[PATCH v19 6/7] firmware: arm_rmm: Ensure the RMM has GPT entries for memory
Shanker Donthineni
sdonthineni at nvidia.com
Tue Sep 29 15:29:41 PDT 2026
Hi Suzuki,
On 9/24/2026 8:52 AM, Suzuki K Poulose wrote:
> External email: Use caution opening links or attachments
>
>
> From: Steven Price <steven.price at arm.com>
>
> The RMM maintains the state of all the granules in the system to make
> sure that the host is abiding by the rules. This state can be maintained
> at different granularity, per page (TRACKING_FINE) or per region
> (TRACKING_COARSE or TRACKING_INTERMEDIATE). The region size depends on the
> underlying "RMI_GRANULE_SIZE". For a "coarse"/"intermediate" region,
> all pages in the region must be of the same state, this implies we need to
> have "fine" tracking for DRAM, so that we can delegate individual pages.
>
> For now we only support a statically carved out memory for tracking
> granules for the "fine" regions. This can be extended in the future to
> allow modifying the tracking granularity and remove the need for a
> static allocation by the firmware.
>
> Similarly, the firmware may create L0 GPT entries describing the total
> address space. But if we change the "PAS" (Physical Address Space) of a
> granule, then the firmware may need to create L1 tables to track the PAS
> at a finer granularity. Linux therefore checks if the platform firmware
> manages the PAR region. i.e., the firmware is in charge of managing the
> L1 GPTs (creation and the required memory for the GPT tables - via static
> carveouts) without host intervention. Support for dynamic GPT creation by
> the host will be added later.
>
> If the firmware requires us to manage the tracking or GPT memory,
> deactivate the RMM and reclaim any memory donated at RMM activation.
>
> Apply the same checks when hotplugged memory is brought online.
>
> Signed-off-by: Steven Price <steven.price at arm.com>
> [ Switch to RMI_GPT_L1_INFO for checking GPTs and deactivate RMM ]
> Co-developed-by: Suzuki K Poulose <suzuki.poulose at arm.com>
> Signed-off-by: Suzuki K Poulose <suzuki.poulose at arm.com>
> ---
> Changes since v19:
> * Avoid mixing gotos with __free cleanups for arm64_init_rmi()
> Changes since v18:
> * Handle buggy RMM to make forward progress for RMI_GPT_INFO and
> RMI_GRANULE_TRACKING_GET
> * Make sure the memory ranges are not inverted and reject such ranges.
> * Move granule_tracking_get/gpt_info wrappers closer to the callers
> Drop "inline", let the compiler do its job
> Changes since v17:
> * Move wrappers that may not be used elsewhere, out of arm-rmi-cmds.h
> Changes since v16:
> * Check fine tracking and create L1 GPTs for hotplug-added memory.
> * Clarify the L1 GPT setup and move the explanatory comment.
> * Switch to using RMI_GPT_INFO command for checking the GPTs.
> * Deactivate the RMM and reclaim the memory if we can't proceed.
> Changes since v15:
> * Skip firmware-reserved NOMAP memory in rmi_init_metadata()
> * Handle negative error codes from wrappers.
> Changes since v14:
> * Move the implementation into drivers/firmware/arm_rmm.
> Changes since v13:
> * Moved out of KVM
> ---
> drivers/firmware/arm_rmm/rmi.c | 218 ++++++++++++++++++++++++++++++++-
> include/linux/arm-rmi-cmds.h | 2 +
> 2 files changed, 219 insertions(+), 1 deletion(-)
>
> diff --git a/drivers/firmware/arm_rmm/rmi.c b/drivers/firmware/arm_rmm/rmi.c
> index 0859f256e192b..1a8f3debd844a 100644
> --- a/drivers/firmware/arm_rmm/rmi.c
> +++ b/drivers/firmware/arm_rmm/rmi.c
> @@ -5,6 +5,7 @@
>
> #include <linux/cpufeature.h>
> #include <linux/memblock.h>
> +#include <linux/memory.h>
> #include <linux/arm-rmi-cmds.h>
> #include <linux/slab.h>
>
> @@ -13,6 +14,7 @@
>
> /* RMM defines RmiFeatureRegister0 to RmiFeatureRegister5. */
> static unsigned long rmi_feat_reg_cache[5] __ro_after_init;
> +static bool arm64_rmi_is_available;
>
> /**
> * rmi_granule_range_undelegate() - Undelegate a range of granules
> @@ -817,6 +819,210 @@ static int rmi_configure(void)
> return ret;
> }
>
> +/**
> + * rmi_granule_tracking_get() - Get configuration of a Granule tracking region
> + * @start: Base PA of the tracking region
> + * @end: End of the PA region
> + * @out_category: Memory category
> + * @out_state: Tracking region state
> + * @out_top: Top of the memory region
> + *
> + * Return: RMI return code
> + */
> +static int rmi_granule_tracking_get(unsigned long start,
> + unsigned long end,
> + unsigned long *out_category,
> + unsigned long *out_state,
> + unsigned long *out_top)
> +{
> + struct arm_smccc_1_2_regs regs = {
> + SMC_RMI_GRANULE_TRACKING_GET, start, end,
> + };
> +
> + rmi_smccc_invoke(®s);
> +
> + if (regs.a0 != RMI_SUCCESS)
> + return regs.a0;
> +
> + if (out_category)
> + *out_category = regs.a1;
> + if (out_state)
> + *out_state = regs.a2;
> + if (out_top)
> + *out_top = regs.a3;
> +
> + return RMI_SUCCESS;
> +}
> +
> +/*
> + * Make sure the area is tracked by RMM at FINE granularity.
> + * We do not support changing the tracking yet.
> + */
> +static int rmi_verify_memory_tracking(phys_addr_t start, phys_addr_t end)
> +{
> + while (start < end) {
> + unsigned long ret, category, state, next;
> +
> + ret = rmi_granule_tracking_get(start, end, &category, &state, &next);
> + if (ret != RMI_SUCCESS)
> + return -ENOMEM;
> +
> + if (WARN_ON(next <= start))
> + return -ENXIO;
> +
> + if (state != RMI_TRACKING_FINE ||
> + category != RMI_MEM_CATEGORY_CONVENTIONAL) {
> + /* TODO: Set granule tracking in this case */
> + pr_err("Granule tracking for region isn't fine/conventional: %llx-%lx\n",
> + start, next);
> + return -ENODEV;
> + }
> + start = next;
> + }
> +
> + return 0;
> +}
> +
> +/*
> + * rmi_gpt_info - Query the GPT info for the given PAR.
> + * @start: Base of the physical address region
> + * @end: Top of the physical address region
> + * @out_top: Top of the physical address region for which
> + * the GPT @out_gpt_par_state is valid
> + * @out_gpt_par_state: State of the GPT covered by [start, out_top)
> + */
> +static long rmi_gpt_info(unsigned long start, unsigned long end,
> + unsigned long *out_top,
> + unsigned long *out_gpt_par_state)
> +{
> + struct arm_smccc_1_2_regs regs = {
> + SMC_RMI_GPT_INFO, start, end,
> + };
> +
> + rmi_smccc_invoke(®s);
> + if (regs.a0 != RMI_SUCCESS)
> + return regs.a0;
> +
> + if (out_top)
> + *out_top = regs.a1;
> + if (out_gpt_par_state)
> + *out_gpt_par_state = regs.a2;
> +
> + return RMI_SUCCESS;
> +}
> +
> +/*
> + * We do not support creating L1 GPTs yet. So, make sure that
> + * all the regions are managed by the firmware.
> + */
> +static int rmi_verify_gpt_firmware_managed(phys_addr_t start, phys_addr_t end)
> +{
> + unsigned long l0gpt_sz;
> + unsigned long next, par_state;
> +
> + l0gpt_sz = 1UL << (30 + FIELD_GET(RMI_FEATURE_REGISTER_1_L0GPTSZ,
> + rmi_feat_reg(1)));
> + start = ALIGN_DOWN(start, l0gpt_sz);
> + end = ALIGN(end, l0gpt_sz);
> +
> + while (start < end) {
> + long ret = rmi_gpt_info(start, end, &next, &par_state);
> +
> + if (ret != RMI_SUCCESS)
> + return -ENOMEM;
> +
> + if (WARN_ON(next <= start))
> + return -ENXIO;
> +
> + if (par_state != RMI_GPT_PAR_PLAT) {
> + pr_err("GPT for the region is not managed by firmware %llx-%lx\n",
> + start, next);
> + return -ENOMEM;
Accepting only RMI_GPT_PAR_PLAT rules out the host ever creating its own
L1 GPTs. On NVIDIA platforms the PARs backing system RAM come up as
RMI_GPT_PAR_HOST_NOT_CREATED, so this returns -ENOMEM and CCA cannot be
enabled at all.
With local changes to promote granule tracking via RMI_GRANULE_TRACKING_SET
and to create the host L1 GPTs, I was able to boot a Realm with multiple
vCPUs and exercise basic boot functionality on NVIDIA silicon.
-Shanker
> + }
> + start = next;
> + }
> +
> + return 0;
> +}
> +
> +static int rmi_prepare_memory(phys_addr_t start, phys_addr_t end)
> +{
> + int ret;
> +
> + if (start >= end)
> + return -EINVAL;
> +
> + ret = rmi_verify_memory_tracking(start, end);
> + if (ret)
> + return ret;
> +
> + return rmi_verify_gpt_firmware_managed(start, end);
> +}
> +
> +static int rmi_init_metadata(void)
> +{
> + phys_addr_t start, end;
> + struct memblock_region *r;
> +
> + for_each_mem_region(r) {
> + int ret;
> +
> + /* Firmware-reserved NOMAP regions are not usable system RAM */
> + if (memblock_is_nomap(r))
> + continue;
> +
> + start = PAGE_ALIGN(r->base);
> + end = PAGE_ALIGN_DOWN(r->base + r->size);
> + /* Too small ? */
> + if (start >= end)
> + continue;
> +
> + ret = rmi_prepare_memory(start, end);
> + if (ret)
> + return ret;
> + }
> +
> + return 0;
> +}
> +
> +static int rmi_memory_notifier(struct notifier_block *nb,
> + unsigned long action, void *data)
> +{
> + struct memory_notify *arg = data;
> + phys_addr_t start, end;
> + int ret;
> +
> + if (action != MEM_GOING_ONLINE)
> + return NOTIFY_DONE;
> +
> + start = PFN_PHYS(arg->start_pfn);
> + end = PFN_PHYS(arg->start_pfn + arg->nr_pages);
> + ret = rmi_prepare_memory(start, end);
> +
> + return notifier_from_errno(ret);
> +}
> +
> +static struct notifier_block rmi_memory_nb = {
> + .notifier_call = rmi_memory_notifier,
> +};
> +
> +bool is_rmi_available(void)
> +{
> + return arm64_rmi_is_available;
> +}
> +EXPORT_SYMBOL_GPL(is_rmi_available);
> +
> +static int rmi_init_memory(void)
> +{
> + int ret;
> +
> + ret = rmi_init_metadata();
> + if (ret)
> + return ret;
> +
> + return register_memory_notifier(&rmi_memory_nb);
> +}
> +
> static int __init arm64_init_rmi(void)
> {
> int ret;
> @@ -843,9 +1049,19 @@ static int __init arm64_init_rmi(void)
> if (ret) {
> pr_err("RMM activate failed (%d)\n", ret);
> ret = ret < 0 ? ret : -ENXIO;
> + return ret;
> }
>
> - return ret;
> + ret = rmi_init_memory();
> + if (ret) {
> + /* Deactivate the RMM */
> + WARN_ON(rmi_sro_memxfer_cmd(sro, GFP_KERNEL, SMC_RMI_RMM_DEACTIVATE));
> + return ret;
> + }
> +
> + arm64_rmi_is_available = true;
> + pr_info("RMI configured\n");
> + return 0;
> }
>
> /*
> diff --git a/include/linux/arm-rmi-cmds.h b/include/linux/arm-rmi-cmds.h
> index b03974fd8168c..5b177bb176326 100644
> --- a/include/linux/arm-rmi-cmds.h
> +++ b/include/linux/arm-rmi-cmds.h
> @@ -70,6 +70,8 @@ static inline int rmi_undelegate_page(phys_addr_t phys)
> return rmi_undelegate_range(phys, PAGE_SIZE);
> }
>
> +bool is_rmi_available(void);
> +
> long rmi_sro_memxfer_execute(struct rmi_sro_state *sro, gfp_t gfp);
> void rmi_sro_free(struct rmi_sro_state *sro);
> long rmi_sro_execute(struct arm_smccc_1_2_regs *regs);
> --
> 2.43.0
>
More information about the linux-arm-kernel
mailing list