[PATCH 04/20] KVM: arm64: Use block-level annotations when setting up the host stage-2
Vincent Donnefort
vdonnefort at google.com
Mon Aug 3 03:08:48 PDT 2026
During pKVM initialisation, fix_host_ownership() configures the host
stage-2 mappings to match the hypervisor stage-1. Previously this was
done page-by-page. Now that the host stage-2 supports block-level
annotations, refactor the walker to apply the ownership state to an
entire range at once.
This reduces host stage-2 page-table fragmentation and memory usage by
avoiding allocation of intermediate page-table pages.
Signed-off-by: Vincent Donnefort <vdonnefort at google.com>
diff --git a/arch/arm64/kvm/hyp/include/nvhe/memory.h b/arch/arm64/kvm/hyp/include/nvhe/memory.h
index b50712d47f6d..86a51cf1d590 100644
--- a/arch/arm64/kvm/hyp/include/nvhe/memory.h
+++ b/arch/arm64/kvm/hyp/include/nvhe/memory.h
@@ -122,6 +122,11 @@ static inline void set_hyp_state(struct hyp_page *p, enum pkvm_page_state state)
p->__hyp_state_comp = state ^ PKVM_PAGE_STATE_VMEMMAP_MASK;
}
+#define for_each_hyp_page(__p, __st, __sz) \
+ for (struct hyp_page *__p = hyp_phys_to_page(__st), \
+ *__e = __p + ((__sz) >> PAGE_SHIFT); \
+ __p < __e; __p++)
+
/*
* Refcounting for 'struct hyp_page'.
* hyp_pool::lock must be held if atomic access to the refcount is required.
diff --git a/arch/arm64/kvm/hyp/nvhe/mem_protect.c b/arch/arm64/kvm/hyp/nvhe/mem_protect.c
index 4e329e39a695..3c3e0c3d66d1 100644
--- a/arch/arm64/kvm/hyp/nvhe/mem_protect.c
+++ b/arch/arm64/kvm/hyp/nvhe/mem_protect.c
@@ -76,11 +76,6 @@ static void hyp_unlock_component(void)
hyp_spin_unlock(&pkvm_pgd_lock);
}
-#define for_each_hyp_page(__p, __st, __sz) \
- for (struct hyp_page *__p = hyp_phys_to_page(__st), \
- *__e = __p + ((__sz) >> PAGE_SHIFT); \
- __p < __e; __p++)
-
static void *host_s2_zalloc_pages_exact(size_t size)
{
void *addr = hyp_alloc_pages(&host_s2_pool, get_order(size));
diff --git a/arch/arm64/kvm/hyp/nvhe/setup.c b/arch/arm64/kvm/hyp/nvhe/setup.c
index 75b00c323310..d7beb2ccf1e1 100644
--- a/arch/arm64/kvm/hyp/nvhe/setup.c
+++ b/arch/arm64/kvm/hyp/nvhe/setup.c
@@ -186,50 +186,46 @@ static void hpool_put_page(void *addr)
hyp_put_page(&hpool, addr);
}
-static int fix_host_ownership_walker(const struct kvm_pgtable_visit_ctx *ctx,
- enum kvm_pgtable_walk_flags visit)
+struct fix_host_ownership_data {
+ phys_addr_t phys;
+ size_t size;
+ enum kvm_pgtable_prot prot;
+};
+
+static int __fix_host_ownership(const struct fix_host_ownership_data *data)
{
- enum pkvm_page_state state;
- struct hyp_page *page;
- phys_addr_t phys;
- enum kvm_pgtable_prot prot;
+ enum pkvm_page_state state = pkvm_getstate(data->prot);
- if (!kvm_pte_valid(ctx->old))
+ if (!data->prot)
return 0;
- if (ctx->level != KVM_PGTABLE_LAST_LEVEL)
- return -EINVAL;
-
- phys = kvm_pte_to_phys(ctx->old);
- if (!addr_is_memory(phys))
- return -EINVAL;
-
- page = hyp_phys_to_page(phys);
-
/*
* Adjust the host stage-2 mappings to match the ownership attributes
* configured in the hypervisor stage-1, and make sure to propagate them
* to the hyp_vmemmap state.
*/
- prot = kvm_pgtable_hyp_pte_prot(ctx->old);
- state = pkvm_getstate(prot);
switch (state) {
case PKVM_PAGE_OWNED:
- set_hyp_state(page, PKVM_PAGE_OWNED);
- /* hyp text is RO in the host stage-2 to be inspected on panic. */
- if (prot == PAGE_HYP_EXEC) {
+ for_each_hyp_page(page, data->phys, data->size) {
+ set_hyp_state(page, PKVM_PAGE_OWNED);
set_host_state(page, PKVM_NOPAGE);
- return host_stage2_idmap_locked(phys, PAGE_SIZE, KVM_PGTABLE_PROT_R);
- } else {
- return host_stage2_set_owner_locked(phys, PAGE_SIZE, PKVM_ID_HYP);
}
+ /* hyp text is RO in the host stage-2 to be inspected on panic. */
+ if (data->prot == PAGE_HYP_EXEC)
+ return host_stage2_idmap_locked(data->phys, data->size, KVM_PGTABLE_PROT_R);
+ else
+ return host_stage2_set_owner_locked(data->phys, data->size, PKVM_ID_HYP);
case PKVM_PAGE_SHARED_OWNED:
- set_hyp_state(page, PKVM_PAGE_SHARED_OWNED);
- set_host_state(page, PKVM_PAGE_SHARED_BORROWED);
+ for_each_hyp_page(page, data->phys, data->size) {
+ set_hyp_state(page, PKVM_PAGE_SHARED_OWNED);
+ set_host_state(page, PKVM_PAGE_SHARED_BORROWED);
+ }
break;
case PKVM_PAGE_SHARED_BORROWED:
- set_hyp_state(page, PKVM_PAGE_SHARED_BORROWED);
- set_host_state(page, PKVM_PAGE_SHARED_OWNED);
+ for_each_hyp_page(page, data->phys, data->size) {
+ set_hyp_state(page, PKVM_PAGE_SHARED_BORROWED);
+ set_host_state(page, PKVM_PAGE_SHARED_OWNED);
+ }
break;
default:
return -EINVAL;
@@ -238,6 +234,78 @@ static int fix_host_ownership_walker(const struct kvm_pgtable_visit_ctx *ctx,
return 0;
}
+static int fix_host_ownership_walker(const struct kvm_pgtable_visit_ctx *ctx,
+ enum kvm_pgtable_walk_flags visit)
+{
+ struct fix_host_ownership_data *data = ctx->arg;
+ enum kvm_pgtable_prot prot = 0;
+ phys_addr_t phys = 0;
+ int ret;
+
+ if (kvm_pte_valid(ctx->old))
+ prot = kvm_pgtable_hyp_pte_prot(ctx->old);
+
+ if (!prot) {
+ /* Unchanged region */
+ if (!data->prot)
+ return 0;
+
+ goto apply_ownership;
+ }
+
+ phys = kvm_pte_to_phys(ctx->old);
+ if (!addr_is_memory(phys))
+ return -EINVAL;
+
+ /* We already know phys is contiguous as we walk the linear map */
+
+ if (prot != data->prot)
+ goto apply_ownership;
+
+ /* Accumulate in the current region */
+ data->size += kvm_granule_size(ctx->level);
+
+ return 0;
+
+apply_ownership:
+ ret = __fix_host_ownership(data);
+ if (ret)
+ return ret;
+
+ data->phys = phys;
+ data->size = kvm_granule_size(ctx->level);
+ data->prot = prot;
+
+ return 0;
+}
+
+static int fix_host_ownership(void)
+{
+ struct fix_host_ownership_data data;
+ struct kvm_pgtable_walker walker = {
+ .cb = fix_host_ownership_walker,
+ .flags = KVM_PGTABLE_WALK_LEAF,
+ .arg = &data,
+ };
+ int i, ret;
+
+ for (i = 0; i < hyp_memblock_nr; i++) {
+ struct memblock_region *reg = &hyp_memory[i];
+ u64 start = (u64)hyp_phys_to_virt(reg->base);
+
+ data = (struct fix_host_ownership_data){ 0 };
+ ret = kvm_pgtable_walk(&pkvm_pgtable, start, reg->size, &walker);
+ if (ret)
+ return ret;
+
+ ret = __fix_host_ownership(&data);
+ if (ret)
+ return ret;
+ }
+
+ return 0;
+}
+
static int fix_hyp_pgtable_refcnt_walker(const struct kvm_pgtable_visit_ctx *ctx,
enum kvm_pgtable_walk_flags visit)
{
@@ -252,26 +320,6 @@ static int fix_hyp_pgtable_refcnt_walker(const struct kvm_pgtable_visit_ctx *ctx
return 0;
}
-static int fix_host_ownership(void)
-{
- struct kvm_pgtable_walker walker = {
- .cb = fix_host_ownership_walker,
- .flags = KVM_PGTABLE_WALK_LEAF,
- };
- int i, ret;
-
- for (i = 0; i < hyp_memblock_nr; i++) {
- struct memblock_region *reg = &hyp_memory[i];
- u64 start = (u64)hyp_phys_to_virt(reg->base);
-
- ret = kvm_pgtable_walk(&pkvm_pgtable, start, reg->size, &walker);
- if (ret)
- return ret;
- }
-
- return 0;
-}
-
static int fix_hyp_pgtable_refcnt(void)
{
struct kvm_pgtable_walker walker = {
--
2.55.0.508.g3f0d502094-goog
More information about the linux-arm-kernel
mailing list