[PATCH v17 19/20] KVM: arm64: Add VM specific callback for S2 MMU operations
Suzuki K Poulose
suzuki.poulose at arm.com
Tue Sep 8 09:22:22 PDT 2026
Add VM type specific S2 MMU operation backends which can be initialized per VM
flavor, to keep the handling cleaner.
Signed-off-by: Suzuki K Poulose <suzuki.poulose at arm.com>
---
arch/arm64/include/asm/kvm_host.h | 15 ++++
arch/arm64/kvm/mmu.c | 129 +++++++++++++++++++++++++-----
2 files changed, 124 insertions(+), 20 deletions(-)
diff --git a/arch/arm64/include/asm/kvm_host.h b/arch/arm64/include/asm/kvm_host.h
index 0f258f16fdf81..e3c308b6e319f 100644
--- a/arch/arm64/include/asm/kvm_host.h
+++ b/arch/arm64/include/asm/kvm_host.h
@@ -157,6 +157,19 @@ struct kvm_vcpu_ops {
void (*vcpu_put)(struct kvm_vcpu *vcpu);
};
+struct kvm_gfn_range;
+
+struct kvm_vm_s2_ops {
+ bool (*vm_age_gfn)(struct kvm *kvm, struct kvm_gfn_range *range);
+ bool (*vm_test_age_gfn)(struct kvm *kvm, struct kvm_gfn_range *range);
+ int (*vm_flush_remote_tlbs)(struct kvm *kvm);
+ int (*vm_flush_remote_tlbs_range)(struct kvm *kvm, gfn_t gfn,
+ u64 nr_pages);
+ void (*vm_stage2_unmap_range)(struct kvm_s2_mmu *mmu,
+ phys_addr_t start, u64 size,
+ bool may_block);
+};
+
struct kvm_s2_mmu {
struct kvm_vmid vmid;
@@ -335,6 +348,8 @@ struct kvm_arch {
*/
u64 fgu[__NR_FGT_GROUP_IDS__];
+ const struct kvm_vm_s2_ops *vm_s2_ops;
+
/*
* Stage 2 paging state for VMs with nested S2 using a virtual
* VMID.
diff --git a/arch/arm64/kvm/mmu.c b/arch/arm64/kvm/mmu.c
index 67852acf7a6f8..e78a56fb73696 100644
--- a/arch/arm64/kvm/mmu.c
+++ b/arch/arm64/kvm/mmu.c
@@ -37,6 +37,10 @@ static unsigned long __ro_after_init io_map_base;
#define KVM_PGT_FN(fn) (!is_protected_kvm_enabled() ? fn : p ## fn)
+static const struct kvm_vm_s2_ops protected_pkvm_s2_ops;
+static const struct kvm_vm_s2_ops unprotected_pkvm_s2_ops;
+static const struct kvm_vm_s2_ops kvm_default_vm_s2_ops;
+
static phys_addr_t __stage2_range_addr_end(phys_addr_t addr, phys_addr_t end,
phys_addr_t size)
{
@@ -166,6 +170,18 @@ static bool memslot_is_logging(struct kvm_memory_slot *memslot)
return memslot->dirty_bitmap && !(memslot->flags & KVM_MEM_READONLY);
}
+static int pkvm_flush_remote_tlbs(struct kvm *kvm)
+{
+ kvm_call_hyp_nvhe(__pkvm_tlb_flush_vmid, kvm->arch.pkvm.handle);
+ return 0;
+}
+
+static int kvm_vm_flush_remote_tlbs(struct kvm *kvm)
+{
+ kvm_call_hyp(__kvm_tlb_flush_vmid, &kvm->arch.mmu);
+ return 0;
+}
+
/**
* kvm_arch_flush_remote_tlbs() - flush all VM TLB entries for v7/8
* @kvm: pointer to kvm structure.
@@ -174,26 +190,36 @@ static bool memslot_is_logging(struct kvm_memory_slot *memslot)
*/
int kvm_arch_flush_remote_tlbs(struct kvm *kvm)
{
- if (is_protected_kvm_enabled())
- kvm_call_hyp_nvhe(__pkvm_tlb_flush_vmid, kvm->arch.pkvm.handle);
- else
- kvm_call_hyp(__kvm_tlb_flush_vmid, &kvm->arch.mmu);
- return 0;
+ if (!kvm->arch.vm_s2_ops->vm_flush_remote_tlbs)
+ return 0;
+ return kvm->arch.vm_s2_ops->vm_flush_remote_tlbs(kvm);
}
-int kvm_arch_flush_remote_tlbs_range(struct kvm *kvm,
- gfn_t gfn, u64 nr_pages)
+static int pkvm_flush_remote_tlbs_range(struct kvm *kvm,
+ gfn_t gfn, u64 nr_pages)
+{
+ return pkvm_flush_remote_tlbs(kvm);
+}
+
+static int kvm_vm_flush_remote_tlbs_range(struct kvm *kvm,
+ gfn_t gfn, u64 nr_pages)
{
u64 size = nr_pages << PAGE_SHIFT;
u64 addr = gfn << PAGE_SHIFT;
- if (is_protected_kvm_enabled())
- kvm_call_hyp_nvhe(__pkvm_tlb_flush_vmid, kvm->arch.pkvm.handle);
- else
- kvm_tlb_flush_vmid_range(&kvm->arch.mmu, addr, size);
+ kvm_tlb_flush_vmid_range(&kvm->arch.mmu, addr, size);
return 0;
}
+int kvm_arch_flush_remote_tlbs_range(struct kvm *kvm,
+ gfn_t gfn, u64 nr_pages)
+{
+ if (!kvm->arch.vm_s2_ops->vm_flush_remote_tlbs_range)
+ return 0;
+
+ return kvm->arch.vm_s2_ops->vm_flush_remote_tlbs_range(kvm, gfn, nr_pages);
+}
+
static void *stage2_memcache_zalloc_page(void *arg)
{
struct kvm_mmu_memory_cache *mc = arg;
@@ -337,13 +363,20 @@ static void __unmap_stage2_range(struct kvm_s2_mmu *mmu, phys_addr_t start, u64
may_block));
}
+static void kvm_vm_stage2_unmap_range(struct kvm_s2_mmu *mmu,
+ phys_addr_t start,
+ u64 size, bool may_block)
+{
+ __unmap_stage2_range(mmu, start, size, may_block);
+}
+
void kvm_stage2_unmap_range(struct kvm_s2_mmu *mmu, phys_addr_t start,
u64 size, bool may_block)
{
- if (kvm_vm_is_protected(kvm_s2_mmu_to_kvm(mmu)))
- return;
+ struct kvm *kvm = kvm_s2_mmu_to_kvm(mmu);
- __unmap_stage2_range(mmu, start, size, may_block);
+ if (kvm->arch.vm_s2_ops->vm_stage2_unmap_range)
+ kvm->arch.vm_s2_ops->vm_stage2_unmap_range(mmu, start, size, may_block);
}
void kvm_stage2_flush_range(struct kvm_s2_mmu *mmu, phys_addr_t addr, phys_addr_t end)
@@ -963,6 +996,18 @@ static void kvm_stage2_destroy(struct kvm_pgtable *pgt)
KVM_PGT_FN(kvm_pgtable_stage2_destroy_pgd)(pgt);
}
+static const struct kvm_vm_s2_ops *arm64_vm_s2_ops[VM_FLAVOR_MAX] = {
+ [VM_VHE] = &kvm_default_vm_s2_ops,
+ [VM_NVHE] = &kvm_default_vm_s2_ops,
+ [VM_PKVM] = &unprotected_pkvm_s2_ops,
+ [VM_PROTECTED_PKVM] = &protected_pkvm_s2_ops,
+};
+
+static void kvm_vm_init_vm_s2_ops(struct kvm *kvm, unsigned long type)
+{
+ kvm->arch.vm_s2_ops = arm64_vm_s2_ops[kvm->arch.vm_flavor];
+}
+
/**
* kvm_init_stage2_mmu - Initialise a S2 MMU structure
* @kvm: The pointer to the KVM structure
@@ -983,6 +1028,9 @@ int kvm_init_stage2_mmu(struct kvm *kvm, struct kvm_s2_mmu *mmu, unsigned long t
int cpu, err;
struct kvm_pgtable *pgt;
+ /* Initialize the VM ops for the VM instance for the first time */
+ if (mmu == &kvm->arch.mmu)
+ kvm_vm_init_vm_s2_ops(kvm, type);
/*
* If we already have our page tables in place, and that the
* MMU context is the canonical one, we have a bug somewhere,
@@ -2447,27 +2495,32 @@ bool kvm_unmap_gfn_range(struct kvm *kvm, struct kvm_gfn_range *range)
return false;
}
-bool kvm_age_gfn(struct kvm *kvm, struct kvm_gfn_range *range)
+static bool kvm_vm_age_gfn(struct kvm *kvm, struct kvm_gfn_range *range)
{
u64 size = (range->end - range->start) << PAGE_SHIFT;
- if (!kvm->arch.mmu.pgt || kvm_vm_is_protected(kvm))
- return false;
-
return KVM_PGT_FN(kvm_pgtable_stage2_test_clear_young)(kvm->arch.mmu.pgt,
range->start << PAGE_SHIFT,
size, true);
+}
+
+bool kvm_age_gfn(struct kvm *kvm, struct kvm_gfn_range *range)
+{
+ if (!kvm->arch.mmu.pgt || !kvm->arch.vm_s2_ops->vm_age_gfn)
+ return false;
+
+ return kvm->arch.vm_s2_ops->vm_age_gfn(kvm, range);
/*
* TODO: Handle nested_mmu structures here using the reverse mapping in
* a later version of patch series.
*/
}
-bool kvm_test_age_gfn(struct kvm *kvm, struct kvm_gfn_range *range)
+static bool kvm_vm_test_age_gfn(struct kvm *kvm, struct kvm_gfn_range *range)
{
u64 size = (range->end - range->start) << PAGE_SHIFT;
- if (!kvm->arch.mmu.pgt || kvm_vm_is_protected(kvm))
+ if (!kvm->arch.mmu.pgt)
return false;
return KVM_PGT_FN(kvm_pgtable_stage2_test_clear_young)(kvm->arch.mmu.pgt,
@@ -2475,6 +2528,15 @@ bool kvm_test_age_gfn(struct kvm *kvm, struct kvm_gfn_range *range)
size, false);
}
+bool kvm_test_age_gfn(struct kvm *kvm, struct kvm_gfn_range *range)
+{
+
+ if (!kvm->arch.mmu.pgt || !kvm->arch.vm_s2_ops->vm_test_age_gfn)
+ return false;
+
+ return kvm->arch.vm_s2_ops->vm_test_age_gfn(kvm, range);
+}
+
phys_addr_t kvm_mmu_get_httbr(void)
{
return __pa(hyp_pgtable->pgd);
@@ -2796,3 +2858,30 @@ void kvm_toggle_cache(struct kvm_vcpu *vcpu, bool was_enabled)
trace_kvm_toggle_cache(*vcpu_pc(vcpu), was_enabled, now_enabled);
}
+
+static const struct kvm_vm_s2_ops protected_vm_s2_ops = {
+ .vm_flush_remote_tlbs = pkvm_flush_remote_tlbs,
+ .vm_flush_remote_tlbs_range = pkvm_flush_remote_tlbs_range,
+ /*
+ * Not supported for Protected VMs under pKVM
+ * .vm_age_gfn
+ * .vm_test_age_gfn
+ * .vm_stage2_unmap_range
+ */
+};
+
+static const struct kvm_vm_s2_ops pkvm_vm_s2_ops = {
+ .vm_flush_remote_tlbs = pkvm_flush_remote_tlbs,
+ .vm_flush_remote_tlbs_range = pkvm_flush_remote_tlbs_range,
+ .vm_age_gfn = kvm_vm_age_gfn,
+ .vm_test_age_gfn = kvm_vm_test_age_gfn,
+ .vm_stage2_unmap_range = kvm_vm_stage2_unmap_range,
+};
+
+static const struct kvm_vm_s2_ops kvm_default_vm_s2_ops = {
+ .vm_flush_remote_tlbs = kvm_vm_flush_remote_tlbs,
+ .vm_flush_remote_tlbs_range = kvm_vm_flush_remote_tlbs_range,
+ .vm_age_gfn = kvm_vm_age_gfn,
+ .vm_test_age_gfn = kvm_vm_test_age_gfn,
+ .vm_stage2_unmap_range = kvm_vm_stage2_unmap_range,
+};
--
2.43.0
More information about the linux-arm-kernel
mailing list