[PATCH 1/4] KVM: arm64: pgtable: Add Stage-2 unmap without TLBI primitive
Marc Zyngier
maz at kernel.org
Sat Sep 12 03:48:31 PDT 2026
kvm_pgtable_stage2_unmap() iterates over a range, unmapping whatever is
within the range, and always guarantees that that the corresponding TLBs
are invalidated when the function returns.
While this is safe, it means that iterating over empty range on a system
that supports range invalidation results in a TLBI per largest block
mapping size (1GB, 32MB or 512MB, depending on the base granule size).
This can be pretty expensive in situation where the whole address space
is being torn down, as it happens with NV (where S2 MMUs are recycled
regularly), and it would be more efficient to elide the per-subrange
TLBIs to solely rely on a VMID-wide TLBI.
For this, provide a kvm_pgtable_stage2_unmap_notlbi() helper that elides
all TLBIs, and relies on the caller to do the work.
Note that for pKVM case, no additional helper is provided, and we
fallback on the TLBI-aware version.
Signed-off-by: Marc Zyngier <maz at kernel.org>
---
arch/arm64/include/asm/kvm_pgtable.h | 17 +++++++++++++
arch/arm64/include/asm/kvm_pkvm.h | 1 +
arch/arm64/kvm/hyp/pgtable.c | 38 +++++++++++++++++++++-------
arch/arm64/kvm/pkvm.c | 2 ++
4 files changed, 49 insertions(+), 9 deletions(-)
diff --git a/arch/arm64/include/asm/kvm_pgtable.h b/arch/arm64/include/asm/kvm_pgtable.h
index 41a8687938eb6..c370196888d1d 100644
--- a/arch/arm64/include/asm/kvm_pgtable.h
+++ b/arch/arm64/include/asm/kvm_pgtable.h
@@ -318,6 +318,8 @@ typedef bool (*kvm_pgtable_force_pte_cb_t)(u64 addr, u64 end,
* @KVM_PGTABLE_WALK_SKIP_CMO: Visit and update table entries
* without Cache maintenance
* operations required.
+ * @KVM_PGTABLE_WALK_SKIP_S2_TLBI: Visit and update table entries
+ * without Stage-2 TLB invalidation.
*/
enum kvm_pgtable_walk_flags {
KVM_PGTABLE_WALK_LEAF = BIT(0),
@@ -327,6 +329,7 @@ enum kvm_pgtable_walk_flags {
KVM_PGTABLE_WALK_IGNORE_EAGAIN = BIT(4),
KVM_PGTABLE_WALK_SKIP_BBM_TLBI = BIT(5),
KVM_PGTABLE_WALK_SKIP_CMO = BIT(6),
+ KVM_PGTABLE_WALK_SKIP_S2_TLBI = BIT(7),
};
struct kvm_pgtable_visit_ctx {
@@ -717,6 +720,20 @@ int kvm_pgtable_stage2_annotate(struct kvm_pgtable *pgt, u64 addr, u64 size,
*/
int kvm_pgtable_stage2_unmap(struct kvm_pgtable *pgt, u64 addr, u64 size);
+/**
+ * kvm_pgtable_stage2_unmap_notlbi() - Remove a mapping from a guest stage-2 page-table
+ * without TLB invalidation.
+ * @pgt: Page-table structure initialised by kvm_pgtable_stage2_init*().
+ * @addr: Intermediate physical address from which to remove the mapping.
+ * @size: Size of the mapping.
+ *
+ * Same as kvm_pgtable_stage2_unmap(), but does not invalidate the
+ * TLBs, which is the responsibility of the caller. Use with caution!
+ *
+ * Return: 0 on success, negative error code on failure.
+ */
+int kvm_pgtable_stage2_unmap_notlbi(struct kvm_pgtable *pgt, u64 addr, u64 size);
+
/**
* kvm_pgtable_stage2_wrprotect() - Write-protect guest stage-2 address range
* without TLB invalidation.
diff --git a/arch/arm64/include/asm/kvm_pkvm.h b/arch/arm64/include/asm/kvm_pkvm.h
index beea00e693a0a..273013c98ff17 100644
--- a/arch/arm64/include/asm/kvm_pkvm.h
+++ b/arch/arm64/include/asm/kvm_pkvm.h
@@ -214,6 +214,7 @@ int pkvm_pgtable_stage2_map(struct kvm_pgtable *pgt, u64 addr, u64 size, u64 phy
enum kvm_pgtable_prot prot, void *mc,
enum kvm_pgtable_walk_flags flags);
int pkvm_pgtable_stage2_unmap(struct kvm_pgtable *pgt, u64 addr, u64 size);
+int pkvm_pgtable_stage2_unmap_notlbi(struct kvm_pgtable *pgt, u64 addr, u64 size);
int pkvm_pgtable_stage2_wrprotect(struct kvm_pgtable *pgt, u64 addr, u64 size);
int pkvm_pgtable_stage2_flush(struct kvm_pgtable *pgt, u64 addr, u64 size);
bool pkvm_pgtable_stage2_test_clear_young(struct kvm_pgtable *pgt, u64 addr, u64 size, bool mkold);
diff --git a/arch/arm64/kvm/hyp/pgtable.c b/arch/arm64/kvm/hyp/pgtable.c
index b74dd5ce1efd3..6603fc236daa2 100644
--- a/arch/arm64/kvm/hyp/pgtable.c
+++ b/arch/arm64/kvm/hyp/pgtable.c
@@ -29,6 +29,11 @@ static bool kvm_pgtable_walk_skip_cmo(const struct kvm_pgtable_visit_ctx *ctx)
return unlikely(ctx->flags & KVM_PGTABLE_WALK_SKIP_CMO);
}
+static bool kvm_pgtable_walk_skip_s2_tlbi(const struct kvm_pgtable_visit_ctx *ctx)
+{
+ return unlikely(ctx->flags & KVM_PGTABLE_WALK_SKIP_S2_TLBI);
+}
+
static bool kvm_block_mapping_supported(const struct kvm_pgtable_visit_ctx *ctx, u64 phys)
{
u64 granule = kvm_granule_size(ctx->level);
@@ -905,12 +910,14 @@ static void stage2_unmap_put_pte(const struct kvm_pgtable_visit_ctx *ctx,
if (kvm_pte_valid(ctx->old)) {
kvm_clear_pte(ctx->ptep);
- if (kvm_pte_table(ctx->old, ctx->level)) {
- kvm_call_hyp(__kvm_tlb_flush_vmid_ipa, mmu, ctx->addr,
- TLBI_TTL_UNKNOWN);
- } else if (!stage2_unmap_defer_tlb_flush(pgt)) {
- kvm_call_hyp(__kvm_tlb_flush_vmid_ipa, mmu, ctx->addr,
- ctx->level);
+ if (!kvm_pgtable_walk_skip_s2_tlbi(ctx)) {
+ if (kvm_pte_table(ctx->old, ctx->level)) {
+ kvm_call_hyp(__kvm_tlb_flush_vmid_ipa, mmu, ctx->addr,
+ TLBI_TTL_UNKNOWN);
+ } else if (!stage2_unmap_defer_tlb_flush(pgt)) {
+ kvm_call_hyp(__kvm_tlb_flush_vmid_ipa, mmu, ctx->addr,
+ ctx->level);
+ }
}
}
@@ -1195,23 +1202,36 @@ static int stage2_unmap_walker(const struct kvm_pgtable_visit_ctx *ctx,
return 0;
}
-int kvm_pgtable_stage2_unmap(struct kvm_pgtable *pgt, u64 addr, u64 size)
+static int __kvm_pgtable_stage2_unmap(struct kvm_pgtable *pgt,
+ enum kvm_pgtable_walk_flags flags,
+ u64 addr, u64 size)
{
int ret;
struct kvm_pgtable_walker walker = {
.cb = stage2_unmap_walker,
.arg = pgt,
- .flags = KVM_PGTABLE_WALK_LEAF | KVM_PGTABLE_WALK_TABLE_POST,
+ .flags = KVM_PGTABLE_WALK_LEAF | KVM_PGTABLE_WALK_TABLE_POST | flags,
};
ret = kvm_pgtable_walk(pgt, addr, size, &walker);
- if (stage2_unmap_defer_tlb_flush(pgt))
+ if (stage2_unmap_defer_tlb_flush(pgt) &&
+ !(flags & KVM_PGTABLE_WALK_SKIP_S2_TLBI))
/* Perform the deferred TLB invalidations */
kvm_tlb_flush_vmid_range(pgt->mmu, addr, size);
return ret;
}
+int kvm_pgtable_stage2_unmap(struct kvm_pgtable *pgt, u64 addr, u64 size)
+{
+ return __kvm_pgtable_stage2_unmap(pgt, 0, addr, size);
+}
+
+int kvm_pgtable_stage2_unmap_notlbi(struct kvm_pgtable *pgt, u64 addr, u64 size)
+{
+ return __kvm_pgtable_stage2_unmap(pgt, KVM_PGTABLE_WALK_SKIP_S2_TLBI, addr, size);
+}
+
struct stage2_attr_data {
kvm_pte_t attr_set;
kvm_pte_t attr_clr;
diff --git a/arch/arm64/kvm/pkvm.c b/arch/arm64/kvm/pkvm.c
index 8e4c6e4bec123..ec151005fbe4d 100644
--- a/arch/arm64/kvm/pkvm.c
+++ b/arch/arm64/kvm/pkvm.c
@@ -488,6 +488,8 @@ int pkvm_pgtable_stage2_unmap(struct kvm_pgtable *pgt, u64 addr, u64 size)
return __pkvm_pgtable_stage2_unshare(pgt, addr, addr + size);
}
+int pkvm_pgtable_stage2_unmap_notlbi(struct kvm_pgtable *pgt, u64 addr, u64 size) __alias(pkvm_pgtable_stage2_unmap);
+
int pkvm_pgtable_stage2_wrprotect(struct kvm_pgtable *pgt, u64 addr, u64 size)
{
struct kvm *kvm = kvm_s2_mmu_to_kvm(pgt->mmu);
--
2.47.3
More information about the linux-arm-kernel
mailing list