[RFC PATCH 5/5] KVM: arm64: Enable HAFDBS for guests not on migration
Leonardo Bras
leo.bras at arm.com
Tue Sep 1 10:15:56 PDT 2026
When dirty-logging is disabled, even non-write faults make a page dirty,
which avoids a second fault when the page is actually written to.
On dirty-logging enable, this approach causes all (writable) pages on the
memslot to be marked clean, even if they were not written to, which can
take a lot of time, while holding the MMU lock, doing atomic writes to
PTEs.
Systems with HAFDBS can use HW to mark a writable-clean page as
writable-dirty when a write occurs, avoiding the mentioned second fault,
while keeping dirty only the pages that have been actually written to.
So, if the system supports VHE + HAFDBS, keep the non-write-faulted page as
writable-clean, and let HAFDBS update that on demand when a write happens.
When dirty-tracking actually starts, disable HAFDBS as having it on
avoids the same fault that is used for dirty-logging.
Signed-off-by: Leonardo Bras <leo.bras at arm.com>
---
arch/arm64/include/asm/kvm_mmu.h | 6 ++++++
arch/arm64/include/asm/kvm_nested.h | 9 +++++++--
arch/arm64/kvm/arm.c | 7 +++++++
arch/arm64/kvm/mmu.c | 26 +++++++++++++++++++++++++-
4 files changed, 45 insertions(+), 3 deletions(-)
diff --git a/arch/arm64/include/asm/kvm_mmu.h b/arch/arm64/include/asm/kvm_mmu.h
index 6eae7e7e2a68..3defa1a988d3 100644
--- a/arch/arm64/include/asm/kvm_mmu.h
+++ b/arch/arm64/include/asm/kvm_mmu.h
@@ -383,20 +383,26 @@ static inline void kvm_fault_unlock(struct kvm *kvm)
* and CMOs are NOP'd. This has the effect of no longer requiring a
* KVA for addresses mapped into the S2. The presence of these features
* are thus necessary to support cacheable S2 mapping of VM_PFNMAP.
*/
static inline bool kvm_supports_cacheable_pfnmap(void)
{
return cpus_have_final_cap(ARM64_HAS_STAGE2_FWB) &&
cpus_have_final_cap(ARM64_HAS_CACHE_DIC);
}
+static inline bool kvm_supports_hafdbs(struct kvm *kvm)
+{
+ return IS_ENABLED(CONFIG_ARM64_HW_AFDBM) && has_vhe() &&
+ !kvm_vcpu_has_nv(kvm) && cpus_have_final_cap(ARM64_HW_DBM);
+}
+
#ifdef CONFIG_PTDUMP_STAGE2_DEBUGFS
void kvm_s2_ptdump_create_debugfs(struct kvm *kvm);
void kvm_nested_s2_ptdump_create_debugfs(struct kvm_s2_mmu *mmu);
void kvm_nested_s2_ptdump_remove_debugfs(struct kvm_s2_mmu *mmu);
#else
static inline void kvm_s2_ptdump_create_debugfs(struct kvm *kvm) {}
static inline void kvm_nested_s2_ptdump_create_debugfs(struct kvm_s2_mmu *mmu) {}
static inline void kvm_nested_s2_ptdump_remove_debugfs(struct kvm_s2_mmu *mmu) {}
#endif /* CONFIG_PTDUMP_STAGE2_DEBUGFS */
diff --git a/arch/arm64/include/asm/kvm_nested.h b/arch/arm64/include/asm/kvm_nested.h
index 1ed708335809..9242b5d665af 100644
--- a/arch/arm64/include/asm/kvm_nested.h
+++ b/arch/arm64/include/asm/kvm_nested.h
@@ -1,24 +1,29 @@
/* SPDX-License-Identifier: GPL-2.0 */
#ifndef __ARM64_KVM_NESTED_H
#define __ARM64_KVM_NESTED_H
#include <linux/bitfield.h>
#include <linux/kvm_host.h>
#include <asm/kvm_emulate.h>
#include <asm/kvm_pgtable.h>
-static inline bool vcpu_has_nv(const struct kvm_vcpu *vcpu)
+static inline bool kvm_vcpu_has_nv(const struct kvm *kvm)
{
return (!__is_defined(__KVM_NVHE_HYPERVISOR__) &&
cpus_have_final_cap(ARM64_HAS_NESTED_VIRT) &&
- vcpu_has_feature(vcpu, KVM_ARM_VCPU_HAS_EL2));
+ kvm_vcpu_has_feature(kvm, KVM_ARM_VCPU_HAS_EL2));
+}
+
+static inline bool vcpu_has_nv(const struct kvm_vcpu *vcpu)
+{
+ return kvm_vcpu_has_nv(vcpu->kvm);
}
/* Translation helpers from non-VHE EL2 to EL1 */
static inline u64 tcr_el2_ps_to_tcr_el1_ips(u64 tcr_el2)
{
return (u64)FIELD_GET(TCR_EL2_PS_MASK, tcr_el2) << TCR_IPS_SHIFT;
}
static inline u64 translate_tcr_el2_to_tcr_el1(u64 tcr)
{
diff --git a/arch/arm64/kvm/arm.c b/arch/arm64/kvm/arm.c
index 1e528d53d093..76aebaa95cda 100644
--- a/arch/arm64/kvm/arm.c
+++ b/arch/arm64/kvm/arm.c
@@ -1676,20 +1676,27 @@ static int kvm_setup_vcpu(struct kvm_vcpu *vcpu)
* KVM_ARM_VCPU_PMU_V3_SET_PMU.
*/
if (kvm_vcpu_has_pmu(vcpu) && !kvm->arch.arm_pmu &&
!kvm_vcpu_has_pmuv3_strict(vcpu))
ret = kvm_arm_set_default_pmu(kvm);
/* Prepare for nested if required */
if (!ret && vcpu_has_nv(vcpu))
ret = kvm_vcpu_init_nested(vcpu);
+ /* Enable HAFDBS by default if VHE && !nested */
+ if (kvm_supports_hafdbs(kvm) &&
+ atomic_read(&kvm->nr_memslots_dirty_logging) == 0)
+ kvm->arch.mmu.vtcr |= VTCR_EL2_HD;
+ else
+ kvm->arch.mmu.vtcr &= ~VTCR_EL2_HD;
+
return ret;
}
static int __kvm_vcpu_set_target(struct kvm_vcpu *vcpu,
const struct kvm_vcpu_init *init)
{
unsigned long features = init->features[0];
struct kvm *kvm = vcpu->kvm;
int ret = -EINVAL;
diff --git a/arch/arm64/kvm/mmu.c b/arch/arm64/kvm/mmu.c
index 9d4f70430bbe..94094ab56d90 100644
--- a/arch/arm64/kvm/mmu.c
+++ b/arch/arm64/kvm/mmu.c
@@ -1982,21 +1982,22 @@ static int kvm_s2_fault_compute_prot(const struct kvm_s2_fault_desc *s2fd,
if (esr_fsc_is_excl_atomic_fault(kvm_vcpu_get_esr(s2fd->vcpu))) {
kvm_inject_dabt_excl_atomic(s2fd->vcpu, kvm_vcpu_get_hfar(s2fd->vcpu));
return 1;
}
*prot = KVM_PGTABLE_PROT_R;
if (s2vi->map_writable) {
*prot |= KVM_PGTABLE_PROT_W;
- if (s2vi->device || !memslot_is_logging(s2fd->memslot) ||
+ if (s2vi->device ||
+ !(memslot_is_logging(s2fd->memslot) || kvm_supports_hafdbs(kvm)) ||
kvm_is_write_fault(s2fd->vcpu))
*prot |= KVM_PGTABLE_PROT_DIRTY;
}
if (s2fd->nested)
*prot = adjust_nested_fault_perms(s2fd->nested, *prot);
if (kvm_vcpu_trap_is_exec_fault(s2fd->vcpu))
*prot |= KVM_PGTABLE_PROT_X;
@@ -2570,53 +2571,76 @@ int __init kvm_mmu_init(u32 hyp_va_bits)
out_destroy_pgtable:
kvm_pgtable_hyp_destroy(hyp_pgtable);
out_free_pgtable:
kfree(hyp_pgtable);
hyp_pgtable = NULL;
out:
return err;
}
+static void kvm_set_hafdbs(struct kvm *kvm, bool set)
+{
+ /* Check if no action required */
+ if (!!(kvm->arch.mmu.vtcr & VTCR_EL2_HD) == set)
+ return;
+
+ if (set)
+ kvm->arch.mmu.vtcr |= VTCR_EL2_HD;
+ else
+ kvm->arch.mmu.vtcr &= ~VTCR_EL2_HD;
+
+ kvm_make_all_cpus_request(kvm, KVM_REQ_RELOAD_STAGE2);
+}
+
void kvm_arch_commit_memory_region(struct kvm *kvm,
struct kvm_memory_slot *old,
const struct kvm_memory_slot *new,
enum kvm_mr_change change)
{
bool log_dirty_pages = new && new->flags & KVM_MEM_LOG_DIRTY_PAGES;
/*
* At this point memslot has been committed and there is an
* allocated dirty_bitmap[], dirty pages will be tracked while the
* memory slot is write protected.
*/
if (log_dirty_pages) {
if (change == KVM_MR_DELETE)
return;
+ /* Disable HAFDBS when dirty-logging starts */
+ if (kvm_supports_hafdbs(kvm))
+ kvm_set_hafdbs(kvm, 0);
+
/*
* Huge and normal pages are write-protected and split
* on either of these two cases:
*
* 1. with initial-all-set: gradually with CLEAR ioctls,
*/
if (kvm_dirty_log_manual_protect_and_init_set(kvm))
return;
/*
* or
* 2. without initial-all-set: all in one shot when
* enabling dirty logging.
*/
kvm_mmu_wp_memory_region(kvm, new->id);
kvm_mmu_split_memory_region(kvm, new->id);
} else {
+ /* If dirty-logging was canceled, set HAFDBS back on */
+ if (kvm_supports_hafdbs(kvm) &&
+ atomic_read(&kvm->nr_memslots_dirty_logging) == 0)
+ kvm_set_hafdbs(kvm, 1);
+
/*
* Free any leftovers from the eager page splitting cache. Do
* this when deleting, moving, disabling dirty logging, or
* creating the memslot (a nop). Doing it for deletes makes
* sure we don't leak memory, and there's no need to keep the
* cache around for any of the other cases.
*/
kvm_mmu_free_memory_cache(&kvm->arch.mmu.split_page_cache);
}
}
--
2.55.0
More information about the linux-arm-kernel
mailing list