[RFC PATCH 46/46] KVM: arm64: Implement Caretaker vCPU run loop and LUO detach/attach lifecycle
Pasha Tatashin
pasha.tatashin at soleen.com
Mon Sep 21 14:01:02 PDT 2026
Complete ARM64 Caretaker vCPU execution loop, detach-time state
serialization, and LUO integration in arch/arm64/kvm/caretaker.c.
Signed-off-by: Pasha Tatashin <pasha.tatashin at soleen.com>
---
arch/arm64/kvm/Kconfig | 1 +
arch/arm64/kvm/Makefile | 12 ++
arch/arm64/kvm/caretaker.c | 243 ++++++++++++++++++++++++++++++++++++-
arch/arm64/kvm/kvm_luo.c | 103 ++++++++++++++++
4 files changed, 357 insertions(+), 2 deletions(-)
diff --git a/arch/arm64/kvm/Kconfig b/arch/arm64/kvm/Kconfig
index fd25b3c0d8ca..9c43010873a3 100644
--- a/arch/arm64/kvm/Kconfig
+++ b/arch/arm64/kvm/Kconfig
@@ -38,6 +38,7 @@ menuconfig KVM
select GUEST_PERF_EVENTS if PERF_EVENTS
select KVM_GUEST_MEMFD
select HAVE_KVM_ARCH_VCPU_PRESERVE
+ select HAVE_KVM_ARCH_CARETAKER
help
Support hosting virtualized guest machines.
diff --git a/arch/arm64/kvm/Makefile b/arch/arm64/kvm/Makefile
index d5e888d1bd50..58550a0a8160 100644
--- a/arch/arm64/kvm/Makefile
+++ b/arch/arm64/kvm/Makefile
@@ -12,6 +12,18 @@ obj-$(CONFIG_KVM) += hyp/
CFLAGS_sys_regs.o += -Wno-override-init
CFLAGS_handle_exit.o += -Wno-override-init
+KASAN_SANITIZE_caretaker.o := n
+KCSAN_SANITIZE_caretaker.o := n
+UBSAN_SANITIZE_caretaker.o := n
+KCOV_INSTRUMENT_caretaker.o := n
+CFLAGS_REMOVE_caretaker.o = $(CC_FLAGS_FTRACE)
+CFLAGS_caretaker.o += $(call cc-option,-mbranch-protection=none) \
+ -fno-stack-protector \
+ $(call cc-option,-ftrivial-auto-var-init=uninitialized) \
+ $(call cc-option,-fno-jump-tables)
+AFLAGS_caretaker_vmenter.o += -D__KVM_VHE_HYPERVISOR__
+
+kvm-$(CONFIG_KVM_CARETAKER) += caretaker.o caretaker_vmenter.o
kvm-y += arm.o mmu.o mmio.o psci.o hypercalls.o pvtime.o \
inject_fault.o va_layout.o handle_exit.o config.o \
diff --git a/arch/arm64/kvm/caretaker.c b/arch/arm64/kvm/caretaker.c
index 6980c1f6b998..8cd73099a60a 100644
--- a/arch/arm64/kvm/caretaker.c
+++ b/arch/arm64/kvm/caretaker.c
@@ -193,6 +193,9 @@ int arm64_kvm_caretaker_preserve(struct kvm_vcpu *vcpu,
cap->ctx.vtcr_el2 = mmu->vtcr;
cap->ctx.vttbr_el2 = kvm_get_vttbr(mmu);
+ if (irqchip_in_kernel(vcpu->kvm) && !vgic_initialized(vcpu->kvm))
+ kvm_vgic_map_resources(vcpu->kvm);
+
if (vgic_initialized(vcpu->kvm)) {
int i;
@@ -504,8 +507,8 @@ __caretaker_text static void caretaker_vgic_v3_save(struct vgic_v3_cpu_if *cpu_i
used_lrs = min3(used_lrs, max_lrs, (unsigned int)VGIC_V3_MAX_LRS);
for (i = 0; i < used_lrs; i++) {
- cpu_if->vgic_lr[i] = __gic_v3_get_lr(i);
- __gic_v3_set_lr(0, i);
+ cpu_if->vgic_lr[i] = caretaker_gic_v3_get_lr(i);
+ caretaker_gic_v3_set_lr(0, i);
}
cpu_if->vgic_vmcr = read_sysreg_s(SYS_ICH_VMCR_EL2);
@@ -921,3 +924,239 @@ static void arm64_caretaker_sync_vcpu(struct kvm_vcpu *vcpu,
kvm_make_request(KVM_REQ_IRQ_PENDING, vcpu);
}
+static __caretaker_text void
+arm64_caretaker_op_pre_run(void *data)
+{
+ struct caretaker_arm64_page *cap = data;
+
+ local_daif_mask();
+ arm64_caretaker_save_ptrauth(&cap->ptrauth_keys);
+
+ /* Pre-job: load guest context */
+ cpu_preserved_inval(cap);
+
+ arm64_caretaker_load_sysregs(&cap->ctx.ctxt);
+
+ if (cap->ctx.vgic_initialized)
+ caretaker_vgic_v3_restore(cap);
+
+ write_sysreg(cap->ctx.cntvoff_el2, cntvoff_el2);
+ write_sysreg_el0(cap->ctx.cntv_cval_el0, SYS_CNTV_CVAL);
+ write_sysreg_el0(cap->ctx.cntv_ctl_el0, SYS_CNTV_CTL);
+ isb();
+
+ if (cap->ctx.vtcr_el2 && cap->ctx.vttbr_el2) {
+ write_sysreg(cap->ctx.vtcr_el2, vtcr_el2);
+ write_sysreg(cap->ctx.vttbr_el2, vttbr_el2);
+ asm(ALTERNATIVE("nop", "isb", ARM64_WORKAROUND_SPECULATIVE_AT));
+ __tlbi(vmalle1);
+ asm volatile("ic iallu");
+ dsb(nsh);
+ isb();
+ }
+
+ write_sysreg(CPACR_EL1_FPEN_EL0EN | CPACR_EL1_FPEN_EL1EN |
+ CPACR_EL1_ZEN_EL0EN | CPACR_EL1_ZEN_EL1EN,
+ cpacr_el1);
+ isb();
+
+ fpsimd_load_state(&cap->ctx.ctxt.fp_regs);
+
+ gicv3_caretaker_enable_sgi();
+ write_sysreg_s(ICC_CTLR_EL1_EOImode_drop, SYS_ICC_CTLR_EL1);
+ write_sysreg_s(ICC_SRE_EL1_SRE, SYS_ICC_SRE_EL1);
+ write_sysreg_s(0, SYS_ICC_BPR1_EL1);
+ gicv3_caretaker_clear_active_priorities();
+ write_sysreg_s(ICC_PMR_EL1_MASK, SYS_ICC_PMR_EL1);
+ write_sysreg_s(ICC_IGRPEN1_EL1_MASK, SYS_ICC_IGRPEN1_EL1);
+ caretaker_gic_drain_iar();
+ dsb(sy);
+ isb();
+}
+
+static __caretaker_text void
+arm64_caretaker_op_post_run(void *data)
+{
+ struct cpu_preserved_stack_context *sctx;
+ struct caretaker_arm64_page *cap = data;
+ int cpu = cap->abi.cb.pcpu_id;
+ phys_addr_t pgd_pa;
+
+ /* Post-run: restore host hypervisor mode then save guest context */
+ write_sysreg_s(0, SYS_CNTHP_CTL_EL2);
+ caretaker_restore_host_el2();
+
+ fpsimd_save_state(&cap->ctx.ctxt.fp_regs);
+
+ cap->ctx.cntv_cval_el0 = read_sysreg_el0(SYS_CNTV_CVAL);
+ cap->ctx.cntv_ctl_el0 = read_sysreg_el0(SYS_CNTV_CTL);
+
+ if (cap->ctx.vgic_initialized)
+ caretaker_vgic_v3_save(&cap->ctx.vgic_v3);
+
+ arm64_caretaker_save_sysregs(&cap->ctx.ctxt);
+
+ sctx = cpu_preserved_get_stack_context();
+ if (sctx && sctx->session_pgd_pa)
+ pgd_pa = sctx->session_pgd_pa;
+ else
+ pgd_pa = cpu_preserved_get_pgd(cpu);
+
+ if (!pgd_pa)
+ pgd_pa = READ_ONCE(arm64_caretaker_pgd_pa);
+
+ write_sysreg(0, ttbr0_el1);
+ if (pgd_pa && read_sysreg(ttbr1_el1) != pgd_pa) {
+ write_sysreg(pgd_pa, ttbr1_el1);
+ isb();
+ arm64_flush_host_tlb_local();
+ }
+
+ cpu_preserved_clean(cap);
+
+ if (smp_load_acquire(&cap->abi.cb.state) >= KVM_CARETAKER_STOPPING ||
+ kvm_caretaker_should_exit(&cap->vcpu)) {
+ local_daif_mask();
+ isb();
+
+ caretaker_gic_drain_iar();
+ gicv3_caretaker_clear_active_priorities();
+ write_sysreg_s(0, SYS_ICC_IGRPEN1_EL1);
+ write_sysreg_s(0, SYS_ICC_PMR_EL1);
+ write_sysreg_s(0, SYS_ICC_BPR1_EL1);
+ gicv3_caretaker_clear_sgi();
+ dsb(sy);
+ isb();
+ }
+
+ arm64_caretaker_detach_serialize(cap);
+
+ arm64_caretaker_restore_ptrauth(&cap->ptrauth_keys);
+ caretaker_restore_host_el2();
+}
+
+static struct kvm_caretaker_ops arm64_caretaker_ops __cpu_preserved_data = {
+ .enter_guest = arm64_caretaker_op_enter,
+ .decode_exit = arm64_caretaker_op_decode_exit,
+ .handle_arch_exit = arm64_caretaker_op_handle_exit,
+ .advance_rip = arm64_caretaker_op_advance_rip,
+ .arm_timer = arm64_caretaker_op_arm_timer,
+ .disarm_timer = arm64_caretaker_op_disarm_timer,
+ .pre_run = arm64_caretaker_op_pre_run,
+ .post_run = arm64_caretaker_op_post_run,
+};
+
+static __caretaker_text enum oncore_exit_reason
+caretaker_arch_run_page(struct caretaker_arm64_page *cap, u64 deadline_ticks)
+{
+ enum oncore_exit_reason reason;
+ int cpu;
+
+ if (!cap)
+ return ONCORE_EXIT_ERROR;
+
+ cpu_preserved_inval(&cap->abi.cb);
+ cpu = cap->abi.cb.pcpu_id;
+ if (cpu < 0 || cpu >= ARRAY_SIZE(arm64_caretaker_faults))
+ cpu = arm64_caretaker_get_pcpu();
+
+ cap->abi.cb.pcpu_id = cpu;
+ if (cmpxchg(&cap->abi.cb.state, KVM_CARETAKER_PAUSED,
+ KVM_CARETAKER_RUNNING) != KVM_CARETAKER_PAUSED ||
+ cpu_preserved_should_exit(cpu)) {
+ arm64_caretaker_detach_serialize(cap);
+ smp_store_release(&cap->abi.cb.state, KVM_CARETAKER_STOPPED);
+ cpu_preserved_clean(&cap->abi.cb);
+ return ONCORE_EXIT_ATTACH_SIGNALED;
+ }
+ cpu_preserved_clean(&cap->abi.cb);
+
+ cap->vcpu.ops = &arm64_caretaker_ops;
+
+ reason = kvm_caretaker_vcpu_run(&cap->vcpu, deadline_ticks);
+
+ cpu_preserved_inval(&cap->abi.cb);
+ if (reason == ONCORE_EXIT_ATTACH_SIGNALED ||
+ reason == ONCORE_EXIT_ERROR ||
+ cpu_preserved_should_exit(cpu) ||
+ cmpxchg(&cap->abi.cb.state, KVM_CARETAKER_RUNNING,
+ KVM_CARETAKER_PAUSED) != KVM_CARETAKER_RUNNING) {
+ reason = ONCORE_EXIT_ATTACH_SIGNALED;
+ arm64_caretaker_detach_serialize(cap);
+ smp_store_release(&cap->abi.cb.state, KVM_CARETAKER_STOPPED);
+ cpu_preserved_clean(&cap->abi.cb);
+ } else {
+ cpu_preserved_clean(&cap->abi.cb);
+ }
+
+ return reason;
+}
+
+__caretaker_text enum oncore_exit_reason
+kvm_arch_vcpu_caretaker_run(void *data, u64 deadline_ticks)
+{
+ struct kvm_caretaker_cb_ser *cb = data;
+
+ /*
+ * @data is always a struct kvm_caretaker_cb_ser: kvm_caretaker_vcpu_preserve()
+ * installs it with oncore_job_set_data() before activating the job.
+ */
+ if (!cb)
+ return ONCORE_EXIT_ERROR;
+
+ return caretaker_arch_run_page(container_of(cb,
+ struct caretaker_arm64_page,
+ abi.cb),
+ deadline_ticks);
+}
+
+void kvm_arch_vcpu_luo_pre_retrieve_caretaker(struct kvm_vcpu *vcpu,
+ struct kvm_vcpu_ser *ser)
+{
+ if ((ser->flags & KVM_VCPU_LUO_FLAG_CARETAKER) && ser->cb.phys) {
+ struct kvm_caretaker_arch_ser *abi = phys_to_virt(ser->cb.phys);
+ int pcpu = abi->cb.pcpu_id;
+
+ if (pcpu >= 0) {
+ kvm_caretaker_wait_for_attach(&abi->cb, pcpu);
+ cpu_preserved_inval(abi);
+ if (ser->arch_state.phys) {
+ struct kvm_vcpu_arch_ser *state =
+ phys_to_virt(ser->arch_state.phys);
+
+ cpu_preserved_inval(state);
+ cpu_preserved_inval_sz(state,
+ struct_size(state, sysregs,
+ state->num_sysregs));
+ }
+ }
+ }
+}
+
+void kvm_arch_vcpu_luo_attach_caretaker(struct kvm_vcpu *vcpu,
+ struct kvm_vcpu_ser *ser)
+{
+ if ((ser->flags & KVM_VCPU_LUO_FLAG_CARETAKER) && ser->cb.phys) {
+ struct kvm_caretaker_arch_ser *abi = phys_to_virt(ser->cb.phys);
+
+ arm64_caretaker_sync_vcpu(vcpu, abi);
+ }
+
+ kvm_caretaker_post_attach_vcpu(vcpu);
+}
+
+void arm64_kvm_caretaker_unpreserve(struct kvm_vcpu_ser *ser)
+{
+ if (ser->cb.phys) {
+ kho_unpreserve_free(phys_to_virt(ser->cb.phys));
+ ser->cb.phys = 0;
+ }
+}
+
+void arm64_kvm_caretaker_finish(struct kvm_vcpu_ser *ser)
+{
+ if (ser->cb.phys) {
+ kho_restore_free(phys_to_virt(ser->cb.phys));
+ ser->cb.phys = 0;
+ }
+}
diff --git a/arch/arm64/kvm/kvm_luo.c b/arch/arm64/kvm/kvm_luo.c
index 59c56836cda8..ae8baef49f97 100644
--- a/arch/arm64/kvm/kvm_luo.c
+++ b/arch/arm64/kvm/kvm_luo.c
@@ -6,6 +6,7 @@
* ARM64 KVM LUO preservation and retrieval handlers.
*/
+#include <linux/cpu_preserve.h>
#include <linux/kexec_handover.h>
#include <linux/kho/abi/kvm_arm64.h>
#include <linux/kvm_host.h>
@@ -17,15 +18,100 @@
#include <kvm/arm_arch_timer.h>
#include <kvm/arm_vgic.h>
+#include "caretaker.h"
#include "sys_regs.h"
#include "vgic/vgic.h"
+#ifdef CONFIG_KVM_CARETAKER
+struct arm64_stage2_kho_walk {
+ struct kvm_kho_folios_ser *kp;
+ unsigned int count;
+};
+
+static int stage2_kho_visitor(const struct kvm_pgtable_visit_ctx *ctx,
+ enum kvm_pgtable_walk_flags visit)
+{
+ struct arm64_stage2_kho_walk *w = ctx->arg;
+
+ if (kvm_pte_valid(ctx->old) && ctx->level != KVM_PGTABLE_LAST_LEVEL &&
+ FIELD_GET(KVM_PTE_TYPE, ctx->old) == KVM_PTE_TYPE_TABLE) {
+ u64 phys = kvm_pte_to_phys(ctx->old);
+ struct page *p = phys_to_page(phys);
+
+ if (!p)
+ return 0;
+
+ if (!w->kp) {
+ w->count++;
+ return 0;
+ }
+
+ if (w->kp->nr_folios >= w->count)
+ return -ENOSPC;
+
+ if (kho_preserve_folio(page_folio(p)))
+ return -ENOMEM;
+
+ w->kp->folios_pa[w->kp->nr_folios++] = phys;
+ }
+ return 0;
+}
+#endif
+
int kvm_arch_vm_luo_preserve(struct kvm *kvm, struct kvm_luo_ser *ser)
{
+#ifdef CONFIG_KVM_CARETAKER
+ struct kvm_s2_mmu *mmu = &kvm->arch.mmu;
+ struct arm64_stage2_kho_walk walk = {};
+ struct kvm_pgtable_walker walker = {
+ .cb = stage2_kho_visitor,
+ .flags = KVM_PGTABLE_WALK_TABLE_PRE,
+ .arg = &walk,
+ };
+ struct kvm_kho_folios_ser *kp;
+ int ret;
+#endif
+
ser->type = kvm_phys_shift(&kvm->arch.mmu);
if (kvm_vm_is_protected(kvm))
ser->type |= KVM_VM_TYPE_ARM_PROTECTED;
+#ifdef CONFIG_KVM_CARETAKER
+ kvm->caretaker_vm = NULL;
+
+ if (mmu->pgd_phys)
+ walk.count++;
+ if (mmu->pgt) {
+ ret = kvm_pgtable_walk(mmu->pgt, 0, BIT(mmu->pgt->ia_bits), &walker);
+ if (ret)
+ return ret;
+ }
+
+ kp = kvm_kho_folios_alloc(walk.count);
+ if (IS_ERR(kp))
+ return PTR_ERR(kp);
+ walk.kp = kp;
+
+ if (mmu->pgd_phys) {
+ ret = kho_preserve_folio(page_folio(phys_to_page(mmu->pgd_phys)));
+ if (ret) {
+ kvm_kho_folios_unpreserve(kp);
+ return ret;
+ }
+ kp->folios_pa[kp->nr_folios++] = mmu->pgd_phys;
+ }
+
+ if (mmu->pgt) {
+ ret = kvm_pgtable_walk(mmu->pgt, 0, BIT(mmu->pgt->ia_bits), &walker);
+ if (ret) {
+ kvm_kho_folios_unpreserve(kp);
+ return ret;
+ }
+ }
+
+ kvm->kho_folios = kp;
+ KHOSER_STORE_PTR(ser->kho_folios, kp);
+#endif
return 0;
}
@@ -36,6 +122,10 @@ int kvm_arch_vm_luo_retrieve(struct kvm *kvm, struct kvm_luo_ser *ser)
void kvm_arch_vm_luo_unpreserve(struct kvm *kvm, struct kvm_luo_ser *ser)
{
+#ifdef CONFIG_KVM_CARETAKER
+ if (kvm)
+ kvm->caretaker_vm = NULL;
+#endif
}
void kvm_arch_vm_luo_finish(struct kvm_luo_ser *ser)
@@ -119,6 +209,17 @@ int kvm_arch_vcpu_luo_preserve(struct kvm_vcpu *vcpu, struct kvm_vcpu_ser *ser)
kfree(indices);
KHOSER_STORE_PTR(ser->arch_state, state);
+
+ if (ser->flags & KVM_VCPU_LUO_FLAG_CARETAKER) {
+ int ret = arm64_kvm_caretaker_preserve(vcpu, ser);
+
+ if (ret) {
+ kho_unpreserve_free(state);
+ ser->arch_state.phys = 0;
+ return ret;
+ }
+ }
+
return 0;
}
@@ -186,6 +287,7 @@ int kvm_arch_vcpu_luo_retrieve(struct kvm_vcpu *vcpu, struct kvm_vcpu_ser *ser)
void kvm_arch_vcpu_luo_unpreserve(struct kvm_vcpu_ser *ser)
{
+ arm64_kvm_caretaker_unpreserve(ser);
if (ser->arch_state.phys) {
kho_unpreserve_free(phys_to_virt(ser->arch_state.phys));
ser->arch_state.phys = 0;
@@ -194,6 +296,7 @@ void kvm_arch_vcpu_luo_unpreserve(struct kvm_vcpu_ser *ser)
void kvm_arch_vcpu_luo_finish(struct kvm_vcpu_ser *ser)
{
+ arm64_kvm_caretaker_finish(ser);
if (ser->arch_state.phys) {
kho_restore_free(phys_to_virt(ser->arch_state.phys));
ser->arch_state.phys = 0;
--
2.55.0.1082.g2b9226bbc0-goog
More information about the kexec
mailing list