[RFC PATCH 39/46] KVM: VMX: Integrate Caretaker VMX detach serialization and KVM registration
Pasha Tatashin
pasha.tatashin at soleen.com
Sun Sep 20 12:36:43 PDT 2026
Complete Intel VMX Caretaker detach-time serialization, vCPU sync,
and ops registration in arch/x86/kvm/vmx/caretaker.c.
Signed-off-by: Pasha Tatashin <pasha.tatashin at soleen.com>
---
arch/x86/kvm/Makefile | 9 +
arch/x86/kvm/vmx/caretaker.c | 322 ++++++++++++++++++++++++++++++++++-
arch/x86/kvm/vmx/vmx.c | 4 +
3 files changed, 333 insertions(+), 2 deletions(-)
diff --git a/arch/x86/kvm/Makefile b/arch/x86/kvm/Makefile
index 11e67c072258..b550a3fd0708 100644
--- a/arch/x86/kvm/Makefile
+++ b/arch/x86/kvm/Makefile
@@ -18,6 +18,8 @@ kvm-$(CONFIG_KVM_SMM) += smm.o
kvm-intel-y += vmx/vmx.o vmx/vmenter.o vmx/pmu_intel.o vmx/vmcs12.o \
vmx/nested.o vmx/posted_intr.o vmx/main.o
+kvm-intel-$(CONFIG_KVM_CARETAKER) += vmx/caretaker.o vmx/caretaker_vmenter.o
+
kvm-intel-$(CONFIG_X86_SGX_KVM) += vmx/sgx.o
kvm-intel-$(CONFIG_KVM_HYPERV) += vmx/hyperv.o vmx/hyperv_evmcs.o
@@ -44,6 +46,13 @@ $(obj)/svm/vmenter.o: $(obj)/kvm-asm-offsets.h
AFLAGS_vmx/vmenter.o := -iquote $(obj)
$(obj)/vmx/vmenter.o: $(obj)/kvm-asm-offsets.h
+AFLAGS_vmx/caretaker_vmenter.o := -iquote $(obj)
+$(obj)/vmx/caretaker_vmenter.o: $(obj)/kvm-asm-offsets.h
+
+CFLAGS_REMOVE_vmx/caretaker.o = $(CC_FLAGS_FTRACE)
+CFLAGS_vmx/caretaker.o := -fno-stack-protector
+
+
$(obj)/kvm-asm-offsets.h: $(obj)/kvm-asm-offsets.s FORCE
$(call filechk,offsets,__KVM_ASM_OFFSETS_H__)
diff --git a/arch/x86/kvm/vmx/caretaker.c b/arch/x86/kvm/vmx/caretaker.c
index 47b4743650e5..efef1c30c0d1 100644
--- a/arch/x86/kvm/vmx/caretaker.c
+++ b/arch/x86/kvm/vmx/caretaker.c
@@ -49,8 +49,8 @@ static int vmx_caretaker_init_page(struct caretaker_vmx_page *cvp,
u64 basic_msr, misc_msr;
int ret;
- if (!vmx->vmcs01.vmcs)
- return -EINVAL;
+ if (!vmx->vmcs01.vmcs || is_guest_mode(vcpu))
+ return -EOPNOTSUPP;
ret = kvm_x86_caretaker_init_common_page(&cvp->common, vcpu, sizeof(*cvp));
if (ret)
@@ -336,5 +336,323 @@ vmx_caretaker_init_host_vmcs(struct caretaker_vmx_page *cvp)
vmx_vmwrite(PLE_WINDOW, 4096);
}
vmx_vmwrite(CPU_BASED_VM_EXEC_CONTROL, cpu_ctl);
+
+ /*
+ * Disable hardware MSR autoload lists while running in Caretaker so
+ * VM-entry/VM-exit do not dereference unpreserved host vmx->msr_autoload
+ * memory after kexec.
+ */
+ vmx_vmwrite(VM_EXIT_MSR_LOAD_COUNT, 0);
+ vmx_vmwrite(VM_EXIT_MSR_STORE_COUNT, 0);
+ vmx_vmwrite(VM_ENTRY_MSR_LOAD_COUNT, 0);
+}
+
+/*
+ * Guest-visible VMCS fields carried from the VMCS the caretaker ran the vCPU
+ * on to the VMCS the new kernel allocated for it.
+ *
+ * The new kernel does not adopt the old VMCS: it belongs to the previous
+ * kernel's struct loaded_vmcs, whose layout is not part of any handover ABI,
+ * and the VMCS region itself is opaque and implementation defined. So the
+ * architecturally defined guest state is copied field by field instead.
+ *
+ * GUEST_IA32_EFER is handled separately because it is only written back when
+ * the caretaker actually recorded a value for it.
+ */
+static const u16 vmx_caretaker_guest_fields[] = {
+ GUEST_CS_SELECTOR, GUEST_CS_LIMIT,
+ GUEST_CS_AR_BYTES, GUEST_CS_BASE,
+ GUEST_SS_SELECTOR, GUEST_SS_LIMIT,
+ GUEST_SS_AR_BYTES, GUEST_SS_BASE,
+ GUEST_DS_SELECTOR, GUEST_DS_LIMIT,
+ GUEST_DS_AR_BYTES, GUEST_DS_BASE,
+ GUEST_ES_SELECTOR, GUEST_ES_LIMIT,
+ GUEST_ES_AR_BYTES, GUEST_ES_BASE,
+ GUEST_FS_SELECTOR, GUEST_FS_LIMIT,
+ GUEST_FS_AR_BYTES, GUEST_FS_BASE,
+ GUEST_GS_SELECTOR, GUEST_GS_LIMIT,
+ GUEST_GS_AR_BYTES, GUEST_GS_BASE,
+ GUEST_TR_SELECTOR, GUEST_TR_LIMIT,
+ GUEST_TR_AR_BYTES, GUEST_TR_BASE,
+ GUEST_LDTR_SELECTOR, GUEST_LDTR_LIMIT,
+ GUEST_LDTR_AR_BYTES, GUEST_LDTR_BASE,
+ GUEST_GDTR_LIMIT, GUEST_GDTR_BASE,
+ GUEST_IDTR_LIMIT, GUEST_IDTR_BASE,
+ GUEST_INTERRUPTIBILITY_INFO,
+ GUEST_ACTIVITY_STATE,
+ GUEST_IA32_DEBUGCTL,
+ GUEST_SYSENTER_CS,
+ GUEST_SYSENTER_ESP,
+ GUEST_SYSENTER_EIP,
+};
+
+static void
+vmx_caretaker_sync_vcpu(struct kvm_vcpu *vcpu, void *vcpu_data)
+{
+ struct kvm_caretaker_arch_ser *abi = vcpu_data;
+ struct vcpu_vmx *vmx = to_vmx(vcpu);
+ phys_addr_t cur_vmcs_pa = vmx->loaded_vmcs ? virt_to_phys(vmx->loaded_vmcs->vmcs) : 0;
+ struct vmcs *prev_vmcs;
+
+ guard(preempt)();
+ prev_vmcs = this_cpu_read(current_vmcs);
+
+ if (abi->vmcs_pa && cur_vmcs_pa && abi->vmcs_pa != cur_vmcs_pa) {
+ /* 42 * 8 bytes; this runs on the host stack, not a preserved one. */
+ unsigned long val[ARRAY_SIZE(vmx_caretaker_guest_fields)];
+ unsigned long guest_efer;
+ int i;
+
+ asm volatile("vmptrld %0" : : "m" (abi->vmcs_pa) : "memory", "cc");
+
+ for (i = 0; i < ARRAY_SIZE(vmx_caretaker_guest_fields); i++)
+ val[i] = vmx_vmread(vmx_caretaker_guest_fields[i]);
+ guest_efer = vmx_caretaker_read_efer();
+
+ asm volatile("vmclear %0" : : "m" (abi->vmcs_pa) : "memory", "cc");
+ asm volatile("vmptrld %0" : : "m" (cur_vmcs_pa) : "memory", "cc");
+
+ for (i = 0; i < ARRAY_SIZE(vmx_caretaker_guest_fields); i++)
+ vmx_vmwrite(vmx_caretaker_guest_fields[i], val[i]);
+ if (guest_efer)
+ vmx_vmwrite(GUEST_IA32_EFER, guest_efer);
+
+ abi->vmcs_pa = cur_vmcs_pa;
+ } else if (cur_vmcs_pa) {
+ asm volatile("vmptrld %0" : : "m" (cur_vmcs_pa) : "memory", "cc");
+ }
+
+ kvm_x86_caretaker_sync_vcpu_common(vcpu);
+
+ if (vmx->loaded_vmcs) {
+ pin_controls_clearbit(vmx, PIN_BASED_VMX_PREEMPTION_TIMER);
+ vmcs_write32(PIN_BASED_VM_EXEC_CONTROL, pin_controls_get(vmx));
+ vmcs_write32(CPU_BASED_VM_EXEC_CONTROL, exec_controls_get(vmx));
+ vmcs_write32(VMX_PREEMPTION_TIMER_VALUE, 0);
+ vmcs_write32(VM_EXIT_MSR_LOAD_COUNT, vmx->msr_autoload.host.nr);
+ vmcs_write32(VM_ENTRY_MSR_LOAD_COUNT, vmx->msr_autoload.guest.nr);
+ memset(&vmx->loaded_vmcs->host_state, 0,
+ sizeof(struct vmcs_host_state));
+ list_del_init(&vmx->loaded_vmcs->loaded_vmcss_on_cpu_link);
+ vmx->loaded_vmcs->cpu = -1;
+ vmx->loaded_vmcs->launched = 0;
+ }
+ vmx_segment_cache_clear(vmx);
+ vmx->vt.guest_state_loaded = false;
+ vmx->guest_uret_msrs_loaded = false;
+
+ vmcs_write32(VM_ENTRY_INTR_INFO_FIELD, 0);
+ vmcs_write32(GUEST_INTERRUPTIBILITY_INFO, 0);
+ vmcs_write32(GUEST_ACTIVITY_STATE, GUEST_ACTIVITY_ACTIVE);
+ vmcs_writel(GUEST_PENDING_DBG_EXCEPTIONS, 0);
+
+ vmcs_writel(GUEST_RIP, kvm_rip_read(vcpu));
+ vmcs_writel(GUEST_RSP, kvm_rsp_read(vcpu));
+ vmcs_writel(GUEST_RFLAGS, kvm_get_rflags(vcpu));
+ vmx_set_cr0(vcpu, vcpu->arch.cr0);
+ vmcs_writel(GUEST_CR3, vcpu->arch.cr3);
+ vmx_set_cr4(vcpu, vcpu->arch.cr4);
+ vmx_set_efer(vcpu, vcpu->arch.efer);
+
+ if (vmx->loaded_vmcs)
+ vmx_set_constant_host_state(vmx);
+
+ if (cur_vmcs_pa)
+ asm volatile("vmclear %0" : : "m" (cur_vmcs_pa) : "memory", "cc");
+
+ if (prev_vmcs && (!vmx->loaded_vmcs || prev_vmcs != vmx->loaded_vmcs->vmcs)) {
+ vmcs_load(prev_vmcs);
+ this_cpu_write(current_vmcs, prev_vmcs);
+ } else {
+ this_cpu_write(current_vmcs, NULL);
+ }
+}
+
+static __cpu_preserved_text void
+vmx_caretaker_detach_serialize(void *page, struct kvm_vcpu_arch_ser *state)
+{
+ struct caretaker_vmx_page *cvp = page;
+
+ kvm_x86_caretaker_detach_serialize_common(&cvp->common, state);
+ kvm_x86_caretaker_update_msr(state, MSR_STAR, cvp->star);
+ kvm_x86_caretaker_update_msr(state, MSR_LSTAR, cvp->lstar);
+ kvm_x86_caretaker_update_msr(state, MSR_SYSCALL_MASK, cvp->fmask);
+ kvm_x86_caretaker_update_msr(state, MSR_KERNEL_GS_BASE,
+ cvp->common.kernel_gs_base);
+}
+
+static __cpu_preserved_text void vmx_caretaker_arm_timer(void *page, u64 deadline_ticks)
+{
+ struct caretaker_vmx_page *cvp = page;
+ u32 shift = (cvp && cvp->timer_shift) ? cvp->timer_shift : VMX_PREEMPTION_TIMER_SHIFT;
+ u32 timer_value = 0;
+ u32 pin;
+
+ if (deadline_ticks) {
+ u64 now = arch_oncore_read_counter();
+
+ if (deadline_ticks > now) {
+ u64 remaining = deadline_ticks - now;
+
+ timer_value = (u32)(remaining >> shift);
+ if (timer_value == 0)
+ timer_value = 1;
+ } else {
+ timer_value = 1;
+ }
+ }
+
+ if (timer_value > 0) {
+ vmx_vmwrite(VMX_PREEMPTION_TIMER_VALUE, timer_value);
+ pin = (u32)vmx_vmread(PIN_BASED_VM_EXEC_CONTROL);
+ pin |= PIN_BASED_VMX_PREEMPTION_TIMER;
+ vmx_vmwrite(PIN_BASED_VM_EXEC_CONTROL, pin);
+ } else {
+ vmx_caretaker_disarm_timer(page);
+ }
}
+static __cpu_preserved_text void
+vmx_caretaker_advance_rip(void *page, u64 rip)
+{
+ struct caretaker_vmx_page *cvp = page;
+
+ cvp->common.last_exit_rip = rip;
+ vmx_vmwrite(GUEST_RIP, rip);
+}
+
+static __cpu_preserved_text void vmx_caretaker_pre_enter(void *page)
+{
+ struct caretaker_vmx_page *cvp = page;
+
+ /* Ensure VMX is active on this core */
+ if (!(__read_cr4() & X86_CR4_VMXE)) {
+ asm volatile("mov %0, %%cr4" : : "r" (__read_cr4() | X86_CR4_VMXE) : "memory");
+ if (cvp->vmxon_pa) {
+ asm volatile("1: vmxon %[vmxon_pa]\n\t"
+ "2:\n\t"
+ _ASM_EXTABLE(1b, 2b)
+ : : [vmxon_pa] "m" (cvp->vmxon_pa)
+ : "memory", "cc");
+ }
+ }
+
+ /* Activate VMCS on this pCPU */
+ asm volatile("vmptrld %0" : : "m" (cvp->common.abi.vmcs_pa) : "memory", "cc");
+
+ /* Configure Caretaker host VMCS */
+ vmx_caretaker_init_host_vmcs(cvp);
+
+ if (cvp->common.arch_state)
+ native_write_cr2(cvp->common.arch_state->sregs.cr2);
+
+ native_wrmsrq(MSR_STAR, cvp->star);
+ native_wrmsrq(MSR_LSTAR, cvp->lstar);
+ native_wrmsrq(MSR_SYSCALL_MASK, cvp->fmask);
+}
+
+static __cpu_preserved_text void
+vmx_caretaker_read_seg(struct kvm_segment *var, u16 sel_field,
+ u16 base_field, u16 limit_field, u16 ar_field)
+{
+ u32 ar = (u32)vmx_vmread(ar_field);
+
+ var->base = vmx_vmread(base_field);
+ var->limit = (u32)vmx_vmread(limit_field);
+ var->selector = (u16)vmx_vmread(sel_field);
+ var->unusable = (ar >> 16) & 1;
+ var->type = ar & 15;
+ var->s = (ar >> 4) & 1;
+ var->dpl = (ar >> 5) & 3;
+ var->present = !var->unusable;
+ var->avl = (ar >> 12) & 1;
+ var->l = (ar >> 13) & 1;
+ var->db = (ar >> 14) & 1;
+ var->g = (ar >> 15) & 1;
+}
+
+static __cpu_preserved_text void vmx_caretaker_post_exit(void *page)
+{
+ struct caretaker_vmx_page *cvp = page;
+ struct kvm_vcpu_arch_ser *state = cvp->common.arch_state;
+ u64 efer;
+
+ cvp->star = native_rdmsrq(MSR_STAR);
+ cvp->lstar = native_rdmsrq(MSR_LSTAR);
+ cvp->fmask = native_rdmsrq(MSR_SYSCALL_MASK);
+
+ cvp->common.cr0 = vmx_caretaker_read_cr0();
+ cvp->common.cr3 = vmx_vmread(GUEST_CR3);
+ cvp->common.cr4 = vmx_caretaker_read_cr4();
+ efer = vmx_caretaker_read_efer();
+ if (efer)
+ cvp->common.efer = efer;
+ cvp->common.last_exit_rip = vmx_vmread(GUEST_RIP);
+ cvp->common.last_exit_rsp = vmx_vmread(GUEST_RSP);
+ cvp->common.last_exit_rflags = vmx_vmread(GUEST_RFLAGS);
+
+ if (state) {
+ state->sregs.cr2 = native_read_cr2();
+ vmx_caretaker_read_seg(&state->sregs.cs, GUEST_CS_SELECTOR,
+ GUEST_CS_BASE, GUEST_CS_LIMIT, GUEST_CS_AR_BYTES);
+ vmx_caretaker_read_seg(&state->sregs.ds, GUEST_DS_SELECTOR,
+ GUEST_DS_BASE, GUEST_DS_LIMIT, GUEST_DS_AR_BYTES);
+ vmx_caretaker_read_seg(&state->sregs.es, GUEST_ES_SELECTOR,
+ GUEST_ES_BASE, GUEST_ES_LIMIT, GUEST_ES_AR_BYTES);
+ vmx_caretaker_read_seg(&state->sregs.fs, GUEST_FS_SELECTOR,
+ GUEST_FS_BASE, GUEST_FS_LIMIT, GUEST_FS_AR_BYTES);
+ vmx_caretaker_read_seg(&state->sregs.gs, GUEST_GS_SELECTOR,
+ GUEST_GS_BASE, GUEST_GS_LIMIT, GUEST_GS_AR_BYTES);
+ vmx_caretaker_read_seg(&state->sregs.ss, GUEST_SS_SELECTOR,
+ GUEST_SS_BASE, GUEST_SS_LIMIT, GUEST_SS_AR_BYTES);
+ vmx_caretaker_read_seg(&state->sregs.tr, GUEST_TR_SELECTOR,
+ GUEST_TR_BASE, GUEST_TR_LIMIT, GUEST_TR_AR_BYTES);
+ vmx_caretaker_read_seg(&state->sregs.ldt, GUEST_LDTR_SELECTOR,
+ GUEST_LDTR_BASE, GUEST_LDTR_LIMIT, GUEST_LDTR_AR_BYTES);
+ state->sregs.gdt.base = vmx_vmread(GUEST_GDTR_BASE);
+ state->sregs.gdt.limit = (u16)vmx_vmread(GUEST_GDTR_LIMIT);
+ state->sregs.idt.base = vmx_vmread(GUEST_IDTR_BASE);
+ state->sregs.idt.limit = (u16)vmx_vmread(GUEST_IDTR_LIMIT);
+
+ kvm_x86_caretaker_update_msr(state, MSR_IA32_SYSENTER_CS,
+ vmx_vmread(GUEST_SYSENTER_CS));
+ kvm_x86_caretaker_update_msr(state, MSR_IA32_SYSENTER_ESP,
+ vmx_vmread(GUEST_SYSENTER_ESP));
+ kvm_x86_caretaker_update_msr(state, MSR_IA32_SYSENTER_EIP,
+ vmx_vmread(GUEST_SYSENTER_EIP));
+ }
+
+ /* Flush VMCS cache so host and incoming kernel see latest guest state */
+ asm volatile("vmclear %0" : : "m" (cvp->common.abi.vmcs_pa) : "memory", "cc");
+}
+
+static const struct kvm_x86_caretaker_runtime_ops vmx_caretaker_runtime_ops __cpu_preserved_data = {
+ .detach_serialize = vmx_caretaker_detach_serialize,
+ .common = {
+ .enter_guest = vmx_caretaker_enter,
+ .decode_exit = vmx_caretaker_decode_exit,
+ .handle_arch_exit = kvm_x86_caretaker_handle_exit,
+ .advance_rip = vmx_caretaker_advance_rip,
+ .arm_timer = vmx_caretaker_arm_timer,
+ .disarm_timer = vmx_caretaker_disarm_timer,
+ .pre_run = vmx_caretaker_pre_enter,
+ .post_run = vmx_caretaker_post_exit,
+ },
+};
+
+static const struct kvm_x86_caretaker_ops vmx_caretaker_ops = {
+ .name = "vmx",
+ .init = vmx_caretaker_init,
+ .sync_vcpu = vmx_caretaker_sync_vcpu,
+ .runtime = &vmx_caretaker_runtime_ops,
+};
+
+void vmx_caretaker_register(void)
+{
+ kvm_x86_caretaker_register_ops(&vmx_caretaker_ops);
+}
+
+void vmx_caretaker_unregister(void)
+{
+ kvm_x86_caretaker_unregister_ops(&vmx_caretaker_ops);
+}
diff --git a/arch/x86/kvm/vmx/vmx.c b/arch/x86/kvm/vmx/vmx.c
index 48b6e69f99bf..817623f7429a 100644
--- a/arch/x86/kvm/vmx/vmx.c
+++ b/arch/x86/kvm/vmx/vmx.c
@@ -54,6 +54,7 @@
#include <trace/events/ipi.h>
#include "capabilities.h"
+#include "caretaker.h"
#include "common.h"
#include "cpuid.h"
#include "hyperv.h"
@@ -8548,6 +8549,7 @@ void vmx_migrate_timers(struct kvm_vcpu *vcpu)
void vmx_hardware_unsetup(void)
{
+ vmx_caretaker_unregister();
kvm_set_posted_intr_wakeup_handler(NULL);
if (nested)
@@ -8855,6 +8857,8 @@ __init int vmx_hardware_setup(void)
kvm_caps.inapplicable_quirks &= ~KVM_X86_QUIRK_IGNORE_GUEST_PAT;
+ vmx_caretaker_register();
+
return 0;
}
--
2.55.0.1082.g2b9226bbc0-goog
More information about the kexec
mailing list