[RFC PATCH 33/46] KVM: x86: Add Caretaker x86 KHO ABI and runtime context headers
Pasha Tatashin
pasha.tatashin at soleen.com
Sun Sep 20 12:36:37 PDT 2026
Add struct kvm_caretaker_arch_ser, x86 Caretaker runtime context headers,
and generated assembly offsets for Caretaker guest entry/exit.
Signed-off-by: Pasha Tatashin <pasha.tatashin at soleen.com>
---
arch/x86/kvm/caretaker.h | 230 ++++++++++++++++++++++++++++++++
arch/x86/kvm/caretaker_asm.h | 117 ++++++++++++++++
arch/x86/kvm/kvm-asm-offsets.c | 32 +++++
include/linux/kho/abi/kvm_x86.h | 22 +++
4 files changed, 401 insertions(+)
create mode 100644 arch/x86/kvm/caretaker.h
create mode 100644 arch/x86/kvm/caretaker_asm.h
diff --git a/arch/x86/kvm/caretaker.h b/arch/x86/kvm/caretaker.h
new file mode 100644
index 000000000000..9dd8815d414b
--- /dev/null
+++ b/arch/x86/kvm/caretaker.h
@@ -0,0 +1,230 @@
+/* SPDX-License-Identifier: GPL-2.0-only */
+/*
+ * Copyright (c) 2026, Google LLC.
+ * Pasha Tatashin <pasha.tatashin at soleen.com>
+ *
+ * Header for x86 KVM Caretaker common C execution engine and helpers.
+ */
+#ifndef __ARCH_X86_KVM_CARETAKER_H
+#define __ARCH_X86_KVM_CARETAKER_H
+
+/*
+ * Size of the caretaker's standalone VMX exit and IST stack in the upper half
+ * of struct caretaker_x86_page ([2048..4096)).
+ */
+#define CXP_STACK_SIZE 2048
+
+/* COM1 serial port range intercepted by Caretaker */
+#define COM1_PORT_BASE 0x3f8
+#define COM1_PORT_END 0x3ff
+
+/* Number of x2APIC MSRs (0x800 - 0x83f) */
+#define X2APIC_MSR_COUNT 0x40
+
+/* Caretaker APIC version: 6 LVT entries (max index 5), version 0x14 */
+#define CARETAKER_APIC_LVR ((5 << 16) | 0x14)
+
+#ifndef __ASSEMBLY__
+
+#include <linux/types.h>
+#include <linux/oncore.h>
+#include <linux/kho/abi/kvm_x86.h>
+#include <linux/kvm_caretaker.h>
+
+/* Architecture-specific VM exit types for x86 */
+#define KVM_CARETAKER_EXIT_CPUID \
+ ((enum kvm_caretaker_exit_type)(KVM_CARETAKER_EXIT_ARCH + 1))
+#define KVM_CARETAKER_EXIT_MSR \
+ ((enum kvm_caretaker_exit_type)(KVM_CARETAKER_EXIT_ARCH + 2))
+#define KVM_CARETAKER_EXIT_RDTSC \
+ ((enum kvm_caretaker_exit_type)(KVM_CARETAKER_EXIT_ARCH + 3))
+
+/* 8250 UART register state for guest early printk emulation */
+struct caretaker_uart {
+ u8 lcr;
+ u8 ier;
+ u8 mcr;
+ u8 scr;
+ u8 dll;
+ u8 dlm;
+};
+
+#include <linux/processor.h>
+#include <asm/desc.h>
+#include <asm/page.h>
+
+static inline void caretaker_set_tss_desc(struct desc_struct *gdt,
+ unsigned long addr, unsigned int size)
+{
+ struct ldttss_desc *desc = (struct ldttss_desc *)&gdt[GDT_ENTRY_TSS];
+
+ memset(desc, 0, sizeof(*desc));
+ desc->limit0 = (u16)size;
+ desc->base0 = (u16)addr;
+ desc->base1 = (addr >> 16) & 0xFF;
+ desc->type = DESC_TSS;
+ desc->p = 1;
+ desc->limit1 = (size >> 16) & 0xF;
+ desc->base2 = (addr >> 24) & 0xFF;
+ desc->base3 = (u32)(addr >> 32);
+}
+
+struct kvm_vcpu;
+struct kvm_caretaker_exit;
+
+/**
+ * struct caretaker_x86_page - Vendor-common x86 Caretaker runtime page (4 KB)
+ * @abi: Cross-kexec KHO ABI header (must remain at offset 0).
+ * @vcpu: Common Caretaker vCPU execution engine descriptor.
+ * @arch_state: Pointer to KHO-preserved vCPU architectural state buffer.
+ * @stack_orig: Saved caller %rsp across vmx_caretaker_enter() guest entry.
+ * @rax: Guest RAX switched in caretaker_vmenter.S.
+ * @rbx: Guest RBX switched in caretaker_vmenter.S.
+ * @rcx: Guest RCX switched in caretaker_vmenter.S.
+ * @rdx: Guest RDX switched in caretaker_vmenter.S.
+ * @rsi: Guest RSI switched in caretaker_vmenter.S.
+ * @rdi: Guest RDI switched in caretaker_vmenter.S.
+ * @rbp: Guest RBP switched in caretaker_vmenter.S.
+ * @r8: Guest R8 switched in caretaker_vmenter.S.
+ * @r9: Guest R9 switched in caretaker_vmenter.S.
+ * @r10: Guest R10 switched in caretaker_vmenter.S.
+ * @r11: Guest R11 switched in caretaker_vmenter.S.
+ * @r12: Guest R12 switched in caretaker_vmenter.S.
+ * @r13: Guest R13 switched in caretaker_vmenter.S.
+ * @r14: Guest R14 switched in caretaker_vmenter.S.
+ * @r15: Guest R15 switched in caretaker_vmenter.S.
+ * @last_exit_rip: Guest RIP captured on VM exit and updated on instruction skip.
+ * @last_exit_rsp: Guest RSP captured on VM exit.
+ * @last_exit_rflags: Guest RFLAGS captured on VM exit.
+ * @host_cr3: Isolated Caretaker page table root PA loaded during run.
+ * @cr3: Guest CR3 shadow value captured on VM exit.
+ * @cr0: Guest CR0 shadow value captured on VM exit.
+ * @cr4: Guest CR4 shadow value captured on VM exit.
+ * @efer: Guest EFER shadow value captured on VM exit.
+ * @kernel_gs_base: Guest MSR_KERNEL_GS_BASE switched across VMX entry/exit.
+ * @uart: Emulated 8250 UART register state for early guest console.
+ * @save_guest_fpu: True if guest FPU state can be captured via XSAVE at detach.
+ * @gdt: Per-vCPU preserved GDT containing the active Caretaker TSS.
+ * @tss: Per-vCPU hardware TSS whose SP0/IST point to @stack.
+ * @stack: 2 KB standalone host stack occupying the upper half of the
+ * page ([2048..4096)). VMX sets HOST_RSP to the top of this
+ * page so vmx_caretaker_exit_handler can recover the base
+ * address of struct caretaker_x86_page via (%rsp & PAGE_MASK).
+ *
+ * Cross-kexec invariant: Only @abi (and @arch_state) may be dereferenced by the
+ * incoming kernel. All remaining fields are private to the preserved Caretaker
+ * text executing on the isolated physical CPU during the kexec handover window.
+ */
+struct caretaker_x86_page {
+ /* KHO ABI prefix (offset 0) and common scheduler descriptor */
+ struct kvm_caretaker_arch_ser abi;
+ struct kvm_caretaker_vcpu vcpu;
+
+ struct kvm_vcpu_arch_ser *arch_state;
+ u64 stack_orig;
+
+ /* Guest GPRs switched in caretaker_vmenter.S */
+ u64 rax, rbx, rcx, rdx, rsi, rdi, rbp;
+ u64 r8, r9, r10, r11, r12, r13, r14, r15;
+
+ /* Guest instruction/stack pointers and control registers at VM exit */
+ u64 last_exit_rip;
+ u64 last_exit_rsp;
+ u64 last_exit_rflags;
+ u64 host_cr3;
+ u64 cr3;
+ u64 cr0;
+ u64 cr4;
+ u64 efer;
+ u64 kernel_gs_base;
+
+ /* Emulated UART and FPU capability state */
+ struct caretaker_uart uart;
+ bool save_guest_fpu;
+
+ /* Isolated host descriptors loaded while Caretaker owns the pCPU */
+ struct desc_struct gdt[GDT_ENTRIES] __aligned(16);
+ struct x86_hw_tss tss __aligned(16);
+
+ /* Upper half of Page 0 (2 KB): standalone VMX exit and IST stack */
+ u8 stack[CXP_STACK_SIZE] __aligned(CXP_STACK_SIZE);
+} __aligned(PAGE_SIZE);
+
+static_assert(offsetof(struct caretaker_x86_page, abi) == 0);
+static_assert(offsetof(struct caretaker_x86_page, abi.cb) == 0);
+static_assert(offsetof(struct caretaker_x86_page, stack) == CXP_STACK_SIZE);
+static_assert(sizeof(struct caretaker_x86_page) == PAGE_SIZE);
+
+int kvm_x86_caretaker_preserve_page(struct kvm_caretaker_arch_ser *abi,
+ struct page *page);
+void kvm_x86_caretaker_unpreserve_pages(struct kvm_caretaker_arch_ser *abi);
+
+/* Shared page table, IDT, and GPR helpers */
+extern gate_desc caretaker_x86_idt[IDT_ENTRIES];
+#ifdef CONFIG_LIVEUPDATE_CPU
+extern phys_addr_t x86_caretaker_pgd_pa;
+#else
+#define x86_caretaker_pgd_pa 0ULL
+#endif
+
+int kvm_x86_caretaker_init_common_page(struct caretaker_x86_page *cxp,
+ struct kvm_vcpu *vcpu,
+ size_t full_page_size);
+
+void kvm_x86_caretaker_sync_vcpu_common(struct kvm_vcpu *vcpu);
+__caretaker_text void
+kvm_x86_caretaker_detach_serialize_common(struct caretaker_x86_page *cxp,
+ struct kvm_vcpu_arch_ser *state);
+__caretaker_text void
+kvm_x86_caretaker_update_msr(struct kvm_vcpu_arch_ser *state,
+ u32 msr, u64 val);
+__caretaker_text bool
+kvm_x86_caretaker_handle_exit(void *data, struct kvm_caretaker_exit *exit);
+
+void x86_preserved_iret_stub(void);
+void x86_preserved_iret_err_stub(void);
+void x86_preserved_apic_eoi_stub(void);
+__caretaker_text void kvm_x86_caretaker_arm_timer(u64 deadline_ticks);
+__caretaker_text void kvm_x86_caretaker_disarm_timer(void);
+
+/**
+ * struct kvm_x86_caretaker_runtime_ops - Preserved runtime vectors for x86 Caretaker
+ * @detach_serialize: Serialize live vendor guest state into struct kvm_vcpu_arch_ser.
+ * @common: Common Caretaker operations table (enter_guest, decode_exit, etc.).
+ */
+struct kvm_x86_caretaker_runtime_ops {
+ void (*detach_serialize)(void *page, struct kvm_vcpu_arch_ser *state);
+ struct kvm_caretaker_ops common;
+};
+
+/**
+ * struct kvm_x86_caretaker_ops - Vendor virtualization vectors for Caretaker
+ * @name: Vendor name identifier ("vmx" or "svm").
+ * @init: Initialize vendor-specific Caretaker page and hardware state for vCPU.
+ * @sync_vcpu: Synchronize preserved hardware state back into @vcpu during attach.
+ * @runtime: Preserved runtime operations table placed in __cpu_preserved_data.
+ */
+struct kvm_x86_caretaker_ops {
+ const char *name;
+ void (*init)(struct kvm_vcpu *vcpu);
+ void (*sync_vcpu)(struct kvm_vcpu *vcpu, void *vcpu_data);
+ const struct kvm_x86_caretaker_runtime_ops *runtime;
+};
+
+void kvm_x86_caretaker_register_ops(const struct kvm_x86_caretaker_ops *ops);
+void kvm_x86_caretaker_unregister_ops(const struct kvm_x86_caretaker_ops *ops);
+
+struct kvm_vcpu_ser;
+
+#ifdef CONFIG_KVM_CARETAKER
+void kvm_arch_vcpu_caretaker_init(struct kvm_vcpu *vcpu);
+void kvm_arch_vcpu_caretaker_unpreserve(struct kvm_vcpu_ser *ser);
+void kvm_arch_vcpu_caretaker_finish(struct kvm_vcpu_ser *ser);
+#else
+static inline void kvm_arch_vcpu_caretaker_init(struct kvm_vcpu *vcpu) {}
+static inline void kvm_arch_vcpu_caretaker_unpreserve(struct kvm_vcpu_ser *ser) {}
+static inline void kvm_arch_vcpu_caretaker_finish(struct kvm_vcpu_ser *ser) {}
+#endif
+#endif /* !__ASSEMBLY__ */
+
+#endif /* __ARCH_X86_KVM_CARETAKER_H */
diff --git a/arch/x86/kvm/caretaker_asm.h b/arch/x86/kvm/caretaker_asm.h
new file mode 100644
index 000000000000..8bae9bf07e16
--- /dev/null
+++ b/arch/x86/kvm/caretaker_asm.h
@@ -0,0 +1,117 @@
+/* SPDX-License-Identifier: GPL-2.0-only */
+/*
+ * Copyright (c) 2026, Google LLC.
+ * Pasha Tatashin <pasha.tatashin at soleen.com>
+ *
+ * Assembly macros for x86 Caretaker host context switch and guest GPRs.
+ */
+#ifndef __ARCH_X86_KVM_CARETAKER_ASM_H
+#define __ARCH_X86_KVM_CARETAKER_ASM_H
+
+#include <linux/objtool.h>
+#include <asm/segment.h>
+#include "caretaker.h"
+#include "kvm-asm-offsets.h"
+
+/* Save host callee-saved registers to caller stack */
+.macro CARETAKER_PUSH_HOST_REGS
+ pushq %rbp
+ movq %rsp, %rbp
+ pushq %rbx
+ pushq %r12
+ pushq %r13
+ pushq %r14
+ pushq %r15
+.endm
+
+/* Restore host callee-saved registers from caller stack */
+.macro CARETAKER_POP_HOST_REGS
+ popq %r15
+ popq %r14
+ popq %r13
+ popq %r12
+ popq %rbx
+ popq %rbp
+.endm
+
+/* Restore guest GPRs (RBX..R15) from preserved page structure */
+.macro CARETAKER_RESTORE_GPRS_NO_RAX base:req
+ movq CXP_REG_RBX(\base), %rbx
+ movq CXP_REG_RCX(\base), %rcx
+ movq CXP_REG_RDX(\base), %rdx
+ movq CXP_REG_RSI(\base), %rsi
+ movq CXP_REG_RBP(\base), %rbp
+ movq CXP_REG_R8(\base), %r8
+ movq CXP_REG_R9(\base), %r9
+ movq CXP_REG_R10(\base), %r10
+ movq CXP_REG_R11(\base), %r11
+ movq CXP_REG_R12(\base), %r12
+ movq CXP_REG_R13(\base), %r13
+ movq CXP_REG_R14(\base), %r14
+ movq CXP_REG_R15(\base), %r15
+.endm
+
+/* Restore guest GPRs including RAX from preserved page structure */
+.macro CARETAKER_RESTORE_GPRS base:req
+ movq CXP_REG_RAX(\base), %rax
+ CARETAKER_RESTORE_GPRS_NO_RAX \base
+.endm
+
+/* Save guest GPRs (RBX..R15) to preserved page structure */
+.macro CARETAKER_SAVE_GPRS_NO_RAX base:req
+ movq %rbx, CXP_REG_RBX(\base)
+ movq %rcx, CXP_REG_RCX(\base)
+ movq %rdx, CXP_REG_RDX(\base)
+ movq %rsi, CXP_REG_RSI(\base)
+ movq %rbp, CXP_REG_RBP(\base)
+ movq %r8, CXP_REG_R8(\base)
+ movq %r9, CXP_REG_R9(\base)
+ movq %r10, CXP_REG_R10(\base)
+ movq %r11, CXP_REG_R11(\base)
+ movq %r12, CXP_REG_R12(\base)
+ movq %r13, CXP_REG_R13(\base)
+ movq %r14, CXP_REG_R14(\base)
+ movq %r15, CXP_REG_R15(\base)
+.endm
+
+/* Save guest GPRs including RAX to preserved page structure */
+.macro CARETAKER_SAVE_GPRS base:req
+ movq %rax, CXP_REG_RAX(\base)
+ CARETAKER_SAVE_GPRS_NO_RAX \base
+.endm
+
+/*
+ * Clear CPU microarchitectural buffers (MDS/TAA/MMIO Stale Data/RFDS) via VERW
+ * before entering guest mode, using a selector stored in .data.cpu_preserved.
+ */
+.macro CARETAKER_CLEAR_CPU_BUFFERS
+ .pushsection .data.cpu_preserved, "aw"
+ .align 2
+.Lcaretaker_verw_sel\@:
+ .word __KERNEL_DS
+ .popsection
+ verw .Lcaretaker_verw_sel\@(%rip)
+.endm
+
+/*
+ * Stuff the 32-entry hardware Return Stack Buffer (RSB) on VM-exit without
+ * referencing external thunks or unmapped per-CPU call-depth variables.
+ */
+.macro CARETAKER_FILL_RETURN_BUFFER reg:req
+ mov $(RSB_CLEAR_LOOPS / 2), \reg
+.Lrsb_loop\@:
+ ANNOTATE_INTRA_FUNCTION_CALL
+ call .Lrsb_1\@
+ int3
+.Lrsb_1\@:
+ ANNOTATE_INTRA_FUNCTION_CALL
+ call .Lrsb_2\@
+ int3
+.Lrsb_2\@:
+ add $16, %rsp
+ dec \reg
+ jnz .Lrsb_loop\@
+ lfence
+.endm
+
+#endif /* __ARCH_X86_KVM_CARETAKER_ASM_H */
diff --git a/arch/x86/kvm/kvm-asm-offsets.c b/arch/x86/kvm/kvm-asm-offsets.c
index 36ac61724dd7..e743ddae255b 100644
--- a/arch/x86/kvm/kvm-asm-offsets.c
+++ b/arch/x86/kvm/kvm-asm-offsets.c
@@ -9,6 +9,9 @@
#include <linux/kbuild.h>
#include "vmx/vmx.h"
#include "svm/svm.h"
+#ifdef CONFIG_KVM_CARETAKER
+#include "caretaker.h"
+#endif
static void __used common(void)
{
@@ -27,4 +30,33 @@ static void __used common(void)
OFFSET(VMX_vcpu_arch_regs, vcpu_vmx, vcpu.arch.regs);
OFFSET(VMX_spec_ctrl, vcpu_vmx, spec_ctrl);
}
+
+#ifdef CONFIG_KVM_CARETAKER
+ /*
+ * Offsets into the caretaker page used by the on-core guest entry and
+ * exit paths in {vmx,svm}/caretaker_vmenter.S. These used to be a
+ * hand-maintained table of literals in caretaker.h.
+ */
+ BLANK();
+ OFFSET(CXP_VMCS_PA, caretaker_x86_page, abi.vmcs_pa);
+ OFFSET(CXP_STACK_ORIG, caretaker_x86_page, stack_orig);
+ OFFSET(CXP_KERNEL_GS_BASE, caretaker_x86_page, kernel_gs_base);
+
+ BLANK();
+ OFFSET(CXP_REG_RAX, caretaker_x86_page, rax);
+ OFFSET(CXP_REG_RBX, caretaker_x86_page, rbx);
+ OFFSET(CXP_REG_RCX, caretaker_x86_page, rcx);
+ OFFSET(CXP_REG_RDX, caretaker_x86_page, rdx);
+ OFFSET(CXP_REG_RSI, caretaker_x86_page, rsi);
+ OFFSET(CXP_REG_RDI, caretaker_x86_page, rdi);
+ OFFSET(CXP_REG_RBP, caretaker_x86_page, rbp);
+ OFFSET(CXP_REG_R8, caretaker_x86_page, r8);
+ OFFSET(CXP_REG_R9, caretaker_x86_page, r9);
+ OFFSET(CXP_REG_R10, caretaker_x86_page, r10);
+ OFFSET(CXP_REG_R11, caretaker_x86_page, r11);
+ OFFSET(CXP_REG_R12, caretaker_x86_page, r12);
+ OFFSET(CXP_REG_R13, caretaker_x86_page, r13);
+ OFFSET(CXP_REG_R14, caretaker_x86_page, r14);
+ OFFSET(CXP_REG_R15, caretaker_x86_page, r15);
+#endif
}
diff --git a/include/linux/kho/abi/kvm_x86.h b/include/linux/kho/abi/kvm_x86.h
index 316250482db5..72038ca65b0f 100644
--- a/include/linux/kho/abi/kvm_x86.h
+++ b/include/linux/kho/abi/kvm_x86.h
@@ -79,6 +79,28 @@ static_assert(offsetof(struct kvm_vcpu_arch_ser, msrs) % sizeof(u64) == 0,
static_assert(offsetof(struct kvm_vcpu_arch_ser, xsave) % 64 == 0,
"xsave must be 64-byte aligned to be XSAVE-able in place");
+#define KVM_X86_CARETAKER_MAX_PAGES 5
+
+/**
+ * struct kvm_caretaker_arch_ser - x86-specific Caretaker control block ABI
+ * @cb: Common Caretaker control block header (must be at offset 0).
+ * @apic_id: Hardware APIC ID of the preserved core, used by the incoming
+ * kernel to deliver the attach-signal IPI.
+ * @nr_preserved_pages: Number of valid physical addresses in @preserved_pages_pa.
+ * @vmcs_pa: Physical address of the VMCS the Caretaker ran the vCPU on
+ * (Intel VMX), used by the incoming kernel to VMCLEAR and
+ * flush the VMCS after the vCPU stops.
+ * @preserved_pages_pa: Physical addresses of per-vCPU hardware control pages
+ * preserved via kho_preserve_pages().
+ */
+struct kvm_caretaker_arch_ser {
+ struct kvm_caretaker_cb_ser cb;
+ u32 apic_id;
+ u32 nr_preserved_pages;
+ u64 vmcs_pa;
+ u64 preserved_pages_pa[KVM_X86_CARETAKER_MAX_PAGES];
+} __packed;
+
#endif /* CONFIG_X86_64 */
#endif /* _LINUX_KHO_ABI_KVM_X86_H */
--
2.55.0.1082.g2b9226bbc0-goog
More information about the kexec
mailing list