[PATCH v10 2/9] riscv: add infrastructure for preemptible this_cpu operations
Yunhui Cui
cuiyunhui at bytedance.com
Thu Nov 12 02:41:42 PST 2026
RISC-V does not provide architecture-specific this_cpu operations, so
read-modify-write operations use generic fallbacks that disable local
interrupts. Disabling preemption around each architecture-specific
operation would avoid the interrupt-off region, but would still add
preemption-count bookkeeping and a reschedule check to the common path.
Use the recoverable-GPR scheme employed by arm64 so the operations can
remain preemptible. Record the registers used for per-CPU address
generation and repair them on exception return if the task migrated.
Exception entry saves and clears the active state so handlers can use
per-CPU operations independently. Keep this protocol in common assembly
macros so NMI-like entry paths which bypass handle_exception, such as SBI
Supervisor Software Events, can preserve an interrupted operation.
The existing exception return path clears LR/SC reservations, so an
interrupted LR/SC sequence retries after its saved address is repaired.
Suggested-by: David Laight <david.laight.linux at gmail.com>
Link: https://lore.kernel.org/r/20261108213254.67805ef0@pumpkin
Suggested-by: Jisheng Zhang <jszhang at kernel.org>
Link: https://lore.kernel.org/r/avG07pmpcLGkq5i-@xhacker
Signed-off-by: Yunhui Cui <cuiyunhui at bytedance.com>
---
arch/riscv/include/asm/asm.h | 19 ++++++++++++
arch/riscv/include/asm/percpu.h | 43 ++++++++++++++++++++++++++++
arch/riscv/include/asm/ptrace.h | 15 ++++++++++
arch/riscv/include/asm/thread_info.h | 1 +
arch/riscv/kernel/Makefile | 1 +
arch/riscv/kernel/asm-offsets.c | 2 ++
arch/riscv/kernel/entry-common.c | 32 +++++++++++++++++++++
arch/riscv/kernel/entry.S | 10 +++++++
8 files changed, 123 insertions(+)
create mode 100644 arch/riscv/kernel/entry-common.c
diff --git a/arch/riscv/include/asm/asm.h b/arch/riscv/include/asm/asm.h
index 06ac38f733b0a..57af10a936d9b 100644
--- a/arch/riscv/include/asm/asm.h
+++ b/arch/riscv/include/asm/asm.h
@@ -81,6 +81,25 @@
/* Common assembly source macros */
+/*
+ * Save and clear the active per-CPU GPR state on exception entry. Entry
+ * paths which bypass handle_exception, such as SBI Supervisor Software
+ * Events (SSE), must pair this with pcpu_gprs_exit around their handlers.
+ */
+.macro pcpu_gprs_entry tsk regs tmp
+ lhu \tmp, TASK_TI_PCPU_GPRS(\tsk)
+ REG_S \tmp, PT_PCPU_GPRS(\regs)
+ sh zero, TASK_TI_PCPU_GPRS(\tsk)
+.endm
+
+/*
+ * Restore the saved state before resuming the interrupted context. tmp must
+ * contain PT_PCPU_GPRS from the interrupted frame.
+ */
+.macro pcpu_gprs_exit tsk tmp
+ sh \tmp, TASK_TI_PCPU_GPRS(\tsk)
+.endm
+
/*
* NOP sequence
*/
diff --git a/arch/riscv/include/asm/percpu.h b/arch/riscv/include/asm/percpu.h
index faf9a75ec66d7..59f7d44f7d3fe 100644
--- a/arch/riscv/include/asm/percpu.h
+++ b/arch/riscv/include/asm/percpu.h
@@ -3,12 +3,55 @@
#ifndef __ASM_PERCPU_H
#define __ASM_PERCPU_H
+#include <linux/bits.h>
+#include <linux/stringify.h>
#include <linux/thread_info.h>
+#include <asm/asm.h>
#include <asm/current.h>
+#include <asm/gpr-num.h>
#define __my_cpu_offset (((struct thread_info *)current)->pcpu_offset)
+#define PCPU_GPR_PCP_SHIFT 0
+#define PCPU_GPR_PCP GENMASK(4, 0)
+#define PCPU_GPR_OFF_SHIFT 5
+#define PCPU_GPR_OFF GENMASK(9, 5)
+#define PCPU_GPR_ADDR_SHIFT 10
+#define PCPU_GPR_ADDR GENMASK(14, 10)
+
+#define __PCPU_GPR_NUM(gpr) ".L__gpr_num_" gpr
+
+#define __PCPU_GPRS_VALUE \
+ "(" \
+ "(" __PCPU_GPR_NUM("%[pcp]") " << " __stringify(PCPU_GPR_PCP_SHIFT) ") | " \
+ "(" __PCPU_GPR_NUM("%[off]") " << " __stringify(PCPU_GPR_OFF_SHIFT) ") | " \
+ "(" __PCPU_GPR_NUM("%[addr]") " << " __stringify(PCPU_GPR_ADDR_SHIFT) ")" \
+ ")"
+
+#define __PCPU_GPRS_ASSERT_DISTINCT \
+ ".if ((" __PCPU_GPR_NUM("%[pcp]") " == " __PCPU_GPR_NUM("%[off]") ") || " \
+ "(" __PCPU_GPR_NUM("%[pcp]") " == " __PCPU_GPR_NUM("%[addr]") ") || " \
+ "(" __PCPU_GPR_NUM("%[off]") " == " __PCPU_GPR_NUM("%[addr]") "))\n" \
+ ".error \"PCPU GPRs overlap\"\n" \
+ ".endif\n"
+
+/*
+ * Mark a region where the per-CPU offset and address registers may need to
+ * be rewritten by exception return after migration. The three registers
+ * must remain distinct for the whole region.
+ */
+#define __PCPU_GPRS_BEGIN \
+ __DEFINE_ASM_GPR_NUMS \
+ __PCPU_GPRS_ASSERT_DISTINCT \
+ "li %[off], " __PCPU_GPRS_VALUE "\n" \
+ "sh %[off], %[gprs]\n" \
+ REG_L " %[off], %[pcpu_offset]\n" \
+ "add %[addr], %[pcp], %[off]\n"
+
+#define __PCPU_GPRS_END \
+ "sh zero, %[gprs]\n"
+
#include <asm-generic/percpu.h>
#endif /* __ASM_PERCPU_H */
diff --git a/arch/riscv/include/asm/ptrace.h b/arch/riscv/include/asm/ptrace.h
index addc8188152f7..ceff8f63bc866 100644
--- a/arch/riscv/include/asm/ptrace.h
+++ b/arch/riscv/include/asm/ptrace.h
@@ -53,6 +53,8 @@ struct pt_regs {
unsigned long cause;
/* a0 value before the syscall */
unsigned long orig_a0;
+ /* Active preemptible per-CPU operation, if any. */
+ unsigned long pcpu_gprs;
};
#define PTRACE_SYSEMU 0x1f
@@ -145,6 +147,19 @@ static inline unsigned long regs_get_register(struct pt_regs *regs,
return *(unsigned long *)((unsigned long)regs + offset);
}
+static inline void regs_set_register(struct pt_regs *regs,
+ unsigned int offset, unsigned long val)
+{
+ if (unlikely(offset > MAX_REG_OFFSET))
+ return;
+
+ /* Register x0 is represented by the first, read-only slot. */
+ if (offset)
+ *(unsigned long *)((unsigned long)regs + offset) = val;
+}
+
+void noinstr riscv_pcpu_regs_fixup(struct pt_regs *regs);
+
/**
* regs_get_kernel_argument() - get Nth function argument in kernel
* @regs: pt_regs of that context
diff --git a/arch/riscv/include/asm/thread_info.h b/arch/riscv/include/asm/thread_info.h
index f10ba62b61016..51c4076763c1b 100644
--- a/arch/riscv/include/asm/thread_info.h
+++ b/arch/riscv/include/asm/thread_info.h
@@ -63,6 +63,7 @@ struct thread_info {
long user_sp; /* User stack pointer */
unsigned long syscall_work; /* SYSCALL_WORK_ flags */
unsigned long pcpu_offset;
+ unsigned short pcpu_gprs;
#ifdef CONFIG_SHADOW_CALL_STACK
void *scs_base;
void *scs_sp;
diff --git a/arch/riscv/kernel/Makefile b/arch/riscv/kernel/Makefile
index f54c336bf547c..bf786d5dd4de3 100644
--- a/arch/riscv/kernel/Makefile
+++ b/arch/riscv/kernel/Makefile
@@ -52,6 +52,7 @@ obj-y += alternative.o
obj-y += cpu.o
obj-y += cpufeature.o
obj-y += entry.o
+obj-y += entry-common.o
obj-y += irq.o
obj-y += process.o
obj-y += ptrace.o
diff --git a/arch/riscv/kernel/asm-offsets.c b/arch/riscv/kernel/asm-offsets.c
index f4f7ee976cd94..38f246a691c60 100644
--- a/arch/riscv/kernel/asm-offsets.c
+++ b/arch/riscv/kernel/asm-offsets.c
@@ -39,6 +39,7 @@ void asm_offsets(void)
OFFSET(TASK_TI_CPU, task_struct, thread_info.cpu);
OFFSET(TASK_TI_PCPU_OFFSET, task_struct, thread_info.pcpu_offset);
+ OFFSET(TASK_TI_PCPU_GPRS, task_struct, thread_info.pcpu_gprs);
OFFSET(TASK_TI_PREEMPT_COUNT, task_struct, thread_info.preempt_count);
OFFSET(TASK_TI_KERNEL_SP, task_struct, thread_info.kernel_sp);
OFFSET(TASK_TI_USER_SP, task_struct, thread_info.user_sp);
@@ -130,6 +131,7 @@ void asm_offsets(void)
OFFSET(PT_STATUS, pt_regs, status);
OFFSET(PT_BADADDR, pt_regs, badaddr);
OFFSET(PT_CAUSE, pt_regs, cause);
+ OFFSET(PT_PCPU_GPRS, pt_regs, pcpu_gprs);
OFFSET(SUSPEND_CONTEXT_REGS, suspend_context, regs);
diff --git a/arch/riscv/kernel/entry-common.c b/arch/riscv/kernel/entry-common.c
new file mode 100644
index 0000000000000..023d856b1679b
--- /dev/null
+++ b/arch/riscv/kernel/entry-common.c
@@ -0,0 +1,32 @@
+// SPDX-License-Identifier: GPL-2.0-only
+
+#include <linux/bitfield.h>
+#include <linux/compiler.h>
+#include <linux/percpu.h>
+
+#include <asm/current.h>
+#include <asm/ptrace.h>
+
+/*
+ * If the interrupted context had an active per-CPU operation, rewrite its
+ * saved offset and address registers for the CPU on which it will resume.
+ */
+void noinstr riscv_pcpu_regs_fixup(struct pt_regs *regs)
+{
+ unsigned long pcpu_offset = __my_cpu_offset;
+ unsigned long gprs = regs->pcpu_gprs;
+ unsigned long pcp;
+ unsigned int reg_pcp;
+ unsigned int reg_off;
+ unsigned int reg_addr;
+
+ reg_pcp = FIELD_GET(PCPU_GPR_PCP, gprs);
+ reg_off = FIELD_GET(PCPU_GPR_OFF, gprs);
+ reg_addr = FIELD_GET(PCPU_GPR_ADDR, gprs);
+
+ /* pt_regs offset zero represents x0, followed by x1 through x31. */
+ pcp = regs_get_register(regs, reg_pcp * sizeof(unsigned long));
+ regs_set_register(regs, reg_off * sizeof(unsigned long), pcpu_offset);
+ regs_set_register(regs, reg_addr * sizeof(unsigned long),
+ pcp + pcpu_offset);
+}
diff --git a/arch/riscv/kernel/entry.S b/arch/riscv/kernel/entry.S
index 628b85e3915a7..65298ac828959 100644
--- a/arch/riscv/kernel/entry.S
+++ b/arch/riscv/kernel/entry.S
@@ -198,6 +198,8 @@ SYM_CODE_START(handle_exception)
REG_S s4, PT_CAUSE(sp)
REG_S s5, PT_TP(sp)
+ pcpu_gprs_entry tp sp s0
+
/*
* Set the scratch register to 0, so that if a recursive exception
* occurs, the exception vector knows it came from the kernel
@@ -281,6 +283,14 @@ SYM_CODE_START_NOALIGN(ret_from_exception)
move a0, sp
call riscv_v_context_nesting_end
#endif
+ REG_L t0, PT_PCPU_GPRS(sp)
+ beqz t0, .Lpcpu_regs_restored
+ move a0, sp
+ call riscv_pcpu_regs_fixup
+ REG_L t0, PT_PCPU_GPRS(sp)
+.Lpcpu_regs_restored:
+ pcpu_gprs_exit tp t0
+
REG_L a0, PT_STATUS(sp)
restore_userssp s3, a0
/*
--
2.39.5
More information about the linux-riscv
mailing list