[PATCH v10 4/9] riscv: implement preemptible this_cpu void RMW operations
Yunhui Cui
cuiyunhui at bytedance.com
Thu Nov 12 02:41:44 PST 2026
Apply the recoverable per-CPU address handling to void read-modify-write
operations.
Use relaxed AMOs for native-width operations, and Zabha or a masked LR/SC
sequence for subword operations. Use value-returning AMOs even when the old
value is unused so CPU-local operations remain likely to execute near the
CPU.
Link: https://lore.kernel.org/r/20260826010748.zphu3wh2hlm4t4o2@offworld
Signed-off-by: Yunhui Cui <cuiyunhui at bytedance.com>
---
arch/riscv/include/asm/percpu.h | 127 ++++++++++++++++++++++++++++++++
1 file changed, 127 insertions(+)
diff --git a/arch/riscv/include/asm/percpu.h b/arch/riscv/include/asm/percpu.h
index ae7c123a48093..46eafa1ef02a1 100644
--- a/arch/riscv/include/asm/percpu.h
+++ b/arch/riscv/include/asm/percpu.h
@@ -9,8 +9,10 @@
#include <linux/thread_info.h>
#include <asm/asm.h>
+#include <asm/cpufeature-macros.h>
#include <asm/current.h>
#include <asm/gpr-num.h>
+#include <asm/hwcap.h>
#define __my_cpu_offset (((struct thread_info *)current)->pcpu_offset)
@@ -102,6 +104,105 @@ PERCPU_RW_OPS("ld", "sd", 64)
PERCPU_RW_OPS("lw", "sw", 32)
#endif
+/*
+ * Use value-returning AMOs even when the old value is unused. Avoiding x0
+ * as rd makes an AMO more likely to execute near the CPU, which better suits
+ * CPU-local data that is commonly accessed again by the same CPU.
+ */
+#define __PERCPU_AMO_OP_CASE(sfx, name, sz, amo_insn) \
+static __always_inline void \
+__percpu_##name##_amo_case_##sz(void __percpu *pcp, unsigned long val) \
+{ \
+ __PCPU_OP_CONTEXT(); \
+ u##sz tmp; \
+ \
+ asm volatile (__PCPU_GPRS_BEGIN \
+ "amo" #amo_insn #sfx " %[tmp], %[val], 0(%[addr])\n" \
+ __PCPU_GPRS_END \
+ : [gprs] "=m" (__ti->pcpu_gprs), \
+ [off] "=&r" (__off), [addr] "=&r" (__addr), \
+ [tmp] "=&r" (tmp) \
+ : [pcpu_offset] "m" (__ti->pcpu_offset), \
+ [pcp] "r" (pcp), [val] "r" ((u##sz)val) \
+ : "memory"); \
+}
+
+__PERCPU_AMO_OP_CASE(.w, add, 32, add)
+__PERCPU_AMO_OP_CASE(.w, and, 32, and)
+__PERCPU_AMO_OP_CASE(.w, or, 32, or)
+
+#ifdef CONFIG_64BIT
+__PERCPU_AMO_OP_CASE(.d, add, 64, add)
+__PERCPU_AMO_OP_CASE(.d, and, 64, and)
+__PERCPU_AMO_OP_CASE(.d, or, 64, or)
+#endif
+
+#define PERCPU_8_16_GET_SHIFT(pcp) \
+ (((__force unsigned long)(pcp) & 0x3) * BITS_PER_BYTE)
+#define PERCPU_8_16_GET_MASK(sz) GENMASK((sz) - 1, 0)
+#define PERCPU_8_16_GET_PCP32(pcp) \
+ ((void __percpu *)((__force unsigned long)(pcp) & ~0x3UL))
+
+#define __PERCPU_8_16_OP_CASE(name, sz, sfx, amo_insn, op_insn) \
+static __always_inline void \
+__percpu_##name##_amo_case_##sz(void __percpu *pcp, unsigned long val) \
+{ \
+ __PCPU_OP_CONTEXT(); \
+ const u32 field_mask = PERCPU_8_16_GET_MASK(sz); \
+ const unsigned long shift = PERCPU_8_16_GET_SHIFT(pcp); \
+ const u32 mask = field_mask << shift; \
+ const u32 val_trunc = val & field_mask; \
+ void __percpu *pcp32 = PERCPU_8_16_GET_PCP32(pcp); \
+ u32 old, new, field; \
+ u##sz tmp; \
+ \
+ if (IS_ENABLED(CONFIG_RISCV_ISA_ZABHA) && \
+ riscv_has_extension_unlikely(RISCV_ISA_EXT_ZABHA)) { \
+ asm volatile (__PCPU_GPRS_BEGIN \
+ ".option push\n" \
+ ".option arch, +zabha\n" \
+ "amo" #amo_insn #sfx " %[tmp], %[val], 0(%[addr])\n" \
+ ".option pop\n" \
+ __PCPU_GPRS_END \
+ : [gprs] "=m" (__ti->pcpu_gprs), \
+ [off] "=&r" (__off), [addr] "=&r" (__addr), \
+ [tmp] "=&r" (tmp) \
+ : [pcpu_offset] "m" (__ti->pcpu_offset), \
+ [pcp] "r" (pcp), [val] "r" ((u##sz)val) \
+ : "memory"); \
+ } else { \
+ asm volatile (__PCPU_GPRS_BEGIN \
+ "0: lr.w %[old], 0(%[addr])\n" \
+ "and %[field], %[old], %[mask]\n" \
+ "srl %[field], %[field], %[shift]\n" \
+ op_insn " %[field], %[field], %[val]\n" \
+ "and %[field], %[field], %[field_mask]\n" \
+ "sll %[field], %[field], %[shift]\n" \
+ "and %[new], %[old], %[inv_mask]\n" \
+ "or %[new], %[new], %[field]\n" \
+ "sc.w %[new], %[new], 0(%[addr])\n" \
+ "bnez %[new], 0b\n" \
+ __PCPU_GPRS_END \
+ : [gprs] "=m" (__ti->pcpu_gprs), \
+ [off] "=&r" (__off), [addr] "=&r" (__addr), \
+ [old] "=&r" (old), [new] "=&r" (new), \
+ [field] "=&r" (field) \
+ : [pcpu_offset] "m" (__ti->pcpu_offset), \
+ [pcp] "r" (pcp32), [mask] "r" (mask), \
+ [shift] "r" (shift), [val] "r" (val_trunc), \
+ [field_mask] "r" (field_mask), \
+ [inv_mask] "r" (~mask) \
+ : "memory"); \
+ } \
+}
+
+__PERCPU_8_16_OP_CASE(add, 8, .b, add, "add")
+__PERCPU_8_16_OP_CASE(add, 16, .h, add, "add")
+__PERCPU_8_16_OP_CASE(and, 8, .b, and, "and")
+__PERCPU_8_16_OP_CASE(and, 16, .h, and, "and")
+__PERCPU_8_16_OP_CASE(or, 8, .b, or, "or")
+__PERCPU_8_16_OP_CASE(or, 16, .h, or, "or")
+
#define _pcp_wrap(op, pcp, ...) \
({ \
op(&(pcp), __VA_ARGS__); \
@@ -134,7 +235,33 @@ PERCPU_RW_OPS("lw", "sw", 32)
_pcp_wrap(__percpu_write_64, pcp, (unsigned long)(val))
#endif
+#define this_cpu_add_1(pcp, val) _pcp_wrap(__percpu_add_amo_case_8, pcp, val)
+#define this_cpu_add_2(pcp, val) _pcp_wrap(__percpu_add_amo_case_16, pcp, val)
+#define this_cpu_add_4(pcp, val) _pcp_wrap(__percpu_add_amo_case_32, pcp, val)
+#ifdef CONFIG_64BIT
+#define this_cpu_add_8(pcp, val) _pcp_wrap(__percpu_add_amo_case_64, pcp, val)
+#endif
+
+#define this_cpu_and_1(pcp, val) _pcp_wrap(__percpu_and_amo_case_8, pcp, val)
+#define this_cpu_and_2(pcp, val) _pcp_wrap(__percpu_and_amo_case_16, pcp, val)
+#define this_cpu_and_4(pcp, val) _pcp_wrap(__percpu_and_amo_case_32, pcp, val)
+#ifdef CONFIG_64BIT
+#define this_cpu_and_8(pcp, val) _pcp_wrap(__percpu_and_amo_case_64, pcp, val)
+#endif
+
+#define this_cpu_or_1(pcp, val) _pcp_wrap(__percpu_or_amo_case_8, pcp, val)
+#define this_cpu_or_2(pcp, val) _pcp_wrap(__percpu_or_amo_case_16, pcp, val)
+#define this_cpu_or_4(pcp, val) _pcp_wrap(__percpu_or_amo_case_32, pcp, val)
+#ifdef CONFIG_64BIT
+#define this_cpu_or_8(pcp, val) _pcp_wrap(__percpu_or_amo_case_64, pcp, val)
+#endif
+
#undef PERCPU_RW_OPS
+#undef __PERCPU_AMO_OP_CASE
+#undef __PERCPU_8_16_OP_CASE
+#undef PERCPU_8_16_GET_SHIFT
+#undef PERCPU_8_16_GET_MASK
+#undef PERCPU_8_16_GET_PCP32
#undef __PCPU_OP_CONTEXT
#include <asm-generic/percpu.h>
--
2.39.5
More information about the linux-riscv
mailing list