[PATCH v10 7/9] riscv: implement preemptible this_cpu cmpxchg operations
Yunhui Cui
cuiyunhui at bytedance.com
Thu Nov 12 02:41:47 PST 2026
Apply the recoverable per-CPU address handling to compare-and-exchange
operations. Providing this_cpu_cmpxchg() also allows asm-generic to derive
this_cpu_try_cmpxchg().
Use Zacas where available and LR/SC otherwise. Keep masked subword operands
XLEN-wide so RV64 LR.W sign extension does not affect the comparison, and
use typed helpers to separate asm-generic from the architecture cmpxchg
macro expansion.
Link: https://lore.kernel.org/r/20260826010748.zphu3wh2hlm4t4o2@offworld
Signed-off-by: Yunhui Cui <cuiyunhui at bytedance.com>
---
arch/riscv/include/asm/percpu.h | 140 ++++++++++++++++++++++++++++++++
1 file changed, 140 insertions(+)
diff --git a/arch/riscv/include/asm/percpu.h b/arch/riscv/include/asm/percpu.h
index 810f4c5eaa508..4d2fc10cbdab1 100644
--- a/arch/riscv/include/asm/percpu.h
+++ b/arch/riscv/include/asm/percpu.h
@@ -401,6 +401,123 @@ __percpu_xchg_case_##sz(void __percpu *pcp, unsigned long val) \
__PERCPU_8_16_XCHG_CASE(8, .b)
__PERCPU_8_16_XCHG_CASE(16, .h)
+
+#define __PERCPU_CMPXCHG_CASE(sz, lr_sfx, sc_sfx, cas_sfx) \
+static __always_inline unsigned long \
+__percpu_cmpxchg_case_##sz(void __percpu *pcp, unsigned long old, \
+ unsigned long new) \
+{ \
+ __PCPU_OP_CONTEXT(); \
+ const long cmp = (long)(s##sz)(u##sz)old; \
+ unsigned long rc; \
+ u##sz ret = (u##sz)old; \
+ \
+ if (IS_ENABLED(CONFIG_RISCV_ISA_ZACAS) && \
+ IS_ENABLED(CONFIG_TOOLCHAIN_HAS_ZACAS) && \
+ riscv_has_extension_unlikely(RISCV_ISA_EXT_ZACAS)) { \
+ asm volatile (__PCPU_GPRS_BEGIN \
+ ".option push\n" \
+ ".option arch, +zacas\n" \
+ "amocas" #cas_sfx " %[ret], %[new], 0(%[addr])\n" \
+ ".option pop\n" \
+ __PCPU_GPRS_END \
+ : [gprs] "=m" (__ti->pcpu_gprs), \
+ [off] "=&r" (__off), [addr] "=&r" (__addr), \
+ [ret] "+&r" (ret) \
+ : [pcpu_offset] "m" (__ti->pcpu_offset), \
+ [pcp] "r" (pcp), [new] "r" ((u##sz)new) \
+ : "memory"); \
+ } else { \
+ asm volatile (__PCPU_GPRS_BEGIN \
+ "0: lr" #lr_sfx " %[ret], 0(%[addr])\n" \
+ "bne %[ret], %[old], 1f\n" \
+ "sc" #sc_sfx " %[rc], %[new], 0(%[addr])\n" \
+ "bnez %[rc], 0b\n" \
+ "1:\n" \
+ __PCPU_GPRS_END \
+ : [gprs] "=m" (__ti->pcpu_gprs), \
+ [off] "=&r" (__off), [addr] "=&r" (__addr), \
+ [ret] "=&r" (ret), [rc] "=&r" (rc) \
+ : [pcpu_offset] "m" (__ti->pcpu_offset), \
+ [pcp] "r" (pcp), [old] "r" (cmp), \
+ [new] "r" ((u##sz)new) \
+ : "memory"); \
+ } \
+ \
+ return ret; \
+}
+
+__PERCPU_CMPXCHG_CASE(32, .w, .w, .w)
+#ifdef CONFIG_64BIT
+__PERCPU_CMPXCHG_CASE(64, .d, .d, .d)
+#endif
+
+/*
+ * Use XLEN-wide masks to clear the upper bits of an RV64 sign-extended
+ * LR.W result before comparing a high byte or halfword.
+ */
+#define __PERCPU_8_16_CMPXCHG_CASE(sz, sfx) \
+static __always_inline unsigned long \
+__percpu_cmpxchg_case_##sz(void __percpu *pcp, unsigned long old, \
+ unsigned long new) \
+{ \
+ __PCPU_OP_CONTEXT(); \
+ const unsigned long shift = PERCPU_8_16_GET_SHIFT(pcp); \
+ const unsigned long field_mask = PERCPU_8_16_GET_MASK(sz); \
+ const unsigned long mask = field_mask << shift; \
+ const unsigned long old_field = (old & field_mask) << shift; \
+ const unsigned long new_field = (new & field_mask) << shift; \
+ void __percpu *pcp32 = PERCPU_8_16_GET_PCP32(pcp); \
+ unsigned long ret32, tmp, rc; \
+ u##sz ret = (u##sz)old; \
+ \
+ if (IS_ENABLED(CONFIG_RISCV_ISA_ZABHA) && \
+ IS_ENABLED(CONFIG_RISCV_ISA_ZACAS) && \
+ IS_ENABLED(CONFIG_TOOLCHAIN_HAS_ZACAS) && \
+ riscv_has_extension_unlikely(RISCV_ISA_EXT_ZABHA) && \
+ riscv_has_extension_unlikely(RISCV_ISA_EXT_ZACAS)) { \
+ asm volatile (__PCPU_GPRS_BEGIN \
+ ".option push\n" \
+ ".option arch, +zacas, +zabha\n" \
+ "amocas" #sfx " %[ret], %[new], 0(%[addr])\n" \
+ ".option pop\n" \
+ __PCPU_GPRS_END \
+ : [gprs] "=m" (__ti->pcpu_gprs), \
+ [off] "=&r" (__off), [addr] "=&r" (__addr), \
+ [ret] "+&r" (ret) \
+ : [pcpu_offset] "m" (__ti->pcpu_offset), \
+ [pcp] "r" (pcp), [new] "r" ((u##sz)new) \
+ : "memory"); \
+ } else { \
+ asm volatile (__PCPU_GPRS_BEGIN \
+ "0: lr.w %[ret], 0(%[addr])\n" \
+ "and %[tmp], %[ret], %[mask]\n" \
+ "bne %[tmp], %[old], 1f\n" \
+ "and %[tmp], %[ret], %[inv_mask]\n" \
+ "or %[tmp], %[tmp], %[new]\n" \
+ "sc.w %[rc], %[tmp], 0(%[addr])\n" \
+ "bnez %[rc], 0b\n" \
+ "1:\n" \
+ __PCPU_GPRS_END \
+ : [gprs] "=m" (__ti->pcpu_gprs), \
+ [off] "=&r" (__off), [addr] "=&r" (__addr), \
+ [ret] "=&r" (ret32), [tmp] "=&r" (tmp), \
+ [rc] "=&r" (rc) \
+ : [pcpu_offset] "m" (__ti->pcpu_offset), \
+ [pcp] "r" (pcp32), [mask] "r" (mask), \
+ [old] "r" (old_field), \
+ [new] "r" (new_field), \
+ [inv_mask] "r" (~mask) \
+ : "memory"); \
+ \
+ ret = (ret32 & mask) >> shift; \
+ } \
+ \
+ return ret; \
+}
+
+__PERCPU_8_16_CMPXCHG_CASE(8, .b)
+__PERCPU_8_16_CMPXCHG_CASE(16, .h)
#define _pcp_wrap(op, pcp, ...) \
({ \
op(&(pcp), __VA_ARGS__); \
@@ -421,6 +538,15 @@ __PERCPU_8_16_XCHG_CASE(16, .h)
(unsigned long)(val)); \
})
+#define _pcp_wrap_cmpxchg(op, pcp, old, new) \
+({ \
+ __auto_type __pcp_ptr = &(pcp); \
+ \
+ (TYPEOF_UNQUAL(*__pcp_ptr))op(__pcp_ptr, \
+ (unsigned long)(old), \
+ (unsigned long)(new)); \
+})
+
#define this_cpu_read_1(pcp) _pcp_wrap_return(__percpu_read_8, pcp)
#define this_cpu_read_2(pcp) _pcp_wrap_return(__percpu_read_16, pcp)
#define this_cpu_read_4(pcp) _pcp_wrap_return(__percpu_read_32, pcp)
@@ -480,11 +606,25 @@ __PERCPU_8_16_XCHG_CASE(16, .h)
#define this_cpu_xchg_8(pcp, val) _pcp_wrap_xchg(__percpu_xchg_case_64, pcp, val)
#endif
+#define this_cpu_cmpxchg_1(pcp, o, n) \
+ _pcp_wrap_cmpxchg(__percpu_cmpxchg_case_8, pcp, o, n)
+#define this_cpu_cmpxchg_2(pcp, o, n) \
+ _pcp_wrap_cmpxchg(__percpu_cmpxchg_case_16, pcp, o, n)
+#define this_cpu_cmpxchg_4(pcp, o, n) \
+ _pcp_wrap_cmpxchg(__percpu_cmpxchg_case_32, pcp, o, n)
+#ifdef CONFIG_64BIT
+#define this_cpu_cmpxchg_8(pcp, o, n) \
+ _pcp_wrap_cmpxchg(__percpu_cmpxchg_case_64, pcp, o, n)
+#define this_cpu_cmpxchg64(pcp, o, n) this_cpu_cmpxchg_8(pcp, o, n)
+#endif
+
#undef PERCPU_RW_OPS
#undef __PERCPU_AMO_OP_CASE
#undef __PERCPU_8_16_OP_CASE
#undef __PERCPU_XCHG_AMO_CASE
#undef __PERCPU_8_16_XCHG_CASE
+#undef __PERCPU_CMPXCHG_CASE
+#undef __PERCPU_8_16_CMPXCHG_CASE
#undef PERCPU_8_16_GET_SHIFT
#undef PERCPU_8_16_GET_MASK
#undef PERCPU_8_16_GET_PCP32
--
2.39.5
More information about the linux-riscv
mailing list