[PATCH v4 16/21] arm64: percpu: Implement preemptible void RMW ops
Vladimir Murzin
vladimir.murzin at arm.com
Tue Sep 15 05:01:29 PDT 2026
On 9/8/26 16:17, Mark Rutland wrote:
> Use the PCPU GPR infrastructure to implement all of the RMW ops which do
> not return a value.
>
> Test case:
>
> | void outline_this_cpu_add_u64(u64 __percpu *p, u64 v)
> | {
> | this_cpu_add(*p, v);
> | }
>
> Generated code before this patch (v7.2-rc4):
>
> | <outline_this_cpu_add_u64>:
> | paciasp
> | stp x29, x30, [sp, #-16]!
> | mrs x2, sp_el0
> | mov x29, sp
> | ldr w3, [x2, #8]
> | add w3, w3, #0x1
> | str w3, [x2, #8]
> | mrs x3, tpidr_el1
> | add x0, x0, x3
> | 1: ldxr x5, [x0]
> | add x5, x5, x1
> | stxr w4, x5, [x0]
> | cbnz w4, 1b
> | ldr x0, [x2, #8]
> | sub x0, x0, #0x1
> | str w0, [x2, #8]
> | cbz x0, 2f
> | ldr x0, [x2, #8]
> | cbnz x0, 3f
> | 2: bl preempt_schedule_notrace
> | 3: ldp x29, x30, [sp], #16
> | autiasp
> | ret
>
> Generated code after this patch:
>
> | <outline_this_cpu_add_u64>:
> | mrs x2, sp_el0
> | mov w4, #0xc80
> | strh w4, [x2, #20]
> | mrs x4, tpidr_el1
> | add x3, x0, x4
> | 1: ldxr x6, [x3]
> | add x6, x6, x1
> | stxr w5, x6, [x3]
> | cbnz w5, 1b
> | strh wzr, [x2, #20]
> | ret
>
> Signed-off-by: Mark Rutland <mark.rutland at arm.com>
> Tested-by: Muhammad Usama Anjum <usama.anjum at arm.com>
> Cc: Ada Couprie Diaz <ada.coupriediaz at arm.com>
> Cc: Ard Biesheuvel <ardb at kernel.org>
> Cc: Catalin Marinas <catalin.marinas at arm.com>
> Cc: James Morse <james.morse at arm.com>
> Cc: Jinjie Ruan <ruanjinjie at huawei.com>
> Cc: Marc Zyngier <maz at kernel.org>
> Cc: Peter Zijlstra <peterz at infradead.org>
> Cc: Vladimir Murzin <vladimir.murzin at arm.com>
> Cc: Will Deacon <will at kernel.org>
> Cc: Yang Shi <yang at os.amperecomputing.com>
> ---
> arch/arm64/include/asm/percpu.h | 52 ++++++++++++++++++++-------------
> 1 file changed, 32 insertions(+), 20 deletions(-)
>
> diff --git a/arch/arm64/include/asm/percpu.h b/arch/arm64/include/asm/percpu.h
> index 984f69846cc09..504a7fefa2015 100644
> --- a/arch/arm64/include/asm/percpu.h
> +++ b/arch/arm64/include/asm/percpu.h
> @@ -168,23 +168,35 @@ static inline void __percpu_write_##sz(void __percpu *pcp, unsigned long val) \
>
> #define __PERCPU_OP_CASE(w, sfx, name, sz, op_llsc, op_lse) \
> static inline void \
> -__percpu_##name##_case_##sz(void *ptr, unsigned long val) \
> +__percpu_##name##_case_##sz(void __percpu *pcp, unsigned long val) \
> { \
> + u16 *gprs = ¤t_thread_info()->pcpu_gprs; \
> + unsigned long addr; \
> + unsigned long off; \
> unsigned int loop; \
> u##sz tmp; \
> \
> - asm volatile (ARM64_LSE_ATOMIC_INSN( \
> + asm volatile ( \
> + __PCPU_GPRS_BEGIN("%[gprs]", "%[pcp]", "%[off]", "%[addr]") \
> + ARM64_LSE_ATOMIC_INSN( \
> /* LL/SC */ \
> - "1: ldxr" #sfx "\t%" #w "[tmp], %[ptr]\n" \
> + "1: ldxr" #sfx "\t%" #w "[tmp], [%[addr]]\n" \
> #op_llsc "\t%" #w "[tmp], %" #w "[tmp], %" #w "[val]\n" \
> - " stxr" #sfx "\t%w[loop], %" #w "[tmp], %[ptr]\n" \
> + " stxr" #sfx "\t%w[loop], %" #w "[tmp], [%[addr]]\n" \
> " cbnz %w[loop], 1b", \
> /* LSE atomics */ \
> - #op_lse #sfx "\t%" #w "[val], %" #w "[tmp], %[ptr]\n" \
> + #op_lse #sfx "\t%" #w "[val], %" #w "[tmp], [%[addr]]\n"\
> __nops(3)) \
> - : [loop] "=&r" (loop), [tmp] "=&r" (tmp), \
> - [ptr] "+Q"(*(u##sz *)ptr) \
> - : [val] "r" ((u##sz)(val))); \
> + __PCPU_GPRS_END("%[gprs]") \
> + : [gprs] "=Qo" (*gprs), \
> + [addr] "=&r" (addr), \
> + [off] "=&r" (off), \
> + [loop] "=&r" (loop), \
> + [tmp] "=&r" (tmp) \
> + : [pcp] "r" (pcp), \
> + [val] "r" ((u##sz)(val)) \
> + : "memory" \
> + ); \
> }
>
> #define __PERCPU_RET_OP_CASE(w, sfx, name, sz, op_llsc, op_lse) \
> @@ -301,13 +313,13 @@ PERCPU_RET_OP(add, add, ldadd)
> _pcp_wrap(__percpu_write_64, pcp, (unsigned long)(val))
>
> #define this_cpu_add_1(pcp, val) \
> - _pcp_protect(__percpu_add_case_8, pcp, val)
> + _pcp_wrap(__percpu_add_case_8, pcp, val)
> #define this_cpu_add_2(pcp, val) \
> - _pcp_protect(__percpu_add_case_16, pcp, val)
> + _pcp_wrap(__percpu_add_case_16, pcp, val)
> #define this_cpu_add_4(pcp, val) \
> - _pcp_protect(__percpu_add_case_32, pcp, val)
> + _pcp_wrap(__percpu_add_case_32, pcp, val)
> #define this_cpu_add_8(pcp, val) \
> - _pcp_protect(__percpu_add_case_64, pcp, val)
> + _pcp_wrap(__percpu_add_case_64, pcp, val)
>
> #define this_cpu_add_return_1(pcp, val) \
> _pcp_protect_return(__percpu_add_return_case_8, pcp, val)
> @@ -319,22 +331,22 @@ PERCPU_RET_OP(add, add, ldadd)
> _pcp_protect_return(__percpu_add_return_case_64, pcp, val)
>
> #define this_cpu_and_1(pcp, val) \
> - _pcp_protect(__percpu_andnot_case_8, pcp, ~(u8)(val))
> + _pcp_wrap(__percpu_andnot_case_8, pcp, ~(u8)(val))
> #define this_cpu_and_2(pcp, val) \
> - _pcp_protect(__percpu_andnot_case_16, pcp, ~(u16)(val))
> + _pcp_wrap(__percpu_andnot_case_16, pcp, ~(u16)(val))
> #define this_cpu_and_4(pcp, val) \
> - _pcp_protect(__percpu_andnot_case_32, pcp, ~(u32)(val))
> + _pcp_wrap(__percpu_andnot_case_32, pcp, ~(u32)(val))
> #define this_cpu_and_8(pcp, val) \
> - _pcp_protect(__percpu_andnot_case_64, pcp, ~(u64)(val))
> + _pcp_wrap(__percpu_andnot_case_64, pcp, ~(u64)(val))
>
> #define this_cpu_or_1(pcp, val) \
> - _pcp_protect(__percpu_or_case_8, pcp, val)
> + _pcp_wrap(__percpu_or_case_8, pcp, val)
> #define this_cpu_or_2(pcp, val) \
> - _pcp_protect(__percpu_or_case_16, pcp, val)
> + _pcp_wrap(__percpu_or_case_16, pcp, val)
> #define this_cpu_or_4(pcp, val) \
> - _pcp_protect(__percpu_or_case_32, pcp, val)
> + _pcp_wrap(__percpu_or_case_32, pcp, val)
> #define this_cpu_or_8(pcp, val) \
> - _pcp_protect(__percpu_or_case_64, pcp, val)
> + _pcp_wrap(__percpu_or_case_64, pcp, val)
>
> #define this_cpu_xchg_1(pcp, val) \
> _pcp_protect_return(xchg_relaxed, pcp, val)
> -- 2.30.2
>
FWIW,
Reviewed-by: Vladimir Murzin <vladimir.murzin at arm.com>
More information about the linux-arm-kernel
mailing list