[PATCH v4 18/21] arm64: percpu: Implement preemptible XCHG ops

[Vladimir Murzin]

On 9/8/26 16:17, Mark Rutland wrote:
> Use the PCPU GPR infrastructure to implement all of the xchg ops
> 
> Test case:
> 
> | u64 outline_xchg_name(u64 *p, u64 v)
> | {
> | 	return xchg(p, v);
> | }
> 
> Generated code before this patch (v7.2-rc4):
> 
> | <outline_this_cpu_xchg_u64>:
> |        paciasp
> |        stp     x29, x30, [sp, #-32]!
> |        mrs     x3, sp_el0
> |        mov     x29, sp
> |        ldr     w2, [x3, #8]
> |        add     w2, w2, #0x1
> |        str     w2, [x3, #8]
> |        mov     x2, x0
> |        mrs     x4, tpidr_el1
> |        add     x2, x2, x4
> |        prfm    pstl1strm, [x2]
> | 1:     ldxr    x0, [x2]
> |        stxr    w5, x1, [x2]
> |        cbnz    w5, 1b
> |        ldr     x1, [x3, #8]
> |        sub     x1, x1, #0x1
> |        str     w1, [x3, #8]
> |        cbz     x1, 2f
> |        ldr     x1, [x3, #8]
> |        cbnz    x1, 3f
> | 2:     str     x0, [sp, #24]
> |        bl      preempt_schedule_notrace
> |        ldr     x0, [sp, #24]
> | 3:     ldp     x29, x30, [sp], #32
> |        autiasp
> |        ret
> 
> Generated code after this patch:
> 
> | <outline_this_cpu_xchg_u64>:
> |        mrs     x3, sp_el0
> |        mov     w5, #0x10a0
> |        strh    w5, [x3, #20]
> |        mrs     x5, tpidr_el1
> |        add     x4, x0, x5
> |        prfm    pstl1strm, [x4]
> | 1:     ldxr    x2, [x4]
> |        stxr    w6, x1, [x4]
> |        cbnz    w6, 1b
> |        strh    wzr, [x3, #20]
> |        mov     x0, x2
> |        ret
> 
> Signed-off-by: Mark Rutland <mark.rutland at arm.com>
> Tested-by: Muhammad Usama Anjum <usama.anjum at arm.com>
> Cc: Ada Couprie Diaz <ada.coupriediaz at arm.com>
> Cc: Ard Biesheuvel <ardb at kernel.org>
> Cc: Catalin Marinas <catalin.marinas at arm.com>
> Cc: James Morse <james.morse at arm.com>
> Cc: Jinjie Ruan <ruanjinjie at huawei.com>
> Cc: Marc Zyngier <maz at kernel.org>
> Cc: Peter Zijlstra <peterz at infradead.org>
> Cc: Vladimir Murzin <vladimir.murzin at arm.com>
> Cc: Will Deacon <will at kernel.org>
> Cc: Yang Shi <yang at os.amperecomputing.com>
> ---
>  arch/arm64/include/asm/percpu.h | 57 ++++++++++++++++++++++++++++++---
>  1 file changed, 53 insertions(+), 4 deletions(-)
> 
> diff --git a/arch/arm64/include/asm/percpu.h b/arch/arm64/include/asm/percpu.h
> index 9fe44d04c871b..820340c61d70f 100644
> --- a/arch/arm64/include/asm/percpu.h
> +++ b/arch/arm64/include/asm/percpu.h
> @@ -269,6 +269,50 @@ PERCPU_RET_OP(add, add, ldadd)
>  #undef PERCPU_OP
>  #undef PERCPU_RET_OP
>  
> +#define PERCPU_XCHG_OP(w, sfx, sz)					\
> +static inline unsigned long						\
> +__percpu_xchg_case_##sz(void __percpu *pcp, u##sz val)			\
> +{									\
> +	u16 *gprs = &current_thread_info()->pcpu_gprs;			\
> +	unsigned long addr;						\
> +	unsigned long off;						\
> +	unsigned int loop;						\
> +	unsigned long ret;						\
> +									\
> +	asm volatile (							\
> +	__PCPU_GPRS_BEGIN("%[gprs]", "%[pcp]", "%[off]", "%[addr]")	\
> +	ARM64_LSE_ATOMIC_INSN(						\
> +	/* LL/SC */							\
> +	"	prfm	pstl1strm, [%[addr]]\n"				\
> +	"1:	ldxr" #sfx "\t%" #w "[ret], [%[addr]]\n"		\
> +	"	stxr" #sfx "\t%w[loop], %" #w "[val], [%[addr]]\n"	\
> +	"	cbnz	%w[loop], 1b\n"					\
> +	,								\
> +	/* LSE atomics */						\
> +	"	swp" #sfx "\t%" #w "[val], %" #w "[ret], [%[addr]]\n"	\
> +		__nops(3)						\
> +	)								\
> +	__PCPU_GPRS_END("%[gprs]")					\
> +	: [gprs] "=Qo" (*gprs),						\
> +	  [addr] "=&r" (addr),						\
> +	  [off] "=&r" (off),						\
> +	  [loop] "=&r" (loop),						\
> +	  [ret] "=&r" (ret)						\
> +	: [pcp] "r" (pcp),						\
> +	  [val] "r" (val)						\
> +	: "memory"							\
> +	);								\
> +									\
> +	return ret;							\
> +}
> +
> +PERCPU_XCHG_OP(w, b, 8)
> +PERCPU_XCHG_OP(w, h, 16)
> +PERCPU_XCHG_OP(w,  , 32)
> +PERCPU_XCHG_OP(x,  , 64)
> +
> +#undef PERCPU_XCHG_OP
> +
>  /*
>   * It would be nice to avoid the conditional call into the scheduler when
>   * re-enabling preemption for preemptible kernels, but doing that in a way
> @@ -306,6 +350,11 @@ PERCPU_RET_OP(add, add, ldadd)
>  	(typeof(pcp))op(&(pcp), ##args);				\
>  })
>  
> +#define _pcp_wrap_xchg(op, pcp, val)					\
> +({									\
> +	(typeof(pcp))op(&(pcp), (unsigned long)(val));			\
> +})
> +
>  #define this_cpu_read_1(pcp)		\
>  	_pcp_wrap_return(__percpu_read_8, pcp)
>  #define this_cpu_read_2(pcp)		\
> @@ -361,13 +410,13 @@ PERCPU_RET_OP(add, add, ldadd)
>  	_pcp_wrap(__percpu_or_case_64, pcp, val)
>  
>  #define this_cpu_xchg_1(pcp, val)	\
> -	_pcp_protect_return(xchg_relaxed, pcp, val)
> +	_pcp_wrap_xchg(__percpu_xchg_case_8, pcp, val)
>  #define this_cpu_xchg_2(pcp, val)	\
> -	_pcp_protect_return(xchg_relaxed, pcp, val)
> +	_pcp_wrap_xchg(__percpu_xchg_case_16, pcp, val)
>  #define this_cpu_xchg_4(pcp, val)	\
> -	_pcp_protect_return(xchg_relaxed, pcp, val)
> +	_pcp_wrap_xchg(__percpu_xchg_case_32, pcp, val)
>  #define this_cpu_xchg_8(pcp, val)	\
> -	_pcp_protect_return(xchg_relaxed, pcp, val)
> +	_pcp_wrap_xchg(__percpu_xchg_case_64, pcp, val)
>  
>  #define this_cpu_cmpxchg_1(pcp, o, n)	\
>  	_pcp_protect_return(cmpxchg_relaxed, pcp, o, n)
> -- 2.30.2
> 

FWIW,

Reviewed-by: Vladimir Murzin <vladimir.murzin at arm.com>