[PATCH] arm64: percpu: Implement this_cpu operations
Ard Biesheuvel
ard.biesheuvel at linaro.org
Thu Nov 6 03:26:46 PST 2014
Hi Steve,
On 6 November 2014 12:12, Steve Capper <steve.capper at linaro.org> wrote:
> The generic this_cpu operations disable interrupts to ensure that the
> requested operation is protected from pre-emption. For arm64, this is
> overkill and can hurt throughput and latency.
>
> This patch provides arm64 specific implementations for the this_cpu
> operations. Rather than disable interrupts, we use the exclusive
> monitor or atomic operations as appropriate.
>
> The following operations are implemented: add, add_return, and, or,
> read, write, xchg. We also wire up a cmpxchg implementation from
> cmpxchg.h.
>
> Testing was performed using the percpu_test module and hackbench on a
> Juno board running 3.18-rc3.
>
Got any numbers?
> Signed-off-by: Steve Capper <steve.capper at linaro.org>
> ---
>
> This patch applies on top of one I sent out earlier:
> "arm64: xchg: Implement cmpxchg_double"
> http://lists.infradead.org/pipermail/linux-arm-kernel/2014-October/294705.html
>
> My two least favourite things in the kernel are excessive pre-processor
> use and assembler, so this patch has been fun to write ;-).
>
> I've tried to make the inline assembler clearer with named parameters,
> it really upset checkpatch, if it upsets people then please shout and I
> will change it.
>
> Cheers,
> --
> Steve
>
> ---
> arch/arm64/include/asm/cmpxchg.h | 6 +-
> arch/arm64/include/asm/percpu.h | 231 +++++++++++++++++++++++++++++++++++++++
> 2 files changed, 235 insertions(+), 2 deletions(-)
>
> diff --git a/arch/arm64/include/asm/cmpxchg.h b/arch/arm64/include/asm/cmpxchg.h
> index 3e02245..3e51f49 100644
> --- a/arch/arm64/include/asm/cmpxchg.h
> +++ b/arch/arm64/include/asm/cmpxchg.h
> @@ -237,8 +237,10 @@ static inline unsigned long __cmpxchg_mb(volatile void *ptr, unsigned long old,
> __ret; \
> })
>
> -#define this_cpu_cmpxchg_8(ptr, o, n) \
> - cmpxchg(raw_cpu_ptr(&(ptr)), o, n);
> +#define this_cpu_cmpxchg_1(ptr, o, n) cmpxchg(raw_cpu_ptr(&(ptr)), o, n)
> +#define this_cpu_cmpxchg_2(ptr, o, n) cmpxchg(raw_cpu_ptr(&(ptr)), o, n)
> +#define this_cpu_cmpxchg_4(ptr, o, n) cmpxchg(raw_cpu_ptr(&(ptr)), o, n)
> +#define this_cpu_cmpxchg_8(ptr, o, n) cmpxchg(raw_cpu_ptr(&(ptr)), o, n)
>
> #define this_cpu_cmpxchg_double_8(ptr1, ptr2, o1, o2, n1, n2) \
> cmpxchg_double(raw_cpu_ptr(&(ptr1)), raw_cpu_ptr(&(ptr2)), \
> diff --git a/arch/arm64/include/asm/percpu.h b/arch/arm64/include/asm/percpu.h
> index 5279e57..e751681 100644
> --- a/arch/arm64/include/asm/percpu.h
> +++ b/arch/arm64/include/asm/percpu.h
> @@ -44,6 +44,237 @@ static inline unsigned long __my_cpu_offset(void)
>
> #endif /* CONFIG_SMP */
>
> +#define PERCPU_OP(op, asm_op) \
> +static inline unsigned long __percpu_##op(void *ptr, \
> + unsigned long val, int size) \
> +{ \
> + unsigned long loop, ret; \
> + \
> + switch (size) { \
> + case 1: \
> + do { \
> + asm ("//__per_cpu_" #op "_1\n" \
> + "ldxrb %w[ret], %[ptr]\n" \
> + #asm_op " %w[ret], %w[ret], %w[val]\n" \
> + "stxrb %w[loop], %w[ret], %[ptr]\n" \
> + : [loop] "=&r" (loop), [ret] "=&r" (ret), \
> + [ptr] "+Q"(*(u8 *)ptr) \
> + : [val] "Ir" (val)); \
> + } while (loop); \
> + break; \
> + case 2: \
> + do { \
> + asm ("//__per_cpu_" #op "_2\n" \
> + "ldxrh %w[ret], %[ptr]\n" \
> + #asm_op " %w[ret], %w[ret], %w[val]\n" \
> + "stxrh %w[loop], %w[ret], %[ptr]\n" \
> + : [loop] "=&r" (loop), [ret] "=&r" (ret), \
> + [ptr] "+Q"(*(u16 *)ptr) \
> + : [val] "Ir" (val)); \
> + } while (loop); \
> + break; \
> + case 4: \
> + do { \
> + asm ("//__per_cpu_" #op "_4\n" \
> + "ldxr %w[ret], %[ptr]\n" \
> + #asm_op " %w[ret], %w[ret], %w[val]\n" \
> + "stxr %w[loop], %w[ret], %[ptr]\n" \
> + : [loop] "=&r" (loop), [ret] "=&r" (ret), \
> + [ptr] "+Q"(*(u32 *)ptr) \
> + : [val] "Ir" (val)); \
> + } while (loop); \
> + break; \
> + case 8: \
> + do { \
> + asm ("//__per_cpu_" #op "_8\n" \
> + "ldxr %[ret], %[ptr]\n" \
> + #asm_op " %[ret], %[ret], %[val]\n" \
> + "stxr %w[loop], %[ret], %[ptr]\n" \
> + : [loop] "=&r" (loop), [ret] "=&r" (ret), \
> + [ptr] "+Q"(*(u64 *)ptr) \
> + : [val] "Ir" (val)); \
> + } while (loop); \
> + break; \
> + default: \
> + BUILD_BUG(); \
> + } \
> + \
> + return ret; \
> +}
> +
> +PERCPU_OP(add, add)
> +PERCPU_OP(and, and)
> +PERCPU_OP(or, orr)
> +#undef PERCPU_OP
> +
> +static inline unsigned long __percpu_read(void *ptr, int size)
> +{
> + unsigned long ret;
> +
> + switch (size) {
> + case 1:
> + asm ("//__per_cpu_read_1\n"
> + "ldrb %w[ret], %[ptr]\n" :
> + [ret] "=&r"(ret), [ptr] "+Q"(*(u8 *)ptr));
> + break;
> + case 2:
> + asm ("//__per_cpu_read_2\n"
> + "ldrh %w[ret], %[ptr]\n" :
> + [ret] "=&r"(ret), [ptr] "+Q"(*(u16 *)ptr));
> + break;
> + case 4:
> + asm ("//__per_cpu_read_4\n"
> + "ldr %w[ret], %[ptr]\n" :
> + [ret] "=&r"(ret), [ptr] "+Q"(*(u32 *)ptr));
> + break;
> + case 8:
> + asm ("//__per_cpu_read_8\n"
> + "ldr %[ret], %[ptr]\n" :
> + [ret] "=&r"(ret), [ptr] "+Q"(*(u64 *)ptr));
> + break;
> + default:
> + BUILD_BUG();
> + }
> +
> + return ret;
> +}
> +
Why are the 'ptr' references '+Q' outputs here rather than 'Q' inputs?
> +static inline void __percpu_write(void *ptr, unsigned long val, int size)
> +{
> + switch (size) {
> + case 1:
> + asm ("//__per_cpu_write_1\n"
> + "strb %w[val], %[ptr]\n" :
> + [ptr] "+Q"(*(u8 *)ptr) : [val] "r"(val));
> + break;
> + case 2:
> + asm ("//__per_cpu_write_2\n"
> + "strh %w[val], %[ptr]\n" :
> + [ptr] "+Q"(*(u16 *)ptr) : [val] "r"(val));
> + break;
> + case 4:
> + asm ("//__per_cpu_write_4\n"
> + "str %w[val], %[ptr]\n" :
> + [ptr] "+Q"(*(u32 *)ptr) : [val] "r"(val));
> + break;
> + case 8:
> + asm ("//__per_cpu_write_8\n"
> + "str %[val], %[ptr]\n" :
> + [ptr] "+Q"(*(u64 *)ptr) : [val] "r"(val));
> + break;
> + default:
> + BUILD_BUG();
> + }
> +}
> +
... and similarly, why are these '+Q' and not just '=Q' ?
> +static inline unsigned long __percpu_xchg(void *ptr, unsigned long val,
> + int size)
> +{
> + unsigned long ret, loop;
> +
> + switch (size) {
> + case 1:
> + do {
> + asm ("//__percpu_xchg_1\n"
> + "ldxrb %w[ret], %[ptr]\n"
> + "stxrb %w[loop], %w[val], %[ptr]\n"
> + : [loop] "=&r"(loop), [ret] "=&r"(ret),
> + [ptr] "+Q"(*(u8 *)ptr)
> + : [val] "r" (val));
> + } while (loop);
> + break;
> + case 2:
> + do {
> + asm ("//__percpu_xchg_2\n"
> + "ldxrh %w[ret], %[ptr]\n"
> + "stxrh %w[loop], %w[val], %[ptr]\n"
> + : [loop] "=&r"(loop), [ret] "=&r"(ret),
> + [ptr] "+Q"(*(u16 *)ptr)
> + : [val] "r" (val));
> + } while (loop);
> + break;
> + case 4:
> + do {
> + asm ("//__percpu_xchg_4\n"
> + "ldxr %w[ret], %[ptr]\n"
> + "stxr %w[loop], %w[val], %[ptr]\n"
> + : [loop] "=&r"(loop), [ret] "=&r"(ret),
> + [ptr] "+Q"(*(u32 *)ptr)
> + : [val] "r" (val));
> + } while (loop);
> + break;
> + case 8:
> + do {
> + asm ("//__percpu_xchg_8\n"
> + "ldxr %[ret], %[ptr]\n"
> + "stxr %w[loop], %[val], %[ptr]\n"
> + : [loop] "=&r"(loop), [ret] "=&r"(ret),
> + [ptr] "+Q"(*(u64 *)ptr)
> + : [val] "r" (val));
> + } while (loop);
> + break;
> + default:
> + BUILD_BUG();
> + }
> +
> + return ret;
> +}
> +
> +#define _percpu_add(pcp, val) \
> + __percpu_add(raw_cpu_ptr(&(pcp)), val, sizeof(pcp))
> +
> +#define _percpu_add_return(pcp, val) (typeof(pcp)) (_percpu_add(pcp, val))
> +
> +#define _percpu_and(pcp, val) \
> + __percpu_and(raw_cpu_ptr(&(pcp)), val, sizeof(pcp))
> +
> +#define _percpu_or(pcp, val) \
> + __percpu_or(raw_cpu_ptr(&(pcp)), val, sizeof(pcp))
> +
> +#define _percpu_read(pcp) (typeof(pcp)) \
> + (__percpu_read(raw_cpu_ptr(&(pcp)), sizeof(pcp)))
> +
> +#define _percpu_write(pcp, val) \
> + __percpu_write(raw_cpu_ptr(&(pcp)), (unsigned long)(val), sizeof(pcp))
> +
> +#define _percpu_xchg(pcp, val) (typeof(pcp)) \
> + (__percpu_xchg(raw_cpu_ptr(&(pcp)), (unsigned long)(val), sizeof(pcp)))
> +
> +#define this_cpu_add_1(pcp, val) _percpu_add(pcp, val)
> +#define this_cpu_add_2(pcp, val) _percpu_add(pcp, val)
> +#define this_cpu_add_4(pcp, val) _percpu_add(pcp, val)
> +#define this_cpu_add_8(pcp, val) _percpu_add(pcp, val)
> +
> +#define this_cpu_add_return_1(pcp, val) _percpu_add_return(pcp, val)
> +#define this_cpu_add_return_2(pcp, val) _percpu_add_return(pcp, val)
> +#define this_cpu_add_return_4(pcp, val) _percpu_add_return(pcp, val)
> +#define this_cpu_add_return_8(pcp, val) _percpu_add_return(pcp, val)
> +
> +#define this_cpu_and_1(pcp, val) _percpu_and(pcp, val)
> +#define this_cpu_and_2(pcp, val) _percpu_and(pcp, val)
> +#define this_cpu_and_4(pcp, val) _percpu_and(pcp, val)
> +#define this_cpu_and_8(pcp, val) _percpu_and(pcp, val)
> +
> +#define this_cpu_or_1(pcp, val) _percpu_or(pcp, val)
> +#define this_cpu_or_2(pcp, val) _percpu_or(pcp, val)
> +#define this_cpu_or_4(pcp, val) _percpu_or(pcp, val)
> +#define this_cpu_or_8(pcp, val) _percpu_or(pcp, val)
> +
> +#define this_cpu_read_1(pcp) _percpu_read(pcp)
> +#define this_cpu_read_2(pcp) _percpu_read(pcp)
> +#define this_cpu_read_4(pcp) _percpu_read(pcp)
> +#define this_cpu_read_8(pcp) _percpu_read(pcp)
> +
> +#define this_cpu_write_1(pcp, val) _percpu_write(pcp, val)
> +#define this_cpu_write_2(pcp, val) _percpu_write(pcp, val)
> +#define this_cpu_write_4(pcp, val) _percpu_write(pcp, val)
> +#define this_cpu_write_8(pcp, val) _percpu_write(pcp, val)
> +
> +#define this_cpu_xchg_1(pcp, val) _percpu_xchg(pcp, val)
> +#define this_cpu_xchg_2(pcp, val) _percpu_xchg(pcp, val)
> +#define this_cpu_xchg_4(pcp, val) _percpu_xchg(pcp, val)
> +#define this_cpu_xchg_8(pcp, val) _percpu_xchg(pcp, val)
> +
> #include <asm-generic/percpu.h>
>
> #endif /* __ASM_PERCPU_H */
> --
> 1.9.3
>
>
> _______________________________________________
> linux-arm-kernel mailing list
> linux-arm-kernel at lists.infradead.org
> http://lists.infradead.org/mailman/listinfo/linux-arm-kernel
More information about the linux-arm-kernel
mailing list