[PATCH RFC 1/2] arm64: atomics: reimplement __xchg* with __lse_ll_sc_body()
Yicong Yang
yangyicong at huawei.com
Fri Sep 19 02:17:46 PDT 2025
From: Yicong Yang <yangyicong at hisilicon.com>
Almost all the atomic operations are implemented with __lse_ll_sc_body()
and split the LL/SC and LSE implementations to separate headers. One
exception is __xchg*, though its variant __cmpxchg128* is already using
__lse_ll_sc_body().
Reimplement __xchg* with __lse_ll_sc_body() and move the LL/SC and LSE
to their own headers, use symbolic names for operands as well. This will
make the code more organized, readable and easy for extension.
No functional changes intended. Tested with kernel's boot time
atomic64_selftest.
Signed-off-by: Yicong Yang <yangyicong at hisilicon.com>
---
arch/arm64/include/asm/atomic_ll_sc.h | 40 +++++++++++++++++
arch/arm64/include/asm/atomic_lse.h | 36 +++++++++++++++
arch/arm64/include/asm/cmpxchg.h | 63 +++++++++------------------
3 files changed, 96 insertions(+), 43 deletions(-)
diff --git a/arch/arm64/include/asm/atomic_ll_sc.h b/arch/arm64/include/asm/atomic_ll_sc.h
index 89d2ba272359..95d91a5fd216 100644
--- a/arch/arm64/include/asm/atomic_ll_sc.h
+++ b/arch/arm64/include/asm/atomic_ll_sc.h
@@ -236,6 +236,46 @@ __ll_sc_atomic64_dec_if_positive(atomic64_t *v)
return result;
}
+#define __XCHG_CASE(w, sfx, name, sz, mb, acq, rel, cl) \
+static __always_inline u##sz \
+__ll_sc__xchg_case_##name##sz(u##sz new, volatile void *ptr) \
+{ \
+ u##sz old; \
+ unsigned long tmp; \
+ \
+ asm volatile( \
+ " prfm pstl1strm, %[v]\n" \
+ "1: ld" #acq "xr" #sfx "\t%" #w "[old], %[v]\n" \
+ " st" #rel "xr" #sfx "\t%w[tmp], %" #w "[new], %[v]\n" \
+ " cbnz %w[tmp], 1b\n" \
+ " " #mb \
+ : [old] "=&r" (old), [tmp] "=&r" (tmp), \
+ [v] "+Q" (*(u##sz *)ptr) \
+ : [new] "r" (new) \
+ : cl); \
+ \
+ return old; \
+}
+
+__XCHG_CASE(w, b, , 8, , , , )
+__XCHG_CASE(w, h, , 16, , , , )
+__XCHG_CASE(w, , , 32, , , , )
+__XCHG_CASE( , , , 64, , , , )
+__XCHG_CASE(w, b, acq_, 8, , a, , "memory")
+__XCHG_CASE(w, h, acq_, 16, , a, , "memory")
+__XCHG_CASE(w, , acq_, 32, , a, , "memory")
+__XCHG_CASE( , , acq_, 64, , a, , "memory")
+__XCHG_CASE(w, b, rel_, 8, , , l, "memory")
+__XCHG_CASE(w, h, rel_, 16, , , l, "memory")
+__XCHG_CASE(w, , rel_, 32, , , l, "memory")
+__XCHG_CASE( , , rel_, 64, , , l, "memory")
+__XCHG_CASE(w, b, mb_, 8, dmb ish, , l, "memory")
+__XCHG_CASE(w, h, mb_, 16, dmb ish, , l, "memory")
+__XCHG_CASE(w, , mb_, 32, dmb ish, , l, "memory")
+__XCHG_CASE( , , mb_, 64, dmb ish, , l, "memory")
+
+#undef __XCHG_CASE
+
#define __CMPXCHG_CASE(w, sfx, name, sz, mb, acq, rel, cl, constraint) \
static __always_inline u##sz \
__ll_sc__cmpxchg_case_##name##sz(volatile void *ptr, \
diff --git a/arch/arm64/include/asm/atomic_lse.h b/arch/arm64/include/asm/atomic_lse.h
index 87f568a94e55..cb38c2120595 100644
--- a/arch/arm64/include/asm/atomic_lse.h
+++ b/arch/arm64/include/asm/atomic_lse.h
@@ -245,6 +245,42 @@ static __always_inline s64 __lse_atomic64_dec_if_positive(atomic64_t *v)
return (long)v;
}
+#define __XCHG_CASE(w, sfx, name, sz, mb, cl...) \
+static __always_inline u##sz \
+__lse__xchg_case_##name##sz(u##sz new, volatile void *ptr) \
+{ \
+ u##sz old; \
+ \
+ asm volatile( \
+ __LSE_PREAMBLE \
+ " swp" #mb #sfx "\t%" #w "[new], %" #w "[old], %[v]\n" \
+ : [old] "=r" (old), \
+ [v] "+Q" (*(u##sz *)ptr) \
+ : [new] "r" (new) \
+ : cl); \
+ \
+ return old; \
+}
+
+__XCHG_CASE(w, b, , 8, )
+__XCHG_CASE(w, h, , 16, )
+__XCHG_CASE(w, , , 32, )
+__XCHG_CASE(x, , , 64, )
+__XCHG_CASE(w, b, acq_, 8, a, "memory")
+__XCHG_CASE(w, h, acq_, 16, a, "memory")
+__XCHG_CASE(w, , acq_, 32, a, "memory")
+__XCHG_CASE(x, , acq_, 64, a, "memory")
+__XCHG_CASE(w, b, rel_, 8, l, "memory")
+__XCHG_CASE(w, h, rel_, 16, l, "memory")
+__XCHG_CASE(w, , rel_, 32, l, "memory")
+__XCHG_CASE(x, , rel_, 64, l, "memory")
+__XCHG_CASE(w, b, mb_, 8, al, "memory")
+__XCHG_CASE(w, h, mb_, 16, al, "memory")
+__XCHG_CASE(w, , mb_, 32, al, "memory")
+__XCHG_CASE(x, , mb_, 64, al, "memory")
+
+#undef __XCHG_CASE
+
#define __CMPXCHG_CASE(w, sfx, name, sz, mb, cl...) \
static __always_inline u##sz \
__lse__cmpxchg_case_##name##sz(volatile void *ptr, \
diff --git a/arch/arm64/include/asm/cmpxchg.h b/arch/arm64/include/asm/cmpxchg.h
index d7a540736741..14cb05cf13cb 100644
--- a/arch/arm64/include/asm/cmpxchg.h
+++ b/arch/arm64/include/asm/cmpxchg.h
@@ -13,51 +13,28 @@
#include <asm/barrier.h>
#include <asm/lse.h>
-/*
- * We need separate acquire parameters for ll/sc and lse, since the full
- * barrier case is generated as release+dmb for the former and
- * acquire+release for the latter.
- */
-#define __XCHG_CASE(w, sfx, name, sz, mb, nop_lse, acq, acq_lse, rel, cl) \
-static inline u##sz __xchg_case_##name##sz(u##sz x, volatile void *ptr) \
-{ \
- u##sz ret; \
- unsigned long tmp; \
- \
- asm volatile(ARM64_LSE_ATOMIC_INSN( \
- /* LL/SC */ \
- " prfm pstl1strm, %2\n" \
- "1: ld" #acq "xr" #sfx "\t%" #w "0, %2\n" \
- " st" #rel "xr" #sfx "\t%w1, %" #w "3, %2\n" \
- " cbnz %w1, 1b\n" \
- " " #mb, \
- /* LSE atomics */ \
- " swp" #acq_lse #rel #sfx "\t%" #w "3, %" #w "0, %2\n" \
- __nops(3) \
- " " #nop_lse) \
- : "=&r" (ret), "=&r" (tmp), "+Q" (*(u##sz *)ptr) \
- : "r" (x) \
- : cl); \
- \
- return ret; \
+#define __XCHG_CASE(name, sz) \
+static inline u##sz __xchg_case_##name##sz(u##sz x, volatile void *ptr) \
+{ \
+ return __lse_ll_sc_body(_xchg_case_##name##sz, x, ptr); \
}
-__XCHG_CASE(w, b, , 8, , , , , , )
-__XCHG_CASE(w, h, , 16, , , , , , )
-__XCHG_CASE(w, , , 32, , , , , , )
-__XCHG_CASE( , , , 64, , , , , , )
-__XCHG_CASE(w, b, acq_, 8, , , a, a, , "memory")
-__XCHG_CASE(w, h, acq_, 16, , , a, a, , "memory")
-__XCHG_CASE(w, , acq_, 32, , , a, a, , "memory")
-__XCHG_CASE( , , acq_, 64, , , a, a, , "memory")
-__XCHG_CASE(w, b, rel_, 8, , , , , l, "memory")
-__XCHG_CASE(w, h, rel_, 16, , , , , l, "memory")
-__XCHG_CASE(w, , rel_, 32, , , , , l, "memory")
-__XCHG_CASE( , , rel_, 64, , , , , l, "memory")
-__XCHG_CASE(w, b, mb_, 8, dmb ish, nop, , a, l, "memory")
-__XCHG_CASE(w, h, mb_, 16, dmb ish, nop, , a, l, "memory")
-__XCHG_CASE(w, , mb_, 32, dmb ish, nop, , a, l, "memory")
-__XCHG_CASE( , , mb_, 64, dmb ish, nop, , a, l, "memory")
+__XCHG_CASE( , 8)
+__XCHG_CASE( , 16)
+__XCHG_CASE( , 32)
+__XCHG_CASE( , 64)
+__XCHG_CASE(acq_, 8)
+__XCHG_CASE(acq_, 16)
+__XCHG_CASE(acq_, 32)
+__XCHG_CASE(acq_, 64)
+__XCHG_CASE(rel_, 8)
+__XCHG_CASE(rel_, 16)
+__XCHG_CASE(rel_, 32)
+__XCHG_CASE(rel_, 64)
+__XCHG_CASE(mb_, 8)
+__XCHG_CASE(mb_, 16)
+__XCHG_CASE(mb_, 32)
+__XCHG_CASE(mb_, 64)
#undef __XCHG_CASE
--
2.24.0
More information about the linux-arm-kernel
mailing list