[PATCH v5 1/5] riscv: Strengthen fallback barriers with Zalasr

[Qingwei Hu]

RISC-V acquire and release primitives currently use directional fences
around plain loads, stores, and relaxed atomic operations. Later changes
can use Zalasr acquire and release annotations instead. Mixing the two
mappings needs a full fence on the fence-based side to retain the RCsc
guarantees expected by the Linux memory model.

Before introducing annotated operations, use alternatives to upgrade the
remaining directional fences in load/store, xchg/cmpxchg, and generic
atomic fallbacks on Zalasr systems. Keep the existing fences when Zalasr
is unavailable.

Signed-off-by: Qingwei Hu <qingwei.hu at bytedance.com>
---
 arch/riscv/include/asm/atomic.h  |  8 ++++++--
 arch/riscv/include/asm/barrier.h |  8 ++++++--
 arch/riscv/include/asm/cmpxchg.h | 25 +++++++++++++++++++------
 arch/riscv/include/asm/fence.h   |  9 +++++++++
 4 files changed, 40 insertions(+), 10 deletions(-)

diff --git a/arch/riscv/include/asm/atomic.h b/arch/riscv/include/asm/atomic.h
index 616b8b332ac56..03d477a65dc6e 100644
--- a/arch/riscv/include/asm/atomic.h
+++ b/arch/riscv/include/asm/atomic.h
@@ -19,10 +19,14 @@
 #include <asm/cmpxchg.h>
 
 #define __atomic_acquire_fence()					\
-	__asm__ __volatile__(RISCV_ACQUIRE_BARRIER "" ::: "memory")
+	__asm__ __volatile__(RISCV_ZALASR_SMP_ALTERNATIVE(		\
+		RISCV_ACQUIRE_BARRIER, RISCV_FULL_BARRIER)		\
+		::: "memory")
 
 #define __atomic_release_fence()					\
-	__asm__ __volatile__(RISCV_RELEASE_BARRIER "" ::: "memory");
+	__asm__ __volatile__(RISCV_ZALASR_SMP_ALTERNATIVE(		\
+		RISCV_RELEASE_BARRIER, RISCV_FULL_BARRIER)		\
+		::: "memory");
 
 static __always_inline int arch_atomic_read(const atomic_t *v)
 {
diff --git a/arch/riscv/include/asm/barrier.h b/arch/riscv/include/asm/barrier.h
index 700ba3f922cb5..51a73712b6a12 100644
--- a/arch/riscv/include/asm/barrier.h
+++ b/arch/riscv/include/asm/barrier.h
@@ -54,7 +54,9 @@
 #define __smp_store_release(p, v)					\
 do {									\
 	compiletime_assert_atomic_type(*p);				\
-	RISCV_FENCE(rw, w);						\
+	__asm__ __volatile__(RISCV_ZALASR_ALTERNATIVE(			\
+		RISCV_FENCE_ASM(rw, w), RISCV_FENCE_ASM(rw, rw))	\
+		::: "memory");						\
 	WRITE_ONCE(*p, v);						\
 } while (0)
 
@@ -62,7 +64,9 @@ do {									\
 ({									\
 	typeof(*p) ___p1 = READ_ONCE(*p);				\
 	compiletime_assert_atomic_type(*p);				\
-	RISCV_FENCE(r, rw);						\
+	__asm__ __volatile__(RISCV_ZALASR_ALTERNATIVE(			\
+		RISCV_FENCE_ASM(r, rw), RISCV_FENCE_ASM(rw, rw))	\
+		::: "memory");						\
 	___p1;								\
 })
 
diff --git a/arch/riscv/include/asm/cmpxchg.h b/arch/riscv/include/asm/cmpxchg.h
index 662e160b05229..39f8195c9fd6c 100644
--- a/arch/riscv/include/asm/cmpxchg.h
+++ b/arch/riscv/include/asm/cmpxchg.h
@@ -105,10 +105,15 @@
 
 #define arch_xchg_acquire(ptr, x)					\
 	_arch_xchg(ptr, x, "", "", "",					\
-		   RISCV_ACQUIRE_BARRIER, RISCV_ACQUIRE_BARRIER)
+		   RISCV_ZALASR_SMP_ALTERNATIVE(			\
+			RISCV_ACQUIRE_BARRIER, RISCV_FULL_BARRIER),	\
+		   RISCV_ZALASR_SMP_ALTERNATIVE(			\
+			RISCV_ACQUIRE_BARRIER, RISCV_FULL_BARRIER))
 
 #define arch_xchg_release(ptr, x)					\
-	_arch_xchg(ptr, x, "", "", RISCV_RELEASE_BARRIER, "", "")
+	_arch_xchg(ptr, x, "", "",					\
+		   RISCV_ZALASR_SMP_ALTERNATIVE(			\
+			RISCV_RELEASE_BARRIER, RISCV_FULL_BARRIER), "", "")
 
 #define arch_xchg(ptr, x)						\
 	_arch_xchg(ptr, x, ".rl", ".aqrl", "", RISCV_FULL_BARRIER, "")
@@ -279,14 +284,22 @@
 #define arch_cmpxchg_acquire(ptr, o, n)					\
 	_arch_cmpxchg((ptr), (o), (n),					\
 		      SC_SFX(""), CAS_SFX(""),				\
-		      SC_PREPEND(""), SC_APPEND(RISCV_ACQUIRE_BARRIER),	\
-		      CAS_PREPEND(""), CAS_APPEND(RISCV_ACQUIRE_BARRIER))
+		      SC_PREPEND(""), SC_APPEND(			\
+			RISCV_ZALASR_SMP_ALTERNATIVE(			\
+				RISCV_ACQUIRE_BARRIER, RISCV_FULL_BARRIER)), \
+		      CAS_PREPEND(""), CAS_APPEND(			\
+			RISCV_ZALASR_SMP_ALTERNATIVE(			\
+				RISCV_ACQUIRE_BARRIER, RISCV_FULL_BARRIER)))
 
 #define arch_cmpxchg_release(ptr, o, n)					\
 	_arch_cmpxchg((ptr), (o), (n),					\
 		      SC_SFX(""), CAS_SFX(""),				\
-		      SC_PREPEND(RISCV_RELEASE_BARRIER), SC_APPEND(""),	\
-		      CAS_PREPEND(RISCV_RELEASE_BARRIER), CAS_APPEND(""))
+		      SC_PREPEND(RISCV_ZALASR_SMP_ALTERNATIVE(		\
+			RISCV_RELEASE_BARRIER, RISCV_FULL_BARRIER)),	\
+		      SC_APPEND(""),					\
+		      CAS_PREPEND(RISCV_ZALASR_SMP_ALTERNATIVE(	\
+			RISCV_RELEASE_BARRIER, RISCV_FULL_BARRIER)),	\
+		      CAS_APPEND(""))
 
 #define arch_cmpxchg(ptr, o, n)						\
 	_arch_cmpxchg((ptr), (o), (n),					\
diff --git a/arch/riscv/include/asm/fence.h b/arch/riscv/include/asm/fence.h
index 182db7930edc2..607f56c9b8516 100644
--- a/arch/riscv/include/asm/fence.h
+++ b/arch/riscv/include/asm/fence.h
@@ -2,6 +2,15 @@
 #ifndef _ASM_RISCV_FENCE_H
 #define _ASM_RISCV_FENCE_H
 
+#include <asm/alternative-macros.h>
+#include <asm/hwcap.h>
+
+#define RISCV_ZALASR_ALTERNATIVE(old, new) \
+	ALTERNATIVE(old, new, 0, RISCV_ISA_EXT_ZALASR, 1)
+
+#define RISCV_ZALASR_SMP_ALTERNATIVE(old, new) \
+	ALTERNATIVE(old, new, 0, RISCV_ISA_EXT_ZALASR, CONFIG_SMP)
+
 #define RISCV_FENCE_ASM(p, s)		"\tfence " #p "," #s "\n"
 #define RISCV_FENCE(p, s) \
 	({ __asm__ __volatile__ (RISCV_FENCE_ASM(p, s) : : : "memory"); })
-- 
2.39.5