[PATCH v5 2/5] riscv: Apply Zalasr to smp_load_acquire/smp_store_release

[Qingwei Hu]

From: Xu Lu <luxu.kernel at bytedance.com>

Replace fence instructions with Zalasr instructions during
smp_load_acquire() and smp_store_release() operations.

|----------------------------------|
|    |     __smp_store_release     |
|    |-----------------------------|
|    | zalasr        | !zalasr     |
| rl |-----------------------------|
|    | s{b|h|w|d}.rl | fence rw, w |
|    |               | s{b|h|w|d}  |
|----------------------------------|
|    |    __smp_load_acquire       |
|    |-----------------------------|
|    | zalasr        | !zalasr     |
| aq |-----------------------------|
|    | l{b|h|w|d}.aq | l{b|h|w|d}  |
|    |               | fence r, rw |
|----------------------------------|

Signed-off-by: Xu Lu <luxu.kernel at bytedance.com>
Co-developed-by: Qingwei Hu <qingwei.hu at bytedance.com>
Signed-off-by: Qingwei Hu <qingwei.hu at bytedance.com>
---
 arch/riscv/include/asm/barrier.h | 78 ++++++++++++++++++++++++++++----
 1 file changed, 69 insertions(+), 9 deletions(-)

diff --git a/arch/riscv/include/asm/barrier.h b/arch/riscv/include/asm/barrier.h
index 51a73712b6a12..d2200e826ec00 100644
--- a/arch/riscv/include/asm/barrier.h
+++ b/arch/riscv/include/asm/barrier.h
@@ -13,6 +13,7 @@
 #ifndef __ASSEMBLER__
 #include <asm/cmpxchg.h>
 #include <asm/fence.h>
+#include <asm/insn-def.h>
 
 /* These barriers need to enforce ordering on both devices or memory. */
 #define __mb()		RISCV_FENCE(iorw, iorw)
@@ -53,21 +54,80 @@
 
 #define __smp_store_release(p, v)					\
 do {									\
+	typeof(p) __p = (p);						\
+	union { __unqual_scalar_typeof(*p) __val; char __c[1]; } __u =	\
+		{ .__val = (__force __unqual_scalar_typeof(*p)) (v) };	\
 	compiletime_assert_atomic_type(*p);				\
-	__asm__ __volatile__(RISCV_ZALASR_ALTERNATIVE(			\
-		RISCV_FENCE_ASM(rw, w), RISCV_FENCE_ASM(rw, rw))	\
-		::: "memory");						\
-	WRITE_ONCE(*p, v);						\
+	kasan_check_write(__p, sizeof(*p));				\
+	switch (sizeof(*p)) {						\
+	case 1:								\
+		asm volatile(RISCV_ZALASR_ALTERNATIVE(			\
+			RISCV_FENCE_ASM(rw, w) "sb %0, 0(%1)\n",	\
+			SB_RL(%0, %1) "\n" __nops(1))			\
+			: : "r" (*(__u8 *)__u.__c), "r" (__p)		\
+			: "memory");					\
+		break;							\
+	case 2:								\
+		asm volatile(RISCV_ZALASR_ALTERNATIVE(			\
+			RISCV_FENCE_ASM(rw, w) "sh %0, 0(%1)\n",	\
+			SH_RL(%0, %1) "\n" __nops(1))			\
+			: : "r" (*(__u16 *)__u.__c), "r" (__p)	\
+			: "memory");					\
+		break;							\
+	case 4:								\
+		asm volatile(RISCV_ZALASR_ALTERNATIVE(			\
+			RISCV_FENCE_ASM(rw, w) "sw %0, 0(%1)\n",	\
+			SW_RL(%0, %1) "\n" __nops(1))			\
+			: : "r" (*(__u32 *)__u.__c), "r" (__p)	\
+			: "memory");					\
+		break;							\
+	case 8:								\
+		asm volatile(RISCV_ZALASR_ALTERNATIVE(			\
+			RISCV_FENCE_ASM(rw, w) "sd %0, 0(%1)\n",	\
+			SD_RL(%0, %1) "\n" __nops(1))			\
+			: : "r" (*(__u64 *)__u.__c), "r" (__p)	\
+			: "memory");					\
+		break;							\
+	}								\
 } while (0)
 
 #define __smp_load_acquire(p)						\
 ({									\
-	typeof(*p) ___p1 = READ_ONCE(*p);				\
+	union { __unqual_scalar_typeof(*p) __val; char __c[1]; } __u;	\
+	typeof(p) __p = (p);						\
 	compiletime_assert_atomic_type(*p);				\
-	__asm__ __volatile__(RISCV_ZALASR_ALTERNATIVE(			\
-		RISCV_FENCE_ASM(r, rw), RISCV_FENCE_ASM(rw, rw))	\
-		::: "memory");						\
-	___p1;								\
+	kasan_check_read(__p, sizeof(*p));				\
+	switch (sizeof(*p)) {						\
+	case 1:								\
+		asm volatile(RISCV_ZALASR_ALTERNATIVE(			\
+			"lb %0, 0(%1)\n" RISCV_FENCE_ASM(r, rw),	\
+			LB_AQ(%0, %1) "\n" __nops(1))			\
+			: "=r" (*(__u8 *)__u.__c) : "r" (__p)		\
+			: "memory");					\
+		break;							\
+	case 2:								\
+		asm volatile(RISCV_ZALASR_ALTERNATIVE(			\
+			"lh %0, 0(%1)\n" RISCV_FENCE_ASM(r, rw),	\
+			LH_AQ(%0, %1) "\n" __nops(1))			\
+			: "=r" (*(__u16 *)__u.__c) : "r" (__p)	\
+			: "memory");					\
+		break;							\
+	case 4:								\
+		asm volatile(RISCV_ZALASR_ALTERNATIVE(			\
+			"lw %0, 0(%1)\n" RISCV_FENCE_ASM(r, rw),	\
+			LW_AQ(%0, %1) "\n" __nops(1))			\
+			: "=r" (*(__u32 *)__u.__c) : "r" (__p)	\
+			: "memory");					\
+		break;							\
+	case 8:								\
+		asm volatile(RISCV_ZALASR_ALTERNATIVE(			\
+			"ld %0, 0(%1)\n" RISCV_FENCE_ASM(r, rw),	\
+			LD_AQ(%0, %1) "\n" __nops(1))			\
+			: "=r" (*(__u64 *)__u.__c) : "r" (__p)	\
+			: "memory");					\
+		break;							\
+	}								\
+	(typeof(*p))__u.__val;						\
 })
 
 #ifdef CONFIG_RISCV_ISA_ZAWRS
-- 
2.39.5