[PATCH v5 7/7] riscv: vector: optimize vstate operations

[Andy Chiu]

Introduce the CONFIG_RISCV_VSTATE_OPT to optimize vector state operations
in kernel space. By default, the kernel disables the Vector unit upon
entering kernel space to guard illegal use of vector, and nulls out
vector registers at syscall stops to catch autovectorizer errors such as
using stale vector states across syscalls.

This patch enables Vector in kernel mode and bypasses vector register
nulling for user space on the syscall fast path when the new
configuration is selected.

Enabling this optimization yields the following performance benefits:
 - Cut 80 cycles latency for getpid on Ascalon.
 - Delivers a 3.5% request throughput improvement for an nginx workload
   serving 1KB web pages on a four-core Ascalon-S.

Signed-off-by: Andy Chiu <tchiu at tenstorrent.com>
---
Changelog v5:
 - new patch since v5
---
 arch/riscv/Kconfig              | 11 +++++++++++
 arch/riscv/include/asm/vector.h |  9 ++++++++-
 arch/riscv/kernel/entry.S       | 13 +++++++++++--
 arch/riscv/kernel/process.c     |  4 ++++
 arch/riscv/kernel/vector.c      |  3 +++
 5 files changed, 37 insertions(+), 3 deletions(-)

diff --git a/arch/riscv/Kconfig b/arch/riscv/Kconfig
index 3f0a647218e4..89c0de5d1243 100644
--- a/arch/riscv/Kconfig
+++ b/arch/riscv/Kconfig
@@ -647,6 +647,17 @@ config RISCV_ISA_V_UCOPY_THRESHOLD
 	  Prefer using vectorized copy_to_user()/copy_from_user() when the
 	  workload size exceeds this value.
 
+config RISCV_VSTATE_OPT
+	bool "Optimize vector state operation in the kernel space"
+	default n
+	help
+	  Say N here if you want to catch illegal use of Vector in the kernel
+	  space. This config turns on V when entering the kernel space and the
+	  kernel would not perform register nulling for the user space. This
+	  rougly cut ~30 ns latency for getpid on blackhole x280 and gives 3.5%
+	  request throughput improvement for nginx workload serving 1KB web page
+	  on a four-core Ascalon-S.
+
 config RISCV_ISA_V_PREEMPTIVE
 	bool "Run kernel-mode Vector with kernel preemption"
 	depends on PREEMPTION
diff --git a/arch/riscv/include/asm/vector.h b/arch/riscv/include/asm/vector.h
index 2e6b88e35664..bed0958c0cb2 100644
--- a/arch/riscv/include/asm/vector.h
+++ b/arch/riscv/include/asm/vector.h
@@ -122,6 +122,9 @@ static inline bool riscv_v_vstate_query(struct pt_regs *regs)
 
 static __always_inline void riscv_v_enable(void)
 {
+	if (IS_ENABLED(CONFIG_RISCV_VSTATE_OPT))
+		return;
+
 	if (has_xtheadvector())
 		csr_set(CSR_SSTATUS, SR_VS_THEAD);
 	else
@@ -130,6 +133,9 @@ static __always_inline void riscv_v_enable(void)
 
 static __always_inline void riscv_v_disable(void)
 {
+	if (IS_ENABLED(CONFIG_RISCV_VSTATE_OPT))
+		return;
+
 	if (has_xtheadvector())
 		csr_clear(CSR_SSTATUS, SR_VS_THEAD);
 	else
@@ -335,7 +341,8 @@ static inline void riscv_v_vstate_set_restore(struct task_struct *task,
 static inline void riscv_v_vstate_discard(struct pt_regs *regs)
 {
 	if (__riscv_v_vstate_check_gt(regs->status, INITIAL)) {
-		riscv_v_vstate_set_restore(current, regs);
+		if (!IS_ENABLED(CONFIG_RISCV_VSTATE_OPT))
+			riscv_v_vstate_set_restore(current, regs);
 		riscv_v_vstate_init(regs);
 	}
 }
diff --git a/arch/riscv/kernel/entry.S b/arch/riscv/kernel/entry.S
index c6988983cdf7..6c9fd81b9583 100644
--- a/arch/riscv/kernel/entry.S
+++ b/arch/riscv/kernel/entry.S
@@ -6,6 +6,7 @@
 
 #include <linux/init.h>
 #include <linux/linkage.h>
+#include <linux/stringify.h>
 
 #include <asm/alternative-macros.h>
 #include <asm/asm.h>
@@ -175,10 +176,12 @@ SYM_CODE_START(handle_exception)
 	 * Disable user-mode memory access as it should only be set in the
 	 * actual user copy routines.
 	 *
-	 * Disable the FPU/Vector to detect illegal usage of floating point
+	 * Disable the FPU to detect illegal usage of floating point
 	 * or vector in kernel space.
 	 */
-	li t0, SR_SUM | SR_FS_VS
+	li t0, SR_SUM | SR_FS
+	ALTERNATIVE(__stringify(ori t0, t0, SR_VS), "nop",
+		    0, RISCV_ISA_EXT_ZVE32X, CONFIG_RISCV_VSTATE_OPT)
 #ifdef CONFIG_64BIT
 	li t1, SR_ELP
 	or t0, t0, t1
@@ -186,6 +189,12 @@ SYM_CODE_START(handle_exception)
 
 	REG_L s0, TASK_TI_USER_SP(tp)
 	csrrc s1, CSR_STATUS, t0
+	ALTERNATIVE("j .Lskip_v_enable", __stringify(andi t0, s1, SR_VS),
+		    0, RISCV_ISA_EXT_ZVE32X, CONFIG_RISCV_VSTATE_OPT)
+	bnez t0, .Lskip_v_enable
+	li a0, SR_VS_INITIAL
+	csrs CSR_STATUS, a0
+.Lskip_v_enable:
 	save_userssp s2, s1
 	csrr s2, CSR_EPC
 	csrr s3, CSR_TVAL
diff --git a/arch/riscv/kernel/process.c b/arch/riscv/kernel/process.c
index b2df7f72241a..b17ca4f70740 100644
--- a/arch/riscv/kernel/process.c
+++ b/arch/riscv/kernel/process.c
@@ -258,6 +258,10 @@ int copy_thread(struct task_struct *p, const struct kernel_clone_args *args)
 		/* Supervisor/Machine, irqs on: */
 		childregs->status = SR_PP | SR_PIE;
 
+		if (IS_ENABLED(CONFIG_RISCV_VSTATE_OPT) &&
+		    (has_vector() || has_xtheadvector()))
+			riscv_v_vstate_init(childregs);
+
 		p->thread.s[0] = (unsigned long)args->fn;
 		p->thread.s[1] = (unsigned long)args->fn_arg;
 		p->thread.ra = (unsigned long)ret_from_fork_kernel_asm;
diff --git a/arch/riscv/kernel/vector.c b/arch/riscv/kernel/vector.c
index 6fd541f5d5cb..050c09cc63dd 100644
--- a/arch/riscv/kernel/vector.c
+++ b/arch/riscv/kernel/vector.c
@@ -70,6 +70,9 @@ int riscv_v_setup_vsize(void)
 {
 	unsigned long this_vsize;
 
+	if (IS_ENABLED(CONFIG_RISCV_VSTATE_OPT))
+		csr_set(CSR_SSTATUS, SR_VS_INITIAL);
+
 	/*
 	 * There are 32 vector registers with vlenb length.
 	 *
-- 
2.43.0