[PATCH 2/3] arm64: fpsimd: preserve/restore kernel mode NEON at context switch
Ard Biesheuvel
ardb at google.com
Tue Nov 21 05:52:15 PST 2023
From: Ard Biesheuvel <ardb at kernel.org>
Currently, the SIMD register file is not preserved and restored along
with the general registers on exception entry/exit or context switch.
For this reason, we disable preemption when enabling the NEON for kernel
mode use in task context, and suspend the processing of softirqs so that
there are no concurrent uses in the kernel. (Kernel mode NEON may not be
used at all in other contexts).
Disabling preemption while doing CPU intensive work on inputs of
potentially unbounded size is bad for real-time performance, which is
why we try and ensure that SIMD crypto code does not operate on more
than ~4k at a time, which is an arbitrary value and requires assembler
code to implement efficiently.
We can avoid the need for disabling preemption if we can ensure that any
in-kernel users of the NEON will not lose the FPSIMD register state
across a context switch. And given that disabling softirqs implicitly
disables preemption as well, we will also have to ensure that a softirq
that uses the NEON can safely interrupt an in-kernel user.
So introduce a thread_info flag TIF_USING_KMODE_NEON, and modify the
context switch hook for FPSIMD to preserve resp. restore the kernel mode
FPSIMD to/from struct thread_struct when it is set. This avoids any
scheduling blackouts due to prolonged use of the NEON in kernel mode,
without the need for manual yielding.
In order to support softirq processing while the NEON is being used in
kernel task context, use the same flag to decide whether the kernel mode
FPSIMD state needs to be preserved and restored before allowing the NEON
to be used in softirq context.
Signed-off-by: Ard Biesheuvel <ardb at kernel.org>
---
arch/arm64/include/asm/processor.h | 2 +
arch/arm64/include/asm/thread_info.h | 1 +
arch/arm64/kernel/fpsimd.c | 75 ++++++++++++++++----
3 files changed, 63 insertions(+), 15 deletions(-)
diff --git a/arch/arm64/include/asm/processor.h b/arch/arm64/include/asm/processor.h
index e5bc54522e71..dcb51c0571af 100644
--- a/arch/arm64/include/asm/processor.h
+++ b/arch/arm64/include/asm/processor.h
@@ -167,6 +167,8 @@ struct thread_struct {
unsigned long fault_address; /* fault info */
unsigned long fault_code; /* ESR_EL1 value */
struct debug_info debug; /* debugging */
+
+ struct user_fpsimd_state kmode_fpsimd_state;
#ifdef CONFIG_ARM64_PTR_AUTH
struct ptrauth_keys_user keys_user;
#ifdef CONFIG_ARM64_PTR_AUTH_KERNEL
diff --git a/arch/arm64/include/asm/thread_info.h b/arch/arm64/include/asm/thread_info.h
index 553d1bc559c6..b65324d1ffc8 100644
--- a/arch/arm64/include/asm/thread_info.h
+++ b/arch/arm64/include/asm/thread_info.h
@@ -80,6 +80,7 @@ void arch_setup_new_exec(void);
#define TIF_TAGGED_ADDR 26 /* Allow tagged user addresses */
#define TIF_SME 27 /* SME in use */
#define TIF_SME_VL_INHERIT 28 /* Inherit SME vl_onexec across exec */
+#define TIF_USING_KMODE_NEON 29 /* Task is in a kernel mode NEON section */
#define _TIF_SIGPENDING (1 << TIF_SIGPENDING)
#define _TIF_NEED_RESCHED (1 << TIF_NEED_RESCHED)
diff --git a/arch/arm64/kernel/fpsimd.c b/arch/arm64/kernel/fpsimd.c
index ccc4a78a70e4..326c1613091e 100644
--- a/arch/arm64/kernel/fpsimd.c
+++ b/arch/arm64/kernel/fpsimd.c
@@ -357,6 +357,7 @@ static void task_fpsimd_load(void)
WARN_ON(!system_supports_fpsimd());
WARN_ON(preemptible());
+ WARN_ON(test_thread_flag(TIF_USING_KMODE_NEON));
if (system_supports_sve() || system_supports_sme()) {
switch (current->thread.fp_type) {
@@ -1482,20 +1483,29 @@ void fpsimd_thread_switch(struct task_struct *next)
WARN_ON_ONCE(!irqs_disabled());
- /* Save unsaved fpsimd state, if any: */
- fpsimd_save();
+ if (!test_thread_flag(TIF_USING_KMODE_NEON)) {
+ /* Save unsaved user mode fpsimd state, if any: */
+ fpsimd_save();
+ } else {
+ fpsimd_save_state(¤t->thread.kmode_fpsimd_state);
+ }
- /*
- * Fix up TIF_FOREIGN_FPSTATE to correctly describe next's
- * state. For kernel threads, FPSIMD registers are never loaded
- * and wrong_task and wrong_cpu will always be true.
- */
- wrong_task = __this_cpu_read(fpsimd_last_state.st) !=
- &next->thread.uw.fpsimd_state;
- wrong_cpu = next->thread.fpsimd_cpu != smp_processor_id();
+ if (test_tsk_thread_flag(next, TIF_USING_KMODE_NEON)) {
+ fpsimd_load_state(&next->thread.kmode_fpsimd_state);
+ set_tsk_thread_flag(next, TIF_FOREIGN_FPSTATE);
+ } else {
+ /*
+ * Fix up TIF_FOREIGN_FPSTATE to correctly describe next's
+ * state. For kernel threads, FPSIMD registers are never loaded
+ * and wrong_task and wrong_cpu will always be true.
+ */
+ wrong_task = __this_cpu_read(fpsimd_last_state.st) !=
+ &next->thread.uw.fpsimd_state;
+ wrong_cpu = next->thread.fpsimd_cpu != smp_processor_id();
- update_tsk_thread_flag(next, TIF_FOREIGN_FPSTATE,
- wrong_task || wrong_cpu);
+ update_tsk_thread_flag(next, TIF_FOREIGN_FPSTATE,
+ wrong_task || wrong_cpu);
+ }
}
static void fpsimd_flush_thread_vl(enum vec_type type)
@@ -1834,11 +1844,37 @@ void kernel_neon_begin(void)
get_cpu_fpsimd_context();
- /* Save unsaved fpsimd state, if any: */
- fpsimd_save();
+ if (!test_thread_flag(TIF_USING_KMODE_NEON)) {
+ /* Save unsaved user mode fpsimd state, if any: */
+ fpsimd_save();
+
+ /*
+ * Set the thread flag so that the kernel mode NEON state will
+ * be context switched along with the rest of the task state.
+ *
+ * On non-PREEMPT_RT, softirqs may interrupt task level kernel
+ * mode NEON, but the task will not be preemptible so setting
+ * TIF_USING_KMODE_NEON for those would be both wrong (as it
+ * would mark the task context NEON state as requiring a
+ * context switch) and unnecessary.
+ *
+ * On PREEMPT_RT, softirqs are serviced from a separate thread,
+ * which is scheduled as usual, and this guarantees that these
+ * softirqs are not interrupting use of the NEON in kernel mode
+ * in task context. So in this case, setting the flag here is
+ * always appropriate.
+ */
+ if (IS_ENABLED(CONFIG_PREEMPT_RT) || !in_serving_softirq())
+ set_thread_flag(TIF_USING_KMODE_NEON);
+ } else {
+ BUG_ON(IS_ENABLED(CONFIG_PREEMPT_RT) || !in_serving_softirq());
+ fpsimd_save_state(¤t->thread.kmode_fpsimd_state);
+ }
/* Invalidate any task state remaining in the fpsimd regs: */
fpsimd_flush_cpu_state();
+
+ put_cpu_fpsimd_context();
}
EXPORT_SYMBOL_GPL(kernel_neon_begin);
@@ -1856,7 +1892,16 @@ void kernel_neon_end(void)
if (!system_supports_fpsimd())
return;
- put_cpu_fpsimd_context();
+ /*
+ * If we are returning from a nested use of the kernel mode NEON,
+ * restore the task context kernel mode NEON state. This can only
+ * happen when running in softirq context on non-PREEMPT_RT.
+ */
+ if (!IS_ENABLED(CONFIG_PREEMPT_RT) && in_serving_softirq() &&
+ test_thread_flag(TIF_USING_KMODE_NEON))
+ fpsimd_load_state(¤t->thread.kmode_fpsimd_state);
+ else
+ clear_thread_flag(TIF_USING_KMODE_NEON);
}
EXPORT_SYMBOL_GPL(kernel_neon_end);
--
2.43.0.rc1.413.gea7ed67945-goog
More information about the linux-arm-kernel
mailing list