[PATCH 06/21] arm64: Store 'current' in TPIDRRO_EL0 instead of SP_EL0
David Laight
david.laight.linux at gmail.com
Tue Sep 8 06:19:50 PDT 2026
On Mon, 7 Sep 2026 17:42:31 +0100
Will Deacon <will at kernel.org> wrote:
> To make SP_EL0 available to point at the per-cpu kernel overflow stack,
> we must relocate 'current' elsewhere. With TPIDRRO_EL0 now restored late
> in the return-to-user path, we can otherwise use it to hold 'current'
> and update the various accessors accordingly.
>
> Move 'current' from SP_EL0 to TPIDRRO_EL0.
Is it worth having a #define for the name of the register that stores 'current'?
(And probably the other similar registers.)
That would avoid the bisection problems with the following two patches
and might make other code a bit more readable.
David
>
> Cc: Mark Rutland <mark.rutland at arm.com>
> Signed-off-by: Will Deacon <will at kernel.org>
> ---
> arch/arm64/include/asm/assembler.h | 2 +-
> arch/arm64/include/asm/current.h | 6 +++---
> arch/arm64/kernel/entry.S | 20 ++++++++++----------
> arch/arm64/kernel/head.S | 2 +-
> arch/arm64/kernel/process.c | 5 +++--
> arch/arm64/net/bpf_jit_comp.c | 4 ++--
> 6 files changed, 20 insertions(+), 19 deletions(-)
>
> diff --git a/arch/arm64/include/asm/assembler.h b/arch/arm64/include/asm/assembler.h
> index 0b58b550e8dc..fd1ae5337935 100644
> --- a/arch/arm64/include/asm/assembler.h
> +++ b/arch/arm64/include/asm/assembler.h
> @@ -587,7 +587,7 @@ alternative_else_nop_endif
> * Return the current task_struct.
> */
> .macro get_current_task, rd
> - mrs \rd, sp_el0
> + mrs \rd, tpidrro_el0
> .endm
>
> /*
> diff --git a/arch/arm64/include/asm/current.h b/arch/arm64/include/asm/current.h
> index c92912eaf186..8c7efac05394 100644
> --- a/arch/arm64/include/asm/current.h
> +++ b/arch/arm64/include/asm/current.h
> @@ -14,11 +14,11 @@ struct task_struct;
> */
> static __always_inline struct task_struct *get_current(void)
> {
> - unsigned long sp_el0;
> + unsigned long tpidrro_el0;
>
> - asm ("mrs %0, sp_el0" : "=r" (sp_el0));
> + asm ("mrs %0, tpidrro_el0" : "=r" (tpidrro_el0));
>
> - return (struct task_struct *)sp_el0;
> + return (struct task_struct *)tpidrro_el0;
> }
>
> #define current get_current()
> diff --git a/arch/arm64/kernel/entry.S b/arch/arm64/kernel/entry.S
> index b5d8277f608a..a45be0a837c8 100644
> --- a/arch/arm64/kernel/entry.S
> +++ b/arch/arm64/kernel/entry.S
> @@ -81,9 +81,9 @@
> /* Stash the original SP (minus PT_REGS_SIZE) in tpidr_el0. */
> msr tpidr_el0, x0
>
> - /* Recover the original x0 value and stash it in tpidrro_el0 */
> + /* Recover the original x0 value and stash it in sp_el0 */
> sub x0, sp, x0
> - msr tpidrro_el0, x0
> + msr sp_el0, x0
>
> /* Switch to the overflow stack */
> adr_this_cpu sp, overflow_stack + OVERFLOW_STACK_SIZE, x0
> @@ -99,7 +99,7 @@
>
> /* We were already on the overflow stack. Restore sp/x0 and carry on. */
> sub sp, sp, x0
> - mrs x0, tpidrro_el0
> + mrs x0, sp_el0
> b el\el\ht\()_\regsize\()_\label
> .endif
> .org .Lventry_start\@ + 128 // Did we overflow the ventry slot?
> @@ -226,7 +226,7 @@ alternative_cb_end
> clear_gp_regs
> mrs x21, sp_el0
> ldr_this_cpu tsk, __entry_task, x20
> - msr sp_el0, tsk
> + msr tpidrro_el0, tsk
>
> /*
> * Ensure MDSCR_EL1.SS is clear, since we can unmask debug exceptions
> @@ -548,7 +548,7 @@ SYM_CODE_START_LOCAL(__bad_stack)
> */
>
> /* Restore the original x0 value */
> - mrs x0, tpidrro_el0
> + mrs x0, sp_el0
>
> /*
> * Store the original GPRs to the new stack. The orginal SP (minus
> @@ -843,7 +843,7 @@ SYM_FUNC_START(cpu_switch_to)
> ldp x29, x9, [x8], #16
> ldr lr, [x8]
> mov sp, x9
> - msr sp_el0, x1
> + msr tpidrro_el0, x1
> ptrauth_keys_install_kernel x1, x8, x9, x10
> scs_save x0
> scs_load_current
> @@ -1031,11 +1031,11 @@ SYM_CODE_START(__sdei_asm_handler)
>
> /*
> * We may have interrupted userspace, or a guest, or exit-from or
> - * return-to either of these. We can't trust sp_el0, restore it.
> + * return-to either of these. We can't trust tpidrro_el0, restore it.
> */
> - mrs x28, sp_el0
> + mrs x28, tpidrro_el0
> ldr_this_cpu dst=x0, sym=__entry_task, tmp=x1
> - msr sp_el0, x0
> + msr tpidrro_el0, x0
>
> /* If we interrupted the kernel point to the previous stack/frame. */
> and x0, x3, #0xc
> @@ -1051,7 +1051,7 @@ SYM_CODE_START(__sdei_asm_handler)
> mov x1, x19
> bl __sdei_handler
>
> - msr sp_el0, x28
> + msr tpidrro_el0, x28
> /* restore regs >x17 that we clobbered */
> mov x4, x19 // keep x4 for __sdei_asm_exit_trampoline
> ldp x28, x29, [x4, #SDEI_EVENT_INTREGS + 16 * 14]
> diff --git a/arch/arm64/kernel/head.S b/arch/arm64/kernel/head.S
> index 87a822e5c4ca..c33551ee6572 100644
> --- a/arch/arm64/kernel/head.S
> +++ b/arch/arm64/kernel/head.S
> @@ -193,7 +193,7 @@ SYM_CODE_END(preserve_boot_args)
> * for consistency with user tasks and kthreads.
> */
> .macro init_cpu_task tsk, tmp1, tmp2
> - msr sp_el0, \tsk
> + msr tpidrro_el0, \tsk
>
> ldr \tmp1, [\tsk, #TSK_STACK]
> add sp, \tmp1, #THREAD_SIZE
> diff --git a/arch/arm64/kernel/process.c b/arch/arm64/kernel/process.c
> index bfdc12166895..79a19be88ced 100644
> --- a/arch/arm64/kernel/process.c
> +++ b/arch/arm64/kernel/process.c
> @@ -551,8 +551,9 @@ static void ssbs_thread_switch(struct task_struct *next)
> }
>
> /*
> - * We store our current task in sp_el0, which is clobbered by userspace. Keep a
> - * shadow copy so that we can restore this upon entry from userspace.
> + * We store our current task in tpidrro_el0, which is clobbered when returning
> + * to userspace. Keep a shadow copy so that we can restore this upon entry from
> + * userspace.
> *
> * This is *only* for exception entry from EL0, and is not valid until we
> * __switch_to() a user task.
> diff --git a/arch/arm64/net/bpf_jit_comp.c b/arch/arm64/net/bpf_jit_comp.c
> index c18e005a41db..51d853b92443 100644
> --- a/arch/arm64/net/bpf_jit_comp.c
> +++ b/arch/arm64/net/bpf_jit_comp.c
> @@ -1694,7 +1694,7 @@ static int build_insn(const struct bpf_verifier_env *env, const struct bpf_insn
> if (insn->src_reg == 0 && insn->imm == BPF_FUNC_get_smp_processor_id) {
> cpu_offset = offsetof(struct thread_info, cpu);
>
> - emit(A64_MRS_SP_EL0(tmp), ctx);
> + emit(A64_MRS_TPIDRRO_EL0(tmp), ctx);
> if (is_lsi_offset(cpu_offset, 2)) {
> emit(A64_LDR32I(r0, tmp, cpu_offset), ctx);
> } else {
> @@ -1707,7 +1707,7 @@ static int build_insn(const struct bpf_verifier_env *env, const struct bpf_insn
> /* Implement helper call to bpf_get_current_task/_btf() inline */
> if (insn->src_reg == 0 && (insn->imm == BPF_FUNC_get_current_task ||
> insn->imm == BPF_FUNC_get_current_task_btf)) {
> - emit(A64_MRS_SP_EL0(r0), ctx);
> + emit(A64_MRS_TPIDRRO_EL0(r0), ctx);
> break;
> }
>
More information about the linux-arm-kernel
mailing list