[PATCH v2 14/19] arm64: entry: handle all vectors with C
Joey Gouly
joey.gouly at arm.com
Fri May 21 08:59:52 PDT 2021
Hi Mark,
I like these clean ups to entry.S!
On Wed, May 19, 2021 at 01:38:57PM +0100, Mark Rutland wrote:
> We have 16 architectural exception vectors, and depending on kernel
> configuration we handle 8 or 12 of these with C code, and we handle 8 or
> 4 of these as sepcial cases in the entry assembly.
>
> It would be nicer if the entry assembly were uniform for all exceptions,
> and we deferred any specific handling of the exceptions to C code. This
> way the entry assembly can be more easily templated without ifdeffery or
> special cases, and it's easier to modify the handling of these cases in
> future (e.g. to dump additional registers other context).
>
> This patch reworks the entry code so that we always have a C handle for
s/handle/handler/
> every architectural exception vector, with the entry assembly being
> completely uniform. We now have to handle exceptions from EL1t and EL1h,
> and also have to handle exceptions from AArch32 even when the kernel is
> built without CONFIG_COMPAT. To make this clear and to simplify
> templating, we rename the top-level exception handlers with a consistent
> naming scheme:
>
> asm: <el>_<regsize>_<type>
> c: <el>_<regsize>_<type>_handler
>
> .. where:
>
> <el> is `el1t`, `el1h`, or `el0`
Is there a reason against using `el0t`? `el0t` is used in the Arm ARM.
It would get rid of the weird empty arguments in the `kernel_ventry`
and `entry_handler` macros.
> <regsize> is `64` or `32`
> <type> is `sync`, `irq`, `fiq`, or `error`
>
> ... e.g.
>
> asm: el1h_64_sync
> c: el1h_64_sync_handler
>
> ... with lower-level handlers simply using "el1" and "compat" as today.
>
> For unexpected exceptions, this information is passed to
> panic_unandled(), so it can report the specific vector an unexpected
> exception was taken from, e.g.
>
> | Unexpected 64-bit el1t sync exception
>
> For vectors we never expect to enter legitimately, the C code is
> gnerated using a macro to avoid code duplication.
>
> The `kernel_ventry` and `entry_handler` assembly macros are update to
s/update/updated/
> handle the new naming scheme. In theory it should be possible to
> generate the entry functions at the same time as the vectors using a
> single table, but this will require reworking the linker script to split
> the two into separate sections, so for now we duplicate the two.
>
> Signed-off-by: Mark Rutland <mark.rutland at arm.com>
> Cc: Catalin Marinas <catalin.marinas at arm.com>
> Cc: James Morse <james.morse at arm.com>
> Cc: Marc Zyngier <maz at kernel.org>
> Cc: Will Deacon <will at kernel.org>
> ---
> arch/arm64/include/asm/exception.h | 31 +++++---
> arch/arm64/kernel/entry-common.c | 51 +++++++------
> arch/arm64/kernel/entry.S | 146 ++++++++++++-------------------------
> 3 files changed, 92 insertions(+), 136 deletions(-)
>
> diff --git a/arch/arm64/include/asm/exception.h b/arch/arm64/include/asm/exception.h
> index 4284ee57a9a5..40a3a20dca1c 100644
> --- a/arch/arm64/include/asm/exception.h
> +++ b/arch/arm64/include/asm/exception.h
> @@ -31,18 +31,25 @@ static inline u32 disr_to_esr(u64 disr)
> return esr;
> }
>
> -asmlinkage void el1_sync_handler(struct pt_regs *regs);
> -asmlinkage void el1_irq_handler(struct pt_regs *regs);
> -asmlinkage void el1_fiq_handler(struct pt_regs *regs);
> -asmlinkage void el1_error_handler(struct pt_regs *regs);
> -asmlinkage void el0_sync_handler(struct pt_regs *regs);
> -asmlinkage void el0_irq_handler(struct pt_regs *regs);
> -asmlinkage void el0_fiq_handler(struct pt_regs *regs);
> -asmlinkage void el0_error_handler(struct pt_regs *regs);
> -asmlinkage void el0_sync_compat_handler(struct pt_regs *regs);
> -asmlinkage void el0_irq_compat_handler(struct pt_regs *regs);
> -asmlinkage void el0_fiq_compat_handler(struct pt_regs *regs);
> -asmlinkage void el0_error_compat_handler(struct pt_regs *regs);
> +asmlinkage void el1t_64_sync_handler(struct pt_regs *regs);
> +asmlinkage void el1t_64_irq_handler(struct pt_regs *regs);
> +asmlinkage void el1t_64_fiq_handler(struct pt_regs *regs);
> +asmlinkage void el1t_64_error_handler(struct pt_regs *regs);
> +
> +asmlinkage void el1h_64_sync_handler(struct pt_regs *regs);
> +asmlinkage void el1h_64_irq_handler(struct pt_regs *regs);
> +asmlinkage void el1h_64_fiq_handler(struct pt_regs *regs);
> +asmlinkage void el1h_64_error_handler(struct pt_regs *regs);
> +
> +asmlinkage void el0_64_sync_handler(struct pt_regs *regs);
> +asmlinkage void el0_64_irq_handler(struct pt_regs *regs);
> +asmlinkage void el0_64_fiq_handler(struct pt_regs *regs);
> +asmlinkage void el0_64_error_handler(struct pt_regs *regs);
> +
> +asmlinkage void el0_32_sync_handler(struct pt_regs *regs);
> +asmlinkage void el0_32_irq_handler(struct pt_regs *regs);
> +asmlinkage void el0_32_fiq_handler(struct pt_regs *regs);
> +asmlinkage void el0_32_error_handler(struct pt_regs *regs);
>
> asmlinkage void call_on_irq_stack(struct pt_regs *regs,
> void (*func)(struct pt_regs *));
Can you remove `bad_mode` from this header? (Further down, not shown here)
Also there is a reference to `bad_mode` in `traps.c`.
> diff --git a/arch/arm64/kernel/entry-common.c b/arch/arm64/kernel/entry-common.c
> index b43ef1a918a4..6f1caec913a9 100644
> --- a/arch/arm64/kernel/entry-common.c
> +++ b/arch/arm64/kernel/entry-common.c
> @@ -175,16 +175,11 @@ static void noinstr __panic_unhandled(struct pt_regs *regs, const char *vector,
> panic("Unhandled exception");
> }
>
> -asmlinkage void noinstr bad_mode(struct pt_regs *regs, int reason, unsigned int esr)
> -{
> - const char *handler[] = {
> - "Synchronous Abort",
> - "IRQ",
> - "FIQ",
> - "Error"
> - };
> -
> - __panic_unhandled(regs, handler[reason], esr);
> +#define UNHANDLED(el, regsize, vector) \
> +asmlinkage void noinstr el##_##regsize##_##vector##_handler(struct pt_regs *regs) \
> +{ \
> + const char *desc = #regsize "-bit " #el " " #vector; \
> + __panic_unhandled(regs, desc, read_sysreg(esr_el1)); \
> }
>
> #ifdef CONFIG_ARM64_ERRATUM_1463225
> @@ -236,6 +231,11 @@ static bool cortex_a76_erratum_1463225_debug_handler(struct pt_regs *regs)
> }
> #endif /* CONFIG_ARM64_ERRATUM_1463225 */
>
> +UNHANDLED(el1t, 64, sync)
> +UNHANDLED(el1t, 64, irq)
> +UNHANDLED(el1t, 64, fiq)
> +UNHANDLED(el1t, 64, error)
> +
> static void noinstr el1_abort(struct pt_regs *regs, unsigned long esr)
> {
> unsigned long far = read_sysreg(far_el1);
> @@ -271,7 +271,7 @@ static void noinstr el1_inv(struct pt_regs *regs, unsigned long esr)
> {
> enter_from_kernel_mode(regs);
> local_daif_inherit(regs);
> - bad_mode(regs, 0, esr);
> + __panic_unhandled(regs, "el1h sync", esr);
> local_daif_mask();
> exit_to_kernel_mode(regs);
This is never going to actually exit to kernel mode, is it? The panic
should stop that.
> }
> @@ -319,7 +319,7 @@ static void noinstr el1_fpac(struct pt_regs *regs, unsigned long esr)
> exit_to_kernel_mode(regs);
> }
>
> -asmlinkage void noinstr el1_sync_handler(struct pt_regs *regs)
> +asmlinkage void noinstr el1h_64_sync_handler(struct pt_regs *regs)
> {
> unsigned long esr = read_sysreg(esr_el1);
>
> @@ -364,17 +364,17 @@ static void noinstr el1_interrupt(struct pt_regs *regs,
> exit_el1_irq_or_nmi(regs);
> }
>
> -asmlinkage void noinstr el1_irq_handler(struct pt_regs *regs)
> +asmlinkage void noinstr el1h_64_irq_handler(struct pt_regs *regs)
> {
> el1_interrupt(regs, handle_arch_irq);
> }
>
> -asmlinkage void noinstr el1_fiq_handler(struct pt_regs *regs)
> +asmlinkage void noinstr el1h_64_fiq_handler(struct pt_regs *regs)
> {
> el1_interrupt(regs, handle_arch_fiq);
> }
>
> -asmlinkage void noinstr el1_error_handler(struct pt_regs *regs)
> +asmlinkage void noinstr el1h_64_error_handler(struct pt_regs *regs)
> {
> unsigned long esr = read_sysreg(esr_el1);
>
> @@ -520,7 +520,7 @@ static void noinstr el0_fpac(struct pt_regs *regs, unsigned long esr)
> do_ptrauth_fault(regs, esr);
> }
>
> -asmlinkage void noinstr el0_sync_handler(struct pt_regs *regs)
> +asmlinkage void noinstr el0_64_sync_handler(struct pt_regs *regs)
> {
> unsigned long esr = read_sysreg(esr_el1);
>
> @@ -591,7 +591,7 @@ static void noinstr __el0_irq_handler_common(struct pt_regs *regs)
> el0_interrupt(regs, handle_arch_irq);
> }
>
> -asmlinkage void noinstr el0_irq_handler(struct pt_regs *regs)
> +asmlinkage void noinstr el0_64_irq_handler(struct pt_regs *regs)
> {
> __el0_irq_handler_common(regs);
> }
> @@ -601,7 +601,7 @@ static void noinstr __el0_fiq_handler_common(struct pt_regs *regs)
> el0_interrupt(regs, handle_arch_fiq);
> }
>
> -asmlinkage void noinstr el0_fiq_handler(struct pt_regs *regs)
> +asmlinkage void noinstr el0_64_fiq_handler(struct pt_regs *regs)
> {
> __el0_fiq_handler_common(regs);
> }
> @@ -618,7 +618,7 @@ static void __el0_error_handler_common(struct pt_regs *regs)
> local_daif_restore(DAIF_PROCCTX);
> }
>
> -asmlinkage void noinstr el0_error_handler(struct pt_regs *regs)
> +asmlinkage void noinstr el0_64_error_handler(struct pt_regs *regs)
> {
> __el0_error_handler_common(regs);
> }
> @@ -638,7 +638,7 @@ static void noinstr el0_svc_compat(struct pt_regs *regs)
> do_el0_svc_compat(regs);
> }
>
> -asmlinkage void noinstr el0_sync_compat_handler(struct pt_regs *regs)
> +asmlinkage void noinstr el0_32_sync_handler(struct pt_regs *regs)
> {
> unsigned long esr = read_sysreg(esr_el1);
>
> @@ -682,18 +682,23 @@ asmlinkage void noinstr el0_sync_compat_handler(struct pt_regs *regs)
> }
> }
>
> -asmlinkage void noinstr el0_irq_compat_handler(struct pt_regs *regs)
> +asmlinkage void noinstr el0_32_irq_handler(struct pt_regs *regs)
> {
> __el0_irq_handler_common(regs);
> }
>
> -asmlinkage void noinstr el0_fiq_compat_handler(struct pt_regs *regs)
> +asmlinkage void noinstr el0_32_fiq_handler(struct pt_regs *regs)
> {
> __el0_fiq_handler_common(regs);
> }
>
> -asmlinkage void noinstr el0_error_compat_handler(struct pt_regs *regs)
> +asmlinkage void noinstr el0_32_error_handler(struct pt_regs *regs)
> {
> __el0_error_handler_common(regs);
> }
> +#else /* CONFIG_COMPAT */
> +UNHANDLED(el0, 32, sync)
> +UNHANDLED(el0, 32, irq)
> +UNHANDLED(el0, 32, fiq)
> +UNHANDLED(el0, 32, error)
> #endif /* CONFIG_COMPAT */
> diff --git a/arch/arm64/kernel/entry.S b/arch/arm64/kernel/entry.S
> index d4f80b9df621..257e8192e8d8 100644
> --- a/arch/arm64/kernel/entry.S
> +++ b/arch/arm64/kernel/entry.S
> @@ -51,16 +51,7 @@
> .endr
> .endm
>
> -/*
> - * Bad Abort numbers
> - *-----------------
> - */
> -#define BAD_SYNC 0
> -#define BAD_IRQ 1
> -#define BAD_FIQ 2
> -#define BAD_ERROR 3
> -
> - .macro kernel_ventry, el:req, regsize:req, label:req
> + .macro kernel_ventry, el:req, ht, regsize:req, label:req
> .align 7
> #ifdef CONFIG_UNMAP_KERNEL_AT_EL0
> .if \el == 0
> @@ -87,7 +78,7 @@ alternative_else_nop_endif
> tbnz x0, #THREAD_SHIFT, 0f
> sub x0, sp, x0 // x0'' = sp' - x0' = (sp + x0) - sp = x0
> sub sp, sp, x0 // sp'' = sp' - x0 = (sp + x0) - x0 = sp
> - b el\()\el\()_\label
> + b el\el\ht\()_\regsize\()_\label
>
> 0:
> /*
> @@ -119,7 +110,7 @@ alternative_else_nop_endif
> sub sp, sp, x0
> mrs x0, tpidrro_el0
> #endif
> - b el\()\el\()_\label
> + b el\el\ht\()_\regsize\()_\label
> .endm
>
> .macro tramp_alias, dst, sym
> @@ -510,32 +501,25 @@ tsk .req x28 // current thread_info
>
> .align 11
> SYM_CODE_START(vectors)
> - kernel_ventry 1, 64, sync_invalid // Synchronous EL1t
> - kernel_ventry 1, 64, irq_invalid // IRQ EL1t
> - kernel_ventry 1, 64, fiq_invalid // FIQ EL1t
> - kernel_ventry 1, 64, error_invalid // Error EL1t
> -
> - kernel_ventry 1, 64, sync // Synchronous EL1h
> - kernel_ventry 1, 64, irq // IRQ EL1h
> - kernel_ventry 1, 64, fiq // FIQ EL1h
> - kernel_ventry 1, 64, error // Error EL1h
> -
> - kernel_ventry 0, 64, sync // Synchronous 64-bit EL0
> - kernel_ventry 0, 64, irq // IRQ 64-bit EL0
> - kernel_ventry 0, 64, fiq // FIQ 64-bit EL0
> - kernel_ventry 0, 64, error // Error 64-bit EL0
> -
> -#ifdef CONFIG_COMPAT
> - kernel_ventry 0, 32, sync_compat // Synchronous 32-bit EL0
> - kernel_ventry 0, 32, irq_compat // IRQ 32-bit EL0
> - kernel_ventry 0, 32, fiq_compat // FIQ 32-bit EL0
> - kernel_ventry 0, 32, error_compat // Error 32-bit EL0
> -#else
> - kernel_ventry 0, 32, sync_invalid // Synchronous 32-bit EL0
> - kernel_ventry 0, 32, irq_invalid // IRQ 32-bit EL0
> - kernel_ventry 0, 32, fiq_invalid // FIQ 32-bit EL0
> - kernel_ventry 0, 32, error_invalid // Error 32-bit EL0
> -#endif
> + kernel_ventry 1, t, 64, sync // Synchronous EL1t
> + kernel_ventry 1, t, 64, irq // IRQ EL1t
> + kernel_ventry 1, t, 64, fiq // FIQ EL1h
> + kernel_ventry 1, t, 64, error // Error EL1t
> +
> + kernel_ventry 1, h, 64, sync // Synchronous EL1h
> + kernel_ventry 1, h, 64, irq // IRQ EL1h
> + kernel_ventry 1, h, 64, fiq // FIQ EL1h
> + kernel_ventry 1, h, 64, error // Error EL1h
> +
> + kernel_ventry 0, , 64, sync // Synchronous 64-bit EL0
> + kernel_ventry 0, , 64, irq // IRQ 64-bit EL0
> + kernel_ventry 0, , 64, fiq // FIQ 64-bit EL0
> + kernel_ventry 0, , 64, error // Error 64-bit EL0
> +
> + kernel_ventry 0, , 32, sync // Synchronous 32-bit EL0
> + kernel_ventry 0, , 32, irq // IRQ 32-bit EL0
> + kernel_ventry 0, , 32, fiq // FIQ 32-bit EL0
> + kernel_ventry 0, , 32, error // Error 32-bit EL0
> SYM_CODE_END(vectors)
>
> #ifdef CONFIG_VMAP_STACK
> @@ -566,83 +550,43 @@ __bad_stack:
> ASM_BUG()
> #endif /* CONFIG_VMAP_STACK */
>
> -/*
> - * Invalid mode handlers
> - */
> - .macro inv_entry, el, reason, regsize = 64
> - kernel_entry \el, \regsize
> - mov x0, sp
> - mov x1, #\reason
> - mrs x2, esr_el1
> - bl bad_mode
> - ASM_BUG()
> - .endm
> -
> -SYM_CODE_START_LOCAL(el0_sync_invalid)
> - inv_entry 0, BAD_SYNC
> -SYM_CODE_END(el0_sync_invalid)
> -
> -SYM_CODE_START_LOCAL(el0_irq_invalid)
> - inv_entry 0, BAD_IRQ
> -SYM_CODE_END(el0_irq_invalid)
> -
> -SYM_CODE_START_LOCAL(el0_fiq_invalid)
> - inv_entry 0, BAD_FIQ
> -SYM_CODE_END(el0_fiq_invalid)
> -
> -SYM_CODE_START_LOCAL(el0_error_invalid)
> - inv_entry 0, BAD_ERROR
> -SYM_CODE_END(el0_error_invalid)
>
> -SYM_CODE_START_LOCAL(el1_sync_invalid)
> - inv_entry 1, BAD_SYNC
> -SYM_CODE_END(el1_sync_invalid)
> -
> -SYM_CODE_START_LOCAL(el1_irq_invalid)
> - inv_entry 1, BAD_IRQ
> -SYM_CODE_END(el1_irq_invalid)
> -
> -SYM_CODE_START_LOCAL(el1_fiq_invalid)
> - inv_entry 1, BAD_FIQ
> -SYM_CODE_END(el1_fiq_invalid)
> -
> -SYM_CODE_START_LOCAL(el1_error_invalid)
> - inv_entry 1, BAD_ERROR
> -SYM_CODE_END(el1_error_invalid)
> -
> - .macro entry_handler el:req, regsize:req, label:req
> + .macro entry_handler el:req, ht, regsize:req, label:req
> .align 6
> -SYM_CODE_START_LOCAL_NOALIGN(el\el\()_\label)
> +SYM_CODE_START_LOCAL_NOALIGN(el\el\ht\()_\regsize\()_\label)
> kernel_entry \el, \regsize
> mov x0, sp
> - bl el\el\()_\label\()_handler
> + bl el\el\ht\()_\regsize\()_\label\()_handler
> .if \el == 0
> b ret_to_user
> .else
> b ret_to_kernel
> .endif
> -SYM_CODE_END(el\el\()_\label)
> +SYM_CODE_END(el\el\ht\()_\regsize\()_\label)
> .endm
>
> /*
> * Early exception handlers
> */
> - entry_handler 1, 64, sync
> - entry_handler 1, 64, irq
> - entry_handler 1, 64, fiq
> - entry_handler 1, 64, error
> -
> - entry_handler 0, 64, sync
> - entry_handler 0, 64, irq
> - entry_handler 0, 64, fiq
> - entry_handler 0, 64, error
> -
> -#ifdef CONFIG_COMPAT
> - entry_handler 0, 32, sync_compat
> - entry_handler 0, 32, irq_compat
> - entry_handler 0, 32, fiq_compat
> - entry_handler 0, 32, error_compat
> -#endif
> + entry_handler 1, t, 64, sync
> + entry_handler 1, t, 64, irq
> + entry_handler 1, t, 64, fiq
> + entry_handler 1, t, 64, error
> +
> + entry_handler 1, h, 64, sync
> + entry_handler 1, h, 64, irq
> + entry_handler 1, h, 64, fiq
> + entry_handler 1, h, 64, error
> +
> + entry_handler 0, , 64, sync
> + entry_handler 0, , 64, irq
> + entry_handler 0, , 64, fiq
> + entry_handler 0, , 64, error
> +
> + entry_handler 0, , 32, sync
> + entry_handler 0, , 32, irq
> + entry_handler 0, , 32, fiq
> + entry_handler 0, , 32, error
>
> SYM_CODE_START_LOCAL(ret_to_kernel)
> kernel_exit 1
> --
> 2.11.0
>
Minor comments / questions, but otherwise:
Reviewed-by: Joey Gouly <joey.gouly at arm.com>
Thanks,
Joey
More information about the linux-arm-kernel
mailing list