[PATCH v2 1/3] arm64: extable: Add fixup handling for uaccess CPY* instructions

[Robin Murphy]

On 28/02/2025 5:00 pm, Kristina Martšenko wrote:
> A subsequent patch will use CPY* instructions to copy between user and
> kernel memory. Add a new exception fixup type to avoid fixing up faults
> on kernel memory accesses, in order to make it easier to debug kernel
> bugs and to keep the same behavior as with regular loads/stores.

Trusting my "the rest looks OK to me" from last time since I've already 
paged the details back out again... :)

Reviewed-by: Robin Murphy <robin.murphy at arm.com>

> Signed-off-by: Kristina Martšenko <kristina.martsenko at arm.com>
> ---
>   arch/arm64/include/asm/asm-extable.h | 10 +++++++++-
>   arch/arm64/include/asm/extable.h     |  2 +-
>   arch/arm64/mm/extable.c              | 25 ++++++++++++++++++++++++-
>   arch/arm64/mm/fault.c                |  2 +-
>   4 files changed, 35 insertions(+), 4 deletions(-)
> 
> diff --git a/arch/arm64/include/asm/asm-extable.h b/arch/arm64/include/asm/asm-extable.h
> index b8a5861dc7b7..292f2687a12e 100644
> --- a/arch/arm64/include/asm/asm-extable.h
> +++ b/arch/arm64/include/asm/asm-extable.h
> @@ -9,7 +9,8 @@
>   #define EX_TYPE_BPF			1
>   #define EX_TYPE_UACCESS_ERR_ZERO	2
>   #define EX_TYPE_KACCESS_ERR_ZERO	3
> -#define EX_TYPE_LOAD_UNALIGNED_ZEROPAD	4
> +#define EX_TYPE_UACCESS_CPY		4
> +#define EX_TYPE_LOAD_UNALIGNED_ZEROPAD	5
>   
>   /* Data fields for EX_TYPE_UACCESS_ERR_ZERO */
>   #define EX_DATA_REG_ERR_SHIFT	0
> @@ -23,6 +24,9 @@
>   #define EX_DATA_REG_ADDR_SHIFT	5
>   #define EX_DATA_REG_ADDR	GENMASK(9, 5)
>   
> +/* Data fields for EX_TYPE_UACCESS_CPY */
> +#define EX_DATA_UACCESS_WRITE	BIT(0)
> +
>   #ifdef __ASSEMBLY__
>   
>   #define __ASM_EXTABLE_RAW(insn, fixup, type, data)	\
> @@ -69,6 +73,10 @@
>   	.endif
>   	.endm
>   
> +	.macro		_asm_extable_uaccess_cpy, insn, fixup, uaccess_is_write
> +	__ASM_EXTABLE_RAW(\insn, \fixup, EX_TYPE_UACCESS_CPY, \uaccess_is_write)
> +	.endm
> +
>   #else /* __ASSEMBLY__ */
>   
>   #include <linux/stringify.h>
> diff --git a/arch/arm64/include/asm/extable.h b/arch/arm64/include/asm/extable.h
> index 72b0e71cc3de..5892b8977710 100644
> --- a/arch/arm64/include/asm/extable.h
> +++ b/arch/arm64/include/asm/extable.h
> @@ -45,5 +45,5 @@ bool ex_handler_bpf(const struct exception_table_entry *ex,
>   }
>   #endif /* !CONFIG_BPF_JIT */
>   
> -bool fixup_exception(struct pt_regs *regs);
> +bool fixup_exception(struct pt_regs *regs, unsigned long esr);
>   #endif
> diff --git a/arch/arm64/mm/extable.c b/arch/arm64/mm/extable.c
> index 228d681a8715..afb5241e4d91 100644
> --- a/arch/arm64/mm/extable.c
> +++ b/arch/arm64/mm/extable.c
> @@ -8,8 +8,18 @@
>   #include <linux/uaccess.h>
>   
>   #include <asm/asm-extable.h>
> +#include <asm/esr.h>
>   #include <asm/ptrace.h>
>   
> +static bool cpy_faulted_on_uaccess(const struct exception_table_entry *ex,
> +				   unsigned long esr)
> +{
> +	bool uaccess_is_write = FIELD_GET(EX_DATA_UACCESS_WRITE, ex->data);
> +	bool fault_on_write = esr & ESR_ELx_WNR;
> +
> +	return uaccess_is_write == fault_on_write;
> +}
> +
>   static inline unsigned long
>   get_ex_fixup(const struct exception_table_entry *ex)
>   {
> @@ -29,6 +39,17 @@ static bool ex_handler_uaccess_err_zero(const struct exception_table_entry *ex,
>   	return true;
>   }
>   
> +static bool ex_handler_uaccess_cpy(const struct exception_table_entry *ex,
> +				   struct pt_regs *regs, unsigned long esr)
> +{
> +	/* Do not fix up faults on kernel memory accesses */
> +	if (!cpy_faulted_on_uaccess(ex, esr))
> +		return false;
> +
> +	regs->pc = get_ex_fixup(ex);
> +	return true;
> +}
> +
>   static bool
>   ex_handler_load_unaligned_zeropad(const struct exception_table_entry *ex,
>   				  struct pt_regs *regs)
> @@ -56,7 +77,7 @@ ex_handler_load_unaligned_zeropad(const struct exception_table_entry *ex,
>   	return true;
>   }
>   
> -bool fixup_exception(struct pt_regs *regs)
> +bool fixup_exception(struct pt_regs *regs, unsigned long esr)
>   {
>   	const struct exception_table_entry *ex;
>   
> @@ -70,6 +91,8 @@ bool fixup_exception(struct pt_regs *regs)
>   	case EX_TYPE_UACCESS_ERR_ZERO:
>   	case EX_TYPE_KACCESS_ERR_ZERO:
>   		return ex_handler_uaccess_err_zero(ex, regs);
> +	case EX_TYPE_UACCESS_CPY:
> +		return ex_handler_uaccess_cpy(ex, regs, esr);
>   	case EX_TYPE_LOAD_UNALIGNED_ZEROPAD:
>   		return ex_handler_load_unaligned_zeropad(ex, regs);
>   	}
> diff --git a/arch/arm64/mm/fault.c b/arch/arm64/mm/fault.c
> index ef63651099a9..da4854fc6150 100644
> --- a/arch/arm64/mm/fault.c
> +++ b/arch/arm64/mm/fault.c
> @@ -375,7 +375,7 @@ static void __do_kernel_fault(unsigned long addr, unsigned long esr,
>   	 * Are we prepared to handle this kernel fault?
>   	 * We are almost certainly not prepared to handle instruction faults.
>   	 */
> -	if (!is_el1_instruction_abort(esr) && fixup_exception(regs))
> +	if (!is_el1_instruction_abort(esr) && fixup_exception(regs, esr))
>   		return;
>   
>   	if (WARN_RATELIMIT(is_spurious_el1_translation_fault(addr, esr, regs),