[PATCH bpf-next 1/1] arm64: bpf: Add BPF exception tables

[Qian Cai]

On Tue, Jul 28, 2020 at 05:21:26PM +0200, Jean-Philippe Brucker wrote:
> When a tracing BPF program attempts to read memory without using the
> bpf_probe_read() helper, the verifier marks the load instruction with
> the BPF_PROBE_MEM flag. Since the arm64 JIT does not currently recognize
> this flag it falls back to the interpreter.
> 
> Add support for BPF_PROBE_MEM, by appending an exception table to the
> BPF program. If the load instruction causes a data abort, the fixup
> infrastructure finds the exception table and fixes up the fault, by
> clearing the destination register and jumping over the faulting
> instruction.
> 
> To keep the compact exception table entry format, inspect the pc in
> fixup_exception(). A more generic solution would add a "handler" field
> to the table entry, like on x86 and s390.
> 
> Signed-off-by: Jean-Philippe Brucker <jean-philippe at linaro.org>

This will fail to compile on arm64,

https://gitlab.com/cailca/linux-mm/-/blob/master/arm64.config

arch/arm64/mm/extable.o: In function `fixup_exception':
arch/arm64/mm/extable.c:19: undefined reference to `arm64_bpf_fixup_exception'

> ---
> Note: the extable is aligned on 32 bits. Given that extable entries have
> 32-bit members I figured we don't need to align it to 64 bits.
> ---
>  arch/arm64/include/asm/extable.h |  3 ++
>  arch/arm64/mm/extable.c          | 11 ++--
>  arch/arm64/net/bpf_jit_comp.c    | 93 +++++++++++++++++++++++++++++---
>  3 files changed, 98 insertions(+), 9 deletions(-)
> 
> diff --git a/arch/arm64/include/asm/extable.h b/arch/arm64/include/asm/extable.h
> index 56a4f68b262e..bcee40df1586 100644
> --- a/arch/arm64/include/asm/extable.h
> +++ b/arch/arm64/include/asm/extable.h
> @@ -22,5 +22,8 @@ struct exception_table_entry
>  
>  #define ARCH_HAS_RELATIVE_EXTABLE
>  
> +int arm64_bpf_fixup_exception(const struct exception_table_entry *ex,
> +			      struct pt_regs *regs);
> +
>  extern int fixup_exception(struct pt_regs *regs);
>  #endif
> diff --git a/arch/arm64/mm/extable.c b/arch/arm64/mm/extable.c
> index 81e694af5f8c..1f42991cacdd 100644
> --- a/arch/arm64/mm/extable.c
> +++ b/arch/arm64/mm/extable.c
> @@ -11,8 +11,13 @@ int fixup_exception(struct pt_regs *regs)
>  	const struct exception_table_entry *fixup;
>  
>  	fixup = search_exception_tables(instruction_pointer(regs));
> -	if (fixup)
> -		regs->pc = (unsigned long)&fixup->fixup + fixup->fixup;
> +	if (!fixup)
> +		return 0;
>  
> -	return fixup != NULL;
> +	if (regs->pc >= BPF_JIT_REGION_START &&
> +	    regs->pc < BPF_JIT_REGION_END)
> +		return arm64_bpf_fixup_exception(fixup, regs);
> +
> +	regs->pc = (unsigned long)&fixup->fixup + fixup->fixup;
> +	return 1;
>  }
> diff --git a/arch/arm64/net/bpf_jit_comp.c b/arch/arm64/net/bpf_jit_comp.c
> index 3cb25b43b368..f8912e45be7a 100644
> --- a/arch/arm64/net/bpf_jit_comp.c
> +++ b/arch/arm64/net/bpf_jit_comp.c
> @@ -7,6 +7,7 @@
>  
>  #define pr_fmt(fmt) "bpf_jit: " fmt
>  
> +#include <linux/bitfield.h>
>  #include <linux/bpf.h>
>  #include <linux/filter.h>
>  #include <linux/printk.h>
> @@ -56,6 +57,7 @@ struct jit_ctx {
>  	int idx;
>  	int epilogue_offset;
>  	int *offset;
> +	int exentry_idx;
>  	__le32 *image;
>  	u32 stack_size;
>  };
> @@ -351,6 +353,67 @@ static void build_epilogue(struct jit_ctx *ctx)
>  	emit(A64_RET(A64_LR), ctx);
>  }
>  
> +#define BPF_FIXUP_OFFSET_MASK	GENMASK(26, 0)
> +#define BPF_FIXUP_REG_MASK	GENMASK(31, 27)
> +
> +int arm64_bpf_fixup_exception(const struct exception_table_entry *ex,
> +			      struct pt_regs *regs)
> +{
> +	off_t offset = FIELD_GET(BPF_FIXUP_OFFSET_MASK, ex->fixup);
> +	int dst_reg = FIELD_GET(BPF_FIXUP_REG_MASK, ex->fixup);
> +
> +	regs->regs[dst_reg] = 0;
> +	regs->pc = (unsigned long)&ex->fixup - offset;
> +	return 1;
> +}
> +
[]