[PATCH v3 3/3] kasan: arm64: support specialized outlined tag mismatch checks

Andrey Konovalov andreyknvl at google.com
Thu Jan 7 10:20:58 EST 2021


On Thu, Dec 3, 2020 at 6:12 AM Peter Collingbourne <pcc at google.com> wrote:
>
> By using outlined checks we can achieve a significant code size
> improvement by moving the tag-based ASAN checks into separate
> functions. Unlike the existing CONFIG_KASAN_OUTLINE mode these
> functions have a custom calling convention that preserves most
> registers and is specialized to the register containing the address
> and the type of access, and as a result we can eliminate the code
> size and performance overhead of a standard calling convention such
> as AAPCS for these functions.
>
> This change depends on a separate series of changes to Clang [1] to
> support outlined checks in the kernel, although the change works fine
> without them (we just don't get outlined checks). This is because the
> flag -mllvm -hwasan-inline-all-checks=0 has no effect until the Clang
> changes land. The flag was introduced in the Clang 9.0 timeframe as
> part of the support for outlined checks in userspace and because our
> minimum Clang version is 10.0 we can pass it unconditionally.
>
> Outlined checks require a new runtime function with a custom calling
> convention. Add this function to arch/arm64/lib.
>
> I measured the code size of defconfig + tag-based KASAN, as well
> as boot time (i.e. time to init launch) on a DragonBoard 845c with
> an Android arm64 GKI kernel. The results are below:
>
>                                code size    boot time
> CONFIG_KASAN_INLINE=y before    92824064      6.18s
> CONFIG_KASAN_INLINE=y after     38822400      6.65s
> CONFIG_KASAN_OUTLINE=y          39215616     11.48s
>
> We can see straight away that specialized outlined checks beat the
> existing CONFIG_KASAN_OUTLINE=y on both code size and boot time
> for tag-based ASAN.
>
> As for the comparison between CONFIG_KASAN_INLINE=y before and after
> we saw similar performance numbers in userspace [2] and decided
> that since the performance overhead is minimal compared to the
> overhead of tag-based ASAN itself as well as compared to the code
> size improvements we would just replace the inlined checks with the
> specialized outlined checks without the option to select between them,
> and that is what I have implemented in this patch. But we may make a
> different decision for the kernel such as having CONFIG_KASAN_OUTLINE=y
> turn on specialized outlined checks if Clang is new enough.
>
> Signed-off-by: Peter Collingbourne <pcc at google.com>
> Link: https://linux-review.googlesource.com/id/I1a30036c70ab3c3ee78d75ed9b87ef7cdc3fdb76
> Link: [1] https://reviews.llvm.org/D90426
> Link: [2] https://reviews.llvm.org/D56954
> ---
> v3:
> - adopt Mark Rutland's suggested changes
> - move frame record alignment patches behind this one
>
> v2:
> - use calculations in the stack spills and restores
> - improve the comment at the top of the function
> - add a BTI instruction
>
>  arch/arm64/include/asm/asm-prototypes.h |  6 ++
>  arch/arm64/include/asm/module.lds.h     | 17 +++++-
>  arch/arm64/lib/Makefile                 |  2 +
>  arch/arm64/lib/kasan_sw_tags.S          | 76 +++++++++++++++++++++++++
>  mm/kasan/tags.c                         |  7 +++
>  scripts/Makefile.kasan                  |  1 +
>  6 files changed, 107 insertions(+), 2 deletions(-)
>  create mode 100644 arch/arm64/lib/kasan_sw_tags.S
>
> diff --git a/arch/arm64/include/asm/asm-prototypes.h b/arch/arm64/include/asm/asm-prototypes.h
> index 1c9a3a0c5fa5..ec1d9655f885 100644
> --- a/arch/arm64/include/asm/asm-prototypes.h
> +++ b/arch/arm64/include/asm/asm-prototypes.h
> @@ -23,4 +23,10 @@ long long __ashlti3(long long a, int b);
>  long long __ashrti3(long long a, int b);
>  long long __lshrti3(long long a, int b);
>
> +/*
> + * This function uses a custom calling convention and cannot be called from C so
> + * this prototype is not entirely accurate.
> + */
> +void __hwasan_tag_mismatch(unsigned long addr, unsigned long access_info);
> +
>  #endif /* __ASM_PROTOTYPES_H */
> diff --git a/arch/arm64/include/asm/module.lds.h b/arch/arm64/include/asm/module.lds.h
> index 691f15af788e..4a6d717f75f3 100644
> --- a/arch/arm64/include/asm/module.lds.h
> +++ b/arch/arm64/include/asm/module.lds.h
> @@ -1,7 +1,20 @@
> -#ifdef CONFIG_ARM64_MODULE_PLTS
>  SECTIONS {
> +#ifdef CONFIG_ARM64_MODULE_PLTS
>         .plt (NOLOAD) : { BYTE(0) }
>         .init.plt (NOLOAD) : { BYTE(0) }
>         .text.ftrace_trampoline (NOLOAD) : { BYTE(0) }
> -}
>  #endif
> +
> +#ifdef CONFIG_KASAN_SW_TAGS
> +       /*
> +        * Outlined checks go into comdat-deduplicated sections named .text.hot.
> +        * Because they are in comdats they are not combined by the linker and
> +        * we otherwise end up with multiple sections with the same .text.hot
> +        * name in the .ko file. The kernel module loader warns if it sees
> +        * multiple sections with the same name so we use this sections
> +        * directive to force them into a single section and silence the
> +        * warning.
> +        */
> +       .text.hot : { *(.text.hot) }
> +#endif
> +}
> diff --git a/arch/arm64/lib/Makefile b/arch/arm64/lib/Makefile
> index d31e1169d9b8..8e60d76a1b47 100644
> --- a/arch/arm64/lib/Makefile
> +++ b/arch/arm64/lib/Makefile
> @@ -18,3 +18,5 @@ obj-$(CONFIG_CRC32) += crc32.o
>  obj-$(CONFIG_FUNCTION_ERROR_INJECTION) += error-inject.o
>
>  obj-$(CONFIG_ARM64_MTE) += mte.o
> +
> +obj-$(CONFIG_KASAN_SW_TAGS) += kasan_sw_tags.o
> diff --git a/arch/arm64/lib/kasan_sw_tags.S b/arch/arm64/lib/kasan_sw_tags.S
> new file mode 100644
> index 000000000000..5b04464c045e
> --- /dev/null
> +++ b/arch/arm64/lib/kasan_sw_tags.S
> @@ -0,0 +1,76 @@
> +/* SPDX-License-Identifier: GPL-2.0-only */
> +/*
> + * Copyright (C) 2020 Google LLC
> + */
> +
> +#include <linux/linkage.h>
> +#include <asm/assembler.h>
> +
> +/*
> + * Report a tag mismatch detected by tag-based KASAN.
> + *
> + * A compiler-generated thunk calls this with a non-AAPCS calling
> + * convention. Upon entry to this function, registers are as follows:
> + *
> + * x0:         fault address (see below for restore)
> + * x1:         fault description (see below for restore)
> + * x2 to x15:  callee-saved
> + * x16 to x17: safe to clobber
> + * x18 to x30: callee-saved
> + * sp:         pre-decremented by 256 bytes (see below for restore)
> + *
> + * The caller has decremented the SP by 256 bytes, and created a
> + * structure on the stack as follows:
> + *
> + * sp + 0..15:    x0 and x1 to be restored
> + * sp + 16..231:  free for use
> + * sp + 232..247: x29 and x30 (same as in GPRs)
> + * sp + 248..255: free for use
> + *
> + * Note that this is not a struct pt_regs.
> + *
> + * To call a regular AAPCS function we must save x2 to x15 (which we can
> + * store in the gaps), and create a frame record (for which we can use
> + * x29 and x30 spilled by the caller as those match the GPRs).
> + *
> + * The caller expects x0 and x1 to be restored from the structure, and
> + * for the structure to be removed from the stack (i.e. the SP must be
> + * incremented by 256 prior to return).
> + */
> +SYM_CODE_START(__hwasan_tag_mismatch)
> +#ifdef BTI_C
> +       BTI_C
> +#endif
> +       add     x29, sp, #232
> +       stp     x2, x3, [sp, #8 * 2]
> +       stp     x4, x5, [sp, #8 * 4]
> +       stp     x6, x7, [sp, #8 * 6]
> +       stp     x8, x9, [sp, #8 * 8]
> +       stp     x10, x11, [sp, #8 * 10]
> +       stp     x12, x13, [sp, #8 * 12]
> +       stp     x14, x15, [sp, #8 * 14]
> +#ifndef CONFIG_SHADOW_CALL_STACK
> +       str     x18, [sp, #8 * 18]
> +#endif
> +
> +       mov     x2, x30
> +       bl      kasan_tag_mismatch
> +
> +       ldp     x0, x1, [sp]
> +       ldp     x2, x3, [sp, #8 * 2]
> +       ldp     x4, x5, [sp, #8 * 4]
> +       ldp     x6, x7, [sp, #8 * 6]
> +       ldp     x8, x9, [sp, #8 * 8]
> +       ldp     x10, x11, [sp, #8 * 10]
> +       ldp     x12, x13, [sp, #8 * 12]
> +       ldp     x14, x15, [sp, #8 * 14]
> +#ifndef CONFIG_SHADOW_CALL_STACK
> +       ldr     x18, [sp, #8 * 18]
> +#endif
> +       ldp     x29, x30, [sp, #8 * 29]
> +
> +       /* remove the structure from the stack */
> +       add     sp, sp, #256
> +       ret
> +SYM_CODE_END(__hwasan_tag_mismatch)
> +EXPORT_SYMBOL(__hwasan_tag_mismatch)
> diff --git a/mm/kasan/tags.c b/mm/kasan/tags.c
> index e02a36a51f42..d00613956c79 100644
> --- a/mm/kasan/tags.c
> +++ b/mm/kasan/tags.c
> @@ -198,3 +198,10 @@ struct kasan_track *kasan_get_free_track(struct kmem_cache *cache,
>
>         return &alloc_meta->free_track[i];
>  }
> +
> +void kasan_tag_mismatch(unsigned long addr, unsigned long access_info,
> +                       unsigned long ret_ip)
> +{
> +       kasan_report(addr, 1 << (access_info & 0xf), access_info & 0x10,
> +                    ret_ip);
> +}
> diff --git a/scripts/Makefile.kasan b/scripts/Makefile.kasan
> index 1e000cc2e7b4..1f2cccc264d8 100644
> --- a/scripts/Makefile.kasan
> +++ b/scripts/Makefile.kasan
> @@ -44,6 +44,7 @@ endif
>  CFLAGS_KASAN := -fsanitize=kernel-hwaddress \
>                 -mllvm -hwasan-instrument-stack=$(CONFIG_KASAN_STACK) \
>                 -mllvm -hwasan-use-short-granules=0 \
> +               -mllvm -hwasan-inline-all-checks=0 \
>                 $(instrumentation_flags)
>
>  endif # CONFIG_KASAN_SW_TAGS
> --
> 2.29.2.454.gaff20da3a2-goog
>

Acked-by: Andrey Konovalov <andreyknvl at google.com>



More information about the linux-arm-kernel mailing list