[PATCH] arm64: kasan: mte: move GCR_EL1 switch to task switch when KASAN disabled

[Peter Collingbourne]

On Sun, Aug 29, 2021 at 7:10 AM Andrey Konovalov <andreyknvl at gmail.com> wrote:
>
> On Fri, Aug 27, 2021 at 5:07 AM Peter Collingbourne <pcc at google.com> wrote:
> >
> > It is not necessary to write to GCR_EL1 on every kernel entry and
> > exit when HW tag-based KASAN is disabled because the kernel will not
> > execute any IRG instructions in that mode. Since accessing GCR_EL1
> > can be expensive on some microarchitectures, avoid doing so by moving
> > the access to task switch when HW tag-based KASAN is disabled.
> >
> > Signed-off-by: Peter Collingbourne <pcc at google.com>
> > ---
> >  arch/arm64/kernel/entry.S | 10 +++++-----
> >  arch/arm64/kernel/mte.c   | 24 +++++++++++++++++++++---
> >  include/linux/kasan.h     |  9 +++++++--
> >  3 files changed, 33 insertions(+), 10 deletions(-)
> >
> > diff --git a/arch/arm64/kernel/entry.S b/arch/arm64/kernel/entry.S
> > index 923ee2ac85fd..e12ed892cde8 100644
> > --- a/arch/arm64/kernel/entry.S
> > +++ b/arch/arm64/kernel/entry.S
> > @@ -178,9 +178,9 @@ alternative_else_nop_endif
> >
> >         .macro mte_set_kernel_gcr, tmp, tmp2
> >  #ifdef CONFIG_KASAN_HW_TAGS
> > -alternative_if_not ARM64_MTE
> > +alternative_cb kasan_hw_tags_enable
> >         b       1f
> > -alternative_else_nop_endif
> > +alternative_cb_end
> >         mov     \tmp, KERNEL_GCR_EL1
> >         msr_s   SYS_GCR_EL1, \tmp
> >  1:
> > @@ -188,10 +188,10 @@ alternative_else_nop_endif
> >         .endm
> >
> >         .macro mte_set_user_gcr, tsk, tmp, tmp2
> > -#ifdef CONFIG_ARM64_MTE
> > -alternative_if_not ARM64_MTE
> > +#ifdef CONFIG_KASAN_HW_TAGS
> > +alternative_cb kasan_hw_tags_enable
> >         b       1f
> > -alternative_else_nop_endif
> > +alternative_cb_end
> >         ldr     \tmp, [\tsk, #THREAD_MTE_CTRL]
> >
> >         mte_set_gcr \tmp, \tmp2
> > diff --git a/arch/arm64/kernel/mte.c b/arch/arm64/kernel/mte.c
> > index 9d314a3bad3b..0e8edb5d5861 100644
> > --- a/arch/arm64/kernel/mte.c
> > +++ b/arch/arm64/kernel/mte.c
> > @@ -162,7 +162,7 @@ void mte_check_tfsr_el1(void)
> >  }
> >  #endif
> >
> > -static void mte_update_sctlr_user(struct task_struct *task)
> > +static void mte_update_sctlr_user_and_gcr_excl(struct task_struct *task)
> >  {
> >         /*
> >          * This must be called with preemption disabled and can only be called
> > @@ -182,6 +182,24 @@ static void mte_update_sctlr_user(struct task_struct *task)
> >         else if (resolved_mte_tcf & MTE_CTRL_TCF_SYNC)
> >                 sctlr |= SCTLR_EL1_TCF0_SYNC;
> >         task->thread.sctlr_user = sctlr;
> > +
> > +       /*
> > +        * SYS_GCR_EL1 will be set to current->thread.mte_ctrl value by
> > +        * mte_set_user_gcr() in kernel_exit, but only if KASAN is enabled.
> > +        */
> > +       if (!kasan_hw_tags_enabled())
> > +               write_sysreg_s(((mte_ctrl >> MTE_CTRL_GCR_USER_EXCL_SHIFT) &
> > +                               SYS_GCR_EL1_EXCL_MASK) | SYS_GCR_EL1_RRND,
> > +                              SYS_GCR_EL1);
> > +}
> > +
> > +void __init kasan_hw_tags_enable(struct alt_instr *alt, __le32 *origptr,
> > +                                __le32 *updptr, int nr_inst)
> > +{
> > +       BUG_ON(nr_inst != 1); /* Branch -> NOP */
> > +
> > +       if (kasan_hw_tags_enabled())
> > +               *updptr = cpu_to_le32(aarch64_insn_gen_nop());
> >  }
> >
> >  void mte_thread_init_user(void)
> > @@ -199,7 +217,7 @@ void mte_thread_init_user(void)
> >
> >  void mte_thread_switch(struct task_struct *next)
> >  {
> > -       mte_update_sctlr_user(next);
> > +       mte_update_sctlr_user_and_gcr_excl(next);
> >
> >         /*
> >          * Check if an async tag exception occurred at EL1.
> > @@ -244,7 +262,7 @@ long set_mte_ctrl(struct task_struct *task, unsigned long arg)
> >         task->thread.mte_ctrl = mte_ctrl;
> >         if (task == current) {
> >                 preempt_disable();
> > -               mte_update_sctlr_user(task);
> > +               mte_update_sctlr_user_and_gcr_excl(task);
> >                 update_sctlr_el1(task->thread.sctlr_user);
> >                 preempt_enable();
> >         }
> > diff --git a/include/linux/kasan.h b/include/linux/kasan.h
> > index dd874a1ee862..de5f5913374d 100644
> > --- a/include/linux/kasan.h
> > +++ b/include/linux/kasan.h
> > @@ -89,7 +89,7 @@ static __always_inline bool kasan_enabled(void)
> >         return static_branch_likely(&kasan_flag_enabled);
> >  }
> >
> > -static inline bool kasan_has_integrated_init(void)
> > +static inline bool kasan_hw_tags_enabled(void)
> >  {
> >         return kasan_enabled();
> >  }
> > @@ -104,7 +104,7 @@ static inline bool kasan_enabled(void)
> >         return IS_ENABLED(CONFIG_KASAN);
> >  }
> >
> > -static inline bool kasan_has_integrated_init(void)
> > +static inline bool kasan_hw_tags_enabled(void)
> >  {
> >         return false;
> >  }
> > @@ -125,6 +125,11 @@ static __always_inline void kasan_free_pages(struct page *page,
> >
> >  #endif /* CONFIG_KASAN_HW_TAGS */
> >
> > +static inline bool kasan_has_integrated_init(void)
> > +{
> > +       return kasan_hw_tags_enabled();
> > +}
> > +
> >  #ifdef CONFIG_KASAN
> >
> >  struct kasan_cache {
> > --
> > 2.33.0.259.gc128427fd7-goog
> >
>
> Acked-by: Andrey Konovalov <andreyknvl at gmail.com>
>
> for non-arm-specific KASAN parts.

Hi Catalin,

Would you be able to take a look please? Do you think it's possible
that we could get this in before the end of the merge window?

Peter