[PATCH 1/5] arm64: kasan: allow to init memory when setting tags

[Marco Elver]

On Sat, Mar 06, 2021 at 01:15AM +0100, Andrey Konovalov wrote:
> This change adds an argument to mte_set_mem_tag_range() that allows
> to enable memory initialization when settinh the allocation tags.
> The implementation uses stzg instruction instead of stg when this
> argument indicates to initialize memory.
> 
> Combining setting allocation tags with memory initialization will
> improve HW_TAGS KASAN performance when init_on_alloc/free is enabled.
> 
> This change doesn't integrate memory initialization with KASAN,
> this is done is subsequent patches in this series.
> 
> Signed-off-by: Andrey Konovalov <andreyknvl at google.com>

Acked-by: Marco Elver <elver at google.com>

> ---
>  arch/arm64/include/asm/memory.h    |  4 ++--
>  arch/arm64/include/asm/mte-kasan.h | 20 ++++++++++++++------
>  mm/kasan/kasan.h                   |  9 +++++----
>  3 files changed, 21 insertions(+), 12 deletions(-)
> 
> diff --git a/arch/arm64/include/asm/memory.h b/arch/arm64/include/asm/memory.h
> index c759faf7a1ff..f1ba48b4347d 100644
> --- a/arch/arm64/include/asm/memory.h
> +++ b/arch/arm64/include/asm/memory.h
> @@ -248,8 +248,8 @@ static inline const void *__tag_set(const void *addr, u8 tag)
>  #define arch_init_tags(max_tag)			mte_init_tags(max_tag)
>  #define arch_get_random_tag()			mte_get_random_tag()
>  #define arch_get_mem_tag(addr)			mte_get_mem_tag(addr)
> -#define arch_set_mem_tag_range(addr, size, tag)	\
> -			mte_set_mem_tag_range((addr), (size), (tag))
> +#define arch_set_mem_tag_range(addr, size, tag, init)	\
> +			mte_set_mem_tag_range((addr), (size), (tag), (init))
>  #endif /* CONFIG_KASAN_HW_TAGS */
>  
>  /*
> diff --git a/arch/arm64/include/asm/mte-kasan.h b/arch/arm64/include/asm/mte-kasan.h
> index 7ab500e2ad17..35fe549f7ea4 100644
> --- a/arch/arm64/include/asm/mte-kasan.h
> +++ b/arch/arm64/include/asm/mte-kasan.h
> @@ -53,7 +53,8 @@ static inline u8 mte_get_random_tag(void)
>   * Note: The address must be non-NULL and MTE_GRANULE_SIZE aligned and
>   * size must be non-zero and MTE_GRANULE_SIZE aligned.
>   */
> -static inline void mte_set_mem_tag_range(void *addr, size_t size, u8 tag)
> +static inline void mte_set_mem_tag_range(void *addr, size_t size,
> +						u8 tag, bool init)
>  {
>  	u64 curr, end;
>  
> @@ -68,10 +69,16 @@ static inline void mte_set_mem_tag_range(void *addr, size_t size, u8 tag)
>  		 * 'asm volatile' is required to prevent the compiler to move
>  		 * the statement outside of the loop.
>  		 */
> -		asm volatile(__MTE_PREAMBLE "stg %0, [%0]"
> -			     :
> -			     : "r" (curr)
> -			     : "memory");
> +		if (init)
> +			asm volatile(__MTE_PREAMBLE "stzg %0, [%0]"
> +				     :
> +				     : "r" (curr)
> +				     : "memory");
> +		else
> +			asm volatile(__MTE_PREAMBLE "stg %0, [%0]"
> +				     :
> +				     : "r" (curr)
> +				     : "memory");
>  
>  		curr += MTE_GRANULE_SIZE;
>  	} while (curr != end);
> @@ -100,7 +107,8 @@ static inline u8 mte_get_random_tag(void)
>  	return 0xFF;
>  }
>  
> -static inline void mte_set_mem_tag_range(void *addr, size_t size, u8 tag)
> +static inline void mte_set_mem_tag_range(void *addr, size_t size,
> +						u8 tag, bool init)
>  {
>  }
>  
> diff --git a/mm/kasan/kasan.h b/mm/kasan/kasan.h
> index 8c55634d6edd..7fbb32234414 100644
> --- a/mm/kasan/kasan.h
> +++ b/mm/kasan/kasan.h
> @@ -291,7 +291,7 @@ static inline const void *arch_kasan_set_tag(const void *addr, u8 tag)
>  #define arch_get_mem_tag(addr)	(0xFF)
>  #endif
>  #ifndef arch_set_mem_tag_range
> -#define arch_set_mem_tag_range(addr, size, tag) ((void *)(addr))
> +#define arch_set_mem_tag_range(addr, size, tag, init) ((void *)(addr))
>  #endif
>  
>  #define hw_enable_tagging()			arch_enable_tagging()
> @@ -299,7 +299,8 @@ static inline const void *arch_kasan_set_tag(const void *addr, u8 tag)
>  #define hw_set_tagging_report_once(state)	arch_set_tagging_report_once(state)
>  #define hw_get_random_tag()			arch_get_random_tag()
>  #define hw_get_mem_tag(addr)			arch_get_mem_tag(addr)
> -#define hw_set_mem_tag_range(addr, size, tag)	arch_set_mem_tag_range((addr), (size), (tag))
> +#define hw_set_mem_tag_range(addr, size, tag, init) \
> +			arch_set_mem_tag_range((addr), (size), (tag), (init))
>  
>  #else /* CONFIG_KASAN_HW_TAGS */
>  
> @@ -343,7 +344,7 @@ static inline void kasan_poison(const void *addr, size_t size, u8 value)
>  	if (WARN_ON(size & KASAN_GRANULE_MASK))
>  		return;
>  
> -	hw_set_mem_tag_range((void *)addr, size, value);
> +	hw_set_mem_tag_range((void *)addr, size, value, false);
>  }
>  
>  static inline void kasan_unpoison(const void *addr, size_t size)
> @@ -360,7 +361,7 @@ static inline void kasan_unpoison(const void *addr, size_t size)
>  		return;
>  	size = round_up(size, KASAN_GRANULE_SIZE);
>  
> -	hw_set_mem_tag_range((void *)addr, size, tag);
> +	hw_set_mem_tag_range((void *)addr, size, tag, false);
>  }
>  
>  static inline bool kasan_byte_accessible(const void *addr)
> -- 
> 2.30.1.766.gb4fecdf3b7-goog
>