[PATCH 2/2] arm64/xor: use EOR3 instructions when available

[Catalin Marinas]

Hi Ard,

I trust you on the algorithm but some minor issues below.

On Tue, Nov 09, 2021 at 01:03:36PM +0100, Ard Biesheuvel wrote:
> diff --git a/arch/arm64/Kconfig b/arch/arm64/Kconfig
> index 6f2d3e31fb54..14354acba5b4 100644
> --- a/arch/arm64/Kconfig
> +++ b/arch/arm64/Kconfig
> @@ -2034,6 +2034,9 @@ config SYSVIPC_COMPAT
>  	def_bool y
>  	depends on COMPAT && SYSVIPC
>  
> +config CC_HAVE_SHA3
> +	def_bool $(cc-option, -march=armv8.2-a+sha3)

Is it the compiler or the assembler that we need to support this? I
think it's sufficient to only check the latter.

I'd also move it to the ARMv8.2 section.

> +
>  menu "Power management options"
>  
>  source "kernel/power/Kconfig"
> diff --git a/arch/arm64/lib/xor-neon.c b/arch/arm64/lib/xor-neon.c
> index ee4795f3e166..0415cb94c781 100644
> --- a/arch/arm64/lib/xor-neon.c
> +++ b/arch/arm64/lib/xor-neon.c
> @@ -172,6 +172,135 @@ void xor_arm64_neon_5(unsigned long bytes, unsigned long *p1,
>  }
>  EXPORT_SYMBOL(xor_arm64_neon_5);
>  
> +static inline uint64x2_t eor3(uint64x2_t p, uint64x2_t q, uint64x2_t r)
> +{
> +	uint64x2_t res;
> +
> +	asm(".arch	armv8.2-a+sha3		\n"
> +	    "eor3 %0.16b, %1.16b, %2.16b, %3.16b"
> +	    : "=w"(res) : "w"(p), "w"(q), "w"(r));
> +	return res;
> +}

The .arch here may confuse the compiler/assembler since it overrides any
other .arch. I think this diff on top would do but I haven't extensively
tested it. I can fold it in if you give it a try:

diff --git a/arch/arm64/Kconfig b/arch/arm64/Kconfig
index 5adae54c98d8..c5104e8829e5 100644
--- a/arch/arm64/Kconfig
+++ b/arch/arm64/Kconfig
@@ -1545,6 +1545,12 @@ endmenu
 
 menu "ARMv8.2 architectural features"
 
+config AS_HAS_ARMV8_2
+	def_bool $(cc-option,-Wa$(comma)-march=armv8.2-a)
+
+config AS_HAS_SHA3
+	def_bool $(as-instr,.arch armv8.2-a+sha3)
+
 config ARM64_PMEM
 	bool "Enable support for persistent memory"
 	select ARCH_HAS_PMEM_API
@@ -2032,9 +2038,6 @@ config SYSVIPC_COMPAT
 	def_bool y
 	depends on COMPAT && SYSVIPC
 
-config CC_HAVE_SHA3
-	def_bool $(cc-option, -march=armv8.2-a+sha3)
-
 menu "Power management options"
 
 source "kernel/power/Kconfig"
diff --git a/arch/arm64/Makefile b/arch/arm64/Makefile
index e8cfc5868aa8..2f1de88651e6 100644
--- a/arch/arm64/Makefile
+++ b/arch/arm64/Makefile
@@ -58,6 +58,11 @@ stack_protector_prepare: prepare0
 					include/generated/asm-offsets.h))
 endif
 
+ifeq ($(CONFIG_AS_HAS_ARMV8_2), y)
+# make sure to pass the newest target architecture to -march.
+asm-arch := armv8.2-a
+endif
+
 # Ensure that if the compiler supports branch protection we default it
 # off, this will be overridden if we are using branch protection.
 branch-prot-flags-y += $(call cc-option,-mbranch-protection=none)
diff --git a/arch/arm64/lib/xor-neon.c b/arch/arm64/lib/xor-neon.c
index 0415cb94c781..2ca823825363 100644
--- a/arch/arm64/lib/xor-neon.c
+++ b/arch/arm64/lib/xor-neon.c
@@ -176,7 +176,7 @@ static inline uint64x2_t eor3(uint64x2_t p, uint64x2_t q, uint64x2_t r)
 {
 	uint64x2_t res;
 
-	asm(".arch	armv8.2-a+sha3		\n"
+	asm(ARM64_ASM_PREAMBLE ".arch_extension sha3\n"
 	    "eor3 %0.16b, %1.16b, %2.16b, %3.16b"
 	    : "=w"(res) : "w"(p), "w"(q), "w"(r));
 	return res;
@@ -311,7 +311,7 @@ EXPORT_STATIC_CALL(xor_arm64_5);
 
 static int __init xor_neon_init(void)
 {
-	if (IS_ENABLED(CONFIG_CC_HAVE_SHA3) && cpu_have_named_feature(SHA3)) {
+	if (IS_ENABLED(CONFIG_AS_HAS_SHA3) && cpu_have_named_feature(SHA3)) {
 		static_call_update(xor_arm64_3, xor_arm64_eor3_3);
 		static_call_update(xor_arm64_4, xor_arm64_eor3_4);
 		static_call_update(xor_arm64_5, xor_arm64_eor3_5);

-- 
Catalin