[PATCH v3] arm64: mte: optimize GCR_EL1 modification on kernel entry/exit

[Peter Collingbourne]

Accessing GCR_EL1 and issuing an ISB can be expensive on some
microarchitectures. Although we must write to GCR_EL1, we can
restructure the code to avoid reading from it because the new value
can be derived entirely from the exclusion mask, which is already in
a GPR. Do so.

Furthermore, although an ISB is required in order to make this system
register update effective, and the same is true for PAC-related updates
to SCTLR_EL1 or APIAKey{Hi,Lo}_EL1, we issue two ISBs on machines
that support both features while we only need to issue one. To avoid
the unnecessary additional ISB, remove the ISBs from the PAC and
MTE-specific alternative blocks and add an ISB in a separate block
that is activated only if either feature is supported.

Signed-off-by: Peter Collingbourne <pcc at google.com>
Link: https://linux-review.googlesource.com/id/I560a190a74176ca4cc5191dad08f77f6b1577c75
---
v3:
- go back to modifying on entry/exit; optimize that path instead

v2:
- rebase onto v9 of the tag checking mode preference series

 arch/arm64/kernel/cpufeature.c | 19 +++++++++++++++++++
 arch/arm64/kernel/entry.S      | 23 +++++++++++++----------
 arch/arm64/tools/cpucaps       |  1 +
 3 files changed, 33 insertions(+), 10 deletions(-)

diff --git a/arch/arm64/kernel/cpufeature.c b/arch/arm64/kernel/cpufeature.c
index efed2830d141..740e09ade2ea 100644
--- a/arch/arm64/kernel/cpufeature.c
+++ b/arch/arm64/kernel/cpufeature.c
@@ -1717,6 +1717,20 @@ static bool has_generic_auth(const struct arm64_cpu_capabilities *entry,
 }
 #endif /* CONFIG_ARM64_PTR_AUTH */
 
+static bool has_address_auth_or_mte(const struct arm64_cpu_capabilities *entry,
+				    int scope)
+{
+#ifdef CONFIG_ARM64_PTR_AUTH
+	if (has_address_auth_metacap(entry, scope))
+		return true;
+#endif
+#ifdef CONFIG_ARM64_MTE
+	if (__system_matches_cap(ARM64_MTE))
+		return true;
+#endif
+	return false;
+}
+
 #ifdef CONFIG_ARM64_E0PD
 static void cpu_enable_e0pd(struct arm64_cpu_capabilities const *cap)
 {
@@ -2218,6 +2232,11 @@ static const struct arm64_cpu_capabilities arm64_features[] = {
 		.matches = has_cpuid_feature,
 		.min_field_value = 1,
 	},
+	{
+		.capability = ARM64_HAS_ADDRESS_AUTH_OR_MTE,
+		.type = ARM64_CPUCAP_BOOT_CPU_FEATURE,
+		.matches = has_address_auth_or_mte,
+	},
 	{},
 };
 
diff --git a/arch/arm64/kernel/entry.S b/arch/arm64/kernel/entry.S
index ce59280355c5..deb3ee45b018 100644
--- a/arch/arm64/kernel/entry.S
+++ b/arch/arm64/kernel/entry.S
@@ -175,15 +175,11 @@ alternative_else_nop_endif
 #endif
 	.endm
 
-	.macro mte_set_gcr, tmp, tmp2
+	.macro mte_set_gcr, mte_ctrl, tmp
 #ifdef CONFIG_ARM64_MTE
-	/*
-	 * Calculate and set the exclude mask preserving
-	 * the RRND (bit[16]) setting.
-	 */
-	mrs_s	\tmp2, SYS_GCR_EL1
-	bfxil	\tmp2, \tmp, #MTE_CTRL_GCR_USER_EXCL_SHIFT, #16
-	msr_s	SYS_GCR_EL1, \tmp2
+	ubfx	\tmp, \mte_ctrl, #MTE_CTRL_GCR_USER_EXCL_SHIFT, #16
+	orr	\tmp, \tmp, #SYS_GCR_EL1_RRND
+	msr_s	SYS_GCR_EL1, \tmp
 #endif
 	.endm
 
@@ -195,7 +191,6 @@ alternative_else_nop_endif
 	ldr_l	\tmp, gcr_kernel_excl
 
 	mte_set_gcr \tmp, \tmp2
-	isb
 1:
 #endif
 	.endm
@@ -269,12 +264,20 @@ alternative_if ARM64_HAS_ADDRESS_AUTH
 	orr	x0, x0, SCTLR_ELx_ENIA
 	msr	sctlr_el1, x0
 2:
-	isb
 alternative_else_nop_endif
 #endif
 
 	mte_set_kernel_gcr x22, x23
 
+alternative_if ARM64_HAS_ADDRESS_AUTH_OR_MTE
+	/*
+	 * Any non-self-synchronizing system register updates required for
+	 * kernel entry should be placed before this point. If necessary,
+	 * the alternative condition should be adjusted.
+	 */
+	isb
+alternative_else_nop_endif
+
 	scs_load tsk, x20
 	.else
 	add	x21, sp, #PT_REGS_SIZE
diff --git a/arch/arm64/tools/cpucaps b/arch/arm64/tools/cpucaps
index 21fbdda7086e..a938260ae5bc 100644
--- a/arch/arm64/tools/cpucaps
+++ b/arch/arm64/tools/cpucaps
@@ -8,6 +8,7 @@ HAS_32BIT_EL1
 HAS_ADDRESS_AUTH
 HAS_ADDRESS_AUTH_ARCH
 HAS_ADDRESS_AUTH_IMP_DEF
+HAS_ADDRESS_AUTH_OR_MTE
 HAS_AMU_EXTN
 HAS_ARMv8_4_TTL
 HAS_CACHE_DIC
-- 
2.32.0.93.g670b81a890-goog