[PATCH 2/4] arm64: assembler: Add macros for return address protection

[Ard Biesheuvel]

When in-kernel pointer authentication is configured, emit PACIASP and
AUTIASP instructions as well as shadow call stack pushes and pops,
depending on the configuration.

Note that dynamic shadow call stack makes this slightly tricky, as it
depends on in-kernel BTI as well. The resulting code will never contain
both PAC and shadow call stack operations, even if shadow call stack
support is not configured as dynamic.

Signed-off-by: Ard Biesheuvel <ardb at kernel.org>
---
 arch/arm64/include/asm/assembler.h | 81 ++++++++++++++++++++
 1 file changed, 81 insertions(+)

diff --git a/arch/arm64/include/asm/assembler.h b/arch/arm64/include/asm/assembler.h
index 3d1714a7eb6411ba..99d74c29ab3cbe05 100644
--- a/arch/arm64/include/asm/assembler.h
+++ b/arch/arm64/include/asm/assembler.h
@@ -692,6 +692,85 @@ alternative_endif
 #endif
 	.endm
 
+	/*
+	 * protect_return_address - protect the return address value in
+	 * register @reg, either by signing it using PAC and/or by storing it
+	 * on the shadow call stack.
+	 *
+	 * The sequence below emits a shadow call stack push if the feature is
+	 * enabled, and if in-kernel PAC is enabled as well, the instruction
+	 * will be patched into a PACIA instruction involving the same register
+	 * address (and SP as the modifier) if PAC is detected at runtime.
+	 *
+	 * If in-kernel BTI and dynamic shadow call stacks are also configured,
+	 * it becomes a bit more tricky, because then, shadow call stacks will
+	 * only be enabled on non-BTI hardware, regardless of the PAUTH state.
+	 * In that case, we emit one of the following sequences.
+	 *
+	 *     PAC+BTI enabled  No PAC or BTI	BTI without PAC	PAC without BTI
+	 *
+	 *     B   0f		NOP		B   0f		NOP
+	 *     NOP		SCS push	SCS push	NOP
+	 *  0: PACIA		NOP		NOP		PACIA
+	 *
+	 * Note that, due to the code patching occuring at function entry and
+	 * exit, these macros must not be used in code that may execute before
+	 * the boot CPU feature based code patching has completed.
+	 */
+	.macro		protect_return_address, reg=x30
+#ifdef CONFIG_ARM64_PTR_AUTH_KERNEL
+#if defined(CONFIG_DYNAMIC_SCS) && defined(CONFIG_ARM64_BTI_KERNEL)
+alternative_if ARM64_BTI
+	b		.L0_\@
+alternative_else_nop_endif
+#endif
+alternative_if_not ARM64_HAS_ADDRESS_AUTH
+#endif
+#ifdef CONFIG_SHADOW_CALL_STACK
+	str		\reg, [x18], #8
+#endif
+#ifdef CONFIG_ARM64_PTR_AUTH_KERNEL
+#if !defined(CONFIG_SHADOW_CALL_STACK) || \
+    (defined(CONFIG_DYNAMIC_SCS) && defined(CONFIG_ARM64_BTI_KERNEL))
+.L0_\@:	nop
+#endif
+alternative_else
+#if defined(CONFIG_DYNAMIC_SCS) && defined(CONFIG_ARM64_BTI_KERNEL)
+	nop
+#endif
+	.arch_extension	pauth
+	pacia		\reg, sp
+alternative_endif
+#endif
+	.endm
+
+	/*
+	 * restore_return_address - restore the return address value in
+	 * register @reg, either by authenticating it using PAC and/or
+	 * reloading it from the shadow call stack.
+	 */
+	.macro		restore_return_address, reg=x30
+#ifdef CONFIG_ARM64_PTR_AUTH_KERNEL
+alternative_if ARM64_HAS_ADDRESS_AUTH
+	.arch_extension	pauth
+	autia		\reg, sp
+alternative_else_nop_endif
+#if defined(CONFIG_DYNAMIC_SCS) && defined(CONFIG_ARM64_BTI_KERNEL)
+alternative_if ARM64_BTI
+	b		.L0_\@
+alternative_else_nop_endif
+#endif
+alternative_if_not ARM64_HAS_ADDRESS_AUTH
+#endif
+#ifdef CONFIG_SHADOW_CALL_STACK
+	ldr		\reg, [x18, #-8]!
+#endif
+#ifdef CONFIG_ARM64_PTR_AUTH_KERNEL
+alternative_else_nop_endif
+.L0_\@:
+#endif
+	.endm
+
 	/*
 	 * frame_push - Push @regcount callee saved registers to the stack,
 	 *              starting at x19, as well as x29/x30, and set x29 to
@@ -699,6 +778,7 @@ alternative_endif
 	 *              for locals.
 	 */
 	.macro		frame_push, regcount:req, extra
+	protect_return_address
 	__frame		st, \regcount, \extra
 	.endm
 
@@ -710,6 +790,7 @@ alternative_endif
 	 */
 	.macro		frame_pop
 	__frame		ld
+	restore_return_address
 	.endm
 
 	.macro		__frame_regs, reg1, reg2, op, num
-- 
2.35.1