[PATCH v1 2/2] arm64: errata: add module load workaround for erratum #843419

[Ard Biesheuvel]

In order to work around Cortex-A53 erratum #843419, this patch updates
the module loading logic to either change potentially problematic adrp
instructions into adr instructions (if the symbol turns out to be in
range), or emit a veneer that is guaranteed to be at an offset that does
not trigger the issue.

Signed-off-by: Ard Biesheuvel <ard.biesheuvel at linaro.org>
---
 arch/arm64/Kconfig                   | 21 ++++++++++++
 arch/arm64/include/asm/mod_veneers.h | 20 ++++++++++++
 arch/arm64/kernel/mod_veneers.c      | 33 +++++++++++++++++++
 arch/arm64/kernel/module.c           | 34 ++++++++++++++++++++
 4 files changed, 108 insertions(+)

diff --git a/arch/arm64/Kconfig b/arch/arm64/Kconfig
index b591aac06fed..0f9e1d1235cb 100644
--- a/arch/arm64/Kconfig
+++ b/arch/arm64/Kconfig
@@ -337,6 +337,27 @@ endmenu
 config ARM64_MOD_VENEERS
 	bool
 
+config ARM64_ERRATUM_843419
+	bool "Cortex-A53: 843419: A load or store might access an incorrect address"
+	depends on MODULES
+	select ARM64_MOD_VENEERS
+	default y
+	help
+	  This option applies special relocation logic when loading kernel
+	  modules to prevent ADRP instructions from appearing in either of the
+	  last two instruction slots in a 4 KB page. This can cause a subsequent
+	  memory access to use an incorrect address on Cortex-A53 parts up to
+	  r0p4. If such an instruction is subject to relocation, we either
+	  change the ADRP instruction into an ADR instruction (if the symbol it
+	  refers to is within 1 MB), or else we replace it by a call to a veneer
+	  which executes the same instruction at a safe offset before branching
+	  back past the original location.
+
+	  Note that the kernel itself must be linked with a version of ld which
+	  fixes potentially affected ADRP instructions at build time.
+
+	  If unsure, say Y.
+
 choice
 	prompt "Page size"
 	default ARM64_4K_PAGES
diff --git a/arch/arm64/include/asm/mod_veneers.h b/arch/arm64/include/asm/mod_veneers.h
new file mode 100644
index 000000000000..3cf36ce91549
--- /dev/null
+++ b/arch/arm64/include/asm/mod_veneers.h
@@ -0,0 +1,20 @@
+
+#include <linux/types.h>
+
+struct veneer_erratum_843419 {
+	u32	adrp;
+	u32	branch;
+};
+
+static inline bool erratum_843419_affects_adrp_insn(void *addr)
+{
+	/*
+	 * The workaround for erratum 843419 only needs to be
+	 * applied if the adrp instruction appears in either of
+	 * the last two instruction slots in the 4 KB page.
+	 */
+	return ((u64)addr % SZ_4K) >= (SZ_4K - 8);
+}
+
+void *emit_erratum_843419_veneer(struct module *mod, u32 *insn,
+				 Elf64_Shdr *sechdrs);
diff --git a/arch/arm64/kernel/mod_veneers.c b/arch/arm64/kernel/mod_veneers.c
index 9649c4f305bd..7da300e59749 100644
--- a/arch/arm64/kernel/mod_veneers.c
+++ b/arch/arm64/kernel/mod_veneers.c
@@ -10,6 +10,8 @@
 #include <linux/kernel.h>
 #include <linux/module.h>
 
+#include <asm/mod_veneers.h>
+
 static bool in_init(const struct module *mod, u64 addr)
 {
 	return addr - (u64)mod->module_init < mod->init_size;
@@ -32,6 +34,31 @@ static void *alloc_veneer(struct module *mod, u64 loc, int size,
 	return ret;
 }
 
+#ifdef CONFIG_ARM64_ERRATUM_843419
+void *emit_erratum_843419_veneer(struct module *mod, u32 *insn,
+				 Elf64_Shdr *sechdrs)
+{
+	struct veneer_erratum_843419 *veneer;
+
+	veneer = alloc_veneer(mod, (u64)insn, 2 * sizeof(*veneer), sechdrs);
+	if (erratum_843419_affects_adrp_insn(&veneer->adrp))
+		/*
+		 * We allocated a veneer that is susceptible to the same problem
+		 * as the original location. We allocated twice the space, so
+		 * just advance to the next slot.
+		 */
+		veneer++;
+
+	veneer->adrp = *insn;
+	veneer->branch = aarch64_insn_gen_branch_imm(
+				(unsigned long)&veneer->branch,
+				(unsigned long)(insn + 1),
+				AARCH64_INSN_BRANCH_NOLINK);
+
+	return veneer;
+}
+#endif
+
 /* estimate the maximum size of the veneer for this relocation */
 static unsigned long get_veneers_size(Elf64_Addr base, const Elf64_Rela *rel,
 				      int num)
@@ -41,6 +68,12 @@ static unsigned long get_veneers_size(Elf64_Addr base, const Elf64_Rela *rel,
 
 	for (i = 0; i < num; i++)
 		switch (ELF64_R_TYPE(rel[i].r_info)) {
+		case R_AARCH64_ADR_PREL_PG_HI21_NC:
+		case R_AARCH64_ADR_PREL_PG_HI21:
+#ifdef CONFIG_ARM64_ERRATUM_843419
+			ret += 2 * sizeof(struct veneer_erratum_843419);
+#endif
+			break;
 		}
 	return ret;
 }
diff --git a/arch/arm64/kernel/module.c b/arch/arm64/kernel/module.c
index 67bf4107f6ef..5d46f63c0111 100644
--- a/arch/arm64/kernel/module.c
+++ b/arch/arm64/kernel/module.c
@@ -27,6 +27,7 @@
 #include <linux/vmalloc.h>
 #include <asm/alternative.h>
 #include <asm/insn.h>
+#include <asm/mod_veneers.h>
 #include <asm/sections.h>
 
 #define	AARCH64_INSN_IMM_MOVNZ		AARCH64_INSN_IMM_MAX
@@ -335,6 +336,39 @@ int apply_relocate_add(Elf64_Shdr *sechdrs,
 		case R_AARCH64_ADR_PREL_PG_HI21_NC:
 			overflow_check = false;
 		case R_AARCH64_ADR_PREL_PG_HI21:
+#ifdef CONFIG_ARM64_ERRATUM_843419
+			/*
+			 * TODO check for presence of affected A53 cores
+			 */
+			if (erratum_843419_affects_adrp_insn(loc)) {
+				struct veneer_erratum_843419 *v;
+
+				/*
+				 * This adrp instruction appears at an offset
+				 * that may be problematic on older Cortex-A53
+				 * cores. So first, try to convert it into a
+				 * simple adr instruction.
+				 */
+				ovf = reloc_insn_imm(RELOC_OP_PREL, loc,
+						     val & ~(SZ_4K - 1), 0, 21,
+						     AARCH64_INSN_IMM_ADR);
+				if (ovf == 0) {
+					/* success! convert adrp -> adr */
+					*(u32 *)loc &= 0x7fffffff;
+					break;
+				}
+
+				/* symbol out of range for adr -> emit veneer */
+				v = emit_erratum_843419_veneer(me, loc,
+							       sechdrs);
+				*(u32 *)loc = aarch64_insn_gen_branch_imm(
+						(unsigned long)loc,
+						(unsigned long)v,
+						AARCH64_INSN_BRANCH_NOLINK);
+				loc = &v->adrp;
+				/* fall through */
+			}
+#endif
 			ovf = reloc_insn_imm(RELOC_OP_PAGE, loc, val, 12, 21,
 					     AARCH64_INSN_IMM_ADR);
 			break;
-- 
1.9.1