[PATCH v3 11/12] arm64: smp: Rework early boot data into per-CPU arrays
Jinjie Ruan
ruanjinjie at huawei.com
Wed Jun 24 02:25:36 PDT 2026
Rework the global `secondary_data` and `__early_cpu_boot_status`
accessed during early boot into per-CPU arrays to allow secondary
CPUs to boot in parallel.
And reuse `__cpu_logical_map` array in the early boot code in head.S
to resolve each secondary CPU's logical ID concurrently.
Signed-off-by: Jinjie Ruan <ruanjinjie at huawei.com>
---
arch/arm64/include/asm/smp.h | 16 ++++++------
arch/arm64/kernel/asm-offsets.c | 2 ++
arch/arm64/kernel/head.S | 44 +++++++++++++++++++++++++++------
arch/arm64/kernel/setup.c | 9 +++++++
arch/arm64/kernel/smp.c | 11 +++++----
arch/arm64/mm/mmu.c | 2 +-
6 files changed, 62 insertions(+), 22 deletions(-)
diff --git a/arch/arm64/include/asm/smp.h b/arch/arm64/include/asm/smp.h
index e2151a01731f..7e69219e6e33 100644
--- a/arch/arm64/include/asm/smp.h
+++ b/arch/arm64/include/asm/smp.h
@@ -34,14 +34,9 @@
/*
* Logical CPU mapping.
*/
-extern u64 __cpu_logical_map[NR_CPUS];
+extern void set_cpu_logical_map(unsigned int cpu, u64 hwid);
extern u64 cpu_logical_map(unsigned int cpu);
-static inline void set_cpu_logical_map(unsigned int cpu, u64 hwid)
-{
- __cpu_logical_map[cpu] = hwid;
-}
-
struct seq_file;
/*
@@ -92,8 +87,11 @@ struct secondary_data {
long status;
};
-extern struct secondary_data secondary_data;
-extern long __early_cpu_boot_status;
+static_assert((sizeof(struct secondary_data) & (sizeof(struct secondary_data) - 1)) == 0,
+ "secondary_data size must be a power of 2 for assembly lsl assembly!");
+
+extern struct secondary_data cpu_boot_data[NR_CPUS];
+extern long __early_cpu_boot_status[NR_CPUS];
extern void secondary_entry(void);
extern void arch_send_call_function_single_ipi(int cpu);
@@ -124,7 +122,7 @@ static inline void __noreturn cpu_park_loop(void)
static inline void update_cpu_boot_status(unsigned int cpu, int val)
{
- WRITE_ONCE(secondary_data.status, val);
+ WRITE_ONCE(cpu_boot_data[cpu].status, val);
/* Ensure the visibility of the status update */
dsb(ishst);
}
diff --git a/arch/arm64/kernel/asm-offsets.c b/arch/arm64/kernel/asm-offsets.c
index b6367ff3a49c..566e2222af5b 100644
--- a/arch/arm64/kernel/asm-offsets.c
+++ b/arch/arm64/kernel/asm-offsets.c
@@ -11,6 +11,7 @@
#include <linux/arm_sdei.h>
#include <linux/sched.h>
#include <linux/ftrace.h>
+#include <linux/log2.h>
#include <linux/kexec.h>
#include <linux/mm.h>
#include <linux/kvm_host.h>
@@ -97,6 +98,7 @@ int main(void)
BLANK();
#endif
DEFINE(CPU_BOOT_TASK, offsetof(struct secondary_data, task));
+ DEFINE(SECONDARY_DATA_SHIFT, ilog2(sizeof(struct secondary_data)));
BLANK();
DEFINE(FTR_OVR_VAL_OFFSET, offsetof(struct arm64_ftr_override, val));
DEFINE(FTR_OVR_MASK_OFFSET, offsetof(struct arm64_ftr_override, mask));
diff --git a/arch/arm64/kernel/head.S b/arch/arm64/kernel/head.S
index 87a822e5c4ca..f58de58c4edc 100644
--- a/arch/arm64/kernel/head.S
+++ b/arch/arm64/kernel/head.S
@@ -12,6 +12,7 @@
#include <linux/linkage.h>
#include <linux/init.h>
#include <linux/pgtable.h>
+#include <linux/threads.h>
#include <asm/asm_pointer_auth.h>
#include <asm/assembler.h>
@@ -348,6 +349,31 @@ pen: ldr x4, [x3]
b pen
SYM_FUNC_END(secondary_holding_pen)
+ /*
+ * Convert the physical MPIDR of the current secondary CPU
+ * to its logical CPUID by traversing __cpu_logical_map
+ * in parallel.
+ */
+ .macro mpidr_to_cpuid, mpidr, cpuid, tmp1, tmp2
+ mov_q \tmp1, MPIDR_HWID_BITMASK
+ and \mpidr, \mpidr, \tmp1
+
+ adr_l \tmp1, __cpu_logical_map
+ mov \cpuid, #0
+.Lfind_cpuid\@:
+ ldr \tmp2, [\tmp1, \cpuid, lsl #3]
+ cmp \tmp2, #-1
+ b.eq .Lnext_cpu\@
+ cmp \tmp2, \mpidr
+ b.eq .Lfound_cpuid\@
+.Lnext_cpu\@:
+ add \cpuid, \cpuid, #1
+ cmp \cpuid, #NR_CPUS
+ b.ne .Lfind_cpuid\@
+ b __secondary_too_slow
+.Lfound_cpuid\@:
+ .endm
+
/*
* Secondary entry point that jumps straight into the kernel. Only to
* be used where CPUs are brought online dynamically by the kernel.
@@ -363,6 +389,8 @@ SYM_FUNC_START_LOCAL(secondary_startup)
* Common entry point for secondary CPUs.
*/
mov x20, x0 // preserve boot mode
+ mrs x0, mpidr_el1
+ mpidr_to_cpuid mpidr=x0, cpuid=x19, tmp1=x1, tmp2=x3
#ifdef CONFIG_ARM64_VA_BITS_52
alternative_if ARM64_HAS_VA52
@@ -386,12 +414,12 @@ SYM_FUNC_START_LOCAL(__secondary_switched)
mov x0, x20
bl finalise_el2
- str_l xzr, __early_cpu_boot_status, x3
adr_l x5, vectors
msr vbar_el1, x5
isb
- adr_l x0, secondary_data
+ adr_l x0, cpu_boot_data
+ add x0, x0, x19, lsl #SECONDARY_DATA_SHIFT
ldr x2, [x0, #CPU_BOOT_TASK]
cbz x2, __secondary_too_slow
@@ -430,13 +458,14 @@ SYM_FUNC_END(set_cpu_boot_mode_flag)
*
* update_early_cpu_boot_status tmp, status
* - Corrupts tmp1, tmp2
- * - Writes 'status' to __early_cpu_boot_status and makes sure
+ * - Writes 'status' to __early_cpu_boot_status[cpu] and makes sure
* it is committed to memory.
*/
- .macro update_early_cpu_boot_status status, tmp1, tmp2
- mov \tmp2, #\status
+ .macro update_early_cpu_boot_status status, cpu_reg, tmp1, tmp2
adr_l \tmp1, __early_cpu_boot_status
+ mov \tmp2, #\status
+ add \tmp1, \tmp1, \cpu_reg, lsl #3
str \tmp2, [\tmp1]
dmb sy
dc ivac, \tmp1 // Invalidate potentially stale cache line
@@ -486,7 +515,7 @@ SYM_FUNC_START(__cpu_secondary_check52bitva)
#endif
update_early_cpu_boot_status \
- CPU_STUCK_IN_KERNEL | CPU_STUCK_REASON_52_BIT_VA, x0, x1
+ CPU_STUCK_IN_KERNEL | CPU_STUCK_REASON_52_BIT_VA, x19, x0, x1
1: wfe
wfi
b 1b
@@ -498,7 +527,7 @@ SYM_FUNC_END(__cpu_secondary_check52bitva)
SYM_FUNC_START_LOCAL(__no_granule_support)
/* Indicate that this CPU can't boot and is stuck in the kernel */
update_early_cpu_boot_status \
- CPU_STUCK_IN_KERNEL | CPU_STUCK_REASON_NO_GRAN, x1, x2
+ CPU_STUCK_IN_KERNEL | CPU_STUCK_REASON_NO_GRAN, x19, x1, x2
1:
wfe
wfi
@@ -508,6 +537,7 @@ SYM_FUNC_END(__no_granule_support)
SYM_FUNC_START_LOCAL(__primary_switch)
adrp x1, reserved_pg_dir
adrp x2, __pi_init_idmap_pg_dir
+ mov x19, #0
bl __enable_mmu
adrp x1, early_init_stack
diff --git a/arch/arm64/kernel/setup.c b/arch/arm64/kernel/setup.c
index 23c05dc7a8f2..856c00ab6f19 100644
--- a/arch/arm64/kernel/setup.c
+++ b/arch/arm64/kernel/setup.c
@@ -273,6 +273,15 @@ arch_initcall(reserve_memblock_reserved_regions);
u64 __cpu_logical_map[NR_CPUS] = { [0 ... NR_CPUS-1] = INVALID_HWID };
+void set_cpu_logical_map(unsigned int cpu, u64 hwid)
+{
+ unsigned long start = (unsigned long)&__cpu_logical_map[cpu];
+ unsigned long end = start + sizeof(__cpu_logical_map[cpu]);
+
+ __cpu_logical_map[cpu] = hwid;
+ dcache_clean_inval_poc(start, end);
+}
+
u64 cpu_logical_map(unsigned int cpu)
{
return __cpu_logical_map[cpu];
diff --git a/arch/arm64/kernel/smp.c b/arch/arm64/kernel/smp.c
index 14b94df26b44..98ddbe50081d 100644
--- a/arch/arm64/kernel/smp.c
+++ b/arch/arm64/kernel/smp.c
@@ -61,7 +61,8 @@
* so we need some other way of telling a new secondary core
* where to place its SVC stack
*/
-struct secondary_data secondary_data;
+struct secondary_data cpu_boot_data[NR_CPUS] ____cacheline_aligned;
+
/* Number of CPUs which aren't online, but looping in kernel text. */
static int cpus_stuck_in_kernel;
@@ -115,7 +116,7 @@ int arch_cpuhp_kick_ap_alive(unsigned int cpu, struct task_struct *idle)
* We need to tell the secondary core where to find its stack and the
* page tables.
*/
- secondary_data.task = idle;
+ cpu_boot_data[cpu].task = idle;
update_cpu_boot_status(cpu, CPU_MMU_OFF);
/* Now bring the CPU into our world */
@@ -136,10 +137,10 @@ void arch_cpuhp_cleanup_kick_cpu(unsigned int cpu, bool is_alive)
* We failed to synchronise with the CPU, so check if it left us
* any breadcrumbs.
*/
- secondary_data.task = NULL;
- status = READ_ONCE(secondary_data.status);
+ cpu_boot_data[cpu].task = NULL;
+ status = READ_ONCE(cpu_boot_data[cpu].status);
if (status == CPU_MMU_OFF)
- status = READ_ONCE(__early_cpu_boot_status);
+ status = READ_ONCE(__early_cpu_boot_status[cpu]);
switch (status & CPU_BOOT_STATUS_MASK) {
default:
diff --git a/arch/arm64/mm/mmu.c b/arch/arm64/mm/mmu.c
index dd85e093ffdb..71c160c6c383 100644
--- a/arch/arm64/mm/mmu.c
+++ b/arch/arm64/mm/mmu.c
@@ -62,7 +62,7 @@ static bool rodata_is_rw __ro_after_init = true;
* The booting CPU updates the failed status @__early_cpu_boot_status,
* with MMU turned off.
*/
-long __section(".mmuoff.data.write") __early_cpu_boot_status;
+long __section(".mmuoff.data.write") __early_cpu_boot_status[NR_CPUS];
static DEFINE_SPINLOCK(swapper_pgdir_lock);
static DEFINE_MUTEX(fixmap_lock);
--
2.34.1
More information about the linux-arm-kernel
mailing list