[PATCH 03/19] cpu/hotplug: Avoid busy-polling on archs where cpu_relax() is a no-op

[Will Deacon]

On some architectures (such as arm64), cpu_relax() is effectively a NOP
and so isn't particularly efficient when used in a tight polling loop
such as the CPU state synchronisation in cpuhp_wait_for_sync_state().

Once an incoming CPU has reached the SYNC_STATE_ALIVE state, we know
that it is executing within the kernel and so we can use the more
efficient polling mechanism provided by the atomic_cond_read* API.

Extend the generic implementation of arch_cpuhp_sync_state_poll() to
take the state details as additional parameters and polling using
atomic_cond_read_relaxed() instead of cpu_relax() once we have reached
the alive state. No change on x86.

Signed-off-by: Will Deacon <will at kernel.org>
---
 arch/x86/kernel/smpboot.c  |  2 +-
 include/linux/cpuhotplug.h |  2 +-
 kernel/cpu.c               | 13 +++++++++----
 3 files changed, 11 insertions(+), 6 deletions(-)

diff --git a/arch/x86/kernel/smpboot.c b/arch/x86/kernel/smpboot.c
index ba01a9e919b7..362f85cbdbaf 100644
--- a/arch/x86/kernel/smpboot.c
+++ b/arch/x86/kernel/smpboot.c
@@ -1138,7 +1138,7 @@ void arch_cpuhp_cleanup_dead_cpu(unsigned int cpu)
 		pr_info("CPU %u is now offline\n", cpu);
 }
 
-void arch_cpuhp_sync_state_poll(void)
+void arch_cpuhp_sync_state_poll(atomic_t *st, int old)
 {
 	if (smp_ops.poll_sync_state)
 		smp_ops.poll_sync_state();
diff --git a/include/linux/cpuhotplug.h b/include/linux/cpuhotplug.h
index feb32949aeea..bbcee650155f 100644
--- a/include/linux/cpuhotplug.h
+++ b/include/linux/cpuhotplug.h
@@ -509,7 +509,7 @@ static inline void cpuhp_online_idle(enum cpuhp_state state) { }
 struct task_struct;
 
 void cpuhp_ap_sync_alive(void);
-void arch_cpuhp_sync_state_poll(void);
+void arch_cpuhp_sync_state_poll(atomic_t *st, int old);
 void arch_cpuhp_cleanup_kick_cpu(unsigned int cpu);
 int arch_cpuhp_kick_ap_alive(unsigned int cpu, struct task_struct *tidle);
 bool arch_cpuhp_init_parallel_bringup(void);
diff --git a/kernel/cpu.c b/kernel/cpu.c
index 97a9bfe4edad..d9fe204f02cb 100644
--- a/kernel/cpu.c
+++ b/kernel/cpu.c
@@ -303,7 +303,13 @@ static inline void cpuhp_ap_update_sync_state(enum cpuhp_sync_state state)
 	(void)atomic_xchg(st, state);
 }
 
-void __weak arch_cpuhp_sync_state_poll(void) { cpu_relax(); }
+void __weak arch_cpuhp_sync_state_poll(atomic_t *st, int old)
+{
+	if (old < SYNC_STATE_ALIVE)
+		cpu_relax();
+	else
+		atomic_cond_read_relaxed(st, VAL != old);
+}
 
 static bool cpuhp_wait_for_sync_state(unsigned int cpu, enum cpuhp_sync_state state,
 				      enum cpuhp_sync_state next_state)
@@ -328,7 +334,7 @@ static bool cpuhp_wait_for_sync_state(unsigned int cpu, enum cpuhp_sync_state st
 			return false;
 		} else if (now - start < NSEC_PER_MSEC) {
 			/* Poll for one millisecond */
-			arch_cpuhp_sync_state_poll();
+			arch_cpuhp_sync_state_poll(st, sync);
 		} else {
 			usleep_range(USEC_PER_MSEC, 2 * USEC_PER_MSEC);
 		}
@@ -395,8 +401,7 @@ void cpuhp_ap_sync_alive(void)
 	cpuhp_ap_update_sync_state(SYNC_STATE_ALIVE);
 
 	/* Wait for the control CPU to release it. */
-	while (atomic_read(st) != SYNC_STATE_SHOULD_ONLINE)
-		cpu_relax();
+	atomic_cond_read_acquire(st, VAL == SYNC_STATE_SHOULD_ONLINE);
 }
 
 static bool cpuhp_can_boot_ap(unsigned int cpu)
-- 
2.55.0.979.g7e5102b832-goog