[PATCH v5 08/13] arm64: ftrace: Take a Tasks Trace reader around ftrace_caller's call-out
Josef Bacik
josef at toxicpanda.com
Mon Sep 21 19:23:27 PDT 2026
For HAVE_RCU_TRAMPOLINE_READERS the ftrace trampoline must be a Tasks
Trace RCU reader while it calls out, since that is what
synchronize_rcu_tasks() will wait for before ftrace_shutdown() frees an
ftrace_ops (or, with CALL_OPS, lets its owner free it) under a task
preempted in the callback.
Open-code rcu_read_lock_trace() and rcu_read_unlock_trace() around the
call to ops->func in ftrace_caller: bump current->trc_reader_nesting via
sp_el0 and, for the outermost reader, do the SRCU-fast per-CPU increment
on rcu_tasks_trace_srcu_struct and stash the counter pointer in
current->trc_reader_scp, as the C inlines do (including the dmb when
CONFIG_TASKS_TRACE_RCU_NO_MB is not set). The per-CPU increment is an
LL/SC add on this CPU's counter; being migrated between reading the
per-CPU offset and the store-exclusive only means another CPU's counter
is incremented atomically instead, which SRCU sums over anyway.
x12-x16 are free at both points.
arm64 has no return thunks and, with CALL_OPS, no dynamic ftrace
trampolines, but ftrace_caller itself carries the ops pointer in x11
from before the reader is entered and a direct-call BPF trampoline
address in x17 until the final br/ret after it is left, so mark the end
of the static trampoline text and provide arch_rcu_tasks_trampoline_text()
covering [ftrace_caller, ftrace_static_tramp_end).
Built only under CONFIG_TASKS_RCU_TRAMPOLINE_READERS, which arm64 does
not enable until a later patch.
Assisted-by: LLM
Signed-off-by: Josef Bacik <josef at toxicpanda.com>
---
arch/arm64/kernel/asm-offsets.c | 8 +++++
arch/arm64/kernel/entry-ftrace.S | 74 ++++++++++++++++++++++++++++++++++++++++
arch/arm64/kernel/ftrace.c | 20 +++++++++++
3 files changed, 102 insertions(+)
diff --git a/arch/arm64/kernel/asm-offsets.c b/arch/arm64/kernel/asm-offsets.c
index 9c853ed3ceab..f6a8fb1f9b43 100644
--- a/arch/arm64/kernel/asm-offsets.c
+++ b/arch/arm64/kernel/asm-offsets.c
@@ -10,6 +10,7 @@
#include <linux/arm_sdei.h>
#include <linux/sched.h>
+#include <linux/srcu.h>
#include <linux/ftrace.h>
#include <linux/kexec.h>
#include <linux/mm.h>
@@ -39,6 +40,13 @@ int main(void)
DEFINE(TSK_STACK, offsetof(struct task_struct, stack));
#ifdef CONFIG_STACKPROTECTOR
DEFINE(TSK_STACK_CANARY, offsetof(struct task_struct, stack_canary));
+#endif
+#ifdef CONFIG_TASKS_RCU_TRAMPOLINE_READERS
+ DEFINE(TSK_TRC_READER_NESTING, offsetof(struct task_struct, trc_reader_nesting));
+ DEFINE(TSK_TRC_READER_SCP, offsetof(struct task_struct, trc_reader_scp));
+ DEFINE(SRCU_SRCU_CTRP, offsetof(struct srcu_struct, srcu_ctrp));
+ DEFINE(SRCU_CTR_SRCU_LOCKS, offsetof(struct srcu_ctr, srcu_locks));
+ DEFINE(SRCU_CTR_SRCU_UNLOCKS, offsetof(struct srcu_ctr, srcu_unlocks));
#endif
BLANK();
DEFINE(THREAD_CPU_CONTEXT, offsetof(struct task_struct, thread.cpu_context));
diff --git a/arch/arm64/kernel/entry-ftrace.S b/arch/arm64/kernel/entry-ftrace.S
index 025140caafe7..fc2805eb9e15 100644
--- a/arch/arm64/kernel/entry-ftrace.S
+++ b/arch/arm64/kernel/entry-ftrace.S
@@ -14,6 +14,72 @@
#include <asm/insn.h>
#ifdef CONFIG_DYNAMIC_FTRACE_WITH_ARGS
+/*
+ * Open-coded rcu_read_lock_trace() / rcu_read_unlock_trace(), see
+ * include/linux/rcupdate_trace.h and CONFIG_HAVE_RCU_TRAMPOLINE_READERS. The
+ * whole of ftrace_caller is treated as trampoline text by the irq-exit check
+ * (see arch_rcu_tasks_trampoline_text()), so these only need to bracket the
+ * call out to ops->func; everything before the lock and after the unlock,
+ * including the direct-call tails that carry a BPF trampoline address in x17,
+ * is covered by that.
+ *
+ * The SRCU-fast per-CPU increment is done LL/SC on this CPU's counter; being
+ * migrated between reading the per-CPU offset and the store-exclusive only
+ * means another CPU's counter is (atomically) incremented, which SRCU sums
+ * over anyway. Ordering between the nesting count and the scp stash only
+ * matters against interrupts on this CPU, which observe program order.
+ * Clobbers x12-x16 and the flags.
+ */
+ .macro trace_rcu_srcu_inc, addr:req, tmp:req, wtmp2:req
+8888: ldxr \tmp, [\addr]
+ add \tmp, \tmp, #1
+ stxr \wtmp2, \tmp, [\addr]
+ cbnz \wtmp2, 8888b
+ .endm
+
+ .macro trace_rcu_read_lock
+#ifdef CONFIG_TASKS_RCU_TRAMPOLINE_READERS
+ mrs x12, sp_el0 // current
+ ldr w13, [x12, #TSK_TRC_READER_NESTING]
+ add w14, w13, #1
+ str w14, [x12, #TSK_TRC_READER_NESTING]
+ cbnz w13, .Ltrl_nested\@ // interrupted a reader: done
+ ldr_l x13, rcu_tasks_trace_srcu_struct + SRCU_SRCU_CTRP
+ str x13, [x12, #TSK_TRC_READER_SCP]
+ get_this_cpu_offset x14
+ add x14, x14, x13
+ add x14, x14, #SRCU_CTR_SRCU_LOCKS
+ trace_rcu_srcu_inc x14, x15, w16
+#ifndef CONFIG_TASKS_TRACE_RCU_NO_MB
+ dmb ish
+#endif
+.Ltrl_nested\@:
+#endif
+ .endm
+
+ .macro trace_rcu_read_unlock
+#ifdef CONFIG_TASKS_RCU_TRAMPOLINE_READERS
+ mrs x12, sp_el0 // current
+ ldr w13, [x12, #TSK_TRC_READER_NESTING]
+ subs w13, w13, #1
+ b.ne .Ltru_nested\@
+ /* Outermost: pick up scp before an interrupt can see nesting == 0. */
+ ldr x14, [x12, #TSK_TRC_READER_SCP]
+ str wzr, [x12, #TSK_TRC_READER_NESTING]
+#ifndef CONFIG_TASKS_TRACE_RCU_NO_MB
+ dmb ish
+#endif
+ get_this_cpu_offset x15
+ add x14, x14, x15
+ add x14, x14, #SRCU_CTR_SRCU_UNLOCKS
+ trace_rcu_srcu_inc x14, x15, w16
+ b .Ltru_done\@
+.Ltru_nested\@:
+ str w13, [x12, #TSK_TRC_READER_NESTING]
+.Ltru_done\@:
+#endif
+ .endm
+
/*
* Due to -fpatchable-function-entry=2, the compiler has placed two NOPs before
* the regular function prologue. For an enabled callsite, ftrace_init_nop() and
@@ -94,6 +160,8 @@ SYM_CODE_START(ftrace_caller)
stp x29, x30, [sp, #FREGS_SIZE]
add x29, sp, #FREGS_SIZE
+ trace_rcu_read_lock
+
/* Prepare arguments for the tracer func */
sub x0, x30, #AARCH64_INSN_SIZE // ip (callsite's BL insn)
mov x1, x9 // parent_ip (callsite's LR)
@@ -111,6 +179,8 @@ SYM_INNER_LABEL(ftrace_call, SYM_L_GLOBAL)
bl ftrace_stub // func(ip, parent_ip, op, regs)
#endif
+ trace_rcu_read_unlock
+
/*
* At the callsite x0-x8 and x19-x30 were live. Any C code will have preserved
* x19-x29 per the AAPCS, and we created frame records upon entry, so we need
@@ -178,6 +248,10 @@ SYM_CODE_START(ftrace_stub_direct_tramp)
SYM_CODE_END(ftrace_stub_direct_tramp)
#endif /* CONFIG_DYNAMIC_FTRACE_WITH_DIRECT_CALLS */
+/* End of [ftrace_caller, ...) for arch_rcu_tasks_trampoline_text(). */
+SYM_CODE_START(ftrace_static_tramp_end)
+SYM_CODE_END(ftrace_static_tramp_end)
+
#else /* CONFIG_DYNAMIC_FTRACE_WITH_ARGS */
/*
diff --git a/arch/arm64/kernel/ftrace.c b/arch/arm64/kernel/ftrace.c
index e1a3c0b3a051..5f4193f15cd9 100644
--- a/arch/arm64/kernel/ftrace.c
+++ b/arch/arm64/kernel/ftrace.c
@@ -17,6 +17,26 @@
#include <asm/insn.h>
#include <asm/text-patching.h>
+#ifdef CONFIG_TASKS_RCU_TRAMPOLINE_READERS
+extern void ftrace_static_tramp_end(void);
+
+/* The SRCU-fast increments in entry-ftrace.S are the this_cpu_inc() form. */
+static_assert(!IS_ENABLED(CONFIG_NEED_SRCU_NMI_SAFE));
+
+/*
+ * See rcu_tasks_trampoline_text(). ftrace_caller and ftrace_stub_direct_tramp
+ * are core kernel text but must be treated as trampolines: a task interrupted
+ * in them outside the Tasks Trace reader may be carrying an ops pointer (x11)
+ * or a direct-call BPF trampoline address (x17) whose lifetime is guarded only
+ * by Tasks RCU.
+ */
+bool arch_rcu_tasks_trampoline_text(unsigned long ip)
+{
+ return ip >= (unsigned long)ftrace_caller &&
+ ip < (unsigned long)ftrace_static_tramp_end;
+}
+#endif
+
#ifdef CONFIG_DYNAMIC_FTRACE_WITH_ARGS
struct fregs_offset {
const char *name;
--
2.55.0
More information about the linux-arm-kernel
mailing list