[PATCH v5 10/13] samples: ftrace: Make the direct-call trampolines Tasks Trace readers
Josef Bacik
josef at toxicpanda.com
Mon Sep 21 19:23:29 PDT 2026
The sample direct trampolines are exactly the kind of out-of-line
register_ftrace_direct() user whose lifetime depends on Tasks RCU
waiting for a task inside them: nothing else stops rmmod while a task is
preempted in my_direct_func(). On HAVE_RCU_TRAMPOLINE_READERS
architectures that wait only covers Tasks Trace RCU readers, so give the
samples a small shared header with rcu_read_lock_trace() and
rcu_read_unlock_trace() open-coded as instruction strings for x86-64 and
arm64 -- the same sequences as ftrace_64.S and entry-ftrace.S, using
caller-saved non-argument scratch registers -- and bracket every
call-out with them. The instructions outside the bracket are module
text, covered by ftrace_direct_mark_module(). Other architectures get
empty definitions.
Assisted-by: LLM
Signed-off-by: Josef Bacik <josef at toxicpanda.com>
---
samples/ftrace/ftrace-direct-modify.c | 9 ++
samples/ftrace/ftrace-direct-multi-modify.c | 9 ++
samples/ftrace/ftrace-direct-multi.c | 5 ++
samples/ftrace/ftrace-direct-too.c | 5 ++
samples/ftrace/ftrace-direct.c | 5 ++
samples/ftrace/ftrace-direct.h | 126 ++++++++++++++++++++++++++++
6 files changed, 159 insertions(+)
diff --git a/samples/ftrace/ftrace-direct-modify.c b/samples/ftrace/ftrace-direct-modify.c
index 164d9dd6fd92..937c8d8c2a1b 100644
--- a/samples/ftrace/ftrace-direct-modify.c
+++ b/samples/ftrace/ftrace-direct-modify.c
@@ -2,6 +2,7 @@
#include <linux/module.h>
#include <linux/kthread.h>
#include <linux/ftrace.h>
+#include "ftrace-direct.h"
#if !defined(CONFIG_ARM64) && !defined(CONFIG_PPC32)
#include <asm/asm-offsets.h>
#endif
@@ -73,7 +74,9 @@ asm (
" pushq %rbp\n"
" movq %rsp, %rbp\n"
CALL_DEPTH_ACCOUNT
+ TRACE_RCU_READ_LOCK
" call my_direct_func1\n"
+ TRACE_RCU_READ_UNLOCK
" leave\n"
ASM_RET
" .size my_tramp1, .-my_tramp1\n"
@@ -85,7 +88,9 @@ asm (
" pushq %rbp\n"
" movq %rsp, %rbp\n"
CALL_DEPTH_ACCOUNT
+ TRACE_RCU_READ_LOCK
" call my_direct_func2\n"
+ TRACE_RCU_READ_UNLOCK
" leave\n"
ASM_RET
" .size my_tramp2, .-my_tramp2\n"
@@ -141,11 +146,13 @@ asm (
" .globl my_tramp1\n"
" my_tramp1:"
" hint 34\n" // bti c
+ TRACE_RCU_READ_LOCK
" sub sp, sp, #16\n"
" stp x9, x30, [sp]\n"
" bl my_direct_func1\n"
" ldp x30, x9, [sp]\n"
" add sp, sp, #16\n"
+ TRACE_RCU_READ_UNLOCK
" ret x9\n"
" .size my_tramp1, .-my_tramp1\n"
@@ -153,11 +160,13 @@ asm (
" .globl my_tramp2\n"
" my_tramp2:"
" hint 34\n" // bti c
+ TRACE_RCU_READ_LOCK
" sub sp, sp, #16\n"
" stp x9, x30, [sp]\n"
" bl my_direct_func2\n"
" ldp x30, x9, [sp]\n"
" add sp, sp, #16\n"
+ TRACE_RCU_READ_UNLOCK
" ret x9\n"
" .size my_tramp2, .-my_tramp2\n"
" .popsection\n"
diff --git a/samples/ftrace/ftrace-direct-multi-modify.c b/samples/ftrace/ftrace-direct-multi-modify.c
index b03766c6217b..e12e5c8b83f0 100644
--- a/samples/ftrace/ftrace-direct-multi-modify.c
+++ b/samples/ftrace/ftrace-direct-multi-modify.c
@@ -2,6 +2,7 @@
#include <linux/module.h>
#include <linux/kthread.h>
#include <linux/ftrace.h>
+#include "ftrace-direct.h"
#if !defined(CONFIG_ARM64) && !defined(CONFIG_PPC32)
#include <asm/asm-offsets.h>
#endif
@@ -77,10 +78,12 @@ asm (
" pushq %rbp\n"
" movq %rsp, %rbp\n"
CALL_DEPTH_ACCOUNT
+ TRACE_RCU_READ_LOCK
" pushq %rdi\n"
" movq 8(%rbp), %rdi\n"
" call my_direct_func1\n"
" popq %rdi\n"
+ TRACE_RCU_READ_UNLOCK
" leave\n"
ASM_RET
" .size my_tramp1, .-my_tramp1\n"
@@ -92,10 +95,12 @@ asm (
" pushq %rbp\n"
" movq %rsp, %rbp\n"
CALL_DEPTH_ACCOUNT
+ TRACE_RCU_READ_LOCK
" pushq %rdi\n"
" movq 8(%rbp), %rdi\n"
" call my_direct_func2\n"
" popq %rdi\n"
+ TRACE_RCU_READ_UNLOCK
" leave\n"
ASM_RET
" .size my_tramp2, .-my_tramp2\n"
@@ -154,6 +159,7 @@ asm (
" .globl my_tramp1\n"
" my_tramp1:"
" hint 34\n" // bti c
+ TRACE_RCU_READ_LOCK
" sub sp, sp, #32\n"
" stp x9, x30, [sp]\n"
" str x0, [sp, #16]\n"
@@ -162,6 +168,7 @@ asm (
" ldp x30, x9, [sp]\n"
" ldr x0, [sp, #16]\n"
" add sp, sp, #32\n"
+ TRACE_RCU_READ_UNLOCK
" ret x9\n"
" .size my_tramp1, .-my_tramp1\n"
@@ -169,6 +176,7 @@ asm (
" .globl my_tramp2\n"
" my_tramp2:"
" hint 34\n" // bti c
+ TRACE_RCU_READ_LOCK
" sub sp, sp, #32\n"
" stp x9, x30, [sp]\n"
" str x0, [sp, #16]\n"
@@ -177,6 +185,7 @@ asm (
" ldp x30, x9, [sp]\n"
" ldr x0, [sp, #16]\n"
" add sp, sp, #32\n"
+ TRACE_RCU_READ_UNLOCK
" ret x9\n"
" .size my_tramp2, .-my_tramp2\n"
" .popsection\n"
diff --git a/samples/ftrace/ftrace-direct-multi.c b/samples/ftrace/ftrace-direct-multi.c
index 3fe6ddaf0b69..a970464ed378 100644
--- a/samples/ftrace/ftrace-direct-multi.c
+++ b/samples/ftrace/ftrace-direct-multi.c
@@ -3,6 +3,7 @@
#include <linux/mm.h> /* for handle_mm_fault() */
#include <linux/ftrace.h>
+#include "ftrace-direct.h"
#include <linux/sched/stat.h>
#if !defined(CONFIG_ARM64) && !defined(CONFIG_PPC32)
#include <asm/asm-offsets.h>
@@ -56,10 +57,12 @@ asm (
" pushq %rbp\n"
" movq %rsp, %rbp\n"
CALL_DEPTH_ACCOUNT
+ TRACE_RCU_READ_LOCK
" pushq %rdi\n"
" movq 8(%rbp), %rdi\n"
" call my_direct_func\n"
" popq %rdi\n"
+ TRACE_RCU_READ_UNLOCK
" leave\n"
ASM_RET
" .size my_tramp, .-my_tramp\n"
@@ -101,6 +104,7 @@ asm (
" .globl my_tramp\n"
" my_tramp:"
" hint 34\n" // bti c
+ TRACE_RCU_READ_LOCK
" sub sp, sp, #32\n"
" stp x9, x30, [sp]\n"
" str x0, [sp, #16]\n"
@@ -109,6 +113,7 @@ asm (
" ldp x30, x9, [sp]\n"
" ldr x0, [sp, #16]\n"
" add sp, sp, #32\n"
+ TRACE_RCU_READ_UNLOCK
" ret x9\n"
" .size my_tramp, .-my_tramp\n"
" .popsection\n"
diff --git a/samples/ftrace/ftrace-direct-too.c b/samples/ftrace/ftrace-direct-too.c
index bf2411aa6fd7..abc098c2ab7a 100644
--- a/samples/ftrace/ftrace-direct-too.c
+++ b/samples/ftrace/ftrace-direct-too.c
@@ -3,6 +3,7 @@
#include <linux/mm.h> /* for handle_mm_fault() */
#include <linux/ftrace.h>
+#include "ftrace-direct.h"
#if !defined(CONFIG_ARM64) && !defined(CONFIG_PPC32)
#include <asm/asm-offsets.h>
#endif
@@ -61,6 +62,7 @@ asm (
" pushq %rbp\n"
" movq %rsp, %rbp\n"
CALL_DEPTH_ACCOUNT
+ TRACE_RCU_READ_LOCK
" pushq %rdi\n"
" pushq %rsi\n"
" pushq %rdx\n"
@@ -70,6 +72,7 @@ asm (
" popq %rdx\n"
" popq %rsi\n"
" popq %rdi\n"
+ TRACE_RCU_READ_UNLOCK
" leave\n"
ASM_RET
" .size my_tramp, .-my_tramp\n"
@@ -110,6 +113,7 @@ asm (
" .globl my_tramp\n"
" my_tramp:"
" hint 34\n" // bti c
+ TRACE_RCU_READ_LOCK
" sub sp, sp, #48\n"
" stp x9, x30, [sp]\n"
" stp x0, x1, [sp, #16]\n"
@@ -119,6 +123,7 @@ asm (
" ldp x0, x1, [sp, #16]\n"
" ldp x2, x3, [sp, #32]\n"
" add sp, sp, #48\n"
+ TRACE_RCU_READ_UNLOCK
" ret x9\n"
" .size my_tramp, .-my_tramp\n"
" .popsection\n"
diff --git a/samples/ftrace/ftrace-direct.c b/samples/ftrace/ftrace-direct.c
index 5368c8c39cbb..99b65ad2fccc 100644
--- a/samples/ftrace/ftrace-direct.c
+++ b/samples/ftrace/ftrace-direct.c
@@ -3,6 +3,7 @@
#include <linux/sched.h> /* for wake_up_process() */
#include <linux/ftrace.h>
+#include "ftrace-direct.h"
#if !defined(CONFIG_ARM64) && !defined(CONFIG_PPC32)
#include <asm/asm-offsets.h>
#endif
@@ -54,9 +55,11 @@ asm (
" pushq %rbp\n"
" movq %rsp, %rbp\n"
CALL_DEPTH_ACCOUNT
+ TRACE_RCU_READ_LOCK
" pushq %rdi\n"
" call my_direct_func\n"
" popq %rdi\n"
+ TRACE_RCU_READ_UNLOCK
" leave\n"
ASM_RET
" .size my_tramp, .-my_tramp\n"
@@ -97,6 +100,7 @@ asm (
" .globl my_tramp\n"
" my_tramp:"
" hint 34\n" // bti c
+ TRACE_RCU_READ_LOCK
" sub sp, sp, #32\n"
" stp x9, x30, [sp]\n"
" str x0, [sp, #16]\n"
@@ -104,6 +108,7 @@ asm (
" ldp x30, x9, [sp]\n"
" ldr x0, [sp, #16]\n"
" add sp, sp, #32\n"
+ TRACE_RCU_READ_UNLOCK
" ret x9\n"
" .size my_tramp, .-my_tramp\n"
" .popsection\n"
diff --git a/samples/ftrace/ftrace-direct.h b/samples/ftrace/ftrace-direct.h
new file mode 100644
index 000000000000..726f67048ff5
--- /dev/null
+++ b/samples/ftrace/ftrace-direct.h
@@ -0,0 +1,126 @@
+/* SPDX-License-Identifier: GPL-2.0-only */
+#ifndef _SAMPLES_FTRACE_DIRECT_H
+#define _SAMPLES_FTRACE_DIRECT_H
+
+#include <linux/stringify.h>
+
+/*
+ * A direct-call trampoline is entered with no lock, refcount or RCU marker
+ * held; only Tasks RCU keeps it (and, for a module, its text) alive while a
+ * task is inside it or preempted in something it called. On architectures
+ * that select HAVE_RCU_TRAMPOLINE_READERS, Tasks RCU only waits for such a
+ * task while it is a Tasks Trace RCU reader, so the trampoline must enter one
+ * before calling out and leave it afterwards, exactly like the ftrace and BPF
+ * trampolines do. See register_ftrace_direct(). The instructions before the
+ * lock and after the unlock are covered by ftrace_direct_mark_module().
+ *
+ * These are rcu_read_lock_trace() / rcu_read_unlock_trace() open-coded as
+ * instruction strings for use inside the samples' asm() trampolines, after
+ * the versions in arch/x86/kernel/ftrace_64.S and
+ * arch/arm64/kernel/entry-ftrace.S. The scratch registers are caller-saved
+ * and not argument registers, so they are dead on entry to and exit from an
+ * fentry trampoline; the flags are clobbered.
+ *
+ * The generated asm-offsets.h is only pulled in on the architectures that need
+ * it here: it is not generally safe to include from C (e.g. PPC32's TASK_SIZE
+ * and arm64's TRAMP_VALIAS clash with the C definitions).
+ */
+#if defined(CONFIG_TASKS_RCU_TRAMPOLINE_READERS) && defined(CONFIG_X86_64)
+
+#include <asm/asm-offsets.h>
+
+#ifndef CONFIG_TASKS_TRACE_RCU_NO_MB
+#define TRACE_RCU_MB " lock addl $0, -4(%rsp)\n"
+#else
+#define TRACE_RCU_MB
+#endif
+
+#define TRACE_RCU_READ_LOCK \
+ " movq %gs:current_task(%rip), %r11\n" \
+ " movl " __stringify(TASK_trc_reader_nesting) "(%r11), %r10d\n" \
+ " incl " __stringify(TASK_trc_reader_nesting) "(%r11)\n" \
+ " testl %r10d, %r10d\n" \
+ " jnz 771f\n" \
+ " movq rcu_tasks_trace_srcu_struct+" __stringify(SRCU_srcu_ctrp) "(%rip), %r10\n" \
+ " incq %gs:" __stringify(SRCU_CTR_srcu_locks) "(%r10)\n" \
+ " movq %r10, " __stringify(TASK_trc_reader_scp) "(%r11)\n" \
+ TRACE_RCU_MB \
+ "771:\n"
+
+#define TRACE_RCU_READ_UNLOCK \
+ " movq %gs:current_task(%rip), %r11\n" \
+ " movl " __stringify(TASK_trc_reader_nesting) "(%r11), %r10d\n" \
+ " subl $1, %r10d\n" \
+ " jnz 772f\n" \
+ " movq " __stringify(TASK_trc_reader_scp) "(%r11), %r10\n" \
+ " movl $0, " __stringify(TASK_trc_reader_nesting) "(%r11)\n" \
+ TRACE_RCU_MB \
+ " incq %gs:" __stringify(SRCU_CTR_srcu_unlocks) "(%r10)\n" \
+ " jmp 773f\n" \
+ "772: movl %r10d, " __stringify(TASK_trc_reader_nesting) "(%r11)\n" \
+ "773:\n"
+
+#elif defined(CONFIG_TASKS_RCU_TRAMPOLINE_READERS) && defined(CONFIG_ARM64)
+
+#include <asm/alternative-macros.h>
+#include <asm/cpucaps.h>
+/* arm64's asm-offsets.h redefines TRAMP_VALIAS from <asm/fixmap.h>. */
+#pragma push_macro("TRAMP_VALIAS")
+#undef TRAMP_VALIAS
+#include <asm/asm-offsets.h>
+#pragma pop_macro("TRAMP_VALIAS")
+
+#ifndef CONFIG_TASKS_TRACE_RCU_NO_MB
+#define TRACE_RCU_MB " dmb ish\n"
+#else
+#define TRACE_RCU_MB
+#endif
+
+#define TRACE_RCU_SRCU_CTRP "rcu_tasks_trace_srcu_struct+" __stringify(SRCU_SRCU_CTRP)
+
+/* x14 = this CPU's offset; then atomically increment the long at x14 + \areg */
+#define TRACE_RCU_PERCPU_INC(areg) \
+ ALTERNATIVE(" mrs x14, tpidr_el1\n", " mrs x14, tpidr_el2\n", \
+ ARM64_HAS_VIRT_HOST_EXTN) \
+ " add x14, x14, " areg "\n" \
+ "778: ldxr x15, [x14]\n" \
+ " add x15, x15, #1\n" \
+ " stxr w16, x15, [x14]\n" \
+ " cbnz w16, 778b\n"
+
+#define TRACE_RCU_READ_LOCK \
+ " mrs x12, sp_el0\n" \
+ " ldr w13, [x12, #" __stringify(TSK_TRC_READER_NESTING) "]\n" \
+ " add w14, w13, #1\n" \
+ " str w14, [x12, #" __stringify(TSK_TRC_READER_NESTING) "]\n" \
+ " cbnz w13, 771f\n" \
+ " adrp x13, " TRACE_RCU_SRCU_CTRP "\n" \
+ " ldr x13, [x13, #:lo12:" TRACE_RCU_SRCU_CTRP "]\n" \
+ " str x13, [x12, #" __stringify(TSK_TRC_READER_SCP) "]\n" \
+ " add x13, x13, #" __stringify(SRCU_CTR_SRCU_LOCKS) "\n" \
+ TRACE_RCU_PERCPU_INC("x13") \
+ TRACE_RCU_MB \
+ "771:\n"
+
+#define TRACE_RCU_READ_UNLOCK \
+ " mrs x12, sp_el0\n" \
+ " ldr w13, [x12, #" __stringify(TSK_TRC_READER_NESTING) "]\n" \
+ " subs w13, w13, #1\n" \
+ " b.ne 772f\n" \
+ " ldr x13, [x12, #" __stringify(TSK_TRC_READER_SCP) "]\n" \
+ " str wzr, [x12, #" __stringify(TSK_TRC_READER_NESTING) "]\n" \
+ TRACE_RCU_MB \
+ " add x13, x13, #" __stringify(SRCU_CTR_SRCU_UNLOCKS) "\n" \
+ TRACE_RCU_PERCPU_INC("x13") \
+ " b 773f\n" \
+ "772: str w13, [x12, #" __stringify(TSK_TRC_READER_NESTING) "]\n" \
+ "773:\n"
+
+#else
+
+#define TRACE_RCU_READ_LOCK
+#define TRACE_RCU_READ_UNLOCK
+
+#endif
+
+#endif /* _SAMPLES_FTRACE_DIRECT_H */
--
2.55.0
More information about the linux-arm-kernel
mailing list