[PATCH bpf-next v2 0/4] Add ftrace direct call for arm64
Xu Kuohai
xukuohai at huawei.com
Fri Oct 7 03:13:36 PDT 2022
On 10/7/2022 12:29 AM, Steven Rostedt wrote:
> On Thu, 6 Oct 2022 18:19:12 +0200
> Florent Revest <revest at chromium.org> wrote:
>
>> Sure, we can give this a try, I'll work on a macro that generates the
>> 7 callbacks and we can check how much that helps. My belief right now
>> is that ftrace's iteration over all ops on arm64 is where we lose most
>> time but now that we have numbers it's pretty easy to check hypothesis
>> :)
>
> Ah, I forgot that's what Mark's code is doing. But yes, that needs to be
> fixed first. I forget that arm64 doesn't have the dedicated trampolines yet.
>
> So, let's hold off until that is complete.
>
> -- Steve
>
> .
Here is the perf data I captured.
1. perf report
99.94% 0.00% ld-linux-aarch6 bench [.] trigger_producer
|
---trigger_producer
|
|--98.04%--syscall
| |
| --81.35%--el0t_64_sync
| el0t_64_sync_handler
| el0_svc
| do_el0_svc
| |
| |--80.75%--el0_svc_common.constprop.0
| | |
| | |--49.70%--invoke_syscall
| | | |
| | | --46.66%--__arm64_sys_getpgid
| | | |
| | | |--40.73%--ftrace_call
| | | | |
| | | | |--38.71%--ftrace_ops_list_func
| | | | | |
| | | | | |--25.06%--fprobe_handler
| | | | | | |
| | | | | | |--13.20%--bpf_fprobe_entry
| | | | | | | |
| | | | | | | --11.47%--call_bpf_prog.isra.0
| | | | | | | |
| | | | | | | |--4.08%--__bpf_prog_exit
| | | | | | | | |
| | | | | | | | --0.87%--migrate_enable
| | | | | | | |
| | | | | | | |--2.46%--__bpf_prog_enter
| | | | | | | |
| | | | | | | --2.18%--bpf_prog_21856463590f61f1_bench_trigger_fentry
| | | | | | |
| | | | | | |--8.68%--rethook_trampoline_handler
| | | | | | |
| | | | | | --1.59%--rethook_try_get
| | | | | | |
| | | | | | --0.58%--rcu_is_watching
| | | | | |
| | | | | |--6.65%--rethook_trampoline_handler
| | | | | |
| | | | | --0.77%--rethook_recycle
| | | | |
| | | | --1.74%--hash_contains_ip.isra.0
| | | |
| | | --3.62%--find_task_by_vpid
| | | |
| | | --2.75%--idr_find
| | | |
| | | --2.17%--__radix_tree_lookup
| | |
| | --1.30%--ftrace_caller
| |
| --0.60%--invoke_syscall
|
|--0.88%--0xffffb2807594
|
--0.87%--syscall at plt
2. perf annotate
2.1 ftrace_caller
: 39 SYM_CODE_START(ftrace_caller)
: 40 bti c
0.00 : ffff80000802e0c4: bti c
:
: 39 /* Save original SP */
: 40 mov x10, sp
0.00 : ffff80000802e0c8: mov x10, sp
:
: 42 /* Make room for pt_regs, plus two frame records */
: 43 sub sp, sp, #(FREGS_SIZE + 32)
0.00 : ffff80000802e0cc: sub sp, sp, #0x90
:
: 45 /* Save function arguments */
: 46 stp x0, x1, [sp, #FREGS_X0]
0.00 : ffff80000802e0d0: stp x0, x1, [sp]
: 45 stp x2, x3, [sp, #FREGS_X2]
0.00 : ffff80000802e0d4: stp x2, x3, [sp, #16]
: 46 stp x4, x5, [sp, #FREGS_X4]
16.67 : ffff80000802e0d8: stp x4, x5, [sp, #32] // entry-ftrace.S:46
: 47 stp x6, x7, [sp, #FREGS_X6]
8.33 : ffff80000802e0dc: stp x6, x7, [sp, #48] // entry-ftrace.S:47
: 48 str x8, [sp, #FREGS_X8]
0.00 : ffff80000802e0e0: str x8, [sp, #64]
:
: 52 /* Save the callsite's FP, LR, SP */
: 53 str x29, [sp, #FREGS_FP]
8.33 : ffff80000802e0e4: str x29, [sp, #80] // entry-ftrace.S:51
: 52 str x9, [sp, #FREGS_LR]
8.33 : ffff80000802e0e8: str x9, [sp, #88] // entry-ftrace.S:52
: 53 str x10, [sp, #FREGS_SP]
0.00 : ffff80000802e0ec: str x10, [sp, #96]
:
: 57 /* Save the PC after the ftrace callsite */
: 58 str x30, [sp, #FREGS_PC]
16.67 : ffff80000802e0f0: str x30, [sp, #104] // entry-ftrace.S:56
:
: 60 /* Create a frame record for the callsite above the ftrace regs */
: 61 stp x29, x9, [sp, #FREGS_SIZE + 16]
16.67 : ffff80000802e0f4: stp x29, x9, [sp, #128] // entry-ftrace.S:59
: 60 add x29, sp, #FREGS_SIZE + 16
0.00 : ffff80000802e0f8: add x29, sp, #0x80
:
: 64 /* Create our frame record above the ftrace regs */
: 65 stp x29, x30, [sp, #FREGS_SIZE]
16.67 : ffff80000802e0fc: stp x29, x30, [sp, #112] // entry-ftrace.S:63
: 64 add x29, sp, #FREGS_SIZE
0.00 : ffff80000802e100: add x29, sp, #0x70
:
: 67 sub x0, x30, #AARCH64_INSN_SIZE // ip (callsite's BL insn)
0.00 : ffff80000802e104: sub x0, x30, #0x4
: 67 mov x1, x9 // parent_ip (callsite's LR)
0.00 : ffff80000802e108: mov x1, x9
: 68 ldr_l x2, function_trace_op // op
0.00 : ffff80000802e10c: adrp x2, ffff800009638000 <folio_wait_table+0x14c0>
0.00 : ffff80000802e110: ldr x2, [x2, #3320]
: 69 mov x3, sp // regs
0.00 : ffff80000802e114: mov x3, sp
:
: 72 ffff80000802e118 <ftrace_call>:
:
: 73 SYM_INNER_LABEL(ftrace_call, SYM_L_GLOBAL)
: 74 bl ftrace_stub
0.00 : ffff80000802e118: bl ffff80000802e144 <ftrace_stub>
: 80 * At the callsite x0-x8 and x19-x30 were live. Any C code will have preserved
: 81 * x19-x29 per the AAPCS, and we created frame records upon entry, so we need
: 82 * to restore x0-x8, x29, and x30.
: 83 */
: 84 /* Restore function arguments */
: 85 ldp x0, x1, [sp, #FREGS_X0]
8.33 : ffff80000802e11c: ldp x0, x1, [sp] // entry-ftrace.S:80
: 81 ldp x2, x3, [sp, #FREGS_X2]
0.00 : ffff80000802e120: ldp x2, x3, [sp, #16]
: 82 ldp x4, x5, [sp, #FREGS_X4]
0.00 : ffff80000802e124: ldp x4, x5, [sp, #32]
: 83 ldp x6, x7, [sp, #FREGS_X6]
0.00 : ffff80000802e128: ldp x6, x7, [sp, #48]
: 84 ldr x8, [sp, #FREGS_X8]
0.00 : ffff80000802e12c: ldr x8, [sp, #64]
:
: 88 /* Restore the callsite's FP, LR, PC */
: 89 ldr x29, [sp, #FREGS_FP]
0.00 : ffff80000802e130: ldr x29, [sp, #80]
: 88 ldr x30, [sp, #FREGS_LR]
0.00 : ffff80000802e134: ldr x30, [sp, #88]
: 89 ldr x9, [sp, #FREGS_PC]
0.00 : ffff80000802e138: ldr x9, [sp, #104]
:
: 93 /* Restore the callsite's SP */
: 94 add sp, sp, #FREGS_SIZE + 32
0.00 : ffff80000802e13c: add sp, sp, #0x90
:
: 95 ret x9
0.00 : ffff80000802e140: ret x9
2.2 arch_ftrace_ops_list_func
: 7554 void arch_ftrace_ops_list_func(unsigned long ip, unsigned long parent_ip,
: 7555 struct ftrace_ops *op, struct ftrace_regs *fregs)
: 7556 {
0.00 : ffff80000815bdf0: paciasp
4.65 : ffff80000815bdf4: stp x29, x30, [sp, #-144]! // ftrace.c:7551
0.00 : ffff80000815bdf8: mrs x2, sp_el0
0.00 : ffff80000815bdfc: mov x29, sp
2.32 : ffff80000815be00: stp x19, x20, [sp, #16]
0.00 : ffff80000815be04: mov x20, x1
: 7563 trace_test_and_set_recursion():
: 147 int start)
: 148 {
: 149 unsigned int val = READ_ONCE(current->trace_recursion);
: 150 int bit;
:
: 152 bit = trace_get_context_bit() + start;
0.00 : ffff80000815be08: mov w5, #0x8 // #8
: 154 arch_ftrace_ops_list_func():
0.00 : ffff80000815be0c: stp x21, x22, [sp, #32]
0.00 : ffff80000815be10: mov x21, x3
2.32 : ffff80000815be14: stp x23, x24, [sp, #48]
0.00 : ffff80000815be18: mov x23, x0
0.00 : ffff80000815be1c: ldr x4, [x2, #1168]
2.32 : ffff80000815be20: str x4, [sp, #136]
0.00 : ffff80000815be24: mov x4, #0x0 // #0
: 7558 trace_test_and_set_recursion():
: 148 if (unlikely(val & (1 << bit))) {
0.00 : ffff80000815be28: mov w2, #0x1 // #1
: 150 get_current():
: 19 */
: 20 static __always_inline struct task_struct *get_current(void)
: 21 {
: 22 unsigned long sp_el0;
:
: 24 asm ("mrs %0, sp_el0" : "=r" (sp_el0));
0.00 : ffff80000815be2c: mrs x4, sp_el0
: 26 trace_test_and_set_recursion():
: 144 unsigned int val = READ_ONCE(current->trace_recursion);
0.00 : ffff80000815be30: ldr x7, [x4, #2520]
: 146 preempt_count():
: 13 #define PREEMPT_NEED_RESCHED BIT(32)
: 14 #define PREEMPT_ENABLED (PREEMPT_NEED_RESCHED)
:
: 16 static inline int preempt_count(void)
: 17 {
: 18 return READ_ONCE(current_thread_info()->preempt.count);
0.00 : ffff80000815be34: ldr w6, [x4, #8]
: 20 interrupt_context_level():
: 94 static __always_inline unsigned char interrupt_context_level(void)
: 95 {
: 96 unsigned long pc = preempt_count();
: 97 unsigned char level = 0;
:
: 99 level += !!(pc & (NMI_MASK));
0.00 : ffff80000815be38: tst w6, #0xf00000
: 96 level += !!(pc & (NMI_MASK | HARDIRQ_MASK));
: 97 level += !!(pc & (NMI_MASK | HARDIRQ_MASK | SOFTIRQ_OFFSET));
0.00 : ffff80000815be3c: and w1, w6, #0xffff00
: 94 level += !!(pc & (NMI_MASK));
0.00 : ffff80000815be40: cset w4, ne // ne = any
: 96 level += !!(pc & (NMI_MASK | HARDIRQ_MASK | SOFTIRQ_OFFSET));
0.00 : ffff80000815be44: and w1, w1, #0xffff01ff
: 95 level += !!(pc & (NMI_MASK | HARDIRQ_MASK));
0.00 : ffff80000815be48: tst w6, #0xff0000
0.00 : ffff80000815be4c: cinc w4, w4, ne // ne = any
: 96 level += !!(pc & (NMI_MASK | HARDIRQ_MASK | SOFTIRQ_OFFSET));
0.00 : ffff80000815be50: cmp w1, #0x0
: 98 trace_get_context_bit():
: 121 return TRACE_CTX_NORMAL - bit;
0.00 : ffff80000815be54: cinc w4, w4, ne // ne = any
: 123 trace_test_and_set_recursion():
: 147 bit = trace_get_context_bit() + start;
0.00 : ffff80000815be58: sub w5, w5, w4
: 148 if (unlikely(val & (1 << bit))) {
0.00 : ffff80000815be5c: lsl w2, w2, w5
0.00 : ffff80000815be60: tst w2, w7
0.00 : ffff80000815be64: b.ne ffff80000815bf84 <arch_ftrace_ops_list_func+0x194> // b.any
: 152 trace_clear_recursion():
: 180 */
: 181 static __always_inline void trace_clear_recursion(int bit)
: 182 {
: 183 preempt_enable_notrace();
: 184 barrier();
: 185 trace_recursion_clear(bit);
4.65 : ffff80000815be68: mvn w22, w2 // trace_recursion.h:180
0.00 : ffff80000815be6c: str x25, [sp, #64]
0.00 : ffff80000815be70: sxtw x22, w22
: 189 trace_test_and_set_recursion():
: 165 current->trace_recursion = val;
0.00 : ffff80000815be74: orr w2, w2, w7
: 167 get_current():
0.00 : ffff80000815be78: mrs x4, sp_el0
: 20 trace_test_and_set_recursion():
2.32 : ffff80000815be7c: str x2, [x4, #2520] // trace_recursion.h:165
: 166 __preempt_count_add():
: 47 return !current_thread_info()->preempt.need_resched;
: 48 }
:
: 50 static inline void __preempt_count_add(int val)
: 51 {
: 52 u32 pc = READ_ONCE(current_thread_info()->preempt.count);
0.00 : ffff80000815be80: ldr w1, [x4, #8]
: 48 pc += val;
0.00 : ffff80000815be84: add w1, w1, #0x1
: 49 WRITE_ONCE(current_thread_info()->preempt.count, pc);
2.32 : ffff80000815be88: str w1, [x4, #8] // preempt.h:49
: 51 __ftrace_ops_list_func():
: 7506 do_for_each_ftrace_op(op, ftrace_ops_list) {
0.00 : ffff80000815be8c: adrp x0, ffff800009638000 <folio_wait_table+0x14c0>
0.00 : ffff80000815be90: add x25, x0, #0xc28
: 7527 } while_for_each_ftrace_op(op);
0.00 : ffff80000815be94: add x24, x25, #0x8
: 7506 do_for_each_ftrace_op(op, ftrace_ops_list) {
0.00 : ffff80000815be98: ldr x19, [x0, #3112]
: 7508 if (op->flags & FTRACE_OPS_FL_STUB)
4.72 : ffff80000815be9c: ldr x0, [x19, #16] // ftrace.c:7508
0.00 : ffff80000815bea0: tbnz w0, #5, ffff80000815bef8 <arch_ftrace_ops_list_func+0x108>
: 7519 if ((!(op->flags & FTRACE_OPS_FL_RCU) || rcu_is_watching()) &&
2.32 : ffff80000815bea4: tbnz w0, #14, ffff80000815bf74 <arch_ftrace_ops_list_func+0x184> // ftrace.c:7519
: 7521 ftrace_ops_test():
: 1486 rcu_assign_pointer(hash.filter_hash, ops->func_hash->filter_hash);
2.32 : ffff80000815bea8: ldr x0, [x19, #88] // ftrace.c:1486
0.00 : ffff80000815beac: add x1, sp, #0x60
0.00 : ffff80000815beb0: ldr x0, [x0, #8]
0.00 : ffff80000815beb4: stlr x0, [x1]
: 1487 rcu_assign_pointer(hash.notrace_hash, ops->func_hash->notrace_hash);
0.00 : ffff80000815beb8: ldr x0, [x19, #88]
0.00 : ffff80000815bebc: add x1, sp, #0x58
0.00 : ffff80000815bec0: ldr x0, [x0]
2.32 : ffff80000815bec4: stlr x0, [x1] // ftrace.c:1487
: 1489 if (hash_contains_ip(ip, &hash))
44.15 : ffff80000815bec8: ldp x1, x2, [sp, #88] // ftrace.c:1489
0.00 : ffff80000815becc: mov x0, x23
0.00 : ffff80000815bed0: bl ffff80000815b530 <hash_contains_ip.isra.0>
0.00 : ffff80000815bed4: tst w0, #0xff
0.00 : ffff80000815bed8: b.eq ffff80000815bef8 <arch_ftrace_ops_list_func+0x108> // b.none
: 1495 __ftrace_ops_list_func():
: 7521 if (FTRACE_WARN_ON(!op->func)) {
0.00 : ffff80000815bedc: ldr x4, [x19]
0.00 : ffff80000815bee0: cbz x4, ffff80000815bfa0 <arch_ftrace_ops_list_func+0x1b0>
: 7525 op->func(ip, parent_ip, op, fregs);
0.00 : ffff80000815bee4: mov x3, x21
0.00 : ffff80000815bee8: mov x2, x19
0.00 : ffff80000815beec: mov x1, x20
0.00 : ffff80000815bef0: mov x0, x23
0.00 : ffff80000815bef4: blr x4
: 7527 } while_for_each_ftrace_op(op);
0.00 : ffff80000815bef8: ldr x19, [x19, #8]
0.00 : ffff80000815befc: cmp x19, #0x0
0.00 : ffff80000815bf00: ccmp x19, x24, #0x4, ne // ne = any
0.00 : ffff80000815bf04: b.ne ffff80000815be9c <arch_ftrace_ops_list_func+0xac> // b.any
: 7532 get_current():
0.00 : ffff80000815bf08: mrs x1, sp_el0
: 20 __preempt_count_dec_and_test():
: 62 }
:
: 64 static inline bool __preempt_count_dec_and_test(void)
: 65 {
: 66 struct thread_info *ti = current_thread_info();
: 67 u64 pc = READ_ONCE(ti->preempt_count);
0.00 : ffff80000815bf0c: ldr x0, [x1, #8]
:
: 66 /* Update only the count field, leaving need_resched unchanged */
: 67 WRITE_ONCE(ti->preempt.count, --pc);
0.00 : ffff80000815bf10: sub x0, x0, #0x1
0.00 : ffff80000815bf14: str w0, [x1, #8]
: 74 * need of a reschedule. Otherwise, we need to reload the
: 75 * preempt_count in case the need_resched flag was cleared by an
: 76 * interrupt occurring between the non-atomic READ_ONCE/WRITE_ONCE
: 77 * pair.
: 78 */
: 79 return !pc || !READ_ONCE(ti->preempt_count);
0.00 : ffff80000815bf18: cbnz x0, ffff80000815bf64 <arch_ftrace_ops_list_func+0x174>
: 81 trace_clear_recursion():
: 178 preempt_enable_notrace();
0.00 : ffff80000815bf1c: bl ffff800008ae88d0 <preempt_schedule_notrace>
: 180 get_current():
2.32 : ffff80000815bf20: mrs x1, sp_el0 // current.h:19
: 20 trace_clear_recursion():
: 180 trace_recursion_clear(bit);
0.00 : ffff80000815bf24: ldr x0, [x1, #2520]
0.00 : ffff80000815bf28: and x0, x0, x22
2.32 : ffff80000815bf2c: str x0, [x1, #2520] // trace_recursion.h:180
: 184 arch_ftrace_ops_list_func():
: 7553 __ftrace_ops_list_func(ip, parent_ip, NULL, fregs);
: 7554 }
0.00 : ffff80000815bf30: ldr x25, [sp, #64]
0.00 : ffff80000815bf34: mrs x0, sp_el0
2.32 : ffff80000815bf38: ldr x2, [sp, #136] // ftrace.c:7553
0.00 : ffff80000815bf3c: ldr x1, [x0, #1168]
0.00 : ffff80000815bf40: subs x2, x2, x1
0.00 : ffff80000815bf44: mov x1, #0x0 // #0
0.00 : ffff80000815bf48: b.ne ffff80000815bf98 <arch_ftrace_ops_list_func+0x1a8> // b.any
2.32 : ffff80000815bf4c: ldp x19, x20, [sp, #16]
0.00 : ffff80000815bf50: ldp x21, x22, [sp, #32]
2.32 : ffff80000815bf54: ldp x23, x24, [sp, #48]
0.00 : ffff80000815bf58: ldp x29, x30, [sp], #144
0.00 : ffff80000815bf5c: autiasp
0.00 : ffff80000815bf60: ret
: 7568 __preempt_count_dec_and_test():
11.62 : ffff80000815bf64: ldr x0, [x1, #8] // preempt.h:74
0.00 : ffff80000815bf68: cbnz x0, ffff80000815bf20 <arch_ftrace_ops_list_func+0x130>
: 76 trace_clear_recursion():
: 178 preempt_enable_notrace();
0.00 : ffff80000815bf6c: bl ffff800008ae88d0 <preempt_schedule_notrace>
0.00 : ffff80000815bf70: b ffff80000815bf20 <arch_ftrace_ops_list_func+0x130>
: 181 __ftrace_ops_list_func():
: 7519 if ((!(op->flags & FTRACE_OPS_FL_RCU) || rcu_is_watching()) &&
0.00 : ffff80000815bf74: bl ffff8000080e5770 <rcu_is_watching>
0.00 : ffff80000815bf78: tst w0, #0xff
0.00 : ffff80000815bf7c: b.ne ffff80000815bea8 <arch_ftrace_ops_list_func+0xb8> // b.any
0.00 : ffff80000815bf80: b ffff80000815bef8 <arch_ftrace_ops_list_func+0x108>
: 7524 trace_test_and_set_recursion():
: 158 if (val & (1 << bit)) {
0.00 : ffff80000815bf84: tbnz w7, #9, ffff80000815bf34 <arch_ftrace_ops_list_func+0x144>
0.00 : ffff80000815bf88: mov x22, #0xfffffffffffffdff // #-513
0.00 : ffff80000815bf8c: mov w2, #0x200 // #512
0.00 : ffff80000815bf90: str x25, [sp, #64]
0.00 : ffff80000815bf94: b ffff80000815be74 <arch_ftrace_ops_list_func+0x84>
0.00 : ffff80000815bf98: str x25, [sp, #64]
: 165 arch_ftrace_ops_list_func():
: 7553 }
0.00 : ffff80000815bf9c: bl ffff800008ae5de0 <__stack_chk_fail>
: 7555 __ftrace_ops_list_func():
: 7521 if (FTRACE_WARN_ON(!op->func)) {
0.00 : ffff80000815bfa0: brk #0x800
: 7523 ftrace_kill():
: 8040 */
: 8041 void ftrace_kill(void)
: 8042 {
: 8043 ftrace_disabled = 1;
: 8044 ftrace_enabled = 0;
: 8045 ftrace_trace_function = ftrace_stub;
0.00 : ffff80000815bfa4: adrp x3, ffff80000802e000 <arch_ftrace_update_code+0x10>
0.00 : ffff80000815bfa8: add x3, x3, #0x144
: 8038 ftrace_disabled = 1;
0.00 : ffff80000815bfac: mov w4, #0x1 // #1
: 8040 __ftrace_ops_list_func():
: 7522 pr_warn("op=%p %pS\n", op, op);
0.00 : ffff80000815bfb0: mov x2, x19
0.00 : ffff80000815bfb4: mov x1, x19
0.00 : ffff80000815bfb8: adrp x0, ffff800008d80000 <kallsyms_token_index+0x17f60>
0.00 : ffff80000815bfbc: add x0, x0, #0x678
: 7527 ftrace_kill():
: 8040 ftrace_trace_function = ftrace_stub;
0.00 : ffff80000815bfc0: str x3, [x25, #192]
: 8039 ftrace_enabled = 0;
0.00 : ffff80000815bfc4: stp w4, wzr, [x25, #200]
: 8041 __ftrace_ops_list_func():
: 7522 pr_warn("op=%p %pS\n", op, op);
0.00 : ffff80000815bfc8: bl ffff800008ad5220 <_printk>
: 7523 goto out;
0.00 : ffff80000815bfcc: b ffff80000815bf08 <arch_ftrace_ops_list_func+0x118>
2.3 fprobe_handler
: 28 static void fprobe_handler(unsigned long ip, unsigned long parent_ip,
: 29 struct ftrace_ops *ops, struct ftrace_regs *fregs)
: 30 {
0.00 : ffff8000081a2020: paciasp
0.00 : ffff8000081a2024: stp x29, x30, [sp, #-64]!
0.00 : ffff8000081a2028: mov x29, sp
0.00 : ffff8000081a202c: stp x19, x20, [sp, #16]
0.00 : ffff8000081a2030: mov x19, x2
0.00 : ffff8000081a2034: stp x21, x22, [sp, #32]
0.00 : ffff8000081a2038: mov x22, x3
0.00 : ffff8000081a203c: str x23, [sp, #48]
0.00 : ffff8000081a2040: mov x23, x0
: 40 fprobe_disabled():
: 49 */
: 50 #define FPROBE_FL_KPROBE_SHARED 2
:
: 52 static inline bool fprobe_disabled(struct fprobe *fp)
: 53 {
: 54 return (fp) ? fp->flags & FPROBE_FL_DISABLED : false;
0.00 : ffff8000081a2044: cbz x2, ffff8000081a2050 <fprobe_handler+0x30>
20.00 : ffff8000081a2048: ldr w0, [x2, #192] // fprobe.h:49
0.00 : ffff8000081a204c: tbnz w0, #0, ffff8000081a2128 <fprobe_handler+0x108>
: 58 get_current():
: 19 */
: 20 static __always_inline struct task_struct *get_current(void)
: 21 {
: 22 unsigned long sp_el0;
:
: 24 asm ("mrs %0, sp_el0" : "=r" (sp_el0));
0.00 : ffff8000081a2050: mrs x0, sp_el0
: 26 trace_test_and_set_recursion():
: 144 * Preemption is promised to be disabled when return bit >= 0.
: 145 */
: 146 static __always_inline int trace_test_and_set_recursion(unsigned long ip, unsigned long pip,
: 147 int start)
: 148 {
: 149 unsigned int val = READ_ONCE(current->trace_recursion);
10.00 : ffff8000081a2054: ldr x9, [x0, #2520] // trace_recursion.h:144
: 151 trace_get_context_bit():
: 121 return TRACE_CTX_NORMAL - bit;
0.00 : ffff8000081a2058: mov w6, #0x3 // #3
: 123 preempt_count():
: 13 #define PREEMPT_NEED_RESCHED BIT(32)
: 14 #define PREEMPT_ENABLED (PREEMPT_NEED_RESCHED)
:
: 16 static inline int preempt_count(void)
: 17 {
: 18 return READ_ONCE(current_thread_info()->preempt.count);
0.00 : ffff8000081a205c: ldr w8, [x0, #8]
: 20 trace_test_and_set_recursion():
: 148 int bit;
:
: 150 bit = trace_get_context_bit() + start;
: 151 if (unlikely(val & (1 << bit))) {
0.00 : ffff8000081a2060: mov w4, #0x1 // #1
: 153 interrupt_context_level():
: 94 static __always_inline unsigned char interrupt_context_level(void)
: 95 {
: 96 unsigned long pc = preempt_count();
: 97 unsigned char level = 0;
:
: 99 level += !!(pc & (NMI_MASK));
0.00 : ffff8000081a2064: tst w8, #0xf00000
: 96 level += !!(pc & (NMI_MASK | HARDIRQ_MASK));
: 97 level += !!(pc & (NMI_MASK | HARDIRQ_MASK | SOFTIRQ_OFFSET));
0.00 : ffff8000081a2068: and w7, w8, #0xffff00
: 94 level += !!(pc & (NMI_MASK));
0.00 : ffff8000081a206c: cset w5, ne // ne = any
: 96 level += !!(pc & (NMI_MASK | HARDIRQ_MASK | SOFTIRQ_OFFSET));
0.00 : ffff8000081a2070: and w7, w7, #0xffff01ff
: 95 level += !!(pc & (NMI_MASK | HARDIRQ_MASK));
0.00 : ffff8000081a2074: tst w8, #0xff0000
0.00 : ffff8000081a2078: cinc w5, w5, ne // ne = any
: 96 level += !!(pc & (NMI_MASK | HARDIRQ_MASK | SOFTIRQ_OFFSET));
0.00 : ffff8000081a207c: cmp w7, #0x0
: 98 trace_get_context_bit():
: 121 return TRACE_CTX_NORMAL - bit;
0.00 : ffff8000081a2080: cinc w5, w5, ne // ne = any
0.00 : ffff8000081a2084: sub w5, w6, w5
: 124 trace_test_and_set_recursion():
: 148 if (unlikely(val & (1 << bit))) {
0.00 : ffff8000081a2088: lsl w4, w4, w5
: 150 trace_clear_recursion():
: 180 */
: 181 static __always_inline void trace_clear_recursion(int bit)
: 182 {
: 183 preempt_enable_notrace();
: 184 barrier();
: 185 trace_recursion_clear(bit);
10.00 : ffff8000081a208c: mvn w20, w4 // trace_recursion.h:180
0.00 : ffff8000081a2090: sxtw x20, w20
: 188 trace_test_and_set_recursion():
: 148 if (unlikely(val & (1 << bit))) {
0.00 : ffff8000081a2094: tst w4, w9
0.00 : ffff8000081a2098: b.ne ffff8000081a2194 <fprobe_handler+0x174> // b.any
: 165 current->trace_recursion = val;
0.00 : ffff8000081a209c: orr w4, w4, w9
: 167 get_current():
0.00 : ffff8000081a20a0: mrs x5, sp_el0
: 20 trace_test_and_set_recursion():
0.00 : ffff8000081a20a4: str x4, [x5, #2520]
: 166 __preempt_count_add():
: 47 return !current_thread_info()->preempt.need_resched;
: 48 }
:
: 50 static inline void __preempt_count_add(int val)
: 51 {
: 52 u32 pc = READ_ONCE(current_thread_info()->preempt.count);
0.00 : ffff8000081a20a8: ldr w4, [x5, #8]
: 48 pc += val;
0.00 : ffff8000081a20ac: add w4, w4, #0x1
: 49 WRITE_ONCE(current_thread_info()->preempt.count, pc);
0.00 : ffff8000081a20b0: str w4, [x5, #8]
: 51 fprobe_handler():
: 43 if (bit < 0) {
: 44 fp->nmissed++;
: 45 return;
: 46 }
:
: 48 if (fp->exit_handler) {
0.00 : ffff8000081a20b4: ldr x0, [x19, #224]
0.00 : ffff8000081a20b8: cbz x0, ffff8000081a2140 <fprobe_handler+0x120>
: 44 rh = rethook_try_get(fp->rethook);
10.00 : ffff8000081a20bc: ldr x0, [x19, #200] // fprobe.c:44
0.00 : ffff8000081a20c0: bl ffff8000081a2a54 <rethook_try_get>
0.00 : ffff8000081a20c4: mov x21, x0
: 45 if (!rh) {
0.00 : ffff8000081a20c8: cbz x0, ffff8000081a21a4 <fprobe_handler+0x184>
: 50 fp->nmissed++;
: 51 goto out;
: 52 }
: 53 fpr = container_of(rh, struct fprobe_rethook_node, node);
: 54 fpr->entry_ip = ip;
0.00 : ffff8000081a20cc: str x23, [x0, #48]
: 54 private = fpr->private;
: 55 }
:
: 57 if (fp->entry_handler)
0.00 : ffff8000081a20d0: ldr x4, [x19, #216]
0.00 : ffff8000081a20d4: cbz x4, ffff8000081a2180 <fprobe_handler+0x160>
: 55 should_rethook = fp->entry_handler(fp, ip, fregs, fpr->private);
0.00 : ffff8000081a20d8: mov x1, x23
0.00 : ffff8000081a20dc: mov x0, x19
0.00 : ffff8000081a20e0: add x3, x21, #0x38
0.00 : ffff8000081a20e4: mov x2, x22
0.00 : ffff8000081a20e8: blr x4
:
: 59 if (rh) {
: 60 if (should_rethook)
0.00 : ffff8000081a20ec: tst w0, #0xff
0.00 : ffff8000081a20f0: b.ne ffff8000081a2180 <fprobe_handler+0x160> // b.any
: 61 rethook_hook(rh, fregs, true);
: 62 else
: 63 rethook_recycle(rh);
0.00 : ffff8000081a20f4: mov x0, x21
0.00 : ffff8000081a20f8: bl ffff8000081a2bf0 <rethook_recycle>
: 66 get_current():
0.00 : ffff8000081a20fc: mrs x1, sp_el0
: 20 __preempt_count_dec_and_test():
: 62 }
:
: 64 static inline bool __preempt_count_dec_and_test(void)
: 65 {
: 66 struct thread_info *ti = current_thread_info();
: 67 u64 pc = READ_ONCE(ti->preempt_count);
0.00 : ffff8000081a2100: ldr x0, [x1, #8]
:
: 66 /* Update only the count field, leaving need_resched unchanged */
: 67 WRITE_ONCE(ti->preempt.count, --pc);
0.00 : ffff8000081a2104: sub x0, x0, #0x1
0.00 : ffff8000081a2108: str w0, [x1, #8]
: 74 * need of a reschedule. Otherwise, we need to reload the
: 75 * preempt_count in case the need_resched flag was cleared by an
: 76 * interrupt occurring between the non-atomic READ_ONCE/WRITE_ONCE
: 77 * pair.
: 78 */
: 79 return !pc || !READ_ONCE(ti->preempt_count);
0.00 : ffff8000081a210c: cbnz x0, ffff8000081a2170 <fprobe_handler+0x150>
: 81 trace_clear_recursion():
: 178 preempt_enable_notrace();
0.00 : ffff8000081a2110: bl ffff800008ae88d0 <preempt_schedule_notrace>
0.00 : ffff8000081a2114: nop
: 181 get_current():
10.00 : ffff8000081a2118: mrs x1, sp_el0 // current.h:19
: 20 trace_clear_recursion():
: 180 trace_recursion_clear(bit);
0.00 : ffff8000081a211c: ldr x0, [x1, #2520]
0.00 : ffff8000081a2120: and x0, x0, x20
10.00 : ffff8000081a2124: str x0, [x1, #2520] // trace_recursion.h:180
: 184 fprobe_handler():
: 66 }
:
: 68 out:
: 69 ftrace_test_recursion_unlock(bit);
: 70 }
0.00 : ffff8000081a2128: ldp x19, x20, [sp, #16]
0.00 : ffff8000081a212c: ldp x21, x22, [sp, #32]
0.00 : ffff8000081a2130: ldr x23, [sp, #48]
20.00 : ffff8000081a2134: ldp x29, x30, [sp], #64 // fprobe.c:66
0.00 : ffff8000081a2138: autiasp
10.00 : ffff8000081a213c: ret
: 54 if (fp->entry_handler)
0.00 : ffff8000081a2140: ldr x4, [x19, #216]
0.00 : ffff8000081a2144: cbz x4, ffff8000081a215c <fprobe_handler+0x13c>
: 55 should_rethook = fp->entry_handler(fp, ip, fregs, fpr->private);
0.00 : ffff8000081a2148: mov x2, x22
0.00 : ffff8000081a214c: mov x1, x23
0.00 : ffff8000081a2150: mov x0, x19
0.00 : ffff8000081a2154: mov x3, #0x38 // #56
0.00 : ffff8000081a2158: blr x4
: 61 get_current():
0.00 : ffff8000081a215c: mrs x1, sp_el0
: 20 __preempt_count_dec_and_test():
: 62 u64 pc = READ_ONCE(ti->preempt_count);
0.00 : ffff8000081a2160: ldr x0, [x1, #8]
: 65 WRITE_ONCE(ti->preempt.count, --pc);
0.00 : ffff8000081a2164: sub x0, x0, #0x1
0.00 : ffff8000081a2168: str w0, [x1, #8]
: 74 return !pc || !READ_ONCE(ti->preempt_count);
0.00 : ffff8000081a216c: cbz x0, ffff8000081a2110 <fprobe_handler+0xf0>
0.00 : ffff8000081a2170: ldr x0, [x1, #8]
0.00 : ffff8000081a2174: cbnz x0, ffff8000081a2118 <fprobe_handler+0xf8>
: 78 trace_clear_recursion():
: 178 preempt_enable_notrace();
0.00 : ffff8000081a2178: bl ffff800008ae88d0 <preempt_schedule_notrace>
0.00 : ffff8000081a217c: b ffff8000081a2118 <fprobe_handler+0xf8>
: 181 fprobe_handler():
: 59 rethook_hook(rh, fregs, true);
0.00 : ffff8000081a2180: mov x1, x22
0.00 : ffff8000081a2184: mov x0, x21
0.00 : ffff8000081a2188: mov w2, #0x1 // #1
0.00 : ffff8000081a218c: bl ffff8000081a27d0 <rethook_hook>
0.00 : ffff8000081a2190: b ffff8000081a215c <fprobe_handler+0x13c>
: 65 trace_test_and_set_recursion():
: 158 if (val & (1 << bit)) {
0.00 : ffff8000081a2194: tbnz w9, #4, ffff8000081a21b4 <fprobe_handler+0x194>
0.00 : ffff8000081a2198: mov x20, #0xffffffffffffffef // #-17
0.00 : ffff8000081a219c: mov w4, #0x10 // #16
0.00 : ffff8000081a21a0: b ffff8000081a209c <fprobe_handler+0x7c>
: 163 fprobe_handler():
: 46 fp->nmissed++;
0.00 : ffff8000081a21a4: ldr x0, [x19, #184]
0.00 : ffff8000081a21a8: add x0, x0, #0x1
0.00 : ffff8000081a21ac: str x0, [x19, #184]
: 47 goto out;
0.00 : ffff8000081a21b0: b ffff8000081a215c <fprobe_handler+0x13c>
: 39 fp->nmissed++;
0.00 : ffff8000081a21b4: ldr x0, [x19, #184]
0.00 : ffff8000081a21b8: add x0, x0, #0x1
0.00 : ffff8000081a21bc: str x0, [x19, #184]
: 40 return;
0.00 : ffff8000081a21c0: b ffff8000081a2128 <fprobe_handler+0x108>
2.4 bpf_fprobe_entry
: 5 ffff8000081e19f0 <bpf_fprobe_entry>:
: 6 bpf_fprobe_entry():
: 1057 flags = u64_stats_update_begin_irqsave(&stats->syncp);
: 1058 u64_stats_inc(&stats->cnt);
: 1059 u64_stats_add(&stats->nsecs, sched_clock() - start);
: 1060 u64_stats_update_end_irqrestore(&stats->syncp, flags);
: 1061 }
: 1062 }
0.00 : ffff8000081e19f0: bti c
0.00 : ffff8000081e19f4: nop
0.00 : ffff8000081e19f8: nop
: 165 {
0.00 : ffff8000081e19fc: paciasp
0.00 : ffff8000081e1a00: stp x29, x30, [sp, #-80]!
0.00 : ffff8000081e1a04: mov w4, #0x0 // #0
0.00 : ffff8000081e1a08: mov x29, sp
0.00 : ffff8000081e1a0c: stp x19, x20, [sp, #16]
0.00 : ffff8000081e1a10: mov x19, x3
0.00 : ffff8000081e1a14: stp x21, x22, [sp, #32]
0.00 : ffff8000081e1a18: mov x22, x0
0.00 : ffff8000081e1a1c: mov x21, x2
0.00 : ffff8000081e1a20: stp x23, x24, [sp, #48]
0.00 : ffff8000081e1a24: str x25, [sp, #64]
: 167 struct bpf_fprobe_context *fprobe_ctx = fp->ops.private;
0.00 : ffff8000081e1a28: ldr x24, [x0, #24]
: 168 struct bpf_tramp_links *links = fprobe_ctx->links;
0.00 : ffff8000081e1a2c: ldr x23, [x24]
: 174 memset(&call_ctx->ctx, 0, sizeof(call_ctx->ctx));
0.00 : ffff8000081e1a30: stp xzr, xzr, [x3]
: 175 call_ctx->ip = ip;
0.00 : ffff8000081e1a34: str x1, [x3, #16]
: 176 for (i = 0; i < fprobe_ctx->nr_args; i++)
0.00 : ffff8000081e1a38: ldr w0, [x24, #8]
0.00 : ffff8000081e1a3c: cmp w0, #0x0
0.00 : ffff8000081e1a40: b.gt ffff8000081e1a64 <bpf_fprobe_entry+0x74>
0.00 : ffff8000081e1a44: b ffff8000081e1a90 <bpf_fprobe_entry+0xa0>
: 177 call_ctx->args[i] = ftrace_regs_get_argument(regs, i);
0.00 : ffff8000081e1a48: ldr x0, [x21, x1, lsl #3]
0.00 : ffff8000081e1a4c: add x1, x19, x1, lsl #3
: 176 for (i = 0; i < fprobe_ctx->nr_args; i++)
0.00 : ffff8000081e1a50: add w4, w4, #0x1
: 177 call_ctx->args[i] = ftrace_regs_get_argument(regs, i);
16.67 : ffff8000081e1a54: str x0, [x1, #24] // trampoline.c:177
: 176 for (i = 0; i < fprobe_ctx->nr_args; i++)
0.00 : ffff8000081e1a58: ldr w0, [x24, #8]
0.00 : ffff8000081e1a5c: cmp w0, w4
0.00 : ffff8000081e1a60: b.le ffff8000081e1a90 <bpf_fprobe_entry+0xa0>
: 177 call_ctx->args[i] = ftrace_regs_get_argument(regs, i);
8.33 : ffff8000081e1a64: sxtw x1, w4
0.00 : ffff8000081e1a68: mov x0, #0x0 // #0
0.00 : ffff8000081e1a6c: cmp w4, #0x7
0.00 : ffff8000081e1a70: b.le ffff8000081e1a48 <bpf_fprobe_entry+0x58>
0.00 : ffff8000081e1a74: sxtw x1, w4
: 176 for (i = 0; i < fprobe_ctx->nr_args; i++)
0.00 : ffff8000081e1a78: add w4, w4, #0x1
: 177 call_ctx->args[i] = ftrace_regs_get_argument(regs, i);
0.00 : ffff8000081e1a7c: add x1, x19, x1, lsl #3
0.00 : ffff8000081e1a80: str x0, [x1, #24]
: 176 for (i = 0; i < fprobe_ctx->nr_args; i++)
0.00 : ffff8000081e1a84: ldr w0, [x24, #8]
0.00 : ffff8000081e1a88: cmp w0, w4
0.00 : ffff8000081e1a8c: b.gt ffff8000081e1a64 <bpf_fprobe_entry+0x74>
: 179 for (i = 0; i < fentry->nr_links; i++)
0.00 : ffff8000081e1a90: ldr w1, [x23, #304]
: 185 call_ctx->args);
0.00 : ffff8000081e1a94: add x25, x19, #0x18
0.00 : ffff8000081e1a98: mov x20, #0x0 // #0
: 179 for (i = 0; i < fentry->nr_links; i++)
0.00 : ffff8000081e1a9c: cmp w1, #0x0
0.00 : ffff8000081e1aa0: b.le ffff8000081e1ad4 <bpf_fprobe_entry+0xe4>
0.00 : ffff8000081e1aa4: nop
: 180 call_bpf_prog(fentry->links[i], &call_ctx->ctx, call_ctx->args);
0.00 : ffff8000081e1aa8: ldr x1, [x23, x20, lsl #3]
0.00 : ffff8000081e1aac: mov x3, x25
0.00 : ffff8000081e1ab0: mov x2, x19
: 179 for (i = 0; i < fentry->nr_links; i++)
0.00 : ffff8000081e1ab4: add x20, x20, #0x1
: 180 call_bpf_prog(fentry->links[i], &call_ctx->ctx, call_ctx->args);
16.67 : ffff8000081e1ab8: ldr x0, [x1, #24] // trampoline.c:180
0.00 : ffff8000081e1abc: ldr x1, [x1, #80]
0.00 : ffff8000081e1ac0: bl ffff8000081e1800 <call_bpf_prog.isra.0>
: 179 for (i = 0; i < fentry->nr_links; i++)
0.00 : ffff8000081e1ac4: ldr w0, [x23, #304]
0.00 : ffff8000081e1ac8: cmp w0, w20
0.00 : ffff8000081e1acc: b.gt ffff8000081e1aa8 <bpf_fprobe_entry+0xb8>
0.00 : ffff8000081e1ad0: ldr w0, [x24, #8]
: 182 call_ctx->args[fprobe_ctx->nr_args] = 0;
0.00 : ffff8000081e1ad4: add x0, x19, w0, sxtw #3
: 183 for (i = 0; i < fmod_ret->nr_links; i++) {
0.00 : ffff8000081e1ad8: add x25, x23, #0x270
: 185 call_ctx->args);
0.00 : ffff8000081e1adc: add x24, x19, #0x18
0.00 : ffff8000081e1ae0: mov x20, #0x0 // #0
: 182 call_ctx->args[fprobe_ctx->nr_args] = 0;
25.00 : ffff8000081e1ae4: str xzr, [x0, #24] // trampoline.c:182
: 183 for (i = 0; i < fmod_ret->nr_links; i++) {
0.00 : ffff8000081e1ae8: ldr w0, [x25, #304]
0.00 : ffff8000081e1aec: cmp w0, #0x0
0.00 : ffff8000081e1af0: b.gt ffff8000081e1b04 <bpf_fprobe_entry+0x114>
16.67 : ffff8000081e1af4: b ffff8000081e1ba8 <bpf_fprobe_entry+0x1b8> // trampoline.c:183
0.00 : ffff8000081e1af8: ldr w0, [x25, #304]
0.00 : ffff8000081e1afc: cmp w0, w20
0.00 : ffff8000081e1b00: b.le ffff8000081e1ba8 <bpf_fprobe_entry+0x1b8>
: 184 ret = call_bpf_prog(fmod_ret->links[i], &call_ctx->ctx,
0.00 : ffff8000081e1b04: ldr x1, [x25, x20, lsl #3]
0.00 : ffff8000081e1b08: mov x3, x24
0.00 : ffff8000081e1b0c: mov x2, x19
: 183 for (i = 0; i < fmod_ret->nr_links; i++) {
0.00 : ffff8000081e1b10: add x20, x20, #0x1
: 184 ret = call_bpf_prog(fmod_ret->links[i], &call_ctx->ctx,
0.00 : ffff8000081e1b14: ldr x0, [x1, #24]
0.00 : ffff8000081e1b18: ldr x1, [x1, #80]
0.00 : ffff8000081e1b1c: bl ffff8000081e1800 <call_bpf_prog.isra.0>
: 187 if (ret) {
0.00 : ffff8000081e1b20: cbz w0, ffff8000081e1af8 <bpf_fprobe_entry+0x108>
: 189 ftrace_override_function_with_return(regs);
0.00 : ffff8000081e1b24: ldr x2, [x21, #88]
: 188 ftrace_regs_set_return_value(regs, ret);
0.00 : ffff8000081e1b28: sxtw x1, w0
0.00 : ffff8000081e1b2c: str x1, [x21]
: 191 bpf_fprobe_exit():
: 160 for (i = 0; i < fexit->nr_links; i++)
0.00 : ffff8000081e1b30: mov x20, #0x0 // #0
: 162 bpf_fprobe_entry():
: 189 ftrace_override_function_with_return(regs);
0.00 : ffff8000081e1b34: str x2, [x21, #104]
: 191 bpf_fprobe_exit():
: 153 struct bpf_fprobe_context *fprobe_ctx = fp->ops.private;
0.00 : ffff8000081e1b38: ldr x2, [x22, #24]
: 158 call_ctx->args[fprobe_ctx->nr_args] = ftrace_regs_return_value(regs);
0.00 : ffff8000081e1b3c: ldrsw x0, [x2, #8]
: 154 struct bpf_tramp_links *links = fprobe_ctx->links;
0.00 : ffff8000081e1b40: ldr x21, [x2]
: 158 call_ctx->args[fprobe_ctx->nr_args] = ftrace_regs_return_value(regs);
0.00 : ffff8000081e1b44: add x0, x19, x0, lsl #3
: 160 for (i = 0; i < fexit->nr_links; i++)
0.00 : ffff8000081e1b48: add x21, x21, #0x138
: 158 call_ctx->args[fprobe_ctx->nr_args] = ftrace_regs_return_value(regs);
0.00 : ffff8000081e1b4c: str x1, [x0, #24]
: 160 for (i = 0; i < fexit->nr_links; i++)
0.00 : ffff8000081e1b50: ldr w0, [x21, #304]
0.00 : ffff8000081e1b54: cmp w0, #0x0
0.00 : ffff8000081e1b58: b.le ffff8000081e1b88 <bpf_fprobe_entry+0x198>
0.00 : ffff8000081e1b5c: nop
: 161 call_bpf_prog(fexit->links[i], &call_ctx->ctx, call_ctx->args);
0.00 : ffff8000081e1b60: ldr x1, [x21, x20, lsl #3]
0.00 : ffff8000081e1b64: mov x3, x24
0.00 : ffff8000081e1b68: mov x2, x19
: 160 for (i = 0; i < fexit->nr_links; i++)
0.00 : ffff8000081e1b6c: add x20, x20, #0x1
: 161 call_bpf_prog(fexit->links[i], &call_ctx->ctx, call_ctx->args);
0.00 : ffff8000081e1b70: ldr x0, [x1, #24]
0.00 : ffff8000081e1b74: ldr x1, [x1, #80]
0.00 : ffff8000081e1b78: bl ffff8000081e1800 <call_bpf_prog.isra.0>
: 160 for (i = 0; i < fexit->nr_links; i++)
0.00 : ffff8000081e1b7c: ldr w0, [x21, #304]
0.00 : ffff8000081e1b80: cmp w0, w20
0.00 : ffff8000081e1b84: b.gt ffff8000081e1b60 <bpf_fprobe_entry+0x170>
: 164 bpf_fprobe_entry():
: 192 return false;
0.00 : ffff8000081e1b88: mov w0, #0x0 // #0
: 197 }
0.00 : ffff8000081e1b8c: ldp x19, x20, [sp, #16]
0.00 : ffff8000081e1b90: ldp x21, x22, [sp, #32]
0.00 : ffff8000081e1b94: ldp x23, x24, [sp, #48]
0.00 : ffff8000081e1b98: ldr x25, [sp, #64]
0.00 : ffff8000081e1b9c: ldp x29, x30, [sp], #80
0.00 : ffff8000081e1ba0: autiasp
0.00 : ffff8000081e1ba4: ret
: 196 return fexit->nr_links;
0.00 : ffff8000081e1ba8: ldr w0, [x23, #616]
: 197 }
0.00 : ffff8000081e1bac: ldp x19, x20, [sp, #16]
: 196 return fexit->nr_links;
0.00 : ffff8000081e1bb0: cmp w0, #0x0
0.00 : ffff8000081e1bb4: cset w0, ne // ne = any
: 197 }
0.00 : ffff8000081e1bb8: ldp x21, x22, [sp, #32]
0.00 : ffff8000081e1bbc: ldp x23, x24, [sp, #48]
0.00 : ffff8000081e1bc0: ldr x25, [sp, #64]
0.00 : ffff8000081e1bc4: ldp x29, x30, [sp], #80
0.00 : ffff8000081e1bc8: autiasp
16.67 : ffff8000081e1bcc: ret // trampoline.c:197
2.5 call_bpf_prog
: 5 ffff8000081e1800 <call_bpf_prog.isra.0>:
: 6 call_bpf_prog.isra.0():
:
: 200 if (oldp)
: 201 *oldp = old;
:
: 203 if (unlikely(!old))
: 204 refcount_warn_saturate(r, REFCOUNT_ADD_UAF);
13.33 : ffff8000081e1800: nop // refcount.h:199
0.00 : ffff8000081e1804: nop
: 207 call_bpf_prog():
:
: 108 mutex_unlock(&tr->mutex);
: 109 return ret;
: 110 }
: 111 #else
: 112 static unsigned int call_bpf_prog(struct bpf_tramp_link *l,
0.00 : ffff8000081e1808: paciasp
0.00 : ffff8000081e180c: stp x29, x30, [sp, #-64]!
0.00 : ffff8000081e1810: mov x29, sp
0.00 : ffff8000081e1814: stp x19, x20, [sp, #16]
0.00 : ffff8000081e1818: mov x19, x0
0.00 : ffff8000081e181c: mov x20, x2
0.00 : ffff8000081e1820: stp x21, x22, [sp, #32]
6.67 : ffff8000081e1824: stp x23, x24, [sp, #48] // trampoline.c:107
0.00 : ffff8000081e1828: mov x24, x3
: 118 struct bpf_tramp_run_ctx *run_ctx) = __bpf_prog_exit;
: 119 struct bpf_prog *p = l->link.prog;
: 120 unsigned int ret;
: 121 u64 start_time;
:
: 123 if (p->aux->sleepable) {
60.00 : ffff8000081e182c: ldr x0, [x0, #56] // trampoline.c:118
13.33 : ffff8000081e1830: ldrb w0, [x0, #140]
0.00 : ffff8000081e1834: cbnz w0, ffff8000081e1858 <call_bpf_prog.isra.0+0x58>
: 121 enter = __bpf_prog_enter_sleepable;
: 122 exit = __bpf_prog_exit_sleepable;
: 123 } else if (p->expected_attach_type == BPF_LSM_CGROUP) {
0.00 : ffff8000081e1838: ldr w0, [x19, #8]
0.00 : ffff8000081e183c: cmp w0, #0x2b
0.00 : ffff8000081e1840: b.eq ffff8000081e18c4 <call_bpf_prog.isra.0+0xc4> // b.none
: 112 void (*exit)(struct bpf_prog *prog, u64 start,
0.00 : ffff8000081e1844: adrp x22, ffff8000081e1000 <print_bpf_insn+0x580>
: 110 u64 (*enter)(struct bpf_prog *prog,
0.00 : ffff8000081e1848: adrp x2, ffff8000081e1000 <print_bpf_insn+0x580>
: 112 void (*exit)(struct bpf_prog *prog, u64 start,
0.00 : ffff8000081e184c: add x22, x22, #0xbd0
: 110 u64 (*enter)(struct bpf_prog *prog,
0.00 : ffff8000081e1850: add x2, x2, #0xd20
0.00 : ffff8000081e1854: b ffff8000081e1868 <call_bpf_prog.isra.0+0x68>
: 120 exit = __bpf_prog_exit_sleepable;
0.00 : ffff8000081e1858: adrp x22, ffff8000081e1000 <print_bpf_insn+0x580>
: 119 enter = __bpf_prog_enter_sleepable;
0.00 : ffff8000081e185c: adrp x2, ffff8000081e1000 <print_bpf_insn+0x580>
: 120 exit = __bpf_prog_exit_sleepable;
0.00 : ffff8000081e1860: add x22, x22, #0xc60
: 119 enter = __bpf_prog_enter_sleepable;
0.00 : ffff8000081e1864: add x2, x2, #0xe10
: 126 enter = __bpf_prog_enter_lsm_cgroup;
: 127 exit = __bpf_prog_exit_lsm_cgroup;
: 128 }
:
: 130 ctx->bpf_cookie = l->cookie;
0.00 : ffff8000081e1868: str x1, [x20]
:
: 129 start_time = enter(p, ctx);
0.00 : ffff8000081e186c: mov x0, x19
0.00 : ffff8000081e1870: mov x1, x20
: 130 if (!start_time)
: 131 return 0;
0.00 : ffff8000081e1874: mov w23, #0x0 // #0
: 128 start_time = enter(p, ctx);
0.00 : ffff8000081e1878: blr x2
0.00 : ffff8000081e187c: mov x21, x0
: 129 if (!start_time)
0.00 : ffff8000081e1880: cbz x0, ffff8000081e18a8 <call_bpf_prog.isra.0+0xa8>
:
: 133 ret = p->bpf_func(args, p->insnsi);
0.00 : ffff8000081e1884: ldr x2, [x19, #48]
0.00 : ffff8000081e1888: add x1, x19, #0x48
0.00 : ffff8000081e188c: mov x0, x24
0.00 : ffff8000081e1890: blr x2
0.00 : ffff8000081e1894: mov w23, w0
:
: 135 exit(p, start_time, ctx);
0.00 : ffff8000081e1898: mov x2, x20
0.00 : ffff8000081e189c: mov x1, x21
0.00 : ffff8000081e18a0: mov x0, x19
0.00 : ffff8000081e18a4: blr x22
:
: 138 return ret;
: 139 }
0.00 : ffff8000081e18a8: mov w0, w23
0.00 : ffff8000081e18ac: ldp x19, x20, [sp, #16]
0.00 : ffff8000081e18b0: ldp x21, x22, [sp, #32]
0.00 : ffff8000081e18b4: ldp x23, x24, [sp, #48]
6.67 : ffff8000081e18b8: ldp x29, x30, [sp], #64 // trampoline.c:137
0.00 : ffff8000081e18bc: autiasp
0.00 : ffff8000081e18c0: ret
: 123 exit = __bpf_prog_exit_lsm_cgroup;
0.00 : ffff8000081e18c4: adrp x22, ffff8000081e1000 <print_bpf_insn+0x580>
: 122 enter = __bpf_prog_enter_lsm_cgroup;
0.00 : ffff8000081e18c8: adrp x2, ffff8000081e1000 <print_bpf_insn+0x580>
: 123 exit = __bpf_prog_exit_lsm_cgroup;
0.00 : ffff8000081e18cc: add x22, x22, #0x200
: 122 enter = __bpf_prog_enter_lsm_cgroup;
0.00 : ffff8000081e18d0: add x2, x2, #0x1c0
0.00 : ffff8000081e18d4: b ffff8000081e1868 <call_bpf_prog.isra.0+0x68>
More information about the linux-arm-kernel
mailing list