kernel/trace/trace_preemptirq.c

0001 // SPDX-License-Identifier: GPL-2.0
0002 /*
0003  * preemptoff and irqoff tracepoints
0004  *
0005  * Copyright (C) Joel Fernandes (Google) <joel@joelfernandes.org>
0006  */
0007
0008 #include <linux/kallsyms.h>
0009 #include <linux/uaccess.h>
0010 #include <linux/module.h>
0011 #include <linux/ftrace.h>
0012 #include <linux/kprobes.h>
0013 #include "trace.h"
0014
0015 #define CREATE_TRACE_POINTS
0016 #include <trace/events/preemptirq.h>
0017
0018 #ifdef CONFIG_TRACE_IRQFLAGS
0019 /* Per-cpu variable to prevent redundant calls when IRQs already off */
0020 static DEFINE_PER_CPU(int, tracing_irq_cpu);
0021
0022 /*
0023  * Like trace_hardirqs_on() but without the lockdep invocation. This is
0024  * used in the low level entry code where the ordering vs. RCU is important
0025  * and lockdep uses a staged approach which splits the lockdep hardirq
0026  * tracking into a RCU on and a RCU off section.
0027  */
0028 void trace_hardirqs_on_prepare(void)
0029 {
0030     if (this_cpu_read(tracing_irq_cpu)) {
0031         if (!in_nmi())
0032             trace_irq_enable(CALLER_ADDR0, CALLER_ADDR1);
0033         tracer_hardirqs_on(CALLER_ADDR0, CALLER_ADDR1);
0034         this_cpu_write(tracing_irq_cpu, 0);
0035     }
0036 }
0037 EXPORT_SYMBOL(trace_hardirqs_on_prepare);
0038 NOKPROBE_SYMBOL(trace_hardirqs_on_prepare);
0039
0040 void trace_hardirqs_on(void)
0041 {
0042     if (this_cpu_read(tracing_irq_cpu)) {
0043         if (!in_nmi())
0044             trace_irq_enable_rcuidle(CALLER_ADDR0, CALLER_ADDR1);
0045         tracer_hardirqs_on(CALLER_ADDR0, CALLER_ADDR1);
0046         this_cpu_write(tracing_irq_cpu, 0);
0047     }
0048
0049     lockdep_hardirqs_on_prepare();
0050     lockdep_hardirqs_on(CALLER_ADDR0);
0051 }
0052 EXPORT_SYMBOL(trace_hardirqs_on);
0053 NOKPROBE_SYMBOL(trace_hardirqs_on);
0054
0055 /*
0056  * Like trace_hardirqs_off() but without the lockdep invocation. This is
0057  * used in the low level entry code where the ordering vs. RCU is important
0058  * and lockdep uses a staged approach which splits the lockdep hardirq
0059  * tracking into a RCU on and a RCU off section.
0060  */
0061 void trace_hardirqs_off_finish(void)
0062 {
0063     if (!this_cpu_read(tracing_irq_cpu)) {
0064         this_cpu_write(tracing_irq_cpu, 1);
0065         tracer_hardirqs_off(CALLER_ADDR0, CALLER_ADDR1);
0066         if (!in_nmi())
0067             trace_irq_disable(CALLER_ADDR0, CALLER_ADDR1);
0068     }
0069
0070 }
0071 EXPORT_SYMBOL(trace_hardirqs_off_finish);
0072 NOKPROBE_SYMBOL(trace_hardirqs_off_finish);
0073
0074 void trace_hardirqs_off(void)
0075 {
0076     lockdep_hardirqs_off(CALLER_ADDR0);
0077
0078     if (!this_cpu_read(tracing_irq_cpu)) {
0079         this_cpu_write(tracing_irq_cpu, 1);
0080         tracer_hardirqs_off(CALLER_ADDR0, CALLER_ADDR1);
0081         if (!in_nmi())
0082             trace_irq_disable_rcuidle(CALLER_ADDR0, CALLER_ADDR1);
0083     }
0084 }
0085 EXPORT_SYMBOL(trace_hardirqs_off);
0086 NOKPROBE_SYMBOL(trace_hardirqs_off);
0087
0088 __visible void trace_hardirqs_on_caller(unsigned long caller_addr)
0089 {
0090     if (this_cpu_read(tracing_irq_cpu)) {
0091         if (!in_nmi())
0092             trace_irq_enable_rcuidle(CALLER_ADDR0, caller_addr);
0093         tracer_hardirqs_on(CALLER_ADDR0, caller_addr);
0094         this_cpu_write(tracing_irq_cpu, 0);
0095     }
0096
0097     lockdep_hardirqs_on_prepare();
0098     lockdep_hardirqs_on(caller_addr);
0099 }
0100 EXPORT_SYMBOL(trace_hardirqs_on_caller);
0101 NOKPROBE_SYMBOL(trace_hardirqs_on_caller);
0102
0103 __visible void trace_hardirqs_off_caller(unsigned long caller_addr)
0104 {
0105     lockdep_hardirqs_off(caller_addr);
0106
0107     if (!this_cpu_read(tracing_irq_cpu)) {
0108         this_cpu_write(tracing_irq_cpu, 1);
0109         tracer_hardirqs_off(CALLER_ADDR0, caller_addr);
0110         if (!in_nmi())
0111             trace_irq_disable_rcuidle(CALLER_ADDR0, caller_addr);
0112     }
0113 }
0114 EXPORT_SYMBOL(trace_hardirqs_off_caller);
0115 NOKPROBE_SYMBOL(trace_hardirqs_off_caller);
0116 #endif /* CONFIG_TRACE_IRQFLAGS */
0117
0118 #ifdef CONFIG_TRACE_PREEMPT_TOGGLE
0119
0120 void trace_preempt_on(unsigned long a0, unsigned long a1)
0121 {
0122     if (!in_nmi())
0123         trace_preempt_enable_rcuidle(a0, a1);
0124     tracer_preempt_on(a0, a1);
0125 }
0126
0127 void trace_preempt_off(unsigned long a0, unsigned long a1)
0128 {
0129     if (!in_nmi())
0130         trace_preempt_disable_rcuidle(a0, a1);
0131     tracer_preempt_off(a0, a1);
0132 }
0133 #endif