include/linux/trace_recursion.h

0001 /* SPDX-License-Identifier: GPL-2.0 */
0002 #ifndef _LINUX_TRACE_RECURSION_H
0003 #define _LINUX_TRACE_RECURSION_H
0004
0005 #include <linux/interrupt.h>
0006 #include <linux/sched.h>
0007
0008 #ifdef CONFIG_TRACING
0009
0010 /* Only current can touch trace_recursion */
0011
0012 /*
0013  * For function tracing recursion:
0014  *  The order of these bits are important.
0015  *
0016  *  When function tracing occurs, the following steps are made:
0017  *   If arch does not support a ftrace feature:
0018  *    call internal function (uses INTERNAL bits) which calls...
0019  *   The function callback, which can use the FTRACE bits to
0020  *    check for recursion.
0021  */
0022 enum {
0023     /* Function recursion bits */
0024     TRACE_FTRACE_BIT,
0025     TRACE_FTRACE_NMI_BIT,
0026     TRACE_FTRACE_IRQ_BIT,
0027     TRACE_FTRACE_SIRQ_BIT,
0028     TRACE_FTRACE_TRANSITION_BIT,
0029
0030     /* Internal use recursion bits */
0031     TRACE_INTERNAL_BIT,
0032     TRACE_INTERNAL_NMI_BIT,
0033     TRACE_INTERNAL_IRQ_BIT,
0034     TRACE_INTERNAL_SIRQ_BIT,
0035     TRACE_INTERNAL_TRANSITION_BIT,
0036
0037     TRACE_BRANCH_BIT,
0038 /*
0039  * Abuse of the trace_recursion.
0040  * As we need a way to maintain state if we are tracing the function
0041  * graph in irq because we want to trace a particular function that
0042  * was called in irq context but we have irq tracing off. Since this
0043  * can only be modified by current, we can reuse trace_recursion.
0044  */
0045     TRACE_IRQ_BIT,
0046
0047     /* Set if the function is in the set_graph_function file */
0048     TRACE_GRAPH_BIT,
0049
0050     /*
0051      * In the very unlikely case that an interrupt came in
0052      * at a start of graph tracing, and we want to trace
0053      * the function in that interrupt, the depth can be greater
0054      * than zero, because of the preempted start of a previous
0055      * trace. In an even more unlikely case, depth could be 2
0056      * if a softirq interrupted the start of graph tracing,
0057      * followed by an interrupt preempting a start of graph
0058      * tracing in the softirq, and depth can even be 3
0059      * if an NMI came in at the start of an interrupt function
0060      * that preempted a softirq start of a function that
0061      * preempted normal context!!!! Luckily, it can't be
0062      * greater than 3, so the next two bits are a mask
0063      * of what the depth is when we set TRACE_GRAPH_BIT
0064      */
0065
0066     TRACE_GRAPH_DEPTH_START_BIT,
0067     TRACE_GRAPH_DEPTH_END_BIT,
0068
0069     /*
0070      * To implement set_graph_notrace, if this bit is set, we ignore
0071      * function graph tracing of called functions, until the return
0072      * function is called to clear it.
0073      */
0074     TRACE_GRAPH_NOTRACE_BIT,
0075
0076     /* Used to prevent recursion recording from recursing. */
0077     TRACE_RECORD_RECURSION_BIT,
0078 };
0079
0080 #define trace_recursion_set(bit)    do { (current)->trace_recursion |= (1<<(bit)); } while (0)
0081 #define trace_recursion_clear(bit)  do { (current)->trace_recursion &= ~(1<<(bit)); } while (0)
0082 #define trace_recursion_test(bit)   ((current)->trace_recursion & (1<<(bit)))
0083
0084 #define trace_recursion_depth() \
0085     (((current)->trace_recursion >> TRACE_GRAPH_DEPTH_START_BIT) & 3)
0086 #define trace_recursion_set_depth(depth) \
0087     do {                                \
0088         current->trace_recursion &=             \
0089             ~(3 << TRACE_GRAPH_DEPTH_START_BIT);        \
0090         current->trace_recursion |=             \
0091             ((depth) & 3) << TRACE_GRAPH_DEPTH_START_BIT;   \
0092     } while (0)
0093
0094 #define TRACE_CONTEXT_BITS  4
0095
0096 #define TRACE_FTRACE_START  TRACE_FTRACE_BIT
0097
0098 #define TRACE_LIST_START    TRACE_INTERNAL_BIT
0099
0100 #define TRACE_CONTEXT_MASK  ((1 << (TRACE_LIST_START + TRACE_CONTEXT_BITS)) - 1)
0101
0102 /*
0103  * Used for setting context
0104  *  NMI     = 0
0105  *  IRQ     = 1
0106  *  SOFTIRQ = 2
0107  *  NORMAL  = 3
0108  */
0109 enum {
0110     TRACE_CTX_NMI,
0111     TRACE_CTX_IRQ,
0112     TRACE_CTX_SOFTIRQ,
0113     TRACE_CTX_NORMAL,
0114     TRACE_CTX_TRANSITION,
0115 };
0116
0117 static __always_inline int trace_get_context_bit(void)
0118 {
0119     unsigned char bit = interrupt_context_level();
0120
0121     return TRACE_CTX_NORMAL - bit;
0122 }
0123
0124 #ifdef CONFIG_FTRACE_RECORD_RECURSION
0125 extern void ftrace_record_recursion(unsigned long ip, unsigned long parent_ip);
0126 # define do_ftrace_record_recursion(ip, pip)                \
0127     do {                                \
0128         if (!trace_recursion_test(TRACE_RECORD_RECURSION_BIT)) { \
0129             trace_recursion_set(TRACE_RECORD_RECURSION_BIT); \
0130             ftrace_record_recursion(ip, pip);       \
0131             trace_recursion_clear(TRACE_RECORD_RECURSION_BIT); \
0132         }                           \
0133     } while (0)
0134 #else
0135 # define do_ftrace_record_recursion(ip, pip)    do { } while (0)
0136 #endif
0137
0138 /*
0139  * Preemption is promised to be disabled when return bit >= 0.
0140  */
0141 static __always_inline int trace_test_and_set_recursion(unsigned long ip, unsigned long pip,
0142                             int start)
0143 {
0144     unsigned int val = READ_ONCE(current->trace_recursion);
0145     int bit;
0146
0147     bit = trace_get_context_bit() + start;
0148     if (unlikely(val & (1 << bit))) {
0149         /*
0150          * If an interrupt occurs during a trace, and another trace
0151          * happens in that interrupt but before the preempt_count is
0152          * updated to reflect the new interrupt context, then this
0153          * will think a recursion occurred, and the event will be dropped.
0154          * Let a single instance happen via the TRANSITION_BIT to
0155          * not drop those events.
0156          */
0157         bit = TRACE_CTX_TRANSITION + start;
0158         if (val & (1 << bit)) {
0159             do_ftrace_record_recursion(ip, pip);
0160             return -1;
0161         }
0162     }
0163
0164     val |= 1 << bit;
0165     current->trace_recursion = val;
0166     barrier();
0167
0168     preempt_disable_notrace();
0169
0170     return bit;
0171 }
0172
0173 /*
0174  * Preemption will be enabled (if it was previously enabled).
0175  */
0176 static __always_inline void trace_clear_recursion(int bit)
0177 {
0178     preempt_enable_notrace();
0179     barrier();
0180     trace_recursion_clear(bit);
0181 }
0182
0183 /**
0184  * ftrace_test_recursion_trylock - tests for recursion in same context
0185  *
0186  * Use this for ftrace callbacks. This will detect if the function
0187  * tracing recursed in the same context (normal vs interrupt),
0188  *
0189  * Returns: -1 if a recursion happened.
0190  *           >= 0 if no recursion.
0191  */
0192 static __always_inline int ftrace_test_recursion_trylock(unsigned long ip,
0193                              unsigned long parent_ip)
0194 {
0195     return trace_test_and_set_recursion(ip, parent_ip, TRACE_FTRACE_START);
0196 }
0197
0198 /**
0199  * ftrace_test_recursion_unlock - called when function callback is complete
0200  * @bit: The return of a successful ftrace_test_recursion_trylock()
0201  *
0202  * This is used at the end of a ftrace callback.
0203  */
0204 static __always_inline void ftrace_test_recursion_unlock(int bit)
0205 {
0206     trace_clear_recursion(bit);
0207 }
0208
0209 #endif /* CONFIG_TRACING */
0210 #endif /* _LINUX_TRACE_RECURSION_H */