include/linux/context_tracking.h

0001 /* SPDX-License-Identifier: GPL-2.0 */
0002 #ifndef _LINUX_CONTEXT_TRACKING_H
0003 #define _LINUX_CONTEXT_TRACKING_H
0004
0005 #include <linux/sched.h>
0006 #include <linux/vtime.h>
0007 #include <linux/context_tracking_state.h>
0008 #include <linux/instrumentation.h>
0009
0010 #include <asm/ptrace.h>
0011
0012
0013 #ifdef CONFIG_CONTEXT_TRACKING_USER
0014 extern void ct_cpu_track_user(int cpu);
0015
0016 /* Called with interrupts disabled.  */
0017 extern void __ct_user_enter(enum ctx_state state);
0018 extern void __ct_user_exit(enum ctx_state state);
0019
0020 extern void ct_user_enter(enum ctx_state state);
0021 extern void ct_user_exit(enum ctx_state state);
0022
0023 extern void user_enter_callable(void);
0024 extern void user_exit_callable(void);
0025
0026 static inline void user_enter(void)
0027 {
0028     if (context_tracking_enabled())
0029         ct_user_enter(CONTEXT_USER);
0030
0031 }
0032 static inline void user_exit(void)
0033 {
0034     if (context_tracking_enabled())
0035         ct_user_exit(CONTEXT_USER);
0036 }
0037
0038 /* Called with interrupts disabled.  */
0039 static __always_inline void user_enter_irqoff(void)
0040 {
0041     if (context_tracking_enabled())
0042         __ct_user_enter(CONTEXT_USER);
0043
0044 }
0045 static __always_inline void user_exit_irqoff(void)
0046 {
0047     if (context_tracking_enabled())
0048         __ct_user_exit(CONTEXT_USER);
0049 }
0050
0051 static inline enum ctx_state exception_enter(void)
0052 {
0053     enum ctx_state prev_ctx;
0054
0055     if (IS_ENABLED(CONFIG_HAVE_CONTEXT_TRACKING_USER_OFFSTACK) ||
0056         !context_tracking_enabled())
0057         return 0;
0058
0059     prev_ctx = __ct_state();
0060     if (prev_ctx != CONTEXT_KERNEL)
0061         ct_user_exit(prev_ctx);
0062
0063     return prev_ctx;
0064 }
0065
0066 static inline void exception_exit(enum ctx_state prev_ctx)
0067 {
0068     if (!IS_ENABLED(CONFIG_HAVE_CONTEXT_TRACKING_USER_OFFSTACK) &&
0069         context_tracking_enabled()) {
0070         if (prev_ctx != CONTEXT_KERNEL)
0071             ct_user_enter(prev_ctx);
0072     }
0073 }
0074
0075 static __always_inline bool context_tracking_guest_enter(void)
0076 {
0077     if (context_tracking_enabled())
0078         __ct_user_enter(CONTEXT_GUEST);
0079
0080     return context_tracking_enabled_this_cpu();
0081 }
0082
0083 static __always_inline void context_tracking_guest_exit(void)
0084 {
0085     if (context_tracking_enabled())
0086         __ct_user_exit(CONTEXT_GUEST);
0087 }
0088
0089 #define CT_WARN_ON(cond) WARN_ON(context_tracking_enabled() && (cond))
0090
0091 #else
0092 static inline void user_enter(void) { }
0093 static inline void user_exit(void) { }
0094 static inline void user_enter_irqoff(void) { }
0095 static inline void user_exit_irqoff(void) { }
0096 static inline int exception_enter(void) { return 0; }
0097 static inline void exception_exit(enum ctx_state prev_ctx) { }
0098 static inline int ct_state(void) { return -1; }
0099 static __always_inline bool context_tracking_guest_enter(void) { return false; }
0100 static inline void context_tracking_guest_exit(void) { }
0101 #define CT_WARN_ON(cond) do { } while (0)
0102 #endif /* !CONFIG_CONTEXT_TRACKING_USER */
0103
0104 #ifdef CONFIG_CONTEXT_TRACKING_USER_FORCE
0105 extern void context_tracking_init(void);
0106 #else
0107 static inline void context_tracking_init(void) { }
0108 #endif /* CONFIG_CONTEXT_TRACKING_USER_FORCE */
0109
0110 #ifdef CONFIG_CONTEXT_TRACKING_IDLE
0111 extern void ct_idle_enter(void);
0112 extern void ct_idle_exit(void);
0113
0114 /*
0115  * Is the current CPU in an extended quiescent state?
0116  *
0117  * No ordering, as we are sampling CPU-local information.
0118  */
0119 static __always_inline bool rcu_dynticks_curr_cpu_in_eqs(void)
0120 {
0121     return !(arch_atomic_read(this_cpu_ptr(&context_tracking.state)) & RCU_DYNTICKS_IDX);
0122 }
0123
0124 /*
0125  * Increment the current CPU's context_tracking structure's ->state field
0126  * with ordering.  Return the new value.
0127  */
0128 static __always_inline unsigned long ct_state_inc(int incby)
0129 {
0130     return arch_atomic_add_return(incby, this_cpu_ptr(&context_tracking.state));
0131 }
0132
0133 #else
0134 static inline void ct_idle_enter(void) { }
0135 static inline void ct_idle_exit(void) { }
0136 #endif /* !CONFIG_CONTEXT_TRACKING_IDLE */
0137
0138 #endif