Back to home page

OSCL-LXR
 
 

     

0001 /* SPDX-License-Identifier: GPL-2.0 */
0002 #ifndef _LINUX_JUMP_LABEL_H
0003 #define _LINUX_JUMP_LABEL_H
0004 
0005 /*
0006  * Jump label support
0007  *
0008  * Copyright (C) 2009-2012 Jason Baron <jbaron@redhat.com>
0009  * Copyright (C) 2011-2012 Red Hat, Inc., Peter Zijlstra
0010  *
0011  * DEPRECATED API:
0012  *
0013  * The use of 'struct static_key' directly, is now DEPRECATED. In addition
0014  * static_key_{true,false}() is also DEPRECATED. IE DO NOT use the following:
0015  *
0016  * struct static_key false = STATIC_KEY_INIT_FALSE;
0017  * struct static_key true = STATIC_KEY_INIT_TRUE;
0018  * static_key_true()
0019  * static_key_false()
0020  *
0021  * The updated API replacements are:
0022  *
0023  * DEFINE_STATIC_KEY_TRUE(key);
0024  * DEFINE_STATIC_KEY_FALSE(key);
0025  * DEFINE_STATIC_KEY_ARRAY_TRUE(keys, count);
0026  * DEFINE_STATIC_KEY_ARRAY_FALSE(keys, count);
0027  * static_branch_likely()
0028  * static_branch_unlikely()
0029  *
0030  * Jump labels provide an interface to generate dynamic branches using
0031  * self-modifying code. Assuming toolchain and architecture support, if we
0032  * define a "key" that is initially false via "DEFINE_STATIC_KEY_FALSE(key)",
0033  * an "if (static_branch_unlikely(&key))" statement is an unconditional branch
0034  * (which defaults to false - and the true block is placed out of line).
0035  * Similarly, we can define an initially true key via
0036  * "DEFINE_STATIC_KEY_TRUE(key)", and use it in the same
0037  * "if (static_branch_unlikely(&key))", in which case we will generate an
0038  * unconditional branch to the out-of-line true branch. Keys that are
0039  * initially true or false can be using in both static_branch_unlikely()
0040  * and static_branch_likely() statements.
0041  *
0042  * At runtime we can change the branch target by setting the key
0043  * to true via a call to static_branch_enable(), or false using
0044  * static_branch_disable(). If the direction of the branch is switched by
0045  * these calls then we run-time modify the branch target via a
0046  * no-op -> jump or jump -> no-op conversion. For example, for an
0047  * initially false key that is used in an "if (static_branch_unlikely(&key))"
0048  * statement, setting the key to true requires us to patch in a jump
0049  * to the out-of-line of true branch.
0050  *
0051  * In addition to static_branch_{enable,disable}, we can also reference count
0052  * the key or branch direction via static_branch_{inc,dec}. Thus,
0053  * static_branch_inc() can be thought of as a 'make more true' and
0054  * static_branch_dec() as a 'make more false'.
0055  *
0056  * Since this relies on modifying code, the branch modifying functions
0057  * must be considered absolute slow paths (machine wide synchronization etc.).
0058  * OTOH, since the affected branches are unconditional, their runtime overhead
0059  * will be absolutely minimal, esp. in the default (off) case where the total
0060  * effect is a single NOP of appropriate size. The on case will patch in a jump
0061  * to the out-of-line block.
0062  *
0063  * When the control is directly exposed to userspace, it is prudent to delay the
0064  * decrement to avoid high frequency code modifications which can (and do)
0065  * cause significant performance degradation. Struct static_key_deferred and
0066  * static_key_slow_dec_deferred() provide for this.
0067  *
0068  * Lacking toolchain and or architecture support, static keys fall back to a
0069  * simple conditional branch.
0070  *
0071  * Additional babbling in: Documentation/staging/static-keys.rst
0072  */
0073 
0074 #ifndef __ASSEMBLY__
0075 
0076 #include <linux/types.h>
0077 #include <linux/compiler.h>
0078 
0079 extern bool static_key_initialized;
0080 
0081 #define STATIC_KEY_CHECK_USE(key) WARN(!static_key_initialized,           \
0082                     "%s(): static key '%pS' used before call to jump_label_init()", \
0083                     __func__, (key))
0084 
0085 struct static_key {
0086     atomic_t enabled;
0087 #ifdef CONFIG_JUMP_LABEL
0088 /*
0089  * Note:
0090  *   To make anonymous unions work with old compilers, the static
0091  *   initialization of them requires brackets. This creates a dependency
0092  *   on the order of the struct with the initializers. If any fields
0093  *   are added, STATIC_KEY_INIT_TRUE and STATIC_KEY_INIT_FALSE may need
0094  *   to be modified.
0095  *
0096  * bit 0 => 1 if key is initially true
0097  *      0 if initially false
0098  * bit 1 => 1 if points to struct static_key_mod
0099  *      0 if points to struct jump_entry
0100  */
0101     union {
0102         unsigned long type;
0103         struct jump_entry *entries;
0104         struct static_key_mod *next;
0105     };
0106 #endif  /* CONFIG_JUMP_LABEL */
0107 };
0108 
0109 #endif /* __ASSEMBLY__ */
0110 
0111 #ifdef CONFIG_JUMP_LABEL
0112 #include <asm/jump_label.h>
0113 
0114 #ifndef __ASSEMBLY__
0115 #ifdef CONFIG_HAVE_ARCH_JUMP_LABEL_RELATIVE
0116 
0117 struct jump_entry {
0118     s32 code;
0119     s32 target;
0120     long key;   // key may be far away from the core kernel under KASLR
0121 };
0122 
0123 static inline unsigned long jump_entry_code(const struct jump_entry *entry)
0124 {
0125     return (unsigned long)&entry->code + entry->code;
0126 }
0127 
0128 static inline unsigned long jump_entry_target(const struct jump_entry *entry)
0129 {
0130     return (unsigned long)&entry->target + entry->target;
0131 }
0132 
0133 static inline struct static_key *jump_entry_key(const struct jump_entry *entry)
0134 {
0135     long offset = entry->key & ~3L;
0136 
0137     return (struct static_key *)((unsigned long)&entry->key + offset);
0138 }
0139 
0140 #else
0141 
0142 static inline unsigned long jump_entry_code(const struct jump_entry *entry)
0143 {
0144     return entry->code;
0145 }
0146 
0147 static inline unsigned long jump_entry_target(const struct jump_entry *entry)
0148 {
0149     return entry->target;
0150 }
0151 
0152 static inline struct static_key *jump_entry_key(const struct jump_entry *entry)
0153 {
0154     return (struct static_key *)((unsigned long)entry->key & ~3UL);
0155 }
0156 
0157 #endif
0158 
0159 static inline bool jump_entry_is_branch(const struct jump_entry *entry)
0160 {
0161     return (unsigned long)entry->key & 1UL;
0162 }
0163 
0164 static inline bool jump_entry_is_init(const struct jump_entry *entry)
0165 {
0166     return (unsigned long)entry->key & 2UL;
0167 }
0168 
0169 static inline void jump_entry_set_init(struct jump_entry *entry, bool set)
0170 {
0171     if (set)
0172         entry->key |= 2;
0173     else
0174         entry->key &= ~2;
0175 }
0176 
0177 static inline int jump_entry_size(struct jump_entry *entry)
0178 {
0179 #ifdef JUMP_LABEL_NOP_SIZE
0180     return JUMP_LABEL_NOP_SIZE;
0181 #else
0182     return arch_jump_entry_size(entry);
0183 #endif
0184 }
0185 
0186 #endif
0187 #endif
0188 
0189 #ifndef __ASSEMBLY__
0190 
0191 enum jump_label_type {
0192     JUMP_LABEL_NOP = 0,
0193     JUMP_LABEL_JMP,
0194 };
0195 
0196 struct module;
0197 
0198 #ifdef CONFIG_JUMP_LABEL
0199 
0200 #define JUMP_TYPE_FALSE     0UL
0201 #define JUMP_TYPE_TRUE      1UL
0202 #define JUMP_TYPE_LINKED    2UL
0203 #define JUMP_TYPE_MASK      3UL
0204 
0205 static __always_inline bool static_key_false(struct static_key *key)
0206 {
0207     return arch_static_branch(key, false);
0208 }
0209 
0210 static __always_inline bool static_key_true(struct static_key *key)
0211 {
0212     return !arch_static_branch(key, true);
0213 }
0214 
0215 extern struct jump_entry __start___jump_table[];
0216 extern struct jump_entry __stop___jump_table[];
0217 
0218 extern void jump_label_init(void);
0219 extern void jump_label_lock(void);
0220 extern void jump_label_unlock(void);
0221 extern void arch_jump_label_transform(struct jump_entry *entry,
0222                       enum jump_label_type type);
0223 extern bool arch_jump_label_transform_queue(struct jump_entry *entry,
0224                         enum jump_label_type type);
0225 extern void arch_jump_label_transform_apply(void);
0226 extern int jump_label_text_reserved(void *start, void *end);
0227 extern void static_key_slow_inc(struct static_key *key);
0228 extern void static_key_slow_dec(struct static_key *key);
0229 extern void static_key_slow_inc_cpuslocked(struct static_key *key);
0230 extern void static_key_slow_dec_cpuslocked(struct static_key *key);
0231 extern int static_key_count(struct static_key *key);
0232 extern void static_key_enable(struct static_key *key);
0233 extern void static_key_disable(struct static_key *key);
0234 extern void static_key_enable_cpuslocked(struct static_key *key);
0235 extern void static_key_disable_cpuslocked(struct static_key *key);
0236 extern enum jump_label_type jump_label_init_type(struct jump_entry *entry);
0237 
0238 /*
0239  * We should be using ATOMIC_INIT() for initializing .enabled, but
0240  * the inclusion of atomic.h is problematic for inclusion of jump_label.h
0241  * in 'low-level' headers. Thus, we are initializing .enabled with a
0242  * raw value, but have added a BUILD_BUG_ON() to catch any issues in
0243  * jump_label_init() see: kernel/jump_label.c.
0244  */
0245 #define STATIC_KEY_INIT_TRUE                    \
0246     { .enabled = { 1 },                 \
0247       { .type = JUMP_TYPE_TRUE } }
0248 #define STATIC_KEY_INIT_FALSE                   \
0249     { .enabled = { 0 },                 \
0250       { .type = JUMP_TYPE_FALSE } }
0251 
0252 #else  /* !CONFIG_JUMP_LABEL */
0253 
0254 #include <linux/atomic.h>
0255 #include <linux/bug.h>
0256 
0257 static __always_inline int static_key_count(struct static_key *key)
0258 {
0259     return arch_atomic_read(&key->enabled);
0260 }
0261 
0262 static __always_inline void jump_label_init(void)
0263 {
0264     static_key_initialized = true;
0265 }
0266 
0267 static __always_inline bool static_key_false(struct static_key *key)
0268 {
0269     if (unlikely_notrace(static_key_count(key) > 0))
0270         return true;
0271     return false;
0272 }
0273 
0274 static __always_inline bool static_key_true(struct static_key *key)
0275 {
0276     if (likely_notrace(static_key_count(key) > 0))
0277         return true;
0278     return false;
0279 }
0280 
0281 static inline void static_key_slow_inc(struct static_key *key)
0282 {
0283     STATIC_KEY_CHECK_USE(key);
0284     atomic_inc(&key->enabled);
0285 }
0286 
0287 static inline void static_key_slow_dec(struct static_key *key)
0288 {
0289     STATIC_KEY_CHECK_USE(key);
0290     atomic_dec(&key->enabled);
0291 }
0292 
0293 #define static_key_slow_inc_cpuslocked(key) static_key_slow_inc(key)
0294 #define static_key_slow_dec_cpuslocked(key) static_key_slow_dec(key)
0295 
0296 static inline int jump_label_text_reserved(void *start, void *end)
0297 {
0298     return 0;
0299 }
0300 
0301 static inline void jump_label_lock(void) {}
0302 static inline void jump_label_unlock(void) {}
0303 
0304 static inline void static_key_enable(struct static_key *key)
0305 {
0306     STATIC_KEY_CHECK_USE(key);
0307 
0308     if (atomic_read(&key->enabled) != 0) {
0309         WARN_ON_ONCE(atomic_read(&key->enabled) != 1);
0310         return;
0311     }
0312     atomic_set(&key->enabled, 1);
0313 }
0314 
0315 static inline void static_key_disable(struct static_key *key)
0316 {
0317     STATIC_KEY_CHECK_USE(key);
0318 
0319     if (atomic_read(&key->enabled) != 1) {
0320         WARN_ON_ONCE(atomic_read(&key->enabled) != 0);
0321         return;
0322     }
0323     atomic_set(&key->enabled, 0);
0324 }
0325 
0326 #define static_key_enable_cpuslocked(k)     static_key_enable((k))
0327 #define static_key_disable_cpuslocked(k)    static_key_disable((k))
0328 
0329 #define STATIC_KEY_INIT_TRUE    { .enabled = ATOMIC_INIT(1) }
0330 #define STATIC_KEY_INIT_FALSE   { .enabled = ATOMIC_INIT(0) }
0331 
0332 #endif  /* CONFIG_JUMP_LABEL */
0333 
0334 #define STATIC_KEY_INIT STATIC_KEY_INIT_FALSE
0335 #define jump_label_enabled static_key_enabled
0336 
0337 /* -------------------------------------------------------------------------- */
0338 
0339 /*
0340  * Two type wrappers around static_key, such that we can use compile time
0341  * type differentiation to emit the right code.
0342  *
0343  * All the below code is macros in order to play type games.
0344  */
0345 
0346 struct static_key_true {
0347     struct static_key key;
0348 };
0349 
0350 struct static_key_false {
0351     struct static_key key;
0352 };
0353 
0354 #define STATIC_KEY_TRUE_INIT  (struct static_key_true) { .key = STATIC_KEY_INIT_TRUE,  }
0355 #define STATIC_KEY_FALSE_INIT (struct static_key_false){ .key = STATIC_KEY_INIT_FALSE, }
0356 
0357 #define DEFINE_STATIC_KEY_TRUE(name)    \
0358     struct static_key_true name = STATIC_KEY_TRUE_INIT
0359 
0360 #define DEFINE_STATIC_KEY_TRUE_RO(name) \
0361     struct static_key_true name __ro_after_init = STATIC_KEY_TRUE_INIT
0362 
0363 #define DECLARE_STATIC_KEY_TRUE(name)   \
0364     extern struct static_key_true name
0365 
0366 #define DEFINE_STATIC_KEY_FALSE(name)   \
0367     struct static_key_false name = STATIC_KEY_FALSE_INIT
0368 
0369 #define DEFINE_STATIC_KEY_FALSE_RO(name)    \
0370     struct static_key_false name __ro_after_init = STATIC_KEY_FALSE_INIT
0371 
0372 #define DECLARE_STATIC_KEY_FALSE(name)  \
0373     extern struct static_key_false name
0374 
0375 #define DEFINE_STATIC_KEY_ARRAY_TRUE(name, count)       \
0376     struct static_key_true name[count] = {          \
0377         [0 ... (count) - 1] = STATIC_KEY_TRUE_INIT, \
0378     }
0379 
0380 #define DEFINE_STATIC_KEY_ARRAY_FALSE(name, count)      \
0381     struct static_key_false name[count] = {         \
0382         [0 ... (count) - 1] = STATIC_KEY_FALSE_INIT,    \
0383     }
0384 
0385 #define _DEFINE_STATIC_KEY_1(name)  DEFINE_STATIC_KEY_TRUE(name)
0386 #define _DEFINE_STATIC_KEY_0(name)  DEFINE_STATIC_KEY_FALSE(name)
0387 #define DEFINE_STATIC_KEY_MAYBE(cfg, name)          \
0388     __PASTE(_DEFINE_STATIC_KEY_, IS_ENABLED(cfg))(name)
0389 
0390 #define _DEFINE_STATIC_KEY_RO_1(name)   DEFINE_STATIC_KEY_TRUE_RO(name)
0391 #define _DEFINE_STATIC_KEY_RO_0(name)   DEFINE_STATIC_KEY_FALSE_RO(name)
0392 #define DEFINE_STATIC_KEY_MAYBE_RO(cfg, name)           \
0393     __PASTE(_DEFINE_STATIC_KEY_RO_, IS_ENABLED(cfg))(name)
0394 
0395 #define _DECLARE_STATIC_KEY_1(name) DECLARE_STATIC_KEY_TRUE(name)
0396 #define _DECLARE_STATIC_KEY_0(name) DECLARE_STATIC_KEY_FALSE(name)
0397 #define DECLARE_STATIC_KEY_MAYBE(cfg, name)         \
0398     __PASTE(_DECLARE_STATIC_KEY_, IS_ENABLED(cfg))(name)
0399 
0400 extern bool ____wrong_branch_error(void);
0401 
0402 #define static_key_enabled(x)                           \
0403 ({                                      \
0404     if (!__builtin_types_compatible_p(typeof(*x), struct static_key) && \
0405         !__builtin_types_compatible_p(typeof(*x), struct static_key_true) &&\
0406         !__builtin_types_compatible_p(typeof(*x), struct static_key_false)) \
0407         ____wrong_branch_error();                   \
0408     static_key_count((struct static_key *)x) > 0;               \
0409 })
0410 
0411 #ifdef CONFIG_JUMP_LABEL
0412 
0413 /*
0414  * Combine the right initial value (type) with the right branch order
0415  * to generate the desired result.
0416  *
0417  *
0418  * type\branch| likely (1)        | unlikely (0)
0419  * -----------+-----------------------+------------------
0420  *            |                       |
0421  *  true (1)  |    ...            |    ...
0422  *            |    NOP            |    JMP L
0423  *            |    <br-stmts>         | 1: ...
0424  *            | L: ...            |
0425  *            |               |
0426  *            |               | L: <br-stmts>
0427  *            |               |    jmp 1b
0428  *            |                       |
0429  * -----------+-----------------------+------------------
0430  *            |                       |
0431  *  false (0) |    ...            |    ...
0432  *            |    JMP L          |    NOP
0433  *            |    <br-stmts>         | 1: ...
0434  *            | L: ...            |
0435  *            |               |
0436  *            |               | L: <br-stmts>
0437  *            |               |    jmp 1b
0438  *            |                       |
0439  * -----------+-----------------------+------------------
0440  *
0441  * The initial value is encoded in the LSB of static_key::entries,
0442  * type: 0 = false, 1 = true.
0443  *
0444  * The branch type is encoded in the LSB of jump_entry::key,
0445  * branch: 0 = unlikely, 1 = likely.
0446  *
0447  * This gives the following logic table:
0448  *
0449  *  enabled type    branch    instuction
0450  * -----------------------------+-----------
0451  *  0   0   0   | NOP
0452  *  0   0   1   | JMP
0453  *  0   1   0   | NOP
0454  *  0   1   1   | JMP
0455  *
0456  *  1   0   0   | JMP
0457  *  1   0   1   | NOP
0458  *  1   1   0   | JMP
0459  *  1   1   1   | NOP
0460  *
0461  * Which gives the following functions:
0462  *
0463  *   dynamic: instruction = enabled ^ branch
0464  *   static:  instruction = type ^ branch
0465  *
0466  * See jump_label_type() / jump_label_init_type().
0467  */
0468 
0469 #define static_branch_likely(x)                         \
0470 ({                                      \
0471     bool branch;                                \
0472     if (__builtin_types_compatible_p(typeof(*x), struct static_key_true))   \
0473         branch = !arch_static_branch(&(x)->key, true);          \
0474     else if (__builtin_types_compatible_p(typeof(*x), struct static_key_false)) \
0475         branch = !arch_static_branch_jump(&(x)->key, true);     \
0476     else                                    \
0477         branch = ____wrong_branch_error();              \
0478     likely_notrace(branch);                             \
0479 })
0480 
0481 #define static_branch_unlikely(x)                       \
0482 ({                                      \
0483     bool branch;                                \
0484     if (__builtin_types_compatible_p(typeof(*x), struct static_key_true))   \
0485         branch = arch_static_branch_jump(&(x)->key, false);     \
0486     else if (__builtin_types_compatible_p(typeof(*x), struct static_key_false)) \
0487         branch = arch_static_branch(&(x)->key, false);          \
0488     else                                    \
0489         branch = ____wrong_branch_error();              \
0490     unlikely_notrace(branch);                           \
0491 })
0492 
0493 #else /* !CONFIG_JUMP_LABEL */
0494 
0495 #define static_branch_likely(x)     likely_notrace(static_key_enabled(&(x)->key))
0496 #define static_branch_unlikely(x)   unlikely_notrace(static_key_enabled(&(x)->key))
0497 
0498 #endif /* CONFIG_JUMP_LABEL */
0499 
0500 #define static_branch_maybe(config, x)                  \
0501     (IS_ENABLED(config) ? static_branch_likely(x)           \
0502                 : static_branch_unlikely(x))
0503 
0504 /*
0505  * Advanced usage; refcount, branch is enabled when: count != 0
0506  */
0507 
0508 #define static_branch_inc(x)        static_key_slow_inc(&(x)->key)
0509 #define static_branch_dec(x)        static_key_slow_dec(&(x)->key)
0510 #define static_branch_inc_cpuslocked(x) static_key_slow_inc_cpuslocked(&(x)->key)
0511 #define static_branch_dec_cpuslocked(x) static_key_slow_dec_cpuslocked(&(x)->key)
0512 
0513 /*
0514  * Normal usage; boolean enable/disable.
0515  */
0516 
0517 #define static_branch_enable(x)         static_key_enable(&(x)->key)
0518 #define static_branch_disable(x)        static_key_disable(&(x)->key)
0519 #define static_branch_enable_cpuslocked(x)  static_key_enable_cpuslocked(&(x)->key)
0520 #define static_branch_disable_cpuslocked(x) static_key_disable_cpuslocked(&(x)->key)
0521 
0522 #endif /* __ASSEMBLY__ */
0523 
0524 #endif  /* _LINUX_JUMP_LABEL_H */
[ Source navigation ] [ Diff markup ] [ Identifier search ] [ general search ]

This page was automatically generated by the 2.1.0 LXR engine.

The LXR team
Valid CSS 2.1! Valid HTML 4.01!