Back to home page

LXR

 
 

    


0001 #ifndef _ASM_X86_JUMP_LABEL_H
0002 #define _ASM_X86_JUMP_LABEL_H
0003 
0004 #ifndef HAVE_JUMP_LABEL
0005 /*
0006  * For better or for worse, if jump labels (the gcc extension) are missing,
0007  * then the entire static branch patching infrastructure is compiled out.
0008  * If that happens, the code in here will malfunction.  Raise a compiler
0009  * error instead.
0010  *
0011  * In theory, jump labels and the static branch patching infrastructure
0012  * could be decoupled to fix this.
0013  */
0014 #error asm/jump_label.h included on a non-jump-label kernel
0015 #endif
0016 
0017 #define JUMP_LABEL_NOP_SIZE 5
0018 
0019 #ifdef CONFIG_X86_64
0020 # define STATIC_KEY_INIT_NOP P6_NOP5_ATOMIC
0021 #else
0022 # define STATIC_KEY_INIT_NOP GENERIC_NOP5_ATOMIC
0023 #endif
0024 
0025 #include <asm/asm.h>
0026 #include <asm/nops.h>
0027 
0028 #ifndef __ASSEMBLY__
0029 
0030 #include <linux/stringify.h>
0031 #include <linux/types.h>
0032 
0033 static __always_inline bool arch_static_branch(struct static_key *key, bool branch)
0034 {
0035     asm_volatile_goto("1:"
0036         ".byte " __stringify(STATIC_KEY_INIT_NOP) "\n\t"
0037         ".pushsection __jump_table,  \"aw\" \n\t"
0038         _ASM_ALIGN "\n\t"
0039         _ASM_PTR "1b, %l[l_yes], %c0 + %c1 \n\t"
0040         ".popsection \n\t"
0041         : :  "i" (key), "i" (branch) : : l_yes);
0042 
0043     return false;
0044 l_yes:
0045     return true;
0046 }
0047 
0048 static __always_inline bool arch_static_branch_jump(struct static_key *key, bool branch)
0049 {
0050     asm_volatile_goto("1:"
0051         ".byte 0xe9\n\t .long %l[l_yes] - 2f\n\t"
0052         "2:\n\t"
0053         ".pushsection __jump_table,  \"aw\" \n\t"
0054         _ASM_ALIGN "\n\t"
0055         _ASM_PTR "1b, %l[l_yes], %c0 + %c1 \n\t"
0056         ".popsection \n\t"
0057         : :  "i" (key), "i" (branch) : : l_yes);
0058 
0059     return false;
0060 l_yes:
0061     return true;
0062 }
0063 
0064 #ifdef CONFIG_X86_64
0065 typedef u64 jump_label_t;
0066 #else
0067 typedef u32 jump_label_t;
0068 #endif
0069 
0070 struct jump_entry {
0071     jump_label_t code;
0072     jump_label_t target;
0073     jump_label_t key;
0074 };
0075 
0076 #else   /* __ASSEMBLY__ */
0077 
0078 .macro STATIC_JUMP_IF_TRUE target, key, def
0079 .Lstatic_jump_\@:
0080     .if \def
0081     /* Equivalent to "jmp.d32 \target" */
0082     .byte       0xe9
0083     .long       \target - .Lstatic_jump_after_\@
0084 .Lstatic_jump_after_\@:
0085     .else
0086     .byte       STATIC_KEY_INIT_NOP
0087     .endif
0088     .pushsection __jump_table, "aw"
0089     _ASM_ALIGN
0090     _ASM_PTR    .Lstatic_jump_\@, \target, \key
0091     .popsection
0092 .endm
0093 
0094 .macro STATIC_JUMP_IF_FALSE target, key, def
0095 .Lstatic_jump_\@:
0096     .if \def
0097     .byte       STATIC_KEY_INIT_NOP
0098     .else
0099     /* Equivalent to "jmp.d32 \target" */
0100     .byte       0xe9
0101     .long       \target - .Lstatic_jump_after_\@
0102 .Lstatic_jump_after_\@:
0103     .endif
0104     .pushsection __jump_table, "aw"
0105     _ASM_ALIGN
0106     _ASM_PTR    .Lstatic_jump_\@, \target, \key + 1
0107     .popsection
0108 .endm
0109 
0110 #endif  /* __ASSEMBLY__ */
0111 
0112 #endif