include/asm/alternative.h

0001 /* SPDX-License-Identifier: GPL-2.0 */
0002 #ifndef _ASM_X86_ALTERNATIVE_H
0003 #define _ASM_X86_ALTERNATIVE_H
0004
0005 #include <linux/types.h>
0006 #include <linux/stringify.h>
0007 #include <asm/asm.h>
0008
0009 #define ALTINSTR_FLAG_INV   (1 << 15)
0010 #define ALT_NOT(feat)       ((feat) | ALTINSTR_FLAG_INV)
0011
0012 #ifndef __ASSEMBLY__
0013
0014 #include <linux/stddef.h>
0015
0016 /*
0017  * Alternative inline assembly for SMP.
0018  *
0019  * The LOCK_PREFIX macro defined here replaces the LOCK and
0020  * LOCK_PREFIX macros used everywhere in the source tree.
0021  *
0022  * SMP alternatives use the same data structures as the other
0023  * alternatives and the X86_FEATURE_UP flag to indicate the case of a
0024  * UP system running a SMP kernel.  The existing apply_alternatives()
0025  * works fine for patching a SMP kernel for UP.
0026  *
0027  * The SMP alternative tables can be kept after boot and contain both
0028  * UP and SMP versions of the instructions to allow switching back to
0029  * SMP at runtime, when hotplugging in a new CPU, which is especially
0030  * useful in virtualized environments.
0031  *
0032  * The very common lock prefix is handled as special case in a
0033  * separate table which is a pure address list without replacement ptr
0034  * and size information.  That keeps the table sizes small.
0035  */
0036
0037 #ifdef CONFIG_SMP
0038 #define LOCK_PREFIX_HERE \
0039         ".pushsection .smp_locks,\"a\"\n"   \
0040         ".balign 4\n"               \
0041         ".long 671f - .\n" /* offset */     \
0042         ".popsection\n"             \
0043         "671:"
0044
0045 #define LOCK_PREFIX LOCK_PREFIX_HERE "\n\tlock; "
0046
0047 #else /* ! CONFIG_SMP */
0048 #define LOCK_PREFIX_HERE ""
0049 #define LOCK_PREFIX ""
0050 #endif
0051
0052 /*
0053  * objtool annotation to ignore the alternatives and only consider the original
0054  * instruction(s).
0055  */
0056 #define ANNOTATE_IGNORE_ALTERNATIVE             \
0057     "999:\n\t"                      \
0058     ".pushsection .discard.ignore_alts\n\t"         \
0059     ".long 999b - .\n\t"                    \
0060     ".popsection\n\t"
0061
0062 struct alt_instr {
0063     s32 instr_offset;   /* original instruction */
0064     s32 repl_offset;    /* offset to replacement instruction */
0065     u16 cpuid;      /* cpuid bit set for replacement */
0066     u8  instrlen;       /* length of original instruction */
0067     u8  replacementlen; /* length of new instruction */
0068 } __packed;
0069
0070 /*
0071  * Debug flag that can be tested to see whether alternative
0072  * instructions were patched in already:
0073  */
0074 extern int alternatives_patched;
0075
0076 extern void alternative_instructions(void);
0077 extern void apply_alternatives(struct alt_instr *start, struct alt_instr *end);
0078 extern void apply_retpolines(s32 *start, s32 *end);
0079 extern void apply_returns(s32 *start, s32 *end);
0080 extern void apply_ibt_endbr(s32 *start, s32 *end);
0081
0082 struct module;
0083
0084 #ifdef CONFIG_SMP
0085 extern void alternatives_smp_module_add(struct module *mod, char *name,
0086                     void *locks, void *locks_end,
0087                     void *text, void *text_end);
0088 extern void alternatives_smp_module_del(struct module *mod);
0089 extern void alternatives_enable_smp(void);
0090 extern int alternatives_text_reserved(void *start, void *end);
0091 extern bool skip_smp_alternatives;
0092 #else
0093 static inline void alternatives_smp_module_add(struct module *mod, char *name,
0094                            void *locks, void *locks_end,
0095                            void *text, void *text_end) {}
0096 static inline void alternatives_smp_module_del(struct module *mod) {}
0097 static inline void alternatives_enable_smp(void) {}
0098 static inline int alternatives_text_reserved(void *start, void *end)
0099 {
0100     return 0;
0101 }
0102 #endif  /* CONFIG_SMP */
0103
0104 #define b_replacement(num)  "664"#num
0105 #define e_replacement(num)  "665"#num
0106
0107 #define alt_end_marker      "663"
0108 #define alt_slen        "662b-661b"
0109 #define alt_total_slen      alt_end_marker"b-661b"
0110 #define alt_rlen(num)       e_replacement(num)"f-"b_replacement(num)"f"
0111
0112 #define OLDINSTR(oldinstr, num)                     \
0113     "# ALT: oldnstr\n"                      \
0114     "661:\n\t" oldinstr "\n662:\n"                  \
0115     "# ALT: padding\n"                      \
0116     ".skip -(((" alt_rlen(num) ")-(" alt_slen ")) > 0) * "      \
0117         "((" alt_rlen(num) ")-(" alt_slen ")),0x90\n"       \
0118     alt_end_marker ":\n"
0119
0120 /*
0121  * gas compatible max based on the idea from:
0122  * http://graphics.stanford.edu/~seander/bithacks.html#IntegerMinOrMax
0123  *
0124  * The additional "-" is needed because gas uses a "true" value of -1.
0125  */
0126 #define alt_max_short(a, b) "((" a ") ^ (((" a ") ^ (" b ")) & -(-((" a ") < (" b ")))))"
0127
0128 /*
0129  * Pad the second replacement alternative with additional NOPs if it is
0130  * additionally longer than the first replacement alternative.
0131  */
0132 #define OLDINSTR_2(oldinstr, num1, num2) \
0133     "# ALT: oldinstr2\n"                                    \
0134     "661:\n\t" oldinstr "\n662:\n"                              \
0135     "# ALT: padding2\n"                                 \
0136     ".skip -((" alt_max_short(alt_rlen(num1), alt_rlen(num2)) " - (" alt_slen ")) > 0) * "  \
0137         "(" alt_max_short(alt_rlen(num1), alt_rlen(num2)) " - (" alt_slen ")), 0x90\n"  \
0138     alt_end_marker ":\n"
0139
0140 #define OLDINSTR_3(oldinsn, n1, n2, n3)                             \
0141     "# ALT: oldinstr3\n"                                    \
0142     "661:\n\t" oldinsn "\n662:\n"                               \
0143     "# ALT: padding3\n"                                 \
0144     ".skip -((" alt_max_short(alt_max_short(alt_rlen(n1), alt_rlen(n2)), alt_rlen(n3))  \
0145         " - (" alt_slen ")) > 0) * "                            \
0146         "(" alt_max_short(alt_max_short(alt_rlen(n1), alt_rlen(n2)), alt_rlen(n3))  \
0147         " - (" alt_slen ")), 0x90\n"                            \
0148     alt_end_marker ":\n"
0149
0150 #define ALTINSTR_ENTRY(feature, num)                          \
0151     " .long 661b - .\n"             /* label           */ \
0152     " .long " b_replacement(num)"f - .\n"       /* new instruction */ \
0153     " .word " __stringify(feature) "\n"     /* feature bit     */ \
0154     " .byte " alt_total_slen "\n"           /* source len      */ \
0155     " .byte " alt_rlen(num) "\n"            /* replacement len */
0156
0157 #define ALTINSTR_REPLACEMENT(newinstr, num)     /* replacement */   \
0158     "# ALT: replacement " #num "\n"                     \
0159     b_replacement(num)":\n\t" newinstr "\n" e_replacement(num) ":\n"
0160
0161 /* alternative assembly primitive: */
0162 #define ALTERNATIVE(oldinstr, newinstr, feature)            \
0163     OLDINSTR(oldinstr, 1)                       \
0164     ".pushsection .altinstructions,\"a\"\n"             \
0165     ALTINSTR_ENTRY(feature, 1)                  \
0166     ".popsection\n"                         \
0167     ".pushsection .altinstr_replacement, \"ax\"\n"          \
0168     ALTINSTR_REPLACEMENT(newinstr, 1)               \
0169     ".popsection\n"
0170
0171 #define ALTERNATIVE_2(oldinstr, newinstr1, feature1, newinstr2, feature2)\
0172     OLDINSTR_2(oldinstr, 1, 2)                  \
0173     ".pushsection .altinstructions,\"a\"\n"             \
0174     ALTINSTR_ENTRY(feature1, 1)                 \
0175     ALTINSTR_ENTRY(feature2, 2)                 \
0176     ".popsection\n"                         \
0177     ".pushsection .altinstr_replacement, \"ax\"\n"          \
0178     ALTINSTR_REPLACEMENT(newinstr1, 1)              \
0179     ALTINSTR_REPLACEMENT(newinstr2, 2)              \
0180     ".popsection\n"
0181
0182 /* If @feature is set, patch in @newinstr_yes, otherwise @newinstr_no. */
0183 #define ALTERNATIVE_TERNARY(oldinstr, feature, newinstr_yes, newinstr_no) \
0184     ALTERNATIVE_2(oldinstr, newinstr_no, X86_FEATURE_ALWAYS,    \
0185               newinstr_yes, feature)
0186
0187 #define ALTERNATIVE_3(oldinsn, newinsn1, feat1, newinsn2, feat2, newinsn3, feat3) \
0188     OLDINSTR_3(oldinsn, 1, 2, 3)                        \
0189     ".pushsection .altinstructions,\"a\"\n"                 \
0190     ALTINSTR_ENTRY(feat1, 1)                        \
0191     ALTINSTR_ENTRY(feat2, 2)                        \
0192     ALTINSTR_ENTRY(feat3, 3)                        \
0193     ".popsection\n"                             \
0194     ".pushsection .altinstr_replacement, \"ax\"\n"              \
0195     ALTINSTR_REPLACEMENT(newinsn1, 1)                   \
0196     ALTINSTR_REPLACEMENT(newinsn2, 2)                   \
0197     ALTINSTR_REPLACEMENT(newinsn3, 3)                   \
0198     ".popsection\n"
0199
0200 /*
0201  * Alternative instructions for different CPU types or capabilities.
0202  *
0203  * This allows to use optimized instructions even on generic binary
0204  * kernels.
0205  *
0206  * length of oldinstr must be longer or equal the length of newinstr
0207  * It can be padded with nops as needed.
0208  *
0209  * For non barrier like inlines please define new variants
0210  * without volatile and memory clobber.
0211  */
0212 #define alternative(oldinstr, newinstr, feature)            \
0213     asm_inline volatile (ALTERNATIVE(oldinstr, newinstr, feature) : : : "memory")
0214
0215 #define alternative_2(oldinstr, newinstr1, feature1, newinstr2, feature2) \
0216     asm_inline volatile(ALTERNATIVE_2(oldinstr, newinstr1, feature1, newinstr2, feature2) ::: "memory")
0217
0218 #define alternative_ternary(oldinstr, feature, newinstr_yes, newinstr_no) \
0219     asm_inline volatile(ALTERNATIVE_TERNARY(oldinstr, feature, newinstr_yes, newinstr_no) ::: "memory")
0220
0221 /*
0222  * Alternative inline assembly with input.
0223  *
0224  * Peculiarities:
0225  * No memory clobber here.
0226  * Argument numbers start with 1.
0227  * Leaving an unused argument 0 to keep API compatibility.
0228  */
0229 #define alternative_input(oldinstr, newinstr, feature, input...)    \
0230     asm_inline volatile (ALTERNATIVE(oldinstr, newinstr, feature)   \
0231         : : "i" (0), ## input)
0232
0233 /*
0234  * This is similar to alternative_input. But it has two features and
0235  * respective instructions.
0236  *
0237  * If CPU has feature2, newinstr2 is used.
0238  * Otherwise, if CPU has feature1, newinstr1 is used.
0239  * Otherwise, oldinstr is used.
0240  */
0241 #define alternative_input_2(oldinstr, newinstr1, feature1, newinstr2,        \
0242                feature2, input...)                   \
0243     asm_inline volatile(ALTERNATIVE_2(oldinstr, newinstr1, feature1,     \
0244         newinstr2, feature2)                         \
0245         : : "i" (0), ## input)
0246
0247 /* Like alternative_input, but with a single output argument */
0248 #define alternative_io(oldinstr, newinstr, feature, output, input...)   \
0249     asm_inline volatile (ALTERNATIVE(oldinstr, newinstr, feature)   \
0250         : output : "i" (0), ## input)
0251
0252 /* Like alternative_io, but for replacing a direct call with another one. */
0253 #define alternative_call(oldfunc, newfunc, feature, output, input...)   \
0254     asm_inline volatile (ALTERNATIVE("call %P[old]", "call %P[new]", feature) \
0255         : output : [old] "i" (oldfunc), [new] "i" (newfunc), ## input)
0256
0257 /*
0258  * Like alternative_call, but there are two features and respective functions.
0259  * If CPU has feature2, function2 is used.
0260  * Otherwise, if CPU has feature1, function1 is used.
0261  * Otherwise, old function is used.
0262  */
0263 #define alternative_call_2(oldfunc, newfunc1, feature1, newfunc2, feature2,   \
0264                output, input...)                      \
0265     asm_inline volatile (ALTERNATIVE_2("call %P[old]", "call %P[new1]", feature1,\
0266         "call %P[new2]", feature2)                    \
0267         : output, ASM_CALL_CONSTRAINT                     \
0268         : [old] "i" (oldfunc), [new1] "i" (newfunc1),             \
0269           [new2] "i" (newfunc2), ## input)
0270
0271 /*
0272  * use this macro(s) if you need more than one output parameter
0273  * in alternative_io
0274  */
0275 #define ASM_OUTPUT2(a...) a
0276
0277 /*
0278  * use this macro if you need clobbers but no inputs in
0279  * alternative_{input,io,call}()
0280  */
0281 #define ASM_NO_INPUT_CLOBBER(clbr...) "i" (0) : clbr
0282
0283 #else /* __ASSEMBLY__ */
0284
0285 #ifdef CONFIG_SMP
0286     .macro LOCK_PREFIX
0287 672:    lock
0288     .pushsection .smp_locks,"a"
0289     .balign 4
0290     .long 672b - .
0291     .popsection
0292     .endm
0293 #else
0294     .macro LOCK_PREFIX
0295     .endm
0296 #endif
0297
0298 /*
0299  * objtool annotation to ignore the alternatives and only consider the original
0300  * instruction(s).
0301  */
0302 .macro ANNOTATE_IGNORE_ALTERNATIVE
0303     .Lannotate_\@:
0304     .pushsection .discard.ignore_alts
0305     .long .Lannotate_\@ - .
0306     .popsection
0307 .endm
0308
0309 /*
0310  * Issue one struct alt_instr descriptor entry (need to put it into
0311  * the section .altinstructions, see below). This entry contains
0312  * enough information for the alternatives patching code to patch an
0313  * instruction. See apply_alternatives().
0314  */
0315 .macro altinstruction_entry orig alt feature orig_len alt_len
0316     .long \orig - .
0317     .long \alt - .
0318     .word \feature
0319     .byte \orig_len
0320     .byte \alt_len
0321 .endm
0322
0323 /*
0324  * Define an alternative between two instructions. If @feature is
0325  * present, early code in apply_alternatives() replaces @oldinstr with
0326  * @newinstr. ".skip" directive takes care of proper instruction padding
0327  * in case @newinstr is longer than @oldinstr.
0328  */
0329 .macro ALTERNATIVE oldinstr, newinstr, feature
0330 140:
0331     \oldinstr
0332 141:
0333     .skip -(((144f-143f)-(141b-140b)) > 0) * ((144f-143f)-(141b-140b)),0x90
0334 142:
0335
0336     .pushsection .altinstructions,"a"
0337     altinstruction_entry 140b,143f,\feature,142b-140b,144f-143f
0338     .popsection
0339
0340     .pushsection .altinstr_replacement,"ax"
0341 143:
0342     \newinstr
0343 144:
0344     .popsection
0345 .endm
0346
0347 #define old_len         141b-140b
0348 #define new_len1        144f-143f
0349 #define new_len2        145f-144f
0350
0351 /*
0352  * gas compatible max based on the idea from:
0353  * http://graphics.stanford.edu/~seander/bithacks.html#IntegerMinOrMax
0354  *
0355  * The additional "-" is needed because gas uses a "true" value of -1.
0356  */
0357 #define alt_max_short(a, b) ((a) ^ (((a) ^ (b)) & -(-((a) < (b)))))
0358
0359
0360 /*
0361  * Same as ALTERNATIVE macro above but for two alternatives. If CPU
0362  * has @feature1, it replaces @oldinstr with @newinstr1. If CPU has
0363  * @feature2, it replaces @oldinstr with @feature2.
0364  */
0365 .macro ALTERNATIVE_2 oldinstr, newinstr1, feature1, newinstr2, feature2
0366 140:
0367     \oldinstr
0368 141:
0369     .skip -((alt_max_short(new_len1, new_len2) - (old_len)) > 0) * \
0370         (alt_max_short(new_len1, new_len2) - (old_len)),0x90
0371 142:
0372
0373     .pushsection .altinstructions,"a"
0374     altinstruction_entry 140b,143f,\feature1,142b-140b,144f-143f
0375     altinstruction_entry 140b,144f,\feature2,142b-140b,145f-144f
0376     .popsection
0377
0378     .pushsection .altinstr_replacement,"ax"
0379 143:
0380     \newinstr1
0381 144:
0382     \newinstr2
0383 145:
0384     .popsection
0385 .endm
0386
0387 /* If @feature is set, patch in @newinstr_yes, otherwise @newinstr_no. */
0388 #define ALTERNATIVE_TERNARY(oldinstr, feature, newinstr_yes, newinstr_no) \
0389     ALTERNATIVE_2 oldinstr, newinstr_no, X86_FEATURE_ALWAYS,    \
0390     newinstr_yes, feature
0391
0392 #endif /* __ASSEMBLY__ */
0393
0394 #endif /* _ASM_X86_ALTERNATIVE_H */