primitives/asm/feature-fixups.h

0001 /* SPDX-License-Identifier: GPL-2.0-or-later */
0002 #ifndef __ASM_POWERPC_FEATURE_FIXUPS_H
0003 #define __ASM_POWERPC_FEATURE_FIXUPS_H
0004
0005 #include <asm/asm-const.h>
0006
0007 /*
0008  */
0009
0010 /*
0011  * Feature section common macros
0012  *
0013  * Note that the entries now contain offsets between the table entry
0014  * and the code rather than absolute code pointers in order to be
0015  * useable with the vdso shared library. There is also an assumption
0016  * that values will be negative, that is, the fixup table has to be
0017  * located after the code it fixes up.
0018  */
0019 #if defined(CONFIG_PPC64) && !defined(__powerpc64__)
0020 /* 64 bits kernel, 32 bits code (ie. vdso32) */
0021 #define FTR_ENTRY_LONG      .8byte
0022 #define FTR_ENTRY_OFFSET    .long 0xffffffff; .long
0023 #elif defined(CONFIG_PPC64)
0024 #define FTR_ENTRY_LONG      .8byte
0025 #define FTR_ENTRY_OFFSET    .8byte
0026 #else
0027 #define FTR_ENTRY_LONG      .long
0028 #define FTR_ENTRY_OFFSET    .long
0029 #endif
0030
0031 #define START_FTR_SECTION(label)    label##1:
0032
0033 #define FTR_SECTION_ELSE_NESTED(label)          \
0034 label##2:                       \
0035     .pushsection __ftr_alt_##label,"a";     \
0036     .align 2;                   \
0037 label##3:
0038
0039
0040 #ifndef CONFIG_CC_IS_CLANG
0041 #define CHECK_ALT_SIZE(else_size, body_size)            \
0042     .ifgt (else_size) - (body_size);            \
0043     .error "Feature section else case larger than body";    \
0044     .endif;
0045 #else
0046 /*
0047  * If we use the ifgt syntax above, clang's assembler complains about the
0048  * expression being non-absolute when the code appears in an inline assembly
0049  * statement.
0050  * As a workaround use an .org directive that has no effect if the else case
0051  * instructions are smaller than the body, but fails otherwise.
0052  */
0053 #define CHECK_ALT_SIZE(else_size, body_size)            \
0054     .org . + ((else_size) > (body_size));
0055 #endif
0056
0057 #define MAKE_FTR_SECTION_ENTRY(msk, val, label, sect)       \
0058 label##4:                           \
0059     .popsection;                        \
0060     .pushsection sect,"a";                  \
0061     .align 3;                       \
0062 label##5:                           \
0063     FTR_ENTRY_LONG msk;                 \
0064     FTR_ENTRY_LONG val;                 \
0065     FTR_ENTRY_OFFSET label##1b-label##5b;           \
0066     FTR_ENTRY_OFFSET label##2b-label##5b;           \
0067     FTR_ENTRY_OFFSET label##3b-label##5b;           \
0068     FTR_ENTRY_OFFSET label##4b-label##5b;           \
0069     CHECK_ALT_SIZE((label##4b-label##3b), (label##2b-label##1b)); \
0070     .popsection;
0071
0072
0073 /* CPU feature dependent sections */
0074 #define BEGIN_FTR_SECTION_NESTED(label) START_FTR_SECTION(label)
0075 #define BEGIN_FTR_SECTION       START_FTR_SECTION(97)
0076
0077 #define END_FTR_SECTION_NESTED(msk, val, label)         \
0078     FTR_SECTION_ELSE_NESTED(label)              \
0079     MAKE_FTR_SECTION_ENTRY(msk, val, label, __ftr_fixup)
0080
0081 #define END_FTR_SECTION(msk, val)       \
0082     END_FTR_SECTION_NESTED(msk, val, 97)
0083
0084 #define END_FTR_SECTION_NESTED_IFSET(msk, label)    \
0085     END_FTR_SECTION_NESTED((msk), (msk), label)
0086
0087 #define END_FTR_SECTION_IFSET(msk)  END_FTR_SECTION((msk), (msk))
0088 #define END_FTR_SECTION_IFCLR(msk)  END_FTR_SECTION((msk), 0)
0089
0090 /* CPU feature sections with alternatives, use BEGIN_FTR_SECTION to start */
0091 #define FTR_SECTION_ELSE    FTR_SECTION_ELSE_NESTED(97)
0092 #define ALT_FTR_SECTION_END_NESTED(msk, val, label) \
0093     MAKE_FTR_SECTION_ENTRY(msk, val, label, __ftr_fixup)
0094 #define ALT_FTR_SECTION_END_NESTED_IFSET(msk, label)    \
0095     ALT_FTR_SECTION_END_NESTED(msk, msk, label)
0096 #define ALT_FTR_SECTION_END_NESTED_IFCLR(msk, label)    \
0097     ALT_FTR_SECTION_END_NESTED(msk, 0, label)
0098 #define ALT_FTR_SECTION_END(msk, val)   \
0099     ALT_FTR_SECTION_END_NESTED(msk, val, 97)
0100 #define ALT_FTR_SECTION_END_IFSET(msk)  \
0101     ALT_FTR_SECTION_END_NESTED_IFSET(msk, 97)
0102 #define ALT_FTR_SECTION_END_IFCLR(msk)  \
0103     ALT_FTR_SECTION_END_NESTED_IFCLR(msk, 97)
0104
0105 /* MMU feature dependent sections */
0106 #define BEGIN_MMU_FTR_SECTION_NESTED(label) START_FTR_SECTION(label)
0107 #define BEGIN_MMU_FTR_SECTION           START_FTR_SECTION(97)
0108
0109 #define END_MMU_FTR_SECTION_NESTED(msk, val, label)         \
0110     FTR_SECTION_ELSE_NESTED(label)              \
0111     MAKE_FTR_SECTION_ENTRY(msk, val, label, __mmu_ftr_fixup)
0112
0113 #define END_MMU_FTR_SECTION(msk, val)       \
0114     END_MMU_FTR_SECTION_NESTED(msk, val, 97)
0115
0116 #define END_MMU_FTR_SECTION_NESTED_IFSET(msk, label)    \
0117     END_MMU_FTR_SECTION_NESTED((msk), (msk), label)
0118
0119 #define END_MMU_FTR_SECTION_NESTED_IFCLR(msk, label)    \
0120     END_MMU_FTR_SECTION_NESTED((msk), 0, label)
0121
0122 #define END_MMU_FTR_SECTION_IFSET(msk)  END_MMU_FTR_SECTION((msk), (msk))
0123 #define END_MMU_FTR_SECTION_IFCLR(msk)  END_MMU_FTR_SECTION((msk), 0)
0124
0125 /* MMU feature sections with alternatives, use BEGIN_FTR_SECTION to start */
0126 #define MMU_FTR_SECTION_ELSE_NESTED(label)  FTR_SECTION_ELSE_NESTED(label)
0127 #define MMU_FTR_SECTION_ELSE    MMU_FTR_SECTION_ELSE_NESTED(97)
0128 #define ALT_MMU_FTR_SECTION_END_NESTED(msk, val, label) \
0129     MAKE_FTR_SECTION_ENTRY(msk, val, label, __mmu_ftr_fixup)
0130 #define ALT_MMU_FTR_SECTION_END_NESTED_IFSET(msk, label)    \
0131     ALT_MMU_FTR_SECTION_END_NESTED(msk, msk, label)
0132 #define ALT_MMU_FTR_SECTION_END_NESTED_IFCLR(msk, label)    \
0133     ALT_MMU_FTR_SECTION_END_NESTED(msk, 0, label)
0134 #define ALT_MMU_FTR_SECTION_END(msk, val)   \
0135     ALT_MMU_FTR_SECTION_END_NESTED(msk, val, 97)
0136 #define ALT_MMU_FTR_SECTION_END_IFSET(msk)  \
0137     ALT_MMU_FTR_SECTION_END_NESTED_IFSET(msk, 97)
0138 #define ALT_MMU_FTR_SECTION_END_IFCLR(msk)  \
0139     ALT_MMU_FTR_SECTION_END_NESTED_IFCLR(msk, 97)
0140
0141 /* Firmware feature dependent sections */
0142 #define BEGIN_FW_FTR_SECTION_NESTED(label)  START_FTR_SECTION(label)
0143 #define BEGIN_FW_FTR_SECTION            START_FTR_SECTION(97)
0144
0145 #define END_FW_FTR_SECTION_NESTED(msk, val, label)      \
0146     FTR_SECTION_ELSE_NESTED(label)              \
0147     MAKE_FTR_SECTION_ENTRY(msk, val, label, __fw_ftr_fixup)
0148
0149 #define END_FW_FTR_SECTION(msk, val)        \
0150     END_FW_FTR_SECTION_NESTED(msk, val, 97)
0151
0152 #define END_FW_FTR_SECTION_IFSET(msk)   END_FW_FTR_SECTION((msk), (msk))
0153 #define END_FW_FTR_SECTION_IFCLR(msk)   END_FW_FTR_SECTION((msk), 0)
0154
0155 /* Firmware feature sections with alternatives */
0156 #define FW_FTR_SECTION_ELSE_NESTED(label)   FTR_SECTION_ELSE_NESTED(label)
0157 #define FW_FTR_SECTION_ELSE FTR_SECTION_ELSE_NESTED(97)
0158 #define ALT_FW_FTR_SECTION_END_NESTED(msk, val, label)  \
0159     MAKE_FTR_SECTION_ENTRY(msk, val, label, __fw_ftr_fixup)
0160 #define ALT_FW_FTR_SECTION_END_NESTED_IFSET(msk, label) \
0161     ALT_FW_FTR_SECTION_END_NESTED(msk, msk, label)
0162 #define ALT_FW_FTR_SECTION_END_NESTED_IFCLR(msk, label) \
0163     ALT_FW_FTR_SECTION_END_NESTED(msk, 0, label)
0164 #define ALT_FW_FTR_SECTION_END(msk, val)    \
0165     ALT_FW_FTR_SECTION_END_NESTED(msk, val, 97)
0166 #define ALT_FW_FTR_SECTION_END_IFSET(msk)   \
0167     ALT_FW_FTR_SECTION_END_NESTED_IFSET(msk, 97)
0168 #define ALT_FW_FTR_SECTION_END_IFCLR(msk)   \
0169     ALT_FW_FTR_SECTION_END_NESTED_IFCLR(msk, 97)
0170
0171 #ifndef __ASSEMBLY__
0172
0173 #define ASM_FTR_IF(section_if, section_else, msk, val)  \
0174     stringify_in_c(BEGIN_FTR_SECTION)           \
0175     section_if "; "                     \
0176     stringify_in_c(FTR_SECTION_ELSE)            \
0177     section_else "; "                   \
0178     stringify_in_c(ALT_FTR_SECTION_END((msk), (val)))
0179
0180 #define ASM_FTR_IFSET(section_if, section_else, msk)    \
0181     ASM_FTR_IF(section_if, section_else, (msk), (msk))
0182
0183 #define ASM_FTR_IFCLR(section_if, section_else, msk)    \
0184     ASM_FTR_IF(section_if, section_else, (msk), 0)
0185
0186 #define ASM_MMU_FTR_IF(section_if, section_else, msk, val)  \
0187     stringify_in_c(BEGIN_MMU_FTR_SECTION)           \
0188     section_if "; "                     \
0189     stringify_in_c(MMU_FTR_SECTION_ELSE)            \
0190     section_else "; "                   \
0191     stringify_in_c(ALT_MMU_FTR_SECTION_END((msk), (val)))
0192
0193 #define ASM_MMU_FTR_IFSET(section_if, section_else, msk)    \
0194     ASM_MMU_FTR_IF(section_if, section_else, (msk), (msk))
0195
0196 #define ASM_MMU_FTR_IFCLR(section_if, section_else, msk)    \
0197     ASM_MMU_FTR_IF(section_if, section_else, (msk), 0)
0198
0199 #endif /* __ASSEMBLY__ */
0200
0201 /* LWSYNC feature sections */
0202 #define START_LWSYNC_SECTION(label) label##1:
0203 #define MAKE_LWSYNC_SECTION_ENTRY(label, sect)      \
0204 label##2:                       \
0205     .pushsection sect,"a";              \
0206     .align 2;                   \
0207 label##3:                           \
0208     FTR_ENTRY_OFFSET label##1b-label##3b;       \
0209     .popsection;
0210
0211 #define STF_ENTRY_BARRIER_FIXUP_SECTION         \
0212 953:                            \
0213     .pushsection __stf_entry_barrier_fixup,"a"; \
0214     .align 2;                   \
0215 954:                            \
0216     FTR_ENTRY_OFFSET 953b-954b;         \
0217     .popsection;
0218
0219 #define STF_EXIT_BARRIER_FIXUP_SECTION          \
0220 955:                            \
0221     .pushsection __stf_exit_barrier_fixup,"a";  \
0222     .align 2;                   \
0223 956:                            \
0224     FTR_ENTRY_OFFSET 955b-956b;         \
0225     .popsection;
0226
0227 #define UACCESS_FLUSH_FIXUP_SECTION         \
0228 959:                            \
0229     .pushsection __uaccess_flush_fixup,"a";     \
0230     .align 2;                   \
0231 960:                            \
0232     FTR_ENTRY_OFFSET 959b-960b;         \
0233     .popsection;
0234
0235 #define ENTRY_FLUSH_FIXUP_SECTION           \
0236 957:                            \
0237     .pushsection __entry_flush_fixup,"a";       \
0238     .align 2;                   \
0239 958:                            \
0240     FTR_ENTRY_OFFSET 957b-958b;         \
0241     .popsection;
0242
0243 #define SCV_ENTRY_FLUSH_FIXUP_SECTION           \
0244 957:                            \
0245     .pushsection __scv_entry_flush_fixup,"a";   \
0246     .align 2;                   \
0247 958:                            \
0248     FTR_ENTRY_OFFSET 957b-958b;         \
0249     .popsection;
0250
0251 #define RFI_FLUSH_FIXUP_SECTION             \
0252 951:                            \
0253     .pushsection __rfi_flush_fixup,"a";     \
0254     .align 2;                   \
0255 952:                            \
0256     FTR_ENTRY_OFFSET 951b-952b;         \
0257     .popsection;
0258
0259 #define NOSPEC_BARRIER_FIXUP_SECTION            \
0260 953:                            \
0261     .pushsection __barrier_nospec_fixup,"a";    \
0262     .align 2;                   \
0263 954:                            \
0264     FTR_ENTRY_OFFSET 953b-954b;         \
0265     .popsection;
0266
0267 #define START_BTB_FLUSH_SECTION         \
0268 955:                            \
0269
0270 #define END_BTB_FLUSH_SECTION           \
0271 956:                            \
0272     .pushsection __btb_flush_fixup,"a"; \
0273     .align 2;                           \
0274 957:                        \
0275     FTR_ENTRY_OFFSET 955b-957b;         \
0276     FTR_ENTRY_OFFSET 956b-957b;         \
0277     .popsection;
0278
0279 #ifndef __ASSEMBLY__
0280 #include <linux/types.h>
0281
0282 extern long stf_barrier_fallback;
0283 extern long entry_flush_fallback;
0284 extern long scv_entry_flush_fallback;
0285 extern long __start___stf_entry_barrier_fixup, __stop___stf_entry_barrier_fixup;
0286 extern long __start___stf_exit_barrier_fixup, __stop___stf_exit_barrier_fixup;
0287 extern long __start___uaccess_flush_fixup, __stop___uaccess_flush_fixup;
0288 extern long __start___entry_flush_fixup, __stop___entry_flush_fixup;
0289 extern long __start___scv_entry_flush_fixup, __stop___scv_entry_flush_fixup;
0290 extern long __start___rfi_flush_fixup, __stop___rfi_flush_fixup;
0291 extern long __start___barrier_nospec_fixup, __stop___barrier_nospec_fixup;
0292 extern long __start__btb_flush_fixup, __stop__btb_flush_fixup;
0293
0294 void apply_feature_fixups(void);
0295 void setup_feature_keys(void);
0296 #endif
0297
0298 #endif /* __ASM_POWERPC_FEATURE_FIXUPS_H */