include/asm/mmu.h

0001 /* SPDX-License-Identifier: GPL-2.0 */
0002 #ifndef _ASM_POWERPC_MMU_H_
0003 #define _ASM_POWERPC_MMU_H_
0004 #ifdef __KERNEL__
0005
0006 #include <linux/types.h>
0007
0008 #include <asm/asm-const.h>
0009
0010 /*
0011  * MMU features bit definitions
0012  */
0013
0014 /*
0015  * MMU families
0016  */
0017 #define MMU_FTR_HPTE_TABLE      ASM_CONST(0x00000001)
0018 #define MMU_FTR_TYPE_8xx        ASM_CONST(0x00000002)
0019 #define MMU_FTR_TYPE_40x        ASM_CONST(0x00000004)
0020 #define MMU_FTR_TYPE_44x        ASM_CONST(0x00000008)
0021 #define MMU_FTR_TYPE_FSL_E      ASM_CONST(0x00000010)
0022 #define MMU_FTR_TYPE_47x        ASM_CONST(0x00000020)
0023
0024 /* Radix page table supported and enabled */
0025 #define MMU_FTR_TYPE_RADIX      ASM_CONST(0x00000040)
0026
0027 /*
0028  * Individual features below.
0029  */
0030
0031 /*
0032  * Supports KUAP feature
0033  * key 0 controlling userspace addresses on radix
0034  * Key 3 on hash
0035  */
0036 #define MMU_FTR_BOOK3S_KUAP     ASM_CONST(0x00000200)
0037
0038 /*
0039  * Supports KUEP feature
0040  * key 0 controlling userspace addresses on radix
0041  * Key 3 on hash
0042  */
0043 #define MMU_FTR_BOOK3S_KUEP     ASM_CONST(0x00000400)
0044
0045 /*
0046  * Support for memory protection keys.
0047  */
0048 #define MMU_FTR_PKEY            ASM_CONST(0x00000800)
0049
0050 /* Guest Translation Shootdown Enable */
0051 #define MMU_FTR_GTSE            ASM_CONST(0x00001000)
0052
0053 /*
0054  * Support for 68 bit VA space. We added that from ISA 2.05
0055  */
0056 #define MMU_FTR_68_BIT_VA       ASM_CONST(0x00002000)
0057 /*
0058  * Kernel read only support.
0059  * We added the ppp value 0b110 in ISA 2.04.
0060  */
0061 #define MMU_FTR_KERNEL_RO       ASM_CONST(0x00004000)
0062
0063 /*
0064  * We need to clear top 16bits of va (from the remaining 64 bits )in
0065  * tlbie* instructions
0066  */
0067 #define MMU_FTR_TLBIE_CROP_VA       ASM_CONST(0x00008000)
0068
0069 /* Enable use of high BAT registers */
0070 #define MMU_FTR_USE_HIGH_BATS       ASM_CONST(0x00010000)
0071
0072 /* Enable >32-bit physical addresses on 32-bit processor, only used
0073  * by CONFIG_PPC_BOOK3S_32 currently as BookE supports that from day 1
0074  */
0075 #define MMU_FTR_BIG_PHYS        ASM_CONST(0x00020000)
0076
0077 /* Enable use of broadcast TLB invalidations. We don't always set it
0078  * on processors that support it due to other constraints with the
0079  * use of such invalidations
0080  */
0081 #define MMU_FTR_USE_TLBIVAX_BCAST   ASM_CONST(0x00040000)
0082
0083 /* Enable use of tlbilx invalidate instructions.
0084  */
0085 #define MMU_FTR_USE_TLBILX      ASM_CONST(0x00080000)
0086
0087 /* This indicates that the processor cannot handle multiple outstanding
0088  * broadcast tlbivax or tlbsync. This makes the code use a spinlock
0089  * around such invalidate forms.
0090  */
0091 #define MMU_FTR_LOCK_BCAST_INVAL    ASM_CONST(0x00100000)
0092
0093 /* This indicates that the processor doesn't handle way selection
0094  * properly and needs SW to track and update the LRU state.  This
0095  * is specific to an errata on e300c2/c3/c4 class parts
0096  */
0097 #define MMU_FTR_NEED_DTLB_SW_LRU    ASM_CONST(0x00200000)
0098
0099 /* Doesn't support the B bit (1T segment) in SLBIE
0100  */
0101 #define MMU_FTR_NO_SLBIE_B      ASM_CONST(0x02000000)
0102
0103 /* Support 16M large pages
0104  */
0105 #define MMU_FTR_16M_PAGE        ASM_CONST(0x04000000)
0106
0107 /* Supports TLBIEL variant
0108  */
0109 #define MMU_FTR_TLBIEL          ASM_CONST(0x08000000)
0110
0111 /* Supports tlbies w/o locking
0112  */
0113 #define MMU_FTR_LOCKLESS_TLBIE      ASM_CONST(0x10000000)
0114
0115 /* Large pages can be marked CI
0116  */
0117 #define MMU_FTR_CI_LARGE_PAGE       ASM_CONST(0x20000000)
0118
0119 /* 1T segments available
0120  */
0121 #define MMU_FTR_1T_SEGMENT      ASM_CONST(0x40000000)
0122
0123 /* MMU feature bit sets for various CPUs */
0124 #define MMU_FTRS_DEFAULT_HPTE_ARCH_V2   (MMU_FTR_HPTE_TABLE | MMU_FTR_TLBIEL | MMU_FTR_16M_PAGE)
0125 #define MMU_FTRS_POWER      MMU_FTRS_DEFAULT_HPTE_ARCH_V2
0126 #define MMU_FTRS_PPC970     MMU_FTRS_POWER | MMU_FTR_TLBIE_CROP_VA
0127 #define MMU_FTRS_POWER5     MMU_FTRS_POWER | MMU_FTR_LOCKLESS_TLBIE
0128 #define MMU_FTRS_POWER6     MMU_FTRS_POWER5 | MMU_FTR_KERNEL_RO | MMU_FTR_68_BIT_VA
0129 #define MMU_FTRS_POWER7     MMU_FTRS_POWER6
0130 #define MMU_FTRS_POWER8     MMU_FTRS_POWER6
0131 #define MMU_FTRS_POWER9     MMU_FTRS_POWER6
0132 #define MMU_FTRS_POWER10    MMU_FTRS_POWER6
0133 #define MMU_FTRS_CELL       MMU_FTRS_DEFAULT_HPTE_ARCH_V2 | \
0134                 MMU_FTR_CI_LARGE_PAGE
0135 #define MMU_FTRS_PA6T       MMU_FTRS_DEFAULT_HPTE_ARCH_V2 | \
0136                 MMU_FTR_CI_LARGE_PAGE | MMU_FTR_NO_SLBIE_B
0137 #ifndef __ASSEMBLY__
0138 #include <linux/bug.h>
0139 #include <asm/cputable.h>
0140 #include <asm/page.h>
0141
0142 typedef pte_t *pgtable_t;
0143
0144 #ifdef CONFIG_PPC_FSL_BOOK3E
0145 #include <asm/percpu.h>
0146 DECLARE_PER_CPU(int, next_tlbcam_idx);
0147 #endif
0148
0149 enum {
0150     MMU_FTRS_POSSIBLE =
0151 #if defined(CONFIG_PPC_BOOK3S_604)
0152         MMU_FTR_HPTE_TABLE |
0153 #endif
0154 #ifdef CONFIG_PPC_8xx
0155         MMU_FTR_TYPE_8xx |
0156 #endif
0157 #ifdef CONFIG_40x
0158         MMU_FTR_TYPE_40x |
0159 #endif
0160 #ifdef CONFIG_PPC_47x
0161         MMU_FTR_TYPE_47x | MMU_FTR_USE_TLBIVAX_BCAST | MMU_FTR_LOCK_BCAST_INVAL |
0162 #elif defined(CONFIG_44x)
0163         MMU_FTR_TYPE_44x |
0164 #endif
0165 #ifdef CONFIG_E500
0166         MMU_FTR_TYPE_FSL_E | MMU_FTR_BIG_PHYS | MMU_FTR_USE_TLBILX |
0167 #endif
0168 #ifdef CONFIG_PPC_BOOK3S_32
0169         MMU_FTR_USE_HIGH_BATS |
0170 #endif
0171 #ifdef CONFIG_PPC_83xx
0172         MMU_FTR_NEED_DTLB_SW_LRU |
0173 #endif
0174 #ifdef CONFIG_PPC_BOOK3S_64
0175         MMU_FTR_KERNEL_RO |
0176 #ifdef CONFIG_PPC_64S_HASH_MMU
0177         MMU_FTR_NO_SLBIE_B | MMU_FTR_16M_PAGE | MMU_FTR_TLBIEL |
0178         MMU_FTR_LOCKLESS_TLBIE | MMU_FTR_CI_LARGE_PAGE |
0179         MMU_FTR_1T_SEGMENT | MMU_FTR_TLBIE_CROP_VA |
0180         MMU_FTR_68_BIT_VA | MMU_FTR_HPTE_TABLE |
0181 #endif
0182 #ifdef CONFIG_PPC_RADIX_MMU
0183         MMU_FTR_TYPE_RADIX |
0184         MMU_FTR_GTSE |
0185 #endif /* CONFIG_PPC_RADIX_MMU */
0186 #endif
0187 #ifdef CONFIG_PPC_KUAP
0188     MMU_FTR_BOOK3S_KUAP |
0189 #endif /* CONFIG_PPC_KUAP */
0190 #ifdef CONFIG_PPC_MEM_KEYS
0191     MMU_FTR_PKEY |
0192 #endif
0193 #ifdef CONFIG_PPC_KUEP
0194     MMU_FTR_BOOK3S_KUEP |
0195 #endif /* CONFIG_PPC_KUAP */
0196
0197         0,
0198 };
0199
0200 #if defined(CONFIG_PPC_BOOK3S_604) && !defined(CONFIG_PPC_BOOK3S_603)
0201 #define MMU_FTRS_ALWAYS     MMU_FTR_HPTE_TABLE
0202 #endif
0203 #ifdef CONFIG_PPC_8xx
0204 #define MMU_FTRS_ALWAYS     MMU_FTR_TYPE_8xx
0205 #endif
0206 #ifdef CONFIG_40x
0207 #define MMU_FTRS_ALWAYS     MMU_FTR_TYPE_40x
0208 #endif
0209 #ifdef CONFIG_PPC_47x
0210 #define MMU_FTRS_ALWAYS     MMU_FTR_TYPE_47x
0211 #elif defined(CONFIG_44x)
0212 #define MMU_FTRS_ALWAYS     MMU_FTR_TYPE_44x
0213 #endif
0214 #ifdef CONFIG_E500
0215 #define MMU_FTRS_ALWAYS     MMU_FTR_TYPE_FSL_E
0216 #endif
0217
0218 /* BOOK3S_64 options */
0219 #if defined(CONFIG_PPC_RADIX_MMU) && !defined(CONFIG_PPC_64S_HASH_MMU)
0220 #define MMU_FTRS_ALWAYS     MMU_FTR_TYPE_RADIX
0221 #elif !defined(CONFIG_PPC_RADIX_MMU) && defined(CONFIG_PPC_64S_HASH_MMU)
0222 #define MMU_FTRS_ALWAYS     MMU_FTR_HPTE_TABLE
0223 #endif
0224
0225 #ifndef MMU_FTRS_ALWAYS
0226 #define MMU_FTRS_ALWAYS     0
0227 #endif
0228
0229 static __always_inline bool early_mmu_has_feature(unsigned long feature)
0230 {
0231     if (MMU_FTRS_ALWAYS & feature)
0232         return true;
0233
0234     return !!(MMU_FTRS_POSSIBLE & cur_cpu_spec->mmu_features & feature);
0235 }
0236
0237 #ifdef CONFIG_JUMP_LABEL_FEATURE_CHECKS
0238 #include <linux/jump_label.h>
0239
0240 #define NUM_MMU_FTR_KEYS    32
0241
0242 extern struct static_key_true mmu_feature_keys[NUM_MMU_FTR_KEYS];
0243
0244 extern void mmu_feature_keys_init(void);
0245
0246 static __always_inline bool mmu_has_feature(unsigned long feature)
0247 {
0248     int i;
0249
0250 #ifndef __clang__ /* clang can't cope with this */
0251     BUILD_BUG_ON(!__builtin_constant_p(feature));
0252 #endif
0253
0254 #ifdef CONFIG_JUMP_LABEL_FEATURE_CHECK_DEBUG
0255     if (!static_key_initialized) {
0256         printk("Warning! mmu_has_feature() used prior to jump label init!\n");
0257         dump_stack();
0258         return early_mmu_has_feature(feature);
0259     }
0260 #endif
0261
0262     if (MMU_FTRS_ALWAYS & feature)
0263         return true;
0264
0265     if (!(MMU_FTRS_POSSIBLE & feature))
0266         return false;
0267
0268     i = __builtin_ctzl(feature);
0269     return static_branch_likely(&mmu_feature_keys[i]);
0270 }
0271
0272 static inline void mmu_clear_feature(unsigned long feature)
0273 {
0274     int i;
0275
0276     i = __builtin_ctzl(feature);
0277     cur_cpu_spec->mmu_features &= ~feature;
0278     static_branch_disable(&mmu_feature_keys[i]);
0279 }
0280 #else
0281
0282 static inline void mmu_feature_keys_init(void)
0283 {
0284
0285 }
0286
0287 static __always_inline bool mmu_has_feature(unsigned long feature)
0288 {
0289     return early_mmu_has_feature(feature);
0290 }
0291
0292 static inline void mmu_clear_feature(unsigned long feature)
0293 {
0294     cur_cpu_spec->mmu_features &= ~feature;
0295 }
0296 #endif /* CONFIG_JUMP_LABEL */
0297
0298 extern unsigned int __start___mmu_ftr_fixup, __stop___mmu_ftr_fixup;
0299
0300 #ifdef CONFIG_PPC64
0301 /* This is our real memory area size on ppc64 server, on embedded, we
0302  * make it match the size our of bolted TLB area
0303  */
0304 extern u64 ppc64_rma_size;
0305
0306 /* Cleanup function used by kexec */
0307 extern void mmu_cleanup_all(void);
0308 extern void radix__mmu_cleanup_all(void);
0309
0310 /* Functions for creating and updating partition table on POWER9 */
0311 extern void mmu_partition_table_init(void);
0312 extern void mmu_partition_table_set_entry(unsigned int lpid, unsigned long dw0,
0313                       unsigned long dw1, bool flush);
0314 #endif /* CONFIG_PPC64 */
0315
0316 struct mm_struct;
0317 #ifdef CONFIG_DEBUG_VM
0318 extern void assert_pte_locked(struct mm_struct *mm, unsigned long addr);
0319 #else /* CONFIG_DEBUG_VM */
0320 static inline void assert_pte_locked(struct mm_struct *mm, unsigned long addr)
0321 {
0322 }
0323 #endif /* !CONFIG_DEBUG_VM */
0324
0325 static __always_inline bool radix_enabled(void)
0326 {
0327     return mmu_has_feature(MMU_FTR_TYPE_RADIX);
0328 }
0329
0330 static __always_inline bool early_radix_enabled(void)
0331 {
0332     return early_mmu_has_feature(MMU_FTR_TYPE_RADIX);
0333 }
0334
0335 #ifdef CONFIG_STRICT_KERNEL_RWX
0336 static inline bool strict_kernel_rwx_enabled(void)
0337 {
0338     return rodata_enabled;
0339 }
0340 #else
0341 static inline bool strict_kernel_rwx_enabled(void)
0342 {
0343     return false;
0344 }
0345 #endif
0346
0347 static inline bool strict_module_rwx_enabled(void)
0348 {
0349     return IS_ENABLED(CONFIG_STRICT_MODULE_RWX) && strict_kernel_rwx_enabled();
0350 }
0351 #endif /* !__ASSEMBLY__ */
0352
0353 /* The kernel use the constants below to index in the page sizes array.
0354  * The use of fixed constants for this purpose is better for performances
0355  * of the low level hash refill handlers.
0356  *
0357  * A non supported page size has a "shift" field set to 0
0358  *
0359  * Any new page size being implemented can get a new entry in here. Whether
0360  * the kernel will use it or not is a different matter though. The actual page
0361  * size used by hugetlbfs is not defined here and may be made variable
0362  *
0363  * Note: This array ended up being a false good idea as it's growing to the
0364  * point where I wonder if we should replace it with something different,
0365  * to think about, feedback welcome. --BenH.
0366  */
0367
0368 /* These are #defines as they have to be used in assembly */
0369 #define MMU_PAGE_4K 0
0370 #define MMU_PAGE_16K    1
0371 #define MMU_PAGE_64K    2
0372 #define MMU_PAGE_64K_AP 3   /* "Admixed pages" (hash64 only) */
0373 #define MMU_PAGE_256K   4
0374 #define MMU_PAGE_512K   5
0375 #define MMU_PAGE_1M 6
0376 #define MMU_PAGE_2M 7
0377 #define MMU_PAGE_4M 8
0378 #define MMU_PAGE_8M 9
0379 #define MMU_PAGE_16M    10
0380 #define MMU_PAGE_64M    11
0381 #define MMU_PAGE_256M   12
0382 #define MMU_PAGE_1G 13
0383 #define MMU_PAGE_16G    14
0384 #define MMU_PAGE_64G    15
0385
0386 /*
0387  * N.B. we need to change the type of hpte_page_sizes if this gets to be > 16
0388  * Also we need to change he type of mm_context.low/high_slices_psize.
0389  */
0390 #define MMU_PAGE_COUNT  16
0391
0392 #ifdef CONFIG_PPC_BOOK3S_64
0393 #include <asm/book3s/64/mmu.h>
0394 #else /* CONFIG_PPC_BOOK3S_64 */
0395
0396 #ifndef __ASSEMBLY__
0397 /* MMU initialization */
0398 extern void early_init_mmu(void);
0399 extern void early_init_mmu_secondary(void);
0400 extern void setup_initial_memory_limit(phys_addr_t first_memblock_base,
0401                        phys_addr_t first_memblock_size);
0402 static inline void mmu_early_init_devtree(void) { }
0403
0404 static inline void pkey_early_init_devtree(void) {}
0405
0406 extern void *abatron_pteptrs[2];
0407 #endif /* __ASSEMBLY__ */
0408 #endif
0409
0410 #if defined(CONFIG_PPC_BOOK3S_32)
0411 /* 32-bit classic hash table MMU */
0412 #include <asm/book3s/32/mmu-hash.h>
0413 #elif defined(CONFIG_PPC_MMU_NOHASH)
0414 #include <asm/nohash/mmu.h>
0415 #endif
0416
0417 #endif /* __KERNEL__ */
0418 #endif /* _ASM_POWERPC_MMU_H_ */