mm/nohash/fsl_book3e.c

0001 // SPDX-License-Identifier: GPL-2.0-or-later
0002 /*
0003  * Modifications by Kumar Gala (galak@kernel.crashing.org) to support
0004  * E500 Book E processors.
0005  *
0006  * Copyright 2004,2010 Freescale Semiconductor, Inc.
0007  *
0008  * This file contains the routines for initializing the MMU
0009  * on the 4xx series of chips.
0010  *  -- paulus
0011  *
0012  *  Derived from arch/ppc/mm/init.c:
0013  *    Copyright (C) 1995-1996 Gary Thomas (gdt@linuxppc.org)
0014  *
0015  *  Modifications by Paul Mackerras (PowerMac) (paulus@cs.anu.edu.au)
0016  *  and Cort Dougan (PReP) (cort@cs.nmt.edu)
0017  *    Copyright (C) 1996 Paul Mackerras
0018  *
0019  *  Derived from "arch/i386/mm/init.c"
0020  *    Copyright (C) 1991, 1992, 1993, 1994  Linus Torvalds
0021  */
0022
0023 #include <linux/signal.h>
0024 #include <linux/sched.h>
0025 #include <linux/kernel.h>
0026 #include <linux/errno.h>
0027 #include <linux/string.h>
0028 #include <linux/types.h>
0029 #include <linux/ptrace.h>
0030 #include <linux/mman.h>
0031 #include <linux/mm.h>
0032 #include <linux/swap.h>
0033 #include <linux/stddef.h>
0034 #include <linux/vmalloc.h>
0035 #include <linux/init.h>
0036 #include <linux/delay.h>
0037 #include <linux/highmem.h>
0038 #include <linux/memblock.h>
0039 #include <linux/of_fdt.h>
0040
0041 #include <asm/io.h>
0042 #include <asm/mmu_context.h>
0043 #include <asm/mmu.h>
0044 #include <linux/uaccess.h>
0045 #include <asm/smp.h>
0046 #include <asm/machdep.h>
0047 #include <asm/setup.h>
0048 #include <asm/paca.h>
0049
0050 #include <mm/mmu_decl.h>
0051
0052 unsigned int tlbcam_index;
0053
0054 struct tlbcam TLBCAM[NUM_TLBCAMS];
0055
0056 static struct {
0057     unsigned long start;
0058     unsigned long limit;
0059     phys_addr_t phys;
0060 } tlbcam_addrs[NUM_TLBCAMS];
0061
0062 #ifdef CONFIG_FSL_BOOKE
0063 /*
0064  * Return PA for this VA if it is mapped by a CAM, or 0
0065  */
0066 phys_addr_t v_block_mapped(unsigned long va)
0067 {
0068     int b;
0069     for (b = 0; b < tlbcam_index; ++b)
0070         if (va >= tlbcam_addrs[b].start && va < tlbcam_addrs[b].limit)
0071             return tlbcam_addrs[b].phys + (va - tlbcam_addrs[b].start);
0072     return 0;
0073 }
0074
0075 /*
0076  * Return VA for a given PA or 0 if not mapped
0077  */
0078 unsigned long p_block_mapped(phys_addr_t pa)
0079 {
0080     int b;
0081     for (b = 0; b < tlbcam_index; ++b)
0082         if (pa >= tlbcam_addrs[b].phys
0083             && pa < (tlbcam_addrs[b].limit-tlbcam_addrs[b].start)
0084                       +tlbcam_addrs[b].phys)
0085             return tlbcam_addrs[b].start+(pa-tlbcam_addrs[b].phys);
0086     return 0;
0087 }
0088 #endif
0089
0090 /*
0091  * Set up a variable-size TLB entry (tlbcam). The parameters are not checked;
0092  * in particular size must be a power of 4 between 4k and the max supported by
0093  * an implementation; max may further be limited by what can be represented in
0094  * an unsigned long (for example, 32-bit implementations cannot support a 4GB
0095  * size).
0096  */
0097 static void settlbcam(int index, unsigned long virt, phys_addr_t phys,
0098         unsigned long size, unsigned long flags, unsigned int pid)
0099 {
0100     unsigned int tsize;
0101
0102     tsize = __ilog2(size) - 10;
0103
0104 #if defined(CONFIG_SMP) || defined(CONFIG_PPC_E500MC)
0105     if ((flags & _PAGE_NO_CACHE) == 0)
0106         flags |= _PAGE_COHERENT;
0107 #endif
0108
0109     TLBCAM[index].MAS0 = MAS0_TLBSEL(1) | MAS0_ESEL(index) | MAS0_NV(index+1);
0110     TLBCAM[index].MAS1 = MAS1_VALID | MAS1_IPROT | MAS1_TSIZE(tsize) | MAS1_TID(pid);
0111     TLBCAM[index].MAS2 = virt & PAGE_MASK;
0112
0113     TLBCAM[index].MAS2 |= (flags & _PAGE_WRITETHRU) ? MAS2_W : 0;
0114     TLBCAM[index].MAS2 |= (flags & _PAGE_NO_CACHE) ? MAS2_I : 0;
0115     TLBCAM[index].MAS2 |= (flags & _PAGE_COHERENT) ? MAS2_M : 0;
0116     TLBCAM[index].MAS2 |= (flags & _PAGE_GUARDED) ? MAS2_G : 0;
0117     TLBCAM[index].MAS2 |= (flags & _PAGE_ENDIAN) ? MAS2_E : 0;
0118
0119     TLBCAM[index].MAS3 = (phys & MAS3_RPN) | MAS3_SR;
0120     TLBCAM[index].MAS3 |= (flags & _PAGE_RW) ? MAS3_SW : 0;
0121     if (mmu_has_feature(MMU_FTR_BIG_PHYS))
0122         TLBCAM[index].MAS7 = (u64)phys >> 32;
0123
0124     /* Below is unlikely -- only for large user pages or similar */
0125     if (pte_user(__pte(flags))) {
0126         TLBCAM[index].MAS3 |= MAS3_UR;
0127         TLBCAM[index].MAS3 |= (flags & _PAGE_EXEC) ? MAS3_UX : 0;
0128         TLBCAM[index].MAS3 |= (flags & _PAGE_RW) ? MAS3_UW : 0;
0129     } else {
0130         TLBCAM[index].MAS3 |= (flags & _PAGE_EXEC) ? MAS3_SX : 0;
0131     }
0132
0133     tlbcam_addrs[index].start = virt;
0134     tlbcam_addrs[index].limit = virt + size - 1;
0135     tlbcam_addrs[index].phys = phys;
0136 }
0137
0138 unsigned long calc_cam_sz(unsigned long ram, unsigned long virt,
0139               phys_addr_t phys)
0140 {
0141     unsigned int camsize = __ilog2(ram);
0142     unsigned int align = __ffs(virt | phys);
0143     unsigned long max_cam;
0144
0145     if ((mfspr(SPRN_MMUCFG) & MMUCFG_MAVN) == MMUCFG_MAVN_V1) {
0146         /* Convert (4^max) kB to (2^max) bytes */
0147         max_cam = ((mfspr(SPRN_TLB1CFG) >> 16) & 0xf) * 2 + 10;
0148         camsize &= ~1U;
0149         align &= ~1U;
0150     } else {
0151         /* Convert (2^max) kB to (2^max) bytes */
0152         max_cam = __ilog2(mfspr(SPRN_TLB1PS)) + 10;
0153     }
0154
0155     if (camsize > align)
0156         camsize = align;
0157     if (camsize > max_cam)
0158         camsize = max_cam;
0159
0160     return 1UL << camsize;
0161 }
0162
0163 static unsigned long map_mem_in_cams_addr(phys_addr_t phys, unsigned long virt,
0164                     unsigned long ram, int max_cam_idx,
0165                     bool dryrun, bool init)
0166 {
0167     int i;
0168     unsigned long amount_mapped = 0;
0169     unsigned long boundary;
0170
0171     if (strict_kernel_rwx_enabled())
0172         boundary = (unsigned long)(_sinittext - _stext);
0173     else
0174         boundary = ram;
0175
0176     /* Calculate CAM values */
0177     for (i = 0; boundary && i < max_cam_idx; i++) {
0178         unsigned long cam_sz;
0179         pgprot_t prot = init ? PAGE_KERNEL_X : PAGE_KERNEL_ROX;
0180
0181         cam_sz = calc_cam_sz(boundary, virt, phys);
0182         if (!dryrun)
0183             settlbcam(i, virt, phys, cam_sz, pgprot_val(prot), 0);
0184
0185         boundary -= cam_sz;
0186         amount_mapped += cam_sz;
0187         virt += cam_sz;
0188         phys += cam_sz;
0189     }
0190     for (ram -= amount_mapped; ram && i < max_cam_idx; i++) {
0191         unsigned long cam_sz;
0192         pgprot_t prot = init ? PAGE_KERNEL_X : PAGE_KERNEL;
0193
0194         cam_sz = calc_cam_sz(ram, virt, phys);
0195         if (!dryrun)
0196             settlbcam(i, virt, phys, cam_sz, pgprot_val(prot), 0);
0197
0198         ram -= cam_sz;
0199         amount_mapped += cam_sz;
0200         virt += cam_sz;
0201         phys += cam_sz;
0202     }
0203
0204     if (dryrun)
0205         return amount_mapped;
0206
0207     if (init) {
0208         loadcam_multi(0, i, max_cam_idx);
0209         tlbcam_index = i;
0210     } else {
0211         loadcam_multi(0, i, 0);
0212         WARN_ON(i > tlbcam_index);
0213     }
0214
0215 #ifdef CONFIG_PPC64
0216     get_paca()->tcd.esel_next = i;
0217     get_paca()->tcd.esel_max = mfspr(SPRN_TLB1CFG) & TLBnCFG_N_ENTRY;
0218     get_paca()->tcd.esel_first = i;
0219 #endif
0220
0221     return amount_mapped;
0222 }
0223
0224 unsigned long map_mem_in_cams(unsigned long ram, int max_cam_idx, bool dryrun, bool init)
0225 {
0226     unsigned long virt = PAGE_OFFSET;
0227     phys_addr_t phys = memstart_addr;
0228
0229     return map_mem_in_cams_addr(phys, virt, ram, max_cam_idx, dryrun, init);
0230 }
0231
0232 #ifdef CONFIG_PPC32
0233
0234 #if defined(CONFIG_LOWMEM_CAM_NUM_BOOL) && (CONFIG_LOWMEM_CAM_NUM >= NUM_TLBCAMS)
0235 #error "LOWMEM_CAM_NUM must be less than NUM_TLBCAMS"
0236 #endif
0237
0238 unsigned long __init mmu_mapin_ram(unsigned long base, unsigned long top)
0239 {
0240     return tlbcam_addrs[tlbcam_index - 1].limit - PAGE_OFFSET + 1;
0241 }
0242
0243 void flush_instruction_cache(void)
0244 {
0245     unsigned long tmp;
0246
0247     tmp = mfspr(SPRN_L1CSR1);
0248     tmp |= L1CSR1_ICFI | L1CSR1_ICLFR;
0249     mtspr(SPRN_L1CSR1, tmp);
0250     isync();
0251 }
0252
0253 /*
0254  * MMU_init_hw does the chip-specific initialization of the MMU hardware.
0255  */
0256 void __init MMU_init_hw(void)
0257 {
0258     flush_instruction_cache();
0259 }
0260
0261 static unsigned long __init tlbcam_sz(int idx)
0262 {
0263     return tlbcam_addrs[idx].limit - tlbcam_addrs[idx].start + 1;
0264 }
0265
0266 void __init adjust_total_lowmem(void)
0267 {
0268     unsigned long ram;
0269     int i;
0270
0271     /* adjust lowmem size to __max_low_memory */
0272     ram = min((phys_addr_t)__max_low_memory, (phys_addr_t)total_lowmem);
0273
0274     i = switch_to_as1();
0275     __max_low_memory = map_mem_in_cams(ram, CONFIG_LOWMEM_CAM_NUM, false, true);
0276     restore_to_as0(i, 0, NULL, 1);
0277
0278     pr_info("Memory CAM mapping: ");
0279     for (i = 0; i < tlbcam_index - 1; i++)
0280         pr_cont("%lu/", tlbcam_sz(i) >> 20);
0281     pr_cont("%lu Mb, residual: %dMb\n", tlbcam_sz(tlbcam_index - 1) >> 20,
0282             (unsigned int)((total_lowmem - __max_low_memory) >> 20));
0283
0284     memblock_set_current_limit(memstart_addr + __max_low_memory);
0285 }
0286
0287 #ifdef CONFIG_STRICT_KERNEL_RWX
0288 void mmu_mark_rodata_ro(void)
0289 {
0290     unsigned long remapped;
0291
0292     remapped = map_mem_in_cams(__max_low_memory, CONFIG_LOWMEM_CAM_NUM, false, false);
0293
0294     WARN_ON(__max_low_memory != remapped);
0295 }
0296 #endif
0297
0298 void mmu_mark_initmem_nx(void)
0299 {
0300     /* Everything is done in mmu_mark_rodata_ro() */
0301 }
0302
0303 void setup_initial_memory_limit(phys_addr_t first_memblock_base,
0304                 phys_addr_t first_memblock_size)
0305 {
0306     phys_addr_t limit = first_memblock_base + first_memblock_size;
0307
0308     /* 64M mapped initially according to head_fsl_booke.S */
0309     memblock_set_current_limit(min_t(u64, limit, 0x04000000));
0310 }
0311
0312 #ifdef CONFIG_RELOCATABLE
0313 int __initdata is_second_reloc;
0314 notrace void __init relocate_init(u64 dt_ptr, phys_addr_t start)
0315 {
0316     unsigned long base = kernstart_virt_addr;
0317     phys_addr_t size;
0318
0319     kernstart_addr = start;
0320     if (is_second_reloc) {
0321         virt_phys_offset = PAGE_OFFSET - memstart_addr;
0322         kaslr_late_init();
0323         return;
0324     }
0325
0326     /*
0327      * Relocatable kernel support based on processing of dynamic
0328      * relocation entries. Before we get the real memstart_addr,
0329      * We will compute the virt_phys_offset like this:
0330      * virt_phys_offset = stext.run - kernstart_addr
0331      *
0332      * stext.run = (KERNELBASE & ~0x3ffffff) +
0333      *              (kernstart_addr & 0x3ffffff)
0334      * When we relocate, we have :
0335      *
0336      *  (kernstart_addr & 0x3ffffff) = (stext.run & 0x3ffffff)
0337      *
0338      * hence:
0339      *  virt_phys_offset = (KERNELBASE & ~0x3ffffff) -
0340      *                              (kernstart_addr & ~0x3ffffff)
0341      *
0342      */
0343     start &= ~0x3ffffff;
0344     base &= ~0x3ffffff;
0345     virt_phys_offset = base - start;
0346     early_get_first_memblock_info(__va(dt_ptr), &size);
0347     /*
0348      * We now get the memstart_addr, then we should check if this
0349      * address is the same as what the PAGE_OFFSET map to now. If
0350      * not we have to change the map of PAGE_OFFSET to memstart_addr
0351      * and do a second relocation.
0352      */
0353     if (start != memstart_addr) {
0354         int n;
0355         long offset = start - memstart_addr;
0356
0357         is_second_reloc = 1;
0358         n = switch_to_as1();
0359         /* map a 64M area for the second relocation */
0360         if (memstart_addr > start)
0361             map_mem_in_cams(0x4000000, CONFIG_LOWMEM_CAM_NUM,
0362                     false, true);
0363         else
0364             map_mem_in_cams_addr(start, PAGE_OFFSET + offset,
0365                     0x4000000, CONFIG_LOWMEM_CAM_NUM,
0366                     false, true);
0367         restore_to_as0(n, offset, __va(dt_ptr), 1);
0368         /* We should never reach here */
0369         panic("Relocation error");
0370     }
0371
0372     kaslr_early_init(__va(dt_ptr), size);
0373 }
0374 #endif
0375 #endif