arm/mm/init.c

0001 // SPDX-License-Identifier: GPL-2.0-only
0002 /*
0003  *  linux/arch/arm/mm/init.c
0004  *
0005  *  Copyright (C) 1995-2005 Russell King
0006  */
0007 #include <linux/kernel.h>
0008 #include <linux/errno.h>
0009 #include <linux/swap.h>
0010 #include <linux/init.h>
0011 #include <linux/mman.h>
0012 #include <linux/sched/signal.h>
0013 #include <linux/sched/task.h>
0014 #include <linux/export.h>
0015 #include <linux/nodemask.h>
0016 #include <linux/initrd.h>
0017 #include <linux/of_fdt.h>
0018 #include <linux/highmem.h>
0019 #include <linux/gfp.h>
0020 #include <linux/memblock.h>
0021 #include <linux/dma-map-ops.h>
0022 #include <linux/sizes.h>
0023 #include <linux/stop_machine.h>
0024 #include <linux/swiotlb.h>
0025
0026 #include <asm/cp15.h>
0027 #include <asm/mach-types.h>
0028 #include <asm/memblock.h>
0029 #include <asm/memory.h>
0030 #include <asm/prom.h>
0031 #include <asm/sections.h>
0032 #include <asm/setup.h>
0033 #include <asm/set_memory.h>
0034 #include <asm/system_info.h>
0035 #include <asm/tlb.h>
0036 #include <asm/fixmap.h>
0037 #include <asm/ptdump.h>
0038
0039 #include <asm/mach/arch.h>
0040 #include <asm/mach/map.h>
0041
0042 #include "mm.h"
0043
0044 #ifdef CONFIG_CPU_CP15_MMU
0045 unsigned long __init __clear_cr(unsigned long mask)
0046 {
0047     cr_alignment = cr_alignment & ~mask;
0048     return cr_alignment;
0049 }
0050 #endif
0051
0052 #ifdef CONFIG_BLK_DEV_INITRD
0053 static int __init parse_tag_initrd(const struct tag *tag)
0054 {
0055     pr_warn("ATAG_INITRD is deprecated; "
0056         "please update your bootloader.\n");
0057     phys_initrd_start = __virt_to_phys(tag->u.initrd.start);
0058     phys_initrd_size = tag->u.initrd.size;
0059     return 0;
0060 }
0061
0062 __tagtable(ATAG_INITRD, parse_tag_initrd);
0063
0064 static int __init parse_tag_initrd2(const struct tag *tag)
0065 {
0066     phys_initrd_start = tag->u.initrd.start;
0067     phys_initrd_size = tag->u.initrd.size;
0068     return 0;
0069 }
0070
0071 __tagtable(ATAG_INITRD2, parse_tag_initrd2);
0072 #endif
0073
0074 static void __init find_limits(unsigned long *min, unsigned long *max_low,
0075                    unsigned long *max_high)
0076 {
0077     *max_low = PFN_DOWN(memblock_get_current_limit());
0078     *min = PFN_UP(memblock_start_of_DRAM());
0079     *max_high = PFN_DOWN(memblock_end_of_DRAM());
0080 }
0081
0082 #ifdef CONFIG_ZONE_DMA
0083
0084 phys_addr_t arm_dma_zone_size __read_mostly;
0085 EXPORT_SYMBOL(arm_dma_zone_size);
0086
0087 /*
0088  * The DMA mask corresponding to the maximum bus address allocatable
0089  * using GFP_DMA.  The default here places no restriction on DMA
0090  * allocations.  This must be the smallest DMA mask in the system,
0091  * so a successful GFP_DMA allocation will always satisfy this.
0092  */
0093 phys_addr_t arm_dma_limit;
0094 unsigned long arm_dma_pfn_limit;
0095 #endif
0096
0097 void __init setup_dma_zone(const struct machine_desc *mdesc)
0098 {
0099 #ifdef CONFIG_ZONE_DMA
0100     if (mdesc->dma_zone_size) {
0101         arm_dma_zone_size = mdesc->dma_zone_size;
0102         arm_dma_limit = PHYS_OFFSET + arm_dma_zone_size - 1;
0103     } else
0104         arm_dma_limit = 0xffffffff;
0105     arm_dma_pfn_limit = arm_dma_limit >> PAGE_SHIFT;
0106 #endif
0107 }
0108
0109 static void __init zone_sizes_init(unsigned long min, unsigned long max_low,
0110     unsigned long max_high)
0111 {
0112     unsigned long max_zone_pfn[MAX_NR_ZONES] = { 0 };
0113
0114 #ifdef CONFIG_ZONE_DMA
0115     max_zone_pfn[ZONE_DMA] = min(arm_dma_pfn_limit, max_low);
0116 #endif
0117     max_zone_pfn[ZONE_NORMAL] = max_low;
0118 #ifdef CONFIG_HIGHMEM
0119     max_zone_pfn[ZONE_HIGHMEM] = max_high;
0120 #endif
0121     free_area_init(max_zone_pfn);
0122 }
0123
0124 #ifdef CONFIG_HAVE_ARCH_PFN_VALID
0125 int pfn_valid(unsigned long pfn)
0126 {
0127     phys_addr_t addr = __pfn_to_phys(pfn);
0128     unsigned long pageblock_size = PAGE_SIZE * pageblock_nr_pages;
0129
0130     if (__phys_to_pfn(addr) != pfn)
0131         return 0;
0132
0133     /*
0134      * If address less than pageblock_size bytes away from a present
0135      * memory chunk there still will be a memory map entry for it
0136      * because we round freed memory map to the pageblock boundaries.
0137      */
0138     if (memblock_overlaps_region(&memblock.memory,
0139                      ALIGN_DOWN(addr, pageblock_size),
0140                      pageblock_size))
0141         return 1;
0142
0143     return 0;
0144 }
0145 EXPORT_SYMBOL(pfn_valid);
0146 #endif
0147
0148 static bool arm_memblock_steal_permitted = true;
0149
0150 phys_addr_t __init arm_memblock_steal(phys_addr_t size, phys_addr_t align)
0151 {
0152     phys_addr_t phys;
0153
0154     BUG_ON(!arm_memblock_steal_permitted);
0155
0156     phys = memblock_phys_alloc(size, align);
0157     if (!phys)
0158         panic("Failed to steal %pa bytes at %pS\n",
0159               &size, (void *)_RET_IP_);
0160
0161     memblock_phys_free(phys, size);
0162     memblock_remove(phys, size);
0163
0164     return phys;
0165 }
0166
0167 #ifdef CONFIG_CPU_ICACHE_MISMATCH_WORKAROUND
0168 void check_cpu_icache_size(int cpuid)
0169 {
0170     u32 size, ctr;
0171
0172     asm("mrc p15, 0, %0, c0, c0, 1" : "=r" (ctr));
0173
0174     size = 1 << ((ctr & 0xf) + 2);
0175     if (cpuid != 0 && icache_size != size)
0176         pr_info("CPU%u: detected I-Cache line size mismatch, workaround enabled\n",
0177             cpuid);
0178     if (icache_size > size)
0179         icache_size = size;
0180 }
0181 #endif
0182
0183 void __init arm_memblock_init(const struct machine_desc *mdesc)
0184 {
0185     /* Register the kernel text, kernel data and initrd with memblock. */
0186     memblock_reserve(__pa(KERNEL_START), KERNEL_END - KERNEL_START);
0187
0188     reserve_initrd_mem();
0189
0190     arm_mm_memblock_reserve();
0191
0192     /* reserve any platform specific memblock areas */
0193     if (mdesc->reserve)
0194         mdesc->reserve();
0195
0196     early_init_fdt_scan_reserved_mem();
0197
0198     /* reserve memory for DMA contiguous allocations */
0199     dma_contiguous_reserve(arm_dma_limit);
0200
0201     arm_memblock_steal_permitted = false;
0202     memblock_dump_all();
0203 }
0204
0205 void __init bootmem_init(void)
0206 {
0207     memblock_allow_resize();
0208
0209     find_limits(&min_low_pfn, &max_low_pfn, &max_pfn);
0210
0211     early_memtest((phys_addr_t)min_low_pfn << PAGE_SHIFT,
0212               (phys_addr_t)max_low_pfn << PAGE_SHIFT);
0213
0214     /*
0215      * sparse_init() tries to allocate memory from memblock, so must be
0216      * done after the fixed reservations
0217      */
0218     sparse_init();
0219
0220     /*
0221      * Now free the memory - free_area_init needs
0222      * the sparse mem_map arrays initialized by sparse_init()
0223      * for memmap_init_zone(), otherwise all PFNs are invalid.
0224      */
0225     zone_sizes_init(min_low_pfn, max_low_pfn, max_pfn);
0226 }
0227
0228 /*
0229  * Poison init memory with an undefined instruction (ARM) or a branch to an
0230  * undefined instruction (Thumb).
0231  */
0232 static inline void poison_init_mem(void *s, size_t count)
0233 {
0234     u32 *p = (u32 *)s;
0235     for (; count != 0; count -= 4)
0236         *p++ = 0xe7fddef0;
0237 }
0238
0239 static void __init free_highpages(void)
0240 {
0241 #ifdef CONFIG_HIGHMEM
0242     unsigned long max_low = max_low_pfn;
0243     phys_addr_t range_start, range_end;
0244     u64 i;
0245
0246     /* set highmem page free */
0247     for_each_free_mem_range(i, NUMA_NO_NODE, MEMBLOCK_NONE,
0248                 &range_start, &range_end, NULL) {
0249         unsigned long start = PFN_UP(range_start);
0250         unsigned long end = PFN_DOWN(range_end);
0251
0252         /* Ignore complete lowmem entries */
0253         if (end <= max_low)
0254             continue;
0255
0256         /* Truncate partial highmem entries */
0257         if (start < max_low)
0258             start = max_low;
0259
0260         for (; start < end; start++)
0261             free_highmem_page(pfn_to_page(start));
0262     }
0263 #endif
0264 }
0265
0266 /*
0267  * mem_init() marks the free areas in the mem_map and tells us how much
0268  * memory is free.  This is done after various parts of the system have
0269  * claimed their memory after the kernel image.
0270  */
0271 void __init mem_init(void)
0272 {
0273 #ifdef CONFIG_ARM_LPAE
0274     swiotlb_init(max_pfn > arm_dma_pfn_limit, SWIOTLB_VERBOSE);
0275 #endif
0276
0277     set_max_mapnr(pfn_to_page(max_pfn) - mem_map);
0278
0279     /* this will put all unused low memory onto the freelists */
0280     memblock_free_all();
0281
0282 #ifdef CONFIG_SA1111
0283     /* now that our DMA memory is actually so designated, we can free it */
0284     free_reserved_area(__va(PHYS_OFFSET), swapper_pg_dir, -1, NULL);
0285 #endif
0286
0287     free_highpages();
0288
0289     /*
0290      * Check boundaries twice: Some fundamental inconsistencies can
0291      * be detected at build time already.
0292      */
0293 #ifdef CONFIG_MMU
0294     BUILD_BUG_ON(TASK_SIZE              > MODULES_VADDR);
0295     BUG_ON(TASK_SIZE                > MODULES_VADDR);
0296 #endif
0297
0298 #ifdef CONFIG_HIGHMEM
0299     BUILD_BUG_ON(PKMAP_BASE + LAST_PKMAP * PAGE_SIZE > PAGE_OFFSET);
0300     BUG_ON(PKMAP_BASE + LAST_PKMAP * PAGE_SIZE  > PAGE_OFFSET);
0301 #endif
0302 }
0303
0304 #ifdef CONFIG_STRICT_KERNEL_RWX
0305 struct section_perm {
0306     const char *name;
0307     unsigned long start;
0308     unsigned long end;
0309     pmdval_t mask;
0310     pmdval_t prot;
0311     pmdval_t clear;
0312 };
0313
0314 /* First section-aligned location at or after __start_rodata. */
0315 extern char __start_rodata_section_aligned[];
0316
0317 static struct section_perm nx_perms[] = {
0318     /* Make pages tables, etc before _stext RW (set NX). */
0319     {
0320         .name   = "pre-text NX",
0321         .start  = PAGE_OFFSET,
0322         .end    = (unsigned long)_stext,
0323         .mask   = ~PMD_SECT_XN,
0324         .prot   = PMD_SECT_XN,
0325     },
0326     /* Make init RW (set NX). */
0327     {
0328         .name   = "init NX",
0329         .start  = (unsigned long)__init_begin,
0330         .end    = (unsigned long)_sdata,
0331         .mask   = ~PMD_SECT_XN,
0332         .prot   = PMD_SECT_XN,
0333     },
0334     /* Make rodata NX (set RO in ro_perms below). */
0335     {
0336         .name   = "rodata NX",
0337         .start  = (unsigned long)__start_rodata_section_aligned,
0338         .end    = (unsigned long)__init_begin,
0339         .mask   = ~PMD_SECT_XN,
0340         .prot   = PMD_SECT_XN,
0341     },
0342 };
0343
0344 static struct section_perm ro_perms[] = {
0345     /* Make kernel code and rodata RX (set RO). */
0346     {
0347         .name   = "text/rodata RO",
0348         .start  = (unsigned long)_stext,
0349         .end    = (unsigned long)__init_begin,
0350 #ifdef CONFIG_ARM_LPAE
0351         .mask   = ~(L_PMD_SECT_RDONLY | PMD_SECT_AP2),
0352         .prot   = L_PMD_SECT_RDONLY | PMD_SECT_AP2,
0353 #else
0354         .mask   = ~(PMD_SECT_APX | PMD_SECT_AP_WRITE),
0355         .prot   = PMD_SECT_APX | PMD_SECT_AP_WRITE,
0356         .clear  = PMD_SECT_AP_WRITE,
0357 #endif
0358     },
0359 };
0360
0361 /*
0362  * Updates section permissions only for the current mm (sections are
0363  * copied into each mm). During startup, this is the init_mm. Is only
0364  * safe to be called with preemption disabled, as under stop_machine().
0365  */
0366 static inline void section_update(unsigned long addr, pmdval_t mask,
0367                   pmdval_t prot, struct mm_struct *mm)
0368 {
0369     pmd_t *pmd;
0370
0371     pmd = pmd_offset(pud_offset(p4d_offset(pgd_offset(mm, addr), addr), addr), addr);
0372
0373 #ifdef CONFIG_ARM_LPAE
0374     pmd[0] = __pmd((pmd_val(pmd[0]) & mask) | prot);
0375 #else
0376     if (addr & SECTION_SIZE)
0377         pmd[1] = __pmd((pmd_val(pmd[1]) & mask) | prot);
0378     else
0379         pmd[0] = __pmd((pmd_val(pmd[0]) & mask) | prot);
0380 #endif
0381     flush_pmd_entry(pmd);
0382     local_flush_tlb_kernel_range(addr, addr + SECTION_SIZE);
0383 }
0384
0385 /* Make sure extended page tables are in use. */
0386 static inline bool arch_has_strict_perms(void)
0387 {
0388     if (cpu_architecture() < CPU_ARCH_ARMv6)
0389         return false;
0390
0391     return !!(get_cr() & CR_XP);
0392 }
0393
0394 static void set_section_perms(struct section_perm *perms, int n, bool set,
0395                   struct mm_struct *mm)
0396 {
0397     size_t i;
0398     unsigned long addr;
0399
0400     if (!arch_has_strict_perms())
0401         return;
0402
0403     for (i = 0; i < n; i++) {
0404         if (!IS_ALIGNED(perms[i].start, SECTION_SIZE) ||
0405             !IS_ALIGNED(perms[i].end, SECTION_SIZE)) {
0406             pr_err("BUG: %s section %lx-%lx not aligned to %lx\n",
0407                 perms[i].name, perms[i].start, perms[i].end,
0408                 SECTION_SIZE);
0409             continue;
0410         }
0411
0412         for (addr = perms[i].start;
0413              addr < perms[i].end;
0414              addr += SECTION_SIZE)
0415             section_update(addr, perms[i].mask,
0416                 set ? perms[i].prot : perms[i].clear, mm);
0417     }
0418
0419 }
0420
0421 /**
0422  * update_sections_early intended to be called only through stop_machine
0423  * framework and executed by only one CPU while all other CPUs will spin and
0424  * wait, so no locking is required in this function.
0425  */
0426 static void update_sections_early(struct section_perm perms[], int n)
0427 {
0428     struct task_struct *t, *s;
0429
0430     for_each_process(t) {
0431         if (t->flags & PF_KTHREAD)
0432             continue;
0433         for_each_thread(t, s)
0434             if (s->mm)
0435                 set_section_perms(perms, n, true, s->mm);
0436     }
0437     set_section_perms(perms, n, true, current->active_mm);
0438     set_section_perms(perms, n, true, &init_mm);
0439 }
0440
0441 static int __fix_kernmem_perms(void *unused)
0442 {
0443     update_sections_early(nx_perms, ARRAY_SIZE(nx_perms));
0444     return 0;
0445 }
0446
0447 static void fix_kernmem_perms(void)
0448 {
0449     stop_machine(__fix_kernmem_perms, NULL, NULL);
0450 }
0451
0452 static int __mark_rodata_ro(void *unused)
0453 {
0454     update_sections_early(ro_perms, ARRAY_SIZE(ro_perms));
0455     return 0;
0456 }
0457
0458 void mark_rodata_ro(void)
0459 {
0460     stop_machine(__mark_rodata_ro, NULL, NULL);
0461     debug_checkwx();
0462 }
0463
0464 #else
0465 static inline void fix_kernmem_perms(void) { }
0466 #endif /* CONFIG_STRICT_KERNEL_RWX */
0467
0468 void free_initmem(void)
0469 {
0470     fix_kernmem_perms();
0471
0472     poison_init_mem(__init_begin, __init_end - __init_begin);
0473     if (!machine_is_integrator() && !machine_is_cintegrator())
0474         free_initmem_default(-1);
0475 }
0476
0477 #ifdef CONFIG_BLK_DEV_INITRD
0478 void free_initrd_mem(unsigned long start, unsigned long end)
0479 {
0480     if (start == initrd_start)
0481         start = round_down(start, PAGE_SIZE);
0482     if (end == initrd_end)
0483         end = round_up(end, PAGE_SIZE);
0484
0485     poison_init_mem((void *)start, PAGE_ALIGN(end) - start);
0486     free_reserved_area((void *)start, (void *)end, -1, "initrd");
0487 }
0488 #endif