mm/kasan/init_32.c

0001 // SPDX-License-Identifier: GPL-2.0
0002
0003 #define DISABLE_BRANCH_PROFILING
0004
0005 #include <linux/kasan.h>
0006 #include <linux/printk.h>
0007 #include <linux/memblock.h>
0008 #include <linux/sched/task.h>
0009 #include <asm/pgalloc.h>
0010 #include <asm/code-patching.h>
0011 #include <mm/mmu_decl.h>
0012
0013 static pgprot_t __init kasan_prot_ro(void)
0014 {
0015     if (early_mmu_has_feature(MMU_FTR_HPTE_TABLE))
0016         return PAGE_READONLY;
0017
0018     return PAGE_KERNEL_RO;
0019 }
0020
0021 static void __init kasan_populate_pte(pte_t *ptep, pgprot_t prot)
0022 {
0023     unsigned long va = (unsigned long)kasan_early_shadow_page;
0024     phys_addr_t pa = __pa(kasan_early_shadow_page);
0025     int i;
0026
0027     for (i = 0; i < PTRS_PER_PTE; i++, ptep++)
0028         __set_pte_at(&init_mm, va, ptep, pfn_pte(PHYS_PFN(pa), prot), 1);
0029 }
0030
0031 int __init kasan_init_shadow_page_tables(unsigned long k_start, unsigned long k_end)
0032 {
0033     pmd_t *pmd;
0034     unsigned long k_cur, k_next;
0035
0036     pmd = pmd_off_k(k_start);
0037
0038     for (k_cur = k_start; k_cur != k_end; k_cur = k_next, pmd++) {
0039         pte_t *new;
0040
0041         k_next = pgd_addr_end(k_cur, k_end);
0042         if ((void *)pmd_page_vaddr(*pmd) != kasan_early_shadow_pte)
0043             continue;
0044
0045         new = memblock_alloc(PTE_FRAG_SIZE, PTE_FRAG_SIZE);
0046
0047         if (!new)
0048             return -ENOMEM;
0049         kasan_populate_pte(new, PAGE_KERNEL);
0050         pmd_populate_kernel(&init_mm, pmd, new);
0051     }
0052     return 0;
0053 }
0054
0055 int __init __weak kasan_init_region(void *start, size_t size)
0056 {
0057     unsigned long k_start = (unsigned long)kasan_mem_to_shadow(start);
0058     unsigned long k_end = (unsigned long)kasan_mem_to_shadow(start + size);
0059     unsigned long k_cur;
0060     int ret;
0061     void *block;
0062
0063     ret = kasan_init_shadow_page_tables(k_start, k_end);
0064     if (ret)
0065         return ret;
0066
0067     block = memblock_alloc(k_end - k_start, PAGE_SIZE);
0068     if (!block)
0069         return -ENOMEM;
0070
0071     for (k_cur = k_start & PAGE_MASK; k_cur < k_end; k_cur += PAGE_SIZE) {
0072         pmd_t *pmd = pmd_off_k(k_cur);
0073         void *va = block + k_cur - k_start;
0074         pte_t pte = pfn_pte(PHYS_PFN(__pa(va)), PAGE_KERNEL);
0075
0076         __set_pte_at(&init_mm, k_cur, pte_offset_kernel(pmd, k_cur), pte, 0);
0077     }
0078     flush_tlb_kernel_range(k_start, k_end);
0079     return 0;
0080 }
0081
0082 void __init
0083 kasan_update_early_region(unsigned long k_start, unsigned long k_end, pte_t pte)
0084 {
0085     unsigned long k_cur;
0086
0087     for (k_cur = k_start; k_cur != k_end; k_cur += PAGE_SIZE) {
0088         pmd_t *pmd = pmd_off_k(k_cur);
0089         pte_t *ptep = pte_offset_kernel(pmd, k_cur);
0090
0091         if (pte_page(*ptep) != virt_to_page(lm_alias(kasan_early_shadow_page)))
0092             continue;
0093
0094         __set_pte_at(&init_mm, k_cur, ptep, pte, 0);
0095     }
0096
0097     flush_tlb_kernel_range(k_start, k_end);
0098 }
0099
0100 static void __init kasan_remap_early_shadow_ro(void)
0101 {
0102     pgprot_t prot = kasan_prot_ro();
0103     phys_addr_t pa = __pa(kasan_early_shadow_page);
0104
0105     kasan_populate_pte(kasan_early_shadow_pte, prot);
0106
0107     kasan_update_early_region(KASAN_SHADOW_START, KASAN_SHADOW_END,
0108                   pfn_pte(PHYS_PFN(pa), prot));
0109 }
0110
0111 static void __init kasan_unmap_early_shadow_vmalloc(void)
0112 {
0113     unsigned long k_start = (unsigned long)kasan_mem_to_shadow((void *)VMALLOC_START);
0114     unsigned long k_end = (unsigned long)kasan_mem_to_shadow((void *)VMALLOC_END);
0115
0116     kasan_update_early_region(k_start, k_end, __pte(0));
0117
0118 #ifdef MODULES_VADDR
0119     k_start = (unsigned long)kasan_mem_to_shadow((void *)MODULES_VADDR);
0120     k_end = (unsigned long)kasan_mem_to_shadow((void *)MODULES_END);
0121     kasan_update_early_region(k_start, k_end, __pte(0));
0122 #endif
0123 }
0124
0125 void __init kasan_mmu_init(void)
0126 {
0127     int ret;
0128
0129     if (early_mmu_has_feature(MMU_FTR_HPTE_TABLE)) {
0130         ret = kasan_init_shadow_page_tables(KASAN_SHADOW_START, KASAN_SHADOW_END);
0131
0132         if (ret)
0133             panic("kasan: kasan_init_shadow_page_tables() failed");
0134     }
0135 }
0136
0137 void __init kasan_init(void)
0138 {
0139     phys_addr_t base, end;
0140     u64 i;
0141     int ret;
0142
0143     for_each_mem_range(i, &base, &end) {
0144         phys_addr_t top = min(end, total_lowmem);
0145
0146         if (base >= top)
0147             continue;
0148
0149         ret = kasan_init_region(__va(base), top - base);
0150         if (ret)
0151             panic("kasan: kasan_init_region() failed");
0152     }
0153
0154     if (IS_ENABLED(CONFIG_KASAN_VMALLOC)) {
0155         ret = kasan_init_shadow_page_tables(KASAN_SHADOW_START, KASAN_SHADOW_END);
0156
0157         if (ret)
0158             panic("kasan: kasan_init_shadow_page_tables() failed");
0159     }
0160
0161     kasan_remap_early_shadow_ro();
0162
0163     clear_page(kasan_early_shadow_page);
0164
0165     /* At this point kasan is fully initialized. Enable error messages */
0166     init_task.kasan_depth = 0;
0167     pr_info("KASAN init done\n");
0168 }
0169
0170 void __init kasan_late_init(void)
0171 {
0172     if (IS_ENABLED(CONFIG_KASAN_VMALLOC))
0173         kasan_unmap_early_shadow_vmalloc();
0174 }
0175
0176 void __init kasan_early_init(void)
0177 {
0178     unsigned long addr = KASAN_SHADOW_START;
0179     unsigned long end = KASAN_SHADOW_END;
0180     unsigned long next;
0181     pmd_t *pmd = pmd_off_k(addr);
0182
0183     BUILD_BUG_ON(KASAN_SHADOW_START & ~PGDIR_MASK);
0184
0185     kasan_populate_pte(kasan_early_shadow_pte, PAGE_KERNEL);
0186
0187     do {
0188         next = pgd_addr_end(addr, end);
0189         pmd_populate_kernel(&init_mm, pmd, kasan_early_shadow_pte);
0190     } while (pmd++, addr = next, addr != end);
0191 }