arm/mm/pgd.c

0001 // SPDX-License-Identifier: GPL-2.0-only
0002 /*
0003  *  linux/arch/arm/mm/pgd.c
0004  *
0005  *  Copyright (C) 1998-2005 Russell King
0006  */
0007 #include <linux/mm.h>
0008 #include <linux/gfp.h>
0009 #include <linux/highmem.h>
0010 #include <linux/slab.h>
0011
0012 #include <asm/cp15.h>
0013 #include <asm/pgalloc.h>
0014 #include <asm/page.h>
0015 #include <asm/tlbflush.h>
0016
0017 #include "mm.h"
0018
0019 #ifdef CONFIG_ARM_LPAE
0020 #define __pgd_alloc()   kmalloc_array(PTRS_PER_PGD, sizeof(pgd_t), GFP_KERNEL)
0021 #define __pgd_free(pgd) kfree(pgd)
0022 #else
0023 #define __pgd_alloc()   (pgd_t *)__get_free_pages(GFP_KERNEL, 2)
0024 #define __pgd_free(pgd) free_pages((unsigned long)pgd, 2)
0025 #endif
0026
0027 /*
0028  * need to get a 16k page for level 1
0029  */
0030 pgd_t *pgd_alloc(struct mm_struct *mm)
0031 {
0032     pgd_t *new_pgd, *init_pgd;
0033     p4d_t *new_p4d, *init_p4d;
0034     pud_t *new_pud, *init_pud;
0035     pmd_t *new_pmd, *init_pmd;
0036     pte_t *new_pte, *init_pte;
0037
0038     new_pgd = __pgd_alloc();
0039     if (!new_pgd)
0040         goto no_pgd;
0041
0042     memset(new_pgd, 0, USER_PTRS_PER_PGD * sizeof(pgd_t));
0043
0044     /*
0045      * Copy over the kernel and IO PGD entries
0046      */
0047     init_pgd = pgd_offset_k(0);
0048     memcpy(new_pgd + USER_PTRS_PER_PGD, init_pgd + USER_PTRS_PER_PGD,
0049                (PTRS_PER_PGD - USER_PTRS_PER_PGD) * sizeof(pgd_t));
0050
0051     clean_dcache_area(new_pgd, PTRS_PER_PGD * sizeof(pgd_t));
0052
0053 #ifdef CONFIG_ARM_LPAE
0054     /*
0055      * Allocate PMD table for modules and pkmap mappings.
0056      */
0057     new_p4d = p4d_alloc(mm, new_pgd + pgd_index(MODULES_VADDR),
0058                 MODULES_VADDR);
0059     if (!new_p4d)
0060         goto no_p4d;
0061
0062     new_pud = pud_alloc(mm, new_p4d, MODULES_VADDR);
0063     if (!new_pud)
0064         goto no_pud;
0065
0066     new_pmd = pmd_alloc(mm, new_pud, 0);
0067     if (!new_pmd)
0068         goto no_pmd;
0069 #ifdef CONFIG_KASAN
0070     /*
0071      * Copy PMD table for KASAN shadow mappings.
0072      */
0073     init_pgd = pgd_offset_k(TASK_SIZE);
0074     init_p4d = p4d_offset(init_pgd, TASK_SIZE);
0075     init_pud = pud_offset(init_p4d, TASK_SIZE);
0076     init_pmd = pmd_offset(init_pud, TASK_SIZE);
0077     new_pmd = pmd_offset(new_pud, TASK_SIZE);
0078     memcpy(new_pmd, init_pmd,
0079            (pmd_index(MODULES_VADDR) - pmd_index(TASK_SIZE))
0080            * sizeof(pmd_t));
0081     clean_dcache_area(new_pmd, PTRS_PER_PMD * sizeof(pmd_t));
0082 #endif /* CONFIG_KASAN */
0083 #endif /* CONFIG_LPAE */
0084
0085     if (!vectors_high()) {
0086         /*
0087          * On ARM, first page must always be allocated since it
0088          * contains the machine vectors. The vectors are always high
0089          * with LPAE.
0090          */
0091         new_p4d = p4d_alloc(mm, new_pgd, 0);
0092         if (!new_p4d)
0093             goto no_p4d;
0094
0095         new_pud = pud_alloc(mm, new_p4d, 0);
0096         if (!new_pud)
0097             goto no_pud;
0098
0099         new_pmd = pmd_alloc(mm, new_pud, 0);
0100         if (!new_pmd)
0101             goto no_pmd;
0102
0103         new_pte = pte_alloc_map(mm, new_pmd, 0);
0104         if (!new_pte)
0105             goto no_pte;
0106
0107 #ifndef CONFIG_ARM_LPAE
0108         /*
0109          * Modify the PTE pointer to have the correct domain.  This
0110          * needs to be the vectors domain to avoid the low vectors
0111          * being unmapped.
0112          */
0113         pmd_val(*new_pmd) &= ~PMD_DOMAIN_MASK;
0114         pmd_val(*new_pmd) |= PMD_DOMAIN(DOMAIN_VECTORS);
0115 #endif
0116
0117         init_p4d = p4d_offset(init_pgd, 0);
0118         init_pud = pud_offset(init_p4d, 0);
0119         init_pmd = pmd_offset(init_pud, 0);
0120         init_pte = pte_offset_map(init_pmd, 0);
0121         set_pte_ext(new_pte + 0, init_pte[0], 0);
0122         set_pte_ext(new_pte + 1, init_pte[1], 0);
0123         pte_unmap(init_pte);
0124         pte_unmap(new_pte);
0125     }
0126
0127     return new_pgd;
0128
0129 no_pte:
0130     pmd_free(mm, new_pmd);
0131     mm_dec_nr_pmds(mm);
0132 no_pmd:
0133     pud_free(mm, new_pud);
0134 no_pud:
0135     p4d_free(mm, new_p4d);
0136 no_p4d:
0137     __pgd_free(new_pgd);
0138 no_pgd:
0139     return NULL;
0140 }
0141
0142 void pgd_free(struct mm_struct *mm, pgd_t *pgd_base)
0143 {
0144     pgd_t *pgd;
0145     p4d_t *p4d;
0146     pud_t *pud;
0147     pmd_t *pmd;
0148     pgtable_t pte;
0149
0150     if (!pgd_base)
0151         return;
0152
0153     pgd = pgd_base + pgd_index(0);
0154     if (pgd_none_or_clear_bad(pgd))
0155         goto no_pgd;
0156
0157     p4d = p4d_offset(pgd, 0);
0158     if (p4d_none_or_clear_bad(p4d))
0159         goto no_p4d;
0160
0161     pud = pud_offset(p4d, 0);
0162     if (pud_none_or_clear_bad(pud))
0163         goto no_pud;
0164
0165     pmd = pmd_offset(pud, 0);
0166     if (pmd_none_or_clear_bad(pmd))
0167         goto no_pmd;
0168
0169     pte = pmd_pgtable(*pmd);
0170     pmd_clear(pmd);
0171     pte_free(mm, pte);
0172     mm_dec_nr_ptes(mm);
0173 no_pmd:
0174     pud_clear(pud);
0175     pmd_free(mm, pmd);
0176     mm_dec_nr_pmds(mm);
0177 no_pud:
0178     p4d_clear(p4d);
0179     pud_free(mm, pud);
0180 no_p4d:
0181     pgd_clear(pgd);
0182     p4d_free(mm, p4d);
0183 no_pgd:
0184 #ifdef CONFIG_ARM_LPAE
0185     /*
0186      * Free modules/pkmap or identity pmd tables.
0187      */
0188     for (pgd = pgd_base; pgd < pgd_base + PTRS_PER_PGD; pgd++) {
0189         if (pgd_none_or_clear_bad(pgd))
0190             continue;
0191         if (pgd_val(*pgd) & L_PGD_SWAPPER)
0192             continue;
0193         p4d = p4d_offset(pgd, 0);
0194         if (p4d_none_or_clear_bad(p4d))
0195             continue;
0196         pud = pud_offset(p4d, 0);
0197         if (pud_none_or_clear_bad(pud))
0198             continue;
0199         pmd = pmd_offset(pud, 0);
0200         pud_clear(pud);
0201         pmd_free(mm, pmd);
0202         mm_dec_nr_pmds(mm);
0203         p4d_clear(p4d);
0204         pud_free(mm, pud);
0205         mm_dec_nr_puds(mm);
0206         pgd_clear(pgd);
0207         p4d_free(mm, p4d);
0208     }
0209 #endif
0210     __pgd_free(pgd_base);
0211 }