x86/mm/ident_map.c

0001 // SPDX-License-Identifier: GPL-2.0
0002 /*
0003  * Helper routines for building identity mapping page tables. This is
0004  * included by both the compressed kernel and the regular kernel.
0005  */
0006
0007 static void ident_pmd_init(struct x86_mapping_info *info, pmd_t *pmd_page,
0008                unsigned long addr, unsigned long end)
0009 {
0010     addr &= PMD_MASK;
0011     for (; addr < end; addr += PMD_SIZE) {
0012         pmd_t *pmd = pmd_page + pmd_index(addr);
0013
0014         if (pmd_present(*pmd))
0015             continue;
0016
0017         set_pmd(pmd, __pmd((addr - info->offset) | info->page_flag));
0018     }
0019 }
0020
0021 static int ident_pud_init(struct x86_mapping_info *info, pud_t *pud_page,
0022               unsigned long addr, unsigned long end)
0023 {
0024     unsigned long next;
0025
0026     for (; addr < end; addr = next) {
0027         pud_t *pud = pud_page + pud_index(addr);
0028         pmd_t *pmd;
0029
0030         next = (addr & PUD_MASK) + PUD_SIZE;
0031         if (next > end)
0032             next = end;
0033
0034         if (info->direct_gbpages) {
0035             pud_t pudval;
0036
0037             if (pud_present(*pud))
0038                 continue;
0039
0040             addr &= PUD_MASK;
0041             pudval = __pud((addr - info->offset) | info->page_flag);
0042             set_pud(pud, pudval);
0043             continue;
0044         }
0045
0046         if (pud_present(*pud)) {
0047             pmd = pmd_offset(pud, 0);
0048             ident_pmd_init(info, pmd, addr, next);
0049             continue;
0050         }
0051         pmd = (pmd_t *)info->alloc_pgt_page(info->context);
0052         if (!pmd)
0053             return -ENOMEM;
0054         ident_pmd_init(info, pmd, addr, next);
0055         set_pud(pud, __pud(__pa(pmd) | info->kernpg_flag));
0056     }
0057
0058     return 0;
0059 }
0060
0061 static int ident_p4d_init(struct x86_mapping_info *info, p4d_t *p4d_page,
0062               unsigned long addr, unsigned long end)
0063 {
0064     unsigned long next;
0065     int result;
0066
0067     for (; addr < end; addr = next) {
0068         p4d_t *p4d = p4d_page + p4d_index(addr);
0069         pud_t *pud;
0070
0071         next = (addr & P4D_MASK) + P4D_SIZE;
0072         if (next > end)
0073             next = end;
0074
0075         if (p4d_present(*p4d)) {
0076             pud = pud_offset(p4d, 0);
0077             result = ident_pud_init(info, pud, addr, next);
0078             if (result)
0079                 return result;
0080
0081             continue;
0082         }
0083         pud = (pud_t *)info->alloc_pgt_page(info->context);
0084         if (!pud)
0085             return -ENOMEM;
0086
0087         result = ident_pud_init(info, pud, addr, next);
0088         if (result)
0089             return result;
0090
0091         set_p4d(p4d, __p4d(__pa(pud) | info->kernpg_flag));
0092     }
0093
0094     return 0;
0095 }
0096
0097 int kernel_ident_mapping_init(struct x86_mapping_info *info, pgd_t *pgd_page,
0098                   unsigned long pstart, unsigned long pend)
0099 {
0100     unsigned long addr = pstart + info->offset;
0101     unsigned long end = pend + info->offset;
0102     unsigned long next;
0103     int result;
0104
0105     /* Set the default pagetable flags if not supplied */
0106     if (!info->kernpg_flag)
0107         info->kernpg_flag = _KERNPG_TABLE;
0108
0109     /* Filter out unsupported __PAGE_KERNEL_* bits: */
0110     info->kernpg_flag &= __default_kernel_pte_mask;
0111
0112     for (; addr < end; addr = next) {
0113         pgd_t *pgd = pgd_page + pgd_index(addr);
0114         p4d_t *p4d;
0115
0116         next = (addr & PGDIR_MASK) + PGDIR_SIZE;
0117         if (next > end)
0118             next = end;
0119
0120         if (pgd_present(*pgd)) {
0121             p4d = p4d_offset(pgd, 0);
0122             result = ident_p4d_init(info, p4d, addr, next);
0123             if (result)
0124                 return result;
0125             continue;
0126         }
0127
0128         p4d = (p4d_t *)info->alloc_pgt_page(info->context);
0129         if (!p4d)
0130             return -ENOMEM;
0131         result = ident_p4d_init(info, p4d, addr, next);
0132         if (result)
0133             return result;
0134         if (pgtable_l5_enabled()) {
0135             set_pgd(pgd, __pgd(__pa(p4d) | info->kernpg_flag));
0136         } else {
0137             /*
0138              * With p4d folded, pgd is equal to p4d.
0139              * The pgd entry has to point to the pud page table in this case.
0140              */
0141             pud_t *pud = pud_offset(p4d, 0);
0142             set_pgd(pgd, __pgd(__pa(pud) | info->kernpg_flag));
0143         }
0144     }
0145
0146     return 0;
0147 }