include/asm/pgalloc.h

0001 /* SPDX-License-Identifier: GPL-2.0 */
0002 #ifndef _ASM_X86_PGALLOC_H
0003 #define _ASM_X86_PGALLOC_H
0004
0005 #include <linux/threads.h>
0006 #include <linux/mm.h>       /* for struct page */
0007 #include <linux/pagemap.h>
0008
0009 #define __HAVE_ARCH_PTE_ALLOC_ONE
0010 #define __HAVE_ARCH_PGD_FREE
0011 #include <asm-generic/pgalloc.h>
0012
0013 static inline int  __paravirt_pgd_alloc(struct mm_struct *mm) { return 0; }
0014
0015 #ifdef CONFIG_PARAVIRT_XXL
0016 #include <asm/paravirt.h>
0017 #else
0018 #define paravirt_pgd_alloc(mm)  __paravirt_pgd_alloc(mm)
0019 static inline void paravirt_pgd_free(struct mm_struct *mm, pgd_t *pgd) {}
0020 static inline void paravirt_alloc_pte(struct mm_struct *mm, unsigned long pfn)  {}
0021 static inline void paravirt_alloc_pmd(struct mm_struct *mm, unsigned long pfn)  {}
0022 static inline void paravirt_alloc_pmd_clone(unsigned long pfn, unsigned long clonepfn,
0023                         unsigned long start, unsigned long count) {}
0024 static inline void paravirt_alloc_pud(struct mm_struct *mm, unsigned long pfn)  {}
0025 static inline void paravirt_alloc_p4d(struct mm_struct *mm, unsigned long pfn)  {}
0026 static inline void paravirt_release_pte(unsigned long pfn) {}
0027 static inline void paravirt_release_pmd(unsigned long pfn) {}
0028 static inline void paravirt_release_pud(unsigned long pfn) {}
0029 static inline void paravirt_release_p4d(unsigned long pfn) {}
0030 #endif
0031
0032 /*
0033  * Flags to use when allocating a user page table page.
0034  */
0035 extern gfp_t __userpte_alloc_gfp;
0036
0037 #ifdef CONFIG_PAGE_TABLE_ISOLATION
0038 /*
0039  * Instead of one PGD, we acquire two PGDs.  Being order-1, it is
0040  * both 8k in size and 8k-aligned.  That lets us just flip bit 12
0041  * in a pointer to swap between the two 4k halves.
0042  */
0043 #define PGD_ALLOCATION_ORDER 1
0044 #else
0045 #define PGD_ALLOCATION_ORDER 0
0046 #endif
0047
0048 /*
0049  * Allocate and free page tables.
0050  */
0051 extern pgd_t *pgd_alloc(struct mm_struct *);
0052 extern void pgd_free(struct mm_struct *mm, pgd_t *pgd);
0053
0054 extern pgtable_t pte_alloc_one(struct mm_struct *);
0055
0056 extern void ___pte_free_tlb(struct mmu_gather *tlb, struct page *pte);
0057
0058 static inline void __pte_free_tlb(struct mmu_gather *tlb, struct page *pte,
0059                   unsigned long address)
0060 {
0061     ___pte_free_tlb(tlb, pte);
0062 }
0063
0064 static inline void pmd_populate_kernel(struct mm_struct *mm,
0065                        pmd_t *pmd, pte_t *pte)
0066 {
0067     paravirt_alloc_pte(mm, __pa(pte) >> PAGE_SHIFT);
0068     set_pmd(pmd, __pmd(__pa(pte) | _PAGE_TABLE));
0069 }
0070
0071 static inline void pmd_populate_kernel_safe(struct mm_struct *mm,
0072                        pmd_t *pmd, pte_t *pte)
0073 {
0074     paravirt_alloc_pte(mm, __pa(pte) >> PAGE_SHIFT);
0075     set_pmd_safe(pmd, __pmd(__pa(pte) | _PAGE_TABLE));
0076 }
0077
0078 static inline void pmd_populate(struct mm_struct *mm, pmd_t *pmd,
0079                 struct page *pte)
0080 {
0081     unsigned long pfn = page_to_pfn(pte);
0082
0083     paravirt_alloc_pte(mm, pfn);
0084     set_pmd(pmd, __pmd(((pteval_t)pfn << PAGE_SHIFT) | _PAGE_TABLE));
0085 }
0086
0087 #if CONFIG_PGTABLE_LEVELS > 2
0088 extern void ___pmd_free_tlb(struct mmu_gather *tlb, pmd_t *pmd);
0089
0090 static inline void __pmd_free_tlb(struct mmu_gather *tlb, pmd_t *pmd,
0091                   unsigned long address)
0092 {
0093     ___pmd_free_tlb(tlb, pmd);
0094 }
0095
0096 #ifdef CONFIG_X86_PAE
0097 extern void pud_populate(struct mm_struct *mm, pud_t *pudp, pmd_t *pmd);
0098 #else   /* !CONFIG_X86_PAE */
0099 static inline void pud_populate(struct mm_struct *mm, pud_t *pud, pmd_t *pmd)
0100 {
0101     paravirt_alloc_pmd(mm, __pa(pmd) >> PAGE_SHIFT);
0102     set_pud(pud, __pud(_PAGE_TABLE | __pa(pmd)));
0103 }
0104
0105 static inline void pud_populate_safe(struct mm_struct *mm, pud_t *pud, pmd_t *pmd)
0106 {
0107     paravirt_alloc_pmd(mm, __pa(pmd) >> PAGE_SHIFT);
0108     set_pud_safe(pud, __pud(_PAGE_TABLE | __pa(pmd)));
0109 }
0110 #endif  /* CONFIG_X86_PAE */
0111
0112 #if CONFIG_PGTABLE_LEVELS > 3
0113 static inline void p4d_populate(struct mm_struct *mm, p4d_t *p4d, pud_t *pud)
0114 {
0115     paravirt_alloc_pud(mm, __pa(pud) >> PAGE_SHIFT);
0116     set_p4d(p4d, __p4d(_PAGE_TABLE | __pa(pud)));
0117 }
0118
0119 static inline void p4d_populate_safe(struct mm_struct *mm, p4d_t *p4d, pud_t *pud)
0120 {
0121     paravirt_alloc_pud(mm, __pa(pud) >> PAGE_SHIFT);
0122     set_p4d_safe(p4d, __p4d(_PAGE_TABLE | __pa(pud)));
0123 }
0124
0125 extern void ___pud_free_tlb(struct mmu_gather *tlb, pud_t *pud);
0126
0127 static inline void __pud_free_tlb(struct mmu_gather *tlb, pud_t *pud,
0128                   unsigned long address)
0129 {
0130     ___pud_free_tlb(tlb, pud);
0131 }
0132
0133 #if CONFIG_PGTABLE_LEVELS > 4
0134 static inline void pgd_populate(struct mm_struct *mm, pgd_t *pgd, p4d_t *p4d)
0135 {
0136     if (!pgtable_l5_enabled())
0137         return;
0138     paravirt_alloc_p4d(mm, __pa(p4d) >> PAGE_SHIFT);
0139     set_pgd(pgd, __pgd(_PAGE_TABLE | __pa(p4d)));
0140 }
0141
0142 static inline void pgd_populate_safe(struct mm_struct *mm, pgd_t *pgd, p4d_t *p4d)
0143 {
0144     if (!pgtable_l5_enabled())
0145         return;
0146     paravirt_alloc_p4d(mm, __pa(p4d) >> PAGE_SHIFT);
0147     set_pgd_safe(pgd, __pgd(_PAGE_TABLE | __pa(p4d)));
0148 }
0149
0150 static inline p4d_t *p4d_alloc_one(struct mm_struct *mm, unsigned long addr)
0151 {
0152     gfp_t gfp = GFP_KERNEL_ACCOUNT;
0153
0154     if (mm == &init_mm)
0155         gfp &= ~__GFP_ACCOUNT;
0156     return (p4d_t *)get_zeroed_page(gfp);
0157 }
0158
0159 static inline void p4d_free(struct mm_struct *mm, p4d_t *p4d)
0160 {
0161     if (!pgtable_l5_enabled())
0162         return;
0163
0164     BUG_ON((unsigned long)p4d & (PAGE_SIZE-1));
0165     free_page((unsigned long)p4d);
0166 }
0167
0168 extern void ___p4d_free_tlb(struct mmu_gather *tlb, p4d_t *p4d);
0169
0170 static inline void __p4d_free_tlb(struct mmu_gather *tlb, p4d_t *p4d,
0171                   unsigned long address)
0172 {
0173     if (pgtable_l5_enabled())
0174         ___p4d_free_tlb(tlb, p4d);
0175 }
0176
0177 #endif  /* CONFIG_PGTABLE_LEVELS > 4 */
0178 #endif  /* CONFIG_PGTABLE_LEVELS > 3 */
0179 #endif  /* CONFIG_PGTABLE_LEVELS > 2 */
0180
0181 #endif /* _ASM_X86_PGALLOC_H */