powerpc/mm/pgtable-frag.c

0001 // SPDX-License-Identifier: GPL-2.0
0002
0003 /*
0004  *  Handling Page Tables through page fragments
0005  *
0006  */
0007
0008 #include <linux/kernel.h>
0009 #include <linux/gfp.h>
0010 #include <linux/mm.h>
0011 #include <linux/percpu.h>
0012 #include <linux/hardirq.h>
0013 #include <linux/hugetlb.h>
0014 #include <asm/pgalloc.h>
0015 #include <asm/tlbflush.h>
0016 #include <asm/tlb.h>
0017
0018 void pte_frag_destroy(void *pte_frag)
0019 {
0020     int count;
0021     struct page *page;
0022
0023     page = virt_to_page(pte_frag);
0024     /* drop all the pending references */
0025     count = ((unsigned long)pte_frag & ~PAGE_MASK) >> PTE_FRAG_SIZE_SHIFT;
0026     /* We allow PTE_FRAG_NR fragments from a PTE page */
0027     if (atomic_sub_and_test(PTE_FRAG_NR - count, &page->pt_frag_refcount)) {
0028         pgtable_pte_page_dtor(page);
0029         __free_page(page);
0030     }
0031 }
0032
0033 static pte_t *get_pte_from_cache(struct mm_struct *mm)
0034 {
0035     void *pte_frag, *ret;
0036
0037     if (PTE_FRAG_NR == 1)
0038         return NULL;
0039
0040     spin_lock(&mm->page_table_lock);
0041     ret = pte_frag_get(&mm->context);
0042     if (ret) {
0043         pte_frag = ret + PTE_FRAG_SIZE;
0044         /*
0045          * If we have taken up all the fragments mark PTE page NULL
0046          */
0047         if (((unsigned long)pte_frag & ~PAGE_MASK) == 0)
0048             pte_frag = NULL;
0049         pte_frag_set(&mm->context, pte_frag);
0050     }
0051     spin_unlock(&mm->page_table_lock);
0052     return (pte_t *)ret;
0053 }
0054
0055 static pte_t *__alloc_for_ptecache(struct mm_struct *mm, int kernel)
0056 {
0057     void *ret = NULL;
0058     struct page *page;
0059
0060     if (!kernel) {
0061         page = alloc_page(PGALLOC_GFP | __GFP_ACCOUNT);
0062         if (!page)
0063             return NULL;
0064         if (!pgtable_pte_page_ctor(page)) {
0065             __free_page(page);
0066             return NULL;
0067         }
0068     } else {
0069         page = alloc_page(PGALLOC_GFP);
0070         if (!page)
0071             return NULL;
0072     }
0073
0074     atomic_set(&page->pt_frag_refcount, 1);
0075
0076     ret = page_address(page);
0077     /*
0078      * if we support only one fragment just return the
0079      * allocated page.
0080      */
0081     if (PTE_FRAG_NR == 1)
0082         return ret;
0083     spin_lock(&mm->page_table_lock);
0084     /*
0085      * If we find pgtable_page set, we return
0086      * the allocated page with single fragment
0087      * count.
0088      */
0089     if (likely(!pte_frag_get(&mm->context))) {
0090         atomic_set(&page->pt_frag_refcount, PTE_FRAG_NR);
0091         pte_frag_set(&mm->context, ret + PTE_FRAG_SIZE);
0092     }
0093     spin_unlock(&mm->page_table_lock);
0094
0095     return (pte_t *)ret;
0096 }
0097
0098 pte_t *pte_fragment_alloc(struct mm_struct *mm, int kernel)
0099 {
0100     pte_t *pte;
0101
0102     pte = get_pte_from_cache(mm);
0103     if (pte)
0104         return pte;
0105
0106     return __alloc_for_ptecache(mm, kernel);
0107 }
0108
0109 void pte_fragment_free(unsigned long *table, int kernel)
0110 {
0111     struct page *page = virt_to_page(table);
0112
0113     if (PageReserved(page))
0114         return free_reserved_page(page);
0115
0116     BUG_ON(atomic_read(&page->pt_frag_refcount) <= 0);
0117     if (atomic_dec_and_test(&page->pt_frag_refcount)) {
0118         if (!kernel)
0119             pgtable_pte_page_dtor(page);
0120         __free_page(page);
0121     }
0122 }