sparc/mm/tlb.c

0001 // SPDX-License-Identifier: GPL-2.0
0002 /* arch/sparc64/mm/tlb.c
0003  *
0004  * Copyright (C) 2004 David S. Miller <davem@redhat.com>
0005  */
0006
0007 #include <linux/kernel.h>
0008 #include <linux/percpu.h>
0009 #include <linux/mm.h>
0010 #include <linux/swap.h>
0011 #include <linux/preempt.h>
0012 #include <linux/pagemap.h>
0013
0014 #include <asm/tlbflush.h>
0015 #include <asm/cacheflush.h>
0016 #include <asm/mmu_context.h>
0017 #include <asm/tlb.h>
0018
0019 /* Heavily inspired by the ppc64 code.  */
0020
0021 static DEFINE_PER_CPU(struct tlb_batch, tlb_batch);
0022
0023 void flush_tlb_pending(void)
0024 {
0025     struct tlb_batch *tb = &get_cpu_var(tlb_batch);
0026     struct mm_struct *mm = tb->mm;
0027
0028     if (!tb->tlb_nr)
0029         goto out;
0030
0031     flush_tsb_user(tb);
0032
0033     if (CTX_VALID(mm->context)) {
0034         if (tb->tlb_nr == 1) {
0035             global_flush_tlb_page(mm, tb->vaddrs[0]);
0036         } else {
0037 #ifdef CONFIG_SMP
0038             smp_flush_tlb_pending(tb->mm, tb->tlb_nr,
0039                           &tb->vaddrs[0]);
0040 #else
0041             __flush_tlb_pending(CTX_HWBITS(tb->mm->context),
0042                         tb->tlb_nr, &tb->vaddrs[0]);
0043 #endif
0044         }
0045     }
0046
0047     tb->tlb_nr = 0;
0048
0049 out:
0050     put_cpu_var(tlb_batch);
0051 }
0052
0053 void arch_enter_lazy_mmu_mode(void)
0054 {
0055     struct tlb_batch *tb = this_cpu_ptr(&tlb_batch);
0056
0057     tb->active = 1;
0058 }
0059
0060 void arch_leave_lazy_mmu_mode(void)
0061 {
0062     struct tlb_batch *tb = this_cpu_ptr(&tlb_batch);
0063
0064     if (tb->tlb_nr)
0065         flush_tlb_pending();
0066     tb->active = 0;
0067 }
0068
0069 static void tlb_batch_add_one(struct mm_struct *mm, unsigned long vaddr,
0070                   bool exec, unsigned int hugepage_shift)
0071 {
0072     struct tlb_batch *tb = &get_cpu_var(tlb_batch);
0073     unsigned long nr;
0074
0075     vaddr &= PAGE_MASK;
0076     if (exec)
0077         vaddr |= 0x1UL;
0078
0079     nr = tb->tlb_nr;
0080
0081     if (unlikely(nr != 0 && mm != tb->mm)) {
0082         flush_tlb_pending();
0083         nr = 0;
0084     }
0085
0086     if (!tb->active) {
0087         flush_tsb_user_page(mm, vaddr, hugepage_shift);
0088         global_flush_tlb_page(mm, vaddr);
0089         goto out;
0090     }
0091
0092     if (nr == 0) {
0093         tb->mm = mm;
0094         tb->hugepage_shift = hugepage_shift;
0095     }
0096
0097     if (tb->hugepage_shift != hugepage_shift) {
0098         flush_tlb_pending();
0099         tb->hugepage_shift = hugepage_shift;
0100         nr = 0;
0101     }
0102
0103     tb->vaddrs[nr] = vaddr;
0104     tb->tlb_nr = ++nr;
0105     if (nr >= TLB_BATCH_NR)
0106         flush_tlb_pending();
0107
0108 out:
0109     put_cpu_var(tlb_batch);
0110 }
0111
0112 void tlb_batch_add(struct mm_struct *mm, unsigned long vaddr,
0113            pte_t *ptep, pte_t orig, int fullmm,
0114            unsigned int hugepage_shift)
0115 {
0116     if (tlb_type != hypervisor &&
0117         pte_dirty(orig)) {
0118         unsigned long paddr, pfn = pte_pfn(orig);
0119         struct address_space *mapping;
0120         struct page *page;
0121
0122         if (!pfn_valid(pfn))
0123             goto no_cache_flush;
0124
0125         page = pfn_to_page(pfn);
0126         if (PageReserved(page))
0127             goto no_cache_flush;
0128
0129         /* A real file page? */
0130         mapping = page_mapping_file(page);
0131         if (!mapping)
0132             goto no_cache_flush;
0133
0134         paddr = (unsigned long) page_address(page);
0135         if ((paddr ^ vaddr) & (1 << 13))
0136             flush_dcache_page_all(mm, page);
0137     }
0138
0139 no_cache_flush:
0140     if (!fullmm)
0141         tlb_batch_add_one(mm, vaddr, pte_exec(orig), hugepage_shift);
0142 }
0143
0144 #ifdef CONFIG_TRANSPARENT_HUGEPAGE
0145 static void tlb_batch_pmd_scan(struct mm_struct *mm, unsigned long vaddr,
0146                    pmd_t pmd)
0147 {
0148     unsigned long end;
0149     pte_t *pte;
0150
0151     pte = pte_offset_map(&pmd, vaddr);
0152     end = vaddr + HPAGE_SIZE;
0153     while (vaddr < end) {
0154         if (pte_val(*pte) & _PAGE_VALID) {
0155             bool exec = pte_exec(*pte);
0156
0157             tlb_batch_add_one(mm, vaddr, exec, PAGE_SHIFT);
0158         }
0159         pte++;
0160         vaddr += PAGE_SIZE;
0161     }
0162     pte_unmap(pte);
0163 }
0164
0165
0166 static void __set_pmd_acct(struct mm_struct *mm, unsigned long addr,
0167                pmd_t orig, pmd_t pmd)
0168 {
0169     if (mm == &init_mm)
0170         return;
0171
0172     if ((pmd_val(pmd) ^ pmd_val(orig)) & _PAGE_PMD_HUGE) {
0173         /*
0174          * Note that this routine only sets pmds for THP pages.
0175          * Hugetlb pages are handled elsewhere.  We need to check
0176          * for huge zero page.  Huge zero pages are like hugetlb
0177          * pages in that there is no RSS, but there is the need
0178          * for TSB entries.  So, huge zero page counts go into
0179          * hugetlb_pte_count.
0180          */
0181         if (pmd_val(pmd) & _PAGE_PMD_HUGE) {
0182             if (is_huge_zero_page(pmd_page(pmd)))
0183                 mm->context.hugetlb_pte_count++;
0184             else
0185                 mm->context.thp_pte_count++;
0186         } else {
0187             if (is_huge_zero_page(pmd_page(orig)))
0188                 mm->context.hugetlb_pte_count--;
0189             else
0190                 mm->context.thp_pte_count--;
0191         }
0192
0193         /* Do not try to allocate the TSB hash table if we
0194          * don't have one already.  We have various locks held
0195          * and thus we'll end up doing a GFP_KERNEL allocation
0196          * in an atomic context.
0197          *
0198          * Instead, we let the first TLB miss on a hugepage
0199          * take care of this.
0200          */
0201     }
0202
0203     if (!pmd_none(orig)) {
0204         addr &= HPAGE_MASK;
0205         if (pmd_trans_huge(orig)) {
0206             pte_t orig_pte = __pte(pmd_val(orig));
0207             bool exec = pte_exec(orig_pte);
0208
0209             tlb_batch_add_one(mm, addr, exec, REAL_HPAGE_SHIFT);
0210             tlb_batch_add_one(mm, addr + REAL_HPAGE_SIZE, exec,
0211                       REAL_HPAGE_SHIFT);
0212         } else {
0213             tlb_batch_pmd_scan(mm, addr, orig);
0214         }
0215     }
0216 }
0217
0218 void set_pmd_at(struct mm_struct *mm, unsigned long addr,
0219         pmd_t *pmdp, pmd_t pmd)
0220 {
0221     pmd_t orig = *pmdp;
0222
0223     *pmdp = pmd;
0224     __set_pmd_acct(mm, addr, orig, pmd);
0225 }
0226
0227 static inline pmd_t pmdp_establish(struct vm_area_struct *vma,
0228         unsigned long address, pmd_t *pmdp, pmd_t pmd)
0229 {
0230     pmd_t old;
0231
0232     do {
0233         old = *pmdp;
0234     } while (cmpxchg64(&pmdp->pmd, old.pmd, pmd.pmd) != old.pmd);
0235     __set_pmd_acct(vma->vm_mm, address, old, pmd);
0236
0237     return old;
0238 }
0239
0240 /*
0241  * This routine is only called when splitting a THP
0242  */
0243 pmd_t pmdp_invalidate(struct vm_area_struct *vma, unsigned long address,
0244              pmd_t *pmdp)
0245 {
0246     pmd_t old, entry;
0247
0248     entry = __pmd(pmd_val(*pmdp) & ~_PAGE_VALID);
0249     old = pmdp_establish(vma, address, pmdp, entry);
0250     flush_tlb_range(vma, address, address + HPAGE_PMD_SIZE);
0251
0252     /*
0253      * set_pmd_at() will not be called in a way to decrement
0254      * thp_pte_count when splitting a THP, so do it now.
0255      * Sanity check pmd before doing the actual decrement.
0256      */
0257     if ((pmd_val(entry) & _PAGE_PMD_HUGE) &&
0258         !is_huge_zero_page(pmd_page(entry)))
0259         (vma->vm_mm)->context.thp_pte_count--;
0260
0261     return old;
0262 }
0263
0264 void pgtable_trans_huge_deposit(struct mm_struct *mm, pmd_t *pmdp,
0265                 pgtable_t pgtable)
0266 {
0267     struct list_head *lh = (struct list_head *) pgtable;
0268
0269     assert_spin_locked(&mm->page_table_lock);
0270
0271     /* FIFO */
0272     if (!pmd_huge_pte(mm, pmdp))
0273         INIT_LIST_HEAD(lh);
0274     else
0275         list_add(lh, (struct list_head *) pmd_huge_pte(mm, pmdp));
0276     pmd_huge_pte(mm, pmdp) = pgtable;
0277 }
0278
0279 pgtable_t pgtable_trans_huge_withdraw(struct mm_struct *mm, pmd_t *pmdp)
0280 {
0281     struct list_head *lh;
0282     pgtable_t pgtable;
0283
0284     assert_spin_locked(&mm->page_table_lock);
0285
0286     /* FIFO */
0287     pgtable = pmd_huge_pte(mm, pmdp);
0288     lh = (struct list_head *) pgtable;
0289     if (list_empty(lh))
0290         pmd_huge_pte(mm, pmdp) = NULL;
0291     else {
0292         pmd_huge_pte(mm, pmdp) = (pgtable_t) lh->next;
0293         list_del(lh);
0294     }
0295     pte_val(pgtable[0]) = 0;
0296     pte_val(pgtable[1]) = 0;
0297
0298     return pgtable;
0299 }
0300 #endif /* CONFIG_TRANSPARENT_HUGEPAGE */