include/asm/tlb.h

0001 /* SPDX-License-Identifier: GPL-2.0-only */
0002 /*
0003  * Based on arch/arm/include/asm/tlb.h
0004  *
0005  * Copyright (C) 2002 Russell King
0006  * Copyright (C) 2012 ARM Ltd.
0007  */
0008 #ifndef __ASM_TLB_H
0009 #define __ASM_TLB_H
0010
0011 #include <linux/pagemap.h>
0012 #include <linux/swap.h>
0013
0014 static inline void __tlb_remove_table(void *_table)
0015 {
0016     free_page_and_swap_cache((struct page *)_table);
0017 }
0018
0019 #define tlb_flush tlb_flush
0020 static void tlb_flush(struct mmu_gather *tlb);
0021
0022 #include <asm-generic/tlb.h>
0023
0024 /*
0025  * get the tlbi levels in arm64.  Default value is 0 if more than one
0026  * of cleared_* is set or neither is set.
0027  * Arm64 doesn't support p4ds now.
0028  */
0029 static inline int tlb_get_level(struct mmu_gather *tlb)
0030 {
0031     /* The TTL field is only valid for the leaf entry. */
0032     if (tlb->freed_tables)
0033         return 0;
0034
0035     if (tlb->cleared_ptes && !(tlb->cleared_pmds ||
0036                    tlb->cleared_puds ||
0037                    tlb->cleared_p4ds))
0038         return 3;
0039
0040     if (tlb->cleared_pmds && !(tlb->cleared_ptes ||
0041                    tlb->cleared_puds ||
0042                    tlb->cleared_p4ds))
0043         return 2;
0044
0045     if (tlb->cleared_puds && !(tlb->cleared_ptes ||
0046                    tlb->cleared_pmds ||
0047                    tlb->cleared_p4ds))
0048         return 1;
0049
0050     return 0;
0051 }
0052
0053 static inline void tlb_flush(struct mmu_gather *tlb)
0054 {
0055     struct vm_area_struct vma = TLB_FLUSH_VMA(tlb->mm, 0);
0056     bool last_level = !tlb->freed_tables;
0057     unsigned long stride = tlb_get_unmap_size(tlb);
0058     int tlb_level = tlb_get_level(tlb);
0059
0060     /*
0061      * If we're tearing down the address space then we only care about
0062      * invalidating the walk-cache, since the ASID allocator won't
0063      * reallocate our ASID without invalidating the entire TLB.
0064      */
0065     if (tlb->fullmm) {
0066         if (!last_level)
0067             flush_tlb_mm(tlb->mm);
0068         return;
0069     }
0070
0071     __flush_tlb_range(&vma, tlb->start, tlb->end, stride,
0072               last_level, tlb_level);
0073 }
0074
0075 static inline void __pte_free_tlb(struct mmu_gather *tlb, pgtable_t pte,
0076                   unsigned long addr)
0077 {
0078     pgtable_pte_page_dtor(pte);
0079     tlb_remove_table(tlb, pte);
0080 }
0081
0082 #if CONFIG_PGTABLE_LEVELS > 2
0083 static inline void __pmd_free_tlb(struct mmu_gather *tlb, pmd_t *pmdp,
0084                   unsigned long addr)
0085 {
0086     struct page *page = virt_to_page(pmdp);
0087
0088     pgtable_pmd_page_dtor(page);
0089     tlb_remove_table(tlb, page);
0090 }
0091 #endif
0092
0093 #if CONFIG_PGTABLE_LEVELS > 3
0094 static inline void __pud_free_tlb(struct mmu_gather *tlb, pud_t *pudp,
0095                   unsigned long addr)
0096 {
0097     tlb_remove_table(tlb, virt_to_page(pudp));
0098 }
0099 #endif
0100
0101 #endif