Back to home page

OSCL-LXR
 
 

     

0001 /* SPDX-License-Identifier: GPL-2.0 */
0002 #ifndef _LINUX_HUGE_MM_H
0003 #define _LINUX_HUGE_MM_H
0004 
0005 #include <linux/sched/coredump.h>
0006 #include <linux/mm_types.h>
0007 
0008 #include <linux/fs.h> /* only for vma_is_dax() */
0009 
0010 vm_fault_t do_huge_pmd_anonymous_page(struct vm_fault *vmf);
0011 int copy_huge_pmd(struct mm_struct *dst_mm, struct mm_struct *src_mm,
0012           pmd_t *dst_pmd, pmd_t *src_pmd, unsigned long addr,
0013           struct vm_area_struct *dst_vma, struct vm_area_struct *src_vma);
0014 void huge_pmd_set_accessed(struct vm_fault *vmf);
0015 int copy_huge_pud(struct mm_struct *dst_mm, struct mm_struct *src_mm,
0016           pud_t *dst_pud, pud_t *src_pud, unsigned long addr,
0017           struct vm_area_struct *vma);
0018 
0019 #ifdef CONFIG_HAVE_ARCH_TRANSPARENT_HUGEPAGE_PUD
0020 void huge_pud_set_accessed(struct vm_fault *vmf, pud_t orig_pud);
0021 #else
0022 static inline void huge_pud_set_accessed(struct vm_fault *vmf, pud_t orig_pud)
0023 {
0024 }
0025 #endif
0026 
0027 vm_fault_t do_huge_pmd_wp_page(struct vm_fault *vmf);
0028 struct page *follow_trans_huge_pmd(struct vm_area_struct *vma,
0029                    unsigned long addr, pmd_t *pmd,
0030                    unsigned int flags);
0031 bool madvise_free_huge_pmd(struct mmu_gather *tlb, struct vm_area_struct *vma,
0032                pmd_t *pmd, unsigned long addr, unsigned long next);
0033 int zap_huge_pmd(struct mmu_gather *tlb, struct vm_area_struct *vma, pmd_t *pmd,
0034          unsigned long addr);
0035 int zap_huge_pud(struct mmu_gather *tlb, struct vm_area_struct *vma, pud_t *pud,
0036          unsigned long addr);
0037 bool move_huge_pmd(struct vm_area_struct *vma, unsigned long old_addr,
0038            unsigned long new_addr, pmd_t *old_pmd, pmd_t *new_pmd);
0039 int change_huge_pmd(struct mmu_gather *tlb, struct vm_area_struct *vma,
0040             pmd_t *pmd, unsigned long addr, pgprot_t newprot,
0041             unsigned long cp_flags);
0042 vm_fault_t vmf_insert_pfn_pmd_prot(struct vm_fault *vmf, pfn_t pfn,
0043                    pgprot_t pgprot, bool write);
0044 
0045 /**
0046  * vmf_insert_pfn_pmd - insert a pmd size pfn
0047  * @vmf: Structure describing the fault
0048  * @pfn: pfn to insert
0049  * @pgprot: page protection to use
0050  * @write: whether it's a write fault
0051  *
0052  * Insert a pmd size pfn. See vmf_insert_pfn() for additional info.
0053  *
0054  * Return: vm_fault_t value.
0055  */
0056 static inline vm_fault_t vmf_insert_pfn_pmd(struct vm_fault *vmf, pfn_t pfn,
0057                         bool write)
0058 {
0059     return vmf_insert_pfn_pmd_prot(vmf, pfn, vmf->vma->vm_page_prot, write);
0060 }
0061 vm_fault_t vmf_insert_pfn_pud_prot(struct vm_fault *vmf, pfn_t pfn,
0062                    pgprot_t pgprot, bool write);
0063 
0064 /**
0065  * vmf_insert_pfn_pud - insert a pud size pfn
0066  * @vmf: Structure describing the fault
0067  * @pfn: pfn to insert
0068  * @pgprot: page protection to use
0069  * @write: whether it's a write fault
0070  *
0071  * Insert a pud size pfn. See vmf_insert_pfn() for additional info.
0072  *
0073  * Return: vm_fault_t value.
0074  */
0075 static inline vm_fault_t vmf_insert_pfn_pud(struct vm_fault *vmf, pfn_t pfn,
0076                         bool write)
0077 {
0078     return vmf_insert_pfn_pud_prot(vmf, pfn, vmf->vma->vm_page_prot, write);
0079 }
0080 
0081 enum transparent_hugepage_flag {
0082     TRANSPARENT_HUGEPAGE_NEVER_DAX,
0083     TRANSPARENT_HUGEPAGE_FLAG,
0084     TRANSPARENT_HUGEPAGE_REQ_MADV_FLAG,
0085     TRANSPARENT_HUGEPAGE_DEFRAG_DIRECT_FLAG,
0086     TRANSPARENT_HUGEPAGE_DEFRAG_KSWAPD_FLAG,
0087     TRANSPARENT_HUGEPAGE_DEFRAG_KSWAPD_OR_MADV_FLAG,
0088     TRANSPARENT_HUGEPAGE_DEFRAG_REQ_MADV_FLAG,
0089     TRANSPARENT_HUGEPAGE_DEFRAG_KHUGEPAGED_FLAG,
0090     TRANSPARENT_HUGEPAGE_USE_ZERO_PAGE_FLAG,
0091 };
0092 
0093 struct kobject;
0094 struct kobj_attribute;
0095 
0096 ssize_t single_hugepage_flag_store(struct kobject *kobj,
0097                    struct kobj_attribute *attr,
0098                    const char *buf, size_t count,
0099                    enum transparent_hugepage_flag flag);
0100 ssize_t single_hugepage_flag_show(struct kobject *kobj,
0101                   struct kobj_attribute *attr, char *buf,
0102                   enum transparent_hugepage_flag flag);
0103 extern struct kobj_attribute shmem_enabled_attr;
0104 
0105 #define HPAGE_PMD_ORDER (HPAGE_PMD_SHIFT-PAGE_SHIFT)
0106 #define HPAGE_PMD_NR (1<<HPAGE_PMD_ORDER)
0107 
0108 #ifdef CONFIG_TRANSPARENT_HUGEPAGE
0109 #define HPAGE_PMD_SHIFT PMD_SHIFT
0110 #define HPAGE_PMD_SIZE  ((1UL) << HPAGE_PMD_SHIFT)
0111 #define HPAGE_PMD_MASK  (~(HPAGE_PMD_SIZE - 1))
0112 
0113 #define HPAGE_PUD_SHIFT PUD_SHIFT
0114 #define HPAGE_PUD_SIZE  ((1UL) << HPAGE_PUD_SHIFT)
0115 #define HPAGE_PUD_MASK  (~(HPAGE_PUD_SIZE - 1))
0116 
0117 extern unsigned long transparent_hugepage_flags;
0118 
0119 #define hugepage_flags_enabled()                           \
0120     (transparent_hugepage_flags &                      \
0121      ((1<<TRANSPARENT_HUGEPAGE_FLAG) |             \
0122       (1<<TRANSPARENT_HUGEPAGE_REQ_MADV_FLAG)))
0123 #define hugepage_flags_always()             \
0124     (transparent_hugepage_flags &           \
0125      (1<<TRANSPARENT_HUGEPAGE_FLAG))
0126 
0127 /*
0128  * Do the below checks:
0129  *   - For file vma, check if the linear page offset of vma is
0130  *     HPAGE_PMD_NR aligned within the file.  The hugepage is
0131  *     guaranteed to be hugepage-aligned within the file, but we must
0132  *     check that the PMD-aligned addresses in the VMA map to
0133  *     PMD-aligned offsets within the file, else the hugepage will
0134  *     not be PMD-mappable.
0135  *   - For all vmas, check if the haddr is in an aligned HPAGE_PMD_SIZE
0136  *     area.
0137  */
0138 static inline bool transhuge_vma_suitable(struct vm_area_struct *vma,
0139         unsigned long addr)
0140 {
0141     unsigned long haddr;
0142 
0143     /* Don't have to check pgoff for anonymous vma */
0144     if (!vma_is_anonymous(vma)) {
0145         if (!IS_ALIGNED((vma->vm_start >> PAGE_SHIFT) - vma->vm_pgoff,
0146                 HPAGE_PMD_NR))
0147             return false;
0148     }
0149 
0150     haddr = addr & HPAGE_PMD_MASK;
0151 
0152     if (haddr < vma->vm_start || haddr + HPAGE_PMD_SIZE > vma->vm_end)
0153         return false;
0154     return true;
0155 }
0156 
0157 static inline bool file_thp_enabled(struct vm_area_struct *vma)
0158 {
0159     struct inode *inode;
0160 
0161     if (!vma->vm_file)
0162         return false;
0163 
0164     inode = vma->vm_file->f_inode;
0165 
0166     return (IS_ENABLED(CONFIG_READ_ONLY_THP_FOR_FS)) &&
0167            (vma->vm_flags & VM_EXEC) &&
0168            !inode_is_open_for_write(inode) && S_ISREG(inode->i_mode);
0169 }
0170 
0171 bool hugepage_vma_check(struct vm_area_struct *vma,
0172             unsigned long vm_flags,
0173             bool smaps, bool in_pf);
0174 
0175 #define transparent_hugepage_use_zero_page()                \
0176     (transparent_hugepage_flags &                   \
0177      (1<<TRANSPARENT_HUGEPAGE_USE_ZERO_PAGE_FLAG))
0178 
0179 unsigned long thp_get_unmapped_area(struct file *filp, unsigned long addr,
0180         unsigned long len, unsigned long pgoff, unsigned long flags);
0181 
0182 void prep_transhuge_page(struct page *page);
0183 void free_transhuge_page(struct page *page);
0184 
0185 bool can_split_folio(struct folio *folio, int *pextra_pins);
0186 int split_huge_page_to_list(struct page *page, struct list_head *list);
0187 static inline int split_huge_page(struct page *page)
0188 {
0189     return split_huge_page_to_list(page, NULL);
0190 }
0191 void deferred_split_huge_page(struct page *page);
0192 
0193 void __split_huge_pmd(struct vm_area_struct *vma, pmd_t *pmd,
0194         unsigned long address, bool freeze, struct folio *folio);
0195 
0196 #define split_huge_pmd(__vma, __pmd, __address)             \
0197     do {                                \
0198         pmd_t *____pmd = (__pmd);               \
0199         if (is_swap_pmd(*____pmd) || pmd_trans_huge(*____pmd)   \
0200                     || pmd_devmap(*____pmd))    \
0201             __split_huge_pmd(__vma, __pmd, __address,   \
0202                         false, NULL);       \
0203     }  while (0)
0204 
0205 
0206 void split_huge_pmd_address(struct vm_area_struct *vma, unsigned long address,
0207         bool freeze, struct folio *folio);
0208 
0209 void __split_huge_pud(struct vm_area_struct *vma, pud_t *pud,
0210         unsigned long address);
0211 
0212 #define split_huge_pud(__vma, __pud, __address)             \
0213     do {                                \
0214         pud_t *____pud = (__pud);               \
0215         if (pud_trans_huge(*____pud)                \
0216                     || pud_devmap(*____pud))    \
0217             __split_huge_pud(__vma, __pud, __address);  \
0218     }  while (0)
0219 
0220 int hugepage_madvise(struct vm_area_struct *vma, unsigned long *vm_flags,
0221              int advice);
0222 void vma_adjust_trans_huge(struct vm_area_struct *vma, unsigned long start,
0223                unsigned long end, long adjust_next);
0224 spinlock_t *__pmd_trans_huge_lock(pmd_t *pmd, struct vm_area_struct *vma);
0225 spinlock_t *__pud_trans_huge_lock(pud_t *pud, struct vm_area_struct *vma);
0226 
0227 static inline int is_swap_pmd(pmd_t pmd)
0228 {
0229     return !pmd_none(pmd) && !pmd_present(pmd);
0230 }
0231 
0232 /* mmap_lock must be held on entry */
0233 static inline spinlock_t *pmd_trans_huge_lock(pmd_t *pmd,
0234         struct vm_area_struct *vma)
0235 {
0236     if (is_swap_pmd(*pmd) || pmd_trans_huge(*pmd) || pmd_devmap(*pmd))
0237         return __pmd_trans_huge_lock(pmd, vma);
0238     else
0239         return NULL;
0240 }
0241 static inline spinlock_t *pud_trans_huge_lock(pud_t *pud,
0242         struct vm_area_struct *vma)
0243 {
0244     if (pud_trans_huge(*pud) || pud_devmap(*pud))
0245         return __pud_trans_huge_lock(pud, vma);
0246     else
0247         return NULL;
0248 }
0249 
0250 /**
0251  * folio_test_pmd_mappable - Can we map this folio with a PMD?
0252  * @folio: The folio to test
0253  */
0254 static inline bool folio_test_pmd_mappable(struct folio *folio)
0255 {
0256     return folio_order(folio) >= HPAGE_PMD_ORDER;
0257 }
0258 
0259 struct page *follow_devmap_pmd(struct vm_area_struct *vma, unsigned long addr,
0260         pmd_t *pmd, int flags, struct dev_pagemap **pgmap);
0261 struct page *follow_devmap_pud(struct vm_area_struct *vma, unsigned long addr,
0262         pud_t *pud, int flags, struct dev_pagemap **pgmap);
0263 
0264 vm_fault_t do_huge_pmd_numa_page(struct vm_fault *vmf);
0265 
0266 extern struct page *huge_zero_page;
0267 extern unsigned long huge_zero_pfn;
0268 
0269 static inline bool is_huge_zero_page(struct page *page)
0270 {
0271     return READ_ONCE(huge_zero_page) == page;
0272 }
0273 
0274 static inline bool is_huge_zero_pmd(pmd_t pmd)
0275 {
0276     return pmd_present(pmd) && READ_ONCE(huge_zero_pfn) == pmd_pfn(pmd);
0277 }
0278 
0279 static inline bool is_huge_zero_pud(pud_t pud)
0280 {
0281     return false;
0282 }
0283 
0284 struct page *mm_get_huge_zero_page(struct mm_struct *mm);
0285 void mm_put_huge_zero_page(struct mm_struct *mm);
0286 
0287 #define mk_huge_pmd(page, prot) pmd_mkhuge(mk_pmd(page, prot))
0288 
0289 static inline bool thp_migration_supported(void)
0290 {
0291     return IS_ENABLED(CONFIG_ARCH_ENABLE_THP_MIGRATION);
0292 }
0293 
0294 static inline struct list_head *page_deferred_list(struct page *page)
0295 {
0296     /*
0297      * See organization of tail pages of compound page in
0298      * "struct page" definition.
0299      */
0300     return &page[2].deferred_list;
0301 }
0302 
0303 #else /* CONFIG_TRANSPARENT_HUGEPAGE */
0304 #define HPAGE_PMD_SHIFT ({ BUILD_BUG(); 0; })
0305 #define HPAGE_PMD_MASK ({ BUILD_BUG(); 0; })
0306 #define HPAGE_PMD_SIZE ({ BUILD_BUG(); 0; })
0307 
0308 #define HPAGE_PUD_SHIFT ({ BUILD_BUG(); 0; })
0309 #define HPAGE_PUD_MASK ({ BUILD_BUG(); 0; })
0310 #define HPAGE_PUD_SIZE ({ BUILD_BUG(); 0; })
0311 
0312 static inline bool folio_test_pmd_mappable(struct folio *folio)
0313 {
0314     return false;
0315 }
0316 
0317 static inline bool transhuge_vma_suitable(struct vm_area_struct *vma,
0318         unsigned long addr)
0319 {
0320     return false;
0321 }
0322 
0323 static inline bool hugepage_vma_check(struct vm_area_struct *vma,
0324                        unsigned long vm_flags,
0325                        bool smaps, bool in_pf)
0326 {
0327     return false;
0328 }
0329 
0330 static inline void prep_transhuge_page(struct page *page) {}
0331 
0332 #define transparent_hugepage_flags 0UL
0333 
0334 #define thp_get_unmapped_area   NULL
0335 
0336 static inline bool
0337 can_split_folio(struct folio *folio, int *pextra_pins)
0338 {
0339     return false;
0340 }
0341 static inline int
0342 split_huge_page_to_list(struct page *page, struct list_head *list)
0343 {
0344     return 0;
0345 }
0346 static inline int split_huge_page(struct page *page)
0347 {
0348     return 0;
0349 }
0350 static inline void deferred_split_huge_page(struct page *page) {}
0351 #define split_huge_pmd(__vma, __pmd, __address) \
0352     do { } while (0)
0353 
0354 static inline void __split_huge_pmd(struct vm_area_struct *vma, pmd_t *pmd,
0355         unsigned long address, bool freeze, struct folio *folio) {}
0356 static inline void split_huge_pmd_address(struct vm_area_struct *vma,
0357         unsigned long address, bool freeze, struct folio *folio) {}
0358 
0359 #define split_huge_pud(__vma, __pmd, __address) \
0360     do { } while (0)
0361 
0362 static inline int hugepage_madvise(struct vm_area_struct *vma,
0363                    unsigned long *vm_flags, int advice)
0364 {
0365     BUG();
0366     return 0;
0367 }
0368 static inline void vma_adjust_trans_huge(struct vm_area_struct *vma,
0369                      unsigned long start,
0370                      unsigned long end,
0371                      long adjust_next)
0372 {
0373 }
0374 static inline int is_swap_pmd(pmd_t pmd)
0375 {
0376     return 0;
0377 }
0378 static inline spinlock_t *pmd_trans_huge_lock(pmd_t *pmd,
0379         struct vm_area_struct *vma)
0380 {
0381     return NULL;
0382 }
0383 static inline spinlock_t *pud_trans_huge_lock(pud_t *pud,
0384         struct vm_area_struct *vma)
0385 {
0386     return NULL;
0387 }
0388 
0389 static inline vm_fault_t do_huge_pmd_numa_page(struct vm_fault *vmf)
0390 {
0391     return 0;
0392 }
0393 
0394 static inline bool is_huge_zero_page(struct page *page)
0395 {
0396     return false;
0397 }
0398 
0399 static inline bool is_huge_zero_pmd(pmd_t pmd)
0400 {
0401     return false;
0402 }
0403 
0404 static inline bool is_huge_zero_pud(pud_t pud)
0405 {
0406     return false;
0407 }
0408 
0409 static inline void mm_put_huge_zero_page(struct mm_struct *mm)
0410 {
0411     return;
0412 }
0413 
0414 static inline struct page *follow_devmap_pmd(struct vm_area_struct *vma,
0415     unsigned long addr, pmd_t *pmd, int flags, struct dev_pagemap **pgmap)
0416 {
0417     return NULL;
0418 }
0419 
0420 static inline struct page *follow_devmap_pud(struct vm_area_struct *vma,
0421     unsigned long addr, pud_t *pud, int flags, struct dev_pagemap **pgmap)
0422 {
0423     return NULL;
0424 }
0425 
0426 static inline bool thp_migration_supported(void)
0427 {
0428     return false;
0429 }
0430 #endif /* CONFIG_TRANSPARENT_HUGEPAGE */
0431 
0432 static inline int split_folio_to_list(struct folio *folio,
0433         struct list_head *list)
0434 {
0435     return split_huge_page_to_list(&folio->page, list);
0436 }
0437 
0438 /*
0439  * archs that select ARCH_WANTS_THP_SWAP but don't support THP_SWP due to
0440  * limitations in the implementation like arm64 MTE can override this to
0441  * false
0442  */
0443 #ifndef arch_thp_swp_supported
0444 static inline bool arch_thp_swp_supported(void)
0445 {
0446     return true;
0447 }
0448 #endif
0449 
0450 #endif /* _LINUX_HUGE_MM_H */
[ Source navigation ] [ Diff markup ] [ Identifier search ] [ general search ]

This page was automatically generated by the 2.1.0 LXR engine.

The LXR team
Valid CSS 2.1! Valid HTML 4.01!