Back to home page

OSCL-LXR
 
 

     

0001 /* SPDX-License-Identifier: GPL-2.0 */
0002 #ifndef _LINUX_SWAPOPS_H
0003 #define _LINUX_SWAPOPS_H
0004 
0005 #include <linux/radix-tree.h>
0006 #include <linux/bug.h>
0007 #include <linux/mm_types.h>
0008 
0009 #ifdef CONFIG_MMU
0010 
0011 /*
0012  * swapcache pages are stored in the swapper_space radix tree.  We want to
0013  * get good packing density in that tree, so the index should be dense in
0014  * the low-order bits.
0015  *
0016  * We arrange the `type' and `offset' fields so that `type' is at the six
0017  * high-order bits of the swp_entry_t and `offset' is right-aligned in the
0018  * remaining bits.  Although `type' itself needs only five bits, we allow for
0019  * shmem/tmpfs to shift it all up a further one bit: see swp_to_radix_entry().
0020  *
0021  * swp_entry_t's are *never* stored anywhere in their arch-dependent format.
0022  */
0023 #define SWP_TYPE_SHIFT  (BITS_PER_XA_VALUE - MAX_SWAPFILES_SHIFT)
0024 #define SWP_OFFSET_MASK ((1UL << SWP_TYPE_SHIFT) - 1)
0025 
0026 /* Clear all flags but only keep swp_entry_t related information */
0027 static inline pte_t pte_swp_clear_flags(pte_t pte)
0028 {
0029     if (pte_swp_exclusive(pte))
0030         pte = pte_swp_clear_exclusive(pte);
0031     if (pte_swp_soft_dirty(pte))
0032         pte = pte_swp_clear_soft_dirty(pte);
0033     if (pte_swp_uffd_wp(pte))
0034         pte = pte_swp_clear_uffd_wp(pte);
0035     return pte;
0036 }
0037 
0038 /*
0039  * Store a type+offset into a swp_entry_t in an arch-independent format
0040  */
0041 static inline swp_entry_t swp_entry(unsigned long type, pgoff_t offset)
0042 {
0043     swp_entry_t ret;
0044 
0045     ret.val = (type << SWP_TYPE_SHIFT) | (offset & SWP_OFFSET_MASK);
0046     return ret;
0047 }
0048 
0049 /*
0050  * Extract the `type' field from a swp_entry_t.  The swp_entry_t is in
0051  * arch-independent format
0052  */
0053 static inline unsigned swp_type(swp_entry_t entry)
0054 {
0055     return (entry.val >> SWP_TYPE_SHIFT);
0056 }
0057 
0058 /*
0059  * Extract the `offset' field from a swp_entry_t.  The swp_entry_t is in
0060  * arch-independent format
0061  */
0062 static inline pgoff_t swp_offset(swp_entry_t entry)
0063 {
0064     return entry.val & SWP_OFFSET_MASK;
0065 }
0066 
0067 /* check whether a pte points to a swap entry */
0068 static inline int is_swap_pte(pte_t pte)
0069 {
0070     return !pte_none(pte) && !pte_present(pte);
0071 }
0072 
0073 /*
0074  * Convert the arch-dependent pte representation of a swp_entry_t into an
0075  * arch-independent swp_entry_t.
0076  */
0077 static inline swp_entry_t pte_to_swp_entry(pte_t pte)
0078 {
0079     swp_entry_t arch_entry;
0080 
0081     pte = pte_swp_clear_flags(pte);
0082     arch_entry = __pte_to_swp_entry(pte);
0083     return swp_entry(__swp_type(arch_entry), __swp_offset(arch_entry));
0084 }
0085 
0086 /*
0087  * Convert the arch-independent representation of a swp_entry_t into the
0088  * arch-dependent pte representation.
0089  */
0090 static inline pte_t swp_entry_to_pte(swp_entry_t entry)
0091 {
0092     swp_entry_t arch_entry;
0093 
0094     arch_entry = __swp_entry(swp_type(entry), swp_offset(entry));
0095     return __swp_entry_to_pte(arch_entry);
0096 }
0097 
0098 static inline swp_entry_t radix_to_swp_entry(void *arg)
0099 {
0100     swp_entry_t entry;
0101 
0102     entry.val = xa_to_value(arg);
0103     return entry;
0104 }
0105 
0106 static inline void *swp_to_radix_entry(swp_entry_t entry)
0107 {
0108     return xa_mk_value(entry.val);
0109 }
0110 
0111 static inline swp_entry_t make_swapin_error_entry(struct page *page)
0112 {
0113     return swp_entry(SWP_SWAPIN_ERROR, page_to_pfn(page));
0114 }
0115 
0116 static inline int is_swapin_error_entry(swp_entry_t entry)
0117 {
0118     return swp_type(entry) == SWP_SWAPIN_ERROR;
0119 }
0120 
0121 #if IS_ENABLED(CONFIG_DEVICE_PRIVATE)
0122 static inline swp_entry_t make_readable_device_private_entry(pgoff_t offset)
0123 {
0124     return swp_entry(SWP_DEVICE_READ, offset);
0125 }
0126 
0127 static inline swp_entry_t make_writable_device_private_entry(pgoff_t offset)
0128 {
0129     return swp_entry(SWP_DEVICE_WRITE, offset);
0130 }
0131 
0132 static inline bool is_device_private_entry(swp_entry_t entry)
0133 {
0134     int type = swp_type(entry);
0135     return type == SWP_DEVICE_READ || type == SWP_DEVICE_WRITE;
0136 }
0137 
0138 static inline bool is_writable_device_private_entry(swp_entry_t entry)
0139 {
0140     return unlikely(swp_type(entry) == SWP_DEVICE_WRITE);
0141 }
0142 
0143 static inline swp_entry_t make_readable_device_exclusive_entry(pgoff_t offset)
0144 {
0145     return swp_entry(SWP_DEVICE_EXCLUSIVE_READ, offset);
0146 }
0147 
0148 static inline swp_entry_t make_writable_device_exclusive_entry(pgoff_t offset)
0149 {
0150     return swp_entry(SWP_DEVICE_EXCLUSIVE_WRITE, offset);
0151 }
0152 
0153 static inline bool is_device_exclusive_entry(swp_entry_t entry)
0154 {
0155     return swp_type(entry) == SWP_DEVICE_EXCLUSIVE_READ ||
0156         swp_type(entry) == SWP_DEVICE_EXCLUSIVE_WRITE;
0157 }
0158 
0159 static inline bool is_writable_device_exclusive_entry(swp_entry_t entry)
0160 {
0161     return unlikely(swp_type(entry) == SWP_DEVICE_EXCLUSIVE_WRITE);
0162 }
0163 #else /* CONFIG_DEVICE_PRIVATE */
0164 static inline swp_entry_t make_readable_device_private_entry(pgoff_t offset)
0165 {
0166     return swp_entry(0, 0);
0167 }
0168 
0169 static inline swp_entry_t make_writable_device_private_entry(pgoff_t offset)
0170 {
0171     return swp_entry(0, 0);
0172 }
0173 
0174 static inline bool is_device_private_entry(swp_entry_t entry)
0175 {
0176     return false;
0177 }
0178 
0179 static inline bool is_writable_device_private_entry(swp_entry_t entry)
0180 {
0181     return false;
0182 }
0183 
0184 static inline swp_entry_t make_readable_device_exclusive_entry(pgoff_t offset)
0185 {
0186     return swp_entry(0, 0);
0187 }
0188 
0189 static inline swp_entry_t make_writable_device_exclusive_entry(pgoff_t offset)
0190 {
0191     return swp_entry(0, 0);
0192 }
0193 
0194 static inline bool is_device_exclusive_entry(swp_entry_t entry)
0195 {
0196     return false;
0197 }
0198 
0199 static inline bool is_writable_device_exclusive_entry(swp_entry_t entry)
0200 {
0201     return false;
0202 }
0203 #endif /* CONFIG_DEVICE_PRIVATE */
0204 
0205 #ifdef CONFIG_MIGRATION
0206 static inline int is_migration_entry(swp_entry_t entry)
0207 {
0208     return unlikely(swp_type(entry) == SWP_MIGRATION_READ ||
0209             swp_type(entry) == SWP_MIGRATION_READ_EXCLUSIVE ||
0210             swp_type(entry) == SWP_MIGRATION_WRITE);
0211 }
0212 
0213 static inline int is_writable_migration_entry(swp_entry_t entry)
0214 {
0215     return unlikely(swp_type(entry) == SWP_MIGRATION_WRITE);
0216 }
0217 
0218 static inline int is_readable_migration_entry(swp_entry_t entry)
0219 {
0220     return unlikely(swp_type(entry) == SWP_MIGRATION_READ);
0221 }
0222 
0223 static inline int is_readable_exclusive_migration_entry(swp_entry_t entry)
0224 {
0225     return unlikely(swp_type(entry) == SWP_MIGRATION_READ_EXCLUSIVE);
0226 }
0227 
0228 static inline swp_entry_t make_readable_migration_entry(pgoff_t offset)
0229 {
0230     return swp_entry(SWP_MIGRATION_READ, offset);
0231 }
0232 
0233 static inline swp_entry_t make_readable_exclusive_migration_entry(pgoff_t offset)
0234 {
0235     return swp_entry(SWP_MIGRATION_READ_EXCLUSIVE, offset);
0236 }
0237 
0238 static inline swp_entry_t make_writable_migration_entry(pgoff_t offset)
0239 {
0240     return swp_entry(SWP_MIGRATION_WRITE, offset);
0241 }
0242 
0243 extern void __migration_entry_wait(struct mm_struct *mm, pte_t *ptep,
0244                     spinlock_t *ptl);
0245 extern void migration_entry_wait(struct mm_struct *mm, pmd_t *pmd,
0246                     unsigned long address);
0247 #ifdef CONFIG_HUGETLB_PAGE
0248 extern void __migration_entry_wait_huge(pte_t *ptep, spinlock_t *ptl);
0249 extern void migration_entry_wait_huge(struct vm_area_struct *vma, pte_t *pte);
0250 #endif
0251 #else
0252 static inline swp_entry_t make_readable_migration_entry(pgoff_t offset)
0253 {
0254     return swp_entry(0, 0);
0255 }
0256 
0257 static inline swp_entry_t make_readable_exclusive_migration_entry(pgoff_t offset)
0258 {
0259     return swp_entry(0, 0);
0260 }
0261 
0262 static inline swp_entry_t make_writable_migration_entry(pgoff_t offset)
0263 {
0264     return swp_entry(0, 0);
0265 }
0266 
0267 static inline int is_migration_entry(swp_entry_t swp)
0268 {
0269     return 0;
0270 }
0271 
0272 static inline void __migration_entry_wait(struct mm_struct *mm, pte_t *ptep,
0273                     spinlock_t *ptl) { }
0274 static inline void migration_entry_wait(struct mm_struct *mm, pmd_t *pmd,
0275                      unsigned long address) { }
0276 #ifdef CONFIG_HUGETLB_PAGE
0277 static inline void __migration_entry_wait_huge(pte_t *ptep, spinlock_t *ptl) { }
0278 static inline void migration_entry_wait_huge(struct vm_area_struct *vma, pte_t *pte) { }
0279 #endif
0280 static inline int is_writable_migration_entry(swp_entry_t entry)
0281 {
0282     return 0;
0283 }
0284 static inline int is_readable_migration_entry(swp_entry_t entry)
0285 {
0286     return 0;
0287 }
0288 
0289 #endif
0290 
0291 typedef unsigned long pte_marker;
0292 
0293 #define  PTE_MARKER_UFFD_WP  BIT(0)
0294 #define  PTE_MARKER_MASK     (PTE_MARKER_UFFD_WP)
0295 
0296 #ifdef CONFIG_PTE_MARKER
0297 
0298 static inline swp_entry_t make_pte_marker_entry(pte_marker marker)
0299 {
0300     return swp_entry(SWP_PTE_MARKER, marker);
0301 }
0302 
0303 static inline bool is_pte_marker_entry(swp_entry_t entry)
0304 {
0305     return swp_type(entry) == SWP_PTE_MARKER;
0306 }
0307 
0308 static inline pte_marker pte_marker_get(swp_entry_t entry)
0309 {
0310     return swp_offset(entry) & PTE_MARKER_MASK;
0311 }
0312 
0313 static inline bool is_pte_marker(pte_t pte)
0314 {
0315     return is_swap_pte(pte) && is_pte_marker_entry(pte_to_swp_entry(pte));
0316 }
0317 
0318 #else /* CONFIG_PTE_MARKER */
0319 
0320 static inline swp_entry_t make_pte_marker_entry(pte_marker marker)
0321 {
0322     /* This should never be called if !CONFIG_PTE_MARKER */
0323     WARN_ON_ONCE(1);
0324     return swp_entry(0, 0);
0325 }
0326 
0327 static inline bool is_pte_marker_entry(swp_entry_t entry)
0328 {
0329     return false;
0330 }
0331 
0332 static inline pte_marker pte_marker_get(swp_entry_t entry)
0333 {
0334     return 0;
0335 }
0336 
0337 static inline bool is_pte_marker(pte_t pte)
0338 {
0339     return false;
0340 }
0341 
0342 #endif /* CONFIG_PTE_MARKER */
0343 
0344 static inline pte_t make_pte_marker(pte_marker marker)
0345 {
0346     return swp_entry_to_pte(make_pte_marker_entry(marker));
0347 }
0348 
0349 /*
0350  * This is a special version to check pte_none() just to cover the case when
0351  * the pte is a pte marker.  It existed because in many cases the pte marker
0352  * should be seen as a none pte; it's just that we have stored some information
0353  * onto the none pte so it becomes not-none any more.
0354  *
0355  * It should be used when the pte is file-backed, ram-based and backing
0356  * userspace pages, like shmem.  It is not needed upon pgtables that do not
0357  * support pte markers at all.  For example, it's not needed on anonymous
0358  * memory, kernel-only memory (including when the system is during-boot),
0359  * non-ram based generic file-system.  It's fine to be used even there, but the
0360  * extra pte marker check will be pure overhead.
0361  *
0362  * For systems configured with !CONFIG_PTE_MARKER this will be automatically
0363  * optimized to pte_none().
0364  */
0365 static inline int pte_none_mostly(pte_t pte)
0366 {
0367     return pte_none(pte) || is_pte_marker(pte);
0368 }
0369 
0370 static inline struct page *pfn_swap_entry_to_page(swp_entry_t entry)
0371 {
0372     struct page *p = pfn_to_page(swp_offset(entry));
0373 
0374     /*
0375      * Any use of migration entries may only occur while the
0376      * corresponding page is locked
0377      */
0378     BUG_ON(is_migration_entry(entry) && !PageLocked(p));
0379 
0380     return p;
0381 }
0382 
0383 /*
0384  * A pfn swap entry is a special type of swap entry that always has a pfn stored
0385  * in the swap offset. They are used to represent unaddressable device memory
0386  * and to restrict access to a page undergoing migration.
0387  */
0388 static inline bool is_pfn_swap_entry(swp_entry_t entry)
0389 {
0390     return is_migration_entry(entry) || is_device_private_entry(entry) ||
0391            is_device_exclusive_entry(entry);
0392 }
0393 
0394 struct page_vma_mapped_walk;
0395 
0396 #ifdef CONFIG_ARCH_ENABLE_THP_MIGRATION
0397 extern int set_pmd_migration_entry(struct page_vma_mapped_walk *pvmw,
0398         struct page *page);
0399 
0400 extern void remove_migration_pmd(struct page_vma_mapped_walk *pvmw,
0401         struct page *new);
0402 
0403 extern void pmd_migration_entry_wait(struct mm_struct *mm, pmd_t *pmd);
0404 
0405 static inline swp_entry_t pmd_to_swp_entry(pmd_t pmd)
0406 {
0407     swp_entry_t arch_entry;
0408 
0409     if (pmd_swp_soft_dirty(pmd))
0410         pmd = pmd_swp_clear_soft_dirty(pmd);
0411     if (pmd_swp_uffd_wp(pmd))
0412         pmd = pmd_swp_clear_uffd_wp(pmd);
0413     arch_entry = __pmd_to_swp_entry(pmd);
0414     return swp_entry(__swp_type(arch_entry), __swp_offset(arch_entry));
0415 }
0416 
0417 static inline pmd_t swp_entry_to_pmd(swp_entry_t entry)
0418 {
0419     swp_entry_t arch_entry;
0420 
0421     arch_entry = __swp_entry(swp_type(entry), swp_offset(entry));
0422     return __swp_entry_to_pmd(arch_entry);
0423 }
0424 
0425 static inline int is_pmd_migration_entry(pmd_t pmd)
0426 {
0427     return is_swap_pmd(pmd) && is_migration_entry(pmd_to_swp_entry(pmd));
0428 }
0429 #else
0430 static inline int set_pmd_migration_entry(struct page_vma_mapped_walk *pvmw,
0431         struct page *page)
0432 {
0433     BUILD_BUG();
0434 }
0435 
0436 static inline void remove_migration_pmd(struct page_vma_mapped_walk *pvmw,
0437         struct page *new)
0438 {
0439     BUILD_BUG();
0440 }
0441 
0442 static inline void pmd_migration_entry_wait(struct mm_struct *m, pmd_t *p) { }
0443 
0444 static inline swp_entry_t pmd_to_swp_entry(pmd_t pmd)
0445 {
0446     return swp_entry(0, 0);
0447 }
0448 
0449 static inline pmd_t swp_entry_to_pmd(swp_entry_t entry)
0450 {
0451     return __pmd(0);
0452 }
0453 
0454 static inline int is_pmd_migration_entry(pmd_t pmd)
0455 {
0456     return 0;
0457 }
0458 #endif
0459 
0460 #ifdef CONFIG_MEMORY_FAILURE
0461 
0462 extern atomic_long_t num_poisoned_pages __read_mostly;
0463 
0464 /*
0465  * Support for hardware poisoned pages
0466  */
0467 static inline swp_entry_t make_hwpoison_entry(struct page *page)
0468 {
0469     BUG_ON(!PageLocked(page));
0470     return swp_entry(SWP_HWPOISON, page_to_pfn(page));
0471 }
0472 
0473 static inline int is_hwpoison_entry(swp_entry_t entry)
0474 {
0475     return swp_type(entry) == SWP_HWPOISON;
0476 }
0477 
0478 static inline unsigned long hwpoison_entry_to_pfn(swp_entry_t entry)
0479 {
0480     return swp_offset(entry);
0481 }
0482 
0483 static inline void num_poisoned_pages_inc(void)
0484 {
0485     atomic_long_inc(&num_poisoned_pages);
0486 }
0487 
0488 static inline void num_poisoned_pages_dec(void)
0489 {
0490     atomic_long_dec(&num_poisoned_pages);
0491 }
0492 
0493 static inline void num_poisoned_pages_sub(long i)
0494 {
0495     atomic_long_sub(i, &num_poisoned_pages);
0496 }
0497 
0498 #else
0499 
0500 static inline swp_entry_t make_hwpoison_entry(struct page *page)
0501 {
0502     return swp_entry(0, 0);
0503 }
0504 
0505 static inline int is_hwpoison_entry(swp_entry_t swp)
0506 {
0507     return 0;
0508 }
0509 
0510 static inline void num_poisoned_pages_inc(void)
0511 {
0512 }
0513 
0514 static inline void num_poisoned_pages_sub(long i)
0515 {
0516 }
0517 #endif
0518 
0519 static inline int non_swap_entry(swp_entry_t entry)
0520 {
0521     return swp_type(entry) >= MAX_SWAPFILES;
0522 }
0523 
0524 #endif /* CONFIG_MMU */
0525 #endif /* _LINUX_SWAPOPS_H */
[ Source navigation ] [ Diff markup ] [ Identifier search ] [ general search ]

This page was automatically generated by the 2.1.0 LXR engine.

The LXR team
Valid CSS 2.1! Valid HTML 4.01!