include/linux/userfaultfd_k.h

0001 /* SPDX-License-Identifier: GPL-2.0 */
0002 /*
0003  *  include/linux/userfaultfd_k.h
0004  *
0005  *  Copyright (C) 2015  Red Hat, Inc.
0006  *
0007  */
0008
0009 #ifndef _LINUX_USERFAULTFD_K_H
0010 #define _LINUX_USERFAULTFD_K_H
0011
0012 #ifdef CONFIG_USERFAULTFD
0013
0014 #include <linux/userfaultfd.h> /* linux/include/uapi/linux/userfaultfd.h */
0015
0016 #include <linux/fcntl.h>
0017 #include <linux/mm.h>
0018 #include <linux/swap.h>
0019 #include <linux/swapops.h>
0020 #include <asm-generic/pgtable_uffd.h>
0021 #include <linux/hugetlb_inline.h>
0022
0023 /* The set of all possible UFFD-related VM flags. */
0024 #define __VM_UFFD_FLAGS (VM_UFFD_MISSING | VM_UFFD_WP | VM_UFFD_MINOR)
0025
0026 /*
0027  * CAREFUL: Check include/uapi/asm-generic/fcntl.h when defining
0028  * new flags, since they might collide with O_* ones. We want
0029  * to re-use O_* flags that couldn't possibly have a meaning
0030  * from userfaultfd, in order to leave a free define-space for
0031  * shared O_* flags.
0032  */
0033 #define UFFD_CLOEXEC O_CLOEXEC
0034 #define UFFD_NONBLOCK O_NONBLOCK
0035
0036 #define UFFD_SHARED_FCNTL_FLAGS (O_CLOEXEC | O_NONBLOCK)
0037 #define UFFD_FLAGS_SET (EFD_SHARED_FCNTL_FLAGS)
0038
0039 extern int sysctl_unprivileged_userfaultfd;
0040
0041 extern vm_fault_t handle_userfault(struct vm_fault *vmf, unsigned long reason);
0042
0043 /*
0044  * The mode of operation for __mcopy_atomic and its helpers.
0045  *
0046  * This is almost an implementation detail (mcopy_atomic below doesn't take this
0047  * as a parameter), but it's exposed here because memory-kind-specific
0048  * implementations (e.g. hugetlbfs) need to know the mode of operation.
0049  */
0050 enum mcopy_atomic_mode {
0051     /* A normal copy_from_user into the destination range. */
0052     MCOPY_ATOMIC_NORMAL,
0053     /* Don't copy; map the destination range to the zero page. */
0054     MCOPY_ATOMIC_ZEROPAGE,
0055     /* Just install pte(s) with the existing page(s) in the page cache. */
0056     MCOPY_ATOMIC_CONTINUE,
0057 };
0058
0059 extern int mfill_atomic_install_pte(struct mm_struct *dst_mm, pmd_t *dst_pmd,
0060                     struct vm_area_struct *dst_vma,
0061                     unsigned long dst_addr, struct page *page,
0062                     bool newly_allocated, bool wp_copy);
0063
0064 extern ssize_t mcopy_atomic(struct mm_struct *dst_mm, unsigned long dst_start,
0065                 unsigned long src_start, unsigned long len,
0066                 atomic_t *mmap_changing, __u64 mode);
0067 extern ssize_t mfill_zeropage(struct mm_struct *dst_mm,
0068                   unsigned long dst_start,
0069                   unsigned long len,
0070                   atomic_t *mmap_changing);
0071 extern ssize_t mcopy_continue(struct mm_struct *dst_mm, unsigned long dst_start,
0072                   unsigned long len, atomic_t *mmap_changing);
0073 extern int mwriteprotect_range(struct mm_struct *dst_mm,
0074                    unsigned long start, unsigned long len,
0075                    bool enable_wp, atomic_t *mmap_changing);
0076 extern void uffd_wp_range(struct mm_struct *dst_mm, struct vm_area_struct *vma,
0077               unsigned long start, unsigned long len, bool enable_wp);
0078
0079 /* mm helpers */
0080 static inline bool is_mergeable_vm_userfaultfd_ctx(struct vm_area_struct *vma,
0081                     struct vm_userfaultfd_ctx vm_ctx)
0082 {
0083     return vma->vm_userfaultfd_ctx.ctx == vm_ctx.ctx;
0084 }
0085
0086 /*
0087  * Never enable huge pmd sharing on some uffd registered vmas:
0088  *
0089  * - VM_UFFD_WP VMAs, because write protect information is per pgtable entry.
0090  *
0091  * - VM_UFFD_MINOR VMAs, because otherwise we would never get minor faults for
0092  *   VMAs which share huge pmds. (If you have two mappings to the same
0093  *   underlying pages, and fault in the non-UFFD-registered one with a write,
0094  *   with huge pmd sharing this would *also* setup the second UFFD-registered
0095  *   mapping, and we'd not get minor faults.)
0096  */
0097 static inline bool uffd_disable_huge_pmd_share(struct vm_area_struct *vma)
0098 {
0099     return vma->vm_flags & (VM_UFFD_WP | VM_UFFD_MINOR);
0100 }
0101
0102 /*
0103  * Don't do fault around for either WP or MINOR registered uffd range.  For
0104  * MINOR registered range, fault around will be a total disaster and ptes can
0105  * be installed without notifications; for WP it should mostly be fine as long
0106  * as the fault around checks for pte_none() before the installation, however
0107  * to be super safe we just forbid it.
0108  */
0109 static inline bool uffd_disable_fault_around(struct vm_area_struct *vma)
0110 {
0111     return vma->vm_flags & (VM_UFFD_WP | VM_UFFD_MINOR);
0112 }
0113
0114 static inline bool userfaultfd_missing(struct vm_area_struct *vma)
0115 {
0116     return vma->vm_flags & VM_UFFD_MISSING;
0117 }
0118
0119 static inline bool userfaultfd_wp(struct vm_area_struct *vma)
0120 {
0121     return vma->vm_flags & VM_UFFD_WP;
0122 }
0123
0124 static inline bool userfaultfd_minor(struct vm_area_struct *vma)
0125 {
0126     return vma->vm_flags & VM_UFFD_MINOR;
0127 }
0128
0129 static inline bool userfaultfd_pte_wp(struct vm_area_struct *vma,
0130                       pte_t pte)
0131 {
0132     return userfaultfd_wp(vma) && pte_uffd_wp(pte);
0133 }
0134
0135 static inline bool userfaultfd_huge_pmd_wp(struct vm_area_struct *vma,
0136                        pmd_t pmd)
0137 {
0138     return userfaultfd_wp(vma) && pmd_uffd_wp(pmd);
0139 }
0140
0141 static inline bool userfaultfd_armed(struct vm_area_struct *vma)
0142 {
0143     return vma->vm_flags & __VM_UFFD_FLAGS;
0144 }
0145
0146 static inline bool vma_can_userfault(struct vm_area_struct *vma,
0147                      unsigned long vm_flags)
0148 {
0149     if (vm_flags & VM_UFFD_MINOR)
0150         return is_vm_hugetlb_page(vma) || vma_is_shmem(vma);
0151
0152 #ifndef CONFIG_PTE_MARKER_UFFD_WP
0153     /*
0154      * If user requested uffd-wp but not enabled pte markers for
0155      * uffd-wp, then shmem & hugetlbfs are not supported but only
0156      * anonymous.
0157      */
0158     if ((vm_flags & VM_UFFD_WP) && !vma_is_anonymous(vma))
0159         return false;
0160 #endif
0161     return vma_is_anonymous(vma) || is_vm_hugetlb_page(vma) ||
0162         vma_is_shmem(vma);
0163 }
0164
0165 extern int dup_userfaultfd(struct vm_area_struct *, struct list_head *);
0166 extern void dup_userfaultfd_complete(struct list_head *);
0167
0168 extern void mremap_userfaultfd_prep(struct vm_area_struct *,
0169                     struct vm_userfaultfd_ctx *);
0170 extern void mremap_userfaultfd_complete(struct vm_userfaultfd_ctx *,
0171                     unsigned long from, unsigned long to,
0172                     unsigned long len);
0173
0174 extern bool userfaultfd_remove(struct vm_area_struct *vma,
0175                    unsigned long start,
0176                    unsigned long end);
0177
0178 extern int userfaultfd_unmap_prep(struct vm_area_struct *vma,
0179                   unsigned long start, unsigned long end,
0180                   struct list_head *uf);
0181 extern void userfaultfd_unmap_complete(struct mm_struct *mm,
0182                        struct list_head *uf);
0183
0184 #else /* CONFIG_USERFAULTFD */
0185
0186 /* mm helpers */
0187 static inline vm_fault_t handle_userfault(struct vm_fault *vmf,
0188                 unsigned long reason)
0189 {
0190     return VM_FAULT_SIGBUS;
0191 }
0192
0193 static inline bool is_mergeable_vm_userfaultfd_ctx(struct vm_area_struct *vma,
0194                     struct vm_userfaultfd_ctx vm_ctx)
0195 {
0196     return true;
0197 }
0198
0199 static inline bool userfaultfd_missing(struct vm_area_struct *vma)
0200 {
0201     return false;
0202 }
0203
0204 static inline bool userfaultfd_wp(struct vm_area_struct *vma)
0205 {
0206     return false;
0207 }
0208
0209 static inline bool userfaultfd_minor(struct vm_area_struct *vma)
0210 {
0211     return false;
0212 }
0213
0214 static inline bool userfaultfd_pte_wp(struct vm_area_struct *vma,
0215                       pte_t pte)
0216 {
0217     return false;
0218 }
0219
0220 static inline bool userfaultfd_huge_pmd_wp(struct vm_area_struct *vma,
0221                        pmd_t pmd)
0222 {
0223     return false;
0224 }
0225
0226
0227 static inline bool userfaultfd_armed(struct vm_area_struct *vma)
0228 {
0229     return false;
0230 }
0231
0232 static inline int dup_userfaultfd(struct vm_area_struct *vma,
0233                   struct list_head *l)
0234 {
0235     return 0;
0236 }
0237
0238 static inline void dup_userfaultfd_complete(struct list_head *l)
0239 {
0240 }
0241
0242 static inline void mremap_userfaultfd_prep(struct vm_area_struct *vma,
0243                        struct vm_userfaultfd_ctx *ctx)
0244 {
0245 }
0246
0247 static inline void mremap_userfaultfd_complete(struct vm_userfaultfd_ctx *ctx,
0248                            unsigned long from,
0249                            unsigned long to,
0250                            unsigned long len)
0251 {
0252 }
0253
0254 static inline bool userfaultfd_remove(struct vm_area_struct *vma,
0255                       unsigned long start,
0256                       unsigned long end)
0257 {
0258     return true;
0259 }
0260
0261 static inline int userfaultfd_unmap_prep(struct vm_area_struct *vma,
0262                      unsigned long start, unsigned long end,
0263                      struct list_head *uf)
0264 {
0265     return 0;
0266 }
0267
0268 static inline void userfaultfd_unmap_complete(struct mm_struct *mm,
0269                           struct list_head *uf)
0270 {
0271 }
0272
0273 static inline bool uffd_disable_fault_around(struct vm_area_struct *vma)
0274 {
0275     return false;
0276 }
0277
0278 #endif /* CONFIG_USERFAULTFD */
0279
0280 static inline bool pte_marker_entry_uffd_wp(swp_entry_t entry)
0281 {
0282 #ifdef CONFIG_PTE_MARKER_UFFD_WP
0283     return is_pte_marker_entry(entry) &&
0284         (pte_marker_get(entry) & PTE_MARKER_UFFD_WP);
0285 #else
0286     return false;
0287 #endif
0288 }
0289
0290 static inline bool pte_marker_uffd_wp(pte_t pte)
0291 {
0292 #ifdef CONFIG_PTE_MARKER_UFFD_WP
0293     swp_entry_t entry;
0294
0295     if (!is_swap_pte(pte))
0296         return false;
0297
0298     entry = pte_to_swp_entry(pte);
0299
0300     return pte_marker_entry_uffd_wp(entry);
0301 #else
0302     return false;
0303 #endif
0304 }
0305
0306 /*
0307  * Returns true if this is a swap pte and was uffd-wp wr-protected in either
0308  * forms (pte marker or a normal swap pte), false otherwise.
0309  */
0310 static inline bool pte_swp_uffd_wp_any(pte_t pte)
0311 {
0312 #ifdef CONFIG_PTE_MARKER_UFFD_WP
0313     if (!is_swap_pte(pte))
0314         return false;
0315
0316     if (pte_swp_uffd_wp(pte))
0317         return true;
0318
0319     if (pte_marker_uffd_wp(pte))
0320         return true;
0321 #endif
0322     return false;
0323 }
0324
0325 #endif /* _LINUX_USERFAULTFD_K_H */