kernel/bpf/mmap_unlock_work.h

0001 /* SPDX-License-Identifier: GPL-2.0-only */
0002 /* Copyright (c) 2021 Facebook
0003  */
0004
0005 #ifndef __MMAP_UNLOCK_WORK_H__
0006 #define __MMAP_UNLOCK_WORK_H__
0007 #include <linux/irq_work.h>
0008
0009 /* irq_work to run mmap_read_unlock() in irq_work */
0010 struct mmap_unlock_irq_work {
0011     struct irq_work irq_work;
0012     struct mm_struct *mm;
0013 };
0014
0015 DECLARE_PER_CPU(struct mmap_unlock_irq_work, mmap_unlock_work);
0016
0017 /*
0018  * We cannot do mmap_read_unlock() when the irq is disabled, because of
0019  * risk to deadlock with rq_lock. To look up vma when the irqs are
0020  * disabled, we need to run mmap_read_unlock() in irq_work. We use a
0021  * percpu variable to do the irq_work. If the irq_work is already used
0022  * by another lookup, we fall over.
0023  */
0024 static inline bool bpf_mmap_unlock_get_irq_work(struct mmap_unlock_irq_work **work_ptr)
0025 {
0026     struct mmap_unlock_irq_work *work = NULL;
0027     bool irq_work_busy = false;
0028
0029     if (irqs_disabled()) {
0030         if (!IS_ENABLED(CONFIG_PREEMPT_RT)) {
0031             work = this_cpu_ptr(&mmap_unlock_work);
0032             if (irq_work_is_busy(&work->irq_work)) {
0033                 /* cannot queue more up_read, fallback */
0034                 irq_work_busy = true;
0035             }
0036         } else {
0037             /*
0038              * PREEMPT_RT does not allow to trylock mmap sem in
0039              * interrupt disabled context. Force the fallback code.
0040              */
0041             irq_work_busy = true;
0042         }
0043     }
0044
0045     *work_ptr = work;
0046     return irq_work_busy;
0047 }
0048
0049 static inline void bpf_mmap_unlock_mm(struct mmap_unlock_irq_work *work, struct mm_struct *mm)
0050 {
0051     if (!work) {
0052         mmap_read_unlock(mm);
0053     } else {
0054         work->mm = mm;
0055
0056         /* The lock will be released once we're out of interrupt
0057          * context. Tell lockdep that we've released it now so
0058          * it doesn't complain that we forgot to release it.
0059          */
0060         rwsem_release(&mm->mmap_lock.dep_map, _RET_IP_);
0061         irq_work_queue(&work->irq_work);
0062     }
0063 }
0064
0065 #endif /* __MMAP_UNLOCK_WORK_H__ */