Back to home page

LXR

 
 

    


0001 /*
0002  * Copyright (C) 2014 Davidlohr Bueso.
0003  */
0004 #include <linux/sched.h>
0005 #include <linux/mm.h>
0006 #include <linux/vmacache.h>
0007 
0008 /*
0009  * Flush vma caches for threads that share a given mm.
0010  *
0011  * The operation is safe because the caller holds the mmap_sem
0012  * exclusively and other threads accessing the vma cache will
0013  * have mmap_sem held at least for read, so no extra locking
0014  * is required to maintain the vma cache.
0015  */
0016 void vmacache_flush_all(struct mm_struct *mm)
0017 {
0018     struct task_struct *g, *p;
0019 
0020     count_vm_vmacache_event(VMACACHE_FULL_FLUSHES);
0021 
0022     /*
0023      * Single threaded tasks need not iterate the entire
0024      * list of process. We can avoid the flushing as well
0025      * since the mm's seqnum was increased and don't have
0026      * to worry about other threads' seqnum. Current's
0027      * flush will occur upon the next lookup.
0028      */
0029     if (atomic_read(&mm->mm_users) == 1)
0030         return;
0031 
0032     rcu_read_lock();
0033     for_each_process_thread(g, p) {
0034         /*
0035          * Only flush the vmacache pointers as the
0036          * mm seqnum is already set and curr's will
0037          * be set upon invalidation when the next
0038          * lookup is done.
0039          */
0040         if (mm == p->mm)
0041             vmacache_flush(p);
0042     }
0043     rcu_read_unlock();
0044 }
0045 
0046 /*
0047  * This task may be accessing a foreign mm via (for example)
0048  * get_user_pages()->find_vma().  The vmacache is task-local and this
0049  * task's vmacache pertains to a different mm (ie, its own).  There is
0050  * nothing we can do here.
0051  *
0052  * Also handle the case where a kernel thread has adopted this mm via use_mm().
0053  * That kernel thread's vmacache is not applicable to this mm.
0054  */
0055 static inline bool vmacache_valid_mm(struct mm_struct *mm)
0056 {
0057     return current->mm == mm && !(current->flags & PF_KTHREAD);
0058 }
0059 
0060 void vmacache_update(unsigned long addr, struct vm_area_struct *newvma)
0061 {
0062     if (vmacache_valid_mm(newvma->vm_mm))
0063         current->vmacache[VMACACHE_HASH(addr)] = newvma;
0064 }
0065 
0066 static bool vmacache_valid(struct mm_struct *mm)
0067 {
0068     struct task_struct *curr;
0069 
0070     if (!vmacache_valid_mm(mm))
0071         return false;
0072 
0073     curr = current;
0074     if (mm->vmacache_seqnum != curr->vmacache_seqnum) {
0075         /*
0076          * First attempt will always be invalid, initialize
0077          * the new cache for this task here.
0078          */
0079         curr->vmacache_seqnum = mm->vmacache_seqnum;
0080         vmacache_flush(curr);
0081         return false;
0082     }
0083     return true;
0084 }
0085 
0086 struct vm_area_struct *vmacache_find(struct mm_struct *mm, unsigned long addr)
0087 {
0088     int i;
0089 
0090     count_vm_vmacache_event(VMACACHE_FIND_CALLS);
0091 
0092     if (!vmacache_valid(mm))
0093         return NULL;
0094 
0095     for (i = 0; i < VMACACHE_SIZE; i++) {
0096         struct vm_area_struct *vma = current->vmacache[i];
0097 
0098         if (!vma)
0099             continue;
0100         if (WARN_ON_ONCE(vma->vm_mm != mm))
0101             break;
0102         if (vma->vm_start <= addr && vma->vm_end > addr) {
0103             count_vm_vmacache_event(VMACACHE_FIND_HITS);
0104             return vma;
0105         }
0106     }
0107 
0108     return NULL;
0109 }
0110 
0111 #ifndef CONFIG_MMU
0112 struct vm_area_struct *vmacache_find_exact(struct mm_struct *mm,
0113                        unsigned long start,
0114                        unsigned long end)
0115 {
0116     int i;
0117 
0118     count_vm_vmacache_event(VMACACHE_FIND_CALLS);
0119 
0120     if (!vmacache_valid(mm))
0121         return NULL;
0122 
0123     for (i = 0; i < VMACACHE_SIZE; i++) {
0124         struct vm_area_struct *vma = current->vmacache[i];
0125 
0126         if (vma && vma->vm_start == start && vma->vm_end == end) {
0127             count_vm_vmacache_event(VMACACHE_FIND_HITS);
0128             return vma;
0129         }
0130     }
0131 
0132     return NULL;
0133 }
0134 #endif