the-tree/mm/gup_test.c

0001 #include <linux/kernel.h>
0002 #include <linux/mm.h>
0003 #include <linux/slab.h>
0004 #include <linux/uaccess.h>
0005 #include <linux/ktime.h>
0006 #include <linux/debugfs.h>
0007 #include "gup_test.h"
0008
0009 static void put_back_pages(unsigned int cmd, struct page **pages,
0010                unsigned long nr_pages, unsigned int gup_test_flags)
0011 {
0012     unsigned long i;
0013
0014     switch (cmd) {
0015     case GUP_FAST_BENCHMARK:
0016     case GUP_BASIC_TEST:
0017         for (i = 0; i < nr_pages; i++)
0018             put_page(pages[i]);
0019         break;
0020
0021     case PIN_FAST_BENCHMARK:
0022     case PIN_BASIC_TEST:
0023     case PIN_LONGTERM_BENCHMARK:
0024         unpin_user_pages(pages, nr_pages);
0025         break;
0026     case DUMP_USER_PAGES_TEST:
0027         if (gup_test_flags & GUP_TEST_FLAG_DUMP_PAGES_USE_PIN) {
0028             unpin_user_pages(pages, nr_pages);
0029         } else {
0030             for (i = 0; i < nr_pages; i++)
0031                 put_page(pages[i]);
0032
0033         }
0034         break;
0035     }
0036 }
0037
0038 static void verify_dma_pinned(unsigned int cmd, struct page **pages,
0039                   unsigned long nr_pages)
0040 {
0041     unsigned long i;
0042     struct page *page;
0043
0044     switch (cmd) {
0045     case PIN_FAST_BENCHMARK:
0046     case PIN_BASIC_TEST:
0047     case PIN_LONGTERM_BENCHMARK:
0048         for (i = 0; i < nr_pages; i++) {
0049             page = pages[i];
0050             if (WARN(!page_maybe_dma_pinned(page),
0051                  "pages[%lu] is NOT dma-pinned\n", i)) {
0052
0053                 dump_page(page, "gup_test failure");
0054                 break;
0055             } else if (cmd == PIN_LONGTERM_BENCHMARK &&
0056                 WARN(!is_longterm_pinnable_page(page),
0057                      "pages[%lu] is NOT pinnable but pinned\n",
0058                      i)) {
0059                 dump_page(page, "gup_test failure");
0060                 break;
0061             }
0062         }
0063         break;
0064     }
0065 }
0066
0067 static void dump_pages_test(struct gup_test *gup, struct page **pages,
0068                 unsigned long nr_pages)
0069 {
0070     unsigned int index_to_dump;
0071     unsigned int i;
0072
0073     /*
0074      * Zero out any user-supplied page index that is out of range. Remember:
0075      * .which_pages[] contains a 1-based set of page indices.
0076      */
0077     for (i = 0; i < GUP_TEST_MAX_PAGES_TO_DUMP; i++) {
0078         if (gup->which_pages[i] > nr_pages) {
0079             pr_warn("ZEROING due to out of range: .which_pages[%u]: %u\n",
0080                 i, gup->which_pages[i]);
0081             gup->which_pages[i] = 0;
0082         }
0083     }
0084
0085     for (i = 0; i < GUP_TEST_MAX_PAGES_TO_DUMP; i++) {
0086         index_to_dump = gup->which_pages[i];
0087
0088         if (index_to_dump) {
0089             index_to_dump--; // Decode from 1-based, to 0-based
0090             pr_info("---- page #%u, starting from user virt addr: 0x%llx\n",
0091                 index_to_dump, gup->addr);
0092             dump_page(pages[index_to_dump],
0093                   "gup_test: dump_pages() test");
0094         }
0095     }
0096 }
0097
0098 static int __gup_test_ioctl(unsigned int cmd,
0099         struct gup_test *gup)
0100 {
0101     ktime_t start_time, end_time;
0102     unsigned long i, nr_pages, addr, next;
0103     long nr;
0104     struct page **pages;
0105     int ret = 0;
0106     bool needs_mmap_lock =
0107         cmd != GUP_FAST_BENCHMARK && cmd != PIN_FAST_BENCHMARK;
0108
0109     if (gup->size > ULONG_MAX)
0110         return -EINVAL;
0111
0112     nr_pages = gup->size / PAGE_SIZE;
0113     pages = kvcalloc(nr_pages, sizeof(void *), GFP_KERNEL);
0114     if (!pages)
0115         return -ENOMEM;
0116
0117     if (needs_mmap_lock && mmap_read_lock_killable(current->mm)) {
0118         ret = -EINTR;
0119         goto free_pages;
0120     }
0121
0122     i = 0;
0123     nr = gup->nr_pages_per_call;
0124     start_time = ktime_get();
0125     for (addr = gup->addr; addr < gup->addr + gup->size; addr = next) {
0126         if (nr != gup->nr_pages_per_call)
0127             break;
0128
0129         next = addr + nr * PAGE_SIZE;
0130         if (next > gup->addr + gup->size) {
0131             next = gup->addr + gup->size;
0132             nr = (next - addr) / PAGE_SIZE;
0133         }
0134
0135         switch (cmd) {
0136         case GUP_FAST_BENCHMARK:
0137             nr = get_user_pages_fast(addr, nr, gup->gup_flags,
0138                          pages + i);
0139             break;
0140         case GUP_BASIC_TEST:
0141             nr = get_user_pages(addr, nr, gup->gup_flags, pages + i,
0142                         NULL);
0143             break;
0144         case PIN_FAST_BENCHMARK:
0145             nr = pin_user_pages_fast(addr, nr, gup->gup_flags,
0146                          pages + i);
0147             break;
0148         case PIN_BASIC_TEST:
0149             nr = pin_user_pages(addr, nr, gup->gup_flags, pages + i,
0150                         NULL);
0151             break;
0152         case PIN_LONGTERM_BENCHMARK:
0153             nr = pin_user_pages(addr, nr,
0154                         gup->gup_flags | FOLL_LONGTERM,
0155                         pages + i, NULL);
0156             break;
0157         case DUMP_USER_PAGES_TEST:
0158             if (gup->test_flags & GUP_TEST_FLAG_DUMP_PAGES_USE_PIN)
0159                 nr = pin_user_pages(addr, nr, gup->gup_flags,
0160                             pages + i, NULL);
0161             else
0162                 nr = get_user_pages(addr, nr, gup->gup_flags,
0163                             pages + i, NULL);
0164             break;
0165         default:
0166             ret = -EINVAL;
0167             goto unlock;
0168         }
0169
0170         if (nr <= 0)
0171             break;
0172         i += nr;
0173     }
0174     end_time = ktime_get();
0175
0176     /* Shifting the meaning of nr_pages: now it is actual number pinned: */
0177     nr_pages = i;
0178
0179     gup->get_delta_usec = ktime_us_delta(end_time, start_time);
0180     gup->size = addr - gup->addr;
0181
0182     /*
0183      * Take an un-benchmark-timed moment to verify DMA pinned
0184      * state: print a warning if any non-dma-pinned pages are found:
0185      */
0186     verify_dma_pinned(cmd, pages, nr_pages);
0187
0188     if (cmd == DUMP_USER_PAGES_TEST)
0189         dump_pages_test(gup, pages, nr_pages);
0190
0191     start_time = ktime_get();
0192
0193     put_back_pages(cmd, pages, nr_pages, gup->test_flags);
0194
0195     end_time = ktime_get();
0196     gup->put_delta_usec = ktime_us_delta(end_time, start_time);
0197
0198 unlock:
0199     if (needs_mmap_lock)
0200         mmap_read_unlock(current->mm);
0201 free_pages:
0202     kvfree(pages);
0203     return ret;
0204 }
0205
0206 static long gup_test_ioctl(struct file *filep, unsigned int cmd,
0207         unsigned long arg)
0208 {
0209     struct gup_test gup;
0210     int ret;
0211
0212     switch (cmd) {
0213     case GUP_FAST_BENCHMARK:
0214     case PIN_FAST_BENCHMARK:
0215     case PIN_LONGTERM_BENCHMARK:
0216     case GUP_BASIC_TEST:
0217     case PIN_BASIC_TEST:
0218     case DUMP_USER_PAGES_TEST:
0219         break;
0220     default:
0221         return -EINVAL;
0222     }
0223
0224     if (copy_from_user(&gup, (void __user *)arg, sizeof(gup)))
0225         return -EFAULT;
0226
0227     ret = __gup_test_ioctl(cmd, &gup);
0228     if (ret)
0229         return ret;
0230
0231     if (copy_to_user((void __user *)arg, &gup, sizeof(gup)))
0232         return -EFAULT;
0233
0234     return 0;
0235 }
0236
0237 static const struct file_operations gup_test_fops = {
0238     .open = nonseekable_open,
0239     .unlocked_ioctl = gup_test_ioctl,
0240 };
0241
0242 static int __init gup_test_init(void)
0243 {
0244     debugfs_create_file_unsafe("gup_test", 0600, NULL, NULL,
0245                    &gup_test_fops);
0246
0247     return 0;
0248 }
0249
0250 late_initcall(gup_test_init);