kernel/dma/map_benchmark.c

0001 // SPDX-License-Identifier: GPL-2.0-only
0002 /*
0003  * Copyright (C) 2020 HiSilicon Limited.
0004  */
0005
0006 #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
0007
0008 #include <linux/debugfs.h>
0009 #include <linux/delay.h>
0010 #include <linux/device.h>
0011 #include <linux/dma-mapping.h>
0012 #include <linux/kernel.h>
0013 #include <linux/kthread.h>
0014 #include <linux/map_benchmark.h>
0015 #include <linux/math64.h>
0016 #include <linux/module.h>
0017 #include <linux/pci.h>
0018 #include <linux/platform_device.h>
0019 #include <linux/slab.h>
0020 #include <linux/timekeeping.h>
0021
0022 struct map_benchmark_data {
0023     struct map_benchmark bparam;
0024     struct device *dev;
0025     struct dentry  *debugfs;
0026     enum dma_data_direction dir;
0027     atomic64_t sum_map_100ns;
0028     atomic64_t sum_unmap_100ns;
0029     atomic64_t sum_sq_map;
0030     atomic64_t sum_sq_unmap;
0031     atomic64_t loops;
0032 };
0033
0034 static int map_benchmark_thread(void *data)
0035 {
0036     void *buf;
0037     dma_addr_t dma_addr;
0038     struct map_benchmark_data *map = data;
0039     int npages = map->bparam.granule;
0040     u64 size = npages * PAGE_SIZE;
0041     int ret = 0;
0042
0043     buf = alloc_pages_exact(size, GFP_KERNEL);
0044     if (!buf)
0045         return -ENOMEM;
0046
0047     while (!kthread_should_stop())  {
0048         u64 map_100ns, unmap_100ns, map_sq, unmap_sq;
0049         ktime_t map_stime, map_etime, unmap_stime, unmap_etime;
0050         ktime_t map_delta, unmap_delta;
0051
0052         /*
0053          * for a non-coherent device, if we don't stain them in the
0054          * cache, this will give an underestimate of the real-world
0055          * overhead of BIDIRECTIONAL or TO_DEVICE mappings;
0056          * 66 means evertything goes well! 66 is lucky.
0057          */
0058         if (map->dir != DMA_FROM_DEVICE)
0059             memset(buf, 0x66, size);
0060
0061         map_stime = ktime_get();
0062         dma_addr = dma_map_single(map->dev, buf, size, map->dir);
0063         if (unlikely(dma_mapping_error(map->dev, dma_addr))) {
0064             pr_err("dma_map_single failed on %s\n",
0065                 dev_name(map->dev));
0066             ret = -ENOMEM;
0067             goto out;
0068         }
0069         map_etime = ktime_get();
0070         map_delta = ktime_sub(map_etime, map_stime);
0071
0072         /* Pretend DMA is transmitting */
0073         ndelay(map->bparam.dma_trans_ns);
0074
0075         unmap_stime = ktime_get();
0076         dma_unmap_single(map->dev, dma_addr, size, map->dir);
0077         unmap_etime = ktime_get();
0078         unmap_delta = ktime_sub(unmap_etime, unmap_stime);
0079
0080         /* calculate sum and sum of squares */
0081
0082         map_100ns = div64_ul(map_delta,  100);
0083         unmap_100ns = div64_ul(unmap_delta, 100);
0084         map_sq = map_100ns * map_100ns;
0085         unmap_sq = unmap_100ns * unmap_100ns;
0086
0087         atomic64_add(map_100ns, &map->sum_map_100ns);
0088         atomic64_add(unmap_100ns, &map->sum_unmap_100ns);
0089         atomic64_add(map_sq, &map->sum_sq_map);
0090         atomic64_add(unmap_sq, &map->sum_sq_unmap);
0091         atomic64_inc(&map->loops);
0092     }
0093
0094 out:
0095     free_pages_exact(buf, size);
0096     return ret;
0097 }
0098
0099 static int do_map_benchmark(struct map_benchmark_data *map)
0100 {
0101     struct task_struct **tsk;
0102     int threads = map->bparam.threads;
0103     int node = map->bparam.node;
0104     const cpumask_t *cpu_mask = cpumask_of_node(node);
0105     u64 loops;
0106     int ret = 0;
0107     int i;
0108
0109     tsk = kmalloc_array(threads, sizeof(*tsk), GFP_KERNEL);
0110     if (!tsk)
0111         return -ENOMEM;
0112
0113     get_device(map->dev);
0114
0115     for (i = 0; i < threads; i++) {
0116         tsk[i] = kthread_create_on_node(map_benchmark_thread, map,
0117                 map->bparam.node, "dma-map-benchmark/%d", i);
0118         if (IS_ERR(tsk[i])) {
0119             pr_err("create dma_map thread failed\n");
0120             ret = PTR_ERR(tsk[i]);
0121             goto out;
0122         }
0123
0124         if (node != NUMA_NO_NODE)
0125             kthread_bind_mask(tsk[i], cpu_mask);
0126     }
0127
0128     /* clear the old value in the previous benchmark */
0129     atomic64_set(&map->sum_map_100ns, 0);
0130     atomic64_set(&map->sum_unmap_100ns, 0);
0131     atomic64_set(&map->sum_sq_map, 0);
0132     atomic64_set(&map->sum_sq_unmap, 0);
0133     atomic64_set(&map->loops, 0);
0134
0135     for (i = 0; i < threads; i++) {
0136         get_task_struct(tsk[i]);
0137         wake_up_process(tsk[i]);
0138     }
0139
0140     msleep_interruptible(map->bparam.seconds * 1000);
0141
0142     /* wait for the completion of benchmark threads */
0143     for (i = 0; i < threads; i++) {
0144         ret = kthread_stop(tsk[i]);
0145         if (ret)
0146             goto out;
0147     }
0148
0149     loops = atomic64_read(&map->loops);
0150     if (likely(loops > 0)) {
0151         u64 map_variance, unmap_variance;
0152         u64 sum_map = atomic64_read(&map->sum_map_100ns);
0153         u64 sum_unmap = atomic64_read(&map->sum_unmap_100ns);
0154         u64 sum_sq_map = atomic64_read(&map->sum_sq_map);
0155         u64 sum_sq_unmap = atomic64_read(&map->sum_sq_unmap);
0156
0157         /* average latency */
0158         map->bparam.avg_map_100ns = div64_u64(sum_map, loops);
0159         map->bparam.avg_unmap_100ns = div64_u64(sum_unmap, loops);
0160
0161         /* standard deviation of latency */
0162         map_variance = div64_u64(sum_sq_map, loops) -
0163                 map->bparam.avg_map_100ns *
0164                 map->bparam.avg_map_100ns;
0165         unmap_variance = div64_u64(sum_sq_unmap, loops) -
0166                 map->bparam.avg_unmap_100ns *
0167                 map->bparam.avg_unmap_100ns;
0168         map->bparam.map_stddev = int_sqrt64(map_variance);
0169         map->bparam.unmap_stddev = int_sqrt64(unmap_variance);
0170     }
0171
0172 out:
0173     for (i = 0; i < threads; i++)
0174         put_task_struct(tsk[i]);
0175     put_device(map->dev);
0176     kfree(tsk);
0177     return ret;
0178 }
0179
0180 static long map_benchmark_ioctl(struct file *file, unsigned int cmd,
0181         unsigned long arg)
0182 {
0183     struct map_benchmark_data *map = file->private_data;
0184     void __user *argp = (void __user *)arg;
0185     u64 old_dma_mask;
0186     int ret;
0187
0188     if (copy_from_user(&map->bparam, argp, sizeof(map->bparam)))
0189         return -EFAULT;
0190
0191     switch (cmd) {
0192     case DMA_MAP_BENCHMARK:
0193         if (map->bparam.threads == 0 ||
0194             map->bparam.threads > DMA_MAP_MAX_THREADS) {
0195             pr_err("invalid thread number\n");
0196             return -EINVAL;
0197         }
0198
0199         if (map->bparam.seconds == 0 ||
0200             map->bparam.seconds > DMA_MAP_MAX_SECONDS) {
0201             pr_err("invalid duration seconds\n");
0202             return -EINVAL;
0203         }
0204
0205         if (map->bparam.dma_trans_ns > DMA_MAP_MAX_TRANS_DELAY) {
0206             pr_err("invalid transmission delay\n");
0207             return -EINVAL;
0208         }
0209
0210         if (map->bparam.node != NUMA_NO_NODE &&
0211             !node_possible(map->bparam.node)) {
0212             pr_err("invalid numa node\n");
0213             return -EINVAL;
0214         }
0215
0216         if (map->bparam.granule < 1 || map->bparam.granule > 1024) {
0217             pr_err("invalid granule size\n");
0218             return -EINVAL;
0219         }
0220
0221         switch (map->bparam.dma_dir) {
0222         case DMA_MAP_BIDIRECTIONAL:
0223             map->dir = DMA_BIDIRECTIONAL;
0224             break;
0225         case DMA_MAP_FROM_DEVICE:
0226             map->dir = DMA_FROM_DEVICE;
0227             break;
0228         case DMA_MAP_TO_DEVICE:
0229             map->dir = DMA_TO_DEVICE;
0230             break;
0231         default:
0232             pr_err("invalid DMA direction\n");
0233             return -EINVAL;
0234         }
0235
0236         old_dma_mask = dma_get_mask(map->dev);
0237
0238         ret = dma_set_mask(map->dev,
0239                    DMA_BIT_MASK(map->bparam.dma_bits));
0240         if (ret) {
0241             pr_err("failed to set dma_mask on device %s\n",
0242                 dev_name(map->dev));
0243             return -EINVAL;
0244         }
0245
0246         ret = do_map_benchmark(map);
0247
0248         /*
0249          * restore the original dma_mask as many devices' dma_mask are
0250          * set by architectures, acpi, busses. When we bind them back
0251          * to their original drivers, those drivers shouldn't see
0252          * dma_mask changed by benchmark
0253          */
0254         dma_set_mask(map->dev, old_dma_mask);
0255         break;
0256     default:
0257         return -EINVAL;
0258     }
0259
0260     if (copy_to_user(argp, &map->bparam, sizeof(map->bparam)))
0261         return -EFAULT;
0262
0263     return ret;
0264 }
0265
0266 static const struct file_operations map_benchmark_fops = {
0267     .open           = simple_open,
0268     .unlocked_ioctl     = map_benchmark_ioctl,
0269 };
0270
0271 static void map_benchmark_remove_debugfs(void *data)
0272 {
0273     struct map_benchmark_data *map = (struct map_benchmark_data *)data;
0274
0275     debugfs_remove(map->debugfs);
0276 }
0277
0278 static int __map_benchmark_probe(struct device *dev)
0279 {
0280     struct dentry *entry;
0281     struct map_benchmark_data *map;
0282     int ret;
0283
0284     map = devm_kzalloc(dev, sizeof(*map), GFP_KERNEL);
0285     if (!map)
0286         return -ENOMEM;
0287     map->dev = dev;
0288
0289     ret = devm_add_action(dev, map_benchmark_remove_debugfs, map);
0290     if (ret) {
0291         pr_err("Can't add debugfs remove action\n");
0292         return ret;
0293     }
0294
0295     /*
0296      * we only permit a device bound with this driver, 2nd probe
0297      * will fail
0298      */
0299     entry = debugfs_create_file("dma_map_benchmark", 0600, NULL, map,
0300             &map_benchmark_fops);
0301     if (IS_ERR(entry))
0302         return PTR_ERR(entry);
0303     map->debugfs = entry;
0304
0305     return 0;
0306 }
0307
0308 static int map_benchmark_platform_probe(struct platform_device *pdev)
0309 {
0310     return __map_benchmark_probe(&pdev->dev);
0311 }
0312
0313 static struct platform_driver map_benchmark_platform_driver = {
0314     .driver     = {
0315         .name   = "dma_map_benchmark",
0316     },
0317     .probe = map_benchmark_platform_probe,
0318 };
0319
0320 static int
0321 map_benchmark_pci_probe(struct pci_dev *pdev, const struct pci_device_id *id)
0322 {
0323     return __map_benchmark_probe(&pdev->dev);
0324 }
0325
0326 static struct pci_driver map_benchmark_pci_driver = {
0327     .name   = "dma_map_benchmark",
0328     .probe  = map_benchmark_pci_probe,
0329 };
0330
0331 static int __init map_benchmark_init(void)
0332 {
0333     int ret;
0334
0335     ret = pci_register_driver(&map_benchmark_pci_driver);
0336     if (ret)
0337         return ret;
0338
0339     ret = platform_driver_register(&map_benchmark_platform_driver);
0340     if (ret) {
0341         pci_unregister_driver(&map_benchmark_pci_driver);
0342         return ret;
0343     }
0344
0345     return 0;
0346 }
0347
0348 static void __exit map_benchmark_cleanup(void)
0349 {
0350     platform_driver_unregister(&map_benchmark_platform_driver);
0351     pci_unregister_driver(&map_benchmark_pci_driver);
0352 }
0353
0354 module_init(map_benchmark_init);
0355 module_exit(map_benchmark_cleanup);
0356
0357 MODULE_AUTHOR("Barry Song <song.bao.hua@hisilicon.com>");
0358 MODULE_DESCRIPTION("dma_map benchmark driver");
0359 MODULE_LICENSE("GPL");