the-tree/crypto/xor.c

0001 // SPDX-License-Identifier: GPL-2.0-or-later
0002 /*
0003  * xor.c : Multiple Devices driver for Linux
0004  *
0005  * Copyright (C) 1996, 1997, 1998, 1999, 2000,
0006  * Ingo Molnar, Matti Aarnio, Jakub Jelinek, Richard Henderson.
0007  *
0008  * Dispatch optimized RAID-5 checksumming functions.
0009  */
0010
0011 #define BH_TRACE 0
0012 #include <linux/module.h>
0013 #include <linux/gfp.h>
0014 #include <linux/raid/xor.h>
0015 #include <linux/jiffies.h>
0016 #include <linux/preempt.h>
0017 #include <asm/xor.h>
0018
0019 #ifndef XOR_SELECT_TEMPLATE
0020 #define XOR_SELECT_TEMPLATE(x) (x)
0021 #endif
0022
0023 /* The xor routines to use.  */
0024 static struct xor_block_template *active_template;
0025
0026 void
0027 xor_blocks(unsigned int src_count, unsigned int bytes, void *dest, void **srcs)
0028 {
0029     unsigned long *p1, *p2, *p3, *p4;
0030
0031     p1 = (unsigned long *) srcs[0];
0032     if (src_count == 1) {
0033         active_template->do_2(bytes, dest, p1);
0034         return;
0035     }
0036
0037     p2 = (unsigned long *) srcs[1];
0038     if (src_count == 2) {
0039         active_template->do_3(bytes, dest, p1, p2);
0040         return;
0041     }
0042
0043     p3 = (unsigned long *) srcs[2];
0044     if (src_count == 3) {
0045         active_template->do_4(bytes, dest, p1, p2, p3);
0046         return;
0047     }
0048
0049     p4 = (unsigned long *) srcs[3];
0050     active_template->do_5(bytes, dest, p1, p2, p3, p4);
0051 }
0052 EXPORT_SYMBOL(xor_blocks);
0053
0054 /* Set of all registered templates.  */
0055 static struct xor_block_template *__initdata template_list;
0056
0057 #ifndef MODULE
0058 static void __init do_xor_register(struct xor_block_template *tmpl)
0059 {
0060     tmpl->next = template_list;
0061     template_list = tmpl;
0062 }
0063
0064 static int __init register_xor_blocks(void)
0065 {
0066     active_template = XOR_SELECT_TEMPLATE(NULL);
0067
0068     if (!active_template) {
0069 #define xor_speed   do_xor_register
0070         // register all the templates and pick the first as the default
0071         XOR_TRY_TEMPLATES;
0072 #undef xor_speed
0073         active_template = template_list;
0074     }
0075     return 0;
0076 }
0077 #endif
0078
0079 #define BENCH_SIZE  4096
0080 #define REPS        800U
0081
0082 static void __init
0083 do_xor_speed(struct xor_block_template *tmpl, void *b1, void *b2)
0084 {
0085     int speed;
0086     int i, j;
0087     ktime_t min, start, diff;
0088
0089     tmpl->next = template_list;
0090     template_list = tmpl;
0091
0092     preempt_disable();
0093
0094     min = (ktime_t)S64_MAX;
0095     for (i = 0; i < 3; i++) {
0096         start = ktime_get();
0097         for (j = 0; j < REPS; j++) {
0098             mb(); /* prevent loop optimization */
0099             tmpl->do_2(BENCH_SIZE, b1, b2);
0100             mb();
0101         }
0102         diff = ktime_sub(ktime_get(), start);
0103         if (diff < min)
0104             min = diff;
0105     }
0106
0107     preempt_enable();
0108
0109     // bytes/ns == GB/s, multiply by 1000 to get MB/s [not MiB/s]
0110     if (!min)
0111         min = 1;
0112     speed = (1000 * REPS * BENCH_SIZE) / (unsigned int)ktime_to_ns(min);
0113     tmpl->speed = speed;
0114
0115     pr_info("   %-16s: %5d MB/sec\n", tmpl->name, speed);
0116 }
0117
0118 static int __init
0119 calibrate_xor_blocks(void)
0120 {
0121     void *b1, *b2;
0122     struct xor_block_template *f, *fastest;
0123
0124     fastest = XOR_SELECT_TEMPLATE(NULL);
0125
0126     if (fastest) {
0127         printk(KERN_INFO "xor: automatically using best "
0128                  "checksumming function   %-10s\n",
0129                fastest->name);
0130         goto out;
0131     }
0132
0133     b1 = (void *) __get_free_pages(GFP_KERNEL, 2);
0134     if (!b1) {
0135         printk(KERN_WARNING "xor: Yikes!  No memory available.\n");
0136         return -ENOMEM;
0137     }
0138     b2 = b1 + 2*PAGE_SIZE + BENCH_SIZE;
0139
0140     /*
0141      * If this arch/cpu has a short-circuited selection, don't loop through
0142      * all the possible functions, just test the best one
0143      */
0144
0145 #define xor_speed(templ)    do_xor_speed((templ), b1, b2)
0146
0147     printk(KERN_INFO "xor: measuring software checksum speed\n");
0148     template_list = NULL;
0149     XOR_TRY_TEMPLATES;
0150     fastest = template_list;
0151     for (f = fastest; f; f = f->next)
0152         if (f->speed > fastest->speed)
0153             fastest = f;
0154
0155     pr_info("xor: using function: %s (%d MB/sec)\n",
0156            fastest->name, fastest->speed);
0157
0158 #undef xor_speed
0159
0160     free_pages((unsigned long)b1, 2);
0161 out:
0162     active_template = fastest;
0163     return 0;
0164 }
0165
0166 static __exit void xor_exit(void) { }
0167
0168 MODULE_LICENSE("GPL");
0169
0170 #ifndef MODULE
0171 /* when built-in xor.o must initialize before drivers/md/md.o */
0172 core_initcall(register_xor_blocks);
0173 #endif
0174
0175 module_init(calibrate_xor_blocks);
0176 module_exit(xor_exit);