lib/raid6/algos.c

0001 // SPDX-License-Identifier: GPL-2.0-or-later
0002 /* -*- linux-c -*- ------------------------------------------------------- *
0003  *
0004  *   Copyright 2002 H. Peter Anvin - All Rights Reserved
0005  *
0006  * ----------------------------------------------------------------------- */
0007
0008 /*
0009  * raid6/algos.c
0010  *
0011  * Algorithm list and algorithm selection for RAID-6
0012  */
0013
0014 #include <linux/raid/pq.h>
0015 #ifndef __KERNEL__
0016 #include <sys/mman.h>
0017 #include <stdio.h>
0018 #else
0019 #include <linux/module.h>
0020 #include <linux/gfp.h>
0021 #if !RAID6_USE_EMPTY_ZERO_PAGE
0022 /* In .bss so it's zeroed */
0023 const char raid6_empty_zero_page[PAGE_SIZE] __attribute__((aligned(256)));
0024 EXPORT_SYMBOL(raid6_empty_zero_page);
0025 #endif
0026 #endif
0027
0028 struct raid6_calls raid6_call;
0029 EXPORT_SYMBOL_GPL(raid6_call);
0030
0031 const struct raid6_calls * const raid6_algos[] = {
0032 #if defined(__i386__) && !defined(__arch_um__)
0033 #ifdef CONFIG_AS_AVX512
0034     &raid6_avx512x2,
0035     &raid6_avx512x1,
0036 #endif
0037     &raid6_avx2x2,
0038     &raid6_avx2x1,
0039     &raid6_sse2x2,
0040     &raid6_sse2x1,
0041     &raid6_sse1x2,
0042     &raid6_sse1x1,
0043     &raid6_mmxx2,
0044     &raid6_mmxx1,
0045 #endif
0046 #if defined(__x86_64__) && !defined(__arch_um__)
0047 #ifdef CONFIG_AS_AVX512
0048     &raid6_avx512x4,
0049     &raid6_avx512x2,
0050     &raid6_avx512x1,
0051 #endif
0052     &raid6_avx2x4,
0053     &raid6_avx2x2,
0054     &raid6_avx2x1,
0055     &raid6_sse2x4,
0056     &raid6_sse2x2,
0057     &raid6_sse2x1,
0058 #endif
0059 #ifdef CONFIG_ALTIVEC
0060     &raid6_vpermxor8,
0061     &raid6_vpermxor4,
0062     &raid6_vpermxor2,
0063     &raid6_vpermxor1,
0064     &raid6_altivec8,
0065     &raid6_altivec4,
0066     &raid6_altivec2,
0067     &raid6_altivec1,
0068 #endif
0069 #if defined(CONFIG_S390)
0070     &raid6_s390vx8,
0071 #endif
0072 #ifdef CONFIG_KERNEL_MODE_NEON
0073     &raid6_neonx8,
0074     &raid6_neonx4,
0075     &raid6_neonx2,
0076     &raid6_neonx1,
0077 #endif
0078 #if defined(__ia64__)
0079     &raid6_intx32,
0080     &raid6_intx16,
0081 #endif
0082     &raid6_intx8,
0083     &raid6_intx4,
0084     &raid6_intx2,
0085     &raid6_intx1,
0086     NULL
0087 };
0088
0089 void (*raid6_2data_recov)(int, size_t, int, int, void **);
0090 EXPORT_SYMBOL_GPL(raid6_2data_recov);
0091
0092 void (*raid6_datap_recov)(int, size_t, int, void **);
0093 EXPORT_SYMBOL_GPL(raid6_datap_recov);
0094
0095 const struct raid6_recov_calls *const raid6_recov_algos[] = {
0096 #ifdef CONFIG_X86
0097 #ifdef CONFIG_AS_AVX512
0098     &raid6_recov_avx512,
0099 #endif
0100     &raid6_recov_avx2,
0101     &raid6_recov_ssse3,
0102 #endif
0103 #ifdef CONFIG_S390
0104     &raid6_recov_s390xc,
0105 #endif
0106 #if defined(CONFIG_KERNEL_MODE_NEON)
0107     &raid6_recov_neon,
0108 #endif
0109     &raid6_recov_intx1,
0110     NULL
0111 };
0112
0113 #ifdef __KERNEL__
0114 #define RAID6_TIME_JIFFIES_LG2  4
0115 #else
0116 /* Need more time to be stable in userspace */
0117 #define RAID6_TIME_JIFFIES_LG2  9
0118 #define time_before(x, y) ((x) < (y))
0119 #endif
0120
0121 #define RAID6_TEST_DISKS    8
0122 #define RAID6_TEST_DISKS_ORDER  3
0123
0124 static inline const struct raid6_recov_calls *raid6_choose_recov(void)
0125 {
0126     const struct raid6_recov_calls *const *algo;
0127     const struct raid6_recov_calls *best;
0128
0129     for (best = NULL, algo = raid6_recov_algos; *algo; algo++)
0130         if (!best || (*algo)->priority > best->priority)
0131             if (!(*algo)->valid || (*algo)->valid())
0132                 best = *algo;
0133
0134     if (best) {
0135         raid6_2data_recov = best->data2;
0136         raid6_datap_recov = best->datap;
0137
0138         pr_info("raid6: using %s recovery algorithm\n", best->name);
0139     } else
0140         pr_err("raid6: Yikes! No recovery algorithm found!\n");
0141
0142     return best;
0143 }
0144
0145 static inline const struct raid6_calls *raid6_choose_gen(
0146     void *(*const dptrs)[RAID6_TEST_DISKS], const int disks)
0147 {
0148     unsigned long perf, bestgenperf, j0, j1;
0149     int start = (disks>>1)-1, stop = disks-3;   /* work on the second half of the disks */
0150     const struct raid6_calls *const *algo;
0151     const struct raid6_calls *best;
0152
0153     for (bestgenperf = 0, best = NULL, algo = raid6_algos; *algo; algo++) {
0154         if (!best || (*algo)->priority >= best->priority) {
0155             if ((*algo)->valid && !(*algo)->valid())
0156                 continue;
0157
0158             if (!IS_ENABLED(CONFIG_RAID6_PQ_BENCHMARK)) {
0159                 best = *algo;
0160                 break;
0161             }
0162
0163             perf = 0;
0164
0165             preempt_disable();
0166             j0 = jiffies;
0167             while ((j1 = jiffies) == j0)
0168                 cpu_relax();
0169             while (time_before(jiffies,
0170                         j1 + (1<<RAID6_TIME_JIFFIES_LG2))) {
0171                 (*algo)->gen_syndrome(disks, PAGE_SIZE, *dptrs);
0172                 perf++;
0173             }
0174             preempt_enable();
0175
0176             if (perf > bestgenperf) {
0177                 bestgenperf = perf;
0178                 best = *algo;
0179             }
0180             pr_info("raid6: %-8s gen() %5ld MB/s\n", (*algo)->name,
0181                 (perf * HZ * (disks-2)) >>
0182                 (20 - PAGE_SHIFT + RAID6_TIME_JIFFIES_LG2));
0183         }
0184     }
0185
0186     if (!best) {
0187         pr_err("raid6: Yikes! No algorithm found!\n");
0188         goto out;
0189     }
0190
0191     raid6_call = *best;
0192
0193     if (!IS_ENABLED(CONFIG_RAID6_PQ_BENCHMARK)) {
0194         pr_info("raid6: skipped pq benchmark and selected %s\n",
0195             best->name);
0196         goto out;
0197     }
0198
0199     pr_info("raid6: using algorithm %s gen() %ld MB/s\n",
0200         best->name,
0201         (bestgenperf * HZ * (disks - 2)) >>
0202         (20 - PAGE_SHIFT + RAID6_TIME_JIFFIES_LG2));
0203
0204     if (best->xor_syndrome) {
0205         perf = 0;
0206
0207         preempt_disable();
0208         j0 = jiffies;
0209         while ((j1 = jiffies) == j0)
0210             cpu_relax();
0211         while (time_before(jiffies,
0212                    j1 + (1 << RAID6_TIME_JIFFIES_LG2))) {
0213             best->xor_syndrome(disks, start, stop,
0214                        PAGE_SIZE, *dptrs);
0215             perf++;
0216         }
0217         preempt_enable();
0218
0219         pr_info("raid6: .... xor() %ld MB/s, rmw enabled\n",
0220             (perf * HZ * (disks - 2)) >>
0221             (20 - PAGE_SHIFT + RAID6_TIME_JIFFIES_LG2 + 1));
0222     }
0223
0224 out:
0225     return best;
0226 }
0227
0228
0229 /* Try to pick the best algorithm */
0230 /* This code uses the gfmul table as convenient data set to abuse */
0231
0232 int __init raid6_select_algo(void)
0233 {
0234     const int disks = RAID6_TEST_DISKS;
0235
0236     const struct raid6_calls *gen_best;
0237     const struct raid6_recov_calls *rec_best;
0238     char *disk_ptr, *p;
0239     void *dptrs[RAID6_TEST_DISKS];
0240     int i, cycle;
0241
0242     /* prepare the buffer and fill it circularly with gfmul table */
0243     disk_ptr = (char *)__get_free_pages(GFP_KERNEL, RAID6_TEST_DISKS_ORDER);
0244     if (!disk_ptr) {
0245         pr_err("raid6: Yikes!  No memory available.\n");
0246         return -ENOMEM;
0247     }
0248
0249     p = disk_ptr;
0250     for (i = 0; i < disks; i++)
0251         dptrs[i] = p + PAGE_SIZE * i;
0252
0253     cycle = ((disks - 2) * PAGE_SIZE) / 65536;
0254     for (i = 0; i < cycle; i++) {
0255         memcpy(p, raid6_gfmul, 65536);
0256         p += 65536;
0257     }
0258
0259     if ((disks - 2) * PAGE_SIZE % 65536)
0260         memcpy(p, raid6_gfmul, (disks - 2) * PAGE_SIZE % 65536);
0261
0262     /* select raid gen_syndrome function */
0263     gen_best = raid6_choose_gen(&dptrs, disks);
0264
0265     /* select raid recover functions */
0266     rec_best = raid6_choose_recov();
0267
0268     free_pages((unsigned long)disk_ptr, RAID6_TEST_DISKS_ORDER);
0269
0270     return gen_best && rec_best ? 0 : -EINVAL;
0271 }
0272
0273 static void raid6_exit(void)
0274 {
0275     do { } while (0);
0276 }
0277
0278 subsys_initcall(raid6_select_algo);
0279 module_exit(raid6_exit);
0280 MODULE_LICENSE("GPL");
0281 MODULE_DESCRIPTION("RAID6 Q-syndrome calculations");