lib/raid6/recov_neon.c

0001 // SPDX-License-Identifier: GPL-2.0-only
0002 /*
0003  * Copyright (C) 2012 Intel Corporation
0004  * Copyright (C) 2017 Linaro Ltd. <ard.biesheuvel@linaro.org>
0005  */
0006
0007 #include <linux/raid/pq.h>
0008
0009 #ifdef __KERNEL__
0010 #include <asm/neon.h>
0011 #else
0012 #define kernel_neon_begin()
0013 #define kernel_neon_end()
0014 #define cpu_has_neon()      (1)
0015 #endif
0016
0017 static int raid6_has_neon(void)
0018 {
0019     return cpu_has_neon();
0020 }
0021
0022 void __raid6_2data_recov_neon(int bytes, uint8_t *p, uint8_t *q, uint8_t *dp,
0023                   uint8_t *dq, const uint8_t *pbmul,
0024                   const uint8_t *qmul);
0025
0026 void __raid6_datap_recov_neon(int bytes, uint8_t *p, uint8_t *q, uint8_t *dq,
0027                   const uint8_t *qmul);
0028
0029 static void raid6_2data_recov_neon(int disks, size_t bytes, int faila,
0030         int failb, void **ptrs)
0031 {
0032     u8 *p, *q, *dp, *dq;
0033     const u8 *pbmul;    /* P multiplier table for B data */
0034     const u8 *qmul;     /* Q multiplier table (for both) */
0035
0036     p = (u8 *)ptrs[disks - 2];
0037     q = (u8 *)ptrs[disks - 1];
0038
0039     /*
0040      * Compute syndrome with zero for the missing data pages
0041      * Use the dead data pages as temporary storage for
0042      * delta p and delta q
0043      */
0044     dp = (u8 *)ptrs[faila];
0045     ptrs[faila] = (void *)raid6_empty_zero_page;
0046     ptrs[disks - 2] = dp;
0047     dq = (u8 *)ptrs[failb];
0048     ptrs[failb] = (void *)raid6_empty_zero_page;
0049     ptrs[disks - 1] = dq;
0050
0051     raid6_call.gen_syndrome(disks, bytes, ptrs);
0052
0053     /* Restore pointer table */
0054     ptrs[faila]     = dp;
0055     ptrs[failb]     = dq;
0056     ptrs[disks - 2] = p;
0057     ptrs[disks - 1] = q;
0058
0059     /* Now, pick the proper data tables */
0060     pbmul = raid6_vgfmul[raid6_gfexi[failb-faila]];
0061     qmul  = raid6_vgfmul[raid6_gfinv[raid6_gfexp[faila] ^
0062                      raid6_gfexp[failb]]];
0063
0064     kernel_neon_begin();
0065     __raid6_2data_recov_neon(bytes, p, q, dp, dq, pbmul, qmul);
0066     kernel_neon_end();
0067 }
0068
0069 static void raid6_datap_recov_neon(int disks, size_t bytes, int faila,
0070         void **ptrs)
0071 {
0072     u8 *p, *q, *dq;
0073     const u8 *qmul;     /* Q multiplier table */
0074
0075     p = (u8 *)ptrs[disks - 2];
0076     q = (u8 *)ptrs[disks - 1];
0077
0078     /*
0079      * Compute syndrome with zero for the missing data page
0080      * Use the dead data page as temporary storage for delta q
0081      */
0082     dq = (u8 *)ptrs[faila];
0083     ptrs[faila] = (void *)raid6_empty_zero_page;
0084     ptrs[disks - 1] = dq;
0085
0086     raid6_call.gen_syndrome(disks, bytes, ptrs);
0087
0088     /* Restore pointer table */
0089     ptrs[faila]     = dq;
0090     ptrs[disks - 1] = q;
0091
0092     /* Now, pick the proper data tables */
0093     qmul = raid6_vgfmul[raid6_gfinv[raid6_gfexp[faila]]];
0094
0095     kernel_neon_begin();
0096     __raid6_datap_recov_neon(bytes, p, q, dq, qmul);
0097     kernel_neon_end();
0098 }
0099
0100 const struct raid6_recov_calls raid6_recov_neon = {
0101     .data2      = raid6_2data_recov_neon,
0102     .datap      = raid6_datap_recov_neon,
0103     .valid      = raid6_has_neon,
0104     .name       = "neon",
0105     .priority   = 10,
0106 };