lib/raid6/neon.c

0001 // SPDX-License-Identifier: GPL-2.0-only
0002 /*
0003  * linux/lib/raid6/neon.c - RAID6 syndrome calculation using ARM NEON intrinsics
0004  *
0005  * Copyright (C) 2013 Linaro Ltd <ard.biesheuvel@linaro.org>
0006  */
0007
0008 #include <linux/raid/pq.h>
0009
0010 #ifdef __KERNEL__
0011 #include <asm/neon.h>
0012 #else
0013 #define kernel_neon_begin()
0014 #define kernel_neon_end()
0015 #define cpu_has_neon()      (1)
0016 #endif
0017
0018 /*
0019  * There are 2 reasons these wrappers are kept in a separate compilation unit
0020  * from the actual implementations in neonN.c (generated from neon.uc by
0021  * unroll.awk):
0022  * - the actual implementations use NEON intrinsics, and the GCC support header
0023  *   (arm_neon.h) is not fully compatible (type wise) with the kernel;
0024  * - the neonN.c files are compiled with -mfpu=neon and optimization enabled,
0025  *   and we have to make sure that we never use *any* NEON/VFP instructions
0026  *   outside a kernel_neon_begin()/kernel_neon_end() pair.
0027  */
0028
0029 #define RAID6_NEON_WRAPPER(_n)                      \
0030     static void raid6_neon ## _n ## _gen_syndrome(int disks,    \
0031                     size_t bytes, void **ptrs)  \
0032     {                               \
0033         void raid6_neon ## _n  ## _gen_syndrome_real(int,   \
0034                         unsigned long, void**); \
0035         kernel_neon_begin();                    \
0036         raid6_neon ## _n ## _gen_syndrome_real(disks,       \
0037                     (unsigned long)bytes, ptrs);    \
0038         kernel_neon_end();                  \
0039     }                               \
0040     static void raid6_neon ## _n ## _xor_syndrome(int disks,    \
0041                     int start, int stop,        \
0042                     size_t bytes, void **ptrs)  \
0043     {                               \
0044         void raid6_neon ## _n  ## _xor_syndrome_real(int,   \
0045                 int, int, unsigned long, void**);   \
0046         kernel_neon_begin();                    \
0047         raid6_neon ## _n ## _xor_syndrome_real(disks,       \
0048             start, stop, (unsigned long)bytes, ptrs);   \
0049         kernel_neon_end();                  \
0050     }                               \
0051     struct raid6_calls const raid6_neonx ## _n = {          \
0052         raid6_neon ## _n ## _gen_syndrome,          \
0053         raid6_neon ## _n ## _xor_syndrome,          \
0054         raid6_have_neon,                    \
0055         "neonx" #_n,                        \
0056         0                           \
0057     }
0058
0059 static int raid6_have_neon(void)
0060 {
0061     return cpu_has_neon();
0062 }
0063
0064 RAID6_NEON_WRAPPER(1);
0065 RAID6_NEON_WRAPPER(2);
0066 RAID6_NEON_WRAPPER(4);
0067 RAID6_NEON_WRAPPER(8);