lib/raid6/s390vx.uc

0001 // SPDX-License-Identifier: GPL-2.0
0002 /*
0003  * raid6_vx$#.c
0004  *
0005  * $#-way unrolled RAID6 gen/xor functions for s390
0006  * based on the vector facility
0007  *
0008  * Copyright IBM Corp. 2016
0009  * Author(s): Martin Schwidefsky <schwidefsky@de.ibm.com>
0010  *
0011  * This file is postprocessed using unroll.awk.
0012  */
0013
0014 #include <linux/raid/pq.h>
0015 #include <asm/fpu/api.h>
0016
0017 asm(".include \"asm/vx-insn.h\"\n");
0018
0019 #define NSIZE 16
0020
0021 static inline void LOAD_CONST(void)
0022 {
0023         asm volatile("VREPIB %v24,7");
0024         asm volatile("VREPIB %v25,0x1d");
0025 }
0026
0027 /*
0028  * The SHLBYTE() operation shifts each of the 16 bytes in
0029  * vector register y left by 1 bit and stores the result in
0030  * vector register x.
0031  */
0032 static inline void SHLBYTE(int x, int y)
0033 {
0034         asm volatile ("VAB %0,%1,%1" : : "i" (x), "i" (y));
0035 }
0036
0037 /*
0038  * For each of the 16 bytes in the vector register y the MASK()
0039  * operation returns 0xFF if the high bit of the byte is 1,
0040  * or 0x00 if the high bit is 0. The result is stored in vector
0041  * register x.
0042  */
0043 static inline void MASK(int x, int y)
0044 {
0045         asm volatile ("VESRAVB  %0,%1,24" : : "i" (x), "i" (y));
0046 }
0047
0048 static inline void AND(int x, int y, int z)
0049 {
0050         asm volatile ("VN %0,%1,%2" : : "i" (x), "i" (y), "i" (z));
0051 }
0052
0053 static inline void XOR(int x, int y, int z)
0054 {
0055         asm volatile ("VX %0,%1,%2" : : "i" (x), "i" (y), "i" (z));
0056 }
0057
0058 static inline void LOAD_DATA(int x, u8 *ptr)
0059 {
0060         typedef struct { u8 _[16 * $#]; } addrtype;
0061         register addrtype *__ptr asm("1") = (addrtype *) ptr;
0062
0063         asm volatile ("VLM %2,%3,0,%1"
0064                       : : "m" (*__ptr), "a" (__ptr), "i" (x),
0065                           "i" (x + $# - 1));
0066 }
0067
0068 static inline void STORE_DATA(int x, u8 *ptr)
0069 {
0070         typedef struct { u8 _[16 * $#]; } addrtype;
0071         register addrtype *__ptr asm("1") = (addrtype *) ptr;
0072
0073         asm volatile ("VSTM %2,%3,0,1"
0074                       : "=m" (*__ptr) : "a" (__ptr), "i" (x),
0075                         "i" (x + $# - 1));
0076 }
0077
0078 static inline void COPY_VEC(int x, int y)
0079 {
0080         asm volatile ("VLR %0,%1" : : "i" (x), "i" (y));
0081 }
0082
0083 static void raid6_s390vx$#_gen_syndrome(int disks, size_t bytes, void **ptrs)
0084 {
0085         struct kernel_fpu vxstate;
0086         u8 **dptr, *p, *q;
0087         int d, z, z0;
0088
0089         kernel_fpu_begin(&vxstate, KERNEL_VXR);
0090         LOAD_CONST();
0091
0092         dptr = (u8 **) ptrs;
0093         z0 = disks - 3;         /* Highest data disk */
0094         p = dptr[z0 + 1];       /* XOR parity */
0095         q = dptr[z0 + 2];       /* RS syndrome */
0096
0097         for (d = 0; d < bytes; d += $#*NSIZE) {
0098                 LOAD_DATA(0,&dptr[z0][d]);
0099                 COPY_VEC(8+$$,0+$$);
0100                 for (z = z0 - 1; z >= 0; z--) {
0101                         MASK(16+$$,8+$$);
0102                         AND(16+$$,16+$$,25);
0103                         SHLBYTE(8+$$,8+$$);
0104                         XOR(8+$$,8+$$,16+$$);
0105                         LOAD_DATA(16,&dptr[z][d]);
0106                         XOR(0+$$,0+$$,16+$$);
0107                         XOR(8+$$,8+$$,16+$$);
0108                 }
0109                 STORE_DATA(0,&p[d]);
0110                 STORE_DATA(8,&q[d]);
0111         }
0112         kernel_fpu_end(&vxstate, KERNEL_VXR);
0113 }
0114
0115 static void raid6_s390vx$#_xor_syndrome(int disks, int start, int stop,
0116                                         size_t bytes, void **ptrs)
0117 {
0118         struct kernel_fpu vxstate;
0119         u8 **dptr, *p, *q;
0120         int d, z, z0;
0121
0122         dptr = (u8 **) ptrs;
0123         z0 = stop;              /* P/Q right side optimization */
0124         p = dptr[disks - 2];    /* XOR parity */
0125         q = dptr[disks - 1];    /* RS syndrome */
0126
0127         kernel_fpu_begin(&vxstate, KERNEL_VXR);
0128         LOAD_CONST();
0129
0130         for (d = 0; d < bytes; d += $#*NSIZE) {
0131                 /* P/Q data pages */
0132                 LOAD_DATA(0,&dptr[z0][d]);
0133                 COPY_VEC(8+$$,0+$$);
0134                 for (z = z0 - 1; z >= start; z--) {
0135                         MASK(16+$$,8+$$);
0136                         AND(16+$$,16+$$,25);
0137                         SHLBYTE(8+$$,8+$$);
0138                         XOR(8+$$,8+$$,16+$$);
0139                         LOAD_DATA(16,&dptr[z][d]);
0140                         XOR(0+$$,0+$$,16+$$);
0141                         XOR(8+$$,8+$$,16+$$);
0142                 }
0143                 /* P/Q left side optimization */
0144                 for (z = start - 1; z >= 0; z--) {
0145                         MASK(16+$$,8+$$);
0146                         AND(16+$$,16+$$,25);
0147                         SHLBYTE(8+$$,8+$$);
0148                         XOR(8+$$,8+$$,16+$$);
0149                 }
0150                 LOAD_DATA(16,&p[d]);
0151                 XOR(16+$$,16+$$,0+$$);
0152                 STORE_DATA(16,&p[d]);
0153                 LOAD_DATA(16,&q[d]);
0154                 XOR(16+$$,16+$$,8+$$);
0155                 STORE_DATA(16,&q[d]);
0156         }
0157         kernel_fpu_end(&vxstate, KERNEL_VXR);
0158 }
0159
0160 static int raid6_s390vx$#_valid(void)
0161 {
0162         return MACHINE_HAS_VX;
0163 }
0164
0165 const struct raid6_calls raid6_s390vx$# = {
0166         raid6_s390vx$#_gen_syndrome,
0167         raid6_s390vx$#_xor_syndrome,
0168         raid6_s390vx$#_valid,
0169         "vx128x$#",
0170         1
0171 };