s390/lib/xor.c

0001 // SPDX-License-Identifier: GPL-2.0
0002 /*
0003  * Optimized xor_block operation for RAID4/5
0004  *
0005  * Copyright IBM Corp. 2016
0006  * Author(s): Martin Schwidefsky <schwidefsky@de.ibm.com>
0007  */
0008
0009 #include <linux/types.h>
0010 #include <linux/export.h>
0011 #include <linux/raid/xor.h>
0012 #include <asm/xor.h>
0013
0014 static void xor_xc_2(unsigned long bytes, unsigned long * __restrict p1,
0015              const unsigned long * __restrict p2)
0016 {
0017     asm volatile(
0018         "   larl    1,2f\n"
0019         "   aghi    %0,-1\n"
0020         "   jm  3f\n"
0021         "   srlg    0,%0,8\n"
0022         "   ltgr    0,0\n"
0023         "   jz  1f\n"
0024         "0: xc  0(256,%1),0(%2)\n"
0025         "   la  %1,256(%1)\n"
0026         "   la  %2,256(%2)\n"
0027         "   brctg   0,0b\n"
0028         "1: ex  %0,0(1)\n"
0029         "   j   3f\n"
0030         "2: xc  0(1,%1),0(%2)\n"
0031         "3:\n"
0032         : : "d" (bytes), "a" (p1), "a" (p2)
0033         : "0", "1", "cc", "memory");
0034 }
0035
0036 static void xor_xc_3(unsigned long bytes, unsigned long * __restrict p1,
0037              const unsigned long * __restrict p2,
0038              const unsigned long * __restrict p3)
0039 {
0040     asm volatile(
0041         "   larl    1,2f\n"
0042         "   aghi    %0,-1\n"
0043         "   jm  3f\n"
0044         "   srlg    0,%0,8\n"
0045         "   ltgr    0,0\n"
0046         "   jz  1f\n"
0047         "0: xc  0(256,%1),0(%2)\n"
0048         "   xc  0(256,%1),0(%3)\n"
0049         "   la  %1,256(%1)\n"
0050         "   la  %2,256(%2)\n"
0051         "   la  %3,256(%3)\n"
0052         "   brctg   0,0b\n"
0053         "1: ex  %0,0(1)\n"
0054         "   ex  %0,6(1)\n"
0055         "   j   3f\n"
0056         "2: xc  0(1,%1),0(%2)\n"
0057         "   xc  0(1,%1),0(%3)\n"
0058         "3:\n"
0059         : "+d" (bytes), "+a" (p1), "+a" (p2), "+a" (p3)
0060         : : "0", "1", "cc", "memory");
0061 }
0062
0063 static void xor_xc_4(unsigned long bytes, unsigned long * __restrict p1,
0064              const unsigned long * __restrict p2,
0065              const unsigned long * __restrict p3,
0066              const unsigned long * __restrict p4)
0067 {
0068     asm volatile(
0069         "   larl    1,2f\n"
0070         "   aghi    %0,-1\n"
0071         "   jm  3f\n"
0072         "   srlg    0,%0,8\n"
0073         "   ltgr    0,0\n"
0074         "   jz  1f\n"
0075         "0: xc  0(256,%1),0(%2)\n"
0076         "   xc  0(256,%1),0(%3)\n"
0077         "   xc  0(256,%1),0(%4)\n"
0078         "   la  %1,256(%1)\n"
0079         "   la  %2,256(%2)\n"
0080         "   la  %3,256(%3)\n"
0081         "   la  %4,256(%4)\n"
0082         "   brctg   0,0b\n"
0083         "1: ex  %0,0(1)\n"
0084         "   ex  %0,6(1)\n"
0085         "   ex  %0,12(1)\n"
0086         "   j   3f\n"
0087         "2: xc  0(1,%1),0(%2)\n"
0088         "   xc  0(1,%1),0(%3)\n"
0089         "   xc  0(1,%1),0(%4)\n"
0090         "3:\n"
0091         : "+d" (bytes), "+a" (p1), "+a" (p2), "+a" (p3), "+a" (p4)
0092         : : "0", "1", "cc", "memory");
0093 }
0094
0095 static void xor_xc_5(unsigned long bytes, unsigned long * __restrict p1,
0096              const unsigned long * __restrict p2,
0097              const unsigned long * __restrict p3,
0098              const unsigned long * __restrict p4,
0099              const unsigned long * __restrict p5)
0100 {
0101     asm volatile(
0102         "   larl    1,2f\n"
0103         "   aghi    %0,-1\n"
0104         "   jm  3f\n"
0105         "   srlg    0,%0,8\n"
0106         "   ltgr    0,0\n"
0107         "   jz  1f\n"
0108         "0: xc  0(256,%1),0(%2)\n"
0109         "   xc  0(256,%1),0(%3)\n"
0110         "   xc  0(256,%1),0(%4)\n"
0111         "   xc  0(256,%1),0(%5)\n"
0112         "   la  %1,256(%1)\n"
0113         "   la  %2,256(%2)\n"
0114         "   la  %3,256(%3)\n"
0115         "   la  %4,256(%4)\n"
0116         "   la  %5,256(%5)\n"
0117         "   brctg   0,0b\n"
0118         "1: ex  %0,0(1)\n"
0119         "   ex  %0,6(1)\n"
0120         "   ex  %0,12(1)\n"
0121         "   ex  %0,18(1)\n"
0122         "   j   3f\n"
0123         "2: xc  0(1,%1),0(%2)\n"
0124         "   xc  0(1,%1),0(%3)\n"
0125         "   xc  0(1,%1),0(%4)\n"
0126         "   xc  0(1,%1),0(%5)\n"
0127         "3:\n"
0128         : "+d" (bytes), "+a" (p1), "+a" (p2), "+a" (p3), "+a" (p4),
0129           "+a" (p5)
0130         : : "0", "1", "cc", "memory");
0131 }
0132
0133 struct xor_block_template xor_block_xc = {
0134     .name = "xc",
0135     .do_2 = xor_xc_2,
0136     .do_3 = xor_xc_3,
0137     .do_4 = xor_xc_4,
0138     .do_5 = xor_xc_5,
0139 };
0140 EXPORT_SYMBOL(xor_block_xc);