include/asm-generic/xor.h

0001 /* SPDX-License-Identifier: GPL-2.0-or-later */
0002 /*
0003  * include/asm-generic/xor.h
0004  *
0005  * Generic optimized RAID-5 checksumming functions.
0006  */
0007
0008 #include <linux/prefetch.h>
0009
0010 static void
0011 xor_8regs_2(unsigned long bytes, unsigned long * __restrict p1,
0012         const unsigned long * __restrict p2)
0013 {
0014     long lines = bytes / (sizeof (long)) / 8;
0015
0016     do {
0017         p1[0] ^= p2[0];
0018         p1[1] ^= p2[1];
0019         p1[2] ^= p2[2];
0020         p1[3] ^= p2[3];
0021         p1[4] ^= p2[4];
0022         p1[5] ^= p2[5];
0023         p1[6] ^= p2[6];
0024         p1[7] ^= p2[7];
0025         p1 += 8;
0026         p2 += 8;
0027     } while (--lines > 0);
0028 }
0029
0030 static void
0031 xor_8regs_3(unsigned long bytes, unsigned long * __restrict p1,
0032         const unsigned long * __restrict p2,
0033         const unsigned long * __restrict p3)
0034 {
0035     long lines = bytes / (sizeof (long)) / 8;
0036
0037     do {
0038         p1[0] ^= p2[0] ^ p3[0];
0039         p1[1] ^= p2[1] ^ p3[1];
0040         p1[2] ^= p2[2] ^ p3[2];
0041         p1[3] ^= p2[3] ^ p3[3];
0042         p1[4] ^= p2[4] ^ p3[4];
0043         p1[5] ^= p2[5] ^ p3[5];
0044         p1[6] ^= p2[6] ^ p3[6];
0045         p1[7] ^= p2[7] ^ p3[7];
0046         p1 += 8;
0047         p2 += 8;
0048         p3 += 8;
0049     } while (--lines > 0);
0050 }
0051
0052 static void
0053 xor_8regs_4(unsigned long bytes, unsigned long * __restrict p1,
0054         const unsigned long * __restrict p2,
0055         const unsigned long * __restrict p3,
0056         const unsigned long * __restrict p4)
0057 {
0058     long lines = bytes / (sizeof (long)) / 8;
0059
0060     do {
0061         p1[0] ^= p2[0] ^ p3[0] ^ p4[0];
0062         p1[1] ^= p2[1] ^ p3[1] ^ p4[1];
0063         p1[2] ^= p2[2] ^ p3[2] ^ p4[2];
0064         p1[3] ^= p2[3] ^ p3[3] ^ p4[3];
0065         p1[4] ^= p2[4] ^ p3[4] ^ p4[4];
0066         p1[5] ^= p2[5] ^ p3[5] ^ p4[5];
0067         p1[6] ^= p2[6] ^ p3[6] ^ p4[6];
0068         p1[7] ^= p2[7] ^ p3[7] ^ p4[7];
0069         p1 += 8;
0070         p2 += 8;
0071         p3 += 8;
0072         p4 += 8;
0073     } while (--lines > 0);
0074 }
0075
0076 static void
0077 xor_8regs_5(unsigned long bytes, unsigned long * __restrict p1,
0078         const unsigned long * __restrict p2,
0079         const unsigned long * __restrict p3,
0080         const unsigned long * __restrict p4,
0081         const unsigned long * __restrict p5)
0082 {
0083     long lines = bytes / (sizeof (long)) / 8;
0084
0085     do {
0086         p1[0] ^= p2[0] ^ p3[0] ^ p4[0] ^ p5[0];
0087         p1[1] ^= p2[1] ^ p3[1] ^ p4[1] ^ p5[1];
0088         p1[2] ^= p2[2] ^ p3[2] ^ p4[2] ^ p5[2];
0089         p1[3] ^= p2[3] ^ p3[3] ^ p4[3] ^ p5[3];
0090         p1[4] ^= p2[4] ^ p3[4] ^ p4[4] ^ p5[4];
0091         p1[5] ^= p2[5] ^ p3[5] ^ p4[5] ^ p5[5];
0092         p1[6] ^= p2[6] ^ p3[6] ^ p4[6] ^ p5[6];
0093         p1[7] ^= p2[7] ^ p3[7] ^ p4[7] ^ p5[7];
0094         p1 += 8;
0095         p2 += 8;
0096         p3 += 8;
0097         p4 += 8;
0098         p5 += 8;
0099     } while (--lines > 0);
0100 }
0101
0102 static void
0103 xor_32regs_2(unsigned long bytes, unsigned long * __restrict p1,
0104          const unsigned long * __restrict p2)
0105 {
0106     long lines = bytes / (sizeof (long)) / 8;
0107
0108     do {
0109         register long d0, d1, d2, d3, d4, d5, d6, d7;
0110         d0 = p1[0]; /* Pull the stuff into registers    */
0111         d1 = p1[1]; /*  ... in bursts, if possible.     */
0112         d2 = p1[2];
0113         d3 = p1[3];
0114         d4 = p1[4];
0115         d5 = p1[5];
0116         d6 = p1[6];
0117         d7 = p1[7];
0118         d0 ^= p2[0];
0119         d1 ^= p2[1];
0120         d2 ^= p2[2];
0121         d3 ^= p2[3];
0122         d4 ^= p2[4];
0123         d5 ^= p2[5];
0124         d6 ^= p2[6];
0125         d7 ^= p2[7];
0126         p1[0] = d0; /* Store the result (in bursts)     */
0127         p1[1] = d1;
0128         p1[2] = d2;
0129         p1[3] = d3;
0130         p1[4] = d4;
0131         p1[5] = d5;
0132         p1[6] = d6;
0133         p1[7] = d7;
0134         p1 += 8;
0135         p2 += 8;
0136     } while (--lines > 0);
0137 }
0138
0139 static void
0140 xor_32regs_3(unsigned long bytes, unsigned long * __restrict p1,
0141          const unsigned long * __restrict p2,
0142          const unsigned long * __restrict p3)
0143 {
0144     long lines = bytes / (sizeof (long)) / 8;
0145
0146     do {
0147         register long d0, d1, d2, d3, d4, d5, d6, d7;
0148         d0 = p1[0]; /* Pull the stuff into registers    */
0149         d1 = p1[1]; /*  ... in bursts, if possible.     */
0150         d2 = p1[2];
0151         d3 = p1[3];
0152         d4 = p1[4];
0153         d5 = p1[5];
0154         d6 = p1[6];
0155         d7 = p1[7];
0156         d0 ^= p2[0];
0157         d1 ^= p2[1];
0158         d2 ^= p2[2];
0159         d3 ^= p2[3];
0160         d4 ^= p2[4];
0161         d5 ^= p2[5];
0162         d6 ^= p2[6];
0163         d7 ^= p2[7];
0164         d0 ^= p3[0];
0165         d1 ^= p3[1];
0166         d2 ^= p3[2];
0167         d3 ^= p3[3];
0168         d4 ^= p3[4];
0169         d5 ^= p3[5];
0170         d6 ^= p3[6];
0171         d7 ^= p3[7];
0172         p1[0] = d0; /* Store the result (in bursts)     */
0173         p1[1] = d1;
0174         p1[2] = d2;
0175         p1[3] = d3;
0176         p1[4] = d4;
0177         p1[5] = d5;
0178         p1[6] = d6;
0179         p1[7] = d7;
0180         p1 += 8;
0181         p2 += 8;
0182         p3 += 8;
0183     } while (--lines > 0);
0184 }
0185
0186 static void
0187 xor_32regs_4(unsigned long bytes, unsigned long * __restrict p1,
0188          const unsigned long * __restrict p2,
0189          const unsigned long * __restrict p3,
0190          const unsigned long * __restrict p4)
0191 {
0192     long lines = bytes / (sizeof (long)) / 8;
0193
0194     do {
0195         register long d0, d1, d2, d3, d4, d5, d6, d7;
0196         d0 = p1[0]; /* Pull the stuff into registers    */
0197         d1 = p1[1]; /*  ... in bursts, if possible.     */
0198         d2 = p1[2];
0199         d3 = p1[3];
0200         d4 = p1[4];
0201         d5 = p1[5];
0202         d6 = p1[6];
0203         d7 = p1[7];
0204         d0 ^= p2[0];
0205         d1 ^= p2[1];
0206         d2 ^= p2[2];
0207         d3 ^= p2[3];
0208         d4 ^= p2[4];
0209         d5 ^= p2[5];
0210         d6 ^= p2[6];
0211         d7 ^= p2[7];
0212         d0 ^= p3[0];
0213         d1 ^= p3[1];
0214         d2 ^= p3[2];
0215         d3 ^= p3[3];
0216         d4 ^= p3[4];
0217         d5 ^= p3[5];
0218         d6 ^= p3[6];
0219         d7 ^= p3[7];
0220         d0 ^= p4[0];
0221         d1 ^= p4[1];
0222         d2 ^= p4[2];
0223         d3 ^= p4[3];
0224         d4 ^= p4[4];
0225         d5 ^= p4[5];
0226         d6 ^= p4[6];
0227         d7 ^= p4[7];
0228         p1[0] = d0; /* Store the result (in bursts)     */
0229         p1[1] = d1;
0230         p1[2] = d2;
0231         p1[3] = d3;
0232         p1[4] = d4;
0233         p1[5] = d5;
0234         p1[6] = d6;
0235         p1[7] = d7;
0236         p1 += 8;
0237         p2 += 8;
0238         p3 += 8;
0239         p4 += 8;
0240     } while (--lines > 0);
0241 }
0242
0243 static void
0244 xor_32regs_5(unsigned long bytes, unsigned long * __restrict p1,
0245          const unsigned long * __restrict p2,
0246          const unsigned long * __restrict p3,
0247          const unsigned long * __restrict p4,
0248          const unsigned long * __restrict p5)
0249 {
0250     long lines = bytes / (sizeof (long)) / 8;
0251
0252     do {
0253         register long d0, d1, d2, d3, d4, d5, d6, d7;
0254         d0 = p1[0]; /* Pull the stuff into registers    */
0255         d1 = p1[1]; /*  ... in bursts, if possible.     */
0256         d2 = p1[2];
0257         d3 = p1[3];
0258         d4 = p1[4];
0259         d5 = p1[5];
0260         d6 = p1[6];
0261         d7 = p1[7];
0262         d0 ^= p2[0];
0263         d1 ^= p2[1];
0264         d2 ^= p2[2];
0265         d3 ^= p2[3];
0266         d4 ^= p2[4];
0267         d5 ^= p2[5];
0268         d6 ^= p2[6];
0269         d7 ^= p2[7];
0270         d0 ^= p3[0];
0271         d1 ^= p3[1];
0272         d2 ^= p3[2];
0273         d3 ^= p3[3];
0274         d4 ^= p3[4];
0275         d5 ^= p3[5];
0276         d6 ^= p3[6];
0277         d7 ^= p3[7];
0278         d0 ^= p4[0];
0279         d1 ^= p4[1];
0280         d2 ^= p4[2];
0281         d3 ^= p4[3];
0282         d4 ^= p4[4];
0283         d5 ^= p4[5];
0284         d6 ^= p4[6];
0285         d7 ^= p4[7];
0286         d0 ^= p5[0];
0287         d1 ^= p5[1];
0288         d2 ^= p5[2];
0289         d3 ^= p5[3];
0290         d4 ^= p5[4];
0291         d5 ^= p5[5];
0292         d6 ^= p5[6];
0293         d7 ^= p5[7];
0294         p1[0] = d0; /* Store the result (in bursts)     */
0295         p1[1] = d1;
0296         p1[2] = d2;
0297         p1[3] = d3;
0298         p1[4] = d4;
0299         p1[5] = d5;
0300         p1[6] = d6;
0301         p1[7] = d7;
0302         p1 += 8;
0303         p2 += 8;
0304         p3 += 8;
0305         p4 += 8;
0306         p5 += 8;
0307     } while (--lines > 0);
0308 }
0309
0310 static void
0311 xor_8regs_p_2(unsigned long bytes, unsigned long * __restrict p1,
0312           const unsigned long * __restrict p2)
0313 {
0314     long lines = bytes / (sizeof (long)) / 8 - 1;
0315     prefetchw(p1);
0316     prefetch(p2);
0317
0318     do {
0319         prefetchw(p1+8);
0320         prefetch(p2+8);
0321  once_more:
0322         p1[0] ^= p2[0];
0323         p1[1] ^= p2[1];
0324         p1[2] ^= p2[2];
0325         p1[3] ^= p2[3];
0326         p1[4] ^= p2[4];
0327         p1[5] ^= p2[5];
0328         p1[6] ^= p2[6];
0329         p1[7] ^= p2[7];
0330         p1 += 8;
0331         p2 += 8;
0332     } while (--lines > 0);
0333     if (lines == 0)
0334         goto once_more;
0335 }
0336
0337 static void
0338 xor_8regs_p_3(unsigned long bytes, unsigned long * __restrict p1,
0339           const unsigned long * __restrict p2,
0340           const unsigned long * __restrict p3)
0341 {
0342     long lines = bytes / (sizeof (long)) / 8 - 1;
0343     prefetchw(p1);
0344     prefetch(p2);
0345     prefetch(p3);
0346
0347     do {
0348         prefetchw(p1+8);
0349         prefetch(p2+8);
0350         prefetch(p3+8);
0351  once_more:
0352         p1[0] ^= p2[0] ^ p3[0];
0353         p1[1] ^= p2[1] ^ p3[1];
0354         p1[2] ^= p2[2] ^ p3[2];
0355         p1[3] ^= p2[3] ^ p3[3];
0356         p1[4] ^= p2[4] ^ p3[4];
0357         p1[5] ^= p2[5] ^ p3[5];
0358         p1[6] ^= p2[6] ^ p3[6];
0359         p1[7] ^= p2[7] ^ p3[7];
0360         p1 += 8;
0361         p2 += 8;
0362         p3 += 8;
0363     } while (--lines > 0);
0364     if (lines == 0)
0365         goto once_more;
0366 }
0367
0368 static void
0369 xor_8regs_p_4(unsigned long bytes, unsigned long * __restrict p1,
0370           const unsigned long * __restrict p2,
0371           const unsigned long * __restrict p3,
0372           const unsigned long * __restrict p4)
0373 {
0374     long lines = bytes / (sizeof (long)) / 8 - 1;
0375
0376     prefetchw(p1);
0377     prefetch(p2);
0378     prefetch(p3);
0379     prefetch(p4);
0380
0381     do {
0382         prefetchw(p1+8);
0383         prefetch(p2+8);
0384         prefetch(p3+8);
0385         prefetch(p4+8);
0386  once_more:
0387         p1[0] ^= p2[0] ^ p3[0] ^ p4[0];
0388         p1[1] ^= p2[1] ^ p3[1] ^ p4[1];
0389         p1[2] ^= p2[2] ^ p3[2] ^ p4[2];
0390         p1[3] ^= p2[3] ^ p3[3] ^ p4[3];
0391         p1[4] ^= p2[4] ^ p3[4] ^ p4[4];
0392         p1[5] ^= p2[5] ^ p3[5] ^ p4[5];
0393         p1[6] ^= p2[6] ^ p3[6] ^ p4[6];
0394         p1[7] ^= p2[7] ^ p3[7] ^ p4[7];
0395         p1 += 8;
0396         p2 += 8;
0397         p3 += 8;
0398         p4 += 8;
0399     } while (--lines > 0);
0400     if (lines == 0)
0401         goto once_more;
0402 }
0403
0404 static void
0405 xor_8regs_p_5(unsigned long bytes, unsigned long * __restrict p1,
0406           const unsigned long * __restrict p2,
0407           const unsigned long * __restrict p3,
0408           const unsigned long * __restrict p4,
0409           const unsigned long * __restrict p5)
0410 {
0411     long lines = bytes / (sizeof (long)) / 8 - 1;
0412
0413     prefetchw(p1);
0414     prefetch(p2);
0415     prefetch(p3);
0416     prefetch(p4);
0417     prefetch(p5);
0418
0419     do {
0420         prefetchw(p1+8);
0421         prefetch(p2+8);
0422         prefetch(p3+8);
0423         prefetch(p4+8);
0424         prefetch(p5+8);
0425  once_more:
0426         p1[0] ^= p2[0] ^ p3[0] ^ p4[0] ^ p5[0];
0427         p1[1] ^= p2[1] ^ p3[1] ^ p4[1] ^ p5[1];
0428         p1[2] ^= p2[2] ^ p3[2] ^ p4[2] ^ p5[2];
0429         p1[3] ^= p2[3] ^ p3[3] ^ p4[3] ^ p5[3];
0430         p1[4] ^= p2[4] ^ p3[4] ^ p4[4] ^ p5[4];
0431         p1[5] ^= p2[5] ^ p3[5] ^ p4[5] ^ p5[5];
0432         p1[6] ^= p2[6] ^ p3[6] ^ p4[6] ^ p5[6];
0433         p1[7] ^= p2[7] ^ p3[7] ^ p4[7] ^ p5[7];
0434         p1 += 8;
0435         p2 += 8;
0436         p3 += 8;
0437         p4 += 8;
0438         p5 += 8;
0439     } while (--lines > 0);
0440     if (lines == 0)
0441         goto once_more;
0442 }
0443
0444 static void
0445 xor_32regs_p_2(unsigned long bytes, unsigned long * __restrict p1,
0446            const unsigned long * __restrict p2)
0447 {
0448     long lines = bytes / (sizeof (long)) / 8 - 1;
0449
0450     prefetchw(p1);
0451     prefetch(p2);
0452
0453     do {
0454         register long d0, d1, d2, d3, d4, d5, d6, d7;
0455
0456         prefetchw(p1+8);
0457         prefetch(p2+8);
0458  once_more:
0459         d0 = p1[0]; /* Pull the stuff into registers    */
0460         d1 = p1[1]; /*  ... in bursts, if possible.     */
0461         d2 = p1[2];
0462         d3 = p1[3];
0463         d4 = p1[4];
0464         d5 = p1[5];
0465         d6 = p1[6];
0466         d7 = p1[7];
0467         d0 ^= p2[0];
0468         d1 ^= p2[1];
0469         d2 ^= p2[2];
0470         d3 ^= p2[3];
0471         d4 ^= p2[4];
0472         d5 ^= p2[5];
0473         d6 ^= p2[6];
0474         d7 ^= p2[7];
0475         p1[0] = d0; /* Store the result (in bursts)     */
0476         p1[1] = d1;
0477         p1[2] = d2;
0478         p1[3] = d3;
0479         p1[4] = d4;
0480         p1[5] = d5;
0481         p1[6] = d6;
0482         p1[7] = d7;
0483         p1 += 8;
0484         p2 += 8;
0485     } while (--lines > 0);
0486     if (lines == 0)
0487         goto once_more;
0488 }
0489
0490 static void
0491 xor_32regs_p_3(unsigned long bytes, unsigned long * __restrict p1,
0492            const unsigned long * __restrict p2,
0493            const unsigned long * __restrict p3)
0494 {
0495     long lines = bytes / (sizeof (long)) / 8 - 1;
0496
0497     prefetchw(p1);
0498     prefetch(p2);
0499     prefetch(p3);
0500
0501     do {
0502         register long d0, d1, d2, d3, d4, d5, d6, d7;
0503
0504         prefetchw(p1+8);
0505         prefetch(p2+8);
0506         prefetch(p3+8);
0507  once_more:
0508         d0 = p1[0]; /* Pull the stuff into registers    */
0509         d1 = p1[1]; /*  ... in bursts, if possible.     */
0510         d2 = p1[2];
0511         d3 = p1[3];
0512         d4 = p1[4];
0513         d5 = p1[5];
0514         d6 = p1[6];
0515         d7 = p1[7];
0516         d0 ^= p2[0];
0517         d1 ^= p2[1];
0518         d2 ^= p2[2];
0519         d3 ^= p2[3];
0520         d4 ^= p2[4];
0521         d5 ^= p2[5];
0522         d6 ^= p2[6];
0523         d7 ^= p2[7];
0524         d0 ^= p3[0];
0525         d1 ^= p3[1];
0526         d2 ^= p3[2];
0527         d3 ^= p3[3];
0528         d4 ^= p3[4];
0529         d5 ^= p3[5];
0530         d6 ^= p3[6];
0531         d7 ^= p3[7];
0532         p1[0] = d0; /* Store the result (in bursts)     */
0533         p1[1] = d1;
0534         p1[2] = d2;
0535         p1[3] = d3;
0536         p1[4] = d4;
0537         p1[5] = d5;
0538         p1[6] = d6;
0539         p1[7] = d7;
0540         p1 += 8;
0541         p2 += 8;
0542         p3 += 8;
0543     } while (--lines > 0);
0544     if (lines == 0)
0545         goto once_more;
0546 }
0547
0548 static void
0549 xor_32regs_p_4(unsigned long bytes, unsigned long * __restrict p1,
0550            const unsigned long * __restrict p2,
0551            const unsigned long * __restrict p3,
0552            const unsigned long * __restrict p4)
0553 {
0554     long lines = bytes / (sizeof (long)) / 8 - 1;
0555
0556     prefetchw(p1);
0557     prefetch(p2);
0558     prefetch(p3);
0559     prefetch(p4);
0560
0561     do {
0562         register long d0, d1, d2, d3, d4, d5, d6, d7;
0563
0564         prefetchw(p1+8);
0565         prefetch(p2+8);
0566         prefetch(p3+8);
0567         prefetch(p4+8);
0568  once_more:
0569         d0 = p1[0]; /* Pull the stuff into registers    */
0570         d1 = p1[1]; /*  ... in bursts, if possible.     */
0571         d2 = p1[2];
0572         d3 = p1[3];
0573         d4 = p1[4];
0574         d5 = p1[5];
0575         d6 = p1[6];
0576         d7 = p1[7];
0577         d0 ^= p2[0];
0578         d1 ^= p2[1];
0579         d2 ^= p2[2];
0580         d3 ^= p2[3];
0581         d4 ^= p2[4];
0582         d5 ^= p2[5];
0583         d6 ^= p2[6];
0584         d7 ^= p2[7];
0585         d0 ^= p3[0];
0586         d1 ^= p3[1];
0587         d2 ^= p3[2];
0588         d3 ^= p3[3];
0589         d4 ^= p3[4];
0590         d5 ^= p3[5];
0591         d6 ^= p3[6];
0592         d7 ^= p3[7];
0593         d0 ^= p4[0];
0594         d1 ^= p4[1];
0595         d2 ^= p4[2];
0596         d3 ^= p4[3];
0597         d4 ^= p4[4];
0598         d5 ^= p4[5];
0599         d6 ^= p4[6];
0600         d7 ^= p4[7];
0601         p1[0] = d0; /* Store the result (in bursts)     */
0602         p1[1] = d1;
0603         p1[2] = d2;
0604         p1[3] = d3;
0605         p1[4] = d4;
0606         p1[5] = d5;
0607         p1[6] = d6;
0608         p1[7] = d7;
0609         p1 += 8;
0610         p2 += 8;
0611         p3 += 8;
0612         p4 += 8;
0613     } while (--lines > 0);
0614     if (lines == 0)
0615         goto once_more;
0616 }
0617
0618 static void
0619 xor_32regs_p_5(unsigned long bytes, unsigned long * __restrict p1,
0620            const unsigned long * __restrict p2,
0621            const unsigned long * __restrict p3,
0622            const unsigned long * __restrict p4,
0623            const unsigned long * __restrict p5)
0624 {
0625     long lines = bytes / (sizeof (long)) / 8 - 1;
0626
0627     prefetchw(p1);
0628     prefetch(p2);
0629     prefetch(p3);
0630     prefetch(p4);
0631     prefetch(p5);
0632
0633     do {
0634         register long d0, d1, d2, d3, d4, d5, d6, d7;
0635
0636         prefetchw(p1+8);
0637         prefetch(p2+8);
0638         prefetch(p3+8);
0639         prefetch(p4+8);
0640         prefetch(p5+8);
0641  once_more:
0642         d0 = p1[0]; /* Pull the stuff into registers    */
0643         d1 = p1[1]; /*  ... in bursts, if possible.     */
0644         d2 = p1[2];
0645         d3 = p1[3];
0646         d4 = p1[4];
0647         d5 = p1[5];
0648         d6 = p1[6];
0649         d7 = p1[7];
0650         d0 ^= p2[0];
0651         d1 ^= p2[1];
0652         d2 ^= p2[2];
0653         d3 ^= p2[3];
0654         d4 ^= p2[4];
0655         d5 ^= p2[5];
0656         d6 ^= p2[6];
0657         d7 ^= p2[7];
0658         d0 ^= p3[0];
0659         d1 ^= p3[1];
0660         d2 ^= p3[2];
0661         d3 ^= p3[3];
0662         d4 ^= p3[4];
0663         d5 ^= p3[5];
0664         d6 ^= p3[6];
0665         d7 ^= p3[7];
0666         d0 ^= p4[0];
0667         d1 ^= p4[1];
0668         d2 ^= p4[2];
0669         d3 ^= p4[3];
0670         d4 ^= p4[4];
0671         d5 ^= p4[5];
0672         d6 ^= p4[6];
0673         d7 ^= p4[7];
0674         d0 ^= p5[0];
0675         d1 ^= p5[1];
0676         d2 ^= p5[2];
0677         d3 ^= p5[3];
0678         d4 ^= p5[4];
0679         d5 ^= p5[5];
0680         d6 ^= p5[6];
0681         d7 ^= p5[7];
0682         p1[0] = d0; /* Store the result (in bursts)     */
0683         p1[1] = d1;
0684         p1[2] = d2;
0685         p1[3] = d3;
0686         p1[4] = d4;
0687         p1[5] = d5;
0688         p1[6] = d6;
0689         p1[7] = d7;
0690         p1 += 8;
0691         p2 += 8;
0692         p3 += 8;
0693         p4 += 8;
0694         p5 += 8;
0695     } while (--lines > 0);
0696     if (lines == 0)
0697         goto once_more;
0698 }
0699
0700 static struct xor_block_template xor_block_8regs = {
0701     .name = "8regs",
0702     .do_2 = xor_8regs_2,
0703     .do_3 = xor_8regs_3,
0704     .do_4 = xor_8regs_4,
0705     .do_5 = xor_8regs_5,
0706 };
0707
0708 static struct xor_block_template xor_block_32regs = {
0709     .name = "32regs",
0710     .do_2 = xor_32regs_2,
0711     .do_3 = xor_32regs_3,
0712     .do_4 = xor_32regs_4,
0713     .do_5 = xor_32regs_5,
0714 };
0715
0716 static struct xor_block_template xor_block_8regs_p __maybe_unused = {
0717     .name = "8regs_prefetch",
0718     .do_2 = xor_8regs_p_2,
0719     .do_3 = xor_8regs_p_3,
0720     .do_4 = xor_8regs_p_4,
0721     .do_5 = xor_8regs_p_5,
0722 };
0723
0724 static struct xor_block_template xor_block_32regs_p __maybe_unused = {
0725     .name = "32regs_prefetch",
0726     .do_2 = xor_32regs_p_2,
0727     .do_3 = xor_32regs_p_3,
0728     .do_4 = xor_32regs_p_4,
0729     .do_5 = xor_32regs_p_5,
0730 };
0731
0732 #define XOR_TRY_TEMPLATES           \
0733     do {                    \
0734         xor_speed(&xor_block_8regs);    \
0735         xor_speed(&xor_block_8regs_p);  \
0736         xor_speed(&xor_block_32regs);   \
0737         xor_speed(&xor_block_32regs_p); \
0738     } while (0)