lib/raid6/recov_neon_inner.c

0001 // SPDX-License-Identifier: GPL-2.0-only
0002 /*
0003  * Copyright (C) 2012 Intel Corporation
0004  * Copyright (C) 2017 Linaro Ltd. <ard.biesheuvel@linaro.org>
0005  */
0006
0007 #include <arm_neon.h>
0008
0009 #ifdef CONFIG_ARM
0010 /*
0011  * AArch32 does not provide this intrinsic natively because it does not
0012  * implement the underlying instruction. AArch32 only provides a 64-bit
0013  * wide vtbl.8 instruction, so use that instead.
0014  */
0015 static uint8x16_t vqtbl1q_u8(uint8x16_t a, uint8x16_t b)
0016 {
0017     union {
0018         uint8x16_t  val;
0019         uint8x8x2_t pair;
0020     } __a = { a };
0021
0022     return vcombine_u8(vtbl2_u8(__a.pair, vget_low_u8(b)),
0023                vtbl2_u8(__a.pair, vget_high_u8(b)));
0024 }
0025 #endif
0026
0027 void __raid6_2data_recov_neon(int bytes, uint8_t *p, uint8_t *q, uint8_t *dp,
0028                   uint8_t *dq, const uint8_t *pbmul,
0029                   const uint8_t *qmul)
0030 {
0031     uint8x16_t pm0 = vld1q_u8(pbmul);
0032     uint8x16_t pm1 = vld1q_u8(pbmul + 16);
0033     uint8x16_t qm0 = vld1q_u8(qmul);
0034     uint8x16_t qm1 = vld1q_u8(qmul + 16);
0035     uint8x16_t x0f = vdupq_n_u8(0x0f);
0036
0037     /*
0038      * while ( bytes-- ) {
0039      *  uint8_t px, qx, db;
0040      *
0041      *  px    = *p ^ *dp;
0042      *  qx    = qmul[*q ^ *dq];
0043      *  *dq++ = db = pbmul[px] ^ qx;
0044      *  *dp++ = db ^ px;
0045      *  p++; q++;
0046      * }
0047      */
0048
0049     while (bytes) {
0050         uint8x16_t vx, vy, px, qx, db;
0051
0052         px = veorq_u8(vld1q_u8(p), vld1q_u8(dp));
0053         vx = veorq_u8(vld1q_u8(q), vld1q_u8(dq));
0054
0055         vy = vshrq_n_u8(vx, 4);
0056         vx = vqtbl1q_u8(qm0, vandq_u8(vx, x0f));
0057         vy = vqtbl1q_u8(qm1, vy);
0058         qx = veorq_u8(vx, vy);
0059
0060         vy = vshrq_n_u8(px, 4);
0061         vx = vqtbl1q_u8(pm0, vandq_u8(px, x0f));
0062         vy = vqtbl1q_u8(pm1, vy);
0063         vx = veorq_u8(vx, vy);
0064         db = veorq_u8(vx, qx);
0065
0066         vst1q_u8(dq, db);
0067         vst1q_u8(dp, veorq_u8(db, px));
0068
0069         bytes -= 16;
0070         p += 16;
0071         q += 16;
0072         dp += 16;
0073         dq += 16;
0074     }
0075 }
0076
0077 void __raid6_datap_recov_neon(int bytes, uint8_t *p, uint8_t *q, uint8_t *dq,
0078                   const uint8_t *qmul)
0079 {
0080     uint8x16_t qm0 = vld1q_u8(qmul);
0081     uint8x16_t qm1 = vld1q_u8(qmul + 16);
0082     uint8x16_t x0f = vdupq_n_u8(0x0f);
0083
0084     /*
0085      * while (bytes--) {
0086      *  *p++ ^= *dq = qmul[*q ^ *dq];
0087      *  q++; dq++;
0088      * }
0089      */
0090
0091     while (bytes) {
0092         uint8x16_t vx, vy;
0093
0094         vx = veorq_u8(vld1q_u8(q), vld1q_u8(dq));
0095
0096         vy = vshrq_n_u8(vx, 4);
0097         vx = vqtbl1q_u8(qm0, vandq_u8(vx, x0f));
0098         vy = vqtbl1q_u8(qm1, vy);
0099         vx = veorq_u8(vx, vy);
0100         vy = veorq_u8(vx, vld1q_u8(p));
0101
0102         vst1q_u8(dq, vx);
0103         vst1q_u8(p, vy);
0104
0105         bytes -= 16;
0106         p += 16;
0107         q += 16;
0108         dq += 16;
0109     }
0110 }