x86/crypto/nh-avx2-x86_64.S

0001 /* SPDX-License-Identifier: GPL-2.0 */
0002 /*
0003  * NH - ε-almost-universal hash function, x86_64 AVX2 accelerated
0004  *
0005  * Copyright 2018 Google LLC
0006  *
0007  * Author: Eric Biggers <ebiggers@google.com>
0008  */
0009
0010 #include <linux/linkage.h>
0011
0012 #define     PASS0_SUMS  %ymm0
0013 #define     PASS1_SUMS  %ymm1
0014 #define     PASS2_SUMS  %ymm2
0015 #define     PASS3_SUMS  %ymm3
0016 #define     K0      %ymm4
0017 #define     K0_XMM      %xmm4
0018 #define     K1      %ymm5
0019 #define     K1_XMM      %xmm5
0020 #define     K2      %ymm6
0021 #define     K2_XMM      %xmm6
0022 #define     K3      %ymm7
0023 #define     K3_XMM      %xmm7
0024 #define     T0      %ymm8
0025 #define     T1      %ymm9
0026 #define     T2      %ymm10
0027 #define     T2_XMM      %xmm10
0028 #define     T3      %ymm11
0029 #define     T3_XMM      %xmm11
0030 #define     T4      %ymm12
0031 #define     T5      %ymm13
0032 #define     T6      %ymm14
0033 #define     T7      %ymm15
0034 #define     KEY     %rdi
0035 #define     MESSAGE     %rsi
0036 #define     MESSAGE_LEN %rdx
0037 #define     HASH        %rcx
0038
0039 .macro _nh_2xstride k0, k1, k2, k3
0040
0041     // Add message words to key words
0042     vpaddd      \k0, T3, T0
0043     vpaddd      \k1, T3, T1
0044     vpaddd      \k2, T3, T2
0045     vpaddd      \k3, T3, T3
0046
0047     // Multiply 32x32 => 64 and accumulate
0048     vpshufd     $0x10, T0, T4
0049     vpshufd     $0x32, T0, T0
0050     vpshufd     $0x10, T1, T5
0051     vpshufd     $0x32, T1, T1
0052     vpshufd     $0x10, T2, T6
0053     vpshufd     $0x32, T2, T2
0054     vpshufd     $0x10, T3, T7
0055     vpshufd     $0x32, T3, T3
0056     vpmuludq    T4, T0, T0
0057     vpmuludq    T5, T1, T1
0058     vpmuludq    T6, T2, T2
0059     vpmuludq    T7, T3, T3
0060     vpaddq      T0, PASS0_SUMS, PASS0_SUMS
0061     vpaddq      T1, PASS1_SUMS, PASS1_SUMS
0062     vpaddq      T2, PASS2_SUMS, PASS2_SUMS
0063     vpaddq      T3, PASS3_SUMS, PASS3_SUMS
0064 .endm
0065
0066 /*
0067  * void nh_avx2(const u32 *key, const u8 *message, size_t message_len,
0068  *      u8 hash[NH_HASH_BYTES])
0069  *
0070  * It's guaranteed that message_len % 16 == 0.
0071  */
0072 SYM_FUNC_START(nh_avx2)
0073
0074     vmovdqu     0x00(KEY), K0
0075     vmovdqu     0x10(KEY), K1
0076     add     $0x20, KEY
0077     vpxor       PASS0_SUMS, PASS0_SUMS, PASS0_SUMS
0078     vpxor       PASS1_SUMS, PASS1_SUMS, PASS1_SUMS
0079     vpxor       PASS2_SUMS, PASS2_SUMS, PASS2_SUMS
0080     vpxor       PASS3_SUMS, PASS3_SUMS, PASS3_SUMS
0081
0082     sub     $0x40, MESSAGE_LEN
0083     jl      .Lloop4_done
0084 .Lloop4:
0085     vmovdqu     (MESSAGE), T3
0086     vmovdqu     0x00(KEY), K2
0087     vmovdqu     0x10(KEY), K3
0088     _nh_2xstride    K0, K1, K2, K3
0089
0090     vmovdqu     0x20(MESSAGE), T3
0091     vmovdqu     0x20(KEY), K0
0092     vmovdqu     0x30(KEY), K1
0093     _nh_2xstride    K2, K3, K0, K1
0094
0095     add     $0x40, MESSAGE
0096     add     $0x40, KEY
0097     sub     $0x40, MESSAGE_LEN
0098     jge     .Lloop4
0099
0100 .Lloop4_done:
0101     and     $0x3f, MESSAGE_LEN
0102     jz      .Ldone
0103
0104     cmp     $0x20, MESSAGE_LEN
0105     jl      .Llast
0106
0107     // 2 or 3 strides remain; do 2 more.
0108     vmovdqu     (MESSAGE), T3
0109     vmovdqu     0x00(KEY), K2
0110     vmovdqu     0x10(KEY), K3
0111     _nh_2xstride    K0, K1, K2, K3
0112     add     $0x20, MESSAGE
0113     add     $0x20, KEY
0114     sub     $0x20, MESSAGE_LEN
0115     jz      .Ldone
0116     vmovdqa     K2, K0
0117     vmovdqa     K3, K1
0118 .Llast:
0119     // Last stride.  Zero the high 128 bits of the message and keys so they
0120     // don't affect the result when processing them like 2 strides.
0121     vmovdqu     (MESSAGE), T3_XMM
0122     vmovdqa     K0_XMM, K0_XMM
0123     vmovdqa     K1_XMM, K1_XMM
0124     vmovdqu     0x00(KEY), K2_XMM
0125     vmovdqu     0x10(KEY), K3_XMM
0126     _nh_2xstride    K0, K1, K2, K3
0127
0128 .Ldone:
0129     // Sum the accumulators for each pass, then store the sums to 'hash'
0130
0131     // PASS0_SUMS is (0A 0B 0C 0D)
0132     // PASS1_SUMS is (1A 1B 1C 1D)
0133     // PASS2_SUMS is (2A 2B 2C 2D)
0134     // PASS3_SUMS is (3A 3B 3C 3D)
0135     // We need the horizontal sums:
0136     //     (0A + 0B + 0C + 0D,
0137     //  1A + 1B + 1C + 1D,
0138     //  2A + 2B + 2C + 2D,
0139     //  3A + 3B + 3C + 3D)
0140     //
0141
0142     vpunpcklqdq PASS1_SUMS, PASS0_SUMS, T0  // T0 = (0A 1A 0C 1C)
0143     vpunpckhqdq PASS1_SUMS, PASS0_SUMS, T1  // T1 = (0B 1B 0D 1D)
0144     vpunpcklqdq PASS3_SUMS, PASS2_SUMS, T2  // T2 = (2A 3A 2C 3C)
0145     vpunpckhqdq PASS3_SUMS, PASS2_SUMS, T3  // T3 = (2B 3B 2D 3D)
0146
0147     vinserti128 $0x1, T2_XMM, T0, T4        // T4 = (0A 1A 2A 3A)
0148     vinserti128 $0x1, T3_XMM, T1, T5        // T5 = (0B 1B 2B 3B)
0149     vperm2i128  $0x31, T2, T0, T0       // T0 = (0C 1C 2C 3C)
0150     vperm2i128  $0x31, T3, T1, T1       // T1 = (0D 1D 2D 3D)
0151
0152     vpaddq      T5, T4, T4
0153     vpaddq      T1, T0, T0
0154     vpaddq      T4, T0, T0
0155     vmovdqu     T0, (HASH)
0156     RET
0157 SYM_FUNC_END(nh_avx2)