x86/crypto/poly1305_glue.c

0001 // SPDX-License-Identifier: GPL-2.0 OR MIT
0002 /*
0003  * Copyright (C) 2015-2019 Jason A. Donenfeld <Jason@zx2c4.com>. All Rights Reserved.
0004  */
0005
0006 #include <crypto/algapi.h>
0007 #include <crypto/internal/hash.h>
0008 #include <crypto/internal/poly1305.h>
0009 #include <crypto/internal/simd.h>
0010 #include <linux/crypto.h>
0011 #include <linux/jump_label.h>
0012 #include <linux/kernel.h>
0013 #include <linux/module.h>
0014 #include <linux/sizes.h>
0015 #include <asm/intel-family.h>
0016 #include <asm/simd.h>
0017
0018 asmlinkage void poly1305_init_x86_64(void *ctx,
0019                      const u8 key[POLY1305_BLOCK_SIZE]);
0020 asmlinkage void poly1305_blocks_x86_64(void *ctx, const u8 *inp,
0021                        const size_t len, const u32 padbit);
0022 asmlinkage void poly1305_emit_x86_64(void *ctx, u8 mac[POLY1305_DIGEST_SIZE],
0023                      const u32 nonce[4]);
0024 asmlinkage void poly1305_emit_avx(void *ctx, u8 mac[POLY1305_DIGEST_SIZE],
0025                   const u32 nonce[4]);
0026 asmlinkage void poly1305_blocks_avx(void *ctx, const u8 *inp, const size_t len,
0027                     const u32 padbit);
0028 asmlinkage void poly1305_blocks_avx2(void *ctx, const u8 *inp, const size_t len,
0029                      const u32 padbit);
0030 asmlinkage void poly1305_blocks_avx512(void *ctx, const u8 *inp,
0031                        const size_t len, const u32 padbit);
0032
0033 static __ro_after_init DEFINE_STATIC_KEY_FALSE(poly1305_use_avx);
0034 static __ro_after_init DEFINE_STATIC_KEY_FALSE(poly1305_use_avx2);
0035 static __ro_after_init DEFINE_STATIC_KEY_FALSE(poly1305_use_avx512);
0036
0037 struct poly1305_arch_internal {
0038     union {
0039         struct {
0040             u32 h[5];
0041             u32 is_base2_26;
0042         };
0043         u64 hs[3];
0044     };
0045     u64 r[2];
0046     u64 pad;
0047     struct { u32 r2, r1, r4, r3; } rn[9];
0048 };
0049
0050 /* The AVX code uses base 2^26, while the scalar code uses base 2^64. If we hit
0051  * the unfortunate situation of using AVX and then having to go back to scalar
0052  * -- because the user is silly and has called the update function from two
0053  * separate contexts -- then we need to convert back to the original base before
0054  * proceeding. It is possible to reason that the initial reduction below is
0055  * sufficient given the implementation invariants. However, for an avoidance of
0056  * doubt and because this is not performance critical, we do the full reduction
0057  * anyway. Z3 proof of below function: https://xn--4db.cc/ltPtHCKN/py
0058  */
0059 static void convert_to_base2_64(void *ctx)
0060 {
0061     struct poly1305_arch_internal *state = ctx;
0062     u32 cy;
0063
0064     if (!state->is_base2_26)
0065         return;
0066
0067     cy = state->h[0] >> 26; state->h[0] &= 0x3ffffff; state->h[1] += cy;
0068     cy = state->h[1] >> 26; state->h[1] &= 0x3ffffff; state->h[2] += cy;
0069     cy = state->h[2] >> 26; state->h[2] &= 0x3ffffff; state->h[3] += cy;
0070     cy = state->h[3] >> 26; state->h[3] &= 0x3ffffff; state->h[4] += cy;
0071     state->hs[0] = ((u64)state->h[2] << 52) | ((u64)state->h[1] << 26) | state->h[0];
0072     state->hs[1] = ((u64)state->h[4] << 40) | ((u64)state->h[3] << 14) | (state->h[2] >> 12);
0073     state->hs[2] = state->h[4] >> 24;
0074 #define ULT(a, b) ((a ^ ((a ^ b) | ((a - b) ^ b))) >> (sizeof(a) * 8 - 1))
0075     cy = (state->hs[2] >> 2) + (state->hs[2] & ~3ULL);
0076     state->hs[2] &= 3;
0077     state->hs[0] += cy;
0078     state->hs[1] += (cy = ULT(state->hs[0], cy));
0079     state->hs[2] += ULT(state->hs[1], cy);
0080 #undef ULT
0081     state->is_base2_26 = 0;
0082 }
0083
0084 static void poly1305_simd_init(void *ctx, const u8 key[POLY1305_BLOCK_SIZE])
0085 {
0086     poly1305_init_x86_64(ctx, key);
0087 }
0088
0089 static void poly1305_simd_blocks(void *ctx, const u8 *inp, size_t len,
0090                  const u32 padbit)
0091 {
0092     struct poly1305_arch_internal *state = ctx;
0093
0094     /* SIMD disables preemption, so relax after processing each page. */
0095     BUILD_BUG_ON(SZ_4K < POLY1305_BLOCK_SIZE ||
0096              SZ_4K % POLY1305_BLOCK_SIZE);
0097
0098     if (!static_branch_likely(&poly1305_use_avx) ||
0099         (len < (POLY1305_BLOCK_SIZE * 18) && !state->is_base2_26) ||
0100         !crypto_simd_usable()) {
0101         convert_to_base2_64(ctx);
0102         poly1305_blocks_x86_64(ctx, inp, len, padbit);
0103         return;
0104     }
0105
0106     do {
0107         const size_t bytes = min_t(size_t, len, SZ_4K);
0108
0109         kernel_fpu_begin();
0110         if (IS_ENABLED(CONFIG_AS_AVX512) && static_branch_likely(&poly1305_use_avx512))
0111             poly1305_blocks_avx512(ctx, inp, bytes, padbit);
0112         else if (static_branch_likely(&poly1305_use_avx2))
0113             poly1305_blocks_avx2(ctx, inp, bytes, padbit);
0114         else
0115             poly1305_blocks_avx(ctx, inp, bytes, padbit);
0116         kernel_fpu_end();
0117
0118         len -= bytes;
0119         inp += bytes;
0120     } while (len);
0121 }
0122
0123 static void poly1305_simd_emit(void *ctx, u8 mac[POLY1305_DIGEST_SIZE],
0124                    const u32 nonce[4])
0125 {
0126     if (!static_branch_likely(&poly1305_use_avx))
0127         poly1305_emit_x86_64(ctx, mac, nonce);
0128     else
0129         poly1305_emit_avx(ctx, mac, nonce);
0130 }
0131
0132 void poly1305_init_arch(struct poly1305_desc_ctx *dctx, const u8 key[POLY1305_KEY_SIZE])
0133 {
0134     poly1305_simd_init(&dctx->h, key);
0135     dctx->s[0] = get_unaligned_le32(&key[16]);
0136     dctx->s[1] = get_unaligned_le32(&key[20]);
0137     dctx->s[2] = get_unaligned_le32(&key[24]);
0138     dctx->s[3] = get_unaligned_le32(&key[28]);
0139     dctx->buflen = 0;
0140     dctx->sset = true;
0141 }
0142 EXPORT_SYMBOL(poly1305_init_arch);
0143
0144 static unsigned int crypto_poly1305_setdctxkey(struct poly1305_desc_ctx *dctx,
0145                            const u8 *inp, unsigned int len)
0146 {
0147     unsigned int acc = 0;
0148     if (unlikely(!dctx->sset)) {
0149         if (!dctx->rset && len >= POLY1305_BLOCK_SIZE) {
0150             poly1305_simd_init(&dctx->h, inp);
0151             inp += POLY1305_BLOCK_SIZE;
0152             len -= POLY1305_BLOCK_SIZE;
0153             acc += POLY1305_BLOCK_SIZE;
0154             dctx->rset = 1;
0155         }
0156         if (len >= POLY1305_BLOCK_SIZE) {
0157             dctx->s[0] = get_unaligned_le32(&inp[0]);
0158             dctx->s[1] = get_unaligned_le32(&inp[4]);
0159             dctx->s[2] = get_unaligned_le32(&inp[8]);
0160             dctx->s[3] = get_unaligned_le32(&inp[12]);
0161             acc += POLY1305_BLOCK_SIZE;
0162             dctx->sset = true;
0163         }
0164     }
0165     return acc;
0166 }
0167
0168 void poly1305_update_arch(struct poly1305_desc_ctx *dctx, const u8 *src,
0169               unsigned int srclen)
0170 {
0171     unsigned int bytes, used;
0172
0173     if (unlikely(dctx->buflen)) {
0174         bytes = min(srclen, POLY1305_BLOCK_SIZE - dctx->buflen);
0175         memcpy(dctx->buf + dctx->buflen, src, bytes);
0176         src += bytes;
0177         srclen -= bytes;
0178         dctx->buflen += bytes;
0179
0180         if (dctx->buflen == POLY1305_BLOCK_SIZE) {
0181             if (likely(!crypto_poly1305_setdctxkey(dctx, dctx->buf, POLY1305_BLOCK_SIZE)))
0182                 poly1305_simd_blocks(&dctx->h, dctx->buf, POLY1305_BLOCK_SIZE, 1);
0183             dctx->buflen = 0;
0184         }
0185     }
0186
0187     if (likely(srclen >= POLY1305_BLOCK_SIZE)) {
0188         bytes = round_down(srclen, POLY1305_BLOCK_SIZE);
0189         srclen -= bytes;
0190         used = crypto_poly1305_setdctxkey(dctx, src, bytes);
0191         if (likely(bytes - used))
0192             poly1305_simd_blocks(&dctx->h, src + used, bytes - used, 1);
0193         src += bytes;
0194     }
0195
0196     if (unlikely(srclen)) {
0197         dctx->buflen = srclen;
0198         memcpy(dctx->buf, src, srclen);
0199     }
0200 }
0201 EXPORT_SYMBOL(poly1305_update_arch);
0202
0203 void poly1305_final_arch(struct poly1305_desc_ctx *dctx, u8 *dst)
0204 {
0205     if (unlikely(dctx->buflen)) {
0206         dctx->buf[dctx->buflen++] = 1;
0207         memset(dctx->buf + dctx->buflen, 0,
0208                POLY1305_BLOCK_SIZE - dctx->buflen);
0209         poly1305_simd_blocks(&dctx->h, dctx->buf, POLY1305_BLOCK_SIZE, 0);
0210     }
0211
0212     poly1305_simd_emit(&dctx->h, dst, dctx->s);
0213     memzero_explicit(dctx, sizeof(*dctx));
0214 }
0215 EXPORT_SYMBOL(poly1305_final_arch);
0216
0217 static int crypto_poly1305_init(struct shash_desc *desc)
0218 {
0219     struct poly1305_desc_ctx *dctx = shash_desc_ctx(desc);
0220
0221     *dctx = (struct poly1305_desc_ctx){};
0222     return 0;
0223 }
0224
0225 static int crypto_poly1305_update(struct shash_desc *desc,
0226                   const u8 *src, unsigned int srclen)
0227 {
0228     struct poly1305_desc_ctx *dctx = shash_desc_ctx(desc);
0229
0230     poly1305_update_arch(dctx, src, srclen);
0231     return 0;
0232 }
0233
0234 static int crypto_poly1305_final(struct shash_desc *desc, u8 *dst)
0235 {
0236     struct poly1305_desc_ctx *dctx = shash_desc_ctx(desc);
0237
0238     if (unlikely(!dctx->sset))
0239         return -ENOKEY;
0240
0241     poly1305_final_arch(dctx, dst);
0242     return 0;
0243 }
0244
0245 static struct shash_alg alg = {
0246     .digestsize = POLY1305_DIGEST_SIZE,
0247     .init       = crypto_poly1305_init,
0248     .update     = crypto_poly1305_update,
0249     .final      = crypto_poly1305_final,
0250     .descsize   = sizeof(struct poly1305_desc_ctx),
0251     .base       = {
0252         .cra_name       = "poly1305",
0253         .cra_driver_name    = "poly1305-simd",
0254         .cra_priority       = 300,
0255         .cra_blocksize      = POLY1305_BLOCK_SIZE,
0256         .cra_module     = THIS_MODULE,
0257     },
0258 };
0259
0260 static int __init poly1305_simd_mod_init(void)
0261 {
0262     if (boot_cpu_has(X86_FEATURE_AVX) &&
0263         cpu_has_xfeatures(XFEATURE_MASK_SSE | XFEATURE_MASK_YMM, NULL))
0264         static_branch_enable(&poly1305_use_avx);
0265     if (boot_cpu_has(X86_FEATURE_AVX) && boot_cpu_has(X86_FEATURE_AVX2) &&
0266         cpu_has_xfeatures(XFEATURE_MASK_SSE | XFEATURE_MASK_YMM, NULL))
0267         static_branch_enable(&poly1305_use_avx2);
0268     if (IS_ENABLED(CONFIG_AS_AVX512) && boot_cpu_has(X86_FEATURE_AVX) &&
0269         boot_cpu_has(X86_FEATURE_AVX2) && boot_cpu_has(X86_FEATURE_AVX512F) &&
0270         cpu_has_xfeatures(XFEATURE_MASK_SSE | XFEATURE_MASK_YMM | XFEATURE_MASK_AVX512, NULL) &&
0271         /* Skylake downclocks unacceptably much when using zmm, but later generations are fast. */
0272         boot_cpu_data.x86_model != INTEL_FAM6_SKYLAKE_X)
0273         static_branch_enable(&poly1305_use_avx512);
0274     return IS_REACHABLE(CONFIG_CRYPTO_HASH) ? crypto_register_shash(&alg) : 0;
0275 }
0276
0277 static void __exit poly1305_simd_mod_exit(void)
0278 {
0279     if (IS_REACHABLE(CONFIG_CRYPTO_HASH))
0280         crypto_unregister_shash(&alg);
0281 }
0282
0283 module_init(poly1305_simd_mod_init);
0284 module_exit(poly1305_simd_mod_exit);
0285
0286 MODULE_LICENSE("GPL");
0287 MODULE_AUTHOR("Jason A. Donenfeld <Jason@zx2c4.com>");
0288 MODULE_DESCRIPTION("Poly1305 authenticator");
0289 MODULE_ALIAS_CRYPTO("poly1305");
0290 MODULE_ALIAS_CRYPTO("poly1305-simd");