x86/crypto/polyval-clmulni_glue.c

0001 // SPDX-License-Identifier: GPL-2.0-only
0002 /*
0003  * Glue code for POLYVAL using PCMULQDQ-NI
0004  *
0005  * Copyright (c) 2007 Nokia Siemens Networks - Mikko Herranen <mh1@iki.fi>
0006  * Copyright (c) 2009 Intel Corp.
0007  *   Author: Huang Ying <ying.huang@intel.com>
0008  * Copyright 2021 Google LLC
0009  */
0010
0011 /*
0012  * Glue code based on ghash-clmulni-intel_glue.c.
0013  *
0014  * This implementation of POLYVAL uses montgomery multiplication
0015  * accelerated by PCLMULQDQ-NI to implement the finite field
0016  * operations.
0017  */
0018
0019 #include <crypto/algapi.h>
0020 #include <crypto/internal/hash.h>
0021 #include <crypto/internal/simd.h>
0022 #include <crypto/polyval.h>
0023 #include <linux/crypto.h>
0024 #include <linux/init.h>
0025 #include <linux/kernel.h>
0026 #include <linux/module.h>
0027 #include <asm/cpu_device_id.h>
0028 #include <asm/simd.h>
0029
0030 #define NUM_KEY_POWERS  8
0031
0032 struct polyval_tfm_ctx {
0033     /*
0034      * These powers must be in the order h^8, ..., h^1.
0035      */
0036     u8 key_powers[NUM_KEY_POWERS][POLYVAL_BLOCK_SIZE];
0037 };
0038
0039 struct polyval_desc_ctx {
0040     u8 buffer[POLYVAL_BLOCK_SIZE];
0041     u32 bytes;
0042 };
0043
0044 asmlinkage void clmul_polyval_update(const struct polyval_tfm_ctx *keys,
0045     const u8 *in, size_t nblocks, u8 *accumulator);
0046 asmlinkage void clmul_polyval_mul(u8 *op1, const u8 *op2);
0047
0048 static void internal_polyval_update(const struct polyval_tfm_ctx *keys,
0049     const u8 *in, size_t nblocks, u8 *accumulator)
0050 {
0051     if (likely(crypto_simd_usable())) {
0052         kernel_fpu_begin();
0053         clmul_polyval_update(keys, in, nblocks, accumulator);
0054         kernel_fpu_end();
0055     } else {
0056         polyval_update_non4k(keys->key_powers[NUM_KEY_POWERS-1], in,
0057             nblocks, accumulator);
0058     }
0059 }
0060
0061 static void internal_polyval_mul(u8 *op1, const u8 *op2)
0062 {
0063     if (likely(crypto_simd_usable())) {
0064         kernel_fpu_begin();
0065         clmul_polyval_mul(op1, op2);
0066         kernel_fpu_end();
0067     } else {
0068         polyval_mul_non4k(op1, op2);
0069     }
0070 }
0071
0072 static int polyval_x86_setkey(struct crypto_shash *tfm,
0073             const u8 *key, unsigned int keylen)
0074 {
0075     struct polyval_tfm_ctx *tctx = crypto_shash_ctx(tfm);
0076     int i;
0077
0078     if (keylen != POLYVAL_BLOCK_SIZE)
0079         return -EINVAL;
0080
0081     memcpy(tctx->key_powers[NUM_KEY_POWERS-1], key, POLYVAL_BLOCK_SIZE);
0082
0083     for (i = NUM_KEY_POWERS-2; i >= 0; i--) {
0084         memcpy(tctx->key_powers[i], key, POLYVAL_BLOCK_SIZE);
0085         internal_polyval_mul(tctx->key_powers[i],
0086                      tctx->key_powers[i+1]);
0087     }
0088
0089     return 0;
0090 }
0091
0092 static int polyval_x86_init(struct shash_desc *desc)
0093 {
0094     struct polyval_desc_ctx *dctx = shash_desc_ctx(desc);
0095
0096     memset(dctx, 0, sizeof(*dctx));
0097
0098     return 0;
0099 }
0100
0101 static int polyval_x86_update(struct shash_desc *desc,
0102              const u8 *src, unsigned int srclen)
0103 {
0104     struct polyval_desc_ctx *dctx = shash_desc_ctx(desc);
0105     const struct polyval_tfm_ctx *tctx = crypto_shash_ctx(desc->tfm);
0106     u8 *pos;
0107     unsigned int nblocks;
0108     unsigned int n;
0109
0110     if (dctx->bytes) {
0111         n = min(srclen, dctx->bytes);
0112         pos = dctx->buffer + POLYVAL_BLOCK_SIZE - dctx->bytes;
0113
0114         dctx->bytes -= n;
0115         srclen -= n;
0116
0117         while (n--)
0118             *pos++ ^= *src++;
0119
0120         if (!dctx->bytes)
0121             internal_polyval_mul(dctx->buffer,
0122                         tctx->key_powers[NUM_KEY_POWERS-1]);
0123     }
0124
0125     while (srclen >= POLYVAL_BLOCK_SIZE) {
0126         /* Allow rescheduling every 4K bytes. */
0127         nblocks = min(srclen, 4096U) / POLYVAL_BLOCK_SIZE;
0128         internal_polyval_update(tctx, src, nblocks, dctx->buffer);
0129         srclen -= nblocks * POLYVAL_BLOCK_SIZE;
0130         src += nblocks * POLYVAL_BLOCK_SIZE;
0131     }
0132
0133     if (srclen) {
0134         dctx->bytes = POLYVAL_BLOCK_SIZE - srclen;
0135         pos = dctx->buffer;
0136         while (srclen--)
0137             *pos++ ^= *src++;
0138     }
0139
0140     return 0;
0141 }
0142
0143 static int polyval_x86_final(struct shash_desc *desc, u8 *dst)
0144 {
0145     struct polyval_desc_ctx *dctx = shash_desc_ctx(desc);
0146     const struct polyval_tfm_ctx *tctx = crypto_shash_ctx(desc->tfm);
0147
0148     if (dctx->bytes) {
0149         internal_polyval_mul(dctx->buffer,
0150                      tctx->key_powers[NUM_KEY_POWERS-1]);
0151     }
0152
0153     memcpy(dst, dctx->buffer, POLYVAL_BLOCK_SIZE);
0154
0155     return 0;
0156 }
0157
0158 static struct shash_alg polyval_alg = {
0159     .digestsize = POLYVAL_DIGEST_SIZE,
0160     .init       = polyval_x86_init,
0161     .update     = polyval_x86_update,
0162     .final      = polyval_x86_final,
0163     .setkey     = polyval_x86_setkey,
0164     .descsize   = sizeof(struct polyval_desc_ctx),
0165     .base       = {
0166         .cra_name       = "polyval",
0167         .cra_driver_name    = "polyval-clmulni",
0168         .cra_priority       = 200,
0169         .cra_blocksize      = POLYVAL_BLOCK_SIZE,
0170         .cra_ctxsize        = sizeof(struct polyval_tfm_ctx),
0171         .cra_module     = THIS_MODULE,
0172     },
0173 };
0174
0175 __maybe_unused static const struct x86_cpu_id pcmul_cpu_id[] = {
0176     X86_MATCH_FEATURE(X86_FEATURE_PCLMULQDQ, NULL),
0177     {}
0178 };
0179 MODULE_DEVICE_TABLE(x86cpu, pcmul_cpu_id);
0180
0181 static int __init polyval_clmulni_mod_init(void)
0182 {
0183     if (!x86_match_cpu(pcmul_cpu_id))
0184         return -ENODEV;
0185
0186     if (!boot_cpu_has(X86_FEATURE_AVX))
0187         return -ENODEV;
0188
0189     return crypto_register_shash(&polyval_alg);
0190 }
0191
0192 static void __exit polyval_clmulni_mod_exit(void)
0193 {
0194     crypto_unregister_shash(&polyval_alg);
0195 }
0196
0197 module_init(polyval_clmulni_mod_init);
0198 module_exit(polyval_clmulni_mod_exit);
0199
0200 MODULE_LICENSE("GPL");
0201 MODULE_DESCRIPTION("POLYVAL hash function accelerated by PCLMULQDQ-NI");
0202 MODULE_ALIAS_CRYPTO("polyval");
0203 MODULE_ALIAS_CRYPTO("polyval-clmulni");