arm64/crypto/polyval-ce-glue.c

0001 // SPDX-License-Identifier: GPL-2.0-only
0002 /*
0003  * Glue code for POLYVAL using ARMv8 Crypto Extensions
0004  *
0005  * Copyright (c) 2007 Nokia Siemens Networks - Mikko Herranen <mh1@iki.fi>
0006  * Copyright (c) 2009 Intel Corp.
0007  *   Author: Huang Ying <ying.huang@intel.com>
0008  * Copyright 2021 Google LLC
0009  */
0010
0011 /*
0012  * Glue code based on ghash-clmulni-intel_glue.c.
0013  *
0014  * This implementation of POLYVAL uses montgomery multiplication accelerated by
0015  * ARMv8 Crypto Extensions instructions to implement the finite field operations.
0016  */
0017
0018 #include <crypto/algapi.h>
0019 #include <crypto/internal/hash.h>
0020 #include <crypto/internal/simd.h>
0021 #include <crypto/polyval.h>
0022 #include <linux/crypto.h>
0023 #include <linux/init.h>
0024 #include <linux/kernel.h>
0025 #include <linux/module.h>
0026 #include <linux/cpufeature.h>
0027 #include <asm/neon.h>
0028 #include <asm/simd.h>
0029
0030 #define NUM_KEY_POWERS  8
0031
0032 struct polyval_tfm_ctx {
0033     /*
0034      * These powers must be in the order h^8, ..., h^1.
0035      */
0036     u8 key_powers[NUM_KEY_POWERS][POLYVAL_BLOCK_SIZE];
0037 };
0038
0039 struct polyval_desc_ctx {
0040     u8 buffer[POLYVAL_BLOCK_SIZE];
0041     u32 bytes;
0042 };
0043
0044 asmlinkage void pmull_polyval_update(const struct polyval_tfm_ctx *keys,
0045     const u8 *in, size_t nblocks, u8 *accumulator);
0046 asmlinkage void pmull_polyval_mul(u8 *op1, const u8 *op2);
0047
0048 static void internal_polyval_update(const struct polyval_tfm_ctx *keys,
0049     const u8 *in, size_t nblocks, u8 *accumulator)
0050 {
0051     if (likely(crypto_simd_usable())) {
0052         kernel_neon_begin();
0053         pmull_polyval_update(keys, in, nblocks, accumulator);
0054         kernel_neon_end();
0055     } else {
0056         polyval_update_non4k(keys->key_powers[NUM_KEY_POWERS-1], in,
0057             nblocks, accumulator);
0058     }
0059 }
0060
0061 static void internal_polyval_mul(u8 *op1, const u8 *op2)
0062 {
0063     if (likely(crypto_simd_usable())) {
0064         kernel_neon_begin();
0065         pmull_polyval_mul(op1, op2);
0066         kernel_neon_end();
0067     } else {
0068         polyval_mul_non4k(op1, op2);
0069     }
0070 }
0071
0072 static int polyval_arm64_setkey(struct crypto_shash *tfm,
0073             const u8 *key, unsigned int keylen)
0074 {
0075     struct polyval_tfm_ctx *tctx = crypto_shash_ctx(tfm);
0076     int i;
0077
0078     if (keylen != POLYVAL_BLOCK_SIZE)
0079         return -EINVAL;
0080
0081     memcpy(tctx->key_powers[NUM_KEY_POWERS-1], key, POLYVAL_BLOCK_SIZE);
0082
0083     for (i = NUM_KEY_POWERS-2; i >= 0; i--) {
0084         memcpy(tctx->key_powers[i], key, POLYVAL_BLOCK_SIZE);
0085         internal_polyval_mul(tctx->key_powers[i],
0086                      tctx->key_powers[i+1]);
0087     }
0088
0089     return 0;
0090 }
0091
0092 static int polyval_arm64_init(struct shash_desc *desc)
0093 {
0094     struct polyval_desc_ctx *dctx = shash_desc_ctx(desc);
0095
0096     memset(dctx, 0, sizeof(*dctx));
0097
0098     return 0;
0099 }
0100
0101 static int polyval_arm64_update(struct shash_desc *desc,
0102              const u8 *src, unsigned int srclen)
0103 {
0104     struct polyval_desc_ctx *dctx = shash_desc_ctx(desc);
0105     const struct polyval_tfm_ctx *tctx = crypto_shash_ctx(desc->tfm);
0106     u8 *pos;
0107     unsigned int nblocks;
0108     unsigned int n;
0109
0110     if (dctx->bytes) {
0111         n = min(srclen, dctx->bytes);
0112         pos = dctx->buffer + POLYVAL_BLOCK_SIZE - dctx->bytes;
0113
0114         dctx->bytes -= n;
0115         srclen -= n;
0116
0117         while (n--)
0118             *pos++ ^= *src++;
0119
0120         if (!dctx->bytes)
0121             internal_polyval_mul(dctx->buffer,
0122                         tctx->key_powers[NUM_KEY_POWERS-1]);
0123     }
0124
0125     while (srclen >= POLYVAL_BLOCK_SIZE) {
0126         /* allow rescheduling every 4K bytes */
0127         nblocks = min(srclen, 4096U) / POLYVAL_BLOCK_SIZE;
0128         internal_polyval_update(tctx, src, nblocks, dctx->buffer);
0129         srclen -= nblocks * POLYVAL_BLOCK_SIZE;
0130         src += nblocks * POLYVAL_BLOCK_SIZE;
0131     }
0132
0133     if (srclen) {
0134         dctx->bytes = POLYVAL_BLOCK_SIZE - srclen;
0135         pos = dctx->buffer;
0136         while (srclen--)
0137             *pos++ ^= *src++;
0138     }
0139
0140     return 0;
0141 }
0142
0143 static int polyval_arm64_final(struct shash_desc *desc, u8 *dst)
0144 {
0145     struct polyval_desc_ctx *dctx = shash_desc_ctx(desc);
0146     const struct polyval_tfm_ctx *tctx = crypto_shash_ctx(desc->tfm);
0147
0148     if (dctx->bytes) {
0149         internal_polyval_mul(dctx->buffer,
0150                      tctx->key_powers[NUM_KEY_POWERS-1]);
0151     }
0152
0153     memcpy(dst, dctx->buffer, POLYVAL_BLOCK_SIZE);
0154
0155     return 0;
0156 }
0157
0158 static struct shash_alg polyval_alg = {
0159     .digestsize = POLYVAL_DIGEST_SIZE,
0160     .init       = polyval_arm64_init,
0161     .update     = polyval_arm64_update,
0162     .final      = polyval_arm64_final,
0163     .setkey     = polyval_arm64_setkey,
0164     .descsize   = sizeof(struct polyval_desc_ctx),
0165     .base       = {
0166         .cra_name       = "polyval",
0167         .cra_driver_name    = "polyval-ce",
0168         .cra_priority       = 200,
0169         .cra_blocksize      = POLYVAL_BLOCK_SIZE,
0170         .cra_ctxsize        = sizeof(struct polyval_tfm_ctx),
0171         .cra_module     = THIS_MODULE,
0172     },
0173 };
0174
0175 static int __init polyval_ce_mod_init(void)
0176 {
0177     return crypto_register_shash(&polyval_alg);
0178 }
0179
0180 static void __exit polyval_ce_mod_exit(void)
0181 {
0182     crypto_unregister_shash(&polyval_alg);
0183 }
0184
0185 module_cpu_feature_match(PMULL, polyval_ce_mod_init)
0186 module_exit(polyval_ce_mod_exit);
0187
0188 MODULE_LICENSE("GPL");
0189 MODULE_DESCRIPTION("POLYVAL hash function accelerated by ARMv8 Crypto Extensions");
0190 MODULE_ALIAS_CRYPTO("polyval");
0191 MODULE_ALIAS_CRYPTO("polyval-ce");