arm/crypto/blake2b-neon-glue.c

0001 // SPDX-License-Identifier: GPL-2.0-or-later
0002 /*
0003  * BLAKE2b digest algorithm, NEON accelerated
0004  *
0005  * Copyright 2020 Google LLC
0006  */
0007
0008 #include <crypto/internal/blake2b.h>
0009 #include <crypto/internal/hash.h>
0010 #include <crypto/internal/simd.h>
0011
0012 #include <linux/module.h>
0013 #include <linux/sizes.h>
0014
0015 #include <asm/neon.h>
0016 #include <asm/simd.h>
0017
0018 asmlinkage void blake2b_compress_neon(struct blake2b_state *state,
0019                       const u8 *block, size_t nblocks, u32 inc);
0020
0021 static void blake2b_compress_arch(struct blake2b_state *state,
0022                   const u8 *block, size_t nblocks, u32 inc)
0023 {
0024     if (!crypto_simd_usable()) {
0025         blake2b_compress_generic(state, block, nblocks, inc);
0026         return;
0027     }
0028
0029     do {
0030         const size_t blocks = min_t(size_t, nblocks,
0031                         SZ_4K / BLAKE2B_BLOCK_SIZE);
0032
0033         kernel_neon_begin();
0034         blake2b_compress_neon(state, block, blocks, inc);
0035         kernel_neon_end();
0036
0037         nblocks -= blocks;
0038         block += blocks * BLAKE2B_BLOCK_SIZE;
0039     } while (nblocks);
0040 }
0041
0042 static int crypto_blake2b_update_neon(struct shash_desc *desc,
0043                       const u8 *in, unsigned int inlen)
0044 {
0045     return crypto_blake2b_update(desc, in, inlen, blake2b_compress_arch);
0046 }
0047
0048 static int crypto_blake2b_final_neon(struct shash_desc *desc, u8 *out)
0049 {
0050     return crypto_blake2b_final(desc, out, blake2b_compress_arch);
0051 }
0052
0053 #define BLAKE2B_ALG(name, driver_name, digest_size)         \
0054     {                               \
0055         .base.cra_name      = name,             \
0056         .base.cra_driver_name   = driver_name,          \
0057         .base.cra_priority  = 200,              \
0058         .base.cra_flags     = CRYPTO_ALG_OPTIONAL_KEY,  \
0059         .base.cra_blocksize = BLAKE2B_BLOCK_SIZE,       \
0060         .base.cra_ctxsize   = sizeof(struct blake2b_tfm_ctx), \
0061         .base.cra_module    = THIS_MODULE,          \
0062         .digestsize     = digest_size,          \
0063         .setkey         = crypto_blake2b_setkey,    \
0064         .init           = crypto_blake2b_init,      \
0065         .update         = crypto_blake2b_update_neon,   \
0066         .final          = crypto_blake2b_final_neon,    \
0067         .descsize       = sizeof(struct blake2b_state), \
0068     }
0069
0070 static struct shash_alg blake2b_neon_algs[] = {
0071     BLAKE2B_ALG("blake2b-160", "blake2b-160-neon", BLAKE2B_160_HASH_SIZE),
0072     BLAKE2B_ALG("blake2b-256", "blake2b-256-neon", BLAKE2B_256_HASH_SIZE),
0073     BLAKE2B_ALG("blake2b-384", "blake2b-384-neon", BLAKE2B_384_HASH_SIZE),
0074     BLAKE2B_ALG("blake2b-512", "blake2b-512-neon", BLAKE2B_512_HASH_SIZE),
0075 };
0076
0077 static int __init blake2b_neon_mod_init(void)
0078 {
0079     if (!(elf_hwcap & HWCAP_NEON))
0080         return -ENODEV;
0081
0082     return crypto_register_shashes(blake2b_neon_algs,
0083                        ARRAY_SIZE(blake2b_neon_algs));
0084 }
0085
0086 static void __exit blake2b_neon_mod_exit(void)
0087 {
0088     crypto_unregister_shashes(blake2b_neon_algs,
0089                   ARRAY_SIZE(blake2b_neon_algs));
0090 }
0091
0092 module_init(blake2b_neon_mod_init);
0093 module_exit(blake2b_neon_mod_exit);
0094
0095 MODULE_DESCRIPTION("BLAKE2b digest algorithm, NEON accelerated");
0096 MODULE_LICENSE("GPL");
0097 MODULE_AUTHOR("Eric Biggers <ebiggers@google.com>");
0098 MODULE_ALIAS_CRYPTO("blake2b-160");
0099 MODULE_ALIAS_CRYPTO("blake2b-160-neon");
0100 MODULE_ALIAS_CRYPTO("blake2b-256");
0101 MODULE_ALIAS_CRYPTO("blake2b-256-neon");
0102 MODULE_ALIAS_CRYPTO("blake2b-384");
0103 MODULE_ALIAS_CRYPTO("blake2b-384-neon");
0104 MODULE_ALIAS_CRYPTO("blake2b-512");
0105 MODULE_ALIAS_CRYPTO("blake2b-512-neon");