x86/crypto/serpent_avx2_glue.c

0001 // SPDX-License-Identifier: GPL-2.0-or-later
0002 /*
0003  * Glue Code for x86_64/AVX2 assembler optimized version of Serpent
0004  *
0005  * Copyright © 2012-2013 Jussi Kivilinna <jussi.kivilinna@mbnet.fi>
0006  */
0007
0008 #include <linux/module.h>
0009 #include <linux/types.h>
0010 #include <linux/crypto.h>
0011 #include <linux/err.h>
0012 #include <crypto/algapi.h>
0013 #include <crypto/internal/simd.h>
0014 #include <crypto/serpent.h>
0015
0016 #include "serpent-avx.h"
0017 #include "ecb_cbc_helpers.h"
0018
0019 #define SERPENT_AVX2_PARALLEL_BLOCKS 16
0020
0021 /* 16-way AVX2 parallel cipher functions */
0022 asmlinkage void serpent_ecb_enc_16way(const void *ctx, u8 *dst, const u8 *src);
0023 asmlinkage void serpent_ecb_dec_16way(const void *ctx, u8 *dst, const u8 *src);
0024 asmlinkage void serpent_cbc_dec_16way(const void *ctx, u8 *dst, const u8 *src);
0025
0026 static int serpent_setkey_skcipher(struct crypto_skcipher *tfm,
0027                    const u8 *key, unsigned int keylen)
0028 {
0029     return __serpent_setkey(crypto_skcipher_ctx(tfm), key, keylen);
0030 }
0031
0032 static int ecb_encrypt(struct skcipher_request *req)
0033 {
0034     ECB_WALK_START(req, SERPENT_BLOCK_SIZE, SERPENT_PARALLEL_BLOCKS);
0035     ECB_BLOCK(SERPENT_AVX2_PARALLEL_BLOCKS, serpent_ecb_enc_16way);
0036     ECB_BLOCK(SERPENT_PARALLEL_BLOCKS, serpent_ecb_enc_8way_avx);
0037     ECB_BLOCK(1, __serpent_encrypt);
0038     ECB_WALK_END();
0039 }
0040
0041 static int ecb_decrypt(struct skcipher_request *req)
0042 {
0043     ECB_WALK_START(req, SERPENT_BLOCK_SIZE, SERPENT_PARALLEL_BLOCKS);
0044     ECB_BLOCK(SERPENT_AVX2_PARALLEL_BLOCKS, serpent_ecb_dec_16way);
0045     ECB_BLOCK(SERPENT_PARALLEL_BLOCKS, serpent_ecb_dec_8way_avx);
0046     ECB_BLOCK(1, __serpent_decrypt);
0047     ECB_WALK_END();
0048 }
0049
0050 static int cbc_encrypt(struct skcipher_request *req)
0051 {
0052     CBC_WALK_START(req, SERPENT_BLOCK_SIZE, -1);
0053     CBC_ENC_BLOCK(__serpent_encrypt);
0054     CBC_WALK_END();
0055 }
0056
0057 static int cbc_decrypt(struct skcipher_request *req)
0058 {
0059     CBC_WALK_START(req, SERPENT_BLOCK_SIZE, SERPENT_PARALLEL_BLOCKS);
0060     CBC_DEC_BLOCK(SERPENT_AVX2_PARALLEL_BLOCKS, serpent_cbc_dec_16way);
0061     CBC_DEC_BLOCK(SERPENT_PARALLEL_BLOCKS, serpent_cbc_dec_8way_avx);
0062     CBC_DEC_BLOCK(1, __serpent_decrypt);
0063     CBC_WALK_END();
0064 }
0065
0066 static struct skcipher_alg serpent_algs[] = {
0067     {
0068         .base.cra_name      = "__ecb(serpent)",
0069         .base.cra_driver_name   = "__ecb-serpent-avx2",
0070         .base.cra_priority  = 600,
0071         .base.cra_flags     = CRYPTO_ALG_INTERNAL,
0072         .base.cra_blocksize = SERPENT_BLOCK_SIZE,
0073         .base.cra_ctxsize   = sizeof(struct serpent_ctx),
0074         .base.cra_module    = THIS_MODULE,
0075         .min_keysize        = SERPENT_MIN_KEY_SIZE,
0076         .max_keysize        = SERPENT_MAX_KEY_SIZE,
0077         .setkey         = serpent_setkey_skcipher,
0078         .encrypt        = ecb_encrypt,
0079         .decrypt        = ecb_decrypt,
0080     }, {
0081         .base.cra_name      = "__cbc(serpent)",
0082         .base.cra_driver_name   = "__cbc-serpent-avx2",
0083         .base.cra_priority  = 600,
0084         .base.cra_flags     = CRYPTO_ALG_INTERNAL,
0085         .base.cra_blocksize = SERPENT_BLOCK_SIZE,
0086         .base.cra_ctxsize   = sizeof(struct serpent_ctx),
0087         .base.cra_module    = THIS_MODULE,
0088         .min_keysize        = SERPENT_MIN_KEY_SIZE,
0089         .max_keysize        = SERPENT_MAX_KEY_SIZE,
0090         .ivsize         = SERPENT_BLOCK_SIZE,
0091         .setkey         = serpent_setkey_skcipher,
0092         .encrypt        = cbc_encrypt,
0093         .decrypt        = cbc_decrypt,
0094     },
0095 };
0096
0097 static struct simd_skcipher_alg *serpent_simd_algs[ARRAY_SIZE(serpent_algs)];
0098
0099 static int __init serpent_avx2_init(void)
0100 {
0101     const char *feature_name;
0102
0103     if (!boot_cpu_has(X86_FEATURE_AVX2) || !boot_cpu_has(X86_FEATURE_OSXSAVE)) {
0104         pr_info("AVX2 instructions are not detected.\n");
0105         return -ENODEV;
0106     }
0107     if (!cpu_has_xfeatures(XFEATURE_MASK_SSE | XFEATURE_MASK_YMM,
0108                 &feature_name)) {
0109         pr_info("CPU feature '%s' is not supported.\n", feature_name);
0110         return -ENODEV;
0111     }
0112
0113     return simd_register_skciphers_compat(serpent_algs,
0114                           ARRAY_SIZE(serpent_algs),
0115                           serpent_simd_algs);
0116 }
0117
0118 static void __exit serpent_avx2_fini(void)
0119 {
0120     simd_unregister_skciphers(serpent_algs, ARRAY_SIZE(serpent_algs),
0121                   serpent_simd_algs);
0122 }
0123
0124 module_init(serpent_avx2_init);
0125 module_exit(serpent_avx2_fini);
0126
0127 MODULE_LICENSE("GPL");
0128 MODULE_DESCRIPTION("Serpent Cipher Algorithm, AVX2 optimized");
0129 MODULE_ALIAS_CRYPTO("serpent");
0130 MODULE_ALIAS_CRYPTO("serpent-asm");