x86/crypto/sm4_aesni_avx2_glue.c

0001 /* SPDX-License-Identifier: GPL-2.0-or-later */
0002 /*
0003  * SM4 Cipher Algorithm, AES-NI/AVX2 optimized.
0004  * as specified in
0005  * https://tools.ietf.org/id/draft-ribose-cfrg-sm4-10.html
0006  *
0007  * Copyright (c) 2021, Alibaba Group.
0008  * Copyright (c) 2021 Tianjia Zhang <tianjia.zhang@linux.alibaba.com>
0009  */
0010
0011 #include <linux/module.h>
0012 #include <linux/crypto.h>
0013 #include <linux/kernel.h>
0014 #include <asm/simd.h>
0015 #include <crypto/internal/simd.h>
0016 #include <crypto/internal/skcipher.h>
0017 #include <crypto/sm4.h>
0018 #include "sm4-avx.h"
0019
0020 #define SM4_CRYPT16_BLOCK_SIZE  (SM4_BLOCK_SIZE * 16)
0021
0022 asmlinkage void sm4_aesni_avx2_ctr_enc_blk16(const u32 *rk, u8 *dst,
0023                     const u8 *src, u8 *iv);
0024 asmlinkage void sm4_aesni_avx2_cbc_dec_blk16(const u32 *rk, u8 *dst,
0025                     const u8 *src, u8 *iv);
0026 asmlinkage void sm4_aesni_avx2_cfb_dec_blk16(const u32 *rk, u8 *dst,
0027                     const u8 *src, u8 *iv);
0028
0029 static int sm4_skcipher_setkey(struct crypto_skcipher *tfm, const u8 *key,
0030             unsigned int key_len)
0031 {
0032     struct sm4_ctx *ctx = crypto_skcipher_ctx(tfm);
0033
0034     return sm4_expandkey(ctx, key, key_len);
0035 }
0036
0037 static int cbc_decrypt(struct skcipher_request *req)
0038 {
0039     return sm4_avx_cbc_decrypt(req, SM4_CRYPT16_BLOCK_SIZE,
0040                 sm4_aesni_avx2_cbc_dec_blk16);
0041 }
0042
0043
0044 static int cfb_decrypt(struct skcipher_request *req)
0045 {
0046     return sm4_avx_cfb_decrypt(req, SM4_CRYPT16_BLOCK_SIZE,
0047                 sm4_aesni_avx2_cfb_dec_blk16);
0048 }
0049
0050 static int ctr_crypt(struct skcipher_request *req)
0051 {
0052     return sm4_avx_ctr_crypt(req, SM4_CRYPT16_BLOCK_SIZE,
0053                 sm4_aesni_avx2_ctr_enc_blk16);
0054 }
0055
0056 static struct skcipher_alg sm4_aesni_avx2_skciphers[] = {
0057     {
0058         .base = {
0059             .cra_name       = "__ecb(sm4)",
0060             .cra_driver_name    = "__ecb-sm4-aesni-avx2",
0061             .cra_priority       = 500,
0062             .cra_flags      = CRYPTO_ALG_INTERNAL,
0063             .cra_blocksize      = SM4_BLOCK_SIZE,
0064             .cra_ctxsize        = sizeof(struct sm4_ctx),
0065             .cra_module     = THIS_MODULE,
0066         },
0067         .min_keysize    = SM4_KEY_SIZE,
0068         .max_keysize    = SM4_KEY_SIZE,
0069         .walksize   = 16 * SM4_BLOCK_SIZE,
0070         .setkey     = sm4_skcipher_setkey,
0071         .encrypt    = sm4_avx_ecb_encrypt,
0072         .decrypt    = sm4_avx_ecb_decrypt,
0073     }, {
0074         .base = {
0075             .cra_name       = "__cbc(sm4)",
0076             .cra_driver_name    = "__cbc-sm4-aesni-avx2",
0077             .cra_priority       = 500,
0078             .cra_flags      = CRYPTO_ALG_INTERNAL,
0079             .cra_blocksize      = SM4_BLOCK_SIZE,
0080             .cra_ctxsize        = sizeof(struct sm4_ctx),
0081             .cra_module     = THIS_MODULE,
0082         },
0083         .min_keysize    = SM4_KEY_SIZE,
0084         .max_keysize    = SM4_KEY_SIZE,
0085         .ivsize     = SM4_BLOCK_SIZE,
0086         .walksize   = 16 * SM4_BLOCK_SIZE,
0087         .setkey     = sm4_skcipher_setkey,
0088         .encrypt    = sm4_cbc_encrypt,
0089         .decrypt    = cbc_decrypt,
0090     }, {
0091         .base = {
0092             .cra_name       = "__cfb(sm4)",
0093             .cra_driver_name    = "__cfb-sm4-aesni-avx2",
0094             .cra_priority       = 500,
0095             .cra_flags      = CRYPTO_ALG_INTERNAL,
0096             .cra_blocksize      = 1,
0097             .cra_ctxsize        = sizeof(struct sm4_ctx),
0098             .cra_module     = THIS_MODULE,
0099         },
0100         .min_keysize    = SM4_KEY_SIZE,
0101         .max_keysize    = SM4_KEY_SIZE,
0102         .ivsize     = SM4_BLOCK_SIZE,
0103         .chunksize  = SM4_BLOCK_SIZE,
0104         .walksize   = 16 * SM4_BLOCK_SIZE,
0105         .setkey     = sm4_skcipher_setkey,
0106         .encrypt    = sm4_cfb_encrypt,
0107         .decrypt    = cfb_decrypt,
0108     }, {
0109         .base = {
0110             .cra_name       = "__ctr(sm4)",
0111             .cra_driver_name    = "__ctr-sm4-aesni-avx2",
0112             .cra_priority       = 500,
0113             .cra_flags      = CRYPTO_ALG_INTERNAL,
0114             .cra_blocksize      = 1,
0115             .cra_ctxsize        = sizeof(struct sm4_ctx),
0116             .cra_module     = THIS_MODULE,
0117         },
0118         .min_keysize    = SM4_KEY_SIZE,
0119         .max_keysize    = SM4_KEY_SIZE,
0120         .ivsize     = SM4_BLOCK_SIZE,
0121         .chunksize  = SM4_BLOCK_SIZE,
0122         .walksize   = 16 * SM4_BLOCK_SIZE,
0123         .setkey     = sm4_skcipher_setkey,
0124         .encrypt    = ctr_crypt,
0125         .decrypt    = ctr_crypt,
0126     }
0127 };
0128
0129 static struct simd_skcipher_alg *
0130 simd_sm4_aesni_avx2_skciphers[ARRAY_SIZE(sm4_aesni_avx2_skciphers)];
0131
0132 static int __init sm4_init(void)
0133 {
0134     const char *feature_name;
0135
0136     if (!boot_cpu_has(X86_FEATURE_AVX) ||
0137         !boot_cpu_has(X86_FEATURE_AVX2) ||
0138         !boot_cpu_has(X86_FEATURE_AES) ||
0139         !boot_cpu_has(X86_FEATURE_OSXSAVE)) {
0140         pr_info("AVX2 or AES-NI instructions are not detected.\n");
0141         return -ENODEV;
0142     }
0143
0144     if (!cpu_has_xfeatures(XFEATURE_MASK_SSE | XFEATURE_MASK_YMM,
0145                 &feature_name)) {
0146         pr_info("CPU feature '%s' is not supported.\n", feature_name);
0147         return -ENODEV;
0148     }
0149
0150     return simd_register_skciphers_compat(sm4_aesni_avx2_skciphers,
0151                     ARRAY_SIZE(sm4_aesni_avx2_skciphers),
0152                     simd_sm4_aesni_avx2_skciphers);
0153 }
0154
0155 static void __exit sm4_exit(void)
0156 {
0157     simd_unregister_skciphers(sm4_aesni_avx2_skciphers,
0158                 ARRAY_SIZE(sm4_aesni_avx2_skciphers),
0159                 simd_sm4_aesni_avx2_skciphers);
0160 }
0161
0162 module_init(sm4_init);
0163 module_exit(sm4_exit);
0164
0165 MODULE_LICENSE("GPL v2");
0166 MODULE_AUTHOR("Tianjia Zhang <tianjia.zhang@linux.alibaba.com>");
0167 MODULE_DESCRIPTION("SM4 Cipher Algorithm, AES-NI/AVX2 optimized");
0168 MODULE_ALIAS_CRYPTO("sm4");
0169 MODULE_ALIAS_CRYPTO("sm4-aesni-avx2");