x86/crypto/des3_ede_glue.c

0001 // SPDX-License-Identifier: GPL-2.0-or-later
0002 /*
0003  * Glue Code for assembler optimized version of 3DES
0004  *
0005  * Copyright © 2014 Jussi Kivilinna <jussi.kivilinna@mbnet.fi>
0006  *
0007  * CBC & ECB parts based on code (crypto/cbc.c,ecb.c) by:
0008  *   Copyright (c) 2006 Herbert Xu <herbert@gondor.apana.org.au>
0009  */
0010
0011 #include <crypto/algapi.h>
0012 #include <crypto/des.h>
0013 #include <crypto/internal/skcipher.h>
0014 #include <linux/crypto.h>
0015 #include <linux/init.h>
0016 #include <linux/module.h>
0017 #include <linux/types.h>
0018
0019 struct des3_ede_x86_ctx {
0020     struct des3_ede_ctx enc;
0021     struct des3_ede_ctx dec;
0022 };
0023
0024 /* regular block cipher functions */
0025 asmlinkage void des3_ede_x86_64_crypt_blk(const u32 *expkey, u8 *dst,
0026                       const u8 *src);
0027
0028 /* 3-way parallel cipher functions */
0029 asmlinkage void des3_ede_x86_64_crypt_blk_3way(const u32 *expkey, u8 *dst,
0030                            const u8 *src);
0031
0032 static inline void des3_ede_enc_blk(struct des3_ede_x86_ctx *ctx, u8 *dst,
0033                     const u8 *src)
0034 {
0035     u32 *enc_ctx = ctx->enc.expkey;
0036
0037     des3_ede_x86_64_crypt_blk(enc_ctx, dst, src);
0038 }
0039
0040 static inline void des3_ede_dec_blk(struct des3_ede_x86_ctx *ctx, u8 *dst,
0041                     const u8 *src)
0042 {
0043     u32 *dec_ctx = ctx->dec.expkey;
0044
0045     des3_ede_x86_64_crypt_blk(dec_ctx, dst, src);
0046 }
0047
0048 static inline void des3_ede_dec_blk_3way(struct des3_ede_x86_ctx *ctx, u8 *dst,
0049                      const u8 *src)
0050 {
0051     u32 *dec_ctx = ctx->dec.expkey;
0052
0053     des3_ede_x86_64_crypt_blk_3way(dec_ctx, dst, src);
0054 }
0055
0056 static void des3_ede_x86_encrypt(struct crypto_tfm *tfm, u8 *dst, const u8 *src)
0057 {
0058     des3_ede_enc_blk(crypto_tfm_ctx(tfm), dst, src);
0059 }
0060
0061 static void des3_ede_x86_decrypt(struct crypto_tfm *tfm, u8 *dst, const u8 *src)
0062 {
0063     des3_ede_dec_blk(crypto_tfm_ctx(tfm), dst, src);
0064 }
0065
0066 static int ecb_crypt(struct skcipher_request *req, const u32 *expkey)
0067 {
0068     const unsigned int bsize = DES3_EDE_BLOCK_SIZE;
0069     struct skcipher_walk walk;
0070     unsigned int nbytes;
0071     int err;
0072
0073     err = skcipher_walk_virt(&walk, req, false);
0074
0075     while ((nbytes = walk.nbytes)) {
0076         u8 *wsrc = walk.src.virt.addr;
0077         u8 *wdst = walk.dst.virt.addr;
0078
0079         /* Process four block batch */
0080         if (nbytes >= bsize * 3) {
0081             do {
0082                 des3_ede_x86_64_crypt_blk_3way(expkey, wdst,
0083                                    wsrc);
0084
0085                 wsrc += bsize * 3;
0086                 wdst += bsize * 3;
0087                 nbytes -= bsize * 3;
0088             } while (nbytes >= bsize * 3);
0089
0090             if (nbytes < bsize)
0091                 goto done;
0092         }
0093
0094         /* Handle leftovers */
0095         do {
0096             des3_ede_x86_64_crypt_blk(expkey, wdst, wsrc);
0097
0098             wsrc += bsize;
0099             wdst += bsize;
0100             nbytes -= bsize;
0101         } while (nbytes >= bsize);
0102
0103 done:
0104         err = skcipher_walk_done(&walk, nbytes);
0105     }
0106
0107     return err;
0108 }
0109
0110 static int ecb_encrypt(struct skcipher_request *req)
0111 {
0112     struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req);
0113     struct des3_ede_x86_ctx *ctx = crypto_skcipher_ctx(tfm);
0114
0115     return ecb_crypt(req, ctx->enc.expkey);
0116 }
0117
0118 static int ecb_decrypt(struct skcipher_request *req)
0119 {
0120     struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req);
0121     struct des3_ede_x86_ctx *ctx = crypto_skcipher_ctx(tfm);
0122
0123     return ecb_crypt(req, ctx->dec.expkey);
0124 }
0125
0126 static unsigned int __cbc_encrypt(struct des3_ede_x86_ctx *ctx,
0127                   struct skcipher_walk *walk)
0128 {
0129     unsigned int bsize = DES3_EDE_BLOCK_SIZE;
0130     unsigned int nbytes = walk->nbytes;
0131     u64 *src = (u64 *)walk->src.virt.addr;
0132     u64 *dst = (u64 *)walk->dst.virt.addr;
0133     u64 *iv = (u64 *)walk->iv;
0134
0135     do {
0136         *dst = *src ^ *iv;
0137         des3_ede_enc_blk(ctx, (u8 *)dst, (u8 *)dst);
0138         iv = dst;
0139
0140         src += 1;
0141         dst += 1;
0142         nbytes -= bsize;
0143     } while (nbytes >= bsize);
0144
0145     *(u64 *)walk->iv = *iv;
0146     return nbytes;
0147 }
0148
0149 static int cbc_encrypt(struct skcipher_request *req)
0150 {
0151     struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req);
0152     struct des3_ede_x86_ctx *ctx = crypto_skcipher_ctx(tfm);
0153     struct skcipher_walk walk;
0154     unsigned int nbytes;
0155     int err;
0156
0157     err = skcipher_walk_virt(&walk, req, false);
0158
0159     while (walk.nbytes) {
0160         nbytes = __cbc_encrypt(ctx, &walk);
0161         err = skcipher_walk_done(&walk, nbytes);
0162     }
0163
0164     return err;
0165 }
0166
0167 static unsigned int __cbc_decrypt(struct des3_ede_x86_ctx *ctx,
0168                   struct skcipher_walk *walk)
0169 {
0170     unsigned int bsize = DES3_EDE_BLOCK_SIZE;
0171     unsigned int nbytes = walk->nbytes;
0172     u64 *src = (u64 *)walk->src.virt.addr;
0173     u64 *dst = (u64 *)walk->dst.virt.addr;
0174     u64 ivs[3 - 1];
0175     u64 last_iv;
0176
0177     /* Start of the last block. */
0178     src += nbytes / bsize - 1;
0179     dst += nbytes / bsize - 1;
0180
0181     last_iv = *src;
0182
0183     /* Process four block batch */
0184     if (nbytes >= bsize * 3) {
0185         do {
0186             nbytes -= bsize * 3 - bsize;
0187             src -= 3 - 1;
0188             dst -= 3 - 1;
0189
0190             ivs[0] = src[0];
0191             ivs[1] = src[1];
0192
0193             des3_ede_dec_blk_3way(ctx, (u8 *)dst, (u8 *)src);
0194
0195             dst[1] ^= ivs[0];
0196             dst[2] ^= ivs[1];
0197
0198             nbytes -= bsize;
0199             if (nbytes < bsize)
0200                 goto done;
0201
0202             *dst ^= *(src - 1);
0203             src -= 1;
0204             dst -= 1;
0205         } while (nbytes >= bsize * 3);
0206     }
0207
0208     /* Handle leftovers */
0209     for (;;) {
0210         des3_ede_dec_blk(ctx, (u8 *)dst, (u8 *)src);
0211
0212         nbytes -= bsize;
0213         if (nbytes < bsize)
0214             break;
0215
0216         *dst ^= *(src - 1);
0217         src -= 1;
0218         dst -= 1;
0219     }
0220
0221 done:
0222     *dst ^= *(u64 *)walk->iv;
0223     *(u64 *)walk->iv = last_iv;
0224
0225     return nbytes;
0226 }
0227
0228 static int cbc_decrypt(struct skcipher_request *req)
0229 {
0230     struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req);
0231     struct des3_ede_x86_ctx *ctx = crypto_skcipher_ctx(tfm);
0232     struct skcipher_walk walk;
0233     unsigned int nbytes;
0234     int err;
0235
0236     err = skcipher_walk_virt(&walk, req, false);
0237
0238     while (walk.nbytes) {
0239         nbytes = __cbc_decrypt(ctx, &walk);
0240         err = skcipher_walk_done(&walk, nbytes);
0241     }
0242
0243     return err;
0244 }
0245
0246 static int des3_ede_x86_setkey(struct crypto_tfm *tfm, const u8 *key,
0247                    unsigned int keylen)
0248 {
0249     struct des3_ede_x86_ctx *ctx = crypto_tfm_ctx(tfm);
0250     u32 i, j, tmp;
0251     int err;
0252
0253     err = des3_ede_expand_key(&ctx->enc, key, keylen);
0254     if (err == -ENOKEY) {
0255         if (crypto_tfm_get_flags(tfm) & CRYPTO_TFM_REQ_FORBID_WEAK_KEYS)
0256             err = -EINVAL;
0257         else
0258             err = 0;
0259     }
0260
0261     if (err) {
0262         memset(ctx, 0, sizeof(*ctx));
0263         return err;
0264     }
0265
0266     /* Fix encryption context for this implementation and form decryption
0267      * context. */
0268     j = DES3_EDE_EXPKEY_WORDS - 2;
0269     for (i = 0; i < DES3_EDE_EXPKEY_WORDS; i += 2, j -= 2) {
0270         tmp = ror32(ctx->enc.expkey[i + 1], 4);
0271         ctx->enc.expkey[i + 1] = tmp;
0272
0273         ctx->dec.expkey[j + 0] = ctx->enc.expkey[i + 0];
0274         ctx->dec.expkey[j + 1] = tmp;
0275     }
0276
0277     return 0;
0278 }
0279
0280 static int des3_ede_x86_setkey_skcipher(struct crypto_skcipher *tfm,
0281                     const u8 *key,
0282                     unsigned int keylen)
0283 {
0284     return des3_ede_x86_setkey(&tfm->base, key, keylen);
0285 }
0286
0287 static struct crypto_alg des3_ede_cipher = {
0288     .cra_name       = "des3_ede",
0289     .cra_driver_name    = "des3_ede-asm",
0290     .cra_priority       = 200,
0291     .cra_flags      = CRYPTO_ALG_TYPE_CIPHER,
0292     .cra_blocksize      = DES3_EDE_BLOCK_SIZE,
0293     .cra_ctxsize        = sizeof(struct des3_ede_x86_ctx),
0294     .cra_alignmask      = 0,
0295     .cra_module     = THIS_MODULE,
0296     .cra_u = {
0297         .cipher = {
0298             .cia_min_keysize    = DES3_EDE_KEY_SIZE,
0299             .cia_max_keysize    = DES3_EDE_KEY_SIZE,
0300             .cia_setkey     = des3_ede_x86_setkey,
0301             .cia_encrypt        = des3_ede_x86_encrypt,
0302             .cia_decrypt        = des3_ede_x86_decrypt,
0303         }
0304     }
0305 };
0306
0307 static struct skcipher_alg des3_ede_skciphers[] = {
0308     {
0309         .base.cra_name      = "ecb(des3_ede)",
0310         .base.cra_driver_name   = "ecb-des3_ede-asm",
0311         .base.cra_priority  = 300,
0312         .base.cra_blocksize = DES3_EDE_BLOCK_SIZE,
0313         .base.cra_ctxsize   = sizeof(struct des3_ede_x86_ctx),
0314         .base.cra_module    = THIS_MODULE,
0315         .min_keysize        = DES3_EDE_KEY_SIZE,
0316         .max_keysize        = DES3_EDE_KEY_SIZE,
0317         .setkey         = des3_ede_x86_setkey_skcipher,
0318         .encrypt        = ecb_encrypt,
0319         .decrypt        = ecb_decrypt,
0320     }, {
0321         .base.cra_name      = "cbc(des3_ede)",
0322         .base.cra_driver_name   = "cbc-des3_ede-asm",
0323         .base.cra_priority  = 300,
0324         .base.cra_blocksize = DES3_EDE_BLOCK_SIZE,
0325         .base.cra_ctxsize   = sizeof(struct des3_ede_x86_ctx),
0326         .base.cra_module    = THIS_MODULE,
0327         .min_keysize        = DES3_EDE_KEY_SIZE,
0328         .max_keysize        = DES3_EDE_KEY_SIZE,
0329         .ivsize         = DES3_EDE_BLOCK_SIZE,
0330         .setkey         = des3_ede_x86_setkey_skcipher,
0331         .encrypt        = cbc_encrypt,
0332         .decrypt        = cbc_decrypt,
0333     }
0334 };
0335
0336 static bool is_blacklisted_cpu(void)
0337 {
0338     if (boot_cpu_data.x86_vendor != X86_VENDOR_INTEL)
0339         return false;
0340
0341     if (boot_cpu_data.x86 == 0x0f) {
0342         /*
0343          * On Pentium 4, des3_ede-x86_64 is slower than generic C
0344          * implementation because use of 64bit rotates (which are really
0345          * slow on P4). Therefore blacklist P4s.
0346          */
0347         return true;
0348     }
0349
0350     return false;
0351 }
0352
0353 static int force;
0354 module_param(force, int, 0);
0355 MODULE_PARM_DESC(force, "Force module load, ignore CPU blacklist");
0356
0357 static int __init des3_ede_x86_init(void)
0358 {
0359     int err;
0360
0361     if (!force && is_blacklisted_cpu()) {
0362         pr_info("des3_ede-x86_64: performance on this CPU would be suboptimal: disabling des3_ede-x86_64.\n");
0363         return -ENODEV;
0364     }
0365
0366     err = crypto_register_alg(&des3_ede_cipher);
0367     if (err)
0368         return err;
0369
0370     err = crypto_register_skciphers(des3_ede_skciphers,
0371                     ARRAY_SIZE(des3_ede_skciphers));
0372     if (err)
0373         crypto_unregister_alg(&des3_ede_cipher);
0374
0375     return err;
0376 }
0377
0378 static void __exit des3_ede_x86_fini(void)
0379 {
0380     crypto_unregister_alg(&des3_ede_cipher);
0381     crypto_unregister_skciphers(des3_ede_skciphers,
0382                     ARRAY_SIZE(des3_ede_skciphers));
0383 }
0384
0385 module_init(des3_ede_x86_init);
0386 module_exit(des3_ede_x86_fini);
0387
0388 MODULE_LICENSE("GPL");
0389 MODULE_DESCRIPTION("Triple DES EDE Cipher Algorithm, asm optimized");
0390 MODULE_ALIAS_CRYPTO("des3_ede");
0391 MODULE_ALIAS_CRYPTO("des3_ede-asm");
0392 MODULE_AUTHOR("Jussi Kivilinna <jussi.kivilinna@iki.fi>");