x86/crypto/blowfish_glue.c

0001 // SPDX-License-Identifier: GPL-2.0-or-later
0002 /*
0003  * Glue Code for assembler optimized version of Blowfish
0004  *
0005  * Copyright (c) 2011 Jussi Kivilinna <jussi.kivilinna@mbnet.fi>
0006  *
0007  * CBC & ECB parts based on code (crypto/cbc.c,ecb.c) by:
0008  *   Copyright (c) 2006 Herbert Xu <herbert@gondor.apana.org.au>
0009  */
0010
0011 #include <crypto/algapi.h>
0012 #include <crypto/blowfish.h>
0013 #include <crypto/internal/skcipher.h>
0014 #include <linux/crypto.h>
0015 #include <linux/init.h>
0016 #include <linux/module.h>
0017 #include <linux/types.h>
0018
0019 /* regular block cipher functions */
0020 asmlinkage void __blowfish_enc_blk(struct bf_ctx *ctx, u8 *dst, const u8 *src,
0021                    bool xor);
0022 asmlinkage void blowfish_dec_blk(struct bf_ctx *ctx, u8 *dst, const u8 *src);
0023
0024 /* 4-way parallel cipher functions */
0025 asmlinkage void __blowfish_enc_blk_4way(struct bf_ctx *ctx, u8 *dst,
0026                     const u8 *src, bool xor);
0027 asmlinkage void blowfish_dec_blk_4way(struct bf_ctx *ctx, u8 *dst,
0028                       const u8 *src);
0029
0030 static inline void blowfish_enc_blk(struct bf_ctx *ctx, u8 *dst, const u8 *src)
0031 {
0032     __blowfish_enc_blk(ctx, dst, src, false);
0033 }
0034
0035 static inline void blowfish_enc_blk_4way(struct bf_ctx *ctx, u8 *dst,
0036                      const u8 *src)
0037 {
0038     __blowfish_enc_blk_4way(ctx, dst, src, false);
0039 }
0040
0041 static void blowfish_encrypt(struct crypto_tfm *tfm, u8 *dst, const u8 *src)
0042 {
0043     blowfish_enc_blk(crypto_tfm_ctx(tfm), dst, src);
0044 }
0045
0046 static void blowfish_decrypt(struct crypto_tfm *tfm, u8 *dst, const u8 *src)
0047 {
0048     blowfish_dec_blk(crypto_tfm_ctx(tfm), dst, src);
0049 }
0050
0051 static int blowfish_setkey_skcipher(struct crypto_skcipher *tfm,
0052                     const u8 *key, unsigned int keylen)
0053 {
0054     return blowfish_setkey(&tfm->base, key, keylen);
0055 }
0056
0057 static int ecb_crypt(struct skcipher_request *req,
0058              void (*fn)(struct bf_ctx *, u8 *, const u8 *),
0059              void (*fn_4way)(struct bf_ctx *, u8 *, const u8 *))
0060 {
0061     unsigned int bsize = BF_BLOCK_SIZE;
0062     struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req);
0063     struct bf_ctx *ctx = crypto_skcipher_ctx(tfm);
0064     struct skcipher_walk walk;
0065     unsigned int nbytes;
0066     int err;
0067
0068     err = skcipher_walk_virt(&walk, req, false);
0069
0070     while ((nbytes = walk.nbytes)) {
0071         u8 *wsrc = walk.src.virt.addr;
0072         u8 *wdst = walk.dst.virt.addr;
0073
0074         /* Process four block batch */
0075         if (nbytes >= bsize * 4) {
0076             do {
0077                 fn_4way(ctx, wdst, wsrc);
0078
0079                 wsrc += bsize * 4;
0080                 wdst += bsize * 4;
0081                 nbytes -= bsize * 4;
0082             } while (nbytes >= bsize * 4);
0083
0084             if (nbytes < bsize)
0085                 goto done;
0086         }
0087
0088         /* Handle leftovers */
0089         do {
0090             fn(ctx, wdst, wsrc);
0091
0092             wsrc += bsize;
0093             wdst += bsize;
0094             nbytes -= bsize;
0095         } while (nbytes >= bsize);
0096
0097 done:
0098         err = skcipher_walk_done(&walk, nbytes);
0099     }
0100
0101     return err;
0102 }
0103
0104 static int ecb_encrypt(struct skcipher_request *req)
0105 {
0106     return ecb_crypt(req, blowfish_enc_blk, blowfish_enc_blk_4way);
0107 }
0108
0109 static int ecb_decrypt(struct skcipher_request *req)
0110 {
0111     return ecb_crypt(req, blowfish_dec_blk, blowfish_dec_blk_4way);
0112 }
0113
0114 static unsigned int __cbc_encrypt(struct bf_ctx *ctx,
0115                   struct skcipher_walk *walk)
0116 {
0117     unsigned int bsize = BF_BLOCK_SIZE;
0118     unsigned int nbytes = walk->nbytes;
0119     u64 *src = (u64 *)walk->src.virt.addr;
0120     u64 *dst = (u64 *)walk->dst.virt.addr;
0121     u64 *iv = (u64 *)walk->iv;
0122
0123     do {
0124         *dst = *src ^ *iv;
0125         blowfish_enc_blk(ctx, (u8 *)dst, (u8 *)dst);
0126         iv = dst;
0127
0128         src += 1;
0129         dst += 1;
0130         nbytes -= bsize;
0131     } while (nbytes >= bsize);
0132
0133     *(u64 *)walk->iv = *iv;
0134     return nbytes;
0135 }
0136
0137 static int cbc_encrypt(struct skcipher_request *req)
0138 {
0139     struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req);
0140     struct bf_ctx *ctx = crypto_skcipher_ctx(tfm);
0141     struct skcipher_walk walk;
0142     unsigned int nbytes;
0143     int err;
0144
0145     err = skcipher_walk_virt(&walk, req, false);
0146
0147     while (walk.nbytes) {
0148         nbytes = __cbc_encrypt(ctx, &walk);
0149         err = skcipher_walk_done(&walk, nbytes);
0150     }
0151
0152     return err;
0153 }
0154
0155 static unsigned int __cbc_decrypt(struct bf_ctx *ctx,
0156                   struct skcipher_walk *walk)
0157 {
0158     unsigned int bsize = BF_BLOCK_SIZE;
0159     unsigned int nbytes = walk->nbytes;
0160     u64 *src = (u64 *)walk->src.virt.addr;
0161     u64 *dst = (u64 *)walk->dst.virt.addr;
0162     u64 ivs[4 - 1];
0163     u64 last_iv;
0164
0165     /* Start of the last block. */
0166     src += nbytes / bsize - 1;
0167     dst += nbytes / bsize - 1;
0168
0169     last_iv = *src;
0170
0171     /* Process four block batch */
0172     if (nbytes >= bsize * 4) {
0173         do {
0174             nbytes -= bsize * 4 - bsize;
0175             src -= 4 - 1;
0176             dst -= 4 - 1;
0177
0178             ivs[0] = src[0];
0179             ivs[1] = src[1];
0180             ivs[2] = src[2];
0181
0182             blowfish_dec_blk_4way(ctx, (u8 *)dst, (u8 *)src);
0183
0184             dst[1] ^= ivs[0];
0185             dst[2] ^= ivs[1];
0186             dst[3] ^= ivs[2];
0187
0188             nbytes -= bsize;
0189             if (nbytes < bsize)
0190                 goto done;
0191
0192             *dst ^= *(src - 1);
0193             src -= 1;
0194             dst -= 1;
0195         } while (nbytes >= bsize * 4);
0196     }
0197
0198     /* Handle leftovers */
0199     for (;;) {
0200         blowfish_dec_blk(ctx, (u8 *)dst, (u8 *)src);
0201
0202         nbytes -= bsize;
0203         if (nbytes < bsize)
0204             break;
0205
0206         *dst ^= *(src - 1);
0207         src -= 1;
0208         dst -= 1;
0209     }
0210
0211 done:
0212     *dst ^= *(u64 *)walk->iv;
0213     *(u64 *)walk->iv = last_iv;
0214
0215     return nbytes;
0216 }
0217
0218 static int cbc_decrypt(struct skcipher_request *req)
0219 {
0220     struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req);
0221     struct bf_ctx *ctx = crypto_skcipher_ctx(tfm);
0222     struct skcipher_walk walk;
0223     unsigned int nbytes;
0224     int err;
0225
0226     err = skcipher_walk_virt(&walk, req, false);
0227
0228     while (walk.nbytes) {
0229         nbytes = __cbc_decrypt(ctx, &walk);
0230         err = skcipher_walk_done(&walk, nbytes);
0231     }
0232
0233     return err;
0234 }
0235
0236 static struct crypto_alg bf_cipher_alg = {
0237     .cra_name       = "blowfish",
0238     .cra_driver_name    = "blowfish-asm",
0239     .cra_priority       = 200,
0240     .cra_flags      = CRYPTO_ALG_TYPE_CIPHER,
0241     .cra_blocksize      = BF_BLOCK_SIZE,
0242     .cra_ctxsize        = sizeof(struct bf_ctx),
0243     .cra_alignmask      = 0,
0244     .cra_module     = THIS_MODULE,
0245     .cra_u = {
0246         .cipher = {
0247             .cia_min_keysize    = BF_MIN_KEY_SIZE,
0248             .cia_max_keysize    = BF_MAX_KEY_SIZE,
0249             .cia_setkey     = blowfish_setkey,
0250             .cia_encrypt        = blowfish_encrypt,
0251             .cia_decrypt        = blowfish_decrypt,
0252         }
0253     }
0254 };
0255
0256 static struct skcipher_alg bf_skcipher_algs[] = {
0257     {
0258         .base.cra_name      = "ecb(blowfish)",
0259         .base.cra_driver_name   = "ecb-blowfish-asm",
0260         .base.cra_priority  = 300,
0261         .base.cra_blocksize = BF_BLOCK_SIZE,
0262         .base.cra_ctxsize   = sizeof(struct bf_ctx),
0263         .base.cra_module    = THIS_MODULE,
0264         .min_keysize        = BF_MIN_KEY_SIZE,
0265         .max_keysize        = BF_MAX_KEY_SIZE,
0266         .setkey         = blowfish_setkey_skcipher,
0267         .encrypt        = ecb_encrypt,
0268         .decrypt        = ecb_decrypt,
0269     }, {
0270         .base.cra_name      = "cbc(blowfish)",
0271         .base.cra_driver_name   = "cbc-blowfish-asm",
0272         .base.cra_priority  = 300,
0273         .base.cra_blocksize = BF_BLOCK_SIZE,
0274         .base.cra_ctxsize   = sizeof(struct bf_ctx),
0275         .base.cra_module    = THIS_MODULE,
0276         .min_keysize        = BF_MIN_KEY_SIZE,
0277         .max_keysize        = BF_MAX_KEY_SIZE,
0278         .ivsize         = BF_BLOCK_SIZE,
0279         .setkey         = blowfish_setkey_skcipher,
0280         .encrypt        = cbc_encrypt,
0281         .decrypt        = cbc_decrypt,
0282     },
0283 };
0284
0285 static bool is_blacklisted_cpu(void)
0286 {
0287     if (boot_cpu_data.x86_vendor != X86_VENDOR_INTEL)
0288         return false;
0289
0290     if (boot_cpu_data.x86 == 0x0f) {
0291         /*
0292          * On Pentium 4, blowfish-x86_64 is slower than generic C
0293          * implementation because use of 64bit rotates (which are really
0294          * slow on P4). Therefore blacklist P4s.
0295          */
0296         return true;
0297     }
0298
0299     return false;
0300 }
0301
0302 static int force;
0303 module_param(force, int, 0);
0304 MODULE_PARM_DESC(force, "Force module load, ignore CPU blacklist");
0305
0306 static int __init blowfish_init(void)
0307 {
0308     int err;
0309
0310     if (!force && is_blacklisted_cpu()) {
0311         printk(KERN_INFO
0312             "blowfish-x86_64: performance on this CPU "
0313             "would be suboptimal: disabling "
0314             "blowfish-x86_64.\n");
0315         return -ENODEV;
0316     }
0317
0318     err = crypto_register_alg(&bf_cipher_alg);
0319     if (err)
0320         return err;
0321
0322     err = crypto_register_skciphers(bf_skcipher_algs,
0323                     ARRAY_SIZE(bf_skcipher_algs));
0324     if (err)
0325         crypto_unregister_alg(&bf_cipher_alg);
0326
0327     return err;
0328 }
0329
0330 static void __exit blowfish_fini(void)
0331 {
0332     crypto_unregister_alg(&bf_cipher_alg);
0333     crypto_unregister_skciphers(bf_skcipher_algs,
0334                     ARRAY_SIZE(bf_skcipher_algs));
0335 }
0336
0337 module_init(blowfish_init);
0338 module_exit(blowfish_fini);
0339
0340 MODULE_LICENSE("GPL");
0341 MODULE_DESCRIPTION("Blowfish Cipher Algorithm, asm optimized");
0342 MODULE_ALIAS_CRYPTO("blowfish");
0343 MODULE_ALIAS_CRYPTO("blowfish-asm");