sparc/crypto/des_glue.c

0001 // SPDX-License-Identifier: GPL-2.0-only
0002 /* Glue code for DES encryption optimized for sparc64 crypto opcodes.
0003  *
0004  * Copyright (C) 2012 David S. Miller <davem@davemloft.net>
0005  */
0006
0007 #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
0008
0009 #include <linux/crypto.h>
0010 #include <linux/init.h>
0011 #include <linux/module.h>
0012 #include <linux/mm.h>
0013 #include <linux/types.h>
0014 #include <crypto/algapi.h>
0015 #include <crypto/internal/des.h>
0016 #include <crypto/internal/skcipher.h>
0017
0018 #include <asm/fpumacro.h>
0019 #include <asm/pstate.h>
0020 #include <asm/elf.h>
0021
0022 #include "opcodes.h"
0023
0024 struct des_sparc64_ctx {
0025     u64 encrypt_expkey[DES_EXPKEY_WORDS / 2];
0026     u64 decrypt_expkey[DES_EXPKEY_WORDS / 2];
0027 };
0028
0029 struct des3_ede_sparc64_ctx {
0030     u64 encrypt_expkey[DES3_EDE_EXPKEY_WORDS / 2];
0031     u64 decrypt_expkey[DES3_EDE_EXPKEY_WORDS / 2];
0032 };
0033
0034 static void encrypt_to_decrypt(u64 *d, const u64 *e)
0035 {
0036     const u64 *s = e + (DES_EXPKEY_WORDS / 2) - 1;
0037     int i;
0038
0039     for (i = 0; i < DES_EXPKEY_WORDS / 2; i++)
0040         *d++ = *s--;
0041 }
0042
0043 extern void des_sparc64_key_expand(const u32 *input_key, u64 *key);
0044
0045 static int des_set_key(struct crypto_tfm *tfm, const u8 *key,
0046                unsigned int keylen)
0047 {
0048     struct des_sparc64_ctx *dctx = crypto_tfm_ctx(tfm);
0049     int err;
0050
0051     /* Even though we have special instructions for key expansion,
0052      * we call des_verify_key() so that we don't have to write our own
0053      * weak key detection code.
0054      */
0055     err = crypto_des_verify_key(tfm, key);
0056     if (err)
0057         return err;
0058
0059     des_sparc64_key_expand((const u32 *) key, &dctx->encrypt_expkey[0]);
0060     encrypt_to_decrypt(&dctx->decrypt_expkey[0], &dctx->encrypt_expkey[0]);
0061
0062     return 0;
0063 }
0064
0065 static int des_set_key_skcipher(struct crypto_skcipher *tfm, const u8 *key,
0066                 unsigned int keylen)
0067 {
0068     return des_set_key(crypto_skcipher_tfm(tfm), key, keylen);
0069 }
0070
0071 extern void des_sparc64_crypt(const u64 *key, const u64 *input,
0072                   u64 *output);
0073
0074 static void sparc_des_encrypt(struct crypto_tfm *tfm, u8 *dst, const u8 *src)
0075 {
0076     struct des_sparc64_ctx *ctx = crypto_tfm_ctx(tfm);
0077     const u64 *K = ctx->encrypt_expkey;
0078
0079     des_sparc64_crypt(K, (const u64 *) src, (u64 *) dst);
0080 }
0081
0082 static void sparc_des_decrypt(struct crypto_tfm *tfm, u8 *dst, const u8 *src)
0083 {
0084     struct des_sparc64_ctx *ctx = crypto_tfm_ctx(tfm);
0085     const u64 *K = ctx->decrypt_expkey;
0086
0087     des_sparc64_crypt(K, (const u64 *) src, (u64 *) dst);
0088 }
0089
0090 extern void des_sparc64_load_keys(const u64 *key);
0091
0092 extern void des_sparc64_ecb_crypt(const u64 *input, u64 *output,
0093                   unsigned int len);
0094
0095 static int __ecb_crypt(struct skcipher_request *req, bool encrypt)
0096 {
0097     struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req);
0098     const struct des_sparc64_ctx *ctx = crypto_skcipher_ctx(tfm);
0099     struct skcipher_walk walk;
0100     unsigned int nbytes;
0101     int err;
0102
0103     err = skcipher_walk_virt(&walk, req, true);
0104     if (err)
0105         return err;
0106
0107     if (encrypt)
0108         des_sparc64_load_keys(&ctx->encrypt_expkey[0]);
0109     else
0110         des_sparc64_load_keys(&ctx->decrypt_expkey[0]);
0111     while ((nbytes = walk.nbytes) != 0) {
0112         des_sparc64_ecb_crypt(walk.src.virt.addr, walk.dst.virt.addr,
0113                       round_down(nbytes, DES_BLOCK_SIZE));
0114         err = skcipher_walk_done(&walk, nbytes % DES_BLOCK_SIZE);
0115     }
0116     fprs_write(0);
0117     return err;
0118 }
0119
0120 static int ecb_encrypt(struct skcipher_request *req)
0121 {
0122     return __ecb_crypt(req, true);
0123 }
0124
0125 static int ecb_decrypt(struct skcipher_request *req)
0126 {
0127     return __ecb_crypt(req, false);
0128 }
0129
0130 extern void des_sparc64_cbc_encrypt(const u64 *input, u64 *output,
0131                     unsigned int len, u64 *iv);
0132
0133 extern void des_sparc64_cbc_decrypt(const u64 *input, u64 *output,
0134                     unsigned int len, u64 *iv);
0135
0136 static int __cbc_crypt(struct skcipher_request *req, bool encrypt)
0137 {
0138     struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req);
0139     const struct des_sparc64_ctx *ctx = crypto_skcipher_ctx(tfm);
0140     struct skcipher_walk walk;
0141     unsigned int nbytes;
0142     int err;
0143
0144     err = skcipher_walk_virt(&walk, req, true);
0145     if (err)
0146         return err;
0147
0148     if (encrypt)
0149         des_sparc64_load_keys(&ctx->encrypt_expkey[0]);
0150     else
0151         des_sparc64_load_keys(&ctx->decrypt_expkey[0]);
0152     while ((nbytes = walk.nbytes) != 0) {
0153         if (encrypt)
0154             des_sparc64_cbc_encrypt(walk.src.virt.addr,
0155                         walk.dst.virt.addr,
0156                         round_down(nbytes,
0157                                DES_BLOCK_SIZE),
0158                         walk.iv);
0159         else
0160             des_sparc64_cbc_decrypt(walk.src.virt.addr,
0161                         walk.dst.virt.addr,
0162                         round_down(nbytes,
0163                                DES_BLOCK_SIZE),
0164                         walk.iv);
0165         err = skcipher_walk_done(&walk, nbytes % DES_BLOCK_SIZE);
0166     }
0167     fprs_write(0);
0168     return err;
0169 }
0170
0171 static int cbc_encrypt(struct skcipher_request *req)
0172 {
0173     return __cbc_crypt(req, true);
0174 }
0175
0176 static int cbc_decrypt(struct skcipher_request *req)
0177 {
0178     return __cbc_crypt(req, false);
0179 }
0180
0181 static int des3_ede_set_key(struct crypto_tfm *tfm, const u8 *key,
0182                 unsigned int keylen)
0183 {
0184     struct des3_ede_sparc64_ctx *dctx = crypto_tfm_ctx(tfm);
0185     u64 k1[DES_EXPKEY_WORDS / 2];
0186     u64 k2[DES_EXPKEY_WORDS / 2];
0187     u64 k3[DES_EXPKEY_WORDS / 2];
0188     int err;
0189
0190     err = crypto_des3_ede_verify_key(tfm, key);
0191     if (err)
0192         return err;
0193
0194     des_sparc64_key_expand((const u32 *)key, k1);
0195     key += DES_KEY_SIZE;
0196     des_sparc64_key_expand((const u32 *)key, k2);
0197     key += DES_KEY_SIZE;
0198     des_sparc64_key_expand((const u32 *)key, k3);
0199
0200     memcpy(&dctx->encrypt_expkey[0], &k1[0], sizeof(k1));
0201     encrypt_to_decrypt(&dctx->encrypt_expkey[DES_EXPKEY_WORDS / 2], &k2[0]);
0202     memcpy(&dctx->encrypt_expkey[(DES_EXPKEY_WORDS / 2) * 2],
0203            &k3[0], sizeof(k3));
0204
0205     encrypt_to_decrypt(&dctx->decrypt_expkey[0], &k3[0]);
0206     memcpy(&dctx->decrypt_expkey[DES_EXPKEY_WORDS / 2],
0207            &k2[0], sizeof(k2));
0208     encrypt_to_decrypt(&dctx->decrypt_expkey[(DES_EXPKEY_WORDS / 2) * 2],
0209                &k1[0]);
0210
0211     return 0;
0212 }
0213
0214 static int des3_ede_set_key_skcipher(struct crypto_skcipher *tfm, const u8 *key,
0215                      unsigned int keylen)
0216 {
0217     return des3_ede_set_key(crypto_skcipher_tfm(tfm), key, keylen);
0218 }
0219
0220 extern void des3_ede_sparc64_crypt(const u64 *key, const u64 *input,
0221                    u64 *output);
0222
0223 static void sparc_des3_ede_encrypt(struct crypto_tfm *tfm, u8 *dst, const u8 *src)
0224 {
0225     struct des3_ede_sparc64_ctx *ctx = crypto_tfm_ctx(tfm);
0226     const u64 *K = ctx->encrypt_expkey;
0227
0228     des3_ede_sparc64_crypt(K, (const u64 *) src, (u64 *) dst);
0229 }
0230
0231 static void sparc_des3_ede_decrypt(struct crypto_tfm *tfm, u8 *dst, const u8 *src)
0232 {
0233     struct des3_ede_sparc64_ctx *ctx = crypto_tfm_ctx(tfm);
0234     const u64 *K = ctx->decrypt_expkey;
0235
0236     des3_ede_sparc64_crypt(K, (const u64 *) src, (u64 *) dst);
0237 }
0238
0239 extern void des3_ede_sparc64_load_keys(const u64 *key);
0240
0241 extern void des3_ede_sparc64_ecb_crypt(const u64 *expkey, const u64 *input,
0242                        u64 *output, unsigned int len);
0243
0244 static int __ecb3_crypt(struct skcipher_request *req, bool encrypt)
0245 {
0246     struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req);
0247     const struct des3_ede_sparc64_ctx *ctx = crypto_skcipher_ctx(tfm);
0248     struct skcipher_walk walk;
0249     const u64 *K;
0250     unsigned int nbytes;
0251     int err;
0252
0253     err = skcipher_walk_virt(&walk, req, true);
0254     if (err)
0255         return err;
0256
0257     if (encrypt)
0258         K = &ctx->encrypt_expkey[0];
0259     else
0260         K = &ctx->decrypt_expkey[0];
0261     des3_ede_sparc64_load_keys(K);
0262     while ((nbytes = walk.nbytes) != 0) {
0263         des3_ede_sparc64_ecb_crypt(K, walk.src.virt.addr,
0264                        walk.dst.virt.addr,
0265                        round_down(nbytes, DES_BLOCK_SIZE));
0266         err = skcipher_walk_done(&walk, nbytes % DES_BLOCK_SIZE);
0267     }
0268     fprs_write(0);
0269     return err;
0270 }
0271
0272 static int ecb3_encrypt(struct skcipher_request *req)
0273 {
0274     return __ecb3_crypt(req, true);
0275 }
0276
0277 static int ecb3_decrypt(struct skcipher_request *req)
0278 {
0279     return __ecb3_crypt(req, false);
0280 }
0281
0282 extern void des3_ede_sparc64_cbc_encrypt(const u64 *expkey, const u64 *input,
0283                      u64 *output, unsigned int len,
0284                      u64 *iv);
0285
0286 extern void des3_ede_sparc64_cbc_decrypt(const u64 *expkey, const u64 *input,
0287                      u64 *output, unsigned int len,
0288                      u64 *iv);
0289
0290 static int __cbc3_crypt(struct skcipher_request *req, bool encrypt)
0291 {
0292     struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req);
0293     const struct des3_ede_sparc64_ctx *ctx = crypto_skcipher_ctx(tfm);
0294     struct skcipher_walk walk;
0295     const u64 *K;
0296     unsigned int nbytes;
0297     int err;
0298
0299     err = skcipher_walk_virt(&walk, req, true);
0300     if (err)
0301         return err;
0302
0303     if (encrypt)
0304         K = &ctx->encrypt_expkey[0];
0305     else
0306         K = &ctx->decrypt_expkey[0];
0307     des3_ede_sparc64_load_keys(K);
0308     while ((nbytes = walk.nbytes) != 0) {
0309         if (encrypt)
0310             des3_ede_sparc64_cbc_encrypt(K, walk.src.virt.addr,
0311                              walk.dst.virt.addr,
0312                              round_down(nbytes,
0313                                 DES_BLOCK_SIZE),
0314                              walk.iv);
0315         else
0316             des3_ede_sparc64_cbc_decrypt(K, walk.src.virt.addr,
0317                              walk.dst.virt.addr,
0318                              round_down(nbytes,
0319                                 DES_BLOCK_SIZE),
0320                              walk.iv);
0321         err = skcipher_walk_done(&walk, nbytes % DES_BLOCK_SIZE);
0322     }
0323     fprs_write(0);
0324     return err;
0325 }
0326
0327 static int cbc3_encrypt(struct skcipher_request *req)
0328 {
0329     return __cbc3_crypt(req, true);
0330 }
0331
0332 static int cbc3_decrypt(struct skcipher_request *req)
0333 {
0334     return __cbc3_crypt(req, false);
0335 }
0336
0337 static struct crypto_alg cipher_algs[] = {
0338     {
0339         .cra_name       = "des",
0340         .cra_driver_name    = "des-sparc64",
0341         .cra_priority       = SPARC_CR_OPCODE_PRIORITY,
0342         .cra_flags      = CRYPTO_ALG_TYPE_CIPHER,
0343         .cra_blocksize      = DES_BLOCK_SIZE,
0344         .cra_ctxsize        = sizeof(struct des_sparc64_ctx),
0345         .cra_alignmask      = 7,
0346         .cra_module     = THIS_MODULE,
0347         .cra_u  = {
0348             .cipher = {
0349                 .cia_min_keysize    = DES_KEY_SIZE,
0350                 .cia_max_keysize    = DES_KEY_SIZE,
0351                 .cia_setkey     = des_set_key,
0352                 .cia_encrypt        = sparc_des_encrypt,
0353                 .cia_decrypt        = sparc_des_decrypt
0354             }
0355         }
0356     }, {
0357         .cra_name       = "des3_ede",
0358         .cra_driver_name    = "des3_ede-sparc64",
0359         .cra_priority       = SPARC_CR_OPCODE_PRIORITY,
0360         .cra_flags      = CRYPTO_ALG_TYPE_CIPHER,
0361         .cra_blocksize      = DES3_EDE_BLOCK_SIZE,
0362         .cra_ctxsize        = sizeof(struct des3_ede_sparc64_ctx),
0363         .cra_alignmask      = 7,
0364         .cra_module     = THIS_MODULE,
0365         .cra_u  = {
0366             .cipher = {
0367                 .cia_min_keysize    = DES3_EDE_KEY_SIZE,
0368                 .cia_max_keysize    = DES3_EDE_KEY_SIZE,
0369                 .cia_setkey     = des3_ede_set_key,
0370                 .cia_encrypt        = sparc_des3_ede_encrypt,
0371                 .cia_decrypt        = sparc_des3_ede_decrypt
0372             }
0373         }
0374     }
0375 };
0376
0377 static struct skcipher_alg skcipher_algs[] = {
0378     {
0379         .base.cra_name      = "ecb(des)",
0380         .base.cra_driver_name   = "ecb-des-sparc64",
0381         .base.cra_priority  = SPARC_CR_OPCODE_PRIORITY,
0382         .base.cra_blocksize = DES_BLOCK_SIZE,
0383         .base.cra_ctxsize   = sizeof(struct des_sparc64_ctx),
0384         .base.cra_alignmask = 7,
0385         .base.cra_module    = THIS_MODULE,
0386         .min_keysize        = DES_KEY_SIZE,
0387         .max_keysize        = DES_KEY_SIZE,
0388         .setkey         = des_set_key_skcipher,
0389         .encrypt        = ecb_encrypt,
0390         .decrypt        = ecb_decrypt,
0391     }, {
0392         .base.cra_name      = "cbc(des)",
0393         .base.cra_driver_name   = "cbc-des-sparc64",
0394         .base.cra_priority  = SPARC_CR_OPCODE_PRIORITY,
0395         .base.cra_blocksize = DES_BLOCK_SIZE,
0396         .base.cra_ctxsize   = sizeof(struct des_sparc64_ctx),
0397         .base.cra_alignmask = 7,
0398         .base.cra_module    = THIS_MODULE,
0399         .min_keysize        = DES_KEY_SIZE,
0400         .max_keysize        = DES_KEY_SIZE,
0401         .ivsize         = DES_BLOCK_SIZE,
0402         .setkey         = des_set_key_skcipher,
0403         .encrypt        = cbc_encrypt,
0404         .decrypt        = cbc_decrypt,
0405     }, {
0406         .base.cra_name      = "ecb(des3_ede)",
0407         .base.cra_driver_name   = "ecb-des3_ede-sparc64",
0408         .base.cra_priority  = SPARC_CR_OPCODE_PRIORITY,
0409         .base.cra_blocksize = DES3_EDE_BLOCK_SIZE,
0410         .base.cra_ctxsize   = sizeof(struct des3_ede_sparc64_ctx),
0411         .base.cra_alignmask = 7,
0412         .base.cra_module    = THIS_MODULE,
0413         .min_keysize        = DES3_EDE_KEY_SIZE,
0414         .max_keysize        = DES3_EDE_KEY_SIZE,
0415         .setkey         = des3_ede_set_key_skcipher,
0416         .encrypt        = ecb3_encrypt,
0417         .decrypt        = ecb3_decrypt,
0418     }, {
0419         .base.cra_name      = "cbc(des3_ede)",
0420         .base.cra_driver_name   = "cbc-des3_ede-sparc64",
0421         .base.cra_priority  = SPARC_CR_OPCODE_PRIORITY,
0422         .base.cra_blocksize = DES3_EDE_BLOCK_SIZE,
0423         .base.cra_ctxsize   = sizeof(struct des3_ede_sparc64_ctx),
0424         .base.cra_alignmask = 7,
0425         .base.cra_module    = THIS_MODULE,
0426         .min_keysize        = DES3_EDE_KEY_SIZE,
0427         .max_keysize        = DES3_EDE_KEY_SIZE,
0428         .ivsize         = DES3_EDE_BLOCK_SIZE,
0429         .setkey         = des3_ede_set_key_skcipher,
0430         .encrypt        = cbc3_encrypt,
0431         .decrypt        = cbc3_decrypt,
0432     }
0433 };
0434
0435 static bool __init sparc64_has_des_opcode(void)
0436 {
0437     unsigned long cfr;
0438
0439     if (!(sparc64_elf_hwcap & HWCAP_SPARC_CRYPTO))
0440         return false;
0441
0442     __asm__ __volatile__("rd %%asr26, %0" : "=r" (cfr));
0443     if (!(cfr & CFR_DES))
0444         return false;
0445
0446     return true;
0447 }
0448
0449 static int __init des_sparc64_mod_init(void)
0450 {
0451     int err;
0452
0453     if (!sparc64_has_des_opcode()) {
0454         pr_info("sparc64 des opcodes not available.\n");
0455         return -ENODEV;
0456     }
0457     pr_info("Using sparc64 des opcodes optimized DES implementation\n");
0458     err = crypto_register_algs(cipher_algs, ARRAY_SIZE(cipher_algs));
0459     if (err)
0460         return err;
0461     err = crypto_register_skciphers(skcipher_algs,
0462                     ARRAY_SIZE(skcipher_algs));
0463     if (err)
0464         crypto_unregister_algs(cipher_algs, ARRAY_SIZE(cipher_algs));
0465     return err;
0466 }
0467
0468 static void __exit des_sparc64_mod_fini(void)
0469 {
0470     crypto_unregister_algs(cipher_algs, ARRAY_SIZE(cipher_algs));
0471     crypto_unregister_skciphers(skcipher_algs, ARRAY_SIZE(skcipher_algs));
0472 }
0473
0474 module_init(des_sparc64_mod_init);
0475 module_exit(des_sparc64_mod_fini);
0476
0477 MODULE_LICENSE("GPL");
0478 MODULE_DESCRIPTION("DES & Triple DES EDE Cipher Algorithms, sparc64 des opcode accelerated");
0479
0480 MODULE_ALIAS_CRYPTO("des");
0481 MODULE_ALIAS_CRYPTO("des3_ede");
0482
0483 #include "crop_devid.c"