the-tree/crypto/simd.c

0001 // SPDX-License-Identifier: GPL-2.0-or-later
0002 /*
0003  * Shared crypto simd helpers
0004  *
0005  * Copyright (c) 2012 Jussi Kivilinna <jussi.kivilinna@mbnet.fi>
0006  * Copyright (c) 2016 Herbert Xu <herbert@gondor.apana.org.au>
0007  * Copyright (c) 2019 Google LLC
0008  *
0009  * Based on aesni-intel_glue.c by:
0010  *  Copyright (C) 2008, Intel Corp.
0011  *    Author: Huang Ying <ying.huang@intel.com>
0012  */
0013
0014 /*
0015  * Shared crypto SIMD helpers.  These functions dynamically create and register
0016  * an skcipher or AEAD algorithm that wraps another, internal algorithm.  The
0017  * wrapper ensures that the internal algorithm is only executed in a context
0018  * where SIMD instructions are usable, i.e. where may_use_simd() returns true.
0019  * If SIMD is already usable, the wrapper directly calls the internal algorithm.
0020  * Otherwise it defers execution to a workqueue via cryptd.
0021  *
0022  * This is an alternative to the internal algorithm implementing a fallback for
0023  * the !may_use_simd() case itself.
0024  *
0025  * Note that the wrapper algorithm is asynchronous, i.e. it has the
0026  * CRYPTO_ALG_ASYNC flag set.  Therefore it won't be found by users who
0027  * explicitly allocate a synchronous algorithm.
0028  */
0029
0030 #include <crypto/cryptd.h>
0031 #include <crypto/internal/aead.h>
0032 #include <crypto/internal/simd.h>
0033 #include <crypto/internal/skcipher.h>
0034 #include <linux/kernel.h>
0035 #include <linux/module.h>
0036 #include <linux/preempt.h>
0037 #include <asm/simd.h>
0038
0039 /* skcipher support */
0040
0041 struct simd_skcipher_alg {
0042     const char *ialg_name;
0043     struct skcipher_alg alg;
0044 };
0045
0046 struct simd_skcipher_ctx {
0047     struct cryptd_skcipher *cryptd_tfm;
0048 };
0049
0050 static int simd_skcipher_setkey(struct crypto_skcipher *tfm, const u8 *key,
0051                 unsigned int key_len)
0052 {
0053     struct simd_skcipher_ctx *ctx = crypto_skcipher_ctx(tfm);
0054     struct crypto_skcipher *child = &ctx->cryptd_tfm->base;
0055
0056     crypto_skcipher_clear_flags(child, CRYPTO_TFM_REQ_MASK);
0057     crypto_skcipher_set_flags(child, crypto_skcipher_get_flags(tfm) &
0058                      CRYPTO_TFM_REQ_MASK);
0059     return crypto_skcipher_setkey(child, key, key_len);
0060 }
0061
0062 static int simd_skcipher_encrypt(struct skcipher_request *req)
0063 {
0064     struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req);
0065     struct simd_skcipher_ctx *ctx = crypto_skcipher_ctx(tfm);
0066     struct skcipher_request *subreq;
0067     struct crypto_skcipher *child;
0068
0069     subreq = skcipher_request_ctx(req);
0070     *subreq = *req;
0071
0072     if (!crypto_simd_usable() ||
0073         (in_atomic() && cryptd_skcipher_queued(ctx->cryptd_tfm)))
0074         child = &ctx->cryptd_tfm->base;
0075     else
0076         child = cryptd_skcipher_child(ctx->cryptd_tfm);
0077
0078     skcipher_request_set_tfm(subreq, child);
0079
0080     return crypto_skcipher_encrypt(subreq);
0081 }
0082
0083 static int simd_skcipher_decrypt(struct skcipher_request *req)
0084 {
0085     struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req);
0086     struct simd_skcipher_ctx *ctx = crypto_skcipher_ctx(tfm);
0087     struct skcipher_request *subreq;
0088     struct crypto_skcipher *child;
0089
0090     subreq = skcipher_request_ctx(req);
0091     *subreq = *req;
0092
0093     if (!crypto_simd_usable() ||
0094         (in_atomic() && cryptd_skcipher_queued(ctx->cryptd_tfm)))
0095         child = &ctx->cryptd_tfm->base;
0096     else
0097         child = cryptd_skcipher_child(ctx->cryptd_tfm);
0098
0099     skcipher_request_set_tfm(subreq, child);
0100
0101     return crypto_skcipher_decrypt(subreq);
0102 }
0103
0104 static void simd_skcipher_exit(struct crypto_skcipher *tfm)
0105 {
0106     struct simd_skcipher_ctx *ctx = crypto_skcipher_ctx(tfm);
0107
0108     cryptd_free_skcipher(ctx->cryptd_tfm);
0109 }
0110
0111 static int simd_skcipher_init(struct crypto_skcipher *tfm)
0112 {
0113     struct simd_skcipher_ctx *ctx = crypto_skcipher_ctx(tfm);
0114     struct cryptd_skcipher *cryptd_tfm;
0115     struct simd_skcipher_alg *salg;
0116     struct skcipher_alg *alg;
0117     unsigned reqsize;
0118
0119     alg = crypto_skcipher_alg(tfm);
0120     salg = container_of(alg, struct simd_skcipher_alg, alg);
0121
0122     cryptd_tfm = cryptd_alloc_skcipher(salg->ialg_name,
0123                        CRYPTO_ALG_INTERNAL,
0124                        CRYPTO_ALG_INTERNAL);
0125     if (IS_ERR(cryptd_tfm))
0126         return PTR_ERR(cryptd_tfm);
0127
0128     ctx->cryptd_tfm = cryptd_tfm;
0129
0130     reqsize = crypto_skcipher_reqsize(cryptd_skcipher_child(cryptd_tfm));
0131     reqsize = max(reqsize, crypto_skcipher_reqsize(&cryptd_tfm->base));
0132     reqsize += sizeof(struct skcipher_request);
0133
0134     crypto_skcipher_set_reqsize(tfm, reqsize);
0135
0136     return 0;
0137 }
0138
0139 struct simd_skcipher_alg *simd_skcipher_create_compat(const char *algname,
0140                               const char *drvname,
0141                               const char *basename)
0142 {
0143     struct simd_skcipher_alg *salg;
0144     struct crypto_skcipher *tfm;
0145     struct skcipher_alg *ialg;
0146     struct skcipher_alg *alg;
0147     int err;
0148
0149     tfm = crypto_alloc_skcipher(basename, CRYPTO_ALG_INTERNAL,
0150                     CRYPTO_ALG_INTERNAL | CRYPTO_ALG_ASYNC);
0151     if (IS_ERR(tfm))
0152         return ERR_CAST(tfm);
0153
0154     ialg = crypto_skcipher_alg(tfm);
0155
0156     salg = kzalloc(sizeof(*salg), GFP_KERNEL);
0157     if (!salg) {
0158         salg = ERR_PTR(-ENOMEM);
0159         goto out_put_tfm;
0160     }
0161
0162     salg->ialg_name = basename;
0163     alg = &salg->alg;
0164
0165     err = -ENAMETOOLONG;
0166     if (snprintf(alg->base.cra_name, CRYPTO_MAX_ALG_NAME, "%s", algname) >=
0167         CRYPTO_MAX_ALG_NAME)
0168         goto out_free_salg;
0169
0170     if (snprintf(alg->base.cra_driver_name, CRYPTO_MAX_ALG_NAME, "%s",
0171              drvname) >= CRYPTO_MAX_ALG_NAME)
0172         goto out_free_salg;
0173
0174     alg->base.cra_flags = CRYPTO_ALG_ASYNC |
0175         (ialg->base.cra_flags & CRYPTO_ALG_INHERITED_FLAGS);
0176     alg->base.cra_priority = ialg->base.cra_priority;
0177     alg->base.cra_blocksize = ialg->base.cra_blocksize;
0178     alg->base.cra_alignmask = ialg->base.cra_alignmask;
0179     alg->base.cra_module = ialg->base.cra_module;
0180     alg->base.cra_ctxsize = sizeof(struct simd_skcipher_ctx);
0181
0182     alg->ivsize = ialg->ivsize;
0183     alg->chunksize = ialg->chunksize;
0184     alg->min_keysize = ialg->min_keysize;
0185     alg->max_keysize = ialg->max_keysize;
0186
0187     alg->init = simd_skcipher_init;
0188     alg->exit = simd_skcipher_exit;
0189
0190     alg->setkey = simd_skcipher_setkey;
0191     alg->encrypt = simd_skcipher_encrypt;
0192     alg->decrypt = simd_skcipher_decrypt;
0193
0194     err = crypto_register_skcipher(alg);
0195     if (err)
0196         goto out_free_salg;
0197
0198 out_put_tfm:
0199     crypto_free_skcipher(tfm);
0200     return salg;
0201
0202 out_free_salg:
0203     kfree(salg);
0204     salg = ERR_PTR(err);
0205     goto out_put_tfm;
0206 }
0207 EXPORT_SYMBOL_GPL(simd_skcipher_create_compat);
0208
0209 struct simd_skcipher_alg *simd_skcipher_create(const char *algname,
0210                            const char *basename)
0211 {
0212     char drvname[CRYPTO_MAX_ALG_NAME];
0213
0214     if (snprintf(drvname, CRYPTO_MAX_ALG_NAME, "simd-%s", basename) >=
0215         CRYPTO_MAX_ALG_NAME)
0216         return ERR_PTR(-ENAMETOOLONG);
0217
0218     return simd_skcipher_create_compat(algname, drvname, basename);
0219 }
0220 EXPORT_SYMBOL_GPL(simd_skcipher_create);
0221
0222 void simd_skcipher_free(struct simd_skcipher_alg *salg)
0223 {
0224     crypto_unregister_skcipher(&salg->alg);
0225     kfree(salg);
0226 }
0227 EXPORT_SYMBOL_GPL(simd_skcipher_free);
0228
0229 int simd_register_skciphers_compat(struct skcipher_alg *algs, int count,
0230                    struct simd_skcipher_alg **simd_algs)
0231 {
0232     int err;
0233     int i;
0234     const char *algname;
0235     const char *drvname;
0236     const char *basename;
0237     struct simd_skcipher_alg *simd;
0238
0239     err = crypto_register_skciphers(algs, count);
0240     if (err)
0241         return err;
0242
0243     for (i = 0; i < count; i++) {
0244         WARN_ON(strncmp(algs[i].base.cra_name, "__", 2));
0245         WARN_ON(strncmp(algs[i].base.cra_driver_name, "__", 2));
0246         algname = algs[i].base.cra_name + 2;
0247         drvname = algs[i].base.cra_driver_name + 2;
0248         basename = algs[i].base.cra_driver_name;
0249         simd = simd_skcipher_create_compat(algname, drvname, basename);
0250         err = PTR_ERR(simd);
0251         if (IS_ERR(simd))
0252             goto err_unregister;
0253         simd_algs[i] = simd;
0254     }
0255     return 0;
0256
0257 err_unregister:
0258     simd_unregister_skciphers(algs, count, simd_algs);
0259     return err;
0260 }
0261 EXPORT_SYMBOL_GPL(simd_register_skciphers_compat);
0262
0263 void simd_unregister_skciphers(struct skcipher_alg *algs, int count,
0264                    struct simd_skcipher_alg **simd_algs)
0265 {
0266     int i;
0267
0268     crypto_unregister_skciphers(algs, count);
0269
0270     for (i = 0; i < count; i++) {
0271         if (simd_algs[i]) {
0272             simd_skcipher_free(simd_algs[i]);
0273             simd_algs[i] = NULL;
0274         }
0275     }
0276 }
0277 EXPORT_SYMBOL_GPL(simd_unregister_skciphers);
0278
0279 /* AEAD support */
0280
0281 struct simd_aead_alg {
0282     const char *ialg_name;
0283     struct aead_alg alg;
0284 };
0285
0286 struct simd_aead_ctx {
0287     struct cryptd_aead *cryptd_tfm;
0288 };
0289
0290 static int simd_aead_setkey(struct crypto_aead *tfm, const u8 *key,
0291                 unsigned int key_len)
0292 {
0293     struct simd_aead_ctx *ctx = crypto_aead_ctx(tfm);
0294     struct crypto_aead *child = &ctx->cryptd_tfm->base;
0295
0296     crypto_aead_clear_flags(child, CRYPTO_TFM_REQ_MASK);
0297     crypto_aead_set_flags(child, crypto_aead_get_flags(tfm) &
0298                      CRYPTO_TFM_REQ_MASK);
0299     return crypto_aead_setkey(child, key, key_len);
0300 }
0301
0302 static int simd_aead_setauthsize(struct crypto_aead *tfm, unsigned int authsize)
0303 {
0304     struct simd_aead_ctx *ctx = crypto_aead_ctx(tfm);
0305     struct crypto_aead *child = &ctx->cryptd_tfm->base;
0306
0307     return crypto_aead_setauthsize(child, authsize);
0308 }
0309
0310 static int simd_aead_encrypt(struct aead_request *req)
0311 {
0312     struct crypto_aead *tfm = crypto_aead_reqtfm(req);
0313     struct simd_aead_ctx *ctx = crypto_aead_ctx(tfm);
0314     struct aead_request *subreq;
0315     struct crypto_aead *child;
0316
0317     subreq = aead_request_ctx(req);
0318     *subreq = *req;
0319
0320     if (!crypto_simd_usable() ||
0321         (in_atomic() && cryptd_aead_queued(ctx->cryptd_tfm)))
0322         child = &ctx->cryptd_tfm->base;
0323     else
0324         child = cryptd_aead_child(ctx->cryptd_tfm);
0325
0326     aead_request_set_tfm(subreq, child);
0327
0328     return crypto_aead_encrypt(subreq);
0329 }
0330
0331 static int simd_aead_decrypt(struct aead_request *req)
0332 {
0333     struct crypto_aead *tfm = crypto_aead_reqtfm(req);
0334     struct simd_aead_ctx *ctx = crypto_aead_ctx(tfm);
0335     struct aead_request *subreq;
0336     struct crypto_aead *child;
0337
0338     subreq = aead_request_ctx(req);
0339     *subreq = *req;
0340
0341     if (!crypto_simd_usable() ||
0342         (in_atomic() && cryptd_aead_queued(ctx->cryptd_tfm)))
0343         child = &ctx->cryptd_tfm->base;
0344     else
0345         child = cryptd_aead_child(ctx->cryptd_tfm);
0346
0347     aead_request_set_tfm(subreq, child);
0348
0349     return crypto_aead_decrypt(subreq);
0350 }
0351
0352 static void simd_aead_exit(struct crypto_aead *tfm)
0353 {
0354     struct simd_aead_ctx *ctx = crypto_aead_ctx(tfm);
0355
0356     cryptd_free_aead(ctx->cryptd_tfm);
0357 }
0358
0359 static int simd_aead_init(struct crypto_aead *tfm)
0360 {
0361     struct simd_aead_ctx *ctx = crypto_aead_ctx(tfm);
0362     struct cryptd_aead *cryptd_tfm;
0363     struct simd_aead_alg *salg;
0364     struct aead_alg *alg;
0365     unsigned reqsize;
0366
0367     alg = crypto_aead_alg(tfm);
0368     salg = container_of(alg, struct simd_aead_alg, alg);
0369
0370     cryptd_tfm = cryptd_alloc_aead(salg->ialg_name, CRYPTO_ALG_INTERNAL,
0371                        CRYPTO_ALG_INTERNAL);
0372     if (IS_ERR(cryptd_tfm))
0373         return PTR_ERR(cryptd_tfm);
0374
0375     ctx->cryptd_tfm = cryptd_tfm;
0376
0377     reqsize = crypto_aead_reqsize(cryptd_aead_child(cryptd_tfm));
0378     reqsize = max(reqsize, crypto_aead_reqsize(&cryptd_tfm->base));
0379     reqsize += sizeof(struct aead_request);
0380
0381     crypto_aead_set_reqsize(tfm, reqsize);
0382
0383     return 0;
0384 }
0385
0386 struct simd_aead_alg *simd_aead_create_compat(const char *algname,
0387                           const char *drvname,
0388                           const char *basename)
0389 {
0390     struct simd_aead_alg *salg;
0391     struct crypto_aead *tfm;
0392     struct aead_alg *ialg;
0393     struct aead_alg *alg;
0394     int err;
0395
0396     tfm = crypto_alloc_aead(basename, CRYPTO_ALG_INTERNAL,
0397                 CRYPTO_ALG_INTERNAL | CRYPTO_ALG_ASYNC);
0398     if (IS_ERR(tfm))
0399         return ERR_CAST(tfm);
0400
0401     ialg = crypto_aead_alg(tfm);
0402
0403     salg = kzalloc(sizeof(*salg), GFP_KERNEL);
0404     if (!salg) {
0405         salg = ERR_PTR(-ENOMEM);
0406         goto out_put_tfm;
0407     }
0408
0409     salg->ialg_name = basename;
0410     alg = &salg->alg;
0411
0412     err = -ENAMETOOLONG;
0413     if (snprintf(alg->base.cra_name, CRYPTO_MAX_ALG_NAME, "%s", algname) >=
0414         CRYPTO_MAX_ALG_NAME)
0415         goto out_free_salg;
0416
0417     if (snprintf(alg->base.cra_driver_name, CRYPTO_MAX_ALG_NAME, "%s",
0418              drvname) >= CRYPTO_MAX_ALG_NAME)
0419         goto out_free_salg;
0420
0421     alg->base.cra_flags = CRYPTO_ALG_ASYNC |
0422         (ialg->base.cra_flags & CRYPTO_ALG_INHERITED_FLAGS);
0423     alg->base.cra_priority = ialg->base.cra_priority;
0424     alg->base.cra_blocksize = ialg->base.cra_blocksize;
0425     alg->base.cra_alignmask = ialg->base.cra_alignmask;
0426     alg->base.cra_module = ialg->base.cra_module;
0427     alg->base.cra_ctxsize = sizeof(struct simd_aead_ctx);
0428
0429     alg->ivsize = ialg->ivsize;
0430     alg->maxauthsize = ialg->maxauthsize;
0431     alg->chunksize = ialg->chunksize;
0432
0433     alg->init = simd_aead_init;
0434     alg->exit = simd_aead_exit;
0435
0436     alg->setkey = simd_aead_setkey;
0437     alg->setauthsize = simd_aead_setauthsize;
0438     alg->encrypt = simd_aead_encrypt;
0439     alg->decrypt = simd_aead_decrypt;
0440
0441     err = crypto_register_aead(alg);
0442     if (err)
0443         goto out_free_salg;
0444
0445 out_put_tfm:
0446     crypto_free_aead(tfm);
0447     return salg;
0448
0449 out_free_salg:
0450     kfree(salg);
0451     salg = ERR_PTR(err);
0452     goto out_put_tfm;
0453 }
0454 EXPORT_SYMBOL_GPL(simd_aead_create_compat);
0455
0456 struct simd_aead_alg *simd_aead_create(const char *algname,
0457                        const char *basename)
0458 {
0459     char drvname[CRYPTO_MAX_ALG_NAME];
0460
0461     if (snprintf(drvname, CRYPTO_MAX_ALG_NAME, "simd-%s", basename) >=
0462         CRYPTO_MAX_ALG_NAME)
0463         return ERR_PTR(-ENAMETOOLONG);
0464
0465     return simd_aead_create_compat(algname, drvname, basename);
0466 }
0467 EXPORT_SYMBOL_GPL(simd_aead_create);
0468
0469 void simd_aead_free(struct simd_aead_alg *salg)
0470 {
0471     crypto_unregister_aead(&salg->alg);
0472     kfree(salg);
0473 }
0474 EXPORT_SYMBOL_GPL(simd_aead_free);
0475
0476 int simd_register_aeads_compat(struct aead_alg *algs, int count,
0477                    struct simd_aead_alg **simd_algs)
0478 {
0479     int err;
0480     int i;
0481     const char *algname;
0482     const char *drvname;
0483     const char *basename;
0484     struct simd_aead_alg *simd;
0485
0486     err = crypto_register_aeads(algs, count);
0487     if (err)
0488         return err;
0489
0490     for (i = 0; i < count; i++) {
0491         WARN_ON(strncmp(algs[i].base.cra_name, "__", 2));
0492         WARN_ON(strncmp(algs[i].base.cra_driver_name, "__", 2));
0493         algname = algs[i].base.cra_name + 2;
0494         drvname = algs[i].base.cra_driver_name + 2;
0495         basename = algs[i].base.cra_driver_name;
0496         simd = simd_aead_create_compat(algname, drvname, basename);
0497         err = PTR_ERR(simd);
0498         if (IS_ERR(simd))
0499             goto err_unregister;
0500         simd_algs[i] = simd;
0501     }
0502     return 0;
0503
0504 err_unregister:
0505     simd_unregister_aeads(algs, count, simd_algs);
0506     return err;
0507 }
0508 EXPORT_SYMBOL_GPL(simd_register_aeads_compat);
0509
0510 void simd_unregister_aeads(struct aead_alg *algs, int count,
0511                struct simd_aead_alg **simd_algs)
0512 {
0513     int i;
0514
0515     crypto_unregister_aeads(algs, count);
0516
0517     for (i = 0; i < count; i++) {
0518         if (simd_algs[i]) {
0519             simd_aead_free(simd_algs[i]);
0520             simd_algs[i] = NULL;
0521         }
0522     }
0523 }
0524 EXPORT_SYMBOL_GPL(simd_unregister_aeads);
0525
0526 MODULE_LICENSE("GPL");