the-tree/crypto/algapi.c

0001 // SPDX-License-Identifier: GPL-2.0-or-later
0002 /*
0003  * Cryptographic API for algorithms (i.e., low-level API).
0004  *
0005  * Copyright (c) 2006 Herbert Xu <herbert@gondor.apana.org.au>
0006  */
0007
0008 #include <crypto/algapi.h>
0009 #include <crypto/internal/simd.h>
0010 #include <linux/err.h>
0011 #include <linux/errno.h>
0012 #include <linux/fips.h>
0013 #include <linux/init.h>
0014 #include <linux/kernel.h>
0015 #include <linux/list.h>
0016 #include <linux/module.h>
0017 #include <linux/rtnetlink.h>
0018 #include <linux/slab.h>
0019 #include <linux/string.h>
0020
0021 #include "internal.h"
0022
0023 static LIST_HEAD(crypto_template_list);
0024
0025 #ifdef CONFIG_CRYPTO_MANAGER_EXTRA_TESTS
0026 DEFINE_PER_CPU(bool, crypto_simd_disabled_for_test);
0027 EXPORT_PER_CPU_SYMBOL_GPL(crypto_simd_disabled_for_test);
0028 #endif
0029
0030 static inline void crypto_check_module_sig(struct module *mod)
0031 {
0032     if (fips_enabled && mod && !module_sig_ok(mod))
0033         panic("Module %s signature verification failed in FIPS mode\n",
0034               module_name(mod));
0035 }
0036
0037 static int crypto_check_alg(struct crypto_alg *alg)
0038 {
0039     crypto_check_module_sig(alg->cra_module);
0040
0041     if (!alg->cra_name[0] || !alg->cra_driver_name[0])
0042         return -EINVAL;
0043
0044     if (alg->cra_alignmask & (alg->cra_alignmask + 1))
0045         return -EINVAL;
0046
0047     /* General maximums for all algs. */
0048     if (alg->cra_alignmask > MAX_ALGAPI_ALIGNMASK)
0049         return -EINVAL;
0050
0051     if (alg->cra_blocksize > MAX_ALGAPI_BLOCKSIZE)
0052         return -EINVAL;
0053
0054     /* Lower maximums for specific alg types. */
0055     if (!alg->cra_type && (alg->cra_flags & CRYPTO_ALG_TYPE_MASK) ==
0056                    CRYPTO_ALG_TYPE_CIPHER) {
0057         if (alg->cra_alignmask > MAX_CIPHER_ALIGNMASK)
0058             return -EINVAL;
0059
0060         if (alg->cra_blocksize > MAX_CIPHER_BLOCKSIZE)
0061             return -EINVAL;
0062     }
0063
0064     if (alg->cra_priority < 0)
0065         return -EINVAL;
0066
0067     refcount_set(&alg->cra_refcnt, 1);
0068
0069     return 0;
0070 }
0071
0072 static void crypto_free_instance(struct crypto_instance *inst)
0073 {
0074     inst->alg.cra_type->free(inst);
0075 }
0076
0077 static void crypto_destroy_instance(struct crypto_alg *alg)
0078 {
0079     struct crypto_instance *inst = (void *)alg;
0080     struct crypto_template *tmpl = inst->tmpl;
0081
0082     crypto_free_instance(inst);
0083     crypto_tmpl_put(tmpl);
0084 }
0085
0086 /*
0087  * This function adds a spawn to the list secondary_spawns which
0088  * will be used at the end of crypto_remove_spawns to unregister
0089  * instances, unless the spawn happens to be one that is depended
0090  * on by the new algorithm (nalg in crypto_remove_spawns).
0091  *
0092  * This function is also responsible for resurrecting any algorithms
0093  * in the dependency chain of nalg by unsetting n->dead.
0094  */
0095 static struct list_head *crypto_more_spawns(struct crypto_alg *alg,
0096                         struct list_head *stack,
0097                         struct list_head *top,
0098                         struct list_head *secondary_spawns)
0099 {
0100     struct crypto_spawn *spawn, *n;
0101
0102     spawn = list_first_entry_or_null(stack, struct crypto_spawn, list);
0103     if (!spawn)
0104         return NULL;
0105
0106     n = list_prev_entry(spawn, list);
0107     list_move(&spawn->list, secondary_spawns);
0108
0109     if (list_is_last(&n->list, stack))
0110         return top;
0111
0112     n = list_next_entry(n, list);
0113     if (!spawn->dead)
0114         n->dead = false;
0115
0116     return &n->inst->alg.cra_users;
0117 }
0118
0119 static void crypto_remove_instance(struct crypto_instance *inst,
0120                    struct list_head *list)
0121 {
0122     struct crypto_template *tmpl = inst->tmpl;
0123
0124     if (crypto_is_dead(&inst->alg))
0125         return;
0126
0127     inst->alg.cra_flags |= CRYPTO_ALG_DEAD;
0128
0129     if (!tmpl || !crypto_tmpl_get(tmpl))
0130         return;
0131
0132     list_move(&inst->alg.cra_list, list);
0133     hlist_del(&inst->list);
0134     inst->alg.cra_destroy = crypto_destroy_instance;
0135
0136     BUG_ON(!list_empty(&inst->alg.cra_users));
0137 }
0138
0139 /*
0140  * Given an algorithm alg, remove all algorithms that depend on it
0141  * through spawns.  If nalg is not null, then exempt any algorithms
0142  * that is depended on by nalg.  This is useful when nalg itself
0143  * depends on alg.
0144  */
0145 void crypto_remove_spawns(struct crypto_alg *alg, struct list_head *list,
0146               struct crypto_alg *nalg)
0147 {
0148     u32 new_type = (nalg ?: alg)->cra_flags;
0149     struct crypto_spawn *spawn, *n;
0150     LIST_HEAD(secondary_spawns);
0151     struct list_head *spawns;
0152     LIST_HEAD(stack);
0153     LIST_HEAD(top);
0154
0155     spawns = &alg->cra_users;
0156     list_for_each_entry_safe(spawn, n, spawns, list) {
0157         if ((spawn->alg->cra_flags ^ new_type) & spawn->mask)
0158             continue;
0159
0160         list_move(&spawn->list, &top);
0161     }
0162
0163     /*
0164      * Perform a depth-first walk starting from alg through
0165      * the cra_users tree.  The list stack records the path
0166      * from alg to the current spawn.
0167      */
0168     spawns = &top;
0169     do {
0170         while (!list_empty(spawns)) {
0171             struct crypto_instance *inst;
0172
0173             spawn = list_first_entry(spawns, struct crypto_spawn,
0174                          list);
0175             inst = spawn->inst;
0176
0177             list_move(&spawn->list, &stack);
0178             spawn->dead = !spawn->registered || &inst->alg != nalg;
0179
0180             if (!spawn->registered)
0181                 break;
0182
0183             BUG_ON(&inst->alg == alg);
0184
0185             if (&inst->alg == nalg)
0186                 break;
0187
0188             spawns = &inst->alg.cra_users;
0189
0190             /*
0191              * Even if spawn->registered is true, the
0192              * instance itself may still be unregistered.
0193              * This is because it may have failed during
0194              * registration.  Therefore we still need to
0195              * make the following test.
0196              *
0197              * We may encounter an unregistered instance here, since
0198              * an instance's spawns are set up prior to the instance
0199              * being registered.  An unregistered instance will have
0200              * NULL ->cra_users.next, since ->cra_users isn't
0201              * properly initialized until registration.  But an
0202              * unregistered instance cannot have any users, so treat
0203              * it the same as ->cra_users being empty.
0204              */
0205             if (spawns->next == NULL)
0206                 break;
0207         }
0208     } while ((spawns = crypto_more_spawns(alg, &stack, &top,
0209                           &secondary_spawns)));
0210
0211     /*
0212      * Remove all instances that are marked as dead.  Also
0213      * complete the resurrection of the others by moving them
0214      * back to the cra_users list.
0215      */
0216     list_for_each_entry_safe(spawn, n, &secondary_spawns, list) {
0217         if (!spawn->dead)
0218             list_move(&spawn->list, &spawn->alg->cra_users);
0219         else if (spawn->registered)
0220             crypto_remove_instance(spawn->inst, list);
0221     }
0222 }
0223 EXPORT_SYMBOL_GPL(crypto_remove_spawns);
0224
0225 static struct crypto_larval *crypto_alloc_test_larval(struct crypto_alg *alg)
0226 {
0227     struct crypto_larval *larval;
0228
0229     if (!IS_ENABLED(CONFIG_CRYPTO_MANAGER))
0230         return NULL;
0231
0232     larval = crypto_larval_alloc(alg->cra_name,
0233                      alg->cra_flags | CRYPTO_ALG_TESTED, 0);
0234     if (IS_ERR(larval))
0235         return larval;
0236
0237     larval->adult = crypto_mod_get(alg);
0238     if (!larval->adult) {
0239         kfree(larval);
0240         return ERR_PTR(-ENOENT);
0241     }
0242
0243     refcount_set(&larval->alg.cra_refcnt, 1);
0244     memcpy(larval->alg.cra_driver_name, alg->cra_driver_name,
0245            CRYPTO_MAX_ALG_NAME);
0246     larval->alg.cra_priority = alg->cra_priority;
0247
0248     return larval;
0249 }
0250
0251 static struct crypto_larval *__crypto_register_alg(struct crypto_alg *alg)
0252 {
0253     struct crypto_alg *q;
0254     struct crypto_larval *larval;
0255     int ret = -EAGAIN;
0256
0257     if (crypto_is_dead(alg))
0258         goto err;
0259
0260     INIT_LIST_HEAD(&alg->cra_users);
0261
0262     /* No cheating! */
0263     alg->cra_flags &= ~CRYPTO_ALG_TESTED;
0264
0265     ret = -EEXIST;
0266
0267     list_for_each_entry(q, &crypto_alg_list, cra_list) {
0268         if (q == alg)
0269             goto err;
0270
0271         if (crypto_is_moribund(q))
0272             continue;
0273
0274         if (crypto_is_larval(q)) {
0275             if (!strcmp(alg->cra_driver_name, q->cra_driver_name))
0276                 goto err;
0277             continue;
0278         }
0279
0280         if (!strcmp(q->cra_driver_name, alg->cra_name) ||
0281             !strcmp(q->cra_name, alg->cra_driver_name))
0282             goto err;
0283     }
0284
0285     larval = crypto_alloc_test_larval(alg);
0286     if (IS_ERR(larval))
0287         goto out;
0288
0289     list_add(&alg->cra_list, &crypto_alg_list);
0290
0291     if (larval)
0292         list_add(&larval->alg.cra_list, &crypto_alg_list);
0293     else
0294         alg->cra_flags |= CRYPTO_ALG_TESTED;
0295
0296     crypto_stats_init(alg);
0297
0298 out:
0299     return larval;
0300
0301 err:
0302     larval = ERR_PTR(ret);
0303     goto out;
0304 }
0305
0306 void crypto_alg_tested(const char *name, int err)
0307 {
0308     struct crypto_larval *test;
0309     struct crypto_alg *alg;
0310     struct crypto_alg *q;
0311     LIST_HEAD(list);
0312     bool best;
0313
0314     down_write(&crypto_alg_sem);
0315     list_for_each_entry(q, &crypto_alg_list, cra_list) {
0316         if (crypto_is_moribund(q) || !crypto_is_larval(q))
0317             continue;
0318
0319         test = (struct crypto_larval *)q;
0320
0321         if (!strcmp(q->cra_driver_name, name))
0322             goto found;
0323     }
0324
0325     pr_err("alg: Unexpected test result for %s: %d\n", name, err);
0326     goto unlock;
0327
0328 found:
0329     q->cra_flags |= CRYPTO_ALG_DEAD;
0330     alg = test->adult;
0331
0332     if (list_empty(&alg->cra_list))
0333         goto complete;
0334
0335     if (err == -ECANCELED)
0336         alg->cra_flags |= CRYPTO_ALG_FIPS_INTERNAL;
0337     else if (err)
0338         goto complete;
0339     else
0340         alg->cra_flags &= ~CRYPTO_ALG_FIPS_INTERNAL;
0341
0342     alg->cra_flags |= CRYPTO_ALG_TESTED;
0343
0344     /* Only satisfy larval waiters if we are the best. */
0345     best = true;
0346     list_for_each_entry(q, &crypto_alg_list, cra_list) {
0347         if (crypto_is_moribund(q) || !crypto_is_larval(q))
0348             continue;
0349
0350         if (strcmp(alg->cra_name, q->cra_name))
0351             continue;
0352
0353         if (q->cra_priority > alg->cra_priority) {
0354             best = false;
0355             break;
0356         }
0357     }
0358
0359     list_for_each_entry(q, &crypto_alg_list, cra_list) {
0360         if (q == alg)
0361             continue;
0362
0363         if (crypto_is_moribund(q))
0364             continue;
0365
0366         if (crypto_is_larval(q)) {
0367             struct crypto_larval *larval = (void *)q;
0368
0369             /*
0370              * Check to see if either our generic name or
0371              * specific name can satisfy the name requested
0372              * by the larval entry q.
0373              */
0374             if (strcmp(alg->cra_name, q->cra_name) &&
0375                 strcmp(alg->cra_driver_name, q->cra_name))
0376                 continue;
0377
0378             if (larval->adult)
0379                 continue;
0380             if ((q->cra_flags ^ alg->cra_flags) & larval->mask)
0381                 continue;
0382
0383             if (best && crypto_mod_get(alg))
0384                 larval->adult = alg;
0385             else
0386                 larval->adult = ERR_PTR(-EAGAIN);
0387
0388             continue;
0389         }
0390
0391         if (strcmp(alg->cra_name, q->cra_name))
0392             continue;
0393
0394         if (strcmp(alg->cra_driver_name, q->cra_driver_name) &&
0395             q->cra_priority > alg->cra_priority)
0396             continue;
0397
0398         crypto_remove_spawns(q, &list, alg);
0399     }
0400
0401 complete:
0402     complete_all(&test->completion);
0403
0404 unlock:
0405     up_write(&crypto_alg_sem);
0406
0407     crypto_remove_final(&list);
0408 }
0409 EXPORT_SYMBOL_GPL(crypto_alg_tested);
0410
0411 void crypto_remove_final(struct list_head *list)
0412 {
0413     struct crypto_alg *alg;
0414     struct crypto_alg *n;
0415
0416     list_for_each_entry_safe(alg, n, list, cra_list) {
0417         list_del_init(&alg->cra_list);
0418         crypto_alg_put(alg);
0419     }
0420 }
0421 EXPORT_SYMBOL_GPL(crypto_remove_final);
0422
0423 int crypto_register_alg(struct crypto_alg *alg)
0424 {
0425     struct crypto_larval *larval;
0426     bool test_started;
0427     int err;
0428
0429     alg->cra_flags &= ~CRYPTO_ALG_DEAD;
0430     err = crypto_check_alg(alg);
0431     if (err)
0432         return err;
0433
0434     down_write(&crypto_alg_sem);
0435     larval = __crypto_register_alg(alg);
0436     test_started = static_key_enabled(&crypto_boot_test_finished);
0437     if (!IS_ERR_OR_NULL(larval))
0438         larval->test_started = test_started;
0439     up_write(&crypto_alg_sem);
0440
0441     if (IS_ERR_OR_NULL(larval))
0442         return PTR_ERR(larval);
0443
0444     if (test_started)
0445         crypto_wait_for_test(larval);
0446     return 0;
0447 }
0448 EXPORT_SYMBOL_GPL(crypto_register_alg);
0449
0450 static int crypto_remove_alg(struct crypto_alg *alg, struct list_head *list)
0451 {
0452     if (unlikely(list_empty(&alg->cra_list)))
0453         return -ENOENT;
0454
0455     alg->cra_flags |= CRYPTO_ALG_DEAD;
0456
0457     list_del_init(&alg->cra_list);
0458     crypto_remove_spawns(alg, list, NULL);
0459
0460     return 0;
0461 }
0462
0463 void crypto_unregister_alg(struct crypto_alg *alg)
0464 {
0465     int ret;
0466     LIST_HEAD(list);
0467
0468     down_write(&crypto_alg_sem);
0469     ret = crypto_remove_alg(alg, &list);
0470     up_write(&crypto_alg_sem);
0471
0472     if (WARN(ret, "Algorithm %s is not registered", alg->cra_driver_name))
0473         return;
0474
0475     BUG_ON(refcount_read(&alg->cra_refcnt) != 1);
0476     if (alg->cra_destroy)
0477         alg->cra_destroy(alg);
0478
0479     crypto_remove_final(&list);
0480 }
0481 EXPORT_SYMBOL_GPL(crypto_unregister_alg);
0482
0483 int crypto_register_algs(struct crypto_alg *algs, int count)
0484 {
0485     int i, ret;
0486
0487     for (i = 0; i < count; i++) {
0488         ret = crypto_register_alg(&algs[i]);
0489         if (ret)
0490             goto err;
0491     }
0492
0493     return 0;
0494
0495 err:
0496     for (--i; i >= 0; --i)
0497         crypto_unregister_alg(&algs[i]);
0498
0499     return ret;
0500 }
0501 EXPORT_SYMBOL_GPL(crypto_register_algs);
0502
0503 void crypto_unregister_algs(struct crypto_alg *algs, int count)
0504 {
0505     int i;
0506
0507     for (i = 0; i < count; i++)
0508         crypto_unregister_alg(&algs[i]);
0509 }
0510 EXPORT_SYMBOL_GPL(crypto_unregister_algs);
0511
0512 int crypto_register_template(struct crypto_template *tmpl)
0513 {
0514     struct crypto_template *q;
0515     int err = -EEXIST;
0516
0517     down_write(&crypto_alg_sem);
0518
0519     crypto_check_module_sig(tmpl->module);
0520
0521     list_for_each_entry(q, &crypto_template_list, list) {
0522         if (q == tmpl)
0523             goto out;
0524     }
0525
0526     list_add(&tmpl->list, &crypto_template_list);
0527     err = 0;
0528 out:
0529     up_write(&crypto_alg_sem);
0530     return err;
0531 }
0532 EXPORT_SYMBOL_GPL(crypto_register_template);
0533
0534 int crypto_register_templates(struct crypto_template *tmpls, int count)
0535 {
0536     int i, err;
0537
0538     for (i = 0; i < count; i++) {
0539         err = crypto_register_template(&tmpls[i]);
0540         if (err)
0541             goto out;
0542     }
0543     return 0;
0544
0545 out:
0546     for (--i; i >= 0; --i)
0547         crypto_unregister_template(&tmpls[i]);
0548     return err;
0549 }
0550 EXPORT_SYMBOL_GPL(crypto_register_templates);
0551
0552 void crypto_unregister_template(struct crypto_template *tmpl)
0553 {
0554     struct crypto_instance *inst;
0555     struct hlist_node *n;
0556     struct hlist_head *list;
0557     LIST_HEAD(users);
0558
0559     down_write(&crypto_alg_sem);
0560
0561     BUG_ON(list_empty(&tmpl->list));
0562     list_del_init(&tmpl->list);
0563
0564     list = &tmpl->instances;
0565     hlist_for_each_entry(inst, list, list) {
0566         int err = crypto_remove_alg(&inst->alg, &users);
0567
0568         BUG_ON(err);
0569     }
0570
0571     up_write(&crypto_alg_sem);
0572
0573     hlist_for_each_entry_safe(inst, n, list, list) {
0574         BUG_ON(refcount_read(&inst->alg.cra_refcnt) != 1);
0575         crypto_free_instance(inst);
0576     }
0577     crypto_remove_final(&users);
0578 }
0579 EXPORT_SYMBOL_GPL(crypto_unregister_template);
0580
0581 void crypto_unregister_templates(struct crypto_template *tmpls, int count)
0582 {
0583     int i;
0584
0585     for (i = count - 1; i >= 0; --i)
0586         crypto_unregister_template(&tmpls[i]);
0587 }
0588 EXPORT_SYMBOL_GPL(crypto_unregister_templates);
0589
0590 static struct crypto_template *__crypto_lookup_template(const char *name)
0591 {
0592     struct crypto_template *q, *tmpl = NULL;
0593
0594     down_read(&crypto_alg_sem);
0595     list_for_each_entry(q, &crypto_template_list, list) {
0596         if (strcmp(q->name, name))
0597             continue;
0598         if (unlikely(!crypto_tmpl_get(q)))
0599             continue;
0600
0601         tmpl = q;
0602         break;
0603     }
0604     up_read(&crypto_alg_sem);
0605
0606     return tmpl;
0607 }
0608
0609 struct crypto_template *crypto_lookup_template(const char *name)
0610 {
0611     return try_then_request_module(__crypto_lookup_template(name),
0612                        "crypto-%s", name);
0613 }
0614 EXPORT_SYMBOL_GPL(crypto_lookup_template);
0615
0616 int crypto_register_instance(struct crypto_template *tmpl,
0617                  struct crypto_instance *inst)
0618 {
0619     struct crypto_larval *larval;
0620     struct crypto_spawn *spawn;
0621     u32 fips_internal = 0;
0622     int err;
0623
0624     err = crypto_check_alg(&inst->alg);
0625     if (err)
0626         return err;
0627
0628     inst->alg.cra_module = tmpl->module;
0629     inst->alg.cra_flags |= CRYPTO_ALG_INSTANCE;
0630
0631     down_write(&crypto_alg_sem);
0632
0633     larval = ERR_PTR(-EAGAIN);
0634     for (spawn = inst->spawns; spawn;) {
0635         struct crypto_spawn *next;
0636
0637         if (spawn->dead)
0638             goto unlock;
0639
0640         next = spawn->next;
0641         spawn->inst = inst;
0642         spawn->registered = true;
0643
0644         fips_internal |= spawn->alg->cra_flags;
0645
0646         crypto_mod_put(spawn->alg);
0647
0648         spawn = next;
0649     }
0650
0651     inst->alg.cra_flags |= (fips_internal & CRYPTO_ALG_FIPS_INTERNAL);
0652
0653     larval = __crypto_register_alg(&inst->alg);
0654     if (IS_ERR(larval))
0655         goto unlock;
0656     else if (larval)
0657         larval->test_started = true;
0658
0659     hlist_add_head(&inst->list, &tmpl->instances);
0660     inst->tmpl = tmpl;
0661
0662 unlock:
0663     up_write(&crypto_alg_sem);
0664
0665     err = PTR_ERR(larval);
0666     if (IS_ERR_OR_NULL(larval))
0667         goto err;
0668
0669     crypto_wait_for_test(larval);
0670     err = 0;
0671
0672 err:
0673     return err;
0674 }
0675 EXPORT_SYMBOL_GPL(crypto_register_instance);
0676
0677 void crypto_unregister_instance(struct crypto_instance *inst)
0678 {
0679     LIST_HEAD(list);
0680
0681     down_write(&crypto_alg_sem);
0682
0683     crypto_remove_spawns(&inst->alg, &list, NULL);
0684     crypto_remove_instance(inst, &list);
0685
0686     up_write(&crypto_alg_sem);
0687
0688     crypto_remove_final(&list);
0689 }
0690 EXPORT_SYMBOL_GPL(crypto_unregister_instance);
0691
0692 int crypto_grab_spawn(struct crypto_spawn *spawn, struct crypto_instance *inst,
0693               const char *name, u32 type, u32 mask)
0694 {
0695     struct crypto_alg *alg;
0696     int err = -EAGAIN;
0697
0698     if (WARN_ON_ONCE(inst == NULL))
0699         return -EINVAL;
0700
0701     /* Allow the result of crypto_attr_alg_name() to be passed directly */
0702     if (IS_ERR(name))
0703         return PTR_ERR(name);
0704
0705     alg = crypto_find_alg(name, spawn->frontend,
0706                   type | CRYPTO_ALG_FIPS_INTERNAL, mask);
0707     if (IS_ERR(alg))
0708         return PTR_ERR(alg);
0709
0710     down_write(&crypto_alg_sem);
0711     if (!crypto_is_moribund(alg)) {
0712         list_add(&spawn->list, &alg->cra_users);
0713         spawn->alg = alg;
0714         spawn->mask = mask;
0715         spawn->next = inst->spawns;
0716         inst->spawns = spawn;
0717         inst->alg.cra_flags |=
0718             (alg->cra_flags & CRYPTO_ALG_INHERITED_FLAGS);
0719         err = 0;
0720     }
0721     up_write(&crypto_alg_sem);
0722     if (err)
0723         crypto_mod_put(alg);
0724     return err;
0725 }
0726 EXPORT_SYMBOL_GPL(crypto_grab_spawn);
0727
0728 void crypto_drop_spawn(struct crypto_spawn *spawn)
0729 {
0730     if (!spawn->alg) /* not yet initialized? */
0731         return;
0732
0733     down_write(&crypto_alg_sem);
0734     if (!spawn->dead)
0735         list_del(&spawn->list);
0736     up_write(&crypto_alg_sem);
0737
0738     if (!spawn->registered)
0739         crypto_mod_put(spawn->alg);
0740 }
0741 EXPORT_SYMBOL_GPL(crypto_drop_spawn);
0742
0743 static struct crypto_alg *crypto_spawn_alg(struct crypto_spawn *spawn)
0744 {
0745     struct crypto_alg *alg = ERR_PTR(-EAGAIN);
0746     struct crypto_alg *target;
0747     bool shoot = false;
0748
0749     down_read(&crypto_alg_sem);
0750     if (!spawn->dead) {
0751         alg = spawn->alg;
0752         if (!crypto_mod_get(alg)) {
0753             target = crypto_alg_get(alg);
0754             shoot = true;
0755             alg = ERR_PTR(-EAGAIN);
0756         }
0757     }
0758     up_read(&crypto_alg_sem);
0759
0760     if (shoot) {
0761         crypto_shoot_alg(target);
0762         crypto_alg_put(target);
0763     }
0764
0765     return alg;
0766 }
0767
0768 struct crypto_tfm *crypto_spawn_tfm(struct crypto_spawn *spawn, u32 type,
0769                     u32 mask)
0770 {
0771     struct crypto_alg *alg;
0772     struct crypto_tfm *tfm;
0773
0774     alg = crypto_spawn_alg(spawn);
0775     if (IS_ERR(alg))
0776         return ERR_CAST(alg);
0777
0778     tfm = ERR_PTR(-EINVAL);
0779     if (unlikely((alg->cra_flags ^ type) & mask))
0780         goto out_put_alg;
0781
0782     tfm = __crypto_alloc_tfm(alg, type, mask);
0783     if (IS_ERR(tfm))
0784         goto out_put_alg;
0785
0786     return tfm;
0787
0788 out_put_alg:
0789     crypto_mod_put(alg);
0790     return tfm;
0791 }
0792 EXPORT_SYMBOL_GPL(crypto_spawn_tfm);
0793
0794 void *crypto_spawn_tfm2(struct crypto_spawn *spawn)
0795 {
0796     struct crypto_alg *alg;
0797     struct crypto_tfm *tfm;
0798
0799     alg = crypto_spawn_alg(spawn);
0800     if (IS_ERR(alg))
0801         return ERR_CAST(alg);
0802
0803     tfm = crypto_create_tfm(alg, spawn->frontend);
0804     if (IS_ERR(tfm))
0805         goto out_put_alg;
0806
0807     return tfm;
0808
0809 out_put_alg:
0810     crypto_mod_put(alg);
0811     return tfm;
0812 }
0813 EXPORT_SYMBOL_GPL(crypto_spawn_tfm2);
0814
0815 int crypto_register_notifier(struct notifier_block *nb)
0816 {
0817     return blocking_notifier_chain_register(&crypto_chain, nb);
0818 }
0819 EXPORT_SYMBOL_GPL(crypto_register_notifier);
0820
0821 int crypto_unregister_notifier(struct notifier_block *nb)
0822 {
0823     return blocking_notifier_chain_unregister(&crypto_chain, nb);
0824 }
0825 EXPORT_SYMBOL_GPL(crypto_unregister_notifier);
0826
0827 struct crypto_attr_type *crypto_get_attr_type(struct rtattr **tb)
0828 {
0829     struct rtattr *rta = tb[0];
0830     struct crypto_attr_type *algt;
0831
0832     if (!rta)
0833         return ERR_PTR(-ENOENT);
0834     if (RTA_PAYLOAD(rta) < sizeof(*algt))
0835         return ERR_PTR(-EINVAL);
0836     if (rta->rta_type != CRYPTOA_TYPE)
0837         return ERR_PTR(-EINVAL);
0838
0839     algt = RTA_DATA(rta);
0840
0841     return algt;
0842 }
0843 EXPORT_SYMBOL_GPL(crypto_get_attr_type);
0844
0845 /**
0846  * crypto_check_attr_type() - check algorithm type and compute inherited mask
0847  * @tb: the template parameters
0848  * @type: the algorithm type the template would be instantiated as
0849  * @mask_ret: (output) the mask that should be passed to crypto_grab_*()
0850  *        to restrict the flags of any inner algorithms
0851  *
0852  * Validate that the algorithm type the user requested is compatible with the
0853  * one the template would actually be instantiated as.  E.g., if the user is
0854  * doing crypto_alloc_shash("cbc(aes)", ...), this would return an error because
0855  * the "cbc" template creates an "skcipher" algorithm, not an "shash" algorithm.
0856  *
0857  * Also compute the mask to use to restrict the flags of any inner algorithms.
0858  *
0859  * Return: 0 on success; -errno on failure
0860  */
0861 int crypto_check_attr_type(struct rtattr **tb, u32 type, u32 *mask_ret)
0862 {
0863     struct crypto_attr_type *algt;
0864
0865     algt = crypto_get_attr_type(tb);
0866     if (IS_ERR(algt))
0867         return PTR_ERR(algt);
0868
0869     if ((algt->type ^ type) & algt->mask)
0870         return -EINVAL;
0871
0872     *mask_ret = crypto_algt_inherited_mask(algt);
0873     return 0;
0874 }
0875 EXPORT_SYMBOL_GPL(crypto_check_attr_type);
0876
0877 const char *crypto_attr_alg_name(struct rtattr *rta)
0878 {
0879     struct crypto_attr_alg *alga;
0880
0881     if (!rta)
0882         return ERR_PTR(-ENOENT);
0883     if (RTA_PAYLOAD(rta) < sizeof(*alga))
0884         return ERR_PTR(-EINVAL);
0885     if (rta->rta_type != CRYPTOA_ALG)
0886         return ERR_PTR(-EINVAL);
0887
0888     alga = RTA_DATA(rta);
0889     alga->name[CRYPTO_MAX_ALG_NAME - 1] = 0;
0890
0891     return alga->name;
0892 }
0893 EXPORT_SYMBOL_GPL(crypto_attr_alg_name);
0894
0895 int crypto_inst_setname(struct crypto_instance *inst, const char *name,
0896             struct crypto_alg *alg)
0897 {
0898     if (snprintf(inst->alg.cra_name, CRYPTO_MAX_ALG_NAME, "%s(%s)", name,
0899              alg->cra_name) >= CRYPTO_MAX_ALG_NAME)
0900         return -ENAMETOOLONG;
0901
0902     if (snprintf(inst->alg.cra_driver_name, CRYPTO_MAX_ALG_NAME, "%s(%s)",
0903              name, alg->cra_driver_name) >= CRYPTO_MAX_ALG_NAME)
0904         return -ENAMETOOLONG;
0905
0906     return 0;
0907 }
0908 EXPORT_SYMBOL_GPL(crypto_inst_setname);
0909
0910 void crypto_init_queue(struct crypto_queue *queue, unsigned int max_qlen)
0911 {
0912     INIT_LIST_HEAD(&queue->list);
0913     queue->backlog = &queue->list;
0914     queue->qlen = 0;
0915     queue->max_qlen = max_qlen;
0916 }
0917 EXPORT_SYMBOL_GPL(crypto_init_queue);
0918
0919 int crypto_enqueue_request(struct crypto_queue *queue,
0920                struct crypto_async_request *request)
0921 {
0922     int err = -EINPROGRESS;
0923
0924     if (unlikely(queue->qlen >= queue->max_qlen)) {
0925         if (!(request->flags & CRYPTO_TFM_REQ_MAY_BACKLOG)) {
0926             err = -ENOSPC;
0927             goto out;
0928         }
0929         err = -EBUSY;
0930         if (queue->backlog == &queue->list)
0931             queue->backlog = &request->list;
0932     }
0933
0934     queue->qlen++;
0935     list_add_tail(&request->list, &queue->list);
0936
0937 out:
0938     return err;
0939 }
0940 EXPORT_SYMBOL_GPL(crypto_enqueue_request);
0941
0942 void crypto_enqueue_request_head(struct crypto_queue *queue,
0943                  struct crypto_async_request *request)
0944 {
0945     queue->qlen++;
0946     list_add(&request->list, &queue->list);
0947 }
0948 EXPORT_SYMBOL_GPL(crypto_enqueue_request_head);
0949
0950 struct crypto_async_request *crypto_dequeue_request(struct crypto_queue *queue)
0951 {
0952     struct list_head *request;
0953
0954     if (unlikely(!queue->qlen))
0955         return NULL;
0956
0957     queue->qlen--;
0958
0959     if (queue->backlog != &queue->list)
0960         queue->backlog = queue->backlog->next;
0961
0962     request = queue->list.next;
0963     list_del(request);
0964
0965     return list_entry(request, struct crypto_async_request, list);
0966 }
0967 EXPORT_SYMBOL_GPL(crypto_dequeue_request);
0968
0969 static inline void crypto_inc_byte(u8 *a, unsigned int size)
0970 {
0971     u8 *b = (a + size);
0972     u8 c;
0973
0974     for (; size; size--) {
0975         c = *--b + 1;
0976         *b = c;
0977         if (c)
0978             break;
0979     }
0980 }
0981
0982 void crypto_inc(u8 *a, unsigned int size)
0983 {
0984     __be32 *b = (__be32 *)(a + size);
0985     u32 c;
0986
0987     if (IS_ENABLED(CONFIG_HAVE_EFFICIENT_UNALIGNED_ACCESS) ||
0988         IS_ALIGNED((unsigned long)b, __alignof__(*b)))
0989         for (; size >= 4; size -= 4) {
0990             c = be32_to_cpu(*--b) + 1;
0991             *b = cpu_to_be32(c);
0992             if (likely(c))
0993                 return;
0994         }
0995
0996     crypto_inc_byte(a, size);
0997 }
0998 EXPORT_SYMBOL_GPL(crypto_inc);
0999
1000 void __crypto_xor(u8 *dst, const u8 *src1, const u8 *src2, unsigned int len)
1001 {
1002     int relalign = 0;
1003
1004     if (!IS_ENABLED(CONFIG_HAVE_EFFICIENT_UNALIGNED_ACCESS)) {
1005         int size = sizeof(unsigned long);
1006         int d = (((unsigned long)dst ^ (unsigned long)src1) |
1007              ((unsigned long)dst ^ (unsigned long)src2)) &
1008             (size - 1);
1009
1010         relalign = d ? 1 << __ffs(d) : size;
1011
1012         /*
1013          * If we care about alignment, process as many bytes as
1014          * needed to advance dst and src to values whose alignments
1015          * equal their relative alignment. This will allow us to
1016          * process the remainder of the input using optimal strides.
1017          */
1018         while (((unsigned long)dst & (relalign - 1)) && len > 0) {
1019             *dst++ = *src1++ ^ *src2++;
1020             len--;
1021         }
1022     }
1023
1024     while (IS_ENABLED(CONFIG_64BIT) && len >= 8 && !(relalign & 7)) {
1025         if (IS_ENABLED(CONFIG_HAVE_EFFICIENT_UNALIGNED_ACCESS)) {
1026             u64 l = get_unaligned((u64 *)src1) ^
1027                 get_unaligned((u64 *)src2);
1028             put_unaligned(l, (u64 *)dst);
1029         } else {
1030             *(u64 *)dst = *(u64 *)src1 ^ *(u64 *)src2;
1031         }
1032         dst += 8;
1033         src1 += 8;
1034         src2 += 8;
1035         len -= 8;
1036     }
1037
1038     while (len >= 4 && !(relalign & 3)) {
1039         if (IS_ENABLED(CONFIG_HAVE_EFFICIENT_UNALIGNED_ACCESS)) {
1040             u32 l = get_unaligned((u32 *)src1) ^
1041                 get_unaligned((u32 *)src2);
1042             put_unaligned(l, (u32 *)dst);
1043         } else {
1044             *(u32 *)dst = *(u32 *)src1 ^ *(u32 *)src2;
1045         }
1046         dst += 4;
1047         src1 += 4;
1048         src2 += 4;
1049         len -= 4;
1050     }
1051
1052     while (len >= 2 && !(relalign & 1)) {
1053         if (IS_ENABLED(CONFIG_HAVE_EFFICIENT_UNALIGNED_ACCESS)) {
1054             u16 l = get_unaligned((u16 *)src1) ^
1055                 get_unaligned((u16 *)src2);
1056             put_unaligned(l, (u16 *)dst);
1057         } else {
1058             *(u16 *)dst = *(u16 *)src1 ^ *(u16 *)src2;
1059         }
1060         dst += 2;
1061         src1 += 2;
1062         src2 += 2;
1063         len -= 2;
1064     }
1065
1066     while (len--)
1067         *dst++ = *src1++ ^ *src2++;
1068 }
1069 EXPORT_SYMBOL_GPL(__crypto_xor);
1070
1071 unsigned int crypto_alg_extsize(struct crypto_alg *alg)
1072 {
1073     return alg->cra_ctxsize +
1074            (alg->cra_alignmask & ~(crypto_tfm_ctx_alignment() - 1));
1075 }
1076 EXPORT_SYMBOL_GPL(crypto_alg_extsize);
1077
1078 int crypto_type_has_alg(const char *name, const struct crypto_type *frontend,
1079             u32 type, u32 mask)
1080 {
1081     int ret = 0;
1082     struct crypto_alg *alg = crypto_find_alg(name, frontend, type, mask);
1083
1084     if (!IS_ERR(alg)) {
1085         crypto_mod_put(alg);
1086         ret = 1;
1087     }
1088
1089     return ret;
1090 }
1091 EXPORT_SYMBOL_GPL(crypto_type_has_alg);
1092
1093 #ifdef CONFIG_CRYPTO_STATS
1094 void crypto_stats_init(struct crypto_alg *alg)
1095 {
1096     memset(&alg->stats, 0, sizeof(alg->stats));
1097 }
1098 EXPORT_SYMBOL_GPL(crypto_stats_init);
1099
1100 void crypto_stats_get(struct crypto_alg *alg)
1101 {
1102     crypto_alg_get(alg);
1103 }
1104 EXPORT_SYMBOL_GPL(crypto_stats_get);
1105
1106 void crypto_stats_aead_encrypt(unsigned int cryptlen, struct crypto_alg *alg,
1107                    int ret)
1108 {
1109     if (ret && ret != -EINPROGRESS && ret != -EBUSY) {
1110         atomic64_inc(&alg->stats.aead.err_cnt);
1111     } else {
1112         atomic64_inc(&alg->stats.aead.encrypt_cnt);
1113         atomic64_add(cryptlen, &alg->stats.aead.encrypt_tlen);
1114     }
1115     crypto_alg_put(alg);
1116 }
1117 EXPORT_SYMBOL_GPL(crypto_stats_aead_encrypt);
1118
1119 void crypto_stats_aead_decrypt(unsigned int cryptlen, struct crypto_alg *alg,
1120                    int ret)
1121 {
1122     if (ret && ret != -EINPROGRESS && ret != -EBUSY) {
1123         atomic64_inc(&alg->stats.aead.err_cnt);
1124     } else {
1125         atomic64_inc(&alg->stats.aead.decrypt_cnt);
1126         atomic64_add(cryptlen, &alg->stats.aead.decrypt_tlen);
1127     }
1128     crypto_alg_put(alg);
1129 }
1130 EXPORT_SYMBOL_GPL(crypto_stats_aead_decrypt);
1131
1132 void crypto_stats_akcipher_encrypt(unsigned int src_len, int ret,
1133                    struct crypto_alg *alg)
1134 {
1135     if (ret && ret != -EINPROGRESS && ret != -EBUSY) {
1136         atomic64_inc(&alg->stats.akcipher.err_cnt);
1137     } else {
1138         atomic64_inc(&alg->stats.akcipher.encrypt_cnt);
1139         atomic64_add(src_len, &alg->stats.akcipher.encrypt_tlen);
1140     }
1141     crypto_alg_put(alg);
1142 }
1143 EXPORT_SYMBOL_GPL(crypto_stats_akcipher_encrypt);
1144
1145 void crypto_stats_akcipher_decrypt(unsigned int src_len, int ret,
1146                    struct crypto_alg *alg)
1147 {
1148     if (ret && ret != -EINPROGRESS && ret != -EBUSY) {
1149         atomic64_inc(&alg->stats.akcipher.err_cnt);
1150     } else {
1151         atomic64_inc(&alg->stats.akcipher.decrypt_cnt);
1152         atomic64_add(src_len, &alg->stats.akcipher.decrypt_tlen);
1153     }
1154     crypto_alg_put(alg);
1155 }
1156 EXPORT_SYMBOL_GPL(crypto_stats_akcipher_decrypt);
1157
1158 void crypto_stats_akcipher_sign(int ret, struct crypto_alg *alg)
1159 {
1160     if (ret && ret != -EINPROGRESS && ret != -EBUSY)
1161         atomic64_inc(&alg->stats.akcipher.err_cnt);
1162     else
1163         atomic64_inc(&alg->stats.akcipher.sign_cnt);
1164     crypto_alg_put(alg);
1165 }
1166 EXPORT_SYMBOL_GPL(crypto_stats_akcipher_sign);
1167
1168 void crypto_stats_akcipher_verify(int ret, struct crypto_alg *alg)
1169 {
1170     if (ret && ret != -EINPROGRESS && ret != -EBUSY)
1171         atomic64_inc(&alg->stats.akcipher.err_cnt);
1172     else
1173         atomic64_inc(&alg->stats.akcipher.verify_cnt);
1174     crypto_alg_put(alg);
1175 }
1176 EXPORT_SYMBOL_GPL(crypto_stats_akcipher_verify);
1177
1178 void crypto_stats_compress(unsigned int slen, int ret, struct crypto_alg *alg)
1179 {
1180     if (ret && ret != -EINPROGRESS && ret != -EBUSY) {
1181         atomic64_inc(&alg->stats.compress.err_cnt);
1182     } else {
1183         atomic64_inc(&alg->stats.compress.compress_cnt);
1184         atomic64_add(slen, &alg->stats.compress.compress_tlen);
1185     }
1186     crypto_alg_put(alg);
1187 }
1188 EXPORT_SYMBOL_GPL(crypto_stats_compress);
1189
1190 void crypto_stats_decompress(unsigned int slen, int ret, struct crypto_alg *alg)
1191 {
1192     if (ret && ret != -EINPROGRESS && ret != -EBUSY) {
1193         atomic64_inc(&alg->stats.compress.err_cnt);
1194     } else {
1195         atomic64_inc(&alg->stats.compress.decompress_cnt);
1196         atomic64_add(slen, &alg->stats.compress.decompress_tlen);
1197     }
1198     crypto_alg_put(alg);
1199 }
1200 EXPORT_SYMBOL_GPL(crypto_stats_decompress);
1201
1202 void crypto_stats_ahash_update(unsigned int nbytes, int ret,
1203                    struct crypto_alg *alg)
1204 {
1205     if (ret && ret != -EINPROGRESS && ret != -EBUSY)
1206         atomic64_inc(&alg->stats.hash.err_cnt);
1207     else
1208         atomic64_add(nbytes, &alg->stats.hash.hash_tlen);
1209     crypto_alg_put(alg);
1210 }
1211 EXPORT_SYMBOL_GPL(crypto_stats_ahash_update);
1212
1213 void crypto_stats_ahash_final(unsigned int nbytes, int ret,
1214                   struct crypto_alg *alg)
1215 {
1216     if (ret && ret != -EINPROGRESS && ret != -EBUSY) {
1217         atomic64_inc(&alg->stats.hash.err_cnt);
1218     } else {
1219         atomic64_inc(&alg->stats.hash.hash_cnt);
1220         atomic64_add(nbytes, &alg->stats.hash.hash_tlen);
1221     }
1222     crypto_alg_put(alg);
1223 }
1224 EXPORT_SYMBOL_GPL(crypto_stats_ahash_final);
1225
1226 void crypto_stats_kpp_set_secret(struct crypto_alg *alg, int ret)
1227 {
1228     if (ret)
1229         atomic64_inc(&alg->stats.kpp.err_cnt);
1230     else
1231         atomic64_inc(&alg->stats.kpp.setsecret_cnt);
1232     crypto_alg_put(alg);
1233 }
1234 EXPORT_SYMBOL_GPL(crypto_stats_kpp_set_secret);
1235
1236 void crypto_stats_kpp_generate_public_key(struct crypto_alg *alg, int ret)
1237 {
1238     if (ret)
1239         atomic64_inc(&alg->stats.kpp.err_cnt);
1240     else
1241         atomic64_inc(&alg->stats.kpp.generate_public_key_cnt);
1242     crypto_alg_put(alg);
1243 }
1244 EXPORT_SYMBOL_GPL(crypto_stats_kpp_generate_public_key);
1245
1246 void crypto_stats_kpp_compute_shared_secret(struct crypto_alg *alg, int ret)
1247 {
1248     if (ret)
1249         atomic64_inc(&alg->stats.kpp.err_cnt);
1250     else
1251         atomic64_inc(&alg->stats.kpp.compute_shared_secret_cnt);
1252     crypto_alg_put(alg);
1253 }
1254 EXPORT_SYMBOL_GPL(crypto_stats_kpp_compute_shared_secret);
1255
1256 void crypto_stats_rng_seed(struct crypto_alg *alg, int ret)
1257 {
1258     if (ret && ret != -EINPROGRESS && ret != -EBUSY)
1259         atomic64_inc(&alg->stats.rng.err_cnt);
1260     else
1261         atomic64_inc(&alg->stats.rng.seed_cnt);
1262     crypto_alg_put(alg);
1263 }
1264 EXPORT_SYMBOL_GPL(crypto_stats_rng_seed);
1265
1266 void crypto_stats_rng_generate(struct crypto_alg *alg, unsigned int dlen,
1267                    int ret)
1268 {
1269     if (ret && ret != -EINPROGRESS && ret != -EBUSY) {
1270         atomic64_inc(&alg->stats.rng.err_cnt);
1271     } else {
1272         atomic64_inc(&alg->stats.rng.generate_cnt);
1273         atomic64_add(dlen, &alg->stats.rng.generate_tlen);
1274     }
1275     crypto_alg_put(alg);
1276 }
1277 EXPORT_SYMBOL_GPL(crypto_stats_rng_generate);
1278
1279 void crypto_stats_skcipher_encrypt(unsigned int cryptlen, int ret,
1280                    struct crypto_alg *alg)
1281 {
1282     if (ret && ret != -EINPROGRESS && ret != -EBUSY) {
1283         atomic64_inc(&alg->stats.cipher.err_cnt);
1284     } else {
1285         atomic64_inc(&alg->stats.cipher.encrypt_cnt);
1286         atomic64_add(cryptlen, &alg->stats.cipher.encrypt_tlen);
1287     }
1288     crypto_alg_put(alg);
1289 }
1290 EXPORT_SYMBOL_GPL(crypto_stats_skcipher_encrypt);
1291
1292 void crypto_stats_skcipher_decrypt(unsigned int cryptlen, int ret,
1293                    struct crypto_alg *alg)
1294 {
1295     if (ret && ret != -EINPROGRESS && ret != -EBUSY) {
1296         atomic64_inc(&alg->stats.cipher.err_cnt);
1297     } else {
1298         atomic64_inc(&alg->stats.cipher.decrypt_cnt);
1299         atomic64_add(cryptlen, &alg->stats.cipher.decrypt_tlen);
1300     }
1301     crypto_alg_put(alg);
1302 }
1303 EXPORT_SYMBOL_GPL(crypto_stats_skcipher_decrypt);
1304 #endif
1305
1306 static void __init crypto_start_tests(void)
1307 {
1308     for (;;) {
1309         struct crypto_larval *larval = NULL;
1310         struct crypto_alg *q;
1311
1312         down_write(&crypto_alg_sem);
1313
1314         list_for_each_entry(q, &crypto_alg_list, cra_list) {
1315             struct crypto_larval *l;
1316
1317             if (!crypto_is_larval(q))
1318                 continue;
1319
1320             l = (void *)q;
1321
1322             if (!crypto_is_test_larval(l))
1323                 continue;
1324
1325             if (l->test_started)
1326                 continue;
1327
1328             l->test_started = true;
1329             larval = l;
1330             break;
1331         }
1332
1333         up_write(&crypto_alg_sem);
1334
1335         if (!larval)
1336             break;
1337
1338         crypto_wait_for_test(larval);
1339     }
1340
1341     static_branch_enable(&crypto_boot_test_finished);
1342 }
1343
1344 static int __init crypto_algapi_init(void)
1345 {
1346     crypto_init_proc();
1347     crypto_start_tests();
1348     return 0;
1349 }
1350
1351 static void __exit crypto_algapi_exit(void)
1352 {
1353     crypto_exit_proc();
1354 }
1355
1356 /*
1357  * We run this at late_initcall so that all the built-in algorithms
1358  * have had a chance to register themselves first.
1359  */
1360 late_initcall(crypto_algapi_init);
1361 module_exit(crypto_algapi_exit);
1362
1363 MODULE_LICENSE("GPL");
1364 MODULE_DESCRIPTION("Cryptographic algorithms API");
1365 MODULE_SOFTDEP("pre: cryptomgr");