Back to home page

LXR

 
 

    


0001 /*
0002  * Software async crypto daemon.
0003  *
0004  * Copyright (c) 2006 Herbert Xu <herbert@gondor.apana.org.au>
0005  *
0006  * Added AEAD support to cryptd.
0007  *    Authors: Tadeusz Struk (tadeusz.struk@intel.com)
0008  *             Adrian Hoban <adrian.hoban@intel.com>
0009  *             Gabriele Paoloni <gabriele.paoloni@intel.com>
0010  *             Aidan O'Mahony (aidan.o.mahony@intel.com)
0011  *    Copyright (c) 2010, Intel Corporation.
0012  *
0013  * This program is free software; you can redistribute it and/or modify it
0014  * under the terms of the GNU General Public License as published by the Free
0015  * Software Foundation; either version 2 of the License, or (at your option)
0016  * any later version.
0017  *
0018  */
0019 
0020 #include <crypto/internal/hash.h>
0021 #include <crypto/internal/aead.h>
0022 #include <crypto/internal/skcipher.h>
0023 #include <crypto/cryptd.h>
0024 #include <crypto/crypto_wq.h>
0025 #include <linux/atomic.h>
0026 #include <linux/err.h>
0027 #include <linux/init.h>
0028 #include <linux/kernel.h>
0029 #include <linux/list.h>
0030 #include <linux/module.h>
0031 #include <linux/scatterlist.h>
0032 #include <linux/sched.h>
0033 #include <linux/slab.h>
0034 
0035 #define CRYPTD_MAX_CPU_QLEN 1000
0036 
0037 struct cryptd_cpu_queue {
0038     struct crypto_queue queue;
0039     struct work_struct work;
0040 };
0041 
0042 struct cryptd_queue {
0043     struct cryptd_cpu_queue __percpu *cpu_queue;
0044 };
0045 
0046 struct cryptd_instance_ctx {
0047     struct crypto_spawn spawn;
0048     struct cryptd_queue *queue;
0049 };
0050 
0051 struct skcipherd_instance_ctx {
0052     struct crypto_skcipher_spawn spawn;
0053     struct cryptd_queue *queue;
0054 };
0055 
0056 struct hashd_instance_ctx {
0057     struct crypto_shash_spawn spawn;
0058     struct cryptd_queue *queue;
0059 };
0060 
0061 struct aead_instance_ctx {
0062     struct crypto_aead_spawn aead_spawn;
0063     struct cryptd_queue *queue;
0064 };
0065 
0066 struct cryptd_blkcipher_ctx {
0067     atomic_t refcnt;
0068     struct crypto_blkcipher *child;
0069 };
0070 
0071 struct cryptd_blkcipher_request_ctx {
0072     crypto_completion_t complete;
0073 };
0074 
0075 struct cryptd_skcipher_ctx {
0076     atomic_t refcnt;
0077     struct crypto_skcipher *child;
0078 };
0079 
0080 struct cryptd_skcipher_request_ctx {
0081     crypto_completion_t complete;
0082 };
0083 
0084 struct cryptd_hash_ctx {
0085     atomic_t refcnt;
0086     struct crypto_shash *child;
0087 };
0088 
0089 struct cryptd_hash_request_ctx {
0090     crypto_completion_t complete;
0091     struct shash_desc desc;
0092 };
0093 
0094 struct cryptd_aead_ctx {
0095     atomic_t refcnt;
0096     struct crypto_aead *child;
0097 };
0098 
0099 struct cryptd_aead_request_ctx {
0100     crypto_completion_t complete;
0101 };
0102 
0103 static void cryptd_queue_worker(struct work_struct *work);
0104 
0105 static int cryptd_init_queue(struct cryptd_queue *queue,
0106                  unsigned int max_cpu_qlen)
0107 {
0108     int cpu;
0109     struct cryptd_cpu_queue *cpu_queue;
0110 
0111     queue->cpu_queue = alloc_percpu(struct cryptd_cpu_queue);
0112     if (!queue->cpu_queue)
0113         return -ENOMEM;
0114     for_each_possible_cpu(cpu) {
0115         cpu_queue = per_cpu_ptr(queue->cpu_queue, cpu);
0116         crypto_init_queue(&cpu_queue->queue, max_cpu_qlen);
0117         INIT_WORK(&cpu_queue->work, cryptd_queue_worker);
0118     }
0119     return 0;
0120 }
0121 
0122 static void cryptd_fini_queue(struct cryptd_queue *queue)
0123 {
0124     int cpu;
0125     struct cryptd_cpu_queue *cpu_queue;
0126 
0127     for_each_possible_cpu(cpu) {
0128         cpu_queue = per_cpu_ptr(queue->cpu_queue, cpu);
0129         BUG_ON(cpu_queue->queue.qlen);
0130     }
0131     free_percpu(queue->cpu_queue);
0132 }
0133 
0134 static int cryptd_enqueue_request(struct cryptd_queue *queue,
0135                   struct crypto_async_request *request)
0136 {
0137     int cpu, err;
0138     struct cryptd_cpu_queue *cpu_queue;
0139     atomic_t *refcnt;
0140     bool may_backlog;
0141 
0142     cpu = get_cpu();
0143     cpu_queue = this_cpu_ptr(queue->cpu_queue);
0144     err = crypto_enqueue_request(&cpu_queue->queue, request);
0145 
0146     refcnt = crypto_tfm_ctx(request->tfm);
0147     may_backlog = request->flags & CRYPTO_TFM_REQ_MAY_BACKLOG;
0148 
0149     if (err == -EBUSY && !may_backlog)
0150         goto out_put_cpu;
0151 
0152     queue_work_on(cpu, kcrypto_wq, &cpu_queue->work);
0153 
0154     if (!atomic_read(refcnt))
0155         goto out_put_cpu;
0156 
0157     atomic_inc(refcnt);
0158 
0159 out_put_cpu:
0160     put_cpu();
0161 
0162     return err;
0163 }
0164 
0165 /* Called in workqueue context, do one real cryption work (via
0166  * req->complete) and reschedule itself if there are more work to
0167  * do. */
0168 static void cryptd_queue_worker(struct work_struct *work)
0169 {
0170     struct cryptd_cpu_queue *cpu_queue;
0171     struct crypto_async_request *req, *backlog;
0172 
0173     cpu_queue = container_of(work, struct cryptd_cpu_queue, work);
0174     /*
0175      * Only handle one request at a time to avoid hogging crypto workqueue.
0176      * preempt_disable/enable is used to prevent being preempted by
0177      * cryptd_enqueue_request(). local_bh_disable/enable is used to prevent
0178      * cryptd_enqueue_request() being accessed from software interrupts.
0179      */
0180     local_bh_disable();
0181     preempt_disable();
0182     backlog = crypto_get_backlog(&cpu_queue->queue);
0183     req = crypto_dequeue_request(&cpu_queue->queue);
0184     preempt_enable();
0185     local_bh_enable();
0186 
0187     if (!req)
0188         return;
0189 
0190     if (backlog)
0191         backlog->complete(backlog, -EINPROGRESS);
0192     req->complete(req, 0);
0193 
0194     if (cpu_queue->queue.qlen)
0195         queue_work(kcrypto_wq, &cpu_queue->work);
0196 }
0197 
0198 static inline struct cryptd_queue *cryptd_get_queue(struct crypto_tfm *tfm)
0199 {
0200     struct crypto_instance *inst = crypto_tfm_alg_instance(tfm);
0201     struct cryptd_instance_ctx *ictx = crypto_instance_ctx(inst);
0202     return ictx->queue;
0203 }
0204 
0205 static inline void cryptd_check_internal(struct rtattr **tb, u32 *type,
0206                      u32 *mask)
0207 {
0208     struct crypto_attr_type *algt;
0209 
0210     algt = crypto_get_attr_type(tb);
0211     if (IS_ERR(algt))
0212         return;
0213 
0214     *type |= algt->type & CRYPTO_ALG_INTERNAL;
0215     *mask |= algt->mask & CRYPTO_ALG_INTERNAL;
0216 }
0217 
0218 static int cryptd_blkcipher_setkey(struct crypto_ablkcipher *parent,
0219                    const u8 *key, unsigned int keylen)
0220 {
0221     struct cryptd_blkcipher_ctx *ctx = crypto_ablkcipher_ctx(parent);
0222     struct crypto_blkcipher *child = ctx->child;
0223     int err;
0224 
0225     crypto_blkcipher_clear_flags(child, CRYPTO_TFM_REQ_MASK);
0226     crypto_blkcipher_set_flags(child, crypto_ablkcipher_get_flags(parent) &
0227                       CRYPTO_TFM_REQ_MASK);
0228     err = crypto_blkcipher_setkey(child, key, keylen);
0229     crypto_ablkcipher_set_flags(parent, crypto_blkcipher_get_flags(child) &
0230                         CRYPTO_TFM_RES_MASK);
0231     return err;
0232 }
0233 
0234 static void cryptd_blkcipher_crypt(struct ablkcipher_request *req,
0235                    struct crypto_blkcipher *child,
0236                    int err,
0237                    int (*crypt)(struct blkcipher_desc *desc,
0238                         struct scatterlist *dst,
0239                         struct scatterlist *src,
0240                         unsigned int len))
0241 {
0242     struct cryptd_blkcipher_request_ctx *rctx;
0243     struct cryptd_blkcipher_ctx *ctx;
0244     struct crypto_ablkcipher *tfm;
0245     struct blkcipher_desc desc;
0246     int refcnt;
0247 
0248     rctx = ablkcipher_request_ctx(req);
0249 
0250     if (unlikely(err == -EINPROGRESS))
0251         goto out;
0252 
0253     desc.tfm = child;
0254     desc.info = req->info;
0255     desc.flags = CRYPTO_TFM_REQ_MAY_SLEEP;
0256 
0257     err = crypt(&desc, req->dst, req->src, req->nbytes);
0258 
0259     req->base.complete = rctx->complete;
0260 
0261 out:
0262     tfm = crypto_ablkcipher_reqtfm(req);
0263     ctx = crypto_ablkcipher_ctx(tfm);
0264     refcnt = atomic_read(&ctx->refcnt);
0265 
0266     local_bh_disable();
0267     rctx->complete(&req->base, err);
0268     local_bh_enable();
0269 
0270     if (err != -EINPROGRESS && refcnt && atomic_dec_and_test(&ctx->refcnt))
0271         crypto_free_ablkcipher(tfm);
0272 }
0273 
0274 static void cryptd_blkcipher_encrypt(struct crypto_async_request *req, int err)
0275 {
0276     struct cryptd_blkcipher_ctx *ctx = crypto_tfm_ctx(req->tfm);
0277     struct crypto_blkcipher *child = ctx->child;
0278 
0279     cryptd_blkcipher_crypt(ablkcipher_request_cast(req), child, err,
0280                    crypto_blkcipher_crt(child)->encrypt);
0281 }
0282 
0283 static void cryptd_blkcipher_decrypt(struct crypto_async_request *req, int err)
0284 {
0285     struct cryptd_blkcipher_ctx *ctx = crypto_tfm_ctx(req->tfm);
0286     struct crypto_blkcipher *child = ctx->child;
0287 
0288     cryptd_blkcipher_crypt(ablkcipher_request_cast(req), child, err,
0289                    crypto_blkcipher_crt(child)->decrypt);
0290 }
0291 
0292 static int cryptd_blkcipher_enqueue(struct ablkcipher_request *req,
0293                     crypto_completion_t compl)
0294 {
0295     struct cryptd_blkcipher_request_ctx *rctx = ablkcipher_request_ctx(req);
0296     struct crypto_ablkcipher *tfm = crypto_ablkcipher_reqtfm(req);
0297     struct cryptd_queue *queue;
0298 
0299     queue = cryptd_get_queue(crypto_ablkcipher_tfm(tfm));
0300     rctx->complete = req->base.complete;
0301     req->base.complete = compl;
0302 
0303     return cryptd_enqueue_request(queue, &req->base);
0304 }
0305 
0306 static int cryptd_blkcipher_encrypt_enqueue(struct ablkcipher_request *req)
0307 {
0308     return cryptd_blkcipher_enqueue(req, cryptd_blkcipher_encrypt);
0309 }
0310 
0311 static int cryptd_blkcipher_decrypt_enqueue(struct ablkcipher_request *req)
0312 {
0313     return cryptd_blkcipher_enqueue(req, cryptd_blkcipher_decrypt);
0314 }
0315 
0316 static int cryptd_blkcipher_init_tfm(struct crypto_tfm *tfm)
0317 {
0318     struct crypto_instance *inst = crypto_tfm_alg_instance(tfm);
0319     struct cryptd_instance_ctx *ictx = crypto_instance_ctx(inst);
0320     struct crypto_spawn *spawn = &ictx->spawn;
0321     struct cryptd_blkcipher_ctx *ctx = crypto_tfm_ctx(tfm);
0322     struct crypto_blkcipher *cipher;
0323 
0324     cipher = crypto_spawn_blkcipher(spawn);
0325     if (IS_ERR(cipher))
0326         return PTR_ERR(cipher);
0327 
0328     ctx->child = cipher;
0329     tfm->crt_ablkcipher.reqsize =
0330         sizeof(struct cryptd_blkcipher_request_ctx);
0331     return 0;
0332 }
0333 
0334 static void cryptd_blkcipher_exit_tfm(struct crypto_tfm *tfm)
0335 {
0336     struct cryptd_blkcipher_ctx *ctx = crypto_tfm_ctx(tfm);
0337 
0338     crypto_free_blkcipher(ctx->child);
0339 }
0340 
0341 static int cryptd_init_instance(struct crypto_instance *inst,
0342                 struct crypto_alg *alg)
0343 {
0344     if (snprintf(inst->alg.cra_driver_name, CRYPTO_MAX_ALG_NAME,
0345              "cryptd(%s)",
0346              alg->cra_driver_name) >= CRYPTO_MAX_ALG_NAME)
0347         return -ENAMETOOLONG;
0348 
0349     memcpy(inst->alg.cra_name, alg->cra_name, CRYPTO_MAX_ALG_NAME);
0350 
0351     inst->alg.cra_priority = alg->cra_priority + 50;
0352     inst->alg.cra_blocksize = alg->cra_blocksize;
0353     inst->alg.cra_alignmask = alg->cra_alignmask;
0354 
0355     return 0;
0356 }
0357 
0358 static void *cryptd_alloc_instance(struct crypto_alg *alg, unsigned int head,
0359                    unsigned int tail)
0360 {
0361     char *p;
0362     struct crypto_instance *inst;
0363     int err;
0364 
0365     p = kzalloc(head + sizeof(*inst) + tail, GFP_KERNEL);
0366     if (!p)
0367         return ERR_PTR(-ENOMEM);
0368 
0369     inst = (void *)(p + head);
0370 
0371     err = cryptd_init_instance(inst, alg);
0372     if (err)
0373         goto out_free_inst;
0374 
0375 out:
0376     return p;
0377 
0378 out_free_inst:
0379     kfree(p);
0380     p = ERR_PTR(err);
0381     goto out;
0382 }
0383 
0384 static int cryptd_create_blkcipher(struct crypto_template *tmpl,
0385                    struct rtattr **tb,
0386                    struct cryptd_queue *queue)
0387 {
0388     struct cryptd_instance_ctx *ctx;
0389     struct crypto_instance *inst;
0390     struct crypto_alg *alg;
0391     u32 type = CRYPTO_ALG_TYPE_BLKCIPHER;
0392     u32 mask = CRYPTO_ALG_TYPE_MASK;
0393     int err;
0394 
0395     cryptd_check_internal(tb, &type, &mask);
0396 
0397     alg = crypto_get_attr_alg(tb, type, mask);
0398     if (IS_ERR(alg))
0399         return PTR_ERR(alg);
0400 
0401     inst = cryptd_alloc_instance(alg, 0, sizeof(*ctx));
0402     err = PTR_ERR(inst);
0403     if (IS_ERR(inst))
0404         goto out_put_alg;
0405 
0406     ctx = crypto_instance_ctx(inst);
0407     ctx->queue = queue;
0408 
0409     err = crypto_init_spawn(&ctx->spawn, alg, inst,
0410                 CRYPTO_ALG_TYPE_MASK | CRYPTO_ALG_ASYNC);
0411     if (err)
0412         goto out_free_inst;
0413 
0414     type = CRYPTO_ALG_TYPE_ABLKCIPHER | CRYPTO_ALG_ASYNC;
0415     if (alg->cra_flags & CRYPTO_ALG_INTERNAL)
0416         type |= CRYPTO_ALG_INTERNAL;
0417     inst->alg.cra_flags = type;
0418     inst->alg.cra_type = &crypto_ablkcipher_type;
0419 
0420     inst->alg.cra_ablkcipher.ivsize = alg->cra_blkcipher.ivsize;
0421     inst->alg.cra_ablkcipher.min_keysize = alg->cra_blkcipher.min_keysize;
0422     inst->alg.cra_ablkcipher.max_keysize = alg->cra_blkcipher.max_keysize;
0423 
0424     inst->alg.cra_ablkcipher.geniv = alg->cra_blkcipher.geniv;
0425 
0426     inst->alg.cra_ctxsize = sizeof(struct cryptd_blkcipher_ctx);
0427 
0428     inst->alg.cra_init = cryptd_blkcipher_init_tfm;
0429     inst->alg.cra_exit = cryptd_blkcipher_exit_tfm;
0430 
0431     inst->alg.cra_ablkcipher.setkey = cryptd_blkcipher_setkey;
0432     inst->alg.cra_ablkcipher.encrypt = cryptd_blkcipher_encrypt_enqueue;
0433     inst->alg.cra_ablkcipher.decrypt = cryptd_blkcipher_decrypt_enqueue;
0434 
0435     err = crypto_register_instance(tmpl, inst);
0436     if (err) {
0437         crypto_drop_spawn(&ctx->spawn);
0438 out_free_inst:
0439         kfree(inst);
0440     }
0441 
0442 out_put_alg:
0443     crypto_mod_put(alg);
0444     return err;
0445 }
0446 
0447 static int cryptd_skcipher_setkey(struct crypto_skcipher *parent,
0448                   const u8 *key, unsigned int keylen)
0449 {
0450     struct cryptd_skcipher_ctx *ctx = crypto_skcipher_ctx(parent);
0451     struct crypto_skcipher *child = ctx->child;
0452     int err;
0453 
0454     crypto_skcipher_clear_flags(child, CRYPTO_TFM_REQ_MASK);
0455     crypto_skcipher_set_flags(child, crypto_skcipher_get_flags(parent) &
0456                      CRYPTO_TFM_REQ_MASK);
0457     err = crypto_skcipher_setkey(child, key, keylen);
0458     crypto_skcipher_set_flags(parent, crypto_skcipher_get_flags(child) &
0459                       CRYPTO_TFM_RES_MASK);
0460     return err;
0461 }
0462 
0463 static void cryptd_skcipher_complete(struct skcipher_request *req, int err)
0464 {
0465     struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req);
0466     struct cryptd_skcipher_ctx *ctx = crypto_skcipher_ctx(tfm);
0467     struct cryptd_skcipher_request_ctx *rctx = skcipher_request_ctx(req);
0468     int refcnt = atomic_read(&ctx->refcnt);
0469 
0470     local_bh_disable();
0471     rctx->complete(&req->base, err);
0472     local_bh_enable();
0473 
0474     if (err != -EINPROGRESS && refcnt && atomic_dec_and_test(&ctx->refcnt))
0475         crypto_free_skcipher(tfm);
0476 }
0477 
0478 static void cryptd_skcipher_encrypt(struct crypto_async_request *base,
0479                     int err)
0480 {
0481     struct skcipher_request *req = skcipher_request_cast(base);
0482     struct cryptd_skcipher_request_ctx *rctx = skcipher_request_ctx(req);
0483     struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req);
0484     struct cryptd_skcipher_ctx *ctx = crypto_skcipher_ctx(tfm);
0485     struct crypto_skcipher *child = ctx->child;
0486     SKCIPHER_REQUEST_ON_STACK(subreq, child);
0487 
0488     if (unlikely(err == -EINPROGRESS))
0489         goto out;
0490 
0491     skcipher_request_set_tfm(subreq, child);
0492     skcipher_request_set_callback(subreq, CRYPTO_TFM_REQ_MAY_SLEEP,
0493                       NULL, NULL);
0494     skcipher_request_set_crypt(subreq, req->src, req->dst, req->cryptlen,
0495                    req->iv);
0496 
0497     err = crypto_skcipher_encrypt(subreq);
0498     skcipher_request_zero(subreq);
0499 
0500     req->base.complete = rctx->complete;
0501 
0502 out:
0503     cryptd_skcipher_complete(req, err);
0504 }
0505 
0506 static void cryptd_skcipher_decrypt(struct crypto_async_request *base,
0507                     int err)
0508 {
0509     struct skcipher_request *req = skcipher_request_cast(base);
0510     struct cryptd_skcipher_request_ctx *rctx = skcipher_request_ctx(req);
0511     struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req);
0512     struct cryptd_skcipher_ctx *ctx = crypto_skcipher_ctx(tfm);
0513     struct crypto_skcipher *child = ctx->child;
0514     SKCIPHER_REQUEST_ON_STACK(subreq, child);
0515 
0516     if (unlikely(err == -EINPROGRESS))
0517         goto out;
0518 
0519     skcipher_request_set_tfm(subreq, child);
0520     skcipher_request_set_callback(subreq, CRYPTO_TFM_REQ_MAY_SLEEP,
0521                       NULL, NULL);
0522     skcipher_request_set_crypt(subreq, req->src, req->dst, req->cryptlen,
0523                    req->iv);
0524 
0525     err = crypto_skcipher_decrypt(subreq);
0526     skcipher_request_zero(subreq);
0527 
0528     req->base.complete = rctx->complete;
0529 
0530 out:
0531     cryptd_skcipher_complete(req, err);
0532 }
0533 
0534 static int cryptd_skcipher_enqueue(struct skcipher_request *req,
0535                    crypto_completion_t compl)
0536 {
0537     struct cryptd_skcipher_request_ctx *rctx = skcipher_request_ctx(req);
0538     struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req);
0539     struct cryptd_queue *queue;
0540 
0541     queue = cryptd_get_queue(crypto_skcipher_tfm(tfm));
0542     rctx->complete = req->base.complete;
0543     req->base.complete = compl;
0544 
0545     return cryptd_enqueue_request(queue, &req->base);
0546 }
0547 
0548 static int cryptd_skcipher_encrypt_enqueue(struct skcipher_request *req)
0549 {
0550     return cryptd_skcipher_enqueue(req, cryptd_skcipher_encrypt);
0551 }
0552 
0553 static int cryptd_skcipher_decrypt_enqueue(struct skcipher_request *req)
0554 {
0555     return cryptd_skcipher_enqueue(req, cryptd_skcipher_decrypt);
0556 }
0557 
0558 static int cryptd_skcipher_init_tfm(struct crypto_skcipher *tfm)
0559 {
0560     struct skcipher_instance *inst = skcipher_alg_instance(tfm);
0561     struct skcipherd_instance_ctx *ictx = skcipher_instance_ctx(inst);
0562     struct crypto_skcipher_spawn *spawn = &ictx->spawn;
0563     struct cryptd_skcipher_ctx *ctx = crypto_skcipher_ctx(tfm);
0564     struct crypto_skcipher *cipher;
0565 
0566     cipher = crypto_spawn_skcipher(spawn);
0567     if (IS_ERR(cipher))
0568         return PTR_ERR(cipher);
0569 
0570     ctx->child = cipher;
0571     crypto_skcipher_set_reqsize(
0572         tfm, sizeof(struct cryptd_skcipher_request_ctx));
0573     return 0;
0574 }
0575 
0576 static void cryptd_skcipher_exit_tfm(struct crypto_skcipher *tfm)
0577 {
0578     struct cryptd_skcipher_ctx *ctx = crypto_skcipher_ctx(tfm);
0579 
0580     crypto_free_skcipher(ctx->child);
0581 }
0582 
0583 static void cryptd_skcipher_free(struct skcipher_instance *inst)
0584 {
0585     struct skcipherd_instance_ctx *ctx = skcipher_instance_ctx(inst);
0586 
0587     crypto_drop_skcipher(&ctx->spawn);
0588 }
0589 
0590 static int cryptd_create_skcipher(struct crypto_template *tmpl,
0591                   struct rtattr **tb,
0592                   struct cryptd_queue *queue)
0593 {
0594     struct skcipherd_instance_ctx *ctx;
0595     struct skcipher_instance *inst;
0596     struct skcipher_alg *alg;
0597     const char *name;
0598     u32 type;
0599     u32 mask;
0600     int err;
0601 
0602     type = 0;
0603     mask = CRYPTO_ALG_ASYNC;
0604 
0605     cryptd_check_internal(tb, &type, &mask);
0606 
0607     name = crypto_attr_alg_name(tb[1]);
0608     if (IS_ERR(name))
0609         return PTR_ERR(name);
0610 
0611     inst = kzalloc(sizeof(*inst) + sizeof(*ctx), GFP_KERNEL);
0612     if (!inst)
0613         return -ENOMEM;
0614 
0615     ctx = skcipher_instance_ctx(inst);
0616     ctx->queue = queue;
0617 
0618     crypto_set_skcipher_spawn(&ctx->spawn, skcipher_crypto_instance(inst));
0619     err = crypto_grab_skcipher(&ctx->spawn, name, type, mask);
0620     if (err)
0621         goto out_free_inst;
0622 
0623     alg = crypto_spawn_skcipher_alg(&ctx->spawn);
0624     err = cryptd_init_instance(skcipher_crypto_instance(inst), &alg->base);
0625     if (err)
0626         goto out_drop_skcipher;
0627 
0628     inst->alg.base.cra_flags = CRYPTO_ALG_ASYNC |
0629                    (alg->base.cra_flags & CRYPTO_ALG_INTERNAL);
0630 
0631     inst->alg.ivsize = crypto_skcipher_alg_ivsize(alg);
0632     inst->alg.chunksize = crypto_skcipher_alg_chunksize(alg);
0633     inst->alg.min_keysize = crypto_skcipher_alg_min_keysize(alg);
0634     inst->alg.max_keysize = crypto_skcipher_alg_max_keysize(alg);
0635 
0636     inst->alg.base.cra_ctxsize = sizeof(struct cryptd_skcipher_ctx);
0637 
0638     inst->alg.init = cryptd_skcipher_init_tfm;
0639     inst->alg.exit = cryptd_skcipher_exit_tfm;
0640 
0641     inst->alg.setkey = cryptd_skcipher_setkey;
0642     inst->alg.encrypt = cryptd_skcipher_encrypt_enqueue;
0643     inst->alg.decrypt = cryptd_skcipher_decrypt_enqueue;
0644 
0645     inst->free = cryptd_skcipher_free;
0646 
0647     err = skcipher_register_instance(tmpl, inst);
0648     if (err) {
0649 out_drop_skcipher:
0650         crypto_drop_skcipher(&ctx->spawn);
0651 out_free_inst:
0652         kfree(inst);
0653     }
0654     return err;
0655 }
0656 
0657 static int cryptd_hash_init_tfm(struct crypto_tfm *tfm)
0658 {
0659     struct crypto_instance *inst = crypto_tfm_alg_instance(tfm);
0660     struct hashd_instance_ctx *ictx = crypto_instance_ctx(inst);
0661     struct crypto_shash_spawn *spawn = &ictx->spawn;
0662     struct cryptd_hash_ctx *ctx = crypto_tfm_ctx(tfm);
0663     struct crypto_shash *hash;
0664 
0665     hash = crypto_spawn_shash(spawn);
0666     if (IS_ERR(hash))
0667         return PTR_ERR(hash);
0668 
0669     ctx->child = hash;
0670     crypto_ahash_set_reqsize(__crypto_ahash_cast(tfm),
0671                  sizeof(struct cryptd_hash_request_ctx) +
0672                  crypto_shash_descsize(hash));
0673     return 0;
0674 }
0675 
0676 static void cryptd_hash_exit_tfm(struct crypto_tfm *tfm)
0677 {
0678     struct cryptd_hash_ctx *ctx = crypto_tfm_ctx(tfm);
0679 
0680     crypto_free_shash(ctx->child);
0681 }
0682 
0683 static int cryptd_hash_setkey(struct crypto_ahash *parent,
0684                    const u8 *key, unsigned int keylen)
0685 {
0686     struct cryptd_hash_ctx *ctx   = crypto_ahash_ctx(parent);
0687     struct crypto_shash *child = ctx->child;
0688     int err;
0689 
0690     crypto_shash_clear_flags(child, CRYPTO_TFM_REQ_MASK);
0691     crypto_shash_set_flags(child, crypto_ahash_get_flags(parent) &
0692                       CRYPTO_TFM_REQ_MASK);
0693     err = crypto_shash_setkey(child, key, keylen);
0694     crypto_ahash_set_flags(parent, crypto_shash_get_flags(child) &
0695                        CRYPTO_TFM_RES_MASK);
0696     return err;
0697 }
0698 
0699 static int cryptd_hash_enqueue(struct ahash_request *req,
0700                 crypto_completion_t compl)
0701 {
0702     struct cryptd_hash_request_ctx *rctx = ahash_request_ctx(req);
0703     struct crypto_ahash *tfm = crypto_ahash_reqtfm(req);
0704     struct cryptd_queue *queue =
0705         cryptd_get_queue(crypto_ahash_tfm(tfm));
0706 
0707     rctx->complete = req->base.complete;
0708     req->base.complete = compl;
0709 
0710     return cryptd_enqueue_request(queue, &req->base);
0711 }
0712 
0713 static void cryptd_hash_complete(struct ahash_request *req, int err)
0714 {
0715     struct crypto_ahash *tfm = crypto_ahash_reqtfm(req);
0716     struct cryptd_hash_ctx *ctx = crypto_ahash_ctx(tfm);
0717     struct cryptd_hash_request_ctx *rctx = ahash_request_ctx(req);
0718     int refcnt = atomic_read(&ctx->refcnt);
0719 
0720     local_bh_disable();
0721     rctx->complete(&req->base, err);
0722     local_bh_enable();
0723 
0724     if (err != -EINPROGRESS && refcnt && atomic_dec_and_test(&ctx->refcnt))
0725         crypto_free_ahash(tfm);
0726 }
0727 
0728 static void cryptd_hash_init(struct crypto_async_request *req_async, int err)
0729 {
0730     struct cryptd_hash_ctx *ctx = crypto_tfm_ctx(req_async->tfm);
0731     struct crypto_shash *child = ctx->child;
0732     struct ahash_request *req = ahash_request_cast(req_async);
0733     struct cryptd_hash_request_ctx *rctx = ahash_request_ctx(req);
0734     struct shash_desc *desc = &rctx->desc;
0735 
0736     if (unlikely(err == -EINPROGRESS))
0737         goto out;
0738 
0739     desc->tfm = child;
0740     desc->flags = CRYPTO_TFM_REQ_MAY_SLEEP;
0741 
0742     err = crypto_shash_init(desc);
0743 
0744     req->base.complete = rctx->complete;
0745 
0746 out:
0747     cryptd_hash_complete(req, err);
0748 }
0749 
0750 static int cryptd_hash_init_enqueue(struct ahash_request *req)
0751 {
0752     return cryptd_hash_enqueue(req, cryptd_hash_init);
0753 }
0754 
0755 static void cryptd_hash_update(struct crypto_async_request *req_async, int err)
0756 {
0757     struct ahash_request *req = ahash_request_cast(req_async);
0758     struct cryptd_hash_request_ctx *rctx;
0759 
0760     rctx = ahash_request_ctx(req);
0761 
0762     if (unlikely(err == -EINPROGRESS))
0763         goto out;
0764 
0765     err = shash_ahash_update(req, &rctx->desc);
0766 
0767     req->base.complete = rctx->complete;
0768 
0769 out:
0770     cryptd_hash_complete(req, err);
0771 }
0772 
0773 static int cryptd_hash_update_enqueue(struct ahash_request *req)
0774 {
0775     return cryptd_hash_enqueue(req, cryptd_hash_update);
0776 }
0777 
0778 static void cryptd_hash_final(struct crypto_async_request *req_async, int err)
0779 {
0780     struct ahash_request *req = ahash_request_cast(req_async);
0781     struct cryptd_hash_request_ctx *rctx = ahash_request_ctx(req);
0782 
0783     if (unlikely(err == -EINPROGRESS))
0784         goto out;
0785 
0786     err = crypto_shash_final(&rctx->desc, req->result);
0787 
0788     req->base.complete = rctx->complete;
0789 
0790 out:
0791     cryptd_hash_complete(req, err);
0792 }
0793 
0794 static int cryptd_hash_final_enqueue(struct ahash_request *req)
0795 {
0796     return cryptd_hash_enqueue(req, cryptd_hash_final);
0797 }
0798 
0799 static void cryptd_hash_finup(struct crypto_async_request *req_async, int err)
0800 {
0801     struct ahash_request *req = ahash_request_cast(req_async);
0802     struct cryptd_hash_request_ctx *rctx = ahash_request_ctx(req);
0803 
0804     if (unlikely(err == -EINPROGRESS))
0805         goto out;
0806 
0807     err = shash_ahash_finup(req, &rctx->desc);
0808 
0809     req->base.complete = rctx->complete;
0810 
0811 out:
0812     cryptd_hash_complete(req, err);
0813 }
0814 
0815 static int cryptd_hash_finup_enqueue(struct ahash_request *req)
0816 {
0817     return cryptd_hash_enqueue(req, cryptd_hash_finup);
0818 }
0819 
0820 static void cryptd_hash_digest(struct crypto_async_request *req_async, int err)
0821 {
0822     struct cryptd_hash_ctx *ctx = crypto_tfm_ctx(req_async->tfm);
0823     struct crypto_shash *child = ctx->child;
0824     struct ahash_request *req = ahash_request_cast(req_async);
0825     struct cryptd_hash_request_ctx *rctx = ahash_request_ctx(req);
0826     struct shash_desc *desc = &rctx->desc;
0827 
0828     if (unlikely(err == -EINPROGRESS))
0829         goto out;
0830 
0831     desc->tfm = child;
0832     desc->flags = CRYPTO_TFM_REQ_MAY_SLEEP;
0833 
0834     err = shash_ahash_digest(req, desc);
0835 
0836     req->base.complete = rctx->complete;
0837 
0838 out:
0839     cryptd_hash_complete(req, err);
0840 }
0841 
0842 static int cryptd_hash_digest_enqueue(struct ahash_request *req)
0843 {
0844     return cryptd_hash_enqueue(req, cryptd_hash_digest);
0845 }
0846 
0847 static int cryptd_hash_export(struct ahash_request *req, void *out)
0848 {
0849     struct cryptd_hash_request_ctx *rctx = ahash_request_ctx(req);
0850 
0851     return crypto_shash_export(&rctx->desc, out);
0852 }
0853 
0854 static int cryptd_hash_import(struct ahash_request *req, const void *in)
0855 {
0856     struct crypto_ahash *tfm = crypto_ahash_reqtfm(req);
0857     struct cryptd_hash_ctx *ctx = crypto_ahash_ctx(tfm);
0858     struct shash_desc *desc = cryptd_shash_desc(req);
0859 
0860     desc->tfm = ctx->child;
0861     desc->flags = req->base.flags;
0862 
0863     return crypto_shash_import(desc, in);
0864 }
0865 
0866 static int cryptd_create_hash(struct crypto_template *tmpl, struct rtattr **tb,
0867                   struct cryptd_queue *queue)
0868 {
0869     struct hashd_instance_ctx *ctx;
0870     struct ahash_instance *inst;
0871     struct shash_alg *salg;
0872     struct crypto_alg *alg;
0873     u32 type = 0;
0874     u32 mask = 0;
0875     int err;
0876 
0877     cryptd_check_internal(tb, &type, &mask);
0878 
0879     salg = shash_attr_alg(tb[1], type, mask);
0880     if (IS_ERR(salg))
0881         return PTR_ERR(salg);
0882 
0883     alg = &salg->base;
0884     inst = cryptd_alloc_instance(alg, ahash_instance_headroom(),
0885                      sizeof(*ctx));
0886     err = PTR_ERR(inst);
0887     if (IS_ERR(inst))
0888         goto out_put_alg;
0889 
0890     ctx = ahash_instance_ctx(inst);
0891     ctx->queue = queue;
0892 
0893     err = crypto_init_shash_spawn(&ctx->spawn, salg,
0894                       ahash_crypto_instance(inst));
0895     if (err)
0896         goto out_free_inst;
0897 
0898     type = CRYPTO_ALG_ASYNC;
0899     if (alg->cra_flags & CRYPTO_ALG_INTERNAL)
0900         type |= CRYPTO_ALG_INTERNAL;
0901     inst->alg.halg.base.cra_flags = type;
0902 
0903     inst->alg.halg.digestsize = salg->digestsize;
0904     inst->alg.halg.statesize = salg->statesize;
0905     inst->alg.halg.base.cra_ctxsize = sizeof(struct cryptd_hash_ctx);
0906 
0907     inst->alg.halg.base.cra_init = cryptd_hash_init_tfm;
0908     inst->alg.halg.base.cra_exit = cryptd_hash_exit_tfm;
0909 
0910     inst->alg.init   = cryptd_hash_init_enqueue;
0911     inst->alg.update = cryptd_hash_update_enqueue;
0912     inst->alg.final  = cryptd_hash_final_enqueue;
0913     inst->alg.finup  = cryptd_hash_finup_enqueue;
0914     inst->alg.export = cryptd_hash_export;
0915     inst->alg.import = cryptd_hash_import;
0916     inst->alg.setkey = cryptd_hash_setkey;
0917     inst->alg.digest = cryptd_hash_digest_enqueue;
0918 
0919     err = ahash_register_instance(tmpl, inst);
0920     if (err) {
0921         crypto_drop_shash(&ctx->spawn);
0922 out_free_inst:
0923         kfree(inst);
0924     }
0925 
0926 out_put_alg:
0927     crypto_mod_put(alg);
0928     return err;
0929 }
0930 
0931 static int cryptd_aead_setkey(struct crypto_aead *parent,
0932                   const u8 *key, unsigned int keylen)
0933 {
0934     struct cryptd_aead_ctx *ctx = crypto_aead_ctx(parent);
0935     struct crypto_aead *child = ctx->child;
0936 
0937     return crypto_aead_setkey(child, key, keylen);
0938 }
0939 
0940 static int cryptd_aead_setauthsize(struct crypto_aead *parent,
0941                    unsigned int authsize)
0942 {
0943     struct cryptd_aead_ctx *ctx = crypto_aead_ctx(parent);
0944     struct crypto_aead *child = ctx->child;
0945 
0946     return crypto_aead_setauthsize(child, authsize);
0947 }
0948 
0949 static void cryptd_aead_crypt(struct aead_request *req,
0950             struct crypto_aead *child,
0951             int err,
0952             int (*crypt)(struct aead_request *req))
0953 {
0954     struct cryptd_aead_request_ctx *rctx;
0955     struct cryptd_aead_ctx *ctx;
0956     crypto_completion_t compl;
0957     struct crypto_aead *tfm;
0958     int refcnt;
0959 
0960     rctx = aead_request_ctx(req);
0961     compl = rctx->complete;
0962 
0963     tfm = crypto_aead_reqtfm(req);
0964 
0965     if (unlikely(err == -EINPROGRESS))
0966         goto out;
0967     aead_request_set_tfm(req, child);
0968     err = crypt( req );
0969 
0970 out:
0971     ctx = crypto_aead_ctx(tfm);
0972     refcnt = atomic_read(&ctx->refcnt);
0973 
0974     local_bh_disable();
0975     compl(&req->base, err);
0976     local_bh_enable();
0977 
0978     if (err != -EINPROGRESS && refcnt && atomic_dec_and_test(&ctx->refcnt))
0979         crypto_free_aead(tfm);
0980 }
0981 
0982 static void cryptd_aead_encrypt(struct crypto_async_request *areq, int err)
0983 {
0984     struct cryptd_aead_ctx *ctx = crypto_tfm_ctx(areq->tfm);
0985     struct crypto_aead *child = ctx->child;
0986     struct aead_request *req;
0987 
0988     req = container_of(areq, struct aead_request, base);
0989     cryptd_aead_crypt(req, child, err, crypto_aead_alg(child)->encrypt);
0990 }
0991 
0992 static void cryptd_aead_decrypt(struct crypto_async_request *areq, int err)
0993 {
0994     struct cryptd_aead_ctx *ctx = crypto_tfm_ctx(areq->tfm);
0995     struct crypto_aead *child = ctx->child;
0996     struct aead_request *req;
0997 
0998     req = container_of(areq, struct aead_request, base);
0999     cryptd_aead_crypt(req, child, err, crypto_aead_alg(child)->decrypt);
1000 }
1001 
1002 static int cryptd_aead_enqueue(struct aead_request *req,
1003                     crypto_completion_t compl)
1004 {
1005     struct cryptd_aead_request_ctx *rctx = aead_request_ctx(req);
1006     struct crypto_aead *tfm = crypto_aead_reqtfm(req);
1007     struct cryptd_queue *queue = cryptd_get_queue(crypto_aead_tfm(tfm));
1008 
1009     rctx->complete = req->base.complete;
1010     req->base.complete = compl;
1011     return cryptd_enqueue_request(queue, &req->base);
1012 }
1013 
1014 static int cryptd_aead_encrypt_enqueue(struct aead_request *req)
1015 {
1016     return cryptd_aead_enqueue(req, cryptd_aead_encrypt );
1017 }
1018 
1019 static int cryptd_aead_decrypt_enqueue(struct aead_request *req)
1020 {
1021     return cryptd_aead_enqueue(req, cryptd_aead_decrypt );
1022 }
1023 
1024 static int cryptd_aead_init_tfm(struct crypto_aead *tfm)
1025 {
1026     struct aead_instance *inst = aead_alg_instance(tfm);
1027     struct aead_instance_ctx *ictx = aead_instance_ctx(inst);
1028     struct crypto_aead_spawn *spawn = &ictx->aead_spawn;
1029     struct cryptd_aead_ctx *ctx = crypto_aead_ctx(tfm);
1030     struct crypto_aead *cipher;
1031 
1032     cipher = crypto_spawn_aead(spawn);
1033     if (IS_ERR(cipher))
1034         return PTR_ERR(cipher);
1035 
1036     ctx->child = cipher;
1037     crypto_aead_set_reqsize(
1038         tfm, max((unsigned)sizeof(struct cryptd_aead_request_ctx),
1039              crypto_aead_reqsize(cipher)));
1040     return 0;
1041 }
1042 
1043 static void cryptd_aead_exit_tfm(struct crypto_aead *tfm)
1044 {
1045     struct cryptd_aead_ctx *ctx = crypto_aead_ctx(tfm);
1046     crypto_free_aead(ctx->child);
1047 }
1048 
1049 static int cryptd_create_aead(struct crypto_template *tmpl,
1050                       struct rtattr **tb,
1051                   struct cryptd_queue *queue)
1052 {
1053     struct aead_instance_ctx *ctx;
1054     struct aead_instance *inst;
1055     struct aead_alg *alg;
1056     const char *name;
1057     u32 type = 0;
1058     u32 mask = CRYPTO_ALG_ASYNC;
1059     int err;
1060 
1061     cryptd_check_internal(tb, &type, &mask);
1062 
1063     name = crypto_attr_alg_name(tb[1]);
1064     if (IS_ERR(name))
1065         return PTR_ERR(name);
1066 
1067     inst = kzalloc(sizeof(*inst) + sizeof(*ctx), GFP_KERNEL);
1068     if (!inst)
1069         return -ENOMEM;
1070 
1071     ctx = aead_instance_ctx(inst);
1072     ctx->queue = queue;
1073 
1074     crypto_set_aead_spawn(&ctx->aead_spawn, aead_crypto_instance(inst));
1075     err = crypto_grab_aead(&ctx->aead_spawn, name, type, mask);
1076     if (err)
1077         goto out_free_inst;
1078 
1079     alg = crypto_spawn_aead_alg(&ctx->aead_spawn);
1080     err = cryptd_init_instance(aead_crypto_instance(inst), &alg->base);
1081     if (err)
1082         goto out_drop_aead;
1083 
1084     inst->alg.base.cra_flags = CRYPTO_ALG_ASYNC |
1085                    (alg->base.cra_flags & CRYPTO_ALG_INTERNAL);
1086     inst->alg.base.cra_ctxsize = sizeof(struct cryptd_aead_ctx);
1087 
1088     inst->alg.ivsize = crypto_aead_alg_ivsize(alg);
1089     inst->alg.maxauthsize = crypto_aead_alg_maxauthsize(alg);
1090 
1091     inst->alg.init = cryptd_aead_init_tfm;
1092     inst->alg.exit = cryptd_aead_exit_tfm;
1093     inst->alg.setkey = cryptd_aead_setkey;
1094     inst->alg.setauthsize = cryptd_aead_setauthsize;
1095     inst->alg.encrypt = cryptd_aead_encrypt_enqueue;
1096     inst->alg.decrypt = cryptd_aead_decrypt_enqueue;
1097 
1098     err = aead_register_instance(tmpl, inst);
1099     if (err) {
1100 out_drop_aead:
1101         crypto_drop_aead(&ctx->aead_spawn);
1102 out_free_inst:
1103         kfree(inst);
1104     }
1105     return err;
1106 }
1107 
1108 static struct cryptd_queue queue;
1109 
1110 static int cryptd_create(struct crypto_template *tmpl, struct rtattr **tb)
1111 {
1112     struct crypto_attr_type *algt;
1113 
1114     algt = crypto_get_attr_type(tb);
1115     if (IS_ERR(algt))
1116         return PTR_ERR(algt);
1117 
1118     switch (algt->type & algt->mask & CRYPTO_ALG_TYPE_MASK) {
1119     case CRYPTO_ALG_TYPE_BLKCIPHER:
1120         if ((algt->type & CRYPTO_ALG_TYPE_MASK) ==
1121             CRYPTO_ALG_TYPE_BLKCIPHER)
1122             return cryptd_create_blkcipher(tmpl, tb, &queue);
1123 
1124         return cryptd_create_skcipher(tmpl, tb, &queue);
1125     case CRYPTO_ALG_TYPE_DIGEST:
1126         return cryptd_create_hash(tmpl, tb, &queue);
1127     case CRYPTO_ALG_TYPE_AEAD:
1128         return cryptd_create_aead(tmpl, tb, &queue);
1129     }
1130 
1131     return -EINVAL;
1132 }
1133 
1134 static void cryptd_free(struct crypto_instance *inst)
1135 {
1136     struct cryptd_instance_ctx *ctx = crypto_instance_ctx(inst);
1137     struct hashd_instance_ctx *hctx = crypto_instance_ctx(inst);
1138     struct aead_instance_ctx *aead_ctx = crypto_instance_ctx(inst);
1139 
1140     switch (inst->alg.cra_flags & CRYPTO_ALG_TYPE_MASK) {
1141     case CRYPTO_ALG_TYPE_AHASH:
1142         crypto_drop_shash(&hctx->spawn);
1143         kfree(ahash_instance(inst));
1144         return;
1145     case CRYPTO_ALG_TYPE_AEAD:
1146         crypto_drop_aead(&aead_ctx->aead_spawn);
1147         kfree(aead_instance(inst));
1148         return;
1149     default:
1150         crypto_drop_spawn(&ctx->spawn);
1151         kfree(inst);
1152     }
1153 }
1154 
1155 static struct crypto_template cryptd_tmpl = {
1156     .name = "cryptd",
1157     .create = cryptd_create,
1158     .free = cryptd_free,
1159     .module = THIS_MODULE,
1160 };
1161 
1162 struct cryptd_ablkcipher *cryptd_alloc_ablkcipher(const char *alg_name,
1163                           u32 type, u32 mask)
1164 {
1165     char cryptd_alg_name[CRYPTO_MAX_ALG_NAME];
1166     struct cryptd_blkcipher_ctx *ctx;
1167     struct crypto_tfm *tfm;
1168 
1169     if (snprintf(cryptd_alg_name, CRYPTO_MAX_ALG_NAME,
1170              "cryptd(%s)", alg_name) >= CRYPTO_MAX_ALG_NAME)
1171         return ERR_PTR(-EINVAL);
1172     type = crypto_skcipher_type(type);
1173     mask &= ~CRYPTO_ALG_TYPE_MASK;
1174     mask |= (CRYPTO_ALG_GENIV | CRYPTO_ALG_TYPE_BLKCIPHER_MASK);
1175     tfm = crypto_alloc_base(cryptd_alg_name, type, mask);
1176     if (IS_ERR(tfm))
1177         return ERR_CAST(tfm);
1178     if (tfm->__crt_alg->cra_module != THIS_MODULE) {
1179         crypto_free_tfm(tfm);
1180         return ERR_PTR(-EINVAL);
1181     }
1182 
1183     ctx = crypto_tfm_ctx(tfm);
1184     atomic_set(&ctx->refcnt, 1);
1185 
1186     return __cryptd_ablkcipher_cast(__crypto_ablkcipher_cast(tfm));
1187 }
1188 EXPORT_SYMBOL_GPL(cryptd_alloc_ablkcipher);
1189 
1190 struct crypto_blkcipher *cryptd_ablkcipher_child(struct cryptd_ablkcipher *tfm)
1191 {
1192     struct cryptd_blkcipher_ctx *ctx = crypto_ablkcipher_ctx(&tfm->base);
1193     return ctx->child;
1194 }
1195 EXPORT_SYMBOL_GPL(cryptd_ablkcipher_child);
1196 
1197 bool cryptd_ablkcipher_queued(struct cryptd_ablkcipher *tfm)
1198 {
1199     struct cryptd_blkcipher_ctx *ctx = crypto_ablkcipher_ctx(&tfm->base);
1200 
1201     return atomic_read(&ctx->refcnt) - 1;
1202 }
1203 EXPORT_SYMBOL_GPL(cryptd_ablkcipher_queued);
1204 
1205 void cryptd_free_ablkcipher(struct cryptd_ablkcipher *tfm)
1206 {
1207     struct cryptd_blkcipher_ctx *ctx = crypto_ablkcipher_ctx(&tfm->base);
1208 
1209     if (atomic_dec_and_test(&ctx->refcnt))
1210         crypto_free_ablkcipher(&tfm->base);
1211 }
1212 EXPORT_SYMBOL_GPL(cryptd_free_ablkcipher);
1213 
1214 struct cryptd_skcipher *cryptd_alloc_skcipher(const char *alg_name,
1215                           u32 type, u32 mask)
1216 {
1217     char cryptd_alg_name[CRYPTO_MAX_ALG_NAME];
1218     struct cryptd_skcipher_ctx *ctx;
1219     struct crypto_skcipher *tfm;
1220 
1221     if (snprintf(cryptd_alg_name, CRYPTO_MAX_ALG_NAME,
1222              "cryptd(%s)", alg_name) >= CRYPTO_MAX_ALG_NAME)
1223         return ERR_PTR(-EINVAL);
1224 
1225     tfm = crypto_alloc_skcipher(cryptd_alg_name, type, mask);
1226     if (IS_ERR(tfm))
1227         return ERR_CAST(tfm);
1228 
1229     if (tfm->base.__crt_alg->cra_module != THIS_MODULE) {
1230         crypto_free_skcipher(tfm);
1231         return ERR_PTR(-EINVAL);
1232     }
1233 
1234     ctx = crypto_skcipher_ctx(tfm);
1235     atomic_set(&ctx->refcnt, 1);
1236 
1237     return container_of(tfm, struct cryptd_skcipher, base);
1238 }
1239 EXPORT_SYMBOL_GPL(cryptd_alloc_skcipher);
1240 
1241 struct crypto_skcipher *cryptd_skcipher_child(struct cryptd_skcipher *tfm)
1242 {
1243     struct cryptd_skcipher_ctx *ctx = crypto_skcipher_ctx(&tfm->base);
1244 
1245     return ctx->child;
1246 }
1247 EXPORT_SYMBOL_GPL(cryptd_skcipher_child);
1248 
1249 bool cryptd_skcipher_queued(struct cryptd_skcipher *tfm)
1250 {
1251     struct cryptd_skcipher_ctx *ctx = crypto_skcipher_ctx(&tfm->base);
1252 
1253     return atomic_read(&ctx->refcnt) - 1;
1254 }
1255 EXPORT_SYMBOL_GPL(cryptd_skcipher_queued);
1256 
1257 void cryptd_free_skcipher(struct cryptd_skcipher *tfm)
1258 {
1259     struct cryptd_skcipher_ctx *ctx = crypto_skcipher_ctx(&tfm->base);
1260 
1261     if (atomic_dec_and_test(&ctx->refcnt))
1262         crypto_free_skcipher(&tfm->base);
1263 }
1264 EXPORT_SYMBOL_GPL(cryptd_free_skcipher);
1265 
1266 struct cryptd_ahash *cryptd_alloc_ahash(const char *alg_name,
1267                     u32 type, u32 mask)
1268 {
1269     char cryptd_alg_name[CRYPTO_MAX_ALG_NAME];
1270     struct cryptd_hash_ctx *ctx;
1271     struct crypto_ahash *tfm;
1272 
1273     if (snprintf(cryptd_alg_name, CRYPTO_MAX_ALG_NAME,
1274              "cryptd(%s)", alg_name) >= CRYPTO_MAX_ALG_NAME)
1275         return ERR_PTR(-EINVAL);
1276     tfm = crypto_alloc_ahash(cryptd_alg_name, type, mask);
1277     if (IS_ERR(tfm))
1278         return ERR_CAST(tfm);
1279     if (tfm->base.__crt_alg->cra_module != THIS_MODULE) {
1280         crypto_free_ahash(tfm);
1281         return ERR_PTR(-EINVAL);
1282     }
1283 
1284     ctx = crypto_ahash_ctx(tfm);
1285     atomic_set(&ctx->refcnt, 1);
1286 
1287     return __cryptd_ahash_cast(tfm);
1288 }
1289 EXPORT_SYMBOL_GPL(cryptd_alloc_ahash);
1290 
1291 struct crypto_shash *cryptd_ahash_child(struct cryptd_ahash *tfm)
1292 {
1293     struct cryptd_hash_ctx *ctx = crypto_ahash_ctx(&tfm->base);
1294 
1295     return ctx->child;
1296 }
1297 EXPORT_SYMBOL_GPL(cryptd_ahash_child);
1298 
1299 struct shash_desc *cryptd_shash_desc(struct ahash_request *req)
1300 {
1301     struct cryptd_hash_request_ctx *rctx = ahash_request_ctx(req);
1302     return &rctx->desc;
1303 }
1304 EXPORT_SYMBOL_GPL(cryptd_shash_desc);
1305 
1306 bool cryptd_ahash_queued(struct cryptd_ahash *tfm)
1307 {
1308     struct cryptd_hash_ctx *ctx = crypto_ahash_ctx(&tfm->base);
1309 
1310     return atomic_read(&ctx->refcnt) - 1;
1311 }
1312 EXPORT_SYMBOL_GPL(cryptd_ahash_queued);
1313 
1314 void cryptd_free_ahash(struct cryptd_ahash *tfm)
1315 {
1316     struct cryptd_hash_ctx *ctx = crypto_ahash_ctx(&tfm->base);
1317 
1318     if (atomic_dec_and_test(&ctx->refcnt))
1319         crypto_free_ahash(&tfm->base);
1320 }
1321 EXPORT_SYMBOL_GPL(cryptd_free_ahash);
1322 
1323 struct cryptd_aead *cryptd_alloc_aead(const char *alg_name,
1324                           u32 type, u32 mask)
1325 {
1326     char cryptd_alg_name[CRYPTO_MAX_ALG_NAME];
1327     struct cryptd_aead_ctx *ctx;
1328     struct crypto_aead *tfm;
1329 
1330     if (snprintf(cryptd_alg_name, CRYPTO_MAX_ALG_NAME,
1331              "cryptd(%s)", alg_name) >= CRYPTO_MAX_ALG_NAME)
1332         return ERR_PTR(-EINVAL);
1333     tfm = crypto_alloc_aead(cryptd_alg_name, type, mask);
1334     if (IS_ERR(tfm))
1335         return ERR_CAST(tfm);
1336     if (tfm->base.__crt_alg->cra_module != THIS_MODULE) {
1337         crypto_free_aead(tfm);
1338         return ERR_PTR(-EINVAL);
1339     }
1340 
1341     ctx = crypto_aead_ctx(tfm);
1342     atomic_set(&ctx->refcnt, 1);
1343 
1344     return __cryptd_aead_cast(tfm);
1345 }
1346 EXPORT_SYMBOL_GPL(cryptd_alloc_aead);
1347 
1348 struct crypto_aead *cryptd_aead_child(struct cryptd_aead *tfm)
1349 {
1350     struct cryptd_aead_ctx *ctx;
1351     ctx = crypto_aead_ctx(&tfm->base);
1352     return ctx->child;
1353 }
1354 EXPORT_SYMBOL_GPL(cryptd_aead_child);
1355 
1356 bool cryptd_aead_queued(struct cryptd_aead *tfm)
1357 {
1358     struct cryptd_aead_ctx *ctx = crypto_aead_ctx(&tfm->base);
1359 
1360     return atomic_read(&ctx->refcnt) - 1;
1361 }
1362 EXPORT_SYMBOL_GPL(cryptd_aead_queued);
1363 
1364 void cryptd_free_aead(struct cryptd_aead *tfm)
1365 {
1366     struct cryptd_aead_ctx *ctx = crypto_aead_ctx(&tfm->base);
1367 
1368     if (atomic_dec_and_test(&ctx->refcnt))
1369         crypto_free_aead(&tfm->base);
1370 }
1371 EXPORT_SYMBOL_GPL(cryptd_free_aead);
1372 
1373 static int __init cryptd_init(void)
1374 {
1375     int err;
1376 
1377     err = cryptd_init_queue(&queue, CRYPTD_MAX_CPU_QLEN);
1378     if (err)
1379         return err;
1380 
1381     err = crypto_register_template(&cryptd_tmpl);
1382     if (err)
1383         cryptd_fini_queue(&queue);
1384 
1385     return err;
1386 }
1387 
1388 static void __exit cryptd_exit(void)
1389 {
1390     cryptd_fini_queue(&queue);
1391     crypto_unregister_template(&cryptd_tmpl);
1392 }
1393 
1394 subsys_initcall(cryptd_init);
1395 module_exit(cryptd_exit);
1396 
1397 MODULE_LICENSE("GPL");
1398 MODULE_DESCRIPTION("Software async crypto daemon");
1399 MODULE_ALIAS_CRYPTO("cryptd");