Back to home page

LXR

 
 

    


0001 /*
0002  * Software multibuffer async crypto daemon.
0003  *
0004  * Copyright (c) 2014 Tim Chen <tim.c.chen@linux.intel.com>
0005  *
0006  * Adapted from crypto daemon.
0007  *
0008  * This program is free software; you can redistribute it and/or modify it
0009  * under the terms of the GNU General Public License as published by the Free
0010  * Software Foundation; either version 2 of the License, or (at your option)
0011  * any later version.
0012  *
0013  */
0014 
0015 #include <crypto/algapi.h>
0016 #include <crypto/internal/hash.h>
0017 #include <crypto/internal/aead.h>
0018 #include <crypto/mcryptd.h>
0019 #include <crypto/crypto_wq.h>
0020 #include <linux/err.h>
0021 #include <linux/init.h>
0022 #include <linux/kernel.h>
0023 #include <linux/list.h>
0024 #include <linux/module.h>
0025 #include <linux/scatterlist.h>
0026 #include <linux/sched.h>
0027 #include <linux/slab.h>
0028 #include <linux/hardirq.h>
0029 
0030 #define MCRYPTD_MAX_CPU_QLEN 100
0031 #define MCRYPTD_BATCH 9
0032 
0033 static void *mcryptd_alloc_instance(struct crypto_alg *alg, unsigned int head,
0034                    unsigned int tail);
0035 
0036 struct mcryptd_flush_list {
0037     struct list_head list;
0038     struct mutex lock;
0039 };
0040 
0041 static struct mcryptd_flush_list __percpu *mcryptd_flist;
0042 
0043 struct hashd_instance_ctx {
0044     struct crypto_ahash_spawn spawn;
0045     struct mcryptd_queue *queue;
0046 };
0047 
0048 static void mcryptd_queue_worker(struct work_struct *work);
0049 
0050 void mcryptd_arm_flusher(struct mcryptd_alg_cstate *cstate, unsigned long delay)
0051 {
0052     struct mcryptd_flush_list *flist;
0053 
0054     if (!cstate->flusher_engaged) {
0055         /* put the flusher on the flush list */
0056         flist = per_cpu_ptr(mcryptd_flist, smp_processor_id());
0057         mutex_lock(&flist->lock);
0058         list_add_tail(&cstate->flush_list, &flist->list);
0059         cstate->flusher_engaged = true;
0060         cstate->next_flush = jiffies + delay;
0061         queue_delayed_work_on(smp_processor_id(), kcrypto_wq,
0062             &cstate->flush, delay);
0063         mutex_unlock(&flist->lock);
0064     }
0065 }
0066 EXPORT_SYMBOL(mcryptd_arm_flusher);
0067 
0068 static int mcryptd_init_queue(struct mcryptd_queue *queue,
0069                  unsigned int max_cpu_qlen)
0070 {
0071     int cpu;
0072     struct mcryptd_cpu_queue *cpu_queue;
0073 
0074     queue->cpu_queue = alloc_percpu(struct mcryptd_cpu_queue);
0075     pr_debug("mqueue:%p mcryptd_cpu_queue %p\n", queue, queue->cpu_queue);
0076     if (!queue->cpu_queue)
0077         return -ENOMEM;
0078     for_each_possible_cpu(cpu) {
0079         cpu_queue = per_cpu_ptr(queue->cpu_queue, cpu);
0080         pr_debug("cpu_queue #%d %p\n", cpu, queue->cpu_queue);
0081         crypto_init_queue(&cpu_queue->queue, max_cpu_qlen);
0082         INIT_WORK(&cpu_queue->work, mcryptd_queue_worker);
0083     }
0084     return 0;
0085 }
0086 
0087 static void mcryptd_fini_queue(struct mcryptd_queue *queue)
0088 {
0089     int cpu;
0090     struct mcryptd_cpu_queue *cpu_queue;
0091 
0092     for_each_possible_cpu(cpu) {
0093         cpu_queue = per_cpu_ptr(queue->cpu_queue, cpu);
0094         BUG_ON(cpu_queue->queue.qlen);
0095     }
0096     free_percpu(queue->cpu_queue);
0097 }
0098 
0099 static int mcryptd_enqueue_request(struct mcryptd_queue *queue,
0100                   struct crypto_async_request *request,
0101                   struct mcryptd_hash_request_ctx *rctx)
0102 {
0103     int cpu, err;
0104     struct mcryptd_cpu_queue *cpu_queue;
0105 
0106     cpu = get_cpu();
0107     cpu_queue = this_cpu_ptr(queue->cpu_queue);
0108     rctx->tag.cpu = cpu;
0109 
0110     err = crypto_enqueue_request(&cpu_queue->queue, request);
0111     pr_debug("enqueue request: cpu %d cpu_queue %p request %p\n",
0112          cpu, cpu_queue, request);
0113     queue_work_on(cpu, kcrypto_wq, &cpu_queue->work);
0114     put_cpu();
0115 
0116     return err;
0117 }
0118 
0119 /*
0120  * Try to opportunisticlly flush the partially completed jobs if
0121  * crypto daemon is the only task running.
0122  */
0123 static void mcryptd_opportunistic_flush(void)
0124 {
0125     struct mcryptd_flush_list *flist;
0126     struct mcryptd_alg_cstate *cstate;
0127 
0128     flist = per_cpu_ptr(mcryptd_flist, smp_processor_id());
0129     while (single_task_running()) {
0130         mutex_lock(&flist->lock);
0131         cstate = list_first_entry_or_null(&flist->list,
0132                 struct mcryptd_alg_cstate, flush_list);
0133         if (!cstate || !cstate->flusher_engaged) {
0134             mutex_unlock(&flist->lock);
0135             return;
0136         }
0137         list_del(&cstate->flush_list);
0138         cstate->flusher_engaged = false;
0139         mutex_unlock(&flist->lock);
0140         cstate->alg_state->flusher(cstate);
0141     }
0142 }
0143 
0144 /*
0145  * Called in workqueue context, do one real cryption work (via
0146  * req->complete) and reschedule itself if there are more work to
0147  * do.
0148  */
0149 static void mcryptd_queue_worker(struct work_struct *work)
0150 {
0151     struct mcryptd_cpu_queue *cpu_queue;
0152     struct crypto_async_request *req, *backlog;
0153     int i;
0154 
0155     /*
0156      * Need to loop through more than once for multi-buffer to
0157      * be effective.
0158      */
0159 
0160     cpu_queue = container_of(work, struct mcryptd_cpu_queue, work);
0161     i = 0;
0162     while (i < MCRYPTD_BATCH || single_task_running()) {
0163         /*
0164          * preempt_disable/enable is used to prevent
0165          * being preempted by mcryptd_enqueue_request()
0166          */
0167         local_bh_disable();
0168         preempt_disable();
0169         backlog = crypto_get_backlog(&cpu_queue->queue);
0170         req = crypto_dequeue_request(&cpu_queue->queue);
0171         preempt_enable();
0172         local_bh_enable();
0173 
0174         if (!req) {
0175             mcryptd_opportunistic_flush();
0176             return;
0177         }
0178 
0179         if (backlog)
0180             backlog->complete(backlog, -EINPROGRESS);
0181         req->complete(req, 0);
0182         if (!cpu_queue->queue.qlen)
0183             return;
0184         ++i;
0185     }
0186     if (cpu_queue->queue.qlen)
0187         queue_work(kcrypto_wq, &cpu_queue->work);
0188 }
0189 
0190 void mcryptd_flusher(struct work_struct *__work)
0191 {
0192     struct  mcryptd_alg_cstate  *alg_cpu_state;
0193     struct  mcryptd_alg_state   *alg_state;
0194     struct  mcryptd_flush_list  *flist;
0195     int cpu;
0196 
0197     cpu = smp_processor_id();
0198     alg_cpu_state = container_of(to_delayed_work(__work),
0199                      struct mcryptd_alg_cstate, flush);
0200     alg_state = alg_cpu_state->alg_state;
0201     if (alg_cpu_state->cpu != cpu)
0202         pr_debug("mcryptd error: work on cpu %d, should be cpu %d\n",
0203                 cpu, alg_cpu_state->cpu);
0204 
0205     if (alg_cpu_state->flusher_engaged) {
0206         flist = per_cpu_ptr(mcryptd_flist, cpu);
0207         mutex_lock(&flist->lock);
0208         list_del(&alg_cpu_state->flush_list);
0209         alg_cpu_state->flusher_engaged = false;
0210         mutex_unlock(&flist->lock);
0211         alg_state->flusher(alg_cpu_state);
0212     }
0213 }
0214 EXPORT_SYMBOL_GPL(mcryptd_flusher);
0215 
0216 static inline struct mcryptd_queue *mcryptd_get_queue(struct crypto_tfm *tfm)
0217 {
0218     struct crypto_instance *inst = crypto_tfm_alg_instance(tfm);
0219     struct mcryptd_instance_ctx *ictx = crypto_instance_ctx(inst);
0220 
0221     return ictx->queue;
0222 }
0223 
0224 static void *mcryptd_alloc_instance(struct crypto_alg *alg, unsigned int head,
0225                    unsigned int tail)
0226 {
0227     char *p;
0228     struct crypto_instance *inst;
0229     int err;
0230 
0231     p = kzalloc(head + sizeof(*inst) + tail, GFP_KERNEL);
0232     if (!p)
0233         return ERR_PTR(-ENOMEM);
0234 
0235     inst = (void *)(p + head);
0236 
0237     err = -ENAMETOOLONG;
0238     if (snprintf(inst->alg.cra_driver_name, CRYPTO_MAX_ALG_NAME,
0239             "mcryptd(%s)", alg->cra_driver_name) >= CRYPTO_MAX_ALG_NAME)
0240         goto out_free_inst;
0241 
0242     memcpy(inst->alg.cra_name, alg->cra_name, CRYPTO_MAX_ALG_NAME);
0243 
0244     inst->alg.cra_priority = alg->cra_priority + 50;
0245     inst->alg.cra_blocksize = alg->cra_blocksize;
0246     inst->alg.cra_alignmask = alg->cra_alignmask;
0247 
0248 out:
0249     return p;
0250 
0251 out_free_inst:
0252     kfree(p);
0253     p = ERR_PTR(err);
0254     goto out;
0255 }
0256 
0257 static inline bool mcryptd_check_internal(struct rtattr **tb, u32 *type,
0258                       u32 *mask)
0259 {
0260     struct crypto_attr_type *algt;
0261 
0262     algt = crypto_get_attr_type(tb);
0263     if (IS_ERR(algt))
0264         return false;
0265 
0266     *type |= algt->type & CRYPTO_ALG_INTERNAL;
0267     *mask |= algt->mask & CRYPTO_ALG_INTERNAL;
0268 
0269     if (*type & *mask & CRYPTO_ALG_INTERNAL)
0270         return true;
0271     else
0272         return false;
0273 }
0274 
0275 static int mcryptd_hash_init_tfm(struct crypto_tfm *tfm)
0276 {
0277     struct crypto_instance *inst = crypto_tfm_alg_instance(tfm);
0278     struct hashd_instance_ctx *ictx = crypto_instance_ctx(inst);
0279     struct crypto_ahash_spawn *spawn = &ictx->spawn;
0280     struct mcryptd_hash_ctx *ctx = crypto_tfm_ctx(tfm);
0281     struct crypto_ahash *hash;
0282 
0283     hash = crypto_spawn_ahash(spawn);
0284     if (IS_ERR(hash))
0285         return PTR_ERR(hash);
0286 
0287     ctx->child = hash;
0288     crypto_ahash_set_reqsize(__crypto_ahash_cast(tfm),
0289                  sizeof(struct mcryptd_hash_request_ctx) +
0290                  crypto_ahash_reqsize(hash));
0291     return 0;
0292 }
0293 
0294 static void mcryptd_hash_exit_tfm(struct crypto_tfm *tfm)
0295 {
0296     struct mcryptd_hash_ctx *ctx = crypto_tfm_ctx(tfm);
0297 
0298     crypto_free_ahash(ctx->child);
0299 }
0300 
0301 static int mcryptd_hash_setkey(struct crypto_ahash *parent,
0302                    const u8 *key, unsigned int keylen)
0303 {
0304     struct mcryptd_hash_ctx *ctx   = crypto_ahash_ctx(parent);
0305     struct crypto_ahash *child = ctx->child;
0306     int err;
0307 
0308     crypto_ahash_clear_flags(child, CRYPTO_TFM_REQ_MASK);
0309     crypto_ahash_set_flags(child, crypto_ahash_get_flags(parent) &
0310                       CRYPTO_TFM_REQ_MASK);
0311     err = crypto_ahash_setkey(child, key, keylen);
0312     crypto_ahash_set_flags(parent, crypto_ahash_get_flags(child) &
0313                        CRYPTO_TFM_RES_MASK);
0314     return err;
0315 }
0316 
0317 static int mcryptd_hash_enqueue(struct ahash_request *req,
0318                 crypto_completion_t complete)
0319 {
0320     int ret;
0321 
0322     struct mcryptd_hash_request_ctx *rctx = ahash_request_ctx(req);
0323     struct crypto_ahash *tfm = crypto_ahash_reqtfm(req);
0324     struct mcryptd_queue *queue =
0325         mcryptd_get_queue(crypto_ahash_tfm(tfm));
0326 
0327     rctx->complete = req->base.complete;
0328     req->base.complete = complete;
0329 
0330     ret = mcryptd_enqueue_request(queue, &req->base, rctx);
0331 
0332     return ret;
0333 }
0334 
0335 static void mcryptd_hash_init(struct crypto_async_request *req_async, int err)
0336 {
0337     struct mcryptd_hash_ctx *ctx = crypto_tfm_ctx(req_async->tfm);
0338     struct crypto_ahash *child = ctx->child;
0339     struct ahash_request *req = ahash_request_cast(req_async);
0340     struct mcryptd_hash_request_ctx *rctx = ahash_request_ctx(req);
0341     struct ahash_request *desc = &rctx->areq;
0342 
0343     if (unlikely(err == -EINPROGRESS))
0344         goto out;
0345 
0346     ahash_request_set_tfm(desc, child);
0347     ahash_request_set_callback(desc, CRYPTO_TFM_REQ_MAY_SLEEP,
0348                         rctx->complete, req_async);
0349 
0350     rctx->out = req->result;
0351     err = crypto_ahash_init(desc);
0352 
0353 out:
0354     local_bh_disable();
0355     rctx->complete(&req->base, err);
0356     local_bh_enable();
0357 }
0358 
0359 static int mcryptd_hash_init_enqueue(struct ahash_request *req)
0360 {
0361     return mcryptd_hash_enqueue(req, mcryptd_hash_init);
0362 }
0363 
0364 static void mcryptd_hash_update(struct crypto_async_request *req_async, int err)
0365 {
0366     struct ahash_request *req = ahash_request_cast(req_async);
0367     struct mcryptd_hash_request_ctx *rctx = ahash_request_ctx(req);
0368 
0369     if (unlikely(err == -EINPROGRESS))
0370         goto out;
0371 
0372     rctx->out = req->result;
0373     err = ahash_mcryptd_update(&rctx->areq);
0374     if (err) {
0375         req->base.complete = rctx->complete;
0376         goto out;
0377     }
0378 
0379     return;
0380 out:
0381     local_bh_disable();
0382     rctx->complete(&req->base, err);
0383     local_bh_enable();
0384 }
0385 
0386 static int mcryptd_hash_update_enqueue(struct ahash_request *req)
0387 {
0388     return mcryptd_hash_enqueue(req, mcryptd_hash_update);
0389 }
0390 
0391 static void mcryptd_hash_final(struct crypto_async_request *req_async, int err)
0392 {
0393     struct ahash_request *req = ahash_request_cast(req_async);
0394     struct mcryptd_hash_request_ctx *rctx = ahash_request_ctx(req);
0395 
0396     if (unlikely(err == -EINPROGRESS))
0397         goto out;
0398 
0399     rctx->out = req->result;
0400     err = ahash_mcryptd_final(&rctx->areq);
0401     if (err) {
0402         req->base.complete = rctx->complete;
0403         goto out;
0404     }
0405 
0406     return;
0407 out:
0408     local_bh_disable();
0409     rctx->complete(&req->base, err);
0410     local_bh_enable();
0411 }
0412 
0413 static int mcryptd_hash_final_enqueue(struct ahash_request *req)
0414 {
0415     return mcryptd_hash_enqueue(req, mcryptd_hash_final);
0416 }
0417 
0418 static void mcryptd_hash_finup(struct crypto_async_request *req_async, int err)
0419 {
0420     struct ahash_request *req = ahash_request_cast(req_async);
0421     struct mcryptd_hash_request_ctx *rctx = ahash_request_ctx(req);
0422 
0423     if (unlikely(err == -EINPROGRESS))
0424         goto out;
0425     rctx->out = req->result;
0426     err = ahash_mcryptd_finup(&rctx->areq);
0427 
0428     if (err) {
0429         req->base.complete = rctx->complete;
0430         goto out;
0431     }
0432 
0433     return;
0434 out:
0435     local_bh_disable();
0436     rctx->complete(&req->base, err);
0437     local_bh_enable();
0438 }
0439 
0440 static int mcryptd_hash_finup_enqueue(struct ahash_request *req)
0441 {
0442     return mcryptd_hash_enqueue(req, mcryptd_hash_finup);
0443 }
0444 
0445 static void mcryptd_hash_digest(struct crypto_async_request *req_async, int err)
0446 {
0447     struct mcryptd_hash_ctx *ctx = crypto_tfm_ctx(req_async->tfm);
0448     struct crypto_ahash *child = ctx->child;
0449     struct ahash_request *req = ahash_request_cast(req_async);
0450     struct mcryptd_hash_request_ctx *rctx = ahash_request_ctx(req);
0451     struct ahash_request *desc = &rctx->areq;
0452 
0453     if (unlikely(err == -EINPROGRESS))
0454         goto out;
0455 
0456     ahash_request_set_tfm(desc, child);
0457     ahash_request_set_callback(desc, CRYPTO_TFM_REQ_MAY_SLEEP,
0458                         rctx->complete, req_async);
0459 
0460     rctx->out = req->result;
0461     err = ahash_mcryptd_digest(desc);
0462 
0463 out:
0464     local_bh_disable();
0465     rctx->complete(&req->base, err);
0466     local_bh_enable();
0467 }
0468 
0469 static int mcryptd_hash_digest_enqueue(struct ahash_request *req)
0470 {
0471     return mcryptd_hash_enqueue(req, mcryptd_hash_digest);
0472 }
0473 
0474 static int mcryptd_hash_export(struct ahash_request *req, void *out)
0475 {
0476     struct mcryptd_hash_request_ctx *rctx = ahash_request_ctx(req);
0477 
0478     return crypto_ahash_export(&rctx->areq, out);
0479 }
0480 
0481 static int mcryptd_hash_import(struct ahash_request *req, const void *in)
0482 {
0483     struct mcryptd_hash_request_ctx *rctx = ahash_request_ctx(req);
0484 
0485     return crypto_ahash_import(&rctx->areq, in);
0486 }
0487 
0488 static int mcryptd_create_hash(struct crypto_template *tmpl, struct rtattr **tb,
0489                   struct mcryptd_queue *queue)
0490 {
0491     struct hashd_instance_ctx *ctx;
0492     struct ahash_instance *inst;
0493     struct hash_alg_common *halg;
0494     struct crypto_alg *alg;
0495     u32 type = 0;
0496     u32 mask = 0;
0497     int err;
0498 
0499     if (!mcryptd_check_internal(tb, &type, &mask))
0500         return -EINVAL;
0501 
0502     halg = ahash_attr_alg(tb[1], type, mask);
0503     if (IS_ERR(halg))
0504         return PTR_ERR(halg);
0505 
0506     alg = &halg->base;
0507     pr_debug("crypto: mcryptd hash alg: %s\n", alg->cra_name);
0508     inst = mcryptd_alloc_instance(alg, ahash_instance_headroom(),
0509                     sizeof(*ctx));
0510     err = PTR_ERR(inst);
0511     if (IS_ERR(inst))
0512         goto out_put_alg;
0513 
0514     ctx = ahash_instance_ctx(inst);
0515     ctx->queue = queue;
0516 
0517     err = crypto_init_ahash_spawn(&ctx->spawn, halg,
0518                       ahash_crypto_instance(inst));
0519     if (err)
0520         goto out_free_inst;
0521 
0522     type = CRYPTO_ALG_ASYNC;
0523     if (alg->cra_flags & CRYPTO_ALG_INTERNAL)
0524         type |= CRYPTO_ALG_INTERNAL;
0525     inst->alg.halg.base.cra_flags = type;
0526 
0527     inst->alg.halg.digestsize = halg->digestsize;
0528     inst->alg.halg.statesize = halg->statesize;
0529     inst->alg.halg.base.cra_ctxsize = sizeof(struct mcryptd_hash_ctx);
0530 
0531     inst->alg.halg.base.cra_init = mcryptd_hash_init_tfm;
0532     inst->alg.halg.base.cra_exit = mcryptd_hash_exit_tfm;
0533 
0534     inst->alg.init   = mcryptd_hash_init_enqueue;
0535     inst->alg.update = mcryptd_hash_update_enqueue;
0536     inst->alg.final  = mcryptd_hash_final_enqueue;
0537     inst->alg.finup  = mcryptd_hash_finup_enqueue;
0538     inst->alg.export = mcryptd_hash_export;
0539     inst->alg.import = mcryptd_hash_import;
0540     inst->alg.setkey = mcryptd_hash_setkey;
0541     inst->alg.digest = mcryptd_hash_digest_enqueue;
0542 
0543     err = ahash_register_instance(tmpl, inst);
0544     if (err) {
0545         crypto_drop_ahash(&ctx->spawn);
0546 out_free_inst:
0547         kfree(inst);
0548     }
0549 
0550 out_put_alg:
0551     crypto_mod_put(alg);
0552     return err;
0553 }
0554 
0555 static struct mcryptd_queue mqueue;
0556 
0557 static int mcryptd_create(struct crypto_template *tmpl, struct rtattr **tb)
0558 {
0559     struct crypto_attr_type *algt;
0560 
0561     algt = crypto_get_attr_type(tb);
0562     if (IS_ERR(algt))
0563         return PTR_ERR(algt);
0564 
0565     switch (algt->type & algt->mask & CRYPTO_ALG_TYPE_MASK) {
0566     case CRYPTO_ALG_TYPE_DIGEST:
0567         return mcryptd_create_hash(tmpl, tb, &mqueue);
0568     break;
0569     }
0570 
0571     return -EINVAL;
0572 }
0573 
0574 static void mcryptd_free(struct crypto_instance *inst)
0575 {
0576     struct mcryptd_instance_ctx *ctx = crypto_instance_ctx(inst);
0577     struct hashd_instance_ctx *hctx = crypto_instance_ctx(inst);
0578 
0579     switch (inst->alg.cra_flags & CRYPTO_ALG_TYPE_MASK) {
0580     case CRYPTO_ALG_TYPE_AHASH:
0581         crypto_drop_ahash(&hctx->spawn);
0582         kfree(ahash_instance(inst));
0583         return;
0584     default:
0585         crypto_drop_spawn(&ctx->spawn);
0586         kfree(inst);
0587     }
0588 }
0589 
0590 static struct crypto_template mcryptd_tmpl = {
0591     .name = "mcryptd",
0592     .create = mcryptd_create,
0593     .free = mcryptd_free,
0594     .module = THIS_MODULE,
0595 };
0596 
0597 struct mcryptd_ahash *mcryptd_alloc_ahash(const char *alg_name,
0598                     u32 type, u32 mask)
0599 {
0600     char mcryptd_alg_name[CRYPTO_MAX_ALG_NAME];
0601     struct crypto_ahash *tfm;
0602 
0603     if (snprintf(mcryptd_alg_name, CRYPTO_MAX_ALG_NAME,
0604              "mcryptd(%s)", alg_name) >= CRYPTO_MAX_ALG_NAME)
0605         return ERR_PTR(-EINVAL);
0606     tfm = crypto_alloc_ahash(mcryptd_alg_name, type, mask);
0607     if (IS_ERR(tfm))
0608         return ERR_CAST(tfm);
0609     if (tfm->base.__crt_alg->cra_module != THIS_MODULE) {
0610         crypto_free_ahash(tfm);
0611         return ERR_PTR(-EINVAL);
0612     }
0613 
0614     return __mcryptd_ahash_cast(tfm);
0615 }
0616 EXPORT_SYMBOL_GPL(mcryptd_alloc_ahash);
0617 
0618 int ahash_mcryptd_digest(struct ahash_request *desc)
0619 {
0620     return crypto_ahash_init(desc) ?: ahash_mcryptd_finup(desc);
0621 }
0622 
0623 int ahash_mcryptd_update(struct ahash_request *desc)
0624 {
0625     /* alignment is to be done by multi-buffer crypto algorithm if needed */
0626 
0627     return crypto_ahash_update(desc);
0628 }
0629 
0630 int ahash_mcryptd_finup(struct ahash_request *desc)
0631 {
0632     /* alignment is to be done by multi-buffer crypto algorithm if needed */
0633 
0634     return crypto_ahash_finup(desc);
0635 }
0636 
0637 int ahash_mcryptd_final(struct ahash_request *desc)
0638 {
0639     /* alignment is to be done by multi-buffer crypto algorithm if needed */
0640 
0641     return crypto_ahash_final(desc);
0642 }
0643 
0644 struct crypto_ahash *mcryptd_ahash_child(struct mcryptd_ahash *tfm)
0645 {
0646     struct mcryptd_hash_ctx *ctx = crypto_ahash_ctx(&tfm->base);
0647 
0648     return ctx->child;
0649 }
0650 EXPORT_SYMBOL_GPL(mcryptd_ahash_child);
0651 
0652 struct ahash_request *mcryptd_ahash_desc(struct ahash_request *req)
0653 {
0654     struct mcryptd_hash_request_ctx *rctx = ahash_request_ctx(req);
0655     return &rctx->areq;
0656 }
0657 EXPORT_SYMBOL_GPL(mcryptd_ahash_desc);
0658 
0659 void mcryptd_free_ahash(struct mcryptd_ahash *tfm)
0660 {
0661     crypto_free_ahash(&tfm->base);
0662 }
0663 EXPORT_SYMBOL_GPL(mcryptd_free_ahash);
0664 
0665 static int __init mcryptd_init(void)
0666 {
0667     int err, cpu;
0668     struct mcryptd_flush_list *flist;
0669 
0670     mcryptd_flist = alloc_percpu(struct mcryptd_flush_list);
0671     for_each_possible_cpu(cpu) {
0672         flist = per_cpu_ptr(mcryptd_flist, cpu);
0673         INIT_LIST_HEAD(&flist->list);
0674         mutex_init(&flist->lock);
0675     }
0676 
0677     err = mcryptd_init_queue(&mqueue, MCRYPTD_MAX_CPU_QLEN);
0678     if (err) {
0679         free_percpu(mcryptd_flist);
0680         return err;
0681     }
0682 
0683     err = crypto_register_template(&mcryptd_tmpl);
0684     if (err) {
0685         mcryptd_fini_queue(&mqueue);
0686         free_percpu(mcryptd_flist);
0687     }
0688 
0689     return err;
0690 }
0691 
0692 static void __exit mcryptd_exit(void)
0693 {
0694     mcryptd_fini_queue(&mqueue);
0695     crypto_unregister_template(&mcryptd_tmpl);
0696     free_percpu(mcryptd_flist);
0697 }
0698 
0699 subsys_initcall(mcryptd_init);
0700 module_exit(mcryptd_exit);
0701 
0702 MODULE_LICENSE("GPL");
0703 MODULE_DESCRIPTION("Software async multibuffer crypto daemon");
0704 MODULE_ALIAS_CRYPTO("mcryptd");