the-tree/crypto/pcrypt.c

0001 // SPDX-License-Identifier: GPL-2.0-only
0002 /*
0003  * pcrypt - Parallel crypto wrapper.
0004  *
0005  * Copyright (C) 2009 secunet Security Networks AG
0006  * Copyright (C) 2009 Steffen Klassert <steffen.klassert@secunet.com>
0007  */
0008
0009 #include <crypto/algapi.h>
0010 #include <crypto/internal/aead.h>
0011 #include <linux/atomic.h>
0012 #include <linux/err.h>
0013 #include <linux/init.h>
0014 #include <linux/module.h>
0015 #include <linux/slab.h>
0016 #include <linux/kobject.h>
0017 #include <linux/cpu.h>
0018 #include <crypto/pcrypt.h>
0019
0020 static struct padata_instance *pencrypt;
0021 static struct padata_instance *pdecrypt;
0022 static struct kset           *pcrypt_kset;
0023
0024 struct pcrypt_instance_ctx {
0025     struct crypto_aead_spawn spawn;
0026     struct padata_shell *psenc;
0027     struct padata_shell *psdec;
0028     atomic_t tfm_count;
0029 };
0030
0031 struct pcrypt_aead_ctx {
0032     struct crypto_aead *child;
0033     unsigned int cb_cpu;
0034 };
0035
0036 static inline struct pcrypt_instance_ctx *pcrypt_tfm_ictx(
0037     struct crypto_aead *tfm)
0038 {
0039     return aead_instance_ctx(aead_alg_instance(tfm));
0040 }
0041
0042 static int pcrypt_aead_setkey(struct crypto_aead *parent,
0043                   const u8 *key, unsigned int keylen)
0044 {
0045     struct pcrypt_aead_ctx *ctx = crypto_aead_ctx(parent);
0046
0047     return crypto_aead_setkey(ctx->child, key, keylen);
0048 }
0049
0050 static int pcrypt_aead_setauthsize(struct crypto_aead *parent,
0051                    unsigned int authsize)
0052 {
0053     struct pcrypt_aead_ctx *ctx = crypto_aead_ctx(parent);
0054
0055     return crypto_aead_setauthsize(ctx->child, authsize);
0056 }
0057
0058 static void pcrypt_aead_serial(struct padata_priv *padata)
0059 {
0060     struct pcrypt_request *preq = pcrypt_padata_request(padata);
0061     struct aead_request *req = pcrypt_request_ctx(preq);
0062
0063     aead_request_complete(req->base.data, padata->info);
0064 }
0065
0066 static void pcrypt_aead_done(struct crypto_async_request *areq, int err)
0067 {
0068     struct aead_request *req = areq->data;
0069     struct pcrypt_request *preq = aead_request_ctx(req);
0070     struct padata_priv *padata = pcrypt_request_padata(preq);
0071
0072     padata->info = err;
0073
0074     padata_do_serial(padata);
0075 }
0076
0077 static void pcrypt_aead_enc(struct padata_priv *padata)
0078 {
0079     struct pcrypt_request *preq = pcrypt_padata_request(padata);
0080     struct aead_request *req = pcrypt_request_ctx(preq);
0081     int ret;
0082
0083     ret = crypto_aead_encrypt(req);
0084
0085     if (ret == -EINPROGRESS)
0086         return;
0087
0088     padata->info = ret;
0089     padata_do_serial(padata);
0090 }
0091
0092 static int pcrypt_aead_encrypt(struct aead_request *req)
0093 {
0094     int err;
0095     struct pcrypt_request *preq = aead_request_ctx(req);
0096     struct aead_request *creq = pcrypt_request_ctx(preq);
0097     struct padata_priv *padata = pcrypt_request_padata(preq);
0098     struct crypto_aead *aead = crypto_aead_reqtfm(req);
0099     struct pcrypt_aead_ctx *ctx = crypto_aead_ctx(aead);
0100     u32 flags = aead_request_flags(req);
0101     struct pcrypt_instance_ctx *ictx;
0102
0103     ictx = pcrypt_tfm_ictx(aead);
0104
0105     memset(padata, 0, sizeof(struct padata_priv));
0106
0107     padata->parallel = pcrypt_aead_enc;
0108     padata->serial = pcrypt_aead_serial;
0109
0110     aead_request_set_tfm(creq, ctx->child);
0111     aead_request_set_callback(creq, flags & ~CRYPTO_TFM_REQ_MAY_SLEEP,
0112                   pcrypt_aead_done, req);
0113     aead_request_set_crypt(creq, req->src, req->dst,
0114                    req->cryptlen, req->iv);
0115     aead_request_set_ad(creq, req->assoclen);
0116
0117     err = padata_do_parallel(ictx->psenc, padata, &ctx->cb_cpu);
0118     if (!err)
0119         return -EINPROGRESS;
0120
0121     return err;
0122 }
0123
0124 static void pcrypt_aead_dec(struct padata_priv *padata)
0125 {
0126     struct pcrypt_request *preq = pcrypt_padata_request(padata);
0127     struct aead_request *req = pcrypt_request_ctx(preq);
0128     int ret;
0129
0130     ret = crypto_aead_decrypt(req);
0131
0132     if (ret == -EINPROGRESS)
0133         return;
0134
0135     padata->info = ret;
0136     padata_do_serial(padata);
0137 }
0138
0139 static int pcrypt_aead_decrypt(struct aead_request *req)
0140 {
0141     int err;
0142     struct pcrypt_request *preq = aead_request_ctx(req);
0143     struct aead_request *creq = pcrypt_request_ctx(preq);
0144     struct padata_priv *padata = pcrypt_request_padata(preq);
0145     struct crypto_aead *aead = crypto_aead_reqtfm(req);
0146     struct pcrypt_aead_ctx *ctx = crypto_aead_ctx(aead);
0147     u32 flags = aead_request_flags(req);
0148     struct pcrypt_instance_ctx *ictx;
0149
0150     ictx = pcrypt_tfm_ictx(aead);
0151
0152     memset(padata, 0, sizeof(struct padata_priv));
0153
0154     padata->parallel = pcrypt_aead_dec;
0155     padata->serial = pcrypt_aead_serial;
0156
0157     aead_request_set_tfm(creq, ctx->child);
0158     aead_request_set_callback(creq, flags & ~CRYPTO_TFM_REQ_MAY_SLEEP,
0159                   pcrypt_aead_done, req);
0160     aead_request_set_crypt(creq, req->src, req->dst,
0161                    req->cryptlen, req->iv);
0162     aead_request_set_ad(creq, req->assoclen);
0163
0164     err = padata_do_parallel(ictx->psdec, padata, &ctx->cb_cpu);
0165     if (!err)
0166         return -EINPROGRESS;
0167
0168     return err;
0169 }
0170
0171 static int pcrypt_aead_init_tfm(struct crypto_aead *tfm)
0172 {
0173     int cpu, cpu_index;
0174     struct aead_instance *inst = aead_alg_instance(tfm);
0175     struct pcrypt_instance_ctx *ictx = aead_instance_ctx(inst);
0176     struct pcrypt_aead_ctx *ctx = crypto_aead_ctx(tfm);
0177     struct crypto_aead *cipher;
0178
0179     cpu_index = (unsigned int)atomic_inc_return(&ictx->tfm_count) %
0180             cpumask_weight(cpu_online_mask);
0181
0182     ctx->cb_cpu = cpumask_first(cpu_online_mask);
0183     for (cpu = 0; cpu < cpu_index; cpu++)
0184         ctx->cb_cpu = cpumask_next(ctx->cb_cpu, cpu_online_mask);
0185
0186     cipher = crypto_spawn_aead(&ictx->spawn);
0187
0188     if (IS_ERR(cipher))
0189         return PTR_ERR(cipher);
0190
0191     ctx->child = cipher;
0192     crypto_aead_set_reqsize(tfm, sizeof(struct pcrypt_request) +
0193                      sizeof(struct aead_request) +
0194                      crypto_aead_reqsize(cipher));
0195
0196     return 0;
0197 }
0198
0199 static void pcrypt_aead_exit_tfm(struct crypto_aead *tfm)
0200 {
0201     struct pcrypt_aead_ctx *ctx = crypto_aead_ctx(tfm);
0202
0203     crypto_free_aead(ctx->child);
0204 }
0205
0206 static void pcrypt_free(struct aead_instance *inst)
0207 {
0208     struct pcrypt_instance_ctx *ctx = aead_instance_ctx(inst);
0209
0210     crypto_drop_aead(&ctx->spawn);
0211     padata_free_shell(ctx->psdec);
0212     padata_free_shell(ctx->psenc);
0213     kfree(inst);
0214 }
0215
0216 static int pcrypt_init_instance(struct crypto_instance *inst,
0217                 struct crypto_alg *alg)
0218 {
0219     if (snprintf(inst->alg.cra_driver_name, CRYPTO_MAX_ALG_NAME,
0220              "pcrypt(%s)", alg->cra_driver_name) >= CRYPTO_MAX_ALG_NAME)
0221         return -ENAMETOOLONG;
0222
0223     memcpy(inst->alg.cra_name, alg->cra_name, CRYPTO_MAX_ALG_NAME);
0224
0225     inst->alg.cra_priority = alg->cra_priority + 100;
0226     inst->alg.cra_blocksize = alg->cra_blocksize;
0227     inst->alg.cra_alignmask = alg->cra_alignmask;
0228
0229     return 0;
0230 }
0231
0232 static int pcrypt_create_aead(struct crypto_template *tmpl, struct rtattr **tb,
0233                   struct crypto_attr_type *algt)
0234 {
0235     struct pcrypt_instance_ctx *ctx;
0236     struct aead_instance *inst;
0237     struct aead_alg *alg;
0238     u32 mask = crypto_algt_inherited_mask(algt);
0239     int err;
0240
0241     inst = kzalloc(sizeof(*inst) + sizeof(*ctx), GFP_KERNEL);
0242     if (!inst)
0243         return -ENOMEM;
0244
0245     err = -ENOMEM;
0246
0247     ctx = aead_instance_ctx(inst);
0248     ctx->psenc = padata_alloc_shell(pencrypt);
0249     if (!ctx->psenc)
0250         goto err_free_inst;
0251
0252     ctx->psdec = padata_alloc_shell(pdecrypt);
0253     if (!ctx->psdec)
0254         goto err_free_inst;
0255
0256     err = crypto_grab_aead(&ctx->spawn, aead_crypto_instance(inst),
0257                    crypto_attr_alg_name(tb[1]), 0, mask);
0258     if (err)
0259         goto err_free_inst;
0260
0261     alg = crypto_spawn_aead_alg(&ctx->spawn);
0262     err = pcrypt_init_instance(aead_crypto_instance(inst), &alg->base);
0263     if (err)
0264         goto err_free_inst;
0265
0266     inst->alg.base.cra_flags |= CRYPTO_ALG_ASYNC;
0267
0268     inst->alg.ivsize = crypto_aead_alg_ivsize(alg);
0269     inst->alg.maxauthsize = crypto_aead_alg_maxauthsize(alg);
0270
0271     inst->alg.base.cra_ctxsize = sizeof(struct pcrypt_aead_ctx);
0272
0273     inst->alg.init = pcrypt_aead_init_tfm;
0274     inst->alg.exit = pcrypt_aead_exit_tfm;
0275
0276     inst->alg.setkey = pcrypt_aead_setkey;
0277     inst->alg.setauthsize = pcrypt_aead_setauthsize;
0278     inst->alg.encrypt = pcrypt_aead_encrypt;
0279     inst->alg.decrypt = pcrypt_aead_decrypt;
0280
0281     inst->free = pcrypt_free;
0282
0283     err = aead_register_instance(tmpl, inst);
0284     if (err) {
0285 err_free_inst:
0286         pcrypt_free(inst);
0287     }
0288     return err;
0289 }
0290
0291 static int pcrypt_create(struct crypto_template *tmpl, struct rtattr **tb)
0292 {
0293     struct crypto_attr_type *algt;
0294
0295     algt = crypto_get_attr_type(tb);
0296     if (IS_ERR(algt))
0297         return PTR_ERR(algt);
0298
0299     switch (algt->type & algt->mask & CRYPTO_ALG_TYPE_MASK) {
0300     case CRYPTO_ALG_TYPE_AEAD:
0301         return pcrypt_create_aead(tmpl, tb, algt);
0302     }
0303
0304     return -EINVAL;
0305 }
0306
0307 static int pcrypt_sysfs_add(struct padata_instance *pinst, const char *name)
0308 {
0309     int ret;
0310
0311     pinst->kobj.kset = pcrypt_kset;
0312     ret = kobject_add(&pinst->kobj, NULL, "%s", name);
0313     if (!ret)
0314         kobject_uevent(&pinst->kobj, KOBJ_ADD);
0315
0316     return ret;
0317 }
0318
0319 static int pcrypt_init_padata(struct padata_instance **pinst, const char *name)
0320 {
0321     int ret = -ENOMEM;
0322
0323     *pinst = padata_alloc(name);
0324     if (!*pinst)
0325         return ret;
0326
0327     ret = pcrypt_sysfs_add(*pinst, name);
0328     if (ret)
0329         padata_free(*pinst);
0330
0331     return ret;
0332 }
0333
0334 static struct crypto_template pcrypt_tmpl = {
0335     .name = "pcrypt",
0336     .create = pcrypt_create,
0337     .module = THIS_MODULE,
0338 };
0339
0340 static int __init pcrypt_init(void)
0341 {
0342     int err = -ENOMEM;
0343
0344     pcrypt_kset = kset_create_and_add("pcrypt", NULL, kernel_kobj);
0345     if (!pcrypt_kset)
0346         goto err;
0347
0348     err = pcrypt_init_padata(&pencrypt, "pencrypt");
0349     if (err)
0350         goto err_unreg_kset;
0351
0352     err = pcrypt_init_padata(&pdecrypt, "pdecrypt");
0353     if (err)
0354         goto err_deinit_pencrypt;
0355
0356     return crypto_register_template(&pcrypt_tmpl);
0357
0358 err_deinit_pencrypt:
0359     padata_free(pencrypt);
0360 err_unreg_kset:
0361     kset_unregister(pcrypt_kset);
0362 err:
0363     return err;
0364 }
0365
0366 static void __exit pcrypt_exit(void)
0367 {
0368     crypto_unregister_template(&pcrypt_tmpl);
0369
0370     padata_free(pencrypt);
0371     padata_free(pdecrypt);
0372
0373     kset_unregister(pcrypt_kset);
0374 }
0375
0376 subsys_initcall(pcrypt_init);
0377 module_exit(pcrypt_exit);
0378
0379 MODULE_LICENSE("GPL");
0380 MODULE_AUTHOR("Steffen Klassert <steffen.klassert@secunet.com>");
0381 MODULE_DESCRIPTION("Parallel crypto wrapper");
0382 MODULE_ALIAS_CRYPTO("pcrypt");