the-tree/crypto/algif_aead.c

0001 // SPDX-License-Identifier: GPL-2.0-or-later
0002 /*
0003  * algif_aead: User-space interface for AEAD algorithms
0004  *
0005  * Copyright (C) 2014, Stephan Mueller <smueller@chronox.de>
0006  *
0007  * This file provides the user-space API for AEAD ciphers.
0008  *
0009  * The following concept of the memory management is used:
0010  *
0011  * The kernel maintains two SGLs, the TX SGL and the RX SGL. The TX SGL is
0012  * filled by user space with the data submitted via sendpage/sendmsg. Filling
0013  * up the TX SGL does not cause a crypto operation -- the data will only be
0014  * tracked by the kernel. Upon receipt of one recvmsg call, the caller must
0015  * provide a buffer which is tracked with the RX SGL.
0016  *
0017  * During the processing of the recvmsg operation, the cipher request is
0018  * allocated and prepared. As part of the recvmsg operation, the processed
0019  * TX buffers are extracted from the TX SGL into a separate SGL.
0020  *
0021  * After the completion of the crypto operation, the RX SGL and the cipher
0022  * request is released. The extracted TX SGL parts are released together with
0023  * the RX SGL release.
0024  */
0025
0026 #include <crypto/internal/aead.h>
0027 #include <crypto/scatterwalk.h>
0028 #include <crypto/if_alg.h>
0029 #include <crypto/skcipher.h>
0030 #include <crypto/null.h>
0031 #include <linux/init.h>
0032 #include <linux/list.h>
0033 #include <linux/kernel.h>
0034 #include <linux/mm.h>
0035 #include <linux/module.h>
0036 #include <linux/net.h>
0037 #include <net/sock.h>
0038
0039 struct aead_tfm {
0040     struct crypto_aead *aead;
0041     struct crypto_sync_skcipher *null_tfm;
0042 };
0043
0044 static inline bool aead_sufficient_data(struct sock *sk)
0045 {
0046     struct alg_sock *ask = alg_sk(sk);
0047     struct sock *psk = ask->parent;
0048     struct alg_sock *pask = alg_sk(psk);
0049     struct af_alg_ctx *ctx = ask->private;
0050     struct aead_tfm *aeadc = pask->private;
0051     struct crypto_aead *tfm = aeadc->aead;
0052     unsigned int as = crypto_aead_authsize(tfm);
0053
0054     /*
0055      * The minimum amount of memory needed for an AEAD cipher is
0056      * the AAD and in case of decryption the tag.
0057      */
0058     return ctx->used >= ctx->aead_assoclen + (ctx->enc ? 0 : as);
0059 }
0060
0061 static int aead_sendmsg(struct socket *sock, struct msghdr *msg, size_t size)
0062 {
0063     struct sock *sk = sock->sk;
0064     struct alg_sock *ask = alg_sk(sk);
0065     struct sock *psk = ask->parent;
0066     struct alg_sock *pask = alg_sk(psk);
0067     struct aead_tfm *aeadc = pask->private;
0068     struct crypto_aead *tfm = aeadc->aead;
0069     unsigned int ivsize = crypto_aead_ivsize(tfm);
0070
0071     return af_alg_sendmsg(sock, msg, size, ivsize);
0072 }
0073
0074 static int crypto_aead_copy_sgl(struct crypto_sync_skcipher *null_tfm,
0075                 struct scatterlist *src,
0076                 struct scatterlist *dst, unsigned int len)
0077 {
0078     SYNC_SKCIPHER_REQUEST_ON_STACK(skreq, null_tfm);
0079
0080     skcipher_request_set_sync_tfm(skreq, null_tfm);
0081     skcipher_request_set_callback(skreq, CRYPTO_TFM_REQ_MAY_SLEEP,
0082                       NULL, NULL);
0083     skcipher_request_set_crypt(skreq, src, dst, len, NULL);
0084
0085     return crypto_skcipher_encrypt(skreq);
0086 }
0087
0088 static int _aead_recvmsg(struct socket *sock, struct msghdr *msg,
0089              size_t ignored, int flags)
0090 {
0091     struct sock *sk = sock->sk;
0092     struct alg_sock *ask = alg_sk(sk);
0093     struct sock *psk = ask->parent;
0094     struct alg_sock *pask = alg_sk(psk);
0095     struct af_alg_ctx *ctx = ask->private;
0096     struct aead_tfm *aeadc = pask->private;
0097     struct crypto_aead *tfm = aeadc->aead;
0098     struct crypto_sync_skcipher *null_tfm = aeadc->null_tfm;
0099     unsigned int i, as = crypto_aead_authsize(tfm);
0100     struct af_alg_async_req *areq;
0101     struct af_alg_tsgl *tsgl, *tmp;
0102     struct scatterlist *rsgl_src, *tsgl_src = NULL;
0103     int err = 0;
0104     size_t used = 0;        /* [in]  TX bufs to be en/decrypted */
0105     size_t outlen = 0;      /* [out] RX bufs produced by kernel */
0106     size_t usedpages = 0;       /* [in]  RX bufs to be used from user */
0107     size_t processed = 0;       /* [in]  TX bufs to be consumed */
0108
0109     if (!ctx->init || ctx->more) {
0110         err = af_alg_wait_for_data(sk, flags, 0);
0111         if (err)
0112             return err;
0113     }
0114
0115     /*
0116      * Data length provided by caller via sendmsg/sendpage that has not
0117      * yet been processed.
0118      */
0119     used = ctx->used;
0120
0121     /*
0122      * Make sure sufficient data is present -- note, the same check is
0123      * also present in sendmsg/sendpage. The checks in sendpage/sendmsg
0124      * shall provide an information to the data sender that something is
0125      * wrong, but they are irrelevant to maintain the kernel integrity.
0126      * We need this check here too in case user space decides to not honor
0127      * the error message in sendmsg/sendpage and still call recvmsg. This
0128      * check here protects the kernel integrity.
0129      */
0130     if (!aead_sufficient_data(sk))
0131         return -EINVAL;
0132
0133     /*
0134      * Calculate the minimum output buffer size holding the result of the
0135      * cipher operation. When encrypting data, the receiving buffer is
0136      * larger by the tag length compared to the input buffer as the
0137      * encryption operation generates the tag. For decryption, the input
0138      * buffer provides the tag which is consumed resulting in only the
0139      * plaintext without a buffer for the tag returned to the caller.
0140      */
0141     if (ctx->enc)
0142         outlen = used + as;
0143     else
0144         outlen = used - as;
0145
0146     /*
0147      * The cipher operation input data is reduced by the associated data
0148      * length as this data is processed separately later on.
0149      */
0150     used -= ctx->aead_assoclen;
0151
0152     /* Allocate cipher request for current operation. */
0153     areq = af_alg_alloc_areq(sk, sizeof(struct af_alg_async_req) +
0154                      crypto_aead_reqsize(tfm));
0155     if (IS_ERR(areq))
0156         return PTR_ERR(areq);
0157
0158     /* convert iovecs of output buffers into RX SGL */
0159     err = af_alg_get_rsgl(sk, msg, flags, areq, outlen, &usedpages);
0160     if (err)
0161         goto free;
0162
0163     /*
0164      * Ensure output buffer is sufficiently large. If the caller provides
0165      * less buffer space, only use the relative required input size. This
0166      * allows AIO operation where the caller sent all data to be processed
0167      * and the AIO operation performs the operation on the different chunks
0168      * of the input data.
0169      */
0170     if (usedpages < outlen) {
0171         size_t less = outlen - usedpages;
0172
0173         if (used < less) {
0174             err = -EINVAL;
0175             goto free;
0176         }
0177         used -= less;
0178         outlen -= less;
0179     }
0180
0181     processed = used + ctx->aead_assoclen;
0182     list_for_each_entry_safe(tsgl, tmp, &ctx->tsgl_list, list) {
0183         for (i = 0; i < tsgl->cur; i++) {
0184             struct scatterlist *process_sg = tsgl->sg + i;
0185
0186             if (!(process_sg->length) || !sg_page(process_sg))
0187                 continue;
0188             tsgl_src = process_sg;
0189             break;
0190         }
0191         if (tsgl_src)
0192             break;
0193     }
0194     if (processed && !tsgl_src) {
0195         err = -EFAULT;
0196         goto free;
0197     }
0198
0199     /*
0200      * Copy of AAD from source to destination
0201      *
0202      * The AAD is copied to the destination buffer without change. Even
0203      * when user space uses an in-place cipher operation, the kernel
0204      * will copy the data as it does not see whether such in-place operation
0205      * is initiated.
0206      *
0207      * To ensure efficiency, the following implementation ensure that the
0208      * ciphers are invoked to perform a crypto operation in-place. This
0209      * is achieved by memory management specified as follows.
0210      */
0211
0212     /* Use the RX SGL as source (and destination) for crypto op. */
0213     rsgl_src = areq->first_rsgl.sgl.sg;
0214
0215     if (ctx->enc) {
0216         /*
0217          * Encryption operation - The in-place cipher operation is
0218          * achieved by the following operation:
0219          *
0220          * TX SGL: AAD || PT
0221          *      |      |
0222          *      | copy |
0223          *      v      v
0224          * RX SGL: AAD || PT || Tag
0225          */
0226         err = crypto_aead_copy_sgl(null_tfm, tsgl_src,
0227                        areq->first_rsgl.sgl.sg, processed);
0228         if (err)
0229             goto free;
0230         af_alg_pull_tsgl(sk, processed, NULL, 0);
0231     } else {
0232         /*
0233          * Decryption operation - To achieve an in-place cipher
0234          * operation, the following  SGL structure is used:
0235          *
0236          * TX SGL: AAD || CT || Tag
0237          *      |      |     ^
0238          *      | copy |     | Create SGL link.
0239          *      v      v     |
0240          * RX SGL: AAD || CT ----+
0241          */
0242
0243          /* Copy AAD || CT to RX SGL buffer for in-place operation. */
0244         err = crypto_aead_copy_sgl(null_tfm, tsgl_src,
0245                        areq->first_rsgl.sgl.sg, outlen);
0246         if (err)
0247             goto free;
0248
0249         /* Create TX SGL for tag and chain it to RX SGL. */
0250         areq->tsgl_entries = af_alg_count_tsgl(sk, processed,
0251                                processed - as);
0252         if (!areq->tsgl_entries)
0253             areq->tsgl_entries = 1;
0254         areq->tsgl = sock_kmalloc(sk, array_size(sizeof(*areq->tsgl),
0255                              areq->tsgl_entries),
0256                       GFP_KERNEL);
0257         if (!areq->tsgl) {
0258             err = -ENOMEM;
0259             goto free;
0260         }
0261         sg_init_table(areq->tsgl, areq->tsgl_entries);
0262
0263         /* Release TX SGL, except for tag data and reassign tag data. */
0264         af_alg_pull_tsgl(sk, processed, areq->tsgl, processed - as);
0265
0266         /* chain the areq TX SGL holding the tag with RX SGL */
0267         if (usedpages) {
0268             /* RX SGL present */
0269             struct af_alg_sgl *sgl_prev = &areq->last_rsgl->sgl;
0270
0271             sg_unmark_end(sgl_prev->sg + sgl_prev->npages - 1);
0272             sg_chain(sgl_prev->sg, sgl_prev->npages + 1,
0273                  areq->tsgl);
0274         } else
0275             /* no RX SGL present (e.g. authentication only) */
0276             rsgl_src = areq->tsgl;
0277     }
0278
0279     /* Initialize the crypto operation */
0280     aead_request_set_crypt(&areq->cra_u.aead_req, rsgl_src,
0281                    areq->first_rsgl.sgl.sg, used, ctx->iv);
0282     aead_request_set_ad(&areq->cra_u.aead_req, ctx->aead_assoclen);
0283     aead_request_set_tfm(&areq->cra_u.aead_req, tfm);
0284
0285     if (msg->msg_iocb && !is_sync_kiocb(msg->msg_iocb)) {
0286         /* AIO operation */
0287         sock_hold(sk);
0288         areq->iocb = msg->msg_iocb;
0289
0290         /* Remember output size that will be generated. */
0291         areq->outlen = outlen;
0292
0293         aead_request_set_callback(&areq->cra_u.aead_req,
0294                       CRYPTO_TFM_REQ_MAY_SLEEP,
0295                       af_alg_async_cb, areq);
0296         err = ctx->enc ? crypto_aead_encrypt(&areq->cra_u.aead_req) :
0297                  crypto_aead_decrypt(&areq->cra_u.aead_req);
0298
0299         /* AIO operation in progress */
0300         if (err == -EINPROGRESS)
0301             return -EIOCBQUEUED;
0302
0303         sock_put(sk);
0304     } else {
0305         /* Synchronous operation */
0306         aead_request_set_callback(&areq->cra_u.aead_req,
0307                       CRYPTO_TFM_REQ_MAY_SLEEP |
0308                       CRYPTO_TFM_REQ_MAY_BACKLOG,
0309                       crypto_req_done, &ctx->wait);
0310         err = crypto_wait_req(ctx->enc ?
0311                 crypto_aead_encrypt(&areq->cra_u.aead_req) :
0312                 crypto_aead_decrypt(&areq->cra_u.aead_req),
0313                 &ctx->wait);
0314     }
0315
0316
0317 free:
0318     af_alg_free_resources(areq);
0319
0320     return err ? err : outlen;
0321 }
0322
0323 static int aead_recvmsg(struct socket *sock, struct msghdr *msg,
0324             size_t ignored, int flags)
0325 {
0326     struct sock *sk = sock->sk;
0327     int ret = 0;
0328
0329     lock_sock(sk);
0330     while (msg_data_left(msg)) {
0331         int err = _aead_recvmsg(sock, msg, ignored, flags);
0332
0333         /*
0334          * This error covers -EIOCBQUEUED which implies that we can
0335          * only handle one AIO request. If the caller wants to have
0336          * multiple AIO requests in parallel, he must make multiple
0337          * separate AIO calls.
0338          *
0339          * Also return the error if no data has been processed so far.
0340          */
0341         if (err <= 0) {
0342             if (err == -EIOCBQUEUED || err == -EBADMSG || !ret)
0343                 ret = err;
0344             goto out;
0345         }
0346
0347         ret += err;
0348     }
0349
0350 out:
0351     af_alg_wmem_wakeup(sk);
0352     release_sock(sk);
0353     return ret;
0354 }
0355
0356 static struct proto_ops algif_aead_ops = {
0357     .family     =   PF_ALG,
0358
0359     .connect    =   sock_no_connect,
0360     .socketpair =   sock_no_socketpair,
0361     .getname    =   sock_no_getname,
0362     .ioctl      =   sock_no_ioctl,
0363     .listen     =   sock_no_listen,
0364     .shutdown   =   sock_no_shutdown,
0365     .mmap       =   sock_no_mmap,
0366     .bind       =   sock_no_bind,
0367     .accept     =   sock_no_accept,
0368
0369     .release    =   af_alg_release,
0370     .sendmsg    =   aead_sendmsg,
0371     .sendpage   =   af_alg_sendpage,
0372     .recvmsg    =   aead_recvmsg,
0373     .poll       =   af_alg_poll,
0374 };
0375
0376 static int aead_check_key(struct socket *sock)
0377 {
0378     int err = 0;
0379     struct sock *psk;
0380     struct alg_sock *pask;
0381     struct aead_tfm *tfm;
0382     struct sock *sk = sock->sk;
0383     struct alg_sock *ask = alg_sk(sk);
0384
0385     lock_sock(sk);
0386     if (!atomic_read(&ask->nokey_refcnt))
0387         goto unlock_child;
0388
0389     psk = ask->parent;
0390     pask = alg_sk(ask->parent);
0391     tfm = pask->private;
0392
0393     err = -ENOKEY;
0394     lock_sock_nested(psk, SINGLE_DEPTH_NESTING);
0395     if (crypto_aead_get_flags(tfm->aead) & CRYPTO_TFM_NEED_KEY)
0396         goto unlock;
0397
0398     atomic_dec(&pask->nokey_refcnt);
0399     atomic_set(&ask->nokey_refcnt, 0);
0400
0401     err = 0;
0402
0403 unlock:
0404     release_sock(psk);
0405 unlock_child:
0406     release_sock(sk);
0407
0408     return err;
0409 }
0410
0411 static int aead_sendmsg_nokey(struct socket *sock, struct msghdr *msg,
0412                   size_t size)
0413 {
0414     int err;
0415
0416     err = aead_check_key(sock);
0417     if (err)
0418         return err;
0419
0420     return aead_sendmsg(sock, msg, size);
0421 }
0422
0423 static ssize_t aead_sendpage_nokey(struct socket *sock, struct page *page,
0424                        int offset, size_t size, int flags)
0425 {
0426     int err;
0427
0428     err = aead_check_key(sock);
0429     if (err)
0430         return err;
0431
0432     return af_alg_sendpage(sock, page, offset, size, flags);
0433 }
0434
0435 static int aead_recvmsg_nokey(struct socket *sock, struct msghdr *msg,
0436                   size_t ignored, int flags)
0437 {
0438     int err;
0439
0440     err = aead_check_key(sock);
0441     if (err)
0442         return err;
0443
0444     return aead_recvmsg(sock, msg, ignored, flags);
0445 }
0446
0447 static struct proto_ops algif_aead_ops_nokey = {
0448     .family     =   PF_ALG,
0449
0450     .connect    =   sock_no_connect,
0451     .socketpair =   sock_no_socketpair,
0452     .getname    =   sock_no_getname,
0453     .ioctl      =   sock_no_ioctl,
0454     .listen     =   sock_no_listen,
0455     .shutdown   =   sock_no_shutdown,
0456     .mmap       =   sock_no_mmap,
0457     .bind       =   sock_no_bind,
0458     .accept     =   sock_no_accept,
0459
0460     .release    =   af_alg_release,
0461     .sendmsg    =   aead_sendmsg_nokey,
0462     .sendpage   =   aead_sendpage_nokey,
0463     .recvmsg    =   aead_recvmsg_nokey,
0464     .poll       =   af_alg_poll,
0465 };
0466
0467 static void *aead_bind(const char *name, u32 type, u32 mask)
0468 {
0469     struct aead_tfm *tfm;
0470     struct crypto_aead *aead;
0471     struct crypto_sync_skcipher *null_tfm;
0472
0473     tfm = kzalloc(sizeof(*tfm), GFP_KERNEL);
0474     if (!tfm)
0475         return ERR_PTR(-ENOMEM);
0476
0477     aead = crypto_alloc_aead(name, type, mask);
0478     if (IS_ERR(aead)) {
0479         kfree(tfm);
0480         return ERR_CAST(aead);
0481     }
0482
0483     null_tfm = crypto_get_default_null_skcipher();
0484     if (IS_ERR(null_tfm)) {
0485         crypto_free_aead(aead);
0486         kfree(tfm);
0487         return ERR_CAST(null_tfm);
0488     }
0489
0490     tfm->aead = aead;
0491     tfm->null_tfm = null_tfm;
0492
0493     return tfm;
0494 }
0495
0496 static void aead_release(void *private)
0497 {
0498     struct aead_tfm *tfm = private;
0499
0500     crypto_free_aead(tfm->aead);
0501     crypto_put_default_null_skcipher();
0502     kfree(tfm);
0503 }
0504
0505 static int aead_setauthsize(void *private, unsigned int authsize)
0506 {
0507     struct aead_tfm *tfm = private;
0508
0509     return crypto_aead_setauthsize(tfm->aead, authsize);
0510 }
0511
0512 static int aead_setkey(void *private, const u8 *key, unsigned int keylen)
0513 {
0514     struct aead_tfm *tfm = private;
0515
0516     return crypto_aead_setkey(tfm->aead, key, keylen);
0517 }
0518
0519 static void aead_sock_destruct(struct sock *sk)
0520 {
0521     struct alg_sock *ask = alg_sk(sk);
0522     struct af_alg_ctx *ctx = ask->private;
0523     struct sock *psk = ask->parent;
0524     struct alg_sock *pask = alg_sk(psk);
0525     struct aead_tfm *aeadc = pask->private;
0526     struct crypto_aead *tfm = aeadc->aead;
0527     unsigned int ivlen = crypto_aead_ivsize(tfm);
0528
0529     af_alg_pull_tsgl(sk, ctx->used, NULL, 0);
0530     sock_kzfree_s(sk, ctx->iv, ivlen);
0531     sock_kfree_s(sk, ctx, ctx->len);
0532     af_alg_release_parent(sk);
0533 }
0534
0535 static int aead_accept_parent_nokey(void *private, struct sock *sk)
0536 {
0537     struct af_alg_ctx *ctx;
0538     struct alg_sock *ask = alg_sk(sk);
0539     struct aead_tfm *tfm = private;
0540     struct crypto_aead *aead = tfm->aead;
0541     unsigned int len = sizeof(*ctx);
0542     unsigned int ivlen = crypto_aead_ivsize(aead);
0543
0544     ctx = sock_kmalloc(sk, len, GFP_KERNEL);
0545     if (!ctx)
0546         return -ENOMEM;
0547     memset(ctx, 0, len);
0548
0549     ctx->iv = sock_kmalloc(sk, ivlen, GFP_KERNEL);
0550     if (!ctx->iv) {
0551         sock_kfree_s(sk, ctx, len);
0552         return -ENOMEM;
0553     }
0554     memset(ctx->iv, 0, ivlen);
0555
0556     INIT_LIST_HEAD(&ctx->tsgl_list);
0557     ctx->len = len;
0558     crypto_init_wait(&ctx->wait);
0559
0560     ask->private = ctx;
0561
0562     sk->sk_destruct = aead_sock_destruct;
0563
0564     return 0;
0565 }
0566
0567 static int aead_accept_parent(void *private, struct sock *sk)
0568 {
0569     struct aead_tfm *tfm = private;
0570
0571     if (crypto_aead_get_flags(tfm->aead) & CRYPTO_TFM_NEED_KEY)
0572         return -ENOKEY;
0573
0574     return aead_accept_parent_nokey(private, sk);
0575 }
0576
0577 static const struct af_alg_type algif_type_aead = {
0578     .bind       =   aead_bind,
0579     .release    =   aead_release,
0580     .setkey     =   aead_setkey,
0581     .setauthsize    =   aead_setauthsize,
0582     .accept     =   aead_accept_parent,
0583     .accept_nokey   =   aead_accept_parent_nokey,
0584     .ops        =   &algif_aead_ops,
0585     .ops_nokey  =   &algif_aead_ops_nokey,
0586     .name       =   "aead",
0587     .owner      =   THIS_MODULE
0588 };
0589
0590 static int __init algif_aead_init(void)
0591 {
0592     return af_alg_register_type(&algif_type_aead);
0593 }
0594
0595 static void __exit algif_aead_exit(void)
0596 {
0597     int err = af_alg_unregister_type(&algif_type_aead);
0598     BUG_ON(err);
0599 }
0600
0601 module_init(algif_aead_init);
0602 module_exit(algif_aead_exit);
0603 MODULE_LICENSE("GPL");
0604 MODULE_AUTHOR("Stephan Mueller <smueller@chronox.de>");
0605 MODULE_DESCRIPTION("AEAD kernel crypto API user space interface");