the-tree/io_uring/tctx.c

0001 // SPDX-License-Identifier: GPL-2.0
0002 #include <linux/kernel.h>
0003 #include <linux/errno.h>
0004 #include <linux/file.h>
0005 #include <linux/mm.h>
0006 #include <linux/slab.h>
0007 #include <linux/nospec.h>
0008 #include <linux/io_uring.h>
0009
0010 #include <uapi/linux/io_uring.h>
0011
0012 #include "io_uring.h"
0013 #include "tctx.h"
0014
0015 static struct io_wq *io_init_wq_offload(struct io_ring_ctx *ctx,
0016                     struct task_struct *task)
0017 {
0018     struct io_wq_hash *hash;
0019     struct io_wq_data data;
0020     unsigned int concurrency;
0021
0022     mutex_lock(&ctx->uring_lock);
0023     hash = ctx->hash_map;
0024     if (!hash) {
0025         hash = kzalloc(sizeof(*hash), GFP_KERNEL);
0026         if (!hash) {
0027             mutex_unlock(&ctx->uring_lock);
0028             return ERR_PTR(-ENOMEM);
0029         }
0030         refcount_set(&hash->refs, 1);
0031         init_waitqueue_head(&hash->wait);
0032         ctx->hash_map = hash;
0033     }
0034     mutex_unlock(&ctx->uring_lock);
0035
0036     data.hash = hash;
0037     data.task = task;
0038     data.free_work = io_wq_free_work;
0039     data.do_work = io_wq_submit_work;
0040
0041     /* Do QD, or 4 * CPUS, whatever is smallest */
0042     concurrency = min(ctx->sq_entries, 4 * num_online_cpus());
0043
0044     return io_wq_create(concurrency, &data);
0045 }
0046
0047 void __io_uring_free(struct task_struct *tsk)
0048 {
0049     struct io_uring_task *tctx = tsk->io_uring;
0050
0051     WARN_ON_ONCE(!xa_empty(&tctx->xa));
0052     WARN_ON_ONCE(tctx->io_wq);
0053     WARN_ON_ONCE(tctx->cached_refs);
0054
0055     percpu_counter_destroy(&tctx->inflight);
0056     kfree(tctx);
0057     tsk->io_uring = NULL;
0058 }
0059
0060 __cold int io_uring_alloc_task_context(struct task_struct *task,
0061                        struct io_ring_ctx *ctx)
0062 {
0063     struct io_uring_task *tctx;
0064     int ret;
0065
0066     tctx = kzalloc(sizeof(*tctx), GFP_KERNEL);
0067     if (unlikely(!tctx))
0068         return -ENOMEM;
0069
0070     ret = percpu_counter_init(&tctx->inflight, 0, GFP_KERNEL);
0071     if (unlikely(ret)) {
0072         kfree(tctx);
0073         return ret;
0074     }
0075
0076     tctx->io_wq = io_init_wq_offload(ctx, task);
0077     if (IS_ERR(tctx->io_wq)) {
0078         ret = PTR_ERR(tctx->io_wq);
0079         percpu_counter_destroy(&tctx->inflight);
0080         kfree(tctx);
0081         return ret;
0082     }
0083
0084     xa_init(&tctx->xa);
0085     init_waitqueue_head(&tctx->wait);
0086     atomic_set(&tctx->in_idle, 0);
0087     atomic_set(&tctx->inflight_tracked, 0);
0088     task->io_uring = tctx;
0089     init_llist_head(&tctx->task_list);
0090     init_task_work(&tctx->task_work, tctx_task_work);
0091     return 0;
0092 }
0093
0094 static int io_register_submitter(struct io_ring_ctx *ctx)
0095 {
0096     int ret = 0;
0097
0098     mutex_lock(&ctx->uring_lock);
0099     if (!ctx->submitter_task)
0100         ctx->submitter_task = get_task_struct(current);
0101     else if (ctx->submitter_task != current)
0102         ret = -EEXIST;
0103     mutex_unlock(&ctx->uring_lock);
0104
0105     return ret;
0106 }
0107
0108 int __io_uring_add_tctx_node(struct io_ring_ctx *ctx, bool submitter)
0109 {
0110     struct io_uring_task *tctx = current->io_uring;
0111     struct io_tctx_node *node;
0112     int ret;
0113
0114     if ((ctx->flags & IORING_SETUP_SINGLE_ISSUER) && submitter) {
0115         ret = io_register_submitter(ctx);
0116         if (ret)
0117             return ret;
0118     }
0119
0120     if (unlikely(!tctx)) {
0121         ret = io_uring_alloc_task_context(current, ctx);
0122         if (unlikely(ret))
0123             return ret;
0124
0125         tctx = current->io_uring;
0126         if (ctx->iowq_limits_set) {
0127             unsigned int limits[2] = { ctx->iowq_limits[0],
0128                            ctx->iowq_limits[1], };
0129
0130             ret = io_wq_max_workers(tctx->io_wq, limits);
0131             if (ret)
0132                 return ret;
0133         }
0134     }
0135     if (!xa_load(&tctx->xa, (unsigned long)ctx)) {
0136         node = kmalloc(sizeof(*node), GFP_KERNEL);
0137         if (!node)
0138             return -ENOMEM;
0139         node->ctx = ctx;
0140         node->task = current;
0141
0142         ret = xa_err(xa_store(&tctx->xa, (unsigned long)ctx,
0143                     node, GFP_KERNEL));
0144         if (ret) {
0145             kfree(node);
0146             return ret;
0147         }
0148
0149         mutex_lock(&ctx->uring_lock);
0150         list_add(&node->ctx_node, &ctx->tctx_list);
0151         mutex_unlock(&ctx->uring_lock);
0152     }
0153     if (submitter)
0154         tctx->last = ctx;
0155     return 0;
0156 }
0157
0158 /*
0159  * Remove this io_uring_file -> task mapping.
0160  */
0161 __cold void io_uring_del_tctx_node(unsigned long index)
0162 {
0163     struct io_uring_task *tctx = current->io_uring;
0164     struct io_tctx_node *node;
0165
0166     if (!tctx)
0167         return;
0168     node = xa_erase(&tctx->xa, index);
0169     if (!node)
0170         return;
0171
0172     WARN_ON_ONCE(current != node->task);
0173     WARN_ON_ONCE(list_empty(&node->ctx_node));
0174
0175     mutex_lock(&node->ctx->uring_lock);
0176     list_del(&node->ctx_node);
0177     mutex_unlock(&node->ctx->uring_lock);
0178
0179     if (tctx->last == node->ctx)
0180         tctx->last = NULL;
0181     kfree(node);
0182 }
0183
0184 __cold void io_uring_clean_tctx(struct io_uring_task *tctx)
0185 {
0186     struct io_wq *wq = tctx->io_wq;
0187     struct io_tctx_node *node;
0188     unsigned long index;
0189
0190     xa_for_each(&tctx->xa, index, node) {
0191         io_uring_del_tctx_node(index);
0192         cond_resched();
0193     }
0194     if (wq) {
0195         /*
0196          * Must be after io_uring_del_tctx_node() (removes nodes under
0197          * uring_lock) to avoid race with io_uring_try_cancel_iowq().
0198          */
0199         io_wq_put_and_exit(wq);
0200         tctx->io_wq = NULL;
0201     }
0202 }
0203
0204 void io_uring_unreg_ringfd(void)
0205 {
0206     struct io_uring_task *tctx = current->io_uring;
0207     int i;
0208
0209     for (i = 0; i < IO_RINGFD_REG_MAX; i++) {
0210         if (tctx->registered_rings[i]) {
0211             fput(tctx->registered_rings[i]);
0212             tctx->registered_rings[i] = NULL;
0213         }
0214     }
0215 }
0216
0217 static int io_ring_add_registered_fd(struct io_uring_task *tctx, int fd,
0218                      int start, int end)
0219 {
0220     struct file *file;
0221     int offset;
0222
0223     for (offset = start; offset < end; offset++) {
0224         offset = array_index_nospec(offset, IO_RINGFD_REG_MAX);
0225         if (tctx->registered_rings[offset])
0226             continue;
0227
0228         file = fget(fd);
0229         if (!file) {
0230             return -EBADF;
0231         } else if (!io_is_uring_fops(file)) {
0232             fput(file);
0233             return -EOPNOTSUPP;
0234         }
0235         tctx->registered_rings[offset] = file;
0236         return offset;
0237     }
0238
0239     return -EBUSY;
0240 }
0241
0242 /*
0243  * Register a ring fd to avoid fdget/fdput for each io_uring_enter()
0244  * invocation. User passes in an array of struct io_uring_rsrc_update
0245  * with ->data set to the ring_fd, and ->offset given for the desired
0246  * index. If no index is desired, application may set ->offset == -1U
0247  * and we'll find an available index. Returns number of entries
0248  * successfully processed, or < 0 on error if none were processed.
0249  */
0250 int io_ringfd_register(struct io_ring_ctx *ctx, void __user *__arg,
0251                unsigned nr_args)
0252 {
0253     struct io_uring_rsrc_update __user *arg = __arg;
0254     struct io_uring_rsrc_update reg;
0255     struct io_uring_task *tctx;
0256     int ret, i;
0257
0258     if (!nr_args || nr_args > IO_RINGFD_REG_MAX)
0259         return -EINVAL;
0260
0261     mutex_unlock(&ctx->uring_lock);
0262     ret = __io_uring_add_tctx_node(ctx, false);
0263     mutex_lock(&ctx->uring_lock);
0264     if (ret)
0265         return ret;
0266
0267     tctx = current->io_uring;
0268     for (i = 0; i < nr_args; i++) {
0269         int start, end;
0270
0271         if (copy_from_user(&reg, &arg[i], sizeof(reg))) {
0272             ret = -EFAULT;
0273             break;
0274         }
0275
0276         if (reg.resv) {
0277             ret = -EINVAL;
0278             break;
0279         }
0280
0281         if (reg.offset == -1U) {
0282             start = 0;
0283             end = IO_RINGFD_REG_MAX;
0284         } else {
0285             if (reg.offset >= IO_RINGFD_REG_MAX) {
0286                 ret = -EINVAL;
0287                 break;
0288             }
0289             start = reg.offset;
0290             end = start + 1;
0291         }
0292
0293         ret = io_ring_add_registered_fd(tctx, reg.data, start, end);
0294         if (ret < 0)
0295             break;
0296
0297         reg.offset = ret;
0298         if (copy_to_user(&arg[i], &reg, sizeof(reg))) {
0299             fput(tctx->registered_rings[reg.offset]);
0300             tctx->registered_rings[reg.offset] = NULL;
0301             ret = -EFAULT;
0302             break;
0303         }
0304     }
0305
0306     return i ? i : ret;
0307 }
0308
0309 int io_ringfd_unregister(struct io_ring_ctx *ctx, void __user *__arg,
0310              unsigned nr_args)
0311 {
0312     struct io_uring_rsrc_update __user *arg = __arg;
0313     struct io_uring_task *tctx = current->io_uring;
0314     struct io_uring_rsrc_update reg;
0315     int ret = 0, i;
0316
0317     if (!nr_args || nr_args > IO_RINGFD_REG_MAX)
0318         return -EINVAL;
0319     if (!tctx)
0320         return 0;
0321
0322     for (i = 0; i < nr_args; i++) {
0323         if (copy_from_user(&reg, &arg[i], sizeof(reg))) {
0324             ret = -EFAULT;
0325             break;
0326         }
0327         if (reg.resv || reg.data || reg.offset >= IO_RINGFD_REG_MAX) {
0328             ret = -EINVAL;
0329             break;
0330         }
0331
0332         reg.offset = array_index_nospec(reg.offset, IO_RINGFD_REG_MAX);
0333         if (tctx->registered_rings[reg.offset]) {
0334             fput(tctx->registered_rings[reg.offset]);
0335             tctx->registered_rings[reg.offset] = NULL;
0336         }
0337     }
0338
0339     return i ? i : ret;
0340 }