the-tree/io_uring/filetable.c

0001 // SPDX-License-Identifier: GPL-2.0
0002 #include <linux/kernel.h>
0003 #include <linux/errno.h>
0004 #include <linux/file.h>
0005 #include <linux/mm.h>
0006 #include <linux/slab.h>
0007 #include <linux/nospec.h>
0008 #include <linux/io_uring.h>
0009
0010 #include <uapi/linux/io_uring.h>
0011
0012 #include "io_uring.h"
0013 #include "rsrc.h"
0014 #include "filetable.h"
0015
0016 static int io_file_bitmap_get(struct io_ring_ctx *ctx)
0017 {
0018     struct io_file_table *table = &ctx->file_table;
0019     unsigned long nr = ctx->file_alloc_end;
0020     int ret;
0021
0022     do {
0023         ret = find_next_zero_bit(table->bitmap, nr, table->alloc_hint);
0024         if (ret != nr)
0025             return ret;
0026
0027         if (table->alloc_hint == ctx->file_alloc_start)
0028             break;
0029         nr = table->alloc_hint;
0030         table->alloc_hint = ctx->file_alloc_start;
0031     } while (1);
0032
0033     return -ENFILE;
0034 }
0035
0036 bool io_alloc_file_tables(struct io_file_table *table, unsigned nr_files)
0037 {
0038     table->files = kvcalloc(nr_files, sizeof(table->files[0]),
0039                 GFP_KERNEL_ACCOUNT);
0040     if (unlikely(!table->files))
0041         return false;
0042
0043     table->bitmap = bitmap_zalloc(nr_files, GFP_KERNEL_ACCOUNT);
0044     if (unlikely(!table->bitmap)) {
0045         kvfree(table->files);
0046         return false;
0047     }
0048
0049     return true;
0050 }
0051
0052 void io_free_file_tables(struct io_file_table *table)
0053 {
0054     kvfree(table->files);
0055     bitmap_free(table->bitmap);
0056     table->files = NULL;
0057     table->bitmap = NULL;
0058 }
0059
0060 static int io_install_fixed_file(struct io_ring_ctx *ctx, struct file *file,
0061                  u32 slot_index)
0062     __must_hold(&req->ctx->uring_lock)
0063 {
0064     bool needs_switch = false;
0065     struct io_fixed_file *file_slot;
0066     int ret;
0067
0068     if (io_is_uring_fops(file))
0069         return -EBADF;
0070     if (!ctx->file_data)
0071         return -ENXIO;
0072     if (slot_index >= ctx->nr_user_files)
0073         return -EINVAL;
0074
0075     slot_index = array_index_nospec(slot_index, ctx->nr_user_files);
0076     file_slot = io_fixed_file_slot(&ctx->file_table, slot_index);
0077
0078     if (file_slot->file_ptr) {
0079         struct file *old_file;
0080
0081         ret = io_rsrc_node_switch_start(ctx);
0082         if (ret)
0083             goto err;
0084
0085         old_file = (struct file *)(file_slot->file_ptr & FFS_MASK);
0086         ret = io_queue_rsrc_removal(ctx->file_data, slot_index,
0087                         ctx->rsrc_node, old_file);
0088         if (ret)
0089             goto err;
0090         file_slot->file_ptr = 0;
0091         io_file_bitmap_clear(&ctx->file_table, slot_index);
0092         needs_switch = true;
0093     }
0094
0095     ret = io_scm_file_account(ctx, file);
0096     if (!ret) {
0097         *io_get_tag_slot(ctx->file_data, slot_index) = 0;
0098         io_fixed_file_set(file_slot, file);
0099         io_file_bitmap_set(&ctx->file_table, slot_index);
0100     }
0101 err:
0102     if (needs_switch)
0103         io_rsrc_node_switch(ctx, ctx->file_data);
0104     if (ret)
0105         fput(file);
0106     return ret;
0107 }
0108
0109 int __io_fixed_fd_install(struct io_ring_ctx *ctx, struct file *file,
0110               unsigned int file_slot)
0111 {
0112     bool alloc_slot = file_slot == IORING_FILE_INDEX_ALLOC;
0113     int ret;
0114
0115     if (alloc_slot) {
0116         ret = io_file_bitmap_get(ctx);
0117         if (unlikely(ret < 0))
0118             return ret;
0119         file_slot = ret;
0120     } else {
0121         file_slot--;
0122     }
0123
0124     ret = io_install_fixed_file(ctx, file, file_slot);
0125     if (!ret && alloc_slot)
0126         ret = file_slot;
0127     return ret;
0128 }
0129 /*
0130  * Note when io_fixed_fd_install() returns error value, it will ensure
0131  * fput() is called correspondingly.
0132  */
0133 int io_fixed_fd_install(struct io_kiocb *req, unsigned int issue_flags,
0134             struct file *file, unsigned int file_slot)
0135 {
0136     struct io_ring_ctx *ctx = req->ctx;
0137     int ret;
0138
0139     io_ring_submit_lock(ctx, issue_flags);
0140     ret = __io_fixed_fd_install(ctx, file, file_slot);
0141     io_ring_submit_unlock(ctx, issue_flags);
0142
0143     if (unlikely(ret < 0))
0144         fput(file);
0145     return ret;
0146 }
0147
0148 int io_fixed_fd_remove(struct io_ring_ctx *ctx, unsigned int offset)
0149 {
0150     struct io_fixed_file *file_slot;
0151     struct file *file;
0152     int ret;
0153
0154     if (unlikely(!ctx->file_data))
0155         return -ENXIO;
0156     if (offset >= ctx->nr_user_files)
0157         return -EINVAL;
0158     ret = io_rsrc_node_switch_start(ctx);
0159     if (ret)
0160         return ret;
0161
0162     offset = array_index_nospec(offset, ctx->nr_user_files);
0163     file_slot = io_fixed_file_slot(&ctx->file_table, offset);
0164     if (!file_slot->file_ptr)
0165         return -EBADF;
0166
0167     file = (struct file *)(file_slot->file_ptr & FFS_MASK);
0168     ret = io_queue_rsrc_removal(ctx->file_data, offset, ctx->rsrc_node, file);
0169     if (ret)
0170         return ret;
0171
0172     file_slot->file_ptr = 0;
0173     io_file_bitmap_clear(&ctx->file_table, offset);
0174     io_rsrc_node_switch(ctx, ctx->file_data);
0175     return 0;
0176 }
0177
0178 int io_register_file_alloc_range(struct io_ring_ctx *ctx,
0179                  struct io_uring_file_index_range __user *arg)
0180 {
0181     struct io_uring_file_index_range range;
0182     u32 end;
0183
0184     if (copy_from_user(&range, arg, sizeof(range)))
0185         return -EFAULT;
0186     if (check_add_overflow(range.off, range.len, &end))
0187         return -EOVERFLOW;
0188     if (range.resv || end > ctx->nr_user_files)
0189         return -EINVAL;
0190
0191     io_file_table_set_alloc_range(ctx, range.off, range.len);
0192     return 0;
0193 }