the-tree/io_uring/rsrc.h

0001 // SPDX-License-Identifier: GPL-2.0
0002 #ifndef IOU_RSRC_H
0003 #define IOU_RSRC_H
0004
0005 #include <net/af_unix.h>
0006
0007 #define IO_RSRC_TAG_TABLE_SHIFT (PAGE_SHIFT - 3)
0008 #define IO_RSRC_TAG_TABLE_MAX   (1U << IO_RSRC_TAG_TABLE_SHIFT)
0009 #define IO_RSRC_TAG_TABLE_MASK  (IO_RSRC_TAG_TABLE_MAX - 1)
0010
0011 enum {
0012     IORING_RSRC_FILE        = 0,
0013     IORING_RSRC_BUFFER      = 1,
0014 };
0015
0016 struct io_rsrc_put {
0017     struct list_head list;
0018     u64 tag;
0019     union {
0020         void *rsrc;
0021         struct file *file;
0022         struct io_mapped_ubuf *buf;
0023     };
0024 };
0025
0026 typedef void (rsrc_put_fn)(struct io_ring_ctx *ctx, struct io_rsrc_put *prsrc);
0027
0028 struct io_rsrc_data {
0029     struct io_ring_ctx      *ctx;
0030
0031     u64             **tags;
0032     unsigned int            nr;
0033     rsrc_put_fn         *do_put;
0034     atomic_t            refs;
0035     struct completion       done;
0036     bool                quiesce;
0037 };
0038
0039 struct io_rsrc_node {
0040     struct percpu_ref       refs;
0041     struct list_head        node;
0042     struct list_head        rsrc_list;
0043     struct io_rsrc_data     *rsrc_data;
0044     struct llist_node       llist;
0045     bool                done;
0046 };
0047
0048 struct io_mapped_ubuf {
0049     u64     ubuf;
0050     u64     ubuf_end;
0051     unsigned int    nr_bvecs;
0052     unsigned long   acct_pages;
0053     struct bio_vec  bvec[];
0054 };
0055
0056 void io_rsrc_put_work(struct work_struct *work);
0057 void io_rsrc_refs_refill(struct io_ring_ctx *ctx);
0058 void io_wait_rsrc_data(struct io_rsrc_data *data);
0059 void io_rsrc_node_destroy(struct io_rsrc_node *ref_node);
0060 void io_rsrc_refs_drop(struct io_ring_ctx *ctx);
0061 int io_rsrc_node_switch_start(struct io_ring_ctx *ctx);
0062 int io_queue_rsrc_removal(struct io_rsrc_data *data, unsigned idx,
0063               struct io_rsrc_node *node, void *rsrc);
0064 void io_rsrc_node_switch(struct io_ring_ctx *ctx,
0065              struct io_rsrc_data *data_to_kill);
0066
0067 int io_import_fixed(int ddir, struct iov_iter *iter,
0068                struct io_mapped_ubuf *imu,
0069                u64 buf_addr, size_t len);
0070
0071 void __io_sqe_buffers_unregister(struct io_ring_ctx *ctx);
0072 int io_sqe_buffers_unregister(struct io_ring_ctx *ctx);
0073 int io_sqe_buffers_register(struct io_ring_ctx *ctx, void __user *arg,
0074                 unsigned int nr_args, u64 __user *tags);
0075 void __io_sqe_files_unregister(struct io_ring_ctx *ctx);
0076 int io_sqe_files_unregister(struct io_ring_ctx *ctx);
0077 int io_sqe_files_register(struct io_ring_ctx *ctx, void __user *arg,
0078               unsigned nr_args, u64 __user *tags);
0079
0080 int __io_scm_file_account(struct io_ring_ctx *ctx, struct file *file);
0081
0082 #if defined(CONFIG_UNIX)
0083 static inline bool io_file_need_scm(struct file *filp)
0084 {
0085 #if defined(IO_URING_SCM_ALL)
0086     return true;
0087 #else
0088     return !!unix_get_socket(filp);
0089 #endif
0090 }
0091 #else
0092 static inline bool io_file_need_scm(struct file *filp)
0093 {
0094     return false;
0095 }
0096 #endif
0097
0098 static inline int io_scm_file_account(struct io_ring_ctx *ctx,
0099                       struct file *file)
0100 {
0101     if (likely(!io_file_need_scm(file)))
0102         return 0;
0103     return __io_scm_file_account(ctx, file);
0104 }
0105
0106 int io_register_files_update(struct io_ring_ctx *ctx, void __user *arg,
0107                  unsigned nr_args);
0108 int io_register_rsrc_update(struct io_ring_ctx *ctx, void __user *arg,
0109                 unsigned size, unsigned type);
0110 int io_register_rsrc(struct io_ring_ctx *ctx, void __user *arg,
0111             unsigned int size, unsigned int type);
0112
0113 static inline void io_rsrc_put_node(struct io_rsrc_node *node, int nr)
0114 {
0115     percpu_ref_put_many(&node->refs, nr);
0116 }
0117
0118 static inline void io_req_put_rsrc(struct io_kiocb *req)
0119 {
0120     if (req->rsrc_node)
0121         io_rsrc_put_node(req->rsrc_node, 1);
0122 }
0123
0124 static inline void io_req_put_rsrc_locked(struct io_kiocb *req,
0125                       struct io_ring_ctx *ctx)
0126     __must_hold(&ctx->uring_lock)
0127 {
0128     struct io_rsrc_node *node = req->rsrc_node;
0129
0130     if (node) {
0131         if (node == ctx->rsrc_node)
0132             ctx->rsrc_cached_refs++;
0133         else
0134             io_rsrc_put_node(node, 1);
0135     }
0136 }
0137
0138 static inline void io_charge_rsrc_node(struct io_ring_ctx *ctx)
0139 {
0140     ctx->rsrc_cached_refs--;
0141     if (unlikely(ctx->rsrc_cached_refs < 0))
0142         io_rsrc_refs_refill(ctx);
0143 }
0144
0145 static inline void io_req_set_rsrc_node(struct io_kiocb *req,
0146                     struct io_ring_ctx *ctx,
0147                     unsigned int issue_flags)
0148 {
0149     if (!req->rsrc_node) {
0150         req->rsrc_node = ctx->rsrc_node;
0151
0152         if (!(issue_flags & IO_URING_F_UNLOCKED)) {
0153             lockdep_assert_held(&ctx->uring_lock);
0154
0155             io_charge_rsrc_node(ctx);
0156         } else {
0157             percpu_ref_get(&req->rsrc_node->refs);
0158         }
0159     }
0160 }
0161
0162 static inline u64 *io_get_tag_slot(struct io_rsrc_data *data, unsigned int idx)
0163 {
0164     unsigned int off = idx & IO_RSRC_TAG_TABLE_MASK;
0165     unsigned int table_idx = idx >> IO_RSRC_TAG_TABLE_SHIFT;
0166
0167     return &data->tags[table_idx][off];
0168 }
0169
0170 int io_files_update(struct io_kiocb *req, unsigned int issue_flags);
0171 int io_files_update_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe);
0172
0173 int __io_account_mem(struct user_struct *user, unsigned long nr_pages);
0174
0175 static inline void __io_unaccount_mem(struct user_struct *user,
0176                       unsigned long nr_pages)
0177 {
0178     atomic_long_sub(nr_pages, &user->locked_vm);
0179 }
0180
0181 #endif