0001
0002
0003
0004
0005 #include <linux/kernel.h>
0006 #include <linux/errno.h>
0007 #include <linux/fs.h>
0008 #include <linux/file.h>
0009 #include <linux/io_uring.h>
0010
0011 #include "io_uring.h"
0012 #include "opdef.h"
0013 #include "refs.h"
0014 #include "tctx.h"
0015 #include "sqpoll.h"
0016 #include "fdinfo.h"
0017 #include "kbuf.h"
0018 #include "rsrc.h"
0019
0020 #include "xattr.h"
0021 #include "nop.h"
0022 #include "fs.h"
0023 #include "splice.h"
0024 #include "sync.h"
0025 #include "advise.h"
0026 #include "openclose.h"
0027 #include "uring_cmd.h"
0028 #include "epoll.h"
0029 #include "statx.h"
0030 #include "net.h"
0031 #include "msg_ring.h"
0032 #include "timeout.h"
0033 #include "poll.h"
0034 #include "cancel.h"
0035 #include "rw.h"
0036
0037 static int io_no_issue(struct io_kiocb *req, unsigned int issue_flags)
0038 {
0039 WARN_ON_ONCE(1);
0040 return -ECANCELED;
0041 }
0042
0043 static __maybe_unused int io_eopnotsupp_prep(struct io_kiocb *kiocb,
0044 const struct io_uring_sqe *sqe)
0045 {
0046 return -EOPNOTSUPP;
0047 }
0048
0049 const struct io_op_def io_op_defs[] = {
0050 [IORING_OP_NOP] = {
0051 .audit_skip = 1,
0052 .iopoll = 1,
0053 .name = "NOP",
0054 .prep = io_nop_prep,
0055 .issue = io_nop,
0056 },
0057 [IORING_OP_READV] = {
0058 .needs_file = 1,
0059 .unbound_nonreg_file = 1,
0060 .pollin = 1,
0061 .buffer_select = 1,
0062 .plug = 1,
0063 .audit_skip = 1,
0064 .ioprio = 1,
0065 .iopoll = 1,
0066 .async_size = sizeof(struct io_async_rw),
0067 .name = "READV",
0068 .prep = io_prep_rw,
0069 .issue = io_read,
0070 .prep_async = io_readv_prep_async,
0071 .cleanup = io_readv_writev_cleanup,
0072 },
0073 [IORING_OP_WRITEV] = {
0074 .needs_file = 1,
0075 .hash_reg_file = 1,
0076 .unbound_nonreg_file = 1,
0077 .pollout = 1,
0078 .plug = 1,
0079 .audit_skip = 1,
0080 .ioprio = 1,
0081 .iopoll = 1,
0082 .async_size = sizeof(struct io_async_rw),
0083 .name = "WRITEV",
0084 .prep = io_prep_rw,
0085 .issue = io_write,
0086 .prep_async = io_writev_prep_async,
0087 .cleanup = io_readv_writev_cleanup,
0088 },
0089 [IORING_OP_FSYNC] = {
0090 .needs_file = 1,
0091 .audit_skip = 1,
0092 .name = "FSYNC",
0093 .prep = io_fsync_prep,
0094 .issue = io_fsync,
0095 },
0096 [IORING_OP_READ_FIXED] = {
0097 .needs_file = 1,
0098 .unbound_nonreg_file = 1,
0099 .pollin = 1,
0100 .plug = 1,
0101 .audit_skip = 1,
0102 .ioprio = 1,
0103 .iopoll = 1,
0104 .async_size = sizeof(struct io_async_rw),
0105 .name = "READ_FIXED",
0106 .prep = io_prep_rw,
0107 .issue = io_read,
0108 },
0109 [IORING_OP_WRITE_FIXED] = {
0110 .needs_file = 1,
0111 .hash_reg_file = 1,
0112 .unbound_nonreg_file = 1,
0113 .pollout = 1,
0114 .plug = 1,
0115 .audit_skip = 1,
0116 .ioprio = 1,
0117 .iopoll = 1,
0118 .async_size = sizeof(struct io_async_rw),
0119 .name = "WRITE_FIXED",
0120 .prep = io_prep_rw,
0121 .issue = io_write,
0122 },
0123 [IORING_OP_POLL_ADD] = {
0124 .needs_file = 1,
0125 .unbound_nonreg_file = 1,
0126 .audit_skip = 1,
0127 .name = "POLL_ADD",
0128 .prep = io_poll_add_prep,
0129 .issue = io_poll_add,
0130 },
0131 [IORING_OP_POLL_REMOVE] = {
0132 .audit_skip = 1,
0133 .name = "POLL_REMOVE",
0134 .prep = io_poll_remove_prep,
0135 .issue = io_poll_remove,
0136 },
0137 [IORING_OP_SYNC_FILE_RANGE] = {
0138 .needs_file = 1,
0139 .audit_skip = 1,
0140 .name = "SYNC_FILE_RANGE",
0141 .prep = io_sfr_prep,
0142 .issue = io_sync_file_range,
0143 },
0144 [IORING_OP_SENDMSG] = {
0145 .needs_file = 1,
0146 .unbound_nonreg_file = 1,
0147 .pollout = 1,
0148 .ioprio = 1,
0149 .name = "SENDMSG",
0150 #if defined(CONFIG_NET)
0151 .async_size = sizeof(struct io_async_msghdr),
0152 .prep = io_sendmsg_prep,
0153 .issue = io_sendmsg,
0154 .prep_async = io_sendmsg_prep_async,
0155 .cleanup = io_sendmsg_recvmsg_cleanup,
0156 #else
0157 .prep = io_eopnotsupp_prep,
0158 #endif
0159 },
0160 [IORING_OP_RECVMSG] = {
0161 .needs_file = 1,
0162 .unbound_nonreg_file = 1,
0163 .pollin = 1,
0164 .buffer_select = 1,
0165 .ioprio = 1,
0166 .name = "RECVMSG",
0167 #if defined(CONFIG_NET)
0168 .async_size = sizeof(struct io_async_msghdr),
0169 .prep = io_recvmsg_prep,
0170 .issue = io_recvmsg,
0171 .prep_async = io_recvmsg_prep_async,
0172 .cleanup = io_sendmsg_recvmsg_cleanup,
0173 #else
0174 .prep = io_eopnotsupp_prep,
0175 #endif
0176 },
0177 [IORING_OP_TIMEOUT] = {
0178 .audit_skip = 1,
0179 .async_size = sizeof(struct io_timeout_data),
0180 .name = "TIMEOUT",
0181 .prep = io_timeout_prep,
0182 .issue = io_timeout,
0183 },
0184 [IORING_OP_TIMEOUT_REMOVE] = {
0185
0186 .audit_skip = 1,
0187 .name = "TIMEOUT_REMOVE",
0188 .prep = io_timeout_remove_prep,
0189 .issue = io_timeout_remove,
0190 },
0191 [IORING_OP_ACCEPT] = {
0192 .needs_file = 1,
0193 .unbound_nonreg_file = 1,
0194 .pollin = 1,
0195 .poll_exclusive = 1,
0196 .ioprio = 1,
0197 .name = "ACCEPT",
0198 #if defined(CONFIG_NET)
0199 .prep = io_accept_prep,
0200 .issue = io_accept,
0201 #else
0202 .prep = io_eopnotsupp_prep,
0203 #endif
0204 },
0205 [IORING_OP_ASYNC_CANCEL] = {
0206 .audit_skip = 1,
0207 .name = "ASYNC_CANCEL",
0208 .prep = io_async_cancel_prep,
0209 .issue = io_async_cancel,
0210 },
0211 [IORING_OP_LINK_TIMEOUT] = {
0212 .audit_skip = 1,
0213 .async_size = sizeof(struct io_timeout_data),
0214 .name = "LINK_TIMEOUT",
0215 .prep = io_link_timeout_prep,
0216 .issue = io_no_issue,
0217 },
0218 [IORING_OP_CONNECT] = {
0219 .needs_file = 1,
0220 .unbound_nonreg_file = 1,
0221 .pollout = 1,
0222 .name = "CONNECT",
0223 #if defined(CONFIG_NET)
0224 .async_size = sizeof(struct io_async_connect),
0225 .prep = io_connect_prep,
0226 .issue = io_connect,
0227 .prep_async = io_connect_prep_async,
0228 #else
0229 .prep = io_eopnotsupp_prep,
0230 #endif
0231 },
0232 [IORING_OP_FALLOCATE] = {
0233 .needs_file = 1,
0234 .name = "FALLOCATE",
0235 .prep = io_fallocate_prep,
0236 .issue = io_fallocate,
0237 },
0238 [IORING_OP_OPENAT] = {
0239 .name = "OPENAT",
0240 .prep = io_openat_prep,
0241 .issue = io_openat,
0242 .cleanup = io_open_cleanup,
0243 },
0244 [IORING_OP_CLOSE] = {
0245 .name = "CLOSE",
0246 .prep = io_close_prep,
0247 .issue = io_close,
0248 },
0249 [IORING_OP_FILES_UPDATE] = {
0250 .audit_skip = 1,
0251 .iopoll = 1,
0252 .name = "FILES_UPDATE",
0253 .prep = io_files_update_prep,
0254 .issue = io_files_update,
0255 },
0256 [IORING_OP_STATX] = {
0257 .audit_skip = 1,
0258 .name = "STATX",
0259 .prep = io_statx_prep,
0260 .issue = io_statx,
0261 .cleanup = io_statx_cleanup,
0262 },
0263 [IORING_OP_READ] = {
0264 .needs_file = 1,
0265 .unbound_nonreg_file = 1,
0266 .pollin = 1,
0267 .buffer_select = 1,
0268 .plug = 1,
0269 .audit_skip = 1,
0270 .ioprio = 1,
0271 .iopoll = 1,
0272 .async_size = sizeof(struct io_async_rw),
0273 .name = "READ",
0274 .prep = io_prep_rw,
0275 .issue = io_read,
0276 },
0277 [IORING_OP_WRITE] = {
0278 .needs_file = 1,
0279 .hash_reg_file = 1,
0280 .unbound_nonreg_file = 1,
0281 .pollout = 1,
0282 .plug = 1,
0283 .audit_skip = 1,
0284 .ioprio = 1,
0285 .iopoll = 1,
0286 .async_size = sizeof(struct io_async_rw),
0287 .name = "WRITE",
0288 .prep = io_prep_rw,
0289 .issue = io_write,
0290 },
0291 [IORING_OP_FADVISE] = {
0292 .needs_file = 1,
0293 .audit_skip = 1,
0294 .name = "FADVISE",
0295 .prep = io_fadvise_prep,
0296 .issue = io_fadvise,
0297 },
0298 [IORING_OP_MADVISE] = {
0299 .name = "MADVISE",
0300 .prep = io_madvise_prep,
0301 .issue = io_madvise,
0302 },
0303 [IORING_OP_SEND] = {
0304 .needs_file = 1,
0305 .unbound_nonreg_file = 1,
0306 .pollout = 1,
0307 .audit_skip = 1,
0308 .ioprio = 1,
0309 .name = "SEND",
0310 #if defined(CONFIG_NET)
0311 .prep = io_sendmsg_prep,
0312 .issue = io_send,
0313 #else
0314 .prep = io_eopnotsupp_prep,
0315 #endif
0316 },
0317 [IORING_OP_RECV] = {
0318 .needs_file = 1,
0319 .unbound_nonreg_file = 1,
0320 .pollin = 1,
0321 .buffer_select = 1,
0322 .audit_skip = 1,
0323 .ioprio = 1,
0324 .name = "RECV",
0325 #if defined(CONFIG_NET)
0326 .prep = io_recvmsg_prep,
0327 .issue = io_recv,
0328 #else
0329 .prep = io_eopnotsupp_prep,
0330 #endif
0331 },
0332 [IORING_OP_OPENAT2] = {
0333 .name = "OPENAT2",
0334 .prep = io_openat2_prep,
0335 .issue = io_openat2,
0336 .cleanup = io_open_cleanup,
0337 },
0338 [IORING_OP_EPOLL_CTL] = {
0339 .unbound_nonreg_file = 1,
0340 .audit_skip = 1,
0341 .name = "EPOLL",
0342 #if defined(CONFIG_EPOLL)
0343 .prep = io_epoll_ctl_prep,
0344 .issue = io_epoll_ctl,
0345 #else
0346 .prep = io_eopnotsupp_prep,
0347 #endif
0348 },
0349 [IORING_OP_SPLICE] = {
0350 .needs_file = 1,
0351 .hash_reg_file = 1,
0352 .unbound_nonreg_file = 1,
0353 .audit_skip = 1,
0354 .name = "SPLICE",
0355 .prep = io_splice_prep,
0356 .issue = io_splice,
0357 },
0358 [IORING_OP_PROVIDE_BUFFERS] = {
0359 .audit_skip = 1,
0360 .iopoll = 1,
0361 .name = "PROVIDE_BUFFERS",
0362 .prep = io_provide_buffers_prep,
0363 .issue = io_provide_buffers,
0364 },
0365 [IORING_OP_REMOVE_BUFFERS] = {
0366 .audit_skip = 1,
0367 .iopoll = 1,
0368 .name = "REMOVE_BUFFERS",
0369 .prep = io_remove_buffers_prep,
0370 .issue = io_remove_buffers,
0371 },
0372 [IORING_OP_TEE] = {
0373 .needs_file = 1,
0374 .hash_reg_file = 1,
0375 .unbound_nonreg_file = 1,
0376 .audit_skip = 1,
0377 .name = "TEE",
0378 .prep = io_tee_prep,
0379 .issue = io_tee,
0380 },
0381 [IORING_OP_SHUTDOWN] = {
0382 .needs_file = 1,
0383 .name = "SHUTDOWN",
0384 #if defined(CONFIG_NET)
0385 .prep = io_shutdown_prep,
0386 .issue = io_shutdown,
0387 #else
0388 .prep = io_eopnotsupp_prep,
0389 #endif
0390 },
0391 [IORING_OP_RENAMEAT] = {
0392 .name = "RENAMEAT",
0393 .prep = io_renameat_prep,
0394 .issue = io_renameat,
0395 .cleanup = io_renameat_cleanup,
0396 },
0397 [IORING_OP_UNLINKAT] = {
0398 .name = "UNLINKAT",
0399 .prep = io_unlinkat_prep,
0400 .issue = io_unlinkat,
0401 .cleanup = io_unlinkat_cleanup,
0402 },
0403 [IORING_OP_MKDIRAT] = {
0404 .name = "MKDIRAT",
0405 .prep = io_mkdirat_prep,
0406 .issue = io_mkdirat,
0407 .cleanup = io_mkdirat_cleanup,
0408 },
0409 [IORING_OP_SYMLINKAT] = {
0410 .name = "SYMLINKAT",
0411 .prep = io_symlinkat_prep,
0412 .issue = io_symlinkat,
0413 .cleanup = io_link_cleanup,
0414 },
0415 [IORING_OP_LINKAT] = {
0416 .name = "LINKAT",
0417 .prep = io_linkat_prep,
0418 .issue = io_linkat,
0419 .cleanup = io_link_cleanup,
0420 },
0421 [IORING_OP_MSG_RING] = {
0422 .needs_file = 1,
0423 .iopoll = 1,
0424 .name = "MSG_RING",
0425 .prep = io_msg_ring_prep,
0426 .issue = io_msg_ring,
0427 },
0428 [IORING_OP_FSETXATTR] = {
0429 .needs_file = 1,
0430 .name = "FSETXATTR",
0431 .prep = io_fsetxattr_prep,
0432 .issue = io_fsetxattr,
0433 .cleanup = io_xattr_cleanup,
0434 },
0435 [IORING_OP_SETXATTR] = {
0436 .name = "SETXATTR",
0437 .prep = io_setxattr_prep,
0438 .issue = io_setxattr,
0439 .cleanup = io_xattr_cleanup,
0440 },
0441 [IORING_OP_FGETXATTR] = {
0442 .needs_file = 1,
0443 .name = "FGETXATTR",
0444 .prep = io_fgetxattr_prep,
0445 .issue = io_fgetxattr,
0446 .cleanup = io_xattr_cleanup,
0447 },
0448 [IORING_OP_GETXATTR] = {
0449 .name = "GETXATTR",
0450 .prep = io_getxattr_prep,
0451 .issue = io_getxattr,
0452 .cleanup = io_xattr_cleanup,
0453 },
0454 [IORING_OP_SOCKET] = {
0455 .audit_skip = 1,
0456 .name = "SOCKET",
0457 #if defined(CONFIG_NET)
0458 .prep = io_socket_prep,
0459 .issue = io_socket,
0460 #else
0461 .prep = io_eopnotsupp_prep,
0462 #endif
0463 },
0464 [IORING_OP_URING_CMD] = {
0465 .needs_file = 1,
0466 .plug = 1,
0467 .name = "URING_CMD",
0468 .async_size = uring_cmd_pdu_size(1),
0469 .prep = io_uring_cmd_prep,
0470 .issue = io_uring_cmd,
0471 .prep_async = io_uring_cmd_prep_async,
0472 },
0473 [IORING_OP_SEND_ZC] = {
0474 .name = "SEND_ZC",
0475 .needs_file = 1,
0476 .unbound_nonreg_file = 1,
0477 .pollout = 1,
0478 .audit_skip = 1,
0479 .ioprio = 1,
0480 .manual_alloc = 1,
0481 #if defined(CONFIG_NET)
0482 .async_size = sizeof(struct io_async_msghdr),
0483 .prep = io_sendzc_prep,
0484 .issue = io_sendzc,
0485 .prep_async = io_sendzc_prep_async,
0486 .cleanup = io_sendzc_cleanup,
0487 #else
0488 .prep = io_eopnotsupp_prep,
0489 #endif
0490 },
0491 };
0492
0493 const char *io_uring_get_opcode(u8 opcode)
0494 {
0495 if (opcode < IORING_OP_LAST)
0496 return io_op_defs[opcode].name;
0497 return "INVALID";
0498 }
0499
0500 void __init io_uring_optable_init(void)
0501 {
0502 int i;
0503
0504 BUILD_BUG_ON(ARRAY_SIZE(io_op_defs) != IORING_OP_LAST);
0505
0506 for (i = 0; i < ARRAY_SIZE(io_op_defs); i++) {
0507 BUG_ON(!io_op_defs[i].prep);
0508 if (io_op_defs[i].prep != io_eopnotsupp_prep)
0509 BUG_ON(!io_op_defs[i].issue);
0510 WARN_ON_ONCE(!io_op_defs[i].name);
0511 }
0512 }
