tools/io_uring/queue.c

0001 #include <sys/types.h>
0002 #include <sys/stat.h>
0003 #include <sys/mman.h>
0004 #include <unistd.h>
0005 #include <errno.h>
0006 #include <string.h>
0007
0008 #include "liburing.h"
0009 #include "barrier.h"
0010
0011 static int __io_uring_get_cqe(struct io_uring *ring,
0012                   struct io_uring_cqe **cqe_ptr, int wait)
0013 {
0014     struct io_uring_cq *cq = &ring->cq;
0015     const unsigned mask = *cq->kring_mask;
0016     unsigned head;
0017     int ret;
0018
0019     *cqe_ptr = NULL;
0020     head = *cq->khead;
0021     do {
0022         /*
0023          * It's necessary to use a read_barrier() before reading
0024          * the CQ tail, since the kernel updates it locklessly. The
0025          * kernel has the matching store barrier for the update. The
0026          * kernel also ensures that previous stores to CQEs are ordered
0027          * with the tail update.
0028          */
0029         read_barrier();
0030         if (head != *cq->ktail) {
0031             *cqe_ptr = &cq->cqes[head & mask];
0032             break;
0033         }
0034         if (!wait)
0035             break;
0036         ret = io_uring_enter(ring->ring_fd, 0, 1,
0037                     IORING_ENTER_GETEVENTS, NULL);
0038         if (ret < 0)
0039             return -errno;
0040     } while (1);
0041
0042     return 0;
0043 }
0044
0045 /*
0046  * Return an IO completion, if one is readily available. Returns 0 with
0047  * cqe_ptr filled in on success, -errno on failure.
0048  */
0049 int io_uring_peek_cqe(struct io_uring *ring, struct io_uring_cqe **cqe_ptr)
0050 {
0051     return __io_uring_get_cqe(ring, cqe_ptr, 0);
0052 }
0053
0054 /*
0055  * Return an IO completion, waiting for it if necessary. Returns 0 with
0056  * cqe_ptr filled in on success, -errno on failure.
0057  */
0058 int io_uring_wait_cqe(struct io_uring *ring, struct io_uring_cqe **cqe_ptr)
0059 {
0060     return __io_uring_get_cqe(ring, cqe_ptr, 1);
0061 }
0062
0063 /*
0064  * Submit sqes acquired from io_uring_get_sqe() to the kernel.
0065  *
0066  * Returns number of sqes submitted
0067  */
0068 int io_uring_submit(struct io_uring *ring)
0069 {
0070     struct io_uring_sq *sq = &ring->sq;
0071     const unsigned mask = *sq->kring_mask;
0072     unsigned ktail, ktail_next, submitted, to_submit;
0073     int ret;
0074
0075     /*
0076      * If we have pending IO in the kring, submit it first. We need a
0077      * read barrier here to match the kernels store barrier when updating
0078      * the SQ head.
0079      */
0080     read_barrier();
0081     if (*sq->khead != *sq->ktail) {
0082         submitted = *sq->kring_entries;
0083         goto submit;
0084     }
0085
0086     if (sq->sqe_head == sq->sqe_tail)
0087         return 0;
0088
0089     /*
0090      * Fill in sqes that we have queued up, adding them to the kernel ring
0091      */
0092     submitted = 0;
0093     ktail = ktail_next = *sq->ktail;
0094     to_submit = sq->sqe_tail - sq->sqe_head;
0095     while (to_submit--) {
0096         ktail_next++;
0097         read_barrier();
0098
0099         sq->array[ktail & mask] = sq->sqe_head & mask;
0100         ktail = ktail_next;
0101
0102         sq->sqe_head++;
0103         submitted++;
0104     }
0105
0106     if (!submitted)
0107         return 0;
0108
0109     if (*sq->ktail != ktail) {
0110         /*
0111          * First write barrier ensures that the SQE stores are updated
0112          * with the tail update. This is needed so that the kernel
0113          * will never see a tail update without the preceeding sQE
0114          * stores being done.
0115          */
0116         write_barrier();
0117         *sq->ktail = ktail;
0118         /*
0119          * The kernel has the matching read barrier for reading the
0120          * SQ tail.
0121          */
0122         write_barrier();
0123     }
0124
0125 submit:
0126     ret = io_uring_enter(ring->ring_fd, submitted, 0,
0127                 IORING_ENTER_GETEVENTS, NULL);
0128     if (ret < 0)
0129         return -errno;
0130
0131     return ret;
0132 }
0133
0134 /*
0135  * Return an sqe to fill. Application must later call io_uring_submit()
0136  * when it's ready to tell the kernel about it. The caller may call this
0137  * function multiple times before calling io_uring_submit().
0138  *
0139  * Returns a vacant sqe, or NULL if we're full.
0140  */
0141 struct io_uring_sqe *io_uring_get_sqe(struct io_uring *ring)
0142 {
0143     struct io_uring_sq *sq = &ring->sq;
0144     unsigned next = sq->sqe_tail + 1;
0145     struct io_uring_sqe *sqe;
0146
0147     /*
0148      * All sqes are used
0149      */
0150     if (next - sq->sqe_head > *sq->kring_entries)
0151         return NULL;
0152
0153     sqe = &sq->sqes[sq->sqe_tail & *sq->kring_mask];
0154     sq->sqe_tail = next;
0155     return sqe;
0156 }