virtio/ringtest/virtio_ring_0_9.c

0001 // SPDX-License-Identifier: GPL-2.0-only
0002 /*
0003  * Copyright (C) 2016 Red Hat, Inc.
0004  * Author: Michael S. Tsirkin <mst@redhat.com>
0005  *
0006  * Partial implementation of virtio 0.9. event index is used for signalling,
0007  * unconditionally. Design roughly follows linux kernel implementation in order
0008  * to be able to judge its performance.
0009  */
0010 #define _GNU_SOURCE
0011 #include "main.h"
0012 #include <stdlib.h>
0013 #include <stdio.h>
0014 #include <assert.h>
0015 #include <string.h>
0016 #include <linux/virtio_ring.h>
0017
0018 struct data {
0019     void *data;
0020 } *data;
0021
0022 struct vring ring;
0023
0024 /* enabling the below activates experimental ring polling code
0025  * (which skips index reads on consumer in favor of looking at
0026  * high bits of ring id ^ 0x8000).
0027  */
0028 /* #ifdef RING_POLL */
0029 /* enabling the below activates experimental in-order code
0030  * (which skips ring updates and reads and writes len in descriptor).
0031  */
0032 /* #ifdef INORDER */
0033
0034 #if defined(RING_POLL) && defined(INORDER)
0035 #error "RING_POLL and INORDER are mutually exclusive"
0036 #endif
0037
0038 /* how much padding is needed to avoid false cache sharing */
0039 #define HOST_GUEST_PADDING 0x80
0040
0041 struct guest {
0042     unsigned short avail_idx;
0043     unsigned short last_used_idx;
0044     unsigned short num_free;
0045     unsigned short kicked_avail_idx;
0046 #ifndef INORDER
0047     unsigned short free_head;
0048 #else
0049     unsigned short reserved_free_head;
0050 #endif
0051     unsigned char reserved[HOST_GUEST_PADDING - 10];
0052 } guest;
0053
0054 struct host {
0055     /* we do not need to track last avail index
0056      * unless we have more than one in flight.
0057      */
0058     unsigned short used_idx;
0059     unsigned short called_used_idx;
0060     unsigned char reserved[HOST_GUEST_PADDING - 4];
0061 } host;
0062
0063 /* implemented by ring */
0064 void alloc_ring(void)
0065 {
0066     int ret;
0067     int i;
0068     void *p;
0069
0070     ret = posix_memalign(&p, 0x1000, vring_size(ring_size, 0x1000));
0071     if (ret) {
0072         perror("Unable to allocate ring buffer.\n");
0073         exit(3);
0074     }
0075     memset(p, 0, vring_size(ring_size, 0x1000));
0076     vring_init(&ring, ring_size, p, 0x1000);
0077
0078     guest.avail_idx = 0;
0079     guest.kicked_avail_idx = -1;
0080     guest.last_used_idx = 0;
0081 #ifndef INORDER
0082     /* Put everything in free lists. */
0083     guest.free_head = 0;
0084 #endif
0085     for (i = 0; i < ring_size - 1; i++)
0086         ring.desc[i].next = i + 1;
0087     host.used_idx = 0;
0088     host.called_used_idx = -1;
0089     guest.num_free = ring_size;
0090     data = malloc(ring_size * sizeof *data);
0091     if (!data) {
0092         perror("Unable to allocate data buffer.\n");
0093         exit(3);
0094     }
0095     memset(data, 0, ring_size * sizeof *data);
0096 }
0097
0098 /* guest side */
0099 int add_inbuf(unsigned len, void *buf, void *datap)
0100 {
0101     unsigned head;
0102 #ifndef INORDER
0103     unsigned avail;
0104 #endif
0105     struct vring_desc *desc;
0106
0107     if (!guest.num_free)
0108         return -1;
0109
0110 #ifdef INORDER
0111     head = (ring_size - 1) & (guest.avail_idx++);
0112 #else
0113     head = guest.free_head;
0114 #endif
0115     guest.num_free--;
0116
0117     desc = ring.desc;
0118     desc[head].flags = VRING_DESC_F_NEXT;
0119     desc[head].addr = (unsigned long)(void *)buf;
0120     desc[head].len = len;
0121     /* We do it like this to simulate the way
0122      * we'd have to flip it if we had multiple
0123      * descriptors.
0124      */
0125     desc[head].flags &= ~VRING_DESC_F_NEXT;
0126 #ifndef INORDER
0127     guest.free_head = desc[head].next;
0128 #endif
0129
0130     data[head].data = datap;
0131
0132 #ifdef RING_POLL
0133     /* Barrier A (for pairing) */
0134     smp_release();
0135     avail = guest.avail_idx++;
0136     ring.avail->ring[avail & (ring_size - 1)] =
0137         (head | (avail & ~(ring_size - 1))) ^ 0x8000;
0138 #else
0139 #ifndef INORDER
0140     /* Barrier A (for pairing) */
0141     smp_release();
0142     avail = (ring_size - 1) & (guest.avail_idx++);
0143     ring.avail->ring[avail] = head;
0144 #endif
0145     /* Barrier A (for pairing) */
0146     smp_release();
0147 #endif
0148     ring.avail->idx = guest.avail_idx;
0149     return 0;
0150 }
0151
0152 void *get_buf(unsigned *lenp, void **bufp)
0153 {
0154     unsigned head;
0155     unsigned index;
0156     void *datap;
0157
0158 #ifdef RING_POLL
0159     head = (ring_size - 1) & guest.last_used_idx;
0160     index = ring.used->ring[head].id;
0161     if ((index ^ guest.last_used_idx ^ 0x8000) & ~(ring_size - 1))
0162         return NULL;
0163     /* Barrier B (for pairing) */
0164     smp_acquire();
0165     index &= ring_size - 1;
0166 #else
0167     if (ring.used->idx == guest.last_used_idx)
0168         return NULL;
0169     /* Barrier B (for pairing) */
0170     smp_acquire();
0171 #ifdef INORDER
0172     head = (ring_size - 1) & guest.last_used_idx;
0173     index = head;
0174 #else
0175     head = (ring_size - 1) & guest.last_used_idx;
0176     index = ring.used->ring[head].id;
0177 #endif
0178
0179 #endif
0180 #ifdef INORDER
0181     *lenp = ring.desc[index].len;
0182 #else
0183     *lenp = ring.used->ring[head].len;
0184 #endif
0185     datap = data[index].data;
0186     *bufp = (void*)(unsigned long)ring.desc[index].addr;
0187     data[index].data = NULL;
0188 #ifndef INORDER
0189     ring.desc[index].next = guest.free_head;
0190     guest.free_head = index;
0191 #endif
0192     guest.num_free++;
0193     guest.last_used_idx++;
0194     return datap;
0195 }
0196
0197 bool used_empty()
0198 {
0199     unsigned short last_used_idx = guest.last_used_idx;
0200 #ifdef RING_POLL
0201     unsigned short head = last_used_idx & (ring_size - 1);
0202     unsigned index = ring.used->ring[head].id;
0203
0204     return (index ^ last_used_idx ^ 0x8000) & ~(ring_size - 1);
0205 #else
0206     return ring.used->idx == last_used_idx;
0207 #endif
0208 }
0209
0210 void disable_call()
0211 {
0212     /* Doing nothing to disable calls might cause
0213      * extra interrupts, but reduces the number of cache misses.
0214      */
0215 }
0216
0217 bool enable_call()
0218 {
0219     vring_used_event(&ring) = guest.last_used_idx;
0220     /* Flush call index write */
0221     /* Barrier D (for pairing) */
0222     smp_mb();
0223     return used_empty();
0224 }
0225
0226 void kick_available(void)
0227 {
0228     bool need;
0229
0230     /* Flush in previous flags write */
0231     /* Barrier C (for pairing) */
0232     smp_mb();
0233     need = vring_need_event(vring_avail_event(&ring),
0234                 guest.avail_idx,
0235                 guest.kicked_avail_idx);
0236
0237     guest.kicked_avail_idx = guest.avail_idx;
0238     if (need)
0239         kick();
0240 }
0241
0242 /* host side */
0243 void disable_kick()
0244 {
0245     /* Doing nothing to disable kicks might cause
0246      * extra interrupts, but reduces the number of cache misses.
0247      */
0248 }
0249
0250 bool enable_kick()
0251 {
0252     vring_avail_event(&ring) = host.used_idx;
0253     /* Barrier C (for pairing) */
0254     smp_mb();
0255     return avail_empty();
0256 }
0257
0258 bool avail_empty()
0259 {
0260     unsigned head = host.used_idx;
0261 #ifdef RING_POLL
0262     unsigned index = ring.avail->ring[head & (ring_size - 1)];
0263
0264     return ((index ^ head ^ 0x8000) & ~(ring_size - 1));
0265 #else
0266     return head == ring.avail->idx;
0267 #endif
0268 }
0269
0270 bool use_buf(unsigned *lenp, void **bufp)
0271 {
0272     unsigned used_idx = host.used_idx;
0273     struct vring_desc *desc;
0274     unsigned head;
0275
0276 #ifdef RING_POLL
0277     head = ring.avail->ring[used_idx & (ring_size - 1)];
0278     if ((used_idx ^ head ^ 0x8000) & ~(ring_size - 1))
0279         return false;
0280     /* Barrier A (for pairing) */
0281     smp_acquire();
0282
0283     used_idx &= ring_size - 1;
0284     desc = &ring.desc[head & (ring_size - 1)];
0285 #else
0286     if (used_idx == ring.avail->idx)
0287         return false;
0288
0289     /* Barrier A (for pairing) */
0290     smp_acquire();
0291
0292     used_idx &= ring_size - 1;
0293 #ifdef INORDER
0294     head = used_idx;
0295 #else
0296     head = ring.avail->ring[used_idx];
0297 #endif
0298     desc = &ring.desc[head];
0299 #endif
0300
0301     *lenp = desc->len;
0302     *bufp = (void *)(unsigned long)desc->addr;
0303
0304 #ifdef INORDER
0305     desc->len = desc->len - 1;
0306 #else
0307     /* now update used ring */
0308     ring.used->ring[used_idx].id = head;
0309     ring.used->ring[used_idx].len = desc->len - 1;
0310 #endif
0311     /* Barrier B (for pairing) */
0312     smp_release();
0313     host.used_idx++;
0314     ring.used->idx = host.used_idx;
0315
0316     return true;
0317 }
0318
0319 void call_used(void)
0320 {
0321     bool need;
0322
0323     /* Flush in previous flags write */
0324     /* Barrier D (for pairing) */
0325     smp_mb();
0326     need = vring_need_event(vring_used_event(&ring),
0327                 host.used_idx,
0328                 host.called_used_idx);
0329
0330     host.called_used_idx = host.used_idx;
0331     if (need)
0332         call();
0333 }