0001 ==================================
0002 VDUSE - "vDPA Device in Userspace"
0003 ==================================
0004
0005 vDPA (virtio data path acceleration) device is a device that uses a
0006 datapath which complies with the virtio specifications with vendor
0007 specific control path. vDPA devices can be both physically located on
0008 the hardware or emulated by software. VDUSE is a framework that makes it
0009 possible to implement software-emulated vDPA devices in userspace. And
0010 to make the device emulation more secure, the emulated vDPA device's
0011 control path is handled in the kernel and only the data path is
0012 implemented in the userspace.
0013
0014 Note that only virtio block device is supported by VDUSE framework now,
0015 which can reduce security risks when the userspace process that implements
0016 the data path is run by an unprivileged user. The support for other device
0017 types can be added after the security issue of corresponding device driver
0018 is clarified or fixed in the future.
0019
0020 Create/Destroy VDUSE devices
0021 ----------------------------
0022
0023 VDUSE devices are created as follows:
0024
0025 1. Create a new VDUSE instance with ioctl(VDUSE_CREATE_DEV) on
0026 /dev/vduse/control.
0027
0028 2. Setup each virtqueue with ioctl(VDUSE_VQ_SETUP) on /dev/vduse/$NAME.
0029
0030 3. Begin processing VDUSE messages from /dev/vduse/$NAME. The first
0031 messages will arrive while attaching the VDUSE instance to vDPA bus.
0032
0033 4. Send the VDPA_CMD_DEV_NEW netlink message to attach the VDUSE
0034 instance to vDPA bus.
0035
0036 VDUSE devices are destroyed as follows:
0037
0038 1. Send the VDPA_CMD_DEV_DEL netlink message to detach the VDUSE
0039 instance from vDPA bus.
0040
0041 2. Close the file descriptor referring to /dev/vduse/$NAME.
0042
0043 3. Destroy the VDUSE instance with ioctl(VDUSE_DESTROY_DEV) on
0044 /dev/vduse/control.
0045
0046 The netlink messages can be sent via vdpa tool in iproute2 or use the
0047 below sample codes:
0048
0049 .. code-block:: c
0050
0051 static int netlink_add_vduse(const char *name, enum vdpa_command cmd)
0052 {
0053 struct nl_sock *nlsock;
0054 struct nl_msg *msg;
0055 int famid;
0056
0057 nlsock = nl_socket_alloc();
0058 if (!nlsock)
0059 return -ENOMEM;
0060
0061 if (genl_connect(nlsock))
0062 goto free_sock;
0063
0064 famid = genl_ctrl_resolve(nlsock, VDPA_GENL_NAME);
0065 if (famid < 0)
0066 goto close_sock;
0067
0068 msg = nlmsg_alloc();
0069 if (!msg)
0070 goto close_sock;
0071
0072 if (!genlmsg_put(msg, NL_AUTO_PORT, NL_AUTO_SEQ, famid, 0, 0, cmd, 0))
0073 goto nla_put_failure;
0074
0075 NLA_PUT_STRING(msg, VDPA_ATTR_DEV_NAME, name);
0076 if (cmd == VDPA_CMD_DEV_NEW)
0077 NLA_PUT_STRING(msg, VDPA_ATTR_MGMTDEV_DEV_NAME, "vduse");
0078
0079 if (nl_send_sync(nlsock, msg))
0080 goto close_sock;
0081
0082 nl_close(nlsock);
0083 nl_socket_free(nlsock);
0084
0085 return 0;
0086 nla_put_failure:
0087 nlmsg_free(msg);
0088 close_sock:
0089 nl_close(nlsock);
0090 free_sock:
0091 nl_socket_free(nlsock);
0092 return -1;
0093 }
0094
0095 How VDUSE works
0096 ---------------
0097
0098 As mentioned above, a VDUSE device is created by ioctl(VDUSE_CREATE_DEV) on
0099 /dev/vduse/control. With this ioctl, userspace can specify some basic configuration
0100 such as device name (uniquely identify a VDUSE device), virtio features, virtio
0101 configuration space, the number of virtqueues and so on for this emulated device.
0102 Then a char device interface (/dev/vduse/$NAME) is exported to userspace for device
0103 emulation. Userspace can use the VDUSE_VQ_SETUP ioctl on /dev/vduse/$NAME to
0104 add per-virtqueue configuration such as the max size of virtqueue to the device.
0105
0106 After the initialization, the VDUSE device can be attached to vDPA bus via
0107 the VDPA_CMD_DEV_NEW netlink message. Userspace needs to read()/write() on
0108 /dev/vduse/$NAME to receive/reply some control messages from/to VDUSE kernel
0109 module as follows:
0110
0111 .. code-block:: c
0112
0113 static int vduse_message_handler(int dev_fd)
0114 {
0115 int len;
0116 struct vduse_dev_request req;
0117 struct vduse_dev_response resp;
0118
0119 len = read(dev_fd, &req, sizeof(req));
0120 if (len != sizeof(req))
0121 return -1;
0122
0123 resp.request_id = req.request_id;
0124
0125 switch (req.type) {
0126
0127 /* handle different types of messages */
0128
0129 }
0130
0131 len = write(dev_fd, &resp, sizeof(resp));
0132 if (len != sizeof(resp))
0133 return -1;
0134
0135 return 0;
0136 }
0137
0138 There are now three types of messages introduced by VDUSE framework:
0139
0140 - VDUSE_GET_VQ_STATE: Get the state for virtqueue, userspace should return
0141 avail index for split virtqueue or the device/driver ring wrap counters and
0142 the avail and used index for packed virtqueue.
0143
0144 - VDUSE_SET_STATUS: Set the device status, userspace should follow
0145 the virtio spec: https://docs.oasis-open.org/virtio/virtio/v1.1/virtio-v1.1.html
0146 to process this message. For example, fail to set the FEATURES_OK device
0147 status bit if the device can not accept the negotiated virtio features
0148 get from the VDUSE_DEV_GET_FEATURES ioctl.
0149
0150 - VDUSE_UPDATE_IOTLB: Notify userspace to update the memory mapping for specified
0151 IOVA range, userspace should firstly remove the old mapping, then setup the new
0152 mapping via the VDUSE_IOTLB_GET_FD ioctl.
0153
0154 After DRIVER_OK status bit is set via the VDUSE_SET_STATUS message, userspace is
0155 able to start the dataplane processing as follows:
0156
0157 1. Get the specified virtqueue's information with the VDUSE_VQ_GET_INFO ioctl,
0158 including the size, the IOVAs of descriptor table, available ring and used ring,
0159 the state and the ready status.
0160
0161 2. Pass the above IOVAs to the VDUSE_IOTLB_GET_FD ioctl so that those IOVA regions
0162 can be mapped into userspace. Some sample codes is shown below:
0163
0164 .. code-block:: c
0165
0166 static int perm_to_prot(uint8_t perm)
0167 {
0168 int prot = 0;
0169
0170 switch (perm) {
0171 case VDUSE_ACCESS_WO:
0172 prot |= PROT_WRITE;
0173 break;
0174 case VDUSE_ACCESS_RO:
0175 prot |= PROT_READ;
0176 break;
0177 case VDUSE_ACCESS_RW:
0178 prot |= PROT_READ | PROT_WRITE;
0179 break;
0180 }
0181
0182 return prot;
0183 }
0184
0185 static void *iova_to_va(int dev_fd, uint64_t iova, uint64_t *len)
0186 {
0187 int fd;
0188 void *addr;
0189 size_t size;
0190 struct vduse_iotlb_entry entry;
0191
0192 entry.start = iova;
0193 entry.last = iova;
0194
0195 /*
0196 * Find the first IOVA region that overlaps with the specified
0197 * range [start, last] and return the corresponding file descriptor.
0198 */
0199 fd = ioctl(dev_fd, VDUSE_IOTLB_GET_FD, &entry);
0200 if (fd < 0)
0201 return NULL;
0202
0203 size = entry.last - entry.start + 1;
0204 *len = entry.last - iova + 1;
0205 addr = mmap(0, size, perm_to_prot(entry.perm), MAP_SHARED,
0206 fd, entry.offset);
0207 close(fd);
0208 if (addr == MAP_FAILED)
0209 return NULL;
0210
0211 /*
0212 * Using some data structures such as linked list to store
0213 * the iotlb mapping. The munmap(2) should be called for the
0214 * cached mapping when the corresponding VDUSE_UPDATE_IOTLB
0215 * message is received or the device is reset.
0216 */
0217
0218 return addr + iova - entry.start;
0219 }
0220
0221 3. Setup the kick eventfd for the specified virtqueues with the VDUSE_VQ_SETUP_KICKFD
0222 ioctl. The kick eventfd is used by VDUSE kernel module to notify userspace to
0223 consume the available ring. This is optional since userspace can choose to poll the
0224 available ring instead.
0225
0226 4. Listen to the kick eventfd (optional) and consume the available ring. The buffer
0227 described by the descriptors in the descriptor table should be also mapped into
0228 userspace via the VDUSE_IOTLB_GET_FD ioctl before accessing.
0229
0230 5. Inject an interrupt for specific virtqueue with the VDUSE_INJECT_VQ_IRQ ioctl
0231 after the used ring is filled.
0232
0233 For more details on the uAPI, please see include/uapi/linux/vduse.h.
