include/linux/vringh.h

0001 /* SPDX-License-Identifier: GPL-2.0-or-later */
0002 /*
0003  * Linux host-side vring helpers; for when the kernel needs to access
0004  * someone else's vring.
0005  *
0006  * Copyright IBM Corporation, 2013.
0007  * Parts taken from drivers/vhost/vhost.c Copyright 2009 Red Hat, Inc.
0008  *
0009  * Written by: Rusty Russell <rusty@rustcorp.com.au>
0010  */
0011 #ifndef _LINUX_VRINGH_H
0012 #define _LINUX_VRINGH_H
0013 #include <uapi/linux/virtio_ring.h>
0014 #include <linux/virtio_byteorder.h>
0015 #include <linux/uio.h>
0016 #include <linux/slab.h>
0017 #include <linux/spinlock.h>
0018 #if IS_REACHABLE(CONFIG_VHOST_IOTLB)
0019 #include <linux/dma-direction.h>
0020 #include <linux/vhost_iotlb.h>
0021 #endif
0022 #include <asm/barrier.h>
0023
0024 /* virtio_ring with information needed for host access. */
0025 struct vringh {
0026     /* Everything is little endian */
0027     bool little_endian;
0028
0029     /* Guest publishes used event idx (note: we always do). */
0030     bool event_indices;
0031
0032     /* Can we get away with weak barriers? */
0033     bool weak_barriers;
0034
0035     /* Last available index we saw (ie. where we're up to). */
0036     u16 last_avail_idx;
0037
0038     /* Last index we used. */
0039     u16 last_used_idx;
0040
0041     /* How many descriptors we've completed since last need_notify(). */
0042     u32 completed;
0043
0044     /* The vring (note: it may contain user pointers!) */
0045     struct vring vring;
0046
0047     /* IOTLB for this vring */
0048     struct vhost_iotlb *iotlb;
0049
0050     /* spinlock to synchronize IOTLB accesses */
0051     spinlock_t *iotlb_lock;
0052
0053     /* The function to call to notify the guest about added buffers */
0054     void (*notify)(struct vringh *);
0055 };
0056
0057 /**
0058  * struct vringh_config_ops - ops for creating a host vring from a virtio driver
0059  * @find_vrhs: find the host vrings and instantiate them
0060  *  vdev: the virtio_device
0061  *  nhvrs: the number of host vrings to find
0062  *  hvrs: on success, includes new host vrings
0063  *  callbacks: array of driver callbacks, for each host vring
0064  *      include a NULL entry for vqs that do not need a callback
0065  *  Returns 0 on success or error status
0066  * @del_vrhs: free the host vrings found by find_vrhs().
0067  */
0068 struct virtio_device;
0069 typedef void vrh_callback_t(struct virtio_device *, struct vringh *);
0070 struct vringh_config_ops {
0071     int (*find_vrhs)(struct virtio_device *vdev, unsigned nhvrs,
0072              struct vringh *vrhs[], vrh_callback_t *callbacks[]);
0073     void (*del_vrhs)(struct virtio_device *vdev);
0074 };
0075
0076 /* The memory the vring can access, and what offset to apply. */
0077 struct vringh_range {
0078     u64 start, end_incl;
0079     u64 offset;
0080 };
0081
0082 /**
0083  * struct vringh_iov - iovec mangler.
0084  *
0085  * Mangles iovec in place, and restores it.
0086  * Remaining data is iov + i, of used - i elements.
0087  */
0088 struct vringh_iov {
0089     struct iovec *iov;
0090     size_t consumed; /* Within iov[i] */
0091     unsigned i, used, max_num;
0092 };
0093
0094 /**
0095  * struct vringh_iov - kvec mangler.
0096  *
0097  * Mangles kvec in place, and restores it.
0098  * Remaining data is iov + i, of used - i elements.
0099  */
0100 struct vringh_kiov {
0101     struct kvec *iov;
0102     size_t consumed; /* Within iov[i] */
0103     unsigned i, used, max_num;
0104 };
0105
0106 /* Flag on max_num to indicate we're kmalloced. */
0107 #define VRINGH_IOV_ALLOCATED 0x8000000
0108
0109 /* Helpers for userspace vrings. */
0110 int vringh_init_user(struct vringh *vrh, u64 features,
0111              unsigned int num, bool weak_barriers,
0112              vring_desc_t __user *desc,
0113              vring_avail_t __user *avail,
0114              vring_used_t __user *used);
0115
0116 static inline void vringh_iov_init(struct vringh_iov *iov,
0117                    struct iovec *iovec, unsigned num)
0118 {
0119     iov->used = iov->i = 0;
0120     iov->consumed = 0;
0121     iov->max_num = num;
0122     iov->iov = iovec;
0123 }
0124
0125 static inline void vringh_iov_reset(struct vringh_iov *iov)
0126 {
0127     iov->iov[iov->i].iov_len += iov->consumed;
0128     iov->iov[iov->i].iov_base -= iov->consumed;
0129     iov->consumed = 0;
0130     iov->i = 0;
0131 }
0132
0133 static inline void vringh_iov_cleanup(struct vringh_iov *iov)
0134 {
0135     if (iov->max_num & VRINGH_IOV_ALLOCATED)
0136         kfree(iov->iov);
0137     iov->max_num = iov->used = iov->i = iov->consumed = 0;
0138     iov->iov = NULL;
0139 }
0140
0141 /* Convert a descriptor into iovecs. */
0142 int vringh_getdesc_user(struct vringh *vrh,
0143             struct vringh_iov *riov,
0144             struct vringh_iov *wiov,
0145             bool (*getrange)(struct vringh *vrh,
0146                      u64 addr, struct vringh_range *r),
0147             u16 *head);
0148
0149 /* Copy bytes from readable vsg, consuming it (and incrementing wiov->i). */
0150 ssize_t vringh_iov_pull_user(struct vringh_iov *riov, void *dst, size_t len);
0151
0152 /* Copy bytes into writable vsg, consuming it (and incrementing wiov->i). */
0153 ssize_t vringh_iov_push_user(struct vringh_iov *wiov,
0154                  const void *src, size_t len);
0155
0156 /* Mark a descriptor as used. */
0157 int vringh_complete_user(struct vringh *vrh, u16 head, u32 len);
0158 int vringh_complete_multi_user(struct vringh *vrh,
0159                    const struct vring_used_elem used[],
0160                    unsigned num_used);
0161
0162 /* Pretend we've never seen descriptor (for easy error handling). */
0163 void vringh_abandon_user(struct vringh *vrh, unsigned int num);
0164
0165 /* Do we need to fire the eventfd to notify the other side? */
0166 int vringh_need_notify_user(struct vringh *vrh);
0167
0168 bool vringh_notify_enable_user(struct vringh *vrh);
0169 void vringh_notify_disable_user(struct vringh *vrh);
0170
0171 /* Helpers for kernelspace vrings. */
0172 int vringh_init_kern(struct vringh *vrh, u64 features,
0173              unsigned int num, bool weak_barriers,
0174              struct vring_desc *desc,
0175              struct vring_avail *avail,
0176              struct vring_used *used);
0177
0178 static inline void vringh_kiov_init(struct vringh_kiov *kiov,
0179                     struct kvec *kvec, unsigned num)
0180 {
0181     kiov->used = kiov->i = 0;
0182     kiov->consumed = 0;
0183     kiov->max_num = num;
0184     kiov->iov = kvec;
0185 }
0186
0187 static inline void vringh_kiov_reset(struct vringh_kiov *kiov)
0188 {
0189     kiov->iov[kiov->i].iov_len += kiov->consumed;
0190     kiov->iov[kiov->i].iov_base -= kiov->consumed;
0191     kiov->consumed = 0;
0192     kiov->i = 0;
0193 }
0194
0195 static inline void vringh_kiov_cleanup(struct vringh_kiov *kiov)
0196 {
0197     if (kiov->max_num & VRINGH_IOV_ALLOCATED)
0198         kfree(kiov->iov);
0199     kiov->max_num = kiov->used = kiov->i = kiov->consumed = 0;
0200     kiov->iov = NULL;
0201 }
0202
0203 static inline size_t vringh_kiov_length(struct vringh_kiov *kiov)
0204 {
0205     size_t len = 0;
0206     int i;
0207
0208     for (i = kiov->i; i < kiov->used; i++)
0209         len += kiov->iov[i].iov_len;
0210
0211     return len;
0212 }
0213
0214 void vringh_kiov_advance(struct vringh_kiov *kiov, size_t len);
0215
0216 int vringh_getdesc_kern(struct vringh *vrh,
0217             struct vringh_kiov *riov,
0218             struct vringh_kiov *wiov,
0219             u16 *head,
0220             gfp_t gfp);
0221
0222 ssize_t vringh_iov_pull_kern(struct vringh_kiov *riov, void *dst, size_t len);
0223 ssize_t vringh_iov_push_kern(struct vringh_kiov *wiov,
0224                  const void *src, size_t len);
0225 void vringh_abandon_kern(struct vringh *vrh, unsigned int num);
0226 int vringh_complete_kern(struct vringh *vrh, u16 head, u32 len);
0227
0228 bool vringh_notify_enable_kern(struct vringh *vrh);
0229 void vringh_notify_disable_kern(struct vringh *vrh);
0230
0231 int vringh_need_notify_kern(struct vringh *vrh);
0232
0233 /* Notify the guest about buffers added to the used ring */
0234 static inline void vringh_notify(struct vringh *vrh)
0235 {
0236     if (vrh->notify)
0237         vrh->notify(vrh);
0238 }
0239
0240 static inline bool vringh_is_little_endian(const struct vringh *vrh)
0241 {
0242     return vrh->little_endian ||
0243         virtio_legacy_is_little_endian();
0244 }
0245
0246 static inline u16 vringh16_to_cpu(const struct vringh *vrh, __virtio16 val)
0247 {
0248     return __virtio16_to_cpu(vringh_is_little_endian(vrh), val);
0249 }
0250
0251 static inline __virtio16 cpu_to_vringh16(const struct vringh *vrh, u16 val)
0252 {
0253     return __cpu_to_virtio16(vringh_is_little_endian(vrh), val);
0254 }
0255
0256 static inline u32 vringh32_to_cpu(const struct vringh *vrh, __virtio32 val)
0257 {
0258     return __virtio32_to_cpu(vringh_is_little_endian(vrh), val);
0259 }
0260
0261 static inline __virtio32 cpu_to_vringh32(const struct vringh *vrh, u32 val)
0262 {
0263     return __cpu_to_virtio32(vringh_is_little_endian(vrh), val);
0264 }
0265
0266 static inline u64 vringh64_to_cpu(const struct vringh *vrh, __virtio64 val)
0267 {
0268     return __virtio64_to_cpu(vringh_is_little_endian(vrh), val);
0269 }
0270
0271 static inline __virtio64 cpu_to_vringh64(const struct vringh *vrh, u64 val)
0272 {
0273     return __cpu_to_virtio64(vringh_is_little_endian(vrh), val);
0274 }
0275
0276 #if IS_REACHABLE(CONFIG_VHOST_IOTLB)
0277
0278 void vringh_set_iotlb(struct vringh *vrh, struct vhost_iotlb *iotlb,
0279               spinlock_t *iotlb_lock);
0280
0281 int vringh_init_iotlb(struct vringh *vrh, u64 features,
0282               unsigned int num, bool weak_barriers,
0283               struct vring_desc *desc,
0284               struct vring_avail *avail,
0285               struct vring_used *used);
0286
0287 int vringh_getdesc_iotlb(struct vringh *vrh,
0288              struct vringh_kiov *riov,
0289              struct vringh_kiov *wiov,
0290              u16 *head,
0291              gfp_t gfp);
0292
0293 ssize_t vringh_iov_pull_iotlb(struct vringh *vrh,
0294                   struct vringh_kiov *riov,
0295                   void *dst, size_t len);
0296 ssize_t vringh_iov_push_iotlb(struct vringh *vrh,
0297                   struct vringh_kiov *wiov,
0298                   const void *src, size_t len);
0299
0300 void vringh_abandon_iotlb(struct vringh *vrh, unsigned int num);
0301
0302 int vringh_complete_iotlb(struct vringh *vrh, u16 head, u32 len);
0303
0304 bool vringh_notify_enable_iotlb(struct vringh *vrh);
0305 void vringh_notify_disable_iotlb(struct vringh *vrh);
0306
0307 int vringh_need_notify_iotlb(struct vringh *vrh);
0308
0309 #endif /* CONFIG_VHOST_IOTLB */
0310
0311 #endif /* _LINUX_VRINGH_H */