include/linux/virtio_config.h

0001 /* SPDX-License-Identifier: GPL-2.0 */
0002 #ifndef _LINUX_VIRTIO_CONFIG_H
0003 #define _LINUX_VIRTIO_CONFIG_H
0004
0005 #include <linux/err.h>
0006 #include <linux/bug.h>
0007 #include <linux/virtio.h>
0008 #include <linux/virtio_byteorder.h>
0009 #include <linux/compiler_types.h>
0010 #include <uapi/linux/virtio_config.h>
0011
0012 struct irq_affinity;
0013
0014 struct virtio_shm_region {
0015     u64 addr;
0016     u64 len;
0017 };
0018
0019 /**
0020  * virtio_config_ops - operations for configuring a virtio device
0021  * Note: Do not assume that a transport implements all of the operations
0022  *       getting/setting a value as a simple read/write! Generally speaking,
0023  *       any of @get/@set, @get_status/@set_status, or @get_features/
0024  *       @finalize_features are NOT safe to be called from an atomic
0025  *       context.
0026  * @get: read the value of a configuration field
0027  *  vdev: the virtio_device
0028  *  offset: the offset of the configuration field
0029  *  buf: the buffer to write the field value into.
0030  *  len: the length of the buffer
0031  * @set: write the value of a configuration field
0032  *  vdev: the virtio_device
0033  *  offset: the offset of the configuration field
0034  *  buf: the buffer to read the field value from.
0035  *  len: the length of the buffer
0036  * @generation: config generation counter (optional)
0037  *  vdev: the virtio_device
0038  *  Returns the config generation counter
0039  * @get_status: read the status byte
0040  *  vdev: the virtio_device
0041  *  Returns the status byte
0042  * @set_status: write the status byte
0043  *  vdev: the virtio_device
0044  *  status: the new status byte
0045  * @reset: reset the device
0046  *  vdev: the virtio device
0047  *  After this, status and feature negotiation must be done again
0048  *  Device must not be reset from its vq/config callbacks, or in
0049  *  parallel with being added/removed.
0050  * @find_vqs: find virtqueues and instantiate them.
0051  *  vdev: the virtio_device
0052  *  nvqs: the number of virtqueues to find
0053  *  vqs: on success, includes new virtqueues
0054  *  callbacks: array of callbacks, for each virtqueue
0055  *      include a NULL entry for vqs that do not need a callback
0056  *  names: array of virtqueue names (mainly for debugging)
0057  *      include a NULL entry for vqs unused by driver
0058  *  Returns 0 on success or error status
0059  * @del_vqs: free virtqueues found by find_vqs().
0060  * @synchronize_cbs: synchronize with the virtqueue callbacks (optional)
0061  *      The function guarantees that all memory operations on the
0062  *      queue before it are visible to the vring_interrupt() that is
0063  *      called after it.
0064  *      vdev: the virtio_device
0065  * @get_features: get the array of feature bits for this device.
0066  *  vdev: the virtio_device
0067  *  Returns the first 64 feature bits (all we currently need).
0068  * @finalize_features: confirm what device features we'll be using.
0069  *  vdev: the virtio_device
0070  *  This sends the driver feature bits to the device: it can change
0071  *  the dev->feature bits if it wants.
0072  * Note: despite the name this can be called any number of times.
0073  *  Returns 0 on success or error status
0074  * @bus_name: return the bus name associated with the device (optional)
0075  *  vdev: the virtio_device
0076  *      This returns a pointer to the bus name a la pci_name from which
0077  *      the caller can then copy.
0078  * @set_vq_affinity: set the affinity for a virtqueue (optional).
0079  * @get_vq_affinity: get the affinity for a virtqueue (optional).
0080  * @get_shm_region: get a shared memory region based on the index.
0081  * @disable_vq_and_reset: reset a queue individually (optional).
0082  *  vq: the virtqueue
0083  *  Returns 0 on success or error status
0084  *  disable_vq_and_reset will guarantee that the callbacks are disabled and
0085  *  synchronized.
0086  *  Except for the callback, the caller should guarantee that the vring is
0087  *  not accessed by any functions of virtqueue.
0088  * @enable_vq_after_reset: enable a reset queue
0089  *  vq: the virtqueue
0090  *  Returns 0 on success or error status
0091  *  If disable_vq_and_reset is set, then enable_vq_after_reset must also be
0092  *  set.
0093  */
0094 typedef void vq_callback_t(struct virtqueue *);
0095 struct virtio_config_ops {
0096     void (*get)(struct virtio_device *vdev, unsigned offset,
0097             void *buf, unsigned len);
0098     void (*set)(struct virtio_device *vdev, unsigned offset,
0099             const void *buf, unsigned len);
0100     u32 (*generation)(struct virtio_device *vdev);
0101     u8 (*get_status)(struct virtio_device *vdev);
0102     void (*set_status)(struct virtio_device *vdev, u8 status);
0103     void (*reset)(struct virtio_device *vdev);
0104     int (*find_vqs)(struct virtio_device *, unsigned nvqs,
0105             struct virtqueue *vqs[], vq_callback_t *callbacks[],
0106             const char * const names[], const bool *ctx,
0107             struct irq_affinity *desc);
0108     void (*del_vqs)(struct virtio_device *);
0109     void (*synchronize_cbs)(struct virtio_device *);
0110     u64 (*get_features)(struct virtio_device *vdev);
0111     int (*finalize_features)(struct virtio_device *vdev);
0112     const char *(*bus_name)(struct virtio_device *vdev);
0113     int (*set_vq_affinity)(struct virtqueue *vq,
0114                    const struct cpumask *cpu_mask);
0115     const struct cpumask *(*get_vq_affinity)(struct virtio_device *vdev,
0116             int index);
0117     bool (*get_shm_region)(struct virtio_device *vdev,
0118                    struct virtio_shm_region *region, u8 id);
0119     int (*disable_vq_and_reset)(struct virtqueue *vq);
0120     int (*enable_vq_after_reset)(struct virtqueue *vq);
0121 };
0122
0123 /* If driver didn't advertise the feature, it will never appear. */
0124 void virtio_check_driver_offered_feature(const struct virtio_device *vdev,
0125                      unsigned int fbit);
0126
0127 /**
0128  * __virtio_test_bit - helper to test feature bits. For use by transports.
0129  *                     Devices should normally use virtio_has_feature,
0130  *                     which includes more checks.
0131  * @vdev: the device
0132  * @fbit: the feature bit
0133  */
0134 static inline bool __virtio_test_bit(const struct virtio_device *vdev,
0135                      unsigned int fbit)
0136 {
0137     /* Did you forget to fix assumptions on max features? */
0138     if (__builtin_constant_p(fbit))
0139         BUILD_BUG_ON(fbit >= 64);
0140     else
0141         BUG_ON(fbit >= 64);
0142
0143     return vdev->features & BIT_ULL(fbit);
0144 }
0145
0146 /**
0147  * __virtio_set_bit - helper to set feature bits. For use by transports.
0148  * @vdev: the device
0149  * @fbit: the feature bit
0150  */
0151 static inline void __virtio_set_bit(struct virtio_device *vdev,
0152                     unsigned int fbit)
0153 {
0154     /* Did you forget to fix assumptions on max features? */
0155     if (__builtin_constant_p(fbit))
0156         BUILD_BUG_ON(fbit >= 64);
0157     else
0158         BUG_ON(fbit >= 64);
0159
0160     vdev->features |= BIT_ULL(fbit);
0161 }
0162
0163 /**
0164  * __virtio_clear_bit - helper to clear feature bits. For use by transports.
0165  * @vdev: the device
0166  * @fbit: the feature bit
0167  */
0168 static inline void __virtio_clear_bit(struct virtio_device *vdev,
0169                       unsigned int fbit)
0170 {
0171     /* Did you forget to fix assumptions on max features? */
0172     if (__builtin_constant_p(fbit))
0173         BUILD_BUG_ON(fbit >= 64);
0174     else
0175         BUG_ON(fbit >= 64);
0176
0177     vdev->features &= ~BIT_ULL(fbit);
0178 }
0179
0180 /**
0181  * virtio_has_feature - helper to determine if this device has this feature.
0182  * @vdev: the device
0183  * @fbit: the feature bit
0184  */
0185 static inline bool virtio_has_feature(const struct virtio_device *vdev,
0186                       unsigned int fbit)
0187 {
0188     if (fbit < VIRTIO_TRANSPORT_F_START)
0189         virtio_check_driver_offered_feature(vdev, fbit);
0190
0191     return __virtio_test_bit(vdev, fbit);
0192 }
0193
0194 /**
0195  * virtio_has_dma_quirk - determine whether this device has the DMA quirk
0196  * @vdev: the device
0197  */
0198 static inline bool virtio_has_dma_quirk(const struct virtio_device *vdev)
0199 {
0200     /*
0201      * Note the reverse polarity of the quirk feature (compared to most
0202      * other features), this is for compatibility with legacy systems.
0203      */
0204     return !virtio_has_feature(vdev, VIRTIO_F_ACCESS_PLATFORM);
0205 }
0206
0207 static inline
0208 struct virtqueue *virtio_find_single_vq(struct virtio_device *vdev,
0209                     vq_callback_t *c, const char *n)
0210 {
0211     vq_callback_t *callbacks[] = { c };
0212     const char *names[] = { n };
0213     struct virtqueue *vq;
0214     int err = vdev->config->find_vqs(vdev, 1, &vq, callbacks, names, NULL,
0215                      NULL);
0216     if (err < 0)
0217         return ERR_PTR(err);
0218     return vq;
0219 }
0220
0221 static inline
0222 int virtio_find_vqs(struct virtio_device *vdev, unsigned nvqs,
0223             struct virtqueue *vqs[], vq_callback_t *callbacks[],
0224             const char * const names[],
0225             struct irq_affinity *desc)
0226 {
0227     return vdev->config->find_vqs(vdev, nvqs, vqs, callbacks, names, NULL, desc);
0228 }
0229
0230 static inline
0231 int virtio_find_vqs_ctx(struct virtio_device *vdev, unsigned nvqs,
0232             struct virtqueue *vqs[], vq_callback_t *callbacks[],
0233             const char * const names[], const bool *ctx,
0234             struct irq_affinity *desc)
0235 {
0236     return vdev->config->find_vqs(vdev, nvqs, vqs, callbacks, names, ctx,
0237                       desc);
0238 }
0239
0240 /**
0241  * virtio_synchronize_cbs - synchronize with virtqueue callbacks
0242  * @dev: the virtio device
0243  */
0244 static inline
0245 void virtio_synchronize_cbs(struct virtio_device *dev)
0246 {
0247     if (dev->config->synchronize_cbs) {
0248         dev->config->synchronize_cbs(dev);
0249     } else {
0250         /*
0251          * A best effort fallback to synchronize with
0252          * interrupts, preemption and softirq disabled
0253          * regions. See comment above synchronize_rcu().
0254          */
0255         synchronize_rcu();
0256     }
0257 }
0258
0259 /**
0260  * virtio_device_ready - enable vq use in probe function
0261  * @dev: the virtio device
0262  *
0263  * Driver must call this to use vqs in the probe function.
0264  *
0265  * Note: vqs are enabled automatically after probe returns.
0266  */
0267 static inline
0268 void virtio_device_ready(struct virtio_device *dev)
0269 {
0270     unsigned status = dev->config->get_status(dev);
0271
0272     WARN_ON(status & VIRTIO_CONFIG_S_DRIVER_OK);
0273
0274 #ifdef CONFIG_VIRTIO_HARDEN_NOTIFICATION
0275     /*
0276      * The virtio_synchronize_cbs() makes sure vring_interrupt()
0277      * will see the driver specific setup if it sees vq->broken
0278      * as false (even if the notifications come before DRIVER_OK).
0279      */
0280     virtio_synchronize_cbs(dev);
0281     __virtio_unbreak_device(dev);
0282 #endif
0283     /*
0284      * The transport should ensure the visibility of vq->broken
0285      * before setting DRIVER_OK. See the comments for the transport
0286      * specific set_status() method.
0287      *
0288      * A well behaved device will only notify a virtqueue after
0289      * DRIVER_OK, this means the device should "see" the coherenct
0290      * memory write that set vq->broken as false which is done by
0291      * the driver when it sees DRIVER_OK, then the following
0292      * driver's vring_interrupt() will see vq->broken as false so
0293      * we won't lose any notification.
0294      */
0295     dev->config->set_status(dev, status | VIRTIO_CONFIG_S_DRIVER_OK);
0296 }
0297
0298 static inline
0299 const char *virtio_bus_name(struct virtio_device *vdev)
0300 {
0301     if (!vdev->config->bus_name)
0302         return "virtio";
0303     return vdev->config->bus_name(vdev);
0304 }
0305
0306 /**
0307  * virtqueue_set_affinity - setting affinity for a virtqueue
0308  * @vq: the virtqueue
0309  * @cpu_mask: the cpu mask
0310  *
0311  * Pay attention the function are best-effort: the affinity hint may not be set
0312  * due to config support, irq type and sharing.
0313  *
0314  */
0315 static inline
0316 int virtqueue_set_affinity(struct virtqueue *vq, const struct cpumask *cpu_mask)
0317 {
0318     struct virtio_device *vdev = vq->vdev;
0319     if (vdev->config->set_vq_affinity)
0320         return vdev->config->set_vq_affinity(vq, cpu_mask);
0321     return 0;
0322 }
0323
0324 static inline
0325 bool virtio_get_shm_region(struct virtio_device *vdev,
0326                struct virtio_shm_region *region, u8 id)
0327 {
0328     if (!vdev->config->get_shm_region)
0329         return false;
0330     return vdev->config->get_shm_region(vdev, region, id);
0331 }
0332
0333 static inline bool virtio_is_little_endian(struct virtio_device *vdev)
0334 {
0335     return virtio_has_feature(vdev, VIRTIO_F_VERSION_1) ||
0336         virtio_legacy_is_little_endian();
0337 }
0338
0339 /* Memory accessors */
0340 static inline u16 virtio16_to_cpu(struct virtio_device *vdev, __virtio16 val)
0341 {
0342     return __virtio16_to_cpu(virtio_is_little_endian(vdev), val);
0343 }
0344
0345 static inline __virtio16 cpu_to_virtio16(struct virtio_device *vdev, u16 val)
0346 {
0347     return __cpu_to_virtio16(virtio_is_little_endian(vdev), val);
0348 }
0349
0350 static inline u32 virtio32_to_cpu(struct virtio_device *vdev, __virtio32 val)
0351 {
0352     return __virtio32_to_cpu(virtio_is_little_endian(vdev), val);
0353 }
0354
0355 static inline __virtio32 cpu_to_virtio32(struct virtio_device *vdev, u32 val)
0356 {
0357     return __cpu_to_virtio32(virtio_is_little_endian(vdev), val);
0358 }
0359
0360 static inline u64 virtio64_to_cpu(struct virtio_device *vdev, __virtio64 val)
0361 {
0362     return __virtio64_to_cpu(virtio_is_little_endian(vdev), val);
0363 }
0364
0365 static inline __virtio64 cpu_to_virtio64(struct virtio_device *vdev, u64 val)
0366 {
0367     return __cpu_to_virtio64(virtio_is_little_endian(vdev), val);
0368 }
0369
0370 #define virtio_to_cpu(vdev, x) \
0371     _Generic((x), \
0372         __u8: (x), \
0373         __virtio16: virtio16_to_cpu((vdev), (x)), \
0374         __virtio32: virtio32_to_cpu((vdev), (x)), \
0375         __virtio64: virtio64_to_cpu((vdev), (x)) \
0376         )
0377
0378 #define cpu_to_virtio(vdev, x, m) \
0379     _Generic((m), \
0380         __u8: (x), \
0381         __virtio16: cpu_to_virtio16((vdev), (x)), \
0382         __virtio32: cpu_to_virtio32((vdev), (x)), \
0383         __virtio64: cpu_to_virtio64((vdev), (x)) \
0384         )
0385
0386 #define __virtio_native_type(structname, member) \
0387     typeof(virtio_to_cpu(NULL, ((structname*)0)->member))
0388
0389 /* Config space accessors. */
0390 #define virtio_cread(vdev, structname, member, ptr)         \
0391     do {                                \
0392         typeof(((structname*)0)->member) virtio_cread_v;    \
0393                                     \
0394         might_sleep();                      \
0395         /* Sanity check: must match the member's type */    \
0396         typecheck(typeof(virtio_to_cpu((vdev), virtio_cread_v)), *(ptr)); \
0397                                     \
0398         switch (sizeof(virtio_cread_v)) {           \
0399         case 1:                         \
0400         case 2:                         \
0401         case 4:                         \
0402             vdev->config->get((vdev),           \
0403                       offsetof(structname, member), \
0404                       &virtio_cread_v,      \
0405                       sizeof(virtio_cread_v));  \
0406             break;                      \
0407         default:                        \
0408             __virtio_cread_many((vdev),             \
0409                       offsetof(structname, member), \
0410                       &virtio_cread_v,      \
0411                       1,                \
0412                       sizeof(virtio_cread_v));  \
0413             break;                      \
0414         }                           \
0415         *(ptr) = virtio_to_cpu(vdev, virtio_cread_v);       \
0416     } while(0)
0417
0418 /* Config space accessors. */
0419 #define virtio_cwrite(vdev, structname, member, ptr)            \
0420     do {                                \
0421         typeof(((structname*)0)->member) virtio_cwrite_v =  \
0422             cpu_to_virtio(vdev, *(ptr), ((structname*)0)->member); \
0423                                     \
0424         might_sleep();                      \
0425         /* Sanity check: must match the member's type */    \
0426         typecheck(typeof(virtio_to_cpu((vdev), virtio_cwrite_v)), *(ptr)); \
0427                                     \
0428         vdev->config->set((vdev), offsetof(structname, member), \
0429                   &virtio_cwrite_v,         \
0430                   sizeof(virtio_cwrite_v));     \
0431     } while(0)
0432
0433 /*
0434  * Nothing virtio-specific about these, but let's worry about generalizing
0435  * these later.
0436  */
0437 #define virtio_le_to_cpu(x) \
0438     _Generic((x), \
0439         __u8: (u8)(x), \
0440          __le16: (u16)le16_to_cpu(x), \
0441          __le32: (u32)le32_to_cpu(x), \
0442          __le64: (u64)le64_to_cpu(x) \
0443         )
0444
0445 #define virtio_cpu_to_le(x, m) \
0446     _Generic((m), \
0447          __u8: (x), \
0448          __le16: cpu_to_le16(x), \
0449          __le32: cpu_to_le32(x), \
0450          __le64: cpu_to_le64(x) \
0451         )
0452
0453 /* LE (e.g. modern) Config space accessors. */
0454 #define virtio_cread_le(vdev, structname, member, ptr)          \
0455     do {                                \
0456         typeof(((structname*)0)->member) virtio_cread_v;    \
0457                                     \
0458         might_sleep();                      \
0459         /* Sanity check: must match the member's type */    \
0460         typecheck(typeof(virtio_le_to_cpu(virtio_cread_v)), *(ptr)); \
0461                                     \
0462         switch (sizeof(virtio_cread_v)) {           \
0463         case 1:                         \
0464         case 2:                         \
0465         case 4:                         \
0466             vdev->config->get((vdev),           \
0467                       offsetof(structname, member), \
0468                       &virtio_cread_v,      \
0469                       sizeof(virtio_cread_v));  \
0470             break;                      \
0471         default:                        \
0472             __virtio_cread_many((vdev),             \
0473                       offsetof(structname, member), \
0474                       &virtio_cread_v,      \
0475                       1,                \
0476                       sizeof(virtio_cread_v));  \
0477             break;                      \
0478         }                           \
0479         *(ptr) = virtio_le_to_cpu(virtio_cread_v);      \
0480     } while(0)
0481
0482 #define virtio_cwrite_le(vdev, structname, member, ptr)         \
0483     do {                                \
0484         typeof(((structname*)0)->member) virtio_cwrite_v =  \
0485             virtio_cpu_to_le(*(ptr), ((structname*)0)->member); \
0486                                     \
0487         might_sleep();                      \
0488         /* Sanity check: must match the member's type */    \
0489         typecheck(typeof(virtio_le_to_cpu(virtio_cwrite_v)), *(ptr)); \
0490                                     \
0491         vdev->config->set((vdev), offsetof(structname, member), \
0492                   &virtio_cwrite_v,         \
0493                   sizeof(virtio_cwrite_v));     \
0494     } while(0)
0495
0496
0497 /* Read @count fields, @bytes each. */
0498 static inline void __virtio_cread_many(struct virtio_device *vdev,
0499                        unsigned int offset,
0500                        void *buf, size_t count, size_t bytes)
0501 {
0502     u32 old, gen = vdev->config->generation ?
0503         vdev->config->generation(vdev) : 0;
0504     int i;
0505
0506     might_sleep();
0507     do {
0508         old = gen;
0509
0510         for (i = 0; i < count; i++)
0511             vdev->config->get(vdev, offset + bytes * i,
0512                       buf + i * bytes, bytes);
0513
0514         gen = vdev->config->generation ?
0515             vdev->config->generation(vdev) : 0;
0516     } while (gen != old);
0517 }
0518
0519 static inline void virtio_cread_bytes(struct virtio_device *vdev,
0520                       unsigned int offset,
0521                       void *buf, size_t len)
0522 {
0523     __virtio_cread_many(vdev, offset, buf, len, 1);
0524 }
0525
0526 static inline u8 virtio_cread8(struct virtio_device *vdev, unsigned int offset)
0527 {
0528     u8 ret;
0529
0530     might_sleep();
0531     vdev->config->get(vdev, offset, &ret, sizeof(ret));
0532     return ret;
0533 }
0534
0535 static inline void virtio_cwrite8(struct virtio_device *vdev,
0536                   unsigned int offset, u8 val)
0537 {
0538     might_sleep();
0539     vdev->config->set(vdev, offset, &val, sizeof(val));
0540 }
0541
0542 static inline u16 virtio_cread16(struct virtio_device *vdev,
0543                  unsigned int offset)
0544 {
0545     __virtio16 ret;
0546
0547     might_sleep();
0548     vdev->config->get(vdev, offset, &ret, sizeof(ret));
0549     return virtio16_to_cpu(vdev, ret);
0550 }
0551
0552 static inline void virtio_cwrite16(struct virtio_device *vdev,
0553                    unsigned int offset, u16 val)
0554 {
0555     __virtio16 v;
0556
0557     might_sleep();
0558     v = cpu_to_virtio16(vdev, val);
0559     vdev->config->set(vdev, offset, &v, sizeof(v));
0560 }
0561
0562 static inline u32 virtio_cread32(struct virtio_device *vdev,
0563                  unsigned int offset)
0564 {
0565     __virtio32 ret;
0566
0567     might_sleep();
0568     vdev->config->get(vdev, offset, &ret, sizeof(ret));
0569     return virtio32_to_cpu(vdev, ret);
0570 }
0571
0572 static inline void virtio_cwrite32(struct virtio_device *vdev,
0573                    unsigned int offset, u32 val)
0574 {
0575     __virtio32 v;
0576
0577     might_sleep();
0578     v = cpu_to_virtio32(vdev, val);
0579     vdev->config->set(vdev, offset, &v, sizeof(v));
0580 }
0581
0582 static inline u64 virtio_cread64(struct virtio_device *vdev,
0583                  unsigned int offset)
0584 {
0585     __virtio64 ret;
0586
0587     __virtio_cread_many(vdev, offset, &ret, 1, sizeof(ret));
0588     return virtio64_to_cpu(vdev, ret);
0589 }
0590
0591 static inline void virtio_cwrite64(struct virtio_device *vdev,
0592                    unsigned int offset, u64 val)
0593 {
0594     __virtio64 v;
0595
0596     might_sleep();
0597     v = cpu_to_virtio64(vdev, val);
0598     vdev->config->set(vdev, offset, &v, sizeof(v));
0599 }
0600
0601 /* Conditional config space accessors. */
0602 #define virtio_cread_feature(vdev, fbit, structname, member, ptr)   \
0603     ({                              \
0604         int _r = 0;                     \
0605         if (!virtio_has_feature(vdev, fbit))            \
0606             _r = -ENOENT;                   \
0607         else                            \
0608             virtio_cread((vdev), structname, member, ptr);  \
0609         _r;                         \
0610     })
0611
0612 /* Conditional config space accessors. */
0613 #define virtio_cread_le_feature(vdev, fbit, structname, member, ptr)    \
0614     ({                              \
0615         int _r = 0;                     \
0616         if (!virtio_has_feature(vdev, fbit))            \
0617             _r = -ENOENT;                   \
0618         else                            \
0619             virtio_cread_le((vdev), structname, member, ptr); \
0620         _r;                         \
0621     })
0622
0623 #endif /* _LINUX_VIRTIO_CONFIG_H */