drivers/virtio/virtio_vdpa.c

0001 // SPDX-License-Identifier: GPL-2.0-only
0002 /*
0003  * VIRTIO based driver for vDPA device
0004  *
0005  * Copyright (c) 2020, Red Hat. All rights reserved.
0006  *     Author: Jason Wang <jasowang@redhat.com>
0007  *
0008  */
0009
0010 #include <linux/init.h>
0011 #include <linux/module.h>
0012 #include <linux/device.h>
0013 #include <linux/kernel.h>
0014 #include <linux/slab.h>
0015 #include <linux/uuid.h>
0016 #include <linux/virtio.h>
0017 #include <linux/vdpa.h>
0018 #include <linux/virtio_config.h>
0019 #include <linux/virtio_ring.h>
0020
0021 #define MOD_VERSION  "0.1"
0022 #define MOD_AUTHOR   "Jason Wang <jasowang@redhat.com>"
0023 #define MOD_DESC     "vDPA bus driver for virtio devices"
0024 #define MOD_LICENSE  "GPL v2"
0025
0026 struct virtio_vdpa_device {
0027     struct virtio_device vdev;
0028     struct vdpa_device *vdpa;
0029     u64 features;
0030
0031     /* The lock to protect virtqueue list */
0032     spinlock_t lock;
0033     /* List of virtio_vdpa_vq_info */
0034     struct list_head virtqueues;
0035 };
0036
0037 struct virtio_vdpa_vq_info {
0038     /* the actual virtqueue */
0039     struct virtqueue *vq;
0040
0041     /* the list node for the virtqueues list */
0042     struct list_head node;
0043 };
0044
0045 static inline struct virtio_vdpa_device *
0046 to_virtio_vdpa_device(struct virtio_device *dev)
0047 {
0048     return container_of(dev, struct virtio_vdpa_device, vdev);
0049 }
0050
0051 static struct vdpa_device *vd_get_vdpa(struct virtio_device *vdev)
0052 {
0053     return to_virtio_vdpa_device(vdev)->vdpa;
0054 }
0055
0056 static void virtio_vdpa_get(struct virtio_device *vdev, unsigned int offset,
0057                 void *buf, unsigned int len)
0058 {
0059     struct vdpa_device *vdpa = vd_get_vdpa(vdev);
0060
0061     vdpa_get_config(vdpa, offset, buf, len);
0062 }
0063
0064 static void virtio_vdpa_set(struct virtio_device *vdev, unsigned int offset,
0065                 const void *buf, unsigned int len)
0066 {
0067     struct vdpa_device *vdpa = vd_get_vdpa(vdev);
0068
0069     vdpa_set_config(vdpa, offset, buf, len);
0070 }
0071
0072 static u32 virtio_vdpa_generation(struct virtio_device *vdev)
0073 {
0074     struct vdpa_device *vdpa = vd_get_vdpa(vdev);
0075     const struct vdpa_config_ops *ops = vdpa->config;
0076
0077     if (ops->get_generation)
0078         return ops->get_generation(vdpa);
0079
0080     return 0;
0081 }
0082
0083 static u8 virtio_vdpa_get_status(struct virtio_device *vdev)
0084 {
0085     struct vdpa_device *vdpa = vd_get_vdpa(vdev);
0086     const struct vdpa_config_ops *ops = vdpa->config;
0087
0088     return ops->get_status(vdpa);
0089 }
0090
0091 static void virtio_vdpa_set_status(struct virtio_device *vdev, u8 status)
0092 {
0093     struct vdpa_device *vdpa = vd_get_vdpa(vdev);
0094
0095     return vdpa_set_status(vdpa, status);
0096 }
0097
0098 static void virtio_vdpa_reset(struct virtio_device *vdev)
0099 {
0100     struct vdpa_device *vdpa = vd_get_vdpa(vdev);
0101
0102     vdpa_reset(vdpa);
0103 }
0104
0105 static bool virtio_vdpa_notify(struct virtqueue *vq)
0106 {
0107     struct vdpa_device *vdpa = vd_get_vdpa(vq->vdev);
0108     const struct vdpa_config_ops *ops = vdpa->config;
0109
0110     ops->kick_vq(vdpa, vq->index);
0111
0112     return true;
0113 }
0114
0115 static irqreturn_t virtio_vdpa_config_cb(void *private)
0116 {
0117     struct virtio_vdpa_device *vd_dev = private;
0118
0119     virtio_config_changed(&vd_dev->vdev);
0120
0121     return IRQ_HANDLED;
0122 }
0123
0124 static irqreturn_t virtio_vdpa_virtqueue_cb(void *private)
0125 {
0126     struct virtio_vdpa_vq_info *info = private;
0127
0128     return vring_interrupt(0, info->vq);
0129 }
0130
0131 static struct virtqueue *
0132 virtio_vdpa_setup_vq(struct virtio_device *vdev, unsigned int index,
0133              void (*callback)(struct virtqueue *vq),
0134              const char *name, bool ctx)
0135 {
0136     struct virtio_vdpa_device *vd_dev = to_virtio_vdpa_device(vdev);
0137     struct vdpa_device *vdpa = vd_get_vdpa(vdev);
0138     const struct vdpa_config_ops *ops = vdpa->config;
0139     struct virtio_vdpa_vq_info *info;
0140     struct vdpa_callback cb;
0141     struct virtqueue *vq;
0142     u64 desc_addr, driver_addr, device_addr;
0143     /* Assume split virtqueue, switch to packed if necessary */
0144     struct vdpa_vq_state state = {0};
0145     unsigned long flags;
0146     u32 align, max_num, min_num = 1;
0147     bool may_reduce_num = true;
0148     int err;
0149
0150     if (!name)
0151         return NULL;
0152
0153     if (index >= vdpa->nvqs)
0154         return ERR_PTR(-ENOENT);
0155
0156     /* Queue shouldn't already be set up. */
0157     if (ops->get_vq_ready(vdpa, index))
0158         return ERR_PTR(-ENOENT);
0159
0160     /* Allocate and fill out our active queue description */
0161     info = kmalloc(sizeof(*info), GFP_KERNEL);
0162     if (!info)
0163         return ERR_PTR(-ENOMEM);
0164
0165     max_num = ops->get_vq_num_max(vdpa);
0166     if (max_num == 0) {
0167         err = -ENOENT;
0168         goto error_new_virtqueue;
0169     }
0170
0171     if (ops->get_vq_num_min)
0172         min_num = ops->get_vq_num_min(vdpa);
0173
0174     may_reduce_num = (max_num == min_num) ? false : true;
0175
0176     /* Create the vring */
0177     align = ops->get_vq_align(vdpa);
0178     vq = vring_create_virtqueue(index, max_num, align, vdev,
0179                     true, may_reduce_num, ctx,
0180                     virtio_vdpa_notify, callback, name);
0181     if (!vq) {
0182         err = -ENOMEM;
0183         goto error_new_virtqueue;
0184     }
0185
0186     vq->num_max = max_num;
0187
0188     /* Setup virtqueue callback */
0189     cb.callback = callback ? virtio_vdpa_virtqueue_cb : NULL;
0190     cb.private = info;
0191     ops->set_vq_cb(vdpa, index, &cb);
0192     ops->set_vq_num(vdpa, index, virtqueue_get_vring_size(vq));
0193
0194     desc_addr = virtqueue_get_desc_addr(vq);
0195     driver_addr = virtqueue_get_avail_addr(vq);
0196     device_addr = virtqueue_get_used_addr(vq);
0197
0198     if (ops->set_vq_address(vdpa, index,
0199                 desc_addr, driver_addr,
0200                 device_addr)) {
0201         err = -EINVAL;
0202         goto err_vq;
0203     }
0204
0205     /* reset virtqueue state index */
0206     if (virtio_has_feature(vdev, VIRTIO_F_RING_PACKED)) {
0207         struct vdpa_vq_state_packed *s = &state.packed;
0208
0209         s->last_avail_counter = 1;
0210         s->last_avail_idx = 0;
0211         s->last_used_counter = 1;
0212         s->last_used_idx = 0;
0213     }
0214     err = ops->set_vq_state(vdpa, index, &state);
0215     if (err)
0216         goto err_vq;
0217
0218     ops->set_vq_ready(vdpa, index, 1);
0219
0220     vq->priv = info;
0221     info->vq = vq;
0222
0223     spin_lock_irqsave(&vd_dev->lock, flags);
0224     list_add(&info->node, &vd_dev->virtqueues);
0225     spin_unlock_irqrestore(&vd_dev->lock, flags);
0226
0227     return vq;
0228
0229 err_vq:
0230     vring_del_virtqueue(vq);
0231 error_new_virtqueue:
0232     ops->set_vq_ready(vdpa, index, 0);
0233     /* VDPA driver should make sure vq is stopeed here */
0234     WARN_ON(ops->get_vq_ready(vdpa, index));
0235     kfree(info);
0236     return ERR_PTR(err);
0237 }
0238
0239 static void virtio_vdpa_del_vq(struct virtqueue *vq)
0240 {
0241     struct virtio_vdpa_device *vd_dev = to_virtio_vdpa_device(vq->vdev);
0242     struct vdpa_device *vdpa = vd_dev->vdpa;
0243     const struct vdpa_config_ops *ops = vdpa->config;
0244     struct virtio_vdpa_vq_info *info = vq->priv;
0245     unsigned int index = vq->index;
0246     unsigned long flags;
0247
0248     spin_lock_irqsave(&vd_dev->lock, flags);
0249     list_del(&info->node);
0250     spin_unlock_irqrestore(&vd_dev->lock, flags);
0251
0252     /* Select and deactivate the queue (best effort) */
0253     ops->set_vq_ready(vdpa, index, 0);
0254
0255     vring_del_virtqueue(vq);
0256
0257     kfree(info);
0258 }
0259
0260 static void virtio_vdpa_del_vqs(struct virtio_device *vdev)
0261 {
0262     struct virtqueue *vq, *n;
0263
0264     list_for_each_entry_safe(vq, n, &vdev->vqs, list)
0265         virtio_vdpa_del_vq(vq);
0266 }
0267
0268 static int virtio_vdpa_find_vqs(struct virtio_device *vdev, unsigned int nvqs,
0269                 struct virtqueue *vqs[],
0270                 vq_callback_t *callbacks[],
0271                 const char * const names[],
0272                 const bool *ctx,
0273                 struct irq_affinity *desc)
0274 {
0275     struct virtio_vdpa_device *vd_dev = to_virtio_vdpa_device(vdev);
0276     struct vdpa_device *vdpa = vd_get_vdpa(vdev);
0277     const struct vdpa_config_ops *ops = vdpa->config;
0278     struct vdpa_callback cb;
0279     int i, err, queue_idx = 0;
0280
0281     for (i = 0; i < nvqs; ++i) {
0282         if (!names[i]) {
0283             vqs[i] = NULL;
0284             continue;
0285         }
0286
0287         vqs[i] = virtio_vdpa_setup_vq(vdev, queue_idx++,
0288                           callbacks[i], names[i], ctx ?
0289                           ctx[i] : false);
0290         if (IS_ERR(vqs[i])) {
0291             err = PTR_ERR(vqs[i]);
0292             goto err_setup_vq;
0293         }
0294     }
0295
0296     cb.callback = virtio_vdpa_config_cb;
0297     cb.private = vd_dev;
0298     ops->set_config_cb(vdpa, &cb);
0299
0300     return 0;
0301
0302 err_setup_vq:
0303     virtio_vdpa_del_vqs(vdev);
0304     return err;
0305 }
0306
0307 static u64 virtio_vdpa_get_features(struct virtio_device *vdev)
0308 {
0309     struct vdpa_device *vdpa = vd_get_vdpa(vdev);
0310     const struct vdpa_config_ops *ops = vdpa->config;
0311
0312     return ops->get_device_features(vdpa);
0313 }
0314
0315 static int virtio_vdpa_finalize_features(struct virtio_device *vdev)
0316 {
0317     struct vdpa_device *vdpa = vd_get_vdpa(vdev);
0318
0319     /* Give virtio_ring a chance to accept features. */
0320     vring_transport_features(vdev);
0321
0322     return vdpa_set_features(vdpa, vdev->features);
0323 }
0324
0325 static const char *virtio_vdpa_bus_name(struct virtio_device *vdev)
0326 {
0327     struct virtio_vdpa_device *vd_dev = to_virtio_vdpa_device(vdev);
0328     struct vdpa_device *vdpa = vd_dev->vdpa;
0329
0330     return dev_name(&vdpa->dev);
0331 }
0332
0333 static const struct virtio_config_ops virtio_vdpa_config_ops = {
0334     .get        = virtio_vdpa_get,
0335     .set        = virtio_vdpa_set,
0336     .generation = virtio_vdpa_generation,
0337     .get_status = virtio_vdpa_get_status,
0338     .set_status = virtio_vdpa_set_status,
0339     .reset      = virtio_vdpa_reset,
0340     .find_vqs   = virtio_vdpa_find_vqs,
0341     .del_vqs    = virtio_vdpa_del_vqs,
0342     .get_features   = virtio_vdpa_get_features,
0343     .finalize_features = virtio_vdpa_finalize_features,
0344     .bus_name   = virtio_vdpa_bus_name,
0345 };
0346
0347 static void virtio_vdpa_release_dev(struct device *_d)
0348 {
0349     struct virtio_device *vdev =
0350            container_of(_d, struct virtio_device, dev);
0351     struct virtio_vdpa_device *vd_dev =
0352            container_of(vdev, struct virtio_vdpa_device, vdev);
0353
0354     kfree(vd_dev);
0355 }
0356
0357 static int virtio_vdpa_probe(struct vdpa_device *vdpa)
0358 {
0359     const struct vdpa_config_ops *ops = vdpa->config;
0360     struct virtio_vdpa_device *vd_dev, *reg_dev = NULL;
0361     int ret = -EINVAL;
0362
0363     vd_dev = kzalloc(sizeof(*vd_dev), GFP_KERNEL);
0364     if (!vd_dev)
0365         return -ENOMEM;
0366
0367     vd_dev->vdev.dev.parent = vdpa_get_dma_dev(vdpa);
0368     vd_dev->vdev.dev.release = virtio_vdpa_release_dev;
0369     vd_dev->vdev.config = &virtio_vdpa_config_ops;
0370     vd_dev->vdpa = vdpa;
0371     INIT_LIST_HEAD(&vd_dev->virtqueues);
0372     spin_lock_init(&vd_dev->lock);
0373
0374     vd_dev->vdev.id.device = ops->get_device_id(vdpa);
0375     if (vd_dev->vdev.id.device == 0)
0376         goto err;
0377
0378     vd_dev->vdev.id.vendor = ops->get_vendor_id(vdpa);
0379     ret = register_virtio_device(&vd_dev->vdev);
0380     reg_dev = vd_dev;
0381     if (ret)
0382         goto err;
0383
0384     vdpa_set_drvdata(vdpa, vd_dev);
0385
0386     return 0;
0387
0388 err:
0389     if (reg_dev)
0390         put_device(&vd_dev->vdev.dev);
0391     else
0392         kfree(vd_dev);
0393     return ret;
0394 }
0395
0396 static void virtio_vdpa_remove(struct vdpa_device *vdpa)
0397 {
0398     struct virtio_vdpa_device *vd_dev = vdpa_get_drvdata(vdpa);
0399
0400     unregister_virtio_device(&vd_dev->vdev);
0401 }
0402
0403 static struct vdpa_driver virtio_vdpa_driver = {
0404     .driver = {
0405         .name   = "virtio_vdpa",
0406     },
0407     .probe  = virtio_vdpa_probe,
0408     .remove = virtio_vdpa_remove,
0409 };
0410
0411 module_vdpa_driver(virtio_vdpa_driver);
0412
0413 MODULE_VERSION(MOD_VERSION);
0414 MODULE_LICENSE(MOD_LICENSE);
0415 MODULE_AUTHOR(MOD_AUTHOR);
0416 MODULE_DESCRIPTION(MOD_DESC);