Back to home page

OSCL-LXR
 
 

     

0001 // SPDX-License-Identifier: GPL-2.0-or-later
0002 /*
0003  * Virtio PCI driver - modern (virtio 1.0) device support
0004  *
0005  * This module allows virtio devices to be used over a virtual PCI device.
0006  * This can be used with QEMU based VMMs like KVM or Xen.
0007  *
0008  * Copyright IBM Corp. 2007
0009  * Copyright Red Hat, Inc. 2014
0010  *
0011  * Authors:
0012  *  Anthony Liguori  <aliguori@us.ibm.com>
0013  *  Rusty Russell <rusty@rustcorp.com.au>
0014  *  Michael S. Tsirkin <mst@redhat.com>
0015  */
0016 
0017 #include <linux/delay.h>
0018 #define VIRTIO_PCI_NO_LEGACY
0019 #define VIRTIO_RING_NO_LEGACY
0020 #include "virtio_pci_common.h"
0021 
0022 static u64 vp_get_features(struct virtio_device *vdev)
0023 {
0024     struct virtio_pci_device *vp_dev = to_vp_device(vdev);
0025 
0026     return vp_modern_get_features(&vp_dev->mdev);
0027 }
0028 
0029 static void vp_transport_features(struct virtio_device *vdev, u64 features)
0030 {
0031     struct virtio_pci_device *vp_dev = to_vp_device(vdev);
0032     struct pci_dev *pci_dev = vp_dev->pci_dev;
0033 
0034     if ((features & BIT_ULL(VIRTIO_F_SR_IOV)) &&
0035             pci_find_ext_capability(pci_dev, PCI_EXT_CAP_ID_SRIOV))
0036         __virtio_set_bit(vdev, VIRTIO_F_SR_IOV);
0037 
0038     if (features & BIT_ULL(VIRTIO_F_RING_RESET))
0039         __virtio_set_bit(vdev, VIRTIO_F_RING_RESET);
0040 }
0041 
0042 /* virtio config->finalize_features() implementation */
0043 static int vp_finalize_features(struct virtio_device *vdev)
0044 {
0045     struct virtio_pci_device *vp_dev = to_vp_device(vdev);
0046     u64 features = vdev->features;
0047 
0048     /* Give virtio_ring a chance to accept features. */
0049     vring_transport_features(vdev);
0050 
0051     /* Give virtio_pci a chance to accept features. */
0052     vp_transport_features(vdev, features);
0053 
0054     if (!__virtio_test_bit(vdev, VIRTIO_F_VERSION_1)) {
0055         dev_err(&vdev->dev, "virtio: device uses modern interface "
0056             "but does not have VIRTIO_F_VERSION_1\n");
0057         return -EINVAL;
0058     }
0059 
0060     vp_modern_set_features(&vp_dev->mdev, vdev->features);
0061 
0062     return 0;
0063 }
0064 
0065 /* virtio config->get() implementation */
0066 static void vp_get(struct virtio_device *vdev, unsigned int offset,
0067            void *buf, unsigned int len)
0068 {
0069     struct virtio_pci_device *vp_dev = to_vp_device(vdev);
0070     struct virtio_pci_modern_device *mdev = &vp_dev->mdev;
0071     void __iomem *device = mdev->device;
0072     u8 b;
0073     __le16 w;
0074     __le32 l;
0075 
0076     BUG_ON(offset + len > mdev->device_len);
0077 
0078     switch (len) {
0079     case 1:
0080         b = ioread8(device + offset);
0081         memcpy(buf, &b, sizeof b);
0082         break;
0083     case 2:
0084         w = cpu_to_le16(ioread16(device + offset));
0085         memcpy(buf, &w, sizeof w);
0086         break;
0087     case 4:
0088         l = cpu_to_le32(ioread32(device + offset));
0089         memcpy(buf, &l, sizeof l);
0090         break;
0091     case 8:
0092         l = cpu_to_le32(ioread32(device + offset));
0093         memcpy(buf, &l, sizeof l);
0094         l = cpu_to_le32(ioread32(device + offset + sizeof l));
0095         memcpy(buf + sizeof l, &l, sizeof l);
0096         break;
0097     default:
0098         BUG();
0099     }
0100 }
0101 
0102 /* the config->set() implementation.  it's symmetric to the config->get()
0103  * implementation */
0104 static void vp_set(struct virtio_device *vdev, unsigned int offset,
0105            const void *buf, unsigned int len)
0106 {
0107     struct virtio_pci_device *vp_dev = to_vp_device(vdev);
0108     struct virtio_pci_modern_device *mdev = &vp_dev->mdev;
0109     void __iomem *device = mdev->device;
0110     u8 b;
0111     __le16 w;
0112     __le32 l;
0113 
0114     BUG_ON(offset + len > mdev->device_len);
0115 
0116     switch (len) {
0117     case 1:
0118         memcpy(&b, buf, sizeof b);
0119         iowrite8(b, device + offset);
0120         break;
0121     case 2:
0122         memcpy(&w, buf, sizeof w);
0123         iowrite16(le16_to_cpu(w), device + offset);
0124         break;
0125     case 4:
0126         memcpy(&l, buf, sizeof l);
0127         iowrite32(le32_to_cpu(l), device + offset);
0128         break;
0129     case 8:
0130         memcpy(&l, buf, sizeof l);
0131         iowrite32(le32_to_cpu(l), device + offset);
0132         memcpy(&l, buf + sizeof l, sizeof l);
0133         iowrite32(le32_to_cpu(l), device + offset + sizeof l);
0134         break;
0135     default:
0136         BUG();
0137     }
0138 }
0139 
0140 static u32 vp_generation(struct virtio_device *vdev)
0141 {
0142     struct virtio_pci_device *vp_dev = to_vp_device(vdev);
0143 
0144     return vp_modern_generation(&vp_dev->mdev);
0145 }
0146 
0147 /* config->{get,set}_status() implementations */
0148 static u8 vp_get_status(struct virtio_device *vdev)
0149 {
0150     struct virtio_pci_device *vp_dev = to_vp_device(vdev);
0151 
0152     return vp_modern_get_status(&vp_dev->mdev);
0153 }
0154 
0155 static void vp_set_status(struct virtio_device *vdev, u8 status)
0156 {
0157     struct virtio_pci_device *vp_dev = to_vp_device(vdev);
0158 
0159     /* We should never be setting status to 0. */
0160     BUG_ON(status == 0);
0161     vp_modern_set_status(&vp_dev->mdev, status);
0162 }
0163 
0164 static void vp_reset(struct virtio_device *vdev)
0165 {
0166     struct virtio_pci_device *vp_dev = to_vp_device(vdev);
0167     struct virtio_pci_modern_device *mdev = &vp_dev->mdev;
0168 
0169     /* 0 status means a reset. */
0170     vp_modern_set_status(mdev, 0);
0171     /* After writing 0 to device_status, the driver MUST wait for a read of
0172      * device_status to return 0 before reinitializing the device.
0173      * This will flush out the status write, and flush in device writes,
0174      * including MSI-X interrupts, if any.
0175      */
0176     while (vp_modern_get_status(mdev))
0177         msleep(1);
0178     /* Flush pending VQ/configuration callbacks. */
0179     vp_synchronize_vectors(vdev);
0180 }
0181 
0182 static int vp_active_vq(struct virtqueue *vq, u16 msix_vec)
0183 {
0184     struct virtio_pci_device *vp_dev = to_vp_device(vq->vdev);
0185     struct virtio_pci_modern_device *mdev = &vp_dev->mdev;
0186     unsigned long index;
0187 
0188     index = vq->index;
0189 
0190     /* activate the queue */
0191     vp_modern_set_queue_size(mdev, index, virtqueue_get_vring_size(vq));
0192     vp_modern_queue_address(mdev, index, virtqueue_get_desc_addr(vq),
0193                 virtqueue_get_avail_addr(vq),
0194                 virtqueue_get_used_addr(vq));
0195 
0196     if (msix_vec != VIRTIO_MSI_NO_VECTOR) {
0197         msix_vec = vp_modern_queue_vector(mdev, index, msix_vec);
0198         if (msix_vec == VIRTIO_MSI_NO_VECTOR)
0199             return -EBUSY;
0200     }
0201 
0202     return 0;
0203 }
0204 
0205 static int vp_modern_disable_vq_and_reset(struct virtqueue *vq)
0206 {
0207     struct virtio_pci_device *vp_dev = to_vp_device(vq->vdev);
0208     struct virtio_pci_modern_device *mdev = &vp_dev->mdev;
0209     struct virtio_pci_vq_info *info;
0210     unsigned long flags;
0211 
0212     if (!virtio_has_feature(vq->vdev, VIRTIO_F_RING_RESET))
0213         return -ENOENT;
0214 
0215     vp_modern_set_queue_reset(mdev, vq->index);
0216 
0217     info = vp_dev->vqs[vq->index];
0218 
0219     /* delete vq from irq handler */
0220     spin_lock_irqsave(&vp_dev->lock, flags);
0221     list_del(&info->node);
0222     spin_unlock_irqrestore(&vp_dev->lock, flags);
0223 
0224     INIT_LIST_HEAD(&info->node);
0225 
0226 #ifdef CONFIG_VIRTIO_HARDEN_NOTIFICATION
0227     __virtqueue_break(vq);
0228 #endif
0229 
0230     /* For the case where vq has an exclusive irq, call synchronize_irq() to
0231      * wait for completion.
0232      *
0233      * note: We can't use disable_irq() since it conflicts with the affinity
0234      * managed IRQ that is used by some drivers.
0235      */
0236     if (vp_dev->per_vq_vectors && info->msix_vector != VIRTIO_MSI_NO_VECTOR)
0237         synchronize_irq(pci_irq_vector(vp_dev->pci_dev, info->msix_vector));
0238 
0239     vq->reset = true;
0240 
0241     return 0;
0242 }
0243 
0244 static int vp_modern_enable_vq_after_reset(struct virtqueue *vq)
0245 {
0246     struct virtio_pci_device *vp_dev = to_vp_device(vq->vdev);
0247     struct virtio_pci_modern_device *mdev = &vp_dev->mdev;
0248     struct virtio_pci_vq_info *info;
0249     unsigned long flags, index;
0250     int err;
0251 
0252     if (!vq->reset)
0253         return -EBUSY;
0254 
0255     index = vq->index;
0256     info = vp_dev->vqs[index];
0257 
0258     if (vp_modern_get_queue_reset(mdev, index))
0259         return -EBUSY;
0260 
0261     if (vp_modern_get_queue_enable(mdev, index))
0262         return -EBUSY;
0263 
0264     err = vp_active_vq(vq, info->msix_vector);
0265     if (err)
0266         return err;
0267 
0268     if (vq->callback) {
0269         spin_lock_irqsave(&vp_dev->lock, flags);
0270         list_add(&info->node, &vp_dev->virtqueues);
0271         spin_unlock_irqrestore(&vp_dev->lock, flags);
0272     } else {
0273         INIT_LIST_HEAD(&info->node);
0274     }
0275 
0276 #ifdef CONFIG_VIRTIO_HARDEN_NOTIFICATION
0277     __virtqueue_unbreak(vq);
0278 #endif
0279 
0280     vp_modern_set_queue_enable(&vp_dev->mdev, index, true);
0281     vq->reset = false;
0282 
0283     return 0;
0284 }
0285 
0286 static u16 vp_config_vector(struct virtio_pci_device *vp_dev, u16 vector)
0287 {
0288     return vp_modern_config_vector(&vp_dev->mdev, vector);
0289 }
0290 
0291 static struct virtqueue *setup_vq(struct virtio_pci_device *vp_dev,
0292                   struct virtio_pci_vq_info *info,
0293                   unsigned int index,
0294                   void (*callback)(struct virtqueue *vq),
0295                   const char *name,
0296                   bool ctx,
0297                   u16 msix_vec)
0298 {
0299 
0300     struct virtio_pci_modern_device *mdev = &vp_dev->mdev;
0301     struct virtqueue *vq;
0302     u16 num;
0303     int err;
0304 
0305     if (index >= vp_modern_get_num_queues(mdev))
0306         return ERR_PTR(-ENOENT);
0307 
0308     /* Check if queue is either not available or already active. */
0309     num = vp_modern_get_queue_size(mdev, index);
0310     if (!num || vp_modern_get_queue_enable(mdev, index))
0311         return ERR_PTR(-ENOENT);
0312 
0313     if (num & (num - 1)) {
0314         dev_warn(&vp_dev->pci_dev->dev, "bad queue size %u", num);
0315         return ERR_PTR(-EINVAL);
0316     }
0317 
0318     info->msix_vector = msix_vec;
0319 
0320     /* create the vring */
0321     vq = vring_create_virtqueue(index, num,
0322                     SMP_CACHE_BYTES, &vp_dev->vdev,
0323                     true, true, ctx,
0324                     vp_notify, callback, name);
0325     if (!vq)
0326         return ERR_PTR(-ENOMEM);
0327 
0328     vq->num_max = num;
0329 
0330     err = vp_active_vq(vq, msix_vec);
0331     if (err)
0332         goto err;
0333 
0334     vq->priv = (void __force *)vp_modern_map_vq_notify(mdev, index, NULL);
0335     if (!vq->priv) {
0336         err = -ENOMEM;
0337         goto err;
0338     }
0339 
0340     return vq;
0341 
0342 err:
0343     vring_del_virtqueue(vq);
0344     return ERR_PTR(err);
0345 }
0346 
0347 static int vp_modern_find_vqs(struct virtio_device *vdev, unsigned int nvqs,
0348                   struct virtqueue *vqs[],
0349                   vq_callback_t *callbacks[],
0350                   const char * const names[], const bool *ctx,
0351                   struct irq_affinity *desc)
0352 {
0353     struct virtio_pci_device *vp_dev = to_vp_device(vdev);
0354     struct virtqueue *vq;
0355     int rc = vp_find_vqs(vdev, nvqs, vqs, callbacks, names, ctx, desc);
0356 
0357     if (rc)
0358         return rc;
0359 
0360     /* Select and activate all queues. Has to be done last: once we do
0361      * this, there's no way to go back except reset.
0362      */
0363     list_for_each_entry(vq, &vdev->vqs, list)
0364         vp_modern_set_queue_enable(&vp_dev->mdev, vq->index, true);
0365 
0366     return 0;
0367 }
0368 
0369 static void del_vq(struct virtio_pci_vq_info *info)
0370 {
0371     struct virtqueue *vq = info->vq;
0372     struct virtio_pci_device *vp_dev = to_vp_device(vq->vdev);
0373     struct virtio_pci_modern_device *mdev = &vp_dev->mdev;
0374 
0375     if (vp_dev->msix_enabled)
0376         vp_modern_queue_vector(mdev, vq->index,
0377                        VIRTIO_MSI_NO_VECTOR);
0378 
0379     if (!mdev->notify_base)
0380         pci_iounmap(mdev->pci_dev, (void __force __iomem *)vq->priv);
0381 
0382     vring_del_virtqueue(vq);
0383 }
0384 
0385 static int virtio_pci_find_shm_cap(struct pci_dev *dev, u8 required_id,
0386                    u8 *bar, u64 *offset, u64 *len)
0387 {
0388     int pos;
0389 
0390     for (pos = pci_find_capability(dev, PCI_CAP_ID_VNDR); pos > 0;
0391          pos = pci_find_next_capability(dev, pos, PCI_CAP_ID_VNDR)) {
0392         u8 type, cap_len, id, res_bar;
0393         u32 tmp32;
0394         u64 res_offset, res_length;
0395 
0396         pci_read_config_byte(dev, pos + offsetof(struct virtio_pci_cap,
0397                              cfg_type), &type);
0398         if (type != VIRTIO_PCI_CAP_SHARED_MEMORY_CFG)
0399             continue;
0400 
0401         pci_read_config_byte(dev, pos + offsetof(struct virtio_pci_cap,
0402                              cap_len), &cap_len);
0403         if (cap_len != sizeof(struct virtio_pci_cap64)) {
0404             dev_err(&dev->dev, "%s: shm cap with bad size offset:"
0405                 " %d size: %d\n", __func__, pos, cap_len);
0406             continue;
0407         }
0408 
0409         pci_read_config_byte(dev, pos + offsetof(struct virtio_pci_cap,
0410                              id), &id);
0411         if (id != required_id)
0412             continue;
0413 
0414         pci_read_config_byte(dev, pos + offsetof(struct virtio_pci_cap,
0415                              bar), &res_bar);
0416         if (res_bar >= PCI_STD_NUM_BARS)
0417             continue;
0418 
0419         /* Type and ID match, and the BAR value isn't reserved.
0420          * Looks good.
0421          */
0422 
0423         /* Read the lower 32bit of length and offset */
0424         pci_read_config_dword(dev, pos + offsetof(struct virtio_pci_cap,
0425                               offset), &tmp32);
0426         res_offset = tmp32;
0427         pci_read_config_dword(dev, pos + offsetof(struct virtio_pci_cap,
0428                               length), &tmp32);
0429         res_length = tmp32;
0430 
0431         /* and now the top half */
0432         pci_read_config_dword(dev,
0433                       pos + offsetof(struct virtio_pci_cap64,
0434                              offset_hi), &tmp32);
0435         res_offset |= ((u64)tmp32) << 32;
0436         pci_read_config_dword(dev,
0437                       pos + offsetof(struct virtio_pci_cap64,
0438                              length_hi), &tmp32);
0439         res_length |= ((u64)tmp32) << 32;
0440 
0441         *bar = res_bar;
0442         *offset = res_offset;
0443         *len = res_length;
0444 
0445         return pos;
0446     }
0447     return 0;
0448 }
0449 
0450 static bool vp_get_shm_region(struct virtio_device *vdev,
0451                   struct virtio_shm_region *region, u8 id)
0452 {
0453     struct virtio_pci_device *vp_dev = to_vp_device(vdev);
0454     struct pci_dev *pci_dev = vp_dev->pci_dev;
0455     u8 bar;
0456     u64 offset, len;
0457     phys_addr_t phys_addr;
0458     size_t bar_len;
0459 
0460     if (!virtio_pci_find_shm_cap(pci_dev, id, &bar, &offset, &len))
0461         return false;
0462 
0463     phys_addr = pci_resource_start(pci_dev, bar);
0464     bar_len = pci_resource_len(pci_dev, bar);
0465 
0466     if ((offset + len) < offset) {
0467         dev_err(&pci_dev->dev, "%s: cap offset+len overflow detected\n",
0468             __func__);
0469         return false;
0470     }
0471 
0472     if (offset + len > bar_len) {
0473         dev_err(&pci_dev->dev, "%s: bar shorter than cap offset+len\n",
0474             __func__);
0475         return false;
0476     }
0477 
0478     region->len = len;
0479     region->addr = (u64) phys_addr + offset;
0480 
0481     return true;
0482 }
0483 
0484 static const struct virtio_config_ops virtio_pci_config_nodev_ops = {
0485     .get        = NULL,
0486     .set        = NULL,
0487     .generation = vp_generation,
0488     .get_status = vp_get_status,
0489     .set_status = vp_set_status,
0490     .reset      = vp_reset,
0491     .find_vqs   = vp_modern_find_vqs,
0492     .del_vqs    = vp_del_vqs,
0493     .synchronize_cbs = vp_synchronize_vectors,
0494     .get_features   = vp_get_features,
0495     .finalize_features = vp_finalize_features,
0496     .bus_name   = vp_bus_name,
0497     .set_vq_affinity = vp_set_vq_affinity,
0498     .get_vq_affinity = vp_get_vq_affinity,
0499     .get_shm_region  = vp_get_shm_region,
0500     .disable_vq_and_reset = vp_modern_disable_vq_and_reset,
0501     .enable_vq_after_reset = vp_modern_enable_vq_after_reset,
0502 };
0503 
0504 static const struct virtio_config_ops virtio_pci_config_ops = {
0505     .get        = vp_get,
0506     .set        = vp_set,
0507     .generation = vp_generation,
0508     .get_status = vp_get_status,
0509     .set_status = vp_set_status,
0510     .reset      = vp_reset,
0511     .find_vqs   = vp_modern_find_vqs,
0512     .del_vqs    = vp_del_vqs,
0513     .synchronize_cbs = vp_synchronize_vectors,
0514     .get_features   = vp_get_features,
0515     .finalize_features = vp_finalize_features,
0516     .bus_name   = vp_bus_name,
0517     .set_vq_affinity = vp_set_vq_affinity,
0518     .get_vq_affinity = vp_get_vq_affinity,
0519     .get_shm_region  = vp_get_shm_region,
0520     .disable_vq_and_reset = vp_modern_disable_vq_and_reset,
0521     .enable_vq_after_reset = vp_modern_enable_vq_after_reset,
0522 };
0523 
0524 /* the PCI probing function */
0525 int virtio_pci_modern_probe(struct virtio_pci_device *vp_dev)
0526 {
0527     struct virtio_pci_modern_device *mdev = &vp_dev->mdev;
0528     struct pci_dev *pci_dev = vp_dev->pci_dev;
0529     int err;
0530 
0531     mdev->pci_dev = pci_dev;
0532 
0533     err = vp_modern_probe(mdev);
0534     if (err)
0535         return err;
0536 
0537     if (mdev->device)
0538         vp_dev->vdev.config = &virtio_pci_config_ops;
0539     else
0540         vp_dev->vdev.config = &virtio_pci_config_nodev_ops;
0541 
0542     vp_dev->config_vector = vp_config_vector;
0543     vp_dev->setup_vq = setup_vq;
0544     vp_dev->del_vq = del_vq;
0545     vp_dev->isr = mdev->isr;
0546     vp_dev->vdev.id = mdev->id;
0547 
0548     return 0;
0549 }
0550 
0551 void virtio_pci_modern_remove(struct virtio_pci_device *vp_dev)
0552 {
0553     struct virtio_pci_modern_device *mdev = &vp_dev->mdev;
0554 
0555     vp_modern_remove(mdev);
0556 }
[ Source navigation ] [ Diff markup ] [ Identifier search ] [ general search ]

This page was automatically generated by the 2.1.0 LXR engine.

The LXR team
Valid CSS 2.1! Valid HTML 4.01!