vfio/pci/vfio_pci_rdwr.c

0001 // SPDX-License-Identifier: GPL-2.0-only
0002 /*
0003  * VFIO PCI I/O Port & MMIO access
0004  *
0005  * Copyright (C) 2012 Red Hat, Inc.  All rights reserved.
0006  *     Author: Alex Williamson <alex.williamson@redhat.com>
0007  *
0008  * Derived from original vfio:
0009  * Copyright 2010 Cisco Systems, Inc.  All rights reserved.
0010  * Author: Tom Lyon, pugs@cisco.com
0011  */
0012
0013 #include <linux/fs.h>
0014 #include <linux/pci.h>
0015 #include <linux/uaccess.h>
0016 #include <linux/io.h>
0017 #include <linux/vfio.h>
0018 #include <linux/vgaarb.h>
0019
0020 #include <linux/vfio_pci_core.h>
0021
0022 #ifdef __LITTLE_ENDIAN
0023 #define vfio_ioread64   ioread64
0024 #define vfio_iowrite64  iowrite64
0025 #define vfio_ioread32   ioread32
0026 #define vfio_iowrite32  iowrite32
0027 #define vfio_ioread16   ioread16
0028 #define vfio_iowrite16  iowrite16
0029 #else
0030 #define vfio_ioread64   ioread64be
0031 #define vfio_iowrite64  iowrite64be
0032 #define vfio_ioread32   ioread32be
0033 #define vfio_iowrite32  iowrite32be
0034 #define vfio_ioread16   ioread16be
0035 #define vfio_iowrite16  iowrite16be
0036 #endif
0037 #define vfio_ioread8    ioread8
0038 #define vfio_iowrite8   iowrite8
0039
0040 #define VFIO_IOWRITE(size) \
0041 static int vfio_pci_iowrite##size(struct vfio_pci_core_device *vdev,        \
0042             bool test_mem, u##size val, void __iomem *io)   \
0043 {                                   \
0044     if (test_mem) {                         \
0045         down_read(&vdev->memory_lock);              \
0046         if (!__vfio_pci_memory_enabled(vdev)) {         \
0047             up_read(&vdev->memory_lock);            \
0048             return -EIO;                    \
0049         }                           \
0050     }                               \
0051                                     \
0052     vfio_iowrite##size(val, io);                    \
0053                                     \
0054     if (test_mem)                           \
0055         up_read(&vdev->memory_lock);                \
0056                                     \
0057     return 0;                           \
0058 }
0059
0060 VFIO_IOWRITE(8)
0061 VFIO_IOWRITE(16)
0062 VFIO_IOWRITE(32)
0063 #ifdef iowrite64
0064 VFIO_IOWRITE(64)
0065 #endif
0066
0067 #define VFIO_IOREAD(size) \
0068 static int vfio_pci_ioread##size(struct vfio_pci_core_device *vdev,     \
0069             bool test_mem, u##size *val, void __iomem *io)  \
0070 {                                   \
0071     if (test_mem) {                         \
0072         down_read(&vdev->memory_lock);              \
0073         if (!__vfio_pci_memory_enabled(vdev)) {         \
0074             up_read(&vdev->memory_lock);            \
0075             return -EIO;                    \
0076         }                           \
0077     }                               \
0078                                     \
0079     *val = vfio_ioread##size(io);                   \
0080                                     \
0081     if (test_mem)                           \
0082         up_read(&vdev->memory_lock);                \
0083                                     \
0084     return 0;                           \
0085 }
0086
0087 VFIO_IOREAD(8)
0088 VFIO_IOREAD(16)
0089 VFIO_IOREAD(32)
0090
0091 /*
0092  * Read or write from an __iomem region (MMIO or I/O port) with an excluded
0093  * range which is inaccessible.  The excluded range drops writes and fills
0094  * reads with -1.  This is intended for handling MSI-X vector tables and
0095  * leftover space for ROM BARs.
0096  */
0097 static ssize_t do_io_rw(struct vfio_pci_core_device *vdev, bool test_mem,
0098             void __iomem *io, char __user *buf,
0099             loff_t off, size_t count, size_t x_start,
0100             size_t x_end, bool iswrite)
0101 {
0102     ssize_t done = 0;
0103     int ret;
0104
0105     while (count) {
0106         size_t fillable, filled;
0107
0108         if (off < x_start)
0109             fillable = min(count, (size_t)(x_start - off));
0110         else if (off >= x_end)
0111             fillable = count;
0112         else
0113             fillable = 0;
0114
0115         if (fillable >= 4 && !(off % 4)) {
0116             u32 val;
0117
0118             if (iswrite) {
0119                 if (copy_from_user(&val, buf, 4))
0120                     return -EFAULT;
0121
0122                 ret = vfio_pci_iowrite32(vdev, test_mem,
0123                              val, io + off);
0124                 if (ret)
0125                     return ret;
0126             } else {
0127                 ret = vfio_pci_ioread32(vdev, test_mem,
0128                             &val, io + off);
0129                 if (ret)
0130                     return ret;
0131
0132                 if (copy_to_user(buf, &val, 4))
0133                     return -EFAULT;
0134             }
0135
0136             filled = 4;
0137         } else if (fillable >= 2 && !(off % 2)) {
0138             u16 val;
0139
0140             if (iswrite) {
0141                 if (copy_from_user(&val, buf, 2))
0142                     return -EFAULT;
0143
0144                 ret = vfio_pci_iowrite16(vdev, test_mem,
0145                              val, io + off);
0146                 if (ret)
0147                     return ret;
0148             } else {
0149                 ret = vfio_pci_ioread16(vdev, test_mem,
0150                             &val, io + off);
0151                 if (ret)
0152                     return ret;
0153
0154                 if (copy_to_user(buf, &val, 2))
0155                     return -EFAULT;
0156             }
0157
0158             filled = 2;
0159         } else if (fillable) {
0160             u8 val;
0161
0162             if (iswrite) {
0163                 if (copy_from_user(&val, buf, 1))
0164                     return -EFAULT;
0165
0166                 ret = vfio_pci_iowrite8(vdev, test_mem,
0167                             val, io + off);
0168                 if (ret)
0169                     return ret;
0170             } else {
0171                 ret = vfio_pci_ioread8(vdev, test_mem,
0172                                &val, io + off);
0173                 if (ret)
0174                     return ret;
0175
0176                 if (copy_to_user(buf, &val, 1))
0177                     return -EFAULT;
0178             }
0179
0180             filled = 1;
0181         } else {
0182             /* Fill reads with -1, drop writes */
0183             filled = min(count, (size_t)(x_end - off));
0184             if (!iswrite) {
0185                 u8 val = 0xFF;
0186                 size_t i;
0187
0188                 for (i = 0; i < filled; i++)
0189                     if (copy_to_user(buf + i, &val, 1))
0190                         return -EFAULT;
0191             }
0192         }
0193
0194         count -= filled;
0195         done += filled;
0196         off += filled;
0197         buf += filled;
0198     }
0199
0200     return done;
0201 }
0202
0203 static int vfio_pci_setup_barmap(struct vfio_pci_core_device *vdev, int bar)
0204 {
0205     struct pci_dev *pdev = vdev->pdev;
0206     int ret;
0207     void __iomem *io;
0208
0209     if (vdev->barmap[bar])
0210         return 0;
0211
0212     ret = pci_request_selected_regions(pdev, 1 << bar, "vfio");
0213     if (ret)
0214         return ret;
0215
0216     io = pci_iomap(pdev, bar, 0);
0217     if (!io) {
0218         pci_release_selected_regions(pdev, 1 << bar);
0219         return -ENOMEM;
0220     }
0221
0222     vdev->barmap[bar] = io;
0223
0224     return 0;
0225 }
0226
0227 ssize_t vfio_pci_bar_rw(struct vfio_pci_core_device *vdev, char __user *buf,
0228             size_t count, loff_t *ppos, bool iswrite)
0229 {
0230     struct pci_dev *pdev = vdev->pdev;
0231     loff_t pos = *ppos & VFIO_PCI_OFFSET_MASK;
0232     int bar = VFIO_PCI_OFFSET_TO_INDEX(*ppos);
0233     size_t x_start = 0, x_end = 0;
0234     resource_size_t end;
0235     void __iomem *io;
0236     struct resource *res = &vdev->pdev->resource[bar];
0237     ssize_t done;
0238
0239     if (pci_resource_start(pdev, bar))
0240         end = pci_resource_len(pdev, bar);
0241     else if (bar == PCI_ROM_RESOURCE &&
0242          pdev->resource[bar].flags & IORESOURCE_ROM_SHADOW)
0243         end = 0x20000;
0244     else
0245         return -EINVAL;
0246
0247     if (pos >= end)
0248         return -EINVAL;
0249
0250     count = min(count, (size_t)(end - pos));
0251
0252     if (bar == PCI_ROM_RESOURCE) {
0253         /*
0254          * The ROM can fill less space than the BAR, so we start the
0255          * excluded range at the end of the actual ROM.  This makes
0256          * filling large ROM BARs much faster.
0257          */
0258         io = pci_map_rom(pdev, &x_start);
0259         if (!io) {
0260             done = -ENOMEM;
0261             goto out;
0262         }
0263         x_end = end;
0264     } else {
0265         int ret = vfio_pci_setup_barmap(vdev, bar);
0266         if (ret) {
0267             done = ret;
0268             goto out;
0269         }
0270
0271         io = vdev->barmap[bar];
0272     }
0273
0274     if (bar == vdev->msix_bar) {
0275         x_start = vdev->msix_offset;
0276         x_end = vdev->msix_offset + vdev->msix_size;
0277     }
0278
0279     done = do_io_rw(vdev, res->flags & IORESOURCE_MEM, io, buf, pos,
0280             count, x_start, x_end, iswrite);
0281
0282     if (done >= 0)
0283         *ppos += done;
0284
0285     if (bar == PCI_ROM_RESOURCE)
0286         pci_unmap_rom(pdev, io);
0287 out:
0288     return done;
0289 }
0290
0291 #ifdef CONFIG_VFIO_PCI_VGA
0292 ssize_t vfio_pci_vga_rw(struct vfio_pci_core_device *vdev, char __user *buf,
0293                    size_t count, loff_t *ppos, bool iswrite)
0294 {
0295     int ret;
0296     loff_t off, pos = *ppos & VFIO_PCI_OFFSET_MASK;
0297     void __iomem *iomem = NULL;
0298     unsigned int rsrc;
0299     bool is_ioport;
0300     ssize_t done;
0301
0302     if (!vdev->has_vga)
0303         return -EINVAL;
0304
0305     if (pos > 0xbfffful)
0306         return -EINVAL;
0307
0308     switch ((u32)pos) {
0309     case 0xa0000 ... 0xbffff:
0310         count = min(count, (size_t)(0xc0000 - pos));
0311         iomem = ioremap(0xa0000, 0xbffff - 0xa0000 + 1);
0312         off = pos - 0xa0000;
0313         rsrc = VGA_RSRC_LEGACY_MEM;
0314         is_ioport = false;
0315         break;
0316     case 0x3b0 ... 0x3bb:
0317         count = min(count, (size_t)(0x3bc - pos));
0318         iomem = ioport_map(0x3b0, 0x3bb - 0x3b0 + 1);
0319         off = pos - 0x3b0;
0320         rsrc = VGA_RSRC_LEGACY_IO;
0321         is_ioport = true;
0322         break;
0323     case 0x3c0 ... 0x3df:
0324         count = min(count, (size_t)(0x3e0 - pos));
0325         iomem = ioport_map(0x3c0, 0x3df - 0x3c0 + 1);
0326         off = pos - 0x3c0;
0327         rsrc = VGA_RSRC_LEGACY_IO;
0328         is_ioport = true;
0329         break;
0330     default:
0331         return -EINVAL;
0332     }
0333
0334     if (!iomem)
0335         return -ENOMEM;
0336
0337     ret = vga_get_interruptible(vdev->pdev, rsrc);
0338     if (ret) {
0339         is_ioport ? ioport_unmap(iomem) : iounmap(iomem);
0340         return ret;
0341     }
0342
0343     /*
0344      * VGA MMIO is a legacy, non-BAR resource that hopefully allows
0345      * probing, so we don't currently worry about access in relation
0346      * to the memory enable bit in the command register.
0347      */
0348     done = do_io_rw(vdev, false, iomem, buf, off, count, 0, 0, iswrite);
0349
0350     vga_put(vdev->pdev, rsrc);
0351
0352     is_ioport ? ioport_unmap(iomem) : iounmap(iomem);
0353
0354     if (done >= 0)
0355         *ppos += done;
0356
0357     return done;
0358 }
0359 #endif
0360
0361 static void vfio_pci_ioeventfd_do_write(struct vfio_pci_ioeventfd *ioeventfd,
0362                     bool test_mem)
0363 {
0364     switch (ioeventfd->count) {
0365     case 1:
0366         vfio_pci_iowrite8(ioeventfd->vdev, test_mem,
0367                   ioeventfd->data, ioeventfd->addr);
0368         break;
0369     case 2:
0370         vfio_pci_iowrite16(ioeventfd->vdev, test_mem,
0371                    ioeventfd->data, ioeventfd->addr);
0372         break;
0373     case 4:
0374         vfio_pci_iowrite32(ioeventfd->vdev, test_mem,
0375                    ioeventfd->data, ioeventfd->addr);
0376         break;
0377 #ifdef iowrite64
0378     case 8:
0379         vfio_pci_iowrite64(ioeventfd->vdev, test_mem,
0380                    ioeventfd->data, ioeventfd->addr);
0381         break;
0382 #endif
0383     }
0384 }
0385
0386 static int vfio_pci_ioeventfd_handler(void *opaque, void *unused)
0387 {
0388     struct vfio_pci_ioeventfd *ioeventfd = opaque;
0389     struct vfio_pci_core_device *vdev = ioeventfd->vdev;
0390
0391     if (ioeventfd->test_mem) {
0392         if (!down_read_trylock(&vdev->memory_lock))
0393             return 1; /* Lock contended, use thread */
0394         if (!__vfio_pci_memory_enabled(vdev)) {
0395             up_read(&vdev->memory_lock);
0396             return 0;
0397         }
0398     }
0399
0400     vfio_pci_ioeventfd_do_write(ioeventfd, false);
0401
0402     if (ioeventfd->test_mem)
0403         up_read(&vdev->memory_lock);
0404
0405     return 0;
0406 }
0407
0408 static void vfio_pci_ioeventfd_thread(void *opaque, void *unused)
0409 {
0410     struct vfio_pci_ioeventfd *ioeventfd = opaque;
0411
0412     vfio_pci_ioeventfd_do_write(ioeventfd, ioeventfd->test_mem);
0413 }
0414
0415 long vfio_pci_ioeventfd(struct vfio_pci_core_device *vdev, loff_t offset,
0416             uint64_t data, int count, int fd)
0417 {
0418     struct pci_dev *pdev = vdev->pdev;
0419     loff_t pos = offset & VFIO_PCI_OFFSET_MASK;
0420     int ret, bar = VFIO_PCI_OFFSET_TO_INDEX(offset);
0421     struct vfio_pci_ioeventfd *ioeventfd;
0422
0423     /* Only support ioeventfds into BARs */
0424     if (bar > VFIO_PCI_BAR5_REGION_INDEX)
0425         return -EINVAL;
0426
0427     if (pos + count > pci_resource_len(pdev, bar))
0428         return -EINVAL;
0429
0430     /* Disallow ioeventfds working around MSI-X table writes */
0431     if (bar == vdev->msix_bar &&
0432         !(pos + count <= vdev->msix_offset ||
0433           pos >= vdev->msix_offset + vdev->msix_size))
0434         return -EINVAL;
0435
0436 #ifndef iowrite64
0437     if (count == 8)
0438         return -EINVAL;
0439 #endif
0440
0441     ret = vfio_pci_setup_barmap(vdev, bar);
0442     if (ret)
0443         return ret;
0444
0445     mutex_lock(&vdev->ioeventfds_lock);
0446
0447     list_for_each_entry(ioeventfd, &vdev->ioeventfds_list, next) {
0448         if (ioeventfd->pos == pos && ioeventfd->bar == bar &&
0449             ioeventfd->data == data && ioeventfd->count == count) {
0450             if (fd == -1) {
0451                 vfio_virqfd_disable(&ioeventfd->virqfd);
0452                 list_del(&ioeventfd->next);
0453                 vdev->ioeventfds_nr--;
0454                 kfree(ioeventfd);
0455                 ret = 0;
0456             } else
0457                 ret = -EEXIST;
0458
0459             goto out_unlock;
0460         }
0461     }
0462
0463     if (fd < 0) {
0464         ret = -ENODEV;
0465         goto out_unlock;
0466     }
0467
0468     if (vdev->ioeventfds_nr >= VFIO_PCI_IOEVENTFD_MAX) {
0469         ret = -ENOSPC;
0470         goto out_unlock;
0471     }
0472
0473     ioeventfd = kzalloc(sizeof(*ioeventfd), GFP_KERNEL);
0474     if (!ioeventfd) {
0475         ret = -ENOMEM;
0476         goto out_unlock;
0477     }
0478
0479     ioeventfd->vdev = vdev;
0480     ioeventfd->addr = vdev->barmap[bar] + pos;
0481     ioeventfd->data = data;
0482     ioeventfd->pos = pos;
0483     ioeventfd->bar = bar;
0484     ioeventfd->count = count;
0485     ioeventfd->test_mem = vdev->pdev->resource[bar].flags & IORESOURCE_MEM;
0486
0487     ret = vfio_virqfd_enable(ioeventfd, vfio_pci_ioeventfd_handler,
0488                  vfio_pci_ioeventfd_thread, NULL,
0489                  &ioeventfd->virqfd, fd);
0490     if (ret) {
0491         kfree(ioeventfd);
0492         goto out_unlock;
0493     }
0494
0495     list_add(&ioeventfd->next, &vdev->ioeventfds_list);
0496     vdev->ioeventfds_nr++;
0497
0498 out_unlock:
0499     mutex_unlock(&vdev->ioeventfds_lock);
0500
0501     return ret;
0502 }