drivers/iommu/s390-iommu.c

0001 // SPDX-License-Identifier: GPL-2.0
0002 /*
0003  * IOMMU API for s390 PCI devices
0004  *
0005  * Copyright IBM Corp. 2015
0006  * Author(s): Gerald Schaefer <gerald.schaefer@de.ibm.com>
0007  */
0008
0009 #include <linux/pci.h>
0010 #include <linux/iommu.h>
0011 #include <linux/iommu-helper.h>
0012 #include <linux/sizes.h>
0013 #include <asm/pci_dma.h>
0014
0015 /*
0016  * Physically contiguous memory regions can be mapped with 4 KiB alignment,
0017  * we allow all page sizes that are an order of 4KiB (no special large page
0018  * support so far).
0019  */
0020 #define S390_IOMMU_PGSIZES  (~0xFFFUL)
0021
0022 static const struct iommu_ops s390_iommu_ops;
0023
0024 struct s390_domain {
0025     struct iommu_domain domain;
0026     struct list_head    devices;
0027     unsigned long       *dma_table;
0028     spinlock_t      dma_table_lock;
0029     spinlock_t      list_lock;
0030 };
0031
0032 struct s390_domain_device {
0033     struct list_head    list;
0034     struct zpci_dev     *zdev;
0035 };
0036
0037 static struct s390_domain *to_s390_domain(struct iommu_domain *dom)
0038 {
0039     return container_of(dom, struct s390_domain, domain);
0040 }
0041
0042 static bool s390_iommu_capable(enum iommu_cap cap)
0043 {
0044     switch (cap) {
0045     case IOMMU_CAP_CACHE_COHERENCY:
0046         return true;
0047     case IOMMU_CAP_INTR_REMAP:
0048         return true;
0049     default:
0050         return false;
0051     }
0052 }
0053
0054 static struct iommu_domain *s390_domain_alloc(unsigned domain_type)
0055 {
0056     struct s390_domain *s390_domain;
0057
0058     if (domain_type != IOMMU_DOMAIN_UNMANAGED)
0059         return NULL;
0060
0061     s390_domain = kzalloc(sizeof(*s390_domain), GFP_KERNEL);
0062     if (!s390_domain)
0063         return NULL;
0064
0065     s390_domain->dma_table = dma_alloc_cpu_table();
0066     if (!s390_domain->dma_table) {
0067         kfree(s390_domain);
0068         return NULL;
0069     }
0070
0071     spin_lock_init(&s390_domain->dma_table_lock);
0072     spin_lock_init(&s390_domain->list_lock);
0073     INIT_LIST_HEAD(&s390_domain->devices);
0074
0075     return &s390_domain->domain;
0076 }
0077
0078 static void s390_domain_free(struct iommu_domain *domain)
0079 {
0080     struct s390_domain *s390_domain = to_s390_domain(domain);
0081
0082     dma_cleanup_tables(s390_domain->dma_table);
0083     kfree(s390_domain);
0084 }
0085
0086 static int s390_iommu_attach_device(struct iommu_domain *domain,
0087                     struct device *dev)
0088 {
0089     struct s390_domain *s390_domain = to_s390_domain(domain);
0090     struct zpci_dev *zdev = to_zpci_dev(dev);
0091     struct s390_domain_device *domain_device;
0092     unsigned long flags;
0093     int cc, rc;
0094
0095     if (!zdev)
0096         return -ENODEV;
0097
0098     domain_device = kzalloc(sizeof(*domain_device), GFP_KERNEL);
0099     if (!domain_device)
0100         return -ENOMEM;
0101
0102     if (zdev->dma_table && !zdev->s390_domain) {
0103         cc = zpci_dma_exit_device(zdev);
0104         if (cc) {
0105             rc = -EIO;
0106             goto out_free;
0107         }
0108     }
0109
0110     if (zdev->s390_domain)
0111         zpci_unregister_ioat(zdev, 0);
0112
0113     zdev->dma_table = s390_domain->dma_table;
0114     cc = zpci_register_ioat(zdev, 0, zdev->start_dma, zdev->end_dma,
0115                 virt_to_phys(zdev->dma_table));
0116     if (cc) {
0117         rc = -EIO;
0118         goto out_restore;
0119     }
0120
0121     spin_lock_irqsave(&s390_domain->list_lock, flags);
0122     /* First device defines the DMA range limits */
0123     if (list_empty(&s390_domain->devices)) {
0124         domain->geometry.aperture_start = zdev->start_dma;
0125         domain->geometry.aperture_end = zdev->end_dma;
0126         domain->geometry.force_aperture = true;
0127     /* Allow only devices with identical DMA range limits */
0128     } else if (domain->geometry.aperture_start != zdev->start_dma ||
0129            domain->geometry.aperture_end != zdev->end_dma) {
0130         rc = -EINVAL;
0131         spin_unlock_irqrestore(&s390_domain->list_lock, flags);
0132         goto out_restore;
0133     }
0134     domain_device->zdev = zdev;
0135     zdev->s390_domain = s390_domain;
0136     list_add(&domain_device->list, &s390_domain->devices);
0137     spin_unlock_irqrestore(&s390_domain->list_lock, flags);
0138
0139     return 0;
0140
0141 out_restore:
0142     if (!zdev->s390_domain) {
0143         zpci_dma_init_device(zdev);
0144     } else {
0145         zdev->dma_table = zdev->s390_domain->dma_table;
0146         zpci_register_ioat(zdev, 0, zdev->start_dma, zdev->end_dma,
0147                    virt_to_phys(zdev->dma_table));
0148     }
0149 out_free:
0150     kfree(domain_device);
0151
0152     return rc;
0153 }
0154
0155 static void s390_iommu_detach_device(struct iommu_domain *domain,
0156                      struct device *dev)
0157 {
0158     struct s390_domain *s390_domain = to_s390_domain(domain);
0159     struct zpci_dev *zdev = to_zpci_dev(dev);
0160     struct s390_domain_device *domain_device, *tmp;
0161     unsigned long flags;
0162     int found = 0;
0163
0164     if (!zdev)
0165         return;
0166
0167     spin_lock_irqsave(&s390_domain->list_lock, flags);
0168     list_for_each_entry_safe(domain_device, tmp, &s390_domain->devices,
0169                  list) {
0170         if (domain_device->zdev == zdev) {
0171             list_del(&domain_device->list);
0172             kfree(domain_device);
0173             found = 1;
0174             break;
0175         }
0176     }
0177     spin_unlock_irqrestore(&s390_domain->list_lock, flags);
0178
0179     if (found && (zdev->s390_domain == s390_domain)) {
0180         zdev->s390_domain = NULL;
0181         zpci_unregister_ioat(zdev, 0);
0182         zpci_dma_init_device(zdev);
0183     }
0184 }
0185
0186 static struct iommu_device *s390_iommu_probe_device(struct device *dev)
0187 {
0188     struct zpci_dev *zdev = to_zpci_dev(dev);
0189
0190     return &zdev->iommu_dev;
0191 }
0192
0193 static void s390_iommu_release_device(struct device *dev)
0194 {
0195     struct zpci_dev *zdev = to_zpci_dev(dev);
0196     struct iommu_domain *domain;
0197
0198     /*
0199      * This is a workaround for a scenario where the IOMMU API common code
0200      * "forgets" to call the detach_dev callback: After binding a device
0201      * to vfio-pci and completing the VFIO_SET_IOMMU ioctl (which triggers
0202      * the attach_dev), removing the device via
0203      * "echo 1 > /sys/bus/pci/devices/.../remove" won't trigger detach_dev,
0204      * only release_device will be called via the BUS_NOTIFY_REMOVED_DEVICE
0205      * notifier.
0206      *
0207      * So let's call detach_dev from here if it hasn't been called before.
0208      */
0209     if (zdev && zdev->s390_domain) {
0210         domain = iommu_get_domain_for_dev(dev);
0211         if (domain)
0212             s390_iommu_detach_device(domain, dev);
0213     }
0214 }
0215
0216 static int s390_iommu_update_trans(struct s390_domain *s390_domain,
0217                    phys_addr_t pa, dma_addr_t dma_addr,
0218                    size_t size, int flags)
0219 {
0220     struct s390_domain_device *domain_device;
0221     phys_addr_t page_addr = pa & PAGE_MASK;
0222     dma_addr_t start_dma_addr = dma_addr;
0223     unsigned long irq_flags, nr_pages, i;
0224     unsigned long *entry;
0225     int rc = 0;
0226
0227     if (dma_addr < s390_domain->domain.geometry.aperture_start ||
0228         dma_addr + size > s390_domain->domain.geometry.aperture_end)
0229         return -EINVAL;
0230
0231     nr_pages = PAGE_ALIGN(size) >> PAGE_SHIFT;
0232     if (!nr_pages)
0233         return 0;
0234
0235     spin_lock_irqsave(&s390_domain->dma_table_lock, irq_flags);
0236     for (i = 0; i < nr_pages; i++) {
0237         entry = dma_walk_cpu_trans(s390_domain->dma_table, dma_addr);
0238         if (!entry) {
0239             rc = -ENOMEM;
0240             goto undo_cpu_trans;
0241         }
0242         dma_update_cpu_trans(entry, page_addr, flags);
0243         page_addr += PAGE_SIZE;
0244         dma_addr += PAGE_SIZE;
0245     }
0246
0247     spin_lock(&s390_domain->list_lock);
0248     list_for_each_entry(domain_device, &s390_domain->devices, list) {
0249         rc = zpci_refresh_trans((u64) domain_device->zdev->fh << 32,
0250                     start_dma_addr, nr_pages * PAGE_SIZE);
0251         if (rc)
0252             break;
0253     }
0254     spin_unlock(&s390_domain->list_lock);
0255
0256 undo_cpu_trans:
0257     if (rc && ((flags & ZPCI_PTE_VALID_MASK) == ZPCI_PTE_VALID)) {
0258         flags = ZPCI_PTE_INVALID;
0259         while (i-- > 0) {
0260             page_addr -= PAGE_SIZE;
0261             dma_addr -= PAGE_SIZE;
0262             entry = dma_walk_cpu_trans(s390_domain->dma_table,
0263                            dma_addr);
0264             if (!entry)
0265                 break;
0266             dma_update_cpu_trans(entry, page_addr, flags);
0267         }
0268     }
0269     spin_unlock_irqrestore(&s390_domain->dma_table_lock, irq_flags);
0270
0271     return rc;
0272 }
0273
0274 static int s390_iommu_map(struct iommu_domain *domain, unsigned long iova,
0275               phys_addr_t paddr, size_t size, int prot, gfp_t gfp)
0276 {
0277     struct s390_domain *s390_domain = to_s390_domain(domain);
0278     int flags = ZPCI_PTE_VALID, rc = 0;
0279
0280     if (!(prot & IOMMU_READ))
0281         return -EINVAL;
0282
0283     if (!(prot & IOMMU_WRITE))
0284         flags |= ZPCI_TABLE_PROTECTED;
0285
0286     rc = s390_iommu_update_trans(s390_domain, paddr, iova,
0287                      size, flags);
0288
0289     return rc;
0290 }
0291
0292 static phys_addr_t s390_iommu_iova_to_phys(struct iommu_domain *domain,
0293                        dma_addr_t iova)
0294 {
0295     struct s390_domain *s390_domain = to_s390_domain(domain);
0296     unsigned long *sto, *pto, *rto, flags;
0297     unsigned int rtx, sx, px;
0298     phys_addr_t phys = 0;
0299
0300     if (iova < domain->geometry.aperture_start ||
0301         iova > domain->geometry.aperture_end)
0302         return 0;
0303
0304     rtx = calc_rtx(iova);
0305     sx = calc_sx(iova);
0306     px = calc_px(iova);
0307     rto = s390_domain->dma_table;
0308
0309     spin_lock_irqsave(&s390_domain->dma_table_lock, flags);
0310     if (rto && reg_entry_isvalid(rto[rtx])) {
0311         sto = get_rt_sto(rto[rtx]);
0312         if (sto && reg_entry_isvalid(sto[sx])) {
0313             pto = get_st_pto(sto[sx]);
0314             if (pto && pt_entry_isvalid(pto[px]))
0315                 phys = pto[px] & ZPCI_PTE_ADDR_MASK;
0316         }
0317     }
0318     spin_unlock_irqrestore(&s390_domain->dma_table_lock, flags);
0319
0320     return phys;
0321 }
0322
0323 static size_t s390_iommu_unmap(struct iommu_domain *domain,
0324                    unsigned long iova, size_t size,
0325                    struct iommu_iotlb_gather *gather)
0326 {
0327     struct s390_domain *s390_domain = to_s390_domain(domain);
0328     int flags = ZPCI_PTE_INVALID;
0329     phys_addr_t paddr;
0330     int rc;
0331
0332     paddr = s390_iommu_iova_to_phys(domain, iova);
0333     if (!paddr)
0334         return 0;
0335
0336     rc = s390_iommu_update_trans(s390_domain, paddr, iova,
0337                      size, flags);
0338     if (rc)
0339         return 0;
0340
0341     return size;
0342 }
0343
0344 int zpci_init_iommu(struct zpci_dev *zdev)
0345 {
0346     int rc = 0;
0347
0348     rc = iommu_device_sysfs_add(&zdev->iommu_dev, NULL, NULL,
0349                     "s390-iommu.%08x", zdev->fid);
0350     if (rc)
0351         goto out_err;
0352
0353     rc = iommu_device_register(&zdev->iommu_dev, &s390_iommu_ops, NULL);
0354     if (rc)
0355         goto out_sysfs;
0356
0357     return 0;
0358
0359 out_sysfs:
0360     iommu_device_sysfs_remove(&zdev->iommu_dev);
0361
0362 out_err:
0363     return rc;
0364 }
0365
0366 void zpci_destroy_iommu(struct zpci_dev *zdev)
0367 {
0368     iommu_device_unregister(&zdev->iommu_dev);
0369     iommu_device_sysfs_remove(&zdev->iommu_dev);
0370 }
0371
0372 static const struct iommu_ops s390_iommu_ops = {
0373     .capable = s390_iommu_capable,
0374     .domain_alloc = s390_domain_alloc,
0375     .probe_device = s390_iommu_probe_device,
0376     .release_device = s390_iommu_release_device,
0377     .device_group = generic_device_group,
0378     .pgsize_bitmap = S390_IOMMU_PGSIZES,
0379     .default_domain_ops = &(const struct iommu_domain_ops) {
0380         .attach_dev = s390_iommu_attach_device,
0381         .detach_dev = s390_iommu_detach_device,
0382         .map        = s390_iommu_map,
0383         .unmap      = s390_iommu_unmap,
0384         .iova_to_phys   = s390_iommu_iova_to_phys,
0385         .free       = s390_domain_free,
0386     }
0387 };
0388
0389 static int __init s390_iommu_init(void)
0390 {
0391     return bus_set_iommu(&pci_bus_type, &s390_iommu_ops);
0392 }
0393 subsys_initcall(s390_iommu_init);