platforms/powernv/pci-ioda-tce.c

0001 // SPDX-License-Identifier: GPL-2.0+
0002 /*
0003  * TCE helpers for IODA PCI/PCIe on PowerNV platforms
0004  *
0005  * Copyright 2018 IBM Corp.
0006  *
0007  * This program is free software; you can redistribute it and/or
0008  * modify it under the terms of the GNU General Public License
0009  * as published by the Free Software Foundation; either version
0010  * 2 of the License, or (at your option) any later version.
0011  */
0012
0013 #include <linux/kernel.h>
0014 #include <linux/iommu.h>
0015
0016 #include <asm/iommu.h>
0017 #include <asm/tce.h>
0018 #include "pci.h"
0019
0020 unsigned long pnv_ioda_parse_tce_sizes(struct pnv_phb *phb)
0021 {
0022     struct pci_controller *hose = phb->hose;
0023     struct device_node *dn = hose->dn;
0024     unsigned long mask = 0;
0025     int i, rc, count;
0026     u32 val;
0027
0028     count = of_property_count_u32_elems(dn, "ibm,supported-tce-sizes");
0029     if (count <= 0) {
0030         mask = SZ_4K | SZ_64K;
0031         /* Add 16M for POWER8 by default */
0032         if (cpu_has_feature(CPU_FTR_ARCH_207S) &&
0033                 !cpu_has_feature(CPU_FTR_ARCH_300))
0034             mask |= SZ_16M | SZ_256M;
0035         return mask;
0036     }
0037
0038     for (i = 0; i < count; i++) {
0039         rc = of_property_read_u32_index(dn, "ibm,supported-tce-sizes",
0040                         i, &val);
0041         if (rc == 0)
0042             mask |= 1ULL << val;
0043     }
0044
0045     return mask;
0046 }
0047
0048 void pnv_pci_setup_iommu_table(struct iommu_table *tbl,
0049         void *tce_mem, u64 tce_size,
0050         u64 dma_offset, unsigned int page_shift)
0051 {
0052     tbl->it_blocksize = 16;
0053     tbl->it_base = (unsigned long)tce_mem;
0054     tbl->it_page_shift = page_shift;
0055     tbl->it_offset = dma_offset >> tbl->it_page_shift;
0056     tbl->it_index = 0;
0057     tbl->it_size = tce_size >> 3;
0058     tbl->it_busno = 0;
0059     tbl->it_type = TCE_PCI;
0060 }
0061
0062 static __be64 *pnv_alloc_tce_level(int nid, unsigned int shift)
0063 {
0064     struct page *tce_mem = NULL;
0065     __be64 *addr;
0066
0067     tce_mem = alloc_pages_node(nid, GFP_ATOMIC | __GFP_NOWARN,
0068             shift - PAGE_SHIFT);
0069     if (!tce_mem) {
0070         pr_err("Failed to allocate a TCE memory, level shift=%d\n",
0071                 shift);
0072         return NULL;
0073     }
0074     addr = page_address(tce_mem);
0075     memset(addr, 0, 1UL << shift);
0076
0077     return addr;
0078 }
0079
0080 static void pnv_pci_ioda2_table_do_free_pages(__be64 *addr,
0081         unsigned long size, unsigned int levels);
0082
0083 static __be64 *pnv_tce(struct iommu_table *tbl, bool user, long idx, bool alloc)
0084 {
0085     __be64 *tmp = user ? tbl->it_userspace : (__be64 *) tbl->it_base;
0086     int  level = tbl->it_indirect_levels;
0087     const long shift = ilog2(tbl->it_level_size);
0088     unsigned long mask = (tbl->it_level_size - 1) << (level * shift);
0089
0090     while (level) {
0091         int n = (idx & mask) >> (level * shift);
0092         unsigned long oldtce, tce = be64_to_cpu(READ_ONCE(tmp[n]));
0093
0094         if (!tce) {
0095             __be64 *tmp2;
0096
0097             if (!alloc)
0098                 return NULL;
0099
0100             tmp2 = pnv_alloc_tce_level(tbl->it_nid,
0101                     ilog2(tbl->it_level_size) + 3);
0102             if (!tmp2)
0103                 return NULL;
0104
0105             tce = __pa(tmp2) | TCE_PCI_READ | TCE_PCI_WRITE;
0106             oldtce = be64_to_cpu(cmpxchg(&tmp[n], 0,
0107                     cpu_to_be64(tce)));
0108             if (oldtce) {
0109                 pnv_pci_ioda2_table_do_free_pages(tmp2,
0110                     ilog2(tbl->it_level_size) + 3, 1);
0111                 tce = oldtce;
0112             }
0113         }
0114
0115         tmp = __va(tce & ~(TCE_PCI_READ | TCE_PCI_WRITE));
0116         idx &= ~mask;
0117         mask >>= shift;
0118         --level;
0119     }
0120
0121     return tmp + idx;
0122 }
0123
0124 int pnv_tce_build(struct iommu_table *tbl, long index, long npages,
0125         unsigned long uaddr, enum dma_data_direction direction,
0126         unsigned long attrs)
0127 {
0128     u64 proto_tce = iommu_direction_to_tce_perm(direction);
0129     u64 rpn = __pa(uaddr) >> tbl->it_page_shift;
0130     long i;
0131
0132     if (proto_tce & TCE_PCI_WRITE)
0133         proto_tce |= TCE_PCI_READ;
0134
0135     for (i = 0; i < npages; i++) {
0136         unsigned long newtce = proto_tce |
0137             ((rpn + i) << tbl->it_page_shift);
0138         unsigned long idx = index - tbl->it_offset + i;
0139
0140         *(pnv_tce(tbl, false, idx, true)) = cpu_to_be64(newtce);
0141     }
0142
0143     return 0;
0144 }
0145
0146 #ifdef CONFIG_IOMMU_API
0147 int pnv_tce_xchg(struct iommu_table *tbl, long index,
0148         unsigned long *hpa, enum dma_data_direction *direction)
0149 {
0150     u64 proto_tce = iommu_direction_to_tce_perm(*direction);
0151     unsigned long newtce = *hpa | proto_tce, oldtce;
0152     unsigned long idx = index - tbl->it_offset;
0153     __be64 *ptce = NULL;
0154
0155     BUG_ON(*hpa & ~IOMMU_PAGE_MASK(tbl));
0156
0157     if (*direction == DMA_NONE) {
0158         ptce = pnv_tce(tbl, false, idx, false);
0159         if (!ptce) {
0160             *hpa = 0;
0161             return 0;
0162         }
0163     }
0164
0165     if (!ptce) {
0166         ptce = pnv_tce(tbl, false, idx, true);
0167         if (!ptce)
0168             return -ENOMEM;
0169     }
0170
0171     if (newtce & TCE_PCI_WRITE)
0172         newtce |= TCE_PCI_READ;
0173
0174     oldtce = be64_to_cpu(xchg(ptce, cpu_to_be64(newtce)));
0175     *hpa = oldtce & ~(TCE_PCI_READ | TCE_PCI_WRITE);
0176     *direction = iommu_tce_direction(oldtce);
0177
0178     return 0;
0179 }
0180
0181 __be64 *pnv_tce_useraddrptr(struct iommu_table *tbl, long index, bool alloc)
0182 {
0183     if (WARN_ON_ONCE(!tbl->it_userspace))
0184         return NULL;
0185
0186     return pnv_tce(tbl, true, index - tbl->it_offset, alloc);
0187 }
0188 #endif
0189
0190 void pnv_tce_free(struct iommu_table *tbl, long index, long npages)
0191 {
0192     long i;
0193
0194     for (i = 0; i < npages; i++) {
0195         unsigned long idx = index - tbl->it_offset + i;
0196         __be64 *ptce = pnv_tce(tbl, false, idx, false);
0197
0198         if (ptce)
0199             *ptce = cpu_to_be64(0);
0200         else
0201             /* Skip the rest of the level */
0202             i |= tbl->it_level_size - 1;
0203     }
0204 }
0205
0206 unsigned long pnv_tce_get(struct iommu_table *tbl, long index)
0207 {
0208     __be64 *ptce = pnv_tce(tbl, false, index - tbl->it_offset, false);
0209
0210     if (!ptce)
0211         return 0;
0212
0213     return be64_to_cpu(*ptce);
0214 }
0215
0216 static void pnv_pci_ioda2_table_do_free_pages(__be64 *addr,
0217         unsigned long size, unsigned int levels)
0218 {
0219     const unsigned long addr_ul = (unsigned long) addr &
0220             ~(TCE_PCI_READ | TCE_PCI_WRITE);
0221
0222     if (levels) {
0223         long i;
0224         u64 *tmp = (u64 *) addr_ul;
0225
0226         for (i = 0; i < size; ++i) {
0227             unsigned long hpa = be64_to_cpu(tmp[i]);
0228
0229             if (!(hpa & (TCE_PCI_READ | TCE_PCI_WRITE)))
0230                 continue;
0231
0232             pnv_pci_ioda2_table_do_free_pages(__va(hpa), size,
0233                     levels - 1);
0234         }
0235     }
0236
0237     free_pages(addr_ul, get_order(size << 3));
0238 }
0239
0240 void pnv_pci_ioda2_table_free_pages(struct iommu_table *tbl)
0241 {
0242     const unsigned long size = tbl->it_indirect_levels ?
0243             tbl->it_level_size : tbl->it_size;
0244
0245     if (!tbl->it_size)
0246         return;
0247
0248     pnv_pci_ioda2_table_do_free_pages((__be64 *)tbl->it_base, size,
0249             tbl->it_indirect_levels);
0250     if (tbl->it_userspace) {
0251         pnv_pci_ioda2_table_do_free_pages(tbl->it_userspace, size,
0252                 tbl->it_indirect_levels);
0253     }
0254 }
0255
0256 static __be64 *pnv_pci_ioda2_table_do_alloc_pages(int nid, unsigned int shift,
0257         unsigned int levels, unsigned long limit,
0258         unsigned long *current_offset, unsigned long *total_allocated)
0259 {
0260     __be64 *addr, *tmp;
0261     unsigned long allocated = 1UL << shift;
0262     unsigned int entries = 1UL << (shift - 3);
0263     long i;
0264
0265     addr = pnv_alloc_tce_level(nid, shift);
0266     *total_allocated += allocated;
0267
0268     --levels;
0269     if (!levels) {
0270         *current_offset += allocated;
0271         return addr;
0272     }
0273
0274     for (i = 0; i < entries; ++i) {
0275         tmp = pnv_pci_ioda2_table_do_alloc_pages(nid, shift,
0276                 levels, limit, current_offset, total_allocated);
0277         if (!tmp)
0278             break;
0279
0280         addr[i] = cpu_to_be64(__pa(tmp) |
0281                 TCE_PCI_READ | TCE_PCI_WRITE);
0282
0283         if (*current_offset >= limit)
0284             break;
0285     }
0286
0287     return addr;
0288 }
0289
0290 long pnv_pci_ioda2_table_alloc_pages(int nid, __u64 bus_offset,
0291         __u32 page_shift, __u64 window_size, __u32 levels,
0292         bool alloc_userspace_copy, struct iommu_table *tbl)
0293 {
0294     void *addr, *uas = NULL;
0295     unsigned long offset = 0, level_shift, total_allocated = 0;
0296     unsigned long total_allocated_uas = 0;
0297     const unsigned int window_shift = ilog2(window_size);
0298     unsigned int entries_shift = window_shift - page_shift;
0299     unsigned int table_shift = max_t(unsigned int, entries_shift + 3,
0300             PAGE_SHIFT);
0301     const unsigned long tce_table_size = 1UL << table_shift;
0302
0303     if (!levels || (levels > POWERNV_IOMMU_MAX_LEVELS))
0304         return -EINVAL;
0305
0306     if (!is_power_of_2(window_size))
0307         return -EINVAL;
0308
0309     /* Adjust direct table size from window_size and levels */
0310     entries_shift = (entries_shift + levels - 1) / levels;
0311     level_shift = entries_shift + 3;
0312     level_shift = max_t(unsigned int, level_shift, PAGE_SHIFT);
0313
0314     if ((level_shift - 3) * levels + page_shift >= 55)
0315         return -EINVAL;
0316
0317     /* Allocate TCE table */
0318     addr = pnv_pci_ioda2_table_do_alloc_pages(nid, level_shift,
0319             1, tce_table_size, &offset, &total_allocated);
0320
0321     /* addr==NULL means that the first level allocation failed */
0322     if (!addr)
0323         return -ENOMEM;
0324
0325     /*
0326      * First level was allocated but some lower level failed as
0327      * we did not allocate as much as we wanted,
0328      * release partially allocated table.
0329      */
0330     if (levels == 1 && offset < tce_table_size)
0331         goto free_tces_exit;
0332
0333     /* Allocate userspace view of the TCE table */
0334     if (alloc_userspace_copy) {
0335         offset = 0;
0336         uas = pnv_pci_ioda2_table_do_alloc_pages(nid, level_shift,
0337                 1, tce_table_size, &offset,
0338                 &total_allocated_uas);
0339         if (!uas)
0340             goto free_tces_exit;
0341         if (levels == 1 && (offset < tce_table_size ||
0342                 total_allocated_uas != total_allocated))
0343             goto free_uas_exit;
0344     }
0345
0346     /* Setup linux iommu table */
0347     pnv_pci_setup_iommu_table(tbl, addr, tce_table_size, bus_offset,
0348             page_shift);
0349     tbl->it_level_size = 1ULL << (level_shift - 3);
0350     tbl->it_indirect_levels = levels - 1;
0351     tbl->it_userspace = uas;
0352     tbl->it_nid = nid;
0353
0354     pr_debug("Created TCE table: ws=%08llx ts=%lx @%08llx base=%lx uas=%p levels=%d/%d\n",
0355             window_size, tce_table_size, bus_offset, tbl->it_base,
0356             tbl->it_userspace, 1, levels);
0357
0358     return 0;
0359
0360 free_uas_exit:
0361     pnv_pci_ioda2_table_do_free_pages(uas,
0362             1ULL << (level_shift - 3), levels - 1);
0363 free_tces_exit:
0364     pnv_pci_ioda2_table_do_free_pages(addr,
0365             1ULL << (level_shift - 3), levels - 1);
0366
0367     return -ENOMEM;
0368 }
0369
0370 void pnv_pci_unlink_table_and_group(struct iommu_table *tbl,
0371         struct iommu_table_group *table_group)
0372 {
0373     long i;
0374     bool found;
0375     struct iommu_table_group_link *tgl;
0376
0377     if (!tbl || !table_group)
0378         return;
0379
0380     /* Remove link to a group from table's list of attached groups */
0381     found = false;
0382
0383     rcu_read_lock();
0384     list_for_each_entry_rcu(tgl, &tbl->it_group_list, next) {
0385         if (tgl->table_group == table_group) {
0386             list_del_rcu(&tgl->next);
0387             kfree_rcu(tgl, rcu);
0388             found = true;
0389             break;
0390         }
0391     }
0392     rcu_read_unlock();
0393
0394     if (WARN_ON(!found))
0395         return;
0396
0397     /* Clean a pointer to iommu_table in iommu_table_group::tables[] */
0398     found = false;
0399     for (i = 0; i < IOMMU_TABLE_GROUP_MAX_TABLES; ++i) {
0400         if (table_group->tables[i] == tbl) {
0401             iommu_tce_table_put(tbl);
0402             table_group->tables[i] = NULL;
0403             found = true;
0404             break;
0405         }
0406     }
0407     WARN_ON(!found);
0408 }
0409
0410 long pnv_pci_link_table_and_group(int node, int num,
0411         struct iommu_table *tbl,
0412         struct iommu_table_group *table_group)
0413 {
0414     struct iommu_table_group_link *tgl = NULL;
0415
0416     if (WARN_ON(!tbl || !table_group))
0417         return -EINVAL;
0418
0419     tgl = kzalloc_node(sizeof(struct iommu_table_group_link), GFP_KERNEL,
0420             node);
0421     if (!tgl)
0422         return -ENOMEM;
0423
0424     tgl->table_group = table_group;
0425     list_add_rcu(&tgl->next, &tbl->it_group_list);
0426
0427     table_group->tables[num] = iommu_tce_table_get(tbl);
0428
0429     return 0;
0430 }