controller/cadence/pcie-cadence-host.c

0001 // SPDX-License-Identifier: GPL-2.0
0002 // Copyright (c) 2017 Cadence
0003 // Cadence PCIe host controller driver.
0004 // Author: Cyrille Pitchen <cyrille.pitchen@free-electrons.com>
0005
0006 #include <linux/delay.h>
0007 #include <linux/kernel.h>
0008 #include <linux/list_sort.h>
0009 #include <linux/of_address.h>
0010 #include <linux/of_pci.h>
0011 #include <linux/platform_device.h>
0012
0013 #include "pcie-cadence.h"
0014
0015 static u64 bar_max_size[] = {
0016     [RP_BAR0] = _ULL(128 * SZ_2G),
0017     [RP_BAR1] = SZ_2G,
0018     [RP_NO_BAR] = _BITULL(63),
0019 };
0020
0021 static u8 bar_aperture_mask[] = {
0022     [RP_BAR0] = 0x1F,
0023     [RP_BAR1] = 0xF,
0024 };
0025
0026 void __iomem *cdns_pci_map_bus(struct pci_bus *bus, unsigned int devfn,
0027                    int where)
0028 {
0029     struct pci_host_bridge *bridge = pci_find_host_bridge(bus);
0030     struct cdns_pcie_rc *rc = pci_host_bridge_priv(bridge);
0031     struct cdns_pcie *pcie = &rc->pcie;
0032     unsigned int busn = bus->number;
0033     u32 addr0, desc0;
0034
0035     if (pci_is_root_bus(bus)) {
0036         /*
0037          * Only the root port (devfn == 0) is connected to this bus.
0038          * All other PCI devices are behind some bridge hence on another
0039          * bus.
0040          */
0041         if (devfn)
0042             return NULL;
0043
0044         return pcie->reg_base + (where & 0xfff);
0045     }
0046     /* Check that the link is up */
0047     if (!(cdns_pcie_readl(pcie, CDNS_PCIE_LM_BASE) & 0x1))
0048         return NULL;
0049     /* Clear AXI link-down status */
0050     cdns_pcie_writel(pcie, CDNS_PCIE_AT_LINKDOWN, 0x0);
0051
0052     /* Update Output registers for AXI region 0. */
0053     addr0 = CDNS_PCIE_AT_OB_REGION_PCI_ADDR0_NBITS(12) |
0054         CDNS_PCIE_AT_OB_REGION_PCI_ADDR0_DEVFN(devfn) |
0055         CDNS_PCIE_AT_OB_REGION_PCI_ADDR0_BUS(busn);
0056     cdns_pcie_writel(pcie, CDNS_PCIE_AT_OB_REGION_PCI_ADDR0(0), addr0);
0057
0058     /* Configuration Type 0 or Type 1 access. */
0059     desc0 = CDNS_PCIE_AT_OB_REGION_DESC0_HARDCODED_RID |
0060         CDNS_PCIE_AT_OB_REGION_DESC0_DEVFN(0);
0061     /*
0062      * The bus number was already set once for all in desc1 by
0063      * cdns_pcie_host_init_address_translation().
0064      */
0065     if (busn == bridge->busnr + 1)
0066         desc0 |= CDNS_PCIE_AT_OB_REGION_DESC0_TYPE_CONF_TYPE0;
0067     else
0068         desc0 |= CDNS_PCIE_AT_OB_REGION_DESC0_TYPE_CONF_TYPE1;
0069     cdns_pcie_writel(pcie, CDNS_PCIE_AT_OB_REGION_DESC0(0), desc0);
0070
0071     return rc->cfg_base + (where & 0xfff);
0072 }
0073
0074 static struct pci_ops cdns_pcie_host_ops = {
0075     .map_bus    = cdns_pci_map_bus,
0076     .read       = pci_generic_config_read,
0077     .write      = pci_generic_config_write,
0078 };
0079
0080 static int cdns_pcie_host_wait_for_link(struct cdns_pcie *pcie)
0081 {
0082     struct device *dev = pcie->dev;
0083     int retries;
0084
0085     /* Check if the link is up or not */
0086     for (retries = 0; retries < LINK_WAIT_MAX_RETRIES; retries++) {
0087         if (cdns_pcie_link_up(pcie)) {
0088             dev_info(dev, "Link up\n");
0089             return 0;
0090         }
0091         usleep_range(LINK_WAIT_USLEEP_MIN, LINK_WAIT_USLEEP_MAX);
0092     }
0093
0094     return -ETIMEDOUT;
0095 }
0096
0097 static int cdns_pcie_retrain(struct cdns_pcie *pcie)
0098 {
0099     u32 lnk_cap_sls, pcie_cap_off = CDNS_PCIE_RP_CAP_OFFSET;
0100     u16 lnk_stat, lnk_ctl;
0101     int ret = 0;
0102
0103     /*
0104      * Set retrain bit if current speed is 2.5 GB/s,
0105      * but the PCIe root port support is > 2.5 GB/s.
0106      */
0107
0108     lnk_cap_sls = cdns_pcie_readl(pcie, (CDNS_PCIE_RP_BASE + pcie_cap_off +
0109                          PCI_EXP_LNKCAP));
0110     if ((lnk_cap_sls & PCI_EXP_LNKCAP_SLS) <= PCI_EXP_LNKCAP_SLS_2_5GB)
0111         return ret;
0112
0113     lnk_stat = cdns_pcie_rp_readw(pcie, pcie_cap_off + PCI_EXP_LNKSTA);
0114     if ((lnk_stat & PCI_EXP_LNKSTA_CLS) == PCI_EXP_LNKSTA_CLS_2_5GB) {
0115         lnk_ctl = cdns_pcie_rp_readw(pcie,
0116                          pcie_cap_off + PCI_EXP_LNKCTL);
0117         lnk_ctl |= PCI_EXP_LNKCTL_RL;
0118         cdns_pcie_rp_writew(pcie, pcie_cap_off + PCI_EXP_LNKCTL,
0119                     lnk_ctl);
0120
0121         ret = cdns_pcie_host_wait_for_link(pcie);
0122     }
0123     return ret;
0124 }
0125
0126 static void cdns_pcie_host_enable_ptm_response(struct cdns_pcie *pcie)
0127 {
0128     u32 val;
0129
0130     val = cdns_pcie_readl(pcie, CDNS_PCIE_LM_PTM_CTRL);
0131     cdns_pcie_writel(pcie, CDNS_PCIE_LM_PTM_CTRL, val | CDNS_PCIE_LM_TPM_CTRL_PTMRSEN);
0132 }
0133
0134 static int cdns_pcie_host_start_link(struct cdns_pcie_rc *rc)
0135 {
0136     struct cdns_pcie *pcie = &rc->pcie;
0137     int ret;
0138
0139     ret = cdns_pcie_host_wait_for_link(pcie);
0140
0141     /*
0142      * Retrain link for Gen2 training defect
0143      * if quirk flag is set.
0144      */
0145     if (!ret && rc->quirk_retrain_flag)
0146         ret = cdns_pcie_retrain(pcie);
0147
0148     return ret;
0149 }
0150
0151 static int cdns_pcie_host_init_root_port(struct cdns_pcie_rc *rc)
0152 {
0153     struct cdns_pcie *pcie = &rc->pcie;
0154     u32 value, ctrl;
0155     u32 id;
0156
0157     /*
0158      * Set the root complex BAR configuration register:
0159      * - disable both BAR0 and BAR1.
0160      * - enable Prefetchable Memory Base and Limit registers in type 1
0161      *   config space (64 bits).
0162      * - enable IO Base and Limit registers in type 1 config
0163      *   space (32 bits).
0164      */
0165     ctrl = CDNS_PCIE_LM_BAR_CFG_CTRL_DISABLED;
0166     value = CDNS_PCIE_LM_RC_BAR_CFG_BAR0_CTRL(ctrl) |
0167         CDNS_PCIE_LM_RC_BAR_CFG_BAR1_CTRL(ctrl) |
0168         CDNS_PCIE_LM_RC_BAR_CFG_PREFETCH_MEM_ENABLE |
0169         CDNS_PCIE_LM_RC_BAR_CFG_PREFETCH_MEM_64BITS |
0170         CDNS_PCIE_LM_RC_BAR_CFG_IO_ENABLE |
0171         CDNS_PCIE_LM_RC_BAR_CFG_IO_32BITS;
0172     cdns_pcie_writel(pcie, CDNS_PCIE_LM_RC_BAR_CFG, value);
0173
0174     /* Set root port configuration space */
0175     if (rc->vendor_id != 0xffff) {
0176         id = CDNS_PCIE_LM_ID_VENDOR(rc->vendor_id) |
0177             CDNS_PCIE_LM_ID_SUBSYS(rc->vendor_id);
0178         cdns_pcie_writel(pcie, CDNS_PCIE_LM_ID, id);
0179     }
0180
0181     if (rc->device_id != 0xffff)
0182         cdns_pcie_rp_writew(pcie, PCI_DEVICE_ID, rc->device_id);
0183
0184     cdns_pcie_rp_writeb(pcie, PCI_CLASS_REVISION, 0);
0185     cdns_pcie_rp_writeb(pcie, PCI_CLASS_PROG, 0);
0186     cdns_pcie_rp_writew(pcie, PCI_CLASS_DEVICE, PCI_CLASS_BRIDGE_PCI);
0187
0188     return 0;
0189 }
0190
0191 static int cdns_pcie_host_bar_ib_config(struct cdns_pcie_rc *rc,
0192                     enum cdns_pcie_rp_bar bar,
0193                     u64 cpu_addr, u64 size,
0194                     unsigned long flags)
0195 {
0196     struct cdns_pcie *pcie = &rc->pcie;
0197     u32 addr0, addr1, aperture, value;
0198
0199     if (!rc->avail_ib_bar[bar])
0200         return -EBUSY;
0201
0202     rc->avail_ib_bar[bar] = false;
0203
0204     aperture = ilog2(size);
0205     addr0 = CDNS_PCIE_AT_IB_RP_BAR_ADDR0_NBITS(aperture) |
0206         (lower_32_bits(cpu_addr) & GENMASK(31, 8));
0207     addr1 = upper_32_bits(cpu_addr);
0208     cdns_pcie_writel(pcie, CDNS_PCIE_AT_IB_RP_BAR_ADDR0(bar), addr0);
0209     cdns_pcie_writel(pcie, CDNS_PCIE_AT_IB_RP_BAR_ADDR1(bar), addr1);
0210
0211     if (bar == RP_NO_BAR)
0212         return 0;
0213
0214     value = cdns_pcie_readl(pcie, CDNS_PCIE_LM_RC_BAR_CFG);
0215     value &= ~(LM_RC_BAR_CFG_CTRL_MEM_64BITS(bar) |
0216            LM_RC_BAR_CFG_CTRL_PREF_MEM_64BITS(bar) |
0217            LM_RC_BAR_CFG_CTRL_MEM_32BITS(bar) |
0218            LM_RC_BAR_CFG_CTRL_PREF_MEM_32BITS(bar) |
0219            LM_RC_BAR_CFG_APERTURE(bar, bar_aperture_mask[bar] + 2));
0220     if (size + cpu_addr >= SZ_4G) {
0221         if (!(flags & IORESOURCE_PREFETCH))
0222             value |= LM_RC_BAR_CFG_CTRL_MEM_64BITS(bar);
0223         value |= LM_RC_BAR_CFG_CTRL_PREF_MEM_64BITS(bar);
0224     } else {
0225         if (!(flags & IORESOURCE_PREFETCH))
0226             value |= LM_RC_BAR_CFG_CTRL_MEM_32BITS(bar);
0227         value |= LM_RC_BAR_CFG_CTRL_PREF_MEM_32BITS(bar);
0228     }
0229
0230     value |= LM_RC_BAR_CFG_APERTURE(bar, aperture);
0231     cdns_pcie_writel(pcie, CDNS_PCIE_LM_RC_BAR_CFG, value);
0232
0233     return 0;
0234 }
0235
0236 static enum cdns_pcie_rp_bar
0237 cdns_pcie_host_find_min_bar(struct cdns_pcie_rc *rc, u64 size)
0238 {
0239     enum cdns_pcie_rp_bar bar, sel_bar;
0240
0241     sel_bar = RP_BAR_UNDEFINED;
0242     for (bar = RP_BAR0; bar <= RP_NO_BAR; bar++) {
0243         if (!rc->avail_ib_bar[bar])
0244             continue;
0245
0246         if (size <= bar_max_size[bar]) {
0247             if (sel_bar == RP_BAR_UNDEFINED) {
0248                 sel_bar = bar;
0249                 continue;
0250             }
0251
0252             if (bar_max_size[bar] < bar_max_size[sel_bar])
0253                 sel_bar = bar;
0254         }
0255     }
0256
0257     return sel_bar;
0258 }
0259
0260 static enum cdns_pcie_rp_bar
0261 cdns_pcie_host_find_max_bar(struct cdns_pcie_rc *rc, u64 size)
0262 {
0263     enum cdns_pcie_rp_bar bar, sel_bar;
0264
0265     sel_bar = RP_BAR_UNDEFINED;
0266     for (bar = RP_BAR0; bar <= RP_NO_BAR; bar++) {
0267         if (!rc->avail_ib_bar[bar])
0268             continue;
0269
0270         if (size >= bar_max_size[bar]) {
0271             if (sel_bar == RP_BAR_UNDEFINED) {
0272                 sel_bar = bar;
0273                 continue;
0274             }
0275
0276             if (bar_max_size[bar] > bar_max_size[sel_bar])
0277                 sel_bar = bar;
0278         }
0279     }
0280
0281     return sel_bar;
0282 }
0283
0284 static int cdns_pcie_host_bar_config(struct cdns_pcie_rc *rc,
0285                      struct resource_entry *entry)
0286 {
0287     u64 cpu_addr, pci_addr, size, winsize;
0288     struct cdns_pcie *pcie = &rc->pcie;
0289     struct device *dev = pcie->dev;
0290     enum cdns_pcie_rp_bar bar;
0291     unsigned long flags;
0292     int ret;
0293
0294     cpu_addr = entry->res->start;
0295     pci_addr = entry->res->start - entry->offset;
0296     flags = entry->res->flags;
0297     size = resource_size(entry->res);
0298
0299     if (entry->offset) {
0300         dev_err(dev, "PCI addr: %llx must be equal to CPU addr: %llx\n",
0301             pci_addr, cpu_addr);
0302         return -EINVAL;
0303     }
0304
0305     while (size > 0) {
0306         /*
0307          * Try to find a minimum BAR whose size is greater than
0308          * or equal to the remaining resource_entry size. This will
0309          * fail if the size of each of the available BARs is less than
0310          * the remaining resource_entry size.
0311          * If a minimum BAR is found, IB ATU will be configured and
0312          * exited.
0313          */
0314         bar = cdns_pcie_host_find_min_bar(rc, size);
0315         if (bar != RP_BAR_UNDEFINED) {
0316             ret = cdns_pcie_host_bar_ib_config(rc, bar, cpu_addr,
0317                                size, flags);
0318             if (ret)
0319                 dev_err(dev, "IB BAR: %d config failed\n", bar);
0320             return ret;
0321         }
0322
0323         /*
0324          * If the control reaches here, it would mean the remaining
0325          * resource_entry size cannot be fitted in a single BAR. So we
0326          * find a maximum BAR whose size is less than or equal to the
0327          * remaining resource_entry size and split the resource entry
0328          * so that part of resource entry is fitted inside the maximum
0329          * BAR. The remaining size would be fitted during the next
0330          * iteration of the loop.
0331          * If a maximum BAR is not found, there is no way we can fit
0332          * this resource_entry, so we error out.
0333          */
0334         bar = cdns_pcie_host_find_max_bar(rc, size);
0335         if (bar == RP_BAR_UNDEFINED) {
0336             dev_err(dev, "No free BAR to map cpu_addr %llx\n",
0337                 cpu_addr);
0338             return -EINVAL;
0339         }
0340
0341         winsize = bar_max_size[bar];
0342         ret = cdns_pcie_host_bar_ib_config(rc, bar, cpu_addr, winsize,
0343                            flags);
0344         if (ret) {
0345             dev_err(dev, "IB BAR: %d config failed\n", bar);
0346             return ret;
0347         }
0348
0349         size -= winsize;
0350         cpu_addr += winsize;
0351     }
0352
0353     return 0;
0354 }
0355
0356 static int cdns_pcie_host_dma_ranges_cmp(void *priv, const struct list_head *a,
0357                      const struct list_head *b)
0358 {
0359     struct resource_entry *entry1, *entry2;
0360
0361         entry1 = container_of(a, struct resource_entry, node);
0362         entry2 = container_of(b, struct resource_entry, node);
0363
0364         return resource_size(entry2->res) - resource_size(entry1->res);
0365 }
0366
0367 static int cdns_pcie_host_map_dma_ranges(struct cdns_pcie_rc *rc)
0368 {
0369     struct cdns_pcie *pcie = &rc->pcie;
0370     struct device *dev = pcie->dev;
0371     struct device_node *np = dev->of_node;
0372     struct pci_host_bridge *bridge;
0373     struct resource_entry *entry;
0374     u32 no_bar_nbits = 32;
0375     int err;
0376
0377     bridge = pci_host_bridge_from_priv(rc);
0378     if (!bridge)
0379         return -ENOMEM;
0380
0381     if (list_empty(&bridge->dma_ranges)) {
0382         of_property_read_u32(np, "cdns,no-bar-match-nbits",
0383                      &no_bar_nbits);
0384         err = cdns_pcie_host_bar_ib_config(rc, RP_NO_BAR, 0x0,
0385                            (u64)1 << no_bar_nbits, 0);
0386         if (err)
0387             dev_err(dev, "IB BAR: %d config failed\n", RP_NO_BAR);
0388         return err;
0389     }
0390
0391     list_sort(NULL, &bridge->dma_ranges, cdns_pcie_host_dma_ranges_cmp);
0392
0393     resource_list_for_each_entry(entry, &bridge->dma_ranges) {
0394         err = cdns_pcie_host_bar_config(rc, entry);
0395         if (err) {
0396             dev_err(dev, "Fail to configure IB using dma-ranges\n");
0397             return err;
0398         }
0399     }
0400
0401     return 0;
0402 }
0403
0404 static int cdns_pcie_host_init_address_translation(struct cdns_pcie_rc *rc)
0405 {
0406     struct cdns_pcie *pcie = &rc->pcie;
0407     struct pci_host_bridge *bridge = pci_host_bridge_from_priv(rc);
0408     struct resource *cfg_res = rc->cfg_res;
0409     struct resource_entry *entry;
0410     u64 cpu_addr = cfg_res->start;
0411     u32 addr0, addr1, desc1;
0412     int r, busnr = 0;
0413
0414     entry = resource_list_first_type(&bridge->windows, IORESOURCE_BUS);
0415     if (entry)
0416         busnr = entry->res->start;
0417
0418     /*
0419      * Reserve region 0 for PCI configure space accesses:
0420      * OB_REGION_PCI_ADDR0 and OB_REGION_DESC0 are updated dynamically by
0421      * cdns_pci_map_bus(), other region registers are set here once for all.
0422      */
0423     addr1 = 0; /* Should be programmed to zero. */
0424     desc1 = CDNS_PCIE_AT_OB_REGION_DESC1_BUS(busnr);
0425     cdns_pcie_writel(pcie, CDNS_PCIE_AT_OB_REGION_PCI_ADDR1(0), addr1);
0426     cdns_pcie_writel(pcie, CDNS_PCIE_AT_OB_REGION_DESC1(0), desc1);
0427
0428     if (pcie->ops->cpu_addr_fixup)
0429         cpu_addr = pcie->ops->cpu_addr_fixup(pcie, cpu_addr);
0430
0431     addr0 = CDNS_PCIE_AT_OB_REGION_CPU_ADDR0_NBITS(12) |
0432         (lower_32_bits(cpu_addr) & GENMASK(31, 8));
0433     addr1 = upper_32_bits(cpu_addr);
0434     cdns_pcie_writel(pcie, CDNS_PCIE_AT_OB_REGION_CPU_ADDR0(0), addr0);
0435     cdns_pcie_writel(pcie, CDNS_PCIE_AT_OB_REGION_CPU_ADDR1(0), addr1);
0436
0437     r = 1;
0438     resource_list_for_each_entry(entry, &bridge->windows) {
0439         struct resource *res = entry->res;
0440         u64 pci_addr = res->start - entry->offset;
0441
0442         if (resource_type(res) == IORESOURCE_IO)
0443             cdns_pcie_set_outbound_region(pcie, busnr, 0, r,
0444                               true,
0445                               pci_pio_to_address(res->start),
0446                               pci_addr,
0447                               resource_size(res));
0448         else
0449             cdns_pcie_set_outbound_region(pcie, busnr, 0, r,
0450                               false,
0451                               res->start,
0452                               pci_addr,
0453                               resource_size(res));
0454
0455         r++;
0456     }
0457
0458     return cdns_pcie_host_map_dma_ranges(rc);
0459 }
0460
0461 static int cdns_pcie_host_init(struct device *dev,
0462                    struct cdns_pcie_rc *rc)
0463 {
0464     int err;
0465
0466     err = cdns_pcie_host_init_root_port(rc);
0467     if (err)
0468         return err;
0469
0470     return cdns_pcie_host_init_address_translation(rc);
0471 }
0472
0473 int cdns_pcie_host_setup(struct cdns_pcie_rc *rc)
0474 {
0475     struct device *dev = rc->pcie.dev;
0476     struct platform_device *pdev = to_platform_device(dev);
0477     struct device_node *np = dev->of_node;
0478     struct pci_host_bridge *bridge;
0479     enum cdns_pcie_rp_bar bar;
0480     struct cdns_pcie *pcie;
0481     struct resource *res;
0482     int ret;
0483
0484     bridge = pci_host_bridge_from_priv(rc);
0485     if (!bridge)
0486         return -ENOMEM;
0487
0488     pcie = &rc->pcie;
0489     pcie->is_rc = true;
0490
0491     rc->vendor_id = 0xffff;
0492     of_property_read_u32(np, "vendor-id", &rc->vendor_id);
0493
0494     rc->device_id = 0xffff;
0495     of_property_read_u32(np, "device-id", &rc->device_id);
0496
0497     pcie->reg_base = devm_platform_ioremap_resource_byname(pdev, "reg");
0498     if (IS_ERR(pcie->reg_base)) {
0499         dev_err(dev, "missing \"reg\"\n");
0500         return PTR_ERR(pcie->reg_base);
0501     }
0502
0503     res = platform_get_resource_byname(pdev, IORESOURCE_MEM, "cfg");
0504     rc->cfg_base = devm_pci_remap_cfg_resource(dev, res);
0505     if (IS_ERR(rc->cfg_base))
0506         return PTR_ERR(rc->cfg_base);
0507     rc->cfg_res = res;
0508
0509     if (rc->quirk_detect_quiet_flag)
0510         cdns_pcie_detect_quiet_min_delay_set(&rc->pcie);
0511
0512     cdns_pcie_host_enable_ptm_response(pcie);
0513
0514     ret = cdns_pcie_start_link(pcie);
0515     if (ret) {
0516         dev_err(dev, "Failed to start link\n");
0517         return ret;
0518     }
0519
0520     ret = cdns_pcie_host_start_link(rc);
0521     if (ret)
0522         dev_dbg(dev, "PCIe link never came up\n");
0523
0524     for (bar = RP_BAR0; bar <= RP_NO_BAR; bar++)
0525         rc->avail_ib_bar[bar] = true;
0526
0527     ret = cdns_pcie_host_init(dev, rc);
0528     if (ret)
0529         return ret;
0530
0531     if (!bridge->ops)
0532         bridge->ops = &cdns_pcie_host_ops;
0533
0534     ret = pci_host_probe(bridge);
0535     if (ret < 0)
0536         goto err_init;
0537
0538     return 0;
0539
0540  err_init:
0541     pm_runtime_put_sync(dev);
0542
0543     return ret;
0544 }