Back to home page

OSCL-LXR
 
 

     

0001 // SPDX-License-Identifier: GPL-2.0
0002 /*
0003  * PCIe host bridge driver for Apple system-on-chips.
0004  *
0005  * The HW is ECAM compliant, so once the controller is initialized,
0006  * the driver mostly deals MSI mapping and handling of per-port
0007  * interrupts (INTx, management and error signals).
0008  *
0009  * Initialization requires enabling power and clocks, along with a
0010  * number of register pokes.
0011  *
0012  * Copyright (C) 2021 Alyssa Rosenzweig <alyssa@rosenzweig.io>
0013  * Copyright (C) 2021 Google LLC
0014  * Copyright (C) 2021 Corellium LLC
0015  * Copyright (C) 2021 Mark Kettenis <kettenis@openbsd.org>
0016  *
0017  * Author: Alyssa Rosenzweig <alyssa@rosenzweig.io>
0018  * Author: Marc Zyngier <maz@kernel.org>
0019  */
0020 
0021 #include <linux/gpio/consumer.h>
0022 #include <linux/kernel.h>
0023 #include <linux/iopoll.h>
0024 #include <linux/irqchip/chained_irq.h>
0025 #include <linux/irqdomain.h>
0026 #include <linux/list.h>
0027 #include <linux/module.h>
0028 #include <linux/msi.h>
0029 #include <linux/notifier.h>
0030 #include <linux/of_irq.h>
0031 #include <linux/pci-ecam.h>
0032 
0033 #define CORE_RC_PHYIF_CTL       0x00024
0034 #define   CORE_RC_PHYIF_CTL_RUN     BIT(0)
0035 #define CORE_RC_PHYIF_STAT      0x00028
0036 #define   CORE_RC_PHYIF_STAT_REFCLK BIT(4)
0037 #define CORE_RC_CTL         0x00050
0038 #define   CORE_RC_CTL_RUN       BIT(0)
0039 #define CORE_RC_STAT            0x00058
0040 #define   CORE_RC_STAT_READY        BIT(0)
0041 #define CORE_FABRIC_STAT        0x04000
0042 #define   CORE_FABRIC_STAT_MASK     0x001F001F
0043 #define CORE_LANE_CFG(port)     (0x84000 + 0x4000 * (port))
0044 #define   CORE_LANE_CFG_REFCLK0REQ  BIT(0)
0045 #define   CORE_LANE_CFG_REFCLK1REQ  BIT(1)
0046 #define   CORE_LANE_CFG_REFCLK0ACK  BIT(2)
0047 #define   CORE_LANE_CFG_REFCLK1ACK  BIT(3)
0048 #define   CORE_LANE_CFG_REFCLKEN    (BIT(9) | BIT(10))
0049 #define CORE_LANE_CTL(port)     (0x84004 + 0x4000 * (port))
0050 #define   CORE_LANE_CTL_CFGACC      BIT(15)
0051 
0052 #define PORT_LTSSMCTL           0x00080
0053 #define   PORT_LTSSMCTL_START       BIT(0)
0054 #define PORT_INTSTAT            0x00100
0055 #define   PORT_INT_TUNNEL_ERR       31
0056 #define   PORT_INT_CPL_TIMEOUT      23
0057 #define   PORT_INT_RID2SID_MAPERR   22
0058 #define   PORT_INT_CPL_ABORT        21
0059 #define   PORT_INT_MSI_BAD_DATA     19
0060 #define   PORT_INT_MSI_ERR      18
0061 #define   PORT_INT_REQADDR_GT32     17
0062 #define   PORT_INT_AF_TIMEOUT       15
0063 #define   PORT_INT_LINK_DOWN        14
0064 #define   PORT_INT_LINK_UP      12
0065 #define   PORT_INT_LINK_BWMGMT      11
0066 #define   PORT_INT_AER_MASK     (15 << 4)
0067 #define   PORT_INT_PORT_ERR     4
0068 #define   PORT_INT_INTx(i)      i
0069 #define   PORT_INT_INTx_MASK        15
0070 #define PORT_INTMSK         0x00104
0071 #define PORT_INTMSKSET          0x00108
0072 #define PORT_INTMSKCLR          0x0010c
0073 #define PORT_MSICFG         0x00124
0074 #define   PORT_MSICFG_EN        BIT(0)
0075 #define   PORT_MSICFG_L2MSINUM_SHIFT    4
0076 #define PORT_MSIBASE            0x00128
0077 #define   PORT_MSIBASE_1_SHIFT      16
0078 #define PORT_MSIADDR            0x00168
0079 #define PORT_LINKSTS            0x00208
0080 #define   PORT_LINKSTS_UP       BIT(0)
0081 #define   PORT_LINKSTS_BUSY     BIT(2)
0082 #define PORT_LINKCMDSTS         0x00210
0083 #define PORT_OUTS_NPREQS        0x00284
0084 #define   PORT_OUTS_NPREQS_REQ      BIT(24)
0085 #define   PORT_OUTS_NPREQS_CPL      BIT(16)
0086 #define PORT_RXWR_FIFO          0x00288
0087 #define   PORT_RXWR_FIFO_HDR        GENMASK(15, 10)
0088 #define   PORT_RXWR_FIFO_DATA       GENMASK(9, 0)
0089 #define PORT_RXRD_FIFO          0x0028C
0090 #define   PORT_RXRD_FIFO_REQ        GENMASK(6, 0)
0091 #define PORT_OUTS_CPLS          0x00290
0092 #define   PORT_OUTS_CPLS_SHRD       GENMASK(14, 8)
0093 #define   PORT_OUTS_CPLS_WAIT       GENMASK(6, 0)
0094 #define PORT_APPCLK         0x00800
0095 #define   PORT_APPCLK_EN        BIT(0)
0096 #define   PORT_APPCLK_CGDIS     BIT(8)
0097 #define PORT_STATUS         0x00804
0098 #define   PORT_STATUS_READY     BIT(0)
0099 #define PORT_REFCLK         0x00810
0100 #define   PORT_REFCLK_EN        BIT(0)
0101 #define   PORT_REFCLK_CGDIS     BIT(8)
0102 #define PORT_PERST          0x00814
0103 #define   PORT_PERST_OFF        BIT(0)
0104 #define PORT_RID2SID(i16)       (0x00828 + 4 * (i16))
0105 #define   PORT_RID2SID_VALID        BIT(31)
0106 #define   PORT_RID2SID_SID_SHIFT    16
0107 #define   PORT_RID2SID_BUS_SHIFT    8
0108 #define   PORT_RID2SID_DEV_SHIFT    3
0109 #define   PORT_RID2SID_FUNC_SHIFT   0
0110 #define PORT_OUTS_PREQS_HDR     0x00980
0111 #define   PORT_OUTS_PREQS_HDR_MASK  GENMASK(9, 0)
0112 #define PORT_OUTS_PREQS_DATA        0x00984
0113 #define   PORT_OUTS_PREQS_DATA_MASK GENMASK(15, 0)
0114 #define PORT_TUNCTRL            0x00988
0115 #define   PORT_TUNCTRL_PERST_ON     BIT(0)
0116 #define   PORT_TUNCTRL_PERST_ACK_REQ    BIT(1)
0117 #define PORT_TUNSTAT            0x0098c
0118 #define   PORT_TUNSTAT_PERST_ON     BIT(0)
0119 #define   PORT_TUNSTAT_PERST_ACK_PEND   BIT(1)
0120 #define PORT_PREFMEM_ENABLE     0x00994
0121 
0122 #define MAX_RID2SID         64
0123 
0124 /*
0125  * The doorbell address is set to 0xfffff000, which by convention
0126  * matches what MacOS does, and it is possible to use any other
0127  * address (in the bottom 4GB, as the base register is only 32bit).
0128  * However, it has to be excluded from the IOVA range, and the DART
0129  * driver has to know about it.
0130  */
0131 #define DOORBELL_ADDR       CONFIG_PCIE_APPLE_MSI_DOORBELL_ADDR
0132 
0133 struct apple_pcie {
0134     struct mutex        lock;
0135     struct device       *dev;
0136     void __iomem            *base;
0137     struct irq_domain   *domain;
0138     unsigned long       *bitmap;
0139     struct list_head    ports;
0140     struct completion   event;
0141     struct irq_fwspec   fwspec;
0142     u32         nvecs;
0143 };
0144 
0145 struct apple_pcie_port {
0146     struct apple_pcie   *pcie;
0147     struct device_node  *np;
0148     void __iomem        *base;
0149     struct irq_domain   *domain;
0150     struct list_head    entry;
0151     DECLARE_BITMAP(sid_map, MAX_RID2SID);
0152     int         sid_map_sz;
0153     int         idx;
0154 };
0155 
0156 static void rmw_set(u32 set, void __iomem *addr)
0157 {
0158     writel_relaxed(readl_relaxed(addr) | set, addr);
0159 }
0160 
0161 static void rmw_clear(u32 clr, void __iomem *addr)
0162 {
0163     writel_relaxed(readl_relaxed(addr) & ~clr, addr);
0164 }
0165 
0166 static void apple_msi_top_irq_mask(struct irq_data *d)
0167 {
0168     pci_msi_mask_irq(d);
0169     irq_chip_mask_parent(d);
0170 }
0171 
0172 static void apple_msi_top_irq_unmask(struct irq_data *d)
0173 {
0174     pci_msi_unmask_irq(d);
0175     irq_chip_unmask_parent(d);
0176 }
0177 
0178 static struct irq_chip apple_msi_top_chip = {
0179     .name           = "PCIe MSI",
0180     .irq_mask       = apple_msi_top_irq_mask,
0181     .irq_unmask     = apple_msi_top_irq_unmask,
0182     .irq_eoi        = irq_chip_eoi_parent,
0183     .irq_set_affinity   = irq_chip_set_affinity_parent,
0184     .irq_set_type       = irq_chip_set_type_parent,
0185 };
0186 
0187 static void apple_msi_compose_msg(struct irq_data *data, struct msi_msg *msg)
0188 {
0189     msg->address_hi = upper_32_bits(DOORBELL_ADDR);
0190     msg->address_lo = lower_32_bits(DOORBELL_ADDR);
0191     msg->data = data->hwirq;
0192 }
0193 
0194 static struct irq_chip apple_msi_bottom_chip = {
0195     .name           = "MSI",
0196     .irq_mask       = irq_chip_mask_parent,
0197     .irq_unmask     = irq_chip_unmask_parent,
0198     .irq_eoi        = irq_chip_eoi_parent,
0199     .irq_set_affinity   = irq_chip_set_affinity_parent,
0200     .irq_set_type       = irq_chip_set_type_parent,
0201     .irq_compose_msi_msg    = apple_msi_compose_msg,
0202 };
0203 
0204 static int apple_msi_domain_alloc(struct irq_domain *domain, unsigned int virq,
0205                   unsigned int nr_irqs, void *args)
0206 {
0207     struct apple_pcie *pcie = domain->host_data;
0208     struct irq_fwspec fwspec = pcie->fwspec;
0209     unsigned int i;
0210     int ret, hwirq;
0211 
0212     mutex_lock(&pcie->lock);
0213 
0214     hwirq = bitmap_find_free_region(pcie->bitmap, pcie->nvecs,
0215                     order_base_2(nr_irqs));
0216 
0217     mutex_unlock(&pcie->lock);
0218 
0219     if (hwirq < 0)
0220         return -ENOSPC;
0221 
0222     fwspec.param[fwspec.param_count - 2] += hwirq;
0223 
0224     ret = irq_domain_alloc_irqs_parent(domain, virq, nr_irqs, &fwspec);
0225     if (ret)
0226         return ret;
0227 
0228     for (i = 0; i < nr_irqs; i++) {
0229         irq_domain_set_hwirq_and_chip(domain, virq + i, hwirq + i,
0230                           &apple_msi_bottom_chip,
0231                           domain->host_data);
0232     }
0233 
0234     return 0;
0235 }
0236 
0237 static void apple_msi_domain_free(struct irq_domain *domain, unsigned int virq,
0238                   unsigned int nr_irqs)
0239 {
0240     struct irq_data *d = irq_domain_get_irq_data(domain, virq);
0241     struct apple_pcie *pcie = domain->host_data;
0242 
0243     mutex_lock(&pcie->lock);
0244 
0245     bitmap_release_region(pcie->bitmap, d->hwirq, order_base_2(nr_irqs));
0246 
0247     mutex_unlock(&pcie->lock);
0248 }
0249 
0250 static const struct irq_domain_ops apple_msi_domain_ops = {
0251     .alloc  = apple_msi_domain_alloc,
0252     .free   = apple_msi_domain_free,
0253 };
0254 
0255 static struct msi_domain_info apple_msi_info = {
0256     .flags  = (MSI_FLAG_USE_DEF_DOM_OPS | MSI_FLAG_USE_DEF_CHIP_OPS |
0257            MSI_FLAG_MULTI_PCI_MSI | MSI_FLAG_PCI_MSIX),
0258     .chip   = &apple_msi_top_chip,
0259 };
0260 
0261 static void apple_port_irq_mask(struct irq_data *data)
0262 {
0263     struct apple_pcie_port *port = irq_data_get_irq_chip_data(data);
0264 
0265     writel_relaxed(BIT(data->hwirq), port->base + PORT_INTMSKSET);
0266 }
0267 
0268 static void apple_port_irq_unmask(struct irq_data *data)
0269 {
0270     struct apple_pcie_port *port = irq_data_get_irq_chip_data(data);
0271 
0272     writel_relaxed(BIT(data->hwirq), port->base + PORT_INTMSKCLR);
0273 }
0274 
0275 static bool hwirq_is_intx(unsigned int hwirq)
0276 {
0277     return BIT(hwirq) & PORT_INT_INTx_MASK;
0278 }
0279 
0280 static void apple_port_irq_ack(struct irq_data *data)
0281 {
0282     struct apple_pcie_port *port = irq_data_get_irq_chip_data(data);
0283 
0284     if (!hwirq_is_intx(data->hwirq))
0285         writel_relaxed(BIT(data->hwirq), port->base + PORT_INTSTAT);
0286 }
0287 
0288 static int apple_port_irq_set_type(struct irq_data *data, unsigned int type)
0289 {
0290     /*
0291      * It doesn't seem that there is any way to configure the
0292      * trigger, so assume INTx have to be level (as per the spec),
0293      * and the rest is edge (which looks likely).
0294      */
0295     if (hwirq_is_intx(data->hwirq) ^ !!(type & IRQ_TYPE_LEVEL_MASK))
0296         return -EINVAL;
0297 
0298     irqd_set_trigger_type(data, type);
0299     return 0;
0300 }
0301 
0302 static struct irq_chip apple_port_irqchip = {
0303     .name       = "PCIe",
0304     .irq_ack    = apple_port_irq_ack,
0305     .irq_mask   = apple_port_irq_mask,
0306     .irq_unmask = apple_port_irq_unmask,
0307     .irq_set_type   = apple_port_irq_set_type,
0308 };
0309 
0310 static int apple_port_irq_domain_alloc(struct irq_domain *domain,
0311                        unsigned int virq, unsigned int nr_irqs,
0312                        void *args)
0313 {
0314     struct apple_pcie_port *port = domain->host_data;
0315     struct irq_fwspec *fwspec = args;
0316     int i;
0317 
0318     for (i = 0; i < nr_irqs; i++) {
0319         irq_flow_handler_t flow = handle_edge_irq;
0320         unsigned int type = IRQ_TYPE_EDGE_RISING;
0321 
0322         if (hwirq_is_intx(fwspec->param[0] + i)) {
0323             flow = handle_level_irq;
0324             type = IRQ_TYPE_LEVEL_HIGH;
0325         }
0326 
0327         irq_domain_set_info(domain, virq + i, fwspec->param[0] + i,
0328                     &apple_port_irqchip, port, flow,
0329                     NULL, NULL);
0330 
0331         irq_set_irq_type(virq + i, type);
0332     }
0333 
0334     return 0;
0335 }
0336 
0337 static void apple_port_irq_domain_free(struct irq_domain *domain,
0338                        unsigned int virq, unsigned int nr_irqs)
0339 {
0340     int i;
0341 
0342     for (i = 0; i < nr_irqs; i++) {
0343         struct irq_data *d = irq_domain_get_irq_data(domain, virq + i);
0344 
0345         irq_set_handler(virq + i, NULL);
0346         irq_domain_reset_irq_data(d);
0347     }
0348 }
0349 
0350 static const struct irq_domain_ops apple_port_irq_domain_ops = {
0351     .translate  = irq_domain_translate_onecell,
0352     .alloc      = apple_port_irq_domain_alloc,
0353     .free       = apple_port_irq_domain_free,
0354 };
0355 
0356 static void apple_port_irq_handler(struct irq_desc *desc)
0357 {
0358     struct apple_pcie_port *port = irq_desc_get_handler_data(desc);
0359     struct irq_chip *chip = irq_desc_get_chip(desc);
0360     unsigned long stat;
0361     int i;
0362 
0363     chained_irq_enter(chip, desc);
0364 
0365     stat = readl_relaxed(port->base + PORT_INTSTAT);
0366 
0367     for_each_set_bit(i, &stat, 32)
0368         generic_handle_domain_irq(port->domain, i);
0369 
0370     chained_irq_exit(chip, desc);
0371 }
0372 
0373 static int apple_pcie_port_setup_irq(struct apple_pcie_port *port)
0374 {
0375     struct fwnode_handle *fwnode = &port->np->fwnode;
0376     unsigned int irq;
0377 
0378     /* FIXME: consider moving each interrupt under each port */
0379     irq = irq_of_parse_and_map(to_of_node(dev_fwnode(port->pcie->dev)),
0380                    port->idx);
0381     if (!irq)
0382         return -ENXIO;
0383 
0384     port->domain = irq_domain_create_linear(fwnode, 32,
0385                         &apple_port_irq_domain_ops,
0386                         port);
0387     if (!port->domain)
0388         return -ENOMEM;
0389 
0390     /* Disable all interrupts */
0391     writel_relaxed(~0, port->base + PORT_INTMSKSET);
0392     writel_relaxed(~0, port->base + PORT_INTSTAT);
0393 
0394     irq_set_chained_handler_and_data(irq, apple_port_irq_handler, port);
0395 
0396     /* Configure MSI base address */
0397     BUILD_BUG_ON(upper_32_bits(DOORBELL_ADDR));
0398     writel_relaxed(lower_32_bits(DOORBELL_ADDR), port->base + PORT_MSIADDR);
0399 
0400     /* Enable MSIs, shared between all ports */
0401     writel_relaxed(0, port->base + PORT_MSIBASE);
0402     writel_relaxed((ilog2(port->pcie->nvecs) << PORT_MSICFG_L2MSINUM_SHIFT) |
0403                PORT_MSICFG_EN, port->base + PORT_MSICFG);
0404 
0405     return 0;
0406 }
0407 
0408 static irqreturn_t apple_pcie_port_irq(int irq, void *data)
0409 {
0410     struct apple_pcie_port *port = data;
0411     unsigned int hwirq = irq_domain_get_irq_data(port->domain, irq)->hwirq;
0412 
0413     switch (hwirq) {
0414     case PORT_INT_LINK_UP:
0415         dev_info_ratelimited(port->pcie->dev, "Link up on %pOF\n",
0416                      port->np);
0417         complete_all(&port->pcie->event);
0418         break;
0419     case PORT_INT_LINK_DOWN:
0420         dev_info_ratelimited(port->pcie->dev, "Link down on %pOF\n",
0421                      port->np);
0422         break;
0423     default:
0424         return IRQ_NONE;
0425     }
0426 
0427     return IRQ_HANDLED;
0428 }
0429 
0430 static int apple_pcie_port_register_irqs(struct apple_pcie_port *port)
0431 {
0432     static struct {
0433         unsigned int    hwirq;
0434         const char  *name;
0435     } port_irqs[] = {
0436         { PORT_INT_LINK_UP, "Link up",  },
0437         { PORT_INT_LINK_DOWN,   "Link down",    },
0438     };
0439     int i;
0440 
0441     for (i = 0; i < ARRAY_SIZE(port_irqs); i++) {
0442         struct irq_fwspec fwspec = {
0443             .fwnode     = &port->np->fwnode,
0444             .param_count    = 1,
0445             .param      = {
0446                 [0] = port_irqs[i].hwirq,
0447             },
0448         };
0449         unsigned int irq;
0450         int ret;
0451 
0452         irq = irq_domain_alloc_irqs(port->domain, 1, NUMA_NO_NODE,
0453                         &fwspec);
0454         if (WARN_ON(!irq))
0455             continue;
0456 
0457         ret = request_irq(irq, apple_pcie_port_irq, 0,
0458                   port_irqs[i].name, port);
0459         WARN_ON(ret);
0460     }
0461 
0462     return 0;
0463 }
0464 
0465 static int apple_pcie_setup_refclk(struct apple_pcie *pcie,
0466                    struct apple_pcie_port *port)
0467 {
0468     u32 stat;
0469     int res;
0470 
0471     res = readl_relaxed_poll_timeout(pcie->base + CORE_RC_PHYIF_STAT, stat,
0472                      stat & CORE_RC_PHYIF_STAT_REFCLK,
0473                      100, 50000);
0474     if (res < 0)
0475         return res;
0476 
0477     rmw_set(CORE_LANE_CTL_CFGACC, pcie->base + CORE_LANE_CTL(port->idx));
0478     rmw_set(CORE_LANE_CFG_REFCLK0REQ, pcie->base + CORE_LANE_CFG(port->idx));
0479 
0480     res = readl_relaxed_poll_timeout(pcie->base + CORE_LANE_CFG(port->idx),
0481                      stat, stat & CORE_LANE_CFG_REFCLK0ACK,
0482                      100, 50000);
0483     if (res < 0)
0484         return res;
0485 
0486     rmw_set(CORE_LANE_CFG_REFCLK1REQ, pcie->base + CORE_LANE_CFG(port->idx));
0487     res = readl_relaxed_poll_timeout(pcie->base + CORE_LANE_CFG(port->idx),
0488                      stat, stat & CORE_LANE_CFG_REFCLK1ACK,
0489                      100, 50000);
0490 
0491     if (res < 0)
0492         return res;
0493 
0494     rmw_clear(CORE_LANE_CTL_CFGACC, pcie->base + CORE_LANE_CTL(port->idx));
0495 
0496     rmw_set(CORE_LANE_CFG_REFCLKEN, pcie->base + CORE_LANE_CFG(port->idx));
0497     rmw_set(PORT_REFCLK_EN, port->base + PORT_REFCLK);
0498 
0499     return 0;
0500 }
0501 
0502 static u32 apple_pcie_rid2sid_write(struct apple_pcie_port *port,
0503                     int idx, u32 val)
0504 {
0505     writel_relaxed(val, port->base + PORT_RID2SID(idx));
0506     /* Read back to ensure completion of the write */
0507     return readl_relaxed(port->base + PORT_RID2SID(idx));
0508 }
0509 
0510 static int apple_pcie_setup_port(struct apple_pcie *pcie,
0511                  struct device_node *np)
0512 {
0513     struct platform_device *platform = to_platform_device(pcie->dev);
0514     struct apple_pcie_port *port;
0515     struct gpio_desc *reset;
0516     u32 stat, idx;
0517     int ret, i;
0518 
0519     reset = gpiod_get_from_of_node(np, "reset-gpios", 0,
0520                        GPIOD_OUT_LOW, "PERST#");
0521     if (IS_ERR(reset))
0522         return PTR_ERR(reset);
0523 
0524     port = devm_kzalloc(pcie->dev, sizeof(*port), GFP_KERNEL);
0525     if (!port)
0526         return -ENOMEM;
0527 
0528     ret = of_property_read_u32_index(np, "reg", 0, &idx);
0529     if (ret)
0530         return ret;
0531 
0532     /* Use the first reg entry to work out the port index */
0533     port->idx = idx >> 11;
0534     port->pcie = pcie;
0535     port->np = np;
0536 
0537     port->base = devm_platform_ioremap_resource(platform, port->idx + 2);
0538     if (IS_ERR(port->base))
0539         return PTR_ERR(port->base);
0540 
0541     rmw_set(PORT_APPCLK_EN, port->base + PORT_APPCLK);
0542 
0543     /* Assert PERST# before setting up the clock */
0544     gpiod_set_value(reset, 1);
0545 
0546     ret = apple_pcie_setup_refclk(pcie, port);
0547     if (ret < 0)
0548         return ret;
0549 
0550     /* The minimal Tperst-clk value is 100us (PCIe CEM r5.0, 2.9.2) */
0551     usleep_range(100, 200);
0552 
0553     /* Deassert PERST# */
0554     rmw_set(PORT_PERST_OFF, port->base + PORT_PERST);
0555     gpiod_set_value(reset, 0);
0556 
0557     /* Wait for 100ms after PERST# deassertion (PCIe r5.0, 6.6.1) */
0558     msleep(100);
0559 
0560     ret = readl_relaxed_poll_timeout(port->base + PORT_STATUS, stat,
0561                      stat & PORT_STATUS_READY, 100, 250000);
0562     if (ret < 0) {
0563         dev_err(pcie->dev, "port %pOF ready wait timeout\n", np);
0564         return ret;
0565     }
0566 
0567     rmw_clear(PORT_REFCLK_CGDIS, port->base + PORT_REFCLK);
0568     rmw_clear(PORT_APPCLK_CGDIS, port->base + PORT_APPCLK);
0569 
0570     ret = apple_pcie_port_setup_irq(port);
0571     if (ret)
0572         return ret;
0573 
0574     /* Reset all RID/SID mappings, and check for RAZ/WI registers */
0575     for (i = 0; i < MAX_RID2SID; i++) {
0576         if (apple_pcie_rid2sid_write(port, i, 0xbad1d) != 0xbad1d)
0577             break;
0578         apple_pcie_rid2sid_write(port, i, 0);
0579     }
0580 
0581     dev_dbg(pcie->dev, "%pOF: %d RID/SID mapping entries\n", np, i);
0582 
0583     port->sid_map_sz = i;
0584 
0585     list_add_tail(&port->entry, &pcie->ports);
0586     init_completion(&pcie->event);
0587 
0588     ret = apple_pcie_port_register_irqs(port);
0589     WARN_ON(ret);
0590 
0591     writel_relaxed(PORT_LTSSMCTL_START, port->base + PORT_LTSSMCTL);
0592 
0593     if (!wait_for_completion_timeout(&pcie->event, HZ / 10))
0594         dev_warn(pcie->dev, "%pOF link didn't come up\n", np);
0595 
0596     return 0;
0597 }
0598 
0599 static int apple_msi_init(struct apple_pcie *pcie)
0600 {
0601     struct fwnode_handle *fwnode = dev_fwnode(pcie->dev);
0602     struct of_phandle_args args = {};
0603     struct irq_domain *parent;
0604     int ret;
0605 
0606     ret = of_parse_phandle_with_args(to_of_node(fwnode), "msi-ranges",
0607                      "#interrupt-cells", 0, &args);
0608     if (ret)
0609         return ret;
0610 
0611     ret = of_property_read_u32_index(to_of_node(fwnode), "msi-ranges",
0612                      args.args_count + 1, &pcie->nvecs);
0613     if (ret)
0614         return ret;
0615 
0616     of_phandle_args_to_fwspec(args.np, args.args, args.args_count,
0617                   &pcie->fwspec);
0618 
0619     pcie->bitmap = devm_bitmap_zalloc(pcie->dev, pcie->nvecs, GFP_KERNEL);
0620     if (!pcie->bitmap)
0621         return -ENOMEM;
0622 
0623     parent = irq_find_matching_fwspec(&pcie->fwspec, DOMAIN_BUS_WIRED);
0624     if (!parent) {
0625         dev_err(pcie->dev, "failed to find parent domain\n");
0626         return -ENXIO;
0627     }
0628 
0629     parent = irq_domain_create_hierarchy(parent, 0, pcie->nvecs, fwnode,
0630                          &apple_msi_domain_ops, pcie);
0631     if (!parent) {
0632         dev_err(pcie->dev, "failed to create IRQ domain\n");
0633         return -ENOMEM;
0634     }
0635     irq_domain_update_bus_token(parent, DOMAIN_BUS_NEXUS);
0636 
0637     pcie->domain = pci_msi_create_irq_domain(fwnode, &apple_msi_info,
0638                          parent);
0639     if (!pcie->domain) {
0640         dev_err(pcie->dev, "failed to create MSI domain\n");
0641         irq_domain_remove(parent);
0642         return -ENOMEM;
0643     }
0644 
0645     return 0;
0646 }
0647 
0648 static struct apple_pcie_port *apple_pcie_get_port(struct pci_dev *pdev)
0649 {
0650     struct pci_config_window *cfg = pdev->sysdata;
0651     struct apple_pcie *pcie = cfg->priv;
0652     struct pci_dev *port_pdev;
0653     struct apple_pcie_port *port;
0654 
0655     /* Find the root port this device is on */
0656     port_pdev = pcie_find_root_port(pdev);
0657 
0658     /* If finding the port itself, nothing to do */
0659     if (WARN_ON(!port_pdev) || pdev == port_pdev)
0660         return NULL;
0661 
0662     list_for_each_entry(port, &pcie->ports, entry) {
0663         if (port->idx == PCI_SLOT(port_pdev->devfn))
0664             return port;
0665     }
0666 
0667     return NULL;
0668 }
0669 
0670 static int apple_pcie_add_device(struct apple_pcie_port *port,
0671                  struct pci_dev *pdev)
0672 {
0673     u32 sid, rid = PCI_DEVID(pdev->bus->number, pdev->devfn);
0674     int idx, err;
0675 
0676     dev_dbg(&pdev->dev, "added to bus %s, index %d\n",
0677         pci_name(pdev->bus->self), port->idx);
0678 
0679     err = of_map_id(port->pcie->dev->of_node, rid, "iommu-map",
0680             "iommu-map-mask", NULL, &sid);
0681     if (err)
0682         return err;
0683 
0684     mutex_lock(&port->pcie->lock);
0685 
0686     idx = bitmap_find_free_region(port->sid_map, port->sid_map_sz, 0);
0687     if (idx >= 0) {
0688         apple_pcie_rid2sid_write(port, idx,
0689                      PORT_RID2SID_VALID |
0690                      (sid << PORT_RID2SID_SID_SHIFT) | rid);
0691 
0692         dev_dbg(&pdev->dev, "mapping RID%x to SID%x (index %d)\n",
0693             rid, sid, idx);
0694     }
0695 
0696     mutex_unlock(&port->pcie->lock);
0697 
0698     return idx >= 0 ? 0 : -ENOSPC;
0699 }
0700 
0701 static void apple_pcie_release_device(struct apple_pcie_port *port,
0702                       struct pci_dev *pdev)
0703 {
0704     u32 rid = PCI_DEVID(pdev->bus->number, pdev->devfn);
0705     int idx;
0706 
0707     mutex_lock(&port->pcie->lock);
0708 
0709     for_each_set_bit(idx, port->sid_map, port->sid_map_sz) {
0710         u32 val;
0711 
0712         val = readl_relaxed(port->base + PORT_RID2SID(idx));
0713         if ((val & 0xffff) == rid) {
0714             apple_pcie_rid2sid_write(port, idx, 0);
0715             bitmap_release_region(port->sid_map, idx, 0);
0716             dev_dbg(&pdev->dev, "Released %x (%d)\n", val, idx);
0717             break;
0718         }
0719     }
0720 
0721     mutex_unlock(&port->pcie->lock);
0722 }
0723 
0724 static int apple_pcie_bus_notifier(struct notifier_block *nb,
0725                    unsigned long action,
0726                    void *data)
0727 {
0728     struct device *dev = data;
0729     struct pci_dev *pdev = to_pci_dev(dev);
0730     struct apple_pcie_port *port;
0731     int err;
0732 
0733     /*
0734      * This is a bit ugly. We assume that if we get notified for
0735      * any PCI device, we must be in charge of it, and that there
0736      * is no other PCI controller in the whole system. It probably
0737      * holds for now, but who knows for how long?
0738      */
0739     port = apple_pcie_get_port(pdev);
0740     if (!port)
0741         return NOTIFY_DONE;
0742 
0743     switch (action) {
0744     case BUS_NOTIFY_ADD_DEVICE:
0745         err = apple_pcie_add_device(port, pdev);
0746         if (err)
0747             return notifier_from_errno(err);
0748         break;
0749     case BUS_NOTIFY_DEL_DEVICE:
0750         apple_pcie_release_device(port, pdev);
0751         break;
0752     default:
0753         return NOTIFY_DONE;
0754     }
0755 
0756     return NOTIFY_OK;
0757 }
0758 
0759 static struct notifier_block apple_pcie_nb = {
0760     .notifier_call = apple_pcie_bus_notifier,
0761 };
0762 
0763 static int apple_pcie_init(struct pci_config_window *cfg)
0764 {
0765     struct device *dev = cfg->parent;
0766     struct platform_device *platform = to_platform_device(dev);
0767     struct device_node *of_port;
0768     struct apple_pcie *pcie;
0769     int ret;
0770 
0771     pcie = devm_kzalloc(dev, sizeof(*pcie), GFP_KERNEL);
0772     if (!pcie)
0773         return -ENOMEM;
0774 
0775     pcie->dev = dev;
0776 
0777     mutex_init(&pcie->lock);
0778 
0779     pcie->base = devm_platform_ioremap_resource(platform, 1);
0780     if (IS_ERR(pcie->base))
0781         return PTR_ERR(pcie->base);
0782 
0783     cfg->priv = pcie;
0784     INIT_LIST_HEAD(&pcie->ports);
0785 
0786     for_each_child_of_node(dev->of_node, of_port) {
0787         ret = apple_pcie_setup_port(pcie, of_port);
0788         if (ret) {
0789             dev_err(pcie->dev, "Port %pOF setup fail: %d\n", of_port, ret);
0790             of_node_put(of_port);
0791             return ret;
0792         }
0793     }
0794 
0795     return apple_msi_init(pcie);
0796 }
0797 
0798 static int apple_pcie_probe(struct platform_device *pdev)
0799 {
0800     int ret;
0801 
0802     ret = bus_register_notifier(&pci_bus_type, &apple_pcie_nb);
0803     if (ret)
0804         return ret;
0805 
0806     ret = pci_host_common_probe(pdev);
0807     if (ret)
0808         bus_unregister_notifier(&pci_bus_type, &apple_pcie_nb);
0809 
0810     return ret;
0811 }
0812 
0813 static const struct pci_ecam_ops apple_pcie_cfg_ecam_ops = {
0814     .init       = apple_pcie_init,
0815     .pci_ops    = {
0816         .map_bus    = pci_ecam_map_bus,
0817         .read       = pci_generic_config_read,
0818         .write      = pci_generic_config_write,
0819     }
0820 };
0821 
0822 static const struct of_device_id apple_pcie_of_match[] = {
0823     { .compatible = "apple,pcie", .data = &apple_pcie_cfg_ecam_ops },
0824     { }
0825 };
0826 MODULE_DEVICE_TABLE(of, apple_pcie_of_match);
0827 
0828 static struct platform_driver apple_pcie_driver = {
0829     .probe  = apple_pcie_probe,
0830     .driver = {
0831         .name           = "pcie-apple",
0832         .of_match_table     = apple_pcie_of_match,
0833         .suppress_bind_attrs    = true,
0834     },
0835 };
0836 module_platform_driver(apple_pcie_driver);
0837 
0838 MODULE_LICENSE("GPL v2");
[ Source navigation ] [ Diff markup ] [ Identifier search ] [ general search ]

This page was automatically generated by the 2.1.0 LXR engine.

The LXR team
Valid CSS 2.1! Valid HTML 4.01!