pci/controller/pcie-iproc-msi.c

0001 // SPDX-License-Identifier: GPL-2.0
0002 /*
0003  * Copyright (C) 2015 Broadcom Corporation
0004  */
0005
0006 #include <linux/interrupt.h>
0007 #include <linux/irqchip/chained_irq.h>
0008 #include <linux/irqdomain.h>
0009 #include <linux/msi.h>
0010 #include <linux/of_irq.h>
0011 #include <linux/of_pci.h>
0012 #include <linux/pci.h>
0013
0014 #include "pcie-iproc.h"
0015
0016 #define IPROC_MSI_INTR_EN_SHIFT        11
0017 #define IPROC_MSI_INTR_EN              BIT(IPROC_MSI_INTR_EN_SHIFT)
0018 #define IPROC_MSI_INT_N_EVENT_SHIFT    1
0019 #define IPROC_MSI_INT_N_EVENT          BIT(IPROC_MSI_INT_N_EVENT_SHIFT)
0020 #define IPROC_MSI_EQ_EN_SHIFT          0
0021 #define IPROC_MSI_EQ_EN                BIT(IPROC_MSI_EQ_EN_SHIFT)
0022
0023 #define IPROC_MSI_EQ_MASK              0x3f
0024
0025 /* Max number of GIC interrupts */
0026 #define NR_HW_IRQS                     6
0027
0028 /* Number of entries in each event queue */
0029 #define EQ_LEN                         64
0030
0031 /* Size of each event queue memory region */
0032 #define EQ_MEM_REGION_SIZE             SZ_4K
0033
0034 /* Size of each MSI address region */
0035 #define MSI_MEM_REGION_SIZE            SZ_4K
0036
0037 enum iproc_msi_reg {
0038     IPROC_MSI_EQ_PAGE = 0,
0039     IPROC_MSI_EQ_PAGE_UPPER,
0040     IPROC_MSI_PAGE,
0041     IPROC_MSI_PAGE_UPPER,
0042     IPROC_MSI_CTRL,
0043     IPROC_MSI_EQ_HEAD,
0044     IPROC_MSI_EQ_TAIL,
0045     IPROC_MSI_INTS_EN,
0046     IPROC_MSI_REG_SIZE,
0047 };
0048
0049 struct iproc_msi;
0050
0051 /**
0052  * struct iproc_msi_grp - iProc MSI group
0053  *
0054  * One MSI group is allocated per GIC interrupt, serviced by one iProc MSI
0055  * event queue.
0056  *
0057  * @msi: pointer to iProc MSI data
0058  * @gic_irq: GIC interrupt
0059  * @eq: Event queue number
0060  */
0061 struct iproc_msi_grp {
0062     struct iproc_msi *msi;
0063     int gic_irq;
0064     unsigned int eq;
0065 };
0066
0067 /**
0068  * struct iproc_msi - iProc event queue based MSI
0069  *
0070  * Only meant to be used on platforms without MSI support integrated into the
0071  * GIC.
0072  *
0073  * @pcie: pointer to iProc PCIe data
0074  * @reg_offsets: MSI register offsets
0075  * @grps: MSI groups
0076  * @nr_irqs: number of total interrupts connected to GIC
0077  * @nr_cpus: number of toal CPUs
0078  * @has_inten_reg: indicates the MSI interrupt enable register needs to be
0079  * set explicitly (required for some legacy platforms)
0080  * @bitmap: MSI vector bitmap
0081  * @bitmap_lock: lock to protect access to the MSI bitmap
0082  * @nr_msi_vecs: total number of MSI vectors
0083  * @inner_domain: inner IRQ domain
0084  * @msi_domain: MSI IRQ domain
0085  * @nr_eq_region: required number of 4K aligned memory region for MSI event
0086  * queues
0087  * @nr_msi_region: required number of 4K aligned address region for MSI posted
0088  * writes
0089  * @eq_cpu: pointer to allocated memory region for MSI event queues
0090  * @eq_dma: DMA address of MSI event queues
0091  * @msi_addr: MSI address
0092  */
0093 struct iproc_msi {
0094     struct iproc_pcie *pcie;
0095     const u16 (*reg_offsets)[IPROC_MSI_REG_SIZE];
0096     struct iproc_msi_grp *grps;
0097     int nr_irqs;
0098     int nr_cpus;
0099     bool has_inten_reg;
0100     unsigned long *bitmap;
0101     struct mutex bitmap_lock;
0102     unsigned int nr_msi_vecs;
0103     struct irq_domain *inner_domain;
0104     struct irq_domain *msi_domain;
0105     unsigned int nr_eq_region;
0106     unsigned int nr_msi_region;
0107     void *eq_cpu;
0108     dma_addr_t eq_dma;
0109     phys_addr_t msi_addr;
0110 };
0111
0112 static const u16 iproc_msi_reg_paxb[NR_HW_IRQS][IPROC_MSI_REG_SIZE] = {
0113     { 0x200, 0x2c0, 0x204, 0x2c4, 0x210, 0x250, 0x254, 0x208 },
0114     { 0x200, 0x2c0, 0x204, 0x2c4, 0x214, 0x258, 0x25c, 0x208 },
0115     { 0x200, 0x2c0, 0x204, 0x2c4, 0x218, 0x260, 0x264, 0x208 },
0116     { 0x200, 0x2c0, 0x204, 0x2c4, 0x21c, 0x268, 0x26c, 0x208 },
0117     { 0x200, 0x2c0, 0x204, 0x2c4, 0x220, 0x270, 0x274, 0x208 },
0118     { 0x200, 0x2c0, 0x204, 0x2c4, 0x224, 0x278, 0x27c, 0x208 },
0119 };
0120
0121 static const u16 iproc_msi_reg_paxc[NR_HW_IRQS][IPROC_MSI_REG_SIZE] = {
0122     { 0xc00, 0xc04, 0xc08, 0xc0c, 0xc40, 0xc50, 0xc60 },
0123     { 0xc10, 0xc14, 0xc18, 0xc1c, 0xc44, 0xc54, 0xc64 },
0124     { 0xc20, 0xc24, 0xc28, 0xc2c, 0xc48, 0xc58, 0xc68 },
0125     { 0xc30, 0xc34, 0xc38, 0xc3c, 0xc4c, 0xc5c, 0xc6c },
0126 };
0127
0128 static inline u32 iproc_msi_read_reg(struct iproc_msi *msi,
0129                      enum iproc_msi_reg reg,
0130                      unsigned int eq)
0131 {
0132     struct iproc_pcie *pcie = msi->pcie;
0133
0134     return readl_relaxed(pcie->base + msi->reg_offsets[eq][reg]);
0135 }
0136
0137 static inline void iproc_msi_write_reg(struct iproc_msi *msi,
0138                        enum iproc_msi_reg reg,
0139                        int eq, u32 val)
0140 {
0141     struct iproc_pcie *pcie = msi->pcie;
0142
0143     writel_relaxed(val, pcie->base + msi->reg_offsets[eq][reg]);
0144 }
0145
0146 static inline u32 hwirq_to_group(struct iproc_msi *msi, unsigned long hwirq)
0147 {
0148     return (hwirq % msi->nr_irqs);
0149 }
0150
0151 static inline unsigned int iproc_msi_addr_offset(struct iproc_msi *msi,
0152                          unsigned long hwirq)
0153 {
0154     if (msi->nr_msi_region > 1)
0155         return hwirq_to_group(msi, hwirq) * MSI_MEM_REGION_SIZE;
0156     else
0157         return hwirq_to_group(msi, hwirq) * sizeof(u32);
0158 }
0159
0160 static inline unsigned int iproc_msi_eq_offset(struct iproc_msi *msi, u32 eq)
0161 {
0162     if (msi->nr_eq_region > 1)
0163         return eq * EQ_MEM_REGION_SIZE;
0164     else
0165         return eq * EQ_LEN * sizeof(u32);
0166 }
0167
0168 static struct irq_chip iproc_msi_irq_chip = {
0169     .name = "iProc-MSI",
0170 };
0171
0172 static struct msi_domain_info iproc_msi_domain_info = {
0173     .flags = MSI_FLAG_USE_DEF_DOM_OPS | MSI_FLAG_USE_DEF_CHIP_OPS |
0174         MSI_FLAG_PCI_MSIX,
0175     .chip = &iproc_msi_irq_chip,
0176 };
0177
0178 /*
0179  * In iProc PCIe core, each MSI group is serviced by a GIC interrupt and a
0180  * dedicated event queue.  Each MSI group can support up to 64 MSI vectors.
0181  *
0182  * The number of MSI groups varies between different iProc SoCs.  The total
0183  * number of CPU cores also varies.  To support MSI IRQ affinity, we
0184  * distribute GIC interrupts across all available CPUs.  MSI vector is moved
0185  * from one GIC interrupt to another to steer to the target CPU.
0186  *
0187  * Assuming:
0188  * - the number of MSI groups is M
0189  * - the number of CPU cores is N
0190  * - M is always a multiple of N
0191  *
0192  * Total number of raw MSI vectors = M * 64
0193  * Total number of supported MSI vectors = (M * 64) / N
0194  */
0195 static inline int hwirq_to_cpu(struct iproc_msi *msi, unsigned long hwirq)
0196 {
0197     return (hwirq % msi->nr_cpus);
0198 }
0199
0200 static inline unsigned long hwirq_to_canonical_hwirq(struct iproc_msi *msi,
0201                              unsigned long hwirq)
0202 {
0203     return (hwirq - hwirq_to_cpu(msi, hwirq));
0204 }
0205
0206 static int iproc_msi_irq_set_affinity(struct irq_data *data,
0207                       const struct cpumask *mask, bool force)
0208 {
0209     struct iproc_msi *msi = irq_data_get_irq_chip_data(data);
0210     int target_cpu = cpumask_first(mask);
0211     int curr_cpu;
0212     int ret;
0213
0214     curr_cpu = hwirq_to_cpu(msi, data->hwirq);
0215     if (curr_cpu == target_cpu)
0216         ret = IRQ_SET_MASK_OK_DONE;
0217     else {
0218         /* steer MSI to the target CPU */
0219         data->hwirq = hwirq_to_canonical_hwirq(msi, data->hwirq) + target_cpu;
0220         ret = IRQ_SET_MASK_OK;
0221     }
0222
0223     irq_data_update_effective_affinity(data, cpumask_of(target_cpu));
0224
0225     return ret;
0226 }
0227
0228 static void iproc_msi_irq_compose_msi_msg(struct irq_data *data,
0229                       struct msi_msg *msg)
0230 {
0231     struct iproc_msi *msi = irq_data_get_irq_chip_data(data);
0232     dma_addr_t addr;
0233
0234     addr = msi->msi_addr + iproc_msi_addr_offset(msi, data->hwirq);
0235     msg->address_lo = lower_32_bits(addr);
0236     msg->address_hi = upper_32_bits(addr);
0237     msg->data = data->hwirq << 5;
0238 }
0239
0240 static struct irq_chip iproc_msi_bottom_irq_chip = {
0241     .name = "MSI",
0242     .irq_set_affinity = iproc_msi_irq_set_affinity,
0243     .irq_compose_msi_msg = iproc_msi_irq_compose_msi_msg,
0244 };
0245
0246 static int iproc_msi_irq_domain_alloc(struct irq_domain *domain,
0247                       unsigned int virq, unsigned int nr_irqs,
0248                       void *args)
0249 {
0250     struct iproc_msi *msi = domain->host_data;
0251     int hwirq, i;
0252
0253     if (msi->nr_cpus > 1 && nr_irqs > 1)
0254         return -EINVAL;
0255
0256     mutex_lock(&msi->bitmap_lock);
0257
0258     /*
0259      * Allocate 'nr_irqs' multiplied by 'nr_cpus' number of MSI vectors
0260      * each time
0261      */
0262     hwirq = bitmap_find_free_region(msi->bitmap, msi->nr_msi_vecs,
0263                     order_base_2(msi->nr_cpus * nr_irqs));
0264
0265     mutex_unlock(&msi->bitmap_lock);
0266
0267     if (hwirq < 0)
0268         return -ENOSPC;
0269
0270     for (i = 0; i < nr_irqs; i++) {
0271         irq_domain_set_info(domain, virq + i, hwirq + i,
0272                     &iproc_msi_bottom_irq_chip,
0273                     domain->host_data, handle_simple_irq,
0274                     NULL, NULL);
0275     }
0276
0277     return 0;
0278 }
0279
0280 static void iproc_msi_irq_domain_free(struct irq_domain *domain,
0281                       unsigned int virq, unsigned int nr_irqs)
0282 {
0283     struct irq_data *data = irq_domain_get_irq_data(domain, virq);
0284     struct iproc_msi *msi = irq_data_get_irq_chip_data(data);
0285     unsigned int hwirq;
0286
0287     mutex_lock(&msi->bitmap_lock);
0288
0289     hwirq = hwirq_to_canonical_hwirq(msi, data->hwirq);
0290     bitmap_release_region(msi->bitmap, hwirq,
0291                   order_base_2(msi->nr_cpus * nr_irqs));
0292
0293     mutex_unlock(&msi->bitmap_lock);
0294
0295     irq_domain_free_irqs_parent(domain, virq, nr_irqs);
0296 }
0297
0298 static const struct irq_domain_ops msi_domain_ops = {
0299     .alloc = iproc_msi_irq_domain_alloc,
0300     .free = iproc_msi_irq_domain_free,
0301 };
0302
0303 static inline u32 decode_msi_hwirq(struct iproc_msi *msi, u32 eq, u32 head)
0304 {
0305     u32 __iomem *msg;
0306     u32 hwirq;
0307     unsigned int offs;
0308
0309     offs = iproc_msi_eq_offset(msi, eq) + head * sizeof(u32);
0310     msg = (u32 __iomem *)(msi->eq_cpu + offs);
0311     hwirq = readl(msg);
0312     hwirq = (hwirq >> 5) + (hwirq & 0x1f);
0313
0314     /*
0315      * Since we have multiple hwirq mapped to a single MSI vector,
0316      * now we need to derive the hwirq at CPU0.  It can then be used to
0317      * mapped back to virq.
0318      */
0319     return hwirq_to_canonical_hwirq(msi, hwirq);
0320 }
0321
0322 static void iproc_msi_handler(struct irq_desc *desc)
0323 {
0324     struct irq_chip *chip = irq_desc_get_chip(desc);
0325     struct iproc_msi_grp *grp;
0326     struct iproc_msi *msi;
0327     u32 eq, head, tail, nr_events;
0328     unsigned long hwirq;
0329
0330     chained_irq_enter(chip, desc);
0331
0332     grp = irq_desc_get_handler_data(desc);
0333     msi = grp->msi;
0334     eq = grp->eq;
0335
0336     /*
0337      * iProc MSI event queue is tracked by head and tail pointers.  Head
0338      * pointer indicates the next entry (MSI data) to be consumed by SW in
0339      * the queue and needs to be updated by SW.  iProc MSI core uses the
0340      * tail pointer as the next data insertion point.
0341      *
0342      * Entries between head and tail pointers contain valid MSI data.  MSI
0343      * data is guaranteed to be in the event queue memory before the tail
0344      * pointer is updated by the iProc MSI core.
0345      */
0346     head = iproc_msi_read_reg(msi, IPROC_MSI_EQ_HEAD,
0347                   eq) & IPROC_MSI_EQ_MASK;
0348     do {
0349         tail = iproc_msi_read_reg(msi, IPROC_MSI_EQ_TAIL,
0350                       eq) & IPROC_MSI_EQ_MASK;
0351
0352         /*
0353          * Figure out total number of events (MSI data) to be
0354          * processed.
0355          */
0356         nr_events = (tail < head) ?
0357             (EQ_LEN - (head - tail)) : (tail - head);
0358         if (!nr_events)
0359             break;
0360
0361         /* process all outstanding events */
0362         while (nr_events--) {
0363             hwirq = decode_msi_hwirq(msi, eq, head);
0364             generic_handle_domain_irq(msi->inner_domain, hwirq);
0365
0366             head++;
0367             head %= EQ_LEN;
0368         }
0369
0370         /*
0371          * Now all outstanding events have been processed.  Update the
0372          * head pointer.
0373          */
0374         iproc_msi_write_reg(msi, IPROC_MSI_EQ_HEAD, eq, head);
0375
0376         /*
0377          * Now go read the tail pointer again to see if there are new
0378          * outstanding events that came in during the above window.
0379          */
0380     } while (true);
0381
0382     chained_irq_exit(chip, desc);
0383 }
0384
0385 static void iproc_msi_enable(struct iproc_msi *msi)
0386 {
0387     int i, eq;
0388     u32 val;
0389
0390     /* Program memory region for each event queue */
0391     for (i = 0; i < msi->nr_eq_region; i++) {
0392         dma_addr_t addr = msi->eq_dma + (i * EQ_MEM_REGION_SIZE);
0393
0394         iproc_msi_write_reg(msi, IPROC_MSI_EQ_PAGE, i,
0395                     lower_32_bits(addr));
0396         iproc_msi_write_reg(msi, IPROC_MSI_EQ_PAGE_UPPER, i,
0397                     upper_32_bits(addr));
0398     }
0399
0400     /* Program address region for MSI posted writes */
0401     for (i = 0; i < msi->nr_msi_region; i++) {
0402         phys_addr_t addr = msi->msi_addr + (i * MSI_MEM_REGION_SIZE);
0403
0404         iproc_msi_write_reg(msi, IPROC_MSI_PAGE, i,
0405                     lower_32_bits(addr));
0406         iproc_msi_write_reg(msi, IPROC_MSI_PAGE_UPPER, i,
0407                     upper_32_bits(addr));
0408     }
0409
0410     for (eq = 0; eq < msi->nr_irqs; eq++) {
0411         /* Enable MSI event queue */
0412         val = IPROC_MSI_INTR_EN | IPROC_MSI_INT_N_EVENT |
0413             IPROC_MSI_EQ_EN;
0414         iproc_msi_write_reg(msi, IPROC_MSI_CTRL, eq, val);
0415
0416         /*
0417          * Some legacy platforms require the MSI interrupt enable
0418          * register to be set explicitly.
0419          */
0420         if (msi->has_inten_reg) {
0421             val = iproc_msi_read_reg(msi, IPROC_MSI_INTS_EN, eq);
0422             val |= BIT(eq);
0423             iproc_msi_write_reg(msi, IPROC_MSI_INTS_EN, eq, val);
0424         }
0425     }
0426 }
0427
0428 static void iproc_msi_disable(struct iproc_msi *msi)
0429 {
0430     u32 eq, val;
0431
0432     for (eq = 0; eq < msi->nr_irqs; eq++) {
0433         if (msi->has_inten_reg) {
0434             val = iproc_msi_read_reg(msi, IPROC_MSI_INTS_EN, eq);
0435             val &= ~BIT(eq);
0436             iproc_msi_write_reg(msi, IPROC_MSI_INTS_EN, eq, val);
0437         }
0438
0439         val = iproc_msi_read_reg(msi, IPROC_MSI_CTRL, eq);
0440         val &= ~(IPROC_MSI_INTR_EN | IPROC_MSI_INT_N_EVENT |
0441              IPROC_MSI_EQ_EN);
0442         iproc_msi_write_reg(msi, IPROC_MSI_CTRL, eq, val);
0443     }
0444 }
0445
0446 static int iproc_msi_alloc_domains(struct device_node *node,
0447                    struct iproc_msi *msi)
0448 {
0449     msi->inner_domain = irq_domain_add_linear(NULL, msi->nr_msi_vecs,
0450                           &msi_domain_ops, msi);
0451     if (!msi->inner_domain)
0452         return -ENOMEM;
0453
0454     msi->msi_domain = pci_msi_create_irq_domain(of_node_to_fwnode(node),
0455                             &iproc_msi_domain_info,
0456                             msi->inner_domain);
0457     if (!msi->msi_domain) {
0458         irq_domain_remove(msi->inner_domain);
0459         return -ENOMEM;
0460     }
0461
0462     return 0;
0463 }
0464
0465 static void iproc_msi_free_domains(struct iproc_msi *msi)
0466 {
0467     if (msi->msi_domain)
0468         irq_domain_remove(msi->msi_domain);
0469
0470     if (msi->inner_domain)
0471         irq_domain_remove(msi->inner_domain);
0472 }
0473
0474 static void iproc_msi_irq_free(struct iproc_msi *msi, unsigned int cpu)
0475 {
0476     int i;
0477
0478     for (i = cpu; i < msi->nr_irqs; i += msi->nr_cpus) {
0479         irq_set_chained_handler_and_data(msi->grps[i].gic_irq,
0480                          NULL, NULL);
0481     }
0482 }
0483
0484 static int iproc_msi_irq_setup(struct iproc_msi *msi, unsigned int cpu)
0485 {
0486     int i, ret;
0487     cpumask_var_t mask;
0488     struct iproc_pcie *pcie = msi->pcie;
0489
0490     for (i = cpu; i < msi->nr_irqs; i += msi->nr_cpus) {
0491         irq_set_chained_handler_and_data(msi->grps[i].gic_irq,
0492                          iproc_msi_handler,
0493                          &msi->grps[i]);
0494         /* Dedicate GIC interrupt to each CPU core */
0495         if (alloc_cpumask_var(&mask, GFP_KERNEL)) {
0496             cpumask_clear(mask);
0497             cpumask_set_cpu(cpu, mask);
0498             ret = irq_set_affinity(msi->grps[i].gic_irq, mask);
0499             if (ret)
0500                 dev_err(pcie->dev,
0501                     "failed to set affinity for IRQ%d\n",
0502                     msi->grps[i].gic_irq);
0503             free_cpumask_var(mask);
0504         } else {
0505             dev_err(pcie->dev, "failed to alloc CPU mask\n");
0506             ret = -EINVAL;
0507         }
0508
0509         if (ret) {
0510             /* Free all configured/unconfigured IRQs */
0511             iproc_msi_irq_free(msi, cpu);
0512             return ret;
0513         }
0514     }
0515
0516     return 0;
0517 }
0518
0519 int iproc_msi_init(struct iproc_pcie *pcie, struct device_node *node)
0520 {
0521     struct iproc_msi *msi;
0522     int i, ret;
0523     unsigned int cpu;
0524
0525     if (!of_device_is_compatible(node, "brcm,iproc-msi"))
0526         return -ENODEV;
0527
0528     if (!of_find_property(node, "msi-controller", NULL))
0529         return -ENODEV;
0530
0531     if (pcie->msi)
0532         return -EBUSY;
0533
0534     msi = devm_kzalloc(pcie->dev, sizeof(*msi), GFP_KERNEL);
0535     if (!msi)
0536         return -ENOMEM;
0537
0538     msi->pcie = pcie;
0539     pcie->msi = msi;
0540     msi->msi_addr = pcie->base_addr;
0541     mutex_init(&msi->bitmap_lock);
0542     msi->nr_cpus = num_possible_cpus();
0543
0544     if (msi->nr_cpus == 1)
0545         iproc_msi_domain_info.flags |=  MSI_FLAG_MULTI_PCI_MSI;
0546
0547     msi->nr_irqs = of_irq_count(node);
0548     if (!msi->nr_irqs) {
0549         dev_err(pcie->dev, "found no MSI GIC interrupt\n");
0550         return -ENODEV;
0551     }
0552
0553     if (msi->nr_irqs > NR_HW_IRQS) {
0554         dev_warn(pcie->dev, "too many MSI GIC interrupts defined %d\n",
0555              msi->nr_irqs);
0556         msi->nr_irqs = NR_HW_IRQS;
0557     }
0558
0559     if (msi->nr_irqs < msi->nr_cpus) {
0560         dev_err(pcie->dev,
0561             "not enough GIC interrupts for MSI affinity\n");
0562         return -EINVAL;
0563     }
0564
0565     if (msi->nr_irqs % msi->nr_cpus != 0) {
0566         msi->nr_irqs -= msi->nr_irqs % msi->nr_cpus;
0567         dev_warn(pcie->dev, "Reducing number of interrupts to %d\n",
0568              msi->nr_irqs);
0569     }
0570
0571     switch (pcie->type) {
0572     case IPROC_PCIE_PAXB_BCMA:
0573     case IPROC_PCIE_PAXB:
0574         msi->reg_offsets = iproc_msi_reg_paxb;
0575         msi->nr_eq_region = 1;
0576         msi->nr_msi_region = 1;
0577         break;
0578     case IPROC_PCIE_PAXC:
0579         msi->reg_offsets = iproc_msi_reg_paxc;
0580         msi->nr_eq_region = msi->nr_irqs;
0581         msi->nr_msi_region = msi->nr_irqs;
0582         break;
0583     default:
0584         dev_err(pcie->dev, "incompatible iProc PCIe interface\n");
0585         return -EINVAL;
0586     }
0587
0588     if (of_find_property(node, "brcm,pcie-msi-inten", NULL))
0589         msi->has_inten_reg = true;
0590
0591     msi->nr_msi_vecs = msi->nr_irqs * EQ_LEN;
0592     msi->bitmap = devm_bitmap_zalloc(pcie->dev, msi->nr_msi_vecs,
0593                      GFP_KERNEL);
0594     if (!msi->bitmap)
0595         return -ENOMEM;
0596
0597     msi->grps = devm_kcalloc(pcie->dev, msi->nr_irqs, sizeof(*msi->grps),
0598                  GFP_KERNEL);
0599     if (!msi->grps)
0600         return -ENOMEM;
0601
0602     for (i = 0; i < msi->nr_irqs; i++) {
0603         unsigned int irq = irq_of_parse_and_map(node, i);
0604
0605         if (!irq) {
0606             dev_err(pcie->dev, "unable to parse/map interrupt\n");
0607             ret = -ENODEV;
0608             goto free_irqs;
0609         }
0610         msi->grps[i].gic_irq = irq;
0611         msi->grps[i].msi = msi;
0612         msi->grps[i].eq = i;
0613     }
0614
0615     /* Reserve memory for event queue and make sure memories are zeroed */
0616     msi->eq_cpu = dma_alloc_coherent(pcie->dev,
0617                      msi->nr_eq_region * EQ_MEM_REGION_SIZE,
0618                      &msi->eq_dma, GFP_KERNEL);
0619     if (!msi->eq_cpu) {
0620         ret = -ENOMEM;
0621         goto free_irqs;
0622     }
0623
0624     ret = iproc_msi_alloc_domains(node, msi);
0625     if (ret) {
0626         dev_err(pcie->dev, "failed to create MSI domains\n");
0627         goto free_eq_dma;
0628     }
0629
0630     for_each_online_cpu(cpu) {
0631         ret = iproc_msi_irq_setup(msi, cpu);
0632         if (ret)
0633             goto free_msi_irq;
0634     }
0635
0636     iproc_msi_enable(msi);
0637
0638     return 0;
0639
0640 free_msi_irq:
0641     for_each_online_cpu(cpu)
0642         iproc_msi_irq_free(msi, cpu);
0643     iproc_msi_free_domains(msi);
0644
0645 free_eq_dma:
0646     dma_free_coherent(pcie->dev, msi->nr_eq_region * EQ_MEM_REGION_SIZE,
0647               msi->eq_cpu, msi->eq_dma);
0648
0649 free_irqs:
0650     for (i = 0; i < msi->nr_irqs; i++) {
0651         if (msi->grps[i].gic_irq)
0652             irq_dispose_mapping(msi->grps[i].gic_irq);
0653     }
0654     pcie->msi = NULL;
0655     return ret;
0656 }
0657 EXPORT_SYMBOL(iproc_msi_init);
0658
0659 void iproc_msi_exit(struct iproc_pcie *pcie)
0660 {
0661     struct iproc_msi *msi = pcie->msi;
0662     unsigned int i, cpu;
0663
0664     if (!msi)
0665         return;
0666
0667     iproc_msi_disable(msi);
0668
0669     for_each_online_cpu(cpu)
0670         iproc_msi_irq_free(msi, cpu);
0671
0672     iproc_msi_free_domains(msi);
0673
0674     dma_free_coherent(pcie->dev, msi->nr_eq_region * EQ_MEM_REGION_SIZE,
0675               msi->eq_cpu, msi->eq_dma);
0676
0677     for (i = 0; i < msi->nr_irqs; i++) {
0678         if (msi->grps[i].gic_irq)
0679             irq_dispose_mapping(msi->grps[i].gic_irq);
0680     }
0681 }
0682 EXPORT_SYMBOL(iproc_msi_exit);