powerpc/sysdev/fsl_msi.c

0001 // SPDX-License-Identifier: GPL-2.0-only
0002 /*
0003  * Copyright (C) 2007-2011 Freescale Semiconductor, Inc.
0004  *
0005  * Author: Tony Li <tony.li@freescale.com>
0006  *     Jason Jin <Jason.jin@freescale.com>
0007  *
0008  * The hwirq alloc and free code reuse from sysdev/mpic_msi.c
0009  */
0010 #include <linux/irq.h>
0011 #include <linux/msi.h>
0012 #include <linux/pci.h>
0013 #include <linux/slab.h>
0014 #include <linux/of_address.h>
0015 #include <linux/of_irq.h>
0016 #include <linux/of_platform.h>
0017 #include <linux/interrupt.h>
0018 #include <linux/irqdomain.h>
0019 #include <linux/seq_file.h>
0020 #include <sysdev/fsl_soc.h>
0021 #include <asm/hw_irq.h>
0022 #include <asm/ppc-pci.h>
0023 #include <asm/mpic.h>
0024 #include <asm/fsl_hcalls.h>
0025
0026 #include "fsl_msi.h"
0027 #include "fsl_pci.h"
0028
0029 #define MSIIR_OFFSET_MASK   0xfffff
0030 #define MSIIR_IBS_SHIFT     0
0031 #define MSIIR_SRS_SHIFT     5
0032 #define MSIIR1_IBS_SHIFT    4
0033 #define MSIIR1_SRS_SHIFT    0
0034 #define MSI_SRS_MASK        0xf
0035 #define MSI_IBS_MASK        0x1f
0036
0037 #define msi_hwirq(msi, msir_index, intr_index) \
0038         ((msir_index) << (msi)->srs_shift | \
0039          ((intr_index) << (msi)->ibs_shift))
0040
0041 static LIST_HEAD(msi_head);
0042
0043 struct fsl_msi_feature {
0044     u32 fsl_pic_ip;
0045     u32 msiir_offset; /* Offset of MSIIR, relative to start of MSIR bank */
0046 };
0047
0048 struct fsl_msi_cascade_data {
0049     struct fsl_msi *msi_data;
0050     int index;
0051     int virq;
0052 };
0053
0054 static inline u32 fsl_msi_read(u32 __iomem *base, unsigned int reg)
0055 {
0056     return in_be32(base + (reg >> 2));
0057 }
0058
0059 /*
0060  * We do not need this actually. The MSIR register has been read once
0061  * in the cascade interrupt. So, this MSI interrupt has been acked
0062 */
0063 static void fsl_msi_end_irq(struct irq_data *d)
0064 {
0065 }
0066
0067 static void fsl_msi_print_chip(struct irq_data *irqd, struct seq_file *p)
0068 {
0069     struct fsl_msi *msi_data = irqd->domain->host_data;
0070     irq_hw_number_t hwirq = irqd_to_hwirq(irqd);
0071     int cascade_virq, srs;
0072
0073     srs = (hwirq >> msi_data->srs_shift) & MSI_SRS_MASK;
0074     cascade_virq = msi_data->cascade_array[srs]->virq;
0075
0076     seq_printf(p, " fsl-msi-%d", cascade_virq);
0077 }
0078
0079
0080 static struct irq_chip fsl_msi_chip = {
0081     .irq_mask   = pci_msi_mask_irq,
0082     .irq_unmask = pci_msi_unmask_irq,
0083     .irq_ack    = fsl_msi_end_irq,
0084     .irq_print_chip = fsl_msi_print_chip,
0085 };
0086
0087 static int fsl_msi_host_map(struct irq_domain *h, unsigned int virq,
0088                 irq_hw_number_t hw)
0089 {
0090     struct fsl_msi *msi_data = h->host_data;
0091     struct irq_chip *chip = &fsl_msi_chip;
0092
0093     irq_set_status_flags(virq, IRQ_TYPE_EDGE_FALLING);
0094
0095     irq_set_chip_data(virq, msi_data);
0096     irq_set_chip_and_handler(virq, chip, handle_edge_irq);
0097
0098     return 0;
0099 }
0100
0101 static const struct irq_domain_ops fsl_msi_host_ops = {
0102     .map = fsl_msi_host_map,
0103 };
0104
0105 static int fsl_msi_init_allocator(struct fsl_msi *msi_data)
0106 {
0107     int rc, hwirq;
0108
0109     rc = msi_bitmap_alloc(&msi_data->bitmap, NR_MSI_IRQS_MAX,
0110                   irq_domain_get_of_node(msi_data->irqhost));
0111     if (rc)
0112         return rc;
0113
0114     /*
0115      * Reserve all the hwirqs
0116      * The available hwirqs will be released in fsl_msi_setup_hwirq()
0117      */
0118     for (hwirq = 0; hwirq < NR_MSI_IRQS_MAX; hwirq++)
0119         msi_bitmap_reserve_hwirq(&msi_data->bitmap, hwirq);
0120
0121     return 0;
0122 }
0123
0124 static void fsl_teardown_msi_irqs(struct pci_dev *pdev)
0125 {
0126     struct msi_desc *entry;
0127     struct fsl_msi *msi_data;
0128     irq_hw_number_t hwirq;
0129
0130     msi_for_each_desc(entry, &pdev->dev, MSI_DESC_ASSOCIATED) {
0131         hwirq = virq_to_hw(entry->irq);
0132         msi_data = irq_get_chip_data(entry->irq);
0133         irq_set_msi_desc(entry->irq, NULL);
0134         irq_dispose_mapping(entry->irq);
0135         msi_bitmap_free_hwirqs(&msi_data->bitmap, hwirq, 1);
0136     }
0137 }
0138
0139 static void fsl_compose_msi_msg(struct pci_dev *pdev, int hwirq,
0140                 struct msi_msg *msg,
0141                 struct fsl_msi *fsl_msi_data)
0142 {
0143     struct fsl_msi *msi_data = fsl_msi_data;
0144     struct pci_controller *hose = pci_bus_to_host(pdev->bus);
0145     u64 address; /* Physical address of the MSIIR */
0146     int len;
0147     const __be64 *reg;
0148
0149     /* If the msi-address-64 property exists, then use it */
0150     reg = of_get_property(hose->dn, "msi-address-64", &len);
0151     if (reg && (len == sizeof(u64)))
0152         address = be64_to_cpup(reg);
0153     else
0154         address = fsl_pci_immrbar_base(hose) + msi_data->msiir_offset;
0155
0156     msg->address_lo = lower_32_bits(address);
0157     msg->address_hi = upper_32_bits(address);
0158
0159     /*
0160      * MPIC version 2.0 has erratum PIC1. It causes
0161      * that neither MSI nor MSI-X can work fine.
0162      * This is a workaround to allow MSI-X to function
0163      * properly. It only works for MSI-X, we prevent
0164      * MSI on buggy chips in fsl_setup_msi_irqs().
0165      */
0166     if (msi_data->feature & MSI_HW_ERRATA_ENDIAN)
0167         msg->data = __swab32(hwirq);
0168     else
0169         msg->data = hwirq;
0170
0171     pr_debug("%s: allocated srs: %d, ibs: %d\n", __func__,
0172          (hwirq >> msi_data->srs_shift) & MSI_SRS_MASK,
0173          (hwirq >> msi_data->ibs_shift) & MSI_IBS_MASK);
0174 }
0175
0176 static int fsl_setup_msi_irqs(struct pci_dev *pdev, int nvec, int type)
0177 {
0178     struct pci_controller *hose = pci_bus_to_host(pdev->bus);
0179     struct device_node *np;
0180     phandle phandle = 0;
0181     int rc, hwirq = -ENOMEM;
0182     unsigned int virq;
0183     struct msi_desc *entry;
0184     struct msi_msg msg;
0185     struct fsl_msi *msi_data;
0186
0187     if (type == PCI_CAP_ID_MSI) {
0188         /*
0189          * MPIC version 2.0 has erratum PIC1. For now MSI
0190          * could not work. So check to prevent MSI from
0191          * being used on the board with this erratum.
0192          */
0193         list_for_each_entry(msi_data, &msi_head, list)
0194             if (msi_data->feature & MSI_HW_ERRATA_ENDIAN)
0195                 return -EINVAL;
0196     }
0197
0198     /*
0199      * If the PCI node has an fsl,msi property, then we need to use it
0200      * to find the specific MSI.
0201      */
0202     np = of_parse_phandle(hose->dn, "fsl,msi", 0);
0203     if (np) {
0204         if (of_device_is_compatible(np, "fsl,mpic-msi") ||
0205             of_device_is_compatible(np, "fsl,vmpic-msi") ||
0206             of_device_is_compatible(np, "fsl,vmpic-msi-v4.3"))
0207             phandle = np->phandle;
0208         else {
0209             dev_err(&pdev->dev,
0210                 "node %pOF has an invalid fsl,msi phandle %u\n",
0211                 hose->dn, np->phandle);
0212             return -EINVAL;
0213         }
0214     }
0215
0216     msi_for_each_desc(entry, &pdev->dev, MSI_DESC_NOTASSOCIATED) {
0217         /*
0218          * Loop over all the MSI devices until we find one that has an
0219          * available interrupt.
0220          */
0221         list_for_each_entry(msi_data, &msi_head, list) {
0222             /*
0223              * If the PCI node has an fsl,msi property, then we
0224              * restrict our search to the corresponding MSI node.
0225              * The simplest way is to skip over MSI nodes with the
0226              * wrong phandle. Under the Freescale hypervisor, this
0227              * has the additional benefit of skipping over MSI
0228              * nodes that are not mapped in the PAMU.
0229              */
0230             if (phandle && (phandle != msi_data->phandle))
0231                 continue;
0232
0233             hwirq = msi_bitmap_alloc_hwirqs(&msi_data->bitmap, 1);
0234             if (hwirq >= 0)
0235                 break;
0236         }
0237
0238         if (hwirq < 0) {
0239             rc = hwirq;
0240             dev_err(&pdev->dev, "could not allocate MSI interrupt\n");
0241             goto out_free;
0242         }
0243
0244         virq = irq_create_mapping(msi_data->irqhost, hwirq);
0245
0246         if (!virq) {
0247             dev_err(&pdev->dev, "fail mapping hwirq %i\n", hwirq);
0248             msi_bitmap_free_hwirqs(&msi_data->bitmap, hwirq, 1);
0249             rc = -ENOSPC;
0250             goto out_free;
0251         }
0252         /* chip_data is msi_data via host->hostdata in host->map() */
0253         irq_set_msi_desc(virq, entry);
0254
0255         fsl_compose_msi_msg(pdev, hwirq, &msg, msi_data);
0256         pci_write_msi_msg(virq, &msg);
0257     }
0258     return 0;
0259
0260 out_free:
0261     /* free by the caller of this function */
0262     return rc;
0263 }
0264
0265 static irqreturn_t fsl_msi_cascade(int irq, void *data)
0266 {
0267     struct fsl_msi *msi_data;
0268     int msir_index = -1;
0269     u32 msir_value = 0;
0270     u32 intr_index;
0271     u32 have_shift = 0;
0272     struct fsl_msi_cascade_data *cascade_data = data;
0273     irqreturn_t ret = IRQ_NONE;
0274
0275     msi_data = cascade_data->msi_data;
0276
0277     msir_index = cascade_data->index;
0278
0279     switch (msi_data->feature & FSL_PIC_IP_MASK) {
0280     case FSL_PIC_IP_MPIC:
0281         msir_value = fsl_msi_read(msi_data->msi_regs,
0282             msir_index * 0x10);
0283         break;
0284     case FSL_PIC_IP_IPIC:
0285         msir_value = fsl_msi_read(msi_data->msi_regs, msir_index * 0x4);
0286         break;
0287 #ifdef CONFIG_EPAPR_PARAVIRT
0288     case FSL_PIC_IP_VMPIC: {
0289         unsigned int ret;
0290         ret = fh_vmpic_get_msir(virq_to_hw(irq), &msir_value);
0291         if (ret) {
0292             pr_err("fsl-msi: fh_vmpic_get_msir() failed for "
0293                    "irq %u (ret=%u)\n", irq, ret);
0294             msir_value = 0;
0295         }
0296         break;
0297     }
0298 #endif
0299     }
0300
0301     while (msir_value) {
0302         int err;
0303         intr_index = ffs(msir_value) - 1;
0304
0305         err = generic_handle_domain_irq(msi_data->irqhost,
0306                 msi_hwirq(msi_data, msir_index,
0307                       intr_index + have_shift));
0308         if (!err)
0309             ret = IRQ_HANDLED;
0310
0311         have_shift += intr_index + 1;
0312         msir_value = msir_value >> (intr_index + 1);
0313     }
0314
0315     return ret;
0316 }
0317
0318 static int fsl_of_msi_remove(struct platform_device *ofdev)
0319 {
0320     struct fsl_msi *msi = platform_get_drvdata(ofdev);
0321     int virq, i;
0322
0323     if (msi->list.prev != NULL)
0324         list_del(&msi->list);
0325     for (i = 0; i < NR_MSI_REG_MAX; i++) {
0326         if (msi->cascade_array[i]) {
0327             virq = msi->cascade_array[i]->virq;
0328
0329             BUG_ON(!virq);
0330
0331             free_irq(virq, msi->cascade_array[i]);
0332             kfree(msi->cascade_array[i]);
0333             irq_dispose_mapping(virq);
0334         }
0335     }
0336     if (msi->bitmap.bitmap)
0337         msi_bitmap_free(&msi->bitmap);
0338     if ((msi->feature & FSL_PIC_IP_MASK) != FSL_PIC_IP_VMPIC)
0339         iounmap(msi->msi_regs);
0340     kfree(msi);
0341
0342     return 0;
0343 }
0344
0345 static struct lock_class_key fsl_msi_irq_class;
0346 static struct lock_class_key fsl_msi_irq_request_class;
0347
0348 static int fsl_msi_setup_hwirq(struct fsl_msi *msi, struct platform_device *dev,
0349                    int offset, int irq_index)
0350 {
0351     struct fsl_msi_cascade_data *cascade_data = NULL;
0352     int virt_msir, i, ret;
0353
0354     virt_msir = irq_of_parse_and_map(dev->dev.of_node, irq_index);
0355     if (!virt_msir) {
0356         dev_err(&dev->dev, "%s: Cannot translate IRQ index %d\n",
0357             __func__, irq_index);
0358         return 0;
0359     }
0360
0361     cascade_data = kzalloc(sizeof(struct fsl_msi_cascade_data), GFP_KERNEL);
0362     if (!cascade_data) {
0363         dev_err(&dev->dev, "No memory for MSI cascade data\n");
0364         return -ENOMEM;
0365     }
0366     irq_set_lockdep_class(virt_msir, &fsl_msi_irq_class,
0367                   &fsl_msi_irq_request_class);
0368     cascade_data->index = offset;
0369     cascade_data->msi_data = msi;
0370     cascade_data->virq = virt_msir;
0371     msi->cascade_array[irq_index] = cascade_data;
0372
0373     ret = request_irq(virt_msir, fsl_msi_cascade, IRQF_NO_THREAD,
0374               "fsl-msi-cascade", cascade_data);
0375     if (ret) {
0376         dev_err(&dev->dev, "failed to request_irq(%d), ret = %d\n",
0377             virt_msir, ret);
0378         return ret;
0379     }
0380
0381     /* Release the hwirqs corresponding to this MSI register */
0382     for (i = 0; i < IRQS_PER_MSI_REG; i++)
0383         msi_bitmap_free_hwirqs(&msi->bitmap,
0384                        msi_hwirq(msi, offset, i), 1);
0385
0386     return 0;
0387 }
0388
0389 static const struct of_device_id fsl_of_msi_ids[];
0390 static int fsl_of_msi_probe(struct platform_device *dev)
0391 {
0392     const struct of_device_id *match;
0393     struct fsl_msi *msi;
0394     struct resource res, msiir;
0395     int err, i, j, irq_index, count;
0396     const u32 *p;
0397     const struct fsl_msi_feature *features;
0398     int len;
0399     u32 offset;
0400     struct pci_controller *phb;
0401
0402     match = of_match_device(fsl_of_msi_ids, &dev->dev);
0403     if (!match)
0404         return -EINVAL;
0405     features = match->data;
0406
0407     printk(KERN_DEBUG "Setting up Freescale MSI support\n");
0408
0409     msi = kzalloc(sizeof(struct fsl_msi), GFP_KERNEL);
0410     if (!msi) {
0411         dev_err(&dev->dev, "No memory for MSI structure\n");
0412         return -ENOMEM;
0413     }
0414     platform_set_drvdata(dev, msi);
0415
0416     msi->irqhost = irq_domain_add_linear(dev->dev.of_node,
0417                       NR_MSI_IRQS_MAX, &fsl_msi_host_ops, msi);
0418
0419     if (msi->irqhost == NULL) {
0420         dev_err(&dev->dev, "No memory for MSI irqhost\n");
0421         err = -ENOMEM;
0422         goto error_out;
0423     }
0424
0425     /*
0426      * Under the Freescale hypervisor, the msi nodes don't have a 'reg'
0427      * property.  Instead, we use hypercalls to access the MSI.
0428      */
0429     if ((features->fsl_pic_ip & FSL_PIC_IP_MASK) != FSL_PIC_IP_VMPIC) {
0430         err = of_address_to_resource(dev->dev.of_node, 0, &res);
0431         if (err) {
0432             dev_err(&dev->dev, "invalid resource for node %pOF\n",
0433                 dev->dev.of_node);
0434             goto error_out;
0435         }
0436
0437         msi->msi_regs = ioremap(res.start, resource_size(&res));
0438         if (!msi->msi_regs) {
0439             err = -ENOMEM;
0440             dev_err(&dev->dev, "could not map node %pOF\n",
0441                 dev->dev.of_node);
0442             goto error_out;
0443         }
0444         msi->msiir_offset =
0445             features->msiir_offset + (res.start & 0xfffff);
0446
0447         /*
0448          * First read the MSIIR/MSIIR1 offset from dts
0449          * On failure use the hardcode MSIIR offset
0450          */
0451         if (of_address_to_resource(dev->dev.of_node, 1, &msiir))
0452             msi->msiir_offset = features->msiir_offset +
0453                         (res.start & MSIIR_OFFSET_MASK);
0454         else
0455             msi->msiir_offset = msiir.start & MSIIR_OFFSET_MASK;
0456     }
0457
0458     msi->feature = features->fsl_pic_ip;
0459
0460     /* For erratum PIC1 on MPIC version 2.0*/
0461     if ((features->fsl_pic_ip & FSL_PIC_IP_MASK) == FSL_PIC_IP_MPIC
0462             && (fsl_mpic_primary_get_version() == 0x0200))
0463         msi->feature |= MSI_HW_ERRATA_ENDIAN;
0464
0465     /*
0466      * Remember the phandle, so that we can match with any PCI nodes
0467      * that have an "fsl,msi" property.
0468      */
0469     msi->phandle = dev->dev.of_node->phandle;
0470
0471     err = fsl_msi_init_allocator(msi);
0472     if (err) {
0473         dev_err(&dev->dev, "Error allocating MSI bitmap\n");
0474         goto error_out;
0475     }
0476
0477     p = of_get_property(dev->dev.of_node, "msi-available-ranges", &len);
0478
0479     if (of_device_is_compatible(dev->dev.of_node, "fsl,mpic-msi-v4.3") ||
0480         of_device_is_compatible(dev->dev.of_node, "fsl,vmpic-msi-v4.3")) {
0481         msi->srs_shift = MSIIR1_SRS_SHIFT;
0482         msi->ibs_shift = MSIIR1_IBS_SHIFT;
0483         if (p)
0484             dev_warn(&dev->dev, "%s: dose not support msi-available-ranges property\n",
0485                 __func__);
0486
0487         for (irq_index = 0; irq_index < NR_MSI_REG_MSIIR1;
0488              irq_index++) {
0489             err = fsl_msi_setup_hwirq(msi, dev,
0490                           irq_index, irq_index);
0491             if (err)
0492                 goto error_out;
0493         }
0494     } else {
0495         static const u32 all_avail[] =
0496             { 0, NR_MSI_REG_MSIIR * IRQS_PER_MSI_REG };
0497
0498         msi->srs_shift = MSIIR_SRS_SHIFT;
0499         msi->ibs_shift = MSIIR_IBS_SHIFT;
0500
0501         if (p && len % (2 * sizeof(u32)) != 0) {
0502             dev_err(&dev->dev, "%s: Malformed msi-available-ranges property\n",
0503                 __func__);
0504             err = -EINVAL;
0505             goto error_out;
0506         }
0507
0508         if (!p) {
0509             p = all_avail;
0510             len = sizeof(all_avail);
0511         }
0512
0513         for (irq_index = 0, i = 0; i < len / (2 * sizeof(u32)); i++) {
0514             if (p[i * 2] % IRQS_PER_MSI_REG ||
0515                 p[i * 2 + 1] % IRQS_PER_MSI_REG) {
0516                 pr_warn("%s: %pOF: msi available range of %u at %u is not IRQ-aligned\n",
0517                        __func__, dev->dev.of_node,
0518                        p[i * 2 + 1], p[i * 2]);
0519                 err = -EINVAL;
0520                 goto error_out;
0521             }
0522
0523             offset = p[i * 2] / IRQS_PER_MSI_REG;
0524             count = p[i * 2 + 1] / IRQS_PER_MSI_REG;
0525
0526             for (j = 0; j < count; j++, irq_index++) {
0527                 err = fsl_msi_setup_hwirq(msi, dev, offset + j,
0528                               irq_index);
0529                 if (err)
0530                     goto error_out;
0531             }
0532         }
0533     }
0534
0535     list_add_tail(&msi->list, &msi_head);
0536
0537     /*
0538      * Apply the MSI ops to all the controllers.
0539      * It doesn't hurt to reassign the same ops,
0540      * but bail out if we find another MSI driver.
0541      */
0542     list_for_each_entry(phb, &hose_list, list_node) {
0543         if (!phb->controller_ops.setup_msi_irqs) {
0544             phb->controller_ops.setup_msi_irqs = fsl_setup_msi_irqs;
0545             phb->controller_ops.teardown_msi_irqs = fsl_teardown_msi_irqs;
0546         } else if (phb->controller_ops.setup_msi_irqs != fsl_setup_msi_irqs) {
0547             dev_err(&dev->dev, "Different MSI driver already installed!\n");
0548             err = -ENODEV;
0549             goto error_out;
0550         }
0551     }
0552     return 0;
0553 error_out:
0554     fsl_of_msi_remove(dev);
0555     return err;
0556 }
0557
0558 static const struct fsl_msi_feature mpic_msi_feature = {
0559     .fsl_pic_ip = FSL_PIC_IP_MPIC,
0560     .msiir_offset = 0x140,
0561 };
0562
0563 static const struct fsl_msi_feature ipic_msi_feature = {
0564     .fsl_pic_ip = FSL_PIC_IP_IPIC,
0565     .msiir_offset = 0x38,
0566 };
0567
0568 static const struct fsl_msi_feature vmpic_msi_feature = {
0569     .fsl_pic_ip = FSL_PIC_IP_VMPIC,
0570     .msiir_offset = 0,
0571 };
0572
0573 static const struct of_device_id fsl_of_msi_ids[] = {
0574     {
0575         .compatible = "fsl,mpic-msi",
0576         .data = &mpic_msi_feature,
0577     },
0578     {
0579         .compatible = "fsl,mpic-msi-v4.3",
0580         .data = &mpic_msi_feature,
0581     },
0582     {
0583         .compatible = "fsl,ipic-msi",
0584         .data = &ipic_msi_feature,
0585     },
0586 #ifdef CONFIG_EPAPR_PARAVIRT
0587     {
0588         .compatible = "fsl,vmpic-msi",
0589         .data = &vmpic_msi_feature,
0590     },
0591     {
0592         .compatible = "fsl,vmpic-msi-v4.3",
0593         .data = &vmpic_msi_feature,
0594     },
0595 #endif
0596     {}
0597 };
0598
0599 static struct platform_driver fsl_of_msi_driver = {
0600     .driver = {
0601         .name = "fsl-msi",
0602         .of_match_table = fsl_of_msi_ids,
0603     },
0604     .probe = fsl_of_msi_probe,
0605     .remove = fsl_of_msi_remove,
0606 };
0607
0608 static __init int fsl_of_msi_init(void)
0609 {
0610     return platform_driver_register(&fsl_of_msi_driver);
0611 }
0612
0613 subsys_initcall(fsl_of_msi_init);