s390/pci/pci_irq.c

0001 // SPDX-License-Identifier: GPL-2.0
0002 #define KMSG_COMPONENT "zpci"
0003 #define pr_fmt(fmt) KMSG_COMPONENT ": " fmt
0004
0005 #include <linux/kernel.h>
0006 #include <linux/irq.h>
0007 #include <linux/kernel_stat.h>
0008 #include <linux/pci.h>
0009 #include <linux/msi.h>
0010 #include <linux/smp.h>
0011
0012 #include <asm/isc.h>
0013 #include <asm/airq.h>
0014 #include <asm/tpi.h>
0015
0016 static enum {FLOATING, DIRECTED} irq_delivery;
0017
0018 /*
0019  * summary bit vector
0020  * FLOATING - summary bit per function
0021  * DIRECTED - summary bit per cpu (only used in fallback path)
0022  */
0023 static struct airq_iv *zpci_sbv;
0024
0025 /*
0026  * interrupt bit vectors
0027  * FLOATING - interrupt bit vector per function
0028  * DIRECTED - interrupt bit vector per cpu
0029  */
0030 static struct airq_iv **zpci_ibv;
0031
0032 /* Modify PCI: Register floating adapter interruptions */
0033 static int zpci_set_airq(struct zpci_dev *zdev)
0034 {
0035     u64 req = ZPCI_CREATE_REQ(zdev->fh, 0, ZPCI_MOD_FC_REG_INT);
0036     struct zpci_fib fib = {0};
0037     u8 status;
0038
0039     fib.fmt0.isc = PCI_ISC;
0040     fib.fmt0.sum = 1;   /* enable summary notifications */
0041     fib.fmt0.noi = airq_iv_end(zdev->aibv);
0042     fib.fmt0.aibv = virt_to_phys(zdev->aibv->vector);
0043     fib.fmt0.aibvo = 0; /* each zdev has its own interrupt vector */
0044     fib.fmt0.aisb = virt_to_phys(zpci_sbv->vector) + (zdev->aisb / 64) * 8;
0045     fib.fmt0.aisbo = zdev->aisb & 63;
0046     fib.gd = zdev->gisa;
0047
0048     return zpci_mod_fc(req, &fib, &status) ? -EIO : 0;
0049 }
0050
0051 /* Modify PCI: Unregister floating adapter interruptions */
0052 static int zpci_clear_airq(struct zpci_dev *zdev)
0053 {
0054     u64 req = ZPCI_CREATE_REQ(zdev->fh, 0, ZPCI_MOD_FC_DEREG_INT);
0055     struct zpci_fib fib = {0};
0056     u8 cc, status;
0057
0058     fib.gd = zdev->gisa;
0059
0060     cc = zpci_mod_fc(req, &fib, &status);
0061     if (cc == 3 || (cc == 1 && status == 24))
0062         /* Function already gone or IRQs already deregistered. */
0063         cc = 0;
0064
0065     return cc ? -EIO : 0;
0066 }
0067
0068 /* Modify PCI: Register CPU directed interruptions */
0069 static int zpci_set_directed_irq(struct zpci_dev *zdev)
0070 {
0071     u64 req = ZPCI_CREATE_REQ(zdev->fh, 0, ZPCI_MOD_FC_REG_INT_D);
0072     struct zpci_fib fib = {0};
0073     u8 status;
0074
0075     fib.fmt = 1;
0076     fib.fmt1.noi = zdev->msi_nr_irqs;
0077     fib.fmt1.dibvo = zdev->msi_first_bit;
0078     fib.gd = zdev->gisa;
0079
0080     return zpci_mod_fc(req, &fib, &status) ? -EIO : 0;
0081 }
0082
0083 /* Modify PCI: Unregister CPU directed interruptions */
0084 static int zpci_clear_directed_irq(struct zpci_dev *zdev)
0085 {
0086     u64 req = ZPCI_CREATE_REQ(zdev->fh, 0, ZPCI_MOD_FC_DEREG_INT_D);
0087     struct zpci_fib fib = {0};
0088     u8 cc, status;
0089
0090     fib.fmt = 1;
0091     fib.gd = zdev->gisa;
0092     cc = zpci_mod_fc(req, &fib, &status);
0093     if (cc == 3 || (cc == 1 && status == 24))
0094         /* Function already gone or IRQs already deregistered. */
0095         cc = 0;
0096
0097     return cc ? -EIO : 0;
0098 }
0099
0100 /* Register adapter interruptions */
0101 static int zpci_set_irq(struct zpci_dev *zdev)
0102 {
0103     int rc;
0104
0105     if (irq_delivery == DIRECTED)
0106         rc = zpci_set_directed_irq(zdev);
0107     else
0108         rc = zpci_set_airq(zdev);
0109
0110     if (!rc)
0111         zdev->irqs_registered = 1;
0112
0113     return rc;
0114 }
0115
0116 /* Clear adapter interruptions */
0117 static int zpci_clear_irq(struct zpci_dev *zdev)
0118 {
0119     int rc;
0120
0121     if (irq_delivery == DIRECTED)
0122         rc = zpci_clear_directed_irq(zdev);
0123     else
0124         rc = zpci_clear_airq(zdev);
0125
0126     if (!rc)
0127         zdev->irqs_registered = 0;
0128
0129     return rc;
0130 }
0131
0132 static int zpci_set_irq_affinity(struct irq_data *data, const struct cpumask *dest,
0133                  bool force)
0134 {
0135     struct msi_desc *entry = irq_get_msi_desc(data->irq);
0136     struct msi_msg msg = entry->msg;
0137     int cpu_addr = smp_cpu_get_cpu_address(cpumask_first(dest));
0138
0139     msg.address_lo &= 0xff0000ff;
0140     msg.address_lo |= (cpu_addr << 8);
0141     pci_write_msi_msg(data->irq, &msg);
0142
0143     return IRQ_SET_MASK_OK;
0144 }
0145
0146 static struct irq_chip zpci_irq_chip = {
0147     .name = "PCI-MSI",
0148     .irq_unmask = pci_msi_unmask_irq,
0149     .irq_mask = pci_msi_mask_irq,
0150 };
0151
0152 static void zpci_handle_cpu_local_irq(bool rescan)
0153 {
0154     struct airq_iv *dibv = zpci_ibv[smp_processor_id()];
0155     union zpci_sic_iib iib = {{0}};
0156     unsigned long bit;
0157     int irqs_on = 0;
0158
0159     for (bit = 0;;) {
0160         /* Scan the directed IRQ bit vector */
0161         bit = airq_iv_scan(dibv, bit, airq_iv_end(dibv));
0162         if (bit == -1UL) {
0163             if (!rescan || irqs_on++)
0164                 /* End of second scan with interrupts on. */
0165                 break;
0166             /* First scan complete, reenable interrupts. */
0167             if (zpci_set_irq_ctrl(SIC_IRQ_MODE_D_SINGLE, PCI_ISC, &iib))
0168                 break;
0169             bit = 0;
0170             continue;
0171         }
0172         inc_irq_stat(IRQIO_MSI);
0173         generic_handle_irq(airq_iv_get_data(dibv, bit));
0174     }
0175 }
0176
0177 struct cpu_irq_data {
0178     call_single_data_t csd;
0179     atomic_t scheduled;
0180 };
0181 static DEFINE_PER_CPU_SHARED_ALIGNED(struct cpu_irq_data, irq_data);
0182
0183 static void zpci_handle_remote_irq(void *data)
0184 {
0185     atomic_t *scheduled = data;
0186
0187     do {
0188         zpci_handle_cpu_local_irq(false);
0189     } while (atomic_dec_return(scheduled));
0190 }
0191
0192 static void zpci_handle_fallback_irq(void)
0193 {
0194     struct cpu_irq_data *cpu_data;
0195     union zpci_sic_iib iib = {{0}};
0196     unsigned long cpu;
0197     int irqs_on = 0;
0198
0199     for (cpu = 0;;) {
0200         cpu = airq_iv_scan(zpci_sbv, cpu, airq_iv_end(zpci_sbv));
0201         if (cpu == -1UL) {
0202             if (irqs_on++)
0203                 /* End of second scan with interrupts on. */
0204                 break;
0205             /* First scan complete, reenable interrupts. */
0206             if (zpci_set_irq_ctrl(SIC_IRQ_MODE_SINGLE, PCI_ISC, &iib))
0207                 break;
0208             cpu = 0;
0209             continue;
0210         }
0211         cpu_data = &per_cpu(irq_data, cpu);
0212         if (atomic_inc_return(&cpu_data->scheduled) > 1)
0213             continue;
0214
0215         INIT_CSD(&cpu_data->csd, zpci_handle_remote_irq, &cpu_data->scheduled);
0216         smp_call_function_single_async(cpu, &cpu_data->csd);
0217     }
0218 }
0219
0220 static void zpci_directed_irq_handler(struct airq_struct *airq,
0221                       struct tpi_info *tpi_info)
0222 {
0223     bool floating = !tpi_info->directed_irq;
0224
0225     if (floating) {
0226         inc_irq_stat(IRQIO_PCF);
0227         zpci_handle_fallback_irq();
0228     } else {
0229         inc_irq_stat(IRQIO_PCD);
0230         zpci_handle_cpu_local_irq(true);
0231     }
0232 }
0233
0234 static void zpci_floating_irq_handler(struct airq_struct *airq,
0235                       struct tpi_info *tpi_info)
0236 {
0237     union zpci_sic_iib iib = {{0}};
0238     unsigned long si, ai;
0239     struct airq_iv *aibv;
0240     int irqs_on = 0;
0241
0242     inc_irq_stat(IRQIO_PCF);
0243     for (si = 0;;) {
0244         /* Scan adapter summary indicator bit vector */
0245         si = airq_iv_scan(zpci_sbv, si, airq_iv_end(zpci_sbv));
0246         if (si == -1UL) {
0247             if (irqs_on++)
0248                 /* End of second scan with interrupts on. */
0249                 break;
0250             /* First scan complete, reenable interrupts. */
0251             if (zpci_set_irq_ctrl(SIC_IRQ_MODE_SINGLE, PCI_ISC, &iib))
0252                 break;
0253             si = 0;
0254             continue;
0255         }
0256
0257         /* Scan the adapter interrupt vector for this device. */
0258         aibv = zpci_ibv[si];
0259         for (ai = 0;;) {
0260             ai = airq_iv_scan(aibv, ai, airq_iv_end(aibv));
0261             if (ai == -1UL)
0262                 break;
0263             inc_irq_stat(IRQIO_MSI);
0264             airq_iv_lock(aibv, ai);
0265             generic_handle_irq(airq_iv_get_data(aibv, ai));
0266             airq_iv_unlock(aibv, ai);
0267         }
0268     }
0269 }
0270
0271 int arch_setup_msi_irqs(struct pci_dev *pdev, int nvec, int type)
0272 {
0273     struct zpci_dev *zdev = to_zpci(pdev);
0274     unsigned int hwirq, msi_vecs, cpu;
0275     unsigned long bit;
0276     struct msi_desc *msi;
0277     struct msi_msg msg;
0278     int cpu_addr;
0279     int rc, irq;
0280
0281     zdev->aisb = -1UL;
0282     zdev->msi_first_bit = -1U;
0283     if (type == PCI_CAP_ID_MSI && nvec > 1)
0284         return 1;
0285     msi_vecs = min_t(unsigned int, nvec, zdev->max_msi);
0286
0287     if (irq_delivery == DIRECTED) {
0288         /* Allocate cpu vector bits */
0289         bit = airq_iv_alloc(zpci_ibv[0], msi_vecs);
0290         if (bit == -1UL)
0291             return -EIO;
0292     } else {
0293         /* Allocate adapter summary indicator bit */
0294         bit = airq_iv_alloc_bit(zpci_sbv);
0295         if (bit == -1UL)
0296             return -EIO;
0297         zdev->aisb = bit;
0298
0299         /* Create adapter interrupt vector */
0300         zdev->aibv = airq_iv_create(msi_vecs, AIRQ_IV_DATA | AIRQ_IV_BITLOCK, NULL);
0301         if (!zdev->aibv)
0302             return -ENOMEM;
0303
0304         /* Wire up shortcut pointer */
0305         zpci_ibv[bit] = zdev->aibv;
0306         /* Each function has its own interrupt vector */
0307         bit = 0;
0308     }
0309
0310     /* Request MSI interrupts */
0311     hwirq = bit;
0312     msi_for_each_desc(msi, &pdev->dev, MSI_DESC_NOTASSOCIATED) {
0313         rc = -EIO;
0314         if (hwirq - bit >= msi_vecs)
0315             break;
0316         irq = __irq_alloc_descs(-1, 0, 1, 0, THIS_MODULE,
0317                 (irq_delivery == DIRECTED) ?
0318                 msi->affinity : NULL);
0319         if (irq < 0)
0320             return -ENOMEM;
0321         rc = irq_set_msi_desc(irq, msi);
0322         if (rc)
0323             return rc;
0324         irq_set_chip_and_handler(irq, &zpci_irq_chip,
0325                      handle_percpu_irq);
0326         msg.data = hwirq - bit;
0327         if (irq_delivery == DIRECTED) {
0328             if (msi->affinity)
0329                 cpu = cpumask_first(&msi->affinity->mask);
0330             else
0331                 cpu = 0;
0332             cpu_addr = smp_cpu_get_cpu_address(cpu);
0333
0334             msg.address_lo = zdev->msi_addr & 0xff0000ff;
0335             msg.address_lo |= (cpu_addr << 8);
0336
0337             for_each_possible_cpu(cpu) {
0338                 airq_iv_set_data(zpci_ibv[cpu], hwirq, irq);
0339             }
0340         } else {
0341             msg.address_lo = zdev->msi_addr & 0xffffffff;
0342             airq_iv_set_data(zdev->aibv, hwirq, irq);
0343         }
0344         msg.address_hi = zdev->msi_addr >> 32;
0345         pci_write_msi_msg(irq, &msg);
0346         hwirq++;
0347     }
0348
0349     zdev->msi_first_bit = bit;
0350     zdev->msi_nr_irqs = msi_vecs;
0351
0352     rc = zpci_set_irq(zdev);
0353     if (rc)
0354         return rc;
0355
0356     return (msi_vecs == nvec) ? 0 : msi_vecs;
0357 }
0358
0359 void arch_teardown_msi_irqs(struct pci_dev *pdev)
0360 {
0361     struct zpci_dev *zdev = to_zpci(pdev);
0362     struct msi_desc *msi;
0363     int rc;
0364
0365     /* Disable interrupts */
0366     rc = zpci_clear_irq(zdev);
0367     if (rc)
0368         return;
0369
0370     /* Release MSI interrupts */
0371     msi_for_each_desc(msi, &pdev->dev, MSI_DESC_ASSOCIATED) {
0372         irq_set_msi_desc(msi->irq, NULL);
0373         irq_free_desc(msi->irq);
0374         msi->msg.address_lo = 0;
0375         msi->msg.address_hi = 0;
0376         msi->msg.data = 0;
0377         msi->irq = 0;
0378     }
0379
0380     if (zdev->aisb != -1UL) {
0381         zpci_ibv[zdev->aisb] = NULL;
0382         airq_iv_free_bit(zpci_sbv, zdev->aisb);
0383         zdev->aisb = -1UL;
0384     }
0385     if (zdev->aibv) {
0386         airq_iv_release(zdev->aibv);
0387         zdev->aibv = NULL;
0388     }
0389
0390     if ((irq_delivery == DIRECTED) && zdev->msi_first_bit != -1U)
0391         airq_iv_free(zpci_ibv[0], zdev->msi_first_bit, zdev->msi_nr_irqs);
0392 }
0393
0394 bool arch_restore_msi_irqs(struct pci_dev *pdev)
0395 {
0396     struct zpci_dev *zdev = to_zpci(pdev);
0397
0398     if (!zdev->irqs_registered)
0399         zpci_set_irq(zdev);
0400     return true;
0401 }
0402
0403 static struct airq_struct zpci_airq = {
0404     .handler = zpci_floating_irq_handler,
0405     .isc = PCI_ISC,
0406 };
0407
0408 static void __init cpu_enable_directed_irq(void *unused)
0409 {
0410     union zpci_sic_iib iib = {{0}};
0411     union zpci_sic_iib ziib = {{0}};
0412
0413     iib.cdiib.dibv_addr = (u64) zpci_ibv[smp_processor_id()]->vector;
0414
0415     zpci_set_irq_ctrl(SIC_IRQ_MODE_SET_CPU, 0, &iib);
0416     zpci_set_irq_ctrl(SIC_IRQ_MODE_D_SINGLE, PCI_ISC, &ziib);
0417 }
0418
0419 static int __init zpci_directed_irq_init(void)
0420 {
0421     union zpci_sic_iib iib = {{0}};
0422     unsigned int cpu;
0423
0424     zpci_sbv = airq_iv_create(num_possible_cpus(), 0, NULL);
0425     if (!zpci_sbv)
0426         return -ENOMEM;
0427
0428     iib.diib.isc = PCI_ISC;
0429     iib.diib.nr_cpus = num_possible_cpus();
0430     iib.diib.disb_addr = virt_to_phys(zpci_sbv->vector);
0431     zpci_set_irq_ctrl(SIC_IRQ_MODE_DIRECT, 0, &iib);
0432
0433     zpci_ibv = kcalloc(num_possible_cpus(), sizeof(*zpci_ibv),
0434                GFP_KERNEL);
0435     if (!zpci_ibv)
0436         return -ENOMEM;
0437
0438     for_each_possible_cpu(cpu) {
0439         /*
0440          * Per CPU IRQ vectors look the same but bit-allocation
0441          * is only done on the first vector.
0442          */
0443         zpci_ibv[cpu] = airq_iv_create(cache_line_size() * BITS_PER_BYTE,
0444                            AIRQ_IV_DATA |
0445                            AIRQ_IV_CACHELINE |
0446                            (!cpu ? AIRQ_IV_ALLOC : 0), NULL);
0447         if (!zpci_ibv[cpu])
0448             return -ENOMEM;
0449     }
0450     on_each_cpu(cpu_enable_directed_irq, NULL, 1);
0451
0452     zpci_irq_chip.irq_set_affinity = zpci_set_irq_affinity;
0453
0454     return 0;
0455 }
0456
0457 static int __init zpci_floating_irq_init(void)
0458 {
0459     zpci_ibv = kcalloc(ZPCI_NR_DEVICES, sizeof(*zpci_ibv), GFP_KERNEL);
0460     if (!zpci_ibv)
0461         return -ENOMEM;
0462
0463     zpci_sbv = airq_iv_create(ZPCI_NR_DEVICES, AIRQ_IV_ALLOC, NULL);
0464     if (!zpci_sbv)
0465         goto out_free;
0466
0467     return 0;
0468
0469 out_free:
0470     kfree(zpci_ibv);
0471     return -ENOMEM;
0472 }
0473
0474 int __init zpci_irq_init(void)
0475 {
0476     union zpci_sic_iib iib = {{0}};
0477     int rc;
0478
0479     irq_delivery = sclp.has_dirq ? DIRECTED : FLOATING;
0480     if (s390_pci_force_floating)
0481         irq_delivery = FLOATING;
0482
0483     if (irq_delivery == DIRECTED)
0484         zpci_airq.handler = zpci_directed_irq_handler;
0485
0486     rc = register_adapter_interrupt(&zpci_airq);
0487     if (rc)
0488         goto out;
0489     /* Set summary to 1 to be called every time for the ISC. */
0490     *zpci_airq.lsi_ptr = 1;
0491
0492     switch (irq_delivery) {
0493     case FLOATING:
0494         rc = zpci_floating_irq_init();
0495         break;
0496     case DIRECTED:
0497         rc = zpci_directed_irq_init();
0498         break;
0499     }
0500
0501     if (rc)
0502         goto out_airq;
0503
0504     /*
0505      * Enable floating IRQs (with suppression after one IRQ). When using
0506      * directed IRQs this enables the fallback path.
0507      */
0508     zpci_set_irq_ctrl(SIC_IRQ_MODE_SINGLE, PCI_ISC, &iib);
0509
0510     return 0;
0511 out_airq:
0512     unregister_adapter_interrupt(&zpci_airq);
0513 out:
0514     return rc;
0515 }
0516
0517 void __init zpci_irq_exit(void)
0518 {
0519     unsigned int cpu;
0520
0521     if (irq_delivery == DIRECTED) {
0522         for_each_possible_cpu(cpu) {
0523             airq_iv_release(zpci_ibv[cpu]);
0524         }
0525     }
0526     kfree(zpci_ibv);
0527     if (zpci_sbv)
0528         airq_iv_release(zpci_sbv);
0529     unregister_adapter_interrupt(&zpci_airq);
0530 }