hw/hfi1/msix.c

0001 // SPDX-License-Identifier: (GPL-2.0 OR BSD-3-Clause)
0002 /*
0003  * Copyright(c) 2018 - 2020 Intel Corporation.
0004  */
0005
0006 #include "hfi.h"
0007 #include "affinity.h"
0008 #include "sdma.h"
0009 #include "netdev.h"
0010
0011 /**
0012  * msix_initialize() - Calculate, request and configure MSIx IRQs
0013  * @dd: valid hfi1 devdata
0014  *
0015  */
0016 int msix_initialize(struct hfi1_devdata *dd)
0017 {
0018     u32 total;
0019     int ret;
0020     struct hfi1_msix_entry *entries;
0021
0022     /*
0023      * MSIx interrupt count:
0024      *  one for the general, "slow path" interrupt
0025      *  one per used SDMA engine
0026      *  one per kernel receive context
0027      *  one for each VNIC context
0028      *      ...any new IRQs should be added here.
0029      */
0030     total = 1 + dd->num_sdma + dd->n_krcv_queues + dd->num_netdev_contexts;
0031
0032     if (total >= CCE_NUM_MSIX_VECTORS)
0033         return -EINVAL;
0034
0035     ret = pci_alloc_irq_vectors(dd->pcidev, total, total, PCI_IRQ_MSIX);
0036     if (ret < 0) {
0037         dd_dev_err(dd, "pci_alloc_irq_vectors() failed: %d\n", ret);
0038         return ret;
0039     }
0040
0041     entries = kcalloc(total, sizeof(*dd->msix_info.msix_entries),
0042               GFP_KERNEL);
0043     if (!entries) {
0044         pci_free_irq_vectors(dd->pcidev);
0045         return -ENOMEM;
0046     }
0047
0048     dd->msix_info.msix_entries = entries;
0049     spin_lock_init(&dd->msix_info.msix_lock);
0050     bitmap_zero(dd->msix_info.in_use_msix, total);
0051     dd->msix_info.max_requested = total;
0052     dd_dev_info(dd, "%u MSI-X interrupts allocated\n", total);
0053
0054     return 0;
0055 }
0056
0057 /**
0058  * msix_request_irq() - Allocate a free MSIx IRQ
0059  * @dd: valid devdata
0060  * @arg: context information for the IRQ
0061  * @handler: IRQ handler
0062  * @thread: IRQ thread handler (could be NULL)
0063  * @type: affinty IRQ type
0064  * @name: IRQ name
0065  *
0066  * Allocated an MSIx vector if available, and then create the appropriate
0067  * meta data needed to keep track of the pci IRQ request.
0068  *
0069  * Return:
0070  *   < 0   Error
0071  *   >= 0  MSIx vector
0072  *
0073  */
0074 static int msix_request_irq(struct hfi1_devdata *dd, void *arg,
0075                 irq_handler_t handler, irq_handler_t thread,
0076                 enum irq_type type, const char *name)
0077 {
0078     unsigned long nr;
0079     int irq;
0080     int ret;
0081     struct hfi1_msix_entry *me;
0082
0083     /* Allocate an MSIx vector */
0084     spin_lock(&dd->msix_info.msix_lock);
0085     nr = find_first_zero_bit(dd->msix_info.in_use_msix,
0086                  dd->msix_info.max_requested);
0087     if (nr < dd->msix_info.max_requested)
0088         __set_bit(nr, dd->msix_info.in_use_msix);
0089     spin_unlock(&dd->msix_info.msix_lock);
0090
0091     if (nr == dd->msix_info.max_requested)
0092         return -ENOSPC;
0093
0094     if (type < IRQ_SDMA || type >= IRQ_OTHER)
0095         return -EINVAL;
0096
0097     irq = pci_irq_vector(dd->pcidev, nr);
0098     ret = pci_request_irq(dd->pcidev, nr, handler, thread, arg, name);
0099     if (ret) {
0100         dd_dev_err(dd,
0101                "%s: request for IRQ %d failed, MSIx %lx, err %d\n",
0102                name, irq, nr, ret);
0103         spin_lock(&dd->msix_info.msix_lock);
0104         __clear_bit(nr, dd->msix_info.in_use_msix);
0105         spin_unlock(&dd->msix_info.msix_lock);
0106         return ret;
0107     }
0108
0109     /*
0110      * assign arg after pci_request_irq call, so it will be
0111      * cleaned up
0112      */
0113     me = &dd->msix_info.msix_entries[nr];
0114     me->irq = irq;
0115     me->arg = arg;
0116     me->type = type;
0117
0118     /* This is a request, so a failure is not fatal */
0119     ret = hfi1_get_irq_affinity(dd, me);
0120     if (ret)
0121         dd_dev_err(dd, "%s: unable to pin IRQ %d\n", name, ret);
0122
0123     return nr;
0124 }
0125
0126 static int msix_request_rcd_irq_common(struct hfi1_ctxtdata *rcd,
0127                        irq_handler_t handler,
0128                        irq_handler_t thread,
0129                        const char *name)
0130 {
0131     int nr = msix_request_irq(rcd->dd, rcd, handler, thread,
0132                   rcd->is_vnic ? IRQ_NETDEVCTXT : IRQ_RCVCTXT,
0133                   name);
0134     if (nr < 0)
0135         return nr;
0136
0137     /*
0138      * Set the interrupt register and mask for this
0139      * context's interrupt.
0140      */
0141     rcd->ireg = (IS_RCVAVAIL_START + rcd->ctxt) / 64;
0142     rcd->imask = ((u64)1) << ((IS_RCVAVAIL_START + rcd->ctxt) % 64);
0143     rcd->msix_intr = nr;
0144     remap_intr(rcd->dd, IS_RCVAVAIL_START + rcd->ctxt, nr);
0145
0146     return 0;
0147 }
0148
0149 /**
0150  * msix_request_rcd_irq() - Helper function for RCVAVAIL IRQs
0151  * @rcd: valid rcd context
0152  *
0153  */
0154 int msix_request_rcd_irq(struct hfi1_ctxtdata *rcd)
0155 {
0156     char name[MAX_NAME_SIZE];
0157
0158     snprintf(name, sizeof(name), DRIVER_NAME "_%d kctxt%d",
0159          rcd->dd->unit, rcd->ctxt);
0160
0161     return msix_request_rcd_irq_common(rcd, receive_context_interrupt,
0162                        receive_context_thread, name);
0163 }
0164
0165 /**
0166  * msix_netdev_request_rcd_irq  - Helper function for RCVAVAIL IRQs
0167  * for netdev context
0168  * @rcd: valid netdev contexti
0169  */
0170 int msix_netdev_request_rcd_irq(struct hfi1_ctxtdata *rcd)
0171 {
0172     char name[MAX_NAME_SIZE];
0173
0174     snprintf(name, sizeof(name), DRIVER_NAME "_%d nd kctxt%d",
0175          rcd->dd->unit, rcd->ctxt);
0176     return msix_request_rcd_irq_common(rcd, receive_context_interrupt_napi,
0177                        NULL, name);
0178 }
0179
0180 /**
0181  * msix_request_sdma_irq  - Helper for getting SDMA IRQ resources
0182  * @sde: valid sdma engine
0183  *
0184  */
0185 int msix_request_sdma_irq(struct sdma_engine *sde)
0186 {
0187     int nr;
0188     char name[MAX_NAME_SIZE];
0189
0190     snprintf(name, sizeof(name), DRIVER_NAME "_%d sdma%d",
0191          sde->dd->unit, sde->this_idx);
0192     nr = msix_request_irq(sde->dd, sde, sdma_interrupt, NULL,
0193                   IRQ_SDMA, name);
0194     if (nr < 0)
0195         return nr;
0196     sde->msix_intr = nr;
0197     remap_sdma_interrupts(sde->dd, sde->this_idx, nr);
0198
0199     return 0;
0200 }
0201
0202 /**
0203  * msix_request_general_irq - Helper for getting general IRQ
0204  * resources
0205  * @dd: valid device data
0206  */
0207 int msix_request_general_irq(struct hfi1_devdata *dd)
0208 {
0209     int nr;
0210     char name[MAX_NAME_SIZE];
0211
0212     snprintf(name, sizeof(name), DRIVER_NAME "_%d", dd->unit);
0213     nr = msix_request_irq(dd, dd, general_interrupt, NULL, IRQ_GENERAL,
0214                   name);
0215     if (nr < 0)
0216         return nr;
0217
0218     /* general interrupt must be MSIx vector 0 */
0219     if (nr) {
0220         msix_free_irq(dd, (u8)nr);
0221         dd_dev_err(dd, "Invalid index %d for GENERAL IRQ\n", nr);
0222         return -EINVAL;
0223     }
0224
0225     return 0;
0226 }
0227
0228 /**
0229  * enable_sdma_srcs - Helper to enable SDMA IRQ srcs
0230  * @dd: valid devdata structure
0231  * @i: index of SDMA engine
0232  */
0233 static void enable_sdma_srcs(struct hfi1_devdata *dd, int i)
0234 {
0235     set_intr_bits(dd, IS_SDMA_START + i, IS_SDMA_START + i, true);
0236     set_intr_bits(dd, IS_SDMA_PROGRESS_START + i,
0237               IS_SDMA_PROGRESS_START + i, true);
0238     set_intr_bits(dd, IS_SDMA_IDLE_START + i, IS_SDMA_IDLE_START + i, true);
0239     set_intr_bits(dd, IS_SDMAENG_ERR_START + i, IS_SDMAENG_ERR_START + i,
0240               true);
0241 }
0242
0243 /**
0244  * msix_request_irqs() - Allocate all MSIx IRQs
0245  * @dd: valid devdata structure
0246  *
0247  * Helper function to request the used MSIx IRQs.
0248  *
0249  */
0250 int msix_request_irqs(struct hfi1_devdata *dd)
0251 {
0252     int i;
0253     int ret = msix_request_general_irq(dd);
0254
0255     if (ret)
0256         return ret;
0257
0258     for (i = 0; i < dd->num_sdma; i++) {
0259         struct sdma_engine *sde = &dd->per_sdma[i];
0260
0261         ret = msix_request_sdma_irq(sde);
0262         if (ret)
0263             return ret;
0264         enable_sdma_srcs(sde->dd, i);
0265     }
0266
0267     for (i = 0; i < dd->n_krcv_queues; i++) {
0268         struct hfi1_ctxtdata *rcd = hfi1_rcd_get_by_index_safe(dd, i);
0269
0270         if (rcd)
0271             ret = msix_request_rcd_irq(rcd);
0272         hfi1_rcd_put(rcd);
0273         if (ret)
0274             return ret;
0275     }
0276
0277     return 0;
0278 }
0279
0280 /**
0281  * msix_free_irq() - Free the specified MSIx resources and IRQ
0282  * @dd: valid devdata
0283  * @msix_intr: MSIx vector to free.
0284  *
0285  */
0286 void msix_free_irq(struct hfi1_devdata *dd, u8 msix_intr)
0287 {
0288     struct hfi1_msix_entry *me;
0289
0290     if (msix_intr >= dd->msix_info.max_requested)
0291         return;
0292
0293     me = &dd->msix_info.msix_entries[msix_intr];
0294
0295     if (!me->arg) /* => no irq, no affinity */
0296         return;
0297
0298     hfi1_put_irq_affinity(dd, me);
0299     pci_free_irq(dd->pcidev, msix_intr, me->arg);
0300
0301     me->arg = NULL;
0302
0303     spin_lock(&dd->msix_info.msix_lock);
0304     __clear_bit(msix_intr, dd->msix_info.in_use_msix);
0305     spin_unlock(&dd->msix_info.msix_lock);
0306 }
0307
0308 /**
0309  * msix_clean_up_interrupts  - Free all MSIx IRQ resources
0310  * @dd: valid device data data structure
0311  *
0312  * Free the MSIx and associated PCI resources, if they have been allocated.
0313  */
0314 void msix_clean_up_interrupts(struct hfi1_devdata *dd)
0315 {
0316     int i;
0317     struct hfi1_msix_entry *me = dd->msix_info.msix_entries;
0318
0319     /* remove irqs - must happen before disabling/turning off */
0320     for (i = 0; i < dd->msix_info.max_requested; i++, me++)
0321         msix_free_irq(dd, i);
0322
0323     /* clean structures */
0324     kfree(dd->msix_info.msix_entries);
0325     dd->msix_info.msix_entries = NULL;
0326     dd->msix_info.max_requested = 0;
0327
0328     pci_free_irq_vectors(dd->pcidev);
0329 }
0330
0331 /**
0332  * msix_netdev_synchronize_irq - netdev IRQ synchronize
0333  * @dd: valid devdata
0334  */
0335 void msix_netdev_synchronize_irq(struct hfi1_devdata *dd)
0336 {
0337     int i;
0338     int ctxt_count = hfi1_netdev_ctxt_count(dd);
0339
0340     for (i = 0; i < ctxt_count; i++) {
0341         struct hfi1_ctxtdata *rcd = hfi1_netdev_get_ctxt(dd, i);
0342         struct hfi1_msix_entry *me;
0343
0344         me = &dd->msix_info.msix_entries[rcd->msix_intr];
0345
0346         synchronize_irq(me->irq);
0347     }
0348 }