dma/idxd/irq.c

0001 // SPDX-License-Identifier: GPL-2.0
0002 /* Copyright(c) 2019 Intel Corporation. All rights rsvd. */
0003 #include <linux/init.h>
0004 #include <linux/kernel.h>
0005 #include <linux/module.h>
0006 #include <linux/pci.h>
0007 #include <linux/io-64-nonatomic-lo-hi.h>
0008 #include <linux/dmaengine.h>
0009 #include <linux/delay.h>
0010 #include <uapi/linux/idxd.h>
0011 #include "../dmaengine.h"
0012 #include "idxd.h"
0013 #include "registers.h"
0014
0015 enum irq_work_type {
0016     IRQ_WORK_NORMAL = 0,
0017     IRQ_WORK_PROCESS_FAULT,
0018 };
0019
0020 struct idxd_fault {
0021     struct work_struct work;
0022     u64 addr;
0023     struct idxd_device *idxd;
0024 };
0025
0026 struct idxd_resubmit {
0027     struct work_struct work;
0028     struct idxd_desc *desc;
0029 };
0030
0031 struct idxd_int_handle_revoke {
0032     struct work_struct work;
0033     struct idxd_device *idxd;
0034 };
0035
0036 static void idxd_device_reinit(struct work_struct *work)
0037 {
0038     struct idxd_device *idxd = container_of(work, struct idxd_device, work);
0039     struct device *dev = &idxd->pdev->dev;
0040     int rc, i;
0041
0042     idxd_device_reset(idxd);
0043     rc = idxd_device_config(idxd);
0044     if (rc < 0)
0045         goto out;
0046
0047     rc = idxd_device_enable(idxd);
0048     if (rc < 0)
0049         goto out;
0050
0051     for (i = 0; i < idxd->max_wqs; i++) {
0052         struct idxd_wq *wq = idxd->wqs[i];
0053
0054         if (wq->state == IDXD_WQ_ENABLED) {
0055             rc = idxd_wq_enable(wq);
0056             if (rc < 0) {
0057                 dev_warn(dev, "Unable to re-enable wq %s\n",
0058                      dev_name(wq_confdev(wq)));
0059             }
0060         }
0061     }
0062
0063     return;
0064
0065  out:
0066     idxd_device_clear_state(idxd);
0067 }
0068
0069 /*
0070  * The function sends a drain descriptor for the interrupt handle. The drain ensures
0071  * all descriptors with this interrupt handle is flushed and the interrupt
0072  * will allow the cleanup of the outstanding descriptors.
0073  */
0074 static void idxd_int_handle_revoke_drain(struct idxd_irq_entry *ie)
0075 {
0076     struct idxd_wq *wq = ie_to_wq(ie);
0077     struct idxd_device *idxd = wq->idxd;
0078     struct device *dev = &idxd->pdev->dev;
0079     struct dsa_hw_desc desc = {};
0080     void __iomem *portal;
0081     int rc;
0082
0083     /* Issue a simple drain operation with interrupt but no completion record */
0084     desc.flags = IDXD_OP_FLAG_RCI;
0085     desc.opcode = DSA_OPCODE_DRAIN;
0086     desc.priv = 1;
0087
0088     if (ie->pasid != INVALID_IOASID)
0089         desc.pasid = ie->pasid;
0090     desc.int_handle = ie->int_handle;
0091     portal = idxd_wq_portal_addr(wq);
0092
0093     /*
0094      * The wmb() makes sure that the descriptor is all there before we
0095      * issue.
0096      */
0097     wmb();
0098     if (wq_dedicated(wq)) {
0099         iosubmit_cmds512(portal, &desc, 1);
0100     } else {
0101         rc = idxd_enqcmds(wq, portal, &desc);
0102         /* This should not fail unless hardware failed. */
0103         if (rc < 0)
0104             dev_warn(dev, "Failed to submit drain desc on wq %d\n", wq->id);
0105     }
0106 }
0107
0108 static void idxd_abort_invalid_int_handle_descs(struct idxd_irq_entry *ie)
0109 {
0110     LIST_HEAD(flist);
0111     struct idxd_desc *d, *t;
0112     struct llist_node *head;
0113
0114     spin_lock(&ie->list_lock);
0115     head = llist_del_all(&ie->pending_llist);
0116     if (head) {
0117         llist_for_each_entry_safe(d, t, head, llnode)
0118             list_add_tail(&d->list, &ie->work_list);
0119     }
0120
0121     list_for_each_entry_safe(d, t, &ie->work_list, list) {
0122         if (d->completion->status == DSA_COMP_INT_HANDLE_INVAL)
0123             list_move_tail(&d->list, &flist);
0124     }
0125     spin_unlock(&ie->list_lock);
0126
0127     list_for_each_entry_safe(d, t, &flist, list) {
0128         list_del(&d->list);
0129         idxd_dma_complete_txd(d, IDXD_COMPLETE_ABORT, true);
0130     }
0131 }
0132
0133 static void idxd_int_handle_revoke(struct work_struct *work)
0134 {
0135     struct idxd_int_handle_revoke *revoke =
0136         container_of(work, struct idxd_int_handle_revoke, work);
0137     struct idxd_device *idxd = revoke->idxd;
0138     struct pci_dev *pdev = idxd->pdev;
0139     struct device *dev = &pdev->dev;
0140     int i, new_handle, rc;
0141
0142     if (!idxd->request_int_handles) {
0143         kfree(revoke);
0144         dev_warn(dev, "Unexpected int handle refresh interrupt.\n");
0145         return;
0146     }
0147
0148     /*
0149      * The loop attempts to acquire new interrupt handle for all interrupt
0150      * vectors that supports a handle. If a new interrupt handle is acquired and the
0151      * wq is kernel type, the driver will kill the percpu_ref to pause all
0152      * ongoing descriptor submissions. The interrupt handle is then changed.
0153      * After change, the percpu_ref is revived and all the pending submissions
0154      * are woken to try again. A drain is sent to for the interrupt handle
0155      * at the end to make sure all invalid int handle descriptors are processed.
0156      */
0157     for (i = 1; i < idxd->irq_cnt; i++) {
0158         struct idxd_irq_entry *ie = idxd_get_ie(idxd, i);
0159         struct idxd_wq *wq = ie_to_wq(ie);
0160
0161         if (ie->int_handle == INVALID_INT_HANDLE)
0162             continue;
0163
0164         rc = idxd_device_request_int_handle(idxd, i, &new_handle, IDXD_IRQ_MSIX);
0165         if (rc < 0) {
0166             dev_warn(dev, "get int handle %d failed: %d\n", i, rc);
0167             /*
0168              * Failed to acquire new interrupt handle. Kill the WQ
0169              * and release all the pending submitters. The submitters will
0170              * get error return code and handle appropriately.
0171              */
0172             ie->int_handle = INVALID_INT_HANDLE;
0173             idxd_wq_quiesce(wq);
0174             idxd_abort_invalid_int_handle_descs(ie);
0175             continue;
0176         }
0177
0178         /* No change in interrupt handle, nothing needs to be done */
0179         if (ie->int_handle == new_handle)
0180             continue;
0181
0182         if (wq->state != IDXD_WQ_ENABLED || wq->type != IDXD_WQT_KERNEL) {
0183             /*
0184              * All the MSIX interrupts are allocated at once during probe.
0185              * Therefore we need to update all interrupts even if the WQ
0186              * isn't supporting interrupt operations.
0187              */
0188             ie->int_handle = new_handle;
0189             continue;
0190         }
0191
0192         mutex_lock(&wq->wq_lock);
0193         reinit_completion(&wq->wq_resurrect);
0194
0195         /* Kill percpu_ref to pause additional descriptor submissions */
0196         percpu_ref_kill(&wq->wq_active);
0197
0198         /* Wait for all submitters quiesce before we change interrupt handle */
0199         wait_for_completion(&wq->wq_dead);
0200
0201         ie->int_handle = new_handle;
0202
0203         /* Revive percpu ref and wake up all the waiting submitters */
0204         percpu_ref_reinit(&wq->wq_active);
0205         complete_all(&wq->wq_resurrect);
0206         mutex_unlock(&wq->wq_lock);
0207
0208         /*
0209          * The delay here is to wait for all possible MOVDIR64B that
0210          * are issued before percpu_ref_kill() has happened to have
0211          * reached the PCIe domain before the drain is issued. The driver
0212          * needs to ensure that the drain descriptor issued does not pass
0213          * all the other issued descriptors that contain the invalid
0214          * interrupt handle in order to ensure that the drain descriptor
0215          * interrupt will allow the cleanup of all the descriptors with
0216          * invalid interrupt handle.
0217          */
0218         if (wq_dedicated(wq))
0219             udelay(100);
0220         idxd_int_handle_revoke_drain(ie);
0221     }
0222     kfree(revoke);
0223 }
0224
0225 static int process_misc_interrupts(struct idxd_device *idxd, u32 cause)
0226 {
0227     struct device *dev = &idxd->pdev->dev;
0228     union gensts_reg gensts;
0229     u32 val = 0;
0230     int i;
0231     bool err = false;
0232
0233     if (cause & IDXD_INTC_HALT_STATE)
0234         goto halt;
0235
0236     if (cause & IDXD_INTC_ERR) {
0237         spin_lock(&idxd->dev_lock);
0238         for (i = 0; i < 4; i++)
0239             idxd->sw_err.bits[i] = ioread64(idxd->reg_base +
0240                     IDXD_SWERR_OFFSET + i * sizeof(u64));
0241
0242         iowrite64(idxd->sw_err.bits[0] & IDXD_SWERR_ACK,
0243               idxd->reg_base + IDXD_SWERR_OFFSET);
0244
0245         if (idxd->sw_err.valid && idxd->sw_err.wq_idx_valid) {
0246             int id = idxd->sw_err.wq_idx;
0247             struct idxd_wq *wq = idxd->wqs[id];
0248
0249             if (wq->type == IDXD_WQT_USER)
0250                 wake_up_interruptible(&wq->err_queue);
0251         } else {
0252             int i;
0253
0254             for (i = 0; i < idxd->max_wqs; i++) {
0255                 struct idxd_wq *wq = idxd->wqs[i];
0256
0257                 if (wq->type == IDXD_WQT_USER)
0258                     wake_up_interruptible(&wq->err_queue);
0259             }
0260         }
0261
0262         spin_unlock(&idxd->dev_lock);
0263         val |= IDXD_INTC_ERR;
0264
0265         for (i = 0; i < 4; i++)
0266             dev_warn(dev, "err[%d]: %#16.16llx\n",
0267                  i, idxd->sw_err.bits[i]);
0268         err = true;
0269     }
0270
0271     if (cause & IDXD_INTC_INT_HANDLE_REVOKED) {
0272         struct idxd_int_handle_revoke *revoke;
0273
0274         val |= IDXD_INTC_INT_HANDLE_REVOKED;
0275
0276         revoke = kzalloc(sizeof(*revoke), GFP_ATOMIC);
0277         if (revoke) {
0278             revoke->idxd = idxd;
0279             INIT_WORK(&revoke->work, idxd_int_handle_revoke);
0280             queue_work(idxd->wq, &revoke->work);
0281
0282         } else {
0283             dev_err(dev, "Failed to allocate work for int handle revoke\n");
0284             idxd_wqs_quiesce(idxd);
0285         }
0286     }
0287
0288     if (cause & IDXD_INTC_CMD) {
0289         val |= IDXD_INTC_CMD;
0290         complete(idxd->cmd_done);
0291     }
0292
0293     if (cause & IDXD_INTC_OCCUPY) {
0294         /* Driver does not utilize occupancy interrupt */
0295         val |= IDXD_INTC_OCCUPY;
0296     }
0297
0298     if (cause & IDXD_INTC_PERFMON_OVFL) {
0299         val |= IDXD_INTC_PERFMON_OVFL;
0300         perfmon_counter_overflow(idxd);
0301     }
0302
0303     val ^= cause;
0304     if (val)
0305         dev_warn_once(dev, "Unexpected interrupt cause bits set: %#x\n",
0306                   val);
0307
0308     if (!err)
0309         return 0;
0310
0311 halt:
0312     gensts.bits = ioread32(idxd->reg_base + IDXD_GENSTATS_OFFSET);
0313     if (gensts.state == IDXD_DEVICE_STATE_HALT) {
0314         idxd->state = IDXD_DEV_HALTED;
0315         if (gensts.reset_type == IDXD_DEVICE_RESET_SOFTWARE) {
0316             /*
0317              * If we need a software reset, we will throw the work
0318              * on a system workqueue in order to allow interrupts
0319              * for the device command completions.
0320              */
0321             INIT_WORK(&idxd->work, idxd_device_reinit);
0322             queue_work(idxd->wq, &idxd->work);
0323         } else {
0324             idxd->state = IDXD_DEV_HALTED;
0325             idxd_wqs_quiesce(idxd);
0326             idxd_wqs_unmap_portal(idxd);
0327             spin_lock(&idxd->dev_lock);
0328             idxd_device_clear_state(idxd);
0329             dev_err(&idxd->pdev->dev,
0330                 "idxd halted, need %s.\n",
0331                 gensts.reset_type == IDXD_DEVICE_RESET_FLR ?
0332                 "FLR" : "system reset");
0333             spin_unlock(&idxd->dev_lock);
0334             return -ENXIO;
0335         }
0336     }
0337
0338     return 0;
0339 }
0340
0341 irqreturn_t idxd_misc_thread(int vec, void *data)
0342 {
0343     struct idxd_irq_entry *irq_entry = data;
0344     struct idxd_device *idxd = ie_to_idxd(irq_entry);
0345     int rc;
0346     u32 cause;
0347
0348     cause = ioread32(idxd->reg_base + IDXD_INTCAUSE_OFFSET);
0349     if (cause)
0350         iowrite32(cause, idxd->reg_base + IDXD_INTCAUSE_OFFSET);
0351
0352     while (cause) {
0353         rc = process_misc_interrupts(idxd, cause);
0354         if (rc < 0)
0355             break;
0356         cause = ioread32(idxd->reg_base + IDXD_INTCAUSE_OFFSET);
0357         if (cause)
0358             iowrite32(cause, idxd->reg_base + IDXD_INTCAUSE_OFFSET);
0359     }
0360
0361     return IRQ_HANDLED;
0362 }
0363
0364 static void idxd_int_handle_resubmit_work(struct work_struct *work)
0365 {
0366     struct idxd_resubmit *irw = container_of(work, struct idxd_resubmit, work);
0367     struct idxd_desc *desc = irw->desc;
0368     struct idxd_wq *wq = desc->wq;
0369     int rc;
0370
0371     desc->completion->status = 0;
0372     rc = idxd_submit_desc(wq, desc);
0373     if (rc < 0) {
0374         dev_dbg(&wq->idxd->pdev->dev, "Failed to resubmit desc %d to wq %d.\n",
0375             desc->id, wq->id);
0376         /*
0377          * If the error is not -EAGAIN, it means the submission failed due to wq
0378          * has been killed instead of ENQCMDS failure. Here the driver needs to
0379          * notify the submitter of the failure by reporting abort status.
0380          *
0381          * -EAGAIN comes from ENQCMDS failure. idxd_submit_desc() will handle the
0382          * abort.
0383          */
0384         if (rc != -EAGAIN) {
0385             desc->completion->status = IDXD_COMP_DESC_ABORT;
0386             idxd_dma_complete_txd(desc, IDXD_COMPLETE_ABORT, false);
0387         }
0388         idxd_free_desc(wq, desc);
0389     }
0390     kfree(irw);
0391 }
0392
0393 bool idxd_queue_int_handle_resubmit(struct idxd_desc *desc)
0394 {
0395     struct idxd_wq *wq = desc->wq;
0396     struct idxd_device *idxd = wq->idxd;
0397     struct idxd_resubmit *irw;
0398
0399     irw = kzalloc(sizeof(*irw), GFP_KERNEL);
0400     if (!irw)
0401         return false;
0402
0403     irw->desc = desc;
0404     INIT_WORK(&irw->work, idxd_int_handle_resubmit_work);
0405     queue_work(idxd->wq, &irw->work);
0406     return true;
0407 }
0408
0409 static void irq_process_pending_llist(struct idxd_irq_entry *irq_entry)
0410 {
0411     struct idxd_desc *desc, *t;
0412     struct llist_node *head;
0413
0414     head = llist_del_all(&irq_entry->pending_llist);
0415     if (!head)
0416         return;
0417
0418     llist_for_each_entry_safe(desc, t, head, llnode) {
0419         u8 status = desc->completion->status & DSA_COMP_STATUS_MASK;
0420
0421         if (status) {
0422             /*
0423              * Check against the original status as ABORT is software defined
0424              * and 0xff, which DSA_COMP_STATUS_MASK can mask out.
0425              */
0426             if (unlikely(desc->completion->status == IDXD_COMP_DESC_ABORT)) {
0427                 idxd_dma_complete_txd(desc, IDXD_COMPLETE_ABORT, true);
0428                 continue;
0429             }
0430
0431             idxd_dma_complete_txd(desc, IDXD_COMPLETE_NORMAL, true);
0432         } else {
0433             spin_lock(&irq_entry->list_lock);
0434             list_add_tail(&desc->list,
0435                       &irq_entry->work_list);
0436             spin_unlock(&irq_entry->list_lock);
0437         }
0438     }
0439 }
0440
0441 static void irq_process_work_list(struct idxd_irq_entry *irq_entry)
0442 {
0443     LIST_HEAD(flist);
0444     struct idxd_desc *desc, *n;
0445
0446     /*
0447      * This lock protects list corruption from access of list outside of the irq handler
0448      * thread.
0449      */
0450     spin_lock(&irq_entry->list_lock);
0451     if (list_empty(&irq_entry->work_list)) {
0452         spin_unlock(&irq_entry->list_lock);
0453         return;
0454     }
0455
0456     list_for_each_entry_safe(desc, n, &irq_entry->work_list, list) {
0457         if (desc->completion->status) {
0458             list_move_tail(&desc->list, &flist);
0459         }
0460     }
0461
0462     spin_unlock(&irq_entry->list_lock);
0463
0464     list_for_each_entry(desc, &flist, list) {
0465         /*
0466          * Check against the original status as ABORT is software defined
0467          * and 0xff, which DSA_COMP_STATUS_MASK can mask out.
0468          */
0469         if (unlikely(desc->completion->status == IDXD_COMP_DESC_ABORT)) {
0470             idxd_dma_complete_txd(desc, IDXD_COMPLETE_ABORT, true);
0471             continue;
0472         }
0473
0474         idxd_dma_complete_txd(desc, IDXD_COMPLETE_NORMAL, true);
0475     }
0476 }
0477
0478 irqreturn_t idxd_wq_thread(int irq, void *data)
0479 {
0480     struct idxd_irq_entry *irq_entry = data;
0481
0482     /*
0483      * There are two lists we are processing. The pending_llist is where
0484      * submmiter adds all the submitted descriptor after sending it to
0485      * the workqueue. It's a lockless singly linked list. The work_list
0486      * is the common linux double linked list. We are in a scenario of
0487      * multiple producers and a single consumer. The producers are all
0488      * the kernel submitters of descriptors, and the consumer is the
0489      * kernel irq handler thread for the msix vector when using threaded
0490      * irq. To work with the restrictions of llist to remain lockless,
0491      * we are doing the following steps:
0492      * 1. Iterate through the work_list and process any completed
0493      *    descriptor. Delete the completed entries during iteration.
0494      * 2. llist_del_all() from the pending list.
0495      * 3. Iterate through the llist that was deleted from the pending list
0496      *    and process the completed entries.
0497      * 4. If the entry is still waiting on hardware, list_add_tail() to
0498      *    the work_list.
0499      */
0500     irq_process_work_list(irq_entry);
0501     irq_process_pending_llist(irq_entry);
0502
0503     return IRQ_HANDLED;
0504 }