Back to home page

OSCL-LXR
 
 

     

0001 // SPDX-License-Identifier: GPL-2.0-only
0002 /*
0003  * Intel I/OAT DMA Linux driver
0004  * Copyright(c) 2004 - 2015 Intel Corporation.
0005  */
0006 
0007 #include <linux/init.h>
0008 #include <linux/module.h>
0009 #include <linux/slab.h>
0010 #include <linux/pci.h>
0011 #include <linux/interrupt.h>
0012 #include <linux/dmaengine.h>
0013 #include <linux/delay.h>
0014 #include <linux/dma-mapping.h>
0015 #include <linux/workqueue.h>
0016 #include <linux/prefetch.h>
0017 #include <linux/dca.h>
0018 #include <linux/aer.h>
0019 #include <linux/sizes.h>
0020 #include "dma.h"
0021 #include "registers.h"
0022 #include "hw.h"
0023 
0024 #include "../dmaengine.h"
0025 
0026 MODULE_VERSION(IOAT_DMA_VERSION);
0027 MODULE_LICENSE("Dual BSD/GPL");
0028 MODULE_AUTHOR("Intel Corporation");
0029 
0030 static const struct pci_device_id ioat_pci_tbl[] = {
0031     /* I/OAT v3 platforms */
0032     { PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_IOAT_TBG0) },
0033     { PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_IOAT_TBG1) },
0034     { PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_IOAT_TBG2) },
0035     { PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_IOAT_TBG3) },
0036     { PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_IOAT_TBG4) },
0037     { PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_IOAT_TBG5) },
0038     { PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_IOAT_TBG6) },
0039     { PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_IOAT_TBG7) },
0040 
0041     /* I/OAT v3.2 platforms */
0042     { PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_IOAT_JSF0) },
0043     { PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_IOAT_JSF1) },
0044     { PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_IOAT_JSF2) },
0045     { PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_IOAT_JSF3) },
0046     { PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_IOAT_JSF4) },
0047     { PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_IOAT_JSF5) },
0048     { PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_IOAT_JSF6) },
0049     { PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_IOAT_JSF7) },
0050     { PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_IOAT_JSF8) },
0051     { PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_IOAT_JSF9) },
0052 
0053     { PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_IOAT_SNB0) },
0054     { PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_IOAT_SNB1) },
0055     { PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_IOAT_SNB2) },
0056     { PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_IOAT_SNB3) },
0057     { PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_IOAT_SNB4) },
0058     { PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_IOAT_SNB5) },
0059     { PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_IOAT_SNB6) },
0060     { PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_IOAT_SNB7) },
0061     { PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_IOAT_SNB8) },
0062     { PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_IOAT_SNB9) },
0063 
0064     { PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_IOAT_IVB0) },
0065     { PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_IOAT_IVB1) },
0066     { PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_IOAT_IVB2) },
0067     { PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_IOAT_IVB3) },
0068     { PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_IOAT_IVB4) },
0069     { PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_IOAT_IVB5) },
0070     { PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_IOAT_IVB6) },
0071     { PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_IOAT_IVB7) },
0072     { PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_IOAT_IVB8) },
0073     { PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_IOAT_IVB9) },
0074 
0075     { PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_IOAT_HSW0) },
0076     { PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_IOAT_HSW1) },
0077     { PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_IOAT_HSW2) },
0078     { PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_IOAT_HSW3) },
0079     { PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_IOAT_HSW4) },
0080     { PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_IOAT_HSW5) },
0081     { PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_IOAT_HSW6) },
0082     { PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_IOAT_HSW7) },
0083     { PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_IOAT_HSW8) },
0084     { PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_IOAT_HSW9) },
0085 
0086     { PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_IOAT_BDX0) },
0087     { PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_IOAT_BDX1) },
0088     { PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_IOAT_BDX2) },
0089     { PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_IOAT_BDX3) },
0090     { PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_IOAT_BDX4) },
0091     { PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_IOAT_BDX5) },
0092     { PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_IOAT_BDX6) },
0093     { PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_IOAT_BDX7) },
0094     { PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_IOAT_BDX8) },
0095     { PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_IOAT_BDX9) },
0096 
0097     { PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_IOAT_SKX) },
0098 
0099     /* I/OAT v3.3 platforms */
0100     { PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_IOAT_BWD0) },
0101     { PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_IOAT_BWD1) },
0102     { PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_IOAT_BWD2) },
0103     { PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_IOAT_BWD3) },
0104 
0105     { PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_IOAT_BDXDE0) },
0106     { PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_IOAT_BDXDE1) },
0107     { PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_IOAT_BDXDE2) },
0108     { PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_IOAT_BDXDE3) },
0109 
0110     /* I/OAT v3.4 platforms */
0111     { PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_IOAT_ICX) },
0112 
0113     { 0, }
0114 };
0115 MODULE_DEVICE_TABLE(pci, ioat_pci_tbl);
0116 
0117 static int ioat_pci_probe(struct pci_dev *pdev, const struct pci_device_id *id);
0118 static void ioat_remove(struct pci_dev *pdev);
0119 static void
0120 ioat_init_channel(struct ioatdma_device *ioat_dma,
0121           struct ioatdma_chan *ioat_chan, int idx);
0122 static void ioat_intr_quirk(struct ioatdma_device *ioat_dma);
0123 static void ioat_enumerate_channels(struct ioatdma_device *ioat_dma);
0124 static int ioat3_dma_self_test(struct ioatdma_device *ioat_dma);
0125 
0126 static int ioat_dca_enabled = 1;
0127 module_param(ioat_dca_enabled, int, 0644);
0128 MODULE_PARM_DESC(ioat_dca_enabled, "control support of dca service (default: 1)");
0129 int ioat_pending_level = 7;
0130 module_param(ioat_pending_level, int, 0644);
0131 MODULE_PARM_DESC(ioat_pending_level,
0132          "high-water mark for pushing ioat descriptors (default: 7)");
0133 static char ioat_interrupt_style[32] = "msix";
0134 module_param_string(ioat_interrupt_style, ioat_interrupt_style,
0135             sizeof(ioat_interrupt_style), 0644);
0136 MODULE_PARM_DESC(ioat_interrupt_style,
0137          "set ioat interrupt style: msix (default), msi, intx");
0138 
0139 struct kmem_cache *ioat_cache;
0140 struct kmem_cache *ioat_sed_cache;
0141 
0142 static bool is_jf_ioat(struct pci_dev *pdev)
0143 {
0144     switch (pdev->device) {
0145     case PCI_DEVICE_ID_INTEL_IOAT_JSF0:
0146     case PCI_DEVICE_ID_INTEL_IOAT_JSF1:
0147     case PCI_DEVICE_ID_INTEL_IOAT_JSF2:
0148     case PCI_DEVICE_ID_INTEL_IOAT_JSF3:
0149     case PCI_DEVICE_ID_INTEL_IOAT_JSF4:
0150     case PCI_DEVICE_ID_INTEL_IOAT_JSF5:
0151     case PCI_DEVICE_ID_INTEL_IOAT_JSF6:
0152     case PCI_DEVICE_ID_INTEL_IOAT_JSF7:
0153     case PCI_DEVICE_ID_INTEL_IOAT_JSF8:
0154     case PCI_DEVICE_ID_INTEL_IOAT_JSF9:
0155         return true;
0156     default:
0157         return false;
0158     }
0159 }
0160 
0161 static bool is_snb_ioat(struct pci_dev *pdev)
0162 {
0163     switch (pdev->device) {
0164     case PCI_DEVICE_ID_INTEL_IOAT_SNB0:
0165     case PCI_DEVICE_ID_INTEL_IOAT_SNB1:
0166     case PCI_DEVICE_ID_INTEL_IOAT_SNB2:
0167     case PCI_DEVICE_ID_INTEL_IOAT_SNB3:
0168     case PCI_DEVICE_ID_INTEL_IOAT_SNB4:
0169     case PCI_DEVICE_ID_INTEL_IOAT_SNB5:
0170     case PCI_DEVICE_ID_INTEL_IOAT_SNB6:
0171     case PCI_DEVICE_ID_INTEL_IOAT_SNB7:
0172     case PCI_DEVICE_ID_INTEL_IOAT_SNB8:
0173     case PCI_DEVICE_ID_INTEL_IOAT_SNB9:
0174         return true;
0175     default:
0176         return false;
0177     }
0178 }
0179 
0180 static bool is_ivb_ioat(struct pci_dev *pdev)
0181 {
0182     switch (pdev->device) {
0183     case PCI_DEVICE_ID_INTEL_IOAT_IVB0:
0184     case PCI_DEVICE_ID_INTEL_IOAT_IVB1:
0185     case PCI_DEVICE_ID_INTEL_IOAT_IVB2:
0186     case PCI_DEVICE_ID_INTEL_IOAT_IVB3:
0187     case PCI_DEVICE_ID_INTEL_IOAT_IVB4:
0188     case PCI_DEVICE_ID_INTEL_IOAT_IVB5:
0189     case PCI_DEVICE_ID_INTEL_IOAT_IVB6:
0190     case PCI_DEVICE_ID_INTEL_IOAT_IVB7:
0191     case PCI_DEVICE_ID_INTEL_IOAT_IVB8:
0192     case PCI_DEVICE_ID_INTEL_IOAT_IVB9:
0193         return true;
0194     default:
0195         return false;
0196     }
0197 
0198 }
0199 
0200 static bool is_hsw_ioat(struct pci_dev *pdev)
0201 {
0202     switch (pdev->device) {
0203     case PCI_DEVICE_ID_INTEL_IOAT_HSW0:
0204     case PCI_DEVICE_ID_INTEL_IOAT_HSW1:
0205     case PCI_DEVICE_ID_INTEL_IOAT_HSW2:
0206     case PCI_DEVICE_ID_INTEL_IOAT_HSW3:
0207     case PCI_DEVICE_ID_INTEL_IOAT_HSW4:
0208     case PCI_DEVICE_ID_INTEL_IOAT_HSW5:
0209     case PCI_DEVICE_ID_INTEL_IOAT_HSW6:
0210     case PCI_DEVICE_ID_INTEL_IOAT_HSW7:
0211     case PCI_DEVICE_ID_INTEL_IOAT_HSW8:
0212     case PCI_DEVICE_ID_INTEL_IOAT_HSW9:
0213         return true;
0214     default:
0215         return false;
0216     }
0217 
0218 }
0219 
0220 static bool is_bdx_ioat(struct pci_dev *pdev)
0221 {
0222     switch (pdev->device) {
0223     case PCI_DEVICE_ID_INTEL_IOAT_BDX0:
0224     case PCI_DEVICE_ID_INTEL_IOAT_BDX1:
0225     case PCI_DEVICE_ID_INTEL_IOAT_BDX2:
0226     case PCI_DEVICE_ID_INTEL_IOAT_BDX3:
0227     case PCI_DEVICE_ID_INTEL_IOAT_BDX4:
0228     case PCI_DEVICE_ID_INTEL_IOAT_BDX5:
0229     case PCI_DEVICE_ID_INTEL_IOAT_BDX6:
0230     case PCI_DEVICE_ID_INTEL_IOAT_BDX7:
0231     case PCI_DEVICE_ID_INTEL_IOAT_BDX8:
0232     case PCI_DEVICE_ID_INTEL_IOAT_BDX9:
0233         return true;
0234     default:
0235         return false;
0236     }
0237 }
0238 
0239 static inline bool is_skx_ioat(struct pci_dev *pdev)
0240 {
0241     return (pdev->device == PCI_DEVICE_ID_INTEL_IOAT_SKX) ? true : false;
0242 }
0243 
0244 static bool is_xeon_cb32(struct pci_dev *pdev)
0245 {
0246     return is_jf_ioat(pdev) || is_snb_ioat(pdev) || is_ivb_ioat(pdev) ||
0247         is_hsw_ioat(pdev) || is_bdx_ioat(pdev) || is_skx_ioat(pdev);
0248 }
0249 
0250 bool is_bwd_ioat(struct pci_dev *pdev)
0251 {
0252     switch (pdev->device) {
0253     case PCI_DEVICE_ID_INTEL_IOAT_BWD0:
0254     case PCI_DEVICE_ID_INTEL_IOAT_BWD1:
0255     case PCI_DEVICE_ID_INTEL_IOAT_BWD2:
0256     case PCI_DEVICE_ID_INTEL_IOAT_BWD3:
0257     /* even though not Atom, BDX-DE has same DMA silicon */
0258     case PCI_DEVICE_ID_INTEL_IOAT_BDXDE0:
0259     case PCI_DEVICE_ID_INTEL_IOAT_BDXDE1:
0260     case PCI_DEVICE_ID_INTEL_IOAT_BDXDE2:
0261     case PCI_DEVICE_ID_INTEL_IOAT_BDXDE3:
0262         return true;
0263     default:
0264         return false;
0265     }
0266 }
0267 
0268 static bool is_bwd_noraid(struct pci_dev *pdev)
0269 {
0270     switch (pdev->device) {
0271     case PCI_DEVICE_ID_INTEL_IOAT_BWD2:
0272     case PCI_DEVICE_ID_INTEL_IOAT_BWD3:
0273     case PCI_DEVICE_ID_INTEL_IOAT_BDXDE0:
0274     case PCI_DEVICE_ID_INTEL_IOAT_BDXDE1:
0275     case PCI_DEVICE_ID_INTEL_IOAT_BDXDE2:
0276     case PCI_DEVICE_ID_INTEL_IOAT_BDXDE3:
0277         return true;
0278     default:
0279         return false;
0280     }
0281 
0282 }
0283 
0284 /*
0285  * Perform a IOAT transaction to verify the HW works.
0286  */
0287 #define IOAT_TEST_SIZE 2000
0288 
0289 static void ioat_dma_test_callback(void *dma_async_param)
0290 {
0291     struct completion *cmp = dma_async_param;
0292 
0293     complete(cmp);
0294 }
0295 
0296 /**
0297  * ioat_dma_self_test - Perform a IOAT transaction to verify the HW works.
0298  * @ioat_dma: dma device to be tested
0299  */
0300 static int ioat_dma_self_test(struct ioatdma_device *ioat_dma)
0301 {
0302     int i;
0303     u8 *src;
0304     u8 *dest;
0305     struct dma_device *dma = &ioat_dma->dma_dev;
0306     struct device *dev = &ioat_dma->pdev->dev;
0307     struct dma_chan *dma_chan;
0308     struct dma_async_tx_descriptor *tx;
0309     dma_addr_t dma_dest, dma_src;
0310     dma_cookie_t cookie;
0311     int err = 0;
0312     struct completion cmp;
0313     unsigned long tmo;
0314     unsigned long flags;
0315 
0316     src = kzalloc(IOAT_TEST_SIZE, GFP_KERNEL);
0317     if (!src)
0318         return -ENOMEM;
0319     dest = kzalloc(IOAT_TEST_SIZE, GFP_KERNEL);
0320     if (!dest) {
0321         kfree(src);
0322         return -ENOMEM;
0323     }
0324 
0325     /* Fill in src buffer */
0326     for (i = 0; i < IOAT_TEST_SIZE; i++)
0327         src[i] = (u8)i;
0328 
0329     /* Start copy, using first DMA channel */
0330     dma_chan = container_of(dma->channels.next, struct dma_chan,
0331                 device_node);
0332     if (dma->device_alloc_chan_resources(dma_chan) < 1) {
0333         dev_err(dev, "selftest cannot allocate chan resource\n");
0334         err = -ENODEV;
0335         goto out;
0336     }
0337 
0338     dma_src = dma_map_single(dev, src, IOAT_TEST_SIZE, DMA_TO_DEVICE);
0339     if (dma_mapping_error(dev, dma_src)) {
0340         dev_err(dev, "mapping src buffer failed\n");
0341         err = -ENOMEM;
0342         goto free_resources;
0343     }
0344     dma_dest = dma_map_single(dev, dest, IOAT_TEST_SIZE, DMA_FROM_DEVICE);
0345     if (dma_mapping_error(dev, dma_dest)) {
0346         dev_err(dev, "mapping dest buffer failed\n");
0347         err = -ENOMEM;
0348         goto unmap_src;
0349     }
0350     flags = DMA_PREP_INTERRUPT;
0351     tx = ioat_dma->dma_dev.device_prep_dma_memcpy(dma_chan, dma_dest,
0352                               dma_src, IOAT_TEST_SIZE,
0353                               flags);
0354     if (!tx) {
0355         dev_err(dev, "Self-test prep failed, disabling\n");
0356         err = -ENODEV;
0357         goto unmap_dma;
0358     }
0359 
0360     async_tx_ack(tx);
0361     init_completion(&cmp);
0362     tx->callback = ioat_dma_test_callback;
0363     tx->callback_param = &cmp;
0364     cookie = tx->tx_submit(tx);
0365     if (cookie < 0) {
0366         dev_err(dev, "Self-test setup failed, disabling\n");
0367         err = -ENODEV;
0368         goto unmap_dma;
0369     }
0370     dma->device_issue_pending(dma_chan);
0371 
0372     tmo = wait_for_completion_timeout(&cmp, msecs_to_jiffies(3000));
0373 
0374     if (tmo == 0 ||
0375         dma->device_tx_status(dma_chan, cookie, NULL)
0376                     != DMA_COMPLETE) {
0377         dev_err(dev, "Self-test copy timed out, disabling\n");
0378         err = -ENODEV;
0379         goto unmap_dma;
0380     }
0381     if (memcmp(src, dest, IOAT_TEST_SIZE)) {
0382         dev_err(dev, "Self-test copy failed compare, disabling\n");
0383         err = -ENODEV;
0384         goto unmap_dma;
0385     }
0386 
0387 unmap_dma:
0388     dma_unmap_single(dev, dma_dest, IOAT_TEST_SIZE, DMA_FROM_DEVICE);
0389 unmap_src:
0390     dma_unmap_single(dev, dma_src, IOAT_TEST_SIZE, DMA_TO_DEVICE);
0391 free_resources:
0392     dma->device_free_chan_resources(dma_chan);
0393 out:
0394     kfree(src);
0395     kfree(dest);
0396     return err;
0397 }
0398 
0399 /**
0400  * ioat_dma_setup_interrupts - setup interrupt handler
0401  * @ioat_dma: ioat dma device
0402  */
0403 int ioat_dma_setup_interrupts(struct ioatdma_device *ioat_dma)
0404 {
0405     struct ioatdma_chan *ioat_chan;
0406     struct pci_dev *pdev = ioat_dma->pdev;
0407     struct device *dev = &pdev->dev;
0408     struct msix_entry *msix;
0409     int i, j, msixcnt;
0410     int err = -EINVAL;
0411     u8 intrctrl = 0;
0412 
0413     if (!strcmp(ioat_interrupt_style, "msix"))
0414         goto msix;
0415     if (!strcmp(ioat_interrupt_style, "msi"))
0416         goto msi;
0417     if (!strcmp(ioat_interrupt_style, "intx"))
0418         goto intx;
0419     dev_err(dev, "invalid ioat_interrupt_style %s\n", ioat_interrupt_style);
0420     goto err_no_irq;
0421 
0422 msix:
0423     /* The number of MSI-X vectors should equal the number of channels */
0424     msixcnt = ioat_dma->dma_dev.chancnt;
0425     for (i = 0; i < msixcnt; i++)
0426         ioat_dma->msix_entries[i].entry = i;
0427 
0428     err = pci_enable_msix_exact(pdev, ioat_dma->msix_entries, msixcnt);
0429     if (err)
0430         goto msi;
0431 
0432     for (i = 0; i < msixcnt; i++) {
0433         msix = &ioat_dma->msix_entries[i];
0434         ioat_chan = ioat_chan_by_index(ioat_dma, i);
0435         err = devm_request_irq(dev, msix->vector,
0436                        ioat_dma_do_interrupt_msix, 0,
0437                        "ioat-msix", ioat_chan);
0438         if (err) {
0439             for (j = 0; j < i; j++) {
0440                 msix = &ioat_dma->msix_entries[j];
0441                 ioat_chan = ioat_chan_by_index(ioat_dma, j);
0442                 devm_free_irq(dev, msix->vector, ioat_chan);
0443             }
0444             goto msi;
0445         }
0446     }
0447     intrctrl |= IOAT_INTRCTRL_MSIX_VECTOR_CONTROL;
0448     ioat_dma->irq_mode = IOAT_MSIX;
0449     goto done;
0450 
0451 msi:
0452     err = pci_enable_msi(pdev);
0453     if (err)
0454         goto intx;
0455 
0456     err = devm_request_irq(dev, pdev->irq, ioat_dma_do_interrupt, 0,
0457                    "ioat-msi", ioat_dma);
0458     if (err) {
0459         pci_disable_msi(pdev);
0460         goto intx;
0461     }
0462     ioat_dma->irq_mode = IOAT_MSI;
0463     goto done;
0464 
0465 intx:
0466     err = devm_request_irq(dev, pdev->irq, ioat_dma_do_interrupt,
0467                    IRQF_SHARED, "ioat-intx", ioat_dma);
0468     if (err)
0469         goto err_no_irq;
0470 
0471     ioat_dma->irq_mode = IOAT_INTX;
0472 done:
0473     if (is_bwd_ioat(pdev))
0474         ioat_intr_quirk(ioat_dma);
0475     intrctrl |= IOAT_INTRCTRL_MASTER_INT_EN;
0476     writeb(intrctrl, ioat_dma->reg_base + IOAT_INTRCTRL_OFFSET);
0477     return 0;
0478 
0479 err_no_irq:
0480     /* Disable all interrupt generation */
0481     writeb(0, ioat_dma->reg_base + IOAT_INTRCTRL_OFFSET);
0482     ioat_dma->irq_mode = IOAT_NOIRQ;
0483     dev_err(dev, "no usable interrupts\n");
0484     return err;
0485 }
0486 
0487 static void ioat_disable_interrupts(struct ioatdma_device *ioat_dma)
0488 {
0489     /* Disable all interrupt generation */
0490     writeb(0, ioat_dma->reg_base + IOAT_INTRCTRL_OFFSET);
0491 }
0492 
0493 static int ioat_probe(struct ioatdma_device *ioat_dma)
0494 {
0495     int err = -ENODEV;
0496     struct dma_device *dma = &ioat_dma->dma_dev;
0497     struct pci_dev *pdev = ioat_dma->pdev;
0498     struct device *dev = &pdev->dev;
0499 
0500     ioat_dma->completion_pool = dma_pool_create("completion_pool", dev,
0501                             sizeof(u64),
0502                             SMP_CACHE_BYTES,
0503                             SMP_CACHE_BYTES);
0504 
0505     if (!ioat_dma->completion_pool) {
0506         err = -ENOMEM;
0507         goto err_out;
0508     }
0509 
0510     ioat_enumerate_channels(ioat_dma);
0511 
0512     dma_cap_set(DMA_MEMCPY, dma->cap_mask);
0513     dma->dev = &pdev->dev;
0514 
0515     if (!dma->chancnt) {
0516         dev_err(dev, "channel enumeration error\n");
0517         goto err_setup_interrupts;
0518     }
0519 
0520     err = ioat_dma_setup_interrupts(ioat_dma);
0521     if (err)
0522         goto err_setup_interrupts;
0523 
0524     err = ioat3_dma_self_test(ioat_dma);
0525     if (err)
0526         goto err_self_test;
0527 
0528     return 0;
0529 
0530 err_self_test:
0531     ioat_disable_interrupts(ioat_dma);
0532 err_setup_interrupts:
0533     dma_pool_destroy(ioat_dma->completion_pool);
0534 err_out:
0535     return err;
0536 }
0537 
0538 static int ioat_register(struct ioatdma_device *ioat_dma)
0539 {
0540     int err = dma_async_device_register(&ioat_dma->dma_dev);
0541 
0542     if (err) {
0543         ioat_disable_interrupts(ioat_dma);
0544         dma_pool_destroy(ioat_dma->completion_pool);
0545     }
0546 
0547     return err;
0548 }
0549 
0550 static void ioat_dma_remove(struct ioatdma_device *ioat_dma)
0551 {
0552     struct dma_device *dma = &ioat_dma->dma_dev;
0553 
0554     ioat_disable_interrupts(ioat_dma);
0555 
0556     ioat_kobject_del(ioat_dma);
0557 
0558     dma_async_device_unregister(dma);
0559 }
0560 
0561 /**
0562  * ioat_enumerate_channels - find and initialize the device's channels
0563  * @ioat_dma: the ioat dma device to be enumerated
0564  */
0565 static void ioat_enumerate_channels(struct ioatdma_device *ioat_dma)
0566 {
0567     struct ioatdma_chan *ioat_chan;
0568     struct device *dev = &ioat_dma->pdev->dev;
0569     struct dma_device *dma = &ioat_dma->dma_dev;
0570     u8 xfercap_log;
0571     int i;
0572 
0573     INIT_LIST_HEAD(&dma->channels);
0574     dma->chancnt = readb(ioat_dma->reg_base + IOAT_CHANCNT_OFFSET);
0575     dma->chancnt &= 0x1f; /* bits [4:0] valid */
0576     if (dma->chancnt > ARRAY_SIZE(ioat_dma->idx)) {
0577         dev_warn(dev, "(%d) exceeds max supported channels (%zu)\n",
0578              dma->chancnt, ARRAY_SIZE(ioat_dma->idx));
0579         dma->chancnt = ARRAY_SIZE(ioat_dma->idx);
0580     }
0581     xfercap_log = readb(ioat_dma->reg_base + IOAT_XFERCAP_OFFSET);
0582     xfercap_log &= 0x1f; /* bits [4:0] valid */
0583     if (xfercap_log == 0)
0584         return;
0585     dev_dbg(dev, "%s: xfercap = %d\n", __func__, 1 << xfercap_log);
0586 
0587     for (i = 0; i < dma->chancnt; i++) {
0588         ioat_chan = kzalloc(sizeof(*ioat_chan), GFP_KERNEL);
0589         if (!ioat_chan)
0590             break;
0591 
0592         ioat_init_channel(ioat_dma, ioat_chan, i);
0593         ioat_chan->xfercap_log = xfercap_log;
0594         spin_lock_init(&ioat_chan->prep_lock);
0595         if (ioat_reset_hw(ioat_chan)) {
0596             i = 0;
0597             break;
0598         }
0599     }
0600     dma->chancnt = i;
0601 }
0602 
0603 /**
0604  * ioat_free_chan_resources - release all the descriptors
0605  * @c: the channel to be cleaned
0606  */
0607 static void ioat_free_chan_resources(struct dma_chan *c)
0608 {
0609     struct ioatdma_chan *ioat_chan = to_ioat_chan(c);
0610     struct ioatdma_device *ioat_dma = ioat_chan->ioat_dma;
0611     struct ioat_ring_ent *desc;
0612     const int total_descs = 1 << ioat_chan->alloc_order;
0613     int descs;
0614     int i;
0615 
0616     /* Before freeing channel resources first check
0617      * if they have been previously allocated for this channel.
0618      */
0619     if (!ioat_chan->ring)
0620         return;
0621 
0622     ioat_stop(ioat_chan);
0623 
0624     if (!test_bit(IOAT_CHAN_DOWN, &ioat_chan->state)) {
0625         ioat_reset_hw(ioat_chan);
0626 
0627         /* Put LTR to idle */
0628         if (ioat_dma->version >= IOAT_VER_3_4)
0629             writeb(IOAT_CHAN_LTR_SWSEL_IDLE,
0630                    ioat_chan->reg_base +
0631                    IOAT_CHAN_LTR_SWSEL_OFFSET);
0632     }
0633 
0634     spin_lock_bh(&ioat_chan->cleanup_lock);
0635     spin_lock_bh(&ioat_chan->prep_lock);
0636     descs = ioat_ring_space(ioat_chan);
0637     dev_dbg(to_dev(ioat_chan), "freeing %d idle descriptors\n", descs);
0638     for (i = 0; i < descs; i++) {
0639         desc = ioat_get_ring_ent(ioat_chan, ioat_chan->head + i);
0640         ioat_free_ring_ent(desc, c);
0641     }
0642 
0643     if (descs < total_descs)
0644         dev_err(to_dev(ioat_chan), "Freeing %d in use descriptors!\n",
0645             total_descs - descs);
0646 
0647     for (i = 0; i < total_descs - descs; i++) {
0648         desc = ioat_get_ring_ent(ioat_chan, ioat_chan->tail + i);
0649         dump_desc_dbg(ioat_chan, desc);
0650         ioat_free_ring_ent(desc, c);
0651     }
0652 
0653     for (i = 0; i < ioat_chan->desc_chunks; i++) {
0654         dma_free_coherent(to_dev(ioat_chan), IOAT_CHUNK_SIZE,
0655                   ioat_chan->descs[i].virt,
0656                   ioat_chan->descs[i].hw);
0657         ioat_chan->descs[i].virt = NULL;
0658         ioat_chan->descs[i].hw = 0;
0659     }
0660     ioat_chan->desc_chunks = 0;
0661 
0662     kfree(ioat_chan->ring);
0663     ioat_chan->ring = NULL;
0664     ioat_chan->alloc_order = 0;
0665     dma_pool_free(ioat_dma->completion_pool, ioat_chan->completion,
0666               ioat_chan->completion_dma);
0667     spin_unlock_bh(&ioat_chan->prep_lock);
0668     spin_unlock_bh(&ioat_chan->cleanup_lock);
0669 
0670     ioat_chan->last_completion = 0;
0671     ioat_chan->completion_dma = 0;
0672     ioat_chan->dmacount = 0;
0673 }
0674 
0675 /* ioat_alloc_chan_resources - allocate/initialize ioat descriptor ring
0676  * @chan: channel to be initialized
0677  */
0678 static int ioat_alloc_chan_resources(struct dma_chan *c)
0679 {
0680     struct ioatdma_chan *ioat_chan = to_ioat_chan(c);
0681     struct ioat_ring_ent **ring;
0682     u64 status;
0683     int order;
0684     int i = 0;
0685     u32 chanerr;
0686 
0687     /* have we already been set up? */
0688     if (ioat_chan->ring)
0689         return 1 << ioat_chan->alloc_order;
0690 
0691     /* Setup register to interrupt and write completion status on error */
0692     writew(IOAT_CHANCTRL_RUN, ioat_chan->reg_base + IOAT_CHANCTRL_OFFSET);
0693 
0694     /* allocate a completion writeback area */
0695     /* doing 2 32bit writes to mmio since 1 64b write doesn't work */
0696     ioat_chan->completion =
0697         dma_pool_zalloc(ioat_chan->ioat_dma->completion_pool,
0698                 GFP_NOWAIT, &ioat_chan->completion_dma);
0699     if (!ioat_chan->completion)
0700         return -ENOMEM;
0701 
0702     writel(((u64)ioat_chan->completion_dma) & 0x00000000FFFFFFFF,
0703            ioat_chan->reg_base + IOAT_CHANCMP_OFFSET_LOW);
0704     writel(((u64)ioat_chan->completion_dma) >> 32,
0705            ioat_chan->reg_base + IOAT_CHANCMP_OFFSET_HIGH);
0706 
0707     order = IOAT_MAX_ORDER;
0708     ring = ioat_alloc_ring(c, order, GFP_NOWAIT);
0709     if (!ring)
0710         return -ENOMEM;
0711 
0712     spin_lock_bh(&ioat_chan->cleanup_lock);
0713     spin_lock_bh(&ioat_chan->prep_lock);
0714     ioat_chan->ring = ring;
0715     ioat_chan->head = 0;
0716     ioat_chan->issued = 0;
0717     ioat_chan->tail = 0;
0718     ioat_chan->alloc_order = order;
0719     set_bit(IOAT_RUN, &ioat_chan->state);
0720     spin_unlock_bh(&ioat_chan->prep_lock);
0721     spin_unlock_bh(&ioat_chan->cleanup_lock);
0722 
0723     /* Setting up LTR values for 3.4 or later */
0724     if (ioat_chan->ioat_dma->version >= IOAT_VER_3_4) {
0725         u32 lat_val;
0726 
0727         lat_val = IOAT_CHAN_LTR_ACTIVE_SNVAL |
0728             IOAT_CHAN_LTR_ACTIVE_SNLATSCALE |
0729             IOAT_CHAN_LTR_ACTIVE_SNREQMNT;
0730         writel(lat_val, ioat_chan->reg_base +
0731                 IOAT_CHAN_LTR_ACTIVE_OFFSET);
0732 
0733         lat_val = IOAT_CHAN_LTR_IDLE_SNVAL |
0734               IOAT_CHAN_LTR_IDLE_SNLATSCALE |
0735               IOAT_CHAN_LTR_IDLE_SNREQMNT;
0736         writel(lat_val, ioat_chan->reg_base +
0737                 IOAT_CHAN_LTR_IDLE_OFFSET);
0738 
0739         /* Select to active */
0740         writeb(IOAT_CHAN_LTR_SWSEL_ACTIVE,
0741                ioat_chan->reg_base +
0742                IOAT_CHAN_LTR_SWSEL_OFFSET);
0743     }
0744 
0745     ioat_start_null_desc(ioat_chan);
0746 
0747     /* check that we got off the ground */
0748     do {
0749         udelay(1);
0750         status = ioat_chansts(ioat_chan);
0751     } while (i++ < 20 && !is_ioat_active(status) && !is_ioat_idle(status));
0752 
0753     if (is_ioat_active(status) || is_ioat_idle(status))
0754         return 1 << ioat_chan->alloc_order;
0755 
0756     chanerr = readl(ioat_chan->reg_base + IOAT_CHANERR_OFFSET);
0757 
0758     dev_WARN(to_dev(ioat_chan),
0759          "failed to start channel chanerr: %#x\n", chanerr);
0760     ioat_free_chan_resources(c);
0761     return -EFAULT;
0762 }
0763 
0764 /* common channel initialization */
0765 static void
0766 ioat_init_channel(struct ioatdma_device *ioat_dma,
0767           struct ioatdma_chan *ioat_chan, int idx)
0768 {
0769     struct dma_device *dma = &ioat_dma->dma_dev;
0770 
0771     ioat_chan->ioat_dma = ioat_dma;
0772     ioat_chan->reg_base = ioat_dma->reg_base + (0x80 * (idx + 1));
0773     spin_lock_init(&ioat_chan->cleanup_lock);
0774     ioat_chan->dma_chan.device = dma;
0775     dma_cookie_init(&ioat_chan->dma_chan);
0776     list_add_tail(&ioat_chan->dma_chan.device_node, &dma->channels);
0777     ioat_dma->idx[idx] = ioat_chan;
0778     timer_setup(&ioat_chan->timer, ioat_timer_event, 0);
0779     tasklet_setup(&ioat_chan->cleanup_task, ioat_cleanup_event);
0780 }
0781 
0782 #define IOAT_NUM_SRC_TEST 6 /* must be <= 8 */
0783 static int ioat_xor_val_self_test(struct ioatdma_device *ioat_dma)
0784 {
0785     int i, src_idx;
0786     struct page *dest;
0787     struct page *xor_srcs[IOAT_NUM_SRC_TEST];
0788     struct page *xor_val_srcs[IOAT_NUM_SRC_TEST + 1];
0789     dma_addr_t dma_srcs[IOAT_NUM_SRC_TEST + 1];
0790     dma_addr_t dest_dma;
0791     struct dma_async_tx_descriptor *tx;
0792     struct dma_chan *dma_chan;
0793     dma_cookie_t cookie;
0794     u8 cmp_byte = 0;
0795     u32 cmp_word;
0796     u32 xor_val_result;
0797     int err = 0;
0798     struct completion cmp;
0799     unsigned long tmo;
0800     struct device *dev = &ioat_dma->pdev->dev;
0801     struct dma_device *dma = &ioat_dma->dma_dev;
0802     u8 op = 0;
0803 
0804     dev_dbg(dev, "%s\n", __func__);
0805 
0806     if (!dma_has_cap(DMA_XOR, dma->cap_mask))
0807         return 0;
0808 
0809     for (src_idx = 0; src_idx < IOAT_NUM_SRC_TEST; src_idx++) {
0810         xor_srcs[src_idx] = alloc_page(GFP_KERNEL);
0811         if (!xor_srcs[src_idx]) {
0812             while (src_idx--)
0813                 __free_page(xor_srcs[src_idx]);
0814             return -ENOMEM;
0815         }
0816     }
0817 
0818     dest = alloc_page(GFP_KERNEL);
0819     if (!dest) {
0820         while (src_idx--)
0821             __free_page(xor_srcs[src_idx]);
0822         return -ENOMEM;
0823     }
0824 
0825     /* Fill in src buffers */
0826     for (src_idx = 0; src_idx < IOAT_NUM_SRC_TEST; src_idx++) {
0827         u8 *ptr = page_address(xor_srcs[src_idx]);
0828 
0829         for (i = 0; i < PAGE_SIZE; i++)
0830             ptr[i] = (1 << src_idx);
0831     }
0832 
0833     for (src_idx = 0; src_idx < IOAT_NUM_SRC_TEST; src_idx++)
0834         cmp_byte ^= (u8) (1 << src_idx);
0835 
0836     cmp_word = (cmp_byte << 24) | (cmp_byte << 16) |
0837             (cmp_byte << 8) | cmp_byte;
0838 
0839     memset(page_address(dest), 0, PAGE_SIZE);
0840 
0841     dma_chan = container_of(dma->channels.next, struct dma_chan,
0842                 device_node);
0843     if (dma->device_alloc_chan_resources(dma_chan) < 1) {
0844         err = -ENODEV;
0845         goto out;
0846     }
0847 
0848     /* test xor */
0849     op = IOAT_OP_XOR;
0850 
0851     dest_dma = dma_map_page(dev, dest, 0, PAGE_SIZE, DMA_FROM_DEVICE);
0852     if (dma_mapping_error(dev, dest_dma)) {
0853         err = -ENOMEM;
0854         goto free_resources;
0855     }
0856 
0857     for (i = 0; i < IOAT_NUM_SRC_TEST; i++) {
0858         dma_srcs[i] = dma_map_page(dev, xor_srcs[i], 0, PAGE_SIZE,
0859                        DMA_TO_DEVICE);
0860         if (dma_mapping_error(dev, dma_srcs[i])) {
0861             err = -ENOMEM;
0862             goto dma_unmap;
0863         }
0864     }
0865     tx = dma->device_prep_dma_xor(dma_chan, dest_dma, dma_srcs,
0866                       IOAT_NUM_SRC_TEST, PAGE_SIZE,
0867                       DMA_PREP_INTERRUPT);
0868 
0869     if (!tx) {
0870         dev_err(dev, "Self-test xor prep failed\n");
0871         err = -ENODEV;
0872         goto dma_unmap;
0873     }
0874 
0875     async_tx_ack(tx);
0876     init_completion(&cmp);
0877     tx->callback = ioat_dma_test_callback;
0878     tx->callback_param = &cmp;
0879     cookie = tx->tx_submit(tx);
0880     if (cookie < 0) {
0881         dev_err(dev, "Self-test xor setup failed\n");
0882         err = -ENODEV;
0883         goto dma_unmap;
0884     }
0885     dma->device_issue_pending(dma_chan);
0886 
0887     tmo = wait_for_completion_timeout(&cmp, msecs_to_jiffies(3000));
0888 
0889     if (tmo == 0 ||
0890         dma->device_tx_status(dma_chan, cookie, NULL) != DMA_COMPLETE) {
0891         dev_err(dev, "Self-test xor timed out\n");
0892         err = -ENODEV;
0893         goto dma_unmap;
0894     }
0895 
0896     for (i = 0; i < IOAT_NUM_SRC_TEST; i++)
0897         dma_unmap_page(dev, dma_srcs[i], PAGE_SIZE, DMA_TO_DEVICE);
0898 
0899     dma_sync_single_for_cpu(dev, dest_dma, PAGE_SIZE, DMA_FROM_DEVICE);
0900     for (i = 0; i < (PAGE_SIZE / sizeof(u32)); i++) {
0901         u32 *ptr = page_address(dest);
0902 
0903         if (ptr[i] != cmp_word) {
0904             dev_err(dev, "Self-test xor failed compare\n");
0905             err = -ENODEV;
0906             goto free_resources;
0907         }
0908     }
0909     dma_sync_single_for_device(dev, dest_dma, PAGE_SIZE, DMA_FROM_DEVICE);
0910 
0911     dma_unmap_page(dev, dest_dma, PAGE_SIZE, DMA_FROM_DEVICE);
0912 
0913     /* skip validate if the capability is not present */
0914     if (!dma_has_cap(DMA_XOR_VAL, dma_chan->device->cap_mask))
0915         goto free_resources;
0916 
0917     op = IOAT_OP_XOR_VAL;
0918 
0919     /* validate the sources with the destintation page */
0920     for (i = 0; i < IOAT_NUM_SRC_TEST; i++)
0921         xor_val_srcs[i] = xor_srcs[i];
0922     xor_val_srcs[i] = dest;
0923 
0924     xor_val_result = 1;
0925 
0926     for (i = 0; i < IOAT_NUM_SRC_TEST + 1; i++) {
0927         dma_srcs[i] = dma_map_page(dev, xor_val_srcs[i], 0, PAGE_SIZE,
0928                        DMA_TO_DEVICE);
0929         if (dma_mapping_error(dev, dma_srcs[i])) {
0930             err = -ENOMEM;
0931             goto dma_unmap;
0932         }
0933     }
0934     tx = dma->device_prep_dma_xor_val(dma_chan, dma_srcs,
0935                       IOAT_NUM_SRC_TEST + 1, PAGE_SIZE,
0936                       &xor_val_result, DMA_PREP_INTERRUPT);
0937     if (!tx) {
0938         dev_err(dev, "Self-test zero prep failed\n");
0939         err = -ENODEV;
0940         goto dma_unmap;
0941     }
0942 
0943     async_tx_ack(tx);
0944     init_completion(&cmp);
0945     tx->callback = ioat_dma_test_callback;
0946     tx->callback_param = &cmp;
0947     cookie = tx->tx_submit(tx);
0948     if (cookie < 0) {
0949         dev_err(dev, "Self-test zero setup failed\n");
0950         err = -ENODEV;
0951         goto dma_unmap;
0952     }
0953     dma->device_issue_pending(dma_chan);
0954 
0955     tmo = wait_for_completion_timeout(&cmp, msecs_to_jiffies(3000));
0956 
0957     if (tmo == 0 ||
0958         dma->device_tx_status(dma_chan, cookie, NULL) != DMA_COMPLETE) {
0959         dev_err(dev, "Self-test validate timed out\n");
0960         err = -ENODEV;
0961         goto dma_unmap;
0962     }
0963 
0964     for (i = 0; i < IOAT_NUM_SRC_TEST + 1; i++)
0965         dma_unmap_page(dev, dma_srcs[i], PAGE_SIZE, DMA_TO_DEVICE);
0966 
0967     if (xor_val_result != 0) {
0968         dev_err(dev, "Self-test validate failed compare\n");
0969         err = -ENODEV;
0970         goto free_resources;
0971     }
0972 
0973     memset(page_address(dest), 0, PAGE_SIZE);
0974 
0975     /* test for non-zero parity sum */
0976     op = IOAT_OP_XOR_VAL;
0977 
0978     xor_val_result = 0;
0979     for (i = 0; i < IOAT_NUM_SRC_TEST + 1; i++) {
0980         dma_srcs[i] = dma_map_page(dev, xor_val_srcs[i], 0, PAGE_SIZE,
0981                        DMA_TO_DEVICE);
0982         if (dma_mapping_error(dev, dma_srcs[i])) {
0983             err = -ENOMEM;
0984             goto dma_unmap;
0985         }
0986     }
0987     tx = dma->device_prep_dma_xor_val(dma_chan, dma_srcs,
0988                       IOAT_NUM_SRC_TEST + 1, PAGE_SIZE,
0989                       &xor_val_result, DMA_PREP_INTERRUPT);
0990     if (!tx) {
0991         dev_err(dev, "Self-test 2nd zero prep failed\n");
0992         err = -ENODEV;
0993         goto dma_unmap;
0994     }
0995 
0996     async_tx_ack(tx);
0997     init_completion(&cmp);
0998     tx->callback = ioat_dma_test_callback;
0999     tx->callback_param = &cmp;
1000     cookie = tx->tx_submit(tx);
1001     if (cookie < 0) {
1002         dev_err(dev, "Self-test  2nd zero setup failed\n");
1003         err = -ENODEV;
1004         goto dma_unmap;
1005     }
1006     dma->device_issue_pending(dma_chan);
1007 
1008     tmo = wait_for_completion_timeout(&cmp, msecs_to_jiffies(3000));
1009 
1010     if (tmo == 0 ||
1011         dma->device_tx_status(dma_chan, cookie, NULL) != DMA_COMPLETE) {
1012         dev_err(dev, "Self-test 2nd validate timed out\n");
1013         err = -ENODEV;
1014         goto dma_unmap;
1015     }
1016 
1017     if (xor_val_result != SUM_CHECK_P_RESULT) {
1018         dev_err(dev, "Self-test validate failed compare\n");
1019         err = -ENODEV;
1020         goto dma_unmap;
1021     }
1022 
1023     for (i = 0; i < IOAT_NUM_SRC_TEST + 1; i++)
1024         dma_unmap_page(dev, dma_srcs[i], PAGE_SIZE, DMA_TO_DEVICE);
1025 
1026     goto free_resources;
1027 dma_unmap:
1028     if (op == IOAT_OP_XOR) {
1029         while (--i >= 0)
1030             dma_unmap_page(dev, dma_srcs[i], PAGE_SIZE,
1031                        DMA_TO_DEVICE);
1032         dma_unmap_page(dev, dest_dma, PAGE_SIZE, DMA_FROM_DEVICE);
1033     } else if (op == IOAT_OP_XOR_VAL) {
1034         while (--i >= 0)
1035             dma_unmap_page(dev, dma_srcs[i], PAGE_SIZE,
1036                        DMA_TO_DEVICE);
1037     }
1038 free_resources:
1039     dma->device_free_chan_resources(dma_chan);
1040 out:
1041     src_idx = IOAT_NUM_SRC_TEST;
1042     while (src_idx--)
1043         __free_page(xor_srcs[src_idx]);
1044     __free_page(dest);
1045     return err;
1046 }
1047 
1048 static int ioat3_dma_self_test(struct ioatdma_device *ioat_dma)
1049 {
1050     int rc;
1051 
1052     rc = ioat_dma_self_test(ioat_dma);
1053     if (rc)
1054         return rc;
1055 
1056     rc = ioat_xor_val_self_test(ioat_dma);
1057 
1058     return rc;
1059 }
1060 
1061 static void ioat_intr_quirk(struct ioatdma_device *ioat_dma)
1062 {
1063     struct dma_device *dma;
1064     struct dma_chan *c;
1065     struct ioatdma_chan *ioat_chan;
1066     u32 errmask;
1067 
1068     dma = &ioat_dma->dma_dev;
1069 
1070     /*
1071      * if we have descriptor write back error status, we mask the
1072      * error interrupts
1073      */
1074     if (ioat_dma->cap & IOAT_CAP_DWBES) {
1075         list_for_each_entry(c, &dma->channels, device_node) {
1076             ioat_chan = to_ioat_chan(c);
1077             errmask = readl(ioat_chan->reg_base +
1078                     IOAT_CHANERR_MASK_OFFSET);
1079             errmask |= IOAT_CHANERR_XOR_P_OR_CRC_ERR |
1080                    IOAT_CHANERR_XOR_Q_ERR;
1081             writel(errmask, ioat_chan->reg_base +
1082                     IOAT_CHANERR_MASK_OFFSET);
1083         }
1084     }
1085 }
1086 
1087 static int ioat3_dma_probe(struct ioatdma_device *ioat_dma, int dca)
1088 {
1089     struct pci_dev *pdev = ioat_dma->pdev;
1090     int dca_en = system_has_dca_enabled(pdev);
1091     struct dma_device *dma;
1092     struct dma_chan *c;
1093     struct ioatdma_chan *ioat_chan;
1094     int err;
1095     u16 val16;
1096 
1097     dma = &ioat_dma->dma_dev;
1098     dma->device_prep_dma_memcpy = ioat_dma_prep_memcpy_lock;
1099     dma->device_issue_pending = ioat_issue_pending;
1100     dma->device_alloc_chan_resources = ioat_alloc_chan_resources;
1101     dma->device_free_chan_resources = ioat_free_chan_resources;
1102 
1103     dma_cap_set(DMA_INTERRUPT, dma->cap_mask);
1104     dma->device_prep_dma_interrupt = ioat_prep_interrupt_lock;
1105 
1106     ioat_dma->cap = readl(ioat_dma->reg_base + IOAT_DMA_CAP_OFFSET);
1107 
1108     if (is_xeon_cb32(pdev) || is_bwd_noraid(pdev))
1109         ioat_dma->cap &=
1110             ~(IOAT_CAP_XOR | IOAT_CAP_PQ | IOAT_CAP_RAID16SS);
1111 
1112     /* dca is incompatible with raid operations */
1113     if (dca_en && (ioat_dma->cap & (IOAT_CAP_XOR|IOAT_CAP_PQ)))
1114         ioat_dma->cap &= ~(IOAT_CAP_XOR|IOAT_CAP_PQ);
1115 
1116     if (ioat_dma->cap & IOAT_CAP_XOR) {
1117         dma->max_xor = 8;
1118 
1119         dma_cap_set(DMA_XOR, dma->cap_mask);
1120         dma->device_prep_dma_xor = ioat_prep_xor;
1121 
1122         dma_cap_set(DMA_XOR_VAL, dma->cap_mask);
1123         dma->device_prep_dma_xor_val = ioat_prep_xor_val;
1124     }
1125 
1126     if (ioat_dma->cap & IOAT_CAP_PQ) {
1127 
1128         dma->device_prep_dma_pq = ioat_prep_pq;
1129         dma->device_prep_dma_pq_val = ioat_prep_pq_val;
1130         dma_cap_set(DMA_PQ, dma->cap_mask);
1131         dma_cap_set(DMA_PQ_VAL, dma->cap_mask);
1132 
1133         if (ioat_dma->cap & IOAT_CAP_RAID16SS)
1134             dma_set_maxpq(dma, 16, 0);
1135         else
1136             dma_set_maxpq(dma, 8, 0);
1137 
1138         if (!(ioat_dma->cap & IOAT_CAP_XOR)) {
1139             dma->device_prep_dma_xor = ioat_prep_pqxor;
1140             dma->device_prep_dma_xor_val = ioat_prep_pqxor_val;
1141             dma_cap_set(DMA_XOR, dma->cap_mask);
1142             dma_cap_set(DMA_XOR_VAL, dma->cap_mask);
1143 
1144             if (ioat_dma->cap & IOAT_CAP_RAID16SS)
1145                 dma->max_xor = 16;
1146             else
1147                 dma->max_xor = 8;
1148         }
1149     }
1150 
1151     dma->device_tx_status = ioat_tx_status;
1152 
1153     /* starting with CB3.3 super extended descriptors are supported */
1154     if (ioat_dma->cap & IOAT_CAP_RAID16SS) {
1155         char pool_name[14];
1156         int i;
1157 
1158         for (i = 0; i < MAX_SED_POOLS; i++) {
1159             snprintf(pool_name, 14, "ioat_hw%d_sed", i);
1160 
1161             /* allocate SED DMA pool */
1162             ioat_dma->sed_hw_pool[i] = dmam_pool_create(pool_name,
1163                     &pdev->dev,
1164                     SED_SIZE * (i + 1), 64, 0);
1165             if (!ioat_dma->sed_hw_pool[i])
1166                 return -ENOMEM;
1167 
1168         }
1169     }
1170 
1171     if (!(ioat_dma->cap & (IOAT_CAP_XOR | IOAT_CAP_PQ)))
1172         dma_cap_set(DMA_PRIVATE, dma->cap_mask);
1173 
1174     err = ioat_probe(ioat_dma);
1175     if (err)
1176         return err;
1177 
1178     list_for_each_entry(c, &dma->channels, device_node) {
1179         ioat_chan = to_ioat_chan(c);
1180         writel(IOAT_DMA_DCA_ANY_CPU,
1181                ioat_chan->reg_base + IOAT_DCACTRL_OFFSET);
1182     }
1183 
1184     err = ioat_register(ioat_dma);
1185     if (err)
1186         return err;
1187 
1188     ioat_kobject_add(ioat_dma, &ioat_ktype);
1189 
1190     if (dca)
1191         ioat_dma->dca = ioat_dca_init(pdev, ioat_dma->reg_base);
1192 
1193     /* disable relaxed ordering */
1194     err = pcie_capability_read_word(pdev, IOAT_DEVCTRL_OFFSET, &val16);
1195     if (err)
1196         return pcibios_err_to_errno(err);
1197 
1198     /* clear relaxed ordering enable */
1199     val16 &= ~IOAT_DEVCTRL_ROE;
1200     err = pcie_capability_write_word(pdev, IOAT_DEVCTRL_OFFSET, val16);
1201     if (err)
1202         return pcibios_err_to_errno(err);
1203 
1204     if (ioat_dma->cap & IOAT_CAP_DPS)
1205         writeb(ioat_pending_level + 1,
1206                ioat_dma->reg_base + IOAT_PREFETCH_LIMIT_OFFSET);
1207 
1208     return 0;
1209 }
1210 
1211 static void ioat_shutdown(struct pci_dev *pdev)
1212 {
1213     struct ioatdma_device *ioat_dma = pci_get_drvdata(pdev);
1214     struct ioatdma_chan *ioat_chan;
1215     int i;
1216 
1217     if (!ioat_dma)
1218         return;
1219 
1220     for (i = 0; i < IOAT_MAX_CHANS; i++) {
1221         ioat_chan = ioat_dma->idx[i];
1222         if (!ioat_chan)
1223             continue;
1224 
1225         spin_lock_bh(&ioat_chan->prep_lock);
1226         set_bit(IOAT_CHAN_DOWN, &ioat_chan->state);
1227         spin_unlock_bh(&ioat_chan->prep_lock);
1228         /*
1229          * Synchronization rule for del_timer_sync():
1230          *  - The caller must not hold locks which would prevent
1231          *    completion of the timer's handler.
1232          * So prep_lock cannot be held before calling it.
1233          */
1234         del_timer_sync(&ioat_chan->timer);
1235 
1236         /* this should quiesce then reset */
1237         ioat_reset_hw(ioat_chan);
1238     }
1239 
1240     ioat_disable_interrupts(ioat_dma);
1241 }
1242 
1243 static void ioat_resume(struct ioatdma_device *ioat_dma)
1244 {
1245     struct ioatdma_chan *ioat_chan;
1246     u32 chanerr;
1247     int i;
1248 
1249     for (i = 0; i < IOAT_MAX_CHANS; i++) {
1250         ioat_chan = ioat_dma->idx[i];
1251         if (!ioat_chan)
1252             continue;
1253 
1254         spin_lock_bh(&ioat_chan->prep_lock);
1255         clear_bit(IOAT_CHAN_DOWN, &ioat_chan->state);
1256         spin_unlock_bh(&ioat_chan->prep_lock);
1257 
1258         chanerr = readl(ioat_chan->reg_base + IOAT_CHANERR_OFFSET);
1259         writel(chanerr, ioat_chan->reg_base + IOAT_CHANERR_OFFSET);
1260 
1261         /* no need to reset as shutdown already did that */
1262     }
1263 }
1264 
1265 #define DRV_NAME "ioatdma"
1266 
1267 static pci_ers_result_t ioat_pcie_error_detected(struct pci_dev *pdev,
1268                          pci_channel_state_t error)
1269 {
1270     dev_dbg(&pdev->dev, "%s: PCIe AER error %d\n", DRV_NAME, error);
1271 
1272     /* quiesce and block I/O */
1273     ioat_shutdown(pdev);
1274 
1275     return PCI_ERS_RESULT_NEED_RESET;
1276 }
1277 
1278 static pci_ers_result_t ioat_pcie_error_slot_reset(struct pci_dev *pdev)
1279 {
1280     pci_ers_result_t result = PCI_ERS_RESULT_RECOVERED;
1281 
1282     dev_dbg(&pdev->dev, "%s post reset handling\n", DRV_NAME);
1283 
1284     if (pci_enable_device_mem(pdev) < 0) {
1285         dev_err(&pdev->dev,
1286             "Failed to enable PCIe device after reset.\n");
1287         result = PCI_ERS_RESULT_DISCONNECT;
1288     } else {
1289         pci_set_master(pdev);
1290         pci_restore_state(pdev);
1291         pci_save_state(pdev);
1292         pci_wake_from_d3(pdev, false);
1293     }
1294 
1295     return result;
1296 }
1297 
1298 static void ioat_pcie_error_resume(struct pci_dev *pdev)
1299 {
1300     struct ioatdma_device *ioat_dma = pci_get_drvdata(pdev);
1301 
1302     dev_dbg(&pdev->dev, "%s: AER handling resuming\n", DRV_NAME);
1303 
1304     /* initialize and bring everything back */
1305     ioat_resume(ioat_dma);
1306 }
1307 
1308 static const struct pci_error_handlers ioat_err_handler = {
1309     .error_detected = ioat_pcie_error_detected,
1310     .slot_reset = ioat_pcie_error_slot_reset,
1311     .resume = ioat_pcie_error_resume,
1312 };
1313 
1314 static struct pci_driver ioat_pci_driver = {
1315     .name       = DRV_NAME,
1316     .id_table   = ioat_pci_tbl,
1317     .probe      = ioat_pci_probe,
1318     .remove     = ioat_remove,
1319     .shutdown   = ioat_shutdown,
1320     .err_handler    = &ioat_err_handler,
1321 };
1322 
1323 static void release_ioatdma(struct dma_device *device)
1324 {
1325     struct ioatdma_device *d = to_ioatdma_device(device);
1326     int i;
1327 
1328     for (i = 0; i < IOAT_MAX_CHANS; i++)
1329         kfree(d->idx[i]);
1330 
1331     dma_pool_destroy(d->completion_pool);
1332     kfree(d);
1333 }
1334 
1335 static struct ioatdma_device *
1336 alloc_ioatdma(struct pci_dev *pdev, void __iomem *iobase)
1337 {
1338     struct ioatdma_device *d = kzalloc(sizeof(*d), GFP_KERNEL);
1339 
1340     if (!d)
1341         return NULL;
1342     d->pdev = pdev;
1343     d->reg_base = iobase;
1344     d->dma_dev.device_release = release_ioatdma;
1345     return d;
1346 }
1347 
1348 static int ioat_pci_probe(struct pci_dev *pdev, const struct pci_device_id *id)
1349 {
1350     void __iomem * const *iomap;
1351     struct device *dev = &pdev->dev;
1352     struct ioatdma_device *device;
1353     int err;
1354 
1355     err = pcim_enable_device(pdev);
1356     if (err)
1357         return err;
1358 
1359     err = pcim_iomap_regions(pdev, 1 << IOAT_MMIO_BAR, DRV_NAME);
1360     if (err)
1361         return err;
1362     iomap = pcim_iomap_table(pdev);
1363     if (!iomap)
1364         return -ENOMEM;
1365 
1366     err = dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(64));
1367     if (err)
1368         return err;
1369 
1370     device = alloc_ioatdma(pdev, iomap[IOAT_MMIO_BAR]);
1371     if (!device)
1372         return -ENOMEM;
1373     pci_set_master(pdev);
1374     pci_set_drvdata(pdev, device);
1375 
1376     device->version = readb(device->reg_base + IOAT_VER_OFFSET);
1377     if (device->version >= IOAT_VER_3_4)
1378         ioat_dca_enabled = 0;
1379     if (device->version >= IOAT_VER_3_0) {
1380         if (is_skx_ioat(pdev))
1381             device->version = IOAT_VER_3_2;
1382         err = ioat3_dma_probe(device, ioat_dca_enabled);
1383 
1384         if (device->version >= IOAT_VER_3_3)
1385             pci_enable_pcie_error_reporting(pdev);
1386     } else
1387         return -ENODEV;
1388 
1389     if (err) {
1390         dev_err(dev, "Intel(R) I/OAT DMA Engine init failed\n");
1391         pci_disable_pcie_error_reporting(pdev);
1392         return -ENODEV;
1393     }
1394 
1395     return 0;
1396 }
1397 
1398 static void ioat_remove(struct pci_dev *pdev)
1399 {
1400     struct ioatdma_device *device = pci_get_drvdata(pdev);
1401 
1402     if (!device)
1403         return;
1404 
1405     ioat_shutdown(pdev);
1406 
1407     dev_err(&pdev->dev, "Removing dma and dca services\n");
1408     if (device->dca) {
1409         unregister_dca_provider(device->dca, &pdev->dev);
1410         free_dca_provider(device->dca);
1411         device->dca = NULL;
1412     }
1413 
1414     pci_disable_pcie_error_reporting(pdev);
1415     ioat_dma_remove(device);
1416 }
1417 
1418 static int __init ioat_init_module(void)
1419 {
1420     int err = -ENOMEM;
1421 
1422     pr_info("%s: Intel(R) QuickData Technology Driver %s\n",
1423         DRV_NAME, IOAT_DMA_VERSION);
1424 
1425     ioat_cache = kmem_cache_create("ioat", sizeof(struct ioat_ring_ent),
1426                     0, SLAB_HWCACHE_ALIGN, NULL);
1427     if (!ioat_cache)
1428         return -ENOMEM;
1429 
1430     ioat_sed_cache = KMEM_CACHE(ioat_sed_ent, 0);
1431     if (!ioat_sed_cache)
1432         goto err_ioat_cache;
1433 
1434     err = pci_register_driver(&ioat_pci_driver);
1435     if (err)
1436         goto err_ioat3_cache;
1437 
1438     return 0;
1439 
1440  err_ioat3_cache:
1441     kmem_cache_destroy(ioat_sed_cache);
1442 
1443  err_ioat_cache:
1444     kmem_cache_destroy(ioat_cache);
1445 
1446     return err;
1447 }
1448 module_init(ioat_init_module);
1449 
1450 static void __exit ioat_exit_module(void)
1451 {
1452     pci_unregister_driver(&ioat_pci_driver);
1453     kmem_cache_destroy(ioat_cache);
1454 }
1455 module_exit(ioat_exit_module);
[ Source navigation ] [ Diff markup ] [ Identifier search ] [ general search ]

This page was automatically generated by the 2.1.0 LXR engine.

The LXR team
Valid CSS 2.1! Valid HTML 4.01!