Back to home page

OSCL-LXR
 
 

     

0001 // SPDX-License-Identifier: GPL-2.0
0002 /* Copyright(c) 2019 Intel Corporation. All rights rsvd. */
0003 #include <linux/init.h>
0004 #include <linux/kernel.h>
0005 #include <linux/module.h>
0006 #include <linux/slab.h>
0007 #include <linux/pci.h>
0008 #include <linux/interrupt.h>
0009 #include <linux/delay.h>
0010 #include <linux/dma-mapping.h>
0011 #include <linux/workqueue.h>
0012 #include <linux/aer.h>
0013 #include <linux/fs.h>
0014 #include <linux/io-64-nonatomic-lo-hi.h>
0015 #include <linux/device.h>
0016 #include <linux/idr.h>
0017 #include <linux/intel-svm.h>
0018 #include <linux/iommu.h>
0019 #include <uapi/linux/idxd.h>
0020 #include <linux/dmaengine.h>
0021 #include "../dmaengine.h"
0022 #include "registers.h"
0023 #include "idxd.h"
0024 #include "perfmon.h"
0025 
0026 MODULE_VERSION(IDXD_DRIVER_VERSION);
0027 MODULE_LICENSE("GPL v2");
0028 MODULE_AUTHOR("Intel Corporation");
0029 MODULE_IMPORT_NS(IDXD);
0030 
0031 static bool sva = true;
0032 module_param(sva, bool, 0644);
0033 MODULE_PARM_DESC(sva, "Toggle SVA support on/off");
0034 
0035 bool tc_override;
0036 module_param(tc_override, bool, 0644);
0037 MODULE_PARM_DESC(tc_override, "Override traffic class defaults");
0038 
0039 #define DRV_NAME "idxd"
0040 
0041 bool support_enqcmd;
0042 DEFINE_IDA(idxd_ida);
0043 
0044 static struct idxd_driver_data idxd_driver_data[] = {
0045     [IDXD_TYPE_DSA] = {
0046         .name_prefix = "dsa",
0047         .type = IDXD_TYPE_DSA,
0048         .compl_size = sizeof(struct dsa_completion_record),
0049         .align = 32,
0050         .dev_type = &dsa_device_type,
0051     },
0052     [IDXD_TYPE_IAX] = {
0053         .name_prefix = "iax",
0054         .type = IDXD_TYPE_IAX,
0055         .compl_size = sizeof(struct iax_completion_record),
0056         .align = 64,
0057         .dev_type = &iax_device_type,
0058     },
0059 };
0060 
0061 static struct pci_device_id idxd_pci_tbl[] = {
0062     /* DSA ver 1.0 platforms */
0063     { PCI_DEVICE_DATA(INTEL, DSA_SPR0, &idxd_driver_data[IDXD_TYPE_DSA]) },
0064 
0065     /* IAX ver 1.0 platforms */
0066     { PCI_DEVICE_DATA(INTEL, IAX_SPR0, &idxd_driver_data[IDXD_TYPE_IAX]) },
0067     { 0, }
0068 };
0069 MODULE_DEVICE_TABLE(pci, idxd_pci_tbl);
0070 
0071 static int idxd_setup_interrupts(struct idxd_device *idxd)
0072 {
0073     struct pci_dev *pdev = idxd->pdev;
0074     struct device *dev = &pdev->dev;
0075     struct idxd_irq_entry *ie;
0076     int i, msixcnt;
0077     int rc = 0;
0078 
0079     msixcnt = pci_msix_vec_count(pdev);
0080     if (msixcnt < 0) {
0081         dev_err(dev, "Not MSI-X interrupt capable.\n");
0082         return -ENOSPC;
0083     }
0084     idxd->irq_cnt = msixcnt;
0085 
0086     rc = pci_alloc_irq_vectors(pdev, msixcnt, msixcnt, PCI_IRQ_MSIX);
0087     if (rc != msixcnt) {
0088         dev_err(dev, "Failed enabling %d MSIX entries: %d\n", msixcnt, rc);
0089         return -ENOSPC;
0090     }
0091     dev_dbg(dev, "Enabled %d msix vectors\n", msixcnt);
0092 
0093 
0094     ie = idxd_get_ie(idxd, 0);
0095     ie->vector = pci_irq_vector(pdev, 0);
0096     rc = request_threaded_irq(ie->vector, NULL, idxd_misc_thread, 0, "idxd-misc", ie);
0097     if (rc < 0) {
0098         dev_err(dev, "Failed to allocate misc interrupt.\n");
0099         goto err_misc_irq;
0100     }
0101     dev_dbg(dev, "Requested idxd-misc handler on msix vector %d\n", ie->vector);
0102 
0103     for (i = 0; i < idxd->max_wqs; i++) {
0104         int msix_idx = i + 1;
0105 
0106         ie = idxd_get_ie(idxd, msix_idx);
0107         ie->id = msix_idx;
0108         ie->int_handle = INVALID_INT_HANDLE;
0109         ie->pasid = INVALID_IOASID;
0110 
0111         spin_lock_init(&ie->list_lock);
0112         init_llist_head(&ie->pending_llist);
0113         INIT_LIST_HEAD(&ie->work_list);
0114     }
0115 
0116     idxd_unmask_error_interrupts(idxd);
0117     return 0;
0118 
0119  err_misc_irq:
0120     idxd_mask_error_interrupts(idxd);
0121     pci_free_irq_vectors(pdev);
0122     dev_err(dev, "No usable interrupts\n");
0123     return rc;
0124 }
0125 
0126 static void idxd_cleanup_interrupts(struct idxd_device *idxd)
0127 {
0128     struct pci_dev *pdev = idxd->pdev;
0129     struct idxd_irq_entry *ie;
0130     int msixcnt;
0131 
0132     msixcnt = pci_msix_vec_count(pdev);
0133     if (msixcnt <= 0)
0134         return;
0135 
0136     ie = idxd_get_ie(idxd, 0);
0137     idxd_mask_error_interrupts(idxd);
0138     free_irq(ie->vector, ie);
0139     pci_free_irq_vectors(pdev);
0140 }
0141 
0142 static int idxd_setup_wqs(struct idxd_device *idxd)
0143 {
0144     struct device *dev = &idxd->pdev->dev;
0145     struct idxd_wq *wq;
0146     struct device *conf_dev;
0147     int i, rc;
0148 
0149     idxd->wqs = kcalloc_node(idxd->max_wqs, sizeof(struct idxd_wq *),
0150                  GFP_KERNEL, dev_to_node(dev));
0151     if (!idxd->wqs)
0152         return -ENOMEM;
0153 
0154     for (i = 0; i < idxd->max_wqs; i++) {
0155         wq = kzalloc_node(sizeof(*wq), GFP_KERNEL, dev_to_node(dev));
0156         if (!wq) {
0157             rc = -ENOMEM;
0158             goto err;
0159         }
0160 
0161         idxd_dev_set_type(&wq->idxd_dev, IDXD_DEV_WQ);
0162         conf_dev = wq_confdev(wq);
0163         wq->id = i;
0164         wq->idxd = idxd;
0165         device_initialize(wq_confdev(wq));
0166         conf_dev->parent = idxd_confdev(idxd);
0167         conf_dev->bus = &dsa_bus_type;
0168         conf_dev->type = &idxd_wq_device_type;
0169         rc = dev_set_name(conf_dev, "wq%d.%d", idxd->id, wq->id);
0170         if (rc < 0) {
0171             put_device(conf_dev);
0172             goto err;
0173         }
0174 
0175         mutex_init(&wq->wq_lock);
0176         init_waitqueue_head(&wq->err_queue);
0177         init_completion(&wq->wq_dead);
0178         init_completion(&wq->wq_resurrect);
0179         wq->max_xfer_bytes = WQ_DEFAULT_MAX_XFER;
0180         wq->max_batch_size = WQ_DEFAULT_MAX_BATCH;
0181         wq->enqcmds_retries = IDXD_ENQCMDS_RETRIES;
0182         wq->wqcfg = kzalloc_node(idxd->wqcfg_size, GFP_KERNEL, dev_to_node(dev));
0183         if (!wq->wqcfg) {
0184             put_device(conf_dev);
0185             rc = -ENOMEM;
0186             goto err;
0187         }
0188         idxd->wqs[i] = wq;
0189     }
0190 
0191     return 0;
0192 
0193  err:
0194     while (--i >= 0) {
0195         wq = idxd->wqs[i];
0196         conf_dev = wq_confdev(wq);
0197         put_device(conf_dev);
0198     }
0199     return rc;
0200 }
0201 
0202 static int idxd_setup_engines(struct idxd_device *idxd)
0203 {
0204     struct idxd_engine *engine;
0205     struct device *dev = &idxd->pdev->dev;
0206     struct device *conf_dev;
0207     int i, rc;
0208 
0209     idxd->engines = kcalloc_node(idxd->max_engines, sizeof(struct idxd_engine *),
0210                      GFP_KERNEL, dev_to_node(dev));
0211     if (!idxd->engines)
0212         return -ENOMEM;
0213 
0214     for (i = 0; i < idxd->max_engines; i++) {
0215         engine = kzalloc_node(sizeof(*engine), GFP_KERNEL, dev_to_node(dev));
0216         if (!engine) {
0217             rc = -ENOMEM;
0218             goto err;
0219         }
0220 
0221         idxd_dev_set_type(&engine->idxd_dev, IDXD_DEV_ENGINE);
0222         conf_dev = engine_confdev(engine);
0223         engine->id = i;
0224         engine->idxd = idxd;
0225         device_initialize(conf_dev);
0226         conf_dev->parent = idxd_confdev(idxd);
0227         conf_dev->bus = &dsa_bus_type;
0228         conf_dev->type = &idxd_engine_device_type;
0229         rc = dev_set_name(conf_dev, "engine%d.%d", idxd->id, engine->id);
0230         if (rc < 0) {
0231             put_device(conf_dev);
0232             goto err;
0233         }
0234 
0235         idxd->engines[i] = engine;
0236     }
0237 
0238     return 0;
0239 
0240  err:
0241     while (--i >= 0) {
0242         engine = idxd->engines[i];
0243         conf_dev = engine_confdev(engine);
0244         put_device(conf_dev);
0245     }
0246     return rc;
0247 }
0248 
0249 static int idxd_setup_groups(struct idxd_device *idxd)
0250 {
0251     struct device *dev = &idxd->pdev->dev;
0252     struct device *conf_dev;
0253     struct idxd_group *group;
0254     int i, rc;
0255 
0256     idxd->groups = kcalloc_node(idxd->max_groups, sizeof(struct idxd_group *),
0257                     GFP_KERNEL, dev_to_node(dev));
0258     if (!idxd->groups)
0259         return -ENOMEM;
0260 
0261     for (i = 0; i < idxd->max_groups; i++) {
0262         group = kzalloc_node(sizeof(*group), GFP_KERNEL, dev_to_node(dev));
0263         if (!group) {
0264             rc = -ENOMEM;
0265             goto err;
0266         }
0267 
0268         idxd_dev_set_type(&group->idxd_dev, IDXD_DEV_GROUP);
0269         conf_dev = group_confdev(group);
0270         group->id = i;
0271         group->idxd = idxd;
0272         device_initialize(conf_dev);
0273         conf_dev->parent = idxd_confdev(idxd);
0274         conf_dev->bus = &dsa_bus_type;
0275         conf_dev->type = &idxd_group_device_type;
0276         rc = dev_set_name(conf_dev, "group%d.%d", idxd->id, group->id);
0277         if (rc < 0) {
0278             put_device(conf_dev);
0279             goto err;
0280         }
0281 
0282         idxd->groups[i] = group;
0283         if (idxd->hw.version < DEVICE_VERSION_2 && !tc_override) {
0284             group->tc_a = 1;
0285             group->tc_b = 1;
0286         } else {
0287             group->tc_a = -1;
0288             group->tc_b = -1;
0289         }
0290     }
0291 
0292     return 0;
0293 
0294  err:
0295     while (--i >= 0) {
0296         group = idxd->groups[i];
0297         put_device(group_confdev(group));
0298     }
0299     return rc;
0300 }
0301 
0302 static void idxd_cleanup_internals(struct idxd_device *idxd)
0303 {
0304     int i;
0305 
0306     for (i = 0; i < idxd->max_groups; i++)
0307         put_device(group_confdev(idxd->groups[i]));
0308     for (i = 0; i < idxd->max_engines; i++)
0309         put_device(engine_confdev(idxd->engines[i]));
0310     for (i = 0; i < idxd->max_wqs; i++)
0311         put_device(wq_confdev(idxd->wqs[i]));
0312     destroy_workqueue(idxd->wq);
0313 }
0314 
0315 static int idxd_setup_internals(struct idxd_device *idxd)
0316 {
0317     struct device *dev = &idxd->pdev->dev;
0318     int rc, i;
0319 
0320     init_waitqueue_head(&idxd->cmd_waitq);
0321 
0322     rc = idxd_setup_wqs(idxd);
0323     if (rc < 0)
0324         goto err_wqs;
0325 
0326     rc = idxd_setup_engines(idxd);
0327     if (rc < 0)
0328         goto err_engine;
0329 
0330     rc = idxd_setup_groups(idxd);
0331     if (rc < 0)
0332         goto err_group;
0333 
0334     idxd->wq = create_workqueue(dev_name(dev));
0335     if (!idxd->wq) {
0336         rc = -ENOMEM;
0337         goto err_wkq_create;
0338     }
0339 
0340     return 0;
0341 
0342  err_wkq_create:
0343     for (i = 0; i < idxd->max_groups; i++)
0344         put_device(group_confdev(idxd->groups[i]));
0345  err_group:
0346     for (i = 0; i < idxd->max_engines; i++)
0347         put_device(engine_confdev(idxd->engines[i]));
0348  err_engine:
0349     for (i = 0; i < idxd->max_wqs; i++)
0350         put_device(wq_confdev(idxd->wqs[i]));
0351  err_wqs:
0352     return rc;
0353 }
0354 
0355 static void idxd_read_table_offsets(struct idxd_device *idxd)
0356 {
0357     union offsets_reg offsets;
0358     struct device *dev = &idxd->pdev->dev;
0359 
0360     offsets.bits[0] = ioread64(idxd->reg_base + IDXD_TABLE_OFFSET);
0361     offsets.bits[1] = ioread64(idxd->reg_base + IDXD_TABLE_OFFSET + sizeof(u64));
0362     idxd->grpcfg_offset = offsets.grpcfg * IDXD_TABLE_MULT;
0363     dev_dbg(dev, "IDXD Group Config Offset: %#x\n", idxd->grpcfg_offset);
0364     idxd->wqcfg_offset = offsets.wqcfg * IDXD_TABLE_MULT;
0365     dev_dbg(dev, "IDXD Work Queue Config Offset: %#x\n", idxd->wqcfg_offset);
0366     idxd->msix_perm_offset = offsets.msix_perm * IDXD_TABLE_MULT;
0367     dev_dbg(dev, "IDXD MSIX Permission Offset: %#x\n", idxd->msix_perm_offset);
0368     idxd->perfmon_offset = offsets.perfmon * IDXD_TABLE_MULT;
0369     dev_dbg(dev, "IDXD Perfmon Offset: %#x\n", idxd->perfmon_offset);
0370 }
0371 
0372 static void idxd_read_caps(struct idxd_device *idxd)
0373 {
0374     struct device *dev = &idxd->pdev->dev;
0375     int i;
0376 
0377     /* reading generic capabilities */
0378     idxd->hw.gen_cap.bits = ioread64(idxd->reg_base + IDXD_GENCAP_OFFSET);
0379     dev_dbg(dev, "gen_cap: %#llx\n", idxd->hw.gen_cap.bits);
0380 
0381     if (idxd->hw.gen_cap.cmd_cap) {
0382         idxd->hw.cmd_cap = ioread32(idxd->reg_base + IDXD_CMDCAP_OFFSET);
0383         dev_dbg(dev, "cmd_cap: %#x\n", idxd->hw.cmd_cap);
0384     }
0385 
0386     /* reading command capabilities */
0387     if (idxd->hw.cmd_cap & BIT(IDXD_CMD_REQUEST_INT_HANDLE))
0388         idxd->request_int_handles = true;
0389 
0390     idxd->max_xfer_bytes = 1ULL << idxd->hw.gen_cap.max_xfer_shift;
0391     dev_dbg(dev, "max xfer size: %llu bytes\n", idxd->max_xfer_bytes);
0392     idxd->max_batch_size = 1U << idxd->hw.gen_cap.max_batch_shift;
0393     dev_dbg(dev, "max batch size: %u\n", idxd->max_batch_size);
0394     if (idxd->hw.gen_cap.config_en)
0395         set_bit(IDXD_FLAG_CONFIGURABLE, &idxd->flags);
0396 
0397     /* reading group capabilities */
0398     idxd->hw.group_cap.bits =
0399         ioread64(idxd->reg_base + IDXD_GRPCAP_OFFSET);
0400     dev_dbg(dev, "group_cap: %#llx\n", idxd->hw.group_cap.bits);
0401     idxd->max_groups = idxd->hw.group_cap.num_groups;
0402     dev_dbg(dev, "max groups: %u\n", idxd->max_groups);
0403     idxd->max_rdbufs = idxd->hw.group_cap.total_rdbufs;
0404     dev_dbg(dev, "max read buffers: %u\n", idxd->max_rdbufs);
0405     idxd->nr_rdbufs = idxd->max_rdbufs;
0406 
0407     /* read engine capabilities */
0408     idxd->hw.engine_cap.bits =
0409         ioread64(idxd->reg_base + IDXD_ENGCAP_OFFSET);
0410     dev_dbg(dev, "engine_cap: %#llx\n", idxd->hw.engine_cap.bits);
0411     idxd->max_engines = idxd->hw.engine_cap.num_engines;
0412     dev_dbg(dev, "max engines: %u\n", idxd->max_engines);
0413 
0414     /* read workqueue capabilities */
0415     idxd->hw.wq_cap.bits = ioread64(idxd->reg_base + IDXD_WQCAP_OFFSET);
0416     dev_dbg(dev, "wq_cap: %#llx\n", idxd->hw.wq_cap.bits);
0417     idxd->max_wq_size = idxd->hw.wq_cap.total_wq_size;
0418     dev_dbg(dev, "total workqueue size: %u\n", idxd->max_wq_size);
0419     idxd->max_wqs = idxd->hw.wq_cap.num_wqs;
0420     dev_dbg(dev, "max workqueues: %u\n", idxd->max_wqs);
0421     idxd->wqcfg_size = 1 << (idxd->hw.wq_cap.wqcfg_size + IDXD_WQCFG_MIN);
0422     dev_dbg(dev, "wqcfg size: %u\n", idxd->wqcfg_size);
0423 
0424     /* reading operation capabilities */
0425     for (i = 0; i < 4; i++) {
0426         idxd->hw.opcap.bits[i] = ioread64(idxd->reg_base +
0427                 IDXD_OPCAP_OFFSET + i * sizeof(u64));
0428         dev_dbg(dev, "opcap[%d]: %#llx\n", i, idxd->hw.opcap.bits[i]);
0429     }
0430 }
0431 
0432 static struct idxd_device *idxd_alloc(struct pci_dev *pdev, struct idxd_driver_data *data)
0433 {
0434     struct device *dev = &pdev->dev;
0435     struct device *conf_dev;
0436     struct idxd_device *idxd;
0437     int rc;
0438 
0439     idxd = kzalloc_node(sizeof(*idxd), GFP_KERNEL, dev_to_node(dev));
0440     if (!idxd)
0441         return NULL;
0442 
0443     conf_dev = idxd_confdev(idxd);
0444     idxd->pdev = pdev;
0445     idxd->data = data;
0446     idxd_dev_set_type(&idxd->idxd_dev, idxd->data->type);
0447     idxd->id = ida_alloc(&idxd_ida, GFP_KERNEL);
0448     if (idxd->id < 0)
0449         return NULL;
0450 
0451     device_initialize(conf_dev);
0452     conf_dev->parent = dev;
0453     conf_dev->bus = &dsa_bus_type;
0454     conf_dev->type = idxd->data->dev_type;
0455     rc = dev_set_name(conf_dev, "%s%d", idxd->data->name_prefix, idxd->id);
0456     if (rc < 0) {
0457         put_device(conf_dev);
0458         return NULL;
0459     }
0460 
0461     spin_lock_init(&idxd->dev_lock);
0462     spin_lock_init(&idxd->cmd_lock);
0463 
0464     return idxd;
0465 }
0466 
0467 static int idxd_enable_system_pasid(struct idxd_device *idxd)
0468 {
0469     int flags;
0470     unsigned int pasid;
0471     struct iommu_sva *sva;
0472 
0473     flags = SVM_FLAG_SUPERVISOR_MODE;
0474 
0475     sva = iommu_sva_bind_device(&idxd->pdev->dev, NULL, &flags);
0476     if (IS_ERR(sva)) {
0477         dev_warn(&idxd->pdev->dev,
0478              "iommu sva bind failed: %ld\n", PTR_ERR(sva));
0479         return PTR_ERR(sva);
0480     }
0481 
0482     pasid = iommu_sva_get_pasid(sva);
0483     if (pasid == IOMMU_PASID_INVALID) {
0484         iommu_sva_unbind_device(sva);
0485         return -ENODEV;
0486     }
0487 
0488     idxd->sva = sva;
0489     idxd->pasid = pasid;
0490     dev_dbg(&idxd->pdev->dev, "system pasid: %u\n", pasid);
0491     return 0;
0492 }
0493 
0494 static void idxd_disable_system_pasid(struct idxd_device *idxd)
0495 {
0496 
0497     iommu_sva_unbind_device(idxd->sva);
0498     idxd->sva = NULL;
0499 }
0500 
0501 static int idxd_probe(struct idxd_device *idxd)
0502 {
0503     struct pci_dev *pdev = idxd->pdev;
0504     struct device *dev = &pdev->dev;
0505     int rc;
0506 
0507     dev_dbg(dev, "%s entered and resetting device\n", __func__);
0508     rc = idxd_device_init_reset(idxd);
0509     if (rc < 0)
0510         return rc;
0511 
0512     dev_dbg(dev, "IDXD reset complete\n");
0513 
0514     if (IS_ENABLED(CONFIG_INTEL_IDXD_SVM) && sva) {
0515         if (iommu_dev_enable_feature(dev, IOMMU_DEV_FEAT_SVA)) {
0516             dev_warn(dev, "Unable to turn on user SVA feature.\n");
0517         } else {
0518             set_bit(IDXD_FLAG_USER_PASID_ENABLED, &idxd->flags);
0519 
0520             if (idxd_enable_system_pasid(idxd))
0521                 dev_warn(dev, "No in-kernel DMA with PASID.\n");
0522             else
0523                 set_bit(IDXD_FLAG_PASID_ENABLED, &idxd->flags);
0524         }
0525     } else if (!sva) {
0526         dev_warn(dev, "User forced SVA off via module param.\n");
0527     }
0528 
0529     idxd_read_caps(idxd);
0530     idxd_read_table_offsets(idxd);
0531 
0532     rc = idxd_setup_internals(idxd);
0533     if (rc)
0534         goto err;
0535 
0536     /* If the configs are readonly, then load them from device */
0537     if (!test_bit(IDXD_FLAG_CONFIGURABLE, &idxd->flags)) {
0538         dev_dbg(dev, "Loading RO device config\n");
0539         rc = idxd_device_load_config(idxd);
0540         if (rc < 0)
0541             goto err_config;
0542     }
0543 
0544     rc = idxd_setup_interrupts(idxd);
0545     if (rc)
0546         goto err_config;
0547 
0548     idxd->major = idxd_cdev_get_major(idxd);
0549 
0550     rc = perfmon_pmu_init(idxd);
0551     if (rc < 0)
0552         dev_warn(dev, "Failed to initialize perfmon. No PMU support: %d\n", rc);
0553 
0554     dev_dbg(dev, "IDXD device %d probed successfully\n", idxd->id);
0555     return 0;
0556 
0557  err_config:
0558     idxd_cleanup_internals(idxd);
0559  err:
0560     if (device_pasid_enabled(idxd))
0561         idxd_disable_system_pasid(idxd);
0562     if (device_user_pasid_enabled(idxd))
0563         iommu_dev_disable_feature(dev, IOMMU_DEV_FEAT_SVA);
0564     return rc;
0565 }
0566 
0567 static void idxd_cleanup(struct idxd_device *idxd)
0568 {
0569     struct device *dev = &idxd->pdev->dev;
0570 
0571     perfmon_pmu_remove(idxd);
0572     idxd_cleanup_interrupts(idxd);
0573     idxd_cleanup_internals(idxd);
0574     if (device_pasid_enabled(idxd))
0575         idxd_disable_system_pasid(idxd);
0576     if (device_user_pasid_enabled(idxd))
0577         iommu_dev_disable_feature(dev, IOMMU_DEV_FEAT_SVA);
0578 }
0579 
0580 static int idxd_pci_probe(struct pci_dev *pdev, const struct pci_device_id *id)
0581 {
0582     struct device *dev = &pdev->dev;
0583     struct idxd_device *idxd;
0584     struct idxd_driver_data *data = (struct idxd_driver_data *)id->driver_data;
0585     int rc;
0586 
0587     rc = pci_enable_device(pdev);
0588     if (rc)
0589         return rc;
0590 
0591     dev_dbg(dev, "Alloc IDXD context\n");
0592     idxd = idxd_alloc(pdev, data);
0593     if (!idxd) {
0594         rc = -ENOMEM;
0595         goto err_idxd_alloc;
0596     }
0597 
0598     dev_dbg(dev, "Mapping BARs\n");
0599     idxd->reg_base = pci_iomap(pdev, IDXD_MMIO_BAR, 0);
0600     if (!idxd->reg_base) {
0601         rc = -ENOMEM;
0602         goto err_iomap;
0603     }
0604 
0605     dev_dbg(dev, "Set DMA masks\n");
0606     rc = dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(64));
0607     if (rc)
0608         goto err;
0609 
0610     dev_dbg(dev, "Set PCI master\n");
0611     pci_set_master(pdev);
0612     pci_set_drvdata(pdev, idxd);
0613 
0614     idxd->hw.version = ioread32(idxd->reg_base + IDXD_VER_OFFSET);
0615     rc = idxd_probe(idxd);
0616     if (rc) {
0617         dev_err(dev, "Intel(R) IDXD DMA Engine init failed\n");
0618         goto err;
0619     }
0620 
0621     rc = idxd_register_devices(idxd);
0622     if (rc) {
0623         dev_err(dev, "IDXD sysfs setup failed\n");
0624         goto err_dev_register;
0625     }
0626 
0627     dev_info(&pdev->dev, "Intel(R) Accelerator Device (v%x)\n",
0628          idxd->hw.version);
0629 
0630     return 0;
0631 
0632  err_dev_register:
0633     idxd_cleanup(idxd);
0634  err:
0635     pci_iounmap(pdev, idxd->reg_base);
0636  err_iomap:
0637     put_device(idxd_confdev(idxd));
0638  err_idxd_alloc:
0639     pci_disable_device(pdev);
0640     return rc;
0641 }
0642 
0643 void idxd_wqs_quiesce(struct idxd_device *idxd)
0644 {
0645     struct idxd_wq *wq;
0646     int i;
0647 
0648     for (i = 0; i < idxd->max_wqs; i++) {
0649         wq = idxd->wqs[i];
0650         if (wq->state == IDXD_WQ_ENABLED && wq->type == IDXD_WQT_KERNEL)
0651             idxd_wq_quiesce(wq);
0652     }
0653 }
0654 
0655 static void idxd_shutdown(struct pci_dev *pdev)
0656 {
0657     struct idxd_device *idxd = pci_get_drvdata(pdev);
0658     struct idxd_irq_entry *irq_entry;
0659     int rc;
0660 
0661     rc = idxd_device_disable(idxd);
0662     if (rc)
0663         dev_err(&pdev->dev, "Disabling device failed\n");
0664 
0665     irq_entry = &idxd->ie;
0666     synchronize_irq(irq_entry->vector);
0667     idxd_mask_error_interrupts(idxd);
0668     flush_workqueue(idxd->wq);
0669 }
0670 
0671 static void idxd_remove(struct pci_dev *pdev)
0672 {
0673     struct idxd_device *idxd = pci_get_drvdata(pdev);
0674     struct idxd_irq_entry *irq_entry;
0675 
0676     idxd_unregister_devices(idxd);
0677     /*
0678      * When ->release() is called for the idxd->conf_dev, it frees all the memory related
0679      * to the idxd context. The driver still needs those bits in order to do the rest of
0680      * the cleanup. However, we do need to unbound the idxd sub-driver. So take a ref
0681      * on the device here to hold off the freeing while allowing the idxd sub-driver
0682      * to unbind.
0683      */
0684     get_device(idxd_confdev(idxd));
0685     device_unregister(idxd_confdev(idxd));
0686     idxd_shutdown(pdev);
0687     if (device_pasid_enabled(idxd))
0688         idxd_disable_system_pasid(idxd);
0689 
0690     irq_entry = idxd_get_ie(idxd, 0);
0691     free_irq(irq_entry->vector, irq_entry);
0692     pci_free_irq_vectors(pdev);
0693     pci_iounmap(pdev, idxd->reg_base);
0694     if (device_user_pasid_enabled(idxd))
0695         iommu_dev_disable_feature(&pdev->dev, IOMMU_DEV_FEAT_SVA);
0696     pci_disable_device(pdev);
0697     destroy_workqueue(idxd->wq);
0698     perfmon_pmu_remove(idxd);
0699     put_device(idxd_confdev(idxd));
0700 }
0701 
0702 static struct pci_driver idxd_pci_driver = {
0703     .name       = DRV_NAME,
0704     .id_table   = idxd_pci_tbl,
0705     .probe      = idxd_pci_probe,
0706     .remove     = idxd_remove,
0707     .shutdown   = idxd_shutdown,
0708 };
0709 
0710 static int __init idxd_init_module(void)
0711 {
0712     int err;
0713 
0714     /*
0715      * If the CPU does not support MOVDIR64B or ENQCMDS, there's no point in
0716      * enumerating the device. We can not utilize it.
0717      */
0718     if (!cpu_feature_enabled(X86_FEATURE_MOVDIR64B)) {
0719         pr_warn("idxd driver failed to load without MOVDIR64B.\n");
0720         return -ENODEV;
0721     }
0722 
0723     if (!cpu_feature_enabled(X86_FEATURE_ENQCMD))
0724         pr_warn("Platform does not have ENQCMD(S) support.\n");
0725     else
0726         support_enqcmd = true;
0727 
0728     perfmon_init();
0729 
0730     err = idxd_driver_register(&idxd_drv);
0731     if (err < 0)
0732         goto err_idxd_driver_register;
0733 
0734     err = idxd_driver_register(&idxd_dmaengine_drv);
0735     if (err < 0)
0736         goto err_idxd_dmaengine_driver_register;
0737 
0738     err = idxd_driver_register(&idxd_user_drv);
0739     if (err < 0)
0740         goto err_idxd_user_driver_register;
0741 
0742     err = idxd_cdev_register();
0743     if (err)
0744         goto err_cdev_register;
0745 
0746     err = pci_register_driver(&idxd_pci_driver);
0747     if (err)
0748         goto err_pci_register;
0749 
0750     return 0;
0751 
0752 err_pci_register:
0753     idxd_cdev_remove();
0754 err_cdev_register:
0755     idxd_driver_unregister(&idxd_user_drv);
0756 err_idxd_user_driver_register:
0757     idxd_driver_unregister(&idxd_dmaengine_drv);
0758 err_idxd_dmaengine_driver_register:
0759     idxd_driver_unregister(&idxd_drv);
0760 err_idxd_driver_register:
0761     return err;
0762 }
0763 module_init(idxd_init_module);
0764 
0765 static void __exit idxd_exit_module(void)
0766 {
0767     idxd_driver_unregister(&idxd_user_drv);
0768     idxd_driver_unregister(&idxd_dmaengine_drv);
0769     idxd_driver_unregister(&idxd_drv);
0770     pci_unregister_driver(&idxd_pci_driver);
0771     idxd_cdev_remove();
0772     perfmon_exit();
0773 }
0774 module_exit(idxd_exit_module);
[ Source navigation ] [ Diff markup ] [ Identifier search ] [ general search ]

This page was automatically generated by the 2.1.0 LXR engine.

The LXR team
Valid CSS 2.1! Valid HTML 4.01!