Back to home page

OSCL-LXR
 
 

     

0001 // SPDX-License-Identifier: GPL-2.0 OR BSD-2-Clause
0002 /*
0003  * Copyright 2018-2021 Amazon.com, Inc. or its affiliates. All rights reserved.
0004  */
0005 
0006 #include <linux/module.h>
0007 #include <linux/pci.h>
0008 #include <linux/utsname.h>
0009 #include <linux/version.h>
0010 
0011 #include <rdma/ib_user_verbs.h>
0012 
0013 #include "efa.h"
0014 
0015 #define PCI_DEV_ID_EFA0_VF 0xefa0
0016 #define PCI_DEV_ID_EFA1_VF 0xefa1
0017 
0018 static const struct pci_device_id efa_pci_tbl[] = {
0019     { PCI_VDEVICE(AMAZON, PCI_DEV_ID_EFA0_VF) },
0020     { PCI_VDEVICE(AMAZON, PCI_DEV_ID_EFA1_VF) },
0021     { }
0022 };
0023 
0024 MODULE_AUTHOR("Amazon.com, Inc. or its affiliates");
0025 MODULE_LICENSE("Dual BSD/GPL");
0026 MODULE_DESCRIPTION(DEVICE_NAME);
0027 MODULE_DEVICE_TABLE(pci, efa_pci_tbl);
0028 
0029 #define EFA_REG_BAR 0
0030 #define EFA_MEM_BAR 2
0031 #define EFA_BASE_BAR_MASK (BIT(EFA_REG_BAR) | BIT(EFA_MEM_BAR))
0032 
0033 #define EFA_AENQ_ENABLED_GROUPS \
0034     (BIT(EFA_ADMIN_FATAL_ERROR) | BIT(EFA_ADMIN_WARNING) | \
0035      BIT(EFA_ADMIN_NOTIFICATION) | BIT(EFA_ADMIN_KEEP_ALIVE))
0036 
0037 /* This handler will called for unknown event group or unimplemented handlers */
0038 static void unimplemented_aenq_handler(void *data,
0039                        struct efa_admin_aenq_entry *aenq_e)
0040 {
0041     struct efa_dev *dev = (struct efa_dev *)data;
0042 
0043     ibdev_err(&dev->ibdev,
0044           "Unknown event was received or event with unimplemented handler\n");
0045 }
0046 
0047 static void efa_keep_alive(void *data, struct efa_admin_aenq_entry *aenq_e)
0048 {
0049     struct efa_dev *dev = (struct efa_dev *)data;
0050 
0051     atomic64_inc(&dev->stats.keep_alive_rcvd);
0052 }
0053 
0054 static struct efa_aenq_handlers aenq_handlers = {
0055     .handlers = {
0056         [EFA_ADMIN_KEEP_ALIVE] = efa_keep_alive,
0057     },
0058     .unimplemented_handler = unimplemented_aenq_handler
0059 };
0060 
0061 static void efa_release_bars(struct efa_dev *dev, int bars_mask)
0062 {
0063     struct pci_dev *pdev = dev->pdev;
0064     int release_bars;
0065 
0066     release_bars = pci_select_bars(pdev, IORESOURCE_MEM) & bars_mask;
0067     pci_release_selected_regions(pdev, release_bars);
0068 }
0069 
0070 static void efa_process_comp_eqe(struct efa_dev *dev, struct efa_admin_eqe *eqe)
0071 {
0072     u16 cqn = eqe->u.comp_event.cqn;
0073     struct efa_cq *cq;
0074 
0075     /* Safe to load as we're in irq and removal calls synchronize_irq() */
0076     cq = xa_load(&dev->cqs_xa, cqn);
0077     if (unlikely(!cq)) {
0078         ibdev_err_ratelimited(&dev->ibdev,
0079                       "Completion event on non-existent CQ[%u]",
0080                       cqn);
0081         return;
0082     }
0083 
0084     cq->ibcq.comp_handler(&cq->ibcq, cq->ibcq.cq_context);
0085 }
0086 
0087 static void efa_process_eqe(struct efa_com_eq *eeq, struct efa_admin_eqe *eqe)
0088 {
0089     struct efa_dev *dev = container_of(eeq->edev, struct efa_dev, edev);
0090 
0091     if (likely(EFA_GET(&eqe->common, EFA_ADMIN_EQE_EVENT_TYPE) ==
0092                EFA_ADMIN_EQE_EVENT_TYPE_COMPLETION))
0093         efa_process_comp_eqe(dev, eqe);
0094     else
0095         ibdev_err_ratelimited(&dev->ibdev,
0096                       "Unknown event type received %lu",
0097                       EFA_GET(&eqe->common,
0098                           EFA_ADMIN_EQE_EVENT_TYPE));
0099 }
0100 
0101 static irqreturn_t efa_intr_msix_comp(int irq, void *data)
0102 {
0103     struct efa_eq *eq = data;
0104     struct efa_com_dev *edev = eq->eeq.edev;
0105 
0106     efa_com_eq_comp_intr_handler(edev, &eq->eeq);
0107 
0108     return IRQ_HANDLED;
0109 }
0110 
0111 static irqreturn_t efa_intr_msix_mgmnt(int irq, void *data)
0112 {
0113     struct efa_dev *dev = data;
0114 
0115     efa_com_admin_q_comp_intr_handler(&dev->edev);
0116     efa_com_aenq_intr_handler(&dev->edev, data);
0117 
0118     return IRQ_HANDLED;
0119 }
0120 
0121 static int efa_request_irq(struct efa_dev *dev, struct efa_irq *irq)
0122 {
0123     int err;
0124 
0125     err = request_irq(irq->irqn, irq->handler, 0, irq->name, irq->data);
0126     if (err) {
0127         dev_err(&dev->pdev->dev, "Failed to request irq %s (%d)\n",
0128             irq->name, err);
0129         return err;
0130     }
0131 
0132     irq_set_affinity_hint(irq->irqn, &irq->affinity_hint_mask);
0133 
0134     return 0;
0135 }
0136 
0137 static void efa_setup_comp_irq(struct efa_dev *dev, struct efa_eq *eq,
0138                    int vector)
0139 {
0140     u32 cpu;
0141 
0142     cpu = vector - EFA_COMP_EQS_VEC_BASE;
0143     snprintf(eq->irq.name, EFA_IRQNAME_SIZE, "efa-comp%d@pci:%s", cpu,
0144          pci_name(dev->pdev));
0145     eq->irq.handler = efa_intr_msix_comp;
0146     eq->irq.data = eq;
0147     eq->irq.vector = vector;
0148     eq->irq.irqn = pci_irq_vector(dev->pdev, vector);
0149     cpumask_set_cpu(cpu, &eq->irq.affinity_hint_mask);
0150 }
0151 
0152 static void efa_free_irq(struct efa_dev *dev, struct efa_irq *irq)
0153 {
0154     irq_set_affinity_hint(irq->irqn, NULL);
0155     free_irq(irq->irqn, irq->data);
0156 }
0157 
0158 static void efa_setup_mgmnt_irq(struct efa_dev *dev)
0159 {
0160     u32 cpu;
0161 
0162     snprintf(dev->admin_irq.name, EFA_IRQNAME_SIZE,
0163          "efa-mgmnt@pci:%s", pci_name(dev->pdev));
0164     dev->admin_irq.handler = efa_intr_msix_mgmnt;
0165     dev->admin_irq.data = dev;
0166     dev->admin_irq.vector = dev->admin_msix_vector_idx;
0167     dev->admin_irq.irqn = pci_irq_vector(dev->pdev,
0168                          dev->admin_msix_vector_idx);
0169     cpu = cpumask_first(cpu_online_mask);
0170     cpumask_set_cpu(cpu,
0171             &dev->admin_irq.affinity_hint_mask);
0172     dev_info(&dev->pdev->dev, "Setup irq:%d name:%s\n",
0173          dev->admin_irq.irqn,
0174          dev->admin_irq.name);
0175 }
0176 
0177 static int efa_set_mgmnt_irq(struct efa_dev *dev)
0178 {
0179     efa_setup_mgmnt_irq(dev);
0180 
0181     return efa_request_irq(dev, &dev->admin_irq);
0182 }
0183 
0184 static int efa_request_doorbell_bar(struct efa_dev *dev)
0185 {
0186     u8 db_bar_idx = dev->dev_attr.db_bar;
0187     struct pci_dev *pdev = dev->pdev;
0188     int bars;
0189     int err;
0190 
0191     if (!(BIT(db_bar_idx) & EFA_BASE_BAR_MASK)) {
0192         bars = pci_select_bars(pdev, IORESOURCE_MEM) & BIT(db_bar_idx);
0193 
0194         err = pci_request_selected_regions(pdev, bars, DRV_MODULE_NAME);
0195         if (err) {
0196             dev_err(&dev->pdev->dev,
0197                 "pci_request_selected_regions for bar %d failed %d\n",
0198                 db_bar_idx, err);
0199             return err;
0200         }
0201     }
0202 
0203     dev->db_bar_addr = pci_resource_start(dev->pdev, db_bar_idx);
0204     dev->db_bar_len = pci_resource_len(dev->pdev, db_bar_idx);
0205 
0206     return 0;
0207 }
0208 
0209 static void efa_release_doorbell_bar(struct efa_dev *dev)
0210 {
0211     if (!(BIT(dev->dev_attr.db_bar) & EFA_BASE_BAR_MASK))
0212         efa_release_bars(dev, BIT(dev->dev_attr.db_bar));
0213 }
0214 
0215 static void efa_update_hw_hints(struct efa_dev *dev,
0216                 struct efa_com_get_hw_hints_result *hw_hints)
0217 {
0218     struct efa_com_dev *edev = &dev->edev;
0219 
0220     if (hw_hints->mmio_read_timeout)
0221         edev->mmio_read.mmio_read_timeout =
0222             hw_hints->mmio_read_timeout * 1000;
0223 
0224     if (hw_hints->poll_interval)
0225         edev->aq.poll_interval = hw_hints->poll_interval;
0226 
0227     if (hw_hints->admin_completion_timeout)
0228         edev->aq.completion_timeout =
0229             hw_hints->admin_completion_timeout;
0230 }
0231 
0232 static void efa_stats_init(struct efa_dev *dev)
0233 {
0234     atomic64_t *s = (atomic64_t *)&dev->stats;
0235     int i;
0236 
0237     for (i = 0; i < sizeof(dev->stats) / sizeof(*s); i++, s++)
0238         atomic64_set(s, 0);
0239 }
0240 
0241 static void efa_set_host_info(struct efa_dev *dev)
0242 {
0243     struct efa_admin_set_feature_resp resp = {};
0244     struct efa_admin_set_feature_cmd cmd = {};
0245     struct efa_admin_host_info *hinf;
0246     u32 bufsz = sizeof(*hinf);
0247     dma_addr_t hinf_dma;
0248 
0249     if (!efa_com_check_supported_feature_id(&dev->edev,
0250                         EFA_ADMIN_HOST_INFO))
0251         return;
0252 
0253     /* Failures in host info set shall not disturb probe */
0254     hinf = dma_alloc_coherent(&dev->pdev->dev, bufsz, &hinf_dma,
0255                   GFP_KERNEL);
0256     if (!hinf)
0257         return;
0258 
0259     strscpy(hinf->os_dist_str, utsname()->release,
0260         sizeof(hinf->os_dist_str));
0261     hinf->os_type = EFA_ADMIN_OS_LINUX;
0262     strscpy(hinf->kernel_ver_str, utsname()->version,
0263         sizeof(hinf->kernel_ver_str));
0264     hinf->kernel_ver = LINUX_VERSION_CODE;
0265     EFA_SET(&hinf->driver_ver, EFA_ADMIN_HOST_INFO_DRIVER_MAJOR, 0);
0266     EFA_SET(&hinf->driver_ver, EFA_ADMIN_HOST_INFO_DRIVER_MINOR, 0);
0267     EFA_SET(&hinf->driver_ver, EFA_ADMIN_HOST_INFO_DRIVER_SUB_MINOR, 0);
0268     EFA_SET(&hinf->driver_ver, EFA_ADMIN_HOST_INFO_DRIVER_MODULE_TYPE, 0);
0269     EFA_SET(&hinf->bdf, EFA_ADMIN_HOST_INFO_BUS, dev->pdev->bus->number);
0270     EFA_SET(&hinf->bdf, EFA_ADMIN_HOST_INFO_DEVICE,
0271         PCI_SLOT(dev->pdev->devfn));
0272     EFA_SET(&hinf->bdf, EFA_ADMIN_HOST_INFO_FUNCTION,
0273         PCI_FUNC(dev->pdev->devfn));
0274     EFA_SET(&hinf->spec_ver, EFA_ADMIN_HOST_INFO_SPEC_MAJOR,
0275         EFA_COMMON_SPEC_VERSION_MAJOR);
0276     EFA_SET(&hinf->spec_ver, EFA_ADMIN_HOST_INFO_SPEC_MINOR,
0277         EFA_COMMON_SPEC_VERSION_MINOR);
0278     EFA_SET(&hinf->flags, EFA_ADMIN_HOST_INFO_INTREE, 1);
0279     EFA_SET(&hinf->flags, EFA_ADMIN_HOST_INFO_GDR, 0);
0280 
0281     efa_com_set_feature_ex(&dev->edev, &resp, &cmd, EFA_ADMIN_HOST_INFO,
0282                    hinf_dma, bufsz);
0283 
0284     dma_free_coherent(&dev->pdev->dev, bufsz, hinf, hinf_dma);
0285 }
0286 
0287 static void efa_destroy_eq(struct efa_dev *dev, struct efa_eq *eq)
0288 {
0289     efa_com_eq_destroy(&dev->edev, &eq->eeq);
0290     efa_free_irq(dev, &eq->irq);
0291 }
0292 
0293 static int efa_create_eq(struct efa_dev *dev, struct efa_eq *eq, u8 msix_vec)
0294 {
0295     int err;
0296 
0297     efa_setup_comp_irq(dev, eq, msix_vec);
0298     err = efa_request_irq(dev, &eq->irq);
0299     if (err)
0300         return err;
0301 
0302     err = efa_com_eq_init(&dev->edev, &eq->eeq, efa_process_eqe,
0303                   dev->dev_attr.max_eq_depth, msix_vec);
0304     if (err)
0305         goto err_free_comp_irq;
0306 
0307     return 0;
0308 
0309 err_free_comp_irq:
0310     efa_free_irq(dev, &eq->irq);
0311     return err;
0312 }
0313 
0314 static int efa_create_eqs(struct efa_dev *dev)
0315 {
0316     unsigned int neqs = dev->dev_attr.max_eq;
0317     int err;
0318     int i;
0319 
0320     neqs = min_t(unsigned int, neqs, num_online_cpus());
0321     dev->neqs = neqs;
0322     dev->eqs = kcalloc(neqs, sizeof(*dev->eqs), GFP_KERNEL);
0323     if (!dev->eqs)
0324         return -ENOMEM;
0325 
0326     for (i = 0; i < neqs; i++) {
0327         err = efa_create_eq(dev, &dev->eqs[i],
0328                     i + EFA_COMP_EQS_VEC_BASE);
0329         if (err)
0330             goto err_destroy_eqs;
0331     }
0332 
0333     return 0;
0334 
0335 err_destroy_eqs:
0336     for (i--; i >= 0; i--)
0337         efa_destroy_eq(dev, &dev->eqs[i]);
0338     kfree(dev->eqs);
0339 
0340     return err;
0341 }
0342 
0343 static void efa_destroy_eqs(struct efa_dev *dev)
0344 {
0345     int i;
0346 
0347     for (i = 0; i < dev->neqs; i++)
0348         efa_destroy_eq(dev, &dev->eqs[i]);
0349 
0350     kfree(dev->eqs);
0351 }
0352 
0353 static const struct ib_device_ops efa_dev_ops = {
0354     .owner = THIS_MODULE,
0355     .driver_id = RDMA_DRIVER_EFA,
0356     .uverbs_abi_ver = EFA_UVERBS_ABI_VERSION,
0357 
0358     .alloc_hw_port_stats = efa_alloc_hw_port_stats,
0359     .alloc_hw_device_stats = efa_alloc_hw_device_stats,
0360     .alloc_pd = efa_alloc_pd,
0361     .alloc_ucontext = efa_alloc_ucontext,
0362     .create_cq = efa_create_cq,
0363     .create_qp = efa_create_qp,
0364     .create_user_ah = efa_create_ah,
0365     .dealloc_pd = efa_dealloc_pd,
0366     .dealloc_ucontext = efa_dealloc_ucontext,
0367     .dereg_mr = efa_dereg_mr,
0368     .destroy_ah = efa_destroy_ah,
0369     .destroy_cq = efa_destroy_cq,
0370     .destroy_qp = efa_destroy_qp,
0371     .get_hw_stats = efa_get_hw_stats,
0372     .get_link_layer = efa_port_link_layer,
0373     .get_port_immutable = efa_get_port_immutable,
0374     .mmap = efa_mmap,
0375     .mmap_free = efa_mmap_free,
0376     .modify_qp = efa_modify_qp,
0377     .query_device = efa_query_device,
0378     .query_gid = efa_query_gid,
0379     .query_pkey = efa_query_pkey,
0380     .query_port = efa_query_port,
0381     .query_qp = efa_query_qp,
0382     .reg_user_mr = efa_reg_mr,
0383     .reg_user_mr_dmabuf = efa_reg_user_mr_dmabuf,
0384 
0385     INIT_RDMA_OBJ_SIZE(ib_ah, efa_ah, ibah),
0386     INIT_RDMA_OBJ_SIZE(ib_cq, efa_cq, ibcq),
0387     INIT_RDMA_OBJ_SIZE(ib_pd, efa_pd, ibpd),
0388     INIT_RDMA_OBJ_SIZE(ib_qp, efa_qp, ibqp),
0389     INIT_RDMA_OBJ_SIZE(ib_ucontext, efa_ucontext, ibucontext),
0390 };
0391 
0392 static int efa_ib_device_add(struct efa_dev *dev)
0393 {
0394     struct efa_com_get_hw_hints_result hw_hints;
0395     struct pci_dev *pdev = dev->pdev;
0396     int err;
0397 
0398     efa_stats_init(dev);
0399 
0400     err = efa_com_get_device_attr(&dev->edev, &dev->dev_attr);
0401     if (err)
0402         return err;
0403 
0404     dev_dbg(&dev->pdev->dev, "Doorbells bar (%d)\n", dev->dev_attr.db_bar);
0405     err = efa_request_doorbell_bar(dev);
0406     if (err)
0407         return err;
0408 
0409     err = efa_com_get_hw_hints(&dev->edev, &hw_hints);
0410     if (err)
0411         goto err_release_doorbell_bar;
0412 
0413     efa_update_hw_hints(dev, &hw_hints);
0414 
0415     /* Try to enable all the available aenq groups */
0416     err = efa_com_set_aenq_config(&dev->edev, EFA_AENQ_ENABLED_GROUPS);
0417     if (err)
0418         goto err_release_doorbell_bar;
0419 
0420     err = efa_create_eqs(dev);
0421     if (err)
0422         goto err_release_doorbell_bar;
0423 
0424     efa_set_host_info(dev);
0425 
0426     dev->ibdev.node_type = RDMA_NODE_UNSPECIFIED;
0427     dev->ibdev.phys_port_cnt = 1;
0428     dev->ibdev.num_comp_vectors = dev->neqs ?: 1;
0429     dev->ibdev.dev.parent = &pdev->dev;
0430 
0431     ib_set_device_ops(&dev->ibdev, &efa_dev_ops);
0432 
0433     err = ib_register_device(&dev->ibdev, "efa_%d", &pdev->dev);
0434     if (err)
0435         goto err_destroy_eqs;
0436 
0437     ibdev_info(&dev->ibdev, "IB device registered\n");
0438 
0439     return 0;
0440 
0441 err_destroy_eqs:
0442     efa_destroy_eqs(dev);
0443 err_release_doorbell_bar:
0444     efa_release_doorbell_bar(dev);
0445     return err;
0446 }
0447 
0448 static void efa_ib_device_remove(struct efa_dev *dev)
0449 {
0450     ibdev_info(&dev->ibdev, "Unregister ib device\n");
0451     ib_unregister_device(&dev->ibdev);
0452     efa_destroy_eqs(dev);
0453     efa_com_dev_reset(&dev->edev, EFA_REGS_RESET_NORMAL);
0454     efa_release_doorbell_bar(dev);
0455 }
0456 
0457 static void efa_disable_msix(struct efa_dev *dev)
0458 {
0459     pci_free_irq_vectors(dev->pdev);
0460 }
0461 
0462 static int efa_enable_msix(struct efa_dev *dev)
0463 {
0464     int msix_vecs, irq_num;
0465 
0466     /*
0467      * Reserve the max msix vectors we might need, one vector is reserved
0468      * for admin.
0469      */
0470     msix_vecs = min_t(int, pci_msix_vec_count(dev->pdev),
0471               num_online_cpus() + 1);
0472     dev_dbg(&dev->pdev->dev, "Trying to enable MSI-X, vectors %d\n",
0473         msix_vecs);
0474 
0475     dev->admin_msix_vector_idx = EFA_MGMNT_MSIX_VEC_IDX;
0476     irq_num = pci_alloc_irq_vectors(dev->pdev, msix_vecs,
0477                     msix_vecs, PCI_IRQ_MSIX);
0478 
0479     if (irq_num < 0) {
0480         dev_err(&dev->pdev->dev, "Failed to enable MSI-X. irq_num %d\n",
0481             irq_num);
0482         return -ENOSPC;
0483     }
0484 
0485     if (irq_num != msix_vecs) {
0486         efa_disable_msix(dev);
0487         dev_err(&dev->pdev->dev,
0488             "Allocated %d MSI-X (out of %d requested)\n",
0489             irq_num, msix_vecs);
0490         return -ENOSPC;
0491     }
0492 
0493     return 0;
0494 }
0495 
0496 static int efa_device_init(struct efa_com_dev *edev, struct pci_dev *pdev)
0497 {
0498     int dma_width;
0499     int err;
0500 
0501     err = efa_com_dev_reset(edev, EFA_REGS_RESET_NORMAL);
0502     if (err)
0503         return err;
0504 
0505     err = efa_com_validate_version(edev);
0506     if (err)
0507         return err;
0508 
0509     dma_width = efa_com_get_dma_width(edev);
0510     if (dma_width < 0) {
0511         err = dma_width;
0512         return err;
0513     }
0514 
0515     err = dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(dma_width));
0516     if (err) {
0517         dev_err(&pdev->dev, "dma_set_mask_and_coherent failed %d\n", err);
0518         return err;
0519     }
0520 
0521     dma_set_max_seg_size(&pdev->dev, UINT_MAX);
0522     return 0;
0523 }
0524 
0525 static struct efa_dev *efa_probe_device(struct pci_dev *pdev)
0526 {
0527     struct efa_com_dev *edev;
0528     struct efa_dev *dev;
0529     int bars;
0530     int err;
0531 
0532     err = pci_enable_device_mem(pdev);
0533     if (err) {
0534         dev_err(&pdev->dev, "pci_enable_device_mem() failed!\n");
0535         return ERR_PTR(err);
0536     }
0537 
0538     pci_set_master(pdev);
0539 
0540     dev = ib_alloc_device(efa_dev, ibdev);
0541     if (!dev) {
0542         dev_err(&pdev->dev, "Device alloc failed\n");
0543         err = -ENOMEM;
0544         goto err_disable_device;
0545     }
0546 
0547     pci_set_drvdata(pdev, dev);
0548     edev = &dev->edev;
0549     edev->efa_dev = dev;
0550     edev->dmadev = &pdev->dev;
0551     dev->pdev = pdev;
0552     xa_init(&dev->cqs_xa);
0553 
0554     bars = pci_select_bars(pdev, IORESOURCE_MEM) & EFA_BASE_BAR_MASK;
0555     err = pci_request_selected_regions(pdev, bars, DRV_MODULE_NAME);
0556     if (err) {
0557         dev_err(&pdev->dev, "pci_request_selected_regions failed %d\n",
0558             err);
0559         goto err_ibdev_destroy;
0560     }
0561 
0562     dev->reg_bar_addr = pci_resource_start(pdev, EFA_REG_BAR);
0563     dev->reg_bar_len = pci_resource_len(pdev, EFA_REG_BAR);
0564     dev->mem_bar_addr = pci_resource_start(pdev, EFA_MEM_BAR);
0565     dev->mem_bar_len = pci_resource_len(pdev, EFA_MEM_BAR);
0566 
0567     edev->reg_bar = devm_ioremap(&pdev->dev,
0568                      dev->reg_bar_addr,
0569                      dev->reg_bar_len);
0570     if (!edev->reg_bar) {
0571         dev_err(&pdev->dev, "Failed to remap register bar\n");
0572         err = -EFAULT;
0573         goto err_release_bars;
0574     }
0575 
0576     err = efa_com_mmio_reg_read_init(edev);
0577     if (err) {
0578         dev_err(&pdev->dev, "Failed to init readless MMIO\n");
0579         goto err_iounmap;
0580     }
0581 
0582     err = efa_device_init(edev, pdev);
0583     if (err) {
0584         dev_err(&pdev->dev, "EFA device init failed\n");
0585         if (err == -ETIME)
0586             err = -EPROBE_DEFER;
0587         goto err_reg_read_destroy;
0588     }
0589 
0590     err = efa_enable_msix(dev);
0591     if (err)
0592         goto err_reg_read_destroy;
0593 
0594     edev->aq.msix_vector_idx = dev->admin_msix_vector_idx;
0595     edev->aenq.msix_vector_idx = dev->admin_msix_vector_idx;
0596 
0597     err = efa_set_mgmnt_irq(dev);
0598     if (err)
0599         goto err_disable_msix;
0600 
0601     err = efa_com_admin_init(edev, &aenq_handlers);
0602     if (err)
0603         goto err_free_mgmnt_irq;
0604 
0605     return dev;
0606 
0607 err_free_mgmnt_irq:
0608     efa_free_irq(dev, &dev->admin_irq);
0609 err_disable_msix:
0610     efa_disable_msix(dev);
0611 err_reg_read_destroy:
0612     efa_com_mmio_reg_read_destroy(edev);
0613 err_iounmap:
0614     devm_iounmap(&pdev->dev, edev->reg_bar);
0615 err_release_bars:
0616     efa_release_bars(dev, EFA_BASE_BAR_MASK);
0617 err_ibdev_destroy:
0618     ib_dealloc_device(&dev->ibdev);
0619 err_disable_device:
0620     pci_disable_device(pdev);
0621     return ERR_PTR(err);
0622 }
0623 
0624 static void efa_remove_device(struct pci_dev *pdev)
0625 {
0626     struct efa_dev *dev = pci_get_drvdata(pdev);
0627     struct efa_com_dev *edev;
0628 
0629     edev = &dev->edev;
0630     efa_com_admin_destroy(edev);
0631     efa_free_irq(dev, &dev->admin_irq);
0632     efa_disable_msix(dev);
0633     efa_com_mmio_reg_read_destroy(edev);
0634     devm_iounmap(&pdev->dev, edev->reg_bar);
0635     efa_release_bars(dev, EFA_BASE_BAR_MASK);
0636     xa_destroy(&dev->cqs_xa);
0637     ib_dealloc_device(&dev->ibdev);
0638     pci_disable_device(pdev);
0639 }
0640 
0641 static int efa_probe(struct pci_dev *pdev, const struct pci_device_id *ent)
0642 {
0643     struct efa_dev *dev;
0644     int err;
0645 
0646     dev = efa_probe_device(pdev);
0647     if (IS_ERR(dev))
0648         return PTR_ERR(dev);
0649 
0650     err = efa_ib_device_add(dev);
0651     if (err)
0652         goto err_remove_device;
0653 
0654     return 0;
0655 
0656 err_remove_device:
0657     efa_remove_device(pdev);
0658     return err;
0659 }
0660 
0661 static void efa_remove(struct pci_dev *pdev)
0662 {
0663     struct efa_dev *dev = pci_get_drvdata(pdev);
0664 
0665     efa_ib_device_remove(dev);
0666     efa_remove_device(pdev);
0667 }
0668 
0669 static struct pci_driver efa_pci_driver = {
0670     .name           = DRV_MODULE_NAME,
0671     .id_table       = efa_pci_tbl,
0672     .probe          = efa_probe,
0673     .remove         = efa_remove,
0674 };
0675 
0676 module_pci_driver(efa_pci_driver);
[ Source navigation ] [ Diff markup ] [ Identifier search ] [ general search ]

This page was automatically generated by the 2.1.0 LXR engine.

The LXR team
Valid CSS 2.1! Valid HTML 4.01!