Back to home page

OSCL-LXR
 
 

     

0001 // SPDX-License-Identifier: GPL-2.0-only
0002 /*
0003  * QLogic iSCSI Offload Driver
0004  * Copyright (c) 2016 Cavium Inc.
0005  */
0006 
0007 #include <linux/module.h>
0008 #include <linux/pci.h>
0009 #include <linux/kernel.h>
0010 #include <linux/if_arp.h>
0011 #include <scsi/iscsi_if.h>
0012 #include <linux/inet.h>
0013 #include <net/arp.h>
0014 #include <linux/list.h>
0015 #include <linux/kthread.h>
0016 #include <linux/mm.h>
0017 #include <linux/if_vlan.h>
0018 #include <linux/cpu.h>
0019 #include <linux/iscsi_boot_sysfs.h>
0020 
0021 #include <scsi/scsi_cmnd.h>
0022 #include <scsi/scsi_device.h>
0023 #include <scsi/scsi_eh.h>
0024 #include <scsi/scsi_host.h>
0025 #include <scsi/scsi.h>
0026 
0027 #include "qedi.h"
0028 #include "qedi_gbl.h"
0029 #include "qedi_iscsi.h"
0030 
0031 static uint qedi_qed_debug;
0032 module_param(qedi_qed_debug, uint, 0644);
0033 MODULE_PARM_DESC(qedi_qed_debug, " QED debug level 0 (default)");
0034 
0035 static uint qedi_fw_debug;
0036 module_param(qedi_fw_debug, uint, 0644);
0037 MODULE_PARM_DESC(qedi_fw_debug, " Firmware debug level 0(default) to 3");
0038 
0039 uint qedi_dbg_log = QEDI_LOG_WARN | QEDI_LOG_SCSI_TM;
0040 module_param(qedi_dbg_log, uint, 0644);
0041 MODULE_PARM_DESC(qedi_dbg_log, " Default debug level");
0042 
0043 uint qedi_io_tracing;
0044 module_param(qedi_io_tracing, uint, 0644);
0045 MODULE_PARM_DESC(qedi_io_tracing,
0046          " Enable logging of SCSI requests/completions into trace buffer. (default off).");
0047 
0048 static uint qedi_ll2_buf_size = 0x400;
0049 module_param(qedi_ll2_buf_size, uint, 0644);
0050 MODULE_PARM_DESC(qedi_ll2_buf_size,
0051          "parameter to set ping packet size, default - 0x400, Jumbo packets - 0x2400.");
0052 
0053 static uint qedi_flags_override;
0054 module_param(qedi_flags_override, uint, 0644);
0055 MODULE_PARM_DESC(qedi_flags_override, "Disable/Enable MFW error flags bits action.");
0056 
0057 const struct qed_iscsi_ops *qedi_ops;
0058 static struct scsi_transport_template *qedi_scsi_transport;
0059 static struct pci_driver qedi_pci_driver;
0060 static DEFINE_PER_CPU(struct qedi_percpu_s, qedi_percpu);
0061 static LIST_HEAD(qedi_udev_list);
0062 /* Static function declaration */
0063 static int qedi_alloc_global_queues(struct qedi_ctx *qedi);
0064 static void qedi_free_global_queues(struct qedi_ctx *qedi);
0065 static struct qedi_cmd *qedi_get_cmd_from_tid(struct qedi_ctx *qedi, u32 tid);
0066 static void qedi_reset_uio_rings(struct qedi_uio_dev *udev);
0067 static void qedi_ll2_free_skbs(struct qedi_ctx *qedi);
0068 static struct nvm_iscsi_block *qedi_get_nvram_block(struct qedi_ctx *qedi);
0069 static void qedi_recovery_handler(struct work_struct *work);
0070 static void qedi_schedule_hw_err_handler(void *dev,
0071                      enum qed_hw_err_type err_type);
0072 
0073 static int qedi_iscsi_event_cb(void *context, u8 fw_event_code, void *fw_handle)
0074 {
0075     struct qedi_ctx *qedi;
0076     struct qedi_endpoint *qedi_ep;
0077     struct iscsi_eqe_data *data;
0078     int rval = 0;
0079 
0080     if (!context || !fw_handle) {
0081         QEDI_ERR(NULL, "Recv event with ctx NULL\n");
0082         return -EINVAL;
0083     }
0084 
0085     qedi = (struct qedi_ctx *)context;
0086     QEDI_INFO(&qedi->dbg_ctx, QEDI_LOG_INFO,
0087           "Recv Event %d fw_handle %p\n", fw_event_code, fw_handle);
0088 
0089     data = (struct iscsi_eqe_data *)fw_handle;
0090     QEDI_INFO(&qedi->dbg_ctx, QEDI_LOG_INFO,
0091           "icid=0x%x conn_id=0x%x err-code=0x%x error-pdu-opcode-reserved=0x%x\n",
0092            data->icid, data->conn_id, data->error_code,
0093            data->error_pdu_opcode_reserved);
0094 
0095     qedi_ep = qedi->ep_tbl[data->icid];
0096 
0097     if (!qedi_ep) {
0098         QEDI_WARN(&qedi->dbg_ctx,
0099               "Cannot process event, ep already disconnected, cid=0x%x\n",
0100                data->icid);
0101         WARN_ON(1);
0102         return -ENODEV;
0103     }
0104 
0105     switch (fw_event_code) {
0106     case ISCSI_EVENT_TYPE_ASYN_CONNECT_COMPLETE:
0107         if (qedi_ep->state == EP_STATE_OFLDCONN_START)
0108             qedi_ep->state = EP_STATE_OFLDCONN_COMPL;
0109 
0110         wake_up_interruptible(&qedi_ep->tcp_ofld_wait);
0111         break;
0112     case ISCSI_EVENT_TYPE_ASYN_TERMINATE_DONE:
0113         qedi_ep->state = EP_STATE_DISCONN_COMPL;
0114         wake_up_interruptible(&qedi_ep->tcp_ofld_wait);
0115         break;
0116     case ISCSI_EVENT_TYPE_ISCSI_CONN_ERROR:
0117         qedi_process_iscsi_error(qedi_ep, data);
0118         break;
0119     case ISCSI_EVENT_TYPE_ASYN_ABORT_RCVD:
0120     case ISCSI_EVENT_TYPE_ASYN_SYN_RCVD:
0121     case ISCSI_EVENT_TYPE_ASYN_MAX_RT_TIME:
0122     case ISCSI_EVENT_TYPE_ASYN_MAX_RT_CNT:
0123     case ISCSI_EVENT_TYPE_ASYN_MAX_KA_PROBES_CNT:
0124     case ISCSI_EVENT_TYPE_ASYN_FIN_WAIT2:
0125     case ISCSI_EVENT_TYPE_TCP_CONN_ERROR:
0126         qedi_process_tcp_error(qedi_ep, data);
0127         break;
0128     default:
0129         QEDI_ERR(&qedi->dbg_ctx, "Recv Unknown Event %u\n",
0130              fw_event_code);
0131     }
0132 
0133     return rval;
0134 }
0135 
0136 static int qedi_uio_open(struct uio_info *uinfo, struct inode *inode)
0137 {
0138     struct qedi_uio_dev *udev = uinfo->priv;
0139     struct qedi_ctx *qedi = udev->qedi;
0140 
0141     if (!capable(CAP_NET_ADMIN))
0142         return -EPERM;
0143 
0144     if (udev->uio_dev != -1)
0145         return -EBUSY;
0146 
0147     rtnl_lock();
0148     udev->uio_dev = iminor(inode);
0149     qedi_reset_uio_rings(udev);
0150     set_bit(UIO_DEV_OPENED, &qedi->flags);
0151     rtnl_unlock();
0152 
0153     return 0;
0154 }
0155 
0156 static int qedi_uio_close(struct uio_info *uinfo, struct inode *inode)
0157 {
0158     struct qedi_uio_dev *udev = uinfo->priv;
0159     struct qedi_ctx *qedi = udev->qedi;
0160 
0161     udev->uio_dev = -1;
0162     clear_bit(UIO_DEV_OPENED, &qedi->flags);
0163     qedi_ll2_free_skbs(qedi);
0164     return 0;
0165 }
0166 
0167 static void __qedi_free_uio_rings(struct qedi_uio_dev *udev)
0168 {
0169     if (udev->uctrl) {
0170         free_page((unsigned long)udev->uctrl);
0171         udev->uctrl = NULL;
0172     }
0173 
0174     if (udev->ll2_ring) {
0175         free_page((unsigned long)udev->ll2_ring);
0176         udev->ll2_ring = NULL;
0177     }
0178 
0179     if (udev->ll2_buf) {
0180         free_pages((unsigned long)udev->ll2_buf, 2);
0181         udev->ll2_buf = NULL;
0182     }
0183 }
0184 
0185 static void __qedi_free_uio(struct qedi_uio_dev *udev)
0186 {
0187     uio_unregister_device(&udev->qedi_uinfo);
0188 
0189     __qedi_free_uio_rings(udev);
0190 
0191     pci_dev_put(udev->pdev);
0192     kfree(udev);
0193 }
0194 
0195 static void qedi_free_uio(struct qedi_uio_dev *udev)
0196 {
0197     if (!udev)
0198         return;
0199 
0200     list_del_init(&udev->list);
0201     __qedi_free_uio(udev);
0202 }
0203 
0204 static void qedi_reset_uio_rings(struct qedi_uio_dev *udev)
0205 {
0206     struct qedi_ctx *qedi = NULL;
0207     struct qedi_uio_ctrl *uctrl = NULL;
0208 
0209     qedi = udev->qedi;
0210     uctrl = udev->uctrl;
0211 
0212     spin_lock_bh(&qedi->ll2_lock);
0213     uctrl->host_rx_cons = 0;
0214     uctrl->hw_rx_prod = 0;
0215     uctrl->hw_rx_bd_prod = 0;
0216     uctrl->host_rx_bd_cons = 0;
0217 
0218     memset(udev->ll2_ring, 0, udev->ll2_ring_size);
0219     memset(udev->ll2_buf, 0, udev->ll2_buf_size);
0220     spin_unlock_bh(&qedi->ll2_lock);
0221 }
0222 
0223 static int __qedi_alloc_uio_rings(struct qedi_uio_dev *udev)
0224 {
0225     int rc = 0;
0226 
0227     if (udev->ll2_ring || udev->ll2_buf)
0228         return rc;
0229 
0230     /* Memory for control area.  */
0231     udev->uctrl = (void *)get_zeroed_page(GFP_KERNEL);
0232     if (!udev->uctrl)
0233         return -ENOMEM;
0234 
0235     /* Allocating memory for LL2 ring  */
0236     udev->ll2_ring_size = QEDI_PAGE_SIZE;
0237     udev->ll2_ring = (void *)get_zeroed_page(GFP_KERNEL | __GFP_COMP);
0238     if (!udev->ll2_ring) {
0239         rc = -ENOMEM;
0240         goto exit_alloc_ring;
0241     }
0242 
0243     /* Allocating memory for Tx/Rx pkt buffer */
0244     udev->ll2_buf_size = TX_RX_RING * qedi_ll2_buf_size;
0245     udev->ll2_buf_size = QEDI_PAGE_ALIGN(udev->ll2_buf_size);
0246     udev->ll2_buf = (void *)__get_free_pages(GFP_KERNEL | __GFP_COMP |
0247                          __GFP_ZERO, 2);
0248     if (!udev->ll2_buf) {
0249         rc = -ENOMEM;
0250         goto exit_alloc_buf;
0251     }
0252     return rc;
0253 
0254 exit_alloc_buf:
0255     free_page((unsigned long)udev->ll2_ring);
0256     udev->ll2_ring = NULL;
0257 exit_alloc_ring:
0258     return rc;
0259 }
0260 
0261 static int qedi_alloc_uio_rings(struct qedi_ctx *qedi)
0262 {
0263     struct qedi_uio_dev *udev = NULL;
0264     int rc = 0;
0265 
0266     list_for_each_entry(udev, &qedi_udev_list, list) {
0267         if (udev->pdev == qedi->pdev) {
0268             udev->qedi = qedi;
0269             if (__qedi_alloc_uio_rings(udev)) {
0270                 udev->qedi = NULL;
0271                 return -ENOMEM;
0272             }
0273             qedi->udev = udev;
0274             return 0;
0275         }
0276     }
0277 
0278     udev = kzalloc(sizeof(*udev), GFP_KERNEL);
0279     if (!udev)
0280         goto err_udev;
0281 
0282     udev->uio_dev = -1;
0283 
0284     udev->qedi = qedi;
0285     udev->pdev = qedi->pdev;
0286 
0287     rc = __qedi_alloc_uio_rings(udev);
0288     if (rc)
0289         goto err_uctrl;
0290 
0291     list_add(&udev->list, &qedi_udev_list);
0292 
0293     pci_dev_get(udev->pdev);
0294     qedi->udev = udev;
0295 
0296     udev->tx_pkt = udev->ll2_buf;
0297     udev->rx_pkt = udev->ll2_buf + qedi_ll2_buf_size;
0298     return 0;
0299 
0300  err_uctrl:
0301     kfree(udev);
0302  err_udev:
0303     return -ENOMEM;
0304 }
0305 
0306 static int qedi_init_uio(struct qedi_ctx *qedi)
0307 {
0308     struct qedi_uio_dev *udev = qedi->udev;
0309     struct uio_info *uinfo;
0310     int ret = 0;
0311 
0312     if (!udev)
0313         return -ENOMEM;
0314 
0315     uinfo = &udev->qedi_uinfo;
0316 
0317     uinfo->mem[0].addr = (unsigned long)udev->uctrl;
0318     uinfo->mem[0].size = sizeof(struct qedi_uio_ctrl);
0319     uinfo->mem[0].memtype = UIO_MEM_LOGICAL;
0320 
0321     uinfo->mem[1].addr = (unsigned long)udev->ll2_ring;
0322     uinfo->mem[1].size = udev->ll2_ring_size;
0323     uinfo->mem[1].memtype = UIO_MEM_LOGICAL;
0324 
0325     uinfo->mem[2].addr = (unsigned long)udev->ll2_buf;
0326     uinfo->mem[2].size = udev->ll2_buf_size;
0327     uinfo->mem[2].memtype = UIO_MEM_LOGICAL;
0328 
0329     uinfo->name = "qedi_uio";
0330     uinfo->version = QEDI_MODULE_VERSION;
0331     uinfo->irq = UIO_IRQ_CUSTOM;
0332 
0333     uinfo->open = qedi_uio_open;
0334     uinfo->release = qedi_uio_close;
0335 
0336     if (udev->uio_dev == -1) {
0337         if (!uinfo->priv) {
0338             uinfo->priv = udev;
0339 
0340             ret = uio_register_device(&udev->pdev->dev, uinfo);
0341             if (ret) {
0342                 QEDI_ERR(&qedi->dbg_ctx,
0343                      "UIO registration failed\n");
0344             }
0345         }
0346     }
0347 
0348     return ret;
0349 }
0350 
0351 static int qedi_alloc_and_init_sb(struct qedi_ctx *qedi,
0352                   struct qed_sb_info *sb_info, u16 sb_id)
0353 {
0354     struct status_block *sb_virt;
0355     dma_addr_t sb_phys;
0356     int ret;
0357 
0358     sb_virt = dma_alloc_coherent(&qedi->pdev->dev,
0359                      sizeof(struct status_block), &sb_phys,
0360                      GFP_KERNEL);
0361     if (!sb_virt) {
0362         QEDI_ERR(&qedi->dbg_ctx,
0363              "Status block allocation failed for id = %d.\n",
0364               sb_id);
0365         return -ENOMEM;
0366     }
0367 
0368     ret = qedi_ops->common->sb_init(qedi->cdev, sb_info, sb_virt, sb_phys,
0369                        sb_id, QED_SB_TYPE_STORAGE);
0370     if (ret) {
0371         QEDI_ERR(&qedi->dbg_ctx,
0372              "Status block initialization failed for id = %d.\n",
0373               sb_id);
0374         return ret;
0375     }
0376 
0377     return 0;
0378 }
0379 
0380 static void qedi_free_sb(struct qedi_ctx *qedi)
0381 {
0382     struct qed_sb_info *sb_info;
0383     int id;
0384 
0385     for (id = 0; id < MIN_NUM_CPUS_MSIX(qedi); id++) {
0386         sb_info = &qedi->sb_array[id];
0387         if (sb_info->sb_virt)
0388             dma_free_coherent(&qedi->pdev->dev,
0389                       sizeof(*sb_info->sb_virt),
0390                       (void *)sb_info->sb_virt,
0391                       sb_info->sb_phys);
0392     }
0393 }
0394 
0395 static void qedi_free_fp(struct qedi_ctx *qedi)
0396 {
0397     kfree(qedi->fp_array);
0398     kfree(qedi->sb_array);
0399 }
0400 
0401 static void qedi_destroy_fp(struct qedi_ctx *qedi)
0402 {
0403     qedi_free_sb(qedi);
0404     qedi_free_fp(qedi);
0405 }
0406 
0407 static int qedi_alloc_fp(struct qedi_ctx *qedi)
0408 {
0409     int ret = 0;
0410 
0411     qedi->fp_array = kcalloc(MIN_NUM_CPUS_MSIX(qedi),
0412                  sizeof(struct qedi_fastpath), GFP_KERNEL);
0413     if (!qedi->fp_array) {
0414         QEDI_ERR(&qedi->dbg_ctx,
0415              "fastpath fp array allocation failed.\n");
0416         return -ENOMEM;
0417     }
0418 
0419     qedi->sb_array = kcalloc(MIN_NUM_CPUS_MSIX(qedi),
0420                  sizeof(struct qed_sb_info), GFP_KERNEL);
0421     if (!qedi->sb_array) {
0422         QEDI_ERR(&qedi->dbg_ctx,
0423              "fastpath sb array allocation failed.\n");
0424         ret = -ENOMEM;
0425         goto free_fp;
0426     }
0427 
0428     return ret;
0429 
0430 free_fp:
0431     qedi_free_fp(qedi);
0432     return ret;
0433 }
0434 
0435 static void qedi_int_fp(struct qedi_ctx *qedi)
0436 {
0437     struct qedi_fastpath *fp;
0438     int id;
0439 
0440     memset(qedi->fp_array, 0, MIN_NUM_CPUS_MSIX(qedi) *
0441            sizeof(*qedi->fp_array));
0442     memset(qedi->sb_array, 0, MIN_NUM_CPUS_MSIX(qedi) *
0443            sizeof(*qedi->sb_array));
0444 
0445     for (id = 0; id < MIN_NUM_CPUS_MSIX(qedi); id++) {
0446         fp = &qedi->fp_array[id];
0447         fp->sb_info = &qedi->sb_array[id];
0448         fp->sb_id = id;
0449         fp->qedi = qedi;
0450         snprintf(fp->name, sizeof(fp->name), "%s-fp-%d",
0451              "qedi", id);
0452 
0453         /* fp_array[i] ---- irq cookie
0454          * So init data which is needed in int ctx
0455          */
0456     }
0457 }
0458 
0459 static int qedi_prepare_fp(struct qedi_ctx *qedi)
0460 {
0461     struct qedi_fastpath *fp;
0462     int id, ret = 0;
0463 
0464     ret = qedi_alloc_fp(qedi);
0465     if (ret)
0466         goto err;
0467 
0468     qedi_int_fp(qedi);
0469 
0470     for (id = 0; id < MIN_NUM_CPUS_MSIX(qedi); id++) {
0471         fp = &qedi->fp_array[id];
0472         ret = qedi_alloc_and_init_sb(qedi, fp->sb_info, fp->sb_id);
0473         if (ret) {
0474             QEDI_ERR(&qedi->dbg_ctx,
0475                  "SB allocation and initialization failed.\n");
0476             ret = -EIO;
0477             goto err_init;
0478         }
0479     }
0480 
0481     return 0;
0482 
0483 err_init:
0484     qedi_free_sb(qedi);
0485     qedi_free_fp(qedi);
0486 err:
0487     return ret;
0488 }
0489 
0490 static int qedi_setup_cid_que(struct qedi_ctx *qedi)
0491 {
0492     int i;
0493 
0494     qedi->cid_que.cid_que_base = kmalloc_array(qedi->max_active_conns,
0495                            sizeof(u32), GFP_KERNEL);
0496     if (!qedi->cid_que.cid_que_base)
0497         return -ENOMEM;
0498 
0499     qedi->cid_que.conn_cid_tbl = kmalloc_array(qedi->max_active_conns,
0500                            sizeof(struct qedi_conn *),
0501                            GFP_KERNEL);
0502     if (!qedi->cid_que.conn_cid_tbl) {
0503         kfree(qedi->cid_que.cid_que_base);
0504         qedi->cid_que.cid_que_base = NULL;
0505         return -ENOMEM;
0506     }
0507 
0508     qedi->cid_que.cid_que = (u32 *)qedi->cid_que.cid_que_base;
0509     qedi->cid_que.cid_q_prod_idx = 0;
0510     qedi->cid_que.cid_q_cons_idx = 0;
0511     qedi->cid_que.cid_q_max_idx = qedi->max_active_conns;
0512     qedi->cid_que.cid_free_cnt = qedi->max_active_conns;
0513 
0514     for (i = 0; i < qedi->max_active_conns; i++) {
0515         qedi->cid_que.cid_que[i] = i;
0516         qedi->cid_que.conn_cid_tbl[i] = NULL;
0517     }
0518 
0519     return 0;
0520 }
0521 
0522 static void qedi_release_cid_que(struct qedi_ctx *qedi)
0523 {
0524     kfree(qedi->cid_que.cid_que_base);
0525     qedi->cid_que.cid_que_base = NULL;
0526 
0527     kfree(qedi->cid_que.conn_cid_tbl);
0528     qedi->cid_que.conn_cid_tbl = NULL;
0529 }
0530 
0531 static int qedi_init_id_tbl(struct qedi_portid_tbl *id_tbl, u16 size,
0532                 u16 start_id, u16 next)
0533 {
0534     id_tbl->start = start_id;
0535     id_tbl->max = size;
0536     id_tbl->next = next;
0537     spin_lock_init(&id_tbl->lock);
0538     id_tbl->table = kcalloc(BITS_TO_LONGS(size), sizeof(long), GFP_KERNEL);
0539     if (!id_tbl->table)
0540         return -ENOMEM;
0541 
0542     return 0;
0543 }
0544 
0545 static void qedi_free_id_tbl(struct qedi_portid_tbl *id_tbl)
0546 {
0547     kfree(id_tbl->table);
0548     id_tbl->table = NULL;
0549 }
0550 
0551 int qedi_alloc_id(struct qedi_portid_tbl *id_tbl, u16 id)
0552 {
0553     int ret = -1;
0554 
0555     id -= id_tbl->start;
0556     if (id >= id_tbl->max)
0557         return ret;
0558 
0559     spin_lock(&id_tbl->lock);
0560     if (!test_bit(id, id_tbl->table)) {
0561         set_bit(id, id_tbl->table);
0562         ret = 0;
0563     }
0564     spin_unlock(&id_tbl->lock);
0565     return ret;
0566 }
0567 
0568 u16 qedi_alloc_new_id(struct qedi_portid_tbl *id_tbl)
0569 {
0570     u16 id;
0571 
0572     spin_lock(&id_tbl->lock);
0573     id = find_next_zero_bit(id_tbl->table, id_tbl->max, id_tbl->next);
0574     if (id >= id_tbl->max) {
0575         id = QEDI_LOCAL_PORT_INVALID;
0576         if (id_tbl->next != 0) {
0577             id = find_first_zero_bit(id_tbl->table, id_tbl->next);
0578             if (id >= id_tbl->next)
0579                 id = QEDI_LOCAL_PORT_INVALID;
0580         }
0581     }
0582 
0583     if (id < id_tbl->max) {
0584         set_bit(id, id_tbl->table);
0585         id_tbl->next = (id + 1) & (id_tbl->max - 1);
0586         id += id_tbl->start;
0587     }
0588 
0589     spin_unlock(&id_tbl->lock);
0590 
0591     return id;
0592 }
0593 
0594 void qedi_free_id(struct qedi_portid_tbl *id_tbl, u16 id)
0595 {
0596     if (id == QEDI_LOCAL_PORT_INVALID)
0597         return;
0598 
0599     id -= id_tbl->start;
0600     if (id >= id_tbl->max)
0601         return;
0602 
0603     clear_bit(id, id_tbl->table);
0604 }
0605 
0606 static void qedi_cm_free_mem(struct qedi_ctx *qedi)
0607 {
0608     kfree(qedi->ep_tbl);
0609     qedi->ep_tbl = NULL;
0610     qedi_free_id_tbl(&qedi->lcl_port_tbl);
0611 }
0612 
0613 static int qedi_cm_alloc_mem(struct qedi_ctx *qedi)
0614 {
0615     u16 port_id;
0616 
0617     qedi->ep_tbl = kzalloc((qedi->max_active_conns *
0618                 sizeof(struct qedi_endpoint *)), GFP_KERNEL);
0619     if (!qedi->ep_tbl)
0620         return -ENOMEM;
0621     port_id = prandom_u32() % QEDI_LOCAL_PORT_RANGE;
0622     if (qedi_init_id_tbl(&qedi->lcl_port_tbl, QEDI_LOCAL_PORT_RANGE,
0623                  QEDI_LOCAL_PORT_MIN, port_id)) {
0624         qedi_cm_free_mem(qedi);
0625         return -ENOMEM;
0626     }
0627 
0628     return 0;
0629 }
0630 
0631 static struct qedi_ctx *qedi_host_alloc(struct pci_dev *pdev)
0632 {
0633     struct Scsi_Host *shost;
0634     struct qedi_ctx *qedi = NULL;
0635 
0636     shost = iscsi_host_alloc(&qedi_host_template,
0637                  sizeof(struct qedi_ctx), 0);
0638     if (!shost) {
0639         QEDI_ERR(NULL, "Could not allocate shost\n");
0640         goto exit_setup_shost;
0641     }
0642 
0643     shost->max_id = QEDI_MAX_ISCSI_CONNS_PER_HBA - 1;
0644     shost->max_channel = 0;
0645     shost->max_lun = ~0;
0646     shost->max_cmd_len = 16;
0647     shost->transportt = qedi_scsi_transport;
0648 
0649     qedi = iscsi_host_priv(shost);
0650     memset(qedi, 0, sizeof(*qedi));
0651     qedi->shost = shost;
0652     qedi->dbg_ctx.host_no = shost->host_no;
0653     qedi->pdev = pdev;
0654     qedi->dbg_ctx.pdev = pdev;
0655     qedi->max_active_conns = ISCSI_MAX_SESS_PER_HBA;
0656     qedi->max_sqes = QEDI_SQ_SIZE;
0657 
0658     shost->nr_hw_queues = MIN_NUM_CPUS_MSIX(qedi);
0659 
0660     pci_set_drvdata(pdev, qedi);
0661 
0662 exit_setup_shost:
0663     return qedi;
0664 }
0665 
0666 static int qedi_ll2_rx(void *cookie, struct sk_buff *skb, u32 arg1, u32 arg2)
0667 {
0668     struct qedi_ctx *qedi = (struct qedi_ctx *)cookie;
0669     struct skb_work_list *work;
0670     struct ethhdr *eh;
0671 
0672     if (!qedi) {
0673         QEDI_ERR(NULL, "qedi is NULL\n");
0674         return -1;
0675     }
0676 
0677     if (!test_bit(UIO_DEV_OPENED, &qedi->flags)) {
0678         QEDI_INFO(&qedi->dbg_ctx, QEDI_LOG_UIO,
0679               "UIO DEV is not opened\n");
0680         kfree_skb(skb);
0681         return 0;
0682     }
0683 
0684     eh = (struct ethhdr *)skb->data;
0685     /* Undo VLAN encapsulation */
0686     if (eh->h_proto == htons(ETH_P_8021Q)) {
0687         memmove((u8 *)eh + VLAN_HLEN, eh, ETH_ALEN * 2);
0688         eh = (struct ethhdr *)skb_pull(skb, VLAN_HLEN);
0689         skb_reset_mac_header(skb);
0690     }
0691 
0692     /* Filter out non FIP/FCoE frames here to free them faster */
0693     if (eh->h_proto != htons(ETH_P_ARP) &&
0694         eh->h_proto != htons(ETH_P_IP) &&
0695         eh->h_proto != htons(ETH_P_IPV6)) {
0696         QEDI_INFO(&qedi->dbg_ctx, QEDI_LOG_LL2,
0697               "Dropping frame ethertype [0x%x] len [0x%x].\n",
0698               eh->h_proto, skb->len);
0699         kfree_skb(skb);
0700         return 0;
0701     }
0702 
0703     QEDI_INFO(&qedi->dbg_ctx, QEDI_LOG_LL2,
0704           "Allowed frame ethertype [0x%x] len [0x%x].\n",
0705           eh->h_proto, skb->len);
0706 
0707     work = kzalloc(sizeof(*work), GFP_ATOMIC);
0708     if (!work) {
0709         QEDI_WARN(&qedi->dbg_ctx,
0710               "Could not allocate work so dropping frame.\n");
0711         kfree_skb(skb);
0712         return 0;
0713     }
0714 
0715     INIT_LIST_HEAD(&work->list);
0716     work->skb = skb;
0717 
0718     if (skb_vlan_tag_present(skb))
0719         work->vlan_id = skb_vlan_tag_get(skb);
0720 
0721     if (work->vlan_id)
0722         __vlan_insert_tag(work->skb, htons(ETH_P_8021Q), work->vlan_id);
0723 
0724     spin_lock_bh(&qedi->ll2_lock);
0725     list_add_tail(&work->list, &qedi->ll2_skb_list);
0726     spin_unlock_bh(&qedi->ll2_lock);
0727 
0728     wake_up_process(qedi->ll2_recv_thread);
0729 
0730     return 0;
0731 }
0732 
0733 /* map this skb to iscsiuio mmaped region */
0734 static int qedi_ll2_process_skb(struct qedi_ctx *qedi, struct sk_buff *skb,
0735                 u16 vlan_id)
0736 {
0737     struct qedi_uio_dev *udev = NULL;
0738     struct qedi_uio_ctrl *uctrl = NULL;
0739     struct qedi_rx_bd rxbd;
0740     struct qedi_rx_bd *p_rxbd;
0741     u32 rx_bd_prod;
0742     void *pkt;
0743     int len = 0;
0744     u32 prod;
0745 
0746     if (!qedi) {
0747         QEDI_ERR(NULL, "qedi is NULL\n");
0748         return -1;
0749     }
0750 
0751     udev = qedi->udev;
0752     uctrl = udev->uctrl;
0753 
0754     ++uctrl->hw_rx_prod_cnt;
0755     prod = (uctrl->hw_rx_prod + 1) % RX_RING;
0756 
0757     pkt = udev->rx_pkt + (prod * qedi_ll2_buf_size);
0758     len = min_t(u32, skb->len, (u32)qedi_ll2_buf_size);
0759     memcpy(pkt, skb->data, len);
0760 
0761     memset(&rxbd, 0, sizeof(rxbd));
0762     rxbd.rx_pkt_index = prod;
0763     rxbd.rx_pkt_len = len;
0764     rxbd.vlan_id = vlan_id;
0765 
0766     uctrl->hw_rx_bd_prod = (uctrl->hw_rx_bd_prod + 1) % QEDI_NUM_RX_BD;
0767     rx_bd_prod = uctrl->hw_rx_bd_prod;
0768     p_rxbd = (struct qedi_rx_bd *)udev->ll2_ring;
0769     p_rxbd += rx_bd_prod;
0770 
0771     memcpy(p_rxbd, &rxbd, sizeof(rxbd));
0772 
0773     QEDI_INFO(&qedi->dbg_ctx, QEDI_LOG_LL2,
0774           "hw_rx_prod [%d] prod [%d] hw_rx_bd_prod [%d] rx_pkt_idx [%d] rx_len [%d].\n",
0775           uctrl->hw_rx_prod, prod, uctrl->hw_rx_bd_prod,
0776           rxbd.rx_pkt_index, rxbd.rx_pkt_len);
0777     QEDI_INFO(&qedi->dbg_ctx, QEDI_LOG_LL2,
0778           "host_rx_cons [%d] hw_rx_bd_cons [%d].\n",
0779           uctrl->host_rx_cons, uctrl->host_rx_bd_cons);
0780 
0781     uctrl->hw_rx_prod = prod;
0782 
0783     /* notify the iscsiuio about new packet */
0784     uio_event_notify(&udev->qedi_uinfo);
0785 
0786     return 0;
0787 }
0788 
0789 static void qedi_ll2_free_skbs(struct qedi_ctx *qedi)
0790 {
0791     struct skb_work_list *work, *work_tmp;
0792 
0793     spin_lock_bh(&qedi->ll2_lock);
0794     list_for_each_entry_safe(work, work_tmp, &qedi->ll2_skb_list, list) {
0795         list_del(&work->list);
0796         kfree_skb(work->skb);
0797         kfree(work);
0798     }
0799     spin_unlock_bh(&qedi->ll2_lock);
0800 }
0801 
0802 static int qedi_ll2_recv_thread(void *arg)
0803 {
0804     struct qedi_ctx *qedi = (struct qedi_ctx *)arg;
0805     struct skb_work_list *work, *work_tmp;
0806 
0807     set_user_nice(current, -20);
0808 
0809     while (!kthread_should_stop()) {
0810         spin_lock_bh(&qedi->ll2_lock);
0811         list_for_each_entry_safe(work, work_tmp, &qedi->ll2_skb_list,
0812                      list) {
0813             list_del(&work->list);
0814             qedi_ll2_process_skb(qedi, work->skb, work->vlan_id);
0815             kfree_skb(work->skb);
0816             kfree(work);
0817         }
0818         set_current_state(TASK_INTERRUPTIBLE);
0819         spin_unlock_bh(&qedi->ll2_lock);
0820         schedule();
0821     }
0822 
0823     __set_current_state(TASK_RUNNING);
0824     return 0;
0825 }
0826 
0827 static int qedi_set_iscsi_pf_param(struct qedi_ctx *qedi)
0828 {
0829     u8 num_sq_pages;
0830     u32 log_page_size;
0831     int rval = 0;
0832 
0833 
0834     num_sq_pages = (MAX_OUTSTANDING_TASKS_PER_CON * 8) / QEDI_PAGE_SIZE;
0835 
0836     qedi->num_queues = MIN_NUM_CPUS_MSIX(qedi);
0837 
0838     QEDI_INFO(&qedi->dbg_ctx, QEDI_LOG_INFO,
0839           "Number of CQ count is %d\n", qedi->num_queues);
0840 
0841     memset(&qedi->pf_params.iscsi_pf_params, 0,
0842            sizeof(qedi->pf_params.iscsi_pf_params));
0843 
0844     qedi->p_cpuq = dma_alloc_coherent(&qedi->pdev->dev,
0845             qedi->num_queues * sizeof(struct qedi_glbl_q_params),
0846             &qedi->hw_p_cpuq, GFP_KERNEL);
0847     if (!qedi->p_cpuq) {
0848         QEDI_ERR(&qedi->dbg_ctx, "dma_alloc_coherent fail\n");
0849         rval = -1;
0850         goto err_alloc_mem;
0851     }
0852 
0853     rval = qedi_alloc_global_queues(qedi);
0854     if (rval) {
0855         QEDI_ERR(&qedi->dbg_ctx, "Global queue allocation failed.\n");
0856         rval = -1;
0857         goto err_alloc_mem;
0858     }
0859 
0860     qedi->pf_params.iscsi_pf_params.num_cons = QEDI_MAX_ISCSI_CONNS_PER_HBA;
0861     qedi->pf_params.iscsi_pf_params.num_tasks = QEDI_MAX_ISCSI_TASK;
0862     qedi->pf_params.iscsi_pf_params.half_way_close_timeout = 10;
0863     qedi->pf_params.iscsi_pf_params.num_sq_pages_in_ring = num_sq_pages;
0864     qedi->pf_params.iscsi_pf_params.num_r2tq_pages_in_ring = num_sq_pages;
0865     qedi->pf_params.iscsi_pf_params.num_uhq_pages_in_ring = num_sq_pages;
0866     qedi->pf_params.iscsi_pf_params.num_queues = qedi->num_queues;
0867     qedi->pf_params.iscsi_pf_params.debug_mode = qedi_fw_debug;
0868     qedi->pf_params.iscsi_pf_params.two_msl_timer = QED_TWO_MSL_TIMER_DFLT;
0869     qedi->pf_params.iscsi_pf_params.tx_sws_timer = QED_TX_SWS_TIMER_DFLT;
0870     qedi->pf_params.iscsi_pf_params.max_fin_rt = 2;
0871 
0872     for (log_page_size = 0 ; log_page_size < 32 ; log_page_size++) {
0873         if ((1 << log_page_size) == QEDI_PAGE_SIZE)
0874             break;
0875     }
0876     qedi->pf_params.iscsi_pf_params.log_page_size = log_page_size;
0877 
0878     qedi->pf_params.iscsi_pf_params.glbl_q_params_addr =
0879                                (u64)qedi->hw_p_cpuq;
0880 
0881     /* RQ BDQ initializations.
0882      * rq_num_entries: suggested value for Initiator is 16 (4KB RQ)
0883      * rqe_log_size: 8 for 256B RQE
0884      */
0885     qedi->pf_params.iscsi_pf_params.rqe_log_size = 8;
0886     /* BDQ address and size */
0887     qedi->pf_params.iscsi_pf_params.bdq_pbl_base_addr[BDQ_ID_RQ] =
0888                             qedi->bdq_pbl_list_dma;
0889     qedi->pf_params.iscsi_pf_params.bdq_pbl_num_entries[BDQ_ID_RQ] =
0890                         qedi->bdq_pbl_list_num_entries;
0891     qedi->pf_params.iscsi_pf_params.rq_buffer_size = QEDI_BDQ_BUF_SIZE;
0892 
0893     /* cq_num_entries: num_tasks + rq_num_entries */
0894     qedi->pf_params.iscsi_pf_params.cq_num_entries = 2048;
0895 
0896     qedi->pf_params.iscsi_pf_params.gl_rq_pi = QEDI_PROTO_CQ_PROD_IDX;
0897     qedi->pf_params.iscsi_pf_params.gl_cmd_pi = 1;
0898 
0899 err_alloc_mem:
0900     return rval;
0901 }
0902 
0903 /* Free DMA coherent memory for array of queue pointers we pass to qed */
0904 static void qedi_free_iscsi_pf_param(struct qedi_ctx *qedi)
0905 {
0906     size_t size = 0;
0907 
0908     if (qedi->p_cpuq) {
0909         size = qedi->num_queues * sizeof(struct qedi_glbl_q_params);
0910         dma_free_coherent(&qedi->pdev->dev, size, qedi->p_cpuq,
0911                     qedi->hw_p_cpuq);
0912     }
0913 
0914     qedi_free_global_queues(qedi);
0915 
0916     kfree(qedi->global_queues);
0917 }
0918 
0919 static void qedi_get_boot_tgt_info(struct nvm_iscsi_block *block,
0920                    struct qedi_boot_target *tgt, u8 index)
0921 {
0922     u32 ipv6_en;
0923 
0924     ipv6_en = !!(block->generic.ctrl_flags &
0925              NVM_ISCSI_CFG_GEN_IPV6_ENABLED);
0926 
0927     snprintf(tgt->iscsi_name, sizeof(tgt->iscsi_name), "%s",
0928          block->target[index].target_name.byte);
0929 
0930     tgt->ipv6_en = ipv6_en;
0931 
0932     if (ipv6_en)
0933         snprintf(tgt->ip_addr, IPV6_LEN, "%pI6\n",
0934              block->target[index].ipv6_addr.byte);
0935     else
0936         snprintf(tgt->ip_addr, IPV4_LEN, "%pI4\n",
0937              block->target[index].ipv4_addr.byte);
0938 }
0939 
0940 static int qedi_find_boot_info(struct qedi_ctx *qedi,
0941                    struct qed_mfw_tlv_iscsi *iscsi,
0942                    struct nvm_iscsi_block *block)
0943 {
0944     struct qedi_boot_target *pri_tgt = NULL, *sec_tgt = NULL;
0945     u32 pri_ctrl_flags = 0, sec_ctrl_flags = 0, found = 0;
0946     struct iscsi_cls_session *cls_sess;
0947     struct iscsi_cls_conn *cls_conn;
0948     struct qedi_conn *qedi_conn;
0949     struct iscsi_session *sess;
0950     struct iscsi_conn *conn;
0951     char ep_ip_addr[64];
0952     int i, ret = 0;
0953 
0954     pri_ctrl_flags = !!(block->target[0].ctrl_flags &
0955                     NVM_ISCSI_CFG_TARGET_ENABLED);
0956     if (pri_ctrl_flags) {
0957         pri_tgt = kzalloc(sizeof(*pri_tgt), GFP_KERNEL);
0958         if (!pri_tgt)
0959             return -1;
0960         qedi_get_boot_tgt_info(block, pri_tgt, 0);
0961     }
0962 
0963     sec_ctrl_flags = !!(block->target[1].ctrl_flags &
0964                     NVM_ISCSI_CFG_TARGET_ENABLED);
0965     if (sec_ctrl_flags) {
0966         sec_tgt = kzalloc(sizeof(*sec_tgt), GFP_KERNEL);
0967         if (!sec_tgt) {
0968             ret = -1;
0969             goto free_tgt;
0970         }
0971         qedi_get_boot_tgt_info(block, sec_tgt, 1);
0972     }
0973 
0974     for (i = 0; i < qedi->max_active_conns; i++) {
0975         qedi_conn = qedi_get_conn_from_id(qedi, i);
0976         if (!qedi_conn)
0977             continue;
0978 
0979         if (qedi_conn->ep->ip_type == TCP_IPV4)
0980             snprintf(ep_ip_addr, IPV4_LEN, "%pI4\n",
0981                  qedi_conn->ep->dst_addr);
0982         else
0983             snprintf(ep_ip_addr, IPV6_LEN, "%pI6\n",
0984                  qedi_conn->ep->dst_addr);
0985 
0986         cls_conn = qedi_conn->cls_conn;
0987         conn = cls_conn->dd_data;
0988         cls_sess = iscsi_conn_to_session(cls_conn);
0989         sess = cls_sess->dd_data;
0990 
0991         if (!iscsi_is_session_online(cls_sess))
0992             continue;
0993 
0994         if (!sess->targetname)
0995             continue;
0996 
0997         if (pri_ctrl_flags) {
0998             if (!strcmp(pri_tgt->iscsi_name, sess->targetname) &&
0999                 !strcmp(pri_tgt->ip_addr, ep_ip_addr)) {
1000                 found = 1;
1001                 break;
1002             }
1003         }
1004 
1005         if (sec_ctrl_flags) {
1006             if (!strcmp(sec_tgt->iscsi_name, sess->targetname) &&
1007                 !strcmp(sec_tgt->ip_addr, ep_ip_addr)) {
1008                 found = 1;
1009                 break;
1010             }
1011         }
1012     }
1013 
1014     if (found) {
1015         if (conn->hdrdgst_en) {
1016             iscsi->header_digest_set = true;
1017             iscsi->header_digest = 1;
1018         }
1019 
1020         if (conn->datadgst_en) {
1021             iscsi->data_digest_set = true;
1022             iscsi->data_digest = 1;
1023         }
1024         iscsi->boot_taget_portal_set = true;
1025         iscsi->boot_taget_portal = sess->tpgt;
1026 
1027     } else {
1028         ret = -1;
1029     }
1030 
1031     if (sec_ctrl_flags)
1032         kfree(sec_tgt);
1033 free_tgt:
1034     if (pri_ctrl_flags)
1035         kfree(pri_tgt);
1036 
1037     return ret;
1038 }
1039 
1040 static void qedi_get_generic_tlv_data(void *dev, struct qed_generic_tlvs *data)
1041 {
1042     struct qedi_ctx *qedi;
1043 
1044     if (!dev) {
1045         QEDI_INFO(NULL, QEDI_LOG_EVT,
1046               "dev is NULL so ignoring get_generic_tlv_data request.\n");
1047         return;
1048     }
1049     qedi = (struct qedi_ctx *)dev;
1050 
1051     memset(data, 0, sizeof(struct qed_generic_tlvs));
1052     ether_addr_copy(data->mac[0], qedi->mac);
1053 }
1054 
1055 /*
1056  * Protocol TLV handler
1057  */
1058 static void qedi_get_protocol_tlv_data(void *dev, void *data)
1059 {
1060     struct qed_mfw_tlv_iscsi *iscsi = data;
1061     struct qed_iscsi_stats *fw_iscsi_stats;
1062     struct nvm_iscsi_block *block = NULL;
1063     u32 chap_en = 0, mchap_en = 0;
1064     struct qedi_ctx *qedi = dev;
1065     int rval = 0;
1066 
1067     fw_iscsi_stats = kmalloc(sizeof(*fw_iscsi_stats), GFP_KERNEL);
1068     if (!fw_iscsi_stats) {
1069         QEDI_ERR(&qedi->dbg_ctx,
1070              "Could not allocate memory for fw_iscsi_stats.\n");
1071         goto exit_get_data;
1072     }
1073 
1074     mutex_lock(&qedi->stats_lock);
1075     /* Query firmware for offload stats */
1076     qedi_ops->get_stats(qedi->cdev, fw_iscsi_stats);
1077     mutex_unlock(&qedi->stats_lock);
1078 
1079     iscsi->rx_frames_set = true;
1080     iscsi->rx_frames = fw_iscsi_stats->iscsi_rx_packet_cnt;
1081     iscsi->rx_bytes_set = true;
1082     iscsi->rx_bytes = fw_iscsi_stats->iscsi_rx_bytes_cnt;
1083     iscsi->tx_frames_set = true;
1084     iscsi->tx_frames = fw_iscsi_stats->iscsi_tx_packet_cnt;
1085     iscsi->tx_bytes_set = true;
1086     iscsi->tx_bytes = fw_iscsi_stats->iscsi_tx_bytes_cnt;
1087     iscsi->frame_size_set = true;
1088     iscsi->frame_size = qedi->ll2_mtu;
1089     block = qedi_get_nvram_block(qedi);
1090     if (block) {
1091         chap_en = !!(block->generic.ctrl_flags &
1092                  NVM_ISCSI_CFG_GEN_CHAP_ENABLED);
1093         mchap_en = !!(block->generic.ctrl_flags &
1094                   NVM_ISCSI_CFG_GEN_CHAP_MUTUAL_ENABLED);
1095 
1096         iscsi->auth_method_set = (chap_en || mchap_en) ? true : false;
1097         iscsi->auth_method = 1;
1098         if (chap_en)
1099             iscsi->auth_method = 2;
1100         if (mchap_en)
1101             iscsi->auth_method = 3;
1102 
1103         iscsi->tx_desc_size_set = true;
1104         iscsi->tx_desc_size = QEDI_SQ_SIZE;
1105         iscsi->rx_desc_size_set = true;
1106         iscsi->rx_desc_size = QEDI_CQ_SIZE;
1107 
1108         /* tpgt, hdr digest, data digest */
1109         rval = qedi_find_boot_info(qedi, iscsi, block);
1110         if (rval)
1111             QEDI_INFO(&qedi->dbg_ctx, QEDI_LOG_INFO,
1112                   "Boot target not set");
1113     }
1114 
1115     kfree(fw_iscsi_stats);
1116 exit_get_data:
1117     return;
1118 }
1119 
1120 void qedi_schedule_hw_err_handler(void *dev,
1121                   enum qed_hw_err_type err_type)
1122 {
1123     struct qedi_ctx *qedi = (struct qedi_ctx *)dev;
1124     unsigned long override_flags = qedi_flags_override;
1125 
1126     if (override_flags && test_bit(QEDI_ERR_OVERRIDE_EN, &override_flags))
1127         qedi->qedi_err_flags = qedi_flags_override;
1128 
1129     QEDI_INFO(&qedi->dbg_ctx, QEDI_LOG_INFO,
1130           "HW error handler scheduled, err=%d err_flags=0x%x\n",
1131           err_type, qedi->qedi_err_flags);
1132 
1133     switch (err_type) {
1134     case QED_HW_ERR_FAN_FAIL:
1135         schedule_delayed_work(&qedi->board_disable_work, 0);
1136         break;
1137     case QED_HW_ERR_MFW_RESP_FAIL:
1138     case QED_HW_ERR_HW_ATTN:
1139     case QED_HW_ERR_DMAE_FAIL:
1140     case QED_HW_ERR_RAMROD_FAIL:
1141     case QED_HW_ERR_FW_ASSERT:
1142         /* Prevent HW attentions from being reasserted */
1143         if (test_bit(QEDI_ERR_ATTN_CLR_EN, &qedi->qedi_err_flags))
1144             qedi_ops->common->attn_clr_enable(qedi->cdev, true);
1145 
1146         if (err_type == QED_HW_ERR_RAMROD_FAIL &&
1147             test_bit(QEDI_ERR_IS_RECOVERABLE, &qedi->qedi_err_flags))
1148             qedi_ops->common->recovery_process(qedi->cdev);
1149 
1150         break;
1151     default:
1152         break;
1153     }
1154 }
1155 
1156 static void qedi_schedule_recovery_handler(void *dev)
1157 {
1158     struct qedi_ctx *qedi = dev;
1159 
1160     QEDI_ERR(&qedi->dbg_ctx, "Recovery handler scheduled.\n");
1161 
1162     if (test_and_set_bit(QEDI_IN_RECOVERY, &qedi->flags))
1163         return;
1164 
1165     atomic_set(&qedi->link_state, QEDI_LINK_DOWN);
1166 
1167     schedule_delayed_work(&qedi->recovery_work, 0);
1168 }
1169 
1170 static void qedi_set_conn_recovery(struct iscsi_cls_session *cls_session)
1171 {
1172     struct iscsi_session *session = cls_session->dd_data;
1173     struct iscsi_conn *conn = session->leadconn;
1174     struct qedi_conn *qedi_conn = conn->dd_data;
1175 
1176     qedi_start_conn_recovery(qedi_conn->qedi, qedi_conn);
1177 }
1178 
1179 static void qedi_link_update(void *dev, struct qed_link_output *link)
1180 {
1181     struct qedi_ctx *qedi = (struct qedi_ctx *)dev;
1182 
1183     if (link->link_up) {
1184         QEDI_INFO(&qedi->dbg_ctx, QEDI_LOG_INFO, "Link Up event.\n");
1185         atomic_set(&qedi->link_state, QEDI_LINK_UP);
1186     } else {
1187         QEDI_INFO(&qedi->dbg_ctx, QEDI_LOG_INFO,
1188               "Link Down event.\n");
1189         atomic_set(&qedi->link_state, QEDI_LINK_DOWN);
1190         iscsi_host_for_each_session(qedi->shost, qedi_set_conn_recovery);
1191     }
1192 }
1193 
1194 static struct qed_iscsi_cb_ops qedi_cb_ops = {
1195     {
1196         .link_update =      qedi_link_update,
1197         .schedule_recovery_handler = qedi_schedule_recovery_handler,
1198         .schedule_hw_err_handler = qedi_schedule_hw_err_handler,
1199         .get_protocol_tlv_data = qedi_get_protocol_tlv_data,
1200         .get_generic_tlv_data = qedi_get_generic_tlv_data,
1201     }
1202 };
1203 
1204 static int qedi_queue_cqe(struct qedi_ctx *qedi, union iscsi_cqe *cqe,
1205               u16 que_idx, struct qedi_percpu_s *p)
1206 {
1207     struct qedi_work *qedi_work;
1208     struct qedi_conn *q_conn;
1209     struct qedi_cmd *qedi_cmd;
1210     u32 iscsi_cid;
1211     int rc = 0;
1212 
1213     iscsi_cid  = cqe->cqe_common.conn_id;
1214     q_conn = qedi->cid_que.conn_cid_tbl[iscsi_cid];
1215     if (!q_conn) {
1216         QEDI_WARN(&qedi->dbg_ctx,
1217               "Session no longer exists for cid=0x%x!!\n",
1218               iscsi_cid);
1219         return -1;
1220     }
1221 
1222     switch (cqe->cqe_common.cqe_type) {
1223     case ISCSI_CQE_TYPE_SOLICITED:
1224     case ISCSI_CQE_TYPE_SOLICITED_WITH_SENSE:
1225         qedi_cmd = qedi_get_cmd_from_tid(qedi, cqe->cqe_solicited.itid);
1226         if (!qedi_cmd) {
1227             rc = -1;
1228             break;
1229         }
1230         INIT_LIST_HEAD(&qedi_cmd->cqe_work.list);
1231         qedi_cmd->cqe_work.qedi = qedi;
1232         memcpy(&qedi_cmd->cqe_work.cqe, cqe, sizeof(union iscsi_cqe));
1233         qedi_cmd->cqe_work.que_idx = que_idx;
1234         qedi_cmd->cqe_work.is_solicited = true;
1235         list_add_tail(&qedi_cmd->cqe_work.list, &p->work_list);
1236         break;
1237     case ISCSI_CQE_TYPE_UNSOLICITED:
1238     case ISCSI_CQE_TYPE_DUMMY:
1239     case ISCSI_CQE_TYPE_TASK_CLEANUP:
1240         qedi_work = kzalloc(sizeof(*qedi_work), GFP_ATOMIC);
1241         if (!qedi_work) {
1242             rc = -1;
1243             break;
1244         }
1245         INIT_LIST_HEAD(&qedi_work->list);
1246         qedi_work->qedi = qedi;
1247         memcpy(&qedi_work->cqe, cqe, sizeof(union iscsi_cqe));
1248         qedi_work->que_idx = que_idx;
1249         qedi_work->is_solicited = false;
1250         list_add_tail(&qedi_work->list, &p->work_list);
1251         break;
1252     default:
1253         rc = -1;
1254         QEDI_ERR(&qedi->dbg_ctx, "FW Error cqe.\n");
1255     }
1256     return rc;
1257 }
1258 
1259 static bool qedi_process_completions(struct qedi_fastpath *fp)
1260 {
1261     struct qedi_ctx *qedi = fp->qedi;
1262     struct qed_sb_info *sb_info = fp->sb_info;
1263     struct status_block *sb = sb_info->sb_virt;
1264     struct qedi_percpu_s *p = NULL;
1265     struct global_queue *que;
1266     u16 prod_idx;
1267     unsigned long flags;
1268     union iscsi_cqe *cqe;
1269     int cpu;
1270     int ret;
1271 
1272     /* Get the current firmware producer index */
1273     prod_idx = sb->pi_array[QEDI_PROTO_CQ_PROD_IDX];
1274 
1275     if (prod_idx >= QEDI_CQ_SIZE)
1276         prod_idx = prod_idx % QEDI_CQ_SIZE;
1277 
1278     que = qedi->global_queues[fp->sb_id];
1279     QEDI_INFO(&qedi->dbg_ctx, QEDI_LOG_IO,
1280           "Before: global queue=%p prod_idx=%d cons_idx=%d, sb_id=%d\n",
1281           que, prod_idx, que->cq_cons_idx, fp->sb_id);
1282 
1283     qedi->intr_cpu = fp->sb_id;
1284     cpu = smp_processor_id();
1285     p = &per_cpu(qedi_percpu, cpu);
1286 
1287     if (unlikely(!p->iothread))
1288         WARN_ON(1);
1289 
1290     spin_lock_irqsave(&p->p_work_lock, flags);
1291     while (que->cq_cons_idx != prod_idx) {
1292         cqe = &que->cq[que->cq_cons_idx];
1293 
1294         QEDI_INFO(&qedi->dbg_ctx, QEDI_LOG_IO,
1295               "cqe=%p prod_idx=%d cons_idx=%d.\n",
1296               cqe, prod_idx, que->cq_cons_idx);
1297 
1298         ret = qedi_queue_cqe(qedi, cqe, fp->sb_id, p);
1299         if (ret)
1300             QEDI_WARN(&qedi->dbg_ctx,
1301                   "Dropping CQE 0x%x for cid=0x%x.\n",
1302                   que->cq_cons_idx, cqe->cqe_common.conn_id);
1303 
1304         que->cq_cons_idx++;
1305         if (que->cq_cons_idx == QEDI_CQ_SIZE)
1306             que->cq_cons_idx = 0;
1307     }
1308     wake_up_process(p->iothread);
1309     spin_unlock_irqrestore(&p->p_work_lock, flags);
1310 
1311     return true;
1312 }
1313 
1314 static bool qedi_fp_has_work(struct qedi_fastpath *fp)
1315 {
1316     struct qedi_ctx *qedi = fp->qedi;
1317     struct global_queue *que;
1318     struct qed_sb_info *sb_info = fp->sb_info;
1319     struct status_block *sb = sb_info->sb_virt;
1320     u16 prod_idx;
1321 
1322     barrier();
1323 
1324     /* Get the current firmware producer index */
1325     prod_idx = sb->pi_array[QEDI_PROTO_CQ_PROD_IDX];
1326 
1327     /* Get the pointer to the global CQ this completion is on */
1328     que = qedi->global_queues[fp->sb_id];
1329 
1330     /* prod idx wrap around uint16 */
1331     if (prod_idx >= QEDI_CQ_SIZE)
1332         prod_idx = prod_idx % QEDI_CQ_SIZE;
1333 
1334     return (que->cq_cons_idx != prod_idx);
1335 }
1336 
1337 /* MSI-X fastpath handler code */
1338 static irqreturn_t qedi_msix_handler(int irq, void *dev_id)
1339 {
1340     struct qedi_fastpath *fp = dev_id;
1341     struct qedi_ctx *qedi = fp->qedi;
1342     bool wake_io_thread = true;
1343 
1344     qed_sb_ack(fp->sb_info, IGU_INT_DISABLE, 0);
1345 
1346 process_again:
1347     wake_io_thread = qedi_process_completions(fp);
1348     if (wake_io_thread) {
1349         QEDI_INFO(&qedi->dbg_ctx, QEDI_LOG_DISC,
1350               "process already running\n");
1351     }
1352 
1353     if (!qedi_fp_has_work(fp))
1354         qed_sb_update_sb_idx(fp->sb_info);
1355 
1356     /* Check for more work */
1357     rmb();
1358 
1359     if (!qedi_fp_has_work(fp))
1360         qed_sb_ack(fp->sb_info, IGU_INT_ENABLE, 1);
1361     else
1362         goto process_again;
1363 
1364     return IRQ_HANDLED;
1365 }
1366 
1367 /* simd handler for MSI/INTa */
1368 static void qedi_simd_int_handler(void *cookie)
1369 {
1370     /* Cookie is qedi_ctx struct */
1371     struct qedi_ctx *qedi = (struct qedi_ctx *)cookie;
1372 
1373     QEDI_WARN(&qedi->dbg_ctx, "qedi=%p.\n", qedi);
1374 }
1375 
1376 #define QEDI_SIMD_HANDLER_NUM       0
1377 static void qedi_sync_free_irqs(struct qedi_ctx *qedi)
1378 {
1379     int i;
1380     u16 idx;
1381 
1382     if (qedi->int_info.msix_cnt) {
1383         for (i = 0; i < qedi->int_info.used_cnt; i++) {
1384             idx = i * qedi->dev_info.common.num_hwfns +
1385             qedi_ops->common->get_affin_hwfn_idx(qedi->cdev);
1386 
1387             QEDI_INFO(&qedi->dbg_ctx, QEDI_LOG_INFO,
1388                   "Freeing IRQ #%d vector_idx=%d.\n", i, idx);
1389 
1390             synchronize_irq(qedi->int_info.msix[idx].vector);
1391             irq_set_affinity_hint(qedi->int_info.msix[idx].vector,
1392                           NULL);
1393             free_irq(qedi->int_info.msix[idx].vector,
1394                  &qedi->fp_array[i]);
1395         }
1396     } else {
1397         qedi_ops->common->simd_handler_clean(qedi->cdev,
1398                              QEDI_SIMD_HANDLER_NUM);
1399     }
1400 
1401     qedi->int_info.used_cnt = 0;
1402     qedi_ops->common->set_fp_int(qedi->cdev, 0);
1403 }
1404 
1405 static int qedi_request_msix_irq(struct qedi_ctx *qedi)
1406 {
1407     int i, rc, cpu;
1408     u16 idx;
1409 
1410     cpu = cpumask_first(cpu_online_mask);
1411     for (i = 0; i < qedi->msix_count; i++) {
1412         idx = i * qedi->dev_info.common.num_hwfns +
1413               qedi_ops->common->get_affin_hwfn_idx(qedi->cdev);
1414 
1415         QEDI_INFO(&qedi->dbg_ctx, QEDI_LOG_INFO,
1416               "dev_info: num_hwfns=%d affin_hwfn_idx=%d.\n",
1417               qedi->dev_info.common.num_hwfns,
1418               qedi_ops->common->get_affin_hwfn_idx(qedi->cdev));
1419 
1420         rc = request_irq(qedi->int_info.msix[idx].vector,
1421                  qedi_msix_handler, 0, "qedi",
1422                  &qedi->fp_array[i]);
1423         if (rc) {
1424             QEDI_WARN(&qedi->dbg_ctx, "request_irq failed.\n");
1425             qedi_sync_free_irqs(qedi);
1426             return rc;
1427         }
1428         qedi->int_info.used_cnt++;
1429         rc = irq_set_affinity_hint(qedi->int_info.msix[idx].vector,
1430                        get_cpu_mask(cpu));
1431         cpu = cpumask_next(cpu, cpu_online_mask);
1432     }
1433 
1434     return 0;
1435 }
1436 
1437 static int qedi_setup_int(struct qedi_ctx *qedi)
1438 {
1439     int rc = 0;
1440 
1441     rc = qedi_ops->common->set_fp_int(qedi->cdev, qedi->num_queues);
1442     if (rc < 0)
1443         goto exit_setup_int;
1444 
1445     qedi->msix_count = rc;
1446 
1447     rc = qedi_ops->common->get_fp_int(qedi->cdev, &qedi->int_info);
1448     if (rc)
1449         goto exit_setup_int;
1450 
1451     QEDI_INFO(&qedi->dbg_ctx, QEDI_LOG_DISC,
1452           "Number of msix_cnt = 0x%x num of cpus = 0x%x\n",
1453            qedi->int_info.msix_cnt, num_online_cpus());
1454 
1455     if (qedi->int_info.msix_cnt) {
1456         rc = qedi_request_msix_irq(qedi);
1457         goto exit_setup_int;
1458     } else {
1459         qedi_ops->common->simd_handler_config(qedi->cdev, &qedi,
1460                               QEDI_SIMD_HANDLER_NUM,
1461                               qedi_simd_int_handler);
1462         qedi->int_info.used_cnt = 1;
1463     }
1464 
1465 exit_setup_int:
1466     return rc;
1467 }
1468 
1469 static void qedi_free_nvm_iscsi_cfg(struct qedi_ctx *qedi)
1470 {
1471     if (qedi->iscsi_image)
1472         dma_free_coherent(&qedi->pdev->dev,
1473                   sizeof(struct qedi_nvm_iscsi_image),
1474                   qedi->iscsi_image, qedi->nvm_buf_dma);
1475 }
1476 
1477 static int qedi_alloc_nvm_iscsi_cfg(struct qedi_ctx *qedi)
1478 {
1479     qedi->iscsi_image = dma_alloc_coherent(&qedi->pdev->dev,
1480                            sizeof(struct qedi_nvm_iscsi_image),
1481                            &qedi->nvm_buf_dma, GFP_KERNEL);
1482     if (!qedi->iscsi_image) {
1483         QEDI_ERR(&qedi->dbg_ctx, "Could not allocate NVM BUF.\n");
1484         return -ENOMEM;
1485     }
1486     QEDI_INFO(&qedi->dbg_ctx, QEDI_LOG_INFO,
1487           "NVM BUF addr=0x%p dma=0x%llx.\n", qedi->iscsi_image,
1488           qedi->nvm_buf_dma);
1489 
1490     return 0;
1491 }
1492 
1493 static void qedi_free_bdq(struct qedi_ctx *qedi)
1494 {
1495     int i;
1496 
1497     if (qedi->bdq_pbl_list)
1498         dma_free_coherent(&qedi->pdev->dev, QEDI_PAGE_SIZE,
1499                   qedi->bdq_pbl_list, qedi->bdq_pbl_list_dma);
1500 
1501     if (qedi->bdq_pbl)
1502         dma_free_coherent(&qedi->pdev->dev, qedi->bdq_pbl_mem_size,
1503                   qedi->bdq_pbl, qedi->bdq_pbl_dma);
1504 
1505     for (i = 0; i < QEDI_BDQ_NUM; i++) {
1506         if (qedi->bdq[i].buf_addr) {
1507             dma_free_coherent(&qedi->pdev->dev, QEDI_BDQ_BUF_SIZE,
1508                       qedi->bdq[i].buf_addr,
1509                       qedi->bdq[i].buf_dma);
1510         }
1511     }
1512 }
1513 
1514 static void qedi_free_global_queues(struct qedi_ctx *qedi)
1515 {
1516     int i;
1517     struct global_queue **gl = qedi->global_queues;
1518 
1519     for (i = 0; i < qedi->num_queues; i++) {
1520         if (!gl[i])
1521             continue;
1522 
1523         if (gl[i]->cq)
1524             dma_free_coherent(&qedi->pdev->dev, gl[i]->cq_mem_size,
1525                       gl[i]->cq, gl[i]->cq_dma);
1526         if (gl[i]->cq_pbl)
1527             dma_free_coherent(&qedi->pdev->dev, gl[i]->cq_pbl_size,
1528                       gl[i]->cq_pbl, gl[i]->cq_pbl_dma);
1529 
1530         kfree(gl[i]);
1531     }
1532     qedi_free_bdq(qedi);
1533     qedi_free_nvm_iscsi_cfg(qedi);
1534 }
1535 
1536 static int qedi_alloc_bdq(struct qedi_ctx *qedi)
1537 {
1538     int i;
1539     struct scsi_bd *pbl;
1540     u64 *list;
1541 
1542     /* Alloc dma memory for BDQ buffers */
1543     for (i = 0; i < QEDI_BDQ_NUM; i++) {
1544         qedi->bdq[i].buf_addr =
1545                 dma_alloc_coherent(&qedi->pdev->dev,
1546                            QEDI_BDQ_BUF_SIZE,
1547                            &qedi->bdq[i].buf_dma,
1548                            GFP_KERNEL);
1549         if (!qedi->bdq[i].buf_addr) {
1550             QEDI_ERR(&qedi->dbg_ctx,
1551                  "Could not allocate BDQ buffer %d.\n", i);
1552             return -ENOMEM;
1553         }
1554     }
1555 
1556     /* Alloc dma memory for BDQ page buffer list */
1557     qedi->bdq_pbl_mem_size = QEDI_BDQ_NUM * sizeof(struct scsi_bd);
1558     qedi->bdq_pbl_mem_size = ALIGN(qedi->bdq_pbl_mem_size, QEDI_PAGE_SIZE);
1559     qedi->rq_num_entries = qedi->bdq_pbl_mem_size / sizeof(struct scsi_bd);
1560 
1561     QEDI_INFO(&qedi->dbg_ctx, QEDI_LOG_CONN, "rq_num_entries = %d.\n",
1562           qedi->rq_num_entries);
1563 
1564     qedi->bdq_pbl = dma_alloc_coherent(&qedi->pdev->dev,
1565                        qedi->bdq_pbl_mem_size,
1566                        &qedi->bdq_pbl_dma, GFP_KERNEL);
1567     if (!qedi->bdq_pbl) {
1568         QEDI_ERR(&qedi->dbg_ctx, "Could not allocate BDQ PBL.\n");
1569         return -ENOMEM;
1570     }
1571 
1572     /*
1573      * Populate BDQ PBL with physical and virtual address of individual
1574      * BDQ buffers
1575      */
1576     pbl = (struct scsi_bd  *)qedi->bdq_pbl;
1577     for (i = 0; i < QEDI_BDQ_NUM; i++) {
1578         pbl->address.hi =
1579                 cpu_to_le32(QEDI_U64_HI(qedi->bdq[i].buf_dma));
1580         pbl->address.lo =
1581                 cpu_to_le32(QEDI_U64_LO(qedi->bdq[i].buf_dma));
1582         QEDI_INFO(&qedi->dbg_ctx, QEDI_LOG_CONN,
1583               "pbl [0x%p] pbl->address hi [0x%llx] lo [0x%llx], idx [%d]\n",
1584               pbl, pbl->address.hi, pbl->address.lo, i);
1585         pbl->opaque.iscsi_opaque.reserved_zero[0] = 0;
1586         pbl->opaque.iscsi_opaque.reserved_zero[1] = 0;
1587         pbl->opaque.iscsi_opaque.reserved_zero[2] = 0;
1588         pbl->opaque.iscsi_opaque.opaque = cpu_to_le16(i);
1589         pbl++;
1590     }
1591 
1592     /* Allocate list of PBL pages */
1593     qedi->bdq_pbl_list = dma_alloc_coherent(&qedi->pdev->dev,
1594                         QEDI_PAGE_SIZE,
1595                         &qedi->bdq_pbl_list_dma,
1596                         GFP_KERNEL);
1597     if (!qedi->bdq_pbl_list) {
1598         QEDI_ERR(&qedi->dbg_ctx,
1599              "Could not allocate list of PBL pages.\n");
1600         return -ENOMEM;
1601     }
1602 
1603     /*
1604      * Now populate PBL list with pages that contain pointers to the
1605      * individual buffers.
1606      */
1607     qedi->bdq_pbl_list_num_entries = qedi->bdq_pbl_mem_size /
1608                      QEDI_PAGE_SIZE;
1609     list = (u64 *)qedi->bdq_pbl_list;
1610     for (i = 0; i < qedi->bdq_pbl_list_num_entries; i++) {
1611         *list = qedi->bdq_pbl_dma;
1612         list++;
1613     }
1614 
1615     return 0;
1616 }
1617 
1618 static int qedi_alloc_global_queues(struct qedi_ctx *qedi)
1619 {
1620     u32 *list;
1621     int i;
1622     int status;
1623     u32 *pbl;
1624     dma_addr_t page;
1625     int num_pages;
1626 
1627     /*
1628      * Number of global queues (CQ / RQ). This should
1629      * be <= number of available MSIX vectors for the PF
1630      */
1631     if (!qedi->num_queues) {
1632         QEDI_ERR(&qedi->dbg_ctx, "No MSI-X vectors available!\n");
1633         return -ENOMEM;
1634     }
1635 
1636     /* Make sure we allocated the PBL that will contain the physical
1637      * addresses of our queues
1638      */
1639     if (!qedi->p_cpuq) {
1640         status = -EINVAL;
1641         goto mem_alloc_failure;
1642     }
1643 
1644     qedi->global_queues = kzalloc((sizeof(struct global_queue *) *
1645                        qedi->num_queues), GFP_KERNEL);
1646     if (!qedi->global_queues) {
1647         QEDI_ERR(&qedi->dbg_ctx,
1648              "Unable to allocate global queues array ptr memory\n");
1649         return -ENOMEM;
1650     }
1651     QEDI_INFO(&qedi->dbg_ctx, QEDI_LOG_DISC,
1652           "qedi->global_queues=%p.\n", qedi->global_queues);
1653 
1654     /* Allocate DMA coherent buffers for BDQ */
1655     status = qedi_alloc_bdq(qedi);
1656     if (status)
1657         goto mem_alloc_failure;
1658 
1659     /* Allocate DMA coherent buffers for NVM_ISCSI_CFG */
1660     status = qedi_alloc_nvm_iscsi_cfg(qedi);
1661     if (status)
1662         goto mem_alloc_failure;
1663 
1664     /* Allocate a CQ and an associated PBL for each MSI-X
1665      * vector.
1666      */
1667     for (i = 0; i < qedi->num_queues; i++) {
1668         qedi->global_queues[i] =
1669                     kzalloc(sizeof(*qedi->global_queues[0]),
1670                         GFP_KERNEL);
1671         if (!qedi->global_queues[i]) {
1672             QEDI_ERR(&qedi->dbg_ctx,
1673                  "Unable to allocation global queue %d.\n", i);
1674             status = -ENOMEM;
1675             goto mem_alloc_failure;
1676         }
1677 
1678         qedi->global_queues[i]->cq_mem_size =
1679             (QEDI_CQ_SIZE + 8) * sizeof(union iscsi_cqe);
1680         qedi->global_queues[i]->cq_mem_size =
1681             (qedi->global_queues[i]->cq_mem_size +
1682             (QEDI_PAGE_SIZE - 1));
1683 
1684         qedi->global_queues[i]->cq_pbl_size =
1685             (qedi->global_queues[i]->cq_mem_size /
1686             QEDI_PAGE_SIZE) * sizeof(void *);
1687         qedi->global_queues[i]->cq_pbl_size =
1688             (qedi->global_queues[i]->cq_pbl_size +
1689             (QEDI_PAGE_SIZE - 1));
1690 
1691         qedi->global_queues[i]->cq = dma_alloc_coherent(&qedi->pdev->dev,
1692                                 qedi->global_queues[i]->cq_mem_size,
1693                                 &qedi->global_queues[i]->cq_dma,
1694                                 GFP_KERNEL);
1695 
1696         if (!qedi->global_queues[i]->cq) {
1697             QEDI_WARN(&qedi->dbg_ctx,
1698                   "Could not allocate cq.\n");
1699             status = -ENOMEM;
1700             goto mem_alloc_failure;
1701         }
1702         qedi->global_queues[i]->cq_pbl = dma_alloc_coherent(&qedi->pdev->dev,
1703                                     qedi->global_queues[i]->cq_pbl_size,
1704                                     &qedi->global_queues[i]->cq_pbl_dma,
1705                                     GFP_KERNEL);
1706 
1707         if (!qedi->global_queues[i]->cq_pbl) {
1708             QEDI_WARN(&qedi->dbg_ctx,
1709                   "Could not allocate cq PBL.\n");
1710             status = -ENOMEM;
1711             goto mem_alloc_failure;
1712         }
1713 
1714         /* Create PBL */
1715         num_pages = qedi->global_queues[i]->cq_mem_size /
1716             QEDI_PAGE_SIZE;
1717         page = qedi->global_queues[i]->cq_dma;
1718         pbl = (u32 *)qedi->global_queues[i]->cq_pbl;
1719 
1720         while (num_pages--) {
1721             *pbl = (u32)page;
1722             pbl++;
1723             *pbl = (u32)((u64)page >> 32);
1724             pbl++;
1725             page += QEDI_PAGE_SIZE;
1726         }
1727     }
1728 
1729     list = (u32 *)qedi->p_cpuq;
1730 
1731     /*
1732      * The list is built as follows: CQ#0 PBL pointer, RQ#0 PBL pointer,
1733      * CQ#1 PBL pointer, RQ#1 PBL pointer, etc.  Each PBL pointer points
1734      * to the physical address which contains an array of pointers to the
1735      * physical addresses of the specific queue pages.
1736      */
1737     for (i = 0; i < qedi->num_queues; i++) {
1738         *list = (u32)qedi->global_queues[i]->cq_pbl_dma;
1739         list++;
1740         *list = (u32)((u64)qedi->global_queues[i]->cq_pbl_dma >> 32);
1741         list++;
1742 
1743         *list = (u32)0;
1744         list++;
1745         *list = (u32)((u64)0 >> 32);
1746         list++;
1747     }
1748 
1749     return 0;
1750 
1751 mem_alloc_failure:
1752     qedi_free_global_queues(qedi);
1753     return status;
1754 }
1755 
1756 int qedi_alloc_sq(struct qedi_ctx *qedi, struct qedi_endpoint *ep)
1757 {
1758     int rval = 0;
1759     u32 *pbl;
1760     dma_addr_t page;
1761     int num_pages;
1762 
1763     if (!ep)
1764         return -EIO;
1765 
1766     /* Calculate appropriate queue and PBL sizes */
1767     ep->sq_mem_size = QEDI_SQ_SIZE * sizeof(struct iscsi_wqe);
1768     ep->sq_mem_size += QEDI_PAGE_SIZE - 1;
1769 
1770     ep->sq_pbl_size = (ep->sq_mem_size / QEDI_PAGE_SIZE) * sizeof(void *);
1771     ep->sq_pbl_size = ep->sq_pbl_size + QEDI_PAGE_SIZE;
1772 
1773     ep->sq = dma_alloc_coherent(&qedi->pdev->dev, ep->sq_mem_size,
1774                     &ep->sq_dma, GFP_KERNEL);
1775     if (!ep->sq) {
1776         QEDI_WARN(&qedi->dbg_ctx,
1777               "Could not allocate send queue.\n");
1778         rval = -ENOMEM;
1779         goto out;
1780     }
1781     ep->sq_pbl = dma_alloc_coherent(&qedi->pdev->dev, ep->sq_pbl_size,
1782                     &ep->sq_pbl_dma, GFP_KERNEL);
1783     if (!ep->sq_pbl) {
1784         QEDI_WARN(&qedi->dbg_ctx,
1785               "Could not allocate send queue PBL.\n");
1786         rval = -ENOMEM;
1787         goto out_free_sq;
1788     }
1789 
1790     /* Create PBL */
1791     num_pages = ep->sq_mem_size / QEDI_PAGE_SIZE;
1792     page = ep->sq_dma;
1793     pbl = (u32 *)ep->sq_pbl;
1794 
1795     while (num_pages--) {
1796         *pbl = (u32)page;
1797         pbl++;
1798         *pbl = (u32)((u64)page >> 32);
1799         pbl++;
1800         page += QEDI_PAGE_SIZE;
1801     }
1802 
1803     return rval;
1804 
1805 out_free_sq:
1806     dma_free_coherent(&qedi->pdev->dev, ep->sq_mem_size, ep->sq,
1807               ep->sq_dma);
1808 out:
1809     return rval;
1810 }
1811 
1812 void qedi_free_sq(struct qedi_ctx *qedi, struct qedi_endpoint *ep)
1813 {
1814     if (ep->sq_pbl)
1815         dma_free_coherent(&qedi->pdev->dev, ep->sq_pbl_size, ep->sq_pbl,
1816                   ep->sq_pbl_dma);
1817     if (ep->sq)
1818         dma_free_coherent(&qedi->pdev->dev, ep->sq_mem_size, ep->sq,
1819                   ep->sq_dma);
1820 }
1821 
1822 int qedi_get_task_idx(struct qedi_ctx *qedi)
1823 {
1824     s16 tmp_idx;
1825 
1826 again:
1827     tmp_idx = find_first_zero_bit(qedi->task_idx_map,
1828                       MAX_ISCSI_TASK_ENTRIES);
1829 
1830     if (tmp_idx >= MAX_ISCSI_TASK_ENTRIES) {
1831         QEDI_ERR(&qedi->dbg_ctx, "FW task context pool is full.\n");
1832         tmp_idx = -1;
1833         goto err_idx;
1834     }
1835 
1836     if (test_and_set_bit(tmp_idx, qedi->task_idx_map))
1837         goto again;
1838 
1839 err_idx:
1840     return tmp_idx;
1841 }
1842 
1843 void qedi_clear_task_idx(struct qedi_ctx *qedi, int idx)
1844 {
1845     if (!test_and_clear_bit(idx, qedi->task_idx_map))
1846         QEDI_ERR(&qedi->dbg_ctx,
1847              "FW task context, already cleared, tid=0x%x\n", idx);
1848 }
1849 
1850 void qedi_update_itt_map(struct qedi_ctx *qedi, u32 tid, u32 proto_itt,
1851              struct qedi_cmd *cmd)
1852 {
1853     qedi->itt_map[tid].itt = proto_itt;
1854     qedi->itt_map[tid].p_cmd = cmd;
1855 
1856     QEDI_INFO(&qedi->dbg_ctx, QEDI_LOG_CONN,
1857           "update itt map tid=0x%x, with proto itt=0x%x\n", tid,
1858           qedi->itt_map[tid].itt);
1859 }
1860 
1861 void qedi_get_task_tid(struct qedi_ctx *qedi, u32 itt, s16 *tid)
1862 {
1863     u16 i;
1864 
1865     for (i = 0; i < MAX_ISCSI_TASK_ENTRIES; i++) {
1866         if (qedi->itt_map[i].itt == itt) {
1867             *tid = i;
1868             QEDI_INFO(&qedi->dbg_ctx, QEDI_LOG_CONN,
1869                   "Ref itt=0x%x, found at tid=0x%x\n",
1870                   itt, *tid);
1871             return;
1872         }
1873     }
1874 
1875     WARN_ON(1);
1876 }
1877 
1878 void qedi_get_proto_itt(struct qedi_ctx *qedi, u32 tid, u32 *proto_itt)
1879 {
1880     *proto_itt = qedi->itt_map[tid].itt;
1881     QEDI_INFO(&qedi->dbg_ctx, QEDI_LOG_CONN,
1882           "Get itt map tid [0x%x with proto itt[0x%x]",
1883           tid, *proto_itt);
1884 }
1885 
1886 struct qedi_cmd *qedi_get_cmd_from_tid(struct qedi_ctx *qedi, u32 tid)
1887 {
1888     struct qedi_cmd *cmd = NULL;
1889 
1890     if (tid >= MAX_ISCSI_TASK_ENTRIES)
1891         return NULL;
1892 
1893     cmd = qedi->itt_map[tid].p_cmd;
1894     if (cmd->task_id != tid)
1895         return NULL;
1896 
1897     qedi->itt_map[tid].p_cmd = NULL;
1898 
1899     return cmd;
1900 }
1901 
1902 static int qedi_alloc_itt(struct qedi_ctx *qedi)
1903 {
1904     qedi->itt_map = kcalloc(MAX_ISCSI_TASK_ENTRIES,
1905                 sizeof(struct qedi_itt_map), GFP_KERNEL);
1906     if (!qedi->itt_map) {
1907         QEDI_ERR(&qedi->dbg_ctx,
1908              "Unable to allocate itt map array memory\n");
1909         return -ENOMEM;
1910     }
1911     return 0;
1912 }
1913 
1914 static void qedi_free_itt(struct qedi_ctx *qedi)
1915 {
1916     kfree(qedi->itt_map);
1917 }
1918 
1919 static struct qed_ll2_cb_ops qedi_ll2_cb_ops = {
1920     .rx_cb = qedi_ll2_rx,
1921     .tx_cb = NULL,
1922 };
1923 
1924 static int qedi_percpu_io_thread(void *arg)
1925 {
1926     struct qedi_percpu_s *p = arg;
1927     struct qedi_work *work, *tmp;
1928     unsigned long flags;
1929     LIST_HEAD(work_list);
1930 
1931     set_user_nice(current, -20);
1932 
1933     while (!kthread_should_stop()) {
1934         spin_lock_irqsave(&p->p_work_lock, flags);
1935         while (!list_empty(&p->work_list)) {
1936             list_splice_init(&p->work_list, &work_list);
1937             spin_unlock_irqrestore(&p->p_work_lock, flags);
1938 
1939             list_for_each_entry_safe(work, tmp, &work_list, list) {
1940                 list_del_init(&work->list);
1941                 qedi_fp_process_cqes(work);
1942                 if (!work->is_solicited)
1943                     kfree(work);
1944             }
1945             cond_resched();
1946             spin_lock_irqsave(&p->p_work_lock, flags);
1947         }
1948         set_current_state(TASK_INTERRUPTIBLE);
1949         spin_unlock_irqrestore(&p->p_work_lock, flags);
1950         schedule();
1951     }
1952     __set_current_state(TASK_RUNNING);
1953 
1954     return 0;
1955 }
1956 
1957 static int qedi_cpu_online(unsigned int cpu)
1958 {
1959     struct qedi_percpu_s *p = this_cpu_ptr(&qedi_percpu);
1960     struct task_struct *thread;
1961 
1962     thread = kthread_create_on_node(qedi_percpu_io_thread, (void *)p,
1963                     cpu_to_node(cpu),
1964                     "qedi_thread/%d", cpu);
1965     if (IS_ERR(thread))
1966         return PTR_ERR(thread);
1967 
1968     kthread_bind(thread, cpu);
1969     p->iothread = thread;
1970     wake_up_process(thread);
1971     return 0;
1972 }
1973 
1974 static int qedi_cpu_offline(unsigned int cpu)
1975 {
1976     struct qedi_percpu_s *p = this_cpu_ptr(&qedi_percpu);
1977     struct qedi_work *work, *tmp;
1978     struct task_struct *thread;
1979 
1980     spin_lock_bh(&p->p_work_lock);
1981     thread = p->iothread;
1982     p->iothread = NULL;
1983 
1984     list_for_each_entry_safe(work, tmp, &p->work_list, list) {
1985         list_del_init(&work->list);
1986         qedi_fp_process_cqes(work);
1987         if (!work->is_solicited)
1988             kfree(work);
1989     }
1990 
1991     spin_unlock_bh(&p->p_work_lock);
1992     if (thread)
1993         kthread_stop(thread);
1994     return 0;
1995 }
1996 
1997 void qedi_reset_host_mtu(struct qedi_ctx *qedi, u16 mtu)
1998 {
1999     struct qed_ll2_params params;
2000 
2001     qedi_recover_all_conns(qedi);
2002 
2003     qedi_ops->ll2->stop(qedi->cdev);
2004     qedi_ll2_free_skbs(qedi);
2005 
2006     QEDI_INFO(&qedi->dbg_ctx, QEDI_LOG_INFO, "old MTU %u, new MTU %u\n",
2007           qedi->ll2_mtu, mtu);
2008     memset(&params, 0, sizeof(params));
2009     qedi->ll2_mtu = mtu;
2010     params.mtu = qedi->ll2_mtu + IPV6_HDR_LEN + TCP_HDR_LEN;
2011     params.drop_ttl0_packets = 0;
2012     params.rx_vlan_stripping = 1;
2013     ether_addr_copy(params.ll2_mac_address, qedi->dev_info.common.hw_mac);
2014     qedi_ops->ll2->start(qedi->cdev, &params);
2015 }
2016 
2017 /*
2018  * qedi_get_nvram_block: - Scan through the iSCSI NVRAM block (while accounting
2019  * for gaps) for the matching absolute-pf-id of the QEDI device.
2020  */
2021 static struct nvm_iscsi_block *
2022 qedi_get_nvram_block(struct qedi_ctx *qedi)
2023 {
2024     int i;
2025     u8 pf;
2026     u32 flags;
2027     struct nvm_iscsi_block *block;
2028 
2029     pf = qedi->dev_info.common.abs_pf_id;
2030     block = &qedi->iscsi_image->iscsi_cfg.block[0];
2031     for (i = 0; i < NUM_OF_ISCSI_PF_SUPPORTED; i++, block++) {
2032         flags = ((block->id) & NVM_ISCSI_CFG_BLK_CTRL_FLAG_MASK) >>
2033             NVM_ISCSI_CFG_BLK_CTRL_FLAG_OFFSET;
2034         if (flags & (NVM_ISCSI_CFG_BLK_CTRL_FLAG_IS_NOT_EMPTY |
2035                 NVM_ISCSI_CFG_BLK_CTRL_FLAG_PF_MAPPED) &&
2036             (pf == (block->id & NVM_ISCSI_CFG_BLK_MAPPED_PF_ID_MASK)
2037                 >> NVM_ISCSI_CFG_BLK_MAPPED_PF_ID_OFFSET))
2038             return block;
2039     }
2040     return NULL;
2041 }
2042 
2043 static ssize_t qedi_show_boot_eth_info(void *data, int type, char *buf)
2044 {
2045     struct qedi_ctx *qedi = data;
2046     struct nvm_iscsi_initiator *initiator;
2047     int rc = 1;
2048     u32 ipv6_en, dhcp_en, ip_len;
2049     struct nvm_iscsi_block *block;
2050     char *fmt, *ip, *sub, *gw;
2051 
2052     block = qedi_get_nvram_block(qedi);
2053     if (!block)
2054         return 0;
2055 
2056     initiator = &block->initiator;
2057     ipv6_en = block->generic.ctrl_flags &
2058           NVM_ISCSI_CFG_GEN_IPV6_ENABLED;
2059     dhcp_en = block->generic.ctrl_flags &
2060           NVM_ISCSI_CFG_GEN_DHCP_TCPIP_CONFIG_ENABLED;
2061     /* Static IP assignments. */
2062     fmt = ipv6_en ? "%pI6\n" : "%pI4\n";
2063     ip = ipv6_en ? initiator->ipv6.addr.byte : initiator->ipv4.addr.byte;
2064     ip_len = ipv6_en ? IPV6_LEN : IPV4_LEN;
2065     sub = ipv6_en ? initiator->ipv6.subnet_mask.byte :
2066           initiator->ipv4.subnet_mask.byte;
2067     gw = ipv6_en ? initiator->ipv6.gateway.byte :
2068          initiator->ipv4.gateway.byte;
2069     /* DHCP IP adjustments. */
2070     fmt = dhcp_en ? "%s\n" : fmt;
2071     if (dhcp_en) {
2072         ip = ipv6_en ? "0::0" : "0.0.0.0";
2073         sub = ip;
2074         gw = ip;
2075         ip_len = ipv6_en ? 5 : 8;
2076     }
2077 
2078     switch (type) {
2079     case ISCSI_BOOT_ETH_IP_ADDR:
2080         rc = snprintf(buf, ip_len, fmt, ip);
2081         break;
2082     case ISCSI_BOOT_ETH_SUBNET_MASK:
2083         rc = snprintf(buf, ip_len, fmt, sub);
2084         break;
2085     case ISCSI_BOOT_ETH_GATEWAY:
2086         rc = snprintf(buf, ip_len, fmt, gw);
2087         break;
2088     case ISCSI_BOOT_ETH_FLAGS:
2089         rc = snprintf(buf, 3, "%d\n", (char)SYSFS_FLAG_FW_SEL_BOOT);
2090         break;
2091     case ISCSI_BOOT_ETH_INDEX:
2092         rc = snprintf(buf, 3, "0\n");
2093         break;
2094     case ISCSI_BOOT_ETH_MAC:
2095         rc = sysfs_format_mac(buf, qedi->mac, ETH_ALEN);
2096         break;
2097     case ISCSI_BOOT_ETH_VLAN:
2098         rc = snprintf(buf, 12, "%d\n",
2099                   GET_FIELD2(initiator->generic_cont0,
2100                      NVM_ISCSI_CFG_INITIATOR_VLAN));
2101         break;
2102     case ISCSI_BOOT_ETH_ORIGIN:
2103         if (dhcp_en)
2104             rc = snprintf(buf, 3, "3\n");
2105         break;
2106     default:
2107         rc = 0;
2108         break;
2109     }
2110 
2111     return rc;
2112 }
2113 
2114 static umode_t qedi_eth_get_attr_visibility(void *data, int type)
2115 {
2116     int rc = 1;
2117 
2118     switch (type) {
2119     case ISCSI_BOOT_ETH_FLAGS:
2120     case ISCSI_BOOT_ETH_MAC:
2121     case ISCSI_BOOT_ETH_INDEX:
2122     case ISCSI_BOOT_ETH_IP_ADDR:
2123     case ISCSI_BOOT_ETH_SUBNET_MASK:
2124     case ISCSI_BOOT_ETH_GATEWAY:
2125     case ISCSI_BOOT_ETH_ORIGIN:
2126     case ISCSI_BOOT_ETH_VLAN:
2127         rc = 0444;
2128         break;
2129     default:
2130         rc = 0;
2131         break;
2132     }
2133     return rc;
2134 }
2135 
2136 static ssize_t qedi_show_boot_ini_info(void *data, int type, char *buf)
2137 {
2138     struct qedi_ctx *qedi = data;
2139     struct nvm_iscsi_initiator *initiator;
2140     int rc;
2141     struct nvm_iscsi_block *block;
2142 
2143     block = qedi_get_nvram_block(qedi);
2144     if (!block)
2145         return 0;
2146 
2147     initiator = &block->initiator;
2148 
2149     switch (type) {
2150     case ISCSI_BOOT_INI_INITIATOR_NAME:
2151         rc = sprintf(buf, "%.*s\n", NVM_ISCSI_CFG_ISCSI_NAME_MAX_LEN,
2152                  initiator->initiator_name.byte);
2153         break;
2154     default:
2155         rc = 0;
2156         break;
2157     }
2158     return rc;
2159 }
2160 
2161 static umode_t qedi_ini_get_attr_visibility(void *data, int type)
2162 {
2163     int rc;
2164 
2165     switch (type) {
2166     case ISCSI_BOOT_INI_INITIATOR_NAME:
2167         rc = 0444;
2168         break;
2169     default:
2170         rc = 0;
2171         break;
2172     }
2173     return rc;
2174 }
2175 
2176 static ssize_t
2177 qedi_show_boot_tgt_info(struct qedi_ctx *qedi, int type,
2178             char *buf, enum qedi_nvm_tgts idx)
2179 {
2180     int rc = 1;
2181     u32 ctrl_flags, ipv6_en, chap_en, mchap_en, ip_len;
2182     struct nvm_iscsi_block *block;
2183     char *chap_name, *chap_secret;
2184     char *mchap_name, *mchap_secret;
2185 
2186     block = qedi_get_nvram_block(qedi);
2187     if (!block)
2188         goto exit_show_tgt_info;
2189 
2190     QEDI_INFO(&qedi->dbg_ctx, QEDI_LOG_EVT,
2191           "Port:%d, tgt_idx:%d\n",
2192           GET_FIELD2(block->id, NVM_ISCSI_CFG_BLK_MAPPED_PF_ID), idx);
2193 
2194     ctrl_flags = block->target[idx].ctrl_flags &
2195              NVM_ISCSI_CFG_TARGET_ENABLED;
2196 
2197     if (!ctrl_flags) {
2198         QEDI_INFO(&qedi->dbg_ctx, QEDI_LOG_EVT,
2199               "Target disabled\n");
2200         goto exit_show_tgt_info;
2201     }
2202 
2203     ipv6_en = block->generic.ctrl_flags &
2204           NVM_ISCSI_CFG_GEN_IPV6_ENABLED;
2205     ip_len = ipv6_en ? IPV6_LEN : IPV4_LEN;
2206     chap_en = block->generic.ctrl_flags &
2207           NVM_ISCSI_CFG_GEN_CHAP_ENABLED;
2208     chap_name = chap_en ? block->initiator.chap_name.byte : NULL;
2209     chap_secret = chap_en ? block->initiator.chap_password.byte : NULL;
2210 
2211     mchap_en = block->generic.ctrl_flags &
2212           NVM_ISCSI_CFG_GEN_CHAP_MUTUAL_ENABLED;
2213     mchap_name = mchap_en ? block->target[idx].chap_name.byte : NULL;
2214     mchap_secret = mchap_en ? block->target[idx].chap_password.byte : NULL;
2215 
2216     switch (type) {
2217     case ISCSI_BOOT_TGT_NAME:
2218         rc = sprintf(buf, "%.*s\n", NVM_ISCSI_CFG_ISCSI_NAME_MAX_LEN,
2219                  block->target[idx].target_name.byte);
2220         break;
2221     case ISCSI_BOOT_TGT_IP_ADDR:
2222         if (ipv6_en)
2223             rc = snprintf(buf, ip_len, "%pI6\n",
2224                       block->target[idx].ipv6_addr.byte);
2225         else
2226             rc = snprintf(buf, ip_len, "%pI4\n",
2227                       block->target[idx].ipv4_addr.byte);
2228         break;
2229     case ISCSI_BOOT_TGT_PORT:
2230         rc = snprintf(buf, 12, "%d\n",
2231                   GET_FIELD2(block->target[idx].generic_cont0,
2232                      NVM_ISCSI_CFG_TARGET_TCP_PORT));
2233         break;
2234     case ISCSI_BOOT_TGT_LUN:
2235         rc = snprintf(buf, 22, "%.*d\n",
2236                   block->target[idx].lun.value[1],
2237                   block->target[idx].lun.value[0]);
2238         break;
2239     case ISCSI_BOOT_TGT_CHAP_NAME:
2240         rc = sprintf(buf, "%.*s\n", NVM_ISCSI_CFG_CHAP_NAME_MAX_LEN,
2241                  chap_name);
2242         break;
2243     case ISCSI_BOOT_TGT_CHAP_SECRET:
2244         rc = sprintf(buf, "%.*s\n", NVM_ISCSI_CFG_CHAP_PWD_MAX_LEN,
2245                  chap_secret);
2246         break;
2247     case ISCSI_BOOT_TGT_REV_CHAP_NAME:
2248         rc = sprintf(buf, "%.*s\n", NVM_ISCSI_CFG_CHAP_NAME_MAX_LEN,
2249                  mchap_name);
2250         break;
2251     case ISCSI_BOOT_TGT_REV_CHAP_SECRET:
2252         rc = sprintf(buf, "%.*s\n", NVM_ISCSI_CFG_CHAP_PWD_MAX_LEN,
2253                  mchap_secret);
2254         break;
2255     case ISCSI_BOOT_TGT_FLAGS:
2256         rc = snprintf(buf, 3, "%d\n", (char)SYSFS_FLAG_FW_SEL_BOOT);
2257         break;
2258     case ISCSI_BOOT_TGT_NIC_ASSOC:
2259         rc = snprintf(buf, 3, "0\n");
2260         break;
2261     default:
2262         rc = 0;
2263         break;
2264     }
2265 
2266 exit_show_tgt_info:
2267     return rc;
2268 }
2269 
2270 static ssize_t qedi_show_boot_tgt_pri_info(void *data, int type, char *buf)
2271 {
2272     struct qedi_ctx *qedi = data;
2273 
2274     return qedi_show_boot_tgt_info(qedi, type, buf, QEDI_NVM_TGT_PRI);
2275 }
2276 
2277 static ssize_t qedi_show_boot_tgt_sec_info(void *data, int type, char *buf)
2278 {
2279     struct qedi_ctx *qedi = data;
2280 
2281     return qedi_show_boot_tgt_info(qedi, type, buf, QEDI_NVM_TGT_SEC);
2282 }
2283 
2284 static umode_t qedi_tgt_get_attr_visibility(void *data, int type)
2285 {
2286     int rc;
2287 
2288     switch (type) {
2289     case ISCSI_BOOT_TGT_NAME:
2290     case ISCSI_BOOT_TGT_IP_ADDR:
2291     case ISCSI_BOOT_TGT_PORT:
2292     case ISCSI_BOOT_TGT_LUN:
2293     case ISCSI_BOOT_TGT_CHAP_NAME:
2294     case ISCSI_BOOT_TGT_CHAP_SECRET:
2295     case ISCSI_BOOT_TGT_REV_CHAP_NAME:
2296     case ISCSI_BOOT_TGT_REV_CHAP_SECRET:
2297     case ISCSI_BOOT_TGT_NIC_ASSOC:
2298     case ISCSI_BOOT_TGT_FLAGS:
2299         rc = 0444;
2300         break;
2301     default:
2302         rc = 0;
2303         break;
2304     }
2305     return rc;
2306 }
2307 
2308 static void qedi_boot_release(void *data)
2309 {
2310     struct qedi_ctx *qedi = data;
2311 
2312     scsi_host_put(qedi->shost);
2313 }
2314 
2315 static int qedi_get_boot_info(struct qedi_ctx *qedi)
2316 {
2317     int ret = 1;
2318 
2319     QEDI_INFO(&qedi->dbg_ctx, QEDI_LOG_INFO,
2320           "Get NVM iSCSI CFG image\n");
2321     ret = qedi_ops->common->nvm_get_image(qedi->cdev,
2322                           QED_NVM_IMAGE_ISCSI_CFG,
2323                           (char *)qedi->iscsi_image,
2324                           sizeof(struct qedi_nvm_iscsi_image));
2325     if (ret)
2326         QEDI_ERR(&qedi->dbg_ctx,
2327              "Could not get NVM image. ret = %d\n", ret);
2328 
2329     return ret;
2330 }
2331 
2332 static int qedi_setup_boot_info(struct qedi_ctx *qedi)
2333 {
2334     struct iscsi_boot_kobj *boot_kobj;
2335 
2336     if (qedi_get_boot_info(qedi))
2337         return -EPERM;
2338 
2339     qedi->boot_kset = iscsi_boot_create_host_kset(qedi->shost->host_no);
2340     if (!qedi->boot_kset)
2341         goto kset_free;
2342 
2343     if (!scsi_host_get(qedi->shost))
2344         goto kset_free;
2345 
2346     boot_kobj = iscsi_boot_create_target(qedi->boot_kset, 0, qedi,
2347                          qedi_show_boot_tgt_pri_info,
2348                          qedi_tgt_get_attr_visibility,
2349                          qedi_boot_release);
2350     if (!boot_kobj)
2351         goto put_host;
2352 
2353     if (!scsi_host_get(qedi->shost))
2354         goto kset_free;
2355 
2356     boot_kobj = iscsi_boot_create_target(qedi->boot_kset, 1, qedi,
2357                          qedi_show_boot_tgt_sec_info,
2358                          qedi_tgt_get_attr_visibility,
2359                          qedi_boot_release);
2360     if (!boot_kobj)
2361         goto put_host;
2362 
2363     if (!scsi_host_get(qedi->shost))
2364         goto kset_free;
2365 
2366     boot_kobj = iscsi_boot_create_initiator(qedi->boot_kset, 0, qedi,
2367                         qedi_show_boot_ini_info,
2368                         qedi_ini_get_attr_visibility,
2369                         qedi_boot_release);
2370     if (!boot_kobj)
2371         goto put_host;
2372 
2373     if (!scsi_host_get(qedi->shost))
2374         goto kset_free;
2375 
2376     boot_kobj = iscsi_boot_create_ethernet(qedi->boot_kset, 0, qedi,
2377                            qedi_show_boot_eth_info,
2378                            qedi_eth_get_attr_visibility,
2379                            qedi_boot_release);
2380     if (!boot_kobj)
2381         goto put_host;
2382 
2383     return 0;
2384 
2385 put_host:
2386     scsi_host_put(qedi->shost);
2387 kset_free:
2388     iscsi_boot_destroy_kset(qedi->boot_kset);
2389     return -ENOMEM;
2390 }
2391 
2392 static pci_ers_result_t qedi_io_error_detected(struct pci_dev *pdev,
2393                            pci_channel_state_t state)
2394 {
2395     struct qedi_ctx *qedi = pci_get_drvdata(pdev);
2396 
2397     QEDI_ERR(&qedi->dbg_ctx, "%s: PCI error detected [%d]\n",
2398          __func__, state);
2399 
2400     if (test_and_set_bit(QEDI_IN_RECOVERY, &qedi->flags)) {
2401         QEDI_INFO(&qedi->dbg_ctx, QEDI_LOG_INFO,
2402               "Recovery already in progress.\n");
2403         return PCI_ERS_RESULT_NONE;
2404     }
2405 
2406     qedi_ops->common->recovery_process(qedi->cdev);
2407 
2408     return PCI_ERS_RESULT_CAN_RECOVER;
2409 }
2410 
2411 static void __qedi_remove(struct pci_dev *pdev, int mode)
2412 {
2413     struct qedi_ctx *qedi = pci_get_drvdata(pdev);
2414     int rval;
2415     u16 retry = 10;
2416 
2417     if (mode == QEDI_MODE_NORMAL)
2418         iscsi_host_remove(qedi->shost, false);
2419     else if (mode == QEDI_MODE_SHUTDOWN)
2420         iscsi_host_remove(qedi->shost, true);
2421 
2422     if (mode == QEDI_MODE_NORMAL || mode == QEDI_MODE_SHUTDOWN) {
2423         if (qedi->tmf_thread) {
2424             destroy_workqueue(qedi->tmf_thread);
2425             qedi->tmf_thread = NULL;
2426         }
2427 
2428         if (qedi->offload_thread) {
2429             destroy_workqueue(qedi->offload_thread);
2430             qedi->offload_thread = NULL;
2431         }
2432     }
2433 
2434 #ifdef CONFIG_DEBUG_FS
2435     qedi_dbg_host_exit(&qedi->dbg_ctx);
2436 #endif
2437     if (!test_bit(QEDI_IN_OFFLINE, &qedi->flags))
2438         qedi_ops->common->set_power_state(qedi->cdev, PCI_D0);
2439 
2440     qedi_sync_free_irqs(qedi);
2441 
2442     if (!test_bit(QEDI_IN_OFFLINE, &qedi->flags)) {
2443         while (retry--) {
2444             rval = qedi_ops->stop(qedi->cdev);
2445             if (rval < 0)
2446                 msleep(1000);
2447             else
2448                 break;
2449         }
2450         qedi_ops->ll2->stop(qedi->cdev);
2451     }
2452 
2453     qedi_free_iscsi_pf_param(qedi);
2454 
2455     rval = qedi_ops->common->update_drv_state(qedi->cdev, false);
2456     if (rval)
2457         QEDI_ERR(&qedi->dbg_ctx, "Failed to send drv state to MFW\n");
2458 
2459     if (!test_bit(QEDI_IN_OFFLINE, &qedi->flags)) {
2460         qedi_ops->common->slowpath_stop(qedi->cdev);
2461         qedi_ops->common->remove(qedi->cdev);
2462     }
2463 
2464     qedi_destroy_fp(qedi);
2465 
2466     if (mode == QEDI_MODE_NORMAL || mode == QEDI_MODE_SHUTDOWN) {
2467         qedi_release_cid_que(qedi);
2468         qedi_cm_free_mem(qedi);
2469         qedi_free_uio(qedi->udev);
2470         qedi_free_itt(qedi);
2471 
2472         if (qedi->ll2_recv_thread) {
2473             kthread_stop(qedi->ll2_recv_thread);
2474             qedi->ll2_recv_thread = NULL;
2475         }
2476         qedi_ll2_free_skbs(qedi);
2477 
2478         if (qedi->boot_kset)
2479             iscsi_boot_destroy_kset(qedi->boot_kset);
2480 
2481         iscsi_host_free(qedi->shost);
2482     }
2483 }
2484 
2485 static void qedi_board_disable_work(struct work_struct *work)
2486 {
2487     struct qedi_ctx *qedi =
2488             container_of(work, struct qedi_ctx,
2489                      board_disable_work.work);
2490 
2491     QEDI_INFO(&qedi->dbg_ctx, QEDI_LOG_INFO,
2492           "Fan failure, Unloading firmware context.\n");
2493 
2494     if (test_and_set_bit(QEDI_IN_SHUTDOWN, &qedi->flags))
2495         return;
2496 
2497     __qedi_remove(qedi->pdev, QEDI_MODE_NORMAL);
2498 }
2499 
2500 static void qedi_shutdown(struct pci_dev *pdev)
2501 {
2502     struct qedi_ctx *qedi = pci_get_drvdata(pdev);
2503 
2504     QEDI_ERR(&qedi->dbg_ctx, "%s: Shutdown qedi\n", __func__);
2505     if (test_and_set_bit(QEDI_IN_SHUTDOWN, &qedi->flags))
2506         return;
2507     __qedi_remove(pdev, QEDI_MODE_SHUTDOWN);
2508 }
2509 
2510 static int __qedi_probe(struct pci_dev *pdev, int mode)
2511 {
2512     struct qedi_ctx *qedi;
2513     struct qed_ll2_params params;
2514     u8 dp_level = 0;
2515     bool is_vf = false;
2516     char host_buf[16];
2517     struct qed_link_params link_params;
2518     struct qed_slowpath_params sp_params;
2519     struct qed_probe_params qed_params;
2520     void *task_start, *task_end;
2521     int rc;
2522     u16 retry = 10;
2523 
2524     if (mode != QEDI_MODE_RECOVERY) {
2525         qedi = qedi_host_alloc(pdev);
2526         if (!qedi) {
2527             rc = -ENOMEM;
2528             goto exit_probe;
2529         }
2530     } else {
2531         qedi = pci_get_drvdata(pdev);
2532     }
2533 
2534 retry_probe:
2535     if (mode == QEDI_MODE_RECOVERY)
2536         msleep(2000);
2537 
2538     memset(&qed_params, 0, sizeof(qed_params));
2539     qed_params.protocol = QED_PROTOCOL_ISCSI;
2540     qed_params.dp_module = qedi_qed_debug;
2541     qed_params.dp_level = dp_level;
2542     qed_params.is_vf = is_vf;
2543     qedi->cdev = qedi_ops->common->probe(pdev, &qed_params);
2544     if (!qedi->cdev) {
2545         if (mode == QEDI_MODE_RECOVERY && retry) {
2546             QEDI_INFO(&qedi->dbg_ctx, QEDI_LOG_INFO,
2547                   "Retry %d initialize hardware\n", retry);
2548             retry--;
2549             goto retry_probe;
2550         }
2551 
2552         rc = -ENODEV;
2553         QEDI_ERR(&qedi->dbg_ctx, "Cannot initialize hardware\n");
2554         goto free_host;
2555     }
2556 
2557     set_bit(QEDI_ERR_ATTN_CLR_EN, &qedi->qedi_err_flags);
2558     set_bit(QEDI_ERR_IS_RECOVERABLE, &qedi->qedi_err_flags);
2559     atomic_set(&qedi->link_state, QEDI_LINK_DOWN);
2560 
2561     rc = qedi_ops->fill_dev_info(qedi->cdev, &qedi->dev_info);
2562     if (rc)
2563         goto free_host;
2564 
2565     QEDI_INFO(&qedi->dbg_ctx, QEDI_LOG_INFO,
2566           "dev_info: num_hwfns=%d affin_hwfn_idx=%d.\n",
2567           qedi->dev_info.common.num_hwfns,
2568           qedi_ops->common->get_affin_hwfn_idx(qedi->cdev));
2569 
2570     rc = qedi_set_iscsi_pf_param(qedi);
2571     if (rc) {
2572         rc = -ENOMEM;
2573         QEDI_ERR(&qedi->dbg_ctx,
2574              "Set iSCSI pf param fail\n");
2575         goto free_host;
2576     }
2577 
2578     qedi_ops->common->update_pf_params(qedi->cdev, &qedi->pf_params);
2579 
2580     rc = qedi_prepare_fp(qedi);
2581     if (rc) {
2582         QEDI_ERR(&qedi->dbg_ctx, "Cannot start slowpath.\n");
2583         goto free_pf_params;
2584     }
2585 
2586     /* Start the Slowpath-process */
2587     memset(&sp_params, 0, sizeof(struct qed_slowpath_params));
2588     sp_params.int_mode = QED_INT_MODE_MSIX;
2589     sp_params.drv_major = QEDI_DRIVER_MAJOR_VER;
2590     sp_params.drv_minor = QEDI_DRIVER_MINOR_VER;
2591     sp_params.drv_rev = QEDI_DRIVER_REV_VER;
2592     sp_params.drv_eng = QEDI_DRIVER_ENG_VER;
2593     strlcpy(sp_params.name, "qedi iSCSI", QED_DRV_VER_STR_SIZE);
2594     rc = qedi_ops->common->slowpath_start(qedi->cdev, &sp_params);
2595     if (rc) {
2596         QEDI_ERR(&qedi->dbg_ctx, "Cannot start slowpath\n");
2597         goto stop_hw;
2598     }
2599 
2600     /* update_pf_params needs to be called before and after slowpath
2601      * start
2602      */
2603     qedi_ops->common->update_pf_params(qedi->cdev, &qedi->pf_params);
2604 
2605     rc = qedi_setup_int(qedi);
2606     if (rc)
2607         goto stop_iscsi_func;
2608 
2609     qedi_ops->common->set_power_state(qedi->cdev, PCI_D0);
2610 
2611     /* Learn information crucial for qedi to progress */
2612     rc = qedi_ops->fill_dev_info(qedi->cdev, &qedi->dev_info);
2613     if (rc)
2614         goto stop_iscsi_func;
2615 
2616     /* Record BDQ producer doorbell addresses */
2617     qedi->bdq_primary_prod = qedi->dev_info.primary_dbq_rq_addr;
2618     qedi->bdq_secondary_prod = qedi->dev_info.secondary_bdq_rq_addr;
2619     QEDI_INFO(&qedi->dbg_ctx, QEDI_LOG_DISC,
2620           "BDQ primary_prod=%p secondary_prod=%p.\n",
2621           qedi->bdq_primary_prod,
2622           qedi->bdq_secondary_prod);
2623 
2624     /*
2625      * We need to write the number of BDs in the BDQ we've preallocated so
2626      * the f/w will do a prefetch and we'll get an unsolicited CQE when a
2627      * packet arrives.
2628      */
2629     qedi->bdq_prod_idx = QEDI_BDQ_NUM;
2630     QEDI_INFO(&qedi->dbg_ctx, QEDI_LOG_DISC,
2631           "Writing %d to primary and secondary BDQ doorbell registers.\n",
2632           qedi->bdq_prod_idx);
2633     writew(qedi->bdq_prod_idx, qedi->bdq_primary_prod);
2634     readw(qedi->bdq_primary_prod);
2635     writew(qedi->bdq_prod_idx, qedi->bdq_secondary_prod);
2636     readw(qedi->bdq_secondary_prod);
2637 
2638     ether_addr_copy(qedi->mac, qedi->dev_info.common.hw_mac);
2639     QEDI_INFO(&qedi->dbg_ctx, QEDI_LOG_DISC, "MAC address is %pM.\n",
2640           qedi->mac);
2641 
2642     snprintf(host_buf, sizeof(host_buf), "host_%d", qedi->shost->host_no);
2643     qedi_ops->common->set_name(qedi->cdev, host_buf);
2644 
2645     qedi_ops->register_ops(qedi->cdev, &qedi_cb_ops, qedi);
2646 
2647     memset(&params, 0, sizeof(params));
2648     params.mtu = DEF_PATH_MTU + IPV6_HDR_LEN + TCP_HDR_LEN;
2649     qedi->ll2_mtu = DEF_PATH_MTU;
2650     params.drop_ttl0_packets = 0;
2651     params.rx_vlan_stripping = 1;
2652     ether_addr_copy(params.ll2_mac_address, qedi->dev_info.common.hw_mac);
2653 
2654     if (mode != QEDI_MODE_RECOVERY) {
2655         /* set up rx path */
2656         INIT_LIST_HEAD(&qedi->ll2_skb_list);
2657         spin_lock_init(&qedi->ll2_lock);
2658         /* start qedi context */
2659         spin_lock_init(&qedi->hba_lock);
2660         spin_lock_init(&qedi->task_idx_lock);
2661         mutex_init(&qedi->stats_lock);
2662     }
2663     qedi_ops->ll2->register_cb_ops(qedi->cdev, &qedi_ll2_cb_ops, qedi);
2664     qedi_ops->ll2->start(qedi->cdev, &params);
2665 
2666     if (mode != QEDI_MODE_RECOVERY) {
2667         qedi->ll2_recv_thread = kthread_run(qedi_ll2_recv_thread,
2668                             (void *)qedi,
2669                             "qedi_ll2_thread");
2670     }
2671 
2672     rc = qedi_ops->start(qedi->cdev, &qedi->tasks,
2673                  qedi, qedi_iscsi_event_cb);
2674     if (rc) {
2675         rc = -ENODEV;
2676         QEDI_ERR(&qedi->dbg_ctx, "Cannot start iSCSI function\n");
2677         goto stop_slowpath;
2678     }
2679 
2680     task_start = qedi_get_task_mem(&qedi->tasks, 0);
2681     task_end = qedi_get_task_mem(&qedi->tasks, MAX_TID_BLOCKS_ISCSI - 1);
2682     QEDI_INFO(&qedi->dbg_ctx, QEDI_LOG_DISC,
2683           "Task context start=%p, end=%p block_size=%u.\n",
2684            task_start, task_end, qedi->tasks.size);
2685 
2686     memset(&link_params, 0, sizeof(link_params));
2687     link_params.link_up = true;
2688     rc = qedi_ops->common->set_link(qedi->cdev, &link_params);
2689     if (rc) {
2690         QEDI_WARN(&qedi->dbg_ctx, "Link set up failed.\n");
2691         atomic_set(&qedi->link_state, QEDI_LINK_DOWN);
2692     }
2693 
2694 #ifdef CONFIG_DEBUG_FS
2695     qedi_dbg_host_init(&qedi->dbg_ctx, qedi_debugfs_ops,
2696                qedi_dbg_fops);
2697 #endif
2698     QEDI_INFO(&qedi->dbg_ctx, QEDI_LOG_INFO,
2699           "QLogic FastLinQ iSCSI Module qedi %s, FW %d.%d.%d.%d\n",
2700           QEDI_MODULE_VERSION, FW_MAJOR_VERSION, FW_MINOR_VERSION,
2701           FW_REVISION_VERSION, FW_ENGINEERING_VERSION);
2702 
2703     if (mode == QEDI_MODE_NORMAL) {
2704         if (iscsi_host_add(qedi->shost, &pdev->dev)) {
2705             QEDI_ERR(&qedi->dbg_ctx,
2706                  "Could not add iscsi host\n");
2707             rc = -ENOMEM;
2708             goto remove_host;
2709         }
2710 
2711         /* Allocate uio buffers */
2712         rc = qedi_alloc_uio_rings(qedi);
2713         if (rc) {
2714             QEDI_ERR(&qedi->dbg_ctx,
2715                  "UIO alloc ring failed err=%d\n", rc);
2716             goto remove_host;
2717         }
2718 
2719         rc = qedi_init_uio(qedi);
2720         if (rc) {
2721             QEDI_ERR(&qedi->dbg_ctx,
2722                  "UIO init failed, err=%d\n", rc);
2723             goto free_uio;
2724         }
2725 
2726         /* host the array on iscsi_conn */
2727         rc = qedi_setup_cid_que(qedi);
2728         if (rc) {
2729             QEDI_ERR(&qedi->dbg_ctx,
2730                  "Could not setup cid que\n");
2731             goto free_uio;
2732         }
2733 
2734         rc = qedi_cm_alloc_mem(qedi);
2735         if (rc) {
2736             QEDI_ERR(&qedi->dbg_ctx,
2737                  "Could not alloc cm memory\n");
2738             goto free_cid_que;
2739         }
2740 
2741         rc = qedi_alloc_itt(qedi);
2742         if (rc) {
2743             QEDI_ERR(&qedi->dbg_ctx,
2744                  "Could not alloc itt memory\n");
2745             goto free_cid_que;
2746         }
2747 
2748         sprintf(host_buf, "host_%d", qedi->shost->host_no);
2749         qedi->tmf_thread = create_singlethread_workqueue(host_buf);
2750         if (!qedi->tmf_thread) {
2751             QEDI_ERR(&qedi->dbg_ctx,
2752                  "Unable to start tmf thread!\n");
2753             rc = -ENODEV;
2754             goto free_cid_que;
2755         }
2756 
2757         sprintf(host_buf, "qedi_ofld%d", qedi->shost->host_no);
2758         qedi->offload_thread = create_workqueue(host_buf);
2759         if (!qedi->offload_thread) {
2760             QEDI_ERR(&qedi->dbg_ctx,
2761                  "Unable to start offload thread!\n");
2762             rc = -ENODEV;
2763             goto free_tmf_thread;
2764         }
2765 
2766         INIT_DELAYED_WORK(&qedi->recovery_work, qedi_recovery_handler);
2767         INIT_DELAYED_WORK(&qedi->board_disable_work,
2768                   qedi_board_disable_work);
2769 
2770         /* F/w needs 1st task context memory entry for performance */
2771         set_bit(QEDI_RESERVE_TASK_ID, qedi->task_idx_map);
2772         atomic_set(&qedi->num_offloads, 0);
2773 
2774         if (qedi_setup_boot_info(qedi))
2775             QEDI_ERR(&qedi->dbg_ctx,
2776                  "No iSCSI boot target configured\n");
2777 
2778         rc = qedi_ops->common->update_drv_state(qedi->cdev, true);
2779         if (rc)
2780             QEDI_ERR(&qedi->dbg_ctx,
2781                  "Failed to send drv state to MFW\n");
2782 
2783     }
2784 
2785     return 0;
2786 
2787 free_tmf_thread:
2788     destroy_workqueue(qedi->tmf_thread);
2789 free_cid_que:
2790     qedi_release_cid_que(qedi);
2791 free_uio:
2792     qedi_free_uio(qedi->udev);
2793 remove_host:
2794 #ifdef CONFIG_DEBUG_FS
2795     qedi_dbg_host_exit(&qedi->dbg_ctx);
2796 #endif
2797     iscsi_host_remove(qedi->shost, false);
2798 stop_iscsi_func:
2799     qedi_ops->stop(qedi->cdev);
2800 stop_slowpath:
2801     qedi_ops->common->slowpath_stop(qedi->cdev);
2802 stop_hw:
2803     qedi_ops->common->remove(qedi->cdev);
2804 free_pf_params:
2805     qedi_free_iscsi_pf_param(qedi);
2806 free_host:
2807     iscsi_host_free(qedi->shost);
2808 exit_probe:
2809     return rc;
2810 }
2811 
2812 static void qedi_mark_conn_recovery(struct iscsi_cls_session *cls_session)
2813 {
2814     struct iscsi_session *session = cls_session->dd_data;
2815     struct iscsi_conn *conn = session->leadconn;
2816     struct qedi_conn *qedi_conn = conn->dd_data;
2817 
2818     iscsi_conn_failure(qedi_conn->cls_conn->dd_data, ISCSI_ERR_CONN_FAILED);
2819 }
2820 
2821 static void qedi_recovery_handler(struct work_struct *work)
2822 {
2823     struct qedi_ctx *qedi =
2824             container_of(work, struct qedi_ctx, recovery_work.work);
2825 
2826     iscsi_host_for_each_session(qedi->shost, qedi_mark_conn_recovery);
2827 
2828     /* Call common_ops->recovery_prolog to allow the MFW to quiesce
2829      * any PCI transactions.
2830      */
2831     qedi_ops->common->recovery_prolog(qedi->cdev);
2832 
2833     __qedi_remove(qedi->pdev, QEDI_MODE_RECOVERY);
2834     __qedi_probe(qedi->pdev, QEDI_MODE_RECOVERY);
2835     clear_bit(QEDI_IN_RECOVERY, &qedi->flags);
2836 }
2837 
2838 static int qedi_probe(struct pci_dev *pdev, const struct pci_device_id *id)
2839 {
2840     return __qedi_probe(pdev, QEDI_MODE_NORMAL);
2841 }
2842 
2843 static void qedi_remove(struct pci_dev *pdev)
2844 {
2845     __qedi_remove(pdev, QEDI_MODE_NORMAL);
2846 }
2847 
2848 static struct pci_device_id qedi_pci_tbl[] = {
2849     { PCI_DEVICE(PCI_VENDOR_ID_QLOGIC, 0x165E) },
2850     { PCI_DEVICE(PCI_VENDOR_ID_QLOGIC, 0x8084) },
2851     { 0 },
2852 };
2853 MODULE_DEVICE_TABLE(pci, qedi_pci_tbl);
2854 
2855 static enum cpuhp_state qedi_cpuhp_state;
2856 
2857 static struct pci_error_handlers qedi_err_handler = {
2858     .error_detected = qedi_io_error_detected,
2859 };
2860 
2861 static struct pci_driver qedi_pci_driver = {
2862     .name = QEDI_MODULE_NAME,
2863     .id_table = qedi_pci_tbl,
2864     .probe = qedi_probe,
2865     .remove = qedi_remove,
2866     .shutdown = qedi_shutdown,
2867     .err_handler = &qedi_err_handler,
2868 };
2869 
2870 static int __init qedi_init(void)
2871 {
2872     struct qedi_percpu_s *p;
2873     int cpu, rc = 0;
2874 
2875     qedi_ops = qed_get_iscsi_ops();
2876     if (!qedi_ops) {
2877         QEDI_ERR(NULL, "Failed to get qed iSCSI operations\n");
2878         return -EINVAL;
2879     }
2880 
2881 #ifdef CONFIG_DEBUG_FS
2882     qedi_dbg_init("qedi");
2883 #endif
2884 
2885     qedi_scsi_transport = iscsi_register_transport(&qedi_iscsi_transport);
2886     if (!qedi_scsi_transport) {
2887         QEDI_ERR(NULL, "Could not register qedi transport");
2888         rc = -ENOMEM;
2889         goto exit_qedi_init_1;
2890     }
2891 
2892     for_each_possible_cpu(cpu) {
2893         p = &per_cpu(qedi_percpu, cpu);
2894         INIT_LIST_HEAD(&p->work_list);
2895         spin_lock_init(&p->p_work_lock);
2896         p->iothread = NULL;
2897     }
2898 
2899     rc = cpuhp_setup_state(CPUHP_AP_ONLINE_DYN, "scsi/qedi:online",
2900                    qedi_cpu_online, qedi_cpu_offline);
2901     if (rc < 0)
2902         goto exit_qedi_init_2;
2903     qedi_cpuhp_state = rc;
2904 
2905     rc = pci_register_driver(&qedi_pci_driver);
2906     if (rc) {
2907         QEDI_ERR(NULL, "Failed to register driver\n");
2908         goto exit_qedi_hp;
2909     }
2910 
2911     return 0;
2912 
2913 exit_qedi_hp:
2914     cpuhp_remove_state(qedi_cpuhp_state);
2915 exit_qedi_init_2:
2916     iscsi_unregister_transport(&qedi_iscsi_transport);
2917 exit_qedi_init_1:
2918 #ifdef CONFIG_DEBUG_FS
2919     qedi_dbg_exit();
2920 #endif
2921     qed_put_iscsi_ops();
2922     return rc;
2923 }
2924 
2925 static void __exit qedi_cleanup(void)
2926 {
2927     pci_unregister_driver(&qedi_pci_driver);
2928     cpuhp_remove_state(qedi_cpuhp_state);
2929     iscsi_unregister_transport(&qedi_iscsi_transport);
2930 
2931 #ifdef CONFIG_DEBUG_FS
2932     qedi_dbg_exit();
2933 #endif
2934     qed_put_iscsi_ops();
2935 }
2936 
2937 MODULE_DESCRIPTION("QLogic FastLinQ 4xxxx iSCSI Module");
2938 MODULE_LICENSE("GPL");
2939 MODULE_AUTHOR("QLogic Corporation");
2940 MODULE_VERSION(QEDI_MODULE_VERSION);
2941 module_init(qedi_init);
2942 module_exit(qedi_cleanup);
[ Source navigation ] [ Diff markup ] [ Identifier search ] [ general search ]

This page was automatically generated by the 2.1.0 LXR engine.

The LXR team
Valid CSS 2.1! Valid HTML 4.01!