hw/qedr/verbs.c

0001 /* QLogic qedr NIC Driver
0002  * Copyright (c) 2015-2016  QLogic Corporation
0003  *
0004  * This software is available to you under a choice of one of two
0005  * licenses.  You may choose to be licensed under the terms of the GNU
0006  * General Public License (GPL) Version 2, available from the file
0007  * COPYING in the main directory of this source tree, or the
0008  * OpenIB.org BSD license below:
0009  *
0010  *     Redistribution and use in source and binary forms, with or
0011  *     without modification, are permitted provided that the following
0012  *     conditions are met:
0013  *
0014  *      - Redistributions of source code must retain the above
0015  *        copyright notice, this list of conditions and the following
0016  *        disclaimer.
0017  *
0018  *      - Redistributions in binary form must reproduce the above
0019  *        copyright notice, this list of conditions and the following
0020  *        disclaimer in the documentation and /or other materials
0021  *        provided with the distribution.
0022  *
0023  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
0024  * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
0025  * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
0026  * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
0027  * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
0028  * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
0029  * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
0030  * SOFTWARE.
0031  */
0032 #include <linux/dma-mapping.h>
0033 #include <linux/crc32.h>
0034 #include <net/ip.h>
0035 #include <net/ipv6.h>
0036 #include <net/udp.h>
0037 #include <linux/iommu.h>
0038
0039 #include <rdma/ib_verbs.h>
0040 #include <rdma/ib_user_verbs.h>
0041 #include <rdma/iw_cm.h>
0042 #include <rdma/ib_umem.h>
0043 #include <rdma/ib_addr.h>
0044 #include <rdma/ib_cache.h>
0045 #include <rdma/uverbs_ioctl.h>
0046
0047 #include <linux/qed/common_hsi.h>
0048 #include "qedr_hsi_rdma.h"
0049 #include <linux/qed/qed_if.h>
0050 #include "qedr.h"
0051 #include "verbs.h"
0052 #include <rdma/qedr-abi.h>
0053 #include "qedr_roce_cm.h"
0054 #include "qedr_iw_cm.h"
0055
0056 #define QEDR_SRQ_WQE_ELEM_SIZE  sizeof(union rdma_srq_elm)
0057 #define RDMA_MAX_SGE_PER_SRQ    (4)
0058 #define RDMA_MAX_SRQ_WQE_SIZE   (RDMA_MAX_SGE_PER_SRQ + 1)
0059
0060 #define DB_ADDR_SHIFT(addr)     ((addr) << DB_PWM_ADDR_OFFSET_SHIFT)
0061
0062 enum {
0063     QEDR_USER_MMAP_IO_WC = 0,
0064     QEDR_USER_MMAP_PHYS_PAGE,
0065 };
0066
0067 static inline int qedr_ib_copy_to_udata(struct ib_udata *udata, void *src,
0068                     size_t len)
0069 {
0070     size_t min_len = min_t(size_t, len, udata->outlen);
0071
0072     return ib_copy_to_udata(udata, src, min_len);
0073 }
0074
0075 int qedr_query_pkey(struct ib_device *ibdev, u32 port, u16 index, u16 *pkey)
0076 {
0077     if (index >= QEDR_ROCE_PKEY_TABLE_LEN)
0078         return -EINVAL;
0079
0080     *pkey = QEDR_ROCE_PKEY_DEFAULT;
0081     return 0;
0082 }
0083
0084 int qedr_iw_query_gid(struct ib_device *ibdev, u32 port,
0085               int index, union ib_gid *sgid)
0086 {
0087     struct qedr_dev *dev = get_qedr_dev(ibdev);
0088
0089     memset(sgid->raw, 0, sizeof(sgid->raw));
0090     ether_addr_copy(sgid->raw, dev->ndev->dev_addr);
0091
0092     DP_DEBUG(dev, QEDR_MSG_INIT, "QUERY sgid[%d]=%llx:%llx\n", index,
0093          sgid->global.interface_id, sgid->global.subnet_prefix);
0094
0095     return 0;
0096 }
0097
0098 int qedr_query_srq(struct ib_srq *ibsrq, struct ib_srq_attr *srq_attr)
0099 {
0100     struct qedr_dev *dev = get_qedr_dev(ibsrq->device);
0101     struct qedr_device_attr *qattr = &dev->attr;
0102     struct qedr_srq *srq = get_qedr_srq(ibsrq);
0103
0104     srq_attr->srq_limit = srq->srq_limit;
0105     srq_attr->max_wr = qattr->max_srq_wr;
0106     srq_attr->max_sge = qattr->max_sge;
0107
0108     return 0;
0109 }
0110
0111 int qedr_query_device(struct ib_device *ibdev,
0112               struct ib_device_attr *attr, struct ib_udata *udata)
0113 {
0114     struct qedr_dev *dev = get_qedr_dev(ibdev);
0115     struct qedr_device_attr *qattr = &dev->attr;
0116
0117     if (!dev->rdma_ctx) {
0118         DP_ERR(dev,
0119                "qedr_query_device called with invalid params rdma_ctx=%p\n",
0120                dev->rdma_ctx);
0121         return -EINVAL;
0122     }
0123
0124     memset(attr, 0, sizeof(*attr));
0125
0126     attr->fw_ver = qattr->fw_ver;
0127     attr->sys_image_guid = qattr->sys_image_guid;
0128     attr->max_mr_size = qattr->max_mr_size;
0129     attr->page_size_cap = qattr->page_size_caps;
0130     attr->vendor_id = qattr->vendor_id;
0131     attr->vendor_part_id = qattr->vendor_part_id;
0132     attr->hw_ver = qattr->hw_ver;
0133     attr->max_qp = qattr->max_qp;
0134     attr->max_qp_wr = max_t(u32, qattr->max_sqe, qattr->max_rqe);
0135     attr->device_cap_flags = IB_DEVICE_CURR_QP_STATE_MOD |
0136         IB_DEVICE_RC_RNR_NAK_GEN |
0137         IB_DEVICE_MEM_MGT_EXTENSIONS;
0138     attr->kernel_cap_flags = IBK_LOCAL_DMA_LKEY;
0139
0140     if (!rdma_protocol_iwarp(&dev->ibdev, 1))
0141         attr->device_cap_flags |= IB_DEVICE_XRC;
0142     attr->max_send_sge = qattr->max_sge;
0143     attr->max_recv_sge = qattr->max_sge;
0144     attr->max_sge_rd = qattr->max_sge;
0145     attr->max_cq = qattr->max_cq;
0146     attr->max_cqe = qattr->max_cqe;
0147     attr->max_mr = qattr->max_mr;
0148     attr->max_mw = qattr->max_mw;
0149     attr->max_pd = qattr->max_pd;
0150     attr->atomic_cap = dev->atomic_cap;
0151     attr->max_qp_init_rd_atom =
0152         1 << (fls(qattr->max_qp_req_rd_atomic_resc) - 1);
0153     attr->max_qp_rd_atom =
0154         min(1 << (fls(qattr->max_qp_resp_rd_atomic_resc) - 1),
0155         attr->max_qp_init_rd_atom);
0156
0157     attr->max_srq = qattr->max_srq;
0158     attr->max_srq_sge = qattr->max_srq_sge;
0159     attr->max_srq_wr = qattr->max_srq_wr;
0160
0161     attr->local_ca_ack_delay = qattr->dev_ack_delay;
0162     attr->max_fast_reg_page_list_len = qattr->max_mr / 8;
0163     attr->max_pkeys = qattr->max_pkey;
0164     attr->max_ah = qattr->max_ah;
0165
0166     return 0;
0167 }
0168
0169 static inline void get_link_speed_and_width(int speed, u16 *ib_speed,
0170                         u8 *ib_width)
0171 {
0172     switch (speed) {
0173     case 1000:
0174         *ib_speed = IB_SPEED_SDR;
0175         *ib_width = IB_WIDTH_1X;
0176         break;
0177     case 10000:
0178         *ib_speed = IB_SPEED_QDR;
0179         *ib_width = IB_WIDTH_1X;
0180         break;
0181
0182     case 20000:
0183         *ib_speed = IB_SPEED_DDR;
0184         *ib_width = IB_WIDTH_4X;
0185         break;
0186
0187     case 25000:
0188         *ib_speed = IB_SPEED_EDR;
0189         *ib_width = IB_WIDTH_1X;
0190         break;
0191
0192     case 40000:
0193         *ib_speed = IB_SPEED_QDR;
0194         *ib_width = IB_WIDTH_4X;
0195         break;
0196
0197     case 50000:
0198         *ib_speed = IB_SPEED_HDR;
0199         *ib_width = IB_WIDTH_1X;
0200         break;
0201
0202     case 100000:
0203         *ib_speed = IB_SPEED_EDR;
0204         *ib_width = IB_WIDTH_4X;
0205         break;
0206
0207     default:
0208         /* Unsupported */
0209         *ib_speed = IB_SPEED_SDR;
0210         *ib_width = IB_WIDTH_1X;
0211     }
0212 }
0213
0214 int qedr_query_port(struct ib_device *ibdev, u32 port,
0215             struct ib_port_attr *attr)
0216 {
0217     struct qedr_dev *dev;
0218     struct qed_rdma_port *rdma_port;
0219
0220     dev = get_qedr_dev(ibdev);
0221
0222     if (!dev->rdma_ctx) {
0223         DP_ERR(dev, "rdma_ctx is NULL\n");
0224         return -EINVAL;
0225     }
0226
0227     rdma_port = dev->ops->rdma_query_port(dev->rdma_ctx);
0228
0229     /* *attr being zeroed by the caller, avoid zeroing it here */
0230     if (rdma_port->port_state == QED_RDMA_PORT_UP) {
0231         attr->state = IB_PORT_ACTIVE;
0232         attr->phys_state = IB_PORT_PHYS_STATE_LINK_UP;
0233     } else {
0234         attr->state = IB_PORT_DOWN;
0235         attr->phys_state = IB_PORT_PHYS_STATE_DISABLED;
0236     }
0237     attr->max_mtu = IB_MTU_4096;
0238     attr->lid = 0;
0239     attr->lmc = 0;
0240     attr->sm_lid = 0;
0241     attr->sm_sl = 0;
0242     attr->ip_gids = true;
0243     if (rdma_protocol_iwarp(&dev->ibdev, 1)) {
0244         attr->active_mtu = iboe_get_mtu(dev->iwarp_max_mtu);
0245         attr->gid_tbl_len = 1;
0246     } else {
0247         attr->active_mtu = iboe_get_mtu(dev->ndev->mtu);
0248         attr->gid_tbl_len = QEDR_MAX_SGID;
0249         attr->pkey_tbl_len = QEDR_ROCE_PKEY_TABLE_LEN;
0250     }
0251     attr->bad_pkey_cntr = rdma_port->pkey_bad_counter;
0252     attr->qkey_viol_cntr = 0;
0253     get_link_speed_and_width(rdma_port->link_speed,
0254                  &attr->active_speed, &attr->active_width);
0255     attr->max_msg_sz = rdma_port->max_msg_size;
0256     attr->max_vl_num = 4;
0257
0258     return 0;
0259 }
0260
0261 int qedr_alloc_ucontext(struct ib_ucontext *uctx, struct ib_udata *udata)
0262 {
0263     struct ib_device *ibdev = uctx->device;
0264     int rc;
0265     struct qedr_ucontext *ctx = get_qedr_ucontext(uctx);
0266     struct qedr_alloc_ucontext_resp uresp = {};
0267     struct qedr_alloc_ucontext_req ureq = {};
0268     struct qedr_dev *dev = get_qedr_dev(ibdev);
0269     struct qed_rdma_add_user_out_params oparams;
0270     struct qedr_user_mmap_entry *entry;
0271
0272     if (!udata)
0273         return -EFAULT;
0274
0275     if (udata->inlen) {
0276         rc = ib_copy_from_udata(&ureq, udata,
0277                     min(sizeof(ureq), udata->inlen));
0278         if (rc) {
0279             DP_ERR(dev, "Problem copying data from user space\n");
0280             return -EFAULT;
0281         }
0282         ctx->edpm_mode = !!(ureq.context_flags &
0283                     QEDR_ALLOC_UCTX_EDPM_MODE);
0284         ctx->db_rec = !!(ureq.context_flags & QEDR_ALLOC_UCTX_DB_REC);
0285     }
0286
0287     rc = dev->ops->rdma_add_user(dev->rdma_ctx, &oparams);
0288     if (rc) {
0289         DP_ERR(dev,
0290                "failed to allocate a DPI for a new RoCE application, rc=%d. To overcome this consider to increase the number of DPIs, increase the doorbell BAR size or just close unnecessary RoCE applications. In order to increase the number of DPIs consult the qedr readme\n",
0291                rc);
0292         return rc;
0293     }
0294
0295     ctx->dpi = oparams.dpi;
0296     ctx->dpi_addr = oparams.dpi_addr;
0297     ctx->dpi_phys_addr = oparams.dpi_phys_addr;
0298     ctx->dpi_size = oparams.dpi_size;
0299     entry = kzalloc(sizeof(*entry), GFP_KERNEL);
0300     if (!entry) {
0301         rc = -ENOMEM;
0302         goto err;
0303     }
0304
0305     entry->io_address = ctx->dpi_phys_addr;
0306     entry->length = ctx->dpi_size;
0307     entry->mmap_flag = QEDR_USER_MMAP_IO_WC;
0308     entry->dpi = ctx->dpi;
0309     entry->dev = dev;
0310     rc = rdma_user_mmap_entry_insert(uctx, &entry->rdma_entry,
0311                      ctx->dpi_size);
0312     if (rc) {
0313         kfree(entry);
0314         goto err;
0315     }
0316     ctx->db_mmap_entry = &entry->rdma_entry;
0317
0318     if (!dev->user_dpm_enabled)
0319         uresp.dpm_flags = 0;
0320     else if (rdma_protocol_iwarp(&dev->ibdev, 1))
0321         uresp.dpm_flags = QEDR_DPM_TYPE_IWARP_LEGACY;
0322     else
0323         uresp.dpm_flags = QEDR_DPM_TYPE_ROCE_ENHANCED |
0324                   QEDR_DPM_TYPE_ROCE_LEGACY |
0325                   QEDR_DPM_TYPE_ROCE_EDPM_MODE;
0326
0327     if (ureq.context_flags & QEDR_SUPPORT_DPM_SIZES) {
0328         uresp.dpm_flags |= QEDR_DPM_SIZES_SET;
0329         uresp.ldpm_limit_size = QEDR_LDPM_MAX_SIZE;
0330         uresp.edpm_trans_size = QEDR_EDPM_TRANS_SIZE;
0331         uresp.edpm_limit_size = QEDR_EDPM_MAX_SIZE;
0332     }
0333
0334     uresp.wids_enabled = 1;
0335     uresp.wid_count = oparams.wid_count;
0336     uresp.db_pa = rdma_user_mmap_get_offset(ctx->db_mmap_entry);
0337     uresp.db_size = ctx->dpi_size;
0338     uresp.max_send_wr = dev->attr.max_sqe;
0339     uresp.max_recv_wr = dev->attr.max_rqe;
0340     uresp.max_srq_wr = dev->attr.max_srq_wr;
0341     uresp.sges_per_send_wr = QEDR_MAX_SQE_ELEMENTS_PER_SQE;
0342     uresp.sges_per_recv_wr = QEDR_MAX_RQE_ELEMENTS_PER_RQE;
0343     uresp.sges_per_srq_wr = dev->attr.max_srq_sge;
0344     uresp.max_cqes = QEDR_MAX_CQES;
0345
0346     rc = qedr_ib_copy_to_udata(udata, &uresp, sizeof(uresp));
0347     if (rc)
0348         goto err;
0349
0350     ctx->dev = dev;
0351
0352     DP_DEBUG(dev, QEDR_MSG_INIT, "Allocating user context %p\n",
0353          &ctx->ibucontext);
0354     return 0;
0355
0356 err:
0357     if (!ctx->db_mmap_entry)
0358         dev->ops->rdma_remove_user(dev->rdma_ctx, ctx->dpi);
0359     else
0360         rdma_user_mmap_entry_remove(ctx->db_mmap_entry);
0361
0362     return rc;
0363 }
0364
0365 void qedr_dealloc_ucontext(struct ib_ucontext *ibctx)
0366 {
0367     struct qedr_ucontext *uctx = get_qedr_ucontext(ibctx);
0368
0369     DP_DEBUG(uctx->dev, QEDR_MSG_INIT, "Deallocating user context %p\n",
0370          uctx);
0371
0372     rdma_user_mmap_entry_remove(uctx->db_mmap_entry);
0373 }
0374
0375 void qedr_mmap_free(struct rdma_user_mmap_entry *rdma_entry)
0376 {
0377     struct qedr_user_mmap_entry *entry = get_qedr_mmap_entry(rdma_entry);
0378     struct qedr_dev *dev = entry->dev;
0379
0380     if (entry->mmap_flag == QEDR_USER_MMAP_PHYS_PAGE)
0381         free_page((unsigned long)entry->address);
0382     else if (entry->mmap_flag == QEDR_USER_MMAP_IO_WC)
0383         dev->ops->rdma_remove_user(dev->rdma_ctx, entry->dpi);
0384
0385     kfree(entry);
0386 }
0387
0388 int qedr_mmap(struct ib_ucontext *ucontext, struct vm_area_struct *vma)
0389 {
0390     struct ib_device *dev = ucontext->device;
0391     size_t length = vma->vm_end - vma->vm_start;
0392     struct rdma_user_mmap_entry *rdma_entry;
0393     struct qedr_user_mmap_entry *entry;
0394     int rc = 0;
0395     u64 pfn;
0396
0397     ibdev_dbg(dev,
0398           "start %#lx, end %#lx, length = %#zx, pgoff = %#lx\n",
0399           vma->vm_start, vma->vm_end, length, vma->vm_pgoff);
0400
0401     rdma_entry = rdma_user_mmap_entry_get(ucontext, vma);
0402     if (!rdma_entry) {
0403         ibdev_dbg(dev, "pgoff[%#lx] does not have valid entry\n",
0404               vma->vm_pgoff);
0405         return -EINVAL;
0406     }
0407     entry = get_qedr_mmap_entry(rdma_entry);
0408     ibdev_dbg(dev,
0409           "Mapping address[%#llx], length[%#zx], mmap_flag[%d]\n",
0410           entry->io_address, length, entry->mmap_flag);
0411
0412     switch (entry->mmap_flag) {
0413     case QEDR_USER_MMAP_IO_WC:
0414         pfn = entry->io_address >> PAGE_SHIFT;
0415         rc = rdma_user_mmap_io(ucontext, vma, pfn, length,
0416                        pgprot_writecombine(vma->vm_page_prot),
0417                        rdma_entry);
0418         break;
0419     case QEDR_USER_MMAP_PHYS_PAGE:
0420         rc = vm_insert_page(vma, vma->vm_start,
0421                     virt_to_page(entry->address));
0422         break;
0423     default:
0424         rc = -EINVAL;
0425     }
0426
0427     if (rc)
0428         ibdev_dbg(dev,
0429               "Couldn't mmap address[%#llx] length[%#zx] mmap_flag[%d] err[%d]\n",
0430               entry->io_address, length, entry->mmap_flag, rc);
0431
0432     rdma_user_mmap_entry_put(rdma_entry);
0433     return rc;
0434 }
0435
0436 int qedr_alloc_pd(struct ib_pd *ibpd, struct ib_udata *udata)
0437 {
0438     struct ib_device *ibdev = ibpd->device;
0439     struct qedr_dev *dev = get_qedr_dev(ibdev);
0440     struct qedr_pd *pd = get_qedr_pd(ibpd);
0441     u16 pd_id;
0442     int rc;
0443
0444     DP_DEBUG(dev, QEDR_MSG_INIT, "Function called from: %s\n",
0445          udata ? "User Lib" : "Kernel");
0446
0447     if (!dev->rdma_ctx) {
0448         DP_ERR(dev, "invalid RDMA context\n");
0449         return -EINVAL;
0450     }
0451
0452     rc = dev->ops->rdma_alloc_pd(dev->rdma_ctx, &pd_id);
0453     if (rc)
0454         return rc;
0455
0456     pd->pd_id = pd_id;
0457
0458     if (udata) {
0459         struct qedr_alloc_pd_uresp uresp = {
0460             .pd_id = pd_id,
0461         };
0462         struct qedr_ucontext *context = rdma_udata_to_drv_context(
0463             udata, struct qedr_ucontext, ibucontext);
0464
0465         rc = qedr_ib_copy_to_udata(udata, &uresp, sizeof(uresp));
0466         if (rc) {
0467             DP_ERR(dev, "copy error pd_id=0x%x.\n", pd_id);
0468             dev->ops->rdma_dealloc_pd(dev->rdma_ctx, pd_id);
0469             return rc;
0470         }
0471
0472         pd->uctx = context;
0473         pd->uctx->pd = pd;
0474     }
0475
0476     return 0;
0477 }
0478
0479 int qedr_dealloc_pd(struct ib_pd *ibpd, struct ib_udata *udata)
0480 {
0481     struct qedr_dev *dev = get_qedr_dev(ibpd->device);
0482     struct qedr_pd *pd = get_qedr_pd(ibpd);
0483
0484     DP_DEBUG(dev, QEDR_MSG_INIT, "Deallocating PD %d\n", pd->pd_id);
0485     dev->ops->rdma_dealloc_pd(dev->rdma_ctx, pd->pd_id);
0486     return 0;
0487 }
0488
0489
0490 int qedr_alloc_xrcd(struct ib_xrcd *ibxrcd, struct ib_udata *udata)
0491 {
0492     struct qedr_dev *dev = get_qedr_dev(ibxrcd->device);
0493     struct qedr_xrcd *xrcd = get_qedr_xrcd(ibxrcd);
0494
0495     return dev->ops->rdma_alloc_xrcd(dev->rdma_ctx, &xrcd->xrcd_id);
0496 }
0497
0498 int qedr_dealloc_xrcd(struct ib_xrcd *ibxrcd, struct ib_udata *udata)
0499 {
0500     struct qedr_dev *dev = get_qedr_dev(ibxrcd->device);
0501     u16 xrcd_id = get_qedr_xrcd(ibxrcd)->xrcd_id;
0502
0503     dev->ops->rdma_dealloc_xrcd(dev->rdma_ctx, xrcd_id);
0504     return 0;
0505 }
0506 static void qedr_free_pbl(struct qedr_dev *dev,
0507               struct qedr_pbl_info *pbl_info, struct qedr_pbl *pbl)
0508 {
0509     struct pci_dev *pdev = dev->pdev;
0510     int i;
0511
0512     for (i = 0; i < pbl_info->num_pbls; i++) {
0513         if (!pbl[i].va)
0514             continue;
0515         dma_free_coherent(&pdev->dev, pbl_info->pbl_size,
0516                   pbl[i].va, pbl[i].pa);
0517     }
0518
0519     kfree(pbl);
0520 }
0521
0522 #define MIN_FW_PBL_PAGE_SIZE (4 * 1024)
0523 #define MAX_FW_PBL_PAGE_SIZE (64 * 1024)
0524
0525 #define NUM_PBES_ON_PAGE(_page_size) (_page_size / sizeof(u64))
0526 #define MAX_PBES_ON_PAGE NUM_PBES_ON_PAGE(MAX_FW_PBL_PAGE_SIZE)
0527 #define MAX_PBES_TWO_LAYER (MAX_PBES_ON_PAGE * MAX_PBES_ON_PAGE)
0528
0529 static struct qedr_pbl *qedr_alloc_pbl_tbl(struct qedr_dev *dev,
0530                        struct qedr_pbl_info *pbl_info,
0531                        gfp_t flags)
0532 {
0533     struct pci_dev *pdev = dev->pdev;
0534     struct qedr_pbl *pbl_table;
0535     dma_addr_t *pbl_main_tbl;
0536     dma_addr_t pa;
0537     void *va;
0538     int i;
0539
0540     pbl_table = kcalloc(pbl_info->num_pbls, sizeof(*pbl_table), flags);
0541     if (!pbl_table)
0542         return ERR_PTR(-ENOMEM);
0543
0544     for (i = 0; i < pbl_info->num_pbls; i++) {
0545         va = dma_alloc_coherent(&pdev->dev, pbl_info->pbl_size, &pa,
0546                     flags);
0547         if (!va)
0548             goto err;
0549
0550         pbl_table[i].va = va;
0551         pbl_table[i].pa = pa;
0552     }
0553
0554     /* Two-Layer PBLs, if we have more than one pbl we need to initialize
0555      * the first one with physical pointers to all of the rest
0556      */
0557     pbl_main_tbl = (dma_addr_t *)pbl_table[0].va;
0558     for (i = 0; i < pbl_info->num_pbls - 1; i++)
0559         pbl_main_tbl[i] = pbl_table[i + 1].pa;
0560
0561     return pbl_table;
0562
0563 err:
0564     for (i--; i >= 0; i--)
0565         dma_free_coherent(&pdev->dev, pbl_info->pbl_size,
0566                   pbl_table[i].va, pbl_table[i].pa);
0567
0568     qedr_free_pbl(dev, pbl_info, pbl_table);
0569
0570     return ERR_PTR(-ENOMEM);
0571 }
0572
0573 static int qedr_prepare_pbl_tbl(struct qedr_dev *dev,
0574                 struct qedr_pbl_info *pbl_info,
0575                 u32 num_pbes, int two_layer_capable)
0576 {
0577     u32 pbl_capacity;
0578     u32 pbl_size;
0579     u32 num_pbls;
0580
0581     if ((num_pbes > MAX_PBES_ON_PAGE) && two_layer_capable) {
0582         if (num_pbes > MAX_PBES_TWO_LAYER) {
0583             DP_ERR(dev, "prepare pbl table: too many pages %d\n",
0584                    num_pbes);
0585             return -EINVAL;
0586         }
0587
0588         /* calculate required pbl page size */
0589         pbl_size = MIN_FW_PBL_PAGE_SIZE;
0590         pbl_capacity = NUM_PBES_ON_PAGE(pbl_size) *
0591                    NUM_PBES_ON_PAGE(pbl_size);
0592
0593         while (pbl_capacity < num_pbes) {
0594             pbl_size *= 2;
0595             pbl_capacity = pbl_size / sizeof(u64);
0596             pbl_capacity = pbl_capacity * pbl_capacity;
0597         }
0598
0599         num_pbls = DIV_ROUND_UP(num_pbes, NUM_PBES_ON_PAGE(pbl_size));
0600         num_pbls++; /* One for the layer0 ( points to the pbls) */
0601         pbl_info->two_layered = true;
0602     } else {
0603         /* One layered PBL */
0604         num_pbls = 1;
0605         pbl_size = max_t(u32, MIN_FW_PBL_PAGE_SIZE,
0606                  roundup_pow_of_two((num_pbes * sizeof(u64))));
0607         pbl_info->two_layered = false;
0608     }
0609
0610     pbl_info->num_pbls = num_pbls;
0611     pbl_info->pbl_size = pbl_size;
0612     pbl_info->num_pbes = num_pbes;
0613
0614     DP_DEBUG(dev, QEDR_MSG_MR,
0615          "prepare pbl table: num_pbes=%d, num_pbls=%d, pbl_size=%d\n",
0616          pbl_info->num_pbes, pbl_info->num_pbls, pbl_info->pbl_size);
0617
0618     return 0;
0619 }
0620
0621 static void qedr_populate_pbls(struct qedr_dev *dev, struct ib_umem *umem,
0622                    struct qedr_pbl *pbl,
0623                    struct qedr_pbl_info *pbl_info, u32 pg_shift)
0624 {
0625     int pbe_cnt, total_num_pbes = 0;
0626     struct qedr_pbl *pbl_tbl;
0627     struct ib_block_iter biter;
0628     struct regpair *pbe;
0629
0630     if (!pbl_info->num_pbes)
0631         return;
0632
0633     /* If we have a two layered pbl, the first pbl points to the rest
0634      * of the pbls and the first entry lays on the second pbl in the table
0635      */
0636     if (pbl_info->two_layered)
0637         pbl_tbl = &pbl[1];
0638     else
0639         pbl_tbl = pbl;
0640
0641     pbe = (struct regpair *)pbl_tbl->va;
0642     if (!pbe) {
0643         DP_ERR(dev, "cannot populate PBL due to a NULL PBE\n");
0644         return;
0645     }
0646
0647     pbe_cnt = 0;
0648
0649     rdma_umem_for_each_dma_block (umem, &biter, BIT(pg_shift)) {
0650         u64 pg_addr = rdma_block_iter_dma_address(&biter);
0651
0652         pbe->lo = cpu_to_le32(pg_addr);
0653         pbe->hi = cpu_to_le32(upper_32_bits(pg_addr));
0654
0655         pbe_cnt++;
0656         total_num_pbes++;
0657         pbe++;
0658
0659         if (total_num_pbes == pbl_info->num_pbes)
0660             return;
0661
0662         /* If the given pbl is full storing the pbes, move to next pbl.
0663          */
0664         if (pbe_cnt == (pbl_info->pbl_size / sizeof(u64))) {
0665             pbl_tbl++;
0666             pbe = (struct regpair *)pbl_tbl->va;
0667             pbe_cnt = 0;
0668         }
0669     }
0670 }
0671
0672 static int qedr_db_recovery_add(struct qedr_dev *dev,
0673                 void __iomem *db_addr,
0674                 void *db_data,
0675                 enum qed_db_rec_width db_width,
0676                 enum qed_db_rec_space db_space)
0677 {
0678     if (!db_data) {
0679         DP_DEBUG(dev, QEDR_MSG_INIT, "avoiding db rec since old lib\n");
0680         return 0;
0681     }
0682
0683     return dev->ops->common->db_recovery_add(dev->cdev, db_addr, db_data,
0684                          db_width, db_space);
0685 }
0686
0687 static void qedr_db_recovery_del(struct qedr_dev *dev,
0688                  void __iomem *db_addr,
0689                  void *db_data)
0690 {
0691     if (!db_data) {
0692         DP_DEBUG(dev, QEDR_MSG_INIT, "avoiding db rec since old lib\n");
0693         return;
0694     }
0695
0696     /* Ignore return code as there is not much we can do about it. Error
0697      * log will be printed inside.
0698      */
0699     dev->ops->common->db_recovery_del(dev->cdev, db_addr, db_data);
0700 }
0701
0702 static int qedr_copy_cq_uresp(struct qedr_dev *dev,
0703                   struct qedr_cq *cq, struct ib_udata *udata,
0704                   u32 db_offset)
0705 {
0706     struct qedr_create_cq_uresp uresp;
0707     int rc;
0708
0709     memset(&uresp, 0, sizeof(uresp));
0710
0711     uresp.db_offset = db_offset;
0712     uresp.icid = cq->icid;
0713     if (cq->q.db_mmap_entry)
0714         uresp.db_rec_addr =
0715             rdma_user_mmap_get_offset(cq->q.db_mmap_entry);
0716
0717     rc = qedr_ib_copy_to_udata(udata, &uresp, sizeof(uresp));
0718     if (rc)
0719         DP_ERR(dev, "copy error cqid=0x%x.\n", cq->icid);
0720
0721     return rc;
0722 }
0723
0724 static void consume_cqe(struct qedr_cq *cq)
0725 {
0726     if (cq->latest_cqe == cq->toggle_cqe)
0727         cq->pbl_toggle ^= RDMA_CQE_REQUESTER_TOGGLE_BIT_MASK;
0728
0729     cq->latest_cqe = qed_chain_consume(&cq->pbl);
0730 }
0731
0732 static inline int qedr_align_cq_entries(int entries)
0733 {
0734     u64 size, aligned_size;
0735
0736     /* We allocate an extra entry that we don't report to the FW. */
0737     size = (entries + 1) * QEDR_CQE_SIZE;
0738     aligned_size = ALIGN(size, PAGE_SIZE);
0739
0740     return aligned_size / QEDR_CQE_SIZE;
0741 }
0742
0743 static int qedr_init_user_db_rec(struct ib_udata *udata,
0744                  struct qedr_dev *dev, struct qedr_userq *q,
0745                  bool requires_db_rec)
0746 {
0747     struct qedr_ucontext *uctx =
0748         rdma_udata_to_drv_context(udata, struct qedr_ucontext,
0749                       ibucontext);
0750     struct qedr_user_mmap_entry *entry;
0751     int rc;
0752
0753     /* Aborting for non doorbell userqueue (SRQ) or non-supporting lib */
0754     if (requires_db_rec == 0 || !uctx->db_rec)
0755         return 0;
0756
0757     /* Allocate a page for doorbell recovery, add to mmap */
0758     q->db_rec_data = (void *)get_zeroed_page(GFP_USER);
0759     if (!q->db_rec_data) {
0760         DP_ERR(dev, "get_zeroed_page failed\n");
0761         return -ENOMEM;
0762     }
0763
0764     entry = kzalloc(sizeof(*entry), GFP_KERNEL);
0765     if (!entry)
0766         goto err_free_db_data;
0767
0768     entry->address = q->db_rec_data;
0769     entry->length = PAGE_SIZE;
0770     entry->mmap_flag = QEDR_USER_MMAP_PHYS_PAGE;
0771     rc = rdma_user_mmap_entry_insert(&uctx->ibucontext,
0772                      &entry->rdma_entry,
0773                      PAGE_SIZE);
0774     if (rc)
0775         goto err_free_entry;
0776
0777     q->db_mmap_entry = &entry->rdma_entry;
0778
0779     return 0;
0780
0781 err_free_entry:
0782     kfree(entry);
0783
0784 err_free_db_data:
0785     free_page((unsigned long)q->db_rec_data);
0786     q->db_rec_data = NULL;
0787     return -ENOMEM;
0788 }
0789
0790 static inline int qedr_init_user_queue(struct ib_udata *udata,
0791                        struct qedr_dev *dev,
0792                        struct qedr_userq *q, u64 buf_addr,
0793                        size_t buf_len, bool requires_db_rec,
0794                        int access,
0795                        int alloc_and_init)
0796 {
0797     u32 fw_pages;
0798     int rc;
0799
0800     q->buf_addr = buf_addr;
0801     q->buf_len = buf_len;
0802     q->umem = ib_umem_get(&dev->ibdev, q->buf_addr, q->buf_len, access);
0803     if (IS_ERR(q->umem)) {
0804         DP_ERR(dev, "create user queue: failed ib_umem_get, got %ld\n",
0805                PTR_ERR(q->umem));
0806         return PTR_ERR(q->umem);
0807     }
0808
0809     fw_pages = ib_umem_num_dma_blocks(q->umem, 1 << FW_PAGE_SHIFT);
0810     rc = qedr_prepare_pbl_tbl(dev, &q->pbl_info, fw_pages, 0);
0811     if (rc)
0812         goto err0;
0813
0814     if (alloc_and_init) {
0815         q->pbl_tbl = qedr_alloc_pbl_tbl(dev, &q->pbl_info, GFP_KERNEL);
0816         if (IS_ERR(q->pbl_tbl)) {
0817             rc = PTR_ERR(q->pbl_tbl);
0818             goto err0;
0819         }
0820         qedr_populate_pbls(dev, q->umem, q->pbl_tbl, &q->pbl_info,
0821                    FW_PAGE_SHIFT);
0822     } else {
0823         q->pbl_tbl = kzalloc(sizeof(*q->pbl_tbl), GFP_KERNEL);
0824         if (!q->pbl_tbl) {
0825             rc = -ENOMEM;
0826             goto err0;
0827         }
0828     }
0829
0830     /* mmap the user address used to store doorbell data for recovery */
0831     return qedr_init_user_db_rec(udata, dev, q, requires_db_rec);
0832
0833 err0:
0834     ib_umem_release(q->umem);
0835     q->umem = NULL;
0836
0837     return rc;
0838 }
0839
0840 static inline void qedr_init_cq_params(struct qedr_cq *cq,
0841                        struct qedr_ucontext *ctx,
0842                        struct qedr_dev *dev, int vector,
0843                        int chain_entries, int page_cnt,
0844                        u64 pbl_ptr,
0845                        struct qed_rdma_create_cq_in_params
0846                        *params)
0847 {
0848     memset(params, 0, sizeof(*params));
0849     params->cq_handle_hi = upper_32_bits((uintptr_t)cq);
0850     params->cq_handle_lo = lower_32_bits((uintptr_t)cq);
0851     params->cnq_id = vector;
0852     params->cq_size = chain_entries - 1;
0853     params->dpi = (ctx) ? ctx->dpi : dev->dpi;
0854     params->pbl_num_pages = page_cnt;
0855     params->pbl_ptr = pbl_ptr;
0856     params->pbl_two_level = 0;
0857 }
0858
0859 static void doorbell_cq(struct qedr_cq *cq, u32 cons, u8 flags)
0860 {
0861     cq->db.data.agg_flags = flags;
0862     cq->db.data.value = cpu_to_le32(cons);
0863     writeq(cq->db.raw, cq->db_addr);
0864 }
0865
0866 int qedr_arm_cq(struct ib_cq *ibcq, enum ib_cq_notify_flags flags)
0867 {
0868     struct qedr_cq *cq = get_qedr_cq(ibcq);
0869     unsigned long sflags;
0870     struct qedr_dev *dev;
0871
0872     dev = get_qedr_dev(ibcq->device);
0873
0874     if (cq->destroyed) {
0875         DP_ERR(dev,
0876                "warning: arm was invoked after destroy for cq %p (icid=%d)\n",
0877                cq, cq->icid);
0878         return -EINVAL;
0879     }
0880
0881
0882     if (cq->cq_type == QEDR_CQ_TYPE_GSI)
0883         return 0;
0884
0885     spin_lock_irqsave(&cq->cq_lock, sflags);
0886
0887     cq->arm_flags = 0;
0888
0889     if (flags & IB_CQ_SOLICITED)
0890         cq->arm_flags |= DQ_UCM_ROCE_CQ_ARM_SE_CF_CMD;
0891
0892     if (flags & IB_CQ_NEXT_COMP)
0893         cq->arm_flags |= DQ_UCM_ROCE_CQ_ARM_CF_CMD;
0894
0895     doorbell_cq(cq, cq->cq_cons - 1, cq->arm_flags);
0896
0897     spin_unlock_irqrestore(&cq->cq_lock, sflags);
0898
0899     return 0;
0900 }
0901
0902 int qedr_create_cq(struct ib_cq *ibcq, const struct ib_cq_init_attr *attr,
0903            struct ib_udata *udata)
0904 {
0905     struct ib_device *ibdev = ibcq->device;
0906     struct qedr_ucontext *ctx = rdma_udata_to_drv_context(
0907         udata, struct qedr_ucontext, ibucontext);
0908     struct qed_rdma_destroy_cq_out_params destroy_oparams;
0909     struct qed_rdma_destroy_cq_in_params destroy_iparams;
0910     struct qed_chain_init_params chain_params = {
0911         .mode       = QED_CHAIN_MODE_PBL,
0912         .intended_use   = QED_CHAIN_USE_TO_CONSUME,
0913         .cnt_type   = QED_CHAIN_CNT_TYPE_U32,
0914         .elem_size  = sizeof(union rdma_cqe),
0915     };
0916     struct qedr_dev *dev = get_qedr_dev(ibdev);
0917     struct qed_rdma_create_cq_in_params params;
0918     struct qedr_create_cq_ureq ureq = {};
0919     int vector = attr->comp_vector;
0920     int entries = attr->cqe;
0921     struct qedr_cq *cq = get_qedr_cq(ibcq);
0922     int chain_entries;
0923     u32 db_offset;
0924     int page_cnt;
0925     u64 pbl_ptr;
0926     u16 icid;
0927     int rc;
0928
0929     DP_DEBUG(dev, QEDR_MSG_INIT,
0930          "create_cq: called from %s. entries=%d, vector=%d\n",
0931          udata ? "User Lib" : "Kernel", entries, vector);
0932
0933     if (attr->flags)
0934         return -EOPNOTSUPP;
0935
0936     if (entries > QEDR_MAX_CQES) {
0937         DP_ERR(dev,
0938                "create cq: the number of entries %d is too high. Must be equal or below %d.\n",
0939                entries, QEDR_MAX_CQES);
0940         return -EINVAL;
0941     }
0942
0943     chain_entries = qedr_align_cq_entries(entries);
0944     chain_entries = min_t(int, chain_entries, QEDR_MAX_CQES);
0945     chain_params.num_elems = chain_entries;
0946
0947     /* calc db offset. user will add DPI base, kernel will add db addr */
0948     db_offset = DB_ADDR_SHIFT(DQ_PWM_OFFSET_UCM_RDMA_CQ_CONS_32BIT);
0949
0950     if (udata) {
0951         if (ib_copy_from_udata(&ureq, udata, min(sizeof(ureq),
0952                              udata->inlen))) {
0953             DP_ERR(dev,
0954                    "create cq: problem copying data from user space\n");
0955             goto err0;
0956         }
0957
0958         if (!ureq.len) {
0959             DP_ERR(dev,
0960                    "create cq: cannot create a cq with 0 entries\n");
0961             goto err0;
0962         }
0963
0964         cq->cq_type = QEDR_CQ_TYPE_USER;
0965
0966         rc = qedr_init_user_queue(udata, dev, &cq->q, ureq.addr,
0967                       ureq.len, true, IB_ACCESS_LOCAL_WRITE,
0968                       1);
0969         if (rc)
0970             goto err0;
0971
0972         pbl_ptr = cq->q.pbl_tbl->pa;
0973         page_cnt = cq->q.pbl_info.num_pbes;
0974
0975         cq->ibcq.cqe = chain_entries;
0976         cq->q.db_addr = ctx->dpi_addr + db_offset;
0977     } else {
0978         cq->cq_type = QEDR_CQ_TYPE_KERNEL;
0979
0980         rc = dev->ops->common->chain_alloc(dev->cdev, &cq->pbl,
0981                            &chain_params);
0982         if (rc)
0983             goto err0;
0984
0985         page_cnt = qed_chain_get_page_cnt(&cq->pbl);
0986         pbl_ptr = qed_chain_get_pbl_phys(&cq->pbl);
0987         cq->ibcq.cqe = cq->pbl.capacity;
0988     }
0989
0990     qedr_init_cq_params(cq, ctx, dev, vector, chain_entries, page_cnt,
0991                 pbl_ptr, &params);
0992
0993     rc = dev->ops->rdma_create_cq(dev->rdma_ctx, &params, &icid);
0994     if (rc)
0995         goto err1;
0996
0997     cq->icid = icid;
0998     cq->sig = QEDR_CQ_MAGIC_NUMBER;
0999     spin_lock_init(&cq->cq_lock);
1000
1001     if (udata) {
1002         rc = qedr_copy_cq_uresp(dev, cq, udata, db_offset);
1003         if (rc)
1004             goto err2;
1005
1006         rc = qedr_db_recovery_add(dev, cq->q.db_addr,
1007                       &cq->q.db_rec_data->db_data,
1008                       DB_REC_WIDTH_64B,
1009                       DB_REC_USER);
1010         if (rc)
1011             goto err2;
1012
1013     } else {
1014         /* Generate doorbell address. */
1015         cq->db.data.icid = cq->icid;
1016         cq->db_addr = dev->db_addr + db_offset;
1017         cq->db.data.params = DB_AGG_CMD_MAX <<
1018             RDMA_PWM_VAL32_DATA_AGG_CMD_SHIFT;
1019
1020         /* point to the very last element, passing it we will toggle */
1021         cq->toggle_cqe = qed_chain_get_last_elem(&cq->pbl);
1022         cq->pbl_toggle = RDMA_CQE_REQUESTER_TOGGLE_BIT_MASK;
1023         cq->latest_cqe = NULL;
1024         consume_cqe(cq);
1025         cq->cq_cons = qed_chain_get_cons_idx_u32(&cq->pbl);
1026
1027         rc = qedr_db_recovery_add(dev, cq->db_addr, &cq->db.data,
1028                       DB_REC_WIDTH_64B, DB_REC_KERNEL);
1029         if (rc)
1030             goto err2;
1031     }
1032
1033     DP_DEBUG(dev, QEDR_MSG_CQ,
1034          "create cq: icid=0x%0x, addr=%p, size(entries)=0x%0x\n",
1035          cq->icid, cq, params.cq_size);
1036
1037     return 0;
1038
1039 err2:
1040     destroy_iparams.icid = cq->icid;
1041     dev->ops->rdma_destroy_cq(dev->rdma_ctx, &destroy_iparams,
1042                   &destroy_oparams);
1043 err1:
1044     if (udata) {
1045         qedr_free_pbl(dev, &cq->q.pbl_info, cq->q.pbl_tbl);
1046         ib_umem_release(cq->q.umem);
1047         if (cq->q.db_mmap_entry)
1048             rdma_user_mmap_entry_remove(cq->q.db_mmap_entry);
1049     } else {
1050         dev->ops->common->chain_free(dev->cdev, &cq->pbl);
1051     }
1052 err0:
1053     return -EINVAL;
1054 }
1055
1056 #define QEDR_DESTROY_CQ_MAX_ITERATIONS      (10)
1057 #define QEDR_DESTROY_CQ_ITER_DURATION       (10)
1058
1059 int qedr_destroy_cq(struct ib_cq *ibcq, struct ib_udata *udata)
1060 {
1061     struct qedr_dev *dev = get_qedr_dev(ibcq->device);
1062     struct qed_rdma_destroy_cq_out_params oparams;
1063     struct qed_rdma_destroy_cq_in_params iparams;
1064     struct qedr_cq *cq = get_qedr_cq(ibcq);
1065     int iter;
1066
1067     DP_DEBUG(dev, QEDR_MSG_CQ, "destroy cq %p (icid=%d)\n", cq, cq->icid);
1068
1069     cq->destroyed = 1;
1070
1071     /* GSIs CQs are handled by driver, so they don't exist in the FW */
1072     if (cq->cq_type == QEDR_CQ_TYPE_GSI) {
1073         qedr_db_recovery_del(dev, cq->db_addr, &cq->db.data);
1074         return 0;
1075     }
1076
1077     iparams.icid = cq->icid;
1078     dev->ops->rdma_destroy_cq(dev->rdma_ctx, &iparams, &oparams);
1079     dev->ops->common->chain_free(dev->cdev, &cq->pbl);
1080
1081     if (udata) {
1082         qedr_free_pbl(dev, &cq->q.pbl_info, cq->q.pbl_tbl);
1083         ib_umem_release(cq->q.umem);
1084
1085         if (cq->q.db_rec_data) {
1086             qedr_db_recovery_del(dev, cq->q.db_addr,
1087                          &cq->q.db_rec_data->db_data);
1088             rdma_user_mmap_entry_remove(cq->q.db_mmap_entry);
1089         }
1090     } else {
1091         qedr_db_recovery_del(dev, cq->db_addr, &cq->db.data);
1092     }
1093
1094     /* We don't want the IRQ handler to handle a non-existing CQ so we
1095      * wait until all CNQ interrupts, if any, are received. This will always
1096      * happen and will always happen very fast. If not, then a serious error
1097      * has occured. That is why we can use a long delay.
1098      * We spin for a short time so we don’t lose time on context switching
1099      * in case all the completions are handled in that span. Otherwise
1100      * we sleep for a while and check again. Since the CNQ may be
1101      * associated with (only) the current CPU we use msleep to allow the
1102      * current CPU to be freed.
1103      * The CNQ notification is increased in qedr_irq_handler().
1104      */
1105     iter = QEDR_DESTROY_CQ_MAX_ITERATIONS;
1106     while (oparams.num_cq_notif != READ_ONCE(cq->cnq_notif) && iter) {
1107         udelay(QEDR_DESTROY_CQ_ITER_DURATION);
1108         iter--;
1109     }
1110
1111     iter = QEDR_DESTROY_CQ_MAX_ITERATIONS;
1112     while (oparams.num_cq_notif != READ_ONCE(cq->cnq_notif) && iter) {
1113         msleep(QEDR_DESTROY_CQ_ITER_DURATION);
1114         iter--;
1115     }
1116
1117     /* Note that we don't need to have explicit code to wait for the
1118      * completion of the event handler because it is invoked from the EQ.
1119      * Since the destroy CQ ramrod has also been received on the EQ we can
1120      * be certain that there's no event handler in process.
1121      */
1122     return 0;
1123 }
1124
1125 static inline int get_gid_info_from_table(struct ib_qp *ibqp,
1126                       struct ib_qp_attr *attr,
1127                       int attr_mask,
1128                       struct qed_rdma_modify_qp_in_params
1129                       *qp_params)
1130 {
1131     const struct ib_gid_attr *gid_attr;
1132     enum rdma_network_type nw_type;
1133     const struct ib_global_route *grh = rdma_ah_read_grh(&attr->ah_attr);
1134     u32 ipv4_addr;
1135     int ret;
1136     int i;
1137
1138     gid_attr = grh->sgid_attr;
1139     ret = rdma_read_gid_l2_fields(gid_attr, &qp_params->vlan_id, NULL);
1140     if (ret)
1141         return ret;
1142
1143     nw_type = rdma_gid_attr_network_type(gid_attr);
1144     switch (nw_type) {
1145     case RDMA_NETWORK_IPV6:
1146         memcpy(&qp_params->sgid.bytes[0], &gid_attr->gid.raw[0],
1147                sizeof(qp_params->sgid));
1148         memcpy(&qp_params->dgid.bytes[0],
1149                &grh->dgid,
1150                sizeof(qp_params->dgid));
1151         qp_params->roce_mode = ROCE_V2_IPV6;
1152         SET_FIELD(qp_params->modify_flags,
1153               QED_ROCE_MODIFY_QP_VALID_ROCE_MODE, 1);
1154         break;
1155     case RDMA_NETWORK_ROCE_V1:
1156         memcpy(&qp_params->sgid.bytes[0], &gid_attr->gid.raw[0],
1157                sizeof(qp_params->sgid));
1158         memcpy(&qp_params->dgid.bytes[0],
1159                &grh->dgid,
1160                sizeof(qp_params->dgid));
1161         qp_params->roce_mode = ROCE_V1;
1162         break;
1163     case RDMA_NETWORK_IPV4:
1164         memset(&qp_params->sgid, 0, sizeof(qp_params->sgid));
1165         memset(&qp_params->dgid, 0, sizeof(qp_params->dgid));
1166         ipv4_addr = qedr_get_ipv4_from_gid(gid_attr->gid.raw);
1167         qp_params->sgid.ipv4_addr = ipv4_addr;
1168         ipv4_addr =
1169             qedr_get_ipv4_from_gid(grh->dgid.raw);
1170         qp_params->dgid.ipv4_addr = ipv4_addr;
1171         SET_FIELD(qp_params->modify_flags,
1172               QED_ROCE_MODIFY_QP_VALID_ROCE_MODE, 1);
1173         qp_params->roce_mode = ROCE_V2_IPV4;
1174         break;
1175     default:
1176         return -EINVAL;
1177     }
1178
1179     for (i = 0; i < 4; i++) {
1180         qp_params->sgid.dwords[i] = ntohl(qp_params->sgid.dwords[i]);
1181         qp_params->dgid.dwords[i] = ntohl(qp_params->dgid.dwords[i]);
1182     }
1183
1184     if (qp_params->vlan_id >= VLAN_CFI_MASK)
1185         qp_params->vlan_id = 0;
1186
1187     return 0;
1188 }
1189
1190 static int qedr_check_qp_attrs(struct ib_pd *ibpd, struct qedr_dev *dev,
1191                    struct ib_qp_init_attr *attrs,
1192                    struct ib_udata *udata)
1193 {
1194     struct qedr_device_attr *qattr = &dev->attr;
1195
1196     /* QP0... attrs->qp_type == IB_QPT_GSI */
1197     if (attrs->qp_type != IB_QPT_RC &&
1198         attrs->qp_type != IB_QPT_GSI &&
1199         attrs->qp_type != IB_QPT_XRC_INI &&
1200         attrs->qp_type != IB_QPT_XRC_TGT) {
1201         DP_DEBUG(dev, QEDR_MSG_QP,
1202              "create qp: unsupported qp type=0x%x requested\n",
1203              attrs->qp_type);
1204         return -EOPNOTSUPP;
1205     }
1206
1207     if (attrs->cap.max_send_wr > qattr->max_sqe) {
1208         DP_ERR(dev,
1209                "create qp: cannot create a SQ with %d elements (max_send_wr=0x%x)\n",
1210                attrs->cap.max_send_wr, qattr->max_sqe);
1211         return -EINVAL;
1212     }
1213
1214     if (attrs->cap.max_inline_data > qattr->max_inline) {
1215         DP_ERR(dev,
1216                "create qp: unsupported inline data size=0x%x requested (max_inline=0x%x)\n",
1217                attrs->cap.max_inline_data, qattr->max_inline);
1218         return -EINVAL;
1219     }
1220
1221     if (attrs->cap.max_send_sge > qattr->max_sge) {
1222         DP_ERR(dev,
1223                "create qp: unsupported send_sge=0x%x requested (max_send_sge=0x%x)\n",
1224                attrs->cap.max_send_sge, qattr->max_sge);
1225         return -EINVAL;
1226     }
1227
1228     if (attrs->cap.max_recv_sge > qattr->max_sge) {
1229         DP_ERR(dev,
1230                "create qp: unsupported recv_sge=0x%x requested (max_recv_sge=0x%x)\n",
1231                attrs->cap.max_recv_sge, qattr->max_sge);
1232         return -EINVAL;
1233     }
1234
1235     /* verify consumer QPs are not trying to use GSI QP's CQ.
1236      * TGT QP isn't associated with RQ/SQ
1237      */
1238     if ((attrs->qp_type != IB_QPT_GSI) && (dev->gsi_qp_created) &&
1239         (attrs->qp_type != IB_QPT_XRC_TGT) &&
1240         (attrs->qp_type != IB_QPT_XRC_INI)) {
1241         struct qedr_cq *send_cq = get_qedr_cq(attrs->send_cq);
1242         struct qedr_cq *recv_cq = get_qedr_cq(attrs->recv_cq);
1243
1244         if ((send_cq->cq_type == QEDR_CQ_TYPE_GSI) ||
1245             (recv_cq->cq_type == QEDR_CQ_TYPE_GSI)) {
1246             DP_ERR(dev,
1247                    "create qp: consumer QP cannot use GSI CQs.\n");
1248             return -EINVAL;
1249         }
1250     }
1251
1252     return 0;
1253 }
1254
1255 static int qedr_copy_srq_uresp(struct qedr_dev *dev,
1256                    struct qedr_srq *srq, struct ib_udata *udata)
1257 {
1258     struct qedr_create_srq_uresp uresp = {};
1259     int rc;
1260
1261     uresp.srq_id = srq->srq_id;
1262
1263     rc = ib_copy_to_udata(udata, &uresp, sizeof(uresp));
1264     if (rc)
1265         DP_ERR(dev, "create srq: problem copying data to user space\n");
1266
1267     return rc;
1268 }
1269
1270 static void qedr_copy_rq_uresp(struct qedr_dev *dev,
1271                    struct qedr_create_qp_uresp *uresp,
1272                    struct qedr_qp *qp)
1273 {
1274     /* iWARP requires two doorbells per RQ. */
1275     if (rdma_protocol_iwarp(&dev->ibdev, 1)) {
1276         uresp->rq_db_offset =
1277             DB_ADDR_SHIFT(DQ_PWM_OFFSET_TCM_IWARP_RQ_PROD);
1278         uresp->rq_db2_offset = DB_ADDR_SHIFT(DQ_PWM_OFFSET_TCM_FLAGS);
1279     } else {
1280         uresp->rq_db_offset =
1281             DB_ADDR_SHIFT(DQ_PWM_OFFSET_TCM_ROCE_RQ_PROD);
1282     }
1283
1284     uresp->rq_icid = qp->icid;
1285     if (qp->urq.db_mmap_entry)
1286         uresp->rq_db_rec_addr =
1287             rdma_user_mmap_get_offset(qp->urq.db_mmap_entry);
1288 }
1289
1290 static void qedr_copy_sq_uresp(struct qedr_dev *dev,
1291                    struct qedr_create_qp_uresp *uresp,
1292                    struct qedr_qp *qp)
1293 {
1294     uresp->sq_db_offset = DB_ADDR_SHIFT(DQ_PWM_OFFSET_XCM_RDMA_SQ_PROD);
1295
1296     /* iWARP uses the same cid for rq and sq */
1297     if (rdma_protocol_iwarp(&dev->ibdev, 1))
1298         uresp->sq_icid = qp->icid;
1299     else
1300         uresp->sq_icid = qp->icid + 1;
1301
1302     if (qp->usq.db_mmap_entry)
1303         uresp->sq_db_rec_addr =
1304             rdma_user_mmap_get_offset(qp->usq.db_mmap_entry);
1305 }
1306
1307 static int qedr_copy_qp_uresp(struct qedr_dev *dev,
1308                   struct qedr_qp *qp, struct ib_udata *udata,
1309                   struct qedr_create_qp_uresp *uresp)
1310 {
1311     int rc;
1312
1313     memset(uresp, 0, sizeof(*uresp));
1314
1315     if (qedr_qp_has_sq(qp))
1316         qedr_copy_sq_uresp(dev, uresp, qp);
1317
1318     if (qedr_qp_has_rq(qp))
1319         qedr_copy_rq_uresp(dev, uresp, qp);
1320
1321     uresp->atomic_supported = dev->atomic_cap != IB_ATOMIC_NONE;
1322     uresp->qp_id = qp->qp_id;
1323
1324     rc = qedr_ib_copy_to_udata(udata, uresp, sizeof(*uresp));
1325     if (rc)
1326         DP_ERR(dev,
1327                "create qp: failed a copy to user space with qp icid=0x%x.\n",
1328                qp->icid);
1329
1330     return rc;
1331 }
1332
1333 static void qedr_reset_qp_hwq_info(struct qedr_qp_hwq_info *qph)
1334 {
1335     qed_chain_reset(&qph->pbl);
1336     qph->prod = 0;
1337     qph->cons = 0;
1338     qph->wqe_cons = 0;
1339     qph->db_data.data.value = cpu_to_le16(0);
1340 }
1341
1342 static void qedr_set_common_qp_params(struct qedr_dev *dev,
1343                       struct qedr_qp *qp,
1344                       struct qedr_pd *pd,
1345                       struct ib_qp_init_attr *attrs)
1346 {
1347     spin_lock_init(&qp->q_lock);
1348     if (rdma_protocol_iwarp(&dev->ibdev, 1)) {
1349         kref_init(&qp->refcnt);
1350         init_completion(&qp->iwarp_cm_comp);
1351         init_completion(&qp->qp_rel_comp);
1352     }
1353
1354     qp->pd = pd;
1355     qp->qp_type = attrs->qp_type;
1356     qp->max_inline_data = attrs->cap.max_inline_data;
1357     qp->state = QED_ROCE_QP_STATE_RESET;
1358
1359     qp->prev_wqe_size = 0;
1360
1361     qp->signaled = (attrs->sq_sig_type == IB_SIGNAL_ALL_WR) ? true : false;
1362     qp->dev = dev;
1363     if (qedr_qp_has_sq(qp)) {
1364         qedr_reset_qp_hwq_info(&qp->sq);
1365         qp->sq.max_sges = attrs->cap.max_send_sge;
1366         qp->sq_cq = get_qedr_cq(attrs->send_cq);
1367         DP_DEBUG(dev, QEDR_MSG_QP,
1368              "SQ params:\tsq_max_sges = %d, sq_cq_id = %d\n",
1369              qp->sq.max_sges, qp->sq_cq->icid);
1370     }
1371
1372     if (attrs->srq)
1373         qp->srq = get_qedr_srq(attrs->srq);
1374
1375     if (qedr_qp_has_rq(qp)) {
1376         qedr_reset_qp_hwq_info(&qp->rq);
1377         qp->rq_cq = get_qedr_cq(attrs->recv_cq);
1378         qp->rq.max_sges = attrs->cap.max_recv_sge;
1379         DP_DEBUG(dev, QEDR_MSG_QP,
1380              "RQ params:\trq_max_sges = %d, rq_cq_id = %d\n",
1381              qp->rq.max_sges, qp->rq_cq->icid);
1382     }
1383
1384     DP_DEBUG(dev, QEDR_MSG_QP,
1385          "QP params:\tpd = %d, qp_type = %d, max_inline_data = %d, state = %d, signaled = %d, use_srq=%d\n",
1386          pd->pd_id, qp->qp_type, qp->max_inline_data,
1387          qp->state, qp->signaled, (attrs->srq) ? 1 : 0);
1388     DP_DEBUG(dev, QEDR_MSG_QP,
1389          "SQ params:\tsq_max_sges = %d, sq_cq_id = %d\n",
1390          qp->sq.max_sges, qp->sq_cq->icid);
1391 }
1392
1393 static int qedr_set_roce_db_info(struct qedr_dev *dev, struct qedr_qp *qp)
1394 {
1395     int rc = 0;
1396
1397     if (qedr_qp_has_sq(qp)) {
1398         qp->sq.db = dev->db_addr +
1399                 DB_ADDR_SHIFT(DQ_PWM_OFFSET_XCM_RDMA_SQ_PROD);
1400         qp->sq.db_data.data.icid = qp->icid + 1;
1401         rc = qedr_db_recovery_add(dev, qp->sq.db, &qp->sq.db_data,
1402                       DB_REC_WIDTH_32B, DB_REC_KERNEL);
1403         if (rc)
1404             return rc;
1405     }
1406
1407     if (qedr_qp_has_rq(qp)) {
1408         qp->rq.db = dev->db_addr +
1409                 DB_ADDR_SHIFT(DQ_PWM_OFFSET_TCM_ROCE_RQ_PROD);
1410         qp->rq.db_data.data.icid = qp->icid;
1411         rc = qedr_db_recovery_add(dev, qp->rq.db, &qp->rq.db_data,
1412                       DB_REC_WIDTH_32B, DB_REC_KERNEL);
1413         if (rc && qedr_qp_has_sq(qp))
1414             qedr_db_recovery_del(dev, qp->sq.db, &qp->sq.db_data);
1415     }
1416
1417     return rc;
1418 }
1419
1420 static int qedr_check_srq_params(struct qedr_dev *dev,
1421                  struct ib_srq_init_attr *attrs,
1422                  struct ib_udata *udata)
1423 {
1424     struct qedr_device_attr *qattr = &dev->attr;
1425
1426     if (attrs->attr.max_wr > qattr->max_srq_wr) {
1427         DP_ERR(dev,
1428                "create srq: unsupported srq_wr=0x%x requested (max_srq_wr=0x%x)\n",
1429                attrs->attr.max_wr, qattr->max_srq_wr);
1430         return -EINVAL;
1431     }
1432
1433     if (attrs->attr.max_sge > qattr->max_sge) {
1434         DP_ERR(dev,
1435                "create srq: unsupported sge=0x%x requested (max_srq_sge=0x%x)\n",
1436                attrs->attr.max_sge, qattr->max_sge);
1437     }
1438
1439     if (!udata && attrs->srq_type == IB_SRQT_XRC) {
1440         DP_ERR(dev, "XRC SRQs are not supported in kernel-space\n");
1441         return -EINVAL;
1442     }
1443
1444     return 0;
1445 }
1446
1447 static void qedr_free_srq_user_params(struct qedr_srq *srq)
1448 {
1449     qedr_free_pbl(srq->dev, &srq->usrq.pbl_info, srq->usrq.pbl_tbl);
1450     ib_umem_release(srq->usrq.umem);
1451     ib_umem_release(srq->prod_umem);
1452 }
1453
1454 static void qedr_free_srq_kernel_params(struct qedr_srq *srq)
1455 {
1456     struct qedr_srq_hwq_info *hw_srq = &srq->hw_srq;
1457     struct qedr_dev *dev = srq->dev;
1458
1459     dev->ops->common->chain_free(dev->cdev, &hw_srq->pbl);
1460
1461     dma_free_coherent(&dev->pdev->dev, sizeof(struct rdma_srq_producers),
1462               hw_srq->virt_prod_pair_addr,
1463               hw_srq->phy_prod_pair_addr);
1464 }
1465
1466 static int qedr_init_srq_user_params(struct ib_udata *udata,
1467                      struct qedr_srq *srq,
1468                      struct qedr_create_srq_ureq *ureq,
1469                      int access)
1470 {
1471     struct scatterlist *sg;
1472     int rc;
1473
1474     rc = qedr_init_user_queue(udata, srq->dev, &srq->usrq, ureq->srq_addr,
1475                   ureq->srq_len, false, access, 1);
1476     if (rc)
1477         return rc;
1478
1479     srq->prod_umem = ib_umem_get(srq->ibsrq.device, ureq->prod_pair_addr,
1480                      sizeof(struct rdma_srq_producers), access);
1481     if (IS_ERR(srq->prod_umem)) {
1482         qedr_free_pbl(srq->dev, &srq->usrq.pbl_info, srq->usrq.pbl_tbl);
1483         ib_umem_release(srq->usrq.umem);
1484         DP_ERR(srq->dev,
1485                "create srq: failed ib_umem_get for producer, got %ld\n",
1486                PTR_ERR(srq->prod_umem));
1487         return PTR_ERR(srq->prod_umem);
1488     }
1489
1490     sg = srq->prod_umem->sgt_append.sgt.sgl;
1491     srq->hw_srq.phy_prod_pair_addr = sg_dma_address(sg);
1492
1493     return 0;
1494 }
1495
1496 static int qedr_alloc_srq_kernel_params(struct qedr_srq *srq,
1497                     struct qedr_dev *dev,
1498                     struct ib_srq_init_attr *init_attr)
1499 {
1500     struct qedr_srq_hwq_info *hw_srq = &srq->hw_srq;
1501     struct qed_chain_init_params params = {
1502         .mode       = QED_CHAIN_MODE_PBL,
1503         .intended_use   = QED_CHAIN_USE_TO_CONSUME_PRODUCE,
1504         .cnt_type   = QED_CHAIN_CNT_TYPE_U32,
1505         .elem_size  = QEDR_SRQ_WQE_ELEM_SIZE,
1506     };
1507     dma_addr_t phy_prod_pair_addr;
1508     u32 num_elems;
1509     void *va;
1510     int rc;
1511
1512     va = dma_alloc_coherent(&dev->pdev->dev,
1513                 sizeof(struct rdma_srq_producers),
1514                 &phy_prod_pair_addr, GFP_KERNEL);
1515     if (!va) {
1516         DP_ERR(dev,
1517                "create srq: failed to allocate dma memory for producer\n");
1518         return -ENOMEM;
1519     }
1520
1521     hw_srq->phy_prod_pair_addr = phy_prod_pair_addr;
1522     hw_srq->virt_prod_pair_addr = va;
1523
1524     num_elems = init_attr->attr.max_wr * RDMA_MAX_SRQ_WQE_SIZE;
1525     params.num_elems = num_elems;
1526
1527     rc = dev->ops->common->chain_alloc(dev->cdev, &hw_srq->pbl, &params);
1528     if (rc)
1529         goto err0;
1530
1531     hw_srq->num_elems = num_elems;
1532
1533     return 0;
1534
1535 err0:
1536     dma_free_coherent(&dev->pdev->dev, sizeof(struct rdma_srq_producers),
1537               va, phy_prod_pair_addr);
1538     return rc;
1539 }
1540
1541 int qedr_create_srq(struct ib_srq *ibsrq, struct ib_srq_init_attr *init_attr,
1542             struct ib_udata *udata)
1543 {
1544     struct qed_rdma_destroy_srq_in_params destroy_in_params;
1545     struct qed_rdma_create_srq_in_params in_params = {};
1546     struct qedr_dev *dev = get_qedr_dev(ibsrq->device);
1547     struct qed_rdma_create_srq_out_params out_params;
1548     struct qedr_pd *pd = get_qedr_pd(ibsrq->pd);
1549     struct qedr_create_srq_ureq ureq = {};
1550     u64 pbl_base_addr, phy_prod_pair_addr;
1551     struct qedr_srq_hwq_info *hw_srq;
1552     u32 page_cnt, page_size;
1553     struct qedr_srq *srq = get_qedr_srq(ibsrq);
1554     int rc = 0;
1555
1556     DP_DEBUG(dev, QEDR_MSG_QP,
1557          "create SRQ called from %s (pd %p)\n",
1558          (udata) ? "User lib" : "kernel", pd);
1559
1560     if (init_attr->srq_type != IB_SRQT_BASIC &&
1561         init_attr->srq_type != IB_SRQT_XRC)
1562         return -EOPNOTSUPP;
1563
1564     rc = qedr_check_srq_params(dev, init_attr, udata);
1565     if (rc)
1566         return -EINVAL;
1567
1568     srq->dev = dev;
1569     srq->is_xrc = (init_attr->srq_type == IB_SRQT_XRC);
1570     hw_srq = &srq->hw_srq;
1571     spin_lock_init(&srq->lock);
1572
1573     hw_srq->max_wr = init_attr->attr.max_wr;
1574     hw_srq->max_sges = init_attr->attr.max_sge;
1575
1576     if (udata) {
1577         if (ib_copy_from_udata(&ureq, udata, min(sizeof(ureq),
1578                              udata->inlen))) {
1579             DP_ERR(dev,
1580                    "create srq: problem copying data from user space\n");
1581             goto err0;
1582         }
1583
1584         rc = qedr_init_srq_user_params(udata, srq, &ureq, 0);
1585         if (rc)
1586             goto err0;
1587
1588         page_cnt = srq->usrq.pbl_info.num_pbes;
1589         pbl_base_addr = srq->usrq.pbl_tbl->pa;
1590         phy_prod_pair_addr = hw_srq->phy_prod_pair_addr;
1591         page_size = PAGE_SIZE;
1592     } else {
1593         struct qed_chain *pbl;
1594
1595         rc = qedr_alloc_srq_kernel_params(srq, dev, init_attr);
1596         if (rc)
1597             goto err0;
1598
1599         pbl = &hw_srq->pbl;
1600         page_cnt = qed_chain_get_page_cnt(pbl);
1601         pbl_base_addr = qed_chain_get_pbl_phys(pbl);
1602         phy_prod_pair_addr = hw_srq->phy_prod_pair_addr;
1603         page_size = QED_CHAIN_PAGE_SIZE;
1604     }
1605
1606     in_params.pd_id = pd->pd_id;
1607     in_params.pbl_base_addr = pbl_base_addr;
1608     in_params.prod_pair_addr = phy_prod_pair_addr;
1609     in_params.num_pages = page_cnt;
1610     in_params.page_size = page_size;
1611     if (srq->is_xrc) {
1612         struct qedr_xrcd *xrcd = get_qedr_xrcd(init_attr->ext.xrc.xrcd);
1613         struct qedr_cq *cq = get_qedr_cq(init_attr->ext.cq);
1614
1615         in_params.is_xrc = 1;
1616         in_params.xrcd_id = xrcd->xrcd_id;
1617         in_params.cq_cid = cq->icid;
1618     }
1619
1620     rc = dev->ops->rdma_create_srq(dev->rdma_ctx, &in_params, &out_params);
1621     if (rc)
1622         goto err1;
1623
1624     srq->srq_id = out_params.srq_id;
1625
1626     if (udata) {
1627         rc = qedr_copy_srq_uresp(dev, srq, udata);
1628         if (rc)
1629             goto err2;
1630     }
1631
1632     rc = xa_insert_irq(&dev->srqs, srq->srq_id, srq, GFP_KERNEL);
1633     if (rc)
1634         goto err2;
1635
1636     DP_DEBUG(dev, QEDR_MSG_SRQ,
1637          "create srq: created srq with srq_id=0x%0x\n", srq->srq_id);
1638     return 0;
1639
1640 err2:
1641     destroy_in_params.srq_id = srq->srq_id;
1642
1643     dev->ops->rdma_destroy_srq(dev->rdma_ctx, &destroy_in_params);
1644 err1:
1645     if (udata)
1646         qedr_free_srq_user_params(srq);
1647     else
1648         qedr_free_srq_kernel_params(srq);
1649 err0:
1650     return -EFAULT;
1651 }
1652
1653 int qedr_destroy_srq(struct ib_srq *ibsrq, struct ib_udata *udata)
1654 {
1655     struct qed_rdma_destroy_srq_in_params in_params = {};
1656     struct qedr_dev *dev = get_qedr_dev(ibsrq->device);
1657     struct qedr_srq *srq = get_qedr_srq(ibsrq);
1658
1659     xa_erase_irq(&dev->srqs, srq->srq_id);
1660     in_params.srq_id = srq->srq_id;
1661     in_params.is_xrc = srq->is_xrc;
1662     dev->ops->rdma_destroy_srq(dev->rdma_ctx, &in_params);
1663
1664     if (ibsrq->uobject)
1665         qedr_free_srq_user_params(srq);
1666     else
1667         qedr_free_srq_kernel_params(srq);
1668
1669     DP_DEBUG(dev, QEDR_MSG_SRQ,
1670          "destroy srq: destroyed srq with srq_id=0x%0x\n",
1671          srq->srq_id);
1672     return 0;
1673 }
1674
1675 int qedr_modify_srq(struct ib_srq *ibsrq, struct ib_srq_attr *attr,
1676             enum ib_srq_attr_mask attr_mask, struct ib_udata *udata)
1677 {
1678     struct qed_rdma_modify_srq_in_params in_params = {};
1679     struct qedr_dev *dev = get_qedr_dev(ibsrq->device);
1680     struct qedr_srq *srq = get_qedr_srq(ibsrq);
1681     int rc;
1682
1683     if (attr_mask & IB_SRQ_MAX_WR) {
1684         DP_ERR(dev,
1685                "modify srq: invalid attribute mask=0x%x specified for %p\n",
1686                attr_mask, srq);
1687         return -EINVAL;
1688     }
1689
1690     if (attr_mask & IB_SRQ_LIMIT) {
1691         if (attr->srq_limit >= srq->hw_srq.max_wr) {
1692             DP_ERR(dev,
1693                    "modify srq: invalid srq_limit=0x%x (max_srq_limit=0x%x)\n",
1694                    attr->srq_limit, srq->hw_srq.max_wr);
1695             return -EINVAL;
1696         }
1697
1698         in_params.srq_id = srq->srq_id;
1699         in_params.wqe_limit = attr->srq_limit;
1700         rc = dev->ops->rdma_modify_srq(dev->rdma_ctx, &in_params);
1701         if (rc)
1702             return rc;
1703     }
1704
1705     srq->srq_limit = attr->srq_limit;
1706
1707     DP_DEBUG(dev, QEDR_MSG_SRQ,
1708          "modify srq: modified srq with srq_id=0x%0x\n", srq->srq_id);
1709
1710     return 0;
1711 }
1712
1713 static enum qed_rdma_qp_type qedr_ib_to_qed_qp_type(enum ib_qp_type ib_qp_type)
1714 {
1715     switch (ib_qp_type) {
1716     case IB_QPT_RC:
1717         return QED_RDMA_QP_TYPE_RC;
1718     case IB_QPT_XRC_INI:
1719         return QED_RDMA_QP_TYPE_XRC_INI;
1720     case IB_QPT_XRC_TGT:
1721         return QED_RDMA_QP_TYPE_XRC_TGT;
1722     default:
1723         return QED_RDMA_QP_TYPE_INVAL;
1724     }
1725 }
1726
1727 static inline void
1728 qedr_init_common_qp_in_params(struct qedr_dev *dev,
1729                   struct qedr_pd *pd,
1730                   struct qedr_qp *qp,
1731                   struct ib_qp_init_attr *attrs,
1732                   bool fmr_and_reserved_lkey,
1733                   struct qed_rdma_create_qp_in_params *params)
1734 {
1735     /* QP handle to be written in an async event */
1736     params->qp_handle_async_lo = lower_32_bits((uintptr_t) qp);
1737     params->qp_handle_async_hi = upper_32_bits((uintptr_t) qp);
1738
1739     params->signal_all = (attrs->sq_sig_type == IB_SIGNAL_ALL_WR);
1740     params->fmr_and_reserved_lkey = fmr_and_reserved_lkey;
1741     params->qp_type = qedr_ib_to_qed_qp_type(attrs->qp_type);
1742     params->stats_queue = 0;
1743
1744     if (pd) {
1745         params->pd = pd->pd_id;
1746         params->dpi = pd->uctx ? pd->uctx->dpi : dev->dpi;
1747     }
1748
1749     if (qedr_qp_has_sq(qp))
1750         params->sq_cq_id = get_qedr_cq(attrs->send_cq)->icid;
1751
1752     if (qedr_qp_has_rq(qp))
1753         params->rq_cq_id = get_qedr_cq(attrs->recv_cq)->icid;
1754
1755     if (qedr_qp_has_srq(qp)) {
1756         params->rq_cq_id = get_qedr_cq(attrs->recv_cq)->icid;
1757         params->srq_id = qp->srq->srq_id;
1758         params->use_srq = true;
1759     } else {
1760         params->srq_id = 0;
1761         params->use_srq = false;
1762     }
1763 }
1764
1765 static inline void qedr_qp_user_print(struct qedr_dev *dev, struct qedr_qp *qp)
1766 {
1767     DP_DEBUG(dev, QEDR_MSG_QP, "create qp: successfully created user QP. "
1768          "qp=%p. "
1769          "sq_addr=0x%llx, "
1770          "sq_len=%zd, "
1771          "rq_addr=0x%llx, "
1772          "rq_len=%zd"
1773          "\n",
1774          qp,
1775          qedr_qp_has_sq(qp) ? qp->usq.buf_addr : 0x0,
1776          qedr_qp_has_sq(qp) ? qp->usq.buf_len : 0,
1777          qedr_qp_has_rq(qp) ? qp->urq.buf_addr : 0x0,
1778          qedr_qp_has_sq(qp) ? qp->urq.buf_len : 0);
1779 }
1780
1781 static inline void
1782 qedr_iwarp_populate_user_qp(struct qedr_dev *dev,
1783                 struct qedr_qp *qp,
1784                 struct qed_rdma_create_qp_out_params *out_params)
1785 {
1786     qp->usq.pbl_tbl->va = out_params->sq_pbl_virt;
1787     qp->usq.pbl_tbl->pa = out_params->sq_pbl_phys;
1788
1789     qedr_populate_pbls(dev, qp->usq.umem, qp->usq.pbl_tbl,
1790                &qp->usq.pbl_info, FW_PAGE_SHIFT);
1791     if (!qp->srq) {
1792         qp->urq.pbl_tbl->va = out_params->rq_pbl_virt;
1793         qp->urq.pbl_tbl->pa = out_params->rq_pbl_phys;
1794     }
1795
1796     qedr_populate_pbls(dev, qp->urq.umem, qp->urq.pbl_tbl,
1797                &qp->urq.pbl_info, FW_PAGE_SHIFT);
1798 }
1799
1800 static void qedr_cleanup_user(struct qedr_dev *dev,
1801                   struct qedr_ucontext *ctx,
1802                   struct qedr_qp *qp)
1803 {
1804     if (qedr_qp_has_sq(qp)) {
1805         ib_umem_release(qp->usq.umem);
1806         qp->usq.umem = NULL;
1807     }
1808
1809     if (qedr_qp_has_rq(qp)) {
1810         ib_umem_release(qp->urq.umem);
1811         qp->urq.umem = NULL;
1812     }
1813
1814     if (rdma_protocol_roce(&dev->ibdev, 1)) {
1815         qedr_free_pbl(dev, &qp->usq.pbl_info, qp->usq.pbl_tbl);
1816         qedr_free_pbl(dev, &qp->urq.pbl_info, qp->urq.pbl_tbl);
1817     } else {
1818         kfree(qp->usq.pbl_tbl);
1819         kfree(qp->urq.pbl_tbl);
1820     }
1821
1822     if (qp->usq.db_rec_data) {
1823         qedr_db_recovery_del(dev, qp->usq.db_addr,
1824                      &qp->usq.db_rec_data->db_data);
1825         rdma_user_mmap_entry_remove(qp->usq.db_mmap_entry);
1826     }
1827
1828     if (qp->urq.db_rec_data) {
1829         qedr_db_recovery_del(dev, qp->urq.db_addr,
1830                      &qp->urq.db_rec_data->db_data);
1831         rdma_user_mmap_entry_remove(qp->urq.db_mmap_entry);
1832     }
1833
1834     if (rdma_protocol_iwarp(&dev->ibdev, 1))
1835         qedr_db_recovery_del(dev, qp->urq.db_rec_db2_addr,
1836                      &qp->urq.db_rec_db2_data);
1837 }
1838
1839 static int qedr_create_user_qp(struct qedr_dev *dev,
1840                    struct qedr_qp *qp,
1841                    struct ib_pd *ibpd,
1842                    struct ib_udata *udata,
1843                    struct ib_qp_init_attr *attrs)
1844 {
1845     struct qed_rdma_create_qp_in_params in_params;
1846     struct qed_rdma_create_qp_out_params out_params;
1847     struct qedr_create_qp_uresp uresp = {};
1848     struct qedr_create_qp_ureq ureq = {};
1849     int alloc_and_init = rdma_protocol_roce(&dev->ibdev, 1);
1850     struct qedr_ucontext *ctx = NULL;
1851     struct qedr_pd *pd = NULL;
1852     int rc = 0;
1853
1854     qp->create_type = QEDR_QP_CREATE_USER;
1855
1856     if (ibpd) {
1857         pd = get_qedr_pd(ibpd);
1858         ctx = pd->uctx;
1859     }
1860
1861     if (udata) {
1862         rc = ib_copy_from_udata(&ureq, udata, min(sizeof(ureq),
1863                     udata->inlen));
1864         if (rc) {
1865             DP_ERR(dev, "Problem copying data from user space\n");
1866             return rc;
1867         }
1868     }
1869
1870     if (qedr_qp_has_sq(qp)) {
1871         /* SQ - read access only (0) */
1872         rc = qedr_init_user_queue(udata, dev, &qp->usq, ureq.sq_addr,
1873                       ureq.sq_len, true, 0, alloc_and_init);
1874         if (rc)
1875             return rc;
1876     }
1877
1878     if (qedr_qp_has_rq(qp)) {
1879         /* RQ - read access only (0) */
1880         rc = qedr_init_user_queue(udata, dev, &qp->urq, ureq.rq_addr,
1881                       ureq.rq_len, true, 0, alloc_and_init);
1882         if (rc)
1883             return rc;
1884     }
1885
1886     memset(&in_params, 0, sizeof(in_params));
1887     qedr_init_common_qp_in_params(dev, pd, qp, attrs, false, &in_params);
1888     in_params.qp_handle_lo = ureq.qp_handle_lo;
1889     in_params.qp_handle_hi = ureq.qp_handle_hi;
1890
1891     if (qp->qp_type == IB_QPT_XRC_TGT) {
1892         struct qedr_xrcd *xrcd = get_qedr_xrcd(attrs->xrcd);
1893
1894         in_params.xrcd_id = xrcd->xrcd_id;
1895         in_params.qp_handle_lo = qp->qp_id;
1896         in_params.use_srq = 1;
1897     }
1898
1899     if (qedr_qp_has_sq(qp)) {
1900         in_params.sq_num_pages = qp->usq.pbl_info.num_pbes;
1901         in_params.sq_pbl_ptr = qp->usq.pbl_tbl->pa;
1902     }
1903
1904     if (qedr_qp_has_rq(qp)) {
1905         in_params.rq_num_pages = qp->urq.pbl_info.num_pbes;
1906         in_params.rq_pbl_ptr = qp->urq.pbl_tbl->pa;
1907     }
1908
1909     if (ctx)
1910         SET_FIELD(in_params.flags, QED_ROCE_EDPM_MODE, ctx->edpm_mode);
1911
1912     qp->qed_qp = dev->ops->rdma_create_qp(dev->rdma_ctx,
1913                           &in_params, &out_params);
1914
1915     if (!qp->qed_qp) {
1916         rc = -ENOMEM;
1917         goto err1;
1918     }
1919
1920     if (rdma_protocol_iwarp(&dev->ibdev, 1))
1921         qedr_iwarp_populate_user_qp(dev, qp, &out_params);
1922
1923     qp->qp_id = out_params.qp_id;
1924     qp->icid = out_params.icid;
1925
1926     if (udata) {
1927         rc = qedr_copy_qp_uresp(dev, qp, udata, &uresp);
1928         if (rc)
1929             goto err;
1930     }
1931
1932     /* db offset was calculated in copy_qp_uresp, now set in the user q */
1933     if (qedr_qp_has_sq(qp)) {
1934         qp->usq.db_addr = ctx->dpi_addr + uresp.sq_db_offset;
1935         qp->sq.max_wr = attrs->cap.max_send_wr;
1936         rc = qedr_db_recovery_add(dev, qp->usq.db_addr,
1937                       &qp->usq.db_rec_data->db_data,
1938                       DB_REC_WIDTH_32B,
1939                       DB_REC_USER);
1940         if (rc)
1941             goto err;
1942     }
1943
1944     if (qedr_qp_has_rq(qp)) {
1945         qp->urq.db_addr = ctx->dpi_addr + uresp.rq_db_offset;
1946         qp->rq.max_wr = attrs->cap.max_recv_wr;
1947         rc = qedr_db_recovery_add(dev, qp->urq.db_addr,
1948                       &qp->urq.db_rec_data->db_data,
1949                       DB_REC_WIDTH_32B,
1950                       DB_REC_USER);
1951         if (rc)
1952             goto err;
1953     }
1954
1955     if (rdma_protocol_iwarp(&dev->ibdev, 1)) {
1956         qp->urq.db_rec_db2_addr = ctx->dpi_addr + uresp.rq_db2_offset;
1957
1958         /* calculate the db_rec_db2 data since it is constant so no
1959          * need to reflect from user
1960          */
1961         qp->urq.db_rec_db2_data.data.icid = cpu_to_le16(qp->icid);
1962         qp->urq.db_rec_db2_data.data.value =
1963             cpu_to_le16(DQ_TCM_IWARP_POST_RQ_CF_CMD);
1964
1965         rc = qedr_db_recovery_add(dev, qp->urq.db_rec_db2_addr,
1966                       &qp->urq.db_rec_db2_data,
1967                       DB_REC_WIDTH_32B,
1968                       DB_REC_USER);
1969         if (rc)
1970             goto err;
1971     }
1972     qedr_qp_user_print(dev, qp);
1973     return rc;
1974 err:
1975     rc = dev->ops->rdma_destroy_qp(dev->rdma_ctx, qp->qed_qp);
1976     if (rc)
1977         DP_ERR(dev, "create qp: fatal fault. rc=%d", rc);
1978
1979 err1:
1980     qedr_cleanup_user(dev, ctx, qp);
1981     return rc;
1982 }
1983
1984 static int qedr_set_iwarp_db_info(struct qedr_dev *dev, struct qedr_qp *qp)
1985 {
1986     int rc;
1987
1988     qp->sq.db = dev->db_addr +
1989         DB_ADDR_SHIFT(DQ_PWM_OFFSET_XCM_RDMA_SQ_PROD);
1990     qp->sq.db_data.data.icid = qp->icid;
1991
1992     rc = qedr_db_recovery_add(dev, qp->sq.db,
1993                   &qp->sq.db_data,
1994                   DB_REC_WIDTH_32B,
1995                   DB_REC_KERNEL);
1996     if (rc)
1997         return rc;
1998
1999     qp->rq.db = dev->db_addr +
2000             DB_ADDR_SHIFT(DQ_PWM_OFFSET_TCM_IWARP_RQ_PROD);
2001     qp->rq.db_data.data.icid = qp->icid;
2002     qp->rq.iwarp_db2 = dev->db_addr +
2003                DB_ADDR_SHIFT(DQ_PWM_OFFSET_TCM_FLAGS);
2004     qp->rq.iwarp_db2_data.data.icid = qp->icid;
2005     qp->rq.iwarp_db2_data.data.value = DQ_TCM_IWARP_POST_RQ_CF_CMD;
2006
2007     rc = qedr_db_recovery_add(dev, qp->rq.db,
2008                   &qp->rq.db_data,
2009                   DB_REC_WIDTH_32B,
2010                   DB_REC_KERNEL);
2011     if (rc)
2012         return rc;
2013
2014     rc = qedr_db_recovery_add(dev, qp->rq.iwarp_db2,
2015                   &qp->rq.iwarp_db2_data,
2016                   DB_REC_WIDTH_32B,
2017                   DB_REC_KERNEL);
2018     return rc;
2019 }
2020
2021 static int
2022 qedr_roce_create_kernel_qp(struct qedr_dev *dev,
2023                struct qedr_qp *qp,
2024                struct qed_rdma_create_qp_in_params *in_params,
2025                u32 n_sq_elems, u32 n_rq_elems)
2026 {
2027     struct qed_rdma_create_qp_out_params out_params;
2028     struct qed_chain_init_params params = {
2029         .mode       = QED_CHAIN_MODE_PBL,
2030         .cnt_type   = QED_CHAIN_CNT_TYPE_U32,
2031     };
2032     int rc;
2033
2034     params.intended_use = QED_CHAIN_USE_TO_PRODUCE;
2035     params.num_elems = n_sq_elems;
2036     params.elem_size = QEDR_SQE_ELEMENT_SIZE;
2037
2038     rc = dev->ops->common->chain_alloc(dev->cdev, &qp->sq.pbl, &params);
2039     if (rc)
2040         return rc;
2041
2042     in_params->sq_num_pages = qed_chain_get_page_cnt(&qp->sq.pbl);
2043     in_params->sq_pbl_ptr = qed_chain_get_pbl_phys(&qp->sq.pbl);
2044
2045     params.intended_use = QED_CHAIN_USE_TO_CONSUME_PRODUCE;
2046     params.num_elems = n_rq_elems;
2047     params.elem_size = QEDR_RQE_ELEMENT_SIZE;
2048
2049     rc = dev->ops->common->chain_alloc(dev->cdev, &qp->rq.pbl, &params);
2050     if (rc)
2051         return rc;
2052
2053     in_params->rq_num_pages = qed_chain_get_page_cnt(&qp->rq.pbl);
2054     in_params->rq_pbl_ptr = qed_chain_get_pbl_phys(&qp->rq.pbl);
2055
2056     qp->qed_qp = dev->ops->rdma_create_qp(dev->rdma_ctx,
2057                           in_params, &out_params);
2058
2059     if (!qp->qed_qp)
2060         return -EINVAL;
2061
2062     qp->qp_id = out_params.qp_id;
2063     qp->icid = out_params.icid;
2064
2065     return qedr_set_roce_db_info(dev, qp);
2066 }
2067
2068 static int
2069 qedr_iwarp_create_kernel_qp(struct qedr_dev *dev,
2070                 struct qedr_qp *qp,
2071                 struct qed_rdma_create_qp_in_params *in_params,
2072                 u32 n_sq_elems, u32 n_rq_elems)
2073 {
2074     struct qed_rdma_create_qp_out_params out_params;
2075     struct qed_chain_init_params params = {
2076         .mode       = QED_CHAIN_MODE_PBL,
2077         .cnt_type   = QED_CHAIN_CNT_TYPE_U32,
2078     };
2079     int rc;
2080
2081     in_params->sq_num_pages = QED_CHAIN_PAGE_CNT(n_sq_elems,
2082                              QEDR_SQE_ELEMENT_SIZE,
2083                              QED_CHAIN_PAGE_SIZE,
2084                              QED_CHAIN_MODE_PBL);
2085     in_params->rq_num_pages = QED_CHAIN_PAGE_CNT(n_rq_elems,
2086                              QEDR_RQE_ELEMENT_SIZE,
2087                              QED_CHAIN_PAGE_SIZE,
2088                              QED_CHAIN_MODE_PBL);
2089
2090     qp->qed_qp = dev->ops->rdma_create_qp(dev->rdma_ctx,
2091                           in_params, &out_params);
2092
2093     if (!qp->qed_qp)
2094         return -EINVAL;
2095
2096     /* Now we allocate the chain */
2097
2098     params.intended_use = QED_CHAIN_USE_TO_PRODUCE;
2099     params.num_elems = n_sq_elems;
2100     params.elem_size = QEDR_SQE_ELEMENT_SIZE;
2101     params.ext_pbl_virt = out_params.sq_pbl_virt;
2102     params.ext_pbl_phys = out_params.sq_pbl_phys;
2103
2104     rc = dev->ops->common->chain_alloc(dev->cdev, &qp->sq.pbl, &params);
2105     if (rc)
2106         goto err;
2107
2108     params.intended_use = QED_CHAIN_USE_TO_CONSUME_PRODUCE;
2109     params.num_elems = n_rq_elems;
2110     params.elem_size = QEDR_RQE_ELEMENT_SIZE;
2111     params.ext_pbl_virt = out_params.rq_pbl_virt;
2112     params.ext_pbl_phys = out_params.rq_pbl_phys;
2113
2114     rc = dev->ops->common->chain_alloc(dev->cdev, &qp->rq.pbl, &params);
2115     if (rc)
2116         goto err;
2117
2118     qp->qp_id = out_params.qp_id;
2119     qp->icid = out_params.icid;
2120
2121     return qedr_set_iwarp_db_info(dev, qp);
2122
2123 err:
2124     dev->ops->rdma_destroy_qp(dev->rdma_ctx, qp->qed_qp);
2125
2126     return rc;
2127 }
2128
2129 static void qedr_cleanup_kernel(struct qedr_dev *dev, struct qedr_qp *qp)
2130 {
2131     dev->ops->common->chain_free(dev->cdev, &qp->sq.pbl);
2132     kfree(qp->wqe_wr_id);
2133
2134     dev->ops->common->chain_free(dev->cdev, &qp->rq.pbl);
2135     kfree(qp->rqe_wr_id);
2136
2137     /* GSI qp is not registered to db mechanism so no need to delete */
2138     if (qp->qp_type == IB_QPT_GSI)
2139         return;
2140
2141     qedr_db_recovery_del(dev, qp->sq.db, &qp->sq.db_data);
2142
2143     if (!qp->srq) {
2144         qedr_db_recovery_del(dev, qp->rq.db, &qp->rq.db_data);
2145
2146         if (rdma_protocol_iwarp(&dev->ibdev, 1))
2147             qedr_db_recovery_del(dev, qp->rq.iwarp_db2,
2148                          &qp->rq.iwarp_db2_data);
2149     }
2150 }
2151
2152 static int qedr_create_kernel_qp(struct qedr_dev *dev,
2153                  struct qedr_qp *qp,
2154                  struct ib_pd *ibpd,
2155                  struct ib_qp_init_attr *attrs)
2156 {
2157     struct qed_rdma_create_qp_in_params in_params;
2158     struct qedr_pd *pd = get_qedr_pd(ibpd);
2159     int rc = -EINVAL;
2160     u32 n_rq_elems;
2161     u32 n_sq_elems;
2162     u32 n_sq_entries;
2163
2164     memset(&in_params, 0, sizeof(in_params));
2165     qp->create_type = QEDR_QP_CREATE_KERNEL;
2166
2167     /* A single work request may take up to QEDR_MAX_SQ_WQE_SIZE elements in
2168      * the ring. The ring should allow at least a single WR, even if the
2169      * user requested none, due to allocation issues.
2170      * We should add an extra WR since the prod and cons indices of
2171      * wqe_wr_id are managed in such a way that the WQ is considered full
2172      * when (prod+1)%max_wr==cons. We currently don't do that because we
2173      * double the number of entries due an iSER issue that pushes far more
2174      * WRs than indicated. If we decline its ib_post_send() then we get
2175      * error prints in the dmesg we'd like to avoid.
2176      */
2177     qp->sq.max_wr = min_t(u32, attrs->cap.max_send_wr * dev->wq_multiplier,
2178                   dev->attr.max_sqe);
2179
2180     qp->wqe_wr_id = kcalloc(qp->sq.max_wr, sizeof(*qp->wqe_wr_id),
2181                 GFP_KERNEL);
2182     if (!qp->wqe_wr_id) {
2183         DP_ERR(dev, "create qp: failed SQ shadow memory allocation\n");
2184         return -ENOMEM;
2185     }
2186
2187     /* QP handle to be written in CQE */
2188     in_params.qp_handle_lo = lower_32_bits((uintptr_t) qp);
2189     in_params.qp_handle_hi = upper_32_bits((uintptr_t) qp);
2190
2191     /* A single work request may take up to QEDR_MAX_RQ_WQE_SIZE elements in
2192      * the ring. There ring should allow at least a single WR, even if the
2193      * user requested none, due to allocation issues.
2194      */
2195     qp->rq.max_wr = (u16) max_t(u32, attrs->cap.max_recv_wr, 1);
2196
2197     /* Allocate driver internal RQ array */
2198     qp->rqe_wr_id = kcalloc(qp->rq.max_wr, sizeof(*qp->rqe_wr_id),
2199                 GFP_KERNEL);
2200     if (!qp->rqe_wr_id) {
2201         DP_ERR(dev,
2202                "create qp: failed RQ shadow memory allocation\n");
2203         kfree(qp->wqe_wr_id);
2204         return -ENOMEM;
2205     }
2206
2207     qedr_init_common_qp_in_params(dev, pd, qp, attrs, true, &in_params);
2208
2209     n_sq_entries = attrs->cap.max_send_wr;
2210     n_sq_entries = min_t(u32, n_sq_entries, dev->attr.max_sqe);
2211     n_sq_entries = max_t(u32, n_sq_entries, 1);
2212     n_sq_elems = n_sq_entries * QEDR_MAX_SQE_ELEMENTS_PER_SQE;
2213
2214     n_rq_elems = qp->rq.max_wr * QEDR_MAX_RQE_ELEMENTS_PER_RQE;
2215
2216     if (rdma_protocol_iwarp(&dev->ibdev, 1))
2217         rc = qedr_iwarp_create_kernel_qp(dev, qp, &in_params,
2218                          n_sq_elems, n_rq_elems);
2219     else
2220         rc = qedr_roce_create_kernel_qp(dev, qp, &in_params,
2221                         n_sq_elems, n_rq_elems);
2222     if (rc)
2223         qedr_cleanup_kernel(dev, qp);
2224
2225     return rc;
2226 }
2227
2228 static int qedr_free_qp_resources(struct qedr_dev *dev, struct qedr_qp *qp,
2229                   struct ib_udata *udata)
2230 {
2231     struct qedr_ucontext *ctx =
2232         rdma_udata_to_drv_context(udata, struct qedr_ucontext,
2233                       ibucontext);
2234     int rc;
2235
2236     if (qp->qp_type != IB_QPT_GSI) {
2237         rc = dev->ops->rdma_destroy_qp(dev->rdma_ctx, qp->qed_qp);
2238         if (rc)
2239             return rc;
2240     }
2241
2242     if (qp->create_type == QEDR_QP_CREATE_USER)
2243         qedr_cleanup_user(dev, ctx, qp);
2244     else
2245         qedr_cleanup_kernel(dev, qp);
2246
2247     return 0;
2248 }
2249
2250 int qedr_create_qp(struct ib_qp *ibqp, struct ib_qp_init_attr *attrs,
2251            struct ib_udata *udata)
2252 {
2253     struct qedr_xrcd *xrcd = NULL;
2254     struct ib_pd *ibpd = ibqp->pd;
2255     struct qedr_pd *pd = get_qedr_pd(ibpd);
2256     struct qedr_dev *dev = get_qedr_dev(ibqp->device);
2257     struct qedr_qp *qp = get_qedr_qp(ibqp);
2258     int rc = 0;
2259
2260     if (attrs->create_flags)
2261         return -EOPNOTSUPP;
2262
2263     if (attrs->qp_type == IB_QPT_XRC_TGT)
2264         xrcd = get_qedr_xrcd(attrs->xrcd);
2265     else
2266         pd = get_qedr_pd(ibpd);
2267
2268     DP_DEBUG(dev, QEDR_MSG_QP, "create qp: called from %s, pd=%p\n",
2269          udata ? "user library" : "kernel", pd);
2270
2271     rc = qedr_check_qp_attrs(ibpd, dev, attrs, udata);
2272     if (rc)
2273         return rc;
2274
2275     DP_DEBUG(dev, QEDR_MSG_QP,
2276          "create qp: called from %s, event_handler=%p, eepd=%p sq_cq=%p, sq_icid=%d, rq_cq=%p, rq_icid=%d\n",
2277          udata ? "user library" : "kernel", attrs->event_handler, pd,
2278          get_qedr_cq(attrs->send_cq),
2279          get_qedr_cq(attrs->send_cq)->icid,
2280          get_qedr_cq(attrs->recv_cq),
2281          attrs->recv_cq ? get_qedr_cq(attrs->recv_cq)->icid : 0);
2282
2283     qedr_set_common_qp_params(dev, qp, pd, attrs);
2284
2285     if (attrs->qp_type == IB_QPT_GSI)
2286         return qedr_create_gsi_qp(dev, attrs, qp);
2287
2288     if (udata || xrcd)
2289         rc = qedr_create_user_qp(dev, qp, ibpd, udata, attrs);
2290     else
2291         rc = qedr_create_kernel_qp(dev, qp, ibpd, attrs);
2292
2293     if (rc)
2294         return rc;
2295
2296     qp->ibqp.qp_num = qp->qp_id;
2297
2298     if (rdma_protocol_iwarp(&dev->ibdev, 1)) {
2299         rc = xa_insert(&dev->qps, qp->qp_id, qp, GFP_KERNEL);
2300         if (rc)
2301             goto out_free_qp_resources;
2302     }
2303
2304     return 0;
2305
2306 out_free_qp_resources:
2307     qedr_free_qp_resources(dev, qp, udata);
2308     return -EFAULT;
2309 }
2310
2311 static enum ib_qp_state qedr_get_ibqp_state(enum qed_roce_qp_state qp_state)
2312 {
2313     switch (qp_state) {
2314     case QED_ROCE_QP_STATE_RESET:
2315         return IB_QPS_RESET;
2316     case QED_ROCE_QP_STATE_INIT:
2317         return IB_QPS_INIT;
2318     case QED_ROCE_QP_STATE_RTR:
2319         return IB_QPS_RTR;
2320     case QED_ROCE_QP_STATE_RTS:
2321         return IB_QPS_RTS;
2322     case QED_ROCE_QP_STATE_SQD:
2323         return IB_QPS_SQD;
2324     case QED_ROCE_QP_STATE_ERR:
2325         return IB_QPS_ERR;
2326     case QED_ROCE_QP_STATE_SQE:
2327         return IB_QPS_SQE;
2328     }
2329     return IB_QPS_ERR;
2330 }
2331
2332 static enum qed_roce_qp_state qedr_get_state_from_ibqp(
2333                     enum ib_qp_state qp_state)
2334 {
2335     switch (qp_state) {
2336     case IB_QPS_RESET:
2337         return QED_ROCE_QP_STATE_RESET;
2338     case IB_QPS_INIT:
2339         return QED_ROCE_QP_STATE_INIT;
2340     case IB_QPS_RTR:
2341         return QED_ROCE_QP_STATE_RTR;
2342     case IB_QPS_RTS:
2343         return QED_ROCE_QP_STATE_RTS;
2344     case IB_QPS_SQD:
2345         return QED_ROCE_QP_STATE_SQD;
2346     case IB_QPS_ERR:
2347         return QED_ROCE_QP_STATE_ERR;
2348     default:
2349         return QED_ROCE_QP_STATE_ERR;
2350     }
2351 }
2352
2353 static int qedr_update_qp_state(struct qedr_dev *dev,
2354                 struct qedr_qp *qp,
2355                 enum qed_roce_qp_state cur_state,
2356                 enum qed_roce_qp_state new_state)
2357 {
2358     int status = 0;
2359
2360     if (new_state == cur_state)
2361         return 0;
2362
2363     switch (cur_state) {
2364     case QED_ROCE_QP_STATE_RESET:
2365         switch (new_state) {
2366         case QED_ROCE_QP_STATE_INIT:
2367             break;
2368         default:
2369             status = -EINVAL;
2370             break;
2371         }
2372         break;
2373     case QED_ROCE_QP_STATE_INIT:
2374         switch (new_state) {
2375         case QED_ROCE_QP_STATE_RTR:
2376             /* Update doorbell (in case post_recv was
2377              * done before move to RTR)
2378              */
2379
2380             if (rdma_protocol_roce(&dev->ibdev, 1)) {
2381                 writel(qp->rq.db_data.raw, qp->rq.db);
2382             }
2383             break;
2384         case QED_ROCE_QP_STATE_ERR:
2385             break;
2386         default:
2387             /* Invalid state change. */
2388             status = -EINVAL;
2389             break;
2390         }
2391         break;
2392     case QED_ROCE_QP_STATE_RTR:
2393         /* RTR->XXX */
2394         switch (new_state) {
2395         case QED_ROCE_QP_STATE_RTS:
2396             break;
2397         case QED_ROCE_QP_STATE_ERR:
2398             break;
2399         default:
2400             /* Invalid state change. */
2401             status = -EINVAL;
2402             break;
2403         }
2404         break;
2405     case QED_ROCE_QP_STATE_RTS:
2406         /* RTS->XXX */
2407         switch (new_state) {
2408         case QED_ROCE_QP_STATE_SQD:
2409             break;
2410         case QED_ROCE_QP_STATE_ERR:
2411             break;
2412         default:
2413             /* Invalid state change. */
2414             status = -EINVAL;
2415             break;
2416         }
2417         break;
2418     case QED_ROCE_QP_STATE_SQD:
2419         /* SQD->XXX */
2420         switch (new_state) {
2421         case QED_ROCE_QP_STATE_RTS:
2422         case QED_ROCE_QP_STATE_ERR:
2423             break;
2424         default:
2425             /* Invalid state change. */
2426             status = -EINVAL;
2427             break;
2428         }
2429         break;
2430     case QED_ROCE_QP_STATE_ERR:
2431         /* ERR->XXX */
2432         switch (new_state) {
2433         case QED_ROCE_QP_STATE_RESET:
2434             if ((qp->rq.prod != qp->rq.cons) ||
2435                 (qp->sq.prod != qp->sq.cons)) {
2436                 DP_NOTICE(dev,
2437                       "Error->Reset with rq/sq not empty rq.prod=%x rq.cons=%x sq.prod=%x sq.cons=%x\n",
2438                       qp->rq.prod, qp->rq.cons, qp->sq.prod,
2439                       qp->sq.cons);
2440                 status = -EINVAL;
2441             }
2442             break;
2443         default:
2444             status = -EINVAL;
2445             break;
2446         }
2447         break;
2448     default:
2449         status = -EINVAL;
2450         break;
2451     }
2452
2453     return status;
2454 }
2455
2456 int qedr_modify_qp(struct ib_qp *ibqp, struct ib_qp_attr *attr,
2457            int attr_mask, struct ib_udata *udata)
2458 {
2459     struct qedr_qp *qp = get_qedr_qp(ibqp);
2460     struct qed_rdma_modify_qp_in_params qp_params = { 0 };
2461     struct qedr_dev *dev = get_qedr_dev(&qp->dev->ibdev);
2462     const struct ib_global_route *grh = rdma_ah_read_grh(&attr->ah_attr);
2463     enum ib_qp_state old_qp_state, new_qp_state;
2464     enum qed_roce_qp_state cur_state;
2465     int rc = 0;
2466
2467     DP_DEBUG(dev, QEDR_MSG_QP,
2468          "modify qp: qp %p attr_mask=0x%x, state=%d", qp, attr_mask,
2469          attr->qp_state);
2470
2471     if (attr_mask & ~IB_QP_ATTR_STANDARD_BITS)
2472         return -EOPNOTSUPP;
2473
2474     old_qp_state = qedr_get_ibqp_state(qp->state);
2475     if (attr_mask & IB_QP_STATE)
2476         new_qp_state = attr->qp_state;
2477     else
2478         new_qp_state = old_qp_state;
2479
2480     if (rdma_protocol_roce(&dev->ibdev, 1)) {
2481         if (!ib_modify_qp_is_ok(old_qp_state, new_qp_state,
2482                     ibqp->qp_type, attr_mask)) {
2483             DP_ERR(dev,
2484                    "modify qp: invalid attribute mask=0x%x specified for\n"
2485                    "qpn=0x%x of type=0x%x old_qp_state=0x%x, new_qp_state=0x%x\n",
2486                    attr_mask, qp->qp_id, ibqp->qp_type,
2487                    old_qp_state, new_qp_state);
2488             rc = -EINVAL;
2489             goto err;
2490         }
2491     }
2492
2493     /* Translate the masks... */
2494     if (attr_mask & IB_QP_STATE) {
2495         SET_FIELD(qp_params.modify_flags,
2496               QED_RDMA_MODIFY_QP_VALID_NEW_STATE, 1);
2497         qp_params.new_state = qedr_get_state_from_ibqp(attr->qp_state);
2498     }
2499
2500     if (attr_mask & IB_QP_EN_SQD_ASYNC_NOTIFY)
2501         qp_params.sqd_async = true;
2502
2503     if (attr_mask & IB_QP_PKEY_INDEX) {
2504         SET_FIELD(qp_params.modify_flags,
2505               QED_ROCE_MODIFY_QP_VALID_PKEY, 1);
2506         if (attr->pkey_index >= QEDR_ROCE_PKEY_TABLE_LEN) {
2507             rc = -EINVAL;
2508             goto err;
2509         }
2510
2511         qp_params.pkey = QEDR_ROCE_PKEY_DEFAULT;
2512     }
2513
2514     if (attr_mask & IB_QP_QKEY)
2515         qp->qkey = attr->qkey;
2516
2517     if (attr_mask & IB_QP_ACCESS_FLAGS) {
2518         SET_FIELD(qp_params.modify_flags,
2519               QED_RDMA_MODIFY_QP_VALID_RDMA_OPS_EN, 1);
2520         qp_params.incoming_rdma_read_en = attr->qp_access_flags &
2521                           IB_ACCESS_REMOTE_READ;
2522         qp_params.incoming_rdma_write_en = attr->qp_access_flags &
2523                            IB_ACCESS_REMOTE_WRITE;
2524         qp_params.incoming_atomic_en = attr->qp_access_flags &
2525                            IB_ACCESS_REMOTE_ATOMIC;
2526     }
2527
2528     if (attr_mask & (IB_QP_AV | IB_QP_PATH_MTU)) {
2529         if (rdma_protocol_iwarp(&dev->ibdev, 1))
2530             return -EINVAL;
2531
2532         if (attr_mask & IB_QP_PATH_MTU) {
2533             if (attr->path_mtu < IB_MTU_256 ||
2534                 attr->path_mtu > IB_MTU_4096) {
2535                 pr_err("error: Only MTU sizes of 256, 512, 1024, 2048 and 4096 are supported by RoCE\n");
2536                 rc = -EINVAL;
2537                 goto err;
2538             }
2539             qp->mtu = min(ib_mtu_enum_to_int(attr->path_mtu),
2540                       ib_mtu_enum_to_int(iboe_get_mtu
2541                              (dev->ndev->mtu)));
2542         }
2543
2544         if (!qp->mtu) {
2545             qp->mtu =
2546             ib_mtu_enum_to_int(iboe_get_mtu(dev->ndev->mtu));
2547             pr_err("Fixing zeroed MTU to qp->mtu = %d\n", qp->mtu);
2548         }
2549
2550         SET_FIELD(qp_params.modify_flags,
2551               QED_ROCE_MODIFY_QP_VALID_ADDRESS_VECTOR, 1);
2552
2553         qp_params.traffic_class_tos = grh->traffic_class;
2554         qp_params.flow_label = grh->flow_label;
2555         qp_params.hop_limit_ttl = grh->hop_limit;
2556
2557         qp->sgid_idx = grh->sgid_index;
2558
2559         rc = get_gid_info_from_table(ibqp, attr, attr_mask, &qp_params);
2560         if (rc) {
2561             DP_ERR(dev,
2562                    "modify qp: problems with GID index %d (rc=%d)\n",
2563                    grh->sgid_index, rc);
2564             return rc;
2565         }
2566
2567         rc = qedr_get_dmac(dev, &attr->ah_attr,
2568                    qp_params.remote_mac_addr);
2569         if (rc)
2570             return rc;
2571
2572         qp_params.use_local_mac = true;
2573         ether_addr_copy(qp_params.local_mac_addr, dev->ndev->dev_addr);
2574
2575         DP_DEBUG(dev, QEDR_MSG_QP, "dgid=%x:%x:%x:%x\n",
2576              qp_params.dgid.dwords[0], qp_params.dgid.dwords[1],
2577              qp_params.dgid.dwords[2], qp_params.dgid.dwords[3]);
2578         DP_DEBUG(dev, QEDR_MSG_QP, "sgid=%x:%x:%x:%x\n",
2579              qp_params.sgid.dwords[0], qp_params.sgid.dwords[1],
2580              qp_params.sgid.dwords[2], qp_params.sgid.dwords[3]);
2581         DP_DEBUG(dev, QEDR_MSG_QP, "remote_mac=[%pM]\n",
2582              qp_params.remote_mac_addr);
2583
2584         qp_params.mtu = qp->mtu;
2585         qp_params.lb_indication = false;
2586     }
2587
2588     if (!qp_params.mtu) {
2589         /* Stay with current MTU */
2590         if (qp->mtu)
2591             qp_params.mtu = qp->mtu;
2592         else
2593             qp_params.mtu =
2594                 ib_mtu_enum_to_int(iboe_get_mtu(dev->ndev->mtu));
2595     }
2596
2597     if (attr_mask & IB_QP_TIMEOUT) {
2598         SET_FIELD(qp_params.modify_flags,
2599               QED_ROCE_MODIFY_QP_VALID_ACK_TIMEOUT, 1);
2600
2601         /* The received timeout value is an exponent used like this:
2602          *    "12.7.34 LOCAL ACK TIMEOUT
2603          *    Value representing the transport (ACK) timeout for use by
2604          *    the remote, expressed as: 4.096 * 2^timeout [usec]"
2605          * The FW expects timeout in msec so we need to divide the usec
2606          * result by 1000. We'll approximate 1000~2^10, and 4.096 ~ 2^2,
2607          * so we get: 2^2 * 2^timeout / 2^10 = 2^(timeout - 8).
2608          * The value of zero means infinite so we use a 'max_t' to make
2609          * sure that sub 1 msec values will be configured as 1 msec.
2610          */
2611         if (attr->timeout)
2612             qp_params.ack_timeout =
2613                     1 << max_t(int, attr->timeout - 8, 0);
2614         else
2615             qp_params.ack_timeout = 0;
2616
2617         qp->timeout = attr->timeout;
2618     }
2619
2620     if (attr_mask & IB_QP_RETRY_CNT) {
2621         SET_FIELD(qp_params.modify_flags,
2622               QED_ROCE_MODIFY_QP_VALID_RETRY_CNT, 1);
2623         qp_params.retry_cnt = attr->retry_cnt;
2624     }
2625
2626     if (attr_mask & IB_QP_RNR_RETRY) {
2627         SET_FIELD(qp_params.modify_flags,
2628               QED_ROCE_MODIFY_QP_VALID_RNR_RETRY_CNT, 1);
2629         qp_params.rnr_retry_cnt = attr->rnr_retry;
2630     }
2631
2632     if (attr_mask & IB_QP_RQ_PSN) {
2633         SET_FIELD(qp_params.modify_flags,
2634               QED_ROCE_MODIFY_QP_VALID_RQ_PSN, 1);
2635         qp_params.rq_psn = attr->rq_psn;
2636         qp->rq_psn = attr->rq_psn;
2637     }
2638
2639     if (attr_mask & IB_QP_MAX_QP_RD_ATOMIC) {
2640         if (attr->max_rd_atomic > dev->attr.max_qp_req_rd_atomic_resc) {
2641             rc = -EINVAL;
2642             DP_ERR(dev,
2643                    "unsupported max_rd_atomic=%d, supported=%d\n",
2644                    attr->max_rd_atomic,
2645                    dev->attr.max_qp_req_rd_atomic_resc);
2646             goto err;
2647         }
2648
2649         SET_FIELD(qp_params.modify_flags,
2650               QED_RDMA_MODIFY_QP_VALID_MAX_RD_ATOMIC_REQ, 1);
2651         qp_params.max_rd_atomic_req = attr->max_rd_atomic;
2652     }
2653
2654     if (attr_mask & IB_QP_MIN_RNR_TIMER) {
2655         SET_FIELD(qp_params.modify_flags,
2656               QED_ROCE_MODIFY_QP_VALID_MIN_RNR_NAK_TIMER, 1);
2657         qp_params.min_rnr_nak_timer = attr->min_rnr_timer;
2658     }
2659
2660     if (attr_mask & IB_QP_SQ_PSN) {
2661         SET_FIELD(qp_params.modify_flags,
2662               QED_ROCE_MODIFY_QP_VALID_SQ_PSN, 1);
2663         qp_params.sq_psn = attr->sq_psn;
2664         qp->sq_psn = attr->sq_psn;
2665     }
2666
2667     if (attr_mask & IB_QP_MAX_DEST_RD_ATOMIC) {
2668         if (attr->max_dest_rd_atomic >
2669             dev->attr.max_qp_resp_rd_atomic_resc) {
2670             DP_ERR(dev,
2671                    "unsupported max_dest_rd_atomic=%d, supported=%d\n",
2672                    attr->max_dest_rd_atomic,
2673                    dev->attr.max_qp_resp_rd_atomic_resc);
2674
2675             rc = -EINVAL;
2676             goto err;
2677         }
2678
2679         SET_FIELD(qp_params.modify_flags,
2680               QED_RDMA_MODIFY_QP_VALID_MAX_RD_ATOMIC_RESP, 1);
2681         qp_params.max_rd_atomic_resp = attr->max_dest_rd_atomic;
2682     }
2683
2684     if (attr_mask & IB_QP_DEST_QPN) {
2685         SET_FIELD(qp_params.modify_flags,
2686               QED_ROCE_MODIFY_QP_VALID_DEST_QP, 1);
2687
2688         qp_params.dest_qp = attr->dest_qp_num;
2689         qp->dest_qp_num = attr->dest_qp_num;
2690     }
2691
2692     cur_state = qp->state;
2693
2694     /* Update the QP state before the actual ramrod to prevent a race with
2695      * fast path. Modifying the QP state to error will cause the device to
2696      * flush the CQEs and while polling the flushed CQEs will considered as
2697      * a potential issue if the QP isn't in error state.
2698      */
2699     if ((attr_mask & IB_QP_STATE) && qp->qp_type != IB_QPT_GSI &&
2700         !udata && qp_params.new_state == QED_ROCE_QP_STATE_ERR)
2701         qp->state = QED_ROCE_QP_STATE_ERR;
2702
2703     if (qp->qp_type != IB_QPT_GSI)
2704         rc = dev->ops->rdma_modify_qp(dev->rdma_ctx,
2705                           qp->qed_qp, &qp_params);
2706
2707     if (attr_mask & IB_QP_STATE) {
2708         if ((qp->qp_type != IB_QPT_GSI) && (!udata))
2709             rc = qedr_update_qp_state(dev, qp, cur_state,
2710                           qp_params.new_state);
2711         qp->state = qp_params.new_state;
2712     }
2713
2714 err:
2715     return rc;
2716 }
2717
2718 static int qedr_to_ib_qp_acc_flags(struct qed_rdma_query_qp_out_params *params)
2719 {
2720     int ib_qp_acc_flags = 0;
2721
2722     if (params->incoming_rdma_write_en)
2723         ib_qp_acc_flags |= IB_ACCESS_REMOTE_WRITE;
2724     if (params->incoming_rdma_read_en)
2725         ib_qp_acc_flags |= IB_ACCESS_REMOTE_READ;
2726     if (params->incoming_atomic_en)
2727         ib_qp_acc_flags |= IB_ACCESS_REMOTE_ATOMIC;
2728     ib_qp_acc_flags |= IB_ACCESS_LOCAL_WRITE;
2729     return ib_qp_acc_flags;
2730 }
2731
2732 int qedr_query_qp(struct ib_qp *ibqp,
2733           struct ib_qp_attr *qp_attr,
2734           int attr_mask, struct ib_qp_init_attr *qp_init_attr)
2735 {
2736     struct qed_rdma_query_qp_out_params params;
2737     struct qedr_qp *qp = get_qedr_qp(ibqp);
2738     struct qedr_dev *dev = qp->dev;
2739     int rc = 0;
2740
2741     memset(&params, 0, sizeof(params));
2742     memset(qp_attr, 0, sizeof(*qp_attr));
2743     memset(qp_init_attr, 0, sizeof(*qp_init_attr));
2744
2745     if (qp->qp_type != IB_QPT_GSI) {
2746         rc = dev->ops->rdma_query_qp(dev->rdma_ctx, qp->qed_qp, &params);
2747         if (rc)
2748             goto err;
2749         qp_attr->qp_state = qedr_get_ibqp_state(params.state);
2750     } else {
2751         qp_attr->qp_state = qedr_get_ibqp_state(QED_ROCE_QP_STATE_RTS);
2752     }
2753
2754     qp_attr->cur_qp_state = qedr_get_ibqp_state(params.state);
2755     qp_attr->path_mtu = ib_mtu_int_to_enum(params.mtu);
2756     qp_attr->path_mig_state = IB_MIG_MIGRATED;
2757     qp_attr->rq_psn = params.rq_psn;
2758     qp_attr->sq_psn = params.sq_psn;
2759     qp_attr->dest_qp_num = params.dest_qp;
2760
2761     qp_attr->qp_access_flags = qedr_to_ib_qp_acc_flags(&params);
2762
2763     qp_attr->cap.max_send_wr = qp->sq.max_wr;
2764     qp_attr->cap.max_recv_wr = qp->rq.max_wr;
2765     qp_attr->cap.max_send_sge = qp->sq.max_sges;
2766     qp_attr->cap.max_recv_sge = qp->rq.max_sges;
2767     qp_attr->cap.max_inline_data = dev->attr.max_inline;
2768     qp_init_attr->cap = qp_attr->cap;
2769
2770     qp_attr->ah_attr.type = RDMA_AH_ATTR_TYPE_ROCE;
2771     rdma_ah_set_grh(&qp_attr->ah_attr, NULL,
2772             params.flow_label, qp->sgid_idx,
2773             params.hop_limit_ttl, params.traffic_class_tos);
2774     rdma_ah_set_dgid_raw(&qp_attr->ah_attr, &params.dgid.bytes[0]);
2775     rdma_ah_set_port_num(&qp_attr->ah_attr, 1);
2776     rdma_ah_set_sl(&qp_attr->ah_attr, 0);
2777     qp_attr->timeout = qp->timeout;
2778     qp_attr->rnr_retry = params.rnr_retry;
2779     qp_attr->retry_cnt = params.retry_cnt;
2780     qp_attr->min_rnr_timer = params.min_rnr_nak_timer;
2781     qp_attr->pkey_index = params.pkey_index;
2782     qp_attr->port_num = 1;
2783     rdma_ah_set_path_bits(&qp_attr->ah_attr, 0);
2784     rdma_ah_set_static_rate(&qp_attr->ah_attr, 0);
2785     qp_attr->alt_pkey_index = 0;
2786     qp_attr->alt_port_num = 0;
2787     qp_attr->alt_timeout = 0;
2788     memset(&qp_attr->alt_ah_attr, 0, sizeof(qp_attr->alt_ah_attr));
2789
2790     qp_attr->sq_draining = (params.state == QED_ROCE_QP_STATE_SQD) ? 1 : 0;
2791     qp_attr->max_dest_rd_atomic = params.max_dest_rd_atomic;
2792     qp_attr->max_rd_atomic = params.max_rd_atomic;
2793     qp_attr->en_sqd_async_notify = (params.sqd_async) ? 1 : 0;
2794
2795     DP_DEBUG(dev, QEDR_MSG_QP, "QEDR_QUERY_QP: max_inline_data=%d\n",
2796          qp_attr->cap.max_inline_data);
2797
2798 err:
2799     return rc;
2800 }
2801
2802 int qedr_destroy_qp(struct ib_qp *ibqp, struct ib_udata *udata)
2803 {
2804     struct qedr_qp *qp = get_qedr_qp(ibqp);
2805     struct qedr_dev *dev = qp->dev;
2806     struct ib_qp_attr attr;
2807     int attr_mask = 0;
2808
2809     DP_DEBUG(dev, QEDR_MSG_QP, "destroy qp: destroying %p, qp type=%d\n",
2810          qp, qp->qp_type);
2811
2812     if (rdma_protocol_roce(&dev->ibdev, 1)) {
2813         if ((qp->state != QED_ROCE_QP_STATE_RESET) &&
2814             (qp->state != QED_ROCE_QP_STATE_ERR) &&
2815             (qp->state != QED_ROCE_QP_STATE_INIT)) {
2816
2817             attr.qp_state = IB_QPS_ERR;
2818             attr_mask |= IB_QP_STATE;
2819
2820             /* Change the QP state to ERROR */
2821             qedr_modify_qp(ibqp, &attr, attr_mask, NULL);
2822         }
2823     } else {
2824         /* If connection establishment started the WAIT_FOR_CONNECT
2825          * bit will be on and we need to Wait for the establishment
2826          * to complete before destroying the qp.
2827          */
2828         if (test_and_set_bit(QEDR_IWARP_CM_WAIT_FOR_CONNECT,
2829                      &qp->iwarp_cm_flags))
2830             wait_for_completion(&qp->iwarp_cm_comp);
2831
2832         /* If graceful disconnect started, the WAIT_FOR_DISCONNECT
2833          * bit will be on, and we need to wait for the disconnect to
2834          * complete before continuing. We can use the same completion,
2835          * iwarp_cm_comp, since this is the only place that waits for
2836          * this completion and it is sequential. In addition,
2837          * disconnect can't occur before the connection is fully
2838          * established, therefore if WAIT_FOR_DISCONNECT is on it
2839          * means WAIT_FOR_CONNECT is also on and the completion for
2840          * CONNECT already occurred.
2841          */
2842         if (test_and_set_bit(QEDR_IWARP_CM_WAIT_FOR_DISCONNECT,
2843                      &qp->iwarp_cm_flags))
2844             wait_for_completion(&qp->iwarp_cm_comp);
2845     }
2846
2847     if (qp->qp_type == IB_QPT_GSI)
2848         qedr_destroy_gsi_qp(dev);
2849
2850     /* We need to remove the entry from the xarray before we release the
2851      * qp_id to avoid a race of the qp_id being reallocated and failing
2852      * on xa_insert
2853      */
2854     if (rdma_protocol_iwarp(&dev->ibdev, 1))
2855         xa_erase(&dev->qps, qp->qp_id);
2856
2857     qedr_free_qp_resources(dev, qp, udata);
2858
2859     if (rdma_protocol_iwarp(&dev->ibdev, 1)) {
2860         qedr_iw_qp_rem_ref(&qp->ibqp);
2861         wait_for_completion(&qp->qp_rel_comp);
2862     }
2863
2864     return 0;
2865 }
2866
2867 int qedr_create_ah(struct ib_ah *ibah, struct rdma_ah_init_attr *init_attr,
2868            struct ib_udata *udata)
2869 {
2870     struct qedr_ah *ah = get_qedr_ah(ibah);
2871
2872     rdma_copy_ah_attr(&ah->attr, init_attr->ah_attr);
2873
2874     return 0;
2875 }
2876
2877 int qedr_destroy_ah(struct ib_ah *ibah, u32 flags)
2878 {
2879     struct qedr_ah *ah = get_qedr_ah(ibah);
2880
2881     rdma_destroy_ah_attr(&ah->attr);
2882     return 0;
2883 }
2884
2885 static void free_mr_info(struct qedr_dev *dev, struct mr_info *info)
2886 {
2887     struct qedr_pbl *pbl, *tmp;
2888
2889     if (info->pbl_table)
2890         list_add_tail(&info->pbl_table->list_entry,
2891                   &info->free_pbl_list);
2892
2893     if (!list_empty(&info->inuse_pbl_list))
2894         list_splice(&info->inuse_pbl_list, &info->free_pbl_list);
2895
2896     list_for_each_entry_safe(pbl, tmp, &info->free_pbl_list, list_entry) {
2897         list_del(&pbl->list_entry);
2898         qedr_free_pbl(dev, &info->pbl_info, pbl);
2899     }
2900 }
2901
2902 static int init_mr_info(struct qedr_dev *dev, struct mr_info *info,
2903             size_t page_list_len, bool two_layered)
2904 {
2905     struct qedr_pbl *tmp;
2906     int rc;
2907
2908     INIT_LIST_HEAD(&info->free_pbl_list);
2909     INIT_LIST_HEAD(&info->inuse_pbl_list);
2910
2911     rc = qedr_prepare_pbl_tbl(dev, &info->pbl_info,
2912                   page_list_len, two_layered);
2913     if (rc)
2914         goto done;
2915
2916     info->pbl_table = qedr_alloc_pbl_tbl(dev, &info->pbl_info, GFP_KERNEL);
2917     if (IS_ERR(info->pbl_table)) {
2918         rc = PTR_ERR(info->pbl_table);
2919         goto done;
2920     }
2921
2922     DP_DEBUG(dev, QEDR_MSG_MR, "pbl_table_pa = %pa\n",
2923          &info->pbl_table->pa);
2924
2925     /* in usual case we use 2 PBLs, so we add one to free
2926      * list and allocating another one
2927      */
2928     tmp = qedr_alloc_pbl_tbl(dev, &info->pbl_info, GFP_KERNEL);
2929     if (IS_ERR(tmp)) {
2930         DP_DEBUG(dev, QEDR_MSG_MR, "Extra PBL is not allocated\n");
2931         goto done;
2932     }
2933
2934     list_add_tail(&tmp->list_entry, &info->free_pbl_list);
2935
2936     DP_DEBUG(dev, QEDR_MSG_MR, "extra pbl_table_pa = %pa\n", &tmp->pa);
2937
2938 done:
2939     if (rc)
2940         free_mr_info(dev, info);
2941
2942     return rc;
2943 }
2944
2945 struct ib_mr *qedr_reg_user_mr(struct ib_pd *ibpd, u64 start, u64 len,
2946                    u64 usr_addr, int acc, struct ib_udata *udata)
2947 {
2948     struct qedr_dev *dev = get_qedr_dev(ibpd->device);
2949     struct qedr_mr *mr;
2950     struct qedr_pd *pd;
2951     int rc = -ENOMEM;
2952
2953     pd = get_qedr_pd(ibpd);
2954     DP_DEBUG(dev, QEDR_MSG_MR,
2955          "qedr_register user mr pd = %d start = %lld, len = %lld, usr_addr = %lld, acc = %d\n",
2956          pd->pd_id, start, len, usr_addr, acc);
2957
2958     if (acc & IB_ACCESS_REMOTE_WRITE && !(acc & IB_ACCESS_LOCAL_WRITE))
2959         return ERR_PTR(-EINVAL);
2960
2961     mr = kzalloc(sizeof(*mr), GFP_KERNEL);
2962     if (!mr)
2963         return ERR_PTR(rc);
2964
2965     mr->type = QEDR_MR_USER;
2966
2967     mr->umem = ib_umem_get(ibpd->device, start, len, acc);
2968     if (IS_ERR(mr->umem)) {
2969         rc = -EFAULT;
2970         goto err0;
2971     }
2972
2973     rc = init_mr_info(dev, &mr->info,
2974               ib_umem_num_dma_blocks(mr->umem, PAGE_SIZE), 1);
2975     if (rc)
2976         goto err1;
2977
2978     qedr_populate_pbls(dev, mr->umem, mr->info.pbl_table,
2979                &mr->info.pbl_info, PAGE_SHIFT);
2980
2981     rc = dev->ops->rdma_alloc_tid(dev->rdma_ctx, &mr->hw_mr.itid);
2982     if (rc) {
2983         if (rc == -EINVAL)
2984             DP_ERR(dev, "Out of MR resources\n");
2985         else
2986             DP_ERR(dev, "roce alloc tid returned error %d\n", rc);
2987
2988         goto err1;
2989     }
2990
2991     /* Index only, 18 bit long, lkey = itid << 8 | key */
2992     mr->hw_mr.tid_type = QED_RDMA_TID_REGISTERED_MR;
2993     mr->hw_mr.key = 0;
2994     mr->hw_mr.pd = pd->pd_id;
2995     mr->hw_mr.local_read = 1;
2996     mr->hw_mr.local_write = (acc & IB_ACCESS_LOCAL_WRITE) ? 1 : 0;
2997     mr->hw_mr.remote_read = (acc & IB_ACCESS_REMOTE_READ) ? 1 : 0;
2998     mr->hw_mr.remote_write = (acc & IB_ACCESS_REMOTE_WRITE) ? 1 : 0;
2999     mr->hw_mr.remote_atomic = (acc & IB_ACCESS_REMOTE_ATOMIC) ? 1 : 0;
3000     mr->hw_mr.mw_bind = false;
3001     mr->hw_mr.pbl_ptr = mr->info.pbl_table[0].pa;
3002     mr->hw_mr.pbl_two_level = mr->info.pbl_info.two_layered;
3003     mr->hw_mr.pbl_page_size_log = ilog2(mr->info.pbl_info.pbl_size);
3004     mr->hw_mr.page_size_log = PAGE_SHIFT;
3005     mr->hw_mr.length = len;
3006     mr->hw_mr.vaddr = usr_addr;
3007     mr->hw_mr.phy_mr = false;
3008     mr->hw_mr.dma_mr = false;
3009
3010     rc = dev->ops->rdma_register_tid(dev->rdma_ctx, &mr->hw_mr);
3011     if (rc) {
3012         DP_ERR(dev, "roce register tid returned an error %d\n", rc);
3013         goto err2;
3014     }
3015
3016     mr->ibmr.lkey = mr->hw_mr.itid << 8 | mr->hw_mr.key;
3017     if (mr->hw_mr.remote_write || mr->hw_mr.remote_read ||
3018         mr->hw_mr.remote_atomic)
3019         mr->ibmr.rkey = mr->hw_mr.itid << 8 | mr->hw_mr.key;
3020
3021     DP_DEBUG(dev, QEDR_MSG_MR, "register user mr lkey: %x\n",
3022          mr->ibmr.lkey);
3023     return &mr->ibmr;
3024
3025 err2:
3026     dev->ops->rdma_free_tid(dev->rdma_ctx, mr->hw_mr.itid);
3027 err1:
3028     qedr_free_pbl(dev, &mr->info.pbl_info, mr->info.pbl_table);
3029 err0:
3030     kfree(mr);
3031     return ERR_PTR(rc);
3032 }
3033
3034 int qedr_dereg_mr(struct ib_mr *ib_mr, struct ib_udata *udata)
3035 {
3036     struct qedr_mr *mr = get_qedr_mr(ib_mr);
3037     struct qedr_dev *dev = get_qedr_dev(ib_mr->device);
3038     int rc = 0;
3039
3040     rc = dev->ops->rdma_deregister_tid(dev->rdma_ctx, mr->hw_mr.itid);
3041     if (rc)
3042         return rc;
3043
3044     dev->ops->rdma_free_tid(dev->rdma_ctx, mr->hw_mr.itid);
3045
3046     if (mr->type != QEDR_MR_DMA)
3047         free_mr_info(dev, &mr->info);
3048
3049     /* it could be user registered memory. */
3050     ib_umem_release(mr->umem);
3051
3052     kfree(mr);
3053
3054     return rc;
3055 }
3056
3057 static struct qedr_mr *__qedr_alloc_mr(struct ib_pd *ibpd,
3058                        int max_page_list_len)
3059 {
3060     struct qedr_pd *pd = get_qedr_pd(ibpd);
3061     struct qedr_dev *dev = get_qedr_dev(ibpd->device);
3062     struct qedr_mr *mr;
3063     int rc = -ENOMEM;
3064
3065     DP_DEBUG(dev, QEDR_MSG_MR,
3066          "qedr_alloc_frmr pd = %d max_page_list_len= %d\n", pd->pd_id,
3067          max_page_list_len);
3068
3069     mr = kzalloc(sizeof(*mr), GFP_KERNEL);
3070     if (!mr)
3071         return ERR_PTR(rc);
3072
3073     mr->dev = dev;
3074     mr->type = QEDR_MR_FRMR;
3075
3076     rc = init_mr_info(dev, &mr->info, max_page_list_len, 1);
3077     if (rc)
3078         goto err0;
3079
3080     rc = dev->ops->rdma_alloc_tid(dev->rdma_ctx, &mr->hw_mr.itid);
3081     if (rc) {
3082         if (rc == -EINVAL)
3083             DP_ERR(dev, "Out of MR resources\n");
3084         else
3085             DP_ERR(dev, "roce alloc tid returned error %d\n", rc);
3086
3087         goto err1;
3088     }
3089
3090     /* Index only, 18 bit long, lkey = itid << 8 | key */
3091     mr->hw_mr.tid_type = QED_RDMA_TID_FMR;
3092     mr->hw_mr.key = 0;
3093     mr->hw_mr.pd = pd->pd_id;
3094     mr->hw_mr.local_read = 1;
3095     mr->hw_mr.local_write = 0;
3096     mr->hw_mr.remote_read = 0;
3097     mr->hw_mr.remote_write = 0;
3098     mr->hw_mr.remote_atomic = 0;
3099     mr->hw_mr.mw_bind = false;
3100     mr->hw_mr.pbl_ptr = 0;
3101     mr->hw_mr.pbl_two_level = mr->info.pbl_info.two_layered;
3102     mr->hw_mr.pbl_page_size_log = ilog2(mr->info.pbl_info.pbl_size);
3103     mr->hw_mr.length = 0;
3104     mr->hw_mr.vaddr = 0;
3105     mr->hw_mr.phy_mr = true;
3106     mr->hw_mr.dma_mr = false;
3107
3108     rc = dev->ops->rdma_register_tid(dev->rdma_ctx, &mr->hw_mr);
3109     if (rc) {
3110         DP_ERR(dev, "roce register tid returned an error %d\n", rc);
3111         goto err2;
3112     }
3113
3114     mr->ibmr.lkey = mr->hw_mr.itid << 8 | mr->hw_mr.key;
3115     mr->ibmr.rkey = mr->ibmr.lkey;
3116
3117     DP_DEBUG(dev, QEDR_MSG_MR, "alloc frmr: %x\n", mr->ibmr.lkey);
3118     return mr;
3119
3120 err2:
3121     dev->ops->rdma_free_tid(dev->rdma_ctx, mr->hw_mr.itid);
3122 err1:
3123     qedr_free_pbl(dev, &mr->info.pbl_info, mr->info.pbl_table);
3124 err0:
3125     kfree(mr);
3126     return ERR_PTR(rc);
3127 }
3128
3129 struct ib_mr *qedr_alloc_mr(struct ib_pd *ibpd, enum ib_mr_type mr_type,
3130                 u32 max_num_sg)
3131 {
3132     struct qedr_mr *mr;
3133
3134     if (mr_type != IB_MR_TYPE_MEM_REG)
3135         return ERR_PTR(-EINVAL);
3136
3137     mr = __qedr_alloc_mr(ibpd, max_num_sg);
3138
3139     if (IS_ERR(mr))
3140         return ERR_PTR(-EINVAL);
3141
3142     return &mr->ibmr;
3143 }
3144
3145 static int qedr_set_page(struct ib_mr *ibmr, u64 addr)
3146 {
3147     struct qedr_mr *mr = get_qedr_mr(ibmr);
3148     struct qedr_pbl *pbl_table;
3149     struct regpair *pbe;
3150     u32 pbes_in_page;
3151
3152     if (unlikely(mr->npages == mr->info.pbl_info.num_pbes)) {
3153         DP_ERR(mr->dev, "qedr_set_page fails when %d\n", mr->npages);
3154         return -ENOMEM;
3155     }
3156
3157     DP_DEBUG(mr->dev, QEDR_MSG_MR, "qedr_set_page pages[%d] = 0x%llx\n",
3158          mr->npages, addr);
3159
3160     pbes_in_page = mr->info.pbl_info.pbl_size / sizeof(u64);
3161     pbl_table = mr->info.pbl_table + (mr->npages / pbes_in_page);
3162     pbe = (struct regpair *)pbl_table->va;
3163     pbe +=  mr->npages % pbes_in_page;
3164     pbe->lo = cpu_to_le32((u32)addr);
3165     pbe->hi = cpu_to_le32((u32)upper_32_bits(addr));
3166
3167     mr->npages++;
3168
3169     return 0;
3170 }
3171
3172 static void handle_completed_mrs(struct qedr_dev *dev, struct mr_info *info)
3173 {
3174     int work = info->completed - info->completed_handled - 1;
3175
3176     DP_DEBUG(dev, QEDR_MSG_MR, "Special FMR work = %d\n", work);
3177     while (work-- > 0 && !list_empty(&info->inuse_pbl_list)) {
3178         struct qedr_pbl *pbl;
3179
3180         /* Free all the page list that are possible to be freed
3181          * (all the ones that were invalidated), under the assumption
3182          * that if an FMR was completed successfully that means that
3183          * if there was an invalidate operation before it also ended
3184          */
3185         pbl = list_first_entry(&info->inuse_pbl_list,
3186                        struct qedr_pbl, list_entry);
3187         list_move_tail(&pbl->list_entry, &info->free_pbl_list);
3188         info->completed_handled++;
3189     }
3190 }
3191
3192 int qedr_map_mr_sg(struct ib_mr *ibmr, struct scatterlist *sg,
3193            int sg_nents, unsigned int *sg_offset)
3194 {
3195     struct qedr_mr *mr = get_qedr_mr(ibmr);
3196
3197     mr->npages = 0;
3198
3199     handle_completed_mrs(mr->dev, &mr->info);
3200     return ib_sg_to_pages(ibmr, sg, sg_nents, NULL, qedr_set_page);
3201 }
3202
3203 struct ib_mr *qedr_get_dma_mr(struct ib_pd *ibpd, int acc)
3204 {
3205     struct qedr_dev *dev = get_qedr_dev(ibpd->device);
3206     struct qedr_pd *pd = get_qedr_pd(ibpd);
3207     struct qedr_mr *mr;
3208     int rc;
3209
3210     mr = kzalloc(sizeof(*mr), GFP_KERNEL);
3211     if (!mr)
3212         return ERR_PTR(-ENOMEM);
3213
3214     mr->type = QEDR_MR_DMA;
3215
3216     rc = dev->ops->rdma_alloc_tid(dev->rdma_ctx, &mr->hw_mr.itid);
3217     if (rc) {
3218         if (rc == -EINVAL)
3219             DP_ERR(dev, "Out of MR resources\n");
3220         else
3221             DP_ERR(dev, "roce alloc tid returned error %d\n", rc);
3222
3223         goto err1;
3224     }
3225
3226     /* index only, 18 bit long, lkey = itid << 8 | key */
3227     mr->hw_mr.tid_type = QED_RDMA_TID_REGISTERED_MR;
3228     mr->hw_mr.pd = pd->pd_id;
3229     mr->hw_mr.local_read = 1;
3230     mr->hw_mr.local_write = (acc & IB_ACCESS_LOCAL_WRITE) ? 1 : 0;
3231     mr->hw_mr.remote_read = (acc & IB_ACCESS_REMOTE_READ) ? 1 : 0;
3232     mr->hw_mr.remote_write = (acc & IB_ACCESS_REMOTE_WRITE) ? 1 : 0;
3233     mr->hw_mr.remote_atomic = (acc & IB_ACCESS_REMOTE_ATOMIC) ? 1 : 0;
3234     mr->hw_mr.dma_mr = true;
3235
3236     rc = dev->ops->rdma_register_tid(dev->rdma_ctx, &mr->hw_mr);
3237     if (rc) {
3238         DP_ERR(dev, "roce register tid returned an error %d\n", rc);
3239         goto err2;
3240     }
3241
3242     mr->ibmr.lkey = mr->hw_mr.itid << 8 | mr->hw_mr.key;
3243     if (mr->hw_mr.remote_write || mr->hw_mr.remote_read ||
3244         mr->hw_mr.remote_atomic)
3245         mr->ibmr.rkey = mr->hw_mr.itid << 8 | mr->hw_mr.key;
3246
3247     DP_DEBUG(dev, QEDR_MSG_MR, "get dma mr: lkey = %x\n", mr->ibmr.lkey);
3248     return &mr->ibmr;
3249
3250 err2:
3251     dev->ops->rdma_free_tid(dev->rdma_ctx, mr->hw_mr.itid);
3252 err1:
3253     kfree(mr);
3254     return ERR_PTR(rc);
3255 }
3256
3257 static inline int qedr_wq_is_full(struct qedr_qp_hwq_info *wq)
3258 {
3259     return (((wq->prod + 1) % wq->max_wr) == wq->cons);
3260 }
3261
3262 static int sge_data_len(struct ib_sge *sg_list, int num_sge)
3263 {
3264     int i, len = 0;
3265
3266     for (i = 0; i < num_sge; i++)
3267         len += sg_list[i].length;
3268
3269     return len;
3270 }
3271
3272 static void swap_wqe_data64(u64 *p)
3273 {
3274     int i;
3275
3276     for (i = 0; i < QEDR_SQE_ELEMENT_SIZE / sizeof(u64); i++, p++)
3277         *p = cpu_to_be64(cpu_to_le64(*p));
3278 }
3279
3280 static u32 qedr_prepare_sq_inline_data(struct qedr_dev *dev,
3281                        struct qedr_qp *qp, u8 *wqe_size,
3282                        const struct ib_send_wr *wr,
3283                        const struct ib_send_wr **bad_wr,
3284                        u8 *bits, u8 bit)
3285 {
3286     u32 data_size = sge_data_len(wr->sg_list, wr->num_sge);
3287     char *seg_prt, *wqe;
3288     int i, seg_siz;
3289
3290     if (data_size > ROCE_REQ_MAX_INLINE_DATA_SIZE) {
3291         DP_ERR(dev, "Too much inline data in WR: %d\n", data_size);
3292         *bad_wr = wr;
3293         return 0;
3294     }
3295
3296     if (!data_size)
3297         return data_size;
3298
3299     *bits |= bit;
3300
3301     seg_prt = NULL;
3302     wqe = NULL;
3303     seg_siz = 0;
3304
3305     /* Copy data inline */
3306     for (i = 0; i < wr->num_sge; i++) {
3307         u32 len = wr->sg_list[i].length;
3308         void *src = (void *)(uintptr_t)wr->sg_list[i].addr;
3309
3310         while (len > 0) {
3311             u32 cur;
3312
3313             /* New segment required */
3314             if (!seg_siz) {
3315                 wqe = (char *)qed_chain_produce(&qp->sq.pbl);
3316                 seg_prt = wqe;
3317                 seg_siz = sizeof(struct rdma_sq_common_wqe);
3318                 (*wqe_size)++;
3319             }
3320
3321             /* Calculate currently allowed length */
3322             cur = min_t(u32, len, seg_siz);
3323             memcpy(seg_prt, src, cur);
3324
3325             /* Update segment variables */
3326             seg_prt += cur;
3327             seg_siz -= cur;
3328
3329             /* Update sge variables */
3330             src += cur;
3331             len -= cur;
3332
3333             /* Swap fully-completed segments */
3334             if (!seg_siz)
3335                 swap_wqe_data64((u64 *)wqe);
3336         }
3337     }
3338
3339     /* swap last not completed segment */
3340     if (seg_siz)
3341         swap_wqe_data64((u64 *)wqe);
3342
3343     return data_size;
3344 }
3345
3346 #define RQ_SGE_SET(sge, vaddr, vlength, vflags)         \
3347     do {                            \
3348         DMA_REGPAIR_LE(sge->addr, vaddr);       \
3349         (sge)->length = cpu_to_le32(vlength);       \
3350         (sge)->flags = cpu_to_le32(vflags);     \
3351     } while (0)
3352
3353 #define SRQ_HDR_SET(hdr, vwr_id, num_sge)           \
3354     do {                            \
3355         DMA_REGPAIR_LE(hdr->wr_id, vwr_id);     \
3356         (hdr)->num_sges = num_sge;          \
3357     } while (0)
3358
3359 #define SRQ_SGE_SET(sge, vaddr, vlength, vlkey)         \
3360     do {                            \
3361         DMA_REGPAIR_LE(sge->addr, vaddr);       \
3362         (sge)->length = cpu_to_le32(vlength);       \
3363         (sge)->l_key = cpu_to_le32(vlkey);      \
3364     } while (0)
3365
3366 static u32 qedr_prepare_sq_sges(struct qedr_qp *qp, u8 *wqe_size,
3367                 const struct ib_send_wr *wr)
3368 {
3369     u32 data_size = 0;
3370     int i;
3371
3372     for (i = 0; i < wr->num_sge; i++) {
3373         struct rdma_sq_sge *sge = qed_chain_produce(&qp->sq.pbl);
3374
3375         DMA_REGPAIR_LE(sge->addr, wr->sg_list[i].addr);
3376         sge->l_key = cpu_to_le32(wr->sg_list[i].lkey);
3377         sge->length = cpu_to_le32(wr->sg_list[i].length);
3378         data_size += wr->sg_list[i].length;
3379     }
3380
3381     if (wqe_size)
3382         *wqe_size += wr->num_sge;
3383
3384     return data_size;
3385 }
3386
3387 static u32 qedr_prepare_sq_rdma_data(struct qedr_dev *dev,
3388                      struct qedr_qp *qp,
3389                      struct rdma_sq_rdma_wqe_1st *rwqe,
3390                      struct rdma_sq_rdma_wqe_2nd *rwqe2,
3391                      const struct ib_send_wr *wr,
3392                      const struct ib_send_wr **bad_wr)
3393 {
3394     rwqe2->r_key = cpu_to_le32(rdma_wr(wr)->rkey);
3395     DMA_REGPAIR_LE(rwqe2->remote_va, rdma_wr(wr)->remote_addr);
3396
3397     if (wr->send_flags & IB_SEND_INLINE &&
3398         (wr->opcode == IB_WR_RDMA_WRITE_WITH_IMM ||
3399          wr->opcode == IB_WR_RDMA_WRITE)) {
3400         u8 flags = 0;
3401
3402         SET_FIELD2(flags, RDMA_SQ_RDMA_WQE_1ST_INLINE_FLG, 1);
3403         return qedr_prepare_sq_inline_data(dev, qp, &rwqe->wqe_size, wr,
3404                            bad_wr, &rwqe->flags, flags);
3405     }
3406
3407     return qedr_prepare_sq_sges(qp, &rwqe->wqe_size, wr);
3408 }
3409
3410 static u32 qedr_prepare_sq_send_data(struct qedr_dev *dev,
3411                      struct qedr_qp *qp,
3412                      struct rdma_sq_send_wqe_1st *swqe,
3413                      struct rdma_sq_send_wqe_2st *swqe2,
3414                      const struct ib_send_wr *wr,
3415                      const struct ib_send_wr **bad_wr)
3416 {
3417     memset(swqe2, 0, sizeof(*swqe2));
3418     if (wr->send_flags & IB_SEND_INLINE) {
3419         u8 flags = 0;
3420
3421         SET_FIELD2(flags, RDMA_SQ_SEND_WQE_INLINE_FLG, 1);
3422         return qedr_prepare_sq_inline_data(dev, qp, &swqe->wqe_size, wr,
3423                            bad_wr, &swqe->flags, flags);
3424     }
3425
3426     return qedr_prepare_sq_sges(qp, &swqe->wqe_size, wr);
3427 }
3428
3429 static int qedr_prepare_reg(struct qedr_qp *qp,
3430                 struct rdma_sq_fmr_wqe_1st *fwqe1,
3431                 const struct ib_reg_wr *wr)
3432 {
3433     struct qedr_mr *mr = get_qedr_mr(wr->mr);
3434     struct rdma_sq_fmr_wqe_2nd *fwqe2;
3435
3436     fwqe2 = (struct rdma_sq_fmr_wqe_2nd *)qed_chain_produce(&qp->sq.pbl);
3437     fwqe1->addr.hi = upper_32_bits(mr->ibmr.iova);
3438     fwqe1->addr.lo = lower_32_bits(mr->ibmr.iova);
3439     fwqe1->l_key = wr->key;
3440
3441     fwqe2->access_ctrl = 0;
3442
3443     SET_FIELD2(fwqe2->access_ctrl, RDMA_SQ_FMR_WQE_2ND_REMOTE_READ,
3444            !!(wr->access & IB_ACCESS_REMOTE_READ));
3445     SET_FIELD2(fwqe2->access_ctrl, RDMA_SQ_FMR_WQE_2ND_REMOTE_WRITE,
3446            !!(wr->access & IB_ACCESS_REMOTE_WRITE));
3447     SET_FIELD2(fwqe2->access_ctrl, RDMA_SQ_FMR_WQE_2ND_ENABLE_ATOMIC,
3448            !!(wr->access & IB_ACCESS_REMOTE_ATOMIC));
3449     SET_FIELD2(fwqe2->access_ctrl, RDMA_SQ_FMR_WQE_2ND_LOCAL_READ, 1);
3450     SET_FIELD2(fwqe2->access_ctrl, RDMA_SQ_FMR_WQE_2ND_LOCAL_WRITE,
3451            !!(wr->access & IB_ACCESS_LOCAL_WRITE));
3452     fwqe2->fmr_ctrl = 0;
3453
3454     SET_FIELD2(fwqe2->fmr_ctrl, RDMA_SQ_FMR_WQE_2ND_PAGE_SIZE_LOG,
3455            ilog2(mr->ibmr.page_size) - 12);
3456
3457     fwqe2->length_hi = 0;
3458     fwqe2->length_lo = mr->ibmr.length;
3459     fwqe2->pbl_addr.hi = upper_32_bits(mr->info.pbl_table->pa);
3460     fwqe2->pbl_addr.lo = lower_32_bits(mr->info.pbl_table->pa);
3461
3462     qp->wqe_wr_id[qp->sq.prod].mr = mr;
3463
3464     return 0;
3465 }
3466
3467 static enum ib_wc_opcode qedr_ib_to_wc_opcode(enum ib_wr_opcode opcode)
3468 {
3469     switch (opcode) {
3470     case IB_WR_RDMA_WRITE:
3471     case IB_WR_RDMA_WRITE_WITH_IMM:
3472         return IB_WC_RDMA_WRITE;
3473     case IB_WR_SEND_WITH_IMM:
3474     case IB_WR_SEND:
3475     case IB_WR_SEND_WITH_INV:
3476         return IB_WC_SEND;
3477     case IB_WR_RDMA_READ:
3478     case IB_WR_RDMA_READ_WITH_INV:
3479         return IB_WC_RDMA_READ;
3480     case IB_WR_ATOMIC_CMP_AND_SWP:
3481         return IB_WC_COMP_SWAP;
3482     case IB_WR_ATOMIC_FETCH_AND_ADD:
3483         return IB_WC_FETCH_ADD;
3484     case IB_WR_REG_MR:
3485         return IB_WC_REG_MR;
3486     case IB_WR_LOCAL_INV:
3487         return IB_WC_LOCAL_INV;
3488     default:
3489         return IB_WC_SEND;
3490     }
3491 }
3492
3493 static inline bool qedr_can_post_send(struct qedr_qp *qp,
3494                       const struct ib_send_wr *wr)
3495 {
3496     int wq_is_full, err_wr, pbl_is_full;
3497     struct qedr_dev *dev = qp->dev;
3498
3499     /* prevent SQ overflow and/or processing of a bad WR */
3500     err_wr = wr->num_sge > qp->sq.max_sges;
3501     wq_is_full = qedr_wq_is_full(&qp->sq);
3502     pbl_is_full = qed_chain_get_elem_left_u32(&qp->sq.pbl) <
3503               QEDR_MAX_SQE_ELEMENTS_PER_SQE;
3504     if (wq_is_full || err_wr || pbl_is_full) {
3505         if (wq_is_full && !(qp->err_bitmap & QEDR_QP_ERR_SQ_FULL)) {
3506             DP_ERR(dev,
3507                    "error: WQ is full. Post send on QP %p failed (this error appears only once)\n",
3508                    qp);
3509             qp->err_bitmap |= QEDR_QP_ERR_SQ_FULL;
3510         }
3511
3512         if (err_wr && !(qp->err_bitmap & QEDR_QP_ERR_BAD_SR)) {
3513             DP_ERR(dev,
3514                    "error: WR is bad. Post send on QP %p failed (this error appears only once)\n",
3515                    qp);
3516             qp->err_bitmap |= QEDR_QP_ERR_BAD_SR;
3517         }
3518
3519         if (pbl_is_full &&
3520             !(qp->err_bitmap & QEDR_QP_ERR_SQ_PBL_FULL)) {
3521             DP_ERR(dev,
3522                    "error: WQ PBL is full. Post send on QP %p failed (this error appears only once)\n",
3523                    qp);
3524             qp->err_bitmap |= QEDR_QP_ERR_SQ_PBL_FULL;
3525         }
3526         return false;
3527     }
3528     return true;
3529 }
3530
3531 static int __qedr_post_send(struct ib_qp *ibqp, const struct ib_send_wr *wr,
3532                 const struct ib_send_wr **bad_wr)
3533 {
3534     struct qedr_dev *dev = get_qedr_dev(ibqp->device);
3535     struct qedr_qp *qp = get_qedr_qp(ibqp);
3536     struct rdma_sq_atomic_wqe_1st *awqe1;
3537     struct rdma_sq_atomic_wqe_2nd *awqe2;
3538     struct rdma_sq_atomic_wqe_3rd *awqe3;
3539     struct rdma_sq_send_wqe_2st *swqe2;
3540     struct rdma_sq_local_inv_wqe *iwqe;
3541     struct rdma_sq_rdma_wqe_2nd *rwqe2;
3542     struct rdma_sq_send_wqe_1st *swqe;
3543     struct rdma_sq_rdma_wqe_1st *rwqe;
3544     struct rdma_sq_fmr_wqe_1st *fwqe1;
3545     struct rdma_sq_common_wqe *wqe;
3546     u32 length;
3547     int rc = 0;
3548     bool comp;
3549
3550     if (!qedr_can_post_send(qp, wr)) {
3551         *bad_wr = wr;
3552         return -ENOMEM;
3553     }
3554
3555     wqe = qed_chain_produce(&qp->sq.pbl);
3556     qp->wqe_wr_id[qp->sq.prod].signaled =
3557         !!(wr->send_flags & IB_SEND_SIGNALED) || qp->signaled;
3558
3559     wqe->flags = 0;
3560     SET_FIELD2(wqe->flags, RDMA_SQ_SEND_WQE_SE_FLG,
3561            !!(wr->send_flags & IB_SEND_SOLICITED));
3562     comp = (!!(wr->send_flags & IB_SEND_SIGNALED)) || qp->signaled;
3563     SET_FIELD2(wqe->flags, RDMA_SQ_SEND_WQE_COMP_FLG, comp);
3564     SET_FIELD2(wqe->flags, RDMA_SQ_SEND_WQE_RD_FENCE_FLG,
3565            !!(wr->send_flags & IB_SEND_FENCE));
3566     wqe->prev_wqe_size = qp->prev_wqe_size;
3567
3568     qp->wqe_wr_id[qp->sq.prod].opcode = qedr_ib_to_wc_opcode(wr->opcode);
3569
3570     switch (wr->opcode) {
3571     case IB_WR_SEND_WITH_IMM:
3572         if (unlikely(rdma_protocol_iwarp(&dev->ibdev, 1))) {
3573             rc = -EINVAL;
3574             *bad_wr = wr;
3575             break;
3576         }
3577         wqe->req_type = RDMA_SQ_REQ_TYPE_SEND_WITH_IMM;
3578         swqe = (struct rdma_sq_send_wqe_1st *)wqe;
3579         swqe->wqe_size = 2;
3580         swqe2 = qed_chain_produce(&qp->sq.pbl);
3581
3582         swqe->inv_key_or_imm_data = cpu_to_le32(be32_to_cpu(wr->ex.imm_data));
3583         length = qedr_prepare_sq_send_data(dev, qp, swqe, swqe2,
3584                            wr, bad_wr);
3585         swqe->length = cpu_to_le32(length);
3586         qp->wqe_wr_id[qp->sq.prod].wqe_size = swqe->wqe_size;
3587         qp->prev_wqe_size = swqe->wqe_size;
3588         qp->wqe_wr_id[qp->sq.prod].bytes_len = swqe->length;
3589         break;
3590     case IB_WR_SEND:
3591         wqe->req_type = RDMA_SQ_REQ_TYPE_SEND;
3592         swqe = (struct rdma_sq_send_wqe_1st *)wqe;
3593
3594         swqe->wqe_size = 2;
3595         swqe2 = qed_chain_produce(&qp->sq.pbl);
3596         length = qedr_prepare_sq_send_data(dev, qp, swqe, swqe2,
3597                            wr, bad_wr);
3598         swqe->length = cpu_to_le32(length);
3599         qp->wqe_wr_id[qp->sq.prod].wqe_size = swqe->wqe_size;
3600         qp->prev_wqe_size = swqe->wqe_size;
3601         qp->wqe_wr_id[qp->sq.prod].bytes_len = swqe->length;
3602         break;
3603     case IB_WR_SEND_WITH_INV:
3604         wqe->req_type = RDMA_SQ_REQ_TYPE_SEND_WITH_INVALIDATE;
3605         swqe = (struct rdma_sq_send_wqe_1st *)wqe;
3606         swqe2 = qed_chain_produce(&qp->sq.pbl);
3607         swqe->wqe_size = 2;
3608         swqe->inv_key_or_imm_data = cpu_to_le32(wr->ex.invalidate_rkey);
3609         length = qedr_prepare_sq_send_data(dev, qp, swqe, swqe2,
3610                            wr, bad_wr);
3611         swqe->length = cpu_to_le32(length);
3612         qp->wqe_wr_id[qp->sq.prod].wqe_size = swqe->wqe_size;
3613         qp->prev_wqe_size = swqe->wqe_size;
3614         qp->wqe_wr_id[qp->sq.prod].bytes_len = swqe->length;
3615         break;
3616
3617     case IB_WR_RDMA_WRITE_WITH_IMM:
3618         if (unlikely(rdma_protocol_iwarp(&dev->ibdev, 1))) {
3619             rc = -EINVAL;
3620             *bad_wr = wr;
3621             break;
3622         }
3623         wqe->req_type = RDMA_SQ_REQ_TYPE_RDMA_WR_WITH_IMM;
3624         rwqe = (struct rdma_sq_rdma_wqe_1st *)wqe;
3625
3626         rwqe->wqe_size = 2;
3627         rwqe->imm_data = htonl(cpu_to_le32(wr->ex.imm_data));
3628         rwqe2 = qed_chain_produce(&qp->sq.pbl);
3629         length = qedr_prepare_sq_rdma_data(dev, qp, rwqe, rwqe2,
3630                            wr, bad_wr);
3631         rwqe->length = cpu_to_le32(length);
3632         qp->wqe_wr_id[qp->sq.prod].wqe_size = rwqe->wqe_size;
3633         qp->prev_wqe_size = rwqe->wqe_size;
3634         qp->wqe_wr_id[qp->sq.prod].bytes_len = rwqe->length;
3635         break;
3636     case IB_WR_RDMA_WRITE:
3637         wqe->req_type = RDMA_SQ_REQ_TYPE_RDMA_WR;
3638         rwqe = (struct rdma_sq_rdma_wqe_1st *)wqe;
3639
3640         rwqe->wqe_size = 2;
3641         rwqe2 = qed_chain_produce(&qp->sq.pbl);
3642         length = qedr_prepare_sq_rdma_data(dev, qp, rwqe, rwqe2,
3643                            wr, bad_wr);
3644         rwqe->length = cpu_to_le32(length);
3645         qp->wqe_wr_id[qp->sq.prod].wqe_size = rwqe->wqe_size;
3646         qp->prev_wqe_size = rwqe->wqe_size;
3647         qp->wqe_wr_id[qp->sq.prod].bytes_len = rwqe->length;
3648         break;
3649     case IB_WR_RDMA_READ_WITH_INV:
3650         SET_FIELD2(wqe->flags, RDMA_SQ_RDMA_WQE_1ST_READ_INV_FLG, 1);
3651         fallthrough;    /* same is identical to RDMA READ */
3652
3653     case IB_WR_RDMA_READ:
3654         wqe->req_type = RDMA_SQ_REQ_TYPE_RDMA_RD;
3655         rwqe = (struct rdma_sq_rdma_wqe_1st *)wqe;
3656
3657         rwqe->wqe_size = 2;
3658         rwqe2 = qed_chain_produce(&qp->sq.pbl);
3659         length = qedr_prepare_sq_rdma_data(dev, qp, rwqe, rwqe2,
3660                            wr, bad_wr);
3661         rwqe->length = cpu_to_le32(length);
3662         qp->wqe_wr_id[qp->sq.prod].wqe_size = rwqe->wqe_size;
3663         qp->prev_wqe_size = rwqe->wqe_size;
3664         qp->wqe_wr_id[qp->sq.prod].bytes_len = rwqe->length;
3665         break;
3666
3667     case IB_WR_ATOMIC_CMP_AND_SWP:
3668     case IB_WR_ATOMIC_FETCH_AND_ADD:
3669         awqe1 = (struct rdma_sq_atomic_wqe_1st *)wqe;
3670         awqe1->wqe_size = 4;
3671
3672         awqe2 = qed_chain_produce(&qp->sq.pbl);
3673         DMA_REGPAIR_LE(awqe2->remote_va, atomic_wr(wr)->remote_addr);
3674         awqe2->r_key = cpu_to_le32(atomic_wr(wr)->rkey);
3675
3676         awqe3 = qed_chain_produce(&qp->sq.pbl);
3677
3678         if (wr->opcode == IB_WR_ATOMIC_FETCH_AND_ADD) {
3679             wqe->req_type = RDMA_SQ_REQ_TYPE_ATOMIC_ADD;
3680             DMA_REGPAIR_LE(awqe3->swap_data,
3681                        atomic_wr(wr)->compare_add);
3682         } else {
3683             wqe->req_type = RDMA_SQ_REQ_TYPE_ATOMIC_CMP_AND_SWAP;
3684             DMA_REGPAIR_LE(awqe3->swap_data,
3685                        atomic_wr(wr)->swap);
3686             DMA_REGPAIR_LE(awqe3->cmp_data,
3687                        atomic_wr(wr)->compare_add);
3688         }
3689
3690         qedr_prepare_sq_sges(qp, NULL, wr);
3691
3692         qp->wqe_wr_id[qp->sq.prod].wqe_size = awqe1->wqe_size;
3693         qp->prev_wqe_size = awqe1->wqe_size;
3694         break;
3695
3696     case IB_WR_LOCAL_INV:
3697         iwqe = (struct rdma_sq_local_inv_wqe *)wqe;
3698         iwqe->wqe_size = 1;
3699
3700         iwqe->req_type = RDMA_SQ_REQ_TYPE_LOCAL_INVALIDATE;
3701         iwqe->inv_l_key = wr->ex.invalidate_rkey;
3702         qp->wqe_wr_id[qp->sq.prod].wqe_size = iwqe->wqe_size;
3703         qp->prev_wqe_size = iwqe->wqe_size;
3704         break;
3705     case IB_WR_REG_MR:
3706         DP_DEBUG(dev, QEDR_MSG_CQ, "REG_MR\n");
3707         wqe->req_type = RDMA_SQ_REQ_TYPE_FAST_MR;
3708         fwqe1 = (struct rdma_sq_fmr_wqe_1st *)wqe;
3709         fwqe1->wqe_size = 2;
3710
3711         rc = qedr_prepare_reg(qp, fwqe1, reg_wr(wr));
3712         if (rc) {
3713             DP_ERR(dev, "IB_REG_MR failed rc=%d\n", rc);
3714             *bad_wr = wr;
3715             break;
3716         }
3717
3718         qp->wqe_wr_id[qp->sq.prod].wqe_size = fwqe1->wqe_size;
3719         qp->prev_wqe_size = fwqe1->wqe_size;
3720         break;
3721     default:
3722         DP_ERR(dev, "invalid opcode 0x%x!\n", wr->opcode);
3723         rc = -EINVAL;
3724         *bad_wr = wr;
3725         break;
3726     }
3727
3728     if (*bad_wr) {
3729         u16 value;
3730
3731         /* Restore prod to its position before
3732          * this WR was processed
3733          */
3734         value = le16_to_cpu(qp->sq.db_data.data.value);
3735         qed_chain_set_prod(&qp->sq.pbl, value, wqe);
3736
3737         /* Restore prev_wqe_size */
3738         qp->prev_wqe_size = wqe->prev_wqe_size;
3739         rc = -EINVAL;
3740         DP_ERR(dev, "POST SEND FAILED\n");
3741     }
3742
3743     return rc;
3744 }
3745
3746 int qedr_post_send(struct ib_qp *ibqp, const struct ib_send_wr *wr,
3747            const struct ib_send_wr **bad_wr)
3748 {
3749     struct qedr_dev *dev = get_qedr_dev(ibqp->device);
3750     struct qedr_qp *qp = get_qedr_qp(ibqp);
3751     unsigned long flags;
3752     int rc = 0;
3753
3754     *bad_wr = NULL;
3755
3756     if (qp->qp_type == IB_QPT_GSI)
3757         return qedr_gsi_post_send(ibqp, wr, bad_wr);
3758
3759     spin_lock_irqsave(&qp->q_lock, flags);
3760
3761     if (rdma_protocol_roce(&dev->ibdev, 1)) {
3762         if ((qp->state != QED_ROCE_QP_STATE_RTS) &&
3763             (qp->state != QED_ROCE_QP_STATE_ERR) &&
3764             (qp->state != QED_ROCE_QP_STATE_SQD)) {
3765             spin_unlock_irqrestore(&qp->q_lock, flags);
3766             *bad_wr = wr;
3767             DP_DEBUG(dev, QEDR_MSG_CQ,
3768                  "QP in wrong state! QP icid=0x%x state %d\n",
3769                  qp->icid, qp->state);
3770             return -EINVAL;
3771         }
3772     }
3773
3774     while (wr) {
3775         rc = __qedr_post_send(ibqp, wr, bad_wr);
3776         if (rc)
3777             break;
3778
3779         qp->wqe_wr_id[qp->sq.prod].wr_id = wr->wr_id;
3780
3781         qedr_inc_sw_prod(&qp->sq);
3782
3783         qp->sq.db_data.data.value++;
3784
3785         wr = wr->next;
3786     }
3787
3788     /* Trigger doorbell
3789      * If there was a failure in the first WR then it will be triggered in
3790      * vane. However this is not harmful (as long as the producer value is
3791      * unchanged). For performance reasons we avoid checking for this
3792      * redundant doorbell.
3793      *
3794      * qp->wqe_wr_id is accessed during qedr_poll_cq, as
3795      * soon as we give the doorbell, we could get a completion
3796      * for this wr, therefore we need to make sure that the
3797      * memory is updated before giving the doorbell.
3798      * During qedr_poll_cq, rmb is called before accessing the
3799      * cqe. This covers for the smp_rmb as well.
3800      */
3801     smp_wmb();
3802     writel(qp->sq.db_data.raw, qp->sq.db);
3803
3804     spin_unlock_irqrestore(&qp->q_lock, flags);
3805
3806     return rc;
3807 }
3808
3809 static u32 qedr_srq_elem_left(struct qedr_srq_hwq_info *hw_srq)
3810 {
3811     u32 used;
3812
3813     /* Calculate number of elements used based on producer
3814      * count and consumer count and subtract it from max
3815      * work request supported so that we get elements left.
3816      */
3817     used = hw_srq->wr_prod_cnt - (u32)atomic_read(&hw_srq->wr_cons_cnt);
3818
3819     return hw_srq->max_wr - used;
3820 }
3821
3822 int qedr_post_srq_recv(struct ib_srq *ibsrq, const struct ib_recv_wr *wr,
3823                const struct ib_recv_wr **bad_wr)
3824 {
3825     struct qedr_srq *srq = get_qedr_srq(ibsrq);
3826     struct qedr_srq_hwq_info *hw_srq;
3827     struct qedr_dev *dev = srq->dev;
3828     struct qed_chain *pbl;
3829     unsigned long flags;
3830     int status = 0;
3831     u32 num_sge;
3832
3833     spin_lock_irqsave(&srq->lock, flags);
3834
3835     hw_srq = &srq->hw_srq;
3836     pbl = &srq->hw_srq.pbl;
3837     while (wr) {
3838         struct rdma_srq_wqe_header *hdr;
3839         int i;
3840
3841         if (!qedr_srq_elem_left(hw_srq) ||
3842             wr->num_sge > srq->hw_srq.max_sges) {
3843             DP_ERR(dev, "Can't post WR  (%d,%d) || (%d > %d)\n",
3844                    hw_srq->wr_prod_cnt,
3845                    atomic_read(&hw_srq->wr_cons_cnt),
3846                    wr->num_sge, srq->hw_srq.max_sges);
3847             status = -ENOMEM;
3848             *bad_wr = wr;
3849             break;
3850         }
3851
3852         hdr = qed_chain_produce(pbl);
3853         num_sge = wr->num_sge;
3854         /* Set number of sge and work request id in header */
3855         SRQ_HDR_SET(hdr, wr->wr_id, num_sge);
3856
3857         srq->hw_srq.wr_prod_cnt++;
3858         hw_srq->wqe_prod++;
3859         hw_srq->sge_prod++;
3860
3861         DP_DEBUG(dev, QEDR_MSG_SRQ,
3862              "SRQ WR: SGEs: %d with wr_id[%d] = %llx\n",
3863              wr->num_sge, hw_srq->wqe_prod, wr->wr_id);
3864
3865         for (i = 0; i < wr->num_sge; i++) {
3866             struct rdma_srq_sge *srq_sge = qed_chain_produce(pbl);
3867
3868             /* Set SGE length, lkey and address */
3869             SRQ_SGE_SET(srq_sge, wr->sg_list[i].addr,
3870                     wr->sg_list[i].length, wr->sg_list[i].lkey);
3871
3872             DP_DEBUG(dev, QEDR_MSG_SRQ,
3873                  "[%d]: len %d key %x addr %x:%x\n",
3874                  i, srq_sge->length, srq_sge->l_key,
3875                  srq_sge->addr.hi, srq_sge->addr.lo);
3876             hw_srq->sge_prod++;
3877         }
3878
3879         /* Update WQE and SGE information before
3880          * updating producer.
3881          */
3882         dma_wmb();
3883
3884         /* SRQ producer is 8 bytes. Need to update SGE producer index
3885          * in first 4 bytes and need to update WQE producer in
3886          * next 4 bytes.
3887          */
3888         srq->hw_srq.virt_prod_pair_addr->sge_prod = cpu_to_le32(hw_srq->sge_prod);
3889         /* Make sure sge producer is updated first */
3890         dma_wmb();
3891         srq->hw_srq.virt_prod_pair_addr->wqe_prod = cpu_to_le32(hw_srq->wqe_prod);
3892
3893         wr = wr->next;
3894     }
3895
3896     DP_DEBUG(dev, QEDR_MSG_SRQ, "POST: Elements in S-RQ: %d\n",
3897          qed_chain_get_elem_left(pbl));
3898     spin_unlock_irqrestore(&srq->lock, flags);
3899
3900     return status;
3901 }
3902
3903 int qedr_post_recv(struct ib_qp *ibqp, const struct ib_recv_wr *wr,
3904            const struct ib_recv_wr **bad_wr)
3905 {
3906     struct qedr_qp *qp = get_qedr_qp(ibqp);
3907     struct qedr_dev *dev = qp->dev;
3908     unsigned long flags;
3909     int status = 0;
3910
3911     if (qp->qp_type == IB_QPT_GSI)
3912         return qedr_gsi_post_recv(ibqp, wr, bad_wr);
3913
3914     spin_lock_irqsave(&qp->q_lock, flags);
3915
3916     while (wr) {
3917         int i;
3918
3919         if (qed_chain_get_elem_left_u32(&qp->rq.pbl) <
3920             QEDR_MAX_RQE_ELEMENTS_PER_RQE ||
3921             wr->num_sge > qp->rq.max_sges) {
3922             DP_ERR(dev, "Can't post WR  (%d < %d) || (%d > %d)\n",
3923                    qed_chain_get_elem_left_u32(&qp->rq.pbl),
3924                    QEDR_MAX_RQE_ELEMENTS_PER_RQE, wr->num_sge,
3925                    qp->rq.max_sges);
3926             status = -ENOMEM;
3927             *bad_wr = wr;
3928             break;
3929         }
3930         for (i = 0; i < wr->num_sge; i++) {
3931             u32 flags = 0;
3932             struct rdma_rq_sge *rqe =
3933                 qed_chain_produce(&qp->rq.pbl);
3934
3935             /* First one must include the number
3936              * of SGE in the list
3937              */
3938             if (!i)
3939                 SET_FIELD(flags, RDMA_RQ_SGE_NUM_SGES,
3940                       wr->num_sge);
3941
3942             SET_FIELD(flags, RDMA_RQ_SGE_L_KEY_LO,
3943                   wr->sg_list[i].lkey);
3944
3945             RQ_SGE_SET(rqe, wr->sg_list[i].addr,
3946                    wr->sg_list[i].length, flags);
3947         }
3948
3949         /* Special case of no sges. FW requires between 1-4 sges...
3950          * in this case we need to post 1 sge with length zero. this is
3951          * because rdma write with immediate consumes an RQ.
3952          */
3953         if (!wr->num_sge) {
3954             u32 flags = 0;
3955             struct rdma_rq_sge *rqe =
3956                 qed_chain_produce(&qp->rq.pbl);
3957
3958             /* First one must include the number
3959              * of SGE in the list
3960              */
3961             SET_FIELD(flags, RDMA_RQ_SGE_L_KEY_LO, 0);
3962             SET_FIELD(flags, RDMA_RQ_SGE_NUM_SGES, 1);
3963
3964             RQ_SGE_SET(rqe, 0, 0, flags);
3965             i = 1;
3966         }
3967
3968         qp->rqe_wr_id[qp->rq.prod].wr_id = wr->wr_id;
3969         qp->rqe_wr_id[qp->rq.prod].wqe_size = i;
3970
3971         qedr_inc_sw_prod(&qp->rq);
3972
3973         /* qp->rqe_wr_id is accessed during qedr_poll_cq, as
3974          * soon as we give the doorbell, we could get a completion
3975          * for this wr, therefore we need to make sure that the
3976          * memory is update before giving the doorbell.
3977          * During qedr_poll_cq, rmb is called before accessing the
3978          * cqe. This covers for the smp_rmb as well.
3979          */
3980         smp_wmb();
3981
3982         qp->rq.db_data.data.value++;
3983
3984         writel(qp->rq.db_data.raw, qp->rq.db);
3985
3986         if (rdma_protocol_iwarp(&dev->ibdev, 1)) {
3987             writel(qp->rq.iwarp_db2_data.raw, qp->rq.iwarp_db2);
3988         }
3989
3990         wr = wr->next;
3991     }
3992
3993     spin_unlock_irqrestore(&qp->q_lock, flags);
3994
3995     return status;
3996 }
3997
3998 static int is_valid_cqe(struct qedr_cq *cq, union rdma_cqe *cqe)
3999 {
4000     struct rdma_cqe_requester *resp_cqe = &cqe->req;
4001
4002     return (resp_cqe->flags & RDMA_CQE_REQUESTER_TOGGLE_BIT_MASK) ==
4003         cq->pbl_toggle;
4004 }
4005
4006 static struct qedr_qp *cqe_get_qp(union rdma_cqe *cqe)
4007 {
4008     struct rdma_cqe_requester *resp_cqe = &cqe->req;
4009     struct qedr_qp *qp;
4010
4011     qp = (struct qedr_qp *)(uintptr_t)HILO_GEN(resp_cqe->qp_handle.hi,
4012                            resp_cqe->qp_handle.lo,
4013                            u64);
4014     return qp;
4015 }
4016
4017 static enum rdma_cqe_type cqe_get_type(union rdma_cqe *cqe)
4018 {
4019     struct rdma_cqe_requester *resp_cqe = &cqe->req;
4020
4021     return GET_FIELD(resp_cqe->flags, RDMA_CQE_REQUESTER_TYPE);
4022 }
4023
4024 /* Return latest CQE (needs processing) */
4025 static union rdma_cqe *get_cqe(struct qedr_cq *cq)
4026 {
4027     return cq->latest_cqe;
4028 }
4029
4030 /* In fmr we need to increase the number of fmr completed counter for the fmr
4031  * algorithm determining whether we can free a pbl or not.
4032  * we need to perform this whether the work request was signaled or not. for
4033  * this purpose we call this function from the condition that checks if a wr
4034  * should be skipped, to make sure we don't miss it ( possibly this fmr
4035  * operation was not signalted)
4036  */
4037 static inline void qedr_chk_if_fmr(struct qedr_qp *qp)
4038 {
4039     if (qp->wqe_wr_id[qp->sq.cons].opcode == IB_WC_REG_MR)
4040         qp->wqe_wr_id[qp->sq.cons].mr->info.completed++;
4041 }
4042
4043 static int process_req(struct qedr_dev *dev, struct qedr_qp *qp,
4044                struct qedr_cq *cq, int num_entries,
4045                struct ib_wc *wc, u16 hw_cons, enum ib_wc_status status,
4046                int force)
4047 {
4048     u16 cnt = 0;
4049
4050     while (num_entries && qp->sq.wqe_cons != hw_cons) {
4051         if (!qp->wqe_wr_id[qp->sq.cons].signaled && !force) {
4052             qedr_chk_if_fmr(qp);
4053             /* skip WC */
4054             goto next_cqe;
4055         }
4056
4057         /* fill WC */
4058         wc->status = status;
4059         wc->vendor_err = 0;
4060         wc->wc_flags = 0;
4061         wc->src_qp = qp->id;
4062         wc->qp = &qp->ibqp;
4063
4064         wc->wr_id = qp->wqe_wr_id[qp->sq.cons].wr_id;
4065         wc->opcode = qp->wqe_wr_id[qp->sq.cons].opcode;
4066
4067         switch (wc->opcode) {
4068         case IB_WC_RDMA_WRITE:
4069             wc->byte_len = qp->wqe_wr_id[qp->sq.cons].bytes_len;
4070             break;
4071         case IB_WC_COMP_SWAP:
4072         case IB_WC_FETCH_ADD:
4073             wc->byte_len = 8;
4074             break;
4075         case IB_WC_REG_MR:
4076             qp->wqe_wr_id[qp->sq.cons].mr->info.completed++;
4077             break;
4078         case IB_WC_RDMA_READ:
4079         case IB_WC_SEND:
4080             wc->byte_len = qp->wqe_wr_id[qp->sq.cons].bytes_len;
4081             break;
4082         default:
4083             break;
4084         }
4085
4086         num_entries--;
4087         wc++;
4088         cnt++;
4089 next_cqe:
4090         while (qp->wqe_wr_id[qp->sq.cons].wqe_size--)
4091             qed_chain_consume(&qp->sq.pbl);
4092         qedr_inc_sw_cons(&qp->sq);
4093     }
4094
4095     return cnt;
4096 }
4097
4098 static int qedr_poll_cq_req(struct qedr_dev *dev,
4099                 struct qedr_qp *qp, struct qedr_cq *cq,
4100                 int num_entries, struct ib_wc *wc,
4101                 struct rdma_cqe_requester *req)
4102 {
4103     int cnt = 0;
4104
4105     switch (req->status) {
4106     case RDMA_CQE_REQ_STS_OK:
4107         cnt = process_req(dev, qp, cq, num_entries, wc, req->sq_cons,
4108                   IB_WC_SUCCESS, 0);
4109         break;
4110     case RDMA_CQE_REQ_STS_WORK_REQUEST_FLUSHED_ERR:
4111         if (qp->state != QED_ROCE_QP_STATE_ERR)
4112             DP_DEBUG(dev, QEDR_MSG_CQ,
4113                  "Error: POLL CQ with RDMA_CQE_REQ_STS_WORK_REQUEST_FLUSHED_ERR. CQ icid=0x%x, QP icid=0x%x\n",
4114                  cq->icid, qp->icid);
4115         cnt = process_req(dev, qp, cq, num_entries, wc, req->sq_cons,
4116                   IB_WC_WR_FLUSH_ERR, 1);
4117         break;
4118     default:
4119         /* process all WQE before the cosumer */
4120         qp->state = QED_ROCE_QP_STATE_ERR;
4121         cnt = process_req(dev, qp, cq, num_entries, wc,
4122                   req->sq_cons - 1, IB_WC_SUCCESS, 0);
4123         wc += cnt;
4124         /* if we have extra WC fill it with actual error info */
4125         if (cnt < num_entries) {
4126             enum ib_wc_status wc_status;
4127
4128             switch (req->status) {
4129             case RDMA_CQE_REQ_STS_BAD_RESPONSE_ERR:
4130                 DP_ERR(dev,
4131                        "Error: POLL CQ with RDMA_CQE_REQ_STS_BAD_RESPONSE_ERR. CQ icid=0x%x, QP icid=0x%x\n",
4132                        cq->icid, qp->icid);
4133                 wc_status = IB_WC_BAD_RESP_ERR;
4134                 break;
4135             case RDMA_CQE_REQ_STS_LOCAL_LENGTH_ERR:
4136                 DP_ERR(dev,
4137                        "Error: POLL CQ with RDMA_CQE_REQ_STS_LOCAL_LENGTH_ERR. CQ icid=0x%x, QP icid=0x%x\n",
4138                        cq->icid, qp->icid);
4139                 wc_status = IB_WC_LOC_LEN_ERR;
4140                 break;
4141             case RDMA_CQE_REQ_STS_LOCAL_QP_OPERATION_ERR:
4142                 DP_ERR(dev,
4143                        "Error: POLL CQ with RDMA_CQE_REQ_STS_LOCAL_QP_OPERATION_ERR. CQ icid=0x%x, QP icid=0x%x\n",
4144                        cq->icid, qp->icid);
4145                 wc_status = IB_WC_LOC_QP_OP_ERR;
4146                 break;
4147             case RDMA_CQE_REQ_STS_LOCAL_PROTECTION_ERR:
4148                 DP_ERR(dev,
4149                        "Error: POLL CQ with RDMA_CQE_REQ_STS_LOCAL_PROTECTION_ERR. CQ icid=0x%x, QP icid=0x%x\n",
4150                        cq->icid, qp->icid);
4151                 wc_status = IB_WC_LOC_PROT_ERR;
4152                 break;
4153             case RDMA_CQE_REQ_STS_MEMORY_MGT_OPERATION_ERR:
4154                 DP_ERR(dev,
4155                        "Error: POLL CQ with RDMA_CQE_REQ_STS_MEMORY_MGT_OPERATION_ERR. CQ icid=0x%x, QP icid=0x%x\n",
4156                        cq->icid, qp->icid);
4157                 wc_status = IB_WC_MW_BIND_ERR;
4158                 break;
4159             case RDMA_CQE_REQ_STS_REMOTE_INVALID_REQUEST_ERR:
4160                 DP_ERR(dev,
4161                        "Error: POLL CQ with RDMA_CQE_REQ_STS_REMOTE_INVALID_REQUEST_ERR. CQ icid=0x%x, QP icid=0x%x\n",
4162                        cq->icid, qp->icid);
4163                 wc_status = IB_WC_REM_INV_REQ_ERR;
4164                 break;
4165             case RDMA_CQE_REQ_STS_REMOTE_ACCESS_ERR:
4166                 DP_ERR(dev,
4167                        "Error: POLL CQ with RDMA_CQE_REQ_STS_REMOTE_ACCESS_ERR. CQ icid=0x%x, QP icid=0x%x\n",
4168                        cq->icid, qp->icid);
4169                 wc_status = IB_WC_REM_ACCESS_ERR;
4170                 break;
4171             case RDMA_CQE_REQ_STS_REMOTE_OPERATION_ERR:
4172                 DP_ERR(dev,
4173                        "Error: POLL CQ with RDMA_CQE_REQ_STS_REMOTE_OPERATION_ERR. CQ icid=0x%x, QP icid=0x%x\n",
4174                        cq->icid, qp->icid);
4175                 wc_status = IB_WC_REM_OP_ERR;
4176                 break;
4177             case RDMA_CQE_REQ_STS_RNR_NAK_RETRY_CNT_ERR:
4178                 DP_ERR(dev,
4179                        "Error: POLL CQ with RDMA_CQE_REQ_STS_RNR_NAK_RETRY_CNT_ERR. CQ icid=0x%x, QP icid=0x%x\n",
4180                        cq->icid, qp->icid);
4181                 wc_status = IB_WC_RNR_RETRY_EXC_ERR;
4182                 break;
4183             case RDMA_CQE_REQ_STS_TRANSPORT_RETRY_CNT_ERR:
4184                 DP_ERR(dev,
4185                        "Error: POLL CQ with ROCE_CQE_REQ_STS_TRANSPORT_RETRY_CNT_ERR. CQ icid=0x%x, QP icid=0x%x\n",
4186                        cq->icid, qp->icid);
4187                 wc_status = IB_WC_RETRY_EXC_ERR;
4188                 break;
4189             default:
4190                 DP_ERR(dev,
4191                        "Error: POLL CQ with IB_WC_GENERAL_ERR. CQ icid=0x%x, QP icid=0x%x\n",
4192                        cq->icid, qp->icid);
4193                 wc_status = IB_WC_GENERAL_ERR;
4194             }
4195             cnt += process_req(dev, qp, cq, 1, wc, req->sq_cons,
4196                        wc_status, 1);
4197         }
4198     }
4199
4200     return cnt;
4201 }
4202
4203 static inline int qedr_cqe_resp_status_to_ib(u8 status)
4204 {
4205     switch (status) {
4206     case RDMA_CQE_RESP_STS_LOCAL_ACCESS_ERR:
4207         return IB_WC_LOC_ACCESS_ERR;
4208     case RDMA_CQE_RESP_STS_LOCAL_LENGTH_ERR:
4209         return IB_WC_LOC_LEN_ERR;
4210     case RDMA_CQE_RESP_STS_LOCAL_QP_OPERATION_ERR:
4211         return IB_WC_LOC_QP_OP_ERR;
4212     case RDMA_CQE_RESP_STS_LOCAL_PROTECTION_ERR:
4213         return IB_WC_LOC_PROT_ERR;
4214     case RDMA_CQE_RESP_STS_MEMORY_MGT_OPERATION_ERR:
4215         return IB_WC_MW_BIND_ERR;
4216     case RDMA_CQE_RESP_STS_REMOTE_INVALID_REQUEST_ERR:
4217         return IB_WC_REM_INV_RD_REQ_ERR;
4218     case RDMA_CQE_RESP_STS_OK:
4219         return IB_WC_SUCCESS;
4220     default:
4221         return IB_WC_GENERAL_ERR;
4222     }
4223 }
4224
4225 static inline int qedr_set_ok_cqe_resp_wc(struct rdma_cqe_responder *resp,
4226                       struct ib_wc *wc)
4227 {
4228     wc->status = IB_WC_SUCCESS;
4229     wc->byte_len = le32_to_cpu(resp->length);
4230
4231     if (resp->flags & QEDR_RESP_IMM) {
4232         wc->ex.imm_data = cpu_to_be32(le32_to_cpu(resp->imm_data_or_inv_r_Key));
4233         wc->wc_flags |= IB_WC_WITH_IMM;
4234
4235         if (resp->flags & QEDR_RESP_RDMA)
4236             wc->opcode = IB_WC_RECV_RDMA_WITH_IMM;
4237
4238         if (resp->flags & QEDR_RESP_INV)
4239             return -EINVAL;
4240
4241     } else if (resp->flags & QEDR_RESP_INV) {
4242         wc->ex.imm_data = le32_to_cpu(resp->imm_data_or_inv_r_Key);
4243         wc->wc_flags |= IB_WC_WITH_INVALIDATE;
4244
4245         if (resp->flags & QEDR_RESP_RDMA)
4246             return -EINVAL;
4247
4248     } else if (resp->flags & QEDR_RESP_RDMA) {
4249         return -EINVAL;
4250     }
4251
4252     return 0;
4253 }
4254
4255 static void __process_resp_one(struct qedr_dev *dev, struct qedr_qp *qp,
4256                    struct qedr_cq *cq, struct ib_wc *wc,
4257                    struct rdma_cqe_responder *resp, u64 wr_id)
4258 {
4259     /* Must fill fields before qedr_set_ok_cqe_resp_wc() */
4260     wc->opcode = IB_WC_RECV;
4261     wc->wc_flags = 0;
4262
4263     if (likely(resp->status == RDMA_CQE_RESP_STS_OK)) {
4264         if (qedr_set_ok_cqe_resp_wc(resp, wc))
4265             DP_ERR(dev,
4266                    "CQ %p (icid=%d) has invalid CQE responder flags=0x%x\n",
4267                    cq, cq->icid, resp->flags);
4268
4269     } else {
4270         wc->status = qedr_cqe_resp_status_to_ib(resp->status);
4271         if (wc->status == IB_WC_GENERAL_ERR)
4272             DP_ERR(dev,
4273                    "CQ %p (icid=%d) contains an invalid CQE status %d\n",
4274                    cq, cq->icid, resp->status);
4275     }
4276
4277     /* Fill the rest of the WC */
4278     wc->vendor_err = 0;
4279     wc->src_qp = qp->id;
4280     wc->qp = &qp->ibqp;
4281     wc->wr_id = wr_id;
4282 }
4283
4284 static int process_resp_one_srq(struct qedr_dev *dev, struct qedr_qp *qp,
4285                 struct qedr_cq *cq, struct ib_wc *wc,
4286                 struct rdma_cqe_responder *resp)
4287 {
4288     struct qedr_srq *srq = qp->srq;
4289     u64 wr_id;
4290
4291     wr_id = HILO_GEN(le32_to_cpu(resp->srq_wr_id.hi),
4292              le32_to_cpu(resp->srq_wr_id.lo), u64);
4293
4294     if (resp->status == RDMA_CQE_RESP_STS_WORK_REQUEST_FLUSHED_ERR) {
4295         wc->status = IB_WC_WR_FLUSH_ERR;
4296         wc->vendor_err = 0;
4297         wc->wr_id = wr_id;
4298         wc->byte_len = 0;
4299         wc->src_qp = qp->id;
4300         wc->qp = &qp->ibqp;
4301         wc->wr_id = wr_id;
4302     } else {
4303         __process_resp_one(dev, qp, cq, wc, resp, wr_id);
4304     }
4305     atomic_inc(&srq->hw_srq.wr_cons_cnt);
4306
4307     return 1;
4308 }
4309 static int process_resp_one(struct qedr_dev *dev, struct qedr_qp *qp,
4310                 struct qedr_cq *cq, struct ib_wc *wc,
4311                 struct rdma_cqe_responder *resp)
4312 {
4313     u64 wr_id = qp->rqe_wr_id[qp->rq.cons].wr_id;
4314
4315     __process_resp_one(dev, qp, cq, wc, resp, wr_id);
4316
4317     while (qp->rqe_wr_id[qp->rq.cons].wqe_size--)
4318         qed_chain_consume(&qp->rq.pbl);
4319     qedr_inc_sw_cons(&qp->rq);
4320
4321     return 1;
4322 }
4323
4324 static int process_resp_flush(struct qedr_qp *qp, struct qedr_cq *cq,
4325                   int num_entries, struct ib_wc *wc, u16 hw_cons)
4326 {
4327     u16 cnt = 0;
4328
4329     while (num_entries && qp->rq.wqe_cons != hw_cons) {
4330         /* fill WC */
4331         wc->status = IB_WC_WR_FLUSH_ERR;
4332         wc->vendor_err = 0;
4333         wc->wc_flags = 0;
4334         wc->src_qp = qp->id;
4335         wc->byte_len = 0;
4336         wc->wr_id = qp->rqe_wr_id[qp->rq.cons].wr_id;
4337         wc->qp = &qp->ibqp;
4338         num_entries--;
4339         wc++;
4340         cnt++;
4341         while (qp->rqe_wr_id[qp->rq.cons].wqe_size--)
4342             qed_chain_consume(&qp->rq.pbl);
4343         qedr_inc_sw_cons(&qp->rq);
4344     }
4345
4346     return cnt;
4347 }
4348
4349 static void try_consume_resp_cqe(struct qedr_cq *cq, struct qedr_qp *qp,
4350                  struct rdma_cqe_responder *resp, int *update)
4351 {
4352     if (le16_to_cpu(resp->rq_cons_or_srq_id) == qp->rq.wqe_cons) {
4353         consume_cqe(cq);
4354         *update |= 1;
4355     }
4356 }
4357
4358 static int qedr_poll_cq_resp_srq(struct qedr_dev *dev, struct qedr_qp *qp,
4359                  struct qedr_cq *cq, int num_entries,
4360                  struct ib_wc *wc,
4361                  struct rdma_cqe_responder *resp)
4362 {
4363     int cnt;
4364
4365     cnt = process_resp_one_srq(dev, qp, cq, wc, resp);
4366     consume_cqe(cq);
4367
4368     return cnt;
4369 }
4370
4371 static int qedr_poll_cq_resp(struct qedr_dev *dev, struct qedr_qp *qp,
4372                  struct qedr_cq *cq, int num_entries,
4373                  struct ib_wc *wc, struct rdma_cqe_responder *resp,
4374                  int *update)
4375 {
4376     int cnt;
4377
4378     if (resp->status == RDMA_CQE_RESP_STS_WORK_REQUEST_FLUSHED_ERR) {
4379         cnt = process_resp_flush(qp, cq, num_entries, wc,
4380                      resp->rq_cons_or_srq_id);
4381         try_consume_resp_cqe(cq, qp, resp, update);
4382     } else {
4383         cnt = process_resp_one(dev, qp, cq, wc, resp);
4384         consume_cqe(cq);
4385         *update |= 1;
4386     }
4387
4388     return cnt;
4389 }
4390
4391 static void try_consume_req_cqe(struct qedr_cq *cq, struct qedr_qp *qp,
4392                 struct rdma_cqe_requester *req, int *update)
4393 {
4394     if (le16_to_cpu(req->sq_cons) == qp->sq.wqe_cons) {
4395         consume_cqe(cq);
4396         *update |= 1;
4397     }
4398 }
4399
4400 int qedr_poll_cq(struct ib_cq *ibcq, int num_entries, struct ib_wc *wc)
4401 {
4402     struct qedr_dev *dev = get_qedr_dev(ibcq->device);
4403     struct qedr_cq *cq = get_qedr_cq(ibcq);
4404     union rdma_cqe *cqe;
4405     u32 old_cons, new_cons;
4406     unsigned long flags;
4407     int update = 0;
4408     int done = 0;
4409
4410     if (cq->destroyed) {
4411         DP_ERR(dev,
4412                "warning: poll was invoked after destroy for cq %p (icid=%d)\n",
4413                cq, cq->icid);
4414         return 0;
4415     }
4416
4417     if (cq->cq_type == QEDR_CQ_TYPE_GSI)
4418         return qedr_gsi_poll_cq(ibcq, num_entries, wc);
4419
4420     spin_lock_irqsave(&cq->cq_lock, flags);
4421     cqe = cq->latest_cqe;
4422     old_cons = qed_chain_get_cons_idx_u32(&cq->pbl);
4423     while (num_entries && is_valid_cqe(cq, cqe)) {
4424         struct qedr_qp *qp;
4425         int cnt = 0;
4426
4427         /* prevent speculative reads of any field of CQE */
4428         rmb();
4429
4430         qp = cqe_get_qp(cqe);
4431         if (!qp) {
4432             WARN(1, "Error: CQE QP pointer is NULL. CQE=%p\n", cqe);
4433             break;
4434         }
4435
4436         wc->qp = &qp->ibqp;
4437
4438         switch (cqe_get_type(cqe)) {
4439         case RDMA_CQE_TYPE_REQUESTER:
4440             cnt = qedr_poll_cq_req(dev, qp, cq, num_entries, wc,
4441                            &cqe->req);
4442             try_consume_req_cqe(cq, qp, &cqe->req, &update);
4443             break;
4444         case RDMA_CQE_TYPE_RESPONDER_RQ:
4445             cnt = qedr_poll_cq_resp(dev, qp, cq, num_entries, wc,
4446                         &cqe->resp, &update);
4447             break;
4448         case RDMA_CQE_TYPE_RESPONDER_SRQ:
4449             cnt = qedr_poll_cq_resp_srq(dev, qp, cq, num_entries,
4450                             wc, &cqe->resp);
4451             update = 1;
4452             break;
4453         case RDMA_CQE_TYPE_INVALID:
4454         default:
4455             DP_ERR(dev, "Error: invalid CQE type = %d\n",
4456                    cqe_get_type(cqe));
4457         }
4458         num_entries -= cnt;
4459         wc += cnt;
4460         done += cnt;
4461
4462         cqe = get_cqe(cq);
4463     }
4464     new_cons = qed_chain_get_cons_idx_u32(&cq->pbl);
4465
4466     cq->cq_cons += new_cons - old_cons;
4467
4468     if (update)
4469         /* doorbell notifies abount latest VALID entry,
4470          * but chain already point to the next INVALID one
4471          */
4472         doorbell_cq(cq, cq->cq_cons - 1, cq->arm_flags);
4473
4474     spin_unlock_irqrestore(&cq->cq_lock, flags);
4475     return done;
4476 }
4477
4478 int qedr_process_mad(struct ib_device *ibdev, int process_mad_flags,
4479              u32 port_num, const struct ib_wc *in_wc,
4480              const struct ib_grh *in_grh, const struct ib_mad *in,
4481              struct ib_mad *out_mad, size_t *out_mad_size,
4482              u16 *out_mad_pkey_index)
4483 {
4484     return IB_MAD_RESULT_SUCCESS;
4485 }