Back to home page

OSCL-LXR
 
 

     

0001 // SPDX-License-Identifier: GPL-2.0 or Linux-OpenIB
0002 /* Copyright (c) 2015 - 2021 Intel Corporation */
0003 #include "osdep.h"
0004 #include "hmc.h"
0005 #include "defs.h"
0006 #include "type.h"
0007 #include "protos.h"
0008 #include "puda.h"
0009 #include "ws.h"
0010 
0011 static void irdma_ieq_receive(struct irdma_sc_vsi *vsi,
0012                   struct irdma_puda_buf *buf);
0013 static void irdma_ieq_tx_compl(struct irdma_sc_vsi *vsi, void *sqwrid);
0014 static void irdma_ilq_putback_rcvbuf(struct irdma_sc_qp *qp,
0015                      struct irdma_puda_buf *buf, u32 wqe_idx);
0016 /**
0017  * irdma_puda_get_listbuf - get buffer from puda list
0018  * @list: list to use for buffers (ILQ or IEQ)
0019  */
0020 static struct irdma_puda_buf *irdma_puda_get_listbuf(struct list_head *list)
0021 {
0022     struct irdma_puda_buf *buf = NULL;
0023 
0024     if (!list_empty(list)) {
0025         buf = (struct irdma_puda_buf *)list->next;
0026         list_del((struct list_head *)&buf->list);
0027     }
0028 
0029     return buf;
0030 }
0031 
0032 /**
0033  * irdma_puda_get_bufpool - return buffer from resource
0034  * @rsrc: resource to use for buffer
0035  */
0036 struct irdma_puda_buf *irdma_puda_get_bufpool(struct irdma_puda_rsrc *rsrc)
0037 {
0038     struct irdma_puda_buf *buf = NULL;
0039     struct list_head *list = &rsrc->bufpool;
0040     unsigned long flags;
0041 
0042     spin_lock_irqsave(&rsrc->bufpool_lock, flags);
0043     buf = irdma_puda_get_listbuf(list);
0044     if (buf) {
0045         rsrc->avail_buf_count--;
0046         buf->vsi = rsrc->vsi;
0047     } else {
0048         rsrc->stats_buf_alloc_fail++;
0049     }
0050     spin_unlock_irqrestore(&rsrc->bufpool_lock, flags);
0051 
0052     return buf;
0053 }
0054 
0055 /**
0056  * irdma_puda_ret_bufpool - return buffer to rsrc list
0057  * @rsrc: resource to use for buffer
0058  * @buf: buffer to return to resource
0059  */
0060 void irdma_puda_ret_bufpool(struct irdma_puda_rsrc *rsrc,
0061                 struct irdma_puda_buf *buf)
0062 {
0063     unsigned long flags;
0064 
0065     buf->do_lpb = false;
0066     spin_lock_irqsave(&rsrc->bufpool_lock, flags);
0067     list_add(&buf->list, &rsrc->bufpool);
0068     spin_unlock_irqrestore(&rsrc->bufpool_lock, flags);
0069     rsrc->avail_buf_count++;
0070 }
0071 
0072 /**
0073  * irdma_puda_post_recvbuf - set wqe for rcv buffer
0074  * @rsrc: resource ptr
0075  * @wqe_idx: wqe index to use
0076  * @buf: puda buffer for rcv q
0077  * @initial: flag if during init time
0078  */
0079 static void irdma_puda_post_recvbuf(struct irdma_puda_rsrc *rsrc, u32 wqe_idx,
0080                     struct irdma_puda_buf *buf, bool initial)
0081 {
0082     __le64 *wqe;
0083     struct irdma_sc_qp *qp = &rsrc->qp;
0084     u64 offset24 = 0;
0085 
0086     /* Synch buffer for use by device */
0087     dma_sync_single_for_device(rsrc->dev->hw->device, buf->mem.pa,
0088                    buf->mem.size, DMA_BIDIRECTIONAL);
0089     qp->qp_uk.rq_wrid_array[wqe_idx] = (uintptr_t)buf;
0090     wqe = qp->qp_uk.rq_base[wqe_idx].elem;
0091     if (!initial)
0092         get_64bit_val(wqe, 24, &offset24);
0093 
0094     offset24 = (offset24) ? 0 : FIELD_PREP(IRDMAQPSQ_VALID, 1);
0095 
0096     set_64bit_val(wqe, 16, 0);
0097     set_64bit_val(wqe, 0, buf->mem.pa);
0098     if (qp->qp_uk.uk_attrs->hw_rev == IRDMA_GEN_1) {
0099         set_64bit_val(wqe, 8,
0100                   FIELD_PREP(IRDMAQPSQ_GEN1_FRAG_LEN, buf->mem.size));
0101     } else {
0102         set_64bit_val(wqe, 8,
0103                   FIELD_PREP(IRDMAQPSQ_FRAG_LEN, buf->mem.size) |
0104                   offset24);
0105     }
0106     dma_wmb(); /* make sure WQE is written before valid bit is set */
0107 
0108     set_64bit_val(wqe, 24, offset24);
0109 }
0110 
0111 /**
0112  * irdma_puda_replenish_rq - post rcv buffers
0113  * @rsrc: resource to use for buffer
0114  * @initial: flag if during init time
0115  */
0116 static int irdma_puda_replenish_rq(struct irdma_puda_rsrc *rsrc, bool initial)
0117 {
0118     u32 i;
0119     u32 invalid_cnt = rsrc->rxq_invalid_cnt;
0120     struct irdma_puda_buf *buf = NULL;
0121 
0122     for (i = 0; i < invalid_cnt; i++) {
0123         buf = irdma_puda_get_bufpool(rsrc);
0124         if (!buf)
0125             return -ENOBUFS;
0126         irdma_puda_post_recvbuf(rsrc, rsrc->rx_wqe_idx, buf, initial);
0127         rsrc->rx_wqe_idx = ((rsrc->rx_wqe_idx + 1) % rsrc->rq_size);
0128         rsrc->rxq_invalid_cnt--;
0129     }
0130 
0131     return 0;
0132 }
0133 
0134 /**
0135  * irdma_puda_alloc_buf - allocate mem for buffer
0136  * @dev: iwarp device
0137  * @len: length of buffer
0138  */
0139 static struct irdma_puda_buf *irdma_puda_alloc_buf(struct irdma_sc_dev *dev,
0140                            u32 len)
0141 {
0142     struct irdma_puda_buf *buf;
0143     struct irdma_virt_mem buf_mem;
0144 
0145     buf_mem.size = sizeof(struct irdma_puda_buf);
0146     buf_mem.va = kzalloc(buf_mem.size, GFP_KERNEL);
0147     if (!buf_mem.va)
0148         return NULL;
0149 
0150     buf = buf_mem.va;
0151     buf->mem.size = len;
0152     buf->mem.va = kzalloc(buf->mem.size, GFP_KERNEL);
0153     if (!buf->mem.va)
0154         goto free_virt;
0155     buf->mem.pa = dma_map_single(dev->hw->device, buf->mem.va,
0156                      buf->mem.size, DMA_BIDIRECTIONAL);
0157     if (dma_mapping_error(dev->hw->device, buf->mem.pa)) {
0158         kfree(buf->mem.va);
0159         goto free_virt;
0160     }
0161 
0162     buf->buf_mem.va = buf_mem.va;
0163     buf->buf_mem.size = buf_mem.size;
0164 
0165     return buf;
0166 
0167 free_virt:
0168     kfree(buf_mem.va);
0169     return NULL;
0170 }
0171 
0172 /**
0173  * irdma_puda_dele_buf - delete buffer back to system
0174  * @dev: iwarp device
0175  * @buf: buffer to free
0176  */
0177 static void irdma_puda_dele_buf(struct irdma_sc_dev *dev,
0178                 struct irdma_puda_buf *buf)
0179 {
0180     dma_unmap_single(dev->hw->device, buf->mem.pa, buf->mem.size,
0181              DMA_BIDIRECTIONAL);
0182     kfree(buf->mem.va);
0183     kfree(buf->buf_mem.va);
0184 }
0185 
0186 /**
0187  * irdma_puda_get_next_send_wqe - return next wqe for processing
0188  * @qp: puda qp for wqe
0189  * @wqe_idx: wqe index for caller
0190  */
0191 static __le64 *irdma_puda_get_next_send_wqe(struct irdma_qp_uk *qp,
0192                         u32 *wqe_idx)
0193 {
0194     int ret_code = 0;
0195 
0196     *wqe_idx = IRDMA_RING_CURRENT_HEAD(qp->sq_ring);
0197     if (!*wqe_idx)
0198         qp->swqe_polarity = !qp->swqe_polarity;
0199     IRDMA_RING_MOVE_HEAD(qp->sq_ring, ret_code);
0200     if (ret_code)
0201         return NULL;
0202 
0203     return qp->sq_base[*wqe_idx].elem;
0204 }
0205 
0206 /**
0207  * irdma_puda_poll_info - poll cq for completion
0208  * @cq: cq for poll
0209  * @info: info return for successful completion
0210  */
0211 static int irdma_puda_poll_info(struct irdma_sc_cq *cq,
0212                 struct irdma_puda_cmpl_info *info)
0213 {
0214     struct irdma_cq_uk *cq_uk = &cq->cq_uk;
0215     u64 qword0, qword2, qword3, qword6;
0216     __le64 *cqe;
0217     __le64 *ext_cqe = NULL;
0218     u64 qword7 = 0;
0219     u64 comp_ctx;
0220     bool valid_bit;
0221     bool ext_valid = 0;
0222     u32 major_err, minor_err;
0223     u32 peek_head;
0224     bool error;
0225     u8 polarity;
0226 
0227     cqe = IRDMA_GET_CURRENT_CQ_ELEM(&cq->cq_uk);
0228     get_64bit_val(cqe, 24, &qword3);
0229     valid_bit = (bool)FIELD_GET(IRDMA_CQ_VALID, qword3);
0230     if (valid_bit != cq_uk->polarity)
0231         return -ENOENT;
0232 
0233     if (cq->dev->hw_attrs.uk_attrs.hw_rev >= IRDMA_GEN_2)
0234         ext_valid = (bool)FIELD_GET(IRDMA_CQ_EXTCQE, qword3);
0235 
0236     if (ext_valid) {
0237         peek_head = (cq_uk->cq_ring.head + 1) % cq_uk->cq_ring.size;
0238         ext_cqe = cq_uk->cq_base[peek_head].buf;
0239         get_64bit_val(ext_cqe, 24, &qword7);
0240         polarity = (u8)FIELD_GET(IRDMA_CQ_VALID, qword7);
0241         if (!peek_head)
0242             polarity ^= 1;
0243         if (polarity != cq_uk->polarity)
0244             return -ENOENT;
0245 
0246         IRDMA_RING_MOVE_HEAD_NOCHECK(cq_uk->cq_ring);
0247         if (!IRDMA_RING_CURRENT_HEAD(cq_uk->cq_ring))
0248             cq_uk->polarity = !cq_uk->polarity;
0249         /* update cq tail in cq shadow memory also */
0250         IRDMA_RING_MOVE_TAIL(cq_uk->cq_ring);
0251     }
0252 
0253     print_hex_dump_debug("PUDA: PUDA CQE", DUMP_PREFIX_OFFSET, 16, 8, cqe,
0254                  32, false);
0255     if (ext_valid)
0256         print_hex_dump_debug("PUDA: PUDA EXT-CQE", DUMP_PREFIX_OFFSET,
0257                      16, 8, ext_cqe, 32, false);
0258 
0259     error = (bool)FIELD_GET(IRDMA_CQ_ERROR, qword3);
0260     if (error) {
0261         ibdev_dbg(to_ibdev(cq->dev), "PUDA: receive error\n");
0262         major_err = (u32)(FIELD_GET(IRDMA_CQ_MAJERR, qword3));
0263         minor_err = (u32)(FIELD_GET(IRDMA_CQ_MINERR, qword3));
0264         info->compl_error = major_err << 16 | minor_err;
0265         return -EIO;
0266     }
0267 
0268     get_64bit_val(cqe, 0, &qword0);
0269     get_64bit_val(cqe, 16, &qword2);
0270 
0271     info->q_type = (u8)FIELD_GET(IRDMA_CQ_SQ, qword3);
0272     info->qp_id = (u32)FIELD_GET(IRDMACQ_QPID, qword2);
0273     if (cq->dev->hw_attrs.uk_attrs.hw_rev >= IRDMA_GEN_2)
0274         info->ipv4 = (bool)FIELD_GET(IRDMACQ_IPV4, qword3);
0275 
0276     get_64bit_val(cqe, 8, &comp_ctx);
0277     info->qp = (struct irdma_qp_uk *)(unsigned long)comp_ctx;
0278     info->wqe_idx = (u32)FIELD_GET(IRDMA_CQ_WQEIDX, qword3);
0279 
0280     if (info->q_type == IRDMA_CQE_QTYPE_RQ) {
0281         if (ext_valid) {
0282             info->vlan_valid = (bool)FIELD_GET(IRDMA_CQ_UDVLANVALID, qword7);
0283             if (info->vlan_valid) {
0284                 get_64bit_val(ext_cqe, 16, &qword6);
0285                 info->vlan = (u16)FIELD_GET(IRDMA_CQ_UDVLAN, qword6);
0286             }
0287             info->smac_valid = (bool)FIELD_GET(IRDMA_CQ_UDSMACVALID, qword7);
0288             if (info->smac_valid) {
0289                 get_64bit_val(ext_cqe, 16, &qword6);
0290                 info->smac[0] = (u8)((qword6 >> 40) & 0xFF);
0291                 info->smac[1] = (u8)((qword6 >> 32) & 0xFF);
0292                 info->smac[2] = (u8)((qword6 >> 24) & 0xFF);
0293                 info->smac[3] = (u8)((qword6 >> 16) & 0xFF);
0294                 info->smac[4] = (u8)((qword6 >> 8) & 0xFF);
0295                 info->smac[5] = (u8)(qword6 & 0xFF);
0296             }
0297         }
0298 
0299         if (cq->dev->hw_attrs.uk_attrs.hw_rev == IRDMA_GEN_1) {
0300             info->vlan_valid = (bool)FIELD_GET(IRDMA_VLAN_TAG_VALID, qword3);
0301             info->l4proto = (u8)FIELD_GET(IRDMA_UDA_L4PROTO, qword2);
0302             info->l3proto = (u8)FIELD_GET(IRDMA_UDA_L3PROTO, qword2);
0303         }
0304 
0305         info->payload_len = (u32)FIELD_GET(IRDMACQ_PAYLDLEN, qword0);
0306     }
0307 
0308     return 0;
0309 }
0310 
0311 /**
0312  * irdma_puda_poll_cmpl - processes completion for cq
0313  * @dev: iwarp device
0314  * @cq: cq getting interrupt
0315  * @compl_err: return any completion err
0316  */
0317 int irdma_puda_poll_cmpl(struct irdma_sc_dev *dev, struct irdma_sc_cq *cq,
0318              u32 *compl_err)
0319 {
0320     struct irdma_qp_uk *qp;
0321     struct irdma_cq_uk *cq_uk = &cq->cq_uk;
0322     struct irdma_puda_cmpl_info info = {};
0323     int ret = 0;
0324     struct irdma_puda_buf *buf;
0325     struct irdma_puda_rsrc *rsrc;
0326     u8 cq_type = cq->cq_type;
0327     unsigned long flags;
0328 
0329     if (cq_type == IRDMA_CQ_TYPE_ILQ || cq_type == IRDMA_CQ_TYPE_IEQ) {
0330         rsrc = (cq_type == IRDMA_CQ_TYPE_ILQ) ? cq->vsi->ilq :
0331                             cq->vsi->ieq;
0332     } else {
0333         ibdev_dbg(to_ibdev(dev), "PUDA: qp_type error\n");
0334         return -EINVAL;
0335     }
0336 
0337     ret = irdma_puda_poll_info(cq, &info);
0338     *compl_err = info.compl_error;
0339     if (ret == -ENOENT)
0340         return ret;
0341     if (ret)
0342         goto done;
0343 
0344     qp = info.qp;
0345     if (!qp || !rsrc) {
0346         ret = -EFAULT;
0347         goto done;
0348     }
0349 
0350     if (qp->qp_id != rsrc->qp_id) {
0351         ret = -EFAULT;
0352         goto done;
0353     }
0354 
0355     if (info.q_type == IRDMA_CQE_QTYPE_RQ) {
0356         buf = (struct irdma_puda_buf *)(uintptr_t)
0357                   qp->rq_wrid_array[info.wqe_idx];
0358 
0359         /* reusing so synch the buffer for CPU use */
0360         dma_sync_single_for_cpu(dev->hw->device, buf->mem.pa,
0361                     buf->mem.size, DMA_BIDIRECTIONAL);
0362         /* Get all the tcpip information in the buf header */
0363         ret = irdma_puda_get_tcpip_info(&info, buf);
0364         if (ret) {
0365             rsrc->stats_rcvd_pkt_err++;
0366             if (cq_type == IRDMA_CQ_TYPE_ILQ) {
0367                 irdma_ilq_putback_rcvbuf(&rsrc->qp, buf,
0368                              info.wqe_idx);
0369             } else {
0370                 irdma_puda_ret_bufpool(rsrc, buf);
0371                 irdma_puda_replenish_rq(rsrc, false);
0372             }
0373             goto done;
0374         }
0375 
0376         rsrc->stats_pkt_rcvd++;
0377         rsrc->compl_rxwqe_idx = info.wqe_idx;
0378         ibdev_dbg(to_ibdev(dev), "PUDA: RQ completion\n");
0379         rsrc->receive(rsrc->vsi, buf);
0380         if (cq_type == IRDMA_CQ_TYPE_ILQ)
0381             irdma_ilq_putback_rcvbuf(&rsrc->qp, buf, info.wqe_idx);
0382         else
0383             irdma_puda_replenish_rq(rsrc, false);
0384 
0385     } else {
0386         ibdev_dbg(to_ibdev(dev), "PUDA: SQ completion\n");
0387         buf = (struct irdma_puda_buf *)(uintptr_t)
0388                     qp->sq_wrtrk_array[info.wqe_idx].wrid;
0389 
0390         /* reusing so synch the buffer for CPU use */
0391         dma_sync_single_for_cpu(dev->hw->device, buf->mem.pa,
0392                     buf->mem.size, DMA_BIDIRECTIONAL);
0393         IRDMA_RING_SET_TAIL(qp->sq_ring, info.wqe_idx);
0394         rsrc->xmit_complete(rsrc->vsi, buf);
0395         spin_lock_irqsave(&rsrc->bufpool_lock, flags);
0396         rsrc->tx_wqe_avail_cnt++;
0397         spin_unlock_irqrestore(&rsrc->bufpool_lock, flags);
0398         if (!list_empty(&rsrc->txpend))
0399             irdma_puda_send_buf(rsrc, NULL);
0400     }
0401 
0402 done:
0403     IRDMA_RING_MOVE_HEAD_NOCHECK(cq_uk->cq_ring);
0404     if (!IRDMA_RING_CURRENT_HEAD(cq_uk->cq_ring))
0405         cq_uk->polarity = !cq_uk->polarity;
0406     /* update cq tail in cq shadow memory also */
0407     IRDMA_RING_MOVE_TAIL(cq_uk->cq_ring);
0408     set_64bit_val(cq_uk->shadow_area, 0,
0409               IRDMA_RING_CURRENT_HEAD(cq_uk->cq_ring));
0410 
0411     return ret;
0412 }
0413 
0414 /**
0415  * irdma_puda_send - complete send wqe for transmit
0416  * @qp: puda qp for send
0417  * @info: buffer information for transmit
0418  */
0419 int irdma_puda_send(struct irdma_sc_qp *qp, struct irdma_puda_send_info *info)
0420 {
0421     __le64 *wqe;
0422     u32 iplen, l4len;
0423     u64 hdr[2];
0424     u32 wqe_idx;
0425     u8 iipt;
0426 
0427     /* number of 32 bits DWORDS in header */
0428     l4len = info->tcplen >> 2;
0429     if (info->ipv4) {
0430         iipt = 3;
0431         iplen = 5;
0432     } else {
0433         iipt = 1;
0434         iplen = 10;
0435     }
0436 
0437     wqe = irdma_puda_get_next_send_wqe(&qp->qp_uk, &wqe_idx);
0438     if (!wqe)
0439         return -ENOMEM;
0440 
0441     qp->qp_uk.sq_wrtrk_array[wqe_idx].wrid = (uintptr_t)info->scratch;
0442     /* Third line of WQE descriptor */
0443     /* maclen is in words */
0444 
0445     if (qp->dev->hw_attrs.uk_attrs.hw_rev >= IRDMA_GEN_2) {
0446         hdr[0] = 0; /* Dest_QPN and Dest_QKey only for UD */
0447         hdr[1] = FIELD_PREP(IRDMA_UDA_QPSQ_OPCODE, IRDMA_OP_TYPE_SEND) |
0448              FIELD_PREP(IRDMA_UDA_QPSQ_L4LEN, l4len) |
0449              FIELD_PREP(IRDMAQPSQ_AHID, info->ah_id) |
0450              FIELD_PREP(IRDMA_UDA_QPSQ_SIGCOMPL, 1) |
0451              FIELD_PREP(IRDMA_UDA_QPSQ_VALID,
0452                     qp->qp_uk.swqe_polarity);
0453 
0454         /* Forth line of WQE descriptor */
0455 
0456         set_64bit_val(wqe, 0, info->paddr);
0457         set_64bit_val(wqe, 8,
0458                   FIELD_PREP(IRDMAQPSQ_FRAG_LEN, info->len) |
0459                   FIELD_PREP(IRDMA_UDA_QPSQ_VALID, qp->qp_uk.swqe_polarity));
0460     } else {
0461         hdr[0] = FIELD_PREP(IRDMA_UDA_QPSQ_MACLEN, info->maclen >> 1) |
0462              FIELD_PREP(IRDMA_UDA_QPSQ_IPLEN, iplen) |
0463              FIELD_PREP(IRDMA_UDA_QPSQ_L4T, 1) |
0464              FIELD_PREP(IRDMA_UDA_QPSQ_IIPT, iipt) |
0465              FIELD_PREP(IRDMA_GEN1_UDA_QPSQ_L4LEN, l4len);
0466 
0467         hdr[1] = FIELD_PREP(IRDMA_UDA_QPSQ_OPCODE, IRDMA_OP_TYPE_SEND) |
0468              FIELD_PREP(IRDMA_UDA_QPSQ_SIGCOMPL, 1) |
0469              FIELD_PREP(IRDMA_UDA_QPSQ_DOLOOPBACK, info->do_lpb) |
0470              FIELD_PREP(IRDMA_UDA_QPSQ_VALID, qp->qp_uk.swqe_polarity);
0471 
0472         /* Forth line of WQE descriptor */
0473 
0474         set_64bit_val(wqe, 0, info->paddr);
0475         set_64bit_val(wqe, 8,
0476                   FIELD_PREP(IRDMAQPSQ_GEN1_FRAG_LEN, info->len));
0477     }
0478 
0479     set_64bit_val(wqe, 16, hdr[0]);
0480     dma_wmb(); /* make sure WQE is written before valid bit is set */
0481 
0482     set_64bit_val(wqe, 24, hdr[1]);
0483 
0484     print_hex_dump_debug("PUDA: PUDA SEND WQE", DUMP_PREFIX_OFFSET, 16, 8,
0485                  wqe, 32, false);
0486     irdma_uk_qp_post_wr(&qp->qp_uk);
0487     return 0;
0488 }
0489 
0490 /**
0491  * irdma_puda_send_buf - transmit puda buffer
0492  * @rsrc: resource to use for buffer
0493  * @buf: puda buffer to transmit
0494  */
0495 void irdma_puda_send_buf(struct irdma_puda_rsrc *rsrc,
0496              struct irdma_puda_buf *buf)
0497 {
0498     struct irdma_puda_send_info info;
0499     int ret = 0;
0500     unsigned long flags;
0501 
0502     spin_lock_irqsave(&rsrc->bufpool_lock, flags);
0503     /* if no wqe available or not from a completion and we have
0504      * pending buffers, we must queue new buffer
0505      */
0506     if (!rsrc->tx_wqe_avail_cnt || (buf && !list_empty(&rsrc->txpend))) {
0507         list_add_tail(&buf->list, &rsrc->txpend);
0508         spin_unlock_irqrestore(&rsrc->bufpool_lock, flags);
0509         rsrc->stats_sent_pkt_q++;
0510         if (rsrc->type == IRDMA_PUDA_RSRC_TYPE_ILQ)
0511             ibdev_dbg(to_ibdev(rsrc->dev),
0512                   "PUDA: adding to txpend\n");
0513         return;
0514     }
0515     rsrc->tx_wqe_avail_cnt--;
0516     /* if we are coming from a completion and have pending buffers
0517      * then Get one from pending list
0518      */
0519     if (!buf) {
0520         buf = irdma_puda_get_listbuf(&rsrc->txpend);
0521         if (!buf)
0522             goto done;
0523     }
0524 
0525     info.scratch = buf;
0526     info.paddr = buf->mem.pa;
0527     info.len = buf->totallen;
0528     info.tcplen = buf->tcphlen;
0529     info.ipv4 = buf->ipv4;
0530 
0531     if (rsrc->dev->hw_attrs.uk_attrs.hw_rev >= IRDMA_GEN_2) {
0532         info.ah_id = buf->ah_id;
0533     } else {
0534         info.maclen = buf->maclen;
0535         info.do_lpb = buf->do_lpb;
0536     }
0537 
0538     /* Synch buffer for use by device */
0539     dma_sync_single_for_cpu(rsrc->dev->hw->device, buf->mem.pa,
0540                 buf->mem.size, DMA_BIDIRECTIONAL);
0541     ret = irdma_puda_send(&rsrc->qp, &info);
0542     if (ret) {
0543         rsrc->tx_wqe_avail_cnt++;
0544         rsrc->stats_sent_pkt_q++;
0545         list_add(&buf->list, &rsrc->txpend);
0546         if (rsrc->type == IRDMA_PUDA_RSRC_TYPE_ILQ)
0547             ibdev_dbg(to_ibdev(rsrc->dev),
0548                   "PUDA: adding to puda_send\n");
0549     } else {
0550         rsrc->stats_pkt_sent++;
0551     }
0552 done:
0553     spin_unlock_irqrestore(&rsrc->bufpool_lock, flags);
0554 }
0555 
0556 /**
0557  * irdma_puda_qp_setctx - during init, set qp's context
0558  * @rsrc: qp's resource
0559  */
0560 static void irdma_puda_qp_setctx(struct irdma_puda_rsrc *rsrc)
0561 {
0562     struct irdma_sc_qp *qp = &rsrc->qp;
0563     __le64 *qp_ctx = qp->hw_host_ctx;
0564 
0565     set_64bit_val(qp_ctx, 8, qp->sq_pa);
0566     set_64bit_val(qp_ctx, 16, qp->rq_pa);
0567     set_64bit_val(qp_ctx, 24,
0568               FIELD_PREP(IRDMAQPC_RQSIZE, qp->hw_rq_size) |
0569               FIELD_PREP(IRDMAQPC_SQSIZE, qp->hw_sq_size));
0570     set_64bit_val(qp_ctx, 48,
0571               FIELD_PREP(IRDMAQPC_SNDMSS, rsrc->buf_size));
0572     set_64bit_val(qp_ctx, 56, 0);
0573     if (qp->dev->hw_attrs.uk_attrs.hw_rev == IRDMA_GEN_1)
0574         set_64bit_val(qp_ctx, 64, 1);
0575     set_64bit_val(qp_ctx, 136,
0576               FIELD_PREP(IRDMAQPC_TXCQNUM, rsrc->cq_id) |
0577               FIELD_PREP(IRDMAQPC_RXCQNUM, rsrc->cq_id));
0578     set_64bit_val(qp_ctx, 144,
0579               FIELD_PREP(IRDMAQPC_STAT_INDEX, rsrc->stats_idx));
0580     set_64bit_val(qp_ctx, 160,
0581               FIELD_PREP(IRDMAQPC_PRIVEN, 1) |
0582               FIELD_PREP(IRDMAQPC_USESTATSINSTANCE, rsrc->stats_idx_valid));
0583     set_64bit_val(qp_ctx, 168,
0584               FIELD_PREP(IRDMAQPC_QPCOMPCTX, (uintptr_t)qp));
0585     set_64bit_val(qp_ctx, 176,
0586               FIELD_PREP(IRDMAQPC_SQTPHVAL, qp->sq_tph_val) |
0587               FIELD_PREP(IRDMAQPC_RQTPHVAL, qp->rq_tph_val) |
0588               FIELD_PREP(IRDMAQPC_QSHANDLE, qp->qs_handle));
0589 
0590     print_hex_dump_debug("PUDA: PUDA QP CONTEXT", DUMP_PREFIX_OFFSET, 16,
0591                  8, qp_ctx, IRDMA_QP_CTX_SIZE, false);
0592 }
0593 
0594 /**
0595  * irdma_puda_qp_wqe - setup wqe for qp create
0596  * @dev: Device
0597  * @qp: Resource qp
0598  */
0599 static int irdma_puda_qp_wqe(struct irdma_sc_dev *dev, struct irdma_sc_qp *qp)
0600 {
0601     struct irdma_sc_cqp *cqp;
0602     __le64 *wqe;
0603     u64 hdr;
0604     struct irdma_ccq_cqe_info compl_info;
0605     int status = 0;
0606 
0607     cqp = dev->cqp;
0608     wqe = irdma_sc_cqp_get_next_send_wqe(cqp, 0);
0609     if (!wqe)
0610         return -ENOMEM;
0611 
0612     set_64bit_val(wqe, 16, qp->hw_host_ctx_pa);
0613     set_64bit_val(wqe, 40, qp->shadow_area_pa);
0614 
0615     hdr = qp->qp_uk.qp_id |
0616           FIELD_PREP(IRDMA_CQPSQ_OPCODE, IRDMA_CQP_OP_CREATE_QP) |
0617           FIELD_PREP(IRDMA_CQPSQ_QP_QPTYPE, IRDMA_QP_TYPE_UDA) |
0618           FIELD_PREP(IRDMA_CQPSQ_QP_CQNUMVALID, 1) |
0619           FIELD_PREP(IRDMA_CQPSQ_QP_NEXTIWSTATE, 2) |
0620           FIELD_PREP(IRDMA_CQPSQ_WQEVALID, cqp->polarity);
0621     dma_wmb(); /* make sure WQE is written before valid bit is set */
0622 
0623     set_64bit_val(wqe, 24, hdr);
0624 
0625     print_hex_dump_debug("PUDA: PUDA QP CREATE", DUMP_PREFIX_OFFSET, 16,
0626                  8, wqe, 40, false);
0627     irdma_sc_cqp_post_sq(cqp);
0628     status = irdma_sc_poll_for_cqp_op_done(dev->cqp, IRDMA_CQP_OP_CREATE_QP,
0629                            &compl_info);
0630 
0631     return status;
0632 }
0633 
0634 /**
0635  * irdma_puda_qp_create - create qp for resource
0636  * @rsrc: resource to use for buffer
0637  */
0638 static int irdma_puda_qp_create(struct irdma_puda_rsrc *rsrc)
0639 {
0640     struct irdma_sc_qp *qp = &rsrc->qp;
0641     struct irdma_qp_uk *ukqp = &qp->qp_uk;
0642     int ret = 0;
0643     u32 sq_size, rq_size;
0644     struct irdma_dma_mem *mem;
0645 
0646     sq_size = rsrc->sq_size * IRDMA_QP_WQE_MIN_SIZE;
0647     rq_size = rsrc->rq_size * IRDMA_QP_WQE_MIN_SIZE;
0648     rsrc->qpmem.size = ALIGN((sq_size + rq_size + (IRDMA_SHADOW_AREA_SIZE << 3) + IRDMA_QP_CTX_SIZE),
0649                  IRDMA_HW_PAGE_SIZE);
0650     rsrc->qpmem.va = dma_alloc_coherent(rsrc->dev->hw->device,
0651                         rsrc->qpmem.size, &rsrc->qpmem.pa,
0652                         GFP_KERNEL);
0653     if (!rsrc->qpmem.va)
0654         return -ENOMEM;
0655 
0656     mem = &rsrc->qpmem;
0657     memset(mem->va, 0, rsrc->qpmem.size);
0658     qp->hw_sq_size = irdma_get_encoded_wqe_size(rsrc->sq_size, IRDMA_QUEUE_TYPE_SQ_RQ);
0659     qp->hw_rq_size = irdma_get_encoded_wqe_size(rsrc->rq_size, IRDMA_QUEUE_TYPE_SQ_RQ);
0660     qp->pd = &rsrc->sc_pd;
0661     qp->qp_uk.qp_type = IRDMA_QP_TYPE_UDA;
0662     qp->dev = rsrc->dev;
0663     qp->qp_uk.back_qp = rsrc;
0664     qp->sq_pa = mem->pa;
0665     qp->rq_pa = qp->sq_pa + sq_size;
0666     qp->vsi = rsrc->vsi;
0667     ukqp->sq_base = mem->va;
0668     ukqp->rq_base = &ukqp->sq_base[rsrc->sq_size];
0669     ukqp->shadow_area = ukqp->rq_base[rsrc->rq_size].elem;
0670     ukqp->uk_attrs = &qp->dev->hw_attrs.uk_attrs;
0671     qp->shadow_area_pa = qp->rq_pa + rq_size;
0672     qp->hw_host_ctx = ukqp->shadow_area + IRDMA_SHADOW_AREA_SIZE;
0673     qp->hw_host_ctx_pa = qp->shadow_area_pa + (IRDMA_SHADOW_AREA_SIZE << 3);
0674     qp->push_idx = IRDMA_INVALID_PUSH_PAGE_INDEX;
0675     ukqp->qp_id = rsrc->qp_id;
0676     ukqp->sq_wrtrk_array = rsrc->sq_wrtrk_array;
0677     ukqp->rq_wrid_array = rsrc->rq_wrid_array;
0678     ukqp->sq_size = rsrc->sq_size;
0679     ukqp->rq_size = rsrc->rq_size;
0680 
0681     IRDMA_RING_INIT(ukqp->sq_ring, ukqp->sq_size);
0682     IRDMA_RING_INIT(ukqp->initial_ring, ukqp->sq_size);
0683     IRDMA_RING_INIT(ukqp->rq_ring, ukqp->rq_size);
0684     ukqp->wqe_alloc_db = qp->pd->dev->wqe_alloc_db;
0685 
0686     ret = rsrc->dev->ws_add(qp->vsi, qp->user_pri);
0687     if (ret) {
0688         dma_free_coherent(rsrc->dev->hw->device, rsrc->qpmem.size,
0689                   rsrc->qpmem.va, rsrc->qpmem.pa);
0690         rsrc->qpmem.va = NULL;
0691         return ret;
0692     }
0693 
0694     irdma_qp_add_qos(qp);
0695     irdma_puda_qp_setctx(rsrc);
0696 
0697     if (rsrc->dev->ceq_valid)
0698         ret = irdma_cqp_qp_create_cmd(rsrc->dev, qp);
0699     else
0700         ret = irdma_puda_qp_wqe(rsrc->dev, qp);
0701     if (ret) {
0702         irdma_qp_rem_qos(qp);
0703         rsrc->dev->ws_remove(qp->vsi, qp->user_pri);
0704         dma_free_coherent(rsrc->dev->hw->device, rsrc->qpmem.size,
0705                   rsrc->qpmem.va, rsrc->qpmem.pa);
0706         rsrc->qpmem.va = NULL;
0707     }
0708 
0709     return ret;
0710 }
0711 
0712 /**
0713  * irdma_puda_cq_wqe - setup wqe for CQ create
0714  * @dev: Device
0715  * @cq: resource for cq
0716  */
0717 static int irdma_puda_cq_wqe(struct irdma_sc_dev *dev, struct irdma_sc_cq *cq)
0718 {
0719     __le64 *wqe;
0720     struct irdma_sc_cqp *cqp;
0721     u64 hdr;
0722     struct irdma_ccq_cqe_info compl_info;
0723     int status = 0;
0724 
0725     cqp = dev->cqp;
0726     wqe = irdma_sc_cqp_get_next_send_wqe(cqp, 0);
0727     if (!wqe)
0728         return -ENOMEM;
0729 
0730     set_64bit_val(wqe, 0, cq->cq_uk.cq_size);
0731     set_64bit_val(wqe, 8, (uintptr_t)cq >> 1);
0732     set_64bit_val(wqe, 16,
0733               FIELD_PREP(IRDMA_CQPSQ_CQ_SHADOW_READ_THRESHOLD, cq->shadow_read_threshold));
0734     set_64bit_val(wqe, 32, cq->cq_pa);
0735     set_64bit_val(wqe, 40, cq->shadow_area_pa);
0736     set_64bit_val(wqe, 56,
0737               FIELD_PREP(IRDMA_CQPSQ_TPHVAL, cq->tph_val) |
0738               FIELD_PREP(IRDMA_CQPSQ_VSIIDX, cq->vsi->vsi_idx));
0739 
0740     hdr = cq->cq_uk.cq_id |
0741           FIELD_PREP(IRDMA_CQPSQ_OPCODE, IRDMA_CQP_OP_CREATE_CQ) |
0742           FIELD_PREP(IRDMA_CQPSQ_CQ_CHKOVERFLOW, 1) |
0743           FIELD_PREP(IRDMA_CQPSQ_CQ_ENCEQEMASK, 1) |
0744           FIELD_PREP(IRDMA_CQPSQ_CQ_CEQIDVALID, 1) |
0745           FIELD_PREP(IRDMA_CQPSQ_WQEVALID, cqp->polarity);
0746     dma_wmb(); /* make sure WQE is written before valid bit is set */
0747 
0748     set_64bit_val(wqe, 24, hdr);
0749 
0750     print_hex_dump_debug("PUDA: PUDA CREATE CQ", DUMP_PREFIX_OFFSET, 16,
0751                  8, wqe, IRDMA_CQP_WQE_SIZE * 8, false);
0752     irdma_sc_cqp_post_sq(dev->cqp);
0753     status = irdma_sc_poll_for_cqp_op_done(dev->cqp, IRDMA_CQP_OP_CREATE_CQ,
0754                            &compl_info);
0755     if (!status) {
0756         struct irdma_sc_ceq *ceq = dev->ceq[0];
0757 
0758         if (ceq && ceq->reg_cq)
0759             status = irdma_sc_add_cq_ctx(ceq, cq);
0760     }
0761 
0762     return status;
0763 }
0764 
0765 /**
0766  * irdma_puda_cq_create - create cq for resource
0767  * @rsrc: resource for which cq to create
0768  */
0769 static int irdma_puda_cq_create(struct irdma_puda_rsrc *rsrc)
0770 {
0771     struct irdma_sc_dev *dev = rsrc->dev;
0772     struct irdma_sc_cq *cq = &rsrc->cq;
0773     int ret = 0;
0774     u32 cqsize;
0775     struct irdma_dma_mem *mem;
0776     struct irdma_cq_init_info info = {};
0777     struct irdma_cq_uk_init_info *init_info = &info.cq_uk_init_info;
0778 
0779     cq->vsi = rsrc->vsi;
0780     cqsize = rsrc->cq_size * (sizeof(struct irdma_cqe));
0781     rsrc->cqmem.size = ALIGN(cqsize + sizeof(struct irdma_cq_shadow_area),
0782                  IRDMA_CQ0_ALIGNMENT);
0783     rsrc->cqmem.va = dma_alloc_coherent(dev->hw->device, rsrc->cqmem.size,
0784                         &rsrc->cqmem.pa, GFP_KERNEL);
0785     if (!rsrc->cqmem.va)
0786         return -ENOMEM;
0787 
0788     mem = &rsrc->cqmem;
0789     info.dev = dev;
0790     info.type = (rsrc->type == IRDMA_PUDA_RSRC_TYPE_ILQ) ?
0791             IRDMA_CQ_TYPE_ILQ : IRDMA_CQ_TYPE_IEQ;
0792     info.shadow_read_threshold = rsrc->cq_size >> 2;
0793     info.cq_base_pa = mem->pa;
0794     info.shadow_area_pa = mem->pa + cqsize;
0795     init_info->cq_base = mem->va;
0796     init_info->shadow_area = (__le64 *)((u8 *)mem->va + cqsize);
0797     init_info->cq_size = rsrc->cq_size;
0798     init_info->cq_id = rsrc->cq_id;
0799     info.ceqe_mask = true;
0800     info.ceq_id_valid = true;
0801     info.vsi = rsrc->vsi;
0802 
0803     ret = irdma_sc_cq_init(cq, &info);
0804     if (ret)
0805         goto error;
0806 
0807     if (rsrc->dev->ceq_valid)
0808         ret = irdma_cqp_cq_create_cmd(dev, cq);
0809     else
0810         ret = irdma_puda_cq_wqe(dev, cq);
0811 error:
0812     if (ret) {
0813         dma_free_coherent(dev->hw->device, rsrc->cqmem.size,
0814                   rsrc->cqmem.va, rsrc->cqmem.pa);
0815         rsrc->cqmem.va = NULL;
0816     }
0817 
0818     return ret;
0819 }
0820 
0821 /**
0822  * irdma_puda_free_qp - free qp for resource
0823  * @rsrc: resource for which qp to free
0824  */
0825 static void irdma_puda_free_qp(struct irdma_puda_rsrc *rsrc)
0826 {
0827     int ret;
0828     struct irdma_ccq_cqe_info compl_info;
0829     struct irdma_sc_dev *dev = rsrc->dev;
0830 
0831     if (rsrc->dev->ceq_valid) {
0832         irdma_cqp_qp_destroy_cmd(dev, &rsrc->qp);
0833         rsrc->dev->ws_remove(rsrc->qp.vsi, rsrc->qp.user_pri);
0834         return;
0835     }
0836 
0837     ret = irdma_sc_qp_destroy(&rsrc->qp, 0, false, true, true);
0838     if (ret)
0839         ibdev_dbg(to_ibdev(dev),
0840               "PUDA: error puda qp destroy wqe, status = %d\n",
0841               ret);
0842     if (!ret) {
0843         ret = irdma_sc_poll_for_cqp_op_done(dev->cqp, IRDMA_CQP_OP_DESTROY_QP,
0844                             &compl_info);
0845         if (ret)
0846             ibdev_dbg(to_ibdev(dev),
0847                   "PUDA: error puda qp destroy failed, status = %d\n",
0848                   ret);
0849     }
0850     rsrc->dev->ws_remove(rsrc->qp.vsi, rsrc->qp.user_pri);
0851 }
0852 
0853 /**
0854  * irdma_puda_free_cq - free cq for resource
0855  * @rsrc: resource for which cq to free
0856  */
0857 static void irdma_puda_free_cq(struct irdma_puda_rsrc *rsrc)
0858 {
0859     int ret;
0860     struct irdma_ccq_cqe_info compl_info;
0861     struct irdma_sc_dev *dev = rsrc->dev;
0862 
0863     if (rsrc->dev->ceq_valid) {
0864         irdma_cqp_cq_destroy_cmd(dev, &rsrc->cq);
0865         return;
0866     }
0867 
0868     ret = irdma_sc_cq_destroy(&rsrc->cq, 0, true);
0869     if (ret)
0870         ibdev_dbg(to_ibdev(dev), "PUDA: error ieq cq destroy\n");
0871     if (!ret) {
0872         ret = irdma_sc_poll_for_cqp_op_done(dev->cqp, IRDMA_CQP_OP_DESTROY_CQ,
0873                             &compl_info);
0874         if (ret)
0875             ibdev_dbg(to_ibdev(dev),
0876                   "PUDA: error ieq qp destroy done\n");
0877     }
0878 }
0879 
0880 /**
0881  * irdma_puda_dele_rsrc - delete all resources during close
0882  * @vsi: VSI structure of device
0883  * @type: type of resource to dele
0884  * @reset: true if reset chip
0885  */
0886 void irdma_puda_dele_rsrc(struct irdma_sc_vsi *vsi, enum puda_rsrc_type type,
0887               bool reset)
0888 {
0889     struct irdma_sc_dev *dev = vsi->dev;
0890     struct irdma_puda_rsrc *rsrc;
0891     struct irdma_puda_buf *buf = NULL;
0892     struct irdma_puda_buf *nextbuf = NULL;
0893     struct irdma_virt_mem *vmem;
0894     struct irdma_sc_ceq *ceq;
0895 
0896     ceq = vsi->dev->ceq[0];
0897     switch (type) {
0898     case IRDMA_PUDA_RSRC_TYPE_ILQ:
0899         rsrc = vsi->ilq;
0900         vmem = &vsi->ilq_mem;
0901         vsi->ilq = NULL;
0902         if (ceq && ceq->reg_cq)
0903             irdma_sc_remove_cq_ctx(ceq, &rsrc->cq);
0904         break;
0905     case IRDMA_PUDA_RSRC_TYPE_IEQ:
0906         rsrc = vsi->ieq;
0907         vmem = &vsi->ieq_mem;
0908         vsi->ieq = NULL;
0909         if (ceq && ceq->reg_cq)
0910             irdma_sc_remove_cq_ctx(ceq, &rsrc->cq);
0911         break;
0912     default:
0913         ibdev_dbg(to_ibdev(dev), "PUDA: error resource type = 0x%x\n",
0914               type);
0915         return;
0916     }
0917 
0918     switch (rsrc->cmpl) {
0919     case PUDA_HASH_CRC_COMPLETE:
0920         irdma_free_hash_desc(rsrc->hash_desc);
0921         fallthrough;
0922     case PUDA_QP_CREATED:
0923         irdma_qp_rem_qos(&rsrc->qp);
0924 
0925         if (!reset)
0926             irdma_puda_free_qp(rsrc);
0927 
0928         dma_free_coherent(dev->hw->device, rsrc->qpmem.size,
0929                   rsrc->qpmem.va, rsrc->qpmem.pa);
0930         rsrc->qpmem.va = NULL;
0931         fallthrough;
0932     case PUDA_CQ_CREATED:
0933         if (!reset)
0934             irdma_puda_free_cq(rsrc);
0935 
0936         dma_free_coherent(dev->hw->device, rsrc->cqmem.size,
0937                   rsrc->cqmem.va, rsrc->cqmem.pa);
0938         rsrc->cqmem.va = NULL;
0939         break;
0940     default:
0941         ibdev_dbg(to_ibdev(rsrc->dev), "PUDA: error no resources\n");
0942         break;
0943     }
0944     /* Free all allocated puda buffers for both tx and rx */
0945     buf = rsrc->alloclist;
0946     while (buf) {
0947         nextbuf = buf->next;
0948         irdma_puda_dele_buf(dev, buf);
0949         buf = nextbuf;
0950         rsrc->alloc_buf_count--;
0951     }
0952 
0953     kfree(vmem->va);
0954 }
0955 
0956 /**
0957  * irdma_puda_allocbufs - allocate buffers for resource
0958  * @rsrc: resource for buffer allocation
0959  * @count: number of buffers to create
0960  */
0961 static int irdma_puda_allocbufs(struct irdma_puda_rsrc *rsrc, u32 count)
0962 {
0963     u32 i;
0964     struct irdma_puda_buf *buf;
0965     struct irdma_puda_buf *nextbuf;
0966 
0967     for (i = 0; i < count; i++) {
0968         buf = irdma_puda_alloc_buf(rsrc->dev, rsrc->buf_size);
0969         if (!buf) {
0970             rsrc->stats_buf_alloc_fail++;
0971             return -ENOMEM;
0972         }
0973         irdma_puda_ret_bufpool(rsrc, buf);
0974         rsrc->alloc_buf_count++;
0975         if (!rsrc->alloclist) {
0976             rsrc->alloclist = buf;
0977         } else {
0978             nextbuf = rsrc->alloclist;
0979             rsrc->alloclist = buf;
0980             buf->next = nextbuf;
0981         }
0982     }
0983 
0984     rsrc->avail_buf_count = rsrc->alloc_buf_count;
0985 
0986     return 0;
0987 }
0988 
0989 /**
0990  * irdma_puda_create_rsrc - create resource (ilq or ieq)
0991  * @vsi: sc VSI struct
0992  * @info: resource information
0993  */
0994 int irdma_puda_create_rsrc(struct irdma_sc_vsi *vsi,
0995                struct irdma_puda_rsrc_info *info)
0996 {
0997     struct irdma_sc_dev *dev = vsi->dev;
0998     int ret = 0;
0999     struct irdma_puda_rsrc *rsrc;
1000     u32 pudasize;
1001     u32 sqwridsize, rqwridsize;
1002     struct irdma_virt_mem *vmem;
1003 
1004     info->count = 1;
1005     pudasize = sizeof(struct irdma_puda_rsrc);
1006     sqwridsize = info->sq_size * sizeof(struct irdma_sq_uk_wr_trk_info);
1007     rqwridsize = info->rq_size * 8;
1008     switch (info->type) {
1009     case IRDMA_PUDA_RSRC_TYPE_ILQ:
1010         vmem = &vsi->ilq_mem;
1011         break;
1012     case IRDMA_PUDA_RSRC_TYPE_IEQ:
1013         vmem = &vsi->ieq_mem;
1014         break;
1015     default:
1016         return -EOPNOTSUPP;
1017     }
1018     vmem->size = pudasize + sqwridsize + rqwridsize;
1019     vmem->va = kzalloc(vmem->size, GFP_KERNEL);
1020     if (!vmem->va)
1021         return -ENOMEM;
1022 
1023     rsrc = vmem->va;
1024     spin_lock_init(&rsrc->bufpool_lock);
1025     switch (info->type) {
1026     case IRDMA_PUDA_RSRC_TYPE_ILQ:
1027         vsi->ilq = vmem->va;
1028         vsi->ilq_count = info->count;
1029         rsrc->receive = info->receive;
1030         rsrc->xmit_complete = info->xmit_complete;
1031         break;
1032     case IRDMA_PUDA_RSRC_TYPE_IEQ:
1033         vsi->ieq_count = info->count;
1034         vsi->ieq = vmem->va;
1035         rsrc->receive = irdma_ieq_receive;
1036         rsrc->xmit_complete = irdma_ieq_tx_compl;
1037         break;
1038     default:
1039         return -EOPNOTSUPP;
1040     }
1041 
1042     rsrc->type = info->type;
1043     rsrc->sq_wrtrk_array = (struct irdma_sq_uk_wr_trk_info *)
1044                    ((u8 *)vmem->va + pudasize);
1045     rsrc->rq_wrid_array = (u64 *)((u8 *)vmem->va + pudasize + sqwridsize);
1046     /* Initialize all ieq lists */
1047     INIT_LIST_HEAD(&rsrc->bufpool);
1048     INIT_LIST_HEAD(&rsrc->txpend);
1049 
1050     rsrc->tx_wqe_avail_cnt = info->sq_size - 1;
1051     irdma_sc_pd_init(dev, &rsrc->sc_pd, info->pd_id, info->abi_ver);
1052     rsrc->qp_id = info->qp_id;
1053     rsrc->cq_id = info->cq_id;
1054     rsrc->sq_size = info->sq_size;
1055     rsrc->rq_size = info->rq_size;
1056     rsrc->cq_size = info->rq_size + info->sq_size;
1057     if (dev->hw_attrs.uk_attrs.hw_rev >= IRDMA_GEN_2) {
1058         if (rsrc->type == IRDMA_PUDA_RSRC_TYPE_ILQ)
1059             rsrc->cq_size += info->rq_size;
1060     }
1061     rsrc->buf_size = info->buf_size;
1062     rsrc->dev = dev;
1063     rsrc->vsi = vsi;
1064     rsrc->stats_idx = info->stats_idx;
1065     rsrc->stats_idx_valid = info->stats_idx_valid;
1066 
1067     ret = irdma_puda_cq_create(rsrc);
1068     if (!ret) {
1069         rsrc->cmpl = PUDA_CQ_CREATED;
1070         ret = irdma_puda_qp_create(rsrc);
1071     }
1072     if (ret) {
1073         ibdev_dbg(to_ibdev(dev),
1074               "PUDA: error qp_create type=%d, status=%d\n",
1075               rsrc->type, ret);
1076         goto error;
1077     }
1078     rsrc->cmpl = PUDA_QP_CREATED;
1079 
1080     ret = irdma_puda_allocbufs(rsrc, info->tx_buf_cnt + info->rq_size);
1081     if (ret) {
1082         ibdev_dbg(to_ibdev(dev), "PUDA: error alloc_buf\n");
1083         goto error;
1084     }
1085 
1086     rsrc->rxq_invalid_cnt = info->rq_size;
1087     ret = irdma_puda_replenish_rq(rsrc, true);
1088     if (ret)
1089         goto error;
1090 
1091     if (info->type == IRDMA_PUDA_RSRC_TYPE_IEQ) {
1092         if (!irdma_init_hash_desc(&rsrc->hash_desc)) {
1093             rsrc->check_crc = true;
1094             rsrc->cmpl = PUDA_HASH_CRC_COMPLETE;
1095             ret = 0;
1096         }
1097     }
1098 
1099     irdma_sc_ccq_arm(&rsrc->cq);
1100     return ret;
1101 
1102 error:
1103     irdma_puda_dele_rsrc(vsi, info->type, false);
1104 
1105     return ret;
1106 }
1107 
1108 /**
1109  * irdma_ilq_putback_rcvbuf - ilq buffer to put back on rq
1110  * @qp: ilq's qp resource
1111  * @buf: puda buffer for rcv q
1112  * @wqe_idx:  wqe index of completed rcvbuf
1113  */
1114 static void irdma_ilq_putback_rcvbuf(struct irdma_sc_qp *qp,
1115                      struct irdma_puda_buf *buf, u32 wqe_idx)
1116 {
1117     __le64 *wqe;
1118     u64 offset8, offset24;
1119 
1120     /* Synch buffer for use by device */
1121     dma_sync_single_for_device(qp->dev->hw->device, buf->mem.pa,
1122                    buf->mem.size, DMA_BIDIRECTIONAL);
1123     wqe = qp->qp_uk.rq_base[wqe_idx].elem;
1124     get_64bit_val(wqe, 24, &offset24);
1125     if (qp->dev->hw_attrs.uk_attrs.hw_rev >= IRDMA_GEN_2) {
1126         get_64bit_val(wqe, 8, &offset8);
1127         if (offset24)
1128             offset8 &= ~FIELD_PREP(IRDMAQPSQ_VALID, 1);
1129         else
1130             offset8 |= FIELD_PREP(IRDMAQPSQ_VALID, 1);
1131         set_64bit_val(wqe, 8, offset8);
1132         dma_wmb(); /* make sure WQE is written before valid bit is set */
1133     }
1134     if (offset24)
1135         offset24 = 0;
1136     else
1137         offset24 = FIELD_PREP(IRDMAQPSQ_VALID, 1);
1138 
1139     set_64bit_val(wqe, 24, offset24);
1140 }
1141 
1142 /**
1143  * irdma_ieq_get_fpdu_len - get length of fpdu with or without marker
1144  * @pfpdu: pointer to fpdu
1145  * @datap: pointer to data in the buffer
1146  * @rcv_seq: seqnum of the data buffer
1147  */
1148 static u16 irdma_ieq_get_fpdu_len(struct irdma_pfpdu *pfpdu, u8 *datap,
1149                   u32 rcv_seq)
1150 {
1151     u32 marker_seq, end_seq, blk_start;
1152     u8 marker_len = pfpdu->marker_len;
1153     u16 total_len = 0;
1154     u16 fpdu_len;
1155 
1156     blk_start = (pfpdu->rcv_start_seq - rcv_seq) & (IRDMA_MRK_BLK_SZ - 1);
1157     if (!blk_start) {
1158         total_len = marker_len;
1159         marker_seq = rcv_seq + IRDMA_MRK_BLK_SZ;
1160         if (marker_len && *(u32 *)datap)
1161             return 0;
1162     } else {
1163         marker_seq = rcv_seq + blk_start;
1164     }
1165 
1166     datap += total_len;
1167     fpdu_len = ntohs(*(__be16 *)datap);
1168     fpdu_len += IRDMA_IEQ_MPA_FRAMING;
1169     fpdu_len = (fpdu_len + 3) & 0xfffc;
1170 
1171     if (fpdu_len > pfpdu->max_fpdu_data)
1172         return 0;
1173 
1174     total_len += fpdu_len;
1175     end_seq = rcv_seq + total_len;
1176     while ((int)(marker_seq - end_seq) < 0) {
1177         total_len += marker_len;
1178         end_seq += marker_len;
1179         marker_seq += IRDMA_MRK_BLK_SZ;
1180     }
1181 
1182     return total_len;
1183 }
1184 
1185 /**
1186  * irdma_ieq_copy_to_txbuf - copydata from rcv buf to tx buf
1187  * @buf: rcv buffer with partial
1188  * @txbuf: tx buffer for sending back
1189  * @buf_offset: rcv buffer offset to copy from
1190  * @txbuf_offset: at offset in tx buf to copy
1191  * @len: length of data to copy
1192  */
1193 static void irdma_ieq_copy_to_txbuf(struct irdma_puda_buf *buf,
1194                     struct irdma_puda_buf *txbuf,
1195                     u16 buf_offset, u32 txbuf_offset, u32 len)
1196 {
1197     void *mem1 = (u8 *)buf->mem.va + buf_offset;
1198     void *mem2 = (u8 *)txbuf->mem.va + txbuf_offset;
1199 
1200     memcpy(mem2, mem1, len);
1201 }
1202 
1203 /**
1204  * irdma_ieq_setup_tx_buf - setup tx buffer for partial handling
1205  * @buf: reeive buffer with partial
1206  * @txbuf: buffer to prepare
1207  */
1208 static void irdma_ieq_setup_tx_buf(struct irdma_puda_buf *buf,
1209                    struct irdma_puda_buf *txbuf)
1210 {
1211     txbuf->tcphlen = buf->tcphlen;
1212     txbuf->ipv4 = buf->ipv4;
1213 
1214     if (buf->vsi->dev->hw_attrs.uk_attrs.hw_rev >= IRDMA_GEN_2) {
1215         txbuf->hdrlen = txbuf->tcphlen;
1216         irdma_ieq_copy_to_txbuf(buf, txbuf, IRDMA_TCP_OFFSET, 0,
1217                     txbuf->hdrlen);
1218     } else {
1219         txbuf->maclen = buf->maclen;
1220         txbuf->hdrlen = buf->hdrlen;
1221         irdma_ieq_copy_to_txbuf(buf, txbuf, 0, 0, buf->hdrlen);
1222     }
1223 }
1224 
1225 /**
1226  * irdma_ieq_check_first_buf - check if rcv buffer's seq is in range
1227  * @buf: receive exception buffer
1228  * @fps: first partial sequence number
1229  */
1230 static void irdma_ieq_check_first_buf(struct irdma_puda_buf *buf, u32 fps)
1231 {
1232     u32 offset;
1233 
1234     if (buf->seqnum < fps) {
1235         offset = fps - buf->seqnum;
1236         if (offset > buf->datalen)
1237             return;
1238         buf->data += offset;
1239         buf->datalen -= (u16)offset;
1240         buf->seqnum = fps;
1241     }
1242 }
1243 
1244 /**
1245  * irdma_ieq_compl_pfpdu - write txbuf with full fpdu
1246  * @ieq: ieq resource
1247  * @rxlist: ieq's received buffer list
1248  * @pbufl: temporary list for buffers for fpddu
1249  * @txbuf: tx buffer for fpdu
1250  * @fpdu_len: total length of fpdu
1251  */
1252 static void irdma_ieq_compl_pfpdu(struct irdma_puda_rsrc *ieq,
1253                   struct list_head *rxlist,
1254                   struct list_head *pbufl,
1255                   struct irdma_puda_buf *txbuf, u16 fpdu_len)
1256 {
1257     struct irdma_puda_buf *buf;
1258     u32 nextseqnum;
1259     u16 txoffset, bufoffset;
1260 
1261     buf = irdma_puda_get_listbuf(pbufl);
1262     if (!buf)
1263         return;
1264 
1265     nextseqnum = buf->seqnum + fpdu_len;
1266     irdma_ieq_setup_tx_buf(buf, txbuf);
1267     if (buf->vsi->dev->hw_attrs.uk_attrs.hw_rev >= IRDMA_GEN_2) {
1268         txoffset = txbuf->hdrlen;
1269         txbuf->totallen = txbuf->hdrlen + fpdu_len;
1270         txbuf->data = (u8 *)txbuf->mem.va + txoffset;
1271     } else {
1272         txoffset = buf->hdrlen;
1273         txbuf->totallen = buf->hdrlen + fpdu_len;
1274         txbuf->data = (u8 *)txbuf->mem.va + buf->hdrlen;
1275     }
1276     bufoffset = (u16)(buf->data - (u8 *)buf->mem.va);
1277 
1278     do {
1279         if (buf->datalen >= fpdu_len) {
1280             /* copied full fpdu */
1281             irdma_ieq_copy_to_txbuf(buf, txbuf, bufoffset, txoffset,
1282                         fpdu_len);
1283             buf->datalen -= fpdu_len;
1284             buf->data += fpdu_len;
1285             buf->seqnum = nextseqnum;
1286             break;
1287         }
1288         /* copy partial fpdu */
1289         irdma_ieq_copy_to_txbuf(buf, txbuf, bufoffset, txoffset,
1290                     buf->datalen);
1291         txoffset += buf->datalen;
1292         fpdu_len -= buf->datalen;
1293         irdma_puda_ret_bufpool(ieq, buf);
1294         buf = irdma_puda_get_listbuf(pbufl);
1295         if (!buf)
1296             return;
1297 
1298         bufoffset = (u16)(buf->data - (u8 *)buf->mem.va);
1299     } while (1);
1300 
1301     /* last buffer on the list*/
1302     if (buf->datalen)
1303         list_add(&buf->list, rxlist);
1304     else
1305         irdma_puda_ret_bufpool(ieq, buf);
1306 }
1307 
1308 /**
1309  * irdma_ieq_create_pbufl - create buffer list for single fpdu
1310  * @pfpdu: pointer to fpdu
1311  * @rxlist: resource list for receive ieq buffes
1312  * @pbufl: temp. list for buffers for fpddu
1313  * @buf: first receive buffer
1314  * @fpdu_len: total length of fpdu
1315  */
1316 static int irdma_ieq_create_pbufl(struct irdma_pfpdu *pfpdu,
1317                   struct list_head *rxlist,
1318                   struct list_head *pbufl,
1319                   struct irdma_puda_buf *buf, u16 fpdu_len)
1320 {
1321     int status = 0;
1322     struct irdma_puda_buf *nextbuf;
1323     u32 nextseqnum;
1324     u16 plen = fpdu_len - buf->datalen;
1325     bool done = false;
1326 
1327     nextseqnum = buf->seqnum + buf->datalen;
1328     do {
1329         nextbuf = irdma_puda_get_listbuf(rxlist);
1330         if (!nextbuf) {
1331             status = -ENOBUFS;
1332             break;
1333         }
1334         list_add_tail(&nextbuf->list, pbufl);
1335         if (nextbuf->seqnum != nextseqnum) {
1336             pfpdu->bad_seq_num++;
1337             status = -ERANGE;
1338             break;
1339         }
1340         if (nextbuf->datalen >= plen) {
1341             done = true;
1342         } else {
1343             plen -= nextbuf->datalen;
1344             nextseqnum = nextbuf->seqnum + nextbuf->datalen;
1345         }
1346 
1347     } while (!done);
1348 
1349     return status;
1350 }
1351 
1352 /**
1353  * irdma_ieq_handle_partial - process partial fpdu buffer
1354  * @ieq: ieq resource
1355  * @pfpdu: partial management per user qp
1356  * @buf: receive buffer
1357  * @fpdu_len: fpdu len in the buffer
1358  */
1359 static int irdma_ieq_handle_partial(struct irdma_puda_rsrc *ieq,
1360                     struct irdma_pfpdu *pfpdu,
1361                     struct irdma_puda_buf *buf, u16 fpdu_len)
1362 {
1363     int status = 0;
1364     u8 *crcptr;
1365     u32 mpacrc;
1366     u32 seqnum = buf->seqnum;
1367     struct list_head pbufl; /* partial buffer list */
1368     struct irdma_puda_buf *txbuf = NULL;
1369     struct list_head *rxlist = &pfpdu->rxlist;
1370 
1371     ieq->partials_handled++;
1372 
1373     INIT_LIST_HEAD(&pbufl);
1374     list_add(&buf->list, &pbufl);
1375 
1376     status = irdma_ieq_create_pbufl(pfpdu, rxlist, &pbufl, buf, fpdu_len);
1377     if (status)
1378         goto error;
1379 
1380     txbuf = irdma_puda_get_bufpool(ieq);
1381     if (!txbuf) {
1382         pfpdu->no_tx_bufs++;
1383         status = -ENOBUFS;
1384         goto error;
1385     }
1386 
1387     irdma_ieq_compl_pfpdu(ieq, rxlist, &pbufl, txbuf, fpdu_len);
1388     irdma_ieq_update_tcpip_info(txbuf, fpdu_len, seqnum);
1389 
1390     crcptr = txbuf->data + fpdu_len - 4;
1391     mpacrc = *(u32 *)crcptr;
1392     if (ieq->check_crc) {
1393         status = irdma_ieq_check_mpacrc(ieq->hash_desc, txbuf->data,
1394                         (fpdu_len - 4), mpacrc);
1395         if (status) {
1396             ibdev_dbg(to_ibdev(ieq->dev), "IEQ: error bad crc\n");
1397             goto error;
1398         }
1399     }
1400 
1401     print_hex_dump_debug("IEQ: IEQ TX BUFFER", DUMP_PREFIX_OFFSET, 16, 8,
1402                  txbuf->mem.va, txbuf->totallen, false);
1403     if (ieq->dev->hw_attrs.uk_attrs.hw_rev >= IRDMA_GEN_2)
1404         txbuf->ah_id = pfpdu->ah->ah_info.ah_idx;
1405     txbuf->do_lpb = true;
1406     irdma_puda_send_buf(ieq, txbuf);
1407     pfpdu->rcv_nxt = seqnum + fpdu_len;
1408     return status;
1409 
1410 error:
1411     while (!list_empty(&pbufl)) {
1412         buf = list_last_entry(&pbufl, struct irdma_puda_buf, list);
1413         list_move(&buf->list, rxlist);
1414     }
1415     if (txbuf)
1416         irdma_puda_ret_bufpool(ieq, txbuf);
1417 
1418     return status;
1419 }
1420 
1421 /**
1422  * irdma_ieq_process_buf - process buffer rcvd for ieq
1423  * @ieq: ieq resource
1424  * @pfpdu: partial management per user qp
1425  * @buf: receive buffer
1426  */
1427 static int irdma_ieq_process_buf(struct irdma_puda_rsrc *ieq,
1428                  struct irdma_pfpdu *pfpdu,
1429                  struct irdma_puda_buf *buf)
1430 {
1431     u16 fpdu_len = 0;
1432     u16 datalen = buf->datalen;
1433     u8 *datap = buf->data;
1434     u8 *crcptr;
1435     u16 ioffset = 0;
1436     u32 mpacrc;
1437     u32 seqnum = buf->seqnum;
1438     u16 len = 0;
1439     u16 full = 0;
1440     bool partial = false;
1441     struct irdma_puda_buf *txbuf;
1442     struct list_head *rxlist = &pfpdu->rxlist;
1443     int ret = 0;
1444 
1445     ioffset = (u16)(buf->data - (u8 *)buf->mem.va);
1446     while (datalen) {
1447         fpdu_len = irdma_ieq_get_fpdu_len(pfpdu, datap, buf->seqnum);
1448         if (!fpdu_len) {
1449             ibdev_dbg(to_ibdev(ieq->dev),
1450                   "IEQ: error bad fpdu len\n");
1451             list_add(&buf->list, rxlist);
1452             return -EINVAL;
1453         }
1454 
1455         if (datalen < fpdu_len) {
1456             partial = true;
1457             break;
1458         }
1459         crcptr = datap + fpdu_len - 4;
1460         mpacrc = *(u32 *)crcptr;
1461         if (ieq->check_crc)
1462             ret = irdma_ieq_check_mpacrc(ieq->hash_desc, datap,
1463                              fpdu_len - 4, mpacrc);
1464         if (ret) {
1465             list_add(&buf->list, rxlist);
1466             ibdev_dbg(to_ibdev(ieq->dev),
1467                   "ERR: IRDMA_ERR_MPA_CRC\n");
1468             return -EINVAL;
1469         }
1470         full++;
1471         pfpdu->fpdu_processed++;
1472         ieq->fpdu_processed++;
1473         datap += fpdu_len;
1474         len += fpdu_len;
1475         datalen -= fpdu_len;
1476     }
1477     if (full) {
1478         /* copy full pdu's in the txbuf and send them out */
1479         txbuf = irdma_puda_get_bufpool(ieq);
1480         if (!txbuf) {
1481             pfpdu->no_tx_bufs++;
1482             list_add(&buf->list, rxlist);
1483             return -ENOBUFS;
1484         }
1485         /* modify txbuf's buffer header */
1486         irdma_ieq_setup_tx_buf(buf, txbuf);
1487         /* copy full fpdu's to new buffer */
1488         if (ieq->dev->hw_attrs.uk_attrs.hw_rev >= IRDMA_GEN_2) {
1489             irdma_ieq_copy_to_txbuf(buf, txbuf, ioffset,
1490                         txbuf->hdrlen, len);
1491             txbuf->totallen = txbuf->hdrlen + len;
1492             txbuf->ah_id = pfpdu->ah->ah_info.ah_idx;
1493         } else {
1494             irdma_ieq_copy_to_txbuf(buf, txbuf, ioffset,
1495                         buf->hdrlen, len);
1496             txbuf->totallen = buf->hdrlen + len;
1497         }
1498         irdma_ieq_update_tcpip_info(txbuf, len, buf->seqnum);
1499         print_hex_dump_debug("IEQ: IEQ TX BUFFER", DUMP_PREFIX_OFFSET,
1500                      16, 8, txbuf->mem.va, txbuf->totallen,
1501                      false);
1502         txbuf->do_lpb = true;
1503         irdma_puda_send_buf(ieq, txbuf);
1504 
1505         if (!datalen) {
1506             pfpdu->rcv_nxt = buf->seqnum + len;
1507             irdma_puda_ret_bufpool(ieq, buf);
1508             return 0;
1509         }
1510         buf->data = datap;
1511         buf->seqnum = seqnum + len;
1512         buf->datalen = datalen;
1513         pfpdu->rcv_nxt = buf->seqnum;
1514     }
1515     if (partial)
1516         return irdma_ieq_handle_partial(ieq, pfpdu, buf, fpdu_len);
1517 
1518     return 0;
1519 }
1520 
1521 /**
1522  * irdma_ieq_process_fpdus - process fpdu's buffers on its list
1523  * @qp: qp for which partial fpdus
1524  * @ieq: ieq resource
1525  */
1526 void irdma_ieq_process_fpdus(struct irdma_sc_qp *qp,
1527                  struct irdma_puda_rsrc *ieq)
1528 {
1529     struct irdma_pfpdu *pfpdu = &qp->pfpdu;
1530     struct list_head *rxlist = &pfpdu->rxlist;
1531     struct irdma_puda_buf *buf;
1532     int status;
1533 
1534     do {
1535         if (list_empty(rxlist))
1536             break;
1537         buf = irdma_puda_get_listbuf(rxlist);
1538         if (!buf) {
1539             ibdev_dbg(to_ibdev(ieq->dev), "IEQ: error no buf\n");
1540             break;
1541         }
1542         if (buf->seqnum != pfpdu->rcv_nxt) {
1543             /* This could be out of order or missing packet */
1544             pfpdu->out_of_order++;
1545             list_add(&buf->list, rxlist);
1546             break;
1547         }
1548         /* keep processing buffers from the head of the list */
1549         status = irdma_ieq_process_buf(ieq, pfpdu, buf);
1550         if (status == -EINVAL) {
1551             pfpdu->mpa_crc_err = true;
1552             while (!list_empty(rxlist)) {
1553                 buf = irdma_puda_get_listbuf(rxlist);
1554                 irdma_puda_ret_bufpool(ieq, buf);
1555                 pfpdu->crc_err++;
1556                 ieq->crc_err++;
1557             }
1558             /* create CQP for AE */
1559             irdma_ieq_mpa_crc_ae(ieq->dev, qp);
1560         }
1561     } while (!status);
1562 }
1563 
1564 /**
1565  * irdma_ieq_create_ah - create an address handle for IEQ
1566  * @qp: qp pointer
1567  * @buf: buf received on IEQ used to create AH
1568  */
1569 static int irdma_ieq_create_ah(struct irdma_sc_qp *qp, struct irdma_puda_buf *buf)
1570 {
1571     struct irdma_ah_info ah_info = {};
1572 
1573     qp->pfpdu.ah_buf = buf;
1574     irdma_puda_ieq_get_ah_info(qp, &ah_info);
1575     return irdma_puda_create_ah(qp->vsi->dev, &ah_info, false,
1576                     IRDMA_PUDA_RSRC_TYPE_IEQ, qp,
1577                     &qp->pfpdu.ah);
1578 }
1579 
1580 /**
1581  * irdma_ieq_handle_exception - handle qp's exception
1582  * @ieq: ieq resource
1583  * @qp: qp receiving excpetion
1584  * @buf: receive buffer
1585  */
1586 static void irdma_ieq_handle_exception(struct irdma_puda_rsrc *ieq,
1587                        struct irdma_sc_qp *qp,
1588                        struct irdma_puda_buf *buf)
1589 {
1590     struct irdma_pfpdu *pfpdu = &qp->pfpdu;
1591     u32 *hw_host_ctx = (u32 *)qp->hw_host_ctx;
1592     u32 rcv_wnd = hw_host_ctx[23];
1593     /* first partial seq # in q2 */
1594     u32 fps = *(u32 *)(qp->q2_buf + Q2_FPSN_OFFSET);
1595     struct list_head *rxlist = &pfpdu->rxlist;
1596     unsigned long flags = 0;
1597     u8 hw_rev = qp->dev->hw_attrs.uk_attrs.hw_rev;
1598 
1599     print_hex_dump_debug("IEQ: IEQ RX BUFFER", DUMP_PREFIX_OFFSET, 16, 8,
1600                  buf->mem.va, buf->totallen, false);
1601 
1602     spin_lock_irqsave(&pfpdu->lock, flags);
1603     pfpdu->total_ieq_bufs++;
1604     if (pfpdu->mpa_crc_err) {
1605         pfpdu->crc_err++;
1606         goto error;
1607     }
1608     if (pfpdu->mode && fps != pfpdu->fps) {
1609         /* clean up qp as it is new partial sequence */
1610         irdma_ieq_cleanup_qp(ieq, qp);
1611         ibdev_dbg(to_ibdev(ieq->dev), "IEQ: restarting new partial\n");
1612         pfpdu->mode = false;
1613     }
1614 
1615     if (!pfpdu->mode) {
1616         print_hex_dump_debug("IEQ: Q2 BUFFER", DUMP_PREFIX_OFFSET, 16,
1617                      8, (u64 *)qp->q2_buf, 128, false);
1618         /* First_Partial_Sequence_Number check */
1619         pfpdu->rcv_nxt = fps;
1620         pfpdu->fps = fps;
1621         pfpdu->mode = true;
1622         pfpdu->max_fpdu_data = (buf->ipv4) ?
1623                        (ieq->vsi->mtu - IRDMA_MTU_TO_MSS_IPV4) :
1624                        (ieq->vsi->mtu - IRDMA_MTU_TO_MSS_IPV6);
1625         pfpdu->pmode_count++;
1626         ieq->pmode_count++;
1627         INIT_LIST_HEAD(rxlist);
1628         irdma_ieq_check_first_buf(buf, fps);
1629     }
1630 
1631     if (!(rcv_wnd >= (buf->seqnum - pfpdu->rcv_nxt))) {
1632         pfpdu->bad_seq_num++;
1633         ieq->bad_seq_num++;
1634         goto error;
1635     }
1636 
1637     if (!list_empty(rxlist)) {
1638         if (buf->seqnum != pfpdu->nextseqnum) {
1639             irdma_send_ieq_ack(qp);
1640             /* throw away out-of-order, duplicates*/
1641             goto error;
1642         }
1643     }
1644     /* Insert buf before head */
1645     list_add_tail(&buf->list, rxlist);
1646     pfpdu->nextseqnum = buf->seqnum + buf->datalen;
1647     pfpdu->lastrcv_buf = buf;
1648     if (hw_rev >= IRDMA_GEN_2 && !pfpdu->ah) {
1649         irdma_ieq_create_ah(qp, buf);
1650         if (!pfpdu->ah)
1651             goto error;
1652         goto exit;
1653     }
1654     if (hw_rev == IRDMA_GEN_1)
1655         irdma_ieq_process_fpdus(qp, ieq);
1656     else if (pfpdu->ah && pfpdu->ah->ah_info.ah_valid)
1657         irdma_ieq_process_fpdus(qp, ieq);
1658 exit:
1659     spin_unlock_irqrestore(&pfpdu->lock, flags);
1660 
1661     return;
1662 
1663 error:
1664     irdma_puda_ret_bufpool(ieq, buf);
1665     spin_unlock_irqrestore(&pfpdu->lock, flags);
1666 }
1667 
1668 /**
1669  * irdma_ieq_receive - received exception buffer
1670  * @vsi: VSI of device
1671  * @buf: exception buffer received
1672  */
1673 static void irdma_ieq_receive(struct irdma_sc_vsi *vsi,
1674                   struct irdma_puda_buf *buf)
1675 {
1676     struct irdma_puda_rsrc *ieq = vsi->ieq;
1677     struct irdma_sc_qp *qp = NULL;
1678     u32 wqe_idx = ieq->compl_rxwqe_idx;
1679 
1680     qp = irdma_ieq_get_qp(vsi->dev, buf);
1681     if (!qp) {
1682         ieq->stats_bad_qp_id++;
1683         irdma_puda_ret_bufpool(ieq, buf);
1684     } else {
1685         irdma_ieq_handle_exception(ieq, qp, buf);
1686     }
1687     /*
1688      * ieq->rx_wqe_idx is used by irdma_puda_replenish_rq()
1689      * on which wqe_idx to start replenish rq
1690      */
1691     if (!ieq->rxq_invalid_cnt)
1692         ieq->rx_wqe_idx = wqe_idx;
1693     ieq->rxq_invalid_cnt++;
1694 }
1695 
1696 /**
1697  * irdma_ieq_tx_compl - put back after sending completed exception buffer
1698  * @vsi: sc VSI struct
1699  * @sqwrid: pointer to puda buffer
1700  */
1701 static void irdma_ieq_tx_compl(struct irdma_sc_vsi *vsi, void *sqwrid)
1702 {
1703     struct irdma_puda_rsrc *ieq = vsi->ieq;
1704     struct irdma_puda_buf *buf = sqwrid;
1705 
1706     irdma_puda_ret_bufpool(ieq, buf);
1707 }
1708 
1709 /**
1710  * irdma_ieq_cleanup_qp - qp is being destroyed
1711  * @ieq: ieq resource
1712  * @qp: all pending fpdu buffers
1713  */
1714 void irdma_ieq_cleanup_qp(struct irdma_puda_rsrc *ieq, struct irdma_sc_qp *qp)
1715 {
1716     struct irdma_puda_buf *buf;
1717     struct irdma_pfpdu *pfpdu = &qp->pfpdu;
1718     struct list_head *rxlist = &pfpdu->rxlist;
1719 
1720     if (qp->pfpdu.ah) {
1721         irdma_puda_free_ah(ieq->dev, qp->pfpdu.ah);
1722         qp->pfpdu.ah = NULL;
1723         qp->pfpdu.ah_buf = NULL;
1724     }
1725 
1726     if (!pfpdu->mode)
1727         return;
1728 
1729     while (!list_empty(rxlist)) {
1730         buf = irdma_puda_get_listbuf(rxlist);
1731         irdma_puda_ret_bufpool(ieq, buf);
1732     }
1733 }
[ Source navigation ] [ Diff markup ] [ Identifier search ] [ general search ]

This page was automatically generated by the 2.1.0 LXR engine.

The LXR team
Valid CSS 2.1! Valid HTML 4.01!