Back to home page

OSCL-LXR
 
 

     

0001 // SPDX-License-Identifier: GPL-2.0 OR Linux-OpenIB
0002 /*
0003  * Copyright (c) 2020, Mellanox Technologies inc. All rights reserved.
0004  */
0005 
0006 #include <linux/gfp.h>
0007 #include <linux/mlx5/qp.h>
0008 #include <linux/mlx5/driver.h>
0009 #include "wr.h"
0010 #include "umr.h"
0011 
0012 static const u32 mlx5_ib_opcode[] = {
0013     [IB_WR_SEND]                = MLX5_OPCODE_SEND,
0014     [IB_WR_LSO]             = MLX5_OPCODE_LSO,
0015     [IB_WR_SEND_WITH_IMM]           = MLX5_OPCODE_SEND_IMM,
0016     [IB_WR_RDMA_WRITE]          = MLX5_OPCODE_RDMA_WRITE,
0017     [IB_WR_RDMA_WRITE_WITH_IMM]     = MLX5_OPCODE_RDMA_WRITE_IMM,
0018     [IB_WR_RDMA_READ]           = MLX5_OPCODE_RDMA_READ,
0019     [IB_WR_ATOMIC_CMP_AND_SWP]      = MLX5_OPCODE_ATOMIC_CS,
0020     [IB_WR_ATOMIC_FETCH_AND_ADD]        = MLX5_OPCODE_ATOMIC_FA,
0021     [IB_WR_SEND_WITH_INV]           = MLX5_OPCODE_SEND_INVAL,
0022     [IB_WR_LOCAL_INV]           = MLX5_OPCODE_UMR,
0023     [IB_WR_REG_MR]              = MLX5_OPCODE_UMR,
0024     [IB_WR_MASKED_ATOMIC_CMP_AND_SWP]   = MLX5_OPCODE_ATOMIC_MASKED_CS,
0025     [IB_WR_MASKED_ATOMIC_FETCH_AND_ADD] = MLX5_OPCODE_ATOMIC_MASKED_FA,
0026     [MLX5_IB_WR_UMR]            = MLX5_OPCODE_UMR,
0027 };
0028 
0029 int mlx5r_wq_overflow(struct mlx5_ib_wq *wq, int nreq, struct ib_cq *ib_cq)
0030 {
0031     struct mlx5_ib_cq *cq;
0032     unsigned int cur;
0033 
0034     cur = wq->head - wq->tail;
0035     if (likely(cur + nreq < wq->max_post))
0036         return 0;
0037 
0038     cq = to_mcq(ib_cq);
0039     spin_lock(&cq->lock);
0040     cur = wq->head - wq->tail;
0041     spin_unlock(&cq->lock);
0042 
0043     return cur + nreq >= wq->max_post;
0044 }
0045 
0046 static __always_inline void set_raddr_seg(struct mlx5_wqe_raddr_seg *rseg,
0047                       u64 remote_addr, u32 rkey)
0048 {
0049     rseg->raddr    = cpu_to_be64(remote_addr);
0050     rseg->rkey     = cpu_to_be32(rkey);
0051     rseg->reserved = 0;
0052 }
0053 
0054 static void set_eth_seg(const struct ib_send_wr *wr, struct mlx5_ib_qp *qp,
0055             void **seg, int *size, void **cur_edge)
0056 {
0057     struct mlx5_wqe_eth_seg *eseg = *seg;
0058 
0059     memset(eseg, 0, sizeof(struct mlx5_wqe_eth_seg));
0060 
0061     if (wr->send_flags & IB_SEND_IP_CSUM)
0062         eseg->cs_flags = MLX5_ETH_WQE_L3_CSUM |
0063                  MLX5_ETH_WQE_L4_CSUM;
0064 
0065     if (wr->opcode == IB_WR_LSO) {
0066         struct ib_ud_wr *ud_wr = container_of(wr, struct ib_ud_wr, wr);
0067         size_t left, copysz;
0068         void *pdata = ud_wr->header;
0069         size_t stride;
0070 
0071         left = ud_wr->hlen;
0072         eseg->mss = cpu_to_be16(ud_wr->mss);
0073         eseg->inline_hdr.sz = cpu_to_be16(left);
0074 
0075         /* mlx5r_memcpy_send_wqe should get a 16B align address. Hence,
0076          * we first copy up to the current edge and then, if needed,
0077          * continue to mlx5r_memcpy_send_wqe.
0078          */
0079         copysz = min_t(u64, *cur_edge - (void *)eseg->inline_hdr.start,
0080                    left);
0081         memcpy(eseg->inline_hdr.start, pdata, copysz);
0082         stride = ALIGN(sizeof(struct mlx5_wqe_eth_seg) -
0083                    sizeof(eseg->inline_hdr.start) + copysz, 16);
0084         *size += stride / 16;
0085         *seg += stride;
0086 
0087         if (copysz < left) {
0088             handle_post_send_edge(&qp->sq, seg, *size, cur_edge);
0089             left -= copysz;
0090             pdata += copysz;
0091             mlx5r_memcpy_send_wqe(&qp->sq, cur_edge, seg, size,
0092                           pdata, left);
0093         }
0094 
0095         return;
0096     }
0097 
0098     *seg += sizeof(struct mlx5_wqe_eth_seg);
0099     *size += sizeof(struct mlx5_wqe_eth_seg) / 16;
0100 }
0101 
0102 static void set_datagram_seg(struct mlx5_wqe_datagram_seg *dseg,
0103                  const struct ib_send_wr *wr)
0104 {
0105     memcpy(&dseg->av, &to_mah(ud_wr(wr)->ah)->av, sizeof(struct mlx5_av));
0106     dseg->av.dqp_dct =
0107         cpu_to_be32(ud_wr(wr)->remote_qpn | MLX5_EXTENDED_UD_AV);
0108     dseg->av.key.qkey.qkey = cpu_to_be32(ud_wr(wr)->remote_qkey);
0109 }
0110 
0111 static void set_data_ptr_seg(struct mlx5_wqe_data_seg *dseg, struct ib_sge *sg)
0112 {
0113     dseg->byte_count = cpu_to_be32(sg->length);
0114     dseg->lkey       = cpu_to_be32(sg->lkey);
0115     dseg->addr       = cpu_to_be64(sg->addr);
0116 }
0117 
0118 static __be64 frwr_mkey_mask(bool atomic)
0119 {
0120     u64 result;
0121 
0122     result = MLX5_MKEY_MASK_LEN     |
0123         MLX5_MKEY_MASK_PAGE_SIZE    |
0124         MLX5_MKEY_MASK_START_ADDR   |
0125         MLX5_MKEY_MASK_EN_RINVAL    |
0126         MLX5_MKEY_MASK_KEY      |
0127         MLX5_MKEY_MASK_LR       |
0128         MLX5_MKEY_MASK_LW       |
0129         MLX5_MKEY_MASK_RR       |
0130         MLX5_MKEY_MASK_RW       |
0131         MLX5_MKEY_MASK_SMALL_FENCE  |
0132         MLX5_MKEY_MASK_FREE;
0133 
0134     if (atomic)
0135         result |= MLX5_MKEY_MASK_A;
0136 
0137     return cpu_to_be64(result);
0138 }
0139 
0140 static __be64 sig_mkey_mask(void)
0141 {
0142     u64 result;
0143 
0144     result = MLX5_MKEY_MASK_LEN     |
0145         MLX5_MKEY_MASK_PAGE_SIZE    |
0146         MLX5_MKEY_MASK_START_ADDR   |
0147         MLX5_MKEY_MASK_EN_SIGERR    |
0148         MLX5_MKEY_MASK_EN_RINVAL    |
0149         MLX5_MKEY_MASK_KEY      |
0150         MLX5_MKEY_MASK_LR       |
0151         MLX5_MKEY_MASK_LW       |
0152         MLX5_MKEY_MASK_RR       |
0153         MLX5_MKEY_MASK_RW       |
0154         MLX5_MKEY_MASK_SMALL_FENCE  |
0155         MLX5_MKEY_MASK_FREE     |
0156         MLX5_MKEY_MASK_BSF_EN;
0157 
0158     return cpu_to_be64(result);
0159 }
0160 
0161 static void set_reg_umr_seg(struct mlx5_wqe_umr_ctrl_seg *umr,
0162                 struct mlx5_ib_mr *mr, u8 flags, bool atomic)
0163 {
0164     int size = (mr->mmkey.ndescs + mr->meta_ndescs) * mr->desc_size;
0165 
0166     memset(umr, 0, sizeof(*umr));
0167 
0168     umr->flags = flags;
0169     umr->xlt_octowords = cpu_to_be16(mlx5r_umr_get_xlt_octo(size));
0170     umr->mkey_mask = frwr_mkey_mask(atomic);
0171 }
0172 
0173 static void set_linv_umr_seg(struct mlx5_wqe_umr_ctrl_seg *umr)
0174 {
0175     memset(umr, 0, sizeof(*umr));
0176     umr->mkey_mask = cpu_to_be64(MLX5_MKEY_MASK_FREE);
0177     umr->flags = MLX5_UMR_INLINE;
0178 }
0179 
0180 static u8 get_umr_flags(int acc)
0181 {
0182     return (acc & IB_ACCESS_REMOTE_ATOMIC ? MLX5_PERM_ATOMIC       : 0) |
0183            (acc & IB_ACCESS_REMOTE_WRITE  ? MLX5_PERM_REMOTE_WRITE : 0) |
0184            (acc & IB_ACCESS_REMOTE_READ   ? MLX5_PERM_REMOTE_READ  : 0) |
0185            (acc & IB_ACCESS_LOCAL_WRITE   ? MLX5_PERM_LOCAL_WRITE  : 0) |
0186         MLX5_PERM_LOCAL_READ | MLX5_PERM_UMR_EN;
0187 }
0188 
0189 static void set_reg_mkey_seg(struct mlx5_mkey_seg *seg,
0190                  struct mlx5_ib_mr *mr,
0191                  u32 key, int access)
0192 {
0193     int ndescs = ALIGN(mr->mmkey.ndescs + mr->meta_ndescs, 8) >> 1;
0194 
0195     memset(seg, 0, sizeof(*seg));
0196 
0197     if (mr->access_mode == MLX5_MKC_ACCESS_MODE_MTT)
0198         seg->log2_page_size = ilog2(mr->ibmr.page_size);
0199     else if (mr->access_mode == MLX5_MKC_ACCESS_MODE_KLMS)
0200         /* KLMs take twice the size of MTTs */
0201         ndescs *= 2;
0202 
0203     seg->flags = get_umr_flags(access) | mr->access_mode;
0204     seg->qpn_mkey7_0 = cpu_to_be32((key & 0xff) | 0xffffff00);
0205     seg->flags_pd = cpu_to_be32(MLX5_MKEY_REMOTE_INVAL);
0206     seg->start_addr = cpu_to_be64(mr->ibmr.iova);
0207     seg->len = cpu_to_be64(mr->ibmr.length);
0208     seg->xlt_oct_size = cpu_to_be32(ndescs);
0209 }
0210 
0211 static void set_linv_mkey_seg(struct mlx5_mkey_seg *seg)
0212 {
0213     memset(seg, 0, sizeof(*seg));
0214     seg->status = MLX5_MKEY_STATUS_FREE;
0215 }
0216 
0217 static void set_reg_data_seg(struct mlx5_wqe_data_seg *dseg,
0218                  struct mlx5_ib_mr *mr,
0219                  struct mlx5_ib_pd *pd)
0220 {
0221     int bcount = mr->desc_size * (mr->mmkey.ndescs + mr->meta_ndescs);
0222 
0223     dseg->addr = cpu_to_be64(mr->desc_map);
0224     dseg->byte_count = cpu_to_be32(ALIGN(bcount, 64));
0225     dseg->lkey = cpu_to_be32(pd->ibpd.local_dma_lkey);
0226 }
0227 
0228 static __be32 send_ieth(const struct ib_send_wr *wr)
0229 {
0230     switch (wr->opcode) {
0231     case IB_WR_SEND_WITH_IMM:
0232     case IB_WR_RDMA_WRITE_WITH_IMM:
0233         return wr->ex.imm_data;
0234 
0235     case IB_WR_SEND_WITH_INV:
0236         return cpu_to_be32(wr->ex.invalidate_rkey);
0237 
0238     default:
0239         return 0;
0240     }
0241 }
0242 
0243 static u8 calc_sig(void *wqe, int size)
0244 {
0245     u8 *p = wqe;
0246     u8 res = 0;
0247     int i;
0248 
0249     for (i = 0; i < size; i++)
0250         res ^= p[i];
0251 
0252     return ~res;
0253 }
0254 
0255 static u8 wq_sig(void *wqe)
0256 {
0257     return calc_sig(wqe, (*((u8 *)wqe + 8) & 0x3f) << 4);
0258 }
0259 
0260 static int set_data_inl_seg(struct mlx5_ib_qp *qp, const struct ib_send_wr *wr,
0261                 void **wqe, int *wqe_sz, void **cur_edge)
0262 {
0263     struct mlx5_wqe_inline_seg *seg;
0264     size_t offset;
0265     int inl = 0;
0266     int i;
0267 
0268     seg = *wqe;
0269     *wqe += sizeof(*seg);
0270     offset = sizeof(*seg);
0271 
0272     for (i = 0; i < wr->num_sge; i++) {
0273         size_t len  = wr->sg_list[i].length;
0274         void *addr = (void *)(unsigned long)(wr->sg_list[i].addr);
0275 
0276         inl += len;
0277 
0278         if (unlikely(inl > qp->max_inline_data))
0279             return -ENOMEM;
0280 
0281         while (likely(len)) {
0282             size_t leftlen;
0283             size_t copysz;
0284 
0285             handle_post_send_edge(&qp->sq, wqe,
0286                           *wqe_sz + (offset >> 4),
0287                           cur_edge);
0288 
0289             leftlen = *cur_edge - *wqe;
0290             copysz = min_t(size_t, leftlen, len);
0291 
0292             memcpy(*wqe, addr, copysz);
0293             len -= copysz;
0294             addr += copysz;
0295             *wqe += copysz;
0296             offset += copysz;
0297         }
0298     }
0299 
0300     seg->byte_count = cpu_to_be32(inl | MLX5_INLINE_SEG);
0301 
0302     *wqe_sz +=  ALIGN(inl + sizeof(seg->byte_count), 16) / 16;
0303 
0304     return 0;
0305 }
0306 
0307 static u16 prot_field_size(enum ib_signature_type type)
0308 {
0309     switch (type) {
0310     case IB_SIG_TYPE_T10_DIF:
0311         return MLX5_DIF_SIZE;
0312     default:
0313         return 0;
0314     }
0315 }
0316 
0317 static u8 bs_selector(int block_size)
0318 {
0319     switch (block_size) {
0320     case 512:       return 0x1;
0321     case 520:       return 0x2;
0322     case 4096:      return 0x3;
0323     case 4160:      return 0x4;
0324     case 1073741824:    return 0x5;
0325     default:        return 0;
0326     }
0327 }
0328 
0329 static void mlx5_fill_inl_bsf(struct ib_sig_domain *domain,
0330                   struct mlx5_bsf_inl *inl)
0331 {
0332     /* Valid inline section and allow BSF refresh */
0333     inl->vld_refresh = cpu_to_be16(MLX5_BSF_INL_VALID |
0334                        MLX5_BSF_REFRESH_DIF);
0335     inl->dif_apptag = cpu_to_be16(domain->sig.dif.app_tag);
0336     inl->dif_reftag = cpu_to_be32(domain->sig.dif.ref_tag);
0337     /* repeating block */
0338     inl->rp_inv_seed = MLX5_BSF_REPEAT_BLOCK;
0339     inl->sig_type = domain->sig.dif.bg_type == IB_T10DIF_CRC ?
0340             MLX5_DIF_CRC : MLX5_DIF_IPCS;
0341 
0342     if (domain->sig.dif.ref_remap)
0343         inl->dif_inc_ref_guard_check |= MLX5_BSF_INC_REFTAG;
0344 
0345     if (domain->sig.dif.app_escape) {
0346         if (domain->sig.dif.ref_escape)
0347             inl->dif_inc_ref_guard_check |= MLX5_BSF_APPREF_ESCAPE;
0348         else
0349             inl->dif_inc_ref_guard_check |= MLX5_BSF_APPTAG_ESCAPE;
0350     }
0351 
0352     inl->dif_app_bitmask_check =
0353         cpu_to_be16(domain->sig.dif.apptag_check_mask);
0354 }
0355 
0356 static int mlx5_set_bsf(struct ib_mr *sig_mr,
0357             struct ib_sig_attrs *sig_attrs,
0358             struct mlx5_bsf *bsf, u32 data_size)
0359 {
0360     struct mlx5_core_sig_ctx *msig = to_mmr(sig_mr)->sig;
0361     struct mlx5_bsf_basic *basic = &bsf->basic;
0362     struct ib_sig_domain *mem = &sig_attrs->mem;
0363     struct ib_sig_domain *wire = &sig_attrs->wire;
0364 
0365     memset(bsf, 0, sizeof(*bsf));
0366 
0367     /* Basic + Extended + Inline */
0368     basic->bsf_size_sbs = 1 << 7;
0369     /* Input domain check byte mask */
0370     basic->check_byte_mask = sig_attrs->check_mask;
0371     basic->raw_data_size = cpu_to_be32(data_size);
0372 
0373     /* Memory domain */
0374     switch (sig_attrs->mem.sig_type) {
0375     case IB_SIG_TYPE_NONE:
0376         break;
0377     case IB_SIG_TYPE_T10_DIF:
0378         basic->mem.bs_selector = bs_selector(mem->sig.dif.pi_interval);
0379         basic->m_bfs_psv = cpu_to_be32(msig->psv_memory.psv_idx);
0380         mlx5_fill_inl_bsf(mem, &bsf->m_inl);
0381         break;
0382     default:
0383         return -EINVAL;
0384     }
0385 
0386     /* Wire domain */
0387     switch (sig_attrs->wire.sig_type) {
0388     case IB_SIG_TYPE_NONE:
0389         break;
0390     case IB_SIG_TYPE_T10_DIF:
0391         if (mem->sig.dif.pi_interval == wire->sig.dif.pi_interval &&
0392             mem->sig_type == wire->sig_type) {
0393             /* Same block structure */
0394             basic->bsf_size_sbs |= 1 << 4;
0395             if (mem->sig.dif.bg_type == wire->sig.dif.bg_type)
0396                 basic->wire.copy_byte_mask |= MLX5_CPY_GRD_MASK;
0397             if (mem->sig.dif.app_tag == wire->sig.dif.app_tag)
0398                 basic->wire.copy_byte_mask |= MLX5_CPY_APP_MASK;
0399             if (mem->sig.dif.ref_tag == wire->sig.dif.ref_tag)
0400                 basic->wire.copy_byte_mask |= MLX5_CPY_REF_MASK;
0401         } else
0402             basic->wire.bs_selector =
0403                 bs_selector(wire->sig.dif.pi_interval);
0404 
0405         basic->w_bfs_psv = cpu_to_be32(msig->psv_wire.psv_idx);
0406         mlx5_fill_inl_bsf(wire, &bsf->w_inl);
0407         break;
0408     default:
0409         return -EINVAL;
0410     }
0411 
0412     return 0;
0413 }
0414 
0415 
0416 static int set_sig_data_segment(const struct ib_send_wr *send_wr,
0417                 struct ib_mr *sig_mr,
0418                 struct ib_sig_attrs *sig_attrs,
0419                 struct mlx5_ib_qp *qp, void **seg, int *size,
0420                 void **cur_edge)
0421 {
0422     struct mlx5_bsf *bsf;
0423     u32 data_len;
0424     u32 data_key;
0425     u64 data_va;
0426     u32 prot_len = 0;
0427     u32 prot_key = 0;
0428     u64 prot_va = 0;
0429     bool prot = false;
0430     int ret;
0431     int wqe_size;
0432     struct mlx5_ib_mr *mr = to_mmr(sig_mr);
0433     struct mlx5_ib_mr *pi_mr = mr->pi_mr;
0434 
0435     data_len = pi_mr->data_length;
0436     data_key = pi_mr->ibmr.lkey;
0437     data_va = pi_mr->data_iova;
0438     if (pi_mr->meta_ndescs) {
0439         prot_len = pi_mr->meta_length;
0440         prot_key = pi_mr->ibmr.lkey;
0441         prot_va = pi_mr->pi_iova;
0442         prot = true;
0443     }
0444 
0445     if (!prot || (data_key == prot_key && data_va == prot_va &&
0446               data_len == prot_len)) {
0447         /**
0448          * Source domain doesn't contain signature information
0449          * or data and protection are interleaved in memory.
0450          * So need construct:
0451          *                  ------------------
0452          *                 |     data_klm     |
0453          *                  ------------------
0454          *                 |       BSF        |
0455          *                  ------------------
0456          **/
0457         struct mlx5_klm *data_klm = *seg;
0458 
0459         data_klm->bcount = cpu_to_be32(data_len);
0460         data_klm->key = cpu_to_be32(data_key);
0461         data_klm->va = cpu_to_be64(data_va);
0462         wqe_size = ALIGN(sizeof(*data_klm), 64);
0463     } else {
0464         /**
0465          * Source domain contains signature information
0466          * So need construct a strided block format:
0467          *               ---------------------------
0468          *              |     stride_block_ctrl     |
0469          *               ---------------------------
0470          *              |          data_klm         |
0471          *               ---------------------------
0472          *              |          prot_klm         |
0473          *               ---------------------------
0474          *              |             BSF           |
0475          *               ---------------------------
0476          **/
0477         struct mlx5_stride_block_ctrl_seg *sblock_ctrl;
0478         struct mlx5_stride_block_entry *data_sentry;
0479         struct mlx5_stride_block_entry *prot_sentry;
0480         u16 block_size = sig_attrs->mem.sig.dif.pi_interval;
0481         int prot_size;
0482 
0483         sblock_ctrl = *seg;
0484         data_sentry = (void *)sblock_ctrl + sizeof(*sblock_ctrl);
0485         prot_sentry = (void *)data_sentry + sizeof(*data_sentry);
0486 
0487         prot_size = prot_field_size(sig_attrs->mem.sig_type);
0488         if (!prot_size) {
0489             pr_err("Bad block size given: %u\n", block_size);
0490             return -EINVAL;
0491         }
0492         sblock_ctrl->bcount_per_cycle = cpu_to_be32(block_size +
0493                                 prot_size);
0494         sblock_ctrl->op = cpu_to_be32(MLX5_STRIDE_BLOCK_OP);
0495         sblock_ctrl->repeat_count = cpu_to_be32(data_len / block_size);
0496         sblock_ctrl->num_entries = cpu_to_be16(2);
0497 
0498         data_sentry->bcount = cpu_to_be16(block_size);
0499         data_sentry->key = cpu_to_be32(data_key);
0500         data_sentry->va = cpu_to_be64(data_va);
0501         data_sentry->stride = cpu_to_be16(block_size);
0502 
0503         prot_sentry->bcount = cpu_to_be16(prot_size);
0504         prot_sentry->key = cpu_to_be32(prot_key);
0505         prot_sentry->va = cpu_to_be64(prot_va);
0506         prot_sentry->stride = cpu_to_be16(prot_size);
0507 
0508         wqe_size = ALIGN(sizeof(*sblock_ctrl) + sizeof(*data_sentry) +
0509                  sizeof(*prot_sentry), 64);
0510     }
0511 
0512     *seg += wqe_size;
0513     *size += wqe_size / 16;
0514     handle_post_send_edge(&qp->sq, seg, *size, cur_edge);
0515 
0516     bsf = *seg;
0517     ret = mlx5_set_bsf(sig_mr, sig_attrs, bsf, data_len);
0518     if (ret)
0519         return -EINVAL;
0520 
0521     *seg += sizeof(*bsf);
0522     *size += sizeof(*bsf) / 16;
0523     handle_post_send_edge(&qp->sq, seg, *size, cur_edge);
0524 
0525     return 0;
0526 }
0527 
0528 static void set_sig_mkey_segment(struct mlx5_mkey_seg *seg,
0529                  struct ib_mr *sig_mr, int access_flags,
0530                  u32 size, u32 length, u32 pdn)
0531 {
0532     u32 sig_key = sig_mr->rkey;
0533     u8 sigerr = to_mmr(sig_mr)->sig->sigerr_count & 1;
0534 
0535     memset(seg, 0, sizeof(*seg));
0536 
0537     seg->flags = get_umr_flags(access_flags) | MLX5_MKC_ACCESS_MODE_KLMS;
0538     seg->qpn_mkey7_0 = cpu_to_be32((sig_key & 0xff) | 0xffffff00);
0539     seg->flags_pd = cpu_to_be32(MLX5_MKEY_REMOTE_INVAL | sigerr << 26 |
0540                     MLX5_MKEY_BSF_EN | pdn);
0541     seg->len = cpu_to_be64(length);
0542     seg->xlt_oct_size = cpu_to_be32(mlx5r_umr_get_xlt_octo(size));
0543     seg->bsfs_octo_size = cpu_to_be32(MLX5_MKEY_BSF_OCTO_SIZE);
0544 }
0545 
0546 static void set_sig_umr_segment(struct mlx5_wqe_umr_ctrl_seg *umr,
0547                 u32 size)
0548 {
0549     memset(umr, 0, sizeof(*umr));
0550 
0551     umr->flags = MLX5_FLAGS_INLINE | MLX5_FLAGS_CHECK_FREE;
0552     umr->xlt_octowords = cpu_to_be16(mlx5r_umr_get_xlt_octo(size));
0553     umr->bsf_octowords = cpu_to_be16(MLX5_MKEY_BSF_OCTO_SIZE);
0554     umr->mkey_mask = sig_mkey_mask();
0555 }
0556 
0557 static int set_pi_umr_wr(const struct ib_send_wr *send_wr,
0558              struct mlx5_ib_qp *qp, void **seg, int *size,
0559              void **cur_edge)
0560 {
0561     const struct ib_reg_wr *wr = reg_wr(send_wr);
0562     struct mlx5_ib_mr *sig_mr = to_mmr(wr->mr);
0563     struct mlx5_ib_mr *pi_mr = sig_mr->pi_mr;
0564     struct ib_sig_attrs *sig_attrs = sig_mr->ibmr.sig_attrs;
0565     u32 pdn = to_mpd(qp->ibqp.pd)->pdn;
0566     u32 xlt_size;
0567     int region_len, ret;
0568 
0569     if (unlikely(send_wr->num_sge != 0) ||
0570         unlikely(wr->access & IB_ACCESS_REMOTE_ATOMIC) ||
0571         unlikely(!sig_mr->sig) || unlikely(!qp->ibqp.integrity_en) ||
0572         unlikely(!sig_mr->sig->sig_status_checked))
0573         return -EINVAL;
0574 
0575     /* length of the protected region, data + protection */
0576     region_len = pi_mr->ibmr.length;
0577 
0578     /**
0579      * KLM octoword size - if protection was provided
0580      * then we use strided block format (3 octowords),
0581      * else we use single KLM (1 octoword)
0582      **/
0583     if (sig_attrs->mem.sig_type != IB_SIG_TYPE_NONE)
0584         xlt_size = 0x30;
0585     else
0586         xlt_size = sizeof(struct mlx5_klm);
0587 
0588     set_sig_umr_segment(*seg, xlt_size);
0589     *seg += sizeof(struct mlx5_wqe_umr_ctrl_seg);
0590     *size += sizeof(struct mlx5_wqe_umr_ctrl_seg) / 16;
0591     handle_post_send_edge(&qp->sq, seg, *size, cur_edge);
0592 
0593     set_sig_mkey_segment(*seg, wr->mr, wr->access, xlt_size, region_len,
0594                  pdn);
0595     *seg += sizeof(struct mlx5_mkey_seg);
0596     *size += sizeof(struct mlx5_mkey_seg) / 16;
0597     handle_post_send_edge(&qp->sq, seg, *size, cur_edge);
0598 
0599     ret = set_sig_data_segment(send_wr, wr->mr, sig_attrs, qp, seg, size,
0600                    cur_edge);
0601     if (ret)
0602         return ret;
0603 
0604     sig_mr->sig->sig_status_checked = false;
0605     return 0;
0606 }
0607 
0608 static int set_psv_wr(struct ib_sig_domain *domain,
0609               u32 psv_idx, void **seg, int *size)
0610 {
0611     struct mlx5_seg_set_psv *psv_seg = *seg;
0612 
0613     memset(psv_seg, 0, sizeof(*psv_seg));
0614     psv_seg->psv_num = cpu_to_be32(psv_idx);
0615     switch (domain->sig_type) {
0616     case IB_SIG_TYPE_NONE:
0617         break;
0618     case IB_SIG_TYPE_T10_DIF:
0619         psv_seg->transient_sig = cpu_to_be32(domain->sig.dif.bg << 16 |
0620                              domain->sig.dif.app_tag);
0621         psv_seg->ref_tag = cpu_to_be32(domain->sig.dif.ref_tag);
0622         break;
0623     default:
0624         pr_err("Bad signature type (%d) is given.\n",
0625                domain->sig_type);
0626         return -EINVAL;
0627     }
0628 
0629     *seg += sizeof(*psv_seg);
0630     *size += sizeof(*psv_seg) / 16;
0631 
0632     return 0;
0633 }
0634 
0635 static int set_reg_wr(struct mlx5_ib_qp *qp,
0636               const struct ib_reg_wr *wr,
0637               void **seg, int *size, void **cur_edge,
0638               bool check_not_free)
0639 {
0640     struct mlx5_ib_mr *mr = to_mmr(wr->mr);
0641     struct mlx5_ib_pd *pd = to_mpd(qp->ibqp.pd);
0642     struct mlx5_ib_dev *dev = to_mdev(pd->ibpd.device);
0643     int mr_list_size = (mr->mmkey.ndescs + mr->meta_ndescs) * mr->desc_size;
0644     bool umr_inline = mr_list_size <= MLX5_IB_SQ_UMR_INLINE_THRESHOLD;
0645     bool atomic = wr->access & IB_ACCESS_REMOTE_ATOMIC;
0646     u8 flags = 0;
0647 
0648     /* Matches access in mlx5_set_umr_free_mkey().
0649      * Relaxed Ordering is set implicitly in mlx5_set_umr_free_mkey() and
0650      * kernel ULPs are not aware of it, so we don't set it here.
0651      */
0652     if (!mlx5r_umr_can_reconfig(dev, 0, wr->access)) {
0653         mlx5_ib_warn(
0654             to_mdev(qp->ibqp.device),
0655             "Fast update for MR access flags is not possible\n");
0656         return -EINVAL;
0657     }
0658 
0659     if (unlikely(wr->wr.send_flags & IB_SEND_INLINE)) {
0660         mlx5_ib_warn(to_mdev(qp->ibqp.device),
0661                  "Invalid IB_SEND_INLINE send flag\n");
0662         return -EINVAL;
0663     }
0664 
0665     if (check_not_free)
0666         flags |= MLX5_UMR_CHECK_NOT_FREE;
0667     if (umr_inline)
0668         flags |= MLX5_UMR_INLINE;
0669 
0670     set_reg_umr_seg(*seg, mr, flags, atomic);
0671     *seg += sizeof(struct mlx5_wqe_umr_ctrl_seg);
0672     *size += sizeof(struct mlx5_wqe_umr_ctrl_seg) / 16;
0673     handle_post_send_edge(&qp->sq, seg, *size, cur_edge);
0674 
0675     set_reg_mkey_seg(*seg, mr, wr->key, wr->access);
0676     *seg += sizeof(struct mlx5_mkey_seg);
0677     *size += sizeof(struct mlx5_mkey_seg) / 16;
0678     handle_post_send_edge(&qp->sq, seg, *size, cur_edge);
0679 
0680     if (umr_inline) {
0681         mlx5r_memcpy_send_wqe(&qp->sq, cur_edge, seg, size, mr->descs,
0682                       mr_list_size);
0683         *size = ALIGN(*size, MLX5_SEND_WQE_BB >> 4);
0684     } else {
0685         set_reg_data_seg(*seg, mr, pd);
0686         *seg += sizeof(struct mlx5_wqe_data_seg);
0687         *size += (sizeof(struct mlx5_wqe_data_seg) / 16);
0688     }
0689     return 0;
0690 }
0691 
0692 static void set_linv_wr(struct mlx5_ib_qp *qp, void **seg, int *size,
0693             void **cur_edge)
0694 {
0695     set_linv_umr_seg(*seg);
0696     *seg += sizeof(struct mlx5_wqe_umr_ctrl_seg);
0697     *size += sizeof(struct mlx5_wqe_umr_ctrl_seg) / 16;
0698     handle_post_send_edge(&qp->sq, seg, *size, cur_edge);
0699     set_linv_mkey_seg(*seg);
0700     *seg += sizeof(struct mlx5_mkey_seg);
0701     *size += sizeof(struct mlx5_mkey_seg) / 16;
0702     handle_post_send_edge(&qp->sq, seg, *size, cur_edge);
0703 }
0704 
0705 static void dump_wqe(struct mlx5_ib_qp *qp, u32 idx, int size_16)
0706 {
0707     __be32 *p = NULL;
0708     int i, j;
0709 
0710     pr_debug("dump WQE index %u:\n", idx);
0711     for (i = 0, j = 0; i < size_16 * 4; i += 4, j += 4) {
0712         if ((i & 0xf) == 0) {
0713             p = mlx5_frag_buf_get_wqe(&qp->sq.fbc, idx);
0714             pr_debug("WQBB at %p:\n", (void *)p);
0715             j = 0;
0716             idx = (idx + 1) & (qp->sq.wqe_cnt - 1);
0717         }
0718         pr_debug("%08x %08x %08x %08x\n", be32_to_cpu(p[j]),
0719              be32_to_cpu(p[j + 1]), be32_to_cpu(p[j + 2]),
0720              be32_to_cpu(p[j + 3]));
0721     }
0722 }
0723 
0724 int mlx5r_begin_wqe(struct mlx5_ib_qp *qp, void **seg,
0725             struct mlx5_wqe_ctrl_seg **ctrl, unsigned int *idx,
0726             int *size, void **cur_edge, int nreq, __be32 general_id,
0727             bool send_signaled, bool solicited)
0728 {
0729     if (unlikely(mlx5r_wq_overflow(&qp->sq, nreq, qp->ibqp.send_cq)))
0730         return -ENOMEM;
0731 
0732     *idx = qp->sq.cur_post & (qp->sq.wqe_cnt - 1);
0733     *seg = mlx5_frag_buf_get_wqe(&qp->sq.fbc, *idx);
0734     *ctrl = *seg;
0735     *(uint32_t *)(*seg + 8) = 0;
0736     (*ctrl)->general_id = general_id;
0737     (*ctrl)->fm_ce_se = qp->sq_signal_bits |
0738                 (send_signaled ? MLX5_WQE_CTRL_CQ_UPDATE : 0) |
0739                 (solicited ? MLX5_WQE_CTRL_SOLICITED : 0);
0740 
0741     *seg += sizeof(**ctrl);
0742     *size = sizeof(**ctrl) / 16;
0743     *cur_edge = qp->sq.cur_edge;
0744 
0745     return 0;
0746 }
0747 
0748 static int begin_wqe(struct mlx5_ib_qp *qp, void **seg,
0749              struct mlx5_wqe_ctrl_seg **ctrl,
0750              const struct ib_send_wr *wr, unsigned int *idx, int *size,
0751              void **cur_edge, int nreq)
0752 {
0753     return mlx5r_begin_wqe(qp, seg, ctrl, idx, size, cur_edge, nreq,
0754                    send_ieth(wr), wr->send_flags & IB_SEND_SIGNALED,
0755                    wr->send_flags & IB_SEND_SOLICITED);
0756 }
0757 
0758 void mlx5r_finish_wqe(struct mlx5_ib_qp *qp, struct mlx5_wqe_ctrl_seg *ctrl,
0759               void *seg, u8 size, void *cur_edge, unsigned int idx,
0760               u64 wr_id, int nreq, u8 fence, u32 mlx5_opcode)
0761 {
0762     u8 opmod = 0;
0763 
0764     ctrl->opmod_idx_opcode = cpu_to_be32(((u32)(qp->sq.cur_post) << 8) |
0765                          mlx5_opcode | ((u32)opmod << 24));
0766     ctrl->qpn_ds = cpu_to_be32(size | (qp->trans_qp.base.mqp.qpn << 8));
0767     ctrl->fm_ce_se |= fence;
0768     if (unlikely(qp->flags_en & MLX5_QP_FLAG_SIGNATURE))
0769         ctrl->signature = wq_sig(ctrl);
0770 
0771     qp->sq.wrid[idx] = wr_id;
0772     qp->sq.w_list[idx].opcode = mlx5_opcode;
0773     qp->sq.wqe_head[idx] = qp->sq.head + nreq;
0774     qp->sq.cur_post += DIV_ROUND_UP(size * 16, MLX5_SEND_WQE_BB);
0775     qp->sq.w_list[idx].next = qp->sq.cur_post;
0776 
0777     /* We save the edge which was possibly updated during the WQE
0778      * construction, into SQ's cache.
0779      */
0780     seg = PTR_ALIGN(seg, MLX5_SEND_WQE_BB);
0781     qp->sq.cur_edge = (unlikely(seg == cur_edge)) ?
0782               get_sq_edge(&qp->sq, qp->sq.cur_post &
0783                       (qp->sq.wqe_cnt - 1)) :
0784               cur_edge;
0785 }
0786 
0787 static void handle_rdma_op(const struct ib_send_wr *wr, void **seg, int *size)
0788 {
0789     set_raddr_seg(*seg, rdma_wr(wr)->remote_addr, rdma_wr(wr)->rkey);
0790     *seg += sizeof(struct mlx5_wqe_raddr_seg);
0791     *size += sizeof(struct mlx5_wqe_raddr_seg) / 16;
0792 }
0793 
0794 static void handle_local_inv(struct mlx5_ib_qp *qp, const struct ib_send_wr *wr,
0795                  struct mlx5_wqe_ctrl_seg **ctrl, void **seg,
0796                  int *size, void **cur_edge, unsigned int idx)
0797 {
0798     qp->sq.wr_data[idx] = IB_WR_LOCAL_INV;
0799     (*ctrl)->imm = cpu_to_be32(wr->ex.invalidate_rkey);
0800     set_linv_wr(qp, seg, size, cur_edge);
0801 }
0802 
0803 static int handle_reg_mr(struct mlx5_ib_qp *qp, const struct ib_send_wr *wr,
0804              struct mlx5_wqe_ctrl_seg **ctrl, void **seg, int *size,
0805              void **cur_edge, unsigned int idx)
0806 {
0807     qp->sq.wr_data[idx] = IB_WR_REG_MR;
0808     (*ctrl)->imm = cpu_to_be32(reg_wr(wr)->key);
0809     return set_reg_wr(qp, reg_wr(wr), seg, size, cur_edge, true);
0810 }
0811 
0812 static int handle_psv(struct mlx5_ib_dev *dev, struct mlx5_ib_qp *qp,
0813               const struct ib_send_wr *wr,
0814               struct mlx5_wqe_ctrl_seg **ctrl, void **seg, int *size,
0815               void **cur_edge, unsigned int *idx, int nreq,
0816               struct ib_sig_domain *domain, u32 psv_index,
0817               u8 next_fence)
0818 {
0819     int err;
0820 
0821     /*
0822      * SET_PSV WQEs are not signaled and solicited on error.
0823      */
0824     err = mlx5r_begin_wqe(qp, seg, ctrl, idx, size, cur_edge, nreq,
0825                   send_ieth(wr), false, true);
0826     if (unlikely(err)) {
0827         mlx5_ib_warn(dev, "\n");
0828         err = -ENOMEM;
0829         goto out;
0830     }
0831     err = set_psv_wr(domain, psv_index, seg, size);
0832     if (unlikely(err)) {
0833         mlx5_ib_warn(dev, "\n");
0834         goto out;
0835     }
0836     mlx5r_finish_wqe(qp, *ctrl, *seg, *size, *cur_edge, *idx, wr->wr_id,
0837              nreq, next_fence, MLX5_OPCODE_SET_PSV);
0838 
0839 out:
0840     return err;
0841 }
0842 
0843 static int handle_reg_mr_integrity(struct mlx5_ib_dev *dev,
0844                    struct mlx5_ib_qp *qp,
0845                    const struct ib_send_wr *wr,
0846                    struct mlx5_wqe_ctrl_seg **ctrl, void **seg,
0847                    int *size, void **cur_edge,
0848                    unsigned int *idx, int nreq, u8 fence,
0849                    u8 next_fence)
0850 {
0851     struct mlx5_ib_mr *mr;
0852     struct mlx5_ib_mr *pi_mr;
0853     struct mlx5_ib_mr pa_pi_mr;
0854     struct ib_sig_attrs *sig_attrs;
0855     struct ib_reg_wr reg_pi_wr;
0856     int err;
0857 
0858     qp->sq.wr_data[*idx] = IB_WR_REG_MR_INTEGRITY;
0859 
0860     mr = to_mmr(reg_wr(wr)->mr);
0861     pi_mr = mr->pi_mr;
0862 
0863     if (pi_mr) {
0864         memset(&reg_pi_wr, 0,
0865                sizeof(struct ib_reg_wr));
0866 
0867         reg_pi_wr.mr = &pi_mr->ibmr;
0868         reg_pi_wr.access = reg_wr(wr)->access;
0869         reg_pi_wr.key = pi_mr->ibmr.rkey;
0870 
0871         (*ctrl)->imm = cpu_to_be32(reg_pi_wr.key);
0872         /* UMR for data + prot registration */
0873         err = set_reg_wr(qp, &reg_pi_wr, seg, size, cur_edge, false);
0874         if (unlikely(err))
0875             goto out;
0876 
0877         mlx5r_finish_wqe(qp, *ctrl, *seg, *size, *cur_edge, *idx,
0878                  wr->wr_id, nreq, fence, MLX5_OPCODE_UMR);
0879 
0880         err = begin_wqe(qp, seg, ctrl, wr, idx, size, cur_edge, nreq);
0881         if (unlikely(err)) {
0882             mlx5_ib_warn(dev, "\n");
0883             err = -ENOMEM;
0884             goto out;
0885         }
0886     } else {
0887         memset(&pa_pi_mr, 0, sizeof(struct mlx5_ib_mr));
0888         /* No UMR, use local_dma_lkey */
0889         pa_pi_mr.ibmr.lkey = mr->ibmr.pd->local_dma_lkey;
0890         pa_pi_mr.mmkey.ndescs = mr->mmkey.ndescs;
0891         pa_pi_mr.data_length = mr->data_length;
0892         pa_pi_mr.data_iova = mr->data_iova;
0893         if (mr->meta_ndescs) {
0894             pa_pi_mr.meta_ndescs = mr->meta_ndescs;
0895             pa_pi_mr.meta_length = mr->meta_length;
0896             pa_pi_mr.pi_iova = mr->pi_iova;
0897         }
0898 
0899         pa_pi_mr.ibmr.length = mr->ibmr.length;
0900         mr->pi_mr = &pa_pi_mr;
0901     }
0902     (*ctrl)->imm = cpu_to_be32(mr->ibmr.rkey);
0903     /* UMR for sig MR */
0904     err = set_pi_umr_wr(wr, qp, seg, size, cur_edge);
0905     if (unlikely(err)) {
0906         mlx5_ib_warn(dev, "\n");
0907         goto out;
0908     }
0909     mlx5r_finish_wqe(qp, *ctrl, *seg, *size, *cur_edge, *idx, wr->wr_id,
0910              nreq, fence, MLX5_OPCODE_UMR);
0911 
0912     sig_attrs = mr->ibmr.sig_attrs;
0913     err = handle_psv(dev, qp, wr, ctrl, seg, size, cur_edge, idx, nreq,
0914              &sig_attrs->mem, mr->sig->psv_memory.psv_idx,
0915              next_fence);
0916     if (unlikely(err))
0917         goto out;
0918 
0919     err = handle_psv(dev, qp, wr, ctrl, seg, size, cur_edge, idx, nreq,
0920              &sig_attrs->wire, mr->sig->psv_wire.psv_idx,
0921              next_fence);
0922     if (unlikely(err))
0923         goto out;
0924 
0925     qp->next_fence = MLX5_FENCE_MODE_INITIATOR_SMALL;
0926 
0927 out:
0928     return err;
0929 }
0930 
0931 static int handle_qpt_rc(struct mlx5_ib_dev *dev, struct mlx5_ib_qp *qp,
0932              const struct ib_send_wr *wr,
0933              struct mlx5_wqe_ctrl_seg **ctrl, void **seg, int *size,
0934              void **cur_edge, unsigned int *idx, int nreq, u8 fence,
0935              u8 next_fence, int *num_sge)
0936 {
0937     int err = 0;
0938 
0939     switch (wr->opcode) {
0940     case IB_WR_RDMA_READ:
0941     case IB_WR_RDMA_WRITE:
0942     case IB_WR_RDMA_WRITE_WITH_IMM:
0943         handle_rdma_op(wr, seg, size);
0944         break;
0945 
0946     case IB_WR_ATOMIC_CMP_AND_SWP:
0947     case IB_WR_ATOMIC_FETCH_AND_ADD:
0948     case IB_WR_MASKED_ATOMIC_CMP_AND_SWP:
0949         mlx5_ib_warn(dev, "Atomic operations are not supported yet\n");
0950         err = -EOPNOTSUPP;
0951         goto out;
0952 
0953     case IB_WR_LOCAL_INV:
0954         handle_local_inv(qp, wr, ctrl, seg, size, cur_edge, *idx);
0955         *num_sge = 0;
0956         break;
0957 
0958     case IB_WR_REG_MR:
0959         err = handle_reg_mr(qp, wr, ctrl, seg, size, cur_edge, *idx);
0960         if (unlikely(err))
0961             goto out;
0962         *num_sge = 0;
0963         break;
0964 
0965     case IB_WR_REG_MR_INTEGRITY:
0966         err = handle_reg_mr_integrity(dev, qp, wr, ctrl, seg, size,
0967                           cur_edge, idx, nreq, fence,
0968                           next_fence);
0969         if (unlikely(err))
0970             goto out;
0971         *num_sge = 0;
0972         break;
0973 
0974     default:
0975         break;
0976     }
0977 
0978 out:
0979     return err;
0980 }
0981 
0982 static void handle_qpt_uc(const struct ib_send_wr *wr, void **seg, int *size)
0983 {
0984     switch (wr->opcode) {
0985     case IB_WR_RDMA_WRITE:
0986     case IB_WR_RDMA_WRITE_WITH_IMM:
0987         handle_rdma_op(wr, seg, size);
0988         break;
0989     default:
0990         break;
0991     }
0992 }
0993 
0994 static void handle_qpt_hw_gsi(struct mlx5_ib_qp *qp,
0995                   const struct ib_send_wr *wr, void **seg,
0996                   int *size, void **cur_edge)
0997 {
0998     set_datagram_seg(*seg, wr);
0999     *seg += sizeof(struct mlx5_wqe_datagram_seg);
1000     *size += sizeof(struct mlx5_wqe_datagram_seg) / 16;
1001     handle_post_send_edge(&qp->sq, seg, *size, cur_edge);
1002 }
1003 
1004 static void handle_qpt_ud(struct mlx5_ib_qp *qp, const struct ib_send_wr *wr,
1005               void **seg, int *size, void **cur_edge)
1006 {
1007     set_datagram_seg(*seg, wr);
1008     *seg += sizeof(struct mlx5_wqe_datagram_seg);
1009     *size += sizeof(struct mlx5_wqe_datagram_seg) / 16;
1010     handle_post_send_edge(&qp->sq, seg, *size, cur_edge);
1011 
1012     /* handle qp that supports ud offload */
1013     if (qp->flags & IB_QP_CREATE_IPOIB_UD_LSO) {
1014         struct mlx5_wqe_eth_pad *pad;
1015 
1016         pad = *seg;
1017         memset(pad, 0, sizeof(struct mlx5_wqe_eth_pad));
1018         *seg += sizeof(struct mlx5_wqe_eth_pad);
1019         *size += sizeof(struct mlx5_wqe_eth_pad) / 16;
1020         set_eth_seg(wr, qp, seg, size, cur_edge);
1021         handle_post_send_edge(&qp->sq, seg, *size, cur_edge);
1022     }
1023 }
1024 
1025 void mlx5r_ring_db(struct mlx5_ib_qp *qp, unsigned int nreq,
1026            struct mlx5_wqe_ctrl_seg *ctrl)
1027 {
1028     struct mlx5_bf *bf = &qp->bf;
1029 
1030     qp->sq.head += nreq;
1031 
1032     /* Make sure that descriptors are written before
1033      * updating doorbell record and ringing the doorbell
1034      */
1035     wmb();
1036 
1037     qp->db.db[MLX5_SND_DBR] = cpu_to_be32(qp->sq.cur_post);
1038 
1039     /* Make sure doorbell record is visible to the HCA before
1040      * we hit doorbell.
1041      */
1042     wmb();
1043 
1044     mlx5_write64((__be32 *)ctrl, bf->bfreg->map + bf->offset);
1045     /* Make sure doorbells don't leak out of SQ spinlock
1046      * and reach the HCA out of order.
1047      */
1048     bf->offset ^= bf->buf_size;
1049 }
1050 
1051 int mlx5_ib_post_send(struct ib_qp *ibqp, const struct ib_send_wr *wr,
1052               const struct ib_send_wr **bad_wr, bool drain)
1053 {
1054     struct mlx5_wqe_ctrl_seg *ctrl = NULL;  /* compiler warning */
1055     struct mlx5_ib_dev *dev = to_mdev(ibqp->device);
1056     struct mlx5_core_dev *mdev = dev->mdev;
1057     struct mlx5_ib_qp *qp = to_mqp(ibqp);
1058     struct mlx5_wqe_xrc_seg *xrc;
1059     void *cur_edge;
1060     int size;
1061     unsigned long flags;
1062     unsigned int idx;
1063     int err = 0;
1064     int num_sge;
1065     void *seg;
1066     int nreq;
1067     int i;
1068     u8 next_fence = 0;
1069     u8 fence;
1070 
1071     if (unlikely(mdev->state == MLX5_DEVICE_STATE_INTERNAL_ERROR &&
1072              !drain)) {
1073         *bad_wr = wr;
1074         return -EIO;
1075     }
1076 
1077     if (qp->type == IB_QPT_GSI)
1078         return mlx5_ib_gsi_post_send(ibqp, wr, bad_wr);
1079 
1080     spin_lock_irqsave(&qp->sq.lock, flags);
1081 
1082     for (nreq = 0; wr; nreq++, wr = wr->next) {
1083         if (unlikely(wr->opcode >= ARRAY_SIZE(mlx5_ib_opcode))) {
1084             mlx5_ib_warn(dev, "\n");
1085             err = -EINVAL;
1086             *bad_wr = wr;
1087             goto out;
1088         }
1089 
1090         num_sge = wr->num_sge;
1091         if (unlikely(num_sge > qp->sq.max_gs)) {
1092             mlx5_ib_warn(dev, "\n");
1093             err = -EINVAL;
1094             *bad_wr = wr;
1095             goto out;
1096         }
1097 
1098         err = begin_wqe(qp, &seg, &ctrl, wr, &idx, &size, &cur_edge,
1099                 nreq);
1100         if (err) {
1101             mlx5_ib_warn(dev, "\n");
1102             err = -ENOMEM;
1103             *bad_wr = wr;
1104             goto out;
1105         }
1106 
1107         if (wr->opcode == IB_WR_REG_MR ||
1108             wr->opcode == IB_WR_REG_MR_INTEGRITY) {
1109             fence = dev->umr_fence;
1110             next_fence = MLX5_FENCE_MODE_INITIATOR_SMALL;
1111         } else  {
1112             if (wr->send_flags & IB_SEND_FENCE) {
1113                 if (qp->next_fence)
1114                     fence = MLX5_FENCE_MODE_SMALL_AND_FENCE;
1115                 else
1116                     fence = MLX5_FENCE_MODE_FENCE;
1117             } else {
1118                 fence = qp->next_fence;
1119             }
1120         }
1121 
1122         switch (qp->type) {
1123         case IB_QPT_XRC_INI:
1124             xrc = seg;
1125             seg += sizeof(*xrc);
1126             size += sizeof(*xrc) / 16;
1127             fallthrough;
1128         case IB_QPT_RC:
1129             err = handle_qpt_rc(dev, qp, wr, &ctrl, &seg, &size,
1130                         &cur_edge, &idx, nreq, fence,
1131                         next_fence, &num_sge);
1132             if (unlikely(err)) {
1133                 *bad_wr = wr;
1134                 goto out;
1135             } else if (wr->opcode == IB_WR_REG_MR_INTEGRITY) {
1136                 goto skip_psv;
1137             }
1138             break;
1139 
1140         case IB_QPT_UC:
1141             handle_qpt_uc(wr, &seg, &size);
1142             break;
1143         case IB_QPT_SMI:
1144             if (unlikely(!dev->port_caps[qp->port - 1].has_smi)) {
1145                 mlx5_ib_warn(dev, "Send SMP MADs is not allowed\n");
1146                 err = -EPERM;
1147                 *bad_wr = wr;
1148                 goto out;
1149             }
1150             fallthrough;
1151         case MLX5_IB_QPT_HW_GSI:
1152             handle_qpt_hw_gsi(qp, wr, &seg, &size, &cur_edge);
1153             break;
1154         case IB_QPT_UD:
1155             handle_qpt_ud(qp, wr, &seg, &size, &cur_edge);
1156             break;
1157 
1158         default:
1159             break;
1160         }
1161 
1162         if (wr->send_flags & IB_SEND_INLINE && num_sge) {
1163             err = set_data_inl_seg(qp, wr, &seg, &size, &cur_edge);
1164             if (unlikely(err)) {
1165                 mlx5_ib_warn(dev, "\n");
1166                 *bad_wr = wr;
1167                 goto out;
1168             }
1169         } else {
1170             for (i = 0; i < num_sge; i++) {
1171                 handle_post_send_edge(&qp->sq, &seg, size,
1172                               &cur_edge);
1173                 if (unlikely(!wr->sg_list[i].length))
1174                     continue;
1175 
1176                 set_data_ptr_seg(
1177                     (struct mlx5_wqe_data_seg *)seg,
1178                     wr->sg_list + i);
1179                 size += sizeof(struct mlx5_wqe_data_seg) / 16;
1180                 seg += sizeof(struct mlx5_wqe_data_seg);
1181             }
1182         }
1183 
1184         qp->next_fence = next_fence;
1185         mlx5r_finish_wqe(qp, ctrl, seg, size, cur_edge, idx, wr->wr_id,
1186                  nreq, fence, mlx5_ib_opcode[wr->opcode]);
1187 skip_psv:
1188         if (0)
1189             dump_wqe(qp, idx, size);
1190     }
1191 
1192 out:
1193     if (likely(nreq))
1194         mlx5r_ring_db(qp, nreq, ctrl);
1195 
1196     spin_unlock_irqrestore(&qp->sq.lock, flags);
1197 
1198     return err;
1199 }
1200 
1201 static void set_sig_seg(struct mlx5_rwqe_sig *sig, int max_gs)
1202 {
1203      sig->signature = calc_sig(sig, (max_gs + 1) << 2);
1204 }
1205 
1206 int mlx5_ib_post_recv(struct ib_qp *ibqp, const struct ib_recv_wr *wr,
1207               const struct ib_recv_wr **bad_wr, bool drain)
1208 {
1209     struct mlx5_ib_qp *qp = to_mqp(ibqp);
1210     struct mlx5_wqe_data_seg *scat;
1211     struct mlx5_rwqe_sig *sig;
1212     struct mlx5_ib_dev *dev = to_mdev(ibqp->device);
1213     struct mlx5_core_dev *mdev = dev->mdev;
1214     unsigned long flags;
1215     int err = 0;
1216     int nreq;
1217     int ind;
1218     int i;
1219 
1220     if (unlikely(mdev->state == MLX5_DEVICE_STATE_INTERNAL_ERROR &&
1221              !drain)) {
1222         *bad_wr = wr;
1223         return -EIO;
1224     }
1225 
1226     if (qp->type == IB_QPT_GSI)
1227         return mlx5_ib_gsi_post_recv(ibqp, wr, bad_wr);
1228 
1229     spin_lock_irqsave(&qp->rq.lock, flags);
1230 
1231     ind = qp->rq.head & (qp->rq.wqe_cnt - 1);
1232 
1233     for (nreq = 0; wr; nreq++, wr = wr->next) {
1234         if (mlx5r_wq_overflow(&qp->rq, nreq, qp->ibqp.recv_cq)) {
1235             err = -ENOMEM;
1236             *bad_wr = wr;
1237             goto out;
1238         }
1239 
1240         if (unlikely(wr->num_sge > qp->rq.max_gs)) {
1241             err = -EINVAL;
1242             *bad_wr = wr;
1243             goto out;
1244         }
1245 
1246         scat = mlx5_frag_buf_get_wqe(&qp->rq.fbc, ind);
1247         if (qp->flags_en & MLX5_QP_FLAG_SIGNATURE)
1248             scat++;
1249 
1250         for (i = 0; i < wr->num_sge; i++)
1251             set_data_ptr_seg(scat + i, wr->sg_list + i);
1252 
1253         if (i < qp->rq.max_gs) {
1254             scat[i].byte_count = 0;
1255             scat[i].lkey       = cpu_to_be32(MLX5_INVALID_LKEY);
1256             scat[i].addr       = 0;
1257         }
1258 
1259         if (qp->flags_en & MLX5_QP_FLAG_SIGNATURE) {
1260             sig = (struct mlx5_rwqe_sig *)scat;
1261             set_sig_seg(sig, qp->rq.max_gs);
1262         }
1263 
1264         qp->rq.wrid[ind] = wr->wr_id;
1265 
1266         ind = (ind + 1) & (qp->rq.wqe_cnt - 1);
1267     }
1268 
1269 out:
1270     if (likely(nreq)) {
1271         qp->rq.head += nreq;
1272 
1273         /* Make sure that descriptors are written before
1274          * doorbell record.
1275          */
1276         wmb();
1277 
1278         *qp->db.db = cpu_to_be32(qp->rq.head & 0xffff);
1279     }
1280 
1281     spin_unlock_irqrestore(&qp->rq.lock, flags);
1282 
1283     return err;
1284 }
[ Source navigation ] [ Diff markup ] [ Identifier search ] [ general search ]

This page was automatically generated by the 2.1.0 LXR engine.

The LXR team
Valid CSS 2.1! Valid HTML 4.01!