Back to home page

OSCL-LXR
 
 

     

0001 // SPDX-License-Identifier: GPL-2.0 OR Linux-OpenIB
0002 /*
0003  * Copyright (c) 2016 Mellanox Technologies Ltd. All rights reserved.
0004  * Copyright (c) 2015 System Fabric Works, Inc. All rights reserved.
0005  */
0006 
0007 #include <linux/skbuff.h>
0008 #include <crypto/hash.h>
0009 
0010 #include "rxe.h"
0011 #include "rxe_loc.h"
0012 #include "rxe_queue.h"
0013 
0014 static int next_opcode(struct rxe_qp *qp, struct rxe_send_wqe *wqe,
0015                u32 opcode);
0016 
0017 static inline void retry_first_write_send(struct rxe_qp *qp,
0018                       struct rxe_send_wqe *wqe, int npsn)
0019 {
0020     int i;
0021 
0022     for (i = 0; i < npsn; i++) {
0023         int to_send = (wqe->dma.resid > qp->mtu) ?
0024                 qp->mtu : wqe->dma.resid;
0025 
0026         qp->req.opcode = next_opcode(qp, wqe,
0027                          wqe->wr.opcode);
0028 
0029         if (wqe->wr.send_flags & IB_SEND_INLINE) {
0030             wqe->dma.resid -= to_send;
0031             wqe->dma.sge_offset += to_send;
0032         } else {
0033             advance_dma_data(&wqe->dma, to_send);
0034         }
0035     }
0036 }
0037 
0038 static void req_retry(struct rxe_qp *qp)
0039 {
0040     struct rxe_send_wqe *wqe;
0041     unsigned int wqe_index;
0042     unsigned int mask;
0043     int npsn;
0044     int first = 1;
0045     struct rxe_queue *q = qp->sq.queue;
0046     unsigned int cons;
0047     unsigned int prod;
0048 
0049     cons = queue_get_consumer(q, QUEUE_TYPE_FROM_CLIENT);
0050     prod = queue_get_producer(q, QUEUE_TYPE_FROM_CLIENT);
0051 
0052     qp->req.wqe_index   = cons;
0053     qp->req.psn     = qp->comp.psn;
0054     qp->req.opcode      = -1;
0055 
0056     for (wqe_index = cons; wqe_index != prod;
0057             wqe_index = queue_next_index(q, wqe_index)) {
0058         wqe = queue_addr_from_index(qp->sq.queue, wqe_index);
0059         mask = wr_opcode_mask(wqe->wr.opcode, qp);
0060 
0061         if (wqe->state == wqe_state_posted)
0062             break;
0063 
0064         if (wqe->state == wqe_state_done)
0065             continue;
0066 
0067         wqe->iova = (mask & WR_ATOMIC_MASK) ?
0068                  wqe->wr.wr.atomic.remote_addr :
0069                  (mask & WR_READ_OR_WRITE_MASK) ?
0070                  wqe->wr.wr.rdma.remote_addr :
0071                  0;
0072 
0073         if (!first || (mask & WR_READ_MASK) == 0) {
0074             wqe->dma.resid = wqe->dma.length;
0075             wqe->dma.cur_sge = 0;
0076             wqe->dma.sge_offset = 0;
0077         }
0078 
0079         if (first) {
0080             first = 0;
0081 
0082             if (mask & WR_WRITE_OR_SEND_MASK) {
0083                 npsn = (qp->comp.psn - wqe->first_psn) &
0084                     BTH_PSN_MASK;
0085                 retry_first_write_send(qp, wqe, npsn);
0086             }
0087 
0088             if (mask & WR_READ_MASK) {
0089                 npsn = (wqe->dma.length - wqe->dma.resid) /
0090                     qp->mtu;
0091                 wqe->iova += npsn * qp->mtu;
0092             }
0093         }
0094 
0095         wqe->state = wqe_state_posted;
0096     }
0097 }
0098 
0099 void rnr_nak_timer(struct timer_list *t)
0100 {
0101     struct rxe_qp *qp = from_timer(qp, t, rnr_nak_timer);
0102 
0103     pr_debug("%s: fired for qp#%d\n", __func__, qp_num(qp));
0104 
0105     /* request a send queue retry */
0106     qp->req.need_retry = 1;
0107     qp->req.wait_for_rnr_timer = 0;
0108     rxe_run_task(&qp->req.task, 1);
0109 }
0110 
0111 static struct rxe_send_wqe *req_next_wqe(struct rxe_qp *qp)
0112 {
0113     struct rxe_send_wqe *wqe;
0114     struct rxe_queue *q = qp->sq.queue;
0115     unsigned int index = qp->req.wqe_index;
0116     unsigned int cons;
0117     unsigned int prod;
0118 
0119     wqe = queue_head(q, QUEUE_TYPE_FROM_CLIENT);
0120     cons = queue_get_consumer(q, QUEUE_TYPE_FROM_CLIENT);
0121     prod = queue_get_producer(q, QUEUE_TYPE_FROM_CLIENT);
0122 
0123     if (unlikely(qp->req.state == QP_STATE_DRAIN)) {
0124         /* check to see if we are drained;
0125          * state_lock used by requester and completer
0126          */
0127         spin_lock_bh(&qp->state_lock);
0128         do {
0129             if (qp->req.state != QP_STATE_DRAIN) {
0130                 /* comp just finished */
0131                 spin_unlock_bh(&qp->state_lock);
0132                 break;
0133             }
0134 
0135             if (wqe && ((index != cons) ||
0136                 (wqe->state != wqe_state_posted))) {
0137                 /* comp not done yet */
0138                 spin_unlock_bh(&qp->state_lock);
0139                 break;
0140             }
0141 
0142             qp->req.state = QP_STATE_DRAINED;
0143             spin_unlock_bh(&qp->state_lock);
0144 
0145             if (qp->ibqp.event_handler) {
0146                 struct ib_event ev;
0147 
0148                 ev.device = qp->ibqp.device;
0149                 ev.element.qp = &qp->ibqp;
0150                 ev.event = IB_EVENT_SQ_DRAINED;
0151                 qp->ibqp.event_handler(&ev,
0152                     qp->ibqp.qp_context);
0153             }
0154         } while (0);
0155     }
0156 
0157     if (index == prod)
0158         return NULL;
0159 
0160     wqe = queue_addr_from_index(q, index);
0161 
0162     if (unlikely((qp->req.state == QP_STATE_DRAIN ||
0163               qp->req.state == QP_STATE_DRAINED) &&
0164              (wqe->state != wqe_state_processing)))
0165         return NULL;
0166 
0167     wqe->mask = wr_opcode_mask(wqe->wr.opcode, qp);
0168     return wqe;
0169 }
0170 
0171 /**
0172  * rxe_wqe_is_fenced - check if next wqe is fenced
0173  * @qp: the queue pair
0174  * @wqe: the next wqe
0175  *
0176  * Returns: 1 if wqe needs to wait
0177  *      0 if wqe is ready to go
0178  */
0179 static int rxe_wqe_is_fenced(struct rxe_qp *qp, struct rxe_send_wqe *wqe)
0180 {
0181     /* Local invalidate fence (LIF) see IBA 10.6.5.1
0182      * Requires ALL previous operations on the send queue
0183      * are complete. Make mandatory for the rxe driver.
0184      */
0185     if (wqe->wr.opcode == IB_WR_LOCAL_INV)
0186         return qp->req.wqe_index != queue_get_consumer(qp->sq.queue,
0187                         QUEUE_TYPE_FROM_CLIENT);
0188 
0189     /* Fence see IBA 10.8.3.3
0190      * Requires that all previous read and atomic operations
0191      * are complete.
0192      */
0193     return (wqe->wr.send_flags & IB_SEND_FENCE) &&
0194         atomic_read(&qp->req.rd_atomic) != qp->attr.max_rd_atomic;
0195 }
0196 
0197 static int next_opcode_rc(struct rxe_qp *qp, u32 opcode, int fits)
0198 {
0199     switch (opcode) {
0200     case IB_WR_RDMA_WRITE:
0201         if (qp->req.opcode == IB_OPCODE_RC_RDMA_WRITE_FIRST ||
0202             qp->req.opcode == IB_OPCODE_RC_RDMA_WRITE_MIDDLE)
0203             return fits ?
0204                 IB_OPCODE_RC_RDMA_WRITE_LAST :
0205                 IB_OPCODE_RC_RDMA_WRITE_MIDDLE;
0206         else
0207             return fits ?
0208                 IB_OPCODE_RC_RDMA_WRITE_ONLY :
0209                 IB_OPCODE_RC_RDMA_WRITE_FIRST;
0210 
0211     case IB_WR_RDMA_WRITE_WITH_IMM:
0212         if (qp->req.opcode == IB_OPCODE_RC_RDMA_WRITE_FIRST ||
0213             qp->req.opcode == IB_OPCODE_RC_RDMA_WRITE_MIDDLE)
0214             return fits ?
0215                 IB_OPCODE_RC_RDMA_WRITE_LAST_WITH_IMMEDIATE :
0216                 IB_OPCODE_RC_RDMA_WRITE_MIDDLE;
0217         else
0218             return fits ?
0219                 IB_OPCODE_RC_RDMA_WRITE_ONLY_WITH_IMMEDIATE :
0220                 IB_OPCODE_RC_RDMA_WRITE_FIRST;
0221 
0222     case IB_WR_SEND:
0223         if (qp->req.opcode == IB_OPCODE_RC_SEND_FIRST ||
0224             qp->req.opcode == IB_OPCODE_RC_SEND_MIDDLE)
0225             return fits ?
0226                 IB_OPCODE_RC_SEND_LAST :
0227                 IB_OPCODE_RC_SEND_MIDDLE;
0228         else
0229             return fits ?
0230                 IB_OPCODE_RC_SEND_ONLY :
0231                 IB_OPCODE_RC_SEND_FIRST;
0232 
0233     case IB_WR_SEND_WITH_IMM:
0234         if (qp->req.opcode == IB_OPCODE_RC_SEND_FIRST ||
0235             qp->req.opcode == IB_OPCODE_RC_SEND_MIDDLE)
0236             return fits ?
0237                 IB_OPCODE_RC_SEND_LAST_WITH_IMMEDIATE :
0238                 IB_OPCODE_RC_SEND_MIDDLE;
0239         else
0240             return fits ?
0241                 IB_OPCODE_RC_SEND_ONLY_WITH_IMMEDIATE :
0242                 IB_OPCODE_RC_SEND_FIRST;
0243 
0244     case IB_WR_RDMA_READ:
0245         return IB_OPCODE_RC_RDMA_READ_REQUEST;
0246 
0247     case IB_WR_ATOMIC_CMP_AND_SWP:
0248         return IB_OPCODE_RC_COMPARE_SWAP;
0249 
0250     case IB_WR_ATOMIC_FETCH_AND_ADD:
0251         return IB_OPCODE_RC_FETCH_ADD;
0252 
0253     case IB_WR_SEND_WITH_INV:
0254         if (qp->req.opcode == IB_OPCODE_RC_SEND_FIRST ||
0255             qp->req.opcode == IB_OPCODE_RC_SEND_MIDDLE)
0256             return fits ? IB_OPCODE_RC_SEND_LAST_WITH_INVALIDATE :
0257                 IB_OPCODE_RC_SEND_MIDDLE;
0258         else
0259             return fits ? IB_OPCODE_RC_SEND_ONLY_WITH_INVALIDATE :
0260                 IB_OPCODE_RC_SEND_FIRST;
0261     case IB_WR_REG_MR:
0262     case IB_WR_LOCAL_INV:
0263         return opcode;
0264     }
0265 
0266     return -EINVAL;
0267 }
0268 
0269 static int next_opcode_uc(struct rxe_qp *qp, u32 opcode, int fits)
0270 {
0271     switch (opcode) {
0272     case IB_WR_RDMA_WRITE:
0273         if (qp->req.opcode == IB_OPCODE_UC_RDMA_WRITE_FIRST ||
0274             qp->req.opcode == IB_OPCODE_UC_RDMA_WRITE_MIDDLE)
0275             return fits ?
0276                 IB_OPCODE_UC_RDMA_WRITE_LAST :
0277                 IB_OPCODE_UC_RDMA_WRITE_MIDDLE;
0278         else
0279             return fits ?
0280                 IB_OPCODE_UC_RDMA_WRITE_ONLY :
0281                 IB_OPCODE_UC_RDMA_WRITE_FIRST;
0282 
0283     case IB_WR_RDMA_WRITE_WITH_IMM:
0284         if (qp->req.opcode == IB_OPCODE_UC_RDMA_WRITE_FIRST ||
0285             qp->req.opcode == IB_OPCODE_UC_RDMA_WRITE_MIDDLE)
0286             return fits ?
0287                 IB_OPCODE_UC_RDMA_WRITE_LAST_WITH_IMMEDIATE :
0288                 IB_OPCODE_UC_RDMA_WRITE_MIDDLE;
0289         else
0290             return fits ?
0291                 IB_OPCODE_UC_RDMA_WRITE_ONLY_WITH_IMMEDIATE :
0292                 IB_OPCODE_UC_RDMA_WRITE_FIRST;
0293 
0294     case IB_WR_SEND:
0295         if (qp->req.opcode == IB_OPCODE_UC_SEND_FIRST ||
0296             qp->req.opcode == IB_OPCODE_UC_SEND_MIDDLE)
0297             return fits ?
0298                 IB_OPCODE_UC_SEND_LAST :
0299                 IB_OPCODE_UC_SEND_MIDDLE;
0300         else
0301             return fits ?
0302                 IB_OPCODE_UC_SEND_ONLY :
0303                 IB_OPCODE_UC_SEND_FIRST;
0304 
0305     case IB_WR_SEND_WITH_IMM:
0306         if (qp->req.opcode == IB_OPCODE_UC_SEND_FIRST ||
0307             qp->req.opcode == IB_OPCODE_UC_SEND_MIDDLE)
0308             return fits ?
0309                 IB_OPCODE_UC_SEND_LAST_WITH_IMMEDIATE :
0310                 IB_OPCODE_UC_SEND_MIDDLE;
0311         else
0312             return fits ?
0313                 IB_OPCODE_UC_SEND_ONLY_WITH_IMMEDIATE :
0314                 IB_OPCODE_UC_SEND_FIRST;
0315     }
0316 
0317     return -EINVAL;
0318 }
0319 
0320 static int next_opcode(struct rxe_qp *qp, struct rxe_send_wqe *wqe,
0321                u32 opcode)
0322 {
0323     int fits = (wqe->dma.resid <= qp->mtu);
0324 
0325     switch (qp_type(qp)) {
0326     case IB_QPT_RC:
0327         return next_opcode_rc(qp, opcode, fits);
0328 
0329     case IB_QPT_UC:
0330         return next_opcode_uc(qp, opcode, fits);
0331 
0332     case IB_QPT_UD:
0333     case IB_QPT_GSI:
0334         switch (opcode) {
0335         case IB_WR_SEND:
0336             return IB_OPCODE_UD_SEND_ONLY;
0337 
0338         case IB_WR_SEND_WITH_IMM:
0339             return IB_OPCODE_UD_SEND_ONLY_WITH_IMMEDIATE;
0340         }
0341         break;
0342 
0343     default:
0344         break;
0345     }
0346 
0347     return -EINVAL;
0348 }
0349 
0350 static inline int check_init_depth(struct rxe_qp *qp, struct rxe_send_wqe *wqe)
0351 {
0352     int depth;
0353 
0354     if (wqe->has_rd_atomic)
0355         return 0;
0356 
0357     qp->req.need_rd_atomic = 1;
0358     depth = atomic_dec_return(&qp->req.rd_atomic);
0359 
0360     if (depth >= 0) {
0361         qp->req.need_rd_atomic = 0;
0362         wqe->has_rd_atomic = 1;
0363         return 0;
0364     }
0365 
0366     atomic_inc(&qp->req.rd_atomic);
0367     return -EAGAIN;
0368 }
0369 
0370 static inline int get_mtu(struct rxe_qp *qp)
0371 {
0372     struct rxe_dev *rxe = to_rdev(qp->ibqp.device);
0373 
0374     if ((qp_type(qp) == IB_QPT_RC) || (qp_type(qp) == IB_QPT_UC))
0375         return qp->mtu;
0376 
0377     return rxe->port.mtu_cap;
0378 }
0379 
0380 static struct sk_buff *init_req_packet(struct rxe_qp *qp,
0381                        struct rxe_av *av,
0382                        struct rxe_send_wqe *wqe,
0383                        int opcode, u32 payload,
0384                        struct rxe_pkt_info *pkt)
0385 {
0386     struct rxe_dev      *rxe = to_rdev(qp->ibqp.device);
0387     struct sk_buff      *skb;
0388     struct rxe_send_wr  *ibwr = &wqe->wr;
0389     int         pad = (-payload) & 0x3;
0390     int         paylen;
0391     int         solicited;
0392     u32         qp_num;
0393     int         ack_req;
0394 
0395     /* length from start of bth to end of icrc */
0396     paylen = rxe_opcode[opcode].length + payload + pad + RXE_ICRC_SIZE;
0397     pkt->paylen = paylen;
0398 
0399     /* init skb */
0400     skb = rxe_init_packet(rxe, av, paylen, pkt);
0401     if (unlikely(!skb))
0402         return NULL;
0403 
0404     /* init bth */
0405     solicited = (ibwr->send_flags & IB_SEND_SOLICITED) &&
0406             (pkt->mask & RXE_END_MASK) &&
0407             ((pkt->mask & (RXE_SEND_MASK)) ||
0408             (pkt->mask & (RXE_WRITE_MASK | RXE_IMMDT_MASK)) ==
0409             (RXE_WRITE_MASK | RXE_IMMDT_MASK));
0410 
0411     qp_num = (pkt->mask & RXE_DETH_MASK) ? ibwr->wr.ud.remote_qpn :
0412                      qp->attr.dest_qp_num;
0413 
0414     ack_req = ((pkt->mask & RXE_END_MASK) ||
0415         (qp->req.noack_pkts++ > RXE_MAX_PKT_PER_ACK));
0416     if (ack_req)
0417         qp->req.noack_pkts = 0;
0418 
0419     bth_init(pkt, pkt->opcode, solicited, 0, pad, IB_DEFAULT_PKEY_FULL, qp_num,
0420          ack_req, pkt->psn);
0421 
0422     /* init optional headers */
0423     if (pkt->mask & RXE_RETH_MASK) {
0424         reth_set_rkey(pkt, ibwr->wr.rdma.rkey);
0425         reth_set_va(pkt, wqe->iova);
0426         reth_set_len(pkt, wqe->dma.resid);
0427     }
0428 
0429     if (pkt->mask & RXE_IMMDT_MASK)
0430         immdt_set_imm(pkt, ibwr->ex.imm_data);
0431 
0432     if (pkt->mask & RXE_IETH_MASK)
0433         ieth_set_rkey(pkt, ibwr->ex.invalidate_rkey);
0434 
0435     if (pkt->mask & RXE_ATMETH_MASK) {
0436         atmeth_set_va(pkt, wqe->iova);
0437         if (opcode == IB_OPCODE_RC_COMPARE_SWAP) {
0438             atmeth_set_swap_add(pkt, ibwr->wr.atomic.swap);
0439             atmeth_set_comp(pkt, ibwr->wr.atomic.compare_add);
0440         } else {
0441             atmeth_set_swap_add(pkt, ibwr->wr.atomic.compare_add);
0442         }
0443         atmeth_set_rkey(pkt, ibwr->wr.atomic.rkey);
0444     }
0445 
0446     if (pkt->mask & RXE_DETH_MASK) {
0447         if (qp->ibqp.qp_num == 1)
0448             deth_set_qkey(pkt, GSI_QKEY);
0449         else
0450             deth_set_qkey(pkt, ibwr->wr.ud.remote_qkey);
0451         deth_set_sqp(pkt, qp->ibqp.qp_num);
0452     }
0453 
0454     return skb;
0455 }
0456 
0457 static int finish_packet(struct rxe_qp *qp, struct rxe_av *av,
0458              struct rxe_send_wqe *wqe, struct rxe_pkt_info *pkt,
0459              struct sk_buff *skb, u32 payload)
0460 {
0461     int err;
0462 
0463     err = rxe_prepare(av, pkt, skb);
0464     if (err)
0465         return err;
0466 
0467     if (pkt->mask & RXE_WRITE_OR_SEND_MASK) {
0468         if (wqe->wr.send_flags & IB_SEND_INLINE) {
0469             u8 *tmp = &wqe->dma.inline_data[wqe->dma.sge_offset];
0470 
0471             memcpy(payload_addr(pkt), tmp, payload);
0472 
0473             wqe->dma.resid -= payload;
0474             wqe->dma.sge_offset += payload;
0475         } else {
0476             err = copy_data(qp->pd, 0, &wqe->dma,
0477                     payload_addr(pkt), payload,
0478                     RXE_FROM_MR_OBJ);
0479             if (err)
0480                 return err;
0481         }
0482         if (bth_pad(pkt)) {
0483             u8 *pad = payload_addr(pkt) + payload;
0484 
0485             memset(pad, 0, bth_pad(pkt));
0486         }
0487     }
0488 
0489     return 0;
0490 }
0491 
0492 static void update_wqe_state(struct rxe_qp *qp,
0493         struct rxe_send_wqe *wqe,
0494         struct rxe_pkt_info *pkt)
0495 {
0496     if (pkt->mask & RXE_END_MASK) {
0497         if (qp_type(qp) == IB_QPT_RC)
0498             wqe->state = wqe_state_pending;
0499     } else {
0500         wqe->state = wqe_state_processing;
0501     }
0502 }
0503 
0504 static void update_wqe_psn(struct rxe_qp *qp,
0505                struct rxe_send_wqe *wqe,
0506                struct rxe_pkt_info *pkt,
0507                u32 payload)
0508 {
0509     /* number of packets left to send including current one */
0510     int num_pkt = (wqe->dma.resid + payload + qp->mtu - 1) / qp->mtu;
0511 
0512     /* handle zero length packet case */
0513     if (num_pkt == 0)
0514         num_pkt = 1;
0515 
0516     if (pkt->mask & RXE_START_MASK) {
0517         wqe->first_psn = qp->req.psn;
0518         wqe->last_psn = (qp->req.psn + num_pkt - 1) & BTH_PSN_MASK;
0519     }
0520 
0521     if (pkt->mask & RXE_READ_MASK)
0522         qp->req.psn = (wqe->first_psn + num_pkt) & BTH_PSN_MASK;
0523     else
0524         qp->req.psn = (qp->req.psn + 1) & BTH_PSN_MASK;
0525 }
0526 
0527 static void save_state(struct rxe_send_wqe *wqe,
0528                struct rxe_qp *qp,
0529                struct rxe_send_wqe *rollback_wqe,
0530                u32 *rollback_psn)
0531 {
0532     rollback_wqe->state     = wqe->state;
0533     rollback_wqe->first_psn = wqe->first_psn;
0534     rollback_wqe->last_psn  = wqe->last_psn;
0535     *rollback_psn       = qp->req.psn;
0536 }
0537 
0538 static void rollback_state(struct rxe_send_wqe *wqe,
0539                struct rxe_qp *qp,
0540                struct rxe_send_wqe *rollback_wqe,
0541                u32 rollback_psn)
0542 {
0543     wqe->state     = rollback_wqe->state;
0544     wqe->first_psn = rollback_wqe->first_psn;
0545     wqe->last_psn  = rollback_wqe->last_psn;
0546     qp->req.psn    = rollback_psn;
0547 }
0548 
0549 static void update_state(struct rxe_qp *qp, struct rxe_pkt_info *pkt)
0550 {
0551     qp->req.opcode = pkt->opcode;
0552 
0553     if (pkt->mask & RXE_END_MASK)
0554         qp->req.wqe_index = queue_next_index(qp->sq.queue,
0555                              qp->req.wqe_index);
0556 
0557     qp->need_req_skb = 0;
0558 
0559     if (qp->qp_timeout_jiffies && !timer_pending(&qp->retrans_timer))
0560         mod_timer(&qp->retrans_timer,
0561               jiffies + qp->qp_timeout_jiffies);
0562 }
0563 
0564 static int rxe_do_local_ops(struct rxe_qp *qp, struct rxe_send_wqe *wqe)
0565 {
0566     u8 opcode = wqe->wr.opcode;
0567     u32 rkey;
0568     int ret;
0569 
0570     switch (opcode) {
0571     case IB_WR_LOCAL_INV:
0572         rkey = wqe->wr.ex.invalidate_rkey;
0573         if (rkey_is_mw(rkey))
0574             ret = rxe_invalidate_mw(qp, rkey);
0575         else
0576             ret = rxe_invalidate_mr(qp, rkey);
0577 
0578         if (unlikely(ret)) {
0579             wqe->status = IB_WC_LOC_QP_OP_ERR;
0580             return ret;
0581         }
0582         break;
0583     case IB_WR_REG_MR:
0584         ret = rxe_reg_fast_mr(qp, wqe);
0585         if (unlikely(ret)) {
0586             wqe->status = IB_WC_LOC_QP_OP_ERR;
0587             return ret;
0588         }
0589         break;
0590     case IB_WR_BIND_MW:
0591         ret = rxe_bind_mw(qp, wqe);
0592         if (unlikely(ret)) {
0593             wqe->status = IB_WC_MW_BIND_ERR;
0594             return ret;
0595         }
0596         break;
0597     default:
0598         pr_err("Unexpected send wqe opcode %d\n", opcode);
0599         wqe->status = IB_WC_LOC_QP_OP_ERR;
0600         return -EINVAL;
0601     }
0602 
0603     wqe->state = wqe_state_done;
0604     wqe->status = IB_WC_SUCCESS;
0605     qp->req.wqe_index = queue_next_index(qp->sq.queue, qp->req.wqe_index);
0606 
0607     /* There is no ack coming for local work requests
0608      * which can lead to a deadlock. So go ahead and complete
0609      * it now.
0610      */
0611     rxe_run_task(&qp->comp.task, 1);
0612 
0613     return 0;
0614 }
0615 
0616 int rxe_requester(void *arg)
0617 {
0618     struct rxe_qp *qp = (struct rxe_qp *)arg;
0619     struct rxe_dev *rxe = to_rdev(qp->ibqp.device);
0620     struct rxe_pkt_info pkt;
0621     struct sk_buff *skb;
0622     struct rxe_send_wqe *wqe;
0623     enum rxe_hdr_mask mask;
0624     u32 payload;
0625     int mtu;
0626     int opcode;
0627     int err;
0628     int ret;
0629     struct rxe_send_wqe rollback_wqe;
0630     u32 rollback_psn;
0631     struct rxe_queue *q = qp->sq.queue;
0632     struct rxe_ah *ah;
0633     struct rxe_av *av;
0634 
0635     if (!rxe_get(qp))
0636         return -EAGAIN;
0637 
0638     if (unlikely(!qp->valid))
0639         goto exit;
0640 
0641     if (unlikely(qp->req.state == QP_STATE_ERROR)) {
0642         wqe = req_next_wqe(qp);
0643         if (wqe)
0644             /*
0645              * Generate an error completion for error qp state
0646              */
0647             goto err;
0648         else
0649             goto exit;
0650     }
0651 
0652     if (unlikely(qp->req.state == QP_STATE_RESET)) {
0653         qp->req.wqe_index = queue_get_consumer(q,
0654                         QUEUE_TYPE_FROM_CLIENT);
0655         qp->req.opcode = -1;
0656         qp->req.need_rd_atomic = 0;
0657         qp->req.wait_psn = 0;
0658         qp->req.need_retry = 0;
0659         qp->req.wait_for_rnr_timer = 0;
0660         goto exit;
0661     }
0662 
0663     /* we come here if the retransmit timer has fired
0664      * or if the rnr timer has fired. If the retransmit
0665      * timer fires while we are processing an RNR NAK wait
0666      * until the rnr timer has fired before starting the
0667      * retry flow
0668      */
0669     if (unlikely(qp->req.need_retry && !qp->req.wait_for_rnr_timer)) {
0670         req_retry(qp);
0671         qp->req.need_retry = 0;
0672     }
0673 
0674     wqe = req_next_wqe(qp);
0675     if (unlikely(!wqe))
0676         goto exit;
0677 
0678     if (rxe_wqe_is_fenced(qp, wqe)) {
0679         qp->req.wait_fence = 1;
0680         goto exit;
0681     }
0682 
0683     if (wqe->mask & WR_LOCAL_OP_MASK) {
0684         err = rxe_do_local_ops(qp, wqe);
0685         if (unlikely(err))
0686             goto err;
0687         else
0688             goto done;
0689     }
0690 
0691     if (unlikely(qp_type(qp) == IB_QPT_RC &&
0692         psn_compare(qp->req.psn, (qp->comp.psn +
0693                 RXE_MAX_UNACKED_PSNS)) > 0)) {
0694         qp->req.wait_psn = 1;
0695         goto exit;
0696     }
0697 
0698     /* Limit the number of inflight SKBs per QP */
0699     if (unlikely(atomic_read(&qp->skb_out) >
0700              RXE_INFLIGHT_SKBS_PER_QP_HIGH)) {
0701         qp->need_req_skb = 1;
0702         goto exit;
0703     }
0704 
0705     opcode = next_opcode(qp, wqe, wqe->wr.opcode);
0706     if (unlikely(opcode < 0)) {
0707         wqe->status = IB_WC_LOC_QP_OP_ERR;
0708         goto err;
0709     }
0710 
0711     mask = rxe_opcode[opcode].mask;
0712     if (unlikely(mask & RXE_READ_OR_ATOMIC_MASK)) {
0713         if (check_init_depth(qp, wqe))
0714             goto exit;
0715     }
0716 
0717     mtu = get_mtu(qp);
0718     payload = (mask & RXE_WRITE_OR_SEND_MASK) ? wqe->dma.resid : 0;
0719     if (payload > mtu) {
0720         if (qp_type(qp) == IB_QPT_UD) {
0721             /* C10-93.1.1: If the total sum of all the buffer lengths specified for a
0722              * UD message exceeds the MTU of the port as returned by QueryHCA, the CI
0723              * shall not emit any packets for this message. Further, the CI shall not
0724              * generate an error due to this condition.
0725              */
0726 
0727             /* fake a successful UD send */
0728             wqe->first_psn = qp->req.psn;
0729             wqe->last_psn = qp->req.psn;
0730             qp->req.psn = (qp->req.psn + 1) & BTH_PSN_MASK;
0731             qp->req.opcode = IB_OPCODE_UD_SEND_ONLY;
0732             qp->req.wqe_index = queue_next_index(qp->sq.queue,
0733                                qp->req.wqe_index);
0734             wqe->state = wqe_state_done;
0735             wqe->status = IB_WC_SUCCESS;
0736             rxe_run_task(&qp->comp.task, 0);
0737             goto done;
0738         }
0739         payload = mtu;
0740     }
0741 
0742     pkt.rxe = rxe;
0743     pkt.opcode = opcode;
0744     pkt.qp = qp;
0745     pkt.psn = qp->req.psn;
0746     pkt.mask = rxe_opcode[opcode].mask;
0747     pkt.wqe = wqe;
0748 
0749     av = rxe_get_av(&pkt, &ah);
0750     if (unlikely(!av)) {
0751         pr_err("qp#%d Failed no address vector\n", qp_num(qp));
0752         wqe->status = IB_WC_LOC_QP_OP_ERR;
0753         goto err;
0754     }
0755 
0756     skb = init_req_packet(qp, av, wqe, opcode, payload, &pkt);
0757     if (unlikely(!skb)) {
0758         pr_err("qp#%d Failed allocating skb\n", qp_num(qp));
0759         wqe->status = IB_WC_LOC_QP_OP_ERR;
0760         if (ah)
0761             rxe_put(ah);
0762         goto err;
0763     }
0764 
0765     err = finish_packet(qp, av, wqe, &pkt, skb, payload);
0766     if (unlikely(err)) {
0767         pr_debug("qp#%d Error during finish packet\n", qp_num(qp));
0768         if (err == -EFAULT)
0769             wqe->status = IB_WC_LOC_PROT_ERR;
0770         else
0771             wqe->status = IB_WC_LOC_QP_OP_ERR;
0772         kfree_skb(skb);
0773         if (ah)
0774             rxe_put(ah);
0775         goto err;
0776     }
0777 
0778     if (ah)
0779         rxe_put(ah);
0780 
0781     /*
0782      * To prevent a race on wqe access between requester and completer,
0783      * wqe members state and psn need to be set before calling
0784      * rxe_xmit_packet().
0785      * Otherwise, completer might initiate an unjustified retry flow.
0786      */
0787     save_state(wqe, qp, &rollback_wqe, &rollback_psn);
0788     update_wqe_state(qp, wqe, &pkt);
0789     update_wqe_psn(qp, wqe, &pkt, payload);
0790 
0791     err = rxe_xmit_packet(qp, &pkt, skb);
0792     if (err) {
0793         qp->need_req_skb = 1;
0794 
0795         rollback_state(wqe, qp, &rollback_wqe, rollback_psn);
0796 
0797         if (err == -EAGAIN) {
0798             rxe_run_task(&qp->req.task, 1);
0799             goto exit;
0800         }
0801 
0802         wqe->status = IB_WC_LOC_QP_OP_ERR;
0803         goto err;
0804     }
0805 
0806     update_state(qp, &pkt);
0807 
0808     /* A non-zero return value will cause rxe_do_task to
0809      * exit its loop and end the tasklet. A zero return
0810      * will continue looping and return to rxe_requester
0811      */
0812 done:
0813     ret = 0;
0814     goto out;
0815 err:
0816     /* update wqe_index for each wqe completion */
0817     qp->req.wqe_index = queue_next_index(qp->sq.queue, qp->req.wqe_index);
0818     wqe->state = wqe_state_error;
0819     qp->req.state = QP_STATE_ERROR;
0820     rxe_run_task(&qp->comp.task, 0);
0821 exit:
0822     ret = -EAGAIN;
0823 out:
0824     rxe_put(qp);
0825 
0826     return ret;
0827 }
[ Source navigation ] [ Diff markup ] [ Identifier search ] [ general search ]

This page was automatically generated by the 2.1.0 LXR engine.

The LXR team
Valid CSS 2.1! Valid HTML 4.01!