Back to home page

OSCL-LXR
 
 

     

0001 // SPDX-License-Identifier: GPL-2.0 OR Linux-OpenIB
0002 /*
0003  * Copyright (c) 2016 Mellanox Technologies Ltd. All rights reserved.
0004  * Copyright (c) 2015 System Fabric Works, Inc. All rights reserved.
0005  */
0006 
0007 #include <linux/skbuff.h>
0008 
0009 #include "rxe.h"
0010 #include "rxe_loc.h"
0011 #include "rxe_queue.h"
0012 #include "rxe_task.h"
0013 
0014 enum comp_state {
0015     COMPST_GET_ACK,
0016     COMPST_GET_WQE,
0017     COMPST_COMP_WQE,
0018     COMPST_COMP_ACK,
0019     COMPST_CHECK_PSN,
0020     COMPST_CHECK_ACK,
0021     COMPST_READ,
0022     COMPST_ATOMIC,
0023     COMPST_WRITE_SEND,
0024     COMPST_UPDATE_COMP,
0025     COMPST_ERROR_RETRY,
0026     COMPST_RNR_RETRY,
0027     COMPST_ERROR,
0028     COMPST_EXIT, /* We have an issue, and we want to rerun the completer */
0029     COMPST_DONE, /* The completer finished successflly */
0030 };
0031 
0032 static char *comp_state_name[] =  {
0033     [COMPST_GET_ACK]        = "GET ACK",
0034     [COMPST_GET_WQE]        = "GET WQE",
0035     [COMPST_COMP_WQE]       = "COMP WQE",
0036     [COMPST_COMP_ACK]       = "COMP ACK",
0037     [COMPST_CHECK_PSN]      = "CHECK PSN",
0038     [COMPST_CHECK_ACK]      = "CHECK ACK",
0039     [COMPST_READ]           = "READ",
0040     [COMPST_ATOMIC]         = "ATOMIC",
0041     [COMPST_WRITE_SEND]     = "WRITE/SEND",
0042     [COMPST_UPDATE_COMP]        = "UPDATE COMP",
0043     [COMPST_ERROR_RETRY]        = "ERROR RETRY",
0044     [COMPST_RNR_RETRY]      = "RNR RETRY",
0045     [COMPST_ERROR]          = "ERROR",
0046     [COMPST_EXIT]           = "EXIT",
0047     [COMPST_DONE]           = "DONE",
0048 };
0049 
0050 static unsigned long rnrnak_usec[32] = {
0051     [IB_RNR_TIMER_655_36] = 655360,
0052     [IB_RNR_TIMER_000_01] = 10,
0053     [IB_RNR_TIMER_000_02] = 20,
0054     [IB_RNR_TIMER_000_03] = 30,
0055     [IB_RNR_TIMER_000_04] = 40,
0056     [IB_RNR_TIMER_000_06] = 60,
0057     [IB_RNR_TIMER_000_08] = 80,
0058     [IB_RNR_TIMER_000_12] = 120,
0059     [IB_RNR_TIMER_000_16] = 160,
0060     [IB_RNR_TIMER_000_24] = 240,
0061     [IB_RNR_TIMER_000_32] = 320,
0062     [IB_RNR_TIMER_000_48] = 480,
0063     [IB_RNR_TIMER_000_64] = 640,
0064     [IB_RNR_TIMER_000_96] = 960,
0065     [IB_RNR_TIMER_001_28] = 1280,
0066     [IB_RNR_TIMER_001_92] = 1920,
0067     [IB_RNR_TIMER_002_56] = 2560,
0068     [IB_RNR_TIMER_003_84] = 3840,
0069     [IB_RNR_TIMER_005_12] = 5120,
0070     [IB_RNR_TIMER_007_68] = 7680,
0071     [IB_RNR_TIMER_010_24] = 10240,
0072     [IB_RNR_TIMER_015_36] = 15360,
0073     [IB_RNR_TIMER_020_48] = 20480,
0074     [IB_RNR_TIMER_030_72] = 30720,
0075     [IB_RNR_TIMER_040_96] = 40960,
0076     [IB_RNR_TIMER_061_44] = 61410,
0077     [IB_RNR_TIMER_081_92] = 81920,
0078     [IB_RNR_TIMER_122_88] = 122880,
0079     [IB_RNR_TIMER_163_84] = 163840,
0080     [IB_RNR_TIMER_245_76] = 245760,
0081     [IB_RNR_TIMER_327_68] = 327680,
0082     [IB_RNR_TIMER_491_52] = 491520,
0083 };
0084 
0085 static inline unsigned long rnrnak_jiffies(u8 timeout)
0086 {
0087     return max_t(unsigned long,
0088         usecs_to_jiffies(rnrnak_usec[timeout]), 1);
0089 }
0090 
0091 static enum ib_wc_opcode wr_to_wc_opcode(enum ib_wr_opcode opcode)
0092 {
0093     switch (opcode) {
0094     case IB_WR_RDMA_WRITE:          return IB_WC_RDMA_WRITE;
0095     case IB_WR_RDMA_WRITE_WITH_IMM:     return IB_WC_RDMA_WRITE;
0096     case IB_WR_SEND:            return IB_WC_SEND;
0097     case IB_WR_SEND_WITH_IMM:       return IB_WC_SEND;
0098     case IB_WR_RDMA_READ:           return IB_WC_RDMA_READ;
0099     case IB_WR_ATOMIC_CMP_AND_SWP:      return IB_WC_COMP_SWAP;
0100     case IB_WR_ATOMIC_FETCH_AND_ADD:    return IB_WC_FETCH_ADD;
0101     case IB_WR_LSO:             return IB_WC_LSO;
0102     case IB_WR_SEND_WITH_INV:       return IB_WC_SEND;
0103     case IB_WR_RDMA_READ_WITH_INV:      return IB_WC_RDMA_READ;
0104     case IB_WR_LOCAL_INV:           return IB_WC_LOCAL_INV;
0105     case IB_WR_REG_MR:          return IB_WC_REG_MR;
0106     case IB_WR_BIND_MW:         return IB_WC_BIND_MW;
0107 
0108     default:
0109         return 0xff;
0110     }
0111 }
0112 
0113 void retransmit_timer(struct timer_list *t)
0114 {
0115     struct rxe_qp *qp = from_timer(qp, t, retrans_timer);
0116 
0117     pr_debug("%s: fired for qp#%d\n", __func__, qp->elem.index);
0118 
0119     if (qp->valid) {
0120         qp->comp.timeout = 1;
0121         rxe_run_task(&qp->comp.task, 1);
0122     }
0123 }
0124 
0125 void rxe_comp_queue_pkt(struct rxe_qp *qp, struct sk_buff *skb)
0126 {
0127     int must_sched;
0128 
0129     skb_queue_tail(&qp->resp_pkts, skb);
0130 
0131     must_sched = skb_queue_len(&qp->resp_pkts) > 1;
0132     if (must_sched != 0)
0133         rxe_counter_inc(SKB_TO_PKT(skb)->rxe, RXE_CNT_COMPLETER_SCHED);
0134 
0135     rxe_run_task(&qp->comp.task, must_sched);
0136 }
0137 
0138 static inline enum comp_state get_wqe(struct rxe_qp *qp,
0139                       struct rxe_pkt_info *pkt,
0140                       struct rxe_send_wqe **wqe_p)
0141 {
0142     struct rxe_send_wqe *wqe;
0143 
0144     /* we come here whether or not we found a response packet to see if
0145      * there are any posted WQEs
0146      */
0147     wqe = queue_head(qp->sq.queue, QUEUE_TYPE_FROM_CLIENT);
0148     *wqe_p = wqe;
0149 
0150     /* no WQE or requester has not started it yet */
0151     if (!wqe || wqe->state == wqe_state_posted)
0152         return pkt ? COMPST_DONE : COMPST_EXIT;
0153 
0154     /* WQE does not require an ack */
0155     if (wqe->state == wqe_state_done)
0156         return COMPST_COMP_WQE;
0157 
0158     /* WQE caused an error */
0159     if (wqe->state == wqe_state_error)
0160         return COMPST_ERROR;
0161 
0162     /* we have a WQE, if we also have an ack check its PSN */
0163     return pkt ? COMPST_CHECK_PSN : COMPST_EXIT;
0164 }
0165 
0166 static inline void reset_retry_counters(struct rxe_qp *qp)
0167 {
0168     qp->comp.retry_cnt = qp->attr.retry_cnt;
0169     qp->comp.rnr_retry = qp->attr.rnr_retry;
0170     qp->comp.started_retry = 0;
0171 }
0172 
0173 static inline enum comp_state check_psn(struct rxe_qp *qp,
0174                     struct rxe_pkt_info *pkt,
0175                     struct rxe_send_wqe *wqe)
0176 {
0177     s32 diff;
0178 
0179     /* check to see if response is past the oldest WQE. if it is, complete
0180      * send/write or error read/atomic
0181      */
0182     diff = psn_compare(pkt->psn, wqe->last_psn);
0183     if (diff > 0) {
0184         if (wqe->state == wqe_state_pending) {
0185             if (wqe->mask & WR_ATOMIC_OR_READ_MASK)
0186                 return COMPST_ERROR_RETRY;
0187 
0188             reset_retry_counters(qp);
0189             return COMPST_COMP_WQE;
0190         } else {
0191             return COMPST_DONE;
0192         }
0193     }
0194 
0195     /* compare response packet to expected response */
0196     diff = psn_compare(pkt->psn, qp->comp.psn);
0197     if (diff < 0) {
0198         /* response is most likely a retried packet if it matches an
0199          * uncompleted WQE go complete it else ignore it
0200          */
0201         if (pkt->psn == wqe->last_psn)
0202             return COMPST_COMP_ACK;
0203         else
0204             return COMPST_DONE;
0205     } else if ((diff > 0) && (wqe->mask & WR_ATOMIC_OR_READ_MASK)) {
0206         return COMPST_DONE;
0207     } else {
0208         return COMPST_CHECK_ACK;
0209     }
0210 }
0211 
0212 static inline enum comp_state check_ack(struct rxe_qp *qp,
0213                     struct rxe_pkt_info *pkt,
0214                     struct rxe_send_wqe *wqe)
0215 {
0216     unsigned int mask = pkt->mask;
0217     u8 syn;
0218     struct rxe_dev *rxe = to_rdev(qp->ibqp.device);
0219 
0220     /* Check the sequence only */
0221     switch (qp->comp.opcode) {
0222     case -1:
0223         /* Will catch all *_ONLY cases. */
0224         if (!(mask & RXE_START_MASK))
0225             return COMPST_ERROR;
0226 
0227         break;
0228 
0229     case IB_OPCODE_RC_RDMA_READ_RESPONSE_FIRST:
0230     case IB_OPCODE_RC_RDMA_READ_RESPONSE_MIDDLE:
0231         if (pkt->opcode != IB_OPCODE_RC_RDMA_READ_RESPONSE_MIDDLE &&
0232             pkt->opcode != IB_OPCODE_RC_RDMA_READ_RESPONSE_LAST) {
0233             /* read retries of partial data may restart from
0234              * read response first or response only.
0235              */
0236             if ((pkt->psn == wqe->first_psn &&
0237                  pkt->opcode ==
0238                  IB_OPCODE_RC_RDMA_READ_RESPONSE_FIRST) ||
0239                 (wqe->first_psn == wqe->last_psn &&
0240                  pkt->opcode ==
0241                  IB_OPCODE_RC_RDMA_READ_RESPONSE_ONLY))
0242                 break;
0243 
0244             return COMPST_ERROR;
0245         }
0246         break;
0247     default:
0248         WARN_ON_ONCE(1);
0249     }
0250 
0251     /* Check operation validity. */
0252     switch (pkt->opcode) {
0253     case IB_OPCODE_RC_RDMA_READ_RESPONSE_FIRST:
0254     case IB_OPCODE_RC_RDMA_READ_RESPONSE_LAST:
0255     case IB_OPCODE_RC_RDMA_READ_RESPONSE_ONLY:
0256         syn = aeth_syn(pkt);
0257 
0258         if ((syn & AETH_TYPE_MASK) != AETH_ACK)
0259             return COMPST_ERROR;
0260 
0261         fallthrough;
0262         /* (IB_OPCODE_RC_RDMA_READ_RESPONSE_MIDDLE doesn't have an AETH)
0263          */
0264     case IB_OPCODE_RC_RDMA_READ_RESPONSE_MIDDLE:
0265         if (wqe->wr.opcode != IB_WR_RDMA_READ &&
0266             wqe->wr.opcode != IB_WR_RDMA_READ_WITH_INV) {
0267             wqe->status = IB_WC_FATAL_ERR;
0268             return COMPST_ERROR;
0269         }
0270         reset_retry_counters(qp);
0271         return COMPST_READ;
0272 
0273     case IB_OPCODE_RC_ATOMIC_ACKNOWLEDGE:
0274         syn = aeth_syn(pkt);
0275 
0276         if ((syn & AETH_TYPE_MASK) != AETH_ACK)
0277             return COMPST_ERROR;
0278 
0279         if (wqe->wr.opcode != IB_WR_ATOMIC_CMP_AND_SWP &&
0280             wqe->wr.opcode != IB_WR_ATOMIC_FETCH_AND_ADD)
0281             return COMPST_ERROR;
0282         reset_retry_counters(qp);
0283         return COMPST_ATOMIC;
0284 
0285     case IB_OPCODE_RC_ACKNOWLEDGE:
0286         syn = aeth_syn(pkt);
0287         switch (syn & AETH_TYPE_MASK) {
0288         case AETH_ACK:
0289             reset_retry_counters(qp);
0290             return COMPST_WRITE_SEND;
0291 
0292         case AETH_RNR_NAK:
0293             rxe_counter_inc(rxe, RXE_CNT_RCV_RNR);
0294             return COMPST_RNR_RETRY;
0295 
0296         case AETH_NAK:
0297             switch (syn) {
0298             case AETH_NAK_PSN_SEQ_ERROR:
0299                 /* a nak implicitly acks all packets with psns
0300                  * before
0301                  */
0302                 if (psn_compare(pkt->psn, qp->comp.psn) > 0) {
0303                     rxe_counter_inc(rxe,
0304                             RXE_CNT_RCV_SEQ_ERR);
0305                     qp->comp.psn = pkt->psn;
0306                     if (qp->req.wait_psn) {
0307                         qp->req.wait_psn = 0;
0308                         rxe_run_task(&qp->req.task, 0);
0309                     }
0310                 }
0311                 return COMPST_ERROR_RETRY;
0312 
0313             case AETH_NAK_INVALID_REQ:
0314                 wqe->status = IB_WC_REM_INV_REQ_ERR;
0315                 return COMPST_ERROR;
0316 
0317             case AETH_NAK_REM_ACC_ERR:
0318                 wqe->status = IB_WC_REM_ACCESS_ERR;
0319                 return COMPST_ERROR;
0320 
0321             case AETH_NAK_REM_OP_ERR:
0322                 wqe->status = IB_WC_REM_OP_ERR;
0323                 return COMPST_ERROR;
0324 
0325             default:
0326                 pr_warn("unexpected nak %x\n", syn);
0327                 wqe->status = IB_WC_REM_OP_ERR;
0328                 return COMPST_ERROR;
0329             }
0330 
0331         default:
0332             return COMPST_ERROR;
0333         }
0334         break;
0335 
0336     default:
0337         pr_warn("unexpected opcode\n");
0338     }
0339 
0340     return COMPST_ERROR;
0341 }
0342 
0343 static inline enum comp_state do_read(struct rxe_qp *qp,
0344                       struct rxe_pkt_info *pkt,
0345                       struct rxe_send_wqe *wqe)
0346 {
0347     int ret;
0348 
0349     ret = copy_data(qp->pd, IB_ACCESS_LOCAL_WRITE,
0350             &wqe->dma, payload_addr(pkt),
0351             payload_size(pkt), RXE_TO_MR_OBJ);
0352     if (ret) {
0353         wqe->status = IB_WC_LOC_PROT_ERR;
0354         return COMPST_ERROR;
0355     }
0356 
0357     if (wqe->dma.resid == 0 && (pkt->mask & RXE_END_MASK))
0358         return COMPST_COMP_ACK;
0359 
0360     return COMPST_UPDATE_COMP;
0361 }
0362 
0363 static inline enum comp_state do_atomic(struct rxe_qp *qp,
0364                     struct rxe_pkt_info *pkt,
0365                     struct rxe_send_wqe *wqe)
0366 {
0367     int ret;
0368 
0369     u64 atomic_orig = atmack_orig(pkt);
0370 
0371     ret = copy_data(qp->pd, IB_ACCESS_LOCAL_WRITE,
0372             &wqe->dma, &atomic_orig,
0373             sizeof(u64), RXE_TO_MR_OBJ);
0374     if (ret) {
0375         wqe->status = IB_WC_LOC_PROT_ERR;
0376         return COMPST_ERROR;
0377     }
0378 
0379     return COMPST_COMP_ACK;
0380 }
0381 
0382 static void make_send_cqe(struct rxe_qp *qp, struct rxe_send_wqe *wqe,
0383               struct rxe_cqe *cqe)
0384 {
0385     struct ib_wc *wc = &cqe->ibwc;
0386     struct ib_uverbs_wc *uwc = &cqe->uibwc;
0387 
0388     memset(cqe, 0, sizeof(*cqe));
0389 
0390     if (!qp->is_user) {
0391         wc->wr_id = wqe->wr.wr_id;
0392         wc->status = wqe->status;
0393         wc->qp = &qp->ibqp;
0394     } else {
0395         uwc->wr_id = wqe->wr.wr_id;
0396         uwc->status = wqe->status;
0397         uwc->qp_num = qp->ibqp.qp_num;
0398     }
0399 
0400     if (wqe->status == IB_WC_SUCCESS) {
0401         if (!qp->is_user) {
0402             wc->opcode = wr_to_wc_opcode(wqe->wr.opcode);
0403             if (wqe->wr.opcode == IB_WR_RDMA_WRITE_WITH_IMM ||
0404                 wqe->wr.opcode == IB_WR_SEND_WITH_IMM)
0405                 wc->wc_flags = IB_WC_WITH_IMM;
0406             wc->byte_len = wqe->dma.length;
0407         } else {
0408             uwc->opcode = wr_to_wc_opcode(wqe->wr.opcode);
0409             if (wqe->wr.opcode == IB_WR_RDMA_WRITE_WITH_IMM ||
0410                 wqe->wr.opcode == IB_WR_SEND_WITH_IMM)
0411                 uwc->wc_flags = IB_WC_WITH_IMM;
0412             uwc->byte_len = wqe->dma.length;
0413         }
0414     }
0415 }
0416 
0417 /*
0418  * IBA Spec. Section 10.7.3.1 SIGNALED COMPLETIONS
0419  * ---------8<---------8<-------------
0420  * ...Note that if a completion error occurs, a Work Completion
0421  * will always be generated, even if the signaling
0422  * indicator requests an Unsignaled Completion.
0423  * ---------8<---------8<-------------
0424  */
0425 static void do_complete(struct rxe_qp *qp, struct rxe_send_wqe *wqe)
0426 {
0427     struct rxe_dev *rxe = to_rdev(qp->ibqp.device);
0428     struct rxe_cqe cqe;
0429     bool post;
0430 
0431     /* do we need to post a completion */
0432     post = ((qp->sq_sig_type == IB_SIGNAL_ALL_WR) ||
0433             (wqe->wr.send_flags & IB_SEND_SIGNALED) ||
0434             wqe->status != IB_WC_SUCCESS);
0435 
0436     if (post)
0437         make_send_cqe(qp, wqe, &cqe);
0438 
0439     queue_advance_consumer(qp->sq.queue, QUEUE_TYPE_FROM_CLIENT);
0440 
0441     if (post)
0442         rxe_cq_post(qp->scq, &cqe, 0);
0443 
0444     if (wqe->wr.opcode == IB_WR_SEND ||
0445         wqe->wr.opcode == IB_WR_SEND_WITH_IMM ||
0446         wqe->wr.opcode == IB_WR_SEND_WITH_INV)
0447         rxe_counter_inc(rxe, RXE_CNT_RDMA_SEND);
0448 
0449     /*
0450      * we completed something so let req run again
0451      * if it is trying to fence
0452      */
0453     if (qp->req.wait_fence) {
0454         qp->req.wait_fence = 0;
0455         rxe_run_task(&qp->req.task, 0);
0456     }
0457 }
0458 
0459 static inline enum comp_state complete_ack(struct rxe_qp *qp,
0460                        struct rxe_pkt_info *pkt,
0461                        struct rxe_send_wqe *wqe)
0462 {
0463     if (wqe->has_rd_atomic) {
0464         wqe->has_rd_atomic = 0;
0465         atomic_inc(&qp->req.rd_atomic);
0466         if (qp->req.need_rd_atomic) {
0467             qp->comp.timeout_retry = 0;
0468             qp->req.need_rd_atomic = 0;
0469             rxe_run_task(&qp->req.task, 0);
0470         }
0471     }
0472 
0473     if (unlikely(qp->req.state == QP_STATE_DRAIN)) {
0474         /* state_lock used by requester & completer */
0475         spin_lock_bh(&qp->state_lock);
0476         if ((qp->req.state == QP_STATE_DRAIN) &&
0477             (qp->comp.psn == qp->req.psn)) {
0478             qp->req.state = QP_STATE_DRAINED;
0479             spin_unlock_bh(&qp->state_lock);
0480 
0481             if (qp->ibqp.event_handler) {
0482                 struct ib_event ev;
0483 
0484                 ev.device = qp->ibqp.device;
0485                 ev.element.qp = &qp->ibqp;
0486                 ev.event = IB_EVENT_SQ_DRAINED;
0487                 qp->ibqp.event_handler(&ev,
0488                     qp->ibqp.qp_context);
0489             }
0490         } else {
0491             spin_unlock_bh(&qp->state_lock);
0492         }
0493     }
0494 
0495     do_complete(qp, wqe);
0496 
0497     if (psn_compare(pkt->psn, qp->comp.psn) >= 0)
0498         return COMPST_UPDATE_COMP;
0499     else
0500         return COMPST_DONE;
0501 }
0502 
0503 static inline enum comp_state complete_wqe(struct rxe_qp *qp,
0504                        struct rxe_pkt_info *pkt,
0505                        struct rxe_send_wqe *wqe)
0506 {
0507     if (pkt && wqe->state == wqe_state_pending) {
0508         if (psn_compare(wqe->last_psn, qp->comp.psn) >= 0) {
0509             qp->comp.psn = (wqe->last_psn + 1) & BTH_PSN_MASK;
0510             qp->comp.opcode = -1;
0511         }
0512 
0513         if (qp->req.wait_psn) {
0514             qp->req.wait_psn = 0;
0515             rxe_run_task(&qp->req.task, 1);
0516         }
0517     }
0518 
0519     do_complete(qp, wqe);
0520 
0521     return COMPST_GET_WQE;
0522 }
0523 
0524 static void rxe_drain_resp_pkts(struct rxe_qp *qp, bool notify)
0525 {
0526     struct sk_buff *skb;
0527     struct rxe_send_wqe *wqe;
0528     struct rxe_queue *q = qp->sq.queue;
0529 
0530     while ((skb = skb_dequeue(&qp->resp_pkts))) {
0531         rxe_put(qp);
0532         kfree_skb(skb);
0533         ib_device_put(qp->ibqp.device);
0534     }
0535 
0536     while ((wqe = queue_head(q, q->type))) {
0537         if (notify) {
0538             wqe->status = IB_WC_WR_FLUSH_ERR;
0539             do_complete(qp, wqe);
0540         } else {
0541             queue_advance_consumer(q, q->type);
0542         }
0543     }
0544 }
0545 
0546 static void free_pkt(struct rxe_pkt_info *pkt)
0547 {
0548     struct sk_buff *skb = PKT_TO_SKB(pkt);
0549     struct rxe_qp *qp = pkt->qp;
0550     struct ib_device *dev = qp->ibqp.device;
0551 
0552     kfree_skb(skb);
0553     rxe_put(qp);
0554     ib_device_put(dev);
0555 }
0556 
0557 int rxe_completer(void *arg)
0558 {
0559     struct rxe_qp *qp = (struct rxe_qp *)arg;
0560     struct rxe_dev *rxe = to_rdev(qp->ibqp.device);
0561     struct rxe_send_wqe *wqe = NULL;
0562     struct sk_buff *skb = NULL;
0563     struct rxe_pkt_info *pkt = NULL;
0564     enum comp_state state;
0565     int ret;
0566 
0567     if (!rxe_get(qp))
0568         return -EAGAIN;
0569 
0570     if (!qp->valid || qp->comp.state == QP_STATE_ERROR ||
0571         qp->comp.state == QP_STATE_RESET) {
0572         rxe_drain_resp_pkts(qp, qp->valid &&
0573                     qp->comp.state == QP_STATE_ERROR);
0574         goto exit;
0575     }
0576 
0577     if (qp->comp.timeout) {
0578         qp->comp.timeout_retry = 1;
0579         qp->comp.timeout = 0;
0580     } else {
0581         qp->comp.timeout_retry = 0;
0582     }
0583 
0584     if (qp->req.need_retry)
0585         goto exit;
0586 
0587     state = COMPST_GET_ACK;
0588 
0589     while (1) {
0590         pr_debug("qp#%d state = %s\n", qp_num(qp),
0591              comp_state_name[state]);
0592         switch (state) {
0593         case COMPST_GET_ACK:
0594             skb = skb_dequeue(&qp->resp_pkts);
0595             if (skb) {
0596                 pkt = SKB_TO_PKT(skb);
0597                 qp->comp.timeout_retry = 0;
0598             }
0599             state = COMPST_GET_WQE;
0600             break;
0601 
0602         case COMPST_GET_WQE:
0603             state = get_wqe(qp, pkt, &wqe);
0604             break;
0605 
0606         case COMPST_CHECK_PSN:
0607             state = check_psn(qp, pkt, wqe);
0608             break;
0609 
0610         case COMPST_CHECK_ACK:
0611             state = check_ack(qp, pkt, wqe);
0612             break;
0613 
0614         case COMPST_READ:
0615             state = do_read(qp, pkt, wqe);
0616             break;
0617 
0618         case COMPST_ATOMIC:
0619             state = do_atomic(qp, pkt, wqe);
0620             break;
0621 
0622         case COMPST_WRITE_SEND:
0623             if (wqe->state == wqe_state_pending &&
0624                 wqe->last_psn == pkt->psn)
0625                 state = COMPST_COMP_ACK;
0626             else
0627                 state = COMPST_UPDATE_COMP;
0628             break;
0629 
0630         case COMPST_COMP_ACK:
0631             state = complete_ack(qp, pkt, wqe);
0632             break;
0633 
0634         case COMPST_COMP_WQE:
0635             state = complete_wqe(qp, pkt, wqe);
0636             break;
0637 
0638         case COMPST_UPDATE_COMP:
0639             if (pkt->mask & RXE_END_MASK)
0640                 qp->comp.opcode = -1;
0641             else
0642                 qp->comp.opcode = pkt->opcode;
0643 
0644             if (psn_compare(pkt->psn, qp->comp.psn) >= 0)
0645                 qp->comp.psn = (pkt->psn + 1) & BTH_PSN_MASK;
0646 
0647             if (qp->req.wait_psn) {
0648                 qp->req.wait_psn = 0;
0649                 rxe_run_task(&qp->req.task, 1);
0650             }
0651 
0652             state = COMPST_DONE;
0653             break;
0654 
0655         case COMPST_DONE:
0656             goto done;
0657 
0658         case COMPST_EXIT:
0659             if (qp->comp.timeout_retry && wqe) {
0660                 state = COMPST_ERROR_RETRY;
0661                 break;
0662             }
0663 
0664             /* re reset the timeout counter if
0665              * (1) QP is type RC
0666              * (2) the QP is alive
0667              * (3) there is a packet sent by the requester that
0668              *     might be acked (we still might get spurious
0669              *     timeouts but try to keep them as few as possible)
0670              * (4) the timeout parameter is set
0671              */
0672             if ((qp_type(qp) == IB_QPT_RC) &&
0673                 (qp->req.state == QP_STATE_READY) &&
0674                 (psn_compare(qp->req.psn, qp->comp.psn) > 0) &&
0675                 qp->qp_timeout_jiffies)
0676                 mod_timer(&qp->retrans_timer,
0677                       jiffies + qp->qp_timeout_jiffies);
0678             goto exit;
0679 
0680         case COMPST_ERROR_RETRY:
0681             /* we come here if the retry timer fired and we did
0682              * not receive a response packet. try to retry the send
0683              * queue if that makes sense and the limits have not
0684              * been exceeded. remember that some timeouts are
0685              * spurious since we do not reset the timer but kick
0686              * it down the road or let it expire
0687              */
0688 
0689             /* there is nothing to retry in this case */
0690             if (!wqe || (wqe->state == wqe_state_posted))
0691                 goto exit;
0692 
0693             /* if we've started a retry, don't start another
0694              * retry sequence, unless this is a timeout.
0695              */
0696             if (qp->comp.started_retry &&
0697                 !qp->comp.timeout_retry)
0698                 goto done;
0699 
0700             if (qp->comp.retry_cnt > 0) {
0701                 if (qp->comp.retry_cnt != 7)
0702                     qp->comp.retry_cnt--;
0703 
0704                 /* no point in retrying if we have already
0705                  * seen the last ack that the requester could
0706                  * have caused
0707                  */
0708                 if (psn_compare(qp->req.psn,
0709                         qp->comp.psn) > 0) {
0710                     /* tell the requester to retry the
0711                      * send queue next time around
0712                      */
0713                     rxe_counter_inc(rxe,
0714                             RXE_CNT_COMP_RETRY);
0715                     qp->req.need_retry = 1;
0716                     qp->comp.started_retry = 1;
0717                     rxe_run_task(&qp->req.task, 0);
0718                 }
0719                 goto done;
0720 
0721             } else {
0722                 rxe_counter_inc(rxe, RXE_CNT_RETRY_EXCEEDED);
0723                 wqe->status = IB_WC_RETRY_EXC_ERR;
0724                 state = COMPST_ERROR;
0725             }
0726             break;
0727 
0728         case COMPST_RNR_RETRY:
0729             /* we come here if we received an RNR NAK */
0730             if (qp->comp.rnr_retry > 0) {
0731                 if (qp->comp.rnr_retry != 7)
0732                     qp->comp.rnr_retry--;
0733 
0734                 /* don't start a retry flow until the
0735                  * rnr timer has fired
0736                  */
0737                 qp->req.wait_for_rnr_timer = 1;
0738                 pr_debug("qp#%d set rnr nak timer\n",
0739                      qp_num(qp));
0740                 mod_timer(&qp->rnr_nak_timer,
0741                       jiffies + rnrnak_jiffies(aeth_syn(pkt)
0742                         & ~AETH_TYPE_MASK));
0743                 goto exit;
0744             } else {
0745                 rxe_counter_inc(rxe,
0746                         RXE_CNT_RNR_RETRY_EXCEEDED);
0747                 wqe->status = IB_WC_RNR_RETRY_EXC_ERR;
0748                 state = COMPST_ERROR;
0749             }
0750             break;
0751 
0752         case COMPST_ERROR:
0753             WARN_ON_ONCE(wqe->status == IB_WC_SUCCESS);
0754             do_complete(qp, wqe);
0755             rxe_qp_error(qp);
0756             goto exit;
0757         }
0758     }
0759 
0760     /* A non-zero return value will cause rxe_do_task to
0761      * exit its loop and end the tasklet. A zero return
0762      * will continue looping and return to rxe_completer
0763      */
0764 done:
0765     ret = 0;
0766     goto out;
0767 exit:
0768     ret = -EAGAIN;
0769 out:
0770     if (pkt)
0771         free_pkt(pkt);
0772     rxe_put(qp);
0773 
0774     return ret;
0775 }
[ Source navigation ] [ Diff markup ] [ Identifier search ] [ general search ]

This page was automatically generated by the 2.1.0 LXR engine.

The LXR team
Valid CSS 2.1! Valid HTML 4.01!