sw/rdmavt/rc.c

0001 // SPDX-License-Identifier: GPL-2.0 or BSD-3-Clause
0002 /*
0003  * Copyright(c) 2016 Intel Corporation.
0004  */
0005
0006 #include <rdma/rdmavt_qp.h>
0007 #include <rdma/ib_hdrs.h>
0008
0009 /*
0010  * Convert the AETH credit code into the number of credits.
0011  */
0012 static const u16 credit_table[31] = {
0013     0,                      /* 0 */
0014     1,                      /* 1 */
0015     2,                      /* 2 */
0016     3,                      /* 3 */
0017     4,                      /* 4 */
0018     6,                      /* 5 */
0019     8,                      /* 6 */
0020     12,                     /* 7 */
0021     16,                     /* 8 */
0022     24,                     /* 9 */
0023     32,                     /* A */
0024     48,                     /* B */
0025     64,                     /* C */
0026     96,                     /* D */
0027     128,                    /* E */
0028     192,                    /* F */
0029     256,                    /* 10 */
0030     384,                    /* 11 */
0031     512,                    /* 12 */
0032     768,                    /* 13 */
0033     1024,                   /* 14 */
0034     1536,                   /* 15 */
0035     2048,                   /* 16 */
0036     3072,                   /* 17 */
0037     4096,                   /* 18 */
0038     6144,                   /* 19 */
0039     8192,                   /* 1A */
0040     12288,                  /* 1B */
0041     16384,                  /* 1C */
0042     24576,                  /* 1D */
0043     32768                   /* 1E */
0044 };
0045
0046 /**
0047  * rvt_compute_aeth - compute the AETH (syndrome + MSN)
0048  * @qp: the queue pair to compute the AETH for
0049  *
0050  * Returns the AETH.
0051  */
0052 __be32 rvt_compute_aeth(struct rvt_qp *qp)
0053 {
0054     u32 aeth = qp->r_msn & IB_MSN_MASK;
0055
0056     if (qp->ibqp.srq) {
0057         /*
0058          * Shared receive queues don't generate credits.
0059          * Set the credit field to the invalid value.
0060          */
0061         aeth |= IB_AETH_CREDIT_INVAL << IB_AETH_CREDIT_SHIFT;
0062     } else {
0063         u32 min, max, x;
0064         u32 credits;
0065         u32 head;
0066         u32 tail;
0067
0068         credits = READ_ONCE(qp->r_rq.kwq->count);
0069         if (credits == 0) {
0070             /* sanity check pointers before trusting them */
0071             if (qp->ip) {
0072                 head = RDMA_READ_UAPI_ATOMIC(qp->r_rq.wq->head);
0073                 tail = RDMA_READ_UAPI_ATOMIC(qp->r_rq.wq->tail);
0074             } else {
0075                 head = READ_ONCE(qp->r_rq.kwq->head);
0076                 tail = READ_ONCE(qp->r_rq.kwq->tail);
0077             }
0078             if (head >= qp->r_rq.size)
0079                 head = 0;
0080             if (tail >= qp->r_rq.size)
0081                 tail = 0;
0082             /*
0083              * Compute the number of credits available (RWQEs).
0084              * There is a small chance that the pair of reads are
0085              * not atomic, which is OK, since the fuzziness is
0086              * resolved as further ACKs go out.
0087              */
0088             credits = rvt_get_rq_count(&qp->r_rq, head, tail);
0089         }
0090         /*
0091          * Binary search the credit table to find the code to
0092          * use.
0093          */
0094         min = 0;
0095         max = 31;
0096         for (;;) {
0097             x = (min + max) / 2;
0098             if (credit_table[x] == credits)
0099                 break;
0100             if (credit_table[x] > credits) {
0101                 max = x;
0102             } else {
0103                 if (min == x)
0104                     break;
0105                 min = x;
0106             }
0107         }
0108         aeth |= x << IB_AETH_CREDIT_SHIFT;
0109     }
0110     return cpu_to_be32(aeth);
0111 }
0112 EXPORT_SYMBOL(rvt_compute_aeth);
0113
0114 /**
0115  * rvt_get_credit - flush the send work queue of a QP
0116  * @qp: the qp who's send work queue to flush
0117  * @aeth: the Acknowledge Extended Transport Header
0118  *
0119  * The QP s_lock should be held.
0120  */
0121 void rvt_get_credit(struct rvt_qp *qp, u32 aeth)
0122 {
0123     struct rvt_dev_info *rdi = ib_to_rvt(qp->ibqp.device);
0124     u32 credit = (aeth >> IB_AETH_CREDIT_SHIFT) & IB_AETH_CREDIT_MASK;
0125
0126     lockdep_assert_held(&qp->s_lock);
0127     /*
0128      * If the credit is invalid, we can send
0129      * as many packets as we like.  Otherwise, we have to
0130      * honor the credit field.
0131      */
0132     if (credit == IB_AETH_CREDIT_INVAL) {
0133         if (!(qp->s_flags & RVT_S_UNLIMITED_CREDIT)) {
0134             qp->s_flags |= RVT_S_UNLIMITED_CREDIT;
0135             if (qp->s_flags & RVT_S_WAIT_SSN_CREDIT) {
0136                 qp->s_flags &= ~RVT_S_WAIT_SSN_CREDIT;
0137                 rdi->driver_f.schedule_send(qp);
0138             }
0139         }
0140     } else if (!(qp->s_flags & RVT_S_UNLIMITED_CREDIT)) {
0141         /* Compute new LSN (i.e., MSN + credit) */
0142         credit = (aeth + credit_table[credit]) & IB_MSN_MASK;
0143         if (rvt_cmp_msn(credit, qp->s_lsn) > 0) {
0144             qp->s_lsn = credit;
0145             if (qp->s_flags & RVT_S_WAIT_SSN_CREDIT) {
0146                 qp->s_flags &= ~RVT_S_WAIT_SSN_CREDIT;
0147                 rdi->driver_f.schedule_send(qp);
0148             }
0149         }
0150     }
0151 }
0152 EXPORT_SYMBOL(rvt_get_credit);
0153
0154 /**
0155  * rvt_restart_sge - rewind the sge state for a wqe
0156  * @ss: the sge state pointer
0157  * @wqe: the wqe to rewind
0158  * @len: the data length from the start of the wqe in bytes
0159  *
0160  * Returns the remaining data length.
0161  */
0162 u32 rvt_restart_sge(struct rvt_sge_state *ss, struct rvt_swqe *wqe, u32 len)
0163 {
0164     ss->sge = wqe->sg_list[0];
0165     ss->sg_list = wqe->sg_list + 1;
0166     ss->num_sge = wqe->wr.num_sge;
0167     ss->total_len = wqe->length;
0168     rvt_skip_sge(ss, len, false);
0169     return wqe->length - len;
0170 }
0171 EXPORT_SYMBOL(rvt_restart_sge);
0172