sw/siw/siw_cq.c

0001 // SPDX-License-Identifier: GPL-2.0 or BSD-3-Clause
0002
0003 /* Authors: Bernard Metzler <bmt@zurich.ibm.com> */
0004 /* Copyright (c) 2008-2019, IBM Corporation */
0005
0006 #include <linux/errno.h>
0007 #include <linux/types.h>
0008
0009 #include <rdma/ib_verbs.h>
0010
0011 #include "siw.h"
0012
0013 static int map_wc_opcode[SIW_NUM_OPCODES] = {
0014     [SIW_OP_WRITE] = IB_WC_RDMA_WRITE,
0015     [SIW_OP_SEND] = IB_WC_SEND,
0016     [SIW_OP_SEND_WITH_IMM] = IB_WC_SEND,
0017     [SIW_OP_READ] = IB_WC_RDMA_READ,
0018     [SIW_OP_READ_LOCAL_INV] = IB_WC_RDMA_READ,
0019     [SIW_OP_COMP_AND_SWAP] = IB_WC_COMP_SWAP,
0020     [SIW_OP_FETCH_AND_ADD] = IB_WC_FETCH_ADD,
0021     [SIW_OP_INVAL_STAG] = IB_WC_LOCAL_INV,
0022     [SIW_OP_REG_MR] = IB_WC_REG_MR,
0023     [SIW_OP_RECEIVE] = IB_WC_RECV,
0024     [SIW_OP_READ_RESPONSE] = -1 /* not used */
0025 };
0026
0027 static struct {
0028     enum siw_wc_status siw;
0029     enum ib_wc_status ib;
0030 } map_cqe_status[SIW_NUM_WC_STATUS] = {
0031     { SIW_WC_SUCCESS, IB_WC_SUCCESS },
0032     { SIW_WC_LOC_LEN_ERR, IB_WC_LOC_LEN_ERR },
0033     { SIW_WC_LOC_PROT_ERR, IB_WC_LOC_PROT_ERR },
0034     { SIW_WC_LOC_QP_OP_ERR, IB_WC_LOC_QP_OP_ERR },
0035     { SIW_WC_WR_FLUSH_ERR, IB_WC_WR_FLUSH_ERR },
0036     { SIW_WC_BAD_RESP_ERR, IB_WC_BAD_RESP_ERR },
0037     { SIW_WC_LOC_ACCESS_ERR, IB_WC_LOC_ACCESS_ERR },
0038     { SIW_WC_REM_ACCESS_ERR, IB_WC_REM_ACCESS_ERR },
0039     { SIW_WC_REM_INV_REQ_ERR, IB_WC_REM_INV_REQ_ERR },
0040     { SIW_WC_GENERAL_ERR, IB_WC_GENERAL_ERR }
0041 };
0042
0043 /*
0044  * Reap one CQE from the CQ. Only used by kernel clients
0045  * during CQ normal operation. Might be called during CQ
0046  * flush for user mapped CQE array as well.
0047  */
0048 int siw_reap_cqe(struct siw_cq *cq, struct ib_wc *wc)
0049 {
0050     struct siw_cqe *cqe;
0051     unsigned long flags;
0052
0053     spin_lock_irqsave(&cq->lock, flags);
0054
0055     cqe = &cq->queue[cq->cq_get % cq->num_cqe];
0056     if (READ_ONCE(cqe->flags) & SIW_WQE_VALID) {
0057         memset(wc, 0, sizeof(*wc));
0058         wc->wr_id = cqe->id;
0059         wc->status = map_cqe_status[cqe->status].ib;
0060         wc->opcode = map_wc_opcode[cqe->opcode];
0061         wc->byte_len = cqe->bytes;
0062
0063         /*
0064          * During CQ flush, also user land CQE's may get
0065          * reaped here, which do not hold a QP reference
0066          * and do not qualify for memory extension verbs.
0067          */
0068         if (likely(rdma_is_kernel_res(&cq->base_cq.res))) {
0069             if (cqe->flags & SIW_WQE_REM_INVAL) {
0070                 wc->ex.invalidate_rkey = cqe->inval_stag;
0071                 wc->wc_flags = IB_WC_WITH_INVALIDATE;
0072             }
0073             wc->qp = cqe->base_qp;
0074             siw_dbg_cq(cq,
0075                    "idx %u, type %d, flags %2x, id 0x%pK\n",
0076                    cq->cq_get % cq->num_cqe, cqe->opcode,
0077                    cqe->flags, (void *)(uintptr_t)cqe->id);
0078         }
0079         WRITE_ONCE(cqe->flags, 0);
0080         cq->cq_get++;
0081
0082         spin_unlock_irqrestore(&cq->lock, flags);
0083
0084         return 1;
0085     }
0086     spin_unlock_irqrestore(&cq->lock, flags);
0087
0088     return 0;
0089 }
0090
0091 /*
0092  * siw_cq_flush()
0093  *
0094  * Flush all CQ elements.
0095  */
0096 void siw_cq_flush(struct siw_cq *cq)
0097 {
0098     struct ib_wc wc;
0099
0100     while (siw_reap_cqe(cq, &wc))
0101         ;
0102 }