Back to home page

OSCL-LXR
 
 

     

0001 /*
0002  * Copyright (c) 2007 Cisco Systems, Inc. All rights reserved.
0003  * Copyright (c) 2007, 2008 Mellanox Technologies. All rights reserved.
0004  *
0005  * This software is available to you under a choice of one of two
0006  * licenses.  You may choose to be licensed under the terms of the GNU
0007  * General Public License (GPL) Version 2, available from the file
0008  * COPYING in the main directory of this source tree, or the
0009  * OpenIB.org BSD license below:
0010  *
0011  *     Redistribution and use in source and binary forms, with or
0012  *     without modification, are permitted provided that the following
0013  *     conditions are met:
0014  *
0015  *      - Redistributions of source code must retain the above
0016  *        copyright notice, this list of conditions and the following
0017  *        disclaimer.
0018  *
0019  *      - Redistributions in binary form must reproduce the above
0020  *        copyright notice, this list of conditions and the following
0021  *        disclaimer in the documentation and/or other materials
0022  *        provided with the distribution.
0023  *
0024  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
0025  * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
0026  * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
0027  * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
0028  * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
0029  * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
0030  * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
0031  * SOFTWARE.
0032  */
0033 
0034 #include <linux/mlx4/cq.h>
0035 #include <linux/mlx4/qp.h>
0036 #include <linux/mlx4/srq.h>
0037 #include <linux/slab.h>
0038 
0039 #include "mlx4_ib.h"
0040 #include <rdma/mlx4-abi.h>
0041 #include <rdma/uverbs_ioctl.h>
0042 
0043 static void mlx4_ib_cq_comp(struct mlx4_cq *cq)
0044 {
0045     struct ib_cq *ibcq = &to_mibcq(cq)->ibcq;
0046     ibcq->comp_handler(ibcq, ibcq->cq_context);
0047 }
0048 
0049 static void mlx4_ib_cq_event(struct mlx4_cq *cq, enum mlx4_event type)
0050 {
0051     struct ib_event event;
0052     struct ib_cq *ibcq;
0053 
0054     if (type != MLX4_EVENT_TYPE_CQ_ERROR) {
0055         pr_warn("Unexpected event type %d "
0056                "on CQ %06x\n", type, cq->cqn);
0057         return;
0058     }
0059 
0060     ibcq = &to_mibcq(cq)->ibcq;
0061     if (ibcq->event_handler) {
0062         event.device     = ibcq->device;
0063         event.event      = IB_EVENT_CQ_ERR;
0064         event.element.cq = ibcq;
0065         ibcq->event_handler(&event, ibcq->cq_context);
0066     }
0067 }
0068 
0069 static void *get_cqe_from_buf(struct mlx4_ib_cq_buf *buf, int n)
0070 {
0071     return mlx4_buf_offset(&buf->buf, n * buf->entry_size);
0072 }
0073 
0074 static void *get_cqe(struct mlx4_ib_cq *cq, int n)
0075 {
0076     return get_cqe_from_buf(&cq->buf, n);
0077 }
0078 
0079 static void *get_sw_cqe(struct mlx4_ib_cq *cq, int n)
0080 {
0081     struct mlx4_cqe *cqe = get_cqe(cq, n & cq->ibcq.cqe);
0082     struct mlx4_cqe *tcqe = ((cq->buf.entry_size == 64) ? (cqe + 1) : cqe);
0083 
0084     return (!!(tcqe->owner_sr_opcode & MLX4_CQE_OWNER_MASK) ^
0085         !!(n & (cq->ibcq.cqe + 1))) ? NULL : cqe;
0086 }
0087 
0088 static struct mlx4_cqe *next_cqe_sw(struct mlx4_ib_cq *cq)
0089 {
0090     return get_sw_cqe(cq, cq->mcq.cons_index);
0091 }
0092 
0093 int mlx4_ib_modify_cq(struct ib_cq *cq, u16 cq_count, u16 cq_period)
0094 {
0095     struct mlx4_ib_cq *mcq = to_mcq(cq);
0096     struct mlx4_ib_dev *dev = to_mdev(cq->device);
0097 
0098     return mlx4_cq_modify(dev->dev, &mcq->mcq, cq_count, cq_period);
0099 }
0100 
0101 static int mlx4_ib_alloc_cq_buf(struct mlx4_ib_dev *dev, struct mlx4_ib_cq_buf *buf, int nent)
0102 {
0103     int err;
0104 
0105     err = mlx4_buf_alloc(dev->dev, nent * dev->dev->caps.cqe_size,
0106                  PAGE_SIZE * 2, &buf->buf);
0107 
0108     if (err)
0109         goto out;
0110 
0111     buf->entry_size = dev->dev->caps.cqe_size;
0112     err = mlx4_mtt_init(dev->dev, buf->buf.npages, buf->buf.page_shift,
0113                     &buf->mtt);
0114     if (err)
0115         goto err_buf;
0116 
0117     err = mlx4_buf_write_mtt(dev->dev, &buf->mtt, &buf->buf);
0118     if (err)
0119         goto err_mtt;
0120 
0121     return 0;
0122 
0123 err_mtt:
0124     mlx4_mtt_cleanup(dev->dev, &buf->mtt);
0125 
0126 err_buf:
0127     mlx4_buf_free(dev->dev, nent * buf->entry_size, &buf->buf);
0128 
0129 out:
0130     return err;
0131 }
0132 
0133 static void mlx4_ib_free_cq_buf(struct mlx4_ib_dev *dev, struct mlx4_ib_cq_buf *buf, int cqe)
0134 {
0135     mlx4_buf_free(dev->dev, (cqe + 1) * buf->entry_size, &buf->buf);
0136 }
0137 
0138 static int mlx4_ib_get_cq_umem(struct mlx4_ib_dev *dev,
0139                    struct mlx4_ib_cq_buf *buf,
0140                    struct ib_umem **umem, u64 buf_addr, int cqe)
0141 {
0142     int err;
0143     int cqe_size = dev->dev->caps.cqe_size;
0144     int shift;
0145     int n;
0146 
0147     *umem = ib_umem_get(&dev->ib_dev, buf_addr, cqe * cqe_size,
0148                 IB_ACCESS_LOCAL_WRITE);
0149     if (IS_ERR(*umem))
0150         return PTR_ERR(*umem);
0151 
0152     shift = mlx4_ib_umem_calc_optimal_mtt_size(*umem, 0, &n);
0153     err = mlx4_mtt_init(dev->dev, n, shift, &buf->mtt);
0154 
0155     if (err)
0156         goto err_buf;
0157 
0158     err = mlx4_ib_umem_write_mtt(dev, &buf->mtt, *umem);
0159     if (err)
0160         goto err_mtt;
0161 
0162     return 0;
0163 
0164 err_mtt:
0165     mlx4_mtt_cleanup(dev->dev, &buf->mtt);
0166 
0167 err_buf:
0168     ib_umem_release(*umem);
0169 
0170     return err;
0171 }
0172 
0173 #define CQ_CREATE_FLAGS_SUPPORTED IB_UVERBS_CQ_FLAGS_TIMESTAMP_COMPLETION
0174 int mlx4_ib_create_cq(struct ib_cq *ibcq, const struct ib_cq_init_attr *attr,
0175               struct ib_udata *udata)
0176 {
0177     struct ib_device *ibdev = ibcq->device;
0178     int entries = attr->cqe;
0179     int vector = attr->comp_vector;
0180     struct mlx4_ib_dev *dev = to_mdev(ibdev);
0181     struct mlx4_ib_cq *cq = to_mcq(ibcq);
0182     struct mlx4_uar *uar;
0183     void *buf_addr;
0184     int err;
0185     struct mlx4_ib_ucontext *context = rdma_udata_to_drv_context(
0186         udata, struct mlx4_ib_ucontext, ibucontext);
0187 
0188     if (entries < 1 || entries > dev->dev->caps.max_cqes)
0189         return -EINVAL;
0190 
0191     if (attr->flags & ~CQ_CREATE_FLAGS_SUPPORTED)
0192         return -EINVAL;
0193 
0194     entries      = roundup_pow_of_two(entries + 1);
0195     cq->ibcq.cqe = entries - 1;
0196     mutex_init(&cq->resize_mutex);
0197     spin_lock_init(&cq->lock);
0198     cq->resize_buf = NULL;
0199     cq->resize_umem = NULL;
0200     cq->create_flags = attr->flags;
0201     INIT_LIST_HEAD(&cq->send_qp_list);
0202     INIT_LIST_HEAD(&cq->recv_qp_list);
0203 
0204     if (udata) {
0205         struct mlx4_ib_create_cq ucmd;
0206 
0207         if (ib_copy_from_udata(&ucmd, udata, sizeof ucmd)) {
0208             err = -EFAULT;
0209             goto err_cq;
0210         }
0211 
0212         buf_addr = (void *)(unsigned long)ucmd.buf_addr;
0213         err = mlx4_ib_get_cq_umem(dev, &cq->buf, &cq->umem,
0214                       ucmd.buf_addr, entries);
0215         if (err)
0216             goto err_cq;
0217 
0218         err = mlx4_ib_db_map_user(udata, ucmd.db_addr, &cq->db);
0219         if (err)
0220             goto err_mtt;
0221 
0222         uar = &context->uar;
0223         cq->mcq.usage = MLX4_RES_USAGE_USER_VERBS;
0224     } else {
0225         err = mlx4_db_alloc(dev->dev, &cq->db, 1);
0226         if (err)
0227             goto err_cq;
0228 
0229         cq->mcq.set_ci_db  = cq->db.db;
0230         cq->mcq.arm_db     = cq->db.db + 1;
0231         *cq->mcq.set_ci_db = 0;
0232         *cq->mcq.arm_db    = 0;
0233 
0234         err = mlx4_ib_alloc_cq_buf(dev, &cq->buf, entries);
0235         if (err)
0236             goto err_db;
0237 
0238         buf_addr = &cq->buf.buf;
0239 
0240         uar = &dev->priv_uar;
0241         cq->mcq.usage = MLX4_RES_USAGE_DRIVER;
0242     }
0243 
0244     if (dev->eq_table)
0245         vector = dev->eq_table[vector % ibdev->num_comp_vectors];
0246 
0247     err = mlx4_cq_alloc(dev->dev, entries, &cq->buf.mtt, uar, cq->db.dma,
0248                 &cq->mcq, vector, 0,
0249                 !!(cq->create_flags &
0250                    IB_UVERBS_CQ_FLAGS_TIMESTAMP_COMPLETION),
0251                 buf_addr, !!udata);
0252     if (err)
0253         goto err_dbmap;
0254 
0255     if (udata)
0256         cq->mcq.tasklet_ctx.comp = mlx4_ib_cq_comp;
0257     else
0258         cq->mcq.comp = mlx4_ib_cq_comp;
0259     cq->mcq.event = mlx4_ib_cq_event;
0260 
0261     if (udata)
0262         if (ib_copy_to_udata(udata, &cq->mcq.cqn, sizeof (__u32))) {
0263             err = -EFAULT;
0264             goto err_cq_free;
0265         }
0266 
0267     return 0;
0268 
0269 err_cq_free:
0270     mlx4_cq_free(dev->dev, &cq->mcq);
0271 
0272 err_dbmap:
0273     if (udata)
0274         mlx4_ib_db_unmap_user(context, &cq->db);
0275 
0276 err_mtt:
0277     mlx4_mtt_cleanup(dev->dev, &cq->buf.mtt);
0278 
0279     ib_umem_release(cq->umem);
0280     if (!udata)
0281         mlx4_ib_free_cq_buf(dev, &cq->buf, cq->ibcq.cqe);
0282 
0283 err_db:
0284     if (!udata)
0285         mlx4_db_free(dev->dev, &cq->db);
0286 err_cq:
0287     return err;
0288 }
0289 
0290 static int mlx4_alloc_resize_buf(struct mlx4_ib_dev *dev, struct mlx4_ib_cq *cq,
0291                   int entries)
0292 {
0293     int err;
0294 
0295     if (cq->resize_buf)
0296         return -EBUSY;
0297 
0298     cq->resize_buf = kmalloc(sizeof *cq->resize_buf, GFP_KERNEL);
0299     if (!cq->resize_buf)
0300         return -ENOMEM;
0301 
0302     err = mlx4_ib_alloc_cq_buf(dev, &cq->resize_buf->buf, entries);
0303     if (err) {
0304         kfree(cq->resize_buf);
0305         cq->resize_buf = NULL;
0306         return err;
0307     }
0308 
0309     cq->resize_buf->cqe = entries - 1;
0310 
0311     return 0;
0312 }
0313 
0314 static int mlx4_alloc_resize_umem(struct mlx4_ib_dev *dev, struct mlx4_ib_cq *cq,
0315                    int entries, struct ib_udata *udata)
0316 {
0317     struct mlx4_ib_resize_cq ucmd;
0318     int err;
0319 
0320     if (cq->resize_umem)
0321         return -EBUSY;
0322 
0323     if (ib_copy_from_udata(&ucmd, udata, sizeof ucmd))
0324         return -EFAULT;
0325 
0326     cq->resize_buf = kmalloc(sizeof *cq->resize_buf, GFP_KERNEL);
0327     if (!cq->resize_buf)
0328         return -ENOMEM;
0329 
0330     err = mlx4_ib_get_cq_umem(dev, &cq->resize_buf->buf, &cq->resize_umem,
0331                   ucmd.buf_addr, entries);
0332     if (err) {
0333         kfree(cq->resize_buf);
0334         cq->resize_buf = NULL;
0335         return err;
0336     }
0337 
0338     cq->resize_buf->cqe = entries - 1;
0339 
0340     return 0;
0341 }
0342 
0343 static int mlx4_ib_get_outstanding_cqes(struct mlx4_ib_cq *cq)
0344 {
0345     u32 i;
0346 
0347     i = cq->mcq.cons_index;
0348     while (get_sw_cqe(cq, i))
0349         ++i;
0350 
0351     return i - cq->mcq.cons_index;
0352 }
0353 
0354 static void mlx4_ib_cq_resize_copy_cqes(struct mlx4_ib_cq *cq)
0355 {
0356     struct mlx4_cqe *cqe, *new_cqe;
0357     int i;
0358     int cqe_size = cq->buf.entry_size;
0359     int cqe_inc = cqe_size == 64 ? 1 : 0;
0360 
0361     i = cq->mcq.cons_index;
0362     cqe = get_cqe(cq, i & cq->ibcq.cqe);
0363     cqe += cqe_inc;
0364 
0365     while ((cqe->owner_sr_opcode & MLX4_CQE_OPCODE_MASK) != MLX4_CQE_OPCODE_RESIZE) {
0366         new_cqe = get_cqe_from_buf(&cq->resize_buf->buf,
0367                        (i + 1) & cq->resize_buf->cqe);
0368         memcpy(new_cqe, get_cqe(cq, i & cq->ibcq.cqe), cqe_size);
0369         new_cqe += cqe_inc;
0370 
0371         new_cqe->owner_sr_opcode = (cqe->owner_sr_opcode & ~MLX4_CQE_OWNER_MASK) |
0372             (((i + 1) & (cq->resize_buf->cqe + 1)) ? MLX4_CQE_OWNER_MASK : 0);
0373         cqe = get_cqe(cq, ++i & cq->ibcq.cqe);
0374         cqe += cqe_inc;
0375     }
0376     ++cq->mcq.cons_index;
0377 }
0378 
0379 int mlx4_ib_resize_cq(struct ib_cq *ibcq, int entries, struct ib_udata *udata)
0380 {
0381     struct mlx4_ib_dev *dev = to_mdev(ibcq->device);
0382     struct mlx4_ib_cq *cq = to_mcq(ibcq);
0383     struct mlx4_mtt mtt;
0384     int outst_cqe;
0385     int err;
0386 
0387     mutex_lock(&cq->resize_mutex);
0388     if (entries < 1 || entries > dev->dev->caps.max_cqes) {
0389         err = -EINVAL;
0390         goto out;
0391     }
0392 
0393     entries = roundup_pow_of_two(entries + 1);
0394     if (entries == ibcq->cqe + 1) {
0395         err = 0;
0396         goto out;
0397     }
0398 
0399     if (entries > dev->dev->caps.max_cqes + 1) {
0400         err = -EINVAL;
0401         goto out;
0402     }
0403 
0404     if (ibcq->uobject) {
0405         err = mlx4_alloc_resize_umem(dev, cq, entries, udata);
0406         if (err)
0407             goto out;
0408     } else {
0409         /* Can't be smaller than the number of outstanding CQEs */
0410         outst_cqe = mlx4_ib_get_outstanding_cqes(cq);
0411         if (entries < outst_cqe + 1) {
0412             err = -EINVAL;
0413             goto out;
0414         }
0415 
0416         err = mlx4_alloc_resize_buf(dev, cq, entries);
0417         if (err)
0418             goto out;
0419     }
0420 
0421     mtt = cq->buf.mtt;
0422 
0423     err = mlx4_cq_resize(dev->dev, &cq->mcq, entries, &cq->resize_buf->buf.mtt);
0424     if (err)
0425         goto err_buf;
0426 
0427     mlx4_mtt_cleanup(dev->dev, &mtt);
0428     if (ibcq->uobject) {
0429         cq->buf      = cq->resize_buf->buf;
0430         cq->ibcq.cqe = cq->resize_buf->cqe;
0431         ib_umem_release(cq->umem);
0432         cq->umem     = cq->resize_umem;
0433 
0434         kfree(cq->resize_buf);
0435         cq->resize_buf = NULL;
0436         cq->resize_umem = NULL;
0437     } else {
0438         struct mlx4_ib_cq_buf tmp_buf;
0439         int tmp_cqe = 0;
0440 
0441         spin_lock_irq(&cq->lock);
0442         if (cq->resize_buf) {
0443             mlx4_ib_cq_resize_copy_cqes(cq);
0444             tmp_buf = cq->buf;
0445             tmp_cqe = cq->ibcq.cqe;
0446             cq->buf      = cq->resize_buf->buf;
0447             cq->ibcq.cqe = cq->resize_buf->cqe;
0448 
0449             kfree(cq->resize_buf);
0450             cq->resize_buf = NULL;
0451         }
0452         spin_unlock_irq(&cq->lock);
0453 
0454         if (tmp_cqe)
0455             mlx4_ib_free_cq_buf(dev, &tmp_buf, tmp_cqe);
0456     }
0457 
0458     goto out;
0459 
0460 err_buf:
0461     mlx4_mtt_cleanup(dev->dev, &cq->resize_buf->buf.mtt);
0462     if (!ibcq->uobject)
0463         mlx4_ib_free_cq_buf(dev, &cq->resize_buf->buf,
0464                     cq->resize_buf->cqe);
0465 
0466     kfree(cq->resize_buf);
0467     cq->resize_buf = NULL;
0468 
0469     ib_umem_release(cq->resize_umem);
0470     cq->resize_umem = NULL;
0471 out:
0472     mutex_unlock(&cq->resize_mutex);
0473 
0474     return err;
0475 }
0476 
0477 int mlx4_ib_destroy_cq(struct ib_cq *cq, struct ib_udata *udata)
0478 {
0479     struct mlx4_ib_dev *dev = to_mdev(cq->device);
0480     struct mlx4_ib_cq *mcq = to_mcq(cq);
0481 
0482     mlx4_cq_free(dev->dev, &mcq->mcq);
0483     mlx4_mtt_cleanup(dev->dev, &mcq->buf.mtt);
0484 
0485     if (udata) {
0486         mlx4_ib_db_unmap_user(
0487             rdma_udata_to_drv_context(
0488                 udata,
0489                 struct mlx4_ib_ucontext,
0490                 ibucontext),
0491             &mcq->db);
0492     } else {
0493         mlx4_ib_free_cq_buf(dev, &mcq->buf, cq->cqe);
0494         mlx4_db_free(dev->dev, &mcq->db);
0495     }
0496     ib_umem_release(mcq->umem);
0497     return 0;
0498 }
0499 
0500 static void dump_cqe(void *cqe)
0501 {
0502     __be32 *buf = cqe;
0503 
0504     pr_debug("CQE contents %08x %08x %08x %08x %08x %08x %08x %08x\n",
0505            be32_to_cpu(buf[0]), be32_to_cpu(buf[1]), be32_to_cpu(buf[2]),
0506            be32_to_cpu(buf[3]), be32_to_cpu(buf[4]), be32_to_cpu(buf[5]),
0507            be32_to_cpu(buf[6]), be32_to_cpu(buf[7]));
0508 }
0509 
0510 static void mlx4_ib_handle_error_cqe(struct mlx4_err_cqe *cqe,
0511                      struct ib_wc *wc)
0512 {
0513     if (cqe->syndrome == MLX4_CQE_SYNDROME_LOCAL_QP_OP_ERR) {
0514         pr_debug("local QP operation err "
0515                "(QPN %06x, WQE index %x, vendor syndrome %02x, "
0516                "opcode = %02x)\n",
0517                be32_to_cpu(cqe->my_qpn), be16_to_cpu(cqe->wqe_index),
0518                cqe->vendor_err_syndrome,
0519                cqe->owner_sr_opcode & ~MLX4_CQE_OWNER_MASK);
0520         dump_cqe(cqe);
0521     }
0522 
0523     switch (cqe->syndrome) {
0524     case MLX4_CQE_SYNDROME_LOCAL_LENGTH_ERR:
0525         wc->status = IB_WC_LOC_LEN_ERR;
0526         break;
0527     case MLX4_CQE_SYNDROME_LOCAL_QP_OP_ERR:
0528         wc->status = IB_WC_LOC_QP_OP_ERR;
0529         break;
0530     case MLX4_CQE_SYNDROME_LOCAL_PROT_ERR:
0531         wc->status = IB_WC_LOC_PROT_ERR;
0532         break;
0533     case MLX4_CQE_SYNDROME_WR_FLUSH_ERR:
0534         wc->status = IB_WC_WR_FLUSH_ERR;
0535         break;
0536     case MLX4_CQE_SYNDROME_MW_BIND_ERR:
0537         wc->status = IB_WC_MW_BIND_ERR;
0538         break;
0539     case MLX4_CQE_SYNDROME_BAD_RESP_ERR:
0540         wc->status = IB_WC_BAD_RESP_ERR;
0541         break;
0542     case MLX4_CQE_SYNDROME_LOCAL_ACCESS_ERR:
0543         wc->status = IB_WC_LOC_ACCESS_ERR;
0544         break;
0545     case MLX4_CQE_SYNDROME_REMOTE_INVAL_REQ_ERR:
0546         wc->status = IB_WC_REM_INV_REQ_ERR;
0547         break;
0548     case MLX4_CQE_SYNDROME_REMOTE_ACCESS_ERR:
0549         wc->status = IB_WC_REM_ACCESS_ERR;
0550         break;
0551     case MLX4_CQE_SYNDROME_REMOTE_OP_ERR:
0552         wc->status = IB_WC_REM_OP_ERR;
0553         break;
0554     case MLX4_CQE_SYNDROME_TRANSPORT_RETRY_EXC_ERR:
0555         wc->status = IB_WC_RETRY_EXC_ERR;
0556         break;
0557     case MLX4_CQE_SYNDROME_RNR_RETRY_EXC_ERR:
0558         wc->status = IB_WC_RNR_RETRY_EXC_ERR;
0559         break;
0560     case MLX4_CQE_SYNDROME_REMOTE_ABORTED_ERR:
0561         wc->status = IB_WC_REM_ABORT_ERR;
0562         break;
0563     default:
0564         wc->status = IB_WC_GENERAL_ERR;
0565         break;
0566     }
0567 
0568     wc->vendor_err = cqe->vendor_err_syndrome;
0569 }
0570 
0571 static int mlx4_ib_ipoib_csum_ok(__be16 status, u8 badfcs_enc, __be16 checksum)
0572 {
0573     return ((badfcs_enc & MLX4_CQE_STATUS_L4_CSUM) ||
0574         ((status & cpu_to_be16(MLX4_CQE_STATUS_IPOK)) &&
0575          (status & cpu_to_be16(MLX4_CQE_STATUS_TCP |
0576                        MLX4_CQE_STATUS_UDP)) &&
0577          (checksum == cpu_to_be16(0xffff))));
0578 }
0579 
0580 static void use_tunnel_data(struct mlx4_ib_qp *qp, struct mlx4_ib_cq *cq, struct ib_wc *wc,
0581                 unsigned tail, struct mlx4_cqe *cqe, int is_eth)
0582 {
0583     struct mlx4_ib_proxy_sqp_hdr *hdr;
0584 
0585     ib_dma_sync_single_for_cpu(qp->ibqp.device,
0586                    qp->sqp_proxy_rcv[tail].map,
0587                    sizeof (struct mlx4_ib_proxy_sqp_hdr),
0588                    DMA_FROM_DEVICE);
0589     hdr = (struct mlx4_ib_proxy_sqp_hdr *) (qp->sqp_proxy_rcv[tail].addr);
0590     wc->pkey_index  = be16_to_cpu(hdr->tun.pkey_index);
0591     wc->src_qp  = be32_to_cpu(hdr->tun.flags_src_qp) & 0xFFFFFF;
0592     wc->wc_flags   |= (hdr->tun.g_ml_path & 0x80) ? (IB_WC_GRH) : 0;
0593     wc->dlid_path_bits = 0;
0594 
0595     if (is_eth) {
0596         wc->slid = 0;
0597         wc->vlan_id = be16_to_cpu(hdr->tun.sl_vid);
0598         memcpy(&(wc->smac[0]), (char *)&hdr->tun.mac_31_0, 4);
0599         memcpy(&(wc->smac[4]), (char *)&hdr->tun.slid_mac_47_32, 2);
0600         wc->wc_flags |= (IB_WC_WITH_VLAN | IB_WC_WITH_SMAC);
0601     } else {
0602         wc->slid        = be16_to_cpu(hdr->tun.slid_mac_47_32);
0603         wc->sl          = (u8) (be16_to_cpu(hdr->tun.sl_vid) >> 12);
0604     }
0605 }
0606 
0607 static void mlx4_ib_qp_sw_comp(struct mlx4_ib_qp *qp, int num_entries,
0608                    struct ib_wc *wc, int *npolled, int is_send)
0609 {
0610     struct mlx4_ib_wq *wq;
0611     unsigned cur;
0612     int i;
0613 
0614     wq = is_send ? &qp->sq : &qp->rq;
0615     cur = wq->head - wq->tail;
0616 
0617     if (cur == 0)
0618         return;
0619 
0620     for (i = 0;  i < cur && *npolled < num_entries; i++) {
0621         wc->wr_id = wq->wrid[wq->tail & (wq->wqe_cnt - 1)];
0622         wc->status = IB_WC_WR_FLUSH_ERR;
0623         wc->vendor_err = MLX4_CQE_SYNDROME_WR_FLUSH_ERR;
0624         wq->tail++;
0625         (*npolled)++;
0626         wc->qp = &qp->ibqp;
0627         wc++;
0628     }
0629 }
0630 
0631 static void mlx4_ib_poll_sw_comp(struct mlx4_ib_cq *cq, int num_entries,
0632                  struct ib_wc *wc, int *npolled)
0633 {
0634     struct mlx4_ib_qp *qp;
0635 
0636     *npolled = 0;
0637     /* Find uncompleted WQEs belonging to that cq and return
0638      * simulated FLUSH_ERR completions
0639      */
0640     list_for_each_entry(qp, &cq->send_qp_list, cq_send_list) {
0641         mlx4_ib_qp_sw_comp(qp, num_entries, wc + *npolled, npolled, 1);
0642         if (*npolled >= num_entries)
0643             goto out;
0644     }
0645 
0646     list_for_each_entry(qp, &cq->recv_qp_list, cq_recv_list) {
0647         mlx4_ib_qp_sw_comp(qp, num_entries, wc + *npolled, npolled, 0);
0648         if (*npolled >= num_entries)
0649             goto out;
0650     }
0651 
0652 out:
0653     return;
0654 }
0655 
0656 static int mlx4_ib_poll_one(struct mlx4_ib_cq *cq,
0657                 struct mlx4_ib_qp **cur_qp,
0658                 struct ib_wc *wc)
0659 {
0660     struct mlx4_cqe *cqe;
0661     struct mlx4_qp *mqp;
0662     struct mlx4_ib_wq *wq;
0663     struct mlx4_ib_srq *srq;
0664     struct mlx4_srq *msrq = NULL;
0665     int is_send;
0666     int is_error;
0667     int is_eth;
0668     u32 g_mlpath_rqpn;
0669     u16 wqe_ctr;
0670     unsigned tail = 0;
0671 
0672 repoll:
0673     cqe = next_cqe_sw(cq);
0674     if (!cqe)
0675         return -EAGAIN;
0676 
0677     if (cq->buf.entry_size == 64)
0678         cqe++;
0679 
0680     ++cq->mcq.cons_index;
0681 
0682     /*
0683      * Make sure we read CQ entry contents after we've checked the
0684      * ownership bit.
0685      */
0686     rmb();
0687 
0688     is_send  = cqe->owner_sr_opcode & MLX4_CQE_IS_SEND_MASK;
0689     is_error = (cqe->owner_sr_opcode & MLX4_CQE_OPCODE_MASK) ==
0690         MLX4_CQE_OPCODE_ERROR;
0691 
0692     /* Resize CQ in progress */
0693     if (unlikely((cqe->owner_sr_opcode & MLX4_CQE_OPCODE_MASK) == MLX4_CQE_OPCODE_RESIZE)) {
0694         if (cq->resize_buf) {
0695             struct mlx4_ib_dev *dev = to_mdev(cq->ibcq.device);
0696 
0697             mlx4_ib_free_cq_buf(dev, &cq->buf, cq->ibcq.cqe);
0698             cq->buf      = cq->resize_buf->buf;
0699             cq->ibcq.cqe = cq->resize_buf->cqe;
0700 
0701             kfree(cq->resize_buf);
0702             cq->resize_buf = NULL;
0703         }
0704 
0705         goto repoll;
0706     }
0707 
0708     if (!*cur_qp ||
0709         (be32_to_cpu(cqe->vlan_my_qpn) & MLX4_CQE_QPN_MASK) != (*cur_qp)->mqp.qpn) {
0710         /*
0711          * We do not have to take the QP table lock here,
0712          * because CQs will be locked while QPs are removed
0713          * from the table.
0714          */
0715         mqp = __mlx4_qp_lookup(to_mdev(cq->ibcq.device)->dev,
0716                        be32_to_cpu(cqe->vlan_my_qpn));
0717         *cur_qp = to_mibqp(mqp);
0718     }
0719 
0720     wc->qp = &(*cur_qp)->ibqp;
0721 
0722     if (wc->qp->qp_type == IB_QPT_XRC_TGT) {
0723         u32 srq_num;
0724         g_mlpath_rqpn = be32_to_cpu(cqe->g_mlpath_rqpn);
0725         srq_num       = g_mlpath_rqpn & 0xffffff;
0726         /* SRQ is also in the radix tree */
0727         msrq = mlx4_srq_lookup(to_mdev(cq->ibcq.device)->dev,
0728                        srq_num);
0729     }
0730 
0731     if (is_send) {
0732         wq = &(*cur_qp)->sq;
0733         if (!(*cur_qp)->sq_signal_bits) {
0734             wqe_ctr = be16_to_cpu(cqe->wqe_index);
0735             wq->tail += (u16) (wqe_ctr - (u16) wq->tail);
0736         }
0737         wc->wr_id = wq->wrid[wq->tail & (wq->wqe_cnt - 1)];
0738         ++wq->tail;
0739     } else if ((*cur_qp)->ibqp.srq) {
0740         srq = to_msrq((*cur_qp)->ibqp.srq);
0741         wqe_ctr = be16_to_cpu(cqe->wqe_index);
0742         wc->wr_id = srq->wrid[wqe_ctr];
0743         mlx4_ib_free_srq_wqe(srq, wqe_ctr);
0744     } else if (msrq) {
0745         srq = to_mibsrq(msrq);
0746         wqe_ctr = be16_to_cpu(cqe->wqe_index);
0747         wc->wr_id = srq->wrid[wqe_ctr];
0748         mlx4_ib_free_srq_wqe(srq, wqe_ctr);
0749     } else {
0750         wq    = &(*cur_qp)->rq;
0751         tail      = wq->tail & (wq->wqe_cnt - 1);
0752         wc->wr_id = wq->wrid[tail];
0753         ++wq->tail;
0754     }
0755 
0756     if (unlikely(is_error)) {
0757         mlx4_ib_handle_error_cqe((struct mlx4_err_cqe *) cqe, wc);
0758         return 0;
0759     }
0760 
0761     wc->status = IB_WC_SUCCESS;
0762 
0763     if (is_send) {
0764         wc->wc_flags = 0;
0765         switch (cqe->owner_sr_opcode & MLX4_CQE_OPCODE_MASK) {
0766         case MLX4_OPCODE_RDMA_WRITE_IMM:
0767             wc->wc_flags |= IB_WC_WITH_IMM;
0768             fallthrough;
0769         case MLX4_OPCODE_RDMA_WRITE:
0770             wc->opcode    = IB_WC_RDMA_WRITE;
0771             break;
0772         case MLX4_OPCODE_SEND_IMM:
0773             wc->wc_flags |= IB_WC_WITH_IMM;
0774             fallthrough;
0775         case MLX4_OPCODE_SEND:
0776         case MLX4_OPCODE_SEND_INVAL:
0777             wc->opcode    = IB_WC_SEND;
0778             break;
0779         case MLX4_OPCODE_RDMA_READ:
0780             wc->opcode    = IB_WC_RDMA_READ;
0781             wc->byte_len  = be32_to_cpu(cqe->byte_cnt);
0782             break;
0783         case MLX4_OPCODE_ATOMIC_CS:
0784             wc->opcode    = IB_WC_COMP_SWAP;
0785             wc->byte_len  = 8;
0786             break;
0787         case MLX4_OPCODE_ATOMIC_FA:
0788             wc->opcode    = IB_WC_FETCH_ADD;
0789             wc->byte_len  = 8;
0790             break;
0791         case MLX4_OPCODE_MASKED_ATOMIC_CS:
0792             wc->opcode    = IB_WC_MASKED_COMP_SWAP;
0793             wc->byte_len  = 8;
0794             break;
0795         case MLX4_OPCODE_MASKED_ATOMIC_FA:
0796             wc->opcode    = IB_WC_MASKED_FETCH_ADD;
0797             wc->byte_len  = 8;
0798             break;
0799         case MLX4_OPCODE_LSO:
0800             wc->opcode    = IB_WC_LSO;
0801             break;
0802         case MLX4_OPCODE_FMR:
0803             wc->opcode    = IB_WC_REG_MR;
0804             break;
0805         case MLX4_OPCODE_LOCAL_INVAL:
0806             wc->opcode    = IB_WC_LOCAL_INV;
0807             break;
0808         }
0809     } else {
0810         wc->byte_len = be32_to_cpu(cqe->byte_cnt);
0811 
0812         switch (cqe->owner_sr_opcode & MLX4_CQE_OPCODE_MASK) {
0813         case MLX4_RECV_OPCODE_RDMA_WRITE_IMM:
0814             wc->opcode  = IB_WC_RECV_RDMA_WITH_IMM;
0815             wc->wc_flags    = IB_WC_WITH_IMM;
0816             wc->ex.imm_data = cqe->immed_rss_invalid;
0817             break;
0818         case MLX4_RECV_OPCODE_SEND_INVAL:
0819             wc->opcode  = IB_WC_RECV;
0820             wc->wc_flags    = IB_WC_WITH_INVALIDATE;
0821             wc->ex.invalidate_rkey = be32_to_cpu(cqe->immed_rss_invalid);
0822             break;
0823         case MLX4_RECV_OPCODE_SEND:
0824             wc->opcode   = IB_WC_RECV;
0825             wc->wc_flags = 0;
0826             break;
0827         case MLX4_RECV_OPCODE_SEND_IMM:
0828             wc->opcode  = IB_WC_RECV;
0829             wc->wc_flags    = IB_WC_WITH_IMM;
0830             wc->ex.imm_data = cqe->immed_rss_invalid;
0831             break;
0832         }
0833 
0834         is_eth = (rdma_port_get_link_layer(wc->qp->device,
0835                           (*cur_qp)->port) ==
0836               IB_LINK_LAYER_ETHERNET);
0837         if (mlx4_is_mfunc(to_mdev(cq->ibcq.device)->dev)) {
0838             if ((*cur_qp)->mlx4_ib_qp_type &
0839                 (MLX4_IB_QPT_PROXY_SMI_OWNER |
0840                  MLX4_IB_QPT_PROXY_SMI | MLX4_IB_QPT_PROXY_GSI)) {
0841                 use_tunnel_data(*cur_qp, cq, wc, tail, cqe,
0842                         is_eth);
0843                 return 0;
0844             }
0845         }
0846 
0847         g_mlpath_rqpn      = be32_to_cpu(cqe->g_mlpath_rqpn);
0848         wc->src_qp     = g_mlpath_rqpn & 0xffffff;
0849         wc->dlid_path_bits = (g_mlpath_rqpn >> 24) & 0x7f;
0850         wc->wc_flags      |= g_mlpath_rqpn & 0x80000000 ? IB_WC_GRH : 0;
0851         wc->pkey_index     = be32_to_cpu(cqe->immed_rss_invalid) & 0x7f;
0852         wc->wc_flags      |= mlx4_ib_ipoib_csum_ok(cqe->status,
0853                     cqe->badfcs_enc,
0854                     cqe->checksum) ? IB_WC_IP_CSUM_OK : 0;
0855         if (is_eth) {
0856             wc->slid = 0;
0857             wc->sl  = be16_to_cpu(cqe->sl_vid) >> 13;
0858             if (be32_to_cpu(cqe->vlan_my_qpn) &
0859                     MLX4_CQE_CVLAN_PRESENT_MASK) {
0860                 wc->vlan_id = be16_to_cpu(cqe->sl_vid) &
0861                     MLX4_CQE_VID_MASK;
0862             } else {
0863                 wc->vlan_id = 0xffff;
0864             }
0865             memcpy(wc->smac, cqe->smac, ETH_ALEN);
0866             wc->wc_flags |= (IB_WC_WITH_VLAN | IB_WC_WITH_SMAC);
0867         } else {
0868             wc->slid = be16_to_cpu(cqe->rlid);
0869             wc->sl  = be16_to_cpu(cqe->sl_vid) >> 12;
0870             wc->vlan_id = 0xffff;
0871         }
0872     }
0873 
0874     return 0;
0875 }
0876 
0877 int mlx4_ib_poll_cq(struct ib_cq *ibcq, int num_entries, struct ib_wc *wc)
0878 {
0879     struct mlx4_ib_cq *cq = to_mcq(ibcq);
0880     struct mlx4_ib_qp *cur_qp = NULL;
0881     unsigned long flags;
0882     int npolled;
0883     struct mlx4_ib_dev *mdev = to_mdev(cq->ibcq.device);
0884 
0885     spin_lock_irqsave(&cq->lock, flags);
0886     if (mdev->dev->persist->state & MLX4_DEVICE_STATE_INTERNAL_ERROR) {
0887         mlx4_ib_poll_sw_comp(cq, num_entries, wc, &npolled);
0888         goto out;
0889     }
0890 
0891     for (npolled = 0; npolled < num_entries; ++npolled) {
0892         if (mlx4_ib_poll_one(cq, &cur_qp, wc + npolled))
0893             break;
0894     }
0895 
0896     mlx4_cq_set_ci(&cq->mcq);
0897 
0898 out:
0899     spin_unlock_irqrestore(&cq->lock, flags);
0900 
0901     return npolled;
0902 }
0903 
0904 int mlx4_ib_arm_cq(struct ib_cq *ibcq, enum ib_cq_notify_flags flags)
0905 {
0906     mlx4_cq_arm(&to_mcq(ibcq)->mcq,
0907             (flags & IB_CQ_SOLICITED_MASK) == IB_CQ_SOLICITED ?
0908             MLX4_CQ_DB_REQ_NOT_SOL : MLX4_CQ_DB_REQ_NOT,
0909             to_mdev(ibcq->device)->uar_map,
0910             MLX4_GET_DOORBELL_LOCK(&to_mdev(ibcq->device)->uar_lock));
0911 
0912     return 0;
0913 }
0914 
0915 void __mlx4_ib_cq_clean(struct mlx4_ib_cq *cq, u32 qpn, struct mlx4_ib_srq *srq)
0916 {
0917     u32 prod_index;
0918     int nfreed = 0;
0919     struct mlx4_cqe *cqe, *dest;
0920     u8 owner_bit;
0921     int cqe_inc = cq->buf.entry_size == 64 ? 1 : 0;
0922 
0923     /*
0924      * First we need to find the current producer index, so we
0925      * know where to start cleaning from.  It doesn't matter if HW
0926      * adds new entries after this loop -- the QP we're worried
0927      * about is already in RESET, so the new entries won't come
0928      * from our QP and therefore don't need to be checked.
0929      */
0930     for (prod_index = cq->mcq.cons_index; get_sw_cqe(cq, prod_index); ++prod_index)
0931         if (prod_index == cq->mcq.cons_index + cq->ibcq.cqe)
0932             break;
0933 
0934     /*
0935      * Now sweep backwards through the CQ, removing CQ entries
0936      * that match our QP by copying older entries on top of them.
0937      */
0938     while ((int) --prod_index - (int) cq->mcq.cons_index >= 0) {
0939         cqe = get_cqe(cq, prod_index & cq->ibcq.cqe);
0940         cqe += cqe_inc;
0941 
0942         if ((be32_to_cpu(cqe->vlan_my_qpn) & MLX4_CQE_QPN_MASK) == qpn) {
0943             if (srq && !(cqe->owner_sr_opcode & MLX4_CQE_IS_SEND_MASK))
0944                 mlx4_ib_free_srq_wqe(srq, be16_to_cpu(cqe->wqe_index));
0945             ++nfreed;
0946         } else if (nfreed) {
0947             dest = get_cqe(cq, (prod_index + nfreed) & cq->ibcq.cqe);
0948             dest += cqe_inc;
0949 
0950             owner_bit = dest->owner_sr_opcode & MLX4_CQE_OWNER_MASK;
0951             memcpy(dest, cqe, sizeof *cqe);
0952             dest->owner_sr_opcode = owner_bit |
0953                 (dest->owner_sr_opcode & ~MLX4_CQE_OWNER_MASK);
0954         }
0955     }
0956 
0957     if (nfreed) {
0958         cq->mcq.cons_index += nfreed;
0959         /*
0960          * Make sure update of buffer contents is done before
0961          * updating consumer index.
0962          */
0963         wmb();
0964         mlx4_cq_set_ci(&cq->mcq);
0965     }
0966 }
0967 
0968 void mlx4_ib_cq_clean(struct mlx4_ib_cq *cq, u32 qpn, struct mlx4_ib_srq *srq)
0969 {
0970     spin_lock_irq(&cq->lock);
0971     __mlx4_ib_cq_clean(cq, qpn, srq);
0972     spin_unlock_irq(&cq->lock);
0973 }
[ Source navigation ] [ Diff markup ] [ Identifier search ] [ general search ]

This page was automatically generated by the 2.1.0 LXR engine.

The LXR team
Valid CSS 2.1! Valid HTML 4.01!