hw/mlx5/srq.c

0001 // SPDX-License-Identifier: GPL-2.0 OR Linux-OpenIB
0002 /*
0003  * Copyright (c) 2013-2018, Mellanox Technologies inc.  All rights reserved.
0004  */
0005
0006 #include <linux/mlx5/qp.h>
0007 #include <linux/slab.h>
0008 #include <rdma/ib_umem.h>
0009 #include <rdma/ib_user_verbs.h>
0010 #include "mlx5_ib.h"
0011 #include "srq.h"
0012
0013 static void *get_wqe(struct mlx5_ib_srq *srq, int n)
0014 {
0015     return mlx5_frag_buf_get_wqe(&srq->fbc, n);
0016 }
0017
0018 static void mlx5_ib_srq_event(struct mlx5_core_srq *srq, enum mlx5_event type)
0019 {
0020     struct ib_event event;
0021     struct ib_srq *ibsrq = &to_mibsrq(srq)->ibsrq;
0022
0023     if (ibsrq->event_handler) {
0024         event.device      = ibsrq->device;
0025         event.element.srq = ibsrq;
0026         switch (type) {
0027         case MLX5_EVENT_TYPE_SRQ_RQ_LIMIT:
0028             event.event = IB_EVENT_SRQ_LIMIT_REACHED;
0029             break;
0030         case MLX5_EVENT_TYPE_SRQ_CATAS_ERROR:
0031             event.event = IB_EVENT_SRQ_ERR;
0032             break;
0033         default:
0034             pr_warn("mlx5_ib: Unexpected event type %d on SRQ %06x\n",
0035                 type, srq->srqn);
0036             return;
0037         }
0038
0039         ibsrq->event_handler(&event, ibsrq->srq_context);
0040     }
0041 }
0042
0043 static int create_srq_user(struct ib_pd *pd, struct mlx5_ib_srq *srq,
0044                struct mlx5_srq_attr *in,
0045                struct ib_udata *udata, int buf_size)
0046 {
0047     struct mlx5_ib_dev *dev = to_mdev(pd->device);
0048     struct mlx5_ib_create_srq ucmd = {};
0049     struct mlx5_ib_ucontext *ucontext = rdma_udata_to_drv_context(
0050         udata, struct mlx5_ib_ucontext, ibucontext);
0051     size_t ucmdlen;
0052     int err;
0053     u32 uidx = MLX5_IB_DEFAULT_UIDX;
0054
0055     ucmdlen = min(udata->inlen, sizeof(ucmd));
0056
0057     if (ib_copy_from_udata(&ucmd, udata, ucmdlen)) {
0058         mlx5_ib_dbg(dev, "failed copy udata\n");
0059         return -EFAULT;
0060     }
0061
0062     if (ucmd.reserved0 || ucmd.reserved1)
0063         return -EINVAL;
0064
0065     if (udata->inlen > sizeof(ucmd) &&
0066         !ib_is_udata_cleared(udata, sizeof(ucmd),
0067                  udata->inlen - sizeof(ucmd)))
0068         return -EINVAL;
0069
0070     if (in->type != IB_SRQT_BASIC) {
0071         err = get_srq_user_index(ucontext, &ucmd, udata->inlen, &uidx);
0072         if (err)
0073             return err;
0074     }
0075
0076     srq->wq_sig = !!(ucmd.flags & MLX5_SRQ_FLAG_SIGNATURE);
0077
0078     srq->umem = ib_umem_get(pd->device, ucmd.buf_addr, buf_size, 0);
0079     if (IS_ERR(srq->umem)) {
0080         mlx5_ib_dbg(dev, "failed umem get, size %d\n", buf_size);
0081         err = PTR_ERR(srq->umem);
0082         return err;
0083     }
0084     in->umem = srq->umem;
0085
0086     err = mlx5_ib_db_map_user(ucontext, ucmd.db_addr, &srq->db);
0087     if (err) {
0088         mlx5_ib_dbg(dev, "map doorbell failed\n");
0089         goto err_umem;
0090     }
0091
0092     in->uid = (in->type != IB_SRQT_XRC) ?  to_mpd(pd)->uid : 0;
0093     if (MLX5_CAP_GEN(dev->mdev, cqe_version) == MLX5_CQE_VERSION_V1 &&
0094         in->type != IB_SRQT_BASIC)
0095         in->user_index = uidx;
0096
0097     return 0;
0098
0099 err_umem:
0100     ib_umem_release(srq->umem);
0101
0102     return err;
0103 }
0104
0105 static int create_srq_kernel(struct mlx5_ib_dev *dev, struct mlx5_ib_srq *srq,
0106                  struct mlx5_srq_attr *in, int buf_size)
0107 {
0108     int err;
0109     int i;
0110     struct mlx5_wqe_srq_next_seg *next;
0111
0112     err = mlx5_db_alloc(dev->mdev, &srq->db);
0113     if (err) {
0114         mlx5_ib_warn(dev, "alloc dbell rec failed\n");
0115         return err;
0116     }
0117
0118     if (mlx5_frag_buf_alloc_node(dev->mdev, buf_size, &srq->buf,
0119                      dev->mdev->priv.numa_node)) {
0120         mlx5_ib_dbg(dev, "buf alloc failed\n");
0121         err = -ENOMEM;
0122         goto err_db;
0123     }
0124
0125     mlx5_init_fbc(srq->buf.frags, srq->msrq.wqe_shift, ilog2(srq->msrq.max),
0126               &srq->fbc);
0127
0128     srq->head    = 0;
0129     srq->tail    = srq->msrq.max - 1;
0130     srq->wqe_ctr = 0;
0131
0132     for (i = 0; i < srq->msrq.max; i++) {
0133         next = get_wqe(srq, i);
0134         next->next_wqe_index =
0135             cpu_to_be16((i + 1) & (srq->msrq.max - 1));
0136     }
0137
0138     mlx5_ib_dbg(dev, "srq->buf.page_shift = %d\n", srq->buf.page_shift);
0139     in->pas = kvcalloc(srq->buf.npages, sizeof(*in->pas), GFP_KERNEL);
0140     if (!in->pas) {
0141         err = -ENOMEM;
0142         goto err_buf;
0143     }
0144     mlx5_fill_page_frag_array(&srq->buf, in->pas);
0145
0146     srq->wrid = kvmalloc_array(srq->msrq.max, sizeof(u64), GFP_KERNEL);
0147     if (!srq->wrid) {
0148         err = -ENOMEM;
0149         goto err_in;
0150     }
0151     srq->wq_sig = 0;
0152
0153     in->log_page_size = srq->buf.page_shift - MLX5_ADAPTER_PAGE_SHIFT;
0154     if (MLX5_CAP_GEN(dev->mdev, cqe_version) == MLX5_CQE_VERSION_V1 &&
0155         in->type != IB_SRQT_BASIC)
0156         in->user_index = MLX5_IB_DEFAULT_UIDX;
0157
0158     return 0;
0159
0160 err_in:
0161     kvfree(in->pas);
0162
0163 err_buf:
0164     mlx5_frag_buf_free(dev->mdev, &srq->buf);
0165
0166 err_db:
0167     mlx5_db_free(dev->mdev, &srq->db);
0168     return err;
0169 }
0170
0171 static void destroy_srq_user(struct ib_pd *pd, struct mlx5_ib_srq *srq,
0172                  struct ib_udata *udata)
0173 {
0174     mlx5_ib_db_unmap_user(
0175         rdma_udata_to_drv_context(
0176             udata,
0177             struct mlx5_ib_ucontext,
0178             ibucontext),
0179         &srq->db);
0180     ib_umem_release(srq->umem);
0181 }
0182
0183
0184 static void destroy_srq_kernel(struct mlx5_ib_dev *dev, struct mlx5_ib_srq *srq)
0185 {
0186     kvfree(srq->wrid);
0187     mlx5_frag_buf_free(dev->mdev, &srq->buf);
0188     mlx5_db_free(dev->mdev, &srq->db);
0189 }
0190
0191 int mlx5_ib_create_srq(struct ib_srq *ib_srq,
0192                struct ib_srq_init_attr *init_attr,
0193                struct ib_udata *udata)
0194 {
0195     struct mlx5_ib_dev *dev = to_mdev(ib_srq->device);
0196     struct mlx5_ib_srq *srq = to_msrq(ib_srq);
0197     size_t desc_size;
0198     size_t buf_size;
0199     int err;
0200     struct mlx5_srq_attr in = {};
0201     __u32 max_srq_wqes = 1 << MLX5_CAP_GEN(dev->mdev, log_max_srq_sz);
0202
0203     if (init_attr->srq_type != IB_SRQT_BASIC &&
0204         init_attr->srq_type != IB_SRQT_XRC &&
0205         init_attr->srq_type != IB_SRQT_TM)
0206         return -EOPNOTSUPP;
0207
0208     /* Sanity check SRQ size before proceeding */
0209     if (init_attr->attr.max_wr >= max_srq_wqes) {
0210         mlx5_ib_dbg(dev, "max_wr %d, cap %d\n",
0211                 init_attr->attr.max_wr,
0212                 max_srq_wqes);
0213         return -EINVAL;
0214     }
0215
0216     mutex_init(&srq->mutex);
0217     spin_lock_init(&srq->lock);
0218     srq->msrq.max    = roundup_pow_of_two(init_attr->attr.max_wr + 1);
0219     srq->msrq.max_gs = init_attr->attr.max_sge;
0220
0221     desc_size = sizeof(struct mlx5_wqe_srq_next_seg) +
0222             srq->msrq.max_gs * sizeof(struct mlx5_wqe_data_seg);
0223     if (desc_size == 0 || srq->msrq.max_gs > desc_size)
0224         return -EINVAL;
0225
0226     desc_size = roundup_pow_of_two(desc_size);
0227     desc_size = max_t(size_t, 32, desc_size);
0228     if (desc_size < sizeof(struct mlx5_wqe_srq_next_seg))
0229         return -EINVAL;
0230
0231     srq->msrq.max_avail_gather = (desc_size - sizeof(struct mlx5_wqe_srq_next_seg)) /
0232         sizeof(struct mlx5_wqe_data_seg);
0233     srq->msrq.wqe_shift = ilog2(desc_size);
0234     buf_size = srq->msrq.max * desc_size;
0235     if (buf_size < desc_size)
0236         return -EINVAL;
0237
0238     in.type = init_attr->srq_type;
0239
0240     if (udata)
0241         err = create_srq_user(ib_srq->pd, srq, &in, udata, buf_size);
0242     else
0243         err = create_srq_kernel(dev, srq, &in, buf_size);
0244
0245     if (err) {
0246         mlx5_ib_warn(dev, "create srq %s failed, err %d\n",
0247                  udata ? "user" : "kernel", err);
0248         return err;
0249     }
0250
0251     in.log_size = ilog2(srq->msrq.max);
0252     in.wqe_shift = srq->msrq.wqe_shift - 4;
0253     if (srq->wq_sig)
0254         in.flags |= MLX5_SRQ_FLAG_WQ_SIG;
0255
0256     if (init_attr->srq_type == IB_SRQT_XRC && init_attr->ext.xrc.xrcd)
0257         in.xrcd = to_mxrcd(init_attr->ext.xrc.xrcd)->xrcdn;
0258     else
0259         in.xrcd = dev->devr.xrcdn0;
0260
0261     if (init_attr->srq_type == IB_SRQT_TM) {
0262         in.tm_log_list_size =
0263             ilog2(init_attr->ext.tag_matching.max_num_tags) + 1;
0264         if (in.tm_log_list_size >
0265             MLX5_CAP_GEN(dev->mdev, log_tag_matching_list_sz)) {
0266             mlx5_ib_dbg(dev, "TM SRQ max_num_tags exceeding limit\n");
0267             err = -EINVAL;
0268             goto err_usr_kern_srq;
0269         }
0270         in.flags |= MLX5_SRQ_FLAG_RNDV;
0271     }
0272
0273     if (ib_srq_has_cq(init_attr->srq_type))
0274         in.cqn = to_mcq(init_attr->ext.cq)->mcq.cqn;
0275     else
0276         in.cqn = to_mcq(dev->devr.c0)->mcq.cqn;
0277
0278     in.pd = to_mpd(ib_srq->pd)->pdn;
0279     in.db_record = srq->db.dma;
0280     err = mlx5_cmd_create_srq(dev, &srq->msrq, &in);
0281     kvfree(in.pas);
0282     if (err) {
0283         mlx5_ib_dbg(dev, "create SRQ failed, err %d\n", err);
0284         goto err_usr_kern_srq;
0285     }
0286
0287     mlx5_ib_dbg(dev, "create SRQ with srqn 0x%x\n", srq->msrq.srqn);
0288
0289     srq->msrq.event = mlx5_ib_srq_event;
0290     srq->ibsrq.ext.xrc.srq_num = srq->msrq.srqn;
0291
0292     if (udata) {
0293         struct mlx5_ib_create_srq_resp resp = {
0294             .srqn = srq->msrq.srqn,
0295         };
0296
0297         if (ib_copy_to_udata(udata, &resp, min(udata->outlen,
0298                      sizeof(resp)))) {
0299             mlx5_ib_dbg(dev, "copy to user failed\n");
0300             err = -EFAULT;
0301             goto err_core;
0302         }
0303     }
0304
0305     init_attr->attr.max_wr = srq->msrq.max - 1;
0306
0307     return 0;
0308
0309 err_core:
0310     mlx5_cmd_destroy_srq(dev, &srq->msrq);
0311
0312 err_usr_kern_srq:
0313     if (udata)
0314         destroy_srq_user(ib_srq->pd, srq, udata);
0315     else
0316         destroy_srq_kernel(dev, srq);
0317
0318     return err;
0319 }
0320
0321 int mlx5_ib_modify_srq(struct ib_srq *ibsrq, struct ib_srq_attr *attr,
0322                enum ib_srq_attr_mask attr_mask, struct ib_udata *udata)
0323 {
0324     struct mlx5_ib_dev *dev = to_mdev(ibsrq->device);
0325     struct mlx5_ib_srq *srq = to_msrq(ibsrq);
0326     int ret;
0327
0328     /* We don't support resizing SRQs yet */
0329     if (attr_mask & IB_SRQ_MAX_WR)
0330         return -EINVAL;
0331
0332     if (attr_mask & IB_SRQ_LIMIT) {
0333         if (attr->srq_limit >= srq->msrq.max)
0334             return -EINVAL;
0335
0336         mutex_lock(&srq->mutex);
0337         ret = mlx5_cmd_arm_srq(dev, &srq->msrq, attr->srq_limit, 1);
0338         mutex_unlock(&srq->mutex);
0339
0340         if (ret)
0341             return ret;
0342     }
0343
0344     return 0;
0345 }
0346
0347 int mlx5_ib_query_srq(struct ib_srq *ibsrq, struct ib_srq_attr *srq_attr)
0348 {
0349     struct mlx5_ib_dev *dev = to_mdev(ibsrq->device);
0350     struct mlx5_ib_srq *srq = to_msrq(ibsrq);
0351     int ret;
0352     struct mlx5_srq_attr *out;
0353
0354     out = kzalloc(sizeof(*out), GFP_KERNEL);
0355     if (!out)
0356         return -ENOMEM;
0357
0358     ret = mlx5_cmd_query_srq(dev, &srq->msrq, out);
0359     if (ret)
0360         goto out_box;
0361
0362     srq_attr->srq_limit = out->lwm;
0363     srq_attr->max_wr    = srq->msrq.max - 1;
0364     srq_attr->max_sge   = srq->msrq.max_gs;
0365
0366 out_box:
0367     kfree(out);
0368     return ret;
0369 }
0370
0371 int mlx5_ib_destroy_srq(struct ib_srq *srq, struct ib_udata *udata)
0372 {
0373     struct mlx5_ib_dev *dev = to_mdev(srq->device);
0374     struct mlx5_ib_srq *msrq = to_msrq(srq);
0375     int ret;
0376
0377     ret = mlx5_cmd_destroy_srq(dev, &msrq->msrq);
0378     if (ret)
0379         return ret;
0380
0381     if (udata)
0382         destroy_srq_user(srq->pd, msrq, udata);
0383     else
0384         destroy_srq_kernel(dev, msrq);
0385     return 0;
0386 }
0387
0388 void mlx5_ib_free_srq_wqe(struct mlx5_ib_srq *srq, int wqe_index)
0389 {
0390     struct mlx5_wqe_srq_next_seg *next;
0391
0392     /* always called with interrupts disabled. */
0393     spin_lock(&srq->lock);
0394
0395     next = get_wqe(srq, srq->tail);
0396     next->next_wqe_index = cpu_to_be16(wqe_index);
0397     srq->tail = wqe_index;
0398
0399     spin_unlock(&srq->lock);
0400 }
0401
0402 int mlx5_ib_post_srq_recv(struct ib_srq *ibsrq, const struct ib_recv_wr *wr,
0403               const struct ib_recv_wr **bad_wr)
0404 {
0405     struct mlx5_ib_srq *srq = to_msrq(ibsrq);
0406     struct mlx5_wqe_srq_next_seg *next;
0407     struct mlx5_wqe_data_seg *scat;
0408     struct mlx5_ib_dev *dev = to_mdev(ibsrq->device);
0409     struct mlx5_core_dev *mdev = dev->mdev;
0410     unsigned long flags;
0411     int err = 0;
0412     int nreq;
0413     int i;
0414
0415     spin_lock_irqsave(&srq->lock, flags);
0416
0417     if (mdev->state == MLX5_DEVICE_STATE_INTERNAL_ERROR) {
0418         err = -EIO;
0419         *bad_wr = wr;
0420         goto out;
0421     }
0422
0423     for (nreq = 0; wr; nreq++, wr = wr->next) {
0424         if (unlikely(wr->num_sge > srq->msrq.max_gs)) {
0425             err = -EINVAL;
0426             *bad_wr = wr;
0427             break;
0428         }
0429
0430         if (unlikely(srq->head == srq->tail)) {
0431             err = -ENOMEM;
0432             *bad_wr = wr;
0433             break;
0434         }
0435
0436         srq->wrid[srq->head] = wr->wr_id;
0437
0438         next      = get_wqe(srq, srq->head);
0439         srq->head = be16_to_cpu(next->next_wqe_index);
0440         scat      = (struct mlx5_wqe_data_seg *)(next + 1);
0441
0442         for (i = 0; i < wr->num_sge; i++) {
0443             scat[i].byte_count = cpu_to_be32(wr->sg_list[i].length);
0444             scat[i].lkey       = cpu_to_be32(wr->sg_list[i].lkey);
0445             scat[i].addr       = cpu_to_be64(wr->sg_list[i].addr);
0446         }
0447
0448         if (i < srq->msrq.max_avail_gather) {
0449             scat[i].byte_count = 0;
0450             scat[i].lkey       = cpu_to_be32(MLX5_INVALID_LKEY);
0451             scat[i].addr       = 0;
0452         }
0453     }
0454
0455     if (likely(nreq)) {
0456         srq->wqe_ctr += nreq;
0457
0458         /* Make sure that descriptors are written before
0459          * doorbell record.
0460          */
0461         wmb();
0462
0463         *srq->db.db = cpu_to_be32(srq->wqe_ctr);
0464     }
0465 out:
0466     spin_unlock_irqrestore(&srq->lock, flags);
0467
0468     return err;
0469 }