hw/mlx4/qp.c

0001 /*
0002  * Copyright (c) 2007 Cisco Systems, Inc. All rights reserved.
0003  * Copyright (c) 2007, 2008 Mellanox Technologies. All rights reserved.
0004  *
0005  * This software is available to you under a choice of one of two
0006  * licenses.  You may choose to be licensed under the terms of the GNU
0007  * General Public License (GPL) Version 2, available from the file
0008  * COPYING in the main directory of this source tree, or the
0009  * OpenIB.org BSD license below:
0010  *
0011  *     Redistribution and use in source and binary forms, with or
0012  *     without modification, are permitted provided that the following
0013  *     conditions are met:
0014  *
0015  *      - Redistributions of source code must retain the above
0016  *        copyright notice, this list of conditions and the following
0017  *        disclaimer.
0018  *
0019  *      - Redistributions in binary form must reproduce the above
0020  *        copyright notice, this list of conditions and the following
0021  *        disclaimer in the documentation and/or other materials
0022  *        provided with the distribution.
0023  *
0024  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
0025  * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
0026  * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
0027  * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
0028  * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
0029  * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
0030  * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
0031  * SOFTWARE.
0032  */
0033
0034 #include <linux/log2.h>
0035 #include <linux/etherdevice.h>
0036 #include <net/ip.h>
0037 #include <linux/slab.h>
0038 #include <linux/netdevice.h>
0039
0040 #include <rdma/ib_cache.h>
0041 #include <rdma/ib_pack.h>
0042 #include <rdma/ib_addr.h>
0043 #include <rdma/ib_mad.h>
0044 #include <rdma/uverbs_ioctl.h>
0045
0046 #include <linux/mlx4/driver.h>
0047 #include <linux/mlx4/qp.h>
0048
0049 #include "mlx4_ib.h"
0050 #include <rdma/mlx4-abi.h>
0051
0052 static void mlx4_ib_lock_cqs(struct mlx4_ib_cq *send_cq,
0053                  struct mlx4_ib_cq *recv_cq);
0054 static void mlx4_ib_unlock_cqs(struct mlx4_ib_cq *send_cq,
0055                    struct mlx4_ib_cq *recv_cq);
0056 static int _mlx4_ib_modify_wq(struct ib_wq *ibwq, enum ib_wq_state new_state,
0057                   struct ib_udata *udata);
0058
0059 enum {
0060     MLX4_IB_ACK_REQ_FREQ    = 8,
0061 };
0062
0063 enum {
0064     MLX4_IB_DEFAULT_SCHED_QUEUE = 0x83,
0065     MLX4_IB_DEFAULT_QP0_SCHED_QUEUE = 0x3f,
0066     MLX4_IB_LINK_TYPE_IB        = 0,
0067     MLX4_IB_LINK_TYPE_ETH       = 1
0068 };
0069
0070 enum {
0071     MLX4_IB_MIN_SQ_STRIDE   = 6,
0072     MLX4_IB_CACHE_LINE_SIZE = 64,
0073 };
0074
0075 enum {
0076     MLX4_RAW_QP_MTU     = 7,
0077     MLX4_RAW_QP_MSGMAX  = 31,
0078 };
0079
0080 #ifndef ETH_ALEN
0081 #define ETH_ALEN        6
0082 #endif
0083
0084 static const __be32 mlx4_ib_opcode[] = {
0085     [IB_WR_SEND]                = cpu_to_be32(MLX4_OPCODE_SEND),
0086     [IB_WR_LSO]             = cpu_to_be32(MLX4_OPCODE_LSO),
0087     [IB_WR_SEND_WITH_IMM]           = cpu_to_be32(MLX4_OPCODE_SEND_IMM),
0088     [IB_WR_RDMA_WRITE]          = cpu_to_be32(MLX4_OPCODE_RDMA_WRITE),
0089     [IB_WR_RDMA_WRITE_WITH_IMM]     = cpu_to_be32(MLX4_OPCODE_RDMA_WRITE_IMM),
0090     [IB_WR_RDMA_READ]           = cpu_to_be32(MLX4_OPCODE_RDMA_READ),
0091     [IB_WR_ATOMIC_CMP_AND_SWP]      = cpu_to_be32(MLX4_OPCODE_ATOMIC_CS),
0092     [IB_WR_ATOMIC_FETCH_AND_ADD]        = cpu_to_be32(MLX4_OPCODE_ATOMIC_FA),
0093     [IB_WR_SEND_WITH_INV]           = cpu_to_be32(MLX4_OPCODE_SEND_INVAL),
0094     [IB_WR_LOCAL_INV]           = cpu_to_be32(MLX4_OPCODE_LOCAL_INVAL),
0095     [IB_WR_REG_MR]              = cpu_to_be32(MLX4_OPCODE_FMR),
0096     [IB_WR_MASKED_ATOMIC_CMP_AND_SWP]   = cpu_to_be32(MLX4_OPCODE_MASKED_ATOMIC_CS),
0097     [IB_WR_MASKED_ATOMIC_FETCH_AND_ADD] = cpu_to_be32(MLX4_OPCODE_MASKED_ATOMIC_FA),
0098 };
0099
0100 enum mlx4_ib_source_type {
0101     MLX4_IB_QP_SRC  = 0,
0102     MLX4_IB_RWQ_SRC = 1,
0103 };
0104
0105 static int is_tunnel_qp(struct mlx4_ib_dev *dev, struct mlx4_ib_qp *qp)
0106 {
0107     if (!mlx4_is_master(dev->dev))
0108         return 0;
0109
0110     return qp->mqp.qpn >= dev->dev->phys_caps.base_tunnel_sqpn &&
0111            qp->mqp.qpn < dev->dev->phys_caps.base_tunnel_sqpn +
0112         8 * MLX4_MFUNC_MAX;
0113 }
0114
0115 static int is_sqp(struct mlx4_ib_dev *dev, struct mlx4_ib_qp *qp)
0116 {
0117     int proxy_sqp = 0;
0118     int real_sqp = 0;
0119     int i;
0120     /* PPF or Native -- real SQP */
0121     real_sqp = ((mlx4_is_master(dev->dev) || !mlx4_is_mfunc(dev->dev)) &&
0122             qp->mqp.qpn >= dev->dev->phys_caps.base_sqpn &&
0123             qp->mqp.qpn <= dev->dev->phys_caps.base_sqpn + 3);
0124     if (real_sqp)
0125         return 1;
0126     /* VF or PF -- proxy SQP */
0127     if (mlx4_is_mfunc(dev->dev)) {
0128         for (i = 0; i < dev->dev->caps.num_ports; i++) {
0129             if (qp->mqp.qpn == dev->dev->caps.spec_qps[i].qp0_proxy ||
0130                 qp->mqp.qpn == dev->dev->caps.spec_qps[i].qp1_proxy) {
0131                 proxy_sqp = 1;
0132                 break;
0133             }
0134         }
0135     }
0136     if (proxy_sqp)
0137         return 1;
0138
0139     return !!(qp->flags & MLX4_IB_ROCE_V2_GSI_QP);
0140 }
0141
0142 /* used for INIT/CLOSE port logic */
0143 static int is_qp0(struct mlx4_ib_dev *dev, struct mlx4_ib_qp *qp)
0144 {
0145     int proxy_qp0 = 0;
0146     int real_qp0 = 0;
0147     int i;
0148     /* PPF or Native -- real QP0 */
0149     real_qp0 = ((mlx4_is_master(dev->dev) || !mlx4_is_mfunc(dev->dev)) &&
0150             qp->mqp.qpn >= dev->dev->phys_caps.base_sqpn &&
0151             qp->mqp.qpn <= dev->dev->phys_caps.base_sqpn + 1);
0152     if (real_qp0)
0153         return 1;
0154     /* VF or PF -- proxy QP0 */
0155     if (mlx4_is_mfunc(dev->dev)) {
0156         for (i = 0; i < dev->dev->caps.num_ports; i++) {
0157             if (qp->mqp.qpn == dev->dev->caps.spec_qps[i].qp0_proxy) {
0158                 proxy_qp0 = 1;
0159                 break;
0160             }
0161         }
0162     }
0163     return proxy_qp0;
0164 }
0165
0166 static void *get_wqe(struct mlx4_ib_qp *qp, int offset)
0167 {
0168     return mlx4_buf_offset(&qp->buf, offset);
0169 }
0170
0171 static void *get_recv_wqe(struct mlx4_ib_qp *qp, int n)
0172 {
0173     return get_wqe(qp, qp->rq.offset + (n << qp->rq.wqe_shift));
0174 }
0175
0176 static void *get_send_wqe(struct mlx4_ib_qp *qp, int n)
0177 {
0178     return get_wqe(qp, qp->sq.offset + (n << qp->sq.wqe_shift));
0179 }
0180
0181 /*
0182  * Stamp a SQ WQE so that it is invalid if prefetched by marking the
0183  * first four bytes of every 64 byte chunk with 0xffffffff, except for
0184  * the very first chunk of the WQE.
0185  */
0186 static void stamp_send_wqe(struct mlx4_ib_qp *qp, int n)
0187 {
0188     __be32 *wqe;
0189     int i;
0190     int s;
0191     void *buf;
0192     struct mlx4_wqe_ctrl_seg *ctrl;
0193
0194     buf = get_send_wqe(qp, n & (qp->sq.wqe_cnt - 1));
0195     ctrl = (struct mlx4_wqe_ctrl_seg *)buf;
0196     s = (ctrl->qpn_vlan.fence_size & 0x3f) << 4;
0197     for (i = 64; i < s; i += 64) {
0198         wqe = buf + i;
0199         *wqe = cpu_to_be32(0xffffffff);
0200     }
0201 }
0202
0203 static void mlx4_ib_qp_event(struct mlx4_qp *qp, enum mlx4_event type)
0204 {
0205     struct ib_event event;
0206     struct ib_qp *ibqp = &to_mibqp(qp)->ibqp;
0207
0208     if (type == MLX4_EVENT_TYPE_PATH_MIG)
0209         to_mibqp(qp)->port = to_mibqp(qp)->alt_port;
0210
0211     if (ibqp->event_handler) {
0212         event.device     = ibqp->device;
0213         event.element.qp = ibqp;
0214         switch (type) {
0215         case MLX4_EVENT_TYPE_PATH_MIG:
0216             event.event = IB_EVENT_PATH_MIG;
0217             break;
0218         case MLX4_EVENT_TYPE_COMM_EST:
0219             event.event = IB_EVENT_COMM_EST;
0220             break;
0221         case MLX4_EVENT_TYPE_SQ_DRAINED:
0222             event.event = IB_EVENT_SQ_DRAINED;
0223             break;
0224         case MLX4_EVENT_TYPE_SRQ_QP_LAST_WQE:
0225             event.event = IB_EVENT_QP_LAST_WQE_REACHED;
0226             break;
0227         case MLX4_EVENT_TYPE_WQ_CATAS_ERROR:
0228             event.event = IB_EVENT_QP_FATAL;
0229             break;
0230         case MLX4_EVENT_TYPE_PATH_MIG_FAILED:
0231             event.event = IB_EVENT_PATH_MIG_ERR;
0232             break;
0233         case MLX4_EVENT_TYPE_WQ_INVAL_REQ_ERROR:
0234             event.event = IB_EVENT_QP_REQ_ERR;
0235             break;
0236         case MLX4_EVENT_TYPE_WQ_ACCESS_ERROR:
0237             event.event = IB_EVENT_QP_ACCESS_ERR;
0238             break;
0239         default:
0240             pr_warn("Unexpected event type %d "
0241                    "on QP %06x\n", type, qp->qpn);
0242             return;
0243         }
0244
0245         ibqp->event_handler(&event, ibqp->qp_context);
0246     }
0247 }
0248
0249 static void mlx4_ib_wq_event(struct mlx4_qp *qp, enum mlx4_event type)
0250 {
0251     pr_warn_ratelimited("Unexpected event type %d on WQ 0x%06x. Events are not supported for WQs\n",
0252                 type, qp->qpn);
0253 }
0254
0255 static int send_wqe_overhead(enum mlx4_ib_qp_type type, u32 flags)
0256 {
0257     /*
0258      * UD WQEs must have a datagram segment.
0259      * RC and UC WQEs might have a remote address segment.
0260      * MLX WQEs need two extra inline data segments (for the UD
0261      * header and space for the ICRC).
0262      */
0263     switch (type) {
0264     case MLX4_IB_QPT_UD:
0265         return sizeof (struct mlx4_wqe_ctrl_seg) +
0266             sizeof (struct mlx4_wqe_datagram_seg) +
0267             ((flags & MLX4_IB_QP_LSO) ? MLX4_IB_LSO_HEADER_SPARE : 0);
0268     case MLX4_IB_QPT_PROXY_SMI_OWNER:
0269     case MLX4_IB_QPT_PROXY_SMI:
0270     case MLX4_IB_QPT_PROXY_GSI:
0271         return sizeof (struct mlx4_wqe_ctrl_seg) +
0272             sizeof (struct mlx4_wqe_datagram_seg) + 64;
0273     case MLX4_IB_QPT_TUN_SMI_OWNER:
0274     case MLX4_IB_QPT_TUN_GSI:
0275         return sizeof (struct mlx4_wqe_ctrl_seg) +
0276             sizeof (struct mlx4_wqe_datagram_seg);
0277
0278     case MLX4_IB_QPT_UC:
0279         return sizeof (struct mlx4_wqe_ctrl_seg) +
0280             sizeof (struct mlx4_wqe_raddr_seg);
0281     case MLX4_IB_QPT_RC:
0282         return sizeof (struct mlx4_wqe_ctrl_seg) +
0283             sizeof (struct mlx4_wqe_masked_atomic_seg) +
0284             sizeof (struct mlx4_wqe_raddr_seg);
0285     case MLX4_IB_QPT_SMI:
0286     case MLX4_IB_QPT_GSI:
0287         return sizeof (struct mlx4_wqe_ctrl_seg) +
0288             ALIGN(MLX4_IB_UD_HEADER_SIZE +
0289                   DIV_ROUND_UP(MLX4_IB_UD_HEADER_SIZE,
0290                        MLX4_INLINE_ALIGN) *
0291                   sizeof (struct mlx4_wqe_inline_seg),
0292                   sizeof (struct mlx4_wqe_data_seg)) +
0293             ALIGN(4 +
0294                   sizeof (struct mlx4_wqe_inline_seg),
0295                   sizeof (struct mlx4_wqe_data_seg));
0296     default:
0297         return sizeof (struct mlx4_wqe_ctrl_seg);
0298     }
0299 }
0300
0301 static int set_rq_size(struct mlx4_ib_dev *dev, struct ib_qp_cap *cap,
0302                bool is_user, bool has_rq, struct mlx4_ib_qp *qp,
0303                u32 inl_recv_sz)
0304 {
0305     /* Sanity check RQ size before proceeding */
0306     if (cap->max_recv_wr > dev->dev->caps.max_wqes - MLX4_IB_SQ_MAX_SPARE ||
0307         cap->max_recv_sge > min(dev->dev->caps.max_sq_sg, dev->dev->caps.max_rq_sg))
0308         return -EINVAL;
0309
0310     if (!has_rq) {
0311         if (cap->max_recv_wr || inl_recv_sz)
0312             return -EINVAL;
0313
0314         qp->rq.wqe_cnt = qp->rq.max_gs = 0;
0315     } else {
0316         u32 max_inl_recv_sz = dev->dev->caps.max_rq_sg *
0317             sizeof(struct mlx4_wqe_data_seg);
0318         u32 wqe_size;
0319
0320         /* HW requires >= 1 RQ entry with >= 1 gather entry */
0321         if (is_user && (!cap->max_recv_wr || !cap->max_recv_sge ||
0322                 inl_recv_sz > max_inl_recv_sz))
0323             return -EINVAL;
0324
0325         qp->rq.wqe_cnt   = roundup_pow_of_two(max(1U, cap->max_recv_wr));
0326         qp->rq.max_gs    = roundup_pow_of_two(max(1U, cap->max_recv_sge));
0327         wqe_size = qp->rq.max_gs * sizeof(struct mlx4_wqe_data_seg);
0328         qp->rq.wqe_shift = ilog2(max_t(u32, wqe_size, inl_recv_sz));
0329     }
0330
0331     /* leave userspace return values as they were, so as not to break ABI */
0332     if (is_user) {
0333         cap->max_recv_wr  = qp->rq.max_post = qp->rq.wqe_cnt;
0334         cap->max_recv_sge = qp->rq.max_gs;
0335     } else {
0336         cap->max_recv_wr  = qp->rq.max_post =
0337             min(dev->dev->caps.max_wqes - MLX4_IB_SQ_MAX_SPARE, qp->rq.wqe_cnt);
0338         cap->max_recv_sge = min(qp->rq.max_gs,
0339                     min(dev->dev->caps.max_sq_sg,
0340                         dev->dev->caps.max_rq_sg));
0341     }
0342
0343     return 0;
0344 }
0345
0346 static int set_kernel_sq_size(struct mlx4_ib_dev *dev, struct ib_qp_cap *cap,
0347                   enum mlx4_ib_qp_type type, struct mlx4_ib_qp *qp)
0348 {
0349     int s;
0350
0351     /* Sanity check SQ size before proceeding */
0352     if (cap->max_send_wr  > (dev->dev->caps.max_wqes - MLX4_IB_SQ_MAX_SPARE) ||
0353         cap->max_send_sge > min(dev->dev->caps.max_sq_sg, dev->dev->caps.max_rq_sg) ||
0354         cap->max_inline_data + send_wqe_overhead(type, qp->flags) +
0355         sizeof (struct mlx4_wqe_inline_seg) > dev->dev->caps.max_sq_desc_sz)
0356         return -EINVAL;
0357
0358     /*
0359      * For MLX transport we need 2 extra S/G entries:
0360      * one for the header and one for the checksum at the end
0361      */
0362     if ((type == MLX4_IB_QPT_SMI || type == MLX4_IB_QPT_GSI ||
0363          type & (MLX4_IB_QPT_PROXY_SMI_OWNER | MLX4_IB_QPT_TUN_SMI_OWNER)) &&
0364         cap->max_send_sge + 2 > dev->dev->caps.max_sq_sg)
0365         return -EINVAL;
0366
0367     s = max(cap->max_send_sge * sizeof (struct mlx4_wqe_data_seg),
0368         cap->max_inline_data + sizeof (struct mlx4_wqe_inline_seg)) +
0369         send_wqe_overhead(type, qp->flags);
0370
0371     if (s > dev->dev->caps.max_sq_desc_sz)
0372         return -EINVAL;
0373
0374     qp->sq.wqe_shift = ilog2(roundup_pow_of_two(s));
0375
0376     /*
0377      * We need to leave 2 KB + 1 WR of headroom in the SQ to
0378      * allow HW to prefetch.
0379      */
0380     qp->sq_spare_wqes = MLX4_IB_SQ_HEADROOM(qp->sq.wqe_shift);
0381     qp->sq.wqe_cnt = roundup_pow_of_two(cap->max_send_wr +
0382                         qp->sq_spare_wqes);
0383
0384     qp->sq.max_gs =
0385         (min(dev->dev->caps.max_sq_desc_sz,
0386              (1 << qp->sq.wqe_shift)) -
0387          send_wqe_overhead(type, qp->flags)) /
0388         sizeof (struct mlx4_wqe_data_seg);
0389
0390     qp->buf_size = (qp->rq.wqe_cnt << qp->rq.wqe_shift) +
0391         (qp->sq.wqe_cnt << qp->sq.wqe_shift);
0392     if (qp->rq.wqe_shift > qp->sq.wqe_shift) {
0393         qp->rq.offset = 0;
0394         qp->sq.offset = qp->rq.wqe_cnt << qp->rq.wqe_shift;
0395     } else {
0396         qp->rq.offset = qp->sq.wqe_cnt << qp->sq.wqe_shift;
0397         qp->sq.offset = 0;
0398     }
0399
0400     cap->max_send_wr  = qp->sq.max_post =
0401         qp->sq.wqe_cnt - qp->sq_spare_wqes;
0402     cap->max_send_sge = min(qp->sq.max_gs,
0403                 min(dev->dev->caps.max_sq_sg,
0404                     dev->dev->caps.max_rq_sg));
0405     /* We don't support inline sends for kernel QPs (yet) */
0406     cap->max_inline_data = 0;
0407
0408     return 0;
0409 }
0410
0411 static int set_user_sq_size(struct mlx4_ib_dev *dev,
0412                 struct mlx4_ib_qp *qp,
0413                 struct mlx4_ib_create_qp *ucmd)
0414 {
0415     /* Sanity check SQ size before proceeding */
0416     if ((1 << ucmd->log_sq_bb_count) > dev->dev->caps.max_wqes   ||
0417         ucmd->log_sq_stride >
0418         ilog2(roundup_pow_of_two(dev->dev->caps.max_sq_desc_sz)) ||
0419         ucmd->log_sq_stride < MLX4_IB_MIN_SQ_STRIDE)
0420         return -EINVAL;
0421
0422     qp->sq.wqe_cnt   = 1 << ucmd->log_sq_bb_count;
0423     qp->sq.wqe_shift = ucmd->log_sq_stride;
0424
0425     qp->buf_size = (qp->rq.wqe_cnt << qp->rq.wqe_shift) +
0426         (qp->sq.wqe_cnt << qp->sq.wqe_shift);
0427
0428     return 0;
0429 }
0430
0431 static int alloc_proxy_bufs(struct ib_device *dev, struct mlx4_ib_qp *qp)
0432 {
0433     int i;
0434
0435     qp->sqp_proxy_rcv =
0436         kmalloc_array(qp->rq.wqe_cnt, sizeof(struct mlx4_ib_buf),
0437                   GFP_KERNEL);
0438     if (!qp->sqp_proxy_rcv)
0439         return -ENOMEM;
0440     for (i = 0; i < qp->rq.wqe_cnt; i++) {
0441         qp->sqp_proxy_rcv[i].addr =
0442             kmalloc(sizeof (struct mlx4_ib_proxy_sqp_hdr),
0443                 GFP_KERNEL);
0444         if (!qp->sqp_proxy_rcv[i].addr)
0445             goto err;
0446         qp->sqp_proxy_rcv[i].map =
0447             ib_dma_map_single(dev, qp->sqp_proxy_rcv[i].addr,
0448                       sizeof (struct mlx4_ib_proxy_sqp_hdr),
0449                       DMA_FROM_DEVICE);
0450         if (ib_dma_mapping_error(dev, qp->sqp_proxy_rcv[i].map)) {
0451             kfree(qp->sqp_proxy_rcv[i].addr);
0452             goto err;
0453         }
0454     }
0455     return 0;
0456
0457 err:
0458     while (i > 0) {
0459         --i;
0460         ib_dma_unmap_single(dev, qp->sqp_proxy_rcv[i].map,
0461                     sizeof (struct mlx4_ib_proxy_sqp_hdr),
0462                     DMA_FROM_DEVICE);
0463         kfree(qp->sqp_proxy_rcv[i].addr);
0464     }
0465     kfree(qp->sqp_proxy_rcv);
0466     qp->sqp_proxy_rcv = NULL;
0467     return -ENOMEM;
0468 }
0469
0470 static void free_proxy_bufs(struct ib_device *dev, struct mlx4_ib_qp *qp)
0471 {
0472     int i;
0473
0474     for (i = 0; i < qp->rq.wqe_cnt; i++) {
0475         ib_dma_unmap_single(dev, qp->sqp_proxy_rcv[i].map,
0476                     sizeof (struct mlx4_ib_proxy_sqp_hdr),
0477                     DMA_FROM_DEVICE);
0478         kfree(qp->sqp_proxy_rcv[i].addr);
0479     }
0480     kfree(qp->sqp_proxy_rcv);
0481 }
0482
0483 static bool qp_has_rq(struct ib_qp_init_attr *attr)
0484 {
0485     if (attr->qp_type == IB_QPT_XRC_INI || attr->qp_type == IB_QPT_XRC_TGT)
0486         return false;
0487
0488     return !attr->srq;
0489 }
0490
0491 static int qp0_enabled_vf(struct mlx4_dev *dev, int qpn)
0492 {
0493     int i;
0494     for (i = 0; i < dev->caps.num_ports; i++) {
0495         if (qpn == dev->caps.spec_qps[i].qp0_proxy)
0496             return !!dev->caps.spec_qps[i].qp0_qkey;
0497     }
0498     return 0;
0499 }
0500
0501 static void mlx4_ib_free_qp_counter(struct mlx4_ib_dev *dev,
0502                     struct mlx4_ib_qp *qp)
0503 {
0504     mutex_lock(&dev->counters_table[qp->port - 1].mutex);
0505     mlx4_counter_free(dev->dev, qp->counter_index->index);
0506     list_del(&qp->counter_index->list);
0507     mutex_unlock(&dev->counters_table[qp->port - 1].mutex);
0508
0509     kfree(qp->counter_index);
0510     qp->counter_index = NULL;
0511 }
0512
0513 static int set_qp_rss(struct mlx4_ib_dev *dev, struct mlx4_ib_rss *rss_ctx,
0514               struct ib_qp_init_attr *init_attr,
0515               struct mlx4_ib_create_qp_rss *ucmd)
0516 {
0517     rss_ctx->base_qpn_tbl_sz = init_attr->rwq_ind_tbl->ind_tbl[0]->wq_num |
0518         (init_attr->rwq_ind_tbl->log_ind_tbl_size << 24);
0519
0520     if ((ucmd->rx_hash_function == MLX4_IB_RX_HASH_FUNC_TOEPLITZ) &&
0521         (dev->dev->caps.flags2 & MLX4_DEV_CAP_FLAG2_RSS_TOP)) {
0522         memcpy(rss_ctx->rss_key, ucmd->rx_hash_key,
0523                MLX4_EN_RSS_KEY_SIZE);
0524     } else {
0525         pr_debug("RX Hash function is not supported\n");
0526         return (-EOPNOTSUPP);
0527     }
0528
0529     if (ucmd->rx_hash_fields_mask & ~(MLX4_IB_RX_HASH_SRC_IPV4  |
0530                       MLX4_IB_RX_HASH_DST_IPV4  |
0531                       MLX4_IB_RX_HASH_SRC_IPV6  |
0532                       MLX4_IB_RX_HASH_DST_IPV6  |
0533                       MLX4_IB_RX_HASH_SRC_PORT_TCP  |
0534                       MLX4_IB_RX_HASH_DST_PORT_TCP  |
0535                       MLX4_IB_RX_HASH_SRC_PORT_UDP  |
0536                       MLX4_IB_RX_HASH_DST_PORT_UDP  |
0537                       MLX4_IB_RX_HASH_INNER)) {
0538         pr_debug("RX Hash fields_mask has unsupported mask (0x%llx)\n",
0539              ucmd->rx_hash_fields_mask);
0540         return (-EOPNOTSUPP);
0541     }
0542
0543     if ((ucmd->rx_hash_fields_mask & MLX4_IB_RX_HASH_SRC_IPV4) &&
0544         (ucmd->rx_hash_fields_mask & MLX4_IB_RX_HASH_DST_IPV4)) {
0545         rss_ctx->flags = MLX4_RSS_IPV4;
0546     } else if ((ucmd->rx_hash_fields_mask & MLX4_IB_RX_HASH_SRC_IPV4) ||
0547            (ucmd->rx_hash_fields_mask & MLX4_IB_RX_HASH_DST_IPV4)) {
0548         pr_debug("RX Hash fields_mask is not supported - both IPv4 SRC and DST must be set\n");
0549         return (-EOPNOTSUPP);
0550     }
0551
0552     if ((ucmd->rx_hash_fields_mask & MLX4_IB_RX_HASH_SRC_IPV6) &&
0553         (ucmd->rx_hash_fields_mask & MLX4_IB_RX_HASH_DST_IPV6)) {
0554         rss_ctx->flags |= MLX4_RSS_IPV6;
0555     } else if ((ucmd->rx_hash_fields_mask & MLX4_IB_RX_HASH_SRC_IPV6) ||
0556            (ucmd->rx_hash_fields_mask & MLX4_IB_RX_HASH_DST_IPV6)) {
0557         pr_debug("RX Hash fields_mask is not supported - both IPv6 SRC and DST must be set\n");
0558         return (-EOPNOTSUPP);
0559     }
0560
0561     if ((ucmd->rx_hash_fields_mask & MLX4_IB_RX_HASH_SRC_PORT_UDP) &&
0562         (ucmd->rx_hash_fields_mask & MLX4_IB_RX_HASH_DST_PORT_UDP)) {
0563         if (!(dev->dev->caps.flags & MLX4_DEV_CAP_FLAG_UDP_RSS)) {
0564             pr_debug("RX Hash fields_mask for UDP is not supported\n");
0565             return (-EOPNOTSUPP);
0566         }
0567
0568         if (rss_ctx->flags & MLX4_RSS_IPV4)
0569             rss_ctx->flags |= MLX4_RSS_UDP_IPV4;
0570         if (rss_ctx->flags & MLX4_RSS_IPV6)
0571             rss_ctx->flags |= MLX4_RSS_UDP_IPV6;
0572         if (!(rss_ctx->flags & (MLX4_RSS_IPV6 | MLX4_RSS_IPV4))) {
0573             pr_debug("RX Hash fields_mask is not supported - UDP must be set with IPv4 or IPv6\n");
0574             return (-EOPNOTSUPP);
0575         }
0576     } else if ((ucmd->rx_hash_fields_mask & MLX4_IB_RX_HASH_SRC_PORT_UDP) ||
0577            (ucmd->rx_hash_fields_mask & MLX4_IB_RX_HASH_DST_PORT_UDP)) {
0578         pr_debug("RX Hash fields_mask is not supported - both UDP SRC and DST must be set\n");
0579         return (-EOPNOTSUPP);
0580     }
0581
0582     if ((ucmd->rx_hash_fields_mask & MLX4_IB_RX_HASH_SRC_PORT_TCP) &&
0583         (ucmd->rx_hash_fields_mask & MLX4_IB_RX_HASH_DST_PORT_TCP)) {
0584         if (rss_ctx->flags & MLX4_RSS_IPV4)
0585             rss_ctx->flags |= MLX4_RSS_TCP_IPV4;
0586         if (rss_ctx->flags & MLX4_RSS_IPV6)
0587             rss_ctx->flags |= MLX4_RSS_TCP_IPV6;
0588         if (!(rss_ctx->flags & (MLX4_RSS_IPV6 | MLX4_RSS_IPV4))) {
0589             pr_debug("RX Hash fields_mask is not supported - TCP must be set with IPv4 or IPv6\n");
0590             return (-EOPNOTSUPP);
0591         }
0592     } else if ((ucmd->rx_hash_fields_mask & MLX4_IB_RX_HASH_SRC_PORT_TCP) ||
0593            (ucmd->rx_hash_fields_mask & MLX4_IB_RX_HASH_DST_PORT_TCP)) {
0594         pr_debug("RX Hash fields_mask is not supported - both TCP SRC and DST must be set\n");
0595         return (-EOPNOTSUPP);
0596     }
0597
0598     if (ucmd->rx_hash_fields_mask & MLX4_IB_RX_HASH_INNER) {
0599         if (dev->dev->caps.tunnel_offload_mode ==
0600             MLX4_TUNNEL_OFFLOAD_MODE_VXLAN) {
0601             /*
0602              * Hash according to inner headers if exist, otherwise
0603              * according to outer headers.
0604              */
0605             rss_ctx->flags |= MLX4_RSS_BY_INNER_HEADERS_IPONLY;
0606         } else {
0607             pr_debug("RSS Hash for inner headers isn't supported\n");
0608             return (-EOPNOTSUPP);
0609         }
0610     }
0611
0612     return 0;
0613 }
0614
0615 static int create_qp_rss(struct mlx4_ib_dev *dev,
0616              struct ib_qp_init_attr *init_attr,
0617              struct mlx4_ib_create_qp_rss *ucmd,
0618              struct mlx4_ib_qp *qp)
0619 {
0620     int qpn;
0621     int err;
0622
0623     qp->mqp.usage = MLX4_RES_USAGE_USER_VERBS;
0624
0625     err = mlx4_qp_reserve_range(dev->dev, 1, 1, &qpn, 0, qp->mqp.usage);
0626     if (err)
0627         return err;
0628
0629     err = mlx4_qp_alloc(dev->dev, qpn, &qp->mqp);
0630     if (err)
0631         goto err_qpn;
0632
0633     INIT_LIST_HEAD(&qp->gid_list);
0634     INIT_LIST_HEAD(&qp->steering_rules);
0635
0636     qp->mlx4_ib_qp_type = MLX4_IB_QPT_RAW_PACKET;
0637     qp->state = IB_QPS_RESET;
0638
0639     /* Set dummy send resources to be compatible with HV and PRM */
0640     qp->sq_no_prefetch = 1;
0641     qp->sq.wqe_cnt = 1;
0642     qp->sq.wqe_shift = MLX4_IB_MIN_SQ_STRIDE;
0643     qp->buf_size = qp->sq.wqe_cnt << MLX4_IB_MIN_SQ_STRIDE;
0644     qp->mtt = (to_mqp(
0645            (struct ib_qp *)init_attr->rwq_ind_tbl->ind_tbl[0]))->mtt;
0646
0647     qp->rss_ctx = kzalloc(sizeof(*qp->rss_ctx), GFP_KERNEL);
0648     if (!qp->rss_ctx) {
0649         err = -ENOMEM;
0650         goto err_qp_alloc;
0651     }
0652
0653     err = set_qp_rss(dev, qp->rss_ctx, init_attr, ucmd);
0654     if (err)
0655         goto err;
0656
0657     return 0;
0658
0659 err:
0660     kfree(qp->rss_ctx);
0661
0662 err_qp_alloc:
0663     mlx4_qp_remove(dev->dev, &qp->mqp);
0664     mlx4_qp_free(dev->dev, &qp->mqp);
0665
0666 err_qpn:
0667     mlx4_qp_release_range(dev->dev, qpn, 1);
0668     return err;
0669 }
0670
0671 static int _mlx4_ib_create_qp_rss(struct ib_pd *pd, struct mlx4_ib_qp *qp,
0672                   struct ib_qp_init_attr *init_attr,
0673                   struct ib_udata *udata)
0674 {
0675     struct mlx4_ib_create_qp_rss ucmd = {};
0676     size_t required_cmd_sz;
0677     int err;
0678
0679     if (!udata) {
0680         pr_debug("RSS QP with NULL udata\n");
0681         return -EINVAL;
0682     }
0683
0684     if (udata->outlen)
0685         return -EOPNOTSUPP;
0686
0687     required_cmd_sz = offsetof(typeof(ucmd), reserved1) +
0688                     sizeof(ucmd.reserved1);
0689     if (udata->inlen < required_cmd_sz) {
0690         pr_debug("invalid inlen\n");
0691         return -EINVAL;
0692     }
0693
0694     if (ib_copy_from_udata(&ucmd, udata, min(sizeof(ucmd), udata->inlen))) {
0695         pr_debug("copy failed\n");
0696         return -EFAULT;
0697     }
0698
0699     if (memchr_inv(ucmd.reserved, 0, sizeof(ucmd.reserved)))
0700         return -EOPNOTSUPP;
0701
0702     if (ucmd.comp_mask || ucmd.reserved1)
0703         return -EOPNOTSUPP;
0704
0705     if (udata->inlen > sizeof(ucmd) &&
0706         !ib_is_udata_cleared(udata, sizeof(ucmd),
0707                  udata->inlen - sizeof(ucmd))) {
0708         pr_debug("inlen is not supported\n");
0709         return -EOPNOTSUPP;
0710     }
0711
0712     if (init_attr->qp_type != IB_QPT_RAW_PACKET) {
0713         pr_debug("RSS QP with unsupported QP type %d\n",
0714              init_attr->qp_type);
0715         return -EOPNOTSUPP;
0716     }
0717
0718     if (init_attr->create_flags) {
0719         pr_debug("RSS QP doesn't support create flags\n");
0720         return -EOPNOTSUPP;
0721     }
0722
0723     if (init_attr->send_cq || init_attr->cap.max_send_wr) {
0724         pr_debug("RSS QP with unsupported send attributes\n");
0725         return -EOPNOTSUPP;
0726     }
0727
0728     qp->pri.vid = 0xFFFF;
0729     qp->alt.vid = 0xFFFF;
0730
0731     err = create_qp_rss(to_mdev(pd->device), init_attr, &ucmd, qp);
0732     if (err)
0733         return err;
0734
0735     qp->ibqp.qp_num = qp->mqp.qpn;
0736     return 0;
0737 }
0738
0739 /*
0740  * This function allocates a WQN from a range which is consecutive and aligned
0741  * to its size. In case the range is full, then it creates a new range and
0742  * allocates WQN from it. The new range will be used for following allocations.
0743  */
0744 static int mlx4_ib_alloc_wqn(struct mlx4_ib_ucontext *context,
0745                  struct mlx4_ib_qp *qp, int range_size, int *wqn)
0746 {
0747     struct mlx4_ib_dev *dev = to_mdev(context->ibucontext.device);
0748     struct mlx4_wqn_range *range;
0749     int err = 0;
0750
0751     mutex_lock(&context->wqn_ranges_mutex);
0752
0753     range = list_first_entry_or_null(&context->wqn_ranges_list,
0754                      struct mlx4_wqn_range, list);
0755
0756     if (!range || (range->refcount == range->size) || range->dirty) {
0757         range = kzalloc(sizeof(*range), GFP_KERNEL);
0758         if (!range) {
0759             err = -ENOMEM;
0760             goto out;
0761         }
0762
0763         err = mlx4_qp_reserve_range(dev->dev, range_size,
0764                         range_size, &range->base_wqn, 0,
0765                         qp->mqp.usage);
0766         if (err) {
0767             kfree(range);
0768             goto out;
0769         }
0770
0771         range->size = range_size;
0772         list_add(&range->list, &context->wqn_ranges_list);
0773     } else if (range_size != 1) {
0774         /*
0775          * Requesting a new range (>1) when last range is still open, is
0776          * not valid.
0777          */
0778         err = -EINVAL;
0779         goto out;
0780     }
0781
0782     qp->wqn_range = range;
0783
0784     *wqn = range->base_wqn + range->refcount;
0785
0786     range->refcount++;
0787
0788 out:
0789     mutex_unlock(&context->wqn_ranges_mutex);
0790
0791     return err;
0792 }
0793
0794 static void mlx4_ib_release_wqn(struct mlx4_ib_ucontext *context,
0795                 struct mlx4_ib_qp *qp, bool dirty_release)
0796 {
0797     struct mlx4_ib_dev *dev = to_mdev(context->ibucontext.device);
0798     struct mlx4_wqn_range *range;
0799
0800     mutex_lock(&context->wqn_ranges_mutex);
0801
0802     range = qp->wqn_range;
0803
0804     range->refcount--;
0805     if (!range->refcount) {
0806         mlx4_qp_release_range(dev->dev, range->base_wqn,
0807                       range->size);
0808         list_del(&range->list);
0809         kfree(range);
0810     } else if (dirty_release) {
0811     /*
0812      * A range which one of its WQNs is destroyed, won't be able to be
0813      * reused for further WQN allocations.
0814      * The next created WQ will allocate a new range.
0815      */
0816         range->dirty = true;
0817     }
0818
0819     mutex_unlock(&context->wqn_ranges_mutex);
0820 }
0821
0822 static int create_rq(struct ib_pd *pd, struct ib_qp_init_attr *init_attr,
0823              struct ib_udata *udata, struct mlx4_ib_qp *qp)
0824 {
0825     struct mlx4_ib_dev *dev = to_mdev(pd->device);
0826     int qpn;
0827     int err;
0828     struct mlx4_ib_ucontext *context = rdma_udata_to_drv_context(
0829         udata, struct mlx4_ib_ucontext, ibucontext);
0830     struct mlx4_ib_cq *mcq;
0831     unsigned long flags;
0832     int range_size;
0833     struct mlx4_ib_create_wq wq;
0834     size_t copy_len;
0835     int shift;
0836     int n;
0837
0838     qp->mlx4_ib_qp_type = MLX4_IB_QPT_RAW_PACKET;
0839
0840     spin_lock_init(&qp->sq.lock);
0841     spin_lock_init(&qp->rq.lock);
0842     INIT_LIST_HEAD(&qp->gid_list);
0843     INIT_LIST_HEAD(&qp->steering_rules);
0844
0845     qp->state = IB_QPS_RESET;
0846
0847     copy_len = min(sizeof(struct mlx4_ib_create_wq), udata->inlen);
0848
0849     if (ib_copy_from_udata(&wq, udata, copy_len)) {
0850         err = -EFAULT;
0851         goto err;
0852     }
0853
0854     if (wq.comp_mask || wq.reserved[0] || wq.reserved[1] ||
0855         wq.reserved[2]) {
0856         pr_debug("user command isn't supported\n");
0857         err = -EOPNOTSUPP;
0858         goto err;
0859     }
0860
0861     if (wq.log_range_size > ilog2(dev->dev->caps.max_rss_tbl_sz)) {
0862         pr_debug("WQN range size must be equal or smaller than %d\n",
0863              dev->dev->caps.max_rss_tbl_sz);
0864         err = -EOPNOTSUPP;
0865         goto err;
0866     }
0867     range_size = 1 << wq.log_range_size;
0868
0869     if (init_attr->create_flags & IB_QP_CREATE_SCATTER_FCS)
0870         qp->flags |= MLX4_IB_QP_SCATTER_FCS;
0871
0872     err = set_rq_size(dev, &init_attr->cap, true, true, qp, qp->inl_recv_sz);
0873     if (err)
0874         goto err;
0875
0876     qp->sq_no_prefetch = 1;
0877     qp->sq.wqe_cnt = 1;
0878     qp->sq.wqe_shift = MLX4_IB_MIN_SQ_STRIDE;
0879     qp->buf_size = (qp->rq.wqe_cnt << qp->rq.wqe_shift) +
0880                (qp->sq.wqe_cnt << qp->sq.wqe_shift);
0881
0882     qp->umem = ib_umem_get(pd->device, wq.buf_addr, qp->buf_size, 0);
0883     if (IS_ERR(qp->umem)) {
0884         err = PTR_ERR(qp->umem);
0885         goto err;
0886     }
0887
0888     shift = mlx4_ib_umem_calc_optimal_mtt_size(qp->umem, 0, &n);
0889     err = mlx4_mtt_init(dev->dev, n, shift, &qp->mtt);
0890
0891     if (err)
0892         goto err_buf;
0893
0894     err = mlx4_ib_umem_write_mtt(dev, &qp->mtt, qp->umem);
0895     if (err)
0896         goto err_mtt;
0897
0898     err = mlx4_ib_db_map_user(udata, wq.db_addr, &qp->db);
0899     if (err)
0900         goto err_mtt;
0901     qp->mqp.usage = MLX4_RES_USAGE_USER_VERBS;
0902
0903     err = mlx4_ib_alloc_wqn(context, qp, range_size, &qpn);
0904     if (err)
0905         goto err_wrid;
0906
0907     err = mlx4_qp_alloc(dev->dev, qpn, &qp->mqp);
0908     if (err)
0909         goto err_qpn;
0910
0911     /*
0912      * Hardware wants QPN written in big-endian order (after
0913      * shifting) for send doorbell.  Precompute this value to save
0914      * a little bit when posting sends.
0915      */
0916     qp->doorbell_qpn = swab32(qp->mqp.qpn << 8);
0917
0918     qp->mqp.event = mlx4_ib_wq_event;
0919
0920     spin_lock_irqsave(&dev->reset_flow_resource_lock, flags);
0921     mlx4_ib_lock_cqs(to_mcq(init_attr->send_cq),
0922              to_mcq(init_attr->recv_cq));
0923     /* Maintain device to QPs access, needed for further handling
0924      * via reset flow
0925      */
0926     list_add_tail(&qp->qps_list, &dev->qp_list);
0927     /* Maintain CQ to QPs access, needed for further handling
0928      * via reset flow
0929      */
0930     mcq = to_mcq(init_attr->send_cq);
0931     list_add_tail(&qp->cq_send_list, &mcq->send_qp_list);
0932     mcq = to_mcq(init_attr->recv_cq);
0933     list_add_tail(&qp->cq_recv_list, &mcq->recv_qp_list);
0934     mlx4_ib_unlock_cqs(to_mcq(init_attr->send_cq),
0935                to_mcq(init_attr->recv_cq));
0936     spin_unlock_irqrestore(&dev->reset_flow_resource_lock, flags);
0937     return 0;
0938
0939 err_qpn:
0940     mlx4_ib_release_wqn(context, qp, 0);
0941 err_wrid:
0942     mlx4_ib_db_unmap_user(context, &qp->db);
0943
0944 err_mtt:
0945     mlx4_mtt_cleanup(dev->dev, &qp->mtt);
0946 err_buf:
0947     ib_umem_release(qp->umem);
0948 err:
0949     return err;
0950 }
0951
0952 static int create_qp_common(struct ib_pd *pd, struct ib_qp_init_attr *init_attr,
0953                 struct ib_udata *udata, int sqpn,
0954                 struct mlx4_ib_qp *qp)
0955 {
0956     struct mlx4_ib_dev *dev = to_mdev(pd->device);
0957     int qpn;
0958     int err;
0959     struct mlx4_ib_ucontext *context = rdma_udata_to_drv_context(
0960         udata, struct mlx4_ib_ucontext, ibucontext);
0961     enum mlx4_ib_qp_type qp_type = (enum mlx4_ib_qp_type) init_attr->qp_type;
0962     struct mlx4_ib_cq *mcq;
0963     unsigned long flags;
0964
0965     /* When tunneling special qps, we use a plain UD qp */
0966     if (sqpn) {
0967         if (mlx4_is_mfunc(dev->dev) &&
0968             (!mlx4_is_master(dev->dev) ||
0969              !(init_attr->create_flags & MLX4_IB_SRIOV_SQP))) {
0970             if (init_attr->qp_type == IB_QPT_GSI)
0971                 qp_type = MLX4_IB_QPT_PROXY_GSI;
0972             else {
0973                 if (mlx4_is_master(dev->dev) ||
0974                     qp0_enabled_vf(dev->dev, sqpn))
0975                     qp_type = MLX4_IB_QPT_PROXY_SMI_OWNER;
0976                 else
0977                     qp_type = MLX4_IB_QPT_PROXY_SMI;
0978             }
0979         }
0980         qpn = sqpn;
0981         /* add extra sg entry for tunneling */
0982         init_attr->cap.max_recv_sge++;
0983     } else if (init_attr->create_flags & MLX4_IB_SRIOV_TUNNEL_QP) {
0984         struct mlx4_ib_qp_tunnel_init_attr *tnl_init =
0985             container_of(init_attr,
0986                      struct mlx4_ib_qp_tunnel_init_attr, init_attr);
0987         if ((tnl_init->proxy_qp_type != IB_QPT_SMI &&
0988              tnl_init->proxy_qp_type != IB_QPT_GSI)   ||
0989             !mlx4_is_master(dev->dev))
0990             return -EINVAL;
0991         if (tnl_init->proxy_qp_type == IB_QPT_GSI)
0992             qp_type = MLX4_IB_QPT_TUN_GSI;
0993         else if (tnl_init->slave == mlx4_master_func_num(dev->dev) ||
0994              mlx4_vf_smi_enabled(dev->dev, tnl_init->slave,
0995                          tnl_init->port))
0996             qp_type = MLX4_IB_QPT_TUN_SMI_OWNER;
0997         else
0998             qp_type = MLX4_IB_QPT_TUN_SMI;
0999         /* we are definitely in the PPF here, since we are creating
1000          * tunnel QPs. base_tunnel_sqpn is therefore valid. */
1001         qpn = dev->dev->phys_caps.base_tunnel_sqpn + 8 * tnl_init->slave
1002             + tnl_init->proxy_qp_type * 2 + tnl_init->port - 1;
1003         sqpn = qpn;
1004     }
1005
1006     if (init_attr->qp_type == IB_QPT_SMI ||
1007         init_attr->qp_type == IB_QPT_GSI || qp_type == MLX4_IB_QPT_SMI ||
1008         qp_type == MLX4_IB_QPT_GSI ||
1009         (qp_type & (MLX4_IB_QPT_PROXY_SMI | MLX4_IB_QPT_PROXY_SMI_OWNER |
1010             MLX4_IB_QPT_PROXY_GSI | MLX4_IB_QPT_TUN_SMI_OWNER))) {
1011         qp->sqp = kzalloc(sizeof(struct mlx4_ib_sqp), GFP_KERNEL);
1012         if (!qp->sqp)
1013             return -ENOMEM;
1014     }
1015
1016     qp->mlx4_ib_qp_type = qp_type;
1017
1018     spin_lock_init(&qp->sq.lock);
1019     spin_lock_init(&qp->rq.lock);
1020     INIT_LIST_HEAD(&qp->gid_list);
1021     INIT_LIST_HEAD(&qp->steering_rules);
1022
1023     qp->state = IB_QPS_RESET;
1024     if (init_attr->sq_sig_type == IB_SIGNAL_ALL_WR)
1025         qp->sq_signal_bits = cpu_to_be32(MLX4_WQE_CTRL_CQ_UPDATE);
1026
1027     if (udata) {
1028         struct mlx4_ib_create_qp ucmd;
1029         size_t copy_len;
1030         int shift;
1031         int n;
1032
1033         copy_len = sizeof(struct mlx4_ib_create_qp);
1034
1035         if (ib_copy_from_udata(&ucmd, udata, copy_len)) {
1036             err = -EFAULT;
1037             goto err;
1038         }
1039
1040         qp->inl_recv_sz = ucmd.inl_recv_sz;
1041
1042         if (init_attr->create_flags & IB_QP_CREATE_SCATTER_FCS) {
1043             if (!(dev->dev->caps.flags &
1044                   MLX4_DEV_CAP_FLAG_FCS_KEEP)) {
1045                 pr_debug("scatter FCS is unsupported\n");
1046                 err = -EOPNOTSUPP;
1047                 goto err;
1048             }
1049
1050             qp->flags |= MLX4_IB_QP_SCATTER_FCS;
1051         }
1052
1053         err = set_rq_size(dev, &init_attr->cap, udata,
1054                   qp_has_rq(init_attr), qp, qp->inl_recv_sz);
1055         if (err)
1056             goto err;
1057
1058         qp->sq_no_prefetch = ucmd.sq_no_prefetch;
1059
1060         err = set_user_sq_size(dev, qp, &ucmd);
1061         if (err)
1062             goto err;
1063
1064         qp->umem =
1065             ib_umem_get(pd->device, ucmd.buf_addr, qp->buf_size, 0);
1066         if (IS_ERR(qp->umem)) {
1067             err = PTR_ERR(qp->umem);
1068             goto err;
1069         }
1070
1071         shift = mlx4_ib_umem_calc_optimal_mtt_size(qp->umem, 0, &n);
1072         err = mlx4_mtt_init(dev->dev, n, shift, &qp->mtt);
1073
1074         if (err)
1075             goto err_buf;
1076
1077         err = mlx4_ib_umem_write_mtt(dev, &qp->mtt, qp->umem);
1078         if (err)
1079             goto err_mtt;
1080
1081         if (qp_has_rq(init_attr)) {
1082             err = mlx4_ib_db_map_user(udata, ucmd.db_addr, &qp->db);
1083             if (err)
1084                 goto err_mtt;
1085         }
1086         qp->mqp.usage = MLX4_RES_USAGE_USER_VERBS;
1087     } else {
1088         err = set_rq_size(dev, &init_attr->cap, udata,
1089                   qp_has_rq(init_attr), qp, 0);
1090         if (err)
1091             goto err;
1092
1093         qp->sq_no_prefetch = 0;
1094
1095         if (init_attr->create_flags & IB_QP_CREATE_IPOIB_UD_LSO)
1096             qp->flags |= MLX4_IB_QP_LSO;
1097
1098         if (init_attr->create_flags & IB_QP_CREATE_NETIF_QP) {
1099             if (dev->steering_support ==
1100                 MLX4_STEERING_MODE_DEVICE_MANAGED)
1101                 qp->flags |= MLX4_IB_QP_NETIF;
1102             else {
1103                 err = -EINVAL;
1104                 goto err;
1105             }
1106         }
1107
1108         err = set_kernel_sq_size(dev, &init_attr->cap, qp_type, qp);
1109         if (err)
1110             goto err;
1111
1112         if (qp_has_rq(init_attr)) {
1113             err = mlx4_db_alloc(dev->dev, &qp->db, 0);
1114             if (err)
1115                 goto err;
1116
1117             *qp->db.db = 0;
1118         }
1119
1120         if (mlx4_buf_alloc(dev->dev, qp->buf_size,  PAGE_SIZE * 2,
1121                    &qp->buf)) {
1122             err = -ENOMEM;
1123             goto err_db;
1124         }
1125
1126         err = mlx4_mtt_init(dev->dev, qp->buf.npages, qp->buf.page_shift,
1127                     &qp->mtt);
1128         if (err)
1129             goto err_buf;
1130
1131         err = mlx4_buf_write_mtt(dev->dev, &qp->mtt, &qp->buf);
1132         if (err)
1133             goto err_mtt;
1134
1135         qp->sq.wrid = kvmalloc_array(qp->sq.wqe_cnt,
1136                          sizeof(u64), GFP_KERNEL);
1137         qp->rq.wrid = kvmalloc_array(qp->rq.wqe_cnt,
1138                          sizeof(u64), GFP_KERNEL);
1139         if (!qp->sq.wrid || !qp->rq.wrid) {
1140             err = -ENOMEM;
1141             goto err_wrid;
1142         }
1143         qp->mqp.usage = MLX4_RES_USAGE_DRIVER;
1144     }
1145
1146     if (sqpn) {
1147         if (qp->mlx4_ib_qp_type & (MLX4_IB_QPT_PROXY_SMI_OWNER |
1148             MLX4_IB_QPT_PROXY_SMI | MLX4_IB_QPT_PROXY_GSI)) {
1149             if (alloc_proxy_bufs(pd->device, qp)) {
1150                 err = -ENOMEM;
1151                 goto err_wrid;
1152             }
1153         }
1154     } else {
1155         /* Raw packet QPNs may not have bits 6,7 set in their qp_num;
1156          * otherwise, the WQE BlueFlame setup flow wrongly causes
1157          * VLAN insertion. */
1158         if (init_attr->qp_type == IB_QPT_RAW_PACKET)
1159             err = mlx4_qp_reserve_range(dev->dev, 1, 1, &qpn,
1160                             (init_attr->cap.max_send_wr ?
1161                              MLX4_RESERVE_ETH_BF_QP : 0) |
1162                             (init_attr->cap.max_recv_wr ?
1163                              MLX4_RESERVE_A0_QP : 0),
1164                             qp->mqp.usage);
1165         else
1166             if (qp->flags & MLX4_IB_QP_NETIF)
1167                 err = mlx4_ib_steer_qp_alloc(dev, 1, &qpn);
1168             else
1169                 err = mlx4_qp_reserve_range(dev->dev, 1, 1,
1170                                 &qpn, 0, qp->mqp.usage);
1171         if (err)
1172             goto err_proxy;
1173     }
1174
1175     if (init_attr->create_flags & IB_QP_CREATE_BLOCK_MULTICAST_LOOPBACK)
1176         qp->flags |= MLX4_IB_QP_BLOCK_MULTICAST_LOOPBACK;
1177
1178     err = mlx4_qp_alloc(dev->dev, qpn, &qp->mqp);
1179     if (err)
1180         goto err_qpn;
1181
1182     if (init_attr->qp_type == IB_QPT_XRC_TGT)
1183         qp->mqp.qpn |= (1 << 23);
1184
1185     /*
1186      * Hardware wants QPN written in big-endian order (after
1187      * shifting) for send doorbell.  Precompute this value to save
1188      * a little bit when posting sends.
1189      */
1190     qp->doorbell_qpn = swab32(qp->mqp.qpn << 8);
1191
1192     qp->mqp.event = mlx4_ib_qp_event;
1193
1194     spin_lock_irqsave(&dev->reset_flow_resource_lock, flags);
1195     mlx4_ib_lock_cqs(to_mcq(init_attr->send_cq),
1196              to_mcq(init_attr->recv_cq));
1197     /* Maintain device to QPs access, needed for further handling
1198      * via reset flow
1199      */
1200     list_add_tail(&qp->qps_list, &dev->qp_list);
1201     /* Maintain CQ to QPs access, needed for further handling
1202      * via reset flow
1203      */
1204     mcq = to_mcq(init_attr->send_cq);
1205     list_add_tail(&qp->cq_send_list, &mcq->send_qp_list);
1206     mcq = to_mcq(init_attr->recv_cq);
1207     list_add_tail(&qp->cq_recv_list, &mcq->recv_qp_list);
1208     mlx4_ib_unlock_cqs(to_mcq(init_attr->send_cq),
1209                to_mcq(init_attr->recv_cq));
1210     spin_unlock_irqrestore(&dev->reset_flow_resource_lock, flags);
1211     return 0;
1212
1213 err_qpn:
1214     if (!sqpn) {
1215         if (qp->flags & MLX4_IB_QP_NETIF)
1216             mlx4_ib_steer_qp_free(dev, qpn, 1);
1217         else
1218             mlx4_qp_release_range(dev->dev, qpn, 1);
1219     }
1220 err_proxy:
1221     if (qp->mlx4_ib_qp_type == MLX4_IB_QPT_PROXY_GSI)
1222         free_proxy_bufs(pd->device, qp);
1223 err_wrid:
1224     if (udata) {
1225         if (qp_has_rq(init_attr))
1226             mlx4_ib_db_unmap_user(context, &qp->db);
1227     } else {
1228         kvfree(qp->sq.wrid);
1229         kvfree(qp->rq.wrid);
1230     }
1231
1232 err_mtt:
1233     mlx4_mtt_cleanup(dev->dev, &qp->mtt);
1234
1235 err_buf:
1236     if (!qp->umem)
1237         mlx4_buf_free(dev->dev, qp->buf_size, &qp->buf);
1238     ib_umem_release(qp->umem);
1239
1240 err_db:
1241     if (!udata && qp_has_rq(init_attr))
1242         mlx4_db_free(dev->dev, &qp->db);
1243
1244 err:
1245     kfree(qp->sqp);
1246     return err;
1247 }
1248
1249 static enum mlx4_qp_state to_mlx4_state(enum ib_qp_state state)
1250 {
1251     switch (state) {
1252     case IB_QPS_RESET:  return MLX4_QP_STATE_RST;
1253     case IB_QPS_INIT:   return MLX4_QP_STATE_INIT;
1254     case IB_QPS_RTR:    return MLX4_QP_STATE_RTR;
1255     case IB_QPS_RTS:    return MLX4_QP_STATE_RTS;
1256     case IB_QPS_SQD:    return MLX4_QP_STATE_SQD;
1257     case IB_QPS_SQE:    return MLX4_QP_STATE_SQER;
1258     case IB_QPS_ERR:    return MLX4_QP_STATE_ERR;
1259     default:        return -1;
1260     }
1261 }
1262
1263 static void mlx4_ib_lock_cqs(struct mlx4_ib_cq *send_cq, struct mlx4_ib_cq *recv_cq)
1264     __acquires(&send_cq->lock) __acquires(&recv_cq->lock)
1265 {
1266     if (send_cq == recv_cq) {
1267         spin_lock(&send_cq->lock);
1268         __acquire(&recv_cq->lock);
1269     } else if (send_cq->mcq.cqn < recv_cq->mcq.cqn) {
1270         spin_lock(&send_cq->lock);
1271         spin_lock_nested(&recv_cq->lock, SINGLE_DEPTH_NESTING);
1272     } else {
1273         spin_lock(&recv_cq->lock);
1274         spin_lock_nested(&send_cq->lock, SINGLE_DEPTH_NESTING);
1275     }
1276 }
1277
1278 static void mlx4_ib_unlock_cqs(struct mlx4_ib_cq *send_cq, struct mlx4_ib_cq *recv_cq)
1279     __releases(&send_cq->lock) __releases(&recv_cq->lock)
1280 {
1281     if (send_cq == recv_cq) {
1282         __release(&recv_cq->lock);
1283         spin_unlock(&send_cq->lock);
1284     } else if (send_cq->mcq.cqn < recv_cq->mcq.cqn) {
1285         spin_unlock(&recv_cq->lock);
1286         spin_unlock(&send_cq->lock);
1287     } else {
1288         spin_unlock(&send_cq->lock);
1289         spin_unlock(&recv_cq->lock);
1290     }
1291 }
1292
1293 static void del_gid_entries(struct mlx4_ib_qp *qp)
1294 {
1295     struct mlx4_ib_gid_entry *ge, *tmp;
1296
1297     list_for_each_entry_safe(ge, tmp, &qp->gid_list, list) {
1298         list_del(&ge->list);
1299         kfree(ge);
1300     }
1301 }
1302
1303 static struct mlx4_ib_pd *get_pd(struct mlx4_ib_qp *qp)
1304 {
1305     if (qp->ibqp.qp_type == IB_QPT_XRC_TGT)
1306         return to_mpd(to_mxrcd(qp->ibqp.xrcd)->pd);
1307     else
1308         return to_mpd(qp->ibqp.pd);
1309 }
1310
1311 static void get_cqs(struct mlx4_ib_qp *qp, enum mlx4_ib_source_type src,
1312             struct mlx4_ib_cq **send_cq, struct mlx4_ib_cq **recv_cq)
1313 {
1314     switch (qp->ibqp.qp_type) {
1315     case IB_QPT_XRC_TGT:
1316         *send_cq = to_mcq(to_mxrcd(qp->ibqp.xrcd)->cq);
1317         *recv_cq = *send_cq;
1318         break;
1319     case IB_QPT_XRC_INI:
1320         *send_cq = to_mcq(qp->ibqp.send_cq);
1321         *recv_cq = *send_cq;
1322         break;
1323     default:
1324         *recv_cq = (src == MLX4_IB_QP_SRC) ? to_mcq(qp->ibqp.recv_cq) :
1325                              to_mcq(qp->ibwq.cq);
1326         *send_cq = (src == MLX4_IB_QP_SRC) ? to_mcq(qp->ibqp.send_cq) :
1327                              *recv_cq;
1328         break;
1329     }
1330 }
1331
1332 static void destroy_qp_rss(struct mlx4_ib_dev *dev, struct mlx4_ib_qp *qp)
1333 {
1334     if (qp->state != IB_QPS_RESET) {
1335         int i;
1336
1337         for (i = 0; i < (1 << qp->ibqp.rwq_ind_tbl->log_ind_tbl_size);
1338              i++) {
1339             struct ib_wq *ibwq = qp->ibqp.rwq_ind_tbl->ind_tbl[i];
1340             struct mlx4_ib_qp *wq = to_mqp((struct ib_qp *)ibwq);
1341
1342             mutex_lock(&wq->mutex);
1343
1344             wq->rss_usecnt--;
1345
1346             mutex_unlock(&wq->mutex);
1347         }
1348
1349         if (mlx4_qp_modify(dev->dev, NULL, to_mlx4_state(qp->state),
1350                    MLX4_QP_STATE_RST, NULL, 0, 0, &qp->mqp))
1351             pr_warn("modify QP %06x to RESET failed.\n",
1352                 qp->mqp.qpn);
1353     }
1354
1355     mlx4_qp_remove(dev->dev, &qp->mqp);
1356     mlx4_qp_free(dev->dev, &qp->mqp);
1357     mlx4_qp_release_range(dev->dev, qp->mqp.qpn, 1);
1358     del_gid_entries(qp);
1359 }
1360
1361 static void destroy_qp_common(struct mlx4_ib_dev *dev, struct mlx4_ib_qp *qp,
1362                   enum mlx4_ib_source_type src,
1363                   struct ib_udata *udata)
1364 {
1365     struct mlx4_ib_cq *send_cq, *recv_cq;
1366     unsigned long flags;
1367
1368     if (qp->state != IB_QPS_RESET) {
1369         if (mlx4_qp_modify(dev->dev, NULL, to_mlx4_state(qp->state),
1370                    MLX4_QP_STATE_RST, NULL, 0, 0, &qp->mqp))
1371             pr_warn("modify QP %06x to RESET failed.\n",
1372                    qp->mqp.qpn);
1373         if (qp->pri.smac || (!qp->pri.smac && qp->pri.smac_port)) {
1374             mlx4_unregister_mac(dev->dev, qp->pri.smac_port, qp->pri.smac);
1375             qp->pri.smac = 0;
1376             qp->pri.smac_port = 0;
1377         }
1378         if (qp->alt.smac) {
1379             mlx4_unregister_mac(dev->dev, qp->alt.smac_port, qp->alt.smac);
1380             qp->alt.smac = 0;
1381         }
1382         if (qp->pri.vid < 0x1000) {
1383             mlx4_unregister_vlan(dev->dev, qp->pri.vlan_port, qp->pri.vid);
1384             qp->pri.vid = 0xFFFF;
1385             qp->pri.candidate_vid = 0xFFFF;
1386             qp->pri.update_vid = 0;
1387         }
1388         if (qp->alt.vid < 0x1000) {
1389             mlx4_unregister_vlan(dev->dev, qp->alt.vlan_port, qp->alt.vid);
1390             qp->alt.vid = 0xFFFF;
1391             qp->alt.candidate_vid = 0xFFFF;
1392             qp->alt.update_vid = 0;
1393         }
1394     }
1395
1396     get_cqs(qp, src, &send_cq, &recv_cq);
1397
1398     spin_lock_irqsave(&dev->reset_flow_resource_lock, flags);
1399     mlx4_ib_lock_cqs(send_cq, recv_cq);
1400
1401     /* del from lists under both locks above to protect reset flow paths */
1402     list_del(&qp->qps_list);
1403     list_del(&qp->cq_send_list);
1404     list_del(&qp->cq_recv_list);
1405     if (!udata) {
1406         __mlx4_ib_cq_clean(recv_cq, qp->mqp.qpn,
1407                  qp->ibqp.srq ? to_msrq(qp->ibqp.srq): NULL);
1408         if (send_cq != recv_cq)
1409             __mlx4_ib_cq_clean(send_cq, qp->mqp.qpn, NULL);
1410     }
1411
1412     mlx4_qp_remove(dev->dev, &qp->mqp);
1413
1414     mlx4_ib_unlock_cqs(send_cq, recv_cq);
1415     spin_unlock_irqrestore(&dev->reset_flow_resource_lock, flags);
1416
1417     mlx4_qp_free(dev->dev, &qp->mqp);
1418
1419     if (!is_sqp(dev, qp) && !is_tunnel_qp(dev, qp)) {
1420         if (qp->flags & MLX4_IB_QP_NETIF)
1421             mlx4_ib_steer_qp_free(dev, qp->mqp.qpn, 1);
1422         else if (src == MLX4_IB_RWQ_SRC)
1423             mlx4_ib_release_wqn(
1424                 rdma_udata_to_drv_context(
1425                     udata,
1426                     struct mlx4_ib_ucontext,
1427                     ibucontext),
1428                 qp, 1);
1429         else
1430             mlx4_qp_release_range(dev->dev, qp->mqp.qpn, 1);
1431     }
1432
1433     mlx4_mtt_cleanup(dev->dev, &qp->mtt);
1434
1435     if (udata) {
1436         if (qp->rq.wqe_cnt) {
1437             struct mlx4_ib_ucontext *mcontext =
1438                 rdma_udata_to_drv_context(
1439                     udata,
1440                     struct mlx4_ib_ucontext,
1441                     ibucontext);
1442
1443             mlx4_ib_db_unmap_user(mcontext, &qp->db);
1444         }
1445     } else {
1446         kvfree(qp->sq.wrid);
1447         kvfree(qp->rq.wrid);
1448         if (qp->mlx4_ib_qp_type & (MLX4_IB_QPT_PROXY_SMI_OWNER |
1449             MLX4_IB_QPT_PROXY_SMI | MLX4_IB_QPT_PROXY_GSI))
1450             free_proxy_bufs(&dev->ib_dev, qp);
1451         mlx4_buf_free(dev->dev, qp->buf_size, &qp->buf);
1452         if (qp->rq.wqe_cnt)
1453             mlx4_db_free(dev->dev, &qp->db);
1454     }
1455     ib_umem_release(qp->umem);
1456
1457     del_gid_entries(qp);
1458 }
1459
1460 static u32 get_sqp_num(struct mlx4_ib_dev *dev, struct ib_qp_init_attr *attr)
1461 {
1462     /* Native or PPF */
1463     if (!mlx4_is_mfunc(dev->dev) ||
1464         (mlx4_is_master(dev->dev) &&
1465          attr->create_flags & MLX4_IB_SRIOV_SQP)) {
1466         return  dev->dev->phys_caps.base_sqpn +
1467             (attr->qp_type == IB_QPT_SMI ? 0 : 2) +
1468             attr->port_num - 1;
1469     }
1470     /* PF or VF -- creating proxies */
1471     if (attr->qp_type == IB_QPT_SMI)
1472         return dev->dev->caps.spec_qps[attr->port_num - 1].qp0_proxy;
1473     else
1474         return dev->dev->caps.spec_qps[attr->port_num - 1].qp1_proxy;
1475 }
1476
1477 static int _mlx4_ib_create_qp(struct ib_pd *pd, struct mlx4_ib_qp *qp,
1478                   struct ib_qp_init_attr *init_attr,
1479                   struct ib_udata *udata)
1480 {
1481     int err;
1482     int sup_u_create_flags = MLX4_IB_QP_BLOCK_MULTICAST_LOOPBACK;
1483     u16 xrcdn = 0;
1484
1485     if (init_attr->rwq_ind_tbl)
1486         return _mlx4_ib_create_qp_rss(pd, qp, init_attr, udata);
1487
1488     /*
1489      * We only support LSO, vendor flag1, and multicast loopback blocking,
1490      * and only for kernel UD QPs.
1491      */
1492     if (init_attr->create_flags & ~(MLX4_IB_QP_LSO |
1493                     MLX4_IB_QP_BLOCK_MULTICAST_LOOPBACK |
1494                     MLX4_IB_SRIOV_TUNNEL_QP |
1495                     MLX4_IB_SRIOV_SQP |
1496                     MLX4_IB_QP_NETIF |
1497                     MLX4_IB_QP_CREATE_ROCE_V2_GSI))
1498         return -EOPNOTSUPP;
1499
1500     if (init_attr->create_flags & IB_QP_CREATE_NETIF_QP) {
1501         if (init_attr->qp_type != IB_QPT_UD)
1502             return -EINVAL;
1503     }
1504
1505     if (init_attr->create_flags) {
1506         if (udata && init_attr->create_flags & ~(sup_u_create_flags))
1507             return -EINVAL;
1508
1509         if ((init_attr->create_flags & ~(MLX4_IB_SRIOV_SQP |
1510                          MLX4_IB_QP_CREATE_ROCE_V2_GSI  |
1511                          MLX4_IB_QP_BLOCK_MULTICAST_LOOPBACK) &&
1512              init_attr->qp_type != IB_QPT_UD) ||
1513             (init_attr->create_flags & MLX4_IB_SRIOV_SQP &&
1514              init_attr->qp_type > IB_QPT_GSI) ||
1515             (init_attr->create_flags & MLX4_IB_QP_CREATE_ROCE_V2_GSI &&
1516              init_attr->qp_type != IB_QPT_GSI))
1517             return -EINVAL;
1518     }
1519
1520     switch (init_attr->qp_type) {
1521     case IB_QPT_XRC_TGT:
1522         pd = to_mxrcd(init_attr->xrcd)->pd;
1523         xrcdn = to_mxrcd(init_attr->xrcd)->xrcdn;
1524         init_attr->send_cq = to_mxrcd(init_attr->xrcd)->cq;
1525         fallthrough;
1526     case IB_QPT_XRC_INI:
1527         if (!(to_mdev(pd->device)->dev->caps.flags & MLX4_DEV_CAP_FLAG_XRC))
1528             return -ENOSYS;
1529         init_attr->recv_cq = init_attr->send_cq;
1530         fallthrough;
1531     case IB_QPT_RC:
1532     case IB_QPT_UC:
1533     case IB_QPT_RAW_PACKET:
1534     case IB_QPT_UD:
1535         qp->pri.vid = 0xFFFF;
1536         qp->alt.vid = 0xFFFF;
1537         err = create_qp_common(pd, init_attr, udata, 0, qp);
1538         if (err)
1539             return err;
1540
1541         qp->ibqp.qp_num = qp->mqp.qpn;
1542         qp->xrcdn = xrcdn;
1543         break;
1544     case IB_QPT_SMI:
1545     case IB_QPT_GSI:
1546     {
1547         int sqpn;
1548
1549         if (init_attr->create_flags & MLX4_IB_QP_CREATE_ROCE_V2_GSI) {
1550             int res = mlx4_qp_reserve_range(to_mdev(pd->device)->dev,
1551                             1, 1, &sqpn, 0,
1552                             MLX4_RES_USAGE_DRIVER);
1553
1554             if (res)
1555                 return res;
1556         } else {
1557             sqpn = get_sqp_num(to_mdev(pd->device), init_attr);
1558         }
1559
1560         qp->pri.vid = 0xFFFF;
1561         qp->alt.vid = 0xFFFF;
1562         err = create_qp_common(pd, init_attr, udata, sqpn, qp);
1563         if (err)
1564             return err;
1565
1566         if (init_attr->create_flags &
1567             (MLX4_IB_SRIOV_SQP | MLX4_IB_SRIOV_TUNNEL_QP))
1568             /* Internal QP created with ib_create_qp */
1569             rdma_restrack_no_track(&qp->ibqp.res);
1570
1571         qp->port    = init_attr->port_num;
1572         qp->ibqp.qp_num = init_attr->qp_type == IB_QPT_SMI ? 0 :
1573             init_attr->create_flags & MLX4_IB_QP_CREATE_ROCE_V2_GSI ? sqpn : 1;
1574         break;
1575     }
1576     default:
1577         /* Don't support raw QPs */
1578         return -EOPNOTSUPP;
1579     }
1580     return 0;
1581 }
1582
1583 int mlx4_ib_create_qp(struct ib_qp *ibqp, struct ib_qp_init_attr *init_attr,
1584               struct ib_udata *udata)
1585 {
1586     struct ib_device *device = ibqp->device;
1587     struct mlx4_ib_dev *dev = to_mdev(device);
1588     struct mlx4_ib_qp *qp = to_mqp(ibqp);
1589     struct ib_pd *pd = ibqp->pd;
1590     int ret;
1591
1592     mutex_init(&qp->mutex);
1593     ret = _mlx4_ib_create_qp(pd, qp, init_attr, udata);
1594     if (ret)
1595         return ret;
1596
1597     if (init_attr->qp_type == IB_QPT_GSI &&
1598         !(init_attr->create_flags & MLX4_IB_QP_CREATE_ROCE_V2_GSI)) {
1599         struct mlx4_ib_sqp *sqp = qp->sqp;
1600         int is_eth = rdma_cap_eth_ah(&dev->ib_dev, init_attr->port_num);
1601
1602         if (is_eth &&
1603             dev->dev->caps.flags2 & MLX4_DEV_CAP_FLAG2_ROCE_V1_V2) {
1604             init_attr->create_flags |= MLX4_IB_QP_CREATE_ROCE_V2_GSI;
1605             sqp->roce_v2_gsi = ib_create_qp(pd, init_attr);
1606
1607             if (IS_ERR(sqp->roce_v2_gsi)) {
1608                 pr_err("Failed to create GSI QP for RoCEv2 (%ld)\n", PTR_ERR(sqp->roce_v2_gsi));
1609                 sqp->roce_v2_gsi = NULL;
1610             } else {
1611                 to_mqp(sqp->roce_v2_gsi)->flags |=
1612                     MLX4_IB_ROCE_V2_GSI_QP;
1613             }
1614
1615             init_attr->create_flags &= ~MLX4_IB_QP_CREATE_ROCE_V2_GSI;
1616         }
1617     }
1618     return 0;
1619 }
1620
1621 static int _mlx4_ib_destroy_qp(struct ib_qp *qp, struct ib_udata *udata)
1622 {
1623     struct mlx4_ib_dev *dev = to_mdev(qp->device);
1624     struct mlx4_ib_qp *mqp = to_mqp(qp);
1625
1626     if (is_qp0(dev, mqp))
1627         mlx4_CLOSE_PORT(dev->dev, mqp->port);
1628
1629     if (mqp->mlx4_ib_qp_type == MLX4_IB_QPT_PROXY_GSI &&
1630         dev->qp1_proxy[mqp->port - 1] == mqp) {
1631         mutex_lock(&dev->qp1_proxy_lock[mqp->port - 1]);
1632         dev->qp1_proxy[mqp->port - 1] = NULL;
1633         mutex_unlock(&dev->qp1_proxy_lock[mqp->port - 1]);
1634     }
1635
1636     if (mqp->counter_index)
1637         mlx4_ib_free_qp_counter(dev, mqp);
1638
1639     if (qp->rwq_ind_tbl) {
1640         destroy_qp_rss(dev, mqp);
1641     } else {
1642         destroy_qp_common(dev, mqp, MLX4_IB_QP_SRC, udata);
1643     }
1644
1645     kfree(mqp->sqp);
1646     return 0;
1647 }
1648
1649 int mlx4_ib_destroy_qp(struct ib_qp *qp, struct ib_udata *udata)
1650 {
1651     struct mlx4_ib_qp *mqp = to_mqp(qp);
1652
1653     if (mqp->mlx4_ib_qp_type == MLX4_IB_QPT_GSI) {
1654         struct mlx4_ib_sqp *sqp = mqp->sqp;
1655
1656         if (sqp->roce_v2_gsi)
1657             ib_destroy_qp(sqp->roce_v2_gsi);
1658     }
1659
1660     return _mlx4_ib_destroy_qp(qp, udata);
1661 }
1662
1663 static int to_mlx4_st(struct mlx4_ib_dev *dev, enum mlx4_ib_qp_type type)
1664 {
1665     switch (type) {
1666     case MLX4_IB_QPT_RC:        return MLX4_QP_ST_RC;
1667     case MLX4_IB_QPT_UC:        return MLX4_QP_ST_UC;
1668     case MLX4_IB_QPT_UD:        return MLX4_QP_ST_UD;
1669     case MLX4_IB_QPT_XRC_INI:
1670     case MLX4_IB_QPT_XRC_TGT:   return MLX4_QP_ST_XRC;
1671     case MLX4_IB_QPT_SMI:
1672     case MLX4_IB_QPT_GSI:
1673     case MLX4_IB_QPT_RAW_PACKET:    return MLX4_QP_ST_MLX;
1674
1675     case MLX4_IB_QPT_PROXY_SMI_OWNER:
1676     case MLX4_IB_QPT_TUN_SMI_OWNER: return (mlx4_is_mfunc(dev->dev) ?
1677                         MLX4_QP_ST_MLX : -1);
1678     case MLX4_IB_QPT_PROXY_SMI:
1679     case MLX4_IB_QPT_TUN_SMI:
1680     case MLX4_IB_QPT_PROXY_GSI:
1681     case MLX4_IB_QPT_TUN_GSI:   return (mlx4_is_mfunc(dev->dev) ?
1682                         MLX4_QP_ST_UD : -1);
1683     default:            return -1;
1684     }
1685 }
1686
1687 static __be32 to_mlx4_access_flags(struct mlx4_ib_qp *qp, const struct ib_qp_attr *attr,
1688                    int attr_mask)
1689 {
1690     u8 dest_rd_atomic;
1691     u32 access_flags;
1692     u32 hw_access_flags = 0;
1693
1694     if (attr_mask & IB_QP_MAX_DEST_RD_ATOMIC)
1695         dest_rd_atomic = attr->max_dest_rd_atomic;
1696     else
1697         dest_rd_atomic = qp->resp_depth;
1698
1699     if (attr_mask & IB_QP_ACCESS_FLAGS)
1700         access_flags = attr->qp_access_flags;
1701     else
1702         access_flags = qp->atomic_rd_en;
1703
1704     if (!dest_rd_atomic)
1705         access_flags &= IB_ACCESS_REMOTE_WRITE;
1706
1707     if (access_flags & IB_ACCESS_REMOTE_READ)
1708         hw_access_flags |= MLX4_QP_BIT_RRE;
1709     if (access_flags & IB_ACCESS_REMOTE_ATOMIC)
1710         hw_access_flags |= MLX4_QP_BIT_RAE;
1711     if (access_flags & IB_ACCESS_REMOTE_WRITE)
1712         hw_access_flags |= MLX4_QP_BIT_RWE;
1713
1714     return cpu_to_be32(hw_access_flags);
1715 }
1716
1717 static void store_sqp_attrs(struct mlx4_ib_sqp *sqp, const struct ib_qp_attr *attr,
1718                 int attr_mask)
1719 {
1720     if (attr_mask & IB_QP_PKEY_INDEX)
1721         sqp->pkey_index = attr->pkey_index;
1722     if (attr_mask & IB_QP_QKEY)
1723         sqp->qkey = attr->qkey;
1724     if (attr_mask & IB_QP_SQ_PSN)
1725         sqp->send_psn = attr->sq_psn;
1726 }
1727
1728 static void mlx4_set_sched(struct mlx4_qp_path *path, u8 port)
1729 {
1730     path->sched_queue = (path->sched_queue & 0xbf) | ((port - 1) << 6);
1731 }
1732
1733 static int _mlx4_set_path(struct mlx4_ib_dev *dev,
1734               const struct rdma_ah_attr *ah,
1735               u64 smac, u16 vlan_tag, struct mlx4_qp_path *path,
1736               struct mlx4_roce_smac_vlan_info *smac_info, u8 port)
1737 {
1738     int vidx;
1739     int smac_index;
1740     int err;
1741
1742     path->grh_mylmc = rdma_ah_get_path_bits(ah) & 0x7f;
1743     path->rlid = cpu_to_be16(rdma_ah_get_dlid(ah));
1744     if (rdma_ah_get_static_rate(ah)) {
1745         path->static_rate = rdma_ah_get_static_rate(ah) +
1746                     MLX4_STAT_RATE_OFFSET;
1747         while (path->static_rate > IB_RATE_2_5_GBPS + MLX4_STAT_RATE_OFFSET &&
1748                !(1 << path->static_rate & dev->dev->caps.stat_rate_support))
1749             --path->static_rate;
1750     } else
1751         path->static_rate = 0;
1752
1753     if (rdma_ah_get_ah_flags(ah) & IB_AH_GRH) {
1754         const struct ib_global_route *grh = rdma_ah_read_grh(ah);
1755         int real_sgid_index =
1756             mlx4_ib_gid_index_to_real_index(dev, grh->sgid_attr);
1757
1758         if (real_sgid_index < 0)
1759             return real_sgid_index;
1760         if (real_sgid_index >= dev->dev->caps.gid_table_len[port]) {
1761             pr_err("sgid_index (%u) too large. max is %d\n",
1762                    real_sgid_index, dev->dev->caps.gid_table_len[port] - 1);
1763             return -1;
1764         }
1765
1766         path->grh_mylmc |= 1 << 7;
1767         path->mgid_index = real_sgid_index;
1768         path->hop_limit  = grh->hop_limit;
1769         path->tclass_flowlabel =
1770             cpu_to_be32((grh->traffic_class << 20) |
1771                     (grh->flow_label));
1772         memcpy(path->rgid, grh->dgid.raw, 16);
1773     }
1774
1775     if (ah->type == RDMA_AH_ATTR_TYPE_ROCE) {
1776         if (!(rdma_ah_get_ah_flags(ah) & IB_AH_GRH))
1777             return -1;
1778
1779         path->sched_queue = MLX4_IB_DEFAULT_SCHED_QUEUE |
1780             ((port - 1) << 6) | ((rdma_ah_get_sl(ah) & 7) << 3);
1781
1782         path->feup |= MLX4_FEUP_FORCE_ETH_UP;
1783         if (vlan_tag < 0x1000) {
1784             if (smac_info->vid < 0x1000) {
1785                 /* both valid vlan ids */
1786                 if (smac_info->vid != vlan_tag) {
1787                     /* different VIDs.  unreg old and reg new */
1788                     err = mlx4_register_vlan(dev->dev, port, vlan_tag, &vidx);
1789                     if (err)
1790                         return err;
1791                     smac_info->candidate_vid = vlan_tag;
1792                     smac_info->candidate_vlan_index = vidx;
1793                     smac_info->candidate_vlan_port = port;
1794                     smac_info->update_vid = 1;
1795                     path->vlan_index = vidx;
1796                 } else {
1797                     path->vlan_index = smac_info->vlan_index;
1798                 }
1799             } else {
1800                 /* no current vlan tag in qp */
1801                 err = mlx4_register_vlan(dev->dev, port, vlan_tag, &vidx);
1802                 if (err)
1803                     return err;
1804                 smac_info->candidate_vid = vlan_tag;
1805                 smac_info->candidate_vlan_index = vidx;
1806                 smac_info->candidate_vlan_port = port;
1807                 smac_info->update_vid = 1;
1808                 path->vlan_index = vidx;
1809             }
1810             path->feup |= MLX4_FVL_FORCE_ETH_VLAN;
1811             path->fl = 1 << 6;
1812         } else {
1813             /* have current vlan tag. unregister it at modify-qp success */
1814             if (smac_info->vid < 0x1000) {
1815                 smac_info->candidate_vid = 0xFFFF;
1816                 smac_info->update_vid = 1;
1817             }
1818         }
1819
1820         /* get smac_index for RoCE use.
1821          * If no smac was yet assigned, register one.
1822          * If one was already assigned, but the new mac differs,
1823          * unregister the old one and register the new one.
1824         */
1825         if ((!smac_info->smac && !smac_info->smac_port) ||
1826             smac_info->smac != smac) {
1827             /* register candidate now, unreg if needed, after success */
1828             smac_index = mlx4_register_mac(dev->dev, port, smac);
1829             if (smac_index >= 0) {
1830                 smac_info->candidate_smac_index = smac_index;
1831                 smac_info->candidate_smac = smac;
1832                 smac_info->candidate_smac_port = port;
1833             } else {
1834                 return -EINVAL;
1835             }
1836         } else {
1837             smac_index = smac_info->smac_index;
1838         }
1839         memcpy(path->dmac, ah->roce.dmac, 6);
1840         path->ackto = MLX4_IB_LINK_TYPE_ETH;
1841         /* put MAC table smac index for IBoE */
1842         path->grh_mylmc = (u8) (smac_index) | 0x80;
1843     } else {
1844         path->sched_queue = MLX4_IB_DEFAULT_SCHED_QUEUE |
1845             ((port - 1) << 6) | ((rdma_ah_get_sl(ah) & 0xf) << 2);
1846     }
1847
1848     return 0;
1849 }
1850
1851 static int mlx4_set_path(struct mlx4_ib_dev *dev, const struct ib_qp_attr *qp,
1852              enum ib_qp_attr_mask qp_attr_mask,
1853              struct mlx4_ib_qp *mqp,
1854              struct mlx4_qp_path *path, u8 port,
1855              u16 vlan_id, u8 *smac)
1856 {
1857     return _mlx4_set_path(dev, &qp->ah_attr,
1858                   ether_addr_to_u64(smac),
1859                   vlan_id,
1860                   path, &mqp->pri, port);
1861 }
1862
1863 static int mlx4_set_alt_path(struct mlx4_ib_dev *dev,
1864                  const struct ib_qp_attr *qp,
1865                  enum ib_qp_attr_mask qp_attr_mask,
1866                  struct mlx4_ib_qp *mqp,
1867                  struct mlx4_qp_path *path, u8 port)
1868 {
1869     return _mlx4_set_path(dev, &qp->alt_ah_attr,
1870                   0,
1871                   0xffff,
1872                   path, &mqp->alt, port);
1873 }
1874
1875 static void update_mcg_macs(struct mlx4_ib_dev *dev, struct mlx4_ib_qp *qp)
1876 {
1877     struct mlx4_ib_gid_entry *ge, *tmp;
1878
1879     list_for_each_entry_safe(ge, tmp, &qp->gid_list, list) {
1880         if (!ge->added && mlx4_ib_add_mc(dev, qp, &ge->gid)) {
1881             ge->added = 1;
1882             ge->port = qp->port;
1883         }
1884     }
1885 }
1886
1887 static int handle_eth_ud_smac_index(struct mlx4_ib_dev *dev,
1888                     struct mlx4_ib_qp *qp,
1889                     struct mlx4_qp_context *context)
1890 {
1891     u64 u64_mac;
1892     int smac_index;
1893
1894     u64_mac = atomic64_read(&dev->iboe.mac[qp->port - 1]);
1895
1896     context->pri_path.sched_queue = MLX4_IB_DEFAULT_SCHED_QUEUE | ((qp->port - 1) << 6);
1897     if (!qp->pri.smac && !qp->pri.smac_port) {
1898         smac_index = mlx4_register_mac(dev->dev, qp->port, u64_mac);
1899         if (smac_index >= 0) {
1900             qp->pri.candidate_smac_index = smac_index;
1901             qp->pri.candidate_smac = u64_mac;
1902             qp->pri.candidate_smac_port = qp->port;
1903             context->pri_path.grh_mylmc = 0x80 | (u8) smac_index;
1904         } else {
1905             return -ENOENT;
1906         }
1907     }
1908     return 0;
1909 }
1910
1911 static int create_qp_lb_counter(struct mlx4_ib_dev *dev, struct mlx4_ib_qp *qp)
1912 {
1913     struct counter_index *new_counter_index;
1914     int err;
1915     u32 tmp_idx;
1916
1917     if (rdma_port_get_link_layer(&dev->ib_dev, qp->port) !=
1918         IB_LINK_LAYER_ETHERNET ||
1919         !(qp->flags & MLX4_IB_QP_BLOCK_MULTICAST_LOOPBACK) ||
1920         !(dev->dev->caps.flags2 & MLX4_DEV_CAP_FLAG2_LB_SRC_CHK))
1921         return 0;
1922
1923     err = mlx4_counter_alloc(dev->dev, &tmp_idx, MLX4_RES_USAGE_DRIVER);
1924     if (err)
1925         return err;
1926
1927     new_counter_index = kmalloc(sizeof(*new_counter_index), GFP_KERNEL);
1928     if (!new_counter_index) {
1929         mlx4_counter_free(dev->dev, tmp_idx);
1930         return -ENOMEM;
1931     }
1932
1933     new_counter_index->index = tmp_idx;
1934     new_counter_index->allocated = 1;
1935     qp->counter_index = new_counter_index;
1936
1937     mutex_lock(&dev->counters_table[qp->port - 1].mutex);
1938     list_add_tail(&new_counter_index->list,
1939               &dev->counters_table[qp->port - 1].counters_list);
1940     mutex_unlock(&dev->counters_table[qp->port - 1].mutex);
1941
1942     return 0;
1943 }
1944
1945 enum {
1946     MLX4_QPC_ROCE_MODE_1 = 0,
1947     MLX4_QPC_ROCE_MODE_2 = 2,
1948     MLX4_QPC_ROCE_MODE_UNDEFINED = 0xff
1949 };
1950
1951 static u8 gid_type_to_qpc(enum ib_gid_type gid_type)
1952 {
1953     switch (gid_type) {
1954     case IB_GID_TYPE_ROCE:
1955         return MLX4_QPC_ROCE_MODE_1;
1956     case IB_GID_TYPE_ROCE_UDP_ENCAP:
1957         return MLX4_QPC_ROCE_MODE_2;
1958     default:
1959         return MLX4_QPC_ROCE_MODE_UNDEFINED;
1960     }
1961 }
1962
1963 /*
1964  * Go over all RSS QP's childes (WQs) and apply their HW state according to
1965  * their logic state if the RSS QP is the first RSS QP associated for the WQ.
1966  */
1967 static int bringup_rss_rwqs(struct ib_rwq_ind_table *ind_tbl, u8 port_num,
1968                 struct ib_udata *udata)
1969 {
1970     int err = 0;
1971     int i;
1972
1973     for (i = 0; i < (1 << ind_tbl->log_ind_tbl_size); i++) {
1974         struct ib_wq *ibwq = ind_tbl->ind_tbl[i];
1975         struct mlx4_ib_qp *wq = to_mqp((struct ib_qp *)ibwq);
1976
1977         mutex_lock(&wq->mutex);
1978
1979         /* Mlx4_ib restrictions:
1980          * WQ's is associated to a port according to the RSS QP it is
1981          * associates to.
1982          * In case the WQ is associated to a different port by another
1983          * RSS QP, return a failure.
1984          */
1985         if ((wq->rss_usecnt > 0) && (wq->port != port_num)) {
1986             err = -EINVAL;
1987             mutex_unlock(&wq->mutex);
1988             break;
1989         }
1990         wq->port = port_num;
1991         if ((wq->rss_usecnt == 0) && (ibwq->state == IB_WQS_RDY)) {
1992             err = _mlx4_ib_modify_wq(ibwq, IB_WQS_RDY, udata);
1993             if (err) {
1994                 mutex_unlock(&wq->mutex);
1995                 break;
1996             }
1997         }
1998         wq->rss_usecnt++;
1999
2000         mutex_unlock(&wq->mutex);
2001     }
2002
2003     if (i && err) {
2004         int j;
2005
2006         for (j = (i - 1); j >= 0; j--) {
2007             struct ib_wq *ibwq = ind_tbl->ind_tbl[j];
2008             struct mlx4_ib_qp *wq = to_mqp((struct ib_qp *)ibwq);
2009
2010             mutex_lock(&wq->mutex);
2011
2012             if ((wq->rss_usecnt == 1) &&
2013                 (ibwq->state == IB_WQS_RDY))
2014                 if (_mlx4_ib_modify_wq(ibwq, IB_WQS_RESET,
2015                                udata))
2016                     pr_warn("failed to reverse WQN=0x%06x\n",
2017                         ibwq->wq_num);
2018             wq->rss_usecnt--;
2019
2020             mutex_unlock(&wq->mutex);
2021         }
2022     }
2023
2024     return err;
2025 }
2026
2027 static void bring_down_rss_rwqs(struct ib_rwq_ind_table *ind_tbl,
2028                 struct ib_udata *udata)
2029 {
2030     int i;
2031
2032     for (i = 0; i < (1 << ind_tbl->log_ind_tbl_size); i++) {
2033         struct ib_wq *ibwq = ind_tbl->ind_tbl[i];
2034         struct mlx4_ib_qp *wq = to_mqp((struct ib_qp *)ibwq);
2035
2036         mutex_lock(&wq->mutex);
2037
2038         if ((wq->rss_usecnt == 1) && (ibwq->state == IB_WQS_RDY))
2039             if (_mlx4_ib_modify_wq(ibwq, IB_WQS_RESET, udata))
2040                 pr_warn("failed to reverse WQN=%x\n",
2041                     ibwq->wq_num);
2042         wq->rss_usecnt--;
2043
2044         mutex_unlock(&wq->mutex);
2045     }
2046 }
2047
2048 static void fill_qp_rss_context(struct mlx4_qp_context *context,
2049                 struct mlx4_ib_qp *qp)
2050 {
2051     struct mlx4_rss_context *rss_context;
2052
2053     rss_context = (void *)context + offsetof(struct mlx4_qp_context,
2054             pri_path) + MLX4_RSS_OFFSET_IN_QPC_PRI_PATH;
2055
2056     rss_context->base_qpn = cpu_to_be32(qp->rss_ctx->base_qpn_tbl_sz);
2057     rss_context->default_qpn =
2058         cpu_to_be32(qp->rss_ctx->base_qpn_tbl_sz & 0xffffff);
2059     if (qp->rss_ctx->flags & (MLX4_RSS_UDP_IPV4 | MLX4_RSS_UDP_IPV6))
2060         rss_context->base_qpn_udp = rss_context->default_qpn;
2061     rss_context->flags = qp->rss_ctx->flags;
2062     /* Currently support just toeplitz */
2063     rss_context->hash_fn = MLX4_RSS_HASH_TOP;
2064
2065     memcpy(rss_context->rss_key, qp->rss_ctx->rss_key,
2066            MLX4_EN_RSS_KEY_SIZE);
2067 }
2068
2069 static int __mlx4_ib_modify_qp(void *src, enum mlx4_ib_source_type src_type,
2070                    const struct ib_qp_attr *attr, int attr_mask,
2071                    enum ib_qp_state cur_state,
2072                    enum ib_qp_state new_state,
2073                    struct ib_udata *udata)
2074 {
2075     struct ib_srq  *ibsrq;
2076     const struct ib_gid_attr *gid_attr = NULL;
2077     struct ib_rwq_ind_table *rwq_ind_tbl;
2078     enum ib_qp_type qp_type;
2079     struct mlx4_ib_dev *dev;
2080     struct mlx4_ib_qp *qp;
2081     struct mlx4_ib_pd *pd;
2082     struct mlx4_ib_cq *send_cq, *recv_cq;
2083     struct mlx4_ib_ucontext *ucontext = rdma_udata_to_drv_context(
2084         udata, struct mlx4_ib_ucontext, ibucontext);
2085     struct mlx4_qp_context *context;
2086     enum mlx4_qp_optpar optpar = 0;
2087     int sqd_event;
2088     int steer_qp = 0;
2089     int err = -EINVAL;
2090     int counter_index;
2091
2092     if (src_type == MLX4_IB_RWQ_SRC) {
2093         struct ib_wq *ibwq;
2094
2095         ibwq        = (struct ib_wq *)src;
2096         ibsrq       = NULL;
2097         rwq_ind_tbl = NULL;
2098         qp_type     = IB_QPT_RAW_PACKET;
2099         qp      = to_mqp((struct ib_qp *)ibwq);
2100         dev     = to_mdev(ibwq->device);
2101         pd      = to_mpd(ibwq->pd);
2102     } else {
2103         struct ib_qp *ibqp;
2104
2105         ibqp        = (struct ib_qp *)src;
2106         ibsrq       = ibqp->srq;
2107         rwq_ind_tbl = ibqp->rwq_ind_tbl;
2108         qp_type     = ibqp->qp_type;
2109         qp      = to_mqp(ibqp);
2110         dev     = to_mdev(ibqp->device);
2111         pd      = get_pd(qp);
2112     }
2113
2114     /* APM is not supported under RoCE */
2115     if (attr_mask & IB_QP_ALT_PATH &&
2116         rdma_port_get_link_layer(&dev->ib_dev, qp->port) ==
2117         IB_LINK_LAYER_ETHERNET)
2118         return -ENOTSUPP;
2119
2120     context = kzalloc(sizeof *context, GFP_KERNEL);
2121     if (!context)
2122         return -ENOMEM;
2123
2124     context->flags = cpu_to_be32((to_mlx4_state(new_state) << 28) |
2125                      (to_mlx4_st(dev, qp->mlx4_ib_qp_type) << 16));
2126
2127     if (!(attr_mask & IB_QP_PATH_MIG_STATE))
2128         context->flags |= cpu_to_be32(MLX4_QP_PM_MIGRATED << 11);
2129     else {
2130         optpar |= MLX4_QP_OPTPAR_PM_STATE;
2131         switch (attr->path_mig_state) {
2132         case IB_MIG_MIGRATED:
2133             context->flags |= cpu_to_be32(MLX4_QP_PM_MIGRATED << 11);
2134             break;
2135         case IB_MIG_REARM:
2136             context->flags |= cpu_to_be32(MLX4_QP_PM_REARM << 11);
2137             break;
2138         case IB_MIG_ARMED:
2139             context->flags |= cpu_to_be32(MLX4_QP_PM_ARMED << 11);
2140             break;
2141         }
2142     }
2143
2144     if (qp->inl_recv_sz)
2145         context->param3 |= cpu_to_be32(1 << 25);
2146
2147     if (qp->flags & MLX4_IB_QP_SCATTER_FCS)
2148         context->param3 |= cpu_to_be32(1 << 29);
2149
2150     if (qp_type == IB_QPT_GSI || qp_type == IB_QPT_SMI)
2151         context->mtu_msgmax = (IB_MTU_4096 << 5) | 11;
2152     else if (qp_type == IB_QPT_RAW_PACKET)
2153         context->mtu_msgmax = (MLX4_RAW_QP_MTU << 5) | MLX4_RAW_QP_MSGMAX;
2154     else if (qp_type == IB_QPT_UD) {
2155         if (qp->flags & MLX4_IB_QP_LSO)
2156             context->mtu_msgmax = (IB_MTU_4096 << 5) |
2157                           ilog2(dev->dev->caps.max_gso_sz);
2158         else
2159             context->mtu_msgmax = (IB_MTU_4096 << 5) | 13;
2160     } else if (attr_mask & IB_QP_PATH_MTU) {
2161         if (attr->path_mtu < IB_MTU_256 || attr->path_mtu > IB_MTU_4096) {
2162             pr_err("path MTU (%u) is invalid\n",
2163                    attr->path_mtu);
2164             goto out;
2165         }
2166         context->mtu_msgmax = (attr->path_mtu << 5) |
2167             ilog2(dev->dev->caps.max_msg_sz);
2168     }
2169
2170     if (!rwq_ind_tbl) { /* PRM RSS receive side should be left zeros */
2171         if (qp->rq.wqe_cnt)
2172             context->rq_size_stride = ilog2(qp->rq.wqe_cnt) << 3;
2173         context->rq_size_stride |= qp->rq.wqe_shift - 4;
2174     }
2175
2176     if (qp->sq.wqe_cnt)
2177         context->sq_size_stride = ilog2(qp->sq.wqe_cnt) << 3;
2178     context->sq_size_stride |= qp->sq.wqe_shift - 4;
2179
2180     if (new_state == IB_QPS_RESET && qp->counter_index)
2181         mlx4_ib_free_qp_counter(dev, qp);
2182
2183     if (cur_state == IB_QPS_RESET && new_state == IB_QPS_INIT) {
2184         context->sq_size_stride |= !!qp->sq_no_prefetch << 7;
2185         context->xrcd = cpu_to_be32((u32) qp->xrcdn);
2186         if (qp_type == IB_QPT_RAW_PACKET)
2187             context->param3 |= cpu_to_be32(1 << 30);
2188     }
2189
2190     if (ucontext)
2191         context->usr_page = cpu_to_be32(
2192             mlx4_to_hw_uar_index(dev->dev, ucontext->uar.index));
2193     else
2194         context->usr_page = cpu_to_be32(
2195             mlx4_to_hw_uar_index(dev->dev, dev->priv_uar.index));
2196
2197     if (attr_mask & IB_QP_DEST_QPN)
2198         context->remote_qpn = cpu_to_be32(attr->dest_qp_num);
2199
2200     if (attr_mask & IB_QP_PORT) {
2201         if (cur_state == IB_QPS_SQD && new_state == IB_QPS_SQD &&
2202             !(attr_mask & IB_QP_AV)) {
2203             mlx4_set_sched(&context->pri_path, attr->port_num);
2204             optpar |= MLX4_QP_OPTPAR_SCHED_QUEUE;
2205         }
2206     }
2207
2208     if (cur_state == IB_QPS_INIT && new_state == IB_QPS_RTR) {
2209         err = create_qp_lb_counter(dev, qp);
2210         if (err)
2211             goto out;
2212
2213         counter_index =
2214             dev->counters_table[qp->port - 1].default_counter;
2215         if (qp->counter_index)
2216             counter_index = qp->counter_index->index;
2217
2218         if (counter_index != -1) {
2219             context->pri_path.counter_index = counter_index;
2220             optpar |= MLX4_QP_OPTPAR_COUNTER_INDEX;
2221             if (qp->counter_index) {
2222                 context->pri_path.fl |=
2223                     MLX4_FL_ETH_SRC_CHECK_MC_LB;
2224                 context->pri_path.vlan_control |=
2225                     MLX4_CTRL_ETH_SRC_CHECK_IF_COUNTER;
2226             }
2227         } else
2228             context->pri_path.counter_index =
2229                 MLX4_SINK_COUNTER_INDEX(dev->dev);
2230
2231         if (qp->flags & MLX4_IB_QP_NETIF) {
2232             mlx4_ib_steer_qp_reg(dev, qp, 1);
2233             steer_qp = 1;
2234         }
2235
2236         if (qp_type == IB_QPT_GSI) {
2237             enum ib_gid_type gid_type = qp->flags & MLX4_IB_ROCE_V2_GSI_QP ?
2238                 IB_GID_TYPE_ROCE_UDP_ENCAP : IB_GID_TYPE_ROCE;
2239             u8 qpc_roce_mode = gid_type_to_qpc(gid_type);
2240
2241             context->rlkey_roce_mode |= (qpc_roce_mode << 6);
2242         }
2243     }
2244
2245     if (attr_mask & IB_QP_PKEY_INDEX) {
2246         if (qp->mlx4_ib_qp_type & MLX4_IB_QPT_ANY_SRIOV)
2247             context->pri_path.disable_pkey_check = 0x40;
2248         context->pri_path.pkey_index = attr->pkey_index;
2249         optpar |= MLX4_QP_OPTPAR_PKEY_INDEX;
2250     }
2251
2252     if (attr_mask & IB_QP_AV) {
2253         u8 port_num = mlx4_is_bonded(dev->dev) ? 1 :
2254             attr_mask & IB_QP_PORT ? attr->port_num : qp->port;
2255         u16 vlan = 0xffff;
2256         u8 smac[ETH_ALEN];
2257         int is_eth =
2258             rdma_cap_eth_ah(&dev->ib_dev, port_num) &&
2259             rdma_ah_get_ah_flags(&attr->ah_attr) & IB_AH_GRH;
2260
2261         if (is_eth) {
2262             gid_attr = attr->ah_attr.grh.sgid_attr;
2263             err = rdma_read_gid_l2_fields(gid_attr, &vlan,
2264                               &smac[0]);
2265             if (err)
2266                 goto out;
2267         }
2268
2269         if (mlx4_set_path(dev, attr, attr_mask, qp, &context->pri_path,
2270                   port_num, vlan, smac))
2271             goto out;
2272
2273         optpar |= (MLX4_QP_OPTPAR_PRIMARY_ADDR_PATH |
2274                MLX4_QP_OPTPAR_SCHED_QUEUE);
2275
2276         if (is_eth &&
2277             (cur_state == IB_QPS_INIT && new_state == IB_QPS_RTR)) {
2278             u8 qpc_roce_mode = gid_type_to_qpc(gid_attr->gid_type);
2279
2280             if (qpc_roce_mode == MLX4_QPC_ROCE_MODE_UNDEFINED) {
2281                 err = -EINVAL;
2282                 goto out;
2283             }
2284             context->rlkey_roce_mode |= (qpc_roce_mode << 6);
2285         }
2286
2287     }
2288
2289     if (attr_mask & IB_QP_TIMEOUT) {
2290         context->pri_path.ackto |= attr->timeout << 3;
2291         optpar |= MLX4_QP_OPTPAR_ACK_TIMEOUT;
2292     }
2293
2294     if (attr_mask & IB_QP_ALT_PATH) {
2295         if (attr->alt_port_num == 0 ||
2296             attr->alt_port_num > dev->dev->caps.num_ports)
2297             goto out;
2298
2299         if (attr->alt_pkey_index >=
2300             dev->dev->caps.pkey_table_len[attr->alt_port_num])
2301             goto out;
2302
2303         if (mlx4_set_alt_path(dev, attr, attr_mask, qp,
2304                       &context->alt_path,
2305                       attr->alt_port_num))
2306             goto out;
2307
2308         context->alt_path.pkey_index = attr->alt_pkey_index;
2309         context->alt_path.ackto = attr->alt_timeout << 3;
2310         optpar |= MLX4_QP_OPTPAR_ALT_ADDR_PATH;
2311     }
2312
2313     context->pd = cpu_to_be32(pd->pdn);
2314
2315     if (!rwq_ind_tbl) {
2316         context->params1 = cpu_to_be32(MLX4_IB_ACK_REQ_FREQ << 28);
2317         get_cqs(qp, src_type, &send_cq, &recv_cq);
2318     } else { /* Set dummy CQs to be compatible with HV and PRM */
2319         send_cq = to_mcq(rwq_ind_tbl->ind_tbl[0]->cq);
2320         recv_cq = send_cq;
2321     }
2322     context->cqn_send = cpu_to_be32(send_cq->mcq.cqn);
2323     context->cqn_recv = cpu_to_be32(recv_cq->mcq.cqn);
2324
2325     /* Set "fast registration enabled" for all kernel QPs */
2326     if (!ucontext)
2327         context->params1 |= cpu_to_be32(1 << 11);
2328
2329     if (attr_mask & IB_QP_RNR_RETRY) {
2330         context->params1 |= cpu_to_be32(attr->rnr_retry << 13);
2331         optpar |= MLX4_QP_OPTPAR_RNR_RETRY;
2332     }
2333
2334     if (attr_mask & IB_QP_RETRY_CNT) {
2335         context->params1 |= cpu_to_be32(attr->retry_cnt << 16);
2336         optpar |= MLX4_QP_OPTPAR_RETRY_COUNT;
2337     }
2338
2339     if (attr_mask & IB_QP_MAX_QP_RD_ATOMIC) {
2340         if (attr->max_rd_atomic)
2341             context->params1 |=
2342                 cpu_to_be32(fls(attr->max_rd_atomic - 1) << 21);
2343         optpar |= MLX4_QP_OPTPAR_SRA_MAX;
2344     }
2345
2346     if (attr_mask & IB_QP_SQ_PSN)
2347         context->next_send_psn = cpu_to_be32(attr->sq_psn);
2348
2349     if (attr_mask & IB_QP_MAX_DEST_RD_ATOMIC) {
2350         if (attr->max_dest_rd_atomic)
2351             context->params2 |=
2352                 cpu_to_be32(fls(attr->max_dest_rd_atomic - 1) << 21);
2353         optpar |= MLX4_QP_OPTPAR_RRA_MAX;
2354     }
2355
2356     if (attr_mask & (IB_QP_ACCESS_FLAGS | IB_QP_MAX_DEST_RD_ATOMIC)) {
2357         context->params2 |= to_mlx4_access_flags(qp, attr, attr_mask);
2358         optpar |= MLX4_QP_OPTPAR_RWE | MLX4_QP_OPTPAR_RRE | MLX4_QP_OPTPAR_RAE;
2359     }
2360
2361     if (ibsrq)
2362         context->params2 |= cpu_to_be32(MLX4_QP_BIT_RIC);
2363
2364     if (attr_mask & IB_QP_MIN_RNR_TIMER) {
2365         context->rnr_nextrecvpsn |= cpu_to_be32(attr->min_rnr_timer << 24);
2366         optpar |= MLX4_QP_OPTPAR_RNR_TIMEOUT;
2367     }
2368     if (attr_mask & IB_QP_RQ_PSN)
2369         context->rnr_nextrecvpsn |= cpu_to_be32(attr->rq_psn);
2370
2371     /* proxy and tunnel qp qkeys will be changed in modify-qp wrappers */
2372     if (attr_mask & IB_QP_QKEY) {
2373         if (qp->mlx4_ib_qp_type &
2374             (MLX4_IB_QPT_PROXY_SMI_OWNER | MLX4_IB_QPT_TUN_SMI_OWNER))
2375             context->qkey = cpu_to_be32(IB_QP_SET_QKEY);
2376         else {
2377             if (mlx4_is_mfunc(dev->dev) &&
2378                 !(qp->mlx4_ib_qp_type & MLX4_IB_QPT_ANY_SRIOV) &&
2379                 (attr->qkey & MLX4_RESERVED_QKEY_MASK) ==
2380                 MLX4_RESERVED_QKEY_BASE) {
2381                 pr_err("Cannot use reserved QKEY"
2382                        " 0x%x (range 0xffff0000..0xffffffff"
2383                        " is reserved)\n", attr->qkey);
2384                 err = -EINVAL;
2385                 goto out;
2386             }
2387             context->qkey = cpu_to_be32(attr->qkey);
2388         }
2389         optpar |= MLX4_QP_OPTPAR_Q_KEY;
2390     }
2391
2392     if (ibsrq)
2393         context->srqn = cpu_to_be32(1 << 24 |
2394                         to_msrq(ibsrq)->msrq.srqn);
2395
2396     if (qp->rq.wqe_cnt &&
2397         cur_state == IB_QPS_RESET &&
2398         new_state == IB_QPS_INIT)
2399         context->db_rec_addr = cpu_to_be64(qp->db.dma);
2400
2401     if (cur_state == IB_QPS_INIT &&
2402         new_state == IB_QPS_RTR  &&
2403         (qp_type == IB_QPT_GSI || qp_type == IB_QPT_SMI ||
2404          qp_type == IB_QPT_UD || qp_type == IB_QPT_RAW_PACKET)) {
2405         context->pri_path.sched_queue = (qp->port - 1) << 6;
2406         if (qp->mlx4_ib_qp_type == MLX4_IB_QPT_SMI ||
2407             qp->mlx4_ib_qp_type &
2408             (MLX4_IB_QPT_PROXY_SMI_OWNER | MLX4_IB_QPT_TUN_SMI_OWNER)) {
2409             context->pri_path.sched_queue |= MLX4_IB_DEFAULT_QP0_SCHED_QUEUE;
2410             if (qp->mlx4_ib_qp_type != MLX4_IB_QPT_SMI)
2411                 context->pri_path.fl = 0x80;
2412         } else {
2413             if (qp->mlx4_ib_qp_type & MLX4_IB_QPT_ANY_SRIOV)
2414                 context->pri_path.fl = 0x80;
2415             context->pri_path.sched_queue |= MLX4_IB_DEFAULT_SCHED_QUEUE;
2416         }
2417         if (rdma_port_get_link_layer(&dev->ib_dev, qp->port) ==
2418             IB_LINK_LAYER_ETHERNET) {
2419             if (qp->mlx4_ib_qp_type == MLX4_IB_QPT_TUN_GSI ||
2420                 qp->mlx4_ib_qp_type == MLX4_IB_QPT_GSI)
2421                 context->pri_path.feup = 1 << 7; /* don't fsm */
2422             /* handle smac_index */
2423             if (qp->mlx4_ib_qp_type == MLX4_IB_QPT_UD ||
2424                 qp->mlx4_ib_qp_type == MLX4_IB_QPT_PROXY_GSI ||
2425                 qp->mlx4_ib_qp_type == MLX4_IB_QPT_TUN_GSI) {
2426                 err = handle_eth_ud_smac_index(dev, qp, context);
2427                 if (err) {
2428                     err = -EINVAL;
2429                     goto out;
2430                 }
2431                 if (qp->mlx4_ib_qp_type == MLX4_IB_QPT_PROXY_GSI)
2432                     dev->qp1_proxy[qp->port - 1] = qp;
2433             }
2434         }
2435     }
2436
2437     if (qp_type == IB_QPT_RAW_PACKET) {
2438         context->pri_path.ackto = (context->pri_path.ackto & 0xf8) |
2439                     MLX4_IB_LINK_TYPE_ETH;
2440         if (dev->dev->caps.tunnel_offload_mode ==  MLX4_TUNNEL_OFFLOAD_MODE_VXLAN) {
2441             /* set QP to receive both tunneled & non-tunneled packets */
2442             if (!rwq_ind_tbl)
2443                 context->srqn = cpu_to_be32(7 << 28);
2444         }
2445     }
2446
2447     if (qp_type == IB_QPT_UD && (new_state == IB_QPS_RTR)) {
2448         int is_eth = rdma_port_get_link_layer(
2449                 &dev->ib_dev, qp->port) ==
2450                 IB_LINK_LAYER_ETHERNET;
2451         if (is_eth) {
2452             context->pri_path.ackto = MLX4_IB_LINK_TYPE_ETH;
2453             optpar |= MLX4_QP_OPTPAR_PRIMARY_ADDR_PATH;
2454         }
2455     }
2456
2457     if (cur_state == IB_QPS_RTS && new_state == IB_QPS_SQD  &&
2458         attr_mask & IB_QP_EN_SQD_ASYNC_NOTIFY && attr->en_sqd_async_notify)
2459         sqd_event = 1;
2460     else
2461         sqd_event = 0;
2462
2463     if (!ucontext &&
2464         cur_state == IB_QPS_RESET &&
2465         new_state == IB_QPS_INIT)
2466         context->rlkey_roce_mode |= (1 << 4);
2467
2468     /*
2469      * Before passing a kernel QP to the HW, make sure that the
2470      * ownership bits of the send queue are set and the SQ
2471      * headroom is stamped so that the hardware doesn't start
2472      * processing stale work requests.
2473      */
2474     if (!ucontext &&
2475         cur_state == IB_QPS_RESET &&
2476         new_state == IB_QPS_INIT) {
2477         struct mlx4_wqe_ctrl_seg *ctrl;
2478         int i;
2479
2480         for (i = 0; i < qp->sq.wqe_cnt; ++i) {
2481             ctrl = get_send_wqe(qp, i);
2482             ctrl->owner_opcode = cpu_to_be32(1 << 31);
2483             ctrl->qpn_vlan.fence_size =
2484                 1 << (qp->sq.wqe_shift - 4);
2485             stamp_send_wqe(qp, i);
2486         }
2487     }
2488
2489     if (rwq_ind_tbl &&
2490         cur_state == IB_QPS_RESET &&
2491         new_state == IB_QPS_INIT) {
2492         fill_qp_rss_context(context, qp);
2493         context->flags |= cpu_to_be32(1 << MLX4_RSS_QPC_FLAG_OFFSET);
2494     }
2495
2496     err = mlx4_qp_modify(dev->dev, &qp->mtt, to_mlx4_state(cur_state),
2497                  to_mlx4_state(new_state), context, optpar,
2498                  sqd_event, &qp->mqp);
2499     if (err)
2500         goto out;
2501
2502     qp->state = new_state;
2503
2504     if (attr_mask & IB_QP_ACCESS_FLAGS)
2505         qp->atomic_rd_en = attr->qp_access_flags;
2506     if (attr_mask & IB_QP_MAX_DEST_RD_ATOMIC)
2507         qp->resp_depth = attr->max_dest_rd_atomic;
2508     if (attr_mask & IB_QP_PORT) {
2509         qp->port = attr->port_num;
2510         update_mcg_macs(dev, qp);
2511     }
2512     if (attr_mask & IB_QP_ALT_PATH)
2513         qp->alt_port = attr->alt_port_num;
2514
2515     if (is_sqp(dev, qp))
2516         store_sqp_attrs(qp->sqp, attr, attr_mask);
2517
2518     /*
2519      * If we moved QP0 to RTR, bring the IB link up; if we moved
2520      * QP0 to RESET or ERROR, bring the link back down.
2521      */
2522     if (is_qp0(dev, qp)) {
2523         if (cur_state != IB_QPS_RTR && new_state == IB_QPS_RTR)
2524             if (mlx4_INIT_PORT(dev->dev, qp->port))
2525                 pr_warn("INIT_PORT failed for port %d\n",
2526                        qp->port);
2527
2528         if (cur_state != IB_QPS_RESET && cur_state != IB_QPS_ERR &&
2529             (new_state == IB_QPS_RESET || new_state == IB_QPS_ERR))
2530             mlx4_CLOSE_PORT(dev->dev, qp->port);
2531     }
2532
2533     /*
2534      * If we moved a kernel QP to RESET, clean up all old CQ
2535      * entries and reinitialize the QP.
2536      */
2537     if (new_state == IB_QPS_RESET) {
2538         if (!ucontext) {
2539             mlx4_ib_cq_clean(recv_cq, qp->mqp.qpn,
2540                      ibsrq ? to_msrq(ibsrq) : NULL);
2541             if (send_cq != recv_cq)
2542                 mlx4_ib_cq_clean(send_cq, qp->mqp.qpn, NULL);
2543
2544             qp->rq.head = 0;
2545             qp->rq.tail = 0;
2546             qp->sq.head = 0;
2547             qp->sq.tail = 0;
2548             qp->sq_next_wqe = 0;
2549             if (qp->rq.wqe_cnt)
2550                 *qp->db.db  = 0;
2551
2552             if (qp->flags & MLX4_IB_QP_NETIF)
2553                 mlx4_ib_steer_qp_reg(dev, qp, 0);
2554         }
2555         if (qp->pri.smac || (!qp->pri.smac && qp->pri.smac_port)) {
2556             mlx4_unregister_mac(dev->dev, qp->pri.smac_port, qp->pri.smac);
2557             qp->pri.smac = 0;
2558             qp->pri.smac_port = 0;
2559         }
2560         if (qp->alt.smac) {
2561             mlx4_unregister_mac(dev->dev, qp->alt.smac_port, qp->alt.smac);
2562             qp->alt.smac = 0;
2563         }
2564         if (qp->pri.vid < 0x1000) {
2565             mlx4_unregister_vlan(dev->dev, qp->pri.vlan_port, qp->pri.vid);
2566             qp->pri.vid = 0xFFFF;
2567             qp->pri.candidate_vid = 0xFFFF;
2568             qp->pri.update_vid = 0;
2569         }
2570
2571         if (qp->alt.vid < 0x1000) {
2572             mlx4_unregister_vlan(dev->dev, qp->alt.vlan_port, qp->alt.vid);
2573             qp->alt.vid = 0xFFFF;
2574             qp->alt.candidate_vid = 0xFFFF;
2575             qp->alt.update_vid = 0;
2576         }
2577     }
2578 out:
2579     if (err && qp->counter_index)
2580         mlx4_ib_free_qp_counter(dev, qp);
2581     if (err && steer_qp)
2582         mlx4_ib_steer_qp_reg(dev, qp, 0);
2583     kfree(context);
2584     if (qp->pri.candidate_smac ||
2585         (!qp->pri.candidate_smac && qp->pri.candidate_smac_port)) {
2586         if (err) {
2587             mlx4_unregister_mac(dev->dev, qp->pri.candidate_smac_port, qp->pri.candidate_smac);
2588         } else {
2589             if (qp->pri.smac || (!qp->pri.smac && qp->pri.smac_port))
2590                 mlx4_unregister_mac(dev->dev, qp->pri.smac_port, qp->pri.smac);
2591             qp->pri.smac = qp->pri.candidate_smac;
2592             qp->pri.smac_index = qp->pri.candidate_smac_index;
2593             qp->pri.smac_port = qp->pri.candidate_smac_port;
2594         }
2595         qp->pri.candidate_smac = 0;
2596         qp->pri.candidate_smac_index = 0;
2597         qp->pri.candidate_smac_port = 0;
2598     }
2599     if (qp->alt.candidate_smac) {
2600         if (err) {
2601             mlx4_unregister_mac(dev->dev, qp->alt.candidate_smac_port, qp->alt.candidate_smac);
2602         } else {
2603             if (qp->alt.smac)
2604                 mlx4_unregister_mac(dev->dev, qp->alt.smac_port, qp->alt.smac);
2605             qp->alt.smac = qp->alt.candidate_smac;
2606             qp->alt.smac_index = qp->alt.candidate_smac_index;
2607             qp->alt.smac_port = qp->alt.candidate_smac_port;
2608         }
2609         qp->alt.candidate_smac = 0;
2610         qp->alt.candidate_smac_index = 0;
2611         qp->alt.candidate_smac_port = 0;
2612     }
2613
2614     if (qp->pri.update_vid) {
2615         if (err) {
2616             if (qp->pri.candidate_vid < 0x1000)
2617                 mlx4_unregister_vlan(dev->dev, qp->pri.candidate_vlan_port,
2618                              qp->pri.candidate_vid);
2619         } else {
2620             if (qp->pri.vid < 0x1000)
2621                 mlx4_unregister_vlan(dev->dev, qp->pri.vlan_port,
2622                              qp->pri.vid);
2623             qp->pri.vid = qp->pri.candidate_vid;
2624             qp->pri.vlan_port = qp->pri.candidate_vlan_port;
2625             qp->pri.vlan_index =  qp->pri.candidate_vlan_index;
2626         }
2627         qp->pri.candidate_vid = 0xFFFF;
2628         qp->pri.update_vid = 0;
2629     }
2630
2631     if (qp->alt.update_vid) {
2632         if (err) {
2633             if (qp->alt.candidate_vid < 0x1000)
2634                 mlx4_unregister_vlan(dev->dev, qp->alt.candidate_vlan_port,
2635                              qp->alt.candidate_vid);
2636         } else {
2637             if (qp->alt.vid < 0x1000)
2638                 mlx4_unregister_vlan(dev->dev, qp->alt.vlan_port,
2639                              qp->alt.vid);
2640             qp->alt.vid = qp->alt.candidate_vid;
2641             qp->alt.vlan_port = qp->alt.candidate_vlan_port;
2642             qp->alt.vlan_index =  qp->alt.candidate_vlan_index;
2643         }
2644         qp->alt.candidate_vid = 0xFFFF;
2645         qp->alt.update_vid = 0;
2646     }
2647
2648     return err;
2649 }
2650
2651 enum {
2652     MLX4_IB_MODIFY_QP_RSS_SUP_ATTR_MSK = (IB_QP_STATE   |
2653                           IB_QP_PORT),
2654 };
2655
2656 static int _mlx4_ib_modify_qp(struct ib_qp *ibqp, struct ib_qp_attr *attr,
2657                   int attr_mask, struct ib_udata *udata)
2658 {
2659     struct mlx4_ib_dev *dev = to_mdev(ibqp->device);
2660     struct mlx4_ib_qp *qp = to_mqp(ibqp);
2661     enum ib_qp_state cur_state, new_state;
2662     int err = -EINVAL;
2663     mutex_lock(&qp->mutex);
2664
2665     cur_state = attr_mask & IB_QP_CUR_STATE ? attr->cur_qp_state : qp->state;
2666     new_state = attr_mask & IB_QP_STATE ? attr->qp_state : cur_state;
2667
2668     if (!ib_modify_qp_is_ok(cur_state, new_state, ibqp->qp_type,
2669                 attr_mask)) {
2670         pr_debug("qpn 0x%x: invalid attribute mask specified "
2671              "for transition %d to %d. qp_type %d,"
2672              " attr_mask 0x%x\n",
2673              ibqp->qp_num, cur_state, new_state,
2674              ibqp->qp_type, attr_mask);
2675         goto out;
2676     }
2677
2678     if (ibqp->rwq_ind_tbl) {
2679         if (!(((cur_state == IB_QPS_RESET) &&
2680                (new_state == IB_QPS_INIT)) ||
2681               ((cur_state == IB_QPS_INIT)  &&
2682                (new_state == IB_QPS_RTR)))) {
2683             pr_debug("qpn 0x%x: RSS QP unsupported transition %d to %d\n",
2684                  ibqp->qp_num, cur_state, new_state);
2685
2686             err = -EOPNOTSUPP;
2687             goto out;
2688         }
2689
2690         if (attr_mask & ~MLX4_IB_MODIFY_QP_RSS_SUP_ATTR_MSK) {
2691             pr_debug("qpn 0x%x: RSS QP unsupported attribute mask 0x%x for transition %d to %d\n",
2692                  ibqp->qp_num, attr_mask, cur_state, new_state);
2693
2694             err = -EOPNOTSUPP;
2695             goto out;
2696         }
2697     }
2698
2699     if (mlx4_is_bonded(dev->dev) && (attr_mask & IB_QP_PORT)) {
2700         if ((cur_state == IB_QPS_RESET) && (new_state == IB_QPS_INIT)) {
2701             if ((ibqp->qp_type == IB_QPT_RC) ||
2702                 (ibqp->qp_type == IB_QPT_UD) ||
2703                 (ibqp->qp_type == IB_QPT_UC) ||
2704                 (ibqp->qp_type == IB_QPT_RAW_PACKET) ||
2705                 (ibqp->qp_type == IB_QPT_XRC_INI)) {
2706                 attr->port_num = mlx4_ib_bond_next_port(dev);
2707             }
2708         } else {
2709             /* no sense in changing port_num
2710              * when ports are bonded */
2711             attr_mask &= ~IB_QP_PORT;
2712         }
2713     }
2714
2715     if ((attr_mask & IB_QP_PORT) &&
2716         (attr->port_num == 0 || attr->port_num > dev->num_ports)) {
2717         pr_debug("qpn 0x%x: invalid port number (%d) specified "
2718              "for transition %d to %d. qp_type %d\n",
2719              ibqp->qp_num, attr->port_num, cur_state,
2720              new_state, ibqp->qp_type);
2721         goto out;
2722     }
2723
2724     if ((attr_mask & IB_QP_PORT) && (ibqp->qp_type == IB_QPT_RAW_PACKET) &&
2725         (rdma_port_get_link_layer(&dev->ib_dev, attr->port_num) !=
2726          IB_LINK_LAYER_ETHERNET))
2727         goto out;
2728
2729     if (attr_mask & IB_QP_PKEY_INDEX) {
2730         int p = attr_mask & IB_QP_PORT ? attr->port_num : qp->port;
2731         if (attr->pkey_index >= dev->dev->caps.pkey_table_len[p]) {
2732             pr_debug("qpn 0x%x: invalid pkey index (%d) specified "
2733                  "for transition %d to %d. qp_type %d\n",
2734                  ibqp->qp_num, attr->pkey_index, cur_state,
2735                  new_state, ibqp->qp_type);
2736             goto out;
2737         }
2738     }
2739
2740     if (attr_mask & IB_QP_MAX_QP_RD_ATOMIC &&
2741         attr->max_rd_atomic > dev->dev->caps.max_qp_init_rdma) {
2742         pr_debug("qpn 0x%x: max_rd_atomic (%d) too large. "
2743              "Transition %d to %d. qp_type %d\n",
2744              ibqp->qp_num, attr->max_rd_atomic, cur_state,
2745              new_state, ibqp->qp_type);
2746         goto out;
2747     }
2748
2749     if (attr_mask & IB_QP_MAX_DEST_RD_ATOMIC &&
2750         attr->max_dest_rd_atomic > dev->dev->caps.max_qp_dest_rdma) {
2751         pr_debug("qpn 0x%x: max_dest_rd_atomic (%d) too large. "
2752              "Transition %d to %d. qp_type %d\n",
2753              ibqp->qp_num, attr->max_dest_rd_atomic, cur_state,
2754              new_state, ibqp->qp_type);
2755         goto out;
2756     }
2757
2758     if (cur_state == new_state && cur_state == IB_QPS_RESET) {
2759         err = 0;
2760         goto out;
2761     }
2762
2763     if (ibqp->rwq_ind_tbl && (new_state == IB_QPS_INIT)) {
2764         err = bringup_rss_rwqs(ibqp->rwq_ind_tbl, attr->port_num,
2765                        udata);
2766         if (err)
2767             goto out;
2768     }
2769
2770     err = __mlx4_ib_modify_qp(ibqp, MLX4_IB_QP_SRC, attr, attr_mask,
2771                   cur_state, new_state, udata);
2772
2773     if (ibqp->rwq_ind_tbl && err)
2774         bring_down_rss_rwqs(ibqp->rwq_ind_tbl, udata);
2775
2776     if (mlx4_is_bonded(dev->dev) && (attr_mask & IB_QP_PORT))
2777         attr->port_num = 1;
2778
2779 out:
2780     mutex_unlock(&qp->mutex);
2781     return err;
2782 }
2783
2784 int mlx4_ib_modify_qp(struct ib_qp *ibqp, struct ib_qp_attr *attr,
2785               int attr_mask, struct ib_udata *udata)
2786 {
2787     struct mlx4_ib_qp *mqp = to_mqp(ibqp);
2788     int ret;
2789
2790     if (attr_mask & ~IB_QP_ATTR_STANDARD_BITS)
2791         return -EOPNOTSUPP;
2792
2793     ret = _mlx4_ib_modify_qp(ibqp, attr, attr_mask, udata);
2794
2795     if (mqp->mlx4_ib_qp_type == MLX4_IB_QPT_GSI) {
2796         struct mlx4_ib_sqp *sqp = mqp->sqp;
2797         int err = 0;
2798
2799         if (sqp->roce_v2_gsi)
2800             err = ib_modify_qp(sqp->roce_v2_gsi, attr, attr_mask);
2801         if (err)
2802             pr_err("Failed to modify GSI QP for RoCEv2 (%d)\n",
2803                    err);
2804     }
2805     return ret;
2806 }
2807
2808 static int vf_get_qp0_qkey(struct mlx4_dev *dev, int qpn, u32 *qkey)
2809 {
2810     int i;
2811     for (i = 0; i < dev->caps.num_ports; i++) {
2812         if (qpn == dev->caps.spec_qps[i].qp0_proxy ||
2813             qpn == dev->caps.spec_qps[i].qp0_tunnel) {
2814             *qkey = dev->caps.spec_qps[i].qp0_qkey;
2815             return 0;
2816         }
2817     }
2818     return -EINVAL;
2819 }
2820
2821 static int build_sriov_qp0_header(struct mlx4_ib_qp *qp,
2822                   const struct ib_ud_wr *wr,
2823                   void *wqe, unsigned *mlx_seg_len)
2824 {
2825     struct mlx4_ib_dev *mdev = to_mdev(qp->ibqp.device);
2826     struct mlx4_ib_sqp *sqp = qp->sqp;
2827     struct ib_device *ib_dev = qp->ibqp.device;
2828     struct mlx4_wqe_mlx_seg *mlx = wqe;
2829     struct mlx4_wqe_inline_seg *inl = wqe + sizeof *mlx;
2830     struct mlx4_ib_ah *ah = to_mah(wr->ah);
2831     u16 pkey;
2832     u32 qkey;
2833     int send_size;
2834     int header_size;
2835     int spc;
2836     int err;
2837     int i;
2838
2839     if (wr->wr.opcode != IB_WR_SEND)
2840         return -EINVAL;
2841
2842     send_size = 0;
2843
2844     for (i = 0; i < wr->wr.num_sge; ++i)
2845         send_size += wr->wr.sg_list[i].length;
2846
2847     /* for proxy-qp0 sends, need to add in size of tunnel header */
2848     /* for tunnel-qp0 sends, tunnel header is already in s/g list */
2849     if (qp->mlx4_ib_qp_type == MLX4_IB_QPT_PROXY_SMI_OWNER)
2850         send_size += sizeof (struct mlx4_ib_tunnel_header);
2851
2852     ib_ud_header_init(send_size, 1, 0, 0, 0, 0, 0, 0, &sqp->ud_header);
2853
2854     if (qp->mlx4_ib_qp_type == MLX4_IB_QPT_PROXY_SMI_OWNER) {
2855         sqp->ud_header.lrh.service_level =
2856             be32_to_cpu(ah->av.ib.sl_tclass_flowlabel) >> 28;
2857         sqp->ud_header.lrh.destination_lid =
2858             cpu_to_be16(ah->av.ib.g_slid & 0x7f);
2859         sqp->ud_header.lrh.source_lid =
2860             cpu_to_be16(ah->av.ib.g_slid & 0x7f);
2861     }
2862
2863     mlx->flags &= cpu_to_be32(MLX4_WQE_CTRL_CQ_UPDATE);
2864
2865     /* force loopback */
2866     mlx->flags |= cpu_to_be32(MLX4_WQE_MLX_VL15 | 0x1 | MLX4_WQE_MLX_SLR);
2867     mlx->rlid = sqp->ud_header.lrh.destination_lid;
2868
2869     sqp->ud_header.lrh.virtual_lane    = 0;
2870     sqp->ud_header.bth.solicited_event = !!(wr->wr.send_flags & IB_SEND_SOLICITED);
2871     err = ib_get_cached_pkey(ib_dev, qp->port, 0, &pkey);
2872     if (err)
2873         return err;
2874     sqp->ud_header.bth.pkey = cpu_to_be16(pkey);
2875     if (qp->mlx4_ib_qp_type == MLX4_IB_QPT_TUN_SMI_OWNER)
2876         sqp->ud_header.bth.destination_qpn = cpu_to_be32(wr->remote_qpn);
2877     else
2878         sqp->ud_header.bth.destination_qpn =
2879             cpu_to_be32(mdev->dev->caps.spec_qps[qp->port - 1].qp0_tunnel);
2880
2881     sqp->ud_header.bth.psn = cpu_to_be32((sqp->send_psn++) & ((1 << 24) - 1));
2882     if (mlx4_is_master(mdev->dev)) {
2883         if (mlx4_get_parav_qkey(mdev->dev, qp->mqp.qpn, &qkey))
2884             return -EINVAL;
2885     } else {
2886         if (vf_get_qp0_qkey(mdev->dev, qp->mqp.qpn, &qkey))
2887             return -EINVAL;
2888     }
2889     sqp->ud_header.deth.qkey = cpu_to_be32(qkey);
2890     sqp->ud_header.deth.source_qpn = cpu_to_be32(qp->mqp.qpn);
2891
2892     sqp->ud_header.bth.opcode        = IB_OPCODE_UD_SEND_ONLY;
2893     sqp->ud_header.immediate_present = 0;
2894
2895     header_size = ib_ud_header_pack(&sqp->ud_header, sqp->header_buf);
2896
2897     /*
2898      * Inline data segments may not cross a 64 byte boundary.  If
2899      * our UD header is bigger than the space available up to the
2900      * next 64 byte boundary in the WQE, use two inline data
2901      * segments to hold the UD header.
2902      */
2903     spc = MLX4_INLINE_ALIGN -
2904           ((unsigned long) (inl + 1) & (MLX4_INLINE_ALIGN - 1));
2905     if (header_size <= spc) {
2906         inl->byte_count = cpu_to_be32(1 << 31 | header_size);
2907         memcpy(inl + 1, sqp->header_buf, header_size);
2908         i = 1;
2909     } else {
2910         inl->byte_count = cpu_to_be32(1 << 31 | spc);
2911         memcpy(inl + 1, sqp->header_buf, spc);
2912
2913         inl = (void *) (inl + 1) + spc;
2914         memcpy(inl + 1, sqp->header_buf + spc, header_size - spc);
2915         /*
2916          * Need a barrier here to make sure all the data is
2917          * visible before the byte_count field is set.
2918          * Otherwise the HCA prefetcher could grab the 64-byte
2919          * chunk with this inline segment and get a valid (!=
2920          * 0xffffffff) byte count but stale data, and end up
2921          * generating a packet with bad headers.
2922          *
2923          * The first inline segment's byte_count field doesn't
2924          * need a barrier, because it comes after a
2925          * control/MLX segment and therefore is at an offset
2926          * of 16 mod 64.
2927          */
2928         wmb();
2929         inl->byte_count = cpu_to_be32(1 << 31 | (header_size - spc));
2930         i = 2;
2931     }
2932
2933     *mlx_seg_len =
2934     ALIGN(i * sizeof (struct mlx4_wqe_inline_seg) + header_size, 16);
2935     return 0;
2936 }
2937
2938 static u8 sl_to_vl(struct mlx4_ib_dev *dev, u8 sl, int port_num)
2939 {
2940     union sl2vl_tbl_to_u64 tmp_vltab;
2941     u8 vl;
2942
2943     if (sl > 15)
2944         return 0xf;
2945     tmp_vltab.sl64 = atomic64_read(&dev->sl2vl[port_num - 1]);
2946     vl = tmp_vltab.sl8[sl >> 1];
2947     if (sl & 1)
2948         vl &= 0x0f;
2949     else
2950         vl >>= 4;
2951     return vl;
2952 }
2953
2954 static int fill_gid_by_hw_index(struct mlx4_ib_dev *ibdev, u8 port_num,
2955                 int index, union ib_gid *gid,
2956                 enum ib_gid_type *gid_type)
2957 {
2958     struct mlx4_ib_iboe *iboe = &ibdev->iboe;
2959     struct mlx4_port_gid_table *port_gid_table;
2960     unsigned long flags;
2961
2962     port_gid_table = &iboe->gids[port_num - 1];
2963     spin_lock_irqsave(&iboe->lock, flags);
2964     memcpy(gid, &port_gid_table->gids[index].gid, sizeof(*gid));
2965     *gid_type = port_gid_table->gids[index].gid_type;
2966     spin_unlock_irqrestore(&iboe->lock, flags);
2967     if (rdma_is_zero_gid(gid))
2968         return -ENOENT;
2969
2970     return 0;
2971 }
2972
2973 #define MLX4_ROCEV2_QP1_SPORT 0xC000
2974 static int build_mlx_header(struct mlx4_ib_qp *qp, const struct ib_ud_wr *wr,
2975                 void *wqe, unsigned *mlx_seg_len)
2976 {
2977     struct mlx4_ib_sqp *sqp = qp->sqp;
2978     struct ib_device *ib_dev = qp->ibqp.device;
2979     struct mlx4_ib_dev *ibdev = to_mdev(ib_dev);
2980     struct mlx4_wqe_mlx_seg *mlx = wqe;
2981     struct mlx4_wqe_ctrl_seg *ctrl = wqe;
2982     struct mlx4_wqe_inline_seg *inl = wqe + sizeof *mlx;
2983     struct mlx4_ib_ah *ah = to_mah(wr->ah);
2984     union ib_gid sgid;
2985     u16 pkey;
2986     int send_size;
2987     int header_size;
2988     int spc;
2989     int i;
2990     int err = 0;
2991     u16 vlan = 0xffff;
2992     bool is_eth;
2993     bool is_vlan = false;
2994     bool is_grh;
2995     bool is_udp = false;
2996     int ip_version = 0;
2997
2998     send_size = 0;
2999     for (i = 0; i < wr->wr.num_sge; ++i)
3000         send_size += wr->wr.sg_list[i].length;
3001
3002     is_eth = rdma_port_get_link_layer(qp->ibqp.device, qp->port) == IB_LINK_LAYER_ETHERNET;
3003     is_grh = mlx4_ib_ah_grh_present(ah);
3004     if (is_eth) {
3005         enum ib_gid_type gid_type;
3006         if (mlx4_is_mfunc(to_mdev(ib_dev)->dev)) {
3007             /* When multi-function is enabled, the ib_core gid
3008              * indexes don't necessarily match the hw ones, so
3009              * we must use our own cache */
3010             err = mlx4_get_roce_gid_from_slave(to_mdev(ib_dev)->dev,
3011                                be32_to_cpu(ah->av.ib.port_pd) >> 24,
3012                                ah->av.ib.gid_index, &sgid.raw[0]);
3013             if (err)
3014                 return err;
3015         } else  {
3016             err = fill_gid_by_hw_index(ibdev, qp->port,
3017                            ah->av.ib.gid_index, &sgid,
3018                            &gid_type);
3019             if (!err) {
3020                 is_udp = gid_type == IB_GID_TYPE_ROCE_UDP_ENCAP;
3021                 if (is_udp) {
3022                     if (ipv6_addr_v4mapped((struct in6_addr *)&sgid))
3023                         ip_version = 4;
3024                     else
3025                         ip_version = 6;
3026                     is_grh = false;
3027                 }
3028             } else {
3029                 return err;
3030             }
3031         }
3032         if (ah->av.eth.vlan != cpu_to_be16(0xffff)) {
3033             vlan = be16_to_cpu(ah->av.eth.vlan) & 0x0fff;
3034             is_vlan = true;
3035         }
3036     }
3037     err = ib_ud_header_init(send_size, !is_eth, is_eth, is_vlan, is_grh,
3038               ip_version, is_udp, 0, &sqp->ud_header);
3039     if (err)
3040         return err;
3041
3042     if (!is_eth) {
3043         sqp->ud_header.lrh.service_level =
3044             be32_to_cpu(ah->av.ib.sl_tclass_flowlabel) >> 28;
3045         sqp->ud_header.lrh.destination_lid = ah->av.ib.dlid;
3046         sqp->ud_header.lrh.source_lid = cpu_to_be16(ah->av.ib.g_slid & 0x7f);
3047     }
3048
3049     if (is_grh || (ip_version == 6)) {
3050         sqp->ud_header.grh.traffic_class =
3051             (be32_to_cpu(ah->av.ib.sl_tclass_flowlabel) >> 20) & 0xff;
3052         sqp->ud_header.grh.flow_label    =
3053             ah->av.ib.sl_tclass_flowlabel & cpu_to_be32(0xfffff);
3054         sqp->ud_header.grh.hop_limit     = ah->av.ib.hop_limit;
3055         if (is_eth) {
3056             memcpy(sqp->ud_header.grh.source_gid.raw, sgid.raw, 16);
3057         } else {
3058             if (mlx4_is_mfunc(to_mdev(ib_dev)->dev)) {
3059                 /* When multi-function is enabled, the ib_core gid
3060                  * indexes don't necessarily match the hw ones, so
3061                  * we must use our own cache
3062                  */
3063                 sqp->ud_header.grh.source_gid.global
3064                     .subnet_prefix =
3065                     cpu_to_be64(atomic64_read(
3066                         &(to_mdev(ib_dev)
3067                               ->sriov
3068                               .demux[qp->port - 1]
3069                               .subnet_prefix)));
3070                 sqp->ud_header.grh.source_gid.global
3071                     .interface_id =
3072                     to_mdev(ib_dev)
3073                         ->sriov.demux[qp->port - 1]
3074                         .guid_cache[ah->av.ib.gid_index];
3075             } else {
3076                 sqp->ud_header.grh.source_gid =
3077                     ah->ibah.sgid_attr->gid;
3078             }
3079         }
3080         memcpy(sqp->ud_header.grh.destination_gid.raw,
3081                ah->av.ib.dgid, 16);
3082     }
3083
3084     if (ip_version == 4) {
3085         sqp->ud_header.ip4.tos =
3086             (be32_to_cpu(ah->av.ib.sl_tclass_flowlabel) >> 20) & 0xff;
3087         sqp->ud_header.ip4.id = 0;
3088         sqp->ud_header.ip4.frag_off = htons(IP_DF);
3089         sqp->ud_header.ip4.ttl = ah->av.eth.hop_limit;
3090
3091         memcpy(&sqp->ud_header.ip4.saddr,
3092                sgid.raw + 12, 4);
3093         memcpy(&sqp->ud_header.ip4.daddr, ah->av.ib.dgid + 12, 4);
3094         sqp->ud_header.ip4.check = ib_ud_ip4_csum(&sqp->ud_header);
3095     }
3096
3097     if (is_udp) {
3098         sqp->ud_header.udp.dport = htons(ROCE_V2_UDP_DPORT);
3099         sqp->ud_header.udp.sport = htons(MLX4_ROCEV2_QP1_SPORT);
3100         sqp->ud_header.udp.csum = 0;
3101     }
3102
3103     mlx->flags &= cpu_to_be32(MLX4_WQE_CTRL_CQ_UPDATE);
3104
3105     if (!is_eth) {
3106         mlx->flags |=
3107             cpu_to_be32((!qp->ibqp.qp_num ? MLX4_WQE_MLX_VL15 : 0) |
3108                     (sqp->ud_header.lrh.destination_lid ==
3109                              IB_LID_PERMISSIVE ?
3110                          MLX4_WQE_MLX_SLR :
3111                          0) |
3112                     (sqp->ud_header.lrh.service_level << 8));
3113         if (ah->av.ib.port_pd & cpu_to_be32(0x80000000))
3114             mlx->flags |= cpu_to_be32(0x1); /* force loopback */
3115         mlx->rlid = sqp->ud_header.lrh.destination_lid;
3116     }
3117
3118     switch (wr->wr.opcode) {
3119     case IB_WR_SEND:
3120         sqp->ud_header.bth.opcode    = IB_OPCODE_UD_SEND_ONLY;
3121         sqp->ud_header.immediate_present = 0;
3122         break;
3123     case IB_WR_SEND_WITH_IMM:
3124         sqp->ud_header.bth.opcode    = IB_OPCODE_UD_SEND_ONLY_WITH_IMMEDIATE;
3125         sqp->ud_header.immediate_present = 1;
3126         sqp->ud_header.immediate_data    = wr->wr.ex.imm_data;
3127         break;
3128     default:
3129         return -EINVAL;
3130     }
3131
3132     if (is_eth) {
3133         u16 ether_type;
3134         u16 pcp = (be32_to_cpu(ah->av.ib.sl_tclass_flowlabel) >> 29) << 13;
3135
3136         ether_type = (!is_udp) ? ETH_P_IBOE:
3137             (ip_version == 4 ? ETH_P_IP : ETH_P_IPV6);
3138
3139         mlx->sched_prio = cpu_to_be16(pcp);
3140
3141         ether_addr_copy(sqp->ud_header.eth.smac_h, ah->av.eth.s_mac);
3142         ether_addr_copy(sqp->ud_header.eth.dmac_h, ah->av.eth.mac);
3143         memcpy(&ctrl->srcrb_flags16[0], ah->av.eth.mac, 2);
3144         memcpy(&ctrl->imm, ah->av.eth.mac + 2, 4);
3145
3146         if (!memcmp(sqp->ud_header.eth.smac_h, sqp->ud_header.eth.dmac_h, 6))
3147             mlx->flags |= cpu_to_be32(MLX4_WQE_CTRL_FORCE_LOOPBACK);
3148         if (!is_vlan) {
3149             sqp->ud_header.eth.type = cpu_to_be16(ether_type);
3150         } else {
3151             sqp->ud_header.vlan.type = cpu_to_be16(ether_type);
3152             sqp->ud_header.vlan.tag = cpu_to_be16(vlan | pcp);
3153         }
3154     } else {
3155         sqp->ud_header.lrh.virtual_lane =
3156             !qp->ibqp.qp_num ?
3157                 15 :
3158                 sl_to_vl(to_mdev(ib_dev),
3159                      sqp->ud_header.lrh.service_level,
3160                      qp->port);
3161         if (qp->ibqp.qp_num && sqp->ud_header.lrh.virtual_lane == 15)
3162             return -EINVAL;
3163         if (sqp->ud_header.lrh.destination_lid == IB_LID_PERMISSIVE)
3164             sqp->ud_header.lrh.source_lid = IB_LID_PERMISSIVE;
3165     }
3166     sqp->ud_header.bth.solicited_event = !!(wr->wr.send_flags & IB_SEND_SOLICITED);
3167     if (!qp->ibqp.qp_num)
3168         err = ib_get_cached_pkey(ib_dev, qp->port, sqp->pkey_index,
3169                      &pkey);
3170     else
3171         err = ib_get_cached_pkey(ib_dev, qp->port, wr->pkey_index,
3172                      &pkey);
3173     if (err)
3174         return err;
3175
3176     sqp->ud_header.bth.pkey = cpu_to_be16(pkey);
3177     sqp->ud_header.bth.destination_qpn = cpu_to_be32(wr->remote_qpn);
3178     sqp->ud_header.bth.psn = cpu_to_be32((sqp->send_psn++) & ((1 << 24) - 1));
3179     sqp->ud_header.deth.qkey = cpu_to_be32(wr->remote_qkey & 0x80000000 ?
3180                            sqp->qkey : wr->remote_qkey);
3181     sqp->ud_header.deth.source_qpn = cpu_to_be32(qp->ibqp.qp_num);
3182
3183     header_size = ib_ud_header_pack(&sqp->ud_header, sqp->header_buf);
3184
3185     if (0) {
3186         pr_err("built UD header of size %d:\n", header_size);
3187         for (i = 0; i < header_size / 4; ++i) {
3188             if (i % 8 == 0)
3189                 pr_err("  [%02x] ", i * 4);
3190             pr_cont(" %08x",
3191                 be32_to_cpu(((__be32 *) sqp->header_buf)[i]));
3192             if ((i + 1) % 8 == 0)
3193                 pr_cont("\n");
3194         }
3195         pr_err("\n");
3196     }
3197
3198     /*
3199      * Inline data segments may not cross a 64 byte boundary.  If
3200      * our UD header is bigger than the space available up to the
3201      * next 64 byte boundary in the WQE, use two inline data
3202      * segments to hold the UD header.
3203      */
3204     spc = MLX4_INLINE_ALIGN -
3205         ((unsigned long) (inl + 1) & (MLX4_INLINE_ALIGN - 1));
3206     if (header_size <= spc) {
3207         inl->byte_count = cpu_to_be32(1 << 31 | header_size);
3208         memcpy(inl + 1, sqp->header_buf, header_size);
3209         i = 1;
3210     } else {
3211         inl->byte_count = cpu_to_be32(1 << 31 | spc);
3212         memcpy(inl + 1, sqp->header_buf, spc);
3213
3214         inl = (void *) (inl + 1) + spc;
3215         memcpy(inl + 1, sqp->header_buf + spc, header_size - spc);
3216         /*
3217          * Need a barrier here to make sure all the data is
3218          * visible before the byte_count field is set.
3219          * Otherwise the HCA prefetcher could grab the 64-byte
3220          * chunk with this inline segment and get a valid (!=
3221          * 0xffffffff) byte count but stale data, and end up
3222          * generating a packet with bad headers.
3223          *
3224          * The first inline segment's byte_count field doesn't
3225          * need a barrier, because it comes after a
3226          * control/MLX segment and therefore is at an offset
3227          * of 16 mod 64.
3228          */
3229         wmb();
3230         inl->byte_count = cpu_to_be32(1 << 31 | (header_size - spc));
3231         i = 2;
3232     }
3233
3234     *mlx_seg_len =
3235         ALIGN(i * sizeof (struct mlx4_wqe_inline_seg) + header_size, 16);
3236     return 0;
3237 }
3238
3239 static int mlx4_wq_overflow(struct mlx4_ib_wq *wq, int nreq, struct ib_cq *ib_cq)
3240 {
3241     unsigned cur;
3242     struct mlx4_ib_cq *cq;
3243
3244     cur = wq->head - wq->tail;
3245     if (likely(cur + nreq < wq->max_post))
3246         return 0;
3247
3248     cq = to_mcq(ib_cq);
3249     spin_lock(&cq->lock);
3250     cur = wq->head - wq->tail;
3251     spin_unlock(&cq->lock);
3252
3253     return cur + nreq >= wq->max_post;
3254 }
3255
3256 static __be32 convert_access(int acc)
3257 {
3258     return (acc & IB_ACCESS_REMOTE_ATOMIC ?
3259         cpu_to_be32(MLX4_WQE_FMR_AND_BIND_PERM_ATOMIC)       : 0) |
3260            (acc & IB_ACCESS_REMOTE_WRITE  ?
3261         cpu_to_be32(MLX4_WQE_FMR_AND_BIND_PERM_REMOTE_WRITE) : 0) |
3262            (acc & IB_ACCESS_REMOTE_READ   ?
3263         cpu_to_be32(MLX4_WQE_FMR_AND_BIND_PERM_REMOTE_READ)  : 0) |
3264            (acc & IB_ACCESS_LOCAL_WRITE   ? cpu_to_be32(MLX4_WQE_FMR_PERM_LOCAL_WRITE)  : 0) |
3265         cpu_to_be32(MLX4_WQE_FMR_PERM_LOCAL_READ);
3266 }
3267
3268 static void set_reg_seg(struct mlx4_wqe_fmr_seg *fseg,
3269             const struct ib_reg_wr *wr)
3270 {
3271     struct mlx4_ib_mr *mr = to_mmr(wr->mr);
3272
3273     fseg->flags     = convert_access(wr->access);
3274     fseg->mem_key       = cpu_to_be32(wr->key);
3275     fseg->buf_list      = cpu_to_be64(mr->page_map);
3276     fseg->start_addr    = cpu_to_be64(mr->ibmr.iova);
3277     fseg->reg_len       = cpu_to_be64(mr->ibmr.length);
3278     fseg->offset        = 0; /* XXX -- is this just for ZBVA? */
3279     fseg->page_size     = cpu_to_be32(ilog2(mr->ibmr.page_size));
3280     fseg->reserved[0]   = 0;
3281     fseg->reserved[1]   = 0;
3282 }
3283
3284 static void set_local_inv_seg(struct mlx4_wqe_local_inval_seg *iseg, u32 rkey)
3285 {
3286     memset(iseg, 0, sizeof(*iseg));
3287     iseg->mem_key = cpu_to_be32(rkey);
3288 }
3289
3290 static __always_inline void set_raddr_seg(struct mlx4_wqe_raddr_seg *rseg,
3291                       u64 remote_addr, u32 rkey)
3292 {
3293     rseg->raddr    = cpu_to_be64(remote_addr);
3294     rseg->rkey     = cpu_to_be32(rkey);
3295     rseg->reserved = 0;
3296 }
3297
3298 static void set_atomic_seg(struct mlx4_wqe_atomic_seg *aseg,
3299                const struct ib_atomic_wr *wr)
3300 {
3301     if (wr->wr.opcode == IB_WR_ATOMIC_CMP_AND_SWP) {
3302         aseg->swap_add = cpu_to_be64(wr->swap);
3303         aseg->compare  = cpu_to_be64(wr->compare_add);
3304     } else if (wr->wr.opcode == IB_WR_MASKED_ATOMIC_FETCH_AND_ADD) {
3305         aseg->swap_add = cpu_to_be64(wr->compare_add);
3306         aseg->compare  = cpu_to_be64(wr->compare_add_mask);
3307     } else {
3308         aseg->swap_add = cpu_to_be64(wr->compare_add);
3309         aseg->compare  = 0;
3310     }
3311
3312 }
3313
3314 static void set_masked_atomic_seg(struct mlx4_wqe_masked_atomic_seg *aseg,
3315                   const struct ib_atomic_wr *wr)
3316 {
3317     aseg->swap_add      = cpu_to_be64(wr->swap);
3318     aseg->swap_add_mask = cpu_to_be64(wr->swap_mask);
3319     aseg->compare       = cpu_to_be64(wr->compare_add);
3320     aseg->compare_mask  = cpu_to_be64(wr->compare_add_mask);
3321 }
3322
3323 static void set_datagram_seg(struct mlx4_wqe_datagram_seg *dseg,
3324                  const struct ib_ud_wr *wr)
3325 {
3326     memcpy(dseg->av, &to_mah(wr->ah)->av, sizeof (struct mlx4_av));
3327     dseg->dqpn = cpu_to_be32(wr->remote_qpn);
3328     dseg->qkey = cpu_to_be32(wr->remote_qkey);
3329     dseg->vlan = to_mah(wr->ah)->av.eth.vlan;
3330     memcpy(dseg->mac, to_mah(wr->ah)->av.eth.mac, 6);
3331 }
3332
3333 static void set_tunnel_datagram_seg(struct mlx4_ib_dev *dev,
3334                     struct mlx4_wqe_datagram_seg *dseg,
3335                     const struct ib_ud_wr *wr,
3336                     enum mlx4_ib_qp_type qpt)
3337 {
3338     union mlx4_ext_av *av = &to_mah(wr->ah)->av;
3339     struct mlx4_av sqp_av = {0};
3340     int port = *((u8 *) &av->ib.port_pd) & 0x3;
3341
3342     /* force loopback */
3343     sqp_av.port_pd = av->ib.port_pd | cpu_to_be32(0x80000000);
3344     sqp_av.g_slid = av->ib.g_slid & 0x7f; /* no GRH */
3345     sqp_av.sl_tclass_flowlabel = av->ib.sl_tclass_flowlabel &
3346             cpu_to_be32(0xf0000000);
3347
3348     memcpy(dseg->av, &sqp_av, sizeof (struct mlx4_av));
3349     if (qpt == MLX4_IB_QPT_PROXY_GSI)
3350         dseg->dqpn = cpu_to_be32(dev->dev->caps.spec_qps[port - 1].qp1_tunnel);
3351     else
3352         dseg->dqpn = cpu_to_be32(dev->dev->caps.spec_qps[port - 1].qp0_tunnel);
3353     /* Use QKEY from the QP context, which is set by master */
3354     dseg->qkey = cpu_to_be32(IB_QP_SET_QKEY);
3355 }
3356
3357 static void build_tunnel_header(const struct ib_ud_wr *wr, void *wqe,
3358                 unsigned *mlx_seg_len)
3359 {
3360     struct mlx4_wqe_inline_seg *inl = wqe;
3361     struct mlx4_ib_tunnel_header hdr;
3362     struct mlx4_ib_ah *ah = to_mah(wr->ah);
3363     int spc;
3364     int i;
3365
3366     memcpy(&hdr.av, &ah->av, sizeof hdr.av);
3367     hdr.remote_qpn = cpu_to_be32(wr->remote_qpn);
3368     hdr.pkey_index = cpu_to_be16(wr->pkey_index);
3369     hdr.qkey = cpu_to_be32(wr->remote_qkey);
3370     memcpy(hdr.mac, ah->av.eth.mac, 6);
3371     hdr.vlan = ah->av.eth.vlan;
3372
3373     spc = MLX4_INLINE_ALIGN -
3374         ((unsigned long) (inl + 1) & (MLX4_INLINE_ALIGN - 1));
3375     if (sizeof (hdr) <= spc) {
3376         memcpy(inl + 1, &hdr, sizeof (hdr));
3377         wmb();
3378         inl->byte_count = cpu_to_be32(1 << 31 | sizeof (hdr));
3379         i = 1;
3380     } else {
3381         memcpy(inl + 1, &hdr, spc);
3382         wmb();
3383         inl->byte_count = cpu_to_be32(1 << 31 | spc);
3384
3385         inl = (void *) (inl + 1) + spc;
3386         memcpy(inl + 1, (void *) &hdr + spc, sizeof (hdr) - spc);
3387         wmb();
3388         inl->byte_count = cpu_to_be32(1 << 31 | (sizeof (hdr) - spc));
3389         i = 2;
3390     }
3391
3392     *mlx_seg_len =
3393         ALIGN(i * sizeof (struct mlx4_wqe_inline_seg) + sizeof (hdr), 16);
3394 }
3395
3396 static void set_mlx_icrc_seg(void *dseg)
3397 {
3398     u32 *t = dseg;
3399     struct mlx4_wqe_inline_seg *iseg = dseg;
3400
3401     t[1] = 0;
3402
3403     /*
3404      * Need a barrier here before writing the byte_count field to
3405      * make sure that all the data is visible before the
3406      * byte_count field is set.  Otherwise, if the segment begins
3407      * a new cacheline, the HCA prefetcher could grab the 64-byte
3408      * chunk and get a valid (!= * 0xffffffff) byte count but
3409      * stale data, and end up sending the wrong data.
3410      */
3411     wmb();
3412
3413     iseg->byte_count = cpu_to_be32((1 << 31) | 4);
3414 }
3415
3416 static void set_data_seg(struct mlx4_wqe_data_seg *dseg, struct ib_sge *sg)
3417 {
3418     dseg->lkey       = cpu_to_be32(sg->lkey);
3419     dseg->addr       = cpu_to_be64(sg->addr);
3420
3421     /*
3422      * Need a barrier here before writing the byte_count field to
3423      * make sure that all the data is visible before the
3424      * byte_count field is set.  Otherwise, if the segment begins
3425      * a new cacheline, the HCA prefetcher could grab the 64-byte
3426      * chunk and get a valid (!= * 0xffffffff) byte count but
3427      * stale data, and end up sending the wrong data.
3428      */
3429     wmb();
3430
3431     dseg->byte_count = cpu_to_be32(sg->length);
3432 }
3433
3434 static void __set_data_seg(struct mlx4_wqe_data_seg *dseg, struct ib_sge *sg)
3435 {
3436     dseg->byte_count = cpu_to_be32(sg->length);
3437     dseg->lkey       = cpu_to_be32(sg->lkey);
3438     dseg->addr       = cpu_to_be64(sg->addr);
3439 }
3440
3441 static int build_lso_seg(struct mlx4_wqe_lso_seg *wqe,
3442              const struct ib_ud_wr *wr, struct mlx4_ib_qp *qp,
3443              unsigned *lso_seg_len, __be32 *lso_hdr_sz, __be32 *blh)
3444 {
3445     unsigned halign = ALIGN(sizeof *wqe + wr->hlen, 16);
3446
3447     if (unlikely(halign > MLX4_IB_CACHE_LINE_SIZE))
3448         *blh = cpu_to_be32(1 << 6);
3449
3450     if (unlikely(!(qp->flags & MLX4_IB_QP_LSO) &&
3451              wr->wr.num_sge > qp->sq.max_gs - (halign >> 4)))
3452         return -EINVAL;
3453
3454     memcpy(wqe->header, wr->header, wr->hlen);
3455
3456     *lso_hdr_sz  = cpu_to_be32(wr->mss << 16 | wr->hlen);
3457     *lso_seg_len = halign;
3458     return 0;
3459 }
3460
3461 static __be32 send_ieth(const struct ib_send_wr *wr)
3462 {
3463     switch (wr->opcode) {
3464     case IB_WR_SEND_WITH_IMM:
3465     case IB_WR_RDMA_WRITE_WITH_IMM:
3466         return wr->ex.imm_data;
3467
3468     case IB_WR_SEND_WITH_INV:
3469         return cpu_to_be32(wr->ex.invalidate_rkey);
3470
3471     default:
3472         return 0;
3473     }
3474 }
3475
3476 static void add_zero_len_inline(void *wqe)
3477 {
3478     struct mlx4_wqe_inline_seg *inl = wqe;
3479     memset(wqe, 0, 16);
3480     inl->byte_count = cpu_to_be32(1 << 31);
3481 }
3482
3483 static int _mlx4_ib_post_send(struct ib_qp *ibqp, const struct ib_send_wr *wr,
3484                   const struct ib_send_wr **bad_wr, bool drain)
3485 {
3486     struct mlx4_ib_qp *qp = to_mqp(ibqp);
3487     void *wqe;
3488     struct mlx4_wqe_ctrl_seg *ctrl;
3489     struct mlx4_wqe_data_seg *dseg;
3490     unsigned long flags;
3491     int nreq;
3492     int err = 0;
3493     unsigned ind;
3494     int size;
3495     unsigned seglen;
3496     __be32 dummy;
3497     __be32 *lso_wqe;
3498     __be32 lso_hdr_sz;
3499     __be32 blh;
3500     int i;
3501     struct mlx4_ib_dev *mdev = to_mdev(ibqp->device);
3502
3503     if (qp->mlx4_ib_qp_type == MLX4_IB_QPT_GSI) {
3504         struct mlx4_ib_sqp *sqp = qp->sqp;
3505
3506         if (sqp->roce_v2_gsi) {
3507             struct mlx4_ib_ah *ah = to_mah(ud_wr(wr)->ah);
3508             enum ib_gid_type gid_type;
3509             union ib_gid gid;
3510
3511             if (!fill_gid_by_hw_index(mdev, qp->port,
3512                        ah->av.ib.gid_index,
3513                        &gid, &gid_type))
3514                 qp = (gid_type == IB_GID_TYPE_ROCE_UDP_ENCAP) ?
3515                         to_mqp(sqp->roce_v2_gsi) : qp;
3516             else
3517                 pr_err("Failed to get gid at index %d. RoCEv2 will not work properly\n",
3518                        ah->av.ib.gid_index);
3519         }
3520     }
3521
3522     spin_lock_irqsave(&qp->sq.lock, flags);
3523     if (mdev->dev->persist->state & MLX4_DEVICE_STATE_INTERNAL_ERROR &&
3524         !drain) {
3525         err = -EIO;
3526         *bad_wr = wr;
3527         nreq = 0;
3528         goto out;
3529     }
3530
3531     ind = qp->sq_next_wqe;
3532
3533     for (nreq = 0; wr; ++nreq, wr = wr->next) {
3534         lso_wqe = &dummy;
3535         blh = 0;
3536
3537         if (mlx4_wq_overflow(&qp->sq, nreq, qp->ibqp.send_cq)) {
3538             err = -ENOMEM;
3539             *bad_wr = wr;
3540             goto out;
3541         }
3542
3543         if (unlikely(wr->num_sge > qp->sq.max_gs)) {
3544             err = -EINVAL;
3545             *bad_wr = wr;
3546             goto out;
3547         }
3548
3549         ctrl = wqe = get_send_wqe(qp, ind & (qp->sq.wqe_cnt - 1));
3550         qp->sq.wrid[(qp->sq.head + nreq) & (qp->sq.wqe_cnt - 1)] = wr->wr_id;
3551
3552         ctrl->srcrb_flags =
3553             (wr->send_flags & IB_SEND_SIGNALED ?
3554              cpu_to_be32(MLX4_WQE_CTRL_CQ_UPDATE) : 0) |
3555             (wr->send_flags & IB_SEND_SOLICITED ?
3556              cpu_to_be32(MLX4_WQE_CTRL_SOLICITED) : 0) |
3557             ((wr->send_flags & IB_SEND_IP_CSUM) ?
3558              cpu_to_be32(MLX4_WQE_CTRL_IP_CSUM |
3559                      MLX4_WQE_CTRL_TCP_UDP_CSUM) : 0) |
3560             qp->sq_signal_bits;
3561
3562         ctrl->imm = send_ieth(wr);
3563
3564         wqe += sizeof *ctrl;
3565         size = sizeof *ctrl / 16;
3566
3567         switch (qp->mlx4_ib_qp_type) {
3568         case MLX4_IB_QPT_RC:
3569         case MLX4_IB_QPT_UC:
3570             switch (wr->opcode) {
3571             case IB_WR_ATOMIC_CMP_AND_SWP:
3572             case IB_WR_ATOMIC_FETCH_AND_ADD:
3573             case IB_WR_MASKED_ATOMIC_FETCH_AND_ADD:
3574                 set_raddr_seg(wqe, atomic_wr(wr)->remote_addr,
3575                           atomic_wr(wr)->rkey);
3576                 wqe  += sizeof (struct mlx4_wqe_raddr_seg);
3577
3578                 set_atomic_seg(wqe, atomic_wr(wr));
3579                 wqe  += sizeof (struct mlx4_wqe_atomic_seg);
3580
3581                 size += (sizeof (struct mlx4_wqe_raddr_seg) +
3582                      sizeof (struct mlx4_wqe_atomic_seg)) / 16;
3583
3584                 break;
3585
3586             case IB_WR_MASKED_ATOMIC_CMP_AND_SWP:
3587                 set_raddr_seg(wqe, atomic_wr(wr)->remote_addr,
3588                           atomic_wr(wr)->rkey);
3589                 wqe  += sizeof (struct mlx4_wqe_raddr_seg);
3590
3591                 set_masked_atomic_seg(wqe, atomic_wr(wr));
3592                 wqe  += sizeof (struct mlx4_wqe_masked_atomic_seg);
3593
3594                 size += (sizeof (struct mlx4_wqe_raddr_seg) +
3595                      sizeof (struct mlx4_wqe_masked_atomic_seg)) / 16;
3596
3597                 break;
3598
3599             case IB_WR_RDMA_READ:
3600             case IB_WR_RDMA_WRITE:
3601             case IB_WR_RDMA_WRITE_WITH_IMM:
3602                 set_raddr_seg(wqe, rdma_wr(wr)->remote_addr,
3603                           rdma_wr(wr)->rkey);
3604                 wqe  += sizeof (struct mlx4_wqe_raddr_seg);
3605                 size += sizeof (struct mlx4_wqe_raddr_seg) / 16;
3606                 break;
3607
3608             case IB_WR_LOCAL_INV:
3609                 ctrl->srcrb_flags |=
3610                     cpu_to_be32(MLX4_WQE_CTRL_STRONG_ORDER);
3611                 set_local_inv_seg(wqe, wr->ex.invalidate_rkey);
3612                 wqe  += sizeof (struct mlx4_wqe_local_inval_seg);
3613                 size += sizeof (struct mlx4_wqe_local_inval_seg) / 16;
3614                 break;
3615
3616             case IB_WR_REG_MR:
3617                 ctrl->srcrb_flags |=
3618                     cpu_to_be32(MLX4_WQE_CTRL_STRONG_ORDER);
3619                 set_reg_seg(wqe, reg_wr(wr));
3620                 wqe  += sizeof(struct mlx4_wqe_fmr_seg);
3621                 size += sizeof(struct mlx4_wqe_fmr_seg) / 16;
3622                 break;
3623
3624             default:
3625                 /* No extra segments required for sends */
3626                 break;
3627             }
3628             break;
3629
3630         case MLX4_IB_QPT_TUN_SMI_OWNER:
3631             err = build_sriov_qp0_header(qp, ud_wr(wr), ctrl,
3632                              &seglen);
3633             if (unlikely(err)) {
3634                 *bad_wr = wr;
3635                 goto out;
3636             }
3637             wqe  += seglen;
3638             size += seglen / 16;
3639             break;
3640         case MLX4_IB_QPT_TUN_SMI:
3641         case MLX4_IB_QPT_TUN_GSI:
3642             /* this is a UD qp used in MAD responses to slaves. */
3643             set_datagram_seg(wqe, ud_wr(wr));
3644             /* set the forced-loopback bit in the data seg av */
3645             *(__be32 *) wqe |= cpu_to_be32(0x80000000);
3646             wqe  += sizeof (struct mlx4_wqe_datagram_seg);
3647             size += sizeof (struct mlx4_wqe_datagram_seg) / 16;
3648             break;
3649         case MLX4_IB_QPT_UD:
3650             set_datagram_seg(wqe, ud_wr(wr));
3651             wqe  += sizeof (struct mlx4_wqe_datagram_seg);
3652             size += sizeof (struct mlx4_wqe_datagram_seg) / 16;
3653
3654             if (wr->opcode == IB_WR_LSO) {
3655                 err = build_lso_seg(wqe, ud_wr(wr), qp, &seglen,
3656                         &lso_hdr_sz, &blh);
3657                 if (unlikely(err)) {
3658                     *bad_wr = wr;
3659                     goto out;
3660                 }
3661                 lso_wqe = (__be32 *) wqe;
3662                 wqe  += seglen;
3663                 size += seglen / 16;
3664             }
3665             break;
3666
3667         case MLX4_IB_QPT_PROXY_SMI_OWNER:
3668             err = build_sriov_qp0_header(qp, ud_wr(wr), ctrl,
3669                              &seglen);
3670             if (unlikely(err)) {
3671                 *bad_wr = wr;
3672                 goto out;
3673             }
3674             wqe  += seglen;
3675             size += seglen / 16;
3676             /* to start tunnel header on a cache-line boundary */
3677             add_zero_len_inline(wqe);
3678             wqe += 16;
3679             size++;
3680             build_tunnel_header(ud_wr(wr), wqe, &seglen);
3681             wqe  += seglen;
3682             size += seglen / 16;
3683             break;
3684         case MLX4_IB_QPT_PROXY_SMI:
3685         case MLX4_IB_QPT_PROXY_GSI:
3686             /* If we are tunneling special qps, this is a UD qp.
3687              * In this case we first add a UD segment targeting
3688              * the tunnel qp, and then add a header with address
3689              * information */
3690             set_tunnel_datagram_seg(to_mdev(ibqp->device), wqe,
3691                         ud_wr(wr),
3692                         qp->mlx4_ib_qp_type);
3693             wqe  += sizeof (struct mlx4_wqe_datagram_seg);
3694             size += sizeof (struct mlx4_wqe_datagram_seg) / 16;
3695             build_tunnel_header(ud_wr(wr), wqe, &seglen);
3696             wqe  += seglen;
3697             size += seglen / 16;
3698             break;
3699
3700         case MLX4_IB_QPT_SMI:
3701         case MLX4_IB_QPT_GSI:
3702             err = build_mlx_header(qp, ud_wr(wr), ctrl, &seglen);
3703             if (unlikely(err)) {
3704                 *bad_wr = wr;
3705                 goto out;
3706             }
3707             wqe  += seglen;
3708             size += seglen / 16;
3709             break;
3710
3711         default:
3712             break;
3713         }
3714
3715         /*
3716          * Write data segments in reverse order, so as to
3717          * overwrite cacheline stamp last within each
3718          * cacheline.  This avoids issues with WQE
3719          * prefetching.
3720          */
3721
3722         dseg = wqe;
3723         dseg += wr->num_sge - 1;
3724         size += wr->num_sge * (sizeof (struct mlx4_wqe_data_seg) / 16);
3725
3726         /* Add one more inline data segment for ICRC for MLX sends */
3727         if (unlikely(qp->mlx4_ib_qp_type == MLX4_IB_QPT_SMI ||
3728                  qp->mlx4_ib_qp_type == MLX4_IB_QPT_GSI ||
3729                  qp->mlx4_ib_qp_type &
3730                  (MLX4_IB_QPT_PROXY_SMI_OWNER | MLX4_IB_QPT_TUN_SMI_OWNER))) {
3731             set_mlx_icrc_seg(dseg + 1);
3732             size += sizeof (struct mlx4_wqe_data_seg) / 16;
3733         }
3734
3735         for (i = wr->num_sge - 1; i >= 0; --i, --dseg)
3736             set_data_seg(dseg, wr->sg_list + i);
3737
3738         /*
3739          * Possibly overwrite stamping in cacheline with LSO
3740          * segment only after making sure all data segments
3741          * are written.
3742          */
3743         wmb();
3744         *lso_wqe = lso_hdr_sz;
3745
3746         ctrl->qpn_vlan.fence_size = (wr->send_flags & IB_SEND_FENCE ?
3747                          MLX4_WQE_CTRL_FENCE : 0) | size;
3748
3749         /*
3750          * Make sure descriptor is fully written before
3751          * setting ownership bit (because HW can start
3752          * executing as soon as we do).
3753          */
3754         wmb();
3755
3756         if (wr->opcode < 0 || wr->opcode >= ARRAY_SIZE(mlx4_ib_opcode)) {
3757             *bad_wr = wr;
3758             err = -EINVAL;
3759             goto out;
3760         }
3761
3762         ctrl->owner_opcode = mlx4_ib_opcode[wr->opcode] |
3763             (ind & qp->sq.wqe_cnt ? cpu_to_be32(1 << 31) : 0) | blh;
3764
3765         /*
3766          * We can improve latency by not stamping the last
3767          * send queue WQE until after ringing the doorbell, so
3768          * only stamp here if there are still more WQEs to post.
3769          */
3770         if (wr->next)
3771             stamp_send_wqe(qp, ind + qp->sq_spare_wqes);
3772         ind++;
3773     }
3774
3775 out:
3776     if (likely(nreq)) {
3777         qp->sq.head += nreq;
3778
3779         /*
3780          * Make sure that descriptors are written before
3781          * doorbell record.
3782          */
3783         wmb();
3784
3785         writel_relaxed(qp->doorbell_qpn,
3786             to_mdev(ibqp->device)->uar_map + MLX4_SEND_DOORBELL);
3787
3788         stamp_send_wqe(qp, ind + qp->sq_spare_wqes - 1);
3789
3790         qp->sq_next_wqe = ind;
3791     }
3792
3793     spin_unlock_irqrestore(&qp->sq.lock, flags);
3794
3795     return err;
3796 }
3797
3798 int mlx4_ib_post_send(struct ib_qp *ibqp, const struct ib_send_wr *wr,
3799               const struct ib_send_wr **bad_wr)
3800 {
3801     return _mlx4_ib_post_send(ibqp, wr, bad_wr, false);
3802 }
3803
3804 static int _mlx4_ib_post_recv(struct ib_qp *ibqp, const struct ib_recv_wr *wr,
3805                   const struct ib_recv_wr **bad_wr, bool drain)
3806 {
3807     struct mlx4_ib_qp *qp = to_mqp(ibqp);
3808     struct mlx4_wqe_data_seg *scat;
3809     unsigned long flags;
3810     int err = 0;
3811     int nreq;
3812     int ind;
3813     int max_gs;
3814     int i;
3815     struct mlx4_ib_dev *mdev = to_mdev(ibqp->device);
3816
3817     max_gs = qp->rq.max_gs;
3818     spin_lock_irqsave(&qp->rq.lock, flags);
3819
3820     if (mdev->dev->persist->state & MLX4_DEVICE_STATE_INTERNAL_ERROR &&
3821         !drain) {
3822         err = -EIO;
3823         *bad_wr = wr;
3824         nreq = 0;
3825         goto out;
3826     }
3827
3828     ind = qp->rq.head & (qp->rq.wqe_cnt - 1);
3829
3830     for (nreq = 0; wr; ++nreq, wr = wr->next) {
3831         if (mlx4_wq_overflow(&qp->rq, nreq, qp->ibqp.recv_cq)) {
3832             err = -ENOMEM;
3833             *bad_wr = wr;
3834             goto out;
3835         }
3836
3837         if (unlikely(wr->num_sge > qp->rq.max_gs)) {
3838             err = -EINVAL;
3839             *bad_wr = wr;
3840             goto out;
3841         }
3842
3843         scat = get_recv_wqe(qp, ind);
3844
3845         if (qp->mlx4_ib_qp_type & (MLX4_IB_QPT_PROXY_SMI_OWNER |
3846             MLX4_IB_QPT_PROXY_SMI | MLX4_IB_QPT_PROXY_GSI)) {
3847             ib_dma_sync_single_for_device(ibqp->device,
3848                               qp->sqp_proxy_rcv[ind].map,
3849                               sizeof (struct mlx4_ib_proxy_sqp_hdr),
3850                               DMA_FROM_DEVICE);
3851             scat->byte_count =
3852                 cpu_to_be32(sizeof (struct mlx4_ib_proxy_sqp_hdr));
3853             /* use dma lkey from upper layer entry */
3854             scat->lkey = cpu_to_be32(wr->sg_list->lkey);
3855             scat->addr = cpu_to_be64(qp->sqp_proxy_rcv[ind].map);
3856             scat++;
3857             max_gs--;
3858         }
3859
3860         for (i = 0; i < wr->num_sge; ++i)
3861             __set_data_seg(scat + i, wr->sg_list + i);
3862
3863         if (i < max_gs) {
3864             scat[i].byte_count = 0;
3865             scat[i].lkey       = cpu_to_be32(MLX4_INVALID_LKEY);
3866             scat[i].addr       = 0;
3867         }
3868
3869         qp->rq.wrid[ind] = wr->wr_id;
3870
3871         ind = (ind + 1) & (qp->rq.wqe_cnt - 1);
3872     }
3873
3874 out:
3875     if (likely(nreq)) {
3876         qp->rq.head += nreq;
3877
3878         /*
3879          * Make sure that descriptors are written before
3880          * doorbell record.
3881          */
3882         wmb();
3883
3884         *qp->db.db = cpu_to_be32(qp->rq.head & 0xffff);
3885     }
3886
3887     spin_unlock_irqrestore(&qp->rq.lock, flags);
3888
3889     return err;
3890 }
3891
3892 int mlx4_ib_post_recv(struct ib_qp *ibqp, const struct ib_recv_wr *wr,
3893               const struct ib_recv_wr **bad_wr)
3894 {
3895     return _mlx4_ib_post_recv(ibqp, wr, bad_wr, false);
3896 }
3897
3898 static inline enum ib_qp_state to_ib_qp_state(enum mlx4_qp_state mlx4_state)
3899 {
3900     switch (mlx4_state) {
3901     case MLX4_QP_STATE_RST:      return IB_QPS_RESET;
3902     case MLX4_QP_STATE_INIT:     return IB_QPS_INIT;
3903     case MLX4_QP_STATE_RTR:      return IB_QPS_RTR;
3904     case MLX4_QP_STATE_RTS:      return IB_QPS_RTS;
3905     case MLX4_QP_STATE_SQ_DRAINING:
3906     case MLX4_QP_STATE_SQD:      return IB_QPS_SQD;
3907     case MLX4_QP_STATE_SQER:     return IB_QPS_SQE;
3908     case MLX4_QP_STATE_ERR:      return IB_QPS_ERR;
3909     default:             return -1;
3910     }
3911 }
3912
3913 static inline enum ib_mig_state to_ib_mig_state(int mlx4_mig_state)
3914 {
3915     switch (mlx4_mig_state) {
3916     case MLX4_QP_PM_ARMED:      return IB_MIG_ARMED;
3917     case MLX4_QP_PM_REARM:      return IB_MIG_REARM;
3918     case MLX4_QP_PM_MIGRATED:   return IB_MIG_MIGRATED;
3919     default: return -1;
3920     }
3921 }
3922
3923 static int to_ib_qp_access_flags(int mlx4_flags)
3924 {
3925     int ib_flags = 0;
3926
3927     if (mlx4_flags & MLX4_QP_BIT_RRE)
3928         ib_flags |= IB_ACCESS_REMOTE_READ;
3929     if (mlx4_flags & MLX4_QP_BIT_RWE)
3930         ib_flags |= IB_ACCESS_REMOTE_WRITE;
3931     if (mlx4_flags & MLX4_QP_BIT_RAE)
3932         ib_flags |= IB_ACCESS_REMOTE_ATOMIC;
3933
3934     return ib_flags;
3935 }
3936
3937 static void to_rdma_ah_attr(struct mlx4_ib_dev *ibdev,
3938                 struct rdma_ah_attr *ah_attr,
3939                 struct mlx4_qp_path *path)
3940 {
3941     struct mlx4_dev *dev = ibdev->dev;
3942     u8 port_num = path->sched_queue & 0x40 ? 2 : 1;
3943
3944     memset(ah_attr, 0, sizeof(*ah_attr));
3945     if (port_num == 0 || port_num > dev->caps.num_ports)
3946         return;
3947     ah_attr->type = rdma_ah_find_type(&ibdev->ib_dev, port_num);
3948
3949     if (ah_attr->type == RDMA_AH_ATTR_TYPE_ROCE)
3950         rdma_ah_set_sl(ah_attr, ((path->sched_queue >> 3) & 0x7) |
3951                    ((path->sched_queue & 4) << 1));
3952     else
3953         rdma_ah_set_sl(ah_attr, (path->sched_queue >> 2) & 0xf);
3954     rdma_ah_set_port_num(ah_attr, port_num);
3955
3956     rdma_ah_set_dlid(ah_attr, be16_to_cpu(path->rlid));
3957     rdma_ah_set_path_bits(ah_attr, path->grh_mylmc & 0x7f);
3958     rdma_ah_set_static_rate(ah_attr,
3959                 path->static_rate ? path->static_rate - 5 : 0);
3960     if (path->grh_mylmc & (1 << 7)) {
3961         rdma_ah_set_grh(ah_attr, NULL,
3962                 be32_to_cpu(path->tclass_flowlabel) & 0xfffff,
3963                 path->mgid_index,
3964                 path->hop_limit,
3965                 (be32_to_cpu(path->tclass_flowlabel)
3966                  >> 20) & 0xff);
3967         rdma_ah_set_dgid_raw(ah_attr, path->rgid);
3968     }
3969 }
3970
3971 int mlx4_ib_query_qp(struct ib_qp *ibqp, struct ib_qp_attr *qp_attr, int qp_attr_mask,
3972              struct ib_qp_init_attr *qp_init_attr)
3973 {
3974     struct mlx4_ib_dev *dev = to_mdev(ibqp->device);
3975     struct mlx4_ib_qp *qp = to_mqp(ibqp);
3976     struct mlx4_qp_context context;
3977     int mlx4_state;
3978     int err = 0;
3979
3980     if (ibqp->rwq_ind_tbl)
3981         return -EOPNOTSUPP;
3982
3983     mutex_lock(&qp->mutex);
3984
3985     if (qp->state == IB_QPS_RESET) {
3986         qp_attr->qp_state = IB_QPS_RESET;
3987         goto done;
3988     }
3989
3990     err = mlx4_qp_query(dev->dev, &qp->mqp, &context);
3991     if (err) {
3992         err = -EINVAL;
3993         goto out;
3994     }
3995
3996     mlx4_state = be32_to_cpu(context.flags) >> 28;
3997
3998     qp->state            = to_ib_qp_state(mlx4_state);
3999     qp_attr->qp_state        = qp->state;
4000     qp_attr->path_mtu        = context.mtu_msgmax >> 5;
4001     qp_attr->path_mig_state      =
4002         to_ib_mig_state((be32_to_cpu(context.flags) >> 11) & 0x3);
4003     qp_attr->qkey            = be32_to_cpu(context.qkey);
4004     qp_attr->rq_psn          = be32_to_cpu(context.rnr_nextrecvpsn) & 0xffffff;
4005     qp_attr->sq_psn          = be32_to_cpu(context.next_send_psn) & 0xffffff;
4006     qp_attr->dest_qp_num         = be32_to_cpu(context.remote_qpn) & 0xffffff;
4007     qp_attr->qp_access_flags     =
4008         to_ib_qp_access_flags(be32_to_cpu(context.params2));
4009
4010     if (qp->ibqp.qp_type == IB_QPT_RC || qp->ibqp.qp_type == IB_QPT_UC ||
4011         qp->ibqp.qp_type == IB_QPT_XRC_INI ||
4012         qp->ibqp.qp_type == IB_QPT_XRC_TGT) {
4013         to_rdma_ah_attr(dev, &qp_attr->ah_attr, &context.pri_path);
4014         to_rdma_ah_attr(dev, &qp_attr->alt_ah_attr, &context.alt_path);
4015         qp_attr->alt_pkey_index = context.alt_path.pkey_index & 0x7f;
4016         qp_attr->alt_port_num   =
4017             rdma_ah_get_port_num(&qp_attr->alt_ah_attr);
4018     }
4019
4020     qp_attr->pkey_index = context.pri_path.pkey_index & 0x7f;
4021     if (qp_attr->qp_state == IB_QPS_INIT)
4022         qp_attr->port_num = qp->port;
4023     else
4024         qp_attr->port_num = context.pri_path.sched_queue & 0x40 ? 2 : 1;
4025
4026     /* qp_attr->en_sqd_async_notify is only applicable in modify qp */
4027     qp_attr->sq_draining = mlx4_state == MLX4_QP_STATE_SQ_DRAINING;
4028
4029     qp_attr->max_rd_atomic = 1 << ((be32_to_cpu(context.params1) >> 21) & 0x7);
4030
4031     qp_attr->max_dest_rd_atomic =
4032         1 << ((be32_to_cpu(context.params2) >> 21) & 0x7);
4033     qp_attr->min_rnr_timer      =
4034         (be32_to_cpu(context.rnr_nextrecvpsn) >> 24) & 0x1f;
4035     qp_attr->timeout        = context.pri_path.ackto >> 3;
4036     qp_attr->retry_cnt      = (be32_to_cpu(context.params1) >> 16) & 0x7;
4037     qp_attr->rnr_retry      = (be32_to_cpu(context.params1) >> 13) & 0x7;
4038     qp_attr->alt_timeout        = context.alt_path.ackto >> 3;
4039
4040 done:
4041     qp_attr->cur_qp_state        = qp_attr->qp_state;
4042     qp_attr->cap.max_recv_wr     = qp->rq.wqe_cnt;
4043     qp_attr->cap.max_recv_sge    = qp->rq.max_gs;
4044
4045     if (!ibqp->uobject) {
4046         qp_attr->cap.max_send_wr  = qp->sq.wqe_cnt;
4047         qp_attr->cap.max_send_sge = qp->sq.max_gs;
4048     } else {
4049         qp_attr->cap.max_send_wr  = 0;
4050         qp_attr->cap.max_send_sge = 0;
4051     }
4052
4053     /*
4054      * We don't support inline sends for kernel QPs (yet), and we
4055      * don't know what userspace's value should be.
4056      */
4057     qp_attr->cap.max_inline_data = 0;
4058
4059     qp_init_attr->cap        = qp_attr->cap;
4060
4061     qp_init_attr->create_flags = 0;
4062     if (qp->flags & MLX4_IB_QP_BLOCK_MULTICAST_LOOPBACK)
4063         qp_init_attr->create_flags |= IB_QP_CREATE_BLOCK_MULTICAST_LOOPBACK;
4064
4065     if (qp->flags & MLX4_IB_QP_LSO)
4066         qp_init_attr->create_flags |= IB_QP_CREATE_IPOIB_UD_LSO;
4067
4068     if (qp->flags & MLX4_IB_QP_NETIF)
4069         qp_init_attr->create_flags |= IB_QP_CREATE_NETIF_QP;
4070
4071     qp_init_attr->sq_sig_type =
4072         qp->sq_signal_bits == cpu_to_be32(MLX4_WQE_CTRL_CQ_UPDATE) ?
4073         IB_SIGNAL_ALL_WR : IB_SIGNAL_REQ_WR;
4074
4075 out:
4076     mutex_unlock(&qp->mutex);
4077     return err;
4078 }
4079
4080 struct ib_wq *mlx4_ib_create_wq(struct ib_pd *pd,
4081                 struct ib_wq_init_attr *init_attr,
4082                 struct ib_udata *udata)
4083 {
4084     struct mlx4_dev *dev = to_mdev(pd->device)->dev;
4085     struct ib_qp_init_attr ib_qp_init_attr = {};
4086     struct mlx4_ib_qp *qp;
4087     struct mlx4_ib_create_wq ucmd;
4088     int err, required_cmd_sz;
4089
4090     if (!udata)
4091         return ERR_PTR(-EINVAL);
4092
4093     required_cmd_sz = offsetof(typeof(ucmd), comp_mask) +
4094               sizeof(ucmd.comp_mask);
4095     if (udata->inlen < required_cmd_sz) {
4096         pr_debug("invalid inlen\n");
4097         return ERR_PTR(-EINVAL);
4098     }
4099
4100     if (udata->inlen > sizeof(ucmd) &&
4101         !ib_is_udata_cleared(udata, sizeof(ucmd),
4102                  udata->inlen - sizeof(ucmd))) {
4103         pr_debug("inlen is not supported\n");
4104         return ERR_PTR(-EOPNOTSUPP);
4105     }
4106
4107     if (udata->outlen)
4108         return ERR_PTR(-EOPNOTSUPP);
4109
4110     if (init_attr->wq_type != IB_WQT_RQ) {
4111         pr_debug("unsupported wq type %d\n", init_attr->wq_type);
4112         return ERR_PTR(-EOPNOTSUPP);
4113     }
4114
4115     if (init_attr->create_flags & ~IB_WQ_FLAGS_SCATTER_FCS ||
4116         !(dev->caps.flags & MLX4_DEV_CAP_FLAG_FCS_KEEP)) {
4117         pr_debug("unsupported create_flags %u\n",
4118              init_attr->create_flags);
4119         return ERR_PTR(-EOPNOTSUPP);
4120     }
4121
4122     qp = kzalloc(sizeof(*qp), GFP_KERNEL);
4123     if (!qp)
4124         return ERR_PTR(-ENOMEM);
4125
4126     mutex_init(&qp->mutex);
4127     qp->pri.vid = 0xFFFF;
4128     qp->alt.vid = 0xFFFF;
4129
4130     ib_qp_init_attr.qp_context = init_attr->wq_context;
4131     ib_qp_init_attr.qp_type = IB_QPT_RAW_PACKET;
4132     ib_qp_init_attr.cap.max_recv_wr = init_attr->max_wr;
4133     ib_qp_init_attr.cap.max_recv_sge = init_attr->max_sge;
4134     ib_qp_init_attr.recv_cq = init_attr->cq;
4135     ib_qp_init_attr.send_cq = ib_qp_init_attr.recv_cq; /* Dummy CQ */
4136
4137     if (init_attr->create_flags & IB_WQ_FLAGS_SCATTER_FCS)
4138         ib_qp_init_attr.create_flags |= IB_QP_CREATE_SCATTER_FCS;
4139
4140     err = create_rq(pd, &ib_qp_init_attr, udata, qp);
4141     if (err) {
4142         kfree(qp);
4143         return ERR_PTR(err);
4144     }
4145
4146     qp->ibwq.event_handler = init_attr->event_handler;
4147     qp->ibwq.wq_num = qp->mqp.qpn;
4148     qp->ibwq.state = IB_WQS_RESET;
4149
4150     return &qp->ibwq;
4151 }
4152
4153 static int ib_wq2qp_state(enum ib_wq_state state)
4154 {
4155     switch (state) {
4156     case IB_WQS_RESET:
4157         return IB_QPS_RESET;
4158     case IB_WQS_RDY:
4159         return IB_QPS_RTR;
4160     default:
4161         return IB_QPS_ERR;
4162     }
4163 }
4164
4165 static int _mlx4_ib_modify_wq(struct ib_wq *ibwq, enum ib_wq_state new_state,
4166                   struct ib_udata *udata)
4167 {
4168     struct mlx4_ib_qp *qp = to_mqp((struct ib_qp *)ibwq);
4169     enum ib_qp_state qp_cur_state;
4170     enum ib_qp_state qp_new_state;
4171     int attr_mask;
4172     int err;
4173
4174     /* ib_qp.state represents the WQ HW state while ib_wq.state represents
4175      * the WQ logic state.
4176      */
4177     qp_cur_state = qp->state;
4178     qp_new_state = ib_wq2qp_state(new_state);
4179
4180     if (ib_wq2qp_state(new_state) == qp_cur_state)
4181         return 0;
4182
4183     if (new_state == IB_WQS_RDY) {
4184         struct ib_qp_attr attr = {};
4185
4186         attr.port_num = qp->port;
4187         attr_mask = IB_QP_PORT;
4188
4189         err = __mlx4_ib_modify_qp(ibwq, MLX4_IB_RWQ_SRC, &attr,
4190                       attr_mask, IB_QPS_RESET, IB_QPS_INIT,
4191                       udata);
4192         if (err) {
4193             pr_debug("WQN=0x%06x failed to apply RST->INIT on the HW QP\n",
4194                  ibwq->wq_num);
4195             return err;
4196         }
4197
4198         qp_cur_state = IB_QPS_INIT;
4199     }
4200
4201     attr_mask = 0;
4202     err = __mlx4_ib_modify_qp(ibwq, MLX4_IB_RWQ_SRC, NULL, attr_mask,
4203                   qp_cur_state,  qp_new_state, udata);
4204
4205     if (err && (qp_cur_state == IB_QPS_INIT)) {
4206         qp_new_state = IB_QPS_RESET;
4207         if (__mlx4_ib_modify_qp(ibwq, MLX4_IB_RWQ_SRC, NULL,
4208                     attr_mask, IB_QPS_INIT, IB_QPS_RESET,
4209                     udata)) {
4210             pr_warn("WQN=0x%06x failed with reverting HW's resources failure\n",
4211                 ibwq->wq_num);
4212             qp_new_state = IB_QPS_INIT;
4213         }
4214     }
4215
4216     qp->state = qp_new_state;
4217
4218     return err;
4219 }
4220
4221 int mlx4_ib_modify_wq(struct ib_wq *ibwq, struct ib_wq_attr *wq_attr,
4222               u32 wq_attr_mask, struct ib_udata *udata)
4223 {
4224     struct mlx4_ib_qp *qp = to_mqp((struct ib_qp *)ibwq);
4225     struct mlx4_ib_modify_wq ucmd = {};
4226     size_t required_cmd_sz;
4227     enum ib_wq_state cur_state, new_state;
4228     int err = 0;
4229
4230     required_cmd_sz = offsetof(typeof(ucmd), reserved) +
4231                    sizeof(ucmd.reserved);
4232     if (udata->inlen < required_cmd_sz)
4233         return -EINVAL;
4234
4235     if (udata->inlen > sizeof(ucmd) &&
4236         !ib_is_udata_cleared(udata, sizeof(ucmd),
4237                  udata->inlen - sizeof(ucmd)))
4238         return -EOPNOTSUPP;
4239
4240     if (ib_copy_from_udata(&ucmd, udata, min(sizeof(ucmd), udata->inlen)))
4241         return -EFAULT;
4242
4243     if (ucmd.comp_mask || ucmd.reserved)
4244         return -EOPNOTSUPP;
4245
4246     if (wq_attr_mask & IB_WQ_FLAGS)
4247         return -EOPNOTSUPP;
4248
4249     cur_state = wq_attr->curr_wq_state;
4250     new_state = wq_attr->wq_state;
4251
4252     if ((new_state == IB_WQS_RDY) && (cur_state == IB_WQS_ERR))
4253         return -EINVAL;
4254
4255     if ((new_state == IB_WQS_ERR) && (cur_state == IB_WQS_RESET))
4256         return -EINVAL;
4257
4258     /* Need to protect against the parent RSS which also may modify WQ
4259      * state.
4260      */
4261     mutex_lock(&qp->mutex);
4262
4263     /* Can update HW state only if a RSS QP has already associated to this
4264      * WQ, so we can apply its port on the WQ.
4265      */
4266     if (qp->rss_usecnt)
4267         err = _mlx4_ib_modify_wq(ibwq, new_state, udata);
4268
4269     if (!err)
4270         ibwq->state = new_state;
4271
4272     mutex_unlock(&qp->mutex);
4273
4274     return err;
4275 }
4276
4277 int mlx4_ib_destroy_wq(struct ib_wq *ibwq, struct ib_udata *udata)
4278 {
4279     struct mlx4_ib_dev *dev = to_mdev(ibwq->device);
4280     struct mlx4_ib_qp *qp = to_mqp((struct ib_qp *)ibwq);
4281
4282     if (qp->counter_index)
4283         mlx4_ib_free_qp_counter(dev, qp);
4284
4285     destroy_qp_common(dev, qp, MLX4_IB_RWQ_SRC, udata);
4286
4287     kfree(qp);
4288     return 0;
4289 }
4290
4291 int mlx4_ib_create_rwq_ind_table(struct ib_rwq_ind_table *rwq_ind_table,
4292                  struct ib_rwq_ind_table_init_attr *init_attr,
4293                  struct ib_udata *udata)
4294 {
4295     struct mlx4_ib_create_rwq_ind_tbl_resp resp = {};
4296     unsigned int ind_tbl_size = 1 << init_attr->log_ind_tbl_size;
4297     struct ib_device *device = rwq_ind_table->device;
4298     unsigned int base_wqn;
4299     size_t min_resp_len;
4300     int i, err = 0;
4301
4302     if (udata->inlen > 0 &&
4303         !ib_is_udata_cleared(udata, 0,
4304                  udata->inlen))
4305         return -EOPNOTSUPP;
4306
4307     min_resp_len = offsetof(typeof(resp), reserved) + sizeof(resp.reserved);
4308     if (udata->outlen && udata->outlen < min_resp_len)
4309         return -EINVAL;
4310
4311     if (ind_tbl_size >
4312         device->attrs.rss_caps.max_rwq_indirection_table_size) {
4313         pr_debug("log_ind_tbl_size = %d is bigger than supported = %d\n",
4314              ind_tbl_size,
4315              device->attrs.rss_caps.max_rwq_indirection_table_size);
4316         return -EINVAL;
4317     }
4318
4319     base_wqn = init_attr->ind_tbl[0]->wq_num;
4320
4321     if (base_wqn % ind_tbl_size) {
4322         pr_debug("WQN=0x%x isn't aligned with indirection table size\n",
4323              base_wqn);
4324         return -EINVAL;
4325     }
4326
4327     for (i = 1; i < ind_tbl_size; i++) {
4328         if (++base_wqn != init_attr->ind_tbl[i]->wq_num) {
4329             pr_debug("indirection table's WQNs aren't consecutive\n");
4330             return -EINVAL;
4331         }
4332     }
4333
4334     if (udata->outlen) {
4335         resp.response_length = offsetof(typeof(resp), response_length) +
4336                     sizeof(resp.response_length);
4337         err = ib_copy_to_udata(udata, &resp, resp.response_length);
4338     }
4339
4340     return err;
4341 }
4342
4343 struct mlx4_ib_drain_cqe {
4344     struct ib_cqe cqe;
4345     struct completion done;
4346 };
4347
4348 static void mlx4_ib_drain_qp_done(struct ib_cq *cq, struct ib_wc *wc)
4349 {
4350     struct mlx4_ib_drain_cqe *cqe = container_of(wc->wr_cqe,
4351                              struct mlx4_ib_drain_cqe,
4352                              cqe);
4353
4354     complete(&cqe->done);
4355 }
4356
4357 /* This function returns only once the drained WR was completed */
4358 static void handle_drain_completion(struct ib_cq *cq,
4359                     struct mlx4_ib_drain_cqe *sdrain,
4360                     struct mlx4_ib_dev *dev)
4361 {
4362     struct mlx4_dev *mdev = dev->dev;
4363
4364     if (cq->poll_ctx == IB_POLL_DIRECT) {
4365         while (wait_for_completion_timeout(&sdrain->done, HZ / 10) <= 0)
4366             ib_process_cq_direct(cq, -1);
4367         return;
4368     }
4369
4370     if (mdev->persist->state == MLX4_DEVICE_STATE_INTERNAL_ERROR) {
4371         struct mlx4_ib_cq *mcq = to_mcq(cq);
4372         bool triggered = false;
4373         unsigned long flags;
4374
4375         spin_lock_irqsave(&dev->reset_flow_resource_lock, flags);
4376         /* Make sure that the CQ handler won't run if wasn't run yet */
4377         if (!mcq->mcq.reset_notify_added)
4378             mcq->mcq.reset_notify_added = 1;
4379         else
4380             triggered = true;
4381         spin_unlock_irqrestore(&dev->reset_flow_resource_lock, flags);
4382
4383         if (triggered) {
4384             /* Wait for any scheduled/running task to be ended */
4385             switch (cq->poll_ctx) {
4386             case IB_POLL_SOFTIRQ:
4387                 irq_poll_disable(&cq->iop);
4388                 irq_poll_enable(&cq->iop);
4389                 break;
4390             case IB_POLL_WORKQUEUE:
4391                 cancel_work_sync(&cq->work);
4392                 break;
4393             default:
4394                 WARN_ON_ONCE(1);
4395             }
4396         }
4397
4398         /* Run the CQ handler - this makes sure that the drain WR will
4399          * be processed if wasn't processed yet.
4400          */
4401         mcq->mcq.comp(&mcq->mcq);
4402     }
4403
4404     wait_for_completion(&sdrain->done);
4405 }
4406
4407 void mlx4_ib_drain_sq(struct ib_qp *qp)
4408 {
4409     struct ib_cq *cq = qp->send_cq;
4410     struct ib_qp_attr attr = { .qp_state = IB_QPS_ERR };
4411     struct mlx4_ib_drain_cqe sdrain;
4412     const struct ib_send_wr *bad_swr;
4413     struct ib_rdma_wr swr = {
4414         .wr = {
4415             .next = NULL,
4416             { .wr_cqe   = &sdrain.cqe, },
4417             .opcode = IB_WR_RDMA_WRITE,
4418         },
4419     };
4420     int ret;
4421     struct mlx4_ib_dev *dev = to_mdev(qp->device);
4422     struct mlx4_dev *mdev = dev->dev;
4423
4424     ret = ib_modify_qp(qp, &attr, IB_QP_STATE);
4425     if (ret && mdev->persist->state != MLX4_DEVICE_STATE_INTERNAL_ERROR) {
4426         WARN_ONCE(ret, "failed to drain send queue: %d\n", ret);
4427         return;
4428     }
4429
4430     sdrain.cqe.done = mlx4_ib_drain_qp_done;
4431     init_completion(&sdrain.done);
4432
4433     ret = _mlx4_ib_post_send(qp, &swr.wr, &bad_swr, true);
4434     if (ret) {
4435         WARN_ONCE(ret, "failed to drain send queue: %d\n", ret);
4436         return;
4437     }
4438
4439     handle_drain_completion(cq, &sdrain, dev);
4440 }
4441
4442 void mlx4_ib_drain_rq(struct ib_qp *qp)
4443 {
4444     struct ib_cq *cq = qp->recv_cq;
4445     struct ib_qp_attr attr = { .qp_state = IB_QPS_ERR };
4446     struct mlx4_ib_drain_cqe rdrain;
4447     struct ib_recv_wr rwr = {};
4448     const struct ib_recv_wr *bad_rwr;
4449     int ret;
4450     struct mlx4_ib_dev *dev = to_mdev(qp->device);
4451     struct mlx4_dev *mdev = dev->dev;
4452
4453     ret = ib_modify_qp(qp, &attr, IB_QP_STATE);
4454     if (ret && mdev->persist->state != MLX4_DEVICE_STATE_INTERNAL_ERROR) {
4455         WARN_ONCE(ret, "failed to drain recv queue: %d\n", ret);
4456         return;
4457     }
4458
4459     rwr.wr_cqe = &rdrain.cqe;
4460     rdrain.cqe.done = mlx4_ib_drain_qp_done;
4461     init_completion(&rdrain.done);
4462
4463     ret = _mlx4_ib_post_recv(qp, &rwr, &bad_rwr, true);
4464     if (ret) {
4465         WARN_ONCE(ret, "failed to drain recv queue: %d\n", ret);
4466         return;
4467     }
4468
4469     handle_drain_completion(cq, &rdrain, dev);
4470 }