hw/mlx5/main.c

0001 // SPDX-License-Identifier: GPL-2.0 OR Linux-OpenIB
0002 /*
0003  * Copyright (c) 2013-2020, Mellanox Technologies inc. All rights reserved.
0004  * Copyright (c) 2020, Intel Corporation. All rights reserved.
0005  */
0006
0007 #include <linux/debugfs.h>
0008 #include <linux/highmem.h>
0009 #include <linux/module.h>
0010 #include <linux/init.h>
0011 #include <linux/errno.h>
0012 #include <linux/pci.h>
0013 #include <linux/dma-mapping.h>
0014 #include <linux/slab.h>
0015 #include <linux/bitmap.h>
0016 #include <linux/sched.h>
0017 #include <linux/sched/mm.h>
0018 #include <linux/sched/task.h>
0019 #include <linux/delay.h>
0020 #include <rdma/ib_user_verbs.h>
0021 #include <rdma/ib_addr.h>
0022 #include <rdma/ib_cache.h>
0023 #include <linux/mlx5/port.h>
0024 #include <linux/mlx5/vport.h>
0025 #include <linux/mlx5/fs.h>
0026 #include <linux/mlx5/eswitch.h>
0027 #include <linux/list.h>
0028 #include <rdma/ib_smi.h>
0029 #include <rdma/ib_umem.h>
0030 #include <rdma/lag.h>
0031 #include <linux/in.h>
0032 #include <linux/etherdevice.h>
0033 #include "mlx5_ib.h"
0034 #include "ib_rep.h"
0035 #include "cmd.h"
0036 #include "devx.h"
0037 #include "dm.h"
0038 #include "fs.h"
0039 #include "srq.h"
0040 #include "qp.h"
0041 #include "wr.h"
0042 #include "restrack.h"
0043 #include "counters.h"
0044 #include "umr.h"
0045 #include <rdma/uverbs_std_types.h>
0046 #include <rdma/uverbs_ioctl.h>
0047 #include <rdma/mlx5_user_ioctl_verbs.h>
0048 #include <rdma/mlx5_user_ioctl_cmds.h>
0049 #include <rdma/ib_umem_odp.h>
0050
0051 #define UVERBS_MODULE_NAME mlx5_ib
0052 #include <rdma/uverbs_named_ioctl.h>
0053
0054 MODULE_AUTHOR("Eli Cohen <eli@mellanox.com>");
0055 MODULE_DESCRIPTION("Mellanox 5th generation network adapters (ConnectX series) IB driver");
0056 MODULE_LICENSE("Dual BSD/GPL");
0057
0058 struct mlx5_ib_event_work {
0059     struct work_struct  work;
0060     union {
0061         struct mlx5_ib_dev        *dev;
0062         struct mlx5_ib_multiport_info *mpi;
0063     };
0064     bool            is_slave;
0065     unsigned int        event;
0066     void            *param;
0067 };
0068
0069 enum {
0070     MLX5_ATOMIC_SIZE_QP_8BYTES = 1 << 3,
0071 };
0072
0073 static struct workqueue_struct *mlx5_ib_event_wq;
0074 static LIST_HEAD(mlx5_ib_unaffiliated_port_list);
0075 static LIST_HEAD(mlx5_ib_dev_list);
0076 /*
0077  * This mutex should be held when accessing either of the above lists
0078  */
0079 static DEFINE_MUTEX(mlx5_ib_multiport_mutex);
0080
0081 struct mlx5_ib_dev *mlx5_ib_get_ibdev_from_mpi(struct mlx5_ib_multiport_info *mpi)
0082 {
0083     struct mlx5_ib_dev *dev;
0084
0085     mutex_lock(&mlx5_ib_multiport_mutex);
0086     dev = mpi->ibdev;
0087     mutex_unlock(&mlx5_ib_multiport_mutex);
0088     return dev;
0089 }
0090
0091 static enum rdma_link_layer
0092 mlx5_port_type_cap_to_rdma_ll(int port_type_cap)
0093 {
0094     switch (port_type_cap) {
0095     case MLX5_CAP_PORT_TYPE_IB:
0096         return IB_LINK_LAYER_INFINIBAND;
0097     case MLX5_CAP_PORT_TYPE_ETH:
0098         return IB_LINK_LAYER_ETHERNET;
0099     default:
0100         return IB_LINK_LAYER_UNSPECIFIED;
0101     }
0102 }
0103
0104 static enum rdma_link_layer
0105 mlx5_ib_port_link_layer(struct ib_device *device, u32 port_num)
0106 {
0107     struct mlx5_ib_dev *dev = to_mdev(device);
0108     int port_type_cap = MLX5_CAP_GEN(dev->mdev, port_type);
0109
0110     return mlx5_port_type_cap_to_rdma_ll(port_type_cap);
0111 }
0112
0113 static int get_port_state(struct ib_device *ibdev,
0114               u32 port_num,
0115               enum ib_port_state *state)
0116 {
0117     struct ib_port_attr attr;
0118     int ret;
0119
0120     memset(&attr, 0, sizeof(attr));
0121     ret = ibdev->ops.query_port(ibdev, port_num, &attr);
0122     if (!ret)
0123         *state = attr.state;
0124     return ret;
0125 }
0126
0127 static struct mlx5_roce *mlx5_get_rep_roce(struct mlx5_ib_dev *dev,
0128                        struct net_device *ndev,
0129                        struct net_device *upper,
0130                        u32 *port_num)
0131 {
0132     struct net_device *rep_ndev;
0133     struct mlx5_ib_port *port;
0134     int i;
0135
0136     for (i = 0; i < dev->num_ports; i++) {
0137         port  = &dev->port[i];
0138         if (!port->rep)
0139             continue;
0140
0141         if (upper == ndev && port->rep->vport == MLX5_VPORT_UPLINK) {
0142             *port_num = i + 1;
0143             return &port->roce;
0144         }
0145
0146         if (upper && port->rep->vport == MLX5_VPORT_UPLINK)
0147             continue;
0148
0149         read_lock(&port->roce.netdev_lock);
0150         rep_ndev = mlx5_ib_get_rep_netdev(port->rep->esw,
0151                           port->rep->vport);
0152         if (rep_ndev == ndev) {
0153             read_unlock(&port->roce.netdev_lock);
0154             *port_num = i + 1;
0155             return &port->roce;
0156         }
0157         read_unlock(&port->roce.netdev_lock);
0158     }
0159
0160     return NULL;
0161 }
0162
0163 static int mlx5_netdev_event(struct notifier_block *this,
0164                  unsigned long event, void *ptr)
0165 {
0166     struct mlx5_roce *roce = container_of(this, struct mlx5_roce, nb);
0167     struct net_device *ndev = netdev_notifier_info_to_dev(ptr);
0168     u32 port_num = roce->native_port_num;
0169     struct mlx5_core_dev *mdev;
0170     struct mlx5_ib_dev *ibdev;
0171
0172     ibdev = roce->dev;
0173     mdev = mlx5_ib_get_native_port_mdev(ibdev, port_num, NULL);
0174     if (!mdev)
0175         return NOTIFY_DONE;
0176
0177     switch (event) {
0178     case NETDEV_REGISTER:
0179         /* Should already be registered during the load */
0180         if (ibdev->is_rep)
0181             break;
0182         write_lock(&roce->netdev_lock);
0183         if (ndev->dev.parent == mdev->device)
0184             roce->netdev = ndev;
0185         write_unlock(&roce->netdev_lock);
0186         break;
0187
0188     case NETDEV_UNREGISTER:
0189         /* In case of reps, ib device goes away before the netdevs */
0190         write_lock(&roce->netdev_lock);
0191         if (roce->netdev == ndev)
0192             roce->netdev = NULL;
0193         write_unlock(&roce->netdev_lock);
0194         break;
0195
0196     case NETDEV_CHANGE:
0197     case NETDEV_UP:
0198     case NETDEV_DOWN: {
0199         struct net_device *lag_ndev = mlx5_lag_get_roce_netdev(mdev);
0200         struct net_device *upper = NULL;
0201
0202         if (lag_ndev) {
0203             upper = netdev_master_upper_dev_get(lag_ndev);
0204             dev_put(lag_ndev);
0205         }
0206
0207         if (ibdev->is_rep)
0208             roce = mlx5_get_rep_roce(ibdev, ndev, upper, &port_num);
0209         if (!roce)
0210             return NOTIFY_DONE;
0211         if ((upper == ndev ||
0212              ((!upper || ibdev->is_rep) && ndev == roce->netdev)) &&
0213             ibdev->ib_active) {
0214             struct ib_event ibev = { };
0215             enum ib_port_state port_state;
0216
0217             if (get_port_state(&ibdev->ib_dev, port_num,
0218                        &port_state))
0219                 goto done;
0220
0221             if (roce->last_port_state == port_state)
0222                 goto done;
0223
0224             roce->last_port_state = port_state;
0225             ibev.device = &ibdev->ib_dev;
0226             if (port_state == IB_PORT_DOWN)
0227                 ibev.event = IB_EVENT_PORT_ERR;
0228             else if (port_state == IB_PORT_ACTIVE)
0229                 ibev.event = IB_EVENT_PORT_ACTIVE;
0230             else
0231                 goto done;
0232
0233             ibev.element.port_num = port_num;
0234             ib_dispatch_event(&ibev);
0235         }
0236         break;
0237     }
0238
0239     default:
0240         break;
0241     }
0242 done:
0243     mlx5_ib_put_native_port_mdev(ibdev, port_num);
0244     return NOTIFY_DONE;
0245 }
0246
0247 static struct net_device *mlx5_ib_get_netdev(struct ib_device *device,
0248                          u32 port_num)
0249 {
0250     struct mlx5_ib_dev *ibdev = to_mdev(device);
0251     struct net_device *ndev;
0252     struct mlx5_core_dev *mdev;
0253
0254     mdev = mlx5_ib_get_native_port_mdev(ibdev, port_num, NULL);
0255     if (!mdev)
0256         return NULL;
0257
0258     ndev = mlx5_lag_get_roce_netdev(mdev);
0259     if (ndev)
0260         goto out;
0261
0262     /* Ensure ndev does not disappear before we invoke dev_hold()
0263      */
0264     read_lock(&ibdev->port[port_num - 1].roce.netdev_lock);
0265     ndev = ibdev->port[port_num - 1].roce.netdev;
0266     if (ndev)
0267         dev_hold(ndev);
0268     read_unlock(&ibdev->port[port_num - 1].roce.netdev_lock);
0269
0270 out:
0271     mlx5_ib_put_native_port_mdev(ibdev, port_num);
0272     return ndev;
0273 }
0274
0275 struct mlx5_core_dev *mlx5_ib_get_native_port_mdev(struct mlx5_ib_dev *ibdev,
0276                            u32 ib_port_num,
0277                            u32 *native_port_num)
0278 {
0279     enum rdma_link_layer ll = mlx5_ib_port_link_layer(&ibdev->ib_dev,
0280                               ib_port_num);
0281     struct mlx5_core_dev *mdev = NULL;
0282     struct mlx5_ib_multiport_info *mpi;
0283     struct mlx5_ib_port *port;
0284
0285     if (!mlx5_core_mp_enabled(ibdev->mdev) ||
0286         ll != IB_LINK_LAYER_ETHERNET) {
0287         if (native_port_num)
0288             *native_port_num = ib_port_num;
0289         return ibdev->mdev;
0290     }
0291
0292     if (native_port_num)
0293         *native_port_num = 1;
0294
0295     port = &ibdev->port[ib_port_num - 1];
0296     spin_lock(&port->mp.mpi_lock);
0297     mpi = ibdev->port[ib_port_num - 1].mp.mpi;
0298     if (mpi && !mpi->unaffiliate) {
0299         mdev = mpi->mdev;
0300         /* If it's the master no need to refcount, it'll exist
0301          * as long as the ib_dev exists.
0302          */
0303         if (!mpi->is_master)
0304             mpi->mdev_refcnt++;
0305     }
0306     spin_unlock(&port->mp.mpi_lock);
0307
0308     return mdev;
0309 }
0310
0311 void mlx5_ib_put_native_port_mdev(struct mlx5_ib_dev *ibdev, u32 port_num)
0312 {
0313     enum rdma_link_layer ll = mlx5_ib_port_link_layer(&ibdev->ib_dev,
0314                               port_num);
0315     struct mlx5_ib_multiport_info *mpi;
0316     struct mlx5_ib_port *port;
0317
0318     if (!mlx5_core_mp_enabled(ibdev->mdev) || ll != IB_LINK_LAYER_ETHERNET)
0319         return;
0320
0321     port = &ibdev->port[port_num - 1];
0322
0323     spin_lock(&port->mp.mpi_lock);
0324     mpi = ibdev->port[port_num - 1].mp.mpi;
0325     if (mpi->is_master)
0326         goto out;
0327
0328     mpi->mdev_refcnt--;
0329     if (mpi->unaffiliate)
0330         complete(&mpi->unref_comp);
0331 out:
0332     spin_unlock(&port->mp.mpi_lock);
0333 }
0334
0335 static int translate_eth_legacy_proto_oper(u32 eth_proto_oper,
0336                        u16 *active_speed, u8 *active_width)
0337 {
0338     switch (eth_proto_oper) {
0339     case MLX5E_PROT_MASK(MLX5E_1000BASE_CX_SGMII):
0340     case MLX5E_PROT_MASK(MLX5E_1000BASE_KX):
0341     case MLX5E_PROT_MASK(MLX5E_100BASE_TX):
0342     case MLX5E_PROT_MASK(MLX5E_1000BASE_T):
0343         *active_width = IB_WIDTH_1X;
0344         *active_speed = IB_SPEED_SDR;
0345         break;
0346     case MLX5E_PROT_MASK(MLX5E_10GBASE_T):
0347     case MLX5E_PROT_MASK(MLX5E_10GBASE_CX4):
0348     case MLX5E_PROT_MASK(MLX5E_10GBASE_KX4):
0349     case MLX5E_PROT_MASK(MLX5E_10GBASE_KR):
0350     case MLX5E_PROT_MASK(MLX5E_10GBASE_CR):
0351     case MLX5E_PROT_MASK(MLX5E_10GBASE_SR):
0352     case MLX5E_PROT_MASK(MLX5E_10GBASE_ER):
0353         *active_width = IB_WIDTH_1X;
0354         *active_speed = IB_SPEED_QDR;
0355         break;
0356     case MLX5E_PROT_MASK(MLX5E_25GBASE_CR):
0357     case MLX5E_PROT_MASK(MLX5E_25GBASE_KR):
0358     case MLX5E_PROT_MASK(MLX5E_25GBASE_SR):
0359         *active_width = IB_WIDTH_1X;
0360         *active_speed = IB_SPEED_EDR;
0361         break;
0362     case MLX5E_PROT_MASK(MLX5E_40GBASE_CR4):
0363     case MLX5E_PROT_MASK(MLX5E_40GBASE_KR4):
0364     case MLX5E_PROT_MASK(MLX5E_40GBASE_SR4):
0365     case MLX5E_PROT_MASK(MLX5E_40GBASE_LR4):
0366         *active_width = IB_WIDTH_4X;
0367         *active_speed = IB_SPEED_QDR;
0368         break;
0369     case MLX5E_PROT_MASK(MLX5E_50GBASE_CR2):
0370     case MLX5E_PROT_MASK(MLX5E_50GBASE_KR2):
0371     case MLX5E_PROT_MASK(MLX5E_50GBASE_SR2):
0372         *active_width = IB_WIDTH_1X;
0373         *active_speed = IB_SPEED_HDR;
0374         break;
0375     case MLX5E_PROT_MASK(MLX5E_56GBASE_R4):
0376         *active_width = IB_WIDTH_4X;
0377         *active_speed = IB_SPEED_FDR;
0378         break;
0379     case MLX5E_PROT_MASK(MLX5E_100GBASE_CR4):
0380     case MLX5E_PROT_MASK(MLX5E_100GBASE_SR4):
0381     case MLX5E_PROT_MASK(MLX5E_100GBASE_KR4):
0382     case MLX5E_PROT_MASK(MLX5E_100GBASE_LR4):
0383         *active_width = IB_WIDTH_4X;
0384         *active_speed = IB_SPEED_EDR;
0385         break;
0386     default:
0387         return -EINVAL;
0388     }
0389
0390     return 0;
0391 }
0392
0393 static int translate_eth_ext_proto_oper(u32 eth_proto_oper, u16 *active_speed,
0394                     u8 *active_width)
0395 {
0396     switch (eth_proto_oper) {
0397     case MLX5E_PROT_MASK(MLX5E_SGMII_100M):
0398     case MLX5E_PROT_MASK(MLX5E_1000BASE_X_SGMII):
0399         *active_width = IB_WIDTH_1X;
0400         *active_speed = IB_SPEED_SDR;
0401         break;
0402     case MLX5E_PROT_MASK(MLX5E_5GBASE_R):
0403         *active_width = IB_WIDTH_1X;
0404         *active_speed = IB_SPEED_DDR;
0405         break;
0406     case MLX5E_PROT_MASK(MLX5E_10GBASE_XFI_XAUI_1):
0407         *active_width = IB_WIDTH_1X;
0408         *active_speed = IB_SPEED_QDR;
0409         break;
0410     case MLX5E_PROT_MASK(MLX5E_40GBASE_XLAUI_4_XLPPI_4):
0411         *active_width = IB_WIDTH_4X;
0412         *active_speed = IB_SPEED_QDR;
0413         break;
0414     case MLX5E_PROT_MASK(MLX5E_25GAUI_1_25GBASE_CR_KR):
0415         *active_width = IB_WIDTH_1X;
0416         *active_speed = IB_SPEED_EDR;
0417         break;
0418     case MLX5E_PROT_MASK(MLX5E_50GAUI_2_LAUI_2_50GBASE_CR2_KR2):
0419         *active_width = IB_WIDTH_2X;
0420         *active_speed = IB_SPEED_EDR;
0421         break;
0422     case MLX5E_PROT_MASK(MLX5E_50GAUI_1_LAUI_1_50GBASE_CR_KR):
0423         *active_width = IB_WIDTH_1X;
0424         *active_speed = IB_SPEED_HDR;
0425         break;
0426     case MLX5E_PROT_MASK(MLX5E_CAUI_4_100GBASE_CR4_KR4):
0427         *active_width = IB_WIDTH_4X;
0428         *active_speed = IB_SPEED_EDR;
0429         break;
0430     case MLX5E_PROT_MASK(MLX5E_100GAUI_2_100GBASE_CR2_KR2):
0431         *active_width = IB_WIDTH_2X;
0432         *active_speed = IB_SPEED_HDR;
0433         break;
0434     case MLX5E_PROT_MASK(MLX5E_100GAUI_1_100GBASE_CR_KR):
0435         *active_width = IB_WIDTH_1X;
0436         *active_speed = IB_SPEED_NDR;
0437         break;
0438     case MLX5E_PROT_MASK(MLX5E_200GAUI_4_200GBASE_CR4_KR4):
0439         *active_width = IB_WIDTH_4X;
0440         *active_speed = IB_SPEED_HDR;
0441         break;
0442     case MLX5E_PROT_MASK(MLX5E_200GAUI_2_200GBASE_CR2_KR2):
0443         *active_width = IB_WIDTH_2X;
0444         *active_speed = IB_SPEED_NDR;
0445         break;
0446     case MLX5E_PROT_MASK(MLX5E_400GAUI_4_400GBASE_CR4_KR4):
0447         *active_width = IB_WIDTH_4X;
0448         *active_speed = IB_SPEED_NDR;
0449         break;
0450     default:
0451         return -EINVAL;
0452     }
0453
0454     return 0;
0455 }
0456
0457 static int translate_eth_proto_oper(u32 eth_proto_oper, u16 *active_speed,
0458                     u8 *active_width, bool ext)
0459 {
0460     return ext ?
0461         translate_eth_ext_proto_oper(eth_proto_oper, active_speed,
0462                          active_width) :
0463         translate_eth_legacy_proto_oper(eth_proto_oper, active_speed,
0464                         active_width);
0465 }
0466
0467 static int mlx5_query_port_roce(struct ib_device *device, u32 port_num,
0468                 struct ib_port_attr *props)
0469 {
0470     struct mlx5_ib_dev *dev = to_mdev(device);
0471     u32 out[MLX5_ST_SZ_DW(ptys_reg)] = {0};
0472     struct mlx5_core_dev *mdev;
0473     struct net_device *ndev, *upper;
0474     enum ib_mtu ndev_ib_mtu;
0475     bool put_mdev = true;
0476     u32 eth_prot_oper;
0477     u32 mdev_port_num;
0478     bool ext;
0479     int err;
0480
0481     mdev = mlx5_ib_get_native_port_mdev(dev, port_num, &mdev_port_num);
0482     if (!mdev) {
0483         /* This means the port isn't affiliated yet. Get the
0484          * info for the master port instead.
0485          */
0486         put_mdev = false;
0487         mdev = dev->mdev;
0488         mdev_port_num = 1;
0489         port_num = 1;
0490     }
0491
0492     /* Possible bad flows are checked before filling out props so in case
0493      * of an error it will still be zeroed out.
0494      * Use native port in case of reps
0495      */
0496     if (dev->is_rep)
0497         err = mlx5_query_port_ptys(mdev, out, sizeof(out), MLX5_PTYS_EN,
0498                        1);
0499     else
0500         err = mlx5_query_port_ptys(mdev, out, sizeof(out), MLX5_PTYS_EN,
0501                        mdev_port_num);
0502     if (err)
0503         goto out;
0504     ext = !!MLX5_GET_ETH_PROTO(ptys_reg, out, true, eth_proto_capability);
0505     eth_prot_oper = MLX5_GET_ETH_PROTO(ptys_reg, out, ext, eth_proto_oper);
0506
0507     props->active_width     = IB_WIDTH_4X;
0508     props->active_speed     = IB_SPEED_QDR;
0509
0510     translate_eth_proto_oper(eth_prot_oper, &props->active_speed,
0511                  &props->active_width, ext);
0512
0513     if (!dev->is_rep && dev->mdev->roce.roce_en) {
0514         u16 qkey_viol_cntr;
0515
0516         props->port_cap_flags |= IB_PORT_CM_SUP;
0517         props->ip_gids = true;
0518         props->gid_tbl_len = MLX5_CAP_ROCE(dev->mdev,
0519                            roce_address_table_size);
0520         mlx5_query_nic_vport_qkey_viol_cntr(mdev, &qkey_viol_cntr);
0521         props->qkey_viol_cntr = qkey_viol_cntr;
0522     }
0523     props->max_mtu          = IB_MTU_4096;
0524     props->max_msg_sz       = 1 << MLX5_CAP_GEN(dev->mdev, log_max_msg);
0525     props->pkey_tbl_len     = 1;
0526     props->state            = IB_PORT_DOWN;
0527     props->phys_state       = IB_PORT_PHYS_STATE_DISABLED;
0528
0529     /* If this is a stub query for an unaffiliated port stop here */
0530     if (!put_mdev)
0531         goto out;
0532
0533     ndev = mlx5_ib_get_netdev(device, port_num);
0534     if (!ndev)
0535         goto out;
0536
0537     if (dev->lag_active) {
0538         rcu_read_lock();
0539         upper = netdev_master_upper_dev_get_rcu(ndev);
0540         if (upper) {
0541             dev_put(ndev);
0542             ndev = upper;
0543             dev_hold(ndev);
0544         }
0545         rcu_read_unlock();
0546     }
0547
0548     if (netif_running(ndev) && netif_carrier_ok(ndev)) {
0549         props->state      = IB_PORT_ACTIVE;
0550         props->phys_state = IB_PORT_PHYS_STATE_LINK_UP;
0551     }
0552
0553     ndev_ib_mtu = iboe_get_mtu(ndev->mtu);
0554
0555     dev_put(ndev);
0556
0557     props->active_mtu   = min(props->max_mtu, ndev_ib_mtu);
0558 out:
0559     if (put_mdev)
0560         mlx5_ib_put_native_port_mdev(dev, port_num);
0561     return err;
0562 }
0563
0564 static int set_roce_addr(struct mlx5_ib_dev *dev, u32 port_num,
0565              unsigned int index, const union ib_gid *gid,
0566              const struct ib_gid_attr *attr)
0567 {
0568     enum ib_gid_type gid_type;
0569     u16 vlan_id = 0xffff;
0570     u8 roce_version = 0;
0571     u8 roce_l3_type = 0;
0572     u8 mac[ETH_ALEN];
0573     int ret;
0574
0575     gid_type = attr->gid_type;
0576     if (gid) {
0577         ret = rdma_read_gid_l2_fields(attr, &vlan_id, &mac[0]);
0578         if (ret)
0579             return ret;
0580     }
0581
0582     switch (gid_type) {
0583     case IB_GID_TYPE_ROCE:
0584         roce_version = MLX5_ROCE_VERSION_1;
0585         break;
0586     case IB_GID_TYPE_ROCE_UDP_ENCAP:
0587         roce_version = MLX5_ROCE_VERSION_2;
0588         if (gid && ipv6_addr_v4mapped((void *)gid))
0589             roce_l3_type = MLX5_ROCE_L3_TYPE_IPV4;
0590         else
0591             roce_l3_type = MLX5_ROCE_L3_TYPE_IPV6;
0592         break;
0593
0594     default:
0595         mlx5_ib_warn(dev, "Unexpected GID type %u\n", gid_type);
0596     }
0597
0598     return mlx5_core_roce_gid_set(dev->mdev, index, roce_version,
0599                       roce_l3_type, gid->raw, mac,
0600                       vlan_id < VLAN_CFI_MASK, vlan_id,
0601                       port_num);
0602 }
0603
0604 static int mlx5_ib_add_gid(const struct ib_gid_attr *attr,
0605                __always_unused void **context)
0606 {
0607     return set_roce_addr(to_mdev(attr->device), attr->port_num,
0608                  attr->index, &attr->gid, attr);
0609 }
0610
0611 static int mlx5_ib_del_gid(const struct ib_gid_attr *attr,
0612                __always_unused void **context)
0613 {
0614     return set_roce_addr(to_mdev(attr->device), attr->port_num,
0615                  attr->index, NULL, attr);
0616 }
0617
0618 __be16 mlx5_get_roce_udp_sport_min(const struct mlx5_ib_dev *dev,
0619                    const struct ib_gid_attr *attr)
0620 {
0621     if (attr->gid_type != IB_GID_TYPE_ROCE_UDP_ENCAP)
0622         return 0;
0623
0624     return cpu_to_be16(MLX5_CAP_ROCE(dev->mdev, r_roce_min_src_udp_port));
0625 }
0626
0627 static int mlx5_use_mad_ifc(struct mlx5_ib_dev *dev)
0628 {
0629     if (MLX5_CAP_GEN(dev->mdev, port_type) == MLX5_CAP_PORT_TYPE_IB)
0630         return !MLX5_CAP_GEN(dev->mdev, ib_virt);
0631     return 0;
0632 }
0633
0634 enum {
0635     MLX5_VPORT_ACCESS_METHOD_MAD,
0636     MLX5_VPORT_ACCESS_METHOD_HCA,
0637     MLX5_VPORT_ACCESS_METHOD_NIC,
0638 };
0639
0640 static int mlx5_get_vport_access_method(struct ib_device *ibdev)
0641 {
0642     if (mlx5_use_mad_ifc(to_mdev(ibdev)))
0643         return MLX5_VPORT_ACCESS_METHOD_MAD;
0644
0645     if (mlx5_ib_port_link_layer(ibdev, 1) ==
0646         IB_LINK_LAYER_ETHERNET)
0647         return MLX5_VPORT_ACCESS_METHOD_NIC;
0648
0649     return MLX5_VPORT_ACCESS_METHOD_HCA;
0650 }
0651
0652 static void get_atomic_caps(struct mlx5_ib_dev *dev,
0653                 u8 atomic_size_qp,
0654                 struct ib_device_attr *props)
0655 {
0656     u8 tmp;
0657     u8 atomic_operations = MLX5_CAP_ATOMIC(dev->mdev, atomic_operations);
0658     u8 atomic_req_8B_endianness_mode =
0659         MLX5_CAP_ATOMIC(dev->mdev, atomic_req_8B_endianness_mode);
0660
0661     /* Check if HW supports 8 bytes standard atomic operations and capable
0662      * of host endianness respond
0663      */
0664     tmp = MLX5_ATOMIC_OPS_CMP_SWAP | MLX5_ATOMIC_OPS_FETCH_ADD;
0665     if (((atomic_operations & tmp) == tmp) &&
0666         (atomic_size_qp & MLX5_ATOMIC_SIZE_QP_8BYTES) &&
0667         (atomic_req_8B_endianness_mode)) {
0668         props->atomic_cap = IB_ATOMIC_HCA;
0669     } else {
0670         props->atomic_cap = IB_ATOMIC_NONE;
0671     }
0672 }
0673
0674 static void get_atomic_caps_qp(struct mlx5_ib_dev *dev,
0675                    struct ib_device_attr *props)
0676 {
0677     u8 atomic_size_qp = MLX5_CAP_ATOMIC(dev->mdev, atomic_size_qp);
0678
0679     get_atomic_caps(dev, atomic_size_qp, props);
0680 }
0681
0682 static int mlx5_query_system_image_guid(struct ib_device *ibdev,
0683                     __be64 *sys_image_guid)
0684 {
0685     struct mlx5_ib_dev *dev = to_mdev(ibdev);
0686     struct mlx5_core_dev *mdev = dev->mdev;
0687     u64 tmp;
0688     int err;
0689
0690     switch (mlx5_get_vport_access_method(ibdev)) {
0691     case MLX5_VPORT_ACCESS_METHOD_MAD:
0692         return mlx5_query_mad_ifc_system_image_guid(ibdev,
0693                                 sys_image_guid);
0694
0695     case MLX5_VPORT_ACCESS_METHOD_HCA:
0696         err = mlx5_query_hca_vport_system_image_guid(mdev, &tmp);
0697         break;
0698
0699     case MLX5_VPORT_ACCESS_METHOD_NIC:
0700         err = mlx5_query_nic_vport_system_image_guid(mdev, &tmp);
0701         break;
0702
0703     default:
0704         return -EINVAL;
0705     }
0706
0707     if (!err)
0708         *sys_image_guid = cpu_to_be64(tmp);
0709
0710     return err;
0711
0712 }
0713
0714 static int mlx5_query_max_pkeys(struct ib_device *ibdev,
0715                 u16 *max_pkeys)
0716 {
0717     struct mlx5_ib_dev *dev = to_mdev(ibdev);
0718     struct mlx5_core_dev *mdev = dev->mdev;
0719
0720     switch (mlx5_get_vport_access_method(ibdev)) {
0721     case MLX5_VPORT_ACCESS_METHOD_MAD:
0722         return mlx5_query_mad_ifc_max_pkeys(ibdev, max_pkeys);
0723
0724     case MLX5_VPORT_ACCESS_METHOD_HCA:
0725     case MLX5_VPORT_ACCESS_METHOD_NIC:
0726         *max_pkeys = mlx5_to_sw_pkey_sz(MLX5_CAP_GEN(mdev,
0727                         pkey_table_size));
0728         return 0;
0729
0730     default:
0731         return -EINVAL;
0732     }
0733 }
0734
0735 static int mlx5_query_vendor_id(struct ib_device *ibdev,
0736                 u32 *vendor_id)
0737 {
0738     struct mlx5_ib_dev *dev = to_mdev(ibdev);
0739
0740     switch (mlx5_get_vport_access_method(ibdev)) {
0741     case MLX5_VPORT_ACCESS_METHOD_MAD:
0742         return mlx5_query_mad_ifc_vendor_id(ibdev, vendor_id);
0743
0744     case MLX5_VPORT_ACCESS_METHOD_HCA:
0745     case MLX5_VPORT_ACCESS_METHOD_NIC:
0746         return mlx5_core_query_vendor_id(dev->mdev, vendor_id);
0747
0748     default:
0749         return -EINVAL;
0750     }
0751 }
0752
0753 static int mlx5_query_node_guid(struct mlx5_ib_dev *dev,
0754                 __be64 *node_guid)
0755 {
0756     u64 tmp;
0757     int err;
0758
0759     switch (mlx5_get_vport_access_method(&dev->ib_dev)) {
0760     case MLX5_VPORT_ACCESS_METHOD_MAD:
0761         return mlx5_query_mad_ifc_node_guid(dev, node_guid);
0762
0763     case MLX5_VPORT_ACCESS_METHOD_HCA:
0764         err = mlx5_query_hca_vport_node_guid(dev->mdev, &tmp);
0765         break;
0766
0767     case MLX5_VPORT_ACCESS_METHOD_NIC:
0768         err = mlx5_query_nic_vport_node_guid(dev->mdev, &tmp);
0769         break;
0770
0771     default:
0772         return -EINVAL;
0773     }
0774
0775     if (!err)
0776         *node_guid = cpu_to_be64(tmp);
0777
0778     return err;
0779 }
0780
0781 struct mlx5_reg_node_desc {
0782     u8  desc[IB_DEVICE_NODE_DESC_MAX];
0783 };
0784
0785 static int mlx5_query_node_desc(struct mlx5_ib_dev *dev, char *node_desc)
0786 {
0787     struct mlx5_reg_node_desc in;
0788
0789     if (mlx5_use_mad_ifc(dev))
0790         return mlx5_query_mad_ifc_node_desc(dev, node_desc);
0791
0792     memset(&in, 0, sizeof(in));
0793
0794     return mlx5_core_access_reg(dev->mdev, &in, sizeof(in), node_desc,
0795                     sizeof(struct mlx5_reg_node_desc),
0796                     MLX5_REG_NODE_DESC, 0, 0);
0797 }
0798
0799 static int mlx5_ib_query_device(struct ib_device *ibdev,
0800                 struct ib_device_attr *props,
0801                 struct ib_udata *uhw)
0802 {
0803     size_t uhw_outlen = (uhw) ? uhw->outlen : 0;
0804     struct mlx5_ib_dev *dev = to_mdev(ibdev);
0805     struct mlx5_core_dev *mdev = dev->mdev;
0806     int err = -ENOMEM;
0807     int max_sq_desc;
0808     int max_rq_sg;
0809     int max_sq_sg;
0810     u64 min_page_size = 1ull << MLX5_CAP_GEN(mdev, log_pg_sz);
0811     bool raw_support = !mlx5_core_mp_enabled(mdev);
0812     struct mlx5_ib_query_device_resp resp = {};
0813     size_t resp_len;
0814     u64 max_tso;
0815
0816     resp_len = sizeof(resp.comp_mask) + sizeof(resp.response_length);
0817     if (uhw_outlen && uhw_outlen < resp_len)
0818         return -EINVAL;
0819
0820     resp.response_length = resp_len;
0821
0822     if (uhw && uhw->inlen && !ib_is_udata_cleared(uhw, 0, uhw->inlen))
0823         return -EINVAL;
0824
0825     memset(props, 0, sizeof(*props));
0826     err = mlx5_query_system_image_guid(ibdev,
0827                        &props->sys_image_guid);
0828     if (err)
0829         return err;
0830
0831     props->max_pkeys = dev->pkey_table_len;
0832
0833     err = mlx5_query_vendor_id(ibdev, &props->vendor_id);
0834     if (err)
0835         return err;
0836
0837     props->fw_ver = ((u64)fw_rev_maj(dev->mdev) << 32) |
0838         (fw_rev_min(dev->mdev) << 16) |
0839         fw_rev_sub(dev->mdev);
0840     props->device_cap_flags    = IB_DEVICE_CHANGE_PHY_PORT |
0841         IB_DEVICE_PORT_ACTIVE_EVENT     |
0842         IB_DEVICE_SYS_IMAGE_GUID        |
0843         IB_DEVICE_RC_RNR_NAK_GEN;
0844
0845     if (MLX5_CAP_GEN(mdev, pkv))
0846         props->device_cap_flags |= IB_DEVICE_BAD_PKEY_CNTR;
0847     if (MLX5_CAP_GEN(mdev, qkv))
0848         props->device_cap_flags |= IB_DEVICE_BAD_QKEY_CNTR;
0849     if (MLX5_CAP_GEN(mdev, apm))
0850         props->device_cap_flags |= IB_DEVICE_AUTO_PATH_MIG;
0851     if (MLX5_CAP_GEN(mdev, xrc))
0852         props->device_cap_flags |= IB_DEVICE_XRC;
0853     if (MLX5_CAP_GEN(mdev, imaicl)) {
0854         props->device_cap_flags |= IB_DEVICE_MEM_WINDOW |
0855                        IB_DEVICE_MEM_WINDOW_TYPE_2B;
0856         props->max_mw = 1 << MLX5_CAP_GEN(mdev, log_max_mkey);
0857         /* We support 'Gappy' memory registration too */
0858         props->kernel_cap_flags |= IBK_SG_GAPS_REG;
0859     }
0860     /* IB_WR_REG_MR always requires changing the entity size with UMR */
0861     if (!MLX5_CAP_GEN(dev->mdev, umr_modify_entity_size_disabled))
0862         props->device_cap_flags |= IB_DEVICE_MEM_MGT_EXTENSIONS;
0863     if (MLX5_CAP_GEN(mdev, sho)) {
0864         props->kernel_cap_flags |= IBK_INTEGRITY_HANDOVER;
0865         /* At this stage no support for signature handover */
0866         props->sig_prot_cap = IB_PROT_T10DIF_TYPE_1 |
0867                       IB_PROT_T10DIF_TYPE_2 |
0868                       IB_PROT_T10DIF_TYPE_3;
0869         props->sig_guard_cap = IB_GUARD_T10DIF_CRC |
0870                        IB_GUARD_T10DIF_CSUM;
0871     }
0872     if (MLX5_CAP_GEN(mdev, block_lb_mc))
0873         props->kernel_cap_flags |= IBK_BLOCK_MULTICAST_LOOPBACK;
0874
0875     if (MLX5_CAP_GEN(dev->mdev, eth_net_offloads) && raw_support) {
0876         if (MLX5_CAP_ETH(mdev, csum_cap)) {
0877             /* Legacy bit to support old userspace libraries */
0878             props->device_cap_flags |= IB_DEVICE_RAW_IP_CSUM;
0879             props->raw_packet_caps |= IB_RAW_PACKET_CAP_IP_CSUM;
0880         }
0881
0882         if (MLX5_CAP_ETH(dev->mdev, vlan_cap))
0883             props->raw_packet_caps |=
0884                 IB_RAW_PACKET_CAP_CVLAN_STRIPPING;
0885
0886         if (offsetofend(typeof(resp), tso_caps) <= uhw_outlen) {
0887             max_tso = MLX5_CAP_ETH(mdev, max_lso_cap);
0888             if (max_tso) {
0889                 resp.tso_caps.max_tso = 1 << max_tso;
0890                 resp.tso_caps.supported_qpts |=
0891                     1 << IB_QPT_RAW_PACKET;
0892                 resp.response_length += sizeof(resp.tso_caps);
0893             }
0894         }
0895
0896         if (offsetofend(typeof(resp), rss_caps) <= uhw_outlen) {
0897             resp.rss_caps.rx_hash_function =
0898                         MLX5_RX_HASH_FUNC_TOEPLITZ;
0899             resp.rss_caps.rx_hash_fields_mask =
0900                         MLX5_RX_HASH_SRC_IPV4 |
0901                         MLX5_RX_HASH_DST_IPV4 |
0902                         MLX5_RX_HASH_SRC_IPV6 |
0903                         MLX5_RX_HASH_DST_IPV6 |
0904                         MLX5_RX_HASH_SRC_PORT_TCP |
0905                         MLX5_RX_HASH_DST_PORT_TCP |
0906                         MLX5_RX_HASH_SRC_PORT_UDP |
0907                         MLX5_RX_HASH_DST_PORT_UDP |
0908                         MLX5_RX_HASH_INNER;
0909             resp.response_length += sizeof(resp.rss_caps);
0910         }
0911     } else {
0912         if (offsetofend(typeof(resp), tso_caps) <= uhw_outlen)
0913             resp.response_length += sizeof(resp.tso_caps);
0914         if (offsetofend(typeof(resp), rss_caps) <= uhw_outlen)
0915             resp.response_length += sizeof(resp.rss_caps);
0916     }
0917
0918     if (MLX5_CAP_GEN(mdev, ipoib_basic_offloads)) {
0919         props->device_cap_flags |= IB_DEVICE_UD_IP_CSUM;
0920         props->kernel_cap_flags |= IBK_UD_TSO;
0921     }
0922
0923     if (MLX5_CAP_GEN(dev->mdev, rq_delay_drop) &&
0924         MLX5_CAP_GEN(dev->mdev, general_notification_event) &&
0925         raw_support)
0926         props->raw_packet_caps |= IB_RAW_PACKET_CAP_DELAY_DROP;
0927
0928     if (MLX5_CAP_GEN(mdev, ipoib_enhanced_offloads) &&
0929         MLX5_CAP_IPOIB_ENHANCED(mdev, csum_cap))
0930         props->device_cap_flags |= IB_DEVICE_UD_IP_CSUM;
0931
0932     if (MLX5_CAP_GEN(dev->mdev, eth_net_offloads) &&
0933         MLX5_CAP_ETH(dev->mdev, scatter_fcs) &&
0934         raw_support) {
0935         /* Legacy bit to support old userspace libraries */
0936         props->device_cap_flags |= IB_DEVICE_RAW_SCATTER_FCS;
0937         props->raw_packet_caps |= IB_RAW_PACKET_CAP_SCATTER_FCS;
0938     }
0939
0940     if (MLX5_CAP_DEV_MEM(mdev, memic)) {
0941         props->max_dm_size =
0942             MLX5_CAP_DEV_MEM(mdev, max_memic_size);
0943     }
0944
0945     if (mlx5_get_flow_namespace(dev->mdev, MLX5_FLOW_NAMESPACE_BYPASS))
0946         props->device_cap_flags |= IB_DEVICE_MANAGED_FLOW_STEERING;
0947
0948     if (MLX5_CAP_GEN(mdev, end_pad))
0949         props->device_cap_flags |= IB_DEVICE_PCI_WRITE_END_PADDING;
0950
0951     props->vendor_part_id      = mdev->pdev->device;
0952     props->hw_ver          = mdev->pdev->revision;
0953
0954     props->max_mr_size     = ~0ull;
0955     props->page_size_cap       = ~(min_page_size - 1);
0956     props->max_qp          = 1 << MLX5_CAP_GEN(mdev, log_max_qp);
0957     props->max_qp_wr       = 1 << MLX5_CAP_GEN(mdev, log_max_qp_sz);
0958     max_rq_sg =  MLX5_CAP_GEN(mdev, max_wqe_sz_rq) /
0959              sizeof(struct mlx5_wqe_data_seg);
0960     max_sq_desc = min_t(int, MLX5_CAP_GEN(mdev, max_wqe_sz_sq), 512);
0961     max_sq_sg = (max_sq_desc - sizeof(struct mlx5_wqe_ctrl_seg) -
0962              sizeof(struct mlx5_wqe_raddr_seg)) /
0963         sizeof(struct mlx5_wqe_data_seg);
0964     props->max_send_sge = max_sq_sg;
0965     props->max_recv_sge = max_rq_sg;
0966     props->max_sge_rd      = MLX5_MAX_SGE_RD;
0967     props->max_cq          = 1 << MLX5_CAP_GEN(mdev, log_max_cq);
0968     props->max_cqe = (1 << MLX5_CAP_GEN(mdev, log_max_cq_sz)) - 1;
0969     props->max_mr          = 1 << MLX5_CAP_GEN(mdev, log_max_mkey);
0970     props->max_pd          = 1 << MLX5_CAP_GEN(mdev, log_max_pd);
0971     props->max_qp_rd_atom      = 1 << MLX5_CAP_GEN(mdev, log_max_ra_req_qp);
0972     props->max_qp_init_rd_atom = 1 << MLX5_CAP_GEN(mdev, log_max_ra_res_qp);
0973     props->max_srq         = 1 << MLX5_CAP_GEN(mdev, log_max_srq);
0974     props->max_srq_wr = (1 << MLX5_CAP_GEN(mdev, log_max_srq_sz)) - 1;
0975     props->local_ca_ack_delay  = MLX5_CAP_GEN(mdev, local_ca_ack_delay);
0976     props->max_res_rd_atom     = props->max_qp_rd_atom * props->max_qp;
0977     props->max_srq_sge     = max_rq_sg - 1;
0978     props->max_fast_reg_page_list_len =
0979         1 << MLX5_CAP_GEN(mdev, log_max_klm_list_size);
0980     props->max_pi_fast_reg_page_list_len =
0981         props->max_fast_reg_page_list_len / 2;
0982     props->max_sgl_rd =
0983         MLX5_CAP_GEN(mdev, max_sgl_for_optimized_performance);
0984     get_atomic_caps_qp(dev, props);
0985     props->masked_atomic_cap   = IB_ATOMIC_NONE;
0986     props->max_mcast_grp       = 1 << MLX5_CAP_GEN(mdev, log_max_mcg);
0987     props->max_mcast_qp_attach = MLX5_CAP_GEN(mdev, max_qp_mcg);
0988     props->max_total_mcast_qp_attach = props->max_mcast_qp_attach *
0989                        props->max_mcast_grp;
0990     props->max_ah = INT_MAX;
0991     props->hca_core_clock = MLX5_CAP_GEN(mdev, device_frequency_khz);
0992     props->timestamp_mask = 0x7FFFFFFFFFFFFFFFULL;
0993
0994     if (IS_ENABLED(CONFIG_INFINIBAND_ON_DEMAND_PAGING)) {
0995         if (dev->odp_caps.general_caps & IB_ODP_SUPPORT)
0996             props->kernel_cap_flags |= IBK_ON_DEMAND_PAGING;
0997         props->odp_caps = dev->odp_caps;
0998         if (!uhw) {
0999             /* ODP for kernel QPs is not implemented for receive
1000              * WQEs and SRQ WQEs
1001              */
1002             props->odp_caps.per_transport_caps.rc_odp_caps &=
1003                 ~(IB_ODP_SUPPORT_READ |
1004                   IB_ODP_SUPPORT_SRQ_RECV);
1005             props->odp_caps.per_transport_caps.uc_odp_caps &=
1006                 ~(IB_ODP_SUPPORT_READ |
1007                   IB_ODP_SUPPORT_SRQ_RECV);
1008             props->odp_caps.per_transport_caps.ud_odp_caps &=
1009                 ~(IB_ODP_SUPPORT_READ |
1010                   IB_ODP_SUPPORT_SRQ_RECV);
1011             props->odp_caps.per_transport_caps.xrc_odp_caps &=
1012                 ~(IB_ODP_SUPPORT_READ |
1013                   IB_ODP_SUPPORT_SRQ_RECV);
1014         }
1015     }
1016
1017     if (mlx5_core_is_vf(mdev))
1018         props->kernel_cap_flags |= IBK_VIRTUAL_FUNCTION;
1019
1020     if (mlx5_ib_port_link_layer(ibdev, 1) ==
1021         IB_LINK_LAYER_ETHERNET && raw_support) {
1022         props->rss_caps.max_rwq_indirection_tables =
1023             1 << MLX5_CAP_GEN(dev->mdev, log_max_rqt);
1024         props->rss_caps.max_rwq_indirection_table_size =
1025             1 << MLX5_CAP_GEN(dev->mdev, log_max_rqt_size);
1026         props->rss_caps.supported_qpts = 1 << IB_QPT_RAW_PACKET;
1027         props->max_wq_type_rq =
1028             1 << MLX5_CAP_GEN(dev->mdev, log_max_rq);
1029     }
1030
1031     if (MLX5_CAP_GEN(mdev, tag_matching)) {
1032         props->tm_caps.max_num_tags =
1033             (1 << MLX5_CAP_GEN(mdev, log_tag_matching_list_sz)) - 1;
1034         props->tm_caps.max_ops =
1035             1 << MLX5_CAP_GEN(mdev, log_max_qp_sz);
1036         props->tm_caps.max_sge = MLX5_TM_MAX_SGE;
1037     }
1038
1039     if (MLX5_CAP_GEN(mdev, tag_matching) &&
1040         MLX5_CAP_GEN(mdev, rndv_offload_rc)) {
1041         props->tm_caps.flags = IB_TM_CAP_RNDV_RC;
1042         props->tm_caps.max_rndv_hdr_size = MLX5_TM_MAX_RNDV_MSG_SIZE;
1043     }
1044
1045     if (MLX5_CAP_GEN(dev->mdev, cq_moderation)) {
1046         props->cq_caps.max_cq_moderation_count =
1047                         MLX5_MAX_CQ_COUNT;
1048         props->cq_caps.max_cq_moderation_period =
1049                         MLX5_MAX_CQ_PERIOD;
1050     }
1051
1052     if (offsetofend(typeof(resp), cqe_comp_caps) <= uhw_outlen) {
1053         resp.response_length += sizeof(resp.cqe_comp_caps);
1054
1055         if (MLX5_CAP_GEN(dev->mdev, cqe_compression)) {
1056             resp.cqe_comp_caps.max_num =
1057                 MLX5_CAP_GEN(dev->mdev,
1058                          cqe_compression_max_num);
1059
1060             resp.cqe_comp_caps.supported_format =
1061                 MLX5_IB_CQE_RES_FORMAT_HASH |
1062                 MLX5_IB_CQE_RES_FORMAT_CSUM;
1063
1064             if (MLX5_CAP_GEN(dev->mdev, mini_cqe_resp_stride_index))
1065                 resp.cqe_comp_caps.supported_format |=
1066                     MLX5_IB_CQE_RES_FORMAT_CSUM_STRIDX;
1067         }
1068     }
1069
1070     if (offsetofend(typeof(resp), packet_pacing_caps) <= uhw_outlen &&
1071         raw_support) {
1072         if (MLX5_CAP_QOS(mdev, packet_pacing) &&
1073             MLX5_CAP_GEN(mdev, qos)) {
1074             resp.packet_pacing_caps.qp_rate_limit_max =
1075                 MLX5_CAP_QOS(mdev, packet_pacing_max_rate);
1076             resp.packet_pacing_caps.qp_rate_limit_min =
1077                 MLX5_CAP_QOS(mdev, packet_pacing_min_rate);
1078             resp.packet_pacing_caps.supported_qpts |=
1079                 1 << IB_QPT_RAW_PACKET;
1080             if (MLX5_CAP_QOS(mdev, packet_pacing_burst_bound) &&
1081                 MLX5_CAP_QOS(mdev, packet_pacing_typical_size))
1082                 resp.packet_pacing_caps.cap_flags |=
1083                     MLX5_IB_PP_SUPPORT_BURST;
1084         }
1085         resp.response_length += sizeof(resp.packet_pacing_caps);
1086     }
1087
1088     if (offsetofend(typeof(resp), mlx5_ib_support_multi_pkt_send_wqes) <=
1089         uhw_outlen) {
1090         if (MLX5_CAP_ETH(mdev, multi_pkt_send_wqe))
1091             resp.mlx5_ib_support_multi_pkt_send_wqes =
1092                 MLX5_IB_ALLOW_MPW;
1093
1094         if (MLX5_CAP_ETH(mdev, enhanced_multi_pkt_send_wqe))
1095             resp.mlx5_ib_support_multi_pkt_send_wqes |=
1096                 MLX5_IB_SUPPORT_EMPW;
1097
1098         resp.response_length +=
1099             sizeof(resp.mlx5_ib_support_multi_pkt_send_wqes);
1100     }
1101
1102     if (offsetofend(typeof(resp), flags) <= uhw_outlen) {
1103         resp.response_length += sizeof(resp.flags);
1104
1105         if (MLX5_CAP_GEN(mdev, cqe_compression_128))
1106             resp.flags |=
1107                 MLX5_IB_QUERY_DEV_RESP_FLAGS_CQE_128B_COMP;
1108
1109         if (MLX5_CAP_GEN(mdev, cqe_128_always))
1110             resp.flags |= MLX5_IB_QUERY_DEV_RESP_FLAGS_CQE_128B_PAD;
1111         if (MLX5_CAP_GEN(mdev, qp_packet_based))
1112             resp.flags |=
1113                 MLX5_IB_QUERY_DEV_RESP_PACKET_BASED_CREDIT_MODE;
1114
1115         resp.flags |= MLX5_IB_QUERY_DEV_RESP_FLAGS_SCAT2CQE_DCT;
1116     }
1117
1118     if (offsetofend(typeof(resp), sw_parsing_caps) <= uhw_outlen) {
1119         resp.response_length += sizeof(resp.sw_parsing_caps);
1120         if (MLX5_CAP_ETH(mdev, swp)) {
1121             resp.sw_parsing_caps.sw_parsing_offloads |=
1122                 MLX5_IB_SW_PARSING;
1123
1124             if (MLX5_CAP_ETH(mdev, swp_csum))
1125                 resp.sw_parsing_caps.sw_parsing_offloads |=
1126                     MLX5_IB_SW_PARSING_CSUM;
1127
1128             if (MLX5_CAP_ETH(mdev, swp_lso))
1129                 resp.sw_parsing_caps.sw_parsing_offloads |=
1130                     MLX5_IB_SW_PARSING_LSO;
1131
1132             if (resp.sw_parsing_caps.sw_parsing_offloads)
1133                 resp.sw_parsing_caps.supported_qpts =
1134                     BIT(IB_QPT_RAW_PACKET);
1135         }
1136     }
1137
1138     if (offsetofend(typeof(resp), striding_rq_caps) <= uhw_outlen &&
1139         raw_support) {
1140         resp.response_length += sizeof(resp.striding_rq_caps);
1141         if (MLX5_CAP_GEN(mdev, striding_rq)) {
1142             resp.striding_rq_caps.min_single_stride_log_num_of_bytes =
1143                 MLX5_MIN_SINGLE_STRIDE_LOG_NUM_BYTES;
1144             resp.striding_rq_caps.max_single_stride_log_num_of_bytes =
1145                 MLX5_MAX_SINGLE_STRIDE_LOG_NUM_BYTES;
1146             if (MLX5_CAP_GEN(dev->mdev, ext_stride_num_range))
1147                 resp.striding_rq_caps
1148                     .min_single_wqe_log_num_of_strides =
1149                     MLX5_EXT_MIN_SINGLE_WQE_LOG_NUM_STRIDES;
1150             else
1151                 resp.striding_rq_caps
1152                     .min_single_wqe_log_num_of_strides =
1153                     MLX5_MIN_SINGLE_WQE_LOG_NUM_STRIDES;
1154             resp.striding_rq_caps.max_single_wqe_log_num_of_strides =
1155                 MLX5_MAX_SINGLE_WQE_LOG_NUM_STRIDES;
1156             resp.striding_rq_caps.supported_qpts =
1157                 BIT(IB_QPT_RAW_PACKET);
1158         }
1159     }
1160
1161     if (offsetofend(typeof(resp), tunnel_offloads_caps) <= uhw_outlen) {
1162         resp.response_length += sizeof(resp.tunnel_offloads_caps);
1163         if (MLX5_CAP_ETH(mdev, tunnel_stateless_vxlan))
1164             resp.tunnel_offloads_caps |=
1165                 MLX5_IB_TUNNELED_OFFLOADS_VXLAN;
1166         if (MLX5_CAP_ETH(mdev, tunnel_stateless_geneve_rx))
1167             resp.tunnel_offloads_caps |=
1168                 MLX5_IB_TUNNELED_OFFLOADS_GENEVE;
1169         if (MLX5_CAP_ETH(mdev, tunnel_stateless_gre))
1170             resp.tunnel_offloads_caps |=
1171                 MLX5_IB_TUNNELED_OFFLOADS_GRE;
1172         if (MLX5_CAP_ETH(mdev, tunnel_stateless_mpls_over_gre))
1173             resp.tunnel_offloads_caps |=
1174                 MLX5_IB_TUNNELED_OFFLOADS_MPLS_GRE;
1175         if (MLX5_CAP_ETH(mdev, tunnel_stateless_mpls_over_udp))
1176             resp.tunnel_offloads_caps |=
1177                 MLX5_IB_TUNNELED_OFFLOADS_MPLS_UDP;
1178     }
1179
1180     if (offsetofend(typeof(resp), dci_streams_caps) <= uhw_outlen) {
1181         resp.response_length += sizeof(resp.dci_streams_caps);
1182
1183         resp.dci_streams_caps.max_log_num_concurent =
1184             MLX5_CAP_GEN(mdev, log_max_dci_stream_channels);
1185
1186         resp.dci_streams_caps.max_log_num_errored =
1187             MLX5_CAP_GEN(mdev, log_max_dci_errored_streams);
1188     }
1189
1190     if (uhw_outlen) {
1191         err = ib_copy_to_udata(uhw, &resp, resp.response_length);
1192
1193         if (err)
1194             return err;
1195     }
1196
1197     return 0;
1198 }
1199
1200 static void translate_active_width(struct ib_device *ibdev, u16 active_width,
1201                    u8 *ib_width)
1202 {
1203     struct mlx5_ib_dev *dev = to_mdev(ibdev);
1204
1205     if (active_width & MLX5_PTYS_WIDTH_1X)
1206         *ib_width = IB_WIDTH_1X;
1207     else if (active_width & MLX5_PTYS_WIDTH_2X)
1208         *ib_width = IB_WIDTH_2X;
1209     else if (active_width & MLX5_PTYS_WIDTH_4X)
1210         *ib_width = IB_WIDTH_4X;
1211     else if (active_width & MLX5_PTYS_WIDTH_8X)
1212         *ib_width = IB_WIDTH_8X;
1213     else if (active_width & MLX5_PTYS_WIDTH_12X)
1214         *ib_width = IB_WIDTH_12X;
1215     else {
1216         mlx5_ib_dbg(dev, "Invalid active_width %d, setting width to default value: 4x\n",
1217                 active_width);
1218         *ib_width = IB_WIDTH_4X;
1219     }
1220
1221     return;
1222 }
1223
1224 static int mlx5_mtu_to_ib_mtu(int mtu)
1225 {
1226     switch (mtu) {
1227     case 256: return 1;
1228     case 512: return 2;
1229     case 1024: return 3;
1230     case 2048: return 4;
1231     case 4096: return 5;
1232     default:
1233         pr_warn("invalid mtu\n");
1234         return -1;
1235     }
1236 }
1237
1238 enum ib_max_vl_num {
1239     __IB_MAX_VL_0       = 1,
1240     __IB_MAX_VL_0_1     = 2,
1241     __IB_MAX_VL_0_3     = 3,
1242     __IB_MAX_VL_0_7     = 4,
1243     __IB_MAX_VL_0_14    = 5,
1244 };
1245
1246 enum mlx5_vl_hw_cap {
1247     MLX5_VL_HW_0    = 1,
1248     MLX5_VL_HW_0_1  = 2,
1249     MLX5_VL_HW_0_2  = 3,
1250     MLX5_VL_HW_0_3  = 4,
1251     MLX5_VL_HW_0_4  = 5,
1252     MLX5_VL_HW_0_5  = 6,
1253     MLX5_VL_HW_0_6  = 7,
1254     MLX5_VL_HW_0_7  = 8,
1255     MLX5_VL_HW_0_14 = 15
1256 };
1257
1258 static int translate_max_vl_num(struct ib_device *ibdev, u8 vl_hw_cap,
1259                 u8 *max_vl_num)
1260 {
1261     switch (vl_hw_cap) {
1262     case MLX5_VL_HW_0:
1263         *max_vl_num = __IB_MAX_VL_0;
1264         break;
1265     case MLX5_VL_HW_0_1:
1266         *max_vl_num = __IB_MAX_VL_0_1;
1267         break;
1268     case MLX5_VL_HW_0_3:
1269         *max_vl_num = __IB_MAX_VL_0_3;
1270         break;
1271     case MLX5_VL_HW_0_7:
1272         *max_vl_num = __IB_MAX_VL_0_7;
1273         break;
1274     case MLX5_VL_HW_0_14:
1275         *max_vl_num = __IB_MAX_VL_0_14;
1276         break;
1277
1278     default:
1279         return -EINVAL;
1280     }
1281
1282     return 0;
1283 }
1284
1285 static int mlx5_query_hca_port(struct ib_device *ibdev, u32 port,
1286                    struct ib_port_attr *props)
1287 {
1288     struct mlx5_ib_dev *dev = to_mdev(ibdev);
1289     struct mlx5_core_dev *mdev = dev->mdev;
1290     struct mlx5_hca_vport_context *rep;
1291     u16 max_mtu;
1292     u16 oper_mtu;
1293     int err;
1294     u16 ib_link_width_oper;
1295     u8 vl_hw_cap;
1296
1297     rep = kzalloc(sizeof(*rep), GFP_KERNEL);
1298     if (!rep) {
1299         err = -ENOMEM;
1300         goto out;
1301     }
1302
1303     /* props being zeroed by the caller, avoid zeroing it here */
1304
1305     err = mlx5_query_hca_vport_context(mdev, 0, port, 0, rep);
1306     if (err)
1307         goto out;
1308
1309     props->lid      = rep->lid;
1310     props->lmc      = rep->lmc;
1311     props->sm_lid       = rep->sm_lid;
1312     props->sm_sl        = rep->sm_sl;
1313     props->state        = rep->vport_state;
1314     props->phys_state   = rep->port_physical_state;
1315     props->port_cap_flags   = rep->cap_mask1;
1316     props->gid_tbl_len  = mlx5_get_gid_table_len(MLX5_CAP_GEN(mdev, gid_table_size));
1317     props->max_msg_sz   = 1 << MLX5_CAP_GEN(mdev, log_max_msg);
1318     props->pkey_tbl_len = mlx5_to_sw_pkey_sz(MLX5_CAP_GEN(mdev, pkey_table_size));
1319     props->bad_pkey_cntr    = rep->pkey_violation_counter;
1320     props->qkey_viol_cntr   = rep->qkey_violation_counter;
1321     props->subnet_timeout   = rep->subnet_timeout;
1322     props->init_type_reply  = rep->init_type_reply;
1323
1324     if (props->port_cap_flags & IB_PORT_CAP_MASK2_SUP)
1325         props->port_cap_flags2 = rep->cap_mask2;
1326
1327     err = mlx5_query_ib_port_oper(mdev, &ib_link_width_oper,
1328                       &props->active_speed, port);
1329     if (err)
1330         goto out;
1331
1332     translate_active_width(ibdev, ib_link_width_oper, &props->active_width);
1333
1334     mlx5_query_port_max_mtu(mdev, &max_mtu, port);
1335
1336     props->max_mtu = mlx5_mtu_to_ib_mtu(max_mtu);
1337
1338     mlx5_query_port_oper_mtu(mdev, &oper_mtu, port);
1339
1340     props->active_mtu = mlx5_mtu_to_ib_mtu(oper_mtu);
1341
1342     err = mlx5_query_port_vl_hw_cap(mdev, &vl_hw_cap, port);
1343     if (err)
1344         goto out;
1345
1346     err = translate_max_vl_num(ibdev, vl_hw_cap,
1347                    &props->max_vl_num);
1348 out:
1349     kfree(rep);
1350     return err;
1351 }
1352
1353 int mlx5_ib_query_port(struct ib_device *ibdev, u32 port,
1354                struct ib_port_attr *props)
1355 {
1356     unsigned int count;
1357     int ret;
1358
1359     switch (mlx5_get_vport_access_method(ibdev)) {
1360     case MLX5_VPORT_ACCESS_METHOD_MAD:
1361         ret = mlx5_query_mad_ifc_port(ibdev, port, props);
1362         break;
1363
1364     case MLX5_VPORT_ACCESS_METHOD_HCA:
1365         ret = mlx5_query_hca_port(ibdev, port, props);
1366         break;
1367
1368     case MLX5_VPORT_ACCESS_METHOD_NIC:
1369         ret = mlx5_query_port_roce(ibdev, port, props);
1370         break;
1371
1372     default:
1373         ret = -EINVAL;
1374     }
1375
1376     if (!ret && props) {
1377         struct mlx5_ib_dev *dev = to_mdev(ibdev);
1378         struct mlx5_core_dev *mdev;
1379         bool put_mdev = true;
1380
1381         mdev = mlx5_ib_get_native_port_mdev(dev, port, NULL);
1382         if (!mdev) {
1383             /* If the port isn't affiliated yet query the master.
1384              * The master and slave will have the same values.
1385              */
1386             mdev = dev->mdev;
1387             port = 1;
1388             put_mdev = false;
1389         }
1390         count = mlx5_core_reserved_gids_count(mdev);
1391         if (put_mdev)
1392             mlx5_ib_put_native_port_mdev(dev, port);
1393         props->gid_tbl_len -= count;
1394     }
1395     return ret;
1396 }
1397
1398 static int mlx5_ib_rep_query_port(struct ib_device *ibdev, u32 port,
1399                   struct ib_port_attr *props)
1400 {
1401     return mlx5_query_port_roce(ibdev, port, props);
1402 }
1403
1404 static int mlx5_ib_rep_query_pkey(struct ib_device *ibdev, u32 port, u16 index,
1405                   u16 *pkey)
1406 {
1407     /* Default special Pkey for representor device port as per the
1408      * IB specification 1.3 section 10.9.1.2.
1409      */
1410     *pkey = 0xffff;
1411     return 0;
1412 }
1413
1414 static int mlx5_ib_query_gid(struct ib_device *ibdev, u32 port, int index,
1415                  union ib_gid *gid)
1416 {
1417     struct mlx5_ib_dev *dev = to_mdev(ibdev);
1418     struct mlx5_core_dev *mdev = dev->mdev;
1419
1420     switch (mlx5_get_vport_access_method(ibdev)) {
1421     case MLX5_VPORT_ACCESS_METHOD_MAD:
1422         return mlx5_query_mad_ifc_gids(ibdev, port, index, gid);
1423
1424     case MLX5_VPORT_ACCESS_METHOD_HCA:
1425         return mlx5_query_hca_vport_gid(mdev, 0, port, 0, index, gid);
1426
1427     default:
1428         return -EINVAL;
1429     }
1430
1431 }
1432
1433 static int mlx5_query_hca_nic_pkey(struct ib_device *ibdev, u32 port,
1434                    u16 index, u16 *pkey)
1435 {
1436     struct mlx5_ib_dev *dev = to_mdev(ibdev);
1437     struct mlx5_core_dev *mdev;
1438     bool put_mdev = true;
1439     u32 mdev_port_num;
1440     int err;
1441
1442     mdev = mlx5_ib_get_native_port_mdev(dev, port, &mdev_port_num);
1443     if (!mdev) {
1444         /* The port isn't affiliated yet, get the PKey from the master
1445          * port. For RoCE the PKey tables will be the same.
1446          */
1447         put_mdev = false;
1448         mdev = dev->mdev;
1449         mdev_port_num = 1;
1450     }
1451
1452     err = mlx5_query_hca_vport_pkey(mdev, 0, mdev_port_num, 0,
1453                     index, pkey);
1454     if (put_mdev)
1455         mlx5_ib_put_native_port_mdev(dev, port);
1456
1457     return err;
1458 }
1459
1460 static int mlx5_ib_query_pkey(struct ib_device *ibdev, u32 port, u16 index,
1461                   u16 *pkey)
1462 {
1463     switch (mlx5_get_vport_access_method(ibdev)) {
1464     case MLX5_VPORT_ACCESS_METHOD_MAD:
1465         return mlx5_query_mad_ifc_pkey(ibdev, port, index, pkey);
1466
1467     case MLX5_VPORT_ACCESS_METHOD_HCA:
1468     case MLX5_VPORT_ACCESS_METHOD_NIC:
1469         return mlx5_query_hca_nic_pkey(ibdev, port, index, pkey);
1470     default:
1471         return -EINVAL;
1472     }
1473 }
1474
1475 static int mlx5_ib_modify_device(struct ib_device *ibdev, int mask,
1476                  struct ib_device_modify *props)
1477 {
1478     struct mlx5_ib_dev *dev = to_mdev(ibdev);
1479     struct mlx5_reg_node_desc in;
1480     struct mlx5_reg_node_desc out;
1481     int err;
1482
1483     if (mask & ~IB_DEVICE_MODIFY_NODE_DESC)
1484         return -EOPNOTSUPP;
1485
1486     if (!(mask & IB_DEVICE_MODIFY_NODE_DESC))
1487         return 0;
1488
1489     /*
1490      * If possible, pass node desc to FW, so it can generate
1491      * a 144 trap.  If cmd fails, just ignore.
1492      */
1493     memcpy(&in, props->node_desc, IB_DEVICE_NODE_DESC_MAX);
1494     err = mlx5_core_access_reg(dev->mdev, &in, sizeof(in), &out,
1495                    sizeof(out), MLX5_REG_NODE_DESC, 0, 1);
1496     if (err)
1497         return err;
1498
1499     memcpy(ibdev->node_desc, props->node_desc, IB_DEVICE_NODE_DESC_MAX);
1500
1501     return err;
1502 }
1503
1504 static int set_port_caps_atomic(struct mlx5_ib_dev *dev, u32 port_num, u32 mask,
1505                 u32 value)
1506 {
1507     struct mlx5_hca_vport_context ctx = {};
1508     struct mlx5_core_dev *mdev;
1509     u32 mdev_port_num;
1510     int err;
1511
1512     mdev = mlx5_ib_get_native_port_mdev(dev, port_num, &mdev_port_num);
1513     if (!mdev)
1514         return -ENODEV;
1515
1516     err = mlx5_query_hca_vport_context(mdev, 0, mdev_port_num, 0, &ctx);
1517     if (err)
1518         goto out;
1519
1520     if (~ctx.cap_mask1_perm & mask) {
1521         mlx5_ib_warn(dev, "trying to change bitmask 0x%X but change supported 0x%X\n",
1522                  mask, ctx.cap_mask1_perm);
1523         err = -EINVAL;
1524         goto out;
1525     }
1526
1527     ctx.cap_mask1 = value;
1528     ctx.cap_mask1_perm = mask;
1529     err = mlx5_core_modify_hca_vport_context(mdev, 0, mdev_port_num,
1530                          0, &ctx);
1531
1532 out:
1533     mlx5_ib_put_native_port_mdev(dev, port_num);
1534
1535     return err;
1536 }
1537
1538 static int mlx5_ib_modify_port(struct ib_device *ibdev, u32 port, int mask,
1539                    struct ib_port_modify *props)
1540 {
1541     struct mlx5_ib_dev *dev = to_mdev(ibdev);
1542     struct ib_port_attr attr;
1543     u32 tmp;
1544     int err;
1545     u32 change_mask;
1546     u32 value;
1547     bool is_ib = (mlx5_ib_port_link_layer(ibdev, port) ==
1548               IB_LINK_LAYER_INFINIBAND);
1549
1550     /* CM layer calls ib_modify_port() regardless of the link layer. For
1551      * Ethernet ports, qkey violation and Port capabilities are meaningless.
1552      */
1553     if (!is_ib)
1554         return 0;
1555
1556     if (MLX5_CAP_GEN(dev->mdev, ib_virt) && is_ib) {
1557         change_mask = props->clr_port_cap_mask | props->set_port_cap_mask;
1558         value = ~props->clr_port_cap_mask | props->set_port_cap_mask;
1559         return set_port_caps_atomic(dev, port, change_mask, value);
1560     }
1561
1562     mutex_lock(&dev->cap_mask_mutex);
1563
1564     err = ib_query_port(ibdev, port, &attr);
1565     if (err)
1566         goto out;
1567
1568     tmp = (attr.port_cap_flags | props->set_port_cap_mask) &
1569         ~props->clr_port_cap_mask;
1570
1571     err = mlx5_set_port_caps(dev->mdev, port, tmp);
1572
1573 out:
1574     mutex_unlock(&dev->cap_mask_mutex);
1575     return err;
1576 }
1577
1578 static void print_lib_caps(struct mlx5_ib_dev *dev, u64 caps)
1579 {
1580     mlx5_ib_dbg(dev, "MLX5_LIB_CAP_4K_UAR = %s\n",
1581             caps & MLX5_LIB_CAP_4K_UAR ? "y" : "n");
1582 }
1583
1584 static u16 calc_dynamic_bfregs(int uars_per_sys_page)
1585 {
1586     /* Large page with non 4k uar support might limit the dynamic size */
1587     if (uars_per_sys_page == 1  && PAGE_SIZE > 4096)
1588         return MLX5_MIN_DYN_BFREGS;
1589
1590     return MLX5_MAX_DYN_BFREGS;
1591 }
1592
1593 static int calc_total_bfregs(struct mlx5_ib_dev *dev, bool lib_uar_4k,
1594                  struct mlx5_ib_alloc_ucontext_req_v2 *req,
1595                  struct mlx5_bfreg_info *bfregi)
1596 {
1597     int uars_per_sys_page;
1598     int bfregs_per_sys_page;
1599     int ref_bfregs = req->total_num_bfregs;
1600
1601     if (req->total_num_bfregs == 0)
1602         return -EINVAL;
1603
1604     BUILD_BUG_ON(MLX5_MAX_BFREGS % MLX5_NON_FP_BFREGS_IN_PAGE);
1605     BUILD_BUG_ON(MLX5_MAX_BFREGS < MLX5_NON_FP_BFREGS_IN_PAGE);
1606
1607     if (req->total_num_bfregs > MLX5_MAX_BFREGS)
1608         return -ENOMEM;
1609
1610     uars_per_sys_page = get_uars_per_sys_page(dev, lib_uar_4k);
1611     bfregs_per_sys_page = uars_per_sys_page * MLX5_NON_FP_BFREGS_PER_UAR;
1612     /* This holds the required static allocation asked by the user */
1613     req->total_num_bfregs = ALIGN(req->total_num_bfregs, bfregs_per_sys_page);
1614     if (req->num_low_latency_bfregs > req->total_num_bfregs - 1)
1615         return -EINVAL;
1616
1617     bfregi->num_static_sys_pages = req->total_num_bfregs / bfregs_per_sys_page;
1618     bfregi->num_dyn_bfregs = ALIGN(calc_dynamic_bfregs(uars_per_sys_page), bfregs_per_sys_page);
1619     bfregi->total_num_bfregs = req->total_num_bfregs + bfregi->num_dyn_bfregs;
1620     bfregi->num_sys_pages = bfregi->total_num_bfregs / bfregs_per_sys_page;
1621
1622     mlx5_ib_dbg(dev, "uar_4k: fw support %s, lib support %s, user requested %d bfregs, allocated %d, total bfregs %d, using %d sys pages\n",
1623             MLX5_CAP_GEN(dev->mdev, uar_4k) ? "yes" : "no",
1624             lib_uar_4k ? "yes" : "no", ref_bfregs,
1625             req->total_num_bfregs, bfregi->total_num_bfregs,
1626             bfregi->num_sys_pages);
1627
1628     return 0;
1629 }
1630
1631 static int allocate_uars(struct mlx5_ib_dev *dev, struct mlx5_ib_ucontext *context)
1632 {
1633     struct mlx5_bfreg_info *bfregi;
1634     int err;
1635     int i;
1636
1637     bfregi = &context->bfregi;
1638     for (i = 0; i < bfregi->num_static_sys_pages; i++) {
1639         err = mlx5_cmd_uar_alloc(dev->mdev, &bfregi->sys_pages[i],
1640                      context->devx_uid);
1641         if (err)
1642             goto error;
1643
1644         mlx5_ib_dbg(dev, "allocated uar %d\n", bfregi->sys_pages[i]);
1645     }
1646
1647     for (i = bfregi->num_static_sys_pages; i < bfregi->num_sys_pages; i++)
1648         bfregi->sys_pages[i] = MLX5_IB_INVALID_UAR_INDEX;
1649
1650     return 0;
1651
1652 error:
1653     for (--i; i >= 0; i--)
1654         if (mlx5_cmd_uar_dealloc(dev->mdev, bfregi->sys_pages[i],
1655                      context->devx_uid))
1656             mlx5_ib_warn(dev, "failed to free uar %d\n", i);
1657
1658     return err;
1659 }
1660
1661 static void deallocate_uars(struct mlx5_ib_dev *dev,
1662                 struct mlx5_ib_ucontext *context)
1663 {
1664     struct mlx5_bfreg_info *bfregi;
1665     int i;
1666
1667     bfregi = &context->bfregi;
1668     for (i = 0; i < bfregi->num_sys_pages; i++)
1669         if (i < bfregi->num_static_sys_pages ||
1670             bfregi->sys_pages[i] != MLX5_IB_INVALID_UAR_INDEX)
1671             mlx5_cmd_uar_dealloc(dev->mdev, bfregi->sys_pages[i],
1672                          context->devx_uid);
1673 }
1674
1675 int mlx5_ib_enable_lb(struct mlx5_ib_dev *dev, bool td, bool qp)
1676 {
1677     int err = 0;
1678
1679     mutex_lock(&dev->lb.mutex);
1680     if (td)
1681         dev->lb.user_td++;
1682     if (qp)
1683         dev->lb.qps++;
1684
1685     if (dev->lb.user_td == 2 ||
1686         dev->lb.qps == 1) {
1687         if (!dev->lb.enabled) {
1688             err = mlx5_nic_vport_update_local_lb(dev->mdev, true);
1689             dev->lb.enabled = true;
1690         }
1691     }
1692
1693     mutex_unlock(&dev->lb.mutex);
1694
1695     return err;
1696 }
1697
1698 void mlx5_ib_disable_lb(struct mlx5_ib_dev *dev, bool td, bool qp)
1699 {
1700     mutex_lock(&dev->lb.mutex);
1701     if (td)
1702         dev->lb.user_td--;
1703     if (qp)
1704         dev->lb.qps--;
1705
1706     if (dev->lb.user_td == 1 &&
1707         dev->lb.qps == 0) {
1708         if (dev->lb.enabled) {
1709             mlx5_nic_vport_update_local_lb(dev->mdev, false);
1710             dev->lb.enabled = false;
1711         }
1712     }
1713
1714     mutex_unlock(&dev->lb.mutex);
1715 }
1716
1717 static int mlx5_ib_alloc_transport_domain(struct mlx5_ib_dev *dev, u32 *tdn,
1718                       u16 uid)
1719 {
1720     int err;
1721
1722     if (!MLX5_CAP_GEN(dev->mdev, log_max_transport_domain))
1723         return 0;
1724
1725     err = mlx5_cmd_alloc_transport_domain(dev->mdev, tdn, uid);
1726     if (err)
1727         return err;
1728
1729     if ((MLX5_CAP_GEN(dev->mdev, port_type) != MLX5_CAP_PORT_TYPE_ETH) ||
1730         (!MLX5_CAP_GEN(dev->mdev, disable_local_lb_uc) &&
1731          !MLX5_CAP_GEN(dev->mdev, disable_local_lb_mc)))
1732         return err;
1733
1734     return mlx5_ib_enable_lb(dev, true, false);
1735 }
1736
1737 static void mlx5_ib_dealloc_transport_domain(struct mlx5_ib_dev *dev, u32 tdn,
1738                          u16 uid)
1739 {
1740     if (!MLX5_CAP_GEN(dev->mdev, log_max_transport_domain))
1741         return;
1742
1743     mlx5_cmd_dealloc_transport_domain(dev->mdev, tdn, uid);
1744
1745     if ((MLX5_CAP_GEN(dev->mdev, port_type) != MLX5_CAP_PORT_TYPE_ETH) ||
1746         (!MLX5_CAP_GEN(dev->mdev, disable_local_lb_uc) &&
1747          !MLX5_CAP_GEN(dev->mdev, disable_local_lb_mc)))
1748         return;
1749
1750     mlx5_ib_disable_lb(dev, true, false);
1751 }
1752
1753 static int set_ucontext_resp(struct ib_ucontext *uctx,
1754                  struct mlx5_ib_alloc_ucontext_resp *resp)
1755 {
1756     struct ib_device *ibdev = uctx->device;
1757     struct mlx5_ib_dev *dev = to_mdev(ibdev);
1758     struct mlx5_ib_ucontext *context = to_mucontext(uctx);
1759     struct mlx5_bfreg_info *bfregi = &context->bfregi;
1760     int err;
1761
1762     if (MLX5_CAP_GEN(dev->mdev, dump_fill_mkey)) {
1763         err = mlx5_cmd_dump_fill_mkey(dev->mdev,
1764                           &resp->dump_fill_mkey);
1765         if (err)
1766             return err;
1767         resp->comp_mask |=
1768             MLX5_IB_ALLOC_UCONTEXT_RESP_MASK_DUMP_FILL_MKEY;
1769     }
1770
1771     resp->qp_tab_size = 1 << MLX5_CAP_GEN(dev->mdev, log_max_qp);
1772     if (dev->wc_support)
1773         resp->bf_reg_size = 1 << MLX5_CAP_GEN(dev->mdev,
1774                               log_bf_reg_size);
1775     resp->cache_line_size = cache_line_size();
1776     resp->max_sq_desc_sz = MLX5_CAP_GEN(dev->mdev, max_wqe_sz_sq);
1777     resp->max_rq_desc_sz = MLX5_CAP_GEN(dev->mdev, max_wqe_sz_rq);
1778     resp->max_send_wqebb = 1 << MLX5_CAP_GEN(dev->mdev, log_max_qp_sz);
1779     resp->max_recv_wr = 1 << MLX5_CAP_GEN(dev->mdev, log_max_qp_sz);
1780     resp->max_srq_recv_wr = 1 << MLX5_CAP_GEN(dev->mdev, log_max_srq_sz);
1781     resp->cqe_version = context->cqe_version;
1782     resp->log_uar_size = MLX5_CAP_GEN(dev->mdev, uar_4k) ?
1783                 MLX5_ADAPTER_PAGE_SHIFT : PAGE_SHIFT;
1784     resp->num_uars_per_page = MLX5_CAP_GEN(dev->mdev, uar_4k) ?
1785                     MLX5_CAP_GEN(dev->mdev,
1786                              num_of_uars_per_page) : 1;
1787     resp->tot_bfregs = bfregi->lib_uar_dyn ? 0 :
1788             bfregi->total_num_bfregs - bfregi->num_dyn_bfregs;
1789     resp->num_ports = dev->num_ports;
1790     resp->cmds_supp_uhw |= MLX5_USER_CMDS_SUPP_UHW_QUERY_DEVICE |
1791                       MLX5_USER_CMDS_SUPP_UHW_CREATE_AH;
1792
1793     if (mlx5_ib_port_link_layer(ibdev, 1) == IB_LINK_LAYER_ETHERNET) {
1794         mlx5_query_min_inline(dev->mdev, &resp->eth_min_inline);
1795         resp->eth_min_inline++;
1796     }
1797
1798     if (dev->mdev->clock_info)
1799         resp->clock_info_versions = BIT(MLX5_IB_CLOCK_INFO_V1);
1800
1801     /*
1802      * We don't want to expose information from the PCI bar that is located
1803      * after 4096 bytes, so if the arch only supports larger pages, let's
1804      * pretend we don't support reading the HCA's core clock. This is also
1805      * forced by mmap function.
1806      */
1807     if (PAGE_SIZE <= 4096) {
1808         resp->comp_mask |=
1809             MLX5_IB_ALLOC_UCONTEXT_RESP_MASK_CORE_CLOCK_OFFSET;
1810         resp->hca_core_clock_offset =
1811             offsetof(struct mlx5_init_seg,
1812                  internal_timer_h) % PAGE_SIZE;
1813     }
1814
1815     if (MLX5_CAP_GEN(dev->mdev, ece_support))
1816         resp->comp_mask |= MLX5_IB_ALLOC_UCONTEXT_RESP_MASK_ECE;
1817
1818     if (rt_supported(MLX5_CAP_GEN(dev->mdev, sq_ts_format)) &&
1819         rt_supported(MLX5_CAP_GEN(dev->mdev, rq_ts_format)) &&
1820         rt_supported(MLX5_CAP_ROCE(dev->mdev, qp_ts_format)))
1821         resp->comp_mask |=
1822             MLX5_IB_ALLOC_UCONTEXT_RESP_MASK_REAL_TIME_TS;
1823
1824     resp->num_dyn_bfregs = bfregi->num_dyn_bfregs;
1825
1826     if (MLX5_CAP_GEN(dev->mdev, drain_sigerr))
1827         resp->comp_mask |= MLX5_IB_ALLOC_UCONTEXT_RESP_MASK_SQD2RTS;
1828
1829     return 0;
1830 }
1831
1832 static int mlx5_ib_alloc_ucontext(struct ib_ucontext *uctx,
1833                   struct ib_udata *udata)
1834 {
1835     struct ib_device *ibdev = uctx->device;
1836     struct mlx5_ib_dev *dev = to_mdev(ibdev);
1837     struct mlx5_ib_alloc_ucontext_req_v2 req = {};
1838     struct mlx5_ib_alloc_ucontext_resp resp = {};
1839     struct mlx5_ib_ucontext *context = to_mucontext(uctx);
1840     struct mlx5_bfreg_info *bfregi;
1841     int ver;
1842     int err;
1843     size_t min_req_v2 = offsetof(struct mlx5_ib_alloc_ucontext_req_v2,
1844                      max_cqe_version);
1845     bool lib_uar_4k;
1846     bool lib_uar_dyn;
1847
1848     if (!dev->ib_active)
1849         return -EAGAIN;
1850
1851     if (udata->inlen == sizeof(struct mlx5_ib_alloc_ucontext_req))
1852         ver = 0;
1853     else if (udata->inlen >= min_req_v2)
1854         ver = 2;
1855     else
1856         return -EINVAL;
1857
1858     err = ib_copy_from_udata(&req, udata, min(udata->inlen, sizeof(req)));
1859     if (err)
1860         return err;
1861
1862     if (req.flags & ~MLX5_IB_ALLOC_UCTX_DEVX)
1863         return -EOPNOTSUPP;
1864
1865     if (req.comp_mask || req.reserved0 || req.reserved1 || req.reserved2)
1866         return -EOPNOTSUPP;
1867
1868     req.total_num_bfregs = ALIGN(req.total_num_bfregs,
1869                     MLX5_NON_FP_BFREGS_PER_UAR);
1870     if (req.num_low_latency_bfregs > req.total_num_bfregs - 1)
1871         return -EINVAL;
1872
1873     if (req.flags & MLX5_IB_ALLOC_UCTX_DEVX) {
1874         err = mlx5_ib_devx_create(dev, true);
1875         if (err < 0)
1876             goto out_ctx;
1877         context->devx_uid = err;
1878     }
1879
1880     lib_uar_4k = req.lib_caps & MLX5_LIB_CAP_4K_UAR;
1881     lib_uar_dyn = req.lib_caps & MLX5_LIB_CAP_DYN_UAR;
1882     bfregi = &context->bfregi;
1883
1884     if (lib_uar_dyn) {
1885         bfregi->lib_uar_dyn = lib_uar_dyn;
1886         goto uar_done;
1887     }
1888
1889     /* updates req->total_num_bfregs */
1890     err = calc_total_bfregs(dev, lib_uar_4k, &req, bfregi);
1891     if (err)
1892         goto out_devx;
1893
1894     mutex_init(&bfregi->lock);
1895     bfregi->lib_uar_4k = lib_uar_4k;
1896     bfregi->count = kcalloc(bfregi->total_num_bfregs, sizeof(*bfregi->count),
1897                 GFP_KERNEL);
1898     if (!bfregi->count) {
1899         err = -ENOMEM;
1900         goto out_devx;
1901     }
1902
1903     bfregi->sys_pages = kcalloc(bfregi->num_sys_pages,
1904                     sizeof(*bfregi->sys_pages),
1905                     GFP_KERNEL);
1906     if (!bfregi->sys_pages) {
1907         err = -ENOMEM;
1908         goto out_count;
1909     }
1910
1911     err = allocate_uars(dev, context);
1912     if (err)
1913         goto out_sys_pages;
1914
1915 uar_done:
1916     err = mlx5_ib_alloc_transport_domain(dev, &context->tdn,
1917                          context->devx_uid);
1918     if (err)
1919         goto out_uars;
1920
1921     INIT_LIST_HEAD(&context->db_page_list);
1922     mutex_init(&context->db_page_mutex);
1923
1924     context->cqe_version = min_t(__u8,
1925                  (__u8)MLX5_CAP_GEN(dev->mdev, cqe_version),
1926                  req.max_cqe_version);
1927
1928     err = set_ucontext_resp(uctx, &resp);
1929     if (err)
1930         goto out_mdev;
1931
1932     resp.response_length = min(udata->outlen, sizeof(resp));
1933     err = ib_copy_to_udata(udata, &resp, resp.response_length);
1934     if (err)
1935         goto out_mdev;
1936
1937     bfregi->ver = ver;
1938     bfregi->num_low_latency_bfregs = req.num_low_latency_bfregs;
1939     context->lib_caps = req.lib_caps;
1940     print_lib_caps(dev, context->lib_caps);
1941
1942     if (mlx5_ib_lag_should_assign_affinity(dev)) {
1943         u32 port = mlx5_core_native_port_num(dev->mdev) - 1;
1944
1945         atomic_set(&context->tx_port_affinity,
1946                atomic_add_return(
1947                    1, &dev->port[port].roce.tx_port_affinity));
1948     }
1949
1950     return 0;
1951
1952 out_mdev:
1953     mlx5_ib_dealloc_transport_domain(dev, context->tdn, context->devx_uid);
1954
1955 out_uars:
1956     deallocate_uars(dev, context);
1957
1958 out_sys_pages:
1959     kfree(bfregi->sys_pages);
1960
1961 out_count:
1962     kfree(bfregi->count);
1963
1964 out_devx:
1965     if (req.flags & MLX5_IB_ALLOC_UCTX_DEVX)
1966         mlx5_ib_devx_destroy(dev, context->devx_uid);
1967
1968 out_ctx:
1969     return err;
1970 }
1971
1972 static int mlx5_ib_query_ucontext(struct ib_ucontext *ibcontext,
1973                   struct uverbs_attr_bundle *attrs)
1974 {
1975     struct mlx5_ib_alloc_ucontext_resp uctx_resp = {};
1976     int ret;
1977
1978     ret = set_ucontext_resp(ibcontext, &uctx_resp);
1979     if (ret)
1980         return ret;
1981
1982     uctx_resp.response_length =
1983         min_t(size_t,
1984               uverbs_attr_get_len(attrs,
1985                 MLX5_IB_ATTR_QUERY_CONTEXT_RESP_UCTX),
1986               sizeof(uctx_resp));
1987
1988     ret = uverbs_copy_to_struct_or_zero(attrs,
1989                     MLX5_IB_ATTR_QUERY_CONTEXT_RESP_UCTX,
1990                     &uctx_resp,
1991                     sizeof(uctx_resp));
1992     return ret;
1993 }
1994
1995 static void mlx5_ib_dealloc_ucontext(struct ib_ucontext *ibcontext)
1996 {
1997     struct mlx5_ib_ucontext *context = to_mucontext(ibcontext);
1998     struct mlx5_ib_dev *dev = to_mdev(ibcontext->device);
1999     struct mlx5_bfreg_info *bfregi;
2000
2001     bfregi = &context->bfregi;
2002     mlx5_ib_dealloc_transport_domain(dev, context->tdn, context->devx_uid);
2003
2004     deallocate_uars(dev, context);
2005     kfree(bfregi->sys_pages);
2006     kfree(bfregi->count);
2007
2008     if (context->devx_uid)
2009         mlx5_ib_devx_destroy(dev, context->devx_uid);
2010 }
2011
2012 static phys_addr_t uar_index2pfn(struct mlx5_ib_dev *dev,
2013                  int uar_idx)
2014 {
2015     int fw_uars_per_page;
2016
2017     fw_uars_per_page = MLX5_CAP_GEN(dev->mdev, uar_4k) ? MLX5_UARS_IN_PAGE : 1;
2018
2019     return (dev->mdev->bar_addr >> PAGE_SHIFT) + uar_idx / fw_uars_per_page;
2020 }
2021
2022 static u64 uar_index2paddress(struct mlx5_ib_dev *dev,
2023                  int uar_idx)
2024 {
2025     unsigned int fw_uars_per_page;
2026
2027     fw_uars_per_page = MLX5_CAP_GEN(dev->mdev, uar_4k) ?
2028                 MLX5_UARS_IN_PAGE : 1;
2029
2030     return (dev->mdev->bar_addr + (uar_idx / fw_uars_per_page) * PAGE_SIZE);
2031 }
2032
2033 static int get_command(unsigned long offset)
2034 {
2035     return (offset >> MLX5_IB_MMAP_CMD_SHIFT) & MLX5_IB_MMAP_CMD_MASK;
2036 }
2037
2038 static int get_arg(unsigned long offset)
2039 {
2040     return offset & ((1 << MLX5_IB_MMAP_CMD_SHIFT) - 1);
2041 }
2042
2043 static int get_index(unsigned long offset)
2044 {
2045     return get_arg(offset);
2046 }
2047
2048 /* Index resides in an extra byte to enable larger values than 255 */
2049 static int get_extended_index(unsigned long offset)
2050 {
2051     return get_arg(offset) | ((offset >> 16) & 0xff) << 8;
2052 }
2053
2054
2055 static void mlx5_ib_disassociate_ucontext(struct ib_ucontext *ibcontext)
2056 {
2057 }
2058
2059 static inline char *mmap_cmd2str(enum mlx5_ib_mmap_cmd cmd)
2060 {
2061     switch (cmd) {
2062     case MLX5_IB_MMAP_WC_PAGE:
2063         return "WC";
2064     case MLX5_IB_MMAP_REGULAR_PAGE:
2065         return "best effort WC";
2066     case MLX5_IB_MMAP_NC_PAGE:
2067         return "NC";
2068     case MLX5_IB_MMAP_DEVICE_MEM:
2069         return "Device Memory";
2070     default:
2071         return NULL;
2072     }
2073 }
2074
2075 static int mlx5_ib_mmap_clock_info_page(struct mlx5_ib_dev *dev,
2076                     struct vm_area_struct *vma,
2077                     struct mlx5_ib_ucontext *context)
2078 {
2079     if ((vma->vm_end - vma->vm_start != PAGE_SIZE) ||
2080         !(vma->vm_flags & VM_SHARED))
2081         return -EINVAL;
2082
2083     if (get_index(vma->vm_pgoff) != MLX5_IB_CLOCK_INFO_V1)
2084         return -EOPNOTSUPP;
2085
2086     if (vma->vm_flags & (VM_WRITE | VM_EXEC))
2087         return -EPERM;
2088     vma->vm_flags &= ~VM_MAYWRITE;
2089
2090     if (!dev->mdev->clock_info)
2091         return -EOPNOTSUPP;
2092
2093     return vm_insert_page(vma, vma->vm_start,
2094                   virt_to_page(dev->mdev->clock_info));
2095 }
2096
2097 static void mlx5_ib_mmap_free(struct rdma_user_mmap_entry *entry)
2098 {
2099     struct mlx5_user_mmap_entry *mentry = to_mmmap(entry);
2100     struct mlx5_ib_dev *dev = to_mdev(entry->ucontext->device);
2101     struct mlx5_var_table *var_table = &dev->var_table;
2102     struct mlx5_ib_ucontext *context = to_mucontext(entry->ucontext);
2103
2104     switch (mentry->mmap_flag) {
2105     case MLX5_IB_MMAP_TYPE_MEMIC:
2106     case MLX5_IB_MMAP_TYPE_MEMIC_OP:
2107         mlx5_ib_dm_mmap_free(dev, mentry);
2108         break;
2109     case MLX5_IB_MMAP_TYPE_VAR:
2110         mutex_lock(&var_table->bitmap_lock);
2111         clear_bit(mentry->page_idx, var_table->bitmap);
2112         mutex_unlock(&var_table->bitmap_lock);
2113         kfree(mentry);
2114         break;
2115     case MLX5_IB_MMAP_TYPE_UAR_WC:
2116     case MLX5_IB_MMAP_TYPE_UAR_NC:
2117         mlx5_cmd_uar_dealloc(dev->mdev, mentry->page_idx,
2118                      context->devx_uid);
2119         kfree(mentry);
2120         break;
2121     default:
2122         WARN_ON(true);
2123     }
2124 }
2125
2126 static int uar_mmap(struct mlx5_ib_dev *dev, enum mlx5_ib_mmap_cmd cmd,
2127             struct vm_area_struct *vma,
2128             struct mlx5_ib_ucontext *context)
2129 {
2130     struct mlx5_bfreg_info *bfregi = &context->bfregi;
2131     int err;
2132     unsigned long idx;
2133     phys_addr_t pfn;
2134     pgprot_t prot;
2135     u32 bfreg_dyn_idx = 0;
2136     u32 uar_index;
2137     int dyn_uar = (cmd == MLX5_IB_MMAP_ALLOC_WC);
2138     int max_valid_idx = dyn_uar ? bfregi->num_sys_pages :
2139                 bfregi->num_static_sys_pages;
2140
2141     if (bfregi->lib_uar_dyn)
2142         return -EINVAL;
2143
2144     if (vma->vm_end - vma->vm_start != PAGE_SIZE)
2145         return -EINVAL;
2146
2147     if (dyn_uar)
2148         idx = get_extended_index(vma->vm_pgoff) + bfregi->num_static_sys_pages;
2149     else
2150         idx = get_index(vma->vm_pgoff);
2151
2152     if (idx >= max_valid_idx) {
2153         mlx5_ib_warn(dev, "invalid uar index %lu, max=%d\n",
2154                  idx, max_valid_idx);
2155         return -EINVAL;
2156     }
2157
2158     switch (cmd) {
2159     case MLX5_IB_MMAP_WC_PAGE:
2160     case MLX5_IB_MMAP_ALLOC_WC:
2161     case MLX5_IB_MMAP_REGULAR_PAGE:
2162         /* For MLX5_IB_MMAP_REGULAR_PAGE do the best effort to get WC */
2163         prot = pgprot_writecombine(vma->vm_page_prot);
2164         break;
2165     case MLX5_IB_MMAP_NC_PAGE:
2166         prot = pgprot_noncached(vma->vm_page_prot);
2167         break;
2168     default:
2169         return -EINVAL;
2170     }
2171
2172     if (dyn_uar) {
2173         int uars_per_page;
2174
2175         uars_per_page = get_uars_per_sys_page(dev, bfregi->lib_uar_4k);
2176         bfreg_dyn_idx = idx * (uars_per_page * MLX5_NON_FP_BFREGS_PER_UAR);
2177         if (bfreg_dyn_idx >= bfregi->total_num_bfregs) {
2178             mlx5_ib_warn(dev, "invalid bfreg_dyn_idx %u, max=%u\n",
2179                      bfreg_dyn_idx, bfregi->total_num_bfregs);
2180             return -EINVAL;
2181         }
2182
2183         mutex_lock(&bfregi->lock);
2184         /* Fail if uar already allocated, first bfreg index of each
2185          * page holds its count.
2186          */
2187         if (bfregi->count[bfreg_dyn_idx]) {
2188             mlx5_ib_warn(dev, "wrong offset, idx %lu is busy, bfregn=%u\n", idx, bfreg_dyn_idx);
2189             mutex_unlock(&bfregi->lock);
2190             return -EINVAL;
2191         }
2192
2193         bfregi->count[bfreg_dyn_idx]++;
2194         mutex_unlock(&bfregi->lock);
2195
2196         err = mlx5_cmd_uar_alloc(dev->mdev, &uar_index,
2197                      context->devx_uid);
2198         if (err) {
2199             mlx5_ib_warn(dev, "UAR alloc failed\n");
2200             goto free_bfreg;
2201         }
2202     } else {
2203         uar_index = bfregi->sys_pages[idx];
2204     }
2205
2206     pfn = uar_index2pfn(dev, uar_index);
2207     mlx5_ib_dbg(dev, "uar idx 0x%lx, pfn %pa\n", idx, &pfn);
2208
2209     err = rdma_user_mmap_io(&context->ibucontext, vma, pfn, PAGE_SIZE,
2210                 prot, NULL);
2211     if (err) {
2212         mlx5_ib_err(dev,
2213                 "rdma_user_mmap_io failed with error=%d, mmap_cmd=%s\n",
2214                 err, mmap_cmd2str(cmd));
2215         goto err;
2216     }
2217
2218     if (dyn_uar)
2219         bfregi->sys_pages[idx] = uar_index;
2220     return 0;
2221
2222 err:
2223     if (!dyn_uar)
2224         return err;
2225
2226     mlx5_cmd_uar_dealloc(dev->mdev, idx, context->devx_uid);
2227
2228 free_bfreg:
2229     mlx5_ib_free_bfreg(dev, bfregi, bfreg_dyn_idx);
2230
2231     return err;
2232 }
2233
2234 static unsigned long mlx5_vma_to_pgoff(struct vm_area_struct *vma)
2235 {
2236     unsigned long idx;
2237     u8 command;
2238
2239     command = get_command(vma->vm_pgoff);
2240     idx = get_extended_index(vma->vm_pgoff);
2241
2242     return (command << 16 | idx);
2243 }
2244
2245 static int mlx5_ib_mmap_offset(struct mlx5_ib_dev *dev,
2246                    struct vm_area_struct *vma,
2247                    struct ib_ucontext *ucontext)
2248 {
2249     struct mlx5_user_mmap_entry *mentry;
2250     struct rdma_user_mmap_entry *entry;
2251     unsigned long pgoff;
2252     pgprot_t prot;
2253     phys_addr_t pfn;
2254     int ret;
2255
2256     pgoff = mlx5_vma_to_pgoff(vma);
2257     entry = rdma_user_mmap_entry_get_pgoff(ucontext, pgoff);
2258     if (!entry)
2259         return -EINVAL;
2260
2261     mentry = to_mmmap(entry);
2262     pfn = (mentry->address >> PAGE_SHIFT);
2263     if (mentry->mmap_flag == MLX5_IB_MMAP_TYPE_VAR ||
2264         mentry->mmap_flag == MLX5_IB_MMAP_TYPE_UAR_NC)
2265         prot = pgprot_noncached(vma->vm_page_prot);
2266     else
2267         prot = pgprot_writecombine(vma->vm_page_prot);
2268     ret = rdma_user_mmap_io(ucontext, vma, pfn,
2269                 entry->npages * PAGE_SIZE,
2270                 prot,
2271                 entry);
2272     rdma_user_mmap_entry_put(&mentry->rdma_entry);
2273     return ret;
2274 }
2275
2276 static u64 mlx5_entry_to_mmap_offset(struct mlx5_user_mmap_entry *entry)
2277 {
2278     u64 cmd = (entry->rdma_entry.start_pgoff >> 16) & 0xFFFF;
2279     u64 index = entry->rdma_entry.start_pgoff & 0xFFFF;
2280
2281     return (((index >> 8) << 16) | (cmd << MLX5_IB_MMAP_CMD_SHIFT) |
2282         (index & 0xFF)) << PAGE_SHIFT;
2283 }
2284
2285 static int mlx5_ib_mmap(struct ib_ucontext *ibcontext, struct vm_area_struct *vma)
2286 {
2287     struct mlx5_ib_ucontext *context = to_mucontext(ibcontext);
2288     struct mlx5_ib_dev *dev = to_mdev(ibcontext->device);
2289     unsigned long command;
2290     phys_addr_t pfn;
2291
2292     command = get_command(vma->vm_pgoff);
2293     switch (command) {
2294     case MLX5_IB_MMAP_WC_PAGE:
2295     case MLX5_IB_MMAP_ALLOC_WC:
2296         if (!dev->wc_support)
2297             return -EPERM;
2298         fallthrough;
2299     case MLX5_IB_MMAP_NC_PAGE:
2300     case MLX5_IB_MMAP_REGULAR_PAGE:
2301         return uar_mmap(dev, command, vma, context);
2302
2303     case MLX5_IB_MMAP_GET_CONTIGUOUS_PAGES:
2304         return -ENOSYS;
2305
2306     case MLX5_IB_MMAP_CORE_CLOCK:
2307         if (vma->vm_end - vma->vm_start != PAGE_SIZE)
2308             return -EINVAL;
2309
2310         if (vma->vm_flags & VM_WRITE)
2311             return -EPERM;
2312         vma->vm_flags &= ~VM_MAYWRITE;
2313
2314         /* Don't expose to user-space information it shouldn't have */
2315         if (PAGE_SIZE > 4096)
2316             return -EOPNOTSUPP;
2317
2318         pfn = (dev->mdev->iseg_base +
2319                offsetof(struct mlx5_init_seg, internal_timer_h)) >>
2320             PAGE_SHIFT;
2321         return rdma_user_mmap_io(&context->ibucontext, vma, pfn,
2322                      PAGE_SIZE,
2323                      pgprot_noncached(vma->vm_page_prot),
2324                      NULL);
2325     case MLX5_IB_MMAP_CLOCK_INFO:
2326         return mlx5_ib_mmap_clock_info_page(dev, vma, context);
2327
2328     default:
2329         return mlx5_ib_mmap_offset(dev, vma, ibcontext);
2330     }
2331
2332     return 0;
2333 }
2334
2335 static int mlx5_ib_alloc_pd(struct ib_pd *ibpd, struct ib_udata *udata)
2336 {
2337     struct mlx5_ib_pd *pd = to_mpd(ibpd);
2338     struct ib_device *ibdev = ibpd->device;
2339     struct mlx5_ib_alloc_pd_resp resp;
2340     int err;
2341     u32 out[MLX5_ST_SZ_DW(alloc_pd_out)] = {};
2342     u32 in[MLX5_ST_SZ_DW(alloc_pd_in)] = {};
2343     u16 uid = 0;
2344     struct mlx5_ib_ucontext *context = rdma_udata_to_drv_context(
2345         udata, struct mlx5_ib_ucontext, ibucontext);
2346
2347     uid = context ? context->devx_uid : 0;
2348     MLX5_SET(alloc_pd_in, in, opcode, MLX5_CMD_OP_ALLOC_PD);
2349     MLX5_SET(alloc_pd_in, in, uid, uid);
2350     err = mlx5_cmd_exec_inout(to_mdev(ibdev)->mdev, alloc_pd, in, out);
2351     if (err)
2352         return err;
2353
2354     pd->pdn = MLX5_GET(alloc_pd_out, out, pd);
2355     pd->uid = uid;
2356     if (udata) {
2357         resp.pdn = pd->pdn;
2358         if (ib_copy_to_udata(udata, &resp, sizeof(resp))) {
2359             mlx5_cmd_dealloc_pd(to_mdev(ibdev)->mdev, pd->pdn, uid);
2360             return -EFAULT;
2361         }
2362     }
2363
2364     return 0;
2365 }
2366
2367 static int mlx5_ib_dealloc_pd(struct ib_pd *pd, struct ib_udata *udata)
2368 {
2369     struct mlx5_ib_dev *mdev = to_mdev(pd->device);
2370     struct mlx5_ib_pd *mpd = to_mpd(pd);
2371
2372     return mlx5_cmd_dealloc_pd(mdev->mdev, mpd->pdn, mpd->uid);
2373 }
2374
2375 static int mlx5_ib_mcg_attach(struct ib_qp *ibqp, union ib_gid *gid, u16 lid)
2376 {
2377     struct mlx5_ib_dev *dev = to_mdev(ibqp->device);
2378     struct mlx5_ib_qp *mqp = to_mqp(ibqp);
2379     int err;
2380     u16 uid;
2381
2382     uid = ibqp->pd ?
2383         to_mpd(ibqp->pd)->uid : 0;
2384
2385     if (mqp->flags & IB_QP_CREATE_SOURCE_QPN) {
2386         mlx5_ib_dbg(dev, "Attaching a multi cast group to underlay QP is not supported\n");
2387         return -EOPNOTSUPP;
2388     }
2389
2390     err = mlx5_cmd_attach_mcg(dev->mdev, gid, ibqp->qp_num, uid);
2391     if (err)
2392         mlx5_ib_warn(dev, "failed attaching QPN 0x%x, MGID %pI6\n",
2393                  ibqp->qp_num, gid->raw);
2394
2395     return err;
2396 }
2397
2398 static int mlx5_ib_mcg_detach(struct ib_qp *ibqp, union ib_gid *gid, u16 lid)
2399 {
2400     struct mlx5_ib_dev *dev = to_mdev(ibqp->device);
2401     int err;
2402     u16 uid;
2403
2404     uid = ibqp->pd ?
2405         to_mpd(ibqp->pd)->uid : 0;
2406     err = mlx5_cmd_detach_mcg(dev->mdev, gid, ibqp->qp_num, uid);
2407     if (err)
2408         mlx5_ib_warn(dev, "failed detaching QPN 0x%x, MGID %pI6\n",
2409                  ibqp->qp_num, gid->raw);
2410
2411     return err;
2412 }
2413
2414 static int init_node_data(struct mlx5_ib_dev *dev)
2415 {
2416     int err;
2417
2418     err = mlx5_query_node_desc(dev, dev->ib_dev.node_desc);
2419     if (err)
2420         return err;
2421
2422     dev->mdev->rev_id = dev->mdev->pdev->revision;
2423
2424     return mlx5_query_node_guid(dev, &dev->ib_dev.node_guid);
2425 }
2426
2427 static ssize_t fw_pages_show(struct device *device,
2428                  struct device_attribute *attr, char *buf)
2429 {
2430     struct mlx5_ib_dev *dev =
2431         rdma_device_to_drv_device(device, struct mlx5_ib_dev, ib_dev);
2432
2433     return sysfs_emit(buf, "%d\n", dev->mdev->priv.fw_pages);
2434 }
2435 static DEVICE_ATTR_RO(fw_pages);
2436
2437 static ssize_t reg_pages_show(struct device *device,
2438                   struct device_attribute *attr, char *buf)
2439 {
2440     struct mlx5_ib_dev *dev =
2441         rdma_device_to_drv_device(device, struct mlx5_ib_dev, ib_dev);
2442
2443     return sysfs_emit(buf, "%d\n", atomic_read(&dev->mdev->priv.reg_pages));
2444 }
2445 static DEVICE_ATTR_RO(reg_pages);
2446
2447 static ssize_t hca_type_show(struct device *device,
2448                  struct device_attribute *attr, char *buf)
2449 {
2450     struct mlx5_ib_dev *dev =
2451         rdma_device_to_drv_device(device, struct mlx5_ib_dev, ib_dev);
2452
2453     return sysfs_emit(buf, "MT%d\n", dev->mdev->pdev->device);
2454 }
2455 static DEVICE_ATTR_RO(hca_type);
2456
2457 static ssize_t hw_rev_show(struct device *device,
2458                struct device_attribute *attr, char *buf)
2459 {
2460     struct mlx5_ib_dev *dev =
2461         rdma_device_to_drv_device(device, struct mlx5_ib_dev, ib_dev);
2462
2463     return sysfs_emit(buf, "%x\n", dev->mdev->rev_id);
2464 }
2465 static DEVICE_ATTR_RO(hw_rev);
2466
2467 static ssize_t board_id_show(struct device *device,
2468                  struct device_attribute *attr, char *buf)
2469 {
2470     struct mlx5_ib_dev *dev =
2471         rdma_device_to_drv_device(device, struct mlx5_ib_dev, ib_dev);
2472
2473     return sysfs_emit(buf, "%.*s\n", MLX5_BOARD_ID_LEN,
2474               dev->mdev->board_id);
2475 }
2476 static DEVICE_ATTR_RO(board_id);
2477
2478 static struct attribute *mlx5_class_attributes[] = {
2479     &dev_attr_hw_rev.attr,
2480     &dev_attr_hca_type.attr,
2481     &dev_attr_board_id.attr,
2482     &dev_attr_fw_pages.attr,
2483     &dev_attr_reg_pages.attr,
2484     NULL,
2485 };
2486
2487 static const struct attribute_group mlx5_attr_group = {
2488     .attrs = mlx5_class_attributes,
2489 };
2490
2491 static void pkey_change_handler(struct work_struct *work)
2492 {
2493     struct mlx5_ib_port_resources *ports =
2494         container_of(work, struct mlx5_ib_port_resources,
2495                  pkey_change_work);
2496
2497     if (!ports->gsi)
2498         /*
2499          * We got this event before device was fully configured
2500          * and MAD registration code wasn't called/finished yet.
2501          */
2502         return;
2503
2504     mlx5_ib_gsi_pkey_change(ports->gsi);
2505 }
2506
2507 static void mlx5_ib_handle_internal_error(struct mlx5_ib_dev *ibdev)
2508 {
2509     struct mlx5_ib_qp *mqp;
2510     struct mlx5_ib_cq *send_mcq, *recv_mcq;
2511     struct mlx5_core_cq *mcq;
2512     struct list_head cq_armed_list;
2513     unsigned long flags_qp;
2514     unsigned long flags_cq;
2515     unsigned long flags;
2516
2517     INIT_LIST_HEAD(&cq_armed_list);
2518
2519     /* Go over qp list reside on that ibdev, sync with create/destroy qp.*/
2520     spin_lock_irqsave(&ibdev->reset_flow_resource_lock, flags);
2521     list_for_each_entry(mqp, &ibdev->qp_list, qps_list) {
2522         spin_lock_irqsave(&mqp->sq.lock, flags_qp);
2523         if (mqp->sq.tail != mqp->sq.head) {
2524             send_mcq = to_mcq(mqp->ibqp.send_cq);
2525             spin_lock_irqsave(&send_mcq->lock, flags_cq);
2526             if (send_mcq->mcq.comp &&
2527                 mqp->ibqp.send_cq->comp_handler) {
2528                 if (!send_mcq->mcq.reset_notify_added) {
2529                     send_mcq->mcq.reset_notify_added = 1;
2530                     list_add_tail(&send_mcq->mcq.reset_notify,
2531                               &cq_armed_list);
2532                 }
2533             }
2534             spin_unlock_irqrestore(&send_mcq->lock, flags_cq);
2535         }
2536         spin_unlock_irqrestore(&mqp->sq.lock, flags_qp);
2537         spin_lock_irqsave(&mqp->rq.lock, flags_qp);
2538         /* no handling is needed for SRQ */
2539         if (!mqp->ibqp.srq) {
2540             if (mqp->rq.tail != mqp->rq.head) {
2541                 recv_mcq = to_mcq(mqp->ibqp.recv_cq);
2542                 spin_lock_irqsave(&recv_mcq->lock, flags_cq);
2543                 if (recv_mcq->mcq.comp &&
2544                     mqp->ibqp.recv_cq->comp_handler) {
2545                     if (!recv_mcq->mcq.reset_notify_added) {
2546                         recv_mcq->mcq.reset_notify_added = 1;
2547                         list_add_tail(&recv_mcq->mcq.reset_notify,
2548                                   &cq_armed_list);
2549                     }
2550                 }
2551                 spin_unlock_irqrestore(&recv_mcq->lock,
2552                                flags_cq);
2553             }
2554         }
2555         spin_unlock_irqrestore(&mqp->rq.lock, flags_qp);
2556     }
2557     /*At that point all inflight post send were put to be executed as of we
2558      * lock/unlock above locks Now need to arm all involved CQs.
2559      */
2560     list_for_each_entry(mcq, &cq_armed_list, reset_notify) {
2561         mcq->comp(mcq, NULL);
2562     }
2563     spin_unlock_irqrestore(&ibdev->reset_flow_resource_lock, flags);
2564 }
2565
2566 static void delay_drop_handler(struct work_struct *work)
2567 {
2568     int err;
2569     struct mlx5_ib_delay_drop *delay_drop =
2570         container_of(work, struct mlx5_ib_delay_drop,
2571                  delay_drop_work);
2572
2573     atomic_inc(&delay_drop->events_cnt);
2574
2575     mutex_lock(&delay_drop->lock);
2576     err = mlx5_core_set_delay_drop(delay_drop->dev, delay_drop->timeout);
2577     if (err) {
2578         mlx5_ib_warn(delay_drop->dev, "Failed to set delay drop, timeout=%u\n",
2579                  delay_drop->timeout);
2580         delay_drop->activate = false;
2581     }
2582     mutex_unlock(&delay_drop->lock);
2583 }
2584
2585 static void handle_general_event(struct mlx5_ib_dev *ibdev, struct mlx5_eqe *eqe,
2586                  struct ib_event *ibev)
2587 {
2588     u32 port = (eqe->data.port.port >> 4) & 0xf;
2589
2590     switch (eqe->sub_type) {
2591     case MLX5_GENERAL_SUBTYPE_DELAY_DROP_TIMEOUT:
2592         if (mlx5_ib_port_link_layer(&ibdev->ib_dev, port) ==
2593                         IB_LINK_LAYER_ETHERNET)
2594             schedule_work(&ibdev->delay_drop.delay_drop_work);
2595         break;
2596     default: /* do nothing */
2597         return;
2598     }
2599 }
2600
2601 static int handle_port_change(struct mlx5_ib_dev *ibdev, struct mlx5_eqe *eqe,
2602                   struct ib_event *ibev)
2603 {
2604     u32 port = (eqe->data.port.port >> 4) & 0xf;
2605
2606     ibev->element.port_num = port;
2607
2608     switch (eqe->sub_type) {
2609     case MLX5_PORT_CHANGE_SUBTYPE_ACTIVE:
2610     case MLX5_PORT_CHANGE_SUBTYPE_DOWN:
2611     case MLX5_PORT_CHANGE_SUBTYPE_INITIALIZED:
2612         /* In RoCE, port up/down events are handled in
2613          * mlx5_netdev_event().
2614          */
2615         if (mlx5_ib_port_link_layer(&ibdev->ib_dev, port) ==
2616                         IB_LINK_LAYER_ETHERNET)
2617             return -EINVAL;
2618
2619         ibev->event = (eqe->sub_type == MLX5_PORT_CHANGE_SUBTYPE_ACTIVE) ?
2620                 IB_EVENT_PORT_ACTIVE : IB_EVENT_PORT_ERR;
2621         break;
2622
2623     case MLX5_PORT_CHANGE_SUBTYPE_LID:
2624         ibev->event = IB_EVENT_LID_CHANGE;
2625         break;
2626
2627     case MLX5_PORT_CHANGE_SUBTYPE_PKEY:
2628         ibev->event = IB_EVENT_PKEY_CHANGE;
2629         schedule_work(&ibdev->devr.ports[port - 1].pkey_change_work);
2630         break;
2631
2632     case MLX5_PORT_CHANGE_SUBTYPE_GUID:
2633         ibev->event = IB_EVENT_GID_CHANGE;
2634         break;
2635
2636     case MLX5_PORT_CHANGE_SUBTYPE_CLIENT_REREG:
2637         ibev->event = IB_EVENT_CLIENT_REREGISTER;
2638         break;
2639     default:
2640         return -EINVAL;
2641     }
2642
2643     return 0;
2644 }
2645
2646 static void mlx5_ib_handle_event(struct work_struct *_work)
2647 {
2648     struct mlx5_ib_event_work *work =
2649         container_of(_work, struct mlx5_ib_event_work, work);
2650     struct mlx5_ib_dev *ibdev;
2651     struct ib_event ibev;
2652     bool fatal = false;
2653
2654     if (work->is_slave) {
2655         ibdev = mlx5_ib_get_ibdev_from_mpi(work->mpi);
2656         if (!ibdev)
2657             goto out;
2658     } else {
2659         ibdev = work->dev;
2660     }
2661
2662     switch (work->event) {
2663     case MLX5_DEV_EVENT_SYS_ERROR:
2664         ibev.event = IB_EVENT_DEVICE_FATAL;
2665         mlx5_ib_handle_internal_error(ibdev);
2666         ibev.element.port_num  = (u8)(unsigned long)work->param;
2667         fatal = true;
2668         break;
2669     case MLX5_EVENT_TYPE_PORT_CHANGE:
2670         if (handle_port_change(ibdev, work->param, &ibev))
2671             goto out;
2672         break;
2673     case MLX5_EVENT_TYPE_GENERAL_EVENT:
2674         handle_general_event(ibdev, work->param, &ibev);
2675         fallthrough;
2676     default:
2677         goto out;
2678     }
2679
2680     ibev.device = &ibdev->ib_dev;
2681
2682     if (!rdma_is_port_valid(&ibdev->ib_dev, ibev.element.port_num)) {
2683         mlx5_ib_warn(ibdev, "warning: event on port %d\n",  ibev.element.port_num);
2684         goto out;
2685     }
2686
2687     if (ibdev->ib_active)
2688         ib_dispatch_event(&ibev);
2689
2690     if (fatal)
2691         ibdev->ib_active = false;
2692 out:
2693     kfree(work);
2694 }
2695
2696 static int mlx5_ib_event(struct notifier_block *nb,
2697              unsigned long event, void *param)
2698 {
2699     struct mlx5_ib_event_work *work;
2700
2701     work = kmalloc(sizeof(*work), GFP_ATOMIC);
2702     if (!work)
2703         return NOTIFY_DONE;
2704
2705     INIT_WORK(&work->work, mlx5_ib_handle_event);
2706     work->dev = container_of(nb, struct mlx5_ib_dev, mdev_events);
2707     work->is_slave = false;
2708     work->param = param;
2709     work->event = event;
2710
2711     queue_work(mlx5_ib_event_wq, &work->work);
2712
2713     return NOTIFY_OK;
2714 }
2715
2716 static int mlx5_ib_event_slave_port(struct notifier_block *nb,
2717                     unsigned long event, void *param)
2718 {
2719     struct mlx5_ib_event_work *work;
2720
2721     work = kmalloc(sizeof(*work), GFP_ATOMIC);
2722     if (!work)
2723         return NOTIFY_DONE;
2724
2725     INIT_WORK(&work->work, mlx5_ib_handle_event);
2726     work->mpi = container_of(nb, struct mlx5_ib_multiport_info, mdev_events);
2727     work->is_slave = true;
2728     work->param = param;
2729     work->event = event;
2730     queue_work(mlx5_ib_event_wq, &work->work);
2731
2732     return NOTIFY_OK;
2733 }
2734
2735 static int set_has_smi_cap(struct mlx5_ib_dev *dev)
2736 {
2737     struct mlx5_hca_vport_context vport_ctx;
2738     int err;
2739     int port;
2740
2741     if (MLX5_CAP_GEN(dev->mdev, port_type) != MLX5_CAP_PORT_TYPE_IB)
2742         return 0;
2743
2744     for (port = 1; port <= dev->num_ports; port++) {
2745         if (!MLX5_CAP_GEN(dev->mdev, ib_virt)) {
2746             dev->port_caps[port - 1].has_smi = true;
2747             continue;
2748         }
2749         err = mlx5_query_hca_vport_context(dev->mdev, 0, port, 0,
2750                            &vport_ctx);
2751         if (err) {
2752             mlx5_ib_err(dev, "query_hca_vport_context for port=%d failed %d\n",
2753                     port, err);
2754             return err;
2755         }
2756         dev->port_caps[port - 1].has_smi = vport_ctx.has_smi;
2757     }
2758
2759     return 0;
2760 }
2761
2762 static void get_ext_port_caps(struct mlx5_ib_dev *dev)
2763 {
2764     unsigned int port;
2765
2766     rdma_for_each_port (&dev->ib_dev, port)
2767         mlx5_query_ext_port_caps(dev, port);
2768 }
2769
2770 static u8 mlx5_get_umr_fence(u8 umr_fence_cap)
2771 {
2772     switch (umr_fence_cap) {
2773     case MLX5_CAP_UMR_FENCE_NONE:
2774         return MLX5_FENCE_MODE_NONE;
2775     case MLX5_CAP_UMR_FENCE_SMALL:
2776         return MLX5_FENCE_MODE_INITIATOR_SMALL;
2777     default:
2778         return MLX5_FENCE_MODE_STRONG_ORDERING;
2779     }
2780 }
2781
2782 static int mlx5_ib_dev_res_init(struct mlx5_ib_dev *dev)
2783 {
2784     struct mlx5_ib_resources *devr = &dev->devr;
2785     struct ib_srq_init_attr attr;
2786     struct ib_device *ibdev;
2787     struct ib_cq_init_attr cq_attr = {.cqe = 1};
2788     int port;
2789     int ret = 0;
2790
2791     ibdev = &dev->ib_dev;
2792
2793     if (!MLX5_CAP_GEN(dev->mdev, xrc))
2794         return -EOPNOTSUPP;
2795
2796     devr->p0 = ib_alloc_pd(ibdev, 0);
2797     if (IS_ERR(devr->p0))
2798         return PTR_ERR(devr->p0);
2799
2800     devr->c0 = ib_create_cq(ibdev, NULL, NULL, NULL, &cq_attr);
2801     if (IS_ERR(devr->c0)) {
2802         ret = PTR_ERR(devr->c0);
2803         goto error1;
2804     }
2805
2806     ret = mlx5_cmd_xrcd_alloc(dev->mdev, &devr->xrcdn0, 0);
2807     if (ret)
2808         goto error2;
2809
2810     ret = mlx5_cmd_xrcd_alloc(dev->mdev, &devr->xrcdn1, 0);
2811     if (ret)
2812         goto error3;
2813
2814     memset(&attr, 0, sizeof(attr));
2815     attr.attr.max_sge = 1;
2816     attr.attr.max_wr = 1;
2817     attr.srq_type = IB_SRQT_XRC;
2818     attr.ext.cq = devr->c0;
2819
2820     devr->s0 = ib_create_srq(devr->p0, &attr);
2821     if (IS_ERR(devr->s0)) {
2822         ret = PTR_ERR(devr->s0);
2823         goto err_create;
2824     }
2825
2826     memset(&attr, 0, sizeof(attr));
2827     attr.attr.max_sge = 1;
2828     attr.attr.max_wr = 1;
2829     attr.srq_type = IB_SRQT_BASIC;
2830
2831     devr->s1 = ib_create_srq(devr->p0, &attr);
2832     if (IS_ERR(devr->s1)) {
2833         ret = PTR_ERR(devr->s1);
2834         goto error6;
2835     }
2836
2837     for (port = 0; port < ARRAY_SIZE(devr->ports); ++port)
2838         INIT_WORK(&devr->ports[port].pkey_change_work,
2839               pkey_change_handler);
2840
2841     return 0;
2842
2843 error6:
2844     ib_destroy_srq(devr->s0);
2845 err_create:
2846     mlx5_cmd_xrcd_dealloc(dev->mdev, devr->xrcdn1, 0);
2847 error3:
2848     mlx5_cmd_xrcd_dealloc(dev->mdev, devr->xrcdn0, 0);
2849 error2:
2850     ib_destroy_cq(devr->c0);
2851 error1:
2852     ib_dealloc_pd(devr->p0);
2853     return ret;
2854 }
2855
2856 static void mlx5_ib_dev_res_cleanup(struct mlx5_ib_dev *dev)
2857 {
2858     struct mlx5_ib_resources *devr = &dev->devr;
2859     int port;
2860
2861     /*
2862      * Make sure no change P_Key work items are still executing.
2863      *
2864      * At this stage, the mlx5_ib_event should be unregistered
2865      * and it ensures that no new works are added.
2866      */
2867     for (port = 0; port < ARRAY_SIZE(devr->ports); ++port)
2868         cancel_work_sync(&devr->ports[port].pkey_change_work);
2869
2870     ib_destroy_srq(devr->s1);
2871     ib_destroy_srq(devr->s0);
2872     mlx5_cmd_xrcd_dealloc(dev->mdev, devr->xrcdn1, 0);
2873     mlx5_cmd_xrcd_dealloc(dev->mdev, devr->xrcdn0, 0);
2874     ib_destroy_cq(devr->c0);
2875     ib_dealloc_pd(devr->p0);
2876 }
2877
2878 static u32 get_core_cap_flags(struct ib_device *ibdev,
2879                   struct mlx5_hca_vport_context *rep)
2880 {
2881     struct mlx5_ib_dev *dev = to_mdev(ibdev);
2882     enum rdma_link_layer ll = mlx5_ib_port_link_layer(ibdev, 1);
2883     u8 l3_type_cap = MLX5_CAP_ROCE(dev->mdev, l3_type);
2884     u8 roce_version_cap = MLX5_CAP_ROCE(dev->mdev, roce_version);
2885     bool raw_support = !mlx5_core_mp_enabled(dev->mdev);
2886     u32 ret = 0;
2887
2888     if (rep->grh_required)
2889         ret |= RDMA_CORE_CAP_IB_GRH_REQUIRED;
2890
2891     if (ll == IB_LINK_LAYER_INFINIBAND)
2892         return ret | RDMA_CORE_PORT_IBA_IB;
2893
2894     if (raw_support)
2895         ret |= RDMA_CORE_PORT_RAW_PACKET;
2896
2897     if (!(l3_type_cap & MLX5_ROCE_L3_TYPE_IPV4_CAP))
2898         return ret;
2899
2900     if (!(l3_type_cap & MLX5_ROCE_L3_TYPE_IPV6_CAP))
2901         return ret;
2902
2903     if (roce_version_cap & MLX5_ROCE_VERSION_1_CAP)
2904         ret |= RDMA_CORE_PORT_IBA_ROCE;
2905
2906     if (roce_version_cap & MLX5_ROCE_VERSION_2_CAP)
2907         ret |= RDMA_CORE_PORT_IBA_ROCE_UDP_ENCAP;
2908
2909     return ret;
2910 }
2911
2912 static int mlx5_port_immutable(struct ib_device *ibdev, u32 port_num,
2913                    struct ib_port_immutable *immutable)
2914 {
2915     struct ib_port_attr attr;
2916     struct mlx5_ib_dev *dev = to_mdev(ibdev);
2917     enum rdma_link_layer ll = mlx5_ib_port_link_layer(ibdev, port_num);
2918     struct mlx5_hca_vport_context rep = {0};
2919     int err;
2920
2921     err = ib_query_port(ibdev, port_num, &attr);
2922     if (err)
2923         return err;
2924
2925     if (ll == IB_LINK_LAYER_INFINIBAND) {
2926         err = mlx5_query_hca_vport_context(dev->mdev, 0, port_num, 0,
2927                            &rep);
2928         if (err)
2929             return err;
2930     }
2931
2932     immutable->pkey_tbl_len = attr.pkey_tbl_len;
2933     immutable->gid_tbl_len = attr.gid_tbl_len;
2934     immutable->core_cap_flags = get_core_cap_flags(ibdev, &rep);
2935     immutable->max_mad_size = IB_MGMT_MAD_SIZE;
2936
2937     return 0;
2938 }
2939
2940 static int mlx5_port_rep_immutable(struct ib_device *ibdev, u32 port_num,
2941                    struct ib_port_immutable *immutable)
2942 {
2943     struct ib_port_attr attr;
2944     int err;
2945
2946     immutable->core_cap_flags = RDMA_CORE_PORT_RAW_PACKET;
2947
2948     err = ib_query_port(ibdev, port_num, &attr);
2949     if (err)
2950         return err;
2951
2952     immutable->pkey_tbl_len = attr.pkey_tbl_len;
2953     immutable->gid_tbl_len = attr.gid_tbl_len;
2954     immutable->core_cap_flags = RDMA_CORE_PORT_RAW_PACKET;
2955
2956     return 0;
2957 }
2958
2959 static void get_dev_fw_str(struct ib_device *ibdev, char *str)
2960 {
2961     struct mlx5_ib_dev *dev =
2962         container_of(ibdev, struct mlx5_ib_dev, ib_dev);
2963     snprintf(str, IB_FW_VERSION_NAME_MAX, "%d.%d.%04d",
2964          fw_rev_maj(dev->mdev), fw_rev_min(dev->mdev),
2965          fw_rev_sub(dev->mdev));
2966 }
2967
2968 static int mlx5_eth_lag_init(struct mlx5_ib_dev *dev)
2969 {
2970     struct mlx5_core_dev *mdev = dev->mdev;
2971     struct mlx5_flow_namespace *ns = mlx5_get_flow_namespace(mdev,
2972                                  MLX5_FLOW_NAMESPACE_LAG);
2973     struct mlx5_flow_table *ft;
2974     int err;
2975
2976     if (!ns || !mlx5_lag_is_active(mdev))
2977         return 0;
2978
2979     err = mlx5_cmd_create_vport_lag(mdev);
2980     if (err)
2981         return err;
2982
2983     ft = mlx5_create_lag_demux_flow_table(ns, 0, 0);
2984     if (IS_ERR(ft)) {
2985         err = PTR_ERR(ft);
2986         goto err_destroy_vport_lag;
2987     }
2988
2989     dev->flow_db->lag_demux_ft = ft;
2990     dev->lag_ports = mlx5_lag_get_num_ports(mdev);
2991     dev->lag_active = true;
2992     return 0;
2993
2994 err_destroy_vport_lag:
2995     mlx5_cmd_destroy_vport_lag(mdev);
2996     return err;
2997 }
2998
2999 static void mlx5_eth_lag_cleanup(struct mlx5_ib_dev *dev)
3000 {
3001     struct mlx5_core_dev *mdev = dev->mdev;
3002
3003     if (dev->lag_active) {
3004         dev->lag_active = false;
3005
3006         mlx5_destroy_flow_table(dev->flow_db->lag_demux_ft);
3007         dev->flow_db->lag_demux_ft = NULL;
3008
3009         mlx5_cmd_destroy_vport_lag(mdev);
3010     }
3011 }
3012
3013 static int mlx5_add_netdev_notifier(struct mlx5_ib_dev *dev, u32 port_num)
3014 {
3015     int err;
3016
3017     dev->port[port_num].roce.nb.notifier_call = mlx5_netdev_event;
3018     err = register_netdevice_notifier(&dev->port[port_num].roce.nb);
3019     if (err) {
3020         dev->port[port_num].roce.nb.notifier_call = NULL;
3021         return err;
3022     }
3023
3024     return 0;
3025 }
3026
3027 static void mlx5_remove_netdev_notifier(struct mlx5_ib_dev *dev, u32 port_num)
3028 {
3029     if (dev->port[port_num].roce.nb.notifier_call) {
3030         unregister_netdevice_notifier(&dev->port[port_num].roce.nb);
3031         dev->port[port_num].roce.nb.notifier_call = NULL;
3032     }
3033 }
3034
3035 static int mlx5_enable_eth(struct mlx5_ib_dev *dev)
3036 {
3037     int err;
3038
3039     if (!dev->is_rep && dev->profile != &raw_eth_profile) {
3040         err = mlx5_nic_vport_enable_roce(dev->mdev);
3041         if (err)
3042             return err;
3043     }
3044
3045     err = mlx5_eth_lag_init(dev);
3046     if (err)
3047         goto err_disable_roce;
3048
3049     return 0;
3050
3051 err_disable_roce:
3052     if (!dev->is_rep && dev->profile != &raw_eth_profile)
3053         mlx5_nic_vport_disable_roce(dev->mdev);
3054
3055     return err;
3056 }
3057
3058 static void mlx5_disable_eth(struct mlx5_ib_dev *dev)
3059 {
3060     mlx5_eth_lag_cleanup(dev);
3061     if (!dev->is_rep && dev->profile != &raw_eth_profile)
3062         mlx5_nic_vport_disable_roce(dev->mdev);
3063 }
3064
3065 static int mlx5_ib_rn_get_params(struct ib_device *device, u32 port_num,
3066                  enum rdma_netdev_t type,
3067                  struct rdma_netdev_alloc_params *params)
3068 {
3069     if (type != RDMA_NETDEV_IPOIB)
3070         return -EOPNOTSUPP;
3071
3072     return mlx5_rdma_rn_get_params(to_mdev(device)->mdev, device, params);
3073 }
3074
3075 static ssize_t delay_drop_timeout_read(struct file *filp, char __user *buf,
3076                        size_t count, loff_t *pos)
3077 {
3078     struct mlx5_ib_delay_drop *delay_drop = filp->private_data;
3079     char lbuf[20];
3080     int len;
3081
3082     len = snprintf(lbuf, sizeof(lbuf), "%u\n", delay_drop->timeout);
3083     return simple_read_from_buffer(buf, count, pos, lbuf, len);
3084 }
3085
3086 static ssize_t delay_drop_timeout_write(struct file *filp, const char __user *buf,
3087                     size_t count, loff_t *pos)
3088 {
3089     struct mlx5_ib_delay_drop *delay_drop = filp->private_data;
3090     u32 timeout;
3091     u32 var;
3092
3093     if (kstrtouint_from_user(buf, count, 0, &var))
3094         return -EFAULT;
3095
3096     timeout = min_t(u32, roundup(var, 100), MLX5_MAX_DELAY_DROP_TIMEOUT_MS *
3097             1000);
3098     if (timeout != var)
3099         mlx5_ib_dbg(delay_drop->dev, "Round delay drop timeout to %u usec\n",
3100                 timeout);
3101
3102     delay_drop->timeout = timeout;
3103
3104     return count;
3105 }
3106
3107 static const struct file_operations fops_delay_drop_timeout = {
3108     .owner  = THIS_MODULE,
3109     .open   = simple_open,
3110     .write  = delay_drop_timeout_write,
3111     .read   = delay_drop_timeout_read,
3112 };
3113
3114 static void mlx5_ib_unbind_slave_port(struct mlx5_ib_dev *ibdev,
3115                       struct mlx5_ib_multiport_info *mpi)
3116 {
3117     u32 port_num = mlx5_core_native_port_num(mpi->mdev) - 1;
3118     struct mlx5_ib_port *port = &ibdev->port[port_num];
3119     int comps;
3120     int err;
3121     int i;
3122
3123     lockdep_assert_held(&mlx5_ib_multiport_mutex);
3124
3125     mlx5_ib_cleanup_cong_debugfs(ibdev, port_num);
3126
3127     spin_lock(&port->mp.mpi_lock);
3128     if (!mpi->ibdev) {
3129         spin_unlock(&port->mp.mpi_lock);
3130         return;
3131     }
3132
3133     mpi->ibdev = NULL;
3134
3135     spin_unlock(&port->mp.mpi_lock);
3136     if (mpi->mdev_events.notifier_call)
3137         mlx5_notifier_unregister(mpi->mdev, &mpi->mdev_events);
3138     mpi->mdev_events.notifier_call = NULL;
3139     mlx5_remove_netdev_notifier(ibdev, port_num);
3140     spin_lock(&port->mp.mpi_lock);
3141
3142     comps = mpi->mdev_refcnt;
3143     if (comps) {
3144         mpi->unaffiliate = true;
3145         init_completion(&mpi->unref_comp);
3146         spin_unlock(&port->mp.mpi_lock);
3147
3148         for (i = 0; i < comps; i++)
3149             wait_for_completion(&mpi->unref_comp);
3150
3151         spin_lock(&port->mp.mpi_lock);
3152         mpi->unaffiliate = false;
3153     }
3154
3155     port->mp.mpi = NULL;
3156
3157     spin_unlock(&port->mp.mpi_lock);
3158
3159     err = mlx5_nic_vport_unaffiliate_multiport(mpi->mdev);
3160
3161     mlx5_ib_dbg(ibdev, "unaffiliated port %u\n", port_num + 1);
3162     /* Log an error, still needed to cleanup the pointers and add
3163      * it back to the list.
3164      */
3165     if (err)
3166         mlx5_ib_err(ibdev, "Failed to unaffiliate port %u\n",
3167                 port_num + 1);
3168
3169     ibdev->port[port_num].roce.last_port_state = IB_PORT_DOWN;
3170 }
3171
3172 static bool mlx5_ib_bind_slave_port(struct mlx5_ib_dev *ibdev,
3173                     struct mlx5_ib_multiport_info *mpi)
3174 {
3175     u32 port_num = mlx5_core_native_port_num(mpi->mdev) - 1;
3176     int err;
3177
3178     lockdep_assert_held(&mlx5_ib_multiport_mutex);
3179
3180     spin_lock(&ibdev->port[port_num].mp.mpi_lock);
3181     if (ibdev->port[port_num].mp.mpi) {
3182         mlx5_ib_dbg(ibdev, "port %u already affiliated.\n",
3183                 port_num + 1);
3184         spin_unlock(&ibdev->port[port_num].mp.mpi_lock);
3185         return false;
3186     }
3187
3188     ibdev->port[port_num].mp.mpi = mpi;
3189     mpi->ibdev = ibdev;
3190     mpi->mdev_events.notifier_call = NULL;
3191     spin_unlock(&ibdev->port[port_num].mp.mpi_lock);
3192
3193     err = mlx5_nic_vport_affiliate_multiport(ibdev->mdev, mpi->mdev);
3194     if (err)
3195         goto unbind;
3196
3197     err = mlx5_add_netdev_notifier(ibdev, port_num);
3198     if (err) {
3199         mlx5_ib_err(ibdev, "failed adding netdev notifier for port %u\n",
3200                 port_num + 1);
3201         goto unbind;
3202     }
3203
3204     mpi->mdev_events.notifier_call = mlx5_ib_event_slave_port;
3205     mlx5_notifier_register(mpi->mdev, &mpi->mdev_events);
3206
3207     mlx5_ib_init_cong_debugfs(ibdev, port_num);
3208
3209     return true;
3210
3211 unbind:
3212     mlx5_ib_unbind_slave_port(ibdev, mpi);
3213     return false;
3214 }
3215
3216 static int mlx5_ib_init_multiport_master(struct mlx5_ib_dev *dev)
3217 {
3218     u32 port_num = mlx5_core_native_port_num(dev->mdev) - 1;
3219     enum rdma_link_layer ll = mlx5_ib_port_link_layer(&dev->ib_dev,
3220                               port_num + 1);
3221     struct mlx5_ib_multiport_info *mpi;
3222     int err;
3223     u32 i;
3224
3225     if (!mlx5_core_is_mp_master(dev->mdev) || ll != IB_LINK_LAYER_ETHERNET)
3226         return 0;
3227
3228     err = mlx5_query_nic_vport_system_image_guid(dev->mdev,
3229                              &dev->sys_image_guid);
3230     if (err)
3231         return err;
3232
3233     err = mlx5_nic_vport_enable_roce(dev->mdev);
3234     if (err)
3235         return err;
3236
3237     mutex_lock(&mlx5_ib_multiport_mutex);
3238     for (i = 0; i < dev->num_ports; i++) {
3239         bool bound = false;
3240
3241         /* build a stub multiport info struct for the native port. */
3242         if (i == port_num) {
3243             mpi = kzalloc(sizeof(*mpi), GFP_KERNEL);
3244             if (!mpi) {
3245                 mutex_unlock(&mlx5_ib_multiport_mutex);
3246                 mlx5_nic_vport_disable_roce(dev->mdev);
3247                 return -ENOMEM;
3248             }
3249
3250             mpi->is_master = true;
3251             mpi->mdev = dev->mdev;
3252             mpi->sys_image_guid = dev->sys_image_guid;
3253             dev->port[i].mp.mpi = mpi;
3254             mpi->ibdev = dev;
3255             mpi = NULL;
3256             continue;
3257         }
3258
3259         list_for_each_entry(mpi, &mlx5_ib_unaffiliated_port_list,
3260                     list) {
3261             if (dev->sys_image_guid == mpi->sys_image_guid &&
3262                 (mlx5_core_native_port_num(mpi->mdev) - 1) == i) {
3263                 bound = mlx5_ib_bind_slave_port(dev, mpi);
3264             }
3265
3266             if (bound) {
3267                 dev_dbg(mpi->mdev->device,
3268                     "removing port from unaffiliated list.\n");
3269                 mlx5_ib_dbg(dev, "port %d bound\n", i + 1);
3270                 list_del(&mpi->list);
3271                 break;
3272             }
3273         }
3274         if (!bound)
3275             mlx5_ib_dbg(dev, "no free port found for port %d\n",
3276                     i + 1);
3277     }
3278
3279     list_add_tail(&dev->ib_dev_list, &mlx5_ib_dev_list);
3280     mutex_unlock(&mlx5_ib_multiport_mutex);
3281     return err;
3282 }
3283
3284 static void mlx5_ib_cleanup_multiport_master(struct mlx5_ib_dev *dev)
3285 {
3286     u32 port_num = mlx5_core_native_port_num(dev->mdev) - 1;
3287     enum rdma_link_layer ll = mlx5_ib_port_link_layer(&dev->ib_dev,
3288                               port_num + 1);
3289     u32 i;
3290
3291     if (!mlx5_core_is_mp_master(dev->mdev) || ll != IB_LINK_LAYER_ETHERNET)
3292         return;
3293
3294     mutex_lock(&mlx5_ib_multiport_mutex);
3295     for (i = 0; i < dev->num_ports; i++) {
3296         if (dev->port[i].mp.mpi) {
3297             /* Destroy the native port stub */
3298             if (i == port_num) {
3299                 kfree(dev->port[i].mp.mpi);
3300                 dev->port[i].mp.mpi = NULL;
3301             } else {
3302                 mlx5_ib_dbg(dev, "unbinding port_num: %u\n",
3303                         i + 1);
3304                 list_add_tail(&dev->port[i].mp.mpi->list,
3305                           &mlx5_ib_unaffiliated_port_list);
3306                 mlx5_ib_unbind_slave_port(dev,
3307                               dev->port[i].mp.mpi);
3308             }
3309         }
3310     }
3311
3312     mlx5_ib_dbg(dev, "removing from devlist\n");
3313     list_del(&dev->ib_dev_list);
3314     mutex_unlock(&mlx5_ib_multiport_mutex);
3315
3316     mlx5_nic_vport_disable_roce(dev->mdev);
3317 }
3318
3319 static int mmap_obj_cleanup(struct ib_uobject *uobject,
3320                 enum rdma_remove_reason why,
3321                 struct uverbs_attr_bundle *attrs)
3322 {
3323     struct mlx5_user_mmap_entry *obj = uobject->object;
3324
3325     rdma_user_mmap_entry_remove(&obj->rdma_entry);
3326     return 0;
3327 }
3328
3329 static int mlx5_rdma_user_mmap_entry_insert(struct mlx5_ib_ucontext *c,
3330                         struct mlx5_user_mmap_entry *entry,
3331                         size_t length)
3332 {
3333     return rdma_user_mmap_entry_insert_range(
3334         &c->ibucontext, &entry->rdma_entry, length,
3335         (MLX5_IB_MMAP_OFFSET_START << 16),
3336         ((MLX5_IB_MMAP_OFFSET_END << 16) + (1UL << 16) - 1));
3337 }
3338
3339 static struct mlx5_user_mmap_entry *
3340 alloc_var_entry(struct mlx5_ib_ucontext *c)
3341 {
3342     struct mlx5_user_mmap_entry *entry;
3343     struct mlx5_var_table *var_table;
3344     u32 page_idx;
3345     int err;
3346
3347     var_table = &to_mdev(c->ibucontext.device)->var_table;
3348     entry = kzalloc(sizeof(*entry), GFP_KERNEL);
3349     if (!entry)
3350         return ERR_PTR(-ENOMEM);
3351
3352     mutex_lock(&var_table->bitmap_lock);
3353     page_idx = find_first_zero_bit(var_table->bitmap,
3354                        var_table->num_var_hw_entries);
3355     if (page_idx >= var_table->num_var_hw_entries) {
3356         err = -ENOSPC;
3357         mutex_unlock(&var_table->bitmap_lock);
3358         goto end;
3359     }
3360
3361     set_bit(page_idx, var_table->bitmap);
3362     mutex_unlock(&var_table->bitmap_lock);
3363
3364     entry->address = var_table->hw_start_addr +
3365                 (page_idx * var_table->stride_size);
3366     entry->page_idx = page_idx;
3367     entry->mmap_flag = MLX5_IB_MMAP_TYPE_VAR;
3368
3369     err = mlx5_rdma_user_mmap_entry_insert(c, entry,
3370                            var_table->stride_size);
3371     if (err)
3372         goto err_insert;
3373
3374     return entry;
3375
3376 err_insert:
3377     mutex_lock(&var_table->bitmap_lock);
3378     clear_bit(page_idx, var_table->bitmap);
3379     mutex_unlock(&var_table->bitmap_lock);
3380 end:
3381     kfree(entry);
3382     return ERR_PTR(err);
3383 }
3384
3385 static int UVERBS_HANDLER(MLX5_IB_METHOD_VAR_OBJ_ALLOC)(
3386     struct uverbs_attr_bundle *attrs)
3387 {
3388     struct ib_uobject *uobj = uverbs_attr_get_uobject(
3389         attrs, MLX5_IB_ATTR_VAR_OBJ_ALLOC_HANDLE);
3390     struct mlx5_ib_ucontext *c;
3391     struct mlx5_user_mmap_entry *entry;
3392     u64 mmap_offset;
3393     u32 length;
3394     int err;
3395
3396     c = to_mucontext(ib_uverbs_get_ucontext(attrs));
3397     if (IS_ERR(c))
3398         return PTR_ERR(c);
3399
3400     entry = alloc_var_entry(c);
3401     if (IS_ERR(entry))
3402         return PTR_ERR(entry);
3403
3404     mmap_offset = mlx5_entry_to_mmap_offset(entry);
3405     length = entry->rdma_entry.npages * PAGE_SIZE;
3406     uobj->object = entry;
3407     uverbs_finalize_uobj_create(attrs, MLX5_IB_ATTR_VAR_OBJ_ALLOC_HANDLE);
3408
3409     err = uverbs_copy_to(attrs, MLX5_IB_ATTR_VAR_OBJ_ALLOC_MMAP_OFFSET,
3410                  &mmap_offset, sizeof(mmap_offset));
3411     if (err)
3412         return err;
3413
3414     err = uverbs_copy_to(attrs, MLX5_IB_ATTR_VAR_OBJ_ALLOC_PAGE_ID,
3415                  &entry->page_idx, sizeof(entry->page_idx));
3416     if (err)
3417         return err;
3418
3419     err = uverbs_copy_to(attrs, MLX5_IB_ATTR_VAR_OBJ_ALLOC_MMAP_LENGTH,
3420                  &length, sizeof(length));
3421     return err;
3422 }
3423
3424 DECLARE_UVERBS_NAMED_METHOD(
3425     MLX5_IB_METHOD_VAR_OBJ_ALLOC,
3426     UVERBS_ATTR_IDR(MLX5_IB_ATTR_VAR_OBJ_ALLOC_HANDLE,
3427             MLX5_IB_OBJECT_VAR,
3428             UVERBS_ACCESS_NEW,
3429             UA_MANDATORY),
3430     UVERBS_ATTR_PTR_OUT(MLX5_IB_ATTR_VAR_OBJ_ALLOC_PAGE_ID,
3431                UVERBS_ATTR_TYPE(u32),
3432                UA_MANDATORY),
3433     UVERBS_ATTR_PTR_OUT(MLX5_IB_ATTR_VAR_OBJ_ALLOC_MMAP_LENGTH,
3434                UVERBS_ATTR_TYPE(u32),
3435                UA_MANDATORY),
3436     UVERBS_ATTR_PTR_OUT(MLX5_IB_ATTR_VAR_OBJ_ALLOC_MMAP_OFFSET,
3437                 UVERBS_ATTR_TYPE(u64),
3438                 UA_MANDATORY));
3439
3440 DECLARE_UVERBS_NAMED_METHOD_DESTROY(
3441     MLX5_IB_METHOD_VAR_OBJ_DESTROY,
3442     UVERBS_ATTR_IDR(MLX5_IB_ATTR_VAR_OBJ_DESTROY_HANDLE,
3443             MLX5_IB_OBJECT_VAR,
3444             UVERBS_ACCESS_DESTROY,
3445             UA_MANDATORY));
3446
3447 DECLARE_UVERBS_NAMED_OBJECT(MLX5_IB_OBJECT_VAR,
3448                 UVERBS_TYPE_ALLOC_IDR(mmap_obj_cleanup),
3449                 &UVERBS_METHOD(MLX5_IB_METHOD_VAR_OBJ_ALLOC),
3450                 &UVERBS_METHOD(MLX5_IB_METHOD_VAR_OBJ_DESTROY));
3451
3452 static bool var_is_supported(struct ib_device *device)
3453 {
3454     struct mlx5_ib_dev *dev = to_mdev(device);
3455
3456     return (MLX5_CAP_GEN_64(dev->mdev, general_obj_types) &
3457             MLX5_GENERAL_OBJ_TYPES_CAP_VIRTIO_NET_Q);
3458 }
3459
3460 static struct mlx5_user_mmap_entry *
3461 alloc_uar_entry(struct mlx5_ib_ucontext *c,
3462         enum mlx5_ib_uapi_uar_alloc_type alloc_type)
3463 {
3464     struct mlx5_user_mmap_entry *entry;
3465     struct mlx5_ib_dev *dev;
3466     u32 uar_index;
3467     int err;
3468
3469     entry = kzalloc(sizeof(*entry), GFP_KERNEL);
3470     if (!entry)
3471         return ERR_PTR(-ENOMEM);
3472
3473     dev = to_mdev(c->ibucontext.device);
3474     err = mlx5_cmd_uar_alloc(dev->mdev, &uar_index, c->devx_uid);
3475     if (err)
3476         goto end;
3477
3478     entry->page_idx = uar_index;
3479     entry->address = uar_index2paddress(dev, uar_index);
3480     if (alloc_type == MLX5_IB_UAPI_UAR_ALLOC_TYPE_BF)
3481         entry->mmap_flag = MLX5_IB_MMAP_TYPE_UAR_WC;
3482     else
3483         entry->mmap_flag = MLX5_IB_MMAP_TYPE_UAR_NC;
3484
3485     err = mlx5_rdma_user_mmap_entry_insert(c, entry, PAGE_SIZE);
3486     if (err)
3487         goto err_insert;
3488
3489     return entry;
3490
3491 err_insert:
3492     mlx5_cmd_uar_dealloc(dev->mdev, uar_index, c->devx_uid);
3493 end:
3494     kfree(entry);
3495     return ERR_PTR(err);
3496 }
3497
3498 static int UVERBS_HANDLER(MLX5_IB_METHOD_UAR_OBJ_ALLOC)(
3499     struct uverbs_attr_bundle *attrs)
3500 {
3501     struct ib_uobject *uobj = uverbs_attr_get_uobject(
3502         attrs, MLX5_IB_ATTR_UAR_OBJ_ALLOC_HANDLE);
3503     enum mlx5_ib_uapi_uar_alloc_type alloc_type;
3504     struct mlx5_ib_ucontext *c;
3505     struct mlx5_user_mmap_entry *entry;
3506     u64 mmap_offset;
3507     u32 length;
3508     int err;
3509
3510     c = to_mucontext(ib_uverbs_get_ucontext(attrs));
3511     if (IS_ERR(c))
3512         return PTR_ERR(c);
3513
3514     err = uverbs_get_const(&alloc_type, attrs,
3515                    MLX5_IB_ATTR_UAR_OBJ_ALLOC_TYPE);
3516     if (err)
3517         return err;
3518
3519     if (alloc_type != MLX5_IB_UAPI_UAR_ALLOC_TYPE_BF &&
3520         alloc_type != MLX5_IB_UAPI_UAR_ALLOC_TYPE_NC)
3521         return -EOPNOTSUPP;
3522
3523     if (!to_mdev(c->ibucontext.device)->wc_support &&
3524         alloc_type == MLX5_IB_UAPI_UAR_ALLOC_TYPE_BF)
3525         return -EOPNOTSUPP;
3526
3527     entry = alloc_uar_entry(c, alloc_type);
3528     if (IS_ERR(entry))
3529         return PTR_ERR(entry);
3530
3531     mmap_offset = mlx5_entry_to_mmap_offset(entry);
3532     length = entry->rdma_entry.npages * PAGE_SIZE;
3533     uobj->object = entry;
3534     uverbs_finalize_uobj_create(attrs, MLX5_IB_ATTR_UAR_OBJ_ALLOC_HANDLE);
3535
3536     err = uverbs_copy_to(attrs, MLX5_IB_ATTR_UAR_OBJ_ALLOC_MMAP_OFFSET,
3537                  &mmap_offset, sizeof(mmap_offset));
3538     if (err)
3539         return err;
3540
3541     err = uverbs_copy_to(attrs, MLX5_IB_ATTR_UAR_OBJ_ALLOC_PAGE_ID,
3542                  &entry->page_idx, sizeof(entry->page_idx));
3543     if (err)
3544         return err;
3545
3546     err = uverbs_copy_to(attrs, MLX5_IB_ATTR_UAR_OBJ_ALLOC_MMAP_LENGTH,
3547                  &length, sizeof(length));
3548     return err;
3549 }
3550
3551 DECLARE_UVERBS_NAMED_METHOD(
3552     MLX5_IB_METHOD_UAR_OBJ_ALLOC,
3553     UVERBS_ATTR_IDR(MLX5_IB_ATTR_UAR_OBJ_ALLOC_HANDLE,
3554             MLX5_IB_OBJECT_UAR,
3555             UVERBS_ACCESS_NEW,
3556             UA_MANDATORY),
3557     UVERBS_ATTR_CONST_IN(MLX5_IB_ATTR_UAR_OBJ_ALLOC_TYPE,
3558                  enum mlx5_ib_uapi_uar_alloc_type,
3559                  UA_MANDATORY),
3560     UVERBS_ATTR_PTR_OUT(MLX5_IB_ATTR_UAR_OBJ_ALLOC_PAGE_ID,
3561                UVERBS_ATTR_TYPE(u32),
3562                UA_MANDATORY),
3563     UVERBS_ATTR_PTR_OUT(MLX5_IB_ATTR_UAR_OBJ_ALLOC_MMAP_LENGTH,
3564                UVERBS_ATTR_TYPE(u32),
3565                UA_MANDATORY),
3566     UVERBS_ATTR_PTR_OUT(MLX5_IB_ATTR_UAR_OBJ_ALLOC_MMAP_OFFSET,
3567                 UVERBS_ATTR_TYPE(u64),
3568                 UA_MANDATORY));
3569
3570 DECLARE_UVERBS_NAMED_METHOD_DESTROY(
3571     MLX5_IB_METHOD_UAR_OBJ_DESTROY,
3572     UVERBS_ATTR_IDR(MLX5_IB_ATTR_UAR_OBJ_DESTROY_HANDLE,
3573             MLX5_IB_OBJECT_UAR,
3574             UVERBS_ACCESS_DESTROY,
3575             UA_MANDATORY));
3576
3577 DECLARE_UVERBS_NAMED_OBJECT(MLX5_IB_OBJECT_UAR,
3578                 UVERBS_TYPE_ALLOC_IDR(mmap_obj_cleanup),
3579                 &UVERBS_METHOD(MLX5_IB_METHOD_UAR_OBJ_ALLOC),
3580                 &UVERBS_METHOD(MLX5_IB_METHOD_UAR_OBJ_DESTROY));
3581
3582 ADD_UVERBS_ATTRIBUTES_SIMPLE(
3583     mlx5_ib_query_context,
3584     UVERBS_OBJECT_DEVICE,
3585     UVERBS_METHOD_QUERY_CONTEXT,
3586     UVERBS_ATTR_PTR_OUT(
3587         MLX5_IB_ATTR_QUERY_CONTEXT_RESP_UCTX,
3588         UVERBS_ATTR_STRUCT(struct mlx5_ib_alloc_ucontext_resp,
3589                    dump_fill_mkey),
3590         UA_MANDATORY));
3591
3592 static const struct uapi_definition mlx5_ib_defs[] = {
3593     UAPI_DEF_CHAIN(mlx5_ib_devx_defs),
3594     UAPI_DEF_CHAIN(mlx5_ib_flow_defs),
3595     UAPI_DEF_CHAIN(mlx5_ib_qos_defs),
3596     UAPI_DEF_CHAIN(mlx5_ib_std_types_defs),
3597     UAPI_DEF_CHAIN(mlx5_ib_dm_defs),
3598
3599     UAPI_DEF_CHAIN_OBJ_TREE(UVERBS_OBJECT_DEVICE, &mlx5_ib_query_context),
3600     UAPI_DEF_CHAIN_OBJ_TREE_NAMED(MLX5_IB_OBJECT_VAR,
3601                 UAPI_DEF_IS_OBJ_SUPPORTED(var_is_supported)),
3602     UAPI_DEF_CHAIN_OBJ_TREE_NAMED(MLX5_IB_OBJECT_UAR),
3603     {}
3604 };
3605
3606 static void mlx5_ib_stage_init_cleanup(struct mlx5_ib_dev *dev)
3607 {
3608     mlx5_ib_cleanup_multiport_master(dev);
3609     WARN_ON(!xa_empty(&dev->odp_mkeys));
3610     mutex_destroy(&dev->cap_mask_mutex);
3611     WARN_ON(!xa_empty(&dev->sig_mrs));
3612     WARN_ON(!bitmap_empty(dev->dm.memic_alloc_pages, MLX5_MAX_MEMIC_PAGES));
3613 }
3614
3615 static int mlx5_ib_stage_init_init(struct mlx5_ib_dev *dev)
3616 {
3617     struct mlx5_core_dev *mdev = dev->mdev;
3618     int err;
3619     int i;
3620
3621     dev->ib_dev.node_type = RDMA_NODE_IB_CA;
3622     dev->ib_dev.local_dma_lkey = 0 /* not supported for now */;
3623     dev->ib_dev.phys_port_cnt = dev->num_ports;
3624     dev->ib_dev.dev.parent = mdev->device;
3625     dev->ib_dev.lag_flags = RDMA_LAG_FLAGS_HASH_ALL_SLAVES;
3626
3627     for (i = 0; i < dev->num_ports; i++) {
3628         spin_lock_init(&dev->port[i].mp.mpi_lock);
3629         rwlock_init(&dev->port[i].roce.netdev_lock);
3630         dev->port[i].roce.dev = dev;
3631         dev->port[i].roce.native_port_num = i + 1;
3632         dev->port[i].roce.last_port_state = IB_PORT_DOWN;
3633     }
3634
3635     err = mlx5_ib_init_multiport_master(dev);
3636     if (err)
3637         return err;
3638
3639     err = set_has_smi_cap(dev);
3640     if (err)
3641         goto err_mp;
3642
3643     err = mlx5_query_max_pkeys(&dev->ib_dev, &dev->pkey_table_len);
3644     if (err)
3645         goto err_mp;
3646
3647     if (mlx5_use_mad_ifc(dev))
3648         get_ext_port_caps(dev);
3649
3650     dev->ib_dev.num_comp_vectors    = mlx5_comp_vectors_count(mdev);
3651
3652     mutex_init(&dev->cap_mask_mutex);
3653     INIT_LIST_HEAD(&dev->qp_list);
3654     spin_lock_init(&dev->reset_flow_resource_lock);
3655     xa_init(&dev->odp_mkeys);
3656     xa_init(&dev->sig_mrs);
3657     atomic_set(&dev->mkey_var, 0);
3658
3659     spin_lock_init(&dev->dm.lock);
3660     dev->dm.dev = mdev;
3661     return 0;
3662
3663 err_mp:
3664     mlx5_ib_cleanup_multiport_master(dev);
3665     return err;
3666 }
3667
3668 static int mlx5_ib_enable_driver(struct ib_device *dev)
3669 {
3670     struct mlx5_ib_dev *mdev = to_mdev(dev);
3671     int ret;
3672
3673     ret = mlx5_ib_test_wc(mdev);
3674     mlx5_ib_dbg(mdev, "Write-Combining %s",
3675             mdev->wc_support ? "supported" : "not supported");
3676
3677     return ret;
3678 }
3679
3680 static const struct ib_device_ops mlx5_ib_dev_ops = {
3681     .owner = THIS_MODULE,
3682     .driver_id = RDMA_DRIVER_MLX5,
3683     .uverbs_abi_ver = MLX5_IB_UVERBS_ABI_VERSION,
3684
3685     .add_gid = mlx5_ib_add_gid,
3686     .alloc_mr = mlx5_ib_alloc_mr,
3687     .alloc_mr_integrity = mlx5_ib_alloc_mr_integrity,
3688     .alloc_pd = mlx5_ib_alloc_pd,
3689     .alloc_ucontext = mlx5_ib_alloc_ucontext,
3690     .attach_mcast = mlx5_ib_mcg_attach,
3691     .check_mr_status = mlx5_ib_check_mr_status,
3692     .create_ah = mlx5_ib_create_ah,
3693     .create_cq = mlx5_ib_create_cq,
3694     .create_qp = mlx5_ib_create_qp,
3695     .create_srq = mlx5_ib_create_srq,
3696     .create_user_ah = mlx5_ib_create_ah,
3697     .dealloc_pd = mlx5_ib_dealloc_pd,
3698     .dealloc_ucontext = mlx5_ib_dealloc_ucontext,
3699     .del_gid = mlx5_ib_del_gid,
3700     .dereg_mr = mlx5_ib_dereg_mr,
3701     .destroy_ah = mlx5_ib_destroy_ah,
3702     .destroy_cq = mlx5_ib_destroy_cq,
3703     .destroy_qp = mlx5_ib_destroy_qp,
3704     .destroy_srq = mlx5_ib_destroy_srq,
3705     .detach_mcast = mlx5_ib_mcg_detach,
3706     .disassociate_ucontext = mlx5_ib_disassociate_ucontext,
3707     .drain_rq = mlx5_ib_drain_rq,
3708     .drain_sq = mlx5_ib_drain_sq,
3709     .device_group = &mlx5_attr_group,
3710     .enable_driver = mlx5_ib_enable_driver,
3711     .get_dev_fw_str = get_dev_fw_str,
3712     .get_dma_mr = mlx5_ib_get_dma_mr,
3713     .get_link_layer = mlx5_ib_port_link_layer,
3714     .map_mr_sg = mlx5_ib_map_mr_sg,
3715     .map_mr_sg_pi = mlx5_ib_map_mr_sg_pi,
3716     .mmap = mlx5_ib_mmap,
3717     .mmap_free = mlx5_ib_mmap_free,
3718     .modify_cq = mlx5_ib_modify_cq,
3719     .modify_device = mlx5_ib_modify_device,
3720     .modify_port = mlx5_ib_modify_port,
3721     .modify_qp = mlx5_ib_modify_qp,
3722     .modify_srq = mlx5_ib_modify_srq,
3723     .poll_cq = mlx5_ib_poll_cq,
3724     .post_recv = mlx5_ib_post_recv_nodrain,
3725     .post_send = mlx5_ib_post_send_nodrain,
3726     .post_srq_recv = mlx5_ib_post_srq_recv,
3727     .process_mad = mlx5_ib_process_mad,
3728     .query_ah = mlx5_ib_query_ah,
3729     .query_device = mlx5_ib_query_device,
3730     .query_gid = mlx5_ib_query_gid,
3731     .query_pkey = mlx5_ib_query_pkey,
3732     .query_qp = mlx5_ib_query_qp,
3733     .query_srq = mlx5_ib_query_srq,
3734     .query_ucontext = mlx5_ib_query_ucontext,
3735     .reg_user_mr = mlx5_ib_reg_user_mr,
3736     .reg_user_mr_dmabuf = mlx5_ib_reg_user_mr_dmabuf,
3737     .req_notify_cq = mlx5_ib_arm_cq,
3738     .rereg_user_mr = mlx5_ib_rereg_user_mr,
3739     .resize_cq = mlx5_ib_resize_cq,
3740
3741     INIT_RDMA_OBJ_SIZE(ib_ah, mlx5_ib_ah, ibah),
3742     INIT_RDMA_OBJ_SIZE(ib_counters, mlx5_ib_mcounters, ibcntrs),
3743     INIT_RDMA_OBJ_SIZE(ib_cq, mlx5_ib_cq, ibcq),
3744     INIT_RDMA_OBJ_SIZE(ib_pd, mlx5_ib_pd, ibpd),
3745     INIT_RDMA_OBJ_SIZE(ib_qp, mlx5_ib_qp, ibqp),
3746     INIT_RDMA_OBJ_SIZE(ib_srq, mlx5_ib_srq, ibsrq),
3747     INIT_RDMA_OBJ_SIZE(ib_ucontext, mlx5_ib_ucontext, ibucontext),
3748 };
3749
3750 static const struct ib_device_ops mlx5_ib_dev_ipoib_enhanced_ops = {
3751     .rdma_netdev_get_params = mlx5_ib_rn_get_params,
3752 };
3753
3754 static const struct ib_device_ops mlx5_ib_dev_sriov_ops = {
3755     .get_vf_config = mlx5_ib_get_vf_config,
3756     .get_vf_guid = mlx5_ib_get_vf_guid,
3757     .get_vf_stats = mlx5_ib_get_vf_stats,
3758     .set_vf_guid = mlx5_ib_set_vf_guid,
3759     .set_vf_link_state = mlx5_ib_set_vf_link_state,
3760 };
3761
3762 static const struct ib_device_ops mlx5_ib_dev_mw_ops = {
3763     .alloc_mw = mlx5_ib_alloc_mw,
3764     .dealloc_mw = mlx5_ib_dealloc_mw,
3765
3766     INIT_RDMA_OBJ_SIZE(ib_mw, mlx5_ib_mw, ibmw),
3767 };
3768
3769 static const struct ib_device_ops mlx5_ib_dev_xrc_ops = {
3770     .alloc_xrcd = mlx5_ib_alloc_xrcd,
3771     .dealloc_xrcd = mlx5_ib_dealloc_xrcd,
3772
3773     INIT_RDMA_OBJ_SIZE(ib_xrcd, mlx5_ib_xrcd, ibxrcd),
3774 };
3775
3776 static int mlx5_ib_init_var_table(struct mlx5_ib_dev *dev)
3777 {
3778     struct mlx5_core_dev *mdev = dev->mdev;
3779     struct mlx5_var_table *var_table = &dev->var_table;
3780     u8 log_doorbell_bar_size;
3781     u8 log_doorbell_stride;
3782     u64 bar_size;
3783
3784     log_doorbell_bar_size = MLX5_CAP_DEV_VDPA_EMULATION(mdev,
3785                     log_doorbell_bar_size);
3786     log_doorbell_stride = MLX5_CAP_DEV_VDPA_EMULATION(mdev,
3787                     log_doorbell_stride);
3788     var_table->hw_start_addr = dev->mdev->bar_addr +
3789                 MLX5_CAP64_DEV_VDPA_EMULATION(mdev,
3790                     doorbell_bar_offset);
3791     bar_size = (1ULL << log_doorbell_bar_size) * 4096;
3792     var_table->stride_size = 1ULL << log_doorbell_stride;
3793     var_table->num_var_hw_entries = div_u64(bar_size,
3794                         var_table->stride_size);
3795     mutex_init(&var_table->bitmap_lock);
3796     var_table->bitmap = bitmap_zalloc(var_table->num_var_hw_entries,
3797                       GFP_KERNEL);
3798     return (var_table->bitmap) ? 0 : -ENOMEM;
3799 }
3800
3801 static void mlx5_ib_stage_caps_cleanup(struct mlx5_ib_dev *dev)
3802 {
3803     bitmap_free(dev->var_table.bitmap);
3804 }
3805
3806 static int mlx5_ib_stage_caps_init(struct mlx5_ib_dev *dev)
3807 {
3808     struct mlx5_core_dev *mdev = dev->mdev;
3809     int err;
3810
3811     if (MLX5_CAP_GEN(mdev, ipoib_enhanced_offloads) &&
3812         IS_ENABLED(CONFIG_MLX5_CORE_IPOIB))
3813         ib_set_device_ops(&dev->ib_dev,
3814                   &mlx5_ib_dev_ipoib_enhanced_ops);
3815
3816     if (mlx5_core_is_pf(mdev))
3817         ib_set_device_ops(&dev->ib_dev, &mlx5_ib_dev_sriov_ops);
3818
3819     dev->umr_fence = mlx5_get_umr_fence(MLX5_CAP_GEN(mdev, umr_fence));
3820
3821     if (MLX5_CAP_GEN(mdev, imaicl))
3822         ib_set_device_ops(&dev->ib_dev, &mlx5_ib_dev_mw_ops);
3823
3824     if (MLX5_CAP_GEN(mdev, xrc))
3825         ib_set_device_ops(&dev->ib_dev, &mlx5_ib_dev_xrc_ops);
3826
3827     if (MLX5_CAP_DEV_MEM(mdev, memic) ||
3828         MLX5_CAP_GEN_64(dev->mdev, general_obj_types) &
3829         MLX5_GENERAL_OBJ_TYPES_CAP_SW_ICM)
3830         ib_set_device_ops(&dev->ib_dev, &mlx5_ib_dev_dm_ops);
3831
3832     ib_set_device_ops(&dev->ib_dev, &mlx5_ib_dev_ops);
3833
3834     if (IS_ENABLED(CONFIG_INFINIBAND_USER_ACCESS))
3835         dev->ib_dev.driver_def = mlx5_ib_defs;
3836
3837     err = init_node_data(dev);
3838     if (err)
3839         return err;
3840
3841     if ((MLX5_CAP_GEN(dev->mdev, port_type) == MLX5_CAP_PORT_TYPE_ETH) &&
3842         (MLX5_CAP_GEN(dev->mdev, disable_local_lb_uc) ||
3843          MLX5_CAP_GEN(dev->mdev, disable_local_lb_mc)))
3844         mutex_init(&dev->lb.mutex);
3845
3846     if (MLX5_CAP_GEN_64(dev->mdev, general_obj_types) &
3847             MLX5_GENERAL_OBJ_TYPES_CAP_VIRTIO_NET_Q) {
3848         err = mlx5_ib_init_var_table(dev);
3849         if (err)
3850             return err;
3851     }
3852
3853     dev->ib_dev.use_cq_dim = true;
3854
3855     return 0;
3856 }
3857
3858 static const struct ib_device_ops mlx5_ib_dev_port_ops = {
3859     .get_port_immutable = mlx5_port_immutable,
3860     .query_port = mlx5_ib_query_port,
3861 };
3862
3863 static int mlx5_ib_stage_non_default_cb(struct mlx5_ib_dev *dev)
3864 {
3865     ib_set_device_ops(&dev->ib_dev, &mlx5_ib_dev_port_ops);
3866     return 0;
3867 }
3868
3869 static const struct ib_device_ops mlx5_ib_dev_port_rep_ops = {
3870     .get_port_immutable = mlx5_port_rep_immutable,
3871     .query_port = mlx5_ib_rep_query_port,
3872     .query_pkey = mlx5_ib_rep_query_pkey,
3873 };
3874
3875 static int mlx5_ib_stage_raw_eth_non_default_cb(struct mlx5_ib_dev *dev)
3876 {
3877     ib_set_device_ops(&dev->ib_dev, &mlx5_ib_dev_port_rep_ops);
3878     return 0;
3879 }
3880
3881 static const struct ib_device_ops mlx5_ib_dev_common_roce_ops = {
3882     .create_rwq_ind_table = mlx5_ib_create_rwq_ind_table,
3883     .create_wq = mlx5_ib_create_wq,
3884     .destroy_rwq_ind_table = mlx5_ib_destroy_rwq_ind_table,
3885     .destroy_wq = mlx5_ib_destroy_wq,
3886     .get_netdev = mlx5_ib_get_netdev,
3887     .modify_wq = mlx5_ib_modify_wq,
3888
3889     INIT_RDMA_OBJ_SIZE(ib_rwq_ind_table, mlx5_ib_rwq_ind_table,
3890                ib_rwq_ind_tbl),
3891 };
3892
3893 static int mlx5_ib_roce_init(struct mlx5_ib_dev *dev)
3894 {
3895     struct mlx5_core_dev *mdev = dev->mdev;
3896     enum rdma_link_layer ll;
3897     int port_type_cap;
3898     u32 port_num = 0;
3899     int err;
3900
3901     port_type_cap = MLX5_CAP_GEN(mdev, port_type);
3902     ll = mlx5_port_type_cap_to_rdma_ll(port_type_cap);
3903
3904     if (ll == IB_LINK_LAYER_ETHERNET) {
3905         ib_set_device_ops(&dev->ib_dev, &mlx5_ib_dev_common_roce_ops);
3906
3907         port_num = mlx5_core_native_port_num(dev->mdev) - 1;
3908
3909         /* Register only for native ports */
3910         err = mlx5_add_netdev_notifier(dev, port_num);
3911         if (err)
3912             return err;
3913
3914         err = mlx5_enable_eth(dev);
3915         if (err)
3916             goto cleanup;
3917     }
3918
3919     return 0;
3920 cleanup:
3921     mlx5_remove_netdev_notifier(dev, port_num);
3922     return err;
3923 }
3924
3925 static void mlx5_ib_roce_cleanup(struct mlx5_ib_dev *dev)
3926 {
3927     struct mlx5_core_dev *mdev = dev->mdev;
3928     enum rdma_link_layer ll;
3929     int port_type_cap;
3930     u32 port_num;
3931
3932     port_type_cap = MLX5_CAP_GEN(mdev, port_type);
3933     ll = mlx5_port_type_cap_to_rdma_ll(port_type_cap);
3934
3935     if (ll == IB_LINK_LAYER_ETHERNET) {
3936         mlx5_disable_eth(dev);
3937
3938         port_num = mlx5_core_native_port_num(dev->mdev) - 1;
3939         mlx5_remove_netdev_notifier(dev, port_num);
3940     }
3941 }
3942
3943 static int mlx5_ib_stage_cong_debugfs_init(struct mlx5_ib_dev *dev)
3944 {
3945     mlx5_ib_init_cong_debugfs(dev,
3946                   mlx5_core_native_port_num(dev->mdev) - 1);
3947     return 0;
3948 }
3949
3950 static void mlx5_ib_stage_cong_debugfs_cleanup(struct mlx5_ib_dev *dev)
3951 {
3952     mlx5_ib_cleanup_cong_debugfs(dev,
3953                      mlx5_core_native_port_num(dev->mdev) - 1);
3954 }
3955
3956 static int mlx5_ib_stage_uar_init(struct mlx5_ib_dev *dev)
3957 {
3958     dev->mdev->priv.uar = mlx5_get_uars_page(dev->mdev);
3959     return PTR_ERR_OR_ZERO(dev->mdev->priv.uar);
3960 }
3961
3962 static void mlx5_ib_stage_uar_cleanup(struct mlx5_ib_dev *dev)
3963 {
3964     mlx5_put_uars_page(dev->mdev, dev->mdev->priv.uar);
3965 }
3966
3967 static int mlx5_ib_stage_bfrag_init(struct mlx5_ib_dev *dev)
3968 {
3969     int err;
3970
3971     err = mlx5_alloc_bfreg(dev->mdev, &dev->bfreg, false, false);
3972     if (err)
3973         return err;
3974
3975     err = mlx5_alloc_bfreg(dev->mdev, &dev->fp_bfreg, false, true);
3976     if (err)
3977         mlx5_free_bfreg(dev->mdev, &dev->bfreg);
3978
3979     return err;
3980 }
3981
3982 static void mlx5_ib_stage_bfrag_cleanup(struct mlx5_ib_dev *dev)
3983 {
3984     mlx5_free_bfreg(dev->mdev, &dev->fp_bfreg);
3985     mlx5_free_bfreg(dev->mdev, &dev->bfreg);
3986 }
3987
3988 static int mlx5_ib_stage_ib_reg_init(struct mlx5_ib_dev *dev)
3989 {
3990     const char *name;
3991
3992     if (!mlx5_lag_is_active(dev->mdev))
3993         name = "mlx5_%d";
3994     else
3995         name = "mlx5_bond_%d";
3996     return ib_register_device(&dev->ib_dev, name, &dev->mdev->pdev->dev);
3997 }
3998
3999 static void mlx5_ib_stage_pre_ib_reg_umr_cleanup(struct mlx5_ib_dev *dev)
4000 {
4001     int err;
4002
4003     err = mlx5_mkey_cache_cleanup(dev);
4004     if (err)
4005         mlx5_ib_warn(dev, "mr cache cleanup failed\n");
4006
4007     mlx5r_umr_resource_cleanup(dev);
4008 }
4009
4010 static void mlx5_ib_stage_ib_reg_cleanup(struct mlx5_ib_dev *dev)
4011 {
4012     ib_unregister_device(&dev->ib_dev);
4013 }
4014
4015 static int mlx5_ib_stage_post_ib_reg_umr_init(struct mlx5_ib_dev *dev)
4016 {
4017     int ret;
4018
4019     ret = mlx5r_umr_resource_init(dev);
4020     if (ret)
4021         return ret;
4022
4023     ret = mlx5_mkey_cache_init(dev);
4024     if (ret) {
4025         mlx5_ib_warn(dev, "mr cache init failed %d\n", ret);
4026         mlx5r_umr_resource_cleanup(dev);
4027     }
4028     return ret;
4029 }
4030
4031 static int mlx5_ib_stage_delay_drop_init(struct mlx5_ib_dev *dev)
4032 {
4033     struct dentry *root;
4034
4035     if (!(dev->ib_dev.attrs.raw_packet_caps & IB_RAW_PACKET_CAP_DELAY_DROP))
4036         return 0;
4037
4038     mutex_init(&dev->delay_drop.lock);
4039     dev->delay_drop.dev = dev;
4040     dev->delay_drop.activate = false;
4041     dev->delay_drop.timeout = MLX5_MAX_DELAY_DROP_TIMEOUT_MS * 1000;
4042     INIT_WORK(&dev->delay_drop.delay_drop_work, delay_drop_handler);
4043     atomic_set(&dev->delay_drop.rqs_cnt, 0);
4044     atomic_set(&dev->delay_drop.events_cnt, 0);
4045
4046     if (!mlx5_debugfs_root)
4047         return 0;
4048
4049     root = debugfs_create_dir("delay_drop", mlx5_debugfs_get_dev_root(dev->mdev));
4050     dev->delay_drop.dir_debugfs = root;
4051
4052     debugfs_create_atomic_t("num_timeout_events", 0400, root,
4053                 &dev->delay_drop.events_cnt);
4054     debugfs_create_atomic_t("num_rqs", 0400, root,
4055                 &dev->delay_drop.rqs_cnt);
4056     debugfs_create_file("timeout", 0600, root, &dev->delay_drop,
4057                 &fops_delay_drop_timeout);
4058     return 0;
4059 }
4060
4061 static void mlx5_ib_stage_delay_drop_cleanup(struct mlx5_ib_dev *dev)
4062 {
4063     if (!(dev->ib_dev.attrs.raw_packet_caps & IB_RAW_PACKET_CAP_DELAY_DROP))
4064         return;
4065
4066     cancel_work_sync(&dev->delay_drop.delay_drop_work);
4067     if (!dev->delay_drop.dir_debugfs)
4068         return;
4069
4070     debugfs_remove_recursive(dev->delay_drop.dir_debugfs);
4071     dev->delay_drop.dir_debugfs = NULL;
4072 }
4073
4074 static int mlx5_ib_stage_dev_notifier_init(struct mlx5_ib_dev *dev)
4075 {
4076     dev->mdev_events.notifier_call = mlx5_ib_event;
4077     mlx5_notifier_register(dev->mdev, &dev->mdev_events);
4078     return 0;
4079 }
4080
4081 static void mlx5_ib_stage_dev_notifier_cleanup(struct mlx5_ib_dev *dev)
4082 {
4083     mlx5_notifier_unregister(dev->mdev, &dev->mdev_events);
4084 }
4085
4086 void __mlx5_ib_remove(struct mlx5_ib_dev *dev,
4087               const struct mlx5_ib_profile *profile,
4088               int stage)
4089 {
4090     dev->ib_active = false;
4091
4092     /* Number of stages to cleanup */
4093     while (stage) {
4094         stage--;
4095         if (profile->stage[stage].cleanup)
4096             profile->stage[stage].cleanup(dev);
4097     }
4098
4099     kfree(dev->port);
4100     ib_dealloc_device(&dev->ib_dev);
4101 }
4102
4103 int __mlx5_ib_add(struct mlx5_ib_dev *dev,
4104           const struct mlx5_ib_profile *profile)
4105 {
4106     int err;
4107     int i;
4108
4109     dev->profile = profile;
4110
4111     for (i = 0; i < MLX5_IB_STAGE_MAX; i++) {
4112         if (profile->stage[i].init) {
4113             err = profile->stage[i].init(dev);
4114             if (err)
4115                 goto err_out;
4116         }
4117     }
4118
4119     dev->ib_active = true;
4120     return 0;
4121
4122 err_out:
4123     /* Clean up stages which were initialized */
4124     while (i) {
4125         i--;
4126         if (profile->stage[i].cleanup)
4127             profile->stage[i].cleanup(dev);
4128     }
4129     return -ENOMEM;
4130 }
4131
4132 static const struct mlx5_ib_profile pf_profile = {
4133     STAGE_CREATE(MLX5_IB_STAGE_INIT,
4134              mlx5_ib_stage_init_init,
4135              mlx5_ib_stage_init_cleanup),
4136     STAGE_CREATE(MLX5_IB_STAGE_FS,
4137              mlx5_ib_fs_init,
4138              mlx5_ib_fs_cleanup),
4139     STAGE_CREATE(MLX5_IB_STAGE_CAPS,
4140              mlx5_ib_stage_caps_init,
4141              mlx5_ib_stage_caps_cleanup),
4142     STAGE_CREATE(MLX5_IB_STAGE_NON_DEFAULT_CB,
4143              mlx5_ib_stage_non_default_cb,
4144              NULL),
4145     STAGE_CREATE(MLX5_IB_STAGE_ROCE,
4146              mlx5_ib_roce_init,
4147              mlx5_ib_roce_cleanup),
4148     STAGE_CREATE(MLX5_IB_STAGE_QP,
4149              mlx5_init_qp_table,
4150              mlx5_cleanup_qp_table),
4151     STAGE_CREATE(MLX5_IB_STAGE_SRQ,
4152              mlx5_init_srq_table,
4153              mlx5_cleanup_srq_table),
4154     STAGE_CREATE(MLX5_IB_STAGE_DEVICE_RESOURCES,
4155              mlx5_ib_dev_res_init,
4156              mlx5_ib_dev_res_cleanup),
4157     STAGE_CREATE(MLX5_IB_STAGE_DEVICE_NOTIFIER,
4158              mlx5_ib_stage_dev_notifier_init,
4159              mlx5_ib_stage_dev_notifier_cleanup),
4160     STAGE_CREATE(MLX5_IB_STAGE_ODP,
4161              mlx5_ib_odp_init_one,
4162              mlx5_ib_odp_cleanup_one),
4163     STAGE_CREATE(MLX5_IB_STAGE_COUNTERS,
4164              mlx5_ib_counters_init,
4165              mlx5_ib_counters_cleanup),
4166     STAGE_CREATE(MLX5_IB_STAGE_CONG_DEBUGFS,
4167              mlx5_ib_stage_cong_debugfs_init,
4168              mlx5_ib_stage_cong_debugfs_cleanup),
4169     STAGE_CREATE(MLX5_IB_STAGE_UAR,
4170              mlx5_ib_stage_uar_init,
4171              mlx5_ib_stage_uar_cleanup),
4172     STAGE_CREATE(MLX5_IB_STAGE_BFREG,
4173              mlx5_ib_stage_bfrag_init,
4174              mlx5_ib_stage_bfrag_cleanup),
4175     STAGE_CREATE(MLX5_IB_STAGE_PRE_IB_REG_UMR,
4176              NULL,
4177              mlx5_ib_stage_pre_ib_reg_umr_cleanup),
4178     STAGE_CREATE(MLX5_IB_STAGE_WHITELIST_UID,
4179              mlx5_ib_devx_init,
4180              mlx5_ib_devx_cleanup),
4181     STAGE_CREATE(MLX5_IB_STAGE_IB_REG,
4182              mlx5_ib_stage_ib_reg_init,
4183              mlx5_ib_stage_ib_reg_cleanup),
4184     STAGE_CREATE(MLX5_IB_STAGE_POST_IB_REG_UMR,
4185              mlx5_ib_stage_post_ib_reg_umr_init,
4186              NULL),
4187     STAGE_CREATE(MLX5_IB_STAGE_DELAY_DROP,
4188              mlx5_ib_stage_delay_drop_init,
4189              mlx5_ib_stage_delay_drop_cleanup),
4190     STAGE_CREATE(MLX5_IB_STAGE_RESTRACK,
4191              mlx5_ib_restrack_init,
4192              NULL),
4193 };
4194
4195 const struct mlx5_ib_profile raw_eth_profile = {
4196     STAGE_CREATE(MLX5_IB_STAGE_INIT,
4197              mlx5_ib_stage_init_init,
4198              mlx5_ib_stage_init_cleanup),
4199     STAGE_CREATE(MLX5_IB_STAGE_FS,
4200              mlx5_ib_fs_init,
4201              mlx5_ib_fs_cleanup),
4202     STAGE_CREATE(MLX5_IB_STAGE_CAPS,
4203              mlx5_ib_stage_caps_init,
4204              mlx5_ib_stage_caps_cleanup),
4205     STAGE_CREATE(MLX5_IB_STAGE_NON_DEFAULT_CB,
4206              mlx5_ib_stage_raw_eth_non_default_cb,
4207              NULL),
4208     STAGE_CREATE(MLX5_IB_STAGE_ROCE,
4209              mlx5_ib_roce_init,
4210              mlx5_ib_roce_cleanup),
4211     STAGE_CREATE(MLX5_IB_STAGE_QP,
4212              mlx5_init_qp_table,
4213              mlx5_cleanup_qp_table),
4214     STAGE_CREATE(MLX5_IB_STAGE_SRQ,
4215              mlx5_init_srq_table,
4216              mlx5_cleanup_srq_table),
4217     STAGE_CREATE(MLX5_IB_STAGE_DEVICE_RESOURCES,
4218              mlx5_ib_dev_res_init,
4219              mlx5_ib_dev_res_cleanup),
4220     STAGE_CREATE(MLX5_IB_STAGE_DEVICE_NOTIFIER,
4221              mlx5_ib_stage_dev_notifier_init,
4222              mlx5_ib_stage_dev_notifier_cleanup),
4223     STAGE_CREATE(MLX5_IB_STAGE_COUNTERS,
4224              mlx5_ib_counters_init,
4225              mlx5_ib_counters_cleanup),
4226     STAGE_CREATE(MLX5_IB_STAGE_CONG_DEBUGFS,
4227              mlx5_ib_stage_cong_debugfs_init,
4228              mlx5_ib_stage_cong_debugfs_cleanup),
4229     STAGE_CREATE(MLX5_IB_STAGE_UAR,
4230              mlx5_ib_stage_uar_init,
4231              mlx5_ib_stage_uar_cleanup),
4232     STAGE_CREATE(MLX5_IB_STAGE_BFREG,
4233              mlx5_ib_stage_bfrag_init,
4234              mlx5_ib_stage_bfrag_cleanup),
4235     STAGE_CREATE(MLX5_IB_STAGE_PRE_IB_REG_UMR,
4236              NULL,
4237              mlx5_ib_stage_pre_ib_reg_umr_cleanup),
4238     STAGE_CREATE(MLX5_IB_STAGE_WHITELIST_UID,
4239              mlx5_ib_devx_init,
4240              mlx5_ib_devx_cleanup),
4241     STAGE_CREATE(MLX5_IB_STAGE_IB_REG,
4242              mlx5_ib_stage_ib_reg_init,
4243              mlx5_ib_stage_ib_reg_cleanup),
4244     STAGE_CREATE(MLX5_IB_STAGE_POST_IB_REG_UMR,
4245              mlx5_ib_stage_post_ib_reg_umr_init,
4246              NULL),
4247     STAGE_CREATE(MLX5_IB_STAGE_RESTRACK,
4248              mlx5_ib_restrack_init,
4249              NULL),
4250 };
4251
4252 static int mlx5r_mp_probe(struct auxiliary_device *adev,
4253               const struct auxiliary_device_id *id)
4254 {
4255     struct mlx5_adev *idev = container_of(adev, struct mlx5_adev, adev);
4256     struct mlx5_core_dev *mdev = idev->mdev;
4257     struct mlx5_ib_multiport_info *mpi;
4258     struct mlx5_ib_dev *dev;
4259     bool bound = false;
4260     int err;
4261
4262     mpi = kzalloc(sizeof(*mpi), GFP_KERNEL);
4263     if (!mpi)
4264         return -ENOMEM;
4265
4266     mpi->mdev = mdev;
4267     err = mlx5_query_nic_vport_system_image_guid(mdev,
4268                              &mpi->sys_image_guid);
4269     if (err) {
4270         kfree(mpi);
4271         return err;
4272     }
4273
4274     mutex_lock(&mlx5_ib_multiport_mutex);
4275     list_for_each_entry(dev, &mlx5_ib_dev_list, ib_dev_list) {
4276         if (dev->sys_image_guid == mpi->sys_image_guid)
4277             bound = mlx5_ib_bind_slave_port(dev, mpi);
4278
4279         if (bound) {
4280             rdma_roce_rescan_device(&dev->ib_dev);
4281             mpi->ibdev->ib_active = true;
4282             break;
4283         }
4284     }
4285
4286     if (!bound) {
4287         list_add_tail(&mpi->list, &mlx5_ib_unaffiliated_port_list);
4288         dev_dbg(mdev->device,
4289             "no suitable IB device found to bind to, added to unaffiliated list.\n");
4290     }
4291     mutex_unlock(&mlx5_ib_multiport_mutex);
4292
4293     auxiliary_set_drvdata(adev, mpi);
4294     return 0;
4295 }
4296
4297 static void mlx5r_mp_remove(struct auxiliary_device *adev)
4298 {
4299     struct mlx5_ib_multiport_info *mpi;
4300
4301     mpi = auxiliary_get_drvdata(adev);
4302     mutex_lock(&mlx5_ib_multiport_mutex);
4303     if (mpi->ibdev)
4304         mlx5_ib_unbind_slave_port(mpi->ibdev, mpi);
4305     else
4306         list_del(&mpi->list);
4307     mutex_unlock(&mlx5_ib_multiport_mutex);
4308     kfree(mpi);
4309 }
4310
4311 static int mlx5r_probe(struct auxiliary_device *adev,
4312                const struct auxiliary_device_id *id)
4313 {
4314     struct mlx5_adev *idev = container_of(adev, struct mlx5_adev, adev);
4315     struct mlx5_core_dev *mdev = idev->mdev;
4316     const struct mlx5_ib_profile *profile;
4317     int port_type_cap, num_ports, ret;
4318     enum rdma_link_layer ll;
4319     struct mlx5_ib_dev *dev;
4320
4321     port_type_cap = MLX5_CAP_GEN(mdev, port_type);
4322     ll = mlx5_port_type_cap_to_rdma_ll(port_type_cap);
4323
4324     num_ports = max(MLX5_CAP_GEN(mdev, num_ports),
4325             MLX5_CAP_GEN(mdev, num_vhca_ports));
4326     dev = ib_alloc_device(mlx5_ib_dev, ib_dev);
4327     if (!dev)
4328         return -ENOMEM;
4329     dev->port = kcalloc(num_ports, sizeof(*dev->port),
4330                  GFP_KERNEL);
4331     if (!dev->port) {
4332         ib_dealloc_device(&dev->ib_dev);
4333         return -ENOMEM;
4334     }
4335
4336     dev->mdev = mdev;
4337     dev->num_ports = num_ports;
4338
4339     if (ll == IB_LINK_LAYER_ETHERNET && !mlx5_get_roce_state(mdev))
4340         profile = &raw_eth_profile;
4341     else
4342         profile = &pf_profile;
4343
4344     ret = __mlx5_ib_add(dev, profile);
4345     if (ret) {
4346         kfree(dev->port);
4347         ib_dealloc_device(&dev->ib_dev);
4348         return ret;
4349     }
4350
4351     auxiliary_set_drvdata(adev, dev);
4352     return 0;
4353 }
4354
4355 static void mlx5r_remove(struct auxiliary_device *adev)
4356 {
4357     struct mlx5_ib_dev *dev;
4358
4359     dev = auxiliary_get_drvdata(adev);
4360     __mlx5_ib_remove(dev, dev->profile, MLX5_IB_STAGE_MAX);
4361 }
4362
4363 static const struct auxiliary_device_id mlx5r_mp_id_table[] = {
4364     { .name = MLX5_ADEV_NAME ".multiport", },
4365     {},
4366 };
4367
4368 static const struct auxiliary_device_id mlx5r_id_table[] = {
4369     { .name = MLX5_ADEV_NAME ".rdma", },
4370     {},
4371 };
4372
4373 MODULE_DEVICE_TABLE(auxiliary, mlx5r_mp_id_table);
4374 MODULE_DEVICE_TABLE(auxiliary, mlx5r_id_table);
4375
4376 static struct auxiliary_driver mlx5r_mp_driver = {
4377     .name = "multiport",
4378     .probe = mlx5r_mp_probe,
4379     .remove = mlx5r_mp_remove,
4380     .id_table = mlx5r_mp_id_table,
4381 };
4382
4383 static struct auxiliary_driver mlx5r_driver = {
4384     .name = "rdma",
4385     .probe = mlx5r_probe,
4386     .remove = mlx5r_remove,
4387     .id_table = mlx5r_id_table,
4388 };
4389
4390 static int __init mlx5_ib_init(void)
4391 {
4392     int ret;
4393
4394     xlt_emergency_page = (void *)__get_free_page(GFP_KERNEL);
4395     if (!xlt_emergency_page)
4396         return -ENOMEM;
4397
4398     mlx5_ib_event_wq = alloc_ordered_workqueue("mlx5_ib_event_wq", 0);
4399     if (!mlx5_ib_event_wq) {
4400         free_page((unsigned long)xlt_emergency_page);
4401         return -ENOMEM;
4402     }
4403
4404     mlx5_ib_odp_init();
4405     ret = mlx5r_rep_init();
4406     if (ret)
4407         goto rep_err;
4408     ret = auxiliary_driver_register(&mlx5r_mp_driver);
4409     if (ret)
4410         goto mp_err;
4411     ret = auxiliary_driver_register(&mlx5r_driver);
4412     if (ret)
4413         goto drv_err;
4414     return 0;
4415
4416 drv_err:
4417     auxiliary_driver_unregister(&mlx5r_mp_driver);
4418 mp_err:
4419     mlx5r_rep_cleanup();
4420 rep_err:
4421     destroy_workqueue(mlx5_ib_event_wq);
4422     free_page((unsigned long)xlt_emergency_page);
4423     return ret;
4424 }
4425
4426 static void __exit mlx5_ib_cleanup(void)
4427 {
4428     auxiliary_driver_unregister(&mlx5r_driver);
4429     auxiliary_driver_unregister(&mlx5r_mp_driver);
4430     mlx5r_rep_cleanup();
4431
4432     destroy_workqueue(mlx5_ib_event_wq);
4433     free_page((unsigned long)xlt_emergency_page);
4434 }
4435
4436 module_init(mlx5_ib_init);
4437 module_exit(mlx5_ib_cleanup);