infiniband/core/cm.c

0001 // SPDX-License-Identifier: GPL-2.0 OR Linux-OpenIB
0002 /*
0003  * Copyright (c) 2004-2007 Intel Corporation.  All rights reserved.
0004  * Copyright (c) 2004 Topspin Corporation.  All rights reserved.
0005  * Copyright (c) 2004, 2005 Voltaire Corporation.  All rights reserved.
0006  * Copyright (c) 2005 Sun Microsystems, Inc. All rights reserved.
0007  * Copyright (c) 2019, Mellanox Technologies inc.  All rights reserved.
0008  */
0009
0010 #include <linux/completion.h>
0011 #include <linux/dma-mapping.h>
0012 #include <linux/device.h>
0013 #include <linux/module.h>
0014 #include <linux/err.h>
0015 #include <linux/idr.h>
0016 #include <linux/interrupt.h>
0017 #include <linux/random.h>
0018 #include <linux/rbtree.h>
0019 #include <linux/spinlock.h>
0020 #include <linux/slab.h>
0021 #include <linux/sysfs.h>
0022 #include <linux/workqueue.h>
0023 #include <linux/kdev_t.h>
0024 #include <linux/etherdevice.h>
0025
0026 #include <rdma/ib_cache.h>
0027 #include <rdma/ib_cm.h>
0028 #include <rdma/ib_sysfs.h>
0029 #include "cm_msgs.h"
0030 #include "core_priv.h"
0031 #include "cm_trace.h"
0032
0033 MODULE_AUTHOR("Sean Hefty");
0034 MODULE_DESCRIPTION("InfiniBand CM");
0035 MODULE_LICENSE("Dual BSD/GPL");
0036
0037 static const char * const ibcm_rej_reason_strs[] = {
0038     [IB_CM_REJ_NO_QP]           = "no QP",
0039     [IB_CM_REJ_NO_EEC]          = "no EEC",
0040     [IB_CM_REJ_NO_RESOURCES]        = "no resources",
0041     [IB_CM_REJ_TIMEOUT]         = "timeout",
0042     [IB_CM_REJ_UNSUPPORTED]         = "unsupported",
0043     [IB_CM_REJ_INVALID_COMM_ID]     = "invalid comm ID",
0044     [IB_CM_REJ_INVALID_COMM_INSTANCE]   = "invalid comm instance",
0045     [IB_CM_REJ_INVALID_SERVICE_ID]      = "invalid service ID",
0046     [IB_CM_REJ_INVALID_TRANSPORT_TYPE]  = "invalid transport type",
0047     [IB_CM_REJ_STALE_CONN]          = "stale conn",
0048     [IB_CM_REJ_RDC_NOT_EXIST]       = "RDC not exist",
0049     [IB_CM_REJ_INVALID_GID]         = "invalid GID",
0050     [IB_CM_REJ_INVALID_LID]         = "invalid LID",
0051     [IB_CM_REJ_INVALID_SL]          = "invalid SL",
0052     [IB_CM_REJ_INVALID_TRAFFIC_CLASS]   = "invalid traffic class",
0053     [IB_CM_REJ_INVALID_HOP_LIMIT]       = "invalid hop limit",
0054     [IB_CM_REJ_INVALID_PACKET_RATE]     = "invalid packet rate",
0055     [IB_CM_REJ_INVALID_ALT_GID]     = "invalid alt GID",
0056     [IB_CM_REJ_INVALID_ALT_LID]     = "invalid alt LID",
0057     [IB_CM_REJ_INVALID_ALT_SL]      = "invalid alt SL",
0058     [IB_CM_REJ_INVALID_ALT_TRAFFIC_CLASS]   = "invalid alt traffic class",
0059     [IB_CM_REJ_INVALID_ALT_HOP_LIMIT]   = "invalid alt hop limit",
0060     [IB_CM_REJ_INVALID_ALT_PACKET_RATE] = "invalid alt packet rate",
0061     [IB_CM_REJ_PORT_CM_REDIRECT]        = "port CM redirect",
0062     [IB_CM_REJ_PORT_REDIRECT]       = "port redirect",
0063     [IB_CM_REJ_INVALID_MTU]         = "invalid MTU",
0064     [IB_CM_REJ_INSUFFICIENT_RESP_RESOURCES] = "insufficient resp resources",
0065     [IB_CM_REJ_CONSUMER_DEFINED]        = "consumer defined",
0066     [IB_CM_REJ_INVALID_RNR_RETRY]       = "invalid RNR retry",
0067     [IB_CM_REJ_DUPLICATE_LOCAL_COMM_ID] = "duplicate local comm ID",
0068     [IB_CM_REJ_INVALID_CLASS_VERSION]   = "invalid class version",
0069     [IB_CM_REJ_INVALID_FLOW_LABEL]      = "invalid flow label",
0070     [IB_CM_REJ_INVALID_ALT_FLOW_LABEL]  = "invalid alt flow label",
0071     [IB_CM_REJ_VENDOR_OPTION_NOT_SUPPORTED] =
0072         "vendor option is not supported",
0073 };
0074
0075 const char *__attribute_const__ ibcm_reject_msg(int reason)
0076 {
0077     size_t index = reason;
0078
0079     if (index < ARRAY_SIZE(ibcm_rej_reason_strs) &&
0080         ibcm_rej_reason_strs[index])
0081         return ibcm_rej_reason_strs[index];
0082     else
0083         return "unrecognized reason";
0084 }
0085 EXPORT_SYMBOL(ibcm_reject_msg);
0086
0087 struct cm_id_private;
0088 struct cm_work;
0089 static int cm_add_one(struct ib_device *device);
0090 static void cm_remove_one(struct ib_device *device, void *client_data);
0091 static void cm_process_work(struct cm_id_private *cm_id_priv,
0092                 struct cm_work *work);
0093 static int cm_send_sidr_rep_locked(struct cm_id_private *cm_id_priv,
0094                    struct ib_cm_sidr_rep_param *param);
0095 static int cm_send_dreq_locked(struct cm_id_private *cm_id_priv,
0096                    const void *private_data, u8 private_data_len);
0097 static int cm_send_drep_locked(struct cm_id_private *cm_id_priv,
0098                    void *private_data, u8 private_data_len);
0099 static int cm_send_rej_locked(struct cm_id_private *cm_id_priv,
0100                   enum ib_cm_rej_reason reason, void *ari,
0101                   u8 ari_length, const void *private_data,
0102                   u8 private_data_len);
0103
0104 static struct ib_client cm_client = {
0105     .name   = "cm",
0106     .add    = cm_add_one,
0107     .remove = cm_remove_one
0108 };
0109
0110 static struct ib_cm {
0111     spinlock_t lock;
0112     struct list_head device_list;
0113     rwlock_t device_lock;
0114     struct rb_root listen_service_table;
0115     u64 listen_service_id;
0116     /* struct rb_root peer_service_table; todo: fix peer to peer */
0117     struct rb_root remote_qp_table;
0118     struct rb_root remote_id_table;
0119     struct rb_root remote_sidr_table;
0120     struct xarray local_id_table;
0121     u32 local_id_next;
0122     __be32 random_id_operand;
0123     struct list_head timewait_list;
0124     struct workqueue_struct *wq;
0125 } cm;
0126
0127 /* Counter indexes ordered by attribute ID */
0128 enum {
0129     CM_REQ_COUNTER,
0130     CM_MRA_COUNTER,
0131     CM_REJ_COUNTER,
0132     CM_REP_COUNTER,
0133     CM_RTU_COUNTER,
0134     CM_DREQ_COUNTER,
0135     CM_DREP_COUNTER,
0136     CM_SIDR_REQ_COUNTER,
0137     CM_SIDR_REP_COUNTER,
0138     CM_LAP_COUNTER,
0139     CM_APR_COUNTER,
0140     CM_ATTR_COUNT,
0141     CM_ATTR_ID_OFFSET = 0x0010,
0142 };
0143
0144 enum {
0145     CM_XMIT,
0146     CM_XMIT_RETRIES,
0147     CM_RECV,
0148     CM_RECV_DUPLICATES,
0149     CM_COUNTER_GROUPS
0150 };
0151
0152 struct cm_counter_attribute {
0153     struct ib_port_attribute attr;
0154     unsigned short group;
0155     unsigned short index;
0156 };
0157
0158 struct cm_port {
0159     struct cm_device *cm_dev;
0160     struct ib_mad_agent *mad_agent;
0161     u32 port_num;
0162     atomic_long_t counters[CM_COUNTER_GROUPS][CM_ATTR_COUNT];
0163 };
0164
0165 struct cm_device {
0166     struct kref kref;
0167     struct list_head list;
0168     spinlock_t mad_agent_lock;
0169     struct ib_device *ib_device;
0170     u8 ack_delay;
0171     int going_down;
0172     struct cm_port *port[];
0173 };
0174
0175 struct cm_av {
0176     struct cm_port *port;
0177     struct rdma_ah_attr ah_attr;
0178     u16 pkey_index;
0179     u8 timeout;
0180 };
0181
0182 struct cm_work {
0183     struct delayed_work work;
0184     struct list_head list;
0185     struct cm_port *port;
0186     struct ib_mad_recv_wc *mad_recv_wc; /* Received MADs */
0187     __be32 local_id;            /* Established / timewait */
0188     __be32 remote_id;
0189     struct ib_cm_event cm_event;
0190     struct sa_path_rec path[];
0191 };
0192
0193 struct cm_timewait_info {
0194     struct cm_work work;
0195     struct list_head list;
0196     struct rb_node remote_qp_node;
0197     struct rb_node remote_id_node;
0198     __be64 remote_ca_guid;
0199     __be32 remote_qpn;
0200     u8 inserted_remote_qp;
0201     u8 inserted_remote_id;
0202 };
0203
0204 struct cm_id_private {
0205     struct ib_cm_id id;
0206
0207     struct rb_node service_node;
0208     struct rb_node sidr_id_node;
0209     u32 sidr_slid;
0210     spinlock_t lock;    /* Do not acquire inside cm.lock */
0211     struct completion comp;
0212     refcount_t refcount;
0213     /* Number of clients sharing this ib_cm_id. Only valid for listeners.
0214      * Protected by the cm.lock spinlock.
0215      */
0216     int listen_sharecount;
0217     struct rcu_head rcu;
0218
0219     struct ib_mad_send_buf *msg;
0220     struct cm_timewait_info *timewait_info;
0221     /* todo: use alternate port on send failure */
0222     struct cm_av av;
0223     struct cm_av alt_av;
0224
0225     void *private_data;
0226     __be64 tid;
0227     __be32 local_qpn;
0228     __be32 remote_qpn;
0229     enum ib_qp_type qp_type;
0230     __be32 sq_psn;
0231     __be32 rq_psn;
0232     int timeout_ms;
0233     enum ib_mtu path_mtu;
0234     __be16 pkey;
0235     u8 private_data_len;
0236     u8 max_cm_retries;
0237     u8 responder_resources;
0238     u8 initiator_depth;
0239     u8 retry_count;
0240     u8 rnr_retry_count;
0241     u8 service_timeout;
0242     u8 target_ack_delay;
0243
0244     struct list_head work_list;
0245     atomic_t work_count;
0246
0247     struct rdma_ucm_ece ece;
0248 };
0249
0250 static void cm_dev_release(struct kref *kref)
0251 {
0252     struct cm_device *cm_dev = container_of(kref, struct cm_device, kref);
0253     u32 i;
0254
0255     rdma_for_each_port(cm_dev->ib_device, i)
0256         kfree(cm_dev->port[i - 1]);
0257
0258     kfree(cm_dev);
0259 }
0260
0261 static void cm_device_put(struct cm_device *cm_dev)
0262 {
0263     kref_put(&cm_dev->kref, cm_dev_release);
0264 }
0265
0266 static void cm_work_handler(struct work_struct *work);
0267
0268 static inline void cm_deref_id(struct cm_id_private *cm_id_priv)
0269 {
0270     if (refcount_dec_and_test(&cm_id_priv->refcount))
0271         complete(&cm_id_priv->comp);
0272 }
0273
0274 static struct ib_mad_send_buf *cm_alloc_msg(struct cm_id_private *cm_id_priv)
0275 {
0276     struct ib_mad_agent *mad_agent;
0277     struct ib_mad_send_buf *m;
0278     struct ib_ah *ah;
0279
0280     lockdep_assert_held(&cm_id_priv->lock);
0281
0282     if (!cm_id_priv->av.port)
0283         return ERR_PTR(-EINVAL);
0284
0285     spin_lock(&cm_id_priv->av.port->cm_dev->mad_agent_lock);
0286     mad_agent = cm_id_priv->av.port->mad_agent;
0287     if (!mad_agent) {
0288         m = ERR_PTR(-EINVAL);
0289         goto out;
0290     }
0291
0292     ah = rdma_create_ah(mad_agent->qp->pd, &cm_id_priv->av.ah_attr, 0);
0293     if (IS_ERR(ah)) {
0294         m = ERR_CAST(ah);
0295         goto out;
0296     }
0297
0298     m = ib_create_send_mad(mad_agent, cm_id_priv->id.remote_cm_qpn,
0299                    cm_id_priv->av.pkey_index,
0300                    0, IB_MGMT_MAD_HDR, IB_MGMT_MAD_DATA,
0301                    GFP_ATOMIC,
0302                    IB_MGMT_BASE_VERSION);
0303     if (IS_ERR(m)) {
0304         rdma_destroy_ah(ah, 0);
0305         goto out;
0306     }
0307
0308     /* Timeout set by caller if response is expected. */
0309     m->ah = ah;
0310     m->retries = cm_id_priv->max_cm_retries;
0311
0312     refcount_inc(&cm_id_priv->refcount);
0313     m->context[0] = cm_id_priv;
0314
0315 out:
0316     spin_unlock(&cm_id_priv->av.port->cm_dev->mad_agent_lock);
0317     return m;
0318 }
0319
0320 static void cm_free_msg(struct ib_mad_send_buf *msg)
0321 {
0322     struct cm_id_private *cm_id_priv = msg->context[0];
0323
0324     if (msg->ah)
0325         rdma_destroy_ah(msg->ah, 0);
0326     cm_deref_id(cm_id_priv);
0327     ib_free_send_mad(msg);
0328 }
0329
0330 static struct ib_mad_send_buf *
0331 cm_alloc_priv_msg(struct cm_id_private *cm_id_priv)
0332 {
0333     struct ib_mad_send_buf *msg;
0334
0335     lockdep_assert_held(&cm_id_priv->lock);
0336
0337     msg = cm_alloc_msg(cm_id_priv);
0338     if (IS_ERR(msg))
0339         return msg;
0340     cm_id_priv->msg = msg;
0341     return msg;
0342 }
0343
0344 static void cm_free_priv_msg(struct ib_mad_send_buf *msg)
0345 {
0346     struct cm_id_private *cm_id_priv = msg->context[0];
0347
0348     lockdep_assert_held(&cm_id_priv->lock);
0349
0350     if (!WARN_ON(cm_id_priv->msg != msg))
0351         cm_id_priv->msg = NULL;
0352
0353     if (msg->ah)
0354         rdma_destroy_ah(msg->ah, 0);
0355     cm_deref_id(cm_id_priv);
0356     ib_free_send_mad(msg);
0357 }
0358
0359 static struct ib_mad_send_buf *cm_alloc_response_msg_no_ah(struct cm_port *port,
0360                                struct ib_mad_recv_wc *mad_recv_wc)
0361 {
0362     return ib_create_send_mad(port->mad_agent, 1, mad_recv_wc->wc->pkey_index,
0363                   0, IB_MGMT_MAD_HDR, IB_MGMT_MAD_DATA,
0364                   GFP_ATOMIC,
0365                   IB_MGMT_BASE_VERSION);
0366 }
0367
0368 static int cm_create_response_msg_ah(struct cm_port *port,
0369                      struct ib_mad_recv_wc *mad_recv_wc,
0370                      struct ib_mad_send_buf *msg)
0371 {
0372     struct ib_ah *ah;
0373
0374     ah = ib_create_ah_from_wc(port->mad_agent->qp->pd, mad_recv_wc->wc,
0375                   mad_recv_wc->recv_buf.grh, port->port_num);
0376     if (IS_ERR(ah))
0377         return PTR_ERR(ah);
0378
0379     msg->ah = ah;
0380     return 0;
0381 }
0382
0383 static int cm_alloc_response_msg(struct cm_port *port,
0384                  struct ib_mad_recv_wc *mad_recv_wc,
0385                  struct ib_mad_send_buf **msg)
0386 {
0387     struct ib_mad_send_buf *m;
0388     int ret;
0389
0390     m = cm_alloc_response_msg_no_ah(port, mad_recv_wc);
0391     if (IS_ERR(m))
0392         return PTR_ERR(m);
0393
0394     ret = cm_create_response_msg_ah(port, mad_recv_wc, m);
0395     if (ret) {
0396         ib_free_send_mad(m);
0397         return ret;
0398     }
0399
0400     *msg = m;
0401     return 0;
0402 }
0403
0404 static void cm_free_response_msg(struct ib_mad_send_buf *msg)
0405 {
0406     if (msg->ah)
0407         rdma_destroy_ah(msg->ah, 0);
0408     ib_free_send_mad(msg);
0409 }
0410
0411 static void *cm_copy_private_data(const void *private_data, u8 private_data_len)
0412 {
0413     void *data;
0414
0415     if (!private_data || !private_data_len)
0416         return NULL;
0417
0418     data = kmemdup(private_data, private_data_len, GFP_KERNEL);
0419     if (!data)
0420         return ERR_PTR(-ENOMEM);
0421
0422     return data;
0423 }
0424
0425 static void cm_set_private_data(struct cm_id_private *cm_id_priv,
0426                  void *private_data, u8 private_data_len)
0427 {
0428     if (cm_id_priv->private_data && cm_id_priv->private_data_len)
0429         kfree(cm_id_priv->private_data);
0430
0431     cm_id_priv->private_data = private_data;
0432     cm_id_priv->private_data_len = private_data_len;
0433 }
0434
0435 static void cm_set_av_port(struct cm_av *av, struct cm_port *port)
0436 {
0437     struct cm_port *old_port = av->port;
0438
0439     if (old_port == port)
0440         return;
0441
0442     av->port = port;
0443     if (old_port)
0444         cm_device_put(old_port->cm_dev);
0445     if (port)
0446         kref_get(&port->cm_dev->kref);
0447 }
0448
0449 static void cm_init_av_for_lap(struct cm_port *port, struct ib_wc *wc,
0450                    struct rdma_ah_attr *ah_attr, struct cm_av *av)
0451 {
0452     cm_set_av_port(av, port);
0453     av->pkey_index = wc->pkey_index;
0454     rdma_move_ah_attr(&av->ah_attr, ah_attr);
0455 }
0456
0457 static int cm_init_av_for_response(struct cm_port *port, struct ib_wc *wc,
0458                    struct ib_grh *grh, struct cm_av *av)
0459 {
0460     cm_set_av_port(av, port);
0461     av->pkey_index = wc->pkey_index;
0462     return ib_init_ah_attr_from_wc(port->cm_dev->ib_device,
0463                        port->port_num, wc,
0464                        grh, &av->ah_attr);
0465 }
0466
0467 static struct cm_port *
0468 get_cm_port_from_path(struct sa_path_rec *path, const struct ib_gid_attr *attr)
0469 {
0470     struct cm_device *cm_dev;
0471     struct cm_port *port = NULL;
0472     unsigned long flags;
0473
0474     if (attr) {
0475         read_lock_irqsave(&cm.device_lock, flags);
0476         list_for_each_entry(cm_dev, &cm.device_list, list) {
0477             if (cm_dev->ib_device == attr->device) {
0478                 port = cm_dev->port[attr->port_num - 1];
0479                 break;
0480             }
0481         }
0482         read_unlock_irqrestore(&cm.device_lock, flags);
0483     } else {
0484         /* SGID attribute can be NULL in following
0485          * conditions.
0486          * (a) Alternative path
0487          * (b) IB link layer without GRH
0488          * (c) LAP send messages
0489          */
0490         read_lock_irqsave(&cm.device_lock, flags);
0491         list_for_each_entry(cm_dev, &cm.device_list, list) {
0492             attr = rdma_find_gid(cm_dev->ib_device,
0493                          &path->sgid,
0494                          sa_conv_pathrec_to_gid_type(path),
0495                          NULL);
0496             if (!IS_ERR(attr)) {
0497                 port = cm_dev->port[attr->port_num - 1];
0498                 break;
0499             }
0500         }
0501         read_unlock_irqrestore(&cm.device_lock, flags);
0502         if (port)
0503             rdma_put_gid_attr(attr);
0504     }
0505     return port;
0506 }
0507
0508 static int cm_init_av_by_path(struct sa_path_rec *path,
0509                   const struct ib_gid_attr *sgid_attr,
0510                   struct cm_av *av)
0511 {
0512     struct rdma_ah_attr new_ah_attr;
0513     struct cm_device *cm_dev;
0514     struct cm_port *port;
0515     int ret;
0516
0517     port = get_cm_port_from_path(path, sgid_attr);
0518     if (!port)
0519         return -EINVAL;
0520     cm_dev = port->cm_dev;
0521
0522     ret = ib_find_cached_pkey(cm_dev->ib_device, port->port_num,
0523                   be16_to_cpu(path->pkey), &av->pkey_index);
0524     if (ret)
0525         return ret;
0526
0527     cm_set_av_port(av, port);
0528
0529     /*
0530      * av->ah_attr might be initialized based on wc or during
0531      * request processing time which might have reference to sgid_attr.
0532      * So initialize a new ah_attr on stack.
0533      * If initialization fails, old ah_attr is used for sending any
0534      * responses. If initialization is successful, than new ah_attr
0535      * is used by overwriting the old one. So that right ah_attr
0536      * can be used to return an error response.
0537      */
0538     ret = ib_init_ah_attr_from_path(cm_dev->ib_device, port->port_num, path,
0539                     &new_ah_attr, sgid_attr);
0540     if (ret)
0541         return ret;
0542
0543     av->timeout = path->packet_life_time + 1;
0544     rdma_move_ah_attr(&av->ah_attr, &new_ah_attr);
0545     return 0;
0546 }
0547
0548 /* Move av created by cm_init_av_by_path(), so av.dgid is not moved */
0549 static void cm_move_av_from_path(struct cm_av *dest, struct cm_av *src)
0550 {
0551     cm_set_av_port(dest, src->port);
0552     cm_set_av_port(src, NULL);
0553     dest->pkey_index = src->pkey_index;
0554     rdma_move_ah_attr(&dest->ah_attr, &src->ah_attr);
0555     dest->timeout = src->timeout;
0556 }
0557
0558 static void cm_destroy_av(struct cm_av *av)
0559 {
0560     rdma_destroy_ah_attr(&av->ah_attr);
0561     cm_set_av_port(av, NULL);
0562 }
0563
0564 static u32 cm_local_id(__be32 local_id)
0565 {
0566     return (__force u32) (local_id ^ cm.random_id_operand);
0567 }
0568
0569 static struct cm_id_private *cm_acquire_id(__be32 local_id, __be32 remote_id)
0570 {
0571     struct cm_id_private *cm_id_priv;
0572
0573     rcu_read_lock();
0574     cm_id_priv = xa_load(&cm.local_id_table, cm_local_id(local_id));
0575     if (!cm_id_priv || cm_id_priv->id.remote_id != remote_id ||
0576         !refcount_inc_not_zero(&cm_id_priv->refcount))
0577         cm_id_priv = NULL;
0578     rcu_read_unlock();
0579
0580     return cm_id_priv;
0581 }
0582
0583 /*
0584  * Trivial helpers to strip endian annotation and compare; the
0585  * endianness doesn't actually matter since we just need a stable
0586  * order for the RB tree.
0587  */
0588 static int be32_lt(__be32 a, __be32 b)
0589 {
0590     return (__force u32) a < (__force u32) b;
0591 }
0592
0593 static int be32_gt(__be32 a, __be32 b)
0594 {
0595     return (__force u32) a > (__force u32) b;
0596 }
0597
0598 static int be64_lt(__be64 a, __be64 b)
0599 {
0600     return (__force u64) a < (__force u64) b;
0601 }
0602
0603 static int be64_gt(__be64 a, __be64 b)
0604 {
0605     return (__force u64) a > (__force u64) b;
0606 }
0607
0608 /*
0609  * Inserts a new cm_id_priv into the listen_service_table. Returns cm_id_priv
0610  * if the new ID was inserted, NULL if it could not be inserted due to a
0611  * collision, or the existing cm_id_priv ready for shared usage.
0612  */
0613 static struct cm_id_private *cm_insert_listen(struct cm_id_private *cm_id_priv,
0614                           ib_cm_handler shared_handler)
0615 {
0616     struct rb_node **link = &cm.listen_service_table.rb_node;
0617     struct rb_node *parent = NULL;
0618     struct cm_id_private *cur_cm_id_priv;
0619     __be64 service_id = cm_id_priv->id.service_id;
0620     __be64 service_mask = cm_id_priv->id.service_mask;
0621     unsigned long flags;
0622
0623     spin_lock_irqsave(&cm.lock, flags);
0624     while (*link) {
0625         parent = *link;
0626         cur_cm_id_priv = rb_entry(parent, struct cm_id_private,
0627                       service_node);
0628         if ((cur_cm_id_priv->id.service_mask & service_id) ==
0629             (service_mask & cur_cm_id_priv->id.service_id) &&
0630             (cm_id_priv->id.device == cur_cm_id_priv->id.device)) {
0631             /*
0632              * Sharing an ib_cm_id with different handlers is not
0633              * supported
0634              */
0635             if (cur_cm_id_priv->id.cm_handler != shared_handler ||
0636                 cur_cm_id_priv->id.context ||
0637                 WARN_ON(!cur_cm_id_priv->id.cm_handler)) {
0638                 spin_unlock_irqrestore(&cm.lock, flags);
0639                 return NULL;
0640             }
0641             refcount_inc(&cur_cm_id_priv->refcount);
0642             cur_cm_id_priv->listen_sharecount++;
0643             spin_unlock_irqrestore(&cm.lock, flags);
0644             return cur_cm_id_priv;
0645         }
0646
0647         if (cm_id_priv->id.device < cur_cm_id_priv->id.device)
0648             link = &(*link)->rb_left;
0649         else if (cm_id_priv->id.device > cur_cm_id_priv->id.device)
0650             link = &(*link)->rb_right;
0651         else if (be64_lt(service_id, cur_cm_id_priv->id.service_id))
0652             link = &(*link)->rb_left;
0653         else if (be64_gt(service_id, cur_cm_id_priv->id.service_id))
0654             link = &(*link)->rb_right;
0655         else
0656             link = &(*link)->rb_right;
0657     }
0658     cm_id_priv->listen_sharecount++;
0659     rb_link_node(&cm_id_priv->service_node, parent, link);
0660     rb_insert_color(&cm_id_priv->service_node, &cm.listen_service_table);
0661     spin_unlock_irqrestore(&cm.lock, flags);
0662     return cm_id_priv;
0663 }
0664
0665 static struct cm_id_private *cm_find_listen(struct ib_device *device,
0666                         __be64 service_id)
0667 {
0668     struct rb_node *node = cm.listen_service_table.rb_node;
0669     struct cm_id_private *cm_id_priv;
0670
0671     while (node) {
0672         cm_id_priv = rb_entry(node, struct cm_id_private, service_node);
0673         if ((cm_id_priv->id.service_mask & service_id) ==
0674              cm_id_priv->id.service_id &&
0675             (cm_id_priv->id.device == device)) {
0676             refcount_inc(&cm_id_priv->refcount);
0677             return cm_id_priv;
0678         }
0679         if (device < cm_id_priv->id.device)
0680             node = node->rb_left;
0681         else if (device > cm_id_priv->id.device)
0682             node = node->rb_right;
0683         else if (be64_lt(service_id, cm_id_priv->id.service_id))
0684             node = node->rb_left;
0685         else if (be64_gt(service_id, cm_id_priv->id.service_id))
0686             node = node->rb_right;
0687         else
0688             node = node->rb_right;
0689     }
0690     return NULL;
0691 }
0692
0693 static struct cm_timewait_info *
0694 cm_insert_remote_id(struct cm_timewait_info *timewait_info)
0695 {
0696     struct rb_node **link = &cm.remote_id_table.rb_node;
0697     struct rb_node *parent = NULL;
0698     struct cm_timewait_info *cur_timewait_info;
0699     __be64 remote_ca_guid = timewait_info->remote_ca_guid;
0700     __be32 remote_id = timewait_info->work.remote_id;
0701
0702     while (*link) {
0703         parent = *link;
0704         cur_timewait_info = rb_entry(parent, struct cm_timewait_info,
0705                          remote_id_node);
0706         if (be32_lt(remote_id, cur_timewait_info->work.remote_id))
0707             link = &(*link)->rb_left;
0708         else if (be32_gt(remote_id, cur_timewait_info->work.remote_id))
0709             link = &(*link)->rb_right;
0710         else if (be64_lt(remote_ca_guid, cur_timewait_info->remote_ca_guid))
0711             link = &(*link)->rb_left;
0712         else if (be64_gt(remote_ca_guid, cur_timewait_info->remote_ca_guid))
0713             link = &(*link)->rb_right;
0714         else
0715             return cur_timewait_info;
0716     }
0717     timewait_info->inserted_remote_id = 1;
0718     rb_link_node(&timewait_info->remote_id_node, parent, link);
0719     rb_insert_color(&timewait_info->remote_id_node, &cm.remote_id_table);
0720     return NULL;
0721 }
0722
0723 static struct cm_id_private *cm_find_remote_id(__be64 remote_ca_guid,
0724                            __be32 remote_id)
0725 {
0726     struct rb_node *node = cm.remote_id_table.rb_node;
0727     struct cm_timewait_info *timewait_info;
0728     struct cm_id_private *res = NULL;
0729
0730     spin_lock_irq(&cm.lock);
0731     while (node) {
0732         timewait_info = rb_entry(node, struct cm_timewait_info,
0733                      remote_id_node);
0734         if (be32_lt(remote_id, timewait_info->work.remote_id))
0735             node = node->rb_left;
0736         else if (be32_gt(remote_id, timewait_info->work.remote_id))
0737             node = node->rb_right;
0738         else if (be64_lt(remote_ca_guid, timewait_info->remote_ca_guid))
0739             node = node->rb_left;
0740         else if (be64_gt(remote_ca_guid, timewait_info->remote_ca_guid))
0741             node = node->rb_right;
0742         else {
0743             res = cm_acquire_id(timewait_info->work.local_id,
0744                          timewait_info->work.remote_id);
0745             break;
0746         }
0747     }
0748     spin_unlock_irq(&cm.lock);
0749     return res;
0750 }
0751
0752 static struct cm_timewait_info *
0753 cm_insert_remote_qpn(struct cm_timewait_info *timewait_info)
0754 {
0755     struct rb_node **link = &cm.remote_qp_table.rb_node;
0756     struct rb_node *parent = NULL;
0757     struct cm_timewait_info *cur_timewait_info;
0758     __be64 remote_ca_guid = timewait_info->remote_ca_guid;
0759     __be32 remote_qpn = timewait_info->remote_qpn;
0760
0761     while (*link) {
0762         parent = *link;
0763         cur_timewait_info = rb_entry(parent, struct cm_timewait_info,
0764                          remote_qp_node);
0765         if (be32_lt(remote_qpn, cur_timewait_info->remote_qpn))
0766             link = &(*link)->rb_left;
0767         else if (be32_gt(remote_qpn, cur_timewait_info->remote_qpn))
0768             link = &(*link)->rb_right;
0769         else if (be64_lt(remote_ca_guid, cur_timewait_info->remote_ca_guid))
0770             link = &(*link)->rb_left;
0771         else if (be64_gt(remote_ca_guid, cur_timewait_info->remote_ca_guid))
0772             link = &(*link)->rb_right;
0773         else
0774             return cur_timewait_info;
0775     }
0776     timewait_info->inserted_remote_qp = 1;
0777     rb_link_node(&timewait_info->remote_qp_node, parent, link);
0778     rb_insert_color(&timewait_info->remote_qp_node, &cm.remote_qp_table);
0779     return NULL;
0780 }
0781
0782 static struct cm_id_private *
0783 cm_insert_remote_sidr(struct cm_id_private *cm_id_priv)
0784 {
0785     struct rb_node **link = &cm.remote_sidr_table.rb_node;
0786     struct rb_node *parent = NULL;
0787     struct cm_id_private *cur_cm_id_priv;
0788     __be32 remote_id = cm_id_priv->id.remote_id;
0789
0790     while (*link) {
0791         parent = *link;
0792         cur_cm_id_priv = rb_entry(parent, struct cm_id_private,
0793                       sidr_id_node);
0794         if (be32_lt(remote_id, cur_cm_id_priv->id.remote_id))
0795             link = &(*link)->rb_left;
0796         else if (be32_gt(remote_id, cur_cm_id_priv->id.remote_id))
0797             link = &(*link)->rb_right;
0798         else {
0799             if (cur_cm_id_priv->sidr_slid < cm_id_priv->sidr_slid)
0800                 link = &(*link)->rb_left;
0801             else if (cur_cm_id_priv->sidr_slid > cm_id_priv->sidr_slid)
0802                 link = &(*link)->rb_right;
0803             else
0804                 return cur_cm_id_priv;
0805         }
0806     }
0807     rb_link_node(&cm_id_priv->sidr_id_node, parent, link);
0808     rb_insert_color(&cm_id_priv->sidr_id_node, &cm.remote_sidr_table);
0809     return NULL;
0810 }
0811
0812 static struct cm_id_private *cm_alloc_id_priv(struct ib_device *device,
0813                           ib_cm_handler cm_handler,
0814                           void *context)
0815 {
0816     struct cm_id_private *cm_id_priv;
0817     u32 id;
0818     int ret;
0819
0820     cm_id_priv = kzalloc(sizeof *cm_id_priv, GFP_KERNEL);
0821     if (!cm_id_priv)
0822         return ERR_PTR(-ENOMEM);
0823
0824     cm_id_priv->id.state = IB_CM_IDLE;
0825     cm_id_priv->id.device = device;
0826     cm_id_priv->id.cm_handler = cm_handler;
0827     cm_id_priv->id.context = context;
0828     cm_id_priv->id.remote_cm_qpn = 1;
0829
0830     RB_CLEAR_NODE(&cm_id_priv->service_node);
0831     RB_CLEAR_NODE(&cm_id_priv->sidr_id_node);
0832     spin_lock_init(&cm_id_priv->lock);
0833     init_completion(&cm_id_priv->comp);
0834     INIT_LIST_HEAD(&cm_id_priv->work_list);
0835     atomic_set(&cm_id_priv->work_count, -1);
0836     refcount_set(&cm_id_priv->refcount, 1);
0837
0838     ret = xa_alloc_cyclic(&cm.local_id_table, &id, NULL, xa_limit_32b,
0839                   &cm.local_id_next, GFP_KERNEL);
0840     if (ret < 0)
0841         goto error;
0842     cm_id_priv->id.local_id = (__force __be32)id ^ cm.random_id_operand;
0843
0844     return cm_id_priv;
0845
0846 error:
0847     kfree(cm_id_priv);
0848     return ERR_PTR(ret);
0849 }
0850
0851 /*
0852  * Make the ID visible to the MAD handlers and other threads that use the
0853  * xarray.
0854  */
0855 static void cm_finalize_id(struct cm_id_private *cm_id_priv)
0856 {
0857     xa_store(&cm.local_id_table, cm_local_id(cm_id_priv->id.local_id),
0858          cm_id_priv, GFP_ATOMIC);
0859 }
0860
0861 struct ib_cm_id *ib_create_cm_id(struct ib_device *device,
0862                  ib_cm_handler cm_handler,
0863                  void *context)
0864 {
0865     struct cm_id_private *cm_id_priv;
0866
0867     cm_id_priv = cm_alloc_id_priv(device, cm_handler, context);
0868     if (IS_ERR(cm_id_priv))
0869         return ERR_CAST(cm_id_priv);
0870
0871     cm_finalize_id(cm_id_priv);
0872     return &cm_id_priv->id;
0873 }
0874 EXPORT_SYMBOL(ib_create_cm_id);
0875
0876 static struct cm_work *cm_dequeue_work(struct cm_id_private *cm_id_priv)
0877 {
0878     struct cm_work *work;
0879
0880     if (list_empty(&cm_id_priv->work_list))
0881         return NULL;
0882
0883     work = list_entry(cm_id_priv->work_list.next, struct cm_work, list);
0884     list_del(&work->list);
0885     return work;
0886 }
0887
0888 static void cm_free_work(struct cm_work *work)
0889 {
0890     if (work->mad_recv_wc)
0891         ib_free_recv_mad(work->mad_recv_wc);
0892     kfree(work);
0893 }
0894
0895 static void cm_queue_work_unlock(struct cm_id_private *cm_id_priv,
0896                  struct cm_work *work)
0897     __releases(&cm_id_priv->lock)
0898 {
0899     bool immediate;
0900
0901     /*
0902      * To deliver the event to the user callback we have the drop the
0903      * spinlock, however, we need to ensure that the user callback is single
0904      * threaded and receives events in the temporal order. If there are
0905      * already events being processed then thread new events onto a list,
0906      * the thread currently processing will pick them up.
0907      */
0908     immediate = atomic_inc_and_test(&cm_id_priv->work_count);
0909     if (!immediate) {
0910         list_add_tail(&work->list, &cm_id_priv->work_list);
0911         /*
0912          * This routine always consumes incoming reference. Once queued
0913          * to the work_list then a reference is held by the thread
0914          * currently running cm_process_work() and this reference is not
0915          * needed.
0916          */
0917         cm_deref_id(cm_id_priv);
0918     }
0919     spin_unlock_irq(&cm_id_priv->lock);
0920
0921     if (immediate)
0922         cm_process_work(cm_id_priv, work);
0923 }
0924
0925 static inline int cm_convert_to_ms(int iba_time)
0926 {
0927     /* approximate conversion to ms from 4.096us x 2^iba_time */
0928     return 1 << max(iba_time - 8, 0);
0929 }
0930
0931 /*
0932  * calculate: 4.096x2^ack_timeout = 4.096x2^ack_delay + 2x4.096x2^life_time
0933  * Because of how ack_timeout is stored, adding one doubles the timeout.
0934  * To avoid large timeouts, select the max(ack_delay, life_time + 1), and
0935  * increment it (round up) only if the other is within 50%.
0936  */
0937 static u8 cm_ack_timeout(u8 ca_ack_delay, u8 packet_life_time)
0938 {
0939     int ack_timeout = packet_life_time + 1;
0940
0941     if (ack_timeout >= ca_ack_delay)
0942         ack_timeout += (ca_ack_delay >= (ack_timeout - 1));
0943     else
0944         ack_timeout = ca_ack_delay +
0945                   (ack_timeout >= (ca_ack_delay - 1));
0946
0947     return min(31, ack_timeout);
0948 }
0949
0950 static void cm_remove_remote(struct cm_id_private *cm_id_priv)
0951 {
0952     struct cm_timewait_info *timewait_info = cm_id_priv->timewait_info;
0953
0954     if (timewait_info->inserted_remote_id) {
0955         rb_erase(&timewait_info->remote_id_node, &cm.remote_id_table);
0956         timewait_info->inserted_remote_id = 0;
0957     }
0958
0959     if (timewait_info->inserted_remote_qp) {
0960         rb_erase(&timewait_info->remote_qp_node, &cm.remote_qp_table);
0961         timewait_info->inserted_remote_qp = 0;
0962     }
0963 }
0964
0965 static struct cm_timewait_info *cm_create_timewait_info(__be32 local_id)
0966 {
0967     struct cm_timewait_info *timewait_info;
0968
0969     timewait_info = kzalloc(sizeof *timewait_info, GFP_KERNEL);
0970     if (!timewait_info)
0971         return ERR_PTR(-ENOMEM);
0972
0973     timewait_info->work.local_id = local_id;
0974     INIT_DELAYED_WORK(&timewait_info->work.work, cm_work_handler);
0975     timewait_info->work.cm_event.event = IB_CM_TIMEWAIT_EXIT;
0976     return timewait_info;
0977 }
0978
0979 static void cm_enter_timewait(struct cm_id_private *cm_id_priv)
0980 {
0981     int wait_time;
0982     unsigned long flags;
0983     struct cm_device *cm_dev;
0984
0985     lockdep_assert_held(&cm_id_priv->lock);
0986
0987     cm_dev = ib_get_client_data(cm_id_priv->id.device, &cm_client);
0988     if (!cm_dev)
0989         return;
0990
0991     spin_lock_irqsave(&cm.lock, flags);
0992     cm_remove_remote(cm_id_priv);
0993     list_add_tail(&cm_id_priv->timewait_info->list, &cm.timewait_list);
0994     spin_unlock_irqrestore(&cm.lock, flags);
0995
0996     /*
0997      * The cm_id could be destroyed by the user before we exit timewait.
0998      * To protect against this, we search for the cm_id after exiting
0999      * timewait before notifying the user that we've exited timewait.
1000      */
1001     cm_id_priv->id.state = IB_CM_TIMEWAIT;
1002     wait_time = cm_convert_to_ms(cm_id_priv->av.timeout);
1003
1004     /* Check if the device started its remove_one */
1005     spin_lock_irqsave(&cm.lock, flags);
1006     if (!cm_dev->going_down)
1007         queue_delayed_work(cm.wq, &cm_id_priv->timewait_info->work.work,
1008                    msecs_to_jiffies(wait_time));
1009     spin_unlock_irqrestore(&cm.lock, flags);
1010
1011     /*
1012      * The timewait_info is converted into a work and gets freed during
1013      * cm_free_work() in cm_timewait_handler().
1014      */
1015     BUILD_BUG_ON(offsetof(struct cm_timewait_info, work) != 0);
1016     cm_id_priv->timewait_info = NULL;
1017 }
1018
1019 static void cm_reset_to_idle(struct cm_id_private *cm_id_priv)
1020 {
1021     unsigned long flags;
1022
1023     lockdep_assert_held(&cm_id_priv->lock);
1024
1025     cm_id_priv->id.state = IB_CM_IDLE;
1026     if (cm_id_priv->timewait_info) {
1027         spin_lock_irqsave(&cm.lock, flags);
1028         cm_remove_remote(cm_id_priv);
1029         spin_unlock_irqrestore(&cm.lock, flags);
1030         kfree(cm_id_priv->timewait_info);
1031         cm_id_priv->timewait_info = NULL;
1032     }
1033 }
1034
1035 static void cm_destroy_id(struct ib_cm_id *cm_id, int err)
1036 {
1037     struct cm_id_private *cm_id_priv;
1038     struct cm_work *work;
1039
1040     cm_id_priv = container_of(cm_id, struct cm_id_private, id);
1041     spin_lock_irq(&cm_id_priv->lock);
1042 retest:
1043     switch (cm_id->state) {
1044     case IB_CM_LISTEN:
1045         spin_lock(&cm.lock);
1046         if (--cm_id_priv->listen_sharecount > 0) {
1047             /* The id is still shared. */
1048             WARN_ON(refcount_read(&cm_id_priv->refcount) == 1);
1049             spin_unlock(&cm.lock);
1050             spin_unlock_irq(&cm_id_priv->lock);
1051             cm_deref_id(cm_id_priv);
1052             return;
1053         }
1054         cm_id->state = IB_CM_IDLE;
1055         rb_erase(&cm_id_priv->service_node, &cm.listen_service_table);
1056         RB_CLEAR_NODE(&cm_id_priv->service_node);
1057         spin_unlock(&cm.lock);
1058         break;
1059     case IB_CM_SIDR_REQ_SENT:
1060         cm_id->state = IB_CM_IDLE;
1061         ib_cancel_mad(cm_id_priv->msg);
1062         break;
1063     case IB_CM_SIDR_REQ_RCVD:
1064         cm_send_sidr_rep_locked(cm_id_priv,
1065                     &(struct ib_cm_sidr_rep_param){
1066                         .status = IB_SIDR_REJECT });
1067         /* cm_send_sidr_rep_locked will not move to IDLE if it fails */
1068         cm_id->state = IB_CM_IDLE;
1069         break;
1070     case IB_CM_REQ_SENT:
1071     case IB_CM_MRA_REQ_RCVD:
1072         ib_cancel_mad(cm_id_priv->msg);
1073         cm_send_rej_locked(cm_id_priv, IB_CM_REJ_TIMEOUT,
1074                    &cm_id_priv->id.device->node_guid,
1075                    sizeof(cm_id_priv->id.device->node_guid),
1076                    NULL, 0);
1077         break;
1078     case IB_CM_REQ_RCVD:
1079         if (err == -ENOMEM) {
1080             /* Do not reject to allow future retries. */
1081             cm_reset_to_idle(cm_id_priv);
1082         } else {
1083             cm_send_rej_locked(cm_id_priv,
1084                        IB_CM_REJ_CONSUMER_DEFINED, NULL, 0,
1085                        NULL, 0);
1086         }
1087         break;
1088     case IB_CM_REP_SENT:
1089     case IB_CM_MRA_REP_RCVD:
1090         ib_cancel_mad(cm_id_priv->msg);
1091         cm_send_rej_locked(cm_id_priv, IB_CM_REJ_CONSUMER_DEFINED, NULL,
1092                    0, NULL, 0);
1093         goto retest;
1094     case IB_CM_MRA_REQ_SENT:
1095     case IB_CM_REP_RCVD:
1096     case IB_CM_MRA_REP_SENT:
1097         cm_send_rej_locked(cm_id_priv, IB_CM_REJ_CONSUMER_DEFINED, NULL,
1098                    0, NULL, 0);
1099         break;
1100     case IB_CM_ESTABLISHED:
1101         if (cm_id_priv->qp_type == IB_QPT_XRC_TGT) {
1102             cm_id->state = IB_CM_IDLE;
1103             break;
1104         }
1105         cm_send_dreq_locked(cm_id_priv, NULL, 0);
1106         goto retest;
1107     case IB_CM_DREQ_SENT:
1108         ib_cancel_mad(cm_id_priv->msg);
1109         cm_enter_timewait(cm_id_priv);
1110         goto retest;
1111     case IB_CM_DREQ_RCVD:
1112         cm_send_drep_locked(cm_id_priv, NULL, 0);
1113         WARN_ON(cm_id->state != IB_CM_TIMEWAIT);
1114         goto retest;
1115     case IB_CM_TIMEWAIT:
1116         /*
1117          * The cm_acquire_id in cm_timewait_handler will stop working
1118          * once we do xa_erase below, so just move to idle here for
1119          * consistency.
1120          */
1121         cm_id->state = IB_CM_IDLE;
1122         break;
1123     case IB_CM_IDLE:
1124         break;
1125     }
1126     WARN_ON(cm_id->state != IB_CM_IDLE);
1127
1128     spin_lock(&cm.lock);
1129     /* Required for cleanup paths related cm_req_handler() */
1130     if (cm_id_priv->timewait_info) {
1131         cm_remove_remote(cm_id_priv);
1132         kfree(cm_id_priv->timewait_info);
1133         cm_id_priv->timewait_info = NULL;
1134     }
1135
1136     WARN_ON(cm_id_priv->listen_sharecount);
1137     WARN_ON(!RB_EMPTY_NODE(&cm_id_priv->service_node));
1138     if (!RB_EMPTY_NODE(&cm_id_priv->sidr_id_node))
1139         rb_erase(&cm_id_priv->sidr_id_node, &cm.remote_sidr_table);
1140     spin_unlock(&cm.lock);
1141     spin_unlock_irq(&cm_id_priv->lock);
1142
1143     xa_erase(&cm.local_id_table, cm_local_id(cm_id->local_id));
1144     cm_deref_id(cm_id_priv);
1145     wait_for_completion(&cm_id_priv->comp);
1146     while ((work = cm_dequeue_work(cm_id_priv)) != NULL)
1147         cm_free_work(work);
1148
1149     cm_destroy_av(&cm_id_priv->av);
1150     cm_destroy_av(&cm_id_priv->alt_av);
1151     kfree(cm_id_priv->private_data);
1152     kfree_rcu(cm_id_priv, rcu);
1153 }
1154
1155 void ib_destroy_cm_id(struct ib_cm_id *cm_id)
1156 {
1157     cm_destroy_id(cm_id, 0);
1158 }
1159 EXPORT_SYMBOL(ib_destroy_cm_id);
1160
1161 static int cm_init_listen(struct cm_id_private *cm_id_priv, __be64 service_id,
1162               __be64 service_mask)
1163 {
1164     service_mask = service_mask ? service_mask : ~cpu_to_be64(0);
1165     service_id &= service_mask;
1166     if ((service_id & IB_SERVICE_ID_AGN_MASK) == IB_CM_ASSIGN_SERVICE_ID &&
1167         (service_id != IB_CM_ASSIGN_SERVICE_ID))
1168         return -EINVAL;
1169
1170     if (service_id == IB_CM_ASSIGN_SERVICE_ID) {
1171         cm_id_priv->id.service_id = cpu_to_be64(cm.listen_service_id++);
1172         cm_id_priv->id.service_mask = ~cpu_to_be64(0);
1173     } else {
1174         cm_id_priv->id.service_id = service_id;
1175         cm_id_priv->id.service_mask = service_mask;
1176     }
1177     return 0;
1178 }
1179
1180 /**
1181  * ib_cm_listen - Initiates listening on the specified service ID for
1182  *   connection and service ID resolution requests.
1183  * @cm_id: Connection identifier associated with the listen request.
1184  * @service_id: Service identifier matched against incoming connection
1185  *   and service ID resolution requests.  The service ID should be specified
1186  *   network-byte order.  If set to IB_CM_ASSIGN_SERVICE_ID, the CM will
1187  *   assign a service ID to the caller.
1188  * @service_mask: Mask applied to service ID used to listen across a
1189  *   range of service IDs.  If set to 0, the service ID is matched
1190  *   exactly.  This parameter is ignored if %service_id is set to
1191  *   IB_CM_ASSIGN_SERVICE_ID.
1192  */
1193 int ib_cm_listen(struct ib_cm_id *cm_id, __be64 service_id, __be64 service_mask)
1194 {
1195     struct cm_id_private *cm_id_priv =
1196         container_of(cm_id, struct cm_id_private, id);
1197     unsigned long flags;
1198     int ret;
1199
1200     spin_lock_irqsave(&cm_id_priv->lock, flags);
1201     if (cm_id_priv->id.state != IB_CM_IDLE) {
1202         ret = -EINVAL;
1203         goto out;
1204     }
1205
1206     ret = cm_init_listen(cm_id_priv, service_id, service_mask);
1207     if (ret)
1208         goto out;
1209
1210     if (!cm_insert_listen(cm_id_priv, NULL)) {
1211         ret = -EBUSY;
1212         goto out;
1213     }
1214
1215     cm_id_priv->id.state = IB_CM_LISTEN;
1216     ret = 0;
1217
1218 out:
1219     spin_unlock_irqrestore(&cm_id_priv->lock, flags);
1220     return ret;
1221 }
1222 EXPORT_SYMBOL(ib_cm_listen);
1223
1224 /**
1225  * ib_cm_insert_listen - Create a new listening ib_cm_id and listen on
1226  *           the given service ID.
1227  *
1228  * If there's an existing ID listening on that same device and service ID,
1229  * return it.
1230  *
1231  * @device: Device associated with the cm_id.  All related communication will
1232  * be associated with the specified device.
1233  * @cm_handler: Callback invoked to notify the user of CM events.
1234  * @service_id: Service identifier matched against incoming connection
1235  *   and service ID resolution requests.  The service ID should be specified
1236  *   network-byte order.  If set to IB_CM_ASSIGN_SERVICE_ID, the CM will
1237  *   assign a service ID to the caller.
1238  *
1239  * Callers should call ib_destroy_cm_id when done with the listener ID.
1240  */
1241 struct ib_cm_id *ib_cm_insert_listen(struct ib_device *device,
1242                      ib_cm_handler cm_handler,
1243                      __be64 service_id)
1244 {
1245     struct cm_id_private *listen_id_priv;
1246     struct cm_id_private *cm_id_priv;
1247     int err = 0;
1248
1249     /* Create an ID in advance, since the creation may sleep */
1250     cm_id_priv = cm_alloc_id_priv(device, cm_handler, NULL);
1251     if (IS_ERR(cm_id_priv))
1252         return ERR_CAST(cm_id_priv);
1253
1254     err = cm_init_listen(cm_id_priv, service_id, 0);
1255     if (err) {
1256         ib_destroy_cm_id(&cm_id_priv->id);
1257         return ERR_PTR(err);
1258     }
1259
1260     spin_lock_irq(&cm_id_priv->lock);
1261     listen_id_priv = cm_insert_listen(cm_id_priv, cm_handler);
1262     if (listen_id_priv != cm_id_priv) {
1263         spin_unlock_irq(&cm_id_priv->lock);
1264         ib_destroy_cm_id(&cm_id_priv->id);
1265         if (!listen_id_priv)
1266             return ERR_PTR(-EINVAL);
1267         return &listen_id_priv->id;
1268     }
1269     cm_id_priv->id.state = IB_CM_LISTEN;
1270     spin_unlock_irq(&cm_id_priv->lock);
1271
1272     /*
1273      * A listen ID does not need to be in the xarray since it does not
1274      * receive mads, is not placed in the remote_id or remote_qpn rbtree,
1275      * and does not enter timewait.
1276      */
1277
1278     return &cm_id_priv->id;
1279 }
1280 EXPORT_SYMBOL(ib_cm_insert_listen);
1281
1282 static __be64 cm_form_tid(struct cm_id_private *cm_id_priv)
1283 {
1284     u64 hi_tid = 0, low_tid;
1285
1286     lockdep_assert_held(&cm_id_priv->lock);
1287
1288     low_tid = (u64)cm_id_priv->id.local_id;
1289     if (!cm_id_priv->av.port)
1290         return cpu_to_be64(low_tid);
1291
1292     spin_lock(&cm_id_priv->av.port->cm_dev->mad_agent_lock);
1293     if (cm_id_priv->av.port->mad_agent)
1294         hi_tid = ((u64)cm_id_priv->av.port->mad_agent->hi_tid) << 32;
1295     spin_unlock(&cm_id_priv->av.port->cm_dev->mad_agent_lock);
1296     return cpu_to_be64(hi_tid | low_tid);
1297 }
1298
1299 static void cm_format_mad_hdr(struct ib_mad_hdr *hdr,
1300                   __be16 attr_id, __be64 tid)
1301 {
1302     hdr->base_version  = IB_MGMT_BASE_VERSION;
1303     hdr->mgmt_class    = IB_MGMT_CLASS_CM;
1304     hdr->class_version = IB_CM_CLASS_VERSION;
1305     hdr->method    = IB_MGMT_METHOD_SEND;
1306     hdr->attr_id       = attr_id;
1307     hdr->tid       = tid;
1308 }
1309
1310 static void cm_format_mad_ece_hdr(struct ib_mad_hdr *hdr, __be16 attr_id,
1311                   __be64 tid, u32 attr_mod)
1312 {
1313     cm_format_mad_hdr(hdr, attr_id, tid);
1314     hdr->attr_mod = cpu_to_be32(attr_mod);
1315 }
1316
1317 static void cm_format_req(struct cm_req_msg *req_msg,
1318               struct cm_id_private *cm_id_priv,
1319               struct ib_cm_req_param *param)
1320 {
1321     struct sa_path_rec *pri_path = param->primary_path;
1322     struct sa_path_rec *alt_path = param->alternate_path;
1323     bool pri_ext = false;
1324
1325     if (pri_path->rec_type == SA_PATH_REC_TYPE_OPA)
1326         pri_ext = opa_is_extended_lid(pri_path->opa.dlid,
1327                           pri_path->opa.slid);
1328
1329     cm_format_mad_ece_hdr(&req_msg->hdr, CM_REQ_ATTR_ID,
1330                   cm_form_tid(cm_id_priv), param->ece.attr_mod);
1331
1332     IBA_SET(CM_REQ_LOCAL_COMM_ID, req_msg,
1333         be32_to_cpu(cm_id_priv->id.local_id));
1334     IBA_SET(CM_REQ_SERVICE_ID, req_msg, be64_to_cpu(param->service_id));
1335     IBA_SET(CM_REQ_LOCAL_CA_GUID, req_msg,
1336         be64_to_cpu(cm_id_priv->id.device->node_guid));
1337     IBA_SET(CM_REQ_LOCAL_QPN, req_msg, param->qp_num);
1338     IBA_SET(CM_REQ_INITIATOR_DEPTH, req_msg, param->initiator_depth);
1339     IBA_SET(CM_REQ_REMOTE_CM_RESPONSE_TIMEOUT, req_msg,
1340         param->remote_cm_response_timeout);
1341     cm_req_set_qp_type(req_msg, param->qp_type);
1342     IBA_SET(CM_REQ_END_TO_END_FLOW_CONTROL, req_msg, param->flow_control);
1343     IBA_SET(CM_REQ_STARTING_PSN, req_msg, param->starting_psn);
1344     IBA_SET(CM_REQ_LOCAL_CM_RESPONSE_TIMEOUT, req_msg,
1345         param->local_cm_response_timeout);
1346     IBA_SET(CM_REQ_PARTITION_KEY, req_msg,
1347         be16_to_cpu(param->primary_path->pkey));
1348     IBA_SET(CM_REQ_PATH_PACKET_PAYLOAD_MTU, req_msg,
1349         param->primary_path->mtu);
1350     IBA_SET(CM_REQ_MAX_CM_RETRIES, req_msg, param->max_cm_retries);
1351
1352     if (param->qp_type != IB_QPT_XRC_INI) {
1353         IBA_SET(CM_REQ_RESPONDER_RESOURCES, req_msg,
1354             param->responder_resources);
1355         IBA_SET(CM_REQ_RETRY_COUNT, req_msg, param->retry_count);
1356         IBA_SET(CM_REQ_RNR_RETRY_COUNT, req_msg,
1357             param->rnr_retry_count);
1358         IBA_SET(CM_REQ_SRQ, req_msg, param->srq);
1359     }
1360
1361     *IBA_GET_MEM_PTR(CM_REQ_PRIMARY_LOCAL_PORT_GID, req_msg) =
1362         pri_path->sgid;
1363     *IBA_GET_MEM_PTR(CM_REQ_PRIMARY_REMOTE_PORT_GID, req_msg) =
1364         pri_path->dgid;
1365     if (pri_ext) {
1366         IBA_GET_MEM_PTR(CM_REQ_PRIMARY_LOCAL_PORT_GID, req_msg)
1367             ->global.interface_id =
1368             OPA_MAKE_ID(be32_to_cpu(pri_path->opa.slid));
1369         IBA_GET_MEM_PTR(CM_REQ_PRIMARY_REMOTE_PORT_GID, req_msg)
1370             ->global.interface_id =
1371             OPA_MAKE_ID(be32_to_cpu(pri_path->opa.dlid));
1372     }
1373     if (pri_path->hop_limit <= 1) {
1374         IBA_SET(CM_REQ_PRIMARY_LOCAL_PORT_LID, req_msg,
1375             be16_to_cpu(pri_ext ? 0 :
1376                           htons(ntohl(sa_path_get_slid(
1377                               pri_path)))));
1378         IBA_SET(CM_REQ_PRIMARY_REMOTE_PORT_LID, req_msg,
1379             be16_to_cpu(pri_ext ? 0 :
1380                           htons(ntohl(sa_path_get_dlid(
1381                               pri_path)))));
1382     } else {
1383         /* Work-around until there's a way to obtain remote LID info */
1384         IBA_SET(CM_REQ_PRIMARY_LOCAL_PORT_LID, req_msg,
1385             be16_to_cpu(IB_LID_PERMISSIVE));
1386         IBA_SET(CM_REQ_PRIMARY_REMOTE_PORT_LID, req_msg,
1387             be16_to_cpu(IB_LID_PERMISSIVE));
1388     }
1389     IBA_SET(CM_REQ_PRIMARY_FLOW_LABEL, req_msg,
1390         be32_to_cpu(pri_path->flow_label));
1391     IBA_SET(CM_REQ_PRIMARY_PACKET_RATE, req_msg, pri_path->rate);
1392     IBA_SET(CM_REQ_PRIMARY_TRAFFIC_CLASS, req_msg, pri_path->traffic_class);
1393     IBA_SET(CM_REQ_PRIMARY_HOP_LIMIT, req_msg, pri_path->hop_limit);
1394     IBA_SET(CM_REQ_PRIMARY_SL, req_msg, pri_path->sl);
1395     IBA_SET(CM_REQ_PRIMARY_SUBNET_LOCAL, req_msg,
1396         (pri_path->hop_limit <= 1));
1397     IBA_SET(CM_REQ_PRIMARY_LOCAL_ACK_TIMEOUT, req_msg,
1398         cm_ack_timeout(cm_id_priv->av.port->cm_dev->ack_delay,
1399                    pri_path->packet_life_time));
1400
1401     if (alt_path) {
1402         bool alt_ext = false;
1403
1404         if (alt_path->rec_type == SA_PATH_REC_TYPE_OPA)
1405             alt_ext = opa_is_extended_lid(alt_path->opa.dlid,
1406                               alt_path->opa.slid);
1407
1408         *IBA_GET_MEM_PTR(CM_REQ_ALTERNATE_LOCAL_PORT_GID, req_msg) =
1409             alt_path->sgid;
1410         *IBA_GET_MEM_PTR(CM_REQ_ALTERNATE_REMOTE_PORT_GID, req_msg) =
1411             alt_path->dgid;
1412         if (alt_ext) {
1413             IBA_GET_MEM_PTR(CM_REQ_ALTERNATE_LOCAL_PORT_GID,
1414                     req_msg)
1415                 ->global.interface_id =
1416                 OPA_MAKE_ID(be32_to_cpu(alt_path->opa.slid));
1417             IBA_GET_MEM_PTR(CM_REQ_ALTERNATE_REMOTE_PORT_GID,
1418                     req_msg)
1419                 ->global.interface_id =
1420                 OPA_MAKE_ID(be32_to_cpu(alt_path->opa.dlid));
1421         }
1422         if (alt_path->hop_limit <= 1) {
1423             IBA_SET(CM_REQ_ALTERNATE_LOCAL_PORT_LID, req_msg,
1424                 be16_to_cpu(
1425                     alt_ext ? 0 :
1426                           htons(ntohl(sa_path_get_slid(
1427                               alt_path)))));
1428             IBA_SET(CM_REQ_ALTERNATE_REMOTE_PORT_LID, req_msg,
1429                 be16_to_cpu(
1430                     alt_ext ? 0 :
1431                           htons(ntohl(sa_path_get_dlid(
1432                               alt_path)))));
1433         } else {
1434             IBA_SET(CM_REQ_ALTERNATE_LOCAL_PORT_LID, req_msg,
1435                 be16_to_cpu(IB_LID_PERMISSIVE));
1436             IBA_SET(CM_REQ_ALTERNATE_REMOTE_PORT_LID, req_msg,
1437                 be16_to_cpu(IB_LID_PERMISSIVE));
1438         }
1439         IBA_SET(CM_REQ_ALTERNATE_FLOW_LABEL, req_msg,
1440             be32_to_cpu(alt_path->flow_label));
1441         IBA_SET(CM_REQ_ALTERNATE_PACKET_RATE, req_msg, alt_path->rate);
1442         IBA_SET(CM_REQ_ALTERNATE_TRAFFIC_CLASS, req_msg,
1443             alt_path->traffic_class);
1444         IBA_SET(CM_REQ_ALTERNATE_HOP_LIMIT, req_msg,
1445             alt_path->hop_limit);
1446         IBA_SET(CM_REQ_ALTERNATE_SL, req_msg, alt_path->sl);
1447         IBA_SET(CM_REQ_ALTERNATE_SUBNET_LOCAL, req_msg,
1448             (alt_path->hop_limit <= 1));
1449         IBA_SET(CM_REQ_ALTERNATE_LOCAL_ACK_TIMEOUT, req_msg,
1450             cm_ack_timeout(cm_id_priv->av.port->cm_dev->ack_delay,
1451                        alt_path->packet_life_time));
1452     }
1453     IBA_SET(CM_REQ_VENDOR_ID, req_msg, param->ece.vendor_id);
1454
1455     if (param->private_data && param->private_data_len)
1456         IBA_SET_MEM(CM_REQ_PRIVATE_DATA, req_msg, param->private_data,
1457                 param->private_data_len);
1458 }
1459
1460 static int cm_validate_req_param(struct ib_cm_req_param *param)
1461 {
1462     if (!param->primary_path)
1463         return -EINVAL;
1464
1465     if (param->qp_type != IB_QPT_RC && param->qp_type != IB_QPT_UC &&
1466         param->qp_type != IB_QPT_XRC_INI)
1467         return -EINVAL;
1468
1469     if (param->private_data &&
1470         param->private_data_len > IB_CM_REQ_PRIVATE_DATA_SIZE)
1471         return -EINVAL;
1472
1473     if (param->alternate_path &&
1474         (param->alternate_path->pkey != param->primary_path->pkey ||
1475          param->alternate_path->mtu != param->primary_path->mtu))
1476         return -EINVAL;
1477
1478     return 0;
1479 }
1480
1481 int ib_send_cm_req(struct ib_cm_id *cm_id,
1482            struct ib_cm_req_param *param)
1483 {
1484     struct cm_av av = {}, alt_av = {};
1485     struct cm_id_private *cm_id_priv;
1486     struct ib_mad_send_buf *msg;
1487     struct cm_req_msg *req_msg;
1488     unsigned long flags;
1489     int ret;
1490
1491     ret = cm_validate_req_param(param);
1492     if (ret)
1493         return ret;
1494
1495     /* Verify that we're not in timewait. */
1496     cm_id_priv = container_of(cm_id, struct cm_id_private, id);
1497     spin_lock_irqsave(&cm_id_priv->lock, flags);
1498     if (cm_id->state != IB_CM_IDLE || WARN_ON(cm_id_priv->timewait_info)) {
1499         spin_unlock_irqrestore(&cm_id_priv->lock, flags);
1500         return -EINVAL;
1501     }
1502     spin_unlock_irqrestore(&cm_id_priv->lock, flags);
1503
1504     cm_id_priv->timewait_info = cm_create_timewait_info(cm_id_priv->
1505                                 id.local_id);
1506     if (IS_ERR(cm_id_priv->timewait_info)) {
1507         ret = PTR_ERR(cm_id_priv->timewait_info);
1508         cm_id_priv->timewait_info = NULL;
1509         return ret;
1510     }
1511
1512     ret = cm_init_av_by_path(param->primary_path,
1513                  param->ppath_sgid_attr, &av);
1514     if (ret)
1515         return ret;
1516     if (param->alternate_path) {
1517         ret = cm_init_av_by_path(param->alternate_path, NULL,
1518                      &alt_av);
1519         if (ret) {
1520             cm_destroy_av(&av);
1521             return ret;
1522         }
1523     }
1524     cm_id->service_id = param->service_id;
1525     cm_id->service_mask = ~cpu_to_be64(0);
1526     cm_id_priv->timeout_ms = cm_convert_to_ms(
1527                     param->primary_path->packet_life_time) * 2 +
1528                  cm_convert_to_ms(
1529                     param->remote_cm_response_timeout);
1530     cm_id_priv->max_cm_retries = param->max_cm_retries;
1531     cm_id_priv->initiator_depth = param->initiator_depth;
1532     cm_id_priv->responder_resources = param->responder_resources;
1533     cm_id_priv->retry_count = param->retry_count;
1534     cm_id_priv->path_mtu = param->primary_path->mtu;
1535     cm_id_priv->pkey = param->primary_path->pkey;
1536     cm_id_priv->qp_type = param->qp_type;
1537
1538     spin_lock_irqsave(&cm_id_priv->lock, flags);
1539
1540     cm_move_av_from_path(&cm_id_priv->av, &av);
1541     if (param->alternate_path)
1542         cm_move_av_from_path(&cm_id_priv->alt_av, &alt_av);
1543
1544     msg = cm_alloc_priv_msg(cm_id_priv);
1545     if (IS_ERR(msg)) {
1546         ret = PTR_ERR(msg);
1547         goto out_unlock;
1548     }
1549
1550     req_msg = (struct cm_req_msg *)msg->mad;
1551     cm_format_req(req_msg, cm_id_priv, param);
1552     cm_id_priv->tid = req_msg->hdr.tid;
1553     msg->timeout_ms = cm_id_priv->timeout_ms;
1554     msg->context[1] = (void *)(unsigned long)IB_CM_REQ_SENT;
1555
1556     cm_id_priv->local_qpn = cpu_to_be32(IBA_GET(CM_REQ_LOCAL_QPN, req_msg));
1557     cm_id_priv->rq_psn = cpu_to_be32(IBA_GET(CM_REQ_STARTING_PSN, req_msg));
1558
1559     trace_icm_send_req(&cm_id_priv->id);
1560     ret = ib_post_send_mad(msg, NULL);
1561     if (ret)
1562         goto out_free;
1563     BUG_ON(cm_id->state != IB_CM_IDLE);
1564     cm_id->state = IB_CM_REQ_SENT;
1565     spin_unlock_irqrestore(&cm_id_priv->lock, flags);
1566     return 0;
1567 out_free:
1568     cm_free_priv_msg(msg);
1569 out_unlock:
1570     spin_unlock_irqrestore(&cm_id_priv->lock, flags);
1571     return ret;
1572 }
1573 EXPORT_SYMBOL(ib_send_cm_req);
1574
1575 static int cm_issue_rej(struct cm_port *port,
1576             struct ib_mad_recv_wc *mad_recv_wc,
1577             enum ib_cm_rej_reason reason,
1578             enum cm_msg_response msg_rejected,
1579             void *ari, u8 ari_length)
1580 {
1581     struct ib_mad_send_buf *msg = NULL;
1582     struct cm_rej_msg *rej_msg, *rcv_msg;
1583     int ret;
1584
1585     ret = cm_alloc_response_msg(port, mad_recv_wc, &msg);
1586     if (ret)
1587         return ret;
1588
1589     /* We just need common CM header information.  Cast to any message. */
1590     rcv_msg = (struct cm_rej_msg *) mad_recv_wc->recv_buf.mad;
1591     rej_msg = (struct cm_rej_msg *) msg->mad;
1592
1593     cm_format_mad_hdr(&rej_msg->hdr, CM_REJ_ATTR_ID, rcv_msg->hdr.tid);
1594     IBA_SET(CM_REJ_REMOTE_COMM_ID, rej_msg,
1595         IBA_GET(CM_REJ_LOCAL_COMM_ID, rcv_msg));
1596     IBA_SET(CM_REJ_LOCAL_COMM_ID, rej_msg,
1597         IBA_GET(CM_REJ_REMOTE_COMM_ID, rcv_msg));
1598     IBA_SET(CM_REJ_MESSAGE_REJECTED, rej_msg, msg_rejected);
1599     IBA_SET(CM_REJ_REASON, rej_msg, reason);
1600
1601     if (ari && ari_length) {
1602         IBA_SET(CM_REJ_REJECTED_INFO_LENGTH, rej_msg, ari_length);
1603         IBA_SET_MEM(CM_REJ_ARI, rej_msg, ari, ari_length);
1604     }
1605
1606     trace_icm_issue_rej(
1607         IBA_GET(CM_REJ_LOCAL_COMM_ID, rcv_msg),
1608         IBA_GET(CM_REJ_REMOTE_COMM_ID, rcv_msg));
1609     ret = ib_post_send_mad(msg, NULL);
1610     if (ret)
1611         cm_free_response_msg(msg);
1612
1613     return ret;
1614 }
1615
1616 static bool cm_req_has_alt_path(struct cm_req_msg *req_msg)
1617 {
1618     return ((cpu_to_be16(
1619             IBA_GET(CM_REQ_ALTERNATE_LOCAL_PORT_LID, req_msg))) ||
1620         (ib_is_opa_gid(IBA_GET_MEM_PTR(CM_REQ_ALTERNATE_LOCAL_PORT_GID,
1621                            req_msg))));
1622 }
1623
1624 static void cm_path_set_rec_type(struct ib_device *ib_device, u32 port_num,
1625                  struct sa_path_rec *path, union ib_gid *gid)
1626 {
1627     if (ib_is_opa_gid(gid) && rdma_cap_opa_ah(ib_device, port_num))
1628         path->rec_type = SA_PATH_REC_TYPE_OPA;
1629     else
1630         path->rec_type = SA_PATH_REC_TYPE_IB;
1631 }
1632
1633 static void cm_format_path_lid_from_req(struct cm_req_msg *req_msg,
1634                     struct sa_path_rec *primary_path,
1635                     struct sa_path_rec *alt_path)
1636 {
1637     u32 lid;
1638
1639     if (primary_path->rec_type != SA_PATH_REC_TYPE_OPA) {
1640         sa_path_set_dlid(primary_path,
1641                  IBA_GET(CM_REQ_PRIMARY_LOCAL_PORT_LID,
1642                      req_msg));
1643         sa_path_set_slid(primary_path,
1644                  IBA_GET(CM_REQ_PRIMARY_REMOTE_PORT_LID,
1645                      req_msg));
1646     } else {
1647         lid = opa_get_lid_from_gid(IBA_GET_MEM_PTR(
1648             CM_REQ_PRIMARY_LOCAL_PORT_GID, req_msg));
1649         sa_path_set_dlid(primary_path, lid);
1650
1651         lid = opa_get_lid_from_gid(IBA_GET_MEM_PTR(
1652             CM_REQ_PRIMARY_REMOTE_PORT_GID, req_msg));
1653         sa_path_set_slid(primary_path, lid);
1654     }
1655
1656     if (!cm_req_has_alt_path(req_msg))
1657         return;
1658
1659     if (alt_path->rec_type != SA_PATH_REC_TYPE_OPA) {
1660         sa_path_set_dlid(alt_path,
1661                  IBA_GET(CM_REQ_ALTERNATE_LOCAL_PORT_LID,
1662                      req_msg));
1663         sa_path_set_slid(alt_path,
1664                  IBA_GET(CM_REQ_ALTERNATE_REMOTE_PORT_LID,
1665                      req_msg));
1666     } else {
1667         lid = opa_get_lid_from_gid(IBA_GET_MEM_PTR(
1668             CM_REQ_ALTERNATE_LOCAL_PORT_GID, req_msg));
1669         sa_path_set_dlid(alt_path, lid);
1670
1671         lid = opa_get_lid_from_gid(IBA_GET_MEM_PTR(
1672             CM_REQ_ALTERNATE_REMOTE_PORT_GID, req_msg));
1673         sa_path_set_slid(alt_path, lid);
1674     }
1675 }
1676
1677 static void cm_format_paths_from_req(struct cm_req_msg *req_msg,
1678                      struct sa_path_rec *primary_path,
1679                      struct sa_path_rec *alt_path)
1680 {
1681     primary_path->dgid =
1682         *IBA_GET_MEM_PTR(CM_REQ_PRIMARY_LOCAL_PORT_GID, req_msg);
1683     primary_path->sgid =
1684         *IBA_GET_MEM_PTR(CM_REQ_PRIMARY_REMOTE_PORT_GID, req_msg);
1685     primary_path->flow_label =
1686         cpu_to_be32(IBA_GET(CM_REQ_PRIMARY_FLOW_LABEL, req_msg));
1687     primary_path->hop_limit = IBA_GET(CM_REQ_PRIMARY_HOP_LIMIT, req_msg);
1688     primary_path->traffic_class =
1689         IBA_GET(CM_REQ_PRIMARY_TRAFFIC_CLASS, req_msg);
1690     primary_path->reversible = 1;
1691     primary_path->pkey =
1692         cpu_to_be16(IBA_GET(CM_REQ_PARTITION_KEY, req_msg));
1693     primary_path->sl = IBA_GET(CM_REQ_PRIMARY_SL, req_msg);
1694     primary_path->mtu_selector = IB_SA_EQ;
1695     primary_path->mtu = IBA_GET(CM_REQ_PATH_PACKET_PAYLOAD_MTU, req_msg);
1696     primary_path->rate_selector = IB_SA_EQ;
1697     primary_path->rate = IBA_GET(CM_REQ_PRIMARY_PACKET_RATE, req_msg);
1698     primary_path->packet_life_time_selector = IB_SA_EQ;
1699     primary_path->packet_life_time =
1700         IBA_GET(CM_REQ_PRIMARY_LOCAL_ACK_TIMEOUT, req_msg);
1701     primary_path->packet_life_time -= (primary_path->packet_life_time > 0);
1702     primary_path->service_id =
1703         cpu_to_be64(IBA_GET(CM_REQ_SERVICE_ID, req_msg));
1704     if (sa_path_is_roce(primary_path))
1705         primary_path->roce.route_resolved = false;
1706
1707     if (cm_req_has_alt_path(req_msg)) {
1708         alt_path->dgid = *IBA_GET_MEM_PTR(
1709             CM_REQ_ALTERNATE_LOCAL_PORT_GID, req_msg);
1710         alt_path->sgid = *IBA_GET_MEM_PTR(
1711             CM_REQ_ALTERNATE_REMOTE_PORT_GID, req_msg);
1712         alt_path->flow_label = cpu_to_be32(
1713             IBA_GET(CM_REQ_ALTERNATE_FLOW_LABEL, req_msg));
1714         alt_path->hop_limit =
1715             IBA_GET(CM_REQ_ALTERNATE_HOP_LIMIT, req_msg);
1716         alt_path->traffic_class =
1717             IBA_GET(CM_REQ_ALTERNATE_TRAFFIC_CLASS, req_msg);
1718         alt_path->reversible = 1;
1719         alt_path->pkey =
1720             cpu_to_be16(IBA_GET(CM_REQ_PARTITION_KEY, req_msg));
1721         alt_path->sl = IBA_GET(CM_REQ_ALTERNATE_SL, req_msg);
1722         alt_path->mtu_selector = IB_SA_EQ;
1723         alt_path->mtu =
1724             IBA_GET(CM_REQ_PATH_PACKET_PAYLOAD_MTU, req_msg);
1725         alt_path->rate_selector = IB_SA_EQ;
1726         alt_path->rate = IBA_GET(CM_REQ_ALTERNATE_PACKET_RATE, req_msg);
1727         alt_path->packet_life_time_selector = IB_SA_EQ;
1728         alt_path->packet_life_time =
1729             IBA_GET(CM_REQ_ALTERNATE_LOCAL_ACK_TIMEOUT, req_msg);
1730         alt_path->packet_life_time -= (alt_path->packet_life_time > 0);
1731         alt_path->service_id =
1732             cpu_to_be64(IBA_GET(CM_REQ_SERVICE_ID, req_msg));
1733
1734         if (sa_path_is_roce(alt_path))
1735             alt_path->roce.route_resolved = false;
1736     }
1737     cm_format_path_lid_from_req(req_msg, primary_path, alt_path);
1738 }
1739
1740 static u16 cm_get_bth_pkey(struct cm_work *work)
1741 {
1742     struct ib_device *ib_dev = work->port->cm_dev->ib_device;
1743     u32 port_num = work->port->port_num;
1744     u16 pkey_index = work->mad_recv_wc->wc->pkey_index;
1745     u16 pkey;
1746     int ret;
1747
1748     ret = ib_get_cached_pkey(ib_dev, port_num, pkey_index, &pkey);
1749     if (ret) {
1750         dev_warn_ratelimited(&ib_dev->dev, "ib_cm: Couldn't retrieve pkey for incoming request (port %u, pkey index %u). %d\n",
1751                      port_num, pkey_index, ret);
1752         return 0;
1753     }
1754
1755     return pkey;
1756 }
1757
1758 /**
1759  * cm_opa_to_ib_sgid - Convert OPA SGID to IB SGID
1760  * ULPs (such as IPoIB) do not understand OPA GIDs and will
1761  * reject them as the local_gid will not match the sgid. Therefore,
1762  * change the pathrec's SGID to an IB SGID.
1763  *
1764  * @work: Work completion
1765  * @path: Path record
1766  */
1767 static void cm_opa_to_ib_sgid(struct cm_work *work,
1768                   struct sa_path_rec *path)
1769 {
1770     struct ib_device *dev = work->port->cm_dev->ib_device;
1771     u32 port_num = work->port->port_num;
1772
1773     if (rdma_cap_opa_ah(dev, port_num) &&
1774         (ib_is_opa_gid(&path->sgid))) {
1775         union ib_gid sgid;
1776
1777         if (rdma_query_gid(dev, port_num, 0, &sgid)) {
1778             dev_warn(&dev->dev,
1779                  "Error updating sgid in CM request\n");
1780             return;
1781         }
1782
1783         path->sgid = sgid;
1784     }
1785 }
1786
1787 static void cm_format_req_event(struct cm_work *work,
1788                 struct cm_id_private *cm_id_priv,
1789                 struct ib_cm_id *listen_id)
1790 {
1791     struct cm_req_msg *req_msg;
1792     struct ib_cm_req_event_param *param;
1793
1794     req_msg = (struct cm_req_msg *)work->mad_recv_wc->recv_buf.mad;
1795     param = &work->cm_event.param.req_rcvd;
1796     param->listen_id = listen_id;
1797     param->bth_pkey = cm_get_bth_pkey(work);
1798     param->port = cm_id_priv->av.port->port_num;
1799     param->primary_path = &work->path[0];
1800     cm_opa_to_ib_sgid(work, param->primary_path);
1801     if (cm_req_has_alt_path(req_msg)) {
1802         param->alternate_path = &work->path[1];
1803         cm_opa_to_ib_sgid(work, param->alternate_path);
1804     } else {
1805         param->alternate_path = NULL;
1806     }
1807     param->remote_ca_guid =
1808         cpu_to_be64(IBA_GET(CM_REQ_LOCAL_CA_GUID, req_msg));
1809     param->remote_qkey = IBA_GET(CM_REQ_LOCAL_Q_KEY, req_msg);
1810     param->remote_qpn = IBA_GET(CM_REQ_LOCAL_QPN, req_msg);
1811     param->qp_type = cm_req_get_qp_type(req_msg);
1812     param->starting_psn = IBA_GET(CM_REQ_STARTING_PSN, req_msg);
1813     param->responder_resources = IBA_GET(CM_REQ_INITIATOR_DEPTH, req_msg);
1814     param->initiator_depth = IBA_GET(CM_REQ_RESPONDER_RESOURCES, req_msg);
1815     param->local_cm_response_timeout =
1816         IBA_GET(CM_REQ_REMOTE_CM_RESPONSE_TIMEOUT, req_msg);
1817     param->flow_control = IBA_GET(CM_REQ_END_TO_END_FLOW_CONTROL, req_msg);
1818     param->remote_cm_response_timeout =
1819         IBA_GET(CM_REQ_LOCAL_CM_RESPONSE_TIMEOUT, req_msg);
1820     param->retry_count = IBA_GET(CM_REQ_RETRY_COUNT, req_msg);
1821     param->rnr_retry_count = IBA_GET(CM_REQ_RNR_RETRY_COUNT, req_msg);
1822     param->srq = IBA_GET(CM_REQ_SRQ, req_msg);
1823     param->ppath_sgid_attr = cm_id_priv->av.ah_attr.grh.sgid_attr;
1824     param->ece.vendor_id = IBA_GET(CM_REQ_VENDOR_ID, req_msg);
1825     param->ece.attr_mod = be32_to_cpu(req_msg->hdr.attr_mod);
1826
1827     work->cm_event.private_data =
1828         IBA_GET_MEM_PTR(CM_REQ_PRIVATE_DATA, req_msg);
1829 }
1830
1831 static void cm_process_work(struct cm_id_private *cm_id_priv,
1832                 struct cm_work *work)
1833 {
1834     int ret;
1835
1836     /* We will typically only have the current event to report. */
1837     ret = cm_id_priv->id.cm_handler(&cm_id_priv->id, &work->cm_event);
1838     cm_free_work(work);
1839
1840     while (!ret && !atomic_add_negative(-1, &cm_id_priv->work_count)) {
1841         spin_lock_irq(&cm_id_priv->lock);
1842         work = cm_dequeue_work(cm_id_priv);
1843         spin_unlock_irq(&cm_id_priv->lock);
1844         if (!work)
1845             return;
1846
1847         ret = cm_id_priv->id.cm_handler(&cm_id_priv->id,
1848                         &work->cm_event);
1849         cm_free_work(work);
1850     }
1851     cm_deref_id(cm_id_priv);
1852     if (ret)
1853         cm_destroy_id(&cm_id_priv->id, ret);
1854 }
1855
1856 static void cm_format_mra(struct cm_mra_msg *mra_msg,
1857               struct cm_id_private *cm_id_priv,
1858               enum cm_msg_response msg_mraed, u8 service_timeout,
1859               const void *private_data, u8 private_data_len)
1860 {
1861     cm_format_mad_hdr(&mra_msg->hdr, CM_MRA_ATTR_ID, cm_id_priv->tid);
1862     IBA_SET(CM_MRA_MESSAGE_MRAED, mra_msg, msg_mraed);
1863     IBA_SET(CM_MRA_LOCAL_COMM_ID, mra_msg,
1864         be32_to_cpu(cm_id_priv->id.local_id));
1865     IBA_SET(CM_MRA_REMOTE_COMM_ID, mra_msg,
1866         be32_to_cpu(cm_id_priv->id.remote_id));
1867     IBA_SET(CM_MRA_SERVICE_TIMEOUT, mra_msg, service_timeout);
1868
1869     if (private_data && private_data_len)
1870         IBA_SET_MEM(CM_MRA_PRIVATE_DATA, mra_msg, private_data,
1871                 private_data_len);
1872 }
1873
1874 static void cm_format_rej(struct cm_rej_msg *rej_msg,
1875               struct cm_id_private *cm_id_priv,
1876               enum ib_cm_rej_reason reason, void *ari,
1877               u8 ari_length, const void *private_data,
1878               u8 private_data_len, enum ib_cm_state state)
1879 {
1880     lockdep_assert_held(&cm_id_priv->lock);
1881
1882     cm_format_mad_hdr(&rej_msg->hdr, CM_REJ_ATTR_ID, cm_id_priv->tid);
1883     IBA_SET(CM_REJ_REMOTE_COMM_ID, rej_msg,
1884         be32_to_cpu(cm_id_priv->id.remote_id));
1885
1886     switch (state) {
1887     case IB_CM_REQ_RCVD:
1888         IBA_SET(CM_REJ_LOCAL_COMM_ID, rej_msg, be32_to_cpu(0));
1889         IBA_SET(CM_REJ_MESSAGE_REJECTED, rej_msg, CM_MSG_RESPONSE_REQ);
1890         break;
1891     case IB_CM_MRA_REQ_SENT:
1892         IBA_SET(CM_REJ_LOCAL_COMM_ID, rej_msg,
1893             be32_to_cpu(cm_id_priv->id.local_id));
1894         IBA_SET(CM_REJ_MESSAGE_REJECTED, rej_msg, CM_MSG_RESPONSE_REQ);
1895         break;
1896     case IB_CM_REP_RCVD:
1897     case IB_CM_MRA_REP_SENT:
1898         IBA_SET(CM_REJ_LOCAL_COMM_ID, rej_msg,
1899             be32_to_cpu(cm_id_priv->id.local_id));
1900         IBA_SET(CM_REJ_MESSAGE_REJECTED, rej_msg, CM_MSG_RESPONSE_REP);
1901         break;
1902     default:
1903         IBA_SET(CM_REJ_LOCAL_COMM_ID, rej_msg,
1904             be32_to_cpu(cm_id_priv->id.local_id));
1905         IBA_SET(CM_REJ_MESSAGE_REJECTED, rej_msg,
1906             CM_MSG_RESPONSE_OTHER);
1907         break;
1908     }
1909
1910     IBA_SET(CM_REJ_REASON, rej_msg, reason);
1911     if (ari && ari_length) {
1912         IBA_SET(CM_REJ_REJECTED_INFO_LENGTH, rej_msg, ari_length);
1913         IBA_SET_MEM(CM_REJ_ARI, rej_msg, ari, ari_length);
1914     }
1915
1916     if (private_data && private_data_len)
1917         IBA_SET_MEM(CM_REJ_PRIVATE_DATA, rej_msg, private_data,
1918                 private_data_len);
1919 }
1920
1921 static void cm_dup_req_handler(struct cm_work *work,
1922                    struct cm_id_private *cm_id_priv)
1923 {
1924     struct ib_mad_send_buf *msg = NULL;
1925     int ret;
1926
1927     atomic_long_inc(
1928         &work->port->counters[CM_RECV_DUPLICATES][CM_REQ_COUNTER]);
1929
1930     /* Quick state check to discard duplicate REQs. */
1931     spin_lock_irq(&cm_id_priv->lock);
1932     if (cm_id_priv->id.state == IB_CM_REQ_RCVD) {
1933         spin_unlock_irq(&cm_id_priv->lock);
1934         return;
1935     }
1936     spin_unlock_irq(&cm_id_priv->lock);
1937
1938     ret = cm_alloc_response_msg(work->port, work->mad_recv_wc, &msg);
1939     if (ret)
1940         return;
1941
1942     spin_lock_irq(&cm_id_priv->lock);
1943     switch (cm_id_priv->id.state) {
1944     case IB_CM_MRA_REQ_SENT:
1945         cm_format_mra((struct cm_mra_msg *) msg->mad, cm_id_priv,
1946                   CM_MSG_RESPONSE_REQ, cm_id_priv->service_timeout,
1947                   cm_id_priv->private_data,
1948                   cm_id_priv->private_data_len);
1949         break;
1950     case IB_CM_TIMEWAIT:
1951         cm_format_rej((struct cm_rej_msg *)msg->mad, cm_id_priv,
1952                   IB_CM_REJ_STALE_CONN, NULL, 0, NULL, 0,
1953                   IB_CM_TIMEWAIT);
1954         break;
1955     default:
1956         goto unlock;
1957     }
1958     spin_unlock_irq(&cm_id_priv->lock);
1959
1960     trace_icm_send_dup_req(&cm_id_priv->id);
1961     ret = ib_post_send_mad(msg, NULL);
1962     if (ret)
1963         goto free;
1964     return;
1965
1966 unlock: spin_unlock_irq(&cm_id_priv->lock);
1967 free:   cm_free_response_msg(msg);
1968 }
1969
1970 static struct cm_id_private *cm_match_req(struct cm_work *work,
1971                       struct cm_id_private *cm_id_priv)
1972 {
1973     struct cm_id_private *listen_cm_id_priv, *cur_cm_id_priv;
1974     struct cm_timewait_info *timewait_info;
1975     struct cm_req_msg *req_msg;
1976
1977     req_msg = (struct cm_req_msg *)work->mad_recv_wc->recv_buf.mad;
1978
1979     /* Check for possible duplicate REQ. */
1980     spin_lock_irq(&cm.lock);
1981     timewait_info = cm_insert_remote_id(cm_id_priv->timewait_info);
1982     if (timewait_info) {
1983         cur_cm_id_priv = cm_acquire_id(timewait_info->work.local_id,
1984                        timewait_info->work.remote_id);
1985         spin_unlock_irq(&cm.lock);
1986         if (cur_cm_id_priv) {
1987             cm_dup_req_handler(work, cur_cm_id_priv);
1988             cm_deref_id(cur_cm_id_priv);
1989         }
1990         return NULL;
1991     }
1992
1993     /* Check for stale connections. */
1994     timewait_info = cm_insert_remote_qpn(cm_id_priv->timewait_info);
1995     if (timewait_info) {
1996         cm_remove_remote(cm_id_priv);
1997         cur_cm_id_priv = cm_acquire_id(timewait_info->work.local_id,
1998                        timewait_info->work.remote_id);
1999
2000         spin_unlock_irq(&cm.lock);
2001         cm_issue_rej(work->port, work->mad_recv_wc,
2002                  IB_CM_REJ_STALE_CONN, CM_MSG_RESPONSE_REQ,
2003                  NULL, 0);
2004         if (cur_cm_id_priv) {
2005             ib_send_cm_dreq(&cur_cm_id_priv->id, NULL, 0);
2006             cm_deref_id(cur_cm_id_priv);
2007         }
2008         return NULL;
2009     }
2010
2011     /* Find matching listen request. */
2012     listen_cm_id_priv = cm_find_listen(
2013         cm_id_priv->id.device,
2014         cpu_to_be64(IBA_GET(CM_REQ_SERVICE_ID, req_msg)));
2015     if (!listen_cm_id_priv) {
2016         cm_remove_remote(cm_id_priv);
2017         spin_unlock_irq(&cm.lock);
2018         cm_issue_rej(work->port, work->mad_recv_wc,
2019                  IB_CM_REJ_INVALID_SERVICE_ID, CM_MSG_RESPONSE_REQ,
2020                  NULL, 0);
2021         return NULL;
2022     }
2023     spin_unlock_irq(&cm.lock);
2024     return listen_cm_id_priv;
2025 }
2026
2027 /*
2028  * Work-around for inter-subnet connections.  If the LIDs are permissive,
2029  * we need to override the LID/SL data in the REQ with the LID information
2030  * in the work completion.
2031  */
2032 static void cm_process_routed_req(struct cm_req_msg *req_msg, struct ib_wc *wc)
2033 {
2034     if (!IBA_GET(CM_REQ_PRIMARY_SUBNET_LOCAL, req_msg)) {
2035         if (cpu_to_be16(IBA_GET(CM_REQ_PRIMARY_LOCAL_PORT_LID,
2036                     req_msg)) == IB_LID_PERMISSIVE) {
2037             IBA_SET(CM_REQ_PRIMARY_LOCAL_PORT_LID, req_msg,
2038                 be16_to_cpu(ib_lid_be16(wc->slid)));
2039             IBA_SET(CM_REQ_PRIMARY_SL, req_msg, wc->sl);
2040         }
2041
2042         if (cpu_to_be16(IBA_GET(CM_REQ_PRIMARY_REMOTE_PORT_LID,
2043                     req_msg)) == IB_LID_PERMISSIVE)
2044             IBA_SET(CM_REQ_PRIMARY_REMOTE_PORT_LID, req_msg,
2045                 wc->dlid_path_bits);
2046     }
2047
2048     if (!IBA_GET(CM_REQ_ALTERNATE_SUBNET_LOCAL, req_msg)) {
2049         if (cpu_to_be16(IBA_GET(CM_REQ_ALTERNATE_LOCAL_PORT_LID,
2050                     req_msg)) == IB_LID_PERMISSIVE) {
2051             IBA_SET(CM_REQ_ALTERNATE_LOCAL_PORT_LID, req_msg,
2052                 be16_to_cpu(ib_lid_be16(wc->slid)));
2053             IBA_SET(CM_REQ_ALTERNATE_SL, req_msg, wc->sl);
2054         }
2055
2056         if (cpu_to_be16(IBA_GET(CM_REQ_ALTERNATE_REMOTE_PORT_LID,
2057                     req_msg)) == IB_LID_PERMISSIVE)
2058             IBA_SET(CM_REQ_ALTERNATE_REMOTE_PORT_LID, req_msg,
2059                 wc->dlid_path_bits);
2060     }
2061 }
2062
2063 static int cm_req_handler(struct cm_work *work)
2064 {
2065     struct cm_id_private *cm_id_priv, *listen_cm_id_priv;
2066     struct cm_req_msg *req_msg;
2067     const struct ib_global_route *grh;
2068     const struct ib_gid_attr *gid_attr;
2069     int ret;
2070
2071     req_msg = (struct cm_req_msg *)work->mad_recv_wc->recv_buf.mad;
2072
2073     cm_id_priv =
2074         cm_alloc_id_priv(work->port->cm_dev->ib_device, NULL, NULL);
2075     if (IS_ERR(cm_id_priv))
2076         return PTR_ERR(cm_id_priv);
2077
2078     cm_id_priv->id.remote_id =
2079         cpu_to_be32(IBA_GET(CM_REQ_LOCAL_COMM_ID, req_msg));
2080     cm_id_priv->id.service_id =
2081         cpu_to_be64(IBA_GET(CM_REQ_SERVICE_ID, req_msg));
2082     cm_id_priv->id.service_mask = ~cpu_to_be64(0);
2083     cm_id_priv->tid = req_msg->hdr.tid;
2084     cm_id_priv->timeout_ms = cm_convert_to_ms(
2085         IBA_GET(CM_REQ_LOCAL_CM_RESPONSE_TIMEOUT, req_msg));
2086     cm_id_priv->max_cm_retries = IBA_GET(CM_REQ_MAX_CM_RETRIES, req_msg);
2087     cm_id_priv->remote_qpn =
2088         cpu_to_be32(IBA_GET(CM_REQ_LOCAL_QPN, req_msg));
2089     cm_id_priv->initiator_depth =
2090         IBA_GET(CM_REQ_RESPONDER_RESOURCES, req_msg);
2091     cm_id_priv->responder_resources =
2092         IBA_GET(CM_REQ_INITIATOR_DEPTH, req_msg);
2093     cm_id_priv->path_mtu = IBA_GET(CM_REQ_PATH_PACKET_PAYLOAD_MTU, req_msg);
2094     cm_id_priv->pkey = cpu_to_be16(IBA_GET(CM_REQ_PARTITION_KEY, req_msg));
2095     cm_id_priv->sq_psn = cpu_to_be32(IBA_GET(CM_REQ_STARTING_PSN, req_msg));
2096     cm_id_priv->retry_count = IBA_GET(CM_REQ_RETRY_COUNT, req_msg);
2097     cm_id_priv->rnr_retry_count = IBA_GET(CM_REQ_RNR_RETRY_COUNT, req_msg);
2098     cm_id_priv->qp_type = cm_req_get_qp_type(req_msg);
2099
2100     ret = cm_init_av_for_response(work->port, work->mad_recv_wc->wc,
2101                       work->mad_recv_wc->recv_buf.grh,
2102                       &cm_id_priv->av);
2103     if (ret)
2104         goto destroy;
2105     cm_id_priv->timewait_info = cm_create_timewait_info(cm_id_priv->
2106                                 id.local_id);
2107     if (IS_ERR(cm_id_priv->timewait_info)) {
2108         ret = PTR_ERR(cm_id_priv->timewait_info);
2109         cm_id_priv->timewait_info = NULL;
2110         goto destroy;
2111     }
2112     cm_id_priv->timewait_info->work.remote_id = cm_id_priv->id.remote_id;
2113     cm_id_priv->timewait_info->remote_ca_guid =
2114         cpu_to_be64(IBA_GET(CM_REQ_LOCAL_CA_GUID, req_msg));
2115     cm_id_priv->timewait_info->remote_qpn = cm_id_priv->remote_qpn;
2116
2117     /*
2118      * Note that the ID pointer is not in the xarray at this point,
2119      * so this set is only visible to the local thread.
2120      */
2121     cm_id_priv->id.state = IB_CM_REQ_RCVD;
2122
2123     listen_cm_id_priv = cm_match_req(work, cm_id_priv);
2124     if (!listen_cm_id_priv) {
2125         trace_icm_no_listener_err(&cm_id_priv->id);
2126         cm_id_priv->id.state = IB_CM_IDLE;
2127         ret = -EINVAL;
2128         goto destroy;
2129     }
2130
2131     memset(&work->path[0], 0, sizeof(work->path[0]));
2132     if (cm_req_has_alt_path(req_msg))
2133         memset(&work->path[1], 0, sizeof(work->path[1]));
2134     grh = rdma_ah_read_grh(&cm_id_priv->av.ah_attr);
2135     gid_attr = grh->sgid_attr;
2136
2137     if (cm_id_priv->av.ah_attr.type == RDMA_AH_ATTR_TYPE_ROCE) {
2138         work->path[0].rec_type =
2139             sa_conv_gid_to_pathrec_type(gid_attr->gid_type);
2140     } else {
2141         cm_process_routed_req(req_msg, work->mad_recv_wc->wc);
2142         cm_path_set_rec_type(
2143             work->port->cm_dev->ib_device, work->port->port_num,
2144             &work->path[0],
2145             IBA_GET_MEM_PTR(CM_REQ_PRIMARY_LOCAL_PORT_GID,
2146                     req_msg));
2147     }
2148     if (cm_req_has_alt_path(req_msg))
2149         work->path[1].rec_type = work->path[0].rec_type;
2150     cm_format_paths_from_req(req_msg, &work->path[0],
2151                  &work->path[1]);
2152     if (cm_id_priv->av.ah_attr.type == RDMA_AH_ATTR_TYPE_ROCE)
2153         sa_path_set_dmac(&work->path[0],
2154                  cm_id_priv->av.ah_attr.roce.dmac);
2155     work->path[0].hop_limit = grh->hop_limit;
2156
2157     /* This destroy call is needed to pair with cm_init_av_for_response */
2158     cm_destroy_av(&cm_id_priv->av);
2159     ret = cm_init_av_by_path(&work->path[0], gid_attr, &cm_id_priv->av);
2160     if (ret) {
2161         int err;
2162
2163         err = rdma_query_gid(work->port->cm_dev->ib_device,
2164                      work->port->port_num, 0,
2165                      &work->path[0].sgid);
2166         if (err)
2167             ib_send_cm_rej(&cm_id_priv->id, IB_CM_REJ_INVALID_GID,
2168                        NULL, 0, NULL, 0);
2169         else
2170             ib_send_cm_rej(&cm_id_priv->id, IB_CM_REJ_INVALID_GID,
2171                        &work->path[0].sgid,
2172                        sizeof(work->path[0].sgid),
2173                        NULL, 0);
2174         goto rejected;
2175     }
2176     if (cm_req_has_alt_path(req_msg)) {
2177         ret = cm_init_av_by_path(&work->path[1], NULL,
2178                      &cm_id_priv->alt_av);
2179         if (ret) {
2180             ib_send_cm_rej(&cm_id_priv->id,
2181                        IB_CM_REJ_INVALID_ALT_GID,
2182                        &work->path[0].sgid,
2183                        sizeof(work->path[0].sgid), NULL, 0);
2184             goto rejected;
2185         }
2186     }
2187
2188     cm_id_priv->id.cm_handler = listen_cm_id_priv->id.cm_handler;
2189     cm_id_priv->id.context = listen_cm_id_priv->id.context;
2190     cm_format_req_event(work, cm_id_priv, &listen_cm_id_priv->id);
2191
2192     /* Now MAD handlers can see the new ID */
2193     spin_lock_irq(&cm_id_priv->lock);
2194     cm_finalize_id(cm_id_priv);
2195
2196     /* Refcount belongs to the event, pairs with cm_process_work() */
2197     refcount_inc(&cm_id_priv->refcount);
2198     cm_queue_work_unlock(cm_id_priv, work);
2199     /*
2200      * Since this ID was just created and was not made visible to other MAD
2201      * handlers until the cm_finalize_id() above we know that the
2202      * cm_process_work() will deliver the event and the listen_cm_id
2203      * embedded in the event can be derefed here.
2204      */
2205     cm_deref_id(listen_cm_id_priv);
2206     return 0;
2207
2208 rejected:
2209     cm_deref_id(listen_cm_id_priv);
2210 destroy:
2211     ib_destroy_cm_id(&cm_id_priv->id);
2212     return ret;
2213 }
2214
2215 static void cm_format_rep(struct cm_rep_msg *rep_msg,
2216               struct cm_id_private *cm_id_priv,
2217               struct ib_cm_rep_param *param)
2218 {
2219     cm_format_mad_ece_hdr(&rep_msg->hdr, CM_REP_ATTR_ID, cm_id_priv->tid,
2220                   param->ece.attr_mod);
2221     IBA_SET(CM_REP_LOCAL_COMM_ID, rep_msg,
2222         be32_to_cpu(cm_id_priv->id.local_id));
2223     IBA_SET(CM_REP_REMOTE_COMM_ID, rep_msg,
2224         be32_to_cpu(cm_id_priv->id.remote_id));
2225     IBA_SET(CM_REP_STARTING_PSN, rep_msg, param->starting_psn);
2226     IBA_SET(CM_REP_RESPONDER_RESOURCES, rep_msg,
2227         param->responder_resources);
2228     IBA_SET(CM_REP_TARGET_ACK_DELAY, rep_msg,
2229         cm_id_priv->av.port->cm_dev->ack_delay);
2230     IBA_SET(CM_REP_FAILOVER_ACCEPTED, rep_msg, param->failover_accepted);
2231     IBA_SET(CM_REP_RNR_RETRY_COUNT, rep_msg, param->rnr_retry_count);
2232     IBA_SET(CM_REP_LOCAL_CA_GUID, rep_msg,
2233         be64_to_cpu(cm_id_priv->id.device->node_guid));
2234
2235     if (cm_id_priv->qp_type != IB_QPT_XRC_TGT) {
2236         IBA_SET(CM_REP_INITIATOR_DEPTH, rep_msg,
2237             param->initiator_depth);
2238         IBA_SET(CM_REP_END_TO_END_FLOW_CONTROL, rep_msg,
2239             param->flow_control);
2240         IBA_SET(CM_REP_SRQ, rep_msg, param->srq);
2241         IBA_SET(CM_REP_LOCAL_QPN, rep_msg, param->qp_num);
2242     } else {
2243         IBA_SET(CM_REP_SRQ, rep_msg, 1);
2244         IBA_SET(CM_REP_LOCAL_EE_CONTEXT_NUMBER, rep_msg, param->qp_num);
2245     }
2246
2247     IBA_SET(CM_REP_VENDOR_ID_L, rep_msg, param->ece.vendor_id);
2248     IBA_SET(CM_REP_VENDOR_ID_M, rep_msg, param->ece.vendor_id >> 8);
2249     IBA_SET(CM_REP_VENDOR_ID_H, rep_msg, param->ece.vendor_id >> 16);
2250
2251     if (param->private_data && param->private_data_len)
2252         IBA_SET_MEM(CM_REP_PRIVATE_DATA, rep_msg, param->private_data,
2253                 param->private_data_len);
2254 }
2255
2256 int ib_send_cm_rep(struct ib_cm_id *cm_id,
2257            struct ib_cm_rep_param *param)
2258 {
2259     struct cm_id_private *cm_id_priv;
2260     struct ib_mad_send_buf *msg;
2261     struct cm_rep_msg *rep_msg;
2262     unsigned long flags;
2263     int ret;
2264
2265     if (param->private_data &&
2266         param->private_data_len > IB_CM_REP_PRIVATE_DATA_SIZE)
2267         return -EINVAL;
2268
2269     cm_id_priv = container_of(cm_id, struct cm_id_private, id);
2270     spin_lock_irqsave(&cm_id_priv->lock, flags);
2271     if (cm_id->state != IB_CM_REQ_RCVD &&
2272         cm_id->state != IB_CM_MRA_REQ_SENT) {
2273         trace_icm_send_rep_err(cm_id_priv->id.local_id, cm_id->state);
2274         ret = -EINVAL;
2275         goto out;
2276     }
2277
2278     msg = cm_alloc_priv_msg(cm_id_priv);
2279     if (IS_ERR(msg)) {
2280         ret = PTR_ERR(msg);
2281         goto out;
2282     }
2283
2284     rep_msg = (struct cm_rep_msg *) msg->mad;
2285     cm_format_rep(rep_msg, cm_id_priv, param);
2286     msg->timeout_ms = cm_id_priv->timeout_ms;
2287     msg->context[1] = (void *) (unsigned long) IB_CM_REP_SENT;
2288
2289     trace_icm_send_rep(cm_id);
2290     ret = ib_post_send_mad(msg, NULL);
2291     if (ret)
2292         goto out_free;
2293
2294     cm_id->state = IB_CM_REP_SENT;
2295     cm_id_priv->initiator_depth = param->initiator_depth;
2296     cm_id_priv->responder_resources = param->responder_resources;
2297     cm_id_priv->rq_psn = cpu_to_be32(IBA_GET(CM_REP_STARTING_PSN, rep_msg));
2298     WARN_ONCE(param->qp_num & 0xFF000000,
2299           "IBTA declares QPN to be 24 bits, but it is 0x%X\n",
2300           param->qp_num);
2301     cm_id_priv->local_qpn = cpu_to_be32(param->qp_num & 0xFFFFFF);
2302     spin_unlock_irqrestore(&cm_id_priv->lock, flags);
2303     return 0;
2304
2305 out_free:
2306     cm_free_priv_msg(msg);
2307 out:
2308     spin_unlock_irqrestore(&cm_id_priv->lock, flags);
2309     return ret;
2310 }
2311 EXPORT_SYMBOL(ib_send_cm_rep);
2312
2313 static void cm_format_rtu(struct cm_rtu_msg *rtu_msg,
2314               struct cm_id_private *cm_id_priv,
2315               const void *private_data,
2316               u8 private_data_len)
2317 {
2318     cm_format_mad_hdr(&rtu_msg->hdr, CM_RTU_ATTR_ID, cm_id_priv->tid);
2319     IBA_SET(CM_RTU_LOCAL_COMM_ID, rtu_msg,
2320         be32_to_cpu(cm_id_priv->id.local_id));
2321     IBA_SET(CM_RTU_REMOTE_COMM_ID, rtu_msg,
2322         be32_to_cpu(cm_id_priv->id.remote_id));
2323
2324     if (private_data && private_data_len)
2325         IBA_SET_MEM(CM_RTU_PRIVATE_DATA, rtu_msg, private_data,
2326                 private_data_len);
2327 }
2328
2329 int ib_send_cm_rtu(struct ib_cm_id *cm_id,
2330            const void *private_data,
2331            u8 private_data_len)
2332 {
2333     struct cm_id_private *cm_id_priv;
2334     struct ib_mad_send_buf *msg;
2335     unsigned long flags;
2336     void *data;
2337     int ret;
2338
2339     if (private_data && private_data_len > IB_CM_RTU_PRIVATE_DATA_SIZE)
2340         return -EINVAL;
2341
2342     data = cm_copy_private_data(private_data, private_data_len);
2343     if (IS_ERR(data))
2344         return PTR_ERR(data);
2345
2346     cm_id_priv = container_of(cm_id, struct cm_id_private, id);
2347     spin_lock_irqsave(&cm_id_priv->lock, flags);
2348     if (cm_id->state != IB_CM_REP_RCVD &&
2349         cm_id->state != IB_CM_MRA_REP_SENT) {
2350         trace_icm_send_cm_rtu_err(cm_id);
2351         ret = -EINVAL;
2352         goto error;
2353     }
2354
2355     msg = cm_alloc_msg(cm_id_priv);
2356     if (IS_ERR(msg)) {
2357         ret = PTR_ERR(msg);
2358         goto error;
2359     }
2360
2361     cm_format_rtu((struct cm_rtu_msg *) msg->mad, cm_id_priv,
2362               private_data, private_data_len);
2363
2364     trace_icm_send_rtu(cm_id);
2365     ret = ib_post_send_mad(msg, NULL);
2366     if (ret) {
2367         spin_unlock_irqrestore(&cm_id_priv->lock, flags);
2368         cm_free_msg(msg);
2369         kfree(data);
2370         return ret;
2371     }
2372
2373     cm_id->state = IB_CM_ESTABLISHED;
2374     cm_set_private_data(cm_id_priv, data, private_data_len);
2375     spin_unlock_irqrestore(&cm_id_priv->lock, flags);
2376     return 0;
2377
2378 error:  spin_unlock_irqrestore(&cm_id_priv->lock, flags);
2379     kfree(data);
2380     return ret;
2381 }
2382 EXPORT_SYMBOL(ib_send_cm_rtu);
2383
2384 static void cm_format_rep_event(struct cm_work *work, enum ib_qp_type qp_type)
2385 {
2386     struct cm_rep_msg *rep_msg;
2387     struct ib_cm_rep_event_param *param;
2388
2389     rep_msg = (struct cm_rep_msg *)work->mad_recv_wc->recv_buf.mad;
2390     param = &work->cm_event.param.rep_rcvd;
2391     param->remote_ca_guid =
2392         cpu_to_be64(IBA_GET(CM_REP_LOCAL_CA_GUID, rep_msg));
2393     param->remote_qkey = IBA_GET(CM_REP_LOCAL_Q_KEY, rep_msg);
2394     param->remote_qpn = be32_to_cpu(cm_rep_get_qpn(rep_msg, qp_type));
2395     param->starting_psn = IBA_GET(CM_REP_STARTING_PSN, rep_msg);
2396     param->responder_resources = IBA_GET(CM_REP_INITIATOR_DEPTH, rep_msg);
2397     param->initiator_depth = IBA_GET(CM_REP_RESPONDER_RESOURCES, rep_msg);
2398     param->target_ack_delay = IBA_GET(CM_REP_TARGET_ACK_DELAY, rep_msg);
2399     param->failover_accepted = IBA_GET(CM_REP_FAILOVER_ACCEPTED, rep_msg);
2400     param->flow_control = IBA_GET(CM_REP_END_TO_END_FLOW_CONTROL, rep_msg);
2401     param->rnr_retry_count = IBA_GET(CM_REP_RNR_RETRY_COUNT, rep_msg);
2402     param->srq = IBA_GET(CM_REP_SRQ, rep_msg);
2403     param->ece.vendor_id = IBA_GET(CM_REP_VENDOR_ID_H, rep_msg) << 16;
2404     param->ece.vendor_id |= IBA_GET(CM_REP_VENDOR_ID_M, rep_msg) << 8;
2405     param->ece.vendor_id |= IBA_GET(CM_REP_VENDOR_ID_L, rep_msg);
2406     param->ece.attr_mod = be32_to_cpu(rep_msg->hdr.attr_mod);
2407
2408     work->cm_event.private_data =
2409         IBA_GET_MEM_PTR(CM_REP_PRIVATE_DATA, rep_msg);
2410 }
2411
2412 static void cm_dup_rep_handler(struct cm_work *work)
2413 {
2414     struct cm_id_private *cm_id_priv;
2415     struct cm_rep_msg *rep_msg;
2416     struct ib_mad_send_buf *msg = NULL;
2417     int ret;
2418
2419     rep_msg = (struct cm_rep_msg *) work->mad_recv_wc->recv_buf.mad;
2420     cm_id_priv = cm_acquire_id(
2421         cpu_to_be32(IBA_GET(CM_REP_REMOTE_COMM_ID, rep_msg)),
2422         cpu_to_be32(IBA_GET(CM_REP_LOCAL_COMM_ID, rep_msg)));
2423     if (!cm_id_priv)
2424         return;
2425
2426     atomic_long_inc(
2427         &work->port->counters[CM_RECV_DUPLICATES][CM_REP_COUNTER]);
2428     ret = cm_alloc_response_msg(work->port, work->mad_recv_wc, &msg);
2429     if (ret)
2430         goto deref;
2431
2432     spin_lock_irq(&cm_id_priv->lock);
2433     if (cm_id_priv->id.state == IB_CM_ESTABLISHED)
2434         cm_format_rtu((struct cm_rtu_msg *) msg->mad, cm_id_priv,
2435                   cm_id_priv->private_data,
2436                   cm_id_priv->private_data_len);
2437     else if (cm_id_priv->id.state == IB_CM_MRA_REP_SENT)
2438         cm_format_mra((struct cm_mra_msg *) msg->mad, cm_id_priv,
2439                   CM_MSG_RESPONSE_REP, cm_id_priv->service_timeout,
2440                   cm_id_priv->private_data,
2441                   cm_id_priv->private_data_len);
2442     else
2443         goto unlock;
2444     spin_unlock_irq(&cm_id_priv->lock);
2445
2446     trace_icm_send_dup_rep(&cm_id_priv->id);
2447     ret = ib_post_send_mad(msg, NULL);
2448     if (ret)
2449         goto free;
2450     goto deref;
2451
2452 unlock: spin_unlock_irq(&cm_id_priv->lock);
2453 free:   cm_free_response_msg(msg);
2454 deref:  cm_deref_id(cm_id_priv);
2455 }
2456
2457 static int cm_rep_handler(struct cm_work *work)
2458 {
2459     struct cm_id_private *cm_id_priv;
2460     struct cm_rep_msg *rep_msg;
2461     int ret;
2462     struct cm_id_private *cur_cm_id_priv;
2463     struct cm_timewait_info *timewait_info;
2464
2465     rep_msg = (struct cm_rep_msg *)work->mad_recv_wc->recv_buf.mad;
2466     cm_id_priv = cm_acquire_id(
2467         cpu_to_be32(IBA_GET(CM_REP_REMOTE_COMM_ID, rep_msg)), 0);
2468     if (!cm_id_priv) {
2469         cm_dup_rep_handler(work);
2470         trace_icm_remote_no_priv_err(
2471              IBA_GET(CM_REP_REMOTE_COMM_ID, rep_msg));
2472         return -EINVAL;
2473     }
2474
2475     cm_format_rep_event(work, cm_id_priv->qp_type);
2476
2477     spin_lock_irq(&cm_id_priv->lock);
2478     switch (cm_id_priv->id.state) {
2479     case IB_CM_REQ_SENT:
2480     case IB_CM_MRA_REQ_RCVD:
2481         break;
2482     default:
2483         ret = -EINVAL;
2484         trace_icm_rep_unknown_err(
2485             IBA_GET(CM_REP_LOCAL_COMM_ID, rep_msg),
2486             IBA_GET(CM_REP_REMOTE_COMM_ID, rep_msg),
2487             cm_id_priv->id.state);
2488         spin_unlock_irq(&cm_id_priv->lock);
2489         goto error;
2490     }
2491
2492     cm_id_priv->timewait_info->work.remote_id =
2493         cpu_to_be32(IBA_GET(CM_REP_LOCAL_COMM_ID, rep_msg));
2494     cm_id_priv->timewait_info->remote_ca_guid =
2495         cpu_to_be64(IBA_GET(CM_REP_LOCAL_CA_GUID, rep_msg));
2496     cm_id_priv->timewait_info->remote_qpn = cm_rep_get_qpn(rep_msg, cm_id_priv->qp_type);
2497
2498     spin_lock(&cm.lock);
2499     /* Check for duplicate REP. */
2500     if (cm_insert_remote_id(cm_id_priv->timewait_info)) {
2501         spin_unlock(&cm.lock);
2502         spin_unlock_irq(&cm_id_priv->lock);
2503         ret = -EINVAL;
2504         trace_icm_insert_failed_err(
2505              IBA_GET(CM_REP_REMOTE_COMM_ID, rep_msg));
2506         goto error;
2507     }
2508     /* Check for a stale connection. */
2509     timewait_info = cm_insert_remote_qpn(cm_id_priv->timewait_info);
2510     if (timewait_info) {
2511         cm_remove_remote(cm_id_priv);
2512         cur_cm_id_priv = cm_acquire_id(timewait_info->work.local_id,
2513                        timewait_info->work.remote_id);
2514
2515         spin_unlock(&cm.lock);
2516         spin_unlock_irq(&cm_id_priv->lock);
2517         cm_issue_rej(work->port, work->mad_recv_wc,
2518                  IB_CM_REJ_STALE_CONN, CM_MSG_RESPONSE_REP,
2519                  NULL, 0);
2520         ret = -EINVAL;
2521         trace_icm_staleconn_err(
2522             IBA_GET(CM_REP_LOCAL_COMM_ID, rep_msg),
2523             IBA_GET(CM_REP_REMOTE_COMM_ID, rep_msg));
2524
2525         if (cur_cm_id_priv) {
2526             ib_send_cm_dreq(&cur_cm_id_priv->id, NULL, 0);
2527             cm_deref_id(cur_cm_id_priv);
2528         }
2529
2530         goto error;
2531     }
2532     spin_unlock(&cm.lock);
2533
2534     cm_id_priv->id.state = IB_CM_REP_RCVD;
2535     cm_id_priv->id.remote_id =
2536         cpu_to_be32(IBA_GET(CM_REP_LOCAL_COMM_ID, rep_msg));
2537     cm_id_priv->remote_qpn = cm_rep_get_qpn(rep_msg, cm_id_priv->qp_type);
2538     cm_id_priv->initiator_depth =
2539         IBA_GET(CM_REP_RESPONDER_RESOURCES, rep_msg);
2540     cm_id_priv->responder_resources =
2541         IBA_GET(CM_REP_INITIATOR_DEPTH, rep_msg);
2542     cm_id_priv->sq_psn = cpu_to_be32(IBA_GET(CM_REP_STARTING_PSN, rep_msg));
2543     cm_id_priv->rnr_retry_count = IBA_GET(CM_REP_RNR_RETRY_COUNT, rep_msg);
2544     cm_id_priv->target_ack_delay =
2545         IBA_GET(CM_REP_TARGET_ACK_DELAY, rep_msg);
2546     cm_id_priv->av.timeout =
2547             cm_ack_timeout(cm_id_priv->target_ack_delay,
2548                        cm_id_priv->av.timeout - 1);
2549     cm_id_priv->alt_av.timeout =
2550             cm_ack_timeout(cm_id_priv->target_ack_delay,
2551                        cm_id_priv->alt_av.timeout - 1);
2552
2553     ib_cancel_mad(cm_id_priv->msg);
2554     cm_queue_work_unlock(cm_id_priv, work);
2555     return 0;
2556
2557 error:
2558     cm_deref_id(cm_id_priv);
2559     return ret;
2560 }
2561
2562 static int cm_establish_handler(struct cm_work *work)
2563 {
2564     struct cm_id_private *cm_id_priv;
2565
2566     /* See comment in cm_establish about lookup. */
2567     cm_id_priv = cm_acquire_id(work->local_id, work->remote_id);
2568     if (!cm_id_priv)
2569         return -EINVAL;
2570
2571     spin_lock_irq(&cm_id_priv->lock);
2572     if (cm_id_priv->id.state != IB_CM_ESTABLISHED) {
2573         spin_unlock_irq(&cm_id_priv->lock);
2574         goto out;
2575     }
2576
2577     ib_cancel_mad(cm_id_priv->msg);
2578     cm_queue_work_unlock(cm_id_priv, work);
2579     return 0;
2580 out:
2581     cm_deref_id(cm_id_priv);
2582     return -EINVAL;
2583 }
2584
2585 static int cm_rtu_handler(struct cm_work *work)
2586 {
2587     struct cm_id_private *cm_id_priv;
2588     struct cm_rtu_msg *rtu_msg;
2589
2590     rtu_msg = (struct cm_rtu_msg *)work->mad_recv_wc->recv_buf.mad;
2591     cm_id_priv = cm_acquire_id(
2592         cpu_to_be32(IBA_GET(CM_RTU_REMOTE_COMM_ID, rtu_msg)),
2593         cpu_to_be32(IBA_GET(CM_RTU_LOCAL_COMM_ID, rtu_msg)));
2594     if (!cm_id_priv)
2595         return -EINVAL;
2596
2597     work->cm_event.private_data =
2598         IBA_GET_MEM_PTR(CM_RTU_PRIVATE_DATA, rtu_msg);
2599
2600     spin_lock_irq(&cm_id_priv->lock);
2601     if (cm_id_priv->id.state != IB_CM_REP_SENT &&
2602         cm_id_priv->id.state != IB_CM_MRA_REP_RCVD) {
2603         spin_unlock_irq(&cm_id_priv->lock);
2604         atomic_long_inc(&work->port->counters[CM_RECV_DUPLICATES]
2605                              [CM_RTU_COUNTER]);
2606         goto out;
2607     }
2608     cm_id_priv->id.state = IB_CM_ESTABLISHED;
2609
2610     ib_cancel_mad(cm_id_priv->msg);
2611     cm_queue_work_unlock(cm_id_priv, work);
2612     return 0;
2613 out:
2614     cm_deref_id(cm_id_priv);
2615     return -EINVAL;
2616 }
2617
2618 static void cm_format_dreq(struct cm_dreq_msg *dreq_msg,
2619               struct cm_id_private *cm_id_priv,
2620               const void *private_data,
2621               u8 private_data_len)
2622 {
2623     cm_format_mad_hdr(&dreq_msg->hdr, CM_DREQ_ATTR_ID,
2624               cm_form_tid(cm_id_priv));
2625     IBA_SET(CM_DREQ_LOCAL_COMM_ID, dreq_msg,
2626         be32_to_cpu(cm_id_priv->id.local_id));
2627     IBA_SET(CM_DREQ_REMOTE_COMM_ID, dreq_msg,
2628         be32_to_cpu(cm_id_priv->id.remote_id));
2629     IBA_SET(CM_DREQ_REMOTE_QPN_EECN, dreq_msg,
2630         be32_to_cpu(cm_id_priv->remote_qpn));
2631
2632     if (private_data && private_data_len)
2633         IBA_SET_MEM(CM_DREQ_PRIVATE_DATA, dreq_msg, private_data,
2634                 private_data_len);
2635 }
2636
2637 static int cm_send_dreq_locked(struct cm_id_private *cm_id_priv,
2638                    const void *private_data, u8 private_data_len)
2639 {
2640     struct ib_mad_send_buf *msg;
2641     int ret;
2642
2643     lockdep_assert_held(&cm_id_priv->lock);
2644
2645     if (private_data && private_data_len > IB_CM_DREQ_PRIVATE_DATA_SIZE)
2646         return -EINVAL;
2647
2648     if (cm_id_priv->id.state != IB_CM_ESTABLISHED) {
2649         trace_icm_dreq_skipped(&cm_id_priv->id);
2650         return -EINVAL;
2651     }
2652
2653     if (cm_id_priv->id.lap_state == IB_CM_LAP_SENT ||
2654         cm_id_priv->id.lap_state == IB_CM_MRA_LAP_RCVD)
2655         ib_cancel_mad(cm_id_priv->msg);
2656
2657     msg = cm_alloc_priv_msg(cm_id_priv);
2658     if (IS_ERR(msg)) {
2659         cm_enter_timewait(cm_id_priv);
2660         return PTR_ERR(msg);
2661     }
2662
2663     cm_format_dreq((struct cm_dreq_msg *) msg->mad, cm_id_priv,
2664                private_data, private_data_len);
2665     msg->timeout_ms = cm_id_priv->timeout_ms;
2666     msg->context[1] = (void *) (unsigned long) IB_CM_DREQ_SENT;
2667
2668     trace_icm_send_dreq(&cm_id_priv->id);
2669     ret = ib_post_send_mad(msg, NULL);
2670     if (ret) {
2671         cm_enter_timewait(cm_id_priv);
2672         cm_free_priv_msg(msg);
2673         return ret;
2674     }
2675
2676     cm_id_priv->id.state = IB_CM_DREQ_SENT;
2677     return 0;
2678 }
2679
2680 int ib_send_cm_dreq(struct ib_cm_id *cm_id, const void *private_data,
2681             u8 private_data_len)
2682 {
2683     struct cm_id_private *cm_id_priv =
2684         container_of(cm_id, struct cm_id_private, id);
2685     unsigned long flags;
2686     int ret;
2687
2688     spin_lock_irqsave(&cm_id_priv->lock, flags);
2689     ret = cm_send_dreq_locked(cm_id_priv, private_data, private_data_len);
2690     spin_unlock_irqrestore(&cm_id_priv->lock, flags);
2691     return ret;
2692 }
2693 EXPORT_SYMBOL(ib_send_cm_dreq);
2694
2695 static void cm_format_drep(struct cm_drep_msg *drep_msg,
2696               struct cm_id_private *cm_id_priv,
2697               const void *private_data,
2698               u8 private_data_len)
2699 {
2700     cm_format_mad_hdr(&drep_msg->hdr, CM_DREP_ATTR_ID, cm_id_priv->tid);
2701     IBA_SET(CM_DREP_LOCAL_COMM_ID, drep_msg,
2702         be32_to_cpu(cm_id_priv->id.local_id));
2703     IBA_SET(CM_DREP_REMOTE_COMM_ID, drep_msg,
2704         be32_to_cpu(cm_id_priv->id.remote_id));
2705
2706     if (private_data && private_data_len)
2707         IBA_SET_MEM(CM_DREP_PRIVATE_DATA, drep_msg, private_data,
2708                 private_data_len);
2709 }
2710
2711 static int cm_send_drep_locked(struct cm_id_private *cm_id_priv,
2712                    void *private_data, u8 private_data_len)
2713 {
2714     struct ib_mad_send_buf *msg;
2715     int ret;
2716
2717     lockdep_assert_held(&cm_id_priv->lock);
2718
2719     if (private_data && private_data_len > IB_CM_DREP_PRIVATE_DATA_SIZE)
2720         return -EINVAL;
2721
2722     if (cm_id_priv->id.state != IB_CM_DREQ_RCVD) {
2723         trace_icm_send_drep_err(&cm_id_priv->id);
2724         kfree(private_data);
2725         return -EINVAL;
2726     }
2727
2728     cm_set_private_data(cm_id_priv, private_data, private_data_len);
2729     cm_enter_timewait(cm_id_priv);
2730
2731     msg = cm_alloc_msg(cm_id_priv);
2732     if (IS_ERR(msg))
2733         return PTR_ERR(msg);
2734
2735     cm_format_drep((struct cm_drep_msg *) msg->mad, cm_id_priv,
2736                private_data, private_data_len);
2737
2738     trace_icm_send_drep(&cm_id_priv->id);
2739     ret = ib_post_send_mad(msg, NULL);
2740     if (ret) {
2741         cm_free_msg(msg);
2742         return ret;
2743     }
2744     return 0;
2745 }
2746
2747 int ib_send_cm_drep(struct ib_cm_id *cm_id, const void *private_data,
2748             u8 private_data_len)
2749 {
2750     struct cm_id_private *cm_id_priv =
2751         container_of(cm_id, struct cm_id_private, id);
2752     unsigned long flags;
2753     void *data;
2754     int ret;
2755
2756     data = cm_copy_private_data(private_data, private_data_len);
2757     if (IS_ERR(data))
2758         return PTR_ERR(data);
2759
2760     spin_lock_irqsave(&cm_id_priv->lock, flags);
2761     ret = cm_send_drep_locked(cm_id_priv, data, private_data_len);
2762     spin_unlock_irqrestore(&cm_id_priv->lock, flags);
2763     return ret;
2764 }
2765 EXPORT_SYMBOL(ib_send_cm_drep);
2766
2767 static int cm_issue_drep(struct cm_port *port,
2768              struct ib_mad_recv_wc *mad_recv_wc)
2769 {
2770     struct ib_mad_send_buf *msg = NULL;
2771     struct cm_dreq_msg *dreq_msg;
2772     struct cm_drep_msg *drep_msg;
2773     int ret;
2774
2775     ret = cm_alloc_response_msg(port, mad_recv_wc, &msg);
2776     if (ret)
2777         return ret;
2778
2779     dreq_msg = (struct cm_dreq_msg *) mad_recv_wc->recv_buf.mad;
2780     drep_msg = (struct cm_drep_msg *) msg->mad;
2781
2782     cm_format_mad_hdr(&drep_msg->hdr, CM_DREP_ATTR_ID, dreq_msg->hdr.tid);
2783     IBA_SET(CM_DREP_REMOTE_COMM_ID, drep_msg,
2784         IBA_GET(CM_DREQ_LOCAL_COMM_ID, dreq_msg));
2785     IBA_SET(CM_DREP_LOCAL_COMM_ID, drep_msg,
2786         IBA_GET(CM_DREQ_REMOTE_COMM_ID, dreq_msg));
2787
2788     trace_icm_issue_drep(
2789         IBA_GET(CM_DREQ_LOCAL_COMM_ID, dreq_msg),
2790         IBA_GET(CM_DREQ_REMOTE_COMM_ID, dreq_msg));
2791     ret = ib_post_send_mad(msg, NULL);
2792     if (ret)
2793         cm_free_response_msg(msg);
2794
2795     return ret;
2796 }
2797
2798 static int cm_dreq_handler(struct cm_work *work)
2799 {
2800     struct cm_id_private *cm_id_priv;
2801     struct cm_dreq_msg *dreq_msg;
2802     struct ib_mad_send_buf *msg = NULL;
2803
2804     dreq_msg = (struct cm_dreq_msg *)work->mad_recv_wc->recv_buf.mad;
2805     cm_id_priv = cm_acquire_id(
2806         cpu_to_be32(IBA_GET(CM_DREQ_REMOTE_COMM_ID, dreq_msg)),
2807         cpu_to_be32(IBA_GET(CM_DREQ_LOCAL_COMM_ID, dreq_msg)));
2808     if (!cm_id_priv) {
2809         atomic_long_inc(&work->port->counters[CM_RECV_DUPLICATES]
2810                              [CM_DREQ_COUNTER]);
2811         cm_issue_drep(work->port, work->mad_recv_wc);
2812         trace_icm_no_priv_err(
2813             IBA_GET(CM_DREQ_LOCAL_COMM_ID, dreq_msg),
2814             IBA_GET(CM_DREQ_REMOTE_COMM_ID, dreq_msg));
2815         return -EINVAL;
2816     }
2817
2818     work->cm_event.private_data =
2819         IBA_GET_MEM_PTR(CM_DREQ_PRIVATE_DATA, dreq_msg);
2820
2821     spin_lock_irq(&cm_id_priv->lock);
2822     if (cm_id_priv->local_qpn !=
2823         cpu_to_be32(IBA_GET(CM_DREQ_REMOTE_QPN_EECN, dreq_msg)))
2824         goto unlock;
2825
2826     switch (cm_id_priv->id.state) {
2827     case IB_CM_REP_SENT:
2828     case IB_CM_DREQ_SENT:
2829     case IB_CM_MRA_REP_RCVD:
2830         ib_cancel_mad(cm_id_priv->msg);
2831         break;
2832     case IB_CM_ESTABLISHED:
2833         if (cm_id_priv->id.lap_state == IB_CM_LAP_SENT ||
2834             cm_id_priv->id.lap_state == IB_CM_MRA_LAP_RCVD)
2835             ib_cancel_mad(cm_id_priv->msg);
2836         break;
2837     case IB_CM_TIMEWAIT:
2838         atomic_long_inc(&work->port->counters[CM_RECV_DUPLICATES]
2839                              [CM_DREQ_COUNTER]);
2840         msg = cm_alloc_response_msg_no_ah(work->port, work->mad_recv_wc);
2841         if (IS_ERR(msg))
2842             goto unlock;
2843
2844         cm_format_drep((struct cm_drep_msg *) msg->mad, cm_id_priv,
2845                    cm_id_priv->private_data,
2846                    cm_id_priv->private_data_len);
2847         spin_unlock_irq(&cm_id_priv->lock);
2848
2849         if (cm_create_response_msg_ah(work->port, work->mad_recv_wc, msg) ||
2850             ib_post_send_mad(msg, NULL))
2851             cm_free_response_msg(msg);
2852         goto deref;
2853     case IB_CM_DREQ_RCVD:
2854         atomic_long_inc(&work->port->counters[CM_RECV_DUPLICATES]
2855                              [CM_DREQ_COUNTER]);
2856         goto unlock;
2857     default:
2858         trace_icm_dreq_unknown_err(&cm_id_priv->id);
2859         goto unlock;
2860     }
2861     cm_id_priv->id.state = IB_CM_DREQ_RCVD;
2862     cm_id_priv->tid = dreq_msg->hdr.tid;
2863     cm_queue_work_unlock(cm_id_priv, work);
2864     return 0;
2865
2866 unlock: spin_unlock_irq(&cm_id_priv->lock);
2867 deref:  cm_deref_id(cm_id_priv);
2868     return -EINVAL;
2869 }
2870
2871 static int cm_drep_handler(struct cm_work *work)
2872 {
2873     struct cm_id_private *cm_id_priv;
2874     struct cm_drep_msg *drep_msg;
2875
2876     drep_msg = (struct cm_drep_msg *)work->mad_recv_wc->recv_buf.mad;
2877     cm_id_priv = cm_acquire_id(
2878         cpu_to_be32(IBA_GET(CM_DREP_REMOTE_COMM_ID, drep_msg)),
2879         cpu_to_be32(IBA_GET(CM_DREP_LOCAL_COMM_ID, drep_msg)));
2880     if (!cm_id_priv)
2881         return -EINVAL;
2882
2883     work->cm_event.private_data =
2884         IBA_GET_MEM_PTR(CM_DREP_PRIVATE_DATA, drep_msg);
2885
2886     spin_lock_irq(&cm_id_priv->lock);
2887     if (cm_id_priv->id.state != IB_CM_DREQ_SENT &&
2888         cm_id_priv->id.state != IB_CM_DREQ_RCVD) {
2889         spin_unlock_irq(&cm_id_priv->lock);
2890         goto out;
2891     }
2892     cm_enter_timewait(cm_id_priv);
2893
2894     ib_cancel_mad(cm_id_priv->msg);
2895     cm_queue_work_unlock(cm_id_priv, work);
2896     return 0;
2897 out:
2898     cm_deref_id(cm_id_priv);
2899     return -EINVAL;
2900 }
2901
2902 static int cm_send_rej_locked(struct cm_id_private *cm_id_priv,
2903                   enum ib_cm_rej_reason reason, void *ari,
2904                   u8 ari_length, const void *private_data,
2905                   u8 private_data_len)
2906 {
2907     enum ib_cm_state state = cm_id_priv->id.state;
2908     struct ib_mad_send_buf *msg;
2909     int ret;
2910
2911     lockdep_assert_held(&cm_id_priv->lock);
2912
2913     if ((private_data && private_data_len > IB_CM_REJ_PRIVATE_DATA_SIZE) ||
2914         (ari && ari_length > IB_CM_REJ_ARI_LENGTH))
2915         return -EINVAL;
2916
2917     switch (state) {
2918     case IB_CM_REQ_SENT:
2919     case IB_CM_MRA_REQ_RCVD:
2920     case IB_CM_REQ_RCVD:
2921     case IB_CM_MRA_REQ_SENT:
2922     case IB_CM_REP_RCVD:
2923     case IB_CM_MRA_REP_SENT:
2924         cm_reset_to_idle(cm_id_priv);
2925         msg = cm_alloc_msg(cm_id_priv);
2926         if (IS_ERR(msg))
2927             return PTR_ERR(msg);
2928         cm_format_rej((struct cm_rej_msg *)msg->mad, cm_id_priv, reason,
2929                   ari, ari_length, private_data, private_data_len,
2930                   state);
2931         break;
2932     case IB_CM_REP_SENT:
2933     case IB_CM_MRA_REP_RCVD:
2934         cm_enter_timewait(cm_id_priv);
2935         msg = cm_alloc_msg(cm_id_priv);
2936         if (IS_ERR(msg))
2937             return PTR_ERR(msg);
2938         cm_format_rej((struct cm_rej_msg *)msg->mad, cm_id_priv, reason,
2939                   ari, ari_length, private_data, private_data_len,
2940                   state);
2941         break;
2942     default:
2943         trace_icm_send_unknown_rej_err(&cm_id_priv->id);
2944         return -EINVAL;
2945     }
2946
2947     trace_icm_send_rej(&cm_id_priv->id, reason);
2948     ret = ib_post_send_mad(msg, NULL);
2949     if (ret) {
2950         cm_free_msg(msg);
2951         return ret;
2952     }
2953
2954     return 0;
2955 }
2956
2957 int ib_send_cm_rej(struct ib_cm_id *cm_id, enum ib_cm_rej_reason reason,
2958            void *ari, u8 ari_length, const void *private_data,
2959            u8 private_data_len)
2960 {
2961     struct cm_id_private *cm_id_priv =
2962         container_of(cm_id, struct cm_id_private, id);
2963     unsigned long flags;
2964     int ret;
2965
2966     spin_lock_irqsave(&cm_id_priv->lock, flags);
2967     ret = cm_send_rej_locked(cm_id_priv, reason, ari, ari_length,
2968                  private_data, private_data_len);
2969     spin_unlock_irqrestore(&cm_id_priv->lock, flags);
2970     return ret;
2971 }
2972 EXPORT_SYMBOL(ib_send_cm_rej);
2973
2974 static void cm_format_rej_event(struct cm_work *work)
2975 {
2976     struct cm_rej_msg *rej_msg;
2977     struct ib_cm_rej_event_param *param;
2978
2979     rej_msg = (struct cm_rej_msg *)work->mad_recv_wc->recv_buf.mad;
2980     param = &work->cm_event.param.rej_rcvd;
2981     param->ari = IBA_GET_MEM_PTR(CM_REJ_ARI, rej_msg);
2982     param->ari_length = IBA_GET(CM_REJ_REJECTED_INFO_LENGTH, rej_msg);
2983     param->reason = IBA_GET(CM_REJ_REASON, rej_msg);
2984     work->cm_event.private_data =
2985         IBA_GET_MEM_PTR(CM_REJ_PRIVATE_DATA, rej_msg);
2986 }
2987
2988 static struct cm_id_private *cm_acquire_rejected_id(struct cm_rej_msg *rej_msg)
2989 {
2990     struct cm_id_private *cm_id_priv;
2991     __be32 remote_id;
2992
2993     remote_id = cpu_to_be32(IBA_GET(CM_REJ_LOCAL_COMM_ID, rej_msg));
2994
2995     if (IBA_GET(CM_REJ_REASON, rej_msg) == IB_CM_REJ_TIMEOUT) {
2996         cm_id_priv = cm_find_remote_id(
2997             *((__be64 *)IBA_GET_MEM_PTR(CM_REJ_ARI, rej_msg)),
2998             remote_id);
2999     } else if (IBA_GET(CM_REJ_MESSAGE_REJECTED, rej_msg) ==
3000            CM_MSG_RESPONSE_REQ)
3001         cm_id_priv = cm_acquire_id(
3002             cpu_to_be32(IBA_GET(CM_REJ_REMOTE_COMM_ID, rej_msg)),
3003             0);
3004     else
3005         cm_id_priv = cm_acquire_id(
3006             cpu_to_be32(IBA_GET(CM_REJ_REMOTE_COMM_ID, rej_msg)),
3007             remote_id);
3008
3009     return cm_id_priv;
3010 }
3011
3012 static int cm_rej_handler(struct cm_work *work)
3013 {
3014     struct cm_id_private *cm_id_priv;
3015     struct cm_rej_msg *rej_msg;
3016
3017     rej_msg = (struct cm_rej_msg *)work->mad_recv_wc->recv_buf.mad;
3018     cm_id_priv = cm_acquire_rejected_id(rej_msg);
3019     if (!cm_id_priv)
3020         return -EINVAL;
3021
3022     cm_format_rej_event(work);
3023
3024     spin_lock_irq(&cm_id_priv->lock);
3025     switch (cm_id_priv->id.state) {
3026     case IB_CM_REQ_SENT:
3027     case IB_CM_MRA_REQ_RCVD:
3028     case IB_CM_REP_SENT:
3029     case IB_CM_MRA_REP_RCVD:
3030         ib_cancel_mad(cm_id_priv->msg);
3031         fallthrough;
3032     case IB_CM_REQ_RCVD:
3033     case IB_CM_MRA_REQ_SENT:
3034         if (IBA_GET(CM_REJ_REASON, rej_msg) == IB_CM_REJ_STALE_CONN)
3035             cm_enter_timewait(cm_id_priv);
3036         else
3037             cm_reset_to_idle(cm_id_priv);
3038         break;
3039     case IB_CM_DREQ_SENT:
3040         ib_cancel_mad(cm_id_priv->msg);
3041         fallthrough;
3042     case IB_CM_REP_RCVD:
3043     case IB_CM_MRA_REP_SENT:
3044         cm_enter_timewait(cm_id_priv);
3045         break;
3046     case IB_CM_ESTABLISHED:
3047         if (cm_id_priv->id.lap_state == IB_CM_LAP_UNINIT ||
3048             cm_id_priv->id.lap_state == IB_CM_LAP_SENT) {
3049             if (cm_id_priv->id.lap_state == IB_CM_LAP_SENT)
3050                 ib_cancel_mad(cm_id_priv->msg);
3051             cm_enter_timewait(cm_id_priv);
3052             break;
3053         }
3054         fallthrough;
3055     default:
3056         trace_icm_rej_unknown_err(&cm_id_priv->id);
3057         spin_unlock_irq(&cm_id_priv->lock);
3058         goto out;
3059     }
3060
3061     cm_queue_work_unlock(cm_id_priv, work);
3062     return 0;
3063 out:
3064     cm_deref_id(cm_id_priv);
3065     return -EINVAL;
3066 }
3067
3068 int ib_send_cm_mra(struct ib_cm_id *cm_id,
3069            u8 service_timeout,
3070            const void *private_data,
3071            u8 private_data_len)
3072 {
3073     struct cm_id_private *cm_id_priv;
3074     struct ib_mad_send_buf *msg;
3075     enum ib_cm_state cm_state;
3076     enum ib_cm_lap_state lap_state;
3077     enum cm_msg_response msg_response;
3078     void *data;
3079     unsigned long flags;
3080     int ret;
3081
3082     if (private_data && private_data_len > IB_CM_MRA_PRIVATE_DATA_SIZE)
3083         return -EINVAL;
3084
3085     data = cm_copy_private_data(private_data, private_data_len);
3086     if (IS_ERR(data))
3087         return PTR_ERR(data);
3088
3089     cm_id_priv = container_of(cm_id, struct cm_id_private, id);
3090
3091     spin_lock_irqsave(&cm_id_priv->lock, flags);
3092     switch (cm_id_priv->id.state) {
3093     case IB_CM_REQ_RCVD:
3094         cm_state = IB_CM_MRA_REQ_SENT;
3095         lap_state = cm_id->lap_state;
3096         msg_response = CM_MSG_RESPONSE_REQ;
3097         break;
3098     case IB_CM_REP_RCVD:
3099         cm_state = IB_CM_MRA_REP_SENT;
3100         lap_state = cm_id->lap_state;
3101         msg_response = CM_MSG_RESPONSE_REP;
3102         break;
3103     case IB_CM_ESTABLISHED:
3104         if (cm_id->lap_state == IB_CM_LAP_RCVD) {
3105             cm_state = cm_id->state;
3106             lap_state = IB_CM_MRA_LAP_SENT;
3107             msg_response = CM_MSG_RESPONSE_OTHER;
3108             break;
3109         }
3110         fallthrough;
3111     default:
3112         trace_icm_send_mra_unknown_err(&cm_id_priv->id);
3113         ret = -EINVAL;
3114         goto error_unlock;
3115     }
3116
3117     if (!(service_timeout & IB_CM_MRA_FLAG_DELAY)) {
3118         msg = cm_alloc_msg(cm_id_priv);
3119         if (IS_ERR(msg)) {
3120             ret = PTR_ERR(msg);
3121             goto error_unlock;
3122         }
3123
3124         cm_format_mra((struct cm_mra_msg *) msg->mad, cm_id_priv,
3125                   msg_response, service_timeout,
3126                   private_data, private_data_len);
3127         trace_icm_send_mra(cm_id);
3128         ret = ib_post_send_mad(msg, NULL);
3129         if (ret)
3130             goto error_free_msg;
3131     }
3132
3133     cm_id->state = cm_state;
3134     cm_id->lap_state = lap_state;
3135     cm_id_priv->service_timeout = service_timeout;
3136     cm_set_private_data(cm_id_priv, data, private_data_len);
3137     spin_unlock_irqrestore(&cm_id_priv->lock, flags);
3138     return 0;
3139
3140 error_free_msg:
3141     cm_free_msg(msg);
3142 error_unlock:
3143     spin_unlock_irqrestore(&cm_id_priv->lock, flags);
3144     kfree(data);
3145     return ret;
3146 }
3147 EXPORT_SYMBOL(ib_send_cm_mra);
3148
3149 static struct cm_id_private *cm_acquire_mraed_id(struct cm_mra_msg *mra_msg)
3150 {
3151     switch (IBA_GET(CM_MRA_MESSAGE_MRAED, mra_msg)) {
3152     case CM_MSG_RESPONSE_REQ:
3153         return cm_acquire_id(
3154             cpu_to_be32(IBA_GET(CM_MRA_REMOTE_COMM_ID, mra_msg)),
3155             0);
3156     case CM_MSG_RESPONSE_REP:
3157     case CM_MSG_RESPONSE_OTHER:
3158         return cm_acquire_id(
3159             cpu_to_be32(IBA_GET(CM_MRA_REMOTE_COMM_ID, mra_msg)),
3160             cpu_to_be32(IBA_GET(CM_MRA_LOCAL_COMM_ID, mra_msg)));
3161     default:
3162         return NULL;
3163     }
3164 }
3165
3166 static int cm_mra_handler(struct cm_work *work)
3167 {
3168     struct cm_id_private *cm_id_priv;
3169     struct cm_mra_msg *mra_msg;
3170     int timeout;
3171
3172     mra_msg = (struct cm_mra_msg *)work->mad_recv_wc->recv_buf.mad;
3173     cm_id_priv = cm_acquire_mraed_id(mra_msg);
3174     if (!cm_id_priv)
3175         return -EINVAL;
3176
3177     work->cm_event.private_data =
3178         IBA_GET_MEM_PTR(CM_MRA_PRIVATE_DATA, mra_msg);
3179     work->cm_event.param.mra_rcvd.service_timeout =
3180         IBA_GET(CM_MRA_SERVICE_TIMEOUT, mra_msg);
3181     timeout = cm_convert_to_ms(IBA_GET(CM_MRA_SERVICE_TIMEOUT, mra_msg)) +
3182           cm_convert_to_ms(cm_id_priv->av.timeout);
3183
3184     spin_lock_irq(&cm_id_priv->lock);
3185     switch (cm_id_priv->id.state) {
3186     case IB_CM_REQ_SENT:
3187         if (IBA_GET(CM_MRA_MESSAGE_MRAED, mra_msg) !=
3188                 CM_MSG_RESPONSE_REQ ||
3189             ib_modify_mad(cm_id_priv->msg, timeout))
3190             goto out;
3191         cm_id_priv->id.state = IB_CM_MRA_REQ_RCVD;
3192         break;
3193     case IB_CM_REP_SENT:
3194         if (IBA_GET(CM_MRA_MESSAGE_MRAED, mra_msg) !=
3195                 CM_MSG_RESPONSE_REP ||
3196             ib_modify_mad(cm_id_priv->msg, timeout))
3197             goto out;
3198         cm_id_priv->id.state = IB_CM_MRA_REP_RCVD;
3199         break;
3200     case IB_CM_ESTABLISHED:
3201         if (IBA_GET(CM_MRA_MESSAGE_MRAED, mra_msg) !=
3202                 CM_MSG_RESPONSE_OTHER ||
3203             cm_id_priv->id.lap_state != IB_CM_LAP_SENT ||
3204             ib_modify_mad(cm_id_priv->msg, timeout)) {
3205             if (cm_id_priv->id.lap_state == IB_CM_MRA_LAP_RCVD)
3206                 atomic_long_inc(
3207                     &work->port->counters[CM_RECV_DUPLICATES]
3208                                  [CM_MRA_COUNTER]);
3209             goto out;
3210         }
3211         cm_id_priv->id.lap_state = IB_CM_MRA_LAP_RCVD;
3212         break;
3213     case IB_CM_MRA_REQ_RCVD:
3214     case IB_CM_MRA_REP_RCVD:
3215         atomic_long_inc(&work->port->counters[CM_RECV_DUPLICATES]
3216                              [CM_MRA_COUNTER]);
3217         fallthrough;
3218     default:
3219         trace_icm_mra_unknown_err(&cm_id_priv->id);
3220         goto out;
3221     }
3222
3223     cm_id_priv->msg->context[1] = (void *) (unsigned long)
3224                       cm_id_priv->id.state;
3225     cm_queue_work_unlock(cm_id_priv, work);
3226     return 0;
3227 out:
3228     spin_unlock_irq(&cm_id_priv->lock);
3229     cm_deref_id(cm_id_priv);
3230     return -EINVAL;
3231 }
3232
3233 static void cm_format_path_lid_from_lap(struct cm_lap_msg *lap_msg,
3234                     struct sa_path_rec *path)
3235 {
3236     u32 lid;
3237
3238     if (path->rec_type != SA_PATH_REC_TYPE_OPA) {
3239         sa_path_set_dlid(path, IBA_GET(CM_LAP_ALTERNATE_LOCAL_PORT_LID,
3240                            lap_msg));
3241         sa_path_set_slid(path, IBA_GET(CM_LAP_ALTERNATE_REMOTE_PORT_LID,
3242                            lap_msg));
3243     } else {
3244         lid = opa_get_lid_from_gid(IBA_GET_MEM_PTR(
3245             CM_LAP_ALTERNATE_LOCAL_PORT_GID, lap_msg));
3246         sa_path_set_dlid(path, lid);
3247
3248         lid = opa_get_lid_from_gid(IBA_GET_MEM_PTR(
3249             CM_LAP_ALTERNATE_REMOTE_PORT_GID, lap_msg));
3250         sa_path_set_slid(path, lid);
3251     }
3252 }
3253
3254 static void cm_format_path_from_lap(struct cm_id_private *cm_id_priv,
3255                     struct sa_path_rec *path,
3256                     struct cm_lap_msg *lap_msg)
3257 {
3258     path->dgid = *IBA_GET_MEM_PTR(CM_LAP_ALTERNATE_LOCAL_PORT_GID, lap_msg);
3259     path->sgid =
3260         *IBA_GET_MEM_PTR(CM_LAP_ALTERNATE_REMOTE_PORT_GID, lap_msg);
3261     path->flow_label =
3262         cpu_to_be32(IBA_GET(CM_LAP_ALTERNATE_FLOW_LABEL, lap_msg));
3263     path->hop_limit = IBA_GET(CM_LAP_ALTERNATE_HOP_LIMIT, lap_msg);
3264     path->traffic_class = IBA_GET(CM_LAP_ALTERNATE_TRAFFIC_CLASS, lap_msg);
3265     path->reversible = 1;
3266     path->pkey = cm_id_priv->pkey;
3267     path->sl = IBA_GET(CM_LAP_ALTERNATE_SL, lap_msg);
3268     path->mtu_selector = IB_SA_EQ;
3269     path->mtu = cm_id_priv->path_mtu;
3270     path->rate_selector = IB_SA_EQ;
3271     path->rate = IBA_GET(CM_LAP_ALTERNATE_PACKET_RATE, lap_msg);
3272     path->packet_life_time_selector = IB_SA_EQ;
3273     path->packet_life_time =
3274         IBA_GET(CM_LAP_ALTERNATE_LOCAL_ACK_TIMEOUT, lap_msg);
3275     path->packet_life_time -= (path->packet_life_time > 0);
3276     cm_format_path_lid_from_lap(lap_msg, path);
3277 }
3278
3279 static int cm_lap_handler(struct cm_work *work)
3280 {
3281     struct cm_id_private *cm_id_priv;
3282     struct cm_lap_msg *lap_msg;
3283     struct ib_cm_lap_event_param *param;
3284     struct ib_mad_send_buf *msg = NULL;
3285     struct rdma_ah_attr ah_attr;
3286     struct cm_av alt_av = {};
3287     int ret;
3288
3289     /* Currently Alternate path messages are not supported for
3290      * RoCE link layer.
3291      */
3292     if (rdma_protocol_roce(work->port->cm_dev->ib_device,
3293                    work->port->port_num))
3294         return -EINVAL;
3295
3296     /* todo: verify LAP request and send reject APR if invalid. */
3297     lap_msg = (struct cm_lap_msg *)work->mad_recv_wc->recv_buf.mad;
3298     cm_id_priv = cm_acquire_id(
3299         cpu_to_be32(IBA_GET(CM_LAP_REMOTE_COMM_ID, lap_msg)),
3300         cpu_to_be32(IBA_GET(CM_LAP_LOCAL_COMM_ID, lap_msg)));
3301     if (!cm_id_priv)
3302         return -EINVAL;
3303
3304     param = &work->cm_event.param.lap_rcvd;
3305     memset(&work->path[0], 0, sizeof(work->path[1]));
3306     cm_path_set_rec_type(work->port->cm_dev->ib_device,
3307                  work->port->port_num, &work->path[0],
3308                  IBA_GET_MEM_PTR(CM_LAP_ALTERNATE_LOCAL_PORT_GID,
3309                          lap_msg));
3310     param->alternate_path = &work->path[0];
3311     cm_format_path_from_lap(cm_id_priv, param->alternate_path, lap_msg);
3312     work->cm_event.private_data =
3313         IBA_GET_MEM_PTR(CM_LAP_PRIVATE_DATA, lap_msg);
3314
3315     ret = ib_init_ah_attr_from_wc(work->port->cm_dev->ib_device,
3316                       work->port->port_num,
3317                       work->mad_recv_wc->wc,
3318                       work->mad_recv_wc->recv_buf.grh,
3319                       &ah_attr);
3320     if (ret)
3321         goto deref;
3322
3323     ret = cm_init_av_by_path(param->alternate_path, NULL, &alt_av);
3324     if (ret) {
3325         rdma_destroy_ah_attr(&ah_attr);
3326         goto deref;
3327     }
3328
3329     spin_lock_irq(&cm_id_priv->lock);
3330     cm_init_av_for_lap(work->port, work->mad_recv_wc->wc,
3331                &ah_attr, &cm_id_priv->av);
3332     cm_move_av_from_path(&cm_id_priv->alt_av, &alt_av);
3333
3334     if (cm_id_priv->id.state != IB_CM_ESTABLISHED)
3335         goto unlock;
3336
3337     switch (cm_id_priv->id.lap_state) {
3338     case IB_CM_LAP_UNINIT:
3339     case IB_CM_LAP_IDLE:
3340         break;
3341     case IB_CM_MRA_LAP_SENT:
3342         atomic_long_inc(&work->port->counters[CM_RECV_DUPLICATES]
3343                              [CM_LAP_COUNTER]);
3344         msg = cm_alloc_response_msg_no_ah(work->port, work->mad_recv_wc);
3345         if (IS_ERR(msg))
3346             goto unlock;
3347
3348         cm_format_mra((struct cm_mra_msg *) msg->mad, cm_id_priv,
3349                   CM_MSG_RESPONSE_OTHER,
3350                   cm_id_priv->service_timeout,
3351                   cm_id_priv->private_data,
3352                   cm_id_priv->private_data_len);
3353         spin_unlock_irq(&cm_id_priv->lock);
3354
3355         if (cm_create_response_msg_ah(work->port, work->mad_recv_wc, msg) ||
3356             ib_post_send_mad(msg, NULL))
3357             cm_free_response_msg(msg);
3358         goto deref;
3359     case IB_CM_LAP_RCVD:
3360         atomic_long_inc(&work->port->counters[CM_RECV_DUPLICATES]
3361                              [CM_LAP_COUNTER]);
3362         goto unlock;
3363     default:
3364         goto unlock;
3365     }
3366
3367     cm_id_priv->id.lap_state = IB_CM_LAP_RCVD;
3368     cm_id_priv->tid = lap_msg->hdr.tid;
3369     cm_queue_work_unlock(cm_id_priv, work);
3370     return 0;
3371
3372 unlock: spin_unlock_irq(&cm_id_priv->lock);
3373 deref:  cm_deref_id(cm_id_priv);
3374     return -EINVAL;
3375 }
3376
3377 static int cm_apr_handler(struct cm_work *work)
3378 {
3379     struct cm_id_private *cm_id_priv;
3380     struct cm_apr_msg *apr_msg;
3381
3382     /* Currently Alternate path messages are not supported for
3383      * RoCE link layer.
3384      */
3385     if (rdma_protocol_roce(work->port->cm_dev->ib_device,
3386                    work->port->port_num))
3387         return -EINVAL;
3388
3389     apr_msg = (struct cm_apr_msg *)work->mad_recv_wc->recv_buf.mad;
3390     cm_id_priv = cm_acquire_id(
3391         cpu_to_be32(IBA_GET(CM_APR_REMOTE_COMM_ID, apr_msg)),
3392         cpu_to_be32(IBA_GET(CM_APR_LOCAL_COMM_ID, apr_msg)));
3393     if (!cm_id_priv)
3394         return -EINVAL; /* Unmatched reply. */
3395
3396     work->cm_event.param.apr_rcvd.ap_status =
3397         IBA_GET(CM_APR_AR_STATUS, apr_msg);
3398     work->cm_event.param.apr_rcvd.apr_info =
3399         IBA_GET_MEM_PTR(CM_APR_ADDITIONAL_INFORMATION, apr_msg);
3400     work->cm_event.param.apr_rcvd.info_len =
3401         IBA_GET(CM_APR_ADDITIONAL_INFORMATION_LENGTH, apr_msg);
3402     work->cm_event.private_data =
3403         IBA_GET_MEM_PTR(CM_APR_PRIVATE_DATA, apr_msg);
3404
3405     spin_lock_irq(&cm_id_priv->lock);
3406     if (cm_id_priv->id.state != IB_CM_ESTABLISHED ||
3407         (cm_id_priv->id.lap_state != IB_CM_LAP_SENT &&
3408          cm_id_priv->id.lap_state != IB_CM_MRA_LAP_RCVD)) {
3409         spin_unlock_irq(&cm_id_priv->lock);
3410         goto out;
3411     }
3412     cm_id_priv->id.lap_state = IB_CM_LAP_IDLE;
3413     ib_cancel_mad(cm_id_priv->msg);
3414     cm_queue_work_unlock(cm_id_priv, work);
3415     return 0;
3416 out:
3417     cm_deref_id(cm_id_priv);
3418     return -EINVAL;
3419 }
3420
3421 static int cm_timewait_handler(struct cm_work *work)
3422 {
3423     struct cm_timewait_info *timewait_info;
3424     struct cm_id_private *cm_id_priv;
3425
3426     timewait_info = container_of(work, struct cm_timewait_info, work);
3427     spin_lock_irq(&cm.lock);
3428     list_del(&timewait_info->list);
3429     spin_unlock_irq(&cm.lock);
3430
3431     cm_id_priv = cm_acquire_id(timewait_info->work.local_id,
3432                    timewait_info->work.remote_id);
3433     if (!cm_id_priv)
3434         return -EINVAL;
3435
3436     spin_lock_irq(&cm_id_priv->lock);
3437     if (cm_id_priv->id.state != IB_CM_TIMEWAIT ||
3438         cm_id_priv->remote_qpn != timewait_info->remote_qpn) {
3439         spin_unlock_irq(&cm_id_priv->lock);
3440         goto out;
3441     }
3442     cm_id_priv->id.state = IB_CM_IDLE;
3443     cm_queue_work_unlock(cm_id_priv, work);
3444     return 0;
3445 out:
3446     cm_deref_id(cm_id_priv);
3447     return -EINVAL;
3448 }
3449
3450 static void cm_format_sidr_req(struct cm_sidr_req_msg *sidr_req_msg,
3451                    struct cm_id_private *cm_id_priv,
3452                    struct ib_cm_sidr_req_param *param)
3453 {
3454     cm_format_mad_hdr(&sidr_req_msg->hdr, CM_SIDR_REQ_ATTR_ID,
3455               cm_form_tid(cm_id_priv));
3456     IBA_SET(CM_SIDR_REQ_REQUESTID, sidr_req_msg,
3457         be32_to_cpu(cm_id_priv->id.local_id));
3458     IBA_SET(CM_SIDR_REQ_PARTITION_KEY, sidr_req_msg,
3459         be16_to_cpu(param->path->pkey));
3460     IBA_SET(CM_SIDR_REQ_SERVICEID, sidr_req_msg,
3461         be64_to_cpu(param->service_id));
3462
3463     if (param->private_data && param->private_data_len)
3464         IBA_SET_MEM(CM_SIDR_REQ_PRIVATE_DATA, sidr_req_msg,
3465                 param->private_data, param->private_data_len);
3466 }
3467
3468 int ib_send_cm_sidr_req(struct ib_cm_id *cm_id,
3469             struct ib_cm_sidr_req_param *param)
3470 {
3471     struct cm_id_private *cm_id_priv;
3472     struct ib_mad_send_buf *msg;
3473     struct cm_av av = {};
3474     unsigned long flags;
3475     int ret;
3476
3477     if (!param->path || (param->private_data &&
3478          param->private_data_len > IB_CM_SIDR_REQ_PRIVATE_DATA_SIZE))
3479         return -EINVAL;
3480
3481     cm_id_priv = container_of(cm_id, struct cm_id_private, id);
3482     ret = cm_init_av_by_path(param->path, param->sgid_attr, &av);
3483     if (ret)
3484         return ret;
3485
3486     spin_lock_irqsave(&cm_id_priv->lock, flags);
3487     cm_move_av_from_path(&cm_id_priv->av, &av);
3488     cm_id->service_id = param->service_id;
3489     cm_id->service_mask = ~cpu_to_be64(0);
3490     cm_id_priv->timeout_ms = param->timeout_ms;
3491     cm_id_priv->max_cm_retries = param->max_cm_retries;
3492     if (cm_id->state != IB_CM_IDLE) {
3493         ret = -EINVAL;
3494         goto out_unlock;
3495     }
3496
3497     msg = cm_alloc_priv_msg(cm_id_priv);
3498     if (IS_ERR(msg)) {
3499         ret = PTR_ERR(msg);
3500         goto out_unlock;
3501     }
3502
3503     cm_format_sidr_req((struct cm_sidr_req_msg *)msg->mad, cm_id_priv,
3504                param);
3505     msg->timeout_ms = cm_id_priv->timeout_ms;
3506     msg->context[1] = (void *)(unsigned long)IB_CM_SIDR_REQ_SENT;
3507
3508     trace_icm_send_sidr_req(&cm_id_priv->id);
3509     ret = ib_post_send_mad(msg, NULL);
3510     if (ret)
3511         goto out_free;
3512     cm_id->state = IB_CM_SIDR_REQ_SENT;
3513     spin_unlock_irqrestore(&cm_id_priv->lock, flags);
3514     return 0;
3515 out_free:
3516     cm_free_priv_msg(msg);
3517 out_unlock:
3518     spin_unlock_irqrestore(&cm_id_priv->lock, flags);
3519     return ret;
3520 }
3521 EXPORT_SYMBOL(ib_send_cm_sidr_req);
3522
3523 static void cm_format_sidr_req_event(struct cm_work *work,
3524                      const struct cm_id_private *rx_cm_id,
3525                      struct ib_cm_id *listen_id)
3526 {
3527     struct cm_sidr_req_msg *sidr_req_msg;
3528     struct ib_cm_sidr_req_event_param *param;
3529
3530     sidr_req_msg = (struct cm_sidr_req_msg *)
3531                 work->mad_recv_wc->recv_buf.mad;
3532     param = &work->cm_event.param.sidr_req_rcvd;
3533     param->pkey = IBA_GET(CM_SIDR_REQ_PARTITION_KEY, sidr_req_msg);
3534     param->listen_id = listen_id;
3535     param->service_id =
3536         cpu_to_be64(IBA_GET(CM_SIDR_REQ_SERVICEID, sidr_req_msg));
3537     param->bth_pkey = cm_get_bth_pkey(work);
3538     param->port = work->port->port_num;
3539     param->sgid_attr = rx_cm_id->av.ah_attr.grh.sgid_attr;
3540     work->cm_event.private_data =
3541         IBA_GET_MEM_PTR(CM_SIDR_REQ_PRIVATE_DATA, sidr_req_msg);
3542 }
3543
3544 static int cm_sidr_req_handler(struct cm_work *work)
3545 {
3546     struct cm_id_private *cm_id_priv, *listen_cm_id_priv;
3547     struct cm_sidr_req_msg *sidr_req_msg;
3548     struct ib_wc *wc;
3549     int ret;
3550
3551     cm_id_priv =
3552         cm_alloc_id_priv(work->port->cm_dev->ib_device, NULL, NULL);
3553     if (IS_ERR(cm_id_priv))
3554         return PTR_ERR(cm_id_priv);
3555
3556     /* Record SGID/SLID and request ID for lookup. */
3557     sidr_req_msg = (struct cm_sidr_req_msg *)
3558                 work->mad_recv_wc->recv_buf.mad;
3559
3560     cm_id_priv->id.remote_id =
3561         cpu_to_be32(IBA_GET(CM_SIDR_REQ_REQUESTID, sidr_req_msg));
3562     cm_id_priv->id.service_id =
3563         cpu_to_be64(IBA_GET(CM_SIDR_REQ_SERVICEID, sidr_req_msg));
3564     cm_id_priv->id.service_mask = ~cpu_to_be64(0);
3565     cm_id_priv->tid = sidr_req_msg->hdr.tid;
3566
3567     wc = work->mad_recv_wc->wc;
3568     cm_id_priv->sidr_slid = wc->slid;
3569     ret = cm_init_av_for_response(work->port, work->mad_recv_wc->wc,
3570                       work->mad_recv_wc->recv_buf.grh,
3571                       &cm_id_priv->av);
3572     if (ret)
3573         goto out;
3574
3575     spin_lock_irq(&cm.lock);
3576     listen_cm_id_priv = cm_insert_remote_sidr(cm_id_priv);
3577     if (listen_cm_id_priv) {
3578         spin_unlock_irq(&cm.lock);
3579         atomic_long_inc(&work->port->counters[CM_RECV_DUPLICATES]
3580                              [CM_SIDR_REQ_COUNTER]);
3581         goto out; /* Duplicate message. */
3582     }
3583     cm_id_priv->id.state = IB_CM_SIDR_REQ_RCVD;
3584     listen_cm_id_priv = cm_find_listen(cm_id_priv->id.device,
3585                        cm_id_priv->id.service_id);
3586     if (!listen_cm_id_priv) {
3587         spin_unlock_irq(&cm.lock);
3588         ib_send_cm_sidr_rep(&cm_id_priv->id,
3589                     &(struct ib_cm_sidr_rep_param){
3590                         .status = IB_SIDR_UNSUPPORTED });
3591         goto out; /* No match. */
3592     }
3593     spin_unlock_irq(&cm.lock);
3594
3595     cm_id_priv->id.cm_handler = listen_cm_id_priv->id.cm_handler;
3596     cm_id_priv->id.context = listen_cm_id_priv->id.context;
3597
3598     /*
3599      * A SIDR ID does not need to be in the xarray since it does not receive
3600      * mads, is not placed in the remote_id or remote_qpn rbtree, and does
3601      * not enter timewait.
3602      */
3603
3604     cm_format_sidr_req_event(work, cm_id_priv, &listen_cm_id_priv->id);
3605     ret = cm_id_priv->id.cm_handler(&cm_id_priv->id, &work->cm_event);
3606     cm_free_work(work);
3607     /*
3608      * A pointer to the listen_cm_id is held in the event, so this deref
3609      * must be after the event is delivered above.
3610      */
3611     cm_deref_id(listen_cm_id_priv);
3612     if (ret)
3613         cm_destroy_id(&cm_id_priv->id, ret);
3614     return 0;
3615 out:
3616     ib_destroy_cm_id(&cm_id_priv->id);
3617     return -EINVAL;
3618 }
3619
3620 static void cm_format_sidr_rep(struct cm_sidr_rep_msg *sidr_rep_msg,
3621                    struct cm_id_private *cm_id_priv,
3622                    struct ib_cm_sidr_rep_param *param)
3623 {
3624     cm_format_mad_ece_hdr(&sidr_rep_msg->hdr, CM_SIDR_REP_ATTR_ID,
3625                   cm_id_priv->tid, param->ece.attr_mod);
3626     IBA_SET(CM_SIDR_REP_REQUESTID, sidr_rep_msg,
3627         be32_to_cpu(cm_id_priv->id.remote_id));
3628     IBA_SET(CM_SIDR_REP_STATUS, sidr_rep_msg, param->status);
3629     IBA_SET(CM_SIDR_REP_QPN, sidr_rep_msg, param->qp_num);
3630     IBA_SET(CM_SIDR_REP_SERVICEID, sidr_rep_msg,
3631         be64_to_cpu(cm_id_priv->id.service_id));
3632     IBA_SET(CM_SIDR_REP_Q_KEY, sidr_rep_msg, param->qkey);
3633     IBA_SET(CM_SIDR_REP_VENDOR_ID_L, sidr_rep_msg,
3634         param->ece.vendor_id & 0xFF);
3635     IBA_SET(CM_SIDR_REP_VENDOR_ID_H, sidr_rep_msg,
3636         (param->ece.vendor_id >> 8) & 0xFF);
3637
3638     if (param->info && param->info_length)
3639         IBA_SET_MEM(CM_SIDR_REP_ADDITIONAL_INFORMATION, sidr_rep_msg,
3640                 param->info, param->info_length);
3641
3642     if (param->private_data && param->private_data_len)
3643         IBA_SET_MEM(CM_SIDR_REP_PRIVATE_DATA, sidr_rep_msg,
3644                 param->private_data, param->private_data_len);
3645 }
3646
3647 static int cm_send_sidr_rep_locked(struct cm_id_private *cm_id_priv,
3648                    struct ib_cm_sidr_rep_param *param)
3649 {
3650     struct ib_mad_send_buf *msg;
3651     unsigned long flags;
3652     int ret;
3653
3654     lockdep_assert_held(&cm_id_priv->lock);
3655
3656     if ((param->info && param->info_length > IB_CM_SIDR_REP_INFO_LENGTH) ||
3657         (param->private_data &&
3658          param->private_data_len > IB_CM_SIDR_REP_PRIVATE_DATA_SIZE))
3659         return -EINVAL;
3660
3661     if (cm_id_priv->id.state != IB_CM_SIDR_REQ_RCVD)
3662         return -EINVAL;
3663
3664     msg = cm_alloc_msg(cm_id_priv);
3665     if (IS_ERR(msg))
3666         return PTR_ERR(msg);
3667
3668     cm_format_sidr_rep((struct cm_sidr_rep_msg *) msg->mad, cm_id_priv,
3669                param);
3670     trace_icm_send_sidr_rep(&cm_id_priv->id);
3671     ret = ib_post_send_mad(msg, NULL);
3672     if (ret) {
3673         cm_free_msg(msg);
3674         return ret;
3675     }
3676     cm_id_priv->id.state = IB_CM_IDLE;
3677     spin_lock_irqsave(&cm.lock, flags);
3678     if (!RB_EMPTY_NODE(&cm_id_priv->sidr_id_node)) {
3679         rb_erase(&cm_id_priv->sidr_id_node, &cm.remote_sidr_table);
3680         RB_CLEAR_NODE(&cm_id_priv->sidr_id_node);
3681     }
3682     spin_unlock_irqrestore(&cm.lock, flags);
3683     return 0;
3684 }
3685
3686 int ib_send_cm_sidr_rep(struct ib_cm_id *cm_id,
3687             struct ib_cm_sidr_rep_param *param)
3688 {
3689     struct cm_id_private *cm_id_priv =
3690         container_of(cm_id, struct cm_id_private, id);
3691     unsigned long flags;
3692     int ret;
3693
3694     spin_lock_irqsave(&cm_id_priv->lock, flags);
3695     ret = cm_send_sidr_rep_locked(cm_id_priv, param);
3696     spin_unlock_irqrestore(&cm_id_priv->lock, flags);
3697     return ret;
3698 }
3699 EXPORT_SYMBOL(ib_send_cm_sidr_rep);
3700
3701 static void cm_format_sidr_rep_event(struct cm_work *work,
3702                      const struct cm_id_private *cm_id_priv)
3703 {
3704     struct cm_sidr_rep_msg *sidr_rep_msg;
3705     struct ib_cm_sidr_rep_event_param *param;
3706
3707     sidr_rep_msg = (struct cm_sidr_rep_msg *)
3708                 work->mad_recv_wc->recv_buf.mad;
3709     param = &work->cm_event.param.sidr_rep_rcvd;
3710     param->status = IBA_GET(CM_SIDR_REP_STATUS, sidr_rep_msg);
3711     param->qkey = IBA_GET(CM_SIDR_REP_Q_KEY, sidr_rep_msg);
3712     param->qpn = IBA_GET(CM_SIDR_REP_QPN, sidr_rep_msg);
3713     param->info = IBA_GET_MEM_PTR(CM_SIDR_REP_ADDITIONAL_INFORMATION,
3714                       sidr_rep_msg);
3715     param->info_len = IBA_GET(CM_SIDR_REP_ADDITIONAL_INFORMATION_LENGTH,
3716                   sidr_rep_msg);
3717     param->sgid_attr = cm_id_priv->av.ah_attr.grh.sgid_attr;
3718     work->cm_event.private_data =
3719         IBA_GET_MEM_PTR(CM_SIDR_REP_PRIVATE_DATA, sidr_rep_msg);
3720 }
3721
3722 static int cm_sidr_rep_handler(struct cm_work *work)
3723 {
3724     struct cm_sidr_rep_msg *sidr_rep_msg;
3725     struct cm_id_private *cm_id_priv;
3726
3727     sidr_rep_msg = (struct cm_sidr_rep_msg *)
3728                 work->mad_recv_wc->recv_buf.mad;
3729     cm_id_priv = cm_acquire_id(
3730         cpu_to_be32(IBA_GET(CM_SIDR_REP_REQUESTID, sidr_rep_msg)), 0);
3731     if (!cm_id_priv)
3732         return -EINVAL; /* Unmatched reply. */
3733
3734     spin_lock_irq(&cm_id_priv->lock);
3735     if (cm_id_priv->id.state != IB_CM_SIDR_REQ_SENT) {
3736         spin_unlock_irq(&cm_id_priv->lock);
3737         goto out;
3738     }
3739     cm_id_priv->id.state = IB_CM_IDLE;
3740     ib_cancel_mad(cm_id_priv->msg);
3741     spin_unlock_irq(&cm_id_priv->lock);
3742
3743     cm_format_sidr_rep_event(work, cm_id_priv);
3744     cm_process_work(cm_id_priv, work);
3745     return 0;
3746 out:
3747     cm_deref_id(cm_id_priv);
3748     return -EINVAL;
3749 }
3750
3751 static void cm_process_send_error(struct cm_id_private *cm_id_priv,
3752                   struct ib_mad_send_buf *msg,
3753                   enum ib_cm_state state,
3754                   enum ib_wc_status wc_status)
3755 {
3756     struct ib_cm_event cm_event = {};
3757     int ret;
3758
3759     /* Discard old sends or ones without a response. */
3760     spin_lock_irq(&cm_id_priv->lock);
3761     if (msg != cm_id_priv->msg) {
3762         spin_unlock_irq(&cm_id_priv->lock);
3763         cm_free_msg(msg);
3764         return;
3765     }
3766     cm_free_priv_msg(msg);
3767
3768     if (state != cm_id_priv->id.state || wc_status == IB_WC_SUCCESS ||
3769         wc_status == IB_WC_WR_FLUSH_ERR)
3770         goto out_unlock;
3771
3772     trace_icm_mad_send_err(state, wc_status);
3773     switch (state) {
3774     case IB_CM_REQ_SENT:
3775     case IB_CM_MRA_REQ_RCVD:
3776         cm_reset_to_idle(cm_id_priv);
3777         cm_event.event = IB_CM_REQ_ERROR;
3778         break;
3779     case IB_CM_REP_SENT:
3780     case IB_CM_MRA_REP_RCVD:
3781         cm_reset_to_idle(cm_id_priv);
3782         cm_event.event = IB_CM_REP_ERROR;
3783         break;
3784     case IB_CM_DREQ_SENT:
3785         cm_enter_timewait(cm_id_priv);
3786         cm_event.event = IB_CM_DREQ_ERROR;
3787         break;
3788     case IB_CM_SIDR_REQ_SENT:
3789         cm_id_priv->id.state = IB_CM_IDLE;
3790         cm_event.event = IB_CM_SIDR_REQ_ERROR;
3791         break;
3792     default:
3793         goto out_unlock;
3794     }
3795     spin_unlock_irq(&cm_id_priv->lock);
3796     cm_event.param.send_status = wc_status;
3797
3798     /* No other events can occur on the cm_id at this point. */
3799     ret = cm_id_priv->id.cm_handler(&cm_id_priv->id, &cm_event);
3800     if (ret)
3801         ib_destroy_cm_id(&cm_id_priv->id);
3802     return;
3803 out_unlock:
3804     spin_unlock_irq(&cm_id_priv->lock);
3805 }
3806
3807 static void cm_send_handler(struct ib_mad_agent *mad_agent,
3808                 struct ib_mad_send_wc *mad_send_wc)
3809 {
3810     struct ib_mad_send_buf *msg = mad_send_wc->send_buf;
3811     struct cm_id_private *cm_id_priv = msg->context[0];
3812     enum ib_cm_state state =
3813         (enum ib_cm_state)(unsigned long)msg->context[1];
3814     struct cm_port *port;
3815     u16 attr_index;
3816
3817     port = mad_agent->context;
3818     attr_index = be16_to_cpu(((struct ib_mad_hdr *)
3819                   msg->mad)->attr_id) - CM_ATTR_ID_OFFSET;
3820
3821     /*
3822      * If the send was in response to a received message (context[0] is not
3823      * set to a cm_id), and is not a REJ, then it is a send that was
3824      * manually retried.
3825      */
3826     if (!cm_id_priv && (attr_index != CM_REJ_COUNTER))
3827         msg->retries = 1;
3828
3829     atomic_long_add(1 + msg->retries, &port->counters[CM_XMIT][attr_index]);
3830     if (msg->retries)
3831         atomic_long_add(msg->retries,
3832                 &port->counters[CM_XMIT_RETRIES][attr_index]);
3833
3834     if (cm_id_priv)
3835         cm_process_send_error(cm_id_priv, msg, state,
3836                       mad_send_wc->status);
3837     else
3838         cm_free_response_msg(msg);
3839 }
3840
3841 static void cm_work_handler(struct work_struct *_work)
3842 {
3843     struct cm_work *work = container_of(_work, struct cm_work, work.work);
3844     int ret;
3845
3846     switch (work->cm_event.event) {
3847     case IB_CM_REQ_RECEIVED:
3848         ret = cm_req_handler(work);
3849         break;
3850     case IB_CM_MRA_RECEIVED:
3851         ret = cm_mra_handler(work);
3852         break;
3853     case IB_CM_REJ_RECEIVED:
3854         ret = cm_rej_handler(work);
3855         break;
3856     case IB_CM_REP_RECEIVED:
3857         ret = cm_rep_handler(work);
3858         break;
3859     case IB_CM_RTU_RECEIVED:
3860         ret = cm_rtu_handler(work);
3861         break;
3862     case IB_CM_USER_ESTABLISHED:
3863         ret = cm_establish_handler(work);
3864         break;
3865     case IB_CM_DREQ_RECEIVED:
3866         ret = cm_dreq_handler(work);
3867         break;
3868     case IB_CM_DREP_RECEIVED:
3869         ret = cm_drep_handler(work);
3870         break;
3871     case IB_CM_SIDR_REQ_RECEIVED:
3872         ret = cm_sidr_req_handler(work);
3873         break;
3874     case IB_CM_SIDR_REP_RECEIVED:
3875         ret = cm_sidr_rep_handler(work);
3876         break;
3877     case IB_CM_LAP_RECEIVED:
3878         ret = cm_lap_handler(work);
3879         break;
3880     case IB_CM_APR_RECEIVED:
3881         ret = cm_apr_handler(work);
3882         break;
3883     case IB_CM_TIMEWAIT_EXIT:
3884         ret = cm_timewait_handler(work);
3885         break;
3886     default:
3887         trace_icm_handler_err(work->cm_event.event);
3888         ret = -EINVAL;
3889         break;
3890     }
3891     if (ret)
3892         cm_free_work(work);
3893 }
3894
3895 static int cm_establish(struct ib_cm_id *cm_id)
3896 {
3897     struct cm_id_private *cm_id_priv;
3898     struct cm_work *work;
3899     unsigned long flags;
3900     int ret = 0;
3901     struct cm_device *cm_dev;
3902
3903     cm_dev = ib_get_client_data(cm_id->device, &cm_client);
3904     if (!cm_dev)
3905         return -ENODEV;
3906
3907     work = kmalloc(sizeof *work, GFP_ATOMIC);
3908     if (!work)
3909         return -ENOMEM;
3910
3911     cm_id_priv = container_of(cm_id, struct cm_id_private, id);
3912     spin_lock_irqsave(&cm_id_priv->lock, flags);
3913     switch (cm_id->state) {
3914     case IB_CM_REP_SENT:
3915     case IB_CM_MRA_REP_RCVD:
3916         cm_id->state = IB_CM_ESTABLISHED;
3917         break;
3918     case IB_CM_ESTABLISHED:
3919         ret = -EISCONN;
3920         break;
3921     default:
3922         trace_icm_establish_err(cm_id);
3923         ret = -EINVAL;
3924         break;
3925     }
3926     spin_unlock_irqrestore(&cm_id_priv->lock, flags);
3927
3928     if (ret) {
3929         kfree(work);
3930         goto out;
3931     }
3932
3933     /*
3934      * The CM worker thread may try to destroy the cm_id before it
3935      * can execute this work item.  To prevent potential deadlock,
3936      * we need to find the cm_id once we're in the context of the
3937      * worker thread, rather than holding a reference on it.
3938      */
3939     INIT_DELAYED_WORK(&work->work, cm_work_handler);
3940     work->local_id = cm_id->local_id;
3941     work->remote_id = cm_id->remote_id;
3942     work->mad_recv_wc = NULL;
3943     work->cm_event.event = IB_CM_USER_ESTABLISHED;
3944
3945     /* Check if the device started its remove_one */
3946     spin_lock_irqsave(&cm.lock, flags);
3947     if (!cm_dev->going_down) {
3948         queue_delayed_work(cm.wq, &work->work, 0);
3949     } else {
3950         kfree(work);
3951         ret = -ENODEV;
3952     }
3953     spin_unlock_irqrestore(&cm.lock, flags);
3954
3955 out:
3956     return ret;
3957 }
3958
3959 static int cm_migrate(struct ib_cm_id *cm_id)
3960 {
3961     struct cm_id_private *cm_id_priv;
3962     unsigned long flags;
3963     int ret = 0;
3964
3965     cm_id_priv = container_of(cm_id, struct cm_id_private, id);
3966     spin_lock_irqsave(&cm_id_priv->lock, flags);
3967     if (cm_id->state == IB_CM_ESTABLISHED &&
3968         (cm_id->lap_state == IB_CM_LAP_UNINIT ||
3969          cm_id->lap_state == IB_CM_LAP_IDLE)) {
3970         cm_id->lap_state = IB_CM_LAP_IDLE;
3971         cm_id_priv->av = cm_id_priv->alt_av;
3972     } else
3973         ret = -EINVAL;
3974     spin_unlock_irqrestore(&cm_id_priv->lock, flags);
3975
3976     return ret;
3977 }
3978
3979 int ib_cm_notify(struct ib_cm_id *cm_id, enum ib_event_type event)
3980 {
3981     int ret;
3982
3983     switch (event) {
3984     case IB_EVENT_COMM_EST:
3985         ret = cm_establish(cm_id);
3986         break;
3987     case IB_EVENT_PATH_MIG:
3988         ret = cm_migrate(cm_id);
3989         break;
3990     default:
3991         ret = -EINVAL;
3992     }
3993     return ret;
3994 }
3995 EXPORT_SYMBOL(ib_cm_notify);
3996
3997 static void cm_recv_handler(struct ib_mad_agent *mad_agent,
3998                 struct ib_mad_send_buf *send_buf,
3999                 struct ib_mad_recv_wc *mad_recv_wc)
4000 {
4001     struct cm_port *port = mad_agent->context;
4002     struct cm_work *work;
4003     enum ib_cm_event_type event;
4004     bool alt_path = false;
4005     u16 attr_id;
4006     int paths = 0;
4007     int going_down = 0;
4008
4009     switch (mad_recv_wc->recv_buf.mad->mad_hdr.attr_id) {
4010     case CM_REQ_ATTR_ID:
4011         alt_path = cm_req_has_alt_path((struct cm_req_msg *)
4012                         mad_recv_wc->recv_buf.mad);
4013         paths = 1 + (alt_path != 0);
4014         event = IB_CM_REQ_RECEIVED;
4015         break;
4016     case CM_MRA_ATTR_ID:
4017         event = IB_CM_MRA_RECEIVED;
4018         break;
4019     case CM_REJ_ATTR_ID:
4020         event = IB_CM_REJ_RECEIVED;
4021         break;
4022     case CM_REP_ATTR_ID:
4023         event = IB_CM_REP_RECEIVED;
4024         break;
4025     case CM_RTU_ATTR_ID:
4026         event = IB_CM_RTU_RECEIVED;
4027         break;
4028     case CM_DREQ_ATTR_ID:
4029         event = IB_CM_DREQ_RECEIVED;
4030         break;
4031     case CM_DREP_ATTR_ID:
4032         event = IB_CM_DREP_RECEIVED;
4033         break;
4034     case CM_SIDR_REQ_ATTR_ID:
4035         event = IB_CM_SIDR_REQ_RECEIVED;
4036         break;
4037     case CM_SIDR_REP_ATTR_ID:
4038         event = IB_CM_SIDR_REP_RECEIVED;
4039         break;
4040     case CM_LAP_ATTR_ID:
4041         paths = 1;
4042         event = IB_CM_LAP_RECEIVED;
4043         break;
4044     case CM_APR_ATTR_ID:
4045         event = IB_CM_APR_RECEIVED;
4046         break;
4047     default:
4048         ib_free_recv_mad(mad_recv_wc);
4049         return;
4050     }
4051
4052     attr_id = be16_to_cpu(mad_recv_wc->recv_buf.mad->mad_hdr.attr_id);
4053     atomic_long_inc(&port->counters[CM_RECV][attr_id - CM_ATTR_ID_OFFSET]);
4054
4055     work = kmalloc(struct_size(work, path, paths), GFP_KERNEL);
4056     if (!work) {
4057         ib_free_recv_mad(mad_recv_wc);
4058         return;
4059     }
4060
4061     INIT_DELAYED_WORK(&work->work, cm_work_handler);
4062     work->cm_event.event = event;
4063     work->mad_recv_wc = mad_recv_wc;
4064     work->port = port;
4065
4066     /* Check if the device started its remove_one */
4067     spin_lock_irq(&cm.lock);
4068     if (!port->cm_dev->going_down)
4069         queue_delayed_work(cm.wq, &work->work, 0);
4070     else
4071         going_down = 1;
4072     spin_unlock_irq(&cm.lock);
4073
4074     if (going_down) {
4075         kfree(work);
4076         ib_free_recv_mad(mad_recv_wc);
4077     }
4078 }
4079
4080 static int cm_init_qp_init_attr(struct cm_id_private *cm_id_priv,
4081                 struct ib_qp_attr *qp_attr,
4082                 int *qp_attr_mask)
4083 {
4084     unsigned long flags;
4085     int ret;
4086
4087     spin_lock_irqsave(&cm_id_priv->lock, flags);
4088     switch (cm_id_priv->id.state) {
4089     case IB_CM_REQ_SENT:
4090     case IB_CM_MRA_REQ_RCVD:
4091     case IB_CM_REQ_RCVD:
4092     case IB_CM_MRA_REQ_SENT:
4093     case IB_CM_REP_RCVD:
4094     case IB_CM_MRA_REP_SENT:
4095     case IB_CM_REP_SENT:
4096     case IB_CM_MRA_REP_RCVD:
4097     case IB_CM_ESTABLISHED:
4098         *qp_attr_mask = IB_QP_STATE | IB_QP_ACCESS_FLAGS |
4099                 IB_QP_PKEY_INDEX | IB_QP_PORT;
4100         qp_attr->qp_access_flags = IB_ACCESS_REMOTE_WRITE;
4101         if (cm_id_priv->responder_resources)
4102             qp_attr->qp_access_flags |= IB_ACCESS_REMOTE_READ |
4103                             IB_ACCESS_REMOTE_ATOMIC;
4104         qp_attr->pkey_index = cm_id_priv->av.pkey_index;
4105         if (cm_id_priv->av.port)
4106             qp_attr->port_num = cm_id_priv->av.port->port_num;
4107         ret = 0;
4108         break;
4109     default:
4110         trace_icm_qp_init_err(&cm_id_priv->id);
4111         ret = -EINVAL;
4112         break;
4113     }
4114     spin_unlock_irqrestore(&cm_id_priv->lock, flags);
4115     return ret;
4116 }
4117
4118 static int cm_init_qp_rtr_attr(struct cm_id_private *cm_id_priv,
4119                    struct ib_qp_attr *qp_attr,
4120                    int *qp_attr_mask)
4121 {
4122     unsigned long flags;
4123     int ret;
4124
4125     spin_lock_irqsave(&cm_id_priv->lock, flags);
4126     switch (cm_id_priv->id.state) {
4127     case IB_CM_REQ_RCVD:
4128     case IB_CM_MRA_REQ_SENT:
4129     case IB_CM_REP_RCVD:
4130     case IB_CM_MRA_REP_SENT:
4131     case IB_CM_REP_SENT:
4132     case IB_CM_MRA_REP_RCVD:
4133     case IB_CM_ESTABLISHED:
4134         *qp_attr_mask = IB_QP_STATE | IB_QP_AV | IB_QP_PATH_MTU |
4135                 IB_QP_DEST_QPN | IB_QP_RQ_PSN;
4136         qp_attr->ah_attr = cm_id_priv->av.ah_attr;
4137         qp_attr->path_mtu = cm_id_priv->path_mtu;
4138         qp_attr->dest_qp_num = be32_to_cpu(cm_id_priv->remote_qpn);
4139         qp_attr->rq_psn = be32_to_cpu(cm_id_priv->rq_psn);
4140         if (cm_id_priv->qp_type == IB_QPT_RC ||
4141             cm_id_priv->qp_type == IB_QPT_XRC_TGT) {
4142             *qp_attr_mask |= IB_QP_MAX_DEST_RD_ATOMIC |
4143                      IB_QP_MIN_RNR_TIMER;
4144             qp_attr->max_dest_rd_atomic =
4145                     cm_id_priv->responder_resources;
4146             qp_attr->min_rnr_timer = 0;
4147         }
4148         if (rdma_ah_get_dlid(&cm_id_priv->alt_av.ah_attr) &&
4149             cm_id_priv->alt_av.port) {
4150             *qp_attr_mask |= IB_QP_ALT_PATH;
4151             qp_attr->alt_port_num = cm_id_priv->alt_av.port->port_num;
4152             qp_attr->alt_pkey_index = cm_id_priv->alt_av.pkey_index;
4153             qp_attr->alt_timeout = cm_id_priv->alt_av.timeout;
4154             qp_attr->alt_ah_attr = cm_id_priv->alt_av.ah_attr;
4155         }
4156         ret = 0;
4157         break;
4158     default:
4159         trace_icm_qp_rtr_err(&cm_id_priv->id);
4160         ret = -EINVAL;
4161         break;
4162     }
4163     spin_unlock_irqrestore(&cm_id_priv->lock, flags);
4164     return ret;
4165 }
4166
4167 static int cm_init_qp_rts_attr(struct cm_id_private *cm_id_priv,
4168                    struct ib_qp_attr *qp_attr,
4169                    int *qp_attr_mask)
4170 {
4171     unsigned long flags;
4172     int ret;
4173
4174     spin_lock_irqsave(&cm_id_priv->lock, flags);
4175     switch (cm_id_priv->id.state) {
4176     /* Allow transition to RTS before sending REP */
4177     case IB_CM_REQ_RCVD:
4178     case IB_CM_MRA_REQ_SENT:
4179
4180     case IB_CM_REP_RCVD:
4181     case IB_CM_MRA_REP_SENT:
4182     case IB_CM_REP_SENT:
4183     case IB_CM_MRA_REP_RCVD:
4184     case IB_CM_ESTABLISHED:
4185         if (cm_id_priv->id.lap_state == IB_CM_LAP_UNINIT) {
4186             *qp_attr_mask = IB_QP_STATE | IB_QP_SQ_PSN;
4187             qp_attr->sq_psn = be32_to_cpu(cm_id_priv->sq_psn);
4188             switch (cm_id_priv->qp_type) {
4189             case IB_QPT_RC:
4190             case IB_QPT_XRC_INI:
4191                 *qp_attr_mask |= IB_QP_RETRY_CNT | IB_QP_RNR_RETRY |
4192                          IB_QP_MAX_QP_RD_ATOMIC;
4193                 qp_attr->retry_cnt = cm_id_priv->retry_count;
4194                 qp_attr->rnr_retry = cm_id_priv->rnr_retry_count;
4195                 qp_attr->max_rd_atomic = cm_id_priv->initiator_depth;
4196                 fallthrough;
4197             case IB_QPT_XRC_TGT:
4198                 *qp_attr_mask |= IB_QP_TIMEOUT;
4199                 qp_attr->timeout = cm_id_priv->av.timeout;
4200                 break;
4201             default:
4202                 break;
4203             }
4204             if (rdma_ah_get_dlid(&cm_id_priv->alt_av.ah_attr)) {
4205                 *qp_attr_mask |= IB_QP_PATH_MIG_STATE;
4206                 qp_attr->path_mig_state = IB_MIG_REARM;
4207             }
4208         } else {
4209             *qp_attr_mask = IB_QP_ALT_PATH | IB_QP_PATH_MIG_STATE;
4210             if (cm_id_priv->alt_av.port)
4211                 qp_attr->alt_port_num =
4212                     cm_id_priv->alt_av.port->port_num;
4213             qp_attr->alt_pkey_index = cm_id_priv->alt_av.pkey_index;
4214             qp_attr->alt_timeout = cm_id_priv->alt_av.timeout;
4215             qp_attr->alt_ah_attr = cm_id_priv->alt_av.ah_attr;
4216             qp_attr->path_mig_state = IB_MIG_REARM;
4217         }
4218         ret = 0;
4219         break;
4220     default:
4221         trace_icm_qp_rts_err(&cm_id_priv->id);
4222         ret = -EINVAL;
4223         break;
4224     }
4225     spin_unlock_irqrestore(&cm_id_priv->lock, flags);
4226     return ret;
4227 }
4228
4229 int ib_cm_init_qp_attr(struct ib_cm_id *cm_id,
4230                struct ib_qp_attr *qp_attr,
4231                int *qp_attr_mask)
4232 {
4233     struct cm_id_private *cm_id_priv;
4234     int ret;
4235
4236     cm_id_priv = container_of(cm_id, struct cm_id_private, id);
4237     switch (qp_attr->qp_state) {
4238     case IB_QPS_INIT:
4239         ret = cm_init_qp_init_attr(cm_id_priv, qp_attr, qp_attr_mask);
4240         break;
4241     case IB_QPS_RTR:
4242         ret = cm_init_qp_rtr_attr(cm_id_priv, qp_attr, qp_attr_mask);
4243         break;
4244     case IB_QPS_RTS:
4245         ret = cm_init_qp_rts_attr(cm_id_priv, qp_attr, qp_attr_mask);
4246         break;
4247     default:
4248         ret = -EINVAL;
4249         break;
4250     }
4251     return ret;
4252 }
4253 EXPORT_SYMBOL(ib_cm_init_qp_attr);
4254
4255 static ssize_t cm_show_counter(struct ib_device *ibdev, u32 port_num,
4256                    struct ib_port_attribute *attr, char *buf)
4257 {
4258     struct cm_counter_attribute *cm_attr =
4259         container_of(attr, struct cm_counter_attribute, attr);
4260     struct cm_device *cm_dev = ib_get_client_data(ibdev, &cm_client);
4261
4262     if (WARN_ON(!cm_dev))
4263         return -EINVAL;
4264
4265     return sysfs_emit(
4266         buf, "%ld\n",
4267         atomic_long_read(
4268             &cm_dev->port[port_num - 1]
4269                  ->counters[cm_attr->group][cm_attr->index]));
4270 }
4271
4272 #define CM_COUNTER_ATTR(_name, _group, _index)                                 \
4273     {                                                                      \
4274         .attr = __ATTR(_name, 0444, cm_show_counter, NULL),            \
4275         .group = _group, .index = _index                               \
4276     }
4277
4278 #define CM_COUNTER_GROUP(_group, _name)                                        \
4279     static struct cm_counter_attribute cm_counter_attr_##_group[] = {      \
4280         CM_COUNTER_ATTR(req, _group, CM_REQ_COUNTER),                  \
4281         CM_COUNTER_ATTR(mra, _group, CM_MRA_COUNTER),                  \
4282         CM_COUNTER_ATTR(rej, _group, CM_REJ_COUNTER),                  \
4283         CM_COUNTER_ATTR(rep, _group, CM_REP_COUNTER),                  \
4284         CM_COUNTER_ATTR(rtu, _group, CM_RTU_COUNTER),                  \
4285         CM_COUNTER_ATTR(dreq, _group, CM_DREQ_COUNTER),                \
4286         CM_COUNTER_ATTR(drep, _group, CM_DREP_COUNTER),                \
4287         CM_COUNTER_ATTR(sidr_req, _group, CM_SIDR_REQ_COUNTER),        \
4288         CM_COUNTER_ATTR(sidr_rep, _group, CM_SIDR_REP_COUNTER),        \
4289         CM_COUNTER_ATTR(lap, _group, CM_LAP_COUNTER),                  \
4290         CM_COUNTER_ATTR(apr, _group, CM_APR_COUNTER),                  \
4291     };                                                                     \
4292     static struct attribute *cm_counter_attrs_##_group[] = {               \
4293         &cm_counter_attr_##_group[0].attr.attr,                        \
4294         &cm_counter_attr_##_group[1].attr.attr,                        \
4295         &cm_counter_attr_##_group[2].attr.attr,                        \
4296         &cm_counter_attr_##_group[3].attr.attr,                        \
4297         &cm_counter_attr_##_group[4].attr.attr,                        \
4298         &cm_counter_attr_##_group[5].attr.attr,                        \
4299         &cm_counter_attr_##_group[6].attr.attr,                        \
4300         &cm_counter_attr_##_group[7].attr.attr,                        \
4301         &cm_counter_attr_##_group[8].attr.attr,                        \
4302         &cm_counter_attr_##_group[9].attr.attr,                        \
4303         &cm_counter_attr_##_group[10].attr.attr,                       \
4304         NULL,                                                          \
4305     };                                                                     \
4306     static const struct attribute_group cm_counter_group_##_group = {      \
4307         .name = _name,                                                 \
4308         .attrs = cm_counter_attrs_##_group,                            \
4309     };
4310
4311 CM_COUNTER_GROUP(CM_XMIT, "cm_tx_msgs")
4312 CM_COUNTER_GROUP(CM_XMIT_RETRIES, "cm_tx_retries")
4313 CM_COUNTER_GROUP(CM_RECV, "cm_rx_msgs")
4314 CM_COUNTER_GROUP(CM_RECV_DUPLICATES, "cm_rx_duplicates")
4315
4316 static const struct attribute_group *cm_counter_groups[] = {
4317     &cm_counter_group_CM_XMIT,
4318     &cm_counter_group_CM_XMIT_RETRIES,
4319     &cm_counter_group_CM_RECV,
4320     &cm_counter_group_CM_RECV_DUPLICATES,
4321     NULL,
4322 };
4323
4324 static int cm_add_one(struct ib_device *ib_device)
4325 {
4326     struct cm_device *cm_dev;
4327     struct cm_port *port;
4328     struct ib_mad_reg_req reg_req = {
4329         .mgmt_class = IB_MGMT_CLASS_CM,
4330         .mgmt_class_version = IB_CM_CLASS_VERSION,
4331     };
4332     struct ib_port_modify port_modify = {
4333         .set_port_cap_mask = IB_PORT_CM_SUP
4334     };
4335     unsigned long flags;
4336     int ret;
4337     int count = 0;
4338     u32 i;
4339
4340     cm_dev = kzalloc(struct_size(cm_dev, port, ib_device->phys_port_cnt),
4341              GFP_KERNEL);
4342     if (!cm_dev)
4343         return -ENOMEM;
4344
4345     kref_init(&cm_dev->kref);
4346     spin_lock_init(&cm_dev->mad_agent_lock);
4347     cm_dev->ib_device = ib_device;
4348     cm_dev->ack_delay = ib_device->attrs.local_ca_ack_delay;
4349     cm_dev->going_down = 0;
4350
4351     ib_set_client_data(ib_device, &cm_client, cm_dev);
4352
4353     set_bit(IB_MGMT_METHOD_SEND, reg_req.method_mask);
4354     rdma_for_each_port (ib_device, i) {
4355         if (!rdma_cap_ib_cm(ib_device, i))
4356             continue;
4357
4358         port = kzalloc(sizeof *port, GFP_KERNEL);
4359         if (!port) {
4360             ret = -ENOMEM;
4361             goto error1;
4362         }
4363
4364         cm_dev->port[i-1] = port;
4365         port->cm_dev = cm_dev;
4366         port->port_num = i;
4367
4368         ret = ib_port_register_client_groups(ib_device, i,
4369                              cm_counter_groups);
4370         if (ret)
4371             goto error1;
4372
4373         port->mad_agent = ib_register_mad_agent(ib_device, i,
4374                             IB_QPT_GSI,
4375                             &reg_req,
4376                             0,
4377                             cm_send_handler,
4378                             cm_recv_handler,
4379                             port,
4380                             0);
4381         if (IS_ERR(port->mad_agent)) {
4382             ret = PTR_ERR(port->mad_agent);
4383             goto error2;
4384         }
4385
4386         ret = ib_modify_port(ib_device, i, 0, &port_modify);
4387         if (ret)
4388             goto error3;
4389
4390         count++;
4391     }
4392
4393     if (!count) {
4394         ret = -EOPNOTSUPP;
4395         goto free;
4396     }
4397
4398     write_lock_irqsave(&cm.device_lock, flags);
4399     list_add_tail(&cm_dev->list, &cm.device_list);
4400     write_unlock_irqrestore(&cm.device_lock, flags);
4401     return 0;
4402
4403 error3:
4404     ib_unregister_mad_agent(port->mad_agent);
4405 error2:
4406     ib_port_unregister_client_groups(ib_device, i, cm_counter_groups);
4407 error1:
4408     port_modify.set_port_cap_mask = 0;
4409     port_modify.clr_port_cap_mask = IB_PORT_CM_SUP;
4410     while (--i) {
4411         if (!rdma_cap_ib_cm(ib_device, i))
4412             continue;
4413
4414         port = cm_dev->port[i-1];
4415         ib_modify_port(ib_device, port->port_num, 0, &port_modify);
4416         ib_unregister_mad_agent(port->mad_agent);
4417         ib_port_unregister_client_groups(ib_device, i,
4418                          cm_counter_groups);
4419     }
4420 free:
4421     cm_device_put(cm_dev);
4422     return ret;
4423 }
4424
4425 static void cm_remove_one(struct ib_device *ib_device, void *client_data)
4426 {
4427     struct cm_device *cm_dev = client_data;
4428     struct cm_port *port;
4429     struct ib_port_modify port_modify = {
4430         .clr_port_cap_mask = IB_PORT_CM_SUP
4431     };
4432     unsigned long flags;
4433     u32 i;
4434
4435     write_lock_irqsave(&cm.device_lock, flags);
4436     list_del(&cm_dev->list);
4437     write_unlock_irqrestore(&cm.device_lock, flags);
4438
4439     spin_lock_irq(&cm.lock);
4440     cm_dev->going_down = 1;
4441     spin_unlock_irq(&cm.lock);
4442
4443     rdma_for_each_port (ib_device, i) {
4444         struct ib_mad_agent *mad_agent;
4445
4446         if (!rdma_cap_ib_cm(ib_device, i))
4447             continue;
4448
4449         port = cm_dev->port[i-1];
4450         mad_agent = port->mad_agent;
4451         ib_modify_port(ib_device, port->port_num, 0, &port_modify);
4452         /*
4453          * We flush the queue here after the going_down set, this
4454          * verify that no new works will be queued in the recv handler,
4455          * after that we can call the unregister_mad_agent
4456          */
4457         flush_workqueue(cm.wq);
4458         /*
4459          * The above ensures no call paths from the work are running,
4460          * the remaining paths all take the mad_agent_lock.
4461          */
4462         spin_lock(&cm_dev->mad_agent_lock);
4463         port->mad_agent = NULL;
4464         spin_unlock(&cm_dev->mad_agent_lock);
4465         ib_unregister_mad_agent(mad_agent);
4466         ib_port_unregister_client_groups(ib_device, i,
4467                          cm_counter_groups);
4468     }
4469
4470     cm_device_put(cm_dev);
4471 }
4472
4473 static int __init ib_cm_init(void)
4474 {
4475     int ret;
4476
4477     INIT_LIST_HEAD(&cm.device_list);
4478     rwlock_init(&cm.device_lock);
4479     spin_lock_init(&cm.lock);
4480     cm.listen_service_table = RB_ROOT;
4481     cm.listen_service_id = be64_to_cpu(IB_CM_ASSIGN_SERVICE_ID);
4482     cm.remote_id_table = RB_ROOT;
4483     cm.remote_qp_table = RB_ROOT;
4484     cm.remote_sidr_table = RB_ROOT;
4485     xa_init_flags(&cm.local_id_table, XA_FLAGS_ALLOC);
4486     get_random_bytes(&cm.random_id_operand, sizeof cm.random_id_operand);
4487     INIT_LIST_HEAD(&cm.timewait_list);
4488
4489     cm.wq = alloc_workqueue("ib_cm", 0, 1);
4490     if (!cm.wq) {
4491         ret = -ENOMEM;
4492         goto error2;
4493     }
4494
4495     ret = ib_register_client(&cm_client);
4496     if (ret)
4497         goto error3;
4498
4499     return 0;
4500 error3:
4501     destroy_workqueue(cm.wq);
4502 error2:
4503     return ret;
4504 }
4505
4506 static void __exit ib_cm_cleanup(void)
4507 {
4508     struct cm_timewait_info *timewait_info, *tmp;
4509
4510     spin_lock_irq(&cm.lock);
4511     list_for_each_entry(timewait_info, &cm.timewait_list, list)
4512         cancel_delayed_work(&timewait_info->work.work);
4513     spin_unlock_irq(&cm.lock);
4514
4515     ib_unregister_client(&cm_client);
4516     destroy_workqueue(cm.wq);
4517
4518     list_for_each_entry_safe(timewait_info, tmp, &cm.timewait_list, list) {
4519         list_del(&timewait_info->list);
4520         kfree(timewait_info);
4521     }
4522
4523     WARN_ON(!xa_empty(&cm.local_id_table));
4524 }
4525
4526 module_init(ib_cm_init);
4527 module_exit(ib_cm_cleanup);