hw/irdma/cm.c

0001 // SPDX-License-Identifier: GPL-2.0 or Linux-OpenIB
0002 /* Copyright (c) 2015 - 2021 Intel Corporation */
0003 #include "main.h"
0004 #include "trace.h"
0005
0006 static void irdma_cm_post_event(struct irdma_cm_event *event);
0007 static void irdma_disconnect_worker(struct work_struct *work);
0008
0009 /**
0010  * irdma_free_sqbuf - put back puda buffer if refcount is 0
0011  * @vsi: The VSI structure of the device
0012  * @bufp: puda buffer to free
0013  */
0014 void irdma_free_sqbuf(struct irdma_sc_vsi *vsi, void *bufp)
0015 {
0016     struct irdma_puda_buf *buf = bufp;
0017     struct irdma_puda_rsrc *ilq = vsi->ilq;
0018
0019     if (refcount_dec_and_test(&buf->refcount))
0020         irdma_puda_ret_bufpool(ilq, buf);
0021 }
0022
0023 /**
0024  * irdma_record_ird_ord - Record IRD/ORD passed in
0025  * @cm_node: connection's node
0026  * @conn_ird: connection IRD
0027  * @conn_ord: connection ORD
0028  */
0029 static void irdma_record_ird_ord(struct irdma_cm_node *cm_node, u32 conn_ird,
0030                  u32 conn_ord)
0031 {
0032     if (conn_ird > cm_node->dev->hw_attrs.max_hw_ird)
0033         conn_ird = cm_node->dev->hw_attrs.max_hw_ird;
0034
0035     if (conn_ord > cm_node->dev->hw_attrs.max_hw_ord)
0036         conn_ord = cm_node->dev->hw_attrs.max_hw_ord;
0037     else if (!conn_ord && cm_node->send_rdma0_op == SEND_RDMA_READ_ZERO)
0038         conn_ord = 1;
0039     cm_node->ird_size = conn_ird;
0040     cm_node->ord_size = conn_ord;
0041 }
0042
0043 /**
0044  * irdma_copy_ip_ntohl - copy IP address from  network to host
0045  * @dst: IP address in host order
0046  * @src: IP address in network order (big endian)
0047  */
0048 void irdma_copy_ip_ntohl(u32 *dst, __be32 *src)
0049 {
0050     *dst++ = ntohl(*src++);
0051     *dst++ = ntohl(*src++);
0052     *dst++ = ntohl(*src++);
0053     *dst = ntohl(*src);
0054 }
0055
0056 /**
0057  * irdma_copy_ip_htonl - copy IP address from host to network order
0058  * @dst: IP address in network order (big endian)
0059  * @src: IP address in host order
0060  */
0061 void irdma_copy_ip_htonl(__be32 *dst, u32 *src)
0062 {
0063     *dst++ = htonl(*src++);
0064     *dst++ = htonl(*src++);
0065     *dst++ = htonl(*src++);
0066     *dst = htonl(*src);
0067 }
0068
0069 /**
0070  * irdma_get_addr_info
0071  * @cm_node: contains ip/tcp info
0072  * @cm_info: to get a copy of the cm_node ip/tcp info
0073  */
0074 static void irdma_get_addr_info(struct irdma_cm_node *cm_node,
0075                 struct irdma_cm_info *cm_info)
0076 {
0077     memset(cm_info, 0, sizeof(*cm_info));
0078     cm_info->ipv4 = cm_node->ipv4;
0079     cm_info->vlan_id = cm_node->vlan_id;
0080     memcpy(cm_info->loc_addr, cm_node->loc_addr, sizeof(cm_info->loc_addr));
0081     memcpy(cm_info->rem_addr, cm_node->rem_addr, sizeof(cm_info->rem_addr));
0082     cm_info->loc_port = cm_node->loc_port;
0083     cm_info->rem_port = cm_node->rem_port;
0084 }
0085
0086 /**
0087  * irdma_fill_sockaddr4 - fill in addr info for IPv4 connection
0088  * @cm_node: connection's node
0089  * @event: upper layer's cm event
0090  */
0091 static inline void irdma_fill_sockaddr4(struct irdma_cm_node *cm_node,
0092                     struct iw_cm_event *event)
0093 {
0094     struct sockaddr_in *laddr = (struct sockaddr_in *)&event->local_addr;
0095     struct sockaddr_in *raddr = (struct sockaddr_in *)&event->remote_addr;
0096
0097     laddr->sin_family = AF_INET;
0098     raddr->sin_family = AF_INET;
0099
0100     laddr->sin_port = htons(cm_node->loc_port);
0101     raddr->sin_port = htons(cm_node->rem_port);
0102
0103     laddr->sin_addr.s_addr = htonl(cm_node->loc_addr[0]);
0104     raddr->sin_addr.s_addr = htonl(cm_node->rem_addr[0]);
0105 }
0106
0107 /**
0108  * irdma_fill_sockaddr6 - fill in addr info for IPv6 connection
0109  * @cm_node: connection's node
0110  * @event: upper layer's cm event
0111  */
0112 static inline void irdma_fill_sockaddr6(struct irdma_cm_node *cm_node,
0113                     struct iw_cm_event *event)
0114 {
0115     struct sockaddr_in6 *laddr6 = (struct sockaddr_in6 *)&event->local_addr;
0116     struct sockaddr_in6 *raddr6 = (struct sockaddr_in6 *)&event->remote_addr;
0117
0118     laddr6->sin6_family = AF_INET6;
0119     raddr6->sin6_family = AF_INET6;
0120
0121     laddr6->sin6_port = htons(cm_node->loc_port);
0122     raddr6->sin6_port = htons(cm_node->rem_port);
0123
0124     irdma_copy_ip_htonl(laddr6->sin6_addr.in6_u.u6_addr32,
0125                 cm_node->loc_addr);
0126     irdma_copy_ip_htonl(raddr6->sin6_addr.in6_u.u6_addr32,
0127                 cm_node->rem_addr);
0128 }
0129
0130 /**
0131  * irdma_get_cmevent_info - for cm event upcall
0132  * @cm_node: connection's node
0133  * @cm_id: upper layers cm struct for the event
0134  * @event: upper layer's cm event
0135  */
0136 static inline void irdma_get_cmevent_info(struct irdma_cm_node *cm_node,
0137                       struct iw_cm_id *cm_id,
0138                       struct iw_cm_event *event)
0139 {
0140     memcpy(&event->local_addr, &cm_id->m_local_addr,
0141            sizeof(event->local_addr));
0142     memcpy(&event->remote_addr, &cm_id->m_remote_addr,
0143            sizeof(event->remote_addr));
0144     if (cm_node) {
0145         event->private_data = cm_node->pdata_buf;
0146         event->private_data_len = (u8)cm_node->pdata.size;
0147         event->ird = cm_node->ird_size;
0148         event->ord = cm_node->ord_size;
0149     }
0150 }
0151
0152 /**
0153  * irdma_send_cm_event - upcall cm's event handler
0154  * @cm_node: connection's node
0155  * @cm_id: upper layer's cm info struct
0156  * @type: Event type to indicate
0157  * @status: status for the event type
0158  */
0159 static int irdma_send_cm_event(struct irdma_cm_node *cm_node,
0160                    struct iw_cm_id *cm_id,
0161                    enum iw_cm_event_type type, int status)
0162 {
0163     struct iw_cm_event event = {};
0164
0165     event.event = type;
0166     event.status = status;
0167     trace_irdma_send_cm_event(cm_node, cm_id, type, status,
0168                   __builtin_return_address(0));
0169
0170     ibdev_dbg(&cm_node->iwdev->ibdev,
0171           "CM: cm_node %p cm_id=%p state=%d accel=%d event_type=%d status=%d\n",
0172           cm_node, cm_id, cm_node->accelerated, cm_node->state, type,
0173           status);
0174
0175     switch (type) {
0176     case IW_CM_EVENT_CONNECT_REQUEST:
0177         if (cm_node->ipv4)
0178             irdma_fill_sockaddr4(cm_node, &event);
0179         else
0180             irdma_fill_sockaddr6(cm_node, &event);
0181         event.provider_data = cm_node;
0182         event.private_data = cm_node->pdata_buf;
0183         event.private_data_len = (u8)cm_node->pdata.size;
0184         event.ird = cm_node->ird_size;
0185         break;
0186     case IW_CM_EVENT_CONNECT_REPLY:
0187         irdma_get_cmevent_info(cm_node, cm_id, &event);
0188         break;
0189     case IW_CM_EVENT_ESTABLISHED:
0190         event.ird = cm_node->ird_size;
0191         event.ord = cm_node->ord_size;
0192         break;
0193     case IW_CM_EVENT_DISCONNECT:
0194     case IW_CM_EVENT_CLOSE:
0195         /* Wait if we are in RTS but havent issued the iwcm event upcall */
0196         if (!cm_node->accelerated)
0197             wait_for_completion(&cm_node->establish_comp);
0198         break;
0199     default:
0200         return -EINVAL;
0201     }
0202
0203     return cm_id->event_handler(cm_id, &event);
0204 }
0205
0206 /**
0207  * irdma_timer_list_prep - add connection nodes to a list to perform timer tasks
0208  * @cm_core: cm's core
0209  * @timer_list: a timer list to which cm_node will be selected
0210  */
0211 static void irdma_timer_list_prep(struct irdma_cm_core *cm_core,
0212                   struct list_head *timer_list)
0213 {
0214     struct irdma_cm_node *cm_node;
0215     int bkt;
0216
0217     hash_for_each_rcu(cm_core->cm_hash_tbl, bkt, cm_node, list) {
0218         if ((cm_node->close_entry || cm_node->send_entry) &&
0219             refcount_inc_not_zero(&cm_node->refcnt))
0220             list_add(&cm_node->timer_entry, timer_list);
0221     }
0222 }
0223
0224 /**
0225  * irdma_create_event - create cm event
0226  * @cm_node: connection's node
0227  * @type: Event type to generate
0228  */
0229 static struct irdma_cm_event *irdma_create_event(struct irdma_cm_node *cm_node,
0230                          enum irdma_cm_event_type type)
0231 {
0232     struct irdma_cm_event *event;
0233
0234     if (!cm_node->cm_id)
0235         return NULL;
0236
0237     event = kzalloc(sizeof(*event), GFP_ATOMIC);
0238
0239     if (!event)
0240         return NULL;
0241
0242     event->type = type;
0243     event->cm_node = cm_node;
0244     memcpy(event->cm_info.rem_addr, cm_node->rem_addr,
0245            sizeof(event->cm_info.rem_addr));
0246     memcpy(event->cm_info.loc_addr, cm_node->loc_addr,
0247            sizeof(event->cm_info.loc_addr));
0248     event->cm_info.rem_port = cm_node->rem_port;
0249     event->cm_info.loc_port = cm_node->loc_port;
0250     event->cm_info.cm_id = cm_node->cm_id;
0251     ibdev_dbg(&cm_node->iwdev->ibdev,
0252           "CM: node=%p event=%p type=%u dst=%pI4 src=%pI4\n", cm_node,
0253           event, type, event->cm_info.loc_addr,
0254           event->cm_info.rem_addr);
0255     trace_irdma_create_event(cm_node, type, __builtin_return_address(0));
0256     irdma_cm_post_event(event);
0257
0258     return event;
0259 }
0260
0261 /**
0262  * irdma_free_retrans_entry - free send entry
0263  * @cm_node: connection's node
0264  */
0265 static void irdma_free_retrans_entry(struct irdma_cm_node *cm_node)
0266 {
0267     struct irdma_device *iwdev = cm_node->iwdev;
0268     struct irdma_timer_entry *send_entry;
0269
0270     send_entry = cm_node->send_entry;
0271     if (!send_entry)
0272         return;
0273
0274     cm_node->send_entry = NULL;
0275     irdma_free_sqbuf(&iwdev->vsi, send_entry->sqbuf);
0276     kfree(send_entry);
0277     refcount_dec(&cm_node->refcnt);
0278 }
0279
0280 /**
0281  * irdma_cleanup_retrans_entry - free send entry with lock
0282  * @cm_node: connection's node
0283  */
0284 static void irdma_cleanup_retrans_entry(struct irdma_cm_node *cm_node)
0285 {
0286     unsigned long flags;
0287
0288     spin_lock_irqsave(&cm_node->retrans_list_lock, flags);
0289     irdma_free_retrans_entry(cm_node);
0290     spin_unlock_irqrestore(&cm_node->retrans_list_lock, flags);
0291 }
0292
0293 /**
0294  * irdma_form_ah_cm_frame - get a free packet and build frame with address handle
0295  * @cm_node: connection's node ionfo to use in frame
0296  * @options: pointer to options info
0297  * @hdr: pointer mpa header
0298  * @pdata: pointer to private data
0299  * @flags:  indicates FIN or ACK
0300  */
0301 static struct irdma_puda_buf *irdma_form_ah_cm_frame(struct irdma_cm_node *cm_node,
0302                              struct irdma_kmem_info *options,
0303                              struct irdma_kmem_info *hdr,
0304                              struct irdma_mpa_priv_info *pdata,
0305                              u8 flags)
0306 {
0307     struct irdma_puda_buf *sqbuf;
0308     struct irdma_sc_vsi *vsi = &cm_node->iwdev->vsi;
0309     u8 *buf;
0310     struct tcphdr *tcph;
0311     u16 pktsize;
0312     u32 opts_len = 0;
0313     u32 pd_len = 0;
0314     u32 hdr_len = 0;
0315
0316     if (!cm_node->ah || !cm_node->ah->ah_info.ah_valid) {
0317         ibdev_dbg(&cm_node->iwdev->ibdev, "CM: AH invalid\n");
0318         return NULL;
0319     }
0320
0321     sqbuf = irdma_puda_get_bufpool(vsi->ilq);
0322     if (!sqbuf) {
0323         ibdev_dbg(&cm_node->iwdev->ibdev, "CM: SQ buf NULL\n");
0324         return NULL;
0325     }
0326
0327     sqbuf->ah_id = cm_node->ah->ah_info.ah_idx;
0328     buf = sqbuf->mem.va;
0329     if (options)
0330         opts_len = (u32)options->size;
0331
0332     if (hdr)
0333         hdr_len = hdr->size;
0334
0335     if (pdata)
0336         pd_len = pdata->size;
0337
0338     pktsize = sizeof(*tcph) + opts_len + hdr_len + pd_len;
0339
0340     memset(buf, 0, pktsize);
0341
0342     sqbuf->totallen = pktsize;
0343     sqbuf->tcphlen = sizeof(*tcph) + opts_len;
0344     sqbuf->scratch = cm_node;
0345
0346     tcph = (struct tcphdr *)buf;
0347     buf += sizeof(*tcph);
0348
0349     tcph->source = htons(cm_node->loc_port);
0350     tcph->dest = htons(cm_node->rem_port);
0351     tcph->seq = htonl(cm_node->tcp_cntxt.loc_seq_num);
0352
0353     if (flags & SET_ACK) {
0354         cm_node->tcp_cntxt.loc_ack_num = cm_node->tcp_cntxt.rcv_nxt;
0355         tcph->ack_seq = htonl(cm_node->tcp_cntxt.loc_ack_num);
0356         tcph->ack = 1;
0357     } else {
0358         tcph->ack_seq = 0;
0359     }
0360
0361     if (flags & SET_SYN) {
0362         cm_node->tcp_cntxt.loc_seq_num++;
0363         tcph->syn = 1;
0364     } else {
0365         cm_node->tcp_cntxt.loc_seq_num += hdr_len + pd_len;
0366     }
0367
0368     if (flags & SET_FIN) {
0369         cm_node->tcp_cntxt.loc_seq_num++;
0370         tcph->fin = 1;
0371     }
0372
0373     if (flags & SET_RST)
0374         tcph->rst = 1;
0375
0376     tcph->doff = (u16)((sizeof(*tcph) + opts_len + 3) >> 2);
0377     sqbuf->tcphlen = tcph->doff << 2;
0378     tcph->window = htons(cm_node->tcp_cntxt.rcv_wnd);
0379     tcph->urg_ptr = 0;
0380
0381     if (opts_len) {
0382         memcpy(buf, options->addr, opts_len);
0383         buf += opts_len;
0384     }
0385
0386     if (hdr_len) {
0387         memcpy(buf, hdr->addr, hdr_len);
0388         buf += hdr_len;
0389     }
0390
0391     if (pdata && pdata->addr)
0392         memcpy(buf, pdata->addr, pdata->size);
0393
0394     refcount_set(&sqbuf->refcount, 1);
0395
0396     print_hex_dump_debug("ILQ: TRANSMIT ILQ BUFFER", DUMP_PREFIX_OFFSET,
0397                  16, 8, sqbuf->mem.va, sqbuf->totallen, false);
0398
0399     return sqbuf;
0400 }
0401
0402 /**
0403  * irdma_form_uda_cm_frame - get a free packet and build frame full tcpip packet
0404  * @cm_node: connection's node ionfo to use in frame
0405  * @options: pointer to options info
0406  * @hdr: pointer mpa header
0407  * @pdata: pointer to private data
0408  * @flags:  indicates FIN or ACK
0409  */
0410 static struct irdma_puda_buf *irdma_form_uda_cm_frame(struct irdma_cm_node *cm_node,
0411                               struct irdma_kmem_info *options,
0412                               struct irdma_kmem_info *hdr,
0413                               struct irdma_mpa_priv_info *pdata,
0414                               u8 flags)
0415 {
0416     struct irdma_puda_buf *sqbuf;
0417     struct irdma_sc_vsi *vsi = &cm_node->iwdev->vsi;
0418     u8 *buf;
0419
0420     struct tcphdr *tcph;
0421     struct iphdr *iph;
0422     struct ipv6hdr *ip6h;
0423     struct ethhdr *ethh;
0424     u16 pktsize;
0425     u16 eth_hlen = ETH_HLEN;
0426     u32 opts_len = 0;
0427     u32 pd_len = 0;
0428     u32 hdr_len = 0;
0429
0430     u16 vtag;
0431
0432     sqbuf = irdma_puda_get_bufpool(vsi->ilq);
0433     if (!sqbuf)
0434         return NULL;
0435
0436     buf = sqbuf->mem.va;
0437
0438     if (options)
0439         opts_len = (u32)options->size;
0440
0441     if (hdr)
0442         hdr_len = hdr->size;
0443
0444     if (pdata)
0445         pd_len = pdata->size;
0446
0447     if (cm_node->vlan_id < VLAN_N_VID)
0448         eth_hlen += 4;
0449
0450     if (cm_node->ipv4)
0451         pktsize = sizeof(*iph) + sizeof(*tcph);
0452     else
0453         pktsize = sizeof(*ip6h) + sizeof(*tcph);
0454     pktsize += opts_len + hdr_len + pd_len;
0455
0456     memset(buf, 0, eth_hlen + pktsize);
0457
0458     sqbuf->totallen = pktsize + eth_hlen;
0459     sqbuf->maclen = eth_hlen;
0460     sqbuf->tcphlen = sizeof(*tcph) + opts_len;
0461     sqbuf->scratch = cm_node;
0462
0463     ethh = (struct ethhdr *)buf;
0464     buf += eth_hlen;
0465
0466     if (cm_node->do_lpb)
0467         sqbuf->do_lpb = true;
0468
0469     if (cm_node->ipv4) {
0470         sqbuf->ipv4 = true;
0471
0472         iph = (struct iphdr *)buf;
0473         buf += sizeof(*iph);
0474         tcph = (struct tcphdr *)buf;
0475         buf += sizeof(*tcph);
0476
0477         ether_addr_copy(ethh->h_dest, cm_node->rem_mac);
0478         ether_addr_copy(ethh->h_source, cm_node->loc_mac);
0479         if (cm_node->vlan_id < VLAN_N_VID) {
0480             ((struct vlan_ethhdr *)ethh)->h_vlan_proto =
0481                 htons(ETH_P_8021Q);
0482             vtag = (cm_node->user_pri << VLAN_PRIO_SHIFT) |
0483                    cm_node->vlan_id;
0484             ((struct vlan_ethhdr *)ethh)->h_vlan_TCI = htons(vtag);
0485
0486             ((struct vlan_ethhdr *)ethh)->h_vlan_encapsulated_proto =
0487                 htons(ETH_P_IP);
0488         } else {
0489             ethh->h_proto = htons(ETH_P_IP);
0490         }
0491
0492         iph->version = IPVERSION;
0493         iph->ihl = 5; /* 5 * 4Byte words, IP headr len */
0494         iph->tos = cm_node->tos;
0495         iph->tot_len = htons(pktsize);
0496         iph->id = htons(++cm_node->tcp_cntxt.loc_id);
0497
0498         iph->frag_off = htons(0x4000);
0499         iph->ttl = 0x40;
0500         iph->protocol = IPPROTO_TCP;
0501         iph->saddr = htonl(cm_node->loc_addr[0]);
0502         iph->daddr = htonl(cm_node->rem_addr[0]);
0503     } else {
0504         sqbuf->ipv4 = false;
0505         ip6h = (struct ipv6hdr *)buf;
0506         buf += sizeof(*ip6h);
0507         tcph = (struct tcphdr *)buf;
0508         buf += sizeof(*tcph);
0509
0510         ether_addr_copy(ethh->h_dest, cm_node->rem_mac);
0511         ether_addr_copy(ethh->h_source, cm_node->loc_mac);
0512         if (cm_node->vlan_id < VLAN_N_VID) {
0513             ((struct vlan_ethhdr *)ethh)->h_vlan_proto =
0514                 htons(ETH_P_8021Q);
0515             vtag = (cm_node->user_pri << VLAN_PRIO_SHIFT) |
0516                    cm_node->vlan_id;
0517             ((struct vlan_ethhdr *)ethh)->h_vlan_TCI = htons(vtag);
0518             ((struct vlan_ethhdr *)ethh)->h_vlan_encapsulated_proto =
0519                 htons(ETH_P_IPV6);
0520         } else {
0521             ethh->h_proto = htons(ETH_P_IPV6);
0522         }
0523         ip6h->version = 6;
0524         ip6h->priority = cm_node->tos >> 4;
0525         ip6h->flow_lbl[0] = cm_node->tos << 4;
0526         ip6h->flow_lbl[1] = 0;
0527         ip6h->flow_lbl[2] = 0;
0528         ip6h->payload_len = htons(pktsize - sizeof(*ip6h));
0529         ip6h->nexthdr = 6;
0530         ip6h->hop_limit = 128;
0531         irdma_copy_ip_htonl(ip6h->saddr.in6_u.u6_addr32,
0532                     cm_node->loc_addr);
0533         irdma_copy_ip_htonl(ip6h->daddr.in6_u.u6_addr32,
0534                     cm_node->rem_addr);
0535     }
0536
0537     tcph->source = htons(cm_node->loc_port);
0538     tcph->dest = htons(cm_node->rem_port);
0539     tcph->seq = htonl(cm_node->tcp_cntxt.loc_seq_num);
0540
0541     if (flags & SET_ACK) {
0542         cm_node->tcp_cntxt.loc_ack_num = cm_node->tcp_cntxt.rcv_nxt;
0543         tcph->ack_seq = htonl(cm_node->tcp_cntxt.loc_ack_num);
0544         tcph->ack = 1;
0545     } else {
0546         tcph->ack_seq = 0;
0547     }
0548
0549     if (flags & SET_SYN) {
0550         cm_node->tcp_cntxt.loc_seq_num++;
0551         tcph->syn = 1;
0552     } else {
0553         cm_node->tcp_cntxt.loc_seq_num += hdr_len + pd_len;
0554     }
0555
0556     if (flags & SET_FIN) {
0557         cm_node->tcp_cntxt.loc_seq_num++;
0558         tcph->fin = 1;
0559     }
0560
0561     if (flags & SET_RST)
0562         tcph->rst = 1;
0563
0564     tcph->doff = (u16)((sizeof(*tcph) + opts_len + 3) >> 2);
0565     sqbuf->tcphlen = tcph->doff << 2;
0566     tcph->window = htons(cm_node->tcp_cntxt.rcv_wnd);
0567     tcph->urg_ptr = 0;
0568
0569     if (opts_len) {
0570         memcpy(buf, options->addr, opts_len);
0571         buf += opts_len;
0572     }
0573
0574     if (hdr_len) {
0575         memcpy(buf, hdr->addr, hdr_len);
0576         buf += hdr_len;
0577     }
0578
0579     if (pdata && pdata->addr)
0580         memcpy(buf, pdata->addr, pdata->size);
0581
0582     refcount_set(&sqbuf->refcount, 1);
0583
0584     print_hex_dump_debug("ILQ: TRANSMIT ILQ BUFFER", DUMP_PREFIX_OFFSET,
0585                  16, 8, sqbuf->mem.va, sqbuf->totallen, false);
0586     return sqbuf;
0587 }
0588
0589 /**
0590  * irdma_send_reset - Send RST packet
0591  * @cm_node: connection's node
0592  */
0593 int irdma_send_reset(struct irdma_cm_node *cm_node)
0594 {
0595     struct irdma_puda_buf *sqbuf;
0596     int flags = SET_RST | SET_ACK;
0597
0598     trace_irdma_send_reset(cm_node, 0, __builtin_return_address(0));
0599     sqbuf = cm_node->cm_core->form_cm_frame(cm_node, NULL, NULL, NULL,
0600                         flags);
0601     if (!sqbuf)
0602         return -ENOMEM;
0603
0604     ibdev_dbg(&cm_node->iwdev->ibdev,
0605           "CM: caller: %pS cm_node %p cm_id=%p accel=%d state=%d rem_port=0x%04x, loc_port=0x%04x rem_addr=%pI4 loc_addr=%pI4\n",
0606           __builtin_return_address(0), cm_node, cm_node->cm_id,
0607           cm_node->accelerated, cm_node->state, cm_node->rem_port,
0608           cm_node->loc_port, cm_node->rem_addr, cm_node->loc_addr);
0609
0610     return irdma_schedule_cm_timer(cm_node, sqbuf, IRDMA_TIMER_TYPE_SEND, 0,
0611                        1);
0612 }
0613
0614 /**
0615  * irdma_active_open_err - send event for active side cm error
0616  * @cm_node: connection's node
0617  * @reset: Flag to send reset or not
0618  */
0619 static void irdma_active_open_err(struct irdma_cm_node *cm_node, bool reset)
0620 {
0621     trace_irdma_active_open_err(cm_node, reset,
0622                     __builtin_return_address(0));
0623     irdma_cleanup_retrans_entry(cm_node);
0624     cm_node->cm_core->stats_connect_errs++;
0625     if (reset) {
0626         ibdev_dbg(&cm_node->iwdev->ibdev,
0627               "CM: cm_node=%p state=%d\n", cm_node,
0628               cm_node->state);
0629         refcount_inc(&cm_node->refcnt);
0630         irdma_send_reset(cm_node);
0631     }
0632
0633     cm_node->state = IRDMA_CM_STATE_CLOSED;
0634     irdma_create_event(cm_node, IRDMA_CM_EVENT_ABORTED);
0635 }
0636
0637 /**
0638  * irdma_passive_open_err - handle passive side cm error
0639  * @cm_node: connection's node
0640  * @reset: send reset or just free cm_node
0641  */
0642 static void irdma_passive_open_err(struct irdma_cm_node *cm_node, bool reset)
0643 {
0644     irdma_cleanup_retrans_entry(cm_node);
0645     cm_node->cm_core->stats_passive_errs++;
0646     cm_node->state = IRDMA_CM_STATE_CLOSED;
0647     ibdev_dbg(&cm_node->iwdev->ibdev, "CM: cm_node=%p state =%d\n",
0648           cm_node, cm_node->state);
0649     trace_irdma_passive_open_err(cm_node, reset,
0650                      __builtin_return_address(0));
0651     if (reset)
0652         irdma_send_reset(cm_node);
0653     else
0654         irdma_rem_ref_cm_node(cm_node);
0655 }
0656
0657 /**
0658  * irdma_event_connect_error - to create connect error event
0659  * @event: cm information for connect event
0660  */
0661 static void irdma_event_connect_error(struct irdma_cm_event *event)
0662 {
0663     struct irdma_qp *iwqp;
0664     struct iw_cm_id *cm_id;
0665
0666     cm_id = event->cm_node->cm_id;
0667     if (!cm_id)
0668         return;
0669
0670     iwqp = cm_id->provider_data;
0671
0672     if (!iwqp || !iwqp->iwdev)
0673         return;
0674
0675     iwqp->cm_id = NULL;
0676     cm_id->provider_data = NULL;
0677     irdma_send_cm_event(event->cm_node, cm_id, IW_CM_EVENT_CONNECT_REPLY,
0678                 -ECONNRESET);
0679     irdma_rem_ref_cm_node(event->cm_node);
0680 }
0681
0682 /**
0683  * irdma_process_options - process options from TCP header
0684  * @cm_node: connection's node
0685  * @optionsloc: point to start of options
0686  * @optionsize: size of all options
0687  * @syn_pkt: flag if syn packet
0688  */
0689 static int irdma_process_options(struct irdma_cm_node *cm_node, u8 *optionsloc,
0690                  u32 optionsize, u32 syn_pkt)
0691 {
0692     u32 tmp;
0693     u32 offset = 0;
0694     union all_known_options *all_options;
0695     char got_mss_option = 0;
0696
0697     while (offset < optionsize) {
0698         all_options = (union all_known_options *)(optionsloc + offset);
0699         switch (all_options->base.optionnum) {
0700         case OPTION_NUM_EOL:
0701             offset = optionsize;
0702             break;
0703         case OPTION_NUM_NONE:
0704             offset += 1;
0705             continue;
0706         case OPTION_NUM_MSS:
0707             ibdev_dbg(&cm_node->iwdev->ibdev,
0708                   "CM: MSS Length: %d Offset: %d Size: %d\n",
0709                   all_options->mss.len, offset, optionsize);
0710             got_mss_option = 1;
0711             if (all_options->mss.len != 4)
0712                 return -EINVAL;
0713             tmp = ntohs(all_options->mss.mss);
0714             if ((cm_node->ipv4 &&
0715                  (tmp + IRDMA_MTU_TO_MSS_IPV4) < IRDMA_MIN_MTU_IPV4) ||
0716                 (!cm_node->ipv4 &&
0717                  (tmp + IRDMA_MTU_TO_MSS_IPV6) < IRDMA_MIN_MTU_IPV6))
0718                 return -EINVAL;
0719             if (tmp < cm_node->tcp_cntxt.mss)
0720                 cm_node->tcp_cntxt.mss = tmp;
0721             break;
0722         case OPTION_NUM_WINDOW_SCALE:
0723             cm_node->tcp_cntxt.snd_wscale =
0724                 all_options->windowscale.shiftcount;
0725             break;
0726         default:
0727             ibdev_dbg(&cm_node->iwdev->ibdev,
0728                   "CM: Unsupported TCP Option: %x\n",
0729                   all_options->base.optionnum);
0730             break;
0731         }
0732         offset += all_options->base.len;
0733     }
0734     if (!got_mss_option && syn_pkt)
0735         cm_node->tcp_cntxt.mss = IRDMA_CM_DEFAULT_MSS;
0736
0737     return 0;
0738 }
0739
0740 /**
0741  * irdma_handle_tcp_options - setup TCP context info after parsing TCP options
0742  * @cm_node: connection's node
0743  * @tcph: pointer tcp header
0744  * @optionsize: size of options rcvd
0745  * @passive: active or passive flag
0746  */
0747 static int irdma_handle_tcp_options(struct irdma_cm_node *cm_node,
0748                     struct tcphdr *tcph, int optionsize,
0749                     int passive)
0750 {
0751     u8 *optionsloc = (u8 *)&tcph[1];
0752     int ret;
0753
0754     if (optionsize) {
0755         ret = irdma_process_options(cm_node, optionsloc, optionsize,
0756                         (u32)tcph->syn);
0757         if (ret) {
0758             ibdev_dbg(&cm_node->iwdev->ibdev,
0759                   "CM: Node %p, Sending Reset\n", cm_node);
0760             if (passive)
0761                 irdma_passive_open_err(cm_node, true);
0762             else
0763                 irdma_active_open_err(cm_node, true);
0764             return ret;
0765         }
0766     }
0767
0768     cm_node->tcp_cntxt.snd_wnd = ntohs(tcph->window)
0769                      << cm_node->tcp_cntxt.snd_wscale;
0770
0771     if (cm_node->tcp_cntxt.snd_wnd > cm_node->tcp_cntxt.max_snd_wnd)
0772         cm_node->tcp_cntxt.max_snd_wnd = cm_node->tcp_cntxt.snd_wnd;
0773
0774     return 0;
0775 }
0776
0777 /**
0778  * irdma_build_mpa_v1 - build a MPA V1 frame
0779  * @cm_node: connection's node
0780  * @start_addr: address where to build frame
0781  * @mpa_key: to do read0 or write0
0782  */
0783 static void irdma_build_mpa_v1(struct irdma_cm_node *cm_node, void *start_addr,
0784                    u8 mpa_key)
0785 {
0786     struct ietf_mpa_v1 *mpa_frame = start_addr;
0787
0788     switch (mpa_key) {
0789     case MPA_KEY_REQUEST:
0790         memcpy(mpa_frame->key, IEFT_MPA_KEY_REQ, IETF_MPA_KEY_SIZE);
0791         break;
0792     case MPA_KEY_REPLY:
0793         memcpy(mpa_frame->key, IEFT_MPA_KEY_REP, IETF_MPA_KEY_SIZE);
0794         break;
0795     default:
0796         break;
0797     }
0798     mpa_frame->flags = IETF_MPA_FLAGS_CRC;
0799     mpa_frame->rev = cm_node->mpa_frame_rev;
0800     mpa_frame->priv_data_len = htons(cm_node->pdata.size);
0801 }
0802
0803 /**
0804  * irdma_build_mpa_v2 - build a MPA V2 frame
0805  * @cm_node: connection's node
0806  * @start_addr: buffer start address
0807  * @mpa_key: to do read0 or write0
0808  */
0809 static void irdma_build_mpa_v2(struct irdma_cm_node *cm_node, void *start_addr,
0810                    u8 mpa_key)
0811 {
0812     struct ietf_mpa_v2 *mpa_frame = start_addr;
0813     struct ietf_rtr_msg *rtr_msg = &mpa_frame->rtr_msg;
0814     u16 ctrl_ird, ctrl_ord;
0815
0816     /* initialize the upper 5 bytes of the frame */
0817     irdma_build_mpa_v1(cm_node, start_addr, mpa_key);
0818     mpa_frame->flags |= IETF_MPA_V2_FLAG;
0819     if (cm_node->iwdev->iw_ooo) {
0820         mpa_frame->flags |= IETF_MPA_FLAGS_MARKERS;
0821         cm_node->rcv_mark_en = true;
0822     }
0823     mpa_frame->priv_data_len = cpu_to_be16(be16_to_cpu(mpa_frame->priv_data_len) +
0824                            IETF_RTR_MSG_SIZE);
0825
0826     /* initialize RTR msg */
0827     if (cm_node->mpav2_ird_ord == IETF_NO_IRD_ORD) {
0828         ctrl_ird = IETF_NO_IRD_ORD;
0829         ctrl_ord = IETF_NO_IRD_ORD;
0830     } else {
0831         ctrl_ird = (cm_node->ird_size > IETF_NO_IRD_ORD) ?
0832                    IETF_NO_IRD_ORD :
0833                    cm_node->ird_size;
0834         ctrl_ord = (cm_node->ord_size > IETF_NO_IRD_ORD) ?
0835                    IETF_NO_IRD_ORD :
0836                    cm_node->ord_size;
0837     }
0838     ctrl_ird |= IETF_PEER_TO_PEER;
0839
0840     switch (mpa_key) {
0841     case MPA_KEY_REQUEST:
0842         ctrl_ord |= IETF_RDMA0_WRITE;
0843         ctrl_ord |= IETF_RDMA0_READ;
0844         break;
0845     case MPA_KEY_REPLY:
0846         switch (cm_node->send_rdma0_op) {
0847         case SEND_RDMA_WRITE_ZERO:
0848             ctrl_ord |= IETF_RDMA0_WRITE;
0849             break;
0850         case SEND_RDMA_READ_ZERO:
0851             ctrl_ord |= IETF_RDMA0_READ;
0852             break;
0853         }
0854         break;
0855     default:
0856         break;
0857     }
0858     rtr_msg->ctrl_ird = htons(ctrl_ird);
0859     rtr_msg->ctrl_ord = htons(ctrl_ord);
0860 }
0861
0862 /**
0863  * irdma_cm_build_mpa_frame - build mpa frame for mpa version 1 or version 2
0864  * @cm_node: connection's node
0865  * @mpa: mpa: data buffer
0866  * @mpa_key: to do read0 or write0
0867  */
0868 static int irdma_cm_build_mpa_frame(struct irdma_cm_node *cm_node,
0869                     struct irdma_kmem_info *mpa, u8 mpa_key)
0870 {
0871     int hdr_len = 0;
0872
0873     switch (cm_node->mpa_frame_rev) {
0874     case IETF_MPA_V1:
0875         hdr_len = sizeof(struct ietf_mpa_v1);
0876         irdma_build_mpa_v1(cm_node, mpa->addr, mpa_key);
0877         break;
0878     case IETF_MPA_V2:
0879         hdr_len = sizeof(struct ietf_mpa_v2);
0880         irdma_build_mpa_v2(cm_node, mpa->addr, mpa_key);
0881         break;
0882     default:
0883         break;
0884     }
0885
0886     return hdr_len;
0887 }
0888
0889 /**
0890  * irdma_send_mpa_request - active node send mpa request to passive node
0891  * @cm_node: connection's node
0892  */
0893 static int irdma_send_mpa_request(struct irdma_cm_node *cm_node)
0894 {
0895     struct irdma_puda_buf *sqbuf;
0896
0897     cm_node->mpa_hdr.addr = &cm_node->mpa_v2_frame;
0898     cm_node->mpa_hdr.size = irdma_cm_build_mpa_frame(cm_node,
0899                              &cm_node->mpa_hdr,
0900                              MPA_KEY_REQUEST);
0901     if (!cm_node->mpa_hdr.size) {
0902         ibdev_dbg(&cm_node->iwdev->ibdev,
0903               "CM: mpa size = %d\n", cm_node->mpa_hdr.size);
0904         return -EINVAL;
0905     }
0906
0907     sqbuf = cm_node->cm_core->form_cm_frame(cm_node, NULL,
0908                         &cm_node->mpa_hdr,
0909                         &cm_node->pdata, SET_ACK);
0910     if (!sqbuf)
0911         return -ENOMEM;
0912
0913     return irdma_schedule_cm_timer(cm_node, sqbuf, IRDMA_TIMER_TYPE_SEND, 1,
0914                        0);
0915 }
0916
0917 /**
0918  * irdma_send_mpa_reject -
0919  * @cm_node: connection's node
0920  * @pdata: reject data for connection
0921  * @plen: length of reject data
0922  */
0923 static int irdma_send_mpa_reject(struct irdma_cm_node *cm_node,
0924                  const void *pdata, u8 plen)
0925 {
0926     struct irdma_puda_buf *sqbuf;
0927     struct irdma_mpa_priv_info priv_info;
0928
0929     cm_node->mpa_hdr.addr = &cm_node->mpa_v2_frame;
0930     cm_node->mpa_hdr.size = irdma_cm_build_mpa_frame(cm_node,
0931                              &cm_node->mpa_hdr,
0932                              MPA_KEY_REPLY);
0933
0934     cm_node->mpa_frame.flags |= IETF_MPA_FLAGS_REJECT;
0935     priv_info.addr = pdata;
0936     priv_info.size = plen;
0937
0938     sqbuf = cm_node->cm_core->form_cm_frame(cm_node, NULL,
0939                         &cm_node->mpa_hdr, &priv_info,
0940                         SET_ACK | SET_FIN);
0941     if (!sqbuf)
0942         return -ENOMEM;
0943
0944     cm_node->state = IRDMA_CM_STATE_FIN_WAIT1;
0945
0946     return irdma_schedule_cm_timer(cm_node, sqbuf, IRDMA_TIMER_TYPE_SEND, 1,
0947                        0);
0948 }
0949
0950 /**
0951  * irdma_negotiate_mpa_v2_ird_ord - negotiate MPAv2 IRD/ORD
0952  * @cm_node: connection's node
0953  * @buf: Data pointer
0954  */
0955 static int irdma_negotiate_mpa_v2_ird_ord(struct irdma_cm_node *cm_node,
0956                       u8 *buf)
0957 {
0958     struct ietf_mpa_v2 *mpa_v2_frame;
0959     struct ietf_rtr_msg *rtr_msg;
0960     u16 ird_size;
0961     u16 ord_size;
0962     u16 ctrl_ord;
0963     u16 ctrl_ird;
0964
0965     mpa_v2_frame = (struct ietf_mpa_v2 *)buf;
0966     rtr_msg = &mpa_v2_frame->rtr_msg;
0967
0968     /* parse rtr message */
0969     ctrl_ord = ntohs(rtr_msg->ctrl_ord);
0970     ctrl_ird = ntohs(rtr_msg->ctrl_ird);
0971     ird_size = ctrl_ird & IETF_NO_IRD_ORD;
0972     ord_size = ctrl_ord & IETF_NO_IRD_ORD;
0973
0974     if (!(ctrl_ird & IETF_PEER_TO_PEER))
0975         return -EOPNOTSUPP;
0976
0977     if (ird_size == IETF_NO_IRD_ORD || ord_size == IETF_NO_IRD_ORD) {
0978         cm_node->mpav2_ird_ord = IETF_NO_IRD_ORD;
0979         goto negotiate_done;
0980     }
0981
0982     if (cm_node->state != IRDMA_CM_STATE_MPAREQ_SENT) {
0983         /* responder */
0984         if (!ord_size && (ctrl_ord & IETF_RDMA0_READ))
0985             cm_node->ird_size = 1;
0986         if (cm_node->ord_size > ird_size)
0987             cm_node->ord_size = ird_size;
0988     } else {
0989         /* initiator */
0990         if (!ird_size && (ctrl_ord & IETF_RDMA0_READ))
0991             /* Remote peer doesn't support RDMA0_READ */
0992             return -EOPNOTSUPP;
0993
0994         if (cm_node->ord_size > ird_size)
0995             cm_node->ord_size = ird_size;
0996
0997         if (cm_node->ird_size < ord_size)
0998         /* no resources available */
0999             return -EINVAL;
1000     }
1001
1002 negotiate_done:
1003     if (ctrl_ord & IETF_RDMA0_READ)
1004         cm_node->send_rdma0_op = SEND_RDMA_READ_ZERO;
1005     else if (ctrl_ord & IETF_RDMA0_WRITE)
1006         cm_node->send_rdma0_op = SEND_RDMA_WRITE_ZERO;
1007     else
1008         /* Not supported RDMA0 operation */
1009         return -EOPNOTSUPP;
1010
1011     ibdev_dbg(&cm_node->iwdev->ibdev,
1012           "CM: MPAV2 Negotiated ORD: %d, IRD: %d\n",
1013           cm_node->ord_size, cm_node->ird_size);
1014     trace_irdma_negotiate_mpa_v2(cm_node);
1015     return 0;
1016 }
1017
1018 /**
1019  * irdma_parse_mpa - process an IETF MPA frame
1020  * @cm_node: connection's node
1021  * @buf: Data pointer
1022  * @type: to return accept or reject
1023  * @len: Len of mpa buffer
1024  */
1025 static int irdma_parse_mpa(struct irdma_cm_node *cm_node, u8 *buf, u32 *type,
1026                u32 len)
1027 {
1028     struct ietf_mpa_v1 *mpa_frame;
1029     int mpa_hdr_len, priv_data_len, ret;
1030
1031     *type = IRDMA_MPA_REQUEST_ACCEPT;
1032
1033     if (len < sizeof(struct ietf_mpa_v1)) {
1034         ibdev_dbg(&cm_node->iwdev->ibdev,
1035               "CM: ietf buffer small (%x)\n", len);
1036         return -EINVAL;
1037     }
1038
1039     mpa_frame = (struct ietf_mpa_v1 *)buf;
1040     mpa_hdr_len = sizeof(struct ietf_mpa_v1);
1041     priv_data_len = ntohs(mpa_frame->priv_data_len);
1042
1043     if (priv_data_len > IETF_MAX_PRIV_DATA_LEN) {
1044         ibdev_dbg(&cm_node->iwdev->ibdev,
1045               "CM: private_data too big %d\n", priv_data_len);
1046         return -EOVERFLOW;
1047     }
1048
1049     if (mpa_frame->rev != IETF_MPA_V1 && mpa_frame->rev != IETF_MPA_V2) {
1050         ibdev_dbg(&cm_node->iwdev->ibdev,
1051               "CM: unsupported mpa rev = %d\n", mpa_frame->rev);
1052         return -EINVAL;
1053     }
1054
1055     if (mpa_frame->rev > cm_node->mpa_frame_rev) {
1056         ibdev_dbg(&cm_node->iwdev->ibdev, "CM: rev %d\n",
1057               mpa_frame->rev);
1058         return -EINVAL;
1059     }
1060
1061     cm_node->mpa_frame_rev = mpa_frame->rev;
1062     if (cm_node->state != IRDMA_CM_STATE_MPAREQ_SENT) {
1063         if (memcmp(mpa_frame->key, IEFT_MPA_KEY_REQ,
1064                IETF_MPA_KEY_SIZE)) {
1065             ibdev_dbg(&cm_node->iwdev->ibdev,
1066                   "CM: Unexpected MPA Key received\n");
1067             return -EINVAL;
1068         }
1069     } else {
1070         if (memcmp(mpa_frame->key, IEFT_MPA_KEY_REP,
1071                IETF_MPA_KEY_SIZE)) {
1072             ibdev_dbg(&cm_node->iwdev->ibdev,
1073                   "CM: Unexpected MPA Key received\n");
1074             return -EINVAL;
1075         }
1076     }
1077
1078     if (priv_data_len + mpa_hdr_len > len) {
1079         ibdev_dbg(&cm_node->iwdev->ibdev,
1080               "CM: ietf buffer len(%x + %x != %x)\n",
1081               priv_data_len, mpa_hdr_len, len);
1082         return -EOVERFLOW;
1083     }
1084
1085     if (len > IRDMA_MAX_CM_BUF) {
1086         ibdev_dbg(&cm_node->iwdev->ibdev,
1087               "CM: ietf buffer large len = %d\n", len);
1088         return -EOVERFLOW;
1089     }
1090
1091     switch (mpa_frame->rev) {
1092     case IETF_MPA_V2:
1093         mpa_hdr_len += IETF_RTR_MSG_SIZE;
1094         ret = irdma_negotiate_mpa_v2_ird_ord(cm_node, buf);
1095         if (ret)
1096             return ret;
1097         break;
1098     case IETF_MPA_V1:
1099     default:
1100         break;
1101     }
1102
1103     memcpy(cm_node->pdata_buf, buf + mpa_hdr_len, priv_data_len);
1104     cm_node->pdata.size = priv_data_len;
1105
1106     if (mpa_frame->flags & IETF_MPA_FLAGS_REJECT)
1107         *type = IRDMA_MPA_REQUEST_REJECT;
1108
1109     if (mpa_frame->flags & IETF_MPA_FLAGS_MARKERS)
1110         cm_node->snd_mark_en = true;
1111
1112     return 0;
1113 }
1114
1115 /**
1116  * irdma_schedule_cm_timer
1117  * @cm_node: connection's node
1118  * @sqbuf: buffer to send
1119  * @type: if it is send or close
1120  * @send_retrans: if rexmits to be done
1121  * @close_when_complete: is cm_node to be removed
1122  *
1123  * note - cm_node needs to be protected before calling this. Encase in:
1124  *      irdma_rem_ref_cm_node(cm_core, cm_node);
1125  *      irdma_schedule_cm_timer(...)
1126  *      refcount_inc(&cm_node->refcnt);
1127  */
1128 int irdma_schedule_cm_timer(struct irdma_cm_node *cm_node,
1129                 struct irdma_puda_buf *sqbuf,
1130                 enum irdma_timer_type type, int send_retrans,
1131                 int close_when_complete)
1132 {
1133     struct irdma_sc_vsi *vsi = &cm_node->iwdev->vsi;
1134     struct irdma_cm_core *cm_core = cm_node->cm_core;
1135     struct irdma_timer_entry *new_send;
1136     u32 was_timer_set;
1137     unsigned long flags;
1138
1139     new_send = kzalloc(sizeof(*new_send), GFP_ATOMIC);
1140     if (!new_send) {
1141         if (type != IRDMA_TIMER_TYPE_CLOSE)
1142             irdma_free_sqbuf(vsi, sqbuf);
1143         return -ENOMEM;
1144     }
1145
1146     new_send->retrycount = IRDMA_DEFAULT_RETRYS;
1147     new_send->retranscount = IRDMA_DEFAULT_RETRANS;
1148     new_send->sqbuf = sqbuf;
1149     new_send->timetosend = jiffies;
1150     new_send->type = type;
1151     new_send->send_retrans = send_retrans;
1152     new_send->close_when_complete = close_when_complete;
1153
1154     if (type == IRDMA_TIMER_TYPE_CLOSE) {
1155         new_send->timetosend += (HZ / 10);
1156         if (cm_node->close_entry) {
1157             kfree(new_send);
1158             ibdev_dbg(&cm_node->iwdev->ibdev,
1159                   "CM: already close entry\n");
1160             return -EINVAL;
1161         }
1162
1163         cm_node->close_entry = new_send;
1164     } else { /* type == IRDMA_TIMER_TYPE_SEND */
1165         spin_lock_irqsave(&cm_node->retrans_list_lock, flags);
1166         cm_node->send_entry = new_send;
1167         refcount_inc(&cm_node->refcnt);
1168         spin_unlock_irqrestore(&cm_node->retrans_list_lock, flags);
1169         new_send->timetosend = jiffies + IRDMA_RETRY_TIMEOUT;
1170
1171         refcount_inc(&sqbuf->refcount);
1172         irdma_puda_send_buf(vsi->ilq, sqbuf);
1173         if (!send_retrans) {
1174             irdma_cleanup_retrans_entry(cm_node);
1175             if (close_when_complete)
1176                 irdma_rem_ref_cm_node(cm_node);
1177             return 0;
1178         }
1179     }
1180
1181     spin_lock_irqsave(&cm_core->ht_lock, flags);
1182     was_timer_set = timer_pending(&cm_core->tcp_timer);
1183
1184     if (!was_timer_set) {
1185         cm_core->tcp_timer.expires = new_send->timetosend;
1186         add_timer(&cm_core->tcp_timer);
1187     }
1188     spin_unlock_irqrestore(&cm_core->ht_lock, flags);
1189
1190     return 0;
1191 }
1192
1193 /**
1194  * irdma_retrans_expired - Could not rexmit the packet
1195  * @cm_node: connection's node
1196  */
1197 static void irdma_retrans_expired(struct irdma_cm_node *cm_node)
1198 {
1199     enum irdma_cm_node_state state = cm_node->state;
1200
1201     cm_node->state = IRDMA_CM_STATE_CLOSED;
1202     switch (state) {
1203     case IRDMA_CM_STATE_SYN_RCVD:
1204     case IRDMA_CM_STATE_CLOSING:
1205         irdma_rem_ref_cm_node(cm_node);
1206         break;
1207     case IRDMA_CM_STATE_FIN_WAIT1:
1208     case IRDMA_CM_STATE_LAST_ACK:
1209         irdma_send_reset(cm_node);
1210         break;
1211     default:
1212         refcount_inc(&cm_node->refcnt);
1213         irdma_send_reset(cm_node);
1214         irdma_create_event(cm_node, IRDMA_CM_EVENT_ABORTED);
1215         break;
1216     }
1217 }
1218
1219 /**
1220  * irdma_handle_close_entry - for handling retry/timeouts
1221  * @cm_node: connection's node
1222  * @rem_node: flag for remove cm_node
1223  */
1224 static void irdma_handle_close_entry(struct irdma_cm_node *cm_node,
1225                      u32 rem_node)
1226 {
1227     struct irdma_timer_entry *close_entry = cm_node->close_entry;
1228     struct irdma_qp *iwqp;
1229     unsigned long flags;
1230
1231     if (!close_entry)
1232         return;
1233     iwqp = (struct irdma_qp *)close_entry->sqbuf;
1234     if (iwqp) {
1235         spin_lock_irqsave(&iwqp->lock, flags);
1236         if (iwqp->cm_id) {
1237             iwqp->hw_tcp_state = IRDMA_TCP_STATE_CLOSED;
1238             iwqp->hw_iwarp_state = IRDMA_QP_STATE_ERROR;
1239             iwqp->last_aeq = IRDMA_AE_RESET_SENT;
1240             iwqp->ibqp_state = IB_QPS_ERR;
1241             spin_unlock_irqrestore(&iwqp->lock, flags);
1242             irdma_cm_disconn(iwqp);
1243         } else {
1244             spin_unlock_irqrestore(&iwqp->lock, flags);
1245         }
1246     } else if (rem_node) {
1247         /* TIME_WAIT state */
1248         irdma_rem_ref_cm_node(cm_node);
1249     }
1250
1251     kfree(close_entry);
1252     cm_node->close_entry = NULL;
1253 }
1254
1255 /**
1256  * irdma_cm_timer_tick - system's timer expired callback
1257  * @t: Pointer to timer_list
1258  */
1259 static void irdma_cm_timer_tick(struct timer_list *t)
1260 {
1261     unsigned long nexttimeout = jiffies + IRDMA_LONG_TIME;
1262     struct irdma_cm_node *cm_node;
1263     struct irdma_timer_entry *send_entry, *close_entry;
1264     struct list_head *list_core_temp;
1265     struct list_head *list_node;
1266     struct irdma_cm_core *cm_core = from_timer(cm_core, t, tcp_timer);
1267     struct irdma_sc_vsi *vsi;
1268     u32 settimer = 0;
1269     unsigned long timetosend;
1270     unsigned long flags;
1271     struct list_head timer_list;
1272
1273     INIT_LIST_HEAD(&timer_list);
1274
1275     rcu_read_lock();
1276     irdma_timer_list_prep(cm_core, &timer_list);
1277     rcu_read_unlock();
1278
1279     list_for_each_safe (list_node, list_core_temp, &timer_list) {
1280         cm_node = container_of(list_node, struct irdma_cm_node,
1281                        timer_entry);
1282         close_entry = cm_node->close_entry;
1283
1284         if (close_entry) {
1285             if (time_after(close_entry->timetosend, jiffies)) {
1286                 if (nexttimeout > close_entry->timetosend ||
1287                     !settimer) {
1288                     nexttimeout = close_entry->timetosend;
1289                     settimer = 1;
1290                 }
1291             } else {
1292                 irdma_handle_close_entry(cm_node, 1);
1293             }
1294         }
1295
1296         spin_lock_irqsave(&cm_node->retrans_list_lock, flags);
1297
1298         send_entry = cm_node->send_entry;
1299         if (!send_entry)
1300             goto done;
1301         if (time_after(send_entry->timetosend, jiffies)) {
1302             if (cm_node->state != IRDMA_CM_STATE_OFFLOADED) {
1303                 if (nexttimeout > send_entry->timetosend ||
1304                     !settimer) {
1305                     nexttimeout = send_entry->timetosend;
1306                     settimer = 1;
1307                 }
1308             } else {
1309                 irdma_free_retrans_entry(cm_node);
1310             }
1311             goto done;
1312         }
1313
1314         if (cm_node->state == IRDMA_CM_STATE_OFFLOADED ||
1315             cm_node->state == IRDMA_CM_STATE_CLOSED) {
1316             irdma_free_retrans_entry(cm_node);
1317             goto done;
1318         }
1319
1320         if (!send_entry->retranscount || !send_entry->retrycount) {
1321             irdma_free_retrans_entry(cm_node);
1322
1323             spin_unlock_irqrestore(&cm_node->retrans_list_lock,
1324                            flags);
1325             irdma_retrans_expired(cm_node);
1326             cm_node->state = IRDMA_CM_STATE_CLOSED;
1327             spin_lock_irqsave(&cm_node->retrans_list_lock, flags);
1328             goto done;
1329         }
1330         spin_unlock_irqrestore(&cm_node->retrans_list_lock, flags);
1331
1332         vsi = &cm_node->iwdev->vsi;
1333         if (!cm_node->ack_rcvd) {
1334             refcount_inc(&send_entry->sqbuf->refcount);
1335             irdma_puda_send_buf(vsi->ilq, send_entry->sqbuf);
1336             cm_node->cm_core->stats_pkt_retrans++;
1337         }
1338
1339         spin_lock_irqsave(&cm_node->retrans_list_lock, flags);
1340         if (send_entry->send_retrans) {
1341             send_entry->retranscount--;
1342             timetosend = (IRDMA_RETRY_TIMEOUT <<
1343                       (IRDMA_DEFAULT_RETRANS -
1344                        send_entry->retranscount));
1345
1346             send_entry->timetosend = jiffies +
1347                 min(timetosend, IRDMA_MAX_TIMEOUT);
1348             if (nexttimeout > send_entry->timetosend || !settimer) {
1349                 nexttimeout = send_entry->timetosend;
1350                 settimer = 1;
1351             }
1352         } else {
1353             int close_when_complete;
1354
1355             close_when_complete = send_entry->close_when_complete;
1356             irdma_free_retrans_entry(cm_node);
1357             if (close_when_complete)
1358                 irdma_rem_ref_cm_node(cm_node);
1359         }
1360 done:
1361         spin_unlock_irqrestore(&cm_node->retrans_list_lock, flags);
1362         irdma_rem_ref_cm_node(cm_node);
1363     }
1364
1365     if (settimer) {
1366         spin_lock_irqsave(&cm_core->ht_lock, flags);
1367         if (!timer_pending(&cm_core->tcp_timer)) {
1368             cm_core->tcp_timer.expires = nexttimeout;
1369             add_timer(&cm_core->tcp_timer);
1370         }
1371         spin_unlock_irqrestore(&cm_core->ht_lock, flags);
1372     }
1373 }
1374
1375 /**
1376  * irdma_send_syn - send SYN packet
1377  * @cm_node: connection's node
1378  * @sendack: flag to set ACK bit or not
1379  */
1380 int irdma_send_syn(struct irdma_cm_node *cm_node, u32 sendack)
1381 {
1382     struct irdma_puda_buf *sqbuf;
1383     int flags = SET_SYN;
1384     char optionsbuf[sizeof(struct option_mss) +
1385             sizeof(struct option_windowscale) +
1386             sizeof(struct option_base) + TCP_OPTIONS_PADDING];
1387     struct irdma_kmem_info opts;
1388     int optionssize = 0;
1389     /* Sending MSS option */
1390     union all_known_options *options;
1391
1392     opts.addr = optionsbuf;
1393     if (!cm_node)
1394         return -EINVAL;
1395
1396     options = (union all_known_options *)&optionsbuf[optionssize];
1397     options->mss.optionnum = OPTION_NUM_MSS;
1398     options->mss.len = sizeof(struct option_mss);
1399     options->mss.mss = htons(cm_node->tcp_cntxt.mss);
1400     optionssize += sizeof(struct option_mss);
1401
1402     options = (union all_known_options *)&optionsbuf[optionssize];
1403     options->windowscale.optionnum = OPTION_NUM_WINDOW_SCALE;
1404     options->windowscale.len = sizeof(struct option_windowscale);
1405     options->windowscale.shiftcount = cm_node->tcp_cntxt.rcv_wscale;
1406     optionssize += sizeof(struct option_windowscale);
1407     options = (union all_known_options *)&optionsbuf[optionssize];
1408     options->eol = OPTION_NUM_EOL;
1409     optionssize += 1;
1410
1411     if (sendack)
1412         flags |= SET_ACK;
1413
1414     opts.size = optionssize;
1415
1416     sqbuf = cm_node->cm_core->form_cm_frame(cm_node, &opts, NULL, NULL,
1417                         flags);
1418     if (!sqbuf)
1419         return -ENOMEM;
1420
1421     return irdma_schedule_cm_timer(cm_node, sqbuf, IRDMA_TIMER_TYPE_SEND, 1,
1422                        0);
1423 }
1424
1425 /**
1426  * irdma_send_ack - Send ACK packet
1427  * @cm_node: connection's node
1428  */
1429 void irdma_send_ack(struct irdma_cm_node *cm_node)
1430 {
1431     struct irdma_puda_buf *sqbuf;
1432     struct irdma_sc_vsi *vsi = &cm_node->iwdev->vsi;
1433
1434     sqbuf = cm_node->cm_core->form_cm_frame(cm_node, NULL, NULL, NULL,
1435                         SET_ACK);
1436     if (sqbuf)
1437         irdma_puda_send_buf(vsi->ilq, sqbuf);
1438 }
1439
1440 /**
1441  * irdma_send_fin - Send FIN pkt
1442  * @cm_node: connection's node
1443  */
1444 static int irdma_send_fin(struct irdma_cm_node *cm_node)
1445 {
1446     struct irdma_puda_buf *sqbuf;
1447
1448     sqbuf = cm_node->cm_core->form_cm_frame(cm_node, NULL, NULL, NULL,
1449                         SET_ACK | SET_FIN);
1450     if (!sqbuf)
1451         return -ENOMEM;
1452
1453     return irdma_schedule_cm_timer(cm_node, sqbuf, IRDMA_TIMER_TYPE_SEND, 1,
1454                        0);
1455 }
1456
1457 /**
1458  * irdma_find_listener - find a cm node listening on this addr-port pair
1459  * @cm_core: cm's core
1460  * @dst_addr: listener ip addr
1461  * @dst_port: listener tcp port num
1462  * @vlan_id: virtual LAN ID
1463  * @listener_state: state to match with listen node's
1464  */
1465 static struct irdma_cm_listener *
1466 irdma_find_listener(struct irdma_cm_core *cm_core, u32 *dst_addr, u16 dst_port,
1467             u16 vlan_id, enum irdma_cm_listener_state listener_state)
1468 {
1469     struct irdma_cm_listener *listen_node;
1470     static const u32 ip_zero[4] = { 0, 0, 0, 0 };
1471     u32 listen_addr[4];
1472     u16 listen_port;
1473     unsigned long flags;
1474
1475     /* walk list and find cm_node associated with this session ID */
1476     spin_lock_irqsave(&cm_core->listen_list_lock, flags);
1477     list_for_each_entry (listen_node, &cm_core->listen_list, list) {
1478         memcpy(listen_addr, listen_node->loc_addr, sizeof(listen_addr));
1479         listen_port = listen_node->loc_port;
1480         if (listen_port != dst_port ||
1481             !(listener_state & listen_node->listener_state))
1482             continue;
1483         /* compare node pair, return node handle if a match */
1484         if (!memcmp(listen_addr, ip_zero, sizeof(listen_addr)) ||
1485             (!memcmp(listen_addr, dst_addr, sizeof(listen_addr)) &&
1486              vlan_id == listen_node->vlan_id)) {
1487             refcount_inc(&listen_node->refcnt);
1488             spin_unlock_irqrestore(&cm_core->listen_list_lock,
1489                            flags);
1490             trace_irdma_find_listener(listen_node);
1491             return listen_node;
1492         }
1493     }
1494     spin_unlock_irqrestore(&cm_core->listen_list_lock, flags);
1495
1496     return NULL;
1497 }
1498
1499 /**
1500  * irdma_del_multiple_qhash - Remove qhash and child listens
1501  * @iwdev: iWarp device
1502  * @cm_info: CM info for parent listen node
1503  * @cm_parent_listen_node: The parent listen node
1504  */
1505 static int irdma_del_multiple_qhash(struct irdma_device *iwdev,
1506                     struct irdma_cm_info *cm_info,
1507                     struct irdma_cm_listener *cm_parent_listen_node)
1508 {
1509     struct irdma_cm_listener *child_listen_node;
1510     struct list_head *pos, *tpos;
1511     unsigned long flags;
1512     int ret = -EINVAL;
1513
1514     spin_lock_irqsave(&iwdev->cm_core.listen_list_lock, flags);
1515     list_for_each_safe (pos, tpos,
1516                 &cm_parent_listen_node->child_listen_list) {
1517         child_listen_node = list_entry(pos, struct irdma_cm_listener,
1518                            child_listen_list);
1519         if (child_listen_node->ipv4)
1520             ibdev_dbg(&iwdev->ibdev,
1521                   "CM: removing child listen for IP=%pI4, port=%d, vlan=%d\n",
1522                   child_listen_node->loc_addr,
1523                   child_listen_node->loc_port,
1524                   child_listen_node->vlan_id);
1525         else
1526             ibdev_dbg(&iwdev->ibdev,
1527                   "CM: removing child listen for IP=%pI6, port=%d, vlan=%d\n",
1528                   child_listen_node->loc_addr,
1529                   child_listen_node->loc_port,
1530                   child_listen_node->vlan_id);
1531         trace_irdma_del_multiple_qhash(child_listen_node);
1532         list_del(pos);
1533         memcpy(cm_info->loc_addr, child_listen_node->loc_addr,
1534                sizeof(cm_info->loc_addr));
1535         cm_info->vlan_id = child_listen_node->vlan_id;
1536         if (child_listen_node->qhash_set) {
1537             ret = irdma_manage_qhash(iwdev, cm_info,
1538                          IRDMA_QHASH_TYPE_TCP_SYN,
1539                          IRDMA_QHASH_MANAGE_TYPE_DELETE,
1540                          NULL, false);
1541             child_listen_node->qhash_set = false;
1542         } else {
1543             ret = 0;
1544         }
1545         ibdev_dbg(&iwdev->ibdev,
1546               "CM: Child listen node freed = %p\n",
1547               child_listen_node);
1548         kfree(child_listen_node);
1549         cm_parent_listen_node->cm_core->stats_listen_nodes_destroyed++;
1550     }
1551     spin_unlock_irqrestore(&iwdev->cm_core.listen_list_lock, flags);
1552
1553     return ret;
1554 }
1555
1556 /**
1557  * irdma_netdev_vlan_ipv6 - Gets the netdev and mac
1558  * @addr: local IPv6 address
1559  * @vlan_id: vlan id for the given IPv6 address
1560  * @mac: mac address for the given IPv6 address
1561  *
1562  * Returns the net_device of the IPv6 address and also sets the
1563  * vlan id and mac for that address.
1564  */
1565 struct net_device *irdma_netdev_vlan_ipv6(u32 *addr, u16 *vlan_id, u8 *mac)
1566 {
1567     struct net_device *ip_dev = NULL;
1568     struct in6_addr laddr6;
1569
1570     if (!IS_ENABLED(CONFIG_IPV6))
1571         return NULL;
1572
1573     irdma_copy_ip_htonl(laddr6.in6_u.u6_addr32, addr);
1574     if (vlan_id)
1575         *vlan_id = 0xFFFF;  /* Match rdma_vlan_dev_vlan_id() */
1576     if (mac)
1577         eth_zero_addr(mac);
1578
1579     rcu_read_lock();
1580     for_each_netdev_rcu (&init_net, ip_dev) {
1581         if (ipv6_chk_addr(&init_net, &laddr6, ip_dev, 1)) {
1582             if (vlan_id)
1583                 *vlan_id = rdma_vlan_dev_vlan_id(ip_dev);
1584             if (ip_dev->dev_addr && mac)
1585                 ether_addr_copy(mac, ip_dev->dev_addr);
1586             break;
1587         }
1588     }
1589     rcu_read_unlock();
1590
1591     return ip_dev;
1592 }
1593
1594 /**
1595  * irdma_get_vlan_ipv4 - Returns the vlan_id for IPv4 address
1596  * @addr: local IPv4 address
1597  */
1598 u16 irdma_get_vlan_ipv4(u32 *addr)
1599 {
1600     struct net_device *netdev;
1601     u16 vlan_id = 0xFFFF;
1602
1603     netdev = ip_dev_find(&init_net, htonl(addr[0]));
1604     if (netdev) {
1605         vlan_id = rdma_vlan_dev_vlan_id(netdev);
1606         dev_put(netdev);
1607     }
1608
1609     return vlan_id;
1610 }
1611
1612 /**
1613  * irdma_add_mqh_6 - Adds multiple qhashes for IPv6
1614  * @iwdev: iWarp device
1615  * @cm_info: CM info for parent listen node
1616  * @cm_parent_listen_node: The parent listen node
1617  *
1618  * Adds a qhash and a child listen node for every IPv6 address
1619  * on the adapter and adds the associated qhash filter
1620  */
1621 static int irdma_add_mqh_6(struct irdma_device *iwdev,
1622                struct irdma_cm_info *cm_info,
1623                struct irdma_cm_listener *cm_parent_listen_node)
1624 {
1625     struct net_device *ip_dev;
1626     struct inet6_dev *idev;
1627     struct inet6_ifaddr *ifp, *tmp;
1628     struct irdma_cm_listener *child_listen_node;
1629     unsigned long flags;
1630     int ret = 0;
1631
1632     rtnl_lock();
1633     for_each_netdev(&init_net, ip_dev) {
1634         if (!(ip_dev->flags & IFF_UP))
1635             continue;
1636
1637         if (((rdma_vlan_dev_vlan_id(ip_dev) >= VLAN_N_VID) ||
1638              (rdma_vlan_dev_real_dev(ip_dev) != iwdev->netdev)) &&
1639             ip_dev != iwdev->netdev)
1640             continue;
1641
1642         idev = __in6_dev_get(ip_dev);
1643         if (!idev) {
1644             ibdev_dbg(&iwdev->ibdev, "CM: idev == NULL\n");
1645             break;
1646         }
1647         list_for_each_entry_safe (ifp, tmp, &idev->addr_list, if_list) {
1648             ibdev_dbg(&iwdev->ibdev, "CM: IP=%pI6, vlan_id=%d, MAC=%pM\n",
1649                   &ifp->addr, rdma_vlan_dev_vlan_id(ip_dev),
1650                   ip_dev->dev_addr);
1651             child_listen_node = kzalloc(sizeof(*child_listen_node), GFP_KERNEL);
1652             ibdev_dbg(&iwdev->ibdev, "CM: Allocating child listener %p\n",
1653                   child_listen_node);
1654             if (!child_listen_node) {
1655                 ibdev_dbg(&iwdev->ibdev, "CM: listener memory allocation\n");
1656                 ret = -ENOMEM;
1657                 goto exit;
1658             }
1659
1660             cm_info->vlan_id = rdma_vlan_dev_vlan_id(ip_dev);
1661             cm_parent_listen_node->vlan_id = cm_info->vlan_id;
1662             memcpy(child_listen_node, cm_parent_listen_node,
1663                    sizeof(*child_listen_node));
1664             irdma_copy_ip_ntohl(child_listen_node->loc_addr,
1665                         ifp->addr.in6_u.u6_addr32);
1666             memcpy(cm_info->loc_addr, child_listen_node->loc_addr,
1667                    sizeof(cm_info->loc_addr));
1668             ret = irdma_manage_qhash(iwdev, cm_info,
1669                          IRDMA_QHASH_TYPE_TCP_SYN,
1670                          IRDMA_QHASH_MANAGE_TYPE_ADD,
1671                          NULL, true);
1672             if (ret) {
1673                 kfree(child_listen_node);
1674                 continue;
1675             }
1676
1677             trace_irdma_add_mqh_6(iwdev, child_listen_node,
1678                           ip_dev->dev_addr);
1679
1680             child_listen_node->qhash_set = true;
1681             spin_lock_irqsave(&iwdev->cm_core.listen_list_lock, flags);
1682             list_add(&child_listen_node->child_listen_list,
1683                  &cm_parent_listen_node->child_listen_list);
1684             spin_unlock_irqrestore(&iwdev->cm_core.listen_list_lock, flags);
1685             cm_parent_listen_node->cm_core->stats_listen_nodes_created++;
1686         }
1687     }
1688 exit:
1689     rtnl_unlock();
1690
1691     return ret;
1692 }
1693
1694 /**
1695  * irdma_add_mqh_4 - Adds multiple qhashes for IPv4
1696  * @iwdev: iWarp device
1697  * @cm_info: CM info for parent listen node
1698  * @cm_parent_listen_node: The parent listen node
1699  *
1700  * Adds a qhash and a child listen node for every IPv4 address
1701  * on the adapter and adds the associated qhash filter
1702  */
1703 static int irdma_add_mqh_4(struct irdma_device *iwdev,
1704                struct irdma_cm_info *cm_info,
1705                struct irdma_cm_listener *cm_parent_listen_node)
1706 {
1707     struct net_device *ip_dev;
1708     struct in_device *idev;
1709     struct irdma_cm_listener *child_listen_node;
1710     unsigned long flags;
1711     const struct in_ifaddr *ifa;
1712     int ret = 0;
1713
1714     rtnl_lock();
1715     for_each_netdev(&init_net, ip_dev) {
1716         if (!(ip_dev->flags & IFF_UP))
1717             continue;
1718
1719         if (((rdma_vlan_dev_vlan_id(ip_dev) >= VLAN_N_VID) ||
1720              (rdma_vlan_dev_real_dev(ip_dev) != iwdev->netdev)) &&
1721             ip_dev != iwdev->netdev)
1722             continue;
1723
1724         idev = in_dev_get(ip_dev);
1725         in_dev_for_each_ifa_rtnl(ifa, idev) {
1726             ibdev_dbg(&iwdev->ibdev,
1727                   "CM: Allocating child CM Listener forIP=%pI4, vlan_id=%d, MAC=%pM\n",
1728                   &ifa->ifa_address, rdma_vlan_dev_vlan_id(ip_dev),
1729                   ip_dev->dev_addr);
1730             child_listen_node = kzalloc(sizeof(*child_listen_node), GFP_KERNEL);
1731             cm_parent_listen_node->cm_core->stats_listen_nodes_created++;
1732             ibdev_dbg(&iwdev->ibdev, "CM: Allocating child listener %p\n",
1733                   child_listen_node);
1734             if (!child_listen_node) {
1735                 ibdev_dbg(&iwdev->ibdev, "CM: listener memory allocation\n");
1736                 in_dev_put(idev);
1737                 ret = -ENOMEM;
1738                 goto exit;
1739             }
1740
1741             cm_info->vlan_id = rdma_vlan_dev_vlan_id(ip_dev);
1742             cm_parent_listen_node->vlan_id = cm_info->vlan_id;
1743             memcpy(child_listen_node, cm_parent_listen_node,
1744                    sizeof(*child_listen_node));
1745             child_listen_node->loc_addr[0] =
1746                 ntohl(ifa->ifa_address);
1747             memcpy(cm_info->loc_addr, child_listen_node->loc_addr,
1748                    sizeof(cm_info->loc_addr));
1749             ret = irdma_manage_qhash(iwdev, cm_info,
1750                          IRDMA_QHASH_TYPE_TCP_SYN,
1751                          IRDMA_QHASH_MANAGE_TYPE_ADD,
1752                          NULL, true);
1753             if (ret) {
1754                 kfree(child_listen_node);
1755                 cm_parent_listen_node->cm_core
1756                     ->stats_listen_nodes_created--;
1757                 continue;
1758             }
1759
1760             trace_irdma_add_mqh_4(iwdev, child_listen_node,
1761                           ip_dev->dev_addr);
1762
1763             child_listen_node->qhash_set = true;
1764             spin_lock_irqsave(&iwdev->cm_core.listen_list_lock,
1765                       flags);
1766             list_add(&child_listen_node->child_listen_list,
1767                  &cm_parent_listen_node->child_listen_list);
1768             spin_unlock_irqrestore(&iwdev->cm_core.listen_list_lock, flags);
1769         }
1770         in_dev_put(idev);
1771     }
1772 exit:
1773     rtnl_unlock();
1774
1775     return ret;
1776 }
1777
1778 /**
1779  * irdma_add_mqh - Adds multiple qhashes
1780  * @iwdev: iWarp device
1781  * @cm_info: CM info for parent listen node
1782  * @cm_listen_node: The parent listen node
1783  */
1784 static int irdma_add_mqh(struct irdma_device *iwdev,
1785              struct irdma_cm_info *cm_info,
1786              struct irdma_cm_listener *cm_listen_node)
1787 {
1788     if (cm_info->ipv4)
1789         return irdma_add_mqh_4(iwdev, cm_info, cm_listen_node);
1790     else
1791         return irdma_add_mqh_6(iwdev, cm_info, cm_listen_node);
1792 }
1793
1794 /**
1795  * irdma_reset_list_prep - add connection nodes slated for reset to list
1796  * @cm_core: cm's core
1797  * @listener: pointer to listener node
1798  * @reset_list: a list to which cm_node will be selected
1799  */
1800 static void irdma_reset_list_prep(struct irdma_cm_core *cm_core,
1801                   struct irdma_cm_listener *listener,
1802                   struct list_head *reset_list)
1803 {
1804     struct irdma_cm_node *cm_node;
1805     int bkt;
1806
1807     hash_for_each_rcu(cm_core->cm_hash_tbl, bkt, cm_node, list) {
1808         if (cm_node->listener == listener &&
1809             !cm_node->accelerated &&
1810             refcount_inc_not_zero(&cm_node->refcnt))
1811             list_add(&cm_node->reset_entry, reset_list);
1812     }
1813 }
1814
1815 /**
1816  * irdma_dec_refcnt_listen - delete listener and associated cm nodes
1817  * @cm_core: cm's core
1818  * @listener: pointer to listener node
1819  * @free_hanging_nodes: to free associated cm_nodes
1820  * @apbvt_del: flag to delete the apbvt
1821  */
1822 static int irdma_dec_refcnt_listen(struct irdma_cm_core *cm_core,
1823                    struct irdma_cm_listener *listener,
1824                    int free_hanging_nodes, bool apbvt_del)
1825 {
1826     int err;
1827     struct list_head *list_pos;
1828     struct list_head *list_temp;
1829     struct irdma_cm_node *cm_node;
1830     struct list_head reset_list;
1831     struct irdma_cm_info nfo;
1832     enum irdma_cm_node_state old_state;
1833     unsigned long flags;
1834
1835     trace_irdma_dec_refcnt_listen(listener, __builtin_return_address(0));
1836     /* free non-accelerated child nodes for this listener */
1837     INIT_LIST_HEAD(&reset_list);
1838     if (free_hanging_nodes) {
1839         rcu_read_lock();
1840         irdma_reset_list_prep(cm_core, listener, &reset_list);
1841         rcu_read_unlock();
1842     }
1843
1844     list_for_each_safe (list_pos, list_temp, &reset_list) {
1845         cm_node = container_of(list_pos, struct irdma_cm_node,
1846                        reset_entry);
1847         if (cm_node->state >= IRDMA_CM_STATE_FIN_WAIT1) {
1848             irdma_rem_ref_cm_node(cm_node);
1849             continue;
1850         }
1851
1852         irdma_cleanup_retrans_entry(cm_node);
1853         err = irdma_send_reset(cm_node);
1854         if (err) {
1855             cm_node->state = IRDMA_CM_STATE_CLOSED;
1856             ibdev_dbg(&cm_node->iwdev->ibdev,
1857                   "CM: send reset failed\n");
1858         } else {
1859             old_state = cm_node->state;
1860             cm_node->state = IRDMA_CM_STATE_LISTENER_DESTROYED;
1861             if (old_state != IRDMA_CM_STATE_MPAREQ_RCVD)
1862                 irdma_rem_ref_cm_node(cm_node);
1863         }
1864     }
1865
1866     if (refcount_dec_and_test(&listener->refcnt)) {
1867         spin_lock_irqsave(&cm_core->listen_list_lock, flags);
1868         list_del(&listener->list);
1869         spin_unlock_irqrestore(&cm_core->listen_list_lock, flags);
1870
1871         if (apbvt_del)
1872             irdma_del_apbvt(listener->iwdev,
1873                     listener->apbvt_entry);
1874         memcpy(nfo.loc_addr, listener->loc_addr, sizeof(nfo.loc_addr));
1875         nfo.loc_port = listener->loc_port;
1876         nfo.ipv4 = listener->ipv4;
1877         nfo.vlan_id = listener->vlan_id;
1878         nfo.user_pri = listener->user_pri;
1879         nfo.qh_qpid = listener->iwdev->vsi.ilq->qp_id;
1880
1881         if (!list_empty(&listener->child_listen_list)) {
1882             irdma_del_multiple_qhash(listener->iwdev, &nfo,
1883                          listener);
1884         } else {
1885             if (listener->qhash_set)
1886                 irdma_manage_qhash(listener->iwdev,
1887                            &nfo,
1888                            IRDMA_QHASH_TYPE_TCP_SYN,
1889                            IRDMA_QHASH_MANAGE_TYPE_DELETE,
1890                            NULL, false);
1891         }
1892
1893         cm_core->stats_listen_destroyed++;
1894         cm_core->stats_listen_nodes_destroyed++;
1895         ibdev_dbg(&listener->iwdev->ibdev,
1896               "CM: loc_port=0x%04x loc_addr=%pI4 cm_listen_node=%p cm_id=%p qhash_set=%d vlan_id=%d apbvt_del=%d\n",
1897               listener->loc_port, listener->loc_addr, listener,
1898               listener->cm_id, listener->qhash_set,
1899               listener->vlan_id, apbvt_del);
1900         kfree(listener);
1901         listener = NULL;
1902         return 0;
1903     }
1904
1905     return -EINVAL;
1906 }
1907
1908 /**
1909  * irdma_cm_del_listen - delete a listener
1910  * @cm_core: cm's core
1911  * @listener: passive connection's listener
1912  * @apbvt_del: flag to delete apbvt
1913  */
1914 static int irdma_cm_del_listen(struct irdma_cm_core *cm_core,
1915                    struct irdma_cm_listener *listener,
1916                    bool apbvt_del)
1917 {
1918     listener->listener_state = IRDMA_CM_LISTENER_PASSIVE_STATE;
1919     listener->cm_id = NULL;
1920
1921     return irdma_dec_refcnt_listen(cm_core, listener, 1, apbvt_del);
1922 }
1923
1924 /**
1925  * irdma_addr_resolve_neigh - resolve neighbor address
1926  * @iwdev: iwarp device structure
1927  * @src_ip: local ip address
1928  * @dst_ip: remote ip address
1929  * @arpindex: if there is an arp entry
1930  */
1931 static int irdma_addr_resolve_neigh(struct irdma_device *iwdev, u32 src_ip,
1932                     u32 dst_ip, int arpindex)
1933 {
1934     struct rtable *rt;
1935     struct neighbour *neigh;
1936     int rc = arpindex;
1937     __be32 dst_ipaddr = htonl(dst_ip);
1938     __be32 src_ipaddr = htonl(src_ip);
1939
1940     rt = ip_route_output(&init_net, dst_ipaddr, src_ipaddr, 0, 0);
1941     if (IS_ERR(rt)) {
1942         ibdev_dbg(&iwdev->ibdev, "CM: ip_route_output fail\n");
1943         return -EINVAL;
1944     }
1945
1946     neigh = dst_neigh_lookup(&rt->dst, &dst_ipaddr);
1947     if (!neigh)
1948         goto exit;
1949
1950     if (neigh->nud_state & NUD_VALID)
1951         rc = irdma_add_arp(iwdev->rf, &dst_ip, true, neigh->ha);
1952     else
1953         neigh_event_send(neigh, NULL);
1954     if (neigh)
1955         neigh_release(neigh);
1956 exit:
1957     ip_rt_put(rt);
1958
1959     return rc;
1960 }
1961
1962 /**
1963  * irdma_get_dst_ipv6 - get destination cache entry via ipv6 lookup
1964  * @src_addr: local ipv6 sock address
1965  * @dst_addr: destination ipv6 sock address
1966  */
1967 static struct dst_entry *irdma_get_dst_ipv6(struct sockaddr_in6 *src_addr,
1968                         struct sockaddr_in6 *dst_addr)
1969 {
1970     struct dst_entry *dst = NULL;
1971
1972     if ((IS_ENABLED(CONFIG_IPV6))) {
1973         struct flowi6 fl6 = {};
1974
1975         fl6.daddr = dst_addr->sin6_addr;
1976         fl6.saddr = src_addr->sin6_addr;
1977         if (ipv6_addr_type(&fl6.daddr) & IPV6_ADDR_LINKLOCAL)
1978             fl6.flowi6_oif = dst_addr->sin6_scope_id;
1979
1980         dst = ip6_route_output(&init_net, NULL, &fl6);
1981     }
1982
1983     return dst;
1984 }
1985
1986 /**
1987  * irdma_addr_resolve_neigh_ipv6 - resolve neighbor ipv6 address
1988  * @iwdev: iwarp device structure
1989  * @src: local ip address
1990  * @dest: remote ip address
1991  * @arpindex: if there is an arp entry
1992  */
1993 static int irdma_addr_resolve_neigh_ipv6(struct irdma_device *iwdev, u32 *src,
1994                      u32 *dest, int arpindex)
1995 {
1996     struct neighbour *neigh;
1997     int rc = arpindex;
1998     struct dst_entry *dst;
1999     struct sockaddr_in6 dst_addr = {};
2000     struct sockaddr_in6 src_addr = {};
2001
2002     dst_addr.sin6_family = AF_INET6;
2003     irdma_copy_ip_htonl(dst_addr.sin6_addr.in6_u.u6_addr32, dest);
2004     src_addr.sin6_family = AF_INET6;
2005     irdma_copy_ip_htonl(src_addr.sin6_addr.in6_u.u6_addr32, src);
2006     dst = irdma_get_dst_ipv6(&src_addr, &dst_addr);
2007     if (!dst || dst->error) {
2008         if (dst) {
2009             dst_release(dst);
2010             ibdev_dbg(&iwdev->ibdev,
2011                   "CM: ip6_route_output returned dst->error = %d\n",
2012                   dst->error);
2013         }
2014         return -EINVAL;
2015     }
2016
2017     neigh = dst_neigh_lookup(dst, dst_addr.sin6_addr.in6_u.u6_addr32);
2018     if (!neigh)
2019         goto exit;
2020
2021     ibdev_dbg(&iwdev->ibdev, "CM: dst_neigh_lookup MAC=%pM\n",
2022           neigh->ha);
2023
2024     trace_irdma_addr_resolve(iwdev, neigh->ha);
2025
2026     if (neigh->nud_state & NUD_VALID)
2027         rc = irdma_add_arp(iwdev->rf, dest, false, neigh->ha);
2028     else
2029         neigh_event_send(neigh, NULL);
2030     if (neigh)
2031         neigh_release(neigh);
2032 exit:
2033     dst_release(dst);
2034
2035     return rc;
2036 }
2037
2038 /**
2039  * irdma_find_node - find a cm node that matches the reference cm node
2040  * @cm_core: cm's core
2041  * @rem_port: remote tcp port num
2042  * @rem_addr: remote ip addr
2043  * @loc_port: local tcp port num
2044  * @loc_addr: local ip addr
2045  * @vlan_id: local VLAN ID
2046  */
2047 struct irdma_cm_node *irdma_find_node(struct irdma_cm_core *cm_core,
2048                       u16 rem_port, u32 *rem_addr, u16 loc_port,
2049                       u32 *loc_addr, u16 vlan_id)
2050 {
2051     struct irdma_cm_node *cm_node;
2052     u32 key = (rem_port << 16) | loc_port;
2053
2054     rcu_read_lock();
2055     hash_for_each_possible_rcu(cm_core->cm_hash_tbl, cm_node, list, key) {
2056         if (cm_node->vlan_id == vlan_id &&
2057             cm_node->loc_port == loc_port && cm_node->rem_port == rem_port &&
2058             !memcmp(cm_node->loc_addr, loc_addr, sizeof(cm_node->loc_addr)) &&
2059             !memcmp(cm_node->rem_addr, rem_addr, sizeof(cm_node->rem_addr))) {
2060             if (!refcount_inc_not_zero(&cm_node->refcnt))
2061                 goto exit;
2062             rcu_read_unlock();
2063             trace_irdma_find_node(cm_node, 0, NULL);
2064             return cm_node;
2065         }
2066     }
2067
2068 exit:
2069     rcu_read_unlock();
2070
2071     /* no owner node */
2072     return NULL;
2073 }
2074
2075 /**
2076  * irdma_add_hte_node - add a cm node to the hash table
2077  * @cm_core: cm's core
2078  * @cm_node: connection's node
2079  */
2080 static void irdma_add_hte_node(struct irdma_cm_core *cm_core,
2081                    struct irdma_cm_node *cm_node)
2082 {
2083     unsigned long flags;
2084     u32 key = (cm_node->rem_port << 16) | cm_node->loc_port;
2085
2086     spin_lock_irqsave(&cm_core->ht_lock, flags);
2087     hash_add_rcu(cm_core->cm_hash_tbl, &cm_node->list, key);
2088     spin_unlock_irqrestore(&cm_core->ht_lock, flags);
2089 }
2090
2091 /**
2092  * irdma_ipv4_is_lpb - check if loopback
2093  * @loc_addr: local addr to compare
2094  * @rem_addr: remote address
2095  */
2096 bool irdma_ipv4_is_lpb(u32 loc_addr, u32 rem_addr)
2097 {
2098     return ipv4_is_loopback(htonl(rem_addr)) || (loc_addr == rem_addr);
2099 }
2100
2101 /**
2102  * irdma_ipv6_is_lpb - check if loopback
2103  * @loc_addr: local addr to compare
2104  * @rem_addr: remote address
2105  */
2106 bool irdma_ipv6_is_lpb(u32 *loc_addr, u32 *rem_addr)
2107 {
2108     struct in6_addr raddr6;
2109
2110     irdma_copy_ip_htonl(raddr6.in6_u.u6_addr32, rem_addr);
2111
2112     return !memcmp(loc_addr, rem_addr, 16) || ipv6_addr_loopback(&raddr6);
2113 }
2114
2115 /**
2116  * irdma_cm_create_ah - create a cm address handle
2117  * @cm_node: The connection manager node to create AH for
2118  * @wait: Provides option to wait for ah creation or not
2119  */
2120 static int irdma_cm_create_ah(struct irdma_cm_node *cm_node, bool wait)
2121 {
2122     struct irdma_ah_info ah_info = {};
2123     struct irdma_device *iwdev = cm_node->iwdev;
2124
2125     ether_addr_copy(ah_info.mac_addr, iwdev->netdev->dev_addr);
2126
2127     ah_info.hop_ttl = 0x40;
2128     ah_info.tc_tos = cm_node->tos;
2129     ah_info.vsi = &iwdev->vsi;
2130
2131     if (cm_node->ipv4) {
2132         ah_info.ipv4_valid = true;
2133         ah_info.dest_ip_addr[0] = cm_node->rem_addr[0];
2134         ah_info.src_ip_addr[0] = cm_node->loc_addr[0];
2135         ah_info.do_lpbk = irdma_ipv4_is_lpb(ah_info.src_ip_addr[0],
2136                             ah_info.dest_ip_addr[0]);
2137     } else {
2138         memcpy(ah_info.dest_ip_addr, cm_node->rem_addr,
2139                sizeof(ah_info.dest_ip_addr));
2140         memcpy(ah_info.src_ip_addr, cm_node->loc_addr,
2141                sizeof(ah_info.src_ip_addr));
2142         ah_info.do_lpbk = irdma_ipv6_is_lpb(ah_info.src_ip_addr,
2143                             ah_info.dest_ip_addr);
2144     }
2145
2146     ah_info.vlan_tag = cm_node->vlan_id;
2147     if (cm_node->vlan_id < VLAN_N_VID) {
2148         ah_info.insert_vlan_tag = 1;
2149         ah_info.vlan_tag |= cm_node->user_pri << VLAN_PRIO_SHIFT;
2150     }
2151
2152     ah_info.dst_arpindex =
2153         irdma_arp_table(iwdev->rf, ah_info.dest_ip_addr,
2154                 ah_info.ipv4_valid, NULL, IRDMA_ARP_RESOLVE);
2155
2156     if (irdma_puda_create_ah(&iwdev->rf->sc_dev, &ah_info, wait,
2157                  IRDMA_PUDA_RSRC_TYPE_ILQ, cm_node,
2158                  &cm_node->ah))
2159         return -ENOMEM;
2160
2161     trace_irdma_create_ah(cm_node);
2162     return 0;
2163 }
2164
2165 /**
2166  * irdma_cm_free_ah - free a cm address handle
2167  * @cm_node: The connection manager node to create AH for
2168  */
2169 static void irdma_cm_free_ah(struct irdma_cm_node *cm_node)
2170 {
2171     struct irdma_device *iwdev = cm_node->iwdev;
2172
2173     trace_irdma_cm_free_ah(cm_node);
2174     irdma_puda_free_ah(&iwdev->rf->sc_dev, cm_node->ah);
2175     cm_node->ah = NULL;
2176 }
2177
2178 /**
2179  * irdma_make_cm_node - create a new instance of a cm node
2180  * @cm_core: cm's core
2181  * @iwdev: iwarp device structure
2182  * @cm_info: quad info for connection
2183  * @listener: passive connection's listener
2184  */
2185 static struct irdma_cm_node *
2186 irdma_make_cm_node(struct irdma_cm_core *cm_core, struct irdma_device *iwdev,
2187            struct irdma_cm_info *cm_info,
2188            struct irdma_cm_listener *listener)
2189 {
2190     struct irdma_cm_node *cm_node;
2191     int oldarpindex;
2192     int arpindex;
2193     struct net_device *netdev = iwdev->netdev;
2194
2195     /* create an hte and cm_node for this instance */
2196     cm_node = kzalloc(sizeof(*cm_node), GFP_ATOMIC);
2197     if (!cm_node)
2198         return NULL;
2199
2200     /* set our node specific transport info */
2201     cm_node->ipv4 = cm_info->ipv4;
2202     cm_node->vlan_id = cm_info->vlan_id;
2203     if (cm_node->vlan_id >= VLAN_N_VID && iwdev->dcb_vlan_mode)
2204         cm_node->vlan_id = 0;
2205     cm_node->tos = cm_info->tos;
2206     cm_node->user_pri = cm_info->user_pri;
2207     if (listener) {
2208         if (listener->tos != cm_info->tos)
2209             ibdev_warn(&iwdev->ibdev,
2210                    "application TOS[%d] and remote client TOS[%d] mismatch\n",
2211                    listener->tos, cm_info->tos);
2212         if (iwdev->vsi.dscp_mode) {
2213             cm_node->user_pri = listener->user_pri;
2214         } else {
2215             cm_node->tos = max(listener->tos, cm_info->tos);
2216             cm_node->user_pri = rt_tos2priority(cm_node->tos);
2217         }
2218         ibdev_dbg(&iwdev->ibdev,
2219               "DCB: listener: TOS:[%d] UP:[%d]\n", cm_node->tos,
2220               cm_node->user_pri);
2221         trace_irdma_listener_tos(iwdev, cm_node->tos,
2222                      cm_node->user_pri);
2223     }
2224     memcpy(cm_node->loc_addr, cm_info->loc_addr, sizeof(cm_node->loc_addr));
2225     memcpy(cm_node->rem_addr, cm_info->rem_addr, sizeof(cm_node->rem_addr));
2226     cm_node->loc_port = cm_info->loc_port;
2227     cm_node->rem_port = cm_info->rem_port;
2228
2229     cm_node->mpa_frame_rev = IRDMA_CM_DEFAULT_MPA_VER;
2230     cm_node->send_rdma0_op = SEND_RDMA_READ_ZERO;
2231     cm_node->iwdev = iwdev;
2232     cm_node->dev = &iwdev->rf->sc_dev;
2233
2234     cm_node->ird_size = cm_node->dev->hw_attrs.max_hw_ird;
2235     cm_node->ord_size = cm_node->dev->hw_attrs.max_hw_ord;
2236
2237     cm_node->listener = listener;
2238     cm_node->cm_id = cm_info->cm_id;
2239     ether_addr_copy(cm_node->loc_mac, netdev->dev_addr);
2240     spin_lock_init(&cm_node->retrans_list_lock);
2241     cm_node->ack_rcvd = false;
2242
2243     init_completion(&cm_node->establish_comp);
2244     refcount_set(&cm_node->refcnt, 1);
2245     /* associate our parent CM core */
2246     cm_node->cm_core = cm_core;
2247     cm_node->tcp_cntxt.loc_id = IRDMA_CM_DEFAULT_LOCAL_ID;
2248     cm_node->tcp_cntxt.rcv_wscale = iwdev->rcv_wscale;
2249     cm_node->tcp_cntxt.rcv_wnd = iwdev->rcv_wnd >> cm_node->tcp_cntxt.rcv_wscale;
2250     if (cm_node->ipv4) {
2251         cm_node->tcp_cntxt.loc_seq_num = secure_tcp_seq(htonl(cm_node->loc_addr[0]),
2252                                 htonl(cm_node->rem_addr[0]),
2253                                 htons(cm_node->loc_port),
2254                                 htons(cm_node->rem_port));
2255         cm_node->tcp_cntxt.mss = iwdev->vsi.mtu - IRDMA_MTU_TO_MSS_IPV4;
2256     } else if (IS_ENABLED(CONFIG_IPV6)) {
2257         __be32 loc[4] = {
2258             htonl(cm_node->loc_addr[0]), htonl(cm_node->loc_addr[1]),
2259             htonl(cm_node->loc_addr[2]), htonl(cm_node->loc_addr[3])
2260         };
2261         __be32 rem[4] = {
2262             htonl(cm_node->rem_addr[0]), htonl(cm_node->rem_addr[1]),
2263             htonl(cm_node->rem_addr[2]), htonl(cm_node->rem_addr[3])
2264         };
2265         cm_node->tcp_cntxt.loc_seq_num = secure_tcpv6_seq(loc, rem,
2266                                   htons(cm_node->loc_port),
2267                                   htons(cm_node->rem_port));
2268         cm_node->tcp_cntxt.mss = iwdev->vsi.mtu - IRDMA_MTU_TO_MSS_IPV6;
2269     }
2270
2271     if ((cm_node->ipv4 &&
2272          irdma_ipv4_is_lpb(cm_node->loc_addr[0], cm_node->rem_addr[0])) ||
2273         (!cm_node->ipv4 &&
2274          irdma_ipv6_is_lpb(cm_node->loc_addr, cm_node->rem_addr))) {
2275         cm_node->do_lpb = true;
2276         arpindex = irdma_arp_table(iwdev->rf, cm_node->rem_addr,
2277                        cm_node->ipv4, NULL,
2278                        IRDMA_ARP_RESOLVE);
2279     } else {
2280         oldarpindex = irdma_arp_table(iwdev->rf, cm_node->rem_addr,
2281                           cm_node->ipv4, NULL,
2282                           IRDMA_ARP_RESOLVE);
2283         if (cm_node->ipv4)
2284             arpindex = irdma_addr_resolve_neigh(iwdev,
2285                                 cm_info->loc_addr[0],
2286                                 cm_info->rem_addr[0],
2287                                 oldarpindex);
2288         else if (IS_ENABLED(CONFIG_IPV6))
2289             arpindex = irdma_addr_resolve_neigh_ipv6(iwdev,
2290                                  cm_info->loc_addr,
2291                                  cm_info->rem_addr,
2292                                  oldarpindex);
2293         else
2294             arpindex = -EINVAL;
2295     }
2296
2297     if (arpindex < 0)
2298         goto err;
2299
2300     ether_addr_copy(cm_node->rem_mac,
2301             iwdev->rf->arp_table[arpindex].mac_addr);
2302     irdma_add_hte_node(cm_core, cm_node);
2303     cm_core->stats_nodes_created++;
2304     return cm_node;
2305
2306 err:
2307     kfree(cm_node);
2308
2309     return NULL;
2310 }
2311
2312 static void irdma_destroy_connection(struct irdma_cm_node *cm_node)
2313 {
2314     struct irdma_cm_core *cm_core = cm_node->cm_core;
2315     struct irdma_qp *iwqp;
2316     struct irdma_cm_info nfo;
2317
2318     /* if the node is destroyed before connection was accelerated */
2319     if (!cm_node->accelerated && cm_node->accept_pend) {
2320         ibdev_dbg(&cm_node->iwdev->ibdev,
2321               "CM: node destroyed before established\n");
2322         atomic_dec(&cm_node->listener->pend_accepts_cnt);
2323     }
2324     if (cm_node->close_entry)
2325         irdma_handle_close_entry(cm_node, 0);
2326     if (cm_node->listener) {
2327         irdma_dec_refcnt_listen(cm_core, cm_node->listener, 0, true);
2328     } else {
2329         if (cm_node->apbvt_set) {
2330             irdma_del_apbvt(cm_node->iwdev, cm_node->apbvt_entry);
2331             cm_node->apbvt_set = 0;
2332         }
2333         irdma_get_addr_info(cm_node, &nfo);
2334         if (cm_node->qhash_set) {
2335             nfo.qh_qpid = cm_node->iwdev->vsi.ilq->qp_id;
2336             irdma_manage_qhash(cm_node->iwdev, &nfo,
2337                        IRDMA_QHASH_TYPE_TCP_ESTABLISHED,
2338                        IRDMA_QHASH_MANAGE_TYPE_DELETE, NULL,
2339                        false);
2340             cm_node->qhash_set = 0;
2341         }
2342     }
2343
2344     iwqp = cm_node->iwqp;
2345     if (iwqp) {
2346         cm_node->cm_id->rem_ref(cm_node->cm_id);
2347         cm_node->cm_id = NULL;
2348         iwqp->cm_id = NULL;
2349         irdma_qp_rem_ref(&iwqp->ibqp);
2350         cm_node->iwqp = NULL;
2351     } else if (cm_node->qhash_set) {
2352         irdma_get_addr_info(cm_node, &nfo);
2353         nfo.qh_qpid = cm_node->iwdev->vsi.ilq->qp_id;
2354         irdma_manage_qhash(cm_node->iwdev, &nfo,
2355                    IRDMA_QHASH_TYPE_TCP_ESTABLISHED,
2356                    IRDMA_QHASH_MANAGE_TYPE_DELETE, NULL, false);
2357         cm_node->qhash_set = 0;
2358     }
2359
2360     cm_core->cm_free_ah(cm_node);
2361 }
2362
2363 /**
2364  * irdma_rem_ref_cm_node - destroy an instance of a cm node
2365  * @cm_node: connection's node
2366  */
2367 void irdma_rem_ref_cm_node(struct irdma_cm_node *cm_node)
2368 {
2369     struct irdma_cm_core *cm_core = cm_node->cm_core;
2370     unsigned long flags;
2371
2372     trace_irdma_rem_ref_cm_node(cm_node, 0, __builtin_return_address(0));
2373     spin_lock_irqsave(&cm_core->ht_lock, flags);
2374
2375     if (!refcount_dec_and_test(&cm_node->refcnt)) {
2376         spin_unlock_irqrestore(&cm_core->ht_lock, flags);
2377         return;
2378     }
2379     if (cm_node->iwqp) {
2380         cm_node->iwqp->cm_node = NULL;
2381         cm_node->iwqp->cm_id = NULL;
2382     }
2383     hash_del_rcu(&cm_node->list);
2384     cm_node->cm_core->stats_nodes_destroyed++;
2385
2386     spin_unlock_irqrestore(&cm_core->ht_lock, flags);
2387
2388     irdma_destroy_connection(cm_node);
2389
2390     kfree_rcu(cm_node, rcu_head);
2391 }
2392
2393 /**
2394  * irdma_handle_fin_pkt - FIN packet received
2395  * @cm_node: connection's node
2396  */
2397 static void irdma_handle_fin_pkt(struct irdma_cm_node *cm_node)
2398 {
2399     switch (cm_node->state) {
2400     case IRDMA_CM_STATE_SYN_RCVD:
2401     case IRDMA_CM_STATE_SYN_SENT:
2402     case IRDMA_CM_STATE_ESTABLISHED:
2403     case IRDMA_CM_STATE_MPAREJ_RCVD:
2404         cm_node->tcp_cntxt.rcv_nxt++;
2405         irdma_cleanup_retrans_entry(cm_node);
2406         cm_node->state = IRDMA_CM_STATE_LAST_ACK;
2407         irdma_send_fin(cm_node);
2408         break;
2409     case IRDMA_CM_STATE_MPAREQ_SENT:
2410         irdma_create_event(cm_node, IRDMA_CM_EVENT_ABORTED);
2411         cm_node->tcp_cntxt.rcv_nxt++;
2412         irdma_cleanup_retrans_entry(cm_node);
2413         cm_node->state = IRDMA_CM_STATE_CLOSED;
2414         refcount_inc(&cm_node->refcnt);
2415         irdma_send_reset(cm_node);
2416         break;
2417     case IRDMA_CM_STATE_FIN_WAIT1:
2418         cm_node->tcp_cntxt.rcv_nxt++;
2419         irdma_cleanup_retrans_entry(cm_node);
2420         cm_node->state = IRDMA_CM_STATE_CLOSING;
2421         irdma_send_ack(cm_node);
2422         /*
2423          * Wait for ACK as this is simultaneous close.
2424          * After we receive ACK, do not send anything.
2425          * Just rm the node.
2426          */
2427         break;
2428     case IRDMA_CM_STATE_FIN_WAIT2:
2429         cm_node->tcp_cntxt.rcv_nxt++;
2430         irdma_cleanup_retrans_entry(cm_node);
2431         cm_node->state = IRDMA_CM_STATE_TIME_WAIT;
2432         irdma_send_ack(cm_node);
2433         irdma_schedule_cm_timer(cm_node, NULL, IRDMA_TIMER_TYPE_CLOSE,
2434                     1, 0);
2435         break;
2436     case IRDMA_CM_STATE_TIME_WAIT:
2437         cm_node->tcp_cntxt.rcv_nxt++;
2438         irdma_cleanup_retrans_entry(cm_node);
2439         cm_node->state = IRDMA_CM_STATE_CLOSED;
2440         irdma_rem_ref_cm_node(cm_node);
2441         break;
2442     case IRDMA_CM_STATE_OFFLOADED:
2443     default:
2444         ibdev_dbg(&cm_node->iwdev->ibdev,
2445               "CM: bad state node state = %d\n", cm_node->state);
2446         break;
2447     }
2448 }
2449
2450 /**
2451  * irdma_handle_rst_pkt - process received RST packet
2452  * @cm_node: connection's node
2453  * @rbuf: receive buffer
2454  */
2455 static void irdma_handle_rst_pkt(struct irdma_cm_node *cm_node,
2456                  struct irdma_puda_buf *rbuf)
2457 {
2458     ibdev_dbg(&cm_node->iwdev->ibdev,
2459           "CM: caller: %pS cm_node=%p state=%d rem_port=0x%04x loc_port=0x%04x rem_addr=%pI4 loc_addr=%pI4\n",
2460           __builtin_return_address(0), cm_node, cm_node->state,
2461           cm_node->rem_port, cm_node->loc_port, cm_node->rem_addr,
2462           cm_node->loc_addr);
2463
2464     irdma_cleanup_retrans_entry(cm_node);
2465     switch (cm_node->state) {
2466     case IRDMA_CM_STATE_SYN_SENT:
2467     case IRDMA_CM_STATE_MPAREQ_SENT:
2468         switch (cm_node->mpa_frame_rev) {
2469         case IETF_MPA_V2:
2470             /* Drop down to MPA_V1*/
2471             cm_node->mpa_frame_rev = IETF_MPA_V1;
2472             /* send a syn and goto syn sent state */
2473             cm_node->state = IRDMA_CM_STATE_SYN_SENT;
2474             if (irdma_send_syn(cm_node, 0))
2475                 irdma_active_open_err(cm_node, false);
2476             break;
2477         case IETF_MPA_V1:
2478         default:
2479             irdma_active_open_err(cm_node, false);
2480             break;
2481         }
2482         break;
2483     case IRDMA_CM_STATE_MPAREQ_RCVD:
2484         atomic_inc(&cm_node->passive_state);
2485         break;
2486     case IRDMA_CM_STATE_ESTABLISHED:
2487     case IRDMA_CM_STATE_SYN_RCVD:
2488     case IRDMA_CM_STATE_LISTENING:
2489         irdma_passive_open_err(cm_node, false);
2490         break;
2491     case IRDMA_CM_STATE_OFFLOADED:
2492         irdma_active_open_err(cm_node, false);
2493         break;
2494     case IRDMA_CM_STATE_CLOSED:
2495         break;
2496     case IRDMA_CM_STATE_FIN_WAIT2:
2497     case IRDMA_CM_STATE_FIN_WAIT1:
2498     case IRDMA_CM_STATE_LAST_ACK:
2499     case IRDMA_CM_STATE_TIME_WAIT:
2500         cm_node->state = IRDMA_CM_STATE_CLOSED;
2501         irdma_rem_ref_cm_node(cm_node);
2502         break;
2503     default:
2504         break;
2505     }
2506 }
2507
2508 /**
2509  * irdma_handle_rcv_mpa - Process a recv'd mpa buffer
2510  * @cm_node: connection's node
2511  * @rbuf: receive buffer
2512  */
2513 static void irdma_handle_rcv_mpa(struct irdma_cm_node *cm_node,
2514                  struct irdma_puda_buf *rbuf)
2515 {
2516     int err;
2517     int datasize = rbuf->datalen;
2518     u8 *dataloc = rbuf->data;
2519
2520     enum irdma_cm_event_type type = IRDMA_CM_EVENT_UNKNOWN;
2521     u32 res_type;
2522
2523     err = irdma_parse_mpa(cm_node, dataloc, &res_type, datasize);
2524     if (err) {
2525         if (cm_node->state == IRDMA_CM_STATE_MPAREQ_SENT)
2526             irdma_active_open_err(cm_node, true);
2527         else
2528             irdma_passive_open_err(cm_node, true);
2529         return;
2530     }
2531
2532     switch (cm_node->state) {
2533     case IRDMA_CM_STATE_ESTABLISHED:
2534         if (res_type == IRDMA_MPA_REQUEST_REJECT)
2535             ibdev_dbg(&cm_node->iwdev->ibdev,
2536                   "CM: state for reject\n");
2537         cm_node->state = IRDMA_CM_STATE_MPAREQ_RCVD;
2538         type = IRDMA_CM_EVENT_MPA_REQ;
2539         irdma_send_ack(cm_node); /* ACK received MPA request */
2540         atomic_set(&cm_node->passive_state,
2541                IRDMA_PASSIVE_STATE_INDICATED);
2542         break;
2543     case IRDMA_CM_STATE_MPAREQ_SENT:
2544         irdma_cleanup_retrans_entry(cm_node);
2545         if (res_type == IRDMA_MPA_REQUEST_REJECT) {
2546             type = IRDMA_CM_EVENT_MPA_REJECT;
2547             cm_node->state = IRDMA_CM_STATE_MPAREJ_RCVD;
2548         } else {
2549             type = IRDMA_CM_EVENT_CONNECTED;
2550             cm_node->state = IRDMA_CM_STATE_OFFLOADED;
2551         }
2552         irdma_send_ack(cm_node);
2553         break;
2554     default:
2555         ibdev_dbg(&cm_node->iwdev->ibdev,
2556               "CM: wrong cm_node state =%d\n", cm_node->state);
2557         break;
2558     }
2559     irdma_create_event(cm_node, type);
2560 }
2561
2562 /**
2563  * irdma_check_syn - Check for error on received syn ack
2564  * @cm_node: connection's node
2565  * @tcph: pointer tcp header
2566  */
2567 static int irdma_check_syn(struct irdma_cm_node *cm_node, struct tcphdr *tcph)
2568 {
2569     if (ntohl(tcph->ack_seq) != cm_node->tcp_cntxt.loc_seq_num) {
2570         irdma_active_open_err(cm_node, true);
2571         return 1;
2572     }
2573
2574     return 0;
2575 }
2576
2577 /**
2578  * irdma_check_seq - check seq numbers if OK
2579  * @cm_node: connection's node
2580  * @tcph: pointer tcp header
2581  */
2582 static int irdma_check_seq(struct irdma_cm_node *cm_node, struct tcphdr *tcph)
2583 {
2584     u32 seq;
2585     u32 ack_seq;
2586     u32 loc_seq_num = cm_node->tcp_cntxt.loc_seq_num;
2587     u32 rcv_nxt = cm_node->tcp_cntxt.rcv_nxt;
2588     u32 rcv_wnd;
2589     int err = 0;
2590
2591     seq = ntohl(tcph->seq);
2592     ack_seq = ntohl(tcph->ack_seq);
2593     rcv_wnd = cm_node->tcp_cntxt.rcv_wnd;
2594     if (ack_seq != loc_seq_num ||
2595         !between(seq, rcv_nxt, (rcv_nxt + rcv_wnd)))
2596         err = -1;
2597     if (err)
2598         ibdev_dbg(&cm_node->iwdev->ibdev,
2599               "CM: seq number err\n");
2600
2601     return err;
2602 }
2603
2604 void irdma_add_conn_est_qh(struct irdma_cm_node *cm_node)
2605 {
2606     struct irdma_cm_info nfo;
2607
2608     irdma_get_addr_info(cm_node, &nfo);
2609     nfo.qh_qpid = cm_node->iwdev->vsi.ilq->qp_id;
2610     irdma_manage_qhash(cm_node->iwdev, &nfo,
2611                IRDMA_QHASH_TYPE_TCP_ESTABLISHED,
2612                IRDMA_QHASH_MANAGE_TYPE_ADD,
2613                cm_node, false);
2614     cm_node->qhash_set = true;
2615 }
2616
2617 /**
2618  * irdma_handle_syn_pkt - is for Passive node
2619  * @cm_node: connection's node
2620  * @rbuf: receive buffer
2621  */
2622 static void irdma_handle_syn_pkt(struct irdma_cm_node *cm_node,
2623                  struct irdma_puda_buf *rbuf)
2624 {
2625     struct tcphdr *tcph = (struct tcphdr *)rbuf->tcph;
2626     int err;
2627     u32 inc_sequence;
2628     int optionsize;
2629
2630     optionsize = (tcph->doff << 2) - sizeof(struct tcphdr);
2631     inc_sequence = ntohl(tcph->seq);
2632
2633     switch (cm_node->state) {
2634     case IRDMA_CM_STATE_SYN_SENT:
2635     case IRDMA_CM_STATE_MPAREQ_SENT:
2636         /* Rcvd syn on active open connection */
2637         irdma_active_open_err(cm_node, 1);
2638         break;
2639     case IRDMA_CM_STATE_LISTENING:
2640         /* Passive OPEN */
2641         if (atomic_read(&cm_node->listener->pend_accepts_cnt) >
2642             cm_node->listener->backlog) {
2643             cm_node->cm_core->stats_backlog_drops++;
2644             irdma_passive_open_err(cm_node, false);
2645             break;
2646         }
2647         err = irdma_handle_tcp_options(cm_node, tcph, optionsize, 1);
2648         if (err) {
2649             irdma_passive_open_err(cm_node, false);
2650             /* drop pkt */
2651             break;
2652         }
2653         err = cm_node->cm_core->cm_create_ah(cm_node, false);
2654         if (err) {
2655             irdma_passive_open_err(cm_node, false);
2656             /* drop pkt */
2657             break;
2658         }
2659         cm_node->tcp_cntxt.rcv_nxt = inc_sequence + 1;
2660         cm_node->accept_pend = 1;
2661         atomic_inc(&cm_node->listener->pend_accepts_cnt);
2662
2663         cm_node->state = IRDMA_CM_STATE_SYN_RCVD;
2664         break;
2665     case IRDMA_CM_STATE_CLOSED:
2666         irdma_cleanup_retrans_entry(cm_node);
2667         refcount_inc(&cm_node->refcnt);
2668         irdma_send_reset(cm_node);
2669         break;
2670     case IRDMA_CM_STATE_OFFLOADED:
2671     case IRDMA_CM_STATE_ESTABLISHED:
2672     case IRDMA_CM_STATE_FIN_WAIT1:
2673     case IRDMA_CM_STATE_FIN_WAIT2:
2674     case IRDMA_CM_STATE_MPAREQ_RCVD:
2675     case IRDMA_CM_STATE_LAST_ACK:
2676     case IRDMA_CM_STATE_CLOSING:
2677     case IRDMA_CM_STATE_UNKNOWN:
2678     default:
2679         break;
2680     }
2681 }
2682
2683 /**
2684  * irdma_handle_synack_pkt - Process SYN+ACK packet (active side)
2685  * @cm_node: connection's node
2686  * @rbuf: receive buffer
2687  */
2688 static void irdma_handle_synack_pkt(struct irdma_cm_node *cm_node,
2689                     struct irdma_puda_buf *rbuf)
2690 {
2691     struct tcphdr *tcph = (struct tcphdr *)rbuf->tcph;
2692     int err;
2693     u32 inc_sequence;
2694     int optionsize;
2695
2696     optionsize = (tcph->doff << 2) - sizeof(struct tcphdr);
2697     inc_sequence = ntohl(tcph->seq);
2698     switch (cm_node->state) {
2699     case IRDMA_CM_STATE_SYN_SENT:
2700         irdma_cleanup_retrans_entry(cm_node);
2701         /* active open */
2702         if (irdma_check_syn(cm_node, tcph)) {
2703             ibdev_dbg(&cm_node->iwdev->ibdev,
2704                   "CM: check syn fail\n");
2705             return;
2706         }
2707         cm_node->tcp_cntxt.rem_ack_num = ntohl(tcph->ack_seq);
2708         /* setup options */
2709         err = irdma_handle_tcp_options(cm_node, tcph, optionsize, 0);
2710         if (err) {
2711             ibdev_dbg(&cm_node->iwdev->ibdev,
2712                   "CM: cm_node=%p tcp_options failed\n",
2713                   cm_node);
2714             break;
2715         }
2716         irdma_cleanup_retrans_entry(cm_node);
2717         cm_node->tcp_cntxt.rcv_nxt = inc_sequence + 1;
2718         irdma_send_ack(cm_node); /* ACK  for the syn_ack */
2719         err = irdma_send_mpa_request(cm_node);
2720         if (err) {
2721             ibdev_dbg(&cm_node->iwdev->ibdev,
2722                   "CM: cm_node=%p irdma_send_mpa_request failed\n",
2723                   cm_node);
2724             break;
2725         }
2726         cm_node->state = IRDMA_CM_STATE_MPAREQ_SENT;
2727         break;
2728     case IRDMA_CM_STATE_MPAREQ_RCVD:
2729         irdma_passive_open_err(cm_node, true);
2730         break;
2731     case IRDMA_CM_STATE_LISTENING:
2732         cm_node->tcp_cntxt.loc_seq_num = ntohl(tcph->ack_seq);
2733         irdma_cleanup_retrans_entry(cm_node);
2734         cm_node->state = IRDMA_CM_STATE_CLOSED;
2735         irdma_send_reset(cm_node);
2736         break;
2737     case IRDMA_CM_STATE_CLOSED:
2738         cm_node->tcp_cntxt.loc_seq_num = ntohl(tcph->ack_seq);
2739         irdma_cleanup_retrans_entry(cm_node);
2740         refcount_inc(&cm_node->refcnt);
2741         irdma_send_reset(cm_node);
2742         break;
2743     case IRDMA_CM_STATE_ESTABLISHED:
2744     case IRDMA_CM_STATE_FIN_WAIT1:
2745     case IRDMA_CM_STATE_FIN_WAIT2:
2746     case IRDMA_CM_STATE_LAST_ACK:
2747     case IRDMA_CM_STATE_OFFLOADED:
2748     case IRDMA_CM_STATE_CLOSING:
2749     case IRDMA_CM_STATE_UNKNOWN:
2750     case IRDMA_CM_STATE_MPAREQ_SENT:
2751     default:
2752         break;
2753     }
2754 }
2755
2756 /**
2757  * irdma_handle_ack_pkt - process packet with ACK
2758  * @cm_node: connection's node
2759  * @rbuf: receive buffer
2760  */
2761 static int irdma_handle_ack_pkt(struct irdma_cm_node *cm_node,
2762                 struct irdma_puda_buf *rbuf)
2763 {
2764     struct tcphdr *tcph = (struct tcphdr *)rbuf->tcph;
2765     u32 inc_sequence;
2766     int ret;
2767     int optionsize;
2768     u32 datasize = rbuf->datalen;
2769
2770     optionsize = (tcph->doff << 2) - sizeof(struct tcphdr);
2771
2772     if (irdma_check_seq(cm_node, tcph))
2773         return -EINVAL;
2774
2775     inc_sequence = ntohl(tcph->seq);
2776     switch (cm_node->state) {
2777     case IRDMA_CM_STATE_SYN_RCVD:
2778         irdma_cleanup_retrans_entry(cm_node);
2779         ret = irdma_handle_tcp_options(cm_node, tcph, optionsize, 1);
2780         if (ret)
2781             return ret;
2782         cm_node->tcp_cntxt.rem_ack_num = ntohl(tcph->ack_seq);
2783         cm_node->state = IRDMA_CM_STATE_ESTABLISHED;
2784         if (datasize) {
2785             cm_node->tcp_cntxt.rcv_nxt = inc_sequence + datasize;
2786             irdma_handle_rcv_mpa(cm_node, rbuf);
2787         }
2788         break;
2789     case IRDMA_CM_STATE_ESTABLISHED:
2790         irdma_cleanup_retrans_entry(cm_node);
2791         if (datasize) {
2792             cm_node->tcp_cntxt.rcv_nxt = inc_sequence + datasize;
2793             irdma_handle_rcv_mpa(cm_node, rbuf);
2794         }
2795         break;
2796     case IRDMA_CM_STATE_MPAREQ_SENT:
2797         cm_node->tcp_cntxt.rem_ack_num = ntohl(tcph->ack_seq);
2798         if (datasize) {
2799             cm_node->tcp_cntxt.rcv_nxt = inc_sequence + datasize;
2800             cm_node->ack_rcvd = false;
2801             irdma_handle_rcv_mpa(cm_node, rbuf);
2802         } else {
2803             cm_node->ack_rcvd = true;
2804         }
2805         break;
2806     case IRDMA_CM_STATE_LISTENING:
2807         irdma_cleanup_retrans_entry(cm_node);
2808         cm_node->state = IRDMA_CM_STATE_CLOSED;
2809         irdma_send_reset(cm_node);
2810         break;
2811     case IRDMA_CM_STATE_CLOSED:
2812         irdma_cleanup_retrans_entry(cm_node);
2813         refcount_inc(&cm_node->refcnt);
2814         irdma_send_reset(cm_node);
2815         break;
2816     case IRDMA_CM_STATE_LAST_ACK:
2817     case IRDMA_CM_STATE_CLOSING:
2818         irdma_cleanup_retrans_entry(cm_node);
2819         cm_node->state = IRDMA_CM_STATE_CLOSED;
2820         irdma_rem_ref_cm_node(cm_node);
2821         break;
2822     case IRDMA_CM_STATE_FIN_WAIT1:
2823         irdma_cleanup_retrans_entry(cm_node);
2824         cm_node->state = IRDMA_CM_STATE_FIN_WAIT2;
2825         break;
2826     case IRDMA_CM_STATE_SYN_SENT:
2827     case IRDMA_CM_STATE_FIN_WAIT2:
2828     case IRDMA_CM_STATE_OFFLOADED:
2829     case IRDMA_CM_STATE_MPAREQ_RCVD:
2830     case IRDMA_CM_STATE_UNKNOWN:
2831     default:
2832         irdma_cleanup_retrans_entry(cm_node);
2833         break;
2834     }
2835
2836     return 0;
2837 }
2838
2839 /**
2840  * irdma_process_pkt - process cm packet
2841  * @cm_node: connection's node
2842  * @rbuf: receive buffer
2843  */
2844 static void irdma_process_pkt(struct irdma_cm_node *cm_node,
2845                   struct irdma_puda_buf *rbuf)
2846 {
2847     enum irdma_tcpip_pkt_type pkt_type = IRDMA_PKT_TYPE_UNKNOWN;
2848     struct tcphdr *tcph = (struct tcphdr *)rbuf->tcph;
2849     u32 fin_set = 0;
2850     int err;
2851
2852     if (tcph->rst) {
2853         pkt_type = IRDMA_PKT_TYPE_RST;
2854     } else if (tcph->syn) {
2855         pkt_type = IRDMA_PKT_TYPE_SYN;
2856         if (tcph->ack)
2857             pkt_type = IRDMA_PKT_TYPE_SYNACK;
2858     } else if (tcph->ack) {
2859         pkt_type = IRDMA_PKT_TYPE_ACK;
2860     }
2861     if (tcph->fin)
2862         fin_set = 1;
2863
2864     switch (pkt_type) {
2865     case IRDMA_PKT_TYPE_SYN:
2866         irdma_handle_syn_pkt(cm_node, rbuf);
2867         break;
2868     case IRDMA_PKT_TYPE_SYNACK:
2869         irdma_handle_synack_pkt(cm_node, rbuf);
2870         break;
2871     case IRDMA_PKT_TYPE_ACK:
2872         err = irdma_handle_ack_pkt(cm_node, rbuf);
2873         if (fin_set && !err)
2874             irdma_handle_fin_pkt(cm_node);
2875         break;
2876     case IRDMA_PKT_TYPE_RST:
2877         irdma_handle_rst_pkt(cm_node, rbuf);
2878         break;
2879     default:
2880         if (fin_set &&
2881             (!irdma_check_seq(cm_node, (struct tcphdr *)rbuf->tcph)))
2882             irdma_handle_fin_pkt(cm_node);
2883         break;
2884     }
2885 }
2886
2887 /**
2888  * irdma_make_listen_node - create a listen node with params
2889  * @cm_core: cm's core
2890  * @iwdev: iwarp device structure
2891  * @cm_info: quad info for connection
2892  */
2893 static struct irdma_cm_listener *
2894 irdma_make_listen_node(struct irdma_cm_core *cm_core,
2895                struct irdma_device *iwdev,
2896                struct irdma_cm_info *cm_info)
2897 {
2898     struct irdma_cm_listener *listener;
2899     unsigned long flags;
2900
2901     /* cannot have multiple matching listeners */
2902     listener = irdma_find_listener(cm_core, cm_info->loc_addr,
2903                        cm_info->loc_port, cm_info->vlan_id,
2904                        IRDMA_CM_LISTENER_EITHER_STATE);
2905     if (listener &&
2906         listener->listener_state == IRDMA_CM_LISTENER_ACTIVE_STATE) {
2907         refcount_dec(&listener->refcnt);
2908         return NULL;
2909     }
2910
2911     if (!listener) {
2912         /* create a CM listen node
2913          * 1/2 node to compare incoming traffic to
2914          */
2915         listener = kzalloc(sizeof(*listener), GFP_KERNEL);
2916         if (!listener)
2917             return NULL;
2918         cm_core->stats_listen_nodes_created++;
2919         memcpy(listener->loc_addr, cm_info->loc_addr,
2920                sizeof(listener->loc_addr));
2921         listener->loc_port = cm_info->loc_port;
2922
2923         INIT_LIST_HEAD(&listener->child_listen_list);
2924
2925         refcount_set(&listener->refcnt, 1);
2926     } else {
2927         listener->reused_node = 1;
2928     }
2929
2930     listener->cm_id = cm_info->cm_id;
2931     listener->ipv4 = cm_info->ipv4;
2932     listener->vlan_id = cm_info->vlan_id;
2933     atomic_set(&listener->pend_accepts_cnt, 0);
2934     listener->cm_core = cm_core;
2935     listener->iwdev = iwdev;
2936
2937     listener->backlog = cm_info->backlog;
2938     listener->listener_state = IRDMA_CM_LISTENER_ACTIVE_STATE;
2939
2940     if (!listener->reused_node) {
2941         spin_lock_irqsave(&cm_core->listen_list_lock, flags);
2942         list_add(&listener->list, &cm_core->listen_list);
2943         spin_unlock_irqrestore(&cm_core->listen_list_lock, flags);
2944     }
2945
2946     return listener;
2947 }
2948
2949 /**
2950  * irdma_create_cm_node - make a connection node with params
2951  * @cm_core: cm's core
2952  * @iwdev: iwarp device structure
2953  * @conn_param: connection parameters
2954  * @cm_info: quad info for connection
2955  * @caller_cm_node: pointer to cm_node structure to return
2956  */
2957 static int irdma_create_cm_node(struct irdma_cm_core *cm_core,
2958                 struct irdma_device *iwdev,
2959                 struct iw_cm_conn_param *conn_param,
2960                 struct irdma_cm_info *cm_info,
2961                 struct irdma_cm_node **caller_cm_node)
2962 {
2963     struct irdma_cm_node *cm_node;
2964     u16 private_data_len = conn_param->private_data_len;
2965     const void *private_data = conn_param->private_data;
2966
2967     /* create a CM connection node */
2968     cm_node = irdma_make_cm_node(cm_core, iwdev, cm_info, NULL);
2969     if (!cm_node)
2970         return -ENOMEM;
2971
2972     /* set our node side to client (active) side */
2973     cm_node->tcp_cntxt.client = 1;
2974     cm_node->tcp_cntxt.rcv_wscale = IRDMA_CM_DEFAULT_RCV_WND_SCALE;
2975
2976     irdma_record_ird_ord(cm_node, conn_param->ird, conn_param->ord);
2977
2978     cm_node->pdata.size = private_data_len;
2979     cm_node->pdata.addr = cm_node->pdata_buf;
2980
2981     memcpy(cm_node->pdata_buf, private_data, private_data_len);
2982     *caller_cm_node = cm_node;
2983
2984     return 0;
2985 }
2986
2987 /**
2988  * irdma_cm_reject - reject and teardown a connection
2989  * @cm_node: connection's node
2990  * @pdata: ptr to private data for reject
2991  * @plen: size of private data
2992  */
2993 static int irdma_cm_reject(struct irdma_cm_node *cm_node, const void *pdata,
2994                u8 plen)
2995 {
2996     int ret;
2997     int passive_state;
2998
2999     if (cm_node->tcp_cntxt.client)
3000         return 0;
3001
3002     irdma_cleanup_retrans_entry(cm_node);
3003
3004     passive_state = atomic_add_return(1, &cm_node->passive_state);
3005     if (passive_state == IRDMA_SEND_RESET_EVENT) {
3006         cm_node->state = IRDMA_CM_STATE_CLOSED;
3007         irdma_rem_ref_cm_node(cm_node);
3008         return 0;
3009     }
3010
3011     if (cm_node->state == IRDMA_CM_STATE_LISTENER_DESTROYED) {
3012         irdma_rem_ref_cm_node(cm_node);
3013         return 0;
3014     }
3015
3016     ret = irdma_send_mpa_reject(cm_node, pdata, plen);
3017     if (!ret)
3018         return 0;
3019
3020     cm_node->state = IRDMA_CM_STATE_CLOSED;
3021     if (irdma_send_reset(cm_node))
3022         ibdev_dbg(&cm_node->iwdev->ibdev,
3023               "CM: send reset failed\n");
3024
3025     return ret;
3026 }
3027
3028 /**
3029  * irdma_cm_close - close of cm connection
3030  * @cm_node: connection's node
3031  */
3032 static int irdma_cm_close(struct irdma_cm_node *cm_node)
3033 {
3034     switch (cm_node->state) {
3035     case IRDMA_CM_STATE_SYN_RCVD:
3036     case IRDMA_CM_STATE_SYN_SENT:
3037     case IRDMA_CM_STATE_ONE_SIDE_ESTABLISHED:
3038     case IRDMA_CM_STATE_ESTABLISHED:
3039     case IRDMA_CM_STATE_ACCEPTING:
3040     case IRDMA_CM_STATE_MPAREQ_SENT:
3041     case IRDMA_CM_STATE_MPAREQ_RCVD:
3042         irdma_cleanup_retrans_entry(cm_node);
3043         irdma_send_reset(cm_node);
3044         break;
3045     case IRDMA_CM_STATE_CLOSE_WAIT:
3046         cm_node->state = IRDMA_CM_STATE_LAST_ACK;
3047         irdma_send_fin(cm_node);
3048         break;
3049     case IRDMA_CM_STATE_FIN_WAIT1:
3050     case IRDMA_CM_STATE_FIN_WAIT2:
3051     case IRDMA_CM_STATE_LAST_ACK:
3052     case IRDMA_CM_STATE_TIME_WAIT:
3053     case IRDMA_CM_STATE_CLOSING:
3054         return -EINVAL;
3055     case IRDMA_CM_STATE_LISTENING:
3056         irdma_cleanup_retrans_entry(cm_node);
3057         irdma_send_reset(cm_node);
3058         break;
3059     case IRDMA_CM_STATE_MPAREJ_RCVD:
3060     case IRDMA_CM_STATE_UNKNOWN:
3061     case IRDMA_CM_STATE_INITED:
3062     case IRDMA_CM_STATE_CLOSED:
3063     case IRDMA_CM_STATE_LISTENER_DESTROYED:
3064         irdma_rem_ref_cm_node(cm_node);
3065         break;
3066     case IRDMA_CM_STATE_OFFLOADED:
3067         if (cm_node->send_entry)
3068             ibdev_dbg(&cm_node->iwdev->ibdev,
3069                   "CM: CM send_entry in OFFLOADED state\n");
3070         irdma_rem_ref_cm_node(cm_node);
3071         break;
3072     }
3073
3074     return 0;
3075 }
3076
3077 /**
3078  * irdma_receive_ilq - recv an ETHERNET packet, and process it
3079  * through CM
3080  * @vsi: VSI structure of dev
3081  * @rbuf: receive buffer
3082  */
3083 void irdma_receive_ilq(struct irdma_sc_vsi *vsi, struct irdma_puda_buf *rbuf)
3084 {
3085     struct irdma_cm_node *cm_node;
3086     struct irdma_cm_listener *listener;
3087     struct iphdr *iph;
3088     struct ipv6hdr *ip6h;
3089     struct tcphdr *tcph;
3090     struct irdma_cm_info cm_info = {};
3091     struct irdma_device *iwdev = vsi->back_vsi;
3092     struct irdma_cm_core *cm_core = &iwdev->cm_core;
3093     struct vlan_ethhdr *ethh;
3094     u16 vtag;
3095
3096     /* if vlan, then maclen = 18 else 14 */
3097     iph = (struct iphdr *)rbuf->iph;
3098     print_hex_dump_debug("ILQ: RECEIVE ILQ BUFFER", DUMP_PREFIX_OFFSET,
3099                  16, 8, rbuf->mem.va, rbuf->totallen, false);
3100     if (iwdev->rf->sc_dev.hw_attrs.uk_attrs.hw_rev >= IRDMA_GEN_2) {
3101         if (rbuf->vlan_valid) {
3102             vtag = rbuf->vlan_id;
3103             cm_info.user_pri = (vtag & VLAN_PRIO_MASK) >>
3104                        VLAN_PRIO_SHIFT;
3105             cm_info.vlan_id = vtag & VLAN_VID_MASK;
3106         } else {
3107             cm_info.vlan_id = 0xFFFF;
3108         }
3109     } else {
3110         ethh = rbuf->mem.va;
3111
3112         if (ethh->h_vlan_proto == htons(ETH_P_8021Q)) {
3113             vtag = ntohs(ethh->h_vlan_TCI);
3114             cm_info.user_pri = (vtag & VLAN_PRIO_MASK) >>
3115                        VLAN_PRIO_SHIFT;
3116             cm_info.vlan_id = vtag & VLAN_VID_MASK;
3117             ibdev_dbg(&cm_core->iwdev->ibdev,
3118                   "CM: vlan_id=%d\n", cm_info.vlan_id);
3119         } else {
3120             cm_info.vlan_id = 0xFFFF;
3121         }
3122     }
3123     tcph = (struct tcphdr *)rbuf->tcph;
3124
3125     if (rbuf->ipv4) {
3126         cm_info.loc_addr[0] = ntohl(iph->daddr);
3127         cm_info.rem_addr[0] = ntohl(iph->saddr);
3128         cm_info.ipv4 = true;
3129         cm_info.tos = iph->tos;
3130     } else {
3131         ip6h = (struct ipv6hdr *)rbuf->iph;
3132         irdma_copy_ip_ntohl(cm_info.loc_addr,
3133                     ip6h->daddr.in6_u.u6_addr32);
3134         irdma_copy_ip_ntohl(cm_info.rem_addr,
3135                     ip6h->saddr.in6_u.u6_addr32);
3136         cm_info.ipv4 = false;
3137         cm_info.tos = (ip6h->priority << 4) | (ip6h->flow_lbl[0] >> 4);
3138     }
3139     cm_info.loc_port = ntohs(tcph->dest);
3140     cm_info.rem_port = ntohs(tcph->source);
3141     cm_node = irdma_find_node(cm_core, cm_info.rem_port, cm_info.rem_addr,
3142                   cm_info.loc_port, cm_info.loc_addr, cm_info.vlan_id);
3143
3144     if (!cm_node) {
3145         /* Only type of packet accepted are for the
3146          * PASSIVE open (syn only)
3147          */
3148         if (!tcph->syn || tcph->ack)
3149             return;
3150
3151         listener = irdma_find_listener(cm_core,
3152                            cm_info.loc_addr,
3153                            cm_info.loc_port,
3154                            cm_info.vlan_id,
3155                            IRDMA_CM_LISTENER_ACTIVE_STATE);
3156         if (!listener) {
3157             cm_info.cm_id = NULL;
3158             ibdev_dbg(&cm_core->iwdev->ibdev,
3159                   "CM: no listener found\n");
3160             return;
3161         }
3162
3163         cm_info.cm_id = listener->cm_id;
3164         cm_node = irdma_make_cm_node(cm_core, iwdev, &cm_info,
3165                          listener);
3166         if (!cm_node) {
3167             ibdev_dbg(&cm_core->iwdev->ibdev,
3168                   "CM: allocate node failed\n");
3169             refcount_dec(&listener->refcnt);
3170             return;
3171         }
3172
3173         if (!tcph->rst && !tcph->fin) {
3174             cm_node->state = IRDMA_CM_STATE_LISTENING;
3175         } else {
3176             irdma_rem_ref_cm_node(cm_node);
3177             return;
3178         }
3179
3180         refcount_inc(&cm_node->refcnt);
3181     } else if (cm_node->state == IRDMA_CM_STATE_OFFLOADED) {
3182         irdma_rem_ref_cm_node(cm_node);
3183         return;
3184     }
3185
3186     irdma_process_pkt(cm_node, rbuf);
3187     irdma_rem_ref_cm_node(cm_node);
3188 }
3189
3190 static int irdma_add_qh(struct irdma_cm_node *cm_node, bool active)
3191 {
3192     if (!active)
3193         irdma_add_conn_est_qh(cm_node);
3194     return 0;
3195 }
3196
3197 static void irdma_cm_free_ah_nop(struct irdma_cm_node *cm_node)
3198 {
3199 }
3200
3201 /**
3202  * irdma_setup_cm_core - setup top level instance of a cm core
3203  * @iwdev: iwarp device structure
3204  * @rdma_ver: HW version
3205  */
3206 int irdma_setup_cm_core(struct irdma_device *iwdev, u8 rdma_ver)
3207 {
3208     struct irdma_cm_core *cm_core = &iwdev->cm_core;
3209
3210     cm_core->iwdev = iwdev;
3211     cm_core->dev = &iwdev->rf->sc_dev;
3212
3213     /* Handles CM event work items send to Iwarp core */
3214     cm_core->event_wq = alloc_ordered_workqueue("iwarp-event-wq", 0);
3215     if (!cm_core->event_wq)
3216         return -ENOMEM;
3217
3218     INIT_LIST_HEAD(&cm_core->listen_list);
3219
3220     timer_setup(&cm_core->tcp_timer, irdma_cm_timer_tick, 0);
3221
3222     spin_lock_init(&cm_core->ht_lock);
3223     spin_lock_init(&cm_core->listen_list_lock);
3224     spin_lock_init(&cm_core->apbvt_lock);
3225     switch (rdma_ver) {
3226     case IRDMA_GEN_1:
3227         cm_core->form_cm_frame = irdma_form_uda_cm_frame;
3228         cm_core->cm_create_ah = irdma_add_qh;
3229         cm_core->cm_free_ah = irdma_cm_free_ah_nop;
3230         break;
3231     case IRDMA_GEN_2:
3232     default:
3233         cm_core->form_cm_frame = irdma_form_ah_cm_frame;
3234         cm_core->cm_create_ah = irdma_cm_create_ah;
3235         cm_core->cm_free_ah = irdma_cm_free_ah;
3236     }
3237
3238     return 0;
3239 }
3240
3241 /**
3242  * irdma_cleanup_cm_core - deallocate a top level instance of a
3243  * cm core
3244  * @cm_core: cm's core
3245  */
3246 void irdma_cleanup_cm_core(struct irdma_cm_core *cm_core)
3247 {
3248     if (!cm_core)
3249         return;
3250
3251     del_timer_sync(&cm_core->tcp_timer);
3252
3253     destroy_workqueue(cm_core->event_wq);
3254     cm_core->dev->ws_reset(&cm_core->iwdev->vsi);
3255 }
3256
3257 /**
3258  * irdma_init_tcp_ctx - setup qp context
3259  * @cm_node: connection's node
3260  * @tcp_info: offload info for tcp
3261  * @iwqp: associate qp for the connection
3262  */
3263 static void irdma_init_tcp_ctx(struct irdma_cm_node *cm_node,
3264                    struct irdma_tcp_offload_info *tcp_info,
3265                    struct irdma_qp *iwqp)
3266 {
3267     tcp_info->ipv4 = cm_node->ipv4;
3268     tcp_info->drop_ooo_seg = !iwqp->iwdev->iw_ooo;
3269     tcp_info->wscale = true;
3270     tcp_info->ignore_tcp_opt = true;
3271     tcp_info->ignore_tcp_uns_opt = true;
3272     tcp_info->no_nagle = false;
3273
3274     tcp_info->ttl = IRDMA_DEFAULT_TTL;
3275     tcp_info->rtt_var = IRDMA_DEFAULT_RTT_VAR;
3276     tcp_info->ss_thresh = IRDMA_DEFAULT_SS_THRESH;
3277     tcp_info->rexmit_thresh = IRDMA_DEFAULT_REXMIT_THRESH;
3278
3279     tcp_info->tcp_state = IRDMA_TCP_STATE_ESTABLISHED;
3280     tcp_info->snd_wscale = cm_node->tcp_cntxt.snd_wscale;
3281     tcp_info->rcv_wscale = cm_node->tcp_cntxt.rcv_wscale;
3282
3283     tcp_info->snd_nxt = cm_node->tcp_cntxt.loc_seq_num;
3284     tcp_info->snd_wnd = cm_node->tcp_cntxt.snd_wnd;
3285     tcp_info->rcv_nxt = cm_node->tcp_cntxt.rcv_nxt;
3286     tcp_info->snd_max = cm_node->tcp_cntxt.loc_seq_num;
3287
3288     tcp_info->snd_una = cm_node->tcp_cntxt.loc_seq_num;
3289     tcp_info->cwnd = 2 * cm_node->tcp_cntxt.mss;
3290     tcp_info->snd_wl1 = cm_node->tcp_cntxt.rcv_nxt;
3291     tcp_info->snd_wl2 = cm_node->tcp_cntxt.loc_seq_num;
3292     tcp_info->max_snd_window = cm_node->tcp_cntxt.max_snd_wnd;
3293     tcp_info->rcv_wnd = cm_node->tcp_cntxt.rcv_wnd
3294                 << cm_node->tcp_cntxt.rcv_wscale;
3295
3296     tcp_info->flow_label = 0;
3297     tcp_info->snd_mss = (u32)cm_node->tcp_cntxt.mss;
3298     tcp_info->tos = cm_node->tos;
3299     if (cm_node->vlan_id < VLAN_N_VID) {
3300         tcp_info->insert_vlan_tag = true;
3301         tcp_info->vlan_tag = cm_node->vlan_id;
3302         tcp_info->vlan_tag |= cm_node->user_pri << VLAN_PRIO_SHIFT;
3303     }
3304     if (cm_node->ipv4) {
3305         tcp_info->src_port = cm_node->loc_port;
3306         tcp_info->dst_port = cm_node->rem_port;
3307
3308         tcp_info->dest_ip_addr[3] = cm_node->rem_addr[0];
3309         tcp_info->local_ipaddr[3] = cm_node->loc_addr[0];
3310         tcp_info->arp_idx = (u16)irdma_arp_table(iwqp->iwdev->rf,
3311                              &tcp_info->dest_ip_addr[3],
3312                              true, NULL,
3313                              IRDMA_ARP_RESOLVE);
3314     } else {
3315         tcp_info->src_port = cm_node->loc_port;
3316         tcp_info->dst_port = cm_node->rem_port;
3317         memcpy(tcp_info->dest_ip_addr, cm_node->rem_addr,
3318                sizeof(tcp_info->dest_ip_addr));
3319         memcpy(tcp_info->local_ipaddr, cm_node->loc_addr,
3320                sizeof(tcp_info->local_ipaddr));
3321
3322         tcp_info->arp_idx = (u16)irdma_arp_table(iwqp->iwdev->rf,
3323                              &tcp_info->dest_ip_addr[0],
3324                              false, NULL,
3325                              IRDMA_ARP_RESOLVE);
3326     }
3327 }
3328
3329 /**
3330  * irdma_cm_init_tsa_conn - setup qp for RTS
3331  * @iwqp: associate qp for the connection
3332  * @cm_node: connection's node
3333  */
3334 static void irdma_cm_init_tsa_conn(struct irdma_qp *iwqp,
3335                    struct irdma_cm_node *cm_node)
3336 {
3337     struct irdma_iwarp_offload_info *iwarp_info;
3338     struct irdma_qp_host_ctx_info *ctx_info;
3339
3340     iwarp_info = &iwqp->iwarp_info;
3341     ctx_info = &iwqp->ctx_info;
3342
3343     ctx_info->tcp_info = &iwqp->tcp_info;
3344     ctx_info->send_cq_num = iwqp->iwscq->sc_cq.cq_uk.cq_id;
3345     ctx_info->rcv_cq_num = iwqp->iwrcq->sc_cq.cq_uk.cq_id;
3346
3347     iwarp_info->ord_size = cm_node->ord_size;
3348     iwarp_info->ird_size = cm_node->ird_size;
3349     iwarp_info->rd_en = true;
3350     iwarp_info->rdmap_ver = 1;
3351     iwarp_info->ddp_ver = 1;
3352     iwarp_info->pd_id = iwqp->iwpd->sc_pd.pd_id;
3353
3354     ctx_info->tcp_info_valid = true;
3355     ctx_info->iwarp_info_valid = true;
3356     ctx_info->user_pri = cm_node->user_pri;
3357
3358     irdma_init_tcp_ctx(cm_node, &iwqp->tcp_info, iwqp);
3359     if (cm_node->snd_mark_en) {
3360         iwarp_info->snd_mark_en = true;
3361         iwarp_info->snd_mark_offset = (iwqp->tcp_info.snd_nxt & SNDMARKER_SEQNMASK) +
3362                            cm_node->lsmm_size;
3363     }
3364
3365     cm_node->state = IRDMA_CM_STATE_OFFLOADED;
3366     iwqp->tcp_info.tcp_state = IRDMA_TCP_STATE_ESTABLISHED;
3367     iwqp->tcp_info.src_mac_addr_idx = iwqp->iwdev->mac_ip_table_idx;
3368
3369     if (cm_node->rcv_mark_en) {
3370         iwarp_info->rcv_mark_en = true;
3371         iwarp_info->align_hdrs = true;
3372     }
3373
3374     irdma_sc_qp_setctx(&iwqp->sc_qp, iwqp->host_ctx.va, ctx_info);
3375
3376     /* once tcp_info is set, no need to do it again */
3377     ctx_info->tcp_info_valid = false;
3378     ctx_info->iwarp_info_valid = false;
3379 }
3380
3381 /**
3382  * irdma_cm_disconn - when a connection is being closed
3383  * @iwqp: associated qp for the connection
3384  */
3385 void irdma_cm_disconn(struct irdma_qp *iwqp)
3386 {
3387     struct irdma_device *iwdev = iwqp->iwdev;
3388     struct disconn_work *work;
3389     unsigned long flags;
3390
3391     work = kzalloc(sizeof(*work), GFP_ATOMIC);
3392     if (!work)
3393         return;
3394
3395     spin_lock_irqsave(&iwdev->rf->qptable_lock, flags);
3396     if (!iwdev->rf->qp_table[iwqp->ibqp.qp_num]) {
3397         spin_unlock_irqrestore(&iwdev->rf->qptable_lock, flags);
3398         ibdev_dbg(&iwdev->ibdev,
3399               "CM: qp_id %d is already freed\n",
3400               iwqp->ibqp.qp_num);
3401         kfree(work);
3402         return;
3403     }
3404     irdma_qp_add_ref(&iwqp->ibqp);
3405     spin_unlock_irqrestore(&iwdev->rf->qptable_lock, flags);
3406
3407     work->iwqp = iwqp;
3408     INIT_WORK(&work->work, irdma_disconnect_worker);
3409     queue_work(iwdev->cleanup_wq, &work->work);
3410 }
3411
3412 /**
3413  * irdma_qp_disconnect - free qp and close cm
3414  * @iwqp: associate qp for the connection
3415  */
3416 static void irdma_qp_disconnect(struct irdma_qp *iwqp)
3417 {
3418     struct irdma_device *iwdev = iwqp->iwdev;
3419
3420     iwqp->active_conn = 0;
3421     /* close the CM node down if it is still active */
3422     ibdev_dbg(&iwdev->ibdev, "CM: Call close API\n");
3423     irdma_cm_close(iwqp->cm_node);
3424 }
3425
3426 /**
3427  * irdma_cm_disconn_true - called by worker thread to disconnect qp
3428  * @iwqp: associate qp for the connection
3429  */
3430 static void irdma_cm_disconn_true(struct irdma_qp *iwqp)
3431 {
3432     struct iw_cm_id *cm_id;
3433     struct irdma_device *iwdev;
3434     struct irdma_sc_qp *qp = &iwqp->sc_qp;
3435     u16 last_ae;
3436     u8 original_hw_tcp_state;
3437     u8 original_ibqp_state;
3438     int disconn_status = 0;
3439     int issue_disconn = 0;
3440     int issue_close = 0;
3441     int issue_flush = 0;
3442     unsigned long flags;
3443     int err;
3444
3445     iwdev = iwqp->iwdev;
3446     spin_lock_irqsave(&iwqp->lock, flags);
3447     if (rdma_protocol_roce(&iwdev->ibdev, 1)) {
3448         struct ib_qp_attr attr;
3449
3450         if (iwqp->flush_issued || iwqp->sc_qp.qp_uk.destroy_pending) {
3451             spin_unlock_irqrestore(&iwqp->lock, flags);
3452             return;
3453         }
3454
3455         spin_unlock_irqrestore(&iwqp->lock, flags);
3456
3457         attr.qp_state = IB_QPS_ERR;
3458         irdma_modify_qp_roce(&iwqp->ibqp, &attr, IB_QP_STATE, NULL);
3459         irdma_ib_qp_event(iwqp, qp->event_type);
3460         return;
3461     }
3462
3463     cm_id = iwqp->cm_id;
3464     original_hw_tcp_state = iwqp->hw_tcp_state;
3465     original_ibqp_state = iwqp->ibqp_state;
3466     last_ae = iwqp->last_aeq;
3467
3468     if (qp->term_flags) {
3469         issue_disconn = 1;
3470         issue_close = 1;
3471         iwqp->cm_id = NULL;
3472         irdma_terminate_del_timer(qp);
3473         if (!iwqp->flush_issued) {
3474             iwqp->flush_issued = 1;
3475             issue_flush = 1;
3476         }
3477     } else if ((original_hw_tcp_state == IRDMA_TCP_STATE_CLOSE_WAIT) ||
3478            ((original_ibqp_state == IB_QPS_RTS) &&
3479             (last_ae == IRDMA_AE_LLP_CONNECTION_RESET))) {
3480         issue_disconn = 1;
3481         if (last_ae == IRDMA_AE_LLP_CONNECTION_RESET)
3482             disconn_status = -ECONNRESET;
3483     }
3484
3485     if (original_hw_tcp_state == IRDMA_TCP_STATE_CLOSED ||
3486         original_hw_tcp_state == IRDMA_TCP_STATE_TIME_WAIT ||
3487         last_ae == IRDMA_AE_RDMAP_ROE_BAD_LLP_CLOSE ||
3488         last_ae == IRDMA_AE_BAD_CLOSE ||
3489         last_ae == IRDMA_AE_LLP_CONNECTION_RESET || iwdev->rf->reset || !cm_id) {
3490         issue_close = 1;
3491         iwqp->cm_id = NULL;
3492         qp->term_flags = 0;
3493         if (!iwqp->flush_issued) {
3494             iwqp->flush_issued = 1;
3495             issue_flush = 1;
3496         }
3497     }
3498
3499     spin_unlock_irqrestore(&iwqp->lock, flags);
3500     if (issue_flush && !iwqp->sc_qp.qp_uk.destroy_pending) {
3501         irdma_flush_wqes(iwqp, IRDMA_FLUSH_SQ | IRDMA_FLUSH_RQ |
3502                  IRDMA_FLUSH_WAIT);
3503
3504         if (qp->term_flags)
3505             irdma_ib_qp_event(iwqp, qp->event_type);
3506     }
3507
3508     if (!cm_id || !cm_id->event_handler)
3509         return;
3510
3511     spin_lock_irqsave(&iwdev->cm_core.ht_lock, flags);
3512     if (!iwqp->cm_node) {
3513         spin_unlock_irqrestore(&iwdev->cm_core.ht_lock, flags);
3514         return;
3515     }
3516     refcount_inc(&iwqp->cm_node->refcnt);
3517
3518     spin_unlock_irqrestore(&iwdev->cm_core.ht_lock, flags);
3519
3520     if (issue_disconn) {
3521         err = irdma_send_cm_event(iwqp->cm_node, cm_id,
3522                       IW_CM_EVENT_DISCONNECT,
3523                       disconn_status);
3524         if (err)
3525             ibdev_dbg(&iwdev->ibdev,
3526                   "CM: disconnect event failed: - cm_id = %p\n",
3527                   cm_id);
3528     }
3529     if (issue_close) {
3530         cm_id->provider_data = iwqp;
3531         err = irdma_send_cm_event(iwqp->cm_node, cm_id,
3532                       IW_CM_EVENT_CLOSE, 0);
3533         if (err)
3534             ibdev_dbg(&iwdev->ibdev,
3535                   "CM: close event failed: - cm_id = %p\n",
3536                   cm_id);
3537         irdma_qp_disconnect(iwqp);
3538     }
3539     irdma_rem_ref_cm_node(iwqp->cm_node);
3540 }
3541
3542 /**
3543  * irdma_disconnect_worker - worker for connection close
3544  * @work: points or disconn structure
3545  */
3546 static void irdma_disconnect_worker(struct work_struct *work)
3547 {
3548     struct disconn_work *dwork = container_of(work, struct disconn_work, work);
3549     struct irdma_qp *iwqp = dwork->iwqp;
3550
3551     kfree(dwork);
3552     irdma_cm_disconn_true(iwqp);
3553     irdma_qp_rem_ref(&iwqp->ibqp);
3554 }
3555
3556 /**
3557  * irdma_free_lsmm_rsrc - free lsmm memory and deregister
3558  * @iwqp: associate qp for the connection
3559  */
3560 void irdma_free_lsmm_rsrc(struct irdma_qp *iwqp)
3561 {
3562     struct irdma_device *iwdev;
3563
3564     iwdev = iwqp->iwdev;
3565
3566     if (iwqp->ietf_mem.va) {
3567         if (iwqp->lsmm_mr)
3568             iwdev->ibdev.ops.dereg_mr(iwqp->lsmm_mr, NULL);
3569         dma_free_coherent(iwdev->rf->sc_dev.hw->device,
3570                   iwqp->ietf_mem.size, iwqp->ietf_mem.va,
3571                   iwqp->ietf_mem.pa);
3572         iwqp->ietf_mem.va = NULL;
3573         iwqp->ietf_mem.va = NULL;
3574     }
3575 }
3576
3577 /**
3578  * irdma_accept - registered call for connection to be accepted
3579  * @cm_id: cm information for passive connection
3580  * @conn_param: accpet parameters
3581  */
3582 int irdma_accept(struct iw_cm_id *cm_id, struct iw_cm_conn_param *conn_param)
3583 {
3584     struct ib_qp *ibqp;
3585     struct irdma_qp *iwqp;
3586     struct irdma_device *iwdev;
3587     struct irdma_sc_dev *dev;
3588     struct irdma_cm_node *cm_node;
3589     struct ib_qp_attr attr = {};
3590     int passive_state;
3591     struct ib_mr *ibmr;
3592     struct irdma_pd *iwpd;
3593     u16 buf_len = 0;
3594     struct irdma_kmem_info accept;
3595     u64 tagged_offset;
3596     int wait_ret;
3597     int ret = 0;
3598
3599     ibqp = irdma_get_qp(cm_id->device, conn_param->qpn);
3600     if (!ibqp)
3601         return -EINVAL;
3602
3603     iwqp = to_iwqp(ibqp);
3604     iwdev = iwqp->iwdev;
3605     dev = &iwdev->rf->sc_dev;
3606     cm_node = cm_id->provider_data;
3607
3608     if (((struct sockaddr_in *)&cm_id->local_addr)->sin_family == AF_INET) {
3609         cm_node->ipv4 = true;
3610         cm_node->vlan_id = irdma_get_vlan_ipv4(cm_node->loc_addr);
3611     } else {
3612         cm_node->ipv4 = false;
3613         irdma_netdev_vlan_ipv6(cm_node->loc_addr, &cm_node->vlan_id,
3614                        NULL);
3615     }
3616     ibdev_dbg(&iwdev->ibdev, "CM: Accept vlan_id=%d\n",
3617           cm_node->vlan_id);
3618
3619     trace_irdma_accept(cm_node, 0, NULL);
3620
3621     if (cm_node->state == IRDMA_CM_STATE_LISTENER_DESTROYED) {
3622         ret = -EINVAL;
3623         goto error;
3624     }
3625
3626     passive_state = atomic_add_return(1, &cm_node->passive_state);
3627     if (passive_state == IRDMA_SEND_RESET_EVENT) {
3628         ret = -ECONNRESET;
3629         goto error;
3630     }
3631
3632     buf_len = conn_param->private_data_len + IRDMA_MAX_IETF_SIZE;
3633     iwqp->ietf_mem.size = ALIGN(buf_len, 1);
3634     iwqp->ietf_mem.va = dma_alloc_coherent(dev->hw->device,
3635                            iwqp->ietf_mem.size,
3636                            &iwqp->ietf_mem.pa, GFP_KERNEL);
3637     if (!iwqp->ietf_mem.va) {
3638         ret = -ENOMEM;
3639         goto error;
3640     }
3641
3642     cm_node->pdata.size = conn_param->private_data_len;
3643     accept.addr = iwqp->ietf_mem.va;
3644     accept.size = irdma_cm_build_mpa_frame(cm_node, &accept, MPA_KEY_REPLY);
3645     memcpy((u8 *)accept.addr + accept.size, conn_param->private_data,
3646            conn_param->private_data_len);
3647
3648     if (cm_node->dev->ws_add(iwqp->sc_qp.vsi, cm_node->user_pri)) {
3649         ret = -ENOMEM;
3650         goto error;
3651     }
3652     iwqp->sc_qp.user_pri = cm_node->user_pri;
3653     irdma_qp_add_qos(&iwqp->sc_qp);
3654     /* setup our first outgoing iWarp send WQE (the IETF frame response) */
3655     iwpd = iwqp->iwpd;
3656     tagged_offset = (uintptr_t)iwqp->ietf_mem.va;
3657     ibmr = irdma_reg_phys_mr(&iwpd->ibpd, iwqp->ietf_mem.pa, buf_len,
3658                  IB_ACCESS_LOCAL_WRITE, &tagged_offset);
3659     if (IS_ERR(ibmr)) {
3660         ret = -ENOMEM;
3661         goto error;
3662     }
3663
3664     ibmr->pd = &iwpd->ibpd;
3665     ibmr->device = iwpd->ibpd.device;
3666     iwqp->lsmm_mr = ibmr;
3667     if (iwqp->page)
3668         iwqp->sc_qp.qp_uk.sq_base = kmap_local_page(iwqp->page);
3669
3670     cm_node->lsmm_size = accept.size + conn_param->private_data_len;
3671     irdma_sc_send_lsmm(&iwqp->sc_qp, iwqp->ietf_mem.va, cm_node->lsmm_size,
3672                ibmr->lkey);
3673
3674     if (iwqp->page)
3675         kunmap_local(iwqp->sc_qp.qp_uk.sq_base);
3676
3677     iwqp->cm_id = cm_id;
3678     cm_node->cm_id = cm_id;
3679
3680     cm_id->provider_data = iwqp;
3681     iwqp->active_conn = 0;
3682     iwqp->cm_node = cm_node;
3683     cm_node->iwqp = iwqp;
3684     irdma_cm_init_tsa_conn(iwqp, cm_node);
3685     irdma_qp_add_ref(&iwqp->ibqp);
3686     cm_id->add_ref(cm_id);
3687
3688     attr.qp_state = IB_QPS_RTS;
3689     cm_node->qhash_set = false;
3690     cm_node->cm_core->cm_free_ah(cm_node);
3691
3692     irdma_modify_qp(&iwqp->ibqp, &attr, IB_QP_STATE, NULL);
3693     if (dev->hw_attrs.uk_attrs.feature_flags & IRDMA_FEATURE_RTS_AE) {
3694         wait_ret = wait_event_interruptible_timeout(iwqp->waitq,
3695                                 iwqp->rts_ae_rcvd,
3696                                 IRDMA_MAX_TIMEOUT);
3697         if (!wait_ret) {
3698             ibdev_dbg(&iwdev->ibdev,
3699                   "CM: Slow Connection: cm_node=%p, loc_port=%d, rem_port=%d, cm_id=%p\n",
3700                   cm_node, cm_node->loc_port,
3701                   cm_node->rem_port, cm_node->cm_id);
3702             ret = -ECONNRESET;
3703             goto error;
3704         }
3705     }
3706
3707     irdma_send_cm_event(cm_node, cm_id, IW_CM_EVENT_ESTABLISHED, 0);
3708     cm_node->accelerated = true;
3709     complete(&cm_node->establish_comp);
3710
3711     if (cm_node->accept_pend) {
3712         atomic_dec(&cm_node->listener->pend_accepts_cnt);
3713         cm_node->accept_pend = 0;
3714     }
3715
3716     ibdev_dbg(&iwdev->ibdev,
3717           "CM: rem_port=0x%04x, loc_port=0x%04x rem_addr=%pI4 loc_addr=%pI4 cm_node=%p cm_id=%p qp_id = %d\n\n",
3718           cm_node->rem_port, cm_node->loc_port, cm_node->rem_addr,
3719           cm_node->loc_addr, cm_node, cm_id, ibqp->qp_num);
3720     cm_node->cm_core->stats_accepts++;
3721
3722     return 0;
3723 error:
3724     irdma_free_lsmm_rsrc(iwqp);
3725     irdma_rem_ref_cm_node(cm_node);
3726
3727     return ret;
3728 }
3729
3730 /**
3731  * irdma_reject - registered call for connection to be rejected
3732  * @cm_id: cm information for passive connection
3733  * @pdata: private data to be sent
3734  * @pdata_len: private data length
3735  */
3736 int irdma_reject(struct iw_cm_id *cm_id, const void *pdata, u8 pdata_len)
3737 {
3738     struct irdma_device *iwdev;
3739     struct irdma_cm_node *cm_node;
3740
3741     cm_node = cm_id->provider_data;
3742     cm_node->pdata.size = pdata_len;
3743
3744     trace_irdma_reject(cm_node, 0, NULL);
3745
3746     iwdev = to_iwdev(cm_id->device);
3747     if (!iwdev)
3748         return -EINVAL;
3749
3750     cm_node->cm_core->stats_rejects++;
3751
3752     if (pdata_len + sizeof(struct ietf_mpa_v2) > IRDMA_MAX_CM_BUF)
3753         return -EINVAL;
3754
3755     return irdma_cm_reject(cm_node, pdata, pdata_len);
3756 }
3757
3758 /**
3759  * irdma_connect - registered call for connection to be established
3760  * @cm_id: cm information for passive connection
3761  * @conn_param: Information about the connection
3762  */
3763 int irdma_connect(struct iw_cm_id *cm_id, struct iw_cm_conn_param *conn_param)
3764 {
3765     struct ib_qp *ibqp;
3766     struct irdma_qp *iwqp;
3767     struct irdma_device *iwdev;
3768     struct irdma_cm_node *cm_node;
3769     struct irdma_cm_info cm_info;
3770     struct sockaddr_in *laddr;
3771     struct sockaddr_in *raddr;
3772     struct sockaddr_in6 *laddr6;
3773     struct sockaddr_in6 *raddr6;
3774     int ret = 0;
3775
3776     ibqp = irdma_get_qp(cm_id->device, conn_param->qpn);
3777     if (!ibqp)
3778         return -EINVAL;
3779     iwqp = to_iwqp(ibqp);
3780     if (!iwqp)
3781         return -EINVAL;
3782     iwdev = iwqp->iwdev;
3783     if (!iwdev)
3784         return -EINVAL;
3785
3786     laddr = (struct sockaddr_in *)&cm_id->m_local_addr;
3787     raddr = (struct sockaddr_in *)&cm_id->m_remote_addr;
3788     laddr6 = (struct sockaddr_in6 *)&cm_id->m_local_addr;
3789     raddr6 = (struct sockaddr_in6 *)&cm_id->m_remote_addr;
3790
3791     if (!(laddr->sin_port) || !(raddr->sin_port))
3792         return -EINVAL;
3793
3794     iwqp->active_conn = 1;
3795     iwqp->cm_id = NULL;
3796     cm_id->provider_data = iwqp;
3797
3798     /* set up the connection params for the node */
3799     if (cm_id->remote_addr.ss_family == AF_INET) {
3800         if (iwdev->vsi.mtu < IRDMA_MIN_MTU_IPV4)
3801             return -EINVAL;
3802
3803         cm_info.ipv4 = true;
3804         memset(cm_info.loc_addr, 0, sizeof(cm_info.loc_addr));
3805         memset(cm_info.rem_addr, 0, sizeof(cm_info.rem_addr));
3806         cm_info.loc_addr[0] = ntohl(laddr->sin_addr.s_addr);
3807         cm_info.rem_addr[0] = ntohl(raddr->sin_addr.s_addr);
3808         cm_info.loc_port = ntohs(laddr->sin_port);
3809         cm_info.rem_port = ntohs(raddr->sin_port);
3810         cm_info.vlan_id = irdma_get_vlan_ipv4(cm_info.loc_addr);
3811     } else {
3812         if (iwdev->vsi.mtu < IRDMA_MIN_MTU_IPV6)
3813             return -EINVAL;
3814
3815         cm_info.ipv4 = false;
3816         irdma_copy_ip_ntohl(cm_info.loc_addr,
3817                     laddr6->sin6_addr.in6_u.u6_addr32);
3818         irdma_copy_ip_ntohl(cm_info.rem_addr,
3819                     raddr6->sin6_addr.in6_u.u6_addr32);
3820         cm_info.loc_port = ntohs(laddr6->sin6_port);
3821         cm_info.rem_port = ntohs(raddr6->sin6_port);
3822         irdma_netdev_vlan_ipv6(cm_info.loc_addr, &cm_info.vlan_id,
3823                        NULL);
3824     }
3825     cm_info.cm_id = cm_id;
3826     cm_info.qh_qpid = iwdev->vsi.ilq->qp_id;
3827     cm_info.tos = cm_id->tos;
3828     if (iwdev->vsi.dscp_mode)
3829         cm_info.user_pri =
3830             iwqp->sc_qp.vsi->dscp_map[irdma_tos2dscp(cm_info.tos)];
3831     else
3832         cm_info.user_pri = rt_tos2priority(cm_id->tos);
3833
3834     if (iwqp->sc_qp.dev->ws_add(iwqp->sc_qp.vsi, cm_info.user_pri))
3835         return -ENOMEM;
3836     iwqp->sc_qp.user_pri = cm_info.user_pri;
3837     irdma_qp_add_qos(&iwqp->sc_qp);
3838     ibdev_dbg(&iwdev->ibdev, "DCB: TOS:[%d] UP:[%d]\n", cm_id->tos,
3839           cm_info.user_pri);
3840
3841     trace_irdma_dcb_tos(iwdev, cm_id->tos, cm_info.user_pri);
3842
3843     ret = irdma_create_cm_node(&iwdev->cm_core, iwdev, conn_param, &cm_info,
3844                    &cm_node);
3845     if (ret)
3846         return ret;
3847     ret = cm_node->cm_core->cm_create_ah(cm_node, true);
3848     if (ret)
3849         goto err;
3850     if (irdma_manage_qhash(iwdev, &cm_info,
3851                    IRDMA_QHASH_TYPE_TCP_ESTABLISHED,
3852                    IRDMA_QHASH_MANAGE_TYPE_ADD, NULL, true)) {
3853         ret = -EINVAL;
3854         goto err;
3855     }
3856     cm_node->qhash_set = true;
3857
3858     cm_node->apbvt_entry = irdma_add_apbvt(iwdev, cm_info.loc_port);
3859     if (!cm_node->apbvt_entry) {
3860         ret = -EINVAL;
3861         goto err;
3862     }
3863
3864     cm_node->apbvt_set = true;
3865     iwqp->cm_node = cm_node;
3866     cm_node->iwqp = iwqp;
3867     iwqp->cm_id = cm_id;
3868     irdma_qp_add_ref(&iwqp->ibqp);
3869     cm_id->add_ref(cm_id);
3870
3871     if (cm_node->state != IRDMA_CM_STATE_OFFLOADED) {
3872         cm_node->state = IRDMA_CM_STATE_SYN_SENT;
3873         ret = irdma_send_syn(cm_node, 0);
3874         if (ret)
3875             goto err;
3876     }
3877
3878     ibdev_dbg(&iwdev->ibdev,
3879           "CM: rem_port=0x%04x, loc_port=0x%04x rem_addr=%pI4 loc_addr=%pI4 cm_node=%p cm_id=%p qp_id = %d\n\n",
3880           cm_node->rem_port, cm_node->loc_port, cm_node->rem_addr,
3881           cm_node->loc_addr, cm_node, cm_id, ibqp->qp_num);
3882
3883     trace_irdma_connect(cm_node, 0, NULL);
3884
3885     return 0;
3886
3887 err:
3888     if (cm_info.ipv4)
3889         ibdev_dbg(&iwdev->ibdev,
3890               "CM: connect() FAILED: dest addr=%pI4",
3891               cm_info.rem_addr);
3892     else
3893         ibdev_dbg(&iwdev->ibdev,
3894               "CM: connect() FAILED: dest addr=%pI6",
3895               cm_info.rem_addr);
3896     irdma_rem_ref_cm_node(cm_node);
3897     iwdev->cm_core.stats_connect_errs++;
3898
3899     return ret;
3900 }
3901
3902 /**
3903  * irdma_create_listen - registered call creating listener
3904  * @cm_id: cm information for passive connection
3905  * @backlog: to max accept pending count
3906  */
3907 int irdma_create_listen(struct iw_cm_id *cm_id, int backlog)
3908 {
3909     struct irdma_device *iwdev;
3910     struct irdma_cm_listener *cm_listen_node;
3911     struct irdma_cm_info cm_info = {};
3912     struct sockaddr_in *laddr;
3913     struct sockaddr_in6 *laddr6;
3914     bool wildcard = false;
3915     int err;
3916
3917     iwdev = to_iwdev(cm_id->device);
3918     if (!iwdev)
3919         return -EINVAL;
3920
3921     laddr = (struct sockaddr_in *)&cm_id->m_local_addr;
3922     laddr6 = (struct sockaddr_in6 *)&cm_id->m_local_addr;
3923     cm_info.qh_qpid = iwdev->vsi.ilq->qp_id;
3924
3925     if (laddr->sin_family == AF_INET) {
3926         if (iwdev->vsi.mtu < IRDMA_MIN_MTU_IPV4)
3927             return -EINVAL;
3928
3929         cm_info.ipv4 = true;
3930         cm_info.loc_addr[0] = ntohl(laddr->sin_addr.s_addr);
3931         cm_info.loc_port = ntohs(laddr->sin_port);
3932
3933         if (laddr->sin_addr.s_addr != htonl(INADDR_ANY)) {
3934             cm_info.vlan_id = irdma_get_vlan_ipv4(cm_info.loc_addr);
3935         } else {
3936             cm_info.vlan_id = 0xFFFF;
3937             wildcard = true;
3938         }
3939     } else {
3940         if (iwdev->vsi.mtu < IRDMA_MIN_MTU_IPV6)
3941             return -EINVAL;
3942
3943         cm_info.ipv4 = false;
3944         irdma_copy_ip_ntohl(cm_info.loc_addr,
3945                     laddr6->sin6_addr.in6_u.u6_addr32);
3946         cm_info.loc_port = ntohs(laddr6->sin6_port);
3947         if (ipv6_addr_type(&laddr6->sin6_addr) != IPV6_ADDR_ANY) {
3948             irdma_netdev_vlan_ipv6(cm_info.loc_addr,
3949                            &cm_info.vlan_id, NULL);
3950         } else {
3951             cm_info.vlan_id = 0xFFFF;
3952             wildcard = true;
3953         }
3954     }
3955
3956     if (cm_info.vlan_id >= VLAN_N_VID && iwdev->dcb_vlan_mode)
3957         cm_info.vlan_id = 0;
3958     cm_info.backlog = backlog;
3959     cm_info.cm_id = cm_id;
3960
3961     trace_irdma_create_listen(iwdev, &cm_info);
3962
3963     cm_listen_node = irdma_make_listen_node(&iwdev->cm_core, iwdev,
3964                         &cm_info);
3965     if (!cm_listen_node) {
3966         ibdev_dbg(&iwdev->ibdev,
3967               "CM: cm_listen_node == NULL\n");
3968         return -ENOMEM;
3969     }
3970
3971     cm_id->provider_data = cm_listen_node;
3972
3973     cm_listen_node->tos = cm_id->tos;
3974     if (iwdev->vsi.dscp_mode)
3975         cm_listen_node->user_pri =
3976             iwdev->vsi.dscp_map[irdma_tos2dscp(cm_id->tos)];
3977     else
3978         cm_listen_node->user_pri = rt_tos2priority(cm_id->tos);
3979     cm_info.user_pri = cm_listen_node->user_pri;
3980     if (!cm_listen_node->reused_node) {
3981         if (wildcard) {
3982             err = irdma_add_mqh(iwdev, &cm_info, cm_listen_node);
3983             if (err)
3984                 goto error;
3985         } else {
3986             err = irdma_manage_qhash(iwdev, &cm_info,
3987                          IRDMA_QHASH_TYPE_TCP_SYN,
3988                          IRDMA_QHASH_MANAGE_TYPE_ADD,
3989                          NULL, true);
3990             if (err)
3991                 goto error;
3992
3993             cm_listen_node->qhash_set = true;
3994         }
3995
3996         cm_listen_node->apbvt_entry = irdma_add_apbvt(iwdev,
3997                                   cm_info.loc_port);
3998         if (!cm_listen_node->apbvt_entry)
3999             goto error;
4000     }
4001     cm_id->add_ref(cm_id);
4002     cm_listen_node->cm_core->stats_listen_created++;
4003     ibdev_dbg(&iwdev->ibdev,
4004           "CM: loc_port=0x%04x loc_addr=%pI4 cm_listen_node=%p cm_id=%p qhash_set=%d vlan_id=%d\n",
4005           cm_listen_node->loc_port, cm_listen_node->loc_addr,
4006           cm_listen_node, cm_listen_node->cm_id,
4007           cm_listen_node->qhash_set, cm_listen_node->vlan_id);
4008
4009     return 0;
4010
4011 error:
4012
4013     irdma_cm_del_listen(&iwdev->cm_core, cm_listen_node, false);
4014
4015     return -EINVAL;
4016 }
4017
4018 /**
4019  * irdma_destroy_listen - registered call to destroy listener
4020  * @cm_id: cm information for passive connection
4021  */
4022 int irdma_destroy_listen(struct iw_cm_id *cm_id)
4023 {
4024     struct irdma_device *iwdev;
4025
4026     iwdev = to_iwdev(cm_id->device);
4027     if (cm_id->provider_data)
4028         irdma_cm_del_listen(&iwdev->cm_core, cm_id->provider_data,
4029                     true);
4030     else
4031         ibdev_dbg(&iwdev->ibdev,
4032               "CM: cm_id->provider_data was NULL\n");
4033
4034     cm_id->rem_ref(cm_id);
4035
4036     return 0;
4037 }
4038
4039 /**
4040  * irdma_teardown_list_prep - add conn nodes slated for tear down to list
4041  * @cm_core: cm's core
4042  * @teardown_list: a list to which cm_node will be selected
4043  * @ipaddr: pointer to ip address
4044  * @nfo: pointer to cm_info structure instance
4045  * @disconnect_all: flag indicating disconnect all QPs
4046  */
4047 static void irdma_teardown_list_prep(struct irdma_cm_core *cm_core,
4048                      struct list_head *teardown_list,
4049                      u32 *ipaddr,
4050                      struct irdma_cm_info *nfo,
4051                      bool disconnect_all)
4052 {
4053     struct irdma_cm_node *cm_node;
4054     int bkt;
4055
4056     hash_for_each_rcu(cm_core->cm_hash_tbl, bkt, cm_node, list) {
4057         if ((disconnect_all ||
4058              (nfo->vlan_id == cm_node->vlan_id &&
4059               !memcmp(cm_node->loc_addr, ipaddr, nfo->ipv4 ? 4 : 16))) &&
4060             refcount_inc_not_zero(&cm_node->refcnt))
4061             list_add(&cm_node->teardown_entry, teardown_list);
4062     }
4063 }
4064
4065 /**
4066  * irdma_cm_event_connected - handle connected active node
4067  * @event: the info for cm_node of connection
4068  */
4069 static void irdma_cm_event_connected(struct irdma_cm_event *event)
4070 {
4071     struct irdma_qp *iwqp;
4072     struct irdma_device *iwdev;
4073     struct irdma_cm_node *cm_node;
4074     struct irdma_sc_dev *dev;
4075     struct ib_qp_attr attr = {};
4076     struct iw_cm_id *cm_id;
4077     int status;
4078     bool read0;
4079     int wait_ret = 0;
4080
4081     cm_node = event->cm_node;
4082     cm_id = cm_node->cm_id;
4083     iwqp = cm_id->provider_data;
4084     iwdev = iwqp->iwdev;
4085     dev = &iwdev->rf->sc_dev;
4086     if (iwqp->sc_qp.qp_uk.destroy_pending) {
4087         status = -ETIMEDOUT;
4088         goto error;
4089     }
4090
4091     irdma_cm_init_tsa_conn(iwqp, cm_node);
4092     read0 = (cm_node->send_rdma0_op == SEND_RDMA_READ_ZERO);
4093     if (iwqp->page)
4094         iwqp->sc_qp.qp_uk.sq_base = kmap_local_page(iwqp->page);
4095     irdma_sc_send_rtt(&iwqp->sc_qp, read0);
4096     if (iwqp->page)
4097         kunmap_local(iwqp->sc_qp.qp_uk.sq_base);
4098
4099     attr.qp_state = IB_QPS_RTS;
4100     cm_node->qhash_set = false;
4101     irdma_modify_qp(&iwqp->ibqp, &attr, IB_QP_STATE, NULL);
4102     if (dev->hw_attrs.uk_attrs.feature_flags & IRDMA_FEATURE_RTS_AE) {
4103         wait_ret = wait_event_interruptible_timeout(iwqp->waitq,
4104                                 iwqp->rts_ae_rcvd,
4105                                 IRDMA_MAX_TIMEOUT);
4106         if (!wait_ret)
4107             ibdev_dbg(&iwdev->ibdev,
4108                   "CM: Slow Connection: cm_node=%p, loc_port=%d, rem_port=%d, cm_id=%p\n",
4109                   cm_node, cm_node->loc_port,
4110                   cm_node->rem_port, cm_node->cm_id);
4111     }
4112
4113     irdma_send_cm_event(cm_node, cm_id, IW_CM_EVENT_CONNECT_REPLY, 0);
4114     cm_node->accelerated = true;
4115     complete(&cm_node->establish_comp);
4116     cm_node->cm_core->cm_free_ah(cm_node);
4117     return;
4118
4119 error:
4120     iwqp->cm_id = NULL;
4121     cm_id->provider_data = NULL;
4122     irdma_send_cm_event(event->cm_node, cm_id, IW_CM_EVENT_CONNECT_REPLY,
4123                 status);
4124     irdma_rem_ref_cm_node(event->cm_node);
4125 }
4126
4127 /**
4128  * irdma_cm_event_reset - handle reset
4129  * @event: the info for cm_node of connection
4130  */
4131 static void irdma_cm_event_reset(struct irdma_cm_event *event)
4132 {
4133     struct irdma_cm_node *cm_node = event->cm_node;
4134     struct iw_cm_id *cm_id = cm_node->cm_id;
4135     struct irdma_qp *iwqp;
4136
4137     if (!cm_id)
4138         return;
4139
4140     iwqp = cm_id->provider_data;
4141     if (!iwqp)
4142         return;
4143
4144     ibdev_dbg(&cm_node->iwdev->ibdev,
4145           "CM: reset event %p - cm_id = %p\n", event->cm_node, cm_id);
4146     iwqp->cm_id = NULL;
4147
4148     irdma_send_cm_event(cm_node, cm_node->cm_id, IW_CM_EVENT_DISCONNECT,
4149                 -ECONNRESET);
4150     irdma_send_cm_event(cm_node, cm_node->cm_id, IW_CM_EVENT_CLOSE, 0);
4151 }
4152
4153 /**
4154  * irdma_cm_event_handler - send event to cm upper layer
4155  * @work: pointer of cm event info.
4156  */
4157 static void irdma_cm_event_handler(struct work_struct *work)
4158 {
4159     struct irdma_cm_event *event = container_of(work, struct irdma_cm_event, event_work);
4160     struct irdma_cm_node *cm_node;
4161
4162     if (!event || !event->cm_node || !event->cm_node->cm_core)
4163         return;
4164
4165     cm_node = event->cm_node;
4166     trace_irdma_cm_event_handler(cm_node, event->type, NULL);
4167
4168     switch (event->type) {
4169     case IRDMA_CM_EVENT_MPA_REQ:
4170         irdma_send_cm_event(cm_node, cm_node->cm_id,
4171                     IW_CM_EVENT_CONNECT_REQUEST, 0);
4172         break;
4173     case IRDMA_CM_EVENT_RESET:
4174         irdma_cm_event_reset(event);
4175         break;
4176     case IRDMA_CM_EVENT_CONNECTED:
4177         if (!event->cm_node->cm_id ||
4178             event->cm_node->state != IRDMA_CM_STATE_OFFLOADED)
4179             break;
4180         irdma_cm_event_connected(event);
4181         break;
4182     case IRDMA_CM_EVENT_MPA_REJECT:
4183         if (!event->cm_node->cm_id ||
4184             cm_node->state == IRDMA_CM_STATE_OFFLOADED)
4185             break;
4186         irdma_send_cm_event(cm_node, cm_node->cm_id,
4187                     IW_CM_EVENT_CONNECT_REPLY, -ECONNREFUSED);
4188         break;
4189     case IRDMA_CM_EVENT_ABORTED:
4190         if (!event->cm_node->cm_id ||
4191             event->cm_node->state == IRDMA_CM_STATE_OFFLOADED)
4192             break;
4193         irdma_event_connect_error(event);
4194         break;
4195     default:
4196         ibdev_dbg(&cm_node->iwdev->ibdev,
4197               "CM: bad event type = %d\n", event->type);
4198         break;
4199     }
4200
4201     irdma_rem_ref_cm_node(event->cm_node);
4202     kfree(event);
4203 }
4204
4205 /**
4206  * irdma_cm_post_event - queue event request for worker thread
4207  * @event: cm node's info for up event call
4208  */
4209 static void irdma_cm_post_event(struct irdma_cm_event *event)
4210 {
4211     refcount_inc(&event->cm_node->refcnt);
4212     INIT_WORK(&event->event_work, irdma_cm_event_handler);
4213     queue_work(event->cm_node->cm_core->event_wq, &event->event_work);
4214 }
4215
4216 /**
4217  * irdma_cm_teardown_connections - teardown QPs
4218  * @iwdev: device pointer
4219  * @ipaddr: Pointer to IPv4 or IPv6 address
4220  * @nfo: Connection info
4221  * @disconnect_all: flag indicating disconnect all QPs
4222  *
4223  * teardown QPs where source or destination addr matches ip addr
4224  */
4225 void irdma_cm_teardown_connections(struct irdma_device *iwdev, u32 *ipaddr,
4226                    struct irdma_cm_info *nfo,
4227                    bool disconnect_all)
4228 {
4229     struct irdma_cm_core *cm_core = &iwdev->cm_core;
4230     struct list_head *list_core_temp;
4231     struct list_head *list_node;
4232     struct irdma_cm_node *cm_node;
4233     struct list_head teardown_list;
4234     struct ib_qp_attr attr;
4235
4236     INIT_LIST_HEAD(&teardown_list);
4237
4238     rcu_read_lock();
4239     irdma_teardown_list_prep(cm_core, &teardown_list, ipaddr, nfo, disconnect_all);
4240     rcu_read_unlock();
4241
4242     list_for_each_safe (list_node, list_core_temp, &teardown_list) {
4243         cm_node = container_of(list_node, struct irdma_cm_node,
4244                        teardown_entry);
4245         attr.qp_state = IB_QPS_ERR;
4246         irdma_modify_qp(&cm_node->iwqp->ibqp, &attr, IB_QP_STATE, NULL);
4247         if (iwdev->rf->reset)
4248             irdma_cm_disconn(cm_node->iwqp);
4249         irdma_rem_ref_cm_node(cm_node);
4250     }
4251 }
4252
4253 /**
4254  * irdma_qhash_ctrl - enable/disable qhash for list
4255  * @iwdev: device pointer
4256  * @parent_listen_node: parent listen node
4257  * @nfo: cm info node
4258  * @ipaddr: Pointer to IPv4 or IPv6 address
4259  * @ipv4: flag indicating IPv4 when true
4260  * @ifup: flag indicating interface up when true
4261  *
4262  * Enables or disables the qhash for the node in the child
4263  * listen list that matches ipaddr. If no matching IP was found
4264  * it will allocate and add a new child listen node to the
4265  * parent listen node. The listen_list_lock is assumed to be
4266  * held when called.
4267  */
4268 static void irdma_qhash_ctrl(struct irdma_device *iwdev,
4269                  struct irdma_cm_listener *parent_listen_node,
4270                  struct irdma_cm_info *nfo, u32 *ipaddr, bool ipv4,
4271                  bool ifup)
4272 {
4273     struct list_head *child_listen_list = &parent_listen_node->child_listen_list;
4274     struct irdma_cm_listener *child_listen_node;
4275     struct list_head *pos, *tpos;
4276     bool node_allocated = false;
4277     enum irdma_quad_hash_manage_type op = ifup ?
4278                           IRDMA_QHASH_MANAGE_TYPE_ADD :
4279                           IRDMA_QHASH_MANAGE_TYPE_DELETE;
4280     int err;
4281
4282     list_for_each_safe (pos, tpos, child_listen_list) {
4283         child_listen_node = list_entry(pos, struct irdma_cm_listener,
4284                            child_listen_list);
4285         if (!memcmp(child_listen_node->loc_addr, ipaddr, ipv4 ? 4 : 16))
4286             goto set_qhash;
4287     }
4288
4289     /* if not found then add a child listener if interface is going up */
4290     if (!ifup)
4291         return;
4292     child_listen_node = kmemdup(parent_listen_node,
4293                     sizeof(*child_listen_node), GFP_ATOMIC);
4294     if (!child_listen_node)
4295         return;
4296
4297     node_allocated = true;
4298     memcpy(child_listen_node->loc_addr, ipaddr, ipv4 ? 4 : 16);
4299
4300 set_qhash:
4301     memcpy(nfo->loc_addr, child_listen_node->loc_addr,
4302            sizeof(nfo->loc_addr));
4303     nfo->vlan_id = child_listen_node->vlan_id;
4304     err = irdma_manage_qhash(iwdev, nfo, IRDMA_QHASH_TYPE_TCP_SYN, op, NULL,
4305                  false);
4306     if (!err) {
4307         child_listen_node->qhash_set = ifup;
4308         if (node_allocated)
4309             list_add(&child_listen_node->child_listen_list,
4310                  &parent_listen_node->child_listen_list);
4311     } else if (node_allocated) {
4312         kfree(child_listen_node);
4313     }
4314 }
4315
4316 /**
4317  * irdma_if_notify - process an ifdown on an interface
4318  * @iwdev: device pointer
4319  * @netdev: network device structure
4320  * @ipaddr: Pointer to IPv4 or IPv6 address
4321  * @ipv4: flag indicating IPv4 when true
4322  * @ifup: flag indicating interface up when true
4323  */
4324 void irdma_if_notify(struct irdma_device *iwdev, struct net_device *netdev,
4325              u32 *ipaddr, bool ipv4, bool ifup)
4326 {
4327     struct irdma_cm_core *cm_core = &iwdev->cm_core;
4328     unsigned long flags;
4329     struct irdma_cm_listener *listen_node;
4330     static const u32 ip_zero[4] = { 0, 0, 0, 0 };
4331     struct irdma_cm_info nfo = {};
4332     u16 vlan_id = rdma_vlan_dev_vlan_id(netdev);
4333     enum irdma_quad_hash_manage_type op = ifup ?
4334                           IRDMA_QHASH_MANAGE_TYPE_ADD :
4335                           IRDMA_QHASH_MANAGE_TYPE_DELETE;
4336
4337     nfo.vlan_id = vlan_id;
4338     nfo.ipv4 = ipv4;
4339     nfo.qh_qpid = 1;
4340
4341     /* Disable or enable qhash for listeners */
4342     spin_lock_irqsave(&cm_core->listen_list_lock, flags);
4343     list_for_each_entry (listen_node, &cm_core->listen_list, list) {
4344         if (vlan_id != listen_node->vlan_id ||
4345             (memcmp(listen_node->loc_addr, ipaddr, ipv4 ? 4 : 16) &&
4346              memcmp(listen_node->loc_addr, ip_zero, ipv4 ? 4 : 16)))
4347             continue;
4348
4349         memcpy(nfo.loc_addr, listen_node->loc_addr,
4350                sizeof(nfo.loc_addr));
4351         nfo.loc_port = listen_node->loc_port;
4352         nfo.user_pri = listen_node->user_pri;
4353         if (!list_empty(&listen_node->child_listen_list)) {
4354             irdma_qhash_ctrl(iwdev, listen_node, &nfo, ipaddr, ipv4,
4355                      ifup);
4356         } else if (memcmp(listen_node->loc_addr, ip_zero,
4357                   ipv4 ? 4 : 16)) {
4358             if (!irdma_manage_qhash(iwdev, &nfo,
4359                         IRDMA_QHASH_TYPE_TCP_SYN, op,
4360                         NULL, false))
4361                 listen_node->qhash_set = ifup;
4362         }
4363     }
4364     spin_unlock_irqrestore(&cm_core->listen_list_lock, flags);
4365
4366     /* disconnect any connected qp's on ifdown */
4367     if (!ifup)
4368         irdma_cm_teardown_connections(iwdev, ipaddr, &nfo, false);
4369 }