net/smc/smc_llc.c

0001 // SPDX-License-Identifier: GPL-2.0
0002 /*
0003  *  Shared Memory Communications over RDMA (SMC-R) and RoCE
0004  *
0005  *  Link Layer Control (LLC)
0006  *
0007  *  Copyright IBM Corp. 2016
0008  *
0009  *  Author(s):  Klaus Wacker <Klaus.Wacker@de.ibm.com>
0010  *              Ursula Braun <ubraun@linux.vnet.ibm.com>
0011  */
0012
0013 #include <net/tcp.h>
0014 #include <rdma/ib_verbs.h>
0015
0016 #include "smc.h"
0017 #include "smc_core.h"
0018 #include "smc_clc.h"
0019 #include "smc_llc.h"
0020 #include "smc_pnet.h"
0021
0022 #define SMC_LLC_DATA_LEN        40
0023
0024 struct smc_llc_hdr {
0025     struct smc_wr_rx_hdr common;
0026     union {
0027         struct {
0028             u8 length;  /* 44 */
0029     #if defined(__BIG_ENDIAN_BITFIELD)
0030             u8 reserved:4,
0031                add_link_rej_rsn:4;
0032 #elif defined(__LITTLE_ENDIAN_BITFIELD)
0033             u8 add_link_rej_rsn:4,
0034                reserved:4;
0035 #endif
0036         };
0037         u16 length_v2;  /* 44 - 8192*/
0038     };
0039     u8 flags;
0040 } __packed;     /* format defined in
0041              * IBM Shared Memory Communications Version 2
0042              * (https://www.ibm.com/support/pages/node/6326337)
0043              */
0044
0045 #define SMC_LLC_FLAG_NO_RMBE_EYEC   0x03
0046
0047 struct smc_llc_msg_confirm_link {   /* type 0x01 */
0048     struct smc_llc_hdr hd;
0049     u8 sender_mac[ETH_ALEN];
0050     u8 sender_gid[SMC_GID_SIZE];
0051     u8 sender_qp_num[3];
0052     u8 link_num;
0053     u8 link_uid[SMC_LGR_ID_SIZE];
0054     u8 max_links;
0055     u8 reserved[9];
0056 };
0057
0058 #define SMC_LLC_FLAG_ADD_LNK_REJ    0x40
0059 #define SMC_LLC_REJ_RSN_NO_ALT_PATH 1
0060
0061 #define SMC_LLC_ADD_LNK_MAX_LINKS   2
0062
0063 struct smc_llc_msg_add_link {       /* type 0x02 */
0064     struct smc_llc_hdr hd;
0065     u8 sender_mac[ETH_ALEN];
0066     u8 reserved2[2];
0067     u8 sender_gid[SMC_GID_SIZE];
0068     u8 sender_qp_num[3];
0069     u8 link_num;
0070 #if defined(__BIG_ENDIAN_BITFIELD)
0071     u8 reserved3 : 4,
0072        qp_mtu   : 4;
0073 #elif defined(__LITTLE_ENDIAN_BITFIELD)
0074     u8 qp_mtu   : 4,
0075        reserved3 : 4;
0076 #endif
0077     u8 initial_psn[3];
0078     u8 reserved[8];
0079 };
0080
0081 struct smc_llc_msg_add_link_cont_rt {
0082     __be32 rmb_key;
0083     __be32 rmb_key_new;
0084     __be64 rmb_vaddr_new;
0085 };
0086
0087 struct smc_llc_msg_add_link_v2_ext {
0088 #if defined(__BIG_ENDIAN_BITFIELD)
0089     u8 v2_direct : 1,
0090        reserved  : 7;
0091 #elif defined(__LITTLE_ENDIAN_BITFIELD)
0092     u8 reserved  : 7,
0093        v2_direct : 1;
0094 #endif
0095     u8 reserved2;
0096     u8 client_target_gid[SMC_GID_SIZE];
0097     u8 reserved3[8];
0098     u16 num_rkeys;
0099     struct smc_llc_msg_add_link_cont_rt rt[];
0100 } __packed;     /* format defined in
0101              * IBM Shared Memory Communications Version 2
0102              * (https://www.ibm.com/support/pages/node/6326337)
0103              */
0104
0105 struct smc_llc_msg_req_add_link_v2 {
0106     struct smc_llc_hdr hd;
0107     u8 reserved[20];
0108     u8 gid_cnt;
0109     u8 reserved2[3];
0110     u8 gid[][SMC_GID_SIZE];
0111 };
0112
0113 #define SMC_LLC_RKEYS_PER_CONT_MSG  2
0114
0115 struct smc_llc_msg_add_link_cont {  /* type 0x03 */
0116     struct smc_llc_hdr hd;
0117     u8 link_num;
0118     u8 num_rkeys;
0119     u8 reserved2[2];
0120     struct smc_llc_msg_add_link_cont_rt rt[SMC_LLC_RKEYS_PER_CONT_MSG];
0121     u8 reserved[4];
0122 } __packed;         /* format defined in RFC7609 */
0123
0124 #define SMC_LLC_FLAG_DEL_LINK_ALL   0x40
0125 #define SMC_LLC_FLAG_DEL_LINK_ORDERLY   0x20
0126
0127 struct smc_llc_msg_del_link {       /* type 0x04 */
0128     struct smc_llc_hdr hd;
0129     u8 link_num;
0130     __be32 reason;
0131     u8 reserved[35];
0132 } __packed;         /* format defined in RFC7609 */
0133
0134 struct smc_llc_msg_test_link {      /* type 0x07 */
0135     struct smc_llc_hdr hd;
0136     u8 user_data[16];
0137     u8 reserved[24];
0138 };
0139
0140 struct smc_rmb_rtoken {
0141     union {
0142         u8 num_rkeys;   /* first rtoken byte of CONFIRM LINK msg */
0143                 /* is actually the num of rtokens, first */
0144                 /* rtoken is always for the current link */
0145         u8 link_id; /* link id of the rtoken */
0146     };
0147     __be32 rmb_key;
0148     __be64 rmb_vaddr;
0149 } __packed;         /* format defined in RFC7609 */
0150
0151 #define SMC_LLC_RKEYS_PER_MSG       3
0152 #define SMC_LLC_RKEYS_PER_MSG_V2    255
0153
0154 struct smc_llc_msg_confirm_rkey {   /* type 0x06 */
0155     struct smc_llc_hdr hd;
0156     struct smc_rmb_rtoken rtoken[SMC_LLC_RKEYS_PER_MSG];
0157     u8 reserved;
0158 };
0159
0160 #define SMC_LLC_DEL_RKEY_MAX    8
0161 #define SMC_LLC_FLAG_RKEY_RETRY 0x10
0162 #define SMC_LLC_FLAG_RKEY_NEG   0x20
0163
0164 struct smc_llc_msg_delete_rkey {    /* type 0x09 */
0165     struct smc_llc_hdr hd;
0166     u8 num_rkeys;
0167     u8 err_mask;
0168     u8 reserved[2];
0169     __be32 rkey[8];
0170     u8 reserved2[4];
0171 };
0172
0173 struct smc_llc_msg_delete_rkey_v2 { /* type 0x29 */
0174     struct smc_llc_hdr hd;
0175     u8 num_rkeys;
0176     u8 num_inval_rkeys;
0177     u8 reserved[2];
0178     __be32 rkey[];
0179 };
0180
0181 union smc_llc_msg {
0182     struct smc_llc_msg_confirm_link confirm_link;
0183     struct smc_llc_msg_add_link add_link;
0184     struct smc_llc_msg_req_add_link_v2 req_add_link;
0185     struct smc_llc_msg_add_link_cont add_link_cont;
0186     struct smc_llc_msg_del_link delete_link;
0187
0188     struct smc_llc_msg_confirm_rkey confirm_rkey;
0189     struct smc_llc_msg_delete_rkey delete_rkey;
0190
0191     struct smc_llc_msg_test_link test_link;
0192     struct {
0193         struct smc_llc_hdr hdr;
0194         u8 data[SMC_LLC_DATA_LEN];
0195     } raw;
0196 };
0197
0198 #define SMC_LLC_FLAG_RESP       0x80
0199
0200 struct smc_llc_qentry {
0201     struct list_head list;
0202     struct smc_link *link;
0203     union smc_llc_msg msg;
0204 };
0205
0206 static void smc_llc_enqueue(struct smc_link *link, union smc_llc_msg *llc);
0207
0208 struct smc_llc_qentry *smc_llc_flow_qentry_clr(struct smc_llc_flow *flow)
0209 {
0210     struct smc_llc_qentry *qentry = flow->qentry;
0211
0212     flow->qentry = NULL;
0213     return qentry;
0214 }
0215
0216 void smc_llc_flow_qentry_del(struct smc_llc_flow *flow)
0217 {
0218     struct smc_llc_qentry *qentry;
0219
0220     if (flow->qentry) {
0221         qentry = flow->qentry;
0222         flow->qentry = NULL;
0223         kfree(qentry);
0224     }
0225 }
0226
0227 static inline void smc_llc_flow_qentry_set(struct smc_llc_flow *flow,
0228                        struct smc_llc_qentry *qentry)
0229 {
0230     flow->qentry = qentry;
0231 }
0232
0233 static void smc_llc_flow_parallel(struct smc_link_group *lgr, u8 flow_type,
0234                   struct smc_llc_qentry *qentry)
0235 {
0236     u8 msg_type = qentry->msg.raw.hdr.common.llc_type;
0237
0238     if ((msg_type == SMC_LLC_ADD_LINK || msg_type == SMC_LLC_DELETE_LINK) &&
0239         flow_type != msg_type && !lgr->delayed_event) {
0240         lgr->delayed_event = qentry;
0241         return;
0242     }
0243     /* drop parallel or already-in-progress llc requests */
0244     if (flow_type != msg_type)
0245         pr_warn_once("smc: SMC-R lg %*phN net %llu dropped parallel "
0246                  "LLC msg: msg %d flow %d role %d\n",
0247                  SMC_LGR_ID_SIZE, &lgr->id,
0248                  lgr->net->net_cookie,
0249                  qentry->msg.raw.hdr.common.type,
0250                  flow_type, lgr->role);
0251     kfree(qentry);
0252 }
0253
0254 /* try to start a new llc flow, initiated by an incoming llc msg */
0255 static bool smc_llc_flow_start(struct smc_llc_flow *flow,
0256                    struct smc_llc_qentry *qentry)
0257 {
0258     struct smc_link_group *lgr = qentry->link->lgr;
0259
0260     spin_lock_bh(&lgr->llc_flow_lock);
0261     if (flow->type) {
0262         /* a flow is already active */
0263         smc_llc_flow_parallel(lgr, flow->type, qentry);
0264         spin_unlock_bh(&lgr->llc_flow_lock);
0265         return false;
0266     }
0267     switch (qentry->msg.raw.hdr.common.llc_type) {
0268     case SMC_LLC_ADD_LINK:
0269         flow->type = SMC_LLC_FLOW_ADD_LINK;
0270         break;
0271     case SMC_LLC_DELETE_LINK:
0272         flow->type = SMC_LLC_FLOW_DEL_LINK;
0273         break;
0274     case SMC_LLC_CONFIRM_RKEY:
0275     case SMC_LLC_DELETE_RKEY:
0276         flow->type = SMC_LLC_FLOW_RKEY;
0277         break;
0278     default:
0279         flow->type = SMC_LLC_FLOW_NONE;
0280     }
0281     smc_llc_flow_qentry_set(flow, qentry);
0282     spin_unlock_bh(&lgr->llc_flow_lock);
0283     return true;
0284 }
0285
0286 /* start a new local llc flow, wait till current flow finished */
0287 int smc_llc_flow_initiate(struct smc_link_group *lgr,
0288               enum smc_llc_flowtype type)
0289 {
0290     enum smc_llc_flowtype allowed_remote = SMC_LLC_FLOW_NONE;
0291     int rc;
0292
0293     /* all flows except confirm_rkey and delete_rkey are exclusive,
0294      * confirm/delete rkey flows can run concurrently (local and remote)
0295      */
0296     if (type == SMC_LLC_FLOW_RKEY)
0297         allowed_remote = SMC_LLC_FLOW_RKEY;
0298 again:
0299     if (list_empty(&lgr->list))
0300         return -ENODEV;
0301     spin_lock_bh(&lgr->llc_flow_lock);
0302     if (lgr->llc_flow_lcl.type == SMC_LLC_FLOW_NONE &&
0303         (lgr->llc_flow_rmt.type == SMC_LLC_FLOW_NONE ||
0304          lgr->llc_flow_rmt.type == allowed_remote)) {
0305         lgr->llc_flow_lcl.type = type;
0306         spin_unlock_bh(&lgr->llc_flow_lock);
0307         return 0;
0308     }
0309     spin_unlock_bh(&lgr->llc_flow_lock);
0310     rc = wait_event_timeout(lgr->llc_flow_waiter, (list_empty(&lgr->list) ||
0311                 (lgr->llc_flow_lcl.type == SMC_LLC_FLOW_NONE &&
0312                  (lgr->llc_flow_rmt.type == SMC_LLC_FLOW_NONE ||
0313                   lgr->llc_flow_rmt.type == allowed_remote))),
0314                 SMC_LLC_WAIT_TIME * 10);
0315     if (!rc)
0316         return -ETIMEDOUT;
0317     goto again;
0318 }
0319
0320 /* finish the current llc flow */
0321 void smc_llc_flow_stop(struct smc_link_group *lgr, struct smc_llc_flow *flow)
0322 {
0323     spin_lock_bh(&lgr->llc_flow_lock);
0324     memset(flow, 0, sizeof(*flow));
0325     flow->type = SMC_LLC_FLOW_NONE;
0326     spin_unlock_bh(&lgr->llc_flow_lock);
0327     if (!list_empty(&lgr->list) && lgr->delayed_event &&
0328         flow == &lgr->llc_flow_lcl)
0329         schedule_work(&lgr->llc_event_work);
0330     else
0331         wake_up(&lgr->llc_flow_waiter);
0332 }
0333
0334 /* lnk is optional and used for early wakeup when link goes down, useful in
0335  * cases where we wait for a response on the link after we sent a request
0336  */
0337 struct smc_llc_qentry *smc_llc_wait(struct smc_link_group *lgr,
0338                     struct smc_link *lnk,
0339                     int time_out, u8 exp_msg)
0340 {
0341     struct smc_llc_flow *flow = &lgr->llc_flow_lcl;
0342     u8 rcv_msg;
0343
0344     wait_event_timeout(lgr->llc_msg_waiter,
0345                (flow->qentry ||
0346                 (lnk && !smc_link_usable(lnk)) ||
0347                 list_empty(&lgr->list)),
0348                time_out);
0349     if (!flow->qentry ||
0350         (lnk && !smc_link_usable(lnk)) || list_empty(&lgr->list)) {
0351         smc_llc_flow_qentry_del(flow);
0352         goto out;
0353     }
0354     rcv_msg = flow->qentry->msg.raw.hdr.common.llc_type;
0355     if (exp_msg && rcv_msg != exp_msg) {
0356         if (exp_msg == SMC_LLC_ADD_LINK &&
0357             rcv_msg == SMC_LLC_DELETE_LINK) {
0358             /* flow_start will delay the unexpected msg */
0359             smc_llc_flow_start(&lgr->llc_flow_lcl,
0360                        smc_llc_flow_qentry_clr(flow));
0361             return NULL;
0362         }
0363         pr_warn_once("smc: SMC-R lg %*phN net %llu dropped unexpected LLC msg: "
0364                  "msg %d exp %d flow %d role %d flags %x\n",
0365                  SMC_LGR_ID_SIZE, &lgr->id, lgr->net->net_cookie,
0366                  rcv_msg, exp_msg,
0367                  flow->type, lgr->role,
0368                  flow->qentry->msg.raw.hdr.flags);
0369         smc_llc_flow_qentry_del(flow);
0370     }
0371 out:
0372     return flow->qentry;
0373 }
0374
0375 /********************************** send *************************************/
0376
0377 struct smc_llc_tx_pend {
0378 };
0379
0380 /* handler for send/transmission completion of an LLC msg */
0381 static void smc_llc_tx_handler(struct smc_wr_tx_pend_priv *pend,
0382                    struct smc_link *link,
0383                    enum ib_wc_status wc_status)
0384 {
0385     /* future work: handle wc_status error for recovery and failover */
0386 }
0387
0388 /**
0389  * smc_llc_add_pending_send() - add LLC control message to pending WQE transmits
0390  * @link: Pointer to SMC link used for sending LLC control message.
0391  * @wr_buf: Out variable returning pointer to work request payload buffer.
0392  * @pend: Out variable returning pointer to private pending WR tracking.
0393  *    It's the context the transmit complete handler will get.
0394  *
0395  * Reserves and pre-fills an entry for a pending work request send/tx.
0396  * Used by mid-level smc_llc_send_msg() to prepare for later actual send/tx.
0397  * Can sleep due to smc_get_ctrl_buf (if not in softirq context).
0398  *
0399  * Return: 0 on success, otherwise an error value.
0400  */
0401 static int smc_llc_add_pending_send(struct smc_link *link,
0402                     struct smc_wr_buf **wr_buf,
0403                     struct smc_wr_tx_pend_priv **pend)
0404 {
0405     int rc;
0406
0407     rc = smc_wr_tx_get_free_slot(link, smc_llc_tx_handler, wr_buf, NULL,
0408                      pend);
0409     if (rc < 0)
0410         return rc;
0411     BUILD_BUG_ON_MSG(
0412         sizeof(union smc_llc_msg) > SMC_WR_BUF_SIZE,
0413         "must increase SMC_WR_BUF_SIZE to at least sizeof(struct smc_llc_msg)");
0414     BUILD_BUG_ON_MSG(
0415         sizeof(union smc_llc_msg) != SMC_WR_TX_SIZE,
0416         "must adapt SMC_WR_TX_SIZE to sizeof(struct smc_llc_msg); if not all smc_wr upper layer protocols use the same message size any more, must start to set link->wr_tx_sges[i].length on each individual smc_wr_tx_send()");
0417     BUILD_BUG_ON_MSG(
0418         sizeof(struct smc_llc_tx_pend) > SMC_WR_TX_PEND_PRIV_SIZE,
0419         "must increase SMC_WR_TX_PEND_PRIV_SIZE to at least sizeof(struct smc_llc_tx_pend)");
0420     return 0;
0421 }
0422
0423 static int smc_llc_add_pending_send_v2(struct smc_link *link,
0424                        struct smc_wr_v2_buf **wr_buf,
0425                        struct smc_wr_tx_pend_priv **pend)
0426 {
0427     int rc;
0428
0429     rc = smc_wr_tx_get_v2_slot(link, smc_llc_tx_handler, wr_buf, pend);
0430     if (rc < 0)
0431         return rc;
0432     return 0;
0433 }
0434
0435 static void smc_llc_init_msg_hdr(struct smc_llc_hdr *hdr,
0436                  struct smc_link_group *lgr, size_t len)
0437 {
0438     if (lgr->smc_version == SMC_V2) {
0439         hdr->common.llc_version = SMC_V2;
0440         hdr->length_v2 = len;
0441     } else {
0442         hdr->common.llc_version = 0;
0443         hdr->length = len;
0444     }
0445 }
0446
0447 /* high-level API to send LLC confirm link */
0448 int smc_llc_send_confirm_link(struct smc_link *link,
0449                   enum smc_llc_reqresp reqresp)
0450 {
0451     struct smc_llc_msg_confirm_link *confllc;
0452     struct smc_wr_tx_pend_priv *pend;
0453     struct smc_wr_buf *wr_buf;
0454     int rc;
0455
0456     if (!smc_wr_tx_link_hold(link))
0457         return -ENOLINK;
0458     rc = smc_llc_add_pending_send(link, &wr_buf, &pend);
0459     if (rc)
0460         goto put_out;
0461     confllc = (struct smc_llc_msg_confirm_link *)wr_buf;
0462     memset(confllc, 0, sizeof(*confllc));
0463     confllc->hd.common.llc_type = SMC_LLC_CONFIRM_LINK;
0464     smc_llc_init_msg_hdr(&confllc->hd, link->lgr, sizeof(*confllc));
0465     confllc->hd.flags |= SMC_LLC_FLAG_NO_RMBE_EYEC;
0466     if (reqresp == SMC_LLC_RESP)
0467         confllc->hd.flags |= SMC_LLC_FLAG_RESP;
0468     memcpy(confllc->sender_mac, link->smcibdev->mac[link->ibport - 1],
0469            ETH_ALEN);
0470     memcpy(confllc->sender_gid, link->gid, SMC_GID_SIZE);
0471     hton24(confllc->sender_qp_num, link->roce_qp->qp_num);
0472     confllc->link_num = link->link_id;
0473     memcpy(confllc->link_uid, link->link_uid, SMC_LGR_ID_SIZE);
0474     confllc->max_links = SMC_LLC_ADD_LNK_MAX_LINKS;
0475     /* send llc message */
0476     rc = smc_wr_tx_send(link, pend);
0477 put_out:
0478     smc_wr_tx_link_put(link);
0479     return rc;
0480 }
0481
0482 /* send LLC confirm rkey request */
0483 static int smc_llc_send_confirm_rkey(struct smc_link *send_link,
0484                      struct smc_buf_desc *rmb_desc)
0485 {
0486     struct smc_llc_msg_confirm_rkey *rkeyllc;
0487     struct smc_wr_tx_pend_priv *pend;
0488     struct smc_wr_buf *wr_buf;
0489     struct smc_link *link;
0490     int i, rc, rtok_ix;
0491
0492     if (!smc_wr_tx_link_hold(send_link))
0493         return -ENOLINK;
0494     rc = smc_llc_add_pending_send(send_link, &wr_buf, &pend);
0495     if (rc)
0496         goto put_out;
0497     rkeyllc = (struct smc_llc_msg_confirm_rkey *)wr_buf;
0498     memset(rkeyllc, 0, sizeof(*rkeyllc));
0499     rkeyllc->hd.common.llc_type = SMC_LLC_CONFIRM_RKEY;
0500     smc_llc_init_msg_hdr(&rkeyllc->hd, send_link->lgr, sizeof(*rkeyllc));
0501
0502     rtok_ix = 1;
0503     for (i = 0; i < SMC_LINKS_PER_LGR_MAX; i++) {
0504         link = &send_link->lgr->lnk[i];
0505         if (smc_link_active(link) && link != send_link) {
0506             rkeyllc->rtoken[rtok_ix].link_id = link->link_id;
0507             rkeyllc->rtoken[rtok_ix].rmb_key =
0508                 htonl(rmb_desc->mr[link->link_idx]->rkey);
0509             rkeyllc->rtoken[rtok_ix].rmb_vaddr = rmb_desc->is_vm ?
0510                 cpu_to_be64((uintptr_t)rmb_desc->cpu_addr) :
0511                 cpu_to_be64((u64)sg_dma_address
0512                         (rmb_desc->sgt[link->link_idx].sgl));
0513             rtok_ix++;
0514         }
0515     }
0516     /* rkey of send_link is in rtoken[0] */
0517     rkeyllc->rtoken[0].num_rkeys = rtok_ix - 1;
0518     rkeyllc->rtoken[0].rmb_key =
0519         htonl(rmb_desc->mr[send_link->link_idx]->rkey);
0520     rkeyllc->rtoken[0].rmb_vaddr = rmb_desc->is_vm ?
0521         cpu_to_be64((uintptr_t)rmb_desc->cpu_addr) :
0522         cpu_to_be64((u64)sg_dma_address
0523                 (rmb_desc->sgt[send_link->link_idx].sgl));
0524     /* send llc message */
0525     rc = smc_wr_tx_send(send_link, pend);
0526 put_out:
0527     smc_wr_tx_link_put(send_link);
0528     return rc;
0529 }
0530
0531 /* send LLC delete rkey request */
0532 static int smc_llc_send_delete_rkey(struct smc_link *link,
0533                     struct smc_buf_desc *rmb_desc)
0534 {
0535     struct smc_llc_msg_delete_rkey *rkeyllc;
0536     struct smc_wr_tx_pend_priv *pend;
0537     struct smc_wr_buf *wr_buf;
0538     int rc;
0539
0540     if (!smc_wr_tx_link_hold(link))
0541         return -ENOLINK;
0542     rc = smc_llc_add_pending_send(link, &wr_buf, &pend);
0543     if (rc)
0544         goto put_out;
0545     rkeyllc = (struct smc_llc_msg_delete_rkey *)wr_buf;
0546     memset(rkeyllc, 0, sizeof(*rkeyllc));
0547     rkeyllc->hd.common.llc_type = SMC_LLC_DELETE_RKEY;
0548     smc_llc_init_msg_hdr(&rkeyllc->hd, link->lgr, sizeof(*rkeyllc));
0549     rkeyllc->num_rkeys = 1;
0550     rkeyllc->rkey[0] = htonl(rmb_desc->mr[link->link_idx]->rkey);
0551     /* send llc message */
0552     rc = smc_wr_tx_send(link, pend);
0553 put_out:
0554     smc_wr_tx_link_put(link);
0555     return rc;
0556 }
0557
0558 /* return first buffer from any of the next buf lists */
0559 static struct smc_buf_desc *_smc_llc_get_next_rmb(struct smc_link_group *lgr,
0560                           int *buf_lst)
0561 {
0562     struct smc_buf_desc *buf_pos;
0563
0564     while (*buf_lst < SMC_RMBE_SIZES) {
0565         buf_pos = list_first_entry_or_null(&lgr->rmbs[*buf_lst],
0566                            struct smc_buf_desc, list);
0567         if (buf_pos)
0568             return buf_pos;
0569         (*buf_lst)++;
0570     }
0571     return NULL;
0572 }
0573
0574 /* return next rmb from buffer lists */
0575 static struct smc_buf_desc *smc_llc_get_next_rmb(struct smc_link_group *lgr,
0576                          int *buf_lst,
0577                          struct smc_buf_desc *buf_pos)
0578 {
0579     struct smc_buf_desc *buf_next;
0580
0581     if (!buf_pos || list_is_last(&buf_pos->list, &lgr->rmbs[*buf_lst])) {
0582         (*buf_lst)++;
0583         return _smc_llc_get_next_rmb(lgr, buf_lst);
0584     }
0585     buf_next = list_next_entry(buf_pos, list);
0586     return buf_next;
0587 }
0588
0589 static struct smc_buf_desc *smc_llc_get_first_rmb(struct smc_link_group *lgr,
0590                           int *buf_lst)
0591 {
0592     *buf_lst = 0;
0593     return smc_llc_get_next_rmb(lgr, buf_lst, NULL);
0594 }
0595
0596 static int smc_llc_fill_ext_v2(struct smc_llc_msg_add_link_v2_ext *ext,
0597                    struct smc_link *link, struct smc_link *link_new)
0598 {
0599     struct smc_link_group *lgr = link->lgr;
0600     struct smc_buf_desc *buf_pos;
0601     int prim_lnk_idx, lnk_idx, i;
0602     struct smc_buf_desc *rmb;
0603     int len = sizeof(*ext);
0604     int buf_lst;
0605
0606     ext->v2_direct = !lgr->uses_gateway;
0607     memcpy(ext->client_target_gid, link_new->gid, SMC_GID_SIZE);
0608
0609     prim_lnk_idx = link->link_idx;
0610     lnk_idx = link_new->link_idx;
0611     mutex_lock(&lgr->rmbs_lock);
0612     ext->num_rkeys = lgr->conns_num;
0613     if (!ext->num_rkeys)
0614         goto out;
0615     buf_pos = smc_llc_get_first_rmb(lgr, &buf_lst);
0616     for (i = 0; i < ext->num_rkeys; i++) {
0617         if (!buf_pos)
0618             break;
0619         rmb = buf_pos;
0620         ext->rt[i].rmb_key = htonl(rmb->mr[prim_lnk_idx]->rkey);
0621         ext->rt[i].rmb_key_new = htonl(rmb->mr[lnk_idx]->rkey);
0622         ext->rt[i].rmb_vaddr_new = rmb->is_vm ?
0623             cpu_to_be64((uintptr_t)rmb->cpu_addr) :
0624             cpu_to_be64((u64)sg_dma_address(rmb->sgt[lnk_idx].sgl));
0625         buf_pos = smc_llc_get_next_rmb(lgr, &buf_lst, buf_pos);
0626         while (buf_pos && !(buf_pos)->used)
0627             buf_pos = smc_llc_get_next_rmb(lgr, &buf_lst, buf_pos);
0628     }
0629     len += i * sizeof(ext->rt[0]);
0630 out:
0631     mutex_unlock(&lgr->rmbs_lock);
0632     return len;
0633 }
0634
0635 /* send ADD LINK request or response */
0636 int smc_llc_send_add_link(struct smc_link *link, u8 mac[], u8 gid[],
0637               struct smc_link *link_new,
0638               enum smc_llc_reqresp reqresp)
0639 {
0640     struct smc_llc_msg_add_link_v2_ext *ext = NULL;
0641     struct smc_llc_msg_add_link *addllc;
0642     struct smc_wr_tx_pend_priv *pend;
0643     int len = sizeof(*addllc);
0644     int rc;
0645
0646     if (!smc_wr_tx_link_hold(link))
0647         return -ENOLINK;
0648     if (link->lgr->smc_version == SMC_V2) {
0649         struct smc_wr_v2_buf *wr_buf;
0650
0651         rc = smc_llc_add_pending_send_v2(link, &wr_buf, &pend);
0652         if (rc)
0653             goto put_out;
0654         addllc = (struct smc_llc_msg_add_link *)wr_buf;
0655         ext = (struct smc_llc_msg_add_link_v2_ext *)
0656                         &wr_buf->raw[sizeof(*addllc)];
0657         memset(ext, 0, SMC_WR_TX_SIZE);
0658     } else {
0659         struct smc_wr_buf *wr_buf;
0660
0661         rc = smc_llc_add_pending_send(link, &wr_buf, &pend);
0662         if (rc)
0663             goto put_out;
0664         addllc = (struct smc_llc_msg_add_link *)wr_buf;
0665     }
0666
0667     memset(addllc, 0, sizeof(*addllc));
0668     addllc->hd.common.llc_type = SMC_LLC_ADD_LINK;
0669     if (reqresp == SMC_LLC_RESP)
0670         addllc->hd.flags |= SMC_LLC_FLAG_RESP;
0671     memcpy(addllc->sender_mac, mac, ETH_ALEN);
0672     memcpy(addllc->sender_gid, gid, SMC_GID_SIZE);
0673     if (link_new) {
0674         addllc->link_num = link_new->link_id;
0675         hton24(addllc->sender_qp_num, link_new->roce_qp->qp_num);
0676         hton24(addllc->initial_psn, link_new->psn_initial);
0677         if (reqresp == SMC_LLC_REQ)
0678             addllc->qp_mtu = link_new->path_mtu;
0679         else
0680             addllc->qp_mtu = min(link_new->path_mtu,
0681                          link_new->peer_mtu);
0682     }
0683     if (ext && link_new)
0684         len += smc_llc_fill_ext_v2(ext, link, link_new);
0685     smc_llc_init_msg_hdr(&addllc->hd, link->lgr, len);
0686     /* send llc message */
0687     if (link->lgr->smc_version == SMC_V2)
0688         rc = smc_wr_tx_v2_send(link, pend, len);
0689     else
0690         rc = smc_wr_tx_send(link, pend);
0691 put_out:
0692     smc_wr_tx_link_put(link);
0693     return rc;
0694 }
0695
0696 /* send DELETE LINK request or response */
0697 int smc_llc_send_delete_link(struct smc_link *link, u8 link_del_id,
0698                  enum smc_llc_reqresp reqresp, bool orderly,
0699                  u32 reason)
0700 {
0701     struct smc_llc_msg_del_link *delllc;
0702     struct smc_wr_tx_pend_priv *pend;
0703     struct smc_wr_buf *wr_buf;
0704     int rc;
0705
0706     if (!smc_wr_tx_link_hold(link))
0707         return -ENOLINK;
0708     rc = smc_llc_add_pending_send(link, &wr_buf, &pend);
0709     if (rc)
0710         goto put_out;
0711     delllc = (struct smc_llc_msg_del_link *)wr_buf;
0712
0713     memset(delllc, 0, sizeof(*delllc));
0714     delllc->hd.common.llc_type = SMC_LLC_DELETE_LINK;
0715     smc_llc_init_msg_hdr(&delllc->hd, link->lgr, sizeof(*delllc));
0716     if (reqresp == SMC_LLC_RESP)
0717         delllc->hd.flags |= SMC_LLC_FLAG_RESP;
0718     if (orderly)
0719         delllc->hd.flags |= SMC_LLC_FLAG_DEL_LINK_ORDERLY;
0720     if (link_del_id)
0721         delllc->link_num = link_del_id;
0722     else
0723         delllc->hd.flags |= SMC_LLC_FLAG_DEL_LINK_ALL;
0724     delllc->reason = htonl(reason);
0725     /* send llc message */
0726     rc = smc_wr_tx_send(link, pend);
0727 put_out:
0728     smc_wr_tx_link_put(link);
0729     return rc;
0730 }
0731
0732 /* send LLC test link request */
0733 static int smc_llc_send_test_link(struct smc_link *link, u8 user_data[16])
0734 {
0735     struct smc_llc_msg_test_link *testllc;
0736     struct smc_wr_tx_pend_priv *pend;
0737     struct smc_wr_buf *wr_buf;
0738     int rc;
0739
0740     if (!smc_wr_tx_link_hold(link))
0741         return -ENOLINK;
0742     rc = smc_llc_add_pending_send(link, &wr_buf, &pend);
0743     if (rc)
0744         goto put_out;
0745     testllc = (struct smc_llc_msg_test_link *)wr_buf;
0746     memset(testllc, 0, sizeof(*testllc));
0747     testllc->hd.common.llc_type = SMC_LLC_TEST_LINK;
0748     smc_llc_init_msg_hdr(&testllc->hd, link->lgr, sizeof(*testllc));
0749     memcpy(testllc->user_data, user_data, sizeof(testllc->user_data));
0750     /* send llc message */
0751     rc = smc_wr_tx_send(link, pend);
0752 put_out:
0753     smc_wr_tx_link_put(link);
0754     return rc;
0755 }
0756
0757 /* schedule an llc send on link, may wait for buffers */
0758 static int smc_llc_send_message(struct smc_link *link, void *llcbuf)
0759 {
0760     struct smc_wr_tx_pend_priv *pend;
0761     struct smc_wr_buf *wr_buf;
0762     int rc;
0763
0764     if (!smc_wr_tx_link_hold(link))
0765         return -ENOLINK;
0766     rc = smc_llc_add_pending_send(link, &wr_buf, &pend);
0767     if (rc)
0768         goto put_out;
0769     memcpy(wr_buf, llcbuf, sizeof(union smc_llc_msg));
0770     rc = smc_wr_tx_send(link, pend);
0771 put_out:
0772     smc_wr_tx_link_put(link);
0773     return rc;
0774 }
0775
0776 /* schedule an llc send on link, may wait for buffers,
0777  * and wait for send completion notification.
0778  * @return 0 on success
0779  */
0780 static int smc_llc_send_message_wait(struct smc_link *link, void *llcbuf)
0781 {
0782     struct smc_wr_tx_pend_priv *pend;
0783     struct smc_wr_buf *wr_buf;
0784     int rc;
0785
0786     if (!smc_wr_tx_link_hold(link))
0787         return -ENOLINK;
0788     rc = smc_llc_add_pending_send(link, &wr_buf, &pend);
0789     if (rc)
0790         goto put_out;
0791     memcpy(wr_buf, llcbuf, sizeof(union smc_llc_msg));
0792     rc = smc_wr_tx_send_wait(link, pend, SMC_LLC_WAIT_TIME);
0793 put_out:
0794     smc_wr_tx_link_put(link);
0795     return rc;
0796 }
0797
0798 /********************************* receive ***********************************/
0799
0800 static int smc_llc_alloc_alt_link(struct smc_link_group *lgr,
0801                   enum smc_lgr_type lgr_new_t)
0802 {
0803     int i;
0804
0805     if (lgr->type == SMC_LGR_SYMMETRIC ||
0806         (lgr->type != SMC_LGR_SINGLE &&
0807          (lgr_new_t == SMC_LGR_ASYMMETRIC_LOCAL ||
0808           lgr_new_t == SMC_LGR_ASYMMETRIC_PEER)))
0809         return -EMLINK;
0810
0811     if (lgr_new_t == SMC_LGR_ASYMMETRIC_LOCAL ||
0812         lgr_new_t == SMC_LGR_ASYMMETRIC_PEER) {
0813         for (i = SMC_LINKS_PER_LGR_MAX - 1; i >= 0; i--)
0814             if (lgr->lnk[i].state == SMC_LNK_UNUSED)
0815                 return i;
0816     } else {
0817         for (i = 0; i < SMC_LINKS_PER_LGR_MAX; i++)
0818             if (lgr->lnk[i].state == SMC_LNK_UNUSED)
0819                 return i;
0820     }
0821     return -EMLINK;
0822 }
0823
0824 /* send one add_link_continue msg */
0825 static int smc_llc_add_link_cont(struct smc_link *link,
0826                  struct smc_link *link_new, u8 *num_rkeys_todo,
0827                  int *buf_lst, struct smc_buf_desc **buf_pos)
0828 {
0829     struct smc_llc_msg_add_link_cont *addc_llc;
0830     struct smc_link_group *lgr = link->lgr;
0831     int prim_lnk_idx, lnk_idx, i, rc;
0832     struct smc_wr_tx_pend_priv *pend;
0833     struct smc_wr_buf *wr_buf;
0834     struct smc_buf_desc *rmb;
0835     u8 n;
0836
0837     if (!smc_wr_tx_link_hold(link))
0838         return -ENOLINK;
0839     rc = smc_llc_add_pending_send(link, &wr_buf, &pend);
0840     if (rc)
0841         goto put_out;
0842     addc_llc = (struct smc_llc_msg_add_link_cont *)wr_buf;
0843     memset(addc_llc, 0, sizeof(*addc_llc));
0844
0845     prim_lnk_idx = link->link_idx;
0846     lnk_idx = link_new->link_idx;
0847     addc_llc->link_num = link_new->link_id;
0848     addc_llc->num_rkeys = *num_rkeys_todo;
0849     n = *num_rkeys_todo;
0850     for (i = 0; i < min_t(u8, n, SMC_LLC_RKEYS_PER_CONT_MSG); i++) {
0851         if (!*buf_pos) {
0852             addc_llc->num_rkeys = addc_llc->num_rkeys -
0853                           *num_rkeys_todo;
0854             *num_rkeys_todo = 0;
0855             break;
0856         }
0857         rmb = *buf_pos;
0858
0859         addc_llc->rt[i].rmb_key = htonl(rmb->mr[prim_lnk_idx]->rkey);
0860         addc_llc->rt[i].rmb_key_new = htonl(rmb->mr[lnk_idx]->rkey);
0861         addc_llc->rt[i].rmb_vaddr_new = rmb->is_vm ?
0862             cpu_to_be64((uintptr_t)rmb->cpu_addr) :
0863             cpu_to_be64((u64)sg_dma_address(rmb->sgt[lnk_idx].sgl));
0864
0865         (*num_rkeys_todo)--;
0866         *buf_pos = smc_llc_get_next_rmb(lgr, buf_lst, *buf_pos);
0867         while (*buf_pos && !(*buf_pos)->used)
0868             *buf_pos = smc_llc_get_next_rmb(lgr, buf_lst, *buf_pos);
0869     }
0870     addc_llc->hd.common.llc_type = SMC_LLC_ADD_LINK_CONT;
0871     addc_llc->hd.length = sizeof(struct smc_llc_msg_add_link_cont);
0872     if (lgr->role == SMC_CLNT)
0873         addc_llc->hd.flags |= SMC_LLC_FLAG_RESP;
0874     rc = smc_wr_tx_send(link, pend);
0875 put_out:
0876     smc_wr_tx_link_put(link);
0877     return rc;
0878 }
0879
0880 static int smc_llc_cli_rkey_exchange(struct smc_link *link,
0881                      struct smc_link *link_new)
0882 {
0883     struct smc_llc_msg_add_link_cont *addc_llc;
0884     struct smc_link_group *lgr = link->lgr;
0885     u8 max, num_rkeys_send, num_rkeys_recv;
0886     struct smc_llc_qentry *qentry;
0887     struct smc_buf_desc *buf_pos;
0888     int buf_lst;
0889     int rc = 0;
0890     int i;
0891
0892     mutex_lock(&lgr->rmbs_lock);
0893     num_rkeys_send = lgr->conns_num;
0894     buf_pos = smc_llc_get_first_rmb(lgr, &buf_lst);
0895     do {
0896         qentry = smc_llc_wait(lgr, NULL, SMC_LLC_WAIT_TIME,
0897                       SMC_LLC_ADD_LINK_CONT);
0898         if (!qentry) {
0899             rc = -ETIMEDOUT;
0900             break;
0901         }
0902         addc_llc = &qentry->msg.add_link_cont;
0903         num_rkeys_recv = addc_llc->num_rkeys;
0904         max = min_t(u8, num_rkeys_recv, SMC_LLC_RKEYS_PER_CONT_MSG);
0905         for (i = 0; i < max; i++) {
0906             smc_rtoken_set(lgr, link->link_idx, link_new->link_idx,
0907                        addc_llc->rt[i].rmb_key,
0908                        addc_llc->rt[i].rmb_vaddr_new,
0909                        addc_llc->rt[i].rmb_key_new);
0910             num_rkeys_recv--;
0911         }
0912         smc_llc_flow_qentry_del(&lgr->llc_flow_lcl);
0913         rc = smc_llc_add_link_cont(link, link_new, &num_rkeys_send,
0914                        &buf_lst, &buf_pos);
0915         if (rc)
0916             break;
0917     } while (num_rkeys_send || num_rkeys_recv);
0918
0919     mutex_unlock(&lgr->rmbs_lock);
0920     return rc;
0921 }
0922
0923 /* prepare and send an add link reject response */
0924 static int smc_llc_cli_add_link_reject(struct smc_llc_qentry *qentry)
0925 {
0926     qentry->msg.raw.hdr.flags |= SMC_LLC_FLAG_RESP;
0927     qentry->msg.raw.hdr.flags |= SMC_LLC_FLAG_ADD_LNK_REJ;
0928     qentry->msg.raw.hdr.add_link_rej_rsn = SMC_LLC_REJ_RSN_NO_ALT_PATH;
0929     smc_llc_init_msg_hdr(&qentry->msg.raw.hdr, qentry->link->lgr,
0930                  sizeof(qentry->msg));
0931     return smc_llc_send_message(qentry->link, &qentry->msg);
0932 }
0933
0934 static int smc_llc_cli_conf_link(struct smc_link *link,
0935                  struct smc_init_info *ini,
0936                  struct smc_link *link_new,
0937                  enum smc_lgr_type lgr_new_t)
0938 {
0939     struct smc_link_group *lgr = link->lgr;
0940     struct smc_llc_qentry *qentry = NULL;
0941     int rc = 0;
0942
0943     /* receive CONFIRM LINK request over RoCE fabric */
0944     qentry = smc_llc_wait(lgr, NULL, SMC_LLC_WAIT_FIRST_TIME, 0);
0945     if (!qentry) {
0946         rc = smc_llc_send_delete_link(link, link_new->link_id,
0947                           SMC_LLC_REQ, false,
0948                           SMC_LLC_DEL_LOST_PATH);
0949         return -ENOLINK;
0950     }
0951     if (qentry->msg.raw.hdr.common.llc_type != SMC_LLC_CONFIRM_LINK) {
0952         /* received DELETE_LINK instead */
0953         qentry->msg.raw.hdr.flags |= SMC_LLC_FLAG_RESP;
0954         smc_llc_send_message(link, &qentry->msg);
0955         smc_llc_flow_qentry_del(&lgr->llc_flow_lcl);
0956         return -ENOLINK;
0957     }
0958     smc_llc_save_peer_uid(qentry);
0959     smc_llc_flow_qentry_del(&lgr->llc_flow_lcl);
0960
0961     rc = smc_ib_modify_qp_rts(link_new);
0962     if (rc) {
0963         smc_llc_send_delete_link(link, link_new->link_id, SMC_LLC_REQ,
0964                      false, SMC_LLC_DEL_LOST_PATH);
0965         return -ENOLINK;
0966     }
0967     smc_wr_remember_qp_attr(link_new);
0968
0969     rc = smcr_buf_reg_lgr(link_new);
0970     if (rc) {
0971         smc_llc_send_delete_link(link, link_new->link_id, SMC_LLC_REQ,
0972                      false, SMC_LLC_DEL_LOST_PATH);
0973         return -ENOLINK;
0974     }
0975
0976     /* send CONFIRM LINK response over RoCE fabric */
0977     rc = smc_llc_send_confirm_link(link_new, SMC_LLC_RESP);
0978     if (rc) {
0979         smc_llc_send_delete_link(link, link_new->link_id, SMC_LLC_REQ,
0980                      false, SMC_LLC_DEL_LOST_PATH);
0981         return -ENOLINK;
0982     }
0983     smc_llc_link_active(link_new);
0984     if (lgr_new_t == SMC_LGR_ASYMMETRIC_LOCAL ||
0985         lgr_new_t == SMC_LGR_ASYMMETRIC_PEER)
0986         smcr_lgr_set_type_asym(lgr, lgr_new_t, link_new->link_idx);
0987     else
0988         smcr_lgr_set_type(lgr, lgr_new_t);
0989     return 0;
0990 }
0991
0992 static void smc_llc_save_add_link_rkeys(struct smc_link *link,
0993                     struct smc_link *link_new)
0994 {
0995     struct smc_llc_msg_add_link_v2_ext *ext;
0996     struct smc_link_group *lgr = link->lgr;
0997     int max, i;
0998
0999     ext = (struct smc_llc_msg_add_link_v2_ext *)((u8 *)lgr->wr_rx_buf_v2 +
1000                              SMC_WR_TX_SIZE);
1001     max = min_t(u8, ext->num_rkeys, SMC_LLC_RKEYS_PER_MSG_V2);
1002     mutex_lock(&lgr->rmbs_lock);
1003     for (i = 0; i < max; i++) {
1004         smc_rtoken_set(lgr, link->link_idx, link_new->link_idx,
1005                    ext->rt[i].rmb_key,
1006                    ext->rt[i].rmb_vaddr_new,
1007                    ext->rt[i].rmb_key_new);
1008     }
1009     mutex_unlock(&lgr->rmbs_lock);
1010 }
1011
1012 static void smc_llc_save_add_link_info(struct smc_link *link,
1013                        struct smc_llc_msg_add_link *add_llc)
1014 {
1015     link->peer_qpn = ntoh24(add_llc->sender_qp_num);
1016     memcpy(link->peer_gid, add_llc->sender_gid, SMC_GID_SIZE);
1017     memcpy(link->peer_mac, add_llc->sender_mac, ETH_ALEN);
1018     link->peer_psn = ntoh24(add_llc->initial_psn);
1019     link->peer_mtu = add_llc->qp_mtu;
1020 }
1021
1022 /* as an SMC client, process an add link request */
1023 int smc_llc_cli_add_link(struct smc_link *link, struct smc_llc_qentry *qentry)
1024 {
1025     struct smc_llc_msg_add_link *llc = &qentry->msg.add_link;
1026     enum smc_lgr_type lgr_new_t = SMC_LGR_SYMMETRIC;
1027     struct smc_link_group *lgr = smc_get_lgr(link);
1028     struct smc_init_info *ini = NULL;
1029     struct smc_link *lnk_new = NULL;
1030     int lnk_idx, rc = 0;
1031
1032     if (!llc->qp_mtu)
1033         goto out_reject;
1034
1035     ini = kzalloc(sizeof(*ini), GFP_KERNEL);
1036     if (!ini) {
1037         rc = -ENOMEM;
1038         goto out_reject;
1039     }
1040
1041     ini->vlan_id = lgr->vlan_id;
1042     if (lgr->smc_version == SMC_V2) {
1043         ini->check_smcrv2 = true;
1044         ini->smcrv2.saddr = lgr->saddr;
1045         ini->smcrv2.daddr = smc_ib_gid_to_ipv4(llc->sender_gid);
1046     }
1047     smc_pnet_find_alt_roce(lgr, ini, link->smcibdev);
1048     if (!memcmp(llc->sender_gid, link->peer_gid, SMC_GID_SIZE) &&
1049         (lgr->smc_version == SMC_V2 ||
1050          !memcmp(llc->sender_mac, link->peer_mac, ETH_ALEN))) {
1051         if (!ini->ib_dev && !ini->smcrv2.ib_dev_v2)
1052             goto out_reject;
1053         lgr_new_t = SMC_LGR_ASYMMETRIC_PEER;
1054     }
1055     if (lgr->smc_version == SMC_V2 && !ini->smcrv2.ib_dev_v2) {
1056         lgr_new_t = SMC_LGR_ASYMMETRIC_LOCAL;
1057         ini->smcrv2.ib_dev_v2 = link->smcibdev;
1058         ini->smcrv2.ib_port_v2 = link->ibport;
1059     } else if (lgr->smc_version < SMC_V2 && !ini->ib_dev) {
1060         lgr_new_t = SMC_LGR_ASYMMETRIC_LOCAL;
1061         ini->ib_dev = link->smcibdev;
1062         ini->ib_port = link->ibport;
1063     }
1064     lnk_idx = smc_llc_alloc_alt_link(lgr, lgr_new_t);
1065     if (lnk_idx < 0)
1066         goto out_reject;
1067     lnk_new = &lgr->lnk[lnk_idx];
1068     rc = smcr_link_init(lgr, lnk_new, lnk_idx, ini);
1069     if (rc)
1070         goto out_reject;
1071     smc_llc_save_add_link_info(lnk_new, llc);
1072     lnk_new->link_id = llc->link_num;   /* SMC server assigns link id */
1073     smc_llc_link_set_uid(lnk_new);
1074
1075     rc = smc_ib_ready_link(lnk_new);
1076     if (rc)
1077         goto out_clear_lnk;
1078
1079     rc = smcr_buf_map_lgr(lnk_new);
1080     if (rc)
1081         goto out_clear_lnk;
1082
1083     rc = smc_llc_send_add_link(link,
1084                    lnk_new->smcibdev->mac[lnk_new->ibport - 1],
1085                    lnk_new->gid, lnk_new, SMC_LLC_RESP);
1086     if (rc)
1087         goto out_clear_lnk;
1088     if (lgr->smc_version == SMC_V2) {
1089         smc_llc_save_add_link_rkeys(link, lnk_new);
1090     } else {
1091         rc = smc_llc_cli_rkey_exchange(link, lnk_new);
1092         if (rc) {
1093             rc = 0;
1094             goto out_clear_lnk;
1095         }
1096     }
1097     rc = smc_llc_cli_conf_link(link, ini, lnk_new, lgr_new_t);
1098     if (!rc)
1099         goto out;
1100 out_clear_lnk:
1101     lnk_new->state = SMC_LNK_INACTIVE;
1102     smcr_link_clear(lnk_new, false);
1103 out_reject:
1104     smc_llc_cli_add_link_reject(qentry);
1105 out:
1106     kfree(ini);
1107     kfree(qentry);
1108     return rc;
1109 }
1110
1111 static void smc_llc_send_request_add_link(struct smc_link *link)
1112 {
1113     struct smc_llc_msg_req_add_link_v2 *llc;
1114     struct smc_wr_tx_pend_priv *pend;
1115     struct smc_wr_v2_buf *wr_buf;
1116     struct smc_gidlist gidlist;
1117     int rc, len, i;
1118
1119     if (!smc_wr_tx_link_hold(link))
1120         return;
1121     if (link->lgr->type == SMC_LGR_SYMMETRIC ||
1122         link->lgr->type == SMC_LGR_ASYMMETRIC_PEER)
1123         goto put_out;
1124
1125     smc_fill_gid_list(link->lgr, &gidlist, link->smcibdev, link->gid);
1126     if (gidlist.len <= 1)
1127         goto put_out;
1128
1129     rc = smc_llc_add_pending_send_v2(link, &wr_buf, &pend);
1130     if (rc)
1131         goto put_out;
1132     llc = (struct smc_llc_msg_req_add_link_v2 *)wr_buf;
1133     memset(llc, 0, SMC_WR_TX_SIZE);
1134
1135     llc->hd.common.llc_type = SMC_LLC_REQ_ADD_LINK;
1136     for (i = 0; i < gidlist.len; i++)
1137         memcpy(llc->gid[i], gidlist.list[i], sizeof(gidlist.list[0]));
1138     llc->gid_cnt = gidlist.len;
1139     len = sizeof(*llc) + (gidlist.len * sizeof(gidlist.list[0]));
1140     smc_llc_init_msg_hdr(&llc->hd, link->lgr, len);
1141     rc = smc_wr_tx_v2_send(link, pend, len);
1142     if (!rc)
1143         /* set REQ_ADD_LINK flow and wait for response from peer */
1144         link->lgr->llc_flow_lcl.type = SMC_LLC_FLOW_REQ_ADD_LINK;
1145 put_out:
1146     smc_wr_tx_link_put(link);
1147 }
1148
1149 /* as an SMC client, invite server to start the add_link processing */
1150 static void smc_llc_cli_add_link_invite(struct smc_link *link,
1151                     struct smc_llc_qentry *qentry)
1152 {
1153     struct smc_link_group *lgr = smc_get_lgr(link);
1154     struct smc_init_info *ini = NULL;
1155
1156     if (lgr->smc_version == SMC_V2) {
1157         smc_llc_send_request_add_link(link);
1158         goto out;
1159     }
1160
1161     if (lgr->type == SMC_LGR_SYMMETRIC ||
1162         lgr->type == SMC_LGR_ASYMMETRIC_PEER)
1163         goto out;
1164
1165     ini = kzalloc(sizeof(*ini), GFP_KERNEL);
1166     if (!ini)
1167         goto out;
1168
1169     ini->vlan_id = lgr->vlan_id;
1170     smc_pnet_find_alt_roce(lgr, ini, link->smcibdev);
1171     if (!ini->ib_dev)
1172         goto out;
1173
1174     smc_llc_send_add_link(link, ini->ib_dev->mac[ini->ib_port - 1],
1175                   ini->ib_gid, NULL, SMC_LLC_REQ);
1176 out:
1177     kfree(ini);
1178     kfree(qentry);
1179 }
1180
1181 static bool smc_llc_is_empty_llc_message(union smc_llc_msg *llc)
1182 {
1183     int i;
1184
1185     for (i = 0; i < ARRAY_SIZE(llc->raw.data); i++)
1186         if (llc->raw.data[i])
1187             return false;
1188     return true;
1189 }
1190
1191 static bool smc_llc_is_local_add_link(union smc_llc_msg *llc)
1192 {
1193     if (llc->raw.hdr.common.llc_type == SMC_LLC_ADD_LINK &&
1194         smc_llc_is_empty_llc_message(llc))
1195         return true;
1196     return false;
1197 }
1198
1199 static void smc_llc_process_cli_add_link(struct smc_link_group *lgr)
1200 {
1201     struct smc_llc_qentry *qentry;
1202
1203     qentry = smc_llc_flow_qentry_clr(&lgr->llc_flow_lcl);
1204
1205     mutex_lock(&lgr->llc_conf_mutex);
1206     if (smc_llc_is_local_add_link(&qentry->msg))
1207         smc_llc_cli_add_link_invite(qentry->link, qentry);
1208     else
1209         smc_llc_cli_add_link(qentry->link, qentry);
1210     mutex_unlock(&lgr->llc_conf_mutex);
1211 }
1212
1213 static int smc_llc_active_link_count(struct smc_link_group *lgr)
1214 {
1215     int i, link_count = 0;
1216
1217     for (i = 0; i < SMC_LINKS_PER_LGR_MAX; i++) {
1218         if (!smc_link_active(&lgr->lnk[i]))
1219             continue;
1220         link_count++;
1221     }
1222     return link_count;
1223 }
1224
1225 /* find the asymmetric link when 3 links are established  */
1226 static struct smc_link *smc_llc_find_asym_link(struct smc_link_group *lgr)
1227 {
1228     int asym_idx = -ENOENT;
1229     int i, j, k;
1230     bool found;
1231
1232     /* determine asymmetric link */
1233     found = false;
1234     for (i = 0; i < SMC_LINKS_PER_LGR_MAX; i++) {
1235         for (j = i + 1; j < SMC_LINKS_PER_LGR_MAX; j++) {
1236             if (!smc_link_usable(&lgr->lnk[i]) ||
1237                 !smc_link_usable(&lgr->lnk[j]))
1238                 continue;
1239             if (!memcmp(lgr->lnk[i].gid, lgr->lnk[j].gid,
1240                     SMC_GID_SIZE)) {
1241                 found = true;   /* asym_lnk is i or j */
1242                 break;
1243             }
1244         }
1245         if (found)
1246             break;
1247     }
1248     if (!found)
1249         goto out; /* no asymmetric link */
1250     for (k = 0; k < SMC_LINKS_PER_LGR_MAX; k++) {
1251         if (!smc_link_usable(&lgr->lnk[k]))
1252             continue;
1253         if (k != i &&
1254             !memcmp(lgr->lnk[i].peer_gid, lgr->lnk[k].peer_gid,
1255                 SMC_GID_SIZE)) {
1256             asym_idx = i;
1257             break;
1258         }
1259         if (k != j &&
1260             !memcmp(lgr->lnk[j].peer_gid, lgr->lnk[k].peer_gid,
1261                 SMC_GID_SIZE)) {
1262             asym_idx = j;
1263             break;
1264         }
1265     }
1266 out:
1267     return (asym_idx < 0) ? NULL : &lgr->lnk[asym_idx];
1268 }
1269
1270 static void smc_llc_delete_asym_link(struct smc_link_group *lgr)
1271 {
1272     struct smc_link *lnk_new = NULL, *lnk_asym;
1273     struct smc_llc_qentry *qentry;
1274     int rc;
1275
1276     lnk_asym = smc_llc_find_asym_link(lgr);
1277     if (!lnk_asym)
1278         return; /* no asymmetric link */
1279     if (!smc_link_downing(&lnk_asym->state))
1280         return;
1281     lnk_new = smc_switch_conns(lgr, lnk_asym, false);
1282     smc_wr_tx_wait_no_pending_sends(lnk_asym);
1283     if (!lnk_new)
1284         goto out_free;
1285     /* change flow type from ADD_LINK into DEL_LINK */
1286     lgr->llc_flow_lcl.type = SMC_LLC_FLOW_DEL_LINK;
1287     rc = smc_llc_send_delete_link(lnk_new, lnk_asym->link_id, SMC_LLC_REQ,
1288                       true, SMC_LLC_DEL_NO_ASYM_NEEDED);
1289     if (rc) {
1290         smcr_link_down_cond(lnk_new);
1291         goto out_free;
1292     }
1293     qentry = smc_llc_wait(lgr, lnk_new, SMC_LLC_WAIT_TIME,
1294                   SMC_LLC_DELETE_LINK);
1295     if (!qentry) {
1296         smcr_link_down_cond(lnk_new);
1297         goto out_free;
1298     }
1299     smc_llc_flow_qentry_del(&lgr->llc_flow_lcl);
1300 out_free:
1301     smcr_link_clear(lnk_asym, true);
1302 }
1303
1304 static int smc_llc_srv_rkey_exchange(struct smc_link *link,
1305                      struct smc_link *link_new)
1306 {
1307     struct smc_llc_msg_add_link_cont *addc_llc;
1308     struct smc_link_group *lgr = link->lgr;
1309     u8 max, num_rkeys_send, num_rkeys_recv;
1310     struct smc_llc_qentry *qentry = NULL;
1311     struct smc_buf_desc *buf_pos;
1312     int buf_lst;
1313     int rc = 0;
1314     int i;
1315
1316     mutex_lock(&lgr->rmbs_lock);
1317     num_rkeys_send = lgr->conns_num;
1318     buf_pos = smc_llc_get_first_rmb(lgr, &buf_lst);
1319     do {
1320         smc_llc_add_link_cont(link, link_new, &num_rkeys_send,
1321                       &buf_lst, &buf_pos);
1322         qentry = smc_llc_wait(lgr, link, SMC_LLC_WAIT_TIME,
1323                       SMC_LLC_ADD_LINK_CONT);
1324         if (!qentry) {
1325             rc = -ETIMEDOUT;
1326             goto out;
1327         }
1328         addc_llc = &qentry->msg.add_link_cont;
1329         num_rkeys_recv = addc_llc->num_rkeys;
1330         max = min_t(u8, num_rkeys_recv, SMC_LLC_RKEYS_PER_CONT_MSG);
1331         for (i = 0; i < max; i++) {
1332             smc_rtoken_set(lgr, link->link_idx, link_new->link_idx,
1333                        addc_llc->rt[i].rmb_key,
1334                        addc_llc->rt[i].rmb_vaddr_new,
1335                        addc_llc->rt[i].rmb_key_new);
1336             num_rkeys_recv--;
1337         }
1338         smc_llc_flow_qentry_del(&lgr->llc_flow_lcl);
1339     } while (num_rkeys_send || num_rkeys_recv);
1340 out:
1341     mutex_unlock(&lgr->rmbs_lock);
1342     return rc;
1343 }
1344
1345 static int smc_llc_srv_conf_link(struct smc_link *link,
1346                  struct smc_link *link_new,
1347                  enum smc_lgr_type lgr_new_t)
1348 {
1349     struct smc_link_group *lgr = link->lgr;
1350     struct smc_llc_qentry *qentry = NULL;
1351     int rc;
1352
1353     /* send CONFIRM LINK request over the RoCE fabric */
1354     rc = smc_llc_send_confirm_link(link_new, SMC_LLC_REQ);
1355     if (rc)
1356         return -ENOLINK;
1357     /* receive CONFIRM LINK response over the RoCE fabric */
1358     qentry = smc_llc_wait(lgr, link, SMC_LLC_WAIT_FIRST_TIME, 0);
1359     if (!qentry ||
1360         qentry->msg.raw.hdr.common.llc_type != SMC_LLC_CONFIRM_LINK) {
1361         /* send DELETE LINK */
1362         smc_llc_send_delete_link(link, link_new->link_id, SMC_LLC_REQ,
1363                      false, SMC_LLC_DEL_LOST_PATH);
1364         if (qentry)
1365             smc_llc_flow_qentry_del(&lgr->llc_flow_lcl);
1366         return -ENOLINK;
1367     }
1368     smc_llc_save_peer_uid(qentry);
1369     smc_llc_link_active(link_new);
1370     if (lgr_new_t == SMC_LGR_ASYMMETRIC_LOCAL ||
1371         lgr_new_t == SMC_LGR_ASYMMETRIC_PEER)
1372         smcr_lgr_set_type_asym(lgr, lgr_new_t, link_new->link_idx);
1373     else
1374         smcr_lgr_set_type(lgr, lgr_new_t);
1375     smc_llc_flow_qentry_del(&lgr->llc_flow_lcl);
1376     return 0;
1377 }
1378
1379 static void smc_llc_send_req_add_link_response(struct smc_llc_qentry *qentry)
1380 {
1381     qentry->msg.raw.hdr.flags |= SMC_LLC_FLAG_RESP;
1382     smc_llc_init_msg_hdr(&qentry->msg.raw.hdr, qentry->link->lgr,
1383                  sizeof(qentry->msg));
1384     memset(&qentry->msg.raw.data, 0, sizeof(qentry->msg.raw.data));
1385     smc_llc_send_message(qentry->link, &qentry->msg);
1386 }
1387
1388 int smc_llc_srv_add_link(struct smc_link *link,
1389              struct smc_llc_qentry *req_qentry)
1390 {
1391     enum smc_lgr_type lgr_new_t = SMC_LGR_SYMMETRIC;
1392     struct smc_link_group *lgr = link->lgr;
1393     struct smc_llc_msg_add_link *add_llc;
1394     struct smc_llc_qentry *qentry = NULL;
1395     bool send_req_add_link_resp = false;
1396     struct smc_link *link_new = NULL;
1397     struct smc_init_info *ini = NULL;
1398     int lnk_idx, rc = 0;
1399
1400     if (req_qentry &&
1401         req_qentry->msg.raw.hdr.common.llc_type == SMC_LLC_REQ_ADD_LINK)
1402         send_req_add_link_resp = true;
1403
1404     ini = kzalloc(sizeof(*ini), GFP_KERNEL);
1405     if (!ini) {
1406         rc = -ENOMEM;
1407         goto out;
1408     }
1409
1410     /* ignore client add link recommendation, start new flow */
1411     ini->vlan_id = lgr->vlan_id;
1412     if (lgr->smc_version == SMC_V2) {
1413         ini->check_smcrv2 = true;
1414         ini->smcrv2.saddr = lgr->saddr;
1415         if (send_req_add_link_resp) {
1416             struct smc_llc_msg_req_add_link_v2 *req_add =
1417                 &req_qentry->msg.req_add_link;
1418
1419             ini->smcrv2.daddr = smc_ib_gid_to_ipv4(req_add->gid[0]);
1420         }
1421     }
1422     smc_pnet_find_alt_roce(lgr, ini, link->smcibdev);
1423     if (lgr->smc_version == SMC_V2 && !ini->smcrv2.ib_dev_v2) {
1424         lgr_new_t = SMC_LGR_ASYMMETRIC_LOCAL;
1425         ini->smcrv2.ib_dev_v2 = link->smcibdev;
1426         ini->smcrv2.ib_port_v2 = link->ibport;
1427     } else if (lgr->smc_version < SMC_V2 && !ini->ib_dev) {
1428         lgr_new_t = SMC_LGR_ASYMMETRIC_LOCAL;
1429         ini->ib_dev = link->smcibdev;
1430         ini->ib_port = link->ibport;
1431     }
1432     lnk_idx = smc_llc_alloc_alt_link(lgr, lgr_new_t);
1433     if (lnk_idx < 0) {
1434         rc = 0;
1435         goto out;
1436     }
1437
1438     rc = smcr_link_init(lgr, &lgr->lnk[lnk_idx], lnk_idx, ini);
1439     if (rc)
1440         goto out;
1441     link_new = &lgr->lnk[lnk_idx];
1442
1443     rc = smcr_buf_map_lgr(link_new);
1444     if (rc)
1445         goto out_err;
1446
1447     rc = smc_llc_send_add_link(link,
1448                    link_new->smcibdev->mac[link_new->ibport-1],
1449                    link_new->gid, link_new, SMC_LLC_REQ);
1450     if (rc)
1451         goto out_err;
1452     send_req_add_link_resp = false;
1453     /* receive ADD LINK response over the RoCE fabric */
1454     qentry = smc_llc_wait(lgr, link, SMC_LLC_WAIT_TIME, SMC_LLC_ADD_LINK);
1455     if (!qentry) {
1456         rc = -ETIMEDOUT;
1457         goto out_err;
1458     }
1459     add_llc = &qentry->msg.add_link;
1460     if (add_llc->hd.flags & SMC_LLC_FLAG_ADD_LNK_REJ) {
1461         smc_llc_flow_qentry_del(&lgr->llc_flow_lcl);
1462         rc = -ENOLINK;
1463         goto out_err;
1464     }
1465     if (lgr->type == SMC_LGR_SINGLE &&
1466         (!memcmp(add_llc->sender_gid, link->peer_gid, SMC_GID_SIZE) &&
1467          (lgr->smc_version == SMC_V2 ||
1468           !memcmp(add_llc->sender_mac, link->peer_mac, ETH_ALEN)))) {
1469         lgr_new_t = SMC_LGR_ASYMMETRIC_PEER;
1470     }
1471     smc_llc_save_add_link_info(link_new, add_llc);
1472     smc_llc_flow_qentry_del(&lgr->llc_flow_lcl);
1473
1474     rc = smc_ib_ready_link(link_new);
1475     if (rc)
1476         goto out_err;
1477     rc = smcr_buf_reg_lgr(link_new);
1478     if (rc)
1479         goto out_err;
1480     if (lgr->smc_version == SMC_V2) {
1481         smc_llc_save_add_link_rkeys(link, link_new);
1482     } else {
1483         rc = smc_llc_srv_rkey_exchange(link, link_new);
1484         if (rc)
1485             goto out_err;
1486     }
1487     rc = smc_llc_srv_conf_link(link, link_new, lgr_new_t);
1488     if (rc)
1489         goto out_err;
1490     kfree(ini);
1491     return 0;
1492 out_err:
1493     if (link_new) {
1494         link_new->state = SMC_LNK_INACTIVE;
1495         smcr_link_clear(link_new, false);
1496     }
1497 out:
1498     kfree(ini);
1499     if (send_req_add_link_resp)
1500         smc_llc_send_req_add_link_response(req_qentry);
1501     return rc;
1502 }
1503
1504 static void smc_llc_process_srv_add_link(struct smc_link_group *lgr)
1505 {
1506     struct smc_link *link = lgr->llc_flow_lcl.qentry->link;
1507     struct smc_llc_qentry *qentry;
1508     int rc;
1509
1510     qentry = smc_llc_flow_qentry_clr(&lgr->llc_flow_lcl);
1511
1512     mutex_lock(&lgr->llc_conf_mutex);
1513     rc = smc_llc_srv_add_link(link, qentry);
1514     if (!rc && lgr->type == SMC_LGR_SYMMETRIC) {
1515         /* delete any asymmetric link */
1516         smc_llc_delete_asym_link(lgr);
1517     }
1518     mutex_unlock(&lgr->llc_conf_mutex);
1519     kfree(qentry);
1520 }
1521
1522 /* enqueue a local add_link req to trigger a new add_link flow */
1523 void smc_llc_add_link_local(struct smc_link *link)
1524 {
1525     struct smc_llc_msg_add_link add_llc = {};
1526
1527     add_llc.hd.common.llc_type = SMC_LLC_ADD_LINK;
1528     smc_llc_init_msg_hdr(&add_llc.hd, link->lgr, sizeof(add_llc));
1529     /* no dev and port needed */
1530     smc_llc_enqueue(link, (union smc_llc_msg *)&add_llc);
1531 }
1532
1533 /* worker to process an add link message */
1534 static void smc_llc_add_link_work(struct work_struct *work)
1535 {
1536     struct smc_link_group *lgr = container_of(work, struct smc_link_group,
1537                           llc_add_link_work);
1538
1539     if (list_empty(&lgr->list)) {
1540         /* link group is terminating */
1541         smc_llc_flow_qentry_del(&lgr->llc_flow_lcl);
1542         goto out;
1543     }
1544
1545     if (lgr->role == SMC_CLNT)
1546         smc_llc_process_cli_add_link(lgr);
1547     else
1548         smc_llc_process_srv_add_link(lgr);
1549 out:
1550     if (lgr->llc_flow_lcl.type != SMC_LLC_FLOW_REQ_ADD_LINK)
1551         smc_llc_flow_stop(lgr, &lgr->llc_flow_lcl);
1552 }
1553
1554 /* enqueue a local del_link msg to trigger a new del_link flow,
1555  * called only for role SMC_SERV
1556  */
1557 void smc_llc_srv_delete_link_local(struct smc_link *link, u8 del_link_id)
1558 {
1559     struct smc_llc_msg_del_link del_llc = {};
1560
1561     del_llc.hd.common.llc_type = SMC_LLC_DELETE_LINK;
1562     smc_llc_init_msg_hdr(&del_llc.hd, link->lgr, sizeof(del_llc));
1563     del_llc.link_num = del_link_id;
1564     del_llc.reason = htonl(SMC_LLC_DEL_LOST_PATH);
1565     del_llc.hd.flags |= SMC_LLC_FLAG_DEL_LINK_ORDERLY;
1566     smc_llc_enqueue(link, (union smc_llc_msg *)&del_llc);
1567 }
1568
1569 static void smc_llc_process_cli_delete_link(struct smc_link_group *lgr)
1570 {
1571     struct smc_link *lnk_del = NULL, *lnk_asym, *lnk;
1572     struct smc_llc_msg_del_link *del_llc;
1573     struct smc_llc_qentry *qentry;
1574     int active_links;
1575     int lnk_idx;
1576
1577     qentry = smc_llc_flow_qentry_clr(&lgr->llc_flow_lcl);
1578     lnk = qentry->link;
1579     del_llc = &qentry->msg.delete_link;
1580
1581     if (del_llc->hd.flags & SMC_LLC_FLAG_DEL_LINK_ALL) {
1582         smc_lgr_terminate_sched(lgr);
1583         goto out;
1584     }
1585     mutex_lock(&lgr->llc_conf_mutex);
1586     /* delete single link */
1587     for (lnk_idx = 0; lnk_idx < SMC_LINKS_PER_LGR_MAX; lnk_idx++) {
1588         if (lgr->lnk[lnk_idx].link_id != del_llc->link_num)
1589             continue;
1590         lnk_del = &lgr->lnk[lnk_idx];
1591         break;
1592     }
1593     del_llc->hd.flags |= SMC_LLC_FLAG_RESP;
1594     if (!lnk_del) {
1595         /* link was not found */
1596         del_llc->reason = htonl(SMC_LLC_DEL_NOLNK);
1597         smc_llc_send_message(lnk, &qentry->msg);
1598         goto out_unlock;
1599     }
1600     lnk_asym = smc_llc_find_asym_link(lgr);
1601
1602     del_llc->reason = 0;
1603     smc_llc_send_message(lnk, &qentry->msg); /* response */
1604
1605     if (smc_link_downing(&lnk_del->state))
1606         smc_switch_conns(lgr, lnk_del, false);
1607     smcr_link_clear(lnk_del, true);
1608
1609     active_links = smc_llc_active_link_count(lgr);
1610     if (lnk_del == lnk_asym) {
1611         /* expected deletion of asym link, don't change lgr state */
1612     } else if (active_links == 1) {
1613         smcr_lgr_set_type(lgr, SMC_LGR_SINGLE);
1614     } else if (!active_links) {
1615         smcr_lgr_set_type(lgr, SMC_LGR_NONE);
1616         smc_lgr_terminate_sched(lgr);
1617     }
1618 out_unlock:
1619     mutex_unlock(&lgr->llc_conf_mutex);
1620 out:
1621     kfree(qentry);
1622 }
1623
1624 /* try to send a DELETE LINK ALL request on any active link,
1625  * waiting for send completion
1626  */
1627 void smc_llc_send_link_delete_all(struct smc_link_group *lgr, bool ord, u32 rsn)
1628 {
1629     struct smc_llc_msg_del_link delllc = {};
1630     int i;
1631
1632     delllc.hd.common.llc_type = SMC_LLC_DELETE_LINK;
1633     smc_llc_init_msg_hdr(&delllc.hd, lgr, sizeof(delllc));
1634     if (ord)
1635         delllc.hd.flags |= SMC_LLC_FLAG_DEL_LINK_ORDERLY;
1636     delllc.hd.flags |= SMC_LLC_FLAG_DEL_LINK_ALL;
1637     delllc.reason = htonl(rsn);
1638
1639     for (i = 0; i < SMC_LINKS_PER_LGR_MAX; i++) {
1640         if (!smc_link_sendable(&lgr->lnk[i]))
1641             continue;
1642         if (!smc_llc_send_message_wait(&lgr->lnk[i], &delllc))
1643             break;
1644     }
1645 }
1646
1647 static void smc_llc_process_srv_delete_link(struct smc_link_group *lgr)
1648 {
1649     struct smc_llc_msg_del_link *del_llc;
1650     struct smc_link *lnk, *lnk_del;
1651     struct smc_llc_qentry *qentry;
1652     int active_links;
1653     int i;
1654
1655     mutex_lock(&lgr->llc_conf_mutex);
1656     qentry = smc_llc_flow_qentry_clr(&lgr->llc_flow_lcl);
1657     lnk = qentry->link;
1658     del_llc = &qentry->msg.delete_link;
1659
1660     if (qentry->msg.delete_link.hd.flags & SMC_LLC_FLAG_DEL_LINK_ALL) {
1661         /* delete entire lgr */
1662         smc_llc_send_link_delete_all(lgr, true, ntohl(
1663                           qentry->msg.delete_link.reason));
1664         smc_lgr_terminate_sched(lgr);
1665         goto out;
1666     }
1667     /* delete single link */
1668     lnk_del = NULL;
1669     for (i = 0; i < SMC_LINKS_PER_LGR_MAX; i++) {
1670         if (lgr->lnk[i].link_id == del_llc->link_num) {
1671             lnk_del = &lgr->lnk[i];
1672             break;
1673         }
1674     }
1675     if (!lnk_del)
1676         goto out; /* asymmetric link already deleted */
1677
1678     if (smc_link_downing(&lnk_del->state)) {
1679         if (smc_switch_conns(lgr, lnk_del, false))
1680             smc_wr_tx_wait_no_pending_sends(lnk_del);
1681     }
1682     if (!list_empty(&lgr->list)) {
1683         /* qentry is either a request from peer (send it back to
1684          * initiate the DELETE_LINK processing), or a locally
1685          * enqueued DELETE_LINK request (forward it)
1686          */
1687         if (!smc_llc_send_message(lnk, &qentry->msg)) {
1688             struct smc_llc_qentry *qentry2;
1689
1690             qentry2 = smc_llc_wait(lgr, lnk, SMC_LLC_WAIT_TIME,
1691                            SMC_LLC_DELETE_LINK);
1692             if (qentry2)
1693                 smc_llc_flow_qentry_del(&lgr->llc_flow_lcl);
1694         }
1695     }
1696     smcr_link_clear(lnk_del, true);
1697
1698     active_links = smc_llc_active_link_count(lgr);
1699     if (active_links == 1) {
1700         smcr_lgr_set_type(lgr, SMC_LGR_SINGLE);
1701     } else if (!active_links) {
1702         smcr_lgr_set_type(lgr, SMC_LGR_NONE);
1703         smc_lgr_terminate_sched(lgr);
1704     }
1705
1706     if (lgr->type == SMC_LGR_SINGLE && !list_empty(&lgr->list)) {
1707         /* trigger setup of asymm alt link */
1708         smc_llc_add_link_local(lnk);
1709     }
1710 out:
1711     mutex_unlock(&lgr->llc_conf_mutex);
1712     kfree(qentry);
1713 }
1714
1715 static void smc_llc_delete_link_work(struct work_struct *work)
1716 {
1717     struct smc_link_group *lgr = container_of(work, struct smc_link_group,
1718                           llc_del_link_work);
1719
1720     if (list_empty(&lgr->list)) {
1721         /* link group is terminating */
1722         smc_llc_flow_qentry_del(&lgr->llc_flow_lcl);
1723         goto out;
1724     }
1725
1726     if (lgr->role == SMC_CLNT)
1727         smc_llc_process_cli_delete_link(lgr);
1728     else
1729         smc_llc_process_srv_delete_link(lgr);
1730 out:
1731     smc_llc_flow_stop(lgr, &lgr->llc_flow_lcl);
1732 }
1733
1734 /* process a confirm_rkey request from peer, remote flow */
1735 static void smc_llc_rmt_conf_rkey(struct smc_link_group *lgr)
1736 {
1737     struct smc_llc_msg_confirm_rkey *llc;
1738     struct smc_llc_qentry *qentry;
1739     struct smc_link *link;
1740     int num_entries;
1741     int rk_idx;
1742     int i;
1743
1744     qentry = lgr->llc_flow_rmt.qentry;
1745     llc = &qentry->msg.confirm_rkey;
1746     link = qentry->link;
1747
1748     num_entries = llc->rtoken[0].num_rkeys;
1749     if (num_entries > SMC_LLC_RKEYS_PER_MSG)
1750         goto out_err;
1751     /* first rkey entry is for receiving link */
1752     rk_idx = smc_rtoken_add(link,
1753                 llc->rtoken[0].rmb_vaddr,
1754                 llc->rtoken[0].rmb_key);
1755     if (rk_idx < 0)
1756         goto out_err;
1757
1758     for (i = 1; i <= min_t(u8, num_entries, SMC_LLC_RKEYS_PER_MSG - 1); i++)
1759         smc_rtoken_set2(lgr, rk_idx, llc->rtoken[i].link_id,
1760                 llc->rtoken[i].rmb_vaddr,
1761                 llc->rtoken[i].rmb_key);
1762     /* max links is 3 so there is no need to support conf_rkey_cont msgs */
1763     goto out;
1764 out_err:
1765     llc->hd.flags |= SMC_LLC_FLAG_RKEY_NEG;
1766     llc->hd.flags |= SMC_LLC_FLAG_RKEY_RETRY;
1767 out:
1768     llc->hd.flags |= SMC_LLC_FLAG_RESP;
1769     smc_llc_init_msg_hdr(&llc->hd, link->lgr, sizeof(*llc));
1770     smc_llc_send_message(link, &qentry->msg);
1771     smc_llc_flow_qentry_del(&lgr->llc_flow_rmt);
1772 }
1773
1774 /* process a delete_rkey request from peer, remote flow */
1775 static void smc_llc_rmt_delete_rkey(struct smc_link_group *lgr)
1776 {
1777     struct smc_llc_msg_delete_rkey *llc;
1778     struct smc_llc_qentry *qentry;
1779     struct smc_link *link;
1780     u8 err_mask = 0;
1781     int i, max;
1782
1783     qentry = lgr->llc_flow_rmt.qentry;
1784     llc = &qentry->msg.delete_rkey;
1785     link = qentry->link;
1786
1787     if (lgr->smc_version == SMC_V2) {
1788         struct smc_llc_msg_delete_rkey_v2 *llcv2;
1789
1790         memcpy(lgr->wr_rx_buf_v2, llc, sizeof(*llc));
1791         llcv2 = (struct smc_llc_msg_delete_rkey_v2 *)lgr->wr_rx_buf_v2;
1792         llcv2->num_inval_rkeys = 0;
1793
1794         max = min_t(u8, llcv2->num_rkeys, SMC_LLC_RKEYS_PER_MSG_V2);
1795         for (i = 0; i < max; i++) {
1796             if (smc_rtoken_delete(link, llcv2->rkey[i]))
1797                 llcv2->num_inval_rkeys++;
1798         }
1799         memset(&llc->rkey[0], 0, sizeof(llc->rkey));
1800         memset(&llc->reserved2, 0, sizeof(llc->reserved2));
1801         smc_llc_init_msg_hdr(&llc->hd, link->lgr, sizeof(*llc));
1802         if (llcv2->num_inval_rkeys) {
1803             llc->hd.flags |= SMC_LLC_FLAG_RKEY_NEG;
1804             llc->err_mask = llcv2->num_inval_rkeys;
1805         }
1806         goto finish;
1807     }
1808
1809     max = min_t(u8, llc->num_rkeys, SMC_LLC_DEL_RKEY_MAX);
1810     for (i = 0; i < max; i++) {
1811         if (smc_rtoken_delete(link, llc->rkey[i]))
1812             err_mask |= 1 << (SMC_LLC_DEL_RKEY_MAX - 1 - i);
1813     }
1814     if (err_mask) {
1815         llc->hd.flags |= SMC_LLC_FLAG_RKEY_NEG;
1816         llc->err_mask = err_mask;
1817     }
1818 finish:
1819     llc->hd.flags |= SMC_LLC_FLAG_RESP;
1820     smc_llc_send_message(link, &qentry->msg);
1821     smc_llc_flow_qentry_del(&lgr->llc_flow_rmt);
1822 }
1823
1824 static void smc_llc_protocol_violation(struct smc_link_group *lgr, u8 type)
1825 {
1826     pr_warn_ratelimited("smc: SMC-R lg %*phN net %llu LLC protocol violation: "
1827                 "llc_type %d\n", SMC_LGR_ID_SIZE, &lgr->id,
1828                 lgr->net->net_cookie, type);
1829     smc_llc_set_termination_rsn(lgr, SMC_LLC_DEL_PROT_VIOL);
1830     smc_lgr_terminate_sched(lgr);
1831 }
1832
1833 /* flush the llc event queue */
1834 static void smc_llc_event_flush(struct smc_link_group *lgr)
1835 {
1836     struct smc_llc_qentry *qentry, *q;
1837
1838     spin_lock_bh(&lgr->llc_event_q_lock);
1839     list_for_each_entry_safe(qentry, q, &lgr->llc_event_q, list) {
1840         list_del_init(&qentry->list);
1841         kfree(qentry);
1842     }
1843     spin_unlock_bh(&lgr->llc_event_q_lock);
1844 }
1845
1846 static void smc_llc_event_handler(struct smc_llc_qentry *qentry)
1847 {
1848     union smc_llc_msg *llc = &qentry->msg;
1849     struct smc_link *link = qentry->link;
1850     struct smc_link_group *lgr = link->lgr;
1851
1852     if (!smc_link_usable(link))
1853         goto out;
1854
1855     switch (llc->raw.hdr.common.llc_type) {
1856     case SMC_LLC_TEST_LINK:
1857         llc->test_link.hd.flags |= SMC_LLC_FLAG_RESP;
1858         smc_llc_send_message(link, llc);
1859         break;
1860     case SMC_LLC_ADD_LINK:
1861         if (list_empty(&lgr->list))
1862             goto out;   /* lgr is terminating */
1863         if (lgr->role == SMC_CLNT) {
1864             if (smc_llc_is_local_add_link(llc)) {
1865                 if (lgr->llc_flow_lcl.type ==
1866                     SMC_LLC_FLOW_ADD_LINK)
1867                     break;  /* add_link in progress */
1868                 if (smc_llc_flow_start(&lgr->llc_flow_lcl,
1869                                qentry)) {
1870                     schedule_work(&lgr->llc_add_link_work);
1871                 }
1872                 return;
1873             }
1874             if (lgr->llc_flow_lcl.type == SMC_LLC_FLOW_ADD_LINK &&
1875                 !lgr->llc_flow_lcl.qentry) {
1876                 /* a flow is waiting for this message */
1877                 smc_llc_flow_qentry_set(&lgr->llc_flow_lcl,
1878                             qentry);
1879                 wake_up(&lgr->llc_msg_waiter);
1880                 return;
1881             }
1882             if (lgr->llc_flow_lcl.type ==
1883                     SMC_LLC_FLOW_REQ_ADD_LINK) {
1884                 /* server started add_link processing */
1885                 lgr->llc_flow_lcl.type = SMC_LLC_FLOW_ADD_LINK;
1886                 smc_llc_flow_qentry_set(&lgr->llc_flow_lcl,
1887                             qentry);
1888                 schedule_work(&lgr->llc_add_link_work);
1889                 return;
1890             }
1891             if (smc_llc_flow_start(&lgr->llc_flow_lcl, qentry)) {
1892                 schedule_work(&lgr->llc_add_link_work);
1893             }
1894         } else if (smc_llc_flow_start(&lgr->llc_flow_lcl, qentry)) {
1895             /* as smc server, handle client suggestion */
1896             schedule_work(&lgr->llc_add_link_work);
1897         }
1898         return;
1899     case SMC_LLC_CONFIRM_LINK:
1900     case SMC_LLC_ADD_LINK_CONT:
1901         if (lgr->llc_flow_lcl.type != SMC_LLC_FLOW_NONE) {
1902             /* a flow is waiting for this message */
1903             smc_llc_flow_qentry_set(&lgr->llc_flow_lcl, qentry);
1904             wake_up(&lgr->llc_msg_waiter);
1905             return;
1906         }
1907         break;
1908     case SMC_LLC_DELETE_LINK:
1909         if (lgr->llc_flow_lcl.type == SMC_LLC_FLOW_ADD_LINK &&
1910             !lgr->llc_flow_lcl.qentry) {
1911             /* DEL LINK REQ during ADD LINK SEQ */
1912             smc_llc_flow_qentry_set(&lgr->llc_flow_lcl, qentry);
1913             wake_up(&lgr->llc_msg_waiter);
1914         } else if (smc_llc_flow_start(&lgr->llc_flow_lcl, qentry)) {
1915             schedule_work(&lgr->llc_del_link_work);
1916         }
1917         return;
1918     case SMC_LLC_CONFIRM_RKEY:
1919         /* new request from remote, assign to remote flow */
1920         if (smc_llc_flow_start(&lgr->llc_flow_rmt, qentry)) {
1921             /* process here, does not wait for more llc msgs */
1922             smc_llc_rmt_conf_rkey(lgr);
1923             smc_llc_flow_stop(lgr, &lgr->llc_flow_rmt);
1924         }
1925         return;
1926     case SMC_LLC_CONFIRM_RKEY_CONT:
1927         /* not used because max links is 3, and 3 rkeys fit into
1928          * one CONFIRM_RKEY message
1929          */
1930         break;
1931     case SMC_LLC_DELETE_RKEY:
1932         /* new request from remote, assign to remote flow */
1933         if (smc_llc_flow_start(&lgr->llc_flow_rmt, qentry)) {
1934             /* process here, does not wait for more llc msgs */
1935             smc_llc_rmt_delete_rkey(lgr);
1936             smc_llc_flow_stop(lgr, &lgr->llc_flow_rmt);
1937         }
1938         return;
1939     case SMC_LLC_REQ_ADD_LINK:
1940         /* handle response here, smc_llc_flow_stop() cannot be called
1941          * in tasklet context
1942          */
1943         if (lgr->role == SMC_CLNT &&
1944             lgr->llc_flow_lcl.type == SMC_LLC_FLOW_REQ_ADD_LINK &&
1945             (llc->raw.hdr.flags & SMC_LLC_FLAG_RESP)) {
1946             smc_llc_flow_stop(link->lgr, &lgr->llc_flow_lcl);
1947         } else if (lgr->role == SMC_SERV) {
1948             if (smc_llc_flow_start(&lgr->llc_flow_lcl, qentry)) {
1949                 /* as smc server, handle client suggestion */
1950                 lgr->llc_flow_lcl.type = SMC_LLC_FLOW_ADD_LINK;
1951                 schedule_work(&lgr->llc_add_link_work);
1952             }
1953             return;
1954         }
1955         break;
1956     default:
1957         smc_llc_protocol_violation(lgr, llc->raw.hdr.common.type);
1958         break;
1959     }
1960 out:
1961     kfree(qentry);
1962 }
1963
1964 /* worker to process llc messages on the event queue */
1965 static void smc_llc_event_work(struct work_struct *work)
1966 {
1967     struct smc_link_group *lgr = container_of(work, struct smc_link_group,
1968                           llc_event_work);
1969     struct smc_llc_qentry *qentry;
1970
1971     if (!lgr->llc_flow_lcl.type && lgr->delayed_event) {
1972         qentry = lgr->delayed_event;
1973         lgr->delayed_event = NULL;
1974         if (smc_link_usable(qentry->link))
1975             smc_llc_event_handler(qentry);
1976         else
1977             kfree(qentry);
1978     }
1979
1980 again:
1981     spin_lock_bh(&lgr->llc_event_q_lock);
1982     if (!list_empty(&lgr->llc_event_q)) {
1983         qentry = list_first_entry(&lgr->llc_event_q,
1984                       struct smc_llc_qentry, list);
1985         list_del_init(&qentry->list);
1986         spin_unlock_bh(&lgr->llc_event_q_lock);
1987         smc_llc_event_handler(qentry);
1988         goto again;
1989     }
1990     spin_unlock_bh(&lgr->llc_event_q_lock);
1991 }
1992
1993 /* process llc responses in tasklet context */
1994 static void smc_llc_rx_response(struct smc_link *link,
1995                 struct smc_llc_qentry *qentry)
1996 {
1997     enum smc_llc_flowtype flowtype = link->lgr->llc_flow_lcl.type;
1998     struct smc_llc_flow *flow = &link->lgr->llc_flow_lcl;
1999     u8 llc_type = qentry->msg.raw.hdr.common.llc_type;
2000
2001     switch (llc_type) {
2002     case SMC_LLC_TEST_LINK:
2003         if (smc_link_active(link))
2004             complete(&link->llc_testlink_resp);
2005         break;
2006     case SMC_LLC_ADD_LINK:
2007     case SMC_LLC_ADD_LINK_CONT:
2008     case SMC_LLC_CONFIRM_LINK:
2009         if (flowtype != SMC_LLC_FLOW_ADD_LINK || flow->qentry)
2010             break;  /* drop out-of-flow response */
2011         goto assign;
2012     case SMC_LLC_DELETE_LINK:
2013         if (flowtype != SMC_LLC_FLOW_DEL_LINK || flow->qentry)
2014             break;  /* drop out-of-flow response */
2015         goto assign;
2016     case SMC_LLC_CONFIRM_RKEY:
2017     case SMC_LLC_DELETE_RKEY:
2018         if (flowtype != SMC_LLC_FLOW_RKEY || flow->qentry)
2019             break;  /* drop out-of-flow response */
2020         goto assign;
2021     case SMC_LLC_CONFIRM_RKEY_CONT:
2022         /* not used because max links is 3 */
2023         break;
2024     default:
2025         smc_llc_protocol_violation(link->lgr,
2026                        qentry->msg.raw.hdr.common.type);
2027         break;
2028     }
2029     kfree(qentry);
2030     return;
2031 assign:
2032     /* assign responses to the local flow, we requested them */
2033     smc_llc_flow_qentry_set(&link->lgr->llc_flow_lcl, qentry);
2034     wake_up(&link->lgr->llc_msg_waiter);
2035 }
2036
2037 static void smc_llc_enqueue(struct smc_link *link, union smc_llc_msg *llc)
2038 {
2039     struct smc_link_group *lgr = link->lgr;
2040     struct smc_llc_qentry *qentry;
2041     unsigned long flags;
2042
2043     qentry = kmalloc(sizeof(*qentry), GFP_ATOMIC);
2044     if (!qentry)
2045         return;
2046     qentry->link = link;
2047     INIT_LIST_HEAD(&qentry->list);
2048     memcpy(&qentry->msg, llc, sizeof(union smc_llc_msg));
2049
2050     /* process responses immediately */
2051     if ((llc->raw.hdr.flags & SMC_LLC_FLAG_RESP) &&
2052         llc->raw.hdr.common.llc_type != SMC_LLC_REQ_ADD_LINK) {
2053         smc_llc_rx_response(link, qentry);
2054         return;
2055     }
2056
2057     /* add requests to event queue */
2058     spin_lock_irqsave(&lgr->llc_event_q_lock, flags);
2059     list_add_tail(&qentry->list, &lgr->llc_event_q);
2060     spin_unlock_irqrestore(&lgr->llc_event_q_lock, flags);
2061     queue_work(system_highpri_wq, &lgr->llc_event_work);
2062 }
2063
2064 /* copy received msg and add it to the event queue */
2065 static void smc_llc_rx_handler(struct ib_wc *wc, void *buf)
2066 {
2067     struct smc_link *link = (struct smc_link *)wc->qp->qp_context;
2068     union smc_llc_msg *llc = buf;
2069
2070     if (wc->byte_len < sizeof(*llc))
2071         return; /* short message */
2072     if (!llc->raw.hdr.common.llc_version) {
2073         if (llc->raw.hdr.length != sizeof(*llc))
2074             return; /* invalid message */
2075     } else {
2076         if (llc->raw.hdr.length_v2 < sizeof(*llc))
2077             return; /* invalid message */
2078     }
2079
2080     smc_llc_enqueue(link, llc);
2081 }
2082
2083 /***************************** worker, utils *********************************/
2084
2085 static void smc_llc_testlink_work(struct work_struct *work)
2086 {
2087     struct smc_link *link = container_of(to_delayed_work(work),
2088                          struct smc_link, llc_testlink_wrk);
2089     unsigned long next_interval;
2090     unsigned long expire_time;
2091     u8 user_data[16] = { 0 };
2092     int rc;
2093
2094     if (!smc_link_active(link))
2095         return;     /* don't reschedule worker */
2096     expire_time = link->wr_rx_tstamp + link->llc_testlink_time;
2097     if (time_is_after_jiffies(expire_time)) {
2098         next_interval = expire_time - jiffies;
2099         goto out;
2100     }
2101     reinit_completion(&link->llc_testlink_resp);
2102     smc_llc_send_test_link(link, user_data);
2103     /* receive TEST LINK response over RoCE fabric */
2104     rc = wait_for_completion_interruptible_timeout(&link->llc_testlink_resp,
2105                                SMC_LLC_WAIT_TIME);
2106     if (!smc_link_active(link))
2107         return;     /* link state changed */
2108     if (rc <= 0) {
2109         smcr_link_down_cond_sched(link);
2110         return;
2111     }
2112     next_interval = link->llc_testlink_time;
2113 out:
2114     schedule_delayed_work(&link->llc_testlink_wrk, next_interval);
2115 }
2116
2117 void smc_llc_lgr_init(struct smc_link_group *lgr, struct smc_sock *smc)
2118 {
2119     struct net *net = sock_net(smc->clcsock->sk);
2120
2121     INIT_WORK(&lgr->llc_event_work, smc_llc_event_work);
2122     INIT_WORK(&lgr->llc_add_link_work, smc_llc_add_link_work);
2123     INIT_WORK(&lgr->llc_del_link_work, smc_llc_delete_link_work);
2124     INIT_LIST_HEAD(&lgr->llc_event_q);
2125     spin_lock_init(&lgr->llc_event_q_lock);
2126     spin_lock_init(&lgr->llc_flow_lock);
2127     init_waitqueue_head(&lgr->llc_flow_waiter);
2128     init_waitqueue_head(&lgr->llc_msg_waiter);
2129     mutex_init(&lgr->llc_conf_mutex);
2130     lgr->llc_testlink_time = READ_ONCE(net->ipv4.sysctl_tcp_keepalive_time);
2131 }
2132
2133 /* called after lgr was removed from lgr_list */
2134 void smc_llc_lgr_clear(struct smc_link_group *lgr)
2135 {
2136     smc_llc_event_flush(lgr);
2137     wake_up_all(&lgr->llc_flow_waiter);
2138     wake_up_all(&lgr->llc_msg_waiter);
2139     cancel_work_sync(&lgr->llc_event_work);
2140     cancel_work_sync(&lgr->llc_add_link_work);
2141     cancel_work_sync(&lgr->llc_del_link_work);
2142     if (lgr->delayed_event) {
2143         kfree(lgr->delayed_event);
2144         lgr->delayed_event = NULL;
2145     }
2146 }
2147
2148 int smc_llc_link_init(struct smc_link *link)
2149 {
2150     init_completion(&link->llc_testlink_resp);
2151     INIT_DELAYED_WORK(&link->llc_testlink_wrk, smc_llc_testlink_work);
2152     return 0;
2153 }
2154
2155 void smc_llc_link_active(struct smc_link *link)
2156 {
2157     pr_warn_ratelimited("smc: SMC-R lg %*phN net %llu link added: id %*phN, "
2158                 "peerid %*phN, ibdev %s, ibport %d\n",
2159                 SMC_LGR_ID_SIZE, &link->lgr->id,
2160                 link->lgr->net->net_cookie,
2161                 SMC_LGR_ID_SIZE, &link->link_uid,
2162                 SMC_LGR_ID_SIZE, &link->peer_link_uid,
2163                 link->smcibdev->ibdev->name, link->ibport);
2164     link->state = SMC_LNK_ACTIVE;
2165     if (link->lgr->llc_testlink_time) {
2166         link->llc_testlink_time = link->lgr->llc_testlink_time;
2167         schedule_delayed_work(&link->llc_testlink_wrk,
2168                       link->llc_testlink_time);
2169     }
2170 }
2171
2172 /* called in worker context */
2173 void smc_llc_link_clear(struct smc_link *link, bool log)
2174 {
2175     if (log)
2176         pr_warn_ratelimited("smc: SMC-R lg %*phN net %llu link removed: id %*phN"
2177                     ", peerid %*phN, ibdev %s, ibport %d\n",
2178                     SMC_LGR_ID_SIZE, &link->lgr->id,
2179                     link->lgr->net->net_cookie,
2180                     SMC_LGR_ID_SIZE, &link->link_uid,
2181                     SMC_LGR_ID_SIZE, &link->peer_link_uid,
2182                     link->smcibdev->ibdev->name, link->ibport);
2183     complete(&link->llc_testlink_resp);
2184     cancel_delayed_work_sync(&link->llc_testlink_wrk);
2185 }
2186
2187 /* register a new rtoken at the remote peer (for all links) */
2188 int smc_llc_do_confirm_rkey(struct smc_link *send_link,
2189                 struct smc_buf_desc *rmb_desc)
2190 {
2191     struct smc_link_group *lgr = send_link->lgr;
2192     struct smc_llc_qentry *qentry = NULL;
2193     int rc = 0;
2194
2195     rc = smc_llc_send_confirm_rkey(send_link, rmb_desc);
2196     if (rc)
2197         goto out;
2198     /* receive CONFIRM RKEY response from server over RoCE fabric */
2199     qentry = smc_llc_wait(lgr, send_link, SMC_LLC_WAIT_TIME,
2200                   SMC_LLC_CONFIRM_RKEY);
2201     if (!qentry || (qentry->msg.raw.hdr.flags & SMC_LLC_FLAG_RKEY_NEG))
2202         rc = -EFAULT;
2203 out:
2204     if (qentry)
2205         smc_llc_flow_qentry_del(&lgr->llc_flow_lcl);
2206     return rc;
2207 }
2208
2209 /* unregister an rtoken at the remote peer */
2210 int smc_llc_do_delete_rkey(struct smc_link_group *lgr,
2211                struct smc_buf_desc *rmb_desc)
2212 {
2213     struct smc_llc_qentry *qentry = NULL;
2214     struct smc_link *send_link;
2215     int rc = 0;
2216
2217     send_link = smc_llc_usable_link(lgr);
2218     if (!send_link)
2219         return -ENOLINK;
2220
2221     /* protected by llc_flow control */
2222     rc = smc_llc_send_delete_rkey(send_link, rmb_desc);
2223     if (rc)
2224         goto out;
2225     /* receive DELETE RKEY response from server over RoCE fabric */
2226     qentry = smc_llc_wait(lgr, send_link, SMC_LLC_WAIT_TIME,
2227                   SMC_LLC_DELETE_RKEY);
2228     if (!qentry || (qentry->msg.raw.hdr.flags & SMC_LLC_FLAG_RKEY_NEG))
2229         rc = -EFAULT;
2230 out:
2231     if (qentry)
2232         smc_llc_flow_qentry_del(&lgr->llc_flow_lcl);
2233     return rc;
2234 }
2235
2236 void smc_llc_link_set_uid(struct smc_link *link)
2237 {
2238     __be32 link_uid;
2239
2240     link_uid = htonl(*((u32 *)link->lgr->id) + link->link_id);
2241     memcpy(link->link_uid, &link_uid, SMC_LGR_ID_SIZE);
2242 }
2243
2244 /* save peers link user id, used for debug purposes */
2245 void smc_llc_save_peer_uid(struct smc_llc_qentry *qentry)
2246 {
2247     memcpy(qentry->link->peer_link_uid, qentry->msg.confirm_link.link_uid,
2248            SMC_LGR_ID_SIZE);
2249 }
2250
2251 /* evaluate confirm link request or response */
2252 int smc_llc_eval_conf_link(struct smc_llc_qentry *qentry,
2253                enum smc_llc_reqresp type)
2254 {
2255     if (type == SMC_LLC_REQ) {  /* SMC server assigns link_id */
2256         qentry->link->link_id = qentry->msg.confirm_link.link_num;
2257         smc_llc_link_set_uid(qentry->link);
2258     }
2259     if (!(qentry->msg.raw.hdr.flags & SMC_LLC_FLAG_NO_RMBE_EYEC))
2260         return -ENOTSUPP;
2261     return 0;
2262 }
2263
2264 /***************************** init, exit, misc ******************************/
2265
2266 static struct smc_wr_rx_handler smc_llc_rx_handlers[] = {
2267     {
2268         .handler    = smc_llc_rx_handler,
2269         .type       = SMC_LLC_CONFIRM_LINK
2270     },
2271     {
2272         .handler    = smc_llc_rx_handler,
2273         .type       = SMC_LLC_TEST_LINK
2274     },
2275     {
2276         .handler    = smc_llc_rx_handler,
2277         .type       = SMC_LLC_ADD_LINK
2278     },
2279     {
2280         .handler    = smc_llc_rx_handler,
2281         .type       = SMC_LLC_ADD_LINK_CONT
2282     },
2283     {
2284         .handler    = smc_llc_rx_handler,
2285         .type       = SMC_LLC_DELETE_LINK
2286     },
2287     {
2288         .handler    = smc_llc_rx_handler,
2289         .type       = SMC_LLC_CONFIRM_RKEY
2290     },
2291     {
2292         .handler    = smc_llc_rx_handler,
2293         .type       = SMC_LLC_CONFIRM_RKEY_CONT
2294     },
2295     {
2296         .handler    = smc_llc_rx_handler,
2297         .type       = SMC_LLC_DELETE_RKEY
2298     },
2299     /* V2 types */
2300     {
2301         .handler    = smc_llc_rx_handler,
2302         .type       = SMC_LLC_CONFIRM_LINK_V2
2303     },
2304     {
2305         .handler    = smc_llc_rx_handler,
2306         .type       = SMC_LLC_TEST_LINK_V2
2307     },
2308     {
2309         .handler    = smc_llc_rx_handler,
2310         .type       = SMC_LLC_ADD_LINK_V2
2311     },
2312     {
2313         .handler    = smc_llc_rx_handler,
2314         .type       = SMC_LLC_DELETE_LINK_V2
2315     },
2316     {
2317         .handler    = smc_llc_rx_handler,
2318         .type       = SMC_LLC_REQ_ADD_LINK_V2
2319     },
2320     {
2321         .handler    = smc_llc_rx_handler,
2322         .type       = SMC_LLC_CONFIRM_RKEY_V2
2323     },
2324     {
2325         .handler    = smc_llc_rx_handler,
2326         .type       = SMC_LLC_DELETE_RKEY_V2
2327     },
2328     {
2329         .handler    = NULL,
2330     }
2331 };
2332
2333 int __init smc_llc_init(void)
2334 {
2335     struct smc_wr_rx_handler *handler;
2336     int rc = 0;
2337
2338     for (handler = smc_llc_rx_handlers; handler->handler; handler++) {
2339         INIT_HLIST_NODE(&handler->list);
2340         rc = smc_wr_rx_register_handler(handler);
2341         if (rc)
2342             break;
2343     }
2344     return rc;
2345 }