Back to home page

OSCL-LXR
 
 

     

0001 /*
0002  * cxgb3i_offload.c: Chelsio S3xx iscsi offloaded tcp connection management
0003  *
0004  * Copyright (C) 2003-2015 Chelsio Communications.  All rights reserved.
0005  *
0006  * This program is distributed in the hope that it will be useful, but WITHOUT
0007  * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
0008  * FITNESS FOR A PARTICULAR PURPOSE.  See the LICENSE file included in this
0009  * release for licensing terms and conditions.
0010  *
0011  * Written by:  Dimitris Michailidis (dm@chelsio.com)
0012  *      Karen Xie (kxie@chelsio.com)
0013  */
0014 
0015 #define pr_fmt(fmt) KBUILD_MODNAME ":%s: " fmt, __func__
0016 
0017 #include <linux/module.h>
0018 #include <linux/moduleparam.h>
0019 #include <scsi/scsi_host.h>
0020 
0021 #include "common.h"
0022 #include "t3_cpl.h"
0023 #include "t3cdev.h"
0024 #include "cxgb3_defs.h"
0025 #include "cxgb3_ctl_defs.h"
0026 #include "cxgb3_offload.h"
0027 #include "firmware_exports.h"
0028 #include "cxgb3i.h"
0029 
0030 static unsigned int dbg_level;
0031 #include "../libcxgbi.h"
0032 
0033 #define DRV_MODULE_NAME         "cxgb3i"
0034 #define DRV_MODULE_DESC         "Chelsio T3 iSCSI Driver"
0035 #define DRV_MODULE_VERSION  "2.0.1-ko"
0036 #define DRV_MODULE_RELDATE  "Apr. 2015"
0037 
0038 static char version[] =
0039     DRV_MODULE_DESC " " DRV_MODULE_NAME
0040     " v" DRV_MODULE_VERSION " (" DRV_MODULE_RELDATE ")\n";
0041 
0042 MODULE_AUTHOR("Chelsio Communications, Inc.");
0043 MODULE_DESCRIPTION(DRV_MODULE_DESC);
0044 MODULE_VERSION(DRV_MODULE_VERSION);
0045 MODULE_LICENSE("GPL");
0046 
0047 module_param(dbg_level, uint, 0644);
0048 MODULE_PARM_DESC(dbg_level, "debug flag (default=0)");
0049 
0050 static int cxgb3i_rcv_win = 256 * 1024;
0051 module_param(cxgb3i_rcv_win, int, 0644);
0052 MODULE_PARM_DESC(cxgb3i_rcv_win, "TCP receive window in bytes (default=256KB)");
0053 
0054 static int cxgb3i_snd_win = 128 * 1024;
0055 module_param(cxgb3i_snd_win, int, 0644);
0056 MODULE_PARM_DESC(cxgb3i_snd_win, "TCP send window in bytes (default=128KB)");
0057 
0058 static int cxgb3i_rx_credit_thres = 10 * 1024;
0059 module_param(cxgb3i_rx_credit_thres, int, 0644);
0060 MODULE_PARM_DESC(cxgb3i_rx_credit_thres,
0061          "RX credits return threshold in bytes (default=10KB)");
0062 
0063 static unsigned int cxgb3i_max_connect = 8 * 1024;
0064 module_param(cxgb3i_max_connect, uint, 0644);
0065 MODULE_PARM_DESC(cxgb3i_max_connect, "Max. # of connections (default=8092)");
0066 
0067 static unsigned int cxgb3i_sport_base = 20000;
0068 module_param(cxgb3i_sport_base, uint, 0644);
0069 MODULE_PARM_DESC(cxgb3i_sport_base, "starting port number (default=20000)");
0070 
0071 static void cxgb3i_dev_open(struct t3cdev *);
0072 static void cxgb3i_dev_close(struct t3cdev *);
0073 static void cxgb3i_dev_event_handler(struct t3cdev *, u32, u32);
0074 
0075 static struct cxgb3_client t3_client = {
0076     .name = DRV_MODULE_NAME,
0077     .handlers = cxgb3i_cpl_handlers,
0078     .add = cxgb3i_dev_open,
0079     .remove = cxgb3i_dev_close,
0080     .event_handler = cxgb3i_dev_event_handler,
0081 };
0082 
0083 static struct scsi_host_template cxgb3i_host_template = {
0084     .module     = THIS_MODULE,
0085     .name       = DRV_MODULE_NAME,
0086     .proc_name  = DRV_MODULE_NAME,
0087     .can_queue  = CXGB3I_SCSI_HOST_QDEPTH,
0088     .queuecommand   = iscsi_queuecommand,
0089     .change_queue_depth = scsi_change_queue_depth,
0090     .sg_tablesize   = SG_ALL,
0091     .max_sectors    = 0xFFFF,
0092     .cmd_per_lun    = ISCSI_DEF_CMD_PER_LUN,
0093     .eh_timed_out   = iscsi_eh_cmd_timed_out,
0094     .eh_abort_handler = iscsi_eh_abort,
0095     .eh_device_reset_handler = iscsi_eh_device_reset,
0096     .eh_target_reset_handler = iscsi_eh_recover_target,
0097     .target_alloc   = iscsi_target_alloc,
0098     .dma_boundary   = PAGE_SIZE - 1,
0099     .this_id    = -1,
0100     .track_queue_depth = 1,
0101     .cmd_size   = sizeof(struct iscsi_cmd),
0102 };
0103 
0104 static struct iscsi_transport cxgb3i_iscsi_transport = {
0105     .owner      = THIS_MODULE,
0106     .name       = DRV_MODULE_NAME,
0107     /* owner and name should be set already */
0108     .caps       = CAP_RECOVERY_L0 | CAP_MULTI_R2T | CAP_HDRDGST
0109                 | CAP_DATADGST | CAP_DIGEST_OFFLOAD |
0110                 CAP_PADDING_OFFLOAD | CAP_TEXT_NEGO,
0111     .attr_is_visible    = cxgbi_attr_is_visible,
0112     .get_host_param = cxgbi_get_host_param,
0113     .set_host_param = cxgbi_set_host_param,
0114     /* session management */
0115     .create_session = cxgbi_create_session,
0116     .destroy_session    = cxgbi_destroy_session,
0117     .get_session_param = iscsi_session_get_param,
0118     /* connection management */
0119     .create_conn    = cxgbi_create_conn,
0120     .bind_conn  = cxgbi_bind_conn,
0121     .unbind_conn    = iscsi_conn_unbind,
0122     .destroy_conn   = iscsi_tcp_conn_teardown,
0123     .start_conn = iscsi_conn_start,
0124     .stop_conn  = iscsi_conn_stop,
0125     .get_conn_param = iscsi_conn_get_param,
0126     .set_param  = cxgbi_set_conn_param,
0127     .get_stats  = cxgbi_get_conn_stats,
0128     /* pdu xmit req from user space */
0129     .send_pdu   = iscsi_conn_send_pdu,
0130     /* task */
0131     .init_task  = iscsi_tcp_task_init,
0132     .xmit_task  = iscsi_tcp_task_xmit,
0133     .cleanup_task   = cxgbi_cleanup_task,
0134     /* pdu */
0135     .alloc_pdu  = cxgbi_conn_alloc_pdu,
0136     .init_pdu   = cxgbi_conn_init_pdu,
0137     .xmit_pdu   = cxgbi_conn_xmit_pdu,
0138     .parse_pdu_itt  = cxgbi_parse_pdu_itt,
0139     /* TCP connect/disconnect */
0140     .get_ep_param   = cxgbi_get_ep_param,
0141     .ep_connect = cxgbi_ep_connect,
0142     .ep_poll    = cxgbi_ep_poll,
0143     .ep_disconnect  = cxgbi_ep_disconnect,
0144     /* Error recovery timeout call */
0145     .session_recovery_timedout = iscsi_session_recovery_timedout,
0146 };
0147 
0148 static struct scsi_transport_template *cxgb3i_stt;
0149 
0150 /*
0151  * CPL (Chelsio Protocol Language) defines a message passing interface between
0152  * the host driver and Chelsio asic.
0153  * The section below implments CPLs that related to iscsi tcp connection
0154  * open/close/abort and data send/receive.
0155  */
0156 
0157 static int push_tx_frames(struct cxgbi_sock *csk, int req_completion);
0158 
0159 static void send_act_open_req(struct cxgbi_sock *csk, struct sk_buff *skb,
0160                   const struct l2t_entry *e)
0161 {
0162     unsigned int wscale = cxgbi_sock_compute_wscale(csk->rcv_win);
0163     struct cpl_act_open_req *req = (struct cpl_act_open_req *)skb->head;
0164 
0165     skb->priority = CPL_PRIORITY_SETUP;
0166 
0167     req->wr.wr_hi = htonl(V_WR_OP(FW_WROPCODE_FORWARD));
0168     OPCODE_TID(req) = htonl(MK_OPCODE_TID(CPL_ACT_OPEN_REQ, csk->atid));
0169     req->local_port = csk->saddr.sin_port;
0170     req->peer_port = csk->daddr.sin_port;
0171     req->local_ip = csk->saddr.sin_addr.s_addr;
0172     req->peer_ip = csk->daddr.sin_addr.s_addr;
0173 
0174     req->opt0h = htonl(V_KEEP_ALIVE(1) | F_TCAM_BYPASS |
0175             V_WND_SCALE(wscale) | V_MSS_IDX(csk->mss_idx) |
0176             V_L2T_IDX(e->idx) | V_TX_CHANNEL(e->smt_idx));
0177     req->opt0l = htonl(V_ULP_MODE(ULP2_MODE_ISCSI) |
0178             V_RCV_BUFSIZ(csk->rcv_win >> 10));
0179 
0180     log_debug(1 << CXGBI_DBG_TOE | 1 << CXGBI_DBG_SOCK,
0181         "csk 0x%p,%u,0x%lx,%u, %pI4:%u-%pI4:%u, %u,%u,%u.\n",
0182         csk, csk->state, csk->flags, csk->atid,
0183         &req->local_ip, ntohs(req->local_port),
0184         &req->peer_ip, ntohs(req->peer_port),
0185         csk->mss_idx, e->idx, e->smt_idx);
0186 
0187     l2t_send(csk->cdev->lldev, skb, csk->l2t);
0188 }
0189 
0190 static inline void act_open_arp_failure(struct t3cdev *dev, struct sk_buff *skb)
0191 {
0192     cxgbi_sock_act_open_req_arp_failure(NULL, skb);
0193 }
0194 
0195 /*
0196  * CPL connection close request: host ->
0197  *
0198  * Close a connection by sending a CPL_CLOSE_CON_REQ message and queue it to
0199  * the write queue (i.e., after any unsent txt data).
0200  */
0201 static void send_close_req(struct cxgbi_sock *csk)
0202 {
0203     struct sk_buff *skb = csk->cpl_close;
0204     struct cpl_close_con_req *req = (struct cpl_close_con_req *)skb->head;
0205     unsigned int tid = csk->tid;
0206 
0207     log_debug(1 << CXGBI_DBG_TOE | 1 << CXGBI_DBG_SOCK,
0208         "csk 0x%p,%u,0x%lx,%u.\n",
0209         csk, csk->state, csk->flags, csk->tid);
0210 
0211     csk->cpl_close = NULL;
0212     req->wr.wr_hi = htonl(V_WR_OP(FW_WROPCODE_OFLD_CLOSE_CON));
0213     req->wr.wr_lo = htonl(V_WR_TID(tid));
0214     OPCODE_TID(req) = htonl(MK_OPCODE_TID(CPL_CLOSE_CON_REQ, tid));
0215     req->rsvd = htonl(csk->write_seq);
0216 
0217     cxgbi_sock_skb_entail(csk, skb);
0218     if (csk->state >= CTP_ESTABLISHED)
0219         push_tx_frames(csk, 1);
0220 }
0221 
0222 /*
0223  * CPL connection abort request: host ->
0224  *
0225  * Send an ABORT_REQ message. Makes sure we do not send multiple ABORT_REQs
0226  * for the same connection and also that we do not try to send a message
0227  * after the connection has closed.
0228  */
0229 static void abort_arp_failure(struct t3cdev *tdev, struct sk_buff *skb)
0230 {
0231     struct cpl_abort_req *req = cplhdr(skb);
0232 
0233     log_debug(1 << CXGBI_DBG_TOE | 1 << CXGBI_DBG_SOCK,
0234         "t3dev 0x%p, tid %u, skb 0x%p.\n",
0235         tdev, GET_TID(req), skb);
0236     req->cmd = CPL_ABORT_NO_RST;
0237     cxgb3_ofld_send(tdev, skb);
0238 }
0239 
0240 static void send_abort_req(struct cxgbi_sock *csk)
0241 {
0242     struct sk_buff *skb = csk->cpl_abort_req;
0243     struct cpl_abort_req *req;
0244 
0245     if (unlikely(csk->state == CTP_ABORTING || !skb))
0246         return;
0247     cxgbi_sock_set_state(csk, CTP_ABORTING);
0248     cxgbi_sock_set_flag(csk, CTPF_ABORT_RPL_PENDING);
0249     /* Purge the send queue so we don't send anything after an abort. */
0250     cxgbi_sock_purge_write_queue(csk);
0251 
0252     csk->cpl_abort_req = NULL;
0253     req = (struct cpl_abort_req *)skb->head;
0254     skb->priority = CPL_PRIORITY_DATA;
0255     set_arp_failure_handler(skb, abort_arp_failure);
0256     req->wr.wr_hi = htonl(V_WR_OP(FW_WROPCODE_OFLD_HOST_ABORT_CON_REQ));
0257     req->wr.wr_lo = htonl(V_WR_TID(csk->tid));
0258     OPCODE_TID(req) = htonl(MK_OPCODE_TID(CPL_ABORT_REQ, csk->tid));
0259     req->rsvd0 = htonl(csk->snd_nxt);
0260     req->rsvd1 = !cxgbi_sock_flag(csk, CTPF_TX_DATA_SENT);
0261     req->cmd = CPL_ABORT_SEND_RST;
0262 
0263     log_debug(1 << CXGBI_DBG_TOE | 1 << CXGBI_DBG_SOCK,
0264         "csk 0x%p,%u,0x%lx,%u, snd_nxt %u, 0x%x.\n",
0265         csk, csk->state, csk->flags, csk->tid, csk->snd_nxt,
0266         req->rsvd1);
0267 
0268     l2t_send(csk->cdev->lldev, skb, csk->l2t);
0269 }
0270 
0271 /*
0272  * CPL connection abort reply: host ->
0273  *
0274  * Send an ABORT_RPL message in response of the ABORT_REQ received.
0275  */
0276 static void send_abort_rpl(struct cxgbi_sock *csk, int rst_status)
0277 {
0278     struct sk_buff *skb = csk->cpl_abort_rpl;
0279     struct cpl_abort_rpl *rpl = (struct cpl_abort_rpl *)skb->head;
0280 
0281     log_debug(1 << CXGBI_DBG_TOE | 1 << CXGBI_DBG_SOCK,
0282         "csk 0x%p,%u,0x%lx,%u, status %d.\n",
0283         csk, csk->state, csk->flags, csk->tid, rst_status);
0284 
0285     csk->cpl_abort_rpl = NULL;
0286     skb->priority = CPL_PRIORITY_DATA;
0287     rpl->wr.wr_hi = htonl(V_WR_OP(FW_WROPCODE_OFLD_HOST_ABORT_CON_RPL));
0288     rpl->wr.wr_lo = htonl(V_WR_TID(csk->tid));
0289     OPCODE_TID(rpl) = htonl(MK_OPCODE_TID(CPL_ABORT_RPL, csk->tid));
0290     rpl->cmd = rst_status;
0291     cxgb3_ofld_send(csk->cdev->lldev, skb);
0292 }
0293 
0294 /*
0295  * CPL connection rx data ack: host ->
0296  * Send RX credits through an RX_DATA_ACK CPL message. Returns the number of
0297  * credits sent.
0298  */
0299 static u32 send_rx_credits(struct cxgbi_sock *csk, u32 credits)
0300 {
0301     struct sk_buff *skb;
0302     struct cpl_rx_data_ack *req;
0303     u32 dack = F_RX_DACK_CHANGE | V_RX_DACK_MODE(1);
0304 
0305     log_debug(1 << CXGBI_DBG_TOE | 1 << CXGBI_DBG_PDU_RX,
0306         "csk 0x%p,%u,0x%lx,%u, credit %u, dack %u.\n",
0307         csk, csk->state, csk->flags, csk->tid, credits, dack);
0308 
0309     skb = alloc_wr(sizeof(*req), 0, GFP_ATOMIC);
0310     if (!skb) {
0311         pr_info("csk 0x%p, credit %u, OOM.\n", csk, credits);
0312         return 0;
0313     }
0314     req = (struct cpl_rx_data_ack *)skb->head;
0315     req->wr.wr_hi = htonl(V_WR_OP(FW_WROPCODE_FORWARD));
0316     OPCODE_TID(req) = htonl(MK_OPCODE_TID(CPL_RX_DATA_ACK, csk->tid));
0317     req->credit_dack = htonl(F_RX_DACK_CHANGE | V_RX_DACK_MODE(1) |
0318                 V_RX_CREDITS(credits));
0319     skb->priority = CPL_PRIORITY_ACK;
0320     cxgb3_ofld_send(csk->cdev->lldev, skb);
0321     return credits;
0322 }
0323 
0324 /*
0325  * CPL connection tx data: host ->
0326  *
0327  * Send iscsi PDU via TX_DATA CPL message. Returns the number of
0328  * credits sent.
0329  * Each TX_DATA consumes work request credit (wrs), so we need to keep track of
0330  * how many we've used so far and how many are pending (i.e., yet ack'ed by T3).
0331  */
0332 
0333 static unsigned int wrlen __read_mostly;
0334 static unsigned int skb_wrs[SKB_WR_LIST_SIZE] __read_mostly;
0335 
0336 static void init_wr_tab(unsigned int wr_len)
0337 {
0338     int i;
0339 
0340     if (skb_wrs[1])     /* already initialized */
0341         return;
0342     for (i = 1; i < SKB_WR_LIST_SIZE; i++) {
0343         int sgl_len = (3 * i) / 2 + (i & 1);
0344 
0345         sgl_len += 3;
0346         skb_wrs[i] = (sgl_len <= wr_len
0347                   ? 1 : 1 + (sgl_len - 2) / (wr_len - 1));
0348     }
0349     wrlen = wr_len * 8;
0350 }
0351 
0352 static inline void make_tx_data_wr(struct cxgbi_sock *csk, struct sk_buff *skb,
0353                    int len, int req_completion)
0354 {
0355     struct tx_data_wr *req;
0356     struct l2t_entry *l2t = csk->l2t;
0357 
0358     skb_reset_transport_header(skb);
0359     req = __skb_push(skb, sizeof(*req));
0360     req->wr_hi = htonl(V_WR_OP(FW_WROPCODE_OFLD_TX_DATA) |
0361             (req_completion ? F_WR_COMPL : 0));
0362     req->wr_lo = htonl(V_WR_TID(csk->tid));
0363     /* len includes the length of any HW ULP additions */
0364     req->len = htonl(len);
0365     /* V_TX_ULP_SUBMODE sets both the mode and submode */
0366     req->flags = htonl(V_TX_ULP_SUBMODE(cxgbi_skcb_tx_ulp_mode(skb)) |
0367                V_TX_SHOVE((skb_peek(&csk->write_queue) ? 0 : 1)));
0368     req->sndseq = htonl(csk->snd_nxt);
0369     req->param = htonl(V_TX_PORT(l2t->smt_idx));
0370 
0371     if (!cxgbi_sock_flag(csk, CTPF_TX_DATA_SENT)) {
0372         req->flags |= htonl(V_TX_ACK_PAGES(2) | F_TX_INIT |
0373                     V_TX_CPU_IDX(csk->rss_qid));
0374         /* sendbuffer is in units of 32KB. */
0375         req->param |= htonl(V_TX_SNDBUF(csk->snd_win >> 15));
0376         cxgbi_sock_set_flag(csk, CTPF_TX_DATA_SENT);
0377     }
0378 }
0379 
0380 /*
0381  * push_tx_frames -- start transmit
0382  *
0383  * Prepends TX_DATA_WR or CPL_CLOSE_CON_REQ headers to buffers waiting in a
0384  * connection's send queue and sends them on to T3.  Must be called with the
0385  * connection's lock held.  Returns the amount of send buffer space that was
0386  * freed as a result of sending queued data to T3.
0387  */
0388 
0389 static void arp_failure_skb_discard(struct t3cdev *dev, struct sk_buff *skb)
0390 {
0391     kfree_skb(skb);
0392 }
0393 
0394 static int push_tx_frames(struct cxgbi_sock *csk, int req_completion)
0395 {
0396     int total_size = 0;
0397     struct sk_buff *skb;
0398 
0399     if (unlikely(csk->state < CTP_ESTABLISHED ||
0400         csk->state == CTP_CLOSE_WAIT_1 || csk->state >= CTP_ABORTING)) {
0401             log_debug(1 << CXGBI_DBG_TOE | 1 << CXGBI_DBG_PDU_TX,
0402                 "csk 0x%p,%u,0x%lx,%u, in closing state.\n",
0403                 csk, csk->state, csk->flags, csk->tid);
0404         return 0;
0405     }
0406 
0407     while (csk->wr_cred && (skb = skb_peek(&csk->write_queue)) != NULL) {
0408         int len = skb->len; /* length before skb_push */
0409         int frags = skb_shinfo(skb)->nr_frags + (len != skb->data_len);
0410         int wrs_needed = skb_wrs[frags];
0411 
0412         if (wrs_needed > 1 && len + sizeof(struct tx_data_wr) <= wrlen)
0413             wrs_needed = 1;
0414 
0415         WARN_ON(frags >= SKB_WR_LIST_SIZE || wrs_needed < 1);
0416 
0417         if (csk->wr_cred < wrs_needed) {
0418             log_debug(1 << CXGBI_DBG_PDU_TX,
0419                 "csk 0x%p, skb len %u/%u, frag %u, wr %d<%u.\n",
0420                 csk, skb->len, skb->data_len, frags,
0421                 wrs_needed, csk->wr_cred);
0422             break;
0423         }
0424 
0425         __skb_unlink(skb, &csk->write_queue);
0426         skb->priority = CPL_PRIORITY_DATA;
0427         skb->csum = wrs_needed; /* remember this until the WR_ACK */
0428         csk->wr_cred -= wrs_needed;
0429         csk->wr_una_cred += wrs_needed;
0430         cxgbi_sock_enqueue_wr(csk, skb);
0431 
0432         log_debug(1 << CXGBI_DBG_TOE | 1 << CXGBI_DBG_PDU_TX,
0433             "csk 0x%p, enqueue, skb len %u/%u, frag %u, wr %d, "
0434             "left %u, unack %u.\n",
0435             csk, skb->len, skb->data_len, frags, skb->csum,
0436             csk->wr_cred, csk->wr_una_cred);
0437 
0438         if (likely(cxgbi_skcb_test_flag(skb, SKCBF_TX_NEED_HDR))) {
0439             if ((req_completion &&
0440                 csk->wr_una_cred == wrs_needed) ||
0441                  csk->wr_una_cred >= csk->wr_max_cred / 2) {
0442                 req_completion = 1;
0443                 csk->wr_una_cred = 0;
0444             }
0445             len += cxgbi_ulp_extra_len(cxgbi_skcb_tx_ulp_mode(skb));
0446             make_tx_data_wr(csk, skb, len, req_completion);
0447             csk->snd_nxt += len;
0448             cxgbi_skcb_clear_flag(skb, SKCBF_TX_NEED_HDR);
0449         }
0450         total_size += skb->truesize;
0451         log_debug(1 << CXGBI_DBG_TOE | 1 << CXGBI_DBG_PDU_TX,
0452             "csk 0x%p, tid 0x%x, send skb 0x%p.\n",
0453             csk, csk->tid, skb);
0454         set_arp_failure_handler(skb, arp_failure_skb_discard);
0455         l2t_send(csk->cdev->lldev, skb, csk->l2t);
0456     }
0457     return total_size;
0458 }
0459 
0460 /*
0461  * Process a CPL_ACT_ESTABLISH message: -> host
0462  * Updates connection state from an active establish CPL message.  Runs with
0463  * the connection lock held.
0464  */
0465 
0466 static inline void free_atid(struct cxgbi_sock *csk)
0467 {
0468     if (cxgbi_sock_flag(csk, CTPF_HAS_ATID)) {
0469         cxgb3_free_atid(csk->cdev->lldev, csk->atid);
0470         cxgbi_sock_clear_flag(csk, CTPF_HAS_ATID);
0471         cxgbi_sock_put(csk);
0472     }
0473 }
0474 
0475 static int do_act_establish(struct t3cdev *tdev, struct sk_buff *skb, void *ctx)
0476 {
0477     struct cxgbi_sock *csk = ctx;
0478     struct cpl_act_establish *req = cplhdr(skb);
0479     unsigned int tid = GET_TID(req);
0480     unsigned int atid = G_PASS_OPEN_TID(ntohl(req->tos_tid));
0481     u32 rcv_isn = ntohl(req->rcv_isn);  /* real RCV_ISN + 1 */
0482 
0483     log_debug(1 << CXGBI_DBG_TOE | 1 << CXGBI_DBG_SOCK,
0484         "atid 0x%x,tid 0x%x, csk 0x%p,%u,0x%lx, isn %u.\n",
0485         atid, atid, csk, csk->state, csk->flags, rcv_isn);
0486 
0487     cxgbi_sock_get(csk);
0488     cxgbi_sock_set_flag(csk, CTPF_HAS_TID);
0489     csk->tid = tid;
0490     cxgb3_insert_tid(csk->cdev->lldev, &t3_client, csk, tid);
0491 
0492     free_atid(csk);
0493 
0494     csk->rss_qid = G_QNUM(ntohs(skb->csum));
0495 
0496     spin_lock_bh(&csk->lock);
0497     if (csk->retry_timer.function) {
0498         del_timer(&csk->retry_timer);
0499         csk->retry_timer.function = NULL;
0500     }
0501 
0502     if (unlikely(csk->state != CTP_ACTIVE_OPEN))
0503         pr_info("csk 0x%p,%u,0x%lx,%u, got EST.\n",
0504             csk, csk->state, csk->flags, csk->tid);
0505 
0506     csk->copied_seq = csk->rcv_wup = csk->rcv_nxt = rcv_isn;
0507     if (csk->rcv_win > (M_RCV_BUFSIZ << 10))
0508         csk->rcv_wup -= csk->rcv_win - (M_RCV_BUFSIZ << 10);
0509 
0510     cxgbi_sock_established(csk, ntohl(req->snd_isn), ntohs(req->tcp_opt));
0511 
0512     if (unlikely(cxgbi_sock_flag(csk, CTPF_ACTIVE_CLOSE_NEEDED)))
0513         /* upper layer has requested closing */
0514         send_abort_req(csk);
0515     else {
0516         if (skb_queue_len(&csk->write_queue))
0517             push_tx_frames(csk, 1);
0518         cxgbi_conn_tx_open(csk);
0519     }
0520 
0521     spin_unlock_bh(&csk->lock);
0522     __kfree_skb(skb);
0523     return 0;
0524 }
0525 
0526 /*
0527  * Process a CPL_ACT_OPEN_RPL message: -> host
0528  * Handle active open failures.
0529  */
0530 static int act_open_rpl_status_to_errno(int status)
0531 {
0532     switch (status) {
0533     case CPL_ERR_CONN_RESET:
0534         return -ECONNREFUSED;
0535     case CPL_ERR_ARP_MISS:
0536         return -EHOSTUNREACH;
0537     case CPL_ERR_CONN_TIMEDOUT:
0538         return -ETIMEDOUT;
0539     case CPL_ERR_TCAM_FULL:
0540         return -ENOMEM;
0541     case CPL_ERR_CONN_EXIST:
0542         return -EADDRINUSE;
0543     default:
0544         return -EIO;
0545     }
0546 }
0547 
0548 static void act_open_retry_timer(struct timer_list *t)
0549 {
0550     struct cxgbi_sock *csk = from_timer(csk, t, retry_timer);
0551     struct sk_buff *skb;
0552 
0553     log_debug(1 << CXGBI_DBG_TOE | 1 << CXGBI_DBG_SOCK,
0554         "csk 0x%p,%u,0x%lx,%u.\n",
0555         csk, csk->state, csk->flags, csk->tid);
0556 
0557     cxgbi_sock_get(csk);
0558     spin_lock_bh(&csk->lock);
0559     skb = alloc_wr(sizeof(struct cpl_act_open_req), 0, GFP_ATOMIC);
0560     if (!skb)
0561         cxgbi_sock_fail_act_open(csk, -ENOMEM);
0562     else {
0563         skb->sk = (struct sock *)csk;
0564         set_arp_failure_handler(skb, act_open_arp_failure);
0565         send_act_open_req(csk, skb, csk->l2t);
0566     }
0567     spin_unlock_bh(&csk->lock);
0568     cxgbi_sock_put(csk);
0569 }
0570 
0571 static int do_act_open_rpl(struct t3cdev *tdev, struct sk_buff *skb, void *ctx)
0572 {
0573     struct cxgbi_sock *csk = ctx;
0574     struct cpl_act_open_rpl *rpl = cplhdr(skb);
0575 
0576     pr_info("csk 0x%p,%u,0x%lx,%u, status %u, %pI4:%u-%pI4:%u.\n",
0577         csk, csk->state, csk->flags, csk->atid, rpl->status,
0578         &csk->saddr.sin_addr.s_addr, ntohs(csk->saddr.sin_port),
0579         &csk->daddr.sin_addr.s_addr, ntohs(csk->daddr.sin_port));
0580 
0581     if (rpl->status != CPL_ERR_TCAM_FULL &&
0582         rpl->status != CPL_ERR_CONN_EXIST &&
0583         rpl->status != CPL_ERR_ARP_MISS)
0584         cxgb3_queue_tid_release(tdev, GET_TID(rpl));
0585 
0586     cxgbi_sock_get(csk);
0587     spin_lock_bh(&csk->lock);
0588     if (rpl->status == CPL_ERR_CONN_EXIST &&
0589         csk->retry_timer.function != act_open_retry_timer) {
0590         csk->retry_timer.function = act_open_retry_timer;
0591         mod_timer(&csk->retry_timer, jiffies + HZ / 2);
0592     } else
0593         cxgbi_sock_fail_act_open(csk,
0594                 act_open_rpl_status_to_errno(rpl->status));
0595 
0596     spin_unlock_bh(&csk->lock);
0597     cxgbi_sock_put(csk);
0598     __kfree_skb(skb);
0599     return 0;
0600 }
0601 
0602 /*
0603  * Process PEER_CLOSE CPL messages: -> host
0604  * Handle peer FIN.
0605  */
0606 static int do_peer_close(struct t3cdev *cdev, struct sk_buff *skb, void *ctx)
0607 {
0608     struct cxgbi_sock *csk = ctx;
0609 
0610     log_debug(1 << CXGBI_DBG_TOE | 1 << CXGBI_DBG_SOCK,
0611         "csk 0x%p,%u,0x%lx,%u.\n",
0612         csk, csk->state, csk->flags, csk->tid);
0613 
0614     cxgbi_sock_rcv_peer_close(csk);
0615     __kfree_skb(skb);
0616     return 0;
0617 }
0618 
0619 /*
0620  * Process CLOSE_CONN_RPL CPL message: -> host
0621  * Process a peer ACK to our FIN.
0622  */
0623 static int do_close_con_rpl(struct t3cdev *cdev, struct sk_buff *skb,
0624                 void *ctx)
0625 {
0626     struct cxgbi_sock *csk = ctx;
0627     struct cpl_close_con_rpl *rpl = cplhdr(skb);
0628 
0629     log_debug(1 << CXGBI_DBG_TOE | 1 << CXGBI_DBG_SOCK,
0630         "csk 0x%p,%u,0x%lx,%u, snxt %u.\n",
0631         csk, csk->state, csk->flags, csk->tid, ntohl(rpl->snd_nxt));
0632 
0633     cxgbi_sock_rcv_close_conn_rpl(csk, ntohl(rpl->snd_nxt));
0634     __kfree_skb(skb);
0635     return 0;
0636 }
0637 
0638 /*
0639  * Process ABORT_REQ_RSS CPL message: -> host
0640  * Process abort requests.  If we are waiting for an ABORT_RPL we ignore this
0641  * request except that we need to reply to it.
0642  */
0643 
0644 static int abort_status_to_errno(struct cxgbi_sock *csk, int abort_reason,
0645                  int *need_rst)
0646 {
0647     switch (abort_reason) {
0648     case CPL_ERR_BAD_SYN:
0649     case CPL_ERR_CONN_RESET:
0650         return csk->state > CTP_ESTABLISHED ? -EPIPE : -ECONNRESET;
0651     case CPL_ERR_XMIT_TIMEDOUT:
0652     case CPL_ERR_PERSIST_TIMEDOUT:
0653     case CPL_ERR_FINWAIT2_TIMEDOUT:
0654     case CPL_ERR_KEEPALIVE_TIMEDOUT:
0655         return -ETIMEDOUT;
0656     default:
0657         return -EIO;
0658     }
0659 }
0660 
0661 static int do_abort_req(struct t3cdev *cdev, struct sk_buff *skb, void *ctx)
0662 {
0663     const struct cpl_abort_req_rss *req = cplhdr(skb);
0664     struct cxgbi_sock *csk = ctx;
0665     int rst_status = CPL_ABORT_NO_RST;
0666 
0667     log_debug(1 << CXGBI_DBG_TOE | 1 << CXGBI_DBG_SOCK,
0668         "csk 0x%p,%u,0x%lx,%u.\n",
0669         csk, csk->state, csk->flags, csk->tid);
0670 
0671     if (req->status == CPL_ERR_RTX_NEG_ADVICE ||
0672         req->status == CPL_ERR_PERSIST_NEG_ADVICE) {
0673         goto done;
0674     }
0675 
0676     cxgbi_sock_get(csk);
0677     spin_lock_bh(&csk->lock);
0678 
0679     if (!cxgbi_sock_flag(csk, CTPF_ABORT_REQ_RCVD)) {
0680         cxgbi_sock_set_flag(csk, CTPF_ABORT_REQ_RCVD);
0681         cxgbi_sock_set_state(csk, CTP_ABORTING);
0682         goto out;
0683     }
0684 
0685     cxgbi_sock_clear_flag(csk, CTPF_ABORT_REQ_RCVD);
0686     send_abort_rpl(csk, rst_status);
0687 
0688     if (!cxgbi_sock_flag(csk, CTPF_ABORT_RPL_PENDING)) {
0689         csk->err = abort_status_to_errno(csk, req->status, &rst_status);
0690         cxgbi_sock_closed(csk);
0691     }
0692 
0693 out:
0694     spin_unlock_bh(&csk->lock);
0695     cxgbi_sock_put(csk);
0696 done:
0697     __kfree_skb(skb);
0698     return 0;
0699 }
0700 
0701 /*
0702  * Process ABORT_RPL_RSS CPL message: -> host
0703  * Process abort replies.  We only process these messages if we anticipate
0704  * them as the coordination between SW and HW in this area is somewhat lacking
0705  * and sometimes we get ABORT_RPLs after we are done with the connection that
0706  * originated the ABORT_REQ.
0707  */
0708 static int do_abort_rpl(struct t3cdev *cdev, struct sk_buff *skb, void *ctx)
0709 {
0710     struct cpl_abort_rpl_rss *rpl = cplhdr(skb);
0711     struct cxgbi_sock *csk = ctx;
0712 
0713     log_debug(1 << CXGBI_DBG_TOE | 1 << CXGBI_DBG_SOCK,
0714         "status 0x%x, csk 0x%p, s %u, 0x%lx.\n",
0715         rpl->status, csk, csk ? csk->state : 0,
0716         csk ? csk->flags : 0UL);
0717     /*
0718      * Ignore replies to post-close aborts indicating that the abort was
0719      * requested too late.  These connections are terminated when we get
0720      * PEER_CLOSE or CLOSE_CON_RPL and by the time the abort_rpl_rss
0721      * arrives the TID is either no longer used or it has been recycled.
0722      */
0723     if (rpl->status == CPL_ERR_ABORT_FAILED)
0724         goto rel_skb;
0725     /*
0726      * Sometimes we've already closed the connection, e.g., a post-close
0727      * abort races with ABORT_REQ_RSS, the latter frees the connection
0728      * expecting the ABORT_REQ will fail with CPL_ERR_ABORT_FAILED,
0729      * but FW turns the ABORT_REQ into a regular one and so we get
0730      * ABORT_RPL_RSS with status 0 and no connection.
0731      */
0732     if (csk)
0733         cxgbi_sock_rcv_abort_rpl(csk);
0734 rel_skb:
0735     __kfree_skb(skb);
0736     return 0;
0737 }
0738 
0739 /*
0740  * Process RX_ISCSI_HDR CPL message: -> host
0741  * Handle received PDUs, the payload could be DDP'ed. If not, the payload
0742  * follow after the bhs.
0743  */
0744 static int do_iscsi_hdr(struct t3cdev *t3dev, struct sk_buff *skb, void *ctx)
0745 {
0746     struct cxgbi_sock *csk = ctx;
0747     struct cpl_iscsi_hdr *hdr_cpl = cplhdr(skb);
0748     struct cpl_iscsi_hdr_norss data_cpl;
0749     struct cpl_rx_data_ddp_norss ddp_cpl;
0750     unsigned int hdr_len, data_len, status;
0751     unsigned int len;
0752     int err;
0753 
0754     log_debug(1 << CXGBI_DBG_TOE | 1 << CXGBI_DBG_PDU_RX,
0755         "csk 0x%p,%u,0x%lx,%u, skb 0x%p,%u.\n",
0756         csk, csk->state, csk->flags, csk->tid, skb, skb->len);
0757 
0758     spin_lock_bh(&csk->lock);
0759 
0760     if (unlikely(csk->state >= CTP_PASSIVE_CLOSE)) {
0761         log_debug(1 << CXGBI_DBG_TOE | 1 << CXGBI_DBG_SOCK,
0762             "csk 0x%p,%u,0x%lx,%u, bad state.\n",
0763             csk, csk->state, csk->flags, csk->tid);
0764         if (csk->state != CTP_ABORTING)
0765             goto abort_conn;
0766         else
0767             goto discard;
0768     }
0769 
0770     cxgbi_skcb_tcp_seq(skb) = ntohl(hdr_cpl->seq);
0771     cxgbi_skcb_flags(skb) = 0;
0772 
0773     skb_reset_transport_header(skb);
0774     __skb_pull(skb, sizeof(struct cpl_iscsi_hdr));
0775 
0776     len = hdr_len = ntohs(hdr_cpl->len);
0777     /* msg coalesce is off or not enough data received */
0778     if (skb->len <= hdr_len) {
0779         pr_err("%s: tid %u, CPL_ISCSI_HDR, skb len %u < %u.\n",
0780             csk->cdev->ports[csk->port_id]->name, csk->tid,
0781             skb->len, hdr_len);
0782         goto abort_conn;
0783     }
0784     cxgbi_skcb_set_flag(skb, SKCBF_RX_COALESCED);
0785 
0786     err = skb_copy_bits(skb, skb->len - sizeof(ddp_cpl), &ddp_cpl,
0787                 sizeof(ddp_cpl));
0788     if (err < 0) {
0789         pr_err("%s: tid %u, copy cpl_ddp %u-%zu failed %d.\n",
0790             csk->cdev->ports[csk->port_id]->name, csk->tid,
0791             skb->len, sizeof(ddp_cpl), err);
0792         goto abort_conn;
0793     }
0794 
0795     cxgbi_skcb_set_flag(skb, SKCBF_RX_STATUS);
0796     cxgbi_skcb_rx_pdulen(skb) = ntohs(ddp_cpl.len);
0797     cxgbi_skcb_rx_ddigest(skb) = ntohl(ddp_cpl.ulp_crc);
0798     status = ntohl(ddp_cpl.ddp_status);
0799 
0800     log_debug(1 << CXGBI_DBG_TOE | 1 << CXGBI_DBG_PDU_RX,
0801         "csk 0x%p, skb 0x%p,%u, pdulen %u, status 0x%x.\n",
0802         csk, skb, skb->len, cxgbi_skcb_rx_pdulen(skb), status);
0803 
0804     if (status & (1 << CPL_RX_DDP_STATUS_HCRC_SHIFT))
0805         cxgbi_skcb_set_flag(skb, SKCBF_RX_HCRC_ERR);
0806     if (status & (1 << CPL_RX_DDP_STATUS_DCRC_SHIFT))
0807         cxgbi_skcb_set_flag(skb, SKCBF_RX_DCRC_ERR);
0808     if (status & (1 << CPL_RX_DDP_STATUS_PAD_SHIFT))
0809         cxgbi_skcb_set_flag(skb, SKCBF_RX_PAD_ERR);
0810 
0811     if (skb->len > (hdr_len + sizeof(ddp_cpl))) {
0812         err = skb_copy_bits(skb, hdr_len, &data_cpl, sizeof(data_cpl));
0813         if (err < 0) {
0814             pr_err("%s: tid %u, cp %zu/%u failed %d.\n",
0815                 csk->cdev->ports[csk->port_id]->name,
0816                 csk->tid, sizeof(data_cpl), skb->len, err);
0817             goto abort_conn;
0818         }
0819         data_len = ntohs(data_cpl.len);
0820         log_debug(1 << CXGBI_DBG_DDP | 1 << CXGBI_DBG_PDU_RX,
0821             "skb 0x%p, pdu not ddp'ed %u/%u, status 0x%x.\n",
0822             skb, data_len, cxgbi_skcb_rx_pdulen(skb), status);
0823         len += sizeof(data_cpl) + data_len;
0824     } else if (status & (1 << CPL_RX_DDP_STATUS_DDP_SHIFT))
0825         cxgbi_skcb_set_flag(skb, SKCBF_RX_DATA_DDPD);
0826 
0827     csk->rcv_nxt = ntohl(ddp_cpl.seq) + cxgbi_skcb_rx_pdulen(skb);
0828     __pskb_trim(skb, len);
0829     __skb_queue_tail(&csk->receive_queue, skb);
0830     cxgbi_conn_pdu_ready(csk);
0831 
0832     spin_unlock_bh(&csk->lock);
0833     return 0;
0834 
0835 abort_conn:
0836     send_abort_req(csk);
0837 discard:
0838     spin_unlock_bh(&csk->lock);
0839     __kfree_skb(skb);
0840     return 0;
0841 }
0842 
0843 /*
0844  * Process TX_DATA_ACK CPL messages: -> host
0845  * Process an acknowledgment of WR completion.  Advance snd_una and send the
0846  * next batch of work requests from the write queue.
0847  */
0848 static int do_wr_ack(struct t3cdev *cdev, struct sk_buff *skb, void *ctx)
0849 {
0850     struct cxgbi_sock *csk = ctx;
0851     struct cpl_wr_ack *hdr = cplhdr(skb);
0852 
0853     log_debug(1 << CXGBI_DBG_TOE | 1 << CXGBI_DBG_PDU_RX,
0854         "csk 0x%p,%u,0x%lx,%u, cr %u.\n",
0855         csk, csk->state, csk->flags, csk->tid, ntohs(hdr->credits));
0856 
0857     cxgbi_sock_rcv_wr_ack(csk, ntohs(hdr->credits), ntohl(hdr->snd_una), 1);
0858     __kfree_skb(skb);
0859     return 0;
0860 }
0861 
0862 /*
0863  * for each connection, pre-allocate skbs needed for close/abort requests. So
0864  * that we can service the request right away.
0865  */
0866 static int alloc_cpls(struct cxgbi_sock *csk)
0867 {
0868     csk->cpl_close = alloc_wr(sizeof(struct cpl_close_con_req), 0,
0869                     GFP_KERNEL);
0870     if (!csk->cpl_close)
0871         return -ENOMEM;
0872     csk->cpl_abort_req = alloc_wr(sizeof(struct cpl_abort_req), 0,
0873                     GFP_KERNEL);
0874     if (!csk->cpl_abort_req)
0875         goto free_cpl_skbs;
0876 
0877     csk->cpl_abort_rpl = alloc_wr(sizeof(struct cpl_abort_rpl), 0,
0878                     GFP_KERNEL);
0879     if (!csk->cpl_abort_rpl)
0880         goto free_cpl_skbs;
0881 
0882     return 0;
0883 
0884 free_cpl_skbs:
0885     cxgbi_sock_free_cpl_skbs(csk);
0886     return -ENOMEM;
0887 }
0888 
0889 static void l2t_put(struct cxgbi_sock *csk)
0890 {
0891     struct t3cdev *t3dev = (struct t3cdev *)csk->cdev->lldev;
0892 
0893     if (csk->l2t) {
0894         l2t_release(t3dev, csk->l2t);
0895         csk->l2t = NULL;
0896         cxgbi_sock_put(csk);
0897     }
0898 }
0899 
0900 /*
0901  * release_offload_resources - release offload resource
0902  * Release resources held by an offload connection (TID, L2T entry, etc.)
0903  */
0904 static void release_offload_resources(struct cxgbi_sock *csk)
0905 {
0906     struct t3cdev *t3dev = (struct t3cdev *)csk->cdev->lldev;
0907 
0908     log_debug(1 << CXGBI_DBG_TOE | 1 << CXGBI_DBG_SOCK,
0909         "csk 0x%p,%u,0x%lx,%u.\n",
0910         csk, csk->state, csk->flags, csk->tid);
0911 
0912     csk->rss_qid = 0;
0913     cxgbi_sock_free_cpl_skbs(csk);
0914 
0915     if (csk->wr_cred != csk->wr_max_cred) {
0916         cxgbi_sock_purge_wr_queue(csk);
0917         cxgbi_sock_reset_wr_list(csk);
0918     }
0919     l2t_put(csk);
0920     if (cxgbi_sock_flag(csk, CTPF_HAS_ATID))
0921         free_atid(csk);
0922     else if (cxgbi_sock_flag(csk, CTPF_HAS_TID)) {
0923         cxgb3_remove_tid(t3dev, (void *)csk, csk->tid);
0924         cxgbi_sock_clear_flag(csk, CTPF_HAS_TID);
0925         cxgbi_sock_put(csk);
0926     }
0927     csk->dst = NULL;
0928     csk->cdev = NULL;
0929 }
0930 
0931 static void update_address(struct cxgbi_hba *chba)
0932 {
0933     if (chba->ipv4addr) {
0934         if (chba->vdev &&
0935             chba->ipv4addr != cxgb3i_get_private_ipv4addr(chba->vdev)) {
0936             cxgb3i_set_private_ipv4addr(chba->vdev, chba->ipv4addr);
0937             cxgb3i_set_private_ipv4addr(chba->ndev, 0);
0938             pr_info("%s set %pI4.\n",
0939                 chba->vdev->name, &chba->ipv4addr);
0940         } else if (chba->ipv4addr !=
0941                 cxgb3i_get_private_ipv4addr(chba->ndev)) {
0942             cxgb3i_set_private_ipv4addr(chba->ndev, chba->ipv4addr);
0943             pr_info("%s set %pI4.\n",
0944                 chba->ndev->name, &chba->ipv4addr);
0945         }
0946     } else if (cxgb3i_get_private_ipv4addr(chba->ndev)) {
0947         if (chba->vdev)
0948             cxgb3i_set_private_ipv4addr(chba->vdev, 0);
0949         cxgb3i_set_private_ipv4addr(chba->ndev, 0);
0950     }
0951 }
0952 
0953 static int init_act_open(struct cxgbi_sock *csk)
0954 {
0955     struct dst_entry *dst = csk->dst;
0956     struct cxgbi_device *cdev = csk->cdev;
0957     struct t3cdev *t3dev = (struct t3cdev *)cdev->lldev;
0958     struct net_device *ndev = cdev->ports[csk->port_id];
0959     struct cxgbi_hba *chba = cdev->hbas[csk->port_id];
0960     struct sk_buff *skb = NULL;
0961     int ret;
0962 
0963     log_debug(1 << CXGBI_DBG_TOE | 1 << CXGBI_DBG_SOCK,
0964         "csk 0x%p,%u,0x%lx.\n", csk, csk->state, csk->flags);
0965 
0966     update_address(chba);
0967     if (chba->ipv4addr)
0968         csk->saddr.sin_addr.s_addr = chba->ipv4addr;
0969 
0970     csk->rss_qid = 0;
0971     csk->l2t = t3_l2t_get(t3dev, dst, ndev,
0972                   &csk->daddr.sin_addr.s_addr);
0973     if (!csk->l2t) {
0974         pr_err("NO l2t available.\n");
0975         return -EINVAL;
0976     }
0977     cxgbi_sock_get(csk);
0978 
0979     csk->atid = cxgb3_alloc_atid(t3dev, &t3_client, csk);
0980     if (csk->atid < 0) {
0981         pr_err("NO atid available.\n");
0982         ret = -EINVAL;
0983         goto put_sock;
0984     }
0985     cxgbi_sock_set_flag(csk, CTPF_HAS_ATID);
0986     cxgbi_sock_get(csk);
0987 
0988     skb = alloc_wr(sizeof(struct cpl_act_open_req), 0, GFP_KERNEL);
0989     if (!skb) {
0990         ret = -ENOMEM;
0991         goto free_atid;
0992     }
0993     skb->sk = (struct sock *)csk;
0994     set_arp_failure_handler(skb, act_open_arp_failure);
0995     csk->snd_win = cxgb3i_snd_win;
0996     csk->rcv_win = cxgb3i_rcv_win;
0997 
0998     csk->wr_max_cred = csk->wr_cred = T3C_DATA(t3dev)->max_wrs - 1;
0999     csk->wr_una_cred = 0;
1000     csk->mss_idx = cxgbi_sock_select_mss(csk, dst_mtu(dst));
1001     cxgbi_sock_reset_wr_list(csk);
1002     csk->err = 0;
1003 
1004     log_debug(1 << CXGBI_DBG_TOE | 1 << CXGBI_DBG_SOCK,
1005         "csk 0x%p,%u,0x%lx, %pI4:%u-%pI4:%u.\n",
1006         csk, csk->state, csk->flags,
1007         &csk->saddr.sin_addr.s_addr, ntohs(csk->saddr.sin_port),
1008         &csk->daddr.sin_addr.s_addr, ntohs(csk->daddr.sin_port));
1009 
1010     cxgbi_sock_set_state(csk, CTP_ACTIVE_OPEN);
1011     send_act_open_req(csk, skb, csk->l2t);
1012     return 0;
1013 
1014 free_atid:
1015     cxgb3_free_atid(t3dev, csk->atid);
1016 put_sock:
1017     cxgbi_sock_put(csk);
1018     l2t_release(t3dev, csk->l2t);
1019     csk->l2t = NULL;
1020 
1021     return ret;
1022 }
1023 
1024 cxgb3_cpl_handler_func cxgb3i_cpl_handlers[NUM_CPL_CMDS] = {
1025     [CPL_ACT_ESTABLISH] = do_act_establish,
1026     [CPL_ACT_OPEN_RPL] = do_act_open_rpl,
1027     [CPL_PEER_CLOSE] = do_peer_close,
1028     [CPL_ABORT_REQ_RSS] = do_abort_req,
1029     [CPL_ABORT_RPL_RSS] = do_abort_rpl,
1030     [CPL_CLOSE_CON_RPL] = do_close_con_rpl,
1031     [CPL_TX_DMA_ACK] = do_wr_ack,
1032     [CPL_ISCSI_HDR] = do_iscsi_hdr,
1033 };
1034 
1035 /**
1036  * cxgb3i_ofld_init - allocate and initialize resources for each adapter found
1037  * @cdev:   cxgbi adapter
1038  */
1039 static int cxgb3i_ofld_init(struct cxgbi_device *cdev)
1040 {
1041     struct t3cdev *t3dev = (struct t3cdev *)cdev->lldev;
1042     struct adap_ports port;
1043     struct ofld_page_info rx_page_info;
1044     unsigned int wr_len;
1045     int rc;
1046 
1047     if (t3dev->ctl(t3dev, GET_WR_LEN, &wr_len) < 0 ||
1048         t3dev->ctl(t3dev, GET_PORTS, &port) < 0 ||
1049         t3dev->ctl(t3dev, GET_RX_PAGE_INFO, &rx_page_info) < 0) {
1050         pr_warn("t3 0x%p, offload up, ioctl failed.\n", t3dev);
1051         return -EINVAL;
1052     }
1053 
1054     if (cxgb3i_max_connect > CXGBI_MAX_CONN)
1055         cxgb3i_max_connect = CXGBI_MAX_CONN;
1056 
1057     rc = cxgbi_device_portmap_create(cdev, cxgb3i_sport_base,
1058                     cxgb3i_max_connect);
1059     if (rc < 0)
1060         return rc;
1061 
1062     init_wr_tab(wr_len);
1063     cdev->csk_release_offload_resources = release_offload_resources;
1064     cdev->csk_push_tx_frames = push_tx_frames;
1065     cdev->csk_send_abort_req = send_abort_req;
1066     cdev->csk_send_close_req = send_close_req;
1067     cdev->csk_send_rx_credits = send_rx_credits;
1068     cdev->csk_alloc_cpls = alloc_cpls;
1069     cdev->csk_init_act_open = init_act_open;
1070 
1071     pr_info("cdev 0x%p, offload up, added.\n", cdev);
1072     return 0;
1073 }
1074 
1075 /*
1076  * functions to program the pagepod in h/w
1077  */
1078 static inline void ulp_mem_io_set_hdr(struct sk_buff *skb, unsigned int addr)
1079 {
1080     struct ulp_mem_io *req = (struct ulp_mem_io *)skb->head;
1081 
1082     memset(req, 0, sizeof(*req));
1083 
1084     req->wr.wr_hi = htonl(V_WR_OP(FW_WROPCODE_BYPASS));
1085     req->cmd_lock_addr = htonl(V_ULP_MEMIO_ADDR(addr >> 5) |
1086                    V_ULPTX_CMD(ULP_MEM_WRITE));
1087     req->len = htonl(V_ULP_MEMIO_DATA_LEN(IPPOD_SIZE >> 5) |
1088              V_ULPTX_NFLITS((IPPOD_SIZE >> 3) + 1));
1089 }
1090 
1091 static struct cxgbi_ppm *cdev2ppm(struct cxgbi_device *cdev)
1092 {
1093     return ((struct t3cdev *)cdev->lldev)->ulp_iscsi;
1094 }
1095 
1096 static int ddp_set_map(struct cxgbi_ppm *ppm, struct cxgbi_sock *csk,
1097                struct cxgbi_task_tag_info *ttinfo)
1098 {
1099     unsigned int idx = ttinfo->idx;
1100     unsigned int npods = ttinfo->npods;
1101     struct scatterlist *sg = ttinfo->sgl;
1102     struct cxgbi_pagepod *ppod;
1103     struct ulp_mem_io *req;
1104     unsigned int sg_off;
1105     unsigned int pm_addr = (idx << PPOD_SIZE_SHIFT) + ppm->llimit;
1106     int i;
1107 
1108     for (i = 0; i < npods; i++, idx++, pm_addr += IPPOD_SIZE) {
1109         struct sk_buff *skb = alloc_wr(sizeof(struct ulp_mem_io) +
1110                            IPPOD_SIZE, 0, GFP_ATOMIC);
1111 
1112         if (!skb)
1113             return -ENOMEM;
1114         ulp_mem_io_set_hdr(skb, pm_addr);
1115         req = (struct ulp_mem_io *)skb->head;
1116         ppod = (struct cxgbi_pagepod *)(req + 1);
1117         sg_off = i * PPOD_PAGES_MAX;
1118         cxgbi_ddp_set_one_ppod(ppod, ttinfo, &sg,
1119                        &sg_off);
1120         skb->priority = CPL_PRIORITY_CONTROL;
1121         cxgb3_ofld_send(ppm->lldev, skb);
1122     }
1123     return 0;
1124 }
1125 
1126 static void ddp_clear_map(struct cxgbi_device *cdev, struct cxgbi_ppm *ppm,
1127               struct cxgbi_task_tag_info *ttinfo)
1128 {
1129     unsigned int idx = ttinfo->idx;
1130     unsigned int pm_addr = (idx << PPOD_SIZE_SHIFT) + ppm->llimit;
1131     unsigned int npods = ttinfo->npods;
1132     int i;
1133 
1134     log_debug(1 << CXGBI_DBG_DDP,
1135           "cdev 0x%p, clear idx %u, npods %u.\n",
1136           cdev, idx, npods);
1137 
1138     for (i = 0; i < npods; i++, idx++, pm_addr += IPPOD_SIZE) {
1139         struct sk_buff *skb = alloc_wr(sizeof(struct ulp_mem_io) +
1140                            IPPOD_SIZE, 0, GFP_ATOMIC);
1141 
1142         if (!skb) {
1143             pr_err("cdev 0x%p, clear ddp, %u,%d/%u, skb OOM.\n",
1144                    cdev, idx, i, npods);
1145             continue;
1146         }
1147         ulp_mem_io_set_hdr(skb, pm_addr);
1148         skb->priority = CPL_PRIORITY_CONTROL;
1149         cxgb3_ofld_send(ppm->lldev, skb);
1150     }
1151 }
1152 
1153 static int ddp_setup_conn_pgidx(struct cxgbi_sock *csk,
1154                 unsigned int tid, int pg_idx)
1155 {
1156     struct sk_buff *skb = alloc_wr(sizeof(struct cpl_set_tcb_field), 0,
1157                     GFP_KERNEL);
1158     struct cpl_set_tcb_field *req;
1159     u64 val = pg_idx < DDP_PGIDX_MAX ? pg_idx : 0;
1160 
1161     log_debug(1 << CXGBI_DBG_DDP,
1162         "csk 0x%p, tid %u, pg_idx %d.\n", csk, tid, pg_idx);
1163     if (!skb)
1164         return -ENOMEM;
1165 
1166     /* set up ulp submode and page size */
1167     req = (struct cpl_set_tcb_field *)skb->head;
1168     req->wr.wr_hi = htonl(V_WR_OP(FW_WROPCODE_FORWARD));
1169     OPCODE_TID(req) = htonl(MK_OPCODE_TID(CPL_SET_TCB_FIELD, tid));
1170     req->reply = V_NO_REPLY(1);
1171     req->cpu_idx = 0;
1172     req->word = htons(31);
1173     req->mask = cpu_to_be64(0xF0000000);
1174     req->val = cpu_to_be64(val << 28);
1175     skb->priority = CPL_PRIORITY_CONTROL;
1176 
1177     cxgb3_ofld_send(csk->cdev->lldev, skb);
1178     return 0;
1179 }
1180 
1181 /**
1182  * ddp_setup_conn_digest - setup conn. digest setting
1183  * @csk: cxgb tcp socket
1184  * @tid: connection id
1185  * @hcrc: header digest enabled
1186  * @dcrc: data digest enabled
1187  * set up the iscsi digest settings for a connection identified by tid
1188  */
1189 static int ddp_setup_conn_digest(struct cxgbi_sock *csk, unsigned int tid,
1190                  int hcrc, int dcrc)
1191 {
1192     struct sk_buff *skb = alloc_wr(sizeof(struct cpl_set_tcb_field), 0,
1193                     GFP_KERNEL);
1194     struct cpl_set_tcb_field *req;
1195     u64 val = (hcrc ? 1 : 0) | (dcrc ? 2 : 0);
1196 
1197     log_debug(1 << CXGBI_DBG_DDP,
1198         "csk 0x%p, tid %u, crc %d,%d.\n", csk, tid, hcrc, dcrc);
1199     if (!skb)
1200         return -ENOMEM;
1201 
1202     /* set up ulp submode and page size */
1203     req = (struct cpl_set_tcb_field *)skb->head;
1204     req->wr.wr_hi = htonl(V_WR_OP(FW_WROPCODE_FORWARD));
1205     OPCODE_TID(req) = htonl(MK_OPCODE_TID(CPL_SET_TCB_FIELD, tid));
1206     req->reply = V_NO_REPLY(1);
1207     req->cpu_idx = 0;
1208     req->word = htons(31);
1209     req->mask = cpu_to_be64(0x0F000000);
1210     req->val = cpu_to_be64(val << 24);
1211     skb->priority = CPL_PRIORITY_CONTROL;
1212 
1213     cxgb3_ofld_send(csk->cdev->lldev, skb);
1214     return 0;
1215 }
1216 
1217 /**
1218  * cxgb3i_ddp_init - initialize the cxgb3 adapter's ddp resource
1219  * @cdev: cxgb3i adapter
1220  * initialize the ddp pagepod manager for a given adapter
1221  */
1222 static int cxgb3i_ddp_init(struct cxgbi_device *cdev)
1223 {
1224     struct t3cdev *tdev = (struct t3cdev *)cdev->lldev;
1225     struct net_device *ndev = cdev->ports[0];
1226     struct cxgbi_tag_format tformat;
1227     unsigned int ppmax, tagmask = 0;
1228     struct ulp_iscsi_info uinfo;
1229     int i, err;
1230 
1231     err = tdev->ctl(tdev, ULP_ISCSI_GET_PARAMS, &uinfo);
1232     if (err < 0) {
1233         pr_err("%s, failed to get iscsi param %d.\n",
1234                ndev->name, err);
1235         return err;
1236     }
1237     if (uinfo.llimit >= uinfo.ulimit) {
1238         pr_warn("T3 %s, iscsi NOT enabled %u ~ %u!\n",
1239             ndev->name, uinfo.llimit, uinfo.ulimit);
1240         return -EACCES;
1241     }
1242 
1243     ppmax = (uinfo.ulimit - uinfo.llimit + 1) >> PPOD_SIZE_SHIFT;
1244     tagmask = cxgbi_tagmask_set(ppmax);
1245 
1246     pr_info("T3 %s: 0x%x~0x%x, 0x%x, tagmask 0x%x -> 0x%x.\n",
1247         ndev->name, uinfo.llimit, uinfo.ulimit, ppmax, uinfo.tagmask,
1248         tagmask);
1249 
1250     memset(&tformat, 0, sizeof(struct cxgbi_tag_format));
1251     for (i = 0; i < 4; i++)
1252         tformat.pgsz_order[i] = uinfo.pgsz_factor[i];
1253     cxgbi_tagmask_check(tagmask, &tformat);
1254 
1255     err = cxgbi_ddp_ppm_setup(&tdev->ulp_iscsi, cdev, &tformat,
1256                   (uinfo.ulimit - uinfo.llimit + 1),
1257                   uinfo.llimit, uinfo.llimit, 0, 0, 0);
1258     if (err)
1259         return err;
1260 
1261     if (!(cdev->flags & CXGBI_FLAG_DDP_OFF)) {
1262         uinfo.tagmask = tagmask;
1263         uinfo.ulimit = uinfo.llimit + (ppmax << PPOD_SIZE_SHIFT);
1264 
1265         err = tdev->ctl(tdev, ULP_ISCSI_SET_PARAMS, &uinfo);
1266         if (err < 0) {
1267             pr_err("T3 %s fail to set iscsi param %d.\n",
1268                    ndev->name, err);
1269             cdev->flags |= CXGBI_FLAG_DDP_OFF;
1270         }
1271         err = 0;
1272     }
1273 
1274     cdev->csk_ddp_setup_digest = ddp_setup_conn_digest;
1275     cdev->csk_ddp_setup_pgidx = ddp_setup_conn_pgidx;
1276     cdev->csk_ddp_set_map = ddp_set_map;
1277     cdev->csk_ddp_clear_map = ddp_clear_map;
1278     cdev->cdev2ppm = cdev2ppm;
1279     cdev->tx_max_size = min_t(unsigned int, ULP2_MAX_PDU_PAYLOAD,
1280                   uinfo.max_txsz - ISCSI_PDU_NONPAYLOAD_LEN);
1281     cdev->rx_max_size = min_t(unsigned int, ULP2_MAX_PDU_PAYLOAD,
1282                   uinfo.max_rxsz - ISCSI_PDU_NONPAYLOAD_LEN);
1283 
1284     return 0;
1285 }
1286 
1287 static void cxgb3i_dev_close(struct t3cdev *t3dev)
1288 {
1289     struct cxgbi_device *cdev = cxgbi_device_find_by_lldev(t3dev);
1290 
1291     if (!cdev || cdev->flags & CXGBI_FLAG_ADAPTER_RESET) {
1292         pr_info("0x%p close, f 0x%x.\n", cdev, cdev ? cdev->flags : 0);
1293         return;
1294     }
1295 
1296     cxgbi_device_unregister(cdev);
1297 }
1298 
1299 /**
1300  * cxgb3i_dev_open - init a t3 adapter structure and any h/w settings
1301  * @t3dev: t3cdev adapter
1302  */
1303 static void cxgb3i_dev_open(struct t3cdev *t3dev)
1304 {
1305     struct cxgbi_device *cdev = cxgbi_device_find_by_lldev(t3dev);
1306     struct adapter *adapter = tdev2adap(t3dev);
1307     int i, err;
1308 
1309     if (cdev) {
1310         pr_info("0x%p, updating.\n", cdev);
1311         return;
1312     }
1313 
1314     cdev = cxgbi_device_register(0, adapter->params.nports);
1315     if (!cdev) {
1316         pr_warn("device 0x%p register failed.\n", t3dev);
1317         return;
1318     }
1319 
1320     cdev->flags = CXGBI_FLAG_DEV_T3 | CXGBI_FLAG_IPV4_SET;
1321     cdev->lldev = t3dev;
1322     cdev->pdev = adapter->pdev;
1323     cdev->ports = adapter->port;
1324     cdev->nports = adapter->params.nports;
1325     cdev->mtus = adapter->params.mtus;
1326     cdev->nmtus = NMTUS;
1327     cdev->rx_credit_thres = cxgb3i_rx_credit_thres;
1328     cdev->skb_tx_rsvd = CXGB3I_TX_HEADER_LEN;
1329     cdev->skb_rx_extra = sizeof(struct cpl_iscsi_hdr_norss);
1330     cdev->itp = &cxgb3i_iscsi_transport;
1331 
1332     err = cxgb3i_ddp_init(cdev);
1333     if (err) {
1334         pr_info("0x%p ddp init failed %d\n", cdev, err);
1335         goto err_out;
1336     }
1337 
1338     err = cxgb3i_ofld_init(cdev);
1339     if (err) {
1340         pr_info("0x%p offload init failed\n", cdev);
1341         goto err_out;
1342     }
1343 
1344     err = cxgbi_hbas_add(cdev, CXGB3I_MAX_LUN, CXGBI_MAX_CONN,
1345                 &cxgb3i_host_template, cxgb3i_stt);
1346     if (err)
1347         goto err_out;
1348 
1349     for (i = 0; i < cdev->nports; i++)
1350         cdev->hbas[i]->ipv4addr =
1351             cxgb3i_get_private_ipv4addr(cdev->ports[i]);
1352 
1353     pr_info("cdev 0x%p, f 0x%x, t3dev 0x%p open, err %d.\n",
1354         cdev, cdev ? cdev->flags : 0, t3dev, err);
1355     return;
1356 
1357 err_out:
1358     cxgbi_device_unregister(cdev);
1359 }
1360 
1361 static void cxgb3i_dev_event_handler(struct t3cdev *t3dev, u32 event, u32 port)
1362 {
1363     struct cxgbi_device *cdev = cxgbi_device_find_by_lldev(t3dev);
1364 
1365     log_debug(1 << CXGBI_DBG_TOE,
1366         "0x%p, cdev 0x%p, event 0x%x, port 0x%x.\n",
1367         t3dev, cdev, event, port);
1368     if (!cdev)
1369         return;
1370 
1371     switch (event) {
1372     case OFFLOAD_STATUS_DOWN:
1373         cdev->flags |= CXGBI_FLAG_ADAPTER_RESET;
1374         break;
1375     case OFFLOAD_STATUS_UP:
1376         cdev->flags &= ~CXGBI_FLAG_ADAPTER_RESET;
1377         break;
1378     }
1379 }
1380 
1381 /**
1382  * cxgb3i_init_module - module init entry point
1383  *
1384  * initialize any driver wide global data structures and register itself
1385  *  with the cxgb3 module
1386  */
1387 static int __init cxgb3i_init_module(void)
1388 {
1389     int rc;
1390 
1391     printk(KERN_INFO "%s", version);
1392 
1393     rc = cxgbi_iscsi_init(&cxgb3i_iscsi_transport, &cxgb3i_stt);
1394     if (rc < 0)
1395         return rc;
1396 
1397     cxgb3_register_client(&t3_client);
1398     return 0;
1399 }
1400 
1401 /**
1402  * cxgb3i_exit_module - module cleanup/exit entry point
1403  *
1404  * go through the driver hba list and for each hba, release any resource held.
1405  *  and unregisters iscsi transport and the cxgb3 module
1406  */
1407 static void __exit cxgb3i_exit_module(void)
1408 {
1409     cxgb3_unregister_client(&t3_client);
1410     cxgbi_device_unregister_all(CXGBI_FLAG_DEV_T3);
1411     cxgbi_iscsi_cleanup(&cxgb3i_iscsi_transport, &cxgb3i_stt);
1412 }
1413 
1414 module_init(cxgb3i_init_module);
1415 module_exit(cxgb3i_exit_module);
[ Source navigation ] [ Diff markup ] [ Identifier search ] [ general search ]

This page was automatically generated by the 2.1.0 LXR engine.

The LXR team
Valid CSS 2.1! Valid HTML 4.01!