Back to home page

OSCL-LXR
 
 

     

0001 // SPDX-License-Identifier: (GPL-2.0-only OR BSD-3-Clause)
0002 /* Copyright 2021 Marvell. All rights reserved. */
0003 
0004 #include <linux/kernel.h>
0005 #include <linux/module.h>
0006 #include <linux/pci.h>
0007 #include <linux/list.h>
0008 #include <linux/mm.h>
0009 #include <linux/types.h>
0010 #include <asm/byteorder.h>
0011 #include <linux/qed/common_hsi.h>
0012 #include <linux/qed/storage_common.h>
0013 #include <linux/qed/nvmetcp_common.h>
0014 #include <linux/qed/qed_nvmetcp_if.h>
0015 #include "qed_nvmetcp_fw_funcs.h"
0016 
0017 #define NVMETCP_NUM_SGES_IN_CACHE 0x4
0018 
0019 bool nvmetcp_is_slow_sgl(u16 num_sges, bool small_mid_sge)
0020 {
0021     return (num_sges > SCSI_NUM_SGES_SLOW_SGL_THR && small_mid_sge);
0022 }
0023 
0024 void init_scsi_sgl_context(struct scsi_sgl_params *ctx_sgl_params,
0025                struct scsi_cached_sges *ctx_data_desc,
0026                struct storage_sgl_task_params *sgl_params)
0027 {
0028     u8 num_sges_to_init = (u8)(sgl_params->num_sges > NVMETCP_NUM_SGES_IN_CACHE ?
0029                    NVMETCP_NUM_SGES_IN_CACHE : sgl_params->num_sges);
0030     u8 sge_index;
0031 
0032     /* sgl params */
0033     ctx_sgl_params->sgl_addr.lo = cpu_to_le32(sgl_params->sgl_phys_addr.lo);
0034     ctx_sgl_params->sgl_addr.hi = cpu_to_le32(sgl_params->sgl_phys_addr.hi);
0035     ctx_sgl_params->sgl_total_length = cpu_to_le32(sgl_params->total_buffer_size);
0036     ctx_sgl_params->sgl_num_sges = cpu_to_le16(sgl_params->num_sges);
0037 
0038     for (sge_index = 0; sge_index < num_sges_to_init; sge_index++) {
0039         ctx_data_desc->sge[sge_index].sge_addr.lo =
0040             cpu_to_le32(sgl_params->sgl[sge_index].sge_addr.lo);
0041         ctx_data_desc->sge[sge_index].sge_addr.hi =
0042             cpu_to_le32(sgl_params->sgl[sge_index].sge_addr.hi);
0043         ctx_data_desc->sge[sge_index].sge_len =
0044             cpu_to_le32(sgl_params->sgl[sge_index].sge_len);
0045     }
0046 }
0047 
0048 static inline u32 calc_rw_task_size(struct nvmetcp_task_params *task_params,
0049                     enum nvmetcp_task_type task_type)
0050 {
0051     u32 io_size;
0052 
0053     if (task_type == NVMETCP_TASK_TYPE_HOST_WRITE)
0054         io_size = task_params->tx_io_size;
0055     else
0056         io_size = task_params->rx_io_size;
0057 
0058     if (unlikely(!io_size))
0059         return 0;
0060 
0061     return io_size;
0062 }
0063 
0064 static inline void init_sqe(struct nvmetcp_task_params *task_params,
0065                 struct storage_sgl_task_params *sgl_task_params,
0066                 enum nvmetcp_task_type task_type)
0067 {
0068     if (!task_params->sqe)
0069         return;
0070 
0071     memset(task_params->sqe, 0, sizeof(*task_params->sqe));
0072     task_params->sqe->task_id = cpu_to_le16(task_params->itid);
0073 
0074     switch (task_type) {
0075     case NVMETCP_TASK_TYPE_HOST_WRITE: {
0076         u32 buf_size = 0;
0077         u32 num_sges = 0;
0078 
0079         SET_FIELD(task_params->sqe->contlen_cdbsize,
0080               NVMETCP_WQE_CDB_SIZE_OR_NVMETCP_CMD, 1);
0081         SET_FIELD(task_params->sqe->flags, NVMETCP_WQE_WQE_TYPE,
0082               NVMETCP_WQE_TYPE_NORMAL);
0083         if (task_params->tx_io_size) {
0084             if (task_params->send_write_incapsule)
0085                 buf_size = calc_rw_task_size(task_params, task_type);
0086 
0087             if (nvmetcp_is_slow_sgl(sgl_task_params->num_sges,
0088                         sgl_task_params->small_mid_sge))
0089                 num_sges = NVMETCP_WQE_NUM_SGES_SLOWIO;
0090             else
0091                 num_sges = min((u16)sgl_task_params->num_sges,
0092                            (u16)SCSI_NUM_SGES_SLOW_SGL_THR);
0093         }
0094         SET_FIELD(task_params->sqe->flags, NVMETCP_WQE_NUM_SGES, num_sges);
0095         SET_FIELD(task_params->sqe->contlen_cdbsize, NVMETCP_WQE_CONT_LEN, buf_size);
0096     } break;
0097 
0098     case NVMETCP_TASK_TYPE_HOST_READ: {
0099         SET_FIELD(task_params->sqe->flags, NVMETCP_WQE_WQE_TYPE,
0100               NVMETCP_WQE_TYPE_NORMAL);
0101         SET_FIELD(task_params->sqe->contlen_cdbsize,
0102               NVMETCP_WQE_CDB_SIZE_OR_NVMETCP_CMD, 1);
0103     } break;
0104 
0105     case NVMETCP_TASK_TYPE_INIT_CONN_REQUEST: {
0106         SET_FIELD(task_params->sqe->flags, NVMETCP_WQE_WQE_TYPE,
0107               NVMETCP_WQE_TYPE_MIDDLE_PATH);
0108 
0109         if (task_params->tx_io_size) {
0110             SET_FIELD(task_params->sqe->contlen_cdbsize, NVMETCP_WQE_CONT_LEN,
0111                   task_params->tx_io_size);
0112             SET_FIELD(task_params->sqe->flags, NVMETCP_WQE_NUM_SGES,
0113                   min((u16)sgl_task_params->num_sges,
0114                       (u16)SCSI_NUM_SGES_SLOW_SGL_THR));
0115         }
0116     } break;
0117 
0118     case NVMETCP_TASK_TYPE_CLEANUP:
0119         SET_FIELD(task_params->sqe->flags, NVMETCP_WQE_WQE_TYPE,
0120               NVMETCP_WQE_TYPE_TASK_CLEANUP);
0121 
0122     default:
0123         break;
0124     }
0125 }
0126 
0127 /* The following function initializes of NVMeTCP task params */
0128 static inline void
0129 init_nvmetcp_task_params(struct e5_nvmetcp_task_context *context,
0130              struct nvmetcp_task_params *task_params,
0131              enum nvmetcp_task_type task_type)
0132 {
0133     context->ystorm_st_context.state.cccid = task_params->host_cccid;
0134     SET_FIELD(context->ustorm_st_context.error_flags, USTORM_NVMETCP_TASK_ST_CTX_NVME_TCP, 1);
0135     context->ustorm_st_context.nvme_tcp_opaque_lo = cpu_to_le32(task_params->opq.lo);
0136     context->ustorm_st_context.nvme_tcp_opaque_hi = cpu_to_le32(task_params->opq.hi);
0137 }
0138 
0139 /* The following function initializes default values to all tasks */
0140 static inline void
0141 init_default_nvmetcp_task(struct nvmetcp_task_params *task_params,
0142               void *pdu_header, void *nvme_cmd,
0143               enum nvmetcp_task_type task_type)
0144 {
0145     struct e5_nvmetcp_task_context *context = task_params->context;
0146     const u8 val_byte = context->mstorm_ag_context.cdu_validation;
0147     u8 dw_index;
0148 
0149     memset(context, 0, sizeof(*context));
0150     init_nvmetcp_task_params(context, task_params,
0151                  (enum nvmetcp_task_type)task_type);
0152 
0153     /* Swapping requirements used below, will be removed in future FW versions */
0154     if (task_type == NVMETCP_TASK_TYPE_HOST_WRITE ||
0155         task_type == NVMETCP_TASK_TYPE_HOST_READ) {
0156         for (dw_index = 0;
0157              dw_index < QED_NVMETCP_CMN_HDR_SIZE / sizeof(u32);
0158              dw_index++)
0159             context->ystorm_st_context.pdu_hdr.task_hdr.reg[dw_index] =
0160                 cpu_to_le32(__swab32(((u32 *)pdu_header)[dw_index]));
0161 
0162         for (dw_index = QED_NVMETCP_CMN_HDR_SIZE / sizeof(u32);
0163              dw_index < QED_NVMETCP_CMD_HDR_SIZE / sizeof(u32);
0164              dw_index++)
0165             context->ystorm_st_context.pdu_hdr.task_hdr.reg[dw_index] =
0166                 cpu_to_le32(__swab32(((u32 *)nvme_cmd)[dw_index - 2]));
0167     } else {
0168         for (dw_index = 0;
0169              dw_index < QED_NVMETCP_NON_IO_HDR_SIZE / sizeof(u32);
0170              dw_index++)
0171             context->ystorm_st_context.pdu_hdr.task_hdr.reg[dw_index] =
0172                 cpu_to_le32(__swab32(((u32 *)pdu_header)[dw_index]));
0173     }
0174 
0175     /* M-Storm Context: */
0176     context->mstorm_ag_context.cdu_validation = val_byte;
0177     context->mstorm_st_context.task_type = (u8)(task_type);
0178     context->mstorm_ag_context.task_cid = cpu_to_le16(task_params->conn_icid);
0179 
0180     /* Ustorm Context: */
0181     SET_FIELD(context->ustorm_ag_context.flags1, E5_USTORM_NVMETCP_TASK_AG_CTX_R2T2RECV, 1);
0182     context->ustorm_st_context.task_type = (u8)(task_type);
0183     context->ustorm_st_context.cq_rss_number = task_params->cq_rss_number;
0184     context->ustorm_ag_context.icid = cpu_to_le16(task_params->conn_icid);
0185 }
0186 
0187 /* The following function initializes the U-Storm Task Contexts */
0188 static inline void
0189 init_ustorm_task_contexts(struct ustorm_nvmetcp_task_st_ctx *ustorm_st_context,
0190               struct e5_ustorm_nvmetcp_task_ag_ctx *ustorm_ag_context,
0191               u32 remaining_recv_len,
0192               u32 expected_data_transfer_len, u8 num_sges,
0193               bool tx_dif_conn_err_en)
0194 {
0195     /* Remaining data to be received in bytes. Used in validations*/
0196     ustorm_st_context->rem_rcv_len = cpu_to_le32(remaining_recv_len);
0197     ustorm_ag_context->exp_data_acked = cpu_to_le32(expected_data_transfer_len);
0198     ustorm_st_context->exp_data_transfer_len = cpu_to_le32(expected_data_transfer_len);
0199     SET_FIELD(ustorm_st_context->reg1_map, REG1_NUM_SGES, num_sges);
0200     SET_FIELD(ustorm_ag_context->flags2, E5_USTORM_NVMETCP_TASK_AG_CTX_DIF_ERROR_CF_EN,
0201           tx_dif_conn_err_en ? 1 : 0);
0202 }
0203 
0204 /* The following function initializes Local Completion Contexts: */
0205 static inline void
0206 set_local_completion_context(struct e5_nvmetcp_task_context *context)
0207 {
0208     SET_FIELD(context->ystorm_st_context.state.flags,
0209           YSTORM_NVMETCP_TASK_STATE_LOCAL_COMP, 1);
0210     SET_FIELD(context->ustorm_st_context.flags,
0211           USTORM_NVMETCP_TASK_ST_CTX_LOCAL_COMP, 1);
0212 }
0213 
0214 /* Common Fastpath task init function: */
0215 static inline void
0216 init_rw_nvmetcp_task(struct nvmetcp_task_params *task_params,
0217              enum nvmetcp_task_type task_type,
0218              void *pdu_header, void *nvme_cmd,
0219              struct storage_sgl_task_params *sgl_task_params)
0220 {
0221     struct e5_nvmetcp_task_context *context = task_params->context;
0222     u32 task_size = calc_rw_task_size(task_params, task_type);
0223     bool slow_io = false;
0224     u8 num_sges = 0;
0225 
0226     init_default_nvmetcp_task(task_params, pdu_header, nvme_cmd, task_type);
0227 
0228     /* Tx/Rx: */
0229     if (task_params->tx_io_size) {
0230         /* if data to transmit: */
0231         init_scsi_sgl_context(&context->ystorm_st_context.state.sgl_params,
0232                       &context->ystorm_st_context.state.data_desc,
0233                       sgl_task_params);
0234         slow_io = nvmetcp_is_slow_sgl(sgl_task_params->num_sges,
0235                           sgl_task_params->small_mid_sge);
0236         num_sges =
0237             (u8)(!slow_io ? min((u32)sgl_task_params->num_sges,
0238                         (u32)SCSI_NUM_SGES_SLOW_SGL_THR) :
0239                         NVMETCP_WQE_NUM_SGES_SLOWIO);
0240         if (slow_io) {
0241             SET_FIELD(context->ystorm_st_context.state.flags,
0242                   YSTORM_NVMETCP_TASK_STATE_SLOW_IO, 1);
0243         }
0244     } else if (task_params->rx_io_size) {
0245         /* if data to receive: */
0246         init_scsi_sgl_context(&context->mstorm_st_context.sgl_params,
0247                       &context->mstorm_st_context.data_desc,
0248                       sgl_task_params);
0249         num_sges =
0250             (u8)(!nvmetcp_is_slow_sgl(sgl_task_params->num_sges,
0251                           sgl_task_params->small_mid_sge) ?
0252                           min((u32)sgl_task_params->num_sges,
0253                               (u32)SCSI_NUM_SGES_SLOW_SGL_THR) :
0254                               NVMETCP_WQE_NUM_SGES_SLOWIO);
0255         context->mstorm_st_context.rem_task_size = cpu_to_le32(task_size);
0256     }
0257 
0258     /* Ustorm context: */
0259     init_ustorm_task_contexts(&context->ustorm_st_context,
0260                   &context->ustorm_ag_context,
0261                   /* Remaining Receive length is the Task Size */
0262                   task_size,
0263                   /* The size of the transmitted task */
0264                   task_size,
0265                   /* num_sges */
0266                   num_sges,
0267                   false);
0268 
0269     /* Set exp_data_acked */
0270     if (task_type == NVMETCP_TASK_TYPE_HOST_WRITE) {
0271         if (task_params->send_write_incapsule)
0272             context->ustorm_ag_context.exp_data_acked = task_size;
0273         else
0274             context->ustorm_ag_context.exp_data_acked = 0;
0275     } else if (task_type == NVMETCP_TASK_TYPE_HOST_READ) {
0276         context->ustorm_ag_context.exp_data_acked = 0;
0277     }
0278 
0279     context->ustorm_ag_context.exp_cont_len = 0;
0280     init_sqe(task_params, sgl_task_params, task_type);
0281 }
0282 
0283 static void
0284 init_common_initiator_read_task(struct nvmetcp_task_params *task_params,
0285                 struct nvme_tcp_cmd_pdu *cmd_pdu_header,
0286                 struct nvme_command *nvme_cmd,
0287                 struct storage_sgl_task_params *sgl_task_params)
0288 {
0289     init_rw_nvmetcp_task(task_params, NVMETCP_TASK_TYPE_HOST_READ,
0290                  cmd_pdu_header, nvme_cmd, sgl_task_params);
0291 }
0292 
0293 void init_nvmetcp_host_read_task(struct nvmetcp_task_params *task_params,
0294                  struct nvme_tcp_cmd_pdu *cmd_pdu_header,
0295                  struct nvme_command *nvme_cmd,
0296                  struct storage_sgl_task_params *sgl_task_params)
0297 {
0298     init_common_initiator_read_task(task_params, (void *)cmd_pdu_header,
0299                     (void *)nvme_cmd, sgl_task_params);
0300 }
0301 
0302 static void
0303 init_common_initiator_write_task(struct nvmetcp_task_params *task_params,
0304                  struct nvme_tcp_cmd_pdu *cmd_pdu_header,
0305                  struct nvme_command *nvme_cmd,
0306                  struct storage_sgl_task_params *sgl_task_params)
0307 {
0308     init_rw_nvmetcp_task(task_params, NVMETCP_TASK_TYPE_HOST_WRITE,
0309                  cmd_pdu_header, nvme_cmd, sgl_task_params);
0310 }
0311 
0312 void init_nvmetcp_host_write_task(struct nvmetcp_task_params *task_params,
0313                   struct nvme_tcp_cmd_pdu *cmd_pdu_header,
0314                   struct nvme_command *nvme_cmd,
0315                   struct storage_sgl_task_params *sgl_task_params)
0316 {
0317     init_common_initiator_write_task(task_params, (void *)cmd_pdu_header,
0318                      (void *)nvme_cmd, sgl_task_params);
0319 }
0320 
0321 static void
0322 init_common_login_request_task(struct nvmetcp_task_params *task_params,
0323                    void *login_req_pdu_header,
0324                    struct storage_sgl_task_params *tx_sgl_task_params,
0325                    struct storage_sgl_task_params *rx_sgl_task_params)
0326 {
0327     struct e5_nvmetcp_task_context *context = task_params->context;
0328 
0329     init_default_nvmetcp_task(task_params, (void *)login_req_pdu_header, NULL,
0330                   NVMETCP_TASK_TYPE_INIT_CONN_REQUEST);
0331 
0332     /* Ustorm Context: */
0333     init_ustorm_task_contexts(&context->ustorm_st_context,
0334                   &context->ustorm_ag_context,
0335 
0336                   /* Remaining Receive length is the Task Size */
0337                   task_params->rx_io_size ?
0338                   rx_sgl_task_params->total_buffer_size : 0,
0339 
0340                   /* The size of the transmitted task */
0341                   task_params->tx_io_size ?
0342                   tx_sgl_task_params->total_buffer_size : 0,
0343                   0, /* num_sges */
0344                   0); /* tx_dif_conn_err_en */
0345 
0346     /* SGL context: */
0347     if (task_params->tx_io_size)
0348         init_scsi_sgl_context(&context->ystorm_st_context.state.sgl_params,
0349                       &context->ystorm_st_context.state.data_desc,
0350                       tx_sgl_task_params);
0351     if (task_params->rx_io_size)
0352         init_scsi_sgl_context(&context->mstorm_st_context.sgl_params,
0353                       &context->mstorm_st_context.data_desc,
0354                       rx_sgl_task_params);
0355 
0356     context->mstorm_st_context.rem_task_size =
0357         cpu_to_le32(task_params->rx_io_size ?
0358                  rx_sgl_task_params->total_buffer_size : 0);
0359     init_sqe(task_params, tx_sgl_task_params, NVMETCP_TASK_TYPE_INIT_CONN_REQUEST);
0360 }
0361 
0362 /* The following function initializes Login task in Host mode: */
0363 void init_nvmetcp_init_conn_req_task(struct nvmetcp_task_params *task_params,
0364                      struct nvme_tcp_icreq_pdu *init_conn_req_pdu_hdr,
0365                      struct storage_sgl_task_params *tx_sgl_task_params,
0366                      struct storage_sgl_task_params *rx_sgl_task_params)
0367 {
0368     init_common_login_request_task(task_params, init_conn_req_pdu_hdr,
0369                        tx_sgl_task_params, rx_sgl_task_params);
0370 }
0371 
0372 void init_cleanup_task_nvmetcp(struct nvmetcp_task_params *task_params)
0373 {
0374     init_sqe(task_params, NULL, NVMETCP_TASK_TYPE_CLEANUP);
0375 }
[ Source navigation ] [ Diff markup ] [ Identifier search ] [ general search ]

This page was automatically generated by the 2.1.0 LXR engine.

The LXR team
Valid CSS 2.1! Valid HTML 4.01!