scsi/lpfc/lpfc_sli.h

0001 /*******************************************************************
0002  * This file is part of the Emulex Linux Device Driver for         *
0003  * Fibre Channel Host Bus Adapters.                                *
0004  * Copyright (C) 2017-2022 Broadcom. All Rights Reserved. The term *
0005  * “Broadcom” refers to Broadcom Inc. and/or its subsidiaries.     *
0006  * Copyright (C) 2004-2016 Emulex.  All rights reserved.           *
0007  * EMULEX and SLI are trademarks of Emulex.                        *
0008  * www.broadcom.com                                                *
0009  *                                                                 *
0010  * This program is free software; you can redistribute it and/or   *
0011  * modify it under the terms of version 2 of the GNU General       *
0012  * Public License as published by the Free Software Foundation.    *
0013  * This program is distributed in the hope that it will be useful. *
0014  * ALL EXPRESS OR IMPLIED CONDITIONS, REPRESENTATIONS AND          *
0015  * WARRANTIES, INCLUDING ANY IMPLIED WARRANTY OF MERCHANTABILITY,  *
0016  * FITNESS FOR A PARTICULAR PURPOSE, OR NON-INFRINGEMENT, ARE      *
0017  * DISCLAIMED, EXCEPT TO THE EXTENT THAT SUCH DISCLAIMERS ARE HELD *
0018  * TO BE LEGALLY INVALID.  See the GNU General Public License for  *
0019  * more details, a copy of which can be found in the file COPYING  *
0020  * included with this package.                                     *
0021  *******************************************************************/
0022
0023 #if defined(CONFIG_DEBUG_FS) && !defined(CONFIG_SCSI_LPFC_DEBUG_FS)
0024 #define CONFIG_SCSI_LPFC_DEBUG_FS
0025 #endif
0026
0027 /* forward declaration for LPFC_IOCB_t's use */
0028 struct lpfc_hba;
0029 struct lpfc_vport;
0030
0031 /* Define the context types that SLI handles for abort and sums. */
0032 typedef enum _lpfc_ctx_cmd {
0033     LPFC_CTX_LUN,
0034     LPFC_CTX_TGT,
0035     LPFC_CTX_HOST
0036 } lpfc_ctx_cmd;
0037
0038 /* Enumeration to describe the thread lock context. */
0039 enum lpfc_mbox_ctx {
0040     MBOX_THD_UNLOCKED,
0041     MBOX_THD_LOCKED
0042 };
0043
0044 union lpfc_vmid_tag {
0045     uint32_t app_id;
0046     uint8_t cs_ctl_vmid;
0047     struct lpfc_vmid_context *vmid_context; /* UVEM context information */
0048 };
0049
0050 struct lpfc_cq_event {
0051     struct list_head list;
0052     uint16_t hdwq;
0053     union {
0054         struct lpfc_mcqe        mcqe_cmpl;
0055         struct lpfc_acqe_link       acqe_link;
0056         struct lpfc_acqe_fip        acqe_fip;
0057         struct lpfc_acqe_dcbx       acqe_dcbx;
0058         struct lpfc_acqe_grp5       acqe_grp5;
0059         struct lpfc_acqe_fc_la      acqe_fc;
0060         struct lpfc_acqe_sli        acqe_sli;
0061         struct lpfc_rcqe        rcqe_cmpl;
0062         struct sli4_wcqe_xri_aborted    wcqe_axri;
0063         struct lpfc_wcqe_complete   wcqe_cmpl;
0064     } cqe;
0065 };
0066
0067 /* This structure is used to handle IOCB requests / responses */
0068 struct lpfc_iocbq {
0069     /* lpfc_iocbqs are used in double linked lists */
0070     struct list_head list;
0071     struct list_head clist;
0072     struct list_head dlist;
0073     uint16_t iotag;         /* pre-assigned IO tag */
0074     uint16_t sli4_lxritag;  /* logical pre-assigned XRI. */
0075     uint16_t sli4_xritag;   /* pre-assigned XRI, (OXID) tag. */
0076     uint16_t hba_wqidx;     /* index to HBA work queue */
0077     struct lpfc_cq_event cq_event;
0078     uint64_t isr_timestamp;
0079
0080     union lpfc_wqe128 wqe;  /* SLI-4 */
0081     IOCB_t iocb;        /* SLI-3 */
0082     struct lpfc_wcqe_complete wcqe_cmpl;    /* WQE cmpl */
0083
0084     u32 unsol_rcv_len;  /* Receive len in usol path */
0085
0086     /* Pack the u8's together and make them module-4. */
0087     u8 num_bdes;    /* Number of BDEs */
0088     u8 abort_bls;   /* ABTS by initiator or responder */
0089     u8 abort_rctl;  /* ACC or RJT flag */
0090     u8 priority;    /* OAS priority */
0091     u8 retry;   /* retry counter for IOCB cmd - if needed */
0092     u8 rsvd1;       /* Pad for u32 */
0093     u8 rsvd2;       /* Pad for u32 */
0094     u8 rsvd3;   /* Pad for u32 */
0095
0096     u32 cmd_flag;
0097 #define LPFC_IO_LIBDFC      1   /* libdfc iocb */
0098 #define LPFC_IO_WAKE        2   /* Synchronous I/O completed */
0099 #define LPFC_IO_WAKE_TMO    LPFC_IO_WAKE /* Synchronous I/O timed out */
0100 #define LPFC_IO_FCP     4   /* FCP command -- iocbq in scsi_buf */
0101 #define LPFC_DRIVER_ABORTED 8   /* driver aborted this request */
0102 #define LPFC_IO_FABRIC      0x10    /* Iocb send using fabric scheduler */
0103 #define LPFC_DELAY_MEM_FREE 0x20    /* Defer free'ing of FC data */
0104 #define LPFC_EXCHANGE_BUSY  0x40    /* SLI4 hba reported XB in response */
0105 #define LPFC_USE_FCPWQIDX   0x80    /* Submit to specified FCPWQ index */
0106 #define DSS_SECURITY_OP     0x100   /* security IO */
0107 #define LPFC_IO_ON_TXCMPLQ  0x200   /* The IO is still on the TXCMPLQ */
0108 #define LPFC_IO_DIF_PASS    0x400   /* T10 DIF IO pass-thru prot */
0109 #define LPFC_IO_DIF_STRIP   0x800   /* T10 DIF IO strip prot */
0110 #define LPFC_IO_DIF_INSERT  0x1000  /* T10 DIF IO insert prot */
0111 #define LPFC_IO_CMD_OUTSTANDING 0x2000 /* timeout handler abort window */
0112
0113 #define LPFC_FIP_ELS_ID_MASK    0xc000  /* ELS_ID range 0-3, non-shifted mask */
0114 #define LPFC_FIP_ELS_ID_SHIFT   14
0115
0116 #define LPFC_IO_OAS     0x10000 /* OAS FCP IO */
0117 #define LPFC_IO_FOF     0x20000 /* FOF FCP IO */
0118 #define LPFC_IO_LOOPBACK    0x40000 /* Loopback IO */
0119 #define LPFC_PRLI_NVME_REQ  0x80000 /* This is an NVME PRLI. */
0120 #define LPFC_PRLI_FCP_REQ   0x100000 /* This is an NVME PRLI. */
0121 #define LPFC_IO_NVME            0x200000 /* NVME FCP command */
0122 #define LPFC_IO_NVME_LS     0x400000 /* NVME LS command */
0123 #define LPFC_IO_NVMET       0x800000 /* NVMET command */
0124 #define LPFC_IO_VMID            0x1000000 /* VMID tagged IO */
0125 #define LPFC_IO_CMF     0x4000000 /* CMF command */
0126
0127     uint32_t drvrTimeout;   /* driver timeout in seconds */
0128     struct lpfc_vport *vport;/* virtual port pointer */
0129     struct lpfc_dmabuf *cmd_dmabuf;
0130     struct lpfc_dmabuf *rsp_dmabuf;
0131     struct lpfc_dmabuf *bpl_dmabuf;
0132     uint32_t event_tag; /* LA Event tag */
0133     union {
0134         wait_queue_head_t    *wait_queue;
0135         struct lpfcMboxq     *mbox;
0136         struct lpfc_node_rrq *rrq;
0137         struct nvmefc_ls_req *nvme_lsreq;
0138         struct lpfc_async_xchg_ctx *axchg;
0139         struct bsg_job_data *dd_data;
0140     } context_un;
0141
0142     struct lpfc_io_buf *io_buf;
0143     struct lpfc_iocbq *rsp_iocb;
0144     struct lpfc_nodelist *ndlp;
0145     union lpfc_vmid_tag vmid_tag;
0146     void (*fabric_cmd_cmpl)(struct lpfc_hba *phba, struct lpfc_iocbq *cmd,
0147                 struct lpfc_iocbq *rsp);
0148     void (*wait_cmd_cmpl)(struct lpfc_hba *phba, struct lpfc_iocbq *cmd,
0149                   struct lpfc_iocbq *rsp);
0150     void (*cmd_cmpl)(struct lpfc_hba *phba, struct lpfc_iocbq *cmd,
0151              struct lpfc_iocbq *rsp);
0152 };
0153
0154 #define SLI_IOCB_RET_IOCB      1    /* Return IOCB if cmd ring full */
0155
0156 #define IOCB_SUCCESS        0
0157 #define IOCB_BUSY           1
0158 #define IOCB_ERROR          2
0159 #define IOCB_TIMEDOUT       3
0160 #define IOCB_ABORTED        4
0161 #define IOCB_ABORTING       5
0162 #define IOCB_NORESOURCE     6
0163
0164 #define SLI_WQE_RET_WQE    1    /* Return WQE if cmd ring full */
0165
0166 #define WQE_SUCCESS        0
0167 #define WQE_BUSY           1
0168 #define WQE_ERROR          2
0169 #define WQE_TIMEDOUT       3
0170 #define WQE_ABORTED        4
0171 #define WQE_ABORTING       5
0172 #define WQE_NORESOURCE     6
0173
0174 #define LPFC_MBX_WAKE       1
0175 #define LPFC_MBX_IMED_UNREG 2
0176
0177 typedef struct lpfcMboxq {
0178     /* MBOXQs are used in single linked lists */
0179     struct list_head list;  /* ptr to next mailbox command */
0180     union {
0181         MAILBOX_t mb;       /* Mailbox cmd */
0182         struct lpfc_mqe mqe;
0183     } u;
0184     struct lpfc_vport *vport; /* virtual port pointer */
0185     void *ctx_ndlp;       /* caller ndlp information */
0186     void *ctx_buf;        /* caller buffer information */
0187     void *context3;
0188
0189     void (*mbox_cmpl) (struct lpfc_hba *, struct lpfcMboxq *);
0190     uint8_t mbox_flag;
0191     uint16_t in_ext_byte_len;
0192     uint16_t out_ext_byte_len;
0193     uint8_t  mbox_offset_word;
0194     struct lpfc_mcqe mcqe;
0195     struct lpfc_mbx_nembed_sge_virt *sge_array;
0196 } LPFC_MBOXQ_t;
0197
0198 #define MBX_POLL        1   /* poll mailbox till command done, then
0199                    return */
0200 #define MBX_NOWAIT      2   /* issue command then return immediately */
0201
0202 #define LPFC_MAX_RING_MASK  5   /* max num of rctl/type masks allowed per
0203                    ring */
0204 #define LPFC_SLI3_MAX_RING  4   /* Max num of SLI3 rings used by driver.
0205                    For SLI4, an additional ring for each
0206                    FCP WQ will be allocated.  */
0207
0208 struct lpfc_sli_ring;
0209
0210 struct lpfc_sli_ring_mask {
0211     uint8_t profile;    /* profile associated with ring */
0212     uint8_t rctl;   /* rctl / type pair configured for ring */
0213     uint8_t type;   /* rctl / type pair configured for ring */
0214     uint8_t rsvd;
0215     /* rcv'd unsol event */
0216     void (*lpfc_sli_rcv_unsol_event) (struct lpfc_hba *,
0217                      struct lpfc_sli_ring *,
0218                      struct lpfc_iocbq *);
0219 };
0220
0221
0222 /* Structure used to hold SLI statistical counters and info */
0223 struct lpfc_sli_ring_stat {
0224     uint64_t iocb_event;     /* IOCB event counters */
0225     uint64_t iocb_cmd;   /* IOCB cmd issued */
0226     uint64_t iocb_rsp;   /* IOCB rsp received */
0227     uint64_t iocb_cmd_delay; /* IOCB cmd ring delay */
0228     uint64_t iocb_cmd_full;  /* IOCB cmd ring full */
0229     uint64_t iocb_cmd_empty; /* IOCB cmd ring is now empty */
0230     uint64_t iocb_rsp_full;  /* IOCB rsp ring full */
0231 };
0232
0233 struct lpfc_sli3_ring {
0234     uint32_t local_getidx;  /* last available cmd index (from cmdGetInx) */
0235     uint32_t next_cmdidx;   /* next_cmd index */
0236     uint32_t rspidx;    /* current index in response ring */
0237     uint32_t cmdidx;    /* current index in command ring */
0238     uint16_t numCiocb;  /* number of command iocb's per ring */
0239     uint16_t numRiocb;  /* number of rsp iocb's per ring */
0240     uint16_t sizeCiocb; /* Size of command iocb's in this ring */
0241     uint16_t sizeRiocb; /* Size of response iocb's in this ring */
0242     uint32_t *cmdringaddr;  /* virtual address for cmd rings */
0243     uint32_t *rspringaddr;  /* virtual address for rsp rings */
0244 };
0245
0246 struct lpfc_sli4_ring {
0247     struct lpfc_queue *wqp; /* Pointer to associated WQ */
0248 };
0249
0250
0251 /* Structure used to hold SLI ring information */
0252 struct lpfc_sli_ring {
0253     uint16_t flag;      /* ring flags */
0254 #define LPFC_DEFERRED_RING_EVENT 0x001  /* Deferred processing a ring event */
0255 #define LPFC_CALL_RING_AVAILABLE 0x002  /* indicates cmd was full */
0256 #define LPFC_STOP_IOCB_EVENT     0x020  /* Stop processing IOCB cmds event */
0257     uint16_t abtsiotag; /* tracks next iotag to use for ABTS */
0258
0259     uint8_t rsvd;
0260     uint8_t ringno;     /* ring number */
0261
0262     spinlock_t ring_lock;   /* lock for issuing commands */
0263
0264     uint32_t fast_iotag;    /* max fastlookup based iotag           */
0265     uint32_t iotag_ctr; /* keeps track of the next iotag to use */
0266     uint32_t iotag_max; /* max iotag value to use               */
0267     struct list_head txq;
0268     uint16_t txq_cnt;   /* current length of queue */
0269     uint16_t txq_max;   /* max length */
0270     struct list_head txcmplq;
0271     uint16_t txcmplq_cnt;   /* current length of queue */
0272     uint16_t txcmplq_max;   /* max length */
0273     uint32_t missbufcnt;    /* keep track of buffers to post */
0274     struct list_head postbufq;
0275     uint16_t postbufq_cnt;  /* current length of queue */
0276     uint16_t postbufq_max;  /* max length */
0277     struct list_head iocb_continueq;
0278     uint16_t iocb_continueq_cnt;    /* current length of queue */
0279     uint16_t iocb_continueq_max;    /* max length */
0280     struct list_head iocb_continue_saveq;
0281
0282     struct lpfc_sli_ring_mask prt[LPFC_MAX_RING_MASK];
0283     uint32_t num_mask;  /* number of mask entries in prt array */
0284     void (*lpfc_sli_rcv_async_status) (struct lpfc_hba *,
0285         struct lpfc_sli_ring *, struct lpfc_iocbq *);
0286
0287     struct lpfc_sli_ring_stat stats;    /* SLI statistical info */
0288
0289     /* cmd ring available */
0290     void (*lpfc_sli_cmd_available) (struct lpfc_hba *,
0291                     struct lpfc_sli_ring *);
0292     union {
0293         struct lpfc_sli3_ring sli3;
0294         struct lpfc_sli4_ring sli4;
0295     } sli;
0296 };
0297
0298 /* Structure used for configuring rings to a specific profile or rctl / type */
0299 struct lpfc_hbq_init {
0300     uint32_t rn;        /* Receive buffer notification */
0301     uint32_t entry_count;   /* max # of entries in HBQ */
0302     uint32_t headerLen; /* 0 if not profile 4 or 5 */
0303     uint32_t logEntry;  /* Set to 1 if this HBQ used for LogEntry */
0304     uint32_t profile;   /* Selection profile 0=all, 7=logentry */
0305     uint32_t ring_mask; /* Binds HBQ to a ring e.g. Ring0=b0001,
0306                  * ring2=b0100 */
0307     uint32_t hbq_index; /* index of this hbq in ring .HBQs[] */
0308
0309     uint32_t seqlenoff;
0310     uint32_t maxlen;
0311     uint32_t seqlenbcnt;
0312     uint32_t cmdcodeoff;
0313     uint32_t cmdmatch[8];
0314     uint32_t mask_count;    /* number of mask entries in prt array */
0315     struct hbq_mask hbqMasks[6];
0316
0317     /* Non-config rings fields to keep track of buffer allocations */
0318     uint32_t buffer_count;  /* number of buffers allocated */
0319     uint32_t init_count;    /* number to allocate when initialized */
0320     uint32_t add_count; /* number to allocate when starved */
0321 } ;
0322
0323 /* Structure used to hold SLI statistical counters and info */
0324 struct lpfc_sli_stat {
0325     uint64_t mbox_stat_err;  /* Mbox cmds completed status error */
0326     uint64_t mbox_cmd;       /* Mailbox commands issued */
0327     uint64_t sli_intr;       /* Count of Host Attention interrupts */
0328     uint64_t sli_prev_intr;  /* Previous cnt of Host Attention interrupts */
0329     uint64_t sli_ips;        /* Host Attention interrupts per sec */
0330     uint32_t err_attn_event; /* Error Attn event counters */
0331     uint32_t link_event;     /* Link event counters */
0332     uint32_t mbox_event;     /* Mailbox event counters */
0333     uint32_t mbox_busy;  /* Mailbox cmd busy */
0334 };
0335
0336 /* Structure to store link status values when port stats are reset */
0337 struct lpfc_lnk_stat {
0338     uint32_t link_failure_count;
0339     uint32_t loss_of_sync_count;
0340     uint32_t loss_of_signal_count;
0341     uint32_t prim_seq_protocol_err_count;
0342     uint32_t invalid_tx_word_count;
0343     uint32_t invalid_crc_count;
0344     uint32_t error_frames;
0345     uint32_t link_events;
0346 };
0347
0348 /* Structure used to hold SLI information */
0349 struct lpfc_sli {
0350     uint32_t num_rings;
0351     uint32_t sli_flag;
0352
0353     /* Additional sli_flags */
0354 #define LPFC_SLI_MBOX_ACTIVE      0x100 /* HBA mailbox is currently active */
0355 #define LPFC_SLI_ACTIVE           0x200 /* SLI in firmware is active */
0356 #define LPFC_PROCESS_LA           0x400 /* Able to process link attention */
0357 #define LPFC_BLOCK_MGMT_IO        0x800 /* Don't allow mgmt mbx or iocb cmds */
0358 #define LPFC_SLI_ASYNC_MBX_BLK    0x2000 /* Async mailbox is blocked */
0359 #define LPFC_SLI_SUPPRESS_RSP     0x4000 /* Suppress RSP feature is supported */
0360 #define LPFC_SLI_USE_EQDR         0x8000 /* EQ Delay Register is supported */
0361 #define LPFC_QUEUE_FREE_INIT      0x10000 /* Queue freeing is in progress */
0362 #define LPFC_QUEUE_FREE_WAIT      0x20000 /* Hold Queue free as it is being
0363                        * used outside worker thread
0364                        */
0365
0366     struct lpfc_sli_ring *sli3_ring;
0367
0368     struct lpfc_sli_stat slistat;   /* SLI statistical info */
0369     struct list_head mboxq;
0370     uint16_t mboxq_cnt; /* current length of queue */
0371     uint16_t mboxq_max; /* max length */
0372     LPFC_MBOXQ_t *mbox_active;  /* active mboxq information */
0373     struct list_head mboxq_cmpl;
0374
0375     struct timer_list mbox_tmo; /* Hold clk to timeout active mbox
0376                        cmd */
0377
0378 #define LPFC_IOCBQ_LOOKUP_INCREMENT  1024
0379     struct lpfc_iocbq ** iocbq_lookup; /* array to lookup IOCB by IOTAG */
0380     size_t iocbq_lookup_len;           /* current lengs of the array */
0381     uint16_t  last_iotag;              /* last allocated IOTAG */
0382     time64_t  stats_start;         /* in seconds */
0383     struct lpfc_lnk_stat lnk_stat_offsets;
0384 };
0385
0386 /* Timeout for normal outstanding mbox command (Seconds) */
0387 #define LPFC_MBOX_TMO               30
0388 /* Timeout for non-flash-based outstanding sli_config mbox command (Seconds) */
0389 #define LPFC_MBOX_SLI4_CONFIG_TMO       60
0390 /* Timeout for flash-based outstanding sli_config mbox command (Seconds) */
0391 #define LPFC_MBOX_SLI4_CONFIG_EXTENDED_TMO  300
0392 /* Timeout for other flash-based outstanding mbox command (Seconds) */
0393 #define LPFC_MBOX_TMO_FLASH_CMD         300
0394
0395 struct lpfc_io_buf {
0396     /* Common fields */
0397     struct list_head list;
0398     void *data;
0399
0400     dma_addr_t dma_handle;
0401     dma_addr_t dma_phys_sgl;
0402
0403     struct sli4_sge *dma_sgl; /* initial segment chunk */
0404
0405     /* linked list of extra sli4_hybrid_sge */
0406     struct list_head dma_sgl_xtra_list;
0407
0408     /* list head for fcp_cmd_rsp buf */
0409     struct list_head dma_cmd_rsp_list;
0410
0411     struct lpfc_iocbq cur_iocbq;
0412     struct lpfc_sli4_hdw_queue *hdwq;
0413     uint16_t hdwq_no;
0414     uint16_t cpu;
0415
0416     struct lpfc_nodelist *ndlp;
0417     uint32_t timeout;
0418     uint16_t flags;
0419 #define LPFC_SBUF_XBUSY     0x1 /* SLI4 hba reported XB on WCQE cmpl */
0420 #define LPFC_SBUF_BUMP_QDEPTH   0x2 /* bumped queue depth counter */
0421                     /* External DIF device IO conversions */
0422 #define LPFC_SBUF_NORMAL_DIF    0x4 /* normal mode to insert/strip */
0423 #define LPFC_SBUF_PASS_DIF  0x8 /* insert/strip mode to passthru */
0424 #define LPFC_SBUF_NOT_POSTED    0x10    /* SGL failed post to FW. */
0425     uint16_t status;    /* From IOCB Word 7- ulpStatus */
0426     uint32_t result;    /* From IOCB Word 4. */
0427
0428     uint32_t   seg_cnt; /* Number of scatter-gather segments returned by
0429                  * dma_map_sg.  The driver needs this for calls
0430                  * to dma_unmap_sg.
0431                  */
0432     unsigned long start_time;
0433     spinlock_t buf_lock;    /* lock used in case of simultaneous abort */
0434     bool expedite;      /* this is an expedite io_buf */
0435
0436     union {
0437         /* SCSI specific fields */
0438         struct {
0439             struct scsi_cmnd *pCmd;
0440             struct lpfc_rport_data *rdata;
0441             uint32_t prot_seg_cnt;  /* seg_cnt's counterpart for
0442                          * protection data
0443                          */
0444
0445             /*
0446              * data and dma_handle are the kernel virtual and bus
0447              * address of the dma-able buffer containing the
0448              * fcp_cmd, fcp_rsp and a scatter gather bde list that
0449              * supports the sg_tablesize value.
0450              */
0451             struct fcp_cmnd *fcp_cmnd;
0452             struct fcp_rsp *fcp_rsp;
0453
0454             wait_queue_head_t *waitq;
0455
0456 #ifdef CONFIG_SCSI_LPFC_DEBUG_FS
0457             /* Used to restore any changes to protection data for
0458              * error injection
0459              */
0460             void *prot_data_segment;
0461             uint32_t prot_data;
0462             uint32_t prot_data_type;
0463 #define LPFC_INJERR_REFTAG  1
0464 #define LPFC_INJERR_APPTAG  2
0465 #define LPFC_INJERR_GUARD   3
0466 #endif
0467         };
0468
0469         /* NVME specific fields */
0470         struct {
0471             struct nvmefc_fcp_req *nvmeCmd;
0472             uint16_t qidx;
0473         };
0474     };
0475 #ifdef CONFIG_SCSI_LPFC_DEBUG_FS
0476     uint64_t ts_cmd_start;
0477     uint64_t ts_last_cmd;
0478     uint64_t ts_cmd_wqput;
0479     uint64_t ts_isr_cmpl;
0480     uint64_t ts_data_io;
0481 #endif
0482     uint64_t rx_cmd_start;
0483 };