0001
0002
0003 #ifndef IRDMA_USER_H
0004 #define IRDMA_USER_H
0005
0006 #define irdma_handle void *
0007 #define irdma_adapter_handle irdma_handle
0008 #define irdma_qp_handle irdma_handle
0009 #define irdma_cq_handle irdma_handle
0010 #define irdma_pd_id irdma_handle
0011 #define irdma_stag_handle irdma_handle
0012 #define irdma_stag_index u32
0013 #define irdma_stag u32
0014 #define irdma_stag_key u8
0015 #define irdma_tagged_offset u64
0016 #define irdma_access_privileges u32
0017 #define irdma_physical_fragment u64
0018 #define irdma_address_list u64 *
0019
0020 #define IRDMA_MAX_MR_SIZE 0x200000000000ULL
0021
0022 #define IRDMA_ACCESS_FLAGS_LOCALREAD 0x01
0023 #define IRDMA_ACCESS_FLAGS_LOCALWRITE 0x02
0024 #define IRDMA_ACCESS_FLAGS_REMOTEREAD_ONLY 0x04
0025 #define IRDMA_ACCESS_FLAGS_REMOTEREAD 0x05
0026 #define IRDMA_ACCESS_FLAGS_REMOTEWRITE_ONLY 0x08
0027 #define IRDMA_ACCESS_FLAGS_REMOTEWRITE 0x0a
0028 #define IRDMA_ACCESS_FLAGS_BIND_WINDOW 0x10
0029 #define IRDMA_ACCESS_FLAGS_ZERO_BASED 0x20
0030 #define IRDMA_ACCESS_FLAGS_ALL 0x3f
0031
0032 #define IRDMA_OP_TYPE_RDMA_WRITE 0x00
0033 #define IRDMA_OP_TYPE_RDMA_READ 0x01
0034 #define IRDMA_OP_TYPE_SEND 0x03
0035 #define IRDMA_OP_TYPE_SEND_INV 0x04
0036 #define IRDMA_OP_TYPE_SEND_SOL 0x05
0037 #define IRDMA_OP_TYPE_SEND_SOL_INV 0x06
0038 #define IRDMA_OP_TYPE_RDMA_WRITE_SOL 0x0d
0039 #define IRDMA_OP_TYPE_BIND_MW 0x08
0040 #define IRDMA_OP_TYPE_FAST_REG_NSMR 0x09
0041 #define IRDMA_OP_TYPE_INV_STAG 0x0a
0042 #define IRDMA_OP_TYPE_RDMA_READ_INV_STAG 0x0b
0043 #define IRDMA_OP_TYPE_NOP 0x0c
0044 #define IRDMA_OP_TYPE_REC 0x3e
0045 #define IRDMA_OP_TYPE_REC_IMM 0x3f
0046
0047 #define IRDMA_FLUSH_MAJOR_ERR 1
0048
0049 enum irdma_device_caps_const {
0050 IRDMA_WQE_SIZE = 4,
0051 IRDMA_CQP_WQE_SIZE = 8,
0052 IRDMA_CQE_SIZE = 4,
0053 IRDMA_EXTENDED_CQE_SIZE = 8,
0054 IRDMA_AEQE_SIZE = 2,
0055 IRDMA_CEQE_SIZE = 1,
0056 IRDMA_CQP_CTX_SIZE = 8,
0057 IRDMA_SHADOW_AREA_SIZE = 8,
0058 IRDMA_QUERY_FPM_BUF_SIZE = 176,
0059 IRDMA_COMMIT_FPM_BUF_SIZE = 176,
0060 IRDMA_GATHER_STATS_BUF_SIZE = 1024,
0061 IRDMA_MIN_IW_QP_ID = 0,
0062 IRDMA_MAX_IW_QP_ID = 262143,
0063 IRDMA_MIN_CEQID = 0,
0064 IRDMA_MAX_CEQID = 1023,
0065 IRDMA_CEQ_MAX_COUNT = IRDMA_MAX_CEQID + 1,
0066 IRDMA_MIN_CQID = 0,
0067 IRDMA_MAX_CQID = 524287,
0068 IRDMA_MIN_AEQ_ENTRIES = 1,
0069 IRDMA_MAX_AEQ_ENTRIES = 524287,
0070 IRDMA_MIN_CEQ_ENTRIES = 1,
0071 IRDMA_MAX_CEQ_ENTRIES = 262143,
0072 IRDMA_MIN_CQ_SIZE = 1,
0073 IRDMA_MAX_CQ_SIZE = 1048575,
0074 IRDMA_DB_ID_ZERO = 0,
0075 IRDMA_MAX_WQ_FRAGMENT_COUNT = 13,
0076 IRDMA_MAX_SGE_RD = 13,
0077 IRDMA_MAX_OUTBOUND_MSG_SIZE = 2147483647,
0078 IRDMA_MAX_INBOUND_MSG_SIZE = 2147483647,
0079 IRDMA_MAX_PUSH_PAGE_COUNT = 1024,
0080 IRDMA_MAX_PE_ENA_VF_COUNT = 32,
0081 IRDMA_MAX_VF_FPM_ID = 47,
0082 IRDMA_MAX_SQ_PAYLOAD_SIZE = 2145386496,
0083 IRDMA_MAX_INLINE_DATA_SIZE = 101,
0084 IRDMA_MAX_WQ_ENTRIES = 32768,
0085 IRDMA_Q2_BUF_SIZE = 256,
0086 IRDMA_QP_CTX_SIZE = 256,
0087 IRDMA_MAX_PDS = 262144,
0088 };
0089
0090 enum irdma_addressing_type {
0091 IRDMA_ADDR_TYPE_ZERO_BASED = 0,
0092 IRDMA_ADDR_TYPE_VA_BASED = 1,
0093 };
0094
0095 enum irdma_flush_opcode {
0096 FLUSH_INVALID = 0,
0097 FLUSH_GENERAL_ERR,
0098 FLUSH_PROT_ERR,
0099 FLUSH_REM_ACCESS_ERR,
0100 FLUSH_LOC_QP_OP_ERR,
0101 FLUSH_REM_OP_ERR,
0102 FLUSH_LOC_LEN_ERR,
0103 FLUSH_FATAL_ERR,
0104 FLUSH_RETRY_EXC_ERR,
0105 FLUSH_MW_BIND_ERR,
0106 };
0107
0108 enum irdma_cmpl_status {
0109 IRDMA_COMPL_STATUS_SUCCESS = 0,
0110 IRDMA_COMPL_STATUS_FLUSHED,
0111 IRDMA_COMPL_STATUS_INVALID_WQE,
0112 IRDMA_COMPL_STATUS_QP_CATASTROPHIC,
0113 IRDMA_COMPL_STATUS_REMOTE_TERMINATION,
0114 IRDMA_COMPL_STATUS_INVALID_STAG,
0115 IRDMA_COMPL_STATUS_BASE_BOUND_VIOLATION,
0116 IRDMA_COMPL_STATUS_ACCESS_VIOLATION,
0117 IRDMA_COMPL_STATUS_INVALID_PD_ID,
0118 IRDMA_COMPL_STATUS_WRAP_ERROR,
0119 IRDMA_COMPL_STATUS_STAG_INVALID_PDID,
0120 IRDMA_COMPL_STATUS_RDMA_READ_ZERO_ORD,
0121 IRDMA_COMPL_STATUS_QP_NOT_PRIVLEDGED,
0122 IRDMA_COMPL_STATUS_STAG_NOT_INVALID,
0123 IRDMA_COMPL_STATUS_INVALID_PHYS_BUF_SIZE,
0124 IRDMA_COMPL_STATUS_INVALID_PHYS_BUF_ENTRY,
0125 IRDMA_COMPL_STATUS_INVALID_FBO,
0126 IRDMA_COMPL_STATUS_INVALID_LEN,
0127 IRDMA_COMPL_STATUS_INVALID_ACCESS,
0128 IRDMA_COMPL_STATUS_PHYS_BUF_LIST_TOO_LONG,
0129 IRDMA_COMPL_STATUS_INVALID_VIRT_ADDRESS,
0130 IRDMA_COMPL_STATUS_INVALID_REGION,
0131 IRDMA_COMPL_STATUS_INVALID_WINDOW,
0132 IRDMA_COMPL_STATUS_INVALID_TOTAL_LEN,
0133 IRDMA_COMPL_STATUS_UNKNOWN,
0134 };
0135
0136 enum irdma_cmpl_notify {
0137 IRDMA_CQ_COMPL_EVENT = 0,
0138 IRDMA_CQ_COMPL_SOLICITED = 1,
0139 };
0140
0141 enum irdma_qp_caps {
0142 IRDMA_WRITE_WITH_IMM = 1,
0143 IRDMA_SEND_WITH_IMM = 2,
0144 IRDMA_ROCE = 4,
0145 IRDMA_PUSH_MODE = 8,
0146 };
0147
0148 struct irdma_qp_uk;
0149 struct irdma_cq_uk;
0150 struct irdma_qp_uk_init_info;
0151 struct irdma_cq_uk_init_info;
0152
0153 struct irdma_ring {
0154 u32 head;
0155 u32 tail;
0156 u32 size;
0157 };
0158
0159 struct irdma_cqe {
0160 __le64 buf[IRDMA_CQE_SIZE];
0161 };
0162
0163 struct irdma_extended_cqe {
0164 __le64 buf[IRDMA_EXTENDED_CQE_SIZE];
0165 };
0166
0167 struct irdma_post_send {
0168 struct ib_sge *sg_list;
0169 u32 num_sges;
0170 u32 qkey;
0171 u32 dest_qp;
0172 u32 ah_id;
0173 };
0174
0175 struct irdma_post_inline_send {
0176 void *data;
0177 u32 len;
0178 u32 qkey;
0179 u32 dest_qp;
0180 u32 ah_id;
0181 };
0182
0183 struct irdma_post_rq_info {
0184 u64 wr_id;
0185 struct ib_sge *sg_list;
0186 u32 num_sges;
0187 };
0188
0189 struct irdma_rdma_write {
0190 struct ib_sge *lo_sg_list;
0191 u32 num_lo_sges;
0192 struct ib_sge rem_addr;
0193 };
0194
0195 struct irdma_inline_rdma_write {
0196 void *data;
0197 u32 len;
0198 struct ib_sge rem_addr;
0199 };
0200
0201 struct irdma_rdma_read {
0202 struct ib_sge *lo_sg_list;
0203 u32 num_lo_sges;
0204 struct ib_sge rem_addr;
0205 };
0206
0207 struct irdma_bind_window {
0208 irdma_stag mr_stag;
0209 u64 bind_len;
0210 void *va;
0211 enum irdma_addressing_type addressing_type;
0212 bool ena_reads:1;
0213 bool ena_writes:1;
0214 irdma_stag mw_stag;
0215 bool mem_window_type_1:1;
0216 };
0217
0218 struct irdma_inv_local_stag {
0219 irdma_stag target_stag;
0220 };
0221
0222 struct irdma_post_sq_info {
0223 u64 wr_id;
0224 u8 op_type;
0225 u8 l4len;
0226 bool signaled:1;
0227 bool read_fence:1;
0228 bool local_fence:1;
0229 bool inline_data:1;
0230 bool imm_data_valid:1;
0231 bool push_wqe:1;
0232 bool report_rtt:1;
0233 bool udp_hdr:1;
0234 bool defer_flag:1;
0235 u32 imm_data;
0236 u32 stag_to_inv;
0237 union {
0238 struct irdma_post_send send;
0239 struct irdma_rdma_write rdma_write;
0240 struct irdma_rdma_read rdma_read;
0241 struct irdma_bind_window bind_window;
0242 struct irdma_inv_local_stag inv_local_stag;
0243 struct irdma_inline_rdma_write inline_rdma_write;
0244 struct irdma_post_inline_send inline_send;
0245 } op;
0246 };
0247
0248 struct irdma_cq_poll_info {
0249 u64 wr_id;
0250 irdma_qp_handle qp_handle;
0251 u32 bytes_xfered;
0252 u32 tcp_seq_num_rtt;
0253 u32 qp_id;
0254 u32 ud_src_qpn;
0255 u32 imm_data;
0256 irdma_stag inv_stag;
0257 enum irdma_cmpl_status comp_status;
0258 u16 major_err;
0259 u16 minor_err;
0260 u16 ud_vlan;
0261 u8 ud_smac[6];
0262 u8 op_type;
0263 bool stag_invalid_set:1;
0264 bool push_dropped:1;
0265 bool error:1;
0266 bool solicited_event:1;
0267 bool ipv4:1;
0268 bool ud_vlan_valid:1;
0269 bool ud_smac_valid:1;
0270 bool imm_valid:1;
0271 };
0272
0273 int irdma_uk_inline_rdma_write(struct irdma_qp_uk *qp,
0274 struct irdma_post_sq_info *info, bool post_sq);
0275 int irdma_uk_inline_send(struct irdma_qp_uk *qp,
0276 struct irdma_post_sq_info *info, bool post_sq);
0277 int irdma_uk_post_nop(struct irdma_qp_uk *qp, u64 wr_id, bool signaled,
0278 bool post_sq);
0279 int irdma_uk_post_receive(struct irdma_qp_uk *qp,
0280 struct irdma_post_rq_info *info);
0281 void irdma_uk_qp_post_wr(struct irdma_qp_uk *qp);
0282 int irdma_uk_rdma_read(struct irdma_qp_uk *qp, struct irdma_post_sq_info *info,
0283 bool inv_stag, bool post_sq);
0284 int irdma_uk_rdma_write(struct irdma_qp_uk *qp, struct irdma_post_sq_info *info,
0285 bool post_sq);
0286 int irdma_uk_send(struct irdma_qp_uk *qp, struct irdma_post_sq_info *info,
0287 bool post_sq);
0288 int irdma_uk_stag_local_invalidate(struct irdma_qp_uk *qp,
0289 struct irdma_post_sq_info *info,
0290 bool post_sq);
0291
0292 struct irdma_wqe_uk_ops {
0293 void (*iw_copy_inline_data)(u8 *dest, u8 *src, u32 len, u8 polarity);
0294 u16 (*iw_inline_data_size_to_quanta)(u32 data_size);
0295 void (*iw_set_fragment)(__le64 *wqe, u32 offset, struct ib_sge *sge,
0296 u8 valid);
0297 void (*iw_set_mw_bind_wqe)(__le64 *wqe,
0298 struct irdma_bind_window *op_info);
0299 };
0300
0301 int irdma_uk_cq_poll_cmpl(struct irdma_cq_uk *cq,
0302 struct irdma_cq_poll_info *info);
0303 void irdma_uk_cq_request_notification(struct irdma_cq_uk *cq,
0304 enum irdma_cmpl_notify cq_notify);
0305 void irdma_uk_cq_resize(struct irdma_cq_uk *cq, void *cq_base, int size);
0306 void irdma_uk_cq_set_resized_cnt(struct irdma_cq_uk *qp, u16 cnt);
0307 void irdma_uk_cq_init(struct irdma_cq_uk *cq,
0308 struct irdma_cq_uk_init_info *info);
0309 int irdma_uk_qp_init(struct irdma_qp_uk *qp,
0310 struct irdma_qp_uk_init_info *info);
0311 struct irdma_sq_uk_wr_trk_info {
0312 u64 wrid;
0313 u32 wr_len;
0314 u16 quanta;
0315 u8 reserved[2];
0316 };
0317
0318 struct irdma_qp_quanta {
0319 __le64 elem[IRDMA_WQE_SIZE];
0320 };
0321
0322 struct irdma_qp_uk {
0323 struct irdma_qp_quanta *sq_base;
0324 struct irdma_qp_quanta *rq_base;
0325 struct irdma_uk_attrs *uk_attrs;
0326 u32 __iomem *wqe_alloc_db;
0327 struct irdma_sq_uk_wr_trk_info *sq_wrtrk_array;
0328 u64 *rq_wrid_array;
0329 __le64 *shadow_area;
0330 __le32 *push_db;
0331 __le64 *push_wqe;
0332 struct irdma_ring sq_ring;
0333 struct irdma_ring rq_ring;
0334 struct irdma_ring initial_ring;
0335 u32 qp_id;
0336 u32 qp_caps;
0337 u32 sq_size;
0338 u32 rq_size;
0339 u32 max_sq_frag_cnt;
0340 u32 max_rq_frag_cnt;
0341 u32 max_inline_data;
0342 struct irdma_wqe_uk_ops wqe_ops;
0343 u16 conn_wqes;
0344 u8 qp_type;
0345 u8 swqe_polarity;
0346 u8 swqe_polarity_deferred;
0347 u8 rwqe_polarity;
0348 u8 rq_wqe_size;
0349 u8 rq_wqe_size_multiplier;
0350 bool deferred_flag:1;
0351 bool push_mode:1;
0352 bool push_dropped:1;
0353 bool first_sq_wq:1;
0354 bool sq_flush_complete:1;
0355 bool rq_flush_complete:1;
0356 bool destroy_pending:1;
0357 void *back_qp;
0358 u8 dbg_rq_flushed;
0359 u8 sq_flush_seen;
0360 u8 rq_flush_seen;
0361 };
0362
0363 struct irdma_cq_uk {
0364 struct irdma_cqe *cq_base;
0365 u32 __iomem *cqe_alloc_db;
0366 u32 __iomem *cq_ack_db;
0367 __le64 *shadow_area;
0368 u32 cq_id;
0369 u32 cq_size;
0370 struct irdma_ring cq_ring;
0371 u8 polarity;
0372 bool avoid_mem_cflct:1;
0373 };
0374
0375 struct irdma_qp_uk_init_info {
0376 struct irdma_qp_quanta *sq;
0377 struct irdma_qp_quanta *rq;
0378 struct irdma_uk_attrs *uk_attrs;
0379 u32 __iomem *wqe_alloc_db;
0380 __le64 *shadow_area;
0381 struct irdma_sq_uk_wr_trk_info *sq_wrtrk_array;
0382 u64 *rq_wrid_array;
0383 u32 qp_id;
0384 u32 qp_caps;
0385 u32 sq_size;
0386 u32 rq_size;
0387 u32 max_sq_frag_cnt;
0388 u32 max_rq_frag_cnt;
0389 u32 max_inline_data;
0390 u8 first_sq_wq;
0391 u8 type;
0392 int abi_ver;
0393 bool legacy_mode;
0394 };
0395
0396 struct irdma_cq_uk_init_info {
0397 u32 __iomem *cqe_alloc_db;
0398 u32 __iomem *cq_ack_db;
0399 struct irdma_cqe *cq_base;
0400 __le64 *shadow_area;
0401 u32 cq_size;
0402 u32 cq_id;
0403 bool avoid_mem_cflct;
0404 };
0405
0406 __le64 *irdma_qp_get_next_send_wqe(struct irdma_qp_uk *qp, u32 *wqe_idx,
0407 u16 quanta, u32 total_size,
0408 struct irdma_post_sq_info *info);
0409 __le64 *irdma_qp_get_next_recv_wqe(struct irdma_qp_uk *qp, u32 *wqe_idx);
0410 void irdma_uk_clean_cq(void *q, struct irdma_cq_uk *cq);
0411 int irdma_nop(struct irdma_qp_uk *qp, u64 wr_id, bool signaled, bool post_sq);
0412 int irdma_fragcnt_to_quanta_sq(u32 frag_cnt, u16 *quanta);
0413 int irdma_fragcnt_to_wqesize_rq(u32 frag_cnt, u16 *wqe_size);
0414 void irdma_get_wqe_shift(struct irdma_uk_attrs *uk_attrs, u32 sge,
0415 u32 inline_data, u8 *shift);
0416 int irdma_get_sqdepth(struct irdma_uk_attrs *uk_attrs, u32 sq_size, u8 shift,
0417 u32 *wqdepth);
0418 int irdma_get_rqdepth(struct irdma_uk_attrs *uk_attrs, u32 rq_size, u8 shift,
0419 u32 *wqdepth);
0420 void irdma_qp_push_wqe(struct irdma_qp_uk *qp, __le64 *wqe, u16 quanta,
0421 u32 wqe_idx, bool post_sq);
0422 void irdma_clr_wqes(struct irdma_qp_uk *qp, u32 qp_wqe_idx);
0423 #endif
