0001
0002
0003
0004
0005
0006
0007 #ifndef __ERDMA_VERBS_H__
0008 #define __ERDMA_VERBS_H__
0009
0010 #include <linux/errno.h>
0011
0012 #include <rdma/ib_verbs.h>
0013 #include <rdma/ib_user_verbs.h>
0014 #include <rdma/iw_cm.h>
0015
0016 #include "erdma.h"
0017 #include "erdma_cm.h"
0018 #include "erdma_hw.h"
0019
0020
0021 #define ERDMA_MAX_PD (128 * 1024)
0022 #define ERDMA_MAX_SEND_WR 4096
0023 #define ERDMA_MAX_ORD 128
0024 #define ERDMA_MAX_IRD 128
0025 #define ERDMA_MAX_SGE_RD 1
0026 #define ERDMA_MAX_CONTEXT (128 * 1024)
0027 #define ERDMA_MAX_SEND_SGE 6
0028 #define ERDMA_MAX_RECV_SGE 1
0029 #define ERDMA_MAX_INLINE (sizeof(struct erdma_sge) * (ERDMA_MAX_SEND_SGE))
0030 #define ERDMA_MAX_FRMR_PA 512
0031
0032 enum {
0033 ERDMA_MMAP_IO_NC = 0,
0034 };
0035
0036 struct erdma_user_mmap_entry {
0037 struct rdma_user_mmap_entry rdma_entry;
0038 u64 address;
0039 u8 mmap_flag;
0040 };
0041
0042 struct erdma_ucontext {
0043 struct ib_ucontext ibucontext;
0044
0045 u32 sdb_type;
0046 u32 sdb_idx;
0047 u32 sdb_page_idx;
0048 u32 sdb_page_off;
0049 u64 sdb;
0050 u64 rdb;
0051 u64 cdb;
0052
0053 struct rdma_user_mmap_entry *sq_db_mmap_entry;
0054 struct rdma_user_mmap_entry *rq_db_mmap_entry;
0055 struct rdma_user_mmap_entry *cq_db_mmap_entry;
0056
0057
0058 struct list_head dbrecords_page_list;
0059 struct mutex dbrecords_page_mutex;
0060 };
0061
0062 struct erdma_pd {
0063 struct ib_pd ibpd;
0064 u32 pdn;
0065 };
0066
0067
0068
0069
0070 #define ERDMA_MAX_INLINE_MTT_ENTRIES 4
0071 #define MTT_SIZE(mtt_cnt) (mtt_cnt << 3)
0072 #define ERDMA_MR_MAX_MTT_CNT 524288
0073 #define ERDMA_MTT_ENTRY_SIZE 8
0074
0075 #define ERDMA_MR_TYPE_NORMAL 0
0076 #define ERDMA_MR_TYPE_FRMR 1
0077 #define ERDMA_MR_TYPE_DMA 2
0078
0079 #define ERDMA_MR_INLINE_MTT 0
0080 #define ERDMA_MR_INDIRECT_MTT 1
0081
0082 #define ERDMA_MR_ACC_LR BIT(0)
0083 #define ERDMA_MR_ACC_LW BIT(1)
0084 #define ERDMA_MR_ACC_RR BIT(2)
0085 #define ERDMA_MR_ACC_RW BIT(3)
0086
0087 static inline u8 to_erdma_access_flags(int access)
0088 {
0089 return (access & IB_ACCESS_REMOTE_READ ? ERDMA_MR_ACC_RR : 0) |
0090 (access & IB_ACCESS_LOCAL_WRITE ? ERDMA_MR_ACC_LW : 0) |
0091 (access & IB_ACCESS_REMOTE_WRITE ? ERDMA_MR_ACC_RW : 0);
0092 }
0093
0094 struct erdma_mem {
0095 struct ib_umem *umem;
0096 void *mtt_buf;
0097 u32 mtt_type;
0098 u32 page_size;
0099 u32 page_offset;
0100 u32 page_cnt;
0101 u32 mtt_nents;
0102
0103 u64 va;
0104 u64 len;
0105
0106 u64 mtt_entry[ERDMA_MAX_INLINE_MTT_ENTRIES];
0107 };
0108
0109 struct erdma_mr {
0110 struct ib_mr ibmr;
0111 struct erdma_mem mem;
0112 u8 type;
0113 u8 access;
0114 u8 valid;
0115 };
0116
0117 struct erdma_user_dbrecords_page {
0118 struct list_head list;
0119 struct ib_umem *umem;
0120 u64 va;
0121 int refcnt;
0122 };
0123
0124 struct erdma_uqp {
0125 struct erdma_mem sq_mtt;
0126 struct erdma_mem rq_mtt;
0127
0128 dma_addr_t sq_db_info_dma_addr;
0129 dma_addr_t rq_db_info_dma_addr;
0130
0131 struct erdma_user_dbrecords_page *user_dbr_page;
0132
0133 u32 rq_offset;
0134 };
0135
0136 struct erdma_kqp {
0137 u16 sq_pi;
0138 u16 sq_ci;
0139
0140 u16 rq_pi;
0141 u16 rq_ci;
0142
0143 u64 *swr_tbl;
0144 u64 *rwr_tbl;
0145
0146 void __iomem *hw_sq_db;
0147 void __iomem *hw_rq_db;
0148
0149 void *sq_buf;
0150 dma_addr_t sq_buf_dma_addr;
0151
0152 void *rq_buf;
0153 dma_addr_t rq_buf_dma_addr;
0154
0155 void *sq_db_info;
0156 void *rq_db_info;
0157
0158 u8 sig_all;
0159 };
0160
0161 enum erdma_qp_state {
0162 ERDMA_QP_STATE_IDLE = 0,
0163 ERDMA_QP_STATE_RTR = 1,
0164 ERDMA_QP_STATE_RTS = 2,
0165 ERDMA_QP_STATE_CLOSING = 3,
0166 ERDMA_QP_STATE_TERMINATE = 4,
0167 ERDMA_QP_STATE_ERROR = 5,
0168 ERDMA_QP_STATE_UNDEF = 7,
0169 ERDMA_QP_STATE_COUNT = 8
0170 };
0171
0172 enum erdma_qp_attr_mask {
0173 ERDMA_QP_ATTR_STATE = (1 << 0),
0174 ERDMA_QP_ATTR_LLP_HANDLE = (1 << 2),
0175 ERDMA_QP_ATTR_ORD = (1 << 3),
0176 ERDMA_QP_ATTR_IRD = (1 << 4),
0177 ERDMA_QP_ATTR_SQ_SIZE = (1 << 5),
0178 ERDMA_QP_ATTR_RQ_SIZE = (1 << 6),
0179 ERDMA_QP_ATTR_MPA = (1 << 7)
0180 };
0181
0182 struct erdma_qp_attrs {
0183 enum erdma_qp_state state;
0184 enum erdma_cc_alg cc;
0185 u32 sq_size;
0186 u32 rq_size;
0187 u32 orq_size;
0188 u32 irq_size;
0189 u32 max_send_sge;
0190 u32 max_recv_sge;
0191 u32 cookie;
0192 #define ERDMA_QP_ACTIVE 0
0193 #define ERDMA_QP_PASSIVE 1
0194 u8 qp_type;
0195 u8 pd_len;
0196 };
0197
0198 struct erdma_qp {
0199 struct ib_qp ibqp;
0200 struct kref ref;
0201 struct completion safe_free;
0202 struct erdma_dev *dev;
0203 struct erdma_cep *cep;
0204 struct rw_semaphore state_lock;
0205
0206 union {
0207 struct erdma_kqp kern_qp;
0208 struct erdma_uqp user_qp;
0209 };
0210
0211 struct erdma_cq *scq;
0212 struct erdma_cq *rcq;
0213
0214 struct erdma_qp_attrs attrs;
0215 spinlock_t lock;
0216 };
0217
0218 struct erdma_kcq_info {
0219 void *qbuf;
0220 dma_addr_t qbuf_dma_addr;
0221 u32 ci;
0222 u32 cmdsn;
0223 u32 notify_cnt;
0224
0225 spinlock_t lock;
0226 u8 __iomem *db;
0227 u64 *db_record;
0228 };
0229
0230 struct erdma_ucq_info {
0231 struct erdma_mem qbuf_mtt;
0232 struct erdma_user_dbrecords_page *user_dbr_page;
0233 dma_addr_t db_info_dma_addr;
0234 };
0235
0236 struct erdma_cq {
0237 struct ib_cq ibcq;
0238 u32 cqn;
0239
0240 u32 depth;
0241 u32 assoc_eqn;
0242
0243 union {
0244 struct erdma_kcq_info kern_cq;
0245 struct erdma_ucq_info user_cq;
0246 };
0247 };
0248
0249 #define QP_ID(qp) ((qp)->ibqp.qp_num)
0250
0251 static inline struct erdma_qp *find_qp_by_qpn(struct erdma_dev *dev, int id)
0252 {
0253 return (struct erdma_qp *)xa_load(&dev->qp_xa, id);
0254 }
0255
0256 static inline struct erdma_cq *find_cq_by_cqn(struct erdma_dev *dev, int id)
0257 {
0258 return (struct erdma_cq *)xa_load(&dev->cq_xa, id);
0259 }
0260
0261 void erdma_qp_get(struct erdma_qp *qp);
0262 void erdma_qp_put(struct erdma_qp *qp);
0263 int erdma_modify_qp_internal(struct erdma_qp *qp, struct erdma_qp_attrs *attrs,
0264 enum erdma_qp_attr_mask mask);
0265 void erdma_qp_llp_close(struct erdma_qp *qp);
0266 void erdma_qp_cm_drop(struct erdma_qp *qp);
0267
0268 static inline struct erdma_ucontext *to_ectx(struct ib_ucontext *ibctx)
0269 {
0270 return container_of(ibctx, struct erdma_ucontext, ibucontext);
0271 }
0272
0273 static inline struct erdma_pd *to_epd(struct ib_pd *pd)
0274 {
0275 return container_of(pd, struct erdma_pd, ibpd);
0276 }
0277
0278 static inline struct erdma_mr *to_emr(struct ib_mr *ibmr)
0279 {
0280 return container_of(ibmr, struct erdma_mr, ibmr);
0281 }
0282
0283 static inline struct erdma_qp *to_eqp(struct ib_qp *qp)
0284 {
0285 return container_of(qp, struct erdma_qp, ibqp);
0286 }
0287
0288 static inline struct erdma_cq *to_ecq(struct ib_cq *ibcq)
0289 {
0290 return container_of(ibcq, struct erdma_cq, ibcq);
0291 }
0292
0293 static inline struct erdma_user_mmap_entry *
0294 to_emmap(struct rdma_user_mmap_entry *ibmmap)
0295 {
0296 return container_of(ibmmap, struct erdma_user_mmap_entry, rdma_entry);
0297 }
0298
0299 int erdma_alloc_ucontext(struct ib_ucontext *ibctx, struct ib_udata *data);
0300 void erdma_dealloc_ucontext(struct ib_ucontext *ibctx);
0301 int erdma_query_device(struct ib_device *dev, struct ib_device_attr *attr,
0302 struct ib_udata *data);
0303 int erdma_get_port_immutable(struct ib_device *dev, u32 port,
0304 struct ib_port_immutable *ib_port_immutable);
0305 int erdma_create_cq(struct ib_cq *ibcq, const struct ib_cq_init_attr *attr,
0306 struct ib_udata *data);
0307 int erdma_query_port(struct ib_device *dev, u32 port,
0308 struct ib_port_attr *attr);
0309 int erdma_query_gid(struct ib_device *dev, u32 port, int idx,
0310 union ib_gid *gid);
0311 int erdma_alloc_pd(struct ib_pd *ibpd, struct ib_udata *data);
0312 int erdma_dealloc_pd(struct ib_pd *ibpd, struct ib_udata *udata);
0313 int erdma_create_qp(struct ib_qp *ibqp, struct ib_qp_init_attr *attr,
0314 struct ib_udata *data);
0315 int erdma_query_qp(struct ib_qp *ibqp, struct ib_qp_attr *attr, int mask,
0316 struct ib_qp_init_attr *init_attr);
0317 int erdma_modify_qp(struct ib_qp *ibqp, struct ib_qp_attr *attr, int mask,
0318 struct ib_udata *data);
0319 int erdma_destroy_qp(struct ib_qp *ibqp, struct ib_udata *udata);
0320 int erdma_destroy_cq(struct ib_cq *ibcq, struct ib_udata *udata);
0321 int erdma_req_notify_cq(struct ib_cq *ibcq, enum ib_cq_notify_flags flags);
0322 struct ib_mr *erdma_reg_user_mr(struct ib_pd *ibpd, u64 start, u64 len,
0323 u64 virt, int access, struct ib_udata *udata);
0324 struct ib_mr *erdma_get_dma_mr(struct ib_pd *ibpd, int rights);
0325 int erdma_dereg_mr(struct ib_mr *ibmr, struct ib_udata *data);
0326 int erdma_mmap(struct ib_ucontext *ctx, struct vm_area_struct *vma);
0327 void erdma_mmap_free(struct rdma_user_mmap_entry *rdma_entry);
0328 void erdma_qp_get_ref(struct ib_qp *ibqp);
0329 void erdma_qp_put_ref(struct ib_qp *ibqp);
0330 struct ib_qp *erdma_get_ibqp(struct ib_device *dev, int id);
0331 int erdma_post_send(struct ib_qp *ibqp, const struct ib_send_wr *send_wr,
0332 const struct ib_send_wr **bad_send_wr);
0333 int erdma_post_recv(struct ib_qp *ibqp, const struct ib_recv_wr *recv_wr,
0334 const struct ib_recv_wr **bad_recv_wr);
0335 int erdma_poll_cq(struct ib_cq *ibcq, int num_entries, struct ib_wc *wc);
0336 struct ib_mr *erdma_ib_alloc_mr(struct ib_pd *ibpd, enum ib_mr_type mr_type,
0337 u32 max_num_sg);
0338 int erdma_map_mr_sg(struct ib_mr *ibmr, struct scatterlist *sg, int sg_nents,
0339 unsigned int *sg_offset);
0340 void erdma_port_event(struct erdma_dev *dev, enum ib_event_type reason);
0341
0342 #endif
