Back to home page

OSCL-LXR
 
 

     

0001 /* SPDX-License-Identifier: GPL-2.0 or BSD-3-Clause */
0002 
0003 /* Authors: Bernard Metzler <bmt@zurich.ibm.com> */
0004 /* Copyright (c) 2008-2019, IBM Corporation */
0005 
0006 #ifndef _SIW_H
0007 #define _SIW_H
0008 
0009 #include <rdma/ib_verbs.h>
0010 #include <rdma/restrack.h>
0011 #include <linux/socket.h>
0012 #include <linux/skbuff.h>
0013 #include <crypto/hash.h>
0014 #include <linux/crc32.h>
0015 #include <linux/crc32c.h>
0016 
0017 #include <rdma/siw-abi.h>
0018 #include "iwarp.h"
0019 
0020 #define SIW_VENDOR_ID 0x626d74 /* ascii 'bmt' for now */
0021 #define SIW_VENDORT_PART_ID 0
0022 #define SIW_MAX_QP (1024 * 100)
0023 #define SIW_MAX_QP_WR (1024 * 32)
0024 #define SIW_MAX_ORD_QP 128
0025 #define SIW_MAX_IRD_QP 128
0026 #define SIW_MAX_SGE_PBL 256 /* max num sge's for PBL */
0027 #define SIW_MAX_SGE_RD 1 /* iwarp limitation. we could relax */
0028 #define SIW_MAX_CQ (1024 * 100)
0029 #define SIW_MAX_CQE (SIW_MAX_QP_WR * 100)
0030 #define SIW_MAX_MR (SIW_MAX_QP * 10)
0031 #define SIW_MAX_PD SIW_MAX_QP
0032 #define SIW_MAX_MW 0 /* to be set if MW's are supported */
0033 #define SIW_MAX_SRQ SIW_MAX_QP
0034 #define SIW_MAX_SRQ_WR (SIW_MAX_QP_WR * 10)
0035 #define SIW_MAX_CONTEXT SIW_MAX_PD
0036 
0037 /* Min number of bytes for using zero copy transmit */
0038 #define SENDPAGE_THRESH PAGE_SIZE
0039 
0040 /* Maximum number of frames which can be send in one SQ processing */
0041 #define SQ_USER_MAXBURST 100
0042 
0043 /* Maximum number of consecutive IRQ elements which get served
0044  * if SQ has pending work. Prevents starving local SQ processing
0045  * by serving peer Read Requests.
0046  */
0047 #define SIW_IRQ_MAXBURST_SQ_ACTIVE 4
0048 
0049 struct siw_dev_cap {
0050     int max_qp;
0051     int max_qp_wr;
0052     int max_ord; /* max. outbound read queue depth */
0053     int max_ird; /* max. inbound read queue depth */
0054     int max_sge;
0055     int max_sge_rd;
0056     int max_cq;
0057     int max_cqe;
0058     int max_mr;
0059     int max_pd;
0060     int max_mw;
0061     int max_srq;
0062     int max_srq_wr;
0063     int max_srq_sge;
0064 };
0065 
0066 struct siw_pd {
0067     struct ib_pd base_pd;
0068 };
0069 
0070 struct siw_device {
0071     struct ib_device base_dev;
0072     struct net_device *netdev;
0073     struct siw_dev_cap attrs;
0074 
0075     u32 vendor_part_id;
0076     int numa_node;
0077 
0078     /* physical port state (only one port per device) */
0079     enum ib_port_state state;
0080 
0081     spinlock_t lock;
0082 
0083     struct xarray qp_xa;
0084     struct xarray mem_xa;
0085 
0086     struct list_head cep_list;
0087     struct list_head qp_list;
0088 
0089     /* active objects statistics to enforce limits */
0090     atomic_t num_qp;
0091     atomic_t num_cq;
0092     atomic_t num_pd;
0093     atomic_t num_mr;
0094     atomic_t num_srq;
0095     atomic_t num_ctx;
0096 
0097     struct work_struct netdev_down;
0098 };
0099 
0100 struct siw_ucontext {
0101     struct ib_ucontext base_ucontext;
0102     struct siw_device *sdev;
0103 };
0104 
0105 /*
0106  * The RDMA core does not define LOCAL_READ access, which is always
0107  * enabled implictely.
0108  */
0109 #define IWARP_ACCESS_MASK                   \
0110     (IB_ACCESS_LOCAL_WRITE | IB_ACCESS_REMOTE_WRITE |   \
0111      IB_ACCESS_REMOTE_READ)
0112 
0113 /*
0114  * siw presentation of user memory registered as source
0115  * or target of RDMA operations.
0116  */
0117 
0118 struct siw_page_chunk {
0119     struct page **plist;
0120 };
0121 
0122 struct siw_umem {
0123     struct siw_page_chunk *page_chunk;
0124     int num_pages;
0125     bool writable;
0126     u64 fp_addr; /* First page base address */
0127     struct mm_struct *owning_mm;
0128 };
0129 
0130 struct siw_pble {
0131     dma_addr_t addr; /* Address of assigned buffer */
0132     unsigned int size; /* Size of this entry */
0133     unsigned long pbl_off; /* Total offset from start of PBL */
0134 };
0135 
0136 struct siw_pbl {
0137     unsigned int num_buf;
0138     unsigned int max_buf;
0139     struct siw_pble pbe[];
0140 };
0141 
0142 /*
0143  * Generic memory representation for registered siw memory.
0144  * Memory lookup always via higher 24 bit of STag (STag index).
0145  */
0146 struct siw_mem {
0147     struct siw_device *sdev;
0148     struct kref ref;
0149     u64 va; /* VA of memory */
0150     u64 len; /* length of the memory buffer in bytes */
0151     u32 stag; /* iWarp memory access steering tag */
0152     u8 stag_valid; /* VALID or INVALID */
0153     u8 is_pbl; /* PBL or user space mem */
0154     u8 is_mw; /* Memory Region or Memory Window */
0155     enum ib_access_flags perms; /* local/remote READ & WRITE */
0156     union {
0157         struct siw_umem *umem;
0158         struct siw_pbl *pbl;
0159         void *mem_obj;
0160     };
0161     struct ib_pd *pd;
0162 };
0163 
0164 struct siw_mr {
0165     struct ib_mr base_mr;
0166     struct siw_mem *mem;
0167     struct rcu_head rcu;
0168 };
0169 
0170 /*
0171  * Error codes for local or remote
0172  * access to registered memory
0173  */
0174 enum siw_access_state {
0175     E_ACCESS_OK,
0176     E_STAG_INVALID,
0177     E_BASE_BOUNDS,
0178     E_ACCESS_PERM,
0179     E_PD_MISMATCH
0180 };
0181 
0182 enum siw_wr_state {
0183     SIW_WR_IDLE,
0184     SIW_WR_QUEUED, /* processing has not started yet */
0185     SIW_WR_INPROGRESS /* initiated processing of the WR */
0186 };
0187 
0188 /* The WQE currently being processed (RX or TX) */
0189 struct siw_wqe {
0190     /* Copy of applications SQE or RQE */
0191     union {
0192         struct siw_sqe sqe;
0193         struct siw_rqe rqe;
0194     };
0195     struct siw_mem *mem[SIW_MAX_SGE]; /* per sge's resolved mem */
0196     enum siw_wr_state wr_status;
0197     enum siw_wc_status wc_status;
0198     u32 bytes; /* total bytes to process */
0199     u32 processed; /* bytes processed */
0200 };
0201 
0202 struct siw_cq {
0203     struct ib_cq base_cq;
0204     spinlock_t lock;
0205     struct siw_cq_ctrl *notify;
0206     struct siw_cqe *queue;
0207     u32 cq_put;
0208     u32 cq_get;
0209     u32 num_cqe;
0210     struct rdma_user_mmap_entry *cq_entry; /* mmap info for CQE array */
0211     u32 id; /* For debugging only */
0212 };
0213 
0214 enum siw_qp_state {
0215     SIW_QP_STATE_IDLE,
0216     SIW_QP_STATE_RTR,
0217     SIW_QP_STATE_RTS,
0218     SIW_QP_STATE_CLOSING,
0219     SIW_QP_STATE_TERMINATE,
0220     SIW_QP_STATE_ERROR,
0221     SIW_QP_STATE_COUNT
0222 };
0223 
0224 enum siw_qp_flags {
0225     SIW_RDMA_BIND_ENABLED = (1 << 0),
0226     SIW_RDMA_WRITE_ENABLED = (1 << 1),
0227     SIW_RDMA_READ_ENABLED = (1 << 2),
0228     SIW_SIGNAL_ALL_WR = (1 << 3),
0229     SIW_MPA_CRC = (1 << 4),
0230     SIW_QP_IN_DESTROY = (1 << 5)
0231 };
0232 
0233 enum siw_qp_attr_mask {
0234     SIW_QP_ATTR_STATE = (1 << 0),
0235     SIW_QP_ATTR_ACCESS_FLAGS = (1 << 1),
0236     SIW_QP_ATTR_LLP_HANDLE = (1 << 2),
0237     SIW_QP_ATTR_ORD = (1 << 3),
0238     SIW_QP_ATTR_IRD = (1 << 4),
0239     SIW_QP_ATTR_SQ_SIZE = (1 << 5),
0240     SIW_QP_ATTR_RQ_SIZE = (1 << 6),
0241     SIW_QP_ATTR_MPA = (1 << 7)
0242 };
0243 
0244 struct siw_srq {
0245     struct ib_srq base_srq;
0246     spinlock_t lock;
0247     u32 max_sge;
0248     u32 limit; /* low watermark for async event */
0249     struct siw_rqe *recvq;
0250     u32 rq_put;
0251     u32 rq_get;
0252     u32 num_rqe; /* max # of wqe's allowed */
0253     struct rdma_user_mmap_entry *srq_entry; /* mmap info for SRQ array */
0254     bool armed:1; /* inform user if limit hit */
0255     bool is_kernel_res:1; /* true if kernel client */
0256 };
0257 
0258 struct siw_qp_attrs {
0259     enum siw_qp_state state;
0260     u32 sq_size;
0261     u32 rq_size;
0262     u32 orq_size;
0263     u32 irq_size;
0264     u32 sq_max_sges;
0265     u32 rq_max_sges;
0266     enum siw_qp_flags flags;
0267 
0268     struct socket *sk;
0269 };
0270 
0271 enum siw_tx_ctx {
0272     SIW_SEND_HDR, /* start or continue sending HDR */
0273     SIW_SEND_DATA, /* start or continue sending DDP payload */
0274     SIW_SEND_TRAILER, /* start or continue sending TRAILER */
0275     SIW_SEND_SHORT_FPDU/* send whole FPDU hdr|data|trailer at once */
0276 };
0277 
0278 enum siw_rx_state {
0279     SIW_GET_HDR, /* await new hdr or within hdr */
0280     SIW_GET_DATA_START, /* start of inbound DDP payload */
0281     SIW_GET_DATA_MORE, /* continuation of (misaligned) DDP payload */
0282     SIW_GET_TRAILER/* await new trailer or within trailer */
0283 };
0284 
0285 struct siw_rx_stream {
0286     struct sk_buff *skb;
0287     int skb_new; /* pending unread bytes in skb */
0288     int skb_offset; /* offset in skb */
0289     int skb_copied; /* processed bytes in skb */
0290 
0291     union iwarp_hdr hdr;
0292     struct mpa_trailer trailer;
0293 
0294     enum siw_rx_state state;
0295 
0296     /*
0297      * For each FPDU, main RX loop runs through 3 stages:
0298      * Receiving protocol headers, placing DDP payload and receiving
0299      * trailer information (CRC + possibly padding).
0300      * Next two variables keep state on receive status of the
0301      * current FPDU part (hdr, data, trailer).
0302      */
0303     int fpdu_part_rcvd; /* bytes in pkt part copied */
0304     int fpdu_part_rem; /* bytes in pkt part not seen */
0305 
0306     /*
0307      * Next expected DDP MSN for each QN +
0308      * expected steering tag +
0309      * expected DDP tagget offset (all HBO)
0310      */
0311     u32 ddp_msn[RDMAP_UNTAGGED_QN_COUNT];
0312     u32 ddp_stag;
0313     u64 ddp_to;
0314     u32 inval_stag; /* Stag to be invalidated */
0315 
0316     struct shash_desc *mpa_crc_hd;
0317     u8 rx_suspend : 1;
0318     u8 pad : 2; /* # of pad bytes expected */
0319     u8 rdmap_op : 4; /* opcode of current frame */
0320 };
0321 
0322 struct siw_rx_fpdu {
0323     /*
0324      * Local destination memory of inbound RDMA operation.
0325      * Valid, according to wqe->wr_status
0326      */
0327     struct siw_wqe wqe_active;
0328 
0329     unsigned int pbl_idx; /* Index into current PBL */
0330     unsigned int sge_idx; /* current sge in rx */
0331     unsigned int sge_off; /* already rcvd in curr. sge */
0332 
0333     char first_ddp_seg; /* this is the first DDP seg */
0334     char more_ddp_segs; /* more DDP segs expected */
0335     u8 prev_rdmap_op : 4; /* opcode of prev frame */
0336 };
0337 
0338 /*
0339  * Shorthands for short packets w/o payload
0340  * to be transmitted more efficient.
0341  */
0342 struct siw_send_pkt {
0343     struct iwarp_send send;
0344     __be32 crc;
0345 };
0346 
0347 struct siw_write_pkt {
0348     struct iwarp_rdma_write write;
0349     __be32 crc;
0350 };
0351 
0352 struct siw_rreq_pkt {
0353     struct iwarp_rdma_rreq rreq;
0354     __be32 crc;
0355 };
0356 
0357 struct siw_rresp_pkt {
0358     struct iwarp_rdma_rresp rresp;
0359     __be32 crc;
0360 };
0361 
0362 struct siw_iwarp_tx {
0363     union {
0364         union iwarp_hdr hdr;
0365 
0366         /* Generic part of FPDU header */
0367         struct iwarp_ctrl ctrl;
0368         struct iwarp_ctrl_untagged c_untagged;
0369         struct iwarp_ctrl_tagged c_tagged;
0370 
0371         /* FPDU headers */
0372         struct iwarp_rdma_write rwrite;
0373         struct iwarp_rdma_rreq rreq;
0374         struct iwarp_rdma_rresp rresp;
0375         struct iwarp_terminate terminate;
0376         struct iwarp_send send;
0377         struct iwarp_send_inv send_inv;
0378 
0379         /* complete short FPDUs */
0380         struct siw_send_pkt send_pkt;
0381         struct siw_write_pkt write_pkt;
0382         struct siw_rreq_pkt rreq_pkt;
0383         struct siw_rresp_pkt rresp_pkt;
0384     } pkt;
0385 
0386     struct mpa_trailer trailer;
0387     /* DDP MSN for untagged messages */
0388     u32 ddp_msn[RDMAP_UNTAGGED_QN_COUNT];
0389 
0390     enum siw_tx_ctx state;
0391     u16 ctrl_len; /* ddp+rdmap hdr */
0392     u16 ctrl_sent;
0393     int burst;
0394     int bytes_unsent; /* ddp payload bytes */
0395 
0396     struct shash_desc *mpa_crc_hd;
0397 
0398     u8 do_crc : 1; /* do crc for segment */
0399     u8 use_sendpage : 1; /* send w/o copy */
0400     u8 tx_suspend : 1; /* stop sending DDP segs. */
0401     u8 pad : 2; /* # pad in current fpdu */
0402     u8 orq_fence : 1; /* ORQ full or Send fenced */
0403     u8 in_syscall : 1; /* TX out of user context */
0404     u8 zcopy_tx : 1; /* Use TCP_SENDPAGE if possible */
0405     u8 gso_seg_limit; /* Maximum segments for GSO, 0 = unbound */
0406 
0407     u16 fpdu_len; /* len of FPDU to tx */
0408     unsigned int tcp_seglen; /* remaining tcp seg space */
0409 
0410     struct siw_wqe wqe_active;
0411 
0412     int pbl_idx; /* Index into current PBL */
0413     int sge_idx; /* current sge in tx */
0414     u32 sge_off; /* already sent in curr. sge */
0415 };
0416 
0417 struct siw_qp {
0418     struct ib_qp base_qp;
0419     struct siw_device *sdev;
0420     struct kref ref;
0421     struct list_head devq;
0422     int tx_cpu;
0423     struct siw_qp_attrs attrs;
0424 
0425     struct siw_cep *cep;
0426     struct rw_semaphore state_lock;
0427 
0428     struct ib_pd *pd;
0429     struct siw_cq *scq;
0430     struct siw_cq *rcq;
0431     struct siw_srq *srq;
0432 
0433     struct siw_iwarp_tx tx_ctx; /* Transmit context */
0434     spinlock_t sq_lock;
0435     struct siw_sqe *sendq; /* send queue element array */
0436     uint32_t sq_get; /* consumer index into sq array */
0437     uint32_t sq_put; /* kernel prod. index into sq array */
0438     struct llist_node tx_list;
0439 
0440     struct siw_sqe *orq; /* outbound read queue element array */
0441     spinlock_t orq_lock;
0442     uint32_t orq_get; /* consumer index into orq array */
0443     uint32_t orq_put; /* shared producer index for ORQ */
0444 
0445     struct siw_rx_stream rx_stream;
0446     struct siw_rx_fpdu *rx_fpdu;
0447     struct siw_rx_fpdu rx_tagged;
0448     struct siw_rx_fpdu rx_untagged;
0449     spinlock_t rq_lock;
0450     struct siw_rqe *recvq; /* recv queue element array */
0451     uint32_t rq_get; /* consumer index into rq array */
0452     uint32_t rq_put; /* kernel prod. index into rq array */
0453 
0454     struct siw_sqe *irq; /* inbound read queue element array */
0455     uint32_t irq_get; /* consumer index into irq array */
0456     uint32_t irq_put; /* producer index into irq array */
0457     int irq_burst;
0458 
0459     struct { /* information to be carried in TERMINATE pkt, if valid */
0460         u8 valid;
0461         u8 in_tx;
0462         u8 layer : 4, etype : 4;
0463         u8 ecode;
0464     } term_info;
0465     struct rdma_user_mmap_entry *sq_entry; /* mmap info for SQE array */
0466     struct rdma_user_mmap_entry *rq_entry; /* mmap info for RQE array */
0467     struct rcu_head rcu;
0468 };
0469 
0470 /* helper macros */
0471 #define rx_qp(rx) container_of(rx, struct siw_qp, rx_stream)
0472 #define tx_qp(tx) container_of(tx, struct siw_qp, tx_ctx)
0473 #define tx_wqe(qp) (&(qp)->tx_ctx.wqe_active)
0474 #define rx_wqe(rctx) (&(rctx)->wqe_active)
0475 #define rx_mem(rctx) ((rctx)->wqe_active.mem[0])
0476 #define tx_type(wqe) ((wqe)->sqe.opcode)
0477 #define rx_type(wqe) ((wqe)->rqe.opcode)
0478 #define tx_flags(wqe) ((wqe)->sqe.flags)
0479 
0480 struct iwarp_msg_info {
0481     int hdr_len;
0482     struct iwarp_ctrl ctrl;
0483     int (*rx_data)(struct siw_qp *qp);
0484 };
0485 
0486 struct siw_user_mmap_entry {
0487     struct rdma_user_mmap_entry rdma_entry;
0488     void *address;
0489 };
0490 
0491 /* Global siw parameters. Currently set in siw_main.c */
0492 extern const bool zcopy_tx;
0493 extern const bool try_gso;
0494 extern const bool loopback_enabled;
0495 extern const bool mpa_crc_required;
0496 extern const bool mpa_crc_strict;
0497 extern const bool siw_tcp_nagle;
0498 extern u_char mpa_version;
0499 extern const bool peer_to_peer;
0500 extern struct task_struct *siw_tx_thread[];
0501 
0502 extern struct crypto_shash *siw_crypto_shash;
0503 extern struct iwarp_msg_info iwarp_pktinfo[RDMAP_TERMINATE + 1];
0504 
0505 /* QP general functions */
0506 int siw_qp_modify(struct siw_qp *qp, struct siw_qp_attrs *attr,
0507           enum siw_qp_attr_mask mask);
0508 int siw_qp_mpa_rts(struct siw_qp *qp, enum mpa_v2_ctrl ctrl);
0509 void siw_qp_llp_close(struct siw_qp *qp);
0510 void siw_qp_cm_drop(struct siw_qp *qp, int schedule);
0511 void siw_send_terminate(struct siw_qp *qp);
0512 
0513 void siw_qp_get_ref(struct ib_qp *qp);
0514 void siw_qp_put_ref(struct ib_qp *qp);
0515 int siw_qp_add(struct siw_device *sdev, struct siw_qp *qp);
0516 void siw_free_qp(struct kref *ref);
0517 
0518 void siw_init_terminate(struct siw_qp *qp, enum term_elayer layer,
0519             u8 etype, u8 ecode, int in_tx);
0520 enum ddp_ecode siw_tagged_error(enum siw_access_state state);
0521 enum rdmap_ecode siw_rdmap_error(enum siw_access_state state);
0522 
0523 void siw_read_to_orq(struct siw_sqe *rreq, struct siw_sqe *sqe);
0524 int siw_sqe_complete(struct siw_qp *qp, struct siw_sqe *sqe, u32 bytes,
0525              enum siw_wc_status status);
0526 int siw_rqe_complete(struct siw_qp *qp, struct siw_rqe *rqe, u32 bytes,
0527              u32 inval_stag, enum siw_wc_status status);
0528 void siw_qp_llp_data_ready(struct sock *sk);
0529 void siw_qp_llp_write_space(struct sock *sk);
0530 
0531 /* QP TX path functions */
0532 int siw_run_sq(void *arg);
0533 int siw_qp_sq_process(struct siw_qp *qp);
0534 int siw_sq_start(struct siw_qp *qp);
0535 int siw_activate_tx(struct siw_qp *qp);
0536 void siw_stop_tx_thread(int nr_cpu);
0537 int siw_get_tx_cpu(struct siw_device *sdev);
0538 void siw_put_tx_cpu(int cpu);
0539 
0540 /* QP RX path functions */
0541 int siw_proc_send(struct siw_qp *qp);
0542 int siw_proc_rreq(struct siw_qp *qp);
0543 int siw_proc_rresp(struct siw_qp *qp);
0544 int siw_proc_write(struct siw_qp *qp);
0545 int siw_proc_terminate(struct siw_qp *qp);
0546 
0547 int siw_tcp_rx_data(read_descriptor_t *rd_desc, struct sk_buff *skb,
0548             unsigned int off, size_t len);
0549 
0550 static inline void set_rx_fpdu_context(struct siw_qp *qp, u8 opcode)
0551 {
0552     if (opcode == RDMAP_RDMA_WRITE || opcode == RDMAP_RDMA_READ_RESP)
0553         qp->rx_fpdu = &qp->rx_tagged;
0554     else
0555         qp->rx_fpdu = &qp->rx_untagged;
0556 
0557     qp->rx_stream.rdmap_op = opcode;
0558 }
0559 
0560 static inline struct siw_ucontext *to_siw_ctx(struct ib_ucontext *base_ctx)
0561 {
0562     return container_of(base_ctx, struct siw_ucontext, base_ucontext);
0563 }
0564 
0565 static inline struct siw_qp *to_siw_qp(struct ib_qp *base_qp)
0566 {
0567     return container_of(base_qp, struct siw_qp, base_qp);
0568 }
0569 
0570 static inline struct siw_cq *to_siw_cq(struct ib_cq *base_cq)
0571 {
0572     return container_of(base_cq, struct siw_cq, base_cq);
0573 }
0574 
0575 static inline struct siw_srq *to_siw_srq(struct ib_srq *base_srq)
0576 {
0577     return container_of(base_srq, struct siw_srq, base_srq);
0578 }
0579 
0580 static inline struct siw_device *to_siw_dev(struct ib_device *base_dev)
0581 {
0582     return container_of(base_dev, struct siw_device, base_dev);
0583 }
0584 
0585 static inline struct siw_mr *to_siw_mr(struct ib_mr *base_mr)
0586 {
0587     return container_of(base_mr, struct siw_mr, base_mr);
0588 }
0589 
0590 static inline struct siw_user_mmap_entry *
0591 to_siw_mmap_entry(struct rdma_user_mmap_entry *rdma_mmap)
0592 {
0593     return container_of(rdma_mmap, struct siw_user_mmap_entry, rdma_entry);
0594 }
0595 
0596 static inline struct siw_qp *siw_qp_id2obj(struct siw_device *sdev, int id)
0597 {
0598     struct siw_qp *qp;
0599 
0600     rcu_read_lock();
0601     qp = xa_load(&sdev->qp_xa, id);
0602     if (likely(qp && kref_get_unless_zero(&qp->ref))) {
0603         rcu_read_unlock();
0604         return qp;
0605     }
0606     rcu_read_unlock();
0607     return NULL;
0608 }
0609 
0610 static inline u32 qp_id(struct siw_qp *qp)
0611 {
0612     return qp->base_qp.qp_num;
0613 }
0614 
0615 static inline void siw_qp_get(struct siw_qp *qp)
0616 {
0617     kref_get(&qp->ref);
0618 }
0619 
0620 static inline void siw_qp_put(struct siw_qp *qp)
0621 {
0622     kref_put(&qp->ref, siw_free_qp);
0623 }
0624 
0625 static inline int siw_sq_empty(struct siw_qp *qp)
0626 {
0627     struct siw_sqe *sqe = &qp->sendq[qp->sq_get % qp->attrs.sq_size];
0628 
0629     return READ_ONCE(sqe->flags) == 0;
0630 }
0631 
0632 static inline struct siw_sqe *sq_get_next(struct siw_qp *qp)
0633 {
0634     struct siw_sqe *sqe = &qp->sendq[qp->sq_get % qp->attrs.sq_size];
0635 
0636     if (READ_ONCE(sqe->flags) & SIW_WQE_VALID)
0637         return sqe;
0638 
0639     return NULL;
0640 }
0641 
0642 static inline struct siw_sqe *orq_get_current(struct siw_qp *qp)
0643 {
0644     return &qp->orq[qp->orq_get % qp->attrs.orq_size];
0645 }
0646 
0647 static inline struct siw_sqe *orq_get_free(struct siw_qp *qp)
0648 {
0649     struct siw_sqe *orq_e = &qp->orq[qp->orq_put % qp->attrs.orq_size];
0650 
0651     if (READ_ONCE(orq_e->flags) == 0)
0652         return orq_e;
0653 
0654     return NULL;
0655 }
0656 
0657 static inline int siw_orq_empty(struct siw_qp *qp)
0658 {
0659     return qp->orq[qp->orq_get % qp->attrs.orq_size].flags == 0 ? 1 : 0;
0660 }
0661 
0662 static inline struct siw_sqe *irq_alloc_free(struct siw_qp *qp)
0663 {
0664     struct siw_sqe *irq_e = &qp->irq[qp->irq_put % qp->attrs.irq_size];
0665 
0666     if (READ_ONCE(irq_e->flags) == 0) {
0667         qp->irq_put++;
0668         return irq_e;
0669     }
0670     return NULL;
0671 }
0672 
0673 static inline __wsum siw_csum_update(const void *buff, int len, __wsum sum)
0674 {
0675     return (__force __wsum)crc32c((__force __u32)sum, buff, len);
0676 }
0677 
0678 static inline __wsum siw_csum_combine(__wsum csum, __wsum csum2, int offset,
0679                       int len)
0680 {
0681     return (__force __wsum)__crc32c_le_combine((__force __u32)csum,
0682                            (__force __u32)csum2, len);
0683 }
0684 
0685 static inline void siw_crc_skb(struct siw_rx_stream *srx, unsigned int len)
0686 {
0687     const struct skb_checksum_ops siw_cs_ops = {
0688         .update = siw_csum_update,
0689         .combine = siw_csum_combine,
0690     };
0691     __wsum crc = *(u32 *)shash_desc_ctx(srx->mpa_crc_hd);
0692 
0693     crc = __skb_checksum(srx->skb, srx->skb_offset, len, crc,
0694                  &siw_cs_ops);
0695     *(u32 *)shash_desc_ctx(srx->mpa_crc_hd) = crc;
0696 }
0697 
0698 #define siw_dbg(ibdev, fmt, ...)                                               \
0699     ibdev_dbg(ibdev, "%s: " fmt, __func__, ##__VA_ARGS__)
0700 
0701 #define siw_dbg_qp(qp, fmt, ...)                                               \
0702     ibdev_dbg(&qp->sdev->base_dev, "QP[%u] %s: " fmt, qp_id(qp), __func__, \
0703           ##__VA_ARGS__)
0704 
0705 #define siw_dbg_cq(cq, fmt, ...)                                               \
0706     ibdev_dbg(cq->base_cq.device, "CQ[%u] %s: " fmt, cq->id, __func__,     \
0707           ##__VA_ARGS__)
0708 
0709 #define siw_dbg_pd(pd, fmt, ...)                                               \
0710     ibdev_dbg(pd->device, "PD[%u] %s: " fmt, pd->res.id, __func__,         \
0711           ##__VA_ARGS__)
0712 
0713 #define siw_dbg_mem(mem, fmt, ...)                                             \
0714     ibdev_dbg(&mem->sdev->base_dev,                                        \
0715           "MEM[0x%08x] %s: " fmt, mem->stag, __func__, ##__VA_ARGS__)
0716 
0717 #define siw_dbg_cep(cep, fmt, ...)                                             \
0718     ibdev_dbg(&cep->sdev->base_dev, "CEP[0x%pK] %s: " fmt,                 \
0719           cep, __func__, ##__VA_ARGS__)
0720 
0721 void siw_cq_flush(struct siw_cq *cq);
0722 void siw_sq_flush(struct siw_qp *qp);
0723 void siw_rq_flush(struct siw_qp *qp);
0724 int siw_reap_cqe(struct siw_cq *cq, struct ib_wc *wc);
0725 
0726 #endif
[ Source navigation ] [ Diff markup ] [ Identifier search ] [ general search ]

This page was automatically generated by the 2.1.0 LXR engine.

The LXR team
Valid CSS 2.1! Valid HTML 4.01!