linux/sunrpc/svc_rdma.h

0001 /* SPDX-License-Identifier: GPL-2.0 OR BSD-3-Clause */
0002 /*
0003  * Copyright (c) 2005-2006 Network Appliance, Inc. All rights reserved.
0004  *
0005  * This software is available to you under a choice of one of two
0006  * licenses.  You may choose to be licensed under the terms of the GNU
0007  * General Public License (GPL) Version 2, available from the file
0008  * COPYING in the main directory of this source tree, or the BSD-type
0009  * license below:
0010  *
0011  * Redistribution and use in source and binary forms, with or without
0012  * modification, are permitted provided that the following conditions
0013  * are met:
0014  *
0015  *      Redistributions of source code must retain the above copyright
0016  *      notice, this list of conditions and the following disclaimer.
0017  *
0018  *      Redistributions in binary form must reproduce the above
0019  *      copyright notice, this list of conditions and the following
0020  *      disclaimer in the documentation and/or other materials provided
0021  *      with the distribution.
0022  *
0023  *      Neither the name of the Network Appliance, Inc. nor the names of
0024  *      its contributors may be used to endorse or promote products
0025  *      derived from this software without specific prior written
0026  *      permission.
0027  *
0028  * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
0029  * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
0030  * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
0031  * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
0032  * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
0033  * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
0034  * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
0035  * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
0036  * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
0037  * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
0038  * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
0039  *
0040  * Author: Tom Tucker <tom@opengridcomputing.com>
0041  */
0042
0043 #ifndef SVC_RDMA_H
0044 #define SVC_RDMA_H
0045 #include <linux/llist.h>
0046 #include <linux/sunrpc/xdr.h>
0047 #include <linux/sunrpc/svcsock.h>
0048 #include <linux/sunrpc/rpc_rdma.h>
0049 #include <linux/sunrpc/rpc_rdma_cid.h>
0050 #include <linux/sunrpc/svc_rdma_pcl.h>
0051
0052 #include <linux/percpu_counter.h>
0053 #include <rdma/ib_verbs.h>
0054 #include <rdma/rdma_cm.h>
0055
0056 /* Default and maximum inline threshold sizes */
0057 enum {
0058     RPCRDMA_PULLUP_THRESH = RPCRDMA_V1_DEF_INLINE_SIZE >> 1,
0059     RPCRDMA_DEF_INLINE_THRESH = 4096,
0060     RPCRDMA_MAX_INLINE_THRESH = 65536
0061 };
0062
0063 /* RPC/RDMA parameters and stats */
0064 extern unsigned int svcrdma_ord;
0065 extern unsigned int svcrdma_max_requests;
0066 extern unsigned int svcrdma_max_bc_requests;
0067 extern unsigned int svcrdma_max_req_size;
0068
0069 extern struct percpu_counter svcrdma_stat_read;
0070 extern struct percpu_counter svcrdma_stat_recv;
0071 extern struct percpu_counter svcrdma_stat_sq_starve;
0072 extern struct percpu_counter svcrdma_stat_write;
0073
0074 struct svcxprt_rdma {
0075     struct svc_xprt      sc_xprt;       /* SVC transport structure */
0076     struct rdma_cm_id    *sc_cm_id;     /* RDMA connection id */
0077     struct list_head     sc_accept_q;   /* Conn. waiting accept */
0078     int          sc_ord;        /* RDMA read limit */
0079     int                  sc_max_send_sges;
0080     bool             sc_snd_w_inv;  /* OK to use Send With Invalidate */
0081
0082     atomic_t             sc_sq_avail;   /* SQEs ready to be consumed */
0083     unsigned int         sc_sq_depth;   /* Depth of SQ */
0084     __be32           sc_fc_credits; /* Forward credits */
0085     u32          sc_max_requests;   /* Max requests */
0086     u32          sc_max_bc_requests;/* Backward credits */
0087     int                  sc_max_req_size;   /* Size of each RQ WR buf */
0088     u8           sc_port_num;
0089
0090     struct ib_pd         *sc_pd;
0091
0092     spinlock_t       sc_send_lock;
0093     struct llist_head    sc_send_ctxts;
0094     spinlock_t       sc_rw_ctxt_lock;
0095     struct llist_head    sc_rw_ctxts;
0096
0097     u32          sc_pending_recvs;
0098     u32          sc_recv_batch;
0099     struct list_head     sc_rq_dto_q;
0100     spinlock_t       sc_rq_dto_lock;
0101     struct ib_qp         *sc_qp;
0102     struct ib_cq         *sc_rq_cq;
0103     struct ib_cq         *sc_sq_cq;
0104
0105     spinlock_t       sc_lock;       /* transport lock */
0106
0107     wait_queue_head_t    sc_send_wait;  /* SQ exhaustion waitlist */
0108     unsigned long        sc_flags;
0109     struct work_struct   sc_work;
0110
0111     struct llist_head    sc_recv_ctxts;
0112
0113     atomic_t         sc_completion_ids;
0114 };
0115 /* sc_flags */
0116 #define RDMAXPRT_CONN_PENDING   3
0117
0118 /*
0119  * Default connection parameters
0120  */
0121 enum {
0122     RPCRDMA_LISTEN_BACKLOG  = 10,
0123     RPCRDMA_MAX_REQUESTS    = 64,
0124     RPCRDMA_MAX_BC_REQUESTS = 2,
0125 };
0126
0127 #define RPCSVC_MAXPAYLOAD_RDMA  RPCSVC_MAXPAYLOAD
0128
0129 struct svc_rdma_recv_ctxt {
0130     struct llist_node   rc_node;
0131     struct list_head    rc_list;
0132     struct ib_recv_wr   rc_recv_wr;
0133     struct ib_cqe       rc_cqe;
0134     struct rpc_rdma_cid rc_cid;
0135     struct ib_sge       rc_recv_sge;
0136     void            *rc_recv_buf;
0137     struct xdr_stream   rc_stream;
0138     bool            rc_temp;
0139     u32         rc_byte_len;
0140     unsigned int        rc_page_count;
0141     u32         rc_inv_rkey;
0142     __be32          rc_msgtype;
0143
0144     struct svc_rdma_pcl rc_call_pcl;
0145
0146     struct svc_rdma_pcl rc_read_pcl;
0147     struct svc_rdma_chunk   *rc_cur_result_payload;
0148     struct svc_rdma_pcl rc_write_pcl;
0149     struct svc_rdma_pcl rc_reply_pcl;
0150 };
0151
0152 struct svc_rdma_send_ctxt {
0153     struct llist_node   sc_node;
0154     struct rpc_rdma_cid sc_cid;
0155
0156     struct ib_send_wr   sc_send_wr;
0157     struct ib_cqe       sc_cqe;
0158     struct completion   sc_done;
0159     struct xdr_buf      sc_hdrbuf;
0160     struct xdr_stream   sc_stream;
0161     void            *sc_xprt_buf;
0162     int         sc_cur_sge_no;
0163
0164     struct ib_sge       sc_sges[];
0165 };
0166
0167 /* svc_rdma_backchannel.c */
0168 extern void svc_rdma_handle_bc_reply(struct svc_rqst *rqstp,
0169                      struct svc_rdma_recv_ctxt *rctxt);
0170
0171 /* svc_rdma_recvfrom.c */
0172 extern void svc_rdma_recv_ctxts_destroy(struct svcxprt_rdma *rdma);
0173 extern bool svc_rdma_post_recvs(struct svcxprt_rdma *rdma);
0174 extern struct svc_rdma_recv_ctxt *
0175         svc_rdma_recv_ctxt_get(struct svcxprt_rdma *rdma);
0176 extern void svc_rdma_recv_ctxt_put(struct svcxprt_rdma *rdma,
0177                    struct svc_rdma_recv_ctxt *ctxt);
0178 extern void svc_rdma_flush_recv_queues(struct svcxprt_rdma *rdma);
0179 extern void svc_rdma_release_rqst(struct svc_rqst *rqstp);
0180 extern int svc_rdma_recvfrom(struct svc_rqst *);
0181
0182 /* svc_rdma_rw.c */
0183 extern void svc_rdma_destroy_rw_ctxts(struct svcxprt_rdma *rdma);
0184 extern int svc_rdma_send_write_chunk(struct svcxprt_rdma *rdma,
0185                      const struct svc_rdma_chunk *chunk,
0186                      const struct xdr_buf *xdr);
0187 extern int svc_rdma_send_reply_chunk(struct svcxprt_rdma *rdma,
0188                      const struct svc_rdma_recv_ctxt *rctxt,
0189                      const struct xdr_buf *xdr);
0190 extern int svc_rdma_process_read_list(struct svcxprt_rdma *rdma,
0191                       struct svc_rqst *rqstp,
0192                       struct svc_rdma_recv_ctxt *head);
0193
0194 /* svc_rdma_sendto.c */
0195 extern void svc_rdma_send_ctxts_destroy(struct svcxprt_rdma *rdma);
0196 extern struct svc_rdma_send_ctxt *
0197         svc_rdma_send_ctxt_get(struct svcxprt_rdma *rdma);
0198 extern void svc_rdma_send_ctxt_put(struct svcxprt_rdma *rdma,
0199                    struct svc_rdma_send_ctxt *ctxt);
0200 extern int svc_rdma_send(struct svcxprt_rdma *rdma,
0201              struct svc_rdma_send_ctxt *ctxt);
0202 extern int svc_rdma_map_reply_msg(struct svcxprt_rdma *rdma,
0203                   struct svc_rdma_send_ctxt *sctxt,
0204                   const struct svc_rdma_recv_ctxt *rctxt,
0205                   const struct xdr_buf *xdr);
0206 extern void svc_rdma_send_error_msg(struct svcxprt_rdma *rdma,
0207                     struct svc_rdma_send_ctxt *sctxt,
0208                     struct svc_rdma_recv_ctxt *rctxt,
0209                     int status);
0210 extern void svc_rdma_wake_send_waiters(struct svcxprt_rdma *rdma, int avail);
0211 extern int svc_rdma_sendto(struct svc_rqst *);
0212 extern int svc_rdma_result_payload(struct svc_rqst *rqstp, unsigned int offset,
0213                    unsigned int length);
0214
0215 /* svc_rdma_transport.c */
0216 extern struct svc_xprt_class svc_rdma_class;
0217 #ifdef CONFIG_SUNRPC_BACKCHANNEL
0218 extern struct svc_xprt_class svc_rdma_bc_class;
0219 #endif
0220
0221 /* svc_rdma.c */
0222 extern int svc_rdma_init(void);
0223 extern void svc_rdma_cleanup(void);
0224
0225 #endif