ulp/rtrs/rtrs-srv.h

0001 /* SPDX-License-Identifier: GPL-2.0-or-later */
0002 /*
0003  * RDMA Transport Layer
0004  *
0005  * Copyright (c) 2014 - 2018 ProfitBricks GmbH. All rights reserved.
0006  * Copyright (c) 2018 - 2019 1&1 IONOS Cloud GmbH. All rights reserved.
0007  * Copyright (c) 2019 - 2020 1&1 IONOS SE. All rights reserved.
0008  */
0009
0010 #ifndef RTRS_SRV_H
0011 #define RTRS_SRV_H
0012
0013 #include <linux/device.h>
0014 #include <linux/refcount.h>
0015 #include <linux/percpu.h>
0016 #include "rtrs-pri.h"
0017
0018 /*
0019  * enum rtrs_srv_state - Server states.
0020  */
0021 enum rtrs_srv_state {
0022     RTRS_SRV_CONNECTING,
0023     RTRS_SRV_CONNECTED,
0024     RTRS_SRV_CLOSING,
0025     RTRS_SRV_CLOSED,
0026 };
0027
0028 /* stats for Read and write operation.
0029  * see Documentation/ABI/testing/sysfs-class-rtrs-server for details
0030  */
0031 struct rtrs_srv_stats_rdma_stats {
0032     struct {
0033         u64 cnt;
0034         u64 size_total;
0035     } dir[2];
0036 };
0037
0038 struct rtrs_srv_stats {
0039     struct kobject                  kobj_stats;
0040     struct rtrs_srv_stats_rdma_stats __percpu   *rdma_stats;
0041     struct rtrs_srv_path                *srv_path;
0042 };
0043
0044 struct rtrs_srv_con {
0045     struct rtrs_con     c;
0046     struct list_head    rsp_wr_wait_list;
0047     spinlock_t      rsp_wr_wait_lock;
0048 };
0049
0050 /* IO context in rtrs_srv, each io has one */
0051 struct rtrs_srv_op {
0052     struct rtrs_srv_con     *con;
0053     u32             msg_id;
0054     u8              dir;
0055     struct rtrs_msg_rdma_read   *rd_msg;
0056     struct ib_rdma_wr       tx_wr;
0057     struct ib_sge           tx_sg;
0058     struct list_head        wait_list;
0059     int             status;
0060 };
0061
0062 /*
0063  * server side memory region context, when always_invalidate=Y, we need
0064  * queue_depth of memory region to invalidate each memory region.
0065  */
0066 struct rtrs_srv_mr {
0067     struct ib_mr    *mr;
0068     struct sg_table sgt;
0069     struct ib_cqe   inv_cqe;    /* only for always_invalidate=true */
0070     u32     msg_id;     /* only for always_invalidate=true */
0071     u32     msg_off;    /* only for always_invalidate=true */
0072     struct rtrs_iu  *iu;        /* send buffer for new rkey msg */
0073 };
0074
0075 struct rtrs_srv_path {
0076     struct rtrs_path    s;
0077     struct rtrs_srv_sess    *srv;
0078     struct work_struct  close_work;
0079     enum rtrs_srv_state state;
0080     spinlock_t      state_lock;
0081     int         cur_cq_vector;
0082     struct rtrs_srv_op  **ops_ids;
0083     struct percpu_ref       ids_inflight_ref;
0084     struct completion       complete_done;
0085     struct rtrs_srv_mr  *mrs;
0086     unsigned int        mrs_num;
0087     dma_addr_t      *dma_addr;
0088     bool            established;
0089     unsigned int        mem_bits;
0090     struct kobject      kobj;
0091     struct rtrs_srv_stats   *stats;
0092 };
0093
0094 struct rtrs_srv_sess {
0095     struct list_head    paths_list;
0096     int         paths_up;
0097     struct mutex        paths_ev_mutex;
0098     size_t          paths_num;
0099     struct mutex        paths_mutex;
0100     uuid_t          paths_uuid;
0101     refcount_t      refcount;
0102     struct rtrs_srv_ctx *ctx;
0103     struct list_head    ctx_list;
0104     void            *priv;
0105     size_t          queue_depth;
0106     struct page     **chunks;
0107     struct device       dev;
0108     unsigned int        dev_ref;
0109     struct kobject      *kobj_paths;
0110 };
0111
0112 struct rtrs_srv_ctx {
0113     struct rtrs_srv_ops ops;
0114     struct rdma_cm_id *cm_id_ip;
0115     struct rdma_cm_id *cm_id_ib;
0116     struct mutex srv_mutex;
0117     struct list_head srv_list;
0118 };
0119
0120 struct rtrs_srv_ib_ctx {
0121     struct rtrs_srv_ctx *srv_ctx;
0122     u16         port;
0123     struct mutex            ib_dev_mutex;
0124     int         ib_dev_count;
0125 };
0126
0127 extern struct class *rtrs_dev_class;
0128
0129 void close_path(struct rtrs_srv_path *srv_path);
0130
0131 static inline void rtrs_srv_update_rdma_stats(struct rtrs_srv_stats *s,
0132                           size_t size, int d)
0133 {
0134     this_cpu_inc(s->rdma_stats->dir[d].cnt);
0135     this_cpu_add(s->rdma_stats->dir[d].size_total, size);
0136 }
0137
0138 /* functions which are implemented in rtrs-srv-stats.c */
0139 int rtrs_srv_reset_rdma_stats(struct rtrs_srv_stats *stats, bool enable);
0140 ssize_t rtrs_srv_stats_rdma_to_str(struct rtrs_srv_stats *stats, char *page);
0141 int rtrs_srv_reset_all_stats(struct rtrs_srv_stats *stats, bool enable);
0142 ssize_t rtrs_srv_reset_all_help(struct rtrs_srv_stats *stats,
0143                  char *page, size_t len);
0144
0145 /* functions which are implemented in rtrs-srv-sysfs.c */
0146 int rtrs_srv_create_path_files(struct rtrs_srv_path *srv_path);
0147 void rtrs_srv_destroy_path_files(struct rtrs_srv_path *srv_path);
0148
0149 #endif /* RTRS_SRV_H */