net/smc/smc_cdc.h

0001 /* SPDX-License-Identifier: GPL-2.0 */
0002 /*
0003  * Shared Memory Communications over RDMA (SMC-R) and RoCE
0004  *
0005  * Connection Data Control (CDC)
0006  *
0007  * Copyright IBM Corp. 2016
0008  *
0009  * Author(s):  Ursula Braun <ubraun@linux.vnet.ibm.com>
0010  */
0011
0012 #ifndef SMC_CDC_H
0013 #define SMC_CDC_H
0014
0015 #include <linux/kernel.h> /* max_t */
0016 #include <linux/atomic.h>
0017 #include <linux/in.h>
0018 #include <linux/compiler.h>
0019
0020 #include "smc.h"
0021 #include "smc_core.h"
0022 #include "smc_wr.h"
0023
0024 #define SMC_CDC_MSG_TYPE        0xFE
0025
0026 /* in network byte order */
0027 union smc_cdc_cursor {      /* SMC cursor */
0028     struct {
0029         __be16  reserved;
0030         __be16  wrap;
0031         __be32  count;
0032     };
0033 #ifdef KERNEL_HAS_ATOMIC64
0034     atomic64_t  acurs;      /* for atomic processing */
0035 #else
0036     u64     acurs;      /* for atomic processing */
0037 #endif
0038 } __aligned(8);
0039
0040 /* in network byte order */
0041 struct smc_cdc_msg {
0042     struct smc_wr_rx_hdr        common; /* .type = 0xFE */
0043     u8              len;    /* 44 */
0044     __be16              seqno;
0045     __be32              token;
0046     union smc_cdc_cursor        prod;
0047     union smc_cdc_cursor        cons;   /* piggy backed "ack" */
0048     struct smc_cdc_producer_flags   prod_flags;
0049     struct smc_cdc_conn_state_flags conn_state_flags;
0050     u8              reserved[18];
0051 };
0052
0053 /* SMC-D cursor format */
0054 union smcd_cdc_cursor {
0055     struct {
0056         u16 wrap;
0057         u32 count;
0058         struct smc_cdc_producer_flags   prod_flags;
0059         struct smc_cdc_conn_state_flags conn_state_flags;
0060     } __packed;
0061 #ifdef KERNEL_HAS_ATOMIC64
0062     atomic64_t      acurs;      /* for atomic processing */
0063 #else
0064     u64         acurs;      /* for atomic processing */
0065 #endif
0066 } __aligned(8);
0067
0068 /* CDC message for SMC-D */
0069 struct smcd_cdc_msg {
0070     struct smc_wr_rx_hdr common;    /* Type = 0xFE */
0071     u8 res1[7];
0072     union smcd_cdc_cursor   prod;
0073     union smcd_cdc_cursor   cons;
0074     u8 res3[8];
0075 } __aligned(8);
0076
0077 static inline bool smc_cdc_rxed_any_close(struct smc_connection *conn)
0078 {
0079     return conn->local_rx_ctrl.conn_state_flags.peer_conn_abort ||
0080            conn->local_rx_ctrl.conn_state_flags.peer_conn_closed;
0081 }
0082
0083 static inline bool smc_cdc_rxed_any_close_or_senddone(
0084     struct smc_connection *conn)
0085 {
0086     return smc_cdc_rxed_any_close(conn) ||
0087            conn->local_rx_ctrl.conn_state_flags.peer_done_writing;
0088 }
0089
0090 static inline void smc_curs_add(int size, union smc_host_cursor *curs,
0091                 int value)
0092 {
0093     curs->count += value;
0094     if (curs->count >= size) {
0095         curs->wrap++;
0096         curs->count -= size;
0097     }
0098 }
0099
0100 /* Copy cursor src into tgt */
0101 static inline void smc_curs_copy(union smc_host_cursor *tgt,
0102                  union smc_host_cursor *src,
0103                  struct smc_connection *conn)
0104 {
0105 #ifndef KERNEL_HAS_ATOMIC64
0106     unsigned long flags;
0107
0108     spin_lock_irqsave(&conn->acurs_lock, flags);
0109     tgt->acurs = src->acurs;
0110     spin_unlock_irqrestore(&conn->acurs_lock, flags);
0111 #else
0112     atomic64_set(&tgt->acurs, atomic64_read(&src->acurs));
0113 #endif
0114 }
0115
0116 static inline void smc_curs_copy_net(union smc_cdc_cursor *tgt,
0117                      union smc_cdc_cursor *src,
0118                      struct smc_connection *conn)
0119 {
0120 #ifndef KERNEL_HAS_ATOMIC64
0121     unsigned long flags;
0122
0123     spin_lock_irqsave(&conn->acurs_lock, flags);
0124     tgt->acurs = src->acurs;
0125     spin_unlock_irqrestore(&conn->acurs_lock, flags);
0126 #else
0127     atomic64_set(&tgt->acurs, atomic64_read(&src->acurs));
0128 #endif
0129 }
0130
0131 static inline void smcd_curs_copy(union smcd_cdc_cursor *tgt,
0132                   union smcd_cdc_cursor *src,
0133                   struct smc_connection *conn)
0134 {
0135 #ifndef KERNEL_HAS_ATOMIC64
0136     unsigned long flags;
0137
0138     spin_lock_irqsave(&conn->acurs_lock, flags);
0139     tgt->acurs = src->acurs;
0140     spin_unlock_irqrestore(&conn->acurs_lock, flags);
0141 #else
0142     atomic64_set(&tgt->acurs, atomic64_read(&src->acurs));
0143 #endif
0144 }
0145
0146 /* calculate cursor difference between old and new, where old <= new and
0147  * difference cannot exceed size
0148  */
0149 static inline int smc_curs_diff(unsigned int size,
0150                 union smc_host_cursor *old,
0151                 union smc_host_cursor *new)
0152 {
0153     if (old->wrap != new->wrap)
0154         return max_t(int, 0,
0155                  ((size - old->count) + new->count));
0156
0157     return max_t(int, 0, (new->count - old->count));
0158 }
0159
0160 /* calculate cursor difference between old and new - returns negative
0161  * value in case old > new
0162  */
0163 static inline int smc_curs_comp(unsigned int size,
0164                 union smc_host_cursor *old,
0165                 union smc_host_cursor *new)
0166 {
0167     if (old->wrap > new->wrap ||
0168         (old->wrap == new->wrap && old->count > new->count))
0169         return -smc_curs_diff(size, new, old);
0170     return smc_curs_diff(size, old, new);
0171 }
0172
0173 /* calculate cursor difference between old and new, where old <= new and
0174  * difference may exceed size
0175  */
0176 static inline int smc_curs_diff_large(unsigned int size,
0177                       union smc_host_cursor *old,
0178                       union smc_host_cursor *new)
0179 {
0180     if (old->wrap < new->wrap)
0181         return min_t(int,
0182                  (size - old->count) + new->count +
0183                  (new->wrap - old->wrap - 1) * size,
0184                  size);
0185
0186     if (old->wrap > new->wrap) /* wrap has switched from 0xffff to 0x0000 */
0187         return min_t(int,
0188                  (size - old->count) + new->count +
0189                  (new->wrap + 0xffff - old->wrap) * size,
0190                  size);
0191
0192     return max_t(int, 0, (new->count - old->count));
0193 }
0194
0195 static inline void smc_host_cursor_to_cdc(union smc_cdc_cursor *peer,
0196                       union smc_host_cursor *local,
0197                       union smc_host_cursor *save,
0198                       struct smc_connection *conn)
0199 {
0200     smc_curs_copy(save, local, conn);
0201     peer->count = htonl(save->count);
0202     peer->wrap = htons(save->wrap);
0203     /* peer->reserved = htons(0); must be ensured by caller */
0204 }
0205
0206 static inline void smc_host_msg_to_cdc(struct smc_cdc_msg *peer,
0207                        struct smc_connection *conn,
0208                        union smc_host_cursor *save)
0209 {
0210     struct smc_host_cdc_msg *local = &conn->local_tx_ctrl;
0211
0212     peer->common.type = local->common.type;
0213     peer->len = local->len;
0214     peer->seqno = htons(local->seqno);
0215     peer->token = htonl(local->token);
0216     smc_host_cursor_to_cdc(&peer->prod, &local->prod, save, conn);
0217     smc_host_cursor_to_cdc(&peer->cons, &local->cons, save, conn);
0218     peer->prod_flags = local->prod_flags;
0219     peer->conn_state_flags = local->conn_state_flags;
0220 }
0221
0222 static inline void smc_cdc_cursor_to_host(union smc_host_cursor *local,
0223                       union smc_cdc_cursor *peer,
0224                       struct smc_connection *conn)
0225 {
0226     union smc_host_cursor temp, old;
0227     union smc_cdc_cursor net;
0228
0229     smc_curs_copy(&old, local, conn);
0230     smc_curs_copy_net(&net, peer, conn);
0231     temp.count = ntohl(net.count);
0232     temp.wrap = ntohs(net.wrap);
0233     if ((old.wrap > temp.wrap) && temp.wrap)
0234         return;
0235     if ((old.wrap == temp.wrap) &&
0236         (old.count > temp.count))
0237         return;
0238     smc_curs_copy(local, &temp, conn);
0239 }
0240
0241 static inline void smcr_cdc_msg_to_host(struct smc_host_cdc_msg *local,
0242                     struct smc_cdc_msg *peer,
0243                     struct smc_connection *conn)
0244 {
0245     local->common.type = peer->common.type;
0246     local->len = peer->len;
0247     local->seqno = ntohs(peer->seqno);
0248     local->token = ntohl(peer->token);
0249     smc_cdc_cursor_to_host(&local->prod, &peer->prod, conn);
0250     smc_cdc_cursor_to_host(&local->cons, &peer->cons, conn);
0251     local->prod_flags = peer->prod_flags;
0252     local->conn_state_flags = peer->conn_state_flags;
0253 }
0254
0255 static inline void smcd_cdc_msg_to_host(struct smc_host_cdc_msg *local,
0256                     struct smcd_cdc_msg *peer,
0257                     struct smc_connection *conn)
0258 {
0259     union smc_host_cursor temp;
0260
0261     temp.wrap = peer->prod.wrap;
0262     temp.count = peer->prod.count;
0263     smc_curs_copy(&local->prod, &temp, conn);
0264
0265     temp.wrap = peer->cons.wrap;
0266     temp.count = peer->cons.count;
0267     smc_curs_copy(&local->cons, &temp, conn);
0268     local->prod_flags = peer->cons.prod_flags;
0269     local->conn_state_flags = peer->cons.conn_state_flags;
0270 }
0271
0272 static inline void smc_cdc_msg_to_host(struct smc_host_cdc_msg *local,
0273                        struct smc_cdc_msg *peer,
0274                        struct smc_connection *conn)
0275 {
0276     if (conn->lgr->is_smcd)
0277         smcd_cdc_msg_to_host(local, (struct smcd_cdc_msg *)peer, conn);
0278     else
0279         smcr_cdc_msg_to_host(local, peer, conn);
0280 }
0281
0282 struct smc_cdc_tx_pend {
0283     struct smc_connection   *conn;      /* socket connection */
0284     union smc_host_cursor   cursor;     /* tx sndbuf cursor sent */
0285     union smc_host_cursor   p_cursor;   /* rx RMBE cursor produced */
0286     u16         ctrl_seq;   /* conn. tx sequence # */
0287 };
0288
0289 int smc_cdc_get_free_slot(struct smc_connection *conn,
0290               struct smc_link *link,
0291               struct smc_wr_buf **wr_buf,
0292               struct smc_rdma_wr **wr_rdma_buf,
0293               struct smc_cdc_tx_pend **pend);
0294 void smc_cdc_wait_pend_tx_wr(struct smc_connection *conn);
0295 int smc_cdc_msg_send(struct smc_connection *conn, struct smc_wr_buf *wr_buf,
0296              struct smc_cdc_tx_pend *pend);
0297 int smc_cdc_get_slot_and_msg_send(struct smc_connection *conn);
0298 int smcd_cdc_msg_send(struct smc_connection *conn);
0299 int smcr_cdc_msg_send_validation(struct smc_connection *conn,
0300                  struct smc_cdc_tx_pend *pend,
0301                  struct smc_wr_buf *wr_buf);
0302 int smc_cdc_init(void) __init;
0303 void smcd_cdc_rx_init(struct smc_connection *conn);
0304
0305 #endif /* SMC_CDC_H */