block/drbd/drbd_protocol.h

0001 /* SPDX-License-Identifier: GPL-2.0 */
0002 #ifndef __DRBD_PROTOCOL_H
0003 #define __DRBD_PROTOCOL_H
0004
0005 enum drbd_packet {
0006     /* receiver (data socket) */
0007     P_DATA            = 0x00,
0008     P_DATA_REPLY          = 0x01, /* Response to P_DATA_REQUEST */
0009     P_RS_DATA_REPLY       = 0x02, /* Response to P_RS_DATA_REQUEST */
0010     P_BARRIER         = 0x03,
0011     P_BITMAP          = 0x04,
0012     P_BECOME_SYNC_TARGET  = 0x05,
0013     P_BECOME_SYNC_SOURCE  = 0x06,
0014     P_UNPLUG_REMOTE       = 0x07, /* Used at various times to hint the peer */
0015     P_DATA_REQUEST        = 0x08, /* Used to ask for a data block */
0016     P_RS_DATA_REQUEST     = 0x09, /* Used to ask for a data block for resync */
0017     P_SYNC_PARAM          = 0x0a,
0018     P_PROTOCOL        = 0x0b,
0019     P_UUIDS           = 0x0c,
0020     P_SIZES           = 0x0d,
0021     P_STATE           = 0x0e,
0022     P_SYNC_UUID       = 0x0f,
0023     P_AUTH_CHALLENGE      = 0x10,
0024     P_AUTH_RESPONSE       = 0x11,
0025     P_STATE_CHG_REQ       = 0x12,
0026
0027     /* (meta socket) */
0028     P_PING            = 0x13,
0029     P_PING_ACK        = 0x14,
0030     P_RECV_ACK        = 0x15, /* Used in protocol B */
0031     P_WRITE_ACK       = 0x16, /* Used in protocol C */
0032     P_RS_WRITE_ACK        = 0x17, /* Is a P_WRITE_ACK, additionally call set_in_sync(). */
0033     P_SUPERSEDED          = 0x18, /* Used in proto C, two-primaries conflict detection */
0034     P_NEG_ACK         = 0x19, /* Sent if local disk is unusable */
0035     P_NEG_DREPLY          = 0x1a, /* Local disk is broken... */
0036     P_NEG_RS_DREPLY       = 0x1b, /* Local disk is broken... */
0037     P_BARRIER_ACK         = 0x1c,
0038     P_STATE_CHG_REPLY     = 0x1d,
0039
0040     /* "new" commands, no longer fitting into the ordering scheme above */
0041
0042     P_OV_REQUEST          = 0x1e, /* data socket */
0043     P_OV_REPLY        = 0x1f,
0044     P_OV_RESULT       = 0x20, /* meta socket */
0045     P_CSUM_RS_REQUEST     = 0x21, /* data socket */
0046     P_RS_IS_IN_SYNC       = 0x22, /* meta socket */
0047     P_SYNC_PARAM89        = 0x23, /* data socket, protocol version 89 replacement for P_SYNC_PARAM */
0048     P_COMPRESSED_BITMAP   = 0x24, /* compressed or otherwise encoded bitmap transfer */
0049     /* P_CKPT_FENCE_REQ      = 0x25, * currently reserved for protocol D */
0050     /* P_CKPT_DISABLE_REQ    = 0x26, * currently reserved for protocol D */
0051     P_DELAY_PROBE         = 0x27, /* is used on BOTH sockets */
0052     P_OUT_OF_SYNC         = 0x28, /* Mark as out of sync (Outrunning), data socket */
0053     P_RS_CANCEL           = 0x29, /* meta: Used to cancel RS_DATA_REQUEST packet by SyncSource */
0054     P_CONN_ST_CHG_REQ     = 0x2a, /* data sock: Connection wide state request */
0055     P_CONN_ST_CHG_REPLY   = 0x2b, /* meta sock: Connection side state req reply */
0056     P_RETRY_WRITE         = 0x2c, /* Protocol C: retry conflicting write request */
0057     P_PROTOCOL_UPDATE     = 0x2d, /* data sock: is used in established connections */
0058         /* 0x2e to 0x30 reserved, used in drbd 9 */
0059
0060     /* REQ_OP_DISCARD. We used "discard" in different contexts before,
0061      * which is why I chose TRIM here, to disambiguate. */
0062     P_TRIM                = 0x31,
0063
0064     /* Only use these two if both support FF_THIN_RESYNC */
0065     P_RS_THIN_REQ         = 0x32, /* Request a block for resync or reply P_RS_DEALLOCATED */
0066     P_RS_DEALLOCATED      = 0x33, /* Contains only zeros on sync source node */
0067
0068     /* REQ_WRITE_SAME.
0069      * On a receiving side without REQ_WRITE_SAME,
0070      * we may fall back to an opencoded loop instead. */
0071     P_WSAME               = 0x34,
0072
0073     /* 0x35 already claimed in DRBD 9 */
0074     P_ZEROES              = 0x36, /* data sock: zero-out, WRITE_ZEROES */
0075
0076     /* 0x40 .. 0x48 already claimed in DRBD 9 */
0077
0078     P_MAY_IGNORE          = 0x100, /* Flag to test if (cmd > P_MAY_IGNORE) ... */
0079     P_MAX_OPT_CMD         = 0x101,
0080
0081     /* special command ids for handshake */
0082
0083     P_INITIAL_META        = 0xfff1, /* First Packet on the MetaSock */
0084     P_INITIAL_DATA        = 0xfff2, /* First Packet on the Socket */
0085
0086     P_CONNECTION_FEATURES = 0xfffe  /* FIXED for the next century! */
0087 };
0088
0089 #ifndef __packed
0090 #define __packed __attribute__((packed))
0091 #endif
0092
0093 /* This is the layout for a packet on the wire.
0094  * The byteorder is the network byte order.
0095  *     (except block_id and barrier fields.
0096  *  these are pointers to local structs
0097  *  and have no relevance for the partner,
0098  *  which just echoes them as received.)
0099  *
0100  * NOTE that the payload starts at a long aligned offset,
0101  * regardless of 32 or 64 bit arch!
0102  */
0103 struct p_header80 {
0104     u32   magic;
0105     u16   command;
0106     u16   length;   /* bytes of data after this header */
0107 } __packed;
0108
0109 /* Header for big packets, Used for data packets exceeding 64kB */
0110 struct p_header95 {
0111     u16   magic;    /* use DRBD_MAGIC_BIG here */
0112     u16   command;
0113     u32   length;
0114 } __packed;
0115
0116 struct p_header100 {
0117     u32   magic;
0118     u16   volume;
0119     u16   command;
0120     u32   length;
0121     u32   pad;
0122 } __packed;
0123
0124 /* These defines must not be changed without changing the protocol version.
0125  * New defines may only be introduced together with protocol version bump or
0126  * new protocol feature flags.
0127  */
0128 #define DP_HARDBARRIER        1 /* no longer used */
0129 #define DP_RW_SYNC        2 /* equals REQ_SYNC    */
0130 #define DP_MAY_SET_IN_SYNC    4
0131 #define DP_UNPLUG             8 /* not used anymore   */
0132 #define DP_FUA               16 /* equals REQ_FUA     */
0133 #define DP_FLUSH             32 /* equals REQ_PREFLUSH   */
0134 #define DP_DISCARD           64 /* equals REQ_OP_DISCARD */
0135 #define DP_SEND_RECEIVE_ACK 128 /* This is a proto B write request */
0136 #define DP_SEND_WRITE_ACK   256 /* This is a proto C write request */
0137 #define DP_WSAME            512 /* equiv. REQ_WRITE_SAME */
0138 #define DP_ZEROES          1024 /* equiv. REQ_OP_WRITE_ZEROES */
0139
0140 /* possible combinations:
0141  * REQ_OP_WRITE_ZEROES:  DP_DISCARD | DP_ZEROES
0142  * REQ_OP_WRITE_ZEROES + REQ_NOUNMAP: DP_ZEROES
0143  */
0144
0145 struct p_data {
0146     u64     sector;    /* 64 bits sector number */
0147     u64     block_id;  /* to identify the request in protocol B&C */
0148     u32     seq_num;
0149     u32     dp_flags;
0150 } __packed;
0151
0152 struct p_trim {
0153     struct p_data p_data;
0154     u32     size;   /* == bio->bi_size */
0155 } __packed;
0156
0157 struct p_wsame {
0158     struct p_data p_data;
0159     u32           size;     /* == bio->bi_size */
0160 } __packed;
0161
0162 /*
0163  * commands which share a struct:
0164  *  p_block_ack:
0165  *   P_RECV_ACK (proto B), P_WRITE_ACK (proto C),
0166  *   P_SUPERSEDED (proto C, two-primaries conflict detection)
0167  *  p_block_req:
0168  *   P_DATA_REQUEST, P_RS_DATA_REQUEST
0169  */
0170 struct p_block_ack {
0171     u64     sector;
0172     u64     block_id;
0173     u32     blksize;
0174     u32     seq_num;
0175 } __packed;
0176
0177 struct p_block_req {
0178     u64 sector;
0179     u64 block_id;
0180     u32 blksize;
0181     u32 pad;    /* to multiple of 8 Byte */
0182 } __packed;
0183
0184 /*
0185  * commands with their own struct for additional fields:
0186  *   P_CONNECTION_FEATURES
0187  *   P_BARRIER
0188  *   P_BARRIER_ACK
0189  *   P_SYNC_PARAM
0190  *   ReportParams
0191  */
0192
0193 /* supports TRIM/DISCARD on the "wire" protocol */
0194 #define DRBD_FF_TRIM 1
0195
0196 /* Detect all-zeros during resync, and rather TRIM/UNMAP/DISCARD those blocks
0197  * instead of fully allocate a supposedly thin volume on initial resync */
0198 #define DRBD_FF_THIN_RESYNC 2
0199
0200 /* supports REQ_WRITE_SAME on the "wire" protocol.
0201  * Note: this flag is overloaded,
0202  * its presence also
0203  *   - indicates support for 128 MiB "batch bios",
0204  *     max discard size of 128 MiB
0205  *     instead of 4M before that.
0206  *   - indicates that we exchange additional settings in p_sizes
0207  *     drbd_send_sizes()/receive_sizes()
0208  */
0209 #define DRBD_FF_WSAME 4
0210
0211 /* supports REQ_OP_WRITE_ZEROES on the "wire" protocol.
0212  *
0213  * We used to map that to "discard" on the sending side, and if we cannot
0214  * guarantee that discard zeroes data, the receiving side would map discard
0215  * back to zero-out.
0216  *
0217  * With the introduction of REQ_OP_WRITE_ZEROES,
0218  * we started to use that for both WRITE_ZEROES and DISCARDS,
0219  * hoping that WRITE_ZEROES would "do what we want",
0220  * UNMAP if possible, zero-out the rest.
0221  *
0222  * The example scenario is some LVM "thin" backend.
0223  *
0224  * While an un-allocated block on dm-thin reads as zeroes, on a dm-thin
0225  * with "skip_block_zeroing=true", after a partial block write allocated
0226  * that block, that same block may well map "undefined old garbage" from
0227  * the backends on LBAs that have not yet been written to.
0228  *
0229  * If we cannot distinguish between zero-out and discard on the receiving
0230  * side, to avoid "undefined old garbage" to pop up randomly at later times
0231  * on supposedly zero-initialized blocks, we'd need to map all discards to
0232  * zero-out on the receiving side.  But that would potentially do a full
0233  * alloc on thinly provisioned backends, even when the expectation was to
0234  * unmap/trim/discard/de-allocate.
0235  *
0236  * We need to distinguish on the protocol level, whether we need to guarantee
0237  * zeroes (and thus use zero-out, potentially doing the mentioned full-alloc),
0238  * or if we want to put the emphasis on discard, and only do a "best effort
0239  * zeroing" (by "discarding" blocks aligned to discard-granularity, and zeroing
0240  * only potential unaligned head and tail clippings), to at least *try* to
0241  * avoid "false positives" in an online-verify later, hoping that someone
0242  * set skip_block_zeroing=false.
0243  */
0244 #define DRBD_FF_WZEROES 8
0245
0246
0247 struct p_connection_features {
0248     u32 protocol_min;
0249     u32 feature_flags;
0250     u32 protocol_max;
0251
0252     /* should be more than enough for future enhancements
0253      * for now, feature_flags and the reserved array shall be zero.
0254      */
0255
0256     u32 _pad;
0257     u64 reserved[7];
0258 } __packed;
0259
0260 struct p_barrier {
0261     u32 barrier;    /* barrier number _handle_ only */
0262     u32 pad;    /* to multiple of 8 Byte */
0263 } __packed;
0264
0265 struct p_barrier_ack {
0266     u32 barrier;
0267     u32 set_size;
0268 } __packed;
0269
0270 struct p_rs_param {
0271     u32 resync_rate;
0272
0273           /* Since protocol version 88 and higher. */
0274     char verify_alg[];
0275 } __packed;
0276
0277 struct p_rs_param_89 {
0278     u32 resync_rate;
0279     /* protocol version 89: */
0280     char verify_alg[SHARED_SECRET_MAX];
0281     char csums_alg[SHARED_SECRET_MAX];
0282 } __packed;
0283
0284 struct p_rs_param_95 {
0285     u32 resync_rate;
0286     struct_group(algs,
0287         char verify_alg[SHARED_SECRET_MAX];
0288         char csums_alg[SHARED_SECRET_MAX];
0289     );
0290     u32 c_plan_ahead;
0291     u32 c_delay_target;
0292     u32 c_fill_target;
0293     u32 c_max_rate;
0294 } __packed;
0295
0296 enum drbd_conn_flags {
0297     CF_DISCARD_MY_DATA = 1,
0298     CF_DRY_RUN = 2,
0299 };
0300
0301 struct p_protocol {
0302     u32 protocol;
0303     u32 after_sb_0p;
0304     u32 after_sb_1p;
0305     u32 after_sb_2p;
0306     u32 conn_flags;
0307     u32 two_primaries;
0308
0309     /* Since protocol version 87 and higher. */
0310     char integrity_alg[];
0311
0312 } __packed;
0313
0314 struct p_uuids {
0315     u64 uuid[UI_EXTENDED_SIZE];
0316 } __packed;
0317
0318 struct p_rs_uuid {
0319     u64     uuid;
0320 } __packed;
0321
0322 /* optional queue_limits if (agreed_features & DRBD_FF_WSAME)
0323  * see also struct queue_limits, as of late 2015 */
0324 struct o_qlim {
0325     /* we don't need it yet, but we may as well communicate it now */
0326     u32 physical_block_size;
0327
0328     /* so the original in struct queue_limits is unsigned short,
0329      * but I'd have to put in padding anyways. */
0330     u32 logical_block_size;
0331
0332     /* One incoming bio becomes one DRBD request,
0333      * which may be translated to several bio on the receiving side.
0334      * We don't need to communicate chunk/boundary/segment ... limits.
0335      */
0336
0337     /* various IO hints may be useful with "diskless client" setups */
0338     u32 alignment_offset;
0339     u32 io_min;
0340     u32 io_opt;
0341
0342     /* We may need to communicate integrity stuff at some point,
0343      * but let's not get ahead of ourselves. */
0344
0345     /* Backend discard capabilities.
0346      * Receiving side uses "blkdev_issue_discard()", no need to communicate
0347      * more specifics.  If the backend cannot do discards, the DRBD peer
0348      * may fall back to blkdev_issue_zeroout().
0349      */
0350     u8 discard_enabled;
0351     u8 discard_zeroes_data;
0352     u8 write_same_capable;
0353     u8 _pad;
0354 } __packed;
0355
0356 struct p_sizes {
0357     u64     d_size;  /* size of disk */
0358     u64     u_size;  /* user requested size */
0359     u64     c_size;  /* current exported size */
0360     u32     max_bio_size;  /* Maximal size of a BIO */
0361     u16     queue_order_type;  /* not yet implemented in DRBD*/
0362     u16     dds_flags; /* use enum dds_flags here. */
0363
0364     /* optional queue_limits if (agreed_features & DRBD_FF_WSAME) */
0365     struct o_qlim qlim[];
0366 } __packed;
0367
0368 struct p_state {
0369     u32     state;
0370 } __packed;
0371
0372 struct p_req_state {
0373     u32     mask;
0374     u32     val;
0375 } __packed;
0376
0377 struct p_req_state_reply {
0378     u32     retcode;
0379 } __packed;
0380
0381 struct p_drbd06_param {
0382     u64   size;
0383     u32   state;
0384     u32   blksize;
0385     u32   protocol;
0386     u32   version;
0387     u32   gen_cnt[5];
0388     u32   bit_map_gen[5];
0389 } __packed;
0390
0391 struct p_block_desc {
0392     u64 sector;
0393     u32 blksize;
0394     u32 pad;    /* to multiple of 8 Byte */
0395 } __packed;
0396
0397 /* Valid values for the encoding field.
0398  * Bump proto version when changing this. */
0399 enum drbd_bitmap_code {
0400     /* RLE_VLI_Bytes = 0,
0401      * and other bit variants had been defined during
0402      * algorithm evaluation. */
0403     RLE_VLI_Bits = 2,
0404 };
0405
0406 struct p_compressed_bm {
0407     /* (encoding & 0x0f): actual encoding, see enum drbd_bitmap_code
0408      * (encoding & 0x80): polarity (set/unset) of first runlength
0409      * ((encoding >> 4) & 0x07): pad_bits, number of trailing zero bits
0410      * used to pad up to head.length bytes
0411      */
0412     u8 encoding;
0413
0414     u8 code[];
0415 } __packed;
0416
0417 struct p_delay_probe93 {
0418     u32     seq_num; /* sequence number to match the two probe packets */
0419     u32     offset;  /* usecs the probe got sent after the reference time point */
0420 } __packed;
0421
0422 /*
0423  * Bitmap packets need to fit within a single page on the sender and receiver,
0424  * so we are limited to 4 KiB (and not to PAGE_SIZE, which can be bigger).
0425  */
0426 #define DRBD_SOCKET_BUFFER_SIZE 4096
0427
0428 #endif  /* __DRBD_PROTOCOL_H */