selftests/bpf/bpf_tcp_helpers.h

0001 /* SPDX-License-Identifier: GPL-2.0 */
0002 #ifndef __BPF_TCP_HELPERS_H
0003 #define __BPF_TCP_HELPERS_H
0004
0005 #include <stdbool.h>
0006 #include <linux/types.h>
0007 #include <bpf/bpf_helpers.h>
0008 #include <bpf/bpf_core_read.h>
0009 #include <bpf/bpf_tracing.h>
0010
0011 #define BPF_STRUCT_OPS(name, args...) \
0012 SEC("struct_ops/"#name) \
0013 BPF_PROG(name, args)
0014
0015 #ifndef SOL_TCP
0016 #define SOL_TCP 6
0017 #endif
0018
0019 #ifndef TCP_CA_NAME_MAX
0020 #define TCP_CA_NAME_MAX 16
0021 #endif
0022
0023 #define tcp_jiffies32 ((__u32)bpf_jiffies64())
0024
0025 struct sock_common {
0026     unsigned char   skc_state;
0027     __u16       skc_num;
0028 } __attribute__((preserve_access_index));
0029
0030 enum sk_pacing {
0031     SK_PACING_NONE      = 0,
0032     SK_PACING_NEEDED    = 1,
0033     SK_PACING_FQ        = 2,
0034 };
0035
0036 struct sock {
0037     struct sock_common  __sk_common;
0038 #define sk_state        __sk_common.skc_state
0039     unsigned long       sk_pacing_rate;
0040     __u32           sk_pacing_status; /* see enum sk_pacing */
0041 } __attribute__((preserve_access_index));
0042
0043 struct inet_sock {
0044     struct sock     sk;
0045 } __attribute__((preserve_access_index));
0046
0047 struct inet_connection_sock {
0048     struct inet_sock      icsk_inet;
0049     __u8              icsk_ca_state:6,
0050                   icsk_ca_setsockopt:1,
0051                   icsk_ca_dst_locked:1;
0052     struct {
0053         __u8          pending;
0054     } icsk_ack;
0055     __u64             icsk_ca_priv[104 / sizeof(__u64)];
0056 } __attribute__((preserve_access_index));
0057
0058 struct request_sock {
0059     struct sock_common      __req_common;
0060 } __attribute__((preserve_access_index));
0061
0062 struct tcp_sock {
0063     struct inet_connection_sock inet_conn;
0064
0065     __u32   rcv_nxt;
0066     __u32   snd_nxt;
0067     __u32   snd_una;
0068     __u32   window_clamp;
0069     __u8    ecn_flags;
0070     __u32   delivered;
0071     __u32   delivered_ce;
0072     __u32   snd_cwnd;
0073     __u32   snd_cwnd_cnt;
0074     __u32   snd_cwnd_clamp;
0075     __u32   snd_ssthresh;
0076     __u8    syn_data:1, /* SYN includes data */
0077         syn_fastopen:1, /* SYN includes Fast Open option */
0078         syn_fastopen_exp:1,/* SYN includes Fast Open exp. option */
0079         syn_fastopen_ch:1, /* Active TFO re-enabling probe */
0080         syn_data_acked:1,/* data in SYN is acked by SYN-ACK */
0081         save_syn:1, /* Save headers of SYN packet */
0082         is_cwnd_limited:1,/* forward progress limited by snd_cwnd? */
0083         syn_smc:1;  /* SYN includes SMC */
0084     __u32   max_packets_out;
0085     __u32   lsndtime;
0086     __u32   prior_cwnd;
0087     __u64   tcp_mstamp; /* most recent packet received/sent */
0088     bool    is_mptcp;
0089 } __attribute__((preserve_access_index));
0090
0091 static __always_inline struct inet_connection_sock *inet_csk(const struct sock *sk)
0092 {
0093     return (struct inet_connection_sock *)sk;
0094 }
0095
0096 static __always_inline void *inet_csk_ca(const struct sock *sk)
0097 {
0098     return (void *)inet_csk(sk)->icsk_ca_priv;
0099 }
0100
0101 static __always_inline struct tcp_sock *tcp_sk(const struct sock *sk)
0102 {
0103     return (struct tcp_sock *)sk;
0104 }
0105
0106 static __always_inline bool before(__u32 seq1, __u32 seq2)
0107 {
0108     return (__s32)(seq1-seq2) < 0;
0109 }
0110 #define after(seq2, seq1)   before(seq1, seq2)
0111
0112 #define TCP_ECN_OK      1
0113 #define TCP_ECN_QUEUE_CWR   2
0114 #define TCP_ECN_DEMAND_CWR  4
0115 #define TCP_ECN_SEEN        8
0116
0117 enum inet_csk_ack_state_t {
0118     ICSK_ACK_SCHED  = 1,
0119     ICSK_ACK_TIMER  = 2,
0120     ICSK_ACK_PUSHED = 4,
0121     ICSK_ACK_PUSHED2 = 8,
0122     ICSK_ACK_NOW = 16   /* Send the next ACK immediately (once) */
0123 };
0124
0125 enum tcp_ca_event {
0126     CA_EVENT_TX_START = 0,
0127     CA_EVENT_CWND_RESTART = 1,
0128     CA_EVENT_COMPLETE_CWR = 2,
0129     CA_EVENT_LOSS = 3,
0130     CA_EVENT_ECN_NO_CE = 4,
0131     CA_EVENT_ECN_IS_CE = 5,
0132 };
0133
0134 struct ack_sample {
0135     __u32 pkts_acked;
0136     __s32 rtt_us;
0137     __u32 in_flight;
0138 } __attribute__((preserve_access_index));
0139
0140 struct rate_sample {
0141     __u64  prior_mstamp; /* starting timestamp for interval */
0142     __u32  prior_delivered; /* tp->delivered at "prior_mstamp" */
0143     __s32  delivered;       /* number of packets delivered over interval */
0144     long interval_us;   /* time for tp->delivered to incr "delivered" */
0145     __u32 snd_interval_us;  /* snd interval for delivered packets */
0146     __u32 rcv_interval_us;  /* rcv interval for delivered packets */
0147     long rtt_us;        /* RTT of last (S)ACKed packet (or -1) */
0148     int  losses;        /* number of packets marked lost upon ACK */
0149     __u32  acked_sacked;    /* number of packets newly (S)ACKed upon ACK */
0150     __u32  prior_in_flight; /* in flight before this ACK */
0151     bool is_app_limited;    /* is sample from packet with bubble in pipe? */
0152     bool is_retrans;    /* is sample from retransmission? */
0153     bool is_ack_delayed;    /* is this (likely) a delayed ACK? */
0154 } __attribute__((preserve_access_index));
0155
0156 #define TCP_CA_NAME_MAX     16
0157 #define TCP_CONG_NEEDS_ECN  0x2
0158
0159 struct tcp_congestion_ops {
0160     char name[TCP_CA_NAME_MAX];
0161     __u32 flags;
0162
0163     /* initialize private data (optional) */
0164     void (*init)(struct sock *sk);
0165     /* cleanup private data  (optional) */
0166     void (*release)(struct sock *sk);
0167
0168     /* return slow start threshold (required) */
0169     __u32 (*ssthresh)(struct sock *sk);
0170     /* do new cwnd calculation (required) */
0171     void (*cong_avoid)(struct sock *sk, __u32 ack, __u32 acked);
0172     /* call before changing ca_state (optional) */
0173     void (*set_state)(struct sock *sk, __u8 new_state);
0174     /* call when cwnd event occurs (optional) */
0175     void (*cwnd_event)(struct sock *sk, enum tcp_ca_event ev);
0176     /* call when ack arrives (optional) */
0177     void (*in_ack_event)(struct sock *sk, __u32 flags);
0178     /* new value of cwnd after loss (required) */
0179     __u32  (*undo_cwnd)(struct sock *sk);
0180     /* hook for packet ack accounting (optional) */
0181     void (*pkts_acked)(struct sock *sk, const struct ack_sample *sample);
0182     /* override sysctl_tcp_min_tso_segs */
0183     __u32 (*min_tso_segs)(struct sock *sk);
0184     /* returns the multiplier used in tcp_sndbuf_expand (optional) */
0185     __u32 (*sndbuf_expand)(struct sock *sk);
0186     /* call when packets are delivered to update cwnd and pacing rate,
0187      * after all the ca_state processing. (optional)
0188      */
0189     void (*cong_control)(struct sock *sk, const struct rate_sample *rs);
0190     void *owner;
0191 };
0192
0193 #define min(a, b) ((a) < (b) ? (a) : (b))
0194 #define max(a, b) ((a) > (b) ? (a) : (b))
0195 #define min_not_zero(x, y) ({           \
0196     typeof(x) __x = (x);            \
0197     typeof(y) __y = (y);            \
0198     __x == 0 ? __y : ((__y == 0) ? __x : min(__x, __y)); })
0199
0200 static __always_inline bool tcp_in_slow_start(const struct tcp_sock *tp)
0201 {
0202     return tp->snd_cwnd < tp->snd_ssthresh;
0203 }
0204
0205 static __always_inline bool tcp_is_cwnd_limited(const struct sock *sk)
0206 {
0207     const struct tcp_sock *tp = tcp_sk(sk);
0208
0209     /* If in slow start, ensure cwnd grows to twice what was ACKed. */
0210     if (tcp_in_slow_start(tp))
0211         return tp->snd_cwnd < 2 * tp->max_packets_out;
0212
0213     return !!BPF_CORE_READ_BITFIELD(tp, is_cwnd_limited);
0214 }
0215
0216 static __always_inline bool tcp_cc_eq(const char *a, const char *b)
0217 {
0218     int i;
0219
0220     for (i = 0; i < TCP_CA_NAME_MAX; i++) {
0221         if (a[i] != b[i])
0222             return false;
0223         if (!a[i])
0224             break;
0225     }
0226
0227     return true;
0228 }
0229
0230 extern __u32 tcp_slow_start(struct tcp_sock *tp, __u32 acked) __ksym;
0231 extern void tcp_cong_avoid_ai(struct tcp_sock *tp, __u32 w, __u32 acked) __ksym;
0232
0233 struct mptcp_sock {
0234     struct inet_connection_sock sk;
0235
0236     __u32       token;
0237     struct sock *first;
0238     char        ca_name[TCP_CA_NAME_MAX];
0239 } __attribute__((preserve_access_index));
0240
0241 #endif