Back to home page

OSCL-LXR
 
 

     

0001 /* SPDX-License-Identifier: GPL-2.0 */
0002 /* Multipath TCP
0003  *
0004  * Copyright (c) 2017 - 2019, Intel Corporation.
0005  */
0006 
0007 #ifndef __MPTCP_PROTOCOL_H
0008 #define __MPTCP_PROTOCOL_H
0009 
0010 #include <linux/random.h>
0011 #include <net/tcp.h>
0012 #include <net/inet_connection_sock.h>
0013 #include <uapi/linux/mptcp.h>
0014 #include <net/genetlink.h>
0015 
0016 #define MPTCP_SUPPORTED_VERSION 1
0017 
0018 /* MPTCP option bits */
0019 #define OPTION_MPTCP_MPC_SYN    BIT(0)
0020 #define OPTION_MPTCP_MPC_SYNACK BIT(1)
0021 #define OPTION_MPTCP_MPC_ACK    BIT(2)
0022 #define OPTION_MPTCP_MPJ_SYN    BIT(3)
0023 #define OPTION_MPTCP_MPJ_SYNACK BIT(4)
0024 #define OPTION_MPTCP_MPJ_ACK    BIT(5)
0025 #define OPTION_MPTCP_ADD_ADDR   BIT(6)
0026 #define OPTION_MPTCP_RM_ADDR    BIT(7)
0027 #define OPTION_MPTCP_FASTCLOSE  BIT(8)
0028 #define OPTION_MPTCP_PRIO   BIT(9)
0029 #define OPTION_MPTCP_RST    BIT(10)
0030 #define OPTION_MPTCP_DSS    BIT(11)
0031 #define OPTION_MPTCP_FAIL   BIT(12)
0032 
0033 #define OPTION_MPTCP_CSUMREQD   BIT(13)
0034 
0035 #define OPTIONS_MPTCP_MPC   (OPTION_MPTCP_MPC_SYN | OPTION_MPTCP_MPC_SYNACK | \
0036                  OPTION_MPTCP_MPC_ACK)
0037 #define OPTIONS_MPTCP_MPJ   (OPTION_MPTCP_MPJ_SYN | OPTION_MPTCP_MPJ_SYNACK | \
0038                  OPTION_MPTCP_MPJ_ACK)
0039 
0040 /* MPTCP option subtypes */
0041 #define MPTCPOPT_MP_CAPABLE 0
0042 #define MPTCPOPT_MP_JOIN    1
0043 #define MPTCPOPT_DSS        2
0044 #define MPTCPOPT_ADD_ADDR   3
0045 #define MPTCPOPT_RM_ADDR    4
0046 #define MPTCPOPT_MP_PRIO    5
0047 #define MPTCPOPT_MP_FAIL    6
0048 #define MPTCPOPT_MP_FASTCLOSE   7
0049 #define MPTCPOPT_RST        8
0050 
0051 /* MPTCP suboption lengths */
0052 #define TCPOLEN_MPTCP_MPC_SYN       4
0053 #define TCPOLEN_MPTCP_MPC_SYNACK    12
0054 #define TCPOLEN_MPTCP_MPC_ACK       20
0055 #define TCPOLEN_MPTCP_MPC_ACK_DATA  22
0056 #define TCPOLEN_MPTCP_MPJ_SYN       12
0057 #define TCPOLEN_MPTCP_MPJ_SYNACK    16
0058 #define TCPOLEN_MPTCP_MPJ_ACK       24
0059 #define TCPOLEN_MPTCP_DSS_BASE      4
0060 #define TCPOLEN_MPTCP_DSS_ACK32     4
0061 #define TCPOLEN_MPTCP_DSS_ACK64     8
0062 #define TCPOLEN_MPTCP_DSS_MAP32     10
0063 #define TCPOLEN_MPTCP_DSS_MAP64     14
0064 #define TCPOLEN_MPTCP_DSS_CHECKSUM  2
0065 #define TCPOLEN_MPTCP_ADD_ADDR      16
0066 #define TCPOLEN_MPTCP_ADD_ADDR_PORT 18
0067 #define TCPOLEN_MPTCP_ADD_ADDR_BASE 8
0068 #define TCPOLEN_MPTCP_ADD_ADDR_BASE_PORT    10
0069 #define TCPOLEN_MPTCP_ADD_ADDR6     28
0070 #define TCPOLEN_MPTCP_ADD_ADDR6_PORT    30
0071 #define TCPOLEN_MPTCP_ADD_ADDR6_BASE    20
0072 #define TCPOLEN_MPTCP_ADD_ADDR6_BASE_PORT   22
0073 #define TCPOLEN_MPTCP_PORT_LEN      2
0074 #define TCPOLEN_MPTCP_PORT_ALIGN    2
0075 #define TCPOLEN_MPTCP_RM_ADDR_BASE  3
0076 #define TCPOLEN_MPTCP_PRIO      3
0077 #define TCPOLEN_MPTCP_PRIO_ALIGN    4
0078 #define TCPOLEN_MPTCP_FASTCLOSE     12
0079 #define TCPOLEN_MPTCP_RST       4
0080 #define TCPOLEN_MPTCP_FAIL      12
0081 
0082 #define TCPOLEN_MPTCP_MPC_ACK_DATA_CSUM (TCPOLEN_MPTCP_DSS_CHECKSUM + TCPOLEN_MPTCP_MPC_ACK_DATA)
0083 
0084 /* MPTCP MP_JOIN flags */
0085 #define MPTCPOPT_BACKUP     BIT(0)
0086 #define MPTCPOPT_THMAC_LEN  8
0087 
0088 /* MPTCP MP_CAPABLE flags */
0089 #define MPTCP_VERSION_MASK  (0x0F)
0090 #define MPTCP_CAP_CHECKSUM_REQD BIT(7)
0091 #define MPTCP_CAP_EXTENSIBILITY BIT(6)
0092 #define MPTCP_CAP_DENY_JOIN_ID0 BIT(5)
0093 #define MPTCP_CAP_HMAC_SHA256   BIT(0)
0094 #define MPTCP_CAP_FLAG_MASK (0x1F)
0095 
0096 /* MPTCP DSS flags */
0097 #define MPTCP_DSS_DATA_FIN  BIT(4)
0098 #define MPTCP_DSS_DSN64     BIT(3)
0099 #define MPTCP_DSS_HAS_MAP   BIT(2)
0100 #define MPTCP_DSS_ACK64     BIT(1)
0101 #define MPTCP_DSS_HAS_ACK   BIT(0)
0102 #define MPTCP_DSS_FLAG_MASK (0x1F)
0103 
0104 /* MPTCP ADD_ADDR flags */
0105 #define MPTCP_ADDR_ECHO     BIT(0)
0106 
0107 /* MPTCP MP_PRIO flags */
0108 #define MPTCP_PRIO_BKUP     BIT(0)
0109 
0110 /* MPTCP TCPRST flags */
0111 #define MPTCP_RST_TRANSIENT BIT(0)
0112 
0113 /* MPTCP socket atomic flags */
0114 #define MPTCP_NOSPACE       1
0115 #define MPTCP_WORK_RTX      2
0116 #define MPTCP_WORK_EOF      3
0117 #define MPTCP_FALLBACK_DONE 4
0118 #define MPTCP_WORK_CLOSE_SUBFLOW 5
0119 
0120 /* MPTCP socket release cb flags */
0121 #define MPTCP_PUSH_PENDING  1
0122 #define MPTCP_CLEAN_UNA     2
0123 #define MPTCP_ERROR_REPORT  3
0124 #define MPTCP_RETRANSMIT    4
0125 #define MPTCP_FLUSH_JOIN_LIST   5
0126 #define MPTCP_CONNECTED     6
0127 #define MPTCP_RESET_SCHEDULER   7
0128 
0129 static inline bool before64(__u64 seq1, __u64 seq2)
0130 {
0131     return (__s64)(seq1 - seq2) < 0;
0132 }
0133 
0134 #define after64(seq2, seq1) before64(seq1, seq2)
0135 
0136 struct mptcp_options_received {
0137     u64 sndr_key;
0138     u64 rcvr_key;
0139     u64 data_ack;
0140     u64 data_seq;
0141     u32 subflow_seq;
0142     u16 data_len;
0143     __sum16 csum;
0144     u16 suboptions;
0145     u32 token;
0146     u32 nonce;
0147     u16 use_map:1,
0148         dsn64:1,
0149         data_fin:1,
0150         use_ack:1,
0151         ack64:1,
0152         mpc_map:1,
0153         reset_reason:4,
0154         reset_transient:1,
0155         echo:1,
0156         backup:1,
0157         deny_join_id0:1,
0158         __unused:2;
0159     u8  join_id;
0160     u64 thmac;
0161     u8  hmac[MPTCPOPT_HMAC_LEN];
0162     struct mptcp_addr_info addr;
0163     struct mptcp_rm_list rm_list;
0164     u64 ahmac;
0165     u64 fail_seq;
0166 };
0167 
0168 static inline __be32 mptcp_option(u8 subopt, u8 len, u8 nib, u8 field)
0169 {
0170     return htonl((TCPOPT_MPTCP << 24) | (len << 16) | (subopt << 12) |
0171              ((nib & 0xF) << 8) | field);
0172 }
0173 
0174 enum mptcp_pm_status {
0175     MPTCP_PM_ADD_ADDR_RECEIVED,
0176     MPTCP_PM_ADD_ADDR_SEND_ACK,
0177     MPTCP_PM_RM_ADDR_RECEIVED,
0178     MPTCP_PM_ESTABLISHED,
0179     MPTCP_PM_SUBFLOW_ESTABLISHED,
0180     MPTCP_PM_ALREADY_ESTABLISHED,   /* persistent status, set after ESTABLISHED event */
0181     MPTCP_PM_MPC_ENDPOINT_ACCOUNTED /* persistent status, set after MPC local address is
0182                      * accounted int id_avail_bitmap
0183                      */
0184 };
0185 
0186 enum mptcp_pm_type {
0187     MPTCP_PM_TYPE_KERNEL = 0,
0188     MPTCP_PM_TYPE_USERSPACE,
0189 
0190     __MPTCP_PM_TYPE_NR,
0191     __MPTCP_PM_TYPE_MAX = __MPTCP_PM_TYPE_NR - 1,
0192 };
0193 
0194 /* Status bits below MPTCP_PM_ALREADY_ESTABLISHED need pm worker actions */
0195 #define MPTCP_PM_WORK_MASK ((1 << MPTCP_PM_ALREADY_ESTABLISHED) - 1)
0196 
0197 enum mptcp_addr_signal_status {
0198     MPTCP_ADD_ADDR_SIGNAL,
0199     MPTCP_ADD_ADDR_ECHO,
0200     MPTCP_RM_ADDR_SIGNAL,
0201 };
0202 
0203 /* max value of mptcp_addr_info.id */
0204 #define MPTCP_PM_MAX_ADDR_ID        U8_MAX
0205 
0206 struct mptcp_pm_data {
0207     struct mptcp_addr_info local;
0208     struct mptcp_addr_info remote;
0209     struct list_head anno_list;
0210     struct list_head userspace_pm_local_addr_list;
0211 
0212     spinlock_t  lock;       /*protects the whole PM data */
0213 
0214     u8      addr_signal;
0215     bool        server_side;
0216     bool        work_pending;
0217     bool        accept_addr;
0218     bool        accept_subflow;
0219     bool        remote_deny_join_id0;
0220     u8      add_addr_signaled;
0221     u8      add_addr_accepted;
0222     u8      local_addr_used;
0223     u8      pm_type;
0224     u8      subflows;
0225     u8      status;
0226     DECLARE_BITMAP(id_avail_bitmap, MPTCP_PM_MAX_ADDR_ID + 1);
0227     struct mptcp_rm_list rm_list_tx;
0228     struct mptcp_rm_list rm_list_rx;
0229 };
0230 
0231 struct mptcp_pm_addr_entry {
0232     struct list_head    list;
0233     struct mptcp_addr_info  addr;
0234     u8          flags;
0235     int         ifindex;
0236     struct socket       *lsk;
0237 };
0238 
0239 struct mptcp_data_frag {
0240     struct list_head list;
0241     u64 data_seq;
0242     u16 data_len;
0243     u16 offset;
0244     u16 overhead;
0245     u16 already_sent;
0246     struct page *page;
0247 };
0248 
0249 /* MPTCP connection sock */
0250 struct mptcp_sock {
0251     /* inet_connection_sock must be the first member */
0252     struct inet_connection_sock sk;
0253     u64     local_key;
0254     u64     remote_key;
0255     u64     write_seq;
0256     u64     snd_nxt;
0257     u64     ack_seq;
0258     atomic64_t  rcv_wnd_sent;
0259     u64     rcv_data_fin_seq;
0260     int     rmem_fwd_alloc;
0261     struct sock *last_snd;
0262     int     snd_burst;
0263     int     old_wspace;
0264     u64     recovery_snd_nxt;   /* in recovery mode accept up to this seq;
0265                          * recovery related fields are under data_lock
0266                          * protection
0267                          */
0268     u64     snd_una;
0269     u64     wnd_end;
0270     unsigned long   timer_ival;
0271     u32     token;
0272     int     rmem_released;
0273     unsigned long   flags;
0274     unsigned long   cb_flags;
0275     unsigned long   push_pending;
0276     bool        recovery;       /* closing subflow write queue reinjected */
0277     bool        can_ack;
0278     bool        fully_established;
0279     bool        rcv_data_fin;
0280     bool        snd_data_fin_enable;
0281     bool        rcv_fastclose;
0282     bool        use_64bit_ack; /* Set when we received a 64-bit DSN */
0283     bool        csum_enabled;
0284     bool        allow_infinite_fallback;
0285     u8      mpc_endpoint_id;
0286     u8      recvmsg_inq:1,
0287             cork:1,
0288             nodelay:1;
0289     struct work_struct work;
0290     struct sk_buff  *ooo_last_skb;
0291     struct rb_root  out_of_order_queue;
0292     struct sk_buff_head receive_queue;
0293     struct list_head conn_list;
0294     struct list_head rtx_queue;
0295     struct mptcp_data_frag *first_pending;
0296     struct list_head join_list;
0297     struct socket   *subflow; /* outgoing connect/listener/!mp_capable */
0298     struct sock *first;
0299     struct mptcp_pm_data    pm;
0300     struct {
0301         u32 space;  /* bytes copied in last measurement window */
0302         u32 copied; /* bytes copied in this measurement window */
0303         u64 time;   /* start time of measurement window */
0304         u64 rtt_us; /* last maximum rtt of subflows */
0305     } rcvq_space;
0306 
0307     u32 setsockopt_seq;
0308     char        ca_name[TCP_CA_NAME_MAX];
0309     struct mptcp_sock   *dl_next;
0310 };
0311 
0312 #define mptcp_data_lock(sk) spin_lock_bh(&(sk)->sk_lock.slock)
0313 #define mptcp_data_unlock(sk) spin_unlock_bh(&(sk)->sk_lock.slock)
0314 
0315 #define mptcp_for_each_subflow(__msk, __subflow)            \
0316     list_for_each_entry(__subflow, &((__msk)->conn_list), node)
0317 
0318 static inline void msk_owned_by_me(const struct mptcp_sock *msk)
0319 {
0320     sock_owned_by_me((const struct sock *)msk);
0321 }
0322 
0323 static inline struct mptcp_sock *mptcp_sk(const struct sock *sk)
0324 {
0325     return (struct mptcp_sock *)sk;
0326 }
0327 
0328 /* the msk socket don't use the backlog, also account for the bulk
0329  * free memory
0330  */
0331 static inline int __mptcp_rmem(const struct sock *sk)
0332 {
0333     return atomic_read(&sk->sk_rmem_alloc) - READ_ONCE(mptcp_sk(sk)->rmem_released);
0334 }
0335 
0336 static inline int __mptcp_space(const struct sock *sk)
0337 {
0338     return tcp_win_from_space(sk, READ_ONCE(sk->sk_rcvbuf) - __mptcp_rmem(sk));
0339 }
0340 
0341 static inline struct mptcp_data_frag *mptcp_send_head(const struct sock *sk)
0342 {
0343     const struct mptcp_sock *msk = mptcp_sk(sk);
0344 
0345     return READ_ONCE(msk->first_pending);
0346 }
0347 
0348 static inline struct mptcp_data_frag *mptcp_send_next(struct sock *sk)
0349 {
0350     struct mptcp_sock *msk = mptcp_sk(sk);
0351     struct mptcp_data_frag *cur;
0352 
0353     cur = msk->first_pending;
0354     return list_is_last(&cur->list, &msk->rtx_queue) ? NULL :
0355                              list_next_entry(cur, list);
0356 }
0357 
0358 static inline struct mptcp_data_frag *mptcp_pending_tail(const struct sock *sk)
0359 {
0360     struct mptcp_sock *msk = mptcp_sk(sk);
0361 
0362     if (!msk->first_pending)
0363         return NULL;
0364 
0365     if (WARN_ON_ONCE(list_empty(&msk->rtx_queue)))
0366         return NULL;
0367 
0368     return list_last_entry(&msk->rtx_queue, struct mptcp_data_frag, list);
0369 }
0370 
0371 static inline struct mptcp_data_frag *mptcp_rtx_head(const struct sock *sk)
0372 {
0373     struct mptcp_sock *msk = mptcp_sk(sk);
0374 
0375     if (msk->snd_una == READ_ONCE(msk->snd_nxt))
0376         return NULL;
0377 
0378     return list_first_entry_or_null(&msk->rtx_queue, struct mptcp_data_frag, list);
0379 }
0380 
0381 struct csum_pseudo_header {
0382     __be64 data_seq;
0383     __be32 subflow_seq;
0384     __be16 data_len;
0385     __sum16 csum;
0386 };
0387 
0388 struct mptcp_subflow_request_sock {
0389     struct  tcp_request_sock sk;
0390     u16 mp_capable : 1,
0391         mp_join : 1,
0392         backup : 1,
0393         csum_reqd : 1,
0394         allow_join_id0 : 1;
0395     u8  local_id;
0396     u8  remote_id;
0397     u64 local_key;
0398     u64 idsn;
0399     u32 token;
0400     u32 ssn_offset;
0401     u64 thmac;
0402     u32 local_nonce;
0403     u32 remote_nonce;
0404     struct mptcp_sock   *msk;
0405     struct hlist_nulls_node token_node;
0406 };
0407 
0408 static inline struct mptcp_subflow_request_sock *
0409 mptcp_subflow_rsk(const struct request_sock *rsk)
0410 {
0411     return (struct mptcp_subflow_request_sock *)rsk;
0412 }
0413 
0414 enum mptcp_data_avail {
0415     MPTCP_SUBFLOW_NODATA,
0416     MPTCP_SUBFLOW_DATA_AVAIL,
0417 };
0418 
0419 struct mptcp_delegated_action {
0420     struct napi_struct napi;
0421     struct list_head head;
0422 };
0423 
0424 DECLARE_PER_CPU(struct mptcp_delegated_action, mptcp_delegated_actions);
0425 
0426 #define MPTCP_DELEGATE_SEND     0
0427 #define MPTCP_DELEGATE_ACK      1
0428 
0429 /* MPTCP subflow context */
0430 struct mptcp_subflow_context {
0431     struct  list_head node;/* conn_list of subflows */
0432 
0433     struct_group(reset,
0434 
0435     unsigned long avg_pacing_rate; /* protected by msk socket lock */
0436     u64 local_key;
0437     u64 remote_key;
0438     u64 idsn;
0439     u64 map_seq;
0440     u32 snd_isn;
0441     u32 token;
0442     u32 rel_write_seq;
0443     u32 map_subflow_seq;
0444     u32 ssn_offset;
0445     u32 map_data_len;
0446     __wsum  map_data_csum;
0447     u32 map_csum_len;
0448     u32 request_mptcp : 1,  /* send MP_CAPABLE */
0449         request_join : 1,   /* send MP_JOIN */
0450         request_bkup : 1,
0451         mp_capable : 1,     /* remote is MPTCP capable */
0452         mp_join : 1,        /* remote is JOINing */
0453         fully_established : 1,      /* path validated */
0454         pm_notified : 1,    /* PM hook called for established status */
0455         conn_finished : 1,
0456         map_valid : 1,
0457         map_csum_reqd : 1,
0458         map_data_fin : 1,
0459         mpc_map : 1,
0460         backup : 1,
0461         send_mp_prio : 1,
0462         send_mp_fail : 1,
0463         send_fastclose : 1,
0464         send_infinite_map : 1,
0465         rx_eof : 1,
0466         can_ack : 1,        /* only after processing the remote a key */
0467         disposable : 1,     /* ctx can be free at ulp release time */
0468         stale : 1,      /* unable to snd/rcv data, do not use for xmit */
0469         local_id_valid : 1, /* local_id is correctly initialized */
0470         valid_csum_seen : 1;        /* at least one csum validated */
0471     enum mptcp_data_avail data_avail;
0472     u32 remote_nonce;
0473     u64 thmac;
0474     u32 local_nonce;
0475     u32 remote_token;
0476     u8  hmac[MPTCPOPT_HMAC_LEN];
0477     u8  local_id;
0478     u8  remote_id;
0479     u8  reset_seen:1;
0480     u8  reset_transient:1;
0481     u8  reset_reason:4;
0482     u8  stale_count;
0483 
0484     long    delegated_status;
0485     unsigned long   fail_tout;
0486 
0487     );
0488 
0489     struct  list_head delegated_node;   /* link into delegated_action, protected by local BH */
0490 
0491     u32 setsockopt_seq;
0492     u32 stale_rcv_tstamp;
0493 
0494     struct  sock *tcp_sock;     /* tcp sk backpointer */
0495     struct  sock *conn;     /* parent mptcp_sock */
0496     const   struct inet_connection_sock_af_ops *icsk_af_ops;
0497     void    (*tcp_state_change)(struct sock *sk);
0498     void    (*tcp_error_report)(struct sock *sk);
0499 
0500     struct  rcu_head rcu;
0501 };
0502 
0503 static inline struct mptcp_subflow_context *
0504 mptcp_subflow_ctx(const struct sock *sk)
0505 {
0506     struct inet_connection_sock *icsk = inet_csk(sk);
0507 
0508     /* Use RCU on icsk_ulp_data only for sock diag code */
0509     return (__force struct mptcp_subflow_context *)icsk->icsk_ulp_data;
0510 }
0511 
0512 static inline struct sock *
0513 mptcp_subflow_tcp_sock(const struct mptcp_subflow_context *subflow)
0514 {
0515     return subflow->tcp_sock;
0516 }
0517 
0518 static inline void
0519 mptcp_subflow_ctx_reset(struct mptcp_subflow_context *subflow)
0520 {
0521     memset(&subflow->reset, 0, sizeof(subflow->reset));
0522     subflow->request_mptcp = 1;
0523 }
0524 
0525 static inline u64
0526 mptcp_subflow_get_map_offset(const struct mptcp_subflow_context *subflow)
0527 {
0528     return tcp_sk(mptcp_subflow_tcp_sock(subflow))->copied_seq -
0529               subflow->ssn_offset -
0530               subflow->map_subflow_seq;
0531 }
0532 
0533 static inline u64
0534 mptcp_subflow_get_mapped_dsn(const struct mptcp_subflow_context *subflow)
0535 {
0536     return subflow->map_seq + mptcp_subflow_get_map_offset(subflow);
0537 }
0538 
0539 void mptcp_subflow_process_delegated(struct sock *ssk);
0540 
0541 static inline void mptcp_subflow_delegate(struct mptcp_subflow_context *subflow, int action)
0542 {
0543     struct mptcp_delegated_action *delegated;
0544     bool schedule;
0545 
0546     /* the caller held the subflow bh socket lock */
0547     lockdep_assert_in_softirq();
0548 
0549     /* The implied barrier pairs with mptcp_subflow_delegated_done(), and
0550      * ensures the below list check sees list updates done prior to status
0551      * bit changes
0552      */
0553     if (!test_and_set_bit(action, &subflow->delegated_status)) {
0554         /* still on delegated list from previous scheduling */
0555         if (!list_empty(&subflow->delegated_node))
0556             return;
0557 
0558         delegated = this_cpu_ptr(&mptcp_delegated_actions);
0559         schedule = list_empty(&delegated->head);
0560         list_add_tail(&subflow->delegated_node, &delegated->head);
0561         sock_hold(mptcp_subflow_tcp_sock(subflow));
0562         if (schedule)
0563             napi_schedule(&delegated->napi);
0564     }
0565 }
0566 
0567 static inline struct mptcp_subflow_context *
0568 mptcp_subflow_delegated_next(struct mptcp_delegated_action *delegated)
0569 {
0570     struct mptcp_subflow_context *ret;
0571 
0572     if (list_empty(&delegated->head))
0573         return NULL;
0574 
0575     ret = list_first_entry(&delegated->head, struct mptcp_subflow_context, delegated_node);
0576     list_del_init(&ret->delegated_node);
0577     return ret;
0578 }
0579 
0580 static inline bool mptcp_subflow_has_delegated_action(const struct mptcp_subflow_context *subflow)
0581 {
0582     return !!READ_ONCE(subflow->delegated_status);
0583 }
0584 
0585 static inline void mptcp_subflow_delegated_done(struct mptcp_subflow_context *subflow, int action)
0586 {
0587     /* pairs with mptcp_subflow_delegate, ensures delegate_node is updated before
0588      * touching the status bit
0589      */
0590     smp_wmb();
0591     clear_bit(action, &subflow->delegated_status);
0592 }
0593 
0594 int mptcp_is_enabled(const struct net *net);
0595 unsigned int mptcp_get_add_addr_timeout(const struct net *net);
0596 int mptcp_is_checksum_enabled(const struct net *net);
0597 int mptcp_allow_join_id0(const struct net *net);
0598 unsigned int mptcp_stale_loss_cnt(const struct net *net);
0599 int mptcp_get_pm_type(const struct net *net);
0600 void mptcp_subflow_fully_established(struct mptcp_subflow_context *subflow,
0601                      struct mptcp_options_received *mp_opt);
0602 bool __mptcp_retransmit_pending_data(struct sock *sk);
0603 void mptcp_check_and_set_pending(struct sock *sk);
0604 void __mptcp_push_pending(struct sock *sk, unsigned int flags);
0605 bool mptcp_subflow_data_available(struct sock *sk);
0606 void __init mptcp_subflow_init(void);
0607 void mptcp_subflow_shutdown(struct sock *sk, struct sock *ssk, int how);
0608 void mptcp_close_ssk(struct sock *sk, struct sock *ssk,
0609              struct mptcp_subflow_context *subflow);
0610 void __mptcp_subflow_send_ack(struct sock *ssk);
0611 void mptcp_subflow_reset(struct sock *ssk);
0612 void mptcp_subflow_queue_clean(struct sock *ssk);
0613 void mptcp_sock_graft(struct sock *sk, struct socket *parent);
0614 struct socket *__mptcp_nmpc_socket(const struct mptcp_sock *msk);
0615 bool __mptcp_close(struct sock *sk, long timeout);
0616 void mptcp_cancel_work(struct sock *sk);
0617 
0618 bool mptcp_addresses_equal(const struct mptcp_addr_info *a,
0619                const struct mptcp_addr_info *b, bool use_port);
0620 
0621 /* called with sk socket lock held */
0622 int __mptcp_subflow_connect(struct sock *sk, const struct mptcp_addr_info *loc,
0623                 const struct mptcp_addr_info *remote);
0624 int mptcp_subflow_create_socket(struct sock *sk, struct socket **new_sock);
0625 void mptcp_info2sockaddr(const struct mptcp_addr_info *info,
0626              struct sockaddr_storage *addr,
0627              unsigned short family);
0628 
0629 static inline bool __tcp_can_send(const struct sock *ssk)
0630 {
0631     /* only send if our side has not closed yet */
0632     return ((1 << inet_sk_state_load(ssk)) & (TCPF_ESTABLISHED | TCPF_CLOSE_WAIT));
0633 }
0634 
0635 static inline bool __mptcp_subflow_active(struct mptcp_subflow_context *subflow)
0636 {
0637     /* can't send if JOIN hasn't completed yet (i.e. is usable for mptcp) */
0638     if (subflow->request_join && !subflow->fully_established)
0639         return false;
0640 
0641     return __tcp_can_send(mptcp_subflow_tcp_sock(subflow));
0642 }
0643 
0644 void mptcp_subflow_set_active(struct mptcp_subflow_context *subflow);
0645 
0646 bool mptcp_subflow_active(struct mptcp_subflow_context *subflow);
0647 
0648 static inline void mptcp_subflow_tcp_fallback(struct sock *sk,
0649                           struct mptcp_subflow_context *ctx)
0650 {
0651     sk->sk_data_ready = sock_def_readable;
0652     sk->sk_state_change = ctx->tcp_state_change;
0653     sk->sk_write_space = sk_stream_write_space;
0654     sk->sk_error_report = ctx->tcp_error_report;
0655 
0656     inet_csk(sk)->icsk_af_ops = ctx->icsk_af_ops;
0657 }
0658 
0659 void __init mptcp_proto_init(void);
0660 #if IS_ENABLED(CONFIG_MPTCP_IPV6)
0661 int __init mptcp_proto_v6_init(void);
0662 #endif
0663 
0664 struct sock *mptcp_sk_clone(const struct sock *sk,
0665                 const struct mptcp_options_received *mp_opt,
0666                 struct request_sock *req);
0667 void mptcp_get_options(const struct sk_buff *skb,
0668                struct mptcp_options_received *mp_opt);
0669 
0670 void mptcp_finish_connect(struct sock *sk);
0671 void __mptcp_set_connected(struct sock *sk);
0672 void mptcp_reset_timeout(struct mptcp_sock *msk, unsigned long fail_tout);
0673 static inline bool mptcp_is_fully_established(struct sock *sk)
0674 {
0675     return inet_sk_state_load(sk) == TCP_ESTABLISHED &&
0676            READ_ONCE(mptcp_sk(sk)->fully_established);
0677 }
0678 void mptcp_rcv_space_init(struct mptcp_sock *msk, const struct sock *ssk);
0679 void mptcp_data_ready(struct sock *sk, struct sock *ssk);
0680 bool mptcp_finish_join(struct sock *sk);
0681 bool mptcp_schedule_work(struct sock *sk);
0682 int mptcp_setsockopt(struct sock *sk, int level, int optname,
0683              sockptr_t optval, unsigned int optlen);
0684 int mptcp_getsockopt(struct sock *sk, int level, int optname,
0685              char __user *optval, int __user *option);
0686 
0687 u64 __mptcp_expand_seq(u64 old_seq, u64 cur_seq);
0688 static inline u64 mptcp_expand_seq(u64 old_seq, u64 cur_seq, bool use_64bit)
0689 {
0690     if (use_64bit)
0691         return cur_seq;
0692 
0693     return __mptcp_expand_seq(old_seq, cur_seq);
0694 }
0695 void __mptcp_check_push(struct sock *sk, struct sock *ssk);
0696 void __mptcp_data_acked(struct sock *sk);
0697 void __mptcp_error_report(struct sock *sk);
0698 void mptcp_subflow_eof(struct sock *sk);
0699 bool mptcp_update_rcv_data_fin(struct mptcp_sock *msk, u64 data_fin_seq, bool use_64bit);
0700 static inline bool mptcp_data_fin_enabled(const struct mptcp_sock *msk)
0701 {
0702     return READ_ONCE(msk->snd_data_fin_enable) &&
0703            READ_ONCE(msk->write_seq) == READ_ONCE(msk->snd_nxt);
0704 }
0705 
0706 static inline bool mptcp_propagate_sndbuf(struct sock *sk, struct sock *ssk)
0707 {
0708     if ((sk->sk_userlocks & SOCK_SNDBUF_LOCK) || ssk->sk_sndbuf <= READ_ONCE(sk->sk_sndbuf))
0709         return false;
0710 
0711     WRITE_ONCE(sk->sk_sndbuf, ssk->sk_sndbuf);
0712     return true;
0713 }
0714 
0715 static inline void mptcp_write_space(struct sock *sk)
0716 {
0717     if (sk_stream_is_writeable(sk)) {
0718         /* pairs with memory barrier in mptcp_poll */
0719         smp_mb();
0720         if (test_and_clear_bit(MPTCP_NOSPACE, &mptcp_sk(sk)->flags))
0721             sk_stream_write_space(sk);
0722     }
0723 }
0724 
0725 void mptcp_destroy_common(struct mptcp_sock *msk, unsigned int flags);
0726 
0727 #define MPTCP_TOKEN_MAX_RETRIES 4
0728 
0729 void __init mptcp_token_init(void);
0730 static inline void mptcp_token_init_request(struct request_sock *req)
0731 {
0732     mptcp_subflow_rsk(req)->token_node.pprev = NULL;
0733 }
0734 
0735 int mptcp_token_new_request(struct request_sock *req);
0736 void mptcp_token_destroy_request(struct request_sock *req);
0737 int mptcp_token_new_connect(struct sock *sk);
0738 void mptcp_token_accept(struct mptcp_subflow_request_sock *r,
0739             struct mptcp_sock *msk);
0740 bool mptcp_token_exists(u32 token);
0741 struct mptcp_sock *mptcp_token_get_sock(struct net *net, u32 token);
0742 struct mptcp_sock *mptcp_token_iter_next(const struct net *net, long *s_slot,
0743                      long *s_num);
0744 void mptcp_token_destroy(struct mptcp_sock *msk);
0745 
0746 void mptcp_crypto_key_sha(u64 key, u32 *token, u64 *idsn);
0747 
0748 void mptcp_crypto_hmac_sha(u64 key1, u64 key2, u8 *msg, int len, void *hmac);
0749 __sum16 __mptcp_make_csum(u64 data_seq, u32 subflow_seq, u16 data_len, __wsum sum);
0750 
0751 void __init mptcp_pm_init(void);
0752 void mptcp_pm_data_init(struct mptcp_sock *msk);
0753 void mptcp_pm_data_reset(struct mptcp_sock *msk);
0754 int mptcp_pm_parse_addr(struct nlattr *attr, struct genl_info *info,
0755             struct mptcp_addr_info *addr);
0756 int mptcp_pm_parse_entry(struct nlattr *attr, struct genl_info *info,
0757              bool require_family,
0758              struct mptcp_pm_addr_entry *entry);
0759 void mptcp_pm_subflow_chk_stale(const struct mptcp_sock *msk, struct sock *ssk);
0760 void mptcp_pm_nl_subflow_chk_stale(const struct mptcp_sock *msk, struct sock *ssk);
0761 void mptcp_pm_new_connection(struct mptcp_sock *msk, const struct sock *ssk, int server_side);
0762 void mptcp_pm_fully_established(struct mptcp_sock *msk, const struct sock *ssk, gfp_t gfp);
0763 bool mptcp_pm_allow_new_subflow(struct mptcp_sock *msk);
0764 void mptcp_pm_connection_closed(struct mptcp_sock *msk);
0765 void mptcp_pm_subflow_established(struct mptcp_sock *msk);
0766 bool mptcp_pm_nl_check_work_pending(struct mptcp_sock *msk);
0767 void mptcp_pm_subflow_check_next(struct mptcp_sock *msk, const struct sock *ssk,
0768                  const struct mptcp_subflow_context *subflow);
0769 void mptcp_pm_add_addr_received(const struct sock *ssk,
0770                 const struct mptcp_addr_info *addr);
0771 void mptcp_pm_add_addr_echoed(struct mptcp_sock *msk,
0772                   const struct mptcp_addr_info *addr);
0773 void mptcp_pm_add_addr_send_ack(struct mptcp_sock *msk);
0774 void mptcp_pm_nl_addr_send_ack(struct mptcp_sock *msk);
0775 void mptcp_pm_rm_addr_received(struct mptcp_sock *msk,
0776                    const struct mptcp_rm_list *rm_list);
0777 void mptcp_pm_mp_prio_received(struct sock *sk, u8 bkup);
0778 void mptcp_pm_mp_fail_received(struct sock *sk, u64 fail_seq);
0779 int mptcp_pm_nl_mp_prio_send_ack(struct mptcp_sock *msk,
0780                  struct mptcp_addr_info *addr,
0781                  struct mptcp_addr_info *rem,
0782                  u8 bkup);
0783 bool mptcp_pm_alloc_anno_list(struct mptcp_sock *msk,
0784                   const struct mptcp_pm_addr_entry *entry);
0785 void mptcp_pm_free_anno_list(struct mptcp_sock *msk);
0786 bool mptcp_pm_sport_in_anno_list(struct mptcp_sock *msk, const struct sock *sk);
0787 struct mptcp_pm_add_entry *
0788 mptcp_pm_del_add_timer(struct mptcp_sock *msk,
0789                const struct mptcp_addr_info *addr, bool check_id);
0790 struct mptcp_pm_add_entry *
0791 mptcp_lookup_anno_list_by_saddr(const struct mptcp_sock *msk,
0792                 const struct mptcp_addr_info *addr);
0793 int mptcp_pm_get_flags_and_ifindex_by_id(struct mptcp_sock *msk,
0794                      unsigned int id,
0795                      u8 *flags, int *ifindex);
0796 int mptcp_userspace_pm_get_flags_and_ifindex_by_id(struct mptcp_sock *msk,
0797                            unsigned int id,
0798                            u8 *flags, int *ifindex);
0799 int mptcp_userspace_pm_set_flags(struct net *net, struct nlattr *token,
0800                  struct mptcp_pm_addr_entry *loc,
0801                  struct mptcp_pm_addr_entry *rem, u8 bkup);
0802 int mptcp_pm_announce_addr(struct mptcp_sock *msk,
0803                const struct mptcp_addr_info *addr,
0804                bool echo);
0805 int mptcp_pm_remove_addr(struct mptcp_sock *msk, const struct mptcp_rm_list *rm_list);
0806 int mptcp_pm_remove_subflow(struct mptcp_sock *msk, const struct mptcp_rm_list *rm_list);
0807 void mptcp_pm_remove_addrs_and_subflows(struct mptcp_sock *msk,
0808                     struct list_head *rm_list);
0809 
0810 int mptcp_userspace_pm_append_new_local_addr(struct mptcp_sock *msk,
0811                          struct mptcp_pm_addr_entry *entry);
0812 void mptcp_free_local_addr_list(struct mptcp_sock *msk);
0813 int mptcp_nl_cmd_announce(struct sk_buff *skb, struct genl_info *info);
0814 int mptcp_nl_cmd_remove(struct sk_buff *skb, struct genl_info *info);
0815 int mptcp_nl_cmd_sf_create(struct sk_buff *skb, struct genl_info *info);
0816 int mptcp_nl_cmd_sf_destroy(struct sk_buff *skb, struct genl_info *info);
0817 
0818 void mptcp_event(enum mptcp_event_type type, const struct mptcp_sock *msk,
0819          const struct sock *ssk, gfp_t gfp);
0820 void mptcp_event_addr_announced(const struct sock *ssk, const struct mptcp_addr_info *info);
0821 void mptcp_event_addr_removed(const struct mptcp_sock *msk, u8 id);
0822 bool mptcp_userspace_pm_active(const struct mptcp_sock *msk);
0823 
0824 static inline bool mptcp_pm_should_add_signal(struct mptcp_sock *msk)
0825 {
0826     return READ_ONCE(msk->pm.addr_signal) &
0827         (BIT(MPTCP_ADD_ADDR_SIGNAL) | BIT(MPTCP_ADD_ADDR_ECHO));
0828 }
0829 
0830 static inline bool mptcp_pm_should_add_signal_addr(struct mptcp_sock *msk)
0831 {
0832     return READ_ONCE(msk->pm.addr_signal) & BIT(MPTCP_ADD_ADDR_SIGNAL);
0833 }
0834 
0835 static inline bool mptcp_pm_should_add_signal_echo(struct mptcp_sock *msk)
0836 {
0837     return READ_ONCE(msk->pm.addr_signal) & BIT(MPTCP_ADD_ADDR_ECHO);
0838 }
0839 
0840 static inline bool mptcp_pm_should_rm_signal(struct mptcp_sock *msk)
0841 {
0842     return READ_ONCE(msk->pm.addr_signal) & BIT(MPTCP_RM_ADDR_SIGNAL);
0843 }
0844 
0845 static inline bool mptcp_pm_is_userspace(const struct mptcp_sock *msk)
0846 {
0847     return READ_ONCE(msk->pm.pm_type) == MPTCP_PM_TYPE_USERSPACE;
0848 }
0849 
0850 static inline bool mptcp_pm_is_kernel(const struct mptcp_sock *msk)
0851 {
0852     return READ_ONCE(msk->pm.pm_type) == MPTCP_PM_TYPE_KERNEL;
0853 }
0854 
0855 static inline unsigned int mptcp_add_addr_len(int family, bool echo, bool port)
0856 {
0857     u8 len = TCPOLEN_MPTCP_ADD_ADDR_BASE;
0858 
0859     if (family == AF_INET6)
0860         len = TCPOLEN_MPTCP_ADD_ADDR6_BASE;
0861     if (!echo)
0862         len += MPTCPOPT_THMAC_LEN;
0863     /* account for 2 trailing 'nop' options */
0864     if (port)
0865         len += TCPOLEN_MPTCP_PORT_LEN + TCPOLEN_MPTCP_PORT_ALIGN;
0866 
0867     return len;
0868 }
0869 
0870 static inline int mptcp_rm_addr_len(const struct mptcp_rm_list *rm_list)
0871 {
0872     if (rm_list->nr == 0 || rm_list->nr > MPTCP_RM_IDS_MAX)
0873         return -EINVAL;
0874 
0875     return TCPOLEN_MPTCP_RM_ADDR_BASE + roundup(rm_list->nr - 1, 4) + 1;
0876 }
0877 
0878 bool mptcp_pm_add_addr_signal(struct mptcp_sock *msk, const struct sk_buff *skb,
0879                   unsigned int opt_size, unsigned int remaining,
0880                   struct mptcp_addr_info *addr, bool *echo,
0881                   bool *drop_other_suboptions);
0882 bool mptcp_pm_rm_addr_signal(struct mptcp_sock *msk, unsigned int remaining,
0883                  struct mptcp_rm_list *rm_list);
0884 int mptcp_pm_get_local_id(struct mptcp_sock *msk, struct sock_common *skc);
0885 int mptcp_userspace_pm_get_local_id(struct mptcp_sock *msk, struct mptcp_addr_info *skc);
0886 
0887 void __init mptcp_pm_nl_init(void);
0888 void mptcp_pm_nl_work(struct mptcp_sock *msk);
0889 void mptcp_pm_nl_rm_subflow_received(struct mptcp_sock *msk,
0890                      const struct mptcp_rm_list *rm_list);
0891 int mptcp_pm_nl_get_local_id(struct mptcp_sock *msk, struct sock_common *skc);
0892 unsigned int mptcp_pm_get_add_addr_signal_max(const struct mptcp_sock *msk);
0893 unsigned int mptcp_pm_get_add_addr_accept_max(const struct mptcp_sock *msk);
0894 unsigned int mptcp_pm_get_subflows_max(const struct mptcp_sock *msk);
0895 unsigned int mptcp_pm_get_local_addr_max(const struct mptcp_sock *msk);
0896 
0897 /* called under PM lock */
0898 static inline void __mptcp_pm_close_subflow(struct mptcp_sock *msk)
0899 {
0900     if (--msk->pm.subflows < mptcp_pm_get_subflows_max(msk))
0901         WRITE_ONCE(msk->pm.accept_subflow, true);
0902 }
0903 
0904 static inline void mptcp_pm_close_subflow(struct mptcp_sock *msk)
0905 {
0906     spin_lock_bh(&msk->pm.lock);
0907     __mptcp_pm_close_subflow(msk);
0908     spin_unlock_bh(&msk->pm.lock);
0909 }
0910 
0911 void mptcp_sockopt_sync(struct mptcp_sock *msk, struct sock *ssk);
0912 void mptcp_sockopt_sync_locked(struct mptcp_sock *msk, struct sock *ssk);
0913 
0914 static inline struct mptcp_ext *mptcp_get_ext(const struct sk_buff *skb)
0915 {
0916     return (struct mptcp_ext *)skb_ext_find(skb, SKB_EXT_MPTCP);
0917 }
0918 
0919 void mptcp_diag_subflow_init(struct tcp_ulp_ops *ops);
0920 
0921 static inline bool __mptcp_check_fallback(const struct mptcp_sock *msk)
0922 {
0923     return test_bit(MPTCP_FALLBACK_DONE, &msk->flags);
0924 }
0925 
0926 static inline bool mptcp_check_fallback(const struct sock *sk)
0927 {
0928     struct mptcp_subflow_context *subflow = mptcp_subflow_ctx(sk);
0929     struct mptcp_sock *msk = mptcp_sk(subflow->conn);
0930 
0931     return __mptcp_check_fallback(msk);
0932 }
0933 
0934 static inline void __mptcp_do_fallback(struct mptcp_sock *msk)
0935 {
0936     if (test_bit(MPTCP_FALLBACK_DONE, &msk->flags)) {
0937         pr_debug("TCP fallback already done (msk=%p)", msk);
0938         return;
0939     }
0940     set_bit(MPTCP_FALLBACK_DONE, &msk->flags);
0941 }
0942 
0943 static inline void mptcp_do_fallback(struct sock *ssk)
0944 {
0945     struct mptcp_subflow_context *subflow = mptcp_subflow_ctx(ssk);
0946     struct sock *sk = subflow->conn;
0947     struct mptcp_sock *msk;
0948 
0949     msk = mptcp_sk(sk);
0950     __mptcp_do_fallback(msk);
0951     if (READ_ONCE(msk->snd_data_fin_enable) && !(ssk->sk_shutdown & SEND_SHUTDOWN)) {
0952         gfp_t saved_allocation = ssk->sk_allocation;
0953 
0954         /* we are in a atomic (BH) scope, override ssk default for data
0955          * fin allocation
0956          */
0957         ssk->sk_allocation = GFP_ATOMIC;
0958         ssk->sk_shutdown |= SEND_SHUTDOWN;
0959         tcp_shutdown(ssk, SEND_SHUTDOWN);
0960         ssk->sk_allocation = saved_allocation;
0961     }
0962 }
0963 
0964 #define pr_fallback(a) pr_debug("%s:fallback to TCP (msk=%p)", __func__, a)
0965 
0966 static inline bool mptcp_check_infinite_map(struct sk_buff *skb)
0967 {
0968     struct mptcp_ext *mpext;
0969 
0970     mpext = skb ? mptcp_get_ext(skb) : NULL;
0971     if (mpext && mpext->infinite_map)
0972         return true;
0973 
0974     return false;
0975 }
0976 
0977 static inline bool is_active_ssk(struct mptcp_subflow_context *subflow)
0978 {
0979     return (subflow->request_mptcp || subflow->request_join);
0980 }
0981 
0982 static inline bool subflow_simultaneous_connect(struct sock *sk)
0983 {
0984     struct mptcp_subflow_context *subflow = mptcp_subflow_ctx(sk);
0985 
0986     return sk->sk_state == TCP_ESTABLISHED &&
0987            is_active_ssk(subflow) &&
0988            !subflow->conn_finished;
0989 }
0990 
0991 #ifdef CONFIG_SYN_COOKIES
0992 void subflow_init_req_cookie_join_save(const struct mptcp_subflow_request_sock *subflow_req,
0993                        struct sk_buff *skb);
0994 bool mptcp_token_join_cookie_init_state(struct mptcp_subflow_request_sock *subflow_req,
0995                     struct sk_buff *skb);
0996 void __init mptcp_join_cookie_init(void);
0997 #else
0998 static inline void
0999 subflow_init_req_cookie_join_save(const struct mptcp_subflow_request_sock *subflow_req,
1000                   struct sk_buff *skb) {}
1001 static inline bool
1002 mptcp_token_join_cookie_init_state(struct mptcp_subflow_request_sock *subflow_req,
1003                    struct sk_buff *skb)
1004 {
1005     return false;
1006 }
1007 
1008 static inline void mptcp_join_cookie_init(void) {}
1009 #endif
1010 
1011 #endif /* __MPTCP_PROTOCOL_H */
[ Source navigation ] [ Diff markup ] [ Identifier search ] [ general search ]

This page was automatically generated by the 2.1.0 LXR engine.

The LXR team
Valid CSS 2.1! Valid HTML 4.01!