net/ipv4/udp_bpf.c

0001 // SPDX-License-Identifier: GPL-2.0
0002 /* Copyright (c) 2020 Cloudflare Ltd https://cloudflare.com */
0003
0004 #include <linux/skmsg.h>
0005 #include <net/sock.h>
0006 #include <net/udp.h>
0007 #include <net/inet_common.h>
0008
0009 #include "udp_impl.h"
0010
0011 static struct proto *udpv6_prot_saved __read_mostly;
0012
0013 static int sk_udp_recvmsg(struct sock *sk, struct msghdr *msg, size_t len,
0014               int flags, int *addr_len)
0015 {
0016 #if IS_ENABLED(CONFIG_IPV6)
0017     if (sk->sk_family == AF_INET6)
0018         return udpv6_prot_saved->recvmsg(sk, msg, len, flags, addr_len);
0019 #endif
0020     return udp_prot.recvmsg(sk, msg, len, flags, addr_len);
0021 }
0022
0023 static bool udp_sk_has_data(struct sock *sk)
0024 {
0025     return !skb_queue_empty(&udp_sk(sk)->reader_queue) ||
0026            !skb_queue_empty(&sk->sk_receive_queue);
0027 }
0028
0029 static bool psock_has_data(struct sk_psock *psock)
0030 {
0031     return !skb_queue_empty(&psock->ingress_skb) ||
0032            !sk_psock_queue_empty(psock);
0033 }
0034
0035 #define udp_msg_has_data(__sk, __psock) \
0036         ({ udp_sk_has_data(__sk) || psock_has_data(__psock); })
0037
0038 static int udp_msg_wait_data(struct sock *sk, struct sk_psock *psock,
0039                  long timeo)
0040 {
0041     DEFINE_WAIT_FUNC(wait, woken_wake_function);
0042     int ret = 0;
0043
0044     if (sk->sk_shutdown & RCV_SHUTDOWN)
0045         return 1;
0046
0047     if (!timeo)
0048         return ret;
0049
0050     add_wait_queue(sk_sleep(sk), &wait);
0051     sk_set_bit(SOCKWQ_ASYNC_WAITDATA, sk);
0052     ret = udp_msg_has_data(sk, psock);
0053     if (!ret) {
0054         wait_woken(&wait, TASK_INTERRUPTIBLE, timeo);
0055         ret = udp_msg_has_data(sk, psock);
0056     }
0057     sk_clear_bit(SOCKWQ_ASYNC_WAITDATA, sk);
0058     remove_wait_queue(sk_sleep(sk), &wait);
0059     return ret;
0060 }
0061
0062 static int udp_bpf_recvmsg(struct sock *sk, struct msghdr *msg, size_t len,
0063                int flags, int *addr_len)
0064 {
0065     struct sk_psock *psock;
0066     int copied, ret;
0067
0068     if (unlikely(flags & MSG_ERRQUEUE))
0069         return inet_recv_error(sk, msg, len, addr_len);
0070
0071     psock = sk_psock_get(sk);
0072     if (unlikely(!psock))
0073         return sk_udp_recvmsg(sk, msg, len, flags, addr_len);
0074
0075     if (!psock_has_data(psock)) {
0076         ret = sk_udp_recvmsg(sk, msg, len, flags, addr_len);
0077         goto out;
0078     }
0079
0080 msg_bytes_ready:
0081     copied = sk_msg_recvmsg(sk, psock, msg, len, flags);
0082     if (!copied) {
0083         long timeo;
0084         int data;
0085
0086         timeo = sock_rcvtimeo(sk, flags & MSG_DONTWAIT);
0087         data = udp_msg_wait_data(sk, psock, timeo);
0088         if (data) {
0089             if (psock_has_data(psock))
0090                 goto msg_bytes_ready;
0091             ret = sk_udp_recvmsg(sk, msg, len, flags, addr_len);
0092             goto out;
0093         }
0094         copied = -EAGAIN;
0095     }
0096     ret = copied;
0097 out:
0098     sk_psock_put(sk, psock);
0099     return ret;
0100 }
0101
0102 enum {
0103     UDP_BPF_IPV4,
0104     UDP_BPF_IPV6,
0105     UDP_BPF_NUM_PROTS,
0106 };
0107
0108 static DEFINE_SPINLOCK(udpv6_prot_lock);
0109 static struct proto udp_bpf_prots[UDP_BPF_NUM_PROTS];
0110
0111 static void udp_bpf_rebuild_protos(struct proto *prot, const struct proto *base)
0112 {
0113     *prot        = *base;
0114     prot->close  = sock_map_close;
0115     prot->recvmsg = udp_bpf_recvmsg;
0116     prot->sock_is_readable = sk_msg_is_readable;
0117 }
0118
0119 static void udp_bpf_check_v6_needs_rebuild(struct proto *ops)
0120 {
0121     if (unlikely(ops != smp_load_acquire(&udpv6_prot_saved))) {
0122         spin_lock_bh(&udpv6_prot_lock);
0123         if (likely(ops != udpv6_prot_saved)) {
0124             udp_bpf_rebuild_protos(&udp_bpf_prots[UDP_BPF_IPV6], ops);
0125             smp_store_release(&udpv6_prot_saved, ops);
0126         }
0127         spin_unlock_bh(&udpv6_prot_lock);
0128     }
0129 }
0130
0131 static int __init udp_bpf_v4_build_proto(void)
0132 {
0133     udp_bpf_rebuild_protos(&udp_bpf_prots[UDP_BPF_IPV4], &udp_prot);
0134     return 0;
0135 }
0136 late_initcall(udp_bpf_v4_build_proto);
0137
0138 int udp_bpf_update_proto(struct sock *sk, struct sk_psock *psock, bool restore)
0139 {
0140     int family = sk->sk_family == AF_INET ? UDP_BPF_IPV4 : UDP_BPF_IPV6;
0141
0142     if (restore) {
0143         sk->sk_write_space = psock->saved_write_space;
0144         WRITE_ONCE(sk->sk_prot, psock->sk_proto);
0145         return 0;
0146     }
0147
0148     if (sk->sk_family == AF_INET6)
0149         udp_bpf_check_v6_needs_rebuild(psock->sk_proto);
0150
0151     WRITE_ONCE(sk->sk_prot, &udp_bpf_prots[family]);
0152     return 0;
0153 }
0154 EXPORT_SYMBOL_GPL(udp_bpf_update_proto);