Back to home page

OSCL-LXR
 
 

     

0001 // SPDX-License-Identifier: GPL-2.0-or-later
0002 /*
0003  *  UDP over IPv6
0004  *  Linux INET6 implementation
0005  *
0006  *  Authors:
0007  *  Pedro Roque     <roque@di.fc.ul.pt>
0008  *
0009  *  Based on linux/ipv4/udp.c
0010  *
0011  *  Fixes:
0012  *  Hideaki YOSHIFUJI   :   sin6_scope_id support
0013  *  YOSHIFUJI Hideaki @USAGI and:   Support IPV6_V6ONLY socket option, which
0014  *  Alexey Kuznetsov        allow both IPv4 and IPv6 sockets to bind
0015  *                  a single port at the same time.
0016  *      Kazunori MIYAZAWA @USAGI:       change process style to use ip6_append_data
0017  *      YOSHIFUJI Hideaki @USAGI:   convert /proc/net/udp6 to seq_file.
0018  */
0019 
0020 #include <linux/bpf-cgroup.h>
0021 #include <linux/errno.h>
0022 #include <linux/types.h>
0023 #include <linux/socket.h>
0024 #include <linux/sockios.h>
0025 #include <linux/net.h>
0026 #include <linux/in6.h>
0027 #include <linux/netdevice.h>
0028 #include <linux/if_arp.h>
0029 #include <linux/ipv6.h>
0030 #include <linux/icmpv6.h>
0031 #include <linux/init.h>
0032 #include <linux/module.h>
0033 #include <linux/skbuff.h>
0034 #include <linux/slab.h>
0035 #include <linux/uaccess.h>
0036 #include <linux/indirect_call_wrapper.h>
0037 
0038 #include <net/addrconf.h>
0039 #include <net/ndisc.h>
0040 #include <net/protocol.h>
0041 #include <net/transp_v6.h>
0042 #include <net/ip6_route.h>
0043 #include <net/raw.h>
0044 #include <net/seg6.h>
0045 #include <net/tcp_states.h>
0046 #include <net/ip6_checksum.h>
0047 #include <net/ip6_tunnel.h>
0048 #include <net/xfrm.h>
0049 #include <net/inet_hashtables.h>
0050 #include <net/inet6_hashtables.h>
0051 #include <net/busy_poll.h>
0052 #include <net/sock_reuseport.h>
0053 
0054 #include <linux/proc_fs.h>
0055 #include <linux/seq_file.h>
0056 #include <trace/events/skb.h>
0057 #include "udp_impl.h"
0058 
0059 static u32 udp6_ehashfn(const struct net *net,
0060             const struct in6_addr *laddr,
0061             const u16 lport,
0062             const struct in6_addr *faddr,
0063             const __be16 fport)
0064 {
0065     static u32 udp6_ehash_secret __read_mostly;
0066     static u32 udp_ipv6_hash_secret __read_mostly;
0067 
0068     u32 lhash, fhash;
0069 
0070     net_get_random_once(&udp6_ehash_secret,
0071                 sizeof(udp6_ehash_secret));
0072     net_get_random_once(&udp_ipv6_hash_secret,
0073                 sizeof(udp_ipv6_hash_secret));
0074 
0075     lhash = (__force u32)laddr->s6_addr32[3];
0076     fhash = __ipv6_addr_jhash(faddr, udp_ipv6_hash_secret);
0077 
0078     return __inet6_ehashfn(lhash, lport, fhash, fport,
0079                    udp_ipv6_hash_secret + net_hash_mix(net));
0080 }
0081 
0082 int udp_v6_get_port(struct sock *sk, unsigned short snum)
0083 {
0084     unsigned int hash2_nulladdr =
0085         ipv6_portaddr_hash(sock_net(sk), &in6addr_any, snum);
0086     unsigned int hash2_partial =
0087         ipv6_portaddr_hash(sock_net(sk), &sk->sk_v6_rcv_saddr, 0);
0088 
0089     /* precompute partial secondary hash */
0090     udp_sk(sk)->udp_portaddr_hash = hash2_partial;
0091     return udp_lib_get_port(sk, snum, hash2_nulladdr);
0092 }
0093 
0094 void udp_v6_rehash(struct sock *sk)
0095 {
0096     u16 new_hash = ipv6_portaddr_hash(sock_net(sk),
0097                       &sk->sk_v6_rcv_saddr,
0098                       inet_sk(sk)->inet_num);
0099 
0100     udp_lib_rehash(sk, new_hash);
0101 }
0102 
0103 static int compute_score(struct sock *sk, struct net *net,
0104              const struct in6_addr *saddr, __be16 sport,
0105              const struct in6_addr *daddr, unsigned short hnum,
0106              int dif, int sdif)
0107 {
0108     int bound_dev_if, score;
0109     struct inet_sock *inet;
0110     bool dev_match;
0111 
0112     if (!net_eq(sock_net(sk), net) ||
0113         udp_sk(sk)->udp_port_hash != hnum ||
0114         sk->sk_family != PF_INET6)
0115         return -1;
0116 
0117     if (!ipv6_addr_equal(&sk->sk_v6_rcv_saddr, daddr))
0118         return -1;
0119 
0120     score = 0;
0121     inet = inet_sk(sk);
0122 
0123     if (inet->inet_dport) {
0124         if (inet->inet_dport != sport)
0125             return -1;
0126         score++;
0127     }
0128 
0129     if (!ipv6_addr_any(&sk->sk_v6_daddr)) {
0130         if (!ipv6_addr_equal(&sk->sk_v6_daddr, saddr))
0131             return -1;
0132         score++;
0133     }
0134 
0135     bound_dev_if = READ_ONCE(sk->sk_bound_dev_if);
0136     dev_match = udp_sk_bound_dev_eq(net, bound_dev_if, dif, sdif);
0137     if (!dev_match)
0138         return -1;
0139     if (bound_dev_if)
0140         score++;
0141 
0142     if (READ_ONCE(sk->sk_incoming_cpu) == raw_smp_processor_id())
0143         score++;
0144 
0145     return score;
0146 }
0147 
0148 static struct sock *lookup_reuseport(struct net *net, struct sock *sk,
0149                      struct sk_buff *skb,
0150                      const struct in6_addr *saddr,
0151                      __be16 sport,
0152                      const struct in6_addr *daddr,
0153                      unsigned int hnum)
0154 {
0155     struct sock *reuse_sk = NULL;
0156     u32 hash;
0157 
0158     if (sk->sk_reuseport && sk->sk_state != TCP_ESTABLISHED) {
0159         hash = udp6_ehashfn(net, daddr, hnum, saddr, sport);
0160         reuse_sk = reuseport_select_sock(sk, hash, skb,
0161                          sizeof(struct udphdr));
0162     }
0163     return reuse_sk;
0164 }
0165 
0166 /* called with rcu_read_lock() */
0167 static struct sock *udp6_lib_lookup2(struct net *net,
0168         const struct in6_addr *saddr, __be16 sport,
0169         const struct in6_addr *daddr, unsigned int hnum,
0170         int dif, int sdif, struct udp_hslot *hslot2,
0171         struct sk_buff *skb)
0172 {
0173     struct sock *sk, *result;
0174     int score, badness;
0175 
0176     result = NULL;
0177     badness = -1;
0178     udp_portaddr_for_each_entry_rcu(sk, &hslot2->head) {
0179         score = compute_score(sk, net, saddr, sport,
0180                       daddr, hnum, dif, sdif);
0181         if (score > badness) {
0182             result = lookup_reuseport(net, sk, skb,
0183                           saddr, sport, daddr, hnum);
0184             /* Fall back to scoring if group has connections */
0185             if (result && !reuseport_has_conns(sk, false))
0186                 return result;
0187 
0188             result = result ? : sk;
0189             badness = score;
0190         }
0191     }
0192     return result;
0193 }
0194 
0195 static inline struct sock *udp6_lookup_run_bpf(struct net *net,
0196                            struct udp_table *udptable,
0197                            struct sk_buff *skb,
0198                            const struct in6_addr *saddr,
0199                            __be16 sport,
0200                            const struct in6_addr *daddr,
0201                            u16 hnum, const int dif)
0202 {
0203     struct sock *sk, *reuse_sk;
0204     bool no_reuseport;
0205 
0206     if (udptable != &udp_table)
0207         return NULL; /* only UDP is supported */
0208 
0209     no_reuseport = bpf_sk_lookup_run_v6(net, IPPROTO_UDP, saddr, sport,
0210                         daddr, hnum, dif, &sk);
0211     if (no_reuseport || IS_ERR_OR_NULL(sk))
0212         return sk;
0213 
0214     reuse_sk = lookup_reuseport(net, sk, skb, saddr, sport, daddr, hnum);
0215     if (reuse_sk)
0216         sk = reuse_sk;
0217     return sk;
0218 }
0219 
0220 /* rcu_read_lock() must be held */
0221 struct sock *__udp6_lib_lookup(struct net *net,
0222                    const struct in6_addr *saddr, __be16 sport,
0223                    const struct in6_addr *daddr, __be16 dport,
0224                    int dif, int sdif, struct udp_table *udptable,
0225                    struct sk_buff *skb)
0226 {
0227     unsigned short hnum = ntohs(dport);
0228     unsigned int hash2, slot2;
0229     struct udp_hslot *hslot2;
0230     struct sock *result, *sk;
0231 
0232     hash2 = ipv6_portaddr_hash(net, daddr, hnum);
0233     slot2 = hash2 & udptable->mask;
0234     hslot2 = &udptable->hash2[slot2];
0235 
0236     /* Lookup connected or non-wildcard sockets */
0237     result = udp6_lib_lookup2(net, saddr, sport,
0238                   daddr, hnum, dif, sdif,
0239                   hslot2, skb);
0240     if (!IS_ERR_OR_NULL(result) && result->sk_state == TCP_ESTABLISHED)
0241         goto done;
0242 
0243     /* Lookup redirect from BPF */
0244     if (static_branch_unlikely(&bpf_sk_lookup_enabled)) {
0245         sk = udp6_lookup_run_bpf(net, udptable, skb,
0246                      saddr, sport, daddr, hnum, dif);
0247         if (sk) {
0248             result = sk;
0249             goto done;
0250         }
0251     }
0252 
0253     /* Got non-wildcard socket or error on first lookup */
0254     if (result)
0255         goto done;
0256 
0257     /* Lookup wildcard sockets */
0258     hash2 = ipv6_portaddr_hash(net, &in6addr_any, hnum);
0259     slot2 = hash2 & udptable->mask;
0260     hslot2 = &udptable->hash2[slot2];
0261 
0262     result = udp6_lib_lookup2(net, saddr, sport,
0263                   &in6addr_any, hnum, dif, sdif,
0264                   hslot2, skb);
0265 done:
0266     if (IS_ERR(result))
0267         return NULL;
0268     return result;
0269 }
0270 EXPORT_SYMBOL_GPL(__udp6_lib_lookup);
0271 
0272 static struct sock *__udp6_lib_lookup_skb(struct sk_buff *skb,
0273                       __be16 sport, __be16 dport,
0274                       struct udp_table *udptable)
0275 {
0276     const struct ipv6hdr *iph = ipv6_hdr(skb);
0277 
0278     return __udp6_lib_lookup(dev_net(skb->dev), &iph->saddr, sport,
0279                  &iph->daddr, dport, inet6_iif(skb),
0280                  inet6_sdif(skb), udptable, skb);
0281 }
0282 
0283 struct sock *udp6_lib_lookup_skb(const struct sk_buff *skb,
0284                  __be16 sport, __be16 dport)
0285 {
0286     const struct ipv6hdr *iph = ipv6_hdr(skb);
0287 
0288     return __udp6_lib_lookup(dev_net(skb->dev), &iph->saddr, sport,
0289                  &iph->daddr, dport, inet6_iif(skb),
0290                  inet6_sdif(skb), &udp_table, NULL);
0291 }
0292 
0293 /* Must be called under rcu_read_lock().
0294  * Does increment socket refcount.
0295  */
0296 #if IS_ENABLED(CONFIG_NF_TPROXY_IPV6) || IS_ENABLED(CONFIG_NF_SOCKET_IPV6)
0297 struct sock *udp6_lib_lookup(struct net *net, const struct in6_addr *saddr, __be16 sport,
0298                  const struct in6_addr *daddr, __be16 dport, int dif)
0299 {
0300     struct sock *sk;
0301 
0302     sk =  __udp6_lib_lookup(net, saddr, sport, daddr, dport,
0303                 dif, 0, &udp_table, NULL);
0304     if (sk && !refcount_inc_not_zero(&sk->sk_refcnt))
0305         sk = NULL;
0306     return sk;
0307 }
0308 EXPORT_SYMBOL_GPL(udp6_lib_lookup);
0309 #endif
0310 
0311 /* do not use the scratch area len for jumbogram: their length execeeds the
0312  * scratch area space; note that the IP6CB flags is still in the first
0313  * cacheline, so checking for jumbograms is cheap
0314  */
0315 static int udp6_skb_len(struct sk_buff *skb)
0316 {
0317     return unlikely(inet6_is_jumbogram(skb)) ? skb->len : udp_skb_len(skb);
0318 }
0319 
0320 /*
0321  *  This should be easy, if there is something there we
0322  *  return it, otherwise we block.
0323  */
0324 
0325 int udpv6_recvmsg(struct sock *sk, struct msghdr *msg, size_t len,
0326           int flags, int *addr_len)
0327 {
0328     struct ipv6_pinfo *np = inet6_sk(sk);
0329     struct inet_sock *inet = inet_sk(sk);
0330     struct sk_buff *skb;
0331     unsigned int ulen, copied;
0332     int off, err, peeking = flags & MSG_PEEK;
0333     int is_udplite = IS_UDPLITE(sk);
0334     struct udp_mib __percpu *mib;
0335     bool checksum_valid = false;
0336     int is_udp4;
0337 
0338     if (flags & MSG_ERRQUEUE)
0339         return ipv6_recv_error(sk, msg, len, addr_len);
0340 
0341     if (np->rxpmtu && np->rxopt.bits.rxpmtu)
0342         return ipv6_recv_rxpmtu(sk, msg, len, addr_len);
0343 
0344 try_again:
0345     off = sk_peek_offset(sk, flags);
0346     skb = __skb_recv_udp(sk, flags, &off, &err);
0347     if (!skb)
0348         return err;
0349 
0350     ulen = udp6_skb_len(skb);
0351     copied = len;
0352     if (copied > ulen - off)
0353         copied = ulen - off;
0354     else if (copied < ulen)
0355         msg->msg_flags |= MSG_TRUNC;
0356 
0357     is_udp4 = (skb->protocol == htons(ETH_P_IP));
0358     mib = __UDPX_MIB(sk, is_udp4);
0359 
0360     /*
0361      * If checksum is needed at all, try to do it while copying the
0362      * data.  If the data is truncated, or if we only want a partial
0363      * coverage checksum (UDP-Lite), do it before the copy.
0364      */
0365 
0366     if (copied < ulen || peeking ||
0367         (is_udplite && UDP_SKB_CB(skb)->partial_cov)) {
0368         checksum_valid = udp_skb_csum_unnecessary(skb) ||
0369                 !__udp_lib_checksum_complete(skb);
0370         if (!checksum_valid)
0371             goto csum_copy_err;
0372     }
0373 
0374     if (checksum_valid || udp_skb_csum_unnecessary(skb)) {
0375         if (udp_skb_is_linear(skb))
0376             err = copy_linear_skb(skb, copied, off, &msg->msg_iter);
0377         else
0378             err = skb_copy_datagram_msg(skb, off, msg, copied);
0379     } else {
0380         err = skb_copy_and_csum_datagram_msg(skb, off, msg);
0381         if (err == -EINVAL)
0382             goto csum_copy_err;
0383     }
0384     if (unlikely(err)) {
0385         if (!peeking) {
0386             atomic_inc(&sk->sk_drops);
0387             SNMP_INC_STATS(mib, UDP_MIB_INERRORS);
0388         }
0389         kfree_skb(skb);
0390         return err;
0391     }
0392     if (!peeking)
0393         SNMP_INC_STATS(mib, UDP_MIB_INDATAGRAMS);
0394 
0395     sock_recv_cmsgs(msg, sk, skb);
0396 
0397     /* Copy the address. */
0398     if (msg->msg_name) {
0399         DECLARE_SOCKADDR(struct sockaddr_in6 *, sin6, msg->msg_name);
0400         sin6->sin6_family = AF_INET6;
0401         sin6->sin6_port = udp_hdr(skb)->source;
0402         sin6->sin6_flowinfo = 0;
0403 
0404         if (is_udp4) {
0405             ipv6_addr_set_v4mapped(ip_hdr(skb)->saddr,
0406                            &sin6->sin6_addr);
0407             sin6->sin6_scope_id = 0;
0408         } else {
0409             sin6->sin6_addr = ipv6_hdr(skb)->saddr;
0410             sin6->sin6_scope_id =
0411                 ipv6_iface_scope_id(&sin6->sin6_addr,
0412                             inet6_iif(skb));
0413         }
0414         *addr_len = sizeof(*sin6);
0415 
0416         BPF_CGROUP_RUN_PROG_UDP6_RECVMSG_LOCK(sk,
0417                               (struct sockaddr *)sin6);
0418     }
0419 
0420     if (udp_sk(sk)->gro_enabled)
0421         udp_cmsg_recv(msg, sk, skb);
0422 
0423     if (np->rxopt.all)
0424         ip6_datagram_recv_common_ctl(sk, msg, skb);
0425 
0426     if (is_udp4) {
0427         if (inet->cmsg_flags)
0428             ip_cmsg_recv_offset(msg, sk, skb,
0429                         sizeof(struct udphdr), off);
0430     } else {
0431         if (np->rxopt.all)
0432             ip6_datagram_recv_specific_ctl(sk, msg, skb);
0433     }
0434 
0435     err = copied;
0436     if (flags & MSG_TRUNC)
0437         err = ulen;
0438 
0439     skb_consume_udp(sk, skb, peeking ? -err : err);
0440     return err;
0441 
0442 csum_copy_err:
0443     if (!__sk_queue_drop_skb(sk, &udp_sk(sk)->reader_queue, skb, flags,
0444                  udp_skb_destructor)) {
0445         SNMP_INC_STATS(mib, UDP_MIB_CSUMERRORS);
0446         SNMP_INC_STATS(mib, UDP_MIB_INERRORS);
0447     }
0448     kfree_skb(skb);
0449 
0450     /* starting over for a new packet, but check if we need to yield */
0451     cond_resched();
0452     msg->msg_flags &= ~MSG_TRUNC;
0453     goto try_again;
0454 }
0455 
0456 DEFINE_STATIC_KEY_FALSE(udpv6_encap_needed_key);
0457 void udpv6_encap_enable(void)
0458 {
0459     static_branch_inc(&udpv6_encap_needed_key);
0460 }
0461 EXPORT_SYMBOL(udpv6_encap_enable);
0462 
0463 /* Handler for tunnels with arbitrary destination ports: no socket lookup, go
0464  * through error handlers in encapsulations looking for a match.
0465  */
0466 static int __udp6_lib_err_encap_no_sk(struct sk_buff *skb,
0467                       struct inet6_skb_parm *opt,
0468                       u8 type, u8 code, int offset, __be32 info)
0469 {
0470     int i;
0471 
0472     for (i = 0; i < MAX_IPTUN_ENCAP_OPS; i++) {
0473         int (*handler)(struct sk_buff *skb, struct inet6_skb_parm *opt,
0474                    u8 type, u8 code, int offset, __be32 info);
0475         const struct ip6_tnl_encap_ops *encap;
0476 
0477         encap = rcu_dereference(ip6tun_encaps[i]);
0478         if (!encap)
0479             continue;
0480         handler = encap->err_handler;
0481         if (handler && !handler(skb, opt, type, code, offset, info))
0482             return 0;
0483     }
0484 
0485     return -ENOENT;
0486 }
0487 
0488 /* Try to match ICMP errors to UDP tunnels by looking up a socket without
0489  * reversing source and destination port: this will match tunnels that force the
0490  * same destination port on both endpoints (e.g. VXLAN, GENEVE). Note that
0491  * lwtunnels might actually break this assumption by being configured with
0492  * different destination ports on endpoints, in this case we won't be able to
0493  * trace ICMP messages back to them.
0494  *
0495  * If this doesn't match any socket, probe tunnels with arbitrary destination
0496  * ports (e.g. FoU, GUE): there, the receiving socket is useless, as the port
0497  * we've sent packets to won't necessarily match the local destination port.
0498  *
0499  * Then ask the tunnel implementation to match the error against a valid
0500  * association.
0501  *
0502  * Return an error if we can't find a match, the socket if we need further
0503  * processing, zero otherwise.
0504  */
0505 static struct sock *__udp6_lib_err_encap(struct net *net,
0506                      const struct ipv6hdr *hdr, int offset,
0507                      struct udphdr *uh,
0508                      struct udp_table *udptable,
0509                      struct sock *sk,
0510                      struct sk_buff *skb,
0511                      struct inet6_skb_parm *opt,
0512                      u8 type, u8 code, __be32 info)
0513 {
0514     int (*lookup)(struct sock *sk, struct sk_buff *skb);
0515     int network_offset, transport_offset;
0516     struct udp_sock *up;
0517 
0518     network_offset = skb_network_offset(skb);
0519     transport_offset = skb_transport_offset(skb);
0520 
0521     /* Network header needs to point to the outer IPv6 header inside ICMP */
0522     skb_reset_network_header(skb);
0523 
0524     /* Transport header needs to point to the UDP header */
0525     skb_set_transport_header(skb, offset);
0526 
0527     if (sk) {
0528         up = udp_sk(sk);
0529 
0530         lookup = READ_ONCE(up->encap_err_lookup);
0531         if (lookup && lookup(sk, skb))
0532             sk = NULL;
0533 
0534         goto out;
0535     }
0536 
0537     sk = __udp6_lib_lookup(net, &hdr->daddr, uh->source,
0538                    &hdr->saddr, uh->dest,
0539                    inet6_iif(skb), 0, udptable, skb);
0540     if (sk) {
0541         up = udp_sk(sk);
0542 
0543         lookup = READ_ONCE(up->encap_err_lookup);
0544         if (!lookup || lookup(sk, skb))
0545             sk = NULL;
0546     }
0547 
0548 out:
0549     if (!sk) {
0550         sk = ERR_PTR(__udp6_lib_err_encap_no_sk(skb, opt, type, code,
0551                             offset, info));
0552     }
0553 
0554     skb_set_transport_header(skb, transport_offset);
0555     skb_set_network_header(skb, network_offset);
0556 
0557     return sk;
0558 }
0559 
0560 int __udp6_lib_err(struct sk_buff *skb, struct inet6_skb_parm *opt,
0561            u8 type, u8 code, int offset, __be32 info,
0562            struct udp_table *udptable)
0563 {
0564     struct ipv6_pinfo *np;
0565     const struct ipv6hdr *hdr = (const struct ipv6hdr *)skb->data;
0566     const struct in6_addr *saddr = &hdr->saddr;
0567     const struct in6_addr *daddr = seg6_get_daddr(skb, opt) ? : &hdr->daddr;
0568     struct udphdr *uh = (struct udphdr *)(skb->data+offset);
0569     bool tunnel = false;
0570     struct sock *sk;
0571     int harderr;
0572     int err;
0573     struct net *net = dev_net(skb->dev);
0574 
0575     sk = __udp6_lib_lookup(net, daddr, uh->dest, saddr, uh->source,
0576                    inet6_iif(skb), inet6_sdif(skb), udptable, NULL);
0577 
0578     if (!sk || udp_sk(sk)->encap_type) {
0579         /* No socket for error: try tunnels before discarding */
0580         if (static_branch_unlikely(&udpv6_encap_needed_key)) {
0581             sk = __udp6_lib_err_encap(net, hdr, offset, uh,
0582                           udptable, sk, skb,
0583                           opt, type, code, info);
0584             if (!sk)
0585                 return 0;
0586         } else
0587             sk = ERR_PTR(-ENOENT);
0588 
0589         if (IS_ERR(sk)) {
0590             __ICMP6_INC_STATS(net, __in6_dev_get(skb->dev),
0591                       ICMP6_MIB_INERRORS);
0592             return PTR_ERR(sk);
0593         }
0594 
0595         tunnel = true;
0596     }
0597 
0598     harderr = icmpv6_err_convert(type, code, &err);
0599     np = inet6_sk(sk);
0600 
0601     if (type == ICMPV6_PKT_TOOBIG) {
0602         if (!ip6_sk_accept_pmtu(sk))
0603             goto out;
0604         ip6_sk_update_pmtu(skb, sk, info);
0605         if (np->pmtudisc != IPV6_PMTUDISC_DONT)
0606             harderr = 1;
0607     }
0608     if (type == NDISC_REDIRECT) {
0609         if (tunnel) {
0610             ip6_redirect(skb, sock_net(sk), inet6_iif(skb),
0611                      sk->sk_mark, sk->sk_uid);
0612         } else {
0613             ip6_sk_redirect(skb, sk);
0614         }
0615         goto out;
0616     }
0617 
0618     /* Tunnels don't have an application socket: don't pass errors back */
0619     if (tunnel) {
0620         if (udp_sk(sk)->encap_err_rcv)
0621             udp_sk(sk)->encap_err_rcv(sk, skb, offset);
0622         goto out;
0623     }
0624 
0625     if (!np->recverr) {
0626         if (!harderr || sk->sk_state != TCP_ESTABLISHED)
0627             goto out;
0628     } else {
0629         ipv6_icmp_error(sk, skb, err, uh->dest, ntohl(info), (u8 *)(uh+1));
0630     }
0631 
0632     sk->sk_err = err;
0633     sk_error_report(sk);
0634 out:
0635     return 0;
0636 }
0637 
0638 static int __udpv6_queue_rcv_skb(struct sock *sk, struct sk_buff *skb)
0639 {
0640     int rc;
0641 
0642     if (!ipv6_addr_any(&sk->sk_v6_daddr)) {
0643         sock_rps_save_rxhash(sk, skb);
0644         sk_mark_napi_id(sk, skb);
0645         sk_incoming_cpu_update(sk);
0646     } else {
0647         sk_mark_napi_id_once(sk, skb);
0648     }
0649 
0650     rc = __udp_enqueue_schedule_skb(sk, skb);
0651     if (rc < 0) {
0652         int is_udplite = IS_UDPLITE(sk);
0653 
0654         /* Note that an ENOMEM error is charged twice */
0655         if (rc == -ENOMEM)
0656             UDP6_INC_STATS(sock_net(sk),
0657                      UDP_MIB_RCVBUFERRORS, is_udplite);
0658         else
0659             UDP6_INC_STATS(sock_net(sk),
0660                        UDP_MIB_MEMERRORS, is_udplite);
0661         UDP6_INC_STATS(sock_net(sk), UDP_MIB_INERRORS, is_udplite);
0662         kfree_skb(skb);
0663         return -1;
0664     }
0665 
0666     return 0;
0667 }
0668 
0669 static __inline__ int udpv6_err(struct sk_buff *skb,
0670                 struct inet6_skb_parm *opt, u8 type,
0671                 u8 code, int offset, __be32 info)
0672 {
0673     return __udp6_lib_err(skb, opt, type, code, offset, info, &udp_table);
0674 }
0675 
0676 static int udpv6_queue_rcv_one_skb(struct sock *sk, struct sk_buff *skb)
0677 {
0678     struct udp_sock *up = udp_sk(sk);
0679     int is_udplite = IS_UDPLITE(sk);
0680 
0681     if (!xfrm6_policy_check(sk, XFRM_POLICY_IN, skb))
0682         goto drop;
0683 
0684     if (static_branch_unlikely(&udpv6_encap_needed_key) && up->encap_type) {
0685         int (*encap_rcv)(struct sock *sk, struct sk_buff *skb);
0686 
0687         /*
0688          * This is an encapsulation socket so pass the skb to
0689          * the socket's udp_encap_rcv() hook. Otherwise, just
0690          * fall through and pass this up the UDP socket.
0691          * up->encap_rcv() returns the following value:
0692          * =0 if skb was successfully passed to the encap
0693          *    handler or was discarded by it.
0694          * >0 if skb should be passed on to UDP.
0695          * <0 if skb should be resubmitted as proto -N
0696          */
0697 
0698         /* if we're overly short, let UDP handle it */
0699         encap_rcv = READ_ONCE(up->encap_rcv);
0700         if (encap_rcv) {
0701             int ret;
0702 
0703             /* Verify checksum before giving to encap */
0704             if (udp_lib_checksum_complete(skb))
0705                 goto csum_error;
0706 
0707             ret = encap_rcv(sk, skb);
0708             if (ret <= 0) {
0709                 __UDP6_INC_STATS(sock_net(sk),
0710                          UDP_MIB_INDATAGRAMS,
0711                          is_udplite);
0712                 return -ret;
0713             }
0714         }
0715 
0716         /* FALLTHROUGH -- it's a UDP Packet */
0717     }
0718 
0719     /*
0720      * UDP-Lite specific tests, ignored on UDP sockets (see net/ipv4/udp.c).
0721      */
0722     if ((up->pcflag & UDPLITE_RECV_CC)  &&  UDP_SKB_CB(skb)->partial_cov) {
0723 
0724         if (up->pcrlen == 0) {          /* full coverage was set  */
0725             net_dbg_ratelimited("UDPLITE6: partial coverage %d while full coverage %d requested\n",
0726                         UDP_SKB_CB(skb)->cscov, skb->len);
0727             goto drop;
0728         }
0729         if (UDP_SKB_CB(skb)->cscov  <  up->pcrlen) {
0730             net_dbg_ratelimited("UDPLITE6: coverage %d too small, need min %d\n",
0731                         UDP_SKB_CB(skb)->cscov, up->pcrlen);
0732             goto drop;
0733         }
0734     }
0735 
0736     prefetch(&sk->sk_rmem_alloc);
0737     if (rcu_access_pointer(sk->sk_filter) &&
0738         udp_lib_checksum_complete(skb))
0739         goto csum_error;
0740 
0741     if (sk_filter_trim_cap(sk, skb, sizeof(struct udphdr)))
0742         goto drop;
0743 
0744     udp_csum_pull_header(skb);
0745 
0746     skb_dst_drop(skb);
0747 
0748     return __udpv6_queue_rcv_skb(sk, skb);
0749 
0750 csum_error:
0751     __UDP6_INC_STATS(sock_net(sk), UDP_MIB_CSUMERRORS, is_udplite);
0752 drop:
0753     __UDP6_INC_STATS(sock_net(sk), UDP_MIB_INERRORS, is_udplite);
0754     atomic_inc(&sk->sk_drops);
0755     kfree_skb(skb);
0756     return -1;
0757 }
0758 
0759 static int udpv6_queue_rcv_skb(struct sock *sk, struct sk_buff *skb)
0760 {
0761     struct sk_buff *next, *segs;
0762     int ret;
0763 
0764     if (likely(!udp_unexpected_gso(sk, skb)))
0765         return udpv6_queue_rcv_one_skb(sk, skb);
0766 
0767     __skb_push(skb, -skb_mac_offset(skb));
0768     segs = udp_rcv_segment(sk, skb, false);
0769     skb_list_walk_safe(segs, skb, next) {
0770         __skb_pull(skb, skb_transport_offset(skb));
0771 
0772         udp_post_segment_fix_csum(skb);
0773         ret = udpv6_queue_rcv_one_skb(sk, skb);
0774         if (ret > 0)
0775             ip6_protocol_deliver_rcu(dev_net(skb->dev), skb, ret,
0776                          true);
0777     }
0778     return 0;
0779 }
0780 
0781 static bool __udp_v6_is_mcast_sock(struct net *net, struct sock *sk,
0782                    __be16 loc_port, const struct in6_addr *loc_addr,
0783                    __be16 rmt_port, const struct in6_addr *rmt_addr,
0784                    int dif, int sdif, unsigned short hnum)
0785 {
0786     struct inet_sock *inet = inet_sk(sk);
0787 
0788     if (!net_eq(sock_net(sk), net))
0789         return false;
0790 
0791     if (udp_sk(sk)->udp_port_hash != hnum ||
0792         sk->sk_family != PF_INET6 ||
0793         (inet->inet_dport && inet->inet_dport != rmt_port) ||
0794         (!ipv6_addr_any(&sk->sk_v6_daddr) &&
0795             !ipv6_addr_equal(&sk->sk_v6_daddr, rmt_addr)) ||
0796         !udp_sk_bound_dev_eq(net, READ_ONCE(sk->sk_bound_dev_if), dif, sdif) ||
0797         (!ipv6_addr_any(&sk->sk_v6_rcv_saddr) &&
0798             !ipv6_addr_equal(&sk->sk_v6_rcv_saddr, loc_addr)))
0799         return false;
0800     if (!inet6_mc_check(sk, loc_addr, rmt_addr))
0801         return false;
0802     return true;
0803 }
0804 
0805 static void udp6_csum_zero_error(struct sk_buff *skb)
0806 {
0807     /* RFC 2460 section 8.1 says that we SHOULD log
0808      * this error. Well, it is reasonable.
0809      */
0810     net_dbg_ratelimited("IPv6: udp checksum is 0 for [%pI6c]:%u->[%pI6c]:%u\n",
0811                 &ipv6_hdr(skb)->saddr, ntohs(udp_hdr(skb)->source),
0812                 &ipv6_hdr(skb)->daddr, ntohs(udp_hdr(skb)->dest));
0813 }
0814 
0815 /*
0816  * Note: called only from the BH handler context,
0817  * so we don't need to lock the hashes.
0818  */
0819 static int __udp6_lib_mcast_deliver(struct net *net, struct sk_buff *skb,
0820         const struct in6_addr *saddr, const struct in6_addr *daddr,
0821         struct udp_table *udptable, int proto)
0822 {
0823     struct sock *sk, *first = NULL;
0824     const struct udphdr *uh = udp_hdr(skb);
0825     unsigned short hnum = ntohs(uh->dest);
0826     struct udp_hslot *hslot = udp_hashslot(udptable, net, hnum);
0827     unsigned int offset = offsetof(typeof(*sk), sk_node);
0828     unsigned int hash2 = 0, hash2_any = 0, use_hash2 = (hslot->count > 10);
0829     int dif = inet6_iif(skb);
0830     int sdif = inet6_sdif(skb);
0831     struct hlist_node *node;
0832     struct sk_buff *nskb;
0833 
0834     if (use_hash2) {
0835         hash2_any = ipv6_portaddr_hash(net, &in6addr_any, hnum) &
0836                 udptable->mask;
0837         hash2 = ipv6_portaddr_hash(net, daddr, hnum) & udptable->mask;
0838 start_lookup:
0839         hslot = &udptable->hash2[hash2];
0840         offset = offsetof(typeof(*sk), __sk_common.skc_portaddr_node);
0841     }
0842 
0843     sk_for_each_entry_offset_rcu(sk, node, &hslot->head, offset) {
0844         if (!__udp_v6_is_mcast_sock(net, sk, uh->dest, daddr,
0845                         uh->source, saddr, dif, sdif,
0846                         hnum))
0847             continue;
0848         /* If zero checksum and no_check is not on for
0849          * the socket then skip it.
0850          */
0851         if (!uh->check && !udp_sk(sk)->no_check6_rx)
0852             continue;
0853         if (!first) {
0854             first = sk;
0855             continue;
0856         }
0857         nskb = skb_clone(skb, GFP_ATOMIC);
0858         if (unlikely(!nskb)) {
0859             atomic_inc(&sk->sk_drops);
0860             __UDP6_INC_STATS(net, UDP_MIB_RCVBUFERRORS,
0861                      IS_UDPLITE(sk));
0862             __UDP6_INC_STATS(net, UDP_MIB_INERRORS,
0863                      IS_UDPLITE(sk));
0864             continue;
0865         }
0866 
0867         if (udpv6_queue_rcv_skb(sk, nskb) > 0)
0868             consume_skb(nskb);
0869     }
0870 
0871     /* Also lookup *:port if we are using hash2 and haven't done so yet. */
0872     if (use_hash2 && hash2 != hash2_any) {
0873         hash2 = hash2_any;
0874         goto start_lookup;
0875     }
0876 
0877     if (first) {
0878         if (udpv6_queue_rcv_skb(first, skb) > 0)
0879             consume_skb(skb);
0880     } else {
0881         kfree_skb(skb);
0882         __UDP6_INC_STATS(net, UDP_MIB_IGNOREDMULTI,
0883                  proto == IPPROTO_UDPLITE);
0884     }
0885     return 0;
0886 }
0887 
0888 static void udp6_sk_rx_dst_set(struct sock *sk, struct dst_entry *dst)
0889 {
0890     if (udp_sk_rx_dst_set(sk, dst)) {
0891         const struct rt6_info *rt = (const struct rt6_info *)dst;
0892 
0893         sk->sk_rx_dst_cookie = rt6_get_cookie(rt);
0894     }
0895 }
0896 
0897 /* wrapper for udp_queue_rcv_skb tacking care of csum conversion and
0898  * return code conversion for ip layer consumption
0899  */
0900 static int udp6_unicast_rcv_skb(struct sock *sk, struct sk_buff *skb,
0901                 struct udphdr *uh)
0902 {
0903     int ret;
0904 
0905     if (inet_get_convert_csum(sk) && uh->check && !IS_UDPLITE(sk))
0906         skb_checksum_try_convert(skb, IPPROTO_UDP, ip6_compute_pseudo);
0907 
0908     ret = udpv6_queue_rcv_skb(sk, skb);
0909 
0910     /* a return value > 0 means to resubmit the input */
0911     if (ret > 0)
0912         return ret;
0913     return 0;
0914 }
0915 
0916 int __udp6_lib_rcv(struct sk_buff *skb, struct udp_table *udptable,
0917            int proto)
0918 {
0919     enum skb_drop_reason reason = SKB_DROP_REASON_NOT_SPECIFIED;
0920     const struct in6_addr *saddr, *daddr;
0921     struct net *net = dev_net(skb->dev);
0922     struct udphdr *uh;
0923     struct sock *sk;
0924     bool refcounted;
0925     u32 ulen = 0;
0926 
0927     if (!pskb_may_pull(skb, sizeof(struct udphdr)))
0928         goto discard;
0929 
0930     saddr = &ipv6_hdr(skb)->saddr;
0931     daddr = &ipv6_hdr(skb)->daddr;
0932     uh = udp_hdr(skb);
0933 
0934     ulen = ntohs(uh->len);
0935     if (ulen > skb->len)
0936         goto short_packet;
0937 
0938     if (proto == IPPROTO_UDP) {
0939         /* UDP validates ulen. */
0940 
0941         /* Check for jumbo payload */
0942         if (ulen == 0)
0943             ulen = skb->len;
0944 
0945         if (ulen < sizeof(*uh))
0946             goto short_packet;
0947 
0948         if (ulen < skb->len) {
0949             if (pskb_trim_rcsum(skb, ulen))
0950                 goto short_packet;
0951             saddr = &ipv6_hdr(skb)->saddr;
0952             daddr = &ipv6_hdr(skb)->daddr;
0953             uh = udp_hdr(skb);
0954         }
0955     }
0956 
0957     if (udp6_csum_init(skb, uh, proto))
0958         goto csum_error;
0959 
0960     /* Check if the socket is already available, e.g. due to early demux */
0961     sk = skb_steal_sock(skb, &refcounted);
0962     if (sk) {
0963         struct dst_entry *dst = skb_dst(skb);
0964         int ret;
0965 
0966         if (unlikely(rcu_dereference(sk->sk_rx_dst) != dst))
0967             udp6_sk_rx_dst_set(sk, dst);
0968 
0969         if (!uh->check && !udp_sk(sk)->no_check6_rx) {
0970             if (refcounted)
0971                 sock_put(sk);
0972             goto report_csum_error;
0973         }
0974 
0975         ret = udp6_unicast_rcv_skb(sk, skb, uh);
0976         if (refcounted)
0977             sock_put(sk);
0978         return ret;
0979     }
0980 
0981     /*
0982      *  Multicast receive code
0983      */
0984     if (ipv6_addr_is_multicast(daddr))
0985         return __udp6_lib_mcast_deliver(net, skb,
0986                 saddr, daddr, udptable, proto);
0987 
0988     /* Unicast */
0989     sk = __udp6_lib_lookup_skb(skb, uh->source, uh->dest, udptable);
0990     if (sk) {
0991         if (!uh->check && !udp_sk(sk)->no_check6_rx)
0992             goto report_csum_error;
0993         return udp6_unicast_rcv_skb(sk, skb, uh);
0994     }
0995 
0996     reason = SKB_DROP_REASON_NO_SOCKET;
0997 
0998     if (!uh->check)
0999         goto report_csum_error;
1000 
1001     if (!xfrm6_policy_check(NULL, XFRM_POLICY_IN, skb))
1002         goto discard;
1003 
1004     if (udp_lib_checksum_complete(skb))
1005         goto csum_error;
1006 
1007     __UDP6_INC_STATS(net, UDP_MIB_NOPORTS, proto == IPPROTO_UDPLITE);
1008     icmpv6_send(skb, ICMPV6_DEST_UNREACH, ICMPV6_PORT_UNREACH, 0);
1009 
1010     kfree_skb_reason(skb, reason);
1011     return 0;
1012 
1013 short_packet:
1014     if (reason == SKB_DROP_REASON_NOT_SPECIFIED)
1015         reason = SKB_DROP_REASON_PKT_TOO_SMALL;
1016     net_dbg_ratelimited("UDP%sv6: short packet: From [%pI6c]:%u %d/%d to [%pI6c]:%u\n",
1017                 proto == IPPROTO_UDPLITE ? "-Lite" : "",
1018                 saddr, ntohs(uh->source),
1019                 ulen, skb->len,
1020                 daddr, ntohs(uh->dest));
1021     goto discard;
1022 
1023 report_csum_error:
1024     udp6_csum_zero_error(skb);
1025 csum_error:
1026     if (reason == SKB_DROP_REASON_NOT_SPECIFIED)
1027         reason = SKB_DROP_REASON_UDP_CSUM;
1028     __UDP6_INC_STATS(net, UDP_MIB_CSUMERRORS, proto == IPPROTO_UDPLITE);
1029 discard:
1030     __UDP6_INC_STATS(net, UDP_MIB_INERRORS, proto == IPPROTO_UDPLITE);
1031     kfree_skb_reason(skb, reason);
1032     return 0;
1033 }
1034 
1035 
1036 static struct sock *__udp6_lib_demux_lookup(struct net *net,
1037             __be16 loc_port, const struct in6_addr *loc_addr,
1038             __be16 rmt_port, const struct in6_addr *rmt_addr,
1039             int dif, int sdif)
1040 {
1041     unsigned short hnum = ntohs(loc_port);
1042     unsigned int hash2 = ipv6_portaddr_hash(net, loc_addr, hnum);
1043     unsigned int slot2 = hash2 & udp_table.mask;
1044     struct udp_hslot *hslot2 = &udp_table.hash2[slot2];
1045     const __portpair ports = INET_COMBINED_PORTS(rmt_port, hnum);
1046     struct sock *sk;
1047 
1048     udp_portaddr_for_each_entry_rcu(sk, &hslot2->head) {
1049         if (sk->sk_state == TCP_ESTABLISHED &&
1050             inet6_match(net, sk, rmt_addr, loc_addr, ports, dif, sdif))
1051             return sk;
1052         /* Only check first socket in chain */
1053         break;
1054     }
1055     return NULL;
1056 }
1057 
1058 void udp_v6_early_demux(struct sk_buff *skb)
1059 {
1060     struct net *net = dev_net(skb->dev);
1061     const struct udphdr *uh;
1062     struct sock *sk;
1063     struct dst_entry *dst;
1064     int dif = skb->dev->ifindex;
1065     int sdif = inet6_sdif(skb);
1066 
1067     if (!pskb_may_pull(skb, skb_transport_offset(skb) +
1068         sizeof(struct udphdr)))
1069         return;
1070 
1071     uh = udp_hdr(skb);
1072 
1073     if (skb->pkt_type == PACKET_HOST)
1074         sk = __udp6_lib_demux_lookup(net, uh->dest,
1075                          &ipv6_hdr(skb)->daddr,
1076                          uh->source, &ipv6_hdr(skb)->saddr,
1077                          dif, sdif);
1078     else
1079         return;
1080 
1081     if (!sk || !refcount_inc_not_zero(&sk->sk_refcnt))
1082         return;
1083 
1084     skb->sk = sk;
1085     skb->destructor = sock_efree;
1086     dst = rcu_dereference(sk->sk_rx_dst);
1087 
1088     if (dst)
1089         dst = dst_check(dst, sk->sk_rx_dst_cookie);
1090     if (dst) {
1091         /* set noref for now.
1092          * any place which wants to hold dst has to call
1093          * dst_hold_safe()
1094          */
1095         skb_dst_set_noref(skb, dst);
1096     }
1097 }
1098 
1099 INDIRECT_CALLABLE_SCOPE int udpv6_rcv(struct sk_buff *skb)
1100 {
1101     return __udp6_lib_rcv(skb, &udp_table, IPPROTO_UDP);
1102 }
1103 
1104 /*
1105  * Throw away all pending data and cancel the corking. Socket is locked.
1106  */
1107 static void udp_v6_flush_pending_frames(struct sock *sk)
1108 {
1109     struct udp_sock *up = udp_sk(sk);
1110 
1111     if (up->pending == AF_INET)
1112         udp_flush_pending_frames(sk);
1113     else if (up->pending) {
1114         up->len = 0;
1115         up->pending = 0;
1116         ip6_flush_pending_frames(sk);
1117     }
1118 }
1119 
1120 static int udpv6_pre_connect(struct sock *sk, struct sockaddr *uaddr,
1121                  int addr_len)
1122 {
1123     if (addr_len < offsetofend(struct sockaddr, sa_family))
1124         return -EINVAL;
1125     /* The following checks are replicated from __ip6_datagram_connect()
1126      * and intended to prevent BPF program called below from accessing
1127      * bytes that are out of the bound specified by user in addr_len.
1128      */
1129     if (uaddr->sa_family == AF_INET) {
1130         if (ipv6_only_sock(sk))
1131             return -EAFNOSUPPORT;
1132         return udp_pre_connect(sk, uaddr, addr_len);
1133     }
1134 
1135     if (addr_len < SIN6_LEN_RFC2133)
1136         return -EINVAL;
1137 
1138     return BPF_CGROUP_RUN_PROG_INET6_CONNECT_LOCK(sk, uaddr);
1139 }
1140 
1141 /**
1142  *  udp6_hwcsum_outgoing  -  handle outgoing HW checksumming
1143  *  @sk:    socket we are sending on
1144  *  @skb:   sk_buff containing the filled-in UDP header
1145  *      (checksum field must be zeroed out)
1146  *  @saddr: source address
1147  *  @daddr: destination address
1148  *  @len:   length of packet
1149  */
1150 static void udp6_hwcsum_outgoing(struct sock *sk, struct sk_buff *skb,
1151                  const struct in6_addr *saddr,
1152                  const struct in6_addr *daddr, int len)
1153 {
1154     unsigned int offset;
1155     struct udphdr *uh = udp_hdr(skb);
1156     struct sk_buff *frags = skb_shinfo(skb)->frag_list;
1157     __wsum csum = 0;
1158 
1159     if (!frags) {
1160         /* Only one fragment on the socket.  */
1161         skb->csum_start = skb_transport_header(skb) - skb->head;
1162         skb->csum_offset = offsetof(struct udphdr, check);
1163         uh->check = ~csum_ipv6_magic(saddr, daddr, len, IPPROTO_UDP, 0);
1164     } else {
1165         /*
1166          * HW-checksum won't work as there are two or more
1167          * fragments on the socket so that all csums of sk_buffs
1168          * should be together
1169          */
1170         offset = skb_transport_offset(skb);
1171         skb->csum = skb_checksum(skb, offset, skb->len - offset, 0);
1172         csum = skb->csum;
1173 
1174         skb->ip_summed = CHECKSUM_NONE;
1175 
1176         do {
1177             csum = csum_add(csum, frags->csum);
1178         } while ((frags = frags->next));
1179 
1180         uh->check = csum_ipv6_magic(saddr, daddr, len, IPPROTO_UDP,
1181                         csum);
1182         if (uh->check == 0)
1183             uh->check = CSUM_MANGLED_0;
1184     }
1185 }
1186 
1187 /*
1188  *  Sending
1189  */
1190 
1191 static int udp_v6_send_skb(struct sk_buff *skb, struct flowi6 *fl6,
1192                struct inet_cork *cork)
1193 {
1194     struct sock *sk = skb->sk;
1195     struct udphdr *uh;
1196     int err = 0;
1197     int is_udplite = IS_UDPLITE(sk);
1198     __wsum csum = 0;
1199     int offset = skb_transport_offset(skb);
1200     int len = skb->len - offset;
1201     int datalen = len - sizeof(*uh);
1202 
1203     /*
1204      * Create a UDP header
1205      */
1206     uh = udp_hdr(skb);
1207     uh->source = fl6->fl6_sport;
1208     uh->dest = fl6->fl6_dport;
1209     uh->len = htons(len);
1210     uh->check = 0;
1211 
1212     if (cork->gso_size) {
1213         const int hlen = skb_network_header_len(skb) +
1214                  sizeof(struct udphdr);
1215 
1216         if (hlen + cork->gso_size > cork->fragsize) {
1217             kfree_skb(skb);
1218             return -EINVAL;
1219         }
1220         if (datalen > cork->gso_size * UDP_MAX_SEGMENTS) {
1221             kfree_skb(skb);
1222             return -EINVAL;
1223         }
1224         if (udp_sk(sk)->no_check6_tx) {
1225             kfree_skb(skb);
1226             return -EINVAL;
1227         }
1228         if (skb->ip_summed != CHECKSUM_PARTIAL || is_udplite ||
1229             dst_xfrm(skb_dst(skb))) {
1230             kfree_skb(skb);
1231             return -EIO;
1232         }
1233 
1234         if (datalen > cork->gso_size) {
1235             skb_shinfo(skb)->gso_size = cork->gso_size;
1236             skb_shinfo(skb)->gso_type = SKB_GSO_UDP_L4;
1237             skb_shinfo(skb)->gso_segs = DIV_ROUND_UP(datalen,
1238                                  cork->gso_size);
1239         }
1240         goto csum_partial;
1241     }
1242 
1243     if (is_udplite)
1244         csum = udplite_csum(skb);
1245     else if (udp_sk(sk)->no_check6_tx) {   /* UDP csum disabled */
1246         skb->ip_summed = CHECKSUM_NONE;
1247         goto send;
1248     } else if (skb->ip_summed == CHECKSUM_PARTIAL) { /* UDP hardware csum */
1249 csum_partial:
1250         udp6_hwcsum_outgoing(sk, skb, &fl6->saddr, &fl6->daddr, len);
1251         goto send;
1252     } else
1253         csum = udp_csum(skb);
1254 
1255     /* add protocol-dependent pseudo-header */
1256     uh->check = csum_ipv6_magic(&fl6->saddr, &fl6->daddr,
1257                     len, fl6->flowi6_proto, csum);
1258     if (uh->check == 0)
1259         uh->check = CSUM_MANGLED_0;
1260 
1261 send:
1262     err = ip6_send_skb(skb);
1263     if (err) {
1264         if (err == -ENOBUFS && !inet6_sk(sk)->recverr) {
1265             UDP6_INC_STATS(sock_net(sk),
1266                        UDP_MIB_SNDBUFERRORS, is_udplite);
1267             err = 0;
1268         }
1269     } else {
1270         UDP6_INC_STATS(sock_net(sk),
1271                    UDP_MIB_OUTDATAGRAMS, is_udplite);
1272     }
1273     return err;
1274 }
1275 
1276 static int udp_v6_push_pending_frames(struct sock *sk)
1277 {
1278     struct sk_buff *skb;
1279     struct udp_sock  *up = udp_sk(sk);
1280     int err = 0;
1281 
1282     if (up->pending == AF_INET)
1283         return udp_push_pending_frames(sk);
1284 
1285     skb = ip6_finish_skb(sk);
1286     if (!skb)
1287         goto out;
1288 
1289     err = udp_v6_send_skb(skb, &inet_sk(sk)->cork.fl.u.ip6,
1290                   &inet_sk(sk)->cork.base);
1291 out:
1292     up->len = 0;
1293     up->pending = 0;
1294     return err;
1295 }
1296 
1297 int udpv6_sendmsg(struct sock *sk, struct msghdr *msg, size_t len)
1298 {
1299     struct ipv6_txoptions opt_space;
1300     struct udp_sock *up = udp_sk(sk);
1301     struct inet_sock *inet = inet_sk(sk);
1302     struct ipv6_pinfo *np = inet6_sk(sk);
1303     DECLARE_SOCKADDR(struct sockaddr_in6 *, sin6, msg->msg_name);
1304     struct in6_addr *daddr, *final_p, final;
1305     struct ipv6_txoptions *opt = NULL;
1306     struct ipv6_txoptions *opt_to_free = NULL;
1307     struct ip6_flowlabel *flowlabel = NULL;
1308     struct inet_cork_full cork;
1309     struct flowi6 *fl6 = &cork.fl.u.ip6;
1310     struct dst_entry *dst;
1311     struct ipcm6_cookie ipc6;
1312     int addr_len = msg->msg_namelen;
1313     bool connected = false;
1314     int ulen = len;
1315     int corkreq = READ_ONCE(up->corkflag) || msg->msg_flags&MSG_MORE;
1316     int err;
1317     int is_udplite = IS_UDPLITE(sk);
1318     int (*getfrag)(void *, char *, int, int, int, struct sk_buff *);
1319 
1320     ipcm6_init(&ipc6);
1321     ipc6.gso_size = READ_ONCE(up->gso_size);
1322     ipc6.sockc.tsflags = sk->sk_tsflags;
1323     ipc6.sockc.mark = sk->sk_mark;
1324 
1325     /* destination address check */
1326     if (sin6) {
1327         if (addr_len < offsetof(struct sockaddr, sa_data))
1328             return -EINVAL;
1329 
1330         switch (sin6->sin6_family) {
1331         case AF_INET6:
1332             if (addr_len < SIN6_LEN_RFC2133)
1333                 return -EINVAL;
1334             daddr = &sin6->sin6_addr;
1335             if (ipv6_addr_any(daddr) &&
1336                 ipv6_addr_v4mapped(&np->saddr))
1337                 ipv6_addr_set_v4mapped(htonl(INADDR_LOOPBACK),
1338                                daddr);
1339             break;
1340         case AF_INET:
1341             goto do_udp_sendmsg;
1342         case AF_UNSPEC:
1343             msg->msg_name = sin6 = NULL;
1344             msg->msg_namelen = addr_len = 0;
1345             daddr = NULL;
1346             break;
1347         default:
1348             return -EINVAL;
1349         }
1350     } else if (!up->pending) {
1351         if (sk->sk_state != TCP_ESTABLISHED)
1352             return -EDESTADDRREQ;
1353         daddr = &sk->sk_v6_daddr;
1354     } else
1355         daddr = NULL;
1356 
1357     if (daddr) {
1358         if (ipv6_addr_v4mapped(daddr)) {
1359             struct sockaddr_in sin;
1360             sin.sin_family = AF_INET;
1361             sin.sin_port = sin6 ? sin6->sin6_port : inet->inet_dport;
1362             sin.sin_addr.s_addr = daddr->s6_addr32[3];
1363             msg->msg_name = &sin;
1364             msg->msg_namelen = sizeof(sin);
1365 do_udp_sendmsg:
1366             if (ipv6_only_sock(sk))
1367                 return -ENETUNREACH;
1368             return udp_sendmsg(sk, msg, len);
1369         }
1370     }
1371 
1372     /* Rough check on arithmetic overflow,
1373        better check is made in ip6_append_data().
1374        */
1375     if (len > INT_MAX - sizeof(struct udphdr))
1376         return -EMSGSIZE;
1377 
1378     getfrag  =  is_udplite ?  udplite_getfrag : ip_generic_getfrag;
1379     if (up->pending) {
1380         if (up->pending == AF_INET)
1381             return udp_sendmsg(sk, msg, len);
1382         /*
1383          * There are pending frames.
1384          * The socket lock must be held while it's corked.
1385          */
1386         lock_sock(sk);
1387         if (likely(up->pending)) {
1388             if (unlikely(up->pending != AF_INET6)) {
1389                 release_sock(sk);
1390                 return -EAFNOSUPPORT;
1391             }
1392             dst = NULL;
1393             goto do_append_data;
1394         }
1395         release_sock(sk);
1396     }
1397     ulen += sizeof(struct udphdr);
1398 
1399     memset(fl6, 0, sizeof(*fl6));
1400 
1401     if (sin6) {
1402         if (sin6->sin6_port == 0)
1403             return -EINVAL;
1404 
1405         fl6->fl6_dport = sin6->sin6_port;
1406         daddr = &sin6->sin6_addr;
1407 
1408         if (np->sndflow) {
1409             fl6->flowlabel = sin6->sin6_flowinfo&IPV6_FLOWINFO_MASK;
1410             if (fl6->flowlabel & IPV6_FLOWLABEL_MASK) {
1411                 flowlabel = fl6_sock_lookup(sk, fl6->flowlabel);
1412                 if (IS_ERR(flowlabel))
1413                     return -EINVAL;
1414             }
1415         }
1416 
1417         /*
1418          * Otherwise it will be difficult to maintain
1419          * sk->sk_dst_cache.
1420          */
1421         if (sk->sk_state == TCP_ESTABLISHED &&
1422             ipv6_addr_equal(daddr, &sk->sk_v6_daddr))
1423             daddr = &sk->sk_v6_daddr;
1424 
1425         if (addr_len >= sizeof(struct sockaddr_in6) &&
1426             sin6->sin6_scope_id &&
1427             __ipv6_addr_needs_scope_id(__ipv6_addr_type(daddr)))
1428             fl6->flowi6_oif = sin6->sin6_scope_id;
1429     } else {
1430         if (sk->sk_state != TCP_ESTABLISHED)
1431             return -EDESTADDRREQ;
1432 
1433         fl6->fl6_dport = inet->inet_dport;
1434         daddr = &sk->sk_v6_daddr;
1435         fl6->flowlabel = np->flow_label;
1436         connected = true;
1437     }
1438 
1439     if (!fl6->flowi6_oif)
1440         fl6->flowi6_oif = READ_ONCE(sk->sk_bound_dev_if);
1441 
1442     if (!fl6->flowi6_oif)
1443         fl6->flowi6_oif = np->sticky_pktinfo.ipi6_ifindex;
1444 
1445     fl6->flowi6_uid = sk->sk_uid;
1446 
1447     if (msg->msg_controllen) {
1448         opt = &opt_space;
1449         memset(opt, 0, sizeof(struct ipv6_txoptions));
1450         opt->tot_len = sizeof(*opt);
1451         ipc6.opt = opt;
1452 
1453         err = udp_cmsg_send(sk, msg, &ipc6.gso_size);
1454         if (err > 0)
1455             err = ip6_datagram_send_ctl(sock_net(sk), sk, msg, fl6,
1456                             &ipc6);
1457         if (err < 0) {
1458             fl6_sock_release(flowlabel);
1459             return err;
1460         }
1461         if ((fl6->flowlabel&IPV6_FLOWLABEL_MASK) && !flowlabel) {
1462             flowlabel = fl6_sock_lookup(sk, fl6->flowlabel);
1463             if (IS_ERR(flowlabel))
1464                 return -EINVAL;
1465         }
1466         if (!(opt->opt_nflen|opt->opt_flen))
1467             opt = NULL;
1468         connected = false;
1469     }
1470     if (!opt) {
1471         opt = txopt_get(np);
1472         opt_to_free = opt;
1473     }
1474     if (flowlabel)
1475         opt = fl6_merge_options(&opt_space, flowlabel, opt);
1476     opt = ipv6_fixup_options(&opt_space, opt);
1477     ipc6.opt = opt;
1478 
1479     fl6->flowi6_proto = sk->sk_protocol;
1480     fl6->flowi6_mark = ipc6.sockc.mark;
1481     fl6->daddr = *daddr;
1482     if (ipv6_addr_any(&fl6->saddr) && !ipv6_addr_any(&np->saddr))
1483         fl6->saddr = np->saddr;
1484     fl6->fl6_sport = inet->inet_sport;
1485 
1486     if (cgroup_bpf_enabled(CGROUP_UDP6_SENDMSG) && !connected) {
1487         err = BPF_CGROUP_RUN_PROG_UDP6_SENDMSG_LOCK(sk,
1488                        (struct sockaddr *)sin6,
1489                        &fl6->saddr);
1490         if (err)
1491             goto out_no_dst;
1492         if (sin6) {
1493             if (ipv6_addr_v4mapped(&sin6->sin6_addr)) {
1494                 /* BPF program rewrote IPv6-only by IPv4-mapped
1495                  * IPv6. It's currently unsupported.
1496                  */
1497                 err = -ENOTSUPP;
1498                 goto out_no_dst;
1499             }
1500             if (sin6->sin6_port == 0) {
1501                 /* BPF program set invalid port. Reject it. */
1502                 err = -EINVAL;
1503                 goto out_no_dst;
1504             }
1505             fl6->fl6_dport = sin6->sin6_port;
1506             fl6->daddr = sin6->sin6_addr;
1507         }
1508     }
1509 
1510     if (ipv6_addr_any(&fl6->daddr))
1511         fl6->daddr.s6_addr[15] = 0x1; /* :: means loopback (BSD'ism) */
1512 
1513     final_p = fl6_update_dst(fl6, opt, &final);
1514     if (final_p)
1515         connected = false;
1516 
1517     if (!fl6->flowi6_oif && ipv6_addr_is_multicast(&fl6->daddr)) {
1518         fl6->flowi6_oif = np->mcast_oif;
1519         connected = false;
1520     } else if (!fl6->flowi6_oif)
1521         fl6->flowi6_oif = np->ucast_oif;
1522 
1523     security_sk_classify_flow(sk, flowi6_to_flowi_common(fl6));
1524 
1525     if (ipc6.tclass < 0)
1526         ipc6.tclass = np->tclass;
1527 
1528     fl6->flowlabel = ip6_make_flowinfo(ipc6.tclass, fl6->flowlabel);
1529 
1530     dst = ip6_sk_dst_lookup_flow(sk, fl6, final_p, connected);
1531     if (IS_ERR(dst)) {
1532         err = PTR_ERR(dst);
1533         dst = NULL;
1534         goto out;
1535     }
1536 
1537     if (ipc6.hlimit < 0)
1538         ipc6.hlimit = ip6_sk_dst_hoplimit(np, fl6, dst);
1539 
1540     if (msg->msg_flags&MSG_CONFIRM)
1541         goto do_confirm;
1542 back_from_confirm:
1543 
1544     /* Lockless fast path for the non-corking case */
1545     if (!corkreq) {
1546         struct sk_buff *skb;
1547 
1548         skb = ip6_make_skb(sk, getfrag, msg, ulen,
1549                    sizeof(struct udphdr), &ipc6,
1550                    (struct rt6_info *)dst,
1551                    msg->msg_flags, &cork);
1552         err = PTR_ERR(skb);
1553         if (!IS_ERR_OR_NULL(skb))
1554             err = udp_v6_send_skb(skb, fl6, &cork.base);
1555         /* ip6_make_skb steals dst reference */
1556         goto out_no_dst;
1557     }
1558 
1559     lock_sock(sk);
1560     if (unlikely(up->pending)) {
1561         /* The socket is already corked while preparing it. */
1562         /* ... which is an evident application bug. --ANK */
1563         release_sock(sk);
1564 
1565         net_dbg_ratelimited("udp cork app bug 2\n");
1566         err = -EINVAL;
1567         goto out;
1568     }
1569 
1570     up->pending = AF_INET6;
1571 
1572 do_append_data:
1573     if (ipc6.dontfrag < 0)
1574         ipc6.dontfrag = np->dontfrag;
1575     up->len += ulen;
1576     err = ip6_append_data(sk, getfrag, msg, ulen, sizeof(struct udphdr),
1577                   &ipc6, fl6, (struct rt6_info *)dst,
1578                   corkreq ? msg->msg_flags|MSG_MORE : msg->msg_flags);
1579     if (err)
1580         udp_v6_flush_pending_frames(sk);
1581     else if (!corkreq)
1582         err = udp_v6_push_pending_frames(sk);
1583     else if (unlikely(skb_queue_empty(&sk->sk_write_queue)))
1584         up->pending = 0;
1585 
1586     if (err > 0)
1587         err = np->recverr ? net_xmit_errno(err) : 0;
1588     release_sock(sk);
1589 
1590 out:
1591     dst_release(dst);
1592 out_no_dst:
1593     fl6_sock_release(flowlabel);
1594     txopt_put(opt_to_free);
1595     if (!err)
1596         return len;
1597     /*
1598      * ENOBUFS = no kernel mem, SOCK_NOSPACE = no sndbuf space.  Reporting
1599      * ENOBUFS might not be good (it's not tunable per se), but otherwise
1600      * we don't have a good statistic (IpOutDiscards but it can be too many
1601      * things).  We could add another new stat but at least for now that
1602      * seems like overkill.
1603      */
1604     if (err == -ENOBUFS || test_bit(SOCK_NOSPACE, &sk->sk_socket->flags)) {
1605         UDP6_INC_STATS(sock_net(sk),
1606                    UDP_MIB_SNDBUFERRORS, is_udplite);
1607     }
1608     return err;
1609 
1610 do_confirm:
1611     if (msg->msg_flags & MSG_PROBE)
1612         dst_confirm_neigh(dst, &fl6->daddr);
1613     if (!(msg->msg_flags&MSG_PROBE) || len)
1614         goto back_from_confirm;
1615     err = 0;
1616     goto out;
1617 }
1618 
1619 void udpv6_destroy_sock(struct sock *sk)
1620 {
1621     struct udp_sock *up = udp_sk(sk);
1622     lock_sock(sk);
1623 
1624     /* protects from races with udp_abort() */
1625     sock_set_flag(sk, SOCK_DEAD);
1626     udp_v6_flush_pending_frames(sk);
1627     release_sock(sk);
1628 
1629     if (static_branch_unlikely(&udpv6_encap_needed_key)) {
1630         if (up->encap_type) {
1631             void (*encap_destroy)(struct sock *sk);
1632             encap_destroy = READ_ONCE(up->encap_destroy);
1633             if (encap_destroy)
1634                 encap_destroy(sk);
1635         }
1636         if (up->encap_enabled) {
1637             static_branch_dec(&udpv6_encap_needed_key);
1638             udp_encap_disable();
1639         }
1640     }
1641 
1642     inet6_destroy_sock(sk);
1643 }
1644 
1645 /*
1646  *  Socket option code for UDP
1647  */
1648 int udpv6_setsockopt(struct sock *sk, int level, int optname, sockptr_t optval,
1649              unsigned int optlen)
1650 {
1651     if (level == SOL_UDP  ||  level == SOL_UDPLITE)
1652         return udp_lib_setsockopt(sk, level, optname,
1653                       optval, optlen,
1654                       udp_v6_push_pending_frames);
1655     return ipv6_setsockopt(sk, level, optname, optval, optlen);
1656 }
1657 
1658 int udpv6_getsockopt(struct sock *sk, int level, int optname,
1659              char __user *optval, int __user *optlen)
1660 {
1661     if (level == SOL_UDP  ||  level == SOL_UDPLITE)
1662         return udp_lib_getsockopt(sk, level, optname, optval, optlen);
1663     return ipv6_getsockopt(sk, level, optname, optval, optlen);
1664 }
1665 
1666 static const struct inet6_protocol udpv6_protocol = {
1667     .handler    =   udpv6_rcv,
1668     .err_handler    =   udpv6_err,
1669     .flags      =   INET6_PROTO_NOPOLICY|INET6_PROTO_FINAL,
1670 };
1671 
1672 /* ------------------------------------------------------------------------ */
1673 #ifdef CONFIG_PROC_FS
1674 int udp6_seq_show(struct seq_file *seq, void *v)
1675 {
1676     if (v == SEQ_START_TOKEN) {
1677         seq_puts(seq, IPV6_SEQ_DGRAM_HEADER);
1678     } else {
1679         int bucket = ((struct udp_iter_state *)seq->private)->bucket;
1680         struct inet_sock *inet = inet_sk(v);
1681         __u16 srcp = ntohs(inet->inet_sport);
1682         __u16 destp = ntohs(inet->inet_dport);
1683         __ip6_dgram_sock_seq_show(seq, v, srcp, destp,
1684                       udp_rqueue_get(v), bucket);
1685     }
1686     return 0;
1687 }
1688 
1689 const struct seq_operations udp6_seq_ops = {
1690     .start      = udp_seq_start,
1691     .next       = udp_seq_next,
1692     .stop       = udp_seq_stop,
1693     .show       = udp6_seq_show,
1694 };
1695 EXPORT_SYMBOL(udp6_seq_ops);
1696 
1697 static struct udp_seq_afinfo udp6_seq_afinfo = {
1698     .family     = AF_INET6,
1699     .udp_table  = &udp_table,
1700 };
1701 
1702 int __net_init udp6_proc_init(struct net *net)
1703 {
1704     if (!proc_create_net_data("udp6", 0444, net->proc_net, &udp6_seq_ops,
1705             sizeof(struct udp_iter_state), &udp6_seq_afinfo))
1706         return -ENOMEM;
1707     return 0;
1708 }
1709 
1710 void udp6_proc_exit(struct net *net)
1711 {
1712     remove_proc_entry("udp6", net->proc_net);
1713 }
1714 #endif /* CONFIG_PROC_FS */
1715 
1716 /* ------------------------------------------------------------------------ */
1717 
1718 struct proto udpv6_prot = {
1719     .name           = "UDPv6",
1720     .owner          = THIS_MODULE,
1721     .close          = udp_lib_close,
1722     .pre_connect        = udpv6_pre_connect,
1723     .connect        = ip6_datagram_connect,
1724     .disconnect     = udp_disconnect,
1725     .ioctl          = udp_ioctl,
1726     .init           = udp_init_sock,
1727     .destroy        = udpv6_destroy_sock,
1728     .setsockopt     = udpv6_setsockopt,
1729     .getsockopt     = udpv6_getsockopt,
1730     .sendmsg        = udpv6_sendmsg,
1731     .recvmsg        = udpv6_recvmsg,
1732     .release_cb     = ip6_datagram_release_cb,
1733     .hash           = udp_lib_hash,
1734     .unhash         = udp_lib_unhash,
1735     .rehash         = udp_v6_rehash,
1736     .get_port       = udp_v6_get_port,
1737     .put_port       = udp_lib_unhash,
1738 #ifdef CONFIG_BPF_SYSCALL
1739     .psock_update_sk_prot   = udp_bpf_update_proto,
1740 #endif
1741 
1742     .memory_allocated   = &udp_memory_allocated,
1743     .per_cpu_fw_alloc   = &udp_memory_per_cpu_fw_alloc,
1744 
1745     .sysctl_mem     = sysctl_udp_mem,
1746     .sysctl_wmem_offset     = offsetof(struct net, ipv4.sysctl_udp_wmem_min),
1747     .sysctl_rmem_offset     = offsetof(struct net, ipv4.sysctl_udp_rmem_min),
1748     .obj_size       = sizeof(struct udp6_sock),
1749     .h.udp_table        = &udp_table,
1750     .diag_destroy       = udp_abort,
1751 };
1752 
1753 static struct inet_protosw udpv6_protosw = {
1754     .type =      SOCK_DGRAM,
1755     .protocol =  IPPROTO_UDP,
1756     .prot =      &udpv6_prot,
1757     .ops =       &inet6_dgram_ops,
1758     .flags =     INET_PROTOSW_PERMANENT,
1759 };
1760 
1761 int __init udpv6_init(void)
1762 {
1763     int ret;
1764 
1765     ret = inet6_add_protocol(&udpv6_protocol, IPPROTO_UDP);
1766     if (ret)
1767         goto out;
1768 
1769     ret = inet6_register_protosw(&udpv6_protosw);
1770     if (ret)
1771         goto out_udpv6_protocol;
1772 out:
1773     return ret;
1774 
1775 out_udpv6_protocol:
1776     inet6_del_protocol(&udpv6_protocol, IPPROTO_UDP);
1777     goto out;
1778 }
1779 
1780 void udpv6_exit(void)
1781 {
1782     inet6_unregister_protosw(&udpv6_protosw);
1783     inet6_del_protocol(&udpv6_protocol, IPPROTO_UDP);
1784 }
[ Source navigation ] [ Diff markup ] [ Identifier search ] [ general search ]

This page was automatically generated by the 2.1.0 LXR engine.

The LXR team
Valid CSS 2.1! Valid HTML 4.01!