net/ipv6/udp_offload.c

0001 // SPDX-License-Identifier: GPL-2.0-or-later
0002 /*
0003  *  IPV6 GSO/GRO offload support
0004  *  Linux INET6 implementation
0005  *
0006  *      UDPv6 GSO support
0007  */
0008 #include <linux/skbuff.h>
0009 #include <linux/netdevice.h>
0010 #include <linux/indirect_call_wrapper.h>
0011 #include <net/protocol.h>
0012 #include <net/ipv6.h>
0013 #include <net/udp.h>
0014 #include <net/ip6_checksum.h>
0015 #include "ip6_offload.h"
0016 #include <net/gro.h>
0017
0018 static struct sk_buff *udp6_ufo_fragment(struct sk_buff *skb,
0019                      netdev_features_t features)
0020 {
0021     struct sk_buff *segs = ERR_PTR(-EINVAL);
0022     unsigned int mss;
0023     unsigned int unfrag_ip6hlen, unfrag_len;
0024     struct frag_hdr *fptr;
0025     u8 *packet_start, *prevhdr;
0026     u8 nexthdr;
0027     u8 frag_hdr_sz = sizeof(struct frag_hdr);
0028     __wsum csum;
0029     int tnl_hlen;
0030     int err;
0031
0032     if (skb->encapsulation && skb_shinfo(skb)->gso_type &
0033         (SKB_GSO_UDP_TUNNEL|SKB_GSO_UDP_TUNNEL_CSUM))
0034         segs = skb_udp_tunnel_segment(skb, features, true);
0035     else {
0036         const struct ipv6hdr *ipv6h;
0037         struct udphdr *uh;
0038
0039         if (!(skb_shinfo(skb)->gso_type & (SKB_GSO_UDP | SKB_GSO_UDP_L4)))
0040             goto out;
0041
0042         if (!pskb_may_pull(skb, sizeof(struct udphdr)))
0043             goto out;
0044
0045         if (skb_shinfo(skb)->gso_type & SKB_GSO_UDP_L4)
0046             return __udp_gso_segment(skb, features, true);
0047
0048         mss = skb_shinfo(skb)->gso_size;
0049         if (unlikely(skb->len <= mss))
0050             goto out;
0051
0052         /* Do software UFO. Complete and fill in the UDP checksum as HW cannot
0053          * do checksum of UDP packets sent as multiple IP fragments.
0054          */
0055
0056         uh = udp_hdr(skb);
0057         ipv6h = ipv6_hdr(skb);
0058
0059         uh->check = 0;
0060         csum = skb_checksum(skb, 0, skb->len, 0);
0061         uh->check = udp_v6_check(skb->len, &ipv6h->saddr,
0062                       &ipv6h->daddr, csum);
0063         if (uh->check == 0)
0064             uh->check = CSUM_MANGLED_0;
0065
0066         skb->ip_summed = CHECKSUM_UNNECESSARY;
0067
0068         /* If there is no outer header we can fake a checksum offload
0069          * due to the fact that we have already done the checksum in
0070          * software prior to segmenting the frame.
0071          */
0072         if (!skb->encap_hdr_csum)
0073             features |= NETIF_F_HW_CSUM;
0074
0075         /* Check if there is enough headroom to insert fragment header. */
0076         tnl_hlen = skb_tnl_header_len(skb);
0077         if (skb->mac_header < (tnl_hlen + frag_hdr_sz)) {
0078             if (gso_pskb_expand_head(skb, tnl_hlen + frag_hdr_sz))
0079                 goto out;
0080         }
0081
0082         /* Find the unfragmentable header and shift it left by frag_hdr_sz
0083          * bytes to insert fragment header.
0084          */
0085         err = ip6_find_1stfragopt(skb, &prevhdr);
0086         if (err < 0)
0087             return ERR_PTR(err);
0088         unfrag_ip6hlen = err;
0089         nexthdr = *prevhdr;
0090         *prevhdr = NEXTHDR_FRAGMENT;
0091         unfrag_len = (skb_network_header(skb) - skb_mac_header(skb)) +
0092                  unfrag_ip6hlen + tnl_hlen;
0093         packet_start = (u8 *) skb->head + SKB_GSO_CB(skb)->mac_offset;
0094         memmove(packet_start-frag_hdr_sz, packet_start, unfrag_len);
0095
0096         SKB_GSO_CB(skb)->mac_offset -= frag_hdr_sz;
0097         skb->mac_header -= frag_hdr_sz;
0098         skb->network_header -= frag_hdr_sz;
0099
0100         fptr = (struct frag_hdr *)(skb_network_header(skb) + unfrag_ip6hlen);
0101         fptr->nexthdr = nexthdr;
0102         fptr->reserved = 0;
0103         fptr->identification = ipv6_proxy_select_ident(dev_net(skb->dev), skb);
0104
0105         /* Fragment the skb. ipv6 header and the remaining fields of the
0106          * fragment header are updated in ipv6_gso_segment()
0107          */
0108         segs = skb_segment(skb, features);
0109     }
0110
0111 out:
0112     return segs;
0113 }
0114
0115 static struct sock *udp6_gro_lookup_skb(struct sk_buff *skb, __be16 sport,
0116                     __be16 dport)
0117 {
0118     const struct ipv6hdr *iph = skb_gro_network_header(skb);
0119
0120     return __udp6_lib_lookup(dev_net(skb->dev), &iph->saddr, sport,
0121                  &iph->daddr, dport, inet6_iif(skb),
0122                  inet6_sdif(skb), &udp_table, NULL);
0123 }
0124
0125 INDIRECT_CALLABLE_SCOPE
0126 struct sk_buff *udp6_gro_receive(struct list_head *head, struct sk_buff *skb)
0127 {
0128     struct udphdr *uh = udp_gro_udphdr(skb);
0129     struct sock *sk = NULL;
0130     struct sk_buff *pp;
0131
0132     if (unlikely(!uh))
0133         goto flush;
0134
0135     /* Don't bother verifying checksum if we're going to flush anyway. */
0136     if (NAPI_GRO_CB(skb)->flush)
0137         goto skip;
0138
0139     if (skb_gro_checksum_validate_zero_check(skb, IPPROTO_UDP, uh->check,
0140                          ip6_gro_compute_pseudo))
0141         goto flush;
0142     else if (uh->check)
0143         skb_gro_checksum_try_convert(skb, IPPROTO_UDP,
0144                          ip6_gro_compute_pseudo);
0145
0146 skip:
0147     NAPI_GRO_CB(skb)->is_ipv6 = 1;
0148
0149     if (static_branch_unlikely(&udpv6_encap_needed_key))
0150         sk = udp6_gro_lookup_skb(skb, uh->source, uh->dest);
0151
0152     pp = udp_gro_receive(head, skb, uh, sk);
0153     return pp;
0154
0155 flush:
0156     NAPI_GRO_CB(skb)->flush = 1;
0157     return NULL;
0158 }
0159
0160 INDIRECT_CALLABLE_SCOPE int udp6_gro_complete(struct sk_buff *skb, int nhoff)
0161 {
0162     const struct ipv6hdr *ipv6h = ipv6_hdr(skb);
0163     struct udphdr *uh = (struct udphdr *)(skb->data + nhoff);
0164
0165     /* do fraglist only if there is no outer UDP encap (or we already processed it) */
0166     if (NAPI_GRO_CB(skb)->is_flist && !NAPI_GRO_CB(skb)->encap_mark) {
0167         uh->len = htons(skb->len - nhoff);
0168
0169         skb_shinfo(skb)->gso_type |= (SKB_GSO_FRAGLIST|SKB_GSO_UDP_L4);
0170         skb_shinfo(skb)->gso_segs = NAPI_GRO_CB(skb)->count;
0171
0172         if (skb->ip_summed == CHECKSUM_UNNECESSARY) {
0173             if (skb->csum_level < SKB_MAX_CSUM_LEVEL)
0174                 skb->csum_level++;
0175         } else {
0176             skb->ip_summed = CHECKSUM_UNNECESSARY;
0177             skb->csum_level = 0;
0178         }
0179
0180         return 0;
0181     }
0182
0183     if (uh->check)
0184         uh->check = ~udp_v6_check(skb->len - nhoff, &ipv6h->saddr,
0185                       &ipv6h->daddr, 0);
0186
0187     return udp_gro_complete(skb, nhoff, udp6_lib_lookup_skb);
0188 }
0189
0190 static const struct net_offload udpv6_offload = {
0191     .callbacks = {
0192         .gso_segment    =   udp6_ufo_fragment,
0193         .gro_receive    =   udp6_gro_receive,
0194         .gro_complete   =   udp6_gro_complete,
0195     },
0196 };
0197
0198 int udpv6_offload_init(void)
0199 {
0200     return inet6_add_offload(&udpv6_offload, IPPROTO_UDP);
0201 }
0202
0203 int udpv6_offload_exit(void)
0204 {
0205     return inet6_del_offload(&udpv6_offload, IPPROTO_UDP);
0206 }