net/ipv4/udp_offload.c

0001 // SPDX-License-Identifier: GPL-2.0-or-later
0002 /*
0003  *  IPV4 GSO/GRO offload support
0004  *  Linux INET implementation
0005  *
0006  *  UDPv4 GSO support
0007  */
0008
0009 #include <linux/skbuff.h>
0010 #include <net/gro.h>
0011 #include <net/udp.h>
0012 #include <net/protocol.h>
0013 #include <net/inet_common.h>
0014
0015 static struct sk_buff *__skb_udp_tunnel_segment(struct sk_buff *skb,
0016     netdev_features_t features,
0017     struct sk_buff *(*gso_inner_segment)(struct sk_buff *skb,
0018                          netdev_features_t features),
0019     __be16 new_protocol, bool is_ipv6)
0020 {
0021     int tnl_hlen = skb_inner_mac_header(skb) - skb_transport_header(skb);
0022     bool remcsum, need_csum, offload_csum, gso_partial;
0023     struct sk_buff *segs = ERR_PTR(-EINVAL);
0024     struct udphdr *uh = udp_hdr(skb);
0025     u16 mac_offset = skb->mac_header;
0026     __be16 protocol = skb->protocol;
0027     u16 mac_len = skb->mac_len;
0028     int udp_offset, outer_hlen;
0029     __wsum partial;
0030     bool need_ipsec;
0031
0032     if (unlikely(!pskb_may_pull(skb, tnl_hlen)))
0033         goto out;
0034
0035     /* Adjust partial header checksum to negate old length.
0036      * We cannot rely on the value contained in uh->len as it is
0037      * possible that the actual value exceeds the boundaries of the
0038      * 16 bit length field due to the header being added outside of an
0039      * IP or IPv6 frame that was already limited to 64K - 1.
0040      */
0041     if (skb_shinfo(skb)->gso_type & SKB_GSO_PARTIAL)
0042         partial = (__force __wsum)uh->len;
0043     else
0044         partial = (__force __wsum)htonl(skb->len);
0045     partial = csum_sub(csum_unfold(uh->check), partial);
0046
0047     /* setup inner skb. */
0048     skb->encapsulation = 0;
0049     SKB_GSO_CB(skb)->encap_level = 0;
0050     __skb_pull(skb, tnl_hlen);
0051     skb_reset_mac_header(skb);
0052     skb_set_network_header(skb, skb_inner_network_offset(skb));
0053     skb_set_transport_header(skb, skb_inner_transport_offset(skb));
0054     skb->mac_len = skb_inner_network_offset(skb);
0055     skb->protocol = new_protocol;
0056
0057     need_csum = !!(skb_shinfo(skb)->gso_type & SKB_GSO_UDP_TUNNEL_CSUM);
0058     skb->encap_hdr_csum = need_csum;
0059
0060     remcsum = !!(skb_shinfo(skb)->gso_type & SKB_GSO_TUNNEL_REMCSUM);
0061     skb->remcsum_offload = remcsum;
0062
0063     need_ipsec = skb_dst(skb) && dst_xfrm(skb_dst(skb));
0064     /* Try to offload checksum if possible */
0065     offload_csum = !!(need_csum &&
0066               !need_ipsec &&
0067               (skb->dev->features &
0068                (is_ipv6 ? (NETIF_F_HW_CSUM | NETIF_F_IPV6_CSUM) :
0069                       (NETIF_F_HW_CSUM | NETIF_F_IP_CSUM))));
0070
0071     features &= skb->dev->hw_enc_features;
0072     if (need_csum)
0073         features &= ~NETIF_F_SCTP_CRC;
0074
0075     /* The only checksum offload we care about from here on out is the
0076      * outer one so strip the existing checksum feature flags and
0077      * instead set the flag based on our outer checksum offload value.
0078      */
0079     if (remcsum) {
0080         features &= ~NETIF_F_CSUM_MASK;
0081         if (!need_csum || offload_csum)
0082             features |= NETIF_F_HW_CSUM;
0083     }
0084
0085     /* segment inner packet. */
0086     segs = gso_inner_segment(skb, features);
0087     if (IS_ERR_OR_NULL(segs)) {
0088         skb_gso_error_unwind(skb, protocol, tnl_hlen, mac_offset,
0089                      mac_len);
0090         goto out;
0091     }
0092
0093     gso_partial = !!(skb_shinfo(segs)->gso_type & SKB_GSO_PARTIAL);
0094
0095     outer_hlen = skb_tnl_header_len(skb);
0096     udp_offset = outer_hlen - tnl_hlen;
0097     skb = segs;
0098     do {
0099         unsigned int len;
0100
0101         if (remcsum)
0102             skb->ip_summed = CHECKSUM_NONE;
0103
0104         /* Set up inner headers if we are offloading inner checksum */
0105         if (skb->ip_summed == CHECKSUM_PARTIAL) {
0106             skb_reset_inner_headers(skb);
0107             skb->encapsulation = 1;
0108         }
0109
0110         skb->mac_len = mac_len;
0111         skb->protocol = protocol;
0112
0113         __skb_push(skb, outer_hlen);
0114         skb_reset_mac_header(skb);
0115         skb_set_network_header(skb, mac_len);
0116         skb_set_transport_header(skb, udp_offset);
0117         len = skb->len - udp_offset;
0118         uh = udp_hdr(skb);
0119
0120         /* If we are only performing partial GSO the inner header
0121          * will be using a length value equal to only one MSS sized
0122          * segment instead of the entire frame.
0123          */
0124         if (gso_partial && skb_is_gso(skb)) {
0125             uh->len = htons(skb_shinfo(skb)->gso_size +
0126                     SKB_GSO_CB(skb)->data_offset +
0127                     skb->head - (unsigned char *)uh);
0128         } else {
0129             uh->len = htons(len);
0130         }
0131
0132         if (!need_csum)
0133             continue;
0134
0135         uh->check = ~csum_fold(csum_add(partial,
0136                        (__force __wsum)htonl(len)));
0137
0138         if (skb->encapsulation || !offload_csum) {
0139             uh->check = gso_make_checksum(skb, ~uh->check);
0140             if (uh->check == 0)
0141                 uh->check = CSUM_MANGLED_0;
0142         } else {
0143             skb->ip_summed = CHECKSUM_PARTIAL;
0144             skb->csum_start = skb_transport_header(skb) - skb->head;
0145             skb->csum_offset = offsetof(struct udphdr, check);
0146         }
0147     } while ((skb = skb->next));
0148 out:
0149     return segs;
0150 }
0151
0152 struct sk_buff *skb_udp_tunnel_segment(struct sk_buff *skb,
0153                        netdev_features_t features,
0154                        bool is_ipv6)
0155 {
0156     const struct net_offload __rcu **offloads;
0157     __be16 protocol = skb->protocol;
0158     const struct net_offload *ops;
0159     struct sk_buff *segs = ERR_PTR(-EINVAL);
0160     struct sk_buff *(*gso_inner_segment)(struct sk_buff *skb,
0161                          netdev_features_t features);
0162
0163     rcu_read_lock();
0164
0165     switch (skb->inner_protocol_type) {
0166     case ENCAP_TYPE_ETHER:
0167         protocol = skb->inner_protocol;
0168         gso_inner_segment = skb_mac_gso_segment;
0169         break;
0170     case ENCAP_TYPE_IPPROTO:
0171         offloads = is_ipv6 ? inet6_offloads : inet_offloads;
0172         ops = rcu_dereference(offloads[skb->inner_ipproto]);
0173         if (!ops || !ops->callbacks.gso_segment)
0174             goto out_unlock;
0175         gso_inner_segment = ops->callbacks.gso_segment;
0176         break;
0177     default:
0178         goto out_unlock;
0179     }
0180
0181     segs = __skb_udp_tunnel_segment(skb, features, gso_inner_segment,
0182                     protocol, is_ipv6);
0183
0184 out_unlock:
0185     rcu_read_unlock();
0186
0187     return segs;
0188 }
0189 EXPORT_SYMBOL(skb_udp_tunnel_segment);
0190
0191 static void __udpv4_gso_segment_csum(struct sk_buff *seg,
0192                      __be32 *oldip, __be32 *newip,
0193                      __be16 *oldport, __be16 *newport)
0194 {
0195     struct udphdr *uh;
0196     struct iphdr *iph;
0197
0198     if (*oldip == *newip && *oldport == *newport)
0199         return;
0200
0201     uh = udp_hdr(seg);
0202     iph = ip_hdr(seg);
0203
0204     if (uh->check) {
0205         inet_proto_csum_replace4(&uh->check, seg, *oldip, *newip,
0206                      true);
0207         inet_proto_csum_replace2(&uh->check, seg, *oldport, *newport,
0208                      false);
0209         if (!uh->check)
0210             uh->check = CSUM_MANGLED_0;
0211     }
0212     *oldport = *newport;
0213
0214     csum_replace4(&iph->check, *oldip, *newip);
0215     *oldip = *newip;
0216 }
0217
0218 static struct sk_buff *__udpv4_gso_segment_list_csum(struct sk_buff *segs)
0219 {
0220     struct sk_buff *seg;
0221     struct udphdr *uh, *uh2;
0222     struct iphdr *iph, *iph2;
0223
0224     seg = segs;
0225     uh = udp_hdr(seg);
0226     iph = ip_hdr(seg);
0227
0228     if ((udp_hdr(seg)->dest == udp_hdr(seg->next)->dest) &&
0229         (udp_hdr(seg)->source == udp_hdr(seg->next)->source) &&
0230         (ip_hdr(seg)->daddr == ip_hdr(seg->next)->daddr) &&
0231         (ip_hdr(seg)->saddr == ip_hdr(seg->next)->saddr))
0232         return segs;
0233
0234     while ((seg = seg->next)) {
0235         uh2 = udp_hdr(seg);
0236         iph2 = ip_hdr(seg);
0237
0238         __udpv4_gso_segment_csum(seg,
0239                      &iph2->saddr, &iph->saddr,
0240                      &uh2->source, &uh->source);
0241         __udpv4_gso_segment_csum(seg,
0242                      &iph2->daddr, &iph->daddr,
0243                      &uh2->dest, &uh->dest);
0244     }
0245
0246     return segs;
0247 }
0248
0249 static struct sk_buff *__udp_gso_segment_list(struct sk_buff *skb,
0250                           netdev_features_t features,
0251                           bool is_ipv6)
0252 {
0253     unsigned int mss = skb_shinfo(skb)->gso_size;
0254
0255     skb = skb_segment_list(skb, features, skb_mac_header_len(skb));
0256     if (IS_ERR(skb))
0257         return skb;
0258
0259     udp_hdr(skb)->len = htons(sizeof(struct udphdr) + mss);
0260
0261     return is_ipv6 ? skb : __udpv4_gso_segment_list_csum(skb);
0262 }
0263
0264 struct sk_buff *__udp_gso_segment(struct sk_buff *gso_skb,
0265                   netdev_features_t features, bool is_ipv6)
0266 {
0267     struct sock *sk = gso_skb->sk;
0268     unsigned int sum_truesize = 0;
0269     struct sk_buff *segs, *seg;
0270     struct udphdr *uh;
0271     unsigned int mss;
0272     bool copy_dtor;
0273     __sum16 check;
0274     __be16 newlen;
0275
0276     if (skb_shinfo(gso_skb)->gso_type & SKB_GSO_FRAGLIST)
0277         return __udp_gso_segment_list(gso_skb, features, is_ipv6);
0278
0279     mss = skb_shinfo(gso_skb)->gso_size;
0280     if (gso_skb->len <= sizeof(*uh) + mss)
0281         return ERR_PTR(-EINVAL);
0282
0283     skb_pull(gso_skb, sizeof(*uh));
0284
0285     /* clear destructor to avoid skb_segment assigning it to tail */
0286     copy_dtor = gso_skb->destructor == sock_wfree;
0287     if (copy_dtor)
0288         gso_skb->destructor = NULL;
0289
0290     segs = skb_segment(gso_skb, features);
0291     if (IS_ERR_OR_NULL(segs)) {
0292         if (copy_dtor)
0293             gso_skb->destructor = sock_wfree;
0294         return segs;
0295     }
0296
0297     /* GSO partial and frag_list segmentation only requires splitting
0298      * the frame into an MSS multiple and possibly a remainder, both
0299      * cases return a GSO skb. So update the mss now.
0300      */
0301     if (skb_is_gso(segs))
0302         mss *= skb_shinfo(segs)->gso_segs;
0303
0304     seg = segs;
0305     uh = udp_hdr(seg);
0306
0307     /* preserve TX timestamp flags and TS key for first segment */
0308     skb_shinfo(seg)->tskey = skb_shinfo(gso_skb)->tskey;
0309     skb_shinfo(seg)->tx_flags |=
0310             (skb_shinfo(gso_skb)->tx_flags & SKBTX_ANY_TSTAMP);
0311
0312     /* compute checksum adjustment based on old length versus new */
0313     newlen = htons(sizeof(*uh) + mss);
0314     check = csum16_add(csum16_sub(uh->check, uh->len), newlen);
0315
0316     for (;;) {
0317         if (copy_dtor) {
0318             seg->destructor = sock_wfree;
0319             seg->sk = sk;
0320             sum_truesize += seg->truesize;
0321         }
0322
0323         if (!seg->next)
0324             break;
0325
0326         uh->len = newlen;
0327         uh->check = check;
0328
0329         if (seg->ip_summed == CHECKSUM_PARTIAL)
0330             gso_reset_checksum(seg, ~check);
0331         else
0332             uh->check = gso_make_checksum(seg, ~check) ? :
0333                     CSUM_MANGLED_0;
0334
0335         seg = seg->next;
0336         uh = udp_hdr(seg);
0337     }
0338
0339     /* last packet can be partial gso_size, account for that in checksum */
0340     newlen = htons(skb_tail_pointer(seg) - skb_transport_header(seg) +
0341                seg->data_len);
0342     check = csum16_add(csum16_sub(uh->check, uh->len), newlen);
0343
0344     uh->len = newlen;
0345     uh->check = check;
0346
0347     if (seg->ip_summed == CHECKSUM_PARTIAL)
0348         gso_reset_checksum(seg, ~check);
0349     else
0350         uh->check = gso_make_checksum(seg, ~check) ? : CSUM_MANGLED_0;
0351
0352     /* update refcount for the packet */
0353     if (copy_dtor) {
0354         int delta = sum_truesize - gso_skb->truesize;
0355
0356         /* In some pathological cases, delta can be negative.
0357          * We need to either use refcount_add() or refcount_sub_and_test()
0358          */
0359         if (likely(delta >= 0))
0360             refcount_add(delta, &sk->sk_wmem_alloc);
0361         else
0362             WARN_ON_ONCE(refcount_sub_and_test(-delta, &sk->sk_wmem_alloc));
0363     }
0364     return segs;
0365 }
0366 EXPORT_SYMBOL_GPL(__udp_gso_segment);
0367
0368 static struct sk_buff *udp4_ufo_fragment(struct sk_buff *skb,
0369                      netdev_features_t features)
0370 {
0371     struct sk_buff *segs = ERR_PTR(-EINVAL);
0372     unsigned int mss;
0373     __wsum csum;
0374     struct udphdr *uh;
0375     struct iphdr *iph;
0376
0377     if (skb->encapsulation &&
0378         (skb_shinfo(skb)->gso_type &
0379          (SKB_GSO_UDP_TUNNEL|SKB_GSO_UDP_TUNNEL_CSUM))) {
0380         segs = skb_udp_tunnel_segment(skb, features, false);
0381         goto out;
0382     }
0383
0384     if (!(skb_shinfo(skb)->gso_type & (SKB_GSO_UDP | SKB_GSO_UDP_L4)))
0385         goto out;
0386
0387     if (!pskb_may_pull(skb, sizeof(struct udphdr)))
0388         goto out;
0389
0390     if (skb_shinfo(skb)->gso_type & SKB_GSO_UDP_L4)
0391         return __udp_gso_segment(skb, features, false);
0392
0393     mss = skb_shinfo(skb)->gso_size;
0394     if (unlikely(skb->len <= mss))
0395         goto out;
0396
0397     /* Do software UFO. Complete and fill in the UDP checksum as
0398      * HW cannot do checksum of UDP packets sent as multiple
0399      * IP fragments.
0400      */
0401
0402     uh = udp_hdr(skb);
0403     iph = ip_hdr(skb);
0404
0405     uh->check = 0;
0406     csum = skb_checksum(skb, 0, skb->len, 0);
0407     uh->check = udp_v4_check(skb->len, iph->saddr, iph->daddr, csum);
0408     if (uh->check == 0)
0409         uh->check = CSUM_MANGLED_0;
0410
0411     skb->ip_summed = CHECKSUM_UNNECESSARY;
0412
0413     /* If there is no outer header we can fake a checksum offload
0414      * due to the fact that we have already done the checksum in
0415      * software prior to segmenting the frame.
0416      */
0417     if (!skb->encap_hdr_csum)
0418         features |= NETIF_F_HW_CSUM;
0419
0420     /* Fragment the skb. IP headers of the fragments are updated in
0421      * inet_gso_segment()
0422      */
0423     segs = skb_segment(skb, features);
0424 out:
0425     return segs;
0426 }
0427
0428 static int skb_gro_receive_list(struct sk_buff *p, struct sk_buff *skb)
0429 {
0430     if (unlikely(p->len + skb->len >= 65536))
0431         return -E2BIG;
0432
0433     if (NAPI_GRO_CB(p)->last == p)
0434         skb_shinfo(p)->frag_list = skb;
0435     else
0436         NAPI_GRO_CB(p)->last->next = skb;
0437
0438     skb_pull(skb, skb_gro_offset(skb));
0439
0440     NAPI_GRO_CB(p)->last = skb;
0441     NAPI_GRO_CB(p)->count++;
0442     p->data_len += skb->len;
0443
0444     /* sk owenrship - if any - completely transferred to the aggregated packet */
0445     skb->destructor = NULL;
0446     p->truesize += skb->truesize;
0447     p->len += skb->len;
0448
0449     NAPI_GRO_CB(skb)->same_flow = 1;
0450
0451     return 0;
0452 }
0453
0454
0455 #define UDP_GRO_CNT_MAX 64
0456 static struct sk_buff *udp_gro_receive_segment(struct list_head *head,
0457                            struct sk_buff *skb)
0458 {
0459     struct udphdr *uh = udp_gro_udphdr(skb);
0460     struct sk_buff *pp = NULL;
0461     struct udphdr *uh2;
0462     struct sk_buff *p;
0463     unsigned int ulen;
0464     int ret = 0;
0465
0466     /* requires non zero csum, for symmetry with GSO */
0467     if (!uh->check) {
0468         NAPI_GRO_CB(skb)->flush = 1;
0469         return NULL;
0470     }
0471
0472     /* Do not deal with padded or malicious packets, sorry ! */
0473     ulen = ntohs(uh->len);
0474     if (ulen <= sizeof(*uh) || ulen != skb_gro_len(skb)) {
0475         NAPI_GRO_CB(skb)->flush = 1;
0476         return NULL;
0477     }
0478     /* pull encapsulating udp header */
0479     skb_gro_pull(skb, sizeof(struct udphdr));
0480
0481     list_for_each_entry(p, head, list) {
0482         if (!NAPI_GRO_CB(p)->same_flow)
0483             continue;
0484
0485         uh2 = udp_hdr(p);
0486
0487         /* Match ports only, as csum is always non zero */
0488         if ((*(u32 *)&uh->source != *(u32 *)&uh2->source)) {
0489             NAPI_GRO_CB(p)->same_flow = 0;
0490             continue;
0491         }
0492
0493         if (NAPI_GRO_CB(skb)->is_flist != NAPI_GRO_CB(p)->is_flist) {
0494             NAPI_GRO_CB(skb)->flush = 1;
0495             return p;
0496         }
0497
0498         /* Terminate the flow on len mismatch or if it grow "too much".
0499          * Under small packet flood GRO count could elsewhere grow a lot
0500          * leading to excessive truesize values.
0501          * On len mismatch merge the first packet shorter than gso_size,
0502          * otherwise complete the GRO packet.
0503          */
0504         if (ulen > ntohs(uh2->len)) {
0505             pp = p;
0506         } else {
0507             if (NAPI_GRO_CB(skb)->is_flist) {
0508                 if (!pskb_may_pull(skb, skb_gro_offset(skb))) {
0509                     NAPI_GRO_CB(skb)->flush = 1;
0510                     return NULL;
0511                 }
0512                 if ((skb->ip_summed != p->ip_summed) ||
0513                     (skb->csum_level != p->csum_level)) {
0514                     NAPI_GRO_CB(skb)->flush = 1;
0515                     return NULL;
0516                 }
0517                 ret = skb_gro_receive_list(p, skb);
0518             } else {
0519                 skb_gro_postpull_rcsum(skb, uh,
0520                                sizeof(struct udphdr));
0521
0522                 ret = skb_gro_receive(p, skb);
0523             }
0524         }
0525
0526         if (ret || ulen != ntohs(uh2->len) ||
0527             NAPI_GRO_CB(p)->count >= UDP_GRO_CNT_MAX)
0528             pp = p;
0529
0530         return pp;
0531     }
0532
0533     /* mismatch, but we never need to flush */
0534     return NULL;
0535 }
0536
0537 struct sk_buff *udp_gro_receive(struct list_head *head, struct sk_buff *skb,
0538                 struct udphdr *uh, struct sock *sk)
0539 {
0540     struct sk_buff *pp = NULL;
0541     struct sk_buff *p;
0542     struct udphdr *uh2;
0543     unsigned int off = skb_gro_offset(skb);
0544     int flush = 1;
0545
0546     /* we can do L4 aggregation only if the packet can't land in a tunnel
0547      * otherwise we could corrupt the inner stream
0548      */
0549     NAPI_GRO_CB(skb)->is_flist = 0;
0550     if (!sk || !udp_sk(sk)->gro_receive) {
0551         if (skb->dev->features & NETIF_F_GRO_FRAGLIST)
0552             NAPI_GRO_CB(skb)->is_flist = sk ? !udp_sk(sk)->gro_enabled : 1;
0553
0554         if ((!sk && (skb->dev->features & NETIF_F_GRO_UDP_FWD)) ||
0555             (sk && udp_sk(sk)->gro_enabled) || NAPI_GRO_CB(skb)->is_flist)
0556             return call_gro_receive(udp_gro_receive_segment, head, skb);
0557
0558         /* no GRO, be sure flush the current packet */
0559         goto out;
0560     }
0561
0562     if (NAPI_GRO_CB(skb)->encap_mark ||
0563         (uh->check && skb->ip_summed != CHECKSUM_PARTIAL &&
0564          NAPI_GRO_CB(skb)->csum_cnt == 0 &&
0565          !NAPI_GRO_CB(skb)->csum_valid))
0566         goto out;
0567
0568     /* mark that this skb passed once through the tunnel gro layer */
0569     NAPI_GRO_CB(skb)->encap_mark = 1;
0570
0571     flush = 0;
0572
0573     list_for_each_entry(p, head, list) {
0574         if (!NAPI_GRO_CB(p)->same_flow)
0575             continue;
0576
0577         uh2 = (struct udphdr   *)(p->data + off);
0578
0579         /* Match ports and either checksums are either both zero
0580          * or nonzero.
0581          */
0582         if ((*(u32 *)&uh->source != *(u32 *)&uh2->source) ||
0583             (!uh->check ^ !uh2->check)) {
0584             NAPI_GRO_CB(p)->same_flow = 0;
0585             continue;
0586         }
0587     }
0588
0589     skb_gro_pull(skb, sizeof(struct udphdr)); /* pull encapsulating udp header */
0590     skb_gro_postpull_rcsum(skb, uh, sizeof(struct udphdr));
0591     pp = call_gro_receive_sk(udp_sk(sk)->gro_receive, sk, head, skb);
0592
0593 out:
0594     skb_gro_flush_final(skb, pp, flush);
0595     return pp;
0596 }
0597 EXPORT_SYMBOL(udp_gro_receive);
0598
0599 static struct sock *udp4_gro_lookup_skb(struct sk_buff *skb, __be16 sport,
0600                     __be16 dport)
0601 {
0602     const struct iphdr *iph = skb_gro_network_header(skb);
0603
0604     return __udp4_lib_lookup(dev_net(skb->dev), iph->saddr, sport,
0605                  iph->daddr, dport, inet_iif(skb),
0606                  inet_sdif(skb), &udp_table, NULL);
0607 }
0608
0609 INDIRECT_CALLABLE_SCOPE
0610 struct sk_buff *udp4_gro_receive(struct list_head *head, struct sk_buff *skb)
0611 {
0612     struct udphdr *uh = udp_gro_udphdr(skb);
0613     struct sock *sk = NULL;
0614     struct sk_buff *pp;
0615
0616     if (unlikely(!uh))
0617         goto flush;
0618
0619     /* Don't bother verifying checksum if we're going to flush anyway. */
0620     if (NAPI_GRO_CB(skb)->flush)
0621         goto skip;
0622
0623     if (skb_gro_checksum_validate_zero_check(skb, IPPROTO_UDP, uh->check,
0624                          inet_gro_compute_pseudo))
0625         goto flush;
0626     else if (uh->check)
0627         skb_gro_checksum_try_convert(skb, IPPROTO_UDP,
0628                          inet_gro_compute_pseudo);
0629 skip:
0630     NAPI_GRO_CB(skb)->is_ipv6 = 0;
0631
0632     if (static_branch_unlikely(&udp_encap_needed_key))
0633         sk = udp4_gro_lookup_skb(skb, uh->source, uh->dest);
0634
0635     pp = udp_gro_receive(head, skb, uh, sk);
0636     return pp;
0637
0638 flush:
0639     NAPI_GRO_CB(skb)->flush = 1;
0640     return NULL;
0641 }
0642
0643 static int udp_gro_complete_segment(struct sk_buff *skb)
0644 {
0645     struct udphdr *uh = udp_hdr(skb);
0646
0647     skb->csum_start = (unsigned char *)uh - skb->head;
0648     skb->csum_offset = offsetof(struct udphdr, check);
0649     skb->ip_summed = CHECKSUM_PARTIAL;
0650
0651     skb_shinfo(skb)->gso_segs = NAPI_GRO_CB(skb)->count;
0652     skb_shinfo(skb)->gso_type |= SKB_GSO_UDP_L4;
0653
0654     if (skb->encapsulation)
0655         skb->inner_transport_header = skb->transport_header;
0656
0657     return 0;
0658 }
0659
0660 int udp_gro_complete(struct sk_buff *skb, int nhoff,
0661              udp_lookup_t lookup)
0662 {
0663     __be16 newlen = htons(skb->len - nhoff);
0664     struct udphdr *uh = (struct udphdr *)(skb->data + nhoff);
0665     struct sock *sk;
0666     int err;
0667
0668     uh->len = newlen;
0669
0670     sk = INDIRECT_CALL_INET(lookup, udp6_lib_lookup_skb,
0671                 udp4_lib_lookup_skb, skb, uh->source, uh->dest);
0672     if (sk && udp_sk(sk)->gro_complete) {
0673         skb_shinfo(skb)->gso_type = uh->check ? SKB_GSO_UDP_TUNNEL_CSUM
0674                     : SKB_GSO_UDP_TUNNEL;
0675
0676         /* clear the encap mark, so that inner frag_list gro_complete
0677          * can take place
0678          */
0679         NAPI_GRO_CB(skb)->encap_mark = 0;
0680
0681         /* Set encapsulation before calling into inner gro_complete()
0682          * functions to make them set up the inner offsets.
0683          */
0684         skb->encapsulation = 1;
0685         err = udp_sk(sk)->gro_complete(sk, skb,
0686                 nhoff + sizeof(struct udphdr));
0687     } else {
0688         err = udp_gro_complete_segment(skb);
0689     }
0690
0691     if (skb->remcsum_offload)
0692         skb_shinfo(skb)->gso_type |= SKB_GSO_TUNNEL_REMCSUM;
0693
0694     return err;
0695 }
0696 EXPORT_SYMBOL(udp_gro_complete);
0697
0698 INDIRECT_CALLABLE_SCOPE int udp4_gro_complete(struct sk_buff *skb, int nhoff)
0699 {
0700     const struct iphdr *iph = ip_hdr(skb);
0701     struct udphdr *uh = (struct udphdr *)(skb->data + nhoff);
0702
0703     /* do fraglist only if there is no outer UDP encap (or we already processed it) */
0704     if (NAPI_GRO_CB(skb)->is_flist && !NAPI_GRO_CB(skb)->encap_mark) {
0705         uh->len = htons(skb->len - nhoff);
0706
0707         skb_shinfo(skb)->gso_type |= (SKB_GSO_FRAGLIST|SKB_GSO_UDP_L4);
0708         skb_shinfo(skb)->gso_segs = NAPI_GRO_CB(skb)->count;
0709
0710         if (skb->ip_summed == CHECKSUM_UNNECESSARY) {
0711             if (skb->csum_level < SKB_MAX_CSUM_LEVEL)
0712                 skb->csum_level++;
0713         } else {
0714             skb->ip_summed = CHECKSUM_UNNECESSARY;
0715             skb->csum_level = 0;
0716         }
0717
0718         return 0;
0719     }
0720
0721     if (uh->check)
0722         uh->check = ~udp_v4_check(skb->len - nhoff, iph->saddr,
0723                       iph->daddr, 0);
0724
0725     return udp_gro_complete(skb, nhoff, udp4_lib_lookup_skb);
0726 }
0727
0728 static const struct net_offload udpv4_offload = {
0729     .callbacks = {
0730         .gso_segment = udp4_ufo_fragment,
0731         .gro_receive  = udp4_gro_receive,
0732         .gro_complete = udp4_gro_complete,
0733     },
0734 };
0735
0736 int __init udpv4_offload_init(void)
0737 {
0738     return inet_add_offload(&udpv4_offload, IPPROTO_UDP);
0739 }