net/ipv6/ip6_offload.c

0001 // SPDX-License-Identifier: GPL-2.0-or-later
0002 /*
0003  *  IPV6 GSO/GRO offload support
0004  *  Linux INET6 implementation
0005  */
0006
0007 #include <linux/kernel.h>
0008 #include <linux/socket.h>
0009 #include <linux/netdevice.h>
0010 #include <linux/skbuff.h>
0011 #include <linux/printk.h>
0012
0013 #include <net/protocol.h>
0014 #include <net/ipv6.h>
0015 #include <net/inet_common.h>
0016 #include <net/tcp.h>
0017 #include <net/udp.h>
0018 #include <net/gro.h>
0019
0020 #include "ip6_offload.h"
0021
0022 /* All GRO functions are always builtin, except UDP over ipv6, which lays in
0023  * ipv6 module, as it depends on UDPv6 lookup function, so we need special care
0024  * when ipv6 is built as a module
0025  */
0026 #if IS_BUILTIN(CONFIG_IPV6)
0027 #define INDIRECT_CALL_L4(f, f2, f1, ...) INDIRECT_CALL_2(f, f2, f1, __VA_ARGS__)
0028 #else
0029 #define INDIRECT_CALL_L4(f, f2, f1, ...) INDIRECT_CALL_1(f, f2, __VA_ARGS__)
0030 #endif
0031
0032 #define indirect_call_gro_receive_l4(f2, f1, cb, head, skb) \
0033 ({                              \
0034     unlikely(gro_recursion_inc_test(skb)) ?         \
0035         NAPI_GRO_CB(skb)->flush |= 1, NULL :        \
0036         INDIRECT_CALL_L4(cb, f2, f1, head, skb);    \
0037 })
0038
0039 static int ipv6_gso_pull_exthdrs(struct sk_buff *skb, int proto)
0040 {
0041     const struct net_offload *ops = NULL;
0042
0043     for (;;) {
0044         struct ipv6_opt_hdr *opth;
0045         int len;
0046
0047         if (proto != NEXTHDR_HOP) {
0048             ops = rcu_dereference(inet6_offloads[proto]);
0049
0050             if (unlikely(!ops))
0051                 break;
0052
0053             if (!(ops->flags & INET6_PROTO_GSO_EXTHDR))
0054                 break;
0055         }
0056
0057         if (unlikely(!pskb_may_pull(skb, 8)))
0058             break;
0059
0060         opth = (void *)skb->data;
0061         len = ipv6_optlen(opth);
0062
0063         if (unlikely(!pskb_may_pull(skb, len)))
0064             break;
0065
0066         opth = (void *)skb->data;
0067         proto = opth->nexthdr;
0068         __skb_pull(skb, len);
0069     }
0070
0071     return proto;
0072 }
0073
0074 static struct sk_buff *ipv6_gso_segment(struct sk_buff *skb,
0075     netdev_features_t features)
0076 {
0077     struct sk_buff *segs = ERR_PTR(-EINVAL);
0078     struct ipv6hdr *ipv6h;
0079     const struct net_offload *ops;
0080     int proto, nexthdr;
0081     struct frag_hdr *fptr;
0082     unsigned int payload_len;
0083     u8 *prevhdr;
0084     int offset = 0;
0085     bool encap, udpfrag;
0086     int nhoff;
0087     bool gso_partial;
0088
0089     skb_reset_network_header(skb);
0090     nexthdr = ipv6_has_hopopt_jumbo(skb);
0091     if (nexthdr) {
0092         const int hophdr_len = sizeof(struct hop_jumbo_hdr);
0093         int err;
0094
0095         err = skb_cow_head(skb, 0);
0096         if (err < 0)
0097             return ERR_PTR(err);
0098
0099         /* remove the HBH header.
0100          * Layout: [Ethernet header][IPv6 header][HBH][TCP header]
0101          */
0102         memmove(skb_mac_header(skb) + hophdr_len,
0103             skb_mac_header(skb),
0104             ETH_HLEN + sizeof(struct ipv6hdr));
0105         skb->data += hophdr_len;
0106         skb->len -= hophdr_len;
0107         skb->network_header += hophdr_len;
0108         skb->mac_header += hophdr_len;
0109         ipv6h = (struct ipv6hdr *)skb->data;
0110         ipv6h->nexthdr = nexthdr;
0111     }
0112     nhoff = skb_network_header(skb) - skb_mac_header(skb);
0113     if (unlikely(!pskb_may_pull(skb, sizeof(*ipv6h))))
0114         goto out;
0115
0116     encap = SKB_GSO_CB(skb)->encap_level > 0;
0117     if (encap)
0118         features &= skb->dev->hw_enc_features;
0119     SKB_GSO_CB(skb)->encap_level += sizeof(*ipv6h);
0120
0121     ipv6h = ipv6_hdr(skb);
0122     __skb_pull(skb, sizeof(*ipv6h));
0123     segs = ERR_PTR(-EPROTONOSUPPORT);
0124
0125     proto = ipv6_gso_pull_exthdrs(skb, ipv6h->nexthdr);
0126
0127     if (skb->encapsulation &&
0128         skb_shinfo(skb)->gso_type & (SKB_GSO_IPXIP4 | SKB_GSO_IPXIP6))
0129         udpfrag = proto == IPPROTO_UDP && encap &&
0130               (skb_shinfo(skb)->gso_type & SKB_GSO_UDP);
0131     else
0132         udpfrag = proto == IPPROTO_UDP && !skb->encapsulation &&
0133               (skb_shinfo(skb)->gso_type & SKB_GSO_UDP);
0134
0135     ops = rcu_dereference(inet6_offloads[proto]);
0136     if (likely(ops && ops->callbacks.gso_segment)) {
0137         skb_reset_transport_header(skb);
0138         segs = ops->callbacks.gso_segment(skb, features);
0139         if (!segs)
0140             skb->network_header = skb_mac_header(skb) + nhoff - skb->head;
0141     }
0142
0143     if (IS_ERR_OR_NULL(segs))
0144         goto out;
0145
0146     gso_partial = !!(skb_shinfo(segs)->gso_type & SKB_GSO_PARTIAL);
0147
0148     for (skb = segs; skb; skb = skb->next) {
0149         ipv6h = (struct ipv6hdr *)(skb_mac_header(skb) + nhoff);
0150         if (gso_partial && skb_is_gso(skb))
0151             payload_len = skb_shinfo(skb)->gso_size +
0152                       SKB_GSO_CB(skb)->data_offset +
0153                       skb->head - (unsigned char *)(ipv6h + 1);
0154         else
0155             payload_len = skb->len - nhoff - sizeof(*ipv6h);
0156         ipv6h->payload_len = htons(payload_len);
0157         skb->network_header = (u8 *)ipv6h - skb->head;
0158         skb_reset_mac_len(skb);
0159
0160         if (udpfrag) {
0161             int err = ip6_find_1stfragopt(skb, &prevhdr);
0162             if (err < 0) {
0163                 kfree_skb_list(segs);
0164                 return ERR_PTR(err);
0165             }
0166             fptr = (struct frag_hdr *)((u8 *)ipv6h + err);
0167             fptr->frag_off = htons(offset);
0168             if (skb->next)
0169                 fptr->frag_off |= htons(IP6_MF);
0170             offset += (ntohs(ipv6h->payload_len) -
0171                    sizeof(struct frag_hdr));
0172         }
0173         if (encap)
0174             skb_reset_inner_headers(skb);
0175     }
0176
0177 out:
0178     return segs;
0179 }
0180
0181 /* Return the total length of all the extension hdrs, following the same
0182  * logic in ipv6_gso_pull_exthdrs() when parsing ext-hdrs.
0183  */
0184 static int ipv6_exthdrs_len(struct ipv6hdr *iph,
0185                 const struct net_offload **opps)
0186 {
0187     struct ipv6_opt_hdr *opth = (void *)iph;
0188     int len = 0, proto, optlen = sizeof(*iph);
0189
0190     proto = iph->nexthdr;
0191     for (;;) {
0192         if (proto != NEXTHDR_HOP) {
0193             *opps = rcu_dereference(inet6_offloads[proto]);
0194             if (unlikely(!(*opps)))
0195                 break;
0196             if (!((*opps)->flags & INET6_PROTO_GSO_EXTHDR))
0197                 break;
0198         }
0199         opth = (void *)opth + optlen;
0200         optlen = ipv6_optlen(opth);
0201         len += optlen;
0202         proto = opth->nexthdr;
0203     }
0204     return len;
0205 }
0206
0207 INDIRECT_CALLABLE_SCOPE struct sk_buff *ipv6_gro_receive(struct list_head *head,
0208                              struct sk_buff *skb)
0209 {
0210     const struct net_offload *ops;
0211     struct sk_buff *pp = NULL;
0212     struct sk_buff *p;
0213     struct ipv6hdr *iph;
0214     unsigned int nlen;
0215     unsigned int hlen;
0216     unsigned int off;
0217     u16 flush = 1;
0218     int proto;
0219
0220     off = skb_gro_offset(skb);
0221     hlen = off + sizeof(*iph);
0222     iph = skb_gro_header_fast(skb, off);
0223     if (skb_gro_header_hard(skb, hlen)) {
0224         iph = skb_gro_header_slow(skb, hlen, off);
0225         if (unlikely(!iph))
0226             goto out;
0227     }
0228
0229     skb_set_network_header(skb, off);
0230     skb_gro_pull(skb, sizeof(*iph));
0231     skb_set_transport_header(skb, skb_gro_offset(skb));
0232
0233     flush += ntohs(iph->payload_len) != skb_gro_len(skb);
0234
0235     proto = iph->nexthdr;
0236     ops = rcu_dereference(inet6_offloads[proto]);
0237     if (!ops || !ops->callbacks.gro_receive) {
0238         __pskb_pull(skb, skb_gro_offset(skb));
0239         skb_gro_frag0_invalidate(skb);
0240         proto = ipv6_gso_pull_exthdrs(skb, proto);
0241         skb_gro_pull(skb, -skb_transport_offset(skb));
0242         skb_reset_transport_header(skb);
0243         __skb_push(skb, skb_gro_offset(skb));
0244
0245         ops = rcu_dereference(inet6_offloads[proto]);
0246         if (!ops || !ops->callbacks.gro_receive)
0247             goto out;
0248
0249         iph = ipv6_hdr(skb);
0250     }
0251
0252     NAPI_GRO_CB(skb)->proto = proto;
0253
0254     flush--;
0255     nlen = skb_network_header_len(skb);
0256
0257     list_for_each_entry(p, head, list) {
0258         const struct ipv6hdr *iph2;
0259         __be32 first_word; /* <Version:4><Traffic_Class:8><Flow_Label:20> */
0260
0261         if (!NAPI_GRO_CB(p)->same_flow)
0262             continue;
0263
0264         iph2 = (struct ipv6hdr *)(p->data + off);
0265         first_word = *(__be32 *)iph ^ *(__be32 *)iph2;
0266
0267         /* All fields must match except length and Traffic Class.
0268          * XXX skbs on the gro_list have all been parsed and pulled
0269          * already so we don't need to compare nlen
0270          * (nlen != (sizeof(*iph2) + ipv6_exthdrs_len(iph2, &ops)))
0271          * memcmp() alone below is sufficient, right?
0272          */
0273          if ((first_word & htonl(0xF00FFFFF)) ||
0274              !ipv6_addr_equal(&iph->saddr, &iph2->saddr) ||
0275              !ipv6_addr_equal(&iph->daddr, &iph2->daddr) ||
0276              iph->nexthdr != iph2->nexthdr) {
0277 not_same_flow:
0278             NAPI_GRO_CB(p)->same_flow = 0;
0279             continue;
0280         }
0281         if (unlikely(nlen > sizeof(struct ipv6hdr))) {
0282             if (memcmp(iph + 1, iph2 + 1,
0283                    nlen - sizeof(struct ipv6hdr)))
0284                 goto not_same_flow;
0285         }
0286         /* flush if Traffic Class fields are different */
0287         NAPI_GRO_CB(p)->flush |= !!((first_word & htonl(0x0FF00000)) |
0288             (__force __be32)(iph->hop_limit ^ iph2->hop_limit));
0289         NAPI_GRO_CB(p)->flush |= flush;
0290
0291         /* If the previous IP ID value was based on an atomic
0292          * datagram we can overwrite the value and ignore it.
0293          */
0294         if (NAPI_GRO_CB(skb)->is_atomic)
0295             NAPI_GRO_CB(p)->flush_id = 0;
0296     }
0297
0298     NAPI_GRO_CB(skb)->is_atomic = true;
0299     NAPI_GRO_CB(skb)->flush |= flush;
0300
0301     skb_gro_postpull_rcsum(skb, iph, nlen);
0302
0303     pp = indirect_call_gro_receive_l4(tcp6_gro_receive, udp6_gro_receive,
0304                      ops->callbacks.gro_receive, head, skb);
0305
0306 out:
0307     skb_gro_flush_final(skb, pp, flush);
0308
0309     return pp;
0310 }
0311
0312 static struct sk_buff *sit_ip6ip6_gro_receive(struct list_head *head,
0313                           struct sk_buff *skb)
0314 {
0315     /* Common GRO receive for SIT and IP6IP6 */
0316
0317     if (NAPI_GRO_CB(skb)->encap_mark) {
0318         NAPI_GRO_CB(skb)->flush = 1;
0319         return NULL;
0320     }
0321
0322     NAPI_GRO_CB(skb)->encap_mark = 1;
0323
0324     return ipv6_gro_receive(head, skb);
0325 }
0326
0327 static struct sk_buff *ip4ip6_gro_receive(struct list_head *head,
0328                       struct sk_buff *skb)
0329 {
0330     /* Common GRO receive for SIT and IP6IP6 */
0331
0332     if (NAPI_GRO_CB(skb)->encap_mark) {
0333         NAPI_GRO_CB(skb)->flush = 1;
0334         return NULL;
0335     }
0336
0337     NAPI_GRO_CB(skb)->encap_mark = 1;
0338
0339     return inet_gro_receive(head, skb);
0340 }
0341
0342 INDIRECT_CALLABLE_SCOPE int ipv6_gro_complete(struct sk_buff *skb, int nhoff)
0343 {
0344     const struct net_offload *ops;
0345     struct ipv6hdr *iph;
0346     int err = -ENOSYS;
0347     u32 payload_len;
0348
0349     if (skb->encapsulation) {
0350         skb_set_inner_protocol(skb, cpu_to_be16(ETH_P_IPV6));
0351         skb_set_inner_network_header(skb, nhoff);
0352     }
0353
0354     payload_len = skb->len - nhoff - sizeof(*iph);
0355     if (unlikely(payload_len > IPV6_MAXPLEN)) {
0356         struct hop_jumbo_hdr *hop_jumbo;
0357         int hoplen = sizeof(*hop_jumbo);
0358
0359         /* Move network header left */
0360         memmove(skb_mac_header(skb) - hoplen, skb_mac_header(skb),
0361             skb->transport_header - skb->mac_header);
0362         skb->data -= hoplen;
0363         skb->len += hoplen;
0364         skb->mac_header -= hoplen;
0365         skb->network_header -= hoplen;
0366         iph = (struct ipv6hdr *)(skb->data + nhoff);
0367         hop_jumbo = (struct hop_jumbo_hdr *)(iph + 1);
0368
0369         /* Build hop-by-hop options */
0370         hop_jumbo->nexthdr = iph->nexthdr;
0371         hop_jumbo->hdrlen = 0;
0372         hop_jumbo->tlv_type = IPV6_TLV_JUMBO;
0373         hop_jumbo->tlv_len = 4;
0374         hop_jumbo->jumbo_payload_len = htonl(payload_len + hoplen);
0375
0376         iph->nexthdr = NEXTHDR_HOP;
0377         iph->payload_len = 0;
0378     } else {
0379         iph = (struct ipv6hdr *)(skb->data + nhoff);
0380         iph->payload_len = htons(payload_len);
0381     }
0382
0383     nhoff += sizeof(*iph) + ipv6_exthdrs_len(iph, &ops);
0384     if (WARN_ON(!ops || !ops->callbacks.gro_complete))
0385         goto out;
0386
0387     err = INDIRECT_CALL_L4(ops->callbacks.gro_complete, tcp6_gro_complete,
0388                    udp6_gro_complete, skb, nhoff);
0389
0390 out:
0391     return err;
0392 }
0393
0394 static int sit_gro_complete(struct sk_buff *skb, int nhoff)
0395 {
0396     skb->encapsulation = 1;
0397     skb_shinfo(skb)->gso_type |= SKB_GSO_IPXIP4;
0398     return ipv6_gro_complete(skb, nhoff);
0399 }
0400
0401 static int ip6ip6_gro_complete(struct sk_buff *skb, int nhoff)
0402 {
0403     skb->encapsulation = 1;
0404     skb_shinfo(skb)->gso_type |= SKB_GSO_IPXIP6;
0405     return ipv6_gro_complete(skb, nhoff);
0406 }
0407
0408 static int ip4ip6_gro_complete(struct sk_buff *skb, int nhoff)
0409 {
0410     skb->encapsulation = 1;
0411     skb_shinfo(skb)->gso_type |= SKB_GSO_IPXIP6;
0412     return inet_gro_complete(skb, nhoff);
0413 }
0414
0415 static struct packet_offload ipv6_packet_offload __read_mostly = {
0416     .type = cpu_to_be16(ETH_P_IPV6),
0417     .callbacks = {
0418         .gso_segment = ipv6_gso_segment,
0419         .gro_receive = ipv6_gro_receive,
0420         .gro_complete = ipv6_gro_complete,
0421     },
0422 };
0423
0424 static struct sk_buff *sit_gso_segment(struct sk_buff *skb,
0425                        netdev_features_t features)
0426 {
0427     if (!(skb_shinfo(skb)->gso_type & SKB_GSO_IPXIP4))
0428         return ERR_PTR(-EINVAL);
0429
0430     return ipv6_gso_segment(skb, features);
0431 }
0432
0433 static struct sk_buff *ip4ip6_gso_segment(struct sk_buff *skb,
0434                       netdev_features_t features)
0435 {
0436     if (!(skb_shinfo(skb)->gso_type & SKB_GSO_IPXIP6))
0437         return ERR_PTR(-EINVAL);
0438
0439     return inet_gso_segment(skb, features);
0440 }
0441
0442 static struct sk_buff *ip6ip6_gso_segment(struct sk_buff *skb,
0443                       netdev_features_t features)
0444 {
0445     if (!(skb_shinfo(skb)->gso_type & SKB_GSO_IPXIP6))
0446         return ERR_PTR(-EINVAL);
0447
0448     return ipv6_gso_segment(skb, features);
0449 }
0450
0451 static const struct net_offload sit_offload = {
0452     .callbacks = {
0453         .gso_segment    = sit_gso_segment,
0454         .gro_receive    = sit_ip6ip6_gro_receive,
0455         .gro_complete   = sit_gro_complete,
0456     },
0457 };
0458
0459 static const struct net_offload ip4ip6_offload = {
0460     .callbacks = {
0461         .gso_segment    = ip4ip6_gso_segment,
0462         .gro_receive    = ip4ip6_gro_receive,
0463         .gro_complete   = ip4ip6_gro_complete,
0464     },
0465 };
0466
0467 static const struct net_offload ip6ip6_offload = {
0468     .callbacks = {
0469         .gso_segment    = ip6ip6_gso_segment,
0470         .gro_receive    = sit_ip6ip6_gro_receive,
0471         .gro_complete   = ip6ip6_gro_complete,
0472     },
0473 };
0474 static int __init ipv6_offload_init(void)
0475 {
0476
0477     if (tcpv6_offload_init() < 0)
0478         pr_crit("%s: Cannot add TCP protocol offload\n", __func__);
0479     if (ipv6_exthdrs_offload_init() < 0)
0480         pr_crit("%s: Cannot add EXTHDRS protocol offload\n", __func__);
0481
0482     dev_add_offload(&ipv6_packet_offload);
0483
0484     inet_add_offload(&sit_offload, IPPROTO_IPV6);
0485     inet6_add_offload(&ip6ip6_offload, IPPROTO_IPV6);
0486     inet6_add_offload(&ip4ip6_offload, IPPROTO_IPIP);
0487
0488     return 0;
0489 }
0490
0491 fs_initcall(ipv6_offload_init);