net/ipv6/esp6_offload.c

0001 // SPDX-License-Identifier: GPL-2.0-only
0002 /*
0003  * IPV6 GSO/GRO offload support
0004  * Linux INET implementation
0005  *
0006  * Copyright (C) 2016 secunet Security Networks AG
0007  * Author: Steffen Klassert <steffen.klassert@secunet.com>
0008  *
0009  * ESP GRO support
0010  */
0011
0012 #include <linux/skbuff.h>
0013 #include <linux/init.h>
0014 #include <net/protocol.h>
0015 #include <crypto/aead.h>
0016 #include <crypto/authenc.h>
0017 #include <linux/err.h>
0018 #include <linux/module.h>
0019 #include <net/gro.h>
0020 #include <net/ip.h>
0021 #include <net/xfrm.h>
0022 #include <net/esp.h>
0023 #include <linux/scatterlist.h>
0024 #include <linux/kernel.h>
0025 #include <linux/slab.h>
0026 #include <linux/spinlock.h>
0027 #include <net/ip6_route.h>
0028 #include <net/ipv6.h>
0029 #include <linux/icmpv6.h>
0030
0031 static __u16 esp6_nexthdr_esp_offset(struct ipv6hdr *ipv6_hdr, int nhlen)
0032 {
0033     int off = sizeof(struct ipv6hdr);
0034     struct ipv6_opt_hdr *exthdr;
0035
0036     if (likely(ipv6_hdr->nexthdr == NEXTHDR_ESP))
0037         return offsetof(struct ipv6hdr, nexthdr);
0038
0039     while (off < nhlen) {
0040         exthdr = (void *)ipv6_hdr + off;
0041         if (exthdr->nexthdr == NEXTHDR_ESP)
0042             return off;
0043
0044         off += ipv6_optlen(exthdr);
0045     }
0046
0047     return 0;
0048 }
0049
0050 static struct sk_buff *esp6_gro_receive(struct list_head *head,
0051                     struct sk_buff *skb)
0052 {
0053     int offset = skb_gro_offset(skb);
0054     struct xfrm_offload *xo;
0055     struct xfrm_state *x;
0056     __be32 seq;
0057     __be32 spi;
0058     int nhoff;
0059     int err;
0060
0061     if (!pskb_pull(skb, offset))
0062         return NULL;
0063
0064     if ((err = xfrm_parse_spi(skb, IPPROTO_ESP, &spi, &seq)) != 0)
0065         goto out;
0066
0067     xo = xfrm_offload(skb);
0068     if (!xo || !(xo->flags & CRYPTO_DONE)) {
0069         struct sec_path *sp = secpath_set(skb);
0070
0071         if (!sp)
0072             goto out;
0073
0074         if (sp->len == XFRM_MAX_DEPTH)
0075             goto out_reset;
0076
0077         x = xfrm_state_lookup(dev_net(skb->dev), skb->mark,
0078                       (xfrm_address_t *)&ipv6_hdr(skb)->daddr,
0079                       spi, IPPROTO_ESP, AF_INET6);
0080         if (!x)
0081             goto out_reset;
0082
0083         skb->mark = xfrm_smark_get(skb->mark, x);
0084
0085         sp->xvec[sp->len++] = x;
0086         sp->olen++;
0087
0088         xo = xfrm_offload(skb);
0089         if (!xo)
0090             goto out_reset;
0091     }
0092
0093     xo->flags |= XFRM_GRO;
0094
0095     nhoff = esp6_nexthdr_esp_offset(ipv6_hdr(skb), offset);
0096     if (!nhoff)
0097         goto out;
0098
0099     IP6CB(skb)->nhoff = nhoff;
0100     XFRM_TUNNEL_SKB_CB(skb)->tunnel.ip6 = NULL;
0101     XFRM_SPI_SKB_CB(skb)->family = AF_INET6;
0102     XFRM_SPI_SKB_CB(skb)->daddroff = offsetof(struct ipv6hdr, daddr);
0103     XFRM_SPI_SKB_CB(skb)->seq = seq;
0104
0105     /* We don't need to handle errors from xfrm_input, it does all
0106      * the error handling and frees the resources on error. */
0107     xfrm_input(skb, IPPROTO_ESP, spi, -2);
0108
0109     return ERR_PTR(-EINPROGRESS);
0110 out_reset:
0111     secpath_reset(skb);
0112 out:
0113     skb_push(skb, offset);
0114     NAPI_GRO_CB(skb)->same_flow = 0;
0115     NAPI_GRO_CB(skb)->flush = 1;
0116
0117     return NULL;
0118 }
0119
0120 static void esp6_gso_encap(struct xfrm_state *x, struct sk_buff *skb)
0121 {
0122     struct ip_esp_hdr *esph;
0123     struct ipv6hdr *iph = ipv6_hdr(skb);
0124     struct xfrm_offload *xo = xfrm_offload(skb);
0125     u8 proto = iph->nexthdr;
0126
0127     skb_push(skb, -skb_network_offset(skb));
0128
0129     if (x->outer_mode.encap == XFRM_MODE_TRANSPORT) {
0130         __be16 frag;
0131
0132         ipv6_skip_exthdr(skb, sizeof(struct ipv6hdr), &proto, &frag);
0133     }
0134
0135     esph = ip_esp_hdr(skb);
0136     *skb_mac_header(skb) = IPPROTO_ESP;
0137
0138     esph->spi = x->id.spi;
0139     esph->seq_no = htonl(XFRM_SKB_CB(skb)->seq.output.low);
0140
0141     xo->proto = proto;
0142 }
0143
0144 static struct sk_buff *xfrm6_tunnel_gso_segment(struct xfrm_state *x,
0145                         struct sk_buff *skb,
0146                         netdev_features_t features)
0147 {
0148     return skb_eth_gso_segment(skb, features, htons(ETH_P_IPV6));
0149 }
0150
0151 static struct sk_buff *xfrm6_transport_gso_segment(struct xfrm_state *x,
0152                            struct sk_buff *skb,
0153                            netdev_features_t features)
0154 {
0155     const struct net_offload *ops;
0156     struct sk_buff *segs = ERR_PTR(-EINVAL);
0157     struct xfrm_offload *xo = xfrm_offload(skb);
0158
0159     skb->transport_header += x->props.header_len;
0160     ops = rcu_dereference(inet6_offloads[xo->proto]);
0161     if (likely(ops && ops->callbacks.gso_segment))
0162         segs = ops->callbacks.gso_segment(skb, features);
0163
0164     return segs;
0165 }
0166
0167 static struct sk_buff *xfrm6_beet_gso_segment(struct xfrm_state *x,
0168                           struct sk_buff *skb,
0169                           netdev_features_t features)
0170 {
0171     struct xfrm_offload *xo = xfrm_offload(skb);
0172     struct sk_buff *segs = ERR_PTR(-EINVAL);
0173     const struct net_offload *ops;
0174     u8 proto = xo->proto;
0175
0176     skb->transport_header += x->props.header_len;
0177
0178     if (x->sel.family != AF_INET6) {
0179         skb->transport_header -=
0180             (sizeof(struct ipv6hdr) - sizeof(struct iphdr));
0181
0182         if (proto == IPPROTO_BEETPH) {
0183             struct ip_beet_phdr *ph =
0184                 (struct ip_beet_phdr *)skb->data;
0185
0186             skb->transport_header += ph->hdrlen * 8;
0187             proto = ph->nexthdr;
0188         } else {
0189             skb->transport_header -= IPV4_BEET_PHMAXLEN;
0190         }
0191
0192         if (proto == IPPROTO_TCP)
0193             skb_shinfo(skb)->gso_type |= SKB_GSO_TCPV6;
0194     } else {
0195         __be16 frag;
0196
0197         skb->transport_header +=
0198             ipv6_skip_exthdr(skb, 0, &proto, &frag);
0199     }
0200
0201     if (proto == IPPROTO_IPIP)
0202         skb_shinfo(skb)->gso_type |= SKB_GSO_IPXIP6;
0203
0204     __skb_pull(skb, skb_transport_offset(skb));
0205     ops = rcu_dereference(inet6_offloads[proto]);
0206     if (likely(ops && ops->callbacks.gso_segment))
0207         segs = ops->callbacks.gso_segment(skb, features);
0208
0209     return segs;
0210 }
0211
0212 static struct sk_buff *xfrm6_outer_mode_gso_segment(struct xfrm_state *x,
0213                             struct sk_buff *skb,
0214                             netdev_features_t features)
0215 {
0216     switch (x->outer_mode.encap) {
0217     case XFRM_MODE_TUNNEL:
0218         return xfrm6_tunnel_gso_segment(x, skb, features);
0219     case XFRM_MODE_TRANSPORT:
0220         return xfrm6_transport_gso_segment(x, skb, features);
0221     case XFRM_MODE_BEET:
0222         return xfrm6_beet_gso_segment(x, skb, features);
0223     }
0224
0225     return ERR_PTR(-EOPNOTSUPP);
0226 }
0227
0228 static struct sk_buff *esp6_gso_segment(struct sk_buff *skb,
0229                         netdev_features_t features)
0230 {
0231     struct xfrm_state *x;
0232     struct ip_esp_hdr *esph;
0233     struct crypto_aead *aead;
0234     netdev_features_t esp_features = features;
0235     struct xfrm_offload *xo = xfrm_offload(skb);
0236     struct sec_path *sp;
0237
0238     if (!xo)
0239         return ERR_PTR(-EINVAL);
0240
0241     if (!(skb_shinfo(skb)->gso_type & SKB_GSO_ESP))
0242         return ERR_PTR(-EINVAL);
0243
0244     sp = skb_sec_path(skb);
0245     x = sp->xvec[sp->len - 1];
0246     aead = x->data;
0247     esph = ip_esp_hdr(skb);
0248
0249     if (esph->spi != x->id.spi)
0250         return ERR_PTR(-EINVAL);
0251
0252     if (!pskb_may_pull(skb, sizeof(*esph) + crypto_aead_ivsize(aead)))
0253         return ERR_PTR(-EINVAL);
0254
0255     __skb_pull(skb, sizeof(*esph) + crypto_aead_ivsize(aead));
0256
0257     skb->encap_hdr_csum = 1;
0258
0259     if (!(features & NETIF_F_HW_ESP) || x->xso.dev != skb->dev)
0260         esp_features = features & ~(NETIF_F_SG | NETIF_F_CSUM_MASK |
0261                         NETIF_F_SCTP_CRC);
0262     else if (!(features & NETIF_F_HW_ESP_TX_CSUM))
0263         esp_features = features & ~(NETIF_F_CSUM_MASK |
0264                         NETIF_F_SCTP_CRC);
0265
0266     xo->flags |= XFRM_GSO_SEGMENT;
0267
0268     return xfrm6_outer_mode_gso_segment(x, skb, esp_features);
0269 }
0270
0271 static int esp6_input_tail(struct xfrm_state *x, struct sk_buff *skb)
0272 {
0273     struct crypto_aead *aead = x->data;
0274     struct xfrm_offload *xo = xfrm_offload(skb);
0275
0276     if (!pskb_may_pull(skb, sizeof(struct ip_esp_hdr) + crypto_aead_ivsize(aead)))
0277         return -EINVAL;
0278
0279     if (!(xo->flags & CRYPTO_DONE))
0280         skb->ip_summed = CHECKSUM_NONE;
0281
0282     return esp6_input_done2(skb, 0);
0283 }
0284
0285 static int esp6_xmit(struct xfrm_state *x, struct sk_buff *skb,  netdev_features_t features)
0286 {
0287     int len;
0288     int err;
0289     int alen;
0290     int blksize;
0291     struct xfrm_offload *xo;
0292     struct crypto_aead *aead;
0293     struct esp_info esp;
0294     bool hw_offload = true;
0295     __u32 seq;
0296
0297     esp.inplace = true;
0298
0299     xo = xfrm_offload(skb);
0300
0301     if (!xo)
0302         return -EINVAL;
0303
0304     if (!(features & NETIF_F_HW_ESP) || x->xso.dev != skb->dev) {
0305         xo->flags |= CRYPTO_FALLBACK;
0306         hw_offload = false;
0307     }
0308
0309     esp.proto = xo->proto;
0310
0311     /* skb is pure payload to encrypt */
0312
0313     aead = x->data;
0314     alen = crypto_aead_authsize(aead);
0315
0316     esp.tfclen = 0;
0317     /* XXX: Add support for tfc padding here. */
0318
0319     blksize = ALIGN(crypto_aead_blocksize(aead), 4);
0320     esp.clen = ALIGN(skb->len + 2 + esp.tfclen, blksize);
0321     esp.plen = esp.clen - skb->len - esp.tfclen;
0322     esp.tailen = esp.tfclen + esp.plen + alen;
0323
0324     if (!hw_offload || !skb_is_gso(skb)) {
0325         esp.nfrags = esp6_output_head(x, skb, &esp);
0326         if (esp.nfrags < 0)
0327             return esp.nfrags;
0328     }
0329
0330     seq = xo->seq.low;
0331
0332     esp.esph = ip_esp_hdr(skb);
0333     esp.esph->spi = x->id.spi;
0334
0335     skb_push(skb, -skb_network_offset(skb));
0336
0337     if (xo->flags & XFRM_GSO_SEGMENT) {
0338         esp.esph->seq_no = htonl(seq);
0339
0340         if (!skb_is_gso(skb))
0341             xo->seq.low++;
0342         else
0343             xo->seq.low += skb_shinfo(skb)->gso_segs;
0344     }
0345
0346     esp.seqno = cpu_to_be64(xo->seq.low + ((u64)xo->seq.hi << 32));
0347
0348     len = skb->len - sizeof(struct ipv6hdr);
0349     if (len > IPV6_MAXPLEN)
0350         len = 0;
0351
0352     ipv6_hdr(skb)->payload_len = htons(len);
0353
0354     if (hw_offload) {
0355         if (!skb_ext_add(skb, SKB_EXT_SEC_PATH))
0356             return -ENOMEM;
0357
0358         xo = xfrm_offload(skb);
0359         if (!xo)
0360             return -EINVAL;
0361
0362         xo->flags |= XFRM_XMIT;
0363         return 0;
0364     }
0365
0366     err = esp6_output_tail(x, skb, &esp);
0367     if (err)
0368         return err;
0369
0370     secpath_reset(skb);
0371
0372     return 0;
0373 }
0374
0375 static const struct net_offload esp6_offload = {
0376     .callbacks = {
0377         .gro_receive = esp6_gro_receive,
0378         .gso_segment = esp6_gso_segment,
0379     },
0380 };
0381
0382 static const struct xfrm_type_offload esp6_type_offload = {
0383     .owner      = THIS_MODULE,
0384     .proto          = IPPROTO_ESP,
0385     .input_tail = esp6_input_tail,
0386     .xmit       = esp6_xmit,
0387     .encap      = esp6_gso_encap,
0388 };
0389
0390 static int __init esp6_offload_init(void)
0391 {
0392     if (xfrm_register_type_offload(&esp6_type_offload, AF_INET6) < 0) {
0393         pr_info("%s: can't add xfrm type offload\n", __func__);
0394         return -EAGAIN;
0395     }
0396
0397     return inet6_add_offload(&esp6_offload, IPPROTO_ESP);
0398 }
0399
0400 static void __exit esp6_offload_exit(void)
0401 {
0402     xfrm_unregister_type_offload(&esp6_type_offload, AF_INET6);
0403     inet6_del_offload(&esp6_offload, IPPROTO_ESP);
0404 }
0405
0406 module_init(esp6_offload_init);
0407 module_exit(esp6_offload_exit);
0408 MODULE_LICENSE("GPL");
0409 MODULE_AUTHOR("Steffen Klassert <steffen.klassert@secunet.com>");
0410 MODULE_ALIAS_XFRM_OFFLOAD_TYPE(AF_INET6, XFRM_PROTO_ESP);
0411 MODULE_DESCRIPTION("IPV6 GSO/GRO offload support");