net/xfrm/xfrm_device.c

0001 // SPDX-License-Identifier: GPL-2.0-or-later
0002 /*
0003  * xfrm_device.c - IPsec device offloading code.
0004  *
0005  * Copyright (c) 2015 secunet Security Networks AG
0006  *
0007  * Author:
0008  * Steffen Klassert <steffen.klassert@secunet.com>
0009  */
0010
0011 #include <linux/errno.h>
0012 #include <linux/module.h>
0013 #include <linux/netdevice.h>
0014 #include <linux/skbuff.h>
0015 #include <linux/slab.h>
0016 #include <linux/spinlock.h>
0017 #include <net/dst.h>
0018 #include <net/xfrm.h>
0019 #include <linux/notifier.h>
0020
0021 #ifdef CONFIG_XFRM_OFFLOAD
0022 static void __xfrm_transport_prep(struct xfrm_state *x, struct sk_buff *skb,
0023                   unsigned int hsize)
0024 {
0025     struct xfrm_offload *xo = xfrm_offload(skb);
0026
0027     skb_reset_mac_len(skb);
0028     if (xo->flags & XFRM_GSO_SEGMENT)
0029         skb->transport_header -= x->props.header_len;
0030
0031     pskb_pull(skb, skb_transport_offset(skb) + x->props.header_len);
0032 }
0033
0034 static void __xfrm_mode_tunnel_prep(struct xfrm_state *x, struct sk_buff *skb,
0035                     unsigned int hsize)
0036
0037 {
0038     struct xfrm_offload *xo = xfrm_offload(skb);
0039
0040     if (xo->flags & XFRM_GSO_SEGMENT)
0041         skb->transport_header = skb->network_header + hsize;
0042
0043     skb_reset_mac_len(skb);
0044     pskb_pull(skb, skb->mac_len + x->props.header_len);
0045 }
0046
0047 static void __xfrm_mode_beet_prep(struct xfrm_state *x, struct sk_buff *skb,
0048                   unsigned int hsize)
0049 {
0050     struct xfrm_offload *xo = xfrm_offload(skb);
0051     int phlen = 0;
0052
0053     if (xo->flags & XFRM_GSO_SEGMENT)
0054         skb->transport_header = skb->network_header + hsize;
0055
0056     skb_reset_mac_len(skb);
0057     if (x->sel.family != AF_INET6) {
0058         phlen = IPV4_BEET_PHMAXLEN;
0059         if (x->outer_mode.family == AF_INET6)
0060             phlen += sizeof(struct ipv6hdr) - sizeof(struct iphdr);
0061     }
0062
0063     pskb_pull(skb, skb->mac_len + hsize + (x->props.header_len - phlen));
0064 }
0065
0066 /* Adjust pointers into the packet when IPsec is done at layer2 */
0067 static void xfrm_outer_mode_prep(struct xfrm_state *x, struct sk_buff *skb)
0068 {
0069     switch (x->outer_mode.encap) {
0070     case XFRM_MODE_TUNNEL:
0071         if (x->outer_mode.family == AF_INET)
0072             return __xfrm_mode_tunnel_prep(x, skb,
0073                                sizeof(struct iphdr));
0074         if (x->outer_mode.family == AF_INET6)
0075             return __xfrm_mode_tunnel_prep(x, skb,
0076                                sizeof(struct ipv6hdr));
0077         break;
0078     case XFRM_MODE_TRANSPORT:
0079         if (x->outer_mode.family == AF_INET)
0080             return __xfrm_transport_prep(x, skb,
0081                              sizeof(struct iphdr));
0082         if (x->outer_mode.family == AF_INET6)
0083             return __xfrm_transport_prep(x, skb,
0084                              sizeof(struct ipv6hdr));
0085         break;
0086     case XFRM_MODE_BEET:
0087         if (x->outer_mode.family == AF_INET)
0088             return __xfrm_mode_beet_prep(x, skb,
0089                              sizeof(struct iphdr));
0090         if (x->outer_mode.family == AF_INET6)
0091             return __xfrm_mode_beet_prep(x, skb,
0092                              sizeof(struct ipv6hdr));
0093         break;
0094     case XFRM_MODE_ROUTEOPTIMIZATION:
0095     case XFRM_MODE_IN_TRIGGER:
0096         break;
0097     }
0098 }
0099
0100 struct sk_buff *validate_xmit_xfrm(struct sk_buff *skb, netdev_features_t features, bool *again)
0101 {
0102     int err;
0103     unsigned long flags;
0104     struct xfrm_state *x;
0105     struct softnet_data *sd;
0106     struct sk_buff *skb2, *nskb, *pskb = NULL;
0107     netdev_features_t esp_features = features;
0108     struct xfrm_offload *xo = xfrm_offload(skb);
0109     struct net_device *dev = skb->dev;
0110     struct sec_path *sp;
0111
0112     if (!xo || (xo->flags & XFRM_XMIT))
0113         return skb;
0114
0115     if (!(features & NETIF_F_HW_ESP))
0116         esp_features = features & ~(NETIF_F_SG | NETIF_F_CSUM_MASK);
0117
0118     sp = skb_sec_path(skb);
0119     x = sp->xvec[sp->len - 1];
0120     if (xo->flags & XFRM_GRO || x->xso.dir == XFRM_DEV_OFFLOAD_IN)
0121         return skb;
0122
0123     /* This skb was already validated on the upper/virtual dev */
0124     if ((x->xso.dev != dev) && (x->xso.real_dev == dev))
0125         return skb;
0126
0127     local_irq_save(flags);
0128     sd = this_cpu_ptr(&softnet_data);
0129     err = !skb_queue_empty(&sd->xfrm_backlog);
0130     local_irq_restore(flags);
0131
0132     if (err) {
0133         *again = true;
0134         return skb;
0135     }
0136
0137     if (skb_is_gso(skb) && unlikely(x->xso.dev != dev)) {
0138         struct sk_buff *segs;
0139
0140         /* Packet got rerouted, fixup features and segment it. */
0141         esp_features = esp_features & ~(NETIF_F_HW_ESP | NETIF_F_GSO_ESP);
0142
0143         segs = skb_gso_segment(skb, esp_features);
0144         if (IS_ERR(segs)) {
0145             kfree_skb(skb);
0146             dev_core_stats_tx_dropped_inc(dev);
0147             return NULL;
0148         } else {
0149             consume_skb(skb);
0150             skb = segs;
0151         }
0152     }
0153
0154     if (!skb->next) {
0155         esp_features |= skb->dev->gso_partial_features;
0156         xfrm_outer_mode_prep(x, skb);
0157
0158         xo->flags |= XFRM_DEV_RESUME;
0159
0160         err = x->type_offload->xmit(x, skb, esp_features);
0161         if (err) {
0162             if (err == -EINPROGRESS)
0163                 return NULL;
0164
0165             XFRM_INC_STATS(xs_net(x), LINUX_MIB_XFRMOUTSTATEPROTOERROR);
0166             kfree_skb(skb);
0167             return NULL;
0168         }
0169
0170         skb_push(skb, skb->data - skb_mac_header(skb));
0171
0172         return skb;
0173     }
0174
0175     skb_list_walk_safe(skb, skb2, nskb) {
0176         esp_features |= skb->dev->gso_partial_features;
0177         skb_mark_not_on_list(skb2);
0178
0179         xo = xfrm_offload(skb2);
0180         xo->flags |= XFRM_DEV_RESUME;
0181
0182         xfrm_outer_mode_prep(x, skb2);
0183
0184         err = x->type_offload->xmit(x, skb2, esp_features);
0185         if (!err) {
0186             skb2->next = nskb;
0187         } else if (err != -EINPROGRESS) {
0188             XFRM_INC_STATS(xs_net(x), LINUX_MIB_XFRMOUTSTATEPROTOERROR);
0189             skb2->next = nskb;
0190             kfree_skb_list(skb2);
0191             return NULL;
0192         } else {
0193             if (skb == skb2)
0194                 skb = nskb;
0195             else
0196                 pskb->next = nskb;
0197
0198             continue;
0199         }
0200
0201         skb_push(skb2, skb2->data - skb_mac_header(skb2));
0202         pskb = skb2;
0203     }
0204
0205     return skb;
0206 }
0207 EXPORT_SYMBOL_GPL(validate_xmit_xfrm);
0208
0209 int xfrm_dev_state_add(struct net *net, struct xfrm_state *x,
0210                struct xfrm_user_offload *xuo)
0211 {
0212     int err;
0213     struct dst_entry *dst;
0214     struct net_device *dev;
0215     struct xfrm_dev_offload *xso = &x->xso;
0216     xfrm_address_t *saddr;
0217     xfrm_address_t *daddr;
0218
0219     if (!x->type_offload)
0220         return -EINVAL;
0221
0222     /* We don't yet support UDP encapsulation and TFC padding. */
0223     if (x->encap || x->tfcpad)
0224         return -EINVAL;
0225
0226     if (xuo->flags & ~(XFRM_OFFLOAD_IPV6 | XFRM_OFFLOAD_INBOUND))
0227         return -EINVAL;
0228
0229     dev = dev_get_by_index(net, xuo->ifindex);
0230     if (!dev) {
0231         if (!(xuo->flags & XFRM_OFFLOAD_INBOUND)) {
0232             saddr = &x->props.saddr;
0233             daddr = &x->id.daddr;
0234         } else {
0235             saddr = &x->id.daddr;
0236             daddr = &x->props.saddr;
0237         }
0238
0239         dst = __xfrm_dst_lookup(net, 0, 0, saddr, daddr,
0240                     x->props.family,
0241                     xfrm_smark_get(0, x));
0242         if (IS_ERR(dst))
0243             return 0;
0244
0245         dev = dst->dev;
0246
0247         dev_hold(dev);
0248         dst_release(dst);
0249     }
0250
0251     if (!dev->xfrmdev_ops || !dev->xfrmdev_ops->xdo_dev_state_add) {
0252         xso->dev = NULL;
0253         dev_put(dev);
0254         return 0;
0255     }
0256
0257     if (x->props.flags & XFRM_STATE_ESN &&
0258         !dev->xfrmdev_ops->xdo_dev_state_advance_esn) {
0259         xso->dev = NULL;
0260         dev_put(dev);
0261         return -EINVAL;
0262     }
0263
0264     xso->dev = dev;
0265     netdev_tracker_alloc(dev, &xso->dev_tracker, GFP_ATOMIC);
0266     xso->real_dev = dev;
0267
0268     if (xuo->flags & XFRM_OFFLOAD_INBOUND)
0269         xso->dir = XFRM_DEV_OFFLOAD_IN;
0270     else
0271         xso->dir = XFRM_DEV_OFFLOAD_OUT;
0272
0273     err = dev->xfrmdev_ops->xdo_dev_state_add(x);
0274     if (err) {
0275         xso->dev = NULL;
0276         xso->dir = 0;
0277         xso->real_dev = NULL;
0278         netdev_put(dev, &xso->dev_tracker);
0279
0280         if (err != -EOPNOTSUPP)
0281             return err;
0282     }
0283
0284     return 0;
0285 }
0286 EXPORT_SYMBOL_GPL(xfrm_dev_state_add);
0287
0288 bool xfrm_dev_offload_ok(struct sk_buff *skb, struct xfrm_state *x)
0289 {
0290     int mtu;
0291     struct dst_entry *dst = skb_dst(skb);
0292     struct xfrm_dst *xdst = (struct xfrm_dst *)dst;
0293     struct net_device *dev = x->xso.dev;
0294
0295     if (!x->type_offload || x->encap)
0296         return false;
0297
0298     if ((!dev || (dev == xfrm_dst_path(dst)->dev)) &&
0299         (!xdst->child->xfrm)) {
0300         mtu = xfrm_state_mtu(x, xdst->child_mtu_cached);
0301         if (skb->len <= mtu)
0302             goto ok;
0303
0304         if (skb_is_gso(skb) && skb_gso_validate_network_len(skb, mtu))
0305             goto ok;
0306     }
0307
0308     return false;
0309
0310 ok:
0311     if (dev && dev->xfrmdev_ops && dev->xfrmdev_ops->xdo_dev_offload_ok)
0312         return x->xso.dev->xfrmdev_ops->xdo_dev_offload_ok(skb, x);
0313
0314     return true;
0315 }
0316 EXPORT_SYMBOL_GPL(xfrm_dev_offload_ok);
0317
0318 void xfrm_dev_resume(struct sk_buff *skb)
0319 {
0320     struct net_device *dev = skb->dev;
0321     int ret = NETDEV_TX_BUSY;
0322     struct netdev_queue *txq;
0323     struct softnet_data *sd;
0324     unsigned long flags;
0325
0326     rcu_read_lock();
0327     txq = netdev_core_pick_tx(dev, skb, NULL);
0328
0329     HARD_TX_LOCK(dev, txq, smp_processor_id());
0330     if (!netif_xmit_frozen_or_stopped(txq))
0331         skb = dev_hard_start_xmit(skb, dev, txq, &ret);
0332     HARD_TX_UNLOCK(dev, txq);
0333
0334     if (!dev_xmit_complete(ret)) {
0335         local_irq_save(flags);
0336         sd = this_cpu_ptr(&softnet_data);
0337         skb_queue_tail(&sd->xfrm_backlog, skb);
0338         raise_softirq_irqoff(NET_TX_SOFTIRQ);
0339         local_irq_restore(flags);
0340     }
0341     rcu_read_unlock();
0342 }
0343 EXPORT_SYMBOL_GPL(xfrm_dev_resume);
0344
0345 void xfrm_dev_backlog(struct softnet_data *sd)
0346 {
0347     struct sk_buff_head *xfrm_backlog = &sd->xfrm_backlog;
0348     struct sk_buff_head list;
0349     struct sk_buff *skb;
0350
0351     if (skb_queue_empty(xfrm_backlog))
0352         return;
0353
0354     __skb_queue_head_init(&list);
0355
0356     spin_lock(&xfrm_backlog->lock);
0357     skb_queue_splice_init(xfrm_backlog, &list);
0358     spin_unlock(&xfrm_backlog->lock);
0359
0360     while (!skb_queue_empty(&list)) {
0361         skb = __skb_dequeue(&list);
0362         xfrm_dev_resume(skb);
0363     }
0364
0365 }
0366 #endif
0367
0368 static int xfrm_api_check(struct net_device *dev)
0369 {
0370 #ifdef CONFIG_XFRM_OFFLOAD
0371     if ((dev->features & NETIF_F_HW_ESP_TX_CSUM) &&
0372         !(dev->features & NETIF_F_HW_ESP))
0373         return NOTIFY_BAD;
0374
0375     if ((dev->features & NETIF_F_HW_ESP) &&
0376         (!(dev->xfrmdev_ops &&
0377            dev->xfrmdev_ops->xdo_dev_state_add &&
0378            dev->xfrmdev_ops->xdo_dev_state_delete)))
0379         return NOTIFY_BAD;
0380 #else
0381     if (dev->features & (NETIF_F_HW_ESP | NETIF_F_HW_ESP_TX_CSUM))
0382         return NOTIFY_BAD;
0383 #endif
0384
0385     return NOTIFY_DONE;
0386 }
0387
0388 static int xfrm_dev_down(struct net_device *dev)
0389 {
0390     if (dev->features & NETIF_F_HW_ESP)
0391         xfrm_dev_state_flush(dev_net(dev), dev, true);
0392
0393     return NOTIFY_DONE;
0394 }
0395
0396 static int xfrm_dev_event(struct notifier_block *this, unsigned long event, void *ptr)
0397 {
0398     struct net_device *dev = netdev_notifier_info_to_dev(ptr);
0399
0400     switch (event) {
0401     case NETDEV_REGISTER:
0402         return xfrm_api_check(dev);
0403
0404     case NETDEV_FEAT_CHANGE:
0405         return xfrm_api_check(dev);
0406
0407     case NETDEV_DOWN:
0408     case NETDEV_UNREGISTER:
0409         return xfrm_dev_down(dev);
0410     }
0411     return NOTIFY_DONE;
0412 }
0413
0414 static struct notifier_block xfrm_dev_notifier = {
0415     .notifier_call  = xfrm_dev_event,
0416 };
0417
0418 void __init xfrm_dev_init(void)
0419 {
0420     register_netdevice_notifier(&xfrm_dev_notifier);
0421 }