net/wireguard/device.c

0001 // SPDX-License-Identifier: GPL-2.0
0002 /*
0003  * Copyright (C) 2015-2019 Jason A. Donenfeld <Jason@zx2c4.com>. All Rights Reserved.
0004  */
0005
0006 #include "queueing.h"
0007 #include "socket.h"
0008 #include "timers.h"
0009 #include "device.h"
0010 #include "ratelimiter.h"
0011 #include "peer.h"
0012 #include "messages.h"
0013
0014 #include <linux/module.h>
0015 #include <linux/rtnetlink.h>
0016 #include <linux/inet.h>
0017 #include <linux/netdevice.h>
0018 #include <linux/inetdevice.h>
0019 #include <linux/if_arp.h>
0020 #include <linux/icmp.h>
0021 #include <linux/suspend.h>
0022 #include <net/dst_metadata.h>
0023 #include <net/icmp.h>
0024 #include <net/rtnetlink.h>
0025 #include <net/ip_tunnels.h>
0026 #include <net/addrconf.h>
0027
0028 static LIST_HEAD(device_list);
0029
0030 static int wg_open(struct net_device *dev)
0031 {
0032     struct in_device *dev_v4 = __in_dev_get_rtnl(dev);
0033     struct inet6_dev *dev_v6 = __in6_dev_get(dev);
0034     struct wg_device *wg = netdev_priv(dev);
0035     struct wg_peer *peer;
0036     int ret;
0037
0038     if (dev_v4) {
0039         /* At some point we might put this check near the ip_rt_send_
0040          * redirect call of ip_forward in net/ipv4/ip_forward.c, similar
0041          * to the current secpath check.
0042          */
0043         IN_DEV_CONF_SET(dev_v4, SEND_REDIRECTS, false);
0044         IPV4_DEVCONF_ALL(dev_net(dev), SEND_REDIRECTS) = false;
0045     }
0046     if (dev_v6)
0047         dev_v6->cnf.addr_gen_mode = IN6_ADDR_GEN_MODE_NONE;
0048
0049     mutex_lock(&wg->device_update_lock);
0050     ret = wg_socket_init(wg, wg->incoming_port);
0051     if (ret < 0)
0052         goto out;
0053     list_for_each_entry(peer, &wg->peer_list, peer_list) {
0054         wg_packet_send_staged_packets(peer);
0055         if (peer->persistent_keepalive_interval)
0056             wg_packet_send_keepalive(peer);
0057     }
0058 out:
0059     mutex_unlock(&wg->device_update_lock);
0060     return ret;
0061 }
0062
0063 static int wg_pm_notification(struct notifier_block *nb, unsigned long action, void *data)
0064 {
0065     struct wg_device *wg;
0066     struct wg_peer *peer;
0067
0068     /* If the machine is constantly suspending and resuming, as part of
0069      * its normal operation rather than as a somewhat rare event, then we
0070      * don't actually want to clear keys.
0071      */
0072     if (IS_ENABLED(CONFIG_PM_AUTOSLEEP) ||
0073         IS_ENABLED(CONFIG_PM_USERSPACE_AUTOSLEEP))
0074         return 0;
0075
0076     if (action != PM_HIBERNATION_PREPARE && action != PM_SUSPEND_PREPARE)
0077         return 0;
0078
0079     rtnl_lock();
0080     list_for_each_entry(wg, &device_list, device_list) {
0081         mutex_lock(&wg->device_update_lock);
0082         list_for_each_entry(peer, &wg->peer_list, peer_list) {
0083             del_timer(&peer->timer_zero_key_material);
0084             wg_noise_handshake_clear(&peer->handshake);
0085             wg_noise_keypairs_clear(&peer->keypairs);
0086         }
0087         mutex_unlock(&wg->device_update_lock);
0088     }
0089     rtnl_unlock();
0090     rcu_barrier();
0091     return 0;
0092 }
0093
0094 static struct notifier_block pm_notifier = { .notifier_call = wg_pm_notification };
0095
0096 static int wg_vm_notification(struct notifier_block *nb, unsigned long action, void *data)
0097 {
0098     struct wg_device *wg;
0099     struct wg_peer *peer;
0100
0101     rtnl_lock();
0102     list_for_each_entry(wg, &device_list, device_list) {
0103         mutex_lock(&wg->device_update_lock);
0104         list_for_each_entry(peer, &wg->peer_list, peer_list)
0105             wg_noise_expire_current_peer_keypairs(peer);
0106         mutex_unlock(&wg->device_update_lock);
0107     }
0108     rtnl_unlock();
0109     return 0;
0110 }
0111
0112 static struct notifier_block vm_notifier = { .notifier_call = wg_vm_notification };
0113
0114 static int wg_stop(struct net_device *dev)
0115 {
0116     struct wg_device *wg = netdev_priv(dev);
0117     struct wg_peer *peer;
0118     struct sk_buff *skb;
0119
0120     mutex_lock(&wg->device_update_lock);
0121     list_for_each_entry(peer, &wg->peer_list, peer_list) {
0122         wg_packet_purge_staged_packets(peer);
0123         wg_timers_stop(peer);
0124         wg_noise_handshake_clear(&peer->handshake);
0125         wg_noise_keypairs_clear(&peer->keypairs);
0126         wg_noise_reset_last_sent_handshake(&peer->last_sent_handshake);
0127     }
0128     mutex_unlock(&wg->device_update_lock);
0129     while ((skb = ptr_ring_consume(&wg->handshake_queue.ring)) != NULL)
0130         kfree_skb(skb);
0131     atomic_set(&wg->handshake_queue_len, 0);
0132     wg_socket_reinit(wg, NULL, NULL);
0133     return 0;
0134 }
0135
0136 static netdev_tx_t wg_xmit(struct sk_buff *skb, struct net_device *dev)
0137 {
0138     struct wg_device *wg = netdev_priv(dev);
0139     struct sk_buff_head packets;
0140     struct wg_peer *peer;
0141     struct sk_buff *next;
0142     sa_family_t family;
0143     u32 mtu;
0144     int ret;
0145
0146     if (unlikely(!wg_check_packet_protocol(skb))) {
0147         ret = -EPROTONOSUPPORT;
0148         net_dbg_ratelimited("%s: Invalid IP packet\n", dev->name);
0149         goto err;
0150     }
0151
0152     peer = wg_allowedips_lookup_dst(&wg->peer_allowedips, skb);
0153     if (unlikely(!peer)) {
0154         ret = -ENOKEY;
0155         if (skb->protocol == htons(ETH_P_IP))
0156             net_dbg_ratelimited("%s: No peer has allowed IPs matching %pI4\n",
0157                         dev->name, &ip_hdr(skb)->daddr);
0158         else if (skb->protocol == htons(ETH_P_IPV6))
0159             net_dbg_ratelimited("%s: No peer has allowed IPs matching %pI6\n",
0160                         dev->name, &ipv6_hdr(skb)->daddr);
0161         goto err_icmp;
0162     }
0163
0164     family = READ_ONCE(peer->endpoint.addr.sa_family);
0165     if (unlikely(family != AF_INET && family != AF_INET6)) {
0166         ret = -EDESTADDRREQ;
0167         net_dbg_ratelimited("%s: No valid endpoint has been configured or discovered for peer %llu\n",
0168                     dev->name, peer->internal_id);
0169         goto err_peer;
0170     }
0171
0172     mtu = skb_valid_dst(skb) ? dst_mtu(skb_dst(skb)) : dev->mtu;
0173
0174     __skb_queue_head_init(&packets);
0175     if (!skb_is_gso(skb)) {
0176         skb_mark_not_on_list(skb);
0177     } else {
0178         struct sk_buff *segs = skb_gso_segment(skb, 0);
0179
0180         if (IS_ERR(segs)) {
0181             ret = PTR_ERR(segs);
0182             goto err_peer;
0183         }
0184         dev_kfree_skb(skb);
0185         skb = segs;
0186     }
0187
0188     skb_list_walk_safe(skb, skb, next) {
0189         skb_mark_not_on_list(skb);
0190
0191         skb = skb_share_check(skb, GFP_ATOMIC);
0192         if (unlikely(!skb))
0193             continue;
0194
0195         /* We only need to keep the original dst around for icmp,
0196          * so at this point we're in a position to drop it.
0197          */
0198         skb_dst_drop(skb);
0199
0200         PACKET_CB(skb)->mtu = mtu;
0201
0202         __skb_queue_tail(&packets, skb);
0203     }
0204
0205     spin_lock_bh(&peer->staged_packet_queue.lock);
0206     /* If the queue is getting too big, we start removing the oldest packets
0207      * until it's small again. We do this before adding the new packet, so
0208      * we don't remove GSO segments that are in excess.
0209      */
0210     while (skb_queue_len(&peer->staged_packet_queue) > MAX_STAGED_PACKETS) {
0211         dev_kfree_skb(__skb_dequeue(&peer->staged_packet_queue));
0212         ++dev->stats.tx_dropped;
0213     }
0214     skb_queue_splice_tail(&packets, &peer->staged_packet_queue);
0215     spin_unlock_bh(&peer->staged_packet_queue.lock);
0216
0217     wg_packet_send_staged_packets(peer);
0218
0219     wg_peer_put(peer);
0220     return NETDEV_TX_OK;
0221
0222 err_peer:
0223     wg_peer_put(peer);
0224 err_icmp:
0225     if (skb->protocol == htons(ETH_P_IP))
0226         icmp_ndo_send(skb, ICMP_DEST_UNREACH, ICMP_HOST_UNREACH, 0);
0227     else if (skb->protocol == htons(ETH_P_IPV6))
0228         icmpv6_ndo_send(skb, ICMPV6_DEST_UNREACH, ICMPV6_ADDR_UNREACH, 0);
0229 err:
0230     ++dev->stats.tx_errors;
0231     kfree_skb(skb);
0232     return ret;
0233 }
0234
0235 static const struct net_device_ops netdev_ops = {
0236     .ndo_open       = wg_open,
0237     .ndo_stop       = wg_stop,
0238     .ndo_start_xmit     = wg_xmit,
0239     .ndo_get_stats64    = dev_get_tstats64
0240 };
0241
0242 static void wg_destruct(struct net_device *dev)
0243 {
0244     struct wg_device *wg = netdev_priv(dev);
0245
0246     rtnl_lock();
0247     list_del(&wg->device_list);
0248     rtnl_unlock();
0249     mutex_lock(&wg->device_update_lock);
0250     rcu_assign_pointer(wg->creating_net, NULL);
0251     wg->incoming_port = 0;
0252     wg_socket_reinit(wg, NULL, NULL);
0253     /* The final references are cleared in the below calls to destroy_workqueue. */
0254     wg_peer_remove_all(wg);
0255     destroy_workqueue(wg->handshake_receive_wq);
0256     destroy_workqueue(wg->handshake_send_wq);
0257     destroy_workqueue(wg->packet_crypt_wq);
0258     wg_packet_queue_free(&wg->handshake_queue, true);
0259     wg_packet_queue_free(&wg->decrypt_queue, false);
0260     wg_packet_queue_free(&wg->encrypt_queue, false);
0261     rcu_barrier(); /* Wait for all the peers to be actually freed. */
0262     wg_ratelimiter_uninit();
0263     memzero_explicit(&wg->static_identity, sizeof(wg->static_identity));
0264     free_percpu(dev->tstats);
0265     kvfree(wg->index_hashtable);
0266     kvfree(wg->peer_hashtable);
0267     mutex_unlock(&wg->device_update_lock);
0268
0269     pr_debug("%s: Interface destroyed\n", dev->name);
0270     free_netdev(dev);
0271 }
0272
0273 static const struct device_type device_type = { .name = KBUILD_MODNAME };
0274
0275 static void wg_setup(struct net_device *dev)
0276 {
0277     struct wg_device *wg = netdev_priv(dev);
0278     enum { WG_NETDEV_FEATURES = NETIF_F_HW_CSUM | NETIF_F_RXCSUM |
0279                     NETIF_F_SG | NETIF_F_GSO |
0280                     NETIF_F_GSO_SOFTWARE | NETIF_F_HIGHDMA };
0281     const int overhead = MESSAGE_MINIMUM_LENGTH + sizeof(struct udphdr) +
0282                  max(sizeof(struct ipv6hdr), sizeof(struct iphdr));
0283
0284     dev->netdev_ops = &netdev_ops;
0285     dev->header_ops = &ip_tunnel_header_ops;
0286     dev->hard_header_len = 0;
0287     dev->addr_len = 0;
0288     dev->needed_headroom = DATA_PACKET_HEAD_ROOM;
0289     dev->needed_tailroom = noise_encrypted_len(MESSAGE_PADDING_MULTIPLE);
0290     dev->type = ARPHRD_NONE;
0291     dev->flags = IFF_POINTOPOINT | IFF_NOARP;
0292     dev->priv_flags |= IFF_NO_QUEUE;
0293     dev->features |= NETIF_F_LLTX;
0294     dev->features |= WG_NETDEV_FEATURES;
0295     dev->hw_features |= WG_NETDEV_FEATURES;
0296     dev->hw_enc_features |= WG_NETDEV_FEATURES;
0297     dev->mtu = ETH_DATA_LEN - overhead;
0298     dev->max_mtu = round_down(INT_MAX, MESSAGE_PADDING_MULTIPLE) - overhead;
0299
0300     SET_NETDEV_DEVTYPE(dev, &device_type);
0301
0302     /* We need to keep the dst around in case of icmp replies. */
0303     netif_keep_dst(dev);
0304
0305     memset(wg, 0, sizeof(*wg));
0306     wg->dev = dev;
0307 }
0308
0309 static int wg_newlink(struct net *src_net, struct net_device *dev,
0310               struct nlattr *tb[], struct nlattr *data[],
0311               struct netlink_ext_ack *extack)
0312 {
0313     struct wg_device *wg = netdev_priv(dev);
0314     int ret = -ENOMEM;
0315
0316     rcu_assign_pointer(wg->creating_net, src_net);
0317     init_rwsem(&wg->static_identity.lock);
0318     mutex_init(&wg->socket_update_lock);
0319     mutex_init(&wg->device_update_lock);
0320     wg_allowedips_init(&wg->peer_allowedips);
0321     wg_cookie_checker_init(&wg->cookie_checker, wg);
0322     INIT_LIST_HEAD(&wg->peer_list);
0323     wg->device_update_gen = 1;
0324
0325     wg->peer_hashtable = wg_pubkey_hashtable_alloc();
0326     if (!wg->peer_hashtable)
0327         return ret;
0328
0329     wg->index_hashtable = wg_index_hashtable_alloc();
0330     if (!wg->index_hashtable)
0331         goto err_free_peer_hashtable;
0332
0333     dev->tstats = netdev_alloc_pcpu_stats(struct pcpu_sw_netstats);
0334     if (!dev->tstats)
0335         goto err_free_index_hashtable;
0336
0337     wg->handshake_receive_wq = alloc_workqueue("wg-kex-%s",
0338             WQ_CPU_INTENSIVE | WQ_FREEZABLE, 0, dev->name);
0339     if (!wg->handshake_receive_wq)
0340         goto err_free_tstats;
0341
0342     wg->handshake_send_wq = alloc_workqueue("wg-kex-%s",
0343             WQ_UNBOUND | WQ_FREEZABLE, 0, dev->name);
0344     if (!wg->handshake_send_wq)
0345         goto err_destroy_handshake_receive;
0346
0347     wg->packet_crypt_wq = alloc_workqueue("wg-crypt-%s",
0348             WQ_CPU_INTENSIVE | WQ_MEM_RECLAIM, 0, dev->name);
0349     if (!wg->packet_crypt_wq)
0350         goto err_destroy_handshake_send;
0351
0352     ret = wg_packet_queue_init(&wg->encrypt_queue, wg_packet_encrypt_worker,
0353                    MAX_QUEUED_PACKETS);
0354     if (ret < 0)
0355         goto err_destroy_packet_crypt;
0356
0357     ret = wg_packet_queue_init(&wg->decrypt_queue, wg_packet_decrypt_worker,
0358                    MAX_QUEUED_PACKETS);
0359     if (ret < 0)
0360         goto err_free_encrypt_queue;
0361
0362     ret = wg_packet_queue_init(&wg->handshake_queue, wg_packet_handshake_receive_worker,
0363                    MAX_QUEUED_INCOMING_HANDSHAKES);
0364     if (ret < 0)
0365         goto err_free_decrypt_queue;
0366
0367     ret = wg_ratelimiter_init();
0368     if (ret < 0)
0369         goto err_free_handshake_queue;
0370
0371     ret = register_netdevice(dev);
0372     if (ret < 0)
0373         goto err_uninit_ratelimiter;
0374
0375     list_add(&wg->device_list, &device_list);
0376
0377     /* We wait until the end to assign priv_destructor, so that
0378      * register_netdevice doesn't call it for us if it fails.
0379      */
0380     dev->priv_destructor = wg_destruct;
0381
0382     pr_debug("%s: Interface created\n", dev->name);
0383     return ret;
0384
0385 err_uninit_ratelimiter:
0386     wg_ratelimiter_uninit();
0387 err_free_handshake_queue:
0388     wg_packet_queue_free(&wg->handshake_queue, false);
0389 err_free_decrypt_queue:
0390     wg_packet_queue_free(&wg->decrypt_queue, false);
0391 err_free_encrypt_queue:
0392     wg_packet_queue_free(&wg->encrypt_queue, false);
0393 err_destroy_packet_crypt:
0394     destroy_workqueue(wg->packet_crypt_wq);
0395 err_destroy_handshake_send:
0396     destroy_workqueue(wg->handshake_send_wq);
0397 err_destroy_handshake_receive:
0398     destroy_workqueue(wg->handshake_receive_wq);
0399 err_free_tstats:
0400     free_percpu(dev->tstats);
0401 err_free_index_hashtable:
0402     kvfree(wg->index_hashtable);
0403 err_free_peer_hashtable:
0404     kvfree(wg->peer_hashtable);
0405     return ret;
0406 }
0407
0408 static struct rtnl_link_ops link_ops __read_mostly = {
0409     .kind           = KBUILD_MODNAME,
0410     .priv_size      = sizeof(struct wg_device),
0411     .setup          = wg_setup,
0412     .newlink        = wg_newlink,
0413 };
0414
0415 static void wg_netns_pre_exit(struct net *net)
0416 {
0417     struct wg_device *wg;
0418     struct wg_peer *peer;
0419
0420     rtnl_lock();
0421     list_for_each_entry(wg, &device_list, device_list) {
0422         if (rcu_access_pointer(wg->creating_net) == net) {
0423             pr_debug("%s: Creating namespace exiting\n", wg->dev->name);
0424             netif_carrier_off(wg->dev);
0425             mutex_lock(&wg->device_update_lock);
0426             rcu_assign_pointer(wg->creating_net, NULL);
0427             wg_socket_reinit(wg, NULL, NULL);
0428             list_for_each_entry(peer, &wg->peer_list, peer_list)
0429                 wg_socket_clear_peer_endpoint_src(peer);
0430             mutex_unlock(&wg->device_update_lock);
0431         }
0432     }
0433     rtnl_unlock();
0434 }
0435
0436 static struct pernet_operations pernet_ops = {
0437     .pre_exit = wg_netns_pre_exit
0438 };
0439
0440 int __init wg_device_init(void)
0441 {
0442     int ret;
0443
0444     ret = register_pm_notifier(&pm_notifier);
0445     if (ret)
0446         return ret;
0447
0448     ret = register_random_vmfork_notifier(&vm_notifier);
0449     if (ret)
0450         goto error_pm;
0451
0452     ret = register_pernet_device(&pernet_ops);
0453     if (ret)
0454         goto error_vm;
0455
0456     ret = rtnl_link_register(&link_ops);
0457     if (ret)
0458         goto error_pernet;
0459
0460     return 0;
0461
0462 error_pernet:
0463     unregister_pernet_device(&pernet_ops);
0464 error_vm:
0465     unregister_random_vmfork_notifier(&vm_notifier);
0466 error_pm:
0467     unregister_pm_notifier(&pm_notifier);
0468     return ret;
0469 }
0470
0471 void wg_device_uninit(void)
0472 {
0473     rtnl_link_unregister(&link_ops);
0474     unregister_pernet_device(&pernet_ops);
0475     unregister_random_vmfork_notifier(&vm_notifier);
0476     unregister_pm_notifier(&pm_notifier);
0477     rcu_barrier();
0478 }