net/wireguard/peer.c

0001 // SPDX-License-Identifier: GPL-2.0
0002 /*
0003  * Copyright (C) 2015-2019 Jason A. Donenfeld <Jason@zx2c4.com>. All Rights Reserved.
0004  */
0005
0006 #include "peer.h"
0007 #include "device.h"
0008 #include "queueing.h"
0009 #include "timers.h"
0010 #include "peerlookup.h"
0011 #include "noise.h"
0012
0013 #include <linux/kref.h>
0014 #include <linux/lockdep.h>
0015 #include <linux/rcupdate.h>
0016 #include <linux/list.h>
0017
0018 static struct kmem_cache *peer_cache;
0019 static atomic64_t peer_counter = ATOMIC64_INIT(0);
0020
0021 struct wg_peer *wg_peer_create(struct wg_device *wg,
0022                    const u8 public_key[NOISE_PUBLIC_KEY_LEN],
0023                    const u8 preshared_key[NOISE_SYMMETRIC_KEY_LEN])
0024 {
0025     struct wg_peer *peer;
0026     int ret = -ENOMEM;
0027
0028     lockdep_assert_held(&wg->device_update_lock);
0029
0030     if (wg->num_peers >= MAX_PEERS_PER_DEVICE)
0031         return ERR_PTR(ret);
0032
0033     peer = kmem_cache_zalloc(peer_cache, GFP_KERNEL);
0034     if (unlikely(!peer))
0035         return ERR_PTR(ret);
0036     if (unlikely(dst_cache_init(&peer->endpoint_cache, GFP_KERNEL)))
0037         goto err;
0038
0039     peer->device = wg;
0040     wg_noise_handshake_init(&peer->handshake, &wg->static_identity,
0041                 public_key, preshared_key, peer);
0042     peer->internal_id = atomic64_inc_return(&peer_counter);
0043     peer->serial_work_cpu = nr_cpumask_bits;
0044     wg_cookie_init(&peer->latest_cookie);
0045     wg_timers_init(peer);
0046     wg_cookie_checker_precompute_peer_keys(peer);
0047     spin_lock_init(&peer->keypairs.keypair_update_lock);
0048     INIT_WORK(&peer->transmit_handshake_work, wg_packet_handshake_send_worker);
0049     INIT_WORK(&peer->transmit_packet_work, wg_packet_tx_worker);
0050     wg_prev_queue_init(&peer->tx_queue);
0051     wg_prev_queue_init(&peer->rx_queue);
0052     rwlock_init(&peer->endpoint_lock);
0053     kref_init(&peer->refcount);
0054     skb_queue_head_init(&peer->staged_packet_queue);
0055     wg_noise_reset_last_sent_handshake(&peer->last_sent_handshake);
0056     set_bit(NAPI_STATE_NO_BUSY_POLL, &peer->napi.state);
0057     netif_napi_add(wg->dev, &peer->napi, wg_packet_rx_poll,
0058                NAPI_POLL_WEIGHT);
0059     napi_enable(&peer->napi);
0060     list_add_tail(&peer->peer_list, &wg->peer_list);
0061     INIT_LIST_HEAD(&peer->allowedips_list);
0062     wg_pubkey_hashtable_add(wg->peer_hashtable, peer);
0063     ++wg->num_peers;
0064     pr_debug("%s: Peer %llu created\n", wg->dev->name, peer->internal_id);
0065     return peer;
0066
0067 err:
0068     kmem_cache_free(peer_cache, peer);
0069     return ERR_PTR(ret);
0070 }
0071
0072 struct wg_peer *wg_peer_get_maybe_zero(struct wg_peer *peer)
0073 {
0074     RCU_LOCKDEP_WARN(!rcu_read_lock_bh_held(),
0075              "Taking peer reference without holding the RCU read lock");
0076     if (unlikely(!peer || !kref_get_unless_zero(&peer->refcount)))
0077         return NULL;
0078     return peer;
0079 }
0080
0081 static void peer_make_dead(struct wg_peer *peer)
0082 {
0083     /* Remove from configuration-time lookup structures. */
0084     list_del_init(&peer->peer_list);
0085     wg_allowedips_remove_by_peer(&peer->device->peer_allowedips, peer,
0086                      &peer->device->device_update_lock);
0087     wg_pubkey_hashtable_remove(peer->device->peer_hashtable, peer);
0088
0089     /* Mark as dead, so that we don't allow jumping contexts after. */
0090     WRITE_ONCE(peer->is_dead, true);
0091
0092     /* The caller must now synchronize_net() for this to take effect. */
0093 }
0094
0095 static void peer_remove_after_dead(struct wg_peer *peer)
0096 {
0097     WARN_ON(!peer->is_dead);
0098
0099     /* No more keypairs can be created for this peer, since is_dead protects
0100      * add_new_keypair, so we can now destroy existing ones.
0101      */
0102     wg_noise_keypairs_clear(&peer->keypairs);
0103
0104     /* Destroy all ongoing timers that were in-flight at the beginning of
0105      * this function.
0106      */
0107     wg_timers_stop(peer);
0108
0109     /* The transition between packet encryption/decryption queues isn't
0110      * guarded by is_dead, but each reference's life is strictly bounded by
0111      * two generations: once for parallel crypto and once for serial
0112      * ingestion, so we can simply flush twice, and be sure that we no
0113      * longer have references inside these queues.
0114      */
0115
0116     /* a) For encrypt/decrypt. */
0117     flush_workqueue(peer->device->packet_crypt_wq);
0118     /* b.1) For send (but not receive, since that's napi). */
0119     flush_workqueue(peer->device->packet_crypt_wq);
0120     /* b.2.1) For receive (but not send, since that's wq). */
0121     napi_disable(&peer->napi);
0122     /* b.2.1) It's now safe to remove the napi struct, which must be done
0123      * here from process context.
0124      */
0125     netif_napi_del(&peer->napi);
0126
0127     /* Ensure any workstructs we own (like transmit_handshake_work or
0128      * clear_peer_work) no longer are in use.
0129      */
0130     flush_workqueue(peer->device->handshake_send_wq);
0131
0132     /* After the above flushes, a peer might still be active in a few
0133      * different contexts: 1) from xmit(), before hitting is_dead and
0134      * returning, 2) from wg_packet_consume_data(), before hitting is_dead
0135      * and returning, 3) from wg_receive_handshake_packet() after a point
0136      * where it has processed an incoming handshake packet, but where
0137      * all calls to pass it off to timers fails because of is_dead. We won't
0138      * have new references in (1) eventually, because we're removed from
0139      * allowedips; we won't have new references in (2) eventually, because
0140      * wg_index_hashtable_lookup will always return NULL, since we removed
0141      * all existing keypairs and no more can be created; we won't have new
0142      * references in (3) eventually, because we're removed from the pubkey
0143      * hash table, which allows for a maximum of one handshake response,
0144      * via the still-uncleared index hashtable entry, but not more than one,
0145      * and in wg_cookie_message_consume, the lookup eventually gets a peer
0146      * with a refcount of zero, so no new reference is taken.
0147      */
0148
0149     --peer->device->num_peers;
0150     wg_peer_put(peer);
0151 }
0152
0153 /* We have a separate "remove" function make sure that all active places where
0154  * a peer is currently operating will eventually come to an end and not pass
0155  * their reference onto another context.
0156  */
0157 void wg_peer_remove(struct wg_peer *peer)
0158 {
0159     if (unlikely(!peer))
0160         return;
0161     lockdep_assert_held(&peer->device->device_update_lock);
0162
0163     peer_make_dead(peer);
0164     synchronize_net();
0165     peer_remove_after_dead(peer);
0166 }
0167
0168 void wg_peer_remove_all(struct wg_device *wg)
0169 {
0170     struct wg_peer *peer, *temp;
0171     LIST_HEAD(dead_peers);
0172
0173     lockdep_assert_held(&wg->device_update_lock);
0174
0175     /* Avoid having to traverse individually for each one. */
0176     wg_allowedips_free(&wg->peer_allowedips, &wg->device_update_lock);
0177
0178     list_for_each_entry_safe(peer, temp, &wg->peer_list, peer_list) {
0179         peer_make_dead(peer);
0180         list_add_tail(&peer->peer_list, &dead_peers);
0181     }
0182     synchronize_net();
0183     list_for_each_entry_safe(peer, temp, &dead_peers, peer_list)
0184         peer_remove_after_dead(peer);
0185 }
0186
0187 static void rcu_release(struct rcu_head *rcu)
0188 {
0189     struct wg_peer *peer = container_of(rcu, struct wg_peer, rcu);
0190
0191     dst_cache_destroy(&peer->endpoint_cache);
0192     WARN_ON(wg_prev_queue_peek(&peer->tx_queue) || wg_prev_queue_peek(&peer->rx_queue));
0193
0194     /* The final zeroing takes care of clearing any remaining handshake key
0195      * material and other potentially sensitive information.
0196      */
0197     memzero_explicit(peer, sizeof(*peer));
0198     kmem_cache_free(peer_cache, peer);
0199 }
0200
0201 static void kref_release(struct kref *refcount)
0202 {
0203     struct wg_peer *peer = container_of(refcount, struct wg_peer, refcount);
0204
0205     pr_debug("%s: Peer %llu (%pISpfsc) destroyed\n",
0206          peer->device->dev->name, peer->internal_id,
0207          &peer->endpoint.addr);
0208
0209     /* Remove ourself from dynamic runtime lookup structures, now that the
0210      * last reference is gone.
0211      */
0212     wg_index_hashtable_remove(peer->device->index_hashtable,
0213                   &peer->handshake.entry);
0214
0215     /* Remove any lingering packets that didn't have a chance to be
0216      * transmitted.
0217      */
0218     wg_packet_purge_staged_packets(peer);
0219
0220     /* Free the memory used. */
0221     call_rcu(&peer->rcu, rcu_release);
0222 }
0223
0224 void wg_peer_put(struct wg_peer *peer)
0225 {
0226     if (unlikely(!peer))
0227         return;
0228     kref_put(&peer->refcount, kref_release);
0229 }
0230
0231 int __init wg_peer_init(void)
0232 {
0233     peer_cache = KMEM_CACHE(wg_peer, 0);
0234     return peer_cache ? 0 : -ENOMEM;
0235 }
0236
0237 void wg_peer_uninit(void)
0238 {
0239     kmem_cache_destroy(peer_cache);
0240 }