net/wireguard/send.c

0001 // SPDX-License-Identifier: GPL-2.0
0002 /*
0003  * Copyright (C) 2015-2019 Jason A. Donenfeld <Jason@zx2c4.com>. All Rights Reserved.
0004  */
0005
0006 #include "queueing.h"
0007 #include "timers.h"
0008 #include "device.h"
0009 #include "peer.h"
0010 #include "socket.h"
0011 #include "messages.h"
0012 #include "cookie.h"
0013
0014 #include <linux/uio.h>
0015 #include <linux/inetdevice.h>
0016 #include <linux/socket.h>
0017 #include <net/ip_tunnels.h>
0018 #include <net/udp.h>
0019 #include <net/sock.h>
0020
0021 static void wg_packet_send_handshake_initiation(struct wg_peer *peer)
0022 {
0023     struct message_handshake_initiation packet;
0024
0025     if (!wg_birthdate_has_expired(atomic64_read(&peer->last_sent_handshake),
0026                       REKEY_TIMEOUT))
0027         return; /* This function is rate limited. */
0028
0029     atomic64_set(&peer->last_sent_handshake, ktime_get_coarse_boottime_ns());
0030     net_dbg_ratelimited("%s: Sending handshake initiation to peer %llu (%pISpfsc)\n",
0031                 peer->device->dev->name, peer->internal_id,
0032                 &peer->endpoint.addr);
0033
0034     if (wg_noise_handshake_create_initiation(&packet, &peer->handshake)) {
0035         wg_cookie_add_mac_to_packet(&packet, sizeof(packet), peer);
0036         wg_timers_any_authenticated_packet_traversal(peer);
0037         wg_timers_any_authenticated_packet_sent(peer);
0038         atomic64_set(&peer->last_sent_handshake,
0039                  ktime_get_coarse_boottime_ns());
0040         wg_socket_send_buffer_to_peer(peer, &packet, sizeof(packet),
0041                           HANDSHAKE_DSCP);
0042         wg_timers_handshake_initiated(peer);
0043     }
0044 }
0045
0046 void wg_packet_handshake_send_worker(struct work_struct *work)
0047 {
0048     struct wg_peer *peer = container_of(work, struct wg_peer,
0049                         transmit_handshake_work);
0050
0051     wg_packet_send_handshake_initiation(peer);
0052     wg_peer_put(peer);
0053 }
0054
0055 void wg_packet_send_queued_handshake_initiation(struct wg_peer *peer,
0056                         bool is_retry)
0057 {
0058     if (!is_retry)
0059         peer->timer_handshake_attempts = 0;
0060
0061     rcu_read_lock_bh();
0062     /* We check last_sent_handshake here in addition to the actual function
0063      * we're queueing up, so that we don't queue things if not strictly
0064      * necessary:
0065      */
0066     if (!wg_birthdate_has_expired(atomic64_read(&peer->last_sent_handshake),
0067                       REKEY_TIMEOUT) ||
0068             unlikely(READ_ONCE(peer->is_dead)))
0069         goto out;
0070
0071     wg_peer_get(peer);
0072     /* Queues up calling packet_send_queued_handshakes(peer), where we do a
0073      * peer_put(peer) after:
0074      */
0075     if (!queue_work(peer->device->handshake_send_wq,
0076             &peer->transmit_handshake_work))
0077         /* If the work was already queued, we want to drop the
0078          * extra reference:
0079          */
0080         wg_peer_put(peer);
0081 out:
0082     rcu_read_unlock_bh();
0083 }
0084
0085 void wg_packet_send_handshake_response(struct wg_peer *peer)
0086 {
0087     struct message_handshake_response packet;
0088
0089     atomic64_set(&peer->last_sent_handshake, ktime_get_coarse_boottime_ns());
0090     net_dbg_ratelimited("%s: Sending handshake response to peer %llu (%pISpfsc)\n",
0091                 peer->device->dev->name, peer->internal_id,
0092                 &peer->endpoint.addr);
0093
0094     if (wg_noise_handshake_create_response(&packet, &peer->handshake)) {
0095         wg_cookie_add_mac_to_packet(&packet, sizeof(packet), peer);
0096         if (wg_noise_handshake_begin_session(&peer->handshake,
0097                              &peer->keypairs)) {
0098             wg_timers_session_derived(peer);
0099             wg_timers_any_authenticated_packet_traversal(peer);
0100             wg_timers_any_authenticated_packet_sent(peer);
0101             atomic64_set(&peer->last_sent_handshake,
0102                      ktime_get_coarse_boottime_ns());
0103             wg_socket_send_buffer_to_peer(peer, &packet,
0104                               sizeof(packet),
0105                               HANDSHAKE_DSCP);
0106         }
0107     }
0108 }
0109
0110 void wg_packet_send_handshake_cookie(struct wg_device *wg,
0111                      struct sk_buff *initiating_skb,
0112                      __le32 sender_index)
0113 {
0114     struct message_handshake_cookie packet;
0115
0116     net_dbg_skb_ratelimited("%s: Sending cookie response for denied handshake message for %pISpfsc\n",
0117                 wg->dev->name, initiating_skb);
0118     wg_cookie_message_create(&packet, initiating_skb, sender_index,
0119                  &wg->cookie_checker);
0120     wg_socket_send_buffer_as_reply_to_skb(wg, initiating_skb, &packet,
0121                           sizeof(packet));
0122 }
0123
0124 static void keep_key_fresh(struct wg_peer *peer)
0125 {
0126     struct noise_keypair *keypair;
0127     bool send;
0128
0129     rcu_read_lock_bh();
0130     keypair = rcu_dereference_bh(peer->keypairs.current_keypair);
0131     send = keypair && READ_ONCE(keypair->sending.is_valid) &&
0132            (atomic64_read(&keypair->sending_counter) > REKEY_AFTER_MESSAGES ||
0133         (keypair->i_am_the_initiator &&
0134          wg_birthdate_has_expired(keypair->sending.birthdate, REKEY_AFTER_TIME)));
0135     rcu_read_unlock_bh();
0136
0137     if (unlikely(send))
0138         wg_packet_send_queued_handshake_initiation(peer, false);
0139 }
0140
0141 static unsigned int calculate_skb_padding(struct sk_buff *skb)
0142 {
0143     unsigned int padded_size, last_unit = skb->len;
0144
0145     if (unlikely(!PACKET_CB(skb)->mtu))
0146         return ALIGN(last_unit, MESSAGE_PADDING_MULTIPLE) - last_unit;
0147
0148     /* We do this modulo business with the MTU, just in case the networking
0149      * layer gives us a packet that's bigger than the MTU. In that case, we
0150      * wouldn't want the final subtraction to overflow in the case of the
0151      * padded_size being clamped. Fortunately, that's very rarely the case,
0152      * so we optimize for that not happening.
0153      */
0154     if (unlikely(last_unit > PACKET_CB(skb)->mtu))
0155         last_unit %= PACKET_CB(skb)->mtu;
0156
0157     padded_size = min(PACKET_CB(skb)->mtu,
0158               ALIGN(last_unit, MESSAGE_PADDING_MULTIPLE));
0159     return padded_size - last_unit;
0160 }
0161
0162 static bool encrypt_packet(struct sk_buff *skb, struct noise_keypair *keypair)
0163 {
0164     unsigned int padding_len, plaintext_len, trailer_len;
0165     struct scatterlist sg[MAX_SKB_FRAGS + 8];
0166     struct message_data *header;
0167     struct sk_buff *trailer;
0168     int num_frags;
0169
0170     /* Force hash calculation before encryption so that flow analysis is
0171      * consistent over the inner packet.
0172      */
0173     skb_get_hash(skb);
0174
0175     /* Calculate lengths. */
0176     padding_len = calculate_skb_padding(skb);
0177     trailer_len = padding_len + noise_encrypted_len(0);
0178     plaintext_len = skb->len + padding_len;
0179
0180     /* Expand data section to have room for padding and auth tag. */
0181     num_frags = skb_cow_data(skb, trailer_len, &trailer);
0182     if (unlikely(num_frags < 0 || num_frags > ARRAY_SIZE(sg)))
0183         return false;
0184
0185     /* Set the padding to zeros, and make sure it and the auth tag are part
0186      * of the skb.
0187      */
0188     memset(skb_tail_pointer(trailer), 0, padding_len);
0189
0190     /* Expand head section to have room for our header and the network
0191      * stack's headers.
0192      */
0193     if (unlikely(skb_cow_head(skb, DATA_PACKET_HEAD_ROOM) < 0))
0194         return false;
0195
0196     /* Finalize checksum calculation for the inner packet, if required. */
0197     if (unlikely(skb->ip_summed == CHECKSUM_PARTIAL &&
0198              skb_checksum_help(skb)))
0199         return false;
0200
0201     /* Only after checksumming can we safely add on the padding at the end
0202      * and the header.
0203      */
0204     skb_set_inner_network_header(skb, 0);
0205     header = (struct message_data *)skb_push(skb, sizeof(*header));
0206     header->header.type = cpu_to_le32(MESSAGE_DATA);
0207     header->key_idx = keypair->remote_index;
0208     header->counter = cpu_to_le64(PACKET_CB(skb)->nonce);
0209     pskb_put(skb, trailer, trailer_len);
0210
0211     /* Now we can encrypt the scattergather segments */
0212     sg_init_table(sg, num_frags);
0213     if (skb_to_sgvec(skb, sg, sizeof(struct message_data),
0214              noise_encrypted_len(plaintext_len)) <= 0)
0215         return false;
0216     return chacha20poly1305_encrypt_sg_inplace(sg, plaintext_len, NULL, 0,
0217                            PACKET_CB(skb)->nonce,
0218                            keypair->sending.key);
0219 }
0220
0221 void wg_packet_send_keepalive(struct wg_peer *peer)
0222 {
0223     struct sk_buff *skb;
0224
0225     if (skb_queue_empty(&peer->staged_packet_queue)) {
0226         skb = alloc_skb(DATA_PACKET_HEAD_ROOM + MESSAGE_MINIMUM_LENGTH,
0227                 GFP_ATOMIC);
0228         if (unlikely(!skb))
0229             return;
0230         skb_reserve(skb, DATA_PACKET_HEAD_ROOM);
0231         skb->dev = peer->device->dev;
0232         PACKET_CB(skb)->mtu = skb->dev->mtu;
0233         skb_queue_tail(&peer->staged_packet_queue, skb);
0234         net_dbg_ratelimited("%s: Sending keepalive packet to peer %llu (%pISpfsc)\n",
0235                     peer->device->dev->name, peer->internal_id,
0236                     &peer->endpoint.addr);
0237     }
0238
0239     wg_packet_send_staged_packets(peer);
0240 }
0241
0242 static void wg_packet_create_data_done(struct wg_peer *peer, struct sk_buff *first)
0243 {
0244     struct sk_buff *skb, *next;
0245     bool is_keepalive, data_sent = false;
0246
0247     wg_timers_any_authenticated_packet_traversal(peer);
0248     wg_timers_any_authenticated_packet_sent(peer);
0249     skb_list_walk_safe(first, skb, next) {
0250         is_keepalive = skb->len == message_data_len(0);
0251         if (likely(!wg_socket_send_skb_to_peer(peer, skb,
0252                 PACKET_CB(skb)->ds) && !is_keepalive))
0253             data_sent = true;
0254     }
0255
0256     if (likely(data_sent))
0257         wg_timers_data_sent(peer);
0258
0259     keep_key_fresh(peer);
0260 }
0261
0262 void wg_packet_tx_worker(struct work_struct *work)
0263 {
0264     struct wg_peer *peer = container_of(work, struct wg_peer, transmit_packet_work);
0265     struct noise_keypair *keypair;
0266     enum packet_state state;
0267     struct sk_buff *first;
0268
0269     while ((first = wg_prev_queue_peek(&peer->tx_queue)) != NULL &&
0270            (state = atomic_read_acquire(&PACKET_CB(first)->state)) !=
0271                PACKET_STATE_UNCRYPTED) {
0272         wg_prev_queue_drop_peeked(&peer->tx_queue);
0273         keypair = PACKET_CB(first)->keypair;
0274
0275         if (likely(state == PACKET_STATE_CRYPTED))
0276             wg_packet_create_data_done(peer, first);
0277         else
0278             kfree_skb_list(first);
0279
0280         wg_noise_keypair_put(keypair, false);
0281         wg_peer_put(peer);
0282         if (need_resched())
0283             cond_resched();
0284     }
0285 }
0286
0287 void wg_packet_encrypt_worker(struct work_struct *work)
0288 {
0289     struct crypt_queue *queue = container_of(work, struct multicore_worker,
0290                          work)->ptr;
0291     struct sk_buff *first, *skb, *next;
0292
0293     while ((first = ptr_ring_consume_bh(&queue->ring)) != NULL) {
0294         enum packet_state state = PACKET_STATE_CRYPTED;
0295
0296         skb_list_walk_safe(first, skb, next) {
0297             if (likely(encrypt_packet(skb,
0298                     PACKET_CB(first)->keypair))) {
0299                 wg_reset_packet(skb, true);
0300             } else {
0301                 state = PACKET_STATE_DEAD;
0302                 break;
0303             }
0304         }
0305         wg_queue_enqueue_per_peer_tx(first, state);
0306         if (need_resched())
0307             cond_resched();
0308     }
0309 }
0310
0311 static void wg_packet_create_data(struct wg_peer *peer, struct sk_buff *first)
0312 {
0313     struct wg_device *wg = peer->device;
0314     int ret = -EINVAL;
0315
0316     rcu_read_lock_bh();
0317     if (unlikely(READ_ONCE(peer->is_dead)))
0318         goto err;
0319
0320     ret = wg_queue_enqueue_per_device_and_peer(&wg->encrypt_queue, &peer->tx_queue, first,
0321                            wg->packet_crypt_wq, &wg->encrypt_queue.last_cpu);
0322     if (unlikely(ret == -EPIPE))
0323         wg_queue_enqueue_per_peer_tx(first, PACKET_STATE_DEAD);
0324 err:
0325     rcu_read_unlock_bh();
0326     if (likely(!ret || ret == -EPIPE))
0327         return;
0328     wg_noise_keypair_put(PACKET_CB(first)->keypair, false);
0329     wg_peer_put(peer);
0330     kfree_skb_list(first);
0331 }
0332
0333 void wg_packet_purge_staged_packets(struct wg_peer *peer)
0334 {
0335     spin_lock_bh(&peer->staged_packet_queue.lock);
0336     peer->device->dev->stats.tx_dropped += peer->staged_packet_queue.qlen;
0337     __skb_queue_purge(&peer->staged_packet_queue);
0338     spin_unlock_bh(&peer->staged_packet_queue.lock);
0339 }
0340
0341 void wg_packet_send_staged_packets(struct wg_peer *peer)
0342 {
0343     struct noise_keypair *keypair;
0344     struct sk_buff_head packets;
0345     struct sk_buff *skb;
0346
0347     /* Steal the current queue into our local one. */
0348     __skb_queue_head_init(&packets);
0349     spin_lock_bh(&peer->staged_packet_queue.lock);
0350     skb_queue_splice_init(&peer->staged_packet_queue, &packets);
0351     spin_unlock_bh(&peer->staged_packet_queue.lock);
0352     if (unlikely(skb_queue_empty(&packets)))
0353         return;
0354
0355     /* First we make sure we have a valid reference to a valid key. */
0356     rcu_read_lock_bh();
0357     keypair = wg_noise_keypair_get(
0358         rcu_dereference_bh(peer->keypairs.current_keypair));
0359     rcu_read_unlock_bh();
0360     if (unlikely(!keypair))
0361         goto out_nokey;
0362     if (unlikely(!READ_ONCE(keypair->sending.is_valid)))
0363         goto out_nokey;
0364     if (unlikely(wg_birthdate_has_expired(keypair->sending.birthdate,
0365                           REJECT_AFTER_TIME)))
0366         goto out_invalid;
0367
0368     /* After we know we have a somewhat valid key, we now try to assign
0369      * nonces to all of the packets in the queue. If we can't assign nonces
0370      * for all of them, we just consider it a failure and wait for the next
0371      * handshake.
0372      */
0373     skb_queue_walk(&packets, skb) {
0374         /* 0 for no outer TOS: no leak. TODO: at some later point, we
0375          * might consider using flowi->tos as outer instead.
0376          */
0377         PACKET_CB(skb)->ds = ip_tunnel_ecn_encap(0, ip_hdr(skb), skb);
0378         PACKET_CB(skb)->nonce =
0379                 atomic64_inc_return(&keypair->sending_counter) - 1;
0380         if (unlikely(PACKET_CB(skb)->nonce >= REJECT_AFTER_MESSAGES))
0381             goto out_invalid;
0382     }
0383
0384     packets.prev->next = NULL;
0385     wg_peer_get(keypair->entry.peer);
0386     PACKET_CB(packets.next)->keypair = keypair;
0387     wg_packet_create_data(peer, packets.next);
0388     return;
0389
0390 out_invalid:
0391     WRITE_ONCE(keypair->sending.is_valid, false);
0392 out_nokey:
0393     wg_noise_keypair_put(keypair, false);
0394
0395     /* We orphan the packets if we're waiting on a handshake, so that they
0396      * don't block a socket's pool.
0397      */
0398     skb_queue_walk(&packets, skb)
0399         skb_orphan(skb);
0400     /* Then we put them back on the top of the queue. We're not too
0401      * concerned about accidentally getting things a little out of order if
0402      * packets are being added really fast, because this queue is for before
0403      * packets can even be sent and it's small anyway.
0404      */
0405     spin_lock_bh(&peer->staged_packet_queue.lock);
0406     skb_queue_splice(&packets, &peer->staged_packet_queue);
0407     spin_unlock_bh(&peer->staged_packet_queue.lock);
0408
0409     /* If we're exiting because there's something wrong with the key, it
0410      * means we should initiate a new handshake.
0411      */
0412     wg_packet_send_queued_handshake_initiation(peer, false);
0413 }