sfc/siena/tx_common.c

0001 // SPDX-License-Identifier: GPL-2.0-only
0002 /****************************************************************************
0003  * Driver for Solarflare network controllers and boards
0004  * Copyright 2018 Solarflare Communications Inc.
0005  *
0006  * This program is free software; you can redistribute it and/or modify it
0007  * under the terms of the GNU General Public License version 2 as published
0008  * by the Free Software Foundation, incorporated herein by reference.
0009  */
0010
0011 #include "net_driver.h"
0012 #include "efx.h"
0013 #include "nic_common.h"
0014 #include "tx_common.h"
0015
0016 static unsigned int efx_tx_cb_page_count(struct efx_tx_queue *tx_queue)
0017 {
0018     return DIV_ROUND_UP(tx_queue->ptr_mask + 1,
0019                 PAGE_SIZE >> EFX_TX_CB_ORDER);
0020 }
0021
0022 int efx_siena_probe_tx_queue(struct efx_tx_queue *tx_queue)
0023 {
0024     struct efx_nic *efx = tx_queue->efx;
0025     unsigned int entries;
0026     int rc;
0027
0028     /* Create the smallest power-of-two aligned ring */
0029     entries = max(roundup_pow_of_two(efx->txq_entries), EFX_MIN_DMAQ_SIZE);
0030     EFX_WARN_ON_PARANOID(entries > EFX_MAX_DMAQ_SIZE);
0031     tx_queue->ptr_mask = entries - 1;
0032
0033     netif_dbg(efx, probe, efx->net_dev,
0034           "creating TX queue %d size %#x mask %#x\n",
0035           tx_queue->queue, efx->txq_entries, tx_queue->ptr_mask);
0036
0037     /* Allocate software ring */
0038     tx_queue->buffer = kcalloc(entries, sizeof(*tx_queue->buffer),
0039                    GFP_KERNEL);
0040     if (!tx_queue->buffer)
0041         return -ENOMEM;
0042
0043     tx_queue->cb_page = kcalloc(efx_tx_cb_page_count(tx_queue),
0044                     sizeof(tx_queue->cb_page[0]), GFP_KERNEL);
0045     if (!tx_queue->cb_page) {
0046         rc = -ENOMEM;
0047         goto fail1;
0048     }
0049
0050     /* Allocate hardware ring, determine TXQ type */
0051     rc = efx_nic_probe_tx(tx_queue);
0052     if (rc)
0053         goto fail2;
0054
0055     tx_queue->channel->tx_queue_by_type[tx_queue->type] = tx_queue;
0056     return 0;
0057
0058 fail2:
0059     kfree(tx_queue->cb_page);
0060     tx_queue->cb_page = NULL;
0061 fail1:
0062     kfree(tx_queue->buffer);
0063     tx_queue->buffer = NULL;
0064     return rc;
0065 }
0066
0067 void efx_siena_init_tx_queue(struct efx_tx_queue *tx_queue)
0068 {
0069     struct efx_nic *efx = tx_queue->efx;
0070
0071     netif_dbg(efx, drv, efx->net_dev,
0072           "initialising TX queue %d\n", tx_queue->queue);
0073
0074     tx_queue->insert_count = 0;
0075     tx_queue->notify_count = 0;
0076     tx_queue->write_count = 0;
0077     tx_queue->packet_write_count = 0;
0078     tx_queue->old_write_count = 0;
0079     tx_queue->read_count = 0;
0080     tx_queue->old_read_count = 0;
0081     tx_queue->empty_read_count = 0 | EFX_EMPTY_COUNT_VALID;
0082     tx_queue->xmit_pending = false;
0083     tx_queue->timestamping = (efx_siena_ptp_use_mac_tx_timestamps(efx) &&
0084                   tx_queue->channel == efx_siena_ptp_channel(efx));
0085     tx_queue->completed_timestamp_major = 0;
0086     tx_queue->completed_timestamp_minor = 0;
0087
0088     tx_queue->xdp_tx = efx_channel_is_xdp_tx(tx_queue->channel);
0089     tx_queue->tso_version = 0;
0090
0091     /* Set up TX descriptor ring */
0092     efx_nic_init_tx(tx_queue);
0093
0094     tx_queue->initialised = true;
0095 }
0096
0097 void efx_siena_remove_tx_queue(struct efx_tx_queue *tx_queue)
0098 {
0099     int i;
0100
0101     if (!tx_queue->buffer)
0102         return;
0103
0104     netif_dbg(tx_queue->efx, drv, tx_queue->efx->net_dev,
0105           "destroying TX queue %d\n", tx_queue->queue);
0106     efx_nic_remove_tx(tx_queue);
0107
0108     if (tx_queue->cb_page) {
0109         for (i = 0; i < efx_tx_cb_page_count(tx_queue); i++)
0110             efx_siena_free_buffer(tx_queue->efx,
0111                           &tx_queue->cb_page[i]);
0112         kfree(tx_queue->cb_page);
0113         tx_queue->cb_page = NULL;
0114     }
0115
0116     kfree(tx_queue->buffer);
0117     tx_queue->buffer = NULL;
0118     tx_queue->channel->tx_queue_by_type[tx_queue->type] = NULL;
0119 }
0120
0121 static void efx_dequeue_buffer(struct efx_tx_queue *tx_queue,
0122                    struct efx_tx_buffer *buffer,
0123                    unsigned int *pkts_compl,
0124                    unsigned int *bytes_compl)
0125 {
0126     if (buffer->unmap_len) {
0127         struct device *dma_dev = &tx_queue->efx->pci_dev->dev;
0128         dma_addr_t unmap_addr = buffer->dma_addr - buffer->dma_offset;
0129
0130         if (buffer->flags & EFX_TX_BUF_MAP_SINGLE)
0131             dma_unmap_single(dma_dev, unmap_addr, buffer->unmap_len,
0132                      DMA_TO_DEVICE);
0133         else
0134             dma_unmap_page(dma_dev, unmap_addr, buffer->unmap_len,
0135                        DMA_TO_DEVICE);
0136         buffer->unmap_len = 0;
0137     }
0138
0139     if (buffer->flags & EFX_TX_BUF_SKB) {
0140         struct sk_buff *skb = (struct sk_buff *)buffer->skb;
0141
0142         EFX_WARN_ON_PARANOID(!pkts_compl || !bytes_compl);
0143         (*pkts_compl)++;
0144         (*bytes_compl) += skb->len;
0145         if (tx_queue->timestamping &&
0146             (tx_queue->completed_timestamp_major ||
0147              tx_queue->completed_timestamp_minor)) {
0148             struct skb_shared_hwtstamps hwtstamp;
0149
0150             hwtstamp.hwtstamp =
0151                 efx_siena_ptp_nic_to_kernel_time(tx_queue);
0152             skb_tstamp_tx(skb, &hwtstamp);
0153
0154             tx_queue->completed_timestamp_major = 0;
0155             tx_queue->completed_timestamp_minor = 0;
0156         }
0157         dev_consume_skb_any((struct sk_buff *)buffer->skb);
0158         netif_vdbg(tx_queue->efx, tx_done, tx_queue->efx->net_dev,
0159                "TX queue %d transmission id %x complete\n",
0160                tx_queue->queue, tx_queue->read_count);
0161     } else if (buffer->flags & EFX_TX_BUF_XDP) {
0162         xdp_return_frame_rx_napi(buffer->xdpf);
0163     }
0164
0165     buffer->len = 0;
0166     buffer->flags = 0;
0167 }
0168
0169 void efx_siena_fini_tx_queue(struct efx_tx_queue *tx_queue)
0170 {
0171     struct efx_tx_buffer *buffer;
0172
0173     netif_dbg(tx_queue->efx, drv, tx_queue->efx->net_dev,
0174           "shutting down TX queue %d\n", tx_queue->queue);
0175
0176     if (!tx_queue->buffer)
0177         return;
0178
0179     /* Free any buffers left in the ring */
0180     while (tx_queue->read_count != tx_queue->write_count) {
0181         unsigned int pkts_compl = 0, bytes_compl = 0;
0182
0183         buffer = &tx_queue->buffer[tx_queue->read_count & tx_queue->ptr_mask];
0184         efx_dequeue_buffer(tx_queue, buffer, &pkts_compl, &bytes_compl);
0185
0186         ++tx_queue->read_count;
0187     }
0188     tx_queue->xmit_pending = false;
0189     netdev_tx_reset_queue(tx_queue->core_txq);
0190 }
0191
0192 /* Remove packets from the TX queue
0193  *
0194  * This removes packets from the TX queue, up to and including the
0195  * specified index.
0196  */
0197 static void efx_dequeue_buffers(struct efx_tx_queue *tx_queue,
0198                 unsigned int index,
0199                 unsigned int *pkts_compl,
0200                 unsigned int *bytes_compl)
0201 {
0202     struct efx_nic *efx = tx_queue->efx;
0203     unsigned int stop_index, read_ptr;
0204
0205     stop_index = (index + 1) & tx_queue->ptr_mask;
0206     read_ptr = tx_queue->read_count & tx_queue->ptr_mask;
0207
0208     while (read_ptr != stop_index) {
0209         struct efx_tx_buffer *buffer = &tx_queue->buffer[read_ptr];
0210
0211         if (!efx_tx_buffer_in_use(buffer)) {
0212             netif_err(efx, tx_err, efx->net_dev,
0213                   "TX queue %d spurious TX completion id %d\n",
0214                   tx_queue->queue, read_ptr);
0215             efx_siena_schedule_reset(efx, RESET_TYPE_TX_SKIP);
0216             return;
0217         }
0218
0219         efx_dequeue_buffer(tx_queue, buffer, pkts_compl, bytes_compl);
0220
0221         ++tx_queue->read_count;
0222         read_ptr = tx_queue->read_count & tx_queue->ptr_mask;
0223     }
0224 }
0225
0226 void efx_siena_xmit_done_check_empty(struct efx_tx_queue *tx_queue)
0227 {
0228     if ((int)(tx_queue->read_count - tx_queue->old_write_count) >= 0) {
0229         tx_queue->old_write_count = READ_ONCE(tx_queue->write_count);
0230         if (tx_queue->read_count == tx_queue->old_write_count) {
0231             /* Ensure that read_count is flushed. */
0232             smp_mb();
0233             tx_queue->empty_read_count =
0234                 tx_queue->read_count | EFX_EMPTY_COUNT_VALID;
0235         }
0236     }
0237 }
0238
0239 void efx_siena_xmit_done(struct efx_tx_queue *tx_queue, unsigned int index)
0240 {
0241     unsigned int fill_level, pkts_compl = 0, bytes_compl = 0;
0242     struct efx_nic *efx = tx_queue->efx;
0243
0244     EFX_WARN_ON_ONCE_PARANOID(index > tx_queue->ptr_mask);
0245
0246     efx_dequeue_buffers(tx_queue, index, &pkts_compl, &bytes_compl);
0247     tx_queue->pkts_compl += pkts_compl;
0248     tx_queue->bytes_compl += bytes_compl;
0249
0250     if (pkts_compl > 1)
0251         ++tx_queue->merge_events;
0252
0253     /* See if we need to restart the netif queue.  This memory
0254      * barrier ensures that we write read_count (inside
0255      * efx_dequeue_buffers()) before reading the queue status.
0256      */
0257     smp_mb();
0258     if (unlikely(netif_tx_queue_stopped(tx_queue->core_txq)) &&
0259         likely(efx->port_enabled) &&
0260         likely(netif_device_present(efx->net_dev))) {
0261         fill_level = efx_channel_tx_fill_level(tx_queue->channel);
0262         if (fill_level <= efx->txq_wake_thresh)
0263             netif_tx_wake_queue(tx_queue->core_txq);
0264     }
0265
0266     efx_siena_xmit_done_check_empty(tx_queue);
0267 }
0268
0269 /* Remove buffers put into a tx_queue for the current packet.
0270  * None of the buffers must have an skb attached.
0271  */
0272 void efx_siena_enqueue_unwind(struct efx_tx_queue *tx_queue,
0273                   unsigned int insert_count)
0274 {
0275     struct efx_tx_buffer *buffer;
0276     unsigned int bytes_compl = 0;
0277     unsigned int pkts_compl = 0;
0278
0279     /* Work backwards until we hit the original insert pointer value */
0280     while (tx_queue->insert_count != insert_count) {
0281         --tx_queue->insert_count;
0282         buffer = __efx_tx_queue_get_insert_buffer(tx_queue);
0283         efx_dequeue_buffer(tx_queue, buffer, &pkts_compl, &bytes_compl);
0284     }
0285 }
0286
0287 struct efx_tx_buffer *efx_siena_tx_map_chunk(struct efx_tx_queue *tx_queue,
0288                          dma_addr_t dma_addr, size_t len)
0289 {
0290     const struct efx_nic_type *nic_type = tx_queue->efx->type;
0291     struct efx_tx_buffer *buffer;
0292     unsigned int dma_len;
0293
0294     /* Map the fragment taking account of NIC-dependent DMA limits. */
0295     do {
0296         buffer = efx_tx_queue_get_insert_buffer(tx_queue);
0297
0298         if (nic_type->tx_limit_len)
0299             dma_len = nic_type->tx_limit_len(tx_queue, dma_addr, len);
0300         else
0301             dma_len = len;
0302
0303         buffer->len = dma_len;
0304         buffer->dma_addr = dma_addr;
0305         buffer->flags = EFX_TX_BUF_CONT;
0306         len -= dma_len;
0307         dma_addr += dma_len;
0308         ++tx_queue->insert_count;
0309     } while (len);
0310
0311     return buffer;
0312 }
0313
0314 static int efx_tx_tso_header_length(struct sk_buff *skb)
0315 {
0316     size_t header_len;
0317
0318     if (skb->encapsulation)
0319         header_len = skb_inner_transport_header(skb) -
0320                 skb->data +
0321                 (inner_tcp_hdr(skb)->doff << 2u);
0322     else
0323         header_len = skb_transport_header(skb) - skb->data +
0324                 (tcp_hdr(skb)->doff << 2u);
0325     return header_len;
0326 }
0327
0328 /* Map all data from an SKB for DMA and create descriptors on the queue. */
0329 int efx_siena_tx_map_data(struct efx_tx_queue *tx_queue, struct sk_buff *skb,
0330               unsigned int segment_count)
0331 {
0332     struct efx_nic *efx = tx_queue->efx;
0333     struct device *dma_dev = &efx->pci_dev->dev;
0334     unsigned int frag_index, nr_frags;
0335     dma_addr_t dma_addr, unmap_addr;
0336     unsigned short dma_flags;
0337     size_t len, unmap_len;
0338
0339     nr_frags = skb_shinfo(skb)->nr_frags;
0340     frag_index = 0;
0341
0342     /* Map header data. */
0343     len = skb_headlen(skb);
0344     dma_addr = dma_map_single(dma_dev, skb->data, len, DMA_TO_DEVICE);
0345     dma_flags = EFX_TX_BUF_MAP_SINGLE;
0346     unmap_len = len;
0347     unmap_addr = dma_addr;
0348
0349     if (unlikely(dma_mapping_error(dma_dev, dma_addr)))
0350         return -EIO;
0351
0352     if (segment_count) {
0353         /* For TSO we need to put the header in to a separate
0354          * descriptor. Map this separately if necessary.
0355          */
0356         size_t header_len = efx_tx_tso_header_length(skb);
0357
0358         if (header_len != len) {
0359             tx_queue->tso_long_headers++;
0360             efx_siena_tx_map_chunk(tx_queue, dma_addr, header_len);
0361             len -= header_len;
0362             dma_addr += header_len;
0363         }
0364     }
0365
0366     /* Add descriptors for each fragment. */
0367     do {
0368         struct efx_tx_buffer *buffer;
0369         skb_frag_t *fragment;
0370
0371         buffer = efx_siena_tx_map_chunk(tx_queue, dma_addr, len);
0372
0373         /* The final descriptor for a fragment is responsible for
0374          * unmapping the whole fragment.
0375          */
0376         buffer->flags = EFX_TX_BUF_CONT | dma_flags;
0377         buffer->unmap_len = unmap_len;
0378         buffer->dma_offset = buffer->dma_addr - unmap_addr;
0379
0380         if (frag_index >= nr_frags) {
0381             /* Store SKB details with the final buffer for
0382              * the completion.
0383              */
0384             buffer->skb = skb;
0385             buffer->flags = EFX_TX_BUF_SKB | dma_flags;
0386             return 0;
0387         }
0388
0389         /* Move on to the next fragment. */
0390         fragment = &skb_shinfo(skb)->frags[frag_index++];
0391         len = skb_frag_size(fragment);
0392         dma_addr = skb_frag_dma_map(dma_dev, fragment, 0, len,
0393                         DMA_TO_DEVICE);
0394         dma_flags = 0;
0395         unmap_len = len;
0396         unmap_addr = dma_addr;
0397
0398         if (unlikely(dma_mapping_error(dma_dev, dma_addr)))
0399             return -EIO;
0400     } while (1);
0401 }
0402
0403 unsigned int efx_siena_tx_max_skb_descs(struct efx_nic *efx)
0404 {
0405     /* Header and payload descriptor for each output segment, plus
0406      * one for every input fragment boundary within a segment
0407      */
0408     unsigned int max_descs = EFX_TSO_MAX_SEGS * 2 + MAX_SKB_FRAGS;
0409
0410     /* Possibly one more per segment for option descriptors */
0411     if (efx_nic_rev(efx) >= EFX_REV_HUNT_A0)
0412         max_descs += EFX_TSO_MAX_SEGS;
0413
0414     /* Possibly more for PCIe page boundaries within input fragments */
0415     if (PAGE_SIZE > EFX_PAGE_SIZE)
0416         max_descs += max_t(unsigned int, MAX_SKB_FRAGS,
0417                    DIV_ROUND_UP(GSO_MAX_SIZE, EFX_PAGE_SIZE));
0418
0419     return max_descs;
0420 }
0421
0422 /*
0423  * Fallback to software TSO.
0424  *
0425  * This is used if we are unable to send a GSO packet through hardware TSO.
0426  * This should only ever happen due to per-queue restrictions - unsupported
0427  * packets should first be filtered by the feature flags.
0428  *
0429  * Returns 0 on success, error code otherwise.
0430  */
0431 int efx_siena_tx_tso_fallback(struct efx_tx_queue *tx_queue,
0432                   struct sk_buff *skb)
0433 {
0434     struct sk_buff *segments, *next;
0435
0436     segments = skb_gso_segment(skb, 0);
0437     if (IS_ERR(segments))
0438         return PTR_ERR(segments);
0439
0440     dev_consume_skb_any(skb);
0441
0442     skb_list_walk_safe(segments, skb, next) {
0443         skb_mark_not_on_list(skb);
0444         efx_enqueue_skb(tx_queue, skb);
0445     }
0446
0447     return 0;
0448 }