Back to home page

OSCL-LXR
 
 

     

0001 // SPDX-License-Identifier: GPL-2.0-only
0002 /****************************************************************************
0003  * Driver for Solarflare network controllers and boards
0004  * Copyright 2005-2006 Fen Systems Ltd.
0005  * Copyright 2005-2013 Solarflare Communications Inc.
0006  */
0007 
0008 #include <linux/socket.h>
0009 #include <linux/in.h>
0010 #include <linux/slab.h>
0011 #include <linux/ip.h>
0012 #include <linux/ipv6.h>
0013 #include <linux/tcp.h>
0014 #include <linux/udp.h>
0015 #include <linux/prefetch.h>
0016 #include <linux/moduleparam.h>
0017 #include <linux/iommu.h>
0018 #include <net/ip.h>
0019 #include <net/checksum.h>
0020 #include "net_driver.h"
0021 #include "efx.h"
0022 #include "filter.h"
0023 #include "nic.h"
0024 #include "selftest.h"
0025 #include "workarounds.h"
0026 
0027 /* Preferred number of descriptors to fill at once */
0028 #define EF4_RX_PREFERRED_BATCH 8U
0029 
0030 /* Number of RX buffers to recycle pages for.  When creating the RX page recycle
0031  * ring, this number is divided by the number of buffers per page to calculate
0032  * the number of pages to store in the RX page recycle ring.
0033  */
0034 #define EF4_RECYCLE_RING_SIZE_IOMMU 4096
0035 #define EF4_RECYCLE_RING_SIZE_NOIOMMU (2 * EF4_RX_PREFERRED_BATCH)
0036 
0037 /* Size of buffer allocated for skb header area. */
0038 #define EF4_SKB_HEADERS  128u
0039 
0040 /* This is the percentage fill level below which new RX descriptors
0041  * will be added to the RX descriptor ring.
0042  */
0043 static unsigned int rx_refill_threshold;
0044 
0045 /* Each packet can consume up to ceil(max_frame_len / buffer_size) buffers */
0046 #define EF4_RX_MAX_FRAGS DIV_ROUND_UP(EF4_MAX_FRAME_LEN(EF4_MAX_MTU), \
0047                       EF4_RX_USR_BUF_SIZE)
0048 
0049 /*
0050  * RX maximum head room required.
0051  *
0052  * This must be at least 1 to prevent overflow, plus one packet-worth
0053  * to allow pipelined receives.
0054  */
0055 #define EF4_RXD_HEAD_ROOM (1 + EF4_RX_MAX_FRAGS)
0056 
0057 static inline u8 *ef4_rx_buf_va(struct ef4_rx_buffer *buf)
0058 {
0059     return page_address(buf->page) + buf->page_offset;
0060 }
0061 
0062 static inline u32 ef4_rx_buf_hash(struct ef4_nic *efx, const u8 *eh)
0063 {
0064 #if defined(CONFIG_HAVE_EFFICIENT_UNALIGNED_ACCESS)
0065     return __le32_to_cpup((const __le32 *)(eh + efx->rx_packet_hash_offset));
0066 #else
0067     const u8 *data = eh + efx->rx_packet_hash_offset;
0068     return (u32)data[0]   |
0069            (u32)data[1] << 8  |
0070            (u32)data[2] << 16 |
0071            (u32)data[3] << 24;
0072 #endif
0073 }
0074 
0075 static inline struct ef4_rx_buffer *
0076 ef4_rx_buf_next(struct ef4_rx_queue *rx_queue, struct ef4_rx_buffer *rx_buf)
0077 {
0078     if (unlikely(rx_buf == ef4_rx_buffer(rx_queue, rx_queue->ptr_mask)))
0079         return ef4_rx_buffer(rx_queue, 0);
0080     else
0081         return rx_buf + 1;
0082 }
0083 
0084 static inline void ef4_sync_rx_buffer(struct ef4_nic *efx,
0085                       struct ef4_rx_buffer *rx_buf,
0086                       unsigned int len)
0087 {
0088     dma_sync_single_for_cpu(&efx->pci_dev->dev, rx_buf->dma_addr, len,
0089                 DMA_FROM_DEVICE);
0090 }
0091 
0092 void ef4_rx_config_page_split(struct ef4_nic *efx)
0093 {
0094     efx->rx_page_buf_step = ALIGN(efx->rx_dma_len + efx->rx_ip_align,
0095                       EF4_RX_BUF_ALIGNMENT);
0096     efx->rx_bufs_per_page = efx->rx_buffer_order ? 1 :
0097         ((PAGE_SIZE - sizeof(struct ef4_rx_page_state)) /
0098          efx->rx_page_buf_step);
0099     efx->rx_buffer_truesize = (PAGE_SIZE << efx->rx_buffer_order) /
0100         efx->rx_bufs_per_page;
0101     efx->rx_pages_per_batch = DIV_ROUND_UP(EF4_RX_PREFERRED_BATCH,
0102                            efx->rx_bufs_per_page);
0103 }
0104 
0105 /* Check the RX page recycle ring for a page that can be reused. */
0106 static struct page *ef4_reuse_page(struct ef4_rx_queue *rx_queue)
0107 {
0108     struct ef4_nic *efx = rx_queue->efx;
0109     struct page *page;
0110     struct ef4_rx_page_state *state;
0111     unsigned index;
0112 
0113     if (unlikely(!rx_queue->page_ring))
0114         return NULL;
0115     index = rx_queue->page_remove & rx_queue->page_ptr_mask;
0116     page = rx_queue->page_ring[index];
0117     if (page == NULL)
0118         return NULL;
0119 
0120     rx_queue->page_ring[index] = NULL;
0121     /* page_remove cannot exceed page_add. */
0122     if (rx_queue->page_remove != rx_queue->page_add)
0123         ++rx_queue->page_remove;
0124 
0125     /* If page_count is 1 then we hold the only reference to this page. */
0126     if (page_count(page) == 1) {
0127         ++rx_queue->page_recycle_count;
0128         return page;
0129     } else {
0130         state = page_address(page);
0131         dma_unmap_page(&efx->pci_dev->dev, state->dma_addr,
0132                    PAGE_SIZE << efx->rx_buffer_order,
0133                    DMA_FROM_DEVICE);
0134         put_page(page);
0135         ++rx_queue->page_recycle_failed;
0136     }
0137 
0138     return NULL;
0139 }
0140 
0141 /**
0142  * ef4_init_rx_buffers - create EF4_RX_BATCH page-based RX buffers
0143  *
0144  * @rx_queue:       Efx RX queue
0145  * @atomic:     control memory allocation flags
0146  *
0147  * This allocates a batch of pages, maps them for DMA, and populates
0148  * struct ef4_rx_buffers for each one. Return a negative error code or
0149  * 0 on success. If a single page can be used for multiple buffers,
0150  * then the page will either be inserted fully, or not at all.
0151  */
0152 static int ef4_init_rx_buffers(struct ef4_rx_queue *rx_queue, bool atomic)
0153 {
0154     struct ef4_nic *efx = rx_queue->efx;
0155     struct ef4_rx_buffer *rx_buf;
0156     struct page *page;
0157     unsigned int page_offset;
0158     struct ef4_rx_page_state *state;
0159     dma_addr_t dma_addr;
0160     unsigned index, count;
0161 
0162     count = 0;
0163     do {
0164         page = ef4_reuse_page(rx_queue);
0165         if (page == NULL) {
0166             page = alloc_pages(__GFP_COMP |
0167                        (atomic ? GFP_ATOMIC : GFP_KERNEL),
0168                        efx->rx_buffer_order);
0169             if (unlikely(page == NULL))
0170                 return -ENOMEM;
0171             dma_addr =
0172                 dma_map_page(&efx->pci_dev->dev, page, 0,
0173                          PAGE_SIZE << efx->rx_buffer_order,
0174                          DMA_FROM_DEVICE);
0175             if (unlikely(dma_mapping_error(&efx->pci_dev->dev,
0176                                dma_addr))) {
0177                 __free_pages(page, efx->rx_buffer_order);
0178                 return -EIO;
0179             }
0180             state = page_address(page);
0181             state->dma_addr = dma_addr;
0182         } else {
0183             state = page_address(page);
0184             dma_addr = state->dma_addr;
0185         }
0186 
0187         dma_addr += sizeof(struct ef4_rx_page_state);
0188         page_offset = sizeof(struct ef4_rx_page_state);
0189 
0190         do {
0191             index = rx_queue->added_count & rx_queue->ptr_mask;
0192             rx_buf = ef4_rx_buffer(rx_queue, index);
0193             rx_buf->dma_addr = dma_addr + efx->rx_ip_align;
0194             rx_buf->page = page;
0195             rx_buf->page_offset = page_offset + efx->rx_ip_align;
0196             rx_buf->len = efx->rx_dma_len;
0197             rx_buf->flags = 0;
0198             ++rx_queue->added_count;
0199             get_page(page);
0200             dma_addr += efx->rx_page_buf_step;
0201             page_offset += efx->rx_page_buf_step;
0202         } while (page_offset + efx->rx_page_buf_step <= PAGE_SIZE);
0203 
0204         rx_buf->flags = EF4_RX_BUF_LAST_IN_PAGE;
0205     } while (++count < efx->rx_pages_per_batch);
0206 
0207     return 0;
0208 }
0209 
0210 /* Unmap a DMA-mapped page.  This function is only called for the final RX
0211  * buffer in a page.
0212  */
0213 static void ef4_unmap_rx_buffer(struct ef4_nic *efx,
0214                 struct ef4_rx_buffer *rx_buf)
0215 {
0216     struct page *page = rx_buf->page;
0217 
0218     if (page) {
0219         struct ef4_rx_page_state *state = page_address(page);
0220         dma_unmap_page(&efx->pci_dev->dev,
0221                    state->dma_addr,
0222                    PAGE_SIZE << efx->rx_buffer_order,
0223                    DMA_FROM_DEVICE);
0224     }
0225 }
0226 
0227 static void ef4_free_rx_buffers(struct ef4_rx_queue *rx_queue,
0228                 struct ef4_rx_buffer *rx_buf,
0229                 unsigned int num_bufs)
0230 {
0231     do {
0232         if (rx_buf->page) {
0233             put_page(rx_buf->page);
0234             rx_buf->page = NULL;
0235         }
0236         rx_buf = ef4_rx_buf_next(rx_queue, rx_buf);
0237     } while (--num_bufs);
0238 }
0239 
0240 /* Attempt to recycle the page if there is an RX recycle ring; the page can
0241  * only be added if this is the final RX buffer, to prevent pages being used in
0242  * the descriptor ring and appearing in the recycle ring simultaneously.
0243  */
0244 static void ef4_recycle_rx_page(struct ef4_channel *channel,
0245                 struct ef4_rx_buffer *rx_buf)
0246 {
0247     struct page *page = rx_buf->page;
0248     struct ef4_rx_queue *rx_queue = ef4_channel_get_rx_queue(channel);
0249     struct ef4_nic *efx = rx_queue->efx;
0250     unsigned index;
0251 
0252     /* Only recycle the page after processing the final buffer. */
0253     if (!(rx_buf->flags & EF4_RX_BUF_LAST_IN_PAGE))
0254         return;
0255 
0256     index = rx_queue->page_add & rx_queue->page_ptr_mask;
0257     if (rx_queue->page_ring[index] == NULL) {
0258         unsigned read_index = rx_queue->page_remove &
0259             rx_queue->page_ptr_mask;
0260 
0261         /* The next slot in the recycle ring is available, but
0262          * increment page_remove if the read pointer currently
0263          * points here.
0264          */
0265         if (read_index == index)
0266             ++rx_queue->page_remove;
0267         rx_queue->page_ring[index] = page;
0268         ++rx_queue->page_add;
0269         return;
0270     }
0271     ++rx_queue->page_recycle_full;
0272     ef4_unmap_rx_buffer(efx, rx_buf);
0273     put_page(rx_buf->page);
0274 }
0275 
0276 static void ef4_fini_rx_buffer(struct ef4_rx_queue *rx_queue,
0277                    struct ef4_rx_buffer *rx_buf)
0278 {
0279     /* Release the page reference we hold for the buffer. */
0280     if (rx_buf->page)
0281         put_page(rx_buf->page);
0282 
0283     /* If this is the last buffer in a page, unmap and free it. */
0284     if (rx_buf->flags & EF4_RX_BUF_LAST_IN_PAGE) {
0285         ef4_unmap_rx_buffer(rx_queue->efx, rx_buf);
0286         ef4_free_rx_buffers(rx_queue, rx_buf, 1);
0287     }
0288     rx_buf->page = NULL;
0289 }
0290 
0291 /* Recycle the pages that are used by buffers that have just been received. */
0292 static void ef4_recycle_rx_pages(struct ef4_channel *channel,
0293                  struct ef4_rx_buffer *rx_buf,
0294                  unsigned int n_frags)
0295 {
0296     struct ef4_rx_queue *rx_queue = ef4_channel_get_rx_queue(channel);
0297 
0298     if (unlikely(!rx_queue->page_ring))
0299         return;
0300 
0301     do {
0302         ef4_recycle_rx_page(channel, rx_buf);
0303         rx_buf = ef4_rx_buf_next(rx_queue, rx_buf);
0304     } while (--n_frags);
0305 }
0306 
0307 static void ef4_discard_rx_packet(struct ef4_channel *channel,
0308                   struct ef4_rx_buffer *rx_buf,
0309                   unsigned int n_frags)
0310 {
0311     struct ef4_rx_queue *rx_queue = ef4_channel_get_rx_queue(channel);
0312 
0313     ef4_recycle_rx_pages(channel, rx_buf, n_frags);
0314 
0315     ef4_free_rx_buffers(rx_queue, rx_buf, n_frags);
0316 }
0317 
0318 /**
0319  * ef4_fast_push_rx_descriptors - push new RX descriptors quickly
0320  * @rx_queue:       RX descriptor queue
0321  *
0322  * This will aim to fill the RX descriptor queue up to
0323  * @rx_queue->@max_fill. If there is insufficient atomic
0324  * memory to do so, a slow fill will be scheduled.
0325  * @atomic: control memory allocation flags
0326  *
0327  * The caller must provide serialisation (none is used here). In practise,
0328  * this means this function must run from the NAPI handler, or be called
0329  * when NAPI is disabled.
0330  */
0331 void ef4_fast_push_rx_descriptors(struct ef4_rx_queue *rx_queue, bool atomic)
0332 {
0333     struct ef4_nic *efx = rx_queue->efx;
0334     unsigned int fill_level, batch_size;
0335     int space, rc = 0;
0336 
0337     if (!rx_queue->refill_enabled)
0338         return;
0339 
0340     /* Calculate current fill level, and exit if we don't need to fill */
0341     fill_level = (rx_queue->added_count - rx_queue->removed_count);
0342     EF4_BUG_ON_PARANOID(fill_level > rx_queue->efx->rxq_entries);
0343     if (fill_level >= rx_queue->fast_fill_trigger)
0344         goto out;
0345 
0346     /* Record minimum fill level */
0347     if (unlikely(fill_level < rx_queue->min_fill)) {
0348         if (fill_level)
0349             rx_queue->min_fill = fill_level;
0350     }
0351 
0352     batch_size = efx->rx_pages_per_batch * efx->rx_bufs_per_page;
0353     space = rx_queue->max_fill - fill_level;
0354     EF4_BUG_ON_PARANOID(space < batch_size);
0355 
0356     netif_vdbg(rx_queue->efx, rx_status, rx_queue->efx->net_dev,
0357            "RX queue %d fast-filling descriptor ring from"
0358            " level %d to level %d\n",
0359            ef4_rx_queue_index(rx_queue), fill_level,
0360            rx_queue->max_fill);
0361 
0362 
0363     do {
0364         rc = ef4_init_rx_buffers(rx_queue, atomic);
0365         if (unlikely(rc)) {
0366             /* Ensure that we don't leave the rx queue empty */
0367             if (rx_queue->added_count == rx_queue->removed_count)
0368                 ef4_schedule_slow_fill(rx_queue);
0369             goto out;
0370         }
0371     } while ((space -= batch_size) >= batch_size);
0372 
0373     netif_vdbg(rx_queue->efx, rx_status, rx_queue->efx->net_dev,
0374            "RX queue %d fast-filled descriptor ring "
0375            "to level %d\n", ef4_rx_queue_index(rx_queue),
0376            rx_queue->added_count - rx_queue->removed_count);
0377 
0378  out:
0379     if (rx_queue->notified_count != rx_queue->added_count)
0380         ef4_nic_notify_rx_desc(rx_queue);
0381 }
0382 
0383 void ef4_rx_slow_fill(struct timer_list *t)
0384 {
0385     struct ef4_rx_queue *rx_queue = from_timer(rx_queue, t, slow_fill);
0386 
0387     /* Post an event to cause NAPI to run and refill the queue */
0388     ef4_nic_generate_fill_event(rx_queue);
0389     ++rx_queue->slow_fill_count;
0390 }
0391 
0392 static void ef4_rx_packet__check_len(struct ef4_rx_queue *rx_queue,
0393                      struct ef4_rx_buffer *rx_buf,
0394                      int len)
0395 {
0396     struct ef4_nic *efx = rx_queue->efx;
0397     unsigned max_len = rx_buf->len - efx->type->rx_buffer_padding;
0398 
0399     if (likely(len <= max_len))
0400         return;
0401 
0402     /* The packet must be discarded, but this is only a fatal error
0403      * if the caller indicated it was
0404      */
0405     rx_buf->flags |= EF4_RX_PKT_DISCARD;
0406 
0407     if ((len > rx_buf->len) && EF4_WORKAROUND_8071(efx)) {
0408         if (net_ratelimit())
0409             netif_err(efx, rx_err, efx->net_dev,
0410                   " RX queue %d seriously overlength "
0411                   "RX event (0x%x > 0x%x+0x%x). Leaking\n",
0412                   ef4_rx_queue_index(rx_queue), len, max_len,
0413                   efx->type->rx_buffer_padding);
0414         ef4_schedule_reset(efx, RESET_TYPE_RX_RECOVERY);
0415     } else {
0416         if (net_ratelimit())
0417             netif_err(efx, rx_err, efx->net_dev,
0418                   " RX queue %d overlength RX event "
0419                   "(0x%x > 0x%x)\n",
0420                   ef4_rx_queue_index(rx_queue), len, max_len);
0421     }
0422 
0423     ef4_rx_queue_channel(rx_queue)->n_rx_overlength++;
0424 }
0425 
0426 /* Pass a received packet up through GRO.  GRO can handle pages
0427  * regardless of checksum state and skbs with a good checksum.
0428  */
0429 static void
0430 ef4_rx_packet_gro(struct ef4_channel *channel, struct ef4_rx_buffer *rx_buf,
0431           unsigned int n_frags, u8 *eh)
0432 {
0433     struct napi_struct *napi = &channel->napi_str;
0434     struct ef4_nic *efx = channel->efx;
0435     struct sk_buff *skb;
0436 
0437     skb = napi_get_frags(napi);
0438     if (unlikely(!skb)) {
0439         struct ef4_rx_queue *rx_queue;
0440 
0441         rx_queue = ef4_channel_get_rx_queue(channel);
0442         ef4_free_rx_buffers(rx_queue, rx_buf, n_frags);
0443         return;
0444     }
0445 
0446     if (efx->net_dev->features & NETIF_F_RXHASH)
0447         skb_set_hash(skb, ef4_rx_buf_hash(efx, eh),
0448                  PKT_HASH_TYPE_L3);
0449     skb->ip_summed = ((rx_buf->flags & EF4_RX_PKT_CSUMMED) ?
0450               CHECKSUM_UNNECESSARY : CHECKSUM_NONE);
0451 
0452     for (;;) {
0453         skb_fill_page_desc(skb, skb_shinfo(skb)->nr_frags,
0454                    rx_buf->page, rx_buf->page_offset,
0455                    rx_buf->len);
0456         rx_buf->page = NULL;
0457         skb->len += rx_buf->len;
0458         if (skb_shinfo(skb)->nr_frags == n_frags)
0459             break;
0460 
0461         rx_buf = ef4_rx_buf_next(&channel->rx_queue, rx_buf);
0462     }
0463 
0464     skb->data_len = skb->len;
0465     skb->truesize += n_frags * efx->rx_buffer_truesize;
0466 
0467     skb_record_rx_queue(skb, channel->rx_queue.core_index);
0468 
0469     napi_gro_frags(napi);
0470 }
0471 
0472 /* Allocate and construct an SKB around page fragments */
0473 static struct sk_buff *ef4_rx_mk_skb(struct ef4_channel *channel,
0474                      struct ef4_rx_buffer *rx_buf,
0475                      unsigned int n_frags,
0476                      u8 *eh, int hdr_len)
0477 {
0478     struct ef4_nic *efx = channel->efx;
0479     struct sk_buff *skb;
0480 
0481     /* Allocate an SKB to store the headers */
0482     skb = netdev_alloc_skb(efx->net_dev,
0483                    efx->rx_ip_align + efx->rx_prefix_size +
0484                    hdr_len);
0485     if (unlikely(skb == NULL)) {
0486         atomic_inc(&efx->n_rx_noskb_drops);
0487         return NULL;
0488     }
0489 
0490     EF4_BUG_ON_PARANOID(rx_buf->len < hdr_len);
0491 
0492     memcpy(skb->data + efx->rx_ip_align, eh - efx->rx_prefix_size,
0493            efx->rx_prefix_size + hdr_len);
0494     skb_reserve(skb, efx->rx_ip_align + efx->rx_prefix_size);
0495     __skb_put(skb, hdr_len);
0496 
0497     /* Append the remaining page(s) onto the frag list */
0498     if (rx_buf->len > hdr_len) {
0499         rx_buf->page_offset += hdr_len;
0500         rx_buf->len -= hdr_len;
0501 
0502         for (;;) {
0503             skb_fill_page_desc(skb, skb_shinfo(skb)->nr_frags,
0504                        rx_buf->page, rx_buf->page_offset,
0505                        rx_buf->len);
0506             rx_buf->page = NULL;
0507             skb->len += rx_buf->len;
0508             skb->data_len += rx_buf->len;
0509             if (skb_shinfo(skb)->nr_frags == n_frags)
0510                 break;
0511 
0512             rx_buf = ef4_rx_buf_next(&channel->rx_queue, rx_buf);
0513         }
0514     } else {
0515         __free_pages(rx_buf->page, efx->rx_buffer_order);
0516         rx_buf->page = NULL;
0517         n_frags = 0;
0518     }
0519 
0520     skb->truesize += n_frags * efx->rx_buffer_truesize;
0521 
0522     /* Move past the ethernet header */
0523     skb->protocol = eth_type_trans(skb, efx->net_dev);
0524 
0525     skb_mark_napi_id(skb, &channel->napi_str);
0526 
0527     return skb;
0528 }
0529 
0530 void ef4_rx_packet(struct ef4_rx_queue *rx_queue, unsigned int index,
0531            unsigned int n_frags, unsigned int len, u16 flags)
0532 {
0533     struct ef4_nic *efx = rx_queue->efx;
0534     struct ef4_channel *channel = ef4_rx_queue_channel(rx_queue);
0535     struct ef4_rx_buffer *rx_buf;
0536 
0537     rx_queue->rx_packets++;
0538 
0539     rx_buf = ef4_rx_buffer(rx_queue, index);
0540     rx_buf->flags |= flags;
0541 
0542     /* Validate the number of fragments and completed length */
0543     if (n_frags == 1) {
0544         if (!(flags & EF4_RX_PKT_PREFIX_LEN))
0545             ef4_rx_packet__check_len(rx_queue, rx_buf, len);
0546     } else if (unlikely(n_frags > EF4_RX_MAX_FRAGS) ||
0547            unlikely(len <= (n_frags - 1) * efx->rx_dma_len) ||
0548            unlikely(len > n_frags * efx->rx_dma_len) ||
0549            unlikely(!efx->rx_scatter)) {
0550         /* If this isn't an explicit discard request, either
0551          * the hardware or the driver is broken.
0552          */
0553         WARN_ON(!(len == 0 && rx_buf->flags & EF4_RX_PKT_DISCARD));
0554         rx_buf->flags |= EF4_RX_PKT_DISCARD;
0555     }
0556 
0557     netif_vdbg(efx, rx_status, efx->net_dev,
0558            "RX queue %d received ids %x-%x len %d %s%s\n",
0559            ef4_rx_queue_index(rx_queue), index,
0560            (index + n_frags - 1) & rx_queue->ptr_mask, len,
0561            (rx_buf->flags & EF4_RX_PKT_CSUMMED) ? " [SUMMED]" : "",
0562            (rx_buf->flags & EF4_RX_PKT_DISCARD) ? " [DISCARD]" : "");
0563 
0564     /* Discard packet, if instructed to do so.  Process the
0565      * previous receive first.
0566      */
0567     if (unlikely(rx_buf->flags & EF4_RX_PKT_DISCARD)) {
0568         ef4_rx_flush_packet(channel);
0569         ef4_discard_rx_packet(channel, rx_buf, n_frags);
0570         return;
0571     }
0572 
0573     if (n_frags == 1 && !(flags & EF4_RX_PKT_PREFIX_LEN))
0574         rx_buf->len = len;
0575 
0576     /* Release and/or sync the DMA mapping - assumes all RX buffers
0577      * consumed in-order per RX queue.
0578      */
0579     ef4_sync_rx_buffer(efx, rx_buf, rx_buf->len);
0580 
0581     /* Prefetch nice and early so data will (hopefully) be in cache by
0582      * the time we look at it.
0583      */
0584     prefetch(ef4_rx_buf_va(rx_buf));
0585 
0586     rx_buf->page_offset += efx->rx_prefix_size;
0587     rx_buf->len -= efx->rx_prefix_size;
0588 
0589     if (n_frags > 1) {
0590         /* Release/sync DMA mapping for additional fragments.
0591          * Fix length for last fragment.
0592          */
0593         unsigned int tail_frags = n_frags - 1;
0594 
0595         for (;;) {
0596             rx_buf = ef4_rx_buf_next(rx_queue, rx_buf);
0597             if (--tail_frags == 0)
0598                 break;
0599             ef4_sync_rx_buffer(efx, rx_buf, efx->rx_dma_len);
0600         }
0601         rx_buf->len = len - (n_frags - 1) * efx->rx_dma_len;
0602         ef4_sync_rx_buffer(efx, rx_buf, rx_buf->len);
0603     }
0604 
0605     /* All fragments have been DMA-synced, so recycle pages. */
0606     rx_buf = ef4_rx_buffer(rx_queue, index);
0607     ef4_recycle_rx_pages(channel, rx_buf, n_frags);
0608 
0609     /* Pipeline receives so that we give time for packet headers to be
0610      * prefetched into cache.
0611      */
0612     ef4_rx_flush_packet(channel);
0613     channel->rx_pkt_n_frags = n_frags;
0614     channel->rx_pkt_index = index;
0615 }
0616 
0617 static void ef4_rx_deliver(struct ef4_channel *channel, u8 *eh,
0618                struct ef4_rx_buffer *rx_buf,
0619                unsigned int n_frags)
0620 {
0621     struct sk_buff *skb;
0622     u16 hdr_len = min_t(u16, rx_buf->len, EF4_SKB_HEADERS);
0623 
0624     skb = ef4_rx_mk_skb(channel, rx_buf, n_frags, eh, hdr_len);
0625     if (unlikely(skb == NULL)) {
0626         struct ef4_rx_queue *rx_queue;
0627 
0628         rx_queue = ef4_channel_get_rx_queue(channel);
0629         ef4_free_rx_buffers(rx_queue, rx_buf, n_frags);
0630         return;
0631     }
0632     skb_record_rx_queue(skb, channel->rx_queue.core_index);
0633 
0634     /* Set the SKB flags */
0635     skb_checksum_none_assert(skb);
0636     if (likely(rx_buf->flags & EF4_RX_PKT_CSUMMED))
0637         skb->ip_summed = CHECKSUM_UNNECESSARY;
0638 
0639     if (channel->type->receive_skb)
0640         if (channel->type->receive_skb(channel, skb))
0641             return;
0642 
0643     /* Pass the packet up */
0644     netif_receive_skb(skb);
0645 }
0646 
0647 /* Handle a received packet.  Second half: Touches packet payload. */
0648 void __ef4_rx_packet(struct ef4_channel *channel)
0649 {
0650     struct ef4_nic *efx = channel->efx;
0651     struct ef4_rx_buffer *rx_buf =
0652         ef4_rx_buffer(&channel->rx_queue, channel->rx_pkt_index);
0653     u8 *eh = ef4_rx_buf_va(rx_buf);
0654 
0655     /* Read length from the prefix if necessary.  This already
0656      * excludes the length of the prefix itself.
0657      */
0658     if (rx_buf->flags & EF4_RX_PKT_PREFIX_LEN)
0659         rx_buf->len = le16_to_cpup((__le16 *)
0660                        (eh + efx->rx_packet_len_offset));
0661 
0662     /* If we're in loopback test, then pass the packet directly to the
0663      * loopback layer, and free the rx_buf here
0664      */
0665     if (unlikely(efx->loopback_selftest)) {
0666         struct ef4_rx_queue *rx_queue;
0667 
0668         ef4_loopback_rx_packet(efx, eh, rx_buf->len);
0669         rx_queue = ef4_channel_get_rx_queue(channel);
0670         ef4_free_rx_buffers(rx_queue, rx_buf,
0671                     channel->rx_pkt_n_frags);
0672         goto out;
0673     }
0674 
0675     if (unlikely(!(efx->net_dev->features & NETIF_F_RXCSUM)))
0676         rx_buf->flags &= ~EF4_RX_PKT_CSUMMED;
0677 
0678     if ((rx_buf->flags & EF4_RX_PKT_TCP) && !channel->type->receive_skb)
0679         ef4_rx_packet_gro(channel, rx_buf, channel->rx_pkt_n_frags, eh);
0680     else
0681         ef4_rx_deliver(channel, eh, rx_buf, channel->rx_pkt_n_frags);
0682 out:
0683     channel->rx_pkt_n_frags = 0;
0684 }
0685 
0686 int ef4_probe_rx_queue(struct ef4_rx_queue *rx_queue)
0687 {
0688     struct ef4_nic *efx = rx_queue->efx;
0689     unsigned int entries;
0690     int rc;
0691 
0692     /* Create the smallest power-of-two aligned ring */
0693     entries = max(roundup_pow_of_two(efx->rxq_entries), EF4_MIN_DMAQ_SIZE);
0694     EF4_BUG_ON_PARANOID(entries > EF4_MAX_DMAQ_SIZE);
0695     rx_queue->ptr_mask = entries - 1;
0696 
0697     netif_dbg(efx, probe, efx->net_dev,
0698           "creating RX queue %d size %#x mask %#x\n",
0699           ef4_rx_queue_index(rx_queue), efx->rxq_entries,
0700           rx_queue->ptr_mask);
0701 
0702     /* Allocate RX buffers */
0703     rx_queue->buffer = kcalloc(entries, sizeof(*rx_queue->buffer),
0704                    GFP_KERNEL);
0705     if (!rx_queue->buffer)
0706         return -ENOMEM;
0707 
0708     rc = ef4_nic_probe_rx(rx_queue);
0709     if (rc) {
0710         kfree(rx_queue->buffer);
0711         rx_queue->buffer = NULL;
0712     }
0713 
0714     return rc;
0715 }
0716 
0717 static void ef4_init_rx_recycle_ring(struct ef4_nic *efx,
0718                      struct ef4_rx_queue *rx_queue)
0719 {
0720     unsigned int bufs_in_recycle_ring, page_ring_size;
0721     struct iommu_domain __maybe_unused *domain;
0722 
0723     /* Set the RX recycle ring size */
0724 #ifdef CONFIG_PPC64
0725     bufs_in_recycle_ring = EF4_RECYCLE_RING_SIZE_IOMMU;
0726 #else
0727     domain = iommu_get_domain_for_dev(&efx->pci_dev->dev);
0728     if (domain && domain->type != IOMMU_DOMAIN_IDENTITY)
0729         bufs_in_recycle_ring = EF4_RECYCLE_RING_SIZE_IOMMU;
0730     else
0731         bufs_in_recycle_ring = EF4_RECYCLE_RING_SIZE_NOIOMMU;
0732 #endif /* CONFIG_PPC64 */
0733 
0734     page_ring_size = roundup_pow_of_two(bufs_in_recycle_ring /
0735                         efx->rx_bufs_per_page);
0736     rx_queue->page_ring = kcalloc(page_ring_size,
0737                       sizeof(*rx_queue->page_ring), GFP_KERNEL);
0738     if (!rx_queue->page_ring)
0739         rx_queue->page_ptr_mask = 0;
0740     else
0741         rx_queue->page_ptr_mask = page_ring_size - 1;
0742 }
0743 
0744 void ef4_init_rx_queue(struct ef4_rx_queue *rx_queue)
0745 {
0746     struct ef4_nic *efx = rx_queue->efx;
0747     unsigned int max_fill, trigger, max_trigger;
0748 
0749     netif_dbg(rx_queue->efx, drv, rx_queue->efx->net_dev,
0750           "initialising RX queue %d\n", ef4_rx_queue_index(rx_queue));
0751 
0752     /* Initialise ptr fields */
0753     rx_queue->added_count = 0;
0754     rx_queue->notified_count = 0;
0755     rx_queue->removed_count = 0;
0756     rx_queue->min_fill = -1U;
0757     ef4_init_rx_recycle_ring(efx, rx_queue);
0758 
0759     rx_queue->page_remove = 0;
0760     rx_queue->page_add = rx_queue->page_ptr_mask + 1;
0761     rx_queue->page_recycle_count = 0;
0762     rx_queue->page_recycle_failed = 0;
0763     rx_queue->page_recycle_full = 0;
0764 
0765     /* Initialise limit fields */
0766     max_fill = efx->rxq_entries - EF4_RXD_HEAD_ROOM;
0767     max_trigger =
0768         max_fill - efx->rx_pages_per_batch * efx->rx_bufs_per_page;
0769     if (rx_refill_threshold != 0) {
0770         trigger = max_fill * min(rx_refill_threshold, 100U) / 100U;
0771         if (trigger > max_trigger)
0772             trigger = max_trigger;
0773     } else {
0774         trigger = max_trigger;
0775     }
0776 
0777     rx_queue->max_fill = max_fill;
0778     rx_queue->fast_fill_trigger = trigger;
0779     rx_queue->refill_enabled = true;
0780 
0781     /* Set up RX descriptor ring */
0782     ef4_nic_init_rx(rx_queue);
0783 }
0784 
0785 void ef4_fini_rx_queue(struct ef4_rx_queue *rx_queue)
0786 {
0787     int i;
0788     struct ef4_nic *efx = rx_queue->efx;
0789     struct ef4_rx_buffer *rx_buf;
0790 
0791     netif_dbg(rx_queue->efx, drv, rx_queue->efx->net_dev,
0792           "shutting down RX queue %d\n", ef4_rx_queue_index(rx_queue));
0793 
0794     del_timer_sync(&rx_queue->slow_fill);
0795 
0796     /* Release RX buffers from the current read ptr to the write ptr */
0797     if (rx_queue->buffer) {
0798         for (i = rx_queue->removed_count; i < rx_queue->added_count;
0799              i++) {
0800             unsigned index = i & rx_queue->ptr_mask;
0801             rx_buf = ef4_rx_buffer(rx_queue, index);
0802             ef4_fini_rx_buffer(rx_queue, rx_buf);
0803         }
0804     }
0805 
0806     /* Unmap and release the pages in the recycle ring. Remove the ring. */
0807     for (i = 0; i <= rx_queue->page_ptr_mask; i++) {
0808         struct page *page = rx_queue->page_ring[i];
0809         struct ef4_rx_page_state *state;
0810 
0811         if (page == NULL)
0812             continue;
0813 
0814         state = page_address(page);
0815         dma_unmap_page(&efx->pci_dev->dev, state->dma_addr,
0816                    PAGE_SIZE << efx->rx_buffer_order,
0817                    DMA_FROM_DEVICE);
0818         put_page(page);
0819     }
0820     kfree(rx_queue->page_ring);
0821     rx_queue->page_ring = NULL;
0822 }
0823 
0824 void ef4_remove_rx_queue(struct ef4_rx_queue *rx_queue)
0825 {
0826     netif_dbg(rx_queue->efx, drv, rx_queue->efx->net_dev,
0827           "destroying RX queue %d\n", ef4_rx_queue_index(rx_queue));
0828 
0829     ef4_nic_remove_rx(rx_queue);
0830 
0831     kfree(rx_queue->buffer);
0832     rx_queue->buffer = NULL;
0833 }
0834 
0835 
0836 module_param(rx_refill_threshold, uint, 0444);
0837 MODULE_PARM_DESC(rx_refill_threshold,
0838          "RX descriptor ring refill threshold (%)");
0839 
0840 #ifdef CONFIG_RFS_ACCEL
0841 
0842 int ef4_filter_rfs(struct net_device *net_dev, const struct sk_buff *skb,
0843            u16 rxq_index, u32 flow_id)
0844 {
0845     struct ef4_nic *efx = netdev_priv(net_dev);
0846     struct ef4_channel *channel;
0847     struct ef4_filter_spec spec;
0848     struct flow_keys fk;
0849     int rc;
0850 
0851     if (flow_id == RPS_FLOW_ID_INVALID)
0852         return -EINVAL;
0853 
0854     if (!skb_flow_dissect_flow_keys(skb, &fk, 0))
0855         return -EPROTONOSUPPORT;
0856 
0857     if (fk.basic.n_proto != htons(ETH_P_IP) && fk.basic.n_proto != htons(ETH_P_IPV6))
0858         return -EPROTONOSUPPORT;
0859     if (fk.control.flags & FLOW_DIS_IS_FRAGMENT)
0860         return -EPROTONOSUPPORT;
0861 
0862     ef4_filter_init_rx(&spec, EF4_FILTER_PRI_HINT,
0863                efx->rx_scatter ? EF4_FILTER_FLAG_RX_SCATTER : 0,
0864                rxq_index);
0865     spec.match_flags =
0866         EF4_FILTER_MATCH_ETHER_TYPE | EF4_FILTER_MATCH_IP_PROTO |
0867         EF4_FILTER_MATCH_LOC_HOST | EF4_FILTER_MATCH_LOC_PORT |
0868         EF4_FILTER_MATCH_REM_HOST | EF4_FILTER_MATCH_REM_PORT;
0869     spec.ether_type = fk.basic.n_proto;
0870     spec.ip_proto = fk.basic.ip_proto;
0871 
0872     if (fk.basic.n_proto == htons(ETH_P_IP)) {
0873         spec.rem_host[0] = fk.addrs.v4addrs.src;
0874         spec.loc_host[0] = fk.addrs.v4addrs.dst;
0875     } else {
0876         memcpy(spec.rem_host, &fk.addrs.v6addrs.src, sizeof(struct in6_addr));
0877         memcpy(spec.loc_host, &fk.addrs.v6addrs.dst, sizeof(struct in6_addr));
0878     }
0879 
0880     spec.rem_port = fk.ports.src;
0881     spec.loc_port = fk.ports.dst;
0882 
0883     rc = efx->type->filter_rfs_insert(efx, &spec);
0884     if (rc < 0)
0885         return rc;
0886 
0887     /* Remember this so we can check whether to expire the filter later */
0888     channel = ef4_get_channel(efx, rxq_index);
0889     channel->rps_flow_id[rc] = flow_id;
0890     ++channel->rfs_filters_added;
0891 
0892     if (spec.ether_type == htons(ETH_P_IP))
0893         netif_info(efx, rx_status, efx->net_dev,
0894                "steering %s %pI4:%u:%pI4:%u to queue %u [flow %u filter %d]\n",
0895                (spec.ip_proto == IPPROTO_TCP) ? "TCP" : "UDP",
0896                spec.rem_host, ntohs(spec.rem_port), spec.loc_host,
0897                ntohs(spec.loc_port), rxq_index, flow_id, rc);
0898     else
0899         netif_info(efx, rx_status, efx->net_dev,
0900                "steering %s [%pI6]:%u:[%pI6]:%u to queue %u [flow %u filter %d]\n",
0901                (spec.ip_proto == IPPROTO_TCP) ? "TCP" : "UDP",
0902                spec.rem_host, ntohs(spec.rem_port), spec.loc_host,
0903                ntohs(spec.loc_port), rxq_index, flow_id, rc);
0904 
0905     return rc;
0906 }
0907 
0908 bool __ef4_filter_rfs_expire(struct ef4_nic *efx, unsigned int quota)
0909 {
0910     bool (*expire_one)(struct ef4_nic *efx, u32 flow_id, unsigned int index);
0911     unsigned int channel_idx, index, size;
0912     u32 flow_id;
0913 
0914     if (!spin_trylock_bh(&efx->filter_lock))
0915         return false;
0916 
0917     expire_one = efx->type->filter_rfs_expire_one;
0918     channel_idx = efx->rps_expire_channel;
0919     index = efx->rps_expire_index;
0920     size = efx->type->max_rx_ip_filters;
0921     while (quota--) {
0922         struct ef4_channel *channel = ef4_get_channel(efx, channel_idx);
0923         flow_id = channel->rps_flow_id[index];
0924 
0925         if (flow_id != RPS_FLOW_ID_INVALID &&
0926             expire_one(efx, flow_id, index)) {
0927             netif_info(efx, rx_status, efx->net_dev,
0928                    "expired filter %d [queue %u flow %u]\n",
0929                    index, channel_idx, flow_id);
0930             channel->rps_flow_id[index] = RPS_FLOW_ID_INVALID;
0931         }
0932         if (++index == size) {
0933             if (++channel_idx == efx->n_channels)
0934                 channel_idx = 0;
0935             index = 0;
0936         }
0937     }
0938     efx->rps_expire_channel = channel_idx;
0939     efx->rps_expire_index = index;
0940 
0941     spin_unlock_bh(&efx->filter_lock);
0942     return true;
0943 }
0944 
0945 #endif /* CONFIG_RFS_ACCEL */
0946 
0947 /**
0948  * ef4_filter_is_mc_recipient - test whether spec is a multicast recipient
0949  * @spec: Specification to test
0950  *
0951  * Return: %true if the specification is a non-drop RX filter that
0952  * matches a local MAC address I/G bit value of 1 or matches a local
0953  * IPv4 or IPv6 address value in the respective multicast address
0954  * range.  Otherwise %false.
0955  */
0956 bool ef4_filter_is_mc_recipient(const struct ef4_filter_spec *spec)
0957 {
0958     if (!(spec->flags & EF4_FILTER_FLAG_RX) ||
0959         spec->dmaq_id == EF4_FILTER_RX_DMAQ_ID_DROP)
0960         return false;
0961 
0962     if (spec->match_flags &
0963         (EF4_FILTER_MATCH_LOC_MAC | EF4_FILTER_MATCH_LOC_MAC_IG) &&
0964         is_multicast_ether_addr(spec->loc_mac))
0965         return true;
0966 
0967     if ((spec->match_flags &
0968          (EF4_FILTER_MATCH_ETHER_TYPE | EF4_FILTER_MATCH_LOC_HOST)) ==
0969         (EF4_FILTER_MATCH_ETHER_TYPE | EF4_FILTER_MATCH_LOC_HOST)) {
0970         if (spec->ether_type == htons(ETH_P_IP) &&
0971             ipv4_is_multicast(spec->loc_host[0]))
0972             return true;
0973         if (spec->ether_type == htons(ETH_P_IPV6) &&
0974             ((const u8 *)spec->loc_host)[0] == 0xff)
0975             return true;
0976     }
0977 
0978     return false;
0979 }
[ Source navigation ] [ Diff markup ] [ Identifier search ] [ general search ]

This page was automatically generated by the 2.1.0 LXR engine.

The LXR team
Valid CSS 2.1! Valid HTML 4.01!