broadcom/bnxt/bnxt_xdp.c

0001 /* Broadcom NetXtreme-C/E network driver.
0002  *
0003  * Copyright (c) 2016-2017 Broadcom Limited
0004  *
0005  * This program is free software; you can redistribute it and/or modify
0006  * it under the terms of the GNU General Public License as published by
0007  * the Free Software Foundation.
0008  */
0009 #include <linux/kernel.h>
0010 #include <linux/errno.h>
0011 #include <linux/pci.h>
0012 #include <linux/netdevice.h>
0013 #include <linux/etherdevice.h>
0014 #include <linux/if_vlan.h>
0015 #include <linux/bpf.h>
0016 #include <linux/bpf_trace.h>
0017 #include <linux/filter.h>
0018 #include <net/page_pool.h>
0019 #include "bnxt_hsi.h"
0020 #include "bnxt.h"
0021 #include "bnxt_xdp.h"
0022
0023 DEFINE_STATIC_KEY_FALSE(bnxt_xdp_locking_key);
0024
0025 struct bnxt_sw_tx_bd *bnxt_xmit_bd(struct bnxt *bp,
0026                    struct bnxt_tx_ring_info *txr,
0027                    dma_addr_t mapping, u32 len,
0028                    struct xdp_buff *xdp)
0029 {
0030     struct skb_shared_info *sinfo;
0031     struct bnxt_sw_tx_bd *tx_buf;
0032     struct tx_bd *txbd;
0033     int num_frags = 0;
0034     u32 flags;
0035     u16 prod;
0036     int i;
0037
0038     if (xdp && xdp_buff_has_frags(xdp)) {
0039         sinfo = xdp_get_shared_info_from_buff(xdp);
0040         num_frags = sinfo->nr_frags;
0041     }
0042
0043     /* fill up the first buffer */
0044     prod = txr->tx_prod;
0045     tx_buf = &txr->tx_buf_ring[prod];
0046     tx_buf->nr_frags = num_frags;
0047     if (xdp)
0048         tx_buf->page = virt_to_head_page(xdp->data);
0049
0050     txbd = &txr->tx_desc_ring[TX_RING(prod)][TX_IDX(prod)];
0051     flags = (len << TX_BD_LEN_SHIFT) |
0052         ((num_frags + 1) << TX_BD_FLAGS_BD_CNT_SHIFT) |
0053         bnxt_lhint_arr[len >> 9];
0054     txbd->tx_bd_len_flags_type = cpu_to_le32(flags);
0055     txbd->tx_bd_opaque = prod;
0056     txbd->tx_bd_haddr = cpu_to_le64(mapping);
0057
0058     /* now let us fill up the frags into the next buffers */
0059     for (i = 0; i < num_frags ; i++) {
0060         skb_frag_t *frag = &sinfo->frags[i];
0061         struct bnxt_sw_tx_bd *frag_tx_buf;
0062         struct pci_dev *pdev = bp->pdev;
0063         dma_addr_t frag_mapping;
0064         int frag_len;
0065
0066         prod = NEXT_TX(prod);
0067         txr->tx_prod = prod;
0068
0069         /* first fill up the first buffer */
0070         frag_tx_buf = &txr->tx_buf_ring[prod];
0071         frag_tx_buf->page = skb_frag_page(frag);
0072
0073         txbd = &txr->tx_desc_ring[TX_RING(prod)][TX_IDX(prod)];
0074
0075         frag_len = skb_frag_size(frag);
0076         frag_mapping = skb_frag_dma_map(&pdev->dev, frag, 0,
0077                         frag_len, DMA_TO_DEVICE);
0078
0079         if (unlikely(dma_mapping_error(&pdev->dev, frag_mapping)))
0080             return NULL;
0081
0082         dma_unmap_addr_set(frag_tx_buf, mapping, frag_mapping);
0083
0084         flags = frag_len << TX_BD_LEN_SHIFT;
0085         txbd->tx_bd_len_flags_type = cpu_to_le32(flags);
0086         txbd->tx_bd_haddr = cpu_to_le64(frag_mapping);
0087
0088         len = frag_len;
0089     }
0090
0091     flags &= ~TX_BD_LEN;
0092     txbd->tx_bd_len_flags_type = cpu_to_le32(((len) << TX_BD_LEN_SHIFT) | flags |
0093             TX_BD_FLAGS_PACKET_END);
0094     /* Sync TX BD */
0095     wmb();
0096     prod = NEXT_TX(prod);
0097     txr->tx_prod = prod;
0098
0099     return tx_buf;
0100 }
0101
0102 static void __bnxt_xmit_xdp(struct bnxt *bp, struct bnxt_tx_ring_info *txr,
0103                 dma_addr_t mapping, u32 len, u16 rx_prod,
0104                 struct xdp_buff *xdp)
0105 {
0106     struct bnxt_sw_tx_bd *tx_buf;
0107
0108     tx_buf = bnxt_xmit_bd(bp, txr, mapping, len, xdp);
0109     tx_buf->rx_prod = rx_prod;
0110     tx_buf->action = XDP_TX;
0111
0112 }
0113
0114 static void __bnxt_xmit_xdp_redirect(struct bnxt *bp,
0115                      struct bnxt_tx_ring_info *txr,
0116                      dma_addr_t mapping, u32 len,
0117                      struct xdp_frame *xdpf)
0118 {
0119     struct bnxt_sw_tx_bd *tx_buf;
0120
0121     tx_buf = bnxt_xmit_bd(bp, txr, mapping, len, NULL);
0122     tx_buf->action = XDP_REDIRECT;
0123     tx_buf->xdpf = xdpf;
0124     dma_unmap_addr_set(tx_buf, mapping, mapping);
0125     dma_unmap_len_set(tx_buf, len, 0);
0126 }
0127
0128 void bnxt_tx_int_xdp(struct bnxt *bp, struct bnxt_napi *bnapi, int nr_pkts)
0129 {
0130     struct bnxt_tx_ring_info *txr = bnapi->tx_ring;
0131     struct bnxt_rx_ring_info *rxr = bnapi->rx_ring;
0132     bool rx_doorbell_needed = false;
0133     struct bnxt_sw_tx_bd *tx_buf;
0134     u16 tx_cons = txr->tx_cons;
0135     u16 last_tx_cons = tx_cons;
0136     int i, j, frags;
0137
0138     for (i = 0; i < nr_pkts; i++) {
0139         tx_buf = &txr->tx_buf_ring[tx_cons];
0140
0141         if (tx_buf->action == XDP_REDIRECT) {
0142             struct pci_dev *pdev = bp->pdev;
0143
0144             dma_unmap_single(&pdev->dev,
0145                      dma_unmap_addr(tx_buf, mapping),
0146                      dma_unmap_len(tx_buf, len),
0147                      DMA_TO_DEVICE);
0148             xdp_return_frame(tx_buf->xdpf);
0149             tx_buf->action = 0;
0150             tx_buf->xdpf = NULL;
0151         } else if (tx_buf->action == XDP_TX) {
0152             rx_doorbell_needed = true;
0153             last_tx_cons = tx_cons;
0154
0155             frags = tx_buf->nr_frags;
0156             for (j = 0; j < frags; j++) {
0157                 tx_cons = NEXT_TX(tx_cons);
0158                 tx_buf = &txr->tx_buf_ring[tx_cons];
0159                 page_pool_recycle_direct(rxr->page_pool, tx_buf->page);
0160             }
0161         }
0162         tx_cons = NEXT_TX(tx_cons);
0163     }
0164     txr->tx_cons = tx_cons;
0165     if (rx_doorbell_needed) {
0166         tx_buf = &txr->tx_buf_ring[last_tx_cons];
0167         bnxt_db_write(bp, &rxr->rx_db, tx_buf->rx_prod);
0168
0169     }
0170 }
0171
0172 bool bnxt_xdp_attached(struct bnxt *bp, struct bnxt_rx_ring_info *rxr)
0173 {
0174     struct bpf_prog *xdp_prog = READ_ONCE(rxr->xdp_prog);
0175
0176     return !!xdp_prog;
0177 }
0178
0179 void bnxt_xdp_buff_init(struct bnxt *bp, struct bnxt_rx_ring_info *rxr,
0180             u16 cons, u8 **data_ptr, unsigned int *len,
0181             struct xdp_buff *xdp)
0182 {
0183     struct bnxt_sw_rx_bd *rx_buf;
0184     u32 buflen = PAGE_SIZE;
0185     struct pci_dev *pdev;
0186     dma_addr_t mapping;
0187     u32 offset;
0188
0189     pdev = bp->pdev;
0190     rx_buf = &rxr->rx_buf_ring[cons];
0191     offset = bp->rx_offset;
0192
0193     mapping = rx_buf->mapping - bp->rx_dma_offset;
0194     dma_sync_single_for_cpu(&pdev->dev, mapping + offset, *len, bp->rx_dir);
0195
0196     if (bp->xdp_has_frags)
0197         buflen = BNXT_PAGE_MODE_BUF_SIZE + offset;
0198
0199     xdp_init_buff(xdp, buflen, &rxr->xdp_rxq);
0200     xdp_prepare_buff(xdp, *data_ptr - offset, offset, *len, false);
0201 }
0202
0203 void bnxt_xdp_buff_frags_free(struct bnxt_rx_ring_info *rxr,
0204                   struct xdp_buff *xdp)
0205 {
0206     struct skb_shared_info *shinfo;
0207     int i;
0208
0209     if (!xdp || !xdp_buff_has_frags(xdp))
0210         return;
0211     shinfo = xdp_get_shared_info_from_buff(xdp);
0212     for (i = 0; i < shinfo->nr_frags; i++) {
0213         struct page *page = skb_frag_page(&shinfo->frags[i]);
0214
0215         page_pool_recycle_direct(rxr->page_pool, page);
0216     }
0217     shinfo->nr_frags = 0;
0218 }
0219
0220 /* returns the following:
0221  * true    - packet consumed by XDP and new buffer is allocated.
0222  * false   - packet should be passed to the stack.
0223  */
0224 bool bnxt_rx_xdp(struct bnxt *bp, struct bnxt_rx_ring_info *rxr, u16 cons,
0225          struct xdp_buff xdp, struct page *page, unsigned int *len, u8 *event)
0226 {
0227     struct bpf_prog *xdp_prog = READ_ONCE(rxr->xdp_prog);
0228     struct bnxt_tx_ring_info *txr;
0229     struct bnxt_sw_rx_bd *rx_buf;
0230     struct pci_dev *pdev;
0231     dma_addr_t mapping;
0232     u32 tx_needed = 1;
0233     void *orig_data;
0234     u32 tx_avail;
0235     u32 offset;
0236     u32 act;
0237
0238     if (!xdp_prog)
0239         return false;
0240
0241     pdev = bp->pdev;
0242     offset = bp->rx_offset;
0243
0244     txr = rxr->bnapi->tx_ring;
0245     /* BNXT_RX_PAGE_MODE(bp) when XDP enabled */
0246     orig_data = xdp.data;
0247
0248     act = bpf_prog_run_xdp(xdp_prog, &xdp);
0249
0250     tx_avail = bnxt_tx_avail(bp, txr);
0251     /* If the tx ring is not full, we must not update the rx producer yet
0252      * because we may still be transmitting on some BDs.
0253      */
0254     if (tx_avail != bp->tx_ring_size)
0255         *event &= ~BNXT_RX_EVENT;
0256
0257     *len = xdp.data_end - xdp.data;
0258     if (orig_data != xdp.data)
0259         offset = xdp.data - xdp.data_hard_start;
0260
0261     switch (act) {
0262     case XDP_PASS:
0263         return false;
0264
0265     case XDP_TX:
0266         rx_buf = &rxr->rx_buf_ring[cons];
0267         mapping = rx_buf->mapping - bp->rx_dma_offset;
0268         *event = 0;
0269
0270         if (unlikely(xdp_buff_has_frags(&xdp))) {
0271             struct skb_shared_info *sinfo = xdp_get_shared_info_from_buff(&xdp);
0272
0273             tx_needed += sinfo->nr_frags;
0274             *event = BNXT_AGG_EVENT;
0275         }
0276
0277         if (tx_avail < tx_needed) {
0278             trace_xdp_exception(bp->dev, xdp_prog, act);
0279             bnxt_xdp_buff_frags_free(rxr, &xdp);
0280             bnxt_reuse_rx_data(rxr, cons, page);
0281             return true;
0282         }
0283
0284         dma_sync_single_for_device(&pdev->dev, mapping + offset, *len,
0285                        bp->rx_dir);
0286
0287         *event |= BNXT_TX_EVENT;
0288         __bnxt_xmit_xdp(bp, txr, mapping + offset, *len,
0289                 NEXT_RX(rxr->rx_prod), &xdp);
0290         bnxt_reuse_rx_data(rxr, cons, page);
0291         return true;
0292     case XDP_REDIRECT:
0293         /* if we are calling this here then we know that the
0294          * redirect is coming from a frame received by the
0295          * bnxt_en driver.
0296          */
0297         rx_buf = &rxr->rx_buf_ring[cons];
0298         mapping = rx_buf->mapping - bp->rx_dma_offset;
0299         dma_unmap_page_attrs(&pdev->dev, mapping,
0300                      PAGE_SIZE, bp->rx_dir,
0301                      DMA_ATTR_WEAK_ORDERING);
0302
0303         /* if we are unable to allocate a new buffer, abort and reuse */
0304         if (bnxt_alloc_rx_data(bp, rxr, rxr->rx_prod, GFP_ATOMIC)) {
0305             trace_xdp_exception(bp->dev, xdp_prog, act);
0306             bnxt_xdp_buff_frags_free(rxr, &xdp);
0307             bnxt_reuse_rx_data(rxr, cons, page);
0308             return true;
0309         }
0310
0311         if (xdp_do_redirect(bp->dev, &xdp, xdp_prog)) {
0312             trace_xdp_exception(bp->dev, xdp_prog, act);
0313             page_pool_recycle_direct(rxr->page_pool, page);
0314             return true;
0315         }
0316
0317         *event |= BNXT_REDIRECT_EVENT;
0318         break;
0319     default:
0320         bpf_warn_invalid_xdp_action(bp->dev, xdp_prog, act);
0321         fallthrough;
0322     case XDP_ABORTED:
0323         trace_xdp_exception(bp->dev, xdp_prog, act);
0324         fallthrough;
0325     case XDP_DROP:
0326         bnxt_xdp_buff_frags_free(rxr, &xdp);
0327         bnxt_reuse_rx_data(rxr, cons, page);
0328         break;
0329     }
0330     return true;
0331 }
0332
0333 int bnxt_xdp_xmit(struct net_device *dev, int num_frames,
0334           struct xdp_frame **frames, u32 flags)
0335 {
0336     struct bnxt *bp = netdev_priv(dev);
0337     struct bpf_prog *xdp_prog = READ_ONCE(bp->xdp_prog);
0338     struct pci_dev *pdev = bp->pdev;
0339     struct bnxt_tx_ring_info *txr;
0340     dma_addr_t mapping;
0341     int nxmit = 0;
0342     int ring;
0343     int i;
0344
0345     if (!test_bit(BNXT_STATE_OPEN, &bp->state) ||
0346         !bp->tx_nr_rings_xdp ||
0347         !xdp_prog)
0348         return -EINVAL;
0349
0350     ring = smp_processor_id() % bp->tx_nr_rings_xdp;
0351     txr = &bp->tx_ring[ring];
0352
0353     if (READ_ONCE(txr->dev_state) == BNXT_DEV_STATE_CLOSING)
0354         return -EINVAL;
0355
0356     if (static_branch_unlikely(&bnxt_xdp_locking_key))
0357         spin_lock(&txr->xdp_tx_lock);
0358
0359     for (i = 0; i < num_frames; i++) {
0360         struct xdp_frame *xdp = frames[i];
0361
0362         if (!bnxt_tx_avail(bp, txr))
0363             break;
0364
0365         mapping = dma_map_single(&pdev->dev, xdp->data, xdp->len,
0366                      DMA_TO_DEVICE);
0367
0368         if (dma_mapping_error(&pdev->dev, mapping))
0369             break;
0370
0371         __bnxt_xmit_xdp_redirect(bp, txr, mapping, xdp->len, xdp);
0372         nxmit++;
0373     }
0374
0375     if (flags & XDP_XMIT_FLUSH) {
0376         /* Sync BD data before updating doorbell */
0377         wmb();
0378         bnxt_db_write(bp, &txr->tx_db, txr->tx_prod);
0379     }
0380
0381     if (static_branch_unlikely(&bnxt_xdp_locking_key))
0382         spin_unlock(&txr->xdp_tx_lock);
0383
0384     return nxmit;
0385 }
0386
0387 /* Under rtnl_lock */
0388 static int bnxt_xdp_set(struct bnxt *bp, struct bpf_prog *prog)
0389 {
0390     struct net_device *dev = bp->dev;
0391     int tx_xdp = 0, rc, tc;
0392     struct bpf_prog *old;
0393
0394     if (prog && !prog->aux->xdp_has_frags &&
0395         bp->dev->mtu > BNXT_MAX_PAGE_MODE_MTU) {
0396         netdev_warn(dev, "MTU %d larger than %d without XDP frag support.\n",
0397                 bp->dev->mtu, BNXT_MAX_PAGE_MODE_MTU);
0398         return -EOPNOTSUPP;
0399     }
0400     if (!(bp->flags & BNXT_FLAG_SHARED_RINGS)) {
0401         netdev_warn(dev, "ethtool rx/tx channels must be combined to support XDP.\n");
0402         return -EOPNOTSUPP;
0403     }
0404     if (prog) {
0405         tx_xdp = bp->rx_nr_rings;
0406         bp->xdp_has_frags = prog->aux->xdp_has_frags;
0407     }
0408
0409     tc = netdev_get_num_tc(dev);
0410     if (!tc)
0411         tc = 1;
0412     rc = bnxt_check_rings(bp, bp->tx_nr_rings_per_tc, bp->rx_nr_rings,
0413                   true, tc, tx_xdp);
0414     if (rc) {
0415         netdev_warn(dev, "Unable to reserve enough TX rings to support XDP.\n");
0416         return rc;
0417     }
0418     if (netif_running(dev))
0419         bnxt_close_nic(bp, true, false);
0420
0421     old = xchg(&bp->xdp_prog, prog);
0422     if (old)
0423         bpf_prog_put(old);
0424
0425     if (prog) {
0426         bnxt_set_rx_skb_mode(bp, true);
0427     } else {
0428         int rx, tx;
0429
0430         bnxt_set_rx_skb_mode(bp, false);
0431         bnxt_get_max_rings(bp, &rx, &tx, true);
0432         if (rx > 1) {
0433             bp->flags &= ~BNXT_FLAG_NO_AGG_RINGS;
0434             bp->dev->hw_features |= NETIF_F_LRO;
0435         }
0436     }
0437     bp->tx_nr_rings_xdp = tx_xdp;
0438     bp->tx_nr_rings = bp->tx_nr_rings_per_tc * tc + tx_xdp;
0439     bp->cp_nr_rings = max_t(int, bp->tx_nr_rings, bp->rx_nr_rings);
0440     bnxt_set_tpa_flags(bp);
0441     bnxt_set_ring_params(bp);
0442
0443     if (netif_running(dev))
0444         return bnxt_open_nic(bp, true, false);
0445
0446     return 0;
0447 }
0448
0449 int bnxt_xdp(struct net_device *dev, struct netdev_bpf *xdp)
0450 {
0451     struct bnxt *bp = netdev_priv(dev);
0452     int rc;
0453
0454     switch (xdp->command) {
0455     case XDP_SETUP_PROG:
0456         rc = bnxt_xdp_set(bp, xdp->prog);
0457         break;
0458     default:
0459         rc = -EINVAL;
0460         break;
0461     }
0462     return rc;
0463 }
0464
0465 struct sk_buff *
0466 bnxt_xdp_build_skb(struct bnxt *bp, struct sk_buff *skb, u8 num_frags,
0467            struct page_pool *pool, struct xdp_buff *xdp,
0468            struct rx_cmp_ext *rxcmp1)
0469 {
0470     struct skb_shared_info *sinfo = xdp_get_shared_info_from_buff(xdp);
0471
0472     if (!skb)
0473         return NULL;
0474     skb_checksum_none_assert(skb);
0475     if (RX_CMP_L4_CS_OK(rxcmp1)) {
0476         if (bp->dev->features & NETIF_F_RXCSUM) {
0477             skb->ip_summed = CHECKSUM_UNNECESSARY;
0478             skb->csum_level = RX_CMP_ENCAP(rxcmp1);
0479         }
0480     }
0481     xdp_update_skb_shared_info(skb, num_frags,
0482                    sinfo->xdp_frags_size,
0483                    PAGE_SIZE * sinfo->nr_frags,
0484                    xdp_buff_is_frag_pfmemalloc(xdp));
0485     return skb;
0486 }