netronome/nfp/nfp_net_xsk.c

0001 // SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause)
0002 /* Copyright (C) 2018 Netronome Systems, Inc */
0003 /* Copyright (C) 2021 Corigine, Inc */
0004
0005 #include <linux/dma-direction.h>
0006 #include <linux/dma-mapping.h>
0007 #include <linux/slab.h>
0008 #include <net/xdp_sock_drv.h>
0009 #include <trace/events/xdp.h>
0010
0011 #include "nfp_app.h"
0012 #include "nfp_net.h"
0013 #include "nfp_net_dp.h"
0014 #include "nfp_net_xsk.h"
0015
0016 static void
0017 nfp_net_xsk_rx_bufs_stash(struct nfp_net_rx_ring *rx_ring, unsigned int idx,
0018               struct xdp_buff *xdp)
0019 {
0020     unsigned int headroom;
0021
0022     headroom = xsk_pool_get_headroom(rx_ring->r_vec->xsk_pool);
0023
0024     rx_ring->rxds[idx].fld.reserved = 0;
0025     rx_ring->rxds[idx].fld.meta_len_dd = 0;
0026
0027     rx_ring->xsk_rxbufs[idx].xdp = xdp;
0028     rx_ring->xsk_rxbufs[idx].dma_addr =
0029         xsk_buff_xdp_get_frame_dma(xdp) + headroom;
0030 }
0031
0032 void nfp_net_xsk_rx_unstash(struct nfp_net_xsk_rx_buf *rxbuf)
0033 {
0034     rxbuf->dma_addr = 0;
0035     rxbuf->xdp = NULL;
0036 }
0037
0038 void nfp_net_xsk_rx_free(struct nfp_net_xsk_rx_buf *rxbuf)
0039 {
0040     if (rxbuf->xdp)
0041         xsk_buff_free(rxbuf->xdp);
0042
0043     nfp_net_xsk_rx_unstash(rxbuf);
0044 }
0045
0046 void nfp_net_xsk_rx_bufs_free(struct nfp_net_rx_ring *rx_ring)
0047 {
0048     unsigned int i;
0049
0050     if (!rx_ring->cnt)
0051         return;
0052
0053     for (i = 0; i < rx_ring->cnt - 1; i++)
0054         nfp_net_xsk_rx_free(&rx_ring->xsk_rxbufs[i]);
0055 }
0056
0057 void nfp_net_xsk_rx_ring_fill_freelist(struct nfp_net_rx_ring *rx_ring)
0058 {
0059     struct nfp_net_r_vector *r_vec = rx_ring->r_vec;
0060     struct xsk_buff_pool *pool = r_vec->xsk_pool;
0061     unsigned int wr_idx, wr_ptr_add = 0;
0062     struct xdp_buff *xdp;
0063
0064     while (nfp_net_rx_space(rx_ring)) {
0065         wr_idx = D_IDX(rx_ring, rx_ring->wr_p);
0066
0067         xdp = xsk_buff_alloc(pool);
0068         if (!xdp)
0069             break;
0070
0071         nfp_net_xsk_rx_bufs_stash(rx_ring, wr_idx, xdp);
0072
0073         /* DMA address is expanded to 48-bit width in freelist for NFP3800,
0074          * so the *_48b macro is used accordingly, it's also OK to fill
0075          * a 40-bit address since the top 8 bits are get set to 0.
0076          */
0077         nfp_desc_set_dma_addr_48b(&rx_ring->rxds[wr_idx].fld,
0078                       rx_ring->xsk_rxbufs[wr_idx].dma_addr);
0079
0080         rx_ring->wr_p++;
0081         wr_ptr_add++;
0082     }
0083
0084     /* Ensure all records are visible before incrementing write counter. */
0085     wmb();
0086     nfp_qcp_wr_ptr_add(rx_ring->qcp_fl, wr_ptr_add);
0087 }
0088
0089 void nfp_net_xsk_rx_drop(struct nfp_net_r_vector *r_vec,
0090              struct nfp_net_xsk_rx_buf *xrxbuf)
0091 {
0092     u64_stats_update_begin(&r_vec->rx_sync);
0093     r_vec->rx_drops++;
0094     u64_stats_update_end(&r_vec->rx_sync);
0095
0096     nfp_net_xsk_rx_free(xrxbuf);
0097 }
0098
0099 static void nfp_net_xsk_pool_unmap(struct device *dev,
0100                    struct xsk_buff_pool *pool)
0101 {
0102     return xsk_pool_dma_unmap(pool, 0);
0103 }
0104
0105 static int nfp_net_xsk_pool_map(struct device *dev, struct xsk_buff_pool *pool)
0106 {
0107     return xsk_pool_dma_map(pool, dev, 0);
0108 }
0109
0110 int nfp_net_xsk_setup_pool(struct net_device *netdev,
0111                struct xsk_buff_pool *pool, u16 queue_id)
0112 {
0113     struct nfp_net *nn = netdev_priv(netdev);
0114
0115     struct xsk_buff_pool *prev_pool;
0116     struct nfp_net_dp *dp;
0117     int err;
0118
0119     /* NFDK doesn't implement xsk yet. */
0120     if (nn->dp.ops->version == NFP_NFD_VER_NFDK)
0121         return -EOPNOTSUPP;
0122
0123     /* Reject on old FWs so we can drop some checks on datapath. */
0124     if (nn->dp.rx_offset != NFP_NET_CFG_RX_OFFSET_DYNAMIC)
0125         return -EOPNOTSUPP;
0126     if (!nn->dp.chained_metadata_format)
0127         return -EOPNOTSUPP;
0128
0129     /* Install */
0130     if (pool) {
0131         err = nfp_net_xsk_pool_map(nn->dp.dev, pool);
0132         if (err)
0133             return err;
0134     }
0135
0136     /* Reconfig/swap */
0137     dp = nfp_net_clone_dp(nn);
0138     if (!dp) {
0139         err = -ENOMEM;
0140         goto err_unmap;
0141     }
0142
0143     prev_pool = dp->xsk_pools[queue_id];
0144     dp->xsk_pools[queue_id] = pool;
0145
0146     err = nfp_net_ring_reconfig(nn, dp, NULL);
0147     if (err)
0148         goto err_unmap;
0149
0150     /* Uninstall */
0151     if (prev_pool)
0152         nfp_net_xsk_pool_unmap(nn->dp.dev, prev_pool);
0153
0154     return 0;
0155 err_unmap:
0156     if (pool)
0157         nfp_net_xsk_pool_unmap(nn->dp.dev, pool);
0158
0159     return err;
0160 }
0161
0162 int nfp_net_xsk_wakeup(struct net_device *netdev, u32 queue_id, u32 flags)
0163 {
0164     struct nfp_net *nn = netdev_priv(netdev);
0165
0166     /* queue_id comes from a zero-copy socket, installed with XDP_SETUP_XSK_POOL,
0167      * so it must be within our vector range.  Moreover, our napi structs
0168      * are statically allocated, so we can always kick them without worrying
0169      * if reconfig is in progress or interface down.
0170      */
0171     napi_schedule(&nn->r_vecs[queue_id].napi);
0172
0173     return 0;
0174 }