nfp/nfd3/rings.c

0001 // SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause)
0002 /* Copyright (C) 2015-2019 Netronome Systems, Inc. */
0003
0004 #include <linux/seq_file.h>
0005
0006 #include "../nfp_net.h"
0007 #include "../nfp_net_dp.h"
0008 #include "../nfp_net_xsk.h"
0009 #include "nfd3.h"
0010
0011 static void nfp_nfd3_xsk_tx_bufs_free(struct nfp_net_tx_ring *tx_ring)
0012 {
0013     struct nfp_nfd3_tx_buf *txbuf;
0014     unsigned int idx;
0015
0016     while (tx_ring->rd_p != tx_ring->wr_p) {
0017         idx = D_IDX(tx_ring, tx_ring->rd_p);
0018         txbuf = &tx_ring->txbufs[idx];
0019
0020         txbuf->real_len = 0;
0021
0022         tx_ring->qcp_rd_p++;
0023         tx_ring->rd_p++;
0024
0025         if (tx_ring->r_vec->xsk_pool) {
0026             if (txbuf->is_xsk_tx)
0027                 nfp_nfd3_xsk_tx_free(txbuf);
0028
0029             xsk_tx_completed(tx_ring->r_vec->xsk_pool, 1);
0030         }
0031     }
0032 }
0033
0034 /**
0035  * nfp_nfd3_tx_ring_reset() - Free any untransmitted buffers and reset pointers
0036  * @dp:     NFP Net data path struct
0037  * @tx_ring:    TX ring structure
0038  *
0039  * Assumes that the device is stopped, must be idempotent.
0040  */
0041 static void
0042 nfp_nfd3_tx_ring_reset(struct nfp_net_dp *dp, struct nfp_net_tx_ring *tx_ring)
0043 {
0044     struct netdev_queue *nd_q;
0045     const skb_frag_t *frag;
0046
0047     while (!tx_ring->is_xdp && tx_ring->rd_p != tx_ring->wr_p) {
0048         struct nfp_nfd3_tx_buf *tx_buf;
0049         struct sk_buff *skb;
0050         int idx, nr_frags;
0051
0052         idx = D_IDX(tx_ring, tx_ring->rd_p);
0053         tx_buf = &tx_ring->txbufs[idx];
0054
0055         skb = tx_ring->txbufs[idx].skb;
0056         nr_frags = skb_shinfo(skb)->nr_frags;
0057
0058         if (tx_buf->fidx == -1) {
0059             /* unmap head */
0060             dma_unmap_single(dp->dev, tx_buf->dma_addr,
0061                      skb_headlen(skb), DMA_TO_DEVICE);
0062         } else {
0063             /* unmap fragment */
0064             frag = &skb_shinfo(skb)->frags[tx_buf->fidx];
0065             dma_unmap_page(dp->dev, tx_buf->dma_addr,
0066                        skb_frag_size(frag), DMA_TO_DEVICE);
0067         }
0068
0069         /* check for last gather fragment */
0070         if (tx_buf->fidx == nr_frags - 1)
0071             dev_kfree_skb_any(skb);
0072
0073         tx_buf->dma_addr = 0;
0074         tx_buf->skb = NULL;
0075         tx_buf->fidx = -2;
0076
0077         tx_ring->qcp_rd_p++;
0078         tx_ring->rd_p++;
0079     }
0080
0081     if (tx_ring->is_xdp)
0082         nfp_nfd3_xsk_tx_bufs_free(tx_ring);
0083
0084     memset(tx_ring->txds, 0, tx_ring->size);
0085     tx_ring->wr_p = 0;
0086     tx_ring->rd_p = 0;
0087     tx_ring->qcp_rd_p = 0;
0088     tx_ring->wr_ptr_add = 0;
0089
0090     if (tx_ring->is_xdp || !dp->netdev)
0091         return;
0092
0093     nd_q = netdev_get_tx_queue(dp->netdev, tx_ring->idx);
0094     netdev_tx_reset_queue(nd_q);
0095 }
0096
0097 /**
0098  * nfp_nfd3_tx_ring_free() - Free resources allocated to a TX ring
0099  * @tx_ring:   TX ring to free
0100  */
0101 static void nfp_nfd3_tx_ring_free(struct nfp_net_tx_ring *tx_ring)
0102 {
0103     struct nfp_net_r_vector *r_vec = tx_ring->r_vec;
0104     struct nfp_net_dp *dp = &r_vec->nfp_net->dp;
0105
0106     kvfree(tx_ring->txbufs);
0107
0108     if (tx_ring->txds)
0109         dma_free_coherent(dp->dev, tx_ring->size,
0110                   tx_ring->txds, tx_ring->dma);
0111
0112     tx_ring->cnt = 0;
0113     tx_ring->txbufs = NULL;
0114     tx_ring->txds = NULL;
0115     tx_ring->dma = 0;
0116     tx_ring->size = 0;
0117 }
0118
0119 /**
0120  * nfp_nfd3_tx_ring_alloc() - Allocate resource for a TX ring
0121  * @dp:        NFP Net data path struct
0122  * @tx_ring:   TX Ring structure to allocate
0123  *
0124  * Return: 0 on success, negative errno otherwise.
0125  */
0126 static int
0127 nfp_nfd3_tx_ring_alloc(struct nfp_net_dp *dp, struct nfp_net_tx_ring *tx_ring)
0128 {
0129     struct nfp_net_r_vector *r_vec = tx_ring->r_vec;
0130
0131     tx_ring->cnt = dp->txd_cnt;
0132
0133     tx_ring->size = array_size(tx_ring->cnt, sizeof(*tx_ring->txds));
0134     tx_ring->txds = dma_alloc_coherent(dp->dev, tx_ring->size,
0135                        &tx_ring->dma,
0136                        GFP_KERNEL | __GFP_NOWARN);
0137     if (!tx_ring->txds) {
0138         netdev_warn(dp->netdev, "failed to allocate TX descriptor ring memory, requested descriptor count: %d, consider lowering descriptor count\n",
0139                 tx_ring->cnt);
0140         goto err_alloc;
0141     }
0142
0143     tx_ring->txbufs = kvcalloc(tx_ring->cnt, sizeof(*tx_ring->txbufs),
0144                    GFP_KERNEL);
0145     if (!tx_ring->txbufs)
0146         goto err_alloc;
0147
0148     if (!tx_ring->is_xdp && dp->netdev)
0149         netif_set_xps_queue(dp->netdev, &r_vec->affinity_mask,
0150                     tx_ring->idx);
0151
0152     return 0;
0153
0154 err_alloc:
0155     nfp_nfd3_tx_ring_free(tx_ring);
0156     return -ENOMEM;
0157 }
0158
0159 static void
0160 nfp_nfd3_tx_ring_bufs_free(struct nfp_net_dp *dp,
0161                struct nfp_net_tx_ring *tx_ring)
0162 {
0163     unsigned int i;
0164
0165     if (!tx_ring->is_xdp)
0166         return;
0167
0168     for (i = 0; i < tx_ring->cnt; i++) {
0169         if (!tx_ring->txbufs[i].frag)
0170             return;
0171
0172         nfp_net_dma_unmap_rx(dp, tx_ring->txbufs[i].dma_addr);
0173         __free_page(virt_to_page(tx_ring->txbufs[i].frag));
0174     }
0175 }
0176
0177 static int
0178 nfp_nfd3_tx_ring_bufs_alloc(struct nfp_net_dp *dp,
0179                 struct nfp_net_tx_ring *tx_ring)
0180 {
0181     struct nfp_nfd3_tx_buf *txbufs = tx_ring->txbufs;
0182     unsigned int i;
0183
0184     if (!tx_ring->is_xdp)
0185         return 0;
0186
0187     for (i = 0; i < tx_ring->cnt; i++) {
0188         txbufs[i].frag = nfp_net_rx_alloc_one(dp, &txbufs[i].dma_addr);
0189         if (!txbufs[i].frag) {
0190             nfp_nfd3_tx_ring_bufs_free(dp, tx_ring);
0191             return -ENOMEM;
0192         }
0193     }
0194
0195     return 0;
0196 }
0197
0198 static void
0199 nfp_nfd3_print_tx_descs(struct seq_file *file,
0200             struct nfp_net_r_vector *r_vec,
0201             struct nfp_net_tx_ring *tx_ring,
0202             u32 d_rd_p, u32 d_wr_p)
0203 {
0204     struct nfp_nfd3_tx_desc *txd;
0205     u32 txd_cnt = tx_ring->cnt;
0206     int i;
0207
0208     for (i = 0; i < txd_cnt; i++) {
0209         struct xdp_buff *xdp;
0210         struct sk_buff *skb;
0211
0212         txd = &tx_ring->txds[i];
0213         seq_printf(file, "%04d: 0x%08x 0x%08x 0x%08x 0x%08x", i,
0214                txd->vals[0], txd->vals[1],
0215                txd->vals[2], txd->vals[3]);
0216
0217         if (!tx_ring->is_xdp) {
0218             skb = READ_ONCE(tx_ring->txbufs[i].skb);
0219             if (skb)
0220                 seq_printf(file, " skb->head=%p skb->data=%p",
0221                        skb->head, skb->data);
0222         } else {
0223             xdp = READ_ONCE(tx_ring->txbufs[i].xdp);
0224             if (xdp)
0225                 seq_printf(file, " xdp->data=%p", xdp->data);
0226         }
0227
0228         if (tx_ring->txbufs[i].dma_addr)
0229             seq_printf(file, " dma_addr=%pad",
0230                    &tx_ring->txbufs[i].dma_addr);
0231
0232         if (i == tx_ring->rd_p % txd_cnt)
0233             seq_puts(file, " H_RD");
0234         if (i == tx_ring->wr_p % txd_cnt)
0235             seq_puts(file, " H_WR");
0236         if (i == d_rd_p % txd_cnt)
0237             seq_puts(file, " D_RD");
0238         if (i == d_wr_p % txd_cnt)
0239             seq_puts(file, " D_WR");
0240
0241         seq_putc(file, '\n');
0242     }
0243 }
0244
0245 #define NFP_NFD3_CFG_CTRL_SUPPORTED                 \
0246     (NFP_NET_CFG_CTRL_ENABLE | NFP_NET_CFG_CTRL_PROMISC |       \
0247      NFP_NET_CFG_CTRL_L2BC | NFP_NET_CFG_CTRL_L2MC |        \
0248      NFP_NET_CFG_CTRL_RXCSUM | NFP_NET_CFG_CTRL_TXCSUM |        \
0249      NFP_NET_CFG_CTRL_RXVLAN | NFP_NET_CFG_CTRL_TXVLAN |        \
0250      NFP_NET_CFG_CTRL_RXVLAN_V2 | NFP_NET_CFG_CTRL_RXQINQ |     \
0251      NFP_NET_CFG_CTRL_TXVLAN_V2 |                   \
0252      NFP_NET_CFG_CTRL_GATHER | NFP_NET_CFG_CTRL_LSO |       \
0253      NFP_NET_CFG_CTRL_CTAG_FILTER | NFP_NET_CFG_CTRL_CMSG_DATA |    \
0254      NFP_NET_CFG_CTRL_RINGCFG | NFP_NET_CFG_CTRL_RSS |      \
0255      NFP_NET_CFG_CTRL_IRQMOD | NFP_NET_CFG_CTRL_TXRWB |     \
0256      NFP_NET_CFG_CTRL_VEPA |                    \
0257      NFP_NET_CFG_CTRL_VXLAN | NFP_NET_CFG_CTRL_NVGRE |      \
0258      NFP_NET_CFG_CTRL_BPF | NFP_NET_CFG_CTRL_LSO2 |         \
0259      NFP_NET_CFG_CTRL_RSS2 | NFP_NET_CFG_CTRL_CSUM_COMPLETE |   \
0260      NFP_NET_CFG_CTRL_LIVE_ADDR)
0261
0262 const struct nfp_dp_ops nfp_nfd3_ops = {
0263     .version        = NFP_NFD_VER_NFD3,
0264     .tx_min_desc_per_pkt    = 1,
0265     .cap_mask       = NFP_NFD3_CFG_CTRL_SUPPORTED,
0266     .dma_mask       = DMA_BIT_MASK(40),
0267     .poll           = nfp_nfd3_poll,
0268     .xsk_poll       = nfp_nfd3_xsk_poll,
0269     .ctrl_poll      = nfp_nfd3_ctrl_poll,
0270     .xmit           = nfp_nfd3_tx,
0271     .ctrl_tx_one        = nfp_nfd3_ctrl_tx_one,
0272     .rx_ring_fill_freelist  = nfp_nfd3_rx_ring_fill_freelist,
0273     .tx_ring_alloc      = nfp_nfd3_tx_ring_alloc,
0274     .tx_ring_reset      = nfp_nfd3_tx_ring_reset,
0275     .tx_ring_free       = nfp_nfd3_tx_ring_free,
0276     .tx_ring_bufs_alloc = nfp_nfd3_tx_ring_bufs_alloc,
0277     .tx_ring_bufs_free  = nfp_nfd3_tx_ring_bufs_free,
0278     .print_tx_descs     = nfp_nfd3_print_tx_descs
0279 };