Back to home page

OSCL-LXR
 
 

     

0001 // SPDX-License-Identifier: GPL-2.0
0002 /* Copyright(c) 2018 Intel Corporation. */
0003 
0004 #include <linux/bpf_trace.h>
0005 #include <linux/stringify.h>
0006 #include <net/xdp_sock_drv.h>
0007 #include <net/xdp.h>
0008 
0009 #include "i40e.h"
0010 #include "i40e_txrx_common.h"
0011 #include "i40e_xsk.h"
0012 
0013 int i40e_alloc_rx_bi_zc(struct i40e_ring *rx_ring)
0014 {
0015     unsigned long sz = sizeof(*rx_ring->rx_bi_zc) * rx_ring->count;
0016 
0017     rx_ring->rx_bi_zc = kzalloc(sz, GFP_KERNEL);
0018     return rx_ring->rx_bi_zc ? 0 : -ENOMEM;
0019 }
0020 
0021 void i40e_clear_rx_bi_zc(struct i40e_ring *rx_ring)
0022 {
0023     memset(rx_ring->rx_bi_zc, 0,
0024            sizeof(*rx_ring->rx_bi_zc) * rx_ring->count);
0025 }
0026 
0027 static struct xdp_buff **i40e_rx_bi(struct i40e_ring *rx_ring, u32 idx)
0028 {
0029     return &rx_ring->rx_bi_zc[idx];
0030 }
0031 
0032 /**
0033  * i40e_xsk_pool_enable - Enable/associate an AF_XDP buffer pool to a
0034  * certain ring/qid
0035  * @vsi: Current VSI
0036  * @pool: buffer pool
0037  * @qid: Rx ring to associate buffer pool with
0038  *
0039  * Returns 0 on success, <0 on failure
0040  **/
0041 static int i40e_xsk_pool_enable(struct i40e_vsi *vsi,
0042                 struct xsk_buff_pool *pool,
0043                 u16 qid)
0044 {
0045     struct net_device *netdev = vsi->netdev;
0046     bool if_running;
0047     int err;
0048 
0049     if (vsi->type != I40E_VSI_MAIN)
0050         return -EINVAL;
0051 
0052     if (qid >= vsi->num_queue_pairs)
0053         return -EINVAL;
0054 
0055     if (qid >= netdev->real_num_rx_queues ||
0056         qid >= netdev->real_num_tx_queues)
0057         return -EINVAL;
0058 
0059     err = xsk_pool_dma_map(pool, &vsi->back->pdev->dev, I40E_RX_DMA_ATTR);
0060     if (err)
0061         return err;
0062 
0063     set_bit(qid, vsi->af_xdp_zc_qps);
0064 
0065     if_running = netif_running(vsi->netdev) && i40e_enabled_xdp_vsi(vsi);
0066 
0067     if (if_running) {
0068         err = i40e_queue_pair_disable(vsi, qid);
0069         if (err)
0070             return err;
0071 
0072         err = i40e_queue_pair_enable(vsi, qid);
0073         if (err)
0074             return err;
0075 
0076         /* Kick start the NAPI context so that receiving will start */
0077         err = i40e_xsk_wakeup(vsi->netdev, qid, XDP_WAKEUP_RX);
0078         if (err)
0079             return err;
0080     }
0081 
0082     return 0;
0083 }
0084 
0085 /**
0086  * i40e_xsk_pool_disable - Disassociate an AF_XDP buffer pool from a
0087  * certain ring/qid
0088  * @vsi: Current VSI
0089  * @qid: Rx ring to associate buffer pool with
0090  *
0091  * Returns 0 on success, <0 on failure
0092  **/
0093 static int i40e_xsk_pool_disable(struct i40e_vsi *vsi, u16 qid)
0094 {
0095     struct net_device *netdev = vsi->netdev;
0096     struct xsk_buff_pool *pool;
0097     bool if_running;
0098     int err;
0099 
0100     pool = xsk_get_pool_from_qid(netdev, qid);
0101     if (!pool)
0102         return -EINVAL;
0103 
0104     if_running = netif_running(vsi->netdev) && i40e_enabled_xdp_vsi(vsi);
0105 
0106     if (if_running) {
0107         err = i40e_queue_pair_disable(vsi, qid);
0108         if (err)
0109             return err;
0110     }
0111 
0112     clear_bit(qid, vsi->af_xdp_zc_qps);
0113     xsk_pool_dma_unmap(pool, I40E_RX_DMA_ATTR);
0114 
0115     if (if_running) {
0116         err = i40e_queue_pair_enable(vsi, qid);
0117         if (err)
0118             return err;
0119     }
0120 
0121     return 0;
0122 }
0123 
0124 /**
0125  * i40e_xsk_pool_setup - Enable/disassociate an AF_XDP buffer pool to/from
0126  * a ring/qid
0127  * @vsi: Current VSI
0128  * @pool: Buffer pool to enable/associate to a ring, or NULL to disable
0129  * @qid: Rx ring to (dis)associate buffer pool (from)to
0130  *
0131  * This function enables or disables a buffer pool to a certain ring.
0132  *
0133  * Returns 0 on success, <0 on failure
0134  **/
0135 int i40e_xsk_pool_setup(struct i40e_vsi *vsi, struct xsk_buff_pool *pool,
0136             u16 qid)
0137 {
0138     return pool ? i40e_xsk_pool_enable(vsi, pool, qid) :
0139         i40e_xsk_pool_disable(vsi, qid);
0140 }
0141 
0142 /**
0143  * i40e_run_xdp_zc - Executes an XDP program on an xdp_buff
0144  * @rx_ring: Rx ring
0145  * @xdp: xdp_buff used as input to the XDP program
0146  * @xdp_prog: XDP program to run
0147  *
0148  * Returns any of I40E_XDP_{PASS, CONSUMED, TX, REDIR}
0149  **/
0150 static int i40e_run_xdp_zc(struct i40e_ring *rx_ring, struct xdp_buff *xdp,
0151                struct bpf_prog *xdp_prog)
0152 {
0153     int err, result = I40E_XDP_PASS;
0154     struct i40e_ring *xdp_ring;
0155     u32 act;
0156 
0157     act = bpf_prog_run_xdp(xdp_prog, xdp);
0158 
0159     if (likely(act == XDP_REDIRECT)) {
0160         err = xdp_do_redirect(rx_ring->netdev, xdp, xdp_prog);
0161         if (!err)
0162             return I40E_XDP_REDIR;
0163         if (xsk_uses_need_wakeup(rx_ring->xsk_pool) && err == -ENOBUFS)
0164             result = I40E_XDP_EXIT;
0165         else
0166             result = I40E_XDP_CONSUMED;
0167         goto out_failure;
0168     }
0169 
0170     switch (act) {
0171     case XDP_PASS:
0172         break;
0173     case XDP_TX:
0174         xdp_ring = rx_ring->vsi->xdp_rings[rx_ring->queue_index];
0175         result = i40e_xmit_xdp_tx_ring(xdp, xdp_ring);
0176         if (result == I40E_XDP_CONSUMED)
0177             goto out_failure;
0178         break;
0179     case XDP_DROP:
0180         result = I40E_XDP_CONSUMED;
0181         break;
0182     default:
0183         bpf_warn_invalid_xdp_action(rx_ring->netdev, xdp_prog, act);
0184         fallthrough;
0185     case XDP_ABORTED:
0186         result = I40E_XDP_CONSUMED;
0187 out_failure:
0188         trace_xdp_exception(rx_ring->netdev, xdp_prog, act);
0189     }
0190     return result;
0191 }
0192 
0193 bool i40e_alloc_rx_buffers_zc(struct i40e_ring *rx_ring, u16 count)
0194 {
0195     u16 ntu = rx_ring->next_to_use;
0196     union i40e_rx_desc *rx_desc;
0197     struct xdp_buff **xdp;
0198     u32 nb_buffs, i;
0199     dma_addr_t dma;
0200 
0201     rx_desc = I40E_RX_DESC(rx_ring, ntu);
0202     xdp = i40e_rx_bi(rx_ring, ntu);
0203 
0204     nb_buffs = min_t(u16, count, rx_ring->count - ntu);
0205     nb_buffs = xsk_buff_alloc_batch(rx_ring->xsk_pool, xdp, nb_buffs);
0206     if (!nb_buffs)
0207         return false;
0208 
0209     i = nb_buffs;
0210     while (i--) {
0211         dma = xsk_buff_xdp_get_dma(*xdp);
0212         rx_desc->read.pkt_addr = cpu_to_le64(dma);
0213         rx_desc->read.hdr_addr = 0;
0214 
0215         rx_desc++;
0216         xdp++;
0217     }
0218 
0219     ntu += nb_buffs;
0220     if (ntu == rx_ring->count) {
0221         rx_desc = I40E_RX_DESC(rx_ring, 0);
0222         ntu = 0;
0223     }
0224 
0225     /* clear the status bits for the next_to_use descriptor */
0226     rx_desc->wb.qword1.status_error_len = 0;
0227     i40e_release_rx_desc(rx_ring, ntu);
0228 
0229     return count == nb_buffs;
0230 }
0231 
0232 /**
0233  * i40e_construct_skb_zc - Create skbuff from zero-copy Rx buffer
0234  * @rx_ring: Rx ring
0235  * @xdp: xdp_buff
0236  *
0237  * This functions allocates a new skb from a zero-copy Rx buffer.
0238  *
0239  * Returns the skb, or NULL on failure.
0240  **/
0241 static struct sk_buff *i40e_construct_skb_zc(struct i40e_ring *rx_ring,
0242                          struct xdp_buff *xdp)
0243 {
0244     unsigned int totalsize = xdp->data_end - xdp->data_meta;
0245     unsigned int metasize = xdp->data - xdp->data_meta;
0246     struct sk_buff *skb;
0247 
0248     net_prefetch(xdp->data_meta);
0249 
0250     /* allocate a skb to store the frags */
0251     skb = __napi_alloc_skb(&rx_ring->q_vector->napi, totalsize,
0252                    GFP_ATOMIC | __GFP_NOWARN);
0253     if (unlikely(!skb))
0254         goto out;
0255 
0256     memcpy(__skb_put(skb, totalsize), xdp->data_meta,
0257            ALIGN(totalsize, sizeof(long)));
0258 
0259     if (metasize) {
0260         skb_metadata_set(skb, metasize);
0261         __skb_pull(skb, metasize);
0262     }
0263 
0264 out:
0265     xsk_buff_free(xdp);
0266     return skb;
0267 }
0268 
0269 static void i40e_handle_xdp_result_zc(struct i40e_ring *rx_ring,
0270                       struct xdp_buff *xdp_buff,
0271                       union i40e_rx_desc *rx_desc,
0272                       unsigned int *rx_packets,
0273                       unsigned int *rx_bytes,
0274                       unsigned int size,
0275                       unsigned int xdp_res,
0276                       bool *failure)
0277 {
0278     struct sk_buff *skb;
0279 
0280     *rx_packets = 1;
0281     *rx_bytes = size;
0282 
0283     if (likely(xdp_res == I40E_XDP_REDIR) || xdp_res == I40E_XDP_TX)
0284         return;
0285 
0286     if (xdp_res == I40E_XDP_EXIT) {
0287         *failure = true;
0288         return;
0289     }
0290 
0291     if (xdp_res == I40E_XDP_CONSUMED) {
0292         xsk_buff_free(xdp_buff);
0293         return;
0294     }
0295     if (xdp_res == I40E_XDP_PASS) {
0296         /* NB! We are not checking for errors using
0297          * i40e_test_staterr with
0298          * BIT(I40E_RXD_QW1_ERROR_SHIFT). This is due to that
0299          * SBP is *not* set in PRT_SBPVSI (default not set).
0300          */
0301         skb = i40e_construct_skb_zc(rx_ring, xdp_buff);
0302         if (!skb) {
0303             rx_ring->rx_stats.alloc_buff_failed++;
0304             *rx_packets = 0;
0305             *rx_bytes = 0;
0306             return;
0307         }
0308 
0309         if (eth_skb_pad(skb)) {
0310             *rx_packets = 0;
0311             *rx_bytes = 0;
0312             return;
0313         }
0314 
0315         *rx_bytes = skb->len;
0316         i40e_process_skb_fields(rx_ring, rx_desc, skb);
0317         napi_gro_receive(&rx_ring->q_vector->napi, skb);
0318         return;
0319     }
0320 
0321     /* Should never get here, as all valid cases have been handled already.
0322      */
0323     WARN_ON_ONCE(1);
0324 }
0325 
0326 /**
0327  * i40e_clean_rx_irq_zc - Consumes Rx packets from the hardware ring
0328  * @rx_ring: Rx ring
0329  * @budget: NAPI budget
0330  *
0331  * Returns amount of work completed
0332  **/
0333 int i40e_clean_rx_irq_zc(struct i40e_ring *rx_ring, int budget)
0334 {
0335     unsigned int total_rx_bytes = 0, total_rx_packets = 0;
0336     u16 next_to_clean = rx_ring->next_to_clean;
0337     u16 count_mask = rx_ring->count - 1;
0338     unsigned int xdp_res, xdp_xmit = 0;
0339     struct bpf_prog *xdp_prog;
0340     bool failure = false;
0341     u16 cleaned_count;
0342 
0343     /* NB! xdp_prog will always be !NULL, due to the fact that
0344      * this path is enabled by setting an XDP program.
0345      */
0346     xdp_prog = READ_ONCE(rx_ring->xdp_prog);
0347 
0348     while (likely(total_rx_packets < (unsigned int)budget)) {
0349         union i40e_rx_desc *rx_desc;
0350         unsigned int rx_packets;
0351         unsigned int rx_bytes;
0352         struct xdp_buff *bi;
0353         unsigned int size;
0354         u64 qword;
0355 
0356         rx_desc = I40E_RX_DESC(rx_ring, next_to_clean);
0357         qword = le64_to_cpu(rx_desc->wb.qword1.status_error_len);
0358 
0359         /* This memory barrier is needed to keep us from reading
0360          * any other fields out of the rx_desc until we have
0361          * verified the descriptor has been written back.
0362          */
0363         dma_rmb();
0364 
0365         if (i40e_rx_is_programming_status(qword)) {
0366             i40e_clean_programming_status(rx_ring,
0367                               rx_desc->raw.qword[0],
0368                               qword);
0369             bi = *i40e_rx_bi(rx_ring, next_to_clean);
0370             xsk_buff_free(bi);
0371             next_to_clean = (next_to_clean + 1) & count_mask;
0372             continue;
0373         }
0374 
0375         size = (qword & I40E_RXD_QW1_LENGTH_PBUF_MASK) >>
0376                I40E_RXD_QW1_LENGTH_PBUF_SHIFT;
0377         if (!size)
0378             break;
0379 
0380         bi = *i40e_rx_bi(rx_ring, next_to_clean);
0381         xsk_buff_set_size(bi, size);
0382         xsk_buff_dma_sync_for_cpu(bi, rx_ring->xsk_pool);
0383 
0384         xdp_res = i40e_run_xdp_zc(rx_ring, bi, xdp_prog);
0385         i40e_handle_xdp_result_zc(rx_ring, bi, rx_desc, &rx_packets,
0386                       &rx_bytes, size, xdp_res, &failure);
0387         if (failure)
0388             break;
0389         total_rx_packets += rx_packets;
0390         total_rx_bytes += rx_bytes;
0391         xdp_xmit |= xdp_res & (I40E_XDP_TX | I40E_XDP_REDIR);
0392         next_to_clean = (next_to_clean + 1) & count_mask;
0393     }
0394 
0395     rx_ring->next_to_clean = next_to_clean;
0396     cleaned_count = (next_to_clean - rx_ring->next_to_use - 1) & count_mask;
0397 
0398     if (cleaned_count >= I40E_RX_BUFFER_WRITE)
0399         failure |= !i40e_alloc_rx_buffers_zc(rx_ring, cleaned_count);
0400 
0401     i40e_finalize_xdp_rx(rx_ring, xdp_xmit);
0402     i40e_update_rx_stats(rx_ring, total_rx_bytes, total_rx_packets);
0403 
0404     if (xsk_uses_need_wakeup(rx_ring->xsk_pool)) {
0405         if (failure || next_to_clean == rx_ring->next_to_use)
0406             xsk_set_rx_need_wakeup(rx_ring->xsk_pool);
0407         else
0408             xsk_clear_rx_need_wakeup(rx_ring->xsk_pool);
0409 
0410         return (int)total_rx_packets;
0411     }
0412     return failure ? budget : (int)total_rx_packets;
0413 }
0414 
0415 static void i40e_xmit_pkt(struct i40e_ring *xdp_ring, struct xdp_desc *desc,
0416               unsigned int *total_bytes)
0417 {
0418     struct i40e_tx_desc *tx_desc;
0419     dma_addr_t dma;
0420 
0421     dma = xsk_buff_raw_get_dma(xdp_ring->xsk_pool, desc->addr);
0422     xsk_buff_raw_dma_sync_for_device(xdp_ring->xsk_pool, dma, desc->len);
0423 
0424     tx_desc = I40E_TX_DESC(xdp_ring, xdp_ring->next_to_use++);
0425     tx_desc->buffer_addr = cpu_to_le64(dma);
0426     tx_desc->cmd_type_offset_bsz = build_ctob(I40E_TX_DESC_CMD_ICRC | I40E_TX_DESC_CMD_EOP,
0427                           0, desc->len, 0);
0428 
0429     *total_bytes += desc->len;
0430 }
0431 
0432 static void i40e_xmit_pkt_batch(struct i40e_ring *xdp_ring, struct xdp_desc *desc,
0433                 unsigned int *total_bytes)
0434 {
0435     u16 ntu = xdp_ring->next_to_use;
0436     struct i40e_tx_desc *tx_desc;
0437     dma_addr_t dma;
0438     u32 i;
0439 
0440     loop_unrolled_for(i = 0; i < PKTS_PER_BATCH; i++) {
0441         dma = xsk_buff_raw_get_dma(xdp_ring->xsk_pool, desc[i].addr);
0442         xsk_buff_raw_dma_sync_for_device(xdp_ring->xsk_pool, dma, desc[i].len);
0443 
0444         tx_desc = I40E_TX_DESC(xdp_ring, ntu++);
0445         tx_desc->buffer_addr = cpu_to_le64(dma);
0446         tx_desc->cmd_type_offset_bsz = build_ctob(I40E_TX_DESC_CMD_ICRC |
0447                               I40E_TX_DESC_CMD_EOP,
0448                               0, desc[i].len, 0);
0449 
0450         *total_bytes += desc[i].len;
0451     }
0452 
0453     xdp_ring->next_to_use = ntu;
0454 }
0455 
0456 static void i40e_fill_tx_hw_ring(struct i40e_ring *xdp_ring, struct xdp_desc *descs, u32 nb_pkts,
0457                  unsigned int *total_bytes)
0458 {
0459     u32 batched, leftover, i;
0460 
0461     batched = nb_pkts & ~(PKTS_PER_BATCH - 1);
0462     leftover = nb_pkts & (PKTS_PER_BATCH - 1);
0463     for (i = 0; i < batched; i += PKTS_PER_BATCH)
0464         i40e_xmit_pkt_batch(xdp_ring, &descs[i], total_bytes);
0465     for (i = batched; i < batched + leftover; i++)
0466         i40e_xmit_pkt(xdp_ring, &descs[i], total_bytes);
0467 }
0468 
0469 static void i40e_set_rs_bit(struct i40e_ring *xdp_ring)
0470 {
0471     u16 ntu = xdp_ring->next_to_use ? xdp_ring->next_to_use - 1 : xdp_ring->count - 1;
0472     struct i40e_tx_desc *tx_desc;
0473 
0474     tx_desc = I40E_TX_DESC(xdp_ring, ntu);
0475     tx_desc->cmd_type_offset_bsz |= cpu_to_le64(I40E_TX_DESC_CMD_RS << I40E_TXD_QW1_CMD_SHIFT);
0476 }
0477 
0478 /**
0479  * i40e_xmit_zc - Performs zero-copy Tx AF_XDP
0480  * @xdp_ring: XDP Tx ring
0481  * @budget: NAPI budget
0482  *
0483  * Returns true if the work is finished.
0484  **/
0485 static bool i40e_xmit_zc(struct i40e_ring *xdp_ring, unsigned int budget)
0486 {
0487     struct xdp_desc *descs = xdp_ring->xsk_pool->tx_descs;
0488     u32 nb_pkts, nb_processed = 0;
0489     unsigned int total_bytes = 0;
0490 
0491     nb_pkts = xsk_tx_peek_release_desc_batch(xdp_ring->xsk_pool, budget);
0492     if (!nb_pkts)
0493         return true;
0494 
0495     if (xdp_ring->next_to_use + nb_pkts >= xdp_ring->count) {
0496         nb_processed = xdp_ring->count - xdp_ring->next_to_use;
0497         i40e_fill_tx_hw_ring(xdp_ring, descs, nb_processed, &total_bytes);
0498         xdp_ring->next_to_use = 0;
0499     }
0500 
0501     i40e_fill_tx_hw_ring(xdp_ring, &descs[nb_processed], nb_pkts - nb_processed,
0502                  &total_bytes);
0503 
0504     /* Request an interrupt for the last frame and bump tail ptr. */
0505     i40e_set_rs_bit(xdp_ring);
0506     i40e_xdp_ring_update_tail(xdp_ring);
0507 
0508     i40e_update_tx_stats(xdp_ring, nb_pkts, total_bytes);
0509 
0510     return nb_pkts < budget;
0511 }
0512 
0513 /**
0514  * i40e_clean_xdp_tx_buffer - Frees and unmaps an XDP Tx entry
0515  * @tx_ring: XDP Tx ring
0516  * @tx_bi: Tx buffer info to clean
0517  **/
0518 static void i40e_clean_xdp_tx_buffer(struct i40e_ring *tx_ring,
0519                      struct i40e_tx_buffer *tx_bi)
0520 {
0521     xdp_return_frame(tx_bi->xdpf);
0522     tx_ring->xdp_tx_active--;
0523     dma_unmap_single(tx_ring->dev,
0524              dma_unmap_addr(tx_bi, dma),
0525              dma_unmap_len(tx_bi, len), DMA_TO_DEVICE);
0526     dma_unmap_len_set(tx_bi, len, 0);
0527 }
0528 
0529 /**
0530  * i40e_clean_xdp_tx_irq - Completes AF_XDP entries, and cleans XDP entries
0531  * @vsi: Current VSI
0532  * @tx_ring: XDP Tx ring
0533  *
0534  * Returns true if cleanup/tranmission is done.
0535  **/
0536 bool i40e_clean_xdp_tx_irq(struct i40e_vsi *vsi, struct i40e_ring *tx_ring)
0537 {
0538     struct xsk_buff_pool *bp = tx_ring->xsk_pool;
0539     u32 i, completed_frames, xsk_frames = 0;
0540     u32 head_idx = i40e_get_head(tx_ring);
0541     struct i40e_tx_buffer *tx_bi;
0542     unsigned int ntc;
0543 
0544     if (head_idx < tx_ring->next_to_clean)
0545         head_idx += tx_ring->count;
0546     completed_frames = head_idx - tx_ring->next_to_clean;
0547 
0548     if (completed_frames == 0)
0549         goto out_xmit;
0550 
0551     if (likely(!tx_ring->xdp_tx_active)) {
0552         xsk_frames = completed_frames;
0553         goto skip;
0554     }
0555 
0556     ntc = tx_ring->next_to_clean;
0557 
0558     for (i = 0; i < completed_frames; i++) {
0559         tx_bi = &tx_ring->tx_bi[ntc];
0560 
0561         if (tx_bi->xdpf) {
0562             i40e_clean_xdp_tx_buffer(tx_ring, tx_bi);
0563             tx_bi->xdpf = NULL;
0564         } else {
0565             xsk_frames++;
0566         }
0567 
0568         if (++ntc >= tx_ring->count)
0569             ntc = 0;
0570     }
0571 
0572 skip:
0573     tx_ring->next_to_clean += completed_frames;
0574     if (unlikely(tx_ring->next_to_clean >= tx_ring->count))
0575         tx_ring->next_to_clean -= tx_ring->count;
0576 
0577     if (xsk_frames)
0578         xsk_tx_completed(bp, xsk_frames);
0579 
0580     i40e_arm_wb(tx_ring, vsi, completed_frames);
0581 
0582 out_xmit:
0583     if (xsk_uses_need_wakeup(tx_ring->xsk_pool))
0584         xsk_set_tx_need_wakeup(tx_ring->xsk_pool);
0585 
0586     return i40e_xmit_zc(tx_ring, I40E_DESC_UNUSED(tx_ring));
0587 }
0588 
0589 /**
0590  * i40e_xsk_wakeup - Implements the ndo_xsk_wakeup
0591  * @dev: the netdevice
0592  * @queue_id: queue id to wake up
0593  * @flags: ignored in our case since we have Rx and Tx in the same NAPI.
0594  *
0595  * Returns <0 for errors, 0 otherwise.
0596  **/
0597 int i40e_xsk_wakeup(struct net_device *dev, u32 queue_id, u32 flags)
0598 {
0599     struct i40e_netdev_priv *np = netdev_priv(dev);
0600     struct i40e_vsi *vsi = np->vsi;
0601     struct i40e_pf *pf = vsi->back;
0602     struct i40e_ring *ring;
0603 
0604     if (test_bit(__I40E_CONFIG_BUSY, pf->state))
0605         return -EAGAIN;
0606 
0607     if (test_bit(__I40E_VSI_DOWN, vsi->state))
0608         return -ENETDOWN;
0609 
0610     if (!i40e_enabled_xdp_vsi(vsi))
0611         return -EINVAL;
0612 
0613     if (queue_id >= vsi->num_queue_pairs)
0614         return -EINVAL;
0615 
0616     if (!vsi->xdp_rings[queue_id]->xsk_pool)
0617         return -EINVAL;
0618 
0619     ring = vsi->xdp_rings[queue_id];
0620 
0621     /* The idea here is that if NAPI is running, mark a miss, so
0622      * it will run again. If not, trigger an interrupt and
0623      * schedule the NAPI from interrupt context. If NAPI would be
0624      * scheduled here, the interrupt affinity would not be
0625      * honored.
0626      */
0627     if (!napi_if_scheduled_mark_missed(&ring->q_vector->napi))
0628         i40e_force_wb(vsi, ring->q_vector);
0629 
0630     return 0;
0631 }
0632 
0633 void i40e_xsk_clean_rx_ring(struct i40e_ring *rx_ring)
0634 {
0635     u16 count_mask = rx_ring->count - 1;
0636     u16 ntc = rx_ring->next_to_clean;
0637     u16 ntu = rx_ring->next_to_use;
0638 
0639     for ( ; ntc != ntu; ntc = (ntc + 1)  & count_mask) {
0640         struct xdp_buff *rx_bi = *i40e_rx_bi(rx_ring, ntc);
0641 
0642         xsk_buff_free(rx_bi);
0643     }
0644 }
0645 
0646 /**
0647  * i40e_xsk_clean_tx_ring - Clean the XDP Tx ring on shutdown
0648  * @tx_ring: XDP Tx ring
0649  **/
0650 void i40e_xsk_clean_tx_ring(struct i40e_ring *tx_ring)
0651 {
0652     u16 ntc = tx_ring->next_to_clean, ntu = tx_ring->next_to_use;
0653     struct xsk_buff_pool *bp = tx_ring->xsk_pool;
0654     struct i40e_tx_buffer *tx_bi;
0655     u32 xsk_frames = 0;
0656 
0657     while (ntc != ntu) {
0658         tx_bi = &tx_ring->tx_bi[ntc];
0659 
0660         if (tx_bi->xdpf)
0661             i40e_clean_xdp_tx_buffer(tx_ring, tx_bi);
0662         else
0663             xsk_frames++;
0664 
0665         tx_bi->xdpf = NULL;
0666 
0667         ntc++;
0668         if (ntc >= tx_ring->count)
0669             ntc = 0;
0670     }
0671 
0672     if (xsk_frames)
0673         xsk_tx_completed(bp, xsk_frames);
0674 }
0675 
0676 /**
0677  * i40e_xsk_any_rx_ring_enabled - Checks if Rx rings have an AF_XDP
0678  * buffer pool attached
0679  * @vsi: vsi
0680  *
0681  * Returns true if any of the Rx rings has an AF_XDP buffer pool attached
0682  **/
0683 bool i40e_xsk_any_rx_ring_enabled(struct i40e_vsi *vsi)
0684 {
0685     struct net_device *netdev = vsi->netdev;
0686     int i;
0687 
0688     for (i = 0; i < vsi->num_queue_pairs; i++) {
0689         if (xsk_get_pool_from_qid(netdev, i))
0690             return true;
0691     }
0692 
0693     return false;
0694 }
[ Source navigation ] [ Diff markup ] [ Identifier search ] [ general search ]

This page was automatically generated by the 2.1.0 LXR engine.

The LXR team
Valid CSS 2.1! Valid HTML 4.01!