fungible/funeth/funeth_txrx.h

0001 /* SPDX-License-Identifier: (GPL-2.0-only OR BSD-3-Clause) */
0002
0003 #ifndef _FUNETH_TXRX_H
0004 #define _FUNETH_TXRX_H
0005
0006 #include <linux/netdevice.h>
0007 #include <linux/u64_stats_sync.h>
0008
0009 /* Tx descriptor size */
0010 #define FUNETH_SQE_SIZE 64U
0011
0012 /* Size of device headers per Tx packet */
0013 #define FUNETH_FUNOS_HDR_SZ (sizeof(struct fun_eth_tx_req))
0014
0015 /* Number of gather list entries per Tx descriptor */
0016 #define FUNETH_GLE_PER_DESC (FUNETH_SQE_SIZE / sizeof(struct fun_dataop_gl))
0017
0018 /* Max gather list size in bytes for an sk_buff. */
0019 #define FUNETH_MAX_GL_SZ ((MAX_SKB_FRAGS + 1) * sizeof(struct fun_dataop_gl))
0020
0021 #if IS_ENABLED(CONFIG_TLS_DEVICE)
0022 # define FUNETH_TLS_SZ sizeof(struct fun_eth_tls)
0023 #else
0024 # define FUNETH_TLS_SZ 0
0025 #endif
0026
0027 /* Max number of Tx descriptors for an sk_buff using a gather list. */
0028 #define FUNETH_MAX_GL_DESC \
0029     DIV_ROUND_UP((FUNETH_FUNOS_HDR_SZ + FUNETH_MAX_GL_SZ + FUNETH_TLS_SZ), \
0030              FUNETH_SQE_SIZE)
0031
0032 /* Max number of Tx descriptors for any packet. */
0033 #define FUNETH_MAX_PKT_DESC FUNETH_MAX_GL_DESC
0034
0035 /* Rx CQ descriptor size. */
0036 #define FUNETH_CQE_SIZE 64U
0037
0038 /* Offset of cqe_info within a CQE. */
0039 #define FUNETH_CQE_INFO_OFFSET (FUNETH_CQE_SIZE - sizeof(struct fun_cqe_info))
0040
0041 /* Construct the IRQ portion of a CQ doorbell. The resulting value arms the
0042  * interrupt with the supplied time delay and packet count moderation settings.
0043  */
0044 #define FUN_IRQ_CQ_DB(usec, pkts) \
0045     (FUN_DB_IRQ_ARM_F | ((usec) << FUN_DB_INTCOAL_USEC_S) | \
0046      ((pkts) << FUN_DB_INTCOAL_ENTRIES_S))
0047
0048 /* As above for SQ doorbells. */
0049 #define FUN_IRQ_SQ_DB(usec, pkts) \
0050     (FUN_DB_IRQ_ARM_F | \
0051      ((usec) << FUN_DB_INTCOAL_USEC_S) | \
0052      ((pkts) << FUN_DB_INTCOAL_ENTRIES_S))
0053
0054 /* Per packet tailroom. Present only for 1-frag packets. */
0055 #define FUN_RX_TAILROOM SKB_DATA_ALIGN(sizeof(struct skb_shared_info))
0056
0057 /* Per packet headroom for XDP. Preferred over XDP_PACKET_HEADROOM to
0058  * accommodate two packets per buffer for 4K pages and 1500B MTUs.
0059  */
0060 #define FUN_XDP_HEADROOM 192
0061
0062 /* Initialization state of a queue. */
0063 enum {
0064     FUN_QSTATE_DESTROYED, /* what queue? */
0065     FUN_QSTATE_INIT_SW,   /* exists in SW, not on the device */
0066     FUN_QSTATE_INIT_FULL, /* exists both in SW and on device */
0067 };
0068
0069 /* Initialization state of an interrupt. */
0070 enum {
0071     FUN_IRQ_INIT,      /* initialized and in the XArray but inactive */
0072     FUN_IRQ_REQUESTED, /* request_irq() done */
0073     FUN_IRQ_ENABLED,   /* processing enabled */
0074     FUN_IRQ_DISABLED,  /* processing disabled */
0075 };
0076
0077 struct bpf_prog;
0078
0079 struct funeth_txq_stats {  /* per Tx queue SW counters */
0080     u64 tx_pkts;       /* # of Tx packets */
0081     u64 tx_bytes;      /* total bytes of Tx packets */
0082     u64 tx_cso;        /* # of packets with checksum offload */
0083     u64 tx_tso;        /* # of non-encapsulated TSO super-packets */
0084     u64 tx_encap_tso;  /* # of encapsulated TSO super-packets */
0085     u64 tx_uso;        /* # of non-encapsulated UDP LSO super-packets */
0086     u64 tx_more;       /* # of DBs elided due to xmit_more */
0087     u64 tx_nstops;     /* # of times the queue has stopped */
0088     u64 tx_nrestarts;  /* # of times the queue has restarted */
0089     u64 tx_map_err;    /* # of packets dropped due to DMA mapping errors */
0090     u64 tx_xdp_full;   /* # of XDP packets that could not be enqueued */
0091     u64 tx_tls_pkts;   /* # of Tx TLS packets offloaded to HW */
0092     u64 tx_tls_bytes;  /* Tx bytes of HW-handled TLS payload */
0093     u64 tx_tls_fallback; /* attempted Tx TLS offloads punted to SW */
0094     u64 tx_tls_drops;  /* attempted Tx TLS offloads dropped */
0095 };
0096
0097 struct funeth_tx_info {      /* per Tx descriptor state */
0098     union {
0099         struct sk_buff *skb;    /* associated packet (sk_buff path) */
0100         struct xdp_frame *xdpf; /* associated XDP frame (XDP path) */
0101     };
0102 };
0103
0104 struct funeth_txq {
0105     /* RO cacheline of frequently accessed data */
0106     u32 mask;               /* queue depth - 1 */
0107     u32 hw_qid;             /* device ID of the queue */
0108     void *desc;             /* base address of descriptor ring */
0109     struct funeth_tx_info *info;
0110     struct device *dma_dev; /* device for DMA mappings */
0111     volatile __be64 *hw_wb; /* HW write-back location */
0112     u32 __iomem *db;        /* SQ doorbell register address */
0113     struct netdev_queue *ndq;
0114     dma_addr_t dma_addr;    /* DMA address of descriptor ring */
0115     /* producer R/W cacheline */
0116     u16 qidx;               /* queue index within net_device */
0117     u16 ethid;
0118     u32 prod_cnt;           /* producer counter */
0119     struct funeth_txq_stats stats;
0120     /* shared R/W cacheline, primarily accessed by consumer */
0121     u32 irq_db_val;         /* value written to IRQ doorbell */
0122     u32 cons_cnt;           /* consumer (cleanup) counter */
0123     struct net_device *netdev;
0124     struct fun_irq *irq;
0125     int numa_node;
0126     u8 init_state;          /* queue initialization state */
0127     struct u64_stats_sync syncp;
0128 };
0129
0130 struct funeth_rxq_stats {  /* per Rx queue SW counters */
0131     u64 rx_pkts;       /* # of received packets, including SW drops */
0132     u64 rx_bytes;      /* total size of received packets */
0133     u64 rx_cso;        /* # of packets with checksum offload */
0134     u64 rx_bufs;       /* total # of Rx buffers provided to device */
0135     u64 gro_pkts;      /* # of GRO superpackets */
0136     u64 gro_merged;    /* # of pkts merged into existing GRO superpackets */
0137     u64 rx_page_alloc; /* # of page allocations for Rx buffers */
0138     u64 rx_budget;     /* NAPI iterations that exhausted their budget */
0139     u64 rx_mem_drops;  /* # of packets dropped due to memory shortage */
0140     u64 rx_map_err;    /* # of page DMA mapping errors */
0141     u64 xdp_drops;     /* XDP_DROPped packets */
0142     u64 xdp_tx;        /* successful XDP transmits */
0143     u64 xdp_redir;     /* successful XDP redirects */
0144     u64 xdp_err;       /* packets dropped due to XDP errors */
0145 };
0146
0147 struct funeth_rxbuf {          /* per Rx buffer state */
0148     struct page *page;     /* associated page */
0149     dma_addr_t dma_addr;   /* DMA address of page start */
0150     int pg_refs;           /* page refs held by driver */
0151     int node;              /* page node, or -1 if it is PF_MEMALLOC */
0152 };
0153
0154 struct funeth_rx_cache {       /* cache of DMA-mapped previously used buffers */
0155     struct funeth_rxbuf *bufs; /* base of Rx buffer state ring */
0156     unsigned int prod_cnt;     /* producer counter */
0157     unsigned int cons_cnt;     /* consumer counter */
0158     unsigned int mask;         /* depth - 1 */
0159 };
0160
0161 /* An Rx queue consists of a CQ and an SQ used to provide Rx buffers. */
0162 struct funeth_rxq {
0163     struct net_device *netdev;
0164     struct napi_struct *napi;
0165     struct device *dma_dev;    /* device for DMA mappings */
0166     void *cqes;                /* base of CQ descriptor ring */
0167     const void *next_cqe_info; /* fun_cqe_info of next CQE */
0168     u32 __iomem *cq_db;        /* CQ doorbell register address */
0169     unsigned int cq_head;      /* CQ head index */
0170     unsigned int cq_mask;      /* CQ depth - 1 */
0171     u16 phase;                 /* CQ phase tag */
0172     u16 qidx;                  /* queue index within net_device */
0173     unsigned int irq_db_val;   /* IRQ info for CQ doorbell */
0174     struct fun_eprq_rqbuf *rqes; /* base of RQ descriptor ring */
0175     struct funeth_rxbuf *bufs; /* base of Rx buffer state ring */
0176     struct funeth_rxbuf *cur_buf; /* currently active buffer */
0177     u32 __iomem *rq_db;        /* RQ doorbell register address */
0178     unsigned int rq_cons;      /* RQ consumer counter */
0179     unsigned int rq_mask;      /* RQ depth - 1 */
0180     unsigned int buf_offset;   /* offset of next pkt in head buffer */
0181     u8 xdp_flush;              /* XDP flush types needed at NAPI end */
0182     u8 init_state;             /* queue initialization state */
0183     u16 headroom;              /* per packet headroom */
0184     unsigned int rq_cons_db;   /* value of rq_cons at last RQ db */
0185     unsigned int rq_db_thres;  /* # of new buffers needed to write RQ db */
0186     struct funeth_rxbuf spare_buf; /* spare for next buffer replacement */
0187     struct funeth_rx_cache cache; /* used buffer cache */
0188     struct bpf_prog *xdp_prog; /* optional XDP BPF program */
0189     struct funeth_rxq_stats stats;
0190     dma_addr_t cq_dma_addr;    /* DMA address of CQE ring */
0191     dma_addr_t rq_dma_addr;    /* DMA address of RQE ring */
0192     u16 irq_cnt;
0193     u32 hw_cqid;               /* device ID of the queue's CQ */
0194     u32 hw_sqid;               /* device ID of the queue's SQ */
0195     int numa_node;
0196     struct u64_stats_sync syncp;
0197     struct xdp_rxq_info xdp_rxq;
0198 };
0199
0200 #define FUN_QSTAT_INC(q, counter) \
0201     do { \
0202         u64_stats_update_begin(&(q)->syncp); \
0203         (q)->stats.counter++; \
0204         u64_stats_update_end(&(q)->syncp); \
0205     } while (0)
0206
0207 #define FUN_QSTAT_READ(q, seq, stats_copy) \
0208     do { \
0209         seq = u64_stats_fetch_begin_irq(&(q)->syncp); \
0210         stats_copy = (q)->stats; \
0211     } while (u64_stats_fetch_retry_irq(&(q)->syncp, (seq)))
0212
0213 #define FUN_INT_NAME_LEN (IFNAMSIZ + 16)
0214
0215 struct fun_irq {
0216     struct napi_struct napi;
0217     struct funeth_txq *txq;
0218     struct funeth_rxq *rxq;
0219     u8 state;
0220     u16 irq_idx;              /* index of MSI-X interrupt */
0221     int irq;                  /* Linux IRQ vector */
0222     cpumask_t affinity_mask;  /* IRQ affinity */
0223     struct irq_affinity_notify aff_notify;
0224     char name[FUN_INT_NAME_LEN];
0225 } ____cacheline_internodealigned_in_smp;
0226
0227 /* Return the start address of the idx-th Tx descriptor. */
0228 static inline void *fun_tx_desc_addr(const struct funeth_txq *q,
0229                      unsigned int idx)
0230 {
0231     return q->desc + idx * FUNETH_SQE_SIZE;
0232 }
0233
0234 static inline void fun_txq_wr_db(const struct funeth_txq *q)
0235 {
0236     unsigned int tail = q->prod_cnt & q->mask;
0237
0238     writel(tail, q->db);
0239 }
0240
0241 static inline int fun_irq_node(const struct fun_irq *p)
0242 {
0243     return cpu_to_mem(cpumask_first(&p->affinity_mask));
0244 }
0245
0246 int fun_rxq_napi_poll(struct napi_struct *napi, int budget);
0247 int fun_txq_napi_poll(struct napi_struct *napi, int budget);
0248 netdev_tx_t fun_start_xmit(struct sk_buff *skb, struct net_device *netdev);
0249 bool fun_xdp_tx(struct funeth_txq *q, struct xdp_frame *xdpf);
0250 int fun_xdp_xmit_frames(struct net_device *dev, int n,
0251             struct xdp_frame **frames, u32 flags);
0252
0253 int funeth_txq_create(struct net_device *dev, unsigned int qidx,
0254               unsigned int ndesc, struct fun_irq *irq, int state,
0255               struct funeth_txq **qp);
0256 int fun_txq_create_dev(struct funeth_txq *q, struct fun_irq *irq);
0257 struct funeth_txq *funeth_txq_free(struct funeth_txq *q, int state);
0258 int funeth_rxq_create(struct net_device *dev, unsigned int qidx,
0259               unsigned int ncqe, unsigned int nrqe, struct fun_irq *irq,
0260               int state, struct funeth_rxq **qp);
0261 int fun_rxq_create_dev(struct funeth_rxq *q, struct fun_irq *irq);
0262 struct funeth_rxq *funeth_rxq_free(struct funeth_rxq *q, int state);
0263 int fun_rxq_set_bpf(struct funeth_rxq *q, struct bpf_prog *prog);
0264
0265 #endif /* _FUNETH_TXRX_H */