core/en/txrx.h

0001 /* SPDX-License-Identifier: GPL-2.0 OR Linux-OpenIB */
0002 /* Copyright (c) 2019 Mellanox Technologies. */
0003
0004 #ifndef __MLX5_EN_TXRX_H___
0005 #define __MLX5_EN_TXRX_H___
0006
0007 #include "en.h"
0008 #include <linux/indirect_call_wrapper.h>
0009
0010 #define MLX5E_TX_WQE_EMPTY_DS_COUNT (sizeof(struct mlx5e_tx_wqe) / MLX5_SEND_WQE_DS)
0011
0012 #define INL_HDR_START_SZ (sizeof(((struct mlx5_wqe_eth_seg *)NULL)->inline_hdr.start))
0013
0014 #define MLX5E_RX_ERR_CQE(cqe) (get_cqe_opcode(cqe) != MLX5_CQE_RESP_SEND)
0015
0016 static inline
0017 ktime_t mlx5e_cqe_ts_to_ns(cqe_ts_to_ns func, struct mlx5_clock *clock, u64 cqe_ts)
0018 {
0019     return INDIRECT_CALL_2(func, mlx5_real_time_cyc2time, mlx5_timecounter_cyc2time,
0020                    clock, cqe_ts);
0021 }
0022
0023 enum mlx5e_icosq_wqe_type {
0024     MLX5E_ICOSQ_WQE_NOP,
0025     MLX5E_ICOSQ_WQE_UMR_RX,
0026     MLX5E_ICOSQ_WQE_SHAMPO_HD_UMR,
0027 #ifdef CONFIG_MLX5_EN_TLS
0028     MLX5E_ICOSQ_WQE_UMR_TLS,
0029     MLX5E_ICOSQ_WQE_SET_PSV_TLS,
0030     MLX5E_ICOSQ_WQE_GET_PSV_TLS,
0031 #endif
0032 };
0033
0034 /* General */
0035 static inline bool mlx5e_skb_is_multicast(struct sk_buff *skb)
0036 {
0037     return skb->pkt_type == PACKET_MULTICAST || skb->pkt_type == PACKET_BROADCAST;
0038 }
0039
0040 void mlx5e_trigger_irq(struct mlx5e_icosq *sq);
0041 void mlx5e_completion_event(struct mlx5_core_cq *mcq, struct mlx5_eqe *eqe);
0042 void mlx5e_cq_error_event(struct mlx5_core_cq *mcq, enum mlx5_event event);
0043 int mlx5e_napi_poll(struct napi_struct *napi, int budget);
0044 int mlx5e_poll_ico_cq(struct mlx5e_cq *cq);
0045
0046 /* RX */
0047 void mlx5e_page_dma_unmap(struct mlx5e_rq *rq, struct page *page);
0048 void mlx5e_page_release_dynamic(struct mlx5e_rq *rq, struct page *page, bool recycle);
0049 INDIRECT_CALLABLE_DECLARE(bool mlx5e_post_rx_wqes(struct mlx5e_rq *rq));
0050 INDIRECT_CALLABLE_DECLARE(bool mlx5e_post_rx_mpwqes(struct mlx5e_rq *rq));
0051 int mlx5e_poll_rx_cq(struct mlx5e_cq *cq, int budget);
0052 void mlx5e_free_rx_descs(struct mlx5e_rq *rq);
0053 void mlx5e_free_rx_in_progress_descs(struct mlx5e_rq *rq);
0054
0055 /* TX */
0056 netdev_tx_t mlx5e_xmit(struct sk_buff *skb, struct net_device *dev);
0057 bool mlx5e_poll_tx_cq(struct mlx5e_cq *cq, int napi_budget);
0058 void mlx5e_free_txqsq_descs(struct mlx5e_txqsq *sq);
0059
0060 static inline bool
0061 mlx5e_wqc_has_room_for(struct mlx5_wq_cyc *wq, u16 cc, u16 pc, u16 n)
0062 {
0063     return (mlx5_wq_cyc_ctr2ix(wq, cc - pc) >= n) || (cc == pc);
0064 }
0065
0066 static inline void *mlx5e_fetch_wqe(struct mlx5_wq_cyc *wq, u16 pi, size_t wqe_size)
0067 {
0068     void *wqe;
0069
0070     wqe = mlx5_wq_cyc_get_wqe(wq, pi);
0071     memset(wqe, 0, wqe_size);
0072
0073     return wqe;
0074 }
0075
0076 #define MLX5E_TX_FETCH_WQE(sq, pi) \
0077     ((struct mlx5e_tx_wqe *)mlx5e_fetch_wqe(&(sq)->wq, pi, sizeof(struct mlx5e_tx_wqe)))
0078
0079 static inline struct mlx5e_tx_wqe *
0080 mlx5e_post_nop(struct mlx5_wq_cyc *wq, u32 sqn, u16 *pc)
0081 {
0082     u16                         pi   = mlx5_wq_cyc_ctr2ix(wq, *pc);
0083     struct mlx5e_tx_wqe        *wqe  = mlx5_wq_cyc_get_wqe(wq, pi);
0084     struct mlx5_wqe_ctrl_seg   *cseg = &wqe->ctrl;
0085
0086     memset(cseg, 0, sizeof(*cseg));
0087
0088     cseg->opmod_idx_opcode = cpu_to_be32((*pc << 8) | MLX5_OPCODE_NOP);
0089     cseg->qpn_ds           = cpu_to_be32((sqn << 8) | 0x01);
0090
0091     (*pc)++;
0092
0093     return wqe;
0094 }
0095
0096 static inline struct mlx5e_tx_wqe *
0097 mlx5e_post_nop_fence(struct mlx5_wq_cyc *wq, u32 sqn, u16 *pc)
0098 {
0099     u16                         pi   = mlx5_wq_cyc_ctr2ix(wq, *pc);
0100     struct mlx5e_tx_wqe        *wqe  = mlx5_wq_cyc_get_wqe(wq, pi);
0101     struct mlx5_wqe_ctrl_seg   *cseg = &wqe->ctrl;
0102
0103     memset(cseg, 0, sizeof(*cseg));
0104
0105     cseg->opmod_idx_opcode = cpu_to_be32((*pc << 8) | MLX5_OPCODE_NOP);
0106     cseg->qpn_ds           = cpu_to_be32((sqn << 8) | 0x01);
0107     cseg->fm_ce_se         = MLX5_FENCE_MODE_INITIATOR_SMALL;
0108
0109     (*pc)++;
0110
0111     return wqe;
0112 }
0113
0114 struct mlx5e_tx_wqe_info {
0115     struct sk_buff *skb;
0116     u32 num_bytes;
0117     u8 num_wqebbs;
0118     u8 num_dma;
0119     u8 num_fifo_pkts;
0120 #ifdef CONFIG_MLX5_EN_TLS
0121     struct page *resync_dump_frag_page;
0122 #endif
0123 };
0124
0125 static inline u16 mlx5e_txqsq_get_next_pi(struct mlx5e_txqsq *sq, u16 size)
0126 {
0127     struct mlx5_wq_cyc *wq = &sq->wq;
0128     u16 pi, contig_wqebbs;
0129
0130     pi = mlx5_wq_cyc_ctr2ix(wq, sq->pc);
0131     contig_wqebbs = mlx5_wq_cyc_get_contig_wqebbs(wq, pi);
0132     if (unlikely(contig_wqebbs < size)) {
0133         struct mlx5e_tx_wqe_info *wi, *edge_wi;
0134
0135         wi = &sq->db.wqe_info[pi];
0136         edge_wi = wi + contig_wqebbs;
0137
0138         /* Fill SQ frag edge with NOPs to avoid WQE wrapping two pages. */
0139         for (; wi < edge_wi; wi++) {
0140             *wi = (struct mlx5e_tx_wqe_info) {
0141                 .num_wqebbs = 1,
0142             };
0143             mlx5e_post_nop(wq, sq->sqn, &sq->pc);
0144         }
0145         sq->stats->nop += contig_wqebbs;
0146
0147         pi = mlx5_wq_cyc_ctr2ix(wq, sq->pc);
0148     }
0149
0150     return pi;
0151 }
0152
0153 static inline u16 mlx5e_shampo_get_cqe_header_index(struct mlx5e_rq *rq, struct mlx5_cqe64 *cqe)
0154 {
0155     return be16_to_cpu(cqe->shampo.header_entry_index) & (rq->mpwqe.shampo->hd_per_wq - 1);
0156 }
0157
0158 struct mlx5e_shampo_umr {
0159     u16 len;
0160 };
0161
0162 struct mlx5e_icosq_wqe_info {
0163     u8 wqe_type;
0164     u8 num_wqebbs;
0165
0166     /* Auxiliary data for different wqe types. */
0167     union {
0168         struct {
0169             struct mlx5e_rq *rq;
0170         } umr;
0171         struct mlx5e_shampo_umr shampo;
0172 #ifdef CONFIG_MLX5_EN_TLS
0173         struct {
0174             struct mlx5e_ktls_offload_context_rx *priv_rx;
0175         } tls_set_params;
0176         struct {
0177             struct mlx5e_ktls_rx_resync_buf *buf;
0178         } tls_get_params;
0179 #endif
0180     };
0181 };
0182
0183 void mlx5e_free_icosq_descs(struct mlx5e_icosq *sq);
0184
0185 static inline u16 mlx5e_icosq_get_next_pi(struct mlx5e_icosq *sq, u16 size)
0186 {
0187     struct mlx5_wq_cyc *wq = &sq->wq;
0188     u16 pi, contig_wqebbs;
0189
0190     pi = mlx5_wq_cyc_ctr2ix(wq, sq->pc);
0191     contig_wqebbs = mlx5_wq_cyc_get_contig_wqebbs(wq, pi);
0192     if (unlikely(contig_wqebbs < size)) {
0193         struct mlx5e_icosq_wqe_info *wi, *edge_wi;
0194
0195         wi = &sq->db.wqe_info[pi];
0196         edge_wi = wi + contig_wqebbs;
0197
0198         /* Fill SQ frag edge with NOPs to avoid WQE wrapping two pages. */
0199         for (; wi < edge_wi; wi++) {
0200             *wi = (struct mlx5e_icosq_wqe_info) {
0201                 .wqe_type   = MLX5E_ICOSQ_WQE_NOP,
0202                 .num_wqebbs = 1,
0203             };
0204             mlx5e_post_nop(wq, sq->sqn, &sq->pc);
0205         }
0206
0207         pi = mlx5_wq_cyc_ctr2ix(wq, sq->pc);
0208     }
0209
0210     return pi;
0211 }
0212
0213 static inline void
0214 mlx5e_notify_hw(struct mlx5_wq_cyc *wq, u16 pc, void __iomem *uar_map,
0215         struct mlx5_wqe_ctrl_seg *ctrl)
0216 {
0217     ctrl->fm_ce_se |= MLX5_WQE_CTRL_CQ_UPDATE;
0218     /* ensure wqe is visible to device before updating doorbell record */
0219     dma_wmb();
0220
0221     *wq->db = cpu_to_be32(pc);
0222
0223     /* ensure doorbell record is visible to device before ringing the
0224      * doorbell
0225      */
0226     wmb();
0227
0228     mlx5_write64((__be32 *)ctrl, uar_map);
0229 }
0230
0231 static inline void mlx5e_cq_arm(struct mlx5e_cq *cq)
0232 {
0233     struct mlx5_core_cq *mcq;
0234
0235     mcq = &cq->mcq;
0236     mlx5_cq_arm(mcq, MLX5_CQ_DB_REQ_NOT, mcq->uar->map, cq->wq.cc);
0237 }
0238
0239 static inline struct mlx5e_sq_dma *
0240 mlx5e_dma_get(struct mlx5e_txqsq *sq, u32 i)
0241 {
0242     return &sq->db.dma_fifo[i & sq->dma_fifo_mask];
0243 }
0244
0245 static inline void
0246 mlx5e_dma_push(struct mlx5e_txqsq *sq, dma_addr_t addr, u32 size,
0247            enum mlx5e_dma_map_type map_type)
0248 {
0249     struct mlx5e_sq_dma *dma = mlx5e_dma_get(sq, sq->dma_fifo_pc++);
0250
0251     dma->addr = addr;
0252     dma->size = size;
0253     dma->type = map_type;
0254 }
0255
0256 static inline
0257 struct sk_buff **mlx5e_skb_fifo_get(struct mlx5e_skb_fifo *fifo, u16 i)
0258 {
0259     return &fifo->fifo[i & fifo->mask];
0260 }
0261
0262 static inline
0263 void mlx5e_skb_fifo_push(struct mlx5e_skb_fifo *fifo, struct sk_buff *skb)
0264 {
0265     struct sk_buff **skb_item = mlx5e_skb_fifo_get(fifo, (*fifo->pc)++);
0266
0267     *skb_item = skb;
0268 }
0269
0270 static inline
0271 struct sk_buff *mlx5e_skb_fifo_pop(struct mlx5e_skb_fifo *fifo)
0272 {
0273     return *mlx5e_skb_fifo_get(fifo, (*fifo->cc)++);
0274 }
0275
0276 static inline void
0277 mlx5e_tx_dma_unmap(struct device *pdev, struct mlx5e_sq_dma *dma)
0278 {
0279     switch (dma->type) {
0280     case MLX5E_DMA_MAP_SINGLE:
0281         dma_unmap_single(pdev, dma->addr, dma->size, DMA_TO_DEVICE);
0282         break;
0283     case MLX5E_DMA_MAP_PAGE:
0284         dma_unmap_page(pdev, dma->addr, dma->size, DMA_TO_DEVICE);
0285         break;
0286     default:
0287         WARN_ONCE(true, "mlx5e_tx_dma_unmap unknown DMA type!\n");
0288     }
0289 }
0290
0291 void mlx5e_sq_xmit_simple(struct mlx5e_txqsq *sq, struct sk_buff *skb, bool xmit_more);
0292 void mlx5e_tx_mpwqe_ensure_complete(struct mlx5e_txqsq *sq);
0293
0294 static inline bool mlx5e_tx_mpwqe_is_full(struct mlx5e_tx_mpwqe *session, u8 max_sq_mpw_wqebbs)
0295 {
0296     return session->ds_count == max_sq_mpw_wqebbs * MLX5_SEND_WQEBB_NUM_DS;
0297 }
0298
0299 static inline void mlx5e_rqwq_reset(struct mlx5e_rq *rq)
0300 {
0301     if (rq->wq_type == MLX5_WQ_TYPE_LINKED_LIST_STRIDING_RQ) {
0302         mlx5_wq_ll_reset(&rq->mpwqe.wq);
0303         rq->mpwqe.actual_wq_head = 0;
0304     } else {
0305         mlx5_wq_cyc_reset(&rq->wqe.wq);
0306     }
0307 }
0308
0309 static inline void mlx5e_dump_error_cqe(struct mlx5e_cq *cq, u32 qn,
0310                     struct mlx5_err_cqe *err_cqe)
0311 {
0312     struct mlx5_cqwq *wq = &cq->wq;
0313     u32 ci;
0314
0315     ci = mlx5_cqwq_ctr2ix(wq, wq->cc - 1);
0316
0317     netdev_err(cq->netdev,
0318            "Error cqe on cqn 0x%x, ci 0x%x, qn 0x%x, opcode 0x%x, syndrome 0x%x, vendor syndrome 0x%x\n",
0319            cq->mcq.cqn, ci, qn,
0320            get_cqe_opcode((struct mlx5_cqe64 *)err_cqe),
0321            err_cqe->syndrome, err_cqe->vendor_err_synd);
0322     mlx5_dump_err_cqe(cq->mdev, err_cqe);
0323 }
0324
0325 static inline u32 mlx5e_rqwq_get_size(struct mlx5e_rq *rq)
0326 {
0327     switch (rq->wq_type) {
0328     case MLX5_WQ_TYPE_LINKED_LIST_STRIDING_RQ:
0329         return mlx5_wq_ll_get_size(&rq->mpwqe.wq);
0330     default:
0331         return mlx5_wq_cyc_get_size(&rq->wqe.wq);
0332     }
0333 }
0334
0335 static inline u32 mlx5e_rqwq_get_cur_sz(struct mlx5e_rq *rq)
0336 {
0337     switch (rq->wq_type) {
0338     case MLX5_WQ_TYPE_LINKED_LIST_STRIDING_RQ:
0339         return rq->mpwqe.wq.cur_sz;
0340     default:
0341         return rq->wqe.wq.cur_sz;
0342     }
0343 }
0344
0345 static inline u16 mlx5e_rqwq_get_head(struct mlx5e_rq *rq)
0346 {
0347     switch (rq->wq_type) {
0348     case MLX5_WQ_TYPE_LINKED_LIST_STRIDING_RQ:
0349         return mlx5_wq_ll_get_head(&rq->mpwqe.wq);
0350     default:
0351         return mlx5_wq_cyc_get_head(&rq->wqe.wq);
0352     }
0353 }
0354
0355 static inline u16 mlx5e_rqwq_get_wqe_counter(struct mlx5e_rq *rq)
0356 {
0357     switch (rq->wq_type) {
0358     case MLX5_WQ_TYPE_LINKED_LIST_STRIDING_RQ:
0359         return mlx5_wq_ll_get_counter(&rq->mpwqe.wq);
0360     default:
0361         return mlx5_wq_cyc_get_counter(&rq->wqe.wq);
0362     }
0363 }
0364
0365 /* SW parser related functions */
0366
0367 struct mlx5e_swp_spec {
0368     __be16 l3_proto;
0369     u8 l4_proto;
0370     u8 is_tun;
0371     __be16 tun_l3_proto;
0372     u8 tun_l4_proto;
0373 };
0374
0375 static inline void mlx5e_eseg_swp_offsets_add_vlan(struct mlx5_wqe_eth_seg *eseg)
0376 {
0377     /* SWP offsets are in 2-bytes words */
0378     eseg->swp_outer_l3_offset += VLAN_HLEN / 2;
0379     eseg->swp_outer_l4_offset += VLAN_HLEN / 2;
0380     eseg->swp_inner_l3_offset += VLAN_HLEN / 2;
0381     eseg->swp_inner_l4_offset += VLAN_HLEN / 2;
0382 }
0383
0384 static inline void
0385 mlx5e_set_eseg_swp(struct sk_buff *skb, struct mlx5_wqe_eth_seg *eseg,
0386            struct mlx5e_swp_spec *swp_spec)
0387 {
0388     /* SWP offsets are in 2-bytes words */
0389     eseg->swp_outer_l3_offset = skb_network_offset(skb) / 2;
0390     if (swp_spec->l3_proto == htons(ETH_P_IPV6))
0391         eseg->swp_flags |= MLX5_ETH_WQE_SWP_OUTER_L3_IPV6;
0392     if (swp_spec->l4_proto) {
0393         eseg->swp_outer_l4_offset = skb_transport_offset(skb) / 2;
0394         if (swp_spec->l4_proto == IPPROTO_UDP)
0395             eseg->swp_flags |= MLX5_ETH_WQE_SWP_OUTER_L4_UDP;
0396     }
0397
0398     if (swp_spec->is_tun) {
0399         eseg->swp_inner_l3_offset = skb_inner_network_offset(skb) / 2;
0400         if (swp_spec->tun_l3_proto == htons(ETH_P_IPV6))
0401             eseg->swp_flags |= MLX5_ETH_WQE_SWP_INNER_L3_IPV6;
0402     } else { /* typically for ipsec when xfrm mode != XFRM_MODE_TUNNEL */
0403         eseg->swp_inner_l3_offset = skb_network_offset(skb) / 2;
0404         if (swp_spec->l3_proto == htons(ETH_P_IPV6))
0405             eseg->swp_flags |= MLX5_ETH_WQE_SWP_INNER_L3_IPV6;
0406     }
0407     switch (swp_spec->tun_l4_proto) {
0408     case IPPROTO_UDP:
0409         eseg->swp_flags |= MLX5_ETH_WQE_SWP_INNER_L4_UDP;
0410         fallthrough;
0411     case IPPROTO_TCP:
0412         eseg->swp_inner_l4_offset = skb_inner_transport_offset(skb) / 2;
0413         break;
0414     }
0415 }
0416
0417 #define MLX5E_STOP_ROOM(wqebbs) ((wqebbs) * 2 - 1)
0418
0419 static inline u16 mlx5e_stop_room_for_wqe(struct mlx5_core_dev *mdev, u16 wqe_size)
0420 {
0421     /* A WQE must not cross the page boundary, hence two conditions:
0422      * 1. Its size must not exceed the page size.
0423      * 2. If the WQE size is X, and the space remaining in a page is less
0424      *    than X, this space needs to be padded with NOPs. So, one WQE of
0425      *    size X may require up to X-1 WQEBBs of padding, which makes the
0426      *    stop room of X-1 + X.
0427      * WQE size is also limited by the hardware limit.
0428      */
0429     WARN_ONCE(wqe_size > mlx5e_get_max_sq_wqebbs(mdev),
0430           "wqe_size %u is greater than max SQ WQEBBs %u",
0431           wqe_size, mlx5e_get_max_sq_wqebbs(mdev));
0432
0433
0434     return MLX5E_STOP_ROOM(wqe_size);
0435 }
0436
0437 static inline u16 mlx5e_stop_room_for_max_wqe(struct mlx5_core_dev *mdev)
0438 {
0439     return MLX5E_STOP_ROOM(mlx5e_get_max_sq_wqebbs(mdev));
0440 }
0441
0442 static inline bool mlx5e_icosq_can_post_wqe(struct mlx5e_icosq *sq, u16 wqe_size)
0443 {
0444     u16 room = sq->reserved_room;
0445
0446     WARN_ONCE(wqe_size > sq->max_sq_wqebbs,
0447           "wqe_size %u is greater than max SQ WQEBBs %u",
0448           wqe_size, sq->max_sq_wqebbs);
0449
0450     room += MLX5E_STOP_ROOM(wqe_size);
0451
0452     return mlx5e_wqc_has_room_for(&sq->wq, sq->cc, sq->pc, room);
0453 }
0454 #endif