freescale/enetc/enetc.h

0001 /* SPDX-License-Identifier: (GPL-2.0+ OR BSD-3-Clause) */
0002 /* Copyright 2017-2019 NXP */
0003
0004 #include <linux/timer.h>
0005 #include <linux/pci.h>
0006 #include <linux/netdevice.h>
0007 #include <linux/etherdevice.h>
0008 #include <linux/dma-mapping.h>
0009 #include <linux/skbuff.h>
0010 #include <linux/ethtool.h>
0011 #include <linux/if_vlan.h>
0012 #include <linux/phylink.h>
0013 #include <linux/dim.h>
0014
0015 #include "enetc_hw.h"
0016
0017 #define ENETC_MAC_MAXFRM_SIZE   9600
0018 #define ENETC_MAX_MTU       (ENETC_MAC_MAXFRM_SIZE - \
0019                 (ETH_FCS_LEN + ETH_HLEN + VLAN_HLEN))
0020
0021 #define ENETC_CBD_DATA_MEM_ALIGN 64
0022
0023 struct enetc_tx_swbd {
0024     union {
0025         struct sk_buff *skb;
0026         struct xdp_frame *xdp_frame;
0027     };
0028     dma_addr_t dma;
0029     struct page *page;  /* valid only if is_xdp_tx */
0030     u16 page_offset;    /* valid only if is_xdp_tx */
0031     u16 len;
0032     enum dma_data_direction dir;
0033     u8 is_dma_page:1;
0034     u8 check_wb:1;
0035     u8 do_twostep_tstamp:1;
0036     u8 is_eof:1;
0037     u8 is_xdp_tx:1;
0038     u8 is_xdp_redirect:1;
0039     u8 qbv_en:1;
0040 };
0041
0042 #define ENETC_RX_MAXFRM_SIZE    ENETC_MAC_MAXFRM_SIZE
0043 #define ENETC_RXB_TRUESIZE  2048 /* PAGE_SIZE >> 1 */
0044 #define ENETC_RXB_PAD       NET_SKB_PAD /* add extra space if needed */
0045 #define ENETC_RXB_DMA_SIZE  \
0046     (SKB_WITH_OVERHEAD(ENETC_RXB_TRUESIZE) - ENETC_RXB_PAD)
0047 #define ENETC_RXB_DMA_SIZE_XDP  \
0048     (SKB_WITH_OVERHEAD(ENETC_RXB_TRUESIZE) - XDP_PACKET_HEADROOM)
0049
0050 struct enetc_rx_swbd {
0051     dma_addr_t dma;
0052     struct page *page;
0053     u16 page_offset;
0054     enum dma_data_direction dir;
0055     u16 len;
0056 };
0057
0058 /* ENETC overhead: optional extension BD + 1 BD gap */
0059 #define ENETC_TXBDS_NEEDED(val) ((val) + 2)
0060 /* max # of chained Tx BDs is 15, including head and extension BD */
0061 #define ENETC_MAX_SKB_FRAGS 13
0062 #define ENETC_TXBDS_MAX_NEEDED  ENETC_TXBDS_NEEDED(ENETC_MAX_SKB_FRAGS + 1)
0063
0064 struct enetc_ring_stats {
0065     unsigned int packets;
0066     unsigned int bytes;
0067     unsigned int rx_alloc_errs;
0068     unsigned int xdp_drops;
0069     unsigned int xdp_tx;
0070     unsigned int xdp_tx_drops;
0071     unsigned int xdp_redirect;
0072     unsigned int xdp_redirect_failures;
0073     unsigned int xdp_redirect_sg;
0074     unsigned int recycles;
0075     unsigned int recycle_failures;
0076     unsigned int win_drop;
0077 };
0078
0079 struct enetc_xdp_data {
0080     struct xdp_rxq_info rxq;
0081     struct bpf_prog *prog;
0082     int xdp_tx_in_flight;
0083 };
0084
0085 #define ENETC_RX_RING_DEFAULT_SIZE  2048
0086 #define ENETC_TX_RING_DEFAULT_SIZE  2048
0087 #define ENETC_DEFAULT_TX_WORK       (ENETC_TX_RING_DEFAULT_SIZE / 2)
0088
0089 struct enetc_bdr {
0090     struct device *dev; /* for DMA mapping */
0091     struct net_device *ndev;
0092     void *bd_base; /* points to Rx or Tx BD ring */
0093     union {
0094         void __iomem *tpir;
0095         void __iomem *rcir;
0096     };
0097     u16 index;
0098     int bd_count; /* # of BDs */
0099     int next_to_use;
0100     int next_to_clean;
0101     union {
0102         struct enetc_tx_swbd *tx_swbd;
0103         struct enetc_rx_swbd *rx_swbd;
0104     };
0105     union {
0106         void __iomem *tcir; /* Tx */
0107         int next_to_alloc; /* Rx */
0108     };
0109     void __iomem *idr; /* Interrupt Detect Register pointer */
0110
0111     int buffer_offset;
0112     struct enetc_xdp_data xdp;
0113
0114     struct enetc_ring_stats stats;
0115
0116     dma_addr_t bd_dma_base;
0117     u8 tsd_enable; /* Time specific departure */
0118     bool ext_en; /* enable h/w descriptor extensions */
0119
0120     /* DMA buffer for TSO headers */
0121     char *tso_headers;
0122     dma_addr_t tso_headers_dma;
0123 } ____cacheline_aligned_in_smp;
0124
0125 static inline void enetc_bdr_idx_inc(struct enetc_bdr *bdr, int *i)
0126 {
0127     if (unlikely(++*i == bdr->bd_count))
0128         *i = 0;
0129 }
0130
0131 static inline int enetc_bd_unused(struct enetc_bdr *bdr)
0132 {
0133     if (bdr->next_to_clean > bdr->next_to_use)
0134         return bdr->next_to_clean - bdr->next_to_use - 1;
0135
0136     return bdr->bd_count + bdr->next_to_clean - bdr->next_to_use - 1;
0137 }
0138
0139 static inline int enetc_swbd_unused(struct enetc_bdr *bdr)
0140 {
0141     if (bdr->next_to_clean > bdr->next_to_alloc)
0142         return bdr->next_to_clean - bdr->next_to_alloc - 1;
0143
0144     return bdr->bd_count + bdr->next_to_clean - bdr->next_to_alloc - 1;
0145 }
0146
0147 /* Control BD ring */
0148 #define ENETC_CBDR_DEFAULT_SIZE 64
0149 struct enetc_cbdr {
0150     void *bd_base; /* points to Rx or Tx BD ring */
0151     void __iomem *pir;
0152     void __iomem *cir;
0153     void __iomem *mr; /* mode register */
0154
0155     int bd_count; /* # of BDs */
0156     int next_to_use;
0157     int next_to_clean;
0158
0159     dma_addr_t bd_dma_base;
0160     struct device *dma_dev;
0161 };
0162
0163 #define ENETC_TXBD(BDR, i) (&(((union enetc_tx_bd *)((BDR).bd_base))[i]))
0164
0165 static inline union enetc_rx_bd *enetc_rxbd(struct enetc_bdr *rx_ring, int i)
0166 {
0167     int hw_idx = i;
0168
0169 #ifdef CONFIG_FSL_ENETC_PTP_CLOCK
0170     if (rx_ring->ext_en)
0171         hw_idx = 2 * i;
0172 #endif
0173     return &(((union enetc_rx_bd *)rx_ring->bd_base)[hw_idx]);
0174 }
0175
0176 static inline void enetc_rxbd_next(struct enetc_bdr *rx_ring,
0177                    union enetc_rx_bd **old_rxbd, int *old_index)
0178 {
0179     union enetc_rx_bd *new_rxbd = *old_rxbd;
0180     int new_index = *old_index;
0181
0182     new_rxbd++;
0183
0184 #ifdef CONFIG_FSL_ENETC_PTP_CLOCK
0185     if (rx_ring->ext_en)
0186         new_rxbd++;
0187 #endif
0188
0189     if (unlikely(++new_index == rx_ring->bd_count)) {
0190         new_rxbd = rx_ring->bd_base;
0191         new_index = 0;
0192     }
0193
0194     *old_rxbd = new_rxbd;
0195     *old_index = new_index;
0196 }
0197
0198 static inline union enetc_rx_bd *enetc_rxbd_ext(union enetc_rx_bd *rxbd)
0199 {
0200     return ++rxbd;
0201 }
0202
0203 struct enetc_msg_swbd {
0204     void *vaddr;
0205     dma_addr_t dma;
0206     int size;
0207 };
0208
0209 #define ENETC_REV1  0x1
0210 enum enetc_errata {
0211     ENETC_ERR_VLAN_ISOL = BIT(0),
0212     ENETC_ERR_UCMCSWP   = BIT(1),
0213 };
0214
0215 #define ENETC_SI_F_QBV BIT(0)
0216 #define ENETC_SI_F_PSFP BIT(1)
0217
0218 /* PCI IEP device data */
0219 struct enetc_si {
0220     struct pci_dev *pdev;
0221     struct enetc_hw hw;
0222     enum enetc_errata errata;
0223
0224     struct net_device *ndev; /* back ref. */
0225
0226     struct enetc_cbdr cbd_ring;
0227
0228     int num_rx_rings; /* how many rings are available in the SI */
0229     int num_tx_rings;
0230     int num_fs_entries;
0231     int num_rss; /* number of RSS buckets */
0232     unsigned short pad;
0233     int hw_features;
0234 };
0235
0236 #define ENETC_SI_ALIGN  32
0237
0238 static inline void *enetc_si_priv(const struct enetc_si *si)
0239 {
0240     return (char *)si + ALIGN(sizeof(struct enetc_si), ENETC_SI_ALIGN);
0241 }
0242
0243 static inline bool enetc_si_is_pf(struct enetc_si *si)
0244 {
0245     return !!(si->hw.port);
0246 }
0247
0248 static inline int enetc_pf_to_port(struct pci_dev *pf_pdev)
0249 {
0250     switch (pf_pdev->devfn) {
0251     case 0:
0252         return 0;
0253     case 1:
0254         return 1;
0255     case 2:
0256         return 2;
0257     case 6:
0258         return 3;
0259     default:
0260         return -1;
0261     }
0262 }
0263
0264 #define ENETC_MAX_NUM_TXQS  8
0265 #define ENETC_INT_NAME_MAX  (IFNAMSIZ + 8)
0266
0267 struct enetc_int_vector {
0268     void __iomem *rbier;
0269     void __iomem *tbier_base;
0270     void __iomem *ricr1;
0271     unsigned long tx_rings_map;
0272     int count_tx_rings;
0273     u32 rx_ictt;
0274     u16 comp_cnt;
0275     bool rx_dim_en, rx_napi_work;
0276     struct napi_struct napi ____cacheline_aligned_in_smp;
0277     struct dim rx_dim ____cacheline_aligned_in_smp;
0278     char name[ENETC_INT_NAME_MAX];
0279
0280     struct enetc_bdr rx_ring;
0281     struct enetc_bdr tx_ring[];
0282 } ____cacheline_aligned_in_smp;
0283
0284 struct enetc_cls_rule {
0285     struct ethtool_rx_flow_spec fs;
0286     int used;
0287 };
0288
0289 #define ENETC_MAX_BDR_INT   2 /* fixed to max # of available cpus */
0290 struct psfp_cap {
0291     u32 max_streamid;
0292     u32 max_psfp_filter;
0293     u32 max_psfp_gate;
0294     u32 max_psfp_gatelist;
0295     u32 max_psfp_meter;
0296 };
0297
0298 #define ENETC_F_TX_TSTAMP_MASK  0xff
0299 /* TODO: more hardware offloads */
0300 enum enetc_active_offloads {
0301     /* 8 bits reserved for TX timestamp types (hwtstamp_tx_types) */
0302     ENETC_F_TX_TSTAMP       = BIT(0),
0303     ENETC_F_TX_ONESTEP_SYNC_TSTAMP  = BIT(1),
0304
0305     ENETC_F_RX_TSTAMP       = BIT(8),
0306     ENETC_F_QBV         = BIT(9),
0307     ENETC_F_QCI         = BIT(10),
0308 };
0309
0310 enum enetc_flags_bit {
0311     ENETC_TX_ONESTEP_TSTAMP_IN_PROGRESS = 0,
0312 };
0313
0314 /* interrupt coalescing modes */
0315 enum enetc_ic_mode {
0316     /* one interrupt per frame */
0317     ENETC_IC_NONE = 0,
0318     /* activated when int coalescing time is set to a non-0 value */
0319     ENETC_IC_RX_MANUAL = BIT(0),
0320     ENETC_IC_TX_MANUAL = BIT(1),
0321     /* use dynamic interrupt moderation */
0322     ENETC_IC_RX_ADAPTIVE = BIT(2),
0323 };
0324
0325 #define ENETC_RXIC_PKTTHR   min_t(u32, 256, ENETC_RX_RING_DEFAULT_SIZE / 2)
0326 #define ENETC_TXIC_PKTTHR   min_t(u32, 128, ENETC_TX_RING_DEFAULT_SIZE / 2)
0327 #define ENETC_TXIC_TIMETHR  enetc_usecs_to_cycles(600)
0328
0329 struct enetc_ndev_priv {
0330     struct net_device *ndev;
0331     struct device *dev; /* dma-mapping device */
0332     struct enetc_si *si;
0333
0334     int bdr_int_num; /* number of Rx/Tx ring interrupts */
0335     struct enetc_int_vector *int_vector[ENETC_MAX_BDR_INT];
0336     u16 num_rx_rings, num_tx_rings;
0337     u16 rx_bd_count, tx_bd_count;
0338
0339     u16 msg_enable;
0340     enum enetc_active_offloads active_offloads;
0341
0342     u32 speed; /* store speed for compare update pspeed */
0343
0344     struct enetc_bdr **xdp_tx_ring;
0345     struct enetc_bdr *tx_ring[16];
0346     struct enetc_bdr *rx_ring[16];
0347
0348     struct enetc_cls_rule *cls_rules;
0349
0350     struct psfp_cap psfp_cap;
0351
0352     struct phylink *phylink;
0353     int ic_mode;
0354     u32 tx_ictt;
0355
0356     struct bpf_prog *xdp_prog;
0357
0358     unsigned long flags;
0359
0360     struct work_struct  tx_onestep_tstamp;
0361     struct sk_buff_head tx_skbs;
0362 };
0363
0364 /* Messaging */
0365
0366 /* VF-PF set primary MAC address message format */
0367 struct enetc_msg_cmd_set_primary_mac {
0368     struct enetc_msg_cmd_header header;
0369     struct sockaddr mac;
0370 };
0371
0372 #define ENETC_CBD(R, i) (&(((struct enetc_cbd *)((R).bd_base))[i]))
0373
0374 #define ENETC_CBDR_TIMEOUT  1000 /* usecs */
0375
0376 /* PTP driver exports */
0377 extern int enetc_phc_index;
0378
0379 /* SI common */
0380 int enetc_pci_probe(struct pci_dev *pdev, const char *name, int sizeof_priv);
0381 void enetc_pci_remove(struct pci_dev *pdev);
0382 int enetc_alloc_msix(struct enetc_ndev_priv *priv);
0383 void enetc_free_msix(struct enetc_ndev_priv *priv);
0384 void enetc_get_si_caps(struct enetc_si *si);
0385 void enetc_init_si_rings_params(struct enetc_ndev_priv *priv);
0386 int enetc_alloc_si_resources(struct enetc_ndev_priv *priv);
0387 void enetc_free_si_resources(struct enetc_ndev_priv *priv);
0388 int enetc_configure_si(struct enetc_ndev_priv *priv);
0389
0390 int enetc_open(struct net_device *ndev);
0391 int enetc_close(struct net_device *ndev);
0392 void enetc_start(struct net_device *ndev);
0393 void enetc_stop(struct net_device *ndev);
0394 netdev_tx_t enetc_xmit(struct sk_buff *skb, struct net_device *ndev);
0395 struct net_device_stats *enetc_get_stats(struct net_device *ndev);
0396 void enetc_set_features(struct net_device *ndev, netdev_features_t features);
0397 int enetc_ioctl(struct net_device *ndev, struct ifreq *rq, int cmd);
0398 int enetc_setup_tc_mqprio(struct net_device *ndev, void *type_data);
0399 int enetc_setup_bpf(struct net_device *dev, struct netdev_bpf *xdp);
0400 int enetc_xdp_xmit(struct net_device *ndev, int num_frames,
0401            struct xdp_frame **frames, u32 flags);
0402
0403 /* ethtool */
0404 void enetc_set_ethtool_ops(struct net_device *ndev);
0405
0406 /* control buffer descriptor ring (CBDR) */
0407 int enetc_setup_cbdr(struct device *dev, struct enetc_hw *hw, int bd_count,
0408              struct enetc_cbdr *cbdr);
0409 void enetc_teardown_cbdr(struct enetc_cbdr *cbdr);
0410 int enetc_set_mac_flt_entry(struct enetc_si *si, int index,
0411                 char *mac_addr, int si_map);
0412 int enetc_clear_mac_flt_entry(struct enetc_si *si, int index);
0413 int enetc_set_fs_entry(struct enetc_si *si, struct enetc_cmd_rfse *rfse,
0414                int index);
0415 void enetc_set_rss_key(struct enetc_hw *hw, const u8 *bytes);
0416 int enetc_get_rss_table(struct enetc_si *si, u32 *table, int count);
0417 int enetc_set_rss_table(struct enetc_si *si, const u32 *table, int count);
0418 int enetc_send_cmd(struct enetc_si *si, struct enetc_cbd *cbd);
0419
0420 static inline void *enetc_cbd_alloc_data_mem(struct enetc_si *si,
0421                          struct enetc_cbd *cbd,
0422                          int size, dma_addr_t *dma,
0423                          void **data_align)
0424 {
0425     struct enetc_cbdr *ring = &si->cbd_ring;
0426     dma_addr_t dma_align;
0427     void *data;
0428
0429     data = dma_alloc_coherent(ring->dma_dev,
0430                   size + ENETC_CBD_DATA_MEM_ALIGN,
0431                   dma, GFP_KERNEL);
0432     if (!data) {
0433         dev_err(ring->dma_dev, "CBD alloc data memory failed!\n");
0434         return NULL;
0435     }
0436
0437     dma_align = ALIGN(*dma, ENETC_CBD_DATA_MEM_ALIGN);
0438     *data_align = PTR_ALIGN(data, ENETC_CBD_DATA_MEM_ALIGN);
0439
0440     cbd->addr[0] = cpu_to_le32(lower_32_bits(dma_align));
0441     cbd->addr[1] = cpu_to_le32(upper_32_bits(dma_align));
0442     cbd->length = cpu_to_le16(size);
0443
0444     return data;
0445 }
0446
0447 static inline void enetc_cbd_free_data_mem(struct enetc_si *si, int size,
0448                        void *data, dma_addr_t *dma)
0449 {
0450     struct enetc_cbdr *ring = &si->cbd_ring;
0451
0452     dma_free_coherent(ring->dma_dev, size + ENETC_CBD_DATA_MEM_ALIGN,
0453               data, *dma);
0454 }
0455
0456 #ifdef CONFIG_FSL_ENETC_QOS
0457 int enetc_setup_tc_taprio(struct net_device *ndev, void *type_data);
0458 void enetc_sched_speed_set(struct enetc_ndev_priv *priv, int speed);
0459 int enetc_setup_tc_cbs(struct net_device *ndev, void *type_data);
0460 int enetc_setup_tc_txtime(struct net_device *ndev, void *type_data);
0461 int enetc_setup_tc_block_cb(enum tc_setup_type type, void *type_data,
0462                 void *cb_priv);
0463 int enetc_setup_tc_psfp(struct net_device *ndev, void *type_data);
0464 int enetc_psfp_init(struct enetc_ndev_priv *priv);
0465 int enetc_psfp_clean(struct enetc_ndev_priv *priv);
0466 int enetc_set_psfp(struct net_device *ndev, bool en);
0467
0468 static inline void enetc_get_max_cap(struct enetc_ndev_priv *priv)
0469 {
0470     u32 reg;
0471
0472     reg = enetc_port_rd(&priv->si->hw, ENETC_PSIDCAPR);
0473     priv->psfp_cap.max_streamid = reg & ENETC_PSIDCAPR_MSK;
0474     /* Port stream filter capability */
0475     reg = enetc_port_rd(&priv->si->hw, ENETC_PSFCAPR);
0476     priv->psfp_cap.max_psfp_filter = reg & ENETC_PSFCAPR_MSK;
0477     /* Port stream gate capability */
0478     reg = enetc_port_rd(&priv->si->hw, ENETC_PSGCAPR);
0479     priv->psfp_cap.max_psfp_gate = (reg & ENETC_PSGCAPR_SGIT_MSK);
0480     priv->psfp_cap.max_psfp_gatelist = (reg & ENETC_PSGCAPR_GCL_MSK) >> 16;
0481     /* Port flow meter capability */
0482     reg = enetc_port_rd(&priv->si->hw, ENETC_PFMCAPR);
0483     priv->psfp_cap.max_psfp_meter = reg & ENETC_PFMCAPR_MSK;
0484 }
0485
0486 static inline int enetc_psfp_enable(struct enetc_ndev_priv *priv)
0487 {
0488     struct enetc_hw *hw = &priv->si->hw;
0489     int err;
0490
0491     enetc_get_max_cap(priv);
0492
0493     err = enetc_psfp_init(priv);
0494     if (err)
0495         return err;
0496
0497     enetc_wr(hw, ENETC_PPSFPMR, enetc_rd(hw, ENETC_PPSFPMR) |
0498          ENETC_PPSFPMR_PSFPEN | ENETC_PPSFPMR_VS |
0499          ENETC_PPSFPMR_PVC | ENETC_PPSFPMR_PVZC);
0500
0501     return 0;
0502 }
0503
0504 static inline int enetc_psfp_disable(struct enetc_ndev_priv *priv)
0505 {
0506     struct enetc_hw *hw = &priv->si->hw;
0507     int err;
0508
0509     err = enetc_psfp_clean(priv);
0510     if (err)
0511         return err;
0512
0513     enetc_wr(hw, ENETC_PPSFPMR, enetc_rd(hw, ENETC_PPSFPMR) &
0514          ~ENETC_PPSFPMR_PSFPEN & ~ENETC_PPSFPMR_VS &
0515          ~ENETC_PPSFPMR_PVC & ~ENETC_PPSFPMR_PVZC);
0516
0517     memset(&priv->psfp_cap, 0, sizeof(struct psfp_cap));
0518
0519     return 0;
0520 }
0521
0522 #else
0523 #define enetc_setup_tc_taprio(ndev, type_data) -EOPNOTSUPP
0524 #define enetc_sched_speed_set(priv, speed) (void)0
0525 #define enetc_setup_tc_cbs(ndev, type_data) -EOPNOTSUPP
0526 #define enetc_setup_tc_txtime(ndev, type_data) -EOPNOTSUPP
0527 #define enetc_setup_tc_psfp(ndev, type_data) -EOPNOTSUPP
0528 #define enetc_setup_tc_block_cb NULL
0529
0530 #define enetc_get_max_cap(p)        \
0531     memset(&((p)->psfp_cap), 0, sizeof(struct psfp_cap))
0532
0533 static inline int enetc_psfp_enable(struct enetc_ndev_priv *priv)
0534 {
0535     return 0;
0536 }
0537
0538 static inline int enetc_psfp_disable(struct enetc_ndev_priv *priv)
0539 {
0540     return 0;
0541 }
0542
0543 static inline int enetc_set_psfp(struct net_device *ndev, bool en)
0544 {
0545     return 0;
0546 }
0547 #endif