net/vmxnet3/vmxnet3_int.h

0001 /*
0002  * Linux driver for VMware's vmxnet3 ethernet NIC.
0003  *
0004  * Copyright (C) 2008-2022, VMware, Inc. All Rights Reserved.
0005  *
0006  * This program is free software; you can redistribute it and/or modify it
0007  * under the terms of the GNU General Public License as published by the
0008  * Free Software Foundation; version 2 of the License and no later version.
0009  *
0010  * This program is distributed in the hope that it will be useful, but
0011  * WITHOUT ANY WARRANTY; without even the implied warranty of
0012  * MERCHANTABILITY OR FITNESS FOR A PARTICULAR PURPOSE, GOOD TITLE or
0013  * NON INFRINGEMENT.  See the GNU General Public License for more
0014  * details.
0015  *
0016  * You should have received a copy of the GNU General Public License
0017  * along with this program; if not, write to the Free Software
0018  * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
0019  *
0020  * The full GNU General Public License is included in this distribution in
0021  * the file called "COPYING".
0022  *
0023  * Maintained by: pv-drivers@vmware.com
0024  *
0025  */
0026
0027 #ifndef _VMXNET3_INT_H
0028 #define _VMXNET3_INT_H
0029
0030 #include <linux/bitops.h>
0031 #include <linux/ethtool.h>
0032 #include <linux/delay.h>
0033 #include <linux/netdevice.h>
0034 #include <linux/pci.h>
0035 #include <linux/compiler.h>
0036 #include <linux/slab.h>
0037 #include <linux/spinlock.h>
0038 #include <linux/ioport.h>
0039 #include <linux/highmem.h>
0040 #include <linux/timer.h>
0041 #include <linux/skbuff.h>
0042 #include <linux/interrupt.h>
0043 #include <linux/workqueue.h>
0044 #include <linux/uaccess.h>
0045 #include <asm/dma.h>
0046 #include <asm/page.h>
0047
0048 #include <linux/tcp.h>
0049 #include <linux/udp.h>
0050 #include <linux/ip.h>
0051 #include <linux/ipv6.h>
0052 #include <linux/in.h>
0053 #include <linux/etherdevice.h>
0054 #include <asm/checksum.h>
0055 #include <linux/if_vlan.h>
0056 #include <linux/if_arp.h>
0057 #include <linux/inetdevice.h>
0058 #include <linux/log2.h>
0059
0060 #include "vmxnet3_defs.h"
0061
0062 #ifdef DEBUG
0063 # define VMXNET3_DRIVER_VERSION_REPORT VMXNET3_DRIVER_VERSION_STRING"-NAPI(debug)"
0064 #else
0065 # define VMXNET3_DRIVER_VERSION_REPORT VMXNET3_DRIVER_VERSION_STRING"-NAPI"
0066 #endif
0067
0068
0069 /*
0070  * Version numbers
0071  */
0072 #define VMXNET3_DRIVER_VERSION_STRING   "1.7.0.0-k"
0073
0074 /* Each byte of this 32-bit integer encodes a version number in
0075  * VMXNET3_DRIVER_VERSION_STRING.
0076  */
0077 #define VMXNET3_DRIVER_VERSION_NUM      0x01070000
0078
0079 #if defined(CONFIG_PCI_MSI)
0080     /* RSS only makes sense if MSI-X is supported. */
0081     #define VMXNET3_RSS
0082 #endif
0083
0084 #define VMXNET3_REV_7       6   /* Vmxnet3 Rev. 7 */
0085 #define VMXNET3_REV_6       5   /* Vmxnet3 Rev. 6 */
0086 #define VMXNET3_REV_5       4   /* Vmxnet3 Rev. 5 */
0087 #define VMXNET3_REV_4       3   /* Vmxnet3 Rev. 4 */
0088 #define VMXNET3_REV_3       2   /* Vmxnet3 Rev. 3 */
0089 #define VMXNET3_REV_2       1   /* Vmxnet3 Rev. 2 */
0090 #define VMXNET3_REV_1       0   /* Vmxnet3 Rev. 1 */
0091
0092 /*
0093  * Capabilities
0094  */
0095
0096 enum {
0097     VMNET_CAP_SG            = 0x0001, /* Can do scatter-gather transmits. */
0098     VMNET_CAP_IP4_CSUM      = 0x0002, /* Can checksum only TCP/UDP over
0099                        * IPv4 */
0100     VMNET_CAP_HW_CSUM       = 0x0004, /* Can checksum all packets. */
0101     VMNET_CAP_HIGH_DMA      = 0x0008, /* Can DMA to high memory. */
0102     VMNET_CAP_TOE           = 0x0010, /* Supports TCP/IP offload. */
0103     VMNET_CAP_TSO           = 0x0020, /* Supports TCP Segmentation
0104                        * offload */
0105     VMNET_CAP_SW_TSO        = 0x0040, /* Supports SW TCP Segmentation */
0106     VMNET_CAP_VMXNET_APROM  = 0x0080, /* Vmxnet APROM support */
0107     VMNET_CAP_HW_TX_VLAN    = 0x0100, /* Can we do VLAN tagging in HW */
0108     VMNET_CAP_HW_RX_VLAN    = 0x0200, /* Can we do VLAN untagging in HW */
0109     VMNET_CAP_SW_VLAN       = 0x0400, /* VLAN tagging/untagging in SW */
0110     VMNET_CAP_WAKE_PCKT_RCV = 0x0800, /* Can wake on network packet recv? */
0111     VMNET_CAP_ENABLE_INT_INLINE = 0x1000,  /* Enable Interrupt Inline */
0112     VMNET_CAP_ENABLE_HEADER_COPY = 0x2000,  /* copy header for vmkernel */
0113     VMNET_CAP_TX_CHAIN      = 0x4000, /* Guest can use multiple tx entries
0114                       * for a pkt */
0115     VMNET_CAP_RX_CHAIN      = 0x8000, /* pkt can span multiple rx entries */
0116     VMNET_CAP_LPD           = 0x10000, /* large pkt delivery */
0117     VMNET_CAP_BPF           = 0x20000, /* BPF Support in VMXNET Virtual HW*/
0118     VMNET_CAP_SG_SPAN_PAGES = 0x40000, /* Scatter-gather can span multiple*/
0119                        /* pages transmits */
0120     VMNET_CAP_IP6_CSUM      = 0x80000, /* Can do IPv6 csum offload. */
0121     VMNET_CAP_TSO6         = 0x100000, /* TSO seg. offload for IPv6 pkts. */
0122     VMNET_CAP_TSO256k      = 0x200000, /* Can do TSO seg offload for */
0123                        /* pkts up to 256kB. */
0124     VMNET_CAP_UPT          = 0x400000  /* Support UPT */
0125 };
0126
0127 /*
0128  * Maximum devices supported.
0129  */
0130 #define MAX_ETHERNET_CARDS      10
0131 #define MAX_PCI_PASSTHRU_DEVICE     6
0132
0133 struct vmxnet3_cmd_ring {
0134     union Vmxnet3_GenericDesc *base;
0135     u32     size;
0136     u32     next2fill;
0137     u32     next2comp;
0138     u8      gen;
0139     u8              isOutOfOrder;
0140     dma_addr_t  basePA;
0141 };
0142
0143 static inline void
0144 vmxnet3_cmd_ring_adv_next2fill(struct vmxnet3_cmd_ring *ring)
0145 {
0146     ring->next2fill++;
0147     if (unlikely(ring->next2fill == ring->size)) {
0148         ring->next2fill = 0;
0149         VMXNET3_FLIP_RING_GEN(ring->gen);
0150     }
0151 }
0152
0153 static inline void
0154 vmxnet3_cmd_ring_adv_next2comp(struct vmxnet3_cmd_ring *ring)
0155 {
0156     VMXNET3_INC_RING_IDX_ONLY(ring->next2comp, ring->size);
0157 }
0158
0159 static inline int
0160 vmxnet3_cmd_ring_desc_avail(struct vmxnet3_cmd_ring *ring)
0161 {
0162     return (ring->next2comp > ring->next2fill ? 0 : ring->size) +
0163         ring->next2comp - ring->next2fill - 1;
0164 }
0165
0166 struct vmxnet3_comp_ring {
0167     union Vmxnet3_GenericDesc *base;
0168     u32               size;
0169     u32               next2proc;
0170     u8                gen;
0171     u8                intr_idx;
0172     dma_addr_t           basePA;
0173 };
0174
0175 static inline void
0176 vmxnet3_comp_ring_adv_next2proc(struct vmxnet3_comp_ring *ring)
0177 {
0178     ring->next2proc++;
0179     if (unlikely(ring->next2proc == ring->size)) {
0180         ring->next2proc = 0;
0181         VMXNET3_FLIP_RING_GEN(ring->gen);
0182     }
0183 }
0184
0185 struct vmxnet3_tx_data_ring {
0186     struct Vmxnet3_TxDataDesc *base;
0187     u32              size;
0188     dma_addr_t          basePA;
0189 };
0190
0191 enum vmxnet3_buf_map_type {
0192     VMXNET3_MAP_INVALID = 0,
0193     VMXNET3_MAP_NONE,
0194     VMXNET3_MAP_SINGLE,
0195     VMXNET3_MAP_PAGE,
0196 };
0197
0198 struct vmxnet3_tx_buf_info {
0199     u32      map_type;
0200     u16      len;
0201     u16      sop_idx;
0202     dma_addr_t  dma_addr;
0203     struct sk_buff *skb;
0204 };
0205
0206 struct vmxnet3_tq_driver_stats {
0207     u64 drop_total;     /* # of pkts dropped by the driver, the
0208                 * counters below track droppings due to
0209                 * different reasons
0210                 */
0211     u64 drop_too_many_frags;
0212     u64 drop_oversized_hdr;
0213     u64 drop_hdr_inspect_err;
0214     u64 drop_tso;
0215
0216     u64 tx_ring_full;
0217     u64 linearized;         /* # of pkts linearized */
0218     u64 copy_skb_header;    /* # of times we have to copy skb header */
0219     u64 oversized_hdr;
0220 };
0221
0222 struct vmxnet3_tx_ctx {
0223     bool   ipv4;
0224     bool   ipv6;
0225     u16 mss;
0226     u32    l4_offset;   /* only valid for pkts requesting tso or csum
0227                  * offloading. For encap offload, it refers to
0228                  * inner L4 offset i.e. it includes outer header
0229                  * encap header and inner eth and ip header size
0230                  */
0231
0232     u32 l4_hdr_size;    /* only valid if mss != 0
0233                  * Refers to inner L4 hdr size for encap
0234                  * offload
0235                  */
0236     u32 copy_size;       /* # of bytes copied into the data ring */
0237     union Vmxnet3_GenericDesc *sop_txd;
0238     union Vmxnet3_GenericDesc *eop_txd;
0239 };
0240
0241 struct vmxnet3_tx_queue {
0242     char            name[IFNAMSIZ+8]; /* To identify interrupt */
0243     struct vmxnet3_adapter      *adapter;
0244     spinlock_t                      tx_lock;
0245     struct vmxnet3_cmd_ring         tx_ring;
0246     struct vmxnet3_tx_buf_info      *buf_info;
0247     struct vmxnet3_tx_data_ring     data_ring;
0248     struct vmxnet3_comp_ring        comp_ring;
0249     struct Vmxnet3_TxQueueCtrl      *shared;
0250     struct vmxnet3_tq_driver_stats  stats;
0251     bool                            stopped;
0252     int                             num_stop;  /* # of times the queue is
0253                             * stopped */
0254     int             qid;
0255     u16             txdata_desc_size;
0256 } __attribute__((__aligned__(SMP_CACHE_BYTES)));
0257
0258 enum vmxnet3_rx_buf_type {
0259     VMXNET3_RX_BUF_NONE = 0,
0260     VMXNET3_RX_BUF_SKB = 1,
0261     VMXNET3_RX_BUF_PAGE = 2
0262 };
0263
0264 #define VMXNET3_RXD_COMP_PENDING        0
0265 #define VMXNET3_RXD_COMP_DONE           1
0266
0267 struct vmxnet3_rx_buf_info {
0268     enum vmxnet3_rx_buf_type buf_type;
0269     u16     len;
0270     u8      comp_state;
0271     union {
0272         struct sk_buff *skb;
0273         struct page    *page;
0274     };
0275     dma_addr_t dma_addr;
0276 };
0277
0278 struct vmxnet3_rx_ctx {
0279     struct sk_buff *skb;
0280     u32 sop_idx;
0281 };
0282
0283 struct vmxnet3_rq_driver_stats {
0284     u64 drop_total;
0285     u64 drop_err;
0286     u64 drop_fcs;
0287     u64 rx_buf_alloc_failure;
0288 };
0289
0290 struct vmxnet3_rx_data_ring {
0291     Vmxnet3_RxDataDesc *base;
0292     dma_addr_t basePA;
0293     u16 desc_size;
0294 };
0295
0296 struct vmxnet3_rx_queue {
0297     char            name[IFNAMSIZ + 8]; /* To identify interrupt */
0298     struct vmxnet3_adapter    *adapter;
0299     struct napi_struct        napi;
0300     struct vmxnet3_cmd_ring   rx_ring[2];
0301     struct vmxnet3_rx_data_ring data_ring;
0302     struct vmxnet3_comp_ring  comp_ring;
0303     struct vmxnet3_rx_ctx     rx_ctx;
0304     u32 qid;            /* rqID in RCD for buffer from 1st ring */
0305     u32 qid2;           /* rqID in RCD for buffer from 2nd ring */
0306     u32 dataRingQid;    /* rqID in RCD for buffer from data ring */
0307     struct vmxnet3_rx_buf_info     *buf_info[2];
0308     struct Vmxnet3_RxQueueCtrl            *shared;
0309     struct vmxnet3_rq_driver_stats  stats;
0310 } __attribute__((__aligned__(SMP_CACHE_BYTES)));
0311
0312 #define VMXNET3_DEVICE_MAX_TX_QUEUES 32
0313 #define VMXNET3_DEVICE_MAX_RX_QUEUES 32   /* Keep this value as a power of 2 */
0314
0315 #define VMXNET3_DEVICE_DEFAULT_TX_QUEUES 8
0316 #define VMXNET3_DEVICE_DEFAULT_RX_QUEUES 8   /* Keep this value as a power of 2 */
0317
0318 /* Should be less than UPT1_RSS_MAX_IND_TABLE_SIZE */
0319 #define VMXNET3_RSS_IND_TABLE_SIZE (VMXNET3_DEVICE_MAX_RX_QUEUES * 4)
0320
0321 #define VMXNET3_LINUX_MAX_MSIX_VECT     (VMXNET3_DEVICE_MAX_TX_QUEUES + \
0322                      VMXNET3_DEVICE_MAX_RX_QUEUES + 1)
0323 #define VMXNET3_LINUX_MIN_MSIX_VECT     3 /* 1 for tx, 1 for rx pair and 1 for event */
0324
0325
0326 struct vmxnet3_intr {
0327     enum vmxnet3_intr_mask_mode  mask_mode;
0328     enum vmxnet3_intr_type       type;  /* MSI-X, MSI, or INTx? */
0329     u8  num_intrs;          /* # of intr vectors */
0330     u8  event_intr_idx;     /* idx of the intr vector for event */
0331     u8  mod_levels[VMXNET3_LINUX_MAX_MSIX_VECT]; /* moderation level */
0332     char    event_msi_vector_name[IFNAMSIZ+17];
0333 #ifdef CONFIG_PCI_MSI
0334     struct msix_entry msix_entries[VMXNET3_LINUX_MAX_MSIX_VECT];
0335 #endif
0336 };
0337
0338 /* Interrupt sharing schemes, share_intr */
0339 #define VMXNET3_INTR_BUDDYSHARE 0    /* Corresponding tx,rx queues share irq */
0340 #define VMXNET3_INTR_TXSHARE 1       /* All tx queues share one irq */
0341 #define VMXNET3_INTR_DONTSHARE 2     /* each queue has its own irq */
0342
0343
0344 #define VMXNET3_STATE_BIT_RESETTING   0
0345 #define VMXNET3_STATE_BIT_QUIESCED    1
0346 struct vmxnet3_adapter {
0347     struct vmxnet3_tx_queue     tx_queue[VMXNET3_DEVICE_MAX_TX_QUEUES];
0348     struct vmxnet3_rx_queue     rx_queue[VMXNET3_DEVICE_MAX_RX_QUEUES];
0349     unsigned long           active_vlans[BITS_TO_LONGS(VLAN_N_VID)];
0350     struct vmxnet3_intr     intr;
0351     spinlock_t          cmd_lock;
0352     struct Vmxnet3_DriverShared *shared;
0353     struct Vmxnet3_PMConf       *pm_conf;
0354     struct Vmxnet3_TxQueueDesc  *tqd_start;     /* all tx queue desc */
0355     struct Vmxnet3_RxQueueDesc  *rqd_start; /* all rx queue desc */
0356     struct net_device       *netdev;
0357     struct pci_dev          *pdev;
0358
0359     u8          __iomem *hw_addr0; /* for BAR 0 */
0360     u8          __iomem *hw_addr1; /* for BAR 1 */
0361     u8                              version;
0362
0363 #ifdef VMXNET3_RSS
0364     struct UPT1_RSSConf     *rss_conf;
0365     bool                rss;
0366 #endif
0367     u32             num_rx_queues;
0368     u32             num_tx_queues;
0369
0370     /* rx buffer related */
0371     unsigned            skb_buf_size;
0372     int     rx_buf_per_pkt;  /* only apply to the 1st ring */
0373     dma_addr_t          shared_pa;
0374     dma_addr_t queue_desc_pa;
0375     dma_addr_t coal_conf_pa;
0376
0377     /* Wake-on-LAN */
0378     u32     wol;
0379
0380     /* Link speed */
0381     u32     link_speed; /* in mbps */
0382
0383     u64     tx_timeout_count;
0384
0385     /* Ring sizes */
0386     u32 tx_ring_size;
0387     u32 rx_ring_size;
0388     u32 rx_ring2_size;
0389
0390     /* Size of buffer in the data ring */
0391     u16 txdata_desc_size;
0392     u16 rxdata_desc_size;
0393
0394     bool rxdataring_enabled;
0395     bool default_rss_fields;
0396     enum Vmxnet3_RSSField rss_fields;
0397
0398     struct work_struct work;
0399
0400     unsigned long  state;    /* VMXNET3_STATE_BIT_xxx */
0401
0402     int share_intr;
0403
0404     struct Vmxnet3_CoalesceScheme *coal_conf;
0405     bool   default_coal_mode;
0406
0407     dma_addr_t adapter_pa;
0408     dma_addr_t pm_conf_pa;
0409     dma_addr_t rss_conf_pa;
0410     bool   queuesExtEnabled;
0411     struct Vmxnet3_RingBufferSize     ringBufSize;
0412     u32    devcap_supported[8];
0413     u32    ptcap_supported[8];
0414     u32    dev_caps[8];
0415     u16    tx_prod_offset;
0416     u16    rx_prod_offset;
0417     u16    rx_prod2_offset;
0418 };
0419
0420 #define VMXNET3_WRITE_BAR0_REG(adapter, reg, val)  \
0421     writel((val), (adapter)->hw_addr0 + (reg))
0422 #define VMXNET3_READ_BAR0_REG(adapter, reg)        \
0423     readl((adapter)->hw_addr0 + (reg))
0424
0425 #define VMXNET3_WRITE_BAR1_REG(adapter, reg, val)  \
0426     writel((val), (adapter)->hw_addr1 + (reg))
0427 #define VMXNET3_READ_BAR1_REG(adapter, reg)        \
0428     readl((adapter)->hw_addr1 + (reg))
0429
0430 #define VMXNET3_WAKE_QUEUE_THRESHOLD(tq)  (5)
0431 #define VMXNET3_RX_ALLOC_THRESHOLD(rq, ring_idx, adapter) \
0432     ((rq)->rx_ring[ring_idx].size >> 3)
0433
0434 #define VMXNET3_GET_ADDR_LO(dma)   ((u32)(dma))
0435 #define VMXNET3_GET_ADDR_HI(dma)   ((u32)(((u64)(dma)) >> 32))
0436
0437 #define VMXNET3_VERSION_GE_2(adapter) \
0438     (adapter->version >= VMXNET3_REV_2 + 1)
0439 #define VMXNET3_VERSION_GE_3(adapter) \
0440     (adapter->version >= VMXNET3_REV_3 + 1)
0441 #define VMXNET3_VERSION_GE_4(adapter) \
0442     (adapter->version >= VMXNET3_REV_4 + 1)
0443 #define VMXNET3_VERSION_GE_5(adapter) \
0444     (adapter->version >= VMXNET3_REV_5 + 1)
0445 #define VMXNET3_VERSION_GE_6(adapter) \
0446     (adapter->version >= VMXNET3_REV_6 + 1)
0447 #define VMXNET3_VERSION_GE_7(adapter) \
0448     (adapter->version >= VMXNET3_REV_7 + 1)
0449
0450 /* must be a multiple of VMXNET3_RING_SIZE_ALIGN */
0451 #define VMXNET3_DEF_TX_RING_SIZE    512
0452 #define VMXNET3_DEF_RX_RING_SIZE    1024
0453 #define VMXNET3_DEF_RX_RING2_SIZE   512
0454
0455 #define VMXNET3_DEF_RXDATA_DESC_SIZE 128
0456
0457 #define VMXNET3_MAX_ETH_HDR_SIZE    22
0458 #define VMXNET3_MAX_SKB_BUF_SIZE    (3*1024)
0459
0460 #define VMXNET3_GET_RING_IDX(adapter, rqID)     \
0461     ((rqID >= adapter->num_rx_queues &&     \
0462      rqID < 2 * adapter->num_rx_queues) ? 1 : 0)    \
0463
0464 #define VMXNET3_RX_DATA_RING(adapter, rqID)     \
0465     (rqID >= 2 * adapter->num_rx_queues &&      \
0466     rqID < 3 * adapter->num_rx_queues)      \
0467
0468 #define VMXNET3_COAL_STATIC_DEFAULT_DEPTH   64
0469
0470 #define VMXNET3_COAL_RBC_RATE(usecs) (1000000 / usecs)
0471 #define VMXNET3_COAL_RBC_USECS(rbc_rate) (1000000 / rbc_rate)
0472 #define VMXNET3_RSS_FIELDS_DEFAULT (VMXNET3_RSS_FIELDS_TCPIP4 | \
0473                     VMXNET3_RSS_FIELDS_TCPIP6)
0474
0475 int
0476 vmxnet3_quiesce_dev(struct vmxnet3_adapter *adapter);
0477
0478 int
0479 vmxnet3_activate_dev(struct vmxnet3_adapter *adapter);
0480
0481 void
0482 vmxnet3_force_close(struct vmxnet3_adapter *adapter);
0483
0484 void
0485 vmxnet3_reset_dev(struct vmxnet3_adapter *adapter);
0486
0487 void
0488 vmxnet3_tq_destroy_all(struct vmxnet3_adapter *adapter);
0489
0490 void
0491 vmxnet3_rq_destroy_all(struct vmxnet3_adapter *adapter);
0492
0493 netdev_features_t
0494 vmxnet3_fix_features(struct net_device *netdev, netdev_features_t features);
0495
0496 netdev_features_t
0497 vmxnet3_features_check(struct sk_buff *skb,
0498                struct net_device *netdev, netdev_features_t features);
0499
0500 int
0501 vmxnet3_set_features(struct net_device *netdev, netdev_features_t features);
0502
0503 int
0504 vmxnet3_create_queues(struct vmxnet3_adapter *adapter,
0505               u32 tx_ring_size, u32 rx_ring_size, u32 rx_ring2_size,
0506               u16 txdata_desc_size, u16 rxdata_desc_size);
0507
0508 void vmxnet3_set_ethtool_ops(struct net_device *netdev);
0509
0510 void vmxnet3_get_stats64(struct net_device *dev,
0511              struct rtnl_link_stats64 *stats);
0512 bool vmxnet3_check_ptcapability(u32 cap_supported, u32 cap);
0513
0514 extern char vmxnet3_driver_name[];
0515 #endif