intel/ixgbevf/ixgbevf_main.c

0001 // SPDX-License-Identifier: GPL-2.0
0002 /* Copyright(c) 1999 - 2018 Intel Corporation. */
0003
0004 /******************************************************************************
0005  Copyright (c)2006 - 2007 Myricom, Inc. for some LRO specific code
0006 ******************************************************************************/
0007
0008 #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
0009
0010 #include <linux/types.h>
0011 #include <linux/bitops.h>
0012 #include <linux/module.h>
0013 #include <linux/pci.h>
0014 #include <linux/netdevice.h>
0015 #include <linux/vmalloc.h>
0016 #include <linux/string.h>
0017 #include <linux/in.h>
0018 #include <linux/ip.h>
0019 #include <linux/tcp.h>
0020 #include <linux/sctp.h>
0021 #include <linux/ipv6.h>
0022 #include <linux/slab.h>
0023 #include <net/checksum.h>
0024 #include <net/ip6_checksum.h>
0025 #include <linux/ethtool.h>
0026 #include <linux/if.h>
0027 #include <linux/if_vlan.h>
0028 #include <linux/prefetch.h>
0029 #include <net/mpls.h>
0030 #include <linux/bpf.h>
0031 #include <linux/bpf_trace.h>
0032 #include <linux/atomic.h>
0033 #include <net/xfrm.h>
0034
0035 #include "ixgbevf.h"
0036
0037 const char ixgbevf_driver_name[] = "ixgbevf";
0038 static const char ixgbevf_driver_string[] =
0039     "Intel(R) 10 Gigabit PCI Express Virtual Function Network Driver";
0040
0041 static char ixgbevf_copyright[] =
0042     "Copyright (c) 2009 - 2018 Intel Corporation.";
0043
0044 static const struct ixgbevf_info *ixgbevf_info_tbl[] = {
0045     [board_82599_vf]    = &ixgbevf_82599_vf_info,
0046     [board_82599_vf_hv] = &ixgbevf_82599_vf_hv_info,
0047     [board_X540_vf]     = &ixgbevf_X540_vf_info,
0048     [board_X540_vf_hv]  = &ixgbevf_X540_vf_hv_info,
0049     [board_X550_vf]     = &ixgbevf_X550_vf_info,
0050     [board_X550_vf_hv]  = &ixgbevf_X550_vf_hv_info,
0051     [board_X550EM_x_vf] = &ixgbevf_X550EM_x_vf_info,
0052     [board_X550EM_x_vf_hv]  = &ixgbevf_X550EM_x_vf_hv_info,
0053     [board_x550em_a_vf] = &ixgbevf_x550em_a_vf_info,
0054 };
0055
0056 /* ixgbevf_pci_tbl - PCI Device ID Table
0057  *
0058  * Wildcard entries (PCI_ANY_ID) should come last
0059  * Last entry must be all 0s
0060  *
0061  * { Vendor ID, Device ID, SubVendor ID, SubDevice ID,
0062  *   Class, Class Mask, private data (not used) }
0063  */
0064 static const struct pci_device_id ixgbevf_pci_tbl[] = {
0065     {PCI_VDEVICE(INTEL, IXGBE_DEV_ID_82599_VF), board_82599_vf },
0066     {PCI_VDEVICE(INTEL, IXGBE_DEV_ID_82599_VF_HV), board_82599_vf_hv },
0067     {PCI_VDEVICE(INTEL, IXGBE_DEV_ID_X540_VF), board_X540_vf },
0068     {PCI_VDEVICE(INTEL, IXGBE_DEV_ID_X540_VF_HV), board_X540_vf_hv },
0069     {PCI_VDEVICE(INTEL, IXGBE_DEV_ID_X550_VF), board_X550_vf },
0070     {PCI_VDEVICE(INTEL, IXGBE_DEV_ID_X550_VF_HV), board_X550_vf_hv },
0071     {PCI_VDEVICE(INTEL, IXGBE_DEV_ID_X550EM_X_VF), board_X550EM_x_vf },
0072     {PCI_VDEVICE(INTEL, IXGBE_DEV_ID_X550EM_X_VF_HV), board_X550EM_x_vf_hv},
0073     {PCI_VDEVICE(INTEL, IXGBE_DEV_ID_X550EM_A_VF), board_x550em_a_vf },
0074     /* required last entry */
0075     {0, }
0076 };
0077 MODULE_DEVICE_TABLE(pci, ixgbevf_pci_tbl);
0078
0079 MODULE_AUTHOR("Intel Corporation, <linux.nics@intel.com>");
0080 MODULE_DESCRIPTION("Intel(R) 10 Gigabit Virtual Function Network Driver");
0081 MODULE_LICENSE("GPL v2");
0082
0083 #define DEFAULT_MSG_ENABLE (NETIF_MSG_DRV|NETIF_MSG_PROBE|NETIF_MSG_LINK)
0084 static int debug = -1;
0085 module_param(debug, int, 0);
0086 MODULE_PARM_DESC(debug, "Debug level (0=none,...,16=all)");
0087
0088 static struct workqueue_struct *ixgbevf_wq;
0089
0090 static void ixgbevf_service_event_schedule(struct ixgbevf_adapter *adapter)
0091 {
0092     if (!test_bit(__IXGBEVF_DOWN, &adapter->state) &&
0093         !test_bit(__IXGBEVF_REMOVING, &adapter->state) &&
0094         !test_and_set_bit(__IXGBEVF_SERVICE_SCHED, &adapter->state))
0095         queue_work(ixgbevf_wq, &adapter->service_task);
0096 }
0097
0098 static void ixgbevf_service_event_complete(struct ixgbevf_adapter *adapter)
0099 {
0100     BUG_ON(!test_bit(__IXGBEVF_SERVICE_SCHED, &adapter->state));
0101
0102     /* flush memory to make sure state is correct before next watchdog */
0103     smp_mb__before_atomic();
0104     clear_bit(__IXGBEVF_SERVICE_SCHED, &adapter->state);
0105 }
0106
0107 /* forward decls */
0108 static void ixgbevf_queue_reset_subtask(struct ixgbevf_adapter *adapter);
0109 static void ixgbevf_set_itr(struct ixgbevf_q_vector *q_vector);
0110 static void ixgbevf_free_all_rx_resources(struct ixgbevf_adapter *adapter);
0111 static bool ixgbevf_can_reuse_rx_page(struct ixgbevf_rx_buffer *rx_buffer);
0112 static void ixgbevf_reuse_rx_page(struct ixgbevf_ring *rx_ring,
0113                   struct ixgbevf_rx_buffer *old_buff);
0114
0115 static void ixgbevf_remove_adapter(struct ixgbe_hw *hw)
0116 {
0117     struct ixgbevf_adapter *adapter = hw->back;
0118
0119     if (!hw->hw_addr)
0120         return;
0121     hw->hw_addr = NULL;
0122     dev_err(&adapter->pdev->dev, "Adapter removed\n");
0123     if (test_bit(__IXGBEVF_SERVICE_INITED, &adapter->state))
0124         ixgbevf_service_event_schedule(adapter);
0125 }
0126
0127 static void ixgbevf_check_remove(struct ixgbe_hw *hw, u32 reg)
0128 {
0129     u32 value;
0130
0131     /* The following check not only optimizes a bit by not
0132      * performing a read on the status register when the
0133      * register just read was a status register read that
0134      * returned IXGBE_FAILED_READ_REG. It also blocks any
0135      * potential recursion.
0136      */
0137     if (reg == IXGBE_VFSTATUS) {
0138         ixgbevf_remove_adapter(hw);
0139         return;
0140     }
0141     value = ixgbevf_read_reg(hw, IXGBE_VFSTATUS);
0142     if (value == IXGBE_FAILED_READ_REG)
0143         ixgbevf_remove_adapter(hw);
0144 }
0145
0146 u32 ixgbevf_read_reg(struct ixgbe_hw *hw, u32 reg)
0147 {
0148     u8 __iomem *reg_addr = READ_ONCE(hw->hw_addr);
0149     u32 value;
0150
0151     if (IXGBE_REMOVED(reg_addr))
0152         return IXGBE_FAILED_READ_REG;
0153     value = readl(reg_addr + reg);
0154     if (unlikely(value == IXGBE_FAILED_READ_REG))
0155         ixgbevf_check_remove(hw, reg);
0156     return value;
0157 }
0158
0159 /**
0160  * ixgbevf_set_ivar - set IVAR registers - maps interrupt causes to vectors
0161  * @adapter: pointer to adapter struct
0162  * @direction: 0 for Rx, 1 for Tx, -1 for other causes
0163  * @queue: queue to map the corresponding interrupt to
0164  * @msix_vector: the vector to map to the corresponding queue
0165  **/
0166 static void ixgbevf_set_ivar(struct ixgbevf_adapter *adapter, s8 direction,
0167                  u8 queue, u8 msix_vector)
0168 {
0169     u32 ivar, index;
0170     struct ixgbe_hw *hw = &adapter->hw;
0171
0172     if (direction == -1) {
0173         /* other causes */
0174         msix_vector |= IXGBE_IVAR_ALLOC_VAL;
0175         ivar = IXGBE_READ_REG(hw, IXGBE_VTIVAR_MISC);
0176         ivar &= ~0xFF;
0177         ivar |= msix_vector;
0178         IXGBE_WRITE_REG(hw, IXGBE_VTIVAR_MISC, ivar);
0179     } else {
0180         /* Tx or Rx causes */
0181         msix_vector |= IXGBE_IVAR_ALLOC_VAL;
0182         index = ((16 * (queue & 1)) + (8 * direction));
0183         ivar = IXGBE_READ_REG(hw, IXGBE_VTIVAR(queue >> 1));
0184         ivar &= ~(0xFF << index);
0185         ivar |= (msix_vector << index);
0186         IXGBE_WRITE_REG(hw, IXGBE_VTIVAR(queue >> 1), ivar);
0187     }
0188 }
0189
0190 static u64 ixgbevf_get_tx_completed(struct ixgbevf_ring *ring)
0191 {
0192     return ring->stats.packets;
0193 }
0194
0195 static u32 ixgbevf_get_tx_pending(struct ixgbevf_ring *ring)
0196 {
0197     struct ixgbevf_adapter *adapter = netdev_priv(ring->netdev);
0198     struct ixgbe_hw *hw = &adapter->hw;
0199
0200     u32 head = IXGBE_READ_REG(hw, IXGBE_VFTDH(ring->reg_idx));
0201     u32 tail = IXGBE_READ_REG(hw, IXGBE_VFTDT(ring->reg_idx));
0202
0203     if (head != tail)
0204         return (head < tail) ?
0205             tail - head : (tail + ring->count - head);
0206
0207     return 0;
0208 }
0209
0210 static inline bool ixgbevf_check_tx_hang(struct ixgbevf_ring *tx_ring)
0211 {
0212     u32 tx_done = ixgbevf_get_tx_completed(tx_ring);
0213     u32 tx_done_old = tx_ring->tx_stats.tx_done_old;
0214     u32 tx_pending = ixgbevf_get_tx_pending(tx_ring);
0215
0216     clear_check_for_tx_hang(tx_ring);
0217
0218     /* Check for a hung queue, but be thorough. This verifies
0219      * that a transmit has been completed since the previous
0220      * check AND there is at least one packet pending. The
0221      * ARMED bit is set to indicate a potential hang.
0222      */
0223     if ((tx_done_old == tx_done) && tx_pending) {
0224         /* make sure it is true for two checks in a row */
0225         return test_and_set_bit(__IXGBEVF_HANG_CHECK_ARMED,
0226                     &tx_ring->state);
0227     }
0228     /* reset the countdown */
0229     clear_bit(__IXGBEVF_HANG_CHECK_ARMED, &tx_ring->state);
0230
0231     /* update completed stats and continue */
0232     tx_ring->tx_stats.tx_done_old = tx_done;
0233
0234     return false;
0235 }
0236
0237 static void ixgbevf_tx_timeout_reset(struct ixgbevf_adapter *adapter)
0238 {
0239     /* Do the reset outside of interrupt context */
0240     if (!test_bit(__IXGBEVF_DOWN, &adapter->state)) {
0241         set_bit(__IXGBEVF_RESET_REQUESTED, &adapter->state);
0242         ixgbevf_service_event_schedule(adapter);
0243     }
0244 }
0245
0246 /**
0247  * ixgbevf_tx_timeout - Respond to a Tx Hang
0248  * @netdev: network interface device structure
0249  * @txqueue: transmit queue hanging (unused)
0250  **/
0251 static void ixgbevf_tx_timeout(struct net_device *netdev, unsigned int __always_unused txqueue)
0252 {
0253     struct ixgbevf_adapter *adapter = netdev_priv(netdev);
0254
0255     ixgbevf_tx_timeout_reset(adapter);
0256 }
0257
0258 /**
0259  * ixgbevf_clean_tx_irq - Reclaim resources after transmit completes
0260  * @q_vector: board private structure
0261  * @tx_ring: tx ring to clean
0262  * @napi_budget: Used to determine if we are in netpoll
0263  **/
0264 static bool ixgbevf_clean_tx_irq(struct ixgbevf_q_vector *q_vector,
0265                  struct ixgbevf_ring *tx_ring, int napi_budget)
0266 {
0267     struct ixgbevf_adapter *adapter = q_vector->adapter;
0268     struct ixgbevf_tx_buffer *tx_buffer;
0269     union ixgbe_adv_tx_desc *tx_desc;
0270     unsigned int total_bytes = 0, total_packets = 0, total_ipsec = 0;
0271     unsigned int budget = tx_ring->count / 2;
0272     unsigned int i = tx_ring->next_to_clean;
0273
0274     if (test_bit(__IXGBEVF_DOWN, &adapter->state))
0275         return true;
0276
0277     tx_buffer = &tx_ring->tx_buffer_info[i];
0278     tx_desc = IXGBEVF_TX_DESC(tx_ring, i);
0279     i -= tx_ring->count;
0280
0281     do {
0282         union ixgbe_adv_tx_desc *eop_desc = tx_buffer->next_to_watch;
0283
0284         /* if next_to_watch is not set then there is no work pending */
0285         if (!eop_desc)
0286             break;
0287
0288         /* prevent any other reads prior to eop_desc */
0289         smp_rmb();
0290
0291         /* if DD is not set pending work has not been completed */
0292         if (!(eop_desc->wb.status & cpu_to_le32(IXGBE_TXD_STAT_DD)))
0293             break;
0294
0295         /* clear next_to_watch to prevent false hangs */
0296         tx_buffer->next_to_watch = NULL;
0297
0298         /* update the statistics for this packet */
0299         total_bytes += tx_buffer->bytecount;
0300         total_packets += tx_buffer->gso_segs;
0301         if (tx_buffer->tx_flags & IXGBE_TX_FLAGS_IPSEC)
0302             total_ipsec++;
0303
0304         /* free the skb */
0305         if (ring_is_xdp(tx_ring))
0306             page_frag_free(tx_buffer->data);
0307         else
0308             napi_consume_skb(tx_buffer->skb, napi_budget);
0309
0310         /* unmap skb header data */
0311         dma_unmap_single(tx_ring->dev,
0312                  dma_unmap_addr(tx_buffer, dma),
0313                  dma_unmap_len(tx_buffer, len),
0314                  DMA_TO_DEVICE);
0315
0316         /* clear tx_buffer data */
0317         dma_unmap_len_set(tx_buffer, len, 0);
0318
0319         /* unmap remaining buffers */
0320         while (tx_desc != eop_desc) {
0321             tx_buffer++;
0322             tx_desc++;
0323             i++;
0324             if (unlikely(!i)) {
0325                 i -= tx_ring->count;
0326                 tx_buffer = tx_ring->tx_buffer_info;
0327                 tx_desc = IXGBEVF_TX_DESC(tx_ring, 0);
0328             }
0329
0330             /* unmap any remaining paged data */
0331             if (dma_unmap_len(tx_buffer, len)) {
0332                 dma_unmap_page(tx_ring->dev,
0333                            dma_unmap_addr(tx_buffer, dma),
0334                            dma_unmap_len(tx_buffer, len),
0335                            DMA_TO_DEVICE);
0336                 dma_unmap_len_set(tx_buffer, len, 0);
0337             }
0338         }
0339
0340         /* move us one more past the eop_desc for start of next pkt */
0341         tx_buffer++;
0342         tx_desc++;
0343         i++;
0344         if (unlikely(!i)) {
0345             i -= tx_ring->count;
0346             tx_buffer = tx_ring->tx_buffer_info;
0347             tx_desc = IXGBEVF_TX_DESC(tx_ring, 0);
0348         }
0349
0350         /* issue prefetch for next Tx descriptor */
0351         prefetch(tx_desc);
0352
0353         /* update budget accounting */
0354         budget--;
0355     } while (likely(budget));
0356
0357     i += tx_ring->count;
0358     tx_ring->next_to_clean = i;
0359     u64_stats_update_begin(&tx_ring->syncp);
0360     tx_ring->stats.bytes += total_bytes;
0361     tx_ring->stats.packets += total_packets;
0362     u64_stats_update_end(&tx_ring->syncp);
0363     q_vector->tx.total_bytes += total_bytes;
0364     q_vector->tx.total_packets += total_packets;
0365     adapter->tx_ipsec += total_ipsec;
0366
0367     if (check_for_tx_hang(tx_ring) && ixgbevf_check_tx_hang(tx_ring)) {
0368         struct ixgbe_hw *hw = &adapter->hw;
0369         union ixgbe_adv_tx_desc *eop_desc;
0370
0371         eop_desc = tx_ring->tx_buffer_info[i].next_to_watch;
0372
0373         pr_err("Detected Tx Unit Hang%s\n"
0374                "  Tx Queue             <%d>\n"
0375                "  TDH, TDT             <%x>, <%x>\n"
0376                "  next_to_use          <%x>\n"
0377                "  next_to_clean        <%x>\n"
0378                "tx_buffer_info[next_to_clean]\n"
0379                "  next_to_watch        <%p>\n"
0380                "  eop_desc->wb.status  <%x>\n"
0381                "  time_stamp           <%lx>\n"
0382                "  jiffies              <%lx>\n",
0383                ring_is_xdp(tx_ring) ? " XDP" : "",
0384                tx_ring->queue_index,
0385                IXGBE_READ_REG(hw, IXGBE_VFTDH(tx_ring->reg_idx)),
0386                IXGBE_READ_REG(hw, IXGBE_VFTDT(tx_ring->reg_idx)),
0387                tx_ring->next_to_use, i,
0388                eop_desc, (eop_desc ? eop_desc->wb.status : 0),
0389                tx_ring->tx_buffer_info[i].time_stamp, jiffies);
0390
0391         if (!ring_is_xdp(tx_ring))
0392             netif_stop_subqueue(tx_ring->netdev,
0393                         tx_ring->queue_index);
0394
0395         /* schedule immediate reset if we believe we hung */
0396         ixgbevf_tx_timeout_reset(adapter);
0397
0398         return true;
0399     }
0400
0401     if (ring_is_xdp(tx_ring))
0402         return !!budget;
0403
0404 #define TX_WAKE_THRESHOLD (DESC_NEEDED * 2)
0405     if (unlikely(total_packets && netif_carrier_ok(tx_ring->netdev) &&
0406              (ixgbevf_desc_unused(tx_ring) >= TX_WAKE_THRESHOLD))) {
0407         /* Make sure that anybody stopping the queue after this
0408          * sees the new next_to_clean.
0409          */
0410         smp_mb();
0411
0412         if (__netif_subqueue_stopped(tx_ring->netdev,
0413                          tx_ring->queue_index) &&
0414             !test_bit(__IXGBEVF_DOWN, &adapter->state)) {
0415             netif_wake_subqueue(tx_ring->netdev,
0416                         tx_ring->queue_index);
0417             ++tx_ring->tx_stats.restart_queue;
0418         }
0419     }
0420
0421     return !!budget;
0422 }
0423
0424 /**
0425  * ixgbevf_rx_skb - Helper function to determine proper Rx method
0426  * @q_vector: structure containing interrupt and ring information
0427  * @skb: packet to send up
0428  **/
0429 static void ixgbevf_rx_skb(struct ixgbevf_q_vector *q_vector,
0430                struct sk_buff *skb)
0431 {
0432     napi_gro_receive(&q_vector->napi, skb);
0433 }
0434
0435 #define IXGBE_RSS_L4_TYPES_MASK \
0436     ((1ul << IXGBE_RXDADV_RSSTYPE_IPV4_TCP) | \
0437      (1ul << IXGBE_RXDADV_RSSTYPE_IPV4_UDP) | \
0438      (1ul << IXGBE_RXDADV_RSSTYPE_IPV6_TCP) | \
0439      (1ul << IXGBE_RXDADV_RSSTYPE_IPV6_UDP))
0440
0441 static inline void ixgbevf_rx_hash(struct ixgbevf_ring *ring,
0442                    union ixgbe_adv_rx_desc *rx_desc,
0443                    struct sk_buff *skb)
0444 {
0445     u16 rss_type;
0446
0447     if (!(ring->netdev->features & NETIF_F_RXHASH))
0448         return;
0449
0450     rss_type = le16_to_cpu(rx_desc->wb.lower.lo_dword.hs_rss.pkt_info) &
0451            IXGBE_RXDADV_RSSTYPE_MASK;
0452
0453     if (!rss_type)
0454         return;
0455
0456     skb_set_hash(skb, le32_to_cpu(rx_desc->wb.lower.hi_dword.rss),
0457              (IXGBE_RSS_L4_TYPES_MASK & (1ul << rss_type)) ?
0458              PKT_HASH_TYPE_L4 : PKT_HASH_TYPE_L3);
0459 }
0460
0461 /**
0462  * ixgbevf_rx_checksum - indicate in skb if hw indicated a good cksum
0463  * @ring: structure containig ring specific data
0464  * @rx_desc: current Rx descriptor being processed
0465  * @skb: skb currently being received and modified
0466  **/
0467 static inline void ixgbevf_rx_checksum(struct ixgbevf_ring *ring,
0468                        union ixgbe_adv_rx_desc *rx_desc,
0469                        struct sk_buff *skb)
0470 {
0471     skb_checksum_none_assert(skb);
0472
0473     /* Rx csum disabled */
0474     if (!(ring->netdev->features & NETIF_F_RXCSUM))
0475         return;
0476
0477     /* if IP and error */
0478     if (ixgbevf_test_staterr(rx_desc, IXGBE_RXD_STAT_IPCS) &&
0479         ixgbevf_test_staterr(rx_desc, IXGBE_RXDADV_ERR_IPE)) {
0480         ring->rx_stats.csum_err++;
0481         return;
0482     }
0483
0484     if (!ixgbevf_test_staterr(rx_desc, IXGBE_RXD_STAT_L4CS))
0485         return;
0486
0487     if (ixgbevf_test_staterr(rx_desc, IXGBE_RXDADV_ERR_TCPE)) {
0488         ring->rx_stats.csum_err++;
0489         return;
0490     }
0491
0492     /* It must be a TCP or UDP packet with a valid checksum */
0493     skb->ip_summed = CHECKSUM_UNNECESSARY;
0494 }
0495
0496 /**
0497  * ixgbevf_process_skb_fields - Populate skb header fields from Rx descriptor
0498  * @rx_ring: rx descriptor ring packet is being transacted on
0499  * @rx_desc: pointer to the EOP Rx descriptor
0500  * @skb: pointer to current skb being populated
0501  *
0502  * This function checks the ring, descriptor, and packet information in
0503  * order to populate the checksum, VLAN, protocol, and other fields within
0504  * the skb.
0505  **/
0506 static void ixgbevf_process_skb_fields(struct ixgbevf_ring *rx_ring,
0507                        union ixgbe_adv_rx_desc *rx_desc,
0508                        struct sk_buff *skb)
0509 {
0510     ixgbevf_rx_hash(rx_ring, rx_desc, skb);
0511     ixgbevf_rx_checksum(rx_ring, rx_desc, skb);
0512
0513     if (ixgbevf_test_staterr(rx_desc, IXGBE_RXD_STAT_VP)) {
0514         u16 vid = le16_to_cpu(rx_desc->wb.upper.vlan);
0515         unsigned long *active_vlans = netdev_priv(rx_ring->netdev);
0516
0517         if (test_bit(vid & VLAN_VID_MASK, active_vlans))
0518             __vlan_hwaccel_put_tag(skb, htons(ETH_P_8021Q), vid);
0519     }
0520
0521     if (ixgbevf_test_staterr(rx_desc, IXGBE_RXDADV_STAT_SECP))
0522         ixgbevf_ipsec_rx(rx_ring, rx_desc, skb);
0523
0524     skb->protocol = eth_type_trans(skb, rx_ring->netdev);
0525 }
0526
0527 static
0528 struct ixgbevf_rx_buffer *ixgbevf_get_rx_buffer(struct ixgbevf_ring *rx_ring,
0529                         const unsigned int size)
0530 {
0531     struct ixgbevf_rx_buffer *rx_buffer;
0532
0533     rx_buffer = &rx_ring->rx_buffer_info[rx_ring->next_to_clean];
0534     prefetchw(rx_buffer->page);
0535
0536     /* we are reusing so sync this buffer for CPU use */
0537     dma_sync_single_range_for_cpu(rx_ring->dev,
0538                       rx_buffer->dma,
0539                       rx_buffer->page_offset,
0540                       size,
0541                       DMA_FROM_DEVICE);
0542
0543     rx_buffer->pagecnt_bias--;
0544
0545     return rx_buffer;
0546 }
0547
0548 static void ixgbevf_put_rx_buffer(struct ixgbevf_ring *rx_ring,
0549                   struct ixgbevf_rx_buffer *rx_buffer,
0550                   struct sk_buff *skb)
0551 {
0552     if (ixgbevf_can_reuse_rx_page(rx_buffer)) {
0553         /* hand second half of page back to the ring */
0554         ixgbevf_reuse_rx_page(rx_ring, rx_buffer);
0555     } else {
0556         if (IS_ERR(skb))
0557             /* We are not reusing the buffer so unmap it and free
0558              * any references we are holding to it
0559              */
0560             dma_unmap_page_attrs(rx_ring->dev, rx_buffer->dma,
0561                          ixgbevf_rx_pg_size(rx_ring),
0562                          DMA_FROM_DEVICE,
0563                          IXGBEVF_RX_DMA_ATTR);
0564         __page_frag_cache_drain(rx_buffer->page,
0565                     rx_buffer->pagecnt_bias);
0566     }
0567
0568     /* clear contents of rx_buffer */
0569     rx_buffer->page = NULL;
0570 }
0571
0572 /**
0573  * ixgbevf_is_non_eop - process handling of non-EOP buffers
0574  * @rx_ring: Rx ring being processed
0575  * @rx_desc: Rx descriptor for current buffer
0576  *
0577  * This function updates next to clean.  If the buffer is an EOP buffer
0578  * this function exits returning false, otherwise it will place the
0579  * sk_buff in the next buffer to be chained and return true indicating
0580  * that this is in fact a non-EOP buffer.
0581  **/
0582 static bool ixgbevf_is_non_eop(struct ixgbevf_ring *rx_ring,
0583                    union ixgbe_adv_rx_desc *rx_desc)
0584 {
0585     u32 ntc = rx_ring->next_to_clean + 1;
0586
0587     /* fetch, update, and store next to clean */
0588     ntc = (ntc < rx_ring->count) ? ntc : 0;
0589     rx_ring->next_to_clean = ntc;
0590
0591     prefetch(IXGBEVF_RX_DESC(rx_ring, ntc));
0592
0593     if (likely(ixgbevf_test_staterr(rx_desc, IXGBE_RXD_STAT_EOP)))
0594         return false;
0595
0596     return true;
0597 }
0598
0599 static inline unsigned int ixgbevf_rx_offset(struct ixgbevf_ring *rx_ring)
0600 {
0601     return ring_uses_build_skb(rx_ring) ? IXGBEVF_SKB_PAD : 0;
0602 }
0603
0604 static bool ixgbevf_alloc_mapped_page(struct ixgbevf_ring *rx_ring,
0605                       struct ixgbevf_rx_buffer *bi)
0606 {
0607     struct page *page = bi->page;
0608     dma_addr_t dma;
0609
0610     /* since we are recycling buffers we should seldom need to alloc */
0611     if (likely(page))
0612         return true;
0613
0614     /* alloc new page for storage */
0615     page = dev_alloc_pages(ixgbevf_rx_pg_order(rx_ring));
0616     if (unlikely(!page)) {
0617         rx_ring->rx_stats.alloc_rx_page_failed++;
0618         return false;
0619     }
0620
0621     /* map page for use */
0622     dma = dma_map_page_attrs(rx_ring->dev, page, 0,
0623                  ixgbevf_rx_pg_size(rx_ring),
0624                  DMA_FROM_DEVICE, IXGBEVF_RX_DMA_ATTR);
0625
0626     /* if mapping failed free memory back to system since
0627      * there isn't much point in holding memory we can't use
0628      */
0629     if (dma_mapping_error(rx_ring->dev, dma)) {
0630         __free_pages(page, ixgbevf_rx_pg_order(rx_ring));
0631
0632         rx_ring->rx_stats.alloc_rx_page_failed++;
0633         return false;
0634     }
0635
0636     bi->dma = dma;
0637     bi->page = page;
0638     bi->page_offset = ixgbevf_rx_offset(rx_ring);
0639     bi->pagecnt_bias = 1;
0640     rx_ring->rx_stats.alloc_rx_page++;
0641
0642     return true;
0643 }
0644
0645 /**
0646  * ixgbevf_alloc_rx_buffers - Replace used receive buffers; packet split
0647  * @rx_ring: rx descriptor ring (for a specific queue) to setup buffers on
0648  * @cleaned_count: number of buffers to replace
0649  **/
0650 static void ixgbevf_alloc_rx_buffers(struct ixgbevf_ring *rx_ring,
0651                      u16 cleaned_count)
0652 {
0653     union ixgbe_adv_rx_desc *rx_desc;
0654     struct ixgbevf_rx_buffer *bi;
0655     unsigned int i = rx_ring->next_to_use;
0656
0657     /* nothing to do or no valid netdev defined */
0658     if (!cleaned_count || !rx_ring->netdev)
0659         return;
0660
0661     rx_desc = IXGBEVF_RX_DESC(rx_ring, i);
0662     bi = &rx_ring->rx_buffer_info[i];
0663     i -= rx_ring->count;
0664
0665     do {
0666         if (!ixgbevf_alloc_mapped_page(rx_ring, bi))
0667             break;
0668
0669         /* sync the buffer for use by the device */
0670         dma_sync_single_range_for_device(rx_ring->dev, bi->dma,
0671                          bi->page_offset,
0672                          ixgbevf_rx_bufsz(rx_ring),
0673                          DMA_FROM_DEVICE);
0674
0675         /* Refresh the desc even if pkt_addr didn't change
0676          * because each write-back erases this info.
0677          */
0678         rx_desc->read.pkt_addr = cpu_to_le64(bi->dma + bi->page_offset);
0679
0680         rx_desc++;
0681         bi++;
0682         i++;
0683         if (unlikely(!i)) {
0684             rx_desc = IXGBEVF_RX_DESC(rx_ring, 0);
0685             bi = rx_ring->rx_buffer_info;
0686             i -= rx_ring->count;
0687         }
0688
0689         /* clear the length for the next_to_use descriptor */
0690         rx_desc->wb.upper.length = 0;
0691
0692         cleaned_count--;
0693     } while (cleaned_count);
0694
0695     i += rx_ring->count;
0696
0697     if (rx_ring->next_to_use != i) {
0698         /* record the next descriptor to use */
0699         rx_ring->next_to_use = i;
0700
0701         /* update next to alloc since we have filled the ring */
0702         rx_ring->next_to_alloc = i;
0703
0704         /* Force memory writes to complete before letting h/w
0705          * know there are new descriptors to fetch.  (Only
0706          * applicable for weak-ordered memory model archs,
0707          * such as IA-64).
0708          */
0709         wmb();
0710         ixgbevf_write_tail(rx_ring, i);
0711     }
0712 }
0713
0714 /**
0715  * ixgbevf_cleanup_headers - Correct corrupted or empty headers
0716  * @rx_ring: rx descriptor ring packet is being transacted on
0717  * @rx_desc: pointer to the EOP Rx descriptor
0718  * @skb: pointer to current skb being fixed
0719  *
0720  * Check for corrupted packet headers caused by senders on the local L2
0721  * embedded NIC switch not setting up their Tx Descriptors right.  These
0722  * should be very rare.
0723  *
0724  * Also address the case where we are pulling data in on pages only
0725  * and as such no data is present in the skb header.
0726  *
0727  * In addition if skb is not at least 60 bytes we need to pad it so that
0728  * it is large enough to qualify as a valid Ethernet frame.
0729  *
0730  * Returns true if an error was encountered and skb was freed.
0731  **/
0732 static bool ixgbevf_cleanup_headers(struct ixgbevf_ring *rx_ring,
0733                     union ixgbe_adv_rx_desc *rx_desc,
0734                     struct sk_buff *skb)
0735 {
0736     /* XDP packets use error pointer so abort at this point */
0737     if (IS_ERR(skb))
0738         return true;
0739
0740     /* verify that the packet does not have any known errors */
0741     if (unlikely(ixgbevf_test_staterr(rx_desc,
0742                       IXGBE_RXDADV_ERR_FRAME_ERR_MASK))) {
0743         struct net_device *netdev = rx_ring->netdev;
0744
0745         if (!(netdev->features & NETIF_F_RXALL)) {
0746             dev_kfree_skb_any(skb);
0747             return true;
0748         }
0749     }
0750
0751     /* if eth_skb_pad returns an error the skb was freed */
0752     if (eth_skb_pad(skb))
0753         return true;
0754
0755     return false;
0756 }
0757
0758 /**
0759  * ixgbevf_reuse_rx_page - page flip buffer and store it back on the ring
0760  * @rx_ring: rx descriptor ring to store buffers on
0761  * @old_buff: donor buffer to have page reused
0762  *
0763  * Synchronizes page for reuse by the adapter
0764  **/
0765 static void ixgbevf_reuse_rx_page(struct ixgbevf_ring *rx_ring,
0766                   struct ixgbevf_rx_buffer *old_buff)
0767 {
0768     struct ixgbevf_rx_buffer *new_buff;
0769     u16 nta = rx_ring->next_to_alloc;
0770
0771     new_buff = &rx_ring->rx_buffer_info[nta];
0772
0773     /* update, and store next to alloc */
0774     nta++;
0775     rx_ring->next_to_alloc = (nta < rx_ring->count) ? nta : 0;
0776
0777     /* transfer page from old buffer to new buffer */
0778     new_buff->page = old_buff->page;
0779     new_buff->dma = old_buff->dma;
0780     new_buff->page_offset = old_buff->page_offset;
0781     new_buff->pagecnt_bias = old_buff->pagecnt_bias;
0782 }
0783
0784 static bool ixgbevf_can_reuse_rx_page(struct ixgbevf_rx_buffer *rx_buffer)
0785 {
0786     unsigned int pagecnt_bias = rx_buffer->pagecnt_bias;
0787     struct page *page = rx_buffer->page;
0788
0789     /* avoid re-using remote and pfmemalloc pages */
0790     if (!dev_page_is_reusable(page))
0791         return false;
0792
0793 #if (PAGE_SIZE < 8192)
0794     /* if we are only owner of page we can reuse it */
0795     if (unlikely((page_ref_count(page) - pagecnt_bias) > 1))
0796         return false;
0797 #else
0798 #define IXGBEVF_LAST_OFFSET \
0799     (SKB_WITH_OVERHEAD(PAGE_SIZE) - IXGBEVF_RXBUFFER_2048)
0800
0801     if (rx_buffer->page_offset > IXGBEVF_LAST_OFFSET)
0802         return false;
0803
0804 #endif
0805
0806     /* If we have drained the page fragment pool we need to update
0807      * the pagecnt_bias and page count so that we fully restock the
0808      * number of references the driver holds.
0809      */
0810     if (unlikely(!pagecnt_bias)) {
0811         page_ref_add(page, USHRT_MAX);
0812         rx_buffer->pagecnt_bias = USHRT_MAX;
0813     }
0814
0815     return true;
0816 }
0817
0818 /**
0819  * ixgbevf_add_rx_frag - Add contents of Rx buffer to sk_buff
0820  * @rx_ring: rx descriptor ring to transact packets on
0821  * @rx_buffer: buffer containing page to add
0822  * @skb: sk_buff to place the data into
0823  * @size: size of buffer to be added
0824  *
0825  * This function will add the data contained in rx_buffer->page to the skb.
0826  **/
0827 static void ixgbevf_add_rx_frag(struct ixgbevf_ring *rx_ring,
0828                 struct ixgbevf_rx_buffer *rx_buffer,
0829                 struct sk_buff *skb,
0830                 unsigned int size)
0831 {
0832 #if (PAGE_SIZE < 8192)
0833     unsigned int truesize = ixgbevf_rx_pg_size(rx_ring) / 2;
0834 #else
0835     unsigned int truesize = ring_uses_build_skb(rx_ring) ?
0836                 SKB_DATA_ALIGN(IXGBEVF_SKB_PAD + size) :
0837                 SKB_DATA_ALIGN(size);
0838 #endif
0839     skb_add_rx_frag(skb, skb_shinfo(skb)->nr_frags, rx_buffer->page,
0840             rx_buffer->page_offset, size, truesize);
0841 #if (PAGE_SIZE < 8192)
0842     rx_buffer->page_offset ^= truesize;
0843 #else
0844     rx_buffer->page_offset += truesize;
0845 #endif
0846 }
0847
0848 static
0849 struct sk_buff *ixgbevf_construct_skb(struct ixgbevf_ring *rx_ring,
0850                       struct ixgbevf_rx_buffer *rx_buffer,
0851                       struct xdp_buff *xdp,
0852                       union ixgbe_adv_rx_desc *rx_desc)
0853 {
0854     unsigned int size = xdp->data_end - xdp->data;
0855 #if (PAGE_SIZE < 8192)
0856     unsigned int truesize = ixgbevf_rx_pg_size(rx_ring) / 2;
0857 #else
0858     unsigned int truesize = SKB_DATA_ALIGN(xdp->data_end -
0859                            xdp->data_hard_start);
0860 #endif
0861     unsigned int headlen;
0862     struct sk_buff *skb;
0863
0864     /* prefetch first cache line of first page */
0865     net_prefetch(xdp->data);
0866
0867     /* Note, we get here by enabling legacy-rx via:
0868      *
0869      *    ethtool --set-priv-flags <dev> legacy-rx on
0870      *
0871      * In this mode, we currently get 0 extra XDP headroom as
0872      * opposed to having legacy-rx off, where we process XDP
0873      * packets going to stack via ixgbevf_build_skb().
0874      *
0875      * For ixgbevf_construct_skb() mode it means that the
0876      * xdp->data_meta will always point to xdp->data, since
0877      * the helper cannot expand the head. Should this ever
0878      * changed in future for legacy-rx mode on, then lets also
0879      * add xdp->data_meta handling here.
0880      */
0881
0882     /* allocate a skb to store the frags */
0883     skb = napi_alloc_skb(&rx_ring->q_vector->napi, IXGBEVF_RX_HDR_SIZE);
0884     if (unlikely(!skb))
0885         return NULL;
0886
0887     /* Determine available headroom for copy */
0888     headlen = size;
0889     if (headlen > IXGBEVF_RX_HDR_SIZE)
0890         headlen = eth_get_headlen(skb->dev, xdp->data,
0891                       IXGBEVF_RX_HDR_SIZE);
0892
0893     /* align pull length to size of long to optimize memcpy performance */
0894     memcpy(__skb_put(skb, headlen), xdp->data,
0895            ALIGN(headlen, sizeof(long)));
0896
0897     /* update all of the pointers */
0898     size -= headlen;
0899     if (size) {
0900         skb_add_rx_frag(skb, 0, rx_buffer->page,
0901                 (xdp->data + headlen) -
0902                     page_address(rx_buffer->page),
0903                 size, truesize);
0904 #if (PAGE_SIZE < 8192)
0905         rx_buffer->page_offset ^= truesize;
0906 #else
0907         rx_buffer->page_offset += truesize;
0908 #endif
0909     } else {
0910         rx_buffer->pagecnt_bias++;
0911     }
0912
0913     return skb;
0914 }
0915
0916 static inline void ixgbevf_irq_enable_queues(struct ixgbevf_adapter *adapter,
0917                          u32 qmask)
0918 {
0919     struct ixgbe_hw *hw = &adapter->hw;
0920
0921     IXGBE_WRITE_REG(hw, IXGBE_VTEIMS, qmask);
0922 }
0923
0924 static struct sk_buff *ixgbevf_build_skb(struct ixgbevf_ring *rx_ring,
0925                      struct ixgbevf_rx_buffer *rx_buffer,
0926                      struct xdp_buff *xdp,
0927                      union ixgbe_adv_rx_desc *rx_desc)
0928 {
0929     unsigned int metasize = xdp->data - xdp->data_meta;
0930 #if (PAGE_SIZE < 8192)
0931     unsigned int truesize = ixgbevf_rx_pg_size(rx_ring) / 2;
0932 #else
0933     unsigned int truesize = SKB_DATA_ALIGN(sizeof(struct skb_shared_info)) +
0934                 SKB_DATA_ALIGN(xdp->data_end -
0935                            xdp->data_hard_start);
0936 #endif
0937     struct sk_buff *skb;
0938
0939     /* Prefetch first cache line of first page. If xdp->data_meta
0940      * is unused, this points to xdp->data, otherwise, we likely
0941      * have a consumer accessing first few bytes of meta data,
0942      * and then actual data.
0943      */
0944     net_prefetch(xdp->data_meta);
0945
0946     /* build an skb around the page buffer */
0947     skb = napi_build_skb(xdp->data_hard_start, truesize);
0948     if (unlikely(!skb))
0949         return NULL;
0950
0951     /* update pointers within the skb to store the data */
0952     skb_reserve(skb, xdp->data - xdp->data_hard_start);
0953     __skb_put(skb, xdp->data_end - xdp->data);
0954     if (metasize)
0955         skb_metadata_set(skb, metasize);
0956
0957     /* update buffer offset */
0958 #if (PAGE_SIZE < 8192)
0959     rx_buffer->page_offset ^= truesize;
0960 #else
0961     rx_buffer->page_offset += truesize;
0962 #endif
0963
0964     return skb;
0965 }
0966
0967 #define IXGBEVF_XDP_PASS 0
0968 #define IXGBEVF_XDP_CONSUMED 1
0969 #define IXGBEVF_XDP_TX 2
0970
0971 static int ixgbevf_xmit_xdp_ring(struct ixgbevf_ring *ring,
0972                  struct xdp_buff *xdp)
0973 {
0974     struct ixgbevf_tx_buffer *tx_buffer;
0975     union ixgbe_adv_tx_desc *tx_desc;
0976     u32 len, cmd_type;
0977     dma_addr_t dma;
0978     u16 i;
0979
0980     len = xdp->data_end - xdp->data;
0981
0982     if (unlikely(!ixgbevf_desc_unused(ring)))
0983         return IXGBEVF_XDP_CONSUMED;
0984
0985     dma = dma_map_single(ring->dev, xdp->data, len, DMA_TO_DEVICE);
0986     if (dma_mapping_error(ring->dev, dma))
0987         return IXGBEVF_XDP_CONSUMED;
0988
0989     /* record the location of the first descriptor for this packet */
0990     i = ring->next_to_use;
0991     tx_buffer = &ring->tx_buffer_info[i];
0992
0993     dma_unmap_len_set(tx_buffer, len, len);
0994     dma_unmap_addr_set(tx_buffer, dma, dma);
0995     tx_buffer->data = xdp->data;
0996     tx_buffer->bytecount = len;
0997     tx_buffer->gso_segs = 1;
0998     tx_buffer->protocol = 0;
0999
1000     /* Populate minimal context descriptor that will provide for the
1001      * fact that we are expected to process Ethernet frames.
1002      */
1003     if (!test_bit(__IXGBEVF_TX_XDP_RING_PRIMED, &ring->state)) {
1004         struct ixgbe_adv_tx_context_desc *context_desc;
1005
1006         set_bit(__IXGBEVF_TX_XDP_RING_PRIMED, &ring->state);
1007
1008         context_desc = IXGBEVF_TX_CTXTDESC(ring, 0);
1009         context_desc->vlan_macip_lens   =
1010             cpu_to_le32(ETH_HLEN << IXGBE_ADVTXD_MACLEN_SHIFT);
1011         context_desc->fceof_saidx   = 0;
1012         context_desc->type_tucmd_mlhl   =
1013             cpu_to_le32(IXGBE_TXD_CMD_DEXT |
1014                     IXGBE_ADVTXD_DTYP_CTXT);
1015         context_desc->mss_l4len_idx = 0;
1016
1017         i = 1;
1018     }
1019
1020     /* put descriptor type bits */
1021     cmd_type = IXGBE_ADVTXD_DTYP_DATA |
1022            IXGBE_ADVTXD_DCMD_DEXT |
1023            IXGBE_ADVTXD_DCMD_IFCS;
1024     cmd_type |= len | IXGBE_TXD_CMD;
1025
1026     tx_desc = IXGBEVF_TX_DESC(ring, i);
1027     tx_desc->read.buffer_addr = cpu_to_le64(dma);
1028
1029     tx_desc->read.cmd_type_len = cpu_to_le32(cmd_type);
1030     tx_desc->read.olinfo_status =
1031             cpu_to_le32((len << IXGBE_ADVTXD_PAYLEN_SHIFT) |
1032                     IXGBE_ADVTXD_CC);
1033
1034     /* Avoid any potential race with cleanup */
1035     smp_wmb();
1036
1037     /* set next_to_watch value indicating a packet is present */
1038     i++;
1039     if (i == ring->count)
1040         i = 0;
1041
1042     tx_buffer->next_to_watch = tx_desc;
1043     ring->next_to_use = i;
1044
1045     return IXGBEVF_XDP_TX;
1046 }
1047
1048 static struct sk_buff *ixgbevf_run_xdp(struct ixgbevf_adapter *adapter,
1049                        struct ixgbevf_ring  *rx_ring,
1050                        struct xdp_buff *xdp)
1051 {
1052     int result = IXGBEVF_XDP_PASS;
1053     struct ixgbevf_ring *xdp_ring;
1054     struct bpf_prog *xdp_prog;
1055     u32 act;
1056
1057     xdp_prog = READ_ONCE(rx_ring->xdp_prog);
1058
1059     if (!xdp_prog)
1060         goto xdp_out;
1061
1062     act = bpf_prog_run_xdp(xdp_prog, xdp);
1063     switch (act) {
1064     case XDP_PASS:
1065         break;
1066     case XDP_TX:
1067         xdp_ring = adapter->xdp_ring[rx_ring->queue_index];
1068         result = ixgbevf_xmit_xdp_ring(xdp_ring, xdp);
1069         if (result == IXGBEVF_XDP_CONSUMED)
1070             goto out_failure;
1071         break;
1072     default:
1073         bpf_warn_invalid_xdp_action(rx_ring->netdev, xdp_prog, act);
1074         fallthrough;
1075     case XDP_ABORTED:
1076 out_failure:
1077         trace_xdp_exception(rx_ring->netdev, xdp_prog, act);
1078         fallthrough; /* handle aborts by dropping packet */
1079     case XDP_DROP:
1080         result = IXGBEVF_XDP_CONSUMED;
1081         break;
1082     }
1083 xdp_out:
1084     return ERR_PTR(-result);
1085 }
1086
1087 static unsigned int ixgbevf_rx_frame_truesize(struct ixgbevf_ring *rx_ring,
1088                           unsigned int size)
1089 {
1090     unsigned int truesize;
1091
1092 #if (PAGE_SIZE < 8192)
1093     truesize = ixgbevf_rx_pg_size(rx_ring) / 2; /* Must be power-of-2 */
1094 #else
1095     truesize = ring_uses_build_skb(rx_ring) ?
1096         SKB_DATA_ALIGN(IXGBEVF_SKB_PAD + size) +
1097         SKB_DATA_ALIGN(sizeof(struct skb_shared_info)) :
1098         SKB_DATA_ALIGN(size);
1099 #endif
1100     return truesize;
1101 }
1102
1103 static void ixgbevf_rx_buffer_flip(struct ixgbevf_ring *rx_ring,
1104                    struct ixgbevf_rx_buffer *rx_buffer,
1105                    unsigned int size)
1106 {
1107     unsigned int truesize = ixgbevf_rx_frame_truesize(rx_ring, size);
1108
1109 #if (PAGE_SIZE < 8192)
1110     rx_buffer->page_offset ^= truesize;
1111 #else
1112     rx_buffer->page_offset += truesize;
1113 #endif
1114 }
1115
1116 static int ixgbevf_clean_rx_irq(struct ixgbevf_q_vector *q_vector,
1117                 struct ixgbevf_ring *rx_ring,
1118                 int budget)
1119 {
1120     unsigned int total_rx_bytes = 0, total_rx_packets = 0, frame_sz = 0;
1121     struct ixgbevf_adapter *adapter = q_vector->adapter;
1122     u16 cleaned_count = ixgbevf_desc_unused(rx_ring);
1123     struct sk_buff *skb = rx_ring->skb;
1124     bool xdp_xmit = false;
1125     struct xdp_buff xdp;
1126
1127     /* Frame size depend on rx_ring setup when PAGE_SIZE=4K */
1128 #if (PAGE_SIZE < 8192)
1129     frame_sz = ixgbevf_rx_frame_truesize(rx_ring, 0);
1130 #endif
1131     xdp_init_buff(&xdp, frame_sz, &rx_ring->xdp_rxq);
1132
1133     while (likely(total_rx_packets < budget)) {
1134         struct ixgbevf_rx_buffer *rx_buffer;
1135         union ixgbe_adv_rx_desc *rx_desc;
1136         unsigned int size;
1137
1138         /* return some buffers to hardware, one at a time is too slow */
1139         if (cleaned_count >= IXGBEVF_RX_BUFFER_WRITE) {
1140             ixgbevf_alloc_rx_buffers(rx_ring, cleaned_count);
1141             cleaned_count = 0;
1142         }
1143
1144         rx_desc = IXGBEVF_RX_DESC(rx_ring, rx_ring->next_to_clean);
1145         size = le16_to_cpu(rx_desc->wb.upper.length);
1146         if (!size)
1147             break;
1148
1149         /* This memory barrier is needed to keep us from reading
1150          * any other fields out of the rx_desc until we know the
1151          * RXD_STAT_DD bit is set
1152          */
1153         rmb();
1154
1155         rx_buffer = ixgbevf_get_rx_buffer(rx_ring, size);
1156
1157         /* retrieve a buffer from the ring */
1158         if (!skb) {
1159             unsigned int offset = ixgbevf_rx_offset(rx_ring);
1160             unsigned char *hard_start;
1161
1162             hard_start = page_address(rx_buffer->page) +
1163                      rx_buffer->page_offset - offset;
1164             xdp_prepare_buff(&xdp, hard_start, offset, size, true);
1165 #if (PAGE_SIZE > 4096)
1166             /* At larger PAGE_SIZE, frame_sz depend on len size */
1167             xdp.frame_sz = ixgbevf_rx_frame_truesize(rx_ring, size);
1168 #endif
1169             skb = ixgbevf_run_xdp(adapter, rx_ring, &xdp);
1170         }
1171
1172         if (IS_ERR(skb)) {
1173             if (PTR_ERR(skb) == -IXGBEVF_XDP_TX) {
1174                 xdp_xmit = true;
1175                 ixgbevf_rx_buffer_flip(rx_ring, rx_buffer,
1176                                size);
1177             } else {
1178                 rx_buffer->pagecnt_bias++;
1179             }
1180             total_rx_packets++;
1181             total_rx_bytes += size;
1182         } else if (skb) {
1183             ixgbevf_add_rx_frag(rx_ring, rx_buffer, skb, size);
1184         } else if (ring_uses_build_skb(rx_ring)) {
1185             skb = ixgbevf_build_skb(rx_ring, rx_buffer,
1186                         &xdp, rx_desc);
1187         } else {
1188             skb = ixgbevf_construct_skb(rx_ring, rx_buffer,
1189                             &xdp, rx_desc);
1190         }
1191
1192         /* exit if we failed to retrieve a buffer */
1193         if (!skb) {
1194             rx_ring->rx_stats.alloc_rx_buff_failed++;
1195             rx_buffer->pagecnt_bias++;
1196             break;
1197         }
1198
1199         ixgbevf_put_rx_buffer(rx_ring, rx_buffer, skb);
1200         cleaned_count++;
1201
1202         /* fetch next buffer in frame if non-eop */
1203         if (ixgbevf_is_non_eop(rx_ring, rx_desc))
1204             continue;
1205
1206         /* verify the packet layout is correct */
1207         if (ixgbevf_cleanup_headers(rx_ring, rx_desc, skb)) {
1208             skb = NULL;
1209             continue;
1210         }
1211
1212         /* probably a little skewed due to removing CRC */
1213         total_rx_bytes += skb->len;
1214
1215         /* Workaround hardware that can't do proper VEPA multicast
1216          * source pruning.
1217          */
1218         if ((skb->pkt_type == PACKET_BROADCAST ||
1219              skb->pkt_type == PACKET_MULTICAST) &&
1220             ether_addr_equal(rx_ring->netdev->dev_addr,
1221                      eth_hdr(skb)->h_source)) {
1222             dev_kfree_skb_irq(skb);
1223             continue;
1224         }
1225
1226         /* populate checksum, VLAN, and protocol */
1227         ixgbevf_process_skb_fields(rx_ring, rx_desc, skb);
1228
1229         ixgbevf_rx_skb(q_vector, skb);
1230
1231         /* reset skb pointer */
1232         skb = NULL;
1233
1234         /* update budget accounting */
1235         total_rx_packets++;
1236     }
1237
1238     /* place incomplete frames back on ring for completion */
1239     rx_ring->skb = skb;
1240
1241     if (xdp_xmit) {
1242         struct ixgbevf_ring *xdp_ring =
1243             adapter->xdp_ring[rx_ring->queue_index];
1244
1245         /* Force memory writes to complete before letting h/w
1246          * know there are new descriptors to fetch.
1247          */
1248         wmb();
1249         ixgbevf_write_tail(xdp_ring, xdp_ring->next_to_use);
1250     }
1251
1252     u64_stats_update_begin(&rx_ring->syncp);
1253     rx_ring->stats.packets += total_rx_packets;
1254     rx_ring->stats.bytes += total_rx_bytes;
1255     u64_stats_update_end(&rx_ring->syncp);
1256     q_vector->rx.total_packets += total_rx_packets;
1257     q_vector->rx.total_bytes += total_rx_bytes;
1258
1259     return total_rx_packets;
1260 }
1261
1262 /**
1263  * ixgbevf_poll - NAPI polling calback
1264  * @napi: napi struct with our devices info in it
1265  * @budget: amount of work driver is allowed to do this pass, in packets
1266  *
1267  * This function will clean more than one or more rings associated with a
1268  * q_vector.
1269  **/
1270 static int ixgbevf_poll(struct napi_struct *napi, int budget)
1271 {
1272     struct ixgbevf_q_vector *q_vector =
1273         container_of(napi, struct ixgbevf_q_vector, napi);
1274     struct ixgbevf_adapter *adapter = q_vector->adapter;
1275     struct ixgbevf_ring *ring;
1276     int per_ring_budget, work_done = 0;
1277     bool clean_complete = true;
1278
1279     ixgbevf_for_each_ring(ring, q_vector->tx) {
1280         if (!ixgbevf_clean_tx_irq(q_vector, ring, budget))
1281             clean_complete = false;
1282     }
1283
1284     if (budget <= 0)
1285         return budget;
1286
1287     /* attempt to distribute budget to each queue fairly, but don't allow
1288      * the budget to go below 1 because we'll exit polling
1289      */
1290     if (q_vector->rx.count > 1)
1291         per_ring_budget = max(budget/q_vector->rx.count, 1);
1292     else
1293         per_ring_budget = budget;
1294
1295     ixgbevf_for_each_ring(ring, q_vector->rx) {
1296         int cleaned = ixgbevf_clean_rx_irq(q_vector, ring,
1297                            per_ring_budget);
1298         work_done += cleaned;
1299         if (cleaned >= per_ring_budget)
1300             clean_complete = false;
1301     }
1302
1303     /* If all work not completed, return budget and keep polling */
1304     if (!clean_complete)
1305         return budget;
1306
1307     /* Exit the polling mode, but don't re-enable interrupts if stack might
1308      * poll us due to busy-polling
1309      */
1310     if (likely(napi_complete_done(napi, work_done))) {
1311         if (adapter->rx_itr_setting == 1)
1312             ixgbevf_set_itr(q_vector);
1313         if (!test_bit(__IXGBEVF_DOWN, &adapter->state) &&
1314             !test_bit(__IXGBEVF_REMOVING, &adapter->state))
1315             ixgbevf_irq_enable_queues(adapter,
1316                           BIT(q_vector->v_idx));
1317     }
1318
1319     return min(work_done, budget - 1);
1320 }
1321
1322 /**
1323  * ixgbevf_write_eitr - write VTEITR register in hardware specific way
1324  * @q_vector: structure containing interrupt and ring information
1325  **/
1326 void ixgbevf_write_eitr(struct ixgbevf_q_vector *q_vector)
1327 {
1328     struct ixgbevf_adapter *adapter = q_vector->adapter;
1329     struct ixgbe_hw *hw = &adapter->hw;
1330     int v_idx = q_vector->v_idx;
1331     u32 itr_reg = q_vector->itr & IXGBE_MAX_EITR;
1332
1333     /* set the WDIS bit to not clear the timer bits and cause an
1334      * immediate assertion of the interrupt
1335      */
1336     itr_reg |= IXGBE_EITR_CNT_WDIS;
1337
1338     IXGBE_WRITE_REG(hw, IXGBE_VTEITR(v_idx), itr_reg);
1339 }
1340
1341 /**
1342  * ixgbevf_configure_msix - Configure MSI-X hardware
1343  * @adapter: board private structure
1344  *
1345  * ixgbevf_configure_msix sets up the hardware to properly generate MSI-X
1346  * interrupts.
1347  **/
1348 static void ixgbevf_configure_msix(struct ixgbevf_adapter *adapter)
1349 {
1350     struct ixgbevf_q_vector *q_vector;
1351     int q_vectors, v_idx;
1352
1353     q_vectors = adapter->num_msix_vectors - NON_Q_VECTORS;
1354     adapter->eims_enable_mask = 0;
1355
1356     /* Populate the IVAR table and set the ITR values to the
1357      * corresponding register.
1358      */
1359     for (v_idx = 0; v_idx < q_vectors; v_idx++) {
1360         struct ixgbevf_ring *ring;
1361
1362         q_vector = adapter->q_vector[v_idx];
1363
1364         ixgbevf_for_each_ring(ring, q_vector->rx)
1365             ixgbevf_set_ivar(adapter, 0, ring->reg_idx, v_idx);
1366
1367         ixgbevf_for_each_ring(ring, q_vector->tx)
1368             ixgbevf_set_ivar(adapter, 1, ring->reg_idx, v_idx);
1369
1370         if (q_vector->tx.ring && !q_vector->rx.ring) {
1371             /* Tx only vector */
1372             if (adapter->tx_itr_setting == 1)
1373                 q_vector->itr = IXGBE_12K_ITR;
1374             else
1375                 q_vector->itr = adapter->tx_itr_setting;
1376         } else {
1377             /* Rx or Rx/Tx vector */
1378             if (adapter->rx_itr_setting == 1)
1379                 q_vector->itr = IXGBE_20K_ITR;
1380             else
1381                 q_vector->itr = adapter->rx_itr_setting;
1382         }
1383
1384         /* add q_vector eims value to global eims_enable_mask */
1385         adapter->eims_enable_mask |= BIT(v_idx);
1386
1387         ixgbevf_write_eitr(q_vector);
1388     }
1389
1390     ixgbevf_set_ivar(adapter, -1, 1, v_idx);
1391     /* setup eims_other and add value to global eims_enable_mask */
1392     adapter->eims_other = BIT(v_idx);
1393     adapter->eims_enable_mask |= adapter->eims_other;
1394 }
1395
1396 enum latency_range {
1397     lowest_latency = 0,
1398     low_latency = 1,
1399     bulk_latency = 2,
1400     latency_invalid = 255
1401 };
1402
1403 /**
1404  * ixgbevf_update_itr - update the dynamic ITR value based on statistics
1405  * @q_vector: structure containing interrupt and ring information
1406  * @ring_container: structure containing ring performance data
1407  *
1408  * Stores a new ITR value based on packets and byte
1409  * counts during the last interrupt.  The advantage of per interrupt
1410  * computation is faster updates and more accurate ITR for the current
1411  * traffic pattern.  Constants in this function were computed
1412  * based on theoretical maximum wire speed and thresholds were set based
1413  * on testing data as well as attempting to minimize response time
1414  * while increasing bulk throughput.
1415  **/
1416 static void ixgbevf_update_itr(struct ixgbevf_q_vector *q_vector,
1417                    struct ixgbevf_ring_container *ring_container)
1418 {
1419     int bytes = ring_container->total_bytes;
1420     int packets = ring_container->total_packets;
1421     u32 timepassed_us;
1422     u64 bytes_perint;
1423     u8 itr_setting = ring_container->itr;
1424
1425     if (packets == 0)
1426         return;
1427
1428     /* simple throttle rate management
1429      *    0-20MB/s lowest (100000 ints/s)
1430      *   20-100MB/s low   (20000 ints/s)
1431      *  100-1249MB/s bulk (12000 ints/s)
1432      */
1433     /* what was last interrupt timeslice? */
1434     timepassed_us = q_vector->itr >> 2;
1435     if (timepassed_us == 0)
1436         return;
1437
1438     bytes_perint = bytes / timepassed_us; /* bytes/usec */
1439
1440     switch (itr_setting) {
1441     case lowest_latency:
1442         if (bytes_perint > 10)
1443             itr_setting = low_latency;
1444         break;
1445     case low_latency:
1446         if (bytes_perint > 20)
1447             itr_setting = bulk_latency;
1448         else if (bytes_perint <= 10)
1449             itr_setting = lowest_latency;
1450         break;
1451     case bulk_latency:
1452         if (bytes_perint <= 20)
1453             itr_setting = low_latency;
1454         break;
1455     }
1456
1457     /* clear work counters since we have the values we need */
1458     ring_container->total_bytes = 0;
1459     ring_container->total_packets = 0;
1460
1461     /* write updated itr to ring container */
1462     ring_container->itr = itr_setting;
1463 }
1464
1465 static void ixgbevf_set_itr(struct ixgbevf_q_vector *q_vector)
1466 {
1467     u32 new_itr = q_vector->itr;
1468     u8 current_itr;
1469
1470     ixgbevf_update_itr(q_vector, &q_vector->tx);
1471     ixgbevf_update_itr(q_vector, &q_vector->rx);
1472
1473     current_itr = max(q_vector->rx.itr, q_vector->tx.itr);
1474
1475     switch (current_itr) {
1476     /* counts and packets in update_itr are dependent on these numbers */
1477     case lowest_latency:
1478         new_itr = IXGBE_100K_ITR;
1479         break;
1480     case low_latency:
1481         new_itr = IXGBE_20K_ITR;
1482         break;
1483     case bulk_latency:
1484         new_itr = IXGBE_12K_ITR;
1485         break;
1486     default:
1487         break;
1488     }
1489
1490     if (new_itr != q_vector->itr) {
1491         /* do an exponential smoothing */
1492         new_itr = (10 * new_itr * q_vector->itr) /
1493               ((9 * new_itr) + q_vector->itr);
1494
1495         /* save the algorithm value here */
1496         q_vector->itr = new_itr;
1497
1498         ixgbevf_write_eitr(q_vector);
1499     }
1500 }
1501
1502 static irqreturn_t ixgbevf_msix_other(int irq, void *data)
1503 {
1504     struct ixgbevf_adapter *adapter = data;
1505     struct ixgbe_hw *hw = &adapter->hw;
1506
1507     hw->mac.get_link_status = 1;
1508
1509     ixgbevf_service_event_schedule(adapter);
1510
1511     IXGBE_WRITE_REG(hw, IXGBE_VTEIMS, adapter->eims_other);
1512
1513     return IRQ_HANDLED;
1514 }
1515
1516 /**
1517  * ixgbevf_msix_clean_rings - single unshared vector rx clean (all queues)
1518  * @irq: unused
1519  * @data: pointer to our q_vector struct for this interrupt vector
1520  **/
1521 static irqreturn_t ixgbevf_msix_clean_rings(int irq, void *data)
1522 {
1523     struct ixgbevf_q_vector *q_vector = data;
1524
1525     /* EIAM disabled interrupts (on this vector) for us */
1526     if (q_vector->rx.ring || q_vector->tx.ring)
1527         napi_schedule_irqoff(&q_vector->napi);
1528
1529     return IRQ_HANDLED;
1530 }
1531
1532 /**
1533  * ixgbevf_request_msix_irqs - Initialize MSI-X interrupts
1534  * @adapter: board private structure
1535  *
1536  * ixgbevf_request_msix_irqs allocates MSI-X vectors and requests
1537  * interrupts from the kernel.
1538  **/
1539 static int ixgbevf_request_msix_irqs(struct ixgbevf_adapter *adapter)
1540 {
1541     struct net_device *netdev = adapter->netdev;
1542     int q_vectors = adapter->num_msix_vectors - NON_Q_VECTORS;
1543     unsigned int ri = 0, ti = 0;
1544     int vector, err;
1545
1546     for (vector = 0; vector < q_vectors; vector++) {
1547         struct ixgbevf_q_vector *q_vector = adapter->q_vector[vector];
1548         struct msix_entry *entry = &adapter->msix_entries[vector];
1549
1550         if (q_vector->tx.ring && q_vector->rx.ring) {
1551             snprintf(q_vector->name, sizeof(q_vector->name),
1552                  "%s-TxRx-%u", netdev->name, ri++);
1553             ti++;
1554         } else if (q_vector->rx.ring) {
1555             snprintf(q_vector->name, sizeof(q_vector->name),
1556                  "%s-rx-%u", netdev->name, ri++);
1557         } else if (q_vector->tx.ring) {
1558             snprintf(q_vector->name, sizeof(q_vector->name),
1559                  "%s-tx-%u", netdev->name, ti++);
1560         } else {
1561             /* skip this unused q_vector */
1562             continue;
1563         }
1564         err = request_irq(entry->vector, &ixgbevf_msix_clean_rings, 0,
1565                   q_vector->name, q_vector);
1566         if (err) {
1567             hw_dbg(&adapter->hw,
1568                    "request_irq failed for MSIX interrupt Error: %d\n",
1569                    err);
1570             goto free_queue_irqs;
1571         }
1572     }
1573
1574     err = request_irq(adapter->msix_entries[vector].vector,
1575               &ixgbevf_msix_other, 0, netdev->name, adapter);
1576     if (err) {
1577         hw_dbg(&adapter->hw, "request_irq for msix_other failed: %d\n",
1578                err);
1579         goto free_queue_irqs;
1580     }
1581
1582     return 0;
1583
1584 free_queue_irqs:
1585     while (vector) {
1586         vector--;
1587         free_irq(adapter->msix_entries[vector].vector,
1588              adapter->q_vector[vector]);
1589     }
1590     /* This failure is non-recoverable - it indicates the system is
1591      * out of MSIX vector resources and the VF driver cannot run
1592      * without them.  Set the number of msix vectors to zero
1593      * indicating that not enough can be allocated.  The error
1594      * will be returned to the user indicating device open failed.
1595      * Any further attempts to force the driver to open will also
1596      * fail.  The only way to recover is to unload the driver and
1597      * reload it again.  If the system has recovered some MSIX
1598      * vectors then it may succeed.
1599      */
1600     adapter->num_msix_vectors = 0;
1601     return err;
1602 }
1603
1604 /**
1605  * ixgbevf_request_irq - initialize interrupts
1606  * @adapter: board private structure
1607  *
1608  * Attempts to configure interrupts using the best available
1609  * capabilities of the hardware and kernel.
1610  **/
1611 static int ixgbevf_request_irq(struct ixgbevf_adapter *adapter)
1612 {
1613     int err = ixgbevf_request_msix_irqs(adapter);
1614
1615     if (err)
1616         hw_dbg(&adapter->hw, "request_irq failed, Error %d\n", err);
1617
1618     return err;
1619 }
1620
1621 static void ixgbevf_free_irq(struct ixgbevf_adapter *adapter)
1622 {
1623     int i, q_vectors;
1624
1625     if (!adapter->msix_entries)
1626         return;
1627
1628     q_vectors = adapter->num_msix_vectors;
1629     i = q_vectors - 1;
1630
1631     free_irq(adapter->msix_entries[i].vector, adapter);
1632     i--;
1633
1634     for (; i >= 0; i--) {
1635         /* free only the irqs that were actually requested */
1636         if (!adapter->q_vector[i]->rx.ring &&
1637             !adapter->q_vector[i]->tx.ring)
1638             continue;
1639
1640         free_irq(adapter->msix_entries[i].vector,
1641              adapter->q_vector[i]);
1642     }
1643 }
1644
1645 /**
1646  * ixgbevf_irq_disable - Mask off interrupt generation on the NIC
1647  * @adapter: board private structure
1648  **/
1649 static inline void ixgbevf_irq_disable(struct ixgbevf_adapter *adapter)
1650 {
1651     struct ixgbe_hw *hw = &adapter->hw;
1652     int i;
1653
1654     IXGBE_WRITE_REG(hw, IXGBE_VTEIAM, 0);
1655     IXGBE_WRITE_REG(hw, IXGBE_VTEIMC, ~0);
1656     IXGBE_WRITE_REG(hw, IXGBE_VTEIAC, 0);
1657
1658     IXGBE_WRITE_FLUSH(hw);
1659
1660     for (i = 0; i < adapter->num_msix_vectors; i++)
1661         synchronize_irq(adapter->msix_entries[i].vector);
1662 }
1663
1664 /**
1665  * ixgbevf_irq_enable - Enable default interrupt generation settings
1666  * @adapter: board private structure
1667  **/
1668 static inline void ixgbevf_irq_enable(struct ixgbevf_adapter *adapter)
1669 {
1670     struct ixgbe_hw *hw = &adapter->hw;
1671
1672     IXGBE_WRITE_REG(hw, IXGBE_VTEIAM, adapter->eims_enable_mask);
1673     IXGBE_WRITE_REG(hw, IXGBE_VTEIAC, adapter->eims_enable_mask);
1674     IXGBE_WRITE_REG(hw, IXGBE_VTEIMS, adapter->eims_enable_mask);
1675 }
1676
1677 /**
1678  * ixgbevf_configure_tx_ring - Configure 82599 VF Tx ring after Reset
1679  * @adapter: board private structure
1680  * @ring: structure containing ring specific data
1681  *
1682  * Configure the Tx descriptor ring after a reset.
1683  **/
1684 static void ixgbevf_configure_tx_ring(struct ixgbevf_adapter *adapter,
1685                       struct ixgbevf_ring *ring)
1686 {
1687     struct ixgbe_hw *hw = &adapter->hw;
1688     u64 tdba = ring->dma;
1689     int wait_loop = 10;
1690     u32 txdctl = IXGBE_TXDCTL_ENABLE;
1691     u8 reg_idx = ring->reg_idx;
1692
1693     /* disable queue to avoid issues while updating state */
1694     IXGBE_WRITE_REG(hw, IXGBE_VFTXDCTL(reg_idx), IXGBE_TXDCTL_SWFLSH);
1695     IXGBE_WRITE_FLUSH(hw);
1696
1697     IXGBE_WRITE_REG(hw, IXGBE_VFTDBAL(reg_idx), tdba & DMA_BIT_MASK(32));
1698     IXGBE_WRITE_REG(hw, IXGBE_VFTDBAH(reg_idx), tdba >> 32);
1699     IXGBE_WRITE_REG(hw, IXGBE_VFTDLEN(reg_idx),
1700             ring->count * sizeof(union ixgbe_adv_tx_desc));
1701
1702     /* disable head writeback */
1703     IXGBE_WRITE_REG(hw, IXGBE_VFTDWBAH(reg_idx), 0);
1704     IXGBE_WRITE_REG(hw, IXGBE_VFTDWBAL(reg_idx), 0);
1705
1706     /* enable relaxed ordering */
1707     IXGBE_WRITE_REG(hw, IXGBE_VFDCA_TXCTRL(reg_idx),
1708             (IXGBE_DCA_TXCTRL_DESC_RRO_EN |
1709              IXGBE_DCA_TXCTRL_DATA_RRO_EN));
1710
1711     /* reset head and tail pointers */
1712     IXGBE_WRITE_REG(hw, IXGBE_VFTDH(reg_idx), 0);
1713     IXGBE_WRITE_REG(hw, IXGBE_VFTDT(reg_idx), 0);
1714     ring->tail = adapter->io_addr + IXGBE_VFTDT(reg_idx);
1715
1716     /* reset ntu and ntc to place SW in sync with hardwdare */
1717     ring->next_to_clean = 0;
1718     ring->next_to_use = 0;
1719
1720     /* In order to avoid issues WTHRESH + PTHRESH should always be equal
1721      * to or less than the number of on chip descriptors, which is
1722      * currently 40.
1723      */
1724     txdctl |= (8 << 16);    /* WTHRESH = 8 */
1725
1726     /* Setting PTHRESH to 32 both improves performance */
1727     txdctl |= (1u << 8) |    /* HTHRESH = 1 */
1728            32;           /* PTHRESH = 32 */
1729
1730     /* reinitialize tx_buffer_info */
1731     memset(ring->tx_buffer_info, 0,
1732            sizeof(struct ixgbevf_tx_buffer) * ring->count);
1733
1734     clear_bit(__IXGBEVF_HANG_CHECK_ARMED, &ring->state);
1735     clear_bit(__IXGBEVF_TX_XDP_RING_PRIMED, &ring->state);
1736
1737     IXGBE_WRITE_REG(hw, IXGBE_VFTXDCTL(reg_idx), txdctl);
1738
1739     /* poll to verify queue is enabled */
1740     do {
1741         usleep_range(1000, 2000);
1742         txdctl = IXGBE_READ_REG(hw, IXGBE_VFTXDCTL(reg_idx));
1743     }  while (--wait_loop && !(txdctl & IXGBE_TXDCTL_ENABLE));
1744     if (!wait_loop)
1745         hw_dbg(hw, "Could not enable Tx Queue %d\n", reg_idx);
1746 }
1747
1748 /**
1749  * ixgbevf_configure_tx - Configure 82599 VF Transmit Unit after Reset
1750  * @adapter: board private structure
1751  *
1752  * Configure the Tx unit of the MAC after a reset.
1753  **/
1754 static void ixgbevf_configure_tx(struct ixgbevf_adapter *adapter)
1755 {
1756     u32 i;
1757
1758     /* Setup the HW Tx Head and Tail descriptor pointers */
1759     for (i = 0; i < adapter->num_tx_queues; i++)
1760         ixgbevf_configure_tx_ring(adapter, adapter->tx_ring[i]);
1761     for (i = 0; i < adapter->num_xdp_queues; i++)
1762         ixgbevf_configure_tx_ring(adapter, adapter->xdp_ring[i]);
1763 }
1764
1765 #define IXGBE_SRRCTL_BSIZEHDRSIZE_SHIFT 2
1766
1767 static void ixgbevf_configure_srrctl(struct ixgbevf_adapter *adapter,
1768                      struct ixgbevf_ring *ring, int index)
1769 {
1770     struct ixgbe_hw *hw = &adapter->hw;
1771     u32 srrctl;
1772
1773     srrctl = IXGBE_SRRCTL_DROP_EN;
1774
1775     srrctl |= IXGBEVF_RX_HDR_SIZE << IXGBE_SRRCTL_BSIZEHDRSIZE_SHIFT;
1776     if (ring_uses_large_buffer(ring))
1777         srrctl |= IXGBEVF_RXBUFFER_3072 >> IXGBE_SRRCTL_BSIZEPKT_SHIFT;
1778     else
1779         srrctl |= IXGBEVF_RXBUFFER_2048 >> IXGBE_SRRCTL_BSIZEPKT_SHIFT;
1780     srrctl |= IXGBE_SRRCTL_DESCTYPE_ADV_ONEBUF;
1781
1782     IXGBE_WRITE_REG(hw, IXGBE_VFSRRCTL(index), srrctl);
1783 }
1784
1785 static void ixgbevf_setup_psrtype(struct ixgbevf_adapter *adapter)
1786 {
1787     struct ixgbe_hw *hw = &adapter->hw;
1788
1789     /* PSRTYPE must be initialized in 82599 */
1790     u32 psrtype = IXGBE_PSRTYPE_TCPHDR | IXGBE_PSRTYPE_UDPHDR |
1791               IXGBE_PSRTYPE_IPV4HDR | IXGBE_PSRTYPE_IPV6HDR |
1792               IXGBE_PSRTYPE_L2HDR;
1793
1794     if (adapter->num_rx_queues > 1)
1795         psrtype |= BIT(29);
1796
1797     IXGBE_WRITE_REG(hw, IXGBE_VFPSRTYPE, psrtype);
1798 }
1799
1800 #define IXGBEVF_MAX_RX_DESC_POLL 10
1801 static void ixgbevf_disable_rx_queue(struct ixgbevf_adapter *adapter,
1802                      struct ixgbevf_ring *ring)
1803 {
1804     struct ixgbe_hw *hw = &adapter->hw;
1805     int wait_loop = IXGBEVF_MAX_RX_DESC_POLL;
1806     u32 rxdctl;
1807     u8 reg_idx = ring->reg_idx;
1808
1809     if (IXGBE_REMOVED(hw->hw_addr))
1810         return;
1811     rxdctl = IXGBE_READ_REG(hw, IXGBE_VFRXDCTL(reg_idx));
1812     rxdctl &= ~IXGBE_RXDCTL_ENABLE;
1813
1814     /* write value back with RXDCTL.ENABLE bit cleared */
1815     IXGBE_WRITE_REG(hw, IXGBE_VFRXDCTL(reg_idx), rxdctl);
1816
1817     /* the hardware may take up to 100us to really disable the Rx queue */
1818     do {
1819         udelay(10);
1820         rxdctl = IXGBE_READ_REG(hw, IXGBE_VFRXDCTL(reg_idx));
1821     } while (--wait_loop && (rxdctl & IXGBE_RXDCTL_ENABLE));
1822
1823     if (!wait_loop)
1824         pr_err("RXDCTL.ENABLE queue %d not cleared while polling\n",
1825                reg_idx);
1826 }
1827
1828 static void ixgbevf_rx_desc_queue_enable(struct ixgbevf_adapter *adapter,
1829                      struct ixgbevf_ring *ring)
1830 {
1831     struct ixgbe_hw *hw = &adapter->hw;
1832     int wait_loop = IXGBEVF_MAX_RX_DESC_POLL;
1833     u32 rxdctl;
1834     u8 reg_idx = ring->reg_idx;
1835
1836     if (IXGBE_REMOVED(hw->hw_addr))
1837         return;
1838     do {
1839         usleep_range(1000, 2000);
1840         rxdctl = IXGBE_READ_REG(hw, IXGBE_VFRXDCTL(reg_idx));
1841     } while (--wait_loop && !(rxdctl & IXGBE_RXDCTL_ENABLE));
1842
1843     if (!wait_loop)
1844         pr_err("RXDCTL.ENABLE queue %d not set while polling\n",
1845                reg_idx);
1846 }
1847
1848 /**
1849  * ixgbevf_init_rss_key - Initialize adapter RSS key
1850  * @adapter: device handle
1851  *
1852  * Allocates and initializes the RSS key if it is not allocated.
1853  **/
1854 static inline int ixgbevf_init_rss_key(struct ixgbevf_adapter *adapter)
1855 {
1856     u32 *rss_key;
1857
1858     if (!adapter->rss_key) {
1859         rss_key = kzalloc(IXGBEVF_RSS_HASH_KEY_SIZE, GFP_KERNEL);
1860         if (unlikely(!rss_key))
1861             return -ENOMEM;
1862
1863         netdev_rss_key_fill(rss_key, IXGBEVF_RSS_HASH_KEY_SIZE);
1864         adapter->rss_key = rss_key;
1865     }
1866
1867     return 0;
1868 }
1869
1870 static void ixgbevf_setup_vfmrqc(struct ixgbevf_adapter *adapter)
1871 {
1872     struct ixgbe_hw *hw = &adapter->hw;
1873     u32 vfmrqc = 0, vfreta = 0;
1874     u16 rss_i = adapter->num_rx_queues;
1875     u8 i, j;
1876
1877     /* Fill out hash function seeds */
1878     for (i = 0; i < IXGBEVF_VFRSSRK_REGS; i++)
1879         IXGBE_WRITE_REG(hw, IXGBE_VFRSSRK(i), *(adapter->rss_key + i));
1880
1881     for (i = 0, j = 0; i < IXGBEVF_X550_VFRETA_SIZE; i++, j++) {
1882         if (j == rss_i)
1883             j = 0;
1884
1885         adapter->rss_indir_tbl[i] = j;
1886
1887         vfreta |= j << (i & 0x3) * 8;
1888         if ((i & 3) == 3) {
1889             IXGBE_WRITE_REG(hw, IXGBE_VFRETA(i >> 2), vfreta);
1890             vfreta = 0;
1891         }
1892     }
1893
1894     /* Perform hash on these packet types */
1895     vfmrqc |= IXGBE_VFMRQC_RSS_FIELD_IPV4 |
1896         IXGBE_VFMRQC_RSS_FIELD_IPV4_TCP |
1897         IXGBE_VFMRQC_RSS_FIELD_IPV6 |
1898         IXGBE_VFMRQC_RSS_FIELD_IPV6_TCP;
1899
1900     vfmrqc |= IXGBE_VFMRQC_RSSEN;
1901
1902     IXGBE_WRITE_REG(hw, IXGBE_VFMRQC, vfmrqc);
1903 }
1904
1905 static void ixgbevf_configure_rx_ring(struct ixgbevf_adapter *adapter,
1906                       struct ixgbevf_ring *ring)
1907 {
1908     struct ixgbe_hw *hw = &adapter->hw;
1909     union ixgbe_adv_rx_desc *rx_desc;
1910     u64 rdba = ring->dma;
1911     u32 rxdctl;
1912     u8 reg_idx = ring->reg_idx;
1913
1914     /* disable queue to avoid issues while updating state */
1915     rxdctl = IXGBE_READ_REG(hw, IXGBE_VFRXDCTL(reg_idx));
1916     ixgbevf_disable_rx_queue(adapter, ring);
1917
1918     IXGBE_WRITE_REG(hw, IXGBE_VFRDBAL(reg_idx), rdba & DMA_BIT_MASK(32));
1919     IXGBE_WRITE_REG(hw, IXGBE_VFRDBAH(reg_idx), rdba >> 32);
1920     IXGBE_WRITE_REG(hw, IXGBE_VFRDLEN(reg_idx),
1921             ring->count * sizeof(union ixgbe_adv_rx_desc));
1922
1923 #ifndef CONFIG_SPARC
1924     /* enable relaxed ordering */
1925     IXGBE_WRITE_REG(hw, IXGBE_VFDCA_RXCTRL(reg_idx),
1926             IXGBE_DCA_RXCTRL_DESC_RRO_EN);
1927 #else
1928     IXGBE_WRITE_REG(hw, IXGBE_VFDCA_RXCTRL(reg_idx),
1929             IXGBE_DCA_RXCTRL_DESC_RRO_EN |
1930             IXGBE_DCA_RXCTRL_DATA_WRO_EN);
1931 #endif
1932
1933     /* reset head and tail pointers */
1934     IXGBE_WRITE_REG(hw, IXGBE_VFRDH(reg_idx), 0);
1935     IXGBE_WRITE_REG(hw, IXGBE_VFRDT(reg_idx), 0);
1936     ring->tail = adapter->io_addr + IXGBE_VFRDT(reg_idx);
1937
1938     /* initialize rx_buffer_info */
1939     memset(ring->rx_buffer_info, 0,
1940            sizeof(struct ixgbevf_rx_buffer) * ring->count);
1941
1942     /* initialize Rx descriptor 0 */
1943     rx_desc = IXGBEVF_RX_DESC(ring, 0);
1944     rx_desc->wb.upper.length = 0;
1945
1946     /* reset ntu and ntc to place SW in sync with hardwdare */
1947     ring->next_to_clean = 0;
1948     ring->next_to_use = 0;
1949     ring->next_to_alloc = 0;
1950
1951     ixgbevf_configure_srrctl(adapter, ring, reg_idx);
1952
1953     /* RXDCTL.RLPML does not work on 82599 */
1954     if (adapter->hw.mac.type != ixgbe_mac_82599_vf) {
1955         rxdctl &= ~(IXGBE_RXDCTL_RLPMLMASK |
1956                 IXGBE_RXDCTL_RLPML_EN);
1957
1958 #if (PAGE_SIZE < 8192)
1959         /* Limit the maximum frame size so we don't overrun the skb */
1960         if (ring_uses_build_skb(ring) &&
1961             !ring_uses_large_buffer(ring))
1962             rxdctl |= IXGBEVF_MAX_FRAME_BUILD_SKB |
1963                   IXGBE_RXDCTL_RLPML_EN;
1964 #endif
1965     }
1966
1967     rxdctl |= IXGBE_RXDCTL_ENABLE | IXGBE_RXDCTL_VME;
1968     IXGBE_WRITE_REG(hw, IXGBE_VFRXDCTL(reg_idx), rxdctl);
1969
1970     ixgbevf_rx_desc_queue_enable(adapter, ring);
1971     ixgbevf_alloc_rx_buffers(ring, ixgbevf_desc_unused(ring));
1972 }
1973
1974 static void ixgbevf_set_rx_buffer_len(struct ixgbevf_adapter *adapter,
1975                       struct ixgbevf_ring *rx_ring)
1976 {
1977     struct net_device *netdev = adapter->netdev;
1978     unsigned int max_frame = netdev->mtu + ETH_HLEN + ETH_FCS_LEN;
1979
1980     /* set build_skb and buffer size flags */
1981     clear_ring_build_skb_enabled(rx_ring);
1982     clear_ring_uses_large_buffer(rx_ring);
1983
1984     if (adapter->flags & IXGBEVF_FLAGS_LEGACY_RX)
1985         return;
1986
1987     if (PAGE_SIZE < 8192)
1988         if (max_frame > IXGBEVF_MAX_FRAME_BUILD_SKB)
1989             set_ring_uses_large_buffer(rx_ring);
1990
1991     /* 82599 can't rely on RXDCTL.RLPML to restrict the size of the frame */
1992     if (adapter->hw.mac.type == ixgbe_mac_82599_vf && !ring_uses_large_buffer(rx_ring))
1993         return;
1994
1995     set_ring_build_skb_enabled(rx_ring);
1996 }
1997
1998 /**
1999  * ixgbevf_configure_rx - Configure 82599 VF Receive Unit after Reset
2000  * @adapter: board private structure
2001  *
2002  * Configure the Rx unit of the MAC after a reset.
2003  **/
2004 static void ixgbevf_configure_rx(struct ixgbevf_adapter *adapter)
2005 {
2006     struct ixgbe_hw *hw = &adapter->hw;
2007     struct net_device *netdev = adapter->netdev;
2008     int i, ret;
2009
2010     ixgbevf_setup_psrtype(adapter);
2011     if (hw->mac.type >= ixgbe_mac_X550_vf)
2012         ixgbevf_setup_vfmrqc(adapter);
2013
2014     spin_lock_bh(&adapter->mbx_lock);
2015     /* notify the PF of our intent to use this size of frame */
2016     ret = hw->mac.ops.set_rlpml(hw, netdev->mtu + ETH_HLEN + ETH_FCS_LEN);
2017     spin_unlock_bh(&adapter->mbx_lock);
2018     if (ret)
2019         dev_err(&adapter->pdev->dev,
2020             "Failed to set MTU at %d\n", netdev->mtu);
2021
2022     /* Setup the HW Rx Head and Tail Descriptor Pointers and
2023      * the Base and Length of the Rx Descriptor Ring
2024      */
2025     for (i = 0; i < adapter->num_rx_queues; i++) {
2026         struct ixgbevf_ring *rx_ring = adapter->rx_ring[i];
2027
2028         ixgbevf_set_rx_buffer_len(adapter, rx_ring);
2029         ixgbevf_configure_rx_ring(adapter, rx_ring);
2030     }
2031 }
2032
2033 static int ixgbevf_vlan_rx_add_vid(struct net_device *netdev,
2034                    __be16 proto, u16 vid)
2035 {
2036     struct ixgbevf_adapter *adapter = netdev_priv(netdev);
2037     struct ixgbe_hw *hw = &adapter->hw;
2038     int err;
2039
2040     spin_lock_bh(&adapter->mbx_lock);
2041
2042     /* add VID to filter table */
2043     err = hw->mac.ops.set_vfta(hw, vid, 0, true);
2044
2045     spin_unlock_bh(&adapter->mbx_lock);
2046
2047     /* translate error return types so error makes sense */
2048     if (err == IXGBE_ERR_MBX)
2049         return -EIO;
2050
2051     if (err == IXGBE_ERR_INVALID_ARGUMENT)
2052         return -EACCES;
2053
2054     set_bit(vid, adapter->active_vlans);
2055
2056     return err;
2057 }
2058
2059 static int ixgbevf_vlan_rx_kill_vid(struct net_device *netdev,
2060                     __be16 proto, u16 vid)
2061 {
2062     struct ixgbevf_adapter *adapter = netdev_priv(netdev);
2063     struct ixgbe_hw *hw = &adapter->hw;
2064     int err;
2065
2066     spin_lock_bh(&adapter->mbx_lock);
2067
2068     /* remove VID from filter table */
2069     err = hw->mac.ops.set_vfta(hw, vid, 0, false);
2070
2071     spin_unlock_bh(&adapter->mbx_lock);
2072
2073     clear_bit(vid, adapter->active_vlans);
2074
2075     return err;
2076 }
2077
2078 static void ixgbevf_restore_vlan(struct ixgbevf_adapter *adapter)
2079 {
2080     u16 vid;
2081
2082     for_each_set_bit(vid, adapter->active_vlans, VLAN_N_VID)
2083         ixgbevf_vlan_rx_add_vid(adapter->netdev,
2084                     htons(ETH_P_8021Q), vid);
2085 }
2086
2087 static int ixgbevf_write_uc_addr_list(struct net_device *netdev)
2088 {
2089     struct ixgbevf_adapter *adapter = netdev_priv(netdev);
2090     struct ixgbe_hw *hw = &adapter->hw;
2091     int count = 0;
2092
2093     if (!netdev_uc_empty(netdev)) {
2094         struct netdev_hw_addr *ha;
2095
2096         netdev_for_each_uc_addr(ha, netdev) {
2097             hw->mac.ops.set_uc_addr(hw, ++count, ha->addr);
2098             udelay(200);
2099         }
2100     } else {
2101         /* If the list is empty then send message to PF driver to
2102          * clear all MAC VLANs on this VF.
2103          */
2104         hw->mac.ops.set_uc_addr(hw, 0, NULL);
2105     }
2106
2107     return count;
2108 }
2109
2110 /**
2111  * ixgbevf_set_rx_mode - Multicast and unicast set
2112  * @netdev: network interface device structure
2113  *
2114  * The set_rx_method entry point is called whenever the multicast address
2115  * list, unicast address list or the network interface flags are updated.
2116  * This routine is responsible for configuring the hardware for proper
2117  * multicast mode and configuring requested unicast filters.
2118  **/
2119 static void ixgbevf_set_rx_mode(struct net_device *netdev)
2120 {
2121     struct ixgbevf_adapter *adapter = netdev_priv(netdev);
2122     struct ixgbe_hw *hw = &adapter->hw;
2123     unsigned int flags = netdev->flags;
2124     int xcast_mode;
2125
2126     /* request the most inclusive mode we need */
2127     if (flags & IFF_PROMISC)
2128         xcast_mode = IXGBEVF_XCAST_MODE_PROMISC;
2129     else if (flags & IFF_ALLMULTI)
2130         xcast_mode = IXGBEVF_XCAST_MODE_ALLMULTI;
2131     else if (flags & (IFF_BROADCAST | IFF_MULTICAST))
2132         xcast_mode = IXGBEVF_XCAST_MODE_MULTI;
2133     else
2134         xcast_mode = IXGBEVF_XCAST_MODE_NONE;
2135
2136     spin_lock_bh(&adapter->mbx_lock);
2137
2138     hw->mac.ops.update_xcast_mode(hw, xcast_mode);
2139
2140     /* reprogram multicast list */
2141     hw->mac.ops.update_mc_addr_list(hw, netdev);
2142
2143     ixgbevf_write_uc_addr_list(netdev);
2144
2145     spin_unlock_bh(&adapter->mbx_lock);
2146 }
2147
2148 static void ixgbevf_napi_enable_all(struct ixgbevf_adapter *adapter)
2149 {
2150     int q_idx;
2151     struct ixgbevf_q_vector *q_vector;
2152     int q_vectors = adapter->num_msix_vectors - NON_Q_VECTORS;
2153
2154     for (q_idx = 0; q_idx < q_vectors; q_idx++) {
2155         q_vector = adapter->q_vector[q_idx];
2156         napi_enable(&q_vector->napi);
2157     }
2158 }
2159
2160 static void ixgbevf_napi_disable_all(struct ixgbevf_adapter *adapter)
2161 {
2162     int q_idx;
2163     struct ixgbevf_q_vector *q_vector;
2164     int q_vectors = adapter->num_msix_vectors - NON_Q_VECTORS;
2165
2166     for (q_idx = 0; q_idx < q_vectors; q_idx++) {
2167         q_vector = adapter->q_vector[q_idx];
2168         napi_disable(&q_vector->napi);
2169     }
2170 }
2171
2172 static int ixgbevf_configure_dcb(struct ixgbevf_adapter *adapter)
2173 {
2174     struct ixgbe_hw *hw = &adapter->hw;
2175     unsigned int def_q = 0;
2176     unsigned int num_tcs = 0;
2177     unsigned int num_rx_queues = adapter->num_rx_queues;
2178     unsigned int num_tx_queues = adapter->num_tx_queues;
2179     int err;
2180
2181     spin_lock_bh(&adapter->mbx_lock);
2182
2183     /* fetch queue configuration from the PF */
2184     err = ixgbevf_get_queues(hw, &num_tcs, &def_q);
2185
2186     spin_unlock_bh(&adapter->mbx_lock);
2187
2188     if (err)
2189         return err;
2190
2191     if (num_tcs > 1) {
2192         /* we need only one Tx queue */
2193         num_tx_queues = 1;
2194
2195         /* update default Tx ring register index */
2196         adapter->tx_ring[0]->reg_idx = def_q;
2197
2198         /* we need as many queues as traffic classes */
2199         num_rx_queues = num_tcs;
2200     }
2201
2202     /* if we have a bad config abort request queue reset */
2203     if ((adapter->num_rx_queues != num_rx_queues) ||
2204         (adapter->num_tx_queues != num_tx_queues)) {
2205         /* force mailbox timeout to prevent further messages */
2206         hw->mbx.timeout = 0;
2207
2208         /* wait for watchdog to come around and bail us out */
2209         set_bit(__IXGBEVF_QUEUE_RESET_REQUESTED, &adapter->state);
2210     }
2211
2212     return 0;
2213 }
2214
2215 static void ixgbevf_configure(struct ixgbevf_adapter *adapter)
2216 {
2217     ixgbevf_configure_dcb(adapter);
2218
2219     ixgbevf_set_rx_mode(adapter->netdev);
2220
2221     ixgbevf_restore_vlan(adapter);
2222     ixgbevf_ipsec_restore(adapter);
2223
2224     ixgbevf_configure_tx(adapter);
2225     ixgbevf_configure_rx(adapter);
2226 }
2227
2228 static void ixgbevf_save_reset_stats(struct ixgbevf_adapter *adapter)
2229 {
2230     /* Only save pre-reset stats if there are some */
2231     if (adapter->stats.vfgprc || adapter->stats.vfgptc) {
2232         adapter->stats.saved_reset_vfgprc += adapter->stats.vfgprc -
2233             adapter->stats.base_vfgprc;
2234         adapter->stats.saved_reset_vfgptc += adapter->stats.vfgptc -
2235             adapter->stats.base_vfgptc;
2236         adapter->stats.saved_reset_vfgorc += adapter->stats.vfgorc -
2237             adapter->stats.base_vfgorc;
2238         adapter->stats.saved_reset_vfgotc += adapter->stats.vfgotc -
2239             adapter->stats.base_vfgotc;
2240         adapter->stats.saved_reset_vfmprc += adapter->stats.vfmprc -
2241             adapter->stats.base_vfmprc;
2242     }
2243 }
2244
2245 static void ixgbevf_init_last_counter_stats(struct ixgbevf_adapter *adapter)
2246 {
2247     struct ixgbe_hw *hw = &adapter->hw;
2248
2249     adapter->stats.last_vfgprc = IXGBE_READ_REG(hw, IXGBE_VFGPRC);
2250     adapter->stats.last_vfgorc = IXGBE_READ_REG(hw, IXGBE_VFGORC_LSB);
2251     adapter->stats.last_vfgorc |=
2252         (((u64)(IXGBE_READ_REG(hw, IXGBE_VFGORC_MSB))) << 32);
2253     adapter->stats.last_vfgptc = IXGBE_READ_REG(hw, IXGBE_VFGPTC);
2254     adapter->stats.last_vfgotc = IXGBE_READ_REG(hw, IXGBE_VFGOTC_LSB);
2255     adapter->stats.last_vfgotc |=
2256         (((u64)(IXGBE_READ_REG(hw, IXGBE_VFGOTC_MSB))) << 32);
2257     adapter->stats.last_vfmprc = IXGBE_READ_REG(hw, IXGBE_VFMPRC);
2258
2259     adapter->stats.base_vfgprc = adapter->stats.last_vfgprc;
2260     adapter->stats.base_vfgorc = adapter->stats.last_vfgorc;
2261     adapter->stats.base_vfgptc = adapter->stats.last_vfgptc;
2262     adapter->stats.base_vfgotc = adapter->stats.last_vfgotc;
2263     adapter->stats.base_vfmprc = adapter->stats.last_vfmprc;
2264 }
2265
2266 static void ixgbevf_negotiate_api(struct ixgbevf_adapter *adapter)
2267 {
2268     struct ixgbe_hw *hw = &adapter->hw;
2269     static const int api[] = {
2270         ixgbe_mbox_api_15,
2271         ixgbe_mbox_api_14,
2272         ixgbe_mbox_api_13,
2273         ixgbe_mbox_api_12,
2274         ixgbe_mbox_api_11,
2275         ixgbe_mbox_api_10,
2276         ixgbe_mbox_api_unknown
2277     };
2278     int err, idx = 0;
2279
2280     spin_lock_bh(&adapter->mbx_lock);
2281
2282     while (api[idx] != ixgbe_mbox_api_unknown) {
2283         err = hw->mac.ops.negotiate_api_version(hw, api[idx]);
2284         if (!err)
2285             break;
2286         idx++;
2287     }
2288
2289     if (hw->api_version >= ixgbe_mbox_api_15) {
2290         hw->mbx.ops.init_params(hw);
2291         memcpy(&hw->mbx.ops, &ixgbevf_mbx_ops,
2292                sizeof(struct ixgbe_mbx_operations));
2293     }
2294
2295     spin_unlock_bh(&adapter->mbx_lock);
2296 }
2297
2298 static void ixgbevf_up_complete(struct ixgbevf_adapter *adapter)
2299 {
2300     struct net_device *netdev = adapter->netdev;
2301     struct pci_dev *pdev = adapter->pdev;
2302     struct ixgbe_hw *hw = &adapter->hw;
2303     bool state;
2304
2305     ixgbevf_configure_msix(adapter);
2306
2307     spin_lock_bh(&adapter->mbx_lock);
2308
2309     if (is_valid_ether_addr(hw->mac.addr))
2310         hw->mac.ops.set_rar(hw, 0, hw->mac.addr, 0);
2311     else
2312         hw->mac.ops.set_rar(hw, 0, hw->mac.perm_addr, 0);
2313
2314     spin_unlock_bh(&adapter->mbx_lock);
2315
2316     state = adapter->link_state;
2317     hw->mac.ops.get_link_state(hw, &adapter->link_state);
2318     if (state && state != adapter->link_state)
2319         dev_info(&pdev->dev, "VF is administratively disabled\n");
2320
2321     smp_mb__before_atomic();
2322     clear_bit(__IXGBEVF_DOWN, &adapter->state);
2323     ixgbevf_napi_enable_all(adapter);
2324
2325     /* clear any pending interrupts, may auto mask */
2326     IXGBE_READ_REG(hw, IXGBE_VTEICR);
2327     ixgbevf_irq_enable(adapter);
2328
2329     /* enable transmits */
2330     netif_tx_start_all_queues(netdev);
2331
2332     ixgbevf_save_reset_stats(adapter);
2333     ixgbevf_init_last_counter_stats(adapter);
2334
2335     hw->mac.get_link_status = 1;
2336     mod_timer(&adapter->service_timer, jiffies);
2337 }
2338
2339 void ixgbevf_up(struct ixgbevf_adapter *adapter)
2340 {
2341     ixgbevf_configure(adapter);
2342
2343     ixgbevf_up_complete(adapter);
2344 }
2345
2346 /**
2347  * ixgbevf_clean_rx_ring - Free Rx Buffers per Queue
2348  * @rx_ring: ring to free buffers from
2349  **/
2350 static void ixgbevf_clean_rx_ring(struct ixgbevf_ring *rx_ring)
2351 {
2352     u16 i = rx_ring->next_to_clean;
2353
2354     /* Free Rx ring sk_buff */
2355     if (rx_ring->skb) {
2356         dev_kfree_skb(rx_ring->skb);
2357         rx_ring->skb = NULL;
2358     }
2359
2360     /* Free all the Rx ring pages */
2361     while (i != rx_ring->next_to_alloc) {
2362         struct ixgbevf_rx_buffer *rx_buffer;
2363
2364         rx_buffer = &rx_ring->rx_buffer_info[i];
2365
2366         /* Invalidate cache lines that may have been written to by
2367          * device so that we avoid corrupting memory.
2368          */
2369         dma_sync_single_range_for_cpu(rx_ring->dev,
2370                           rx_buffer->dma,
2371                           rx_buffer->page_offset,
2372                           ixgbevf_rx_bufsz(rx_ring),
2373                           DMA_FROM_DEVICE);
2374
2375         /* free resources associated with mapping */
2376         dma_unmap_page_attrs(rx_ring->dev,
2377                      rx_buffer->dma,
2378                      ixgbevf_rx_pg_size(rx_ring),
2379                      DMA_FROM_DEVICE,
2380                      IXGBEVF_RX_DMA_ATTR);
2381
2382         __page_frag_cache_drain(rx_buffer->page,
2383                     rx_buffer->pagecnt_bias);
2384
2385         i++;
2386         if (i == rx_ring->count)
2387             i = 0;
2388     }
2389
2390     rx_ring->next_to_alloc = 0;
2391     rx_ring->next_to_clean = 0;
2392     rx_ring->next_to_use = 0;
2393 }
2394
2395 /**
2396  * ixgbevf_clean_tx_ring - Free Tx Buffers
2397  * @tx_ring: ring to be cleaned
2398  **/
2399 static void ixgbevf_clean_tx_ring(struct ixgbevf_ring *tx_ring)
2400 {
2401     u16 i = tx_ring->next_to_clean;
2402     struct ixgbevf_tx_buffer *tx_buffer = &tx_ring->tx_buffer_info[i];
2403
2404     while (i != tx_ring->next_to_use) {
2405         union ixgbe_adv_tx_desc *eop_desc, *tx_desc;
2406
2407         /* Free all the Tx ring sk_buffs */
2408         if (ring_is_xdp(tx_ring))
2409             page_frag_free(tx_buffer->data);
2410         else
2411             dev_kfree_skb_any(tx_buffer->skb);
2412
2413         /* unmap skb header data */
2414         dma_unmap_single(tx_ring->dev,
2415                  dma_unmap_addr(tx_buffer, dma),
2416                  dma_unmap_len(tx_buffer, len),
2417                  DMA_TO_DEVICE);
2418
2419         /* check for eop_desc to determine the end of the packet */
2420         eop_desc = tx_buffer->next_to_watch;
2421         tx_desc = IXGBEVF_TX_DESC(tx_ring, i);
2422
2423         /* unmap remaining buffers */
2424         while (tx_desc != eop_desc) {
2425             tx_buffer++;
2426             tx_desc++;
2427             i++;
2428             if (unlikely(i == tx_ring->count)) {
2429                 i = 0;
2430                 tx_buffer = tx_ring->tx_buffer_info;
2431                 tx_desc = IXGBEVF_TX_DESC(tx_ring, 0);
2432             }
2433
2434             /* unmap any remaining paged data */
2435             if (dma_unmap_len(tx_buffer, len))
2436                 dma_unmap_page(tx_ring->dev,
2437                            dma_unmap_addr(tx_buffer, dma),
2438                            dma_unmap_len(tx_buffer, len),
2439                            DMA_TO_DEVICE);
2440         }
2441
2442         /* move us one more past the eop_desc for start of next pkt */
2443         tx_buffer++;
2444         i++;
2445         if (unlikely(i == tx_ring->count)) {
2446             i = 0;
2447             tx_buffer = tx_ring->tx_buffer_info;
2448         }
2449     }
2450
2451     /* reset next_to_use and next_to_clean */
2452     tx_ring->next_to_use = 0;
2453     tx_ring->next_to_clean = 0;
2454
2455 }
2456
2457 /**
2458  * ixgbevf_clean_all_rx_rings - Free Rx Buffers for all queues
2459  * @adapter: board private structure
2460  **/
2461 static void ixgbevf_clean_all_rx_rings(struct ixgbevf_adapter *adapter)
2462 {
2463     int i;
2464
2465     for (i = 0; i < adapter->num_rx_queues; i++)
2466         ixgbevf_clean_rx_ring(adapter->rx_ring[i]);
2467 }
2468
2469 /**
2470  * ixgbevf_clean_all_tx_rings - Free Tx Buffers for all queues
2471  * @adapter: board private structure
2472  **/
2473 static void ixgbevf_clean_all_tx_rings(struct ixgbevf_adapter *adapter)
2474 {
2475     int i;
2476
2477     for (i = 0; i < adapter->num_tx_queues; i++)
2478         ixgbevf_clean_tx_ring(adapter->tx_ring[i]);
2479     for (i = 0; i < adapter->num_xdp_queues; i++)
2480         ixgbevf_clean_tx_ring(adapter->xdp_ring[i]);
2481 }
2482
2483 void ixgbevf_down(struct ixgbevf_adapter *adapter)
2484 {
2485     struct net_device *netdev = adapter->netdev;
2486     struct ixgbe_hw *hw = &adapter->hw;
2487     int i;
2488
2489     /* signal that we are down to the interrupt handler */
2490     if (test_and_set_bit(__IXGBEVF_DOWN, &adapter->state))
2491         return; /* do nothing if already down */
2492
2493     /* disable all enabled Rx queues */
2494     for (i = 0; i < adapter->num_rx_queues; i++)
2495         ixgbevf_disable_rx_queue(adapter, adapter->rx_ring[i]);
2496
2497     usleep_range(10000, 20000);
2498
2499     netif_tx_stop_all_queues(netdev);
2500
2501     /* call carrier off first to avoid false dev_watchdog timeouts */
2502     netif_carrier_off(netdev);
2503     netif_tx_disable(netdev);
2504
2505     ixgbevf_irq_disable(adapter);
2506
2507     ixgbevf_napi_disable_all(adapter);
2508
2509     del_timer_sync(&adapter->service_timer);
2510
2511     /* disable transmits in the hardware now that interrupts are off */
2512     for (i = 0; i < adapter->num_tx_queues; i++) {
2513         u8 reg_idx = adapter->tx_ring[i]->reg_idx;
2514
2515         IXGBE_WRITE_REG(hw, IXGBE_VFTXDCTL(reg_idx),
2516                 IXGBE_TXDCTL_SWFLSH);
2517     }
2518
2519     for (i = 0; i < adapter->num_xdp_queues; i++) {
2520         u8 reg_idx = adapter->xdp_ring[i]->reg_idx;
2521
2522         IXGBE_WRITE_REG(hw, IXGBE_VFTXDCTL(reg_idx),
2523                 IXGBE_TXDCTL_SWFLSH);
2524     }
2525
2526     if (!pci_channel_offline(adapter->pdev))
2527         ixgbevf_reset(adapter);
2528
2529     ixgbevf_clean_all_tx_rings(adapter);
2530     ixgbevf_clean_all_rx_rings(adapter);
2531 }
2532
2533 void ixgbevf_reinit_locked(struct ixgbevf_adapter *adapter)
2534 {
2535     while (test_and_set_bit(__IXGBEVF_RESETTING, &adapter->state))
2536         msleep(1);
2537
2538     ixgbevf_down(adapter);
2539     pci_set_master(adapter->pdev);
2540     ixgbevf_up(adapter);
2541
2542     clear_bit(__IXGBEVF_RESETTING, &adapter->state);
2543 }
2544
2545 void ixgbevf_reset(struct ixgbevf_adapter *adapter)
2546 {
2547     struct ixgbe_hw *hw = &adapter->hw;
2548     struct net_device *netdev = adapter->netdev;
2549
2550     if (hw->mac.ops.reset_hw(hw)) {
2551         hw_dbg(hw, "PF still resetting\n");
2552     } else {
2553         hw->mac.ops.init_hw(hw);
2554         ixgbevf_negotiate_api(adapter);
2555     }
2556
2557     if (is_valid_ether_addr(adapter->hw.mac.addr)) {
2558         eth_hw_addr_set(netdev, adapter->hw.mac.addr);
2559         ether_addr_copy(netdev->perm_addr, adapter->hw.mac.addr);
2560     }
2561
2562     adapter->last_reset = jiffies;
2563 }
2564
2565 static int ixgbevf_acquire_msix_vectors(struct ixgbevf_adapter *adapter,
2566                     int vectors)
2567 {
2568     int vector_threshold;
2569
2570     /* We'll want at least 2 (vector_threshold):
2571      * 1) TxQ[0] + RxQ[0] handler
2572      * 2) Other (Link Status Change, etc.)
2573      */
2574     vector_threshold = MIN_MSIX_COUNT;
2575
2576     /* The more we get, the more we will assign to Tx/Rx Cleanup
2577      * for the separate queues...where Rx Cleanup >= Tx Cleanup.
2578      * Right now, we simply care about how many we'll get; we'll
2579      * set them up later while requesting irq's.
2580      */
2581     vectors = pci_enable_msix_range(adapter->pdev, adapter->msix_entries,
2582                     vector_threshold, vectors);
2583
2584     if (vectors < 0) {
2585         dev_err(&adapter->pdev->dev,
2586             "Unable to allocate MSI-X interrupts\n");
2587         kfree(adapter->msix_entries);
2588         adapter->msix_entries = NULL;
2589         return vectors;
2590     }
2591
2592     /* Adjust for only the vectors we'll use, which is minimum
2593      * of max_msix_q_vectors + NON_Q_VECTORS, or the number of
2594      * vectors we were allocated.
2595      */
2596     adapter->num_msix_vectors = vectors;
2597
2598     return 0;
2599 }
2600
2601 /**
2602  * ixgbevf_set_num_queues - Allocate queues for device, feature dependent
2603  * @adapter: board private structure to initialize
2604  *
2605  * This is the top level queue allocation routine.  The order here is very
2606  * important, starting with the "most" number of features turned on at once,
2607  * and ending with the smallest set of features.  This way large combinations
2608  * can be allocated if they're turned on, and smaller combinations are the
2609  * fall through conditions.
2610  *
2611  **/
2612 static void ixgbevf_set_num_queues(struct ixgbevf_adapter *adapter)
2613 {
2614     struct ixgbe_hw *hw = &adapter->hw;
2615     unsigned int def_q = 0;
2616     unsigned int num_tcs = 0;
2617     int err;
2618
2619     /* Start with base case */
2620     adapter->num_rx_queues = 1;
2621     adapter->num_tx_queues = 1;
2622     adapter->num_xdp_queues = 0;
2623
2624     spin_lock_bh(&adapter->mbx_lock);
2625
2626     /* fetch queue configuration from the PF */
2627     err = ixgbevf_get_queues(hw, &num_tcs, &def_q);
2628
2629     spin_unlock_bh(&adapter->mbx_lock);
2630
2631     if (err)
2632         return;
2633
2634     /* we need as many queues as traffic classes */
2635     if (num_tcs > 1) {
2636         adapter->num_rx_queues = num_tcs;
2637     } else {
2638         u16 rss = min_t(u16, num_online_cpus(), IXGBEVF_MAX_RSS_QUEUES);
2639
2640         switch (hw->api_version) {
2641         case ixgbe_mbox_api_11:
2642         case ixgbe_mbox_api_12:
2643         case ixgbe_mbox_api_13:
2644         case ixgbe_mbox_api_14:
2645         case ixgbe_mbox_api_15:
2646             if (adapter->xdp_prog &&
2647                 hw->mac.max_tx_queues == rss)
2648                 rss = rss > 3 ? 2 : 1;
2649
2650             adapter->num_rx_queues = rss;
2651             adapter->num_tx_queues = rss;
2652             adapter->num_xdp_queues = adapter->xdp_prog ? rss : 0;
2653             break;
2654         default:
2655             break;
2656         }
2657     }
2658 }
2659
2660 /**
2661  * ixgbevf_set_interrupt_capability - set MSI-X or FAIL if not supported
2662  * @adapter: board private structure to initialize
2663  *
2664  * Attempt to configure the interrupts using the best available
2665  * capabilities of the hardware and the kernel.
2666  **/
2667 static int ixgbevf_set_interrupt_capability(struct ixgbevf_adapter *adapter)
2668 {
2669     int vector, v_budget;
2670
2671     /* It's easy to be greedy for MSI-X vectors, but it really
2672      * doesn't do us much good if we have a lot more vectors
2673      * than CPU's.  So let's be conservative and only ask for
2674      * (roughly) the same number of vectors as there are CPU's.
2675      * The default is to use pairs of vectors.
2676      */
2677     v_budget = max(adapter->num_rx_queues, adapter->num_tx_queues);
2678     v_budget = min_t(int, v_budget, num_online_cpus());
2679     v_budget += NON_Q_VECTORS;
2680
2681     adapter->msix_entries = kcalloc(v_budget,
2682                     sizeof(struct msix_entry), GFP_KERNEL);
2683     if (!adapter->msix_entries)
2684         return -ENOMEM;
2685
2686     for (vector = 0; vector < v_budget; vector++)
2687         adapter->msix_entries[vector].entry = vector;
2688
2689     /* A failure in MSI-X entry allocation isn't fatal, but the VF driver
2690      * does not support any other modes, so we will simply fail here. Note
2691      * that we clean up the msix_entries pointer else-where.
2692      */
2693     return ixgbevf_acquire_msix_vectors(adapter, v_budget);
2694 }
2695
2696 static void ixgbevf_add_ring(struct ixgbevf_ring *ring,
2697                  struct ixgbevf_ring_container *head)
2698 {
2699     ring->next = head->ring;
2700     head->ring = ring;
2701     head->count++;
2702 }
2703
2704 /**
2705  * ixgbevf_alloc_q_vector - Allocate memory for a single interrupt vector
2706  * @adapter: board private structure to initialize
2707  * @v_idx: index of vector in adapter struct
2708  * @txr_count: number of Tx rings for q vector
2709  * @txr_idx: index of first Tx ring to assign
2710  * @xdp_count: total number of XDP rings to allocate
2711  * @xdp_idx: index of first XDP ring to allocate
2712  * @rxr_count: number of Rx rings for q vector
2713  * @rxr_idx: index of first Rx ring to assign
2714  *
2715  * We allocate one q_vector.  If allocation fails we return -ENOMEM.
2716  **/
2717 static int ixgbevf_alloc_q_vector(struct ixgbevf_adapter *adapter, int v_idx,
2718                   int txr_count, int txr_idx,
2719                   int xdp_count, int xdp_idx,
2720                   int rxr_count, int rxr_idx)
2721 {
2722     struct ixgbevf_q_vector *q_vector;
2723     int reg_idx = txr_idx + xdp_idx;
2724     struct ixgbevf_ring *ring;
2725     int ring_count, size;
2726
2727     ring_count = txr_count + xdp_count + rxr_count;
2728     size = sizeof(*q_vector) + (sizeof(*ring) * ring_count);
2729
2730     /* allocate q_vector and rings */
2731     q_vector = kzalloc(size, GFP_KERNEL);
2732     if (!q_vector)
2733         return -ENOMEM;
2734
2735     /* initialize NAPI */
2736     netif_napi_add(adapter->netdev, &q_vector->napi, ixgbevf_poll, 64);
2737
2738     /* tie q_vector and adapter together */
2739     adapter->q_vector[v_idx] = q_vector;
2740     q_vector->adapter = adapter;
2741     q_vector->v_idx = v_idx;
2742
2743     /* initialize pointer to rings */
2744     ring = q_vector->ring;
2745
2746     while (txr_count) {
2747         /* assign generic ring traits */
2748         ring->dev = &adapter->pdev->dev;
2749         ring->netdev = adapter->netdev;
2750
2751         /* configure backlink on ring */
2752         ring->q_vector = q_vector;
2753
2754         /* update q_vector Tx values */
2755         ixgbevf_add_ring(ring, &q_vector->tx);
2756
2757         /* apply Tx specific ring traits */
2758         ring->count = adapter->tx_ring_count;
2759         ring->queue_index = txr_idx;
2760         ring->reg_idx = reg_idx;
2761
2762         /* assign ring to adapter */
2763         adapter->tx_ring[txr_idx] = ring;
2764
2765         /* update count and index */
2766         txr_count--;
2767         txr_idx++;
2768         reg_idx++;
2769
2770         /* push pointer to next ring */
2771         ring++;
2772     }
2773
2774     while (xdp_count) {
2775         /* assign generic ring traits */
2776         ring->dev = &adapter->pdev->dev;
2777         ring->netdev = adapter->netdev;
2778
2779         /* configure backlink on ring */
2780         ring->q_vector = q_vector;
2781
2782         /* update q_vector Tx values */
2783         ixgbevf_add_ring(ring, &q_vector->tx);
2784
2785         /* apply Tx specific ring traits */
2786         ring->count = adapter->tx_ring_count;
2787         ring->queue_index = xdp_idx;
2788         ring->reg_idx = reg_idx;
2789         set_ring_xdp(ring);
2790
2791         /* assign ring to adapter */
2792         adapter->xdp_ring[xdp_idx] = ring;
2793
2794         /* update count and index */
2795         xdp_count--;
2796         xdp_idx++;
2797         reg_idx++;
2798
2799         /* push pointer to next ring */
2800         ring++;
2801     }
2802
2803     while (rxr_count) {
2804         /* assign generic ring traits */
2805         ring->dev = &adapter->pdev->dev;
2806         ring->netdev = adapter->netdev;
2807
2808         /* configure backlink on ring */
2809         ring->q_vector = q_vector;
2810
2811         /* update q_vector Rx values */
2812         ixgbevf_add_ring(ring, &q_vector->rx);
2813
2814         /* apply Rx specific ring traits */
2815         ring->count = adapter->rx_ring_count;
2816         ring->queue_index = rxr_idx;
2817         ring->reg_idx = rxr_idx;
2818
2819         /* assign ring to adapter */
2820         adapter->rx_ring[rxr_idx] = ring;
2821
2822         /* update count and index */
2823         rxr_count--;
2824         rxr_idx++;
2825
2826         /* push pointer to next ring */
2827         ring++;
2828     }
2829
2830     return 0;
2831 }
2832
2833 /**
2834  * ixgbevf_free_q_vector - Free memory allocated for specific interrupt vector
2835  * @adapter: board private structure to initialize
2836  * @v_idx: index of vector in adapter struct
2837  *
2838  * This function frees the memory allocated to the q_vector.  In addition if
2839  * NAPI is enabled it will delete any references to the NAPI struct prior
2840  * to freeing the q_vector.
2841  **/
2842 static void ixgbevf_free_q_vector(struct ixgbevf_adapter *adapter, int v_idx)
2843 {
2844     struct ixgbevf_q_vector *q_vector = adapter->q_vector[v_idx];
2845     struct ixgbevf_ring *ring;
2846
2847     ixgbevf_for_each_ring(ring, q_vector->tx) {
2848         if (ring_is_xdp(ring))
2849             adapter->xdp_ring[ring->queue_index] = NULL;
2850         else
2851             adapter->tx_ring[ring->queue_index] = NULL;
2852     }
2853
2854     ixgbevf_for_each_ring(ring, q_vector->rx)
2855         adapter->rx_ring[ring->queue_index] = NULL;
2856
2857     adapter->q_vector[v_idx] = NULL;
2858     netif_napi_del(&q_vector->napi);
2859
2860     /* ixgbevf_get_stats() might access the rings on this vector,
2861      * we must wait a grace period before freeing it.
2862      */
2863     kfree_rcu(q_vector, rcu);
2864 }
2865
2866 /**
2867  * ixgbevf_alloc_q_vectors - Allocate memory for interrupt vectors
2868  * @adapter: board private structure to initialize
2869  *
2870  * We allocate one q_vector per queue interrupt.  If allocation fails we
2871  * return -ENOMEM.
2872  **/
2873 static int ixgbevf_alloc_q_vectors(struct ixgbevf_adapter *adapter)
2874 {
2875     int q_vectors = adapter->num_msix_vectors - NON_Q_VECTORS;
2876     int rxr_remaining = adapter->num_rx_queues;
2877     int txr_remaining = adapter->num_tx_queues;
2878     int xdp_remaining = adapter->num_xdp_queues;
2879     int rxr_idx = 0, txr_idx = 0, xdp_idx = 0, v_idx = 0;
2880     int err;
2881
2882     if (q_vectors >= (rxr_remaining + txr_remaining + xdp_remaining)) {
2883         for (; rxr_remaining; v_idx++, q_vectors--) {
2884             int rqpv = DIV_ROUND_UP(rxr_remaining, q_vectors);
2885
2886             err = ixgbevf_alloc_q_vector(adapter, v_idx,
2887                              0, 0, 0, 0, rqpv, rxr_idx);
2888             if (err)
2889                 goto err_out;
2890
2891             /* update counts and index */
2892             rxr_remaining -= rqpv;
2893             rxr_idx += rqpv;
2894         }
2895     }
2896
2897     for (; q_vectors; v_idx++, q_vectors--) {
2898         int rqpv = DIV_ROUND_UP(rxr_remaining, q_vectors);
2899         int tqpv = DIV_ROUND_UP(txr_remaining, q_vectors);
2900         int xqpv = DIV_ROUND_UP(xdp_remaining, q_vectors);
2901
2902         err = ixgbevf_alloc_q_vector(adapter, v_idx,
2903                          tqpv, txr_idx,
2904                          xqpv, xdp_idx,
2905                          rqpv, rxr_idx);
2906
2907         if (err)
2908             goto err_out;
2909
2910         /* update counts and index */
2911         rxr_remaining -= rqpv;
2912         rxr_idx += rqpv;
2913         txr_remaining -= tqpv;
2914         txr_idx += tqpv;
2915         xdp_remaining -= xqpv;
2916         xdp_idx += xqpv;
2917     }
2918
2919     return 0;
2920
2921 err_out:
2922     while (v_idx) {
2923         v_idx--;
2924         ixgbevf_free_q_vector(adapter, v_idx);
2925     }
2926
2927     return -ENOMEM;
2928 }
2929
2930 /**
2931  * ixgbevf_free_q_vectors - Free memory allocated for interrupt vectors
2932  * @adapter: board private structure to initialize
2933  *
2934  * This function frees the memory allocated to the q_vectors.  In addition if
2935  * NAPI is enabled it will delete any references to the NAPI struct prior
2936  * to freeing the q_vector.
2937  **/
2938 static void ixgbevf_free_q_vectors(struct ixgbevf_adapter *adapter)
2939 {
2940     int q_vectors = adapter->num_msix_vectors - NON_Q_VECTORS;
2941
2942     while (q_vectors) {
2943         q_vectors--;
2944         ixgbevf_free_q_vector(adapter, q_vectors);
2945     }
2946 }
2947
2948 /**
2949  * ixgbevf_reset_interrupt_capability - Reset MSIX setup
2950  * @adapter: board private structure
2951  *
2952  **/
2953 static void ixgbevf_reset_interrupt_capability(struct ixgbevf_adapter *adapter)
2954 {
2955     if (!adapter->msix_entries)
2956         return;
2957
2958     pci_disable_msix(adapter->pdev);
2959     kfree(adapter->msix_entries);
2960     adapter->msix_entries = NULL;
2961 }
2962
2963 /**
2964  * ixgbevf_init_interrupt_scheme - Determine if MSIX is supported and init
2965  * @adapter: board private structure to initialize
2966  *
2967  **/
2968 static int ixgbevf_init_interrupt_scheme(struct ixgbevf_adapter *adapter)
2969 {
2970     int err;
2971
2972     /* Number of supported queues */
2973     ixgbevf_set_num_queues(adapter);
2974
2975     err = ixgbevf_set_interrupt_capability(adapter);
2976     if (err) {
2977         hw_dbg(&adapter->hw,
2978                "Unable to setup interrupt capabilities\n");
2979         goto err_set_interrupt;
2980     }
2981
2982     err = ixgbevf_alloc_q_vectors(adapter);
2983     if (err) {
2984         hw_dbg(&adapter->hw, "Unable to allocate memory for queue vectors\n");
2985         goto err_alloc_q_vectors;
2986     }
2987
2988     hw_dbg(&adapter->hw, "Multiqueue %s: Rx Queue count = %u, Tx Queue count = %u XDP Queue count %u\n",
2989            (adapter->num_rx_queues > 1) ? "Enabled" : "Disabled",
2990            adapter->num_rx_queues, adapter->num_tx_queues,
2991            adapter->num_xdp_queues);
2992
2993     set_bit(__IXGBEVF_DOWN, &adapter->state);
2994
2995     return 0;
2996 err_alloc_q_vectors:
2997     ixgbevf_reset_interrupt_capability(adapter);
2998 err_set_interrupt:
2999     return err;
3000 }
3001
3002 /**
3003  * ixgbevf_clear_interrupt_scheme - Clear the current interrupt scheme settings
3004  * @adapter: board private structure to clear interrupt scheme on
3005  *
3006  * We go through and clear interrupt specific resources and reset the structure
3007  * to pre-load conditions
3008  **/
3009 static void ixgbevf_clear_interrupt_scheme(struct ixgbevf_adapter *adapter)
3010 {
3011     adapter->num_tx_queues = 0;
3012     adapter->num_xdp_queues = 0;
3013     adapter->num_rx_queues = 0;
3014
3015     ixgbevf_free_q_vectors(adapter);
3016     ixgbevf_reset_interrupt_capability(adapter);
3017 }
3018
3019 /**
3020  * ixgbevf_sw_init - Initialize general software structures
3021  * @adapter: board private structure to initialize
3022  *
3023  * ixgbevf_sw_init initializes the Adapter private data structure.
3024  * Fields are initialized based on PCI device information and
3025  * OS network device settings (MTU size).
3026  **/
3027 static int ixgbevf_sw_init(struct ixgbevf_adapter *adapter)
3028 {
3029     struct ixgbe_hw *hw = &adapter->hw;
3030     struct pci_dev *pdev = adapter->pdev;
3031     struct net_device *netdev = adapter->netdev;
3032     int err;
3033
3034     /* PCI config space info */
3035     hw->vendor_id = pdev->vendor;
3036     hw->device_id = pdev->device;
3037     hw->revision_id = pdev->revision;
3038     hw->subsystem_vendor_id = pdev->subsystem_vendor;
3039     hw->subsystem_device_id = pdev->subsystem_device;
3040
3041     hw->mbx.ops.init_params(hw);
3042
3043     if (hw->mac.type >= ixgbe_mac_X550_vf) {
3044         err = ixgbevf_init_rss_key(adapter);
3045         if (err)
3046             goto out;
3047     }
3048
3049     /* assume legacy case in which PF would only give VF 2 queues */
3050     hw->mac.max_tx_queues = 2;
3051     hw->mac.max_rx_queues = 2;
3052
3053     /* lock to protect mailbox accesses */
3054     spin_lock_init(&adapter->mbx_lock);
3055
3056     err = hw->mac.ops.reset_hw(hw);
3057     if (err) {
3058         dev_info(&pdev->dev,
3059              "PF still in reset state.  Is the PF interface up?\n");
3060     } else {
3061         err = hw->mac.ops.init_hw(hw);
3062         if (err) {
3063             pr_err("init_shared_code failed: %d\n", err);
3064             goto out;
3065         }
3066         ixgbevf_negotiate_api(adapter);
3067         err = hw->mac.ops.get_mac_addr(hw, hw->mac.addr);
3068         if (err)
3069             dev_info(&pdev->dev, "Error reading MAC address\n");
3070         else if (is_zero_ether_addr(adapter->hw.mac.addr))
3071             dev_info(&pdev->dev,
3072                  "MAC address not assigned by administrator.\n");
3073         eth_hw_addr_set(netdev, hw->mac.addr);
3074     }
3075
3076     if (!is_valid_ether_addr(netdev->dev_addr)) {
3077         dev_info(&pdev->dev, "Assigning random MAC address\n");
3078         eth_hw_addr_random(netdev);
3079         ether_addr_copy(hw->mac.addr, netdev->dev_addr);
3080         ether_addr_copy(hw->mac.perm_addr, netdev->dev_addr);
3081     }
3082
3083     /* Enable dynamic interrupt throttling rates */
3084     adapter->rx_itr_setting = 1;
3085     adapter->tx_itr_setting = 1;
3086
3087     /* set default ring sizes */
3088     adapter->tx_ring_count = IXGBEVF_DEFAULT_TXD;
3089     adapter->rx_ring_count = IXGBEVF_DEFAULT_RXD;
3090
3091     adapter->link_state = true;
3092
3093     set_bit(__IXGBEVF_DOWN, &adapter->state);
3094     return 0;
3095
3096 out:
3097     return err;
3098 }
3099
3100 #define UPDATE_VF_COUNTER_32bit(reg, last_counter, counter) \
3101     {                           \
3102         u32 current_counter = IXGBE_READ_REG(hw, reg);  \
3103         if (current_counter < last_counter)     \
3104             counter += 0x100000000LL;       \
3105         last_counter = current_counter;         \
3106         counter &= 0xFFFFFFFF00000000LL;        \
3107         counter |= current_counter;         \
3108     }
3109
3110 #define UPDATE_VF_COUNTER_36bit(reg_lsb, reg_msb, last_counter, counter) \
3111     {                                \
3112         u64 current_counter_lsb = IXGBE_READ_REG(hw, reg_lsb);   \
3113         u64 current_counter_msb = IXGBE_READ_REG(hw, reg_msb);   \
3114         u64 current_counter = (current_counter_msb << 32) |  \
3115             current_counter_lsb;                 \
3116         if (current_counter < last_counter)          \
3117             counter += 0x1000000000LL;           \
3118         last_counter = current_counter;              \
3119         counter &= 0xFFFFFFF000000000LL;             \
3120         counter |= current_counter;              \
3121     }
3122 /**
3123  * ixgbevf_update_stats - Update the board statistics counters.
3124  * @adapter: board private structure
3125  **/
3126 void ixgbevf_update_stats(struct ixgbevf_adapter *adapter)
3127 {
3128     struct ixgbe_hw *hw = &adapter->hw;
3129     u64 alloc_rx_page_failed = 0, alloc_rx_buff_failed = 0;
3130     u64 alloc_rx_page = 0, hw_csum_rx_error = 0;
3131     int i;
3132
3133     if (test_bit(__IXGBEVF_DOWN, &adapter->state) ||
3134         test_bit(__IXGBEVF_RESETTING, &adapter->state))
3135         return;
3136
3137     UPDATE_VF_COUNTER_32bit(IXGBE_VFGPRC, adapter->stats.last_vfgprc,
3138                 adapter->stats.vfgprc);
3139     UPDATE_VF_COUNTER_32bit(IXGBE_VFGPTC, adapter->stats.last_vfgptc,
3140                 adapter->stats.vfgptc);
3141     UPDATE_VF_COUNTER_36bit(IXGBE_VFGORC_LSB, IXGBE_VFGORC_MSB,
3142                 adapter->stats.last_vfgorc,
3143                 adapter->stats.vfgorc);
3144     UPDATE_VF_COUNTER_36bit(IXGBE_VFGOTC_LSB, IXGBE_VFGOTC_MSB,
3145                 adapter->stats.last_vfgotc,
3146                 adapter->stats.vfgotc);
3147     UPDATE_VF_COUNTER_32bit(IXGBE_VFMPRC, adapter->stats.last_vfmprc,
3148                 adapter->stats.vfmprc);
3149
3150     for (i = 0;  i  < adapter->num_rx_queues;  i++) {
3151         struct ixgbevf_ring *rx_ring = adapter->rx_ring[i];
3152
3153         hw_csum_rx_error += rx_ring->rx_stats.csum_err;
3154         alloc_rx_page_failed += rx_ring->rx_stats.alloc_rx_page_failed;
3155         alloc_rx_buff_failed += rx_ring->rx_stats.alloc_rx_buff_failed;
3156         alloc_rx_page += rx_ring->rx_stats.alloc_rx_page;
3157     }
3158
3159     adapter->hw_csum_rx_error = hw_csum_rx_error;
3160     adapter->alloc_rx_page_failed = alloc_rx_page_failed;
3161     adapter->alloc_rx_buff_failed = alloc_rx_buff_failed;
3162     adapter->alloc_rx_page = alloc_rx_page;
3163 }
3164
3165 /**
3166  * ixgbevf_service_timer - Timer Call-back
3167  * @t: pointer to timer_list struct
3168  **/
3169 static void ixgbevf_service_timer(struct timer_list *t)
3170 {
3171     struct ixgbevf_adapter *adapter = from_timer(adapter, t,
3172                              service_timer);
3173
3174     /* Reset the timer */
3175     mod_timer(&adapter->service_timer, (HZ * 2) + jiffies);
3176
3177     ixgbevf_service_event_schedule(adapter);
3178 }
3179
3180 static void ixgbevf_reset_subtask(struct ixgbevf_adapter *adapter)
3181 {
3182     if (!test_and_clear_bit(__IXGBEVF_RESET_REQUESTED, &adapter->state))
3183         return;
3184
3185     rtnl_lock();
3186     /* If we're already down or resetting, just bail */
3187     if (test_bit(__IXGBEVF_DOWN, &adapter->state) ||
3188         test_bit(__IXGBEVF_REMOVING, &adapter->state) ||
3189         test_bit(__IXGBEVF_RESETTING, &adapter->state)) {
3190         rtnl_unlock();
3191         return;
3192     }
3193
3194     adapter->tx_timeout_count++;
3195
3196     ixgbevf_reinit_locked(adapter);
3197     rtnl_unlock();
3198 }
3199
3200 /**
3201  * ixgbevf_check_hang_subtask - check for hung queues and dropped interrupts
3202  * @adapter: pointer to the device adapter structure
3203  *
3204  * This function serves two purposes.  First it strobes the interrupt lines
3205  * in order to make certain interrupts are occurring.  Secondly it sets the
3206  * bits needed to check for TX hangs.  As a result we should immediately
3207  * determine if a hang has occurred.
3208  **/
3209 static void ixgbevf_check_hang_subtask(struct ixgbevf_adapter *adapter)
3210 {
3211     struct ixgbe_hw *hw = &adapter->hw;
3212     u32 eics = 0;
3213     int i;
3214
3215     /* If we're down or resetting, just bail */
3216     if (test_bit(__IXGBEVF_DOWN, &adapter->state) ||
3217         test_bit(__IXGBEVF_RESETTING, &adapter->state))
3218         return;
3219
3220     /* Force detection of hung controller */
3221     if (netif_carrier_ok(adapter->netdev)) {
3222         for (i = 0; i < adapter->num_tx_queues; i++)
3223             set_check_for_tx_hang(adapter->tx_ring[i]);
3224         for (i = 0; i < adapter->num_xdp_queues; i++)
3225             set_check_for_tx_hang(adapter->xdp_ring[i]);
3226     }
3227
3228     /* get one bit for every active Tx/Rx interrupt vector */
3229     for (i = 0; i < adapter->num_msix_vectors - NON_Q_VECTORS; i++) {
3230         struct ixgbevf_q_vector *qv = adapter->q_vector[i];
3231
3232         if (qv->rx.ring || qv->tx.ring)
3233             eics |= BIT(i);
3234     }
3235
3236     /* Cause software interrupt to ensure rings are cleaned */
3237     IXGBE_WRITE_REG(hw, IXGBE_VTEICS, eics);
3238 }
3239
3240 /**
3241  * ixgbevf_watchdog_update_link - update the link status
3242  * @adapter: pointer to the device adapter structure
3243  **/
3244 static void ixgbevf_watchdog_update_link(struct ixgbevf_adapter *adapter)
3245 {
3246     struct ixgbe_hw *hw = &adapter->hw;
3247     u32 link_speed = adapter->link_speed;
3248     bool link_up = adapter->link_up;
3249     s32 err;
3250
3251     spin_lock_bh(&adapter->mbx_lock);
3252
3253     err = hw->mac.ops.check_link(hw, &link_speed, &link_up, false);
3254
3255     spin_unlock_bh(&adapter->mbx_lock);
3256
3257     /* if check for link returns error we will need to reset */
3258     if (err && time_after(jiffies, adapter->last_reset + (10 * HZ))) {
3259         set_bit(__IXGBEVF_RESET_REQUESTED, &adapter->state);
3260         link_up = false;
3261     }
3262
3263     adapter->link_up = link_up;
3264     adapter->link_speed = link_speed;
3265 }
3266
3267 /**
3268  * ixgbevf_watchdog_link_is_up - update netif_carrier status and
3269  *               print link up message
3270  * @adapter: pointer to the device adapter structure
3271  **/
3272 static void ixgbevf_watchdog_link_is_up(struct ixgbevf_adapter *adapter)
3273 {
3274     struct net_device *netdev = adapter->netdev;
3275
3276     /* only continue if link was previously down */
3277     if (netif_carrier_ok(netdev))
3278         return;
3279
3280     dev_info(&adapter->pdev->dev, "NIC Link is Up %s\n",
3281          (adapter->link_speed == IXGBE_LINK_SPEED_10GB_FULL) ?
3282          "10 Gbps" :
3283          (adapter->link_speed == IXGBE_LINK_SPEED_1GB_FULL) ?
3284          "1 Gbps" :
3285          (adapter->link_speed == IXGBE_LINK_SPEED_100_FULL) ?
3286          "100 Mbps" :
3287          "unknown speed");
3288
3289     netif_carrier_on(netdev);
3290 }
3291
3292 /**
3293  * ixgbevf_watchdog_link_is_down - update netif_carrier status and
3294  *                 print link down message
3295  * @adapter: pointer to the adapter structure
3296  **/
3297 static void ixgbevf_watchdog_link_is_down(struct ixgbevf_adapter *adapter)
3298 {
3299     struct net_device *netdev = adapter->netdev;
3300
3301     adapter->link_speed = 0;
3302
3303     /* only continue if link was up previously */
3304     if (!netif_carrier_ok(netdev))
3305         return;
3306
3307     dev_info(&adapter->pdev->dev, "NIC Link is Down\n");
3308
3309     netif_carrier_off(netdev);
3310 }
3311
3312 /**
3313  * ixgbevf_watchdog_subtask - worker thread to bring link up
3314  * @adapter: board private structure
3315  **/
3316 static void ixgbevf_watchdog_subtask(struct ixgbevf_adapter *adapter)
3317 {
3318     /* if interface is down do nothing */
3319     if (test_bit(__IXGBEVF_DOWN, &adapter->state) ||
3320         test_bit(__IXGBEVF_RESETTING, &adapter->state))
3321         return;
3322
3323     ixgbevf_watchdog_update_link(adapter);
3324
3325     if (adapter->link_up && adapter->link_state)
3326         ixgbevf_watchdog_link_is_up(adapter);
3327     else
3328         ixgbevf_watchdog_link_is_down(adapter);
3329
3330     ixgbevf_update_stats(adapter);
3331 }
3332
3333 /**
3334  * ixgbevf_service_task - manages and runs subtasks
3335  * @work: pointer to work_struct containing our data
3336  **/
3337 static void ixgbevf_service_task(struct work_struct *work)
3338 {
3339     struct ixgbevf_adapter *adapter = container_of(work,
3340                                struct ixgbevf_adapter,
3341                                service_task);
3342     struct ixgbe_hw *hw = &adapter->hw;
3343
3344     if (IXGBE_REMOVED(hw->hw_addr)) {
3345         if (!test_bit(__IXGBEVF_DOWN, &adapter->state)) {
3346             rtnl_lock();
3347             ixgbevf_down(adapter);
3348             rtnl_unlock();
3349         }
3350         return;
3351     }
3352
3353     ixgbevf_queue_reset_subtask(adapter);
3354     ixgbevf_reset_subtask(adapter);
3355     ixgbevf_watchdog_subtask(adapter);
3356     ixgbevf_check_hang_subtask(adapter);
3357
3358     ixgbevf_service_event_complete(adapter);
3359 }
3360
3361 /**
3362  * ixgbevf_free_tx_resources - Free Tx Resources per Queue
3363  * @tx_ring: Tx descriptor ring for a specific queue
3364  *
3365  * Free all transmit software resources
3366  **/
3367 void ixgbevf_free_tx_resources(struct ixgbevf_ring *tx_ring)
3368 {
3369     ixgbevf_clean_tx_ring(tx_ring);
3370
3371     vfree(tx_ring->tx_buffer_info);
3372     tx_ring->tx_buffer_info = NULL;
3373
3374     /* if not set, then don't free */
3375     if (!tx_ring->desc)
3376         return;
3377
3378     dma_free_coherent(tx_ring->dev, tx_ring->size, tx_ring->desc,
3379               tx_ring->dma);
3380
3381     tx_ring->desc = NULL;
3382 }
3383
3384 /**
3385  * ixgbevf_free_all_tx_resources - Free Tx Resources for All Queues
3386  * @adapter: board private structure
3387  *
3388  * Free all transmit software resources
3389  **/
3390 static void ixgbevf_free_all_tx_resources(struct ixgbevf_adapter *adapter)
3391 {
3392     int i;
3393
3394     for (i = 0; i < adapter->num_tx_queues; i++)
3395         if (adapter->tx_ring[i]->desc)
3396             ixgbevf_free_tx_resources(adapter->tx_ring[i]);
3397     for (i = 0; i < adapter->num_xdp_queues; i++)
3398         if (adapter->xdp_ring[i]->desc)
3399             ixgbevf_free_tx_resources(adapter->xdp_ring[i]);
3400 }
3401
3402 /**
3403  * ixgbevf_setup_tx_resources - allocate Tx resources (Descriptors)
3404  * @tx_ring: Tx descriptor ring (for a specific queue) to setup
3405  *
3406  * Return 0 on success, negative on failure
3407  **/
3408 int ixgbevf_setup_tx_resources(struct ixgbevf_ring *tx_ring)
3409 {
3410     struct ixgbevf_adapter *adapter = netdev_priv(tx_ring->netdev);
3411     int size;
3412
3413     size = sizeof(struct ixgbevf_tx_buffer) * tx_ring->count;
3414     tx_ring->tx_buffer_info = vmalloc(size);
3415     if (!tx_ring->tx_buffer_info)
3416         goto err;
3417
3418     u64_stats_init(&tx_ring->syncp);
3419
3420     /* round up to nearest 4K */
3421     tx_ring->size = tx_ring->count * sizeof(union ixgbe_adv_tx_desc);
3422     tx_ring->size = ALIGN(tx_ring->size, 4096);
3423
3424     tx_ring->desc = dma_alloc_coherent(tx_ring->dev, tx_ring->size,
3425                        &tx_ring->dma, GFP_KERNEL);
3426     if (!tx_ring->desc)
3427         goto err;
3428
3429     return 0;
3430
3431 err:
3432     vfree(tx_ring->tx_buffer_info);
3433     tx_ring->tx_buffer_info = NULL;
3434     hw_dbg(&adapter->hw, "Unable to allocate memory for the transmit descriptor ring\n");
3435     return -ENOMEM;
3436 }
3437
3438 /**
3439  * ixgbevf_setup_all_tx_resources - allocate all queues Tx resources
3440  * @adapter: board private structure
3441  *
3442  * If this function returns with an error, then it's possible one or
3443  * more of the rings is populated (while the rest are not).  It is the
3444  * callers duty to clean those orphaned rings.
3445  *
3446  * Return 0 on success, negative on failure
3447  **/
3448 static int ixgbevf_setup_all_tx_resources(struct ixgbevf_adapter *adapter)
3449 {
3450     int i, j = 0, err = 0;
3451
3452     for (i = 0; i < adapter->num_tx_queues; i++) {
3453         err = ixgbevf_setup_tx_resources(adapter->tx_ring[i]);
3454         if (!err)
3455             continue;
3456         hw_dbg(&adapter->hw, "Allocation for Tx Queue %u failed\n", i);
3457         goto err_setup_tx;
3458     }
3459
3460     for (j = 0; j < adapter->num_xdp_queues; j++) {
3461         err = ixgbevf_setup_tx_resources(adapter->xdp_ring[j]);
3462         if (!err)
3463             continue;
3464         hw_dbg(&adapter->hw, "Allocation for XDP Queue %u failed\n", j);
3465         goto err_setup_tx;
3466     }
3467
3468     return 0;
3469 err_setup_tx:
3470     /* rewind the index freeing the rings as we go */
3471     while (j--)
3472         ixgbevf_free_tx_resources(adapter->xdp_ring[j]);
3473     while (i--)
3474         ixgbevf_free_tx_resources(adapter->tx_ring[i]);
3475
3476     return err;
3477 }
3478
3479 /**
3480  * ixgbevf_setup_rx_resources - allocate Rx resources (Descriptors)
3481  * @adapter: board private structure
3482  * @rx_ring: Rx descriptor ring (for a specific queue) to setup
3483  *
3484  * Returns 0 on success, negative on failure
3485  **/
3486 int ixgbevf_setup_rx_resources(struct ixgbevf_adapter *adapter,
3487                    struct ixgbevf_ring *rx_ring)
3488 {
3489     int size;
3490
3491     size = sizeof(struct ixgbevf_rx_buffer) * rx_ring->count;
3492     rx_ring->rx_buffer_info = vmalloc(size);
3493     if (!rx_ring->rx_buffer_info)
3494         goto err;
3495
3496     u64_stats_init(&rx_ring->syncp);
3497
3498     /* Round up to nearest 4K */
3499     rx_ring->size = rx_ring->count * sizeof(union ixgbe_adv_rx_desc);
3500     rx_ring->size = ALIGN(rx_ring->size, 4096);
3501
3502     rx_ring->desc = dma_alloc_coherent(rx_ring->dev, rx_ring->size,
3503                        &rx_ring->dma, GFP_KERNEL);
3504
3505     if (!rx_ring->desc)
3506         goto err;
3507
3508     /* XDP RX-queue info */
3509     if (xdp_rxq_info_reg(&rx_ring->xdp_rxq, adapter->netdev,
3510                  rx_ring->queue_index, 0) < 0)
3511         goto err;
3512
3513     rx_ring->xdp_prog = adapter->xdp_prog;
3514
3515     return 0;
3516 err:
3517     vfree(rx_ring->rx_buffer_info);
3518     rx_ring->rx_buffer_info = NULL;
3519     dev_err(rx_ring->dev, "Unable to allocate memory for the Rx descriptor ring\n");
3520     return -ENOMEM;
3521 }
3522
3523 /**
3524  * ixgbevf_setup_all_rx_resources - allocate all queues Rx resources
3525  * @adapter: board private structure
3526  *
3527  * If this function returns with an error, then it's possible one or
3528  * more of the rings is populated (while the rest are not).  It is the
3529  * callers duty to clean those orphaned rings.
3530  *
3531  * Return 0 on success, negative on failure
3532  **/
3533 static int ixgbevf_setup_all_rx_resources(struct ixgbevf_adapter *adapter)
3534 {
3535     int i, err = 0;
3536
3537     for (i = 0; i < adapter->num_rx_queues; i++) {
3538         err = ixgbevf_setup_rx_resources(adapter, adapter->rx_ring[i]);
3539         if (!err)
3540             continue;
3541         hw_dbg(&adapter->hw, "Allocation for Rx Queue %u failed\n", i);
3542         goto err_setup_rx;
3543     }
3544
3545     return 0;
3546 err_setup_rx:
3547     /* rewind the index freeing the rings as we go */
3548     while (i--)
3549         ixgbevf_free_rx_resources(adapter->rx_ring[i]);
3550     return err;
3551 }
3552
3553 /**
3554  * ixgbevf_free_rx_resources - Free Rx Resources
3555  * @rx_ring: ring to clean the resources from
3556  *
3557  * Free all receive software resources
3558  **/
3559 void ixgbevf_free_rx_resources(struct ixgbevf_ring *rx_ring)
3560 {
3561     ixgbevf_clean_rx_ring(rx_ring);
3562
3563     rx_ring->xdp_prog = NULL;
3564     xdp_rxq_info_unreg(&rx_ring->xdp_rxq);
3565     vfree(rx_ring->rx_buffer_info);
3566     rx_ring->rx_buffer_info = NULL;
3567
3568     dma_free_coherent(rx_ring->dev, rx_ring->size, rx_ring->desc,
3569               rx_ring->dma);
3570
3571     rx_ring->desc = NULL;
3572 }
3573
3574 /**
3575  * ixgbevf_free_all_rx_resources - Free Rx Resources for All Queues
3576  * @adapter: board private structure
3577  *
3578  * Free all receive software resources
3579  **/
3580 static void ixgbevf_free_all_rx_resources(struct ixgbevf_adapter *adapter)
3581 {
3582     int i;
3583
3584     for (i = 0; i < adapter->num_rx_queues; i++)
3585         if (adapter->rx_ring[i]->desc)
3586             ixgbevf_free_rx_resources(adapter->rx_ring[i]);
3587 }
3588
3589 /**
3590  * ixgbevf_open - Called when a network interface is made active
3591  * @netdev: network interface device structure
3592  *
3593  * Returns 0 on success, negative value on failure
3594  *
3595  * The open entry point is called when a network interface is made
3596  * active by the system (IFF_UP).  At this point all resources needed
3597  * for transmit and receive operations are allocated, the interrupt
3598  * handler is registered with the OS, the watchdog timer is started,
3599  * and the stack is notified that the interface is ready.
3600  **/
3601 int ixgbevf_open(struct net_device *netdev)
3602 {
3603     struct ixgbevf_adapter *adapter = netdev_priv(netdev);
3604     struct ixgbe_hw *hw = &adapter->hw;
3605     int err;
3606
3607     /* A previous failure to open the device because of a lack of
3608      * available MSIX vector resources may have reset the number
3609      * of msix vectors variable to zero.  The only way to recover
3610      * is to unload/reload the driver and hope that the system has
3611      * been able to recover some MSIX vector resources.
3612      */
3613     if (!adapter->num_msix_vectors)
3614         return -ENOMEM;
3615
3616     if (hw->adapter_stopped) {
3617         ixgbevf_reset(adapter);
3618         /* if adapter is still stopped then PF isn't up and
3619          * the VF can't start.
3620          */
3621         if (hw->adapter_stopped) {
3622             err = IXGBE_ERR_MBX;
3623             pr_err("Unable to start - perhaps the PF Driver isn't up yet\n");
3624             goto err_setup_reset;
3625         }
3626     }
3627
3628     /* disallow open during test */
3629     if (test_bit(__IXGBEVF_TESTING, &adapter->state))
3630         return -EBUSY;
3631
3632     netif_carrier_off(netdev);
3633
3634     /* allocate transmit descriptors */
3635     err = ixgbevf_setup_all_tx_resources(adapter);
3636     if (err)
3637         goto err_setup_tx;
3638
3639     /* allocate receive descriptors */
3640     err = ixgbevf_setup_all_rx_resources(adapter);
3641     if (err)
3642         goto err_setup_rx;
3643
3644     ixgbevf_configure(adapter);
3645
3646     err = ixgbevf_request_irq(adapter);
3647     if (err)
3648         goto err_req_irq;
3649
3650     /* Notify the stack of the actual queue counts. */
3651     err = netif_set_real_num_tx_queues(netdev, adapter->num_tx_queues);
3652     if (err)
3653         goto err_set_queues;
3654
3655     err = netif_set_real_num_rx_queues(netdev, adapter->num_rx_queues);
3656     if (err)
3657         goto err_set_queues;
3658
3659     ixgbevf_up_complete(adapter);
3660
3661     return 0;
3662
3663 err_set_queues:
3664     ixgbevf_free_irq(adapter);
3665 err_req_irq:
3666     ixgbevf_free_all_rx_resources(adapter);
3667 err_setup_rx:
3668     ixgbevf_free_all_tx_resources(adapter);
3669 err_setup_tx:
3670     ixgbevf_reset(adapter);
3671 err_setup_reset:
3672
3673     return err;
3674 }
3675
3676 /**
3677  * ixgbevf_close_suspend - actions necessary to both suspend and close flows
3678  * @adapter: the private adapter struct
3679  *
3680  * This function should contain the necessary work common to both suspending
3681  * and closing of the device.
3682  */
3683 static void ixgbevf_close_suspend(struct ixgbevf_adapter *adapter)
3684 {
3685     ixgbevf_down(adapter);
3686     ixgbevf_free_irq(adapter);
3687     ixgbevf_free_all_tx_resources(adapter);
3688     ixgbevf_free_all_rx_resources(adapter);
3689 }
3690
3691 /**
3692  * ixgbevf_close - Disables a network interface
3693  * @netdev: network interface device structure
3694  *
3695  * Returns 0, this is not allowed to fail
3696  *
3697  * The close entry point is called when an interface is de-activated
3698  * by the OS.  The hardware is still under the drivers control, but
3699  * needs to be disabled.  A global MAC reset is issued to stop the
3700  * hardware, and all transmit and receive resources are freed.
3701  **/
3702 int ixgbevf_close(struct net_device *netdev)
3703 {
3704     struct ixgbevf_adapter *adapter = netdev_priv(netdev);
3705
3706     if (netif_device_present(netdev))
3707         ixgbevf_close_suspend(adapter);
3708
3709     return 0;
3710 }
3711
3712 static void ixgbevf_queue_reset_subtask(struct ixgbevf_adapter *adapter)
3713 {
3714     struct net_device *dev = adapter->netdev;
3715
3716     if (!test_and_clear_bit(__IXGBEVF_QUEUE_RESET_REQUESTED,
3717                 &adapter->state))
3718         return;
3719
3720     /* if interface is down do nothing */
3721     if (test_bit(__IXGBEVF_DOWN, &adapter->state) ||
3722         test_bit(__IXGBEVF_RESETTING, &adapter->state))
3723         return;
3724
3725     /* Hardware has to reinitialize queues and interrupts to
3726      * match packet buffer alignment. Unfortunately, the
3727      * hardware is not flexible enough to do this dynamically.
3728      */
3729     rtnl_lock();
3730
3731     if (netif_running(dev))
3732         ixgbevf_close(dev);
3733
3734     ixgbevf_clear_interrupt_scheme(adapter);
3735     ixgbevf_init_interrupt_scheme(adapter);
3736
3737     if (netif_running(dev))
3738         ixgbevf_open(dev);
3739
3740     rtnl_unlock();
3741 }
3742
3743 static void ixgbevf_tx_ctxtdesc(struct ixgbevf_ring *tx_ring,
3744                 u32 vlan_macip_lens, u32 fceof_saidx,
3745                 u32 type_tucmd, u32 mss_l4len_idx)
3746 {
3747     struct ixgbe_adv_tx_context_desc *context_desc;
3748     u16 i = tx_ring->next_to_use;
3749
3750     context_desc = IXGBEVF_TX_CTXTDESC(tx_ring, i);
3751
3752     i++;
3753     tx_ring->next_to_use = (i < tx_ring->count) ? i : 0;
3754
3755     /* set bits to identify this as an advanced context descriptor */
3756     type_tucmd |= IXGBE_TXD_CMD_DEXT | IXGBE_ADVTXD_DTYP_CTXT;
3757
3758     context_desc->vlan_macip_lens   = cpu_to_le32(vlan_macip_lens);
3759     context_desc->fceof_saidx   = cpu_to_le32(fceof_saidx);
3760     context_desc->type_tucmd_mlhl   = cpu_to_le32(type_tucmd);
3761     context_desc->mss_l4len_idx = cpu_to_le32(mss_l4len_idx);
3762 }
3763
3764 static int ixgbevf_tso(struct ixgbevf_ring *tx_ring,
3765                struct ixgbevf_tx_buffer *first,
3766                u8 *hdr_len,
3767                struct ixgbevf_ipsec_tx_data *itd)
3768 {
3769     u32 vlan_macip_lens, type_tucmd, mss_l4len_idx;
3770     struct sk_buff *skb = first->skb;
3771     union {
3772         struct iphdr *v4;
3773         struct ipv6hdr *v6;
3774         unsigned char *hdr;
3775     } ip;
3776     union {
3777         struct tcphdr *tcp;
3778         unsigned char *hdr;
3779     } l4;
3780     u32 paylen, l4_offset;
3781     u32 fceof_saidx = 0;
3782     int err;
3783
3784     if (skb->ip_summed != CHECKSUM_PARTIAL)
3785         return 0;
3786
3787     if (!skb_is_gso(skb))
3788         return 0;
3789
3790     err = skb_cow_head(skb, 0);
3791     if (err < 0)
3792         return err;
3793
3794     if (eth_p_mpls(first->protocol))
3795         ip.hdr = skb_inner_network_header(skb);
3796     else
3797         ip.hdr = skb_network_header(skb);
3798     l4.hdr = skb_checksum_start(skb);
3799
3800     /* ADV DTYP TUCMD MKRLOC/ISCSIHEDLEN */
3801     type_tucmd = IXGBE_ADVTXD_TUCMD_L4T_TCP;
3802
3803     /* initialize outer IP header fields */
3804     if (ip.v4->version == 4) {
3805         unsigned char *csum_start = skb_checksum_start(skb);
3806         unsigned char *trans_start = ip.hdr + (ip.v4->ihl * 4);
3807         int len = csum_start - trans_start;
3808
3809         /* IP header will have to cancel out any data that
3810          * is not a part of the outer IP header, so set to
3811          * a reverse csum if needed, else init check to 0.
3812          */
3813         ip.v4->check = (skb_shinfo(skb)->gso_type & SKB_GSO_PARTIAL) ?
3814                        csum_fold(csum_partial(trans_start,
3815                                   len, 0)) : 0;
3816         type_tucmd |= IXGBE_ADVTXD_TUCMD_IPV4;
3817
3818         ip.v4->tot_len = 0;
3819         first->tx_flags |= IXGBE_TX_FLAGS_TSO |
3820                    IXGBE_TX_FLAGS_CSUM |
3821                    IXGBE_TX_FLAGS_IPV4;
3822     } else {
3823         ip.v6->payload_len = 0;
3824         first->tx_flags |= IXGBE_TX_FLAGS_TSO |
3825                    IXGBE_TX_FLAGS_CSUM;
3826     }
3827
3828     /* determine offset of inner transport header */
3829     l4_offset = l4.hdr - skb->data;
3830
3831     /* compute length of segmentation header */
3832     *hdr_len = (l4.tcp->doff * 4) + l4_offset;
3833
3834     /* remove payload length from inner checksum */
3835     paylen = skb->len - l4_offset;
3836     csum_replace_by_diff(&l4.tcp->check, (__force __wsum)htonl(paylen));
3837
3838     /* update gso size and bytecount with header size */
3839     first->gso_segs = skb_shinfo(skb)->gso_segs;
3840     first->bytecount += (first->gso_segs - 1) * *hdr_len;
3841
3842     /* mss_l4len_id: use 1 as index for TSO */
3843     mss_l4len_idx = (*hdr_len - l4_offset) << IXGBE_ADVTXD_L4LEN_SHIFT;
3844     mss_l4len_idx |= skb_shinfo(skb)->gso_size << IXGBE_ADVTXD_MSS_SHIFT;
3845     mss_l4len_idx |= (1u << IXGBE_ADVTXD_IDX_SHIFT);
3846
3847     fceof_saidx |= itd->pfsa;
3848     type_tucmd |= itd->flags | itd->trailer_len;
3849
3850     /* vlan_macip_lens: HEADLEN, MACLEN, VLAN tag */
3851     vlan_macip_lens = l4.hdr - ip.hdr;
3852     vlan_macip_lens |= (ip.hdr - skb->data) << IXGBE_ADVTXD_MACLEN_SHIFT;
3853     vlan_macip_lens |= first->tx_flags & IXGBE_TX_FLAGS_VLAN_MASK;
3854
3855     ixgbevf_tx_ctxtdesc(tx_ring, vlan_macip_lens, fceof_saidx, type_tucmd,
3856                 mss_l4len_idx);
3857
3858     return 1;
3859 }
3860
3861 static void ixgbevf_tx_csum(struct ixgbevf_ring *tx_ring,
3862                 struct ixgbevf_tx_buffer *first,
3863                 struct ixgbevf_ipsec_tx_data *itd)
3864 {
3865     struct sk_buff *skb = first->skb;
3866     u32 vlan_macip_lens = 0;
3867     u32 fceof_saidx = 0;
3868     u32 type_tucmd = 0;
3869
3870     if (skb->ip_summed != CHECKSUM_PARTIAL)
3871         goto no_csum;
3872
3873     switch (skb->csum_offset) {
3874     case offsetof(struct tcphdr, check):
3875         type_tucmd = IXGBE_ADVTXD_TUCMD_L4T_TCP;
3876         fallthrough;
3877     case offsetof(struct udphdr, check):
3878         break;
3879     case offsetof(struct sctphdr, checksum):
3880         /* validate that this is actually an SCTP request */
3881         if (skb_csum_is_sctp(skb)) {
3882             type_tucmd = IXGBE_ADVTXD_TUCMD_L4T_SCTP;
3883             break;
3884         }
3885         fallthrough;
3886     default:
3887         skb_checksum_help(skb);
3888         goto no_csum;
3889     }
3890
3891     if (first->protocol == htons(ETH_P_IP))
3892         type_tucmd |= IXGBE_ADVTXD_TUCMD_IPV4;
3893
3894     /* update TX checksum flag */
3895     first->tx_flags |= IXGBE_TX_FLAGS_CSUM;
3896     vlan_macip_lens = skb_checksum_start_offset(skb) -
3897               skb_network_offset(skb);
3898 no_csum:
3899     /* vlan_macip_lens: MACLEN, VLAN tag */
3900     vlan_macip_lens |= skb_network_offset(skb) << IXGBE_ADVTXD_MACLEN_SHIFT;
3901     vlan_macip_lens |= first->tx_flags & IXGBE_TX_FLAGS_VLAN_MASK;
3902
3903     fceof_saidx |= itd->pfsa;
3904     type_tucmd |= itd->flags | itd->trailer_len;
3905
3906     ixgbevf_tx_ctxtdesc(tx_ring, vlan_macip_lens,
3907                 fceof_saidx, type_tucmd, 0);
3908 }
3909
3910 static __le32 ixgbevf_tx_cmd_type(u32 tx_flags)
3911 {
3912     /* set type for advanced descriptor with frame checksum insertion */
3913     __le32 cmd_type = cpu_to_le32(IXGBE_ADVTXD_DTYP_DATA |
3914                       IXGBE_ADVTXD_DCMD_IFCS |
3915                       IXGBE_ADVTXD_DCMD_DEXT);
3916
3917     /* set HW VLAN bit if VLAN is present */
3918     if (tx_flags & IXGBE_TX_FLAGS_VLAN)
3919         cmd_type |= cpu_to_le32(IXGBE_ADVTXD_DCMD_VLE);
3920
3921     /* set segmentation enable bits for TSO/FSO */
3922     if (tx_flags & IXGBE_TX_FLAGS_TSO)
3923         cmd_type |= cpu_to_le32(IXGBE_ADVTXD_DCMD_TSE);
3924
3925     return cmd_type;
3926 }
3927
3928 static void ixgbevf_tx_olinfo_status(union ixgbe_adv_tx_desc *tx_desc,
3929                      u32 tx_flags, unsigned int paylen)
3930 {
3931     __le32 olinfo_status = cpu_to_le32(paylen << IXGBE_ADVTXD_PAYLEN_SHIFT);
3932
3933     /* enable L4 checksum for TSO and TX checksum offload */
3934     if (tx_flags & IXGBE_TX_FLAGS_CSUM)
3935         olinfo_status |= cpu_to_le32(IXGBE_ADVTXD_POPTS_TXSM);
3936
3937     /* enble IPv4 checksum for TSO */
3938     if (tx_flags & IXGBE_TX_FLAGS_IPV4)
3939         olinfo_status |= cpu_to_le32(IXGBE_ADVTXD_POPTS_IXSM);
3940
3941     /* enable IPsec */
3942     if (tx_flags & IXGBE_TX_FLAGS_IPSEC)
3943         olinfo_status |= cpu_to_le32(IXGBE_ADVTXD_POPTS_IPSEC);
3944
3945     /* use index 1 context for TSO/FSO/FCOE/IPSEC */
3946     if (tx_flags & (IXGBE_TX_FLAGS_TSO | IXGBE_TX_FLAGS_IPSEC))
3947         olinfo_status |= cpu_to_le32(1u << IXGBE_ADVTXD_IDX_SHIFT);
3948
3949     /* Check Context must be set if Tx switch is enabled, which it
3950      * always is for case where virtual functions are running
3951      */
3952     olinfo_status |= cpu_to_le32(IXGBE_ADVTXD_CC);
3953
3954     tx_desc->read.olinfo_status = olinfo_status;
3955 }
3956
3957 static void ixgbevf_tx_map(struct ixgbevf_ring *tx_ring,
3958                struct ixgbevf_tx_buffer *first,
3959                const u8 hdr_len)
3960 {
3961     struct sk_buff *skb = first->skb;
3962     struct ixgbevf_tx_buffer *tx_buffer;
3963     union ixgbe_adv_tx_desc *tx_desc;
3964     skb_frag_t *frag;
3965     dma_addr_t dma;
3966     unsigned int data_len, size;
3967     u32 tx_flags = first->tx_flags;
3968     __le32 cmd_type = ixgbevf_tx_cmd_type(tx_flags);
3969     u16 i = tx_ring->next_to_use;
3970
3971     tx_desc = IXGBEVF_TX_DESC(tx_ring, i);
3972
3973     ixgbevf_tx_olinfo_status(tx_desc, tx_flags, skb->len - hdr_len);
3974
3975     size = skb_headlen(skb);
3976     data_len = skb->data_len;
3977
3978     dma = dma_map_single(tx_ring->dev, skb->data, size, DMA_TO_DEVICE);
3979
3980     tx_buffer = first;
3981
3982     for (frag = &skb_shinfo(skb)->frags[0];; frag++) {
3983         if (dma_mapping_error(tx_ring->dev, dma))
3984             goto dma_error;
3985
3986         /* record length, and DMA address */
3987         dma_unmap_len_set(tx_buffer, len, size);
3988         dma_unmap_addr_set(tx_buffer, dma, dma);
3989
3990         tx_desc->read.buffer_addr = cpu_to_le64(dma);
3991
3992         while (unlikely(size > IXGBE_MAX_DATA_PER_TXD)) {
3993             tx_desc->read.cmd_type_len =
3994                 cmd_type | cpu_to_le32(IXGBE_MAX_DATA_PER_TXD);
3995
3996             i++;
3997             tx_desc++;
3998             if (i == tx_ring->count) {
3999                 tx_desc = IXGBEVF_TX_DESC(tx_ring, 0);
4000                 i = 0;
4001             }
4002             tx_desc->read.olinfo_status = 0;
4003
4004             dma += IXGBE_MAX_DATA_PER_TXD;
4005             size -= IXGBE_MAX_DATA_PER_TXD;
4006
4007             tx_desc->read.buffer_addr = cpu_to_le64(dma);
4008         }
4009
4010         if (likely(!data_len))
4011             break;
4012
4013         tx_desc->read.cmd_type_len = cmd_type | cpu_to_le32(size);
4014
4015         i++;
4016         tx_desc++;
4017         if (i == tx_ring->count) {
4018             tx_desc = IXGBEVF_TX_DESC(tx_ring, 0);
4019             i = 0;
4020         }
4021         tx_desc->read.olinfo_status = 0;
4022
4023         size = skb_frag_size(frag);
4024         data_len -= size;
4025
4026         dma = skb_frag_dma_map(tx_ring->dev, frag, 0, size,
4027                        DMA_TO_DEVICE);
4028
4029         tx_buffer = &tx_ring->tx_buffer_info[i];
4030     }
4031
4032     /* write last descriptor with RS and EOP bits */
4033     cmd_type |= cpu_to_le32(size) | cpu_to_le32(IXGBE_TXD_CMD);
4034     tx_desc->read.cmd_type_len = cmd_type;
4035
4036     /* set the timestamp */
4037     first->time_stamp = jiffies;
4038
4039     skb_tx_timestamp(skb);
4040
4041     /* Force memory writes to complete before letting h/w know there
4042      * are new descriptors to fetch.  (Only applicable for weak-ordered
4043      * memory model archs, such as IA-64).
4044      *
4045      * We also need this memory barrier (wmb) to make certain all of the
4046      * status bits have been updated before next_to_watch is written.
4047      */
4048     wmb();
4049
4050     /* set next_to_watch value indicating a packet is present */
4051     first->next_to_watch = tx_desc;
4052
4053     i++;
4054     if (i == tx_ring->count)
4055         i = 0;
4056
4057     tx_ring->next_to_use = i;
4058
4059     /* notify HW of packet */
4060     ixgbevf_write_tail(tx_ring, i);
4061
4062     return;
4063 dma_error:
4064     dev_err(tx_ring->dev, "TX DMA map failed\n");
4065     tx_buffer = &tx_ring->tx_buffer_info[i];
4066
4067     /* clear dma mappings for failed tx_buffer_info map */
4068     while (tx_buffer != first) {
4069         if (dma_unmap_len(tx_buffer, len))
4070             dma_unmap_page(tx_ring->dev,
4071                        dma_unmap_addr(tx_buffer, dma),
4072                        dma_unmap_len(tx_buffer, len),
4073                        DMA_TO_DEVICE);
4074         dma_unmap_len_set(tx_buffer, len, 0);
4075
4076         if (i-- == 0)
4077             i += tx_ring->count;
4078         tx_buffer = &tx_ring->tx_buffer_info[i];
4079     }
4080
4081     if (dma_unmap_len(tx_buffer, len))
4082         dma_unmap_single(tx_ring->dev,
4083                  dma_unmap_addr(tx_buffer, dma),
4084                  dma_unmap_len(tx_buffer, len),
4085                  DMA_TO_DEVICE);
4086     dma_unmap_len_set(tx_buffer, len, 0);
4087
4088     dev_kfree_skb_any(tx_buffer->skb);
4089     tx_buffer->skb = NULL;
4090
4091     tx_ring->next_to_use = i;
4092 }
4093
4094 static int __ixgbevf_maybe_stop_tx(struct ixgbevf_ring *tx_ring, int size)
4095 {
4096     netif_stop_subqueue(tx_ring->netdev, tx_ring->queue_index);
4097     /* Herbert's original patch had:
4098      *  smp_mb__after_netif_stop_queue();
4099      * but since that doesn't exist yet, just open code it.
4100      */
4101     smp_mb();
4102
4103     /* We need to check again in a case another CPU has just
4104      * made room available.
4105      */
4106     if (likely(ixgbevf_desc_unused(tx_ring) < size))
4107         return -EBUSY;
4108
4109     /* A reprieve! - use start_queue because it doesn't call schedule */
4110     netif_start_subqueue(tx_ring->netdev, tx_ring->queue_index);
4111     ++tx_ring->tx_stats.restart_queue;
4112
4113     return 0;
4114 }
4115
4116 static int ixgbevf_maybe_stop_tx(struct ixgbevf_ring *tx_ring, int size)
4117 {
4118     if (likely(ixgbevf_desc_unused(tx_ring) >= size))
4119         return 0;
4120     return __ixgbevf_maybe_stop_tx(tx_ring, size);
4121 }
4122
4123 static int ixgbevf_xmit_frame_ring(struct sk_buff *skb,
4124                    struct ixgbevf_ring *tx_ring)
4125 {
4126     struct ixgbevf_tx_buffer *first;
4127     int tso;
4128     u32 tx_flags = 0;
4129     u16 count = TXD_USE_COUNT(skb_headlen(skb));
4130     struct ixgbevf_ipsec_tx_data ipsec_tx = { 0 };
4131 #if PAGE_SIZE > IXGBE_MAX_DATA_PER_TXD
4132     unsigned short f;
4133 #endif
4134     u8 hdr_len = 0;
4135     u8 *dst_mac = skb_header_pointer(skb, 0, 0, NULL);
4136
4137     if (!dst_mac || is_link_local_ether_addr(dst_mac)) {
4138         dev_kfree_skb_any(skb);
4139         return NETDEV_TX_OK;
4140     }
4141
4142     /* need: 1 descriptor per page * PAGE_SIZE/IXGBE_MAX_DATA_PER_TXD,
4143      *       + 1 desc for skb_headlen/IXGBE_MAX_DATA_PER_TXD,
4144      *       + 2 desc gap to keep tail from touching head,
4145      *       + 1 desc for context descriptor,
4146      * otherwise try next time
4147      */
4148 #if PAGE_SIZE > IXGBE_MAX_DATA_PER_TXD
4149     for (f = 0; f < skb_shinfo(skb)->nr_frags; f++) {
4150         skb_frag_t *frag = &skb_shinfo(skb)->frags[f];
4151
4152         count += TXD_USE_COUNT(skb_frag_size(frag));
4153     }
4154 #else
4155     count += skb_shinfo(skb)->nr_frags;
4156 #endif
4157     if (ixgbevf_maybe_stop_tx(tx_ring, count + 3)) {
4158         tx_ring->tx_stats.tx_busy++;
4159         return NETDEV_TX_BUSY;
4160     }
4161
4162     /* record the location of the first descriptor for this packet */
4163     first = &tx_ring->tx_buffer_info[tx_ring->next_to_use];
4164     first->skb = skb;
4165     first->bytecount = skb->len;
4166     first->gso_segs = 1;
4167
4168     if (skb_vlan_tag_present(skb)) {
4169         tx_flags |= skb_vlan_tag_get(skb);
4170         tx_flags <<= IXGBE_TX_FLAGS_VLAN_SHIFT;
4171         tx_flags |= IXGBE_TX_FLAGS_VLAN;
4172     }
4173
4174     /* record initial flags and protocol */
4175     first->tx_flags = tx_flags;
4176     first->protocol = vlan_get_protocol(skb);
4177
4178 #ifdef CONFIG_IXGBEVF_IPSEC
4179     if (xfrm_offload(skb) && !ixgbevf_ipsec_tx(tx_ring, first, &ipsec_tx))
4180         goto out_drop;
4181 #endif
4182     tso = ixgbevf_tso(tx_ring, first, &hdr_len, &ipsec_tx);
4183     if (tso < 0)
4184         goto out_drop;
4185     else if (!tso)
4186         ixgbevf_tx_csum(tx_ring, first, &ipsec_tx);
4187
4188     ixgbevf_tx_map(tx_ring, first, hdr_len);
4189
4190     ixgbevf_maybe_stop_tx(tx_ring, DESC_NEEDED);
4191
4192     return NETDEV_TX_OK;
4193
4194 out_drop:
4195     dev_kfree_skb_any(first->skb);
4196     first->skb = NULL;
4197
4198     return NETDEV_TX_OK;
4199 }
4200
4201 static netdev_tx_t ixgbevf_xmit_frame(struct sk_buff *skb, struct net_device *netdev)
4202 {
4203     struct ixgbevf_adapter *adapter = netdev_priv(netdev);
4204     struct ixgbevf_ring *tx_ring;
4205
4206     if (skb->len <= 0) {
4207         dev_kfree_skb_any(skb);
4208         return NETDEV_TX_OK;
4209     }
4210
4211     /* The minimum packet size for olinfo paylen is 17 so pad the skb
4212      * in order to meet this minimum size requirement.
4213      */
4214     if (skb->len < 17) {
4215         if (skb_padto(skb, 17))
4216             return NETDEV_TX_OK;
4217         skb->len = 17;
4218     }
4219
4220     tx_ring = adapter->tx_ring[skb->queue_mapping];
4221     return ixgbevf_xmit_frame_ring(skb, tx_ring);
4222 }
4223
4224 /**
4225  * ixgbevf_set_mac - Change the Ethernet Address of the NIC
4226  * @netdev: network interface device structure
4227  * @p: pointer to an address structure
4228  *
4229  * Returns 0 on success, negative on failure
4230  **/
4231 static int ixgbevf_set_mac(struct net_device *netdev, void *p)
4232 {
4233     struct ixgbevf_adapter *adapter = netdev_priv(netdev);
4234     struct ixgbe_hw *hw = &adapter->hw;
4235     struct sockaddr *addr = p;
4236     int err;
4237
4238     if (!is_valid_ether_addr(addr->sa_data))
4239         return -EADDRNOTAVAIL;
4240
4241     spin_lock_bh(&adapter->mbx_lock);
4242
4243     err = hw->mac.ops.set_rar(hw, 0, addr->sa_data, 0);
4244
4245     spin_unlock_bh(&adapter->mbx_lock);
4246
4247     if (err)
4248         return -EPERM;
4249
4250     ether_addr_copy(hw->mac.addr, addr->sa_data);
4251     ether_addr_copy(hw->mac.perm_addr, addr->sa_data);
4252     eth_hw_addr_set(netdev, addr->sa_data);
4253
4254     return 0;
4255 }
4256
4257 /**
4258  * ixgbevf_change_mtu - Change the Maximum Transfer Unit
4259  * @netdev: network interface device structure
4260  * @new_mtu: new value for maximum frame size
4261  *
4262  * Returns 0 on success, negative on failure
4263  **/
4264 static int ixgbevf_change_mtu(struct net_device *netdev, int new_mtu)
4265 {
4266     struct ixgbevf_adapter *adapter = netdev_priv(netdev);
4267     struct ixgbe_hw *hw = &adapter->hw;
4268     int max_frame = new_mtu + ETH_HLEN + ETH_FCS_LEN;
4269     int ret;
4270
4271     /* prevent MTU being changed to a size unsupported by XDP */
4272     if (adapter->xdp_prog) {
4273         dev_warn(&adapter->pdev->dev, "MTU cannot be changed while XDP program is loaded\n");
4274         return -EPERM;
4275     }
4276
4277     spin_lock_bh(&adapter->mbx_lock);
4278     /* notify the PF of our intent to use this size of frame */
4279     ret = hw->mac.ops.set_rlpml(hw, max_frame);
4280     spin_unlock_bh(&adapter->mbx_lock);
4281     if (ret)
4282         return -EINVAL;
4283
4284     hw_dbg(hw, "changing MTU from %d to %d\n",
4285            netdev->mtu, new_mtu);
4286
4287     /* must set new MTU before calling down or up */
4288     netdev->mtu = new_mtu;
4289
4290     if (netif_running(netdev))
4291         ixgbevf_reinit_locked(adapter);
4292
4293     return 0;
4294 }
4295
4296 static int __maybe_unused ixgbevf_suspend(struct device *dev_d)
4297 {
4298     struct net_device *netdev = dev_get_drvdata(dev_d);
4299     struct ixgbevf_adapter *adapter = netdev_priv(netdev);
4300
4301     rtnl_lock();
4302     netif_device_detach(netdev);
4303
4304     if (netif_running(netdev))
4305         ixgbevf_close_suspend(adapter);
4306
4307     ixgbevf_clear_interrupt_scheme(adapter);
4308     rtnl_unlock();
4309
4310     return 0;
4311 }
4312
4313 static int __maybe_unused ixgbevf_resume(struct device *dev_d)
4314 {
4315     struct pci_dev *pdev = to_pci_dev(dev_d);
4316     struct net_device *netdev = pci_get_drvdata(pdev);
4317     struct ixgbevf_adapter *adapter = netdev_priv(netdev);
4318     u32 err;
4319
4320     adapter->hw.hw_addr = adapter->io_addr;
4321     smp_mb__before_atomic();
4322     clear_bit(__IXGBEVF_DISABLED, &adapter->state);
4323     pci_set_master(pdev);
4324
4325     ixgbevf_reset(adapter);
4326
4327     rtnl_lock();
4328     err = ixgbevf_init_interrupt_scheme(adapter);
4329     if (!err && netif_running(netdev))
4330         err = ixgbevf_open(netdev);
4331     rtnl_unlock();
4332     if (err)
4333         return err;
4334
4335     netif_device_attach(netdev);
4336
4337     return err;
4338 }
4339
4340 static void ixgbevf_shutdown(struct pci_dev *pdev)
4341 {
4342     ixgbevf_suspend(&pdev->dev);
4343 }
4344
4345 static void ixgbevf_get_tx_ring_stats(struct rtnl_link_stats64 *stats,
4346                       const struct ixgbevf_ring *ring)
4347 {
4348     u64 bytes, packets;
4349     unsigned int start;
4350
4351     if (ring) {
4352         do {
4353             start = u64_stats_fetch_begin_irq(&ring->syncp);
4354             bytes = ring->stats.bytes;
4355             packets = ring->stats.packets;
4356         } while (u64_stats_fetch_retry_irq(&ring->syncp, start));
4357         stats->tx_bytes += bytes;
4358         stats->tx_packets += packets;
4359     }
4360 }
4361
4362 static void ixgbevf_get_stats(struct net_device *netdev,
4363                   struct rtnl_link_stats64 *stats)
4364 {
4365     struct ixgbevf_adapter *adapter = netdev_priv(netdev);
4366     unsigned int start;
4367     u64 bytes, packets;
4368     const struct ixgbevf_ring *ring;
4369     int i;
4370
4371     ixgbevf_update_stats(adapter);
4372
4373     stats->multicast = adapter->stats.vfmprc - adapter->stats.base_vfmprc;
4374
4375     rcu_read_lock();
4376     for (i = 0; i < adapter->num_rx_queues; i++) {
4377         ring = adapter->rx_ring[i];
4378         do {
4379             start = u64_stats_fetch_begin_irq(&ring->syncp);
4380             bytes = ring->stats.bytes;
4381             packets = ring->stats.packets;
4382         } while (u64_stats_fetch_retry_irq(&ring->syncp, start));
4383         stats->rx_bytes += bytes;
4384         stats->rx_packets += packets;
4385     }
4386
4387     for (i = 0; i < adapter->num_tx_queues; i++) {
4388         ring = adapter->tx_ring[i];
4389         ixgbevf_get_tx_ring_stats(stats, ring);
4390     }
4391
4392     for (i = 0; i < adapter->num_xdp_queues; i++) {
4393         ring = adapter->xdp_ring[i];
4394         ixgbevf_get_tx_ring_stats(stats, ring);
4395     }
4396     rcu_read_unlock();
4397 }
4398
4399 #define IXGBEVF_MAX_MAC_HDR_LEN     127
4400 #define IXGBEVF_MAX_NETWORK_HDR_LEN 511
4401
4402 static netdev_features_t
4403 ixgbevf_features_check(struct sk_buff *skb, struct net_device *dev,
4404                netdev_features_t features)
4405 {
4406     unsigned int network_hdr_len, mac_hdr_len;
4407
4408     /* Make certain the headers can be described by a context descriptor */
4409     mac_hdr_len = skb_network_header(skb) - skb->data;
4410     if (unlikely(mac_hdr_len > IXGBEVF_MAX_MAC_HDR_LEN))
4411         return features & ~(NETIF_F_HW_CSUM |
4412                     NETIF_F_SCTP_CRC |
4413                     NETIF_F_HW_VLAN_CTAG_TX |
4414                     NETIF_F_TSO |
4415                     NETIF_F_TSO6);
4416
4417     network_hdr_len = skb_checksum_start(skb) - skb_network_header(skb);
4418     if (unlikely(network_hdr_len >  IXGBEVF_MAX_NETWORK_HDR_LEN))
4419         return features & ~(NETIF_F_HW_CSUM |
4420                     NETIF_F_SCTP_CRC |
4421                     NETIF_F_TSO |
4422                     NETIF_F_TSO6);
4423
4424     /* We can only support IPV4 TSO in tunnels if we can mangle the
4425      * inner IP ID field, so strip TSO if MANGLEID is not supported.
4426      */
4427     if (skb->encapsulation && !(features & NETIF_F_TSO_MANGLEID))
4428         features &= ~NETIF_F_TSO;
4429
4430     return features;
4431 }
4432
4433 static int ixgbevf_xdp_setup(struct net_device *dev, struct bpf_prog *prog)
4434 {
4435     int i, frame_size = dev->mtu + ETH_HLEN + ETH_FCS_LEN + VLAN_HLEN;
4436     struct ixgbevf_adapter *adapter = netdev_priv(dev);
4437     struct bpf_prog *old_prog;
4438
4439     /* verify ixgbevf ring attributes are sufficient for XDP */
4440     for (i = 0; i < adapter->num_rx_queues; i++) {
4441         struct ixgbevf_ring *ring = adapter->rx_ring[i];
4442
4443         if (frame_size > ixgbevf_rx_bufsz(ring))
4444             return -EINVAL;
4445     }
4446
4447     old_prog = xchg(&adapter->xdp_prog, prog);
4448
4449     /* If transitioning XDP modes reconfigure rings */
4450     if (!!prog != !!old_prog) {
4451         /* Hardware has to reinitialize queues and interrupts to
4452          * match packet buffer alignment. Unfortunately, the
4453          * hardware is not flexible enough to do this dynamically.
4454          */
4455         if (netif_running(dev))
4456             ixgbevf_close(dev);
4457
4458         ixgbevf_clear_interrupt_scheme(adapter);
4459         ixgbevf_init_interrupt_scheme(adapter);
4460
4461         if (netif_running(dev))
4462             ixgbevf_open(dev);
4463     } else {
4464         for (i = 0; i < adapter->num_rx_queues; i++)
4465             xchg(&adapter->rx_ring[i]->xdp_prog, adapter->xdp_prog);
4466     }
4467
4468     if (old_prog)
4469         bpf_prog_put(old_prog);
4470
4471     return 0;
4472 }
4473
4474 static int ixgbevf_xdp(struct net_device *dev, struct netdev_bpf *xdp)
4475 {
4476     switch (xdp->command) {
4477     case XDP_SETUP_PROG:
4478         return ixgbevf_xdp_setup(dev, xdp->prog);
4479     default:
4480         return -EINVAL;
4481     }
4482 }
4483
4484 static const struct net_device_ops ixgbevf_netdev_ops = {
4485     .ndo_open       = ixgbevf_open,
4486     .ndo_stop       = ixgbevf_close,
4487     .ndo_start_xmit     = ixgbevf_xmit_frame,
4488     .ndo_set_rx_mode    = ixgbevf_set_rx_mode,
4489     .ndo_get_stats64    = ixgbevf_get_stats,
4490     .ndo_validate_addr  = eth_validate_addr,
4491     .ndo_set_mac_address    = ixgbevf_set_mac,
4492     .ndo_change_mtu     = ixgbevf_change_mtu,
4493     .ndo_tx_timeout     = ixgbevf_tx_timeout,
4494     .ndo_vlan_rx_add_vid    = ixgbevf_vlan_rx_add_vid,
4495     .ndo_vlan_rx_kill_vid   = ixgbevf_vlan_rx_kill_vid,
4496     .ndo_features_check = ixgbevf_features_check,
4497     .ndo_bpf        = ixgbevf_xdp,
4498 };
4499
4500 static void ixgbevf_assign_netdev_ops(struct net_device *dev)
4501 {
4502     dev->netdev_ops = &ixgbevf_netdev_ops;
4503     ixgbevf_set_ethtool_ops(dev);
4504     dev->watchdog_timeo = 5 * HZ;
4505 }
4506
4507 /**
4508  * ixgbevf_probe - Device Initialization Routine
4509  * @pdev: PCI device information struct
4510  * @ent: entry in ixgbevf_pci_tbl
4511  *
4512  * Returns 0 on success, negative on failure
4513  *
4514  * ixgbevf_probe initializes an adapter identified by a pci_dev structure.
4515  * The OS initialization, configuring of the adapter private structure,
4516  * and a hardware reset occur.
4517  **/
4518 static int ixgbevf_probe(struct pci_dev *pdev, const struct pci_device_id *ent)
4519 {
4520     struct net_device *netdev;
4521     struct ixgbevf_adapter *adapter = NULL;
4522     struct ixgbe_hw *hw = NULL;
4523     const struct ixgbevf_info *ii = ixgbevf_info_tbl[ent->driver_data];
4524     bool disable_dev = false;
4525     int err;
4526
4527     err = pci_enable_device(pdev);
4528     if (err)
4529         return err;
4530
4531     err = dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(64));
4532     if (err) {
4533         dev_err(&pdev->dev, "No usable DMA configuration, aborting\n");
4534         goto err_dma;
4535     }
4536
4537     err = pci_request_regions(pdev, ixgbevf_driver_name);
4538     if (err) {
4539         dev_err(&pdev->dev, "pci_request_regions failed 0x%x\n", err);
4540         goto err_pci_reg;
4541     }
4542
4543     pci_set_master(pdev);
4544
4545     netdev = alloc_etherdev_mq(sizeof(struct ixgbevf_adapter),
4546                    MAX_TX_QUEUES);
4547     if (!netdev) {
4548         err = -ENOMEM;
4549         goto err_alloc_etherdev;
4550     }
4551
4552     SET_NETDEV_DEV(netdev, &pdev->dev);
4553
4554     adapter = netdev_priv(netdev);
4555
4556     adapter->netdev = netdev;
4557     adapter->pdev = pdev;
4558     hw = &adapter->hw;
4559     hw->back = adapter;
4560     adapter->msg_enable = netif_msg_init(debug, DEFAULT_MSG_ENABLE);
4561
4562     /* call save state here in standalone driver because it relies on
4563      * adapter struct to exist, and needs to call netdev_priv
4564      */
4565     pci_save_state(pdev);
4566
4567     hw->hw_addr = ioremap(pci_resource_start(pdev, 0),
4568                   pci_resource_len(pdev, 0));
4569     adapter->io_addr = hw->hw_addr;
4570     if (!hw->hw_addr) {
4571         err = -EIO;
4572         goto err_ioremap;
4573     }
4574
4575     ixgbevf_assign_netdev_ops(netdev);
4576
4577     /* Setup HW API */
4578     memcpy(&hw->mac.ops, ii->mac_ops, sizeof(hw->mac.ops));
4579     hw->mac.type  = ii->mac;
4580
4581     memcpy(&hw->mbx.ops, &ixgbevf_mbx_ops_legacy,
4582            sizeof(struct ixgbe_mbx_operations));
4583
4584     /* setup the private structure */
4585     err = ixgbevf_sw_init(adapter);
4586     if (err)
4587         goto err_sw_init;
4588
4589     /* The HW MAC address was set and/or determined in sw_init */
4590     if (!is_valid_ether_addr(netdev->dev_addr)) {
4591         pr_err("invalid MAC address\n");
4592         err = -EIO;
4593         goto err_sw_init;
4594     }
4595
4596     netdev->hw_features = NETIF_F_SG |
4597                   NETIF_F_TSO |
4598                   NETIF_F_TSO6 |
4599                   NETIF_F_RXCSUM |
4600                   NETIF_F_HW_CSUM |
4601                   NETIF_F_SCTP_CRC;
4602
4603 #define IXGBEVF_GSO_PARTIAL_FEATURES (NETIF_F_GSO_GRE | \
4604                       NETIF_F_GSO_GRE_CSUM | \
4605                       NETIF_F_GSO_IPXIP4 | \
4606                       NETIF_F_GSO_IPXIP6 | \
4607                       NETIF_F_GSO_UDP_TUNNEL | \
4608                       NETIF_F_GSO_UDP_TUNNEL_CSUM)
4609
4610     netdev->gso_partial_features = IXGBEVF_GSO_PARTIAL_FEATURES;
4611     netdev->hw_features |= NETIF_F_GSO_PARTIAL |
4612                    IXGBEVF_GSO_PARTIAL_FEATURES;
4613
4614     netdev->features = netdev->hw_features | NETIF_F_HIGHDMA;
4615
4616     netdev->vlan_features |= netdev->features | NETIF_F_TSO_MANGLEID;
4617     netdev->mpls_features |= NETIF_F_SG |
4618                  NETIF_F_TSO |
4619                  NETIF_F_TSO6 |
4620                  NETIF_F_HW_CSUM;
4621     netdev->mpls_features |= IXGBEVF_GSO_PARTIAL_FEATURES;
4622     netdev->hw_enc_features |= netdev->vlan_features;
4623
4624     /* set this bit last since it cannot be part of vlan_features */
4625     netdev->features |= NETIF_F_HW_VLAN_CTAG_FILTER |
4626                 NETIF_F_HW_VLAN_CTAG_RX |
4627                 NETIF_F_HW_VLAN_CTAG_TX;
4628
4629     netdev->priv_flags |= IFF_UNICAST_FLT;
4630
4631     /* MTU range: 68 - 1504 or 9710 */
4632     netdev->min_mtu = ETH_MIN_MTU;
4633     switch (adapter->hw.api_version) {
4634     case ixgbe_mbox_api_11:
4635     case ixgbe_mbox_api_12:
4636     case ixgbe_mbox_api_13:
4637     case ixgbe_mbox_api_14:
4638     case ixgbe_mbox_api_15:
4639         netdev->max_mtu = IXGBE_MAX_JUMBO_FRAME_SIZE -
4640                   (ETH_HLEN + ETH_FCS_LEN);
4641         break;
4642     default:
4643         if (adapter->hw.mac.type != ixgbe_mac_82599_vf)
4644             netdev->max_mtu = IXGBE_MAX_JUMBO_FRAME_SIZE -
4645                       (ETH_HLEN + ETH_FCS_LEN);
4646         else
4647             netdev->max_mtu = ETH_DATA_LEN + ETH_FCS_LEN;
4648         break;
4649     }
4650
4651     if (IXGBE_REMOVED(hw->hw_addr)) {
4652         err = -EIO;
4653         goto err_sw_init;
4654     }
4655
4656     timer_setup(&adapter->service_timer, ixgbevf_service_timer, 0);
4657
4658     INIT_WORK(&adapter->service_task, ixgbevf_service_task);
4659     set_bit(__IXGBEVF_SERVICE_INITED, &adapter->state);
4660     clear_bit(__IXGBEVF_SERVICE_SCHED, &adapter->state);
4661
4662     err = ixgbevf_init_interrupt_scheme(adapter);
4663     if (err)
4664         goto err_sw_init;
4665
4666     strcpy(netdev->name, "eth%d");
4667
4668     err = register_netdev(netdev);
4669     if (err)
4670         goto err_register;
4671
4672     pci_set_drvdata(pdev, netdev);
4673     netif_carrier_off(netdev);
4674     ixgbevf_init_ipsec_offload(adapter);
4675
4676     ixgbevf_init_last_counter_stats(adapter);
4677
4678     /* print the VF info */
4679     dev_info(&pdev->dev, "%pM\n", netdev->dev_addr);
4680     dev_info(&pdev->dev, "MAC: %d\n", hw->mac.type);
4681
4682     switch (hw->mac.type) {
4683     case ixgbe_mac_X550_vf:
4684         dev_info(&pdev->dev, "Intel(R) X550 Virtual Function\n");
4685         break;
4686     case ixgbe_mac_X540_vf:
4687         dev_info(&pdev->dev, "Intel(R) X540 Virtual Function\n");
4688         break;
4689     case ixgbe_mac_82599_vf:
4690     default:
4691         dev_info(&pdev->dev, "Intel(R) 82599 Virtual Function\n");
4692         break;
4693     }
4694
4695     return 0;
4696
4697 err_register:
4698     ixgbevf_clear_interrupt_scheme(adapter);
4699 err_sw_init:
4700     ixgbevf_reset_interrupt_capability(adapter);
4701     iounmap(adapter->io_addr);
4702     kfree(adapter->rss_key);
4703 err_ioremap:
4704     disable_dev = !test_and_set_bit(__IXGBEVF_DISABLED, &adapter->state);
4705     free_netdev(netdev);
4706 err_alloc_etherdev:
4707     pci_release_regions(pdev);
4708 err_pci_reg:
4709 err_dma:
4710     if (!adapter || disable_dev)
4711         pci_disable_device(pdev);
4712     return err;
4713 }
4714
4715 /**
4716  * ixgbevf_remove - Device Removal Routine
4717  * @pdev: PCI device information struct
4718  *
4719  * ixgbevf_remove is called by the PCI subsystem to alert the driver
4720  * that it should release a PCI device.  The could be caused by a
4721  * Hot-Plug event, or because the driver is going to be removed from
4722  * memory.
4723  **/
4724 static void ixgbevf_remove(struct pci_dev *pdev)
4725 {
4726     struct net_device *netdev = pci_get_drvdata(pdev);
4727     struct ixgbevf_adapter *adapter;
4728     bool disable_dev;
4729
4730     if (!netdev)
4731         return;
4732
4733     adapter = netdev_priv(netdev);
4734
4735     set_bit(__IXGBEVF_REMOVING, &adapter->state);
4736     cancel_work_sync(&adapter->service_task);
4737
4738     if (netdev->reg_state == NETREG_REGISTERED)
4739         unregister_netdev(netdev);
4740
4741     ixgbevf_stop_ipsec_offload(adapter);
4742     ixgbevf_clear_interrupt_scheme(adapter);
4743     ixgbevf_reset_interrupt_capability(adapter);
4744
4745     iounmap(adapter->io_addr);
4746     pci_release_regions(pdev);
4747
4748     hw_dbg(&adapter->hw, "Remove complete\n");
4749
4750     kfree(adapter->rss_key);
4751     disable_dev = !test_and_set_bit(__IXGBEVF_DISABLED, &adapter->state);
4752     free_netdev(netdev);
4753
4754     if (disable_dev)
4755         pci_disable_device(pdev);
4756 }
4757
4758 /**
4759  * ixgbevf_io_error_detected - called when PCI error is detected
4760  * @pdev: Pointer to PCI device
4761  * @state: The current pci connection state
4762  *
4763  * This function is called after a PCI bus error affecting
4764  * this device has been detected.
4765  **/
4766 static pci_ers_result_t ixgbevf_io_error_detected(struct pci_dev *pdev,
4767                           pci_channel_state_t state)
4768 {
4769     struct net_device *netdev = pci_get_drvdata(pdev);
4770     struct ixgbevf_adapter *adapter = netdev_priv(netdev);
4771
4772     if (!test_bit(__IXGBEVF_SERVICE_INITED, &adapter->state))
4773         return PCI_ERS_RESULT_DISCONNECT;
4774
4775     rtnl_lock();
4776     netif_device_detach(netdev);
4777
4778     if (netif_running(netdev))
4779         ixgbevf_close_suspend(adapter);
4780
4781     if (state == pci_channel_io_perm_failure) {
4782         rtnl_unlock();
4783         return PCI_ERS_RESULT_DISCONNECT;
4784     }
4785
4786     if (!test_and_set_bit(__IXGBEVF_DISABLED, &adapter->state))
4787         pci_disable_device(pdev);
4788     rtnl_unlock();
4789
4790     /* Request a slot reset. */
4791     return PCI_ERS_RESULT_NEED_RESET;
4792 }
4793
4794 /**
4795  * ixgbevf_io_slot_reset - called after the pci bus has been reset.
4796  * @pdev: Pointer to PCI device
4797  *
4798  * Restart the card from scratch, as if from a cold-boot. Implementation
4799  * resembles the first-half of the ixgbevf_resume routine.
4800  **/
4801 static pci_ers_result_t ixgbevf_io_slot_reset(struct pci_dev *pdev)
4802 {
4803     struct net_device *netdev = pci_get_drvdata(pdev);
4804     struct ixgbevf_adapter *adapter = netdev_priv(netdev);
4805
4806     if (pci_enable_device_mem(pdev)) {
4807         dev_err(&pdev->dev,
4808             "Cannot re-enable PCI device after reset.\n");
4809         return PCI_ERS_RESULT_DISCONNECT;
4810     }
4811
4812     adapter->hw.hw_addr = adapter->io_addr;
4813     smp_mb__before_atomic();
4814     clear_bit(__IXGBEVF_DISABLED, &adapter->state);
4815     pci_set_master(pdev);
4816
4817     ixgbevf_reset(adapter);
4818
4819     return PCI_ERS_RESULT_RECOVERED;
4820 }
4821
4822 /**
4823  * ixgbevf_io_resume - called when traffic can start flowing again.
4824  * @pdev: Pointer to PCI device
4825  *
4826  * This callback is called when the error recovery driver tells us that
4827  * its OK to resume normal operation. Implementation resembles the
4828  * second-half of the ixgbevf_resume routine.
4829  **/
4830 static void ixgbevf_io_resume(struct pci_dev *pdev)
4831 {
4832     struct net_device *netdev = pci_get_drvdata(pdev);
4833
4834     rtnl_lock();
4835     if (netif_running(netdev))
4836         ixgbevf_open(netdev);
4837
4838     netif_device_attach(netdev);
4839     rtnl_unlock();
4840 }
4841
4842 /* PCI Error Recovery (ERS) */
4843 static const struct pci_error_handlers ixgbevf_err_handler = {
4844     .error_detected = ixgbevf_io_error_detected,
4845     .slot_reset = ixgbevf_io_slot_reset,
4846     .resume = ixgbevf_io_resume,
4847 };
4848
4849 static SIMPLE_DEV_PM_OPS(ixgbevf_pm_ops, ixgbevf_suspend, ixgbevf_resume);
4850
4851 static struct pci_driver ixgbevf_driver = {
4852     .name       = ixgbevf_driver_name,
4853     .id_table   = ixgbevf_pci_tbl,
4854     .probe      = ixgbevf_probe,
4855     .remove     = ixgbevf_remove,
4856
4857     /* Power Management Hooks */
4858     .driver.pm  = &ixgbevf_pm_ops,
4859
4860     .shutdown   = ixgbevf_shutdown,
4861     .err_handler    = &ixgbevf_err_handler
4862 };
4863
4864 /**
4865  * ixgbevf_init_module - Driver Registration Routine
4866  *
4867  * ixgbevf_init_module is the first routine called when the driver is
4868  * loaded. All it does is register with the PCI subsystem.
4869  **/
4870 static int __init ixgbevf_init_module(void)
4871 {
4872     pr_info("%s\n", ixgbevf_driver_string);
4873     pr_info("%s\n", ixgbevf_copyright);
4874     ixgbevf_wq = create_singlethread_workqueue(ixgbevf_driver_name);
4875     if (!ixgbevf_wq) {
4876         pr_err("%s: Failed to create workqueue\n", ixgbevf_driver_name);
4877         return -ENOMEM;
4878     }
4879
4880     return pci_register_driver(&ixgbevf_driver);
4881 }
4882
4883 module_init(ixgbevf_init_module);
4884
4885 /**
4886  * ixgbevf_exit_module - Driver Exit Cleanup Routine
4887  *
4888  * ixgbevf_exit_module is called just before the driver is removed
4889  * from memory.
4890  **/
4891 static void __exit ixgbevf_exit_module(void)
4892 {
4893     pci_unregister_driver(&ixgbevf_driver);
4894     if (ixgbevf_wq) {
4895         destroy_workqueue(ixgbevf_wq);
4896         ixgbevf_wq = NULL;
4897     }
4898 }
4899
4900 #ifdef DEBUG
4901 /**
4902  * ixgbevf_get_hw_dev_name - return device name string
4903  * used by hardware layer to print debugging information
4904  * @hw: pointer to private hardware struct
4905  **/
4906 char *ixgbevf_get_hw_dev_name(struct ixgbe_hw *hw)
4907 {
4908     struct ixgbevf_adapter *adapter = hw->back;
4909
4910     return adapter->netdev->name;
4911 }
4912
4913 #endif
4914 module_exit(ixgbevf_exit_module);
4915
4916 /* ixgbevf_main.c */