Back to home page

OSCL-LXR
 
 

     

0001 /*
0002  * Linux driver for VMware's vmxnet3 ethernet NIC.
0003  *
0004  * Copyright (C) 2008-2022, VMware, Inc. All Rights Reserved.
0005  *
0006  * This program is free software; you can redistribute it and/or modify it
0007  * under the terms of the GNU General Public License as published by the
0008  * Free Software Foundation; version 2 of the License and no later version.
0009  *
0010  * This program is distributed in the hope that it will be useful, but
0011  * WITHOUT ANY WARRANTY; without even the implied warranty of
0012  * MERCHANTABILITY OR FITNESS FOR A PARTICULAR PURPOSE, GOOD TITLE or
0013  * NON INFRINGEMENT. See the GNU General Public License for more
0014  * details.
0015  *
0016  * You should have received a copy of the GNU General Public License
0017  * along with this program; if not, write to the Free Software
0018  * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
0019  *
0020  * The full GNU General Public License is included in this distribution in
0021  * the file called "COPYING".
0022  *
0023  * Maintained by: pv-drivers@vmware.com
0024  *
0025  */
0026 
0027 #include <linux/module.h>
0028 #include <net/ip6_checksum.h>
0029 
0030 #include "vmxnet3_int.h"
0031 
0032 char vmxnet3_driver_name[] = "vmxnet3";
0033 #define VMXNET3_DRIVER_DESC "VMware vmxnet3 virtual NIC driver"
0034 
0035 /*
0036  * PCI Device ID Table
0037  * Last entry must be all 0s
0038  */
0039 static const struct pci_device_id vmxnet3_pciid_table[] = {
0040     {PCI_VDEVICE(VMWARE, PCI_DEVICE_ID_VMWARE_VMXNET3)},
0041     {0}
0042 };
0043 
0044 MODULE_DEVICE_TABLE(pci, vmxnet3_pciid_table);
0045 
0046 static int enable_mq = 1;
0047 
0048 static void
0049 vmxnet3_write_mac_addr(struct vmxnet3_adapter *adapter, const u8 *mac);
0050 
0051 /*
0052  *    Enable/Disable the given intr
0053  */
0054 static void
0055 vmxnet3_enable_intr(struct vmxnet3_adapter *adapter, unsigned intr_idx)
0056 {
0057     VMXNET3_WRITE_BAR0_REG(adapter, VMXNET3_REG_IMR + intr_idx * 8, 0);
0058 }
0059 
0060 
0061 static void
0062 vmxnet3_disable_intr(struct vmxnet3_adapter *adapter, unsigned intr_idx)
0063 {
0064     VMXNET3_WRITE_BAR0_REG(adapter, VMXNET3_REG_IMR + intr_idx * 8, 1);
0065 }
0066 
0067 
0068 /*
0069  *    Enable/Disable all intrs used by the device
0070  */
0071 static void
0072 vmxnet3_enable_all_intrs(struct vmxnet3_adapter *adapter)
0073 {
0074     int i;
0075 
0076     for (i = 0; i < adapter->intr.num_intrs; i++)
0077         vmxnet3_enable_intr(adapter, i);
0078     adapter->shared->devRead.intrConf.intrCtrl &=
0079                     cpu_to_le32(~VMXNET3_IC_DISABLE_ALL);
0080 }
0081 
0082 
0083 static void
0084 vmxnet3_disable_all_intrs(struct vmxnet3_adapter *adapter)
0085 {
0086     int i;
0087 
0088     adapter->shared->devRead.intrConf.intrCtrl |=
0089                     cpu_to_le32(VMXNET3_IC_DISABLE_ALL);
0090     for (i = 0; i < adapter->intr.num_intrs; i++)
0091         vmxnet3_disable_intr(adapter, i);
0092 }
0093 
0094 
0095 static void
0096 vmxnet3_ack_events(struct vmxnet3_adapter *adapter, u32 events)
0097 {
0098     VMXNET3_WRITE_BAR1_REG(adapter, VMXNET3_REG_ECR, events);
0099 }
0100 
0101 
0102 static bool
0103 vmxnet3_tq_stopped(struct vmxnet3_tx_queue *tq, struct vmxnet3_adapter *adapter)
0104 {
0105     return tq->stopped;
0106 }
0107 
0108 
0109 static void
0110 vmxnet3_tq_start(struct vmxnet3_tx_queue *tq, struct vmxnet3_adapter *adapter)
0111 {
0112     tq->stopped = false;
0113     netif_start_subqueue(adapter->netdev, tq - adapter->tx_queue);
0114 }
0115 
0116 
0117 static void
0118 vmxnet3_tq_wake(struct vmxnet3_tx_queue *tq, struct vmxnet3_adapter *adapter)
0119 {
0120     tq->stopped = false;
0121     netif_wake_subqueue(adapter->netdev, (tq - adapter->tx_queue));
0122 }
0123 
0124 
0125 static void
0126 vmxnet3_tq_stop(struct vmxnet3_tx_queue *tq, struct vmxnet3_adapter *adapter)
0127 {
0128     tq->stopped = true;
0129     tq->num_stop++;
0130     netif_stop_subqueue(adapter->netdev, (tq - adapter->tx_queue));
0131 }
0132 
0133 /* Check if capability is supported by UPT device or
0134  * UPT is even requested
0135  */
0136 bool
0137 vmxnet3_check_ptcapability(u32 cap_supported, u32 cap)
0138 {
0139     if (cap_supported & (1UL << VMXNET3_DCR_ERROR) ||
0140         cap_supported & (1UL << cap)) {
0141         return true;
0142     }
0143 
0144     return false;
0145 }
0146 
0147 
0148 /*
0149  * Check the link state. This may start or stop the tx queue.
0150  */
0151 static void
0152 vmxnet3_check_link(struct vmxnet3_adapter *adapter, bool affectTxQueue)
0153 {
0154     u32 ret;
0155     int i;
0156     unsigned long flags;
0157 
0158     spin_lock_irqsave(&adapter->cmd_lock, flags);
0159     VMXNET3_WRITE_BAR1_REG(adapter, VMXNET3_REG_CMD, VMXNET3_CMD_GET_LINK);
0160     ret = VMXNET3_READ_BAR1_REG(adapter, VMXNET3_REG_CMD);
0161     spin_unlock_irqrestore(&adapter->cmd_lock, flags);
0162 
0163     adapter->link_speed = ret >> 16;
0164     if (ret & 1) { /* Link is up. */
0165         netdev_info(adapter->netdev, "NIC Link is Up %d Mbps\n",
0166                 adapter->link_speed);
0167         netif_carrier_on(adapter->netdev);
0168 
0169         if (affectTxQueue) {
0170             for (i = 0; i < adapter->num_tx_queues; i++)
0171                 vmxnet3_tq_start(&adapter->tx_queue[i],
0172                          adapter);
0173         }
0174     } else {
0175         netdev_info(adapter->netdev, "NIC Link is Down\n");
0176         netif_carrier_off(adapter->netdev);
0177 
0178         if (affectTxQueue) {
0179             for (i = 0; i < adapter->num_tx_queues; i++)
0180                 vmxnet3_tq_stop(&adapter->tx_queue[i], adapter);
0181         }
0182     }
0183 }
0184 
0185 static void
0186 vmxnet3_process_events(struct vmxnet3_adapter *adapter)
0187 {
0188     int i;
0189     unsigned long flags;
0190     u32 events = le32_to_cpu(adapter->shared->ecr);
0191     if (!events)
0192         return;
0193 
0194     vmxnet3_ack_events(adapter, events);
0195 
0196     /* Check if link state has changed */
0197     if (events & VMXNET3_ECR_LINK)
0198         vmxnet3_check_link(adapter, true);
0199 
0200     /* Check if there is an error on xmit/recv queues */
0201     if (events & (VMXNET3_ECR_TQERR | VMXNET3_ECR_RQERR)) {
0202         spin_lock_irqsave(&adapter->cmd_lock, flags);
0203         VMXNET3_WRITE_BAR1_REG(adapter, VMXNET3_REG_CMD,
0204                        VMXNET3_CMD_GET_QUEUE_STATUS);
0205         spin_unlock_irqrestore(&adapter->cmd_lock, flags);
0206 
0207         for (i = 0; i < adapter->num_tx_queues; i++)
0208             if (adapter->tqd_start[i].status.stopped)
0209                 dev_err(&adapter->netdev->dev,
0210                     "%s: tq[%d] error 0x%x\n",
0211                     adapter->netdev->name, i, le32_to_cpu(
0212                     adapter->tqd_start[i].status.error));
0213         for (i = 0; i < adapter->num_rx_queues; i++)
0214             if (adapter->rqd_start[i].status.stopped)
0215                 dev_err(&adapter->netdev->dev,
0216                     "%s: rq[%d] error 0x%x\n",
0217                     adapter->netdev->name, i,
0218                     adapter->rqd_start[i].status.error);
0219 
0220         schedule_work(&adapter->work);
0221     }
0222 }
0223 
0224 #ifdef __BIG_ENDIAN_BITFIELD
0225 /*
0226  * The device expects the bitfields in shared structures to be written in
0227  * little endian. When CPU is big endian, the following routines are used to
0228  * correctly read and write into ABI.
0229  * The general technique used here is : double word bitfields are defined in
0230  * opposite order for big endian architecture. Then before reading them in
0231  * driver the complete double word is translated using le32_to_cpu. Similarly
0232  * After the driver writes into bitfields, cpu_to_le32 is used to translate the
0233  * double words into required format.
0234  * In order to avoid touching bits in shared structure more than once, temporary
0235  * descriptors are used. These are passed as srcDesc to following functions.
0236  */
0237 static void vmxnet3_RxDescToCPU(const struct Vmxnet3_RxDesc *srcDesc,
0238                 struct Vmxnet3_RxDesc *dstDesc)
0239 {
0240     u32 *src = (u32 *)srcDesc + 2;
0241     u32 *dst = (u32 *)dstDesc + 2;
0242     dstDesc->addr = le64_to_cpu(srcDesc->addr);
0243     *dst = le32_to_cpu(*src);
0244     dstDesc->ext1 = le32_to_cpu(srcDesc->ext1);
0245 }
0246 
0247 static void vmxnet3_TxDescToLe(const struct Vmxnet3_TxDesc *srcDesc,
0248                    struct Vmxnet3_TxDesc *dstDesc)
0249 {
0250     int i;
0251     u32 *src = (u32 *)(srcDesc + 1);
0252     u32 *dst = (u32 *)(dstDesc + 1);
0253 
0254     /* Working backwards so that the gen bit is set at the end. */
0255     for (i = 2; i > 0; i--) {
0256         src--;
0257         dst--;
0258         *dst = cpu_to_le32(*src);
0259     }
0260 }
0261 
0262 
0263 static void vmxnet3_RxCompToCPU(const struct Vmxnet3_RxCompDesc *srcDesc,
0264                 struct Vmxnet3_RxCompDesc *dstDesc)
0265 {
0266     int i = 0;
0267     u32 *src = (u32 *)srcDesc;
0268     u32 *dst = (u32 *)dstDesc;
0269     for (i = 0; i < sizeof(struct Vmxnet3_RxCompDesc) / sizeof(u32); i++) {
0270         *dst = le32_to_cpu(*src);
0271         src++;
0272         dst++;
0273     }
0274 }
0275 
0276 
0277 /* Used to read bitfield values from double words. */
0278 static u32 get_bitfield32(const __le32 *bitfield, u32 pos, u32 size)
0279 {
0280     u32 temp = le32_to_cpu(*bitfield);
0281     u32 mask = ((1 << size) - 1) << pos;
0282     temp &= mask;
0283     temp >>= pos;
0284     return temp;
0285 }
0286 
0287 
0288 
0289 #endif  /* __BIG_ENDIAN_BITFIELD */
0290 
0291 #ifdef __BIG_ENDIAN_BITFIELD
0292 
0293 #   define VMXNET3_TXDESC_GET_GEN(txdesc) get_bitfield32(((const __le32 *) \
0294             txdesc) + VMXNET3_TXD_GEN_DWORD_SHIFT, \
0295             VMXNET3_TXD_GEN_SHIFT, VMXNET3_TXD_GEN_SIZE)
0296 #   define VMXNET3_TXDESC_GET_EOP(txdesc) get_bitfield32(((const __le32 *) \
0297             txdesc) + VMXNET3_TXD_EOP_DWORD_SHIFT, \
0298             VMXNET3_TXD_EOP_SHIFT, VMXNET3_TXD_EOP_SIZE)
0299 #   define VMXNET3_TCD_GET_GEN(tcd) get_bitfield32(((const __le32 *)tcd) + \
0300             VMXNET3_TCD_GEN_DWORD_SHIFT, VMXNET3_TCD_GEN_SHIFT, \
0301             VMXNET3_TCD_GEN_SIZE)
0302 #   define VMXNET3_TCD_GET_TXIDX(tcd) get_bitfield32((const __le32 *)tcd, \
0303             VMXNET3_TCD_TXIDX_SHIFT, VMXNET3_TCD_TXIDX_SIZE)
0304 #   define vmxnet3_getRxComp(dstrcd, rcd, tmp) do { \
0305             (dstrcd) = (tmp); \
0306             vmxnet3_RxCompToCPU((rcd), (tmp)); \
0307         } while (0)
0308 #   define vmxnet3_getRxDesc(dstrxd, rxd, tmp) do { \
0309             (dstrxd) = (tmp); \
0310             vmxnet3_RxDescToCPU((rxd), (tmp)); \
0311         } while (0)
0312 
0313 #else
0314 
0315 #   define VMXNET3_TXDESC_GET_GEN(txdesc) ((txdesc)->gen)
0316 #   define VMXNET3_TXDESC_GET_EOP(txdesc) ((txdesc)->eop)
0317 #   define VMXNET3_TCD_GET_GEN(tcd) ((tcd)->gen)
0318 #   define VMXNET3_TCD_GET_TXIDX(tcd) ((tcd)->txdIdx)
0319 #   define vmxnet3_getRxComp(dstrcd, rcd, tmp) (dstrcd) = (rcd)
0320 #   define vmxnet3_getRxDesc(dstrxd, rxd, tmp) (dstrxd) = (rxd)
0321 
0322 #endif /* __BIG_ENDIAN_BITFIELD  */
0323 
0324 
0325 static void
0326 vmxnet3_unmap_tx_buf(struct vmxnet3_tx_buf_info *tbi,
0327              struct pci_dev *pdev)
0328 {
0329     if (tbi->map_type == VMXNET3_MAP_SINGLE)
0330         dma_unmap_single(&pdev->dev, tbi->dma_addr, tbi->len,
0331                  DMA_TO_DEVICE);
0332     else if (tbi->map_type == VMXNET3_MAP_PAGE)
0333         dma_unmap_page(&pdev->dev, tbi->dma_addr, tbi->len,
0334                    DMA_TO_DEVICE);
0335     else
0336         BUG_ON(tbi->map_type != VMXNET3_MAP_NONE);
0337 
0338     tbi->map_type = VMXNET3_MAP_NONE; /* to help debugging */
0339 }
0340 
0341 
0342 static int
0343 vmxnet3_unmap_pkt(u32 eop_idx, struct vmxnet3_tx_queue *tq,
0344           struct pci_dev *pdev, struct vmxnet3_adapter *adapter)
0345 {
0346     struct sk_buff *skb;
0347     int entries = 0;
0348 
0349     /* no out of order completion */
0350     BUG_ON(tq->buf_info[eop_idx].sop_idx != tq->tx_ring.next2comp);
0351     BUG_ON(VMXNET3_TXDESC_GET_EOP(&(tq->tx_ring.base[eop_idx].txd)) != 1);
0352 
0353     skb = tq->buf_info[eop_idx].skb;
0354     BUG_ON(skb == NULL);
0355     tq->buf_info[eop_idx].skb = NULL;
0356 
0357     VMXNET3_INC_RING_IDX_ONLY(eop_idx, tq->tx_ring.size);
0358 
0359     while (tq->tx_ring.next2comp != eop_idx) {
0360         vmxnet3_unmap_tx_buf(tq->buf_info + tq->tx_ring.next2comp,
0361                      pdev);
0362 
0363         /* update next2comp w/o tx_lock. Since we are marking more,
0364          * instead of less, tx ring entries avail, the worst case is
0365          * that the tx routine incorrectly re-queues a pkt due to
0366          * insufficient tx ring entries.
0367          */
0368         vmxnet3_cmd_ring_adv_next2comp(&tq->tx_ring);
0369         entries++;
0370     }
0371 
0372     dev_kfree_skb_any(skb);
0373     return entries;
0374 }
0375 
0376 
0377 static int
0378 vmxnet3_tq_tx_complete(struct vmxnet3_tx_queue *tq,
0379             struct vmxnet3_adapter *adapter)
0380 {
0381     int completed = 0;
0382     union Vmxnet3_GenericDesc *gdesc;
0383 
0384     gdesc = tq->comp_ring.base + tq->comp_ring.next2proc;
0385     while (VMXNET3_TCD_GET_GEN(&gdesc->tcd) == tq->comp_ring.gen) {
0386         /* Prevent any &gdesc->tcd field from being (speculatively)
0387          * read before (&gdesc->tcd)->gen is read.
0388          */
0389         dma_rmb();
0390 
0391         completed += vmxnet3_unmap_pkt(VMXNET3_TCD_GET_TXIDX(
0392                            &gdesc->tcd), tq, adapter->pdev,
0393                            adapter);
0394 
0395         vmxnet3_comp_ring_adv_next2proc(&tq->comp_ring);
0396         gdesc = tq->comp_ring.base + tq->comp_ring.next2proc;
0397     }
0398 
0399     if (completed) {
0400         spin_lock(&tq->tx_lock);
0401         if (unlikely(vmxnet3_tq_stopped(tq, adapter) &&
0402                  vmxnet3_cmd_ring_desc_avail(&tq->tx_ring) >
0403                  VMXNET3_WAKE_QUEUE_THRESHOLD(tq) &&
0404                  netif_carrier_ok(adapter->netdev))) {
0405             vmxnet3_tq_wake(tq, adapter);
0406         }
0407         spin_unlock(&tq->tx_lock);
0408     }
0409     return completed;
0410 }
0411 
0412 
0413 static void
0414 vmxnet3_tq_cleanup(struct vmxnet3_tx_queue *tq,
0415            struct vmxnet3_adapter *adapter)
0416 {
0417     int i;
0418 
0419     while (tq->tx_ring.next2comp != tq->tx_ring.next2fill) {
0420         struct vmxnet3_tx_buf_info *tbi;
0421 
0422         tbi = tq->buf_info + tq->tx_ring.next2comp;
0423 
0424         vmxnet3_unmap_tx_buf(tbi, adapter->pdev);
0425         if (tbi->skb) {
0426             dev_kfree_skb_any(tbi->skb);
0427             tbi->skb = NULL;
0428         }
0429         vmxnet3_cmd_ring_adv_next2comp(&tq->tx_ring);
0430     }
0431 
0432     /* sanity check, verify all buffers are indeed unmapped and freed */
0433     for (i = 0; i < tq->tx_ring.size; i++) {
0434         BUG_ON(tq->buf_info[i].skb != NULL ||
0435                tq->buf_info[i].map_type != VMXNET3_MAP_NONE);
0436     }
0437 
0438     tq->tx_ring.gen = VMXNET3_INIT_GEN;
0439     tq->tx_ring.next2fill = tq->tx_ring.next2comp = 0;
0440 
0441     tq->comp_ring.gen = VMXNET3_INIT_GEN;
0442     tq->comp_ring.next2proc = 0;
0443 }
0444 
0445 
0446 static void
0447 vmxnet3_tq_destroy(struct vmxnet3_tx_queue *tq,
0448            struct vmxnet3_adapter *adapter)
0449 {
0450     if (tq->tx_ring.base) {
0451         dma_free_coherent(&adapter->pdev->dev, tq->tx_ring.size *
0452                   sizeof(struct Vmxnet3_TxDesc),
0453                   tq->tx_ring.base, tq->tx_ring.basePA);
0454         tq->tx_ring.base = NULL;
0455     }
0456     if (tq->data_ring.base) {
0457         dma_free_coherent(&adapter->pdev->dev,
0458                   tq->data_ring.size * tq->txdata_desc_size,
0459                   tq->data_ring.base, tq->data_ring.basePA);
0460         tq->data_ring.base = NULL;
0461     }
0462     if (tq->comp_ring.base) {
0463         dma_free_coherent(&adapter->pdev->dev, tq->comp_ring.size *
0464                   sizeof(struct Vmxnet3_TxCompDesc),
0465                   tq->comp_ring.base, tq->comp_ring.basePA);
0466         tq->comp_ring.base = NULL;
0467     }
0468     kfree(tq->buf_info);
0469     tq->buf_info = NULL;
0470 }
0471 
0472 
0473 /* Destroy all tx queues */
0474 void
0475 vmxnet3_tq_destroy_all(struct vmxnet3_adapter *adapter)
0476 {
0477     int i;
0478 
0479     for (i = 0; i < adapter->num_tx_queues; i++)
0480         vmxnet3_tq_destroy(&adapter->tx_queue[i], adapter);
0481 }
0482 
0483 
0484 static void
0485 vmxnet3_tq_init(struct vmxnet3_tx_queue *tq,
0486         struct vmxnet3_adapter *adapter)
0487 {
0488     int i;
0489 
0490     /* reset the tx ring contents to 0 and reset the tx ring states */
0491     memset(tq->tx_ring.base, 0, tq->tx_ring.size *
0492            sizeof(struct Vmxnet3_TxDesc));
0493     tq->tx_ring.next2fill = tq->tx_ring.next2comp = 0;
0494     tq->tx_ring.gen = VMXNET3_INIT_GEN;
0495 
0496     memset(tq->data_ring.base, 0,
0497            tq->data_ring.size * tq->txdata_desc_size);
0498 
0499     /* reset the tx comp ring contents to 0 and reset comp ring states */
0500     memset(tq->comp_ring.base, 0, tq->comp_ring.size *
0501            sizeof(struct Vmxnet3_TxCompDesc));
0502     tq->comp_ring.next2proc = 0;
0503     tq->comp_ring.gen = VMXNET3_INIT_GEN;
0504 
0505     /* reset the bookkeeping data */
0506     memset(tq->buf_info, 0, sizeof(tq->buf_info[0]) * tq->tx_ring.size);
0507     for (i = 0; i < tq->tx_ring.size; i++)
0508         tq->buf_info[i].map_type = VMXNET3_MAP_NONE;
0509 
0510     /* stats are not reset */
0511 }
0512 
0513 
0514 static int
0515 vmxnet3_tq_create(struct vmxnet3_tx_queue *tq,
0516           struct vmxnet3_adapter *adapter)
0517 {
0518     BUG_ON(tq->tx_ring.base || tq->data_ring.base ||
0519            tq->comp_ring.base || tq->buf_info);
0520 
0521     tq->tx_ring.base = dma_alloc_coherent(&adapter->pdev->dev,
0522             tq->tx_ring.size * sizeof(struct Vmxnet3_TxDesc),
0523             &tq->tx_ring.basePA, GFP_KERNEL);
0524     if (!tq->tx_ring.base) {
0525         netdev_err(adapter->netdev, "failed to allocate tx ring\n");
0526         goto err;
0527     }
0528 
0529     tq->data_ring.base = dma_alloc_coherent(&adapter->pdev->dev,
0530             tq->data_ring.size * tq->txdata_desc_size,
0531             &tq->data_ring.basePA, GFP_KERNEL);
0532     if (!tq->data_ring.base) {
0533         netdev_err(adapter->netdev, "failed to allocate tx data ring\n");
0534         goto err;
0535     }
0536 
0537     tq->comp_ring.base = dma_alloc_coherent(&adapter->pdev->dev,
0538             tq->comp_ring.size * sizeof(struct Vmxnet3_TxCompDesc),
0539             &tq->comp_ring.basePA, GFP_KERNEL);
0540     if (!tq->comp_ring.base) {
0541         netdev_err(adapter->netdev, "failed to allocate tx comp ring\n");
0542         goto err;
0543     }
0544 
0545     tq->buf_info = kcalloc_node(tq->tx_ring.size, sizeof(tq->buf_info[0]),
0546                     GFP_KERNEL,
0547                     dev_to_node(&adapter->pdev->dev));
0548     if (!tq->buf_info)
0549         goto err;
0550 
0551     return 0;
0552 
0553 err:
0554     vmxnet3_tq_destroy(tq, adapter);
0555     return -ENOMEM;
0556 }
0557 
0558 static void
0559 vmxnet3_tq_cleanup_all(struct vmxnet3_adapter *adapter)
0560 {
0561     int i;
0562 
0563     for (i = 0; i < adapter->num_tx_queues; i++)
0564         vmxnet3_tq_cleanup(&adapter->tx_queue[i], adapter);
0565 }
0566 
0567 /*
0568  *    starting from ring->next2fill, allocate rx buffers for the given ring
0569  *    of the rx queue and update the rx desc. stop after @num_to_alloc buffers
0570  *    are allocated or allocation fails
0571  */
0572 
0573 static int
0574 vmxnet3_rq_alloc_rx_buf(struct vmxnet3_rx_queue *rq, u32 ring_idx,
0575             int num_to_alloc, struct vmxnet3_adapter *adapter)
0576 {
0577     int num_allocated = 0;
0578     struct vmxnet3_rx_buf_info *rbi_base = rq->buf_info[ring_idx];
0579     struct vmxnet3_cmd_ring *ring = &rq->rx_ring[ring_idx];
0580     u32 val;
0581 
0582     while (num_allocated <= num_to_alloc) {
0583         struct vmxnet3_rx_buf_info *rbi;
0584         union Vmxnet3_GenericDesc *gd;
0585 
0586         rbi = rbi_base + ring->next2fill;
0587         gd = ring->base + ring->next2fill;
0588         rbi->comp_state = VMXNET3_RXD_COMP_PENDING;
0589 
0590         if (rbi->buf_type == VMXNET3_RX_BUF_SKB) {
0591             if (rbi->skb == NULL) {
0592                 rbi->skb = __netdev_alloc_skb_ip_align(adapter->netdev,
0593                                        rbi->len,
0594                                        GFP_KERNEL);
0595                 if (unlikely(rbi->skb == NULL)) {
0596                     rq->stats.rx_buf_alloc_failure++;
0597                     break;
0598                 }
0599 
0600                 rbi->dma_addr = dma_map_single(
0601                         &adapter->pdev->dev,
0602                         rbi->skb->data, rbi->len,
0603                         DMA_FROM_DEVICE);
0604                 if (dma_mapping_error(&adapter->pdev->dev,
0605                               rbi->dma_addr)) {
0606                     dev_kfree_skb_any(rbi->skb);
0607                     rbi->skb = NULL;
0608                     rq->stats.rx_buf_alloc_failure++;
0609                     break;
0610                 }
0611             } else {
0612                 /* rx buffer skipped by the device */
0613             }
0614             val = VMXNET3_RXD_BTYPE_HEAD << VMXNET3_RXD_BTYPE_SHIFT;
0615         } else {
0616             BUG_ON(rbi->buf_type != VMXNET3_RX_BUF_PAGE ||
0617                    rbi->len  != PAGE_SIZE);
0618 
0619             if (rbi->page == NULL) {
0620                 rbi->page = alloc_page(GFP_ATOMIC);
0621                 if (unlikely(rbi->page == NULL)) {
0622                     rq->stats.rx_buf_alloc_failure++;
0623                     break;
0624                 }
0625                 rbi->dma_addr = dma_map_page(
0626                         &adapter->pdev->dev,
0627                         rbi->page, 0, PAGE_SIZE,
0628                         DMA_FROM_DEVICE);
0629                 if (dma_mapping_error(&adapter->pdev->dev,
0630                               rbi->dma_addr)) {
0631                     put_page(rbi->page);
0632                     rbi->page = NULL;
0633                     rq->stats.rx_buf_alloc_failure++;
0634                     break;
0635                 }
0636             } else {
0637                 /* rx buffers skipped by the device */
0638             }
0639             val = VMXNET3_RXD_BTYPE_BODY << VMXNET3_RXD_BTYPE_SHIFT;
0640         }
0641 
0642         gd->rxd.addr = cpu_to_le64(rbi->dma_addr);
0643         gd->dword[2] = cpu_to_le32((!ring->gen << VMXNET3_RXD_GEN_SHIFT)
0644                        | val | rbi->len);
0645 
0646         /* Fill the last buffer but dont mark it ready, or else the
0647          * device will think that the queue is full */
0648         if (num_allocated == num_to_alloc) {
0649             rbi->comp_state = VMXNET3_RXD_COMP_DONE;
0650             break;
0651         }
0652 
0653         gd->dword[2] |= cpu_to_le32(ring->gen << VMXNET3_RXD_GEN_SHIFT);
0654         num_allocated++;
0655         vmxnet3_cmd_ring_adv_next2fill(ring);
0656     }
0657 
0658     netdev_dbg(adapter->netdev,
0659         "alloc_rx_buf: %d allocated, next2fill %u, next2comp %u\n",
0660         num_allocated, ring->next2fill, ring->next2comp);
0661 
0662     /* so that the device can distinguish a full ring and an empty ring */
0663     BUG_ON(num_allocated != 0 && ring->next2fill == ring->next2comp);
0664 
0665     return num_allocated;
0666 }
0667 
0668 
0669 static void
0670 vmxnet3_append_frag(struct sk_buff *skb, struct Vmxnet3_RxCompDesc *rcd,
0671             struct vmxnet3_rx_buf_info *rbi)
0672 {
0673     skb_frag_t *frag = skb_shinfo(skb)->frags + skb_shinfo(skb)->nr_frags;
0674 
0675     BUG_ON(skb_shinfo(skb)->nr_frags >= MAX_SKB_FRAGS);
0676 
0677     __skb_frag_set_page(frag, rbi->page);
0678     skb_frag_off_set(frag, 0);
0679     skb_frag_size_set(frag, rcd->len);
0680     skb->data_len += rcd->len;
0681     skb->truesize += PAGE_SIZE;
0682     skb_shinfo(skb)->nr_frags++;
0683 }
0684 
0685 
0686 static int
0687 vmxnet3_map_pkt(struct sk_buff *skb, struct vmxnet3_tx_ctx *ctx,
0688         struct vmxnet3_tx_queue *tq, struct pci_dev *pdev,
0689         struct vmxnet3_adapter *adapter)
0690 {
0691     u32 dw2, len;
0692     unsigned long buf_offset;
0693     int i;
0694     union Vmxnet3_GenericDesc *gdesc;
0695     struct vmxnet3_tx_buf_info *tbi = NULL;
0696 
0697     BUG_ON(ctx->copy_size > skb_headlen(skb));
0698 
0699     /* use the previous gen bit for the SOP desc */
0700     dw2 = (tq->tx_ring.gen ^ 0x1) << VMXNET3_TXD_GEN_SHIFT;
0701 
0702     ctx->sop_txd = tq->tx_ring.base + tq->tx_ring.next2fill;
0703     gdesc = ctx->sop_txd; /* both loops below can be skipped */
0704 
0705     /* no need to map the buffer if headers are copied */
0706     if (ctx->copy_size) {
0707         ctx->sop_txd->txd.addr = cpu_to_le64(tq->data_ring.basePA +
0708                     tq->tx_ring.next2fill *
0709                     tq->txdata_desc_size);
0710         ctx->sop_txd->dword[2] = cpu_to_le32(dw2 | ctx->copy_size);
0711         ctx->sop_txd->dword[3] = 0;
0712 
0713         tbi = tq->buf_info + tq->tx_ring.next2fill;
0714         tbi->map_type = VMXNET3_MAP_NONE;
0715 
0716         netdev_dbg(adapter->netdev,
0717             "txd[%u]: 0x%Lx 0x%x 0x%x\n",
0718             tq->tx_ring.next2fill,
0719             le64_to_cpu(ctx->sop_txd->txd.addr),
0720             ctx->sop_txd->dword[2], ctx->sop_txd->dword[3]);
0721         vmxnet3_cmd_ring_adv_next2fill(&tq->tx_ring);
0722 
0723         /* use the right gen for non-SOP desc */
0724         dw2 = tq->tx_ring.gen << VMXNET3_TXD_GEN_SHIFT;
0725     }
0726 
0727     /* linear part can use multiple tx desc if it's big */
0728     len = skb_headlen(skb) - ctx->copy_size;
0729     buf_offset = ctx->copy_size;
0730     while (len) {
0731         u32 buf_size;
0732 
0733         if (len < VMXNET3_MAX_TX_BUF_SIZE) {
0734             buf_size = len;
0735             dw2 |= len;
0736         } else {
0737             buf_size = VMXNET3_MAX_TX_BUF_SIZE;
0738             /* spec says that for TxDesc.len, 0 == 2^14 */
0739         }
0740 
0741         tbi = tq->buf_info + tq->tx_ring.next2fill;
0742         tbi->map_type = VMXNET3_MAP_SINGLE;
0743         tbi->dma_addr = dma_map_single(&adapter->pdev->dev,
0744                 skb->data + buf_offset, buf_size,
0745                 DMA_TO_DEVICE);
0746         if (dma_mapping_error(&adapter->pdev->dev, tbi->dma_addr))
0747             return -EFAULT;
0748 
0749         tbi->len = buf_size;
0750 
0751         gdesc = tq->tx_ring.base + tq->tx_ring.next2fill;
0752         BUG_ON(gdesc->txd.gen == tq->tx_ring.gen);
0753 
0754         gdesc->txd.addr = cpu_to_le64(tbi->dma_addr);
0755         gdesc->dword[2] = cpu_to_le32(dw2);
0756         gdesc->dword[3] = 0;
0757 
0758         netdev_dbg(adapter->netdev,
0759             "txd[%u]: 0x%Lx 0x%x 0x%x\n",
0760             tq->tx_ring.next2fill, le64_to_cpu(gdesc->txd.addr),
0761             le32_to_cpu(gdesc->dword[2]), gdesc->dword[3]);
0762         vmxnet3_cmd_ring_adv_next2fill(&tq->tx_ring);
0763         dw2 = tq->tx_ring.gen << VMXNET3_TXD_GEN_SHIFT;
0764 
0765         len -= buf_size;
0766         buf_offset += buf_size;
0767     }
0768 
0769     for (i = 0; i < skb_shinfo(skb)->nr_frags; i++) {
0770         const skb_frag_t *frag = &skb_shinfo(skb)->frags[i];
0771         u32 buf_size;
0772 
0773         buf_offset = 0;
0774         len = skb_frag_size(frag);
0775         while (len) {
0776             tbi = tq->buf_info + tq->tx_ring.next2fill;
0777             if (len < VMXNET3_MAX_TX_BUF_SIZE) {
0778                 buf_size = len;
0779                 dw2 |= len;
0780             } else {
0781                 buf_size = VMXNET3_MAX_TX_BUF_SIZE;
0782                 /* spec says that for TxDesc.len, 0 == 2^14 */
0783             }
0784             tbi->map_type = VMXNET3_MAP_PAGE;
0785             tbi->dma_addr = skb_frag_dma_map(&adapter->pdev->dev, frag,
0786                              buf_offset, buf_size,
0787                              DMA_TO_DEVICE);
0788             if (dma_mapping_error(&adapter->pdev->dev, tbi->dma_addr))
0789                 return -EFAULT;
0790 
0791             tbi->len = buf_size;
0792 
0793             gdesc = tq->tx_ring.base + tq->tx_ring.next2fill;
0794             BUG_ON(gdesc->txd.gen == tq->tx_ring.gen);
0795 
0796             gdesc->txd.addr = cpu_to_le64(tbi->dma_addr);
0797             gdesc->dword[2] = cpu_to_le32(dw2);
0798             gdesc->dword[3] = 0;
0799 
0800             netdev_dbg(adapter->netdev,
0801                 "txd[%u]: 0x%llx %u %u\n",
0802                 tq->tx_ring.next2fill, le64_to_cpu(gdesc->txd.addr),
0803                 le32_to_cpu(gdesc->dword[2]), gdesc->dword[3]);
0804             vmxnet3_cmd_ring_adv_next2fill(&tq->tx_ring);
0805             dw2 = tq->tx_ring.gen << VMXNET3_TXD_GEN_SHIFT;
0806 
0807             len -= buf_size;
0808             buf_offset += buf_size;
0809         }
0810     }
0811 
0812     ctx->eop_txd = gdesc;
0813 
0814     /* set the last buf_info for the pkt */
0815     tbi->skb = skb;
0816     tbi->sop_idx = ctx->sop_txd - tq->tx_ring.base;
0817 
0818     return 0;
0819 }
0820 
0821 
0822 /* Init all tx queues */
0823 static void
0824 vmxnet3_tq_init_all(struct vmxnet3_adapter *adapter)
0825 {
0826     int i;
0827 
0828     for (i = 0; i < adapter->num_tx_queues; i++)
0829         vmxnet3_tq_init(&adapter->tx_queue[i], adapter);
0830 }
0831 
0832 
0833 /*
0834  *    parse relevant protocol headers:
0835  *      For a tso pkt, relevant headers are L2/3/4 including options
0836  *      For a pkt requesting csum offloading, they are L2/3 and may include L4
0837  *      if it's a TCP/UDP pkt
0838  *
0839  * Returns:
0840  *    -1:  error happens during parsing
0841  *     0:  protocol headers parsed, but too big to be copied
0842  *     1:  protocol headers parsed and copied
0843  *
0844  * Other effects:
0845  *    1. related *ctx fields are updated.
0846  *    2. ctx->copy_size is # of bytes copied
0847  *    3. the portion to be copied is guaranteed to be in the linear part
0848  *
0849  */
0850 static int
0851 vmxnet3_parse_hdr(struct sk_buff *skb, struct vmxnet3_tx_queue *tq,
0852           struct vmxnet3_tx_ctx *ctx,
0853           struct vmxnet3_adapter *adapter)
0854 {
0855     u8 protocol = 0;
0856 
0857     if (ctx->mss) { /* TSO */
0858         if (VMXNET3_VERSION_GE_4(adapter) && skb->encapsulation) {
0859             ctx->l4_offset = skb_inner_transport_offset(skb);
0860             ctx->l4_hdr_size = inner_tcp_hdrlen(skb);
0861             ctx->copy_size = ctx->l4_offset + ctx->l4_hdr_size;
0862         } else {
0863             ctx->l4_offset = skb_transport_offset(skb);
0864             ctx->l4_hdr_size = tcp_hdrlen(skb);
0865             ctx->copy_size = ctx->l4_offset + ctx->l4_hdr_size;
0866         }
0867     } else {
0868         if (skb->ip_summed == CHECKSUM_PARTIAL) {
0869             /* For encap packets, skb_checksum_start_offset refers
0870              * to inner L4 offset. Thus, below works for encap as
0871              * well as non-encap case
0872              */
0873             ctx->l4_offset = skb_checksum_start_offset(skb);
0874 
0875             if (VMXNET3_VERSION_GE_4(adapter) &&
0876                 skb->encapsulation) {
0877                 struct iphdr *iph = inner_ip_hdr(skb);
0878 
0879                 if (iph->version == 4) {
0880                     protocol = iph->protocol;
0881                 } else {
0882                     const struct ipv6hdr *ipv6h;
0883 
0884                     ipv6h = inner_ipv6_hdr(skb);
0885                     protocol = ipv6h->nexthdr;
0886                 }
0887             } else {
0888                 if (ctx->ipv4) {
0889                     const struct iphdr *iph = ip_hdr(skb);
0890 
0891                     protocol = iph->protocol;
0892                 } else if (ctx->ipv6) {
0893                     const struct ipv6hdr *ipv6h;
0894 
0895                     ipv6h = ipv6_hdr(skb);
0896                     protocol = ipv6h->nexthdr;
0897                 }
0898             }
0899 
0900             switch (protocol) {
0901             case IPPROTO_TCP:
0902                 ctx->l4_hdr_size = skb->encapsulation ? inner_tcp_hdrlen(skb) :
0903                            tcp_hdrlen(skb);
0904                 break;
0905             case IPPROTO_UDP:
0906                 ctx->l4_hdr_size = sizeof(struct udphdr);
0907                 break;
0908             default:
0909                 ctx->l4_hdr_size = 0;
0910                 break;
0911             }
0912 
0913             ctx->copy_size = min(ctx->l4_offset +
0914                      ctx->l4_hdr_size, skb->len);
0915         } else {
0916             ctx->l4_offset = 0;
0917             ctx->l4_hdr_size = 0;
0918             /* copy as much as allowed */
0919             ctx->copy_size = min_t(unsigned int,
0920                            tq->txdata_desc_size,
0921                            skb_headlen(skb));
0922         }
0923 
0924         if (skb->len <= VMXNET3_HDR_COPY_SIZE)
0925             ctx->copy_size = skb->len;
0926 
0927         /* make sure headers are accessible directly */
0928         if (unlikely(!pskb_may_pull(skb, ctx->copy_size)))
0929             goto err;
0930     }
0931 
0932     if (unlikely(ctx->copy_size > tq->txdata_desc_size)) {
0933         tq->stats.oversized_hdr++;
0934         ctx->copy_size = 0;
0935         return 0;
0936     }
0937 
0938     return 1;
0939 err:
0940     return -1;
0941 }
0942 
0943 /*
0944  *    copy relevant protocol headers to the transmit ring:
0945  *      For a tso pkt, relevant headers are L2/3/4 including options
0946  *      For a pkt requesting csum offloading, they are L2/3 and may include L4
0947  *      if it's a TCP/UDP pkt
0948  *
0949  *
0950  *    Note that this requires that vmxnet3_parse_hdr be called first to set the
0951  *      appropriate bits in ctx first
0952  */
0953 static void
0954 vmxnet3_copy_hdr(struct sk_buff *skb, struct vmxnet3_tx_queue *tq,
0955          struct vmxnet3_tx_ctx *ctx,
0956          struct vmxnet3_adapter *adapter)
0957 {
0958     struct Vmxnet3_TxDataDesc *tdd;
0959 
0960     tdd = (struct Vmxnet3_TxDataDesc *)((u8 *)tq->data_ring.base +
0961                         tq->tx_ring.next2fill *
0962                         tq->txdata_desc_size);
0963 
0964     memcpy(tdd->data, skb->data, ctx->copy_size);
0965     netdev_dbg(adapter->netdev,
0966         "copy %u bytes to dataRing[%u]\n",
0967         ctx->copy_size, tq->tx_ring.next2fill);
0968 }
0969 
0970 
0971 static void
0972 vmxnet3_prepare_inner_tso(struct sk_buff *skb,
0973               struct vmxnet3_tx_ctx *ctx)
0974 {
0975     struct tcphdr *tcph = inner_tcp_hdr(skb);
0976     struct iphdr *iph = inner_ip_hdr(skb);
0977 
0978     if (iph->version == 4) {
0979         iph->check = 0;
0980         tcph->check = ~csum_tcpudp_magic(iph->saddr, iph->daddr, 0,
0981                          IPPROTO_TCP, 0);
0982     } else {
0983         struct ipv6hdr *iph = inner_ipv6_hdr(skb);
0984 
0985         tcph->check = ~csum_ipv6_magic(&iph->saddr, &iph->daddr, 0,
0986                            IPPROTO_TCP, 0);
0987     }
0988 }
0989 
0990 static void
0991 vmxnet3_prepare_tso(struct sk_buff *skb,
0992             struct vmxnet3_tx_ctx *ctx)
0993 {
0994     struct tcphdr *tcph = tcp_hdr(skb);
0995 
0996     if (ctx->ipv4) {
0997         struct iphdr *iph = ip_hdr(skb);
0998 
0999         iph->check = 0;
1000         tcph->check = ~csum_tcpudp_magic(iph->saddr, iph->daddr, 0,
1001                          IPPROTO_TCP, 0);
1002     } else if (ctx->ipv6) {
1003         tcp_v6_gso_csum_prep(skb);
1004     }
1005 }
1006 
1007 static int txd_estimate(const struct sk_buff *skb)
1008 {
1009     int count = VMXNET3_TXD_NEEDED(skb_headlen(skb)) + 1;
1010     int i;
1011 
1012     for (i = 0; i < skb_shinfo(skb)->nr_frags; i++) {
1013         const skb_frag_t *frag = &skb_shinfo(skb)->frags[i];
1014 
1015         count += VMXNET3_TXD_NEEDED(skb_frag_size(frag));
1016     }
1017     return count;
1018 }
1019 
1020 /*
1021  * Transmits a pkt thru a given tq
1022  * Returns:
1023  *    NETDEV_TX_OK:      descriptors are setup successfully
1024  *    NETDEV_TX_OK:      error occurred, the pkt is dropped
1025  *    NETDEV_TX_BUSY:    tx ring is full, queue is stopped
1026  *
1027  * Side-effects:
1028  *    1. tx ring may be changed
1029  *    2. tq stats may be updated accordingly
1030  *    3. shared->txNumDeferred may be updated
1031  */
1032 
1033 static int
1034 vmxnet3_tq_xmit(struct sk_buff *skb, struct vmxnet3_tx_queue *tq,
1035         struct vmxnet3_adapter *adapter, struct net_device *netdev)
1036 {
1037     int ret;
1038     u32 count;
1039     int num_pkts;
1040     int tx_num_deferred;
1041     unsigned long flags;
1042     struct vmxnet3_tx_ctx ctx;
1043     union Vmxnet3_GenericDesc *gdesc;
1044 #ifdef __BIG_ENDIAN_BITFIELD
1045     /* Use temporary descriptor to avoid touching bits multiple times */
1046     union Vmxnet3_GenericDesc tempTxDesc;
1047 #endif
1048 
1049     count = txd_estimate(skb);
1050 
1051     ctx.ipv4 = (vlan_get_protocol(skb) == cpu_to_be16(ETH_P_IP));
1052     ctx.ipv6 = (vlan_get_protocol(skb) == cpu_to_be16(ETH_P_IPV6));
1053 
1054     ctx.mss = skb_shinfo(skb)->gso_size;
1055     if (ctx.mss) {
1056         if (skb_header_cloned(skb)) {
1057             if (unlikely(pskb_expand_head(skb, 0, 0,
1058                               GFP_ATOMIC) != 0)) {
1059                 tq->stats.drop_tso++;
1060                 goto drop_pkt;
1061             }
1062             tq->stats.copy_skb_header++;
1063         }
1064         if (unlikely(count > VMXNET3_MAX_TSO_TXD_PER_PKT)) {
1065             /* tso pkts must not use more than
1066              * VMXNET3_MAX_TSO_TXD_PER_PKT entries
1067              */
1068             if (skb_linearize(skb) != 0) {
1069                 tq->stats.drop_too_many_frags++;
1070                 goto drop_pkt;
1071             }
1072             tq->stats.linearized++;
1073 
1074             /* recalculate the # of descriptors to use */
1075             count = VMXNET3_TXD_NEEDED(skb_headlen(skb)) + 1;
1076             if (unlikely(count > VMXNET3_MAX_TSO_TXD_PER_PKT)) {
1077                 tq->stats.drop_too_many_frags++;
1078                 goto drop_pkt;
1079             }
1080         }
1081         if (skb->encapsulation) {
1082             vmxnet3_prepare_inner_tso(skb, &ctx);
1083         } else {
1084             vmxnet3_prepare_tso(skb, &ctx);
1085         }
1086     } else {
1087         if (unlikely(count > VMXNET3_MAX_TXD_PER_PKT)) {
1088 
1089             /* non-tso pkts must not use more than
1090              * VMXNET3_MAX_TXD_PER_PKT entries
1091              */
1092             if (skb_linearize(skb) != 0) {
1093                 tq->stats.drop_too_many_frags++;
1094                 goto drop_pkt;
1095             }
1096             tq->stats.linearized++;
1097 
1098             /* recalculate the # of descriptors to use */
1099             count = VMXNET3_TXD_NEEDED(skb_headlen(skb)) + 1;
1100         }
1101     }
1102 
1103     ret = vmxnet3_parse_hdr(skb, tq, &ctx, adapter);
1104     if (ret >= 0) {
1105         BUG_ON(ret <= 0 && ctx.copy_size != 0);
1106         /* hdrs parsed, check against other limits */
1107         if (ctx.mss) {
1108             if (unlikely(ctx.l4_offset + ctx.l4_hdr_size >
1109                      VMXNET3_MAX_TX_BUF_SIZE)) {
1110                 tq->stats.drop_oversized_hdr++;
1111                 goto drop_pkt;
1112             }
1113         } else {
1114             if (skb->ip_summed == CHECKSUM_PARTIAL) {
1115                 if (unlikely(ctx.l4_offset +
1116                          skb->csum_offset >
1117                          VMXNET3_MAX_CSUM_OFFSET)) {
1118                     tq->stats.drop_oversized_hdr++;
1119                     goto drop_pkt;
1120                 }
1121             }
1122         }
1123     } else {
1124         tq->stats.drop_hdr_inspect_err++;
1125         goto drop_pkt;
1126     }
1127 
1128     spin_lock_irqsave(&tq->tx_lock, flags);
1129 
1130     if (count > vmxnet3_cmd_ring_desc_avail(&tq->tx_ring)) {
1131         tq->stats.tx_ring_full++;
1132         netdev_dbg(adapter->netdev,
1133             "tx queue stopped on %s, next2comp %u"
1134             " next2fill %u\n", adapter->netdev->name,
1135             tq->tx_ring.next2comp, tq->tx_ring.next2fill);
1136 
1137         vmxnet3_tq_stop(tq, adapter);
1138         spin_unlock_irqrestore(&tq->tx_lock, flags);
1139         return NETDEV_TX_BUSY;
1140     }
1141 
1142 
1143     vmxnet3_copy_hdr(skb, tq, &ctx, adapter);
1144 
1145     /* fill tx descs related to addr & len */
1146     if (vmxnet3_map_pkt(skb, &ctx, tq, adapter->pdev, adapter))
1147         goto unlock_drop_pkt;
1148 
1149     /* setup the EOP desc */
1150     ctx.eop_txd->dword[3] = cpu_to_le32(VMXNET3_TXD_CQ | VMXNET3_TXD_EOP);
1151 
1152     /* setup the SOP desc */
1153 #ifdef __BIG_ENDIAN_BITFIELD
1154     gdesc = &tempTxDesc;
1155     gdesc->dword[2] = ctx.sop_txd->dword[2];
1156     gdesc->dword[3] = ctx.sop_txd->dword[3];
1157 #else
1158     gdesc = ctx.sop_txd;
1159 #endif
1160     tx_num_deferred = le32_to_cpu(tq->shared->txNumDeferred);
1161     if (ctx.mss) {
1162         if (VMXNET3_VERSION_GE_4(adapter) && skb->encapsulation) {
1163             gdesc->txd.hlen = ctx.l4_offset + ctx.l4_hdr_size;
1164             if (VMXNET3_VERSION_GE_7(adapter)) {
1165                 gdesc->txd.om = VMXNET3_OM_TSO;
1166                 gdesc->txd.ext1 = 1;
1167             } else {
1168                 gdesc->txd.om = VMXNET3_OM_ENCAP;
1169             }
1170             gdesc->txd.msscof = ctx.mss;
1171 
1172             if (skb_shinfo(skb)->gso_type & SKB_GSO_UDP_TUNNEL_CSUM)
1173                 gdesc->txd.oco = 1;
1174         } else {
1175             gdesc->txd.hlen = ctx.l4_offset + ctx.l4_hdr_size;
1176             gdesc->txd.om = VMXNET3_OM_TSO;
1177             gdesc->txd.msscof = ctx.mss;
1178         }
1179         num_pkts = (skb->len - gdesc->txd.hlen + ctx.mss - 1) / ctx.mss;
1180     } else {
1181         if (skb->ip_summed == CHECKSUM_PARTIAL) {
1182             if (VMXNET3_VERSION_GE_4(adapter) &&
1183                 skb->encapsulation) {
1184                 gdesc->txd.hlen = ctx.l4_offset +
1185                           ctx.l4_hdr_size;
1186                 if (VMXNET3_VERSION_GE_7(adapter)) {
1187                     gdesc->txd.om = VMXNET3_OM_CSUM;
1188                     gdesc->txd.msscof = ctx.l4_offset +
1189                                 skb->csum_offset;
1190                     gdesc->txd.ext1 = 1;
1191                 } else {
1192                     gdesc->txd.om = VMXNET3_OM_ENCAP;
1193                     gdesc->txd.msscof = 0;      /* Reserved */
1194                 }
1195             } else {
1196                 gdesc->txd.hlen = ctx.l4_offset;
1197                 gdesc->txd.om = VMXNET3_OM_CSUM;
1198                 gdesc->txd.msscof = ctx.l4_offset +
1199                             skb->csum_offset;
1200             }
1201         } else {
1202             gdesc->txd.om = 0;
1203             gdesc->txd.msscof = 0;
1204         }
1205         num_pkts = 1;
1206     }
1207     le32_add_cpu(&tq->shared->txNumDeferred, num_pkts);
1208     tx_num_deferred += num_pkts;
1209 
1210     if (skb_vlan_tag_present(skb)) {
1211         gdesc->txd.ti = 1;
1212         gdesc->txd.tci = skb_vlan_tag_get(skb);
1213     }
1214 
1215     /* Ensure that the write to (&gdesc->txd)->gen will be observed after
1216      * all other writes to &gdesc->txd.
1217      */
1218     dma_wmb();
1219 
1220     /* finally flips the GEN bit of the SOP desc. */
1221     gdesc->dword[2] = cpu_to_le32(le32_to_cpu(gdesc->dword[2]) ^
1222                           VMXNET3_TXD_GEN);
1223 #ifdef __BIG_ENDIAN_BITFIELD
1224     /* Finished updating in bitfields of Tx Desc, so write them in original
1225      * place.
1226      */
1227     vmxnet3_TxDescToLe((struct Vmxnet3_TxDesc *)gdesc,
1228                (struct Vmxnet3_TxDesc *)ctx.sop_txd);
1229     gdesc = ctx.sop_txd;
1230 #endif
1231     netdev_dbg(adapter->netdev,
1232         "txd[%u]: SOP 0x%Lx 0x%x 0x%x\n",
1233         (u32)(ctx.sop_txd -
1234         tq->tx_ring.base), le64_to_cpu(gdesc->txd.addr),
1235         le32_to_cpu(gdesc->dword[2]), le32_to_cpu(gdesc->dword[3]));
1236 
1237     spin_unlock_irqrestore(&tq->tx_lock, flags);
1238 
1239     if (tx_num_deferred >= le32_to_cpu(tq->shared->txThreshold)) {
1240         tq->shared->txNumDeferred = 0;
1241         VMXNET3_WRITE_BAR0_REG(adapter,
1242                        adapter->tx_prod_offset + tq->qid * 8,
1243                        tq->tx_ring.next2fill);
1244     }
1245 
1246     return NETDEV_TX_OK;
1247 
1248 unlock_drop_pkt:
1249     spin_unlock_irqrestore(&tq->tx_lock, flags);
1250 drop_pkt:
1251     tq->stats.drop_total++;
1252     dev_kfree_skb_any(skb);
1253     return NETDEV_TX_OK;
1254 }
1255 
1256 
1257 static netdev_tx_t
1258 vmxnet3_xmit_frame(struct sk_buff *skb, struct net_device *netdev)
1259 {
1260     struct vmxnet3_adapter *adapter = netdev_priv(netdev);
1261 
1262     BUG_ON(skb->queue_mapping > adapter->num_tx_queues);
1263     return vmxnet3_tq_xmit(skb,
1264                    &adapter->tx_queue[skb->queue_mapping],
1265                    adapter, netdev);
1266 }
1267 
1268 
1269 static void
1270 vmxnet3_rx_csum(struct vmxnet3_adapter *adapter,
1271         struct sk_buff *skb,
1272         union Vmxnet3_GenericDesc *gdesc)
1273 {
1274     if (!gdesc->rcd.cnc && adapter->netdev->features & NETIF_F_RXCSUM) {
1275         if (gdesc->rcd.v4 &&
1276             (le32_to_cpu(gdesc->dword[3]) &
1277              VMXNET3_RCD_CSUM_OK) == VMXNET3_RCD_CSUM_OK) {
1278             skb->ip_summed = CHECKSUM_UNNECESSARY;
1279             WARN_ON_ONCE(!(gdesc->rcd.tcp || gdesc->rcd.udp) &&
1280                      !(le32_to_cpu(gdesc->dword[0]) &
1281                      (1UL << VMXNET3_RCD_HDR_INNER_SHIFT)));
1282             WARN_ON_ONCE(gdesc->rcd.frg &&
1283                      !(le32_to_cpu(gdesc->dword[0]) &
1284                      (1UL << VMXNET3_RCD_HDR_INNER_SHIFT)));
1285         } else if (gdesc->rcd.v6 && (le32_to_cpu(gdesc->dword[3]) &
1286                          (1 << VMXNET3_RCD_TUC_SHIFT))) {
1287             skb->ip_summed = CHECKSUM_UNNECESSARY;
1288             WARN_ON_ONCE(!(gdesc->rcd.tcp || gdesc->rcd.udp) &&
1289                      !(le32_to_cpu(gdesc->dword[0]) &
1290                      (1UL << VMXNET3_RCD_HDR_INNER_SHIFT)));
1291             WARN_ON_ONCE(gdesc->rcd.frg &&
1292                      !(le32_to_cpu(gdesc->dword[0]) &
1293                      (1UL << VMXNET3_RCD_HDR_INNER_SHIFT)));
1294         } else {
1295             if (gdesc->rcd.csum) {
1296                 skb->csum = htons(gdesc->rcd.csum);
1297                 skb->ip_summed = CHECKSUM_PARTIAL;
1298             } else {
1299                 skb_checksum_none_assert(skb);
1300             }
1301         }
1302     } else {
1303         skb_checksum_none_assert(skb);
1304     }
1305 }
1306 
1307 
1308 static void
1309 vmxnet3_rx_error(struct vmxnet3_rx_queue *rq, struct Vmxnet3_RxCompDesc *rcd,
1310          struct vmxnet3_rx_ctx *ctx,  struct vmxnet3_adapter *adapter)
1311 {
1312     rq->stats.drop_err++;
1313     if (!rcd->fcs)
1314         rq->stats.drop_fcs++;
1315 
1316     rq->stats.drop_total++;
1317 
1318     /*
1319      * We do not unmap and chain the rx buffer to the skb.
1320      * We basically pretend this buffer is not used and will be recycled
1321      * by vmxnet3_rq_alloc_rx_buf()
1322      */
1323 
1324     /*
1325      * ctx->skb may be NULL if this is the first and the only one
1326      * desc for the pkt
1327      */
1328     if (ctx->skb)
1329         dev_kfree_skb_irq(ctx->skb);
1330 
1331     ctx->skb = NULL;
1332 }
1333 
1334 
1335 static u32
1336 vmxnet3_get_hdr_len(struct vmxnet3_adapter *adapter, struct sk_buff *skb,
1337             union Vmxnet3_GenericDesc *gdesc)
1338 {
1339     u32 hlen, maplen;
1340     union {
1341         void *ptr;
1342         struct ethhdr *eth;
1343         struct vlan_ethhdr *veth;
1344         struct iphdr *ipv4;
1345         struct ipv6hdr *ipv6;
1346         struct tcphdr *tcp;
1347     } hdr;
1348     BUG_ON(gdesc->rcd.tcp == 0);
1349 
1350     maplen = skb_headlen(skb);
1351     if (unlikely(sizeof(struct iphdr) + sizeof(struct tcphdr) > maplen))
1352         return 0;
1353 
1354     if (skb->protocol == cpu_to_be16(ETH_P_8021Q) ||
1355         skb->protocol == cpu_to_be16(ETH_P_8021AD))
1356         hlen = sizeof(struct vlan_ethhdr);
1357     else
1358         hlen = sizeof(struct ethhdr);
1359 
1360     hdr.eth = eth_hdr(skb);
1361     if (gdesc->rcd.v4) {
1362         BUG_ON(hdr.eth->h_proto != htons(ETH_P_IP) &&
1363                hdr.veth->h_vlan_encapsulated_proto != htons(ETH_P_IP));
1364         hdr.ptr += hlen;
1365         BUG_ON(hdr.ipv4->protocol != IPPROTO_TCP);
1366         hlen = hdr.ipv4->ihl << 2;
1367         hdr.ptr += hdr.ipv4->ihl << 2;
1368     } else if (gdesc->rcd.v6) {
1369         BUG_ON(hdr.eth->h_proto != htons(ETH_P_IPV6) &&
1370                hdr.veth->h_vlan_encapsulated_proto != htons(ETH_P_IPV6));
1371         hdr.ptr += hlen;
1372         /* Use an estimated value, since we also need to handle
1373          * TSO case.
1374          */
1375         if (hdr.ipv6->nexthdr != IPPROTO_TCP)
1376             return sizeof(struct ipv6hdr) + sizeof(struct tcphdr);
1377         hlen = sizeof(struct ipv6hdr);
1378         hdr.ptr += sizeof(struct ipv6hdr);
1379     } else {
1380         /* Non-IP pkt, dont estimate header length */
1381         return 0;
1382     }
1383 
1384     if (hlen + sizeof(struct tcphdr) > maplen)
1385         return 0;
1386 
1387     return (hlen + (hdr.tcp->doff << 2));
1388 }
1389 
1390 static int
1391 vmxnet3_rq_rx_complete(struct vmxnet3_rx_queue *rq,
1392                struct vmxnet3_adapter *adapter, int quota)
1393 {
1394     u32 rxprod_reg[2] = {
1395         adapter->rx_prod_offset, adapter->rx_prod2_offset
1396     };
1397     u32 num_pkts = 0;
1398     bool skip_page_frags = false;
1399     struct Vmxnet3_RxCompDesc *rcd;
1400     struct vmxnet3_rx_ctx *ctx = &rq->rx_ctx;
1401     u16 segCnt = 0, mss = 0;
1402     int comp_offset, fill_offset;
1403 #ifdef __BIG_ENDIAN_BITFIELD
1404     struct Vmxnet3_RxDesc rxCmdDesc;
1405     struct Vmxnet3_RxCompDesc rxComp;
1406 #endif
1407     vmxnet3_getRxComp(rcd, &rq->comp_ring.base[rq->comp_ring.next2proc].rcd,
1408               &rxComp);
1409     while (rcd->gen == rq->comp_ring.gen) {
1410         struct vmxnet3_rx_buf_info *rbi;
1411         struct sk_buff *skb, *new_skb = NULL;
1412         struct page *new_page = NULL;
1413         dma_addr_t new_dma_addr;
1414         int num_to_alloc;
1415         struct Vmxnet3_RxDesc *rxd;
1416         u32 idx, ring_idx;
1417         struct vmxnet3_cmd_ring *ring = NULL;
1418         if (num_pkts >= quota) {
1419             /* we may stop even before we see the EOP desc of
1420              * the current pkt
1421              */
1422             break;
1423         }
1424 
1425         /* Prevent any rcd field from being (speculatively) read before
1426          * rcd->gen is read.
1427          */
1428         dma_rmb();
1429 
1430         BUG_ON(rcd->rqID != rq->qid && rcd->rqID != rq->qid2 &&
1431                rcd->rqID != rq->dataRingQid);
1432         idx = rcd->rxdIdx;
1433         ring_idx = VMXNET3_GET_RING_IDX(adapter, rcd->rqID);
1434         ring = rq->rx_ring + ring_idx;
1435         vmxnet3_getRxDesc(rxd, &rq->rx_ring[ring_idx].base[idx].rxd,
1436                   &rxCmdDesc);
1437         rbi = rq->buf_info[ring_idx] + idx;
1438 
1439         BUG_ON(rxd->addr != rbi->dma_addr ||
1440                rxd->len != rbi->len);
1441 
1442         if (unlikely(rcd->eop && rcd->err)) {
1443             vmxnet3_rx_error(rq, rcd, ctx, adapter);
1444             goto rcd_done;
1445         }
1446 
1447         if (rcd->sop) { /* first buf of the pkt */
1448             bool rxDataRingUsed;
1449             u16 len;
1450 
1451             BUG_ON(rxd->btype != VMXNET3_RXD_BTYPE_HEAD ||
1452                    (rcd->rqID != rq->qid &&
1453                 rcd->rqID != rq->dataRingQid));
1454 
1455             BUG_ON(rbi->buf_type != VMXNET3_RX_BUF_SKB);
1456             BUG_ON(ctx->skb != NULL || rbi->skb == NULL);
1457 
1458             if (unlikely(rcd->len == 0)) {
1459                 /* Pretend the rx buffer is skipped. */
1460                 BUG_ON(!(rcd->sop && rcd->eop));
1461                 netdev_dbg(adapter->netdev,
1462                     "rxRing[%u][%u] 0 length\n",
1463                     ring_idx, idx);
1464                 goto rcd_done;
1465             }
1466 
1467             skip_page_frags = false;
1468             ctx->skb = rbi->skb;
1469 
1470             rxDataRingUsed =
1471                 VMXNET3_RX_DATA_RING(adapter, rcd->rqID);
1472             len = rxDataRingUsed ? rcd->len : rbi->len;
1473             new_skb = netdev_alloc_skb_ip_align(adapter->netdev,
1474                                 len);
1475             if (new_skb == NULL) {
1476                 /* Skb allocation failed, do not handover this
1477                  * skb to stack. Reuse it. Drop the existing pkt
1478                  */
1479                 rq->stats.rx_buf_alloc_failure++;
1480                 ctx->skb = NULL;
1481                 rq->stats.drop_total++;
1482                 skip_page_frags = true;
1483                 goto rcd_done;
1484             }
1485 
1486             if (rxDataRingUsed) {
1487                 size_t sz;
1488 
1489                 BUG_ON(rcd->len > rq->data_ring.desc_size);
1490 
1491                 ctx->skb = new_skb;
1492                 sz = rcd->rxdIdx * rq->data_ring.desc_size;
1493                 memcpy(new_skb->data,
1494                        &rq->data_ring.base[sz], rcd->len);
1495             } else {
1496                 ctx->skb = rbi->skb;
1497 
1498                 new_dma_addr =
1499                     dma_map_single(&adapter->pdev->dev,
1500                                new_skb->data, rbi->len,
1501                                DMA_FROM_DEVICE);
1502                 if (dma_mapping_error(&adapter->pdev->dev,
1503                               new_dma_addr)) {
1504                     dev_kfree_skb(new_skb);
1505                     /* Skb allocation failed, do not
1506                      * handover this skb to stack. Reuse
1507                      * it. Drop the existing pkt.
1508                      */
1509                     rq->stats.rx_buf_alloc_failure++;
1510                     ctx->skb = NULL;
1511                     rq->stats.drop_total++;
1512                     skip_page_frags = true;
1513                     goto rcd_done;
1514                 }
1515 
1516                 dma_unmap_single(&adapter->pdev->dev,
1517                          rbi->dma_addr,
1518                          rbi->len,
1519                          DMA_FROM_DEVICE);
1520 
1521                 /* Immediate refill */
1522                 rbi->skb = new_skb;
1523                 rbi->dma_addr = new_dma_addr;
1524                 rxd->addr = cpu_to_le64(rbi->dma_addr);
1525                 rxd->len = rbi->len;
1526             }
1527 
1528 #ifdef VMXNET3_RSS
1529             if (rcd->rssType != VMXNET3_RCD_RSS_TYPE_NONE &&
1530                 (adapter->netdev->features & NETIF_F_RXHASH)) {
1531                 enum pkt_hash_types hash_type;
1532 
1533                 switch (rcd->rssType) {
1534                 case VMXNET3_RCD_RSS_TYPE_IPV4:
1535                 case VMXNET3_RCD_RSS_TYPE_IPV6:
1536                     hash_type = PKT_HASH_TYPE_L3;
1537                     break;
1538                 case VMXNET3_RCD_RSS_TYPE_TCPIPV4:
1539                 case VMXNET3_RCD_RSS_TYPE_TCPIPV6:
1540                 case VMXNET3_RCD_RSS_TYPE_UDPIPV4:
1541                 case VMXNET3_RCD_RSS_TYPE_UDPIPV6:
1542                     hash_type = PKT_HASH_TYPE_L4;
1543                     break;
1544                 default:
1545                     hash_type = PKT_HASH_TYPE_L3;
1546                     break;
1547                 }
1548                 skb_set_hash(ctx->skb,
1549                          le32_to_cpu(rcd->rssHash),
1550                          hash_type);
1551             }
1552 #endif
1553             skb_record_rx_queue(ctx->skb, rq->qid);
1554             skb_put(ctx->skb, rcd->len);
1555 
1556             if (VMXNET3_VERSION_GE_2(adapter) &&
1557                 rcd->type == VMXNET3_CDTYPE_RXCOMP_LRO) {
1558                 struct Vmxnet3_RxCompDescExt *rcdlro;
1559                 rcdlro = (struct Vmxnet3_RxCompDescExt *)rcd;
1560 
1561                 segCnt = rcdlro->segCnt;
1562                 WARN_ON_ONCE(segCnt == 0);
1563                 mss = rcdlro->mss;
1564                 if (unlikely(segCnt <= 1))
1565                     segCnt = 0;
1566             } else {
1567                 segCnt = 0;
1568             }
1569         } else {
1570             BUG_ON(ctx->skb == NULL && !skip_page_frags);
1571 
1572             /* non SOP buffer must be type 1 in most cases */
1573             BUG_ON(rbi->buf_type != VMXNET3_RX_BUF_PAGE);
1574             BUG_ON(rxd->btype != VMXNET3_RXD_BTYPE_BODY);
1575 
1576             /* If an sop buffer was dropped, skip all
1577              * following non-sop fragments. They will be reused.
1578              */
1579             if (skip_page_frags)
1580                 goto rcd_done;
1581 
1582             if (rcd->len) {
1583                 new_page = alloc_page(GFP_ATOMIC);
1584                 /* Replacement page frag could not be allocated.
1585                  * Reuse this page. Drop the pkt and free the
1586                  * skb which contained this page as a frag. Skip
1587                  * processing all the following non-sop frags.
1588                  */
1589                 if (unlikely(!new_page)) {
1590                     rq->stats.rx_buf_alloc_failure++;
1591                     dev_kfree_skb(ctx->skb);
1592                     ctx->skb = NULL;
1593                     skip_page_frags = true;
1594                     goto rcd_done;
1595                 }
1596                 new_dma_addr = dma_map_page(&adapter->pdev->dev,
1597                                 new_page,
1598                                 0, PAGE_SIZE,
1599                                 DMA_FROM_DEVICE);
1600                 if (dma_mapping_error(&adapter->pdev->dev,
1601                               new_dma_addr)) {
1602                     put_page(new_page);
1603                     rq->stats.rx_buf_alloc_failure++;
1604                     dev_kfree_skb(ctx->skb);
1605                     ctx->skb = NULL;
1606                     skip_page_frags = true;
1607                     goto rcd_done;
1608                 }
1609 
1610                 dma_unmap_page(&adapter->pdev->dev,
1611                            rbi->dma_addr, rbi->len,
1612                            DMA_FROM_DEVICE);
1613 
1614                 vmxnet3_append_frag(ctx->skb, rcd, rbi);
1615 
1616                 /* Immediate refill */
1617                 rbi->page = new_page;
1618                 rbi->dma_addr = new_dma_addr;
1619                 rxd->addr = cpu_to_le64(rbi->dma_addr);
1620                 rxd->len = rbi->len;
1621             }
1622         }
1623 
1624 
1625         skb = ctx->skb;
1626         if (rcd->eop) {
1627             u32 mtu = adapter->netdev->mtu;
1628             skb->len += skb->data_len;
1629 
1630             vmxnet3_rx_csum(adapter, skb,
1631                     (union Vmxnet3_GenericDesc *)rcd);
1632             skb->protocol = eth_type_trans(skb, adapter->netdev);
1633             if (!rcd->tcp ||
1634                 !(adapter->netdev->features & NETIF_F_LRO))
1635                 goto not_lro;
1636 
1637             if (segCnt != 0 && mss != 0) {
1638                 skb_shinfo(skb)->gso_type = rcd->v4 ?
1639                     SKB_GSO_TCPV4 : SKB_GSO_TCPV6;
1640                 skb_shinfo(skb)->gso_size = mss;
1641                 skb_shinfo(skb)->gso_segs = segCnt;
1642             } else if (segCnt != 0 || skb->len > mtu) {
1643                 u32 hlen;
1644 
1645                 hlen = vmxnet3_get_hdr_len(adapter, skb,
1646                     (union Vmxnet3_GenericDesc *)rcd);
1647                 if (hlen == 0)
1648                     goto not_lro;
1649 
1650                 skb_shinfo(skb)->gso_type =
1651                     rcd->v4 ? SKB_GSO_TCPV4 : SKB_GSO_TCPV6;
1652                 if (segCnt != 0) {
1653                     skb_shinfo(skb)->gso_segs = segCnt;
1654                     skb_shinfo(skb)->gso_size =
1655                         DIV_ROUND_UP(skb->len -
1656                             hlen, segCnt);
1657                 } else {
1658                     skb_shinfo(skb)->gso_size = mtu - hlen;
1659                 }
1660             }
1661 not_lro:
1662             if (unlikely(rcd->ts))
1663                 __vlan_hwaccel_put_tag(skb, htons(ETH_P_8021Q), rcd->tci);
1664 
1665             if (adapter->netdev->features & NETIF_F_LRO)
1666                 netif_receive_skb(skb);
1667             else
1668                 napi_gro_receive(&rq->napi, skb);
1669 
1670             ctx->skb = NULL;
1671             num_pkts++;
1672         }
1673 
1674 rcd_done:
1675         /* device may have skipped some rx descs */
1676         ring = rq->rx_ring + ring_idx;
1677         rbi->comp_state = VMXNET3_RXD_COMP_DONE;
1678 
1679         comp_offset = vmxnet3_cmd_ring_desc_avail(ring);
1680         fill_offset = (idx > ring->next2fill ? 0 : ring->size) +
1681                   idx - ring->next2fill - 1;
1682         if (!ring->isOutOfOrder || fill_offset >= comp_offset)
1683             ring->next2comp = idx;
1684         num_to_alloc = vmxnet3_cmd_ring_desc_avail(ring);
1685 
1686         /* Ensure that the writes to rxd->gen bits will be observed
1687          * after all other writes to rxd objects.
1688          */
1689         dma_wmb();
1690 
1691         while (num_to_alloc) {
1692             rbi = rq->buf_info[ring_idx] + ring->next2fill;
1693             if (!(adapter->dev_caps[0] & (1UL << VMXNET3_CAP_OOORX_COMP)))
1694                 goto refill_buf;
1695             if (ring_idx == 0) {
1696                 /* ring0 Type1 buffers can get skipped; re-fill them */
1697                 if (rbi->buf_type != VMXNET3_RX_BUF_SKB)
1698                     goto refill_buf;
1699             }
1700             if (rbi->comp_state == VMXNET3_RXD_COMP_DONE) {
1701 refill_buf:
1702                 vmxnet3_getRxDesc(rxd, &ring->base[ring->next2fill].rxd,
1703                           &rxCmdDesc);
1704                 WARN_ON(!rxd->addr);
1705 
1706                 /* Recv desc is ready to be used by the device */
1707                 rxd->gen = ring->gen;
1708                 vmxnet3_cmd_ring_adv_next2fill(ring);
1709                 rbi->comp_state = VMXNET3_RXD_COMP_PENDING;
1710                 num_to_alloc--;
1711             } else {
1712                 /* rx completion hasn't occurred */
1713                 ring->isOutOfOrder = 1;
1714                 break;
1715             }
1716         }
1717 
1718         if (num_to_alloc == 0) {
1719             ring->isOutOfOrder = 0;
1720         }
1721 
1722         /* if needed, update the register */
1723         if (unlikely(rq->shared->updateRxProd) && (ring->next2fill & 0xf) == 0) {
1724             VMXNET3_WRITE_BAR0_REG(adapter,
1725                            rxprod_reg[ring_idx] + rq->qid * 8,
1726                            ring->next2fill);
1727         }
1728 
1729         vmxnet3_comp_ring_adv_next2proc(&rq->comp_ring);
1730         vmxnet3_getRxComp(rcd,
1731                   &rq->comp_ring.base[rq->comp_ring.next2proc].rcd, &rxComp);
1732     }
1733 
1734     return num_pkts;
1735 }
1736 
1737 
1738 static void
1739 vmxnet3_rq_cleanup(struct vmxnet3_rx_queue *rq,
1740            struct vmxnet3_adapter *adapter)
1741 {
1742     u32 i, ring_idx;
1743     struct Vmxnet3_RxDesc *rxd;
1744 
1745     /* ring has already been cleaned up */
1746     if (!rq->rx_ring[0].base)
1747         return;
1748 
1749     for (ring_idx = 0; ring_idx < 2; ring_idx++) {
1750         for (i = 0; i < rq->rx_ring[ring_idx].size; i++) {
1751 #ifdef __BIG_ENDIAN_BITFIELD
1752             struct Vmxnet3_RxDesc rxDesc;
1753 #endif
1754             vmxnet3_getRxDesc(rxd,
1755                 &rq->rx_ring[ring_idx].base[i].rxd, &rxDesc);
1756 
1757             if (rxd->btype == VMXNET3_RXD_BTYPE_HEAD &&
1758                     rq->buf_info[ring_idx][i].skb) {
1759                 dma_unmap_single(&adapter->pdev->dev, rxd->addr,
1760                          rxd->len, DMA_FROM_DEVICE);
1761                 dev_kfree_skb(rq->buf_info[ring_idx][i].skb);
1762                 rq->buf_info[ring_idx][i].skb = NULL;
1763             } else if (rxd->btype == VMXNET3_RXD_BTYPE_BODY &&
1764                     rq->buf_info[ring_idx][i].page) {
1765                 dma_unmap_page(&adapter->pdev->dev, rxd->addr,
1766                            rxd->len, DMA_FROM_DEVICE);
1767                 put_page(rq->buf_info[ring_idx][i].page);
1768                 rq->buf_info[ring_idx][i].page = NULL;
1769             }
1770         }
1771 
1772         rq->rx_ring[ring_idx].gen = VMXNET3_INIT_GEN;
1773         rq->rx_ring[ring_idx].next2fill =
1774                     rq->rx_ring[ring_idx].next2comp = 0;
1775     }
1776 
1777     rq->comp_ring.gen = VMXNET3_INIT_GEN;
1778     rq->comp_ring.next2proc = 0;
1779 }
1780 
1781 
1782 static void
1783 vmxnet3_rq_cleanup_all(struct vmxnet3_adapter *adapter)
1784 {
1785     int i;
1786 
1787     for (i = 0; i < adapter->num_rx_queues; i++)
1788         vmxnet3_rq_cleanup(&adapter->rx_queue[i], adapter);
1789 }
1790 
1791 
1792 static void vmxnet3_rq_destroy(struct vmxnet3_rx_queue *rq,
1793                    struct vmxnet3_adapter *adapter)
1794 {
1795     int i;
1796     int j;
1797 
1798     /* all rx buffers must have already been freed */
1799     for (i = 0; i < 2; i++) {
1800         if (rq->buf_info[i]) {
1801             for (j = 0; j < rq->rx_ring[i].size; j++)
1802                 BUG_ON(rq->buf_info[i][j].page != NULL);
1803         }
1804     }
1805 
1806 
1807     for (i = 0; i < 2; i++) {
1808         if (rq->rx_ring[i].base) {
1809             dma_free_coherent(&adapter->pdev->dev,
1810                       rq->rx_ring[i].size
1811                       * sizeof(struct Vmxnet3_RxDesc),
1812                       rq->rx_ring[i].base,
1813                       rq->rx_ring[i].basePA);
1814             rq->rx_ring[i].base = NULL;
1815         }
1816     }
1817 
1818     if (rq->data_ring.base) {
1819         dma_free_coherent(&adapter->pdev->dev,
1820                   rq->rx_ring[0].size * rq->data_ring.desc_size,
1821                   rq->data_ring.base, rq->data_ring.basePA);
1822         rq->data_ring.base = NULL;
1823     }
1824 
1825     if (rq->comp_ring.base) {
1826         dma_free_coherent(&adapter->pdev->dev, rq->comp_ring.size
1827                   * sizeof(struct Vmxnet3_RxCompDesc),
1828                   rq->comp_ring.base, rq->comp_ring.basePA);
1829         rq->comp_ring.base = NULL;
1830     }
1831 
1832     kfree(rq->buf_info[0]);
1833     rq->buf_info[0] = NULL;
1834     rq->buf_info[1] = NULL;
1835 }
1836 
1837 static void
1838 vmxnet3_rq_destroy_all_rxdataring(struct vmxnet3_adapter *adapter)
1839 {
1840     int i;
1841 
1842     for (i = 0; i < adapter->num_rx_queues; i++) {
1843         struct vmxnet3_rx_queue *rq = &adapter->rx_queue[i];
1844 
1845         if (rq->data_ring.base) {
1846             dma_free_coherent(&adapter->pdev->dev,
1847                       (rq->rx_ring[0].size *
1848                       rq->data_ring.desc_size),
1849                       rq->data_ring.base,
1850                       rq->data_ring.basePA);
1851             rq->data_ring.base = NULL;
1852             rq->data_ring.desc_size = 0;
1853         }
1854     }
1855 }
1856 
1857 static int
1858 vmxnet3_rq_init(struct vmxnet3_rx_queue *rq,
1859         struct vmxnet3_adapter  *adapter)
1860 {
1861     int i;
1862 
1863     /* initialize buf_info */
1864     for (i = 0; i < rq->rx_ring[0].size; i++) {
1865 
1866         /* 1st buf for a pkt is skbuff */
1867         if (i % adapter->rx_buf_per_pkt == 0) {
1868             rq->buf_info[0][i].buf_type = VMXNET3_RX_BUF_SKB;
1869             rq->buf_info[0][i].len = adapter->skb_buf_size;
1870         } else { /* subsequent bufs for a pkt is frag */
1871             rq->buf_info[0][i].buf_type = VMXNET3_RX_BUF_PAGE;
1872             rq->buf_info[0][i].len = PAGE_SIZE;
1873         }
1874     }
1875     for (i = 0; i < rq->rx_ring[1].size; i++) {
1876         rq->buf_info[1][i].buf_type = VMXNET3_RX_BUF_PAGE;
1877         rq->buf_info[1][i].len = PAGE_SIZE;
1878     }
1879 
1880     /* reset internal state and allocate buffers for both rings */
1881     for (i = 0; i < 2; i++) {
1882         rq->rx_ring[i].next2fill = rq->rx_ring[i].next2comp = 0;
1883 
1884         memset(rq->rx_ring[i].base, 0, rq->rx_ring[i].size *
1885                sizeof(struct Vmxnet3_RxDesc));
1886         rq->rx_ring[i].gen = VMXNET3_INIT_GEN;
1887         rq->rx_ring[i].isOutOfOrder = 0;
1888     }
1889     if (vmxnet3_rq_alloc_rx_buf(rq, 0, rq->rx_ring[0].size - 1,
1890                     adapter) == 0) {
1891         /* at least has 1 rx buffer for the 1st ring */
1892         return -ENOMEM;
1893     }
1894     vmxnet3_rq_alloc_rx_buf(rq, 1, rq->rx_ring[1].size - 1, adapter);
1895 
1896     /* reset the comp ring */
1897     rq->comp_ring.next2proc = 0;
1898     memset(rq->comp_ring.base, 0, rq->comp_ring.size *
1899            sizeof(struct Vmxnet3_RxCompDesc));
1900     rq->comp_ring.gen = VMXNET3_INIT_GEN;
1901 
1902     /* reset rxctx */
1903     rq->rx_ctx.skb = NULL;
1904 
1905     /* stats are not reset */
1906     return 0;
1907 }
1908 
1909 
1910 static int
1911 vmxnet3_rq_init_all(struct vmxnet3_adapter *adapter)
1912 {
1913     int i, err = 0;
1914 
1915     for (i = 0; i < adapter->num_rx_queues; i++) {
1916         err = vmxnet3_rq_init(&adapter->rx_queue[i], adapter);
1917         if (unlikely(err)) {
1918             dev_err(&adapter->netdev->dev, "%s: failed to "
1919                 "initialize rx queue%i\n",
1920                 adapter->netdev->name, i);
1921             break;
1922         }
1923     }
1924     return err;
1925 
1926 }
1927 
1928 
1929 static int
1930 vmxnet3_rq_create(struct vmxnet3_rx_queue *rq, struct vmxnet3_adapter *adapter)
1931 {
1932     int i;
1933     size_t sz;
1934     struct vmxnet3_rx_buf_info *bi;
1935 
1936     for (i = 0; i < 2; i++) {
1937 
1938         sz = rq->rx_ring[i].size * sizeof(struct Vmxnet3_RxDesc);
1939         rq->rx_ring[i].base = dma_alloc_coherent(
1940                         &adapter->pdev->dev, sz,
1941                         &rq->rx_ring[i].basePA,
1942                         GFP_KERNEL);
1943         if (!rq->rx_ring[i].base) {
1944             netdev_err(adapter->netdev,
1945                    "failed to allocate rx ring %d\n", i);
1946             goto err;
1947         }
1948     }
1949 
1950     if ((adapter->rxdataring_enabled) && (rq->data_ring.desc_size != 0)) {
1951         sz = rq->rx_ring[0].size * rq->data_ring.desc_size;
1952         rq->data_ring.base =
1953             dma_alloc_coherent(&adapter->pdev->dev, sz,
1954                        &rq->data_ring.basePA,
1955                        GFP_KERNEL);
1956         if (!rq->data_ring.base) {
1957             netdev_err(adapter->netdev,
1958                    "rx data ring will be disabled\n");
1959             adapter->rxdataring_enabled = false;
1960         }
1961     } else {
1962         rq->data_ring.base = NULL;
1963         rq->data_ring.desc_size = 0;
1964     }
1965 
1966     sz = rq->comp_ring.size * sizeof(struct Vmxnet3_RxCompDesc);
1967     rq->comp_ring.base = dma_alloc_coherent(&adapter->pdev->dev, sz,
1968                         &rq->comp_ring.basePA,
1969                         GFP_KERNEL);
1970     if (!rq->comp_ring.base) {
1971         netdev_err(adapter->netdev, "failed to allocate rx comp ring\n");
1972         goto err;
1973     }
1974 
1975     bi = kcalloc_node(rq->rx_ring[0].size + rq->rx_ring[1].size,
1976               sizeof(rq->buf_info[0][0]), GFP_KERNEL,
1977               dev_to_node(&adapter->pdev->dev));
1978     if (!bi)
1979         goto err;
1980 
1981     rq->buf_info[0] = bi;
1982     rq->buf_info[1] = bi + rq->rx_ring[0].size;
1983 
1984     return 0;
1985 
1986 err:
1987     vmxnet3_rq_destroy(rq, adapter);
1988     return -ENOMEM;
1989 }
1990 
1991 
1992 static int
1993 vmxnet3_rq_create_all(struct vmxnet3_adapter *adapter)
1994 {
1995     int i, err = 0;
1996 
1997     adapter->rxdataring_enabled = VMXNET3_VERSION_GE_3(adapter);
1998 
1999     for (i = 0; i < adapter->num_rx_queues; i++) {
2000         err = vmxnet3_rq_create(&adapter->rx_queue[i], adapter);
2001         if (unlikely(err)) {
2002             dev_err(&adapter->netdev->dev,
2003                 "%s: failed to create rx queue%i\n",
2004                 adapter->netdev->name, i);
2005             goto err_out;
2006         }
2007     }
2008 
2009     if (!adapter->rxdataring_enabled)
2010         vmxnet3_rq_destroy_all_rxdataring(adapter);
2011 
2012     return err;
2013 err_out:
2014     vmxnet3_rq_destroy_all(adapter);
2015     return err;
2016 
2017 }
2018 
2019 /* Multiple queue aware polling function for tx and rx */
2020 
2021 static int
2022 vmxnet3_do_poll(struct vmxnet3_adapter *adapter, int budget)
2023 {
2024     int rcd_done = 0, i;
2025     if (unlikely(adapter->shared->ecr))
2026         vmxnet3_process_events(adapter);
2027     for (i = 0; i < adapter->num_tx_queues; i++)
2028         vmxnet3_tq_tx_complete(&adapter->tx_queue[i], adapter);
2029 
2030     for (i = 0; i < adapter->num_rx_queues; i++)
2031         rcd_done += vmxnet3_rq_rx_complete(&adapter->rx_queue[i],
2032                            adapter, budget);
2033     return rcd_done;
2034 }
2035 
2036 
2037 static int
2038 vmxnet3_poll(struct napi_struct *napi, int budget)
2039 {
2040     struct vmxnet3_rx_queue *rx_queue = container_of(napi,
2041                       struct vmxnet3_rx_queue, napi);
2042     int rxd_done;
2043 
2044     rxd_done = vmxnet3_do_poll(rx_queue->adapter, budget);
2045 
2046     if (rxd_done < budget) {
2047         napi_complete_done(napi, rxd_done);
2048         vmxnet3_enable_all_intrs(rx_queue->adapter);
2049     }
2050     return rxd_done;
2051 }
2052 
2053 /*
2054  * NAPI polling function for MSI-X mode with multiple Rx queues
2055  * Returns the # of the NAPI credit consumed (# of rx descriptors processed)
2056  */
2057 
2058 static int
2059 vmxnet3_poll_rx_only(struct napi_struct *napi, int budget)
2060 {
2061     struct vmxnet3_rx_queue *rq = container_of(napi,
2062                         struct vmxnet3_rx_queue, napi);
2063     struct vmxnet3_adapter *adapter = rq->adapter;
2064     int rxd_done;
2065 
2066     /* When sharing interrupt with corresponding tx queue, process
2067      * tx completions in that queue as well
2068      */
2069     if (adapter->share_intr == VMXNET3_INTR_BUDDYSHARE) {
2070         struct vmxnet3_tx_queue *tq =
2071                 &adapter->tx_queue[rq - adapter->rx_queue];
2072         vmxnet3_tq_tx_complete(tq, adapter);
2073     }
2074 
2075     rxd_done = vmxnet3_rq_rx_complete(rq, adapter, budget);
2076 
2077     if (rxd_done < budget) {
2078         napi_complete_done(napi, rxd_done);
2079         vmxnet3_enable_intr(adapter, rq->comp_ring.intr_idx);
2080     }
2081     return rxd_done;
2082 }
2083 
2084 
2085 #ifdef CONFIG_PCI_MSI
2086 
2087 /*
2088  * Handle completion interrupts on tx queues
2089  * Returns whether or not the intr is handled
2090  */
2091 
2092 static irqreturn_t
2093 vmxnet3_msix_tx(int irq, void *data)
2094 {
2095     struct vmxnet3_tx_queue *tq = data;
2096     struct vmxnet3_adapter *adapter = tq->adapter;
2097 
2098     if (adapter->intr.mask_mode == VMXNET3_IMM_ACTIVE)
2099         vmxnet3_disable_intr(adapter, tq->comp_ring.intr_idx);
2100 
2101     /* Handle the case where only one irq is allocate for all tx queues */
2102     if (adapter->share_intr == VMXNET3_INTR_TXSHARE) {
2103         int i;
2104         for (i = 0; i < adapter->num_tx_queues; i++) {
2105             struct vmxnet3_tx_queue *txq = &adapter->tx_queue[i];
2106             vmxnet3_tq_tx_complete(txq, adapter);
2107         }
2108     } else {
2109         vmxnet3_tq_tx_complete(tq, adapter);
2110     }
2111     vmxnet3_enable_intr(adapter, tq->comp_ring.intr_idx);
2112 
2113     return IRQ_HANDLED;
2114 }
2115 
2116 
2117 /*
2118  * Handle completion interrupts on rx queues. Returns whether or not the
2119  * intr is handled
2120  */
2121 
2122 static irqreturn_t
2123 vmxnet3_msix_rx(int irq, void *data)
2124 {
2125     struct vmxnet3_rx_queue *rq = data;
2126     struct vmxnet3_adapter *adapter = rq->adapter;
2127 
2128     /* disable intr if needed */
2129     if (adapter->intr.mask_mode == VMXNET3_IMM_ACTIVE)
2130         vmxnet3_disable_intr(adapter, rq->comp_ring.intr_idx);
2131     napi_schedule(&rq->napi);
2132 
2133     return IRQ_HANDLED;
2134 }
2135 
2136 /*
2137  *----------------------------------------------------------------------------
2138  *
2139  * vmxnet3_msix_event --
2140  *
2141  *    vmxnet3 msix event intr handler
2142  *
2143  * Result:
2144  *    whether or not the intr is handled
2145  *
2146  *----------------------------------------------------------------------------
2147  */
2148 
2149 static irqreturn_t
2150 vmxnet3_msix_event(int irq, void *data)
2151 {
2152     struct net_device *dev = data;
2153     struct vmxnet3_adapter *adapter = netdev_priv(dev);
2154 
2155     /* disable intr if needed */
2156     if (adapter->intr.mask_mode == VMXNET3_IMM_ACTIVE)
2157         vmxnet3_disable_intr(adapter, adapter->intr.event_intr_idx);
2158 
2159     if (adapter->shared->ecr)
2160         vmxnet3_process_events(adapter);
2161 
2162     vmxnet3_enable_intr(adapter, adapter->intr.event_intr_idx);
2163 
2164     return IRQ_HANDLED;
2165 }
2166 
2167 #endif /* CONFIG_PCI_MSI  */
2168 
2169 
2170 /* Interrupt handler for vmxnet3  */
2171 static irqreturn_t
2172 vmxnet3_intr(int irq, void *dev_id)
2173 {
2174     struct net_device *dev = dev_id;
2175     struct vmxnet3_adapter *adapter = netdev_priv(dev);
2176 
2177     if (adapter->intr.type == VMXNET3_IT_INTX) {
2178         u32 icr = VMXNET3_READ_BAR1_REG(adapter, VMXNET3_REG_ICR);
2179         if (unlikely(icr == 0))
2180             /* not ours */
2181             return IRQ_NONE;
2182     }
2183 
2184 
2185     /* disable intr if needed */
2186     if (adapter->intr.mask_mode == VMXNET3_IMM_ACTIVE)
2187         vmxnet3_disable_all_intrs(adapter);
2188 
2189     napi_schedule(&adapter->rx_queue[0].napi);
2190 
2191     return IRQ_HANDLED;
2192 }
2193 
2194 #ifdef CONFIG_NET_POLL_CONTROLLER
2195 
2196 /* netpoll callback. */
2197 static void
2198 vmxnet3_netpoll(struct net_device *netdev)
2199 {
2200     struct vmxnet3_adapter *adapter = netdev_priv(netdev);
2201 
2202     switch (adapter->intr.type) {
2203 #ifdef CONFIG_PCI_MSI
2204     case VMXNET3_IT_MSIX: {
2205         int i;
2206         for (i = 0; i < adapter->num_rx_queues; i++)
2207             vmxnet3_msix_rx(0, &adapter->rx_queue[i]);
2208         break;
2209     }
2210 #endif
2211     case VMXNET3_IT_MSI:
2212     default:
2213         vmxnet3_intr(0, adapter->netdev);
2214         break;
2215     }
2216 
2217 }
2218 #endif  /* CONFIG_NET_POLL_CONTROLLER */
2219 
2220 static int
2221 vmxnet3_request_irqs(struct vmxnet3_adapter *adapter)
2222 {
2223     struct vmxnet3_intr *intr = &adapter->intr;
2224     int err = 0, i;
2225     int vector = 0;
2226 
2227 #ifdef CONFIG_PCI_MSI
2228     if (adapter->intr.type == VMXNET3_IT_MSIX) {
2229         for (i = 0; i < adapter->num_tx_queues; i++) {
2230             if (adapter->share_intr != VMXNET3_INTR_BUDDYSHARE) {
2231                 sprintf(adapter->tx_queue[i].name, "%s-tx-%d",
2232                     adapter->netdev->name, vector);
2233                 err = request_irq(
2234                           intr->msix_entries[vector].vector,
2235                           vmxnet3_msix_tx, 0,
2236                           adapter->tx_queue[i].name,
2237                           &adapter->tx_queue[i]);
2238             } else {
2239                 sprintf(adapter->tx_queue[i].name, "%s-rxtx-%d",
2240                     adapter->netdev->name, vector);
2241             }
2242             if (err) {
2243                 dev_err(&adapter->netdev->dev,
2244                     "Failed to request irq for MSIX, %s, "
2245                     "error %d\n",
2246                     adapter->tx_queue[i].name, err);
2247                 return err;
2248             }
2249 
2250             /* Handle the case where only 1 MSIx was allocated for
2251              * all tx queues */
2252             if (adapter->share_intr == VMXNET3_INTR_TXSHARE) {
2253                 for (; i < adapter->num_tx_queues; i++)
2254                     adapter->tx_queue[i].comp_ring.intr_idx
2255                                 = vector;
2256                 vector++;
2257                 break;
2258             } else {
2259                 adapter->tx_queue[i].comp_ring.intr_idx
2260                                 = vector++;
2261             }
2262         }
2263         if (adapter->share_intr == VMXNET3_INTR_BUDDYSHARE)
2264             vector = 0;
2265 
2266         for (i = 0; i < adapter->num_rx_queues; i++) {
2267             if (adapter->share_intr != VMXNET3_INTR_BUDDYSHARE)
2268                 sprintf(adapter->rx_queue[i].name, "%s-rx-%d",
2269                     adapter->netdev->name, vector);
2270             else
2271                 sprintf(adapter->rx_queue[i].name, "%s-rxtx-%d",
2272                     adapter->netdev->name, vector);
2273             err = request_irq(intr->msix_entries[vector].vector,
2274                       vmxnet3_msix_rx, 0,
2275                       adapter->rx_queue[i].name,
2276                       &(adapter->rx_queue[i]));
2277             if (err) {
2278                 netdev_err(adapter->netdev,
2279                        "Failed to request irq for MSIX, "
2280                        "%s, error %d\n",
2281                        adapter->rx_queue[i].name, err);
2282                 return err;
2283             }
2284 
2285             adapter->rx_queue[i].comp_ring.intr_idx = vector++;
2286         }
2287 
2288         sprintf(intr->event_msi_vector_name, "%s-event-%d",
2289             adapter->netdev->name, vector);
2290         err = request_irq(intr->msix_entries[vector].vector,
2291                   vmxnet3_msix_event, 0,
2292                   intr->event_msi_vector_name, adapter->netdev);
2293         intr->event_intr_idx = vector;
2294 
2295     } else if (intr->type == VMXNET3_IT_MSI) {
2296         adapter->num_rx_queues = 1;
2297         err = request_irq(adapter->pdev->irq, vmxnet3_intr, 0,
2298                   adapter->netdev->name, adapter->netdev);
2299     } else {
2300 #endif
2301         adapter->num_rx_queues = 1;
2302         err = request_irq(adapter->pdev->irq, vmxnet3_intr,
2303                   IRQF_SHARED, adapter->netdev->name,
2304                   adapter->netdev);
2305 #ifdef CONFIG_PCI_MSI
2306     }
2307 #endif
2308     intr->num_intrs = vector + 1;
2309     if (err) {
2310         netdev_err(adapter->netdev,
2311                "Failed to request irq (intr type:%d), error %d\n",
2312                intr->type, err);
2313     } else {
2314         /* Number of rx queues will not change after this */
2315         for (i = 0; i < adapter->num_rx_queues; i++) {
2316             struct vmxnet3_rx_queue *rq = &adapter->rx_queue[i];
2317             rq->qid = i;
2318             rq->qid2 = i + adapter->num_rx_queues;
2319             rq->dataRingQid = i + 2 * adapter->num_rx_queues;
2320         }
2321 
2322         /* init our intr settings */
2323         for (i = 0; i < intr->num_intrs; i++)
2324             intr->mod_levels[i] = UPT1_IML_ADAPTIVE;
2325         if (adapter->intr.type != VMXNET3_IT_MSIX) {
2326             adapter->intr.event_intr_idx = 0;
2327             for (i = 0; i < adapter->num_tx_queues; i++)
2328                 adapter->tx_queue[i].comp_ring.intr_idx = 0;
2329             adapter->rx_queue[0].comp_ring.intr_idx = 0;
2330         }
2331 
2332         netdev_info(adapter->netdev,
2333                 "intr type %u, mode %u, %u vectors allocated\n",
2334                 intr->type, intr->mask_mode, intr->num_intrs);
2335     }
2336 
2337     return err;
2338 }
2339 
2340 
2341 static void
2342 vmxnet3_free_irqs(struct vmxnet3_adapter *adapter)
2343 {
2344     struct vmxnet3_intr *intr = &adapter->intr;
2345     BUG_ON(intr->type == VMXNET3_IT_AUTO || intr->num_intrs <= 0);
2346 
2347     switch (intr->type) {
2348 #ifdef CONFIG_PCI_MSI
2349     case VMXNET3_IT_MSIX:
2350     {
2351         int i, vector = 0;
2352 
2353         if (adapter->share_intr != VMXNET3_INTR_BUDDYSHARE) {
2354             for (i = 0; i < adapter->num_tx_queues; i++) {
2355                 free_irq(intr->msix_entries[vector++].vector,
2356                      &(adapter->tx_queue[i]));
2357                 if (adapter->share_intr == VMXNET3_INTR_TXSHARE)
2358                     break;
2359             }
2360         }
2361 
2362         for (i = 0; i < adapter->num_rx_queues; i++) {
2363             free_irq(intr->msix_entries[vector++].vector,
2364                  &(adapter->rx_queue[i]));
2365         }
2366 
2367         free_irq(intr->msix_entries[vector].vector,
2368              adapter->netdev);
2369         BUG_ON(vector >= intr->num_intrs);
2370         break;
2371     }
2372 #endif
2373     case VMXNET3_IT_MSI:
2374         free_irq(adapter->pdev->irq, adapter->netdev);
2375         break;
2376     case VMXNET3_IT_INTX:
2377         free_irq(adapter->pdev->irq, adapter->netdev);
2378         break;
2379     default:
2380         BUG();
2381     }
2382 }
2383 
2384 
2385 static void
2386 vmxnet3_restore_vlan(struct vmxnet3_adapter *adapter)
2387 {
2388     u32 *vfTable = adapter->shared->devRead.rxFilterConf.vfTable;
2389     u16 vid;
2390 
2391     /* allow untagged pkts */
2392     VMXNET3_SET_VFTABLE_ENTRY(vfTable, 0);
2393 
2394     for_each_set_bit(vid, adapter->active_vlans, VLAN_N_VID)
2395         VMXNET3_SET_VFTABLE_ENTRY(vfTable, vid);
2396 }
2397 
2398 
2399 static int
2400 vmxnet3_vlan_rx_add_vid(struct net_device *netdev, __be16 proto, u16 vid)
2401 {
2402     struct vmxnet3_adapter *adapter = netdev_priv(netdev);
2403 
2404     if (!(netdev->flags & IFF_PROMISC)) {
2405         u32 *vfTable = adapter->shared->devRead.rxFilterConf.vfTable;
2406         unsigned long flags;
2407 
2408         VMXNET3_SET_VFTABLE_ENTRY(vfTable, vid);
2409         spin_lock_irqsave(&adapter->cmd_lock, flags);
2410         VMXNET3_WRITE_BAR1_REG(adapter, VMXNET3_REG_CMD,
2411                        VMXNET3_CMD_UPDATE_VLAN_FILTERS);
2412         spin_unlock_irqrestore(&adapter->cmd_lock, flags);
2413     }
2414 
2415     set_bit(vid, adapter->active_vlans);
2416 
2417     return 0;
2418 }
2419 
2420 
2421 static int
2422 vmxnet3_vlan_rx_kill_vid(struct net_device *netdev, __be16 proto, u16 vid)
2423 {
2424     struct vmxnet3_adapter *adapter = netdev_priv(netdev);
2425 
2426     if (!(netdev->flags & IFF_PROMISC)) {
2427         u32 *vfTable = adapter->shared->devRead.rxFilterConf.vfTable;
2428         unsigned long flags;
2429 
2430         VMXNET3_CLEAR_VFTABLE_ENTRY(vfTable, vid);
2431         spin_lock_irqsave(&adapter->cmd_lock, flags);
2432         VMXNET3_WRITE_BAR1_REG(adapter, VMXNET3_REG_CMD,
2433                        VMXNET3_CMD_UPDATE_VLAN_FILTERS);
2434         spin_unlock_irqrestore(&adapter->cmd_lock, flags);
2435     }
2436 
2437     clear_bit(vid, adapter->active_vlans);
2438 
2439     return 0;
2440 }
2441 
2442 
2443 static u8 *
2444 vmxnet3_copy_mc(struct net_device *netdev)
2445 {
2446     u8 *buf = NULL;
2447     u32 sz = netdev_mc_count(netdev) * ETH_ALEN;
2448 
2449     /* struct Vmxnet3_RxFilterConf.mfTableLen is u16. */
2450     if (sz <= 0xffff) {
2451         /* We may be called with BH disabled */
2452         buf = kmalloc(sz, GFP_ATOMIC);
2453         if (buf) {
2454             struct netdev_hw_addr *ha;
2455             int i = 0;
2456 
2457             netdev_for_each_mc_addr(ha, netdev)
2458                 memcpy(buf + i++ * ETH_ALEN, ha->addr,
2459                        ETH_ALEN);
2460         }
2461     }
2462     return buf;
2463 }
2464 
2465 
2466 static void
2467 vmxnet3_set_mc(struct net_device *netdev)
2468 {
2469     struct vmxnet3_adapter *adapter = netdev_priv(netdev);
2470     unsigned long flags;
2471     struct Vmxnet3_RxFilterConf *rxConf =
2472                     &adapter->shared->devRead.rxFilterConf;
2473     u8 *new_table = NULL;
2474     dma_addr_t new_table_pa = 0;
2475     bool new_table_pa_valid = false;
2476     u32 new_mode = VMXNET3_RXM_UCAST;
2477 
2478     if (netdev->flags & IFF_PROMISC) {
2479         u32 *vfTable = adapter->shared->devRead.rxFilterConf.vfTable;
2480         memset(vfTable, 0, VMXNET3_VFT_SIZE * sizeof(*vfTable));
2481 
2482         new_mode |= VMXNET3_RXM_PROMISC;
2483     } else {
2484         vmxnet3_restore_vlan(adapter);
2485     }
2486 
2487     if (netdev->flags & IFF_BROADCAST)
2488         new_mode |= VMXNET3_RXM_BCAST;
2489 
2490     if (netdev->flags & IFF_ALLMULTI)
2491         new_mode |= VMXNET3_RXM_ALL_MULTI;
2492     else
2493         if (!netdev_mc_empty(netdev)) {
2494             new_table = vmxnet3_copy_mc(netdev);
2495             if (new_table) {
2496                 size_t sz = netdev_mc_count(netdev) * ETH_ALEN;
2497 
2498                 rxConf->mfTableLen = cpu_to_le16(sz);
2499                 new_table_pa = dma_map_single(
2500                             &adapter->pdev->dev,
2501                             new_table,
2502                             sz,
2503                             DMA_TO_DEVICE);
2504                 if (!dma_mapping_error(&adapter->pdev->dev,
2505                                new_table_pa)) {
2506                     new_mode |= VMXNET3_RXM_MCAST;
2507                     new_table_pa_valid = true;
2508                     rxConf->mfTablePA = cpu_to_le64(
2509                                 new_table_pa);
2510                 }
2511             }
2512             if (!new_table_pa_valid) {
2513                 netdev_info(netdev,
2514                         "failed to copy mcast list, setting ALL_MULTI\n");
2515                 new_mode |= VMXNET3_RXM_ALL_MULTI;
2516             }
2517         }
2518 
2519     if (!(new_mode & VMXNET3_RXM_MCAST)) {
2520         rxConf->mfTableLen = 0;
2521         rxConf->mfTablePA = 0;
2522     }
2523 
2524     spin_lock_irqsave(&adapter->cmd_lock, flags);
2525     if (new_mode != rxConf->rxMode) {
2526         rxConf->rxMode = cpu_to_le32(new_mode);
2527         VMXNET3_WRITE_BAR1_REG(adapter, VMXNET3_REG_CMD,
2528                        VMXNET3_CMD_UPDATE_RX_MODE);
2529         VMXNET3_WRITE_BAR1_REG(adapter, VMXNET3_REG_CMD,
2530                        VMXNET3_CMD_UPDATE_VLAN_FILTERS);
2531     }
2532 
2533     VMXNET3_WRITE_BAR1_REG(adapter, VMXNET3_REG_CMD,
2534                    VMXNET3_CMD_UPDATE_MAC_FILTERS);
2535     spin_unlock_irqrestore(&adapter->cmd_lock, flags);
2536 
2537     if (new_table_pa_valid)
2538         dma_unmap_single(&adapter->pdev->dev, new_table_pa,
2539                  rxConf->mfTableLen, DMA_TO_DEVICE);
2540     kfree(new_table);
2541 }
2542 
2543 void
2544 vmxnet3_rq_destroy_all(struct vmxnet3_adapter *adapter)
2545 {
2546     int i;
2547 
2548     for (i = 0; i < adapter->num_rx_queues; i++)
2549         vmxnet3_rq_destroy(&adapter->rx_queue[i], adapter);
2550 }
2551 
2552 
2553 /*
2554  *   Set up driver_shared based on settings in adapter.
2555  */
2556 
2557 static void
2558 vmxnet3_setup_driver_shared(struct vmxnet3_adapter *adapter)
2559 {
2560     struct Vmxnet3_DriverShared *shared = adapter->shared;
2561     struct Vmxnet3_DSDevRead *devRead = &shared->devRead;
2562     struct Vmxnet3_DSDevReadExt *devReadExt = &shared->devReadExt;
2563     struct Vmxnet3_TxQueueConf *tqc;
2564     struct Vmxnet3_RxQueueConf *rqc;
2565     int i;
2566 
2567     memset(shared, 0, sizeof(*shared));
2568 
2569     /* driver settings */
2570     shared->magic = cpu_to_le32(VMXNET3_REV1_MAGIC);
2571     devRead->misc.driverInfo.version = cpu_to_le32(
2572                         VMXNET3_DRIVER_VERSION_NUM);
2573     devRead->misc.driverInfo.gos.gosBits = (sizeof(void *) == 4 ?
2574                 VMXNET3_GOS_BITS_32 : VMXNET3_GOS_BITS_64);
2575     devRead->misc.driverInfo.gos.gosType = VMXNET3_GOS_TYPE_LINUX;
2576     *((u32 *)&devRead->misc.driverInfo.gos) = cpu_to_le32(
2577                 *((u32 *)&devRead->misc.driverInfo.gos));
2578     devRead->misc.driverInfo.vmxnet3RevSpt = cpu_to_le32(1);
2579     devRead->misc.driverInfo.uptVerSpt = cpu_to_le32(1);
2580 
2581     devRead->misc.ddPA = cpu_to_le64(adapter->adapter_pa);
2582     devRead->misc.ddLen = cpu_to_le32(sizeof(struct vmxnet3_adapter));
2583 
2584     /* set up feature flags */
2585     if (adapter->netdev->features & NETIF_F_RXCSUM)
2586         devRead->misc.uptFeatures |= UPT1_F_RXCSUM;
2587 
2588     if (adapter->netdev->features & NETIF_F_LRO) {
2589         devRead->misc.uptFeatures |= UPT1_F_LRO;
2590         devRead->misc.maxNumRxSG = cpu_to_le16(1 + MAX_SKB_FRAGS);
2591     }
2592     if (adapter->netdev->features & NETIF_F_HW_VLAN_CTAG_RX)
2593         devRead->misc.uptFeatures |= UPT1_F_RXVLAN;
2594 
2595     if (adapter->netdev->features & (NETIF_F_GSO_UDP_TUNNEL |
2596                      NETIF_F_GSO_UDP_TUNNEL_CSUM))
2597         devRead->misc.uptFeatures |= UPT1_F_RXINNEROFLD;
2598 
2599     devRead->misc.mtu = cpu_to_le32(adapter->netdev->mtu);
2600     devRead->misc.queueDescPA = cpu_to_le64(adapter->queue_desc_pa);
2601     devRead->misc.queueDescLen = cpu_to_le32(
2602         adapter->num_tx_queues * sizeof(struct Vmxnet3_TxQueueDesc) +
2603         adapter->num_rx_queues * sizeof(struct Vmxnet3_RxQueueDesc));
2604 
2605     /* tx queue settings */
2606     devRead->misc.numTxQueues =  adapter->num_tx_queues;
2607     for (i = 0; i < adapter->num_tx_queues; i++) {
2608         struct vmxnet3_tx_queue *tq = &adapter->tx_queue[i];
2609         BUG_ON(adapter->tx_queue[i].tx_ring.base == NULL);
2610         tqc = &adapter->tqd_start[i].conf;
2611         tqc->txRingBasePA   = cpu_to_le64(tq->tx_ring.basePA);
2612         tqc->dataRingBasePA = cpu_to_le64(tq->data_ring.basePA);
2613         tqc->compRingBasePA = cpu_to_le64(tq->comp_ring.basePA);
2614         tqc->ddPA           = cpu_to_le64(~0ULL);
2615         tqc->txRingSize     = cpu_to_le32(tq->tx_ring.size);
2616         tqc->dataRingSize   = cpu_to_le32(tq->data_ring.size);
2617         tqc->txDataRingDescSize = cpu_to_le32(tq->txdata_desc_size);
2618         tqc->compRingSize   = cpu_to_le32(tq->comp_ring.size);
2619         tqc->ddLen          = cpu_to_le32(0);
2620         tqc->intrIdx        = tq->comp_ring.intr_idx;
2621     }
2622 
2623     /* rx queue settings */
2624     devRead->misc.numRxQueues = adapter->num_rx_queues;
2625     for (i = 0; i < adapter->num_rx_queues; i++) {
2626         struct vmxnet3_rx_queue *rq = &adapter->rx_queue[i];
2627         rqc = &adapter->rqd_start[i].conf;
2628         rqc->rxRingBasePA[0] = cpu_to_le64(rq->rx_ring[0].basePA);
2629         rqc->rxRingBasePA[1] = cpu_to_le64(rq->rx_ring[1].basePA);
2630         rqc->compRingBasePA  = cpu_to_le64(rq->comp_ring.basePA);
2631         rqc->ddPA            = cpu_to_le64(~0ULL);
2632         rqc->rxRingSize[0]   = cpu_to_le32(rq->rx_ring[0].size);
2633         rqc->rxRingSize[1]   = cpu_to_le32(rq->rx_ring[1].size);
2634         rqc->compRingSize    = cpu_to_le32(rq->comp_ring.size);
2635         rqc->ddLen           = cpu_to_le32(0);
2636         rqc->intrIdx         = rq->comp_ring.intr_idx;
2637         if (VMXNET3_VERSION_GE_3(adapter)) {
2638             rqc->rxDataRingBasePA =
2639                 cpu_to_le64(rq->data_ring.basePA);
2640             rqc->rxDataRingDescSize =
2641                 cpu_to_le16(rq->data_ring.desc_size);
2642         }
2643     }
2644 
2645 #ifdef VMXNET3_RSS
2646     memset(adapter->rss_conf, 0, sizeof(*adapter->rss_conf));
2647 
2648     if (adapter->rss) {
2649         struct UPT1_RSSConf *rssConf = adapter->rss_conf;
2650 
2651         devRead->misc.uptFeatures |= UPT1_F_RSS;
2652         devRead->misc.numRxQueues = adapter->num_rx_queues;
2653         rssConf->hashType = UPT1_RSS_HASH_TYPE_TCP_IPV4 |
2654                     UPT1_RSS_HASH_TYPE_IPV4 |
2655                     UPT1_RSS_HASH_TYPE_TCP_IPV6 |
2656                     UPT1_RSS_HASH_TYPE_IPV6;
2657         rssConf->hashFunc = UPT1_RSS_HASH_FUNC_TOEPLITZ;
2658         rssConf->hashKeySize = UPT1_RSS_MAX_KEY_SIZE;
2659         rssConf->indTableSize = VMXNET3_RSS_IND_TABLE_SIZE;
2660         netdev_rss_key_fill(rssConf->hashKey, sizeof(rssConf->hashKey));
2661 
2662         for (i = 0; i < rssConf->indTableSize; i++)
2663             rssConf->indTable[i] = ethtool_rxfh_indir_default(
2664                 i, adapter->num_rx_queues);
2665 
2666         devRead->rssConfDesc.confVer = 1;
2667         devRead->rssConfDesc.confLen = cpu_to_le32(sizeof(*rssConf));
2668         devRead->rssConfDesc.confPA =
2669             cpu_to_le64(adapter->rss_conf_pa);
2670     }
2671 
2672 #endif /* VMXNET3_RSS */
2673 
2674     /* intr settings */
2675     if (!VMXNET3_VERSION_GE_6(adapter) ||
2676         !adapter->queuesExtEnabled) {
2677         devRead->intrConf.autoMask = adapter->intr.mask_mode ==
2678                          VMXNET3_IMM_AUTO;
2679         devRead->intrConf.numIntrs = adapter->intr.num_intrs;
2680         for (i = 0; i < adapter->intr.num_intrs; i++)
2681             devRead->intrConf.modLevels[i] = adapter->intr.mod_levels[i];
2682 
2683         devRead->intrConf.eventIntrIdx = adapter->intr.event_intr_idx;
2684         devRead->intrConf.intrCtrl |= cpu_to_le32(VMXNET3_IC_DISABLE_ALL);
2685     } else {
2686         devReadExt->intrConfExt.autoMask = adapter->intr.mask_mode ==
2687                            VMXNET3_IMM_AUTO;
2688         devReadExt->intrConfExt.numIntrs = adapter->intr.num_intrs;
2689         for (i = 0; i < adapter->intr.num_intrs; i++)
2690             devReadExt->intrConfExt.modLevels[i] = adapter->intr.mod_levels[i];
2691 
2692         devReadExt->intrConfExt.eventIntrIdx = adapter->intr.event_intr_idx;
2693         devReadExt->intrConfExt.intrCtrl |= cpu_to_le32(VMXNET3_IC_DISABLE_ALL);
2694     }
2695 
2696     /* rx filter settings */
2697     devRead->rxFilterConf.rxMode = 0;
2698     vmxnet3_restore_vlan(adapter);
2699     vmxnet3_write_mac_addr(adapter, adapter->netdev->dev_addr);
2700 
2701     /* the rest are already zeroed */
2702 }
2703 
2704 static void
2705 vmxnet3_init_bufsize(struct vmxnet3_adapter *adapter)
2706 {
2707     struct Vmxnet3_DriverShared *shared = adapter->shared;
2708     union Vmxnet3_CmdInfo *cmdInfo = &shared->cu.cmdInfo;
2709     unsigned long flags;
2710 
2711     if (!VMXNET3_VERSION_GE_7(adapter))
2712         return;
2713 
2714     cmdInfo->ringBufSize = adapter->ringBufSize;
2715     spin_lock_irqsave(&adapter->cmd_lock, flags);
2716     VMXNET3_WRITE_BAR1_REG(adapter, VMXNET3_REG_CMD,
2717                    VMXNET3_CMD_SET_RING_BUFFER_SIZE);
2718     spin_unlock_irqrestore(&adapter->cmd_lock, flags);
2719 }
2720 
2721 static void
2722 vmxnet3_init_coalesce(struct vmxnet3_adapter *adapter)
2723 {
2724     struct Vmxnet3_DriverShared *shared = adapter->shared;
2725     union Vmxnet3_CmdInfo *cmdInfo = &shared->cu.cmdInfo;
2726     unsigned long flags;
2727 
2728     if (!VMXNET3_VERSION_GE_3(adapter))
2729         return;
2730 
2731     spin_lock_irqsave(&adapter->cmd_lock, flags);
2732     cmdInfo->varConf.confVer = 1;
2733     cmdInfo->varConf.confLen =
2734         cpu_to_le32(sizeof(*adapter->coal_conf));
2735     cmdInfo->varConf.confPA  = cpu_to_le64(adapter->coal_conf_pa);
2736 
2737     if (adapter->default_coal_mode) {
2738         VMXNET3_WRITE_BAR1_REG(adapter, VMXNET3_REG_CMD,
2739                        VMXNET3_CMD_GET_COALESCE);
2740     } else {
2741         VMXNET3_WRITE_BAR1_REG(adapter, VMXNET3_REG_CMD,
2742                        VMXNET3_CMD_SET_COALESCE);
2743     }
2744 
2745     spin_unlock_irqrestore(&adapter->cmd_lock, flags);
2746 }
2747 
2748 static void
2749 vmxnet3_init_rssfields(struct vmxnet3_adapter *adapter)
2750 {
2751     struct Vmxnet3_DriverShared *shared = adapter->shared;
2752     union Vmxnet3_CmdInfo *cmdInfo = &shared->cu.cmdInfo;
2753     unsigned long flags;
2754 
2755     if (!VMXNET3_VERSION_GE_4(adapter))
2756         return;
2757 
2758     spin_lock_irqsave(&adapter->cmd_lock, flags);
2759 
2760     if (adapter->default_rss_fields) {
2761         VMXNET3_WRITE_BAR1_REG(adapter, VMXNET3_REG_CMD,
2762                        VMXNET3_CMD_GET_RSS_FIELDS);
2763         adapter->rss_fields =
2764             VMXNET3_READ_BAR1_REG(adapter, VMXNET3_REG_CMD);
2765     } else {
2766         if (VMXNET3_VERSION_GE_7(adapter)) {
2767             if ((adapter->rss_fields & VMXNET3_RSS_FIELDS_UDPIP4 ||
2768                  adapter->rss_fields & VMXNET3_RSS_FIELDS_UDPIP6) &&
2769                 vmxnet3_check_ptcapability(adapter->ptcap_supported[0],
2770                                VMXNET3_CAP_UDP_RSS)) {
2771                 adapter->dev_caps[0] |= 1UL << VMXNET3_CAP_UDP_RSS;
2772             } else {
2773                 adapter->dev_caps[0] &= ~(1UL << VMXNET3_CAP_UDP_RSS);
2774             }
2775 
2776             if ((adapter->rss_fields & VMXNET3_RSS_FIELDS_ESPIP4) &&
2777                 vmxnet3_check_ptcapability(adapter->ptcap_supported[0],
2778                                VMXNET3_CAP_ESP_RSS_IPV4)) {
2779                 adapter->dev_caps[0] |= 1UL << VMXNET3_CAP_ESP_RSS_IPV4;
2780             } else {
2781                 adapter->dev_caps[0] &= ~(1UL << VMXNET3_CAP_ESP_RSS_IPV4);
2782             }
2783 
2784             if ((adapter->rss_fields & VMXNET3_RSS_FIELDS_ESPIP6) &&
2785                 vmxnet3_check_ptcapability(adapter->ptcap_supported[0],
2786                                VMXNET3_CAP_ESP_RSS_IPV6)) {
2787                 adapter->dev_caps[0] |= 1UL << VMXNET3_CAP_ESP_RSS_IPV6;
2788             } else {
2789                 adapter->dev_caps[0] &= ~(1UL << VMXNET3_CAP_ESP_RSS_IPV6);
2790             }
2791 
2792             VMXNET3_WRITE_BAR1_REG(adapter, VMXNET3_REG_DCR, adapter->dev_caps[0]);
2793             VMXNET3_WRITE_BAR1_REG(adapter, VMXNET3_REG_CMD, VMXNET3_CMD_GET_DCR0_REG);
2794             adapter->dev_caps[0] = VMXNET3_READ_BAR1_REG(adapter, VMXNET3_REG_CMD);
2795         }
2796         cmdInfo->setRssFields = adapter->rss_fields;
2797         VMXNET3_WRITE_BAR1_REG(adapter, VMXNET3_REG_CMD,
2798                        VMXNET3_CMD_SET_RSS_FIELDS);
2799         /* Not all requested RSS may get applied, so get and
2800          * cache what was actually applied.
2801          */
2802         VMXNET3_WRITE_BAR1_REG(adapter, VMXNET3_REG_CMD,
2803                        VMXNET3_CMD_GET_RSS_FIELDS);
2804         adapter->rss_fields =
2805             VMXNET3_READ_BAR1_REG(adapter, VMXNET3_REG_CMD);
2806     }
2807 
2808     spin_unlock_irqrestore(&adapter->cmd_lock, flags);
2809 }
2810 
2811 int
2812 vmxnet3_activate_dev(struct vmxnet3_adapter *adapter)
2813 {
2814     int err, i;
2815     u32 ret;
2816     unsigned long flags;
2817 
2818     netdev_dbg(adapter->netdev, "%s: skb_buf_size %d, rx_buf_per_pkt %d,"
2819         " ring sizes %u %u %u\n", adapter->netdev->name,
2820         adapter->skb_buf_size, adapter->rx_buf_per_pkt,
2821         adapter->tx_queue[0].tx_ring.size,
2822         adapter->rx_queue[0].rx_ring[0].size,
2823         adapter->rx_queue[0].rx_ring[1].size);
2824 
2825     vmxnet3_tq_init_all(adapter);
2826     err = vmxnet3_rq_init_all(adapter);
2827     if (err) {
2828         netdev_err(adapter->netdev,
2829                "Failed to init rx queue error %d\n", err);
2830         goto rq_err;
2831     }
2832 
2833     err = vmxnet3_request_irqs(adapter);
2834     if (err) {
2835         netdev_err(adapter->netdev,
2836                "Failed to setup irq for error %d\n", err);
2837         goto irq_err;
2838     }
2839 
2840     vmxnet3_setup_driver_shared(adapter);
2841 
2842     VMXNET3_WRITE_BAR1_REG(adapter, VMXNET3_REG_DSAL, VMXNET3_GET_ADDR_LO(
2843                    adapter->shared_pa));
2844     VMXNET3_WRITE_BAR1_REG(adapter, VMXNET3_REG_DSAH, VMXNET3_GET_ADDR_HI(
2845                    adapter->shared_pa));
2846     spin_lock_irqsave(&adapter->cmd_lock, flags);
2847     VMXNET3_WRITE_BAR1_REG(adapter, VMXNET3_REG_CMD,
2848                    VMXNET3_CMD_ACTIVATE_DEV);
2849     ret = VMXNET3_READ_BAR1_REG(adapter, VMXNET3_REG_CMD);
2850     spin_unlock_irqrestore(&adapter->cmd_lock, flags);
2851 
2852     if (ret != 0) {
2853         netdev_err(adapter->netdev,
2854                "Failed to activate dev: error %u\n", ret);
2855         err = -EINVAL;
2856         goto activate_err;
2857     }
2858 
2859     vmxnet3_init_bufsize(adapter);
2860     vmxnet3_init_coalesce(adapter);
2861     vmxnet3_init_rssfields(adapter);
2862 
2863     for (i = 0; i < adapter->num_rx_queues; i++) {
2864         VMXNET3_WRITE_BAR0_REG(adapter,
2865                 adapter->rx_prod_offset + i * VMXNET3_REG_ALIGN,
2866                 adapter->rx_queue[i].rx_ring[0].next2fill);
2867         VMXNET3_WRITE_BAR0_REG(adapter, (adapter->rx_prod2_offset +
2868                 (i * VMXNET3_REG_ALIGN)),
2869                 adapter->rx_queue[i].rx_ring[1].next2fill);
2870     }
2871 
2872     /* Apply the rx filter settins last. */
2873     vmxnet3_set_mc(adapter->netdev);
2874 
2875     /*
2876      * Check link state when first activating device. It will start the
2877      * tx queue if the link is up.
2878      */
2879     vmxnet3_check_link(adapter, true);
2880     netif_tx_wake_all_queues(adapter->netdev);
2881     for (i = 0; i < adapter->num_rx_queues; i++)
2882         napi_enable(&adapter->rx_queue[i].napi);
2883     vmxnet3_enable_all_intrs(adapter);
2884     clear_bit(VMXNET3_STATE_BIT_QUIESCED, &adapter->state);
2885     return 0;
2886 
2887 activate_err:
2888     VMXNET3_WRITE_BAR1_REG(adapter, VMXNET3_REG_DSAL, 0);
2889     VMXNET3_WRITE_BAR1_REG(adapter, VMXNET3_REG_DSAH, 0);
2890     vmxnet3_free_irqs(adapter);
2891 irq_err:
2892 rq_err:
2893     /* free up buffers we allocated */
2894     vmxnet3_rq_cleanup_all(adapter);
2895     return err;
2896 }
2897 
2898 
2899 void
2900 vmxnet3_reset_dev(struct vmxnet3_adapter *adapter)
2901 {
2902     unsigned long flags;
2903     spin_lock_irqsave(&adapter->cmd_lock, flags);
2904     VMXNET3_WRITE_BAR1_REG(adapter, VMXNET3_REG_CMD, VMXNET3_CMD_RESET_DEV);
2905     spin_unlock_irqrestore(&adapter->cmd_lock, flags);
2906 }
2907 
2908 
2909 int
2910 vmxnet3_quiesce_dev(struct vmxnet3_adapter *adapter)
2911 {
2912     int i;
2913     unsigned long flags;
2914     if (test_and_set_bit(VMXNET3_STATE_BIT_QUIESCED, &adapter->state))
2915         return 0;
2916 
2917 
2918     spin_lock_irqsave(&adapter->cmd_lock, flags);
2919     VMXNET3_WRITE_BAR1_REG(adapter, VMXNET3_REG_CMD,
2920                    VMXNET3_CMD_QUIESCE_DEV);
2921     spin_unlock_irqrestore(&adapter->cmd_lock, flags);
2922     vmxnet3_disable_all_intrs(adapter);
2923 
2924     for (i = 0; i < adapter->num_rx_queues; i++)
2925         napi_disable(&adapter->rx_queue[i].napi);
2926     netif_tx_disable(adapter->netdev);
2927     adapter->link_speed = 0;
2928     netif_carrier_off(adapter->netdev);
2929 
2930     vmxnet3_tq_cleanup_all(adapter);
2931     vmxnet3_rq_cleanup_all(adapter);
2932     vmxnet3_free_irqs(adapter);
2933     return 0;
2934 }
2935 
2936 
2937 static void
2938 vmxnet3_write_mac_addr(struct vmxnet3_adapter *adapter, const u8 *mac)
2939 {
2940     u32 tmp;
2941 
2942     tmp = *(u32 *)mac;
2943     VMXNET3_WRITE_BAR1_REG(adapter, VMXNET3_REG_MACL, tmp);
2944 
2945     tmp = (mac[5] << 8) | mac[4];
2946     VMXNET3_WRITE_BAR1_REG(adapter, VMXNET3_REG_MACH, tmp);
2947 }
2948 
2949 
2950 static int
2951 vmxnet3_set_mac_addr(struct net_device *netdev, void *p)
2952 {
2953     struct sockaddr *addr = p;
2954     struct vmxnet3_adapter *adapter = netdev_priv(netdev);
2955 
2956     dev_addr_set(netdev, addr->sa_data);
2957     vmxnet3_write_mac_addr(adapter, addr->sa_data);
2958 
2959     return 0;
2960 }
2961 
2962 
2963 /* ==================== initialization and cleanup routines ============ */
2964 
2965 static int
2966 vmxnet3_alloc_pci_resources(struct vmxnet3_adapter *adapter)
2967 {
2968     int err;
2969     unsigned long mmio_start, mmio_len;
2970     struct pci_dev *pdev = adapter->pdev;
2971 
2972     err = pci_enable_device(pdev);
2973     if (err) {
2974         dev_err(&pdev->dev, "Failed to enable adapter: error %d\n", err);
2975         return err;
2976     }
2977 
2978     err = pci_request_selected_regions(pdev, (1 << 2) - 1,
2979                        vmxnet3_driver_name);
2980     if (err) {
2981         dev_err(&pdev->dev,
2982             "Failed to request region for adapter: error %d\n", err);
2983         goto err_enable_device;
2984     }
2985 
2986     pci_set_master(pdev);
2987 
2988     mmio_start = pci_resource_start(pdev, 0);
2989     mmio_len = pci_resource_len(pdev, 0);
2990     adapter->hw_addr0 = ioremap(mmio_start, mmio_len);
2991     if (!adapter->hw_addr0) {
2992         dev_err(&pdev->dev, "Failed to map bar0\n");
2993         err = -EIO;
2994         goto err_ioremap;
2995     }
2996 
2997     mmio_start = pci_resource_start(pdev, 1);
2998     mmio_len = pci_resource_len(pdev, 1);
2999     adapter->hw_addr1 = ioremap(mmio_start, mmio_len);
3000     if (!adapter->hw_addr1) {
3001         dev_err(&pdev->dev, "Failed to map bar1\n");
3002         err = -EIO;
3003         goto err_bar1;
3004     }
3005     return 0;
3006 
3007 err_bar1:
3008     iounmap(adapter->hw_addr0);
3009 err_ioremap:
3010     pci_release_selected_regions(pdev, (1 << 2) - 1);
3011 err_enable_device:
3012     pci_disable_device(pdev);
3013     return err;
3014 }
3015 
3016 
3017 static void
3018 vmxnet3_free_pci_resources(struct vmxnet3_adapter *adapter)
3019 {
3020     BUG_ON(!adapter->pdev);
3021 
3022     iounmap(adapter->hw_addr0);
3023     iounmap(adapter->hw_addr1);
3024     pci_release_selected_regions(adapter->pdev, (1 << 2) - 1);
3025     pci_disable_device(adapter->pdev);
3026 }
3027 
3028 
3029 static void
3030 vmxnet3_adjust_rx_ring_size(struct vmxnet3_adapter *adapter)
3031 {
3032     size_t sz, i, ring0_size, ring1_size, comp_size;
3033     /* With version7 ring1 will have only T0 buffers */
3034     if (!VMXNET3_VERSION_GE_7(adapter)) {
3035         if (adapter->netdev->mtu <= VMXNET3_MAX_SKB_BUF_SIZE -
3036                         VMXNET3_MAX_ETH_HDR_SIZE) {
3037             adapter->skb_buf_size = adapter->netdev->mtu +
3038                         VMXNET3_MAX_ETH_HDR_SIZE;
3039             if (adapter->skb_buf_size < VMXNET3_MIN_T0_BUF_SIZE)
3040                 adapter->skb_buf_size = VMXNET3_MIN_T0_BUF_SIZE;
3041 
3042             adapter->rx_buf_per_pkt = 1;
3043         } else {
3044             adapter->skb_buf_size = VMXNET3_MAX_SKB_BUF_SIZE;
3045             sz = adapter->netdev->mtu - VMXNET3_MAX_SKB_BUF_SIZE +
3046                             VMXNET3_MAX_ETH_HDR_SIZE;
3047             adapter->rx_buf_per_pkt = 1 + (sz + PAGE_SIZE - 1) / PAGE_SIZE;
3048         }
3049     } else {
3050         adapter->skb_buf_size = min((int)adapter->netdev->mtu + VMXNET3_MAX_ETH_HDR_SIZE,
3051                         VMXNET3_MAX_SKB_BUF_SIZE);
3052         adapter->rx_buf_per_pkt = 1;
3053         adapter->ringBufSize.ring1BufSizeType0 = cpu_to_le16(adapter->skb_buf_size);
3054         adapter->ringBufSize.ring1BufSizeType1 = 0;
3055         adapter->ringBufSize.ring2BufSizeType1 = cpu_to_le16(PAGE_SIZE);
3056     }
3057 
3058     /*
3059      * for simplicity, force the ring0 size to be a multiple of
3060      * rx_buf_per_pkt * VMXNET3_RING_SIZE_ALIGN
3061      */
3062     sz = adapter->rx_buf_per_pkt * VMXNET3_RING_SIZE_ALIGN;
3063     ring0_size = adapter->rx_queue[0].rx_ring[0].size;
3064     ring0_size = (ring0_size + sz - 1) / sz * sz;
3065     ring0_size = min_t(u32, ring0_size, VMXNET3_RX_RING_MAX_SIZE /
3066                sz * sz);
3067     ring1_size = adapter->rx_queue[0].rx_ring[1].size;
3068     ring1_size = (ring1_size + sz - 1) / sz * sz;
3069     ring1_size = min_t(u32, ring1_size, VMXNET3_RX_RING2_MAX_SIZE /
3070                sz * sz);
3071     /* For v7 and later, keep ring size power of 2 for UPT */
3072     if (VMXNET3_VERSION_GE_7(adapter)) {
3073         ring0_size = rounddown_pow_of_two(ring0_size);
3074         ring1_size = rounddown_pow_of_two(ring1_size);
3075     }
3076     comp_size = ring0_size + ring1_size;
3077 
3078     for (i = 0; i < adapter->num_rx_queues; i++) {
3079         struct vmxnet3_rx_queue *rq = &adapter->rx_queue[i];
3080 
3081         rq->rx_ring[0].size = ring0_size;
3082         rq->rx_ring[1].size = ring1_size;
3083         rq->comp_ring.size = comp_size;
3084     }
3085 }
3086 
3087 
3088 int
3089 vmxnet3_create_queues(struct vmxnet3_adapter *adapter, u32 tx_ring_size,
3090               u32 rx_ring_size, u32 rx_ring2_size,
3091               u16 txdata_desc_size, u16 rxdata_desc_size)
3092 {
3093     int err = 0, i;
3094 
3095     for (i = 0; i < adapter->num_tx_queues; i++) {
3096         struct vmxnet3_tx_queue *tq = &adapter->tx_queue[i];
3097         tq->tx_ring.size   = tx_ring_size;
3098         tq->data_ring.size = tx_ring_size;
3099         tq->comp_ring.size = tx_ring_size;
3100         tq->txdata_desc_size = txdata_desc_size;
3101         tq->shared = &adapter->tqd_start[i].ctrl;
3102         tq->stopped = true;
3103         tq->adapter = adapter;
3104         tq->qid = i;
3105         err = vmxnet3_tq_create(tq, adapter);
3106         /*
3107          * Too late to change num_tx_queues. We cannot do away with
3108          * lesser number of queues than what we asked for
3109          */
3110         if (err)
3111             goto queue_err;
3112     }
3113 
3114     adapter->rx_queue[0].rx_ring[0].size = rx_ring_size;
3115     adapter->rx_queue[0].rx_ring[1].size = rx_ring2_size;
3116     vmxnet3_adjust_rx_ring_size(adapter);
3117 
3118     adapter->rxdataring_enabled = VMXNET3_VERSION_GE_3(adapter);
3119     for (i = 0; i < adapter->num_rx_queues; i++) {
3120         struct vmxnet3_rx_queue *rq = &adapter->rx_queue[i];
3121         /* qid and qid2 for rx queues will be assigned later when num
3122          * of rx queues is finalized after allocating intrs */
3123         rq->shared = &adapter->rqd_start[i].ctrl;
3124         rq->adapter = adapter;
3125         rq->data_ring.desc_size = rxdata_desc_size;
3126         err = vmxnet3_rq_create(rq, adapter);
3127         if (err) {
3128             if (i == 0) {
3129                 netdev_err(adapter->netdev,
3130                        "Could not allocate any rx queues. "
3131                        "Aborting.\n");
3132                 goto queue_err;
3133             } else {
3134                 netdev_info(adapter->netdev,
3135                         "Number of rx queues changed "
3136                         "to : %d.\n", i);
3137                 adapter->num_rx_queues = i;
3138                 err = 0;
3139                 break;
3140             }
3141         }
3142     }
3143 
3144     if (!adapter->rxdataring_enabled)
3145         vmxnet3_rq_destroy_all_rxdataring(adapter);
3146 
3147     return err;
3148 queue_err:
3149     vmxnet3_tq_destroy_all(adapter);
3150     return err;
3151 }
3152 
3153 static int
3154 vmxnet3_open(struct net_device *netdev)
3155 {
3156     struct vmxnet3_adapter *adapter;
3157     int err, i;
3158 
3159     adapter = netdev_priv(netdev);
3160 
3161     for (i = 0; i < adapter->num_tx_queues; i++)
3162         spin_lock_init(&adapter->tx_queue[i].tx_lock);
3163 
3164     if (VMXNET3_VERSION_GE_3(adapter)) {
3165         unsigned long flags;
3166         u16 txdata_desc_size;
3167 
3168         spin_lock_irqsave(&adapter->cmd_lock, flags);
3169         VMXNET3_WRITE_BAR1_REG(adapter, VMXNET3_REG_CMD,
3170                        VMXNET3_CMD_GET_TXDATA_DESC_SIZE);
3171         txdata_desc_size = VMXNET3_READ_BAR1_REG(adapter,
3172                              VMXNET3_REG_CMD);
3173         spin_unlock_irqrestore(&adapter->cmd_lock, flags);
3174 
3175         if ((txdata_desc_size < VMXNET3_TXDATA_DESC_MIN_SIZE) ||
3176             (txdata_desc_size > VMXNET3_TXDATA_DESC_MAX_SIZE) ||
3177             (txdata_desc_size & VMXNET3_TXDATA_DESC_SIZE_MASK)) {
3178             adapter->txdata_desc_size =
3179                 sizeof(struct Vmxnet3_TxDataDesc);
3180         } else {
3181             adapter->txdata_desc_size = txdata_desc_size;
3182         }
3183     } else {
3184         adapter->txdata_desc_size = sizeof(struct Vmxnet3_TxDataDesc);
3185     }
3186 
3187     err = vmxnet3_create_queues(adapter,
3188                     adapter->tx_ring_size,
3189                     adapter->rx_ring_size,
3190                     adapter->rx_ring2_size,
3191                     adapter->txdata_desc_size,
3192                     adapter->rxdata_desc_size);
3193     if (err)
3194         goto queue_err;
3195 
3196     err = vmxnet3_activate_dev(adapter);
3197     if (err)
3198         goto activate_err;
3199 
3200     return 0;
3201 
3202 activate_err:
3203     vmxnet3_rq_destroy_all(adapter);
3204     vmxnet3_tq_destroy_all(adapter);
3205 queue_err:
3206     return err;
3207 }
3208 
3209 
3210 static int
3211 vmxnet3_close(struct net_device *netdev)
3212 {
3213     struct vmxnet3_adapter *adapter = netdev_priv(netdev);
3214 
3215     /*
3216      * Reset_work may be in the middle of resetting the device, wait for its
3217      * completion.
3218      */
3219     while (test_and_set_bit(VMXNET3_STATE_BIT_RESETTING, &adapter->state))
3220         usleep_range(1000, 2000);
3221 
3222     vmxnet3_quiesce_dev(adapter);
3223 
3224     vmxnet3_rq_destroy_all(adapter);
3225     vmxnet3_tq_destroy_all(adapter);
3226 
3227     clear_bit(VMXNET3_STATE_BIT_RESETTING, &adapter->state);
3228 
3229 
3230     return 0;
3231 }
3232 
3233 
3234 void
3235 vmxnet3_force_close(struct vmxnet3_adapter *adapter)
3236 {
3237     int i;
3238 
3239     /*
3240      * we must clear VMXNET3_STATE_BIT_RESETTING, otherwise
3241      * vmxnet3_close() will deadlock.
3242      */
3243     BUG_ON(test_bit(VMXNET3_STATE_BIT_RESETTING, &adapter->state));
3244 
3245     /* we need to enable NAPI, otherwise dev_close will deadlock */
3246     for (i = 0; i < adapter->num_rx_queues; i++)
3247         napi_enable(&adapter->rx_queue[i].napi);
3248     /*
3249      * Need to clear the quiesce bit to ensure that vmxnet3_close
3250      * can quiesce the device properly
3251      */
3252     clear_bit(VMXNET3_STATE_BIT_QUIESCED, &adapter->state);
3253     dev_close(adapter->netdev);
3254 }
3255 
3256 
3257 static int
3258 vmxnet3_change_mtu(struct net_device *netdev, int new_mtu)
3259 {
3260     struct vmxnet3_adapter *adapter = netdev_priv(netdev);
3261     int err = 0;
3262 
3263     netdev->mtu = new_mtu;
3264 
3265     /*
3266      * Reset_work may be in the middle of resetting the device, wait for its
3267      * completion.
3268      */
3269     while (test_and_set_bit(VMXNET3_STATE_BIT_RESETTING, &adapter->state))
3270         usleep_range(1000, 2000);
3271 
3272     if (netif_running(netdev)) {
3273         vmxnet3_quiesce_dev(adapter);
3274         vmxnet3_reset_dev(adapter);
3275 
3276         /* we need to re-create the rx queue based on the new mtu */
3277         vmxnet3_rq_destroy_all(adapter);
3278         vmxnet3_adjust_rx_ring_size(adapter);
3279         err = vmxnet3_rq_create_all(adapter);
3280         if (err) {
3281             netdev_err(netdev,
3282                    "failed to re-create rx queues, "
3283                    " error %d. Closing it.\n", err);
3284             goto out;
3285         }
3286 
3287         err = vmxnet3_activate_dev(adapter);
3288         if (err) {
3289             netdev_err(netdev,
3290                    "failed to re-activate, error %d. "
3291                    "Closing it\n", err);
3292             goto out;
3293         }
3294     }
3295 
3296 out:
3297     clear_bit(VMXNET3_STATE_BIT_RESETTING, &adapter->state);
3298     if (err)
3299         vmxnet3_force_close(adapter);
3300 
3301     return err;
3302 }
3303 
3304 
3305 static void
3306 vmxnet3_declare_features(struct vmxnet3_adapter *adapter)
3307 {
3308     struct net_device *netdev = adapter->netdev;
3309 
3310     netdev->hw_features = NETIF_F_SG | NETIF_F_RXCSUM |
3311         NETIF_F_HW_CSUM | NETIF_F_HW_VLAN_CTAG_TX |
3312         NETIF_F_HW_VLAN_CTAG_RX | NETIF_F_TSO | NETIF_F_TSO6 |
3313         NETIF_F_LRO | NETIF_F_HIGHDMA;
3314 
3315     if (VMXNET3_VERSION_GE_4(adapter)) {
3316         netdev->hw_features |= NETIF_F_GSO_UDP_TUNNEL |
3317                 NETIF_F_GSO_UDP_TUNNEL_CSUM;
3318 
3319         netdev->hw_enc_features = NETIF_F_SG | NETIF_F_RXCSUM |
3320             NETIF_F_HW_CSUM | NETIF_F_HW_VLAN_CTAG_TX |
3321             NETIF_F_HW_VLAN_CTAG_RX | NETIF_F_TSO | NETIF_F_TSO6 |
3322             NETIF_F_LRO | NETIF_F_GSO_UDP_TUNNEL |
3323             NETIF_F_GSO_UDP_TUNNEL_CSUM;
3324     }
3325 
3326     if (VMXNET3_VERSION_GE_7(adapter)) {
3327         unsigned long flags;
3328 
3329         if (vmxnet3_check_ptcapability(adapter->ptcap_supported[0],
3330                            VMXNET3_CAP_GENEVE_CHECKSUM_OFFLOAD)) {
3331             adapter->dev_caps[0] |= 1UL << VMXNET3_CAP_GENEVE_CHECKSUM_OFFLOAD;
3332         }
3333         if (vmxnet3_check_ptcapability(adapter->ptcap_supported[0],
3334                            VMXNET3_CAP_VXLAN_CHECKSUM_OFFLOAD)) {
3335             adapter->dev_caps[0] |= 1UL << VMXNET3_CAP_VXLAN_CHECKSUM_OFFLOAD;
3336         }
3337         if (vmxnet3_check_ptcapability(adapter->ptcap_supported[0],
3338                            VMXNET3_CAP_GENEVE_TSO)) {
3339             adapter->dev_caps[0] |= 1UL << VMXNET3_CAP_GENEVE_TSO;
3340         }
3341         if (vmxnet3_check_ptcapability(adapter->ptcap_supported[0],
3342                            VMXNET3_CAP_VXLAN_TSO)) {
3343             adapter->dev_caps[0] |= 1UL << VMXNET3_CAP_VXLAN_TSO;
3344         }
3345         if (vmxnet3_check_ptcapability(adapter->ptcap_supported[0],
3346                            VMXNET3_CAP_GENEVE_OUTER_CHECKSUM_OFFLOAD)) {
3347             adapter->dev_caps[0] |= 1UL << VMXNET3_CAP_GENEVE_OUTER_CHECKSUM_OFFLOAD;
3348         }
3349         if (vmxnet3_check_ptcapability(adapter->ptcap_supported[0],
3350                            VMXNET3_CAP_VXLAN_OUTER_CHECKSUM_OFFLOAD)) {
3351             adapter->dev_caps[0] |= 1UL << VMXNET3_CAP_VXLAN_OUTER_CHECKSUM_OFFLOAD;
3352         }
3353 
3354         VMXNET3_WRITE_BAR1_REG(adapter, VMXNET3_REG_DCR, adapter->dev_caps[0]);
3355         spin_lock_irqsave(&adapter->cmd_lock, flags);
3356         VMXNET3_WRITE_BAR1_REG(adapter, VMXNET3_REG_CMD, VMXNET3_CMD_GET_DCR0_REG);
3357         adapter->dev_caps[0] = VMXNET3_READ_BAR1_REG(adapter, VMXNET3_REG_CMD);
3358         spin_unlock_irqrestore(&adapter->cmd_lock, flags);
3359 
3360         if (!(adapter->dev_caps[0] & (1UL << VMXNET3_CAP_GENEVE_CHECKSUM_OFFLOAD)) &&
3361             !(adapter->dev_caps[0] & (1UL << VMXNET3_CAP_VXLAN_CHECKSUM_OFFLOAD)) &&
3362             !(adapter->dev_caps[0] & (1UL << VMXNET3_CAP_GENEVE_TSO)) &&
3363             !(adapter->dev_caps[0] & (1UL << VMXNET3_CAP_VXLAN_TSO))) {
3364             netdev->hw_enc_features &= ~NETIF_F_GSO_UDP_TUNNEL;
3365             netdev->hw_features &= ~NETIF_F_GSO_UDP_TUNNEL;
3366         }
3367         if (!(adapter->dev_caps[0] & (1UL << VMXNET3_CAP_GENEVE_OUTER_CHECKSUM_OFFLOAD)) &&
3368             !(adapter->dev_caps[0] & (1UL << VMXNET3_CAP_VXLAN_OUTER_CHECKSUM_OFFLOAD))) {
3369             netdev->hw_enc_features &= ~NETIF_F_GSO_UDP_TUNNEL_CSUM;
3370             netdev->hw_features &= ~NETIF_F_GSO_UDP_TUNNEL_CSUM;
3371         }
3372     }
3373 
3374     netdev->vlan_features = netdev->hw_features &
3375                 ~(NETIF_F_HW_VLAN_CTAG_TX |
3376                   NETIF_F_HW_VLAN_CTAG_RX);
3377     netdev->features = netdev->hw_features | NETIF_F_HW_VLAN_CTAG_FILTER;
3378 }
3379 
3380 
3381 static void
3382 vmxnet3_read_mac_addr(struct vmxnet3_adapter *adapter, u8 *mac)
3383 {
3384     u32 tmp;
3385 
3386     tmp = VMXNET3_READ_BAR1_REG(adapter, VMXNET3_REG_MACL);
3387     *(u32 *)mac = tmp;
3388 
3389     tmp = VMXNET3_READ_BAR1_REG(adapter, VMXNET3_REG_MACH);
3390     mac[4] = tmp & 0xff;
3391     mac[5] = (tmp >> 8) & 0xff;
3392 }
3393 
3394 #ifdef CONFIG_PCI_MSI
3395 
3396 /*
3397  * Enable MSIx vectors.
3398  * Returns :
3399  *  VMXNET3_LINUX_MIN_MSIX_VECT when only minimum number of vectors required
3400  *   were enabled.
3401  *  number of vectors which were enabled otherwise (this number is greater
3402  *   than VMXNET3_LINUX_MIN_MSIX_VECT)
3403  */
3404 
3405 static int
3406 vmxnet3_acquire_msix_vectors(struct vmxnet3_adapter *adapter, int nvec)
3407 {
3408     int ret = pci_enable_msix_range(adapter->pdev,
3409                     adapter->intr.msix_entries, nvec, nvec);
3410 
3411     if (ret == -ENOSPC && nvec > VMXNET3_LINUX_MIN_MSIX_VECT) {
3412         dev_err(&adapter->netdev->dev,
3413             "Failed to enable %d MSI-X, trying %d\n",
3414             nvec, VMXNET3_LINUX_MIN_MSIX_VECT);
3415 
3416         ret = pci_enable_msix_range(adapter->pdev,
3417                         adapter->intr.msix_entries,
3418                         VMXNET3_LINUX_MIN_MSIX_VECT,
3419                         VMXNET3_LINUX_MIN_MSIX_VECT);
3420     }
3421 
3422     if (ret < 0) {
3423         dev_err(&adapter->netdev->dev,
3424             "Failed to enable MSI-X, error: %d\n", ret);
3425     }
3426 
3427     return ret;
3428 }
3429 
3430 
3431 #endif /* CONFIG_PCI_MSI */
3432 
3433 static void
3434 vmxnet3_alloc_intr_resources(struct vmxnet3_adapter *adapter)
3435 {
3436     u32 cfg;
3437     unsigned long flags;
3438 
3439     /* intr settings */
3440     spin_lock_irqsave(&adapter->cmd_lock, flags);
3441     VMXNET3_WRITE_BAR1_REG(adapter, VMXNET3_REG_CMD,
3442                    VMXNET3_CMD_GET_CONF_INTR);
3443     cfg = VMXNET3_READ_BAR1_REG(adapter, VMXNET3_REG_CMD);
3444     spin_unlock_irqrestore(&adapter->cmd_lock, flags);
3445     adapter->intr.type = cfg & 0x3;
3446     adapter->intr.mask_mode = (cfg >> 2) & 0x3;
3447 
3448     if (adapter->intr.type == VMXNET3_IT_AUTO) {
3449         adapter->intr.type = VMXNET3_IT_MSIX;
3450     }
3451 
3452 #ifdef CONFIG_PCI_MSI
3453     if (adapter->intr.type == VMXNET3_IT_MSIX) {
3454         int i, nvec, nvec_allocated;
3455 
3456         nvec  = adapter->share_intr == VMXNET3_INTR_TXSHARE ?
3457             1 : adapter->num_tx_queues;
3458         nvec += adapter->share_intr == VMXNET3_INTR_BUDDYSHARE ?
3459             0 : adapter->num_rx_queues;
3460         nvec += 1;  /* for link event */
3461         nvec = nvec > VMXNET3_LINUX_MIN_MSIX_VECT ?
3462                nvec : VMXNET3_LINUX_MIN_MSIX_VECT;
3463 
3464         for (i = 0; i < nvec; i++)
3465             adapter->intr.msix_entries[i].entry = i;
3466 
3467         nvec_allocated = vmxnet3_acquire_msix_vectors(adapter, nvec);
3468         if (nvec_allocated < 0)
3469             goto msix_err;
3470 
3471         /* If we cannot allocate one MSIx vector per queue
3472          * then limit the number of rx queues to 1
3473          */
3474         if (nvec_allocated == VMXNET3_LINUX_MIN_MSIX_VECT &&
3475             nvec != VMXNET3_LINUX_MIN_MSIX_VECT) {
3476             if (adapter->share_intr != VMXNET3_INTR_BUDDYSHARE
3477                 || adapter->num_rx_queues != 1) {
3478                 adapter->share_intr = VMXNET3_INTR_TXSHARE;
3479                 netdev_err(adapter->netdev,
3480                        "Number of rx queues : 1\n");
3481                 adapter->num_rx_queues = 1;
3482             }
3483         }
3484 
3485         adapter->intr.num_intrs = nvec_allocated;
3486         return;
3487 
3488 msix_err:
3489         /* If we cannot allocate MSIx vectors use only one rx queue */
3490         dev_info(&adapter->pdev->dev,
3491              "Failed to enable MSI-X, error %d. "
3492              "Limiting #rx queues to 1, try MSI.\n", nvec_allocated);
3493 
3494         adapter->intr.type = VMXNET3_IT_MSI;
3495     }
3496 
3497     if (adapter->intr.type == VMXNET3_IT_MSI) {
3498         if (!pci_enable_msi(adapter->pdev)) {
3499             adapter->num_rx_queues = 1;
3500             adapter->intr.num_intrs = 1;
3501             return;
3502         }
3503     }
3504 #endif /* CONFIG_PCI_MSI */
3505 
3506     adapter->num_rx_queues = 1;
3507     dev_info(&adapter->netdev->dev,
3508          "Using INTx interrupt, #Rx queues: 1.\n");
3509     adapter->intr.type = VMXNET3_IT_INTX;
3510 
3511     /* INT-X related setting */
3512     adapter->intr.num_intrs = 1;
3513 }
3514 
3515 
3516 static void
3517 vmxnet3_free_intr_resources(struct vmxnet3_adapter *adapter)
3518 {
3519     if (adapter->intr.type == VMXNET3_IT_MSIX)
3520         pci_disable_msix(adapter->pdev);
3521     else if (adapter->intr.type == VMXNET3_IT_MSI)
3522         pci_disable_msi(adapter->pdev);
3523     else
3524         BUG_ON(adapter->intr.type != VMXNET3_IT_INTX);
3525 }
3526 
3527 
3528 static void
3529 vmxnet3_tx_timeout(struct net_device *netdev, unsigned int txqueue)
3530 {
3531     struct vmxnet3_adapter *adapter = netdev_priv(netdev);
3532     adapter->tx_timeout_count++;
3533 
3534     netdev_err(adapter->netdev, "tx hang\n");
3535     schedule_work(&adapter->work);
3536 }
3537 
3538 
3539 static void
3540 vmxnet3_reset_work(struct work_struct *data)
3541 {
3542     struct vmxnet3_adapter *adapter;
3543 
3544     adapter = container_of(data, struct vmxnet3_adapter, work);
3545 
3546     /* if another thread is resetting the device, no need to proceed */
3547     if (test_and_set_bit(VMXNET3_STATE_BIT_RESETTING, &adapter->state))
3548         return;
3549 
3550     /* if the device is closed, we must leave it alone */
3551     rtnl_lock();
3552     if (netif_running(adapter->netdev)) {
3553         netdev_notice(adapter->netdev, "resetting\n");
3554         vmxnet3_quiesce_dev(adapter);
3555         vmxnet3_reset_dev(adapter);
3556         vmxnet3_activate_dev(adapter);
3557     } else {
3558         netdev_info(adapter->netdev, "already closed\n");
3559     }
3560     rtnl_unlock();
3561 
3562     netif_wake_queue(adapter->netdev);
3563     clear_bit(VMXNET3_STATE_BIT_RESETTING, &adapter->state);
3564 }
3565 
3566 
3567 static int
3568 vmxnet3_probe_device(struct pci_dev *pdev,
3569              const struct pci_device_id *id)
3570 {
3571     static const struct net_device_ops vmxnet3_netdev_ops = {
3572         .ndo_open = vmxnet3_open,
3573         .ndo_stop = vmxnet3_close,
3574         .ndo_start_xmit = vmxnet3_xmit_frame,
3575         .ndo_set_mac_address = vmxnet3_set_mac_addr,
3576         .ndo_change_mtu = vmxnet3_change_mtu,
3577         .ndo_fix_features = vmxnet3_fix_features,
3578         .ndo_set_features = vmxnet3_set_features,
3579         .ndo_features_check = vmxnet3_features_check,
3580         .ndo_get_stats64 = vmxnet3_get_stats64,
3581         .ndo_tx_timeout = vmxnet3_tx_timeout,
3582         .ndo_set_rx_mode = vmxnet3_set_mc,
3583         .ndo_vlan_rx_add_vid = vmxnet3_vlan_rx_add_vid,
3584         .ndo_vlan_rx_kill_vid = vmxnet3_vlan_rx_kill_vid,
3585 #ifdef CONFIG_NET_POLL_CONTROLLER
3586         .ndo_poll_controller = vmxnet3_netpoll,
3587 #endif
3588     };
3589     int err;
3590     u32 ver;
3591     struct net_device *netdev;
3592     struct vmxnet3_adapter *adapter;
3593     u8 mac[ETH_ALEN];
3594     int size;
3595     int num_tx_queues;
3596     int num_rx_queues;
3597     int queues;
3598     unsigned long flags;
3599 
3600     if (!pci_msi_enabled())
3601         enable_mq = 0;
3602 
3603 #ifdef VMXNET3_RSS
3604     if (enable_mq)
3605         num_rx_queues = min(VMXNET3_DEVICE_MAX_RX_QUEUES,
3606                     (int)num_online_cpus());
3607     else
3608 #endif
3609         num_rx_queues = 1;
3610 
3611     if (enable_mq)
3612         num_tx_queues = min(VMXNET3_DEVICE_MAX_TX_QUEUES,
3613                     (int)num_online_cpus());
3614     else
3615         num_tx_queues = 1;
3616 
3617     netdev = alloc_etherdev_mq(sizeof(struct vmxnet3_adapter),
3618                    max(num_tx_queues, num_rx_queues));
3619     if (!netdev)
3620         return -ENOMEM;
3621 
3622     pci_set_drvdata(pdev, netdev);
3623     adapter = netdev_priv(netdev);
3624     adapter->netdev = netdev;
3625     adapter->pdev = pdev;
3626 
3627     adapter->tx_ring_size = VMXNET3_DEF_TX_RING_SIZE;
3628     adapter->rx_ring_size = VMXNET3_DEF_RX_RING_SIZE;
3629     adapter->rx_ring2_size = VMXNET3_DEF_RX_RING2_SIZE;
3630 
3631     err = dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(64));
3632     if (err) {
3633         dev_err(&pdev->dev, "dma_set_mask failed\n");
3634         goto err_set_mask;
3635     }
3636 
3637     spin_lock_init(&adapter->cmd_lock);
3638     adapter->adapter_pa = dma_map_single(&adapter->pdev->dev, adapter,
3639                          sizeof(struct vmxnet3_adapter),
3640                          DMA_TO_DEVICE);
3641     if (dma_mapping_error(&adapter->pdev->dev, adapter->adapter_pa)) {
3642         dev_err(&pdev->dev, "Failed to map dma\n");
3643         err = -EFAULT;
3644         goto err_set_mask;
3645     }
3646     adapter->shared = dma_alloc_coherent(
3647                 &adapter->pdev->dev,
3648                 sizeof(struct Vmxnet3_DriverShared),
3649                 &adapter->shared_pa, GFP_KERNEL);
3650     if (!adapter->shared) {
3651         dev_err(&pdev->dev, "Failed to allocate memory\n");
3652         err = -ENOMEM;
3653         goto err_alloc_shared;
3654     }
3655 
3656     err = vmxnet3_alloc_pci_resources(adapter);
3657     if (err < 0)
3658         goto err_alloc_pci;
3659 
3660     ver = VMXNET3_READ_BAR1_REG(adapter, VMXNET3_REG_VRRS);
3661     if (ver & (1 << VMXNET3_REV_7)) {
3662         VMXNET3_WRITE_BAR1_REG(adapter,
3663                        VMXNET3_REG_VRRS,
3664                        1 << VMXNET3_REV_7);
3665         adapter->version = VMXNET3_REV_7 + 1;
3666     } else if (ver & (1 << VMXNET3_REV_6)) {
3667         VMXNET3_WRITE_BAR1_REG(adapter,
3668                        VMXNET3_REG_VRRS,
3669                        1 << VMXNET3_REV_6);
3670         adapter->version = VMXNET3_REV_6 + 1;
3671     } else if (ver & (1 << VMXNET3_REV_5)) {
3672         VMXNET3_WRITE_BAR1_REG(adapter,
3673                        VMXNET3_REG_VRRS,
3674                        1 << VMXNET3_REV_5);
3675         adapter->version = VMXNET3_REV_5 + 1;
3676     } else if (ver & (1 << VMXNET3_REV_4)) {
3677         VMXNET3_WRITE_BAR1_REG(adapter,
3678                        VMXNET3_REG_VRRS,
3679                        1 << VMXNET3_REV_4);
3680         adapter->version = VMXNET3_REV_4 + 1;
3681     } else if (ver & (1 << VMXNET3_REV_3)) {
3682         VMXNET3_WRITE_BAR1_REG(adapter,
3683                        VMXNET3_REG_VRRS,
3684                        1 << VMXNET3_REV_3);
3685         adapter->version = VMXNET3_REV_3 + 1;
3686     } else if (ver & (1 << VMXNET3_REV_2)) {
3687         VMXNET3_WRITE_BAR1_REG(adapter,
3688                        VMXNET3_REG_VRRS,
3689                        1 << VMXNET3_REV_2);
3690         adapter->version = VMXNET3_REV_2 + 1;
3691     } else if (ver & (1 << VMXNET3_REV_1)) {
3692         VMXNET3_WRITE_BAR1_REG(adapter,
3693                        VMXNET3_REG_VRRS,
3694                        1 << VMXNET3_REV_1);
3695         adapter->version = VMXNET3_REV_1 + 1;
3696     } else {
3697         dev_err(&pdev->dev,
3698             "Incompatible h/w version (0x%x) for adapter\n", ver);
3699         err = -EBUSY;
3700         goto err_ver;
3701     }
3702     dev_dbg(&pdev->dev, "Using device version %d\n", adapter->version);
3703 
3704     ver = VMXNET3_READ_BAR1_REG(adapter, VMXNET3_REG_UVRS);
3705     if (ver & 1) {
3706         VMXNET3_WRITE_BAR1_REG(adapter, VMXNET3_REG_UVRS, 1);
3707     } else {
3708         dev_err(&pdev->dev,
3709             "Incompatible upt version (0x%x) for adapter\n", ver);
3710         err = -EBUSY;
3711         goto err_ver;
3712     }
3713 
3714     if (VMXNET3_VERSION_GE_7(adapter)) {
3715         adapter->devcap_supported[0] = VMXNET3_READ_BAR1_REG(adapter, VMXNET3_REG_DCR);
3716         adapter->ptcap_supported[0] = VMXNET3_READ_BAR1_REG(adapter, VMXNET3_REG_PTCR);
3717         if (adapter->devcap_supported[0] & (1UL << VMXNET3_CAP_LARGE_BAR)) {
3718             adapter->dev_caps[0] = adapter->devcap_supported[0] &
3719                             (1UL << VMXNET3_CAP_LARGE_BAR);
3720         }
3721         if (!(adapter->ptcap_supported[0] & (1UL << VMXNET3_DCR_ERROR)) &&
3722             adapter->ptcap_supported[0] & (1UL << VMXNET3_CAP_OOORX_COMP) &&
3723             adapter->devcap_supported[0] & (1UL << VMXNET3_CAP_OOORX_COMP)) {
3724             adapter->dev_caps[0] |= adapter->devcap_supported[0] &
3725                         (1UL << VMXNET3_CAP_OOORX_COMP);
3726         }
3727         if (adapter->dev_caps[0])
3728             VMXNET3_WRITE_BAR1_REG(adapter, VMXNET3_REG_DCR, adapter->dev_caps[0]);
3729 
3730         spin_lock_irqsave(&adapter->cmd_lock, flags);
3731         VMXNET3_WRITE_BAR1_REG(adapter, VMXNET3_REG_CMD, VMXNET3_CMD_GET_DCR0_REG);
3732         adapter->dev_caps[0] = VMXNET3_READ_BAR1_REG(adapter, VMXNET3_REG_CMD);
3733         spin_unlock_irqrestore(&adapter->cmd_lock, flags);
3734     }
3735 
3736     if (VMXNET3_VERSION_GE_7(adapter) &&
3737         adapter->dev_caps[0] & (1UL << VMXNET3_CAP_LARGE_BAR)) {
3738         adapter->tx_prod_offset = VMXNET3_REG_LB_TXPROD;
3739         adapter->rx_prod_offset = VMXNET3_REG_LB_RXPROD;
3740         adapter->rx_prod2_offset = VMXNET3_REG_LB_RXPROD2;
3741     } else {
3742         adapter->tx_prod_offset = VMXNET3_REG_TXPROD;
3743         adapter->rx_prod_offset = VMXNET3_REG_RXPROD;
3744         adapter->rx_prod2_offset = VMXNET3_REG_RXPROD2;
3745     }
3746 
3747     if (VMXNET3_VERSION_GE_6(adapter)) {
3748         spin_lock_irqsave(&adapter->cmd_lock, flags);
3749         VMXNET3_WRITE_BAR1_REG(adapter, VMXNET3_REG_CMD,
3750                        VMXNET3_CMD_GET_MAX_QUEUES_CONF);
3751         queues = VMXNET3_READ_BAR1_REG(adapter, VMXNET3_REG_CMD);
3752         spin_unlock_irqrestore(&adapter->cmd_lock, flags);
3753         if (queues > 0) {
3754             adapter->num_rx_queues = min(num_rx_queues, ((queues >> 8) & 0xff));
3755             adapter->num_tx_queues = min(num_tx_queues, (queues & 0xff));
3756         } else {
3757             adapter->num_rx_queues = min(num_rx_queues,
3758                              VMXNET3_DEVICE_DEFAULT_RX_QUEUES);
3759             adapter->num_tx_queues = min(num_tx_queues,
3760                              VMXNET3_DEVICE_DEFAULT_TX_QUEUES);
3761         }
3762         if (adapter->num_rx_queues > VMXNET3_MAX_RX_QUEUES ||
3763             adapter->num_tx_queues > VMXNET3_MAX_TX_QUEUES) {
3764             adapter->queuesExtEnabled = true;
3765         } else {
3766             adapter->queuesExtEnabled = false;
3767         }
3768     } else {
3769         adapter->queuesExtEnabled = false;
3770         num_rx_queues = rounddown_pow_of_two(num_rx_queues);
3771         num_tx_queues = rounddown_pow_of_two(num_tx_queues);
3772         adapter->num_rx_queues = min(num_rx_queues,
3773                          VMXNET3_DEVICE_DEFAULT_RX_QUEUES);
3774         adapter->num_tx_queues = min(num_tx_queues,
3775                          VMXNET3_DEVICE_DEFAULT_TX_QUEUES);
3776     }
3777     dev_info(&pdev->dev,
3778          "# of Tx queues : %d, # of Rx queues : %d\n",
3779          adapter->num_tx_queues, adapter->num_rx_queues);
3780 
3781     adapter->rx_buf_per_pkt = 1;
3782 
3783     size = sizeof(struct Vmxnet3_TxQueueDesc) * adapter->num_tx_queues;
3784     size += sizeof(struct Vmxnet3_RxQueueDesc) * adapter->num_rx_queues;
3785     adapter->tqd_start = dma_alloc_coherent(&adapter->pdev->dev, size,
3786                         &adapter->queue_desc_pa,
3787                         GFP_KERNEL);
3788 
3789     if (!adapter->tqd_start) {
3790         dev_err(&pdev->dev, "Failed to allocate memory\n");
3791         err = -ENOMEM;
3792         goto err_ver;
3793     }
3794     adapter->rqd_start = (struct Vmxnet3_RxQueueDesc *)(adapter->tqd_start +
3795                                 adapter->num_tx_queues);
3796 
3797     adapter->pm_conf = dma_alloc_coherent(&adapter->pdev->dev,
3798                           sizeof(struct Vmxnet3_PMConf),
3799                           &adapter->pm_conf_pa,
3800                           GFP_KERNEL);
3801     if (adapter->pm_conf == NULL) {
3802         err = -ENOMEM;
3803         goto err_alloc_pm;
3804     }
3805 
3806 #ifdef VMXNET3_RSS
3807 
3808     adapter->rss_conf = dma_alloc_coherent(&adapter->pdev->dev,
3809                            sizeof(struct UPT1_RSSConf),
3810                            &adapter->rss_conf_pa,
3811                            GFP_KERNEL);
3812     if (adapter->rss_conf == NULL) {
3813         err = -ENOMEM;
3814         goto err_alloc_rss;
3815     }
3816 #endif /* VMXNET3_RSS */
3817 
3818     if (VMXNET3_VERSION_GE_3(adapter)) {
3819         adapter->coal_conf =
3820             dma_alloc_coherent(&adapter->pdev->dev,
3821                        sizeof(struct Vmxnet3_CoalesceScheme)
3822                        ,
3823                        &adapter->coal_conf_pa,
3824                        GFP_KERNEL);
3825         if (!adapter->coal_conf) {
3826             err = -ENOMEM;
3827             goto err_coal_conf;
3828         }
3829         adapter->coal_conf->coalMode = VMXNET3_COALESCE_DISABLED;
3830         adapter->default_coal_mode = true;
3831     }
3832 
3833     if (VMXNET3_VERSION_GE_4(adapter)) {
3834         adapter->default_rss_fields = true;
3835         adapter->rss_fields = VMXNET3_RSS_FIELDS_DEFAULT;
3836     }
3837 
3838     SET_NETDEV_DEV(netdev, &pdev->dev);
3839     vmxnet3_declare_features(adapter);
3840 
3841     adapter->rxdata_desc_size = VMXNET3_VERSION_GE_3(adapter) ?
3842         VMXNET3_DEF_RXDATA_DESC_SIZE : 0;
3843 
3844     if (adapter->num_tx_queues == adapter->num_rx_queues)
3845         adapter->share_intr = VMXNET3_INTR_BUDDYSHARE;
3846     else
3847         adapter->share_intr = VMXNET3_INTR_DONTSHARE;
3848 
3849     vmxnet3_alloc_intr_resources(adapter);
3850 
3851 #ifdef VMXNET3_RSS
3852     if (adapter->num_rx_queues > 1 &&
3853         adapter->intr.type == VMXNET3_IT_MSIX) {
3854         adapter->rss = true;
3855         netdev->hw_features |= NETIF_F_RXHASH;
3856         netdev->features |= NETIF_F_RXHASH;
3857         dev_dbg(&pdev->dev, "RSS is enabled.\n");
3858     } else {
3859         adapter->rss = false;
3860     }
3861 #endif
3862 
3863     vmxnet3_read_mac_addr(adapter, mac);
3864     dev_addr_set(netdev, mac);
3865 
3866     netdev->netdev_ops = &vmxnet3_netdev_ops;
3867     vmxnet3_set_ethtool_ops(netdev);
3868     netdev->watchdog_timeo = 5 * HZ;
3869 
3870     /* MTU range: 60 - 9190 */
3871     netdev->min_mtu = VMXNET3_MIN_MTU;
3872     if (VMXNET3_VERSION_GE_6(adapter))
3873         netdev->max_mtu = VMXNET3_V6_MAX_MTU;
3874     else
3875         netdev->max_mtu = VMXNET3_MAX_MTU;
3876 
3877     INIT_WORK(&adapter->work, vmxnet3_reset_work);
3878     set_bit(VMXNET3_STATE_BIT_QUIESCED, &adapter->state);
3879 
3880     if (adapter->intr.type == VMXNET3_IT_MSIX) {
3881         int i;
3882         for (i = 0; i < adapter->num_rx_queues; i++) {
3883             netif_napi_add(adapter->netdev,
3884                        &adapter->rx_queue[i].napi,
3885                        vmxnet3_poll_rx_only, 64);
3886         }
3887     } else {
3888         netif_napi_add(adapter->netdev, &adapter->rx_queue[0].napi,
3889                    vmxnet3_poll, 64);
3890     }
3891 
3892     netif_set_real_num_tx_queues(adapter->netdev, adapter->num_tx_queues);
3893     netif_set_real_num_rx_queues(adapter->netdev, adapter->num_rx_queues);
3894 
3895     netif_carrier_off(netdev);
3896     err = register_netdev(netdev);
3897 
3898     if (err) {
3899         dev_err(&pdev->dev, "Failed to register adapter\n");
3900         goto err_register;
3901     }
3902 
3903     vmxnet3_check_link(adapter, false);
3904     return 0;
3905 
3906 err_register:
3907     if (VMXNET3_VERSION_GE_3(adapter)) {
3908         dma_free_coherent(&adapter->pdev->dev,
3909                   sizeof(struct Vmxnet3_CoalesceScheme),
3910                   adapter->coal_conf, adapter->coal_conf_pa);
3911     }
3912     vmxnet3_free_intr_resources(adapter);
3913 err_coal_conf:
3914 #ifdef VMXNET3_RSS
3915     dma_free_coherent(&adapter->pdev->dev, sizeof(struct UPT1_RSSConf),
3916               adapter->rss_conf, adapter->rss_conf_pa);
3917 err_alloc_rss:
3918 #endif
3919     dma_free_coherent(&adapter->pdev->dev, sizeof(struct Vmxnet3_PMConf),
3920               adapter->pm_conf, adapter->pm_conf_pa);
3921 err_alloc_pm:
3922     dma_free_coherent(&adapter->pdev->dev, size, adapter->tqd_start,
3923               adapter->queue_desc_pa);
3924 err_ver:
3925     vmxnet3_free_pci_resources(adapter);
3926 err_alloc_pci:
3927     dma_free_coherent(&adapter->pdev->dev,
3928               sizeof(struct Vmxnet3_DriverShared),
3929               adapter->shared, adapter->shared_pa);
3930 err_alloc_shared:
3931     dma_unmap_single(&adapter->pdev->dev, adapter->adapter_pa,
3932              sizeof(struct vmxnet3_adapter), DMA_TO_DEVICE);
3933 err_set_mask:
3934     free_netdev(netdev);
3935     return err;
3936 }
3937 
3938 
3939 static void
3940 vmxnet3_remove_device(struct pci_dev *pdev)
3941 {
3942     struct net_device *netdev = pci_get_drvdata(pdev);
3943     struct vmxnet3_adapter *adapter = netdev_priv(netdev);
3944     int size = 0;
3945     int num_rx_queues, rx_queues;
3946     unsigned long flags;
3947 
3948 #ifdef VMXNET3_RSS
3949     if (enable_mq)
3950         num_rx_queues = min(VMXNET3_DEVICE_MAX_RX_QUEUES,
3951                     (int)num_online_cpus());
3952     else
3953 #endif
3954         num_rx_queues = 1;
3955     if (!VMXNET3_VERSION_GE_6(adapter)) {
3956         num_rx_queues = rounddown_pow_of_two(num_rx_queues);
3957     }
3958     if (VMXNET3_VERSION_GE_6(adapter)) {
3959         spin_lock_irqsave(&adapter->cmd_lock, flags);
3960         VMXNET3_WRITE_BAR1_REG(adapter, VMXNET3_REG_CMD,
3961                        VMXNET3_CMD_GET_MAX_QUEUES_CONF);
3962         rx_queues = VMXNET3_READ_BAR1_REG(adapter, VMXNET3_REG_CMD);
3963         spin_unlock_irqrestore(&adapter->cmd_lock, flags);
3964         if (rx_queues > 0)
3965             rx_queues = (rx_queues >> 8) & 0xff;
3966         else
3967             rx_queues = min(num_rx_queues, VMXNET3_DEVICE_DEFAULT_RX_QUEUES);
3968         num_rx_queues = min(num_rx_queues, rx_queues);
3969     } else {
3970         num_rx_queues = min(num_rx_queues,
3971                     VMXNET3_DEVICE_DEFAULT_RX_QUEUES);
3972     }
3973 
3974     cancel_work_sync(&adapter->work);
3975 
3976     unregister_netdev(netdev);
3977 
3978     vmxnet3_free_intr_resources(adapter);
3979     vmxnet3_free_pci_resources(adapter);
3980     if (VMXNET3_VERSION_GE_3(adapter)) {
3981         dma_free_coherent(&adapter->pdev->dev,
3982                   sizeof(struct Vmxnet3_CoalesceScheme),
3983                   adapter->coal_conf, adapter->coal_conf_pa);
3984     }
3985 #ifdef VMXNET3_RSS
3986     dma_free_coherent(&adapter->pdev->dev, sizeof(struct UPT1_RSSConf),
3987               adapter->rss_conf, adapter->rss_conf_pa);
3988 #endif
3989     dma_free_coherent(&adapter->pdev->dev, sizeof(struct Vmxnet3_PMConf),
3990               adapter->pm_conf, adapter->pm_conf_pa);
3991 
3992     size = sizeof(struct Vmxnet3_TxQueueDesc) * adapter->num_tx_queues;
3993     size += sizeof(struct Vmxnet3_RxQueueDesc) * num_rx_queues;
3994     dma_free_coherent(&adapter->pdev->dev, size, adapter->tqd_start,
3995               adapter->queue_desc_pa);
3996     dma_free_coherent(&adapter->pdev->dev,
3997               sizeof(struct Vmxnet3_DriverShared),
3998               adapter->shared, adapter->shared_pa);
3999     dma_unmap_single(&adapter->pdev->dev, adapter->adapter_pa,
4000              sizeof(struct vmxnet3_adapter), DMA_TO_DEVICE);
4001     free_netdev(netdev);
4002 }
4003 
4004 static void vmxnet3_shutdown_device(struct pci_dev *pdev)
4005 {
4006     struct net_device *netdev = pci_get_drvdata(pdev);
4007     struct vmxnet3_adapter *adapter = netdev_priv(netdev);
4008     unsigned long flags;
4009 
4010     /* Reset_work may be in the middle of resetting the device, wait for its
4011      * completion.
4012      */
4013     while (test_and_set_bit(VMXNET3_STATE_BIT_RESETTING, &adapter->state))
4014         usleep_range(1000, 2000);
4015 
4016     if (test_and_set_bit(VMXNET3_STATE_BIT_QUIESCED,
4017                  &adapter->state)) {
4018         clear_bit(VMXNET3_STATE_BIT_RESETTING, &adapter->state);
4019         return;
4020     }
4021     spin_lock_irqsave(&adapter->cmd_lock, flags);
4022     VMXNET3_WRITE_BAR1_REG(adapter, VMXNET3_REG_CMD,
4023                    VMXNET3_CMD_QUIESCE_DEV);
4024     spin_unlock_irqrestore(&adapter->cmd_lock, flags);
4025     vmxnet3_disable_all_intrs(adapter);
4026 
4027     clear_bit(VMXNET3_STATE_BIT_RESETTING, &adapter->state);
4028 }
4029 
4030 
4031 #ifdef CONFIG_PM
4032 
4033 static int
4034 vmxnet3_suspend(struct device *device)
4035 {
4036     struct pci_dev *pdev = to_pci_dev(device);
4037     struct net_device *netdev = pci_get_drvdata(pdev);
4038     struct vmxnet3_adapter *adapter = netdev_priv(netdev);
4039     struct Vmxnet3_PMConf *pmConf;
4040     struct ethhdr *ehdr;
4041     struct arphdr *ahdr;
4042     u8 *arpreq;
4043     struct in_device *in_dev;
4044     struct in_ifaddr *ifa;
4045     unsigned long flags;
4046     int i = 0;
4047 
4048     if (!netif_running(netdev))
4049         return 0;
4050 
4051     for (i = 0; i < adapter->num_rx_queues; i++)
4052         napi_disable(&adapter->rx_queue[i].napi);
4053 
4054     vmxnet3_disable_all_intrs(adapter);
4055     vmxnet3_free_irqs(adapter);
4056     vmxnet3_free_intr_resources(adapter);
4057 
4058     netif_device_detach(netdev);
4059 
4060     /* Create wake-up filters. */
4061     pmConf = adapter->pm_conf;
4062     memset(pmConf, 0, sizeof(*pmConf));
4063 
4064     if (adapter->wol & WAKE_UCAST) {
4065         pmConf->filters[i].patternSize = ETH_ALEN;
4066         pmConf->filters[i].maskSize = 1;
4067         memcpy(pmConf->filters[i].pattern, netdev->dev_addr, ETH_ALEN);
4068         pmConf->filters[i].mask[0] = 0x3F; /* LSB ETH_ALEN bits */
4069 
4070         pmConf->wakeUpEvents |= VMXNET3_PM_WAKEUP_FILTER;
4071         i++;
4072     }
4073 
4074     if (adapter->wol & WAKE_ARP) {
4075         rcu_read_lock();
4076 
4077         in_dev = __in_dev_get_rcu(netdev);
4078         if (!in_dev) {
4079             rcu_read_unlock();
4080             goto skip_arp;
4081         }
4082 
4083         ifa = rcu_dereference(in_dev->ifa_list);
4084         if (!ifa) {
4085             rcu_read_unlock();
4086             goto skip_arp;
4087         }
4088 
4089         pmConf->filters[i].patternSize = ETH_HLEN + /* Ethernet header*/
4090             sizeof(struct arphdr) +     /* ARP header */
4091             2 * ETH_ALEN +      /* 2 Ethernet addresses*/
4092             2 * sizeof(u32);    /*2 IPv4 addresses */
4093         pmConf->filters[i].maskSize =
4094             (pmConf->filters[i].patternSize - 1) / 8 + 1;
4095 
4096         /* ETH_P_ARP in Ethernet header. */
4097         ehdr = (struct ethhdr *)pmConf->filters[i].pattern;
4098         ehdr->h_proto = htons(ETH_P_ARP);
4099 
4100         /* ARPOP_REQUEST in ARP header. */
4101         ahdr = (struct arphdr *)&pmConf->filters[i].pattern[ETH_HLEN];
4102         ahdr->ar_op = htons(ARPOP_REQUEST);
4103         arpreq = (u8 *)(ahdr + 1);
4104 
4105         /* The Unicast IPv4 address in 'tip' field. */
4106         arpreq += 2 * ETH_ALEN + sizeof(u32);
4107         *(__be32 *)arpreq = ifa->ifa_address;
4108 
4109         rcu_read_unlock();
4110 
4111         /* The mask for the relevant bits. */
4112         pmConf->filters[i].mask[0] = 0x00;
4113         pmConf->filters[i].mask[1] = 0x30; /* ETH_P_ARP */
4114         pmConf->filters[i].mask[2] = 0x30; /* ARPOP_REQUEST */
4115         pmConf->filters[i].mask[3] = 0x00;
4116         pmConf->filters[i].mask[4] = 0xC0; /* IPv4 TIP */
4117         pmConf->filters[i].mask[5] = 0x03; /* IPv4 TIP */
4118 
4119         pmConf->wakeUpEvents |= VMXNET3_PM_WAKEUP_FILTER;
4120         i++;
4121     }
4122 
4123 skip_arp:
4124     if (adapter->wol & WAKE_MAGIC)
4125         pmConf->wakeUpEvents |= VMXNET3_PM_WAKEUP_MAGIC;
4126 
4127     pmConf->numFilters = i;
4128 
4129     adapter->shared->devRead.pmConfDesc.confVer = cpu_to_le32(1);
4130     adapter->shared->devRead.pmConfDesc.confLen = cpu_to_le32(sizeof(
4131                                   *pmConf));
4132     adapter->shared->devRead.pmConfDesc.confPA =
4133         cpu_to_le64(adapter->pm_conf_pa);
4134 
4135     spin_lock_irqsave(&adapter->cmd_lock, flags);
4136     VMXNET3_WRITE_BAR1_REG(adapter, VMXNET3_REG_CMD,
4137                    VMXNET3_CMD_UPDATE_PMCFG);
4138     spin_unlock_irqrestore(&adapter->cmd_lock, flags);
4139 
4140     pci_save_state(pdev);
4141     pci_enable_wake(pdev, pci_choose_state(pdev, PMSG_SUSPEND),
4142             adapter->wol);
4143     pci_disable_device(pdev);
4144     pci_set_power_state(pdev, pci_choose_state(pdev, PMSG_SUSPEND));
4145 
4146     return 0;
4147 }
4148 
4149 
4150 static int
4151 vmxnet3_resume(struct device *device)
4152 {
4153     int err;
4154     unsigned long flags;
4155     struct pci_dev *pdev = to_pci_dev(device);
4156     struct net_device *netdev = pci_get_drvdata(pdev);
4157     struct vmxnet3_adapter *adapter = netdev_priv(netdev);
4158 
4159     if (!netif_running(netdev))
4160         return 0;
4161 
4162     pci_set_power_state(pdev, PCI_D0);
4163     pci_restore_state(pdev);
4164     err = pci_enable_device_mem(pdev);
4165     if (err != 0)
4166         return err;
4167 
4168     pci_enable_wake(pdev, PCI_D0, 0);
4169 
4170     vmxnet3_alloc_intr_resources(adapter);
4171 
4172     /* During hibernate and suspend, device has to be reinitialized as the
4173      * device state need not be preserved.
4174      */
4175 
4176     /* Need not check adapter state as other reset tasks cannot run during
4177      * device resume.
4178      */
4179     spin_lock_irqsave(&adapter->cmd_lock, flags);
4180     VMXNET3_WRITE_BAR1_REG(adapter, VMXNET3_REG_CMD,
4181                    VMXNET3_CMD_QUIESCE_DEV);
4182     spin_unlock_irqrestore(&adapter->cmd_lock, flags);
4183     vmxnet3_tq_cleanup_all(adapter);
4184     vmxnet3_rq_cleanup_all(adapter);
4185 
4186     vmxnet3_reset_dev(adapter);
4187     err = vmxnet3_activate_dev(adapter);
4188     if (err != 0) {
4189         netdev_err(netdev,
4190                "failed to re-activate on resume, error: %d", err);
4191         vmxnet3_force_close(adapter);
4192         return err;
4193     }
4194     netif_device_attach(netdev);
4195 
4196     return 0;
4197 }
4198 
4199 static const struct dev_pm_ops vmxnet3_pm_ops = {
4200     .suspend = vmxnet3_suspend,
4201     .resume = vmxnet3_resume,
4202     .freeze = vmxnet3_suspend,
4203     .restore = vmxnet3_resume,
4204 };
4205 #endif
4206 
4207 static struct pci_driver vmxnet3_driver = {
4208     .name       = vmxnet3_driver_name,
4209     .id_table   = vmxnet3_pciid_table,
4210     .probe      = vmxnet3_probe_device,
4211     .remove     = vmxnet3_remove_device,
4212     .shutdown   = vmxnet3_shutdown_device,
4213 #ifdef CONFIG_PM
4214     .driver.pm  = &vmxnet3_pm_ops,
4215 #endif
4216 };
4217 
4218 
4219 static int __init
4220 vmxnet3_init_module(void)
4221 {
4222     pr_info("%s - version %s\n", VMXNET3_DRIVER_DESC,
4223         VMXNET3_DRIVER_VERSION_REPORT);
4224     return pci_register_driver(&vmxnet3_driver);
4225 }
4226 
4227 module_init(vmxnet3_init_module);
4228 
4229 
4230 static void
4231 vmxnet3_exit_module(void)
4232 {
4233     pci_unregister_driver(&vmxnet3_driver);
4234 }
4235 
4236 module_exit(vmxnet3_exit_module);
4237 
4238 MODULE_AUTHOR("VMware, Inc.");
4239 MODULE_DESCRIPTION(VMXNET3_DRIVER_DESC);
4240 MODULE_LICENSE("GPL v2");
4241 MODULE_VERSION(VMXNET3_DRIVER_VERSION_STRING);
[ Source navigation ] [ Diff markup ] [ Identifier search ] [ general search ]

This page was automatically generated by the 2.1.0 LXR engine.

The LXR team
Valid CSS 2.1! Valid HTML 4.01!