ethernet/microchip/lan743x_main.c

0001 /* SPDX-License-Identifier: GPL-2.0+ */
0002 /* Copyright (C) 2018 Microchip Technology Inc. */
0003
0004 #include <linux/module.h>
0005 #include <linux/pci.h>
0006 #include <linux/netdevice.h>
0007 #include <linux/etherdevice.h>
0008 #include <linux/crc32.h>
0009 #include <linux/microchipphy.h>
0010 #include <linux/net_tstamp.h>
0011 #include <linux/of_mdio.h>
0012 #include <linux/of_net.h>
0013 #include <linux/phy.h>
0014 #include <linux/phy_fixed.h>
0015 #include <linux/rtnetlink.h>
0016 #include <linux/iopoll.h>
0017 #include <linux/crc16.h>
0018 #include "lan743x_main.h"
0019 #include "lan743x_ethtool.h"
0020
0021 #define MMD_ACCESS_ADDRESS  0
0022 #define MMD_ACCESS_WRITE    1
0023 #define MMD_ACCESS_READ     2
0024 #define MMD_ACCESS_READ_INC 3
0025 #define PCS_POWER_STATE_DOWN    0x6
0026 #define PCS_POWER_STATE_UP  0x4
0027
0028 static void pci11x1x_strap_get_status(struct lan743x_adapter *adapter)
0029 {
0030     u32 chip_rev;
0031     u32 cfg_load;
0032     u32 hw_cfg;
0033     u32 strap;
0034     int ret;
0035
0036     /* Timeout = 100 (i.e. 1 sec (10 msce * 100)) */
0037     ret = lan743x_hs_syslock_acquire(adapter, 100);
0038     if (ret < 0) {
0039         netif_err(adapter, drv, adapter->netdev,
0040               "Sys Lock acquire failed ret:%d\n", ret);
0041         return;
0042     }
0043
0044     cfg_load = lan743x_csr_read(adapter, ETH_SYS_CONFIG_LOAD_STARTED_REG);
0045     lan743x_hs_syslock_release(adapter);
0046     hw_cfg = lan743x_csr_read(adapter, HW_CFG);
0047
0048     if (cfg_load & GEN_SYS_LOAD_STARTED_REG_ETH_ ||
0049         hw_cfg & HW_CFG_RST_PROTECT_) {
0050         strap = lan743x_csr_read(adapter, STRAP_READ);
0051         if (strap & STRAP_READ_SGMII_EN_)
0052             adapter->is_sgmii_en = true;
0053         else
0054             adapter->is_sgmii_en = false;
0055     } else {
0056         chip_rev = lan743x_csr_read(adapter, FPGA_REV);
0057         if (chip_rev) {
0058             if (chip_rev & FPGA_SGMII_OP)
0059                 adapter->is_sgmii_en = true;
0060             else
0061                 adapter->is_sgmii_en = false;
0062         } else {
0063             adapter->is_sgmii_en = false;
0064         }
0065     }
0066     netif_dbg(adapter, drv, adapter->netdev,
0067           "SGMII I/F %sable\n", adapter->is_sgmii_en ? "En" : "Dis");
0068 }
0069
0070 static bool is_pci11x1x_chip(struct lan743x_adapter *adapter)
0071 {
0072     struct lan743x_csr *csr = &adapter->csr;
0073     u32 id_rev = csr->id_rev;
0074
0075     if (((id_rev & 0xFFFF0000) == ID_REV_ID_A011_) ||
0076         ((id_rev & 0xFFFF0000) == ID_REV_ID_A041_)) {
0077         return true;
0078     }
0079     return false;
0080 }
0081
0082 static void lan743x_pci_cleanup(struct lan743x_adapter *adapter)
0083 {
0084     pci_release_selected_regions(adapter->pdev,
0085                      pci_select_bars(adapter->pdev,
0086                              IORESOURCE_MEM));
0087     pci_disable_device(adapter->pdev);
0088 }
0089
0090 static int lan743x_pci_init(struct lan743x_adapter *adapter,
0091                 struct pci_dev *pdev)
0092 {
0093     unsigned long bars = 0;
0094     int ret;
0095
0096     adapter->pdev = pdev;
0097     ret = pci_enable_device_mem(pdev);
0098     if (ret)
0099         goto return_error;
0100
0101     netif_info(adapter, probe, adapter->netdev,
0102            "PCI: Vendor ID = 0x%04X, Device ID = 0x%04X\n",
0103            pdev->vendor, pdev->device);
0104     bars = pci_select_bars(pdev, IORESOURCE_MEM);
0105     if (!test_bit(0, &bars))
0106         goto disable_device;
0107
0108     ret = pci_request_selected_regions(pdev, bars, DRIVER_NAME);
0109     if (ret)
0110         goto disable_device;
0111
0112     pci_set_master(pdev);
0113     return 0;
0114
0115 disable_device:
0116     pci_disable_device(adapter->pdev);
0117
0118 return_error:
0119     return ret;
0120 }
0121
0122 u32 lan743x_csr_read(struct lan743x_adapter *adapter, int offset)
0123 {
0124     return ioread32(&adapter->csr.csr_address[offset]);
0125 }
0126
0127 void lan743x_csr_write(struct lan743x_adapter *adapter, int offset,
0128                u32 data)
0129 {
0130     iowrite32(data, &adapter->csr.csr_address[offset]);
0131 }
0132
0133 #define LAN743X_CSR_READ_OP(offset) lan743x_csr_read(adapter, offset)
0134
0135 static int lan743x_csr_light_reset(struct lan743x_adapter *adapter)
0136 {
0137     u32 data;
0138
0139     data = lan743x_csr_read(adapter, HW_CFG);
0140     data |= HW_CFG_LRST_;
0141     lan743x_csr_write(adapter, HW_CFG, data);
0142
0143     return readx_poll_timeout(LAN743X_CSR_READ_OP, HW_CFG, data,
0144                   !(data & HW_CFG_LRST_), 100000, 10000000);
0145 }
0146
0147 static int lan743x_csr_wait_for_bit(struct lan743x_adapter *adapter,
0148                     int offset, u32 bit_mask,
0149                     int target_value, int usleep_min,
0150                     int usleep_max, int count)
0151 {
0152     u32 data;
0153
0154     return readx_poll_timeout(LAN743X_CSR_READ_OP, offset, data,
0155                   target_value == ((data & bit_mask) ? 1 : 0),
0156                   usleep_max, usleep_min * count);
0157 }
0158
0159 static int lan743x_csr_init(struct lan743x_adapter *adapter)
0160 {
0161     struct lan743x_csr *csr = &adapter->csr;
0162     resource_size_t bar_start, bar_length;
0163     int result;
0164
0165     bar_start = pci_resource_start(adapter->pdev, 0);
0166     bar_length = pci_resource_len(adapter->pdev, 0);
0167     csr->csr_address = devm_ioremap(&adapter->pdev->dev,
0168                     bar_start, bar_length);
0169     if (!csr->csr_address) {
0170         result = -ENOMEM;
0171         goto clean_up;
0172     }
0173
0174     csr->id_rev = lan743x_csr_read(adapter, ID_REV);
0175     csr->fpga_rev = lan743x_csr_read(adapter, FPGA_REV);
0176     netif_info(adapter, probe, adapter->netdev,
0177            "ID_REV = 0x%08X, FPGA_REV = %d.%d\n",
0178            csr->id_rev, FPGA_REV_GET_MAJOR_(csr->fpga_rev),
0179            FPGA_REV_GET_MINOR_(csr->fpga_rev));
0180     if (!ID_REV_IS_VALID_CHIP_ID_(csr->id_rev)) {
0181         result = -ENODEV;
0182         goto clean_up;
0183     }
0184
0185     csr->flags = LAN743X_CSR_FLAG_SUPPORTS_INTR_AUTO_SET_CLR;
0186     switch (csr->id_rev & ID_REV_CHIP_REV_MASK_) {
0187     case ID_REV_CHIP_REV_A0_:
0188         csr->flags |= LAN743X_CSR_FLAG_IS_A0;
0189         csr->flags &= ~LAN743X_CSR_FLAG_SUPPORTS_INTR_AUTO_SET_CLR;
0190         break;
0191     case ID_REV_CHIP_REV_B0_:
0192         csr->flags |= LAN743X_CSR_FLAG_IS_B0;
0193         break;
0194     }
0195
0196     result = lan743x_csr_light_reset(adapter);
0197     if (result)
0198         goto clean_up;
0199     return 0;
0200 clean_up:
0201     return result;
0202 }
0203
0204 static void lan743x_intr_software_isr(struct lan743x_adapter *adapter)
0205 {
0206     struct lan743x_intr *intr = &adapter->intr;
0207
0208     /* disable the interrupt to prevent repeated re-triggering */
0209     lan743x_csr_write(adapter, INT_EN_CLR, INT_BIT_SW_GP_);
0210     intr->software_isr_flag = true;
0211     wake_up(&intr->software_isr_wq);
0212 }
0213
0214 static void lan743x_tx_isr(void *context, u32 int_sts, u32 flags)
0215 {
0216     struct lan743x_tx *tx = context;
0217     struct lan743x_adapter *adapter = tx->adapter;
0218     bool enable_flag = true;
0219
0220     lan743x_csr_read(adapter, INT_EN_SET);
0221     if (flags & LAN743X_VECTOR_FLAG_SOURCE_ENABLE_CLEAR) {
0222         lan743x_csr_write(adapter, INT_EN_CLR,
0223                   INT_BIT_DMA_TX_(tx->channel_number));
0224     }
0225
0226     if (int_sts & INT_BIT_DMA_TX_(tx->channel_number)) {
0227         u32 ioc_bit = DMAC_INT_BIT_TX_IOC_(tx->channel_number);
0228         u32 dmac_int_sts;
0229         u32 dmac_int_en;
0230
0231         if (flags & LAN743X_VECTOR_FLAG_SOURCE_STATUS_READ)
0232             dmac_int_sts = lan743x_csr_read(adapter, DMAC_INT_STS);
0233         else
0234             dmac_int_sts = ioc_bit;
0235         if (flags & LAN743X_VECTOR_FLAG_SOURCE_ENABLE_CHECK)
0236             dmac_int_en = lan743x_csr_read(adapter,
0237                                DMAC_INT_EN_SET);
0238         else
0239             dmac_int_en = ioc_bit;
0240
0241         dmac_int_en &= ioc_bit;
0242         dmac_int_sts &= dmac_int_en;
0243         if (dmac_int_sts & ioc_bit) {
0244             napi_schedule(&tx->napi);
0245             enable_flag = false;/* poll func will enable later */
0246         }
0247     }
0248
0249     if (enable_flag)
0250         /* enable isr */
0251         lan743x_csr_write(adapter, INT_EN_SET,
0252                   INT_BIT_DMA_TX_(tx->channel_number));
0253 }
0254
0255 static void lan743x_rx_isr(void *context, u32 int_sts, u32 flags)
0256 {
0257     struct lan743x_rx *rx = context;
0258     struct lan743x_adapter *adapter = rx->adapter;
0259     bool enable_flag = true;
0260
0261     if (flags & LAN743X_VECTOR_FLAG_SOURCE_ENABLE_CLEAR) {
0262         lan743x_csr_write(adapter, INT_EN_CLR,
0263                   INT_BIT_DMA_RX_(rx->channel_number));
0264     }
0265
0266     if (int_sts & INT_BIT_DMA_RX_(rx->channel_number)) {
0267         u32 rx_frame_bit = DMAC_INT_BIT_RXFRM_(rx->channel_number);
0268         u32 dmac_int_sts;
0269         u32 dmac_int_en;
0270
0271         if (flags & LAN743X_VECTOR_FLAG_SOURCE_STATUS_READ)
0272             dmac_int_sts = lan743x_csr_read(adapter, DMAC_INT_STS);
0273         else
0274             dmac_int_sts = rx_frame_bit;
0275         if (flags & LAN743X_VECTOR_FLAG_SOURCE_ENABLE_CHECK)
0276             dmac_int_en = lan743x_csr_read(adapter,
0277                                DMAC_INT_EN_SET);
0278         else
0279             dmac_int_en = rx_frame_bit;
0280
0281         dmac_int_en &= rx_frame_bit;
0282         dmac_int_sts &= dmac_int_en;
0283         if (dmac_int_sts & rx_frame_bit) {
0284             napi_schedule(&rx->napi);
0285             enable_flag = false;/* poll funct will enable later */
0286         }
0287     }
0288
0289     if (enable_flag) {
0290         /* enable isr */
0291         lan743x_csr_write(adapter, INT_EN_SET,
0292                   INT_BIT_DMA_RX_(rx->channel_number));
0293     }
0294 }
0295
0296 static void lan743x_intr_shared_isr(void *context, u32 int_sts, u32 flags)
0297 {
0298     struct lan743x_adapter *adapter = context;
0299     unsigned int channel;
0300
0301     if (int_sts & INT_BIT_ALL_RX_) {
0302         for (channel = 0; channel < LAN743X_USED_RX_CHANNELS;
0303             channel++) {
0304             u32 int_bit = INT_BIT_DMA_RX_(channel);
0305
0306             if (int_sts & int_bit) {
0307                 lan743x_rx_isr(&adapter->rx[channel],
0308                            int_bit, flags);
0309                 int_sts &= ~int_bit;
0310             }
0311         }
0312     }
0313     if (int_sts & INT_BIT_ALL_TX_) {
0314         for (channel = 0; channel < adapter->used_tx_channels;
0315             channel++) {
0316             u32 int_bit = INT_BIT_DMA_TX_(channel);
0317
0318             if (int_sts & int_bit) {
0319                 lan743x_tx_isr(&adapter->tx[channel],
0320                            int_bit, flags);
0321                 int_sts &= ~int_bit;
0322             }
0323         }
0324     }
0325     if (int_sts & INT_BIT_ALL_OTHER_) {
0326         if (int_sts & INT_BIT_SW_GP_) {
0327             lan743x_intr_software_isr(adapter);
0328             int_sts &= ~INT_BIT_SW_GP_;
0329         }
0330         if (int_sts & INT_BIT_1588_) {
0331             lan743x_ptp_isr(adapter);
0332             int_sts &= ~INT_BIT_1588_;
0333         }
0334     }
0335     if (int_sts)
0336         lan743x_csr_write(adapter, INT_EN_CLR, int_sts);
0337 }
0338
0339 static irqreturn_t lan743x_intr_entry_isr(int irq, void *ptr)
0340 {
0341     struct lan743x_vector *vector = ptr;
0342     struct lan743x_adapter *adapter = vector->adapter;
0343     irqreturn_t result = IRQ_NONE;
0344     u32 int_enables;
0345     u32 int_sts;
0346
0347     if (vector->flags & LAN743X_VECTOR_FLAG_SOURCE_STATUS_READ) {
0348         int_sts = lan743x_csr_read(adapter, INT_STS);
0349     } else if (vector->flags &
0350            (LAN743X_VECTOR_FLAG_SOURCE_STATUS_R2C |
0351            LAN743X_VECTOR_FLAG_SOURCE_ENABLE_R2C)) {
0352         int_sts = lan743x_csr_read(adapter, INT_STS_R2C);
0353     } else {
0354         /* use mask as implied status */
0355         int_sts = vector->int_mask | INT_BIT_MAS_;
0356     }
0357
0358     if (!(int_sts & INT_BIT_MAS_))
0359         goto irq_done;
0360
0361     if (vector->flags & LAN743X_VECTOR_FLAG_VECTOR_ENABLE_ISR_CLEAR)
0362         /* disable vector interrupt */
0363         lan743x_csr_write(adapter,
0364                   INT_VEC_EN_CLR,
0365                   INT_VEC_EN_(vector->vector_index));
0366
0367     if (vector->flags & LAN743X_VECTOR_FLAG_MASTER_ENABLE_CLEAR)
0368         /* disable master interrupt */
0369         lan743x_csr_write(adapter, INT_EN_CLR, INT_BIT_MAS_);
0370
0371     if (vector->flags & LAN743X_VECTOR_FLAG_SOURCE_ENABLE_CHECK) {
0372         int_enables = lan743x_csr_read(adapter, INT_EN_SET);
0373     } else {
0374         /*  use vector mask as implied enable mask */
0375         int_enables = vector->int_mask;
0376     }
0377
0378     int_sts &= int_enables;
0379     int_sts &= vector->int_mask;
0380     if (int_sts) {
0381         if (vector->handler) {
0382             vector->handler(vector->context,
0383                     int_sts, vector->flags);
0384         } else {
0385             /* disable interrupts on this vector */
0386             lan743x_csr_write(adapter, INT_EN_CLR,
0387                       vector->int_mask);
0388         }
0389         result = IRQ_HANDLED;
0390     }
0391
0392     if (vector->flags & LAN743X_VECTOR_FLAG_MASTER_ENABLE_SET)
0393         /* enable master interrupt */
0394         lan743x_csr_write(adapter, INT_EN_SET, INT_BIT_MAS_);
0395
0396     if (vector->flags & LAN743X_VECTOR_FLAG_VECTOR_ENABLE_ISR_SET)
0397         /* enable vector interrupt */
0398         lan743x_csr_write(adapter,
0399                   INT_VEC_EN_SET,
0400                   INT_VEC_EN_(vector->vector_index));
0401 irq_done:
0402     return result;
0403 }
0404
0405 static int lan743x_intr_test_isr(struct lan743x_adapter *adapter)
0406 {
0407     struct lan743x_intr *intr = &adapter->intr;
0408     int ret;
0409
0410     intr->software_isr_flag = false;
0411
0412     /* enable and activate test interrupt */
0413     lan743x_csr_write(adapter, INT_EN_SET, INT_BIT_SW_GP_);
0414     lan743x_csr_write(adapter, INT_SET, INT_BIT_SW_GP_);
0415
0416     ret = wait_event_timeout(intr->software_isr_wq,
0417                  intr->software_isr_flag,
0418                  msecs_to_jiffies(200));
0419
0420     /* disable test interrupt */
0421     lan743x_csr_write(adapter, INT_EN_CLR, INT_BIT_SW_GP_);
0422
0423     return ret > 0 ? 0 : -ENODEV;
0424 }
0425
0426 static int lan743x_intr_register_isr(struct lan743x_adapter *adapter,
0427                      int vector_index, u32 flags,
0428                      u32 int_mask,
0429                      lan743x_vector_handler handler,
0430                      void *context)
0431 {
0432     struct lan743x_vector *vector = &adapter->intr.vector_list
0433                     [vector_index];
0434     int ret;
0435
0436     vector->adapter = adapter;
0437     vector->flags = flags;
0438     vector->vector_index = vector_index;
0439     vector->int_mask = int_mask;
0440     vector->handler = handler;
0441     vector->context = context;
0442
0443     ret = request_irq(vector->irq,
0444               lan743x_intr_entry_isr,
0445               (flags & LAN743X_VECTOR_FLAG_IRQ_SHARED) ?
0446               IRQF_SHARED : 0, DRIVER_NAME, vector);
0447     if (ret) {
0448         vector->handler = NULL;
0449         vector->context = NULL;
0450         vector->int_mask = 0;
0451         vector->flags = 0;
0452     }
0453     return ret;
0454 }
0455
0456 static void lan743x_intr_unregister_isr(struct lan743x_adapter *adapter,
0457                     int vector_index)
0458 {
0459     struct lan743x_vector *vector = &adapter->intr.vector_list
0460                     [vector_index];
0461
0462     free_irq(vector->irq, vector);
0463     vector->handler = NULL;
0464     vector->context = NULL;
0465     vector->int_mask = 0;
0466     vector->flags = 0;
0467 }
0468
0469 static u32 lan743x_intr_get_vector_flags(struct lan743x_adapter *adapter,
0470                      u32 int_mask)
0471 {
0472     int index;
0473
0474     for (index = 0; index < adapter->max_vector_count; index++) {
0475         if (adapter->intr.vector_list[index].int_mask & int_mask)
0476             return adapter->intr.vector_list[index].flags;
0477     }
0478     return 0;
0479 }
0480
0481 static void lan743x_intr_close(struct lan743x_adapter *adapter)
0482 {
0483     struct lan743x_intr *intr = &adapter->intr;
0484     int index = 0;
0485
0486     lan743x_csr_write(adapter, INT_EN_CLR, INT_BIT_MAS_);
0487     if (adapter->is_pci11x1x)
0488         lan743x_csr_write(adapter, INT_VEC_EN_CLR, 0x0000FFFF);
0489     else
0490         lan743x_csr_write(adapter, INT_VEC_EN_CLR, 0x000000FF);
0491
0492     for (index = 0; index < intr->number_of_vectors; index++) {
0493         if (intr->flags & INTR_FLAG_IRQ_REQUESTED(index)) {
0494             lan743x_intr_unregister_isr(adapter, index);
0495             intr->flags &= ~INTR_FLAG_IRQ_REQUESTED(index);
0496         }
0497     }
0498
0499     if (intr->flags & INTR_FLAG_MSI_ENABLED) {
0500         pci_disable_msi(adapter->pdev);
0501         intr->flags &= ~INTR_FLAG_MSI_ENABLED;
0502     }
0503
0504     if (intr->flags & INTR_FLAG_MSIX_ENABLED) {
0505         pci_disable_msix(adapter->pdev);
0506         intr->flags &= ~INTR_FLAG_MSIX_ENABLED;
0507     }
0508 }
0509
0510 static int lan743x_intr_open(struct lan743x_adapter *adapter)
0511 {
0512     struct msix_entry msix_entries[PCI11X1X_MAX_VECTOR_COUNT];
0513     struct lan743x_intr *intr = &adapter->intr;
0514     unsigned int used_tx_channels;
0515     u32 int_vec_en_auto_clr = 0;
0516     u8 max_vector_count;
0517     u32 int_vec_map0 = 0;
0518     u32 int_vec_map1 = 0;
0519     int ret = -ENODEV;
0520     int index = 0;
0521     u32 flags = 0;
0522
0523     intr->number_of_vectors = 0;
0524
0525     /* Try to set up MSIX interrupts */
0526     max_vector_count = adapter->max_vector_count;
0527     memset(&msix_entries[0], 0,
0528            sizeof(struct msix_entry) * max_vector_count);
0529     for (index = 0; index < max_vector_count; index++)
0530         msix_entries[index].entry = index;
0531     used_tx_channels = adapter->used_tx_channels;
0532     ret = pci_enable_msix_range(adapter->pdev,
0533                     msix_entries, 1,
0534                     1 + used_tx_channels +
0535                     LAN743X_USED_RX_CHANNELS);
0536
0537     if (ret > 0) {
0538         intr->flags |= INTR_FLAG_MSIX_ENABLED;
0539         intr->number_of_vectors = ret;
0540         intr->using_vectors = true;
0541         for (index = 0; index < intr->number_of_vectors; index++)
0542             intr->vector_list[index].irq = msix_entries
0543                                [index].vector;
0544         netif_info(adapter, ifup, adapter->netdev,
0545                "using MSIX interrupts, number of vectors = %d\n",
0546                intr->number_of_vectors);
0547     }
0548
0549     /* If MSIX failed try to setup using MSI interrupts */
0550     if (!intr->number_of_vectors) {
0551         if (!(adapter->csr.flags & LAN743X_CSR_FLAG_IS_A0)) {
0552             if (!pci_enable_msi(adapter->pdev)) {
0553                 intr->flags |= INTR_FLAG_MSI_ENABLED;
0554                 intr->number_of_vectors = 1;
0555                 intr->using_vectors = true;
0556                 intr->vector_list[0].irq =
0557                     adapter->pdev->irq;
0558                 netif_info(adapter, ifup, adapter->netdev,
0559                        "using MSI interrupts, number of vectors = %d\n",
0560                        intr->number_of_vectors);
0561             }
0562         }
0563     }
0564
0565     /* If MSIX, and MSI failed, setup using legacy interrupt */
0566     if (!intr->number_of_vectors) {
0567         intr->number_of_vectors = 1;
0568         intr->using_vectors = false;
0569         intr->vector_list[0].irq = intr->irq;
0570         netif_info(adapter, ifup, adapter->netdev,
0571                "using legacy interrupts\n");
0572     }
0573
0574     /* At this point we must have at least one irq */
0575     lan743x_csr_write(adapter, INT_VEC_EN_CLR, 0xFFFFFFFF);
0576
0577     /* map all interrupts to vector 0 */
0578     lan743x_csr_write(adapter, INT_VEC_MAP0, 0x00000000);
0579     lan743x_csr_write(adapter, INT_VEC_MAP1, 0x00000000);
0580     lan743x_csr_write(adapter, INT_VEC_MAP2, 0x00000000);
0581     flags = LAN743X_VECTOR_FLAG_SOURCE_STATUS_READ |
0582         LAN743X_VECTOR_FLAG_SOURCE_STATUS_W2C |
0583         LAN743X_VECTOR_FLAG_SOURCE_ENABLE_CHECK |
0584         LAN743X_VECTOR_FLAG_SOURCE_ENABLE_CLEAR;
0585
0586     if (intr->using_vectors) {
0587         flags |= LAN743X_VECTOR_FLAG_VECTOR_ENABLE_ISR_CLEAR |
0588              LAN743X_VECTOR_FLAG_VECTOR_ENABLE_ISR_SET;
0589     } else {
0590         flags |= LAN743X_VECTOR_FLAG_MASTER_ENABLE_CLEAR |
0591              LAN743X_VECTOR_FLAG_MASTER_ENABLE_SET |
0592              LAN743X_VECTOR_FLAG_IRQ_SHARED;
0593     }
0594
0595     if (adapter->csr.flags & LAN743X_CSR_FLAG_SUPPORTS_INTR_AUTO_SET_CLR) {
0596         flags &= ~LAN743X_VECTOR_FLAG_SOURCE_STATUS_READ;
0597         flags &= ~LAN743X_VECTOR_FLAG_SOURCE_STATUS_W2C;
0598         flags &= ~LAN743X_VECTOR_FLAG_SOURCE_ENABLE_CLEAR;
0599         flags &= ~LAN743X_VECTOR_FLAG_SOURCE_ENABLE_CHECK;
0600         flags |= LAN743X_VECTOR_FLAG_SOURCE_STATUS_R2C;
0601         flags |= LAN743X_VECTOR_FLAG_SOURCE_ENABLE_R2C;
0602     }
0603
0604     init_waitqueue_head(&intr->software_isr_wq);
0605
0606     ret = lan743x_intr_register_isr(adapter, 0, flags,
0607                     INT_BIT_ALL_RX_ | INT_BIT_ALL_TX_ |
0608                     INT_BIT_ALL_OTHER_,
0609                     lan743x_intr_shared_isr, adapter);
0610     if (ret)
0611         goto clean_up;
0612     intr->flags |= INTR_FLAG_IRQ_REQUESTED(0);
0613
0614     if (intr->using_vectors)
0615         lan743x_csr_write(adapter, INT_VEC_EN_SET,
0616                   INT_VEC_EN_(0));
0617
0618     if (!(adapter->csr.flags & LAN743X_CSR_FLAG_IS_A0)) {
0619         lan743x_csr_write(adapter, INT_MOD_CFG0, LAN743X_INT_MOD);
0620         lan743x_csr_write(adapter, INT_MOD_CFG1, LAN743X_INT_MOD);
0621         lan743x_csr_write(adapter, INT_MOD_CFG2, LAN743X_INT_MOD);
0622         lan743x_csr_write(adapter, INT_MOD_CFG3, LAN743X_INT_MOD);
0623         lan743x_csr_write(adapter, INT_MOD_CFG4, LAN743X_INT_MOD);
0624         lan743x_csr_write(adapter, INT_MOD_CFG5, LAN743X_INT_MOD);
0625         lan743x_csr_write(adapter, INT_MOD_CFG6, LAN743X_INT_MOD);
0626         lan743x_csr_write(adapter, INT_MOD_CFG7, LAN743X_INT_MOD);
0627         if (adapter->is_pci11x1x) {
0628             lan743x_csr_write(adapter, INT_MOD_CFG8, LAN743X_INT_MOD);
0629             lan743x_csr_write(adapter, INT_MOD_CFG9, LAN743X_INT_MOD);
0630             lan743x_csr_write(adapter, INT_MOD_MAP0, 0x00007654);
0631             lan743x_csr_write(adapter, INT_MOD_MAP1, 0x00003210);
0632         } else {
0633             lan743x_csr_write(adapter, INT_MOD_MAP0, 0x00005432);
0634             lan743x_csr_write(adapter, INT_MOD_MAP1, 0x00000001);
0635         }
0636         lan743x_csr_write(adapter, INT_MOD_MAP2, 0x00FFFFFF);
0637     }
0638
0639     /* enable interrupts */
0640     lan743x_csr_write(adapter, INT_EN_SET, INT_BIT_MAS_);
0641     ret = lan743x_intr_test_isr(adapter);
0642     if (ret)
0643         goto clean_up;
0644
0645     if (intr->number_of_vectors > 1) {
0646         int number_of_tx_vectors = intr->number_of_vectors - 1;
0647
0648         if (number_of_tx_vectors > used_tx_channels)
0649             number_of_tx_vectors = used_tx_channels;
0650         flags = LAN743X_VECTOR_FLAG_SOURCE_STATUS_READ |
0651             LAN743X_VECTOR_FLAG_SOURCE_STATUS_W2C |
0652             LAN743X_VECTOR_FLAG_SOURCE_ENABLE_CHECK |
0653             LAN743X_VECTOR_FLAG_SOURCE_ENABLE_CLEAR |
0654             LAN743X_VECTOR_FLAG_VECTOR_ENABLE_ISR_CLEAR |
0655             LAN743X_VECTOR_FLAG_VECTOR_ENABLE_ISR_SET;
0656
0657         if (adapter->csr.flags &
0658            LAN743X_CSR_FLAG_SUPPORTS_INTR_AUTO_SET_CLR) {
0659             flags = LAN743X_VECTOR_FLAG_VECTOR_ENABLE_AUTO_SET |
0660                 LAN743X_VECTOR_FLAG_SOURCE_ENABLE_AUTO_SET |
0661                 LAN743X_VECTOR_FLAG_SOURCE_ENABLE_AUTO_CLEAR |
0662                 LAN743X_VECTOR_FLAG_SOURCE_STATUS_AUTO_CLEAR;
0663         }
0664
0665         for (index = 0; index < number_of_tx_vectors; index++) {
0666             u32 int_bit = INT_BIT_DMA_TX_(index);
0667             int vector = index + 1;
0668
0669             /* map TX interrupt to vector */
0670             int_vec_map1 |= INT_VEC_MAP1_TX_VEC_(index, vector);
0671             lan743x_csr_write(adapter, INT_VEC_MAP1, int_vec_map1);
0672
0673             /* Remove TX interrupt from shared mask */
0674             intr->vector_list[0].int_mask &= ~int_bit;
0675             ret = lan743x_intr_register_isr(adapter, vector, flags,
0676                             int_bit, lan743x_tx_isr,
0677                             &adapter->tx[index]);
0678             if (ret)
0679                 goto clean_up;
0680             intr->flags |= INTR_FLAG_IRQ_REQUESTED(vector);
0681             if (!(flags &
0682                 LAN743X_VECTOR_FLAG_VECTOR_ENABLE_AUTO_SET))
0683                 lan743x_csr_write(adapter, INT_VEC_EN_SET,
0684                           INT_VEC_EN_(vector));
0685         }
0686     }
0687     if ((intr->number_of_vectors - used_tx_channels) > 1) {
0688         int number_of_rx_vectors = intr->number_of_vectors -
0689                         used_tx_channels - 1;
0690
0691         if (number_of_rx_vectors > LAN743X_USED_RX_CHANNELS)
0692             number_of_rx_vectors = LAN743X_USED_RX_CHANNELS;
0693
0694         flags = LAN743X_VECTOR_FLAG_SOURCE_STATUS_READ |
0695             LAN743X_VECTOR_FLAG_SOURCE_STATUS_W2C |
0696             LAN743X_VECTOR_FLAG_SOURCE_ENABLE_CHECK |
0697             LAN743X_VECTOR_FLAG_SOURCE_ENABLE_CLEAR |
0698             LAN743X_VECTOR_FLAG_VECTOR_ENABLE_ISR_CLEAR |
0699             LAN743X_VECTOR_FLAG_VECTOR_ENABLE_ISR_SET;
0700
0701         if (adapter->csr.flags &
0702             LAN743X_CSR_FLAG_SUPPORTS_INTR_AUTO_SET_CLR) {
0703             flags = LAN743X_VECTOR_FLAG_VECTOR_ENABLE_AUTO_CLEAR |
0704                 LAN743X_VECTOR_FLAG_VECTOR_ENABLE_AUTO_SET |
0705                 LAN743X_VECTOR_FLAG_SOURCE_ENABLE_AUTO_SET |
0706                 LAN743X_VECTOR_FLAG_SOURCE_ENABLE_AUTO_CLEAR |
0707                 LAN743X_VECTOR_FLAG_SOURCE_STATUS_AUTO_CLEAR;
0708         }
0709         for (index = 0; index < number_of_rx_vectors; index++) {
0710             int vector = index + 1 + used_tx_channels;
0711             u32 int_bit = INT_BIT_DMA_RX_(index);
0712
0713             /* map RX interrupt to vector */
0714             int_vec_map0 |= INT_VEC_MAP0_RX_VEC_(index, vector);
0715             lan743x_csr_write(adapter, INT_VEC_MAP0, int_vec_map0);
0716             if (flags &
0717                 LAN743X_VECTOR_FLAG_VECTOR_ENABLE_AUTO_CLEAR) {
0718                 int_vec_en_auto_clr |= INT_VEC_EN_(vector);
0719                 lan743x_csr_write(adapter, INT_VEC_EN_AUTO_CLR,
0720                           int_vec_en_auto_clr);
0721             }
0722
0723             /* Remove RX interrupt from shared mask */
0724             intr->vector_list[0].int_mask &= ~int_bit;
0725             ret = lan743x_intr_register_isr(adapter, vector, flags,
0726                             int_bit, lan743x_rx_isr,
0727                             &adapter->rx[index]);
0728             if (ret)
0729                 goto clean_up;
0730             intr->flags |= INTR_FLAG_IRQ_REQUESTED(vector);
0731
0732             lan743x_csr_write(adapter, INT_VEC_EN_SET,
0733                       INT_VEC_EN_(vector));
0734         }
0735     }
0736     return 0;
0737
0738 clean_up:
0739     lan743x_intr_close(adapter);
0740     return ret;
0741 }
0742
0743 static int lan743x_dp_write(struct lan743x_adapter *adapter,
0744                 u32 select, u32 addr, u32 length, u32 *buf)
0745 {
0746     u32 dp_sel;
0747     int i;
0748
0749     if (lan743x_csr_wait_for_bit(adapter, DP_SEL, DP_SEL_DPRDY_,
0750                      1, 40, 100, 100))
0751         return -EIO;
0752     dp_sel = lan743x_csr_read(adapter, DP_SEL);
0753     dp_sel &= ~DP_SEL_MASK_;
0754     dp_sel |= select;
0755     lan743x_csr_write(adapter, DP_SEL, dp_sel);
0756
0757     for (i = 0; i < length; i++) {
0758         lan743x_csr_write(adapter, DP_ADDR, addr + i);
0759         lan743x_csr_write(adapter, DP_DATA_0, buf[i]);
0760         lan743x_csr_write(adapter, DP_CMD, DP_CMD_WRITE_);
0761         if (lan743x_csr_wait_for_bit(adapter, DP_SEL, DP_SEL_DPRDY_,
0762                          1, 40, 100, 100))
0763             return -EIO;
0764     }
0765
0766     return 0;
0767 }
0768
0769 static u32 lan743x_mac_mii_access(u16 id, u16 index, int read)
0770 {
0771     u32 ret;
0772
0773     ret = (id << MAC_MII_ACC_PHY_ADDR_SHIFT_) &
0774         MAC_MII_ACC_PHY_ADDR_MASK_;
0775     ret |= (index << MAC_MII_ACC_MIIRINDA_SHIFT_) &
0776         MAC_MII_ACC_MIIRINDA_MASK_;
0777
0778     if (read)
0779         ret |= MAC_MII_ACC_MII_READ_;
0780     else
0781         ret |= MAC_MII_ACC_MII_WRITE_;
0782     ret |= MAC_MII_ACC_MII_BUSY_;
0783
0784     return ret;
0785 }
0786
0787 static int lan743x_mac_mii_wait_till_not_busy(struct lan743x_adapter *adapter)
0788 {
0789     u32 data;
0790
0791     return readx_poll_timeout(LAN743X_CSR_READ_OP, MAC_MII_ACC, data,
0792                   !(data & MAC_MII_ACC_MII_BUSY_), 0, 1000000);
0793 }
0794
0795 static int lan743x_mdiobus_read(struct mii_bus *bus, int phy_id, int index)
0796 {
0797     struct lan743x_adapter *adapter = bus->priv;
0798     u32 val, mii_access;
0799     int ret;
0800
0801     /* comfirm MII not busy */
0802     ret = lan743x_mac_mii_wait_till_not_busy(adapter);
0803     if (ret < 0)
0804         return ret;
0805
0806     /* set the address, index & direction (read from PHY) */
0807     mii_access = lan743x_mac_mii_access(phy_id, index, MAC_MII_READ);
0808     lan743x_csr_write(adapter, MAC_MII_ACC, mii_access);
0809     ret = lan743x_mac_mii_wait_till_not_busy(adapter);
0810     if (ret < 0)
0811         return ret;
0812
0813     val = lan743x_csr_read(adapter, MAC_MII_DATA);
0814     return (int)(val & 0xFFFF);
0815 }
0816
0817 static int lan743x_mdiobus_write(struct mii_bus *bus,
0818                  int phy_id, int index, u16 regval)
0819 {
0820     struct lan743x_adapter *adapter = bus->priv;
0821     u32 val, mii_access;
0822     int ret;
0823
0824     /* confirm MII not busy */
0825     ret = lan743x_mac_mii_wait_till_not_busy(adapter);
0826     if (ret < 0)
0827         return ret;
0828     val = (u32)regval;
0829     lan743x_csr_write(adapter, MAC_MII_DATA, val);
0830
0831     /* set the address, index & direction (write to PHY) */
0832     mii_access = lan743x_mac_mii_access(phy_id, index, MAC_MII_WRITE);
0833     lan743x_csr_write(adapter, MAC_MII_ACC, mii_access);
0834     ret = lan743x_mac_mii_wait_till_not_busy(adapter);
0835     return ret;
0836 }
0837
0838 static u32 lan743x_mac_mmd_access(int id, int index, int op)
0839 {
0840     u16 dev_addr;
0841     u32 ret;
0842
0843     dev_addr = (index >> 16) & 0x1f;
0844     ret = (id << MAC_MII_ACC_PHY_ADDR_SHIFT_) &
0845         MAC_MII_ACC_PHY_ADDR_MASK_;
0846     ret |= (dev_addr << MAC_MII_ACC_MIIMMD_SHIFT_) &
0847         MAC_MII_ACC_MIIMMD_MASK_;
0848     if (op == MMD_ACCESS_WRITE)
0849         ret |= MAC_MII_ACC_MIICMD_WRITE_;
0850     else if (op == MMD_ACCESS_READ)
0851         ret |= MAC_MII_ACC_MIICMD_READ_;
0852     else if (op == MMD_ACCESS_READ_INC)
0853         ret |= MAC_MII_ACC_MIICMD_READ_INC_;
0854     else
0855         ret |= MAC_MII_ACC_MIICMD_ADDR_;
0856     ret |= (MAC_MII_ACC_MII_BUSY_ | MAC_MII_ACC_MIICL45_);
0857
0858     return ret;
0859 }
0860
0861 static int lan743x_mdiobus_c45_read(struct mii_bus *bus, int phy_id, int index)
0862 {
0863     struct lan743x_adapter *adapter = bus->priv;
0864     u32 mmd_access;
0865     int ret;
0866
0867     /* comfirm MII not busy */
0868     ret = lan743x_mac_mii_wait_till_not_busy(adapter);
0869     if (ret < 0)
0870         return ret;
0871     if (index & MII_ADDR_C45) {
0872         /* Load Register Address */
0873         lan743x_csr_write(adapter, MAC_MII_DATA, (u32)(index & 0xffff));
0874         mmd_access = lan743x_mac_mmd_access(phy_id, index,
0875                             MMD_ACCESS_ADDRESS);
0876         lan743x_csr_write(adapter, MAC_MII_ACC, mmd_access);
0877         ret = lan743x_mac_mii_wait_till_not_busy(adapter);
0878         if (ret < 0)
0879             return ret;
0880         /* Read Data */
0881         mmd_access = lan743x_mac_mmd_access(phy_id, index,
0882                             MMD_ACCESS_READ);
0883         lan743x_csr_write(adapter, MAC_MII_ACC, mmd_access);
0884         ret = lan743x_mac_mii_wait_till_not_busy(adapter);
0885         if (ret < 0)
0886             return ret;
0887         ret = lan743x_csr_read(adapter, MAC_MII_DATA);
0888         return (int)(ret & 0xFFFF);
0889     }
0890
0891     ret = lan743x_mdiobus_read(bus, phy_id, index);
0892     return ret;
0893 }
0894
0895 static int lan743x_mdiobus_c45_write(struct mii_bus *bus,
0896                      int phy_id, int index, u16 regval)
0897 {
0898     struct lan743x_adapter *adapter = bus->priv;
0899     u32 mmd_access;
0900     int ret;
0901
0902     /* confirm MII not busy */
0903     ret = lan743x_mac_mii_wait_till_not_busy(adapter);
0904     if (ret < 0)
0905         return ret;
0906     if (index & MII_ADDR_C45) {
0907         /* Load Register Address */
0908         lan743x_csr_write(adapter, MAC_MII_DATA, (u32)(index & 0xffff));
0909         mmd_access = lan743x_mac_mmd_access(phy_id, index,
0910                             MMD_ACCESS_ADDRESS);
0911         lan743x_csr_write(adapter, MAC_MII_ACC, mmd_access);
0912         ret = lan743x_mac_mii_wait_till_not_busy(adapter);
0913         if (ret < 0)
0914             return ret;
0915         /* Write Data */
0916         lan743x_csr_write(adapter, MAC_MII_DATA, (u32)regval);
0917         mmd_access = lan743x_mac_mmd_access(phy_id, index,
0918                             MMD_ACCESS_WRITE);
0919         lan743x_csr_write(adapter, MAC_MII_ACC, mmd_access);
0920         ret = lan743x_mac_mii_wait_till_not_busy(adapter);
0921     } else {
0922         ret = lan743x_mdiobus_write(bus, phy_id, index, regval);
0923     }
0924
0925     return ret;
0926 }
0927
0928 static int lan743x_sgmii_wait_till_not_busy(struct lan743x_adapter *adapter)
0929 {
0930     u32 data;
0931     int ret;
0932
0933     ret = readx_poll_timeout(LAN743X_CSR_READ_OP, SGMII_ACC, data,
0934                  !(data & SGMII_ACC_SGMII_BZY_), 100, 1000000);
0935     if (ret < 0)
0936         netif_err(adapter, drv, adapter->netdev,
0937               "%s: error %d sgmii wait timeout\n", __func__, ret);
0938
0939     return ret;
0940 }
0941
0942 static int lan743x_sgmii_read(struct lan743x_adapter *adapter, u8 mmd, u16 addr)
0943 {
0944     u32 mmd_access;
0945     int ret;
0946     u32 val;
0947
0948     if (mmd > 31) {
0949         netif_err(adapter, probe, adapter->netdev,
0950               "%s mmd should <= 31\n", __func__);
0951         return -EINVAL;
0952     }
0953
0954     mutex_lock(&adapter->sgmii_rw_lock);
0955     /* Load Register Address */
0956     mmd_access = mmd << SGMII_ACC_SGMII_MMD_SHIFT_;
0957     mmd_access |= (addr | SGMII_ACC_SGMII_BZY_);
0958     lan743x_csr_write(adapter, SGMII_ACC, mmd_access);
0959     ret = lan743x_sgmii_wait_till_not_busy(adapter);
0960     if (ret < 0)
0961         goto sgmii_unlock;
0962
0963     val = lan743x_csr_read(adapter, SGMII_DATA);
0964     ret = (int)(val & SGMII_DATA_MASK_);
0965
0966 sgmii_unlock:
0967     mutex_unlock(&adapter->sgmii_rw_lock);
0968
0969     return ret;
0970 }
0971
0972 static int lan743x_sgmii_write(struct lan743x_adapter *adapter,
0973                    u8 mmd, u16 addr, u16 val)
0974 {
0975     u32 mmd_access;
0976     int ret;
0977
0978     if (mmd > 31) {
0979         netif_err(adapter, probe, adapter->netdev,
0980               "%s mmd should <= 31\n", __func__);
0981         return -EINVAL;
0982     }
0983     mutex_lock(&adapter->sgmii_rw_lock);
0984     /* Load Register Data */
0985     lan743x_csr_write(adapter, SGMII_DATA, (u32)(val & SGMII_DATA_MASK_));
0986     /* Load Register Address */
0987     mmd_access = mmd << SGMII_ACC_SGMII_MMD_SHIFT_;
0988     mmd_access |= (addr | SGMII_ACC_SGMII_BZY_ | SGMII_ACC_SGMII_WR_);
0989     lan743x_csr_write(adapter, SGMII_ACC, mmd_access);
0990     ret = lan743x_sgmii_wait_till_not_busy(adapter);
0991     mutex_unlock(&adapter->sgmii_rw_lock);
0992
0993     return ret;
0994 }
0995
0996 static int lan743x_sgmii_mpll_set(struct lan743x_adapter *adapter,
0997                   u16 baud)
0998 {
0999     int mpllctrl0;
1000     int mpllctrl1;
1001     int miscctrl1;
1002     int ret;
1003
1004     mpllctrl0 = lan743x_sgmii_read(adapter, MDIO_MMD_VEND2,
1005                        VR_MII_GEN2_4_MPLL_CTRL0);
1006     if (mpllctrl0 < 0)
1007         return mpllctrl0;
1008
1009     mpllctrl0 &= ~VR_MII_MPLL_CTRL0_USE_REFCLK_PAD_;
1010     if (baud == VR_MII_BAUD_RATE_1P25GBPS) {
1011         mpllctrl1 = VR_MII_MPLL_MULTIPLIER_100;
1012         /* mpll_baud_clk/4 */
1013         miscctrl1 = 0xA;
1014     } else {
1015         mpllctrl1 = VR_MII_MPLL_MULTIPLIER_125;
1016         /* mpll_baud_clk/2 */
1017         miscctrl1 = 0x5;
1018     }
1019
1020     ret = lan743x_sgmii_write(adapter, MDIO_MMD_VEND2,
1021                   VR_MII_GEN2_4_MPLL_CTRL0, mpllctrl0);
1022     if (ret < 0)
1023         return ret;
1024
1025     ret = lan743x_sgmii_write(adapter, MDIO_MMD_VEND2,
1026                   VR_MII_GEN2_4_MPLL_CTRL1, mpllctrl1);
1027     if (ret < 0)
1028         return ret;
1029
1030     return lan743x_sgmii_write(adapter, MDIO_MMD_VEND2,
1031                   VR_MII_GEN2_4_MISC_CTRL1, miscctrl1);
1032 }
1033
1034 static int lan743x_sgmii_2_5G_mode_set(struct lan743x_adapter *adapter,
1035                        bool enable)
1036 {
1037     if (enable)
1038         return lan743x_sgmii_mpll_set(adapter,
1039                           VR_MII_BAUD_RATE_3P125GBPS);
1040     else
1041         return lan743x_sgmii_mpll_set(adapter,
1042                           VR_MII_BAUD_RATE_1P25GBPS);
1043 }
1044
1045 static int lan743x_is_sgmii_2_5G_mode(struct lan743x_adapter *adapter,
1046                       bool *status)
1047 {
1048     int ret;
1049
1050     ret = lan743x_sgmii_read(adapter, MDIO_MMD_VEND2,
1051                  VR_MII_GEN2_4_MPLL_CTRL1);
1052     if (ret < 0)
1053         return ret;
1054
1055     if (ret == VR_MII_MPLL_MULTIPLIER_125 ||
1056         ret == VR_MII_MPLL_MULTIPLIER_50)
1057         *status = true;
1058     else
1059         *status = false;
1060
1061     return 0;
1062 }
1063
1064 static int lan743x_sgmii_aneg_update(struct lan743x_adapter *adapter)
1065 {
1066     enum lan743x_sgmii_lsd lsd = adapter->sgmii_lsd;
1067     int mii_ctrl;
1068     int dgt_ctrl;
1069     int an_ctrl;
1070     int ret;
1071
1072     if (lsd == LINK_2500_MASTER || lsd == LINK_2500_SLAVE)
1073         /* Switch to 2.5 Gbps */
1074         ret = lan743x_sgmii_2_5G_mode_set(adapter, true);
1075     else
1076         /* Switch to 10/100/1000 Mbps clock */
1077         ret = lan743x_sgmii_2_5G_mode_set(adapter, false);
1078     if (ret < 0)
1079         return ret;
1080
1081     /* Enable SGMII Auto NEG */
1082     mii_ctrl = lan743x_sgmii_read(adapter, MDIO_MMD_VEND2, MII_BMCR);
1083     if (mii_ctrl < 0)
1084         return mii_ctrl;
1085
1086     an_ctrl = lan743x_sgmii_read(adapter, MDIO_MMD_VEND2, VR_MII_AN_CTRL);
1087     if (an_ctrl < 0)
1088         return an_ctrl;
1089
1090     dgt_ctrl = lan743x_sgmii_read(adapter, MDIO_MMD_VEND2,
1091                       VR_MII_DIG_CTRL1);
1092     if (dgt_ctrl < 0)
1093         return dgt_ctrl;
1094
1095     if (lsd == LINK_2500_MASTER || lsd == LINK_2500_SLAVE) {
1096         mii_ctrl &= ~(BMCR_ANENABLE | BMCR_ANRESTART | BMCR_SPEED100);
1097         mii_ctrl |= BMCR_SPEED1000;
1098         dgt_ctrl |= VR_MII_DIG_CTRL1_CL37_TMR_OVR_RIDE_;
1099         dgt_ctrl &= ~VR_MII_DIG_CTRL1_MAC_AUTO_SW_;
1100         /* In order for Auto-Negotiation to operate properly at
1101          * 2.5 Gbps the 1.6ms link timer values must be adjusted
1102          * The VR_MII_LINK_TIMER_CTRL Register must be set to
1103          * 16'h7A1 and The CL37_TMR_OVR_RIDE bit of the
1104          * VR_MII_DIG_CTRL1 Register set to 1
1105          */
1106         ret = lan743x_sgmii_write(adapter, MDIO_MMD_VEND2,
1107                       VR_MII_LINK_TIMER_CTRL, 0x7A1);
1108         if (ret < 0)
1109             return ret;
1110     } else {
1111         mii_ctrl |= (BMCR_ANENABLE | BMCR_ANRESTART);
1112         an_ctrl &= ~VR_MII_AN_CTRL_SGMII_LINK_STS_;
1113         dgt_ctrl &= ~VR_MII_DIG_CTRL1_CL37_TMR_OVR_RIDE_;
1114         dgt_ctrl |= VR_MII_DIG_CTRL1_MAC_AUTO_SW_;
1115     }
1116
1117     ret = lan743x_sgmii_write(adapter, MDIO_MMD_VEND2, MII_BMCR,
1118                   mii_ctrl);
1119     if (ret < 0)
1120         return ret;
1121
1122     ret = lan743x_sgmii_write(adapter, MDIO_MMD_VEND2,
1123                   VR_MII_DIG_CTRL1, dgt_ctrl);
1124     if (ret < 0)
1125         return ret;
1126
1127     return lan743x_sgmii_write(adapter, MDIO_MMD_VEND2,
1128                   VR_MII_AN_CTRL, an_ctrl);
1129 }
1130
1131 static int lan743x_pcs_seq_state(struct lan743x_adapter *adapter, u8 state)
1132 {
1133     u8 wait_cnt = 0;
1134     u32 dig_sts;
1135
1136     do {
1137         dig_sts = lan743x_sgmii_read(adapter, MDIO_MMD_VEND2,
1138                          VR_MII_DIG_STS);
1139         if (((dig_sts & VR_MII_DIG_STS_PSEQ_STATE_MASK_) >>
1140               VR_MII_DIG_STS_PSEQ_STATE_POS_) == state)
1141             break;
1142         usleep_range(1000, 2000);
1143     } while (wait_cnt++ < 10);
1144
1145     if (wait_cnt >= 10)
1146         return -ETIMEDOUT;
1147
1148     return 0;
1149 }
1150
1151 static int lan743x_sgmii_config(struct lan743x_adapter *adapter)
1152 {
1153     struct net_device *netdev = adapter->netdev;
1154     struct phy_device *phydev = netdev->phydev;
1155     enum lan743x_sgmii_lsd lsd = POWER_DOWN;
1156     int mii_ctl;
1157     bool status;
1158     int ret;
1159
1160     switch (phydev->speed) {
1161     case SPEED_2500:
1162         if (phydev->master_slave_state == MASTER_SLAVE_STATE_MASTER)
1163             lsd = LINK_2500_MASTER;
1164         else
1165             lsd = LINK_2500_SLAVE;
1166         break;
1167     case SPEED_1000:
1168         if (phydev->master_slave_state == MASTER_SLAVE_STATE_MASTER)
1169             lsd = LINK_1000_MASTER;
1170         else
1171             lsd = LINK_1000_SLAVE;
1172         break;
1173     case SPEED_100:
1174         if (phydev->duplex)
1175             lsd = LINK_100FD;
1176         else
1177             lsd = LINK_100HD;
1178         break;
1179     case SPEED_10:
1180         if (phydev->duplex)
1181             lsd = LINK_10FD;
1182         else
1183             lsd = LINK_10HD;
1184         break;
1185     default:
1186         netif_err(adapter, drv, adapter->netdev,
1187               "Invalid speed %d\n", phydev->speed);
1188         return -EINVAL;
1189     }
1190
1191     adapter->sgmii_lsd = lsd;
1192     ret = lan743x_sgmii_aneg_update(adapter);
1193     if (ret < 0) {
1194         netif_err(adapter, drv, adapter->netdev,
1195               "error %d SGMII cfg failed\n", ret);
1196         return ret;
1197     }
1198
1199     ret = lan743x_is_sgmii_2_5G_mode(adapter, &status);
1200     if (ret < 0) {
1201         netif_err(adapter, drv, adapter->netdev,
1202               "erro %d SGMII get mode failed\n", ret);
1203         return ret;
1204     }
1205
1206     if (status)
1207         netif_dbg(adapter, drv, adapter->netdev,
1208               "SGMII 2.5G mode enable\n");
1209     else
1210         netif_dbg(adapter, drv, adapter->netdev,
1211               "SGMII 1G mode enable\n");
1212
1213     /* SGMII/1000/2500BASE-X PCS power down */
1214     mii_ctl = lan743x_sgmii_read(adapter, MDIO_MMD_VEND2, MII_BMCR);
1215     if (mii_ctl < 0)
1216         return mii_ctl;
1217
1218     mii_ctl |= BMCR_PDOWN;
1219     ret = lan743x_sgmii_write(adapter, MDIO_MMD_VEND2, MII_BMCR, mii_ctl);
1220     if (ret < 0)
1221         return ret;
1222
1223     ret = lan743x_pcs_seq_state(adapter, PCS_POWER_STATE_DOWN);
1224     if (ret < 0)
1225         return ret;
1226
1227     /* SGMII/1000/2500BASE-X PCS power up */
1228     mii_ctl &= ~BMCR_PDOWN;
1229     ret = lan743x_sgmii_write(adapter, MDIO_MMD_VEND2, MII_BMCR, mii_ctl);
1230     if (ret < 0)
1231         return ret;
1232
1233     ret = lan743x_pcs_seq_state(adapter, PCS_POWER_STATE_UP);
1234     if (ret < 0)
1235         return ret;
1236
1237     return 0;
1238 }
1239
1240 static void lan743x_mac_set_address(struct lan743x_adapter *adapter,
1241                     u8 *addr)
1242 {
1243     u32 addr_lo, addr_hi;
1244
1245     addr_lo = addr[0] |
1246         addr[1] << 8 |
1247         addr[2] << 16 |
1248         addr[3] << 24;
1249     addr_hi = addr[4] |
1250         addr[5] << 8;
1251     lan743x_csr_write(adapter, MAC_RX_ADDRL, addr_lo);
1252     lan743x_csr_write(adapter, MAC_RX_ADDRH, addr_hi);
1253
1254     ether_addr_copy(adapter->mac_address, addr);
1255     netif_info(adapter, drv, adapter->netdev,
1256            "MAC address set to %pM\n", addr);
1257 }
1258
1259 static int lan743x_mac_init(struct lan743x_adapter *adapter)
1260 {
1261     bool mac_address_valid = true;
1262     struct net_device *netdev;
1263     u32 mac_addr_hi = 0;
1264     u32 mac_addr_lo = 0;
1265     u32 data;
1266
1267     netdev = adapter->netdev;
1268
1269     /* disable auto duplex, and speed detection. Phylib does that */
1270     data = lan743x_csr_read(adapter, MAC_CR);
1271     data &= ~(MAC_CR_ADD_ | MAC_CR_ASD_);
1272     data |= MAC_CR_CNTR_RST_;
1273     lan743x_csr_write(adapter, MAC_CR, data);
1274
1275     if (!is_valid_ether_addr(adapter->mac_address)) {
1276         mac_addr_hi = lan743x_csr_read(adapter, MAC_RX_ADDRH);
1277         mac_addr_lo = lan743x_csr_read(adapter, MAC_RX_ADDRL);
1278         adapter->mac_address[0] = mac_addr_lo & 0xFF;
1279         adapter->mac_address[1] = (mac_addr_lo >> 8) & 0xFF;
1280         adapter->mac_address[2] = (mac_addr_lo >> 16) & 0xFF;
1281         adapter->mac_address[3] = (mac_addr_lo >> 24) & 0xFF;
1282         adapter->mac_address[4] = mac_addr_hi & 0xFF;
1283         adapter->mac_address[5] = (mac_addr_hi >> 8) & 0xFF;
1284
1285         if (((mac_addr_hi & 0x0000FFFF) == 0x0000FFFF) &&
1286             mac_addr_lo == 0xFFFFFFFF) {
1287             mac_address_valid = false;
1288         } else if (!is_valid_ether_addr(adapter->mac_address)) {
1289             mac_address_valid = false;
1290         }
1291
1292         if (!mac_address_valid)
1293             eth_random_addr(adapter->mac_address);
1294     }
1295     lan743x_mac_set_address(adapter, adapter->mac_address);
1296     eth_hw_addr_set(netdev, adapter->mac_address);
1297
1298     return 0;
1299 }
1300
1301 static int lan743x_mac_open(struct lan743x_adapter *adapter)
1302 {
1303     u32 temp;
1304
1305     temp = lan743x_csr_read(adapter, MAC_RX);
1306     lan743x_csr_write(adapter, MAC_RX, temp | MAC_RX_RXEN_);
1307     temp = lan743x_csr_read(adapter, MAC_TX);
1308     lan743x_csr_write(adapter, MAC_TX, temp | MAC_TX_TXEN_);
1309     return 0;
1310 }
1311
1312 static void lan743x_mac_close(struct lan743x_adapter *adapter)
1313 {
1314     u32 temp;
1315
1316     temp = lan743x_csr_read(adapter, MAC_TX);
1317     temp &= ~MAC_TX_TXEN_;
1318     lan743x_csr_write(adapter, MAC_TX, temp);
1319     lan743x_csr_wait_for_bit(adapter, MAC_TX, MAC_TX_TXD_,
1320                  1, 1000, 20000, 100);
1321
1322     temp = lan743x_csr_read(adapter, MAC_RX);
1323     temp &= ~MAC_RX_RXEN_;
1324     lan743x_csr_write(adapter, MAC_RX, temp);
1325     lan743x_csr_wait_for_bit(adapter, MAC_RX, MAC_RX_RXD_,
1326                  1, 1000, 20000, 100);
1327 }
1328
1329 static void lan743x_mac_flow_ctrl_set_enables(struct lan743x_adapter *adapter,
1330                           bool tx_enable, bool rx_enable)
1331 {
1332     u32 flow_setting = 0;
1333
1334     /* set maximum pause time because when fifo space frees
1335      * up a zero value pause frame will be sent to release the pause
1336      */
1337     flow_setting = MAC_FLOW_CR_FCPT_MASK_;
1338     if (tx_enable)
1339         flow_setting |= MAC_FLOW_CR_TX_FCEN_;
1340     if (rx_enable)
1341         flow_setting |= MAC_FLOW_CR_RX_FCEN_;
1342     lan743x_csr_write(adapter, MAC_FLOW, flow_setting);
1343 }
1344
1345 static int lan743x_mac_set_mtu(struct lan743x_adapter *adapter, int new_mtu)
1346 {
1347     int enabled = 0;
1348     u32 mac_rx = 0;
1349
1350     mac_rx = lan743x_csr_read(adapter, MAC_RX);
1351     if (mac_rx & MAC_RX_RXEN_) {
1352         enabled = 1;
1353         if (mac_rx & MAC_RX_RXD_) {
1354             lan743x_csr_write(adapter, MAC_RX, mac_rx);
1355             mac_rx &= ~MAC_RX_RXD_;
1356         }
1357         mac_rx &= ~MAC_RX_RXEN_;
1358         lan743x_csr_write(adapter, MAC_RX, mac_rx);
1359         lan743x_csr_wait_for_bit(adapter, MAC_RX, MAC_RX_RXD_,
1360                      1, 1000, 20000, 100);
1361         lan743x_csr_write(adapter, MAC_RX, mac_rx | MAC_RX_RXD_);
1362     }
1363
1364     mac_rx &= ~(MAC_RX_MAX_SIZE_MASK_);
1365     mac_rx |= (((new_mtu + ETH_HLEN + ETH_FCS_LEN)
1366           << MAC_RX_MAX_SIZE_SHIFT_) & MAC_RX_MAX_SIZE_MASK_);
1367     lan743x_csr_write(adapter, MAC_RX, mac_rx);
1368
1369     if (enabled) {
1370         mac_rx |= MAC_RX_RXEN_;
1371         lan743x_csr_write(adapter, MAC_RX, mac_rx);
1372     }
1373     return 0;
1374 }
1375
1376 /* PHY */
1377 static int lan743x_phy_reset(struct lan743x_adapter *adapter)
1378 {
1379     u32 data;
1380
1381     /* Only called with in probe, and before mdiobus_register */
1382
1383     data = lan743x_csr_read(adapter, PMT_CTL);
1384     data |= PMT_CTL_ETH_PHY_RST_;
1385     lan743x_csr_write(adapter, PMT_CTL, data);
1386
1387     return readx_poll_timeout(LAN743X_CSR_READ_OP, PMT_CTL, data,
1388                   (!(data & PMT_CTL_ETH_PHY_RST_) &&
1389                   (data & PMT_CTL_READY_)),
1390                   50000, 1000000);
1391 }
1392
1393 static void lan743x_phy_update_flowcontrol(struct lan743x_adapter *adapter,
1394                        u16 local_adv, u16 remote_adv)
1395 {
1396     struct lan743x_phy *phy = &adapter->phy;
1397     u8 cap;
1398
1399     if (phy->fc_autoneg)
1400         cap = mii_resolve_flowctrl_fdx(local_adv, remote_adv);
1401     else
1402         cap = phy->fc_request_control;
1403
1404     lan743x_mac_flow_ctrl_set_enables(adapter,
1405                       cap & FLOW_CTRL_TX,
1406                       cap & FLOW_CTRL_RX);
1407 }
1408
1409 static int lan743x_phy_init(struct lan743x_adapter *adapter)
1410 {
1411     return lan743x_phy_reset(adapter);
1412 }
1413
1414 static void lan743x_phy_link_status_change(struct net_device *netdev)
1415 {
1416     struct lan743x_adapter *adapter = netdev_priv(netdev);
1417     struct phy_device *phydev = netdev->phydev;
1418     u32 data;
1419
1420     phy_print_status(phydev);
1421     if (phydev->state == PHY_RUNNING) {
1422         int remote_advertisement = 0;
1423         int local_advertisement = 0;
1424
1425         data = lan743x_csr_read(adapter, MAC_CR);
1426
1427         /* set interface mode */
1428         if (phy_interface_is_rgmii(phydev))
1429             /* RGMII */
1430             data &= ~MAC_CR_MII_EN_;
1431         else
1432             /* GMII */
1433             data |= MAC_CR_MII_EN_;
1434
1435         /* set duplex mode */
1436         if (phydev->duplex)
1437             data |= MAC_CR_DPX_;
1438         else
1439             data &= ~MAC_CR_DPX_;
1440
1441         /* set bus speed */
1442         switch (phydev->speed) {
1443         case SPEED_10:
1444             data &= ~MAC_CR_CFG_H_;
1445             data &= ~MAC_CR_CFG_L_;
1446         break;
1447         case SPEED_100:
1448             data &= ~MAC_CR_CFG_H_;
1449             data |= MAC_CR_CFG_L_;
1450         break;
1451         case SPEED_1000:
1452             data |= MAC_CR_CFG_H_;
1453             data &= ~MAC_CR_CFG_L_;
1454         break;
1455         case SPEED_2500:
1456             data |= MAC_CR_CFG_H_;
1457             data |= MAC_CR_CFG_L_;
1458         break;
1459         }
1460         lan743x_csr_write(adapter, MAC_CR, data);
1461
1462         local_advertisement =
1463             linkmode_adv_to_mii_adv_t(phydev->advertising);
1464         remote_advertisement =
1465             linkmode_adv_to_mii_adv_t(phydev->lp_advertising);
1466
1467         lan743x_phy_update_flowcontrol(adapter, local_advertisement,
1468                            remote_advertisement);
1469         lan743x_ptp_update_latency(adapter, phydev->speed);
1470         if (phydev->interface == PHY_INTERFACE_MODE_SGMII ||
1471             phydev->interface == PHY_INTERFACE_MODE_1000BASEX ||
1472             phydev->interface == PHY_INTERFACE_MODE_2500BASEX)
1473             lan743x_sgmii_config(adapter);
1474     }
1475 }
1476
1477 static void lan743x_phy_close(struct lan743x_adapter *adapter)
1478 {
1479     struct net_device *netdev = adapter->netdev;
1480
1481     phy_stop(netdev->phydev);
1482     phy_disconnect(netdev->phydev);
1483     netdev->phydev = NULL;
1484 }
1485
1486 static int lan743x_phy_open(struct lan743x_adapter *adapter)
1487 {
1488     struct net_device *netdev = adapter->netdev;
1489     struct lan743x_phy *phy = &adapter->phy;
1490     struct phy_device *phydev;
1491     int ret = -EIO;
1492
1493     /* try devicetree phy, or fixed link */
1494     phydev = of_phy_get_and_connect(netdev, adapter->pdev->dev.of_node,
1495                     lan743x_phy_link_status_change);
1496
1497     if (!phydev) {
1498         /* try internal phy */
1499         phydev = phy_find_first(adapter->mdiobus);
1500         if (!phydev)
1501             goto return_error;
1502
1503         if (adapter->is_pci11x1x)
1504             ret = phy_connect_direct(netdev, phydev,
1505                          lan743x_phy_link_status_change,
1506                          PHY_INTERFACE_MODE_RGMII);
1507         else
1508             ret = phy_connect_direct(netdev, phydev,
1509                          lan743x_phy_link_status_change,
1510                          PHY_INTERFACE_MODE_GMII);
1511         if (ret)
1512             goto return_error;
1513     }
1514
1515     /* MAC doesn't support 1000T Half */
1516     phy_remove_link_mode(phydev, ETHTOOL_LINK_MODE_1000baseT_Half_BIT);
1517
1518     /* support both flow controls */
1519     phy_support_asym_pause(phydev);
1520     phy->fc_request_control = (FLOW_CTRL_RX | FLOW_CTRL_TX);
1521     phy->fc_autoneg = phydev->autoneg;
1522
1523     phy_start(phydev);
1524     phy_start_aneg(phydev);
1525     phy_attached_info(phydev);
1526     return 0;
1527
1528 return_error:
1529     return ret;
1530 }
1531
1532 static void lan743x_rfe_open(struct lan743x_adapter *adapter)
1533 {
1534     lan743x_csr_write(adapter, RFE_RSS_CFG,
1535         RFE_RSS_CFG_UDP_IPV6_EX_ |
1536         RFE_RSS_CFG_TCP_IPV6_EX_ |
1537         RFE_RSS_CFG_IPV6_EX_ |
1538         RFE_RSS_CFG_UDP_IPV6_ |
1539         RFE_RSS_CFG_TCP_IPV6_ |
1540         RFE_RSS_CFG_IPV6_ |
1541         RFE_RSS_CFG_UDP_IPV4_ |
1542         RFE_RSS_CFG_TCP_IPV4_ |
1543         RFE_RSS_CFG_IPV4_ |
1544         RFE_RSS_CFG_VALID_HASH_BITS_ |
1545         RFE_RSS_CFG_RSS_QUEUE_ENABLE_ |
1546         RFE_RSS_CFG_RSS_HASH_STORE_ |
1547         RFE_RSS_CFG_RSS_ENABLE_);
1548 }
1549
1550 static void lan743x_rfe_update_mac_address(struct lan743x_adapter *adapter)
1551 {
1552     u8 *mac_addr;
1553     u32 mac_addr_hi = 0;
1554     u32 mac_addr_lo = 0;
1555
1556     /* Add mac address to perfect Filter */
1557     mac_addr = adapter->mac_address;
1558     mac_addr_lo = ((((u32)(mac_addr[0])) << 0) |
1559               (((u32)(mac_addr[1])) << 8) |
1560               (((u32)(mac_addr[2])) << 16) |
1561               (((u32)(mac_addr[3])) << 24));
1562     mac_addr_hi = ((((u32)(mac_addr[4])) << 0) |
1563               (((u32)(mac_addr[5])) << 8));
1564
1565     lan743x_csr_write(adapter, RFE_ADDR_FILT_LO(0), mac_addr_lo);
1566     lan743x_csr_write(adapter, RFE_ADDR_FILT_HI(0),
1567               mac_addr_hi | RFE_ADDR_FILT_HI_VALID_);
1568 }
1569
1570 static void lan743x_rfe_set_multicast(struct lan743x_adapter *adapter)
1571 {
1572     struct net_device *netdev = adapter->netdev;
1573     u32 hash_table[DP_SEL_VHF_HASH_LEN];
1574     u32 rfctl;
1575     u32 data;
1576
1577     rfctl = lan743x_csr_read(adapter, RFE_CTL);
1578     rfctl &= ~(RFE_CTL_AU_ | RFE_CTL_AM_ |
1579          RFE_CTL_DA_PERFECT_ | RFE_CTL_MCAST_HASH_);
1580     rfctl |= RFE_CTL_AB_;
1581     if (netdev->flags & IFF_PROMISC) {
1582         rfctl |= RFE_CTL_AM_ | RFE_CTL_AU_;
1583     } else {
1584         if (netdev->flags & IFF_ALLMULTI)
1585             rfctl |= RFE_CTL_AM_;
1586     }
1587
1588     memset(hash_table, 0, DP_SEL_VHF_HASH_LEN * sizeof(u32));
1589     if (netdev_mc_count(netdev)) {
1590         struct netdev_hw_addr *ha;
1591         int i;
1592
1593         rfctl |= RFE_CTL_DA_PERFECT_;
1594         i = 1;
1595         netdev_for_each_mc_addr(ha, netdev) {
1596             /* set first 32 into Perfect Filter */
1597             if (i < 33) {
1598                 lan743x_csr_write(adapter,
1599                           RFE_ADDR_FILT_HI(i), 0);
1600                 data = ha->addr[3];
1601                 data = ha->addr[2] | (data << 8);
1602                 data = ha->addr[1] | (data << 8);
1603                 data = ha->addr[0] | (data << 8);
1604                 lan743x_csr_write(adapter,
1605                           RFE_ADDR_FILT_LO(i), data);
1606                 data = ha->addr[5];
1607                 data = ha->addr[4] | (data << 8);
1608                 data |= RFE_ADDR_FILT_HI_VALID_;
1609                 lan743x_csr_write(adapter,
1610                           RFE_ADDR_FILT_HI(i), data);
1611             } else {
1612                 u32 bitnum = (ether_crc(ETH_ALEN, ha->addr) >>
1613                          23) & 0x1FF;
1614                 hash_table[bitnum / 32] |= (1 << (bitnum % 32));
1615                 rfctl |= RFE_CTL_MCAST_HASH_;
1616             }
1617             i++;
1618         }
1619     }
1620
1621     lan743x_dp_write(adapter, DP_SEL_RFE_RAM,
1622              DP_SEL_VHF_VLAN_LEN,
1623              DP_SEL_VHF_HASH_LEN, hash_table);
1624     lan743x_csr_write(adapter, RFE_CTL, rfctl);
1625 }
1626
1627 static int lan743x_dmac_init(struct lan743x_adapter *adapter)
1628 {
1629     u32 data = 0;
1630
1631     lan743x_csr_write(adapter, DMAC_CMD, DMAC_CMD_SWR_);
1632     lan743x_csr_wait_for_bit(adapter, DMAC_CMD, DMAC_CMD_SWR_,
1633                  0, 1000, 20000, 100);
1634     switch (DEFAULT_DMA_DESCRIPTOR_SPACING) {
1635     case DMA_DESCRIPTOR_SPACING_16:
1636         data = DMAC_CFG_MAX_DSPACE_16_;
1637         break;
1638     case DMA_DESCRIPTOR_SPACING_32:
1639         data = DMAC_CFG_MAX_DSPACE_32_;
1640         break;
1641     case DMA_DESCRIPTOR_SPACING_64:
1642         data = DMAC_CFG_MAX_DSPACE_64_;
1643         break;
1644     case DMA_DESCRIPTOR_SPACING_128:
1645         data = DMAC_CFG_MAX_DSPACE_128_;
1646         break;
1647     default:
1648         return -EPERM;
1649     }
1650     if (!(adapter->csr.flags & LAN743X_CSR_FLAG_IS_A0))
1651         data |= DMAC_CFG_COAL_EN_;
1652     data |= DMAC_CFG_CH_ARB_SEL_RX_HIGH_;
1653     data |= DMAC_CFG_MAX_READ_REQ_SET_(6);
1654     lan743x_csr_write(adapter, DMAC_CFG, data);
1655     data = DMAC_COAL_CFG_TIMER_LIMIT_SET_(1);
1656     data |= DMAC_COAL_CFG_TIMER_TX_START_;
1657     data |= DMAC_COAL_CFG_FLUSH_INTS_;
1658     data |= DMAC_COAL_CFG_INT_EXIT_COAL_;
1659     data |= DMAC_COAL_CFG_CSR_EXIT_COAL_;
1660     data |= DMAC_COAL_CFG_TX_THRES_SET_(0x0A);
1661     data |= DMAC_COAL_CFG_RX_THRES_SET_(0x0C);
1662     lan743x_csr_write(adapter, DMAC_COAL_CFG, data);
1663     data = DMAC_OBFF_TX_THRES_SET_(0x08);
1664     data |= DMAC_OBFF_RX_THRES_SET_(0x0A);
1665     lan743x_csr_write(adapter, DMAC_OBFF_CFG, data);
1666     return 0;
1667 }
1668
1669 static int lan743x_dmac_tx_get_state(struct lan743x_adapter *adapter,
1670                      int tx_channel)
1671 {
1672     u32 dmac_cmd = 0;
1673
1674     dmac_cmd = lan743x_csr_read(adapter, DMAC_CMD);
1675     return DMAC_CHANNEL_STATE_SET((dmac_cmd &
1676                       DMAC_CMD_START_T_(tx_channel)),
1677                       (dmac_cmd &
1678                       DMAC_CMD_STOP_T_(tx_channel)));
1679 }
1680
1681 static int lan743x_dmac_tx_wait_till_stopped(struct lan743x_adapter *adapter,
1682                          int tx_channel)
1683 {
1684     int timeout = 100;
1685     int result = 0;
1686
1687     while (timeout &&
1688            ((result = lan743x_dmac_tx_get_state(adapter, tx_channel)) ==
1689            DMAC_CHANNEL_STATE_STOP_PENDING)) {
1690         usleep_range(1000, 20000);
1691         timeout--;
1692     }
1693     if (result == DMAC_CHANNEL_STATE_STOP_PENDING)
1694         result = -ENODEV;
1695     return result;
1696 }
1697
1698 static int lan743x_dmac_rx_get_state(struct lan743x_adapter *adapter,
1699                      int rx_channel)
1700 {
1701     u32 dmac_cmd = 0;
1702
1703     dmac_cmd = lan743x_csr_read(adapter, DMAC_CMD);
1704     return DMAC_CHANNEL_STATE_SET((dmac_cmd &
1705                       DMAC_CMD_START_R_(rx_channel)),
1706                       (dmac_cmd &
1707                       DMAC_CMD_STOP_R_(rx_channel)));
1708 }
1709
1710 static int lan743x_dmac_rx_wait_till_stopped(struct lan743x_adapter *adapter,
1711                          int rx_channel)
1712 {
1713     int timeout = 100;
1714     int result = 0;
1715
1716     while (timeout &&
1717            ((result = lan743x_dmac_rx_get_state(adapter, rx_channel)) ==
1718            DMAC_CHANNEL_STATE_STOP_PENDING)) {
1719         usleep_range(1000, 20000);
1720         timeout--;
1721     }
1722     if (result == DMAC_CHANNEL_STATE_STOP_PENDING)
1723         result = -ENODEV;
1724     return result;
1725 }
1726
1727 static void lan743x_tx_release_desc(struct lan743x_tx *tx,
1728                     int descriptor_index, bool cleanup)
1729 {
1730     struct lan743x_tx_buffer_info *buffer_info = NULL;
1731     struct lan743x_tx_descriptor *descriptor = NULL;
1732     u32 descriptor_type = 0;
1733     bool ignore_sync;
1734
1735     descriptor = &tx->ring_cpu_ptr[descriptor_index];
1736     buffer_info = &tx->buffer_info[descriptor_index];
1737     if (!(buffer_info->flags & TX_BUFFER_INFO_FLAG_ACTIVE))
1738         goto done;
1739
1740     descriptor_type = le32_to_cpu(descriptor->data0) &
1741               TX_DESC_DATA0_DTYPE_MASK_;
1742     if (descriptor_type == TX_DESC_DATA0_DTYPE_DATA_)
1743         goto clean_up_data_descriptor;
1744     else
1745         goto clear_active;
1746
1747 clean_up_data_descriptor:
1748     if (buffer_info->dma_ptr) {
1749         if (buffer_info->flags &
1750             TX_BUFFER_INFO_FLAG_SKB_FRAGMENT) {
1751             dma_unmap_page(&tx->adapter->pdev->dev,
1752                        buffer_info->dma_ptr,
1753                        buffer_info->buffer_length,
1754                        DMA_TO_DEVICE);
1755         } else {
1756             dma_unmap_single(&tx->adapter->pdev->dev,
1757                      buffer_info->dma_ptr,
1758                      buffer_info->buffer_length,
1759                      DMA_TO_DEVICE);
1760         }
1761         buffer_info->dma_ptr = 0;
1762         buffer_info->buffer_length = 0;
1763     }
1764     if (!buffer_info->skb)
1765         goto clear_active;
1766
1767     if (!(buffer_info->flags & TX_BUFFER_INFO_FLAG_TIMESTAMP_REQUESTED)) {
1768         dev_kfree_skb_any(buffer_info->skb);
1769         goto clear_skb;
1770     }
1771
1772     if (cleanup) {
1773         lan743x_ptp_unrequest_tx_timestamp(tx->adapter);
1774         dev_kfree_skb_any(buffer_info->skb);
1775     } else {
1776         ignore_sync = (buffer_info->flags &
1777                    TX_BUFFER_INFO_FLAG_IGNORE_SYNC) != 0;
1778         lan743x_ptp_tx_timestamp_skb(tx->adapter,
1779                          buffer_info->skb, ignore_sync);
1780     }
1781
1782 clear_skb:
1783     buffer_info->skb = NULL;
1784
1785 clear_active:
1786     buffer_info->flags &= ~TX_BUFFER_INFO_FLAG_ACTIVE;
1787
1788 done:
1789     memset(buffer_info, 0, sizeof(*buffer_info));
1790     memset(descriptor, 0, sizeof(*descriptor));
1791 }
1792
1793 static int lan743x_tx_next_index(struct lan743x_tx *tx, int index)
1794 {
1795     return ((++index) % tx->ring_size);
1796 }
1797
1798 static void lan743x_tx_release_completed_descriptors(struct lan743x_tx *tx)
1799 {
1800     while (le32_to_cpu(*tx->head_cpu_ptr) != (tx->last_head)) {
1801         lan743x_tx_release_desc(tx, tx->last_head, false);
1802         tx->last_head = lan743x_tx_next_index(tx, tx->last_head);
1803     }
1804 }
1805
1806 static void lan743x_tx_release_all_descriptors(struct lan743x_tx *tx)
1807 {
1808     u32 original_head = 0;
1809
1810     original_head = tx->last_head;
1811     do {
1812         lan743x_tx_release_desc(tx, tx->last_head, true);
1813         tx->last_head = lan743x_tx_next_index(tx, tx->last_head);
1814     } while (tx->last_head != original_head);
1815     memset(tx->ring_cpu_ptr, 0,
1816            sizeof(*tx->ring_cpu_ptr) * (tx->ring_size));
1817     memset(tx->buffer_info, 0,
1818            sizeof(*tx->buffer_info) * (tx->ring_size));
1819 }
1820
1821 static int lan743x_tx_get_desc_cnt(struct lan743x_tx *tx,
1822                    struct sk_buff *skb)
1823 {
1824     int result = 1; /* 1 for the main skb buffer */
1825     int nr_frags = 0;
1826
1827     if (skb_is_gso(skb))
1828         result++; /* requires an extension descriptor */
1829     nr_frags = skb_shinfo(skb)->nr_frags;
1830     result += nr_frags; /* 1 for each fragment buffer */
1831     return result;
1832 }
1833
1834 static int lan743x_tx_get_avail_desc(struct lan743x_tx *tx)
1835 {
1836     int last_head = tx->last_head;
1837     int last_tail = tx->last_tail;
1838
1839     if (last_tail >= last_head)
1840         return tx->ring_size - last_tail + last_head - 1;
1841     else
1842         return last_head - last_tail - 1;
1843 }
1844
1845 void lan743x_tx_set_timestamping_mode(struct lan743x_tx *tx,
1846                       bool enable_timestamping,
1847                       bool enable_onestep_sync)
1848 {
1849     if (enable_timestamping)
1850         tx->ts_flags |= TX_TS_FLAG_TIMESTAMPING_ENABLED;
1851     else
1852         tx->ts_flags &= ~TX_TS_FLAG_TIMESTAMPING_ENABLED;
1853     if (enable_onestep_sync)
1854         tx->ts_flags |= TX_TS_FLAG_ONE_STEP_SYNC;
1855     else
1856         tx->ts_flags &= ~TX_TS_FLAG_ONE_STEP_SYNC;
1857 }
1858
1859 static int lan743x_tx_frame_start(struct lan743x_tx *tx,
1860                   unsigned char *first_buffer,
1861                   unsigned int first_buffer_length,
1862                   unsigned int frame_length,
1863                   bool time_stamp,
1864                   bool check_sum)
1865 {
1866     /* called only from within lan743x_tx_xmit_frame.
1867      * assuming tx->ring_lock has already been acquired.
1868      */
1869     struct lan743x_tx_descriptor *tx_descriptor = NULL;
1870     struct lan743x_tx_buffer_info *buffer_info = NULL;
1871     struct lan743x_adapter *adapter = tx->adapter;
1872     struct device *dev = &adapter->pdev->dev;
1873     dma_addr_t dma_ptr;
1874
1875     tx->frame_flags |= TX_FRAME_FLAG_IN_PROGRESS;
1876     tx->frame_first = tx->last_tail;
1877     tx->frame_tail = tx->frame_first;
1878
1879     tx_descriptor = &tx->ring_cpu_ptr[tx->frame_tail];
1880     buffer_info = &tx->buffer_info[tx->frame_tail];
1881     dma_ptr = dma_map_single(dev, first_buffer, first_buffer_length,
1882                  DMA_TO_DEVICE);
1883     if (dma_mapping_error(dev, dma_ptr))
1884         return -ENOMEM;
1885
1886     tx_descriptor->data1 = cpu_to_le32(DMA_ADDR_LOW32(dma_ptr));
1887     tx_descriptor->data2 = cpu_to_le32(DMA_ADDR_HIGH32(dma_ptr));
1888     tx_descriptor->data3 = cpu_to_le32((frame_length << 16) &
1889         TX_DESC_DATA3_FRAME_LENGTH_MSS_MASK_);
1890
1891     buffer_info->skb = NULL;
1892     buffer_info->dma_ptr = dma_ptr;
1893     buffer_info->buffer_length = first_buffer_length;
1894     buffer_info->flags |= TX_BUFFER_INFO_FLAG_ACTIVE;
1895
1896     tx->frame_data0 = (first_buffer_length &
1897         TX_DESC_DATA0_BUF_LENGTH_MASK_) |
1898         TX_DESC_DATA0_DTYPE_DATA_ |
1899         TX_DESC_DATA0_FS_ |
1900         TX_DESC_DATA0_FCS_;
1901     if (time_stamp)
1902         tx->frame_data0 |= TX_DESC_DATA0_TSE_;
1903
1904     if (check_sum)
1905         tx->frame_data0 |= TX_DESC_DATA0_ICE_ |
1906                    TX_DESC_DATA0_IPE_ |
1907                    TX_DESC_DATA0_TPE_;
1908
1909     /* data0 will be programmed in one of other frame assembler functions */
1910     return 0;
1911 }
1912
1913 static void lan743x_tx_frame_add_lso(struct lan743x_tx *tx,
1914                      unsigned int frame_length,
1915                      int nr_frags)
1916 {
1917     /* called only from within lan743x_tx_xmit_frame.
1918      * assuming tx->ring_lock has already been acquired.
1919      */
1920     struct lan743x_tx_descriptor *tx_descriptor = NULL;
1921     struct lan743x_tx_buffer_info *buffer_info = NULL;
1922
1923     /* wrap up previous descriptor */
1924     tx->frame_data0 |= TX_DESC_DATA0_EXT_;
1925     if (nr_frags <= 0) {
1926         tx->frame_data0 |= TX_DESC_DATA0_LS_;
1927         tx->frame_data0 |= TX_DESC_DATA0_IOC_;
1928     }
1929     tx_descriptor = &tx->ring_cpu_ptr[tx->frame_tail];
1930     tx_descriptor->data0 = cpu_to_le32(tx->frame_data0);
1931
1932     /* move to next descriptor */
1933     tx->frame_tail = lan743x_tx_next_index(tx, tx->frame_tail);
1934     tx_descriptor = &tx->ring_cpu_ptr[tx->frame_tail];
1935     buffer_info = &tx->buffer_info[tx->frame_tail];
1936
1937     /* add extension descriptor */
1938     tx_descriptor->data1 = 0;
1939     tx_descriptor->data2 = 0;
1940     tx_descriptor->data3 = 0;
1941
1942     buffer_info->skb = NULL;
1943     buffer_info->dma_ptr = 0;
1944     buffer_info->buffer_length = 0;
1945     buffer_info->flags |= TX_BUFFER_INFO_FLAG_ACTIVE;
1946
1947     tx->frame_data0 = (frame_length & TX_DESC_DATA0_EXT_PAY_LENGTH_MASK_) |
1948               TX_DESC_DATA0_DTYPE_EXT_ |
1949               TX_DESC_DATA0_EXT_LSO_;
1950
1951     /* data0 will be programmed in one of other frame assembler functions */
1952 }
1953
1954 static int lan743x_tx_frame_add_fragment(struct lan743x_tx *tx,
1955                      const skb_frag_t *fragment,
1956                      unsigned int frame_length)
1957 {
1958     /* called only from within lan743x_tx_xmit_frame
1959      * assuming tx->ring_lock has already been acquired
1960      */
1961     struct lan743x_tx_descriptor *tx_descriptor = NULL;
1962     struct lan743x_tx_buffer_info *buffer_info = NULL;
1963     struct lan743x_adapter *adapter = tx->adapter;
1964     struct device *dev = &adapter->pdev->dev;
1965     unsigned int fragment_length = 0;
1966     dma_addr_t dma_ptr;
1967
1968     fragment_length = skb_frag_size(fragment);
1969     if (!fragment_length)
1970         return 0;
1971
1972     /* wrap up previous descriptor */
1973     tx_descriptor = &tx->ring_cpu_ptr[tx->frame_tail];
1974     tx_descriptor->data0 = cpu_to_le32(tx->frame_data0);
1975
1976     /* move to next descriptor */
1977     tx->frame_tail = lan743x_tx_next_index(tx, tx->frame_tail);
1978     tx_descriptor = &tx->ring_cpu_ptr[tx->frame_tail];
1979     buffer_info = &tx->buffer_info[tx->frame_tail];
1980     dma_ptr = skb_frag_dma_map(dev, fragment,
1981                    0, fragment_length,
1982                    DMA_TO_DEVICE);
1983     if (dma_mapping_error(dev, dma_ptr)) {
1984         int desc_index;
1985
1986         /* cleanup all previously setup descriptors */
1987         desc_index = tx->frame_first;
1988         while (desc_index != tx->frame_tail) {
1989             lan743x_tx_release_desc(tx, desc_index, true);
1990             desc_index = lan743x_tx_next_index(tx, desc_index);
1991         }
1992         dma_wmb();
1993         tx->frame_flags &= ~TX_FRAME_FLAG_IN_PROGRESS;
1994         tx->frame_first = 0;
1995         tx->frame_data0 = 0;
1996         tx->frame_tail = 0;
1997         return -ENOMEM;
1998     }
1999
2000     tx_descriptor->data1 = cpu_to_le32(DMA_ADDR_LOW32(dma_ptr));
2001     tx_descriptor->data2 = cpu_to_le32(DMA_ADDR_HIGH32(dma_ptr));
2002     tx_descriptor->data3 = cpu_to_le32((frame_length << 16) &
2003                    TX_DESC_DATA3_FRAME_LENGTH_MSS_MASK_);
2004
2005     buffer_info->skb = NULL;
2006     buffer_info->dma_ptr = dma_ptr;
2007     buffer_info->buffer_length = fragment_length;
2008     buffer_info->flags |= TX_BUFFER_INFO_FLAG_ACTIVE;
2009     buffer_info->flags |= TX_BUFFER_INFO_FLAG_SKB_FRAGMENT;
2010
2011     tx->frame_data0 = (fragment_length & TX_DESC_DATA0_BUF_LENGTH_MASK_) |
2012               TX_DESC_DATA0_DTYPE_DATA_ |
2013               TX_DESC_DATA0_FCS_;
2014
2015     /* data0 will be programmed in one of other frame assembler functions */
2016     return 0;
2017 }
2018
2019 static void lan743x_tx_frame_end(struct lan743x_tx *tx,
2020                  struct sk_buff *skb,
2021                  bool time_stamp,
2022                  bool ignore_sync)
2023 {
2024     /* called only from within lan743x_tx_xmit_frame
2025      * assuming tx->ring_lock has already been acquired
2026      */
2027     struct lan743x_tx_descriptor *tx_descriptor = NULL;
2028     struct lan743x_tx_buffer_info *buffer_info = NULL;
2029     struct lan743x_adapter *adapter = tx->adapter;
2030     u32 tx_tail_flags = 0;
2031
2032     /* wrap up previous descriptor */
2033     if ((tx->frame_data0 & TX_DESC_DATA0_DTYPE_MASK_) ==
2034         TX_DESC_DATA0_DTYPE_DATA_) {
2035         tx->frame_data0 |= TX_DESC_DATA0_LS_;
2036         tx->frame_data0 |= TX_DESC_DATA0_IOC_;
2037     }
2038
2039     tx_descriptor = &tx->ring_cpu_ptr[tx->frame_tail];
2040     buffer_info = &tx->buffer_info[tx->frame_tail];
2041     buffer_info->skb = skb;
2042     if (time_stamp)
2043         buffer_info->flags |= TX_BUFFER_INFO_FLAG_TIMESTAMP_REQUESTED;
2044     if (ignore_sync)
2045         buffer_info->flags |= TX_BUFFER_INFO_FLAG_IGNORE_SYNC;
2046
2047     tx_descriptor->data0 = cpu_to_le32(tx->frame_data0);
2048     tx->frame_tail = lan743x_tx_next_index(tx, tx->frame_tail);
2049     tx->last_tail = tx->frame_tail;
2050
2051     dma_wmb();
2052
2053     if (tx->vector_flags & LAN743X_VECTOR_FLAG_VECTOR_ENABLE_AUTO_SET)
2054         tx_tail_flags |= TX_TAIL_SET_TOP_INT_VEC_EN_;
2055     if (tx->vector_flags & LAN743X_VECTOR_FLAG_SOURCE_ENABLE_AUTO_SET)
2056         tx_tail_flags |= TX_TAIL_SET_DMAC_INT_EN_ |
2057         TX_TAIL_SET_TOP_INT_EN_;
2058
2059     lan743x_csr_write(adapter, TX_TAIL(tx->channel_number),
2060               tx_tail_flags | tx->frame_tail);
2061     tx->frame_flags &= ~TX_FRAME_FLAG_IN_PROGRESS;
2062 }
2063
2064 static netdev_tx_t lan743x_tx_xmit_frame(struct lan743x_tx *tx,
2065                      struct sk_buff *skb)
2066 {
2067     int required_number_of_descriptors = 0;
2068     unsigned int start_frame_length = 0;
2069     unsigned int frame_length = 0;
2070     unsigned int head_length = 0;
2071     unsigned long irq_flags = 0;
2072     bool do_timestamp = false;
2073     bool ignore_sync = false;
2074     int nr_frags = 0;
2075     bool gso = false;
2076     int j;
2077
2078     required_number_of_descriptors = lan743x_tx_get_desc_cnt(tx, skb);
2079
2080     spin_lock_irqsave(&tx->ring_lock, irq_flags);
2081     if (required_number_of_descriptors >
2082         lan743x_tx_get_avail_desc(tx)) {
2083         if (required_number_of_descriptors > (tx->ring_size - 1)) {
2084             dev_kfree_skb_irq(skb);
2085         } else {
2086             /* save to overflow buffer */
2087             tx->overflow_skb = skb;
2088             netif_stop_queue(tx->adapter->netdev);
2089         }
2090         goto unlock;
2091     }
2092
2093     /* space available, transmit skb  */
2094     if ((skb_shinfo(skb)->tx_flags & SKBTX_HW_TSTAMP) &&
2095         (tx->ts_flags & TX_TS_FLAG_TIMESTAMPING_ENABLED) &&
2096         (lan743x_ptp_request_tx_timestamp(tx->adapter))) {
2097         skb_shinfo(skb)->tx_flags |= SKBTX_IN_PROGRESS;
2098         do_timestamp = true;
2099         if (tx->ts_flags & TX_TS_FLAG_ONE_STEP_SYNC)
2100             ignore_sync = true;
2101     }
2102     head_length = skb_headlen(skb);
2103     frame_length = skb_pagelen(skb);
2104     nr_frags = skb_shinfo(skb)->nr_frags;
2105     start_frame_length = frame_length;
2106     gso = skb_is_gso(skb);
2107     if (gso) {
2108         start_frame_length = max(skb_shinfo(skb)->gso_size,
2109                      (unsigned short)8);
2110     }
2111
2112     if (lan743x_tx_frame_start(tx,
2113                    skb->data, head_length,
2114                    start_frame_length,
2115                    do_timestamp,
2116                    skb->ip_summed == CHECKSUM_PARTIAL)) {
2117         dev_kfree_skb_irq(skb);
2118         goto unlock;
2119     }
2120     tx->frame_count++;
2121
2122     if (gso)
2123         lan743x_tx_frame_add_lso(tx, frame_length, nr_frags);
2124
2125     if (nr_frags <= 0)
2126         goto finish;
2127
2128     for (j = 0; j < nr_frags; j++) {
2129         const skb_frag_t *frag = &(skb_shinfo(skb)->frags[j]);
2130
2131         if (lan743x_tx_frame_add_fragment(tx, frag, frame_length)) {
2132             /* upon error no need to call
2133              *  lan743x_tx_frame_end
2134              * frame assembler clean up was performed inside
2135              *  lan743x_tx_frame_add_fragment
2136              */
2137             dev_kfree_skb_irq(skb);
2138             goto unlock;
2139         }
2140     }
2141
2142 finish:
2143     lan743x_tx_frame_end(tx, skb, do_timestamp, ignore_sync);
2144
2145 unlock:
2146     spin_unlock_irqrestore(&tx->ring_lock, irq_flags);
2147     return NETDEV_TX_OK;
2148 }
2149
2150 static int lan743x_tx_napi_poll(struct napi_struct *napi, int weight)
2151 {
2152     struct lan743x_tx *tx = container_of(napi, struct lan743x_tx, napi);
2153     struct lan743x_adapter *adapter = tx->adapter;
2154     bool start_transmitter = false;
2155     unsigned long irq_flags = 0;
2156     u32 ioc_bit = 0;
2157
2158     ioc_bit = DMAC_INT_BIT_TX_IOC_(tx->channel_number);
2159     lan743x_csr_read(adapter, DMAC_INT_STS);
2160     if (tx->vector_flags & LAN743X_VECTOR_FLAG_SOURCE_STATUS_W2C)
2161         lan743x_csr_write(adapter, DMAC_INT_STS, ioc_bit);
2162     spin_lock_irqsave(&tx->ring_lock, irq_flags);
2163
2164     /* clean up tx ring */
2165     lan743x_tx_release_completed_descriptors(tx);
2166     if (netif_queue_stopped(adapter->netdev)) {
2167         if (tx->overflow_skb) {
2168             if (lan743x_tx_get_desc_cnt(tx, tx->overflow_skb) <=
2169                 lan743x_tx_get_avail_desc(tx))
2170                 start_transmitter = true;
2171         } else {
2172             netif_wake_queue(adapter->netdev);
2173         }
2174     }
2175     spin_unlock_irqrestore(&tx->ring_lock, irq_flags);
2176
2177     if (start_transmitter) {
2178         /* space is now available, transmit overflow skb */
2179         lan743x_tx_xmit_frame(tx, tx->overflow_skb);
2180         tx->overflow_skb = NULL;
2181         netif_wake_queue(adapter->netdev);
2182     }
2183
2184     if (!napi_complete(napi))
2185         goto done;
2186
2187     /* enable isr */
2188     lan743x_csr_write(adapter, INT_EN_SET,
2189               INT_BIT_DMA_TX_(tx->channel_number));
2190     lan743x_csr_read(adapter, INT_STS);
2191
2192 done:
2193     return 0;
2194 }
2195
2196 static void lan743x_tx_ring_cleanup(struct lan743x_tx *tx)
2197 {
2198     if (tx->head_cpu_ptr) {
2199         dma_free_coherent(&tx->adapter->pdev->dev,
2200                   sizeof(*tx->head_cpu_ptr), tx->head_cpu_ptr,
2201                   tx->head_dma_ptr);
2202         tx->head_cpu_ptr = NULL;
2203         tx->head_dma_ptr = 0;
2204     }
2205     kfree(tx->buffer_info);
2206     tx->buffer_info = NULL;
2207
2208     if (tx->ring_cpu_ptr) {
2209         dma_free_coherent(&tx->adapter->pdev->dev,
2210                   tx->ring_allocation_size, tx->ring_cpu_ptr,
2211                   tx->ring_dma_ptr);
2212         tx->ring_allocation_size = 0;
2213         tx->ring_cpu_ptr = NULL;
2214         tx->ring_dma_ptr = 0;
2215     }
2216     tx->ring_size = 0;
2217 }
2218
2219 static int lan743x_tx_ring_init(struct lan743x_tx *tx)
2220 {
2221     size_t ring_allocation_size = 0;
2222     void *cpu_ptr = NULL;
2223     dma_addr_t dma_ptr;
2224     int ret = -ENOMEM;
2225
2226     tx->ring_size = LAN743X_TX_RING_SIZE;
2227     if (tx->ring_size & ~TX_CFG_B_TX_RING_LEN_MASK_) {
2228         ret = -EINVAL;
2229         goto cleanup;
2230     }
2231     if (dma_set_mask_and_coherent(&tx->adapter->pdev->dev,
2232                       DMA_BIT_MASK(64))) {
2233         dev_warn(&tx->adapter->pdev->dev,
2234              "lan743x_: No suitable DMA available\n");
2235         ret = -ENOMEM;
2236         goto cleanup;
2237     }
2238     ring_allocation_size = ALIGN(tx->ring_size *
2239                      sizeof(struct lan743x_tx_descriptor),
2240                      PAGE_SIZE);
2241     dma_ptr = 0;
2242     cpu_ptr = dma_alloc_coherent(&tx->adapter->pdev->dev,
2243                      ring_allocation_size, &dma_ptr, GFP_KERNEL);
2244     if (!cpu_ptr) {
2245         ret = -ENOMEM;
2246         goto cleanup;
2247     }
2248
2249     tx->ring_allocation_size = ring_allocation_size;
2250     tx->ring_cpu_ptr = (struct lan743x_tx_descriptor *)cpu_ptr;
2251     tx->ring_dma_ptr = dma_ptr;
2252
2253     cpu_ptr = kcalloc(tx->ring_size, sizeof(*tx->buffer_info), GFP_KERNEL);
2254     if (!cpu_ptr) {
2255         ret = -ENOMEM;
2256         goto cleanup;
2257     }
2258     tx->buffer_info = (struct lan743x_tx_buffer_info *)cpu_ptr;
2259     dma_ptr = 0;
2260     cpu_ptr = dma_alloc_coherent(&tx->adapter->pdev->dev,
2261                      sizeof(*tx->head_cpu_ptr), &dma_ptr,
2262                      GFP_KERNEL);
2263     if (!cpu_ptr) {
2264         ret = -ENOMEM;
2265         goto cleanup;
2266     }
2267
2268     tx->head_cpu_ptr = cpu_ptr;
2269     tx->head_dma_ptr = dma_ptr;
2270     if (tx->head_dma_ptr & 0x3) {
2271         ret = -ENOMEM;
2272         goto cleanup;
2273     }
2274
2275     return 0;
2276
2277 cleanup:
2278     lan743x_tx_ring_cleanup(tx);
2279     return ret;
2280 }
2281
2282 static void lan743x_tx_close(struct lan743x_tx *tx)
2283 {
2284     struct lan743x_adapter *adapter = tx->adapter;
2285
2286     lan743x_csr_write(adapter,
2287               DMAC_CMD,
2288               DMAC_CMD_STOP_T_(tx->channel_number));
2289     lan743x_dmac_tx_wait_till_stopped(adapter, tx->channel_number);
2290
2291     lan743x_csr_write(adapter,
2292               DMAC_INT_EN_CLR,
2293               DMAC_INT_BIT_TX_IOC_(tx->channel_number));
2294     lan743x_csr_write(adapter, INT_EN_CLR,
2295               INT_BIT_DMA_TX_(tx->channel_number));
2296     napi_disable(&tx->napi);
2297     netif_napi_del(&tx->napi);
2298
2299     lan743x_csr_write(adapter, FCT_TX_CTL,
2300               FCT_TX_CTL_DIS_(tx->channel_number));
2301     lan743x_csr_wait_for_bit(adapter, FCT_TX_CTL,
2302                  FCT_TX_CTL_EN_(tx->channel_number),
2303                  0, 1000, 20000, 100);
2304
2305     lan743x_tx_release_all_descriptors(tx);
2306
2307     if (tx->overflow_skb) {
2308         dev_kfree_skb(tx->overflow_skb);
2309         tx->overflow_skb = NULL;
2310     }
2311
2312     lan743x_tx_ring_cleanup(tx);
2313 }
2314
2315 static int lan743x_tx_open(struct lan743x_tx *tx)
2316 {
2317     struct lan743x_adapter *adapter = NULL;
2318     u32 data = 0;
2319     int ret;
2320
2321     adapter = tx->adapter;
2322     ret = lan743x_tx_ring_init(tx);
2323     if (ret)
2324         return ret;
2325
2326     /* initialize fifo */
2327     lan743x_csr_write(adapter, FCT_TX_CTL,
2328               FCT_TX_CTL_RESET_(tx->channel_number));
2329     lan743x_csr_wait_for_bit(adapter, FCT_TX_CTL,
2330                  FCT_TX_CTL_RESET_(tx->channel_number),
2331                  0, 1000, 20000, 100);
2332
2333     /* enable fifo */
2334     lan743x_csr_write(adapter, FCT_TX_CTL,
2335               FCT_TX_CTL_EN_(tx->channel_number));
2336
2337     /* reset tx channel */
2338     lan743x_csr_write(adapter, DMAC_CMD,
2339               DMAC_CMD_TX_SWR_(tx->channel_number));
2340     lan743x_csr_wait_for_bit(adapter, DMAC_CMD,
2341                  DMAC_CMD_TX_SWR_(tx->channel_number),
2342                  0, 1000, 20000, 100);
2343
2344     /* Write TX_BASE_ADDR */
2345     lan743x_csr_write(adapter,
2346               TX_BASE_ADDRH(tx->channel_number),
2347               DMA_ADDR_HIGH32(tx->ring_dma_ptr));
2348     lan743x_csr_write(adapter,
2349               TX_BASE_ADDRL(tx->channel_number),
2350               DMA_ADDR_LOW32(tx->ring_dma_ptr));
2351
2352     /* Write TX_CFG_B */
2353     data = lan743x_csr_read(adapter, TX_CFG_B(tx->channel_number));
2354     data &= ~TX_CFG_B_TX_RING_LEN_MASK_;
2355     data |= ((tx->ring_size) & TX_CFG_B_TX_RING_LEN_MASK_);
2356     if (!(adapter->csr.flags & LAN743X_CSR_FLAG_IS_A0))
2357         data |= TX_CFG_B_TDMABL_512_;
2358     lan743x_csr_write(adapter, TX_CFG_B(tx->channel_number), data);
2359
2360     /* Write TX_CFG_A */
2361     data = TX_CFG_A_TX_TMR_HPWB_SEL_IOC_ | TX_CFG_A_TX_HP_WB_EN_;
2362     if (!(adapter->csr.flags & LAN743X_CSR_FLAG_IS_A0)) {
2363         data |= TX_CFG_A_TX_HP_WB_ON_INT_TMR_;
2364         data |= TX_CFG_A_TX_PF_THRES_SET_(0x10);
2365         data |= TX_CFG_A_TX_PF_PRI_THRES_SET_(0x04);
2366         data |= TX_CFG_A_TX_HP_WB_THRES_SET_(0x07);
2367     }
2368     lan743x_csr_write(adapter, TX_CFG_A(tx->channel_number), data);
2369
2370     /* Write TX_HEAD_WRITEBACK_ADDR */
2371     lan743x_csr_write(adapter,
2372               TX_HEAD_WRITEBACK_ADDRH(tx->channel_number),
2373               DMA_ADDR_HIGH32(tx->head_dma_ptr));
2374     lan743x_csr_write(adapter,
2375               TX_HEAD_WRITEBACK_ADDRL(tx->channel_number),
2376               DMA_ADDR_LOW32(tx->head_dma_ptr));
2377
2378     /* set last head */
2379     tx->last_head = lan743x_csr_read(adapter, TX_HEAD(tx->channel_number));
2380
2381     /* write TX_TAIL */
2382     tx->last_tail = 0;
2383     lan743x_csr_write(adapter, TX_TAIL(tx->channel_number),
2384               (u32)(tx->last_tail));
2385     tx->vector_flags = lan743x_intr_get_vector_flags(adapter,
2386                              INT_BIT_DMA_TX_
2387                              (tx->channel_number));
2388     netif_napi_add_tx_weight(adapter->netdev,
2389                  &tx->napi, lan743x_tx_napi_poll,
2390                  tx->ring_size - 1);
2391     napi_enable(&tx->napi);
2392
2393     data = 0;
2394     if (tx->vector_flags & LAN743X_VECTOR_FLAG_SOURCE_ENABLE_AUTO_CLEAR)
2395         data |= TX_CFG_C_TX_TOP_INT_EN_AUTO_CLR_;
2396     if (tx->vector_flags & LAN743X_VECTOR_FLAG_SOURCE_STATUS_AUTO_CLEAR)
2397         data |= TX_CFG_C_TX_DMA_INT_STS_AUTO_CLR_;
2398     if (tx->vector_flags & LAN743X_VECTOR_FLAG_SOURCE_STATUS_R2C)
2399         data |= TX_CFG_C_TX_INT_STS_R2C_MODE_MASK_;
2400     if (tx->vector_flags & LAN743X_VECTOR_FLAG_SOURCE_ENABLE_R2C)
2401         data |= TX_CFG_C_TX_INT_EN_R2C_;
2402     lan743x_csr_write(adapter, TX_CFG_C(tx->channel_number), data);
2403
2404     if (!(tx->vector_flags & LAN743X_VECTOR_FLAG_SOURCE_ENABLE_AUTO_SET))
2405         lan743x_csr_write(adapter, INT_EN_SET,
2406                   INT_BIT_DMA_TX_(tx->channel_number));
2407     lan743x_csr_write(adapter, DMAC_INT_EN_SET,
2408               DMAC_INT_BIT_TX_IOC_(tx->channel_number));
2409
2410     /*  start dmac channel */
2411     lan743x_csr_write(adapter, DMAC_CMD,
2412               DMAC_CMD_START_T_(tx->channel_number));
2413     return 0;
2414 }
2415
2416 static int lan743x_rx_next_index(struct lan743x_rx *rx, int index)
2417 {
2418     return ((++index) % rx->ring_size);
2419 }
2420
2421 static void lan743x_rx_update_tail(struct lan743x_rx *rx, int index)
2422 {
2423     /* update the tail once per 8 descriptors */
2424     if ((index & 7) == 7)
2425         lan743x_csr_write(rx->adapter, RX_TAIL(rx->channel_number),
2426                   index);
2427 }
2428
2429 static int lan743x_rx_init_ring_element(struct lan743x_rx *rx, int index,
2430                     gfp_t gfp)
2431 {
2432     struct net_device *netdev = rx->adapter->netdev;
2433     struct device *dev = &rx->adapter->pdev->dev;
2434     struct lan743x_rx_buffer_info *buffer_info;
2435     unsigned int buffer_length, used_length;
2436     struct lan743x_rx_descriptor *descriptor;
2437     struct sk_buff *skb;
2438     dma_addr_t dma_ptr;
2439
2440     buffer_length = netdev->mtu + ETH_HLEN + ETH_FCS_LEN + RX_HEAD_PADDING;
2441
2442     descriptor = &rx->ring_cpu_ptr[index];
2443     buffer_info = &rx->buffer_info[index];
2444     skb = __netdev_alloc_skb(netdev, buffer_length, gfp);
2445     if (!skb)
2446         return -ENOMEM;
2447     dma_ptr = dma_map_single(dev, skb->data, buffer_length, DMA_FROM_DEVICE);
2448     if (dma_mapping_error(dev, dma_ptr)) {
2449         dev_kfree_skb_any(skb);
2450         return -ENOMEM;
2451     }
2452     if (buffer_info->dma_ptr) {
2453         /* sync used area of buffer only */
2454         if (le32_to_cpu(descriptor->data0) & RX_DESC_DATA0_LS_)
2455             /* frame length is valid only if LS bit is set.
2456              * it's a safe upper bound for the used area in this
2457              * buffer.
2458              */
2459             used_length = min(RX_DESC_DATA0_FRAME_LENGTH_GET_
2460                       (le32_to_cpu(descriptor->data0)),
2461                       buffer_info->buffer_length);
2462         else
2463             used_length = buffer_info->buffer_length;
2464         dma_sync_single_for_cpu(dev, buffer_info->dma_ptr,
2465                     used_length,
2466                     DMA_FROM_DEVICE);
2467         dma_unmap_single_attrs(dev, buffer_info->dma_ptr,
2468                        buffer_info->buffer_length,
2469                        DMA_FROM_DEVICE,
2470                        DMA_ATTR_SKIP_CPU_SYNC);
2471     }
2472
2473     buffer_info->skb = skb;
2474     buffer_info->dma_ptr = dma_ptr;
2475     buffer_info->buffer_length = buffer_length;
2476     descriptor->data1 = cpu_to_le32(DMA_ADDR_LOW32(buffer_info->dma_ptr));
2477     descriptor->data2 = cpu_to_le32(DMA_ADDR_HIGH32(buffer_info->dma_ptr));
2478     descriptor->data3 = 0;
2479     descriptor->data0 = cpu_to_le32((RX_DESC_DATA0_OWN_ |
2480                 (buffer_length & RX_DESC_DATA0_BUF_LENGTH_MASK_)));
2481     lan743x_rx_update_tail(rx, index);
2482
2483     return 0;
2484 }
2485
2486 static void lan743x_rx_reuse_ring_element(struct lan743x_rx *rx, int index)
2487 {
2488     struct lan743x_rx_buffer_info *buffer_info;
2489     struct lan743x_rx_descriptor *descriptor;
2490
2491     descriptor = &rx->ring_cpu_ptr[index];
2492     buffer_info = &rx->buffer_info[index];
2493
2494     descriptor->data1 = cpu_to_le32(DMA_ADDR_LOW32(buffer_info->dma_ptr));
2495     descriptor->data2 = cpu_to_le32(DMA_ADDR_HIGH32(buffer_info->dma_ptr));
2496     descriptor->data3 = 0;
2497     descriptor->data0 = cpu_to_le32((RX_DESC_DATA0_OWN_ |
2498                 ((buffer_info->buffer_length) &
2499                 RX_DESC_DATA0_BUF_LENGTH_MASK_)));
2500     lan743x_rx_update_tail(rx, index);
2501 }
2502
2503 static void lan743x_rx_release_ring_element(struct lan743x_rx *rx, int index)
2504 {
2505     struct lan743x_rx_buffer_info *buffer_info;
2506     struct lan743x_rx_descriptor *descriptor;
2507
2508     descriptor = &rx->ring_cpu_ptr[index];
2509     buffer_info = &rx->buffer_info[index];
2510
2511     memset(descriptor, 0, sizeof(*descriptor));
2512
2513     if (buffer_info->dma_ptr) {
2514         dma_unmap_single(&rx->adapter->pdev->dev,
2515                  buffer_info->dma_ptr,
2516                  buffer_info->buffer_length,
2517                  DMA_FROM_DEVICE);
2518         buffer_info->dma_ptr = 0;
2519     }
2520
2521     if (buffer_info->skb) {
2522         dev_kfree_skb(buffer_info->skb);
2523         buffer_info->skb = NULL;
2524     }
2525
2526     memset(buffer_info, 0, sizeof(*buffer_info));
2527 }
2528
2529 static struct sk_buff *
2530 lan743x_rx_trim_skb(struct sk_buff *skb, int frame_length)
2531 {
2532     if (skb_linearize(skb)) {
2533         dev_kfree_skb_irq(skb);
2534         return NULL;
2535     }
2536     frame_length = max_t(int, 0, frame_length - ETH_FCS_LEN);
2537     if (skb->len > frame_length) {
2538         skb->tail -= skb->len - frame_length;
2539         skb->len = frame_length;
2540     }
2541     return skb;
2542 }
2543
2544 static int lan743x_rx_process_buffer(struct lan743x_rx *rx)
2545 {
2546     int current_head_index = le32_to_cpu(*rx->head_cpu_ptr);
2547     struct lan743x_rx_descriptor *descriptor, *desc_ext;
2548     struct net_device *netdev = rx->adapter->netdev;
2549     int result = RX_PROCESS_RESULT_NOTHING_TO_DO;
2550     struct lan743x_rx_buffer_info *buffer_info;
2551     int frame_length, buffer_length;
2552     int extension_index = -1;
2553     bool is_last, is_first;
2554     struct sk_buff *skb;
2555
2556     if (current_head_index < 0 || current_head_index >= rx->ring_size)
2557         goto done;
2558
2559     if (rx->last_head < 0 || rx->last_head >= rx->ring_size)
2560         goto done;
2561
2562     if (rx->last_head == current_head_index)
2563         goto done;
2564
2565     descriptor = &rx->ring_cpu_ptr[rx->last_head];
2566     if (le32_to_cpu(descriptor->data0) & RX_DESC_DATA0_OWN_)
2567         goto done;
2568     buffer_info = &rx->buffer_info[rx->last_head];
2569
2570     is_last = le32_to_cpu(descriptor->data0) & RX_DESC_DATA0_LS_;
2571     is_first = le32_to_cpu(descriptor->data0) & RX_DESC_DATA0_FS_;
2572
2573     if (is_last && le32_to_cpu(descriptor->data0) & RX_DESC_DATA0_EXT_) {
2574         /* extension is expected to follow */
2575         int index = lan743x_rx_next_index(rx, rx->last_head);
2576
2577         if (index == current_head_index)
2578             /* extension not yet available */
2579             goto done;
2580         desc_ext = &rx->ring_cpu_ptr[index];
2581         if (le32_to_cpu(desc_ext->data0) & RX_DESC_DATA0_OWN_)
2582             /* extension not yet available */
2583             goto done;
2584         if (!(le32_to_cpu(desc_ext->data0) & RX_DESC_DATA0_EXT_))
2585             goto move_forward;
2586         extension_index = index;
2587     }
2588
2589     /* Only the last buffer in a multi-buffer frame contains the total frame
2590      * length. The chip occasionally sends more buffers than strictly
2591      * required to reach the total frame length.
2592      * Handle this by adding all buffers to the skb in their entirety.
2593      * Once the real frame length is known, trim the skb.
2594      */
2595     frame_length =
2596         RX_DESC_DATA0_FRAME_LENGTH_GET_(le32_to_cpu(descriptor->data0));
2597     buffer_length = buffer_info->buffer_length;
2598
2599     netdev_dbg(netdev, "%s%schunk: %d/%d",
2600            is_first ? "first " : "      ",
2601            is_last  ? "last  " : "      ",
2602            frame_length, buffer_length);
2603
2604     /* save existing skb, allocate new skb and map to dma */
2605     skb = buffer_info->skb;
2606     if (lan743x_rx_init_ring_element(rx, rx->last_head,
2607                      GFP_ATOMIC | GFP_DMA)) {
2608         /* failed to allocate next skb.
2609          * Memory is very low.
2610          * Drop this packet and reuse buffer.
2611          */
2612         lan743x_rx_reuse_ring_element(rx, rx->last_head);
2613         /* drop packet that was being assembled */
2614         dev_kfree_skb_irq(rx->skb_head);
2615         rx->skb_head = NULL;
2616         goto process_extension;
2617     }
2618
2619     /* add buffers to skb via skb->frag_list */
2620     if (is_first) {
2621         skb_reserve(skb, RX_HEAD_PADDING);
2622         skb_put(skb, buffer_length - RX_HEAD_PADDING);
2623         if (rx->skb_head)
2624             dev_kfree_skb_irq(rx->skb_head);
2625         rx->skb_head = skb;
2626     } else if (rx->skb_head) {
2627         skb_put(skb, buffer_length);
2628         if (skb_shinfo(rx->skb_head)->frag_list)
2629             rx->skb_tail->next = skb;
2630         else
2631             skb_shinfo(rx->skb_head)->frag_list = skb;
2632         rx->skb_tail = skb;
2633         rx->skb_head->len += skb->len;
2634         rx->skb_head->data_len += skb->len;
2635         rx->skb_head->truesize += skb->truesize;
2636     } else {
2637         /* packet to assemble has already been dropped because one or
2638          * more of its buffers could not be allocated
2639          */
2640         netdev_dbg(netdev, "drop buffer intended for dropped packet");
2641         dev_kfree_skb_irq(skb);
2642     }
2643
2644 process_extension:
2645     if (extension_index >= 0) {
2646         u32 ts_sec;
2647         u32 ts_nsec;
2648
2649         ts_sec = le32_to_cpu(desc_ext->data1);
2650         ts_nsec = (le32_to_cpu(desc_ext->data2) &
2651               RX_DESC_DATA2_TS_NS_MASK_);
2652         if (rx->skb_head)
2653             skb_hwtstamps(rx->skb_head)->hwtstamp =
2654                 ktime_set(ts_sec, ts_nsec);
2655         lan743x_rx_reuse_ring_element(rx, extension_index);
2656         rx->last_head = extension_index;
2657         netdev_dbg(netdev, "process extension");
2658     }
2659
2660     if (is_last && rx->skb_head)
2661         rx->skb_head = lan743x_rx_trim_skb(rx->skb_head, frame_length);
2662
2663     if (is_last && rx->skb_head) {
2664         rx->skb_head->protocol = eth_type_trans(rx->skb_head,
2665                             rx->adapter->netdev);
2666         netdev_dbg(netdev, "sending %d byte frame to OS",
2667                rx->skb_head->len);
2668         napi_gro_receive(&rx->napi, rx->skb_head);
2669         rx->skb_head = NULL;
2670     }
2671
2672 move_forward:
2673     /* push tail and head forward */
2674     rx->last_tail = rx->last_head;
2675     rx->last_head = lan743x_rx_next_index(rx, rx->last_head);
2676     result = RX_PROCESS_RESULT_BUFFER_RECEIVED;
2677 done:
2678     return result;
2679 }
2680
2681 static int lan743x_rx_napi_poll(struct napi_struct *napi, int weight)
2682 {
2683     struct lan743x_rx *rx = container_of(napi, struct lan743x_rx, napi);
2684     struct lan743x_adapter *adapter = rx->adapter;
2685     int result = RX_PROCESS_RESULT_NOTHING_TO_DO;
2686     u32 rx_tail_flags = 0;
2687     int count;
2688
2689     if (rx->vector_flags & LAN743X_VECTOR_FLAG_SOURCE_STATUS_W2C) {
2690         /* clear int status bit before reading packet */
2691         lan743x_csr_write(adapter, DMAC_INT_STS,
2692                   DMAC_INT_BIT_RXFRM_(rx->channel_number));
2693     }
2694     for (count = 0; count < weight; count++) {
2695         result = lan743x_rx_process_buffer(rx);
2696         if (result == RX_PROCESS_RESULT_NOTHING_TO_DO)
2697             break;
2698     }
2699     rx->frame_count += count;
2700     if (count == weight || result == RX_PROCESS_RESULT_BUFFER_RECEIVED)
2701         return weight;
2702
2703     if (!napi_complete_done(napi, count))
2704         return count;
2705
2706     /* re-arm interrupts, must write to rx tail on some chip variants */
2707     if (rx->vector_flags & LAN743X_VECTOR_FLAG_VECTOR_ENABLE_AUTO_SET)
2708         rx_tail_flags |= RX_TAIL_SET_TOP_INT_VEC_EN_;
2709     if (rx->vector_flags & LAN743X_VECTOR_FLAG_SOURCE_ENABLE_AUTO_SET) {
2710         rx_tail_flags |= RX_TAIL_SET_TOP_INT_EN_;
2711     } else {
2712         lan743x_csr_write(adapter, INT_EN_SET,
2713                   INT_BIT_DMA_RX_(rx->channel_number));
2714     }
2715
2716     if (rx_tail_flags)
2717         lan743x_csr_write(adapter, RX_TAIL(rx->channel_number),
2718                   rx_tail_flags | rx->last_tail);
2719
2720     return count;
2721 }
2722
2723 static void lan743x_rx_ring_cleanup(struct lan743x_rx *rx)
2724 {
2725     if (rx->buffer_info && rx->ring_cpu_ptr) {
2726         int index;
2727
2728         for (index = 0; index < rx->ring_size; index++)
2729             lan743x_rx_release_ring_element(rx, index);
2730     }
2731
2732     if (rx->head_cpu_ptr) {
2733         dma_free_coherent(&rx->adapter->pdev->dev,
2734                   sizeof(*rx->head_cpu_ptr), rx->head_cpu_ptr,
2735                   rx->head_dma_ptr);
2736         rx->head_cpu_ptr = NULL;
2737         rx->head_dma_ptr = 0;
2738     }
2739
2740     kfree(rx->buffer_info);
2741     rx->buffer_info = NULL;
2742
2743     if (rx->ring_cpu_ptr) {
2744         dma_free_coherent(&rx->adapter->pdev->dev,
2745                   rx->ring_allocation_size, rx->ring_cpu_ptr,
2746                   rx->ring_dma_ptr);
2747         rx->ring_allocation_size = 0;
2748         rx->ring_cpu_ptr = NULL;
2749         rx->ring_dma_ptr = 0;
2750     }
2751
2752     rx->ring_size = 0;
2753     rx->last_head = 0;
2754 }
2755
2756 static int lan743x_rx_ring_init(struct lan743x_rx *rx)
2757 {
2758     size_t ring_allocation_size = 0;
2759     dma_addr_t dma_ptr = 0;
2760     void *cpu_ptr = NULL;
2761     int ret = -ENOMEM;
2762     int index = 0;
2763
2764     rx->ring_size = LAN743X_RX_RING_SIZE;
2765     if (rx->ring_size <= 1) {
2766         ret = -EINVAL;
2767         goto cleanup;
2768     }
2769     if (rx->ring_size & ~RX_CFG_B_RX_RING_LEN_MASK_) {
2770         ret = -EINVAL;
2771         goto cleanup;
2772     }
2773     if (dma_set_mask_and_coherent(&rx->adapter->pdev->dev,
2774                       DMA_BIT_MASK(64))) {
2775         dev_warn(&rx->adapter->pdev->dev,
2776              "lan743x_: No suitable DMA available\n");
2777         ret = -ENOMEM;
2778         goto cleanup;
2779     }
2780     ring_allocation_size = ALIGN(rx->ring_size *
2781                      sizeof(struct lan743x_rx_descriptor),
2782                      PAGE_SIZE);
2783     dma_ptr = 0;
2784     cpu_ptr = dma_alloc_coherent(&rx->adapter->pdev->dev,
2785                      ring_allocation_size, &dma_ptr, GFP_KERNEL);
2786     if (!cpu_ptr) {
2787         ret = -ENOMEM;
2788         goto cleanup;
2789     }
2790     rx->ring_allocation_size = ring_allocation_size;
2791     rx->ring_cpu_ptr = (struct lan743x_rx_descriptor *)cpu_ptr;
2792     rx->ring_dma_ptr = dma_ptr;
2793
2794     cpu_ptr = kcalloc(rx->ring_size, sizeof(*rx->buffer_info),
2795               GFP_KERNEL);
2796     if (!cpu_ptr) {
2797         ret = -ENOMEM;
2798         goto cleanup;
2799     }
2800     rx->buffer_info = (struct lan743x_rx_buffer_info *)cpu_ptr;
2801     dma_ptr = 0;
2802     cpu_ptr = dma_alloc_coherent(&rx->adapter->pdev->dev,
2803                      sizeof(*rx->head_cpu_ptr), &dma_ptr,
2804                      GFP_KERNEL);
2805     if (!cpu_ptr) {
2806         ret = -ENOMEM;
2807         goto cleanup;
2808     }
2809
2810     rx->head_cpu_ptr = cpu_ptr;
2811     rx->head_dma_ptr = dma_ptr;
2812     if (rx->head_dma_ptr & 0x3) {
2813         ret = -ENOMEM;
2814         goto cleanup;
2815     }
2816
2817     rx->last_head = 0;
2818     for (index = 0; index < rx->ring_size; index++) {
2819         ret = lan743x_rx_init_ring_element(rx, index, GFP_KERNEL);
2820         if (ret)
2821             goto cleanup;
2822     }
2823     return 0;
2824
2825 cleanup:
2826     netif_warn(rx->adapter, ifup, rx->adapter->netdev,
2827            "Error allocating memory for LAN743x\n");
2828
2829     lan743x_rx_ring_cleanup(rx);
2830     return ret;
2831 }
2832
2833 static void lan743x_rx_close(struct lan743x_rx *rx)
2834 {
2835     struct lan743x_adapter *adapter = rx->adapter;
2836
2837     lan743x_csr_write(adapter, FCT_RX_CTL,
2838               FCT_RX_CTL_DIS_(rx->channel_number));
2839     lan743x_csr_wait_for_bit(adapter, FCT_RX_CTL,
2840                  FCT_RX_CTL_EN_(rx->channel_number),
2841                  0, 1000, 20000, 100);
2842
2843     lan743x_csr_write(adapter, DMAC_CMD,
2844               DMAC_CMD_STOP_R_(rx->channel_number));
2845     lan743x_dmac_rx_wait_till_stopped(adapter, rx->channel_number);
2846
2847     lan743x_csr_write(adapter, DMAC_INT_EN_CLR,
2848               DMAC_INT_BIT_RXFRM_(rx->channel_number));
2849     lan743x_csr_write(adapter, INT_EN_CLR,
2850               INT_BIT_DMA_RX_(rx->channel_number));
2851     napi_disable(&rx->napi);
2852
2853     netif_napi_del(&rx->napi);
2854
2855     lan743x_rx_ring_cleanup(rx);
2856 }
2857
2858 static int lan743x_rx_open(struct lan743x_rx *rx)
2859 {
2860     struct lan743x_adapter *adapter = rx->adapter;
2861     u32 data = 0;
2862     int ret;
2863
2864     rx->frame_count = 0;
2865     ret = lan743x_rx_ring_init(rx);
2866     if (ret)
2867         goto return_error;
2868
2869     netif_napi_add(adapter->netdev,
2870                &rx->napi, lan743x_rx_napi_poll,
2871                NAPI_POLL_WEIGHT);
2872
2873     lan743x_csr_write(adapter, DMAC_CMD,
2874               DMAC_CMD_RX_SWR_(rx->channel_number));
2875     lan743x_csr_wait_for_bit(adapter, DMAC_CMD,
2876                  DMAC_CMD_RX_SWR_(rx->channel_number),
2877                  0, 1000, 20000, 100);
2878
2879     /* set ring base address */
2880     lan743x_csr_write(adapter,
2881               RX_BASE_ADDRH(rx->channel_number),
2882               DMA_ADDR_HIGH32(rx->ring_dma_ptr));
2883     lan743x_csr_write(adapter,
2884               RX_BASE_ADDRL(rx->channel_number),
2885               DMA_ADDR_LOW32(rx->ring_dma_ptr));
2886
2887     /* set rx write back address */
2888     lan743x_csr_write(adapter,
2889               RX_HEAD_WRITEBACK_ADDRH(rx->channel_number),
2890               DMA_ADDR_HIGH32(rx->head_dma_ptr));
2891     lan743x_csr_write(adapter,
2892               RX_HEAD_WRITEBACK_ADDRL(rx->channel_number),
2893               DMA_ADDR_LOW32(rx->head_dma_ptr));
2894     data = RX_CFG_A_RX_HP_WB_EN_;
2895     if (!(adapter->csr.flags & LAN743X_CSR_FLAG_IS_A0)) {
2896         data |= (RX_CFG_A_RX_WB_ON_INT_TMR_ |
2897             RX_CFG_A_RX_WB_THRES_SET_(0x7) |
2898             RX_CFG_A_RX_PF_THRES_SET_(16) |
2899             RX_CFG_A_RX_PF_PRI_THRES_SET_(4));
2900     }
2901
2902     /* set RX_CFG_A */
2903     lan743x_csr_write(adapter,
2904               RX_CFG_A(rx->channel_number), data);
2905
2906     /* set RX_CFG_B */
2907     data = lan743x_csr_read(adapter, RX_CFG_B(rx->channel_number));
2908     data &= ~RX_CFG_B_RX_PAD_MASK_;
2909     if (!RX_HEAD_PADDING)
2910         data |= RX_CFG_B_RX_PAD_0_;
2911     else
2912         data |= RX_CFG_B_RX_PAD_2_;
2913     data &= ~RX_CFG_B_RX_RING_LEN_MASK_;
2914     data |= ((rx->ring_size) & RX_CFG_B_RX_RING_LEN_MASK_);
2915     data |= RX_CFG_B_TS_ALL_RX_;
2916     if (!(adapter->csr.flags & LAN743X_CSR_FLAG_IS_A0))
2917         data |= RX_CFG_B_RDMABL_512_;
2918
2919     lan743x_csr_write(adapter, RX_CFG_B(rx->channel_number), data);
2920     rx->vector_flags = lan743x_intr_get_vector_flags(adapter,
2921                              INT_BIT_DMA_RX_
2922                              (rx->channel_number));
2923
2924     /* set RX_CFG_C */
2925     data = 0;
2926     if (rx->vector_flags & LAN743X_VECTOR_FLAG_SOURCE_ENABLE_AUTO_CLEAR)
2927         data |= RX_CFG_C_RX_TOP_INT_EN_AUTO_CLR_;
2928     if (rx->vector_flags & LAN743X_VECTOR_FLAG_SOURCE_STATUS_AUTO_CLEAR)
2929         data |= RX_CFG_C_RX_DMA_INT_STS_AUTO_CLR_;
2930     if (rx->vector_flags & LAN743X_VECTOR_FLAG_SOURCE_STATUS_R2C)
2931         data |= RX_CFG_C_RX_INT_STS_R2C_MODE_MASK_;
2932     if (rx->vector_flags & LAN743X_VECTOR_FLAG_SOURCE_ENABLE_R2C)
2933         data |= RX_CFG_C_RX_INT_EN_R2C_;
2934     lan743x_csr_write(adapter, RX_CFG_C(rx->channel_number), data);
2935
2936     rx->last_tail = ((u32)(rx->ring_size - 1));
2937     lan743x_csr_write(adapter, RX_TAIL(rx->channel_number),
2938               rx->last_tail);
2939     rx->last_head = lan743x_csr_read(adapter, RX_HEAD(rx->channel_number));
2940     if (rx->last_head) {
2941         ret = -EIO;
2942         goto napi_delete;
2943     }
2944
2945     napi_enable(&rx->napi);
2946
2947     lan743x_csr_write(adapter, INT_EN_SET,
2948               INT_BIT_DMA_RX_(rx->channel_number));
2949     lan743x_csr_write(adapter, DMAC_INT_STS,
2950               DMAC_INT_BIT_RXFRM_(rx->channel_number));
2951     lan743x_csr_write(adapter, DMAC_INT_EN_SET,
2952               DMAC_INT_BIT_RXFRM_(rx->channel_number));
2953     lan743x_csr_write(adapter, DMAC_CMD,
2954               DMAC_CMD_START_R_(rx->channel_number));
2955
2956     /* initialize fifo */
2957     lan743x_csr_write(adapter, FCT_RX_CTL,
2958               FCT_RX_CTL_RESET_(rx->channel_number));
2959     lan743x_csr_wait_for_bit(adapter, FCT_RX_CTL,
2960                  FCT_RX_CTL_RESET_(rx->channel_number),
2961                  0, 1000, 20000, 100);
2962     lan743x_csr_write(adapter, FCT_FLOW(rx->channel_number),
2963               FCT_FLOW_CTL_REQ_EN_ |
2964               FCT_FLOW_CTL_ON_THRESHOLD_SET_(0x2A) |
2965               FCT_FLOW_CTL_OFF_THRESHOLD_SET_(0xA));
2966
2967     /* enable fifo */
2968     lan743x_csr_write(adapter, FCT_RX_CTL,
2969               FCT_RX_CTL_EN_(rx->channel_number));
2970     return 0;
2971
2972 napi_delete:
2973     netif_napi_del(&rx->napi);
2974     lan743x_rx_ring_cleanup(rx);
2975
2976 return_error:
2977     return ret;
2978 }
2979
2980 static int lan743x_netdev_close(struct net_device *netdev)
2981 {
2982     struct lan743x_adapter *adapter = netdev_priv(netdev);
2983     int index;
2984
2985     for (index = 0; index < adapter->used_tx_channels; index++)
2986         lan743x_tx_close(&adapter->tx[index]);
2987
2988     for (index = 0; index < LAN743X_USED_RX_CHANNELS; index++)
2989         lan743x_rx_close(&adapter->rx[index]);
2990
2991     lan743x_ptp_close(adapter);
2992
2993     lan743x_phy_close(adapter);
2994
2995     lan743x_mac_close(adapter);
2996
2997     lan743x_intr_close(adapter);
2998
2999     return 0;
3000 }
3001
3002 static int lan743x_netdev_open(struct net_device *netdev)
3003 {
3004     struct lan743x_adapter *adapter = netdev_priv(netdev);
3005     int index;
3006     int ret;
3007
3008     ret = lan743x_intr_open(adapter);
3009     if (ret)
3010         goto return_error;
3011
3012     ret = lan743x_mac_open(adapter);
3013     if (ret)
3014         goto close_intr;
3015
3016     ret = lan743x_phy_open(adapter);
3017     if (ret)
3018         goto close_mac;
3019
3020     ret = lan743x_ptp_open(adapter);
3021     if (ret)
3022         goto close_phy;
3023
3024     lan743x_rfe_open(adapter);
3025
3026     for (index = 0; index < LAN743X_USED_RX_CHANNELS; index++) {
3027         ret = lan743x_rx_open(&adapter->rx[index]);
3028         if (ret)
3029             goto close_rx;
3030     }
3031
3032     for (index = 0; index < adapter->used_tx_channels; index++) {
3033         ret = lan743x_tx_open(&adapter->tx[index]);
3034         if (ret)
3035             goto close_tx;
3036     }
3037     return 0;
3038
3039 close_tx:
3040     for (index = 0; index < adapter->used_tx_channels; index++) {
3041         if (adapter->tx[index].ring_cpu_ptr)
3042             lan743x_tx_close(&adapter->tx[index]);
3043     }
3044
3045 close_rx:
3046     for (index = 0; index < LAN743X_USED_RX_CHANNELS; index++) {
3047         if (adapter->rx[index].ring_cpu_ptr)
3048             lan743x_rx_close(&adapter->rx[index]);
3049     }
3050     lan743x_ptp_close(adapter);
3051
3052 close_phy:
3053     lan743x_phy_close(adapter);
3054
3055 close_mac:
3056     lan743x_mac_close(adapter);
3057
3058 close_intr:
3059     lan743x_intr_close(adapter);
3060
3061 return_error:
3062     netif_warn(adapter, ifup, adapter->netdev,
3063            "Error opening LAN743x\n");
3064     return ret;
3065 }
3066
3067 static netdev_tx_t lan743x_netdev_xmit_frame(struct sk_buff *skb,
3068                          struct net_device *netdev)
3069 {
3070     struct lan743x_adapter *adapter = netdev_priv(netdev);
3071     u8 ch = 0;
3072
3073     if (adapter->is_pci11x1x)
3074         ch = skb->queue_mapping % PCI11X1X_USED_TX_CHANNELS;
3075
3076     return lan743x_tx_xmit_frame(&adapter->tx[ch], skb);
3077 }
3078
3079 static int lan743x_netdev_ioctl(struct net_device *netdev,
3080                 struct ifreq *ifr, int cmd)
3081 {
3082     if (!netif_running(netdev))
3083         return -EINVAL;
3084     if (cmd == SIOCSHWTSTAMP)
3085         return lan743x_ptp_ioctl(netdev, ifr, cmd);
3086     return phy_mii_ioctl(netdev->phydev, ifr, cmd);
3087 }
3088
3089 static void lan743x_netdev_set_multicast(struct net_device *netdev)
3090 {
3091     struct lan743x_adapter *adapter = netdev_priv(netdev);
3092
3093     lan743x_rfe_set_multicast(adapter);
3094 }
3095
3096 static int lan743x_netdev_change_mtu(struct net_device *netdev, int new_mtu)
3097 {
3098     struct lan743x_adapter *adapter = netdev_priv(netdev);
3099     int ret = 0;
3100
3101     ret = lan743x_mac_set_mtu(adapter, new_mtu);
3102     if (!ret)
3103         netdev->mtu = new_mtu;
3104     return ret;
3105 }
3106
3107 static void lan743x_netdev_get_stats64(struct net_device *netdev,
3108                        struct rtnl_link_stats64 *stats)
3109 {
3110     struct lan743x_adapter *adapter = netdev_priv(netdev);
3111
3112     stats->rx_packets = lan743x_csr_read(adapter, STAT_RX_TOTAL_FRAMES);
3113     stats->tx_packets = lan743x_csr_read(adapter, STAT_TX_TOTAL_FRAMES);
3114     stats->rx_bytes = lan743x_csr_read(adapter,
3115                        STAT_RX_UNICAST_BYTE_COUNT) +
3116               lan743x_csr_read(adapter,
3117                        STAT_RX_BROADCAST_BYTE_COUNT) +
3118               lan743x_csr_read(adapter,
3119                        STAT_RX_MULTICAST_BYTE_COUNT);
3120     stats->tx_bytes = lan743x_csr_read(adapter,
3121                        STAT_TX_UNICAST_BYTE_COUNT) +
3122               lan743x_csr_read(adapter,
3123                        STAT_TX_BROADCAST_BYTE_COUNT) +
3124               lan743x_csr_read(adapter,
3125                        STAT_TX_MULTICAST_BYTE_COUNT);
3126     stats->rx_errors = lan743x_csr_read(adapter, STAT_RX_FCS_ERRORS) +
3127                lan743x_csr_read(adapter,
3128                         STAT_RX_ALIGNMENT_ERRORS) +
3129                lan743x_csr_read(adapter, STAT_RX_JABBER_ERRORS) +
3130                lan743x_csr_read(adapter,
3131                         STAT_RX_UNDERSIZE_FRAME_ERRORS) +
3132                lan743x_csr_read(adapter,
3133                         STAT_RX_OVERSIZE_FRAME_ERRORS);
3134     stats->tx_errors = lan743x_csr_read(adapter, STAT_TX_FCS_ERRORS) +
3135                lan743x_csr_read(adapter,
3136                         STAT_TX_EXCESS_DEFERRAL_ERRORS) +
3137                lan743x_csr_read(adapter, STAT_TX_CARRIER_ERRORS);
3138     stats->rx_dropped = lan743x_csr_read(adapter,
3139                          STAT_RX_DROPPED_FRAMES);
3140     stats->tx_dropped = lan743x_csr_read(adapter,
3141                          STAT_TX_EXCESSIVE_COLLISION);
3142     stats->multicast = lan743x_csr_read(adapter,
3143                         STAT_RX_MULTICAST_FRAMES) +
3144                lan743x_csr_read(adapter,
3145                         STAT_TX_MULTICAST_FRAMES);
3146     stats->collisions = lan743x_csr_read(adapter,
3147                          STAT_TX_SINGLE_COLLISIONS) +
3148                 lan743x_csr_read(adapter,
3149                          STAT_TX_MULTIPLE_COLLISIONS) +
3150                 lan743x_csr_read(adapter,
3151                          STAT_TX_LATE_COLLISIONS);
3152 }
3153
3154 static int lan743x_netdev_set_mac_address(struct net_device *netdev,
3155                       void *addr)
3156 {
3157     struct lan743x_adapter *adapter = netdev_priv(netdev);
3158     struct sockaddr *sock_addr = addr;
3159     int ret;
3160
3161     ret = eth_prepare_mac_addr_change(netdev, sock_addr);
3162     if (ret)
3163         return ret;
3164     eth_hw_addr_set(netdev, sock_addr->sa_data);
3165     lan743x_mac_set_address(adapter, sock_addr->sa_data);
3166     lan743x_rfe_update_mac_address(adapter);
3167     return 0;
3168 }
3169
3170 static const struct net_device_ops lan743x_netdev_ops = {
3171     .ndo_open       = lan743x_netdev_open,
3172     .ndo_stop       = lan743x_netdev_close,
3173     .ndo_start_xmit     = lan743x_netdev_xmit_frame,
3174     .ndo_eth_ioctl      = lan743x_netdev_ioctl,
3175     .ndo_set_rx_mode    = lan743x_netdev_set_multicast,
3176     .ndo_change_mtu     = lan743x_netdev_change_mtu,
3177     .ndo_get_stats64    = lan743x_netdev_get_stats64,
3178     .ndo_set_mac_address    = lan743x_netdev_set_mac_address,
3179 };
3180
3181 static void lan743x_hardware_cleanup(struct lan743x_adapter *adapter)
3182 {
3183     lan743x_csr_write(adapter, INT_EN_CLR, 0xFFFFFFFF);
3184 }
3185
3186 static void lan743x_mdiobus_cleanup(struct lan743x_adapter *adapter)
3187 {
3188     mdiobus_unregister(adapter->mdiobus);
3189 }
3190
3191 static void lan743x_full_cleanup(struct lan743x_adapter *adapter)
3192 {
3193     unregister_netdev(adapter->netdev);
3194
3195     lan743x_mdiobus_cleanup(adapter);
3196     lan743x_hardware_cleanup(adapter);
3197     lan743x_pci_cleanup(adapter);
3198 }
3199
3200 static int lan743x_hardware_init(struct lan743x_adapter *adapter,
3201                  struct pci_dev *pdev)
3202 {
3203     struct lan743x_tx *tx;
3204     int index;
3205     int ret;
3206
3207     adapter->is_pci11x1x = is_pci11x1x_chip(adapter);
3208     if (adapter->is_pci11x1x) {
3209         adapter->max_tx_channels = PCI11X1X_MAX_TX_CHANNELS;
3210         adapter->used_tx_channels = PCI11X1X_USED_TX_CHANNELS;
3211         adapter->max_vector_count = PCI11X1X_MAX_VECTOR_COUNT;
3212         pci11x1x_strap_get_status(adapter);
3213         spin_lock_init(&adapter->eth_syslock_spinlock);
3214         mutex_init(&adapter->sgmii_rw_lock);
3215     } else {
3216         adapter->max_tx_channels = LAN743X_MAX_TX_CHANNELS;
3217         adapter->used_tx_channels = LAN743X_USED_TX_CHANNELS;
3218         adapter->max_vector_count = LAN743X_MAX_VECTOR_COUNT;
3219     }
3220
3221     adapter->intr.irq = adapter->pdev->irq;
3222     lan743x_csr_write(adapter, INT_EN_CLR, 0xFFFFFFFF);
3223
3224     ret = lan743x_gpio_init(adapter);
3225     if (ret)
3226         return ret;
3227
3228     ret = lan743x_mac_init(adapter);
3229     if (ret)
3230         return ret;
3231
3232     ret = lan743x_phy_init(adapter);
3233     if (ret)
3234         return ret;
3235
3236     ret = lan743x_ptp_init(adapter);
3237     if (ret)
3238         return ret;
3239
3240     lan743x_rfe_update_mac_address(adapter);
3241
3242     ret = lan743x_dmac_init(adapter);
3243     if (ret)
3244         return ret;
3245
3246     for (index = 0; index < LAN743X_USED_RX_CHANNELS; index++) {
3247         adapter->rx[index].adapter = adapter;
3248         adapter->rx[index].channel_number = index;
3249     }
3250
3251     for (index = 0; index < adapter->used_tx_channels; index++) {
3252         tx = &adapter->tx[index];
3253         tx->adapter = adapter;
3254         tx->channel_number = index;
3255         spin_lock_init(&tx->ring_lock);
3256     }
3257
3258     return 0;
3259 }
3260
3261 static int lan743x_mdiobus_init(struct lan743x_adapter *adapter)
3262 {
3263     u32 sgmii_ctl;
3264     int ret;
3265
3266     adapter->mdiobus = devm_mdiobus_alloc(&adapter->pdev->dev);
3267     if (!(adapter->mdiobus)) {
3268         ret = -ENOMEM;
3269         goto return_error;
3270     }
3271
3272     adapter->mdiobus->priv = (void *)adapter;
3273     if (adapter->is_pci11x1x) {
3274         if (adapter->is_sgmii_en) {
3275             sgmii_ctl = lan743x_csr_read(adapter, SGMII_CTL);
3276             sgmii_ctl |= SGMII_CTL_SGMII_ENABLE_;
3277             sgmii_ctl &= ~SGMII_CTL_SGMII_POWER_DN_;
3278             lan743x_csr_write(adapter, SGMII_CTL, sgmii_ctl);
3279             netif_dbg(adapter, drv, adapter->netdev,
3280                   "SGMII operation\n");
3281             adapter->mdiobus->probe_capabilities = MDIOBUS_C22_C45;
3282             adapter->mdiobus->read = lan743x_mdiobus_c45_read;
3283             adapter->mdiobus->write = lan743x_mdiobus_c45_write;
3284             adapter->mdiobus->name = "lan743x-mdiobus-c45";
3285             netif_dbg(adapter, drv, adapter->netdev,
3286                   "lan743x-mdiobus-c45\n");
3287         } else {
3288             sgmii_ctl = lan743x_csr_read(adapter, SGMII_CTL);
3289             sgmii_ctl &= ~SGMII_CTL_SGMII_ENABLE_;
3290             sgmii_ctl |= SGMII_CTL_SGMII_POWER_DN_;
3291             lan743x_csr_write(adapter, SGMII_CTL, sgmii_ctl);
3292             netif_dbg(adapter, drv, adapter->netdev,
3293                   "RGMII operation\n");
3294             // Only C22 support when RGMII I/F
3295             adapter->mdiobus->probe_capabilities = MDIOBUS_C22;
3296             adapter->mdiobus->read = lan743x_mdiobus_read;
3297             adapter->mdiobus->write = lan743x_mdiobus_write;
3298             adapter->mdiobus->name = "lan743x-mdiobus";
3299             netif_dbg(adapter, drv, adapter->netdev,
3300                   "lan743x-mdiobus\n");
3301         }
3302     } else {
3303         adapter->mdiobus->read = lan743x_mdiobus_read;
3304         adapter->mdiobus->write = lan743x_mdiobus_write;
3305         adapter->mdiobus->name = "lan743x-mdiobus";
3306         netif_dbg(adapter, drv, adapter->netdev, "lan743x-mdiobus\n");
3307     }
3308
3309     snprintf(adapter->mdiobus->id, MII_BUS_ID_SIZE,
3310          "pci-%s", pci_name(adapter->pdev));
3311
3312     if ((adapter->csr.id_rev & ID_REV_ID_MASK_) == ID_REV_ID_LAN7430_)
3313         /* LAN7430 uses internal phy at address 1 */
3314         adapter->mdiobus->phy_mask = ~(u32)BIT(1);
3315
3316     /* register mdiobus */
3317     ret = mdiobus_register(adapter->mdiobus);
3318     if (ret < 0)
3319         goto return_error;
3320     return 0;
3321
3322 return_error:
3323     return ret;
3324 }
3325
3326 /* lan743x_pcidev_probe - Device Initialization Routine
3327  * @pdev: PCI device information struct
3328  * @id: entry in lan743x_pci_tbl
3329  *
3330  * Returns 0 on success, negative on failure
3331  *
3332  * initializes an adapter identified by a pci_dev structure.
3333  * The OS initialization, configuring of the adapter private structure,
3334  * and a hardware reset occur.
3335  **/
3336 static int lan743x_pcidev_probe(struct pci_dev *pdev,
3337                 const struct pci_device_id *id)
3338 {
3339     struct lan743x_adapter *adapter = NULL;
3340     struct net_device *netdev = NULL;
3341     int ret = -ENODEV;
3342
3343     if (id->device == PCI_DEVICE_ID_SMSC_A011 ||
3344         id->device == PCI_DEVICE_ID_SMSC_A041) {
3345         netdev = devm_alloc_etherdev_mqs(&pdev->dev,
3346                          sizeof(struct lan743x_adapter),
3347                          PCI11X1X_USED_TX_CHANNELS,
3348                          LAN743X_USED_RX_CHANNELS);
3349     } else {
3350         netdev = devm_alloc_etherdev(&pdev->dev,
3351                          sizeof(struct lan743x_adapter));
3352     }
3353
3354     if (!netdev)
3355         goto return_error;
3356
3357     SET_NETDEV_DEV(netdev, &pdev->dev);
3358     pci_set_drvdata(pdev, netdev);
3359     adapter = netdev_priv(netdev);
3360     adapter->netdev = netdev;
3361     adapter->msg_enable = NETIF_MSG_DRV | NETIF_MSG_PROBE |
3362                   NETIF_MSG_LINK | NETIF_MSG_IFUP |
3363                   NETIF_MSG_IFDOWN | NETIF_MSG_TX_QUEUED;
3364     netdev->max_mtu = LAN743X_MAX_FRAME_SIZE;
3365
3366     of_get_mac_address(pdev->dev.of_node, adapter->mac_address);
3367
3368     ret = lan743x_pci_init(adapter, pdev);
3369     if (ret)
3370         goto return_error;
3371
3372     ret = lan743x_csr_init(adapter);
3373     if (ret)
3374         goto cleanup_pci;
3375
3376     ret = lan743x_hardware_init(adapter, pdev);
3377     if (ret)
3378         goto cleanup_pci;
3379
3380     ret = lan743x_mdiobus_init(adapter);
3381     if (ret)
3382         goto cleanup_hardware;
3383
3384     adapter->netdev->netdev_ops = &lan743x_netdev_ops;
3385     adapter->netdev->ethtool_ops = &lan743x_ethtool_ops;
3386     adapter->netdev->features = NETIF_F_SG | NETIF_F_TSO | NETIF_F_HW_CSUM;
3387     adapter->netdev->hw_features = adapter->netdev->features;
3388
3389     /* carrier off reporting is important to ethtool even BEFORE open */
3390     netif_carrier_off(netdev);
3391
3392     ret = register_netdev(adapter->netdev);
3393     if (ret < 0)
3394         goto cleanup_mdiobus;
3395     return 0;
3396
3397 cleanup_mdiobus:
3398     lan743x_mdiobus_cleanup(adapter);
3399
3400 cleanup_hardware:
3401     lan743x_hardware_cleanup(adapter);
3402
3403 cleanup_pci:
3404     lan743x_pci_cleanup(adapter);
3405
3406 return_error:
3407     pr_warn("Initialization failed\n");
3408     return ret;
3409 }
3410
3411 /**
3412  * lan743x_pcidev_remove - Device Removal Routine
3413  * @pdev: PCI device information struct
3414  *
3415  * this is called by the PCI subsystem to alert the driver
3416  * that it should release a PCI device.  This could be caused by a
3417  * Hot-Plug event, or because the driver is going to be removed from
3418  * memory.
3419  **/
3420 static void lan743x_pcidev_remove(struct pci_dev *pdev)
3421 {
3422     struct net_device *netdev = pci_get_drvdata(pdev);
3423     struct lan743x_adapter *adapter = netdev_priv(netdev);
3424
3425     lan743x_full_cleanup(adapter);
3426 }
3427
3428 static void lan743x_pcidev_shutdown(struct pci_dev *pdev)
3429 {
3430     struct net_device *netdev = pci_get_drvdata(pdev);
3431     struct lan743x_adapter *adapter = netdev_priv(netdev);
3432
3433     rtnl_lock();
3434     netif_device_detach(netdev);
3435
3436     /* close netdev when netdev is at running state.
3437      * For instance, it is true when system goes to sleep by pm-suspend
3438      * However, it is false when system goes to sleep by suspend GUI menu
3439      */
3440     if (netif_running(netdev))
3441         lan743x_netdev_close(netdev);
3442     rtnl_unlock();
3443
3444 #ifdef CONFIG_PM
3445     pci_save_state(pdev);
3446 #endif
3447
3448     /* clean up lan743x portion */
3449     lan743x_hardware_cleanup(adapter);
3450 }
3451
3452 #ifdef CONFIG_PM_SLEEP
3453 static u16 lan743x_pm_wakeframe_crc16(const u8 *buf, int len)
3454 {
3455     return bitrev16(crc16(0xFFFF, buf, len));
3456 }
3457
3458 static void lan743x_pm_set_wol(struct lan743x_adapter *adapter)
3459 {
3460     const u8 ipv4_multicast[3] = { 0x01, 0x00, 0x5E };
3461     const u8 ipv6_multicast[3] = { 0x33, 0x33 };
3462     const u8 arp_type[2] = { 0x08, 0x06 };
3463     int mask_index;
3464     u32 sopass;
3465     u32 pmtctl;
3466     u32 wucsr;
3467     u32 macrx;
3468     u16 crc;
3469
3470     for (mask_index = 0; mask_index < MAC_NUM_OF_WUF_CFG; mask_index++)
3471         lan743x_csr_write(adapter, MAC_WUF_CFG(mask_index), 0);
3472
3473     /* clear wake settings */
3474     pmtctl = lan743x_csr_read(adapter, PMT_CTL);
3475     pmtctl |= PMT_CTL_WUPS_MASK_;
3476     pmtctl &= ~(PMT_CTL_GPIO_WAKEUP_EN_ | PMT_CTL_EEE_WAKEUP_EN_ |
3477         PMT_CTL_WOL_EN_ | PMT_CTL_MAC_D3_RX_CLK_OVR_ |
3478         PMT_CTL_RX_FCT_RFE_D3_CLK_OVR_ | PMT_CTL_ETH_PHY_WAKE_EN_);
3479
3480     macrx = lan743x_csr_read(adapter, MAC_RX);
3481
3482     wucsr = 0;
3483     mask_index = 0;
3484
3485     pmtctl |= PMT_CTL_ETH_PHY_D3_COLD_OVR_ | PMT_CTL_ETH_PHY_D3_OVR_;
3486
3487     if (adapter->wolopts & WAKE_PHY) {
3488         pmtctl |= PMT_CTL_ETH_PHY_EDPD_PLL_CTL_;
3489         pmtctl |= PMT_CTL_ETH_PHY_WAKE_EN_;
3490     }
3491     if (adapter->wolopts & WAKE_MAGIC) {
3492         wucsr |= MAC_WUCSR_MPEN_;
3493         macrx |= MAC_RX_RXEN_;
3494         pmtctl |= PMT_CTL_WOL_EN_ | PMT_CTL_MAC_D3_RX_CLK_OVR_;
3495     }
3496     if (adapter->wolopts & WAKE_UCAST) {
3497         wucsr |= MAC_WUCSR_RFE_WAKE_EN_ | MAC_WUCSR_PFDA_EN_;
3498         macrx |= MAC_RX_RXEN_;
3499         pmtctl |= PMT_CTL_WOL_EN_ | PMT_CTL_MAC_D3_RX_CLK_OVR_;
3500         pmtctl |= PMT_CTL_RX_FCT_RFE_D3_CLK_OVR_;
3501     }
3502     if (adapter->wolopts & WAKE_BCAST) {
3503         wucsr |= MAC_WUCSR_RFE_WAKE_EN_ | MAC_WUCSR_BCST_EN_;
3504         macrx |= MAC_RX_RXEN_;
3505         pmtctl |= PMT_CTL_WOL_EN_ | PMT_CTL_MAC_D3_RX_CLK_OVR_;
3506         pmtctl |= PMT_CTL_RX_FCT_RFE_D3_CLK_OVR_;
3507     }
3508     if (adapter->wolopts & WAKE_MCAST) {
3509         /* IPv4 multicast */
3510         crc = lan743x_pm_wakeframe_crc16(ipv4_multicast, 3);
3511         lan743x_csr_write(adapter, MAC_WUF_CFG(mask_index),
3512                   MAC_WUF_CFG_EN_ | MAC_WUF_CFG_TYPE_MCAST_ |
3513                   (0 << MAC_WUF_CFG_OFFSET_SHIFT_) |
3514                   (crc & MAC_WUF_CFG_CRC16_MASK_));
3515         lan743x_csr_write(adapter, MAC_WUF_MASK0(mask_index), 7);
3516         lan743x_csr_write(adapter, MAC_WUF_MASK1(mask_index), 0);
3517         lan743x_csr_write(adapter, MAC_WUF_MASK2(mask_index), 0);
3518         lan743x_csr_write(adapter, MAC_WUF_MASK3(mask_index), 0);
3519         mask_index++;
3520
3521         /* IPv6 multicast */
3522         crc = lan743x_pm_wakeframe_crc16(ipv6_multicast, 2);
3523         lan743x_csr_write(adapter, MAC_WUF_CFG(mask_index),
3524                   MAC_WUF_CFG_EN_ | MAC_WUF_CFG_TYPE_MCAST_ |
3525                   (0 << MAC_WUF_CFG_OFFSET_SHIFT_) |
3526                   (crc & MAC_WUF_CFG_CRC16_MASK_));
3527         lan743x_csr_write(adapter, MAC_WUF_MASK0(mask_index), 3);
3528         lan743x_csr_write(adapter, MAC_WUF_MASK1(mask_index), 0);
3529         lan743x_csr_write(adapter, MAC_WUF_MASK2(mask_index), 0);
3530         lan743x_csr_write(adapter, MAC_WUF_MASK3(mask_index), 0);
3531         mask_index++;
3532
3533         wucsr |= MAC_WUCSR_RFE_WAKE_EN_ | MAC_WUCSR_WAKE_EN_;
3534         macrx |= MAC_RX_RXEN_;
3535         pmtctl |= PMT_CTL_WOL_EN_ | PMT_CTL_MAC_D3_RX_CLK_OVR_;
3536         pmtctl |= PMT_CTL_RX_FCT_RFE_D3_CLK_OVR_;
3537     }
3538     if (adapter->wolopts & WAKE_ARP) {
3539         /* set MAC_WUF_CFG & WUF_MASK
3540          * for packettype (offset 12,13) = ARP (0x0806)
3541          */
3542         crc = lan743x_pm_wakeframe_crc16(arp_type, 2);
3543         lan743x_csr_write(adapter, MAC_WUF_CFG(mask_index),
3544                   MAC_WUF_CFG_EN_ | MAC_WUF_CFG_TYPE_ALL_ |
3545                   (0 << MAC_WUF_CFG_OFFSET_SHIFT_) |
3546                   (crc & MAC_WUF_CFG_CRC16_MASK_));
3547         lan743x_csr_write(adapter, MAC_WUF_MASK0(mask_index), 0x3000);
3548         lan743x_csr_write(adapter, MAC_WUF_MASK1(mask_index), 0);
3549         lan743x_csr_write(adapter, MAC_WUF_MASK2(mask_index), 0);
3550         lan743x_csr_write(adapter, MAC_WUF_MASK3(mask_index), 0);
3551         mask_index++;
3552
3553         wucsr |= MAC_WUCSR_RFE_WAKE_EN_ | MAC_WUCSR_WAKE_EN_;
3554         macrx |= MAC_RX_RXEN_;
3555         pmtctl |= PMT_CTL_WOL_EN_ | PMT_CTL_MAC_D3_RX_CLK_OVR_;
3556         pmtctl |= PMT_CTL_RX_FCT_RFE_D3_CLK_OVR_;
3557     }
3558
3559     if (adapter->wolopts & WAKE_MAGICSECURE) {
3560         sopass = *(u32 *)adapter->sopass;
3561         lan743x_csr_write(adapter, MAC_MP_SO_LO, sopass);
3562         sopass = *(u16 *)&adapter->sopass[4];
3563         lan743x_csr_write(adapter, MAC_MP_SO_HI, sopass);
3564         wucsr |= MAC_MP_SO_EN_;
3565     }
3566
3567     lan743x_csr_write(adapter, MAC_WUCSR, wucsr);
3568     lan743x_csr_write(adapter, PMT_CTL, pmtctl);
3569     lan743x_csr_write(adapter, MAC_RX, macrx);
3570 }
3571
3572 static int lan743x_pm_suspend(struct device *dev)
3573 {
3574     struct pci_dev *pdev = to_pci_dev(dev);
3575     struct net_device *netdev = pci_get_drvdata(pdev);
3576     struct lan743x_adapter *adapter = netdev_priv(netdev);
3577     u32 data;
3578
3579     lan743x_pcidev_shutdown(pdev);
3580
3581     /* clear all wakes */
3582     lan743x_csr_write(adapter, MAC_WUCSR, 0);
3583     lan743x_csr_write(adapter, MAC_WUCSR2, 0);
3584     lan743x_csr_write(adapter, MAC_WK_SRC, 0xFFFFFFFF);
3585
3586     if (adapter->wolopts)
3587         lan743x_pm_set_wol(adapter);
3588
3589     if (adapter->is_pci11x1x) {
3590         /* Save HW_CFG to config again in PM resume */
3591         data = lan743x_csr_read(adapter, HW_CFG);
3592         adapter->hw_cfg = data;
3593         data |= (HW_CFG_RST_PROTECT_PCIE_ |
3594              HW_CFG_D3_RESET_DIS_ |
3595              HW_CFG_D3_VAUX_OVR_ |
3596              HW_CFG_HOT_RESET_DIS_ |
3597              HW_CFG_RST_PROTECT_);
3598         lan743x_csr_write(adapter, HW_CFG, data);
3599     }
3600
3601     /* Host sets PME_En, put D3hot */
3602     return pci_prepare_to_sleep(pdev);
3603 }
3604
3605 static int lan743x_pm_resume(struct device *dev)
3606 {
3607     struct pci_dev *pdev = to_pci_dev(dev);
3608     struct net_device *netdev = pci_get_drvdata(pdev);
3609     struct lan743x_adapter *adapter = netdev_priv(netdev);
3610     int ret;
3611
3612     pci_set_power_state(pdev, PCI_D0);
3613     pci_restore_state(pdev);
3614     pci_save_state(pdev);
3615
3616     /* Restore HW_CFG that was saved during pm suspend */
3617     if (adapter->is_pci11x1x)
3618         lan743x_csr_write(adapter, HW_CFG, adapter->hw_cfg);
3619
3620     ret = lan743x_hardware_init(adapter, pdev);
3621     if (ret) {
3622         netif_err(adapter, probe, adapter->netdev,
3623               "lan743x_hardware_init returned %d\n", ret);
3624         lan743x_pci_cleanup(adapter);
3625         return ret;
3626     }
3627
3628     /* open netdev when netdev is at running state while resume.
3629      * For instance, it is true when system wakesup after pm-suspend
3630      * However, it is false when system wakes up after suspend GUI menu
3631      */
3632     if (netif_running(netdev))
3633         lan743x_netdev_open(netdev);
3634
3635     netif_device_attach(netdev);
3636     ret = lan743x_csr_read(adapter, MAC_WK_SRC);
3637     netif_info(adapter, drv, adapter->netdev,
3638            "Wakeup source : 0x%08X\n", ret);
3639
3640     return 0;
3641 }
3642
3643 static const struct dev_pm_ops lan743x_pm_ops = {
3644     SET_SYSTEM_SLEEP_PM_OPS(lan743x_pm_suspend, lan743x_pm_resume)
3645 };
3646 #endif /* CONFIG_PM_SLEEP */
3647
3648 static const struct pci_device_id lan743x_pcidev_tbl[] = {
3649     { PCI_DEVICE(PCI_VENDOR_ID_SMSC, PCI_DEVICE_ID_SMSC_LAN7430) },
3650     { PCI_DEVICE(PCI_VENDOR_ID_SMSC, PCI_DEVICE_ID_SMSC_LAN7431) },
3651     { PCI_DEVICE(PCI_VENDOR_ID_SMSC, PCI_DEVICE_ID_SMSC_A011) },
3652     { PCI_DEVICE(PCI_VENDOR_ID_SMSC, PCI_DEVICE_ID_SMSC_A041) },
3653     { 0, }
3654 };
3655
3656 MODULE_DEVICE_TABLE(pci, lan743x_pcidev_tbl);
3657
3658 static struct pci_driver lan743x_pcidev_driver = {
3659     .name     = DRIVER_NAME,
3660     .id_table = lan743x_pcidev_tbl,
3661     .probe    = lan743x_pcidev_probe,
3662     .remove   = lan743x_pcidev_remove,
3663 #ifdef CONFIG_PM_SLEEP
3664     .driver.pm = &lan743x_pm_ops,
3665 #endif
3666     .shutdown = lan743x_pcidev_shutdown,
3667 };
3668
3669 module_pci_driver(lan743x_pcidev_driver);
3670
3671 MODULE_AUTHOR(DRIVER_AUTHOR);
3672 MODULE_DESCRIPTION(DRIVER_DESC);
3673 MODULE_LICENSE("GPL");