ethernet/mediatek/mtk_eth_soc.c

0001 // SPDX-License-Identifier: GPL-2.0-only
0002 /*
0003  *
0004  *   Copyright (C) 2009-2016 John Crispin <blogic@openwrt.org>
0005  *   Copyright (C) 2009-2016 Felix Fietkau <nbd@openwrt.org>
0006  *   Copyright (C) 2013-2016 Michael Lee <igvtee@gmail.com>
0007  */
0008
0009 #include <linux/of_device.h>
0010 #include <linux/of_mdio.h>
0011 #include <linux/of_net.h>
0012 #include <linux/of_address.h>
0013 #include <linux/mfd/syscon.h>
0014 #include <linux/regmap.h>
0015 #include <linux/clk.h>
0016 #include <linux/pm_runtime.h>
0017 #include <linux/if_vlan.h>
0018 #include <linux/reset.h>
0019 #include <linux/tcp.h>
0020 #include <linux/interrupt.h>
0021 #include <linux/pinctrl/devinfo.h>
0022 #include <linux/phylink.h>
0023 #include <linux/jhash.h>
0024 #include <linux/bitfield.h>
0025 #include <net/dsa.h>
0026
0027 #include "mtk_eth_soc.h"
0028 #include "mtk_wed.h"
0029
0030 static int mtk_msg_level = -1;
0031 module_param_named(msg_level, mtk_msg_level, int, 0);
0032 MODULE_PARM_DESC(msg_level, "Message level (-1=defaults,0=none,...,16=all)");
0033
0034 #define MTK_ETHTOOL_STAT(x) { #x, \
0035                   offsetof(struct mtk_hw_stats, x) / sizeof(u64) }
0036
0037 #define MTK_ETHTOOL_XDP_STAT(x) { #x, \
0038                   offsetof(struct mtk_hw_stats, xdp_stats.x) / \
0039                   sizeof(u64) }
0040
0041 static const struct mtk_reg_map mtk_reg_map = {
0042     .tx_irq_mask        = 0x1a1c,
0043     .tx_irq_status      = 0x1a18,
0044     .pdma = {
0045         .rx_ptr     = 0x0900,
0046         .rx_cnt_cfg = 0x0904,
0047         .pcrx_ptr   = 0x0908,
0048         .glo_cfg    = 0x0a04,
0049         .rst_idx    = 0x0a08,
0050         .delay_irq  = 0x0a0c,
0051         .irq_status = 0x0a20,
0052         .irq_mask   = 0x0a28,
0053         .int_grp    = 0x0a50,
0054     },
0055     .qdma = {
0056         .qtx_cfg    = 0x1800,
0057         .rx_ptr     = 0x1900,
0058         .rx_cnt_cfg = 0x1904,
0059         .qcrx_ptr   = 0x1908,
0060         .glo_cfg    = 0x1a04,
0061         .rst_idx    = 0x1a08,
0062         .delay_irq  = 0x1a0c,
0063         .fc_th      = 0x1a10,
0064         .int_grp    = 0x1a20,
0065         .hred       = 0x1a44,
0066         .ctx_ptr    = 0x1b00,
0067         .dtx_ptr    = 0x1b04,
0068         .crx_ptr    = 0x1b10,
0069         .drx_ptr    = 0x1b14,
0070         .fq_head    = 0x1b20,
0071         .fq_tail    = 0x1b24,
0072         .fq_count   = 0x1b28,
0073         .fq_blen    = 0x1b2c,
0074     },
0075     .gdm1_cnt       = 0x2400,
0076 };
0077
0078 static const struct mtk_reg_map mt7628_reg_map = {
0079     .tx_irq_mask        = 0x0a28,
0080     .tx_irq_status      = 0x0a20,
0081     .pdma = {
0082         .rx_ptr     = 0x0900,
0083         .rx_cnt_cfg = 0x0904,
0084         .pcrx_ptr   = 0x0908,
0085         .glo_cfg    = 0x0a04,
0086         .rst_idx    = 0x0a08,
0087         .delay_irq  = 0x0a0c,
0088         .irq_status = 0x0a20,
0089         .irq_mask   = 0x0a28,
0090         .int_grp    = 0x0a50,
0091     },
0092 };
0093
0094 static const struct mtk_reg_map mt7986_reg_map = {
0095     .tx_irq_mask        = 0x461c,
0096     .tx_irq_status      = 0x4618,
0097     .pdma = {
0098         .rx_ptr     = 0x6100,
0099         .rx_cnt_cfg = 0x6104,
0100         .pcrx_ptr   = 0x6108,
0101         .glo_cfg    = 0x6204,
0102         .rst_idx    = 0x6208,
0103         .delay_irq  = 0x620c,
0104         .irq_status = 0x6220,
0105         .irq_mask   = 0x6228,
0106         .int_grp    = 0x6250,
0107     },
0108     .qdma = {
0109         .qtx_cfg    = 0x4400,
0110         .rx_ptr     = 0x4500,
0111         .rx_cnt_cfg = 0x4504,
0112         .qcrx_ptr   = 0x4508,
0113         .glo_cfg    = 0x4604,
0114         .rst_idx    = 0x4608,
0115         .delay_irq  = 0x460c,
0116         .fc_th      = 0x4610,
0117         .int_grp    = 0x4620,
0118         .hred       = 0x4644,
0119         .ctx_ptr    = 0x4700,
0120         .dtx_ptr    = 0x4704,
0121         .crx_ptr    = 0x4710,
0122         .drx_ptr    = 0x4714,
0123         .fq_head    = 0x4720,
0124         .fq_tail    = 0x4724,
0125         .fq_count   = 0x4728,
0126         .fq_blen    = 0x472c,
0127     },
0128     .gdm1_cnt       = 0x1c00,
0129 };
0130
0131 /* strings used by ethtool */
0132 static const struct mtk_ethtool_stats {
0133     char str[ETH_GSTRING_LEN];
0134     u32 offset;
0135 } mtk_ethtool_stats[] = {
0136     MTK_ETHTOOL_STAT(tx_bytes),
0137     MTK_ETHTOOL_STAT(tx_packets),
0138     MTK_ETHTOOL_STAT(tx_skip),
0139     MTK_ETHTOOL_STAT(tx_collisions),
0140     MTK_ETHTOOL_STAT(rx_bytes),
0141     MTK_ETHTOOL_STAT(rx_packets),
0142     MTK_ETHTOOL_STAT(rx_overflow),
0143     MTK_ETHTOOL_STAT(rx_fcs_errors),
0144     MTK_ETHTOOL_STAT(rx_short_errors),
0145     MTK_ETHTOOL_STAT(rx_long_errors),
0146     MTK_ETHTOOL_STAT(rx_checksum_errors),
0147     MTK_ETHTOOL_STAT(rx_flow_control_packets),
0148     MTK_ETHTOOL_XDP_STAT(rx_xdp_redirect),
0149     MTK_ETHTOOL_XDP_STAT(rx_xdp_pass),
0150     MTK_ETHTOOL_XDP_STAT(rx_xdp_drop),
0151     MTK_ETHTOOL_XDP_STAT(rx_xdp_tx),
0152     MTK_ETHTOOL_XDP_STAT(rx_xdp_tx_errors),
0153     MTK_ETHTOOL_XDP_STAT(tx_xdp_xmit),
0154     MTK_ETHTOOL_XDP_STAT(tx_xdp_xmit_errors),
0155 };
0156
0157 static const char * const mtk_clks_source_name[] = {
0158     "ethif", "sgmiitop", "esw", "gp0", "gp1", "gp2", "fe", "trgpll",
0159     "sgmii_tx250m", "sgmii_rx250m", "sgmii_cdr_ref", "sgmii_cdr_fb",
0160     "sgmii2_tx250m", "sgmii2_rx250m", "sgmii2_cdr_ref", "sgmii2_cdr_fb",
0161     "sgmii_ck", "eth2pll", "wocpu0", "wocpu1", "netsys0", "netsys1"
0162 };
0163
0164 void mtk_w32(struct mtk_eth *eth, u32 val, unsigned reg)
0165 {
0166     __raw_writel(val, eth->base + reg);
0167 }
0168
0169 u32 mtk_r32(struct mtk_eth *eth, unsigned reg)
0170 {
0171     return __raw_readl(eth->base + reg);
0172 }
0173
0174 static u32 mtk_m32(struct mtk_eth *eth, u32 mask, u32 set, unsigned reg)
0175 {
0176     u32 val;
0177
0178     val = mtk_r32(eth, reg);
0179     val &= ~mask;
0180     val |= set;
0181     mtk_w32(eth, val, reg);
0182     return reg;
0183 }
0184
0185 static int mtk_mdio_busy_wait(struct mtk_eth *eth)
0186 {
0187     unsigned long t_start = jiffies;
0188
0189     while (1) {
0190         if (!(mtk_r32(eth, MTK_PHY_IAC) & PHY_IAC_ACCESS))
0191             return 0;
0192         if (time_after(jiffies, t_start + PHY_IAC_TIMEOUT))
0193             break;
0194         cond_resched();
0195     }
0196
0197     dev_err(eth->dev, "mdio: MDIO timeout\n");
0198     return -ETIMEDOUT;
0199 }
0200
0201 static int _mtk_mdio_write(struct mtk_eth *eth, u32 phy_addr, u32 phy_reg,
0202                u32 write_data)
0203 {
0204     int ret;
0205
0206     ret = mtk_mdio_busy_wait(eth);
0207     if (ret < 0)
0208         return ret;
0209
0210     if (phy_reg & MII_ADDR_C45) {
0211         mtk_w32(eth, PHY_IAC_ACCESS |
0212                  PHY_IAC_START_C45 |
0213                  PHY_IAC_CMD_C45_ADDR |
0214                  PHY_IAC_REG(mdiobus_c45_devad(phy_reg)) |
0215                  PHY_IAC_ADDR(phy_addr) |
0216                  PHY_IAC_DATA(mdiobus_c45_regad(phy_reg)),
0217             MTK_PHY_IAC);
0218
0219         ret = mtk_mdio_busy_wait(eth);
0220         if (ret < 0)
0221             return ret;
0222
0223         mtk_w32(eth, PHY_IAC_ACCESS |
0224                  PHY_IAC_START_C45 |
0225                  PHY_IAC_CMD_WRITE |
0226                  PHY_IAC_REG(mdiobus_c45_devad(phy_reg)) |
0227                  PHY_IAC_ADDR(phy_addr) |
0228                  PHY_IAC_DATA(write_data),
0229             MTK_PHY_IAC);
0230     } else {
0231         mtk_w32(eth, PHY_IAC_ACCESS |
0232                  PHY_IAC_START_C22 |
0233                  PHY_IAC_CMD_WRITE |
0234                  PHY_IAC_REG(phy_reg) |
0235                  PHY_IAC_ADDR(phy_addr) |
0236                  PHY_IAC_DATA(write_data),
0237             MTK_PHY_IAC);
0238     }
0239
0240     ret = mtk_mdio_busy_wait(eth);
0241     if (ret < 0)
0242         return ret;
0243
0244     return 0;
0245 }
0246
0247 static int _mtk_mdio_read(struct mtk_eth *eth, u32 phy_addr, u32 phy_reg)
0248 {
0249     int ret;
0250
0251     ret = mtk_mdio_busy_wait(eth);
0252     if (ret < 0)
0253         return ret;
0254
0255     if (phy_reg & MII_ADDR_C45) {
0256         mtk_w32(eth, PHY_IAC_ACCESS |
0257                  PHY_IAC_START_C45 |
0258                  PHY_IAC_CMD_C45_ADDR |
0259                  PHY_IAC_REG(mdiobus_c45_devad(phy_reg)) |
0260                  PHY_IAC_ADDR(phy_addr) |
0261                  PHY_IAC_DATA(mdiobus_c45_regad(phy_reg)),
0262             MTK_PHY_IAC);
0263
0264         ret = mtk_mdio_busy_wait(eth);
0265         if (ret < 0)
0266             return ret;
0267
0268         mtk_w32(eth, PHY_IAC_ACCESS |
0269                  PHY_IAC_START_C45 |
0270                  PHY_IAC_CMD_C45_READ |
0271                  PHY_IAC_REG(mdiobus_c45_devad(phy_reg)) |
0272                  PHY_IAC_ADDR(phy_addr),
0273             MTK_PHY_IAC);
0274     } else {
0275         mtk_w32(eth, PHY_IAC_ACCESS |
0276                  PHY_IAC_START_C22 |
0277                  PHY_IAC_CMD_C22_READ |
0278                  PHY_IAC_REG(phy_reg) |
0279                  PHY_IAC_ADDR(phy_addr),
0280             MTK_PHY_IAC);
0281     }
0282
0283     ret = mtk_mdio_busy_wait(eth);
0284     if (ret < 0)
0285         return ret;
0286
0287     return mtk_r32(eth, MTK_PHY_IAC) & PHY_IAC_DATA_MASK;
0288 }
0289
0290 static int mtk_mdio_write(struct mii_bus *bus, int phy_addr,
0291               int phy_reg, u16 val)
0292 {
0293     struct mtk_eth *eth = bus->priv;
0294
0295     return _mtk_mdio_write(eth, phy_addr, phy_reg, val);
0296 }
0297
0298 static int mtk_mdio_read(struct mii_bus *bus, int phy_addr, int phy_reg)
0299 {
0300     struct mtk_eth *eth = bus->priv;
0301
0302     return _mtk_mdio_read(eth, phy_addr, phy_reg);
0303 }
0304
0305 static int mt7621_gmac0_rgmii_adjust(struct mtk_eth *eth,
0306                      phy_interface_t interface)
0307 {
0308     u32 val;
0309
0310     /* Check DDR memory type.
0311      * Currently TRGMII mode with DDR2 memory is not supported.
0312      */
0313     regmap_read(eth->ethsys, ETHSYS_SYSCFG, &val);
0314     if (interface == PHY_INTERFACE_MODE_TRGMII &&
0315         val & SYSCFG_DRAM_TYPE_DDR2) {
0316         dev_err(eth->dev,
0317             "TRGMII mode with DDR2 memory is not supported!\n");
0318         return -EOPNOTSUPP;
0319     }
0320
0321     val = (interface == PHY_INTERFACE_MODE_TRGMII) ?
0322         ETHSYS_TRGMII_MT7621_DDR_PLL : 0;
0323
0324     regmap_update_bits(eth->ethsys, ETHSYS_CLKCFG0,
0325                ETHSYS_TRGMII_MT7621_MASK, val);
0326
0327     return 0;
0328 }
0329
0330 static void mtk_gmac0_rgmii_adjust(struct mtk_eth *eth,
0331                    phy_interface_t interface, int speed)
0332 {
0333     u32 val;
0334     int ret;
0335
0336     if (interface == PHY_INTERFACE_MODE_TRGMII) {
0337         mtk_w32(eth, TRGMII_MODE, INTF_MODE);
0338         val = 500000000;
0339         ret = clk_set_rate(eth->clks[MTK_CLK_TRGPLL], val);
0340         if (ret)
0341             dev_err(eth->dev, "Failed to set trgmii pll: %d\n", ret);
0342         return;
0343     }
0344
0345     val = (speed == SPEED_1000) ?
0346         INTF_MODE_RGMII_1000 : INTF_MODE_RGMII_10_100;
0347     mtk_w32(eth, val, INTF_MODE);
0348
0349     regmap_update_bits(eth->ethsys, ETHSYS_CLKCFG0,
0350                ETHSYS_TRGMII_CLK_SEL362_5,
0351                ETHSYS_TRGMII_CLK_SEL362_5);
0352
0353     val = (speed == SPEED_1000) ? 250000000 : 500000000;
0354     ret = clk_set_rate(eth->clks[MTK_CLK_TRGPLL], val);
0355     if (ret)
0356         dev_err(eth->dev, "Failed to set trgmii pll: %d\n", ret);
0357
0358     val = (speed == SPEED_1000) ?
0359         RCK_CTRL_RGMII_1000 : RCK_CTRL_RGMII_10_100;
0360     mtk_w32(eth, val, TRGMII_RCK_CTRL);
0361
0362     val = (speed == SPEED_1000) ?
0363         TCK_CTRL_RGMII_1000 : TCK_CTRL_RGMII_10_100;
0364     mtk_w32(eth, val, TRGMII_TCK_CTRL);
0365 }
0366
0367 static struct phylink_pcs *mtk_mac_select_pcs(struct phylink_config *config,
0368                           phy_interface_t interface)
0369 {
0370     struct mtk_mac *mac = container_of(config, struct mtk_mac,
0371                        phylink_config);
0372     struct mtk_eth *eth = mac->hw;
0373     unsigned int sid;
0374
0375     if (interface == PHY_INTERFACE_MODE_SGMII ||
0376         phy_interface_mode_is_8023z(interface)) {
0377         sid = (MTK_HAS_CAPS(eth->soc->caps, MTK_SHARED_SGMII)) ?
0378                0 : mac->id;
0379
0380         return mtk_sgmii_select_pcs(eth->sgmii, sid);
0381     }
0382
0383     return NULL;
0384 }
0385
0386 static void mtk_mac_config(struct phylink_config *config, unsigned int mode,
0387                const struct phylink_link_state *state)
0388 {
0389     struct mtk_mac *mac = container_of(config, struct mtk_mac,
0390                        phylink_config);
0391     struct mtk_eth *eth = mac->hw;
0392     int val, ge_mode, err = 0;
0393     u32 i;
0394
0395     /* MT76x8 has no hardware settings between for the MAC */
0396     if (!MTK_HAS_CAPS(eth->soc->caps, MTK_SOC_MT7628) &&
0397         mac->interface != state->interface) {
0398         /* Setup soc pin functions */
0399         switch (state->interface) {
0400         case PHY_INTERFACE_MODE_TRGMII:
0401             if (mac->id)
0402                 goto err_phy;
0403             if (!MTK_HAS_CAPS(mac->hw->soc->caps,
0404                       MTK_GMAC1_TRGMII))
0405                 goto err_phy;
0406             fallthrough;
0407         case PHY_INTERFACE_MODE_RGMII_TXID:
0408         case PHY_INTERFACE_MODE_RGMII_RXID:
0409         case PHY_INTERFACE_MODE_RGMII_ID:
0410         case PHY_INTERFACE_MODE_RGMII:
0411         case PHY_INTERFACE_MODE_MII:
0412         case PHY_INTERFACE_MODE_REVMII:
0413         case PHY_INTERFACE_MODE_RMII:
0414             if (MTK_HAS_CAPS(eth->soc->caps, MTK_RGMII)) {
0415                 err = mtk_gmac_rgmii_path_setup(eth, mac->id);
0416                 if (err)
0417                     goto init_err;
0418             }
0419             break;
0420         case PHY_INTERFACE_MODE_1000BASEX:
0421         case PHY_INTERFACE_MODE_2500BASEX:
0422         case PHY_INTERFACE_MODE_SGMII:
0423             if (MTK_HAS_CAPS(eth->soc->caps, MTK_SGMII)) {
0424                 err = mtk_gmac_sgmii_path_setup(eth, mac->id);
0425                 if (err)
0426                     goto init_err;
0427             }
0428             break;
0429         case PHY_INTERFACE_MODE_GMII:
0430             if (MTK_HAS_CAPS(eth->soc->caps, MTK_GEPHY)) {
0431                 err = mtk_gmac_gephy_path_setup(eth, mac->id);
0432                 if (err)
0433                     goto init_err;
0434             }
0435             break;
0436         default:
0437             goto err_phy;
0438         }
0439
0440         /* Setup clock for 1st gmac */
0441         if (!mac->id && state->interface != PHY_INTERFACE_MODE_SGMII &&
0442             !phy_interface_mode_is_8023z(state->interface) &&
0443             MTK_HAS_CAPS(mac->hw->soc->caps, MTK_GMAC1_TRGMII)) {
0444             if (MTK_HAS_CAPS(mac->hw->soc->caps,
0445                      MTK_TRGMII_MT7621_CLK)) {
0446                 if (mt7621_gmac0_rgmii_adjust(mac->hw,
0447                                   state->interface))
0448                     goto err_phy;
0449             } else {
0450                 /* FIXME: this is incorrect. Not only does it
0451                  * use state->speed (which is not guaranteed
0452                  * to be correct) but it also makes use of it
0453                  * in a code path that will only be reachable
0454                  * when the PHY interface mode changes, not
0455                  * when the speed changes. Consequently, RGMII
0456                  * is probably broken.
0457                  */
0458                 mtk_gmac0_rgmii_adjust(mac->hw,
0459                                state->interface,
0460                                state->speed);
0461
0462                 /* mt7623_pad_clk_setup */
0463                 for (i = 0 ; i < NUM_TRGMII_CTRL; i++)
0464                     mtk_w32(mac->hw,
0465                         TD_DM_DRVP(8) | TD_DM_DRVN(8),
0466                         TRGMII_TD_ODT(i));
0467
0468                 /* Assert/release MT7623 RXC reset */
0469                 mtk_m32(mac->hw, 0, RXC_RST | RXC_DQSISEL,
0470                     TRGMII_RCK_CTRL);
0471                 mtk_m32(mac->hw, RXC_RST, 0, TRGMII_RCK_CTRL);
0472             }
0473         }
0474
0475         ge_mode = 0;
0476         switch (state->interface) {
0477         case PHY_INTERFACE_MODE_MII:
0478         case PHY_INTERFACE_MODE_GMII:
0479             ge_mode = 1;
0480             break;
0481         case PHY_INTERFACE_MODE_REVMII:
0482             ge_mode = 2;
0483             break;
0484         case PHY_INTERFACE_MODE_RMII:
0485             if (mac->id)
0486                 goto err_phy;
0487             ge_mode = 3;
0488             break;
0489         default:
0490             break;
0491         }
0492
0493         /* put the gmac into the right mode */
0494         regmap_read(eth->ethsys, ETHSYS_SYSCFG0, &val);
0495         val &= ~SYSCFG0_GE_MODE(SYSCFG0_GE_MASK, mac->id);
0496         val |= SYSCFG0_GE_MODE(ge_mode, mac->id);
0497         regmap_write(eth->ethsys, ETHSYS_SYSCFG0, val);
0498
0499         mac->interface = state->interface;
0500     }
0501
0502     /* SGMII */
0503     if (state->interface == PHY_INTERFACE_MODE_SGMII ||
0504         phy_interface_mode_is_8023z(state->interface)) {
0505         /* The path GMAC to SGMII will be enabled once the SGMIISYS is
0506          * being setup done.
0507          */
0508         regmap_read(eth->ethsys, ETHSYS_SYSCFG0, &val);
0509
0510         regmap_update_bits(eth->ethsys, ETHSYS_SYSCFG0,
0511                    SYSCFG0_SGMII_MASK,
0512                    ~(u32)SYSCFG0_SGMII_MASK);
0513
0514         /* Save the syscfg0 value for mac_finish */
0515         mac->syscfg0 = val;
0516     } else if (phylink_autoneg_inband(mode)) {
0517         dev_err(eth->dev,
0518             "In-band mode not supported in non SGMII mode!\n");
0519         return;
0520     }
0521
0522     return;
0523
0524 err_phy:
0525     dev_err(eth->dev, "%s: GMAC%d mode %s not supported!\n", __func__,
0526         mac->id, phy_modes(state->interface));
0527     return;
0528
0529 init_err:
0530     dev_err(eth->dev, "%s: GMAC%d mode %s err: %d!\n", __func__,
0531         mac->id, phy_modes(state->interface), err);
0532 }
0533
0534 static int mtk_mac_finish(struct phylink_config *config, unsigned int mode,
0535               phy_interface_t interface)
0536 {
0537     struct mtk_mac *mac = container_of(config, struct mtk_mac,
0538                        phylink_config);
0539     struct mtk_eth *eth = mac->hw;
0540     u32 mcr_cur, mcr_new;
0541
0542     /* Enable SGMII */
0543     if (interface == PHY_INTERFACE_MODE_SGMII ||
0544         phy_interface_mode_is_8023z(interface))
0545         regmap_update_bits(eth->ethsys, ETHSYS_SYSCFG0,
0546                    SYSCFG0_SGMII_MASK, mac->syscfg0);
0547
0548     /* Setup gmac */
0549     mcr_cur = mtk_r32(mac->hw, MTK_MAC_MCR(mac->id));
0550     mcr_new = mcr_cur;
0551     mcr_new |= MAC_MCR_IPG_CFG | MAC_MCR_FORCE_MODE |
0552            MAC_MCR_BACKOFF_EN | MAC_MCR_BACKPR_EN | MAC_MCR_FORCE_LINK;
0553
0554     /* Only update control register when needed! */
0555     if (mcr_new != mcr_cur)
0556         mtk_w32(mac->hw, mcr_new, MTK_MAC_MCR(mac->id));
0557
0558     return 0;
0559 }
0560
0561 static void mtk_mac_pcs_get_state(struct phylink_config *config,
0562                   struct phylink_link_state *state)
0563 {
0564     struct mtk_mac *mac = container_of(config, struct mtk_mac,
0565                        phylink_config);
0566     u32 pmsr = mtk_r32(mac->hw, MTK_MAC_MSR(mac->id));
0567
0568     state->link = (pmsr & MAC_MSR_LINK);
0569     state->duplex = (pmsr & MAC_MSR_DPX) >> 1;
0570
0571     switch (pmsr & (MAC_MSR_SPEED_1000 | MAC_MSR_SPEED_100)) {
0572     case 0:
0573         state->speed = SPEED_10;
0574         break;
0575     case MAC_MSR_SPEED_100:
0576         state->speed = SPEED_100;
0577         break;
0578     case MAC_MSR_SPEED_1000:
0579         state->speed = SPEED_1000;
0580         break;
0581     default:
0582         state->speed = SPEED_UNKNOWN;
0583         break;
0584     }
0585
0586     state->pause &= (MLO_PAUSE_RX | MLO_PAUSE_TX);
0587     if (pmsr & MAC_MSR_RX_FC)
0588         state->pause |= MLO_PAUSE_RX;
0589     if (pmsr & MAC_MSR_TX_FC)
0590         state->pause |= MLO_PAUSE_TX;
0591 }
0592
0593 static void mtk_mac_link_down(struct phylink_config *config, unsigned int mode,
0594                   phy_interface_t interface)
0595 {
0596     struct mtk_mac *mac = container_of(config, struct mtk_mac,
0597                        phylink_config);
0598     u32 mcr = mtk_r32(mac->hw, MTK_MAC_MCR(mac->id));
0599
0600     mcr &= ~(MAC_MCR_TX_EN | MAC_MCR_RX_EN);
0601     mtk_w32(mac->hw, mcr, MTK_MAC_MCR(mac->id));
0602 }
0603
0604 static void mtk_mac_link_up(struct phylink_config *config,
0605                 struct phy_device *phy,
0606                 unsigned int mode, phy_interface_t interface,
0607                 int speed, int duplex, bool tx_pause, bool rx_pause)
0608 {
0609     struct mtk_mac *mac = container_of(config, struct mtk_mac,
0610                        phylink_config);
0611     u32 mcr;
0612
0613     mcr = mtk_r32(mac->hw, MTK_MAC_MCR(mac->id));
0614     mcr &= ~(MAC_MCR_SPEED_100 | MAC_MCR_SPEED_1000 |
0615          MAC_MCR_FORCE_DPX | MAC_MCR_FORCE_TX_FC |
0616          MAC_MCR_FORCE_RX_FC);
0617
0618     /* Configure speed */
0619     switch (speed) {
0620     case SPEED_2500:
0621     case SPEED_1000:
0622         mcr |= MAC_MCR_SPEED_1000;
0623         break;
0624     case SPEED_100:
0625         mcr |= MAC_MCR_SPEED_100;
0626         break;
0627     }
0628
0629     /* Configure duplex */
0630     if (duplex == DUPLEX_FULL)
0631         mcr |= MAC_MCR_FORCE_DPX;
0632
0633     /* Configure pause modes - phylink will avoid these for half duplex */
0634     if (tx_pause)
0635         mcr |= MAC_MCR_FORCE_TX_FC;
0636     if (rx_pause)
0637         mcr |= MAC_MCR_FORCE_RX_FC;
0638
0639     mcr |= MAC_MCR_TX_EN | MAC_MCR_RX_EN;
0640     mtk_w32(mac->hw, mcr, MTK_MAC_MCR(mac->id));
0641 }
0642
0643 static const struct phylink_mac_ops mtk_phylink_ops = {
0644     .validate = phylink_generic_validate,
0645     .mac_select_pcs = mtk_mac_select_pcs,
0646     .mac_pcs_get_state = mtk_mac_pcs_get_state,
0647     .mac_config = mtk_mac_config,
0648     .mac_finish = mtk_mac_finish,
0649     .mac_link_down = mtk_mac_link_down,
0650     .mac_link_up = mtk_mac_link_up,
0651 };
0652
0653 static int mtk_mdio_init(struct mtk_eth *eth)
0654 {
0655     struct device_node *mii_np;
0656     int ret;
0657
0658     mii_np = of_get_child_by_name(eth->dev->of_node, "mdio-bus");
0659     if (!mii_np) {
0660         dev_err(eth->dev, "no %s child node found", "mdio-bus");
0661         return -ENODEV;
0662     }
0663
0664     if (!of_device_is_available(mii_np)) {
0665         ret = -ENODEV;
0666         goto err_put_node;
0667     }
0668
0669     eth->mii_bus = devm_mdiobus_alloc(eth->dev);
0670     if (!eth->mii_bus) {
0671         ret = -ENOMEM;
0672         goto err_put_node;
0673     }
0674
0675     eth->mii_bus->name = "mdio";
0676     eth->mii_bus->read = mtk_mdio_read;
0677     eth->mii_bus->write = mtk_mdio_write;
0678     eth->mii_bus->probe_capabilities = MDIOBUS_C22_C45;
0679     eth->mii_bus->priv = eth;
0680     eth->mii_bus->parent = eth->dev;
0681
0682     snprintf(eth->mii_bus->id, MII_BUS_ID_SIZE, "%pOFn", mii_np);
0683     ret = of_mdiobus_register(eth->mii_bus, mii_np);
0684
0685 err_put_node:
0686     of_node_put(mii_np);
0687     return ret;
0688 }
0689
0690 static void mtk_mdio_cleanup(struct mtk_eth *eth)
0691 {
0692     if (!eth->mii_bus)
0693         return;
0694
0695     mdiobus_unregister(eth->mii_bus);
0696 }
0697
0698 static inline void mtk_tx_irq_disable(struct mtk_eth *eth, u32 mask)
0699 {
0700     unsigned long flags;
0701     u32 val;
0702
0703     spin_lock_irqsave(&eth->tx_irq_lock, flags);
0704     val = mtk_r32(eth, eth->soc->reg_map->tx_irq_mask);
0705     mtk_w32(eth, val & ~mask, eth->soc->reg_map->tx_irq_mask);
0706     spin_unlock_irqrestore(&eth->tx_irq_lock, flags);
0707 }
0708
0709 static inline void mtk_tx_irq_enable(struct mtk_eth *eth, u32 mask)
0710 {
0711     unsigned long flags;
0712     u32 val;
0713
0714     spin_lock_irqsave(&eth->tx_irq_lock, flags);
0715     val = mtk_r32(eth, eth->soc->reg_map->tx_irq_mask);
0716     mtk_w32(eth, val | mask, eth->soc->reg_map->tx_irq_mask);
0717     spin_unlock_irqrestore(&eth->tx_irq_lock, flags);
0718 }
0719
0720 static inline void mtk_rx_irq_disable(struct mtk_eth *eth, u32 mask)
0721 {
0722     unsigned long flags;
0723     u32 val;
0724
0725     spin_lock_irqsave(&eth->rx_irq_lock, flags);
0726     val = mtk_r32(eth, eth->soc->reg_map->pdma.irq_mask);
0727     mtk_w32(eth, val & ~mask, eth->soc->reg_map->pdma.irq_mask);
0728     spin_unlock_irqrestore(&eth->rx_irq_lock, flags);
0729 }
0730
0731 static inline void mtk_rx_irq_enable(struct mtk_eth *eth, u32 mask)
0732 {
0733     unsigned long flags;
0734     u32 val;
0735
0736     spin_lock_irqsave(&eth->rx_irq_lock, flags);
0737     val = mtk_r32(eth, eth->soc->reg_map->pdma.irq_mask);
0738     mtk_w32(eth, val | mask, eth->soc->reg_map->pdma.irq_mask);
0739     spin_unlock_irqrestore(&eth->rx_irq_lock, flags);
0740 }
0741
0742 static int mtk_set_mac_address(struct net_device *dev, void *p)
0743 {
0744     int ret = eth_mac_addr(dev, p);
0745     struct mtk_mac *mac = netdev_priv(dev);
0746     struct mtk_eth *eth = mac->hw;
0747     const char *macaddr = dev->dev_addr;
0748
0749     if (ret)
0750         return ret;
0751
0752     if (unlikely(test_bit(MTK_RESETTING, &mac->hw->state)))
0753         return -EBUSY;
0754
0755     spin_lock_bh(&mac->hw->page_lock);
0756     if (MTK_HAS_CAPS(eth->soc->caps, MTK_SOC_MT7628)) {
0757         mtk_w32(mac->hw, (macaddr[0] << 8) | macaddr[1],
0758             MT7628_SDM_MAC_ADRH);
0759         mtk_w32(mac->hw, (macaddr[2] << 24) | (macaddr[3] << 16) |
0760             (macaddr[4] << 8) | macaddr[5],
0761             MT7628_SDM_MAC_ADRL);
0762     } else {
0763         mtk_w32(mac->hw, (macaddr[0] << 8) | macaddr[1],
0764             MTK_GDMA_MAC_ADRH(mac->id));
0765         mtk_w32(mac->hw, (macaddr[2] << 24) | (macaddr[3] << 16) |
0766             (macaddr[4] << 8) | macaddr[5],
0767             MTK_GDMA_MAC_ADRL(mac->id));
0768     }
0769     spin_unlock_bh(&mac->hw->page_lock);
0770
0771     return 0;
0772 }
0773
0774 void mtk_stats_update_mac(struct mtk_mac *mac)
0775 {
0776     struct mtk_hw_stats *hw_stats = mac->hw_stats;
0777     struct mtk_eth *eth = mac->hw;
0778
0779     u64_stats_update_begin(&hw_stats->syncp);
0780
0781     if (MTK_HAS_CAPS(eth->soc->caps, MTK_SOC_MT7628)) {
0782         hw_stats->tx_packets += mtk_r32(mac->hw, MT7628_SDM_TPCNT);
0783         hw_stats->tx_bytes += mtk_r32(mac->hw, MT7628_SDM_TBCNT);
0784         hw_stats->rx_packets += mtk_r32(mac->hw, MT7628_SDM_RPCNT);
0785         hw_stats->rx_bytes += mtk_r32(mac->hw, MT7628_SDM_RBCNT);
0786         hw_stats->rx_checksum_errors +=
0787             mtk_r32(mac->hw, MT7628_SDM_CS_ERR);
0788     } else {
0789         const struct mtk_reg_map *reg_map = eth->soc->reg_map;
0790         unsigned int offs = hw_stats->reg_offset;
0791         u64 stats;
0792
0793         hw_stats->rx_bytes += mtk_r32(mac->hw, reg_map->gdm1_cnt + offs);
0794         stats = mtk_r32(mac->hw, reg_map->gdm1_cnt + 0x4 + offs);
0795         if (stats)
0796             hw_stats->rx_bytes += (stats << 32);
0797         hw_stats->rx_packets +=
0798             mtk_r32(mac->hw, reg_map->gdm1_cnt + 0x8 + offs);
0799         hw_stats->rx_overflow +=
0800             mtk_r32(mac->hw, reg_map->gdm1_cnt + 0x10 + offs);
0801         hw_stats->rx_fcs_errors +=
0802             mtk_r32(mac->hw, reg_map->gdm1_cnt + 0x14 + offs);
0803         hw_stats->rx_short_errors +=
0804             mtk_r32(mac->hw, reg_map->gdm1_cnt + 0x18 + offs);
0805         hw_stats->rx_long_errors +=
0806             mtk_r32(mac->hw, reg_map->gdm1_cnt + 0x1c + offs);
0807         hw_stats->rx_checksum_errors +=
0808             mtk_r32(mac->hw, reg_map->gdm1_cnt + 0x20 + offs);
0809         hw_stats->rx_flow_control_packets +=
0810             mtk_r32(mac->hw, reg_map->gdm1_cnt + 0x24 + offs);
0811         hw_stats->tx_skip +=
0812             mtk_r32(mac->hw, reg_map->gdm1_cnt + 0x28 + offs);
0813         hw_stats->tx_collisions +=
0814             mtk_r32(mac->hw, reg_map->gdm1_cnt + 0x2c + offs);
0815         hw_stats->tx_bytes +=
0816             mtk_r32(mac->hw, reg_map->gdm1_cnt + 0x30 + offs);
0817         stats =  mtk_r32(mac->hw, reg_map->gdm1_cnt + 0x34 + offs);
0818         if (stats)
0819             hw_stats->tx_bytes += (stats << 32);
0820         hw_stats->tx_packets +=
0821             mtk_r32(mac->hw, reg_map->gdm1_cnt + 0x38 + offs);
0822     }
0823
0824     u64_stats_update_end(&hw_stats->syncp);
0825 }
0826
0827 static void mtk_stats_update(struct mtk_eth *eth)
0828 {
0829     int i;
0830
0831     for (i = 0; i < MTK_MAC_COUNT; i++) {
0832         if (!eth->mac[i] || !eth->mac[i]->hw_stats)
0833             continue;
0834         if (spin_trylock(&eth->mac[i]->hw_stats->stats_lock)) {
0835             mtk_stats_update_mac(eth->mac[i]);
0836             spin_unlock(&eth->mac[i]->hw_stats->stats_lock);
0837         }
0838     }
0839 }
0840
0841 static void mtk_get_stats64(struct net_device *dev,
0842                 struct rtnl_link_stats64 *storage)
0843 {
0844     struct mtk_mac *mac = netdev_priv(dev);
0845     struct mtk_hw_stats *hw_stats = mac->hw_stats;
0846     unsigned int start;
0847
0848     if (netif_running(dev) && netif_device_present(dev)) {
0849         if (spin_trylock_bh(&hw_stats->stats_lock)) {
0850             mtk_stats_update_mac(mac);
0851             spin_unlock_bh(&hw_stats->stats_lock);
0852         }
0853     }
0854
0855     do {
0856         start = u64_stats_fetch_begin_irq(&hw_stats->syncp);
0857         storage->rx_packets = hw_stats->rx_packets;
0858         storage->tx_packets = hw_stats->tx_packets;
0859         storage->rx_bytes = hw_stats->rx_bytes;
0860         storage->tx_bytes = hw_stats->tx_bytes;
0861         storage->collisions = hw_stats->tx_collisions;
0862         storage->rx_length_errors = hw_stats->rx_short_errors +
0863             hw_stats->rx_long_errors;
0864         storage->rx_over_errors = hw_stats->rx_overflow;
0865         storage->rx_crc_errors = hw_stats->rx_fcs_errors;
0866         storage->rx_errors = hw_stats->rx_checksum_errors;
0867         storage->tx_aborted_errors = hw_stats->tx_skip;
0868     } while (u64_stats_fetch_retry_irq(&hw_stats->syncp, start));
0869
0870     storage->tx_errors = dev->stats.tx_errors;
0871     storage->rx_dropped = dev->stats.rx_dropped;
0872     storage->tx_dropped = dev->stats.tx_dropped;
0873 }
0874
0875 static inline int mtk_max_frag_size(int mtu)
0876 {
0877     /* make sure buf_size will be at least MTK_MAX_RX_LENGTH */
0878     if (mtu + MTK_RX_ETH_HLEN < MTK_MAX_RX_LENGTH_2K)
0879         mtu = MTK_MAX_RX_LENGTH_2K - MTK_RX_ETH_HLEN;
0880
0881     return SKB_DATA_ALIGN(MTK_RX_HLEN + mtu) +
0882         SKB_DATA_ALIGN(sizeof(struct skb_shared_info));
0883 }
0884
0885 static inline int mtk_max_buf_size(int frag_size)
0886 {
0887     int buf_size = frag_size - NET_SKB_PAD - NET_IP_ALIGN -
0888                SKB_DATA_ALIGN(sizeof(struct skb_shared_info));
0889
0890     WARN_ON(buf_size < MTK_MAX_RX_LENGTH_2K);
0891
0892     return buf_size;
0893 }
0894
0895 static bool mtk_rx_get_desc(struct mtk_eth *eth, struct mtk_rx_dma_v2 *rxd,
0896                 struct mtk_rx_dma_v2 *dma_rxd)
0897 {
0898     rxd->rxd2 = READ_ONCE(dma_rxd->rxd2);
0899     if (!(rxd->rxd2 & RX_DMA_DONE))
0900         return false;
0901
0902     rxd->rxd1 = READ_ONCE(dma_rxd->rxd1);
0903     rxd->rxd3 = READ_ONCE(dma_rxd->rxd3);
0904     rxd->rxd4 = READ_ONCE(dma_rxd->rxd4);
0905     if (MTK_HAS_CAPS(eth->soc->caps, MTK_NETSYS_V2)) {
0906         rxd->rxd5 = READ_ONCE(dma_rxd->rxd5);
0907         rxd->rxd6 = READ_ONCE(dma_rxd->rxd6);
0908     }
0909
0910     return true;
0911 }
0912
0913 static void *mtk_max_lro_buf_alloc(gfp_t gfp_mask)
0914 {
0915     unsigned int size = mtk_max_frag_size(MTK_MAX_LRO_RX_LENGTH);
0916     unsigned long data;
0917
0918     data = __get_free_pages(gfp_mask | __GFP_COMP | __GFP_NOWARN,
0919                 get_order(size));
0920
0921     return (void *)data;
0922 }
0923
0924 /* the qdma core needs scratch memory to be setup */
0925 static int mtk_init_fq_dma(struct mtk_eth *eth)
0926 {
0927     const struct mtk_soc_data *soc = eth->soc;
0928     dma_addr_t phy_ring_tail;
0929     int cnt = MTK_DMA_SIZE;
0930     dma_addr_t dma_addr;
0931     int i;
0932
0933     eth->scratch_ring = dma_alloc_coherent(eth->dma_dev,
0934                            cnt * soc->txrx.txd_size,
0935                            &eth->phy_scratch_ring,
0936                            GFP_KERNEL);
0937     if (unlikely(!eth->scratch_ring))
0938         return -ENOMEM;
0939
0940     eth->scratch_head = kcalloc(cnt, MTK_QDMA_PAGE_SIZE, GFP_KERNEL);
0941     if (unlikely(!eth->scratch_head))
0942         return -ENOMEM;
0943
0944     dma_addr = dma_map_single(eth->dma_dev,
0945                   eth->scratch_head, cnt * MTK_QDMA_PAGE_SIZE,
0946                   DMA_FROM_DEVICE);
0947     if (unlikely(dma_mapping_error(eth->dma_dev, dma_addr)))
0948         return -ENOMEM;
0949
0950     phy_ring_tail = eth->phy_scratch_ring + soc->txrx.txd_size * (cnt - 1);
0951
0952     for (i = 0; i < cnt; i++) {
0953         struct mtk_tx_dma_v2 *txd;
0954
0955         txd = eth->scratch_ring + i * soc->txrx.txd_size;
0956         txd->txd1 = dma_addr + i * MTK_QDMA_PAGE_SIZE;
0957         if (i < cnt - 1)
0958             txd->txd2 = eth->phy_scratch_ring +
0959                     (i + 1) * soc->txrx.txd_size;
0960
0961         txd->txd3 = TX_DMA_PLEN0(MTK_QDMA_PAGE_SIZE);
0962         txd->txd4 = 0;
0963         if (MTK_HAS_CAPS(soc->caps, MTK_NETSYS_V2)) {
0964             txd->txd5 = 0;
0965             txd->txd6 = 0;
0966             txd->txd7 = 0;
0967             txd->txd8 = 0;
0968         }
0969     }
0970
0971     mtk_w32(eth, eth->phy_scratch_ring, soc->reg_map->qdma.fq_head);
0972     mtk_w32(eth, phy_ring_tail, soc->reg_map->qdma.fq_tail);
0973     mtk_w32(eth, (cnt << 16) | cnt, soc->reg_map->qdma.fq_count);
0974     mtk_w32(eth, MTK_QDMA_PAGE_SIZE << 16, soc->reg_map->qdma.fq_blen);
0975
0976     return 0;
0977 }
0978
0979 static void *mtk_qdma_phys_to_virt(struct mtk_tx_ring *ring, u32 desc)
0980 {
0981     return ring->dma + (desc - ring->phys);
0982 }
0983
0984 static struct mtk_tx_buf *mtk_desc_to_tx_buf(struct mtk_tx_ring *ring,
0985                          void *txd, u32 txd_size)
0986 {
0987     int idx = (txd - ring->dma) / txd_size;
0988
0989     return &ring->buf[idx];
0990 }
0991
0992 static struct mtk_tx_dma *qdma_to_pdma(struct mtk_tx_ring *ring,
0993                        struct mtk_tx_dma *dma)
0994 {
0995     return ring->dma_pdma - (struct mtk_tx_dma *)ring->dma + dma;
0996 }
0997
0998 static int txd_to_idx(struct mtk_tx_ring *ring, void *dma, u32 txd_size)
0999 {
1000     return (dma - ring->dma) / txd_size;
1001 }
1002
1003 static void mtk_tx_unmap(struct mtk_eth *eth, struct mtk_tx_buf *tx_buf,
1004              struct xdp_frame_bulk *bq, bool napi)
1005 {
1006     if (MTK_HAS_CAPS(eth->soc->caps, MTK_QDMA)) {
1007         if (tx_buf->flags & MTK_TX_FLAGS_SINGLE0) {
1008             dma_unmap_single(eth->dma_dev,
1009                      dma_unmap_addr(tx_buf, dma_addr0),
1010                      dma_unmap_len(tx_buf, dma_len0),
1011                      DMA_TO_DEVICE);
1012         } else if (tx_buf->flags & MTK_TX_FLAGS_PAGE0) {
1013             dma_unmap_page(eth->dma_dev,
1014                        dma_unmap_addr(tx_buf, dma_addr0),
1015                        dma_unmap_len(tx_buf, dma_len0),
1016                        DMA_TO_DEVICE);
1017         }
1018     } else {
1019         if (dma_unmap_len(tx_buf, dma_len0)) {
1020             dma_unmap_page(eth->dma_dev,
1021                        dma_unmap_addr(tx_buf, dma_addr0),
1022                        dma_unmap_len(tx_buf, dma_len0),
1023                        DMA_TO_DEVICE);
1024         }
1025
1026         if (dma_unmap_len(tx_buf, dma_len1)) {
1027             dma_unmap_page(eth->dma_dev,
1028                        dma_unmap_addr(tx_buf, dma_addr1),
1029                        dma_unmap_len(tx_buf, dma_len1),
1030                        DMA_TO_DEVICE);
1031         }
1032     }
1033
1034     if (tx_buf->data && tx_buf->data != (void *)MTK_DMA_DUMMY_DESC) {
1035         if (tx_buf->type == MTK_TYPE_SKB) {
1036             struct sk_buff *skb = tx_buf->data;
1037
1038             if (napi)
1039                 napi_consume_skb(skb, napi);
1040             else
1041                 dev_kfree_skb_any(skb);
1042         } else {
1043             struct xdp_frame *xdpf = tx_buf->data;
1044
1045             if (napi && tx_buf->type == MTK_TYPE_XDP_TX)
1046                 xdp_return_frame_rx_napi(xdpf);
1047             else if (bq)
1048                 xdp_return_frame_bulk(xdpf, bq);
1049             else
1050                 xdp_return_frame(xdpf);
1051         }
1052     }
1053     tx_buf->flags = 0;
1054     tx_buf->data = NULL;
1055 }
1056
1057 static void setup_tx_buf(struct mtk_eth *eth, struct mtk_tx_buf *tx_buf,
1058              struct mtk_tx_dma *txd, dma_addr_t mapped_addr,
1059              size_t size, int idx)
1060 {
1061     if (MTK_HAS_CAPS(eth->soc->caps, MTK_QDMA)) {
1062         dma_unmap_addr_set(tx_buf, dma_addr0, mapped_addr);
1063         dma_unmap_len_set(tx_buf, dma_len0, size);
1064     } else {
1065         if (idx & 1) {
1066             txd->txd3 = mapped_addr;
1067             txd->txd2 |= TX_DMA_PLEN1(size);
1068             dma_unmap_addr_set(tx_buf, dma_addr1, mapped_addr);
1069             dma_unmap_len_set(tx_buf, dma_len1, size);
1070         } else {
1071             tx_buf->data = (void *)MTK_DMA_DUMMY_DESC;
1072             txd->txd1 = mapped_addr;
1073             txd->txd2 = TX_DMA_PLEN0(size);
1074             dma_unmap_addr_set(tx_buf, dma_addr0, mapped_addr);
1075             dma_unmap_len_set(tx_buf, dma_len0, size);
1076         }
1077     }
1078 }
1079
1080 static void mtk_tx_set_dma_desc_v1(struct net_device *dev, void *txd,
1081                    struct mtk_tx_dma_desc_info *info)
1082 {
1083     struct mtk_mac *mac = netdev_priv(dev);
1084     struct mtk_eth *eth = mac->hw;
1085     struct mtk_tx_dma *desc = txd;
1086     u32 data;
1087
1088     WRITE_ONCE(desc->txd1, info->addr);
1089
1090     data = TX_DMA_SWC | TX_DMA_PLEN0(info->size);
1091     if (info->last)
1092         data |= TX_DMA_LS0;
1093     WRITE_ONCE(desc->txd3, data);
1094
1095     data = (mac->id + 1) << TX_DMA_FPORT_SHIFT; /* forward port */
1096     if (info->first) {
1097         if (info->gso)
1098             data |= TX_DMA_TSO;
1099         /* tx checksum offload */
1100         if (info->csum)
1101             data |= TX_DMA_CHKSUM;
1102         /* vlan header offload */
1103         if (info->vlan)
1104             data |= TX_DMA_INS_VLAN | info->vlan_tci;
1105     }
1106     WRITE_ONCE(desc->txd4, data);
1107 }
1108
1109 static void mtk_tx_set_dma_desc_v2(struct net_device *dev, void *txd,
1110                    struct mtk_tx_dma_desc_info *info)
1111 {
1112     struct mtk_mac *mac = netdev_priv(dev);
1113     struct mtk_tx_dma_v2 *desc = txd;
1114     struct mtk_eth *eth = mac->hw;
1115     u32 data;
1116
1117     WRITE_ONCE(desc->txd1, info->addr);
1118
1119     data = TX_DMA_PLEN0(info->size);
1120     if (info->last)
1121         data |= TX_DMA_LS0;
1122     WRITE_ONCE(desc->txd3, data);
1123
1124     if (!info->qid && mac->id)
1125         info->qid = MTK_QDMA_GMAC2_QID;
1126
1127     data = (mac->id + 1) << TX_DMA_FPORT_SHIFT_V2; /* forward port */
1128     data |= TX_DMA_SWC_V2 | QID_BITS_V2(info->qid);
1129     WRITE_ONCE(desc->txd4, data);
1130
1131     data = 0;
1132     if (info->first) {
1133         if (info->gso)
1134             data |= TX_DMA_TSO_V2;
1135         /* tx checksum offload */
1136         if (info->csum)
1137             data |= TX_DMA_CHKSUM_V2;
1138     }
1139     WRITE_ONCE(desc->txd5, data);
1140
1141     data = 0;
1142     if (info->first && info->vlan)
1143         data |= TX_DMA_INS_VLAN_V2 | info->vlan_tci;
1144     WRITE_ONCE(desc->txd6, data);
1145
1146     WRITE_ONCE(desc->txd7, 0);
1147     WRITE_ONCE(desc->txd8, 0);
1148 }
1149
1150 static void mtk_tx_set_dma_desc(struct net_device *dev, void *txd,
1151                 struct mtk_tx_dma_desc_info *info)
1152 {
1153     struct mtk_mac *mac = netdev_priv(dev);
1154     struct mtk_eth *eth = mac->hw;
1155
1156     if (MTK_HAS_CAPS(eth->soc->caps, MTK_NETSYS_V2))
1157         mtk_tx_set_dma_desc_v2(dev, txd, info);
1158     else
1159         mtk_tx_set_dma_desc_v1(dev, txd, info);
1160 }
1161
1162 static int mtk_tx_map(struct sk_buff *skb, struct net_device *dev,
1163               int tx_num, struct mtk_tx_ring *ring, bool gso)
1164 {
1165     struct mtk_tx_dma_desc_info txd_info = {
1166         .size = skb_headlen(skb),
1167         .gso = gso,
1168         .csum = skb->ip_summed == CHECKSUM_PARTIAL,
1169         .vlan = skb_vlan_tag_present(skb),
1170         .qid = skb->mark & MTK_QDMA_TX_MASK,
1171         .vlan_tci = skb_vlan_tag_get(skb),
1172         .first = true,
1173         .last = !skb_is_nonlinear(skb),
1174     };
1175     struct mtk_mac *mac = netdev_priv(dev);
1176     struct mtk_eth *eth = mac->hw;
1177     const struct mtk_soc_data *soc = eth->soc;
1178     struct mtk_tx_dma *itxd, *txd;
1179     struct mtk_tx_dma *itxd_pdma, *txd_pdma;
1180     struct mtk_tx_buf *itx_buf, *tx_buf;
1181     int i, n_desc = 1;
1182     int k = 0;
1183
1184     itxd = ring->next_free;
1185     itxd_pdma = qdma_to_pdma(ring, itxd);
1186     if (itxd == ring->last_free)
1187         return -ENOMEM;
1188
1189     itx_buf = mtk_desc_to_tx_buf(ring, itxd, soc->txrx.txd_size);
1190     memset(itx_buf, 0, sizeof(*itx_buf));
1191
1192     txd_info.addr = dma_map_single(eth->dma_dev, skb->data, txd_info.size,
1193                        DMA_TO_DEVICE);
1194     if (unlikely(dma_mapping_error(eth->dma_dev, txd_info.addr)))
1195         return -ENOMEM;
1196
1197     mtk_tx_set_dma_desc(dev, itxd, &txd_info);
1198
1199     itx_buf->flags |= MTK_TX_FLAGS_SINGLE0;
1200     itx_buf->flags |= (!mac->id) ? MTK_TX_FLAGS_FPORT0 :
1201               MTK_TX_FLAGS_FPORT1;
1202     setup_tx_buf(eth, itx_buf, itxd_pdma, txd_info.addr, txd_info.size,
1203              k++);
1204
1205     /* TX SG offload */
1206     txd = itxd;
1207     txd_pdma = qdma_to_pdma(ring, txd);
1208
1209     for (i = 0; i < skb_shinfo(skb)->nr_frags; i++) {
1210         skb_frag_t *frag = &skb_shinfo(skb)->frags[i];
1211         unsigned int offset = 0;
1212         int frag_size = skb_frag_size(frag);
1213
1214         while (frag_size) {
1215             bool new_desc = true;
1216
1217             if (MTK_HAS_CAPS(soc->caps, MTK_QDMA) ||
1218                 (i & 0x1)) {
1219                 txd = mtk_qdma_phys_to_virt(ring, txd->txd2);
1220                 txd_pdma = qdma_to_pdma(ring, txd);
1221                 if (txd == ring->last_free)
1222                     goto err_dma;
1223
1224                 n_desc++;
1225             } else {
1226                 new_desc = false;
1227             }
1228
1229             memset(&txd_info, 0, sizeof(struct mtk_tx_dma_desc_info));
1230             txd_info.size = min_t(unsigned int, frag_size,
1231                           soc->txrx.dma_max_len);
1232             txd_info.qid = skb->mark & MTK_QDMA_TX_MASK;
1233             txd_info.last = i == skb_shinfo(skb)->nr_frags - 1 &&
1234                     !(frag_size - txd_info.size);
1235             txd_info.addr = skb_frag_dma_map(eth->dma_dev, frag,
1236                              offset, txd_info.size,
1237                              DMA_TO_DEVICE);
1238             if (unlikely(dma_mapping_error(eth->dma_dev, txd_info.addr)))
1239                 goto err_dma;
1240
1241             mtk_tx_set_dma_desc(dev, txd, &txd_info);
1242
1243             tx_buf = mtk_desc_to_tx_buf(ring, txd,
1244                             soc->txrx.txd_size);
1245             if (new_desc)
1246                 memset(tx_buf, 0, sizeof(*tx_buf));
1247             tx_buf->data = (void *)MTK_DMA_DUMMY_DESC;
1248             tx_buf->flags |= MTK_TX_FLAGS_PAGE0;
1249             tx_buf->flags |= (!mac->id) ? MTK_TX_FLAGS_FPORT0 :
1250                      MTK_TX_FLAGS_FPORT1;
1251
1252             setup_tx_buf(eth, tx_buf, txd_pdma, txd_info.addr,
1253                      txd_info.size, k++);
1254
1255             frag_size -= txd_info.size;
1256             offset += txd_info.size;
1257         }
1258     }
1259
1260     /* store skb to cleanup */
1261     itx_buf->type = MTK_TYPE_SKB;
1262     itx_buf->data = skb;
1263
1264     if (!MTK_HAS_CAPS(soc->caps, MTK_QDMA)) {
1265         if (k & 0x1)
1266             txd_pdma->txd2 |= TX_DMA_LS0;
1267         else
1268             txd_pdma->txd2 |= TX_DMA_LS1;
1269     }
1270
1271     netdev_sent_queue(dev, skb->len);
1272     skb_tx_timestamp(skb);
1273
1274     ring->next_free = mtk_qdma_phys_to_virt(ring, txd->txd2);
1275     atomic_sub(n_desc, &ring->free_count);
1276
1277     /* make sure that all changes to the dma ring are flushed before we
1278      * continue
1279      */
1280     wmb();
1281
1282     if (MTK_HAS_CAPS(soc->caps, MTK_QDMA)) {
1283         if (netif_xmit_stopped(netdev_get_tx_queue(dev, 0)) ||
1284             !netdev_xmit_more())
1285             mtk_w32(eth, txd->txd2, soc->reg_map->qdma.ctx_ptr);
1286     } else {
1287         int next_idx;
1288
1289         next_idx = NEXT_DESP_IDX(txd_to_idx(ring, txd, soc->txrx.txd_size),
1290                      ring->dma_size);
1291         mtk_w32(eth, next_idx, MT7628_TX_CTX_IDX0);
1292     }
1293
1294     return 0;
1295
1296 err_dma:
1297     do {
1298         tx_buf = mtk_desc_to_tx_buf(ring, itxd, soc->txrx.txd_size);
1299
1300         /* unmap dma */
1301         mtk_tx_unmap(eth, tx_buf, NULL, false);
1302
1303         itxd->txd3 = TX_DMA_LS0 | TX_DMA_OWNER_CPU;
1304         if (!MTK_HAS_CAPS(soc->caps, MTK_QDMA))
1305             itxd_pdma->txd2 = TX_DMA_DESP2_DEF;
1306
1307         itxd = mtk_qdma_phys_to_virt(ring, itxd->txd2);
1308         itxd_pdma = qdma_to_pdma(ring, itxd);
1309     } while (itxd != txd);
1310
1311     return -ENOMEM;
1312 }
1313
1314 static int mtk_cal_txd_req(struct mtk_eth *eth, struct sk_buff *skb)
1315 {
1316     int i, nfrags = 1;
1317     skb_frag_t *frag;
1318
1319     if (skb_is_gso(skb)) {
1320         for (i = 0; i < skb_shinfo(skb)->nr_frags; i++) {
1321             frag = &skb_shinfo(skb)->frags[i];
1322             nfrags += DIV_ROUND_UP(skb_frag_size(frag),
1323                            eth->soc->txrx.dma_max_len);
1324         }
1325     } else {
1326         nfrags += skb_shinfo(skb)->nr_frags;
1327     }
1328
1329     return nfrags;
1330 }
1331
1332 static int mtk_queue_stopped(struct mtk_eth *eth)
1333 {
1334     int i;
1335
1336     for (i = 0; i < MTK_MAC_COUNT; i++) {
1337         if (!eth->netdev[i])
1338             continue;
1339         if (netif_queue_stopped(eth->netdev[i]))
1340             return 1;
1341     }
1342
1343     return 0;
1344 }
1345
1346 static void mtk_wake_queue(struct mtk_eth *eth)
1347 {
1348     int i;
1349
1350     for (i = 0; i < MTK_MAC_COUNT; i++) {
1351         if (!eth->netdev[i])
1352             continue;
1353         netif_wake_queue(eth->netdev[i]);
1354     }
1355 }
1356
1357 static netdev_tx_t mtk_start_xmit(struct sk_buff *skb, struct net_device *dev)
1358 {
1359     struct mtk_mac *mac = netdev_priv(dev);
1360     struct mtk_eth *eth = mac->hw;
1361     struct mtk_tx_ring *ring = &eth->tx_ring;
1362     struct net_device_stats *stats = &dev->stats;
1363     bool gso = false;
1364     int tx_num;
1365
1366     /* normally we can rely on the stack not calling this more than once,
1367      * however we have 2 queues running on the same ring so we need to lock
1368      * the ring access
1369      */
1370     spin_lock(&eth->page_lock);
1371
1372     if (unlikely(test_bit(MTK_RESETTING, &eth->state)))
1373         goto drop;
1374
1375     tx_num = mtk_cal_txd_req(eth, skb);
1376     if (unlikely(atomic_read(&ring->free_count) <= tx_num)) {
1377         netif_stop_queue(dev);
1378         netif_err(eth, tx_queued, dev,
1379               "Tx Ring full when queue awake!\n");
1380         spin_unlock(&eth->page_lock);
1381         return NETDEV_TX_BUSY;
1382     }
1383
1384     /* TSO: fill MSS info in tcp checksum field */
1385     if (skb_is_gso(skb)) {
1386         if (skb_cow_head(skb, 0)) {
1387             netif_warn(eth, tx_err, dev,
1388                    "GSO expand head fail.\n");
1389             goto drop;
1390         }
1391
1392         if (skb_shinfo(skb)->gso_type &
1393                 (SKB_GSO_TCPV4 | SKB_GSO_TCPV6)) {
1394             gso = true;
1395             tcp_hdr(skb)->check = htons(skb_shinfo(skb)->gso_size);
1396         }
1397     }
1398
1399     if (mtk_tx_map(skb, dev, tx_num, ring, gso) < 0)
1400         goto drop;
1401
1402     if (unlikely(atomic_read(&ring->free_count) <= ring->thresh))
1403         netif_stop_queue(dev);
1404
1405     spin_unlock(&eth->page_lock);
1406
1407     return NETDEV_TX_OK;
1408
1409 drop:
1410     spin_unlock(&eth->page_lock);
1411     stats->tx_dropped++;
1412     dev_kfree_skb_any(skb);
1413     return NETDEV_TX_OK;
1414 }
1415
1416 static struct mtk_rx_ring *mtk_get_rx_ring(struct mtk_eth *eth)
1417 {
1418     int i;
1419     struct mtk_rx_ring *ring;
1420     int idx;
1421
1422     if (!eth->hwlro)
1423         return &eth->rx_ring[0];
1424
1425     for (i = 0; i < MTK_MAX_RX_RING_NUM; i++) {
1426         struct mtk_rx_dma *rxd;
1427
1428         ring = &eth->rx_ring[i];
1429         idx = NEXT_DESP_IDX(ring->calc_idx, ring->dma_size);
1430         rxd = ring->dma + idx * eth->soc->txrx.rxd_size;
1431         if (rxd->rxd2 & RX_DMA_DONE) {
1432             ring->calc_idx_update = true;
1433             return ring;
1434         }
1435     }
1436
1437     return NULL;
1438 }
1439
1440 static void mtk_update_rx_cpu_idx(struct mtk_eth *eth)
1441 {
1442     struct mtk_rx_ring *ring;
1443     int i;
1444
1445     if (!eth->hwlro) {
1446         ring = &eth->rx_ring[0];
1447         mtk_w32(eth, ring->calc_idx, ring->crx_idx_reg);
1448     } else {
1449         for (i = 0; i < MTK_MAX_RX_RING_NUM; i++) {
1450             ring = &eth->rx_ring[i];
1451             if (ring->calc_idx_update) {
1452                 ring->calc_idx_update = false;
1453                 mtk_w32(eth, ring->calc_idx, ring->crx_idx_reg);
1454             }
1455         }
1456     }
1457 }
1458
1459 static bool mtk_page_pool_enabled(struct mtk_eth *eth)
1460 {
1461     return MTK_HAS_CAPS(eth->soc->caps, MTK_NETSYS_V2);
1462 }
1463
1464 static struct page_pool *mtk_create_page_pool(struct mtk_eth *eth,
1465                           struct xdp_rxq_info *xdp_q,
1466                           int id, int size)
1467 {
1468     struct page_pool_params pp_params = {
1469         .order = 0,
1470         .flags = PP_FLAG_DMA_MAP | PP_FLAG_DMA_SYNC_DEV,
1471         .pool_size = size,
1472         .nid = NUMA_NO_NODE,
1473         .dev = eth->dma_dev,
1474         .offset = MTK_PP_HEADROOM,
1475         .max_len = MTK_PP_MAX_BUF_SIZE,
1476     };
1477     struct page_pool *pp;
1478     int err;
1479
1480     pp_params.dma_dir = rcu_access_pointer(eth->prog) ? DMA_BIDIRECTIONAL
1481                               : DMA_FROM_DEVICE;
1482     pp = page_pool_create(&pp_params);
1483     if (IS_ERR(pp))
1484         return pp;
1485
1486     err = __xdp_rxq_info_reg(xdp_q, &eth->dummy_dev, eth->rx_napi.napi_id,
1487                  id, PAGE_SIZE);
1488     if (err < 0)
1489         goto err_free_pp;
1490
1491     err = xdp_rxq_info_reg_mem_model(xdp_q, MEM_TYPE_PAGE_POOL, pp);
1492     if (err)
1493         goto err_unregister_rxq;
1494
1495     return pp;
1496
1497 err_unregister_rxq:
1498     xdp_rxq_info_unreg(xdp_q);
1499 err_free_pp:
1500     page_pool_destroy(pp);
1501
1502     return ERR_PTR(err);
1503 }
1504
1505 static void *mtk_page_pool_get_buff(struct page_pool *pp, dma_addr_t *dma_addr,
1506                     gfp_t gfp_mask)
1507 {
1508     struct page *page;
1509
1510     page = page_pool_alloc_pages(pp, gfp_mask | __GFP_NOWARN);
1511     if (!page)
1512         return NULL;
1513
1514     *dma_addr = page_pool_get_dma_addr(page) + MTK_PP_HEADROOM;
1515     return page_address(page);
1516 }
1517
1518 static void mtk_rx_put_buff(struct mtk_rx_ring *ring, void *data, bool napi)
1519 {
1520     if (ring->page_pool)
1521         page_pool_put_full_page(ring->page_pool,
1522                     virt_to_head_page(data), napi);
1523     else
1524         skb_free_frag(data);
1525 }
1526
1527 static int mtk_xdp_frame_map(struct mtk_eth *eth, struct net_device *dev,
1528                  struct mtk_tx_dma_desc_info *txd_info,
1529                  struct mtk_tx_dma *txd, struct mtk_tx_buf *tx_buf,
1530                  void *data, u16 headroom, int index, bool dma_map)
1531 {
1532     struct mtk_tx_ring *ring = &eth->tx_ring;
1533     struct mtk_mac *mac = netdev_priv(dev);
1534     struct mtk_tx_dma *txd_pdma;
1535
1536     if (dma_map) {  /* ndo_xdp_xmit */
1537         txd_info->addr = dma_map_single(eth->dma_dev, data,
1538                         txd_info->size, DMA_TO_DEVICE);
1539         if (unlikely(dma_mapping_error(eth->dma_dev, txd_info->addr)))
1540             return -ENOMEM;
1541
1542         tx_buf->flags |= MTK_TX_FLAGS_SINGLE0;
1543     } else {
1544         struct page *page = virt_to_head_page(data);
1545
1546         txd_info->addr = page_pool_get_dma_addr(page) +
1547                  sizeof(struct xdp_frame) + headroom;
1548         dma_sync_single_for_device(eth->dma_dev, txd_info->addr,
1549                        txd_info->size, DMA_BIDIRECTIONAL);
1550     }
1551     mtk_tx_set_dma_desc(dev, txd, txd_info);
1552
1553     tx_buf->flags |= !mac->id ? MTK_TX_FLAGS_FPORT0 : MTK_TX_FLAGS_FPORT1;
1554     tx_buf->type = dma_map ? MTK_TYPE_XDP_NDO : MTK_TYPE_XDP_TX;
1555     tx_buf->data = (void *)MTK_DMA_DUMMY_DESC;
1556
1557     txd_pdma = qdma_to_pdma(ring, txd);
1558     setup_tx_buf(eth, tx_buf, txd_pdma, txd_info->addr, txd_info->size,
1559              index);
1560
1561     return 0;
1562 }
1563
1564 static int mtk_xdp_submit_frame(struct mtk_eth *eth, struct xdp_frame *xdpf,
1565                 struct net_device *dev, bool dma_map)
1566 {
1567     struct skb_shared_info *sinfo = xdp_get_shared_info_from_frame(xdpf);
1568     const struct mtk_soc_data *soc = eth->soc;
1569     struct mtk_tx_ring *ring = &eth->tx_ring;
1570     struct mtk_tx_dma_desc_info txd_info = {
1571         .size   = xdpf->len,
1572         .first  = true,
1573         .last   = !xdp_frame_has_frags(xdpf),
1574     };
1575     int err, index = 0, n_desc = 1, nr_frags;
1576     struct mtk_tx_dma *htxd, *txd, *txd_pdma;
1577     struct mtk_tx_buf *htx_buf, *tx_buf;
1578     void *data = xdpf->data;
1579
1580     if (unlikely(test_bit(MTK_RESETTING, &eth->state)))
1581         return -EBUSY;
1582
1583     nr_frags = unlikely(xdp_frame_has_frags(xdpf)) ? sinfo->nr_frags : 0;
1584     if (unlikely(atomic_read(&ring->free_count) <= 1 + nr_frags))
1585         return -EBUSY;
1586
1587     spin_lock(&eth->page_lock);
1588
1589     txd = ring->next_free;
1590     if (txd == ring->last_free) {
1591         spin_unlock(&eth->page_lock);
1592         return -ENOMEM;
1593     }
1594     htxd = txd;
1595
1596     tx_buf = mtk_desc_to_tx_buf(ring, txd, soc->txrx.txd_size);
1597     memset(tx_buf, 0, sizeof(*tx_buf));
1598     htx_buf = tx_buf;
1599
1600     for (;;) {
1601         err = mtk_xdp_frame_map(eth, dev, &txd_info, txd, tx_buf,
1602                     data, xdpf->headroom, index, dma_map);
1603         if (err < 0)
1604             goto unmap;
1605
1606         if (txd_info.last)
1607             break;
1608
1609         if (MTK_HAS_CAPS(soc->caps, MTK_QDMA) || (index & 0x1)) {
1610             txd = mtk_qdma_phys_to_virt(ring, txd->txd2);
1611             txd_pdma = qdma_to_pdma(ring, txd);
1612             if (txd == ring->last_free)
1613                 goto unmap;
1614
1615             tx_buf = mtk_desc_to_tx_buf(ring, txd,
1616                             soc->txrx.txd_size);
1617             memset(tx_buf, 0, sizeof(*tx_buf));
1618             n_desc++;
1619         }
1620
1621         memset(&txd_info, 0, sizeof(struct mtk_tx_dma_desc_info));
1622         txd_info.size = skb_frag_size(&sinfo->frags[index]);
1623         txd_info.last = index + 1 == nr_frags;
1624         data = skb_frag_address(&sinfo->frags[index]);
1625
1626         index++;
1627     }
1628     /* store xdpf for cleanup */
1629     htx_buf->data = xdpf;
1630
1631     if (!MTK_HAS_CAPS(soc->caps, MTK_QDMA)) {
1632         txd_pdma = qdma_to_pdma(ring, txd);
1633         if (index & 1)
1634             txd_pdma->txd2 |= TX_DMA_LS0;
1635         else
1636             txd_pdma->txd2 |= TX_DMA_LS1;
1637     }
1638
1639     ring->next_free = mtk_qdma_phys_to_virt(ring, txd->txd2);
1640     atomic_sub(n_desc, &ring->free_count);
1641
1642     /* make sure that all changes to the dma ring are flushed before we
1643      * continue
1644      */
1645     wmb();
1646
1647     if (MTK_HAS_CAPS(soc->caps, MTK_QDMA)) {
1648         mtk_w32(eth, txd->txd2, soc->reg_map->qdma.ctx_ptr);
1649     } else {
1650         int idx;
1651
1652         idx = txd_to_idx(ring, txd, soc->txrx.txd_size);
1653         mtk_w32(eth, NEXT_DESP_IDX(idx, ring->dma_size),
1654             MT7628_TX_CTX_IDX0);
1655     }
1656
1657     spin_unlock(&eth->page_lock);
1658
1659     return 0;
1660
1661 unmap:
1662     while (htxd != txd) {
1663         txd_pdma = qdma_to_pdma(ring, htxd);
1664         tx_buf = mtk_desc_to_tx_buf(ring, htxd, soc->txrx.txd_size);
1665         mtk_tx_unmap(eth, tx_buf, NULL, false);
1666
1667         htxd->txd3 = TX_DMA_LS0 | TX_DMA_OWNER_CPU;
1668         if (!MTK_HAS_CAPS(soc->caps, MTK_QDMA))
1669             txd_pdma->txd2 = TX_DMA_DESP2_DEF;
1670
1671         htxd = mtk_qdma_phys_to_virt(ring, htxd->txd2);
1672     }
1673
1674     spin_unlock(&eth->page_lock);
1675
1676     return err;
1677 }
1678
1679 static int mtk_xdp_xmit(struct net_device *dev, int num_frame,
1680             struct xdp_frame **frames, u32 flags)
1681 {
1682     struct mtk_mac *mac = netdev_priv(dev);
1683     struct mtk_hw_stats *hw_stats = mac->hw_stats;
1684     struct mtk_eth *eth = mac->hw;
1685     int i, nxmit = 0;
1686
1687     if (unlikely(flags & ~XDP_XMIT_FLAGS_MASK))
1688         return -EINVAL;
1689
1690     for (i = 0; i < num_frame; i++) {
1691         if (mtk_xdp_submit_frame(eth, frames[i], dev, true))
1692             break;
1693         nxmit++;
1694     }
1695
1696     u64_stats_update_begin(&hw_stats->syncp);
1697     hw_stats->xdp_stats.tx_xdp_xmit += nxmit;
1698     hw_stats->xdp_stats.tx_xdp_xmit_errors += num_frame - nxmit;
1699     u64_stats_update_end(&hw_stats->syncp);
1700
1701     return nxmit;
1702 }
1703
1704 static u32 mtk_xdp_run(struct mtk_eth *eth, struct mtk_rx_ring *ring,
1705                struct xdp_buff *xdp, struct net_device *dev)
1706 {
1707     struct mtk_mac *mac = netdev_priv(dev);
1708     struct mtk_hw_stats *hw_stats = mac->hw_stats;
1709     u64 *count = &hw_stats->xdp_stats.rx_xdp_drop;
1710     struct bpf_prog *prog;
1711     u32 act = XDP_PASS;
1712
1713     rcu_read_lock();
1714
1715     prog = rcu_dereference(eth->prog);
1716     if (!prog)
1717         goto out;
1718
1719     act = bpf_prog_run_xdp(prog, xdp);
1720     switch (act) {
1721     case XDP_PASS:
1722         count = &hw_stats->xdp_stats.rx_xdp_pass;
1723         goto update_stats;
1724     case XDP_REDIRECT:
1725         if (unlikely(xdp_do_redirect(dev, xdp, prog))) {
1726             act = XDP_DROP;
1727             break;
1728         }
1729
1730         count = &hw_stats->xdp_stats.rx_xdp_redirect;
1731         goto update_stats;
1732     case XDP_TX: {
1733         struct xdp_frame *xdpf = xdp_convert_buff_to_frame(xdp);
1734
1735         if (!xdpf || mtk_xdp_submit_frame(eth, xdpf, dev, false)) {
1736             count = &hw_stats->xdp_stats.rx_xdp_tx_errors;
1737             act = XDP_DROP;
1738             break;
1739         }
1740
1741         count = &hw_stats->xdp_stats.rx_xdp_tx;
1742         goto update_stats;
1743     }
1744     default:
1745         bpf_warn_invalid_xdp_action(dev, prog, act);
1746         fallthrough;
1747     case XDP_ABORTED:
1748         trace_xdp_exception(dev, prog, act);
1749         fallthrough;
1750     case XDP_DROP:
1751         break;
1752     }
1753
1754     page_pool_put_full_page(ring->page_pool,
1755                 virt_to_head_page(xdp->data), true);
1756
1757 update_stats:
1758     u64_stats_update_begin(&hw_stats->syncp);
1759     *count = *count + 1;
1760     u64_stats_update_end(&hw_stats->syncp);
1761 out:
1762     rcu_read_unlock();
1763
1764     return act;
1765 }
1766
1767 static int mtk_poll_rx(struct napi_struct *napi, int budget,
1768                struct mtk_eth *eth)
1769 {
1770     struct dim_sample dim_sample = {};
1771     struct mtk_rx_ring *ring;
1772     bool xdp_flush = false;
1773     int idx;
1774     struct sk_buff *skb;
1775     u8 *data, *new_data;
1776     struct mtk_rx_dma_v2 *rxd, trxd;
1777     int done = 0, bytes = 0;
1778
1779     while (done < budget) {
1780         unsigned int pktlen, *rxdcsum;
1781         struct net_device *netdev;
1782         dma_addr_t dma_addr;
1783         u32 hash, reason;
1784         int mac = 0;
1785
1786         ring = mtk_get_rx_ring(eth);
1787         if (unlikely(!ring))
1788             goto rx_done;
1789
1790         idx = NEXT_DESP_IDX(ring->calc_idx, ring->dma_size);
1791         rxd = ring->dma + idx * eth->soc->txrx.rxd_size;
1792         data = ring->data[idx];
1793
1794         if (!mtk_rx_get_desc(eth, &trxd, rxd))
1795             break;
1796
1797         /* find out which mac the packet come from. values start at 1 */
1798         if (MTK_HAS_CAPS(eth->soc->caps, MTK_NETSYS_V2))
1799             mac = RX_DMA_GET_SPORT_V2(trxd.rxd5) - 1;
1800         else if (!MTK_HAS_CAPS(eth->soc->caps, MTK_SOC_MT7628) &&
1801              !(trxd.rxd4 & RX_DMA_SPECIAL_TAG))
1802             mac = RX_DMA_GET_SPORT(trxd.rxd4) - 1;
1803
1804         if (unlikely(mac < 0 || mac >= MTK_MAC_COUNT ||
1805                  !eth->netdev[mac]))
1806             goto release_desc;
1807
1808         netdev = eth->netdev[mac];
1809
1810         if (unlikely(test_bit(MTK_RESETTING, &eth->state)))
1811             goto release_desc;
1812
1813         pktlen = RX_DMA_GET_PLEN0(trxd.rxd2);
1814
1815         /* alloc new buffer */
1816         if (ring->page_pool) {
1817             struct page *page = virt_to_head_page(data);
1818             struct xdp_buff xdp;
1819             u32 ret;
1820
1821             new_data = mtk_page_pool_get_buff(ring->page_pool,
1822                               &dma_addr,
1823                               GFP_ATOMIC);
1824             if (unlikely(!new_data)) {
1825                 netdev->stats.rx_dropped++;
1826                 goto release_desc;
1827             }
1828
1829             dma_sync_single_for_cpu(eth->dma_dev,
1830                 page_pool_get_dma_addr(page) + MTK_PP_HEADROOM,
1831                 pktlen, page_pool_get_dma_dir(ring->page_pool));
1832
1833             xdp_init_buff(&xdp, PAGE_SIZE, &ring->xdp_q);
1834             xdp_prepare_buff(&xdp, data, MTK_PP_HEADROOM, pktlen,
1835                      false);
1836             xdp_buff_clear_frags_flag(&xdp);
1837
1838             ret = mtk_xdp_run(eth, ring, &xdp, netdev);
1839             if (ret == XDP_REDIRECT)
1840                 xdp_flush = true;
1841
1842             if (ret != XDP_PASS)
1843                 goto skip_rx;
1844
1845             skb = build_skb(data, PAGE_SIZE);
1846             if (unlikely(!skb)) {
1847                 page_pool_put_full_page(ring->page_pool,
1848                             page, true);
1849                 netdev->stats.rx_dropped++;
1850                 goto skip_rx;
1851             }
1852
1853             skb_reserve(skb, xdp.data - xdp.data_hard_start);
1854             skb_put(skb, xdp.data_end - xdp.data);
1855             skb_mark_for_recycle(skb);
1856         } else {
1857             if (ring->frag_size <= PAGE_SIZE)
1858                 new_data = napi_alloc_frag(ring->frag_size);
1859             else
1860                 new_data = mtk_max_lro_buf_alloc(GFP_ATOMIC);
1861
1862             if (unlikely(!new_data)) {
1863                 netdev->stats.rx_dropped++;
1864                 goto release_desc;
1865             }
1866
1867             dma_addr = dma_map_single(eth->dma_dev,
1868                 new_data + NET_SKB_PAD + eth->ip_align,
1869                 ring->buf_size, DMA_FROM_DEVICE);
1870             if (unlikely(dma_mapping_error(eth->dma_dev,
1871                                dma_addr))) {
1872                 skb_free_frag(new_data);
1873                 netdev->stats.rx_dropped++;
1874                 goto release_desc;
1875             }
1876
1877             dma_unmap_single(eth->dma_dev, trxd.rxd1,
1878                      ring->buf_size, DMA_FROM_DEVICE);
1879
1880             skb = build_skb(data, ring->frag_size);
1881             if (unlikely(!skb)) {
1882                 netdev->stats.rx_dropped++;
1883                 skb_free_frag(data);
1884                 goto skip_rx;
1885             }
1886
1887             skb_reserve(skb, NET_SKB_PAD + NET_IP_ALIGN);
1888             skb_put(skb, pktlen);
1889         }
1890
1891         skb->dev = netdev;
1892         bytes += skb->len;
1893
1894         if (MTK_HAS_CAPS(eth->soc->caps, MTK_NETSYS_V2)) {
1895             hash = trxd.rxd5 & MTK_RXD5_FOE_ENTRY;
1896             if (hash != MTK_RXD5_FOE_ENTRY)
1897                 skb_set_hash(skb, jhash_1word(hash, 0),
1898                          PKT_HASH_TYPE_L4);
1899             rxdcsum = &trxd.rxd3;
1900         } else {
1901             hash = trxd.rxd4 & MTK_RXD4_FOE_ENTRY;
1902             if (hash != MTK_RXD4_FOE_ENTRY)
1903                 skb_set_hash(skb, jhash_1word(hash, 0),
1904                          PKT_HASH_TYPE_L4);
1905             rxdcsum = &trxd.rxd4;
1906         }
1907
1908         if (*rxdcsum & eth->soc->txrx.rx_dma_l4_valid)
1909             skb->ip_summed = CHECKSUM_UNNECESSARY;
1910         else
1911             skb_checksum_none_assert(skb);
1912         skb->protocol = eth_type_trans(skb, netdev);
1913
1914         reason = FIELD_GET(MTK_RXD4_PPE_CPU_REASON, trxd.rxd4);
1915         if (reason == MTK_PPE_CPU_REASON_HIT_UNBIND_RATE_REACHED)
1916             mtk_ppe_check_skb(eth->ppe, skb, hash);
1917
1918         if (netdev->features & NETIF_F_HW_VLAN_CTAG_RX) {
1919             if (MTK_HAS_CAPS(eth->soc->caps, MTK_NETSYS_V2)) {
1920                 if (trxd.rxd3 & RX_DMA_VTAG_V2)
1921                     __vlan_hwaccel_put_tag(skb,
1922                         htons(RX_DMA_VPID(trxd.rxd4)),
1923                         RX_DMA_VID(trxd.rxd4));
1924             } else if (trxd.rxd2 & RX_DMA_VTAG) {
1925                 __vlan_hwaccel_put_tag(skb, htons(ETH_P_8021Q),
1926                                RX_DMA_VID(trxd.rxd3));
1927             }
1928
1929             /* If the device is attached to a dsa switch, the special
1930              * tag inserted in VLAN field by hw switch can * be offloaded
1931              * by RX HW VLAN offload. Clear vlan info.
1932              */
1933             if (netdev_uses_dsa(netdev))
1934                 __vlan_hwaccel_clear_tag(skb);
1935         }
1936
1937         skb_record_rx_queue(skb, 0);
1938         napi_gro_receive(napi, skb);
1939
1940 skip_rx:
1941         ring->data[idx] = new_data;
1942         rxd->rxd1 = (unsigned int)dma_addr;
1943 release_desc:
1944         if (MTK_HAS_CAPS(eth->soc->caps, MTK_SOC_MT7628))
1945             rxd->rxd2 = RX_DMA_LSO;
1946         else
1947             rxd->rxd2 = RX_DMA_PREP_PLEN0(ring->buf_size);
1948
1949         ring->calc_idx = idx;
1950         done++;
1951     }
1952
1953 rx_done:
1954     if (done) {
1955         /* make sure that all changes to the dma ring are flushed before
1956          * we continue
1957          */
1958         wmb();
1959         mtk_update_rx_cpu_idx(eth);
1960     }
1961
1962     eth->rx_packets += done;
1963     eth->rx_bytes += bytes;
1964     dim_update_sample(eth->rx_events, eth->rx_packets, eth->rx_bytes,
1965               &dim_sample);
1966     net_dim(&eth->rx_dim, dim_sample);
1967
1968     if (xdp_flush)
1969         xdp_do_flush_map();
1970
1971     return done;
1972 }
1973
1974 static int mtk_poll_tx_qdma(struct mtk_eth *eth, int budget,
1975                 unsigned int *done, unsigned int *bytes)
1976 {
1977     const struct mtk_reg_map *reg_map = eth->soc->reg_map;
1978     struct mtk_tx_ring *ring = &eth->tx_ring;
1979     struct mtk_tx_buf *tx_buf;
1980     struct xdp_frame_bulk bq;
1981     struct mtk_tx_dma *desc;
1982     u32 cpu, dma;
1983
1984     cpu = ring->last_free_ptr;
1985     dma = mtk_r32(eth, reg_map->qdma.drx_ptr);
1986
1987     desc = mtk_qdma_phys_to_virt(ring, cpu);
1988     xdp_frame_bulk_init(&bq);
1989
1990     while ((cpu != dma) && budget) {
1991         u32 next_cpu = desc->txd2;
1992         int mac = 0;
1993
1994         desc = mtk_qdma_phys_to_virt(ring, desc->txd2);
1995         if ((desc->txd3 & TX_DMA_OWNER_CPU) == 0)
1996             break;
1997
1998         tx_buf = mtk_desc_to_tx_buf(ring, desc,
1999                         eth->soc->txrx.txd_size);
2000         if (tx_buf->flags & MTK_TX_FLAGS_FPORT1)
2001             mac = 1;
2002
2003         if (!tx_buf->data)
2004             break;
2005
2006         if (tx_buf->data != (void *)MTK_DMA_DUMMY_DESC) {
2007             if (tx_buf->type == MTK_TYPE_SKB) {
2008                 struct sk_buff *skb = tx_buf->data;
2009
2010                 bytes[mac] += skb->len;
2011                 done[mac]++;
2012             }
2013             budget--;
2014         }
2015         mtk_tx_unmap(eth, tx_buf, &bq, true);
2016
2017         ring->last_free = desc;
2018         atomic_inc(&ring->free_count);
2019
2020         cpu = next_cpu;
2021     }
2022     xdp_flush_frame_bulk(&bq);
2023
2024     ring->last_free_ptr = cpu;
2025     mtk_w32(eth, cpu, reg_map->qdma.crx_ptr);
2026
2027     return budget;
2028 }
2029
2030 static int mtk_poll_tx_pdma(struct mtk_eth *eth, int budget,
2031                 unsigned int *done, unsigned int *bytes)
2032 {
2033     struct mtk_tx_ring *ring = &eth->tx_ring;
2034     struct mtk_tx_buf *tx_buf;
2035     struct xdp_frame_bulk bq;
2036     struct mtk_tx_dma *desc;
2037     u32 cpu, dma;
2038
2039     cpu = ring->cpu_idx;
2040     dma = mtk_r32(eth, MT7628_TX_DTX_IDX0);
2041     xdp_frame_bulk_init(&bq);
2042
2043     while ((cpu != dma) && budget) {
2044         tx_buf = &ring->buf[cpu];
2045         if (!tx_buf->data)
2046             break;
2047
2048         if (tx_buf->data != (void *)MTK_DMA_DUMMY_DESC) {
2049             if (tx_buf->type == MTK_TYPE_SKB) {
2050                 struct sk_buff *skb = tx_buf->data;
2051
2052                 bytes[0] += skb->len;
2053                 done[0]++;
2054             }
2055             budget--;
2056         }
2057         mtk_tx_unmap(eth, tx_buf, &bq, true);
2058
2059         desc = ring->dma + cpu * eth->soc->txrx.txd_size;
2060         ring->last_free = desc;
2061         atomic_inc(&ring->free_count);
2062
2063         cpu = NEXT_DESP_IDX(cpu, ring->dma_size);
2064     }
2065     xdp_flush_frame_bulk(&bq);
2066
2067     ring->cpu_idx = cpu;
2068
2069     return budget;
2070 }
2071
2072 static int mtk_poll_tx(struct mtk_eth *eth, int budget)
2073 {
2074     struct mtk_tx_ring *ring = &eth->tx_ring;
2075     struct dim_sample dim_sample = {};
2076     unsigned int done[MTK_MAX_DEVS];
2077     unsigned int bytes[MTK_MAX_DEVS];
2078     int total = 0, i;
2079
2080     memset(done, 0, sizeof(done));
2081     memset(bytes, 0, sizeof(bytes));
2082
2083     if (MTK_HAS_CAPS(eth->soc->caps, MTK_QDMA))
2084         budget = mtk_poll_tx_qdma(eth, budget, done, bytes);
2085     else
2086         budget = mtk_poll_tx_pdma(eth, budget, done, bytes);
2087
2088     for (i = 0; i < MTK_MAC_COUNT; i++) {
2089         if (!eth->netdev[i] || !done[i])
2090             continue;
2091         netdev_completed_queue(eth->netdev[i], done[i], bytes[i]);
2092         total += done[i];
2093         eth->tx_packets += done[i];
2094         eth->tx_bytes += bytes[i];
2095     }
2096
2097     dim_update_sample(eth->tx_events, eth->tx_packets, eth->tx_bytes,
2098               &dim_sample);
2099     net_dim(&eth->tx_dim, dim_sample);
2100
2101     if (mtk_queue_stopped(eth) &&
2102         (atomic_read(&ring->free_count) > ring->thresh))
2103         mtk_wake_queue(eth);
2104
2105     return total;
2106 }
2107
2108 static void mtk_handle_status_irq(struct mtk_eth *eth)
2109 {
2110     u32 status2 = mtk_r32(eth, MTK_INT_STATUS2);
2111
2112     if (unlikely(status2 & (MTK_GDM1_AF | MTK_GDM2_AF))) {
2113         mtk_stats_update(eth);
2114         mtk_w32(eth, (MTK_GDM1_AF | MTK_GDM2_AF),
2115             MTK_INT_STATUS2);
2116     }
2117 }
2118
2119 static int mtk_napi_tx(struct napi_struct *napi, int budget)
2120 {
2121     struct mtk_eth *eth = container_of(napi, struct mtk_eth, tx_napi);
2122     const struct mtk_reg_map *reg_map = eth->soc->reg_map;
2123     int tx_done = 0;
2124
2125     if (MTK_HAS_CAPS(eth->soc->caps, MTK_QDMA))
2126         mtk_handle_status_irq(eth);
2127     mtk_w32(eth, MTK_TX_DONE_INT, reg_map->tx_irq_status);
2128     tx_done = mtk_poll_tx(eth, budget);
2129
2130     if (unlikely(netif_msg_intr(eth))) {
2131         dev_info(eth->dev,
2132              "done tx %d, intr 0x%08x/0x%x\n", tx_done,
2133              mtk_r32(eth, reg_map->tx_irq_status),
2134              mtk_r32(eth, reg_map->tx_irq_mask));
2135     }
2136
2137     if (tx_done == budget)
2138         return budget;
2139
2140     if (mtk_r32(eth, reg_map->tx_irq_status) & MTK_TX_DONE_INT)
2141         return budget;
2142
2143     if (napi_complete_done(napi, tx_done))
2144         mtk_tx_irq_enable(eth, MTK_TX_DONE_INT);
2145
2146     return tx_done;
2147 }
2148
2149 static int mtk_napi_rx(struct napi_struct *napi, int budget)
2150 {
2151     struct mtk_eth *eth = container_of(napi, struct mtk_eth, rx_napi);
2152     const struct mtk_reg_map *reg_map = eth->soc->reg_map;
2153     int rx_done_total = 0;
2154
2155     mtk_handle_status_irq(eth);
2156
2157     do {
2158         int rx_done;
2159
2160         mtk_w32(eth, eth->soc->txrx.rx_irq_done_mask,
2161             reg_map->pdma.irq_status);
2162         rx_done = mtk_poll_rx(napi, budget - rx_done_total, eth);
2163         rx_done_total += rx_done;
2164
2165         if (unlikely(netif_msg_intr(eth))) {
2166             dev_info(eth->dev,
2167                  "done rx %d, intr 0x%08x/0x%x\n", rx_done,
2168                  mtk_r32(eth, reg_map->pdma.irq_status),
2169                  mtk_r32(eth, reg_map->pdma.irq_mask));
2170         }
2171
2172         if (rx_done_total == budget)
2173             return budget;
2174
2175     } while (mtk_r32(eth, reg_map->pdma.irq_status) &
2176          eth->soc->txrx.rx_irq_done_mask);
2177
2178     if (napi_complete_done(napi, rx_done_total))
2179         mtk_rx_irq_enable(eth, eth->soc->txrx.rx_irq_done_mask);
2180
2181     return rx_done_total;
2182 }
2183
2184 static int mtk_tx_alloc(struct mtk_eth *eth)
2185 {
2186     const struct mtk_soc_data *soc = eth->soc;
2187     struct mtk_tx_ring *ring = &eth->tx_ring;
2188     int i, sz = soc->txrx.txd_size;
2189     struct mtk_tx_dma_v2 *txd;
2190
2191     ring->buf = kcalloc(MTK_DMA_SIZE, sizeof(*ring->buf),
2192                    GFP_KERNEL);
2193     if (!ring->buf)
2194         goto no_tx_mem;
2195
2196     ring->dma = dma_alloc_coherent(eth->dma_dev, MTK_DMA_SIZE * sz,
2197                        &ring->phys, GFP_KERNEL);
2198     if (!ring->dma)
2199         goto no_tx_mem;
2200
2201     for (i = 0; i < MTK_DMA_SIZE; i++) {
2202         int next = (i + 1) % MTK_DMA_SIZE;
2203         u32 next_ptr = ring->phys + next * sz;
2204
2205         txd = ring->dma + i * sz;
2206         txd->txd2 = next_ptr;
2207         txd->txd3 = TX_DMA_LS0 | TX_DMA_OWNER_CPU;
2208         txd->txd4 = 0;
2209         if (MTK_HAS_CAPS(soc->caps, MTK_NETSYS_V2)) {
2210             txd->txd5 = 0;
2211             txd->txd6 = 0;
2212             txd->txd7 = 0;
2213             txd->txd8 = 0;
2214         }
2215     }
2216
2217     /* On MT7688 (PDMA only) this driver uses the ring->dma structs
2218      * only as the framework. The real HW descriptors are the PDMA
2219      * descriptors in ring->dma_pdma.
2220      */
2221     if (!MTK_HAS_CAPS(soc->caps, MTK_QDMA)) {
2222         ring->dma_pdma = dma_alloc_coherent(eth->dma_dev, MTK_DMA_SIZE * sz,
2223                             &ring->phys_pdma, GFP_KERNEL);
2224         if (!ring->dma_pdma)
2225             goto no_tx_mem;
2226
2227         for (i = 0; i < MTK_DMA_SIZE; i++) {
2228             ring->dma_pdma[i].txd2 = TX_DMA_DESP2_DEF;
2229             ring->dma_pdma[i].txd4 = 0;
2230         }
2231     }
2232
2233     ring->dma_size = MTK_DMA_SIZE;
2234     atomic_set(&ring->free_count, MTK_DMA_SIZE - 2);
2235     ring->next_free = ring->dma;
2236     ring->last_free = (void *)txd;
2237     ring->last_free_ptr = (u32)(ring->phys + ((MTK_DMA_SIZE - 1) * sz));
2238     ring->thresh = MAX_SKB_FRAGS;
2239
2240     /* make sure that all changes to the dma ring are flushed before we
2241      * continue
2242      */
2243     wmb();
2244
2245     if (MTK_HAS_CAPS(soc->caps, MTK_QDMA)) {
2246         mtk_w32(eth, ring->phys, soc->reg_map->qdma.ctx_ptr);
2247         mtk_w32(eth, ring->phys, soc->reg_map->qdma.dtx_ptr);
2248         mtk_w32(eth,
2249             ring->phys + ((MTK_DMA_SIZE - 1) * sz),
2250             soc->reg_map->qdma.crx_ptr);
2251         mtk_w32(eth, ring->last_free_ptr, soc->reg_map->qdma.drx_ptr);
2252         mtk_w32(eth, (QDMA_RES_THRES << 8) | QDMA_RES_THRES,
2253             soc->reg_map->qdma.qtx_cfg);
2254     } else {
2255         mtk_w32(eth, ring->phys_pdma, MT7628_TX_BASE_PTR0);
2256         mtk_w32(eth, MTK_DMA_SIZE, MT7628_TX_MAX_CNT0);
2257         mtk_w32(eth, 0, MT7628_TX_CTX_IDX0);
2258         mtk_w32(eth, MT7628_PST_DTX_IDX0, soc->reg_map->pdma.rst_idx);
2259     }
2260
2261     return 0;
2262
2263 no_tx_mem:
2264     return -ENOMEM;
2265 }
2266
2267 static void mtk_tx_clean(struct mtk_eth *eth)
2268 {
2269     const struct mtk_soc_data *soc = eth->soc;
2270     struct mtk_tx_ring *ring = &eth->tx_ring;
2271     int i;
2272
2273     if (ring->buf) {
2274         for (i = 0; i < MTK_DMA_SIZE; i++)
2275             mtk_tx_unmap(eth, &ring->buf[i], NULL, false);
2276         kfree(ring->buf);
2277         ring->buf = NULL;
2278     }
2279
2280     if (ring->dma) {
2281         dma_free_coherent(eth->dma_dev,
2282                   MTK_DMA_SIZE * soc->txrx.txd_size,
2283                   ring->dma, ring->phys);
2284         ring->dma = NULL;
2285     }
2286
2287     if (ring->dma_pdma) {
2288         dma_free_coherent(eth->dma_dev,
2289                   MTK_DMA_SIZE * soc->txrx.txd_size,
2290                   ring->dma_pdma, ring->phys_pdma);
2291         ring->dma_pdma = NULL;
2292     }
2293 }
2294
2295 static int mtk_rx_alloc(struct mtk_eth *eth, int ring_no, int rx_flag)
2296 {
2297     const struct mtk_reg_map *reg_map = eth->soc->reg_map;
2298     struct mtk_rx_ring *ring;
2299     int rx_data_len, rx_dma_size;
2300     int i;
2301
2302     if (rx_flag == MTK_RX_FLAGS_QDMA) {
2303         if (ring_no)
2304             return -EINVAL;
2305         ring = &eth->rx_ring_qdma;
2306     } else {
2307         ring = &eth->rx_ring[ring_no];
2308     }
2309
2310     if (rx_flag == MTK_RX_FLAGS_HWLRO) {
2311         rx_data_len = MTK_MAX_LRO_RX_LENGTH;
2312         rx_dma_size = MTK_HW_LRO_DMA_SIZE;
2313     } else {
2314         rx_data_len = ETH_DATA_LEN;
2315         rx_dma_size = MTK_DMA_SIZE;
2316     }
2317
2318     ring->frag_size = mtk_max_frag_size(rx_data_len);
2319     ring->buf_size = mtk_max_buf_size(ring->frag_size);
2320     ring->data = kcalloc(rx_dma_size, sizeof(*ring->data),
2321                  GFP_KERNEL);
2322     if (!ring->data)
2323         return -ENOMEM;
2324
2325     if (mtk_page_pool_enabled(eth)) {
2326         struct page_pool *pp;
2327
2328         pp = mtk_create_page_pool(eth, &ring->xdp_q, ring_no,
2329                       rx_dma_size);
2330         if (IS_ERR(pp))
2331             return PTR_ERR(pp);
2332
2333         ring->page_pool = pp;
2334     }
2335
2336     ring->dma = dma_alloc_coherent(eth->dma_dev,
2337                        rx_dma_size * eth->soc->txrx.rxd_size,
2338                        &ring->phys, GFP_KERNEL);
2339     if (!ring->dma)
2340         return -ENOMEM;
2341
2342     for (i = 0; i < rx_dma_size; i++) {
2343         struct mtk_rx_dma_v2 *rxd;
2344         dma_addr_t dma_addr;
2345         void *data;
2346
2347         rxd = ring->dma + i * eth->soc->txrx.rxd_size;
2348         if (ring->page_pool) {
2349             data = mtk_page_pool_get_buff(ring->page_pool,
2350                               &dma_addr, GFP_KERNEL);
2351             if (!data)
2352                 return -ENOMEM;
2353         } else {
2354             if (ring->frag_size <= PAGE_SIZE)
2355                 data = netdev_alloc_frag(ring->frag_size);
2356             else
2357                 data = mtk_max_lro_buf_alloc(GFP_KERNEL);
2358
2359             if (!data)
2360                 return -ENOMEM;
2361
2362             dma_addr = dma_map_single(eth->dma_dev,
2363                 data + NET_SKB_PAD + eth->ip_align,
2364                 ring->buf_size, DMA_FROM_DEVICE);
2365             if (unlikely(dma_mapping_error(eth->dma_dev,
2366                                dma_addr)))
2367                 return -ENOMEM;
2368         }
2369         rxd->rxd1 = (unsigned int)dma_addr;
2370         ring->data[i] = data;
2371
2372         if (MTK_HAS_CAPS(eth->soc->caps, MTK_SOC_MT7628))
2373             rxd->rxd2 = RX_DMA_LSO;
2374         else
2375             rxd->rxd2 = RX_DMA_PREP_PLEN0(ring->buf_size);
2376
2377         rxd->rxd3 = 0;
2378         rxd->rxd4 = 0;
2379         if (MTK_HAS_CAPS(eth->soc->caps, MTK_NETSYS_V2)) {
2380             rxd->rxd5 = 0;
2381             rxd->rxd6 = 0;
2382             rxd->rxd7 = 0;
2383             rxd->rxd8 = 0;
2384         }
2385     }
2386
2387     ring->dma_size = rx_dma_size;
2388     ring->calc_idx_update = false;
2389     ring->calc_idx = rx_dma_size - 1;
2390     if (rx_flag == MTK_RX_FLAGS_QDMA)
2391         ring->crx_idx_reg = reg_map->qdma.qcrx_ptr +
2392                     ring_no * MTK_QRX_OFFSET;
2393     else
2394         ring->crx_idx_reg = reg_map->pdma.pcrx_ptr +
2395                     ring_no * MTK_QRX_OFFSET;
2396     /* make sure that all changes to the dma ring are flushed before we
2397      * continue
2398      */
2399     wmb();
2400
2401     if (rx_flag == MTK_RX_FLAGS_QDMA) {
2402         mtk_w32(eth, ring->phys,
2403             reg_map->qdma.rx_ptr + ring_no * MTK_QRX_OFFSET);
2404         mtk_w32(eth, rx_dma_size,
2405             reg_map->qdma.rx_cnt_cfg + ring_no * MTK_QRX_OFFSET);
2406         mtk_w32(eth, MTK_PST_DRX_IDX_CFG(ring_no),
2407             reg_map->qdma.rst_idx);
2408     } else {
2409         mtk_w32(eth, ring->phys,
2410             reg_map->pdma.rx_ptr + ring_no * MTK_QRX_OFFSET);
2411         mtk_w32(eth, rx_dma_size,
2412             reg_map->pdma.rx_cnt_cfg + ring_no * MTK_QRX_OFFSET);
2413         mtk_w32(eth, MTK_PST_DRX_IDX_CFG(ring_no),
2414             reg_map->pdma.rst_idx);
2415     }
2416     mtk_w32(eth, ring->calc_idx, ring->crx_idx_reg);
2417
2418     return 0;
2419 }
2420
2421 static void mtk_rx_clean(struct mtk_eth *eth, struct mtk_rx_ring *ring)
2422 {
2423     int i;
2424
2425     if (ring->data && ring->dma) {
2426         for (i = 0; i < ring->dma_size; i++) {
2427             struct mtk_rx_dma *rxd;
2428
2429             if (!ring->data[i])
2430                 continue;
2431
2432             rxd = ring->dma + i * eth->soc->txrx.rxd_size;
2433             if (!rxd->rxd1)
2434                 continue;
2435
2436             dma_unmap_single(eth->dma_dev, rxd->rxd1,
2437                      ring->buf_size, DMA_FROM_DEVICE);
2438             mtk_rx_put_buff(ring, ring->data[i], false);
2439         }
2440         kfree(ring->data);
2441         ring->data = NULL;
2442     }
2443
2444     if (ring->dma) {
2445         dma_free_coherent(eth->dma_dev,
2446                   ring->dma_size * eth->soc->txrx.rxd_size,
2447                   ring->dma, ring->phys);
2448         ring->dma = NULL;
2449     }
2450
2451     if (ring->page_pool) {
2452         if (xdp_rxq_info_is_reg(&ring->xdp_q))
2453             xdp_rxq_info_unreg(&ring->xdp_q);
2454         page_pool_destroy(ring->page_pool);
2455         ring->page_pool = NULL;
2456     }
2457 }
2458
2459 static int mtk_hwlro_rx_init(struct mtk_eth *eth)
2460 {
2461     int i;
2462     u32 ring_ctrl_dw1 = 0, ring_ctrl_dw2 = 0, ring_ctrl_dw3 = 0;
2463     u32 lro_ctrl_dw0 = 0, lro_ctrl_dw3 = 0;
2464
2465     /* set LRO rings to auto-learn modes */
2466     ring_ctrl_dw2 |= MTK_RING_AUTO_LERAN_MODE;
2467
2468     /* validate LRO ring */
2469     ring_ctrl_dw2 |= MTK_RING_VLD;
2470
2471     /* set AGE timer (unit: 20us) */
2472     ring_ctrl_dw2 |= MTK_RING_AGE_TIME_H;
2473     ring_ctrl_dw1 |= MTK_RING_AGE_TIME_L;
2474
2475     /* set max AGG timer (unit: 20us) */
2476     ring_ctrl_dw2 |= MTK_RING_MAX_AGG_TIME;
2477
2478     /* set max LRO AGG count */
2479     ring_ctrl_dw2 |= MTK_RING_MAX_AGG_CNT_L;
2480     ring_ctrl_dw3 |= MTK_RING_MAX_AGG_CNT_H;
2481
2482     for (i = 1; i < MTK_MAX_RX_RING_NUM; i++) {
2483         mtk_w32(eth, ring_ctrl_dw1, MTK_LRO_CTRL_DW1_CFG(i));
2484         mtk_w32(eth, ring_ctrl_dw2, MTK_LRO_CTRL_DW2_CFG(i));
2485         mtk_w32(eth, ring_ctrl_dw3, MTK_LRO_CTRL_DW3_CFG(i));
2486     }
2487
2488     /* IPv4 checksum update enable */
2489     lro_ctrl_dw0 |= MTK_L3_CKS_UPD_EN;
2490
2491     /* switch priority comparison to packet count mode */
2492     lro_ctrl_dw0 |= MTK_LRO_ALT_PKT_CNT_MODE;
2493
2494     /* bandwidth threshold setting */
2495     mtk_w32(eth, MTK_HW_LRO_BW_THRE, MTK_PDMA_LRO_CTRL_DW2);
2496
2497     /* auto-learn score delta setting */
2498     mtk_w32(eth, MTK_HW_LRO_REPLACE_DELTA, MTK_PDMA_LRO_ALT_SCORE_DELTA);
2499
2500     /* set refresh timer for altering flows to 1 sec. (unit: 20us) */
2501     mtk_w32(eth, (MTK_HW_LRO_TIMER_UNIT << 16) | MTK_HW_LRO_REFRESH_TIME,
2502         MTK_PDMA_LRO_ALT_REFRESH_TIMER);
2503
2504     /* set HW LRO mode & the max aggregation count for rx packets */
2505     lro_ctrl_dw3 |= MTK_ADMA_MODE | (MTK_HW_LRO_MAX_AGG_CNT & 0xff);
2506
2507     /* the minimal remaining room of SDL0 in RXD for lro aggregation */
2508     lro_ctrl_dw3 |= MTK_LRO_MIN_RXD_SDL;
2509
2510     /* enable HW LRO */
2511     lro_ctrl_dw0 |= MTK_LRO_EN;
2512
2513     mtk_w32(eth, lro_ctrl_dw3, MTK_PDMA_LRO_CTRL_DW3);
2514     mtk_w32(eth, lro_ctrl_dw0, MTK_PDMA_LRO_CTRL_DW0);
2515
2516     return 0;
2517 }
2518
2519 static void mtk_hwlro_rx_uninit(struct mtk_eth *eth)
2520 {
2521     int i;
2522     u32 val;
2523
2524     /* relinquish lro rings, flush aggregated packets */
2525     mtk_w32(eth, MTK_LRO_RING_RELINQUISH_REQ, MTK_PDMA_LRO_CTRL_DW0);
2526
2527     /* wait for relinquishments done */
2528     for (i = 0; i < 10; i++) {
2529         val = mtk_r32(eth, MTK_PDMA_LRO_CTRL_DW0);
2530         if (val & MTK_LRO_RING_RELINQUISH_DONE) {
2531             msleep(20);
2532             continue;
2533         }
2534         break;
2535     }
2536
2537     /* invalidate lro rings */
2538     for (i = 1; i < MTK_MAX_RX_RING_NUM; i++)
2539         mtk_w32(eth, 0, MTK_LRO_CTRL_DW2_CFG(i));
2540
2541     /* disable HW LRO */
2542     mtk_w32(eth, 0, MTK_PDMA_LRO_CTRL_DW0);
2543 }
2544
2545 static void mtk_hwlro_val_ipaddr(struct mtk_eth *eth, int idx, __be32 ip)
2546 {
2547     u32 reg_val;
2548
2549     reg_val = mtk_r32(eth, MTK_LRO_CTRL_DW2_CFG(idx));
2550
2551     /* invalidate the IP setting */
2552     mtk_w32(eth, (reg_val & ~MTK_RING_MYIP_VLD), MTK_LRO_CTRL_DW2_CFG(idx));
2553
2554     mtk_w32(eth, ip, MTK_LRO_DIP_DW0_CFG(idx));
2555
2556     /* validate the IP setting */
2557     mtk_w32(eth, (reg_val | MTK_RING_MYIP_VLD), MTK_LRO_CTRL_DW2_CFG(idx));
2558 }
2559
2560 static void mtk_hwlro_inval_ipaddr(struct mtk_eth *eth, int idx)
2561 {
2562     u32 reg_val;
2563
2564     reg_val = mtk_r32(eth, MTK_LRO_CTRL_DW2_CFG(idx));
2565
2566     /* invalidate the IP setting */
2567     mtk_w32(eth, (reg_val & ~MTK_RING_MYIP_VLD), MTK_LRO_CTRL_DW2_CFG(idx));
2568
2569     mtk_w32(eth, 0, MTK_LRO_DIP_DW0_CFG(idx));
2570 }
2571
2572 static int mtk_hwlro_get_ip_cnt(struct mtk_mac *mac)
2573 {
2574     int cnt = 0;
2575     int i;
2576
2577     for (i = 0; i < MTK_MAX_LRO_IP_CNT; i++) {
2578         if (mac->hwlro_ip[i])
2579             cnt++;
2580     }
2581
2582     return cnt;
2583 }
2584
2585 static int mtk_hwlro_add_ipaddr(struct net_device *dev,
2586                 struct ethtool_rxnfc *cmd)
2587 {
2588     struct ethtool_rx_flow_spec *fsp =
2589         (struct ethtool_rx_flow_spec *)&cmd->fs;
2590     struct mtk_mac *mac = netdev_priv(dev);
2591     struct mtk_eth *eth = mac->hw;
2592     int hwlro_idx;
2593
2594     if ((fsp->flow_type != TCP_V4_FLOW) ||
2595         (!fsp->h_u.tcp_ip4_spec.ip4dst) ||
2596         (fsp->location > 1))
2597         return -EINVAL;
2598
2599     mac->hwlro_ip[fsp->location] = htonl(fsp->h_u.tcp_ip4_spec.ip4dst);
2600     hwlro_idx = (mac->id * MTK_MAX_LRO_IP_CNT) + fsp->location;
2601
2602     mac->hwlro_ip_cnt = mtk_hwlro_get_ip_cnt(mac);
2603
2604     mtk_hwlro_val_ipaddr(eth, hwlro_idx, mac->hwlro_ip[fsp->location]);
2605
2606     return 0;
2607 }
2608
2609 static int mtk_hwlro_del_ipaddr(struct net_device *dev,
2610                 struct ethtool_rxnfc *cmd)
2611 {
2612     struct ethtool_rx_flow_spec *fsp =
2613         (struct ethtool_rx_flow_spec *)&cmd->fs;
2614     struct mtk_mac *mac = netdev_priv(dev);
2615     struct mtk_eth *eth = mac->hw;
2616     int hwlro_idx;
2617
2618     if (fsp->location > 1)
2619         return -EINVAL;
2620
2621     mac->hwlro_ip[fsp->location] = 0;
2622     hwlro_idx = (mac->id * MTK_MAX_LRO_IP_CNT) + fsp->location;
2623
2624     mac->hwlro_ip_cnt = mtk_hwlro_get_ip_cnt(mac);
2625
2626     mtk_hwlro_inval_ipaddr(eth, hwlro_idx);
2627
2628     return 0;
2629 }
2630
2631 static void mtk_hwlro_netdev_disable(struct net_device *dev)
2632 {
2633     struct mtk_mac *mac = netdev_priv(dev);
2634     struct mtk_eth *eth = mac->hw;
2635     int i, hwlro_idx;
2636
2637     for (i = 0; i < MTK_MAX_LRO_IP_CNT; i++) {
2638         mac->hwlro_ip[i] = 0;
2639         hwlro_idx = (mac->id * MTK_MAX_LRO_IP_CNT) + i;
2640
2641         mtk_hwlro_inval_ipaddr(eth, hwlro_idx);
2642     }
2643
2644     mac->hwlro_ip_cnt = 0;
2645 }
2646
2647 static int mtk_hwlro_get_fdir_entry(struct net_device *dev,
2648                     struct ethtool_rxnfc *cmd)
2649 {
2650     struct mtk_mac *mac = netdev_priv(dev);
2651     struct ethtool_rx_flow_spec *fsp =
2652         (struct ethtool_rx_flow_spec *)&cmd->fs;
2653
2654     if (fsp->location >= ARRAY_SIZE(mac->hwlro_ip))
2655         return -EINVAL;
2656
2657     /* only tcp dst ipv4 is meaningful, others are meaningless */
2658     fsp->flow_type = TCP_V4_FLOW;
2659     fsp->h_u.tcp_ip4_spec.ip4dst = ntohl(mac->hwlro_ip[fsp->location]);
2660     fsp->m_u.tcp_ip4_spec.ip4dst = 0;
2661
2662     fsp->h_u.tcp_ip4_spec.ip4src = 0;
2663     fsp->m_u.tcp_ip4_spec.ip4src = 0xffffffff;
2664     fsp->h_u.tcp_ip4_spec.psrc = 0;
2665     fsp->m_u.tcp_ip4_spec.psrc = 0xffff;
2666     fsp->h_u.tcp_ip4_spec.pdst = 0;
2667     fsp->m_u.tcp_ip4_spec.pdst = 0xffff;
2668     fsp->h_u.tcp_ip4_spec.tos = 0;
2669     fsp->m_u.tcp_ip4_spec.tos = 0xff;
2670
2671     return 0;
2672 }
2673
2674 static int mtk_hwlro_get_fdir_all(struct net_device *dev,
2675                   struct ethtool_rxnfc *cmd,
2676                   u32 *rule_locs)
2677 {
2678     struct mtk_mac *mac = netdev_priv(dev);
2679     int cnt = 0;
2680     int i;
2681
2682     for (i = 0; i < MTK_MAX_LRO_IP_CNT; i++) {
2683         if (mac->hwlro_ip[i]) {
2684             rule_locs[cnt] = i;
2685             cnt++;
2686         }
2687     }
2688
2689     cmd->rule_cnt = cnt;
2690
2691     return 0;
2692 }
2693
2694 static netdev_features_t mtk_fix_features(struct net_device *dev,
2695                       netdev_features_t features)
2696 {
2697     if (!(features & NETIF_F_LRO)) {
2698         struct mtk_mac *mac = netdev_priv(dev);
2699         int ip_cnt = mtk_hwlro_get_ip_cnt(mac);
2700
2701         if (ip_cnt) {
2702             netdev_info(dev, "RX flow is programmed, LRO should keep on\n");
2703
2704             features |= NETIF_F_LRO;
2705         }
2706     }
2707
2708     return features;
2709 }
2710
2711 static int mtk_set_features(struct net_device *dev, netdev_features_t features)
2712 {
2713     int err = 0;
2714
2715     if (!((dev->features ^ features) & NETIF_F_LRO))
2716         return 0;
2717
2718     if (!(features & NETIF_F_LRO))
2719         mtk_hwlro_netdev_disable(dev);
2720
2721     return err;
2722 }
2723
2724 /* wait for DMA to finish whatever it is doing before we start using it again */
2725 static int mtk_dma_busy_wait(struct mtk_eth *eth)
2726 {
2727     unsigned int reg;
2728     int ret;
2729     u32 val;
2730
2731     if (MTK_HAS_CAPS(eth->soc->caps, MTK_QDMA))
2732         reg = eth->soc->reg_map->qdma.glo_cfg;
2733     else
2734         reg = eth->soc->reg_map->pdma.glo_cfg;
2735
2736     ret = readx_poll_timeout_atomic(__raw_readl, eth->base + reg, val,
2737                     !(val & (MTK_RX_DMA_BUSY | MTK_TX_DMA_BUSY)),
2738                     5, MTK_DMA_BUSY_TIMEOUT_US);
2739     if (ret)
2740         dev_err(eth->dev, "DMA init timeout\n");
2741
2742     return ret;
2743 }
2744
2745 static int mtk_dma_init(struct mtk_eth *eth)
2746 {
2747     int err;
2748     u32 i;
2749
2750     if (mtk_dma_busy_wait(eth))
2751         return -EBUSY;
2752
2753     if (MTK_HAS_CAPS(eth->soc->caps, MTK_QDMA)) {
2754         /* QDMA needs scratch memory for internal reordering of the
2755          * descriptors
2756          */
2757         err = mtk_init_fq_dma(eth);
2758         if (err)
2759             return err;
2760     }
2761
2762     err = mtk_tx_alloc(eth);
2763     if (err)
2764         return err;
2765
2766     if (MTK_HAS_CAPS(eth->soc->caps, MTK_QDMA)) {
2767         err = mtk_rx_alloc(eth, 0, MTK_RX_FLAGS_QDMA);
2768         if (err)
2769             return err;
2770     }
2771
2772     err = mtk_rx_alloc(eth, 0, MTK_RX_FLAGS_NORMAL);
2773     if (err)
2774         return err;
2775
2776     if (eth->hwlro) {
2777         for (i = 1; i < MTK_MAX_RX_RING_NUM; i++) {
2778             err = mtk_rx_alloc(eth, i, MTK_RX_FLAGS_HWLRO);
2779             if (err)
2780                 return err;
2781         }
2782         err = mtk_hwlro_rx_init(eth);
2783         if (err)
2784             return err;
2785     }
2786
2787     if (MTK_HAS_CAPS(eth->soc->caps, MTK_QDMA)) {
2788         /* Enable random early drop and set drop threshold
2789          * automatically
2790          */
2791         mtk_w32(eth, FC_THRES_DROP_MODE | FC_THRES_DROP_EN |
2792             FC_THRES_MIN, eth->soc->reg_map->qdma.fc_th);
2793         mtk_w32(eth, 0x0, eth->soc->reg_map->qdma.hred);
2794     }
2795
2796     return 0;
2797 }
2798
2799 static void mtk_dma_free(struct mtk_eth *eth)
2800 {
2801     const struct mtk_soc_data *soc = eth->soc;
2802     int i;
2803
2804     for (i = 0; i < MTK_MAC_COUNT; i++)
2805         if (eth->netdev[i])
2806             netdev_reset_queue(eth->netdev[i]);
2807     if (eth->scratch_ring) {
2808         dma_free_coherent(eth->dma_dev,
2809                   MTK_DMA_SIZE * soc->txrx.txd_size,
2810                   eth->scratch_ring, eth->phy_scratch_ring);
2811         eth->scratch_ring = NULL;
2812         eth->phy_scratch_ring = 0;
2813     }
2814     mtk_tx_clean(eth);
2815     mtk_rx_clean(eth, &eth->rx_ring[0]);
2816     mtk_rx_clean(eth, &eth->rx_ring_qdma);
2817
2818     if (eth->hwlro) {
2819         mtk_hwlro_rx_uninit(eth);
2820         for (i = 1; i < MTK_MAX_RX_RING_NUM; i++)
2821             mtk_rx_clean(eth, &eth->rx_ring[i]);
2822     }
2823
2824     kfree(eth->scratch_head);
2825 }
2826
2827 static void mtk_tx_timeout(struct net_device *dev, unsigned int txqueue)
2828 {
2829     struct mtk_mac *mac = netdev_priv(dev);
2830     struct mtk_eth *eth = mac->hw;
2831
2832     eth->netdev[mac->id]->stats.tx_errors++;
2833     netif_err(eth, tx_err, dev,
2834           "transmit timed out\n");
2835     schedule_work(&eth->pending_work);
2836 }
2837
2838 static irqreturn_t mtk_handle_irq_rx(int irq, void *_eth)
2839 {
2840     struct mtk_eth *eth = _eth;
2841
2842     eth->rx_events++;
2843     if (likely(napi_schedule_prep(&eth->rx_napi))) {
2844         __napi_schedule(&eth->rx_napi);
2845         mtk_rx_irq_disable(eth, eth->soc->txrx.rx_irq_done_mask);
2846     }
2847
2848     return IRQ_HANDLED;
2849 }
2850
2851 static irqreturn_t mtk_handle_irq_tx(int irq, void *_eth)
2852 {
2853     struct mtk_eth *eth = _eth;
2854
2855     eth->tx_events++;
2856     if (likely(napi_schedule_prep(&eth->tx_napi))) {
2857         __napi_schedule(&eth->tx_napi);
2858         mtk_tx_irq_disable(eth, MTK_TX_DONE_INT);
2859     }
2860
2861     return IRQ_HANDLED;
2862 }
2863
2864 static irqreturn_t mtk_handle_irq(int irq, void *_eth)
2865 {
2866     struct mtk_eth *eth = _eth;
2867     const struct mtk_reg_map *reg_map = eth->soc->reg_map;
2868
2869     if (mtk_r32(eth, reg_map->pdma.irq_mask) &
2870         eth->soc->txrx.rx_irq_done_mask) {
2871         if (mtk_r32(eth, reg_map->pdma.irq_status) &
2872             eth->soc->txrx.rx_irq_done_mask)
2873             mtk_handle_irq_rx(irq, _eth);
2874     }
2875     if (mtk_r32(eth, reg_map->tx_irq_mask) & MTK_TX_DONE_INT) {
2876         if (mtk_r32(eth, reg_map->tx_irq_status) & MTK_TX_DONE_INT)
2877             mtk_handle_irq_tx(irq, _eth);
2878     }
2879
2880     return IRQ_HANDLED;
2881 }
2882
2883 #ifdef CONFIG_NET_POLL_CONTROLLER
2884 static void mtk_poll_controller(struct net_device *dev)
2885 {
2886     struct mtk_mac *mac = netdev_priv(dev);
2887     struct mtk_eth *eth = mac->hw;
2888
2889     mtk_tx_irq_disable(eth, MTK_TX_DONE_INT);
2890     mtk_rx_irq_disable(eth, eth->soc->txrx.rx_irq_done_mask);
2891     mtk_handle_irq_rx(eth->irq[2], dev);
2892     mtk_tx_irq_enable(eth, MTK_TX_DONE_INT);
2893     mtk_rx_irq_enable(eth, eth->soc->txrx.rx_irq_done_mask);
2894 }
2895 #endif
2896
2897 static int mtk_start_dma(struct mtk_eth *eth)
2898 {
2899     u32 val, rx_2b_offset = (NET_IP_ALIGN == 2) ? MTK_RX_2B_OFFSET : 0;
2900     const struct mtk_reg_map *reg_map = eth->soc->reg_map;
2901     int err;
2902
2903     err = mtk_dma_init(eth);
2904     if (err) {
2905         mtk_dma_free(eth);
2906         return err;
2907     }
2908
2909     if (MTK_HAS_CAPS(eth->soc->caps, MTK_QDMA)) {
2910         val = mtk_r32(eth, reg_map->qdma.glo_cfg);
2911         val |= MTK_TX_DMA_EN | MTK_RX_DMA_EN |
2912                MTK_TX_BT_32DWORDS | MTK_NDP_CO_PRO |
2913                MTK_RX_2B_OFFSET | MTK_TX_WB_DDONE;
2914
2915         if (MTK_HAS_CAPS(eth->soc->caps, MTK_NETSYS_V2))
2916             val |= MTK_MUTLI_CNT | MTK_RESV_BUF |
2917                    MTK_WCOMP_EN | MTK_DMAD_WR_WDONE |
2918                    MTK_CHK_DDONE_EN;
2919         else
2920             val |= MTK_RX_BT_32DWORDS;
2921         mtk_w32(eth, val, reg_map->qdma.glo_cfg);
2922
2923         mtk_w32(eth,
2924             MTK_RX_DMA_EN | rx_2b_offset |
2925             MTK_RX_BT_32DWORDS | MTK_MULTI_EN,
2926             reg_map->pdma.glo_cfg);
2927     } else {
2928         mtk_w32(eth, MTK_TX_WB_DDONE | MTK_TX_DMA_EN | MTK_RX_DMA_EN |
2929             MTK_MULTI_EN | MTK_PDMA_SIZE_8DWORDS,
2930             reg_map->pdma.glo_cfg);
2931     }
2932
2933     return 0;
2934 }
2935
2936 static void mtk_gdm_config(struct mtk_eth *eth, u32 config)
2937 {
2938     int i;
2939
2940     if (MTK_HAS_CAPS(eth->soc->caps, MTK_SOC_MT7628))
2941         return;
2942
2943     for (i = 0; i < MTK_MAC_COUNT; i++) {
2944         u32 val = mtk_r32(eth, MTK_GDMA_FWD_CFG(i));
2945
2946         /* default setup the forward port to send frame to PDMA */
2947         val &= ~0xffff;
2948
2949         /* Enable RX checksum */
2950         val |= MTK_GDMA_ICS_EN | MTK_GDMA_TCS_EN | MTK_GDMA_UCS_EN;
2951
2952         val |= config;
2953
2954         if (!i && eth->netdev[0] && netdev_uses_dsa(eth->netdev[0]))
2955             val |= MTK_GDMA_SPECIAL_TAG;
2956
2957         mtk_w32(eth, val, MTK_GDMA_FWD_CFG(i));
2958     }
2959     /* Reset and enable PSE */
2960     mtk_w32(eth, RST_GL_PSE, MTK_RST_GL);
2961     mtk_w32(eth, 0, MTK_RST_GL);
2962 }
2963
2964 static int mtk_open(struct net_device *dev)
2965 {
2966     struct mtk_mac *mac = netdev_priv(dev);
2967     struct mtk_eth *eth = mac->hw;
2968     int err;
2969
2970     err = phylink_of_phy_connect(mac->phylink, mac->of_node, 0);
2971     if (err) {
2972         netdev_err(dev, "%s: could not attach PHY: %d\n", __func__,
2973                err);
2974         return err;
2975     }
2976
2977     /* we run 2 netdevs on the same dma ring so we only bring it up once */
2978     if (!refcount_read(&eth->dma_refcnt)) {
2979         u32 gdm_config = MTK_GDMA_TO_PDMA;
2980
2981         err = mtk_start_dma(eth);
2982         if (err)
2983             return err;
2984
2985         if (eth->soc->offload_version && mtk_ppe_start(eth->ppe) == 0)
2986             gdm_config = MTK_GDMA_TO_PPE;
2987
2988         mtk_gdm_config(eth, gdm_config);
2989
2990         napi_enable(&eth->tx_napi);
2991         napi_enable(&eth->rx_napi);
2992         mtk_tx_irq_enable(eth, MTK_TX_DONE_INT);
2993         mtk_rx_irq_enable(eth, eth->soc->txrx.rx_irq_done_mask);
2994         refcount_set(&eth->dma_refcnt, 1);
2995     }
2996     else
2997         refcount_inc(&eth->dma_refcnt);
2998
2999     phylink_start(mac->phylink);
3000     netif_start_queue(dev);
3001     return 0;
3002 }
3003
3004 static void mtk_stop_dma(struct mtk_eth *eth, u32 glo_cfg)
3005 {
3006     u32 val;
3007     int i;
3008
3009     /* stop the dma engine */
3010     spin_lock_bh(&eth->page_lock);
3011     val = mtk_r32(eth, glo_cfg);
3012     mtk_w32(eth, val & ~(MTK_TX_WB_DDONE | MTK_RX_DMA_EN | MTK_TX_DMA_EN),
3013         glo_cfg);
3014     spin_unlock_bh(&eth->page_lock);
3015
3016     /* wait for dma stop */
3017     for (i = 0; i < 10; i++) {
3018         val = mtk_r32(eth, glo_cfg);
3019         if (val & (MTK_TX_DMA_BUSY | MTK_RX_DMA_BUSY)) {
3020             msleep(20);
3021             continue;
3022         }
3023         break;
3024     }
3025 }
3026
3027 static int mtk_stop(struct net_device *dev)
3028 {
3029     struct mtk_mac *mac = netdev_priv(dev);
3030     struct mtk_eth *eth = mac->hw;
3031
3032     phylink_stop(mac->phylink);
3033
3034     netif_tx_disable(dev);
3035
3036     phylink_disconnect_phy(mac->phylink);
3037
3038     /* only shutdown DMA if this is the last user */
3039     if (!refcount_dec_and_test(&eth->dma_refcnt))
3040         return 0;
3041
3042     mtk_gdm_config(eth, MTK_GDMA_DROP_ALL);
3043
3044     mtk_tx_irq_disable(eth, MTK_TX_DONE_INT);
3045     mtk_rx_irq_disable(eth, eth->soc->txrx.rx_irq_done_mask);
3046     napi_disable(&eth->tx_napi);
3047     napi_disable(&eth->rx_napi);
3048
3049     cancel_work_sync(&eth->rx_dim.work);
3050     cancel_work_sync(&eth->tx_dim.work);
3051
3052     if (MTK_HAS_CAPS(eth->soc->caps, MTK_QDMA))
3053         mtk_stop_dma(eth, eth->soc->reg_map->qdma.glo_cfg);
3054     mtk_stop_dma(eth, eth->soc->reg_map->pdma.glo_cfg);
3055
3056     mtk_dma_free(eth);
3057
3058     if (eth->soc->offload_version)
3059         mtk_ppe_stop(eth->ppe);
3060
3061     return 0;
3062 }
3063
3064 static int mtk_xdp_setup(struct net_device *dev, struct bpf_prog *prog,
3065              struct netlink_ext_ack *extack)
3066 {
3067     struct mtk_mac *mac = netdev_priv(dev);
3068     struct mtk_eth *eth = mac->hw;
3069     struct bpf_prog *old_prog;
3070     bool need_update;
3071
3072     if (eth->hwlro) {
3073         NL_SET_ERR_MSG_MOD(extack, "XDP not supported with HWLRO");
3074         return -EOPNOTSUPP;
3075     }
3076
3077     if (dev->mtu > MTK_PP_MAX_BUF_SIZE) {
3078         NL_SET_ERR_MSG_MOD(extack, "MTU too large for XDP");
3079         return -EOPNOTSUPP;
3080     }
3081
3082     need_update = !!eth->prog != !!prog;
3083     if (netif_running(dev) && need_update)
3084         mtk_stop(dev);
3085
3086     old_prog = rcu_replace_pointer(eth->prog, prog, lockdep_rtnl_is_held());
3087     if (old_prog)
3088         bpf_prog_put(old_prog);
3089
3090     if (netif_running(dev) && need_update)
3091         return mtk_open(dev);
3092
3093     return 0;
3094 }
3095
3096 static int mtk_xdp(struct net_device *dev, struct netdev_bpf *xdp)
3097 {
3098     switch (xdp->command) {
3099     case XDP_SETUP_PROG:
3100         return mtk_xdp_setup(dev, xdp->prog, xdp->extack);
3101     default:
3102         return -EINVAL;
3103     }
3104 }
3105
3106 static void ethsys_reset(struct mtk_eth *eth, u32 reset_bits)
3107 {
3108     regmap_update_bits(eth->ethsys, ETHSYS_RSTCTRL,
3109                reset_bits,
3110                reset_bits);
3111
3112     usleep_range(1000, 1100);
3113     regmap_update_bits(eth->ethsys, ETHSYS_RSTCTRL,
3114                reset_bits,
3115                ~reset_bits);
3116     mdelay(10);
3117 }
3118
3119 static void mtk_clk_disable(struct mtk_eth *eth)
3120 {
3121     int clk;
3122
3123     for (clk = MTK_CLK_MAX - 1; clk >= 0; clk--)
3124         clk_disable_unprepare(eth->clks[clk]);
3125 }
3126
3127 static int mtk_clk_enable(struct mtk_eth *eth)
3128 {
3129     int clk, ret;
3130
3131     for (clk = 0; clk < MTK_CLK_MAX ; clk++) {
3132         ret = clk_prepare_enable(eth->clks[clk]);
3133         if (ret)
3134             goto err_disable_clks;
3135     }
3136
3137     return 0;
3138
3139 err_disable_clks:
3140     while (--clk >= 0)
3141         clk_disable_unprepare(eth->clks[clk]);
3142
3143     return ret;
3144 }
3145
3146 static void mtk_dim_rx(struct work_struct *work)
3147 {
3148     struct dim *dim = container_of(work, struct dim, work);
3149     struct mtk_eth *eth = container_of(dim, struct mtk_eth, rx_dim);
3150     const struct mtk_reg_map *reg_map = eth->soc->reg_map;
3151     struct dim_cq_moder cur_profile;
3152     u32 val, cur;
3153
3154     cur_profile = net_dim_get_rx_moderation(eth->rx_dim.mode,
3155                         dim->profile_ix);
3156     spin_lock_bh(&eth->dim_lock);
3157
3158     val = mtk_r32(eth, reg_map->pdma.delay_irq);
3159     val &= MTK_PDMA_DELAY_TX_MASK;
3160     val |= MTK_PDMA_DELAY_RX_EN;
3161
3162     cur = min_t(u32, DIV_ROUND_UP(cur_profile.usec, 20), MTK_PDMA_DELAY_PTIME_MASK);
3163     val |= cur << MTK_PDMA_DELAY_RX_PTIME_SHIFT;
3164
3165     cur = min_t(u32, cur_profile.pkts, MTK_PDMA_DELAY_PINT_MASK);
3166     val |= cur << MTK_PDMA_DELAY_RX_PINT_SHIFT;
3167
3168     mtk_w32(eth, val, reg_map->pdma.delay_irq);
3169     if (MTK_HAS_CAPS(eth->soc->caps, MTK_QDMA))
3170         mtk_w32(eth, val, reg_map->qdma.delay_irq);
3171
3172     spin_unlock_bh(&eth->dim_lock);
3173
3174     dim->state = DIM_START_MEASURE;
3175 }
3176
3177 static void mtk_dim_tx(struct work_struct *work)
3178 {
3179     struct dim *dim = container_of(work, struct dim, work);
3180     struct mtk_eth *eth = container_of(dim, struct mtk_eth, tx_dim);
3181     const struct mtk_reg_map *reg_map = eth->soc->reg_map;
3182     struct dim_cq_moder cur_profile;
3183     u32 val, cur;
3184
3185     cur_profile = net_dim_get_tx_moderation(eth->tx_dim.mode,
3186                         dim->profile_ix);
3187     spin_lock_bh(&eth->dim_lock);
3188
3189     val = mtk_r32(eth, reg_map->pdma.delay_irq);
3190     val &= MTK_PDMA_DELAY_RX_MASK;
3191     val |= MTK_PDMA_DELAY_TX_EN;
3192
3193     cur = min_t(u32, DIV_ROUND_UP(cur_profile.usec, 20), MTK_PDMA_DELAY_PTIME_MASK);
3194     val |= cur << MTK_PDMA_DELAY_TX_PTIME_SHIFT;
3195
3196     cur = min_t(u32, cur_profile.pkts, MTK_PDMA_DELAY_PINT_MASK);
3197     val |= cur << MTK_PDMA_DELAY_TX_PINT_SHIFT;
3198
3199     mtk_w32(eth, val, reg_map->pdma.delay_irq);
3200     if (MTK_HAS_CAPS(eth->soc->caps, MTK_QDMA))
3201         mtk_w32(eth, val, reg_map->qdma.delay_irq);
3202
3203     spin_unlock_bh(&eth->dim_lock);
3204
3205     dim->state = DIM_START_MEASURE;
3206 }
3207
3208 static int mtk_hw_init(struct mtk_eth *eth)
3209 {
3210     u32 dma_mask = ETHSYS_DMA_AG_MAP_PDMA | ETHSYS_DMA_AG_MAP_QDMA |
3211                ETHSYS_DMA_AG_MAP_PPE;
3212     const struct mtk_reg_map *reg_map = eth->soc->reg_map;
3213     int i, val, ret;
3214
3215     if (test_and_set_bit(MTK_HW_INIT, &eth->state))
3216         return 0;
3217
3218     pm_runtime_enable(eth->dev);
3219     pm_runtime_get_sync(eth->dev);
3220
3221     ret = mtk_clk_enable(eth);
3222     if (ret)
3223         goto err_disable_pm;
3224
3225     if (eth->ethsys)
3226         regmap_update_bits(eth->ethsys, ETHSYS_DMA_AG_MAP, dma_mask,
3227                    of_dma_is_coherent(eth->dma_dev->of_node) * dma_mask);
3228
3229     if (MTK_HAS_CAPS(eth->soc->caps, MTK_SOC_MT7628)) {
3230         ret = device_reset(eth->dev);
3231         if (ret) {
3232             dev_err(eth->dev, "MAC reset failed!\n");
3233             goto err_disable_pm;
3234         }
3235
3236         /* set interrupt delays based on current Net DIM sample */
3237         mtk_dim_rx(&eth->rx_dim.work);
3238         mtk_dim_tx(&eth->tx_dim.work);
3239
3240         /* disable delay and normal interrupt */
3241         mtk_tx_irq_disable(eth, ~0);
3242         mtk_rx_irq_disable(eth, ~0);
3243
3244         return 0;
3245     }
3246
3247     val = RSTCTRL_FE | RSTCTRL_PPE;
3248     if (MTK_HAS_CAPS(eth->soc->caps, MTK_NETSYS_V2)) {
3249         regmap_write(eth->ethsys, ETHSYS_FE_RST_CHK_IDLE_EN, 0);
3250
3251         val |= RSTCTRL_ETH;
3252         if (MTK_HAS_CAPS(eth->soc->caps, MTK_RSTCTRL_PPE1))
3253             val |= RSTCTRL_PPE1;
3254     }
3255
3256     ethsys_reset(eth, val);
3257
3258     if (MTK_HAS_CAPS(eth->soc->caps, MTK_NETSYS_V2)) {
3259         regmap_write(eth->ethsys, ETHSYS_FE_RST_CHK_IDLE_EN,
3260                  0x3ffffff);
3261
3262         /* Set FE to PDMAv2 if necessary */
3263         val = mtk_r32(eth, MTK_FE_GLO_MISC);
3264         mtk_w32(eth,  val | BIT(4), MTK_FE_GLO_MISC);
3265     }
3266
3267     if (eth->pctl) {
3268         /* Set GE2 driving and slew rate */
3269         regmap_write(eth->pctl, GPIO_DRV_SEL10, 0xa00);
3270
3271         /* set GE2 TDSEL */
3272         regmap_write(eth->pctl, GPIO_OD33_CTRL8, 0x5);
3273
3274         /* set GE2 TUNE */
3275         regmap_write(eth->pctl, GPIO_BIAS_CTRL, 0x0);
3276     }
3277
3278     /* Set linkdown as the default for each GMAC. Its own MCR would be set
3279      * up with the more appropriate value when mtk_mac_config call is being
3280      * invoked.
3281      */
3282     for (i = 0; i < MTK_MAC_COUNT; i++)
3283         mtk_w32(eth, MAC_MCR_FORCE_LINK_DOWN, MTK_MAC_MCR(i));
3284
3285     /* Indicates CDM to parse the MTK special tag from CPU
3286      * which also is working out for untag packets.
3287      */
3288     val = mtk_r32(eth, MTK_CDMQ_IG_CTRL);
3289     mtk_w32(eth, val | MTK_CDMQ_STAG_EN, MTK_CDMQ_IG_CTRL);
3290
3291     /* Enable RX VLan Offloading */
3292     mtk_w32(eth, 1, MTK_CDMP_EG_CTRL);
3293
3294     /* set interrupt delays based on current Net DIM sample */
3295     mtk_dim_rx(&eth->rx_dim.work);
3296     mtk_dim_tx(&eth->tx_dim.work);
3297
3298     /* disable delay and normal interrupt */
3299     mtk_tx_irq_disable(eth, ~0);
3300     mtk_rx_irq_disable(eth, ~0);
3301
3302     /* FE int grouping */
3303     mtk_w32(eth, MTK_TX_DONE_INT, reg_map->pdma.int_grp);
3304     mtk_w32(eth, eth->soc->txrx.rx_irq_done_mask, reg_map->pdma.int_grp + 4);
3305     mtk_w32(eth, MTK_TX_DONE_INT, reg_map->qdma.int_grp);
3306     mtk_w32(eth, eth->soc->txrx.rx_irq_done_mask, reg_map->qdma.int_grp + 4);
3307     mtk_w32(eth, 0x21021000, MTK_FE_INT_GRP);
3308
3309     if (MTK_HAS_CAPS(eth->soc->caps, MTK_NETSYS_V2)) {
3310         /* PSE should not drop port8 and port9 packets */
3311         mtk_w32(eth, 0x00000300, PSE_DROP_CFG);
3312
3313         /* PSE Free Queue Flow Control  */
3314         mtk_w32(eth, 0x01fa01f4, PSE_FQFC_CFG2);
3315
3316         /* PSE config input queue threshold */
3317         mtk_w32(eth, 0x001a000e, PSE_IQ_REV(1));
3318         mtk_w32(eth, 0x01ff001a, PSE_IQ_REV(2));
3319         mtk_w32(eth, 0x000e01ff, PSE_IQ_REV(3));
3320         mtk_w32(eth, 0x000e000e, PSE_IQ_REV(4));
3321         mtk_w32(eth, 0x000e000e, PSE_IQ_REV(5));
3322         mtk_w32(eth, 0x000e000e, PSE_IQ_REV(6));
3323         mtk_w32(eth, 0x000e000e, PSE_IQ_REV(7));
3324         mtk_w32(eth, 0x000e000e, PSE_IQ_REV(8));
3325
3326         /* PSE config output queue threshold */
3327         mtk_w32(eth, 0x000f000a, PSE_OQ_TH(1));
3328         mtk_w32(eth, 0x001a000f, PSE_OQ_TH(2));
3329         mtk_w32(eth, 0x000f001a, PSE_OQ_TH(3));
3330         mtk_w32(eth, 0x01ff000f, PSE_OQ_TH(4));
3331         mtk_w32(eth, 0x000f000f, PSE_OQ_TH(5));
3332         mtk_w32(eth, 0x0006000f, PSE_OQ_TH(6));
3333         mtk_w32(eth, 0x00060006, PSE_OQ_TH(7));
3334         mtk_w32(eth, 0x00060006, PSE_OQ_TH(8));
3335
3336         /* GDM and CDM Threshold */
3337         mtk_w32(eth, 0x00000004, MTK_GDM2_THRES);
3338         mtk_w32(eth, 0x00000004, MTK_CDMW0_THRES);
3339         mtk_w32(eth, 0x00000004, MTK_CDMW1_THRES);
3340         mtk_w32(eth, 0x00000004, MTK_CDME0_THRES);
3341         mtk_w32(eth, 0x00000004, MTK_CDME1_THRES);
3342         mtk_w32(eth, 0x00000004, MTK_CDMM_THRES);
3343     }
3344
3345     return 0;
3346
3347 err_disable_pm:
3348     pm_runtime_put_sync(eth->dev);
3349     pm_runtime_disable(eth->dev);
3350
3351     return ret;
3352 }
3353
3354 static int mtk_hw_deinit(struct mtk_eth *eth)
3355 {
3356     if (!test_and_clear_bit(MTK_HW_INIT, &eth->state))
3357         return 0;
3358
3359     mtk_clk_disable(eth);
3360
3361     pm_runtime_put_sync(eth->dev);
3362     pm_runtime_disable(eth->dev);
3363
3364     return 0;
3365 }
3366
3367 static int __init mtk_init(struct net_device *dev)
3368 {
3369     struct mtk_mac *mac = netdev_priv(dev);
3370     struct mtk_eth *eth = mac->hw;
3371     int ret;
3372
3373     ret = of_get_ethdev_address(mac->of_node, dev);
3374     if (ret) {
3375         /* If the mac address is invalid, use random mac address */
3376         eth_hw_addr_random(dev);
3377         dev_err(eth->dev, "generated random MAC address %pM\n",
3378             dev->dev_addr);
3379     }
3380
3381     return 0;
3382 }
3383
3384 static void mtk_uninit(struct net_device *dev)
3385 {
3386     struct mtk_mac *mac = netdev_priv(dev);
3387     struct mtk_eth *eth = mac->hw;
3388
3389     phylink_disconnect_phy(mac->phylink);
3390     mtk_tx_irq_disable(eth, ~0);
3391     mtk_rx_irq_disable(eth, ~0);
3392 }
3393
3394 static int mtk_change_mtu(struct net_device *dev, int new_mtu)
3395 {
3396     int length = new_mtu + MTK_RX_ETH_HLEN;
3397     struct mtk_mac *mac = netdev_priv(dev);
3398     struct mtk_eth *eth = mac->hw;
3399     u32 mcr_cur, mcr_new;
3400
3401     if (rcu_access_pointer(eth->prog) &&
3402         length > MTK_PP_MAX_BUF_SIZE) {
3403         netdev_err(dev, "Invalid MTU for XDP mode\n");
3404         return -EINVAL;
3405     }
3406
3407     if (!MTK_HAS_CAPS(eth->soc->caps, MTK_SOC_MT7628)) {
3408         mcr_cur = mtk_r32(mac->hw, MTK_MAC_MCR(mac->id));
3409         mcr_new = mcr_cur & ~MAC_MCR_MAX_RX_MASK;
3410
3411         if (length <= 1518)
3412             mcr_new |= MAC_MCR_MAX_RX(MAC_MCR_MAX_RX_1518);
3413         else if (length <= 1536)
3414             mcr_new |= MAC_MCR_MAX_RX(MAC_MCR_MAX_RX_1536);
3415         else if (length <= 1552)
3416             mcr_new |= MAC_MCR_MAX_RX(MAC_MCR_MAX_RX_1552);
3417         else
3418             mcr_new |= MAC_MCR_MAX_RX(MAC_MCR_MAX_RX_2048);
3419
3420         if (mcr_new != mcr_cur)
3421             mtk_w32(mac->hw, mcr_new, MTK_MAC_MCR(mac->id));
3422     }
3423
3424     dev->mtu = new_mtu;
3425
3426     return 0;
3427 }
3428
3429 static int mtk_do_ioctl(struct net_device *dev, struct ifreq *ifr, int cmd)
3430 {
3431     struct mtk_mac *mac = netdev_priv(dev);
3432
3433     switch (cmd) {
3434     case SIOCGMIIPHY:
3435     case SIOCGMIIREG:
3436     case SIOCSMIIREG:
3437         return phylink_mii_ioctl(mac->phylink, ifr, cmd);
3438     default:
3439         break;
3440     }
3441
3442     return -EOPNOTSUPP;
3443 }
3444
3445 static void mtk_pending_work(struct work_struct *work)
3446 {
3447     struct mtk_eth *eth = container_of(work, struct mtk_eth, pending_work);
3448     int err, i;
3449     unsigned long restart = 0;
3450
3451     rtnl_lock();
3452
3453     dev_dbg(eth->dev, "[%s][%d] reset\n", __func__, __LINE__);
3454
3455     while (test_and_set_bit_lock(MTK_RESETTING, &eth->state))
3456         cpu_relax();
3457
3458     dev_dbg(eth->dev, "[%s][%d] mtk_stop starts\n", __func__, __LINE__);
3459     /* stop all devices to make sure that dma is properly shut down */
3460     for (i = 0; i < MTK_MAC_COUNT; i++) {
3461         if (!eth->netdev[i])
3462             continue;
3463         mtk_stop(eth->netdev[i]);
3464         __set_bit(i, &restart);
3465     }
3466     dev_dbg(eth->dev, "[%s][%d] mtk_stop ends\n", __func__, __LINE__);
3467
3468     /* restart underlying hardware such as power, clock, pin mux
3469      * and the connected phy
3470      */
3471     mtk_hw_deinit(eth);
3472
3473     if (eth->dev->pins)
3474         pinctrl_select_state(eth->dev->pins->p,
3475                      eth->dev->pins->default_state);
3476     mtk_hw_init(eth);
3477
3478     /* restart DMA and enable IRQs */
3479     for (i = 0; i < MTK_MAC_COUNT; i++) {
3480         if (!test_bit(i, &restart))
3481             continue;
3482         err = mtk_open(eth->netdev[i]);
3483         if (err) {
3484             netif_alert(eth, ifup, eth->netdev[i],
3485                   "Driver up/down cycle failed, closing device.\n");
3486             dev_close(eth->netdev[i]);
3487         }
3488     }
3489
3490     dev_dbg(eth->dev, "[%s][%d] reset done\n", __func__, __LINE__);
3491
3492     clear_bit_unlock(MTK_RESETTING, &eth->state);
3493
3494     rtnl_unlock();
3495 }
3496
3497 static int mtk_free_dev(struct mtk_eth *eth)
3498 {
3499     int i;
3500
3501     for (i = 0; i < MTK_MAC_COUNT; i++) {
3502         if (!eth->netdev[i])
3503             continue;
3504         free_netdev(eth->netdev[i]);
3505     }
3506
3507     return 0;
3508 }
3509
3510 static int mtk_unreg_dev(struct mtk_eth *eth)
3511 {
3512     int i;
3513
3514     for (i = 0; i < MTK_MAC_COUNT; i++) {
3515         if (!eth->netdev[i])
3516             continue;
3517         unregister_netdev(eth->netdev[i]);
3518     }
3519
3520     return 0;
3521 }
3522
3523 static int mtk_cleanup(struct mtk_eth *eth)
3524 {
3525     mtk_unreg_dev(eth);
3526     mtk_free_dev(eth);
3527     cancel_work_sync(&eth->pending_work);
3528
3529     return 0;
3530 }
3531
3532 static int mtk_get_link_ksettings(struct net_device *ndev,
3533                   struct ethtool_link_ksettings *cmd)
3534 {
3535     struct mtk_mac *mac = netdev_priv(ndev);
3536
3537     if (unlikely(test_bit(MTK_RESETTING, &mac->hw->state)))
3538         return -EBUSY;
3539
3540     return phylink_ethtool_ksettings_get(mac->phylink, cmd);
3541 }
3542
3543 static int mtk_set_link_ksettings(struct net_device *ndev,
3544                   const struct ethtool_link_ksettings *cmd)
3545 {
3546     struct mtk_mac *mac = netdev_priv(ndev);
3547
3548     if (unlikely(test_bit(MTK_RESETTING, &mac->hw->state)))
3549         return -EBUSY;
3550
3551     return phylink_ethtool_ksettings_set(mac->phylink, cmd);
3552 }
3553
3554 static void mtk_get_drvinfo(struct net_device *dev,
3555                 struct ethtool_drvinfo *info)
3556 {
3557     struct mtk_mac *mac = netdev_priv(dev);
3558
3559     strlcpy(info->driver, mac->hw->dev->driver->name, sizeof(info->driver));
3560     strlcpy(info->bus_info, dev_name(mac->hw->dev), sizeof(info->bus_info));
3561     info->n_stats = ARRAY_SIZE(mtk_ethtool_stats);
3562 }
3563
3564 static u32 mtk_get_msglevel(struct net_device *dev)
3565 {
3566     struct mtk_mac *mac = netdev_priv(dev);
3567
3568     return mac->hw->msg_enable;
3569 }
3570
3571 static void mtk_set_msglevel(struct net_device *dev, u32 value)
3572 {
3573     struct mtk_mac *mac = netdev_priv(dev);
3574
3575     mac->hw->msg_enable = value;
3576 }
3577
3578 static int mtk_nway_reset(struct net_device *dev)
3579 {
3580     struct mtk_mac *mac = netdev_priv(dev);
3581
3582     if (unlikely(test_bit(MTK_RESETTING, &mac->hw->state)))
3583         return -EBUSY;
3584
3585     if (!mac->phylink)
3586         return -ENOTSUPP;
3587
3588     return phylink_ethtool_nway_reset(mac->phylink);
3589 }
3590
3591 static void mtk_get_strings(struct net_device *dev, u32 stringset, u8 *data)
3592 {
3593     int i;
3594
3595     switch (stringset) {
3596     case ETH_SS_STATS: {
3597         struct mtk_mac *mac = netdev_priv(dev);
3598
3599         for (i = 0; i < ARRAY_SIZE(mtk_ethtool_stats); i++) {
3600             memcpy(data, mtk_ethtool_stats[i].str, ETH_GSTRING_LEN);
3601             data += ETH_GSTRING_LEN;
3602         }
3603         if (mtk_page_pool_enabled(mac->hw))
3604             page_pool_ethtool_stats_get_strings(data);
3605         break;
3606     }
3607     default:
3608         break;
3609     }
3610 }
3611
3612 static int mtk_get_sset_count(struct net_device *dev, int sset)
3613 {
3614     switch (sset) {
3615     case ETH_SS_STATS: {
3616         int count = ARRAY_SIZE(mtk_ethtool_stats);
3617         struct mtk_mac *mac = netdev_priv(dev);
3618
3619         if (mtk_page_pool_enabled(mac->hw))
3620             count += page_pool_ethtool_stats_get_count();
3621         return count;
3622     }
3623     default:
3624         return -EOPNOTSUPP;
3625     }
3626 }
3627
3628 static void mtk_ethtool_pp_stats(struct mtk_eth *eth, u64 *data)
3629 {
3630     struct page_pool_stats stats = {};
3631     int i;
3632
3633     for (i = 0; i < ARRAY_SIZE(eth->rx_ring); i++) {
3634         struct mtk_rx_ring *ring = &eth->rx_ring[i];
3635
3636         if (!ring->page_pool)
3637             continue;
3638
3639         page_pool_get_stats(ring->page_pool, &stats);
3640     }
3641     page_pool_ethtool_stats_get(data, &stats);
3642 }
3643
3644 static void mtk_get_ethtool_stats(struct net_device *dev,
3645                   struct ethtool_stats *stats, u64 *data)
3646 {
3647     struct mtk_mac *mac = netdev_priv(dev);
3648     struct mtk_hw_stats *hwstats = mac->hw_stats;
3649     u64 *data_src, *data_dst;
3650     unsigned int start;
3651     int i;
3652
3653     if (unlikely(test_bit(MTK_RESETTING, &mac->hw->state)))
3654         return;
3655
3656     if (netif_running(dev) && netif_device_present(dev)) {
3657         if (spin_trylock_bh(&hwstats->stats_lock)) {
3658             mtk_stats_update_mac(mac);
3659             spin_unlock_bh(&hwstats->stats_lock);
3660         }
3661     }
3662
3663     data_src = (u64 *)hwstats;
3664
3665     do {
3666         data_dst = data;
3667         start = u64_stats_fetch_begin_irq(&hwstats->syncp);
3668
3669         for (i = 0; i < ARRAY_SIZE(mtk_ethtool_stats); i++)
3670             *data_dst++ = *(data_src + mtk_ethtool_stats[i].offset);
3671         if (mtk_page_pool_enabled(mac->hw))
3672             mtk_ethtool_pp_stats(mac->hw, data_dst);
3673     } while (u64_stats_fetch_retry_irq(&hwstats->syncp, start));
3674 }
3675
3676 static int mtk_get_rxnfc(struct net_device *dev, struct ethtool_rxnfc *cmd,
3677              u32 *rule_locs)
3678 {
3679     int ret = -EOPNOTSUPP;
3680
3681     switch (cmd->cmd) {
3682     case ETHTOOL_GRXRINGS:
3683         if (dev->hw_features & NETIF_F_LRO) {
3684             cmd->data = MTK_MAX_RX_RING_NUM;
3685             ret = 0;
3686         }
3687         break;
3688     case ETHTOOL_GRXCLSRLCNT:
3689         if (dev->hw_features & NETIF_F_LRO) {
3690             struct mtk_mac *mac = netdev_priv(dev);
3691
3692             cmd->rule_cnt = mac->hwlro_ip_cnt;
3693             ret = 0;
3694         }
3695         break;
3696     case ETHTOOL_GRXCLSRULE:
3697         if (dev->hw_features & NETIF_F_LRO)
3698             ret = mtk_hwlro_get_fdir_entry(dev, cmd);
3699         break;
3700     case ETHTOOL_GRXCLSRLALL:
3701         if (dev->hw_features & NETIF_F_LRO)
3702             ret = mtk_hwlro_get_fdir_all(dev, cmd,
3703                              rule_locs);
3704         break;
3705     default:
3706         break;
3707     }
3708
3709     return ret;
3710 }
3711
3712 static int mtk_set_rxnfc(struct net_device *dev, struct ethtool_rxnfc *cmd)
3713 {
3714     int ret = -EOPNOTSUPP;
3715
3716     switch (cmd->cmd) {
3717     case ETHTOOL_SRXCLSRLINS:
3718         if (dev->hw_features & NETIF_F_LRO)
3719             ret = mtk_hwlro_add_ipaddr(dev, cmd);
3720         break;
3721     case ETHTOOL_SRXCLSRLDEL:
3722         if (dev->hw_features & NETIF_F_LRO)
3723             ret = mtk_hwlro_del_ipaddr(dev, cmd);
3724         break;
3725     default:
3726         break;
3727     }
3728
3729     return ret;
3730 }
3731
3732 static const struct ethtool_ops mtk_ethtool_ops = {
3733     .get_link_ksettings = mtk_get_link_ksettings,
3734     .set_link_ksettings = mtk_set_link_ksettings,
3735     .get_drvinfo        = mtk_get_drvinfo,
3736     .get_msglevel       = mtk_get_msglevel,
3737     .set_msglevel       = mtk_set_msglevel,
3738     .nway_reset     = mtk_nway_reset,
3739     .get_link       = ethtool_op_get_link,
3740     .get_strings        = mtk_get_strings,
3741     .get_sset_count     = mtk_get_sset_count,
3742     .get_ethtool_stats  = mtk_get_ethtool_stats,
3743     .get_rxnfc      = mtk_get_rxnfc,
3744     .set_rxnfc              = mtk_set_rxnfc,
3745 };
3746
3747 static const struct net_device_ops mtk_netdev_ops = {
3748     .ndo_init       = mtk_init,
3749     .ndo_uninit     = mtk_uninit,
3750     .ndo_open       = mtk_open,
3751     .ndo_stop       = mtk_stop,
3752     .ndo_start_xmit     = mtk_start_xmit,
3753     .ndo_set_mac_address    = mtk_set_mac_address,
3754     .ndo_validate_addr  = eth_validate_addr,
3755     .ndo_eth_ioctl      = mtk_do_ioctl,
3756     .ndo_change_mtu     = mtk_change_mtu,
3757     .ndo_tx_timeout     = mtk_tx_timeout,
3758     .ndo_get_stats64        = mtk_get_stats64,
3759     .ndo_fix_features   = mtk_fix_features,
3760     .ndo_set_features   = mtk_set_features,
3761 #ifdef CONFIG_NET_POLL_CONTROLLER
3762     .ndo_poll_controller    = mtk_poll_controller,
3763 #endif
3764     .ndo_setup_tc       = mtk_eth_setup_tc,
3765     .ndo_bpf        = mtk_xdp,
3766     .ndo_xdp_xmit       = mtk_xdp_xmit,
3767 };
3768
3769 static int mtk_add_mac(struct mtk_eth *eth, struct device_node *np)
3770 {
3771     const __be32 *_id = of_get_property(np, "reg", NULL);
3772     phy_interface_t phy_mode;
3773     struct phylink *phylink;
3774     struct mtk_mac *mac;
3775     int id, err;
3776
3777     if (!_id) {
3778         dev_err(eth->dev, "missing mac id\n");
3779         return -EINVAL;
3780     }
3781
3782     id = be32_to_cpup(_id);
3783     if (id >= MTK_MAC_COUNT) {
3784         dev_err(eth->dev, "%d is not a valid mac id\n", id);
3785         return -EINVAL;
3786     }
3787
3788     if (eth->netdev[id]) {
3789         dev_err(eth->dev, "duplicate mac id found: %d\n", id);
3790         return -EINVAL;
3791     }
3792
3793     eth->netdev[id] = alloc_etherdev(sizeof(*mac));
3794     if (!eth->netdev[id]) {
3795         dev_err(eth->dev, "alloc_etherdev failed\n");
3796         return -ENOMEM;
3797     }
3798     mac = netdev_priv(eth->netdev[id]);
3799     eth->mac[id] = mac;
3800     mac->id = id;
3801     mac->hw = eth;
3802     mac->of_node = np;
3803
3804     memset(mac->hwlro_ip, 0, sizeof(mac->hwlro_ip));
3805     mac->hwlro_ip_cnt = 0;
3806
3807     mac->hw_stats = devm_kzalloc(eth->dev,
3808                      sizeof(*mac->hw_stats),
3809                      GFP_KERNEL);
3810     if (!mac->hw_stats) {
3811         dev_err(eth->dev, "failed to allocate counter memory\n");
3812         err = -ENOMEM;
3813         goto free_netdev;
3814     }
3815     spin_lock_init(&mac->hw_stats->stats_lock);
3816     u64_stats_init(&mac->hw_stats->syncp);
3817     mac->hw_stats->reg_offset = id * MTK_STAT_OFFSET;
3818
3819     /* phylink create */
3820     err = of_get_phy_mode(np, &phy_mode);
3821     if (err) {
3822         dev_err(eth->dev, "incorrect phy-mode\n");
3823         goto free_netdev;
3824     }
3825
3826     /* mac config is not set */
3827     mac->interface = PHY_INTERFACE_MODE_NA;
3828     mac->speed = SPEED_UNKNOWN;
3829
3830     mac->phylink_config.dev = &eth->netdev[id]->dev;
3831     mac->phylink_config.type = PHYLINK_NETDEV;
3832     /* This driver makes use of state->speed in mac_config */
3833     mac->phylink_config.legacy_pre_march2020 = true;
3834     mac->phylink_config.mac_capabilities = MAC_ASYM_PAUSE | MAC_SYM_PAUSE |
3835         MAC_10 | MAC_100 | MAC_1000 | MAC_2500FD;
3836
3837     __set_bit(PHY_INTERFACE_MODE_MII,
3838           mac->phylink_config.supported_interfaces);
3839     __set_bit(PHY_INTERFACE_MODE_GMII,
3840           mac->phylink_config.supported_interfaces);
3841
3842     if (MTK_HAS_CAPS(mac->hw->soc->caps, MTK_RGMII))
3843         phy_interface_set_rgmii(mac->phylink_config.supported_interfaces);
3844
3845     if (MTK_HAS_CAPS(mac->hw->soc->caps, MTK_TRGMII) && !mac->id)
3846         __set_bit(PHY_INTERFACE_MODE_TRGMII,
3847               mac->phylink_config.supported_interfaces);
3848
3849     if (MTK_HAS_CAPS(mac->hw->soc->caps, MTK_SGMII)) {
3850         __set_bit(PHY_INTERFACE_MODE_SGMII,
3851               mac->phylink_config.supported_interfaces);
3852         __set_bit(PHY_INTERFACE_MODE_1000BASEX,
3853               mac->phylink_config.supported_interfaces);
3854         __set_bit(PHY_INTERFACE_MODE_2500BASEX,
3855               mac->phylink_config.supported_interfaces);
3856     }
3857
3858     phylink = phylink_create(&mac->phylink_config,
3859                  of_fwnode_handle(mac->of_node),
3860                  phy_mode, &mtk_phylink_ops);
3861     if (IS_ERR(phylink)) {
3862         err = PTR_ERR(phylink);
3863         goto free_netdev;
3864     }
3865
3866     mac->phylink = phylink;
3867
3868     SET_NETDEV_DEV(eth->netdev[id], eth->dev);
3869     eth->netdev[id]->watchdog_timeo = 5 * HZ;
3870     eth->netdev[id]->netdev_ops = &mtk_netdev_ops;
3871     eth->netdev[id]->base_addr = (unsigned long)eth->base;
3872
3873     eth->netdev[id]->hw_features = eth->soc->hw_features;
3874     if (eth->hwlro)
3875         eth->netdev[id]->hw_features |= NETIF_F_LRO;
3876
3877     eth->netdev[id]->vlan_features = eth->soc->hw_features &
3878         ~(NETIF_F_HW_VLAN_CTAG_TX | NETIF_F_HW_VLAN_CTAG_RX);
3879     eth->netdev[id]->features |= eth->soc->hw_features;
3880     eth->netdev[id]->ethtool_ops = &mtk_ethtool_ops;
3881
3882     eth->netdev[id]->irq = eth->irq[0];
3883     eth->netdev[id]->dev.of_node = np;
3884
3885     if (MTK_HAS_CAPS(eth->soc->caps, MTK_SOC_MT7628))
3886         eth->netdev[id]->max_mtu = MTK_MAX_RX_LENGTH - MTK_RX_ETH_HLEN;
3887     else
3888         eth->netdev[id]->max_mtu = MTK_MAX_RX_LENGTH_2K - MTK_RX_ETH_HLEN;
3889
3890     return 0;
3891
3892 free_netdev:
3893     free_netdev(eth->netdev[id]);
3894     return err;
3895 }
3896
3897 void mtk_eth_set_dma_device(struct mtk_eth *eth, struct device *dma_dev)
3898 {
3899     struct net_device *dev, *tmp;
3900     LIST_HEAD(dev_list);
3901     int i;
3902
3903     rtnl_lock();
3904
3905     for (i = 0; i < MTK_MAC_COUNT; i++) {
3906         dev = eth->netdev[i];
3907
3908         if (!dev || !(dev->flags & IFF_UP))
3909             continue;
3910
3911         list_add_tail(&dev->close_list, &dev_list);
3912     }
3913
3914     dev_close_many(&dev_list, false);
3915
3916     eth->dma_dev = dma_dev;
3917
3918     list_for_each_entry_safe(dev, tmp, &dev_list, close_list) {
3919         list_del_init(&dev->close_list);
3920         dev_open(dev, NULL);
3921     }
3922
3923     rtnl_unlock();
3924 }
3925
3926 static int mtk_probe(struct platform_device *pdev)
3927 {
3928     struct device_node *mac_np;
3929     struct mtk_eth *eth;
3930     int err, i;
3931
3932     eth = devm_kzalloc(&pdev->dev, sizeof(*eth), GFP_KERNEL);
3933     if (!eth)
3934         return -ENOMEM;
3935
3936     eth->soc = of_device_get_match_data(&pdev->dev);
3937
3938     eth->dev = &pdev->dev;
3939     eth->dma_dev = &pdev->dev;
3940     eth->base = devm_platform_ioremap_resource(pdev, 0);
3941     if (IS_ERR(eth->base))
3942         return PTR_ERR(eth->base);
3943
3944     if (MTK_HAS_CAPS(eth->soc->caps, MTK_SOC_MT7628))
3945         eth->ip_align = NET_IP_ALIGN;
3946
3947     spin_lock_init(&eth->page_lock);
3948     spin_lock_init(&eth->tx_irq_lock);
3949     spin_lock_init(&eth->rx_irq_lock);
3950     spin_lock_init(&eth->dim_lock);
3951
3952     eth->rx_dim.mode = DIM_CQ_PERIOD_MODE_START_FROM_EQE;
3953     INIT_WORK(&eth->rx_dim.work, mtk_dim_rx);
3954
3955     eth->tx_dim.mode = DIM_CQ_PERIOD_MODE_START_FROM_EQE;
3956     INIT_WORK(&eth->tx_dim.work, mtk_dim_tx);
3957
3958     if (!MTK_HAS_CAPS(eth->soc->caps, MTK_SOC_MT7628)) {
3959         eth->ethsys = syscon_regmap_lookup_by_phandle(pdev->dev.of_node,
3960                                   "mediatek,ethsys");
3961         if (IS_ERR(eth->ethsys)) {
3962             dev_err(&pdev->dev, "no ethsys regmap found\n");
3963             return PTR_ERR(eth->ethsys);
3964         }
3965     }
3966
3967     if (MTK_HAS_CAPS(eth->soc->caps, MTK_INFRA)) {
3968         eth->infra = syscon_regmap_lookup_by_phandle(pdev->dev.of_node,
3969                                  "mediatek,infracfg");
3970         if (IS_ERR(eth->infra)) {
3971             dev_err(&pdev->dev, "no infracfg regmap found\n");
3972             return PTR_ERR(eth->infra);
3973         }
3974     }
3975
3976     if (of_dma_is_coherent(pdev->dev.of_node)) {
3977         struct regmap *cci;
3978
3979         cci = syscon_regmap_lookup_by_phandle(pdev->dev.of_node,
3980                               "cci-control-port");
3981         /* enable CPU/bus coherency */
3982         if (!IS_ERR(cci))
3983             regmap_write(cci, 0, 3);
3984     }
3985
3986     if (MTK_HAS_CAPS(eth->soc->caps, MTK_SGMII)) {
3987         eth->sgmii = devm_kzalloc(eth->dev, sizeof(*eth->sgmii),
3988                       GFP_KERNEL);
3989         if (!eth->sgmii)
3990             return -ENOMEM;
3991
3992         err = mtk_sgmii_init(eth->sgmii, pdev->dev.of_node,
3993                      eth->soc->ana_rgc3);
3994
3995         if (err)
3996             return err;
3997     }
3998
3999     if (eth->soc->required_pctl) {
4000         eth->pctl = syscon_regmap_lookup_by_phandle(pdev->dev.of_node,
4001                                 "mediatek,pctl");
4002         if (IS_ERR(eth->pctl)) {
4003             dev_err(&pdev->dev, "no pctl regmap found\n");
4004             return PTR_ERR(eth->pctl);
4005         }
4006     }
4007
4008     for (i = 0;; i++) {
4009         struct device_node *np = of_parse_phandle(pdev->dev.of_node,
4010                               "mediatek,wed", i);
4011         static const u32 wdma_regs[] = {
4012             MTK_WDMA0_BASE,
4013             MTK_WDMA1_BASE
4014         };
4015         void __iomem *wdma;
4016
4017         if (!np || i >= ARRAY_SIZE(wdma_regs))
4018             break;
4019
4020         wdma = eth->base + wdma_regs[i];
4021         mtk_wed_add_hw(np, eth, wdma, i);
4022     }
4023
4024     for (i = 0; i < 3; i++) {
4025         if (MTK_HAS_CAPS(eth->soc->caps, MTK_SHARED_INT) && i > 0)
4026             eth->irq[i] = eth->irq[0];
4027         else
4028             eth->irq[i] = platform_get_irq(pdev, i);
4029         if (eth->irq[i] < 0) {
4030             dev_err(&pdev->dev, "no IRQ%d resource found\n", i);
4031             return -ENXIO;
4032         }
4033     }
4034     for (i = 0; i < ARRAY_SIZE(eth->clks); i++) {
4035         eth->clks[i] = devm_clk_get(eth->dev,
4036                         mtk_clks_source_name[i]);
4037         if (IS_ERR(eth->clks[i])) {
4038             if (PTR_ERR(eth->clks[i]) == -EPROBE_DEFER)
4039                 return -EPROBE_DEFER;
4040             if (eth->soc->required_clks & BIT(i)) {
4041                 dev_err(&pdev->dev, "clock %s not found\n",
4042                     mtk_clks_source_name[i]);
4043                 return -EINVAL;
4044             }
4045             eth->clks[i] = NULL;
4046         }
4047     }
4048
4049     eth->msg_enable = netif_msg_init(mtk_msg_level, MTK_DEFAULT_MSG_ENABLE);
4050     INIT_WORK(&eth->pending_work, mtk_pending_work);
4051
4052     err = mtk_hw_init(eth);
4053     if (err)
4054         return err;
4055
4056     eth->hwlro = MTK_HAS_CAPS(eth->soc->caps, MTK_HWLRO);
4057
4058     for_each_child_of_node(pdev->dev.of_node, mac_np) {
4059         if (!of_device_is_compatible(mac_np,
4060                          "mediatek,eth-mac"))
4061             continue;
4062
4063         if (!of_device_is_available(mac_np))
4064             continue;
4065
4066         err = mtk_add_mac(eth, mac_np);
4067         if (err) {
4068             of_node_put(mac_np);
4069             goto err_deinit_hw;
4070         }
4071     }
4072
4073     if (MTK_HAS_CAPS(eth->soc->caps, MTK_SHARED_INT)) {
4074         err = devm_request_irq(eth->dev, eth->irq[0],
4075                        mtk_handle_irq, 0,
4076                        dev_name(eth->dev), eth);
4077     } else {
4078         err = devm_request_irq(eth->dev, eth->irq[1],
4079                        mtk_handle_irq_tx, 0,
4080                        dev_name(eth->dev), eth);
4081         if (err)
4082             goto err_free_dev;
4083
4084         err = devm_request_irq(eth->dev, eth->irq[2],
4085                        mtk_handle_irq_rx, 0,
4086                        dev_name(eth->dev), eth);
4087     }
4088     if (err)
4089         goto err_free_dev;
4090
4091     /* No MT7628/88 support yet */
4092     if (!MTK_HAS_CAPS(eth->soc->caps, MTK_SOC_MT7628)) {
4093         err = mtk_mdio_init(eth);
4094         if (err)
4095             goto err_free_dev;
4096     }
4097
4098     if (eth->soc->offload_version) {
4099         eth->ppe = mtk_ppe_init(eth, eth->base + MTK_ETH_PPE_BASE, 2);
4100         if (!eth->ppe) {
4101             err = -ENOMEM;
4102             goto err_free_dev;
4103         }
4104
4105         err = mtk_eth_offload_init(eth);
4106         if (err)
4107             goto err_free_dev;
4108     }
4109
4110     for (i = 0; i < MTK_MAX_DEVS; i++) {
4111         if (!eth->netdev[i])
4112             continue;
4113
4114         err = register_netdev(eth->netdev[i]);
4115         if (err) {
4116             dev_err(eth->dev, "error bringing up device\n");
4117             goto err_deinit_mdio;
4118         } else
4119             netif_info(eth, probe, eth->netdev[i],
4120                    "mediatek frame engine at 0x%08lx, irq %d\n",
4121                    eth->netdev[i]->base_addr, eth->irq[0]);
4122     }
4123
4124     /* we run 2 devices on the same DMA ring so we need a dummy device
4125      * for NAPI to work
4126      */
4127     init_dummy_netdev(&eth->dummy_dev);
4128     netif_napi_add(&eth->dummy_dev, &eth->tx_napi, mtk_napi_tx,
4129                NAPI_POLL_WEIGHT);
4130     netif_napi_add(&eth->dummy_dev, &eth->rx_napi, mtk_napi_rx,
4131                NAPI_POLL_WEIGHT);
4132
4133     platform_set_drvdata(pdev, eth);
4134
4135     return 0;
4136
4137 err_deinit_mdio:
4138     mtk_mdio_cleanup(eth);
4139 err_free_dev:
4140     mtk_free_dev(eth);
4141 err_deinit_hw:
4142     mtk_hw_deinit(eth);
4143
4144     return err;
4145 }
4146
4147 static int mtk_remove(struct platform_device *pdev)
4148 {
4149     struct mtk_eth *eth = platform_get_drvdata(pdev);
4150     struct mtk_mac *mac;
4151     int i;
4152
4153     /* stop all devices to make sure that dma is properly shut down */
4154     for (i = 0; i < MTK_MAC_COUNT; i++) {
4155         if (!eth->netdev[i])
4156             continue;
4157         mtk_stop(eth->netdev[i]);
4158         mac = netdev_priv(eth->netdev[i]);
4159         phylink_disconnect_phy(mac->phylink);
4160     }
4161
4162     mtk_hw_deinit(eth);
4163
4164     netif_napi_del(&eth->tx_napi);
4165     netif_napi_del(&eth->rx_napi);
4166     mtk_cleanup(eth);
4167     mtk_mdio_cleanup(eth);
4168
4169     return 0;
4170 }
4171
4172 static const struct mtk_soc_data mt2701_data = {
4173     .reg_map = &mtk_reg_map,
4174     .caps = MT7623_CAPS | MTK_HWLRO,
4175     .hw_features = MTK_HW_FEATURES,
4176     .required_clks = MT7623_CLKS_BITMAP,
4177     .required_pctl = true,
4178     .txrx = {
4179         .txd_size = sizeof(struct mtk_tx_dma),
4180         .rxd_size = sizeof(struct mtk_rx_dma),
4181         .rx_irq_done_mask = MTK_RX_DONE_INT,
4182         .rx_dma_l4_valid = RX_DMA_L4_VALID,
4183         .dma_max_len = MTK_TX_DMA_BUF_LEN,
4184         .dma_len_offset = 16,
4185     },
4186 };
4187
4188 static const struct mtk_soc_data mt7621_data = {
4189     .reg_map = &mtk_reg_map,
4190     .caps = MT7621_CAPS,
4191     .hw_features = MTK_HW_FEATURES,
4192     .required_clks = MT7621_CLKS_BITMAP,
4193     .required_pctl = false,
4194     .offload_version = 2,
4195     .txrx = {
4196         .txd_size = sizeof(struct mtk_tx_dma),
4197         .rxd_size = sizeof(struct mtk_rx_dma),
4198         .rx_irq_done_mask = MTK_RX_DONE_INT,
4199         .rx_dma_l4_valid = RX_DMA_L4_VALID,
4200         .dma_max_len = MTK_TX_DMA_BUF_LEN,
4201         .dma_len_offset = 16,
4202     },
4203 };
4204
4205 static const struct mtk_soc_data mt7622_data = {
4206     .reg_map = &mtk_reg_map,
4207     .ana_rgc3 = 0x2028,
4208     .caps = MT7622_CAPS | MTK_HWLRO,
4209     .hw_features = MTK_HW_FEATURES,
4210     .required_clks = MT7622_CLKS_BITMAP,
4211     .required_pctl = false,
4212     .offload_version = 2,
4213     .txrx = {
4214         .txd_size = sizeof(struct mtk_tx_dma),
4215         .rxd_size = sizeof(struct mtk_rx_dma),
4216         .rx_irq_done_mask = MTK_RX_DONE_INT,
4217         .rx_dma_l4_valid = RX_DMA_L4_VALID,
4218         .dma_max_len = MTK_TX_DMA_BUF_LEN,
4219         .dma_len_offset = 16,
4220     },
4221 };
4222
4223 static const struct mtk_soc_data mt7623_data = {
4224     .reg_map = &mtk_reg_map,
4225     .caps = MT7623_CAPS | MTK_HWLRO,
4226     .hw_features = MTK_HW_FEATURES,
4227     .required_clks = MT7623_CLKS_BITMAP,
4228     .required_pctl = true,
4229     .offload_version = 2,
4230     .txrx = {
4231         .txd_size = sizeof(struct mtk_tx_dma),
4232         .rxd_size = sizeof(struct mtk_rx_dma),
4233         .rx_irq_done_mask = MTK_RX_DONE_INT,
4234         .rx_dma_l4_valid = RX_DMA_L4_VALID,
4235         .dma_max_len = MTK_TX_DMA_BUF_LEN,
4236         .dma_len_offset = 16,
4237     },
4238 };
4239
4240 static const struct mtk_soc_data mt7629_data = {
4241     .reg_map = &mtk_reg_map,
4242     .ana_rgc3 = 0x128,
4243     .caps = MT7629_CAPS | MTK_HWLRO,
4244     .hw_features = MTK_HW_FEATURES,
4245     .required_clks = MT7629_CLKS_BITMAP,
4246     .required_pctl = false,
4247     .txrx = {
4248         .txd_size = sizeof(struct mtk_tx_dma),
4249         .rxd_size = sizeof(struct mtk_rx_dma),
4250         .rx_irq_done_mask = MTK_RX_DONE_INT,
4251         .rx_dma_l4_valid = RX_DMA_L4_VALID,
4252         .dma_max_len = MTK_TX_DMA_BUF_LEN,
4253         .dma_len_offset = 16,
4254     },
4255 };
4256
4257 static const struct mtk_soc_data mt7986_data = {
4258     .reg_map = &mt7986_reg_map,
4259     .ana_rgc3 = 0x128,
4260     .caps = MT7986_CAPS,
4261     .required_clks = MT7986_CLKS_BITMAP,
4262     .required_pctl = false,
4263     .txrx = {
4264         .txd_size = sizeof(struct mtk_tx_dma_v2),
4265         .rxd_size = sizeof(struct mtk_rx_dma_v2),
4266         .rx_irq_done_mask = MTK_RX_DONE_INT_V2,
4267         .rx_dma_l4_valid = RX_DMA_L4_VALID_V2,
4268         .dma_max_len = MTK_TX_DMA_BUF_LEN_V2,
4269         .dma_len_offset = 8,
4270     },
4271 };
4272
4273 static const struct mtk_soc_data rt5350_data = {
4274     .reg_map = &mt7628_reg_map,
4275     .caps = MT7628_CAPS,
4276     .hw_features = MTK_HW_FEATURES_MT7628,
4277     .required_clks = MT7628_CLKS_BITMAP,
4278     .required_pctl = false,
4279     .txrx = {
4280         .txd_size = sizeof(struct mtk_tx_dma),
4281         .rxd_size = sizeof(struct mtk_rx_dma),
4282         .rx_irq_done_mask = MTK_RX_DONE_INT,
4283         .rx_dma_l4_valid = RX_DMA_L4_VALID_PDMA,
4284         .dma_max_len = MTK_TX_DMA_BUF_LEN,
4285         .dma_len_offset = 16,
4286     },
4287 };
4288
4289 const struct of_device_id of_mtk_match[] = {
4290     { .compatible = "mediatek,mt2701-eth", .data = &mt2701_data},
4291     { .compatible = "mediatek,mt7621-eth", .data = &mt7621_data},
4292     { .compatible = "mediatek,mt7622-eth", .data = &mt7622_data},
4293     { .compatible = "mediatek,mt7623-eth", .data = &mt7623_data},
4294     { .compatible = "mediatek,mt7629-eth", .data = &mt7629_data},
4295     { .compatible = "mediatek,mt7986-eth", .data = &mt7986_data},
4296     { .compatible = "ralink,rt5350-eth", .data = &rt5350_data},
4297     {},
4298 };
4299 MODULE_DEVICE_TABLE(of, of_mtk_match);
4300
4301 static struct platform_driver mtk_driver = {
4302     .probe = mtk_probe,
4303     .remove = mtk_remove,
4304     .driver = {
4305         .name = "mtk_soc_eth",
4306         .of_match_table = of_mtk_match,
4307     },
4308 };
4309
4310 module_platform_driver(mtk_driver);
4311
4312 MODULE_LICENSE("GPL");
4313 MODULE_AUTHOR("John Crispin <blogic@openwrt.org>");
4314 MODULE_DESCRIPTION("Ethernet driver for MediaTek SoC");