Back to home page

OSCL-LXR
 
 

     

0001 // SPDX-License-Identifier: GPL-2.0
0002 /* niu.c: Neptune ethernet driver.
0003  *
0004  * Copyright (C) 2007, 2008 David S. Miller (davem@davemloft.net)
0005  */
0006 
0007 #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
0008 
0009 #include <linux/module.h>
0010 #include <linux/init.h>
0011 #include <linux/interrupt.h>
0012 #include <linux/pci.h>
0013 #include <linux/dma-mapping.h>
0014 #include <linux/netdevice.h>
0015 #include <linux/ethtool.h>
0016 #include <linux/etherdevice.h>
0017 #include <linux/platform_device.h>
0018 #include <linux/delay.h>
0019 #include <linux/bitops.h>
0020 #include <linux/mii.h>
0021 #include <linux/if.h>
0022 #include <linux/if_ether.h>
0023 #include <linux/if_vlan.h>
0024 #include <linux/ip.h>
0025 #include <linux/in.h>
0026 #include <linux/ipv6.h>
0027 #include <linux/log2.h>
0028 #include <linux/jiffies.h>
0029 #include <linux/crc32.h>
0030 #include <linux/list.h>
0031 #include <linux/slab.h>
0032 
0033 #include <linux/io.h>
0034 #include <linux/of_device.h>
0035 
0036 #include "niu.h"
0037 
0038 /* This driver wants to store a link to a "next page" within the
0039  * page struct itself by overloading the content of the "mapping"
0040  * member. This is not expected by the page API, but does currently
0041  * work. However, the randstruct plugin gets very bothered by this
0042  * case because "mapping" (struct address_space) is randomized, so
0043  * casts to/from it trigger warnings. Hide this by way of a union,
0044  * to create a typed alias of "mapping", since that's how it is
0045  * actually being used here.
0046  */
0047 union niu_page {
0048     struct page page;
0049     struct {
0050         unsigned long __flags;  /* unused alias of "flags" */
0051         struct list_head __lru; /* unused alias of "lru" */
0052         struct page *next;  /* alias of "mapping" */
0053     };
0054 };
0055 #define niu_next_page(p)    container_of(p, union niu_page, page)->next
0056 
0057 #define DRV_MODULE_NAME     "niu"
0058 #define DRV_MODULE_VERSION  "1.1"
0059 #define DRV_MODULE_RELDATE  "Apr 22, 2010"
0060 
0061 static char version[] =
0062     DRV_MODULE_NAME ".c:v" DRV_MODULE_VERSION " (" DRV_MODULE_RELDATE ")\n";
0063 
0064 MODULE_AUTHOR("David S. Miller (davem@davemloft.net)");
0065 MODULE_DESCRIPTION("NIU ethernet driver");
0066 MODULE_LICENSE("GPL");
0067 MODULE_VERSION(DRV_MODULE_VERSION);
0068 
0069 #ifndef readq
0070 static u64 readq(void __iomem *reg)
0071 {
0072     return ((u64) readl(reg)) | (((u64) readl(reg + 4UL)) << 32);
0073 }
0074 
0075 static void writeq(u64 val, void __iomem *reg)
0076 {
0077     writel(val & 0xffffffff, reg);
0078     writel(val >> 32, reg + 0x4UL);
0079 }
0080 #endif
0081 
0082 static const struct pci_device_id niu_pci_tbl[] = {
0083     {PCI_DEVICE(PCI_VENDOR_ID_SUN, 0xabcd)},
0084     {}
0085 };
0086 
0087 MODULE_DEVICE_TABLE(pci, niu_pci_tbl);
0088 
0089 #define NIU_TX_TIMEOUT          (5 * HZ)
0090 
0091 #define nr64(reg)       readq(np->regs + (reg))
0092 #define nw64(reg, val)      writeq((val), np->regs + (reg))
0093 
0094 #define nr64_mac(reg)       readq(np->mac_regs + (reg))
0095 #define nw64_mac(reg, val)  writeq((val), np->mac_regs + (reg))
0096 
0097 #define nr64_ipp(reg)       readq(np->regs + np->ipp_off + (reg))
0098 #define nw64_ipp(reg, val)  writeq((val), np->regs + np->ipp_off + (reg))
0099 
0100 #define nr64_pcs(reg)       readq(np->regs + np->pcs_off + (reg))
0101 #define nw64_pcs(reg, val)  writeq((val), np->regs + np->pcs_off + (reg))
0102 
0103 #define nr64_xpcs(reg)      readq(np->regs + np->xpcs_off + (reg))
0104 #define nw64_xpcs(reg, val) writeq((val), np->regs + np->xpcs_off + (reg))
0105 
0106 #define NIU_MSG_DEFAULT (NETIF_MSG_DRV | NETIF_MSG_PROBE | NETIF_MSG_LINK)
0107 
0108 static int niu_debug;
0109 static int debug = -1;
0110 module_param(debug, int, 0);
0111 MODULE_PARM_DESC(debug, "NIU debug level");
0112 
0113 #define niu_lock_parent(np, flags) \
0114     spin_lock_irqsave(&np->parent->lock, flags)
0115 #define niu_unlock_parent(np, flags) \
0116     spin_unlock_irqrestore(&np->parent->lock, flags)
0117 
0118 static int serdes_init_10g_serdes(struct niu *np);
0119 
0120 static int __niu_wait_bits_clear_mac(struct niu *np, unsigned long reg,
0121                      u64 bits, int limit, int delay)
0122 {
0123     while (--limit >= 0) {
0124         u64 val = nr64_mac(reg);
0125 
0126         if (!(val & bits))
0127             break;
0128         udelay(delay);
0129     }
0130     if (limit < 0)
0131         return -ENODEV;
0132     return 0;
0133 }
0134 
0135 static int __niu_set_and_wait_clear_mac(struct niu *np, unsigned long reg,
0136                     u64 bits, int limit, int delay,
0137                     const char *reg_name)
0138 {
0139     int err;
0140 
0141     nw64_mac(reg, bits);
0142     err = __niu_wait_bits_clear_mac(np, reg, bits, limit, delay);
0143     if (err)
0144         netdev_err(np->dev, "bits (%llx) of register %s would not clear, val[%llx]\n",
0145                (unsigned long long)bits, reg_name,
0146                (unsigned long long)nr64_mac(reg));
0147     return err;
0148 }
0149 
0150 #define niu_set_and_wait_clear_mac(NP, REG, BITS, LIMIT, DELAY, REG_NAME) \
0151 ({  BUILD_BUG_ON(LIMIT <= 0 || DELAY < 0); \
0152     __niu_set_and_wait_clear_mac(NP, REG, BITS, LIMIT, DELAY, REG_NAME); \
0153 })
0154 
0155 static int __niu_wait_bits_clear_ipp(struct niu *np, unsigned long reg,
0156                      u64 bits, int limit, int delay)
0157 {
0158     while (--limit >= 0) {
0159         u64 val = nr64_ipp(reg);
0160 
0161         if (!(val & bits))
0162             break;
0163         udelay(delay);
0164     }
0165     if (limit < 0)
0166         return -ENODEV;
0167     return 0;
0168 }
0169 
0170 static int __niu_set_and_wait_clear_ipp(struct niu *np, unsigned long reg,
0171                     u64 bits, int limit, int delay,
0172                     const char *reg_name)
0173 {
0174     int err;
0175     u64 val;
0176 
0177     val = nr64_ipp(reg);
0178     val |= bits;
0179     nw64_ipp(reg, val);
0180 
0181     err = __niu_wait_bits_clear_ipp(np, reg, bits, limit, delay);
0182     if (err)
0183         netdev_err(np->dev, "bits (%llx) of register %s would not clear, val[%llx]\n",
0184                (unsigned long long)bits, reg_name,
0185                (unsigned long long)nr64_ipp(reg));
0186     return err;
0187 }
0188 
0189 #define niu_set_and_wait_clear_ipp(NP, REG, BITS, LIMIT, DELAY, REG_NAME) \
0190 ({  BUILD_BUG_ON(LIMIT <= 0 || DELAY < 0); \
0191     __niu_set_and_wait_clear_ipp(NP, REG, BITS, LIMIT, DELAY, REG_NAME); \
0192 })
0193 
0194 static int __niu_wait_bits_clear(struct niu *np, unsigned long reg,
0195                  u64 bits, int limit, int delay)
0196 {
0197     while (--limit >= 0) {
0198         u64 val = nr64(reg);
0199 
0200         if (!(val & bits))
0201             break;
0202         udelay(delay);
0203     }
0204     if (limit < 0)
0205         return -ENODEV;
0206     return 0;
0207 }
0208 
0209 #define niu_wait_bits_clear(NP, REG, BITS, LIMIT, DELAY) \
0210 ({  BUILD_BUG_ON(LIMIT <= 0 || DELAY < 0); \
0211     __niu_wait_bits_clear(NP, REG, BITS, LIMIT, DELAY); \
0212 })
0213 
0214 static int __niu_set_and_wait_clear(struct niu *np, unsigned long reg,
0215                     u64 bits, int limit, int delay,
0216                     const char *reg_name)
0217 {
0218     int err;
0219 
0220     nw64(reg, bits);
0221     err = __niu_wait_bits_clear(np, reg, bits, limit, delay);
0222     if (err)
0223         netdev_err(np->dev, "bits (%llx) of register %s would not clear, val[%llx]\n",
0224                (unsigned long long)bits, reg_name,
0225                (unsigned long long)nr64(reg));
0226     return err;
0227 }
0228 
0229 #define niu_set_and_wait_clear(NP, REG, BITS, LIMIT, DELAY, REG_NAME) \
0230 ({  BUILD_BUG_ON(LIMIT <= 0 || DELAY < 0); \
0231     __niu_set_and_wait_clear(NP, REG, BITS, LIMIT, DELAY, REG_NAME); \
0232 })
0233 
0234 static void niu_ldg_rearm(struct niu *np, struct niu_ldg *lp, int on)
0235 {
0236     u64 val = (u64) lp->timer;
0237 
0238     if (on)
0239         val |= LDG_IMGMT_ARM;
0240 
0241     nw64(LDG_IMGMT(lp->ldg_num), val);
0242 }
0243 
0244 static int niu_ldn_irq_enable(struct niu *np, int ldn, int on)
0245 {
0246     unsigned long mask_reg, bits;
0247     u64 val;
0248 
0249     if (ldn < 0 || ldn > LDN_MAX)
0250         return -EINVAL;
0251 
0252     if (ldn < 64) {
0253         mask_reg = LD_IM0(ldn);
0254         bits = LD_IM0_MASK;
0255     } else {
0256         mask_reg = LD_IM1(ldn - 64);
0257         bits = LD_IM1_MASK;
0258     }
0259 
0260     val = nr64(mask_reg);
0261     if (on)
0262         val &= ~bits;
0263     else
0264         val |= bits;
0265     nw64(mask_reg, val);
0266 
0267     return 0;
0268 }
0269 
0270 static int niu_enable_ldn_in_ldg(struct niu *np, struct niu_ldg *lp, int on)
0271 {
0272     struct niu_parent *parent = np->parent;
0273     int i;
0274 
0275     for (i = 0; i <= LDN_MAX; i++) {
0276         int err;
0277 
0278         if (parent->ldg_map[i] != lp->ldg_num)
0279             continue;
0280 
0281         err = niu_ldn_irq_enable(np, i, on);
0282         if (err)
0283             return err;
0284     }
0285     return 0;
0286 }
0287 
0288 static int niu_enable_interrupts(struct niu *np, int on)
0289 {
0290     int i;
0291 
0292     for (i = 0; i < np->num_ldg; i++) {
0293         struct niu_ldg *lp = &np->ldg[i];
0294         int err;
0295 
0296         err = niu_enable_ldn_in_ldg(np, lp, on);
0297         if (err)
0298             return err;
0299     }
0300     for (i = 0; i < np->num_ldg; i++)
0301         niu_ldg_rearm(np, &np->ldg[i], on);
0302 
0303     return 0;
0304 }
0305 
0306 static u32 phy_encode(u32 type, int port)
0307 {
0308     return type << (port * 2);
0309 }
0310 
0311 static u32 phy_decode(u32 val, int port)
0312 {
0313     return (val >> (port * 2)) & PORT_TYPE_MASK;
0314 }
0315 
0316 static int mdio_wait(struct niu *np)
0317 {
0318     int limit = 1000;
0319     u64 val;
0320 
0321     while (--limit > 0) {
0322         val = nr64(MIF_FRAME_OUTPUT);
0323         if ((val >> MIF_FRAME_OUTPUT_TA_SHIFT) & 0x1)
0324             return val & MIF_FRAME_OUTPUT_DATA;
0325 
0326         udelay(10);
0327     }
0328 
0329     return -ENODEV;
0330 }
0331 
0332 static int mdio_read(struct niu *np, int port, int dev, int reg)
0333 {
0334     int err;
0335 
0336     nw64(MIF_FRAME_OUTPUT, MDIO_ADDR_OP(port, dev, reg));
0337     err = mdio_wait(np);
0338     if (err < 0)
0339         return err;
0340 
0341     nw64(MIF_FRAME_OUTPUT, MDIO_READ_OP(port, dev));
0342     return mdio_wait(np);
0343 }
0344 
0345 static int mdio_write(struct niu *np, int port, int dev, int reg, int data)
0346 {
0347     int err;
0348 
0349     nw64(MIF_FRAME_OUTPUT, MDIO_ADDR_OP(port, dev, reg));
0350     err = mdio_wait(np);
0351     if (err < 0)
0352         return err;
0353 
0354     nw64(MIF_FRAME_OUTPUT, MDIO_WRITE_OP(port, dev, data));
0355     err = mdio_wait(np);
0356     if (err < 0)
0357         return err;
0358 
0359     return 0;
0360 }
0361 
0362 static int mii_read(struct niu *np, int port, int reg)
0363 {
0364     nw64(MIF_FRAME_OUTPUT, MII_READ_OP(port, reg));
0365     return mdio_wait(np);
0366 }
0367 
0368 static int mii_write(struct niu *np, int port, int reg, int data)
0369 {
0370     int err;
0371 
0372     nw64(MIF_FRAME_OUTPUT, MII_WRITE_OP(port, reg, data));
0373     err = mdio_wait(np);
0374     if (err < 0)
0375         return err;
0376 
0377     return 0;
0378 }
0379 
0380 static int esr2_set_tx_cfg(struct niu *np, unsigned long channel, u32 val)
0381 {
0382     int err;
0383 
0384     err = mdio_write(np, np->port, NIU_ESR2_DEV_ADDR,
0385              ESR2_TI_PLL_TX_CFG_L(channel),
0386              val & 0xffff);
0387     if (!err)
0388         err = mdio_write(np, np->port, NIU_ESR2_DEV_ADDR,
0389                  ESR2_TI_PLL_TX_CFG_H(channel),
0390                  val >> 16);
0391     return err;
0392 }
0393 
0394 static int esr2_set_rx_cfg(struct niu *np, unsigned long channel, u32 val)
0395 {
0396     int err;
0397 
0398     err = mdio_write(np, np->port, NIU_ESR2_DEV_ADDR,
0399              ESR2_TI_PLL_RX_CFG_L(channel),
0400              val & 0xffff);
0401     if (!err)
0402         err = mdio_write(np, np->port, NIU_ESR2_DEV_ADDR,
0403                  ESR2_TI_PLL_RX_CFG_H(channel),
0404                  val >> 16);
0405     return err;
0406 }
0407 
0408 /* Mode is always 10G fiber.  */
0409 static int serdes_init_niu_10g_fiber(struct niu *np)
0410 {
0411     struct niu_link_config *lp = &np->link_config;
0412     u32 tx_cfg, rx_cfg;
0413     unsigned long i;
0414 
0415     tx_cfg = (PLL_TX_CFG_ENTX | PLL_TX_CFG_SWING_1375MV);
0416     rx_cfg = (PLL_RX_CFG_ENRX | PLL_RX_CFG_TERM_0P8VDDT |
0417           PLL_RX_CFG_ALIGN_ENA | PLL_RX_CFG_LOS_LTHRESH |
0418           PLL_RX_CFG_EQ_LP_ADAPTIVE);
0419 
0420     if (lp->loopback_mode == LOOPBACK_PHY) {
0421         u16 test_cfg = PLL_TEST_CFG_LOOPBACK_CML_DIS;
0422 
0423         mdio_write(np, np->port, NIU_ESR2_DEV_ADDR,
0424                ESR2_TI_PLL_TEST_CFG_L, test_cfg);
0425 
0426         tx_cfg |= PLL_TX_CFG_ENTEST;
0427         rx_cfg |= PLL_RX_CFG_ENTEST;
0428     }
0429 
0430     /* Initialize all 4 lanes of the SERDES.  */
0431     for (i = 0; i < 4; i++) {
0432         int err = esr2_set_tx_cfg(np, i, tx_cfg);
0433         if (err)
0434             return err;
0435     }
0436 
0437     for (i = 0; i < 4; i++) {
0438         int err = esr2_set_rx_cfg(np, i, rx_cfg);
0439         if (err)
0440             return err;
0441     }
0442 
0443     return 0;
0444 }
0445 
0446 static int serdes_init_niu_1g_serdes(struct niu *np)
0447 {
0448     struct niu_link_config *lp = &np->link_config;
0449     u16 pll_cfg, pll_sts;
0450     int max_retry = 100;
0451     u64 sig, mask, val;
0452     u32 tx_cfg, rx_cfg;
0453     unsigned long i;
0454     int err;
0455 
0456     tx_cfg = (PLL_TX_CFG_ENTX | PLL_TX_CFG_SWING_1375MV |
0457           PLL_TX_CFG_RATE_HALF);
0458     rx_cfg = (PLL_RX_CFG_ENRX | PLL_RX_CFG_TERM_0P8VDDT |
0459           PLL_RX_CFG_ALIGN_ENA | PLL_RX_CFG_LOS_LTHRESH |
0460           PLL_RX_CFG_RATE_HALF);
0461 
0462     if (np->port == 0)
0463         rx_cfg |= PLL_RX_CFG_EQ_LP_ADAPTIVE;
0464 
0465     if (lp->loopback_mode == LOOPBACK_PHY) {
0466         u16 test_cfg = PLL_TEST_CFG_LOOPBACK_CML_DIS;
0467 
0468         mdio_write(np, np->port, NIU_ESR2_DEV_ADDR,
0469                ESR2_TI_PLL_TEST_CFG_L, test_cfg);
0470 
0471         tx_cfg |= PLL_TX_CFG_ENTEST;
0472         rx_cfg |= PLL_RX_CFG_ENTEST;
0473     }
0474 
0475     /* Initialize PLL for 1G */
0476     pll_cfg = (PLL_CFG_ENPLL | PLL_CFG_MPY_8X);
0477 
0478     err = mdio_write(np, np->port, NIU_ESR2_DEV_ADDR,
0479              ESR2_TI_PLL_CFG_L, pll_cfg);
0480     if (err) {
0481         netdev_err(np->dev, "NIU Port %d %s() mdio write to ESR2_TI_PLL_CFG_L failed\n",
0482                np->port, __func__);
0483         return err;
0484     }
0485 
0486     pll_sts = PLL_CFG_ENPLL;
0487 
0488     err = mdio_write(np, np->port, NIU_ESR2_DEV_ADDR,
0489              ESR2_TI_PLL_STS_L, pll_sts);
0490     if (err) {
0491         netdev_err(np->dev, "NIU Port %d %s() mdio write to ESR2_TI_PLL_STS_L failed\n",
0492                np->port, __func__);
0493         return err;
0494     }
0495 
0496     udelay(200);
0497 
0498     /* Initialize all 4 lanes of the SERDES.  */
0499     for (i = 0; i < 4; i++) {
0500         err = esr2_set_tx_cfg(np, i, tx_cfg);
0501         if (err)
0502             return err;
0503     }
0504 
0505     for (i = 0; i < 4; i++) {
0506         err = esr2_set_rx_cfg(np, i, rx_cfg);
0507         if (err)
0508             return err;
0509     }
0510 
0511     switch (np->port) {
0512     case 0:
0513         val = (ESR_INT_SRDY0_P0 | ESR_INT_DET0_P0);
0514         mask = val;
0515         break;
0516 
0517     case 1:
0518         val = (ESR_INT_SRDY0_P1 | ESR_INT_DET0_P1);
0519         mask = val;
0520         break;
0521 
0522     default:
0523         return -EINVAL;
0524     }
0525 
0526     while (max_retry--) {
0527         sig = nr64(ESR_INT_SIGNALS);
0528         if ((sig & mask) == val)
0529             break;
0530 
0531         mdelay(500);
0532     }
0533 
0534     if ((sig & mask) != val) {
0535         netdev_err(np->dev, "Port %u signal bits [%08x] are not [%08x]\n",
0536                np->port, (int)(sig & mask), (int)val);
0537         return -ENODEV;
0538     }
0539 
0540     return 0;
0541 }
0542 
0543 static int serdes_init_niu_10g_serdes(struct niu *np)
0544 {
0545     struct niu_link_config *lp = &np->link_config;
0546     u32 tx_cfg, rx_cfg, pll_cfg, pll_sts;
0547     int max_retry = 100;
0548     u64 sig, mask, val;
0549     unsigned long i;
0550     int err;
0551 
0552     tx_cfg = (PLL_TX_CFG_ENTX | PLL_TX_CFG_SWING_1375MV);
0553     rx_cfg = (PLL_RX_CFG_ENRX | PLL_RX_CFG_TERM_0P8VDDT |
0554           PLL_RX_CFG_ALIGN_ENA | PLL_RX_CFG_LOS_LTHRESH |
0555           PLL_RX_CFG_EQ_LP_ADAPTIVE);
0556 
0557     if (lp->loopback_mode == LOOPBACK_PHY) {
0558         u16 test_cfg = PLL_TEST_CFG_LOOPBACK_CML_DIS;
0559 
0560         mdio_write(np, np->port, NIU_ESR2_DEV_ADDR,
0561                ESR2_TI_PLL_TEST_CFG_L, test_cfg);
0562 
0563         tx_cfg |= PLL_TX_CFG_ENTEST;
0564         rx_cfg |= PLL_RX_CFG_ENTEST;
0565     }
0566 
0567     /* Initialize PLL for 10G */
0568     pll_cfg = (PLL_CFG_ENPLL | PLL_CFG_MPY_10X);
0569 
0570     err = mdio_write(np, np->port, NIU_ESR2_DEV_ADDR,
0571              ESR2_TI_PLL_CFG_L, pll_cfg & 0xffff);
0572     if (err) {
0573         netdev_err(np->dev, "NIU Port %d %s() mdio write to ESR2_TI_PLL_CFG_L failed\n",
0574                np->port, __func__);
0575         return err;
0576     }
0577 
0578     pll_sts = PLL_CFG_ENPLL;
0579 
0580     err = mdio_write(np, np->port, NIU_ESR2_DEV_ADDR,
0581              ESR2_TI_PLL_STS_L, pll_sts & 0xffff);
0582     if (err) {
0583         netdev_err(np->dev, "NIU Port %d %s() mdio write to ESR2_TI_PLL_STS_L failed\n",
0584                np->port, __func__);
0585         return err;
0586     }
0587 
0588     udelay(200);
0589 
0590     /* Initialize all 4 lanes of the SERDES.  */
0591     for (i = 0; i < 4; i++) {
0592         err = esr2_set_tx_cfg(np, i, tx_cfg);
0593         if (err)
0594             return err;
0595     }
0596 
0597     for (i = 0; i < 4; i++) {
0598         err = esr2_set_rx_cfg(np, i, rx_cfg);
0599         if (err)
0600             return err;
0601     }
0602 
0603     /* check if serdes is ready */
0604 
0605     switch (np->port) {
0606     case 0:
0607         mask = ESR_INT_SIGNALS_P0_BITS;
0608         val = (ESR_INT_SRDY0_P0 |
0609                ESR_INT_DET0_P0 |
0610                ESR_INT_XSRDY_P0 |
0611                ESR_INT_XDP_P0_CH3 |
0612                ESR_INT_XDP_P0_CH2 |
0613                ESR_INT_XDP_P0_CH1 |
0614                ESR_INT_XDP_P0_CH0);
0615         break;
0616 
0617     case 1:
0618         mask = ESR_INT_SIGNALS_P1_BITS;
0619         val = (ESR_INT_SRDY0_P1 |
0620                ESR_INT_DET0_P1 |
0621                ESR_INT_XSRDY_P1 |
0622                ESR_INT_XDP_P1_CH3 |
0623                ESR_INT_XDP_P1_CH2 |
0624                ESR_INT_XDP_P1_CH1 |
0625                ESR_INT_XDP_P1_CH0);
0626         break;
0627 
0628     default:
0629         return -EINVAL;
0630     }
0631 
0632     while (max_retry--) {
0633         sig = nr64(ESR_INT_SIGNALS);
0634         if ((sig & mask) == val)
0635             break;
0636 
0637         mdelay(500);
0638     }
0639 
0640     if ((sig & mask) != val) {
0641         pr_info("NIU Port %u signal bits [%08x] are not [%08x] for 10G...trying 1G\n",
0642             np->port, (int)(sig & mask), (int)val);
0643 
0644         /* 10G failed, try initializing at 1G */
0645         err = serdes_init_niu_1g_serdes(np);
0646         if (!err) {
0647             np->flags &= ~NIU_FLAGS_10G;
0648             np->mac_xcvr = MAC_XCVR_PCS;
0649         }  else {
0650             netdev_err(np->dev, "Port %u 10G/1G SERDES Link Failed\n",
0651                    np->port);
0652             return -ENODEV;
0653         }
0654     }
0655     return 0;
0656 }
0657 
0658 static int esr_read_rxtx_ctrl(struct niu *np, unsigned long chan, u32 *val)
0659 {
0660     int err;
0661 
0662     err = mdio_read(np, np->port, NIU_ESR_DEV_ADDR, ESR_RXTX_CTRL_L(chan));
0663     if (err >= 0) {
0664         *val = (err & 0xffff);
0665         err = mdio_read(np, np->port, NIU_ESR_DEV_ADDR,
0666                 ESR_RXTX_CTRL_H(chan));
0667         if (err >= 0)
0668             *val |= ((err & 0xffff) << 16);
0669         err = 0;
0670     }
0671     return err;
0672 }
0673 
0674 static int esr_read_glue0(struct niu *np, unsigned long chan, u32 *val)
0675 {
0676     int err;
0677 
0678     err = mdio_read(np, np->port, NIU_ESR_DEV_ADDR,
0679             ESR_GLUE_CTRL0_L(chan));
0680     if (err >= 0) {
0681         *val = (err & 0xffff);
0682         err = mdio_read(np, np->port, NIU_ESR_DEV_ADDR,
0683                 ESR_GLUE_CTRL0_H(chan));
0684         if (err >= 0) {
0685             *val |= ((err & 0xffff) << 16);
0686             err = 0;
0687         }
0688     }
0689     return err;
0690 }
0691 
0692 static int esr_read_reset(struct niu *np, u32 *val)
0693 {
0694     int err;
0695 
0696     err = mdio_read(np, np->port, NIU_ESR_DEV_ADDR,
0697             ESR_RXTX_RESET_CTRL_L);
0698     if (err >= 0) {
0699         *val = (err & 0xffff);
0700         err = mdio_read(np, np->port, NIU_ESR_DEV_ADDR,
0701                 ESR_RXTX_RESET_CTRL_H);
0702         if (err >= 0) {
0703             *val |= ((err & 0xffff) << 16);
0704             err = 0;
0705         }
0706     }
0707     return err;
0708 }
0709 
0710 static int esr_write_rxtx_ctrl(struct niu *np, unsigned long chan, u32 val)
0711 {
0712     int err;
0713 
0714     err = mdio_write(np, np->port, NIU_ESR_DEV_ADDR,
0715              ESR_RXTX_CTRL_L(chan), val & 0xffff);
0716     if (!err)
0717         err = mdio_write(np, np->port, NIU_ESR_DEV_ADDR,
0718                  ESR_RXTX_CTRL_H(chan), (val >> 16));
0719     return err;
0720 }
0721 
0722 static int esr_write_glue0(struct niu *np, unsigned long chan, u32 val)
0723 {
0724     int err;
0725 
0726     err = mdio_write(np, np->port, NIU_ESR_DEV_ADDR,
0727             ESR_GLUE_CTRL0_L(chan), val & 0xffff);
0728     if (!err)
0729         err = mdio_write(np, np->port, NIU_ESR_DEV_ADDR,
0730                  ESR_GLUE_CTRL0_H(chan), (val >> 16));
0731     return err;
0732 }
0733 
0734 static int esr_reset(struct niu *np)
0735 {
0736     u32 reset;
0737     int err;
0738 
0739     err = mdio_write(np, np->port, NIU_ESR_DEV_ADDR,
0740              ESR_RXTX_RESET_CTRL_L, 0x0000);
0741     if (err)
0742         return err;
0743     err = mdio_write(np, np->port, NIU_ESR_DEV_ADDR,
0744              ESR_RXTX_RESET_CTRL_H, 0xffff);
0745     if (err)
0746         return err;
0747     udelay(200);
0748 
0749     err = mdio_write(np, np->port, NIU_ESR_DEV_ADDR,
0750              ESR_RXTX_RESET_CTRL_L, 0xffff);
0751     if (err)
0752         return err;
0753     udelay(200);
0754 
0755     err = mdio_write(np, np->port, NIU_ESR_DEV_ADDR,
0756              ESR_RXTX_RESET_CTRL_H, 0x0000);
0757     if (err)
0758         return err;
0759     udelay(200);
0760 
0761     err = esr_read_reset(np, &reset);
0762     if (err)
0763         return err;
0764     if (reset != 0) {
0765         netdev_err(np->dev, "Port %u ESR_RESET did not clear [%08x]\n",
0766                np->port, reset);
0767         return -ENODEV;
0768     }
0769 
0770     return 0;
0771 }
0772 
0773 static int serdes_init_10g(struct niu *np)
0774 {
0775     struct niu_link_config *lp = &np->link_config;
0776     unsigned long ctrl_reg, test_cfg_reg, i;
0777     u64 ctrl_val, test_cfg_val, sig, mask, val;
0778     int err;
0779 
0780     switch (np->port) {
0781     case 0:
0782         ctrl_reg = ENET_SERDES_0_CTRL_CFG;
0783         test_cfg_reg = ENET_SERDES_0_TEST_CFG;
0784         break;
0785     case 1:
0786         ctrl_reg = ENET_SERDES_1_CTRL_CFG;
0787         test_cfg_reg = ENET_SERDES_1_TEST_CFG;
0788         break;
0789 
0790     default:
0791         return -EINVAL;
0792     }
0793     ctrl_val = (ENET_SERDES_CTRL_SDET_0 |
0794             ENET_SERDES_CTRL_SDET_1 |
0795             ENET_SERDES_CTRL_SDET_2 |
0796             ENET_SERDES_CTRL_SDET_3 |
0797             (0x5 << ENET_SERDES_CTRL_EMPH_0_SHIFT) |
0798             (0x5 << ENET_SERDES_CTRL_EMPH_1_SHIFT) |
0799             (0x5 << ENET_SERDES_CTRL_EMPH_2_SHIFT) |
0800             (0x5 << ENET_SERDES_CTRL_EMPH_3_SHIFT) |
0801             (0x1 << ENET_SERDES_CTRL_LADJ_0_SHIFT) |
0802             (0x1 << ENET_SERDES_CTRL_LADJ_1_SHIFT) |
0803             (0x1 << ENET_SERDES_CTRL_LADJ_2_SHIFT) |
0804             (0x1 << ENET_SERDES_CTRL_LADJ_3_SHIFT));
0805     test_cfg_val = 0;
0806 
0807     if (lp->loopback_mode == LOOPBACK_PHY) {
0808         test_cfg_val |= ((ENET_TEST_MD_PAD_LOOPBACK <<
0809                   ENET_SERDES_TEST_MD_0_SHIFT) |
0810                  (ENET_TEST_MD_PAD_LOOPBACK <<
0811                   ENET_SERDES_TEST_MD_1_SHIFT) |
0812                  (ENET_TEST_MD_PAD_LOOPBACK <<
0813                   ENET_SERDES_TEST_MD_2_SHIFT) |
0814                  (ENET_TEST_MD_PAD_LOOPBACK <<
0815                   ENET_SERDES_TEST_MD_3_SHIFT));
0816     }
0817 
0818     nw64(ctrl_reg, ctrl_val);
0819     nw64(test_cfg_reg, test_cfg_val);
0820 
0821     /* Initialize all 4 lanes of the SERDES.  */
0822     for (i = 0; i < 4; i++) {
0823         u32 rxtx_ctrl, glue0;
0824 
0825         err = esr_read_rxtx_ctrl(np, i, &rxtx_ctrl);
0826         if (err)
0827             return err;
0828         err = esr_read_glue0(np, i, &glue0);
0829         if (err)
0830             return err;
0831 
0832         rxtx_ctrl &= ~(ESR_RXTX_CTRL_VMUXLO);
0833         rxtx_ctrl |= (ESR_RXTX_CTRL_ENSTRETCH |
0834                   (2 << ESR_RXTX_CTRL_VMUXLO_SHIFT));
0835 
0836         glue0 &= ~(ESR_GLUE_CTRL0_SRATE |
0837                ESR_GLUE_CTRL0_THCNT |
0838                ESR_GLUE_CTRL0_BLTIME);
0839         glue0 |= (ESR_GLUE_CTRL0_RXLOSENAB |
0840               (0xf << ESR_GLUE_CTRL0_SRATE_SHIFT) |
0841               (0xff << ESR_GLUE_CTRL0_THCNT_SHIFT) |
0842               (BLTIME_300_CYCLES <<
0843                ESR_GLUE_CTRL0_BLTIME_SHIFT));
0844 
0845         err = esr_write_rxtx_ctrl(np, i, rxtx_ctrl);
0846         if (err)
0847             return err;
0848         err = esr_write_glue0(np, i, glue0);
0849         if (err)
0850             return err;
0851     }
0852 
0853     err = esr_reset(np);
0854     if (err)
0855         return err;
0856 
0857     sig = nr64(ESR_INT_SIGNALS);
0858     switch (np->port) {
0859     case 0:
0860         mask = ESR_INT_SIGNALS_P0_BITS;
0861         val = (ESR_INT_SRDY0_P0 |
0862                ESR_INT_DET0_P0 |
0863                ESR_INT_XSRDY_P0 |
0864                ESR_INT_XDP_P0_CH3 |
0865                ESR_INT_XDP_P0_CH2 |
0866                ESR_INT_XDP_P0_CH1 |
0867                ESR_INT_XDP_P0_CH0);
0868         break;
0869 
0870     case 1:
0871         mask = ESR_INT_SIGNALS_P1_BITS;
0872         val = (ESR_INT_SRDY0_P1 |
0873                ESR_INT_DET0_P1 |
0874                ESR_INT_XSRDY_P1 |
0875                ESR_INT_XDP_P1_CH3 |
0876                ESR_INT_XDP_P1_CH2 |
0877                ESR_INT_XDP_P1_CH1 |
0878                ESR_INT_XDP_P1_CH0);
0879         break;
0880 
0881     default:
0882         return -EINVAL;
0883     }
0884 
0885     if ((sig & mask) != val) {
0886         if (np->flags & NIU_FLAGS_HOTPLUG_PHY) {
0887             np->flags &= ~NIU_FLAGS_HOTPLUG_PHY_PRESENT;
0888             return 0;
0889         }
0890         netdev_err(np->dev, "Port %u signal bits [%08x] are not [%08x]\n",
0891                np->port, (int)(sig & mask), (int)val);
0892         return -ENODEV;
0893     }
0894     if (np->flags & NIU_FLAGS_HOTPLUG_PHY)
0895         np->flags |= NIU_FLAGS_HOTPLUG_PHY_PRESENT;
0896     return 0;
0897 }
0898 
0899 static int serdes_init_1g(struct niu *np)
0900 {
0901     u64 val;
0902 
0903     val = nr64(ENET_SERDES_1_PLL_CFG);
0904     val &= ~ENET_SERDES_PLL_FBDIV2;
0905     switch (np->port) {
0906     case 0:
0907         val |= ENET_SERDES_PLL_HRATE0;
0908         break;
0909     case 1:
0910         val |= ENET_SERDES_PLL_HRATE1;
0911         break;
0912     case 2:
0913         val |= ENET_SERDES_PLL_HRATE2;
0914         break;
0915     case 3:
0916         val |= ENET_SERDES_PLL_HRATE3;
0917         break;
0918     default:
0919         return -EINVAL;
0920     }
0921     nw64(ENET_SERDES_1_PLL_CFG, val);
0922 
0923     return 0;
0924 }
0925 
0926 static int serdes_init_1g_serdes(struct niu *np)
0927 {
0928     struct niu_link_config *lp = &np->link_config;
0929     unsigned long ctrl_reg, test_cfg_reg, pll_cfg, i;
0930     u64 ctrl_val, test_cfg_val, sig, mask, val;
0931     int err;
0932     u64 reset_val, val_rd;
0933 
0934     val = ENET_SERDES_PLL_HRATE0 | ENET_SERDES_PLL_HRATE1 |
0935         ENET_SERDES_PLL_HRATE2 | ENET_SERDES_PLL_HRATE3 |
0936         ENET_SERDES_PLL_FBDIV0;
0937     switch (np->port) {
0938     case 0:
0939         reset_val =  ENET_SERDES_RESET_0;
0940         ctrl_reg = ENET_SERDES_0_CTRL_CFG;
0941         test_cfg_reg = ENET_SERDES_0_TEST_CFG;
0942         pll_cfg = ENET_SERDES_0_PLL_CFG;
0943         break;
0944     case 1:
0945         reset_val =  ENET_SERDES_RESET_1;
0946         ctrl_reg = ENET_SERDES_1_CTRL_CFG;
0947         test_cfg_reg = ENET_SERDES_1_TEST_CFG;
0948         pll_cfg = ENET_SERDES_1_PLL_CFG;
0949         break;
0950 
0951     default:
0952         return -EINVAL;
0953     }
0954     ctrl_val = (ENET_SERDES_CTRL_SDET_0 |
0955             ENET_SERDES_CTRL_SDET_1 |
0956             ENET_SERDES_CTRL_SDET_2 |
0957             ENET_SERDES_CTRL_SDET_3 |
0958             (0x5 << ENET_SERDES_CTRL_EMPH_0_SHIFT) |
0959             (0x5 << ENET_SERDES_CTRL_EMPH_1_SHIFT) |
0960             (0x5 << ENET_SERDES_CTRL_EMPH_2_SHIFT) |
0961             (0x5 << ENET_SERDES_CTRL_EMPH_3_SHIFT) |
0962             (0x1 << ENET_SERDES_CTRL_LADJ_0_SHIFT) |
0963             (0x1 << ENET_SERDES_CTRL_LADJ_1_SHIFT) |
0964             (0x1 << ENET_SERDES_CTRL_LADJ_2_SHIFT) |
0965             (0x1 << ENET_SERDES_CTRL_LADJ_3_SHIFT));
0966     test_cfg_val = 0;
0967 
0968     if (lp->loopback_mode == LOOPBACK_PHY) {
0969         test_cfg_val |= ((ENET_TEST_MD_PAD_LOOPBACK <<
0970                   ENET_SERDES_TEST_MD_0_SHIFT) |
0971                  (ENET_TEST_MD_PAD_LOOPBACK <<
0972                   ENET_SERDES_TEST_MD_1_SHIFT) |
0973                  (ENET_TEST_MD_PAD_LOOPBACK <<
0974                   ENET_SERDES_TEST_MD_2_SHIFT) |
0975                  (ENET_TEST_MD_PAD_LOOPBACK <<
0976                   ENET_SERDES_TEST_MD_3_SHIFT));
0977     }
0978 
0979     nw64(ENET_SERDES_RESET, reset_val);
0980     mdelay(20);
0981     val_rd = nr64(ENET_SERDES_RESET);
0982     val_rd &= ~reset_val;
0983     nw64(pll_cfg, val);
0984     nw64(ctrl_reg, ctrl_val);
0985     nw64(test_cfg_reg, test_cfg_val);
0986     nw64(ENET_SERDES_RESET, val_rd);
0987     mdelay(2000);
0988 
0989     /* Initialize all 4 lanes of the SERDES.  */
0990     for (i = 0; i < 4; i++) {
0991         u32 rxtx_ctrl, glue0;
0992 
0993         err = esr_read_rxtx_ctrl(np, i, &rxtx_ctrl);
0994         if (err)
0995             return err;
0996         err = esr_read_glue0(np, i, &glue0);
0997         if (err)
0998             return err;
0999 
1000         rxtx_ctrl &= ~(ESR_RXTX_CTRL_VMUXLO);
1001         rxtx_ctrl |= (ESR_RXTX_CTRL_ENSTRETCH |
1002                   (2 << ESR_RXTX_CTRL_VMUXLO_SHIFT));
1003 
1004         glue0 &= ~(ESR_GLUE_CTRL0_SRATE |
1005                ESR_GLUE_CTRL0_THCNT |
1006                ESR_GLUE_CTRL0_BLTIME);
1007         glue0 |= (ESR_GLUE_CTRL0_RXLOSENAB |
1008               (0xf << ESR_GLUE_CTRL0_SRATE_SHIFT) |
1009               (0xff << ESR_GLUE_CTRL0_THCNT_SHIFT) |
1010               (BLTIME_300_CYCLES <<
1011                ESR_GLUE_CTRL0_BLTIME_SHIFT));
1012 
1013         err = esr_write_rxtx_ctrl(np, i, rxtx_ctrl);
1014         if (err)
1015             return err;
1016         err = esr_write_glue0(np, i, glue0);
1017         if (err)
1018             return err;
1019     }
1020 
1021 
1022     sig = nr64(ESR_INT_SIGNALS);
1023     switch (np->port) {
1024     case 0:
1025         val = (ESR_INT_SRDY0_P0 | ESR_INT_DET0_P0);
1026         mask = val;
1027         break;
1028 
1029     case 1:
1030         val = (ESR_INT_SRDY0_P1 | ESR_INT_DET0_P1);
1031         mask = val;
1032         break;
1033 
1034     default:
1035         return -EINVAL;
1036     }
1037 
1038     if ((sig & mask) != val) {
1039         netdev_err(np->dev, "Port %u signal bits [%08x] are not [%08x]\n",
1040                np->port, (int)(sig & mask), (int)val);
1041         return -ENODEV;
1042     }
1043 
1044     return 0;
1045 }
1046 
1047 static int link_status_1g_serdes(struct niu *np, int *link_up_p)
1048 {
1049     struct niu_link_config *lp = &np->link_config;
1050     int link_up;
1051     u64 val;
1052     u16 current_speed;
1053     unsigned long flags;
1054     u8 current_duplex;
1055 
1056     link_up = 0;
1057     current_speed = SPEED_INVALID;
1058     current_duplex = DUPLEX_INVALID;
1059 
1060     spin_lock_irqsave(&np->lock, flags);
1061 
1062     val = nr64_pcs(PCS_MII_STAT);
1063 
1064     if (val & PCS_MII_STAT_LINK_STATUS) {
1065         link_up = 1;
1066         current_speed = SPEED_1000;
1067         current_duplex = DUPLEX_FULL;
1068     }
1069 
1070     lp->active_speed = current_speed;
1071     lp->active_duplex = current_duplex;
1072     spin_unlock_irqrestore(&np->lock, flags);
1073 
1074     *link_up_p = link_up;
1075     return 0;
1076 }
1077 
1078 static int link_status_10g_serdes(struct niu *np, int *link_up_p)
1079 {
1080     unsigned long flags;
1081     struct niu_link_config *lp = &np->link_config;
1082     int link_up = 0;
1083     int link_ok = 1;
1084     u64 val, val2;
1085     u16 current_speed;
1086     u8 current_duplex;
1087 
1088     if (!(np->flags & NIU_FLAGS_10G))
1089         return link_status_1g_serdes(np, link_up_p);
1090 
1091     current_speed = SPEED_INVALID;
1092     current_duplex = DUPLEX_INVALID;
1093     spin_lock_irqsave(&np->lock, flags);
1094 
1095     val = nr64_xpcs(XPCS_STATUS(0));
1096     val2 = nr64_mac(XMAC_INTER2);
1097     if (val2 & 0x01000000)
1098         link_ok = 0;
1099 
1100     if ((val & 0x1000ULL) && link_ok) {
1101         link_up = 1;
1102         current_speed = SPEED_10000;
1103         current_duplex = DUPLEX_FULL;
1104     }
1105     lp->active_speed = current_speed;
1106     lp->active_duplex = current_duplex;
1107     spin_unlock_irqrestore(&np->lock, flags);
1108     *link_up_p = link_up;
1109     return 0;
1110 }
1111 
1112 static int link_status_mii(struct niu *np, int *link_up_p)
1113 {
1114     struct niu_link_config *lp = &np->link_config;
1115     int err;
1116     int bmsr, advert, ctrl1000, stat1000, lpa, bmcr, estatus;
1117     int supported, advertising, active_speed, active_duplex;
1118 
1119     err = mii_read(np, np->phy_addr, MII_BMCR);
1120     if (unlikely(err < 0))
1121         return err;
1122     bmcr = err;
1123 
1124     err = mii_read(np, np->phy_addr, MII_BMSR);
1125     if (unlikely(err < 0))
1126         return err;
1127     bmsr = err;
1128 
1129     err = mii_read(np, np->phy_addr, MII_ADVERTISE);
1130     if (unlikely(err < 0))
1131         return err;
1132     advert = err;
1133 
1134     err = mii_read(np, np->phy_addr, MII_LPA);
1135     if (unlikely(err < 0))
1136         return err;
1137     lpa = err;
1138 
1139     if (likely(bmsr & BMSR_ESTATEN)) {
1140         err = mii_read(np, np->phy_addr, MII_ESTATUS);
1141         if (unlikely(err < 0))
1142             return err;
1143         estatus = err;
1144 
1145         err = mii_read(np, np->phy_addr, MII_CTRL1000);
1146         if (unlikely(err < 0))
1147             return err;
1148         ctrl1000 = err;
1149 
1150         err = mii_read(np, np->phy_addr, MII_STAT1000);
1151         if (unlikely(err < 0))
1152             return err;
1153         stat1000 = err;
1154     } else
1155         estatus = ctrl1000 = stat1000 = 0;
1156 
1157     supported = 0;
1158     if (bmsr & BMSR_ANEGCAPABLE)
1159         supported |= SUPPORTED_Autoneg;
1160     if (bmsr & BMSR_10HALF)
1161         supported |= SUPPORTED_10baseT_Half;
1162     if (bmsr & BMSR_10FULL)
1163         supported |= SUPPORTED_10baseT_Full;
1164     if (bmsr & BMSR_100HALF)
1165         supported |= SUPPORTED_100baseT_Half;
1166     if (bmsr & BMSR_100FULL)
1167         supported |= SUPPORTED_100baseT_Full;
1168     if (estatus & ESTATUS_1000_THALF)
1169         supported |= SUPPORTED_1000baseT_Half;
1170     if (estatus & ESTATUS_1000_TFULL)
1171         supported |= SUPPORTED_1000baseT_Full;
1172     lp->supported = supported;
1173 
1174     advertising = mii_adv_to_ethtool_adv_t(advert);
1175     advertising |= mii_ctrl1000_to_ethtool_adv_t(ctrl1000);
1176 
1177     if (bmcr & BMCR_ANENABLE) {
1178         int neg, neg1000;
1179 
1180         lp->active_autoneg = 1;
1181         advertising |= ADVERTISED_Autoneg;
1182 
1183         neg = advert & lpa;
1184         neg1000 = (ctrl1000 << 2) & stat1000;
1185 
1186         if (neg1000 & (LPA_1000FULL | LPA_1000HALF))
1187             active_speed = SPEED_1000;
1188         else if (neg & LPA_100)
1189             active_speed = SPEED_100;
1190         else if (neg & (LPA_10HALF | LPA_10FULL))
1191             active_speed = SPEED_10;
1192         else
1193             active_speed = SPEED_INVALID;
1194 
1195         if ((neg1000 & LPA_1000FULL) || (neg & LPA_DUPLEX))
1196             active_duplex = DUPLEX_FULL;
1197         else if (active_speed != SPEED_INVALID)
1198             active_duplex = DUPLEX_HALF;
1199         else
1200             active_duplex = DUPLEX_INVALID;
1201     } else {
1202         lp->active_autoneg = 0;
1203 
1204         if ((bmcr & BMCR_SPEED1000) && !(bmcr & BMCR_SPEED100))
1205             active_speed = SPEED_1000;
1206         else if (bmcr & BMCR_SPEED100)
1207             active_speed = SPEED_100;
1208         else
1209             active_speed = SPEED_10;
1210 
1211         if (bmcr & BMCR_FULLDPLX)
1212             active_duplex = DUPLEX_FULL;
1213         else
1214             active_duplex = DUPLEX_HALF;
1215     }
1216 
1217     lp->active_advertising = advertising;
1218     lp->active_speed = active_speed;
1219     lp->active_duplex = active_duplex;
1220     *link_up_p = !!(bmsr & BMSR_LSTATUS);
1221 
1222     return 0;
1223 }
1224 
1225 static int link_status_1g_rgmii(struct niu *np, int *link_up_p)
1226 {
1227     struct niu_link_config *lp = &np->link_config;
1228     u16 current_speed, bmsr;
1229     unsigned long flags;
1230     u8 current_duplex;
1231     int err, link_up;
1232 
1233     link_up = 0;
1234     current_speed = SPEED_INVALID;
1235     current_duplex = DUPLEX_INVALID;
1236 
1237     spin_lock_irqsave(&np->lock, flags);
1238 
1239     err = mii_read(np, np->phy_addr, MII_BMSR);
1240     if (err < 0)
1241         goto out;
1242 
1243     bmsr = err;
1244     if (bmsr & BMSR_LSTATUS) {
1245         link_up = 1;
1246         current_speed = SPEED_1000;
1247         current_duplex = DUPLEX_FULL;
1248     }
1249     lp->active_speed = current_speed;
1250     lp->active_duplex = current_duplex;
1251     err = 0;
1252 
1253 out:
1254     spin_unlock_irqrestore(&np->lock, flags);
1255 
1256     *link_up_p = link_up;
1257     return err;
1258 }
1259 
1260 static int link_status_1g(struct niu *np, int *link_up_p)
1261 {
1262     struct niu_link_config *lp = &np->link_config;
1263     unsigned long flags;
1264     int err;
1265 
1266     spin_lock_irqsave(&np->lock, flags);
1267 
1268     err = link_status_mii(np, link_up_p);
1269     lp->supported |= SUPPORTED_TP;
1270     lp->active_advertising |= ADVERTISED_TP;
1271 
1272     spin_unlock_irqrestore(&np->lock, flags);
1273     return err;
1274 }
1275 
1276 static int bcm8704_reset(struct niu *np)
1277 {
1278     int err, limit;
1279 
1280     err = mdio_read(np, np->phy_addr,
1281             BCM8704_PHYXS_DEV_ADDR, MII_BMCR);
1282     if (err < 0 || err == 0xffff)
1283         return err;
1284     err |= BMCR_RESET;
1285     err = mdio_write(np, np->phy_addr, BCM8704_PHYXS_DEV_ADDR,
1286              MII_BMCR, err);
1287     if (err)
1288         return err;
1289 
1290     limit = 1000;
1291     while (--limit >= 0) {
1292         err = mdio_read(np, np->phy_addr,
1293                 BCM8704_PHYXS_DEV_ADDR, MII_BMCR);
1294         if (err < 0)
1295             return err;
1296         if (!(err & BMCR_RESET))
1297             break;
1298     }
1299     if (limit < 0) {
1300         netdev_err(np->dev, "Port %u PHY will not reset (bmcr=%04x)\n",
1301                np->port, (err & 0xffff));
1302         return -ENODEV;
1303     }
1304     return 0;
1305 }
1306 
1307 /* When written, certain PHY registers need to be read back twice
1308  * in order for the bits to settle properly.
1309  */
1310 static int bcm8704_user_dev3_readback(struct niu *np, int reg)
1311 {
1312     int err = mdio_read(np, np->phy_addr, BCM8704_USER_DEV3_ADDR, reg);
1313     if (err < 0)
1314         return err;
1315     err = mdio_read(np, np->phy_addr, BCM8704_USER_DEV3_ADDR, reg);
1316     if (err < 0)
1317         return err;
1318     return 0;
1319 }
1320 
1321 static int bcm8706_init_user_dev3(struct niu *np)
1322 {
1323     int err;
1324 
1325 
1326     err = mdio_read(np, np->phy_addr, BCM8704_USER_DEV3_ADDR,
1327             BCM8704_USER_OPT_DIGITAL_CTRL);
1328     if (err < 0)
1329         return err;
1330     err &= ~USER_ODIG_CTRL_GPIOS;
1331     err |= (0x3 << USER_ODIG_CTRL_GPIOS_SHIFT);
1332     err |=  USER_ODIG_CTRL_RESV2;
1333     err = mdio_write(np, np->phy_addr, BCM8704_USER_DEV3_ADDR,
1334              BCM8704_USER_OPT_DIGITAL_CTRL, err);
1335     if (err)
1336         return err;
1337 
1338     mdelay(1000);
1339 
1340     return 0;
1341 }
1342 
1343 static int bcm8704_init_user_dev3(struct niu *np)
1344 {
1345     int err;
1346 
1347     err = mdio_write(np, np->phy_addr,
1348              BCM8704_USER_DEV3_ADDR, BCM8704_USER_CONTROL,
1349              (USER_CONTROL_OPTXRST_LVL |
1350               USER_CONTROL_OPBIASFLT_LVL |
1351               USER_CONTROL_OBTMPFLT_LVL |
1352               USER_CONTROL_OPPRFLT_LVL |
1353               USER_CONTROL_OPTXFLT_LVL |
1354               USER_CONTROL_OPRXLOS_LVL |
1355               USER_CONTROL_OPRXFLT_LVL |
1356               USER_CONTROL_OPTXON_LVL |
1357               (0x3f << USER_CONTROL_RES1_SHIFT)));
1358     if (err)
1359         return err;
1360 
1361     err = mdio_write(np, np->phy_addr,
1362              BCM8704_USER_DEV3_ADDR, BCM8704_USER_PMD_TX_CONTROL,
1363              (USER_PMD_TX_CTL_XFP_CLKEN |
1364               (1 << USER_PMD_TX_CTL_TX_DAC_TXD_SH) |
1365               (2 << USER_PMD_TX_CTL_TX_DAC_TXCK_SH) |
1366               USER_PMD_TX_CTL_TSCK_LPWREN));
1367     if (err)
1368         return err;
1369 
1370     err = bcm8704_user_dev3_readback(np, BCM8704_USER_CONTROL);
1371     if (err)
1372         return err;
1373     err = bcm8704_user_dev3_readback(np, BCM8704_USER_PMD_TX_CONTROL);
1374     if (err)
1375         return err;
1376 
1377     err = mdio_read(np, np->phy_addr, BCM8704_USER_DEV3_ADDR,
1378             BCM8704_USER_OPT_DIGITAL_CTRL);
1379     if (err < 0)
1380         return err;
1381     err &= ~USER_ODIG_CTRL_GPIOS;
1382     err |= (0x3 << USER_ODIG_CTRL_GPIOS_SHIFT);
1383     err = mdio_write(np, np->phy_addr, BCM8704_USER_DEV3_ADDR,
1384              BCM8704_USER_OPT_DIGITAL_CTRL, err);
1385     if (err)
1386         return err;
1387 
1388     mdelay(1000);
1389 
1390     return 0;
1391 }
1392 
1393 static int mrvl88x2011_act_led(struct niu *np, int val)
1394 {
1395     int err;
1396 
1397     err  = mdio_read(np, np->phy_addr, MRVL88X2011_USER_DEV2_ADDR,
1398         MRVL88X2011_LED_8_TO_11_CTL);
1399     if (err < 0)
1400         return err;
1401 
1402     err &= ~MRVL88X2011_LED(MRVL88X2011_LED_ACT,MRVL88X2011_LED_CTL_MASK);
1403     err |=  MRVL88X2011_LED(MRVL88X2011_LED_ACT,val);
1404 
1405     return mdio_write(np, np->phy_addr, MRVL88X2011_USER_DEV2_ADDR,
1406               MRVL88X2011_LED_8_TO_11_CTL, err);
1407 }
1408 
1409 static int mrvl88x2011_led_blink_rate(struct niu *np, int rate)
1410 {
1411     int err;
1412 
1413     err = mdio_read(np, np->phy_addr, MRVL88X2011_USER_DEV2_ADDR,
1414             MRVL88X2011_LED_BLINK_CTL);
1415     if (err >= 0) {
1416         err &= ~MRVL88X2011_LED_BLKRATE_MASK;
1417         err |= (rate << 4);
1418 
1419         err = mdio_write(np, np->phy_addr, MRVL88X2011_USER_DEV2_ADDR,
1420                  MRVL88X2011_LED_BLINK_CTL, err);
1421     }
1422 
1423     return err;
1424 }
1425 
1426 static int xcvr_init_10g_mrvl88x2011(struct niu *np)
1427 {
1428     int err;
1429 
1430     /* Set LED functions */
1431     err = mrvl88x2011_led_blink_rate(np, MRVL88X2011_LED_BLKRATE_134MS);
1432     if (err)
1433         return err;
1434 
1435     /* led activity */
1436     err = mrvl88x2011_act_led(np, MRVL88X2011_LED_CTL_OFF);
1437     if (err)
1438         return err;
1439 
1440     err = mdio_read(np, np->phy_addr, MRVL88X2011_USER_DEV3_ADDR,
1441             MRVL88X2011_GENERAL_CTL);
1442     if (err < 0)
1443         return err;
1444 
1445     err |= MRVL88X2011_ENA_XFPREFCLK;
1446 
1447     err = mdio_write(np, np->phy_addr, MRVL88X2011_USER_DEV3_ADDR,
1448              MRVL88X2011_GENERAL_CTL, err);
1449     if (err < 0)
1450         return err;
1451 
1452     err = mdio_read(np, np->phy_addr, MRVL88X2011_USER_DEV1_ADDR,
1453             MRVL88X2011_PMA_PMD_CTL_1);
1454     if (err < 0)
1455         return err;
1456 
1457     if (np->link_config.loopback_mode == LOOPBACK_MAC)
1458         err |= MRVL88X2011_LOOPBACK;
1459     else
1460         err &= ~MRVL88X2011_LOOPBACK;
1461 
1462     err = mdio_write(np, np->phy_addr, MRVL88X2011_USER_DEV1_ADDR,
1463              MRVL88X2011_PMA_PMD_CTL_1, err);
1464     if (err < 0)
1465         return err;
1466 
1467     /* Enable PMD  */
1468     return mdio_write(np, np->phy_addr, MRVL88X2011_USER_DEV1_ADDR,
1469               MRVL88X2011_10G_PMD_TX_DIS, MRVL88X2011_ENA_PMDTX);
1470 }
1471 
1472 
1473 static int xcvr_diag_bcm870x(struct niu *np)
1474 {
1475     u16 analog_stat0, tx_alarm_status;
1476     int err = 0;
1477 
1478 #if 1
1479     err = mdio_read(np, np->phy_addr, BCM8704_PMA_PMD_DEV_ADDR,
1480             MII_STAT1000);
1481     if (err < 0)
1482         return err;
1483     pr_info("Port %u PMA_PMD(MII_STAT1000) [%04x]\n", np->port, err);
1484 
1485     err = mdio_read(np, np->phy_addr, BCM8704_USER_DEV3_ADDR, 0x20);
1486     if (err < 0)
1487         return err;
1488     pr_info("Port %u USER_DEV3(0x20) [%04x]\n", np->port, err);
1489 
1490     err = mdio_read(np, np->phy_addr, BCM8704_PHYXS_DEV_ADDR,
1491             MII_NWAYTEST);
1492     if (err < 0)
1493         return err;
1494     pr_info("Port %u PHYXS(MII_NWAYTEST) [%04x]\n", np->port, err);
1495 #endif
1496 
1497     /* XXX dig this out it might not be so useful XXX */
1498     err = mdio_read(np, np->phy_addr, BCM8704_USER_DEV3_ADDR,
1499             BCM8704_USER_ANALOG_STATUS0);
1500     if (err < 0)
1501         return err;
1502     err = mdio_read(np, np->phy_addr, BCM8704_USER_DEV3_ADDR,
1503             BCM8704_USER_ANALOG_STATUS0);
1504     if (err < 0)
1505         return err;
1506     analog_stat0 = err;
1507 
1508     err = mdio_read(np, np->phy_addr, BCM8704_USER_DEV3_ADDR,
1509             BCM8704_USER_TX_ALARM_STATUS);
1510     if (err < 0)
1511         return err;
1512     err = mdio_read(np, np->phy_addr, BCM8704_USER_DEV3_ADDR,
1513             BCM8704_USER_TX_ALARM_STATUS);
1514     if (err < 0)
1515         return err;
1516     tx_alarm_status = err;
1517 
1518     if (analog_stat0 != 0x03fc) {
1519         if ((analog_stat0 == 0x43bc) && (tx_alarm_status != 0)) {
1520             pr_info("Port %u cable not connected or bad cable\n",
1521                 np->port);
1522         } else if (analog_stat0 == 0x639c) {
1523             pr_info("Port %u optical module is bad or missing\n",
1524                 np->port);
1525         }
1526     }
1527 
1528     return 0;
1529 }
1530 
1531 static int xcvr_10g_set_lb_bcm870x(struct niu *np)
1532 {
1533     struct niu_link_config *lp = &np->link_config;
1534     int err;
1535 
1536     err = mdio_read(np, np->phy_addr, BCM8704_PCS_DEV_ADDR,
1537             MII_BMCR);
1538     if (err < 0)
1539         return err;
1540 
1541     err &= ~BMCR_LOOPBACK;
1542 
1543     if (lp->loopback_mode == LOOPBACK_MAC)
1544         err |= BMCR_LOOPBACK;
1545 
1546     err = mdio_write(np, np->phy_addr, BCM8704_PCS_DEV_ADDR,
1547              MII_BMCR, err);
1548     if (err)
1549         return err;
1550 
1551     return 0;
1552 }
1553 
1554 static int xcvr_init_10g_bcm8706(struct niu *np)
1555 {
1556     int err = 0;
1557     u64 val;
1558 
1559     if ((np->flags & NIU_FLAGS_HOTPLUG_PHY) &&
1560         (np->flags & NIU_FLAGS_HOTPLUG_PHY_PRESENT) == 0)
1561             return err;
1562 
1563     val = nr64_mac(XMAC_CONFIG);
1564     val &= ~XMAC_CONFIG_LED_POLARITY;
1565     val |= XMAC_CONFIG_FORCE_LED_ON;
1566     nw64_mac(XMAC_CONFIG, val);
1567 
1568     val = nr64(MIF_CONFIG);
1569     val |= MIF_CONFIG_INDIRECT_MODE;
1570     nw64(MIF_CONFIG, val);
1571 
1572     err = bcm8704_reset(np);
1573     if (err)
1574         return err;
1575 
1576     err = xcvr_10g_set_lb_bcm870x(np);
1577     if (err)
1578         return err;
1579 
1580     err = bcm8706_init_user_dev3(np);
1581     if (err)
1582         return err;
1583 
1584     err = xcvr_diag_bcm870x(np);
1585     if (err)
1586         return err;
1587 
1588     return 0;
1589 }
1590 
1591 static int xcvr_init_10g_bcm8704(struct niu *np)
1592 {
1593     int err;
1594 
1595     err = bcm8704_reset(np);
1596     if (err)
1597         return err;
1598 
1599     err = bcm8704_init_user_dev3(np);
1600     if (err)
1601         return err;
1602 
1603     err = xcvr_10g_set_lb_bcm870x(np);
1604     if (err)
1605         return err;
1606 
1607     err =  xcvr_diag_bcm870x(np);
1608     if (err)
1609         return err;
1610 
1611     return 0;
1612 }
1613 
1614 static int xcvr_init_10g(struct niu *np)
1615 {
1616     int phy_id, err;
1617     u64 val;
1618 
1619     val = nr64_mac(XMAC_CONFIG);
1620     val &= ~XMAC_CONFIG_LED_POLARITY;
1621     val |= XMAC_CONFIG_FORCE_LED_ON;
1622     nw64_mac(XMAC_CONFIG, val);
1623 
1624     /* XXX shared resource, lock parent XXX */
1625     val = nr64(MIF_CONFIG);
1626     val |= MIF_CONFIG_INDIRECT_MODE;
1627     nw64(MIF_CONFIG, val);
1628 
1629     phy_id = phy_decode(np->parent->port_phy, np->port);
1630     phy_id = np->parent->phy_probe_info.phy_id[phy_id][np->port];
1631 
1632     /* handle different phy types */
1633     switch (phy_id & NIU_PHY_ID_MASK) {
1634     case NIU_PHY_ID_MRVL88X2011:
1635         err = xcvr_init_10g_mrvl88x2011(np);
1636         break;
1637 
1638     default: /* bcom 8704 */
1639         err = xcvr_init_10g_bcm8704(np);
1640         break;
1641     }
1642 
1643     return err;
1644 }
1645 
1646 static int mii_reset(struct niu *np)
1647 {
1648     int limit, err;
1649 
1650     err = mii_write(np, np->phy_addr, MII_BMCR, BMCR_RESET);
1651     if (err)
1652         return err;
1653 
1654     limit = 1000;
1655     while (--limit >= 0) {
1656         udelay(500);
1657         err = mii_read(np, np->phy_addr, MII_BMCR);
1658         if (err < 0)
1659             return err;
1660         if (!(err & BMCR_RESET))
1661             break;
1662     }
1663     if (limit < 0) {
1664         netdev_err(np->dev, "Port %u MII would not reset, bmcr[%04x]\n",
1665                np->port, err);
1666         return -ENODEV;
1667     }
1668 
1669     return 0;
1670 }
1671 
1672 static int xcvr_init_1g_rgmii(struct niu *np)
1673 {
1674     int err;
1675     u64 val;
1676     u16 bmcr, bmsr, estat;
1677 
1678     val = nr64(MIF_CONFIG);
1679     val &= ~MIF_CONFIG_INDIRECT_MODE;
1680     nw64(MIF_CONFIG, val);
1681 
1682     err = mii_reset(np);
1683     if (err)
1684         return err;
1685 
1686     err = mii_read(np, np->phy_addr, MII_BMSR);
1687     if (err < 0)
1688         return err;
1689     bmsr = err;
1690 
1691     estat = 0;
1692     if (bmsr & BMSR_ESTATEN) {
1693         err = mii_read(np, np->phy_addr, MII_ESTATUS);
1694         if (err < 0)
1695             return err;
1696         estat = err;
1697     }
1698 
1699     bmcr = 0;
1700     err = mii_write(np, np->phy_addr, MII_BMCR, bmcr);
1701     if (err)
1702         return err;
1703 
1704     if (bmsr & BMSR_ESTATEN) {
1705         u16 ctrl1000 = 0;
1706 
1707         if (estat & ESTATUS_1000_TFULL)
1708             ctrl1000 |= ADVERTISE_1000FULL;
1709         err = mii_write(np, np->phy_addr, MII_CTRL1000, ctrl1000);
1710         if (err)
1711             return err;
1712     }
1713 
1714     bmcr = (BMCR_SPEED1000 | BMCR_FULLDPLX);
1715 
1716     err = mii_write(np, np->phy_addr, MII_BMCR, bmcr);
1717     if (err)
1718         return err;
1719 
1720     err = mii_read(np, np->phy_addr, MII_BMCR);
1721     if (err < 0)
1722         return err;
1723     bmcr = mii_read(np, np->phy_addr, MII_BMCR);
1724 
1725     err = mii_read(np, np->phy_addr, MII_BMSR);
1726     if (err < 0)
1727         return err;
1728 
1729     return 0;
1730 }
1731 
1732 static int mii_init_common(struct niu *np)
1733 {
1734     struct niu_link_config *lp = &np->link_config;
1735     u16 bmcr, bmsr, adv, estat;
1736     int err;
1737 
1738     err = mii_reset(np);
1739     if (err)
1740         return err;
1741 
1742     err = mii_read(np, np->phy_addr, MII_BMSR);
1743     if (err < 0)
1744         return err;
1745     bmsr = err;
1746 
1747     estat = 0;
1748     if (bmsr & BMSR_ESTATEN) {
1749         err = mii_read(np, np->phy_addr, MII_ESTATUS);
1750         if (err < 0)
1751             return err;
1752         estat = err;
1753     }
1754 
1755     bmcr = 0;
1756     err = mii_write(np, np->phy_addr, MII_BMCR, bmcr);
1757     if (err)
1758         return err;
1759 
1760     if (lp->loopback_mode == LOOPBACK_MAC) {
1761         bmcr |= BMCR_LOOPBACK;
1762         if (lp->active_speed == SPEED_1000)
1763             bmcr |= BMCR_SPEED1000;
1764         if (lp->active_duplex == DUPLEX_FULL)
1765             bmcr |= BMCR_FULLDPLX;
1766     }
1767 
1768     if (lp->loopback_mode == LOOPBACK_PHY) {
1769         u16 aux;
1770 
1771         aux = (BCM5464R_AUX_CTL_EXT_LB |
1772                BCM5464R_AUX_CTL_WRITE_1);
1773         err = mii_write(np, np->phy_addr, BCM5464R_AUX_CTL, aux);
1774         if (err)
1775             return err;
1776     }
1777 
1778     if (lp->autoneg) {
1779         u16 ctrl1000;
1780 
1781         adv = ADVERTISE_CSMA | ADVERTISE_PAUSE_CAP;
1782         if ((bmsr & BMSR_10HALF) &&
1783             (lp->advertising & ADVERTISED_10baseT_Half))
1784             adv |= ADVERTISE_10HALF;
1785         if ((bmsr & BMSR_10FULL) &&
1786             (lp->advertising & ADVERTISED_10baseT_Full))
1787             adv |= ADVERTISE_10FULL;
1788         if ((bmsr & BMSR_100HALF) &&
1789             (lp->advertising & ADVERTISED_100baseT_Half))
1790             adv |= ADVERTISE_100HALF;
1791         if ((bmsr & BMSR_100FULL) &&
1792             (lp->advertising & ADVERTISED_100baseT_Full))
1793             adv |= ADVERTISE_100FULL;
1794         err = mii_write(np, np->phy_addr, MII_ADVERTISE, adv);
1795         if (err)
1796             return err;
1797 
1798         if (likely(bmsr & BMSR_ESTATEN)) {
1799             ctrl1000 = 0;
1800             if ((estat & ESTATUS_1000_THALF) &&
1801                 (lp->advertising & ADVERTISED_1000baseT_Half))
1802                 ctrl1000 |= ADVERTISE_1000HALF;
1803             if ((estat & ESTATUS_1000_TFULL) &&
1804                 (lp->advertising & ADVERTISED_1000baseT_Full))
1805                 ctrl1000 |= ADVERTISE_1000FULL;
1806             err = mii_write(np, np->phy_addr,
1807                     MII_CTRL1000, ctrl1000);
1808             if (err)
1809                 return err;
1810         }
1811 
1812         bmcr |= (BMCR_ANENABLE | BMCR_ANRESTART);
1813     } else {
1814         /* !lp->autoneg */
1815         int fulldpx;
1816 
1817         if (lp->duplex == DUPLEX_FULL) {
1818             bmcr |= BMCR_FULLDPLX;
1819             fulldpx = 1;
1820         } else if (lp->duplex == DUPLEX_HALF)
1821             fulldpx = 0;
1822         else
1823             return -EINVAL;
1824 
1825         if (lp->speed == SPEED_1000) {
1826             /* if X-full requested while not supported, or
1827                X-half requested while not supported... */
1828             if ((fulldpx && !(estat & ESTATUS_1000_TFULL)) ||
1829                 (!fulldpx && !(estat & ESTATUS_1000_THALF)))
1830                 return -EINVAL;
1831             bmcr |= BMCR_SPEED1000;
1832         } else if (lp->speed == SPEED_100) {
1833             if ((fulldpx && !(bmsr & BMSR_100FULL)) ||
1834                 (!fulldpx && !(bmsr & BMSR_100HALF)))
1835                 return -EINVAL;
1836             bmcr |= BMCR_SPEED100;
1837         } else if (lp->speed == SPEED_10) {
1838             if ((fulldpx && !(bmsr & BMSR_10FULL)) ||
1839                 (!fulldpx && !(bmsr & BMSR_10HALF)))
1840                 return -EINVAL;
1841         } else
1842             return -EINVAL;
1843     }
1844 
1845     err = mii_write(np, np->phy_addr, MII_BMCR, bmcr);
1846     if (err)
1847         return err;
1848 
1849 #if 0
1850     err = mii_read(np, np->phy_addr, MII_BMCR);
1851     if (err < 0)
1852         return err;
1853     bmcr = err;
1854 
1855     err = mii_read(np, np->phy_addr, MII_BMSR);
1856     if (err < 0)
1857         return err;
1858     bmsr = err;
1859 
1860     pr_info("Port %u after MII init bmcr[%04x] bmsr[%04x]\n",
1861         np->port, bmcr, bmsr);
1862 #endif
1863 
1864     return 0;
1865 }
1866 
1867 static int xcvr_init_1g(struct niu *np)
1868 {
1869     u64 val;
1870 
1871     /* XXX shared resource, lock parent XXX */
1872     val = nr64(MIF_CONFIG);
1873     val &= ~MIF_CONFIG_INDIRECT_MODE;
1874     nw64(MIF_CONFIG, val);
1875 
1876     return mii_init_common(np);
1877 }
1878 
1879 static int niu_xcvr_init(struct niu *np)
1880 {
1881     const struct niu_phy_ops *ops = np->phy_ops;
1882     int err;
1883 
1884     err = 0;
1885     if (ops->xcvr_init)
1886         err = ops->xcvr_init(np);
1887 
1888     return err;
1889 }
1890 
1891 static int niu_serdes_init(struct niu *np)
1892 {
1893     const struct niu_phy_ops *ops = np->phy_ops;
1894     int err;
1895 
1896     err = 0;
1897     if (ops->serdes_init)
1898         err = ops->serdes_init(np);
1899 
1900     return err;
1901 }
1902 
1903 static void niu_init_xif(struct niu *);
1904 static void niu_handle_led(struct niu *, int status);
1905 
1906 static int niu_link_status_common(struct niu *np, int link_up)
1907 {
1908     struct niu_link_config *lp = &np->link_config;
1909     struct net_device *dev = np->dev;
1910     unsigned long flags;
1911 
1912     if (!netif_carrier_ok(dev) && link_up) {
1913         netif_info(np, link, dev, "Link is up at %s, %s duplex\n",
1914                lp->active_speed == SPEED_10000 ? "10Gb/sec" :
1915                lp->active_speed == SPEED_1000 ? "1Gb/sec" :
1916                lp->active_speed == SPEED_100 ? "100Mbit/sec" :
1917                "10Mbit/sec",
1918                lp->active_duplex == DUPLEX_FULL ? "full" : "half");
1919 
1920         spin_lock_irqsave(&np->lock, flags);
1921         niu_init_xif(np);
1922         niu_handle_led(np, 1);
1923         spin_unlock_irqrestore(&np->lock, flags);
1924 
1925         netif_carrier_on(dev);
1926     } else if (netif_carrier_ok(dev) && !link_up) {
1927         netif_warn(np, link, dev, "Link is down\n");
1928         spin_lock_irqsave(&np->lock, flags);
1929         niu_handle_led(np, 0);
1930         spin_unlock_irqrestore(&np->lock, flags);
1931         netif_carrier_off(dev);
1932     }
1933 
1934     return 0;
1935 }
1936 
1937 static int link_status_10g_mrvl(struct niu *np, int *link_up_p)
1938 {
1939     int err, link_up, pma_status, pcs_status;
1940 
1941     link_up = 0;
1942 
1943     err = mdio_read(np, np->phy_addr, MRVL88X2011_USER_DEV1_ADDR,
1944             MRVL88X2011_10G_PMD_STATUS_2);
1945     if (err < 0)
1946         goto out;
1947 
1948     /* Check PMA/PMD Register: 1.0001.2 == 1 */
1949     err = mdio_read(np, np->phy_addr, MRVL88X2011_USER_DEV1_ADDR,
1950             MRVL88X2011_PMA_PMD_STATUS_1);
1951     if (err < 0)
1952         goto out;
1953 
1954     pma_status = ((err & MRVL88X2011_LNK_STATUS_OK) ? 1 : 0);
1955 
1956         /* Check PMC Register : 3.0001.2 == 1: read twice */
1957     err = mdio_read(np, np->phy_addr, MRVL88X2011_USER_DEV3_ADDR,
1958             MRVL88X2011_PMA_PMD_STATUS_1);
1959     if (err < 0)
1960         goto out;
1961 
1962     err = mdio_read(np, np->phy_addr, MRVL88X2011_USER_DEV3_ADDR,
1963             MRVL88X2011_PMA_PMD_STATUS_1);
1964     if (err < 0)
1965         goto out;
1966 
1967     pcs_status = ((err & MRVL88X2011_LNK_STATUS_OK) ? 1 : 0);
1968 
1969         /* Check XGXS Register : 4.0018.[0-3,12] */
1970     err = mdio_read(np, np->phy_addr, MRVL88X2011_USER_DEV4_ADDR,
1971             MRVL88X2011_10G_XGXS_LANE_STAT);
1972     if (err < 0)
1973         goto out;
1974 
1975     if (err == (PHYXS_XGXS_LANE_STAT_ALINGED | PHYXS_XGXS_LANE_STAT_LANE3 |
1976             PHYXS_XGXS_LANE_STAT_LANE2 | PHYXS_XGXS_LANE_STAT_LANE1 |
1977             PHYXS_XGXS_LANE_STAT_LANE0 | PHYXS_XGXS_LANE_STAT_MAGIC |
1978             0x800))
1979         link_up = (pma_status && pcs_status) ? 1 : 0;
1980 
1981     np->link_config.active_speed = SPEED_10000;
1982     np->link_config.active_duplex = DUPLEX_FULL;
1983     err = 0;
1984 out:
1985     mrvl88x2011_act_led(np, (link_up ?
1986                  MRVL88X2011_LED_CTL_PCS_ACT :
1987                  MRVL88X2011_LED_CTL_OFF));
1988 
1989     *link_up_p = link_up;
1990     return err;
1991 }
1992 
1993 static int link_status_10g_bcm8706(struct niu *np, int *link_up_p)
1994 {
1995     int err, link_up;
1996     link_up = 0;
1997 
1998     err = mdio_read(np, np->phy_addr, BCM8704_PMA_PMD_DEV_ADDR,
1999             BCM8704_PMD_RCV_SIGDET);
2000     if (err < 0 || err == 0xffff)
2001         goto out;
2002     if (!(err & PMD_RCV_SIGDET_GLOBAL)) {
2003         err = 0;
2004         goto out;
2005     }
2006 
2007     err = mdio_read(np, np->phy_addr, BCM8704_PCS_DEV_ADDR,
2008             BCM8704_PCS_10G_R_STATUS);
2009     if (err < 0)
2010         goto out;
2011 
2012     if (!(err & PCS_10G_R_STATUS_BLK_LOCK)) {
2013         err = 0;
2014         goto out;
2015     }
2016 
2017     err = mdio_read(np, np->phy_addr, BCM8704_PHYXS_DEV_ADDR,
2018             BCM8704_PHYXS_XGXS_LANE_STAT);
2019     if (err < 0)
2020         goto out;
2021     if (err != (PHYXS_XGXS_LANE_STAT_ALINGED |
2022             PHYXS_XGXS_LANE_STAT_MAGIC |
2023             PHYXS_XGXS_LANE_STAT_PATTEST |
2024             PHYXS_XGXS_LANE_STAT_LANE3 |
2025             PHYXS_XGXS_LANE_STAT_LANE2 |
2026             PHYXS_XGXS_LANE_STAT_LANE1 |
2027             PHYXS_XGXS_LANE_STAT_LANE0)) {
2028         err = 0;
2029         np->link_config.active_speed = SPEED_INVALID;
2030         np->link_config.active_duplex = DUPLEX_INVALID;
2031         goto out;
2032     }
2033 
2034     link_up = 1;
2035     np->link_config.active_speed = SPEED_10000;
2036     np->link_config.active_duplex = DUPLEX_FULL;
2037     err = 0;
2038 
2039 out:
2040     *link_up_p = link_up;
2041     return err;
2042 }
2043 
2044 static int link_status_10g_bcom(struct niu *np, int *link_up_p)
2045 {
2046     int err, link_up;
2047 
2048     link_up = 0;
2049 
2050     err = mdio_read(np, np->phy_addr, BCM8704_PMA_PMD_DEV_ADDR,
2051             BCM8704_PMD_RCV_SIGDET);
2052     if (err < 0)
2053         goto out;
2054     if (!(err & PMD_RCV_SIGDET_GLOBAL)) {
2055         err = 0;
2056         goto out;
2057     }
2058 
2059     err = mdio_read(np, np->phy_addr, BCM8704_PCS_DEV_ADDR,
2060             BCM8704_PCS_10G_R_STATUS);
2061     if (err < 0)
2062         goto out;
2063     if (!(err & PCS_10G_R_STATUS_BLK_LOCK)) {
2064         err = 0;
2065         goto out;
2066     }
2067 
2068     err = mdio_read(np, np->phy_addr, BCM8704_PHYXS_DEV_ADDR,
2069             BCM8704_PHYXS_XGXS_LANE_STAT);
2070     if (err < 0)
2071         goto out;
2072 
2073     if (err != (PHYXS_XGXS_LANE_STAT_ALINGED |
2074             PHYXS_XGXS_LANE_STAT_MAGIC |
2075             PHYXS_XGXS_LANE_STAT_LANE3 |
2076             PHYXS_XGXS_LANE_STAT_LANE2 |
2077             PHYXS_XGXS_LANE_STAT_LANE1 |
2078             PHYXS_XGXS_LANE_STAT_LANE0)) {
2079         err = 0;
2080         goto out;
2081     }
2082 
2083     link_up = 1;
2084     np->link_config.active_speed = SPEED_10000;
2085     np->link_config.active_duplex = DUPLEX_FULL;
2086     err = 0;
2087 
2088 out:
2089     *link_up_p = link_up;
2090     return err;
2091 }
2092 
2093 static int link_status_10g(struct niu *np, int *link_up_p)
2094 {
2095     unsigned long flags;
2096     int err = -EINVAL;
2097 
2098     spin_lock_irqsave(&np->lock, flags);
2099 
2100     if (np->link_config.loopback_mode == LOOPBACK_DISABLED) {
2101         int phy_id;
2102 
2103         phy_id = phy_decode(np->parent->port_phy, np->port);
2104         phy_id = np->parent->phy_probe_info.phy_id[phy_id][np->port];
2105 
2106         /* handle different phy types */
2107         switch (phy_id & NIU_PHY_ID_MASK) {
2108         case NIU_PHY_ID_MRVL88X2011:
2109             err = link_status_10g_mrvl(np, link_up_p);
2110             break;
2111 
2112         default: /* bcom 8704 */
2113             err = link_status_10g_bcom(np, link_up_p);
2114             break;
2115         }
2116     }
2117 
2118     spin_unlock_irqrestore(&np->lock, flags);
2119 
2120     return err;
2121 }
2122 
2123 static int niu_10g_phy_present(struct niu *np)
2124 {
2125     u64 sig, mask, val;
2126 
2127     sig = nr64(ESR_INT_SIGNALS);
2128     switch (np->port) {
2129     case 0:
2130         mask = ESR_INT_SIGNALS_P0_BITS;
2131         val = (ESR_INT_SRDY0_P0 |
2132                ESR_INT_DET0_P0 |
2133                ESR_INT_XSRDY_P0 |
2134                ESR_INT_XDP_P0_CH3 |
2135                ESR_INT_XDP_P0_CH2 |
2136                ESR_INT_XDP_P0_CH1 |
2137                ESR_INT_XDP_P0_CH0);
2138         break;
2139 
2140     case 1:
2141         mask = ESR_INT_SIGNALS_P1_BITS;
2142         val = (ESR_INT_SRDY0_P1 |
2143                ESR_INT_DET0_P1 |
2144                ESR_INT_XSRDY_P1 |
2145                ESR_INT_XDP_P1_CH3 |
2146                ESR_INT_XDP_P1_CH2 |
2147                ESR_INT_XDP_P1_CH1 |
2148                ESR_INT_XDP_P1_CH0);
2149         break;
2150 
2151     default:
2152         return 0;
2153     }
2154 
2155     if ((sig & mask) != val)
2156         return 0;
2157     return 1;
2158 }
2159 
2160 static int link_status_10g_hotplug(struct niu *np, int *link_up_p)
2161 {
2162     unsigned long flags;
2163     int err = 0;
2164     int phy_present;
2165     int phy_present_prev;
2166 
2167     spin_lock_irqsave(&np->lock, flags);
2168 
2169     if (np->link_config.loopback_mode == LOOPBACK_DISABLED) {
2170         phy_present_prev = (np->flags & NIU_FLAGS_HOTPLUG_PHY_PRESENT) ?
2171             1 : 0;
2172         phy_present = niu_10g_phy_present(np);
2173         if (phy_present != phy_present_prev) {
2174             /* state change */
2175             if (phy_present) {
2176                 /* A NEM was just plugged in */
2177                 np->flags |= NIU_FLAGS_HOTPLUG_PHY_PRESENT;
2178                 if (np->phy_ops->xcvr_init)
2179                     err = np->phy_ops->xcvr_init(np);
2180                 if (err) {
2181                     err = mdio_read(np, np->phy_addr,
2182                         BCM8704_PHYXS_DEV_ADDR, MII_BMCR);
2183                     if (err == 0xffff) {
2184                         /* No mdio, back-to-back XAUI */
2185                         goto out;
2186                     }
2187                     /* debounce */
2188                     np->flags &= ~NIU_FLAGS_HOTPLUG_PHY_PRESENT;
2189                 }
2190             } else {
2191                 np->flags &= ~NIU_FLAGS_HOTPLUG_PHY_PRESENT;
2192                 *link_up_p = 0;
2193                 netif_warn(np, link, np->dev,
2194                        "Hotplug PHY Removed\n");
2195             }
2196         }
2197 out:
2198         if (np->flags & NIU_FLAGS_HOTPLUG_PHY_PRESENT) {
2199             err = link_status_10g_bcm8706(np, link_up_p);
2200             if (err == 0xffff) {
2201                 /* No mdio, back-to-back XAUI: it is C10NEM */
2202                 *link_up_p = 1;
2203                 np->link_config.active_speed = SPEED_10000;
2204                 np->link_config.active_duplex = DUPLEX_FULL;
2205             }
2206         }
2207     }
2208 
2209     spin_unlock_irqrestore(&np->lock, flags);
2210 
2211     return 0;
2212 }
2213 
2214 static int niu_link_status(struct niu *np, int *link_up_p)
2215 {
2216     const struct niu_phy_ops *ops = np->phy_ops;
2217     int err;
2218 
2219     err = 0;
2220     if (ops->link_status)
2221         err = ops->link_status(np, link_up_p);
2222 
2223     return err;
2224 }
2225 
2226 static void niu_timer(struct timer_list *t)
2227 {
2228     struct niu *np = from_timer(np, t, timer);
2229     unsigned long off;
2230     int err, link_up;
2231 
2232     err = niu_link_status(np, &link_up);
2233     if (!err)
2234         niu_link_status_common(np, link_up);
2235 
2236     if (netif_carrier_ok(np->dev))
2237         off = 5 * HZ;
2238     else
2239         off = 1 * HZ;
2240     np->timer.expires = jiffies + off;
2241 
2242     add_timer(&np->timer);
2243 }
2244 
2245 static const struct niu_phy_ops phy_ops_10g_serdes = {
2246     .serdes_init        = serdes_init_10g_serdes,
2247     .link_status        = link_status_10g_serdes,
2248 };
2249 
2250 static const struct niu_phy_ops phy_ops_10g_serdes_niu = {
2251     .serdes_init        = serdes_init_niu_10g_serdes,
2252     .link_status        = link_status_10g_serdes,
2253 };
2254 
2255 static const struct niu_phy_ops phy_ops_1g_serdes_niu = {
2256     .serdes_init        = serdes_init_niu_1g_serdes,
2257     .link_status        = link_status_1g_serdes,
2258 };
2259 
2260 static const struct niu_phy_ops phy_ops_1g_rgmii = {
2261     .xcvr_init      = xcvr_init_1g_rgmii,
2262     .link_status        = link_status_1g_rgmii,
2263 };
2264 
2265 static const struct niu_phy_ops phy_ops_10g_fiber_niu = {
2266     .serdes_init        = serdes_init_niu_10g_fiber,
2267     .xcvr_init      = xcvr_init_10g,
2268     .link_status        = link_status_10g,
2269 };
2270 
2271 static const struct niu_phy_ops phy_ops_10g_fiber = {
2272     .serdes_init        = serdes_init_10g,
2273     .xcvr_init      = xcvr_init_10g,
2274     .link_status        = link_status_10g,
2275 };
2276 
2277 static const struct niu_phy_ops phy_ops_10g_fiber_hotplug = {
2278     .serdes_init        = serdes_init_10g,
2279     .xcvr_init      = xcvr_init_10g_bcm8706,
2280     .link_status        = link_status_10g_hotplug,
2281 };
2282 
2283 static const struct niu_phy_ops phy_ops_niu_10g_hotplug = {
2284     .serdes_init        = serdes_init_niu_10g_fiber,
2285     .xcvr_init      = xcvr_init_10g_bcm8706,
2286     .link_status        = link_status_10g_hotplug,
2287 };
2288 
2289 static const struct niu_phy_ops phy_ops_10g_copper = {
2290     .serdes_init        = serdes_init_10g,
2291     .link_status        = link_status_10g, /* XXX */
2292 };
2293 
2294 static const struct niu_phy_ops phy_ops_1g_fiber = {
2295     .serdes_init        = serdes_init_1g,
2296     .xcvr_init      = xcvr_init_1g,
2297     .link_status        = link_status_1g,
2298 };
2299 
2300 static const struct niu_phy_ops phy_ops_1g_copper = {
2301     .xcvr_init      = xcvr_init_1g,
2302     .link_status        = link_status_1g,
2303 };
2304 
2305 struct niu_phy_template {
2306     const struct niu_phy_ops    *ops;
2307     u32             phy_addr_base;
2308 };
2309 
2310 static const struct niu_phy_template phy_template_niu_10g_fiber = {
2311     .ops        = &phy_ops_10g_fiber_niu,
2312     .phy_addr_base  = 16,
2313 };
2314 
2315 static const struct niu_phy_template phy_template_niu_10g_serdes = {
2316     .ops        = &phy_ops_10g_serdes_niu,
2317     .phy_addr_base  = 0,
2318 };
2319 
2320 static const struct niu_phy_template phy_template_niu_1g_serdes = {
2321     .ops        = &phy_ops_1g_serdes_niu,
2322     .phy_addr_base  = 0,
2323 };
2324 
2325 static const struct niu_phy_template phy_template_10g_fiber = {
2326     .ops        = &phy_ops_10g_fiber,
2327     .phy_addr_base  = 8,
2328 };
2329 
2330 static const struct niu_phy_template phy_template_10g_fiber_hotplug = {
2331     .ops        = &phy_ops_10g_fiber_hotplug,
2332     .phy_addr_base  = 8,
2333 };
2334 
2335 static const struct niu_phy_template phy_template_niu_10g_hotplug = {
2336     .ops        = &phy_ops_niu_10g_hotplug,
2337     .phy_addr_base  = 8,
2338 };
2339 
2340 static const struct niu_phy_template phy_template_10g_copper = {
2341     .ops        = &phy_ops_10g_copper,
2342     .phy_addr_base  = 10,
2343 };
2344 
2345 static const struct niu_phy_template phy_template_1g_fiber = {
2346     .ops        = &phy_ops_1g_fiber,
2347     .phy_addr_base  = 0,
2348 };
2349 
2350 static const struct niu_phy_template phy_template_1g_copper = {
2351     .ops        = &phy_ops_1g_copper,
2352     .phy_addr_base  = 0,
2353 };
2354 
2355 static const struct niu_phy_template phy_template_1g_rgmii = {
2356     .ops        = &phy_ops_1g_rgmii,
2357     .phy_addr_base  = 0,
2358 };
2359 
2360 static const struct niu_phy_template phy_template_10g_serdes = {
2361     .ops        = &phy_ops_10g_serdes,
2362     .phy_addr_base  = 0,
2363 };
2364 
2365 static int niu_atca_port_num[4] = {
2366     0, 0,  11, 10
2367 };
2368 
2369 static int serdes_init_10g_serdes(struct niu *np)
2370 {
2371     struct niu_link_config *lp = &np->link_config;
2372     unsigned long ctrl_reg, test_cfg_reg, pll_cfg, i;
2373     u64 ctrl_val, test_cfg_val, sig, mask, val;
2374 
2375     switch (np->port) {
2376     case 0:
2377         ctrl_reg = ENET_SERDES_0_CTRL_CFG;
2378         test_cfg_reg = ENET_SERDES_0_TEST_CFG;
2379         pll_cfg = ENET_SERDES_0_PLL_CFG;
2380         break;
2381     case 1:
2382         ctrl_reg = ENET_SERDES_1_CTRL_CFG;
2383         test_cfg_reg = ENET_SERDES_1_TEST_CFG;
2384         pll_cfg = ENET_SERDES_1_PLL_CFG;
2385         break;
2386 
2387     default:
2388         return -EINVAL;
2389     }
2390     ctrl_val = (ENET_SERDES_CTRL_SDET_0 |
2391             ENET_SERDES_CTRL_SDET_1 |
2392             ENET_SERDES_CTRL_SDET_2 |
2393             ENET_SERDES_CTRL_SDET_3 |
2394             (0x5 << ENET_SERDES_CTRL_EMPH_0_SHIFT) |
2395             (0x5 << ENET_SERDES_CTRL_EMPH_1_SHIFT) |
2396             (0x5 << ENET_SERDES_CTRL_EMPH_2_SHIFT) |
2397             (0x5 << ENET_SERDES_CTRL_EMPH_3_SHIFT) |
2398             (0x1 << ENET_SERDES_CTRL_LADJ_0_SHIFT) |
2399             (0x1 << ENET_SERDES_CTRL_LADJ_1_SHIFT) |
2400             (0x1 << ENET_SERDES_CTRL_LADJ_2_SHIFT) |
2401             (0x1 << ENET_SERDES_CTRL_LADJ_3_SHIFT));
2402     test_cfg_val = 0;
2403 
2404     if (lp->loopback_mode == LOOPBACK_PHY) {
2405         test_cfg_val |= ((ENET_TEST_MD_PAD_LOOPBACK <<
2406                   ENET_SERDES_TEST_MD_0_SHIFT) |
2407                  (ENET_TEST_MD_PAD_LOOPBACK <<
2408                   ENET_SERDES_TEST_MD_1_SHIFT) |
2409                  (ENET_TEST_MD_PAD_LOOPBACK <<
2410                   ENET_SERDES_TEST_MD_2_SHIFT) |
2411                  (ENET_TEST_MD_PAD_LOOPBACK <<
2412                   ENET_SERDES_TEST_MD_3_SHIFT));
2413     }
2414 
2415     esr_reset(np);
2416     nw64(pll_cfg, ENET_SERDES_PLL_FBDIV2);
2417     nw64(ctrl_reg, ctrl_val);
2418     nw64(test_cfg_reg, test_cfg_val);
2419 
2420     /* Initialize all 4 lanes of the SERDES.  */
2421     for (i = 0; i < 4; i++) {
2422         u32 rxtx_ctrl, glue0;
2423         int err;
2424 
2425         err = esr_read_rxtx_ctrl(np, i, &rxtx_ctrl);
2426         if (err)
2427             return err;
2428         err = esr_read_glue0(np, i, &glue0);
2429         if (err)
2430             return err;
2431 
2432         rxtx_ctrl &= ~(ESR_RXTX_CTRL_VMUXLO);
2433         rxtx_ctrl |= (ESR_RXTX_CTRL_ENSTRETCH |
2434                   (2 << ESR_RXTX_CTRL_VMUXLO_SHIFT));
2435 
2436         glue0 &= ~(ESR_GLUE_CTRL0_SRATE |
2437                ESR_GLUE_CTRL0_THCNT |
2438                ESR_GLUE_CTRL0_BLTIME);
2439         glue0 |= (ESR_GLUE_CTRL0_RXLOSENAB |
2440               (0xf << ESR_GLUE_CTRL0_SRATE_SHIFT) |
2441               (0xff << ESR_GLUE_CTRL0_THCNT_SHIFT) |
2442               (BLTIME_300_CYCLES <<
2443                ESR_GLUE_CTRL0_BLTIME_SHIFT));
2444 
2445         err = esr_write_rxtx_ctrl(np, i, rxtx_ctrl);
2446         if (err)
2447             return err;
2448         err = esr_write_glue0(np, i, glue0);
2449         if (err)
2450             return err;
2451     }
2452 
2453 
2454     sig = nr64(ESR_INT_SIGNALS);
2455     switch (np->port) {
2456     case 0:
2457         mask = ESR_INT_SIGNALS_P0_BITS;
2458         val = (ESR_INT_SRDY0_P0 |
2459                ESR_INT_DET0_P0 |
2460                ESR_INT_XSRDY_P0 |
2461                ESR_INT_XDP_P0_CH3 |
2462                ESR_INT_XDP_P0_CH2 |
2463                ESR_INT_XDP_P0_CH1 |
2464                ESR_INT_XDP_P0_CH0);
2465         break;
2466 
2467     case 1:
2468         mask = ESR_INT_SIGNALS_P1_BITS;
2469         val = (ESR_INT_SRDY0_P1 |
2470                ESR_INT_DET0_P1 |
2471                ESR_INT_XSRDY_P1 |
2472                ESR_INT_XDP_P1_CH3 |
2473                ESR_INT_XDP_P1_CH2 |
2474                ESR_INT_XDP_P1_CH1 |
2475                ESR_INT_XDP_P1_CH0);
2476         break;
2477 
2478     default:
2479         return -EINVAL;
2480     }
2481 
2482     if ((sig & mask) != val) {
2483         int err;
2484         err = serdes_init_1g_serdes(np);
2485         if (!err) {
2486             np->flags &= ~NIU_FLAGS_10G;
2487             np->mac_xcvr = MAC_XCVR_PCS;
2488         }  else {
2489             netdev_err(np->dev, "Port %u 10G/1G SERDES Link Failed\n",
2490                    np->port);
2491             return -ENODEV;
2492         }
2493     }
2494 
2495     return 0;
2496 }
2497 
2498 static int niu_determine_phy_disposition(struct niu *np)
2499 {
2500     struct niu_parent *parent = np->parent;
2501     u8 plat_type = parent->plat_type;
2502     const struct niu_phy_template *tp;
2503     u32 phy_addr_off = 0;
2504 
2505     if (plat_type == PLAT_TYPE_NIU) {
2506         switch (np->flags &
2507             (NIU_FLAGS_10G |
2508              NIU_FLAGS_FIBER |
2509              NIU_FLAGS_XCVR_SERDES)) {
2510         case NIU_FLAGS_10G | NIU_FLAGS_XCVR_SERDES:
2511             /* 10G Serdes */
2512             tp = &phy_template_niu_10g_serdes;
2513             break;
2514         case NIU_FLAGS_XCVR_SERDES:
2515             /* 1G Serdes */
2516             tp = &phy_template_niu_1g_serdes;
2517             break;
2518         case NIU_FLAGS_10G | NIU_FLAGS_FIBER:
2519             /* 10G Fiber */
2520         default:
2521             if (np->flags & NIU_FLAGS_HOTPLUG_PHY) {
2522                 tp = &phy_template_niu_10g_hotplug;
2523                 if (np->port == 0)
2524                     phy_addr_off = 8;
2525                 if (np->port == 1)
2526                     phy_addr_off = 12;
2527             } else {
2528                 tp = &phy_template_niu_10g_fiber;
2529                 phy_addr_off += np->port;
2530             }
2531             break;
2532         }
2533     } else {
2534         switch (np->flags &
2535             (NIU_FLAGS_10G |
2536              NIU_FLAGS_FIBER |
2537              NIU_FLAGS_XCVR_SERDES)) {
2538         case 0:
2539             /* 1G copper */
2540             tp = &phy_template_1g_copper;
2541             if (plat_type == PLAT_TYPE_VF_P0)
2542                 phy_addr_off = 10;
2543             else if (plat_type == PLAT_TYPE_VF_P1)
2544                 phy_addr_off = 26;
2545 
2546             phy_addr_off += (np->port ^ 0x3);
2547             break;
2548 
2549         case NIU_FLAGS_10G:
2550             /* 10G copper */
2551             tp = &phy_template_10g_copper;
2552             break;
2553 
2554         case NIU_FLAGS_FIBER:
2555             /* 1G fiber */
2556             tp = &phy_template_1g_fiber;
2557             break;
2558 
2559         case NIU_FLAGS_10G | NIU_FLAGS_FIBER:
2560             /* 10G fiber */
2561             tp = &phy_template_10g_fiber;
2562             if (plat_type == PLAT_TYPE_VF_P0 ||
2563                 plat_type == PLAT_TYPE_VF_P1)
2564                 phy_addr_off = 8;
2565             phy_addr_off += np->port;
2566             if (np->flags & NIU_FLAGS_HOTPLUG_PHY) {
2567                 tp = &phy_template_10g_fiber_hotplug;
2568                 if (np->port == 0)
2569                     phy_addr_off = 8;
2570                 if (np->port == 1)
2571                     phy_addr_off = 12;
2572             }
2573             break;
2574 
2575         case NIU_FLAGS_10G | NIU_FLAGS_XCVR_SERDES:
2576         case NIU_FLAGS_XCVR_SERDES | NIU_FLAGS_FIBER:
2577         case NIU_FLAGS_XCVR_SERDES:
2578             switch(np->port) {
2579             case 0:
2580             case 1:
2581                 tp = &phy_template_10g_serdes;
2582                 break;
2583             case 2:
2584             case 3:
2585                 tp = &phy_template_1g_rgmii;
2586                 break;
2587             default:
2588                 return -EINVAL;
2589             }
2590             phy_addr_off = niu_atca_port_num[np->port];
2591             break;
2592 
2593         default:
2594             return -EINVAL;
2595         }
2596     }
2597 
2598     np->phy_ops = tp->ops;
2599     np->phy_addr = tp->phy_addr_base + phy_addr_off;
2600 
2601     return 0;
2602 }
2603 
2604 static int niu_init_link(struct niu *np)
2605 {
2606     struct niu_parent *parent = np->parent;
2607     int err, ignore;
2608 
2609     if (parent->plat_type == PLAT_TYPE_NIU) {
2610         err = niu_xcvr_init(np);
2611         if (err)
2612             return err;
2613         msleep(200);
2614     }
2615     err = niu_serdes_init(np);
2616     if (err && !(np->flags & NIU_FLAGS_HOTPLUG_PHY))
2617         return err;
2618     msleep(200);
2619     err = niu_xcvr_init(np);
2620     if (!err || (np->flags & NIU_FLAGS_HOTPLUG_PHY))
2621         niu_link_status(np, &ignore);
2622     return 0;
2623 }
2624 
2625 static void niu_set_primary_mac(struct niu *np, const unsigned char *addr)
2626 {
2627     u16 reg0 = addr[4] << 8 | addr[5];
2628     u16 reg1 = addr[2] << 8 | addr[3];
2629     u16 reg2 = addr[0] << 8 | addr[1];
2630 
2631     if (np->flags & NIU_FLAGS_XMAC) {
2632         nw64_mac(XMAC_ADDR0, reg0);
2633         nw64_mac(XMAC_ADDR1, reg1);
2634         nw64_mac(XMAC_ADDR2, reg2);
2635     } else {
2636         nw64_mac(BMAC_ADDR0, reg0);
2637         nw64_mac(BMAC_ADDR1, reg1);
2638         nw64_mac(BMAC_ADDR2, reg2);
2639     }
2640 }
2641 
2642 static int niu_num_alt_addr(struct niu *np)
2643 {
2644     if (np->flags & NIU_FLAGS_XMAC)
2645         return XMAC_NUM_ALT_ADDR;
2646     else
2647         return BMAC_NUM_ALT_ADDR;
2648 }
2649 
2650 static int niu_set_alt_mac(struct niu *np, int index, unsigned char *addr)
2651 {
2652     u16 reg0 = addr[4] << 8 | addr[5];
2653     u16 reg1 = addr[2] << 8 | addr[3];
2654     u16 reg2 = addr[0] << 8 | addr[1];
2655 
2656     if (index >= niu_num_alt_addr(np))
2657         return -EINVAL;
2658 
2659     if (np->flags & NIU_FLAGS_XMAC) {
2660         nw64_mac(XMAC_ALT_ADDR0(index), reg0);
2661         nw64_mac(XMAC_ALT_ADDR1(index), reg1);
2662         nw64_mac(XMAC_ALT_ADDR2(index), reg2);
2663     } else {
2664         nw64_mac(BMAC_ALT_ADDR0(index), reg0);
2665         nw64_mac(BMAC_ALT_ADDR1(index), reg1);
2666         nw64_mac(BMAC_ALT_ADDR2(index), reg2);
2667     }
2668 
2669     return 0;
2670 }
2671 
2672 static int niu_enable_alt_mac(struct niu *np, int index, int on)
2673 {
2674     unsigned long reg;
2675     u64 val, mask;
2676 
2677     if (index >= niu_num_alt_addr(np))
2678         return -EINVAL;
2679 
2680     if (np->flags & NIU_FLAGS_XMAC) {
2681         reg = XMAC_ADDR_CMPEN;
2682         mask = 1 << index;
2683     } else {
2684         reg = BMAC_ADDR_CMPEN;
2685         mask = 1 << (index + 1);
2686     }
2687 
2688     val = nr64_mac(reg);
2689     if (on)
2690         val |= mask;
2691     else
2692         val &= ~mask;
2693     nw64_mac(reg, val);
2694 
2695     return 0;
2696 }
2697 
2698 static void __set_rdc_table_num_hw(struct niu *np, unsigned long reg,
2699                    int num, int mac_pref)
2700 {
2701     u64 val = nr64_mac(reg);
2702     val &= ~(HOST_INFO_MACRDCTBLN | HOST_INFO_MPR);
2703     val |= num;
2704     if (mac_pref)
2705         val |= HOST_INFO_MPR;
2706     nw64_mac(reg, val);
2707 }
2708 
2709 static int __set_rdc_table_num(struct niu *np,
2710                    int xmac_index, int bmac_index,
2711                    int rdc_table_num, int mac_pref)
2712 {
2713     unsigned long reg;
2714 
2715     if (rdc_table_num & ~HOST_INFO_MACRDCTBLN)
2716         return -EINVAL;
2717     if (np->flags & NIU_FLAGS_XMAC)
2718         reg = XMAC_HOST_INFO(xmac_index);
2719     else
2720         reg = BMAC_HOST_INFO(bmac_index);
2721     __set_rdc_table_num_hw(np, reg, rdc_table_num, mac_pref);
2722     return 0;
2723 }
2724 
2725 static int niu_set_primary_mac_rdc_table(struct niu *np, int table_num,
2726                      int mac_pref)
2727 {
2728     return __set_rdc_table_num(np, 17, 0, table_num, mac_pref);
2729 }
2730 
2731 static int niu_set_multicast_mac_rdc_table(struct niu *np, int table_num,
2732                        int mac_pref)
2733 {
2734     return __set_rdc_table_num(np, 16, 8, table_num, mac_pref);
2735 }
2736 
2737 static int niu_set_alt_mac_rdc_table(struct niu *np, int idx,
2738                      int table_num, int mac_pref)
2739 {
2740     if (idx >= niu_num_alt_addr(np))
2741         return -EINVAL;
2742     return __set_rdc_table_num(np, idx, idx + 1, table_num, mac_pref);
2743 }
2744 
2745 static u64 vlan_entry_set_parity(u64 reg_val)
2746 {
2747     u64 port01_mask;
2748     u64 port23_mask;
2749 
2750     port01_mask = 0x00ff;
2751     port23_mask = 0xff00;
2752 
2753     if (hweight64(reg_val & port01_mask) & 1)
2754         reg_val |= ENET_VLAN_TBL_PARITY0;
2755     else
2756         reg_val &= ~ENET_VLAN_TBL_PARITY0;
2757 
2758     if (hweight64(reg_val & port23_mask) & 1)
2759         reg_val |= ENET_VLAN_TBL_PARITY1;
2760     else
2761         reg_val &= ~ENET_VLAN_TBL_PARITY1;
2762 
2763     return reg_val;
2764 }
2765 
2766 static void vlan_tbl_write(struct niu *np, unsigned long index,
2767                int port, int vpr, int rdc_table)
2768 {
2769     u64 reg_val = nr64(ENET_VLAN_TBL(index));
2770 
2771     reg_val &= ~((ENET_VLAN_TBL_VPR |
2772               ENET_VLAN_TBL_VLANRDCTBLN) <<
2773              ENET_VLAN_TBL_SHIFT(port));
2774     if (vpr)
2775         reg_val |= (ENET_VLAN_TBL_VPR <<
2776                 ENET_VLAN_TBL_SHIFT(port));
2777     reg_val |= (rdc_table << ENET_VLAN_TBL_SHIFT(port));
2778 
2779     reg_val = vlan_entry_set_parity(reg_val);
2780 
2781     nw64(ENET_VLAN_TBL(index), reg_val);
2782 }
2783 
2784 static void vlan_tbl_clear(struct niu *np)
2785 {
2786     int i;
2787 
2788     for (i = 0; i < ENET_VLAN_TBL_NUM_ENTRIES; i++)
2789         nw64(ENET_VLAN_TBL(i), 0);
2790 }
2791 
2792 static int tcam_wait_bit(struct niu *np, u64 bit)
2793 {
2794     int limit = 1000;
2795 
2796     while (--limit > 0) {
2797         if (nr64(TCAM_CTL) & bit)
2798             break;
2799         udelay(1);
2800     }
2801     if (limit <= 0)
2802         return -ENODEV;
2803 
2804     return 0;
2805 }
2806 
2807 static int tcam_flush(struct niu *np, int index)
2808 {
2809     nw64(TCAM_KEY_0, 0x00);
2810     nw64(TCAM_KEY_MASK_0, 0xff);
2811     nw64(TCAM_CTL, (TCAM_CTL_RWC_TCAM_WRITE | index));
2812 
2813     return tcam_wait_bit(np, TCAM_CTL_STAT);
2814 }
2815 
2816 #if 0
2817 static int tcam_read(struct niu *np, int index,
2818              u64 *key, u64 *mask)
2819 {
2820     int err;
2821 
2822     nw64(TCAM_CTL, (TCAM_CTL_RWC_TCAM_READ | index));
2823     err = tcam_wait_bit(np, TCAM_CTL_STAT);
2824     if (!err) {
2825         key[0] = nr64(TCAM_KEY_0);
2826         key[1] = nr64(TCAM_KEY_1);
2827         key[2] = nr64(TCAM_KEY_2);
2828         key[3] = nr64(TCAM_KEY_3);
2829         mask[0] = nr64(TCAM_KEY_MASK_0);
2830         mask[1] = nr64(TCAM_KEY_MASK_1);
2831         mask[2] = nr64(TCAM_KEY_MASK_2);
2832         mask[3] = nr64(TCAM_KEY_MASK_3);
2833     }
2834     return err;
2835 }
2836 #endif
2837 
2838 static int tcam_write(struct niu *np, int index,
2839               u64 *key, u64 *mask)
2840 {
2841     nw64(TCAM_KEY_0, key[0]);
2842     nw64(TCAM_KEY_1, key[1]);
2843     nw64(TCAM_KEY_2, key[2]);
2844     nw64(TCAM_KEY_3, key[3]);
2845     nw64(TCAM_KEY_MASK_0, mask[0]);
2846     nw64(TCAM_KEY_MASK_1, mask[1]);
2847     nw64(TCAM_KEY_MASK_2, mask[2]);
2848     nw64(TCAM_KEY_MASK_3, mask[3]);
2849     nw64(TCAM_CTL, (TCAM_CTL_RWC_TCAM_WRITE | index));
2850 
2851     return tcam_wait_bit(np, TCAM_CTL_STAT);
2852 }
2853 
2854 #if 0
2855 static int tcam_assoc_read(struct niu *np, int index, u64 *data)
2856 {
2857     int err;
2858 
2859     nw64(TCAM_CTL, (TCAM_CTL_RWC_RAM_READ | index));
2860     err = tcam_wait_bit(np, TCAM_CTL_STAT);
2861     if (!err)
2862         *data = nr64(TCAM_KEY_1);
2863 
2864     return err;
2865 }
2866 #endif
2867 
2868 static int tcam_assoc_write(struct niu *np, int index, u64 assoc_data)
2869 {
2870     nw64(TCAM_KEY_1, assoc_data);
2871     nw64(TCAM_CTL, (TCAM_CTL_RWC_RAM_WRITE | index));
2872 
2873     return tcam_wait_bit(np, TCAM_CTL_STAT);
2874 }
2875 
2876 static void tcam_enable(struct niu *np, int on)
2877 {
2878     u64 val = nr64(FFLP_CFG_1);
2879 
2880     if (on)
2881         val &= ~FFLP_CFG_1_TCAM_DIS;
2882     else
2883         val |= FFLP_CFG_1_TCAM_DIS;
2884     nw64(FFLP_CFG_1, val);
2885 }
2886 
2887 static void tcam_set_lat_and_ratio(struct niu *np, u64 latency, u64 ratio)
2888 {
2889     u64 val = nr64(FFLP_CFG_1);
2890 
2891     val &= ~(FFLP_CFG_1_FFLPINITDONE |
2892          FFLP_CFG_1_CAMLAT |
2893          FFLP_CFG_1_CAMRATIO);
2894     val |= (latency << FFLP_CFG_1_CAMLAT_SHIFT);
2895     val |= (ratio << FFLP_CFG_1_CAMRATIO_SHIFT);
2896     nw64(FFLP_CFG_1, val);
2897 
2898     val = nr64(FFLP_CFG_1);
2899     val |= FFLP_CFG_1_FFLPINITDONE;
2900     nw64(FFLP_CFG_1, val);
2901 }
2902 
2903 static int tcam_user_eth_class_enable(struct niu *np, unsigned long class,
2904                       int on)
2905 {
2906     unsigned long reg;
2907     u64 val;
2908 
2909     if (class < CLASS_CODE_ETHERTYPE1 ||
2910         class > CLASS_CODE_ETHERTYPE2)
2911         return -EINVAL;
2912 
2913     reg = L2_CLS(class - CLASS_CODE_ETHERTYPE1);
2914     val = nr64(reg);
2915     if (on)
2916         val |= L2_CLS_VLD;
2917     else
2918         val &= ~L2_CLS_VLD;
2919     nw64(reg, val);
2920 
2921     return 0;
2922 }
2923 
2924 #if 0
2925 static int tcam_user_eth_class_set(struct niu *np, unsigned long class,
2926                    u64 ether_type)
2927 {
2928     unsigned long reg;
2929     u64 val;
2930 
2931     if (class < CLASS_CODE_ETHERTYPE1 ||
2932         class > CLASS_CODE_ETHERTYPE2 ||
2933         (ether_type & ~(u64)0xffff) != 0)
2934         return -EINVAL;
2935 
2936     reg = L2_CLS(class - CLASS_CODE_ETHERTYPE1);
2937     val = nr64(reg);
2938     val &= ~L2_CLS_ETYPE;
2939     val |= (ether_type << L2_CLS_ETYPE_SHIFT);
2940     nw64(reg, val);
2941 
2942     return 0;
2943 }
2944 #endif
2945 
2946 static int tcam_user_ip_class_enable(struct niu *np, unsigned long class,
2947                      int on)
2948 {
2949     unsigned long reg;
2950     u64 val;
2951 
2952     if (class < CLASS_CODE_USER_PROG1 ||
2953         class > CLASS_CODE_USER_PROG4)
2954         return -EINVAL;
2955 
2956     reg = L3_CLS(class - CLASS_CODE_USER_PROG1);
2957     val = nr64(reg);
2958     if (on)
2959         val |= L3_CLS_VALID;
2960     else
2961         val &= ~L3_CLS_VALID;
2962     nw64(reg, val);
2963 
2964     return 0;
2965 }
2966 
2967 static int tcam_user_ip_class_set(struct niu *np, unsigned long class,
2968                   int ipv6, u64 protocol_id,
2969                   u64 tos_mask, u64 tos_val)
2970 {
2971     unsigned long reg;
2972     u64 val;
2973 
2974     if (class < CLASS_CODE_USER_PROG1 ||
2975         class > CLASS_CODE_USER_PROG4 ||
2976         (protocol_id & ~(u64)0xff) != 0 ||
2977         (tos_mask & ~(u64)0xff) != 0 ||
2978         (tos_val & ~(u64)0xff) != 0)
2979         return -EINVAL;
2980 
2981     reg = L3_CLS(class - CLASS_CODE_USER_PROG1);
2982     val = nr64(reg);
2983     val &= ~(L3_CLS_IPVER | L3_CLS_PID |
2984          L3_CLS_TOSMASK | L3_CLS_TOS);
2985     if (ipv6)
2986         val |= L3_CLS_IPVER;
2987     val |= (protocol_id << L3_CLS_PID_SHIFT);
2988     val |= (tos_mask << L3_CLS_TOSMASK_SHIFT);
2989     val |= (tos_val << L3_CLS_TOS_SHIFT);
2990     nw64(reg, val);
2991 
2992     return 0;
2993 }
2994 
2995 static int tcam_early_init(struct niu *np)
2996 {
2997     unsigned long i;
2998     int err;
2999 
3000     tcam_enable(np, 0);
3001     tcam_set_lat_and_ratio(np,
3002                    DEFAULT_TCAM_LATENCY,
3003                    DEFAULT_TCAM_ACCESS_RATIO);
3004     for (i = CLASS_CODE_ETHERTYPE1; i <= CLASS_CODE_ETHERTYPE2; i++) {
3005         err = tcam_user_eth_class_enable(np, i, 0);
3006         if (err)
3007             return err;
3008     }
3009     for (i = CLASS_CODE_USER_PROG1; i <= CLASS_CODE_USER_PROG4; i++) {
3010         err = tcam_user_ip_class_enable(np, i, 0);
3011         if (err)
3012             return err;
3013     }
3014 
3015     return 0;
3016 }
3017 
3018 static int tcam_flush_all(struct niu *np)
3019 {
3020     unsigned long i;
3021 
3022     for (i = 0; i < np->parent->tcam_num_entries; i++) {
3023         int err = tcam_flush(np, i);
3024         if (err)
3025             return err;
3026     }
3027     return 0;
3028 }
3029 
3030 static u64 hash_addr_regval(unsigned long index, unsigned long num_entries)
3031 {
3032     return (u64)index | (num_entries == 1 ? HASH_TBL_ADDR_AUTOINC : 0);
3033 }
3034 
3035 #if 0
3036 static int hash_read(struct niu *np, unsigned long partition,
3037              unsigned long index, unsigned long num_entries,
3038              u64 *data)
3039 {
3040     u64 val = hash_addr_regval(index, num_entries);
3041     unsigned long i;
3042 
3043     if (partition >= FCRAM_NUM_PARTITIONS ||
3044         index + num_entries > FCRAM_SIZE)
3045         return -EINVAL;
3046 
3047     nw64(HASH_TBL_ADDR(partition), val);
3048     for (i = 0; i < num_entries; i++)
3049         data[i] = nr64(HASH_TBL_DATA(partition));
3050 
3051     return 0;
3052 }
3053 #endif
3054 
3055 static int hash_write(struct niu *np, unsigned long partition,
3056               unsigned long index, unsigned long num_entries,
3057               u64 *data)
3058 {
3059     u64 val = hash_addr_regval(index, num_entries);
3060     unsigned long i;
3061 
3062     if (partition >= FCRAM_NUM_PARTITIONS ||
3063         index + (num_entries * 8) > FCRAM_SIZE)
3064         return -EINVAL;
3065 
3066     nw64(HASH_TBL_ADDR(partition), val);
3067     for (i = 0; i < num_entries; i++)
3068         nw64(HASH_TBL_DATA(partition), data[i]);
3069 
3070     return 0;
3071 }
3072 
3073 static void fflp_reset(struct niu *np)
3074 {
3075     u64 val;
3076 
3077     nw64(FFLP_CFG_1, FFLP_CFG_1_PIO_FIO_RST);
3078     udelay(10);
3079     nw64(FFLP_CFG_1, 0);
3080 
3081     val = FFLP_CFG_1_FCRAMOUTDR_NORMAL | FFLP_CFG_1_FFLPINITDONE;
3082     nw64(FFLP_CFG_1, val);
3083 }
3084 
3085 static void fflp_set_timings(struct niu *np)
3086 {
3087     u64 val = nr64(FFLP_CFG_1);
3088 
3089     val &= ~FFLP_CFG_1_FFLPINITDONE;
3090     val |= (DEFAULT_FCRAMRATIO << FFLP_CFG_1_FCRAMRATIO_SHIFT);
3091     nw64(FFLP_CFG_1, val);
3092 
3093     val = nr64(FFLP_CFG_1);
3094     val |= FFLP_CFG_1_FFLPINITDONE;
3095     nw64(FFLP_CFG_1, val);
3096 
3097     val = nr64(FCRAM_REF_TMR);
3098     val &= ~(FCRAM_REF_TMR_MAX | FCRAM_REF_TMR_MIN);
3099     val |= (DEFAULT_FCRAM_REFRESH_MAX << FCRAM_REF_TMR_MAX_SHIFT);
3100     val |= (DEFAULT_FCRAM_REFRESH_MIN << FCRAM_REF_TMR_MIN_SHIFT);
3101     nw64(FCRAM_REF_TMR, val);
3102 }
3103 
3104 static int fflp_set_partition(struct niu *np, u64 partition,
3105                   u64 mask, u64 base, int enable)
3106 {
3107     unsigned long reg;
3108     u64 val;
3109 
3110     if (partition >= FCRAM_NUM_PARTITIONS ||
3111         (mask & ~(u64)0x1f) != 0 ||
3112         (base & ~(u64)0x1f) != 0)
3113         return -EINVAL;
3114 
3115     reg = FLW_PRT_SEL(partition);
3116 
3117     val = nr64(reg);
3118     val &= ~(FLW_PRT_SEL_EXT | FLW_PRT_SEL_MASK | FLW_PRT_SEL_BASE);
3119     val |= (mask << FLW_PRT_SEL_MASK_SHIFT);
3120     val |= (base << FLW_PRT_SEL_BASE_SHIFT);
3121     if (enable)
3122         val |= FLW_PRT_SEL_EXT;
3123     nw64(reg, val);
3124 
3125     return 0;
3126 }
3127 
3128 static int fflp_disable_all_partitions(struct niu *np)
3129 {
3130     unsigned long i;
3131 
3132     for (i = 0; i < FCRAM_NUM_PARTITIONS; i++) {
3133         int err = fflp_set_partition(np, 0, 0, 0, 0);
3134         if (err)
3135             return err;
3136     }
3137     return 0;
3138 }
3139 
3140 static void fflp_llcsnap_enable(struct niu *np, int on)
3141 {
3142     u64 val = nr64(FFLP_CFG_1);
3143 
3144     if (on)
3145         val |= FFLP_CFG_1_LLCSNAP;
3146     else
3147         val &= ~FFLP_CFG_1_LLCSNAP;
3148     nw64(FFLP_CFG_1, val);
3149 }
3150 
3151 static void fflp_errors_enable(struct niu *np, int on)
3152 {
3153     u64 val = nr64(FFLP_CFG_1);
3154 
3155     if (on)
3156         val &= ~FFLP_CFG_1_ERRORDIS;
3157     else
3158         val |= FFLP_CFG_1_ERRORDIS;
3159     nw64(FFLP_CFG_1, val);
3160 }
3161 
3162 static int fflp_hash_clear(struct niu *np)
3163 {
3164     struct fcram_hash_ipv4 ent;
3165     unsigned long i;
3166 
3167     /* IPV4 hash entry with valid bit clear, rest is don't care.  */
3168     memset(&ent, 0, sizeof(ent));
3169     ent.header = HASH_HEADER_EXT;
3170 
3171     for (i = 0; i < FCRAM_SIZE; i += sizeof(ent)) {
3172         int err = hash_write(np, 0, i, 1, (u64 *) &ent);
3173         if (err)
3174             return err;
3175     }
3176     return 0;
3177 }
3178 
3179 static int fflp_early_init(struct niu *np)
3180 {
3181     struct niu_parent *parent;
3182     unsigned long flags;
3183     int err;
3184 
3185     niu_lock_parent(np, flags);
3186 
3187     parent = np->parent;
3188     err = 0;
3189     if (!(parent->flags & PARENT_FLGS_CLS_HWINIT)) {
3190         if (np->parent->plat_type != PLAT_TYPE_NIU) {
3191             fflp_reset(np);
3192             fflp_set_timings(np);
3193             err = fflp_disable_all_partitions(np);
3194             if (err) {
3195                 netif_printk(np, probe, KERN_DEBUG, np->dev,
3196                          "fflp_disable_all_partitions failed, err=%d\n",
3197                          err);
3198                 goto out;
3199             }
3200         }
3201 
3202         err = tcam_early_init(np);
3203         if (err) {
3204             netif_printk(np, probe, KERN_DEBUG, np->dev,
3205                      "tcam_early_init failed, err=%d\n", err);
3206             goto out;
3207         }
3208         fflp_llcsnap_enable(np, 1);
3209         fflp_errors_enable(np, 0);
3210         nw64(H1POLY, 0);
3211         nw64(H2POLY, 0);
3212 
3213         err = tcam_flush_all(np);
3214         if (err) {
3215             netif_printk(np, probe, KERN_DEBUG, np->dev,
3216                      "tcam_flush_all failed, err=%d\n", err);
3217             goto out;
3218         }
3219         if (np->parent->plat_type != PLAT_TYPE_NIU) {
3220             err = fflp_hash_clear(np);
3221             if (err) {
3222                 netif_printk(np, probe, KERN_DEBUG, np->dev,
3223                          "fflp_hash_clear failed, err=%d\n",
3224                          err);
3225                 goto out;
3226             }
3227         }
3228 
3229         vlan_tbl_clear(np);
3230 
3231         parent->flags |= PARENT_FLGS_CLS_HWINIT;
3232     }
3233 out:
3234     niu_unlock_parent(np, flags);
3235     return err;
3236 }
3237 
3238 static int niu_set_flow_key(struct niu *np, unsigned long class_code, u64 key)
3239 {
3240     if (class_code < CLASS_CODE_USER_PROG1 ||
3241         class_code > CLASS_CODE_SCTP_IPV6)
3242         return -EINVAL;
3243 
3244     nw64(FLOW_KEY(class_code - CLASS_CODE_USER_PROG1), key);
3245     return 0;
3246 }
3247 
3248 static int niu_set_tcam_key(struct niu *np, unsigned long class_code, u64 key)
3249 {
3250     if (class_code < CLASS_CODE_USER_PROG1 ||
3251         class_code > CLASS_CODE_SCTP_IPV6)
3252         return -EINVAL;
3253 
3254     nw64(TCAM_KEY(class_code - CLASS_CODE_USER_PROG1), key);
3255     return 0;
3256 }
3257 
3258 /* Entries for the ports are interleaved in the TCAM */
3259 static u16 tcam_get_index(struct niu *np, u16 idx)
3260 {
3261     /* One entry reserved for IP fragment rule */
3262     if (idx >= (np->clas.tcam_sz - 1))
3263         idx = 0;
3264     return np->clas.tcam_top + ((idx+1) * np->parent->num_ports);
3265 }
3266 
3267 static u16 tcam_get_size(struct niu *np)
3268 {
3269     /* One entry reserved for IP fragment rule */
3270     return np->clas.tcam_sz - 1;
3271 }
3272 
3273 static u16 tcam_get_valid_entry_cnt(struct niu *np)
3274 {
3275     /* One entry reserved for IP fragment rule */
3276     return np->clas.tcam_valid_entries - 1;
3277 }
3278 
3279 static void niu_rx_skb_append(struct sk_buff *skb, struct page *page,
3280                   u32 offset, u32 size, u32 truesize)
3281 {
3282     skb_fill_page_desc(skb, skb_shinfo(skb)->nr_frags, page, offset, size);
3283 
3284     skb->len += size;
3285     skb->data_len += size;
3286     skb->truesize += truesize;
3287 }
3288 
3289 static unsigned int niu_hash_rxaddr(struct rx_ring_info *rp, u64 a)
3290 {
3291     a >>= PAGE_SHIFT;
3292     a ^= (a >> ilog2(MAX_RBR_RING_SIZE));
3293 
3294     return a & (MAX_RBR_RING_SIZE - 1);
3295 }
3296 
3297 static struct page *niu_find_rxpage(struct rx_ring_info *rp, u64 addr,
3298                     struct page ***link)
3299 {
3300     unsigned int h = niu_hash_rxaddr(rp, addr);
3301     struct page *p, **pp;
3302 
3303     addr &= PAGE_MASK;
3304     pp = &rp->rxhash[h];
3305     for (; (p = *pp) != NULL; pp = &niu_next_page(p)) {
3306         if (p->index == addr) {
3307             *link = pp;
3308             goto found;
3309         }
3310     }
3311     BUG();
3312 
3313 found:
3314     return p;
3315 }
3316 
3317 static void niu_hash_page(struct rx_ring_info *rp, struct page *page, u64 base)
3318 {
3319     unsigned int h = niu_hash_rxaddr(rp, base);
3320 
3321     page->index = base;
3322     niu_next_page(page) = rp->rxhash[h];
3323     rp->rxhash[h] = page;
3324 }
3325 
3326 static int niu_rbr_add_page(struct niu *np, struct rx_ring_info *rp,
3327                 gfp_t mask, int start_index)
3328 {
3329     struct page *page;
3330     u64 addr;
3331     int i;
3332 
3333     page = alloc_page(mask);
3334     if (!page)
3335         return -ENOMEM;
3336 
3337     addr = np->ops->map_page(np->device, page, 0,
3338                  PAGE_SIZE, DMA_FROM_DEVICE);
3339     if (!addr) {
3340         __free_page(page);
3341         return -ENOMEM;
3342     }
3343 
3344     niu_hash_page(rp, page, addr);
3345     if (rp->rbr_blocks_per_page > 1)
3346         page_ref_add(page, rp->rbr_blocks_per_page - 1);
3347 
3348     for (i = 0; i < rp->rbr_blocks_per_page; i++) {
3349         __le32 *rbr = &rp->rbr[start_index + i];
3350 
3351         *rbr = cpu_to_le32(addr >> RBR_DESCR_ADDR_SHIFT);
3352         addr += rp->rbr_block_size;
3353     }
3354 
3355     return 0;
3356 }
3357 
3358 static void niu_rbr_refill(struct niu *np, struct rx_ring_info *rp, gfp_t mask)
3359 {
3360     int index = rp->rbr_index;
3361 
3362     rp->rbr_pending++;
3363     if ((rp->rbr_pending % rp->rbr_blocks_per_page) == 0) {
3364         int err = niu_rbr_add_page(np, rp, mask, index);
3365 
3366         if (unlikely(err)) {
3367             rp->rbr_pending--;
3368             return;
3369         }
3370 
3371         rp->rbr_index += rp->rbr_blocks_per_page;
3372         BUG_ON(rp->rbr_index > rp->rbr_table_size);
3373         if (rp->rbr_index == rp->rbr_table_size)
3374             rp->rbr_index = 0;
3375 
3376         if (rp->rbr_pending >= rp->rbr_kick_thresh) {
3377             nw64(RBR_KICK(rp->rx_channel), rp->rbr_pending);
3378             rp->rbr_pending = 0;
3379         }
3380     }
3381 }
3382 
3383 static int niu_rx_pkt_ignore(struct niu *np, struct rx_ring_info *rp)
3384 {
3385     unsigned int index = rp->rcr_index;
3386     int num_rcr = 0;
3387 
3388     rp->rx_dropped++;
3389     while (1) {
3390         struct page *page, **link;
3391         u64 addr, val;
3392         u32 rcr_size;
3393 
3394         num_rcr++;
3395 
3396         val = le64_to_cpup(&rp->rcr[index]);
3397         addr = (val & RCR_ENTRY_PKT_BUF_ADDR) <<
3398             RCR_ENTRY_PKT_BUF_ADDR_SHIFT;
3399         page = niu_find_rxpage(rp, addr, &link);
3400 
3401         rcr_size = rp->rbr_sizes[(val & RCR_ENTRY_PKTBUFSZ) >>
3402                      RCR_ENTRY_PKTBUFSZ_SHIFT];
3403         if ((page->index + PAGE_SIZE) - rcr_size == addr) {
3404             *link = niu_next_page(page);
3405             np->ops->unmap_page(np->device, page->index,
3406                         PAGE_SIZE, DMA_FROM_DEVICE);
3407             page->index = 0;
3408             niu_next_page(page) = NULL;
3409             __free_page(page);
3410             rp->rbr_refill_pending++;
3411         }
3412 
3413         index = NEXT_RCR(rp, index);
3414         if (!(val & RCR_ENTRY_MULTI))
3415             break;
3416 
3417     }
3418     rp->rcr_index = index;
3419 
3420     return num_rcr;
3421 }
3422 
3423 static int niu_process_rx_pkt(struct napi_struct *napi, struct niu *np,
3424                   struct rx_ring_info *rp)
3425 {
3426     unsigned int index = rp->rcr_index;
3427     struct rx_pkt_hdr1 *rh;
3428     struct sk_buff *skb;
3429     int len, num_rcr;
3430 
3431     skb = netdev_alloc_skb(np->dev, RX_SKB_ALLOC_SIZE);
3432     if (unlikely(!skb))
3433         return niu_rx_pkt_ignore(np, rp);
3434 
3435     num_rcr = 0;
3436     while (1) {
3437         struct page *page, **link;
3438         u32 rcr_size, append_size;
3439         u64 addr, val, off;
3440 
3441         num_rcr++;
3442 
3443         val = le64_to_cpup(&rp->rcr[index]);
3444 
3445         len = (val & RCR_ENTRY_L2_LEN) >>
3446             RCR_ENTRY_L2_LEN_SHIFT;
3447         append_size = len + ETH_HLEN + ETH_FCS_LEN;
3448 
3449         addr = (val & RCR_ENTRY_PKT_BUF_ADDR) <<
3450             RCR_ENTRY_PKT_BUF_ADDR_SHIFT;
3451         page = niu_find_rxpage(rp, addr, &link);
3452 
3453         rcr_size = rp->rbr_sizes[(val & RCR_ENTRY_PKTBUFSZ) >>
3454                      RCR_ENTRY_PKTBUFSZ_SHIFT];
3455 
3456         off = addr & ~PAGE_MASK;
3457         if (num_rcr == 1) {
3458             int ptype;
3459 
3460             ptype = (val >> RCR_ENTRY_PKT_TYPE_SHIFT);
3461             if ((ptype == RCR_PKT_TYPE_TCP ||
3462                  ptype == RCR_PKT_TYPE_UDP) &&
3463                 !(val & (RCR_ENTRY_NOPORT |
3464                      RCR_ENTRY_ERROR)))
3465                 skb->ip_summed = CHECKSUM_UNNECESSARY;
3466             else
3467                 skb_checksum_none_assert(skb);
3468         } else if (!(val & RCR_ENTRY_MULTI))
3469             append_size = append_size - skb->len;
3470 
3471         niu_rx_skb_append(skb, page, off, append_size, rcr_size);
3472         if ((page->index + rp->rbr_block_size) - rcr_size == addr) {
3473             *link = niu_next_page(page);
3474             np->ops->unmap_page(np->device, page->index,
3475                         PAGE_SIZE, DMA_FROM_DEVICE);
3476             page->index = 0;
3477             niu_next_page(page) = NULL;
3478             rp->rbr_refill_pending++;
3479         } else
3480             get_page(page);
3481 
3482         index = NEXT_RCR(rp, index);
3483         if (!(val & RCR_ENTRY_MULTI))
3484             break;
3485 
3486     }
3487     rp->rcr_index = index;
3488 
3489     len += sizeof(*rh);
3490     len = min_t(int, len, sizeof(*rh) + VLAN_ETH_HLEN);
3491     __pskb_pull_tail(skb, len);
3492 
3493     rh = (struct rx_pkt_hdr1 *) skb->data;
3494     if (np->dev->features & NETIF_F_RXHASH)
3495         skb_set_hash(skb,
3496                  ((u32)rh->hashval2_0 << 24 |
3497                   (u32)rh->hashval2_1 << 16 |
3498                   (u32)rh->hashval1_1 << 8 |
3499                   (u32)rh->hashval1_2 << 0),
3500                  PKT_HASH_TYPE_L3);
3501     skb_pull(skb, sizeof(*rh));
3502 
3503     rp->rx_packets++;
3504     rp->rx_bytes += skb->len;
3505 
3506     skb->protocol = eth_type_trans(skb, np->dev);
3507     skb_record_rx_queue(skb, rp->rx_channel);
3508     napi_gro_receive(napi, skb);
3509 
3510     return num_rcr;
3511 }
3512 
3513 static int niu_rbr_fill(struct niu *np, struct rx_ring_info *rp, gfp_t mask)
3514 {
3515     int blocks_per_page = rp->rbr_blocks_per_page;
3516     int err, index = rp->rbr_index;
3517 
3518     err = 0;
3519     while (index < (rp->rbr_table_size - blocks_per_page)) {
3520         err = niu_rbr_add_page(np, rp, mask, index);
3521         if (unlikely(err))
3522             break;
3523 
3524         index += blocks_per_page;
3525     }
3526 
3527     rp->rbr_index = index;
3528     return err;
3529 }
3530 
3531 static void niu_rbr_free(struct niu *np, struct rx_ring_info *rp)
3532 {
3533     int i;
3534 
3535     for (i = 0; i < MAX_RBR_RING_SIZE; i++) {
3536         struct page *page;
3537 
3538         page = rp->rxhash[i];
3539         while (page) {
3540             struct page *next = niu_next_page(page);
3541             u64 base = page->index;
3542 
3543             np->ops->unmap_page(np->device, base, PAGE_SIZE,
3544                         DMA_FROM_DEVICE);
3545             page->index = 0;
3546             niu_next_page(page) = NULL;
3547 
3548             __free_page(page);
3549 
3550             page = next;
3551         }
3552     }
3553 
3554     for (i = 0; i < rp->rbr_table_size; i++)
3555         rp->rbr[i] = cpu_to_le32(0);
3556     rp->rbr_index = 0;
3557 }
3558 
3559 static int release_tx_packet(struct niu *np, struct tx_ring_info *rp, int idx)
3560 {
3561     struct tx_buff_info *tb = &rp->tx_buffs[idx];
3562     struct sk_buff *skb = tb->skb;
3563     struct tx_pkt_hdr *tp;
3564     u64 tx_flags;
3565     int i, len;
3566 
3567     tp = (struct tx_pkt_hdr *) skb->data;
3568     tx_flags = le64_to_cpup(&tp->flags);
3569 
3570     rp->tx_packets++;
3571     rp->tx_bytes += (((tx_flags & TXHDR_LEN) >> TXHDR_LEN_SHIFT) -
3572              ((tx_flags & TXHDR_PAD) / 2));
3573 
3574     len = skb_headlen(skb);
3575     np->ops->unmap_single(np->device, tb->mapping,
3576                   len, DMA_TO_DEVICE);
3577 
3578     if (le64_to_cpu(rp->descr[idx]) & TX_DESC_MARK)
3579         rp->mark_pending--;
3580 
3581     tb->skb = NULL;
3582     do {
3583         idx = NEXT_TX(rp, idx);
3584         len -= MAX_TX_DESC_LEN;
3585     } while (len > 0);
3586 
3587     for (i = 0; i < skb_shinfo(skb)->nr_frags; i++) {
3588         tb = &rp->tx_buffs[idx];
3589         BUG_ON(tb->skb != NULL);
3590         np->ops->unmap_page(np->device, tb->mapping,
3591                     skb_frag_size(&skb_shinfo(skb)->frags[i]),
3592                     DMA_TO_DEVICE);
3593         idx = NEXT_TX(rp, idx);
3594     }
3595 
3596     dev_kfree_skb(skb);
3597 
3598     return idx;
3599 }
3600 
3601 #define NIU_TX_WAKEUP_THRESH(rp)        ((rp)->pending / 4)
3602 
3603 static void niu_tx_work(struct niu *np, struct tx_ring_info *rp)
3604 {
3605     struct netdev_queue *txq;
3606     u16 pkt_cnt, tmp;
3607     int cons, index;
3608     u64 cs;
3609 
3610     index = (rp - np->tx_rings);
3611     txq = netdev_get_tx_queue(np->dev, index);
3612 
3613     cs = rp->tx_cs;
3614     if (unlikely(!(cs & (TX_CS_MK | TX_CS_MMK))))
3615         goto out;
3616 
3617     tmp = pkt_cnt = (cs & TX_CS_PKT_CNT) >> TX_CS_PKT_CNT_SHIFT;
3618     pkt_cnt = (pkt_cnt - rp->last_pkt_cnt) &
3619         (TX_CS_PKT_CNT >> TX_CS_PKT_CNT_SHIFT);
3620 
3621     rp->last_pkt_cnt = tmp;
3622 
3623     cons = rp->cons;
3624 
3625     netif_printk(np, tx_done, KERN_DEBUG, np->dev,
3626              "%s() pkt_cnt[%u] cons[%d]\n", __func__, pkt_cnt, cons);
3627 
3628     while (pkt_cnt--)
3629         cons = release_tx_packet(np, rp, cons);
3630 
3631     rp->cons = cons;
3632     smp_mb();
3633 
3634 out:
3635     if (unlikely(netif_tx_queue_stopped(txq) &&
3636              (niu_tx_avail(rp) > NIU_TX_WAKEUP_THRESH(rp)))) {
3637         __netif_tx_lock(txq, smp_processor_id());
3638         if (netif_tx_queue_stopped(txq) &&
3639             (niu_tx_avail(rp) > NIU_TX_WAKEUP_THRESH(rp)))
3640             netif_tx_wake_queue(txq);
3641         __netif_tx_unlock(txq);
3642     }
3643 }
3644 
3645 static inline void niu_sync_rx_discard_stats(struct niu *np,
3646                          struct rx_ring_info *rp,
3647                          const int limit)
3648 {
3649     /* This elaborate scheme is needed for reading the RX discard
3650      * counters, as they are only 16-bit and can overflow quickly,
3651      * and because the overflow indication bit is not usable as
3652      * the counter value does not wrap, but remains at max value
3653      * 0xFFFF.
3654      *
3655      * In theory and in practice counters can be lost in between
3656      * reading nr64() and clearing the counter nw64().  For this
3657      * reason, the number of counter clearings nw64() is
3658      * limited/reduced though the limit parameter.
3659      */
3660     int rx_channel = rp->rx_channel;
3661     u32 misc, wred;
3662 
3663     /* RXMISC (Receive Miscellaneous Discard Count), covers the
3664      * following discard events: IPP (Input Port Process),
3665      * FFLP/TCAM, Full RCR (Receive Completion Ring) RBR (Receive
3666      * Block Ring) prefetch buffer is empty.
3667      */
3668     misc = nr64(RXMISC(rx_channel));
3669     if (unlikely((misc & RXMISC_COUNT) > limit)) {
3670         nw64(RXMISC(rx_channel), 0);
3671         rp->rx_errors += misc & RXMISC_COUNT;
3672 
3673         if (unlikely(misc & RXMISC_OFLOW))
3674             dev_err(np->device, "rx-%d: Counter overflow RXMISC discard\n",
3675                 rx_channel);
3676 
3677         netif_printk(np, rx_err, KERN_DEBUG, np->dev,
3678                  "rx-%d: MISC drop=%u over=%u\n",
3679                  rx_channel, misc, misc-limit);
3680     }
3681 
3682     /* WRED (Weighted Random Early Discard) by hardware */
3683     wred = nr64(RED_DIS_CNT(rx_channel));
3684     if (unlikely((wred & RED_DIS_CNT_COUNT) > limit)) {
3685         nw64(RED_DIS_CNT(rx_channel), 0);
3686         rp->rx_dropped += wred & RED_DIS_CNT_COUNT;
3687 
3688         if (unlikely(wred & RED_DIS_CNT_OFLOW))
3689             dev_err(np->device, "rx-%d: Counter overflow WRED discard\n", rx_channel);
3690 
3691         netif_printk(np, rx_err, KERN_DEBUG, np->dev,
3692                  "rx-%d: WRED drop=%u over=%u\n",
3693                  rx_channel, wred, wred-limit);
3694     }
3695 }
3696 
3697 static int niu_rx_work(struct napi_struct *napi, struct niu *np,
3698                struct rx_ring_info *rp, int budget)
3699 {
3700     int qlen, rcr_done = 0, work_done = 0;
3701     struct rxdma_mailbox *mbox = rp->mbox;
3702     u64 stat;
3703 
3704 #if 1
3705     stat = nr64(RX_DMA_CTL_STAT(rp->rx_channel));
3706     qlen = nr64(RCRSTAT_A(rp->rx_channel)) & RCRSTAT_A_QLEN;
3707 #else
3708     stat = le64_to_cpup(&mbox->rx_dma_ctl_stat);
3709     qlen = (le64_to_cpup(&mbox->rcrstat_a) & RCRSTAT_A_QLEN);
3710 #endif
3711     mbox->rx_dma_ctl_stat = 0;
3712     mbox->rcrstat_a = 0;
3713 
3714     netif_printk(np, rx_status, KERN_DEBUG, np->dev,
3715              "%s(chan[%d]), stat[%llx] qlen=%d\n",
3716              __func__, rp->rx_channel, (unsigned long long)stat, qlen);
3717 
3718     rcr_done = work_done = 0;
3719     qlen = min(qlen, budget);
3720     while (work_done < qlen) {
3721         rcr_done += niu_process_rx_pkt(napi, np, rp);
3722         work_done++;
3723     }
3724 
3725     if (rp->rbr_refill_pending >= rp->rbr_kick_thresh) {
3726         unsigned int i;
3727 
3728         for (i = 0; i < rp->rbr_refill_pending; i++)
3729             niu_rbr_refill(np, rp, GFP_ATOMIC);
3730         rp->rbr_refill_pending = 0;
3731     }
3732 
3733     stat = (RX_DMA_CTL_STAT_MEX |
3734         ((u64)work_done << RX_DMA_CTL_STAT_PKTREAD_SHIFT) |
3735         ((u64)rcr_done << RX_DMA_CTL_STAT_PTRREAD_SHIFT));
3736 
3737     nw64(RX_DMA_CTL_STAT(rp->rx_channel), stat);
3738 
3739     /* Only sync discards stats when qlen indicate potential for drops */
3740     if (qlen > 10)
3741         niu_sync_rx_discard_stats(np, rp, 0x7FFF);
3742 
3743     return work_done;
3744 }
3745 
3746 static int niu_poll_core(struct niu *np, struct niu_ldg *lp, int budget)
3747 {
3748     u64 v0 = lp->v0;
3749     u32 tx_vec = (v0 >> 32);
3750     u32 rx_vec = (v0 & 0xffffffff);
3751     int i, work_done = 0;
3752 
3753     netif_printk(np, intr, KERN_DEBUG, np->dev,
3754              "%s() v0[%016llx]\n", __func__, (unsigned long long)v0);
3755 
3756     for (i = 0; i < np->num_tx_rings; i++) {
3757         struct tx_ring_info *rp = &np->tx_rings[i];
3758         if (tx_vec & (1 << rp->tx_channel))
3759             niu_tx_work(np, rp);
3760         nw64(LD_IM0(LDN_TXDMA(rp->tx_channel)), 0);
3761     }
3762 
3763     for (i = 0; i < np->num_rx_rings; i++) {
3764         struct rx_ring_info *rp = &np->rx_rings[i];
3765 
3766         if (rx_vec & (1 << rp->rx_channel)) {
3767             int this_work_done;
3768 
3769             this_work_done = niu_rx_work(&lp->napi, np, rp,
3770                              budget);
3771 
3772             budget -= this_work_done;
3773             work_done += this_work_done;
3774         }
3775         nw64(LD_IM0(LDN_RXDMA(rp->rx_channel)), 0);
3776     }
3777 
3778     return work_done;
3779 }
3780 
3781 static int niu_poll(struct napi_struct *napi, int budget)
3782 {
3783     struct niu_ldg *lp = container_of(napi, struct niu_ldg, napi);
3784     struct niu *np = lp->np;
3785     int work_done;
3786 
3787     work_done = niu_poll_core(np, lp, budget);
3788 
3789     if (work_done < budget) {
3790         napi_complete_done(napi, work_done);
3791         niu_ldg_rearm(np, lp, 1);
3792     }
3793     return work_done;
3794 }
3795 
3796 static void niu_log_rxchan_errors(struct niu *np, struct rx_ring_info *rp,
3797                   u64 stat)
3798 {
3799     netdev_err(np->dev, "RX channel %u errors ( ", rp->rx_channel);
3800 
3801     if (stat & RX_DMA_CTL_STAT_RBR_TMOUT)
3802         pr_cont("RBR_TMOUT ");
3803     if (stat & RX_DMA_CTL_STAT_RSP_CNT_ERR)
3804         pr_cont("RSP_CNT ");
3805     if (stat & RX_DMA_CTL_STAT_BYTE_EN_BUS)
3806         pr_cont("BYTE_EN_BUS ");
3807     if (stat & RX_DMA_CTL_STAT_RSP_DAT_ERR)
3808         pr_cont("RSP_DAT ");
3809     if (stat & RX_DMA_CTL_STAT_RCR_ACK_ERR)
3810         pr_cont("RCR_ACK ");
3811     if (stat & RX_DMA_CTL_STAT_RCR_SHA_PAR)
3812         pr_cont("RCR_SHA_PAR ");
3813     if (stat & RX_DMA_CTL_STAT_RBR_PRE_PAR)
3814         pr_cont("RBR_PRE_PAR ");
3815     if (stat & RX_DMA_CTL_STAT_CONFIG_ERR)
3816         pr_cont("CONFIG ");
3817     if (stat & RX_DMA_CTL_STAT_RCRINCON)
3818         pr_cont("RCRINCON ");
3819     if (stat & RX_DMA_CTL_STAT_RCRFULL)
3820         pr_cont("RCRFULL ");
3821     if (stat & RX_DMA_CTL_STAT_RBRFULL)
3822         pr_cont("RBRFULL ");
3823     if (stat & RX_DMA_CTL_STAT_RBRLOGPAGE)
3824         pr_cont("RBRLOGPAGE ");
3825     if (stat & RX_DMA_CTL_STAT_CFIGLOGPAGE)
3826         pr_cont("CFIGLOGPAGE ");
3827     if (stat & RX_DMA_CTL_STAT_DC_FIFO_ERR)
3828         pr_cont("DC_FIDO ");
3829 
3830     pr_cont(")\n");
3831 }
3832 
3833 static int niu_rx_error(struct niu *np, struct rx_ring_info *rp)
3834 {
3835     u64 stat = nr64(RX_DMA_CTL_STAT(rp->rx_channel));
3836     int err = 0;
3837 
3838 
3839     if (stat & (RX_DMA_CTL_STAT_CHAN_FATAL |
3840             RX_DMA_CTL_STAT_PORT_FATAL))
3841         err = -EINVAL;
3842 
3843     if (err) {
3844         netdev_err(np->dev, "RX channel %u error, stat[%llx]\n",
3845                rp->rx_channel,
3846                (unsigned long long) stat);
3847 
3848         niu_log_rxchan_errors(np, rp, stat);
3849     }
3850 
3851     nw64(RX_DMA_CTL_STAT(rp->rx_channel),
3852          stat & RX_DMA_CTL_WRITE_CLEAR_ERRS);
3853 
3854     return err;
3855 }
3856 
3857 static void niu_log_txchan_errors(struct niu *np, struct tx_ring_info *rp,
3858                   u64 cs)
3859 {
3860     netdev_err(np->dev, "TX channel %u errors ( ", rp->tx_channel);
3861 
3862     if (cs & TX_CS_MBOX_ERR)
3863         pr_cont("MBOX ");
3864     if (cs & TX_CS_PKT_SIZE_ERR)
3865         pr_cont("PKT_SIZE ");
3866     if (cs & TX_CS_TX_RING_OFLOW)
3867         pr_cont("TX_RING_OFLOW ");
3868     if (cs & TX_CS_PREF_BUF_PAR_ERR)
3869         pr_cont("PREF_BUF_PAR ");
3870     if (cs & TX_CS_NACK_PREF)
3871         pr_cont("NACK_PREF ");
3872     if (cs & TX_CS_NACK_PKT_RD)
3873         pr_cont("NACK_PKT_RD ");
3874     if (cs & TX_CS_CONF_PART_ERR)
3875         pr_cont("CONF_PART ");
3876     if (cs & TX_CS_PKT_PRT_ERR)
3877         pr_cont("PKT_PTR ");
3878 
3879     pr_cont(")\n");
3880 }
3881 
3882 static int niu_tx_error(struct niu *np, struct tx_ring_info *rp)
3883 {
3884     u64 cs, logh, logl;
3885 
3886     cs = nr64(TX_CS(rp->tx_channel));
3887     logh = nr64(TX_RNG_ERR_LOGH(rp->tx_channel));
3888     logl = nr64(TX_RNG_ERR_LOGL(rp->tx_channel));
3889 
3890     netdev_err(np->dev, "TX channel %u error, cs[%llx] logh[%llx] logl[%llx]\n",
3891            rp->tx_channel,
3892            (unsigned long long)cs,
3893            (unsigned long long)logh,
3894            (unsigned long long)logl);
3895 
3896     niu_log_txchan_errors(np, rp, cs);
3897 
3898     return -ENODEV;
3899 }
3900 
3901 static int niu_mif_interrupt(struct niu *np)
3902 {
3903     u64 mif_status = nr64(MIF_STATUS);
3904     int phy_mdint = 0;
3905 
3906     if (np->flags & NIU_FLAGS_XMAC) {
3907         u64 xrxmac_stat = nr64_mac(XRXMAC_STATUS);
3908 
3909         if (xrxmac_stat & XRXMAC_STATUS_PHY_MDINT)
3910             phy_mdint = 1;
3911     }
3912 
3913     netdev_err(np->dev, "MIF interrupt, stat[%llx] phy_mdint(%d)\n",
3914            (unsigned long long)mif_status, phy_mdint);
3915 
3916     return -ENODEV;
3917 }
3918 
3919 static void niu_xmac_interrupt(struct niu *np)
3920 {
3921     struct niu_xmac_stats *mp = &np->mac_stats.xmac;
3922     u64 val;
3923 
3924     val = nr64_mac(XTXMAC_STATUS);
3925     if (val & XTXMAC_STATUS_FRAME_CNT_EXP)
3926         mp->tx_frames += TXMAC_FRM_CNT_COUNT;
3927     if (val & XTXMAC_STATUS_BYTE_CNT_EXP)
3928         mp->tx_bytes += TXMAC_BYTE_CNT_COUNT;
3929     if (val & XTXMAC_STATUS_TXFIFO_XFR_ERR)
3930         mp->tx_fifo_errors++;
3931     if (val & XTXMAC_STATUS_TXMAC_OFLOW)
3932         mp->tx_overflow_errors++;
3933     if (val & XTXMAC_STATUS_MAX_PSIZE_ERR)
3934         mp->tx_max_pkt_size_errors++;
3935     if (val & XTXMAC_STATUS_TXMAC_UFLOW)
3936         mp->tx_underflow_errors++;
3937 
3938     val = nr64_mac(XRXMAC_STATUS);
3939     if (val & XRXMAC_STATUS_LCL_FLT_STATUS)
3940         mp->rx_local_faults++;
3941     if (val & XRXMAC_STATUS_RFLT_DET)
3942         mp->rx_remote_faults++;
3943     if (val & XRXMAC_STATUS_LFLT_CNT_EXP)
3944         mp->rx_link_faults += LINK_FAULT_CNT_COUNT;
3945     if (val & XRXMAC_STATUS_ALIGNERR_CNT_EXP)
3946         mp->rx_align_errors += RXMAC_ALIGN_ERR_CNT_COUNT;
3947     if (val & XRXMAC_STATUS_RXFRAG_CNT_EXP)
3948         mp->rx_frags += RXMAC_FRAG_CNT_COUNT;
3949     if (val & XRXMAC_STATUS_RXMULTF_CNT_EXP)
3950         mp->rx_mcasts += RXMAC_MC_FRM_CNT_COUNT;
3951     if (val & XRXMAC_STATUS_RXBCAST_CNT_EXP)
3952         mp->rx_bcasts += RXMAC_BC_FRM_CNT_COUNT;
3953     if (val & XRXMAC_STATUS_RXHIST1_CNT_EXP)
3954         mp->rx_hist_cnt1 += RXMAC_HIST_CNT1_COUNT;
3955     if (val & XRXMAC_STATUS_RXHIST2_CNT_EXP)
3956         mp->rx_hist_cnt2 += RXMAC_HIST_CNT2_COUNT;
3957     if (val & XRXMAC_STATUS_RXHIST3_CNT_EXP)
3958         mp->rx_hist_cnt3 += RXMAC_HIST_CNT3_COUNT;
3959     if (val & XRXMAC_STATUS_RXHIST4_CNT_EXP)
3960         mp->rx_hist_cnt4 += RXMAC_HIST_CNT4_COUNT;
3961     if (val & XRXMAC_STATUS_RXHIST5_CNT_EXP)
3962         mp->rx_hist_cnt5 += RXMAC_HIST_CNT5_COUNT;
3963     if (val & XRXMAC_STATUS_RXHIST6_CNT_EXP)
3964         mp->rx_hist_cnt6 += RXMAC_HIST_CNT6_COUNT;
3965     if (val & XRXMAC_STATUS_RXHIST7_CNT_EXP)
3966         mp->rx_hist_cnt7 += RXMAC_HIST_CNT7_COUNT;
3967     if (val & XRXMAC_STATUS_RXOCTET_CNT_EXP)
3968         mp->rx_octets += RXMAC_BT_CNT_COUNT;
3969     if (val & XRXMAC_STATUS_CVIOLERR_CNT_EXP)
3970         mp->rx_code_violations += RXMAC_CD_VIO_CNT_COUNT;
3971     if (val & XRXMAC_STATUS_LENERR_CNT_EXP)
3972         mp->rx_len_errors += RXMAC_MPSZER_CNT_COUNT;
3973     if (val & XRXMAC_STATUS_CRCERR_CNT_EXP)
3974         mp->rx_crc_errors += RXMAC_CRC_ER_CNT_COUNT;
3975     if (val & XRXMAC_STATUS_RXUFLOW)
3976         mp->rx_underflows++;
3977     if (val & XRXMAC_STATUS_RXOFLOW)
3978         mp->rx_overflows++;
3979 
3980     val = nr64_mac(XMAC_FC_STAT);
3981     if (val & XMAC_FC_STAT_TX_MAC_NPAUSE)
3982         mp->pause_off_state++;
3983     if (val & XMAC_FC_STAT_TX_MAC_PAUSE)
3984         mp->pause_on_state++;
3985     if (val & XMAC_FC_STAT_RX_MAC_RPAUSE)
3986         mp->pause_received++;
3987 }
3988 
3989 static void niu_bmac_interrupt(struct niu *np)
3990 {
3991     struct niu_bmac_stats *mp = &np->mac_stats.bmac;
3992     u64 val;
3993 
3994     val = nr64_mac(BTXMAC_STATUS);
3995     if (val & BTXMAC_STATUS_UNDERRUN)
3996         mp->tx_underflow_errors++;
3997     if (val & BTXMAC_STATUS_MAX_PKT_ERR)
3998         mp->tx_max_pkt_size_errors++;
3999     if (val & BTXMAC_STATUS_BYTE_CNT_EXP)
4000         mp->tx_bytes += BTXMAC_BYTE_CNT_COUNT;
4001     if (val & BTXMAC_STATUS_FRAME_CNT_EXP)
4002         mp->tx_frames += BTXMAC_FRM_CNT_COUNT;
4003 
4004     val = nr64_mac(BRXMAC_STATUS);
4005     if (val & BRXMAC_STATUS_OVERFLOW)
4006         mp->rx_overflows++;
4007     if (val & BRXMAC_STATUS_FRAME_CNT_EXP)
4008         mp->rx_frames += BRXMAC_FRAME_CNT_COUNT;
4009     if (val & BRXMAC_STATUS_ALIGN_ERR_EXP)
4010         mp->rx_align_errors += BRXMAC_ALIGN_ERR_CNT_COUNT;
4011     if (val & BRXMAC_STATUS_CRC_ERR_EXP)
4012         mp->rx_crc_errors += BRXMAC_ALIGN_ERR_CNT_COUNT;
4013     if (val & BRXMAC_STATUS_LEN_ERR_EXP)
4014         mp->rx_len_errors += BRXMAC_CODE_VIOL_ERR_CNT_COUNT;
4015 
4016     val = nr64_mac(BMAC_CTRL_STATUS);
4017     if (val & BMAC_CTRL_STATUS_NOPAUSE)
4018         mp->pause_off_state++;
4019     if (val & BMAC_CTRL_STATUS_PAUSE)
4020         mp->pause_on_state++;
4021     if (val & BMAC_CTRL_STATUS_PAUSE_RECV)
4022         mp->pause_received++;
4023 }
4024 
4025 static int niu_mac_interrupt(struct niu *np)
4026 {
4027     if (np->flags & NIU_FLAGS_XMAC)
4028         niu_xmac_interrupt(np);
4029     else
4030         niu_bmac_interrupt(np);
4031 
4032     return 0;
4033 }
4034 
4035 static void niu_log_device_error(struct niu *np, u64 stat)
4036 {
4037     netdev_err(np->dev, "Core device errors ( ");
4038 
4039     if (stat & SYS_ERR_MASK_META2)
4040         pr_cont("META2 ");
4041     if (stat & SYS_ERR_MASK_META1)
4042         pr_cont("META1 ");
4043     if (stat & SYS_ERR_MASK_PEU)
4044         pr_cont("PEU ");
4045     if (stat & SYS_ERR_MASK_TXC)
4046         pr_cont("TXC ");
4047     if (stat & SYS_ERR_MASK_RDMC)
4048         pr_cont("RDMC ");
4049     if (stat & SYS_ERR_MASK_TDMC)
4050         pr_cont("TDMC ");
4051     if (stat & SYS_ERR_MASK_ZCP)
4052         pr_cont("ZCP ");
4053     if (stat & SYS_ERR_MASK_FFLP)
4054         pr_cont("FFLP ");
4055     if (stat & SYS_ERR_MASK_IPP)
4056         pr_cont("IPP ");
4057     if (stat & SYS_ERR_MASK_MAC)
4058         pr_cont("MAC ");
4059     if (stat & SYS_ERR_MASK_SMX)
4060         pr_cont("SMX ");
4061 
4062     pr_cont(")\n");
4063 }
4064 
4065 static int niu_device_error(struct niu *np)
4066 {
4067     u64 stat = nr64(SYS_ERR_STAT);
4068 
4069     netdev_err(np->dev, "Core device error, stat[%llx]\n",
4070            (unsigned long long)stat);
4071 
4072     niu_log_device_error(np, stat);
4073 
4074     return -ENODEV;
4075 }
4076 
4077 static int niu_slowpath_interrupt(struct niu *np, struct niu_ldg *lp,
4078                   u64 v0, u64 v1, u64 v2)
4079 {
4080 
4081     int i, err = 0;
4082 
4083     lp->v0 = v0;
4084     lp->v1 = v1;
4085     lp->v2 = v2;
4086 
4087     if (v1 & 0x00000000ffffffffULL) {
4088         u32 rx_vec = (v1 & 0xffffffff);
4089 
4090         for (i = 0; i < np->num_rx_rings; i++) {
4091             struct rx_ring_info *rp = &np->rx_rings[i];
4092 
4093             if (rx_vec & (1 << rp->rx_channel)) {
4094                 int r = niu_rx_error(np, rp);
4095                 if (r) {
4096                     err = r;
4097                 } else {
4098                     if (!v0)
4099                         nw64(RX_DMA_CTL_STAT(rp->rx_channel),
4100                              RX_DMA_CTL_STAT_MEX);
4101                 }
4102             }
4103         }
4104     }
4105     if (v1 & 0x7fffffff00000000ULL) {
4106         u32 tx_vec = (v1 >> 32) & 0x7fffffff;
4107 
4108         for (i = 0; i < np->num_tx_rings; i++) {
4109             struct tx_ring_info *rp = &np->tx_rings[i];
4110 
4111             if (tx_vec & (1 << rp->tx_channel)) {
4112                 int r = niu_tx_error(np, rp);
4113                 if (r)
4114                     err = r;
4115             }
4116         }
4117     }
4118     if ((v0 | v1) & 0x8000000000000000ULL) {
4119         int r = niu_mif_interrupt(np);
4120         if (r)
4121             err = r;
4122     }
4123     if (v2) {
4124         if (v2 & 0x01ef) {
4125             int r = niu_mac_interrupt(np);
4126             if (r)
4127                 err = r;
4128         }
4129         if (v2 & 0x0210) {
4130             int r = niu_device_error(np);
4131             if (r)
4132                 err = r;
4133         }
4134     }
4135 
4136     if (err)
4137         niu_enable_interrupts(np, 0);
4138 
4139     return err;
4140 }
4141 
4142 static void niu_rxchan_intr(struct niu *np, struct rx_ring_info *rp,
4143                 int ldn)
4144 {
4145     struct rxdma_mailbox *mbox = rp->mbox;
4146     u64 stat_write, stat = le64_to_cpup(&mbox->rx_dma_ctl_stat);
4147 
4148     stat_write = (RX_DMA_CTL_STAT_RCRTHRES |
4149               RX_DMA_CTL_STAT_RCRTO);
4150     nw64(RX_DMA_CTL_STAT(rp->rx_channel), stat_write);
4151 
4152     netif_printk(np, intr, KERN_DEBUG, np->dev,
4153              "%s() stat[%llx]\n", __func__, (unsigned long long)stat);
4154 }
4155 
4156 static void niu_txchan_intr(struct niu *np, struct tx_ring_info *rp,
4157                 int ldn)
4158 {
4159     rp->tx_cs = nr64(TX_CS(rp->tx_channel));
4160 
4161     netif_printk(np, intr, KERN_DEBUG, np->dev,
4162              "%s() cs[%llx]\n", __func__, (unsigned long long)rp->tx_cs);
4163 }
4164 
4165 static void __niu_fastpath_interrupt(struct niu *np, int ldg, u64 v0)
4166 {
4167     struct niu_parent *parent = np->parent;
4168     u32 rx_vec, tx_vec;
4169     int i;
4170 
4171     tx_vec = (v0 >> 32);
4172     rx_vec = (v0 & 0xffffffff);
4173 
4174     for (i = 0; i < np->num_rx_rings; i++) {
4175         struct rx_ring_info *rp = &np->rx_rings[i];
4176         int ldn = LDN_RXDMA(rp->rx_channel);
4177 
4178         if (parent->ldg_map[ldn] != ldg)
4179             continue;
4180 
4181         nw64(LD_IM0(ldn), LD_IM0_MASK);
4182         if (rx_vec & (1 << rp->rx_channel))
4183             niu_rxchan_intr(np, rp, ldn);
4184     }
4185 
4186     for (i = 0; i < np->num_tx_rings; i++) {
4187         struct tx_ring_info *rp = &np->tx_rings[i];
4188         int ldn = LDN_TXDMA(rp->tx_channel);
4189 
4190         if (parent->ldg_map[ldn] != ldg)
4191             continue;
4192 
4193         nw64(LD_IM0(ldn), LD_IM0_MASK);
4194         if (tx_vec & (1 << rp->tx_channel))
4195             niu_txchan_intr(np, rp, ldn);
4196     }
4197 }
4198 
4199 static void niu_schedule_napi(struct niu *np, struct niu_ldg *lp,
4200                   u64 v0, u64 v1, u64 v2)
4201 {
4202     if (likely(napi_schedule_prep(&lp->napi))) {
4203         lp->v0 = v0;
4204         lp->v1 = v1;
4205         lp->v2 = v2;
4206         __niu_fastpath_interrupt(np, lp->ldg_num, v0);
4207         __napi_schedule(&lp->napi);
4208     }
4209 }
4210 
4211 static irqreturn_t niu_interrupt(int irq, void *dev_id)
4212 {
4213     struct niu_ldg *lp = dev_id;
4214     struct niu *np = lp->np;
4215     int ldg = lp->ldg_num;
4216     unsigned long flags;
4217     u64 v0, v1, v2;
4218 
4219     if (netif_msg_intr(np))
4220         printk(KERN_DEBUG KBUILD_MODNAME ": " "%s() ldg[%p](%d)",
4221                __func__, lp, ldg);
4222 
4223     spin_lock_irqsave(&np->lock, flags);
4224 
4225     v0 = nr64(LDSV0(ldg));
4226     v1 = nr64(LDSV1(ldg));
4227     v2 = nr64(LDSV2(ldg));
4228 
4229     if (netif_msg_intr(np))
4230         pr_cont(" v0[%llx] v1[%llx] v2[%llx]\n",
4231                (unsigned long long) v0,
4232                (unsigned long long) v1,
4233                (unsigned long long) v2);
4234 
4235     if (unlikely(!v0 && !v1 && !v2)) {
4236         spin_unlock_irqrestore(&np->lock, flags);
4237         return IRQ_NONE;
4238     }
4239 
4240     if (unlikely((v0 & ((u64)1 << LDN_MIF)) || v1 || v2)) {
4241         int err = niu_slowpath_interrupt(np, lp, v0, v1, v2);
4242         if (err)
4243             goto out;
4244     }
4245     if (likely(v0 & ~((u64)1 << LDN_MIF)))
4246         niu_schedule_napi(np, lp, v0, v1, v2);
4247     else
4248         niu_ldg_rearm(np, lp, 1);
4249 out:
4250     spin_unlock_irqrestore(&np->lock, flags);
4251 
4252     return IRQ_HANDLED;
4253 }
4254 
4255 static void niu_free_rx_ring_info(struct niu *np, struct rx_ring_info *rp)
4256 {
4257     if (rp->mbox) {
4258         np->ops->free_coherent(np->device,
4259                        sizeof(struct rxdma_mailbox),
4260                        rp->mbox, rp->mbox_dma);
4261         rp->mbox = NULL;
4262     }
4263     if (rp->rcr) {
4264         np->ops->free_coherent(np->device,
4265                        MAX_RCR_RING_SIZE * sizeof(__le64),
4266                        rp->rcr, rp->rcr_dma);
4267         rp->rcr = NULL;
4268         rp->rcr_table_size = 0;
4269         rp->rcr_index = 0;
4270     }
4271     if (rp->rbr) {
4272         niu_rbr_free(np, rp);
4273 
4274         np->ops->free_coherent(np->device,
4275                        MAX_RBR_RING_SIZE * sizeof(__le32),
4276                        rp->rbr, rp->rbr_dma);
4277         rp->rbr = NULL;
4278         rp->rbr_table_size = 0;
4279         rp->rbr_index = 0;
4280     }
4281     kfree(rp->rxhash);
4282     rp->rxhash = NULL;
4283 }
4284 
4285 static void niu_free_tx_ring_info(struct niu *np, struct tx_ring_info *rp)
4286 {
4287     if (rp->mbox) {
4288         np->ops->free_coherent(np->device,
4289                        sizeof(struct txdma_mailbox),
4290                        rp->mbox, rp->mbox_dma);
4291         rp->mbox = NULL;
4292     }
4293     if (rp->descr) {
4294         int i;
4295 
4296         for (i = 0; i < MAX_TX_RING_SIZE; i++) {
4297             if (rp->tx_buffs[i].skb)
4298                 (void) release_tx_packet(np, rp, i);
4299         }
4300 
4301         np->ops->free_coherent(np->device,
4302                        MAX_TX_RING_SIZE * sizeof(__le64),
4303                        rp->descr, rp->descr_dma);
4304         rp->descr = NULL;
4305         rp->pending = 0;
4306         rp->prod = 0;
4307         rp->cons = 0;
4308         rp->wrap_bit = 0;
4309     }
4310 }
4311 
4312 static void niu_free_channels(struct niu *np)
4313 {
4314     int i;
4315 
4316     if (np->rx_rings) {
4317         for (i = 0; i < np->num_rx_rings; i++) {
4318             struct rx_ring_info *rp = &np->rx_rings[i];
4319 
4320             niu_free_rx_ring_info(np, rp);
4321         }
4322         kfree(np->rx_rings);
4323         np->rx_rings = NULL;
4324         np->num_rx_rings = 0;
4325     }
4326 
4327     if (np->tx_rings) {
4328         for (i = 0; i < np->num_tx_rings; i++) {
4329             struct tx_ring_info *rp = &np->tx_rings[i];
4330 
4331             niu_free_tx_ring_info(np, rp);
4332         }
4333         kfree(np->tx_rings);
4334         np->tx_rings = NULL;
4335         np->num_tx_rings = 0;
4336     }
4337 }
4338 
4339 static int niu_alloc_rx_ring_info(struct niu *np,
4340                   struct rx_ring_info *rp)
4341 {
4342     BUILD_BUG_ON(sizeof(struct rxdma_mailbox) != 64);
4343 
4344     rp->rxhash = kcalloc(MAX_RBR_RING_SIZE, sizeof(struct page *),
4345                  GFP_KERNEL);
4346     if (!rp->rxhash)
4347         return -ENOMEM;
4348 
4349     rp->mbox = np->ops->alloc_coherent(np->device,
4350                        sizeof(struct rxdma_mailbox),
4351                        &rp->mbox_dma, GFP_KERNEL);
4352     if (!rp->mbox)
4353         return -ENOMEM;
4354     if ((unsigned long)rp->mbox & (64UL - 1)) {
4355         netdev_err(np->dev, "Coherent alloc gives misaligned RXDMA mailbox %p\n",
4356                rp->mbox);
4357         return -EINVAL;
4358     }
4359 
4360     rp->rcr = np->ops->alloc_coherent(np->device,
4361                       MAX_RCR_RING_SIZE * sizeof(__le64),
4362                       &rp->rcr_dma, GFP_KERNEL);
4363     if (!rp->rcr)
4364         return -ENOMEM;
4365     if ((unsigned long)rp->rcr & (64UL - 1)) {
4366         netdev_err(np->dev, "Coherent alloc gives misaligned RXDMA RCR table %p\n",
4367                rp->rcr);
4368         return -EINVAL;
4369     }
4370     rp->rcr_table_size = MAX_RCR_RING_SIZE;
4371     rp->rcr_index = 0;
4372 
4373     rp->rbr = np->ops->alloc_coherent(np->device,
4374                       MAX_RBR_RING_SIZE * sizeof(__le32),
4375                       &rp->rbr_dma, GFP_KERNEL);
4376     if (!rp->rbr)
4377         return -ENOMEM;
4378     if ((unsigned long)rp->rbr & (64UL - 1)) {
4379         netdev_err(np->dev, "Coherent alloc gives misaligned RXDMA RBR table %p\n",
4380                rp->rbr);
4381         return -EINVAL;
4382     }
4383     rp->rbr_table_size = MAX_RBR_RING_SIZE;
4384     rp->rbr_index = 0;
4385     rp->rbr_pending = 0;
4386 
4387     return 0;
4388 }
4389 
4390 static void niu_set_max_burst(struct niu *np, struct tx_ring_info *rp)
4391 {
4392     int mtu = np->dev->mtu;
4393 
4394     /* These values are recommended by the HW designers for fair
4395      * utilization of DRR amongst the rings.
4396      */
4397     rp->max_burst = mtu + 32;
4398     if (rp->max_burst > 4096)
4399         rp->max_burst = 4096;
4400 }
4401 
4402 static int niu_alloc_tx_ring_info(struct niu *np,
4403                   struct tx_ring_info *rp)
4404 {
4405     BUILD_BUG_ON(sizeof(struct txdma_mailbox) != 64);
4406 
4407     rp->mbox = np->ops->alloc_coherent(np->device,
4408                        sizeof(struct txdma_mailbox),
4409                        &rp->mbox_dma, GFP_KERNEL);
4410     if (!rp->mbox)
4411         return -ENOMEM;
4412     if ((unsigned long)rp->mbox & (64UL - 1)) {
4413         netdev_err(np->dev, "Coherent alloc gives misaligned TXDMA mailbox %p\n",
4414                rp->mbox);
4415         return -EINVAL;
4416     }
4417 
4418     rp->descr = np->ops->alloc_coherent(np->device,
4419                         MAX_TX_RING_SIZE * sizeof(__le64),
4420                         &rp->descr_dma, GFP_KERNEL);
4421     if (!rp->descr)
4422         return -ENOMEM;
4423     if ((unsigned long)rp->descr & (64UL - 1)) {
4424         netdev_err(np->dev, "Coherent alloc gives misaligned TXDMA descr table %p\n",
4425                rp->descr);
4426         return -EINVAL;
4427     }
4428 
4429     rp->pending = MAX_TX_RING_SIZE;
4430     rp->prod = 0;
4431     rp->cons = 0;
4432     rp->wrap_bit = 0;
4433 
4434     /* XXX make these configurable... XXX */
4435     rp->mark_freq = rp->pending / 4;
4436 
4437     niu_set_max_burst(np, rp);
4438 
4439     return 0;
4440 }
4441 
4442 static void niu_size_rbr(struct niu *np, struct rx_ring_info *rp)
4443 {
4444     u16 bss;
4445 
4446     bss = min(PAGE_SHIFT, 15);
4447 
4448     rp->rbr_block_size = 1 << bss;
4449     rp->rbr_blocks_per_page = 1 << (PAGE_SHIFT-bss);
4450 
4451     rp->rbr_sizes[0] = 256;
4452     rp->rbr_sizes[1] = 1024;
4453     if (np->dev->mtu > ETH_DATA_LEN) {
4454         switch (PAGE_SIZE) {
4455         case 4 * 1024:
4456             rp->rbr_sizes[2] = 4096;
4457             break;
4458 
4459         default:
4460             rp->rbr_sizes[2] = 8192;
4461             break;
4462         }
4463     } else {
4464         rp->rbr_sizes[2] = 2048;
4465     }
4466     rp->rbr_sizes[3] = rp->rbr_block_size;
4467 }
4468 
4469 static int niu_alloc_channels(struct niu *np)
4470 {
4471     struct niu_parent *parent = np->parent;
4472     int first_rx_channel, first_tx_channel;
4473     int num_rx_rings, num_tx_rings;
4474     struct rx_ring_info *rx_rings;
4475     struct tx_ring_info *tx_rings;
4476     int i, port, err;
4477 
4478     port = np->port;
4479     first_rx_channel = first_tx_channel = 0;
4480     for (i = 0; i < port; i++) {
4481         first_rx_channel += parent->rxchan_per_port[i];
4482         first_tx_channel += parent->txchan_per_port[i];
4483     }
4484 
4485     num_rx_rings = parent->rxchan_per_port[port];
4486     num_tx_rings = parent->txchan_per_port[port];
4487 
4488     rx_rings = kcalloc(num_rx_rings, sizeof(struct rx_ring_info),
4489                GFP_KERNEL);
4490     err = -ENOMEM;
4491     if (!rx_rings)
4492         goto out_err;
4493 
4494     np->num_rx_rings = num_rx_rings;
4495     smp_wmb();
4496     np->rx_rings = rx_rings;
4497 
4498     netif_set_real_num_rx_queues(np->dev, num_rx_rings);
4499 
4500     for (i = 0; i < np->num_rx_rings; i++) {
4501         struct rx_ring_info *rp = &np->rx_rings[i];
4502 
4503         rp->np = np;
4504         rp->rx_channel = first_rx_channel + i;
4505 
4506         err = niu_alloc_rx_ring_info(np, rp);
4507         if (err)
4508             goto out_err;
4509 
4510         niu_size_rbr(np, rp);
4511 
4512         /* XXX better defaults, configurable, etc... XXX */
4513         rp->nonsyn_window = 64;
4514         rp->nonsyn_threshold = rp->rcr_table_size - 64;
4515         rp->syn_window = 64;
4516         rp->syn_threshold = rp->rcr_table_size - 64;
4517         rp->rcr_pkt_threshold = 16;
4518         rp->rcr_timeout = 8;
4519         rp->rbr_kick_thresh = RBR_REFILL_MIN;
4520         if (rp->rbr_kick_thresh < rp->rbr_blocks_per_page)
4521             rp->rbr_kick_thresh = rp->rbr_blocks_per_page;
4522 
4523         err = niu_rbr_fill(np, rp, GFP_KERNEL);
4524         if (err)
4525             return err;
4526     }
4527 
4528     tx_rings = kcalloc(num_tx_rings, sizeof(struct tx_ring_info),
4529                GFP_KERNEL);
4530     err = -ENOMEM;
4531     if (!tx_rings)
4532         goto out_err;
4533 
4534     np->num_tx_rings = num_tx_rings;
4535     smp_wmb();
4536     np->tx_rings = tx_rings;
4537 
4538     netif_set_real_num_tx_queues(np->dev, num_tx_rings);
4539 
4540     for (i = 0; i < np->num_tx_rings; i++) {
4541         struct tx_ring_info *rp = &np->tx_rings[i];
4542 
4543         rp->np = np;
4544         rp->tx_channel = first_tx_channel + i;
4545 
4546         err = niu_alloc_tx_ring_info(np, rp);
4547         if (err)
4548             goto out_err;
4549     }
4550 
4551     return 0;
4552 
4553 out_err:
4554     niu_free_channels(np);
4555     return err;
4556 }
4557 
4558 static int niu_tx_cs_sng_poll(struct niu *np, int channel)
4559 {
4560     int limit = 1000;
4561 
4562     while (--limit > 0) {
4563         u64 val = nr64(TX_CS(channel));
4564         if (val & TX_CS_SNG_STATE)
4565             return 0;
4566     }
4567     return -ENODEV;
4568 }
4569 
4570 static int niu_tx_channel_stop(struct niu *np, int channel)
4571 {
4572     u64 val = nr64(TX_CS(channel));
4573 
4574     val |= TX_CS_STOP_N_GO;
4575     nw64(TX_CS(channel), val);
4576 
4577     return niu_tx_cs_sng_poll(np, channel);
4578 }
4579 
4580 static int niu_tx_cs_reset_poll(struct niu *np, int channel)
4581 {
4582     int limit = 1000;
4583 
4584     while (--limit > 0) {
4585         u64 val = nr64(TX_CS(channel));
4586         if (!(val & TX_CS_RST))
4587             return 0;
4588     }
4589     return -ENODEV;
4590 }
4591 
4592 static int niu_tx_channel_reset(struct niu *np, int channel)
4593 {
4594     u64 val = nr64(TX_CS(channel));
4595     int err;
4596 
4597     val |= TX_CS_RST;
4598     nw64(TX_CS(channel), val);
4599 
4600     err = niu_tx_cs_reset_poll(np, channel);
4601     if (!err)
4602         nw64(TX_RING_KICK(channel), 0);
4603 
4604     return err;
4605 }
4606 
4607 static int niu_tx_channel_lpage_init(struct niu *np, int channel)
4608 {
4609     u64 val;
4610 
4611     nw64(TX_LOG_MASK1(channel), 0);
4612     nw64(TX_LOG_VAL1(channel), 0);
4613     nw64(TX_LOG_MASK2(channel), 0);
4614     nw64(TX_LOG_VAL2(channel), 0);
4615     nw64(TX_LOG_PAGE_RELO1(channel), 0);
4616     nw64(TX_LOG_PAGE_RELO2(channel), 0);
4617     nw64(TX_LOG_PAGE_HDL(channel), 0);
4618 
4619     val  = (u64)np->port << TX_LOG_PAGE_VLD_FUNC_SHIFT;
4620     val |= (TX_LOG_PAGE_VLD_PAGE0 | TX_LOG_PAGE_VLD_PAGE1);
4621     nw64(TX_LOG_PAGE_VLD(channel), val);
4622 
4623     /* XXX TXDMA 32bit mode? XXX */
4624 
4625     return 0;
4626 }
4627 
4628 static void niu_txc_enable_port(struct niu *np, int on)
4629 {
4630     unsigned long flags;
4631     u64 val, mask;
4632 
4633     niu_lock_parent(np, flags);
4634     val = nr64(TXC_CONTROL);
4635     mask = (u64)1 << np->port;
4636     if (on) {
4637         val |= TXC_CONTROL_ENABLE | mask;
4638     } else {
4639         val &= ~mask;
4640         if ((val & ~TXC_CONTROL_ENABLE) == 0)
4641             val &= ~TXC_CONTROL_ENABLE;
4642     }
4643     nw64(TXC_CONTROL, val);
4644     niu_unlock_parent(np, flags);
4645 }
4646 
4647 static void niu_txc_set_imask(struct niu *np, u64 imask)
4648 {
4649     unsigned long flags;
4650     u64 val;
4651 
4652     niu_lock_parent(np, flags);
4653     val = nr64(TXC_INT_MASK);
4654     val &= ~TXC_INT_MASK_VAL(np->port);
4655     val |= (imask << TXC_INT_MASK_VAL_SHIFT(np->port));
4656     niu_unlock_parent(np, flags);
4657 }
4658 
4659 static void niu_txc_port_dma_enable(struct niu *np, int on)
4660 {
4661     u64 val = 0;
4662 
4663     if (on) {
4664         int i;
4665 
4666         for (i = 0; i < np->num_tx_rings; i++)
4667             val |= (1 << np->tx_rings[i].tx_channel);
4668     }
4669     nw64(TXC_PORT_DMA(np->port), val);
4670 }
4671 
4672 static int niu_init_one_tx_channel(struct niu *np, struct tx_ring_info *rp)
4673 {
4674     int err, channel = rp->tx_channel;
4675     u64 val, ring_len;
4676 
4677     err = niu_tx_channel_stop(np, channel);
4678     if (err)
4679         return err;
4680 
4681     err = niu_tx_channel_reset(np, channel);
4682     if (err)
4683         return err;
4684 
4685     err = niu_tx_channel_lpage_init(np, channel);
4686     if (err)
4687         return err;
4688 
4689     nw64(TXC_DMA_MAX(channel), rp->max_burst);
4690     nw64(TX_ENT_MSK(channel), 0);
4691 
4692     if (rp->descr_dma & ~(TX_RNG_CFIG_STADDR_BASE |
4693                   TX_RNG_CFIG_STADDR)) {
4694         netdev_err(np->dev, "TX ring channel %d DMA addr (%llx) is not aligned\n",
4695                channel, (unsigned long long)rp->descr_dma);
4696         return -EINVAL;
4697     }
4698 
4699     /* The length field in TX_RNG_CFIG is measured in 64-byte
4700      * blocks.  rp->pending is the number of TX descriptors in
4701      * our ring, 8 bytes each, thus we divide by 8 bytes more
4702      * to get the proper value the chip wants.
4703      */
4704     ring_len = (rp->pending / 8);
4705 
4706     val = ((ring_len << TX_RNG_CFIG_LEN_SHIFT) |
4707            rp->descr_dma);
4708     nw64(TX_RNG_CFIG(channel), val);
4709 
4710     if (((rp->mbox_dma >> 32) & ~TXDMA_MBH_MBADDR) ||
4711         ((u32)rp->mbox_dma & ~TXDMA_MBL_MBADDR)) {
4712         netdev_err(np->dev, "TX ring channel %d MBOX addr (%llx) has invalid bits\n",
4713                 channel, (unsigned long long)rp->mbox_dma);
4714         return -EINVAL;
4715     }
4716     nw64(TXDMA_MBH(channel), rp->mbox_dma >> 32);
4717     nw64(TXDMA_MBL(channel), rp->mbox_dma & TXDMA_MBL_MBADDR);
4718 
4719     nw64(TX_CS(channel), 0);
4720 
4721     rp->last_pkt_cnt = 0;
4722 
4723     return 0;
4724 }
4725 
4726 static void niu_init_rdc_groups(struct niu *np)
4727 {
4728     struct niu_rdc_tables *tp = &np->parent->rdc_group_cfg[np->port];
4729     int i, first_table_num = tp->first_table_num;
4730 
4731     for (i = 0; i < tp->num_tables; i++) {
4732         struct rdc_table *tbl = &tp->tables[i];
4733         int this_table = first_table_num + i;
4734         int slot;
4735 
4736         for (slot = 0; slot < NIU_RDC_TABLE_SLOTS; slot++)
4737             nw64(RDC_TBL(this_table, slot),
4738                  tbl->rxdma_channel[slot]);
4739     }
4740 
4741     nw64(DEF_RDC(np->port), np->parent->rdc_default[np->port]);
4742 }
4743 
4744 static void niu_init_drr_weight(struct niu *np)
4745 {
4746     int type = phy_decode(np->parent->port_phy, np->port);
4747     u64 val;
4748 
4749     switch (type) {
4750     case PORT_TYPE_10G:
4751         val = PT_DRR_WEIGHT_DEFAULT_10G;
4752         break;
4753 
4754     case PORT_TYPE_1G:
4755     default:
4756         val = PT_DRR_WEIGHT_DEFAULT_1G;
4757         break;
4758     }
4759     nw64(PT_DRR_WT(np->port), val);
4760 }
4761 
4762 static int niu_init_hostinfo(struct niu *np)
4763 {
4764     struct niu_parent *parent = np->parent;
4765     struct niu_rdc_tables *tp = &parent->rdc_group_cfg[np->port];
4766     int i, err, num_alt = niu_num_alt_addr(np);
4767     int first_rdc_table = tp->first_table_num;
4768 
4769     err = niu_set_primary_mac_rdc_table(np, first_rdc_table, 1);
4770     if (err)
4771         return err;
4772 
4773     err = niu_set_multicast_mac_rdc_table(np, first_rdc_table, 1);
4774     if (err)
4775         return err;
4776 
4777     for (i = 0; i < num_alt; i++) {
4778         err = niu_set_alt_mac_rdc_table(np, i, first_rdc_table, 1);
4779         if (err)
4780             return err;
4781     }
4782 
4783     return 0;
4784 }
4785 
4786 static int niu_rx_channel_reset(struct niu *np, int channel)
4787 {
4788     return niu_set_and_wait_clear(np, RXDMA_CFIG1(channel),
4789                       RXDMA_CFIG1_RST, 1000, 10,
4790                       "RXDMA_CFIG1");
4791 }
4792 
4793 static int niu_rx_channel_lpage_init(struct niu *np, int channel)
4794 {
4795     u64 val;
4796 
4797     nw64(RX_LOG_MASK1(channel), 0);
4798     nw64(RX_LOG_VAL1(channel), 0);
4799     nw64(RX_LOG_MASK2(channel), 0);
4800     nw64(RX_LOG_VAL2(channel), 0);
4801     nw64(RX_LOG_PAGE_RELO1(channel), 0);
4802     nw64(RX_LOG_PAGE_RELO2(channel), 0);
4803     nw64(RX_LOG_PAGE_HDL(channel), 0);
4804 
4805     val  = (u64)np->port << RX_LOG_PAGE_VLD_FUNC_SHIFT;
4806     val |= (RX_LOG_PAGE_VLD_PAGE0 | RX_LOG_PAGE_VLD_PAGE1);
4807     nw64(RX_LOG_PAGE_VLD(channel), val);
4808 
4809     return 0;
4810 }
4811 
4812 static void niu_rx_channel_wred_init(struct niu *np, struct rx_ring_info *rp)
4813 {
4814     u64 val;
4815 
4816     val = (((u64)rp->nonsyn_window << RDC_RED_PARA_WIN_SHIFT) |
4817            ((u64)rp->nonsyn_threshold << RDC_RED_PARA_THRE_SHIFT) |
4818            ((u64)rp->syn_window << RDC_RED_PARA_WIN_SYN_SHIFT) |
4819            ((u64)rp->syn_threshold << RDC_RED_PARA_THRE_SYN_SHIFT));
4820     nw64(RDC_RED_PARA(rp->rx_channel), val);
4821 }
4822 
4823 static int niu_compute_rbr_cfig_b(struct rx_ring_info *rp, u64 *ret)
4824 {
4825     u64 val = 0;
4826 
4827     *ret = 0;
4828     switch (rp->rbr_block_size) {
4829     case 4 * 1024:
4830         val |= (RBR_BLKSIZE_4K << RBR_CFIG_B_BLKSIZE_SHIFT);
4831         break;
4832     case 8 * 1024:
4833         val |= (RBR_BLKSIZE_8K << RBR_CFIG_B_BLKSIZE_SHIFT);
4834         break;
4835     case 16 * 1024:
4836         val |= (RBR_BLKSIZE_16K << RBR_CFIG_B_BLKSIZE_SHIFT);
4837         break;
4838     case 32 * 1024:
4839         val |= (RBR_BLKSIZE_32K << RBR_CFIG_B_BLKSIZE_SHIFT);
4840         break;
4841     default:
4842         return -EINVAL;
4843     }
4844     val |= RBR_CFIG_B_VLD2;
4845     switch (rp->rbr_sizes[2]) {
4846     case 2 * 1024:
4847         val |= (RBR_BUFSZ2_2K << RBR_CFIG_B_BUFSZ2_SHIFT);
4848         break;
4849     case 4 * 1024:
4850         val |= (RBR_BUFSZ2_4K << RBR_CFIG_B_BUFSZ2_SHIFT);
4851         break;
4852     case 8 * 1024:
4853         val |= (RBR_BUFSZ2_8K << RBR_CFIG_B_BUFSZ2_SHIFT);
4854         break;
4855     case 16 * 1024:
4856         val |= (RBR_BUFSZ2_16K << RBR_CFIG_B_BUFSZ2_SHIFT);
4857         break;
4858 
4859     default:
4860         return -EINVAL;
4861     }
4862     val |= RBR_CFIG_B_VLD1;
4863     switch (rp->rbr_sizes[1]) {
4864     case 1 * 1024:
4865         val |= (RBR_BUFSZ1_1K << RBR_CFIG_B_BUFSZ1_SHIFT);
4866         break;
4867     case 2 * 1024:
4868         val |= (RBR_BUFSZ1_2K << RBR_CFIG_B_BUFSZ1_SHIFT);
4869         break;
4870     case 4 * 1024:
4871         val |= (RBR_BUFSZ1_4K << RBR_CFIG_B_BUFSZ1_SHIFT);
4872         break;
4873     case 8 * 1024:
4874         val |= (RBR_BUFSZ1_8K << RBR_CFIG_B_BUFSZ1_SHIFT);
4875         break;
4876 
4877     default:
4878         return -EINVAL;
4879     }
4880     val |= RBR_CFIG_B_VLD0;
4881     switch (rp->rbr_sizes[0]) {
4882     case 256:
4883         val |= (RBR_BUFSZ0_256 << RBR_CFIG_B_BUFSZ0_SHIFT);
4884         break;
4885     case 512:
4886         val |= (RBR_BUFSZ0_512 << RBR_CFIG_B_BUFSZ0_SHIFT);
4887         break;
4888     case 1 * 1024:
4889         val |= (RBR_BUFSZ0_1K << RBR_CFIG_B_BUFSZ0_SHIFT);
4890         break;
4891     case 2 * 1024:
4892         val |= (RBR_BUFSZ0_2K << RBR_CFIG_B_BUFSZ0_SHIFT);
4893         break;
4894 
4895     default:
4896         return -EINVAL;
4897     }
4898 
4899     *ret = val;
4900     return 0;
4901 }
4902 
4903 static int niu_enable_rx_channel(struct niu *np, int channel, int on)
4904 {
4905     u64 val = nr64(RXDMA_CFIG1(channel));
4906     int limit;
4907 
4908     if (on)
4909         val |= RXDMA_CFIG1_EN;
4910     else
4911         val &= ~RXDMA_CFIG1_EN;
4912     nw64(RXDMA_CFIG1(channel), val);
4913 
4914     limit = 1000;
4915     while (--limit > 0) {
4916         if (nr64(RXDMA_CFIG1(channel)) & RXDMA_CFIG1_QST)
4917             break;
4918         udelay(10);
4919     }
4920     if (limit <= 0)
4921         return -ENODEV;
4922     return 0;
4923 }
4924 
4925 static int niu_init_one_rx_channel(struct niu *np, struct rx_ring_info *rp)
4926 {
4927     int err, channel = rp->rx_channel;
4928     u64 val;
4929 
4930     err = niu_rx_channel_reset(np, channel);
4931     if (err)
4932         return err;
4933 
4934     err = niu_rx_channel_lpage_init(np, channel);
4935     if (err)
4936         return err;
4937 
4938     niu_rx_channel_wred_init(np, rp);
4939 
4940     nw64(RX_DMA_ENT_MSK(channel), RX_DMA_ENT_MSK_RBR_EMPTY);
4941     nw64(RX_DMA_CTL_STAT(channel),
4942          (RX_DMA_CTL_STAT_MEX |
4943           RX_DMA_CTL_STAT_RCRTHRES |
4944           RX_DMA_CTL_STAT_RCRTO |
4945           RX_DMA_CTL_STAT_RBR_EMPTY));
4946     nw64(RXDMA_CFIG1(channel), rp->mbox_dma >> 32);
4947     nw64(RXDMA_CFIG2(channel),
4948          ((rp->mbox_dma & RXDMA_CFIG2_MBADDR_L) |
4949           RXDMA_CFIG2_FULL_HDR));
4950     nw64(RBR_CFIG_A(channel),
4951          ((u64)rp->rbr_table_size << RBR_CFIG_A_LEN_SHIFT) |
4952          (rp->rbr_dma & (RBR_CFIG_A_STADDR_BASE | RBR_CFIG_A_STADDR)));
4953     err = niu_compute_rbr_cfig_b(rp, &val);
4954     if (err)
4955         return err;
4956     nw64(RBR_CFIG_B(channel), val);
4957     nw64(RCRCFIG_A(channel),
4958          ((u64)rp->rcr_table_size << RCRCFIG_A_LEN_SHIFT) |
4959          (rp->rcr_dma & (RCRCFIG_A_STADDR_BASE | RCRCFIG_A_STADDR)));
4960     nw64(RCRCFIG_B(channel),
4961          ((u64)rp->rcr_pkt_threshold << RCRCFIG_B_PTHRES_SHIFT) |
4962          RCRCFIG_B_ENTOUT |
4963          ((u64)rp->rcr_timeout << RCRCFIG_B_TIMEOUT_SHIFT));
4964 
4965     err = niu_enable_rx_channel(np, channel, 1);
4966     if (err)
4967         return err;
4968 
4969     nw64(RBR_KICK(channel), rp->rbr_index);
4970 
4971     val = nr64(RX_DMA_CTL_STAT(channel));
4972     val |= RX_DMA_CTL_STAT_RBR_EMPTY;
4973     nw64(RX_DMA_CTL_STAT(channel), val);
4974 
4975     return 0;
4976 }
4977 
4978 static int niu_init_rx_channels(struct niu *np)
4979 {
4980     unsigned long flags;
4981     u64 seed = jiffies_64;
4982     int err, i;
4983 
4984     niu_lock_parent(np, flags);
4985     nw64(RX_DMA_CK_DIV, np->parent->rxdma_clock_divider);
4986     nw64(RED_RAN_INIT, RED_RAN_INIT_OPMODE | (seed & RED_RAN_INIT_VAL));
4987     niu_unlock_parent(np, flags);
4988 
4989     /* XXX RXDMA 32bit mode? XXX */
4990 
4991     niu_init_rdc_groups(np);
4992     niu_init_drr_weight(np);
4993 
4994     err = niu_init_hostinfo(np);
4995     if (err)
4996         return err;
4997 
4998     for (i = 0; i < np->num_rx_rings; i++) {
4999         struct rx_ring_info *rp = &np->rx_rings[i];
5000 
5001         err = niu_init_one_rx_channel(np, rp);
5002         if (err)
5003             return err;
5004     }
5005 
5006     return 0;
5007 }
5008 
5009 static int niu_set_ip_frag_rule(struct niu *np)
5010 {
5011     struct niu_parent *parent = np->parent;
5012     struct niu_classifier *cp = &np->clas;
5013     struct niu_tcam_entry *tp;
5014     int index, err;
5015 
5016     index = cp->tcam_top;
5017     tp = &parent->tcam[index];
5018 
5019     /* Note that the noport bit is the same in both ipv4 and
5020      * ipv6 format TCAM entries.
5021      */
5022     memset(tp, 0, sizeof(*tp));
5023     tp->key[1] = TCAM_V4KEY1_NOPORT;
5024     tp->key_mask[1] = TCAM_V4KEY1_NOPORT;
5025     tp->assoc_data = (TCAM_ASSOCDATA_TRES_USE_OFFSET |
5026               ((u64)0 << TCAM_ASSOCDATA_OFFSET_SHIFT));
5027     err = tcam_write(np, index, tp->key, tp->key_mask);
5028     if (err)
5029         return err;
5030     err = tcam_assoc_write(np, index, tp->assoc_data);
5031     if (err)
5032         return err;
5033     tp->valid = 1;
5034     cp->tcam_valid_entries++;
5035 
5036     return 0;
5037 }
5038 
5039 static int niu_init_classifier_hw(struct niu *np)
5040 {
5041     struct niu_parent *parent = np->parent;
5042     struct niu_classifier *cp = &np->clas;
5043     int i, err;
5044 
5045     nw64(H1POLY, cp->h1_init);
5046     nw64(H2POLY, cp->h2_init);
5047 
5048     err = niu_init_hostinfo(np);
5049     if (err)
5050         return err;
5051 
5052     for (i = 0; i < ENET_VLAN_TBL_NUM_ENTRIES; i++) {
5053         struct niu_vlan_rdc *vp = &cp->vlan_mappings[i];
5054 
5055         vlan_tbl_write(np, i, np->port,
5056                    vp->vlan_pref, vp->rdc_num);
5057     }
5058 
5059     for (i = 0; i < cp->num_alt_mac_mappings; i++) {
5060         struct niu_altmac_rdc *ap = &cp->alt_mac_mappings[i];
5061 
5062         err = niu_set_alt_mac_rdc_table(np, ap->alt_mac_num,
5063                         ap->rdc_num, ap->mac_pref);
5064         if (err)
5065             return err;
5066     }
5067 
5068     for (i = CLASS_CODE_USER_PROG1; i <= CLASS_CODE_SCTP_IPV6; i++) {
5069         int index = i - CLASS_CODE_USER_PROG1;
5070 
5071         err = niu_set_tcam_key(np, i, parent->tcam_key[index]);
5072         if (err)
5073             return err;
5074         err = niu_set_flow_key(np, i, parent->flow_key[index]);
5075         if (err)
5076             return err;
5077     }
5078 
5079     err = niu_set_ip_frag_rule(np);
5080     if (err)
5081         return err;
5082 
5083     tcam_enable(np, 1);
5084 
5085     return 0;
5086 }
5087 
5088 static int niu_zcp_write(struct niu *np, int index, u64 *data)
5089 {
5090     nw64(ZCP_RAM_DATA0, data[0]);
5091     nw64(ZCP_RAM_DATA1, data[1]);
5092     nw64(ZCP_RAM_DATA2, data[2]);
5093     nw64(ZCP_RAM_DATA3, data[3]);
5094     nw64(ZCP_RAM_DATA4, data[4]);
5095     nw64(ZCP_RAM_BE, ZCP_RAM_BE_VAL);
5096     nw64(ZCP_RAM_ACC,
5097          (ZCP_RAM_ACC_WRITE |
5098           (0 << ZCP_RAM_ACC_ZFCID_SHIFT) |
5099           (ZCP_RAM_SEL_CFIFO(np->port) << ZCP_RAM_ACC_RAM_SEL_SHIFT)));
5100 
5101     return niu_wait_bits_clear(np, ZCP_RAM_ACC, ZCP_RAM_ACC_BUSY,
5102                    1000, 100);
5103 }
5104 
5105 static int niu_zcp_read(struct niu *np, int index, u64 *data)
5106 {
5107     int err;
5108 
5109     err = niu_wait_bits_clear(np, ZCP_RAM_ACC, ZCP_RAM_ACC_BUSY,
5110                   1000, 100);
5111     if (err) {
5112         netdev_err(np->dev, "ZCP read busy won't clear, ZCP_RAM_ACC[%llx]\n",
5113                (unsigned long long)nr64(ZCP_RAM_ACC));
5114         return err;
5115     }
5116 
5117     nw64(ZCP_RAM_ACC,
5118          (ZCP_RAM_ACC_READ |
5119           (0 << ZCP_RAM_ACC_ZFCID_SHIFT) |
5120           (ZCP_RAM_SEL_CFIFO(np->port) << ZCP_RAM_ACC_RAM_SEL_SHIFT)));
5121 
5122     err = niu_wait_bits_clear(np, ZCP_RAM_ACC, ZCP_RAM_ACC_BUSY,
5123                   1000, 100);
5124     if (err) {
5125         netdev_err(np->dev, "ZCP read busy2 won't clear, ZCP_RAM_ACC[%llx]\n",
5126                (unsigned long long)nr64(ZCP_RAM_ACC));
5127         return err;
5128     }
5129 
5130     data[0] = nr64(ZCP_RAM_DATA0);
5131     data[1] = nr64(ZCP_RAM_DATA1);
5132     data[2] = nr64(ZCP_RAM_DATA2);
5133     data[3] = nr64(ZCP_RAM_DATA3);
5134     data[4] = nr64(ZCP_RAM_DATA4);
5135 
5136     return 0;
5137 }
5138 
5139 static void niu_zcp_cfifo_reset(struct niu *np)
5140 {
5141     u64 val = nr64(RESET_CFIFO);
5142 
5143     val |= RESET_CFIFO_RST(np->port);
5144     nw64(RESET_CFIFO, val);
5145     udelay(10);
5146 
5147     val &= ~RESET_CFIFO_RST(np->port);
5148     nw64(RESET_CFIFO, val);
5149 }
5150 
5151 static int niu_init_zcp(struct niu *np)
5152 {
5153     u64 data[5], rbuf[5];
5154     int i, max, err;
5155 
5156     if (np->parent->plat_type != PLAT_TYPE_NIU) {
5157         if (np->port == 0 || np->port == 1)
5158             max = ATLAS_P0_P1_CFIFO_ENTRIES;
5159         else
5160             max = ATLAS_P2_P3_CFIFO_ENTRIES;
5161     } else
5162         max = NIU_CFIFO_ENTRIES;
5163 
5164     data[0] = 0;
5165     data[1] = 0;
5166     data[2] = 0;
5167     data[3] = 0;
5168     data[4] = 0;
5169 
5170     for (i = 0; i < max; i++) {
5171         err = niu_zcp_write(np, i, data);
5172         if (err)
5173             return err;
5174         err = niu_zcp_read(np, i, rbuf);
5175         if (err)
5176             return err;
5177     }
5178 
5179     niu_zcp_cfifo_reset(np);
5180     nw64(CFIFO_ECC(np->port), 0);
5181     nw64(ZCP_INT_STAT, ZCP_INT_STAT_ALL);
5182     (void) nr64(ZCP_INT_STAT);
5183     nw64(ZCP_INT_MASK, ZCP_INT_MASK_ALL);
5184 
5185     return 0;
5186 }
5187 
5188 static void niu_ipp_write(struct niu *np, int index, u64 *data)
5189 {
5190     u64 val = nr64_ipp(IPP_CFIG);
5191 
5192     nw64_ipp(IPP_CFIG, val | IPP_CFIG_DFIFO_PIO_W);
5193     nw64_ipp(IPP_DFIFO_WR_PTR, index);
5194     nw64_ipp(IPP_DFIFO_WR0, data[0]);
5195     nw64_ipp(IPP_DFIFO_WR1, data[1]);
5196     nw64_ipp(IPP_DFIFO_WR2, data[2]);
5197     nw64_ipp(IPP_DFIFO_WR3, data[3]);
5198     nw64_ipp(IPP_DFIFO_WR4, data[4]);
5199     nw64_ipp(IPP_CFIG, val & ~IPP_CFIG_DFIFO_PIO_W);
5200 }
5201 
5202 static void niu_ipp_read(struct niu *np, int index, u64 *data)
5203 {
5204     nw64_ipp(IPP_DFIFO_RD_PTR, index);
5205     data[0] = nr64_ipp(IPP_DFIFO_RD0);
5206     data[1] = nr64_ipp(IPP_DFIFO_RD1);
5207     data[2] = nr64_ipp(IPP_DFIFO_RD2);
5208     data[3] = nr64_ipp(IPP_DFIFO_RD3);
5209     data[4] = nr64_ipp(IPP_DFIFO_RD4);
5210 }
5211 
5212 static int niu_ipp_reset(struct niu *np)
5213 {
5214     return niu_set_and_wait_clear_ipp(np, IPP_CFIG, IPP_CFIG_SOFT_RST,
5215                       1000, 100, "IPP_CFIG");
5216 }
5217 
5218 static int niu_init_ipp(struct niu *np)
5219 {
5220     u64 data[5], rbuf[5], val;
5221     int i, max, err;
5222 
5223     if (np->parent->plat_type != PLAT_TYPE_NIU) {
5224         if (np->port == 0 || np->port == 1)
5225             max = ATLAS_P0_P1_DFIFO_ENTRIES;
5226         else
5227             max = ATLAS_P2_P3_DFIFO_ENTRIES;
5228     } else
5229         max = NIU_DFIFO_ENTRIES;
5230 
5231     data[0] = 0;
5232     data[1] = 0;
5233     data[2] = 0;
5234     data[3] = 0;
5235     data[4] = 0;
5236 
5237     for (i = 0; i < max; i++) {
5238         niu_ipp_write(np, i, data);
5239         niu_ipp_read(np, i, rbuf);
5240     }
5241 
5242     (void) nr64_ipp(IPP_INT_STAT);
5243     (void) nr64_ipp(IPP_INT_STAT);
5244 
5245     err = niu_ipp_reset(np);
5246     if (err)
5247         return err;
5248 
5249     (void) nr64_ipp(IPP_PKT_DIS);
5250     (void) nr64_ipp(IPP_BAD_CS_CNT);
5251     (void) nr64_ipp(IPP_ECC);
5252 
5253     (void) nr64_ipp(IPP_INT_STAT);
5254 
5255     nw64_ipp(IPP_MSK, ~IPP_MSK_ALL);
5256 
5257     val = nr64_ipp(IPP_CFIG);
5258     val &= ~IPP_CFIG_IP_MAX_PKT;
5259     val |= (IPP_CFIG_IPP_ENABLE |
5260         IPP_CFIG_DFIFO_ECC_EN |
5261         IPP_CFIG_DROP_BAD_CRC |
5262         IPP_CFIG_CKSUM_EN |
5263         (0x1ffff << IPP_CFIG_IP_MAX_PKT_SHIFT));
5264     nw64_ipp(IPP_CFIG, val);
5265 
5266     return 0;
5267 }
5268 
5269 static void niu_handle_led(struct niu *np, int status)
5270 {
5271     u64 val;
5272     val = nr64_mac(XMAC_CONFIG);
5273 
5274     if ((np->flags & NIU_FLAGS_10G) != 0 &&
5275         (np->flags & NIU_FLAGS_FIBER) != 0) {
5276         if (status) {
5277             val |= XMAC_CONFIG_LED_POLARITY;
5278             val &= ~XMAC_CONFIG_FORCE_LED_ON;
5279         } else {
5280             val |= XMAC_CONFIG_FORCE_LED_ON;
5281             val &= ~XMAC_CONFIG_LED_POLARITY;
5282         }
5283     }
5284 
5285     nw64_mac(XMAC_CONFIG, val);
5286 }
5287 
5288 static void niu_init_xif_xmac(struct niu *np)
5289 {
5290     struct niu_link_config *lp = &np->link_config;
5291     u64 val;
5292 
5293     if (np->flags & NIU_FLAGS_XCVR_SERDES) {
5294         val = nr64(MIF_CONFIG);
5295         val |= MIF_CONFIG_ATCA_GE;
5296         nw64(MIF_CONFIG, val);
5297     }
5298 
5299     val = nr64_mac(XMAC_CONFIG);
5300     val &= ~XMAC_CONFIG_SEL_POR_CLK_SRC;
5301 
5302     val |= XMAC_CONFIG_TX_OUTPUT_EN;
5303 
5304     if (lp->loopback_mode == LOOPBACK_MAC) {
5305         val &= ~XMAC_CONFIG_SEL_POR_CLK_SRC;
5306         val |= XMAC_CONFIG_LOOPBACK;
5307     } else {
5308         val &= ~XMAC_CONFIG_LOOPBACK;
5309     }
5310 
5311     if (np->flags & NIU_FLAGS_10G) {
5312         val &= ~XMAC_CONFIG_LFS_DISABLE;
5313     } else {
5314         val |= XMAC_CONFIG_LFS_DISABLE;
5315         if (!(np->flags & NIU_FLAGS_FIBER) &&
5316             !(np->flags & NIU_FLAGS_XCVR_SERDES))
5317             val |= XMAC_CONFIG_1G_PCS_BYPASS;
5318         else
5319             val &= ~XMAC_CONFIG_1G_PCS_BYPASS;
5320     }
5321 
5322     val &= ~XMAC_CONFIG_10G_XPCS_BYPASS;
5323 
5324     if (lp->active_speed == SPEED_100)
5325         val |= XMAC_CONFIG_SEL_CLK_25MHZ;
5326     else
5327         val &= ~XMAC_CONFIG_SEL_CLK_25MHZ;
5328 
5329     nw64_mac(XMAC_CONFIG, val);
5330 
5331     val = nr64_mac(XMAC_CONFIG);
5332     val &= ~XMAC_CONFIG_MODE_MASK;
5333     if (np->flags & NIU_FLAGS_10G) {
5334         val |= XMAC_CONFIG_MODE_XGMII;
5335     } else {
5336         if (lp->active_speed == SPEED_1000)
5337             val |= XMAC_CONFIG_MODE_GMII;
5338         else
5339             val |= XMAC_CONFIG_MODE_MII;
5340     }
5341 
5342     nw64_mac(XMAC_CONFIG, val);
5343 }
5344 
5345 static void niu_init_xif_bmac(struct niu *np)
5346 {
5347     struct niu_link_config *lp = &np->link_config;
5348     u64 val;
5349 
5350     val = BMAC_XIF_CONFIG_TX_OUTPUT_EN;
5351 
5352     if (lp->loopback_mode == LOOPBACK_MAC)
5353         val |= BMAC_XIF_CONFIG_MII_LOOPBACK;
5354     else
5355         val &= ~BMAC_XIF_CONFIG_MII_LOOPBACK;
5356 
5357     if (lp->active_speed == SPEED_1000)
5358         val |= BMAC_XIF_CONFIG_GMII_MODE;
5359     else
5360         val &= ~BMAC_XIF_CONFIG_GMII_MODE;
5361 
5362     val &= ~(BMAC_XIF_CONFIG_LINK_LED |
5363          BMAC_XIF_CONFIG_LED_POLARITY);
5364 
5365     if (!(np->flags & NIU_FLAGS_10G) &&
5366         !(np->flags & NIU_FLAGS_FIBER) &&
5367         lp->active_speed == SPEED_100)
5368         val |= BMAC_XIF_CONFIG_25MHZ_CLOCK;
5369     else
5370         val &= ~BMAC_XIF_CONFIG_25MHZ_CLOCK;
5371 
5372     nw64_mac(BMAC_XIF_CONFIG, val);
5373 }
5374 
5375 static void niu_init_xif(struct niu *np)
5376 {
5377     if (np->flags & NIU_FLAGS_XMAC)
5378         niu_init_xif_xmac(np);
5379     else
5380         niu_init_xif_bmac(np);
5381 }
5382 
5383 static void niu_pcs_mii_reset(struct niu *np)
5384 {
5385     int limit = 1000;
5386     u64 val = nr64_pcs(PCS_MII_CTL);
5387     val |= PCS_MII_CTL_RST;
5388     nw64_pcs(PCS_MII_CTL, val);
5389     while ((--limit >= 0) && (val & PCS_MII_CTL_RST)) {
5390         udelay(100);
5391         val = nr64_pcs(PCS_MII_CTL);
5392     }
5393 }
5394 
5395 static void niu_xpcs_reset(struct niu *np)
5396 {
5397     int limit = 1000;
5398     u64 val = nr64_xpcs(XPCS_CONTROL1);
5399     val |= XPCS_CONTROL1_RESET;
5400     nw64_xpcs(XPCS_CONTROL1, val);
5401     while ((--limit >= 0) && (val & XPCS_CONTROL1_RESET)) {
5402         udelay(100);
5403         val = nr64_xpcs(XPCS_CONTROL1);
5404     }
5405 }
5406 
5407 static int niu_init_pcs(struct niu *np)
5408 {
5409     struct niu_link_config *lp = &np->link_config;
5410     u64 val;
5411 
5412     switch (np->flags & (NIU_FLAGS_10G |
5413                  NIU_FLAGS_FIBER |
5414                  NIU_FLAGS_XCVR_SERDES)) {
5415     case NIU_FLAGS_FIBER:
5416         /* 1G fiber */
5417         nw64_pcs(PCS_CONF, PCS_CONF_MASK | PCS_CONF_ENABLE);
5418         nw64_pcs(PCS_DPATH_MODE, 0);
5419         niu_pcs_mii_reset(np);
5420         break;
5421 
5422     case NIU_FLAGS_10G:
5423     case NIU_FLAGS_10G | NIU_FLAGS_FIBER:
5424     case NIU_FLAGS_10G | NIU_FLAGS_XCVR_SERDES:
5425         /* 10G SERDES */
5426         if (!(np->flags & NIU_FLAGS_XMAC))
5427             return -EINVAL;
5428 
5429         /* 10G copper or fiber */
5430         val = nr64_mac(XMAC_CONFIG);
5431         val &= ~XMAC_CONFIG_10G_XPCS_BYPASS;
5432         nw64_mac(XMAC_CONFIG, val);
5433 
5434         niu_xpcs_reset(np);
5435 
5436         val = nr64_xpcs(XPCS_CONTROL1);
5437         if (lp->loopback_mode == LOOPBACK_PHY)
5438             val |= XPCS_CONTROL1_LOOPBACK;
5439         else
5440             val &= ~XPCS_CONTROL1_LOOPBACK;
5441         nw64_xpcs(XPCS_CONTROL1, val);
5442 
5443         nw64_xpcs(XPCS_DESKEW_ERR_CNT, 0);
5444         (void) nr64_xpcs(XPCS_SYMERR_CNT01);
5445         (void) nr64_xpcs(XPCS_SYMERR_CNT23);
5446         break;
5447 
5448 
5449     case NIU_FLAGS_XCVR_SERDES:
5450         /* 1G SERDES */
5451         niu_pcs_mii_reset(np);
5452         nw64_pcs(PCS_CONF, PCS_CONF_MASK | PCS_CONF_ENABLE);
5453         nw64_pcs(PCS_DPATH_MODE, 0);
5454         break;
5455 
5456     case 0:
5457         /* 1G copper */
5458     case NIU_FLAGS_XCVR_SERDES | NIU_FLAGS_FIBER:
5459         /* 1G RGMII FIBER */
5460         nw64_pcs(PCS_DPATH_MODE, PCS_DPATH_MODE_MII);
5461         niu_pcs_mii_reset(np);
5462         break;
5463 
5464     default:
5465         return -EINVAL;
5466     }
5467 
5468     return 0;
5469 }
5470 
5471 static int niu_reset_tx_xmac(struct niu *np)
5472 {
5473     return niu_set_and_wait_clear_mac(np, XTXMAC_SW_RST,
5474                       (XTXMAC_SW_RST_REG_RS |
5475                        XTXMAC_SW_RST_SOFT_RST),
5476                       1000, 100, "XTXMAC_SW_RST");
5477 }
5478 
5479 static int niu_reset_tx_bmac(struct niu *np)
5480 {
5481     int limit;
5482 
5483     nw64_mac(BTXMAC_SW_RST, BTXMAC_SW_RST_RESET);
5484     limit = 1000;
5485     while (--limit >= 0) {
5486         if (!(nr64_mac(BTXMAC_SW_RST) & BTXMAC_SW_RST_RESET))
5487             break;
5488         udelay(100);
5489     }
5490     if (limit < 0) {
5491         dev_err(np->device, "Port %u TX BMAC would not reset, BTXMAC_SW_RST[%llx]\n",
5492             np->port,
5493             (unsigned long long) nr64_mac(BTXMAC_SW_RST));
5494         return -ENODEV;
5495     }
5496 
5497     return 0;
5498 }
5499 
5500 static int niu_reset_tx_mac(struct niu *np)
5501 {
5502     if (np->flags & NIU_FLAGS_XMAC)
5503         return niu_reset_tx_xmac(np);
5504     else
5505         return niu_reset_tx_bmac(np);
5506 }
5507 
5508 static void niu_init_tx_xmac(struct niu *np, u64 min, u64 max)
5509 {
5510     u64 val;
5511 
5512     val = nr64_mac(XMAC_MIN);
5513     val &= ~(XMAC_MIN_TX_MIN_PKT_SIZE |
5514          XMAC_MIN_RX_MIN_PKT_SIZE);
5515     val |= (min << XMAC_MIN_RX_MIN_PKT_SIZE_SHFT);
5516     val |= (min << XMAC_MIN_TX_MIN_PKT_SIZE_SHFT);
5517     nw64_mac(XMAC_MIN, val);
5518 
5519     nw64_mac(XMAC_MAX, max);
5520 
5521     nw64_mac(XTXMAC_STAT_MSK, ~(u64)0);
5522 
5523     val = nr64_mac(XMAC_IPG);
5524     if (np->flags & NIU_FLAGS_10G) {
5525         val &= ~XMAC_IPG_IPG_XGMII;
5526         val |= (IPG_12_15_XGMII << XMAC_IPG_IPG_XGMII_SHIFT);
5527     } else {
5528         val &= ~XMAC_IPG_IPG_MII_GMII;
5529         val |= (IPG_12_MII_GMII << XMAC_IPG_IPG_MII_GMII_SHIFT);
5530     }
5531     nw64_mac(XMAC_IPG, val);
5532 
5533     val = nr64_mac(XMAC_CONFIG);
5534     val &= ~(XMAC_CONFIG_ALWAYS_NO_CRC |
5535          XMAC_CONFIG_STRETCH_MODE |
5536          XMAC_CONFIG_VAR_MIN_IPG_EN |
5537          XMAC_CONFIG_TX_ENABLE);
5538     nw64_mac(XMAC_CONFIG, val);
5539 
5540     nw64_mac(TXMAC_FRM_CNT, 0);
5541     nw64_mac(TXMAC_BYTE_CNT, 0);
5542 }
5543 
5544 static void niu_init_tx_bmac(struct niu *np, u64 min, u64 max)
5545 {
5546     u64 val;
5547 
5548     nw64_mac(BMAC_MIN_FRAME, min);
5549     nw64_mac(BMAC_MAX_FRAME, max);
5550 
5551     nw64_mac(BTXMAC_STATUS_MASK, ~(u64)0);
5552     nw64_mac(BMAC_CTRL_TYPE, 0x8808);
5553     nw64_mac(BMAC_PREAMBLE_SIZE, 7);
5554 
5555     val = nr64_mac(BTXMAC_CONFIG);
5556     val &= ~(BTXMAC_CONFIG_FCS_DISABLE |
5557          BTXMAC_CONFIG_ENABLE);
5558     nw64_mac(BTXMAC_CONFIG, val);
5559 }
5560 
5561 static void niu_init_tx_mac(struct niu *np)
5562 {
5563     u64 min, max;
5564 
5565     min = 64;
5566     if (np->dev->mtu > ETH_DATA_LEN)
5567         max = 9216;
5568     else
5569         max = 1522;
5570 
5571     /* The XMAC_MIN register only accepts values for TX min which
5572      * have the low 3 bits cleared.
5573      */
5574     BUG_ON(min & 0x7);
5575 
5576     if (np->flags & NIU_FLAGS_XMAC)
5577         niu_init_tx_xmac(np, min, max);
5578     else
5579         niu_init_tx_bmac(np, min, max);
5580 }
5581 
5582 static int niu_reset_rx_xmac(struct niu *np)
5583 {
5584     int limit;
5585 
5586     nw64_mac(XRXMAC_SW_RST,
5587          XRXMAC_SW_RST_REG_RS | XRXMAC_SW_RST_SOFT_RST);
5588     limit = 1000;
5589     while (--limit >= 0) {
5590         if (!(nr64_mac(XRXMAC_SW_RST) & (XRXMAC_SW_RST_REG_RS |
5591                          XRXMAC_SW_RST_SOFT_RST)))
5592             break;
5593         udelay(100);
5594     }
5595     if (limit < 0) {
5596         dev_err(np->device, "Port %u RX XMAC would not reset, XRXMAC_SW_RST[%llx]\n",
5597             np->port,
5598             (unsigned long long) nr64_mac(XRXMAC_SW_RST));
5599         return -ENODEV;
5600     }
5601 
5602     return 0;
5603 }
5604 
5605 static int niu_reset_rx_bmac(struct niu *np)
5606 {
5607     int limit;
5608 
5609     nw64_mac(BRXMAC_SW_RST, BRXMAC_SW_RST_RESET);
5610     limit = 1000;
5611     while (--limit >= 0) {
5612         if (!(nr64_mac(BRXMAC_SW_RST) & BRXMAC_SW_RST_RESET))
5613             break;
5614         udelay(100);
5615     }
5616     if (limit < 0) {
5617         dev_err(np->device, "Port %u RX BMAC would not reset, BRXMAC_SW_RST[%llx]\n",
5618             np->port,
5619             (unsigned long long) nr64_mac(BRXMAC_SW_RST));
5620         return -ENODEV;
5621     }
5622 
5623     return 0;
5624 }
5625 
5626 static int niu_reset_rx_mac(struct niu *np)
5627 {
5628     if (np->flags & NIU_FLAGS_XMAC)
5629         return niu_reset_rx_xmac(np);
5630     else
5631         return niu_reset_rx_bmac(np);
5632 }
5633 
5634 static void niu_init_rx_xmac(struct niu *np)
5635 {
5636     struct niu_parent *parent = np->parent;
5637     struct niu_rdc_tables *tp = &parent->rdc_group_cfg[np->port];
5638     int first_rdc_table = tp->first_table_num;
5639     unsigned long i;
5640     u64 val;
5641 
5642     nw64_mac(XMAC_ADD_FILT0, 0);
5643     nw64_mac(XMAC_ADD_FILT1, 0);
5644     nw64_mac(XMAC_ADD_FILT2, 0);
5645     nw64_mac(XMAC_ADD_FILT12_MASK, 0);
5646     nw64_mac(XMAC_ADD_FILT00_MASK, 0);
5647     for (i = 0; i < MAC_NUM_HASH; i++)
5648         nw64_mac(XMAC_HASH_TBL(i), 0);
5649     nw64_mac(XRXMAC_STAT_MSK, ~(u64)0);
5650     niu_set_primary_mac_rdc_table(np, first_rdc_table, 1);
5651     niu_set_multicast_mac_rdc_table(np, first_rdc_table, 1);
5652 
5653     val = nr64_mac(XMAC_CONFIG);
5654     val &= ~(XMAC_CONFIG_RX_MAC_ENABLE |
5655          XMAC_CONFIG_PROMISCUOUS |
5656          XMAC_CONFIG_PROMISC_GROUP |
5657          XMAC_CONFIG_ERR_CHK_DIS |
5658          XMAC_CONFIG_RX_CRC_CHK_DIS |
5659          XMAC_CONFIG_RESERVED_MULTICAST |
5660          XMAC_CONFIG_RX_CODEV_CHK_DIS |
5661          XMAC_CONFIG_ADDR_FILTER_EN |
5662          XMAC_CONFIG_RCV_PAUSE_ENABLE |
5663          XMAC_CONFIG_STRIP_CRC |
5664          XMAC_CONFIG_PASS_FLOW_CTRL |
5665          XMAC_CONFIG_MAC2IPP_PKT_CNT_EN);
5666     val |= (XMAC_CONFIG_HASH_FILTER_EN);
5667     nw64_mac(XMAC_CONFIG, val);
5668 
5669     nw64_mac(RXMAC_BT_CNT, 0);
5670     nw64_mac(RXMAC_BC_FRM_CNT, 0);
5671     nw64_mac(RXMAC_MC_FRM_CNT, 0);
5672     nw64_mac(RXMAC_FRAG_CNT, 0);
5673     nw64_mac(RXMAC_HIST_CNT1, 0);
5674     nw64_mac(RXMAC_HIST_CNT2, 0);
5675     nw64_mac(RXMAC_HIST_CNT3, 0);
5676     nw64_mac(RXMAC_HIST_CNT4, 0);
5677     nw64_mac(RXMAC_HIST_CNT5, 0);
5678     nw64_mac(RXMAC_HIST_CNT6, 0);
5679     nw64_mac(RXMAC_HIST_CNT7, 0);
5680     nw64_mac(RXMAC_MPSZER_CNT, 0);
5681     nw64_mac(RXMAC_CRC_ER_CNT, 0);
5682     nw64_mac(RXMAC_CD_VIO_CNT, 0);
5683     nw64_mac(LINK_FAULT_CNT, 0);
5684 }
5685 
5686 static void niu_init_rx_bmac(struct niu *np)
5687 {
5688     struct niu_parent *parent = np->parent;
5689     struct niu_rdc_tables *tp = &parent->rdc_group_cfg[np->port];
5690     int first_rdc_table = tp->first_table_num;
5691     unsigned long i;
5692     u64 val;
5693 
5694     nw64_mac(BMAC_ADD_FILT0, 0);
5695     nw64_mac(BMAC_ADD_FILT1, 0);
5696     nw64_mac(BMAC_ADD_FILT2, 0);
5697     nw64_mac(BMAC_ADD_FILT12_MASK, 0);
5698     nw64_mac(BMAC_ADD_FILT00_MASK, 0);
5699     for (i = 0; i < MAC_NUM_HASH; i++)
5700         nw64_mac(BMAC_HASH_TBL(i), 0);
5701     niu_set_primary_mac_rdc_table(np, first_rdc_table, 1);
5702     niu_set_multicast_mac_rdc_table(np, first_rdc_table, 1);
5703     nw64_mac(BRXMAC_STATUS_MASK, ~(u64)0);
5704 
5705     val = nr64_mac(BRXMAC_CONFIG);
5706     val &= ~(BRXMAC_CONFIG_ENABLE |
5707          BRXMAC_CONFIG_STRIP_PAD |
5708          BRXMAC_CONFIG_STRIP_FCS |
5709          BRXMAC_CONFIG_PROMISC |
5710          BRXMAC_CONFIG_PROMISC_GRP |
5711          BRXMAC_CONFIG_ADDR_FILT_EN |
5712          BRXMAC_CONFIG_DISCARD_DIS);
5713     val |= (BRXMAC_CONFIG_HASH_FILT_EN);
5714     nw64_mac(BRXMAC_CONFIG, val);
5715 
5716     val = nr64_mac(BMAC_ADDR_CMPEN);
5717     val |= BMAC_ADDR_CMPEN_EN0;
5718     nw64_mac(BMAC_ADDR_CMPEN, val);
5719 }
5720 
5721 static void niu_init_rx_mac(struct niu *np)
5722 {
5723     niu_set_primary_mac(np, np->dev->dev_addr);
5724 
5725     if (np->flags & NIU_FLAGS_XMAC)
5726         niu_init_rx_xmac(np);
5727     else
5728         niu_init_rx_bmac(np);
5729 }
5730 
5731 static void niu_enable_tx_xmac(struct niu *np, int on)
5732 {
5733     u64 val = nr64_mac(XMAC_CONFIG);
5734 
5735     if (on)
5736         val |= XMAC_CONFIG_TX_ENABLE;
5737     else
5738         val &= ~XMAC_CONFIG_TX_ENABLE;
5739     nw64_mac(XMAC_CONFIG, val);
5740 }
5741 
5742 static void niu_enable_tx_bmac(struct niu *np, int on)
5743 {
5744     u64 val = nr64_mac(BTXMAC_CONFIG);
5745 
5746     if (on)
5747         val |= BTXMAC_CONFIG_ENABLE;
5748     else
5749         val &= ~BTXMAC_CONFIG_ENABLE;
5750     nw64_mac(BTXMAC_CONFIG, val);
5751 }
5752 
5753 static void niu_enable_tx_mac(struct niu *np, int on)
5754 {
5755     if (np->flags & NIU_FLAGS_XMAC)
5756         niu_enable_tx_xmac(np, on);
5757     else
5758         niu_enable_tx_bmac(np, on);
5759 }
5760 
5761 static void niu_enable_rx_xmac(struct niu *np, int on)
5762 {
5763     u64 val = nr64_mac(XMAC_CONFIG);
5764 
5765     val &= ~(XMAC_CONFIG_HASH_FILTER_EN |
5766          XMAC_CONFIG_PROMISCUOUS);
5767 
5768     if (np->flags & NIU_FLAGS_MCAST)
5769         val |= XMAC_CONFIG_HASH_FILTER_EN;
5770     if (np->flags & NIU_FLAGS_PROMISC)
5771         val |= XMAC_CONFIG_PROMISCUOUS;
5772 
5773     if (on)
5774         val |= XMAC_CONFIG_RX_MAC_ENABLE;
5775     else
5776         val &= ~XMAC_CONFIG_RX_MAC_ENABLE;
5777     nw64_mac(XMAC_CONFIG, val);
5778 }
5779 
5780 static void niu_enable_rx_bmac(struct niu *np, int on)
5781 {
5782     u64 val = nr64_mac(BRXMAC_CONFIG);
5783 
5784     val &= ~(BRXMAC_CONFIG_HASH_FILT_EN |
5785          BRXMAC_CONFIG_PROMISC);
5786 
5787     if (np->flags & NIU_FLAGS_MCAST)
5788         val |= BRXMAC_CONFIG_HASH_FILT_EN;
5789     if (np->flags & NIU_FLAGS_PROMISC)
5790         val |= BRXMAC_CONFIG_PROMISC;
5791 
5792     if (on)
5793         val |= BRXMAC_CONFIG_ENABLE;
5794     else
5795         val &= ~BRXMAC_CONFIG_ENABLE;
5796     nw64_mac(BRXMAC_CONFIG, val);
5797 }
5798 
5799 static void niu_enable_rx_mac(struct niu *np, int on)
5800 {
5801     if (np->flags & NIU_FLAGS_XMAC)
5802         niu_enable_rx_xmac(np, on);
5803     else
5804         niu_enable_rx_bmac(np, on);
5805 }
5806 
5807 static int niu_init_mac(struct niu *np)
5808 {
5809     int err;
5810 
5811     niu_init_xif(np);
5812     err = niu_init_pcs(np);
5813     if (err)
5814         return err;
5815 
5816     err = niu_reset_tx_mac(np);
5817     if (err)
5818         return err;
5819     niu_init_tx_mac(np);
5820     err = niu_reset_rx_mac(np);
5821     if (err)
5822         return err;
5823     niu_init_rx_mac(np);
5824 
5825     /* This looks hookey but the RX MAC reset we just did will
5826      * undo some of the state we setup in niu_init_tx_mac() so we
5827      * have to call it again.  In particular, the RX MAC reset will
5828      * set the XMAC_MAX register back to it's default value.
5829      */
5830     niu_init_tx_mac(np);
5831     niu_enable_tx_mac(np, 1);
5832 
5833     niu_enable_rx_mac(np, 1);
5834 
5835     return 0;
5836 }
5837 
5838 static void niu_stop_one_tx_channel(struct niu *np, struct tx_ring_info *rp)
5839 {
5840     (void) niu_tx_channel_stop(np, rp->tx_channel);
5841 }
5842 
5843 static void niu_stop_tx_channels(struct niu *np)
5844 {
5845     int i;
5846 
5847     for (i = 0; i < np->num_tx_rings; i++) {
5848         struct tx_ring_info *rp = &np->tx_rings[i];
5849 
5850         niu_stop_one_tx_channel(np, rp);
5851     }
5852 }
5853 
5854 static void niu_reset_one_tx_channel(struct niu *np, struct tx_ring_info *rp)
5855 {
5856     (void) niu_tx_channel_reset(np, rp->tx_channel);
5857 }
5858 
5859 static void niu_reset_tx_channels(struct niu *np)
5860 {
5861     int i;
5862 
5863     for (i = 0; i < np->num_tx_rings; i++) {
5864         struct tx_ring_info *rp = &np->tx_rings[i];
5865 
5866         niu_reset_one_tx_channel(np, rp);
5867     }
5868 }
5869 
5870 static void niu_stop_one_rx_channel(struct niu *np, struct rx_ring_info *rp)
5871 {
5872     (void) niu_enable_rx_channel(np, rp->rx_channel, 0);
5873 }
5874 
5875 static void niu_stop_rx_channels(struct niu *np)
5876 {
5877     int i;
5878 
5879     for (i = 0; i < np->num_rx_rings; i++) {
5880         struct rx_ring_info *rp = &np->rx_rings[i];
5881 
5882         niu_stop_one_rx_channel(np, rp);
5883     }
5884 }
5885 
5886 static void niu_reset_one_rx_channel(struct niu *np, struct rx_ring_info *rp)
5887 {
5888     int channel = rp->rx_channel;
5889 
5890     (void) niu_rx_channel_reset(np, channel);
5891     nw64(RX_DMA_ENT_MSK(channel), RX_DMA_ENT_MSK_ALL);
5892     nw64(RX_DMA_CTL_STAT(channel), 0);
5893     (void) niu_enable_rx_channel(np, channel, 0);
5894 }
5895 
5896 static void niu_reset_rx_channels(struct niu *np)
5897 {
5898     int i;
5899 
5900     for (i = 0; i < np->num_rx_rings; i++) {
5901         struct rx_ring_info *rp = &np->rx_rings[i];
5902 
5903         niu_reset_one_rx_channel(np, rp);
5904     }
5905 }
5906 
5907 static void niu_disable_ipp(struct niu *np)
5908 {
5909     u64 rd, wr, val;
5910     int limit;
5911 
5912     rd = nr64_ipp(IPP_DFIFO_RD_PTR);
5913     wr = nr64_ipp(IPP_DFIFO_WR_PTR);
5914     limit = 100;
5915     while (--limit >= 0 && (rd != wr)) {
5916         rd = nr64_ipp(IPP_DFIFO_RD_PTR);
5917         wr = nr64_ipp(IPP_DFIFO_WR_PTR);
5918     }
5919     if (limit < 0 &&
5920         (rd != 0 && wr != 1)) {
5921         netdev_err(np->dev, "IPP would not quiesce, rd_ptr[%llx] wr_ptr[%llx]\n",
5922                (unsigned long long)nr64_ipp(IPP_DFIFO_RD_PTR),
5923                (unsigned long long)nr64_ipp(IPP_DFIFO_WR_PTR));
5924     }
5925 
5926     val = nr64_ipp(IPP_CFIG);
5927     val &= ~(IPP_CFIG_IPP_ENABLE |
5928          IPP_CFIG_DFIFO_ECC_EN |
5929          IPP_CFIG_DROP_BAD_CRC |
5930          IPP_CFIG_CKSUM_EN);
5931     nw64_ipp(IPP_CFIG, val);
5932 
5933     (void) niu_ipp_reset(np);
5934 }
5935 
5936 static int niu_init_hw(struct niu *np)
5937 {
5938     int i, err;
5939 
5940     netif_printk(np, ifup, KERN_DEBUG, np->dev, "Initialize TXC\n");
5941     niu_txc_enable_port(np, 1);
5942     niu_txc_port_dma_enable(np, 1);
5943     niu_txc_set_imask(np, 0);
5944 
5945     netif_printk(np, ifup, KERN_DEBUG, np->dev, "Initialize TX channels\n");
5946     for (i = 0; i < np->num_tx_rings; i++) {
5947         struct tx_ring_info *rp = &np->tx_rings[i];
5948 
5949         err = niu_init_one_tx_channel(np, rp);
5950         if (err)
5951             return err;
5952     }
5953 
5954     netif_printk(np, ifup, KERN_DEBUG, np->dev, "Initialize RX channels\n");
5955     err = niu_init_rx_channels(np);
5956     if (err)
5957         goto out_uninit_tx_channels;
5958 
5959     netif_printk(np, ifup, KERN_DEBUG, np->dev, "Initialize classifier\n");
5960     err = niu_init_classifier_hw(np);
5961     if (err)
5962         goto out_uninit_rx_channels;
5963 
5964     netif_printk(np, ifup, KERN_DEBUG, np->dev, "Initialize ZCP\n");
5965     err = niu_init_zcp(np);
5966     if (err)
5967         goto out_uninit_rx_channels;
5968 
5969     netif_printk(np, ifup, KERN_DEBUG, np->dev, "Initialize IPP\n");
5970     err = niu_init_ipp(np);
5971     if (err)
5972         goto out_uninit_rx_channels;
5973 
5974     netif_printk(np, ifup, KERN_DEBUG, np->dev, "Initialize MAC\n");
5975     err = niu_init_mac(np);
5976     if (err)
5977         goto out_uninit_ipp;
5978 
5979     return 0;
5980 
5981 out_uninit_ipp:
5982     netif_printk(np, ifup, KERN_DEBUG, np->dev, "Uninit IPP\n");
5983     niu_disable_ipp(np);
5984 
5985 out_uninit_rx_channels:
5986     netif_printk(np, ifup, KERN_DEBUG, np->dev, "Uninit RX channels\n");
5987     niu_stop_rx_channels(np);
5988     niu_reset_rx_channels(np);
5989 
5990 out_uninit_tx_channels:
5991     netif_printk(np, ifup, KERN_DEBUG, np->dev, "Uninit TX channels\n");
5992     niu_stop_tx_channels(np);
5993     niu_reset_tx_channels(np);
5994 
5995     return err;
5996 }
5997 
5998 static void niu_stop_hw(struct niu *np)
5999 {
6000     netif_printk(np, ifdown, KERN_DEBUG, np->dev, "Disable interrupts\n");
6001     niu_enable_interrupts(np, 0);
6002 
6003     netif_printk(np, ifdown, KERN_DEBUG, np->dev, "Disable RX MAC\n");
6004     niu_enable_rx_mac(np, 0);
6005 
6006     netif_printk(np, ifdown, KERN_DEBUG, np->dev, "Disable IPP\n");
6007     niu_disable_ipp(np);
6008 
6009     netif_printk(np, ifdown, KERN_DEBUG, np->dev, "Stop TX channels\n");
6010     niu_stop_tx_channels(np);
6011 
6012     netif_printk(np, ifdown, KERN_DEBUG, np->dev, "Stop RX channels\n");
6013     niu_stop_rx_channels(np);
6014 
6015     netif_printk(np, ifdown, KERN_DEBUG, np->dev, "Reset TX channels\n");
6016     niu_reset_tx_channels(np);
6017 
6018     netif_printk(np, ifdown, KERN_DEBUG, np->dev, "Reset RX channels\n");
6019     niu_reset_rx_channels(np);
6020 }
6021 
6022 static void niu_set_irq_name(struct niu *np)
6023 {
6024     int port = np->port;
6025     int i, j = 1;
6026 
6027     sprintf(np->irq_name[0], "%s:MAC", np->dev->name);
6028 
6029     if (port == 0) {
6030         sprintf(np->irq_name[1], "%s:MIF", np->dev->name);
6031         sprintf(np->irq_name[2], "%s:SYSERR", np->dev->name);
6032         j = 3;
6033     }
6034 
6035     for (i = 0; i < np->num_ldg - j; i++) {
6036         if (i < np->num_rx_rings)
6037             sprintf(np->irq_name[i+j], "%s-rx-%d",
6038                 np->dev->name, i);
6039         else if (i < np->num_tx_rings + np->num_rx_rings)
6040             sprintf(np->irq_name[i+j], "%s-tx-%d", np->dev->name,
6041                 i - np->num_rx_rings);
6042     }
6043 }
6044 
6045 static int niu_request_irq(struct niu *np)
6046 {
6047     int i, j, err;
6048 
6049     niu_set_irq_name(np);
6050 
6051     err = 0;
6052     for (i = 0; i < np->num_ldg; i++) {
6053         struct niu_ldg *lp = &np->ldg[i];
6054 
6055         err = request_irq(lp->irq, niu_interrupt, IRQF_SHARED,
6056                   np->irq_name[i], lp);
6057         if (err)
6058             goto out_free_irqs;
6059 
6060     }
6061 
6062     return 0;
6063 
6064 out_free_irqs:
6065     for (j = 0; j < i; j++) {
6066         struct niu_ldg *lp = &np->ldg[j];
6067 
6068         free_irq(lp->irq, lp);
6069     }
6070     return err;
6071 }
6072 
6073 static void niu_free_irq(struct niu *np)
6074 {
6075     int i;
6076 
6077     for (i = 0; i < np->num_ldg; i++) {
6078         struct niu_ldg *lp = &np->ldg[i];
6079 
6080         free_irq(lp->irq, lp);
6081     }
6082 }
6083 
6084 static void niu_enable_napi(struct niu *np)
6085 {
6086     int i;
6087 
6088     for (i = 0; i < np->num_ldg; i++)
6089         napi_enable(&np->ldg[i].napi);
6090 }
6091 
6092 static void niu_disable_napi(struct niu *np)
6093 {
6094     int i;
6095 
6096     for (i = 0; i < np->num_ldg; i++)
6097         napi_disable(&np->ldg[i].napi);
6098 }
6099 
6100 static int niu_open(struct net_device *dev)
6101 {
6102     struct niu *np = netdev_priv(dev);
6103     int err;
6104 
6105     netif_carrier_off(dev);
6106 
6107     err = niu_alloc_channels(np);
6108     if (err)
6109         goto out_err;
6110 
6111     err = niu_enable_interrupts(np, 0);
6112     if (err)
6113         goto out_free_channels;
6114 
6115     err = niu_request_irq(np);
6116     if (err)
6117         goto out_free_channels;
6118 
6119     niu_enable_napi(np);
6120 
6121     spin_lock_irq(&np->lock);
6122 
6123     err = niu_init_hw(np);
6124     if (!err) {
6125         timer_setup(&np->timer, niu_timer, 0);
6126         np->timer.expires = jiffies + HZ;
6127 
6128         err = niu_enable_interrupts(np, 1);
6129         if (err)
6130             niu_stop_hw(np);
6131     }
6132 
6133     spin_unlock_irq(&np->lock);
6134 
6135     if (err) {
6136         niu_disable_napi(np);
6137         goto out_free_irq;
6138     }
6139 
6140     netif_tx_start_all_queues(dev);
6141 
6142     if (np->link_config.loopback_mode != LOOPBACK_DISABLED)
6143         netif_carrier_on(dev);
6144 
6145     add_timer(&np->timer);
6146 
6147     return 0;
6148 
6149 out_free_irq:
6150     niu_free_irq(np);
6151 
6152 out_free_channels:
6153     niu_free_channels(np);
6154 
6155 out_err:
6156     return err;
6157 }
6158 
6159 static void niu_full_shutdown(struct niu *np, struct net_device *dev)
6160 {
6161     cancel_work_sync(&np->reset_task);
6162 
6163     niu_disable_napi(np);
6164     netif_tx_stop_all_queues(dev);
6165 
6166     del_timer_sync(&np->timer);
6167 
6168     spin_lock_irq(&np->lock);
6169 
6170     niu_stop_hw(np);
6171 
6172     spin_unlock_irq(&np->lock);
6173 }
6174 
6175 static int niu_close(struct net_device *dev)
6176 {
6177     struct niu *np = netdev_priv(dev);
6178 
6179     niu_full_shutdown(np, dev);
6180 
6181     niu_free_irq(np);
6182 
6183     niu_free_channels(np);
6184 
6185     niu_handle_led(np, 0);
6186 
6187     return 0;
6188 }
6189 
6190 static void niu_sync_xmac_stats(struct niu *np)
6191 {
6192     struct niu_xmac_stats *mp = &np->mac_stats.xmac;
6193 
6194     mp->tx_frames += nr64_mac(TXMAC_FRM_CNT);
6195     mp->tx_bytes += nr64_mac(TXMAC_BYTE_CNT);
6196 
6197     mp->rx_link_faults += nr64_mac(LINK_FAULT_CNT);
6198     mp->rx_align_errors += nr64_mac(RXMAC_ALIGN_ERR_CNT);
6199     mp->rx_frags += nr64_mac(RXMAC_FRAG_CNT);
6200     mp->rx_mcasts += nr64_mac(RXMAC_MC_FRM_CNT);
6201     mp->rx_bcasts += nr64_mac(RXMAC_BC_FRM_CNT);
6202     mp->rx_hist_cnt1 += nr64_mac(RXMAC_HIST_CNT1);
6203     mp->rx_hist_cnt2 += nr64_mac(RXMAC_HIST_CNT2);
6204     mp->rx_hist_cnt3 += nr64_mac(RXMAC_HIST_CNT3);
6205     mp->rx_hist_cnt4 += nr64_mac(RXMAC_HIST_CNT4);
6206     mp->rx_hist_cnt5 += nr64_mac(RXMAC_HIST_CNT5);
6207     mp->rx_hist_cnt6 += nr64_mac(RXMAC_HIST_CNT6);
6208     mp->rx_hist_cnt7 += nr64_mac(RXMAC_HIST_CNT7);
6209     mp->rx_octets += nr64_mac(RXMAC_BT_CNT);
6210     mp->rx_code_violations += nr64_mac(RXMAC_CD_VIO_CNT);
6211     mp->rx_len_errors += nr64_mac(RXMAC_MPSZER_CNT);
6212     mp->rx_crc_errors += nr64_mac(RXMAC_CRC_ER_CNT);
6213 }
6214 
6215 static void niu_sync_bmac_stats(struct niu *np)
6216 {
6217     struct niu_bmac_stats *mp = &np->mac_stats.bmac;
6218 
6219     mp->tx_bytes += nr64_mac(BTXMAC_BYTE_CNT);
6220     mp->tx_frames += nr64_mac(BTXMAC_FRM_CNT);
6221 
6222     mp->rx_frames += nr64_mac(BRXMAC_FRAME_CNT);
6223     mp->rx_align_errors += nr64_mac(BRXMAC_ALIGN_ERR_CNT);
6224     mp->rx_crc_errors += nr64_mac(BRXMAC_ALIGN_ERR_CNT);
6225     mp->rx_len_errors += nr64_mac(BRXMAC_CODE_VIOL_ERR_CNT);
6226 }
6227 
6228 static void niu_sync_mac_stats(struct niu *np)
6229 {
6230     if (np->flags & NIU_FLAGS_XMAC)
6231         niu_sync_xmac_stats(np);
6232     else
6233         niu_sync_bmac_stats(np);
6234 }
6235 
6236 static void niu_get_rx_stats(struct niu *np,
6237                  struct rtnl_link_stats64 *stats)
6238 {
6239     u64 pkts, dropped, errors, bytes;
6240     struct rx_ring_info *rx_rings;
6241     int i;
6242 
6243     pkts = dropped = errors = bytes = 0;
6244 
6245     rx_rings = READ_ONCE(np->rx_rings);
6246     if (!rx_rings)
6247         goto no_rings;
6248 
6249     for (i = 0; i < np->num_rx_rings; i++) {
6250         struct rx_ring_info *rp = &rx_rings[i];
6251 
6252         niu_sync_rx_discard_stats(np, rp, 0);
6253 
6254         pkts += rp->rx_packets;
6255         bytes += rp->rx_bytes;
6256         dropped += rp->rx_dropped;
6257         errors += rp->rx_errors;
6258     }
6259 
6260 no_rings:
6261     stats->rx_packets = pkts;
6262     stats->rx_bytes = bytes;
6263     stats->rx_dropped = dropped;
6264     stats->rx_errors = errors;
6265 }
6266 
6267 static void niu_get_tx_stats(struct niu *np,
6268                  struct rtnl_link_stats64 *stats)
6269 {
6270     u64 pkts, errors, bytes;
6271     struct tx_ring_info *tx_rings;
6272     int i;
6273 
6274     pkts = errors = bytes = 0;
6275 
6276     tx_rings = READ_ONCE(np->tx_rings);
6277     if (!tx_rings)
6278         goto no_rings;
6279 
6280     for (i = 0; i < np->num_tx_rings; i++) {
6281         struct tx_ring_info *rp = &tx_rings[i];
6282 
6283         pkts += rp->tx_packets;
6284         bytes += rp->tx_bytes;
6285         errors += rp->tx_errors;
6286     }
6287 
6288 no_rings:
6289     stats->tx_packets = pkts;
6290     stats->tx_bytes = bytes;
6291     stats->tx_errors = errors;
6292 }
6293 
6294 static void niu_get_stats(struct net_device *dev,
6295               struct rtnl_link_stats64 *stats)
6296 {
6297     struct niu *np = netdev_priv(dev);
6298 
6299     if (netif_running(dev)) {
6300         niu_get_rx_stats(np, stats);
6301         niu_get_tx_stats(np, stats);
6302     }
6303 }
6304 
6305 static void niu_load_hash_xmac(struct niu *np, u16 *hash)
6306 {
6307     int i;
6308 
6309     for (i = 0; i < 16; i++)
6310         nw64_mac(XMAC_HASH_TBL(i), hash[i]);
6311 }
6312 
6313 static void niu_load_hash_bmac(struct niu *np, u16 *hash)
6314 {
6315     int i;
6316 
6317     for (i = 0; i < 16; i++)
6318         nw64_mac(BMAC_HASH_TBL(i), hash[i]);
6319 }
6320 
6321 static void niu_load_hash(struct niu *np, u16 *hash)
6322 {
6323     if (np->flags & NIU_FLAGS_XMAC)
6324         niu_load_hash_xmac(np, hash);
6325     else
6326         niu_load_hash_bmac(np, hash);
6327 }
6328 
6329 static void niu_set_rx_mode(struct net_device *dev)
6330 {
6331     struct niu *np = netdev_priv(dev);
6332     int i, alt_cnt, err;
6333     struct netdev_hw_addr *ha;
6334     unsigned long flags;
6335     u16 hash[16] = { 0, };
6336 
6337     spin_lock_irqsave(&np->lock, flags);
6338     niu_enable_rx_mac(np, 0);
6339 
6340     np->flags &= ~(NIU_FLAGS_MCAST | NIU_FLAGS_PROMISC);
6341     if (dev->flags & IFF_PROMISC)
6342         np->flags |= NIU_FLAGS_PROMISC;
6343     if ((dev->flags & IFF_ALLMULTI) || (!netdev_mc_empty(dev)))
6344         np->flags |= NIU_FLAGS_MCAST;
6345 
6346     alt_cnt = netdev_uc_count(dev);
6347     if (alt_cnt > niu_num_alt_addr(np)) {
6348         alt_cnt = 0;
6349         np->flags |= NIU_FLAGS_PROMISC;
6350     }
6351 
6352     if (alt_cnt) {
6353         int index = 0;
6354 
6355         netdev_for_each_uc_addr(ha, dev) {
6356             err = niu_set_alt_mac(np, index, ha->addr);
6357             if (err)
6358                 netdev_warn(dev, "Error %d adding alt mac %d\n",
6359                         err, index);
6360             err = niu_enable_alt_mac(np, index, 1);
6361             if (err)
6362                 netdev_warn(dev, "Error %d enabling alt mac %d\n",
6363                         err, index);
6364 
6365             index++;
6366         }
6367     } else {
6368         int alt_start;
6369         if (np->flags & NIU_FLAGS_XMAC)
6370             alt_start = 0;
6371         else
6372             alt_start = 1;
6373         for (i = alt_start; i < niu_num_alt_addr(np); i++) {
6374             err = niu_enable_alt_mac(np, i, 0);
6375             if (err)
6376                 netdev_warn(dev, "Error %d disabling alt mac %d\n",
6377                         err, i);
6378         }
6379     }
6380     if (dev->flags & IFF_ALLMULTI) {
6381         for (i = 0; i < 16; i++)
6382             hash[i] = 0xffff;
6383     } else if (!netdev_mc_empty(dev)) {
6384         netdev_for_each_mc_addr(ha, dev) {
6385             u32 crc = ether_crc_le(ETH_ALEN, ha->addr);
6386 
6387             crc >>= 24;
6388             hash[crc >> 4] |= (1 << (15 - (crc & 0xf)));
6389         }
6390     }
6391 
6392     if (np->flags & NIU_FLAGS_MCAST)
6393         niu_load_hash(np, hash);
6394 
6395     niu_enable_rx_mac(np, 1);
6396     spin_unlock_irqrestore(&np->lock, flags);
6397 }
6398 
6399 static int niu_set_mac_addr(struct net_device *dev, void *p)
6400 {
6401     struct niu *np = netdev_priv(dev);
6402     struct sockaddr *addr = p;
6403     unsigned long flags;
6404 
6405     if (!is_valid_ether_addr(addr->sa_data))
6406         return -EADDRNOTAVAIL;
6407 
6408     eth_hw_addr_set(dev, addr->sa_data);
6409 
6410     if (!netif_running(dev))
6411         return 0;
6412 
6413     spin_lock_irqsave(&np->lock, flags);
6414     niu_enable_rx_mac(np, 0);
6415     niu_set_primary_mac(np, dev->dev_addr);
6416     niu_enable_rx_mac(np, 1);
6417     spin_unlock_irqrestore(&np->lock, flags);
6418 
6419     return 0;
6420 }
6421 
6422 static int niu_ioctl(struct net_device *dev, struct ifreq *ifr, int cmd)
6423 {
6424     return -EOPNOTSUPP;
6425 }
6426 
6427 static void niu_netif_stop(struct niu *np)
6428 {
6429     netif_trans_update(np->dev);    /* prevent tx timeout */
6430 
6431     niu_disable_napi(np);
6432 
6433     netif_tx_disable(np->dev);
6434 }
6435 
6436 static void niu_netif_start(struct niu *np)
6437 {
6438     /* NOTE: unconditional netif_wake_queue is only appropriate
6439      * so long as all callers are assured to have free tx slots
6440      * (such as after niu_init_hw).
6441      */
6442     netif_tx_wake_all_queues(np->dev);
6443 
6444     niu_enable_napi(np);
6445 
6446     niu_enable_interrupts(np, 1);
6447 }
6448 
6449 static void niu_reset_buffers(struct niu *np)
6450 {
6451     int i, j, k, err;
6452 
6453     if (np->rx_rings) {
6454         for (i = 0; i < np->num_rx_rings; i++) {
6455             struct rx_ring_info *rp = &np->rx_rings[i];
6456 
6457             for (j = 0, k = 0; j < MAX_RBR_RING_SIZE; j++) {
6458                 struct page *page;
6459 
6460                 page = rp->rxhash[j];
6461                 while (page) {
6462                     struct page *next = niu_next_page(page);
6463                     u64 base = page->index;
6464                     base = base >> RBR_DESCR_ADDR_SHIFT;
6465                     rp->rbr[k++] = cpu_to_le32(base);
6466                     page = next;
6467                 }
6468             }
6469             for (; k < MAX_RBR_RING_SIZE; k++) {
6470                 err = niu_rbr_add_page(np, rp, GFP_ATOMIC, k);
6471                 if (unlikely(err))
6472                     break;
6473             }
6474 
6475             rp->rbr_index = rp->rbr_table_size - 1;
6476             rp->rcr_index = 0;
6477             rp->rbr_pending = 0;
6478             rp->rbr_refill_pending = 0;
6479         }
6480     }
6481     if (np->tx_rings) {
6482         for (i = 0; i < np->num_tx_rings; i++) {
6483             struct tx_ring_info *rp = &np->tx_rings[i];
6484 
6485             for (j = 0; j < MAX_TX_RING_SIZE; j++) {
6486                 if (rp->tx_buffs[j].skb)
6487                     (void) release_tx_packet(np, rp, j);
6488             }
6489 
6490             rp->pending = MAX_TX_RING_SIZE;
6491             rp->prod = 0;
6492             rp->cons = 0;
6493             rp->wrap_bit = 0;
6494         }
6495     }
6496 }
6497 
6498 static void niu_reset_task(struct work_struct *work)
6499 {
6500     struct niu *np = container_of(work, struct niu, reset_task);
6501     unsigned long flags;
6502     int err;
6503 
6504     spin_lock_irqsave(&np->lock, flags);
6505     if (!netif_running(np->dev)) {
6506         spin_unlock_irqrestore(&np->lock, flags);
6507         return;
6508     }
6509 
6510     spin_unlock_irqrestore(&np->lock, flags);
6511 
6512     del_timer_sync(&np->timer);
6513 
6514     niu_netif_stop(np);
6515 
6516     spin_lock_irqsave(&np->lock, flags);
6517 
6518     niu_stop_hw(np);
6519 
6520     spin_unlock_irqrestore(&np->lock, flags);
6521 
6522     niu_reset_buffers(np);
6523 
6524     spin_lock_irqsave(&np->lock, flags);
6525 
6526     err = niu_init_hw(np);
6527     if (!err) {
6528         np->timer.expires = jiffies + HZ;
6529         add_timer(&np->timer);
6530         niu_netif_start(np);
6531     }
6532 
6533     spin_unlock_irqrestore(&np->lock, flags);
6534 }
6535 
6536 static void niu_tx_timeout(struct net_device *dev, unsigned int txqueue)
6537 {
6538     struct niu *np = netdev_priv(dev);
6539 
6540     dev_err(np->device, "%s: Transmit timed out, resetting\n",
6541         dev->name);
6542 
6543     schedule_work(&np->reset_task);
6544 }
6545 
6546 static void niu_set_txd(struct tx_ring_info *rp, int index,
6547             u64 mapping, u64 len, u64 mark,
6548             u64 n_frags)
6549 {
6550     __le64 *desc = &rp->descr[index];
6551 
6552     *desc = cpu_to_le64(mark |
6553                 (n_frags << TX_DESC_NUM_PTR_SHIFT) |
6554                 (len << TX_DESC_TR_LEN_SHIFT) |
6555                 (mapping & TX_DESC_SAD));
6556 }
6557 
6558 static u64 niu_compute_tx_flags(struct sk_buff *skb, struct ethhdr *ehdr,
6559                 u64 pad_bytes, u64 len)
6560 {
6561     u16 eth_proto, eth_proto_inner;
6562     u64 csum_bits, l3off, ihl, ret;
6563     u8 ip_proto;
6564     int ipv6;
6565 
6566     eth_proto = be16_to_cpu(ehdr->h_proto);
6567     eth_proto_inner = eth_proto;
6568     if (eth_proto == ETH_P_8021Q) {
6569         struct vlan_ethhdr *vp = (struct vlan_ethhdr *) ehdr;
6570         __be16 val = vp->h_vlan_encapsulated_proto;
6571 
6572         eth_proto_inner = be16_to_cpu(val);
6573     }
6574 
6575     ipv6 = ihl = 0;
6576     switch (skb->protocol) {
6577     case cpu_to_be16(ETH_P_IP):
6578         ip_proto = ip_hdr(skb)->protocol;
6579         ihl = ip_hdr(skb)->ihl;
6580         break;
6581     case cpu_to_be16(ETH_P_IPV6):
6582         ip_proto = ipv6_hdr(skb)->nexthdr;
6583         ihl = (40 >> 2);
6584         ipv6 = 1;
6585         break;
6586     default:
6587         ip_proto = ihl = 0;
6588         break;
6589     }
6590 
6591     csum_bits = TXHDR_CSUM_NONE;
6592     if (skb->ip_summed == CHECKSUM_PARTIAL) {
6593         u64 start, stuff;
6594 
6595         csum_bits = (ip_proto == IPPROTO_TCP ?
6596                  TXHDR_CSUM_TCP :
6597                  (ip_proto == IPPROTO_UDP ?
6598                   TXHDR_CSUM_UDP : TXHDR_CSUM_SCTP));
6599 
6600         start = skb_checksum_start_offset(skb) -
6601             (pad_bytes + sizeof(struct tx_pkt_hdr));
6602         stuff = start + skb->csum_offset;
6603 
6604         csum_bits |= (start / 2) << TXHDR_L4START_SHIFT;
6605         csum_bits |= (stuff / 2) << TXHDR_L4STUFF_SHIFT;
6606     }
6607 
6608     l3off = skb_network_offset(skb) -
6609         (pad_bytes + sizeof(struct tx_pkt_hdr));
6610 
6611     ret = (((pad_bytes / 2) << TXHDR_PAD_SHIFT) |
6612            (len << TXHDR_LEN_SHIFT) |
6613            ((l3off / 2) << TXHDR_L3START_SHIFT) |
6614            (ihl << TXHDR_IHL_SHIFT) |
6615            ((eth_proto_inner < ETH_P_802_3_MIN) ? TXHDR_LLC : 0) |
6616            ((eth_proto == ETH_P_8021Q) ? TXHDR_VLAN : 0) |
6617            (ipv6 ? TXHDR_IP_VER : 0) |
6618            csum_bits);
6619 
6620     return ret;
6621 }
6622 
6623 static netdev_tx_t niu_start_xmit(struct sk_buff *skb,
6624                   struct net_device *dev)
6625 {
6626     struct niu *np = netdev_priv(dev);
6627     unsigned long align, headroom;
6628     struct netdev_queue *txq;
6629     struct tx_ring_info *rp;
6630     struct tx_pkt_hdr *tp;
6631     unsigned int len, nfg;
6632     struct ethhdr *ehdr;
6633     int prod, i, tlen;
6634     u64 mapping, mrk;
6635 
6636     i = skb_get_queue_mapping(skb);
6637     rp = &np->tx_rings[i];
6638     txq = netdev_get_tx_queue(dev, i);
6639 
6640     if (niu_tx_avail(rp) <= (skb_shinfo(skb)->nr_frags + 1)) {
6641         netif_tx_stop_queue(txq);
6642         dev_err(np->device, "%s: BUG! Tx ring full when queue awake!\n", dev->name);
6643         rp->tx_errors++;
6644         return NETDEV_TX_BUSY;
6645     }
6646 
6647     if (eth_skb_pad(skb))
6648         goto out;
6649 
6650     len = sizeof(struct tx_pkt_hdr) + 15;
6651     if (skb_headroom(skb) < len) {
6652         struct sk_buff *skb_new;
6653 
6654         skb_new = skb_realloc_headroom(skb, len);
6655         if (!skb_new)
6656             goto out_drop;
6657         kfree_skb(skb);
6658         skb = skb_new;
6659     } else
6660         skb_orphan(skb);
6661 
6662     align = ((unsigned long) skb->data & (16 - 1));
6663     headroom = align + sizeof(struct tx_pkt_hdr);
6664 
6665     ehdr = (struct ethhdr *) skb->data;
6666     tp = skb_push(skb, headroom);
6667 
6668     len = skb->len - sizeof(struct tx_pkt_hdr);
6669     tp->flags = cpu_to_le64(niu_compute_tx_flags(skb, ehdr, align, len));
6670     tp->resv = 0;
6671 
6672     len = skb_headlen(skb);
6673     mapping = np->ops->map_single(np->device, skb->data,
6674                       len, DMA_TO_DEVICE);
6675 
6676     prod = rp->prod;
6677 
6678     rp->tx_buffs[prod].skb = skb;
6679     rp->tx_buffs[prod].mapping = mapping;
6680 
6681     mrk = TX_DESC_SOP;
6682     if (++rp->mark_counter == rp->mark_freq) {
6683         rp->mark_counter = 0;
6684         mrk |= TX_DESC_MARK;
6685         rp->mark_pending++;
6686     }
6687 
6688     tlen = len;
6689     nfg = skb_shinfo(skb)->nr_frags;
6690     while (tlen > 0) {
6691         tlen -= MAX_TX_DESC_LEN;
6692         nfg++;
6693     }
6694 
6695     while (len > 0) {
6696         unsigned int this_len = len;
6697 
6698         if (this_len > MAX_TX_DESC_LEN)
6699             this_len = MAX_TX_DESC_LEN;
6700 
6701         niu_set_txd(rp, prod, mapping, this_len, mrk, nfg);
6702         mrk = nfg = 0;
6703 
6704         prod = NEXT_TX(rp, prod);
6705         mapping += this_len;
6706         len -= this_len;
6707     }
6708 
6709     for (i = 0; i <  skb_shinfo(skb)->nr_frags; i++) {
6710         const skb_frag_t *frag = &skb_shinfo(skb)->frags[i];
6711 
6712         len = skb_frag_size(frag);
6713         mapping = np->ops->map_page(np->device, skb_frag_page(frag),
6714                         skb_frag_off(frag), len,
6715                         DMA_TO_DEVICE);
6716 
6717         rp->tx_buffs[prod].skb = NULL;
6718         rp->tx_buffs[prod].mapping = mapping;
6719 
6720         niu_set_txd(rp, prod, mapping, len, 0, 0);
6721 
6722         prod = NEXT_TX(rp, prod);
6723     }
6724 
6725     if (prod < rp->prod)
6726         rp->wrap_bit ^= TX_RING_KICK_WRAP;
6727     rp->prod = prod;
6728 
6729     nw64(TX_RING_KICK(rp->tx_channel), rp->wrap_bit | (prod << 3));
6730 
6731     if (unlikely(niu_tx_avail(rp) <= (MAX_SKB_FRAGS + 1))) {
6732         netif_tx_stop_queue(txq);
6733         if (niu_tx_avail(rp) > NIU_TX_WAKEUP_THRESH(rp))
6734             netif_tx_wake_queue(txq);
6735     }
6736 
6737 out:
6738     return NETDEV_TX_OK;
6739 
6740 out_drop:
6741     rp->tx_errors++;
6742     kfree_skb(skb);
6743     goto out;
6744 }
6745 
6746 static int niu_change_mtu(struct net_device *dev, int new_mtu)
6747 {
6748     struct niu *np = netdev_priv(dev);
6749     int err, orig_jumbo, new_jumbo;
6750 
6751     orig_jumbo = (dev->mtu > ETH_DATA_LEN);
6752     new_jumbo = (new_mtu > ETH_DATA_LEN);
6753 
6754     dev->mtu = new_mtu;
6755 
6756     if (!netif_running(dev) ||
6757         (orig_jumbo == new_jumbo))
6758         return 0;
6759 
6760     niu_full_shutdown(np, dev);
6761 
6762     niu_free_channels(np);
6763 
6764     niu_enable_napi(np);
6765 
6766     err = niu_alloc_channels(np);
6767     if (err)
6768         return err;
6769 
6770     spin_lock_irq(&np->lock);
6771 
6772     err = niu_init_hw(np);
6773     if (!err) {
6774         timer_setup(&np->timer, niu_timer, 0);
6775         np->timer.expires = jiffies + HZ;
6776 
6777         err = niu_enable_interrupts(np, 1);
6778         if (err)
6779             niu_stop_hw(np);
6780     }
6781 
6782     spin_unlock_irq(&np->lock);
6783 
6784     if (!err) {
6785         netif_tx_start_all_queues(dev);
6786         if (np->link_config.loopback_mode != LOOPBACK_DISABLED)
6787             netif_carrier_on(dev);
6788 
6789         add_timer(&np->timer);
6790     }
6791 
6792     return err;
6793 }
6794 
6795 static void niu_get_drvinfo(struct net_device *dev,
6796                 struct ethtool_drvinfo *info)
6797 {
6798     struct niu *np = netdev_priv(dev);
6799     struct niu_vpd *vpd = &np->vpd;
6800 
6801     strlcpy(info->driver, DRV_MODULE_NAME, sizeof(info->driver));
6802     strlcpy(info->version, DRV_MODULE_VERSION, sizeof(info->version));
6803     snprintf(info->fw_version, sizeof(info->fw_version), "%d.%d",
6804         vpd->fcode_major, vpd->fcode_minor);
6805     if (np->parent->plat_type != PLAT_TYPE_NIU)
6806         strlcpy(info->bus_info, pci_name(np->pdev),
6807             sizeof(info->bus_info));
6808 }
6809 
6810 static int niu_get_link_ksettings(struct net_device *dev,
6811                   struct ethtool_link_ksettings *cmd)
6812 {
6813     struct niu *np = netdev_priv(dev);
6814     struct niu_link_config *lp;
6815 
6816     lp = &np->link_config;
6817 
6818     memset(cmd, 0, sizeof(*cmd));
6819     cmd->base.phy_address = np->phy_addr;
6820     ethtool_convert_legacy_u32_to_link_mode(cmd->link_modes.supported,
6821                         lp->supported);
6822     ethtool_convert_legacy_u32_to_link_mode(cmd->link_modes.advertising,
6823                         lp->active_advertising);
6824     cmd->base.autoneg = lp->active_autoneg;
6825     cmd->base.speed = lp->active_speed;
6826     cmd->base.duplex = lp->active_duplex;
6827     cmd->base.port = (np->flags & NIU_FLAGS_FIBER) ? PORT_FIBRE : PORT_TP;
6828 
6829     return 0;
6830 }
6831 
6832 static int niu_set_link_ksettings(struct net_device *dev,
6833                   const struct ethtool_link_ksettings *cmd)
6834 {
6835     struct niu *np = netdev_priv(dev);
6836     struct niu_link_config *lp = &np->link_config;
6837 
6838     ethtool_convert_link_mode_to_legacy_u32(&lp->advertising,
6839                         cmd->link_modes.advertising);
6840     lp->speed = cmd->base.speed;
6841     lp->duplex = cmd->base.duplex;
6842     lp->autoneg = cmd->base.autoneg;
6843     return niu_init_link(np);
6844 }
6845 
6846 static u32 niu_get_msglevel(struct net_device *dev)
6847 {
6848     struct niu *np = netdev_priv(dev);
6849     return np->msg_enable;
6850 }
6851 
6852 static void niu_set_msglevel(struct net_device *dev, u32 value)
6853 {
6854     struct niu *np = netdev_priv(dev);
6855     np->msg_enable = value;
6856 }
6857 
6858 static int niu_nway_reset(struct net_device *dev)
6859 {
6860     struct niu *np = netdev_priv(dev);
6861 
6862     if (np->link_config.autoneg)
6863         return niu_init_link(np);
6864 
6865     return 0;
6866 }
6867 
6868 static int niu_get_eeprom_len(struct net_device *dev)
6869 {
6870     struct niu *np = netdev_priv(dev);
6871 
6872     return np->eeprom_len;
6873 }
6874 
6875 static int niu_get_eeprom(struct net_device *dev,
6876               struct ethtool_eeprom *eeprom, u8 *data)
6877 {
6878     struct niu *np = netdev_priv(dev);
6879     u32 offset, len, val;
6880 
6881     offset = eeprom->offset;
6882     len = eeprom->len;
6883 
6884     if (offset + len < offset)
6885         return -EINVAL;
6886     if (offset >= np->eeprom_len)
6887         return -EINVAL;
6888     if (offset + len > np->eeprom_len)
6889         len = eeprom->len = np->eeprom_len - offset;
6890 
6891     if (offset & 3) {
6892         u32 b_offset, b_count;
6893 
6894         b_offset = offset & 3;
6895         b_count = 4 - b_offset;
6896         if (b_count > len)
6897             b_count = len;
6898 
6899         val = nr64(ESPC_NCR((offset - b_offset) / 4));
6900         memcpy(data, ((char *)&val) + b_offset, b_count);
6901         data += b_count;
6902         len -= b_count;
6903         offset += b_count;
6904     }
6905     while (len >= 4) {
6906         val = nr64(ESPC_NCR(offset / 4));
6907         memcpy(data, &val, 4);
6908         data += 4;
6909         len -= 4;
6910         offset += 4;
6911     }
6912     if (len) {
6913         val = nr64(ESPC_NCR(offset / 4));
6914         memcpy(data, &val, len);
6915     }
6916     return 0;
6917 }
6918 
6919 static void niu_ethflow_to_l3proto(int flow_type, u8 *pid)
6920 {
6921     switch (flow_type) {
6922     case TCP_V4_FLOW:
6923     case TCP_V6_FLOW:
6924         *pid = IPPROTO_TCP;
6925         break;
6926     case UDP_V4_FLOW:
6927     case UDP_V6_FLOW:
6928         *pid = IPPROTO_UDP;
6929         break;
6930     case SCTP_V4_FLOW:
6931     case SCTP_V6_FLOW:
6932         *pid = IPPROTO_SCTP;
6933         break;
6934     case AH_V4_FLOW:
6935     case AH_V6_FLOW:
6936         *pid = IPPROTO_AH;
6937         break;
6938     case ESP_V4_FLOW:
6939     case ESP_V6_FLOW:
6940         *pid = IPPROTO_ESP;
6941         break;
6942     default:
6943         *pid = 0;
6944         break;
6945     }
6946 }
6947 
6948 static int niu_class_to_ethflow(u64 class, int *flow_type)
6949 {
6950     switch (class) {
6951     case CLASS_CODE_TCP_IPV4:
6952         *flow_type = TCP_V4_FLOW;
6953         break;
6954     case CLASS_CODE_UDP_IPV4:
6955         *flow_type = UDP_V4_FLOW;
6956         break;
6957     case CLASS_CODE_AH_ESP_IPV4:
6958         *flow_type = AH_V4_FLOW;
6959         break;
6960     case CLASS_CODE_SCTP_IPV4:
6961         *flow_type = SCTP_V4_FLOW;
6962         break;
6963     case CLASS_CODE_TCP_IPV6:
6964         *flow_type = TCP_V6_FLOW;
6965         break;
6966     case CLASS_CODE_UDP_IPV6:
6967         *flow_type = UDP_V6_FLOW;
6968         break;
6969     case CLASS_CODE_AH_ESP_IPV6:
6970         *flow_type = AH_V6_FLOW;
6971         break;
6972     case CLASS_CODE_SCTP_IPV6:
6973         *flow_type = SCTP_V6_FLOW;
6974         break;
6975     case CLASS_CODE_USER_PROG1:
6976     case CLASS_CODE_USER_PROG2:
6977     case CLASS_CODE_USER_PROG3:
6978     case CLASS_CODE_USER_PROG4:
6979         *flow_type = IP_USER_FLOW;
6980         break;
6981     default:
6982         return -EINVAL;
6983     }
6984 
6985     return 0;
6986 }
6987 
6988 static int niu_ethflow_to_class(int flow_type, u64 *class)
6989 {
6990     switch (flow_type) {
6991     case TCP_V4_FLOW:
6992         *class = CLASS_CODE_TCP_IPV4;
6993         break;
6994     case UDP_V4_FLOW:
6995         *class = CLASS_CODE_UDP_IPV4;
6996         break;
6997     case AH_ESP_V4_FLOW:
6998     case AH_V4_FLOW:
6999     case ESP_V4_FLOW:
7000         *class = CLASS_CODE_AH_ESP_IPV4;
7001         break;
7002     case SCTP_V4_FLOW:
7003         *class = CLASS_CODE_SCTP_IPV4;
7004         break;
7005     case TCP_V6_FLOW:
7006         *class = CLASS_CODE_TCP_IPV6;
7007         break;
7008     case UDP_V6_FLOW:
7009         *class = CLASS_CODE_UDP_IPV6;
7010         break;
7011     case AH_ESP_V6_FLOW:
7012     case AH_V6_FLOW:
7013     case ESP_V6_FLOW:
7014         *class = CLASS_CODE_AH_ESP_IPV6;
7015         break;
7016     case SCTP_V6_FLOW:
7017         *class = CLASS_CODE_SCTP_IPV6;
7018         break;
7019     default:
7020         return 0;
7021     }
7022 
7023     return 1;
7024 }
7025 
7026 static u64 niu_flowkey_to_ethflow(u64 flow_key)
7027 {
7028     u64 ethflow = 0;
7029 
7030     if (flow_key & FLOW_KEY_L2DA)
7031         ethflow |= RXH_L2DA;
7032     if (flow_key & FLOW_KEY_VLAN)
7033         ethflow |= RXH_VLAN;
7034     if (flow_key & FLOW_KEY_IPSA)
7035         ethflow |= RXH_IP_SRC;
7036     if (flow_key & FLOW_KEY_IPDA)
7037         ethflow |= RXH_IP_DST;
7038     if (flow_key & FLOW_KEY_PROTO)
7039         ethflow |= RXH_L3_PROTO;
7040     if (flow_key & (FLOW_KEY_L4_BYTE12 << FLOW_KEY_L4_0_SHIFT))
7041         ethflow |= RXH_L4_B_0_1;
7042     if (flow_key & (FLOW_KEY_L4_BYTE12 << FLOW_KEY_L4_1_SHIFT))
7043         ethflow |= RXH_L4_B_2_3;
7044 
7045     return ethflow;
7046 
7047 }
7048 
7049 static int niu_ethflow_to_flowkey(u64 ethflow, u64 *flow_key)
7050 {
7051     u64 key = 0;
7052 
7053     if (ethflow & RXH_L2DA)
7054         key |= FLOW_KEY_L2DA;
7055     if (ethflow & RXH_VLAN)
7056         key |= FLOW_KEY_VLAN;
7057     if (ethflow & RXH_IP_SRC)
7058         key |= FLOW_KEY_IPSA;
7059     if (ethflow & RXH_IP_DST)
7060         key |= FLOW_KEY_IPDA;
7061     if (ethflow & RXH_L3_PROTO)
7062         key |= FLOW_KEY_PROTO;
7063     if (ethflow & RXH_L4_B_0_1)
7064         key |= (FLOW_KEY_L4_BYTE12 << FLOW_KEY_L4_0_SHIFT);
7065     if (ethflow & RXH_L4_B_2_3)
7066         key |= (FLOW_KEY_L4_BYTE12 << FLOW_KEY_L4_1_SHIFT);
7067 
7068     *flow_key = key;
7069 
7070     return 1;
7071 
7072 }
7073 
7074 static int niu_get_hash_opts(struct niu *np, struct ethtool_rxnfc *nfc)
7075 {
7076     u64 class;
7077 
7078     nfc->data = 0;
7079 
7080     if (!niu_ethflow_to_class(nfc->flow_type, &class))
7081         return -EINVAL;
7082 
7083     if (np->parent->tcam_key[class - CLASS_CODE_USER_PROG1] &
7084         TCAM_KEY_DISC)
7085         nfc->data = RXH_DISCARD;
7086     else
7087         nfc->data = niu_flowkey_to_ethflow(np->parent->flow_key[class -
7088                               CLASS_CODE_USER_PROG1]);
7089     return 0;
7090 }
7091 
7092 static void niu_get_ip4fs_from_tcam_key(struct niu_tcam_entry *tp,
7093                     struct ethtool_rx_flow_spec *fsp)
7094 {
7095     u32 tmp;
7096     u16 prt;
7097 
7098     tmp = (tp->key[3] & TCAM_V4KEY3_SADDR) >> TCAM_V4KEY3_SADDR_SHIFT;
7099     fsp->h_u.tcp_ip4_spec.ip4src = cpu_to_be32(tmp);
7100 
7101     tmp = (tp->key[3] & TCAM_V4KEY3_DADDR) >> TCAM_V4KEY3_DADDR_SHIFT;
7102     fsp->h_u.tcp_ip4_spec.ip4dst = cpu_to_be32(tmp);
7103 
7104     tmp = (tp->key_mask[3] & TCAM_V4KEY3_SADDR) >> TCAM_V4KEY3_SADDR_SHIFT;
7105     fsp->m_u.tcp_ip4_spec.ip4src = cpu_to_be32(tmp);
7106 
7107     tmp = (tp->key_mask[3] & TCAM_V4KEY3_DADDR) >> TCAM_V4KEY3_DADDR_SHIFT;
7108     fsp->m_u.tcp_ip4_spec.ip4dst = cpu_to_be32(tmp);
7109 
7110     fsp->h_u.tcp_ip4_spec.tos = (tp->key[2] & TCAM_V4KEY2_TOS) >>
7111         TCAM_V4KEY2_TOS_SHIFT;
7112     fsp->m_u.tcp_ip4_spec.tos = (tp->key_mask[2] & TCAM_V4KEY2_TOS) >>
7113         TCAM_V4KEY2_TOS_SHIFT;
7114 
7115     switch (fsp->flow_type) {
7116     case TCP_V4_FLOW:
7117     case UDP_V4_FLOW:
7118     case SCTP_V4_FLOW:
7119         prt = ((tp->key[2] & TCAM_V4KEY2_PORT_SPI) >>
7120             TCAM_V4KEY2_PORT_SPI_SHIFT) >> 16;
7121         fsp->h_u.tcp_ip4_spec.psrc = cpu_to_be16(prt);
7122 
7123         prt = ((tp->key[2] & TCAM_V4KEY2_PORT_SPI) >>
7124             TCAM_V4KEY2_PORT_SPI_SHIFT) & 0xffff;
7125         fsp->h_u.tcp_ip4_spec.pdst = cpu_to_be16(prt);
7126 
7127         prt = ((tp->key_mask[2] & TCAM_V4KEY2_PORT_SPI) >>
7128             TCAM_V4KEY2_PORT_SPI_SHIFT) >> 16;
7129         fsp->m_u.tcp_ip4_spec.psrc = cpu_to_be16(prt);
7130 
7131         prt = ((tp->key_mask[2] & TCAM_V4KEY2_PORT_SPI) >>
7132              TCAM_V4KEY2_PORT_SPI_SHIFT) & 0xffff;
7133         fsp->m_u.tcp_ip4_spec.pdst = cpu_to_be16(prt);
7134         break;
7135     case AH_V4_FLOW:
7136     case ESP_V4_FLOW:
7137         tmp = (tp->key[2] & TCAM_V4KEY2_PORT_SPI) >>
7138             TCAM_V4KEY2_PORT_SPI_SHIFT;
7139         fsp->h_u.ah_ip4_spec.spi = cpu_to_be32(tmp);
7140 
7141         tmp = (tp->key_mask[2] & TCAM_V4KEY2_PORT_SPI) >>
7142             TCAM_V4KEY2_PORT_SPI_SHIFT;
7143         fsp->m_u.ah_ip4_spec.spi = cpu_to_be32(tmp);
7144         break;
7145     case IP_USER_FLOW:
7146         tmp = (tp->key[2] & TCAM_V4KEY2_PORT_SPI) >>
7147             TCAM_V4KEY2_PORT_SPI_SHIFT;
7148         fsp->h_u.usr_ip4_spec.l4_4_bytes = cpu_to_be32(tmp);
7149 
7150         tmp = (tp->key_mask[2] & TCAM_V4KEY2_PORT_SPI) >>
7151             TCAM_V4KEY2_PORT_SPI_SHIFT;
7152         fsp->m_u.usr_ip4_spec.l4_4_bytes = cpu_to_be32(tmp);
7153 
7154         fsp->h_u.usr_ip4_spec.proto =
7155             (tp->key[2] & TCAM_V4KEY2_PROTO) >>
7156             TCAM_V4KEY2_PROTO_SHIFT;
7157         fsp->m_u.usr_ip4_spec.proto =
7158             (tp->key_mask[2] & TCAM_V4KEY2_PROTO) >>
7159             TCAM_V4KEY2_PROTO_SHIFT;
7160 
7161         fsp->h_u.usr_ip4_spec.ip_ver = ETH_RX_NFC_IP4;
7162         break;
7163     default:
7164         break;
7165     }
7166 }
7167 
7168 static int niu_get_ethtool_tcam_entry(struct niu *np,
7169                       struct ethtool_rxnfc *nfc)
7170 {
7171     struct niu_parent *parent = np->parent;
7172     struct niu_tcam_entry *tp;
7173     struct ethtool_rx_flow_spec *fsp = &nfc->fs;
7174     u16 idx;
7175     u64 class;
7176     int ret = 0;
7177 
7178     idx = tcam_get_index(np, (u16)nfc->fs.location);
7179 
7180     tp = &parent->tcam[idx];
7181     if (!tp->valid) {
7182         netdev_info(np->dev, "niu%d: entry [%d] invalid for idx[%d]\n",
7183                 parent->index, (u16)nfc->fs.location, idx);
7184         return -EINVAL;
7185     }
7186 
7187     /* fill the flow spec entry */
7188     class = (tp->key[0] & TCAM_V4KEY0_CLASS_CODE) >>
7189         TCAM_V4KEY0_CLASS_CODE_SHIFT;
7190     ret = niu_class_to_ethflow(class, &fsp->flow_type);
7191     if (ret < 0) {
7192         netdev_info(np->dev, "niu%d: niu_class_to_ethflow failed\n",
7193                 parent->index);
7194         goto out;
7195     }
7196 
7197     if (fsp->flow_type == AH_V4_FLOW || fsp->flow_type == AH_V6_FLOW) {
7198         u32 proto = (tp->key[2] & TCAM_V4KEY2_PROTO) >>
7199             TCAM_V4KEY2_PROTO_SHIFT;
7200         if (proto == IPPROTO_ESP) {
7201             if (fsp->flow_type == AH_V4_FLOW)
7202                 fsp->flow_type = ESP_V4_FLOW;
7203             else
7204                 fsp->flow_type = ESP_V6_FLOW;
7205         }
7206     }
7207 
7208     switch (fsp->flow_type) {
7209     case TCP_V4_FLOW:
7210     case UDP_V4_FLOW:
7211     case SCTP_V4_FLOW:
7212     case AH_V4_FLOW:
7213     case ESP_V4_FLOW:
7214         niu_get_ip4fs_from_tcam_key(tp, fsp);
7215         break;
7216     case TCP_V6_FLOW:
7217     case UDP_V6_FLOW:
7218     case SCTP_V6_FLOW:
7219     case AH_V6_FLOW:
7220     case ESP_V6_FLOW:
7221         /* Not yet implemented */
7222         ret = -EINVAL;
7223         break;
7224     case IP_USER_FLOW:
7225         niu_get_ip4fs_from_tcam_key(tp, fsp);
7226         break;
7227     default:
7228         ret = -EINVAL;
7229         break;
7230     }
7231 
7232     if (ret < 0)
7233         goto out;
7234 
7235     if (tp->assoc_data & TCAM_ASSOCDATA_DISC)
7236         fsp->ring_cookie = RX_CLS_FLOW_DISC;
7237     else
7238         fsp->ring_cookie = (tp->assoc_data & TCAM_ASSOCDATA_OFFSET) >>
7239             TCAM_ASSOCDATA_OFFSET_SHIFT;
7240 
7241     /* put the tcam size here */
7242     nfc->data = tcam_get_size(np);
7243 out:
7244     return ret;
7245 }
7246 
7247 static int niu_get_ethtool_tcam_all(struct niu *np,
7248                     struct ethtool_rxnfc *nfc,
7249                     u32 *rule_locs)
7250 {
7251     struct niu_parent *parent = np->parent;
7252     struct niu_tcam_entry *tp;
7253     int i, idx, cnt;
7254     unsigned long flags;
7255     int ret = 0;
7256 
7257     /* put the tcam size here */
7258     nfc->data = tcam_get_size(np);
7259 
7260     niu_lock_parent(np, flags);
7261     for (cnt = 0, i = 0; i < nfc->data; i++) {
7262         idx = tcam_get_index(np, i);
7263         tp = &parent->tcam[idx];
7264         if (!tp->valid)
7265             continue;
7266         if (cnt == nfc->rule_cnt) {
7267             ret = -EMSGSIZE;
7268             break;
7269         }
7270         rule_locs[cnt] = i;
7271         cnt++;
7272     }
7273     niu_unlock_parent(np, flags);
7274 
7275     nfc->rule_cnt = cnt;
7276 
7277     return ret;
7278 }
7279 
7280 static int niu_get_nfc(struct net_device *dev, struct ethtool_rxnfc *cmd,
7281                u32 *rule_locs)
7282 {
7283     struct niu *np = netdev_priv(dev);
7284     int ret = 0;
7285 
7286     switch (cmd->cmd) {
7287     case ETHTOOL_GRXFH:
7288         ret = niu_get_hash_opts(np, cmd);
7289         break;
7290     case ETHTOOL_GRXRINGS:
7291         cmd->data = np->num_rx_rings;
7292         break;
7293     case ETHTOOL_GRXCLSRLCNT:
7294         cmd->rule_cnt = tcam_get_valid_entry_cnt(np);
7295         break;
7296     case ETHTOOL_GRXCLSRULE:
7297         ret = niu_get_ethtool_tcam_entry(np, cmd);
7298         break;
7299     case ETHTOOL_GRXCLSRLALL:
7300         ret = niu_get_ethtool_tcam_all(np, cmd, rule_locs);
7301         break;
7302     default:
7303         ret = -EINVAL;
7304         break;
7305     }
7306 
7307     return ret;
7308 }
7309 
7310 static int niu_set_hash_opts(struct niu *np, struct ethtool_rxnfc *nfc)
7311 {
7312     u64 class;
7313     u64 flow_key = 0;
7314     unsigned long flags;
7315 
7316     if (!niu_ethflow_to_class(nfc->flow_type, &class))
7317         return -EINVAL;
7318 
7319     if (class < CLASS_CODE_USER_PROG1 ||
7320         class > CLASS_CODE_SCTP_IPV6)
7321         return -EINVAL;
7322 
7323     if (nfc->data & RXH_DISCARD) {
7324         niu_lock_parent(np, flags);
7325         flow_key = np->parent->tcam_key[class -
7326                            CLASS_CODE_USER_PROG1];
7327         flow_key |= TCAM_KEY_DISC;
7328         nw64(TCAM_KEY(class - CLASS_CODE_USER_PROG1), flow_key);
7329         np->parent->tcam_key[class - CLASS_CODE_USER_PROG1] = flow_key;
7330         niu_unlock_parent(np, flags);
7331         return 0;
7332     } else {
7333         /* Discard was set before, but is not set now */
7334         if (np->parent->tcam_key[class - CLASS_CODE_USER_PROG1] &
7335             TCAM_KEY_DISC) {
7336             niu_lock_parent(np, flags);
7337             flow_key = np->parent->tcam_key[class -
7338                            CLASS_CODE_USER_PROG1];
7339             flow_key &= ~TCAM_KEY_DISC;
7340             nw64(TCAM_KEY(class - CLASS_CODE_USER_PROG1),
7341                  flow_key);
7342             np->parent->tcam_key[class - CLASS_CODE_USER_PROG1] =
7343                 flow_key;
7344             niu_unlock_parent(np, flags);
7345         }
7346     }
7347 
7348     if (!niu_ethflow_to_flowkey(nfc->data, &flow_key))
7349         return -EINVAL;
7350 
7351     niu_lock_parent(np, flags);
7352     nw64(FLOW_KEY(class - CLASS_CODE_USER_PROG1), flow_key);
7353     np->parent->flow_key[class - CLASS_CODE_USER_PROG1] = flow_key;
7354     niu_unlock_parent(np, flags);
7355 
7356     return 0;
7357 }
7358 
7359 static void niu_get_tcamkey_from_ip4fs(struct ethtool_rx_flow_spec *fsp,
7360                        struct niu_tcam_entry *tp,
7361                        int l2_rdc_tab, u64 class)
7362 {
7363     u8 pid = 0;
7364     u32 sip, dip, sipm, dipm, spi, spim;
7365     u16 sport, dport, spm, dpm;
7366 
7367     sip = be32_to_cpu(fsp->h_u.tcp_ip4_spec.ip4src);
7368     sipm = be32_to_cpu(fsp->m_u.tcp_ip4_spec.ip4src);
7369     dip = be32_to_cpu(fsp->h_u.tcp_ip4_spec.ip4dst);
7370     dipm = be32_to_cpu(fsp->m_u.tcp_ip4_spec.ip4dst);
7371 
7372     tp->key[0] = class << TCAM_V4KEY0_CLASS_CODE_SHIFT;
7373     tp->key_mask[0] = TCAM_V4KEY0_CLASS_CODE;
7374     tp->key[1] = (u64)l2_rdc_tab << TCAM_V4KEY1_L2RDCNUM_SHIFT;
7375     tp->key_mask[1] = TCAM_V4KEY1_L2RDCNUM;
7376 
7377     tp->key[3] = (u64)sip << TCAM_V4KEY3_SADDR_SHIFT;
7378     tp->key[3] |= dip;
7379 
7380     tp->key_mask[3] = (u64)sipm << TCAM_V4KEY3_SADDR_SHIFT;
7381     tp->key_mask[3] |= dipm;
7382 
7383     tp->key[2] |= ((u64)fsp->h_u.tcp_ip4_spec.tos <<
7384                TCAM_V4KEY2_TOS_SHIFT);
7385     tp->key_mask[2] |= ((u64)fsp->m_u.tcp_ip4_spec.tos <<
7386                 TCAM_V4KEY2_TOS_SHIFT);
7387     switch (fsp->flow_type) {
7388     case TCP_V4_FLOW:
7389     case UDP_V4_FLOW:
7390     case SCTP_V4_FLOW:
7391         sport = be16_to_cpu(fsp->h_u.tcp_ip4_spec.psrc);
7392         spm = be16_to_cpu(fsp->m_u.tcp_ip4_spec.psrc);
7393         dport = be16_to_cpu(fsp->h_u.tcp_ip4_spec.pdst);
7394         dpm = be16_to_cpu(fsp->m_u.tcp_ip4_spec.pdst);
7395 
7396         tp->key[2] |= (((u64)sport << 16) | dport);
7397         tp->key_mask[2] |= (((u64)spm << 16) | dpm);
7398         niu_ethflow_to_l3proto(fsp->flow_type, &pid);
7399         break;
7400     case AH_V4_FLOW:
7401     case ESP_V4_FLOW:
7402         spi = be32_to_cpu(fsp->h_u.ah_ip4_spec.spi);
7403         spim = be32_to_cpu(fsp->m_u.ah_ip4_spec.spi);
7404 
7405         tp->key[2] |= spi;
7406         tp->key_mask[2] |= spim;
7407         niu_ethflow_to_l3proto(fsp->flow_type, &pid);
7408         break;
7409     case IP_USER_FLOW:
7410         spi = be32_to_cpu(fsp->h_u.usr_ip4_spec.l4_4_bytes);
7411         spim = be32_to_cpu(fsp->m_u.usr_ip4_spec.l4_4_bytes);
7412 
7413         tp->key[2] |= spi;
7414         tp->key_mask[2] |= spim;
7415         pid = fsp->h_u.usr_ip4_spec.proto;
7416         break;
7417     default:
7418         break;
7419     }
7420 
7421     tp->key[2] |= ((u64)pid << TCAM_V4KEY2_PROTO_SHIFT);
7422     if (pid) {
7423         tp->key_mask[2] |= TCAM_V4KEY2_PROTO;
7424     }
7425 }
7426 
7427 static int niu_add_ethtool_tcam_entry(struct niu *np,
7428                       struct ethtool_rxnfc *nfc)
7429 {
7430     struct niu_parent *parent = np->parent;
7431     struct niu_tcam_entry *tp;
7432     struct ethtool_rx_flow_spec *fsp = &nfc->fs;
7433     struct niu_rdc_tables *rdc_table = &parent->rdc_group_cfg[np->port];
7434     int l2_rdc_table = rdc_table->first_table_num;
7435     u16 idx;
7436     u64 class;
7437     unsigned long flags;
7438     int err, ret;
7439 
7440     ret = 0;
7441 
7442     idx = nfc->fs.location;
7443     if (idx >= tcam_get_size(np))
7444         return -EINVAL;
7445 
7446     if (fsp->flow_type == IP_USER_FLOW) {
7447         int i;
7448         int add_usr_cls = 0;
7449         struct ethtool_usrip4_spec *uspec = &fsp->h_u.usr_ip4_spec;
7450         struct ethtool_usrip4_spec *umask = &fsp->m_u.usr_ip4_spec;
7451 
7452         if (uspec->ip_ver != ETH_RX_NFC_IP4)
7453             return -EINVAL;
7454 
7455         niu_lock_parent(np, flags);
7456 
7457         for (i = 0; i < NIU_L3_PROG_CLS; i++) {
7458             if (parent->l3_cls[i]) {
7459                 if (uspec->proto == parent->l3_cls_pid[i]) {
7460                     class = parent->l3_cls[i];
7461                     parent->l3_cls_refcnt[i]++;
7462                     add_usr_cls = 1;
7463                     break;
7464                 }
7465             } else {
7466                 /* Program new user IP class */
7467                 switch (i) {
7468                 case 0:
7469                     class = CLASS_CODE_USER_PROG1;
7470                     break;
7471                 case 1:
7472                     class = CLASS_CODE_USER_PROG2;
7473                     break;
7474                 case 2:
7475                     class = CLASS_CODE_USER_PROG3;
7476                     break;
7477                 case 3:
7478                     class = CLASS_CODE_USER_PROG4;
7479                     break;
7480                 default:
7481                     class = CLASS_CODE_UNRECOG;
7482                     break;
7483                 }
7484                 ret = tcam_user_ip_class_set(np, class, 0,
7485                                  uspec->proto,
7486                                  uspec->tos,
7487                                  umask->tos);
7488                 if (ret)
7489                     goto out;
7490 
7491                 ret = tcam_user_ip_class_enable(np, class, 1);
7492                 if (ret)
7493                     goto out;
7494                 parent->l3_cls[i] = class;
7495                 parent->l3_cls_pid[i] = uspec->proto;
7496                 parent->l3_cls_refcnt[i]++;
7497                 add_usr_cls = 1;
7498                 break;
7499             }
7500         }
7501         if (!add_usr_cls) {
7502             netdev_info(np->dev, "niu%d: %s(): Could not find/insert class for pid %d\n",
7503                     parent->index, __func__, uspec->proto);
7504             ret = -EINVAL;
7505             goto out;
7506         }
7507         niu_unlock_parent(np, flags);
7508     } else {
7509         if (!niu_ethflow_to_class(fsp->flow_type, &class)) {
7510             return -EINVAL;
7511         }
7512     }
7513 
7514     niu_lock_parent(np, flags);
7515 
7516     idx = tcam_get_index(np, idx);
7517     tp = &parent->tcam[idx];
7518 
7519     memset(tp, 0, sizeof(*tp));
7520 
7521     /* fill in the tcam key and mask */
7522     switch (fsp->flow_type) {
7523     case TCP_V4_FLOW:
7524     case UDP_V4_FLOW:
7525     case SCTP_V4_FLOW:
7526     case AH_V4_FLOW:
7527     case ESP_V4_FLOW:
7528         niu_get_tcamkey_from_ip4fs(fsp, tp, l2_rdc_table, class);
7529         break;
7530     case TCP_V6_FLOW:
7531     case UDP_V6_FLOW:
7532     case SCTP_V6_FLOW:
7533     case AH_V6_FLOW:
7534     case ESP_V6_FLOW:
7535         /* Not yet implemented */
7536         netdev_info(np->dev, "niu%d: In %s(): flow %d for IPv6 not implemented\n",
7537                 parent->index, __func__, fsp->flow_type);
7538         ret = -EINVAL;
7539         goto out;
7540     case IP_USER_FLOW:
7541         niu_get_tcamkey_from_ip4fs(fsp, tp, l2_rdc_table, class);
7542         break;
7543     default:
7544         netdev_info(np->dev, "niu%d: In %s(): Unknown flow type %d\n",
7545                 parent->index, __func__, fsp->flow_type);
7546         ret = -EINVAL;
7547         goto out;
7548     }
7549 
7550     /* fill in the assoc data */
7551     if (fsp->ring_cookie == RX_CLS_FLOW_DISC) {
7552         tp->assoc_data = TCAM_ASSOCDATA_DISC;
7553     } else {
7554         if (fsp->ring_cookie >= np->num_rx_rings) {
7555             netdev_info(np->dev, "niu%d: In %s(): Invalid RX ring %lld\n",
7556                     parent->index, __func__,
7557                     (long long)fsp->ring_cookie);
7558             ret = -EINVAL;
7559             goto out;
7560         }
7561         tp->assoc_data = (TCAM_ASSOCDATA_TRES_USE_OFFSET |
7562                   (fsp->ring_cookie <<
7563                    TCAM_ASSOCDATA_OFFSET_SHIFT));
7564     }
7565 
7566     err = tcam_write(np, idx, tp->key, tp->key_mask);
7567     if (err) {
7568         ret = -EINVAL;
7569         goto out;
7570     }
7571     err = tcam_assoc_write(np, idx, tp->assoc_data);
7572     if (err) {
7573         ret = -EINVAL;
7574         goto out;
7575     }
7576 
7577     /* validate the entry */
7578     tp->valid = 1;
7579     np->clas.tcam_valid_entries++;
7580 out:
7581     niu_unlock_parent(np, flags);
7582 
7583     return ret;
7584 }
7585 
7586 static int niu_del_ethtool_tcam_entry(struct niu *np, u32 loc)
7587 {
7588     struct niu_parent *parent = np->parent;
7589     struct niu_tcam_entry *tp;
7590     u16 idx;
7591     unsigned long flags;
7592     u64 class;
7593     int ret = 0;
7594 
7595     if (loc >= tcam_get_size(np))
7596         return -EINVAL;
7597 
7598     niu_lock_parent(np, flags);
7599 
7600     idx = tcam_get_index(np, loc);
7601     tp = &parent->tcam[idx];
7602 
7603     /* if the entry is of a user defined class, then update*/
7604     class = (tp->key[0] & TCAM_V4KEY0_CLASS_CODE) >>
7605         TCAM_V4KEY0_CLASS_CODE_SHIFT;
7606 
7607     if (class >= CLASS_CODE_USER_PROG1 && class <= CLASS_CODE_USER_PROG4) {
7608         int i;
7609         for (i = 0; i < NIU_L3_PROG_CLS; i++) {
7610             if (parent->l3_cls[i] == class) {
7611                 parent->l3_cls_refcnt[i]--;
7612                 if (!parent->l3_cls_refcnt[i]) {
7613                     /* disable class */
7614                     ret = tcam_user_ip_class_enable(np,
7615                                     class,
7616                                     0);
7617                     if (ret)
7618                         goto out;
7619                     parent->l3_cls[i] = 0;
7620                     parent->l3_cls_pid[i] = 0;
7621                 }
7622                 break;
7623             }
7624         }
7625         if (i == NIU_L3_PROG_CLS) {
7626             netdev_info(np->dev, "niu%d: In %s(): Usr class 0x%llx not found\n",
7627                     parent->index, __func__,
7628                     (unsigned long long)class);
7629             ret = -EINVAL;
7630             goto out;
7631         }
7632     }
7633 
7634     ret = tcam_flush(np, idx);
7635     if (ret)
7636         goto out;
7637 
7638     /* invalidate the entry */
7639     tp->valid = 0;
7640     np->clas.tcam_valid_entries--;
7641 out:
7642     niu_unlock_parent(np, flags);
7643 
7644     return ret;
7645 }
7646 
7647 static int niu_set_nfc(struct net_device *dev, struct ethtool_rxnfc *cmd)
7648 {
7649     struct niu *np = netdev_priv(dev);
7650     int ret = 0;
7651 
7652     switch (cmd->cmd) {
7653     case ETHTOOL_SRXFH:
7654         ret = niu_set_hash_opts(np, cmd);
7655         break;
7656     case ETHTOOL_SRXCLSRLINS:
7657         ret = niu_add_ethtool_tcam_entry(np, cmd);
7658         break;
7659     case ETHTOOL_SRXCLSRLDEL:
7660         ret = niu_del_ethtool_tcam_entry(np, cmd->fs.location);
7661         break;
7662     default:
7663         ret = -EINVAL;
7664         break;
7665     }
7666 
7667     return ret;
7668 }
7669 
7670 static const struct {
7671     const char string[ETH_GSTRING_LEN];
7672 } niu_xmac_stat_keys[] = {
7673     { "tx_frames" },
7674     { "tx_bytes" },
7675     { "tx_fifo_errors" },
7676     { "tx_overflow_errors" },
7677     { "tx_max_pkt_size_errors" },
7678     { "tx_underflow_errors" },
7679     { "rx_local_faults" },
7680     { "rx_remote_faults" },
7681     { "rx_link_faults" },
7682     { "rx_align_errors" },
7683     { "rx_frags" },
7684     { "rx_mcasts" },
7685     { "rx_bcasts" },
7686     { "rx_hist_cnt1" },
7687     { "rx_hist_cnt2" },
7688     { "rx_hist_cnt3" },
7689     { "rx_hist_cnt4" },
7690     { "rx_hist_cnt5" },
7691     { "rx_hist_cnt6" },
7692     { "rx_hist_cnt7" },
7693     { "rx_octets" },
7694     { "rx_code_violations" },
7695     { "rx_len_errors" },
7696     { "rx_crc_errors" },
7697     { "rx_underflows" },
7698     { "rx_overflows" },
7699     { "pause_off_state" },
7700     { "pause_on_state" },
7701     { "pause_received" },
7702 };
7703 
7704 #define NUM_XMAC_STAT_KEYS  ARRAY_SIZE(niu_xmac_stat_keys)
7705 
7706 static const struct {
7707     const char string[ETH_GSTRING_LEN];
7708 } niu_bmac_stat_keys[] = {
7709     { "tx_underflow_errors" },
7710     { "tx_max_pkt_size_errors" },
7711     { "tx_bytes" },
7712     { "tx_frames" },
7713     { "rx_overflows" },
7714     { "rx_frames" },
7715     { "rx_align_errors" },
7716     { "rx_crc_errors" },
7717     { "rx_len_errors" },
7718     { "pause_off_state" },
7719     { "pause_on_state" },
7720     { "pause_received" },
7721 };
7722 
7723 #define NUM_BMAC_STAT_KEYS  ARRAY_SIZE(niu_bmac_stat_keys)
7724 
7725 static const struct {
7726     const char string[ETH_GSTRING_LEN];
7727 } niu_rxchan_stat_keys[] = {
7728     { "rx_channel" },
7729     { "rx_packets" },
7730     { "rx_bytes" },
7731     { "rx_dropped" },
7732     { "rx_errors" },
7733 };
7734 
7735 #define NUM_RXCHAN_STAT_KEYS    ARRAY_SIZE(niu_rxchan_stat_keys)
7736 
7737 static const struct {
7738     const char string[ETH_GSTRING_LEN];
7739 } niu_txchan_stat_keys[] = {
7740     { "tx_channel" },
7741     { "tx_packets" },
7742     { "tx_bytes" },
7743     { "tx_errors" },
7744 };
7745 
7746 #define NUM_TXCHAN_STAT_KEYS    ARRAY_SIZE(niu_txchan_stat_keys)
7747 
7748 static void niu_get_strings(struct net_device *dev, u32 stringset, u8 *data)
7749 {
7750     struct niu *np = netdev_priv(dev);
7751     int i;
7752 
7753     if (stringset != ETH_SS_STATS)
7754         return;
7755 
7756     if (np->flags & NIU_FLAGS_XMAC) {
7757         memcpy(data, niu_xmac_stat_keys,
7758                sizeof(niu_xmac_stat_keys));
7759         data += sizeof(niu_xmac_stat_keys);
7760     } else {
7761         memcpy(data, niu_bmac_stat_keys,
7762                sizeof(niu_bmac_stat_keys));
7763         data += sizeof(niu_bmac_stat_keys);
7764     }
7765     for (i = 0; i < np->num_rx_rings; i++) {
7766         memcpy(data, niu_rxchan_stat_keys,
7767                sizeof(niu_rxchan_stat_keys));
7768         data += sizeof(niu_rxchan_stat_keys);
7769     }
7770     for (i = 0; i < np->num_tx_rings; i++) {
7771         memcpy(data, niu_txchan_stat_keys,
7772                sizeof(niu_txchan_stat_keys));
7773         data += sizeof(niu_txchan_stat_keys);
7774     }
7775 }
7776 
7777 static int niu_get_sset_count(struct net_device *dev, int stringset)
7778 {
7779     struct niu *np = netdev_priv(dev);
7780 
7781     if (stringset != ETH_SS_STATS)
7782         return -EINVAL;
7783 
7784     return (np->flags & NIU_FLAGS_XMAC ?
7785          NUM_XMAC_STAT_KEYS :
7786          NUM_BMAC_STAT_KEYS) +
7787         (np->num_rx_rings * NUM_RXCHAN_STAT_KEYS) +
7788         (np->num_tx_rings * NUM_TXCHAN_STAT_KEYS);
7789 }
7790 
7791 static void niu_get_ethtool_stats(struct net_device *dev,
7792                   struct ethtool_stats *stats, u64 *data)
7793 {
7794     struct niu *np = netdev_priv(dev);
7795     int i;
7796 
7797     niu_sync_mac_stats(np);
7798     if (np->flags & NIU_FLAGS_XMAC) {
7799         memcpy(data, &np->mac_stats.xmac,
7800                sizeof(struct niu_xmac_stats));
7801         data += (sizeof(struct niu_xmac_stats) / sizeof(u64));
7802     } else {
7803         memcpy(data, &np->mac_stats.bmac,
7804                sizeof(struct niu_bmac_stats));
7805         data += (sizeof(struct niu_bmac_stats) / sizeof(u64));
7806     }
7807     for (i = 0; i < np->num_rx_rings; i++) {
7808         struct rx_ring_info *rp = &np->rx_rings[i];
7809 
7810         niu_sync_rx_discard_stats(np, rp, 0);
7811 
7812         data[0] = rp->rx_channel;
7813         data[1] = rp->rx_packets;
7814         data[2] = rp->rx_bytes;
7815         data[3] = rp->rx_dropped;
7816         data[4] = rp->rx_errors;
7817         data += 5;
7818     }
7819     for (i = 0; i < np->num_tx_rings; i++) {
7820         struct tx_ring_info *rp = &np->tx_rings[i];
7821 
7822         data[0] = rp->tx_channel;
7823         data[1] = rp->tx_packets;
7824         data[2] = rp->tx_bytes;
7825         data[3] = rp->tx_errors;
7826         data += 4;
7827     }
7828 }
7829 
7830 static u64 niu_led_state_save(struct niu *np)
7831 {
7832     if (np->flags & NIU_FLAGS_XMAC)
7833         return nr64_mac(XMAC_CONFIG);
7834     else
7835         return nr64_mac(BMAC_XIF_CONFIG);
7836 }
7837 
7838 static void niu_led_state_restore(struct niu *np, u64 val)
7839 {
7840     if (np->flags & NIU_FLAGS_XMAC)
7841         nw64_mac(XMAC_CONFIG, val);
7842     else
7843         nw64_mac(BMAC_XIF_CONFIG, val);
7844 }
7845 
7846 static void niu_force_led(struct niu *np, int on)
7847 {
7848     u64 val, reg, bit;
7849 
7850     if (np->flags & NIU_FLAGS_XMAC) {
7851         reg = XMAC_CONFIG;
7852         bit = XMAC_CONFIG_FORCE_LED_ON;
7853     } else {
7854         reg = BMAC_XIF_CONFIG;
7855         bit = BMAC_XIF_CONFIG_LINK_LED;
7856     }
7857 
7858     val = nr64_mac(reg);
7859     if (on)
7860         val |= bit;
7861     else
7862         val &= ~bit;
7863     nw64_mac(reg, val);
7864 }
7865 
7866 static int niu_set_phys_id(struct net_device *dev,
7867                enum ethtool_phys_id_state state)
7868 
7869 {
7870     struct niu *np = netdev_priv(dev);
7871 
7872     if (!netif_running(dev))
7873         return -EAGAIN;
7874 
7875     switch (state) {
7876     case ETHTOOL_ID_ACTIVE:
7877         np->orig_led_state = niu_led_state_save(np);
7878         return 1;   /* cycle on/off once per second */
7879 
7880     case ETHTOOL_ID_ON:
7881         niu_force_led(np, 1);
7882         break;
7883 
7884     case ETHTOOL_ID_OFF:
7885         niu_force_led(np, 0);
7886         break;
7887 
7888     case ETHTOOL_ID_INACTIVE:
7889         niu_led_state_restore(np, np->orig_led_state);
7890     }
7891 
7892     return 0;
7893 }
7894 
7895 static const struct ethtool_ops niu_ethtool_ops = {
7896     .get_drvinfo        = niu_get_drvinfo,
7897     .get_link       = ethtool_op_get_link,
7898     .get_msglevel       = niu_get_msglevel,
7899     .set_msglevel       = niu_set_msglevel,
7900     .nway_reset     = niu_nway_reset,
7901     .get_eeprom_len     = niu_get_eeprom_len,
7902     .get_eeprom     = niu_get_eeprom,
7903     .get_strings        = niu_get_strings,
7904     .get_sset_count     = niu_get_sset_count,
7905     .get_ethtool_stats  = niu_get_ethtool_stats,
7906     .set_phys_id        = niu_set_phys_id,
7907     .get_rxnfc      = niu_get_nfc,
7908     .set_rxnfc      = niu_set_nfc,
7909     .get_link_ksettings = niu_get_link_ksettings,
7910     .set_link_ksettings = niu_set_link_ksettings,
7911 };
7912 
7913 static int niu_ldg_assign_ldn(struct niu *np, struct niu_parent *parent,
7914                   int ldg, int ldn)
7915 {
7916     if (ldg < NIU_LDG_MIN || ldg > NIU_LDG_MAX)
7917         return -EINVAL;
7918     if (ldn < 0 || ldn > LDN_MAX)
7919         return -EINVAL;
7920 
7921     parent->ldg_map[ldn] = ldg;
7922 
7923     if (np->parent->plat_type == PLAT_TYPE_NIU) {
7924         /* On N2 NIU, the ldn-->ldg assignments are setup and fixed by
7925          * the firmware, and we're not supposed to change them.
7926          * Validate the mapping, because if it's wrong we probably
7927          * won't get any interrupts and that's painful to debug.
7928          */
7929         if (nr64(LDG_NUM(ldn)) != ldg) {
7930             dev_err(np->device, "Port %u, mismatched LDG assignment for ldn %d, should be %d is %llu\n",
7931                 np->port, ldn, ldg,
7932                 (unsigned long long) nr64(LDG_NUM(ldn)));
7933             return -EINVAL;
7934         }
7935     } else
7936         nw64(LDG_NUM(ldn), ldg);
7937 
7938     return 0;
7939 }
7940 
7941 static int niu_set_ldg_timer_res(struct niu *np, int res)
7942 {
7943     if (res < 0 || res > LDG_TIMER_RES_VAL)
7944         return -EINVAL;
7945 
7946 
7947     nw64(LDG_TIMER_RES, res);
7948 
7949     return 0;
7950 }
7951 
7952 static int niu_set_ldg_sid(struct niu *np, int ldg, int func, int vector)
7953 {
7954     if ((ldg < NIU_LDG_MIN || ldg > NIU_LDG_MAX) ||
7955         (func < 0 || func > 3) ||
7956         (vector < 0 || vector > 0x1f))
7957         return -EINVAL;
7958 
7959     nw64(SID(ldg), (func << SID_FUNC_SHIFT) | vector);
7960 
7961     return 0;
7962 }
7963 
7964 static int niu_pci_eeprom_read(struct niu *np, u32 addr)
7965 {
7966     u64 frame, frame_base = (ESPC_PIO_STAT_READ_START |
7967                  (addr << ESPC_PIO_STAT_ADDR_SHIFT));
7968     int limit;
7969 
7970     if (addr > (ESPC_PIO_STAT_ADDR >> ESPC_PIO_STAT_ADDR_SHIFT))
7971         return -EINVAL;
7972 
7973     frame = frame_base;
7974     nw64(ESPC_PIO_STAT, frame);
7975     limit = 64;
7976     do {
7977         udelay(5);
7978         frame = nr64(ESPC_PIO_STAT);
7979         if (frame & ESPC_PIO_STAT_READ_END)
7980             break;
7981     } while (limit--);
7982     if (!(frame & ESPC_PIO_STAT_READ_END)) {
7983         dev_err(np->device, "EEPROM read timeout frame[%llx]\n",
7984             (unsigned long long) frame);
7985         return -ENODEV;
7986     }
7987 
7988     frame = frame_base;
7989     nw64(ESPC_PIO_STAT, frame);
7990     limit = 64;
7991     do {
7992         udelay(5);
7993         frame = nr64(ESPC_PIO_STAT);
7994         if (frame & ESPC_PIO_STAT_READ_END)
7995             break;
7996     } while (limit--);
7997     if (!(frame & ESPC_PIO_STAT_READ_END)) {
7998         dev_err(np->device, "EEPROM read timeout frame[%llx]\n",
7999             (unsigned long long) frame);
8000         return -ENODEV;
8001     }
8002 
8003     frame = nr64(ESPC_PIO_STAT);
8004     return (frame & ESPC_PIO_STAT_DATA) >> ESPC_PIO_STAT_DATA_SHIFT;
8005 }
8006 
8007 static int niu_pci_eeprom_read16(struct niu *np, u32 off)
8008 {
8009     int err = niu_pci_eeprom_read(np, off);
8010     u16 val;
8011 
8012     if (err < 0)
8013         return err;
8014     val = (err << 8);
8015     err = niu_pci_eeprom_read(np, off + 1);
8016     if (err < 0)
8017         return err;
8018     val |= (err & 0xff);
8019 
8020     return val;
8021 }
8022 
8023 static int niu_pci_eeprom_read16_swp(struct niu *np, u32 off)
8024 {
8025     int err = niu_pci_eeprom_read(np, off);
8026     u16 val;
8027 
8028     if (err < 0)
8029         return err;
8030 
8031     val = (err & 0xff);
8032     err = niu_pci_eeprom_read(np, off + 1);
8033     if (err < 0)
8034         return err;
8035 
8036     val |= (err & 0xff) << 8;
8037 
8038     return val;
8039 }
8040 
8041 static int niu_pci_vpd_get_propname(struct niu *np, u32 off, char *namebuf,
8042                     int namebuf_len)
8043 {
8044     int i;
8045 
8046     for (i = 0; i < namebuf_len; i++) {
8047         int err = niu_pci_eeprom_read(np, off + i);
8048         if (err < 0)
8049             return err;
8050         *namebuf++ = err;
8051         if (!err)
8052             break;
8053     }
8054     if (i >= namebuf_len)
8055         return -EINVAL;
8056 
8057     return i + 1;
8058 }
8059 
8060 static void niu_vpd_parse_version(struct niu *np)
8061 {
8062     struct niu_vpd *vpd = &np->vpd;
8063     int len = strlen(vpd->version) + 1;
8064     const char *s = vpd->version;
8065     int i;
8066 
8067     for (i = 0; i < len - 5; i++) {
8068         if (!strncmp(s + i, "FCode ", 6))
8069             break;
8070     }
8071     if (i >= len - 5)
8072         return;
8073 
8074     s += i + 5;
8075     sscanf(s, "%d.%d", &vpd->fcode_major, &vpd->fcode_minor);
8076 
8077     netif_printk(np, probe, KERN_DEBUG, np->dev,
8078              "VPD_SCAN: FCODE major(%d) minor(%d)\n",
8079              vpd->fcode_major, vpd->fcode_minor);
8080     if (vpd->fcode_major > NIU_VPD_MIN_MAJOR ||
8081         (vpd->fcode_major == NIU_VPD_MIN_MAJOR &&
8082          vpd->fcode_minor >= NIU_VPD_MIN_MINOR))
8083         np->flags |= NIU_FLAGS_VPD_VALID;
8084 }
8085 
8086 /* ESPC_PIO_EN_ENABLE must be set */
8087 static int niu_pci_vpd_scan_props(struct niu *np, u32 start, u32 end)
8088 {
8089     unsigned int found_mask = 0;
8090 #define FOUND_MASK_MODEL    0x00000001
8091 #define FOUND_MASK_BMODEL   0x00000002
8092 #define FOUND_MASK_VERS     0x00000004
8093 #define FOUND_MASK_MAC      0x00000008
8094 #define FOUND_MASK_NMAC     0x00000010
8095 #define FOUND_MASK_PHY      0x00000020
8096 #define FOUND_MASK_ALL      0x0000003f
8097 
8098     netif_printk(np, probe, KERN_DEBUG, np->dev,
8099              "VPD_SCAN: start[%x] end[%x]\n", start, end);
8100     while (start < end) {
8101         int len, err, prop_len;
8102         char namebuf[64];
8103         u8 *prop_buf;
8104         int max_len;
8105 
8106         if (found_mask == FOUND_MASK_ALL) {
8107             niu_vpd_parse_version(np);
8108             return 1;
8109         }
8110 
8111         err = niu_pci_eeprom_read(np, start + 2);
8112         if (err < 0)
8113             return err;
8114         len = err;
8115         start += 3;
8116 
8117         prop_len = niu_pci_eeprom_read(np, start + 4);
8118         if (prop_len < 0)
8119             return prop_len;
8120         err = niu_pci_vpd_get_propname(np, start + 5, namebuf, 64);
8121         if (err < 0)
8122             return err;
8123 
8124         prop_buf = NULL;
8125         max_len = 0;
8126         if (!strcmp(namebuf, "model")) {
8127             prop_buf = np->vpd.model;
8128             max_len = NIU_VPD_MODEL_MAX;
8129             found_mask |= FOUND_MASK_MODEL;
8130         } else if (!strcmp(namebuf, "board-model")) {
8131             prop_buf = np->vpd.board_model;
8132             max_len = NIU_VPD_BD_MODEL_MAX;
8133             found_mask |= FOUND_MASK_BMODEL;
8134         } else if (!strcmp(namebuf, "version")) {
8135             prop_buf = np->vpd.version;
8136             max_len = NIU_VPD_VERSION_MAX;
8137             found_mask |= FOUND_MASK_VERS;
8138         } else if (!strcmp(namebuf, "local-mac-address")) {
8139             prop_buf = np->vpd.local_mac;
8140             max_len = ETH_ALEN;
8141             found_mask |= FOUND_MASK_MAC;
8142         } else if (!strcmp(namebuf, "num-mac-addresses")) {
8143             prop_buf = &np->vpd.mac_num;
8144             max_len = 1;
8145             found_mask |= FOUND_MASK_NMAC;
8146         } else if (!strcmp(namebuf, "phy-type")) {
8147             prop_buf = np->vpd.phy_type;
8148             max_len = NIU_VPD_PHY_TYPE_MAX;
8149             found_mask |= FOUND_MASK_PHY;
8150         }
8151 
8152         if (max_len && prop_len > max_len) {
8153             dev_err(np->device, "Property '%s' length (%d) is too long\n", namebuf, prop_len);
8154             return -EINVAL;
8155         }
8156 
8157         if (prop_buf) {
8158             u32 off = start + 5 + err;
8159             int i;
8160 
8161             netif_printk(np, probe, KERN_DEBUG, np->dev,
8162                      "VPD_SCAN: Reading in property [%s] len[%d]\n",
8163                      namebuf, prop_len);
8164             for (i = 0; i < prop_len; i++) {
8165                 err =  niu_pci_eeprom_read(np, off + i);
8166                 if (err < 0)
8167                     return err;
8168                 *prop_buf++ = err;
8169             }
8170         }
8171 
8172         start += len;
8173     }
8174 
8175     return 0;
8176 }
8177 
8178 /* ESPC_PIO_EN_ENABLE must be set */
8179 static int niu_pci_vpd_fetch(struct niu *np, u32 start)
8180 {
8181     u32 offset;
8182     int err;
8183 
8184     err = niu_pci_eeprom_read16_swp(np, start + 1);
8185     if (err < 0)
8186         return err;
8187 
8188     offset = err + 3;
8189 
8190     while (start + offset < ESPC_EEPROM_SIZE) {
8191         u32 here = start + offset;
8192         u32 end;
8193 
8194         err = niu_pci_eeprom_read(np, here);
8195         if (err < 0)
8196             return err;
8197         if (err != 0x90)
8198             return -EINVAL;
8199 
8200         err = niu_pci_eeprom_read16_swp(np, here + 1);
8201         if (err < 0)
8202             return err;
8203 
8204         here = start + offset + 3;
8205         end = start + offset + err;
8206 
8207         offset += err;
8208 
8209         err = niu_pci_vpd_scan_props(np, here, end);
8210         if (err < 0)
8211             return err;
8212         /* ret == 1 is not an error */
8213         if (err == 1)
8214             return 0;
8215     }
8216     return 0;
8217 }
8218 
8219 /* ESPC_PIO_EN_ENABLE must be set */
8220 static u32 niu_pci_vpd_offset(struct niu *np)
8221 {
8222     u32 start = 0, end = ESPC_EEPROM_SIZE, ret;
8223     int err;
8224 
8225     while (start < end) {
8226         ret = start;
8227 
8228         /* ROM header signature?  */
8229         err = niu_pci_eeprom_read16(np, start +  0);
8230         if (err != 0x55aa)
8231             return 0;
8232 
8233         /* Apply offset to PCI data structure.  */
8234         err = niu_pci_eeprom_read16(np, start + 23);
8235         if (err < 0)
8236             return 0;
8237         start += err;
8238 
8239         /* Check for "PCIR" signature.  */
8240         err = niu_pci_eeprom_read16(np, start +  0);
8241         if (err != 0x5043)
8242             return 0;
8243         err = niu_pci_eeprom_read16(np, start +  2);
8244         if (err != 0x4952)
8245             return 0;
8246 
8247         /* Check for OBP image type.  */
8248         err = niu_pci_eeprom_read(np, start + 20);
8249         if (err < 0)
8250             return 0;
8251         if (err != 0x01) {
8252             err = niu_pci_eeprom_read(np, ret + 2);
8253             if (err < 0)
8254                 return 0;
8255 
8256             start = ret + (err * 512);
8257             continue;
8258         }
8259 
8260         err = niu_pci_eeprom_read16_swp(np, start + 8);
8261         if (err < 0)
8262             return err;
8263         ret += err;
8264 
8265         err = niu_pci_eeprom_read(np, ret + 0);
8266         if (err != 0x82)
8267             return 0;
8268 
8269         return ret;
8270     }
8271 
8272     return 0;
8273 }
8274 
8275 static int niu_phy_type_prop_decode(struct niu *np, const char *phy_prop)
8276 {
8277     if (!strcmp(phy_prop, "mif")) {
8278         /* 1G copper, MII */
8279         np->flags &= ~(NIU_FLAGS_FIBER |
8280                    NIU_FLAGS_10G);
8281         np->mac_xcvr = MAC_XCVR_MII;
8282     } else if (!strcmp(phy_prop, "xgf")) {
8283         /* 10G fiber, XPCS */
8284         np->flags |= (NIU_FLAGS_10G |
8285                   NIU_FLAGS_FIBER);
8286         np->mac_xcvr = MAC_XCVR_XPCS;
8287     } else if (!strcmp(phy_prop, "pcs")) {
8288         /* 1G fiber, PCS */
8289         np->flags &= ~NIU_FLAGS_10G;
8290         np->flags |= NIU_FLAGS_FIBER;
8291         np->mac_xcvr = MAC_XCVR_PCS;
8292     } else if (!strcmp(phy_prop, "xgc")) {
8293         /* 10G copper, XPCS */
8294         np->flags |= NIU_FLAGS_10G;
8295         np->flags &= ~NIU_FLAGS_FIBER;
8296         np->mac_xcvr = MAC_XCVR_XPCS;
8297     } else if (!strcmp(phy_prop, "xgsd") || !strcmp(phy_prop, "gsd")) {
8298         /* 10G Serdes or 1G Serdes, default to 10G */
8299         np->flags |= NIU_FLAGS_10G;
8300         np->flags &= ~NIU_FLAGS_FIBER;
8301         np->flags |= NIU_FLAGS_XCVR_SERDES;
8302         np->mac_xcvr = MAC_XCVR_XPCS;
8303     } else {
8304         return -EINVAL;
8305     }
8306     return 0;
8307 }
8308 
8309 static int niu_pci_vpd_get_nports(struct niu *np)
8310 {
8311     int ports = 0;
8312 
8313     if ((!strcmp(np->vpd.model, NIU_QGC_LP_MDL_STR)) ||
8314         (!strcmp(np->vpd.model, NIU_QGC_PEM_MDL_STR)) ||
8315         (!strcmp(np->vpd.model, NIU_MARAMBA_MDL_STR)) ||
8316         (!strcmp(np->vpd.model, NIU_KIMI_MDL_STR)) ||
8317         (!strcmp(np->vpd.model, NIU_ALONSO_MDL_STR))) {
8318         ports = 4;
8319     } else if ((!strcmp(np->vpd.model, NIU_2XGF_LP_MDL_STR)) ||
8320            (!strcmp(np->vpd.model, NIU_2XGF_PEM_MDL_STR)) ||
8321            (!strcmp(np->vpd.model, NIU_FOXXY_MDL_STR)) ||
8322            (!strcmp(np->vpd.model, NIU_2XGF_MRVL_MDL_STR))) {
8323         ports = 2;
8324     }
8325 
8326     return ports;
8327 }
8328 
8329 static void niu_pci_vpd_validate(struct niu *np)
8330 {
8331     struct net_device *dev = np->dev;
8332     struct niu_vpd *vpd = &np->vpd;
8333     u8 addr[ETH_ALEN];
8334     u8 val8;
8335 
8336     if (!is_valid_ether_addr(&vpd->local_mac[0])) {
8337         dev_err(np->device, "VPD MAC invalid, falling back to SPROM\n");
8338 
8339         np->flags &= ~NIU_FLAGS_VPD_VALID;
8340         return;
8341     }
8342 
8343     if (!strcmp(np->vpd.model, NIU_ALONSO_MDL_STR) ||
8344         !strcmp(np->vpd.model, NIU_KIMI_MDL_STR)) {
8345         np->flags |= NIU_FLAGS_10G;
8346         np->flags &= ~NIU_FLAGS_FIBER;
8347         np->flags |= NIU_FLAGS_XCVR_SERDES;
8348         np->mac_xcvr = MAC_XCVR_PCS;
8349         if (np->port > 1) {
8350             np->flags |= NIU_FLAGS_FIBER;
8351             np->flags &= ~NIU_FLAGS_10G;
8352         }
8353         if (np->flags & NIU_FLAGS_10G)
8354             np->mac_xcvr = MAC_XCVR_XPCS;
8355     } else if (!strcmp(np->vpd.model, NIU_FOXXY_MDL_STR)) {
8356         np->flags |= (NIU_FLAGS_10G | NIU_FLAGS_FIBER |
8357                   NIU_FLAGS_HOTPLUG_PHY);
8358     } else if (niu_phy_type_prop_decode(np, np->vpd.phy_type)) {
8359         dev_err(np->device, "Illegal phy string [%s]\n",
8360             np->vpd.phy_type);
8361         dev_err(np->device, "Falling back to SPROM\n");
8362         np->flags &= ~NIU_FLAGS_VPD_VALID;
8363         return;
8364     }
8365 
8366     ether_addr_copy(addr, vpd->local_mac);
8367 
8368     val8 = addr[5];
8369     addr[5] += np->port;
8370     if (addr[5] < val8)
8371         addr[4]++;
8372 
8373     eth_hw_addr_set(dev, addr);
8374 }
8375 
8376 static int niu_pci_probe_sprom(struct niu *np)
8377 {
8378     struct net_device *dev = np->dev;
8379     u8 addr[ETH_ALEN];
8380     int len, i;
8381     u64 val, sum;
8382     u8 val8;
8383 
8384     val = (nr64(ESPC_VER_IMGSZ) & ESPC_VER_IMGSZ_IMGSZ);
8385     val >>= ESPC_VER_IMGSZ_IMGSZ_SHIFT;
8386     len = val / 4;
8387 
8388     np->eeprom_len = len;
8389 
8390     netif_printk(np, probe, KERN_DEBUG, np->dev,
8391              "SPROM: Image size %llu\n", (unsigned long long)val);
8392 
8393     sum = 0;
8394     for (i = 0; i < len; i++) {
8395         val = nr64(ESPC_NCR(i));
8396         sum += (val >>  0) & 0xff;
8397         sum += (val >>  8) & 0xff;
8398         sum += (val >> 16) & 0xff;
8399         sum += (val >> 24) & 0xff;
8400     }
8401     netif_printk(np, probe, KERN_DEBUG, np->dev,
8402              "SPROM: Checksum %x\n", (int)(sum & 0xff));
8403     if ((sum & 0xff) != 0xab) {
8404         dev_err(np->device, "Bad SPROM checksum (%x, should be 0xab)\n", (int)(sum & 0xff));
8405         return -EINVAL;
8406     }
8407 
8408     val = nr64(ESPC_PHY_TYPE);
8409     switch (np->port) {
8410     case 0:
8411         val8 = (val & ESPC_PHY_TYPE_PORT0) >>
8412             ESPC_PHY_TYPE_PORT0_SHIFT;
8413         break;
8414     case 1:
8415         val8 = (val & ESPC_PHY_TYPE_PORT1) >>
8416             ESPC_PHY_TYPE_PORT1_SHIFT;
8417         break;
8418     case 2:
8419         val8 = (val & ESPC_PHY_TYPE_PORT2) >>
8420             ESPC_PHY_TYPE_PORT2_SHIFT;
8421         break;
8422     case 3:
8423         val8 = (val & ESPC_PHY_TYPE_PORT3) >>
8424             ESPC_PHY_TYPE_PORT3_SHIFT;
8425         break;
8426     default:
8427         dev_err(np->device, "Bogus port number %u\n",
8428             np->port);
8429         return -EINVAL;
8430     }
8431     netif_printk(np, probe, KERN_DEBUG, np->dev,
8432              "SPROM: PHY type %x\n", val8);
8433 
8434     switch (val8) {
8435     case ESPC_PHY_TYPE_1G_COPPER:
8436         /* 1G copper, MII */
8437         np->flags &= ~(NIU_FLAGS_FIBER |
8438                    NIU_FLAGS_10G);
8439         np->mac_xcvr = MAC_XCVR_MII;
8440         break;
8441 
8442     case ESPC_PHY_TYPE_1G_FIBER:
8443         /* 1G fiber, PCS */
8444         np->flags &= ~NIU_FLAGS_10G;
8445         np->flags |= NIU_FLAGS_FIBER;
8446         np->mac_xcvr = MAC_XCVR_PCS;
8447         break;
8448 
8449     case ESPC_PHY_TYPE_10G_COPPER:
8450         /* 10G copper, XPCS */
8451         np->flags |= NIU_FLAGS_10G;
8452         np->flags &= ~NIU_FLAGS_FIBER;
8453         np->mac_xcvr = MAC_XCVR_XPCS;
8454         break;
8455 
8456     case ESPC_PHY_TYPE_10G_FIBER:
8457         /* 10G fiber, XPCS */
8458         np->flags |= (NIU_FLAGS_10G |
8459                   NIU_FLAGS_FIBER);
8460         np->mac_xcvr = MAC_XCVR_XPCS;
8461         break;
8462 
8463     default:
8464         dev_err(np->device, "Bogus SPROM phy type %u\n", val8);
8465         return -EINVAL;
8466     }
8467 
8468     val = nr64(ESPC_MAC_ADDR0);
8469     netif_printk(np, probe, KERN_DEBUG, np->dev,
8470              "SPROM: MAC_ADDR0[%08llx]\n", (unsigned long long)val);
8471     addr[0] = (val >>  0) & 0xff;
8472     addr[1] = (val >>  8) & 0xff;
8473     addr[2] = (val >> 16) & 0xff;
8474     addr[3] = (val >> 24) & 0xff;
8475 
8476     val = nr64(ESPC_MAC_ADDR1);
8477     netif_printk(np, probe, KERN_DEBUG, np->dev,
8478              "SPROM: MAC_ADDR1[%08llx]\n", (unsigned long long)val);
8479     addr[4] = (val >>  0) & 0xff;
8480     addr[5] = (val >>  8) & 0xff;
8481 
8482     if (!is_valid_ether_addr(addr)) {
8483         dev_err(np->device, "SPROM MAC address invalid [ %pM ]\n",
8484             addr);
8485         return -EINVAL;
8486     }
8487 
8488     val8 = addr[5];
8489     addr[5] += np->port;
8490     if (addr[5] < val8)
8491         addr[4]++;
8492 
8493     eth_hw_addr_set(dev, addr);
8494 
8495     val = nr64(ESPC_MOD_STR_LEN);
8496     netif_printk(np, probe, KERN_DEBUG, np->dev,
8497              "SPROM: MOD_STR_LEN[%llu]\n", (unsigned long long)val);
8498     if (val >= 8 * 4)
8499         return -EINVAL;
8500 
8501     for (i = 0; i < val; i += 4) {
8502         u64 tmp = nr64(ESPC_NCR(5 + (i / 4)));
8503 
8504         np->vpd.model[i + 3] = (tmp >>  0) & 0xff;
8505         np->vpd.model[i + 2] = (tmp >>  8) & 0xff;
8506         np->vpd.model[i + 1] = (tmp >> 16) & 0xff;
8507         np->vpd.model[i + 0] = (tmp >> 24) & 0xff;
8508     }
8509     np->vpd.model[val] = '\0';
8510 
8511     val = nr64(ESPC_BD_MOD_STR_LEN);
8512     netif_printk(np, probe, KERN_DEBUG, np->dev,
8513              "SPROM: BD_MOD_STR_LEN[%llu]\n", (unsigned long long)val);
8514     if (val >= 4 * 4)
8515         return -EINVAL;
8516 
8517     for (i = 0; i < val; i += 4) {
8518         u64 tmp = nr64(ESPC_NCR(14 + (i / 4)));
8519 
8520         np->vpd.board_model[i + 3] = (tmp >>  0) & 0xff;
8521         np->vpd.board_model[i + 2] = (tmp >>  8) & 0xff;
8522         np->vpd.board_model[i + 1] = (tmp >> 16) & 0xff;
8523         np->vpd.board_model[i + 0] = (tmp >> 24) & 0xff;
8524     }
8525     np->vpd.board_model[val] = '\0';
8526 
8527     np->vpd.mac_num =
8528         nr64(ESPC_NUM_PORTS_MACS) & ESPC_NUM_PORTS_MACS_VAL;
8529     netif_printk(np, probe, KERN_DEBUG, np->dev,
8530              "SPROM: NUM_PORTS_MACS[%d]\n", np->vpd.mac_num);
8531 
8532     return 0;
8533 }
8534 
8535 static int niu_get_and_validate_port(struct niu *np)
8536 {
8537     struct niu_parent *parent = np->parent;
8538 
8539     if (np->port <= 1)
8540         np->flags |= NIU_FLAGS_XMAC;
8541 
8542     if (!parent->num_ports) {
8543         if (parent->plat_type == PLAT_TYPE_NIU) {
8544             parent->num_ports = 2;
8545         } else {
8546             parent->num_ports = niu_pci_vpd_get_nports(np);
8547             if (!parent->num_ports) {
8548                 /* Fall back to SPROM as last resort.
8549                  * This will fail on most cards.
8550                  */
8551                 parent->num_ports = nr64(ESPC_NUM_PORTS_MACS) &
8552                     ESPC_NUM_PORTS_MACS_VAL;
8553 
8554                 /* All of the current probing methods fail on
8555                  * Maramba on-board parts.
8556                  */
8557                 if (!parent->num_ports)
8558                     parent->num_ports = 4;
8559             }
8560         }
8561     }
8562 
8563     if (np->port >= parent->num_ports)
8564         return -ENODEV;
8565 
8566     return 0;
8567 }
8568 
8569 static int phy_record(struct niu_parent *parent, struct phy_probe_info *p,
8570               int dev_id_1, int dev_id_2, u8 phy_port, int type)
8571 {
8572     u32 id = (dev_id_1 << 16) | dev_id_2;
8573     u8 idx;
8574 
8575     if (dev_id_1 < 0 || dev_id_2 < 0)
8576         return 0;
8577     if (type == PHY_TYPE_PMA_PMD || type == PHY_TYPE_PCS) {
8578         /* Because of the NIU_PHY_ID_MASK being applied, the 8704
8579          * test covers the 8706 as well.
8580          */
8581         if (((id & NIU_PHY_ID_MASK) != NIU_PHY_ID_BCM8704) &&
8582             ((id & NIU_PHY_ID_MASK) != NIU_PHY_ID_MRVL88X2011))
8583             return 0;
8584     } else {
8585         if ((id & NIU_PHY_ID_MASK) != NIU_PHY_ID_BCM5464R)
8586             return 0;
8587     }
8588 
8589     pr_info("niu%d: Found PHY %08x type %s at phy_port %u\n",
8590         parent->index, id,
8591         type == PHY_TYPE_PMA_PMD ? "PMA/PMD" :
8592         type == PHY_TYPE_PCS ? "PCS" : "MII",
8593         phy_port);
8594 
8595     if (p->cur[type] >= NIU_MAX_PORTS) {
8596         pr_err("Too many PHY ports\n");
8597         return -EINVAL;
8598     }
8599     idx = p->cur[type];
8600     p->phy_id[type][idx] = id;
8601     p->phy_port[type][idx] = phy_port;
8602     p->cur[type] = idx + 1;
8603     return 0;
8604 }
8605 
8606 static int port_has_10g(struct phy_probe_info *p, int port)
8607 {
8608     int i;
8609 
8610     for (i = 0; i < p->cur[PHY_TYPE_PMA_PMD]; i++) {
8611         if (p->phy_port[PHY_TYPE_PMA_PMD][i] == port)
8612             return 1;
8613     }
8614     for (i = 0; i < p->cur[PHY_TYPE_PCS]; i++) {
8615         if (p->phy_port[PHY_TYPE_PCS][i] == port)
8616             return 1;
8617     }
8618 
8619     return 0;
8620 }
8621 
8622 static int count_10g_ports(struct phy_probe_info *p, int *lowest)
8623 {
8624     int port, cnt;
8625 
8626     cnt = 0;
8627     *lowest = 32;
8628     for (port = 8; port < 32; port++) {
8629         if (port_has_10g(p, port)) {
8630             if (!cnt)
8631                 *lowest = port;
8632             cnt++;
8633         }
8634     }
8635 
8636     return cnt;
8637 }
8638 
8639 static int count_1g_ports(struct phy_probe_info *p, int *lowest)
8640 {
8641     *lowest = 32;
8642     if (p->cur[PHY_TYPE_MII])
8643         *lowest = p->phy_port[PHY_TYPE_MII][0];
8644 
8645     return p->cur[PHY_TYPE_MII];
8646 }
8647 
8648 static void niu_n2_divide_channels(struct niu_parent *parent)
8649 {
8650     int num_ports = parent->num_ports;
8651     int i;
8652 
8653     for (i = 0; i < num_ports; i++) {
8654         parent->rxchan_per_port[i] = (16 / num_ports);
8655         parent->txchan_per_port[i] = (16 / num_ports);
8656 
8657         pr_info("niu%d: Port %u [%u RX chans] [%u TX chans]\n",
8658             parent->index, i,
8659             parent->rxchan_per_port[i],
8660             parent->txchan_per_port[i]);
8661     }
8662 }
8663 
8664 static void niu_divide_channels(struct niu_parent *parent,
8665                 int num_10g, int num_1g)
8666 {
8667     int num_ports = parent->num_ports;
8668     int rx_chans_per_10g, rx_chans_per_1g;
8669     int tx_chans_per_10g, tx_chans_per_1g;
8670     int i, tot_rx, tot_tx;
8671 
8672     if (!num_10g || !num_1g) {
8673         rx_chans_per_10g = rx_chans_per_1g =
8674             (NIU_NUM_RXCHAN / num_ports);
8675         tx_chans_per_10g = tx_chans_per_1g =
8676             (NIU_NUM_TXCHAN / num_ports);
8677     } else {
8678         rx_chans_per_1g = NIU_NUM_RXCHAN / 8;
8679         rx_chans_per_10g = (NIU_NUM_RXCHAN -
8680                     (rx_chans_per_1g * num_1g)) /
8681             num_10g;
8682 
8683         tx_chans_per_1g = NIU_NUM_TXCHAN / 6;
8684         tx_chans_per_10g = (NIU_NUM_TXCHAN -
8685                     (tx_chans_per_1g * num_1g)) /
8686             num_10g;
8687     }
8688 
8689     tot_rx = tot_tx = 0;
8690     for (i = 0; i < num_ports; i++) {
8691         int type = phy_decode(parent->port_phy, i);
8692 
8693         if (type == PORT_TYPE_10G) {
8694             parent->rxchan_per_port[i] = rx_chans_per_10g;
8695             parent->txchan_per_port[i] = tx_chans_per_10g;
8696         } else {
8697             parent->rxchan_per_port[i] = rx_chans_per_1g;
8698             parent->txchan_per_port[i] = tx_chans_per_1g;
8699         }
8700         pr_info("niu%d: Port %u [%u RX chans] [%u TX chans]\n",
8701             parent->index, i,
8702             parent->rxchan_per_port[i],
8703             parent->txchan_per_port[i]);
8704         tot_rx += parent->rxchan_per_port[i];
8705         tot_tx += parent->txchan_per_port[i];
8706     }
8707 
8708     if (tot_rx > NIU_NUM_RXCHAN) {
8709         pr_err("niu%d: Too many RX channels (%d), resetting to one per port\n",
8710                parent->index, tot_rx);
8711         for (i = 0; i < num_ports; i++)
8712             parent->rxchan_per_port[i] = 1;
8713     }
8714     if (tot_tx > NIU_NUM_TXCHAN) {
8715         pr_err("niu%d: Too many TX channels (%d), resetting to one per port\n",
8716                parent->index, tot_tx);
8717         for (i = 0; i < num_ports; i++)
8718             parent->txchan_per_port[i] = 1;
8719     }
8720     if (tot_rx < NIU_NUM_RXCHAN || tot_tx < NIU_NUM_TXCHAN) {
8721         pr_warn("niu%d: Driver bug, wasted channels, RX[%d] TX[%d]\n",
8722             parent->index, tot_rx, tot_tx);
8723     }
8724 }
8725 
8726 static void niu_divide_rdc_groups(struct niu_parent *parent,
8727                   int num_10g, int num_1g)
8728 {
8729     int i, num_ports = parent->num_ports;
8730     int rdc_group, rdc_groups_per_port;
8731     int rdc_channel_base;
8732 
8733     rdc_group = 0;
8734     rdc_groups_per_port = NIU_NUM_RDC_TABLES / num_ports;
8735 
8736     rdc_channel_base = 0;
8737 
8738     for (i = 0; i < num_ports; i++) {
8739         struct niu_rdc_tables *tp = &parent->rdc_group_cfg[i];
8740         int grp, num_channels = parent->rxchan_per_port[i];
8741         int this_channel_offset;
8742 
8743         tp->first_table_num = rdc_group;
8744         tp->num_tables = rdc_groups_per_port;
8745         this_channel_offset = 0;
8746         for (grp = 0; grp < tp->num_tables; grp++) {
8747             struct rdc_table *rt = &tp->tables[grp];
8748             int slot;
8749 
8750             pr_info("niu%d: Port %d RDC tbl(%d) [ ",
8751                 parent->index, i, tp->first_table_num + grp);
8752             for (slot = 0; slot < NIU_RDC_TABLE_SLOTS; slot++) {
8753                 rt->rxdma_channel[slot] =
8754                     rdc_channel_base + this_channel_offset;
8755 
8756                 pr_cont("%d ", rt->rxdma_channel[slot]);
8757 
8758                 if (++this_channel_offset == num_channels)
8759                     this_channel_offset = 0;
8760             }
8761             pr_cont("]\n");
8762         }
8763 
8764         parent->rdc_default[i] = rdc_channel_base;
8765 
8766         rdc_channel_base += num_channels;
8767         rdc_group += rdc_groups_per_port;
8768     }
8769 }
8770 
8771 static int fill_phy_probe_info(struct niu *np, struct niu_parent *parent,
8772                    struct phy_probe_info *info)
8773 {
8774     unsigned long flags;
8775     int port, err;
8776 
8777     memset(info, 0, sizeof(*info));
8778 
8779     /* Port 0 to 7 are reserved for onboard Serdes, probe the rest.  */
8780     niu_lock_parent(np, flags);
8781     err = 0;
8782     for (port = 8; port < 32; port++) {
8783         int dev_id_1, dev_id_2;
8784 
8785         dev_id_1 = mdio_read(np, port,
8786                      NIU_PMA_PMD_DEV_ADDR, MII_PHYSID1);
8787         dev_id_2 = mdio_read(np, port,
8788                      NIU_PMA_PMD_DEV_ADDR, MII_PHYSID2);
8789         err = phy_record(parent, info, dev_id_1, dev_id_2, port,
8790                  PHY_TYPE_PMA_PMD);
8791         if (err)
8792             break;
8793         dev_id_1 = mdio_read(np, port,
8794                      NIU_PCS_DEV_ADDR, MII_PHYSID1);
8795         dev_id_2 = mdio_read(np, port,
8796                      NIU_PCS_DEV_ADDR, MII_PHYSID2);
8797         err = phy_record(parent, info, dev_id_1, dev_id_2, port,
8798                  PHY_TYPE_PCS);
8799         if (err)
8800             break;
8801         dev_id_1 = mii_read(np, port, MII_PHYSID1);
8802         dev_id_2 = mii_read(np, port, MII_PHYSID2);
8803         err = phy_record(parent, info, dev_id_1, dev_id_2, port,
8804                  PHY_TYPE_MII);
8805         if (err)
8806             break;
8807     }
8808     niu_unlock_parent(np, flags);
8809 
8810     return err;
8811 }
8812 
8813 static int walk_phys(struct niu *np, struct niu_parent *parent)
8814 {
8815     struct phy_probe_info *info = &parent->phy_probe_info;
8816     int lowest_10g, lowest_1g;
8817     int num_10g, num_1g;
8818     u32 val;
8819     int err;
8820 
8821     num_10g = num_1g = 0;
8822 
8823     if (!strcmp(np->vpd.model, NIU_ALONSO_MDL_STR) ||
8824         !strcmp(np->vpd.model, NIU_KIMI_MDL_STR)) {
8825         num_10g = 0;
8826         num_1g = 2;
8827         parent->plat_type = PLAT_TYPE_ATCA_CP3220;
8828         parent->num_ports = 4;
8829         val = (phy_encode(PORT_TYPE_1G, 0) |
8830                phy_encode(PORT_TYPE_1G, 1) |
8831                phy_encode(PORT_TYPE_1G, 2) |
8832                phy_encode(PORT_TYPE_1G, 3));
8833     } else if (!strcmp(np->vpd.model, NIU_FOXXY_MDL_STR)) {
8834         num_10g = 2;
8835         num_1g = 0;
8836         parent->num_ports = 2;
8837         val = (phy_encode(PORT_TYPE_10G, 0) |
8838                phy_encode(PORT_TYPE_10G, 1));
8839     } else if ((np->flags & NIU_FLAGS_XCVR_SERDES) &&
8840            (parent->plat_type == PLAT_TYPE_NIU)) {
8841         /* this is the Monza case */
8842         if (np->flags & NIU_FLAGS_10G) {
8843             val = (phy_encode(PORT_TYPE_10G, 0) |
8844                    phy_encode(PORT_TYPE_10G, 1));
8845         } else {
8846             val = (phy_encode(PORT_TYPE_1G, 0) |
8847                    phy_encode(PORT_TYPE_1G, 1));
8848         }
8849     } else {
8850         err = fill_phy_probe_info(np, parent, info);
8851         if (err)
8852             return err;
8853 
8854         num_10g = count_10g_ports(info, &lowest_10g);
8855         num_1g = count_1g_ports(info, &lowest_1g);
8856 
8857         switch ((num_10g << 4) | num_1g) {
8858         case 0x24:
8859             if (lowest_1g == 10)
8860                 parent->plat_type = PLAT_TYPE_VF_P0;
8861             else if (lowest_1g == 26)
8862                 parent->plat_type = PLAT_TYPE_VF_P1;
8863             else
8864                 goto unknown_vg_1g_port;
8865 
8866             fallthrough;
8867         case 0x22:
8868             val = (phy_encode(PORT_TYPE_10G, 0) |
8869                    phy_encode(PORT_TYPE_10G, 1) |
8870                    phy_encode(PORT_TYPE_1G, 2) |
8871                    phy_encode(PORT_TYPE_1G, 3));
8872             break;
8873 
8874         case 0x20:
8875             val = (phy_encode(PORT_TYPE_10G, 0) |
8876                    phy_encode(PORT_TYPE_10G, 1));
8877             break;
8878 
8879         case 0x10:
8880             val = phy_encode(PORT_TYPE_10G, np->port);
8881             break;
8882 
8883         case 0x14:
8884             if (lowest_1g == 10)
8885                 parent->plat_type = PLAT_TYPE_VF_P0;
8886             else if (lowest_1g == 26)
8887                 parent->plat_type = PLAT_TYPE_VF_P1;
8888             else
8889                 goto unknown_vg_1g_port;
8890 
8891             fallthrough;
8892         case 0x13:
8893             if ((lowest_10g & 0x7) == 0)
8894                 val = (phy_encode(PORT_TYPE_10G, 0) |
8895                        phy_encode(PORT_TYPE_1G, 1) |
8896                        phy_encode(PORT_TYPE_1G, 2) |
8897                        phy_encode(PORT_TYPE_1G, 3));
8898             else
8899                 val = (phy_encode(PORT_TYPE_1G, 0) |
8900                        phy_encode(PORT_TYPE_10G, 1) |
8901                        phy_encode(PORT_TYPE_1G, 2) |
8902                        phy_encode(PORT_TYPE_1G, 3));
8903             break;
8904 
8905         case 0x04:
8906             if (lowest_1g == 10)
8907                 parent->plat_type = PLAT_TYPE_VF_P0;
8908             else if (lowest_1g == 26)
8909                 parent->plat_type = PLAT_TYPE_VF_P1;
8910             else
8911                 goto unknown_vg_1g_port;
8912 
8913             val = (phy_encode(PORT_TYPE_1G, 0) |
8914                    phy_encode(PORT_TYPE_1G, 1) |
8915                    phy_encode(PORT_TYPE_1G, 2) |
8916                    phy_encode(PORT_TYPE_1G, 3));
8917             break;
8918 
8919         default:
8920             pr_err("Unsupported port config 10G[%d] 1G[%d]\n",
8921                    num_10g, num_1g);
8922             return -EINVAL;
8923         }
8924     }
8925 
8926     parent->port_phy = val;
8927 
8928     if (parent->plat_type == PLAT_TYPE_NIU)
8929         niu_n2_divide_channels(parent);
8930     else
8931         niu_divide_channels(parent, num_10g, num_1g);
8932 
8933     niu_divide_rdc_groups(parent, num_10g, num_1g);
8934 
8935     return 0;
8936 
8937 unknown_vg_1g_port:
8938     pr_err("Cannot identify platform type, 1gport=%d\n", lowest_1g);
8939     return -EINVAL;
8940 }
8941 
8942 static int niu_probe_ports(struct niu *np)
8943 {
8944     struct niu_parent *parent = np->parent;
8945     int err, i;
8946 
8947     if (parent->port_phy == PORT_PHY_UNKNOWN) {
8948         err = walk_phys(np, parent);
8949         if (err)
8950             return err;
8951 
8952         niu_set_ldg_timer_res(np, 2);
8953         for (i = 0; i <= LDN_MAX; i++)
8954             niu_ldn_irq_enable(np, i, 0);
8955     }
8956 
8957     if (parent->port_phy == PORT_PHY_INVALID)
8958         return -EINVAL;
8959 
8960     return 0;
8961 }
8962 
8963 static int niu_classifier_swstate_init(struct niu *np)
8964 {
8965     struct niu_classifier *cp = &np->clas;
8966 
8967     cp->tcam_top = (u16) np->port;
8968     cp->tcam_sz = np->parent->tcam_num_entries / np->parent->num_ports;
8969     cp->h1_init = 0xffffffff;
8970     cp->h2_init = 0xffff;
8971 
8972     return fflp_early_init(np);
8973 }
8974 
8975 static void niu_link_config_init(struct niu *np)
8976 {
8977     struct niu_link_config *lp = &np->link_config;
8978 
8979     lp->advertising = (ADVERTISED_10baseT_Half |
8980                ADVERTISED_10baseT_Full |
8981                ADVERTISED_100baseT_Half |
8982                ADVERTISED_100baseT_Full |
8983                ADVERTISED_1000baseT_Half |
8984                ADVERTISED_1000baseT_Full |
8985                ADVERTISED_10000baseT_Full |
8986                ADVERTISED_Autoneg);
8987     lp->speed = lp->active_speed = SPEED_INVALID;
8988     lp->duplex = DUPLEX_FULL;
8989     lp->active_duplex = DUPLEX_INVALID;
8990     lp->autoneg = 1;
8991 #if 0
8992     lp->loopback_mode = LOOPBACK_MAC;
8993     lp->active_speed = SPEED_10000;
8994     lp->active_duplex = DUPLEX_FULL;
8995 #else
8996     lp->loopback_mode = LOOPBACK_DISABLED;
8997 #endif
8998 }
8999 
9000 static int niu_init_mac_ipp_pcs_base(struct niu *np)
9001 {
9002     switch (np->port) {
9003     case 0:
9004         np->mac_regs = np->regs + XMAC_PORT0_OFF;
9005         np->ipp_off  = 0x00000;
9006         np->pcs_off  = 0x04000;
9007         np->xpcs_off = 0x02000;
9008         break;
9009 
9010     case 1:
9011         np->mac_regs = np->regs + XMAC_PORT1_OFF;
9012         np->ipp_off  = 0x08000;
9013         np->pcs_off  = 0x0a000;
9014         np->xpcs_off = 0x08000;
9015         break;
9016 
9017     case 2:
9018         np->mac_regs = np->regs + BMAC_PORT2_OFF;
9019         np->ipp_off  = 0x04000;
9020         np->pcs_off  = 0x0e000;
9021         np->xpcs_off = ~0UL;
9022         break;
9023 
9024     case 3:
9025         np->mac_regs = np->regs + BMAC_PORT3_OFF;
9026         np->ipp_off  = 0x0c000;
9027         np->pcs_off  = 0x12000;
9028         np->xpcs_off = ~0UL;
9029         break;
9030 
9031     default:
9032         dev_err(np->device, "Port %u is invalid, cannot compute MAC block offset\n", np->port);
9033         return -EINVAL;
9034     }
9035 
9036     return 0;
9037 }
9038 
9039 static void niu_try_msix(struct niu *np, u8 *ldg_num_map)
9040 {
9041     struct msix_entry msi_vec[NIU_NUM_LDG];
9042     struct niu_parent *parent = np->parent;
9043     struct pci_dev *pdev = np->pdev;
9044     int i, num_irqs;
9045     u8 first_ldg;
9046 
9047     first_ldg = (NIU_NUM_LDG / parent->num_ports) * np->port;
9048     for (i = 0; i < (NIU_NUM_LDG / parent->num_ports); i++)
9049         ldg_num_map[i] = first_ldg + i;
9050 
9051     num_irqs = (parent->rxchan_per_port[np->port] +
9052             parent->txchan_per_port[np->port] +
9053             (np->port == 0 ? 3 : 1));
9054     BUG_ON(num_irqs > (NIU_NUM_LDG / parent->num_ports));
9055 
9056     for (i = 0; i < num_irqs; i++) {
9057         msi_vec[i].vector = 0;
9058         msi_vec[i].entry = i;
9059     }
9060 
9061     num_irqs = pci_enable_msix_range(pdev, msi_vec, 1, num_irqs);
9062     if (num_irqs < 0) {
9063         np->flags &= ~NIU_FLAGS_MSIX;
9064         return;
9065     }
9066 
9067     np->flags |= NIU_FLAGS_MSIX;
9068     for (i = 0; i < num_irqs; i++)
9069         np->ldg[i].irq = msi_vec[i].vector;
9070     np->num_ldg = num_irqs;
9071 }
9072 
9073 static int niu_n2_irq_init(struct niu *np, u8 *ldg_num_map)
9074 {
9075 #ifdef CONFIG_SPARC64
9076     struct platform_device *op = np->op;
9077     const u32 *int_prop;
9078     int i;
9079 
9080     int_prop = of_get_property(op->dev.of_node, "interrupts", NULL);
9081     if (!int_prop)
9082         return -ENODEV;
9083 
9084     for (i = 0; i < op->archdata.num_irqs; i++) {
9085         ldg_num_map[i] = int_prop[i];
9086         np->ldg[i].irq = op->archdata.irqs[i];
9087     }
9088 
9089     np->num_ldg = op->archdata.num_irqs;
9090 
9091     return 0;
9092 #else
9093     return -EINVAL;
9094 #endif
9095 }
9096 
9097 static int niu_ldg_init(struct niu *np)
9098 {
9099     struct niu_parent *parent = np->parent;
9100     u8 ldg_num_map[NIU_NUM_LDG];
9101     int first_chan, num_chan;
9102     int i, err, ldg_rotor;
9103     u8 port;
9104 
9105     np->num_ldg = 1;
9106     np->ldg[0].irq = np->dev->irq;
9107     if (parent->plat_type == PLAT_TYPE_NIU) {
9108         err = niu_n2_irq_init(np, ldg_num_map);
9109         if (err)
9110             return err;
9111     } else
9112         niu_try_msix(np, ldg_num_map);
9113 
9114     port = np->port;
9115     for (i = 0; i < np->num_ldg; i++) {
9116         struct niu_ldg *lp = &np->ldg[i];
9117 
9118         netif_napi_add(np->dev, &lp->napi, niu_poll, 64);
9119 
9120         lp->np = np;
9121         lp->ldg_num = ldg_num_map[i];
9122         lp->timer = 2; /* XXX */
9123 
9124         /* On N2 NIU the firmware has setup the SID mappings so they go
9125          * to the correct values that will route the LDG to the proper
9126          * interrupt in the NCU interrupt table.
9127          */
9128         if (np->parent->plat_type != PLAT_TYPE_NIU) {
9129             err = niu_set_ldg_sid(np, lp->ldg_num, port, i);
9130             if (err)
9131                 return err;
9132         }
9133     }
9134 
9135     /* We adopt the LDG assignment ordering used by the N2 NIU
9136      * 'interrupt' properties because that simplifies a lot of
9137      * things.  This ordering is:
9138      *
9139      *  MAC
9140      *  MIF (if port zero)
9141      *  SYSERR  (if port zero)
9142      *  RX channels
9143      *  TX channels
9144      */
9145 
9146     ldg_rotor = 0;
9147 
9148     err = niu_ldg_assign_ldn(np, parent, ldg_num_map[ldg_rotor],
9149                   LDN_MAC(port));
9150     if (err)
9151         return err;
9152 
9153     ldg_rotor++;
9154     if (ldg_rotor == np->num_ldg)
9155         ldg_rotor = 0;
9156 
9157     if (port == 0) {
9158         err = niu_ldg_assign_ldn(np, parent,
9159                      ldg_num_map[ldg_rotor],
9160                      LDN_MIF);
9161         if (err)
9162             return err;
9163 
9164         ldg_rotor++;
9165         if (ldg_rotor == np->num_ldg)
9166             ldg_rotor = 0;
9167 
9168         err = niu_ldg_assign_ldn(np, parent,
9169                      ldg_num_map[ldg_rotor],
9170                      LDN_DEVICE_ERROR);
9171         if (err)
9172             return err;
9173 
9174         ldg_rotor++;
9175         if (ldg_rotor == np->num_ldg)
9176             ldg_rotor = 0;
9177 
9178     }
9179 
9180     first_chan = 0;
9181     for (i = 0; i < port; i++)
9182         first_chan += parent->rxchan_per_port[i];
9183     num_chan = parent->rxchan_per_port[port];
9184 
9185     for (i = first_chan; i < (first_chan + num_chan); i++) {
9186         err = niu_ldg_assign_ldn(np, parent,
9187                      ldg_num_map[ldg_rotor],
9188                      LDN_RXDMA(i));
9189         if (err)
9190             return err;
9191         ldg_rotor++;
9192         if (ldg_rotor == np->num_ldg)
9193             ldg_rotor = 0;
9194     }
9195 
9196     first_chan = 0;
9197     for (i = 0; i < port; i++)
9198         first_chan += parent->txchan_per_port[i];
9199     num_chan = parent->txchan_per_port[port];
9200     for (i = first_chan; i < (first_chan + num_chan); i++) {
9201         err = niu_ldg_assign_ldn(np, parent,
9202                      ldg_num_map[ldg_rotor],
9203                      LDN_TXDMA(i));
9204         if (err)
9205             return err;
9206         ldg_rotor++;
9207         if (ldg_rotor == np->num_ldg)
9208             ldg_rotor = 0;
9209     }
9210 
9211     return 0;
9212 }
9213 
9214 static void niu_ldg_free(struct niu *np)
9215 {
9216     if (np->flags & NIU_FLAGS_MSIX)
9217         pci_disable_msix(np->pdev);
9218 }
9219 
9220 static int niu_get_of_props(struct niu *np)
9221 {
9222 #ifdef CONFIG_SPARC64
9223     struct net_device *dev = np->dev;
9224     struct device_node *dp;
9225     const char *phy_type;
9226     const u8 *mac_addr;
9227     const char *model;
9228     int prop_len;
9229 
9230     if (np->parent->plat_type == PLAT_TYPE_NIU)
9231         dp = np->op->dev.of_node;
9232     else
9233         dp = pci_device_to_OF_node(np->pdev);
9234 
9235     phy_type = of_get_property(dp, "phy-type", NULL);
9236     if (!phy_type) {
9237         netdev_err(dev, "%pOF: OF node lacks phy-type property\n", dp);
9238         return -EINVAL;
9239     }
9240 
9241     if (!strcmp(phy_type, "none"))
9242         return -ENODEV;
9243 
9244     strcpy(np->vpd.phy_type, phy_type);
9245 
9246     if (niu_phy_type_prop_decode(np, np->vpd.phy_type)) {
9247         netdev_err(dev, "%pOF: Illegal phy string [%s]\n",
9248                dp, np->vpd.phy_type);
9249         return -EINVAL;
9250     }
9251 
9252     mac_addr = of_get_property(dp, "local-mac-address", &prop_len);
9253     if (!mac_addr) {
9254         netdev_err(dev, "%pOF: OF node lacks local-mac-address property\n",
9255                dp);
9256         return -EINVAL;
9257     }
9258     if (prop_len != dev->addr_len) {
9259         netdev_err(dev, "%pOF: OF MAC address prop len (%d) is wrong\n",
9260                dp, prop_len);
9261     }
9262     eth_hw_addr_set(dev, mac_addr);
9263     if (!is_valid_ether_addr(&dev->dev_addr[0])) {
9264         netdev_err(dev, "%pOF: OF MAC address is invalid\n", dp);
9265         netdev_err(dev, "%pOF: [ %pM ]\n", dp, dev->dev_addr);
9266         return -EINVAL;
9267     }
9268 
9269     model = of_get_property(dp, "model", NULL);
9270 
9271     if (model)
9272         strcpy(np->vpd.model, model);
9273 
9274     if (of_find_property(dp, "hot-swappable-phy", NULL)) {
9275         np->flags |= (NIU_FLAGS_10G | NIU_FLAGS_FIBER |
9276             NIU_FLAGS_HOTPLUG_PHY);
9277     }
9278 
9279     return 0;
9280 #else
9281     return -EINVAL;
9282 #endif
9283 }
9284 
9285 static int niu_get_invariants(struct niu *np)
9286 {
9287     int err, have_props;
9288     u32 offset;
9289 
9290     err = niu_get_of_props(np);
9291     if (err == -ENODEV)
9292         return err;
9293 
9294     have_props = !err;
9295 
9296     err = niu_init_mac_ipp_pcs_base(np);
9297     if (err)
9298         return err;
9299 
9300     if (have_props) {
9301         err = niu_get_and_validate_port(np);
9302         if (err)
9303             return err;
9304 
9305     } else  {
9306         if (np->parent->plat_type == PLAT_TYPE_NIU)
9307             return -EINVAL;
9308 
9309         nw64(ESPC_PIO_EN, ESPC_PIO_EN_ENABLE);
9310         offset = niu_pci_vpd_offset(np);
9311         netif_printk(np, probe, KERN_DEBUG, np->dev,
9312                  "%s() VPD offset [%08x]\n", __func__, offset);
9313         if (offset) {
9314             err = niu_pci_vpd_fetch(np, offset);
9315             if (err < 0)
9316                 return err;
9317         }
9318         nw64(ESPC_PIO_EN, 0);
9319 
9320         if (np->flags & NIU_FLAGS_VPD_VALID) {
9321             niu_pci_vpd_validate(np);
9322             err = niu_get_and_validate_port(np);
9323             if (err)
9324                 return err;
9325         }
9326 
9327         if (!(np->flags & NIU_FLAGS_VPD_VALID)) {
9328             err = niu_get_and_validate_port(np);
9329             if (err)
9330                 return err;
9331             err = niu_pci_probe_sprom(np);
9332             if (err)
9333                 return err;
9334         }
9335     }
9336 
9337     err = niu_probe_ports(np);
9338     if (err)
9339         return err;
9340 
9341     niu_ldg_init(np);
9342 
9343     niu_classifier_swstate_init(np);
9344     niu_link_config_init(np);
9345 
9346     err = niu_determine_phy_disposition(np);
9347     if (!err)
9348         err = niu_init_link(np);
9349 
9350     return err;
9351 }
9352 
9353 static LIST_HEAD(niu_parent_list);
9354 static DEFINE_MUTEX(niu_parent_lock);
9355 static int niu_parent_index;
9356 
9357 static ssize_t show_port_phy(struct device *dev,
9358                  struct device_attribute *attr, char *buf)
9359 {
9360     struct platform_device *plat_dev = to_platform_device(dev);
9361     struct niu_parent *p = dev_get_platdata(&plat_dev->dev);
9362     u32 port_phy = p->port_phy;
9363     char *orig_buf = buf;
9364     int i;
9365 
9366     if (port_phy == PORT_PHY_UNKNOWN ||
9367         port_phy == PORT_PHY_INVALID)
9368         return 0;
9369 
9370     for (i = 0; i < p->num_ports; i++) {
9371         const char *type_str;
9372         int type;
9373 
9374         type = phy_decode(port_phy, i);
9375         if (type == PORT_TYPE_10G)
9376             type_str = "10G";
9377         else
9378             type_str = "1G";
9379         buf += sprintf(buf,
9380                    (i == 0) ? "%s" : " %s",
9381                    type_str);
9382     }
9383     buf += sprintf(buf, "\n");
9384     return buf - orig_buf;
9385 }
9386 
9387 static ssize_t show_plat_type(struct device *dev,
9388                   struct device_attribute *attr, char *buf)
9389 {
9390     struct platform_device *plat_dev = to_platform_device(dev);
9391     struct niu_parent *p = dev_get_platdata(&plat_dev->dev);
9392     const char *type_str;
9393 
9394     switch (p->plat_type) {
9395     case PLAT_TYPE_ATLAS:
9396         type_str = "atlas";
9397         break;
9398     case PLAT_TYPE_NIU:
9399         type_str = "niu";
9400         break;
9401     case PLAT_TYPE_VF_P0:
9402         type_str = "vf_p0";
9403         break;
9404     case PLAT_TYPE_VF_P1:
9405         type_str = "vf_p1";
9406         break;
9407     default:
9408         type_str = "unknown";
9409         break;
9410     }
9411 
9412     return sprintf(buf, "%s\n", type_str);
9413 }
9414 
9415 static ssize_t __show_chan_per_port(struct device *dev,
9416                     struct device_attribute *attr, char *buf,
9417                     int rx)
9418 {
9419     struct platform_device *plat_dev = to_platform_device(dev);
9420     struct niu_parent *p = dev_get_platdata(&plat_dev->dev);
9421     char *orig_buf = buf;
9422     u8 *arr;
9423     int i;
9424 
9425     arr = (rx ? p->rxchan_per_port : p->txchan_per_port);
9426 
9427     for (i = 0; i < p->num_ports; i++) {
9428         buf += sprintf(buf,
9429                    (i == 0) ? "%d" : " %d",
9430                    arr[i]);
9431     }
9432     buf += sprintf(buf, "\n");
9433 
9434     return buf - orig_buf;
9435 }
9436 
9437 static ssize_t show_rxchan_per_port(struct device *dev,
9438                     struct device_attribute *attr, char *buf)
9439 {
9440     return __show_chan_per_port(dev, attr, buf, 1);
9441 }
9442 
9443 static ssize_t show_txchan_per_port(struct device *dev,
9444                     struct device_attribute *attr, char *buf)
9445 {
9446     return __show_chan_per_port(dev, attr, buf, 1);
9447 }
9448 
9449 static ssize_t show_num_ports(struct device *dev,
9450                   struct device_attribute *attr, char *buf)
9451 {
9452     struct platform_device *plat_dev = to_platform_device(dev);
9453     struct niu_parent *p = dev_get_platdata(&plat_dev->dev);
9454 
9455     return sprintf(buf, "%d\n", p->num_ports);
9456 }
9457 
9458 static struct device_attribute niu_parent_attributes[] = {
9459     __ATTR(port_phy, 0444, show_port_phy, NULL),
9460     __ATTR(plat_type, 0444, show_plat_type, NULL),
9461     __ATTR(rxchan_per_port, 0444, show_rxchan_per_port, NULL),
9462     __ATTR(txchan_per_port, 0444, show_txchan_per_port, NULL),
9463     __ATTR(num_ports, 0444, show_num_ports, NULL),
9464     {}
9465 };
9466 
9467 static struct niu_parent *niu_new_parent(struct niu *np,
9468                      union niu_parent_id *id, u8 ptype)
9469 {
9470     struct platform_device *plat_dev;
9471     struct niu_parent *p;
9472     int i;
9473 
9474     plat_dev = platform_device_register_simple("niu-board", niu_parent_index,
9475                            NULL, 0);
9476     if (IS_ERR(plat_dev))
9477         return NULL;
9478 
9479     for (i = 0; niu_parent_attributes[i].attr.name; i++) {
9480         int err = device_create_file(&plat_dev->dev,
9481                          &niu_parent_attributes[i]);
9482         if (err)
9483             goto fail_unregister;
9484     }
9485 
9486     p = kzalloc(sizeof(*p), GFP_KERNEL);
9487     if (!p)
9488         goto fail_unregister;
9489 
9490     p->index = niu_parent_index++;
9491 
9492     plat_dev->dev.platform_data = p;
9493     p->plat_dev = plat_dev;
9494 
9495     memcpy(&p->id, id, sizeof(*id));
9496     p->plat_type = ptype;
9497     INIT_LIST_HEAD(&p->list);
9498     atomic_set(&p->refcnt, 0);
9499     list_add(&p->list, &niu_parent_list);
9500     spin_lock_init(&p->lock);
9501 
9502     p->rxdma_clock_divider = 7500;
9503 
9504     p->tcam_num_entries = NIU_PCI_TCAM_ENTRIES;
9505     if (p->plat_type == PLAT_TYPE_NIU)
9506         p->tcam_num_entries = NIU_NONPCI_TCAM_ENTRIES;
9507 
9508     for (i = CLASS_CODE_USER_PROG1; i <= CLASS_CODE_SCTP_IPV6; i++) {
9509         int index = i - CLASS_CODE_USER_PROG1;
9510 
9511         p->tcam_key[index] = TCAM_KEY_TSEL;
9512         p->flow_key[index] = (FLOW_KEY_IPSA |
9513                       FLOW_KEY_IPDA |
9514                       FLOW_KEY_PROTO |
9515                       (FLOW_KEY_L4_BYTE12 <<
9516                        FLOW_KEY_L4_0_SHIFT) |
9517                       (FLOW_KEY_L4_BYTE12 <<
9518                        FLOW_KEY_L4_1_SHIFT));
9519     }
9520 
9521     for (i = 0; i < LDN_MAX + 1; i++)
9522         p->ldg_map[i] = LDG_INVALID;
9523 
9524     return p;
9525 
9526 fail_unregister:
9527     platform_device_unregister(plat_dev);
9528     return NULL;
9529 }
9530 
9531 static struct niu_parent *niu_get_parent(struct niu *np,
9532                      union niu_parent_id *id, u8 ptype)
9533 {
9534     struct niu_parent *p, *tmp;
9535     int port = np->port;
9536 
9537     mutex_lock(&niu_parent_lock);
9538     p = NULL;
9539     list_for_each_entry(tmp, &niu_parent_list, list) {
9540         if (!memcmp(id, &tmp->id, sizeof(*id))) {
9541             p = tmp;
9542             break;
9543         }
9544     }
9545     if (!p)
9546         p = niu_new_parent(np, id, ptype);
9547 
9548     if (p) {
9549         char port_name[8];
9550         int err;
9551 
9552         sprintf(port_name, "port%d", port);
9553         err = sysfs_create_link(&p->plat_dev->dev.kobj,
9554                     &np->device->kobj,
9555                     port_name);
9556         if (!err) {
9557             p->ports[port] = np;
9558             atomic_inc(&p->refcnt);
9559         }
9560     }
9561     mutex_unlock(&niu_parent_lock);
9562 
9563     return p;
9564 }
9565 
9566 static void niu_put_parent(struct niu *np)
9567 {
9568     struct niu_parent *p = np->parent;
9569     u8 port = np->port;
9570     char port_name[8];
9571 
9572     BUG_ON(!p || p->ports[port] != np);
9573 
9574     netif_printk(np, probe, KERN_DEBUG, np->dev,
9575              "%s() port[%u]\n", __func__, port);
9576 
9577     sprintf(port_name, "port%d", port);
9578 
9579     mutex_lock(&niu_parent_lock);
9580 
9581     sysfs_remove_link(&p->plat_dev->dev.kobj, port_name);
9582 
9583     p->ports[port] = NULL;
9584     np->parent = NULL;
9585 
9586     if (atomic_dec_and_test(&p->refcnt)) {
9587         list_del(&p->list);
9588         platform_device_unregister(p->plat_dev);
9589     }
9590 
9591     mutex_unlock(&niu_parent_lock);
9592 }
9593 
9594 static void *niu_pci_alloc_coherent(struct device *dev, size_t size,
9595                     u64 *handle, gfp_t flag)
9596 {
9597     dma_addr_t dh;
9598     void *ret;
9599 
9600     ret = dma_alloc_coherent(dev, size, &dh, flag);
9601     if (ret)
9602         *handle = dh;
9603     return ret;
9604 }
9605 
9606 static void niu_pci_free_coherent(struct device *dev, size_t size,
9607                   void *cpu_addr, u64 handle)
9608 {
9609     dma_free_coherent(dev, size, cpu_addr, handle);
9610 }
9611 
9612 static u64 niu_pci_map_page(struct device *dev, struct page *page,
9613                 unsigned long offset, size_t size,
9614                 enum dma_data_direction direction)
9615 {
9616     return dma_map_page(dev, page, offset, size, direction);
9617 }
9618 
9619 static void niu_pci_unmap_page(struct device *dev, u64 dma_address,
9620                    size_t size, enum dma_data_direction direction)
9621 {
9622     dma_unmap_page(dev, dma_address, size, direction);
9623 }
9624 
9625 static u64 niu_pci_map_single(struct device *dev, void *cpu_addr,
9626                   size_t size,
9627                   enum dma_data_direction direction)
9628 {
9629     return dma_map_single(dev, cpu_addr, size, direction);
9630 }
9631 
9632 static void niu_pci_unmap_single(struct device *dev, u64 dma_address,
9633                  size_t size,
9634                  enum dma_data_direction direction)
9635 {
9636     dma_unmap_single(dev, dma_address, size, direction);
9637 }
9638 
9639 static const struct niu_ops niu_pci_ops = {
9640     .alloc_coherent = niu_pci_alloc_coherent,
9641     .free_coherent  = niu_pci_free_coherent,
9642     .map_page   = niu_pci_map_page,
9643     .unmap_page = niu_pci_unmap_page,
9644     .map_single = niu_pci_map_single,
9645     .unmap_single   = niu_pci_unmap_single,
9646 };
9647 
9648 static void niu_driver_version(void)
9649 {
9650     static int niu_version_printed;
9651 
9652     if (niu_version_printed++ == 0)
9653         pr_info("%s", version);
9654 }
9655 
9656 static struct net_device *niu_alloc_and_init(struct device *gen_dev,
9657                          struct pci_dev *pdev,
9658                          struct platform_device *op,
9659                          const struct niu_ops *ops, u8 port)
9660 {
9661     struct net_device *dev;
9662     struct niu *np;
9663 
9664     dev = alloc_etherdev_mq(sizeof(struct niu), NIU_NUM_TXCHAN);
9665     if (!dev)
9666         return NULL;
9667 
9668     SET_NETDEV_DEV(dev, gen_dev);
9669 
9670     np = netdev_priv(dev);
9671     np->dev = dev;
9672     np->pdev = pdev;
9673     np->op = op;
9674     np->device = gen_dev;
9675     np->ops = ops;
9676 
9677     np->msg_enable = niu_debug;
9678 
9679     spin_lock_init(&np->lock);
9680     INIT_WORK(&np->reset_task, niu_reset_task);
9681 
9682     np->port = port;
9683 
9684     return dev;
9685 }
9686 
9687 static const struct net_device_ops niu_netdev_ops = {
9688     .ndo_open       = niu_open,
9689     .ndo_stop       = niu_close,
9690     .ndo_start_xmit     = niu_start_xmit,
9691     .ndo_get_stats64    = niu_get_stats,
9692     .ndo_set_rx_mode    = niu_set_rx_mode,
9693     .ndo_validate_addr  = eth_validate_addr,
9694     .ndo_set_mac_address    = niu_set_mac_addr,
9695     .ndo_eth_ioctl      = niu_ioctl,
9696     .ndo_tx_timeout     = niu_tx_timeout,
9697     .ndo_change_mtu     = niu_change_mtu,
9698 };
9699 
9700 static void niu_assign_netdev_ops(struct net_device *dev)
9701 {
9702     dev->netdev_ops = &niu_netdev_ops;
9703     dev->ethtool_ops = &niu_ethtool_ops;
9704     dev->watchdog_timeo = NIU_TX_TIMEOUT;
9705 }
9706 
9707 static void niu_device_announce(struct niu *np)
9708 {
9709     struct net_device *dev = np->dev;
9710 
9711     pr_info("%s: NIU Ethernet %pM\n", dev->name, dev->dev_addr);
9712 
9713     if (np->parent->plat_type == PLAT_TYPE_ATCA_CP3220) {
9714         pr_info("%s: Port type[%s] mode[%s:%s] XCVR[%s] phy[%s]\n",
9715                 dev->name,
9716                 (np->flags & NIU_FLAGS_XMAC ? "XMAC" : "BMAC"),
9717                 (np->flags & NIU_FLAGS_10G ? "10G" : "1G"),
9718                 (np->flags & NIU_FLAGS_FIBER ? "RGMII FIBER" : "SERDES"),
9719                 (np->mac_xcvr == MAC_XCVR_MII ? "MII" :
9720                  (np->mac_xcvr == MAC_XCVR_PCS ? "PCS" : "XPCS")),
9721                 np->vpd.phy_type);
9722     } else {
9723         pr_info("%s: Port type[%s] mode[%s:%s] XCVR[%s] phy[%s]\n",
9724                 dev->name,
9725                 (np->flags & NIU_FLAGS_XMAC ? "XMAC" : "BMAC"),
9726                 (np->flags & NIU_FLAGS_10G ? "10G" : "1G"),
9727                 (np->flags & NIU_FLAGS_FIBER ? "FIBER" :
9728                  (np->flags & NIU_FLAGS_XCVR_SERDES ? "SERDES" :
9729                   "COPPER")),
9730                 (np->mac_xcvr == MAC_XCVR_MII ? "MII" :
9731                  (np->mac_xcvr == MAC_XCVR_PCS ? "PCS" : "XPCS")),
9732                 np->vpd.phy_type);
9733     }
9734 }
9735 
9736 static void niu_set_basic_features(struct net_device *dev)
9737 {
9738     dev->hw_features = NETIF_F_SG | NETIF_F_HW_CSUM | NETIF_F_RXHASH;
9739     dev->features |= dev->hw_features | NETIF_F_RXCSUM;
9740 }
9741 
9742 static int niu_pci_init_one(struct pci_dev *pdev,
9743                 const struct pci_device_id *ent)
9744 {
9745     union niu_parent_id parent_id;
9746     struct net_device *dev;
9747     struct niu *np;
9748     int err;
9749 
9750     niu_driver_version();
9751 
9752     err = pci_enable_device(pdev);
9753     if (err) {
9754         dev_err(&pdev->dev, "Cannot enable PCI device, aborting\n");
9755         return err;
9756     }
9757 
9758     if (!(pci_resource_flags(pdev, 0) & IORESOURCE_MEM) ||
9759         !(pci_resource_flags(pdev, 2) & IORESOURCE_MEM)) {
9760         dev_err(&pdev->dev, "Cannot find proper PCI device base addresses, aborting\n");
9761         err = -ENODEV;
9762         goto err_out_disable_pdev;
9763     }
9764 
9765     err = pci_request_regions(pdev, DRV_MODULE_NAME);
9766     if (err) {
9767         dev_err(&pdev->dev, "Cannot obtain PCI resources, aborting\n");
9768         goto err_out_disable_pdev;
9769     }
9770 
9771     if (!pci_is_pcie(pdev)) {
9772         dev_err(&pdev->dev, "Cannot find PCI Express capability, aborting\n");
9773         err = -ENODEV;
9774         goto err_out_free_res;
9775     }
9776 
9777     dev = niu_alloc_and_init(&pdev->dev, pdev, NULL,
9778                  &niu_pci_ops, PCI_FUNC(pdev->devfn));
9779     if (!dev) {
9780         err = -ENOMEM;
9781         goto err_out_free_res;
9782     }
9783     np = netdev_priv(dev);
9784 
9785     memset(&parent_id, 0, sizeof(parent_id));
9786     parent_id.pci.domain = pci_domain_nr(pdev->bus);
9787     parent_id.pci.bus = pdev->bus->number;
9788     parent_id.pci.device = PCI_SLOT(pdev->devfn);
9789 
9790     np->parent = niu_get_parent(np, &parent_id,
9791                     PLAT_TYPE_ATLAS);
9792     if (!np->parent) {
9793         err = -ENOMEM;
9794         goto err_out_free_dev;
9795     }
9796 
9797     pcie_capability_clear_and_set_word(pdev, PCI_EXP_DEVCTL,
9798         PCI_EXP_DEVCTL_NOSNOOP_EN,
9799         PCI_EXP_DEVCTL_CERE | PCI_EXP_DEVCTL_NFERE |
9800         PCI_EXP_DEVCTL_FERE | PCI_EXP_DEVCTL_URRE |
9801         PCI_EXP_DEVCTL_RELAX_EN);
9802 
9803     err = dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(44));
9804     if (!err)
9805         dev->features |= NETIF_F_HIGHDMA;
9806     if (err) {
9807         err = dma_set_mask(&pdev->dev, DMA_BIT_MASK(32));
9808         if (err) {
9809             dev_err(&pdev->dev, "No usable DMA configuration, aborting\n");
9810             goto err_out_release_parent;
9811         }
9812     }
9813 
9814     niu_set_basic_features(dev);
9815 
9816     dev->priv_flags |= IFF_UNICAST_FLT;
9817 
9818     np->regs = pci_ioremap_bar(pdev, 0);
9819     if (!np->regs) {
9820         dev_err(&pdev->dev, "Cannot map device registers, aborting\n");
9821         err = -ENOMEM;
9822         goto err_out_release_parent;
9823     }
9824 
9825     pci_set_master(pdev);
9826     pci_save_state(pdev);
9827 
9828     dev->irq = pdev->irq;
9829 
9830     /* MTU range: 68 - 9216 */
9831     dev->min_mtu = ETH_MIN_MTU;
9832     dev->max_mtu = NIU_MAX_MTU;
9833 
9834     niu_assign_netdev_ops(dev);
9835 
9836     err = niu_get_invariants(np);
9837     if (err) {
9838         if (err != -ENODEV)
9839             dev_err(&pdev->dev, "Problem fetching invariants of chip, aborting\n");
9840         goto err_out_iounmap;
9841     }
9842 
9843     err = register_netdev(dev);
9844     if (err) {
9845         dev_err(&pdev->dev, "Cannot register net device, aborting\n");
9846         goto err_out_iounmap;
9847     }
9848 
9849     pci_set_drvdata(pdev, dev);
9850 
9851     niu_device_announce(np);
9852 
9853     return 0;
9854 
9855 err_out_iounmap:
9856     if (np->regs) {
9857         iounmap(np->regs);
9858         np->regs = NULL;
9859     }
9860 
9861 err_out_release_parent:
9862     niu_put_parent(np);
9863 
9864 err_out_free_dev:
9865     free_netdev(dev);
9866 
9867 err_out_free_res:
9868     pci_release_regions(pdev);
9869 
9870 err_out_disable_pdev:
9871     pci_disable_device(pdev);
9872 
9873     return err;
9874 }
9875 
9876 static void niu_pci_remove_one(struct pci_dev *pdev)
9877 {
9878     struct net_device *dev = pci_get_drvdata(pdev);
9879 
9880     if (dev) {
9881         struct niu *np = netdev_priv(dev);
9882 
9883         unregister_netdev(dev);
9884         if (np->regs) {
9885             iounmap(np->regs);
9886             np->regs = NULL;
9887         }
9888 
9889         niu_ldg_free(np);
9890 
9891         niu_put_parent(np);
9892 
9893         free_netdev(dev);
9894         pci_release_regions(pdev);
9895         pci_disable_device(pdev);
9896     }
9897 }
9898 
9899 static int __maybe_unused niu_suspend(struct device *dev_d)
9900 {
9901     struct net_device *dev = dev_get_drvdata(dev_d);
9902     struct niu *np = netdev_priv(dev);
9903     unsigned long flags;
9904 
9905     if (!netif_running(dev))
9906         return 0;
9907 
9908     flush_work(&np->reset_task);
9909     niu_netif_stop(np);
9910 
9911     del_timer_sync(&np->timer);
9912 
9913     spin_lock_irqsave(&np->lock, flags);
9914     niu_enable_interrupts(np, 0);
9915     spin_unlock_irqrestore(&np->lock, flags);
9916 
9917     netif_device_detach(dev);
9918 
9919     spin_lock_irqsave(&np->lock, flags);
9920     niu_stop_hw(np);
9921     spin_unlock_irqrestore(&np->lock, flags);
9922 
9923     return 0;
9924 }
9925 
9926 static int __maybe_unused niu_resume(struct device *dev_d)
9927 {
9928     struct net_device *dev = dev_get_drvdata(dev_d);
9929     struct niu *np = netdev_priv(dev);
9930     unsigned long flags;
9931     int err;
9932 
9933     if (!netif_running(dev))
9934         return 0;
9935 
9936     netif_device_attach(dev);
9937 
9938     spin_lock_irqsave(&np->lock, flags);
9939 
9940     err = niu_init_hw(np);
9941     if (!err) {
9942         np->timer.expires = jiffies + HZ;
9943         add_timer(&np->timer);
9944         niu_netif_start(np);
9945     }
9946 
9947     spin_unlock_irqrestore(&np->lock, flags);
9948 
9949     return err;
9950 }
9951 
9952 static SIMPLE_DEV_PM_OPS(niu_pm_ops, niu_suspend, niu_resume);
9953 
9954 static struct pci_driver niu_pci_driver = {
9955     .name       = DRV_MODULE_NAME,
9956     .id_table   = niu_pci_tbl,
9957     .probe      = niu_pci_init_one,
9958     .remove     = niu_pci_remove_one,
9959     .driver.pm  = &niu_pm_ops,
9960 };
9961 
9962 #ifdef CONFIG_SPARC64
9963 static void *niu_phys_alloc_coherent(struct device *dev, size_t size,
9964                      u64 *dma_addr, gfp_t flag)
9965 {
9966     unsigned long order = get_order(size);
9967     unsigned long page = __get_free_pages(flag, order);
9968 
9969     if (page == 0UL)
9970         return NULL;
9971     memset((char *)page, 0, PAGE_SIZE << order);
9972     *dma_addr = __pa(page);
9973 
9974     return (void *) page;
9975 }
9976 
9977 static void niu_phys_free_coherent(struct device *dev, size_t size,
9978                    void *cpu_addr, u64 handle)
9979 {
9980     unsigned long order = get_order(size);
9981 
9982     free_pages((unsigned long) cpu_addr, order);
9983 }
9984 
9985 static u64 niu_phys_map_page(struct device *dev, struct page *page,
9986                  unsigned long offset, size_t size,
9987                  enum dma_data_direction direction)
9988 {
9989     return page_to_phys(page) + offset;
9990 }
9991 
9992 static void niu_phys_unmap_page(struct device *dev, u64 dma_address,
9993                 size_t size, enum dma_data_direction direction)
9994 {
9995     /* Nothing to do.  */
9996 }
9997 
9998 static u64 niu_phys_map_single(struct device *dev, void *cpu_addr,
9999                    size_t size,
10000                    enum dma_data_direction direction)
10001 {
10002     return __pa(cpu_addr);
10003 }
10004 
10005 static void niu_phys_unmap_single(struct device *dev, u64 dma_address,
10006                   size_t size,
10007                   enum dma_data_direction direction)
10008 {
10009     /* Nothing to do.  */
10010 }
10011 
10012 static const struct niu_ops niu_phys_ops = {
10013     .alloc_coherent = niu_phys_alloc_coherent,
10014     .free_coherent  = niu_phys_free_coherent,
10015     .map_page   = niu_phys_map_page,
10016     .unmap_page = niu_phys_unmap_page,
10017     .map_single = niu_phys_map_single,
10018     .unmap_single   = niu_phys_unmap_single,
10019 };
10020 
10021 static int niu_of_probe(struct platform_device *op)
10022 {
10023     union niu_parent_id parent_id;
10024     struct net_device *dev;
10025     struct niu *np;
10026     const u32 *reg;
10027     int err;
10028 
10029     niu_driver_version();
10030 
10031     reg = of_get_property(op->dev.of_node, "reg", NULL);
10032     if (!reg) {
10033         dev_err(&op->dev, "%pOF: No 'reg' property, aborting\n",
10034             op->dev.of_node);
10035         return -ENODEV;
10036     }
10037 
10038     dev = niu_alloc_and_init(&op->dev, NULL, op,
10039                  &niu_phys_ops, reg[0] & 0x1);
10040     if (!dev) {
10041         err = -ENOMEM;
10042         goto err_out;
10043     }
10044     np = netdev_priv(dev);
10045 
10046     memset(&parent_id, 0, sizeof(parent_id));
10047     parent_id.of = of_get_parent(op->dev.of_node);
10048 
10049     np->parent = niu_get_parent(np, &parent_id,
10050                     PLAT_TYPE_NIU);
10051     if (!np->parent) {
10052         err = -ENOMEM;
10053         goto err_out_free_dev;
10054     }
10055 
10056     niu_set_basic_features(dev);
10057 
10058     np->regs = of_ioremap(&op->resource[1], 0,
10059                   resource_size(&op->resource[1]),
10060                   "niu regs");
10061     if (!np->regs) {
10062         dev_err(&op->dev, "Cannot map device registers, aborting\n");
10063         err = -ENOMEM;
10064         goto err_out_release_parent;
10065     }
10066 
10067     np->vir_regs_1 = of_ioremap(&op->resource[2], 0,
10068                     resource_size(&op->resource[2]),
10069                     "niu vregs-1");
10070     if (!np->vir_regs_1) {
10071         dev_err(&op->dev, "Cannot map device vir registers 1, aborting\n");
10072         err = -ENOMEM;
10073         goto err_out_iounmap;
10074     }
10075 
10076     np->vir_regs_2 = of_ioremap(&op->resource[3], 0,
10077                     resource_size(&op->resource[3]),
10078                     "niu vregs-2");
10079     if (!np->vir_regs_2) {
10080         dev_err(&op->dev, "Cannot map device vir registers 2, aborting\n");
10081         err = -ENOMEM;
10082         goto err_out_iounmap;
10083     }
10084 
10085     niu_assign_netdev_ops(dev);
10086 
10087     err = niu_get_invariants(np);
10088     if (err) {
10089         if (err != -ENODEV)
10090             dev_err(&op->dev, "Problem fetching invariants of chip, aborting\n");
10091         goto err_out_iounmap;
10092     }
10093 
10094     err = register_netdev(dev);
10095     if (err) {
10096         dev_err(&op->dev, "Cannot register net device, aborting\n");
10097         goto err_out_iounmap;
10098     }
10099 
10100     platform_set_drvdata(op, dev);
10101 
10102     niu_device_announce(np);
10103 
10104     return 0;
10105 
10106 err_out_iounmap:
10107     if (np->vir_regs_1) {
10108         of_iounmap(&op->resource[2], np->vir_regs_1,
10109                resource_size(&op->resource[2]));
10110         np->vir_regs_1 = NULL;
10111     }
10112 
10113     if (np->vir_regs_2) {
10114         of_iounmap(&op->resource[3], np->vir_regs_2,
10115                resource_size(&op->resource[3]));
10116         np->vir_regs_2 = NULL;
10117     }
10118 
10119     if (np->regs) {
10120         of_iounmap(&op->resource[1], np->regs,
10121                resource_size(&op->resource[1]));
10122         np->regs = NULL;
10123     }
10124 
10125 err_out_release_parent:
10126     niu_put_parent(np);
10127 
10128 err_out_free_dev:
10129     free_netdev(dev);
10130 
10131 err_out:
10132     return err;
10133 }
10134 
10135 static int niu_of_remove(struct platform_device *op)
10136 {
10137     struct net_device *dev = platform_get_drvdata(op);
10138 
10139     if (dev) {
10140         struct niu *np = netdev_priv(dev);
10141 
10142         unregister_netdev(dev);
10143 
10144         if (np->vir_regs_1) {
10145             of_iounmap(&op->resource[2], np->vir_regs_1,
10146                    resource_size(&op->resource[2]));
10147             np->vir_regs_1 = NULL;
10148         }
10149 
10150         if (np->vir_regs_2) {
10151             of_iounmap(&op->resource[3], np->vir_regs_2,
10152                    resource_size(&op->resource[3]));
10153             np->vir_regs_2 = NULL;
10154         }
10155 
10156         if (np->regs) {
10157             of_iounmap(&op->resource[1], np->regs,
10158                    resource_size(&op->resource[1]));
10159             np->regs = NULL;
10160         }
10161 
10162         niu_ldg_free(np);
10163 
10164         niu_put_parent(np);
10165 
10166         free_netdev(dev);
10167     }
10168     return 0;
10169 }
10170 
10171 static const struct of_device_id niu_match[] = {
10172     {
10173         .name = "network",
10174         .compatible = "SUNW,niusl",
10175     },
10176     {},
10177 };
10178 MODULE_DEVICE_TABLE(of, niu_match);
10179 
10180 static struct platform_driver niu_of_driver = {
10181     .driver = {
10182         .name = "niu",
10183         .of_match_table = niu_match,
10184     },
10185     .probe      = niu_of_probe,
10186     .remove     = niu_of_remove,
10187 };
10188 
10189 #endif /* CONFIG_SPARC64 */
10190 
10191 static int __init niu_init(void)
10192 {
10193     int err = 0;
10194 
10195     BUILD_BUG_ON(PAGE_SIZE < 4 * 1024);
10196 
10197     BUILD_BUG_ON(offsetof(struct page, mapping) !=
10198              offsetof(union niu_page, next));
10199 
10200     niu_debug = netif_msg_init(debug, NIU_MSG_DEFAULT);
10201 
10202 #ifdef CONFIG_SPARC64
10203     err = platform_driver_register(&niu_of_driver);
10204 #endif
10205 
10206     if (!err) {
10207         err = pci_register_driver(&niu_pci_driver);
10208 #ifdef CONFIG_SPARC64
10209         if (err)
10210             platform_driver_unregister(&niu_of_driver);
10211 #endif
10212     }
10213 
10214     return err;
10215 }
10216 
10217 static void __exit niu_exit(void)
10218 {
10219     pci_unregister_driver(&niu_pci_driver);
10220 #ifdef CONFIG_SPARC64
10221     platform_driver_unregister(&niu_of_driver);
10222 #endif
10223 }
10224 
10225 module_init(niu_init);
10226 module_exit(niu_exit);
[ Source navigation ] [ Diff markup ] [ Identifier search ] [ general search ]

This page was automatically generated by the 2.1.0 LXR engine.

The LXR team
Valid CSS 2.1! Valid HTML 4.01!