hisilicon/hns/hns_enet.c

0001 // SPDX-License-Identifier: GPL-2.0-or-later
0002 /*
0003  * Copyright (c) 2014-2015 Hisilicon Limited.
0004  */
0005
0006 #include <linux/clk.h>
0007 #include <linux/cpumask.h>
0008 #include <linux/etherdevice.h>
0009 #include <linux/if_vlan.h>
0010 #include <linux/interrupt.h>
0011 #include <linux/io.h>
0012 #include <linux/ip.h>
0013 #include <linux/ipv6.h>
0014 #include <linux/irq.h>
0015 #include <linux/module.h>
0016 #include <linux/phy.h>
0017 #include <linux/platform_device.h>
0018 #include <linux/skbuff.h>
0019
0020 #include "hnae.h"
0021 #include "hns_enet.h"
0022 #include "hns_dsaf_mac.h"
0023
0024 #define NIC_MAX_Q_PER_VF 16
0025 #define HNS_NIC_TX_TIMEOUT (5 * HZ)
0026
0027 #define SERVICE_TIMER_HZ (1 * HZ)
0028
0029 #define RCB_IRQ_NOT_INITED 0
0030 #define RCB_IRQ_INITED 1
0031 #define HNS_BUFFER_SIZE_2048 2048
0032
0033 #define BD_MAX_SEND_SIZE 8191
0034
0035 static void fill_v2_desc_hw(struct hnae_ring *ring, void *priv, int size,
0036                 int send_sz, dma_addr_t dma, int frag_end,
0037                 int buf_num, enum hns_desc_type type, int mtu)
0038 {
0039     struct hnae_desc *desc = &ring->desc[ring->next_to_use];
0040     struct hnae_desc_cb *desc_cb = &ring->desc_cb[ring->next_to_use];
0041     struct iphdr *iphdr;
0042     struct ipv6hdr *ipv6hdr;
0043     struct sk_buff *skb;
0044     __be16 protocol;
0045     u8 bn_pid = 0;
0046     u8 rrcfv = 0;
0047     u8 ip_offset = 0;
0048     u8 tvsvsn = 0;
0049     u16 mss = 0;
0050     u8 l4_len = 0;
0051     u16 paylen = 0;
0052
0053     desc_cb->priv = priv;
0054     desc_cb->length = size;
0055     desc_cb->dma = dma;
0056     desc_cb->type = type;
0057
0058     desc->addr = cpu_to_le64(dma);
0059     desc->tx.send_size = cpu_to_le16((u16)send_sz);
0060
0061     /* config bd buffer end */
0062     hnae_set_bit(rrcfv, HNSV2_TXD_VLD_B, 1);
0063     hnae_set_field(bn_pid, HNSV2_TXD_BUFNUM_M, 0, buf_num - 1);
0064
0065     /* fill port_id in the tx bd for sending management pkts */
0066     hnae_set_field(bn_pid, HNSV2_TXD_PORTID_M,
0067                HNSV2_TXD_PORTID_S, ring->q->handle->dport_id);
0068
0069     if (type == DESC_TYPE_SKB) {
0070         skb = (struct sk_buff *)priv;
0071
0072         if (skb->ip_summed == CHECKSUM_PARTIAL) {
0073             skb_reset_mac_len(skb);
0074             protocol = skb->protocol;
0075             ip_offset = ETH_HLEN;
0076
0077             if (protocol == htons(ETH_P_8021Q)) {
0078                 ip_offset += VLAN_HLEN;
0079                 protocol = vlan_get_protocol(skb);
0080                 skb->protocol = protocol;
0081             }
0082
0083             if (skb->protocol == htons(ETH_P_IP)) {
0084                 iphdr = ip_hdr(skb);
0085                 hnae_set_bit(rrcfv, HNSV2_TXD_L3CS_B, 1);
0086                 hnae_set_bit(rrcfv, HNSV2_TXD_L4CS_B, 1);
0087
0088                 /* check for tcp/udp header */
0089                 if (iphdr->protocol == IPPROTO_TCP &&
0090                     skb_is_gso(skb)) {
0091                     hnae_set_bit(tvsvsn,
0092                              HNSV2_TXD_TSE_B, 1);
0093                     l4_len = tcp_hdrlen(skb);
0094                     mss = skb_shinfo(skb)->gso_size;
0095                     paylen = skb->len - skb_tcp_all_headers(skb);
0096                 }
0097             } else if (skb->protocol == htons(ETH_P_IPV6)) {
0098                 hnae_set_bit(tvsvsn, HNSV2_TXD_IPV6_B, 1);
0099                 ipv6hdr = ipv6_hdr(skb);
0100                 hnae_set_bit(rrcfv, HNSV2_TXD_L4CS_B, 1);
0101
0102                 /* check for tcp/udp header */
0103                 if (ipv6hdr->nexthdr == IPPROTO_TCP &&
0104                     skb_is_gso(skb) && skb_is_gso_v6(skb)) {
0105                     hnae_set_bit(tvsvsn,
0106                              HNSV2_TXD_TSE_B, 1);
0107                     l4_len = tcp_hdrlen(skb);
0108                     mss = skb_shinfo(skb)->gso_size;
0109                     paylen = skb->len - skb_tcp_all_headers(skb);
0110                 }
0111             }
0112             desc->tx.ip_offset = ip_offset;
0113             desc->tx.tse_vlan_snap_v6_sctp_nth = tvsvsn;
0114             desc->tx.mss = cpu_to_le16(mss);
0115             desc->tx.l4_len = l4_len;
0116             desc->tx.paylen = cpu_to_le16(paylen);
0117         }
0118     }
0119
0120     hnae_set_bit(rrcfv, HNSV2_TXD_FE_B, frag_end);
0121
0122     desc->tx.bn_pid = bn_pid;
0123     desc->tx.ra_ri_cs_fe_vld = rrcfv;
0124
0125     ring_ptr_move_fw(ring, next_to_use);
0126 }
0127
0128 static void fill_v2_desc(struct hnae_ring *ring, void *priv,
0129              int size, dma_addr_t dma, int frag_end,
0130              int buf_num, enum hns_desc_type type, int mtu)
0131 {
0132     fill_v2_desc_hw(ring, priv, size, size, dma, frag_end,
0133             buf_num, type, mtu);
0134 }
0135
0136 static const struct acpi_device_id hns_enet_acpi_match[] = {
0137     { "HISI00C1", 0 },
0138     { "HISI00C2", 0 },
0139     { },
0140 };
0141 MODULE_DEVICE_TABLE(acpi, hns_enet_acpi_match);
0142
0143 static void fill_desc(struct hnae_ring *ring, void *priv,
0144               int size, dma_addr_t dma, int frag_end,
0145               int buf_num, enum hns_desc_type type, int mtu)
0146 {
0147     struct hnae_desc *desc = &ring->desc[ring->next_to_use];
0148     struct hnae_desc_cb *desc_cb = &ring->desc_cb[ring->next_to_use];
0149     struct sk_buff *skb;
0150     __be16 protocol;
0151     u32 ip_offset;
0152     u32 asid_bufnum_pid = 0;
0153     u32 flag_ipoffset = 0;
0154
0155     desc_cb->priv = priv;
0156     desc_cb->length = size;
0157     desc_cb->dma = dma;
0158     desc_cb->type = type;
0159
0160     desc->addr = cpu_to_le64(dma);
0161     desc->tx.send_size = cpu_to_le16((u16)size);
0162
0163     /*config bd buffer end */
0164     flag_ipoffset |= 1 << HNS_TXD_VLD_B;
0165
0166     asid_bufnum_pid |= buf_num << HNS_TXD_BUFNUM_S;
0167
0168     if (type == DESC_TYPE_SKB) {
0169         skb = (struct sk_buff *)priv;
0170
0171         if (skb->ip_summed == CHECKSUM_PARTIAL) {
0172             protocol = skb->protocol;
0173             ip_offset = ETH_HLEN;
0174
0175             /*if it is a SW VLAN check the next protocol*/
0176             if (protocol == htons(ETH_P_8021Q)) {
0177                 ip_offset += VLAN_HLEN;
0178                 protocol = vlan_get_protocol(skb);
0179                 skb->protocol = protocol;
0180             }
0181
0182             if (skb->protocol == htons(ETH_P_IP)) {
0183                 flag_ipoffset |= 1 << HNS_TXD_L3CS_B;
0184                 /* check for tcp/udp header */
0185                 flag_ipoffset |= 1 << HNS_TXD_L4CS_B;
0186
0187             } else if (skb->protocol == htons(ETH_P_IPV6)) {
0188                 /* ipv6 has not l3 cs, check for L4 header */
0189                 flag_ipoffset |= 1 << HNS_TXD_L4CS_B;
0190             }
0191
0192             flag_ipoffset |= ip_offset << HNS_TXD_IPOFFSET_S;
0193         }
0194     }
0195
0196     flag_ipoffset |= frag_end << HNS_TXD_FE_B;
0197
0198     desc->tx.asid_bufnum_pid = cpu_to_le16(asid_bufnum_pid);
0199     desc->tx.flag_ipoffset = cpu_to_le32(flag_ipoffset);
0200
0201     ring_ptr_move_fw(ring, next_to_use);
0202 }
0203
0204 static void unfill_desc(struct hnae_ring *ring)
0205 {
0206     ring_ptr_move_bw(ring, next_to_use);
0207 }
0208
0209 static int hns_nic_maybe_stop_tx(
0210     struct sk_buff **out_skb, int *bnum, struct hnae_ring *ring)
0211 {
0212     struct sk_buff *skb = *out_skb;
0213     struct sk_buff *new_skb = NULL;
0214     int buf_num;
0215
0216     /* no. of segments (plus a header) */
0217     buf_num = skb_shinfo(skb)->nr_frags + 1;
0218
0219     if (unlikely(buf_num > ring->max_desc_num_per_pkt)) {
0220         if (ring_space(ring) < 1)
0221             return -EBUSY;
0222
0223         new_skb = skb_copy(skb, GFP_ATOMIC);
0224         if (!new_skb)
0225             return -ENOMEM;
0226
0227         dev_kfree_skb_any(skb);
0228         *out_skb = new_skb;
0229         buf_num = 1;
0230     } else if (buf_num > ring_space(ring)) {
0231         return -EBUSY;
0232     }
0233
0234     *bnum = buf_num;
0235     return 0;
0236 }
0237
0238 static int hns_nic_maybe_stop_tso(
0239     struct sk_buff **out_skb, int *bnum, struct hnae_ring *ring)
0240 {
0241     int i;
0242     int size;
0243     int buf_num;
0244     int frag_num;
0245     struct sk_buff *skb = *out_skb;
0246     struct sk_buff *new_skb = NULL;
0247     skb_frag_t *frag;
0248
0249     size = skb_headlen(skb);
0250     buf_num = (size + BD_MAX_SEND_SIZE - 1) / BD_MAX_SEND_SIZE;
0251
0252     frag_num = skb_shinfo(skb)->nr_frags;
0253     for (i = 0; i < frag_num; i++) {
0254         frag = &skb_shinfo(skb)->frags[i];
0255         size = skb_frag_size(frag);
0256         buf_num += (size + BD_MAX_SEND_SIZE - 1) / BD_MAX_SEND_SIZE;
0257     }
0258
0259     if (unlikely(buf_num > ring->max_desc_num_per_pkt)) {
0260         buf_num = (skb->len + BD_MAX_SEND_SIZE - 1) / BD_MAX_SEND_SIZE;
0261         if (ring_space(ring) < buf_num)
0262             return -EBUSY;
0263         /* manual split the send packet */
0264         new_skb = skb_copy(skb, GFP_ATOMIC);
0265         if (!new_skb)
0266             return -ENOMEM;
0267         dev_kfree_skb_any(skb);
0268         *out_skb = new_skb;
0269
0270     } else if (ring_space(ring) < buf_num) {
0271         return -EBUSY;
0272     }
0273
0274     *bnum = buf_num;
0275     return 0;
0276 }
0277
0278 static void fill_tso_desc(struct hnae_ring *ring, void *priv,
0279               int size, dma_addr_t dma, int frag_end,
0280               int buf_num, enum hns_desc_type type, int mtu)
0281 {
0282     int frag_buf_num;
0283     int sizeoflast;
0284     int k;
0285
0286     frag_buf_num = (size + BD_MAX_SEND_SIZE - 1) / BD_MAX_SEND_SIZE;
0287     sizeoflast = size % BD_MAX_SEND_SIZE;
0288     sizeoflast = sizeoflast ? sizeoflast : BD_MAX_SEND_SIZE;
0289
0290     /* when the frag size is bigger than hardware, split this frag */
0291     for (k = 0; k < frag_buf_num; k++)
0292         fill_v2_desc_hw(ring, priv, k == 0 ? size : 0,
0293                 (k == frag_buf_num - 1) ?
0294                     sizeoflast : BD_MAX_SEND_SIZE,
0295                 dma + BD_MAX_SEND_SIZE * k,
0296                 frag_end && (k == frag_buf_num - 1) ? 1 : 0,
0297                 buf_num,
0298                 (type == DESC_TYPE_SKB && !k) ?
0299                     DESC_TYPE_SKB : DESC_TYPE_PAGE,
0300                 mtu);
0301 }
0302
0303 netdev_tx_t hns_nic_net_xmit_hw(struct net_device *ndev,
0304                 struct sk_buff *skb,
0305                 struct hns_nic_ring_data *ring_data)
0306 {
0307     struct hns_nic_priv *priv = netdev_priv(ndev);
0308     struct hnae_ring *ring = ring_data->ring;
0309     struct device *dev = ring_to_dev(ring);
0310     struct netdev_queue *dev_queue;
0311     skb_frag_t *frag;
0312     int buf_num;
0313     int seg_num;
0314     dma_addr_t dma;
0315     int size, next_to_use;
0316     int i;
0317
0318     switch (priv->ops.maybe_stop_tx(&skb, &buf_num, ring)) {
0319     case -EBUSY:
0320         ring->stats.tx_busy++;
0321         goto out_net_tx_busy;
0322     case -ENOMEM:
0323         ring->stats.sw_err_cnt++;
0324         netdev_err(ndev, "no memory to xmit!\n");
0325         goto out_err_tx_ok;
0326     default:
0327         break;
0328     }
0329
0330     /* no. of segments (plus a header) */
0331     seg_num = skb_shinfo(skb)->nr_frags + 1;
0332     next_to_use = ring->next_to_use;
0333
0334     /* fill the first part */
0335     size = skb_headlen(skb);
0336     dma = dma_map_single(dev, skb->data, size, DMA_TO_DEVICE);
0337     if (dma_mapping_error(dev, dma)) {
0338         netdev_err(ndev, "TX head DMA map failed\n");
0339         ring->stats.sw_err_cnt++;
0340         goto out_err_tx_ok;
0341     }
0342     priv->ops.fill_desc(ring, skb, size, dma, seg_num == 1 ? 1 : 0,
0343                 buf_num, DESC_TYPE_SKB, ndev->mtu);
0344
0345     /* fill the fragments */
0346     for (i = 1; i < seg_num; i++) {
0347         frag = &skb_shinfo(skb)->frags[i - 1];
0348         size = skb_frag_size(frag);
0349         dma = skb_frag_dma_map(dev, frag, 0, size, DMA_TO_DEVICE);
0350         if (dma_mapping_error(dev, dma)) {
0351             netdev_err(ndev, "TX frag(%d) DMA map failed\n", i);
0352             ring->stats.sw_err_cnt++;
0353             goto out_map_frag_fail;
0354         }
0355         priv->ops.fill_desc(ring, skb_frag_page(frag), size, dma,
0356                     seg_num - 1 == i ? 1 : 0, buf_num,
0357                     DESC_TYPE_PAGE, ndev->mtu);
0358     }
0359
0360     /*complete translate all packets*/
0361     dev_queue = netdev_get_tx_queue(ndev, skb->queue_mapping);
0362     netdev_tx_sent_queue(dev_queue, skb->len);
0363
0364     netif_trans_update(ndev);
0365     ndev->stats.tx_bytes += skb->len;
0366     ndev->stats.tx_packets++;
0367
0368     wmb(); /* commit all data before submit */
0369     assert(skb->queue_mapping < priv->ae_handle->q_num);
0370     hnae_queue_xmit(priv->ae_handle->qs[skb->queue_mapping], buf_num);
0371
0372     return NETDEV_TX_OK;
0373
0374 out_map_frag_fail:
0375
0376     while (ring->next_to_use != next_to_use) {
0377         unfill_desc(ring);
0378         if (ring->next_to_use != next_to_use)
0379             dma_unmap_page(dev,
0380                        ring->desc_cb[ring->next_to_use].dma,
0381                        ring->desc_cb[ring->next_to_use].length,
0382                        DMA_TO_DEVICE);
0383         else
0384             dma_unmap_single(dev,
0385                      ring->desc_cb[next_to_use].dma,
0386                      ring->desc_cb[next_to_use].length,
0387                      DMA_TO_DEVICE);
0388     }
0389
0390 out_err_tx_ok:
0391
0392     dev_kfree_skb_any(skb);
0393     return NETDEV_TX_OK;
0394
0395 out_net_tx_busy:
0396
0397     netif_stop_subqueue(ndev, skb->queue_mapping);
0398
0399     /* Herbert's original patch had:
0400      *  smp_mb__after_netif_stop_queue();
0401      * but since that doesn't exist yet, just open code it.
0402      */
0403     smp_mb();
0404     return NETDEV_TX_BUSY;
0405 }
0406
0407 static void hns_nic_reuse_page(struct sk_buff *skb, int i,
0408                    struct hnae_ring *ring, int pull_len,
0409                    struct hnae_desc_cb *desc_cb)
0410 {
0411     struct hnae_desc *desc;
0412     u32 truesize;
0413     int size;
0414     int last_offset;
0415     bool twobufs;
0416
0417     twobufs = ((PAGE_SIZE < 8192) &&
0418         hnae_buf_size(ring) == HNS_BUFFER_SIZE_2048);
0419
0420     desc = &ring->desc[ring->next_to_clean];
0421     size = le16_to_cpu(desc->rx.size);
0422
0423     if (twobufs) {
0424         truesize = hnae_buf_size(ring);
0425     } else {
0426         truesize = ALIGN(size, L1_CACHE_BYTES);
0427         last_offset = hnae_page_size(ring) - hnae_buf_size(ring);
0428     }
0429
0430     skb_add_rx_frag(skb, i, desc_cb->priv, desc_cb->page_offset + pull_len,
0431             size - pull_len, truesize);
0432
0433      /* avoid re-using remote pages,flag default unreuse */
0434     if (unlikely(page_to_nid(desc_cb->priv) != numa_node_id()))
0435         return;
0436
0437     if (twobufs) {
0438         /* if we are only owner of page we can reuse it */
0439         if (likely(page_count(desc_cb->priv) == 1)) {
0440             /* flip page offset to other buffer */
0441             desc_cb->page_offset ^= truesize;
0442
0443             desc_cb->reuse_flag = 1;
0444             /* bump ref count on page before it is given*/
0445             get_page(desc_cb->priv);
0446         }
0447         return;
0448     }
0449
0450     /* move offset up to the next cache line */
0451     desc_cb->page_offset += truesize;
0452
0453     if (desc_cb->page_offset <= last_offset) {
0454         desc_cb->reuse_flag = 1;
0455         /* bump ref count on page before it is given*/
0456         get_page(desc_cb->priv);
0457     }
0458 }
0459
0460 static void get_v2rx_desc_bnum(u32 bnum_flag, int *out_bnum)
0461 {
0462     *out_bnum = hnae_get_field(bnum_flag,
0463                    HNS_RXD_BUFNUM_M, HNS_RXD_BUFNUM_S) + 1;
0464 }
0465
0466 static void get_rx_desc_bnum(u32 bnum_flag, int *out_bnum)
0467 {
0468     *out_bnum = hnae_get_field(bnum_flag,
0469                    HNS_RXD_BUFNUM_M, HNS_RXD_BUFNUM_S);
0470 }
0471
0472 static void hns_nic_rx_checksum(struct hns_nic_ring_data *ring_data,
0473                 struct sk_buff *skb, u32 flag)
0474 {
0475     struct net_device *netdev = ring_data->napi.dev;
0476     u32 l3id;
0477     u32 l4id;
0478
0479     /* check if RX checksum offload is enabled */
0480     if (unlikely(!(netdev->features & NETIF_F_RXCSUM)))
0481         return;
0482
0483     /* In hardware, we only support checksum for the following protocols:
0484      * 1) IPv4,
0485      * 2) TCP(over IPv4 or IPv6),
0486      * 3) UDP(over IPv4 or IPv6),
0487      * 4) SCTP(over IPv4 or IPv6)
0488      * but we support many L3(IPv4, IPv6, MPLS, PPPoE etc) and L4(TCP,
0489      * UDP, GRE, SCTP, IGMP, ICMP etc.) protocols.
0490      *
0491      * Hardware limitation:
0492      * Our present hardware RX Descriptor lacks L3/L4 checksum "Status &
0493      * Error" bit (which usually can be used to indicate whether checksum
0494      * was calculated by the hardware and if there was any error encountered
0495      * during checksum calculation).
0496      *
0497      * Software workaround:
0498      * We do get info within the RX descriptor about the kind of L3/L4
0499      * protocol coming in the packet and the error status. These errors
0500      * might not just be checksum errors but could be related to version,
0501      * length of IPv4, UDP, TCP etc.
0502      * Because there is no-way of knowing if it is a L3/L4 error due to bad
0503      * checksum or any other L3/L4 error, we will not (cannot) convey
0504      * checksum status for such cases to upper stack and will not maintain
0505      * the RX L3/L4 checksum counters as well.
0506      */
0507
0508     l3id = hnae_get_field(flag, HNS_RXD_L3ID_M, HNS_RXD_L3ID_S);
0509     l4id = hnae_get_field(flag, HNS_RXD_L4ID_M, HNS_RXD_L4ID_S);
0510
0511     /*  check L3 protocol for which checksum is supported */
0512     if ((l3id != HNS_RX_FLAG_L3ID_IPV4) && (l3id != HNS_RX_FLAG_L3ID_IPV6))
0513         return;
0514
0515     /* check for any(not just checksum)flagged L3 protocol errors */
0516     if (unlikely(hnae_get_bit(flag, HNS_RXD_L3E_B)))
0517         return;
0518
0519     /* we do not support checksum of fragmented packets */
0520     if (unlikely(hnae_get_bit(flag, HNS_RXD_FRAG_B)))
0521         return;
0522
0523     /*  check L4 protocol for which checksum is supported */
0524     if ((l4id != HNS_RX_FLAG_L4ID_TCP) &&
0525         (l4id != HNS_RX_FLAG_L4ID_UDP) &&
0526         (l4id != HNS_RX_FLAG_L4ID_SCTP))
0527         return;
0528
0529     /* check for any(not just checksum)flagged L4 protocol errors */
0530     if (unlikely(hnae_get_bit(flag, HNS_RXD_L4E_B)))
0531         return;
0532
0533     /* now, this has to be a packet with valid RX checksum */
0534     skb->ip_summed = CHECKSUM_UNNECESSARY;
0535 }
0536
0537 static int hns_nic_poll_rx_skb(struct hns_nic_ring_data *ring_data,
0538                    struct sk_buff **out_skb, int *out_bnum)
0539 {
0540     struct hnae_ring *ring = ring_data->ring;
0541     struct net_device *ndev = ring_data->napi.dev;
0542     struct hns_nic_priv *priv = netdev_priv(ndev);
0543     struct sk_buff *skb;
0544     struct hnae_desc *desc;
0545     struct hnae_desc_cb *desc_cb;
0546     unsigned char *va;
0547     int bnum, length, i;
0548     int pull_len;
0549     u32 bnum_flag;
0550
0551     desc = &ring->desc[ring->next_to_clean];
0552     desc_cb = &ring->desc_cb[ring->next_to_clean];
0553
0554     prefetch(desc);
0555
0556     va = (unsigned char *)desc_cb->buf + desc_cb->page_offset;
0557
0558     /* prefetch first cache line of first page */
0559     net_prefetch(va);
0560
0561     skb = *out_skb = napi_alloc_skb(&ring_data->napi,
0562                     HNS_RX_HEAD_SIZE);
0563     if (unlikely(!skb)) {
0564         ring->stats.sw_err_cnt++;
0565         return -ENOMEM;
0566     }
0567
0568     prefetchw(skb->data);
0569     length = le16_to_cpu(desc->rx.pkt_len);
0570     bnum_flag = le32_to_cpu(desc->rx.ipoff_bnum_pid_flag);
0571     priv->ops.get_rxd_bnum(bnum_flag, &bnum);
0572     *out_bnum = bnum;
0573
0574     if (length <= HNS_RX_HEAD_SIZE) {
0575         memcpy(__skb_put(skb, length), va, ALIGN(length, sizeof(long)));
0576
0577         /* we can reuse buffer as-is, just make sure it is local */
0578         if (likely(page_to_nid(desc_cb->priv) == numa_node_id()))
0579             desc_cb->reuse_flag = 1;
0580         else /* this page cannot be reused so discard it */
0581             put_page(desc_cb->priv);
0582
0583         ring_ptr_move_fw(ring, next_to_clean);
0584
0585         if (unlikely(bnum != 1)) { /* check err*/
0586             *out_bnum = 1;
0587             goto out_bnum_err;
0588         }
0589     } else {
0590         ring->stats.seg_pkt_cnt++;
0591
0592         pull_len = eth_get_headlen(ndev, va, HNS_RX_HEAD_SIZE);
0593         memcpy(__skb_put(skb, pull_len), va,
0594                ALIGN(pull_len, sizeof(long)));
0595
0596         hns_nic_reuse_page(skb, 0, ring, pull_len, desc_cb);
0597         ring_ptr_move_fw(ring, next_to_clean);
0598
0599         if (unlikely(bnum >= (int)MAX_SKB_FRAGS)) { /* check err*/
0600             *out_bnum = 1;
0601             goto out_bnum_err;
0602         }
0603         for (i = 1; i < bnum; i++) {
0604             desc = &ring->desc[ring->next_to_clean];
0605             desc_cb = &ring->desc_cb[ring->next_to_clean];
0606
0607             hns_nic_reuse_page(skb, i, ring, 0, desc_cb);
0608             ring_ptr_move_fw(ring, next_to_clean);
0609         }
0610     }
0611
0612     /* check except process, free skb and jump the desc */
0613     if (unlikely((!bnum) || (bnum > ring->max_desc_num_per_pkt))) {
0614 out_bnum_err:
0615         *out_bnum = *out_bnum ? *out_bnum : 1; /* ntc moved,cannot 0*/
0616         netdev_err(ndev, "invalid bnum(%d,%d,%d,%d),%016llx,%016llx\n",
0617                bnum, ring->max_desc_num_per_pkt,
0618                length, (int)MAX_SKB_FRAGS,
0619                ((u64 *)desc)[0], ((u64 *)desc)[1]);
0620         ring->stats.err_bd_num++;
0621         dev_kfree_skb_any(skb);
0622         return -EDOM;
0623     }
0624
0625     bnum_flag = le32_to_cpu(desc->rx.ipoff_bnum_pid_flag);
0626
0627     if (unlikely(!hnae_get_bit(bnum_flag, HNS_RXD_VLD_B))) {
0628         netdev_err(ndev, "no valid bd,%016llx,%016llx\n",
0629                ((u64 *)desc)[0], ((u64 *)desc)[1]);
0630         ring->stats.non_vld_descs++;
0631         dev_kfree_skb_any(skb);
0632         return -EINVAL;
0633     }
0634
0635     if (unlikely((!desc->rx.pkt_len) ||
0636              hnae_get_bit(bnum_flag, HNS_RXD_DROP_B))) {
0637         ring->stats.err_pkt_len++;
0638         dev_kfree_skb_any(skb);
0639         return -EFAULT;
0640     }
0641
0642     if (unlikely(hnae_get_bit(bnum_flag, HNS_RXD_L2E_B))) {
0643         ring->stats.l2_err++;
0644         dev_kfree_skb_any(skb);
0645         return -EFAULT;
0646     }
0647
0648     ring->stats.rx_pkts++;
0649     ring->stats.rx_bytes += skb->len;
0650
0651     /* indicate to upper stack if our hardware has already calculated
0652      * the RX checksum
0653      */
0654     hns_nic_rx_checksum(ring_data, skb, bnum_flag);
0655
0656     return 0;
0657 }
0658
0659 static void
0660 hns_nic_alloc_rx_buffers(struct hns_nic_ring_data *ring_data, int cleand_count)
0661 {
0662     int i, ret;
0663     struct hnae_desc_cb res_cbs;
0664     struct hnae_desc_cb *desc_cb;
0665     struct hnae_ring *ring = ring_data->ring;
0666     struct net_device *ndev = ring_data->napi.dev;
0667
0668     for (i = 0; i < cleand_count; i++) {
0669         desc_cb = &ring->desc_cb[ring->next_to_use];
0670         if (desc_cb->reuse_flag) {
0671             ring->stats.reuse_pg_cnt++;
0672             hnae_reuse_buffer(ring, ring->next_to_use);
0673         } else {
0674             ret = hnae_reserve_buffer_map(ring, &res_cbs);
0675             if (ret) {
0676                 ring->stats.sw_err_cnt++;
0677                 netdev_err(ndev, "hnae reserve buffer map failed.\n");
0678                 break;
0679             }
0680             hnae_replace_buffer(ring, ring->next_to_use, &res_cbs);
0681         }
0682
0683         ring_ptr_move_fw(ring, next_to_use);
0684     }
0685
0686     wmb(); /* make all data has been write before submit */
0687     writel_relaxed(i, ring->io_base + RCB_REG_HEAD);
0688 }
0689
0690 /* return error number for error or number of desc left to take
0691  */
0692 static void hns_nic_rx_up_pro(struct hns_nic_ring_data *ring_data,
0693                   struct sk_buff *skb)
0694 {
0695     struct net_device *ndev = ring_data->napi.dev;
0696
0697     skb->protocol = eth_type_trans(skb, ndev);
0698     napi_gro_receive(&ring_data->napi, skb);
0699 }
0700
0701 static int hns_desc_unused(struct hnae_ring *ring)
0702 {
0703     int ntc = ring->next_to_clean;
0704     int ntu = ring->next_to_use;
0705
0706     return ((ntc >= ntu) ? 0 : ring->desc_num) + ntc - ntu;
0707 }
0708
0709 #define HNS_LOWEST_LATENCY_RATE     27  /* 27 MB/s */
0710 #define HNS_LOW_LATENCY_RATE            80  /* 80 MB/s */
0711
0712 #define HNS_COAL_BDNUM          3
0713
0714 static u32 hns_coal_rx_bdnum(struct hnae_ring *ring)
0715 {
0716     bool coal_enable = ring->q->handle->coal_adapt_en;
0717
0718     if (coal_enable &&
0719         ring->coal_last_rx_bytes > HNS_LOWEST_LATENCY_RATE)
0720         return HNS_COAL_BDNUM;
0721     else
0722         return 0;
0723 }
0724
0725 static void hns_update_rx_rate(struct hnae_ring *ring)
0726 {
0727     bool coal_enable = ring->q->handle->coal_adapt_en;
0728     u32 time_passed_ms;
0729     u64 total_bytes;
0730
0731     if (!coal_enable ||
0732         time_before(jiffies, ring->coal_last_jiffies + (HZ >> 4)))
0733         return;
0734
0735     /* ring->stats.rx_bytes overflowed */
0736     if (ring->coal_last_rx_bytes > ring->stats.rx_bytes) {
0737         ring->coal_last_rx_bytes = ring->stats.rx_bytes;
0738         ring->coal_last_jiffies = jiffies;
0739         return;
0740     }
0741
0742     total_bytes = ring->stats.rx_bytes - ring->coal_last_rx_bytes;
0743     time_passed_ms = jiffies_to_msecs(jiffies - ring->coal_last_jiffies);
0744     do_div(total_bytes, time_passed_ms);
0745     ring->coal_rx_rate = total_bytes >> 10;
0746
0747     ring->coal_last_rx_bytes = ring->stats.rx_bytes;
0748     ring->coal_last_jiffies = jiffies;
0749 }
0750
0751 /**
0752  * smooth_alg - smoothing algrithm for adjusting coalesce parameter
0753  * @new_param: new value
0754  * @old_param: old value
0755  **/
0756 static u32 smooth_alg(u32 new_param, u32 old_param)
0757 {
0758     u32 gap = (new_param > old_param) ? new_param - old_param
0759                       : old_param - new_param;
0760
0761     if (gap > 8)
0762         gap >>= 3;
0763
0764     if (new_param > old_param)
0765         return old_param + gap;
0766     else
0767         return old_param - gap;
0768 }
0769
0770 /**
0771  * hns_nic_adpt_coalesce - self adapte coalesce according to rx rate
0772  * @ring_data: pointer to hns_nic_ring_data
0773  **/
0774 static void hns_nic_adpt_coalesce(struct hns_nic_ring_data *ring_data)
0775 {
0776     struct hnae_ring *ring = ring_data->ring;
0777     struct hnae_handle *handle = ring->q->handle;
0778     u32 new_coal_param, old_coal_param = ring->coal_param;
0779
0780     if (ring->coal_rx_rate < HNS_LOWEST_LATENCY_RATE)
0781         new_coal_param = HNAE_LOWEST_LATENCY_COAL_PARAM;
0782     else if (ring->coal_rx_rate < HNS_LOW_LATENCY_RATE)
0783         new_coal_param = HNAE_LOW_LATENCY_COAL_PARAM;
0784     else
0785         new_coal_param = HNAE_BULK_LATENCY_COAL_PARAM;
0786
0787     if (new_coal_param == old_coal_param &&
0788         new_coal_param == handle->coal_param)
0789         return;
0790
0791     new_coal_param = smooth_alg(new_coal_param, old_coal_param);
0792     ring->coal_param = new_coal_param;
0793
0794     /**
0795      * Because all ring in one port has one coalesce param, when one ring
0796      * calculate its own coalesce param, it cannot write to hardware at
0797      * once. There are three conditions as follows:
0798      *       1. current ring's coalesce param is larger than the hardware.
0799      *       2. or ring which adapt last time can change again.
0800      *       3. timeout.
0801      */
0802     if (new_coal_param == handle->coal_param) {
0803         handle->coal_last_jiffies = jiffies;
0804         handle->coal_ring_idx = ring_data->queue_index;
0805     } else if (new_coal_param > handle->coal_param ||
0806            handle->coal_ring_idx == ring_data->queue_index ||
0807            time_after(jiffies, handle->coal_last_jiffies + (HZ >> 4))) {
0808         handle->dev->ops->set_coalesce_usecs(handle,
0809                     new_coal_param);
0810         handle->dev->ops->set_coalesce_frames(handle,
0811                     1, new_coal_param);
0812         handle->coal_param = new_coal_param;
0813         handle->coal_ring_idx = ring_data->queue_index;
0814         handle->coal_last_jiffies = jiffies;
0815     }
0816 }
0817
0818 static int hns_nic_rx_poll_one(struct hns_nic_ring_data *ring_data,
0819                    int budget, void *v)
0820 {
0821     struct hnae_ring *ring = ring_data->ring;
0822     struct sk_buff *skb;
0823     int num, bnum;
0824 #define RCB_NOF_ALLOC_RX_BUFF_ONCE 16
0825     int recv_pkts, recv_bds, clean_count, err;
0826     int unused_count = hns_desc_unused(ring);
0827
0828     num = readl_relaxed(ring->io_base + RCB_REG_FBDNUM);
0829     rmb(); /* make sure num taken effect before the other data is touched */
0830
0831     recv_pkts = 0, recv_bds = 0, clean_count = 0;
0832     num -= unused_count;
0833
0834     while (recv_pkts < budget && recv_bds < num) {
0835         /* reuse or realloc buffers */
0836         if (clean_count + unused_count >= RCB_NOF_ALLOC_RX_BUFF_ONCE) {
0837             hns_nic_alloc_rx_buffers(ring_data,
0838                          clean_count + unused_count);
0839             clean_count = 0;
0840             unused_count = hns_desc_unused(ring);
0841         }
0842
0843         /* poll one pkt */
0844         err = hns_nic_poll_rx_skb(ring_data, &skb, &bnum);
0845         if (unlikely(!skb)) /* this fault cannot be repaired */
0846             goto out;
0847
0848         recv_bds += bnum;
0849         clean_count += bnum;
0850         if (unlikely(err)) {  /* do jump the err */
0851             recv_pkts++;
0852             continue;
0853         }
0854
0855         /* do update ip stack process*/
0856         ((void (*)(struct hns_nic_ring_data *, struct sk_buff *))v)(
0857                             ring_data, skb);
0858         recv_pkts++;
0859     }
0860
0861 out:
0862     /* make all data has been write before submit */
0863     if (clean_count + unused_count > 0)
0864         hns_nic_alloc_rx_buffers(ring_data,
0865                      clean_count + unused_count);
0866
0867     return recv_pkts;
0868 }
0869
0870 static bool hns_nic_rx_fini_pro(struct hns_nic_ring_data *ring_data)
0871 {
0872     struct hnae_ring *ring = ring_data->ring;
0873     int num;
0874     bool rx_stopped;
0875
0876     hns_update_rx_rate(ring);
0877
0878     /* for hardware bug fixed */
0879     ring_data->ring->q->handle->dev->ops->toggle_ring_irq(ring, 0);
0880     num = readl_relaxed(ring->io_base + RCB_REG_FBDNUM);
0881
0882     if (num <= hns_coal_rx_bdnum(ring)) {
0883         if (ring->q->handle->coal_adapt_en)
0884             hns_nic_adpt_coalesce(ring_data);
0885
0886         rx_stopped = true;
0887     } else {
0888         ring_data->ring->q->handle->dev->ops->toggle_ring_irq(
0889             ring_data->ring, 1);
0890
0891         rx_stopped = false;
0892     }
0893
0894     return rx_stopped;
0895 }
0896
0897 static bool hns_nic_rx_fini_pro_v2(struct hns_nic_ring_data *ring_data)
0898 {
0899     struct hnae_ring *ring = ring_data->ring;
0900     int num;
0901
0902     hns_update_rx_rate(ring);
0903     num = readl_relaxed(ring->io_base + RCB_REG_FBDNUM);
0904
0905     if (num <= hns_coal_rx_bdnum(ring)) {
0906         if (ring->q->handle->coal_adapt_en)
0907             hns_nic_adpt_coalesce(ring_data);
0908
0909         return true;
0910     }
0911
0912     return false;
0913 }
0914
0915 static inline void hns_nic_reclaim_one_desc(struct hnae_ring *ring,
0916                         int *bytes, int *pkts)
0917 {
0918     struct hnae_desc_cb *desc_cb = &ring->desc_cb[ring->next_to_clean];
0919
0920     (*pkts) += (desc_cb->type == DESC_TYPE_SKB);
0921     (*bytes) += desc_cb->length;
0922     /* desc_cb will be cleaned, after hnae_free_buffer_detach*/
0923     hnae_free_buffer_detach(ring, ring->next_to_clean);
0924
0925     ring_ptr_move_fw(ring, next_to_clean);
0926 }
0927
0928 static int is_valid_clean_head(struct hnae_ring *ring, int h)
0929 {
0930     int u = ring->next_to_use;
0931     int c = ring->next_to_clean;
0932
0933     if (unlikely(h > ring->desc_num))
0934         return 0;
0935
0936     assert(u > 0 && u < ring->desc_num);
0937     assert(c > 0 && c < ring->desc_num);
0938     assert(u != c && h != c); /* must be checked before call this func */
0939
0940     return u > c ? (h > c && h <= u) : (h > c || h <= u);
0941 }
0942
0943 /* reclaim all desc in one budget
0944  * return error or number of desc left
0945  */
0946 static int hns_nic_tx_poll_one(struct hns_nic_ring_data *ring_data,
0947                    int budget, void *v)
0948 {
0949     struct hnae_ring *ring = ring_data->ring;
0950     struct net_device *ndev = ring_data->napi.dev;
0951     struct netdev_queue *dev_queue;
0952     struct hns_nic_priv *priv = netdev_priv(ndev);
0953     int head;
0954     int bytes, pkts;
0955
0956     head = readl_relaxed(ring->io_base + RCB_REG_HEAD);
0957     rmb(); /* make sure head is ready before touch any data */
0958
0959     if (is_ring_empty(ring) || head == ring->next_to_clean)
0960         return 0; /* no data to poll */
0961
0962     if (!is_valid_clean_head(ring, head)) {
0963         netdev_err(ndev, "wrong head (%d, %d-%d)\n", head,
0964                ring->next_to_use, ring->next_to_clean);
0965         ring->stats.io_err_cnt++;
0966         return -EIO;
0967     }
0968
0969     bytes = 0;
0970     pkts = 0;
0971     while (head != ring->next_to_clean) {
0972         hns_nic_reclaim_one_desc(ring, &bytes, &pkts);
0973         /* issue prefetch for next Tx descriptor */
0974         prefetch(&ring->desc_cb[ring->next_to_clean]);
0975     }
0976     /* update tx ring statistics. */
0977     ring->stats.tx_pkts += pkts;
0978     ring->stats.tx_bytes += bytes;
0979
0980     dev_queue = netdev_get_tx_queue(ndev, ring_data->queue_index);
0981     netdev_tx_completed_queue(dev_queue, pkts, bytes);
0982
0983     if (unlikely(priv->link && !netif_carrier_ok(ndev)))
0984         netif_carrier_on(ndev);
0985
0986     if (unlikely(pkts && netif_carrier_ok(ndev) &&
0987              (ring_space(ring) >= ring->max_desc_num_per_pkt * 2))) {
0988         /* Make sure that anybody stopping the queue after this
0989          * sees the new next_to_clean.
0990          */
0991         smp_mb();
0992         if (netif_tx_queue_stopped(dev_queue) &&
0993             !test_bit(NIC_STATE_DOWN, &priv->state)) {
0994             netif_tx_wake_queue(dev_queue);
0995             ring->stats.restart_queue++;
0996         }
0997     }
0998     return 0;
0999 }
1000
1001 static bool hns_nic_tx_fini_pro(struct hns_nic_ring_data *ring_data)
1002 {
1003     struct hnae_ring *ring = ring_data->ring;
1004     int head;
1005
1006     ring_data->ring->q->handle->dev->ops->toggle_ring_irq(ring, 0);
1007
1008     head = readl_relaxed(ring->io_base + RCB_REG_HEAD);
1009
1010     if (head != ring->next_to_clean) {
1011         ring_data->ring->q->handle->dev->ops->toggle_ring_irq(
1012             ring_data->ring, 1);
1013
1014         return false;
1015     } else {
1016         return true;
1017     }
1018 }
1019
1020 static bool hns_nic_tx_fini_pro_v2(struct hns_nic_ring_data *ring_data)
1021 {
1022     struct hnae_ring *ring = ring_data->ring;
1023     int head = readl_relaxed(ring->io_base + RCB_REG_HEAD);
1024
1025     if (head == ring->next_to_clean)
1026         return true;
1027     else
1028         return false;
1029 }
1030
1031 static void hns_nic_tx_clr_all_bufs(struct hns_nic_ring_data *ring_data)
1032 {
1033     struct hnae_ring *ring = ring_data->ring;
1034     struct net_device *ndev = ring_data->napi.dev;
1035     struct netdev_queue *dev_queue;
1036     int head;
1037     int bytes, pkts;
1038
1039     head = ring->next_to_use; /* ntu :soft setted ring position*/
1040     bytes = 0;
1041     pkts = 0;
1042     while (head != ring->next_to_clean)
1043         hns_nic_reclaim_one_desc(ring, &bytes, &pkts);
1044
1045     dev_queue = netdev_get_tx_queue(ndev, ring_data->queue_index);
1046     netdev_tx_reset_queue(dev_queue);
1047 }
1048
1049 static int hns_nic_common_poll(struct napi_struct *napi, int budget)
1050 {
1051     int clean_complete = 0;
1052     struct hns_nic_ring_data *ring_data =
1053         container_of(napi, struct hns_nic_ring_data, napi);
1054     struct hnae_ring *ring = ring_data->ring;
1055
1056     clean_complete += ring_data->poll_one(
1057                 ring_data, budget - clean_complete,
1058                 ring_data->ex_process);
1059
1060     if (clean_complete < budget) {
1061         if (ring_data->fini_process(ring_data)) {
1062             napi_complete(napi);
1063             ring->q->handle->dev->ops->toggle_ring_irq(ring, 0);
1064         } else {
1065             return budget;
1066         }
1067     }
1068
1069     return clean_complete;
1070 }
1071
1072 static irqreturn_t hns_irq_handle(int irq, void *dev)
1073 {
1074     struct hns_nic_ring_data *ring_data = (struct hns_nic_ring_data *)dev;
1075
1076     ring_data->ring->q->handle->dev->ops->toggle_ring_irq(
1077         ring_data->ring, 1);
1078     napi_schedule(&ring_data->napi);
1079
1080     return IRQ_HANDLED;
1081 }
1082
1083 /**
1084  *hns_nic_adjust_link - adjust net work mode by the phy stat or new param
1085  *@ndev: net device
1086  */
1087 static void hns_nic_adjust_link(struct net_device *ndev)
1088 {
1089     struct hns_nic_priv *priv = netdev_priv(ndev);
1090     struct hnae_handle *h = priv->ae_handle;
1091     int state = 1;
1092
1093     /* If there is no phy, do not need adjust link */
1094     if (ndev->phydev) {
1095         /* When phy link down, do nothing */
1096         if (ndev->phydev->link == 0)
1097             return;
1098
1099         if (h->dev->ops->need_adjust_link(h, ndev->phydev->speed,
1100                           ndev->phydev->duplex)) {
1101             /* because Hi161X chip don't support to change gmac
1102              * speed and duplex with traffic. Delay 200ms to
1103              * make sure there is no more data in chip FIFO.
1104              */
1105             netif_carrier_off(ndev);
1106             msleep(200);
1107             h->dev->ops->adjust_link(h, ndev->phydev->speed,
1108                          ndev->phydev->duplex);
1109             netif_carrier_on(ndev);
1110         }
1111     }
1112
1113     state = state && h->dev->ops->get_status(h);
1114
1115     if (state != priv->link) {
1116         if (state) {
1117             netif_carrier_on(ndev);
1118             netif_tx_wake_all_queues(ndev);
1119             netdev_info(ndev, "link up\n");
1120         } else {
1121             netif_carrier_off(ndev);
1122             netdev_info(ndev, "link down\n");
1123         }
1124         priv->link = state;
1125     }
1126 }
1127
1128 /**
1129  *hns_nic_init_phy - init phy
1130  *@ndev: net device
1131  *@h: ae handle
1132  * Return 0 on success, negative on failure
1133  */
1134 int hns_nic_init_phy(struct net_device *ndev, struct hnae_handle *h)
1135 {
1136     __ETHTOOL_DECLARE_LINK_MODE_MASK(supported) = { 0, };
1137     struct phy_device *phy_dev = h->phy_dev;
1138     int ret;
1139
1140     if (!h->phy_dev)
1141         return 0;
1142
1143     ethtool_convert_legacy_u32_to_link_mode(supported, h->if_support);
1144     linkmode_and(phy_dev->supported, phy_dev->supported, supported);
1145     linkmode_copy(phy_dev->advertising, phy_dev->supported);
1146
1147     if (h->phy_if == PHY_INTERFACE_MODE_XGMII)
1148         phy_dev->autoneg = false;
1149
1150     if (h->phy_if != PHY_INTERFACE_MODE_XGMII) {
1151         phy_dev->dev_flags = 0;
1152
1153         ret = phy_connect_direct(ndev, phy_dev, hns_nic_adjust_link,
1154                      h->phy_if);
1155     } else {
1156         ret = phy_attach_direct(ndev, phy_dev, 0, h->phy_if);
1157     }
1158     if (unlikely(ret))
1159         return -ENODEV;
1160
1161     phy_attached_info(phy_dev);
1162
1163     return 0;
1164 }
1165
1166 static int hns_nic_ring_open(struct net_device *netdev, int idx)
1167 {
1168     struct hns_nic_priv *priv = netdev_priv(netdev);
1169     struct hnae_handle *h = priv->ae_handle;
1170
1171     napi_enable(&priv->ring_data[idx].napi);
1172
1173     enable_irq(priv->ring_data[idx].ring->irq);
1174     h->dev->ops->toggle_ring_irq(priv->ring_data[idx].ring, 0);
1175
1176     return 0;
1177 }
1178
1179 static int hns_nic_net_set_mac_address(struct net_device *ndev, void *p)
1180 {
1181     struct hns_nic_priv *priv = netdev_priv(ndev);
1182     struct hnae_handle *h = priv->ae_handle;
1183     struct sockaddr *mac_addr = p;
1184     int ret;
1185
1186     if (!mac_addr || !is_valid_ether_addr((const u8 *)mac_addr->sa_data))
1187         return -EADDRNOTAVAIL;
1188
1189     ret = h->dev->ops->set_mac_addr(h, mac_addr->sa_data);
1190     if (ret) {
1191         netdev_err(ndev, "set_mac_address fail, ret=%d!\n", ret);
1192         return ret;
1193     }
1194
1195     eth_hw_addr_set(ndev, mac_addr->sa_data);
1196
1197     return 0;
1198 }
1199
1200 static void hns_nic_update_stats(struct net_device *netdev)
1201 {
1202     struct hns_nic_priv *priv = netdev_priv(netdev);
1203     struct hnae_handle *h = priv->ae_handle;
1204
1205     h->dev->ops->update_stats(h, &netdev->stats);
1206 }
1207
1208 /* set mac addr if it is configed. or leave it to the AE driver */
1209 static void hns_init_mac_addr(struct net_device *ndev)
1210 {
1211     struct hns_nic_priv *priv = netdev_priv(ndev);
1212
1213     if (device_get_ethdev_address(priv->dev, ndev)) {
1214         eth_hw_addr_random(ndev);
1215         dev_warn(priv->dev, "No valid mac, use random mac %pM",
1216              ndev->dev_addr);
1217     }
1218 }
1219
1220 static void hns_nic_ring_close(struct net_device *netdev, int idx)
1221 {
1222     struct hns_nic_priv *priv = netdev_priv(netdev);
1223     struct hnae_handle *h = priv->ae_handle;
1224
1225     h->dev->ops->toggle_ring_irq(priv->ring_data[idx].ring, 1);
1226     disable_irq(priv->ring_data[idx].ring->irq);
1227
1228     napi_disable(&priv->ring_data[idx].napi);
1229 }
1230
1231 static int hns_nic_init_affinity_mask(int q_num, int ring_idx,
1232                       struct hnae_ring *ring, cpumask_t *mask)
1233 {
1234     int cpu;
1235
1236     /* Different irq balance between 16core and 32core.
1237      * The cpu mask set by ring index according to the ring flag
1238      * which indicate the ring is tx or rx.
1239      */
1240     if (q_num == num_possible_cpus()) {
1241         if (is_tx_ring(ring))
1242             cpu = ring_idx;
1243         else
1244             cpu = ring_idx - q_num;
1245     } else {
1246         if (is_tx_ring(ring))
1247             cpu = ring_idx * 2;
1248         else
1249             cpu = (ring_idx - q_num) * 2 + 1;
1250     }
1251
1252     cpumask_clear(mask);
1253     cpumask_set_cpu(cpu, mask);
1254
1255     return cpu;
1256 }
1257
1258 static void hns_nic_free_irq(int q_num, struct hns_nic_priv *priv)
1259 {
1260     int i;
1261
1262     for (i = 0; i < q_num * 2; i++) {
1263         if (priv->ring_data[i].ring->irq_init_flag == RCB_IRQ_INITED) {
1264             irq_set_affinity_hint(priv->ring_data[i].ring->irq,
1265                           NULL);
1266             free_irq(priv->ring_data[i].ring->irq,
1267                  &priv->ring_data[i]);
1268             priv->ring_data[i].ring->irq_init_flag =
1269                 RCB_IRQ_NOT_INITED;
1270         }
1271     }
1272 }
1273
1274 static int hns_nic_init_irq(struct hns_nic_priv *priv)
1275 {
1276     struct hnae_handle *h = priv->ae_handle;
1277     struct hns_nic_ring_data *rd;
1278     int i;
1279     int ret;
1280     int cpu;
1281
1282     for (i = 0; i < h->q_num * 2; i++) {
1283         rd = &priv->ring_data[i];
1284
1285         if (rd->ring->irq_init_flag == RCB_IRQ_INITED)
1286             break;
1287
1288         snprintf(rd->ring->ring_name, RCB_RING_NAME_LEN,
1289              "%s-%s%d", priv->netdev->name,
1290              (is_tx_ring(rd->ring) ? "tx" : "rx"), rd->queue_index);
1291
1292         rd->ring->ring_name[RCB_RING_NAME_LEN - 1] = '\0';
1293
1294         irq_set_status_flags(rd->ring->irq, IRQ_NOAUTOEN);
1295         ret = request_irq(rd->ring->irq,
1296                   hns_irq_handle, 0, rd->ring->ring_name, rd);
1297         if (ret) {
1298             netdev_err(priv->netdev, "request irq(%d) fail\n",
1299                    rd->ring->irq);
1300             goto out_free_irq;
1301         }
1302
1303         cpu = hns_nic_init_affinity_mask(h->q_num, i,
1304                          rd->ring, &rd->mask);
1305
1306         if (cpu_online(cpu))
1307             irq_set_affinity_hint(rd->ring->irq,
1308                           &rd->mask);
1309
1310         rd->ring->irq_init_flag = RCB_IRQ_INITED;
1311     }
1312
1313     return 0;
1314
1315 out_free_irq:
1316     hns_nic_free_irq(h->q_num, priv);
1317     return ret;
1318 }
1319
1320 static int hns_nic_net_up(struct net_device *ndev)
1321 {
1322     struct hns_nic_priv *priv = netdev_priv(ndev);
1323     struct hnae_handle *h = priv->ae_handle;
1324     int i, j;
1325     int ret;
1326
1327     if (!test_bit(NIC_STATE_DOWN, &priv->state))
1328         return 0;
1329
1330     ret = hns_nic_init_irq(priv);
1331     if (ret != 0) {
1332         netdev_err(ndev, "hns init irq failed! ret=%d\n", ret);
1333         return ret;
1334     }
1335
1336     for (i = 0; i < h->q_num * 2; i++) {
1337         ret = hns_nic_ring_open(ndev, i);
1338         if (ret)
1339             goto out_has_some_queues;
1340     }
1341
1342     ret = h->dev->ops->set_mac_addr(h, ndev->dev_addr);
1343     if (ret)
1344         goto out_set_mac_addr_err;
1345
1346     ret = h->dev->ops->start ? h->dev->ops->start(h) : 0;
1347     if (ret)
1348         goto out_start_err;
1349
1350     if (ndev->phydev)
1351         phy_start(ndev->phydev);
1352
1353     clear_bit(NIC_STATE_DOWN, &priv->state);
1354     (void)mod_timer(&priv->service_timer, jiffies + SERVICE_TIMER_HZ);
1355
1356     return 0;
1357
1358 out_start_err:
1359     netif_stop_queue(ndev);
1360 out_set_mac_addr_err:
1361 out_has_some_queues:
1362     for (j = i - 1; j >= 0; j--)
1363         hns_nic_ring_close(ndev, j);
1364
1365     hns_nic_free_irq(h->q_num, priv);
1366     set_bit(NIC_STATE_DOWN, &priv->state);
1367
1368     return ret;
1369 }
1370
1371 static void hns_nic_net_down(struct net_device *ndev)
1372 {
1373     int i;
1374     struct hnae_ae_ops *ops;
1375     struct hns_nic_priv *priv = netdev_priv(ndev);
1376
1377     if (test_and_set_bit(NIC_STATE_DOWN, &priv->state))
1378         return;
1379
1380     (void)del_timer_sync(&priv->service_timer);
1381     netif_tx_stop_all_queues(ndev);
1382     netif_carrier_off(ndev);
1383     netif_tx_disable(ndev);
1384     priv->link = 0;
1385
1386     if (ndev->phydev)
1387         phy_stop(ndev->phydev);
1388
1389     ops = priv->ae_handle->dev->ops;
1390
1391     if (ops->stop)
1392         ops->stop(priv->ae_handle);
1393
1394     netif_tx_stop_all_queues(ndev);
1395
1396     for (i = priv->ae_handle->q_num - 1; i >= 0; i--) {
1397         hns_nic_ring_close(ndev, i);
1398         hns_nic_ring_close(ndev, i + priv->ae_handle->q_num);
1399
1400         /* clean tx buffers*/
1401         hns_nic_tx_clr_all_bufs(priv->ring_data + i);
1402     }
1403 }
1404
1405 void hns_nic_net_reset(struct net_device *ndev)
1406 {
1407     struct hns_nic_priv *priv = netdev_priv(ndev);
1408     struct hnae_handle *handle = priv->ae_handle;
1409
1410     while (test_and_set_bit(NIC_STATE_RESETTING, &priv->state))
1411         usleep_range(1000, 2000);
1412
1413     (void)hnae_reinit_handle(handle);
1414
1415     clear_bit(NIC_STATE_RESETTING, &priv->state);
1416 }
1417
1418 void hns_nic_net_reinit(struct net_device *netdev)
1419 {
1420     struct hns_nic_priv *priv = netdev_priv(netdev);
1421     enum hnae_port_type type = priv->ae_handle->port_type;
1422
1423     netif_trans_update(priv->netdev);
1424     while (test_and_set_bit(NIC_STATE_REINITING, &priv->state))
1425         usleep_range(1000, 2000);
1426
1427     hns_nic_net_down(netdev);
1428
1429     /* Only do hns_nic_net_reset in debug mode
1430      * because of hardware limitation.
1431      */
1432     if (type == HNAE_PORT_DEBUG)
1433         hns_nic_net_reset(netdev);
1434
1435     (void)hns_nic_net_up(netdev);
1436     clear_bit(NIC_STATE_REINITING, &priv->state);
1437 }
1438
1439 static int hns_nic_net_open(struct net_device *ndev)
1440 {
1441     struct hns_nic_priv *priv = netdev_priv(ndev);
1442     struct hnae_handle *h = priv->ae_handle;
1443     int ret;
1444
1445     if (test_bit(NIC_STATE_TESTING, &priv->state))
1446         return -EBUSY;
1447
1448     priv->link = 0;
1449     netif_carrier_off(ndev);
1450
1451     ret = netif_set_real_num_tx_queues(ndev, h->q_num);
1452     if (ret < 0) {
1453         netdev_err(ndev, "netif_set_real_num_tx_queues fail, ret=%d!\n",
1454                ret);
1455         return ret;
1456     }
1457
1458     ret = netif_set_real_num_rx_queues(ndev, h->q_num);
1459     if (ret < 0) {
1460         netdev_err(ndev,
1461                "netif_set_real_num_rx_queues fail, ret=%d!\n", ret);
1462         return ret;
1463     }
1464
1465     ret = hns_nic_net_up(ndev);
1466     if (ret) {
1467         netdev_err(ndev,
1468                "hns net up fail, ret=%d!\n", ret);
1469         return ret;
1470     }
1471
1472     return 0;
1473 }
1474
1475 static int hns_nic_net_stop(struct net_device *ndev)
1476 {
1477     hns_nic_net_down(ndev);
1478
1479     return 0;
1480 }
1481
1482 static void hns_tx_timeout_reset(struct hns_nic_priv *priv);
1483 #define HNS_TX_TIMEO_LIMIT (40 * HZ)
1484 static void hns_nic_net_timeout(struct net_device *ndev, unsigned int txqueue)
1485 {
1486     struct hns_nic_priv *priv = netdev_priv(ndev);
1487
1488     if (ndev->watchdog_timeo < HNS_TX_TIMEO_LIMIT) {
1489         ndev->watchdog_timeo *= 2;
1490         netdev_info(ndev, "watchdog_timo changed to %d.\n",
1491                 ndev->watchdog_timeo);
1492     } else {
1493         ndev->watchdog_timeo = HNS_NIC_TX_TIMEOUT;
1494         hns_tx_timeout_reset(priv);
1495     }
1496 }
1497
1498 static netdev_tx_t hns_nic_net_xmit(struct sk_buff *skb,
1499                     struct net_device *ndev)
1500 {
1501     struct hns_nic_priv *priv = netdev_priv(ndev);
1502
1503     assert(skb->queue_mapping < priv->ae_handle->q_num);
1504
1505     return hns_nic_net_xmit_hw(ndev, skb,
1506                    &tx_ring_data(priv, skb->queue_mapping));
1507 }
1508
1509 static void hns_nic_drop_rx_fetch(struct hns_nic_ring_data *ring_data,
1510                   struct sk_buff *skb)
1511 {
1512     dev_kfree_skb_any(skb);
1513 }
1514
1515 #define HNS_LB_TX_RING  0
1516 static struct sk_buff *hns_assemble_skb(struct net_device *ndev)
1517 {
1518     struct sk_buff *skb;
1519     struct ethhdr *ethhdr;
1520     int frame_len;
1521
1522     /* allocate test skb */
1523     skb = alloc_skb(64, GFP_KERNEL);
1524     if (!skb)
1525         return NULL;
1526
1527     skb_put(skb, 64);
1528     skb->dev = ndev;
1529     memset(skb->data, 0xFF, skb->len);
1530
1531     /* must be tcp/ip package */
1532     ethhdr = (struct ethhdr *)skb->data;
1533     ethhdr->h_proto = htons(ETH_P_IP);
1534
1535     frame_len = skb->len & (~1ul);
1536     memset(&skb->data[frame_len / 2], 0xAA,
1537            frame_len / 2 - 1);
1538
1539     skb->queue_mapping = HNS_LB_TX_RING;
1540
1541     return skb;
1542 }
1543
1544 static int hns_enable_serdes_lb(struct net_device *ndev)
1545 {
1546     struct hns_nic_priv *priv = netdev_priv(ndev);
1547     struct hnae_handle *h = priv->ae_handle;
1548     struct hnae_ae_ops *ops = h->dev->ops;
1549     int speed, duplex;
1550     int ret;
1551
1552     ret = ops->set_loopback(h, MAC_INTERNALLOOP_SERDES, 1);
1553     if (ret)
1554         return ret;
1555
1556     ret = ops->start ? ops->start(h) : 0;
1557     if (ret)
1558         return ret;
1559
1560     /* link adjust duplex*/
1561     if (h->phy_if != PHY_INTERFACE_MODE_XGMII)
1562         speed = 1000;
1563     else
1564         speed = 10000;
1565     duplex = 1;
1566
1567     ops->adjust_link(h, speed, duplex);
1568
1569     /* wait h/w ready */
1570     mdelay(300);
1571
1572     return 0;
1573 }
1574
1575 static void hns_disable_serdes_lb(struct net_device *ndev)
1576 {
1577     struct hns_nic_priv *priv = netdev_priv(ndev);
1578     struct hnae_handle *h = priv->ae_handle;
1579     struct hnae_ae_ops *ops = h->dev->ops;
1580
1581     ops->stop(h);
1582     ops->set_loopback(h, MAC_INTERNALLOOP_SERDES, 0);
1583 }
1584
1585 /**
1586  *hns_nic_clear_all_rx_fetch - clear the chip fetched descriptions. The
1587  *function as follows:
1588  *    1. if one rx ring has found the page_offset is not equal 0 between head
1589  *       and tail, it means that the chip fetched the wrong descs for the ring
1590  *       which buffer size is 4096.
1591  *    2. we set the chip serdes loopback and set rss indirection to the ring.
1592  *    3. construct 64-bytes ip broadcast packages, wait the associated rx ring
1593  *       receiving all packages and it will fetch new descriptions.
1594  *    4. recover to the original state.
1595  *
1596  *@ndev: net device
1597  */
1598 static int hns_nic_clear_all_rx_fetch(struct net_device *ndev)
1599 {
1600     struct hns_nic_priv *priv = netdev_priv(ndev);
1601     struct hnae_handle *h = priv->ae_handle;
1602     struct hnae_ae_ops *ops = h->dev->ops;
1603     struct hns_nic_ring_data *rd;
1604     struct hnae_ring *ring;
1605     struct sk_buff *skb;
1606     u32 *org_indir;
1607     u32 *cur_indir;
1608     int indir_size;
1609     int head, tail;
1610     int fetch_num;
1611     int i, j;
1612     bool found;
1613     int retry_times;
1614     int ret = 0;
1615
1616     /* alloc indir memory */
1617     indir_size = ops->get_rss_indir_size(h) * sizeof(*org_indir);
1618     org_indir = kzalloc(indir_size, GFP_KERNEL);
1619     if (!org_indir)
1620         return -ENOMEM;
1621
1622     /* store the original indirection */
1623     ops->get_rss(h, org_indir, NULL, NULL);
1624
1625     cur_indir = kzalloc(indir_size, GFP_KERNEL);
1626     if (!cur_indir) {
1627         ret = -ENOMEM;
1628         goto cur_indir_alloc_err;
1629     }
1630
1631     /* set loopback */
1632     if (hns_enable_serdes_lb(ndev)) {
1633         ret = -EINVAL;
1634         goto enable_serdes_lb_err;
1635     }
1636
1637     /* foreach every rx ring to clear fetch desc */
1638     for (i = 0; i < h->q_num; i++) {
1639         ring = &h->qs[i]->rx_ring;
1640         head = readl_relaxed(ring->io_base + RCB_REG_HEAD);
1641         tail = readl_relaxed(ring->io_base + RCB_REG_TAIL);
1642         found = false;
1643         fetch_num = ring_dist(ring, head, tail);
1644
1645         while (head != tail) {
1646             if (ring->desc_cb[head].page_offset != 0) {
1647                 found = true;
1648                 break;
1649             }
1650
1651             head++;
1652             if (head == ring->desc_num)
1653                 head = 0;
1654         }
1655
1656         if (found) {
1657             for (j = 0; j < indir_size / sizeof(*org_indir); j++)
1658                 cur_indir[j] = i;
1659             ops->set_rss(h, cur_indir, NULL, 0);
1660
1661             for (j = 0; j < fetch_num; j++) {
1662                 /* alloc one skb and init */
1663                 skb = hns_assemble_skb(ndev);
1664                 if (!skb) {
1665                     ret = -ENOMEM;
1666                     goto out;
1667                 }
1668                 rd = &tx_ring_data(priv, skb->queue_mapping);
1669                 hns_nic_net_xmit_hw(ndev, skb, rd);
1670
1671                 retry_times = 0;
1672                 while (retry_times++ < 10) {
1673                     mdelay(10);
1674                     /* clean rx */
1675                     rd = &rx_ring_data(priv, i);
1676                     if (rd->poll_one(rd, fetch_num,
1677                              hns_nic_drop_rx_fetch))
1678                         break;
1679                 }
1680
1681                 retry_times = 0;
1682                 while (retry_times++ < 10) {
1683                     mdelay(10);
1684                     /* clean tx ring 0 send package */
1685                     rd = &tx_ring_data(priv,
1686                                HNS_LB_TX_RING);
1687                     if (rd->poll_one(rd, fetch_num, NULL))
1688                         break;
1689                 }
1690             }
1691         }
1692     }
1693
1694 out:
1695     /* restore everything */
1696     ops->set_rss(h, org_indir, NULL, 0);
1697     hns_disable_serdes_lb(ndev);
1698 enable_serdes_lb_err:
1699     kfree(cur_indir);
1700 cur_indir_alloc_err:
1701     kfree(org_indir);
1702
1703     return ret;
1704 }
1705
1706 static int hns_nic_change_mtu(struct net_device *ndev, int new_mtu)
1707 {
1708     struct hns_nic_priv *priv = netdev_priv(ndev);
1709     struct hnae_handle *h = priv->ae_handle;
1710     bool if_running = netif_running(ndev);
1711     int ret;
1712
1713     /* MTU < 68 is an error and causes problems on some kernels */
1714     if (new_mtu < 68)
1715         return -EINVAL;
1716
1717     /* MTU no change */
1718     if (new_mtu == ndev->mtu)
1719         return 0;
1720
1721     if (!h->dev->ops->set_mtu)
1722         return -ENOTSUPP;
1723
1724     if (if_running) {
1725         (void)hns_nic_net_stop(ndev);
1726         msleep(100);
1727     }
1728
1729     if (priv->enet_ver != AE_VERSION_1 &&
1730         ndev->mtu <= BD_SIZE_2048_MAX_MTU &&
1731         new_mtu > BD_SIZE_2048_MAX_MTU) {
1732         /* update desc */
1733         hnae_reinit_all_ring_desc(h);
1734
1735         /* clear the package which the chip has fetched */
1736         ret = hns_nic_clear_all_rx_fetch(ndev);
1737
1738         /* the page offset must be consist with desc */
1739         hnae_reinit_all_ring_page_off(h);
1740
1741         if (ret) {
1742             netdev_err(ndev, "clear the fetched desc fail\n");
1743             goto out;
1744         }
1745     }
1746
1747     ret = h->dev->ops->set_mtu(h, new_mtu);
1748     if (ret) {
1749         netdev_err(ndev, "set mtu fail, return value %d\n",
1750                ret);
1751         goto out;
1752     }
1753
1754     /* finally, set new mtu to netdevice */
1755     ndev->mtu = new_mtu;
1756
1757 out:
1758     if (if_running) {
1759         if (hns_nic_net_open(ndev)) {
1760             netdev_err(ndev, "hns net open fail\n");
1761             ret = -EINVAL;
1762         }
1763     }
1764
1765     return ret;
1766 }
1767
1768 static int hns_nic_set_features(struct net_device *netdev,
1769                 netdev_features_t features)
1770 {
1771     struct hns_nic_priv *priv = netdev_priv(netdev);
1772
1773     switch (priv->enet_ver) {
1774     case AE_VERSION_1:
1775         if (features & (NETIF_F_TSO | NETIF_F_TSO6))
1776             netdev_info(netdev, "enet v1 do not support tso!\n");
1777         break;
1778     default:
1779         if (features & (NETIF_F_TSO | NETIF_F_TSO6)) {
1780             priv->ops.fill_desc = fill_tso_desc;
1781             priv->ops.maybe_stop_tx = hns_nic_maybe_stop_tso;
1782             /* The chip only support 7*4096 */
1783             netif_set_tso_max_size(netdev, 7 * 4096);
1784         } else {
1785             priv->ops.fill_desc = fill_v2_desc;
1786             priv->ops.maybe_stop_tx = hns_nic_maybe_stop_tx;
1787         }
1788         break;
1789     }
1790     netdev->features = features;
1791     return 0;
1792 }
1793
1794 static netdev_features_t hns_nic_fix_features(
1795         struct net_device *netdev, netdev_features_t features)
1796 {
1797     struct hns_nic_priv *priv = netdev_priv(netdev);
1798
1799     switch (priv->enet_ver) {
1800     case AE_VERSION_1:
1801         features &= ~(NETIF_F_TSO | NETIF_F_TSO6 |
1802                 NETIF_F_HW_VLAN_CTAG_FILTER);
1803         break;
1804     default:
1805         break;
1806     }
1807     return features;
1808 }
1809
1810 static int hns_nic_uc_sync(struct net_device *netdev, const unsigned char *addr)
1811 {
1812     struct hns_nic_priv *priv = netdev_priv(netdev);
1813     struct hnae_handle *h = priv->ae_handle;
1814
1815     if (h->dev->ops->add_uc_addr)
1816         return h->dev->ops->add_uc_addr(h, addr);
1817
1818     return 0;
1819 }
1820
1821 static int hns_nic_uc_unsync(struct net_device *netdev,
1822                  const unsigned char *addr)
1823 {
1824     struct hns_nic_priv *priv = netdev_priv(netdev);
1825     struct hnae_handle *h = priv->ae_handle;
1826
1827     if (h->dev->ops->rm_uc_addr)
1828         return h->dev->ops->rm_uc_addr(h, addr);
1829
1830     return 0;
1831 }
1832
1833 /**
1834  * hns_set_multicast_list - set mutl mac address
1835  * @ndev: net device
1836  *
1837  * return void
1838  */
1839 static void hns_set_multicast_list(struct net_device *ndev)
1840 {
1841     struct hns_nic_priv *priv = netdev_priv(ndev);
1842     struct hnae_handle *h = priv->ae_handle;
1843     struct netdev_hw_addr *ha = NULL;
1844
1845     if (!h) {
1846         netdev_err(ndev, "hnae handle is null\n");
1847         return;
1848     }
1849
1850     if (h->dev->ops->clr_mc_addr)
1851         if (h->dev->ops->clr_mc_addr(h))
1852             netdev_err(ndev, "clear multicast address fail\n");
1853
1854     if (h->dev->ops->set_mc_addr) {
1855         netdev_for_each_mc_addr(ha, ndev)
1856             if (h->dev->ops->set_mc_addr(h, ha->addr))
1857                 netdev_err(ndev, "set multicast fail\n");
1858     }
1859 }
1860
1861 static void hns_nic_set_rx_mode(struct net_device *ndev)
1862 {
1863     struct hns_nic_priv *priv = netdev_priv(ndev);
1864     struct hnae_handle *h = priv->ae_handle;
1865
1866     if (h->dev->ops->set_promisc_mode) {
1867         if (ndev->flags & IFF_PROMISC)
1868             h->dev->ops->set_promisc_mode(h, 1);
1869         else
1870             h->dev->ops->set_promisc_mode(h, 0);
1871     }
1872
1873     hns_set_multicast_list(ndev);
1874
1875     if (__dev_uc_sync(ndev, hns_nic_uc_sync, hns_nic_uc_unsync))
1876         netdev_err(ndev, "sync uc address fail\n");
1877 }
1878
1879 static void hns_nic_get_stats64(struct net_device *ndev,
1880                 struct rtnl_link_stats64 *stats)
1881 {
1882     int idx;
1883     u64 tx_bytes = 0;
1884     u64 rx_bytes = 0;
1885     u64 tx_pkts = 0;
1886     u64 rx_pkts = 0;
1887     struct hns_nic_priv *priv = netdev_priv(ndev);
1888     struct hnae_handle *h = priv->ae_handle;
1889
1890     for (idx = 0; idx < h->q_num; idx++) {
1891         tx_bytes += h->qs[idx]->tx_ring.stats.tx_bytes;
1892         tx_pkts += h->qs[idx]->tx_ring.stats.tx_pkts;
1893         rx_bytes += h->qs[idx]->rx_ring.stats.rx_bytes;
1894         rx_pkts += h->qs[idx]->rx_ring.stats.rx_pkts;
1895     }
1896
1897     stats->tx_bytes = tx_bytes;
1898     stats->tx_packets = tx_pkts;
1899     stats->rx_bytes = rx_bytes;
1900     stats->rx_packets = rx_pkts;
1901
1902     stats->rx_errors = ndev->stats.rx_errors;
1903     stats->multicast = ndev->stats.multicast;
1904     stats->rx_length_errors = ndev->stats.rx_length_errors;
1905     stats->rx_crc_errors = ndev->stats.rx_crc_errors;
1906     stats->rx_missed_errors = ndev->stats.rx_missed_errors;
1907
1908     stats->tx_errors = ndev->stats.tx_errors;
1909     stats->rx_dropped = ndev->stats.rx_dropped;
1910     stats->tx_dropped = ndev->stats.tx_dropped;
1911     stats->collisions = ndev->stats.collisions;
1912     stats->rx_over_errors = ndev->stats.rx_over_errors;
1913     stats->rx_frame_errors = ndev->stats.rx_frame_errors;
1914     stats->rx_fifo_errors = ndev->stats.rx_fifo_errors;
1915     stats->tx_aborted_errors = ndev->stats.tx_aborted_errors;
1916     stats->tx_carrier_errors = ndev->stats.tx_carrier_errors;
1917     stats->tx_fifo_errors = ndev->stats.tx_fifo_errors;
1918     stats->tx_heartbeat_errors = ndev->stats.tx_heartbeat_errors;
1919     stats->tx_window_errors = ndev->stats.tx_window_errors;
1920     stats->rx_compressed = ndev->stats.rx_compressed;
1921     stats->tx_compressed = ndev->stats.tx_compressed;
1922 }
1923
1924 static u16
1925 hns_nic_select_queue(struct net_device *ndev, struct sk_buff *skb,
1926              struct net_device *sb_dev)
1927 {
1928     struct ethhdr *eth_hdr = (struct ethhdr *)skb->data;
1929     struct hns_nic_priv *priv = netdev_priv(ndev);
1930
1931     /* fix hardware broadcast/multicast packets queue loopback */
1932     if (!AE_IS_VER1(priv->enet_ver) &&
1933         is_multicast_ether_addr(eth_hdr->h_dest))
1934         return 0;
1935     else
1936         return netdev_pick_tx(ndev, skb, NULL);
1937 }
1938
1939 static const struct net_device_ops hns_nic_netdev_ops = {
1940     .ndo_open = hns_nic_net_open,
1941     .ndo_stop = hns_nic_net_stop,
1942     .ndo_start_xmit = hns_nic_net_xmit,
1943     .ndo_tx_timeout = hns_nic_net_timeout,
1944     .ndo_set_mac_address = hns_nic_net_set_mac_address,
1945     .ndo_change_mtu = hns_nic_change_mtu,
1946     .ndo_eth_ioctl = phy_do_ioctl_running,
1947     .ndo_set_features = hns_nic_set_features,
1948     .ndo_fix_features = hns_nic_fix_features,
1949     .ndo_get_stats64 = hns_nic_get_stats64,
1950     .ndo_set_rx_mode = hns_nic_set_rx_mode,
1951     .ndo_select_queue = hns_nic_select_queue,
1952 };
1953
1954 static void hns_nic_update_link_status(struct net_device *netdev)
1955 {
1956     struct hns_nic_priv *priv = netdev_priv(netdev);
1957
1958     struct hnae_handle *h = priv->ae_handle;
1959
1960     if (h->phy_dev) {
1961         if (h->phy_if != PHY_INTERFACE_MODE_XGMII)
1962             return;
1963
1964         (void)genphy_read_status(h->phy_dev);
1965     }
1966     hns_nic_adjust_link(netdev);
1967 }
1968
1969 /* for dumping key regs*/
1970 static void hns_nic_dump(struct hns_nic_priv *priv)
1971 {
1972     struct hnae_handle *h = priv->ae_handle;
1973     struct hnae_ae_ops *ops = h->dev->ops;
1974     u32 *data, reg_num, i;
1975
1976     if (ops->get_regs_len && ops->get_regs) {
1977         reg_num = ops->get_regs_len(priv->ae_handle);
1978         reg_num = (reg_num + 3ul) & ~3ul;
1979         data = kcalloc(reg_num, sizeof(u32), GFP_KERNEL);
1980         if (data) {
1981             ops->get_regs(priv->ae_handle, data);
1982             for (i = 0; i < reg_num; i += 4)
1983                 pr_info("0x%08x: 0x%08x 0x%08x 0x%08x 0x%08x\n",
1984                     i, data[i], data[i + 1],
1985                     data[i + 2], data[i + 3]);
1986             kfree(data);
1987         }
1988     }
1989
1990     for (i = 0; i < h->q_num; i++) {
1991         pr_info("tx_queue%d_next_to_clean:%d\n",
1992             i, h->qs[i]->tx_ring.next_to_clean);
1993         pr_info("tx_queue%d_next_to_use:%d\n",
1994             i, h->qs[i]->tx_ring.next_to_use);
1995         pr_info("rx_queue%d_next_to_clean:%d\n",
1996             i, h->qs[i]->rx_ring.next_to_clean);
1997         pr_info("rx_queue%d_next_to_use:%d\n",
1998             i, h->qs[i]->rx_ring.next_to_use);
1999     }
2000 }
2001
2002 /* for resetting subtask */
2003 static void hns_nic_reset_subtask(struct hns_nic_priv *priv)
2004 {
2005     enum hnae_port_type type = priv->ae_handle->port_type;
2006
2007     if (!test_bit(NIC_STATE2_RESET_REQUESTED, &priv->state))
2008         return;
2009     clear_bit(NIC_STATE2_RESET_REQUESTED, &priv->state);
2010
2011     /* If we're already down, removing or resetting, just bail */
2012     if (test_bit(NIC_STATE_DOWN, &priv->state) ||
2013         test_bit(NIC_STATE_REMOVING, &priv->state) ||
2014         test_bit(NIC_STATE_RESETTING, &priv->state))
2015         return;
2016
2017     hns_nic_dump(priv);
2018     netdev_info(priv->netdev, "try to reset %s port!\n",
2019             (type == HNAE_PORT_DEBUG ? "debug" : "service"));
2020
2021     rtnl_lock();
2022     /* put off any impending NetWatchDogTimeout */
2023     netif_trans_update(priv->netdev);
2024     hns_nic_net_reinit(priv->netdev);
2025
2026     rtnl_unlock();
2027 }
2028
2029 /* for doing service complete*/
2030 static void hns_nic_service_event_complete(struct hns_nic_priv *priv)
2031 {
2032     WARN_ON(!test_bit(NIC_STATE_SERVICE_SCHED, &priv->state));
2033     /* make sure to commit the things */
2034     smp_mb__before_atomic();
2035     clear_bit(NIC_STATE_SERVICE_SCHED, &priv->state);
2036 }
2037
2038 static void hns_nic_service_task(struct work_struct *work)
2039 {
2040     struct hns_nic_priv *priv
2041         = container_of(work, struct hns_nic_priv, service_task);
2042     struct hnae_handle *h = priv->ae_handle;
2043
2044     hns_nic_reset_subtask(priv);
2045     hns_nic_update_link_status(priv->netdev);
2046     h->dev->ops->update_led_status(h);
2047     hns_nic_update_stats(priv->netdev);
2048
2049     hns_nic_service_event_complete(priv);
2050 }
2051
2052 static void hns_nic_task_schedule(struct hns_nic_priv *priv)
2053 {
2054     if (!test_bit(NIC_STATE_DOWN, &priv->state) &&
2055         !test_bit(NIC_STATE_REMOVING, &priv->state) &&
2056         !test_and_set_bit(NIC_STATE_SERVICE_SCHED, &priv->state))
2057         (void)schedule_work(&priv->service_task);
2058 }
2059
2060 static void hns_nic_service_timer(struct timer_list *t)
2061 {
2062     struct hns_nic_priv *priv = from_timer(priv, t, service_timer);
2063
2064     (void)mod_timer(&priv->service_timer, jiffies + SERVICE_TIMER_HZ);
2065
2066     hns_nic_task_schedule(priv);
2067 }
2068
2069 /**
2070  * hns_tx_timeout_reset - initiate reset due to Tx timeout
2071  * @priv: driver private struct
2072  **/
2073 static void hns_tx_timeout_reset(struct hns_nic_priv *priv)
2074 {
2075     /* Do the reset outside of interrupt context */
2076     if (!test_bit(NIC_STATE_DOWN, &priv->state)) {
2077         set_bit(NIC_STATE2_RESET_REQUESTED, &priv->state);
2078         netdev_warn(priv->netdev,
2079                 "initiating reset due to tx timeout(%llu,0x%lx)\n",
2080                 priv->tx_timeout_count, priv->state);
2081         priv->tx_timeout_count++;
2082         hns_nic_task_schedule(priv);
2083     }
2084 }
2085
2086 static int hns_nic_init_ring_data(struct hns_nic_priv *priv)
2087 {
2088     struct hnae_handle *h = priv->ae_handle;
2089     struct hns_nic_ring_data *rd;
2090     bool is_ver1 = AE_IS_VER1(priv->enet_ver);
2091     int i;
2092
2093     if (h->q_num > NIC_MAX_Q_PER_VF) {
2094         netdev_err(priv->netdev, "too much queue (%d)\n", h->q_num);
2095         return -EINVAL;
2096     }
2097
2098     priv->ring_data = kzalloc(array3_size(h->q_num,
2099                           sizeof(*priv->ring_data), 2),
2100                   GFP_KERNEL);
2101     if (!priv->ring_data)
2102         return -ENOMEM;
2103
2104     for (i = 0; i < h->q_num; i++) {
2105         rd = &priv->ring_data[i];
2106         rd->queue_index = i;
2107         rd->ring = &h->qs[i]->tx_ring;
2108         rd->poll_one = hns_nic_tx_poll_one;
2109         rd->fini_process = is_ver1 ? hns_nic_tx_fini_pro :
2110             hns_nic_tx_fini_pro_v2;
2111
2112         netif_napi_add(priv->netdev, &rd->napi,
2113                    hns_nic_common_poll, NAPI_POLL_WEIGHT);
2114         rd->ring->irq_init_flag = RCB_IRQ_NOT_INITED;
2115     }
2116     for (i = h->q_num; i < h->q_num * 2; i++) {
2117         rd = &priv->ring_data[i];
2118         rd->queue_index = i - h->q_num;
2119         rd->ring = &h->qs[i - h->q_num]->rx_ring;
2120         rd->poll_one = hns_nic_rx_poll_one;
2121         rd->ex_process = hns_nic_rx_up_pro;
2122         rd->fini_process = is_ver1 ? hns_nic_rx_fini_pro :
2123             hns_nic_rx_fini_pro_v2;
2124
2125         netif_napi_add(priv->netdev, &rd->napi,
2126                    hns_nic_common_poll, NAPI_POLL_WEIGHT);
2127         rd->ring->irq_init_flag = RCB_IRQ_NOT_INITED;
2128     }
2129
2130     return 0;
2131 }
2132
2133 static void hns_nic_uninit_ring_data(struct hns_nic_priv *priv)
2134 {
2135     struct hnae_handle *h = priv->ae_handle;
2136     int i;
2137
2138     for (i = 0; i < h->q_num * 2; i++) {
2139         netif_napi_del(&priv->ring_data[i].napi);
2140         if (priv->ring_data[i].ring->irq_init_flag == RCB_IRQ_INITED) {
2141             (void)irq_set_affinity_hint(
2142                 priv->ring_data[i].ring->irq,
2143                 NULL);
2144             free_irq(priv->ring_data[i].ring->irq,
2145                  &priv->ring_data[i]);
2146         }
2147
2148         priv->ring_data[i].ring->irq_init_flag = RCB_IRQ_NOT_INITED;
2149     }
2150     kfree(priv->ring_data);
2151 }
2152
2153 static void hns_nic_set_priv_ops(struct net_device *netdev)
2154 {
2155     struct hns_nic_priv *priv = netdev_priv(netdev);
2156     struct hnae_handle *h = priv->ae_handle;
2157
2158     if (AE_IS_VER1(priv->enet_ver)) {
2159         priv->ops.fill_desc = fill_desc;
2160         priv->ops.get_rxd_bnum = get_rx_desc_bnum;
2161         priv->ops.maybe_stop_tx = hns_nic_maybe_stop_tx;
2162     } else {
2163         priv->ops.get_rxd_bnum = get_v2rx_desc_bnum;
2164         if ((netdev->features & NETIF_F_TSO) ||
2165             (netdev->features & NETIF_F_TSO6)) {
2166             priv->ops.fill_desc = fill_tso_desc;
2167             priv->ops.maybe_stop_tx = hns_nic_maybe_stop_tso;
2168             /* This chip only support 7*4096 */
2169             netif_set_tso_max_size(netdev, 7 * 4096);
2170         } else {
2171             priv->ops.fill_desc = fill_v2_desc;
2172             priv->ops.maybe_stop_tx = hns_nic_maybe_stop_tx;
2173         }
2174         /* enable tso when init
2175          * control tso on/off through TSE bit in bd
2176          */
2177         h->dev->ops->set_tso_stats(h, 1);
2178     }
2179 }
2180
2181 static int hns_nic_try_get_ae(struct net_device *ndev)
2182 {
2183     struct hns_nic_priv *priv = netdev_priv(ndev);
2184     struct hnae_handle *h;
2185     int ret;
2186
2187     h = hnae_get_handle(&priv->netdev->dev,
2188                 priv->fwnode, priv->port_id, NULL);
2189     if (IS_ERR_OR_NULL(h)) {
2190         ret = -ENODEV;
2191         dev_dbg(priv->dev, "has not handle, register notifier!\n");
2192         goto out;
2193     }
2194     priv->ae_handle = h;
2195
2196     ret = hns_nic_init_phy(ndev, h);
2197     if (ret) {
2198         dev_err(priv->dev, "probe phy device fail!\n");
2199         goto out_init_phy;
2200     }
2201
2202     ret = hns_nic_init_ring_data(priv);
2203     if (ret) {
2204         ret = -ENOMEM;
2205         goto out_init_ring_data;
2206     }
2207
2208     hns_nic_set_priv_ops(ndev);
2209
2210     ret = register_netdev(ndev);
2211     if (ret) {
2212         dev_err(priv->dev, "probe register netdev fail!\n");
2213         goto out_reg_ndev_fail;
2214     }
2215     return 0;
2216
2217 out_reg_ndev_fail:
2218     hns_nic_uninit_ring_data(priv);
2219     priv->ring_data = NULL;
2220 out_init_phy:
2221 out_init_ring_data:
2222     hnae_put_handle(priv->ae_handle);
2223     priv->ae_handle = NULL;
2224 out:
2225     return ret;
2226 }
2227
2228 static int hns_nic_notifier_action(struct notifier_block *nb,
2229                    unsigned long action, void *data)
2230 {
2231     struct hns_nic_priv *priv =
2232         container_of(nb, struct hns_nic_priv, notifier_block);
2233
2234     assert(action == HNAE_AE_REGISTER);
2235
2236     if (!hns_nic_try_get_ae(priv->netdev)) {
2237         hnae_unregister_notifier(&priv->notifier_block);
2238         priv->notifier_block.notifier_call = NULL;
2239     }
2240     return 0;
2241 }
2242
2243 static int hns_nic_dev_probe(struct platform_device *pdev)
2244 {
2245     struct device *dev = &pdev->dev;
2246     struct net_device *ndev;
2247     struct hns_nic_priv *priv;
2248     u32 port_id;
2249     int ret;
2250
2251     ndev = alloc_etherdev_mq(sizeof(struct hns_nic_priv), NIC_MAX_Q_PER_VF);
2252     if (!ndev)
2253         return -ENOMEM;
2254
2255     platform_set_drvdata(pdev, ndev);
2256
2257     priv = netdev_priv(ndev);
2258     priv->dev = dev;
2259     priv->netdev = ndev;
2260
2261     if (dev_of_node(dev)) {
2262         struct device_node *ae_node;
2263
2264         if (of_device_is_compatible(dev->of_node,
2265                         "hisilicon,hns-nic-v1"))
2266             priv->enet_ver = AE_VERSION_1;
2267         else
2268             priv->enet_ver = AE_VERSION_2;
2269
2270         ae_node = of_parse_phandle(dev->of_node, "ae-handle", 0);
2271         if (!ae_node) {
2272             ret = -ENODEV;
2273             dev_err(dev, "not find ae-handle\n");
2274             goto out_read_prop_fail;
2275         }
2276         priv->fwnode = &ae_node->fwnode;
2277     } else if (is_acpi_node(dev->fwnode)) {
2278         struct fwnode_reference_args args;
2279
2280         if (acpi_dev_found(hns_enet_acpi_match[0].id))
2281             priv->enet_ver = AE_VERSION_1;
2282         else if (acpi_dev_found(hns_enet_acpi_match[1].id))
2283             priv->enet_ver = AE_VERSION_2;
2284         else {
2285             ret = -ENXIO;
2286             goto out_read_prop_fail;
2287         }
2288
2289         /* try to find port-idx-in-ae first */
2290         ret = acpi_node_get_property_reference(dev->fwnode,
2291                                "ae-handle", 0, &args);
2292         if (ret) {
2293             dev_err(dev, "not find ae-handle\n");
2294             goto out_read_prop_fail;
2295         }
2296         if (!is_acpi_device_node(args.fwnode)) {
2297             ret = -EINVAL;
2298             goto out_read_prop_fail;
2299         }
2300         priv->fwnode = args.fwnode;
2301     } else {
2302         dev_err(dev, "cannot read cfg data from OF or acpi\n");
2303         ret = -ENXIO;
2304         goto out_read_prop_fail;
2305     }
2306
2307     ret = device_property_read_u32(dev, "port-idx-in-ae", &port_id);
2308     if (ret) {
2309         /* only for old code compatible */
2310         ret = device_property_read_u32(dev, "port-id", &port_id);
2311         if (ret)
2312             goto out_read_prop_fail;
2313         /* for old dts, we need to caculate the port offset */
2314         port_id = port_id < HNS_SRV_OFFSET ? port_id + HNS_DEBUG_OFFSET
2315             : port_id - HNS_SRV_OFFSET;
2316     }
2317     priv->port_id = port_id;
2318
2319     hns_init_mac_addr(ndev);
2320
2321     ndev->watchdog_timeo = HNS_NIC_TX_TIMEOUT;
2322     ndev->priv_flags |= IFF_UNICAST_FLT;
2323     ndev->netdev_ops = &hns_nic_netdev_ops;
2324     hns_ethtool_set_ops(ndev);
2325
2326     ndev->features |= NETIF_F_IP_CSUM | NETIF_F_IPV6_CSUM |
2327         NETIF_F_RXCSUM | NETIF_F_SG | NETIF_F_GSO |
2328         NETIF_F_GRO;
2329     ndev->vlan_features |=
2330         NETIF_F_IP_CSUM | NETIF_F_IPV6_CSUM | NETIF_F_RXCSUM;
2331     ndev->vlan_features |= NETIF_F_SG | NETIF_F_GSO | NETIF_F_GRO;
2332
2333     /* MTU range: 68 - 9578 (v1) or 9706 (v2) */
2334     ndev->min_mtu = MAC_MIN_MTU;
2335     switch (priv->enet_ver) {
2336     case AE_VERSION_2:
2337         ndev->features |= NETIF_F_TSO | NETIF_F_TSO6 | NETIF_F_NTUPLE;
2338         ndev->hw_features |= NETIF_F_IP_CSUM | NETIF_F_IPV6_CSUM |
2339             NETIF_F_RXCSUM | NETIF_F_SG | NETIF_F_GSO |
2340             NETIF_F_GRO | NETIF_F_TSO | NETIF_F_TSO6;
2341         ndev->vlan_features |= NETIF_F_TSO | NETIF_F_TSO6;
2342         ndev->max_mtu = MAC_MAX_MTU_V2 -
2343                 (ETH_HLEN + ETH_FCS_LEN + VLAN_HLEN);
2344         break;
2345     default:
2346         ndev->max_mtu = MAC_MAX_MTU -
2347                 (ETH_HLEN + ETH_FCS_LEN + VLAN_HLEN);
2348         break;
2349     }
2350
2351     SET_NETDEV_DEV(ndev, dev);
2352
2353     if (!dma_set_mask_and_coherent(dev, DMA_BIT_MASK(64)))
2354         dev_dbg(dev, "set mask to 64bit\n");
2355     else
2356         dev_err(dev, "set mask to 64bit fail!\n");
2357
2358     /* carrier off reporting is important to ethtool even BEFORE open */
2359     netif_carrier_off(ndev);
2360
2361     timer_setup(&priv->service_timer, hns_nic_service_timer, 0);
2362     INIT_WORK(&priv->service_task, hns_nic_service_task);
2363
2364     set_bit(NIC_STATE_SERVICE_INITED, &priv->state);
2365     clear_bit(NIC_STATE_SERVICE_SCHED, &priv->state);
2366     set_bit(NIC_STATE_DOWN, &priv->state);
2367
2368     if (hns_nic_try_get_ae(priv->netdev)) {
2369         priv->notifier_block.notifier_call = hns_nic_notifier_action;
2370         ret = hnae_register_notifier(&priv->notifier_block);
2371         if (ret) {
2372             dev_err(dev, "register notifier fail!\n");
2373             goto out_notify_fail;
2374         }
2375         dev_dbg(dev, "has not handle, register notifier!\n");
2376     }
2377
2378     return 0;
2379
2380 out_notify_fail:
2381     (void)cancel_work_sync(&priv->service_task);
2382 out_read_prop_fail:
2383     /* safe for ACPI FW */
2384     of_node_put(to_of_node(priv->fwnode));
2385     free_netdev(ndev);
2386     return ret;
2387 }
2388
2389 static int hns_nic_dev_remove(struct platform_device *pdev)
2390 {
2391     struct net_device *ndev = platform_get_drvdata(pdev);
2392     struct hns_nic_priv *priv = netdev_priv(ndev);
2393
2394     if (ndev->reg_state != NETREG_UNINITIALIZED)
2395         unregister_netdev(ndev);
2396
2397     if (priv->ring_data)
2398         hns_nic_uninit_ring_data(priv);
2399     priv->ring_data = NULL;
2400
2401     if (ndev->phydev)
2402         phy_disconnect(ndev->phydev);
2403
2404     if (!IS_ERR_OR_NULL(priv->ae_handle))
2405         hnae_put_handle(priv->ae_handle);
2406     priv->ae_handle = NULL;
2407     if (priv->notifier_block.notifier_call)
2408         hnae_unregister_notifier(&priv->notifier_block);
2409     priv->notifier_block.notifier_call = NULL;
2410
2411     set_bit(NIC_STATE_REMOVING, &priv->state);
2412     (void)cancel_work_sync(&priv->service_task);
2413
2414     /* safe for ACPI FW */
2415     of_node_put(to_of_node(priv->fwnode));
2416
2417     free_netdev(ndev);
2418     return 0;
2419 }
2420
2421 static const struct of_device_id hns_enet_of_match[] = {
2422     {.compatible = "hisilicon,hns-nic-v1",},
2423     {.compatible = "hisilicon,hns-nic-v2",},
2424     {},
2425 };
2426
2427 MODULE_DEVICE_TABLE(of, hns_enet_of_match);
2428
2429 static struct platform_driver hns_nic_dev_driver = {
2430     .driver = {
2431         .name = "hns-nic",
2432         .of_match_table = hns_enet_of_match,
2433         .acpi_match_table = ACPI_PTR(hns_enet_acpi_match),
2434     },
2435     .probe = hns_nic_dev_probe,
2436     .remove = hns_nic_dev_remove,
2437 };
2438
2439 module_platform_driver(hns_nic_dev_driver);
2440
2441 MODULE_DESCRIPTION("HISILICON HNS Ethernet driver");
2442 MODULE_AUTHOR("Hisilicon, Inc.");
2443 MODULE_LICENSE("GPL");
2444 MODULE_ALIAS("platform:hns-nic");