Back to home page

OSCL-LXR
 
 

     

0001 // SPDX-License-Identifier: GPL-2.0 OR BSD-3-Clause
0002 /* Copyright(c) 2018-2019  Realtek Corporation
0003  */
0004 
0005 #include <linux/module.h>
0006 #include <linux/pci.h>
0007 #include "main.h"
0008 #include "pci.h"
0009 #include "reg.h"
0010 #include "tx.h"
0011 #include "rx.h"
0012 #include "fw.h"
0013 #include "ps.h"
0014 #include "debug.h"
0015 
0016 static bool rtw_disable_msi;
0017 static bool rtw_pci_disable_aspm;
0018 module_param_named(disable_msi, rtw_disable_msi, bool, 0644);
0019 module_param_named(disable_aspm, rtw_pci_disable_aspm, bool, 0644);
0020 MODULE_PARM_DESC(disable_msi, "Set Y to disable MSI interrupt support");
0021 MODULE_PARM_DESC(disable_aspm, "Set Y to disable PCI ASPM support");
0022 
0023 static u32 rtw_pci_tx_queue_idx_addr[] = {
0024     [RTW_TX_QUEUE_BK]   = RTK_PCI_TXBD_IDX_BKQ,
0025     [RTW_TX_QUEUE_BE]   = RTK_PCI_TXBD_IDX_BEQ,
0026     [RTW_TX_QUEUE_VI]   = RTK_PCI_TXBD_IDX_VIQ,
0027     [RTW_TX_QUEUE_VO]   = RTK_PCI_TXBD_IDX_VOQ,
0028     [RTW_TX_QUEUE_MGMT] = RTK_PCI_TXBD_IDX_MGMTQ,
0029     [RTW_TX_QUEUE_HI0]  = RTK_PCI_TXBD_IDX_HI0Q,
0030     [RTW_TX_QUEUE_H2C]  = RTK_PCI_TXBD_IDX_H2CQ,
0031 };
0032 
0033 static u8 rtw_pci_get_tx_qsel(struct sk_buff *skb, u8 queue)
0034 {
0035     switch (queue) {
0036     case RTW_TX_QUEUE_BCN:
0037         return TX_DESC_QSEL_BEACON;
0038     case RTW_TX_QUEUE_H2C:
0039         return TX_DESC_QSEL_H2C;
0040     case RTW_TX_QUEUE_MGMT:
0041         return TX_DESC_QSEL_MGMT;
0042     case RTW_TX_QUEUE_HI0:
0043         return TX_DESC_QSEL_HIGH;
0044     default:
0045         return skb->priority;
0046     }
0047 };
0048 
0049 static u8 rtw_pci_read8(struct rtw_dev *rtwdev, u32 addr)
0050 {
0051     struct rtw_pci *rtwpci = (struct rtw_pci *)rtwdev->priv;
0052 
0053     return readb(rtwpci->mmap + addr);
0054 }
0055 
0056 static u16 rtw_pci_read16(struct rtw_dev *rtwdev, u32 addr)
0057 {
0058     struct rtw_pci *rtwpci = (struct rtw_pci *)rtwdev->priv;
0059 
0060     return readw(rtwpci->mmap + addr);
0061 }
0062 
0063 static u32 rtw_pci_read32(struct rtw_dev *rtwdev, u32 addr)
0064 {
0065     struct rtw_pci *rtwpci = (struct rtw_pci *)rtwdev->priv;
0066 
0067     return readl(rtwpci->mmap + addr);
0068 }
0069 
0070 static void rtw_pci_write8(struct rtw_dev *rtwdev, u32 addr, u8 val)
0071 {
0072     struct rtw_pci *rtwpci = (struct rtw_pci *)rtwdev->priv;
0073 
0074     writeb(val, rtwpci->mmap + addr);
0075 }
0076 
0077 static void rtw_pci_write16(struct rtw_dev *rtwdev, u32 addr, u16 val)
0078 {
0079     struct rtw_pci *rtwpci = (struct rtw_pci *)rtwdev->priv;
0080 
0081     writew(val, rtwpci->mmap + addr);
0082 }
0083 
0084 static void rtw_pci_write32(struct rtw_dev *rtwdev, u32 addr, u32 val)
0085 {
0086     struct rtw_pci *rtwpci = (struct rtw_pci *)rtwdev->priv;
0087 
0088     writel(val, rtwpci->mmap + addr);
0089 }
0090 
0091 static inline void *rtw_pci_get_tx_desc(struct rtw_pci_tx_ring *tx_ring, u8 idx)
0092 {
0093     int offset = tx_ring->r.desc_size * idx;
0094 
0095     return tx_ring->r.head + offset;
0096 }
0097 
0098 static void rtw_pci_free_tx_ring_skbs(struct rtw_dev *rtwdev,
0099                       struct rtw_pci_tx_ring *tx_ring)
0100 {
0101     struct pci_dev *pdev = to_pci_dev(rtwdev->dev);
0102     struct rtw_pci_tx_data *tx_data;
0103     struct sk_buff *skb, *tmp;
0104     dma_addr_t dma;
0105 
0106     /* free every skb remained in tx list */
0107     skb_queue_walk_safe(&tx_ring->queue, skb, tmp) {
0108         __skb_unlink(skb, &tx_ring->queue);
0109         tx_data = rtw_pci_get_tx_data(skb);
0110         dma = tx_data->dma;
0111 
0112         dma_unmap_single(&pdev->dev, dma, skb->len, DMA_TO_DEVICE);
0113         dev_kfree_skb_any(skb);
0114     }
0115 }
0116 
0117 static void rtw_pci_free_tx_ring(struct rtw_dev *rtwdev,
0118                  struct rtw_pci_tx_ring *tx_ring)
0119 {
0120     struct pci_dev *pdev = to_pci_dev(rtwdev->dev);
0121     u8 *head = tx_ring->r.head;
0122     u32 len = tx_ring->r.len;
0123     int ring_sz = len * tx_ring->r.desc_size;
0124 
0125     rtw_pci_free_tx_ring_skbs(rtwdev, tx_ring);
0126 
0127     /* free the ring itself */
0128     dma_free_coherent(&pdev->dev, ring_sz, head, tx_ring->r.dma);
0129     tx_ring->r.head = NULL;
0130 }
0131 
0132 static void rtw_pci_free_rx_ring_skbs(struct rtw_dev *rtwdev,
0133                       struct rtw_pci_rx_ring *rx_ring)
0134 {
0135     struct pci_dev *pdev = to_pci_dev(rtwdev->dev);
0136     struct sk_buff *skb;
0137     int buf_sz = RTK_PCI_RX_BUF_SIZE;
0138     dma_addr_t dma;
0139     int i;
0140 
0141     for (i = 0; i < rx_ring->r.len; i++) {
0142         skb = rx_ring->buf[i];
0143         if (!skb)
0144             continue;
0145 
0146         dma = *((dma_addr_t *)skb->cb);
0147         dma_unmap_single(&pdev->dev, dma, buf_sz, DMA_FROM_DEVICE);
0148         dev_kfree_skb(skb);
0149         rx_ring->buf[i] = NULL;
0150     }
0151 }
0152 
0153 static void rtw_pci_free_rx_ring(struct rtw_dev *rtwdev,
0154                  struct rtw_pci_rx_ring *rx_ring)
0155 {
0156     struct pci_dev *pdev = to_pci_dev(rtwdev->dev);
0157     u8 *head = rx_ring->r.head;
0158     int ring_sz = rx_ring->r.desc_size * rx_ring->r.len;
0159 
0160     rtw_pci_free_rx_ring_skbs(rtwdev, rx_ring);
0161 
0162     dma_free_coherent(&pdev->dev, ring_sz, head, rx_ring->r.dma);
0163 }
0164 
0165 static void rtw_pci_free_trx_ring(struct rtw_dev *rtwdev)
0166 {
0167     struct rtw_pci *rtwpci = (struct rtw_pci *)rtwdev->priv;
0168     struct rtw_pci_tx_ring *tx_ring;
0169     struct rtw_pci_rx_ring *rx_ring;
0170     int i;
0171 
0172     for (i = 0; i < RTK_MAX_TX_QUEUE_NUM; i++) {
0173         tx_ring = &rtwpci->tx_rings[i];
0174         rtw_pci_free_tx_ring(rtwdev, tx_ring);
0175     }
0176 
0177     for (i = 0; i < RTK_MAX_RX_QUEUE_NUM; i++) {
0178         rx_ring = &rtwpci->rx_rings[i];
0179         rtw_pci_free_rx_ring(rtwdev, rx_ring);
0180     }
0181 }
0182 
0183 static int rtw_pci_init_tx_ring(struct rtw_dev *rtwdev,
0184                 struct rtw_pci_tx_ring *tx_ring,
0185                 u8 desc_size, u32 len)
0186 {
0187     struct pci_dev *pdev = to_pci_dev(rtwdev->dev);
0188     int ring_sz = desc_size * len;
0189     dma_addr_t dma;
0190     u8 *head;
0191 
0192     if (len > TRX_BD_IDX_MASK) {
0193         rtw_err(rtwdev, "len %d exceeds maximum TX entries\n", len);
0194         return -EINVAL;
0195     }
0196 
0197     head = dma_alloc_coherent(&pdev->dev, ring_sz, &dma, GFP_KERNEL);
0198     if (!head) {
0199         rtw_err(rtwdev, "failed to allocate tx ring\n");
0200         return -ENOMEM;
0201     }
0202 
0203     skb_queue_head_init(&tx_ring->queue);
0204     tx_ring->r.head = head;
0205     tx_ring->r.dma = dma;
0206     tx_ring->r.len = len;
0207     tx_ring->r.desc_size = desc_size;
0208     tx_ring->r.wp = 0;
0209     tx_ring->r.rp = 0;
0210 
0211     return 0;
0212 }
0213 
0214 static int rtw_pci_reset_rx_desc(struct rtw_dev *rtwdev, struct sk_buff *skb,
0215                  struct rtw_pci_rx_ring *rx_ring,
0216                  u32 idx, u32 desc_sz)
0217 {
0218     struct pci_dev *pdev = to_pci_dev(rtwdev->dev);
0219     struct rtw_pci_rx_buffer_desc *buf_desc;
0220     int buf_sz = RTK_PCI_RX_BUF_SIZE;
0221     dma_addr_t dma;
0222 
0223     if (!skb)
0224         return -EINVAL;
0225 
0226     dma = dma_map_single(&pdev->dev, skb->data, buf_sz, DMA_FROM_DEVICE);
0227     if (dma_mapping_error(&pdev->dev, dma))
0228         return -EBUSY;
0229 
0230     *((dma_addr_t *)skb->cb) = dma;
0231     buf_desc = (struct rtw_pci_rx_buffer_desc *)(rx_ring->r.head +
0232                              idx * desc_sz);
0233     memset(buf_desc, 0, sizeof(*buf_desc));
0234     buf_desc->buf_size = cpu_to_le16(RTK_PCI_RX_BUF_SIZE);
0235     buf_desc->dma = cpu_to_le32(dma);
0236 
0237     return 0;
0238 }
0239 
0240 static void rtw_pci_sync_rx_desc_device(struct rtw_dev *rtwdev, dma_addr_t dma,
0241                     struct rtw_pci_rx_ring *rx_ring,
0242                     u32 idx, u32 desc_sz)
0243 {
0244     struct device *dev = rtwdev->dev;
0245     struct rtw_pci_rx_buffer_desc *buf_desc;
0246     int buf_sz = RTK_PCI_RX_BUF_SIZE;
0247 
0248     dma_sync_single_for_device(dev, dma, buf_sz, DMA_FROM_DEVICE);
0249 
0250     buf_desc = (struct rtw_pci_rx_buffer_desc *)(rx_ring->r.head +
0251                              idx * desc_sz);
0252     memset(buf_desc, 0, sizeof(*buf_desc));
0253     buf_desc->buf_size = cpu_to_le16(RTK_PCI_RX_BUF_SIZE);
0254     buf_desc->dma = cpu_to_le32(dma);
0255 }
0256 
0257 static int rtw_pci_init_rx_ring(struct rtw_dev *rtwdev,
0258                 struct rtw_pci_rx_ring *rx_ring,
0259                 u8 desc_size, u32 len)
0260 {
0261     struct pci_dev *pdev = to_pci_dev(rtwdev->dev);
0262     struct sk_buff *skb = NULL;
0263     dma_addr_t dma;
0264     u8 *head;
0265     int ring_sz = desc_size * len;
0266     int buf_sz = RTK_PCI_RX_BUF_SIZE;
0267     int i, allocated;
0268     int ret = 0;
0269 
0270     head = dma_alloc_coherent(&pdev->dev, ring_sz, &dma, GFP_KERNEL);
0271     if (!head) {
0272         rtw_err(rtwdev, "failed to allocate rx ring\n");
0273         return -ENOMEM;
0274     }
0275     rx_ring->r.head = head;
0276 
0277     for (i = 0; i < len; i++) {
0278         skb = dev_alloc_skb(buf_sz);
0279         if (!skb) {
0280             allocated = i;
0281             ret = -ENOMEM;
0282             goto err_out;
0283         }
0284 
0285         memset(skb->data, 0, buf_sz);
0286         rx_ring->buf[i] = skb;
0287         ret = rtw_pci_reset_rx_desc(rtwdev, skb, rx_ring, i, desc_size);
0288         if (ret) {
0289             allocated = i;
0290             dev_kfree_skb_any(skb);
0291             goto err_out;
0292         }
0293     }
0294 
0295     rx_ring->r.dma = dma;
0296     rx_ring->r.len = len;
0297     rx_ring->r.desc_size = desc_size;
0298     rx_ring->r.wp = 0;
0299     rx_ring->r.rp = 0;
0300 
0301     return 0;
0302 
0303 err_out:
0304     for (i = 0; i < allocated; i++) {
0305         skb = rx_ring->buf[i];
0306         if (!skb)
0307             continue;
0308         dma = *((dma_addr_t *)skb->cb);
0309         dma_unmap_single(&pdev->dev, dma, buf_sz, DMA_FROM_DEVICE);
0310         dev_kfree_skb_any(skb);
0311         rx_ring->buf[i] = NULL;
0312     }
0313     dma_free_coherent(&pdev->dev, ring_sz, head, dma);
0314 
0315     rtw_err(rtwdev, "failed to init rx buffer\n");
0316 
0317     return ret;
0318 }
0319 
0320 static int rtw_pci_init_trx_ring(struct rtw_dev *rtwdev)
0321 {
0322     struct rtw_pci *rtwpci = (struct rtw_pci *)rtwdev->priv;
0323     struct rtw_pci_tx_ring *tx_ring;
0324     struct rtw_pci_rx_ring *rx_ring;
0325     struct rtw_chip_info *chip = rtwdev->chip;
0326     int i = 0, j = 0, tx_alloced = 0, rx_alloced = 0;
0327     int tx_desc_size, rx_desc_size;
0328     u32 len;
0329     int ret;
0330 
0331     tx_desc_size = chip->tx_buf_desc_sz;
0332 
0333     for (i = 0; i < RTK_MAX_TX_QUEUE_NUM; i++) {
0334         tx_ring = &rtwpci->tx_rings[i];
0335         len = max_num_of_tx_queue(i);
0336         ret = rtw_pci_init_tx_ring(rtwdev, tx_ring, tx_desc_size, len);
0337         if (ret)
0338             goto out;
0339     }
0340 
0341     rx_desc_size = chip->rx_buf_desc_sz;
0342 
0343     for (j = 0; j < RTK_MAX_RX_QUEUE_NUM; j++) {
0344         rx_ring = &rtwpci->rx_rings[j];
0345         ret = rtw_pci_init_rx_ring(rtwdev, rx_ring, rx_desc_size,
0346                        RTK_MAX_RX_DESC_NUM);
0347         if (ret)
0348             goto out;
0349     }
0350 
0351     return 0;
0352 
0353 out:
0354     tx_alloced = i;
0355     for (i = 0; i < tx_alloced; i++) {
0356         tx_ring = &rtwpci->tx_rings[i];
0357         rtw_pci_free_tx_ring(rtwdev, tx_ring);
0358     }
0359 
0360     rx_alloced = j;
0361     for (j = 0; j < rx_alloced; j++) {
0362         rx_ring = &rtwpci->rx_rings[j];
0363         rtw_pci_free_rx_ring(rtwdev, rx_ring);
0364     }
0365 
0366     return ret;
0367 }
0368 
0369 static void rtw_pci_deinit(struct rtw_dev *rtwdev)
0370 {
0371     rtw_pci_free_trx_ring(rtwdev);
0372 }
0373 
0374 static int rtw_pci_init(struct rtw_dev *rtwdev)
0375 {
0376     struct rtw_pci *rtwpci = (struct rtw_pci *)rtwdev->priv;
0377     int ret = 0;
0378 
0379     rtwpci->irq_mask[0] = IMR_HIGHDOK |
0380                   IMR_MGNTDOK |
0381                   IMR_BKDOK |
0382                   IMR_BEDOK |
0383                   IMR_VIDOK |
0384                   IMR_VODOK |
0385                   IMR_ROK |
0386                   IMR_BCNDMAINT_E |
0387                   IMR_C2HCMD |
0388                   0;
0389     rtwpci->irq_mask[1] = IMR_TXFOVW |
0390                   0;
0391     rtwpci->irq_mask[3] = IMR_H2CDOK |
0392                   0;
0393     spin_lock_init(&rtwpci->irq_lock);
0394     spin_lock_init(&rtwpci->hwirq_lock);
0395     ret = rtw_pci_init_trx_ring(rtwdev);
0396 
0397     return ret;
0398 }
0399 
0400 static void rtw_pci_reset_buf_desc(struct rtw_dev *rtwdev)
0401 {
0402     struct rtw_pci *rtwpci = (struct rtw_pci *)rtwdev->priv;
0403     u32 len;
0404     u8 tmp;
0405     dma_addr_t dma;
0406 
0407     tmp = rtw_read8(rtwdev, RTK_PCI_CTRL + 3);
0408     rtw_write8(rtwdev, RTK_PCI_CTRL + 3, tmp | 0xf7);
0409 
0410     dma = rtwpci->tx_rings[RTW_TX_QUEUE_BCN].r.dma;
0411     rtw_write32(rtwdev, RTK_PCI_TXBD_DESA_BCNQ, dma);
0412 
0413     if (!rtw_chip_wcpu_11n(rtwdev)) {
0414         len = rtwpci->tx_rings[RTW_TX_QUEUE_H2C].r.len;
0415         dma = rtwpci->tx_rings[RTW_TX_QUEUE_H2C].r.dma;
0416         rtwpci->tx_rings[RTW_TX_QUEUE_H2C].r.rp = 0;
0417         rtwpci->tx_rings[RTW_TX_QUEUE_H2C].r.wp = 0;
0418         rtw_write16(rtwdev, RTK_PCI_TXBD_NUM_H2CQ, len & TRX_BD_IDX_MASK);
0419         rtw_write32(rtwdev, RTK_PCI_TXBD_DESA_H2CQ, dma);
0420     }
0421 
0422     len = rtwpci->tx_rings[RTW_TX_QUEUE_BK].r.len;
0423     dma = rtwpci->tx_rings[RTW_TX_QUEUE_BK].r.dma;
0424     rtwpci->tx_rings[RTW_TX_QUEUE_BK].r.rp = 0;
0425     rtwpci->tx_rings[RTW_TX_QUEUE_BK].r.wp = 0;
0426     rtw_write16(rtwdev, RTK_PCI_TXBD_NUM_BKQ, len & TRX_BD_IDX_MASK);
0427     rtw_write32(rtwdev, RTK_PCI_TXBD_DESA_BKQ, dma);
0428 
0429     len = rtwpci->tx_rings[RTW_TX_QUEUE_BE].r.len;
0430     dma = rtwpci->tx_rings[RTW_TX_QUEUE_BE].r.dma;
0431     rtwpci->tx_rings[RTW_TX_QUEUE_BE].r.rp = 0;
0432     rtwpci->tx_rings[RTW_TX_QUEUE_BE].r.wp = 0;
0433     rtw_write16(rtwdev, RTK_PCI_TXBD_NUM_BEQ, len & TRX_BD_IDX_MASK);
0434     rtw_write32(rtwdev, RTK_PCI_TXBD_DESA_BEQ, dma);
0435 
0436     len = rtwpci->tx_rings[RTW_TX_QUEUE_VO].r.len;
0437     dma = rtwpci->tx_rings[RTW_TX_QUEUE_VO].r.dma;
0438     rtwpci->tx_rings[RTW_TX_QUEUE_VO].r.rp = 0;
0439     rtwpci->tx_rings[RTW_TX_QUEUE_VO].r.wp = 0;
0440     rtw_write16(rtwdev, RTK_PCI_TXBD_NUM_VOQ, len & TRX_BD_IDX_MASK);
0441     rtw_write32(rtwdev, RTK_PCI_TXBD_DESA_VOQ, dma);
0442 
0443     len = rtwpci->tx_rings[RTW_TX_QUEUE_VI].r.len;
0444     dma = rtwpci->tx_rings[RTW_TX_QUEUE_VI].r.dma;
0445     rtwpci->tx_rings[RTW_TX_QUEUE_VI].r.rp = 0;
0446     rtwpci->tx_rings[RTW_TX_QUEUE_VI].r.wp = 0;
0447     rtw_write16(rtwdev, RTK_PCI_TXBD_NUM_VIQ, len & TRX_BD_IDX_MASK);
0448     rtw_write32(rtwdev, RTK_PCI_TXBD_DESA_VIQ, dma);
0449 
0450     len = rtwpci->tx_rings[RTW_TX_QUEUE_MGMT].r.len;
0451     dma = rtwpci->tx_rings[RTW_TX_QUEUE_MGMT].r.dma;
0452     rtwpci->tx_rings[RTW_TX_QUEUE_MGMT].r.rp = 0;
0453     rtwpci->tx_rings[RTW_TX_QUEUE_MGMT].r.wp = 0;
0454     rtw_write16(rtwdev, RTK_PCI_TXBD_NUM_MGMTQ, len & TRX_BD_IDX_MASK);
0455     rtw_write32(rtwdev, RTK_PCI_TXBD_DESA_MGMTQ, dma);
0456 
0457     len = rtwpci->tx_rings[RTW_TX_QUEUE_HI0].r.len;
0458     dma = rtwpci->tx_rings[RTW_TX_QUEUE_HI0].r.dma;
0459     rtwpci->tx_rings[RTW_TX_QUEUE_HI0].r.rp = 0;
0460     rtwpci->tx_rings[RTW_TX_QUEUE_HI0].r.wp = 0;
0461     rtw_write16(rtwdev, RTK_PCI_TXBD_NUM_HI0Q, len & TRX_BD_IDX_MASK);
0462     rtw_write32(rtwdev, RTK_PCI_TXBD_DESA_HI0Q, dma);
0463 
0464     len = rtwpci->rx_rings[RTW_RX_QUEUE_MPDU].r.len;
0465     dma = rtwpci->rx_rings[RTW_RX_QUEUE_MPDU].r.dma;
0466     rtwpci->rx_rings[RTW_RX_QUEUE_MPDU].r.rp = 0;
0467     rtwpci->rx_rings[RTW_RX_QUEUE_MPDU].r.wp = 0;
0468     rtw_write16(rtwdev, RTK_PCI_RXBD_NUM_MPDUQ, len & TRX_BD_IDX_MASK);
0469     rtw_write32(rtwdev, RTK_PCI_RXBD_DESA_MPDUQ, dma);
0470 
0471     /* reset read/write point */
0472     rtw_write32(rtwdev, RTK_PCI_TXBD_RWPTR_CLR, 0xffffffff);
0473 
0474     /* reset H2C Queue index in a single write */
0475     if (rtw_chip_wcpu_11ac(rtwdev))
0476         rtw_write32_set(rtwdev, RTK_PCI_TXBD_H2CQ_CSR,
0477                 BIT_CLR_H2CQ_HOST_IDX | BIT_CLR_H2CQ_HW_IDX);
0478 }
0479 
0480 static void rtw_pci_reset_trx_ring(struct rtw_dev *rtwdev)
0481 {
0482     rtw_pci_reset_buf_desc(rtwdev);
0483 }
0484 
0485 static void rtw_pci_enable_interrupt(struct rtw_dev *rtwdev,
0486                      struct rtw_pci *rtwpci, bool exclude_rx)
0487 {
0488     unsigned long flags;
0489     u32 imr0_unmask = exclude_rx ? IMR_ROK : 0;
0490 
0491     spin_lock_irqsave(&rtwpci->hwirq_lock, flags);
0492 
0493     rtw_write32(rtwdev, RTK_PCI_HIMR0, rtwpci->irq_mask[0] & ~imr0_unmask);
0494     rtw_write32(rtwdev, RTK_PCI_HIMR1, rtwpci->irq_mask[1]);
0495     if (rtw_chip_wcpu_11ac(rtwdev))
0496         rtw_write32(rtwdev, RTK_PCI_HIMR3, rtwpci->irq_mask[3]);
0497 
0498     rtwpci->irq_enabled = true;
0499 
0500     spin_unlock_irqrestore(&rtwpci->hwirq_lock, flags);
0501 }
0502 
0503 static void rtw_pci_disable_interrupt(struct rtw_dev *rtwdev,
0504                       struct rtw_pci *rtwpci)
0505 {
0506     unsigned long flags;
0507 
0508     spin_lock_irqsave(&rtwpci->hwirq_lock, flags);
0509 
0510     if (!rtwpci->irq_enabled)
0511         goto out;
0512 
0513     rtw_write32(rtwdev, RTK_PCI_HIMR0, 0);
0514     rtw_write32(rtwdev, RTK_PCI_HIMR1, 0);
0515     if (rtw_chip_wcpu_11ac(rtwdev))
0516         rtw_write32(rtwdev, RTK_PCI_HIMR3, 0);
0517 
0518     rtwpci->irq_enabled = false;
0519 
0520 out:
0521     spin_unlock_irqrestore(&rtwpci->hwirq_lock, flags);
0522 }
0523 
0524 static void rtw_pci_dma_reset(struct rtw_dev *rtwdev, struct rtw_pci *rtwpci)
0525 {
0526     /* reset dma and rx tag */
0527     rtw_write32_set(rtwdev, RTK_PCI_CTRL,
0528             BIT_RST_TRXDMA_INTF | BIT_RX_TAG_EN);
0529     rtwpci->rx_tag = 0;
0530 }
0531 
0532 static int rtw_pci_setup(struct rtw_dev *rtwdev)
0533 {
0534     struct rtw_pci *rtwpci = (struct rtw_pci *)rtwdev->priv;
0535 
0536     rtw_pci_reset_trx_ring(rtwdev);
0537     rtw_pci_dma_reset(rtwdev, rtwpci);
0538 
0539     return 0;
0540 }
0541 
0542 static void rtw_pci_dma_release(struct rtw_dev *rtwdev, struct rtw_pci *rtwpci)
0543 {
0544     struct rtw_pci_tx_ring *tx_ring;
0545     u8 queue;
0546 
0547     rtw_pci_reset_trx_ring(rtwdev);
0548     for (queue = 0; queue < RTK_MAX_TX_QUEUE_NUM; queue++) {
0549         tx_ring = &rtwpci->tx_rings[queue];
0550         rtw_pci_free_tx_ring_skbs(rtwdev, tx_ring);
0551     }
0552 }
0553 
0554 static void rtw_pci_napi_start(struct rtw_dev *rtwdev)
0555 {
0556     struct rtw_pci *rtwpci = (struct rtw_pci *)rtwdev->priv;
0557 
0558     if (test_and_set_bit(RTW_PCI_FLAG_NAPI_RUNNING, rtwpci->flags))
0559         return;
0560 
0561     napi_enable(&rtwpci->napi);
0562 }
0563 
0564 static void rtw_pci_napi_stop(struct rtw_dev *rtwdev)
0565 {
0566     struct rtw_pci *rtwpci = (struct rtw_pci *)rtwdev->priv;
0567 
0568     if (!test_and_clear_bit(RTW_PCI_FLAG_NAPI_RUNNING, rtwpci->flags))
0569         return;
0570 
0571     napi_synchronize(&rtwpci->napi);
0572     napi_disable(&rtwpci->napi);
0573 }
0574 
0575 static int rtw_pci_start(struct rtw_dev *rtwdev)
0576 {
0577     struct rtw_pci *rtwpci = (struct rtw_pci *)rtwdev->priv;
0578 
0579     rtw_pci_napi_start(rtwdev);
0580 
0581     spin_lock_bh(&rtwpci->irq_lock);
0582     rtwpci->running = true;
0583     rtw_pci_enable_interrupt(rtwdev, rtwpci, false);
0584     spin_unlock_bh(&rtwpci->irq_lock);
0585 
0586     return 0;
0587 }
0588 
0589 static void rtw_pci_stop(struct rtw_dev *rtwdev)
0590 {
0591     struct rtw_pci *rtwpci = (struct rtw_pci *)rtwdev->priv;
0592     struct pci_dev *pdev = rtwpci->pdev;
0593 
0594     spin_lock_bh(&rtwpci->irq_lock);
0595     rtwpci->running = false;
0596     rtw_pci_disable_interrupt(rtwdev, rtwpci);
0597     spin_unlock_bh(&rtwpci->irq_lock);
0598 
0599     synchronize_irq(pdev->irq);
0600     rtw_pci_napi_stop(rtwdev);
0601 
0602     spin_lock_bh(&rtwpci->irq_lock);
0603     rtw_pci_dma_release(rtwdev, rtwpci);
0604     spin_unlock_bh(&rtwpci->irq_lock);
0605 }
0606 
0607 static void rtw_pci_deep_ps_enter(struct rtw_dev *rtwdev)
0608 {
0609     struct rtw_pci *rtwpci = (struct rtw_pci *)rtwdev->priv;
0610     struct rtw_pci_tx_ring *tx_ring;
0611     bool tx_empty = true;
0612     u8 queue;
0613 
0614     if (rtw_fw_feature_check(&rtwdev->fw, FW_FEATURE_TX_WAKE))
0615         goto enter_deep_ps;
0616 
0617     lockdep_assert_held(&rtwpci->irq_lock);
0618 
0619     /* Deep PS state is not allowed to TX-DMA */
0620     for (queue = 0; queue < RTK_MAX_TX_QUEUE_NUM; queue++) {
0621         /* BCN queue is rsvd page, does not have DMA interrupt
0622          * H2C queue is managed by firmware
0623          */
0624         if (queue == RTW_TX_QUEUE_BCN ||
0625             queue == RTW_TX_QUEUE_H2C)
0626             continue;
0627 
0628         tx_ring = &rtwpci->tx_rings[queue];
0629 
0630         /* check if there is any skb DMAing */
0631         if (skb_queue_len(&tx_ring->queue)) {
0632             tx_empty = false;
0633             break;
0634         }
0635     }
0636 
0637     if (!tx_empty) {
0638         rtw_dbg(rtwdev, RTW_DBG_PS,
0639             "TX path not empty, cannot enter deep power save state\n");
0640         return;
0641     }
0642 enter_deep_ps:
0643     set_bit(RTW_FLAG_LEISURE_PS_DEEP, rtwdev->flags);
0644     rtw_power_mode_change(rtwdev, true);
0645 }
0646 
0647 static void rtw_pci_deep_ps_leave(struct rtw_dev *rtwdev)
0648 {
0649     struct rtw_pci *rtwpci = (struct rtw_pci *)rtwdev->priv;
0650 
0651     lockdep_assert_held(&rtwpci->irq_lock);
0652 
0653     if (test_and_clear_bit(RTW_FLAG_LEISURE_PS_DEEP, rtwdev->flags))
0654         rtw_power_mode_change(rtwdev, false);
0655 }
0656 
0657 static void rtw_pci_deep_ps(struct rtw_dev *rtwdev, bool enter)
0658 {
0659     struct rtw_pci *rtwpci = (struct rtw_pci *)rtwdev->priv;
0660 
0661     spin_lock_bh(&rtwpci->irq_lock);
0662 
0663     if (enter && !test_bit(RTW_FLAG_LEISURE_PS_DEEP, rtwdev->flags))
0664         rtw_pci_deep_ps_enter(rtwdev);
0665 
0666     if (!enter && test_bit(RTW_FLAG_LEISURE_PS_DEEP, rtwdev->flags))
0667         rtw_pci_deep_ps_leave(rtwdev);
0668 
0669     spin_unlock_bh(&rtwpci->irq_lock);
0670 }
0671 
0672 static u8 ac_to_hwq[] = {
0673     [IEEE80211_AC_VO] = RTW_TX_QUEUE_VO,
0674     [IEEE80211_AC_VI] = RTW_TX_QUEUE_VI,
0675     [IEEE80211_AC_BE] = RTW_TX_QUEUE_BE,
0676     [IEEE80211_AC_BK] = RTW_TX_QUEUE_BK,
0677 };
0678 
0679 static_assert(ARRAY_SIZE(ac_to_hwq) == IEEE80211_NUM_ACS);
0680 
0681 static u8 rtw_hw_queue_mapping(struct sk_buff *skb)
0682 {
0683     struct ieee80211_hdr *hdr = (struct ieee80211_hdr *)skb->data;
0684     __le16 fc = hdr->frame_control;
0685     u8 q_mapping = skb_get_queue_mapping(skb);
0686     u8 queue;
0687 
0688     if (unlikely(ieee80211_is_beacon(fc)))
0689         queue = RTW_TX_QUEUE_BCN;
0690     else if (unlikely(ieee80211_is_mgmt(fc) || ieee80211_is_ctl(fc)))
0691         queue = RTW_TX_QUEUE_MGMT;
0692     else if (is_broadcast_ether_addr(hdr->addr1) ||
0693          is_multicast_ether_addr(hdr->addr1))
0694         queue = RTW_TX_QUEUE_HI0;
0695     else if (WARN_ON_ONCE(q_mapping >= ARRAY_SIZE(ac_to_hwq)))
0696         queue = ac_to_hwq[IEEE80211_AC_BE];
0697     else
0698         queue = ac_to_hwq[q_mapping];
0699 
0700     return queue;
0701 }
0702 
0703 static void rtw_pci_release_rsvd_page(struct rtw_pci *rtwpci,
0704                       struct rtw_pci_tx_ring *ring)
0705 {
0706     struct sk_buff *prev = skb_dequeue(&ring->queue);
0707     struct rtw_pci_tx_data *tx_data;
0708     dma_addr_t dma;
0709 
0710     if (!prev)
0711         return;
0712 
0713     tx_data = rtw_pci_get_tx_data(prev);
0714     dma = tx_data->dma;
0715     dma_unmap_single(&rtwpci->pdev->dev, dma, prev->len, DMA_TO_DEVICE);
0716     dev_kfree_skb_any(prev);
0717 }
0718 
0719 static void rtw_pci_dma_check(struct rtw_dev *rtwdev,
0720                   struct rtw_pci_rx_ring *rx_ring,
0721                   u32 idx)
0722 {
0723     struct rtw_pci *rtwpci = (struct rtw_pci *)rtwdev->priv;
0724     struct rtw_chip_info *chip = rtwdev->chip;
0725     struct rtw_pci_rx_buffer_desc *buf_desc;
0726     u32 desc_sz = chip->rx_buf_desc_sz;
0727     u16 total_pkt_size;
0728 
0729     buf_desc = (struct rtw_pci_rx_buffer_desc *)(rx_ring->r.head +
0730                              idx * desc_sz);
0731     total_pkt_size = le16_to_cpu(buf_desc->total_pkt_size);
0732 
0733     /* rx tag mismatch, throw a warning */
0734     if (total_pkt_size != rtwpci->rx_tag)
0735         rtw_warn(rtwdev, "pci bus timeout, check dma status\n");
0736 
0737     rtwpci->rx_tag = (rtwpci->rx_tag + 1) % RX_TAG_MAX;
0738 }
0739 
0740 static u32 __pci_get_hw_tx_ring_rp(struct rtw_dev *rtwdev, u8 pci_q)
0741 {
0742     u32 bd_idx_addr = rtw_pci_tx_queue_idx_addr[pci_q];
0743     u32 bd_idx = rtw_read16(rtwdev, bd_idx_addr + 2);
0744 
0745     return FIELD_GET(TRX_BD_IDX_MASK, bd_idx);
0746 }
0747 
0748 static void __pci_flush_queue(struct rtw_dev *rtwdev, u8 pci_q, bool drop)
0749 {
0750     struct rtw_pci *rtwpci = (struct rtw_pci *)rtwdev->priv;
0751     struct rtw_pci_tx_ring *ring = &rtwpci->tx_rings[pci_q];
0752     u32 cur_rp;
0753     u8 i;
0754 
0755     /* Because the time taked by the I/O in __pci_get_hw_tx_ring_rp is a
0756      * bit dynamic, it's hard to define a reasonable fixed total timeout to
0757      * use read_poll_timeout* helper. Instead, we can ensure a reasonable
0758      * polling times, so we just use for loop with udelay here.
0759      */
0760     for (i = 0; i < 30; i++) {
0761         cur_rp = __pci_get_hw_tx_ring_rp(rtwdev, pci_q);
0762         if (cur_rp == ring->r.wp)
0763             return;
0764 
0765         udelay(1);
0766     }
0767 
0768     if (!drop)
0769         rtw_warn(rtwdev, "timed out to flush pci tx ring[%d]\n", pci_q);
0770 }
0771 
0772 static void __rtw_pci_flush_queues(struct rtw_dev *rtwdev, u32 pci_queues,
0773                    bool drop)
0774 {
0775     u8 q;
0776 
0777     for (q = 0; q < RTK_MAX_TX_QUEUE_NUM; q++) {
0778         /* It may be not necessary to flush BCN and H2C tx queues. */
0779         if (q == RTW_TX_QUEUE_BCN || q == RTW_TX_QUEUE_H2C)
0780             continue;
0781 
0782         if (pci_queues & BIT(q))
0783             __pci_flush_queue(rtwdev, q, drop);
0784     }
0785 }
0786 
0787 static void rtw_pci_flush_queues(struct rtw_dev *rtwdev, u32 queues, bool drop)
0788 {
0789     u32 pci_queues = 0;
0790     u8 i;
0791 
0792     /* If all of the hardware queues are requested to flush,
0793      * flush all of the pci queues.
0794      */
0795     if (queues == BIT(rtwdev->hw->queues) - 1) {
0796         pci_queues = BIT(RTK_MAX_TX_QUEUE_NUM) - 1;
0797     } else {
0798         for (i = 0; i < rtwdev->hw->queues; i++)
0799             if (queues & BIT(i))
0800                 pci_queues |= BIT(ac_to_hwq[i]);
0801     }
0802 
0803     __rtw_pci_flush_queues(rtwdev, pci_queues, drop);
0804 }
0805 
0806 static void rtw_pci_tx_kick_off_queue(struct rtw_dev *rtwdev, u8 queue)
0807 {
0808     struct rtw_pci *rtwpci = (struct rtw_pci *)rtwdev->priv;
0809     struct rtw_pci_tx_ring *ring;
0810     u32 bd_idx;
0811 
0812     ring = &rtwpci->tx_rings[queue];
0813     bd_idx = rtw_pci_tx_queue_idx_addr[queue];
0814 
0815     spin_lock_bh(&rtwpci->irq_lock);
0816     if (!rtw_fw_feature_check(&rtwdev->fw, FW_FEATURE_TX_WAKE))
0817         rtw_pci_deep_ps_leave(rtwdev);
0818     rtw_write16(rtwdev, bd_idx, ring->r.wp & TRX_BD_IDX_MASK);
0819     spin_unlock_bh(&rtwpci->irq_lock);
0820 }
0821 
0822 static void rtw_pci_tx_kick_off(struct rtw_dev *rtwdev)
0823 {
0824     struct rtw_pci *rtwpci = (struct rtw_pci *)rtwdev->priv;
0825     u8 queue;
0826 
0827     for (queue = 0; queue < RTK_MAX_TX_QUEUE_NUM; queue++)
0828         if (test_and_clear_bit(queue, rtwpci->tx_queued))
0829             rtw_pci_tx_kick_off_queue(rtwdev, queue);
0830 }
0831 
0832 static int rtw_pci_tx_write_data(struct rtw_dev *rtwdev,
0833                  struct rtw_tx_pkt_info *pkt_info,
0834                  struct sk_buff *skb, u8 queue)
0835 {
0836     struct rtw_pci *rtwpci = (struct rtw_pci *)rtwdev->priv;
0837     struct rtw_chip_info *chip = rtwdev->chip;
0838     struct rtw_pci_tx_ring *ring;
0839     struct rtw_pci_tx_data *tx_data;
0840     dma_addr_t dma;
0841     u32 tx_pkt_desc_sz = chip->tx_pkt_desc_sz;
0842     u32 tx_buf_desc_sz = chip->tx_buf_desc_sz;
0843     u32 size;
0844     u32 psb_len;
0845     u8 *pkt_desc;
0846     struct rtw_pci_tx_buffer_desc *buf_desc;
0847 
0848     ring = &rtwpci->tx_rings[queue];
0849 
0850     size = skb->len;
0851 
0852     if (queue == RTW_TX_QUEUE_BCN)
0853         rtw_pci_release_rsvd_page(rtwpci, ring);
0854     else if (!avail_desc(ring->r.wp, ring->r.rp, ring->r.len))
0855         return -ENOSPC;
0856 
0857     pkt_desc = skb_push(skb, chip->tx_pkt_desc_sz);
0858     memset(pkt_desc, 0, tx_pkt_desc_sz);
0859     pkt_info->qsel = rtw_pci_get_tx_qsel(skb, queue);
0860     rtw_tx_fill_tx_desc(pkt_info, skb);
0861     dma = dma_map_single(&rtwpci->pdev->dev, skb->data, skb->len,
0862                  DMA_TO_DEVICE);
0863     if (dma_mapping_error(&rtwpci->pdev->dev, dma))
0864         return -EBUSY;
0865 
0866     /* after this we got dma mapped, there is no way back */
0867     buf_desc = get_tx_buffer_desc(ring, tx_buf_desc_sz);
0868     memset(buf_desc, 0, tx_buf_desc_sz);
0869     psb_len = (skb->len - 1) / 128 + 1;
0870     if (queue == RTW_TX_QUEUE_BCN)
0871         psb_len |= 1 << RTK_PCI_TXBD_OWN_OFFSET;
0872 
0873     buf_desc[0].psb_len = cpu_to_le16(psb_len);
0874     buf_desc[0].buf_size = cpu_to_le16(tx_pkt_desc_sz);
0875     buf_desc[0].dma = cpu_to_le32(dma);
0876     buf_desc[1].buf_size = cpu_to_le16(size);
0877     buf_desc[1].dma = cpu_to_le32(dma + tx_pkt_desc_sz);
0878 
0879     tx_data = rtw_pci_get_tx_data(skb);
0880     tx_data->dma = dma;
0881     tx_data->sn = pkt_info->sn;
0882 
0883     spin_lock_bh(&rtwpci->irq_lock);
0884 
0885     skb_queue_tail(&ring->queue, skb);
0886 
0887     if (queue == RTW_TX_QUEUE_BCN)
0888         goto out_unlock;
0889 
0890     /* update write-index, and kick it off later */
0891     set_bit(queue, rtwpci->tx_queued);
0892     if (++ring->r.wp >= ring->r.len)
0893         ring->r.wp = 0;
0894 
0895 out_unlock:
0896     spin_unlock_bh(&rtwpci->irq_lock);
0897 
0898     return 0;
0899 }
0900 
0901 static int rtw_pci_write_data_rsvd_page(struct rtw_dev *rtwdev, u8 *buf,
0902                     u32 size)
0903 {
0904     struct sk_buff *skb;
0905     struct rtw_tx_pkt_info pkt_info = {0};
0906     u8 reg_bcn_work;
0907     int ret;
0908 
0909     skb = rtw_tx_write_data_rsvd_page_get(rtwdev, &pkt_info, buf, size);
0910     if (!skb)
0911         return -ENOMEM;
0912 
0913     ret = rtw_pci_tx_write_data(rtwdev, &pkt_info, skb, RTW_TX_QUEUE_BCN);
0914     if (ret) {
0915         rtw_err(rtwdev, "failed to write rsvd page data\n");
0916         return ret;
0917     }
0918 
0919     /* reserved pages go through beacon queue */
0920     reg_bcn_work = rtw_read8(rtwdev, RTK_PCI_TXBD_BCN_WORK);
0921     reg_bcn_work |= BIT_PCI_BCNQ_FLAG;
0922     rtw_write8(rtwdev, RTK_PCI_TXBD_BCN_WORK, reg_bcn_work);
0923 
0924     return 0;
0925 }
0926 
0927 static int rtw_pci_write_data_h2c(struct rtw_dev *rtwdev, u8 *buf, u32 size)
0928 {
0929     struct sk_buff *skb;
0930     struct rtw_tx_pkt_info pkt_info = {0};
0931     int ret;
0932 
0933     skb = rtw_tx_write_data_h2c_get(rtwdev, &pkt_info, buf, size);
0934     if (!skb)
0935         return -ENOMEM;
0936 
0937     ret = rtw_pci_tx_write_data(rtwdev, &pkt_info, skb, RTW_TX_QUEUE_H2C);
0938     if (ret) {
0939         rtw_err(rtwdev, "failed to write h2c data\n");
0940         return ret;
0941     }
0942 
0943     rtw_pci_tx_kick_off_queue(rtwdev, RTW_TX_QUEUE_H2C);
0944 
0945     return 0;
0946 }
0947 
0948 static int rtw_pci_tx_write(struct rtw_dev *rtwdev,
0949                 struct rtw_tx_pkt_info *pkt_info,
0950                 struct sk_buff *skb)
0951 {
0952     struct rtw_pci *rtwpci = (struct rtw_pci *)rtwdev->priv;
0953     struct rtw_pci_tx_ring *ring;
0954     u8 queue = rtw_hw_queue_mapping(skb);
0955     int ret;
0956 
0957     ret = rtw_pci_tx_write_data(rtwdev, pkt_info, skb, queue);
0958     if (ret)
0959         return ret;
0960 
0961     ring = &rtwpci->tx_rings[queue];
0962     spin_lock_bh(&rtwpci->irq_lock);
0963     if (avail_desc(ring->r.wp, ring->r.rp, ring->r.len) < 2) {
0964         ieee80211_stop_queue(rtwdev->hw, skb_get_queue_mapping(skb));
0965         ring->queue_stopped = true;
0966     }
0967     spin_unlock_bh(&rtwpci->irq_lock);
0968 
0969     return 0;
0970 }
0971 
0972 static void rtw_pci_tx_isr(struct rtw_dev *rtwdev, struct rtw_pci *rtwpci,
0973                u8 hw_queue)
0974 {
0975     struct ieee80211_hw *hw = rtwdev->hw;
0976     struct ieee80211_tx_info *info;
0977     struct rtw_pci_tx_ring *ring;
0978     struct rtw_pci_tx_data *tx_data;
0979     struct sk_buff *skb;
0980     u32 count;
0981     u32 bd_idx_addr;
0982     u32 bd_idx, cur_rp, rp_idx;
0983     u16 q_map;
0984 
0985     ring = &rtwpci->tx_rings[hw_queue];
0986 
0987     bd_idx_addr = rtw_pci_tx_queue_idx_addr[hw_queue];
0988     bd_idx = rtw_read32(rtwdev, bd_idx_addr);
0989     cur_rp = bd_idx >> 16;
0990     cur_rp &= TRX_BD_IDX_MASK;
0991     rp_idx = ring->r.rp;
0992     if (cur_rp >= ring->r.rp)
0993         count = cur_rp - ring->r.rp;
0994     else
0995         count = ring->r.len - (ring->r.rp - cur_rp);
0996 
0997     while (count--) {
0998         skb = skb_dequeue(&ring->queue);
0999         if (!skb) {
1000             rtw_err(rtwdev, "failed to dequeue %d skb TX queue %d, BD=0x%08x, rp %d -> %d\n",
1001                 count, hw_queue, bd_idx, ring->r.rp, cur_rp);
1002             break;
1003         }
1004         tx_data = rtw_pci_get_tx_data(skb);
1005         dma_unmap_single(&rtwpci->pdev->dev, tx_data->dma, skb->len,
1006                  DMA_TO_DEVICE);
1007 
1008         /* just free command packets from host to card */
1009         if (hw_queue == RTW_TX_QUEUE_H2C) {
1010             dev_kfree_skb_irq(skb);
1011             continue;
1012         }
1013 
1014         if (ring->queue_stopped &&
1015             avail_desc(ring->r.wp, rp_idx, ring->r.len) > 4) {
1016             q_map = skb_get_queue_mapping(skb);
1017             ieee80211_wake_queue(hw, q_map);
1018             ring->queue_stopped = false;
1019         }
1020 
1021         if (++rp_idx >= ring->r.len)
1022             rp_idx = 0;
1023 
1024         skb_pull(skb, rtwdev->chip->tx_pkt_desc_sz);
1025 
1026         info = IEEE80211_SKB_CB(skb);
1027 
1028         /* enqueue to wait for tx report */
1029         if (info->flags & IEEE80211_TX_CTL_REQ_TX_STATUS) {
1030             rtw_tx_report_enqueue(rtwdev, skb, tx_data->sn);
1031             continue;
1032         }
1033 
1034         /* always ACK for others, then they won't be marked as drop */
1035         if (info->flags & IEEE80211_TX_CTL_NO_ACK)
1036             info->flags |= IEEE80211_TX_STAT_NOACK_TRANSMITTED;
1037         else
1038             info->flags |= IEEE80211_TX_STAT_ACK;
1039 
1040         ieee80211_tx_info_clear_status(info);
1041         ieee80211_tx_status_irqsafe(hw, skb);
1042     }
1043 
1044     ring->r.rp = cur_rp;
1045 }
1046 
1047 static void rtw_pci_rx_isr(struct rtw_dev *rtwdev)
1048 {
1049     struct rtw_pci *rtwpci = (struct rtw_pci *)rtwdev->priv;
1050     struct napi_struct *napi = &rtwpci->napi;
1051 
1052     napi_schedule(napi);
1053 }
1054 
1055 static int rtw_pci_get_hw_rx_ring_nr(struct rtw_dev *rtwdev,
1056                      struct rtw_pci *rtwpci)
1057 {
1058     struct rtw_pci_rx_ring *ring;
1059     int count = 0;
1060     u32 tmp, cur_wp;
1061 
1062     ring = &rtwpci->rx_rings[RTW_RX_QUEUE_MPDU];
1063     tmp = rtw_read32(rtwdev, RTK_PCI_RXBD_IDX_MPDUQ);
1064     cur_wp = u32_get_bits(tmp, TRX_BD_HW_IDX_MASK);
1065     if (cur_wp >= ring->r.wp)
1066         count = cur_wp - ring->r.wp;
1067     else
1068         count = ring->r.len - (ring->r.wp - cur_wp);
1069 
1070     return count;
1071 }
1072 
1073 static u32 rtw_pci_rx_napi(struct rtw_dev *rtwdev, struct rtw_pci *rtwpci,
1074                u8 hw_queue, u32 limit)
1075 {
1076     struct rtw_chip_info *chip = rtwdev->chip;
1077     struct napi_struct *napi = &rtwpci->napi;
1078     struct rtw_pci_rx_ring *ring = &rtwpci->rx_rings[RTW_RX_QUEUE_MPDU];
1079     struct rtw_rx_pkt_stat pkt_stat;
1080     struct ieee80211_rx_status rx_status;
1081     struct sk_buff *skb, *new;
1082     u32 cur_rp = ring->r.rp;
1083     u32 count, rx_done = 0;
1084     u32 pkt_offset;
1085     u32 pkt_desc_sz = chip->rx_pkt_desc_sz;
1086     u32 buf_desc_sz = chip->rx_buf_desc_sz;
1087     u32 new_len;
1088     u8 *rx_desc;
1089     dma_addr_t dma;
1090 
1091     count = rtw_pci_get_hw_rx_ring_nr(rtwdev, rtwpci);
1092     count = min(count, limit);
1093 
1094     while (count--) {
1095         rtw_pci_dma_check(rtwdev, ring, cur_rp);
1096         skb = ring->buf[cur_rp];
1097         dma = *((dma_addr_t *)skb->cb);
1098         dma_sync_single_for_cpu(rtwdev->dev, dma, RTK_PCI_RX_BUF_SIZE,
1099                     DMA_FROM_DEVICE);
1100         rx_desc = skb->data;
1101         chip->ops->query_rx_desc(rtwdev, rx_desc, &pkt_stat, &rx_status);
1102 
1103         /* offset from rx_desc to payload */
1104         pkt_offset = pkt_desc_sz + pkt_stat.drv_info_sz +
1105                  pkt_stat.shift;
1106 
1107         /* allocate a new skb for this frame,
1108          * discard the frame if none available
1109          */
1110         new_len = pkt_stat.pkt_len + pkt_offset;
1111         new = dev_alloc_skb(new_len);
1112         if (WARN_ONCE(!new, "rx routine starvation\n"))
1113             goto next_rp;
1114 
1115         /* put the DMA data including rx_desc from phy to new skb */
1116         skb_put_data(new, skb->data, new_len);
1117 
1118         if (pkt_stat.is_c2h) {
1119             rtw_fw_c2h_cmd_rx_irqsafe(rtwdev, pkt_offset, new);
1120         } else {
1121             /* remove rx_desc */
1122             skb_pull(new, pkt_offset);
1123 
1124             rtw_rx_stats(rtwdev, pkt_stat.vif, new);
1125             memcpy(new->cb, &rx_status, sizeof(rx_status));
1126             ieee80211_rx_napi(rtwdev->hw, NULL, new, napi);
1127             rx_done++;
1128         }
1129 
1130 next_rp:
1131         /* new skb delivered to mac80211, re-enable original skb DMA */
1132         rtw_pci_sync_rx_desc_device(rtwdev, dma, ring, cur_rp,
1133                         buf_desc_sz);
1134 
1135         /* host read next element in ring */
1136         if (++cur_rp >= ring->r.len)
1137             cur_rp = 0;
1138     }
1139 
1140     ring->r.rp = cur_rp;
1141     /* 'rp', the last position we have read, is seen as previous posistion
1142      * of 'wp' that is used to calculate 'count' next time.
1143      */
1144     ring->r.wp = cur_rp;
1145     rtw_write16(rtwdev, RTK_PCI_RXBD_IDX_MPDUQ, ring->r.rp);
1146 
1147     return rx_done;
1148 }
1149 
1150 static void rtw_pci_irq_recognized(struct rtw_dev *rtwdev,
1151                    struct rtw_pci *rtwpci, u32 *irq_status)
1152 {
1153     unsigned long flags;
1154 
1155     spin_lock_irqsave(&rtwpci->hwirq_lock, flags);
1156 
1157     irq_status[0] = rtw_read32(rtwdev, RTK_PCI_HISR0);
1158     irq_status[1] = rtw_read32(rtwdev, RTK_PCI_HISR1);
1159     if (rtw_chip_wcpu_11ac(rtwdev))
1160         irq_status[3] = rtw_read32(rtwdev, RTK_PCI_HISR3);
1161     else
1162         irq_status[3] = 0;
1163     irq_status[0] &= rtwpci->irq_mask[0];
1164     irq_status[1] &= rtwpci->irq_mask[1];
1165     irq_status[3] &= rtwpci->irq_mask[3];
1166     rtw_write32(rtwdev, RTK_PCI_HISR0, irq_status[0]);
1167     rtw_write32(rtwdev, RTK_PCI_HISR1, irq_status[1]);
1168     if (rtw_chip_wcpu_11ac(rtwdev))
1169         rtw_write32(rtwdev, RTK_PCI_HISR3, irq_status[3]);
1170 
1171     spin_unlock_irqrestore(&rtwpci->hwirq_lock, flags);
1172 }
1173 
1174 static irqreturn_t rtw_pci_interrupt_handler(int irq, void *dev)
1175 {
1176     struct rtw_dev *rtwdev = dev;
1177     struct rtw_pci *rtwpci = (struct rtw_pci *)rtwdev->priv;
1178 
1179     /* disable RTW PCI interrupt to avoid more interrupts before the end of
1180      * thread function
1181      *
1182      * disable HIMR here to also avoid new HISR flag being raised before
1183      * the HISRs have been Write-1-cleared for MSI. If not all of the HISRs
1184      * are cleared, the edge-triggered interrupt will not be generated when
1185      * a new HISR flag is set.
1186      */
1187     rtw_pci_disable_interrupt(rtwdev, rtwpci);
1188 
1189     return IRQ_WAKE_THREAD;
1190 }
1191 
1192 static irqreturn_t rtw_pci_interrupt_threadfn(int irq, void *dev)
1193 {
1194     struct rtw_dev *rtwdev = dev;
1195     struct rtw_pci *rtwpci = (struct rtw_pci *)rtwdev->priv;
1196     u32 irq_status[4];
1197     bool rx = false;
1198 
1199     spin_lock_bh(&rtwpci->irq_lock);
1200     rtw_pci_irq_recognized(rtwdev, rtwpci, irq_status);
1201 
1202     if (irq_status[0] & IMR_MGNTDOK)
1203         rtw_pci_tx_isr(rtwdev, rtwpci, RTW_TX_QUEUE_MGMT);
1204     if (irq_status[0] & IMR_HIGHDOK)
1205         rtw_pci_tx_isr(rtwdev, rtwpci, RTW_TX_QUEUE_HI0);
1206     if (irq_status[0] & IMR_BEDOK)
1207         rtw_pci_tx_isr(rtwdev, rtwpci, RTW_TX_QUEUE_BE);
1208     if (irq_status[0] & IMR_BKDOK)
1209         rtw_pci_tx_isr(rtwdev, rtwpci, RTW_TX_QUEUE_BK);
1210     if (irq_status[0] & IMR_VODOK)
1211         rtw_pci_tx_isr(rtwdev, rtwpci, RTW_TX_QUEUE_VO);
1212     if (irq_status[0] & IMR_VIDOK)
1213         rtw_pci_tx_isr(rtwdev, rtwpci, RTW_TX_QUEUE_VI);
1214     if (irq_status[3] & IMR_H2CDOK)
1215         rtw_pci_tx_isr(rtwdev, rtwpci, RTW_TX_QUEUE_H2C);
1216     if (irq_status[0] & IMR_ROK) {
1217         rtw_pci_rx_isr(rtwdev);
1218         rx = true;
1219     }
1220     if (unlikely(irq_status[0] & IMR_C2HCMD))
1221         rtw_fw_c2h_cmd_isr(rtwdev);
1222 
1223     /* all of the jobs for this interrupt have been done */
1224     if (rtwpci->running)
1225         rtw_pci_enable_interrupt(rtwdev, rtwpci, rx);
1226     spin_unlock_bh(&rtwpci->irq_lock);
1227 
1228     return IRQ_HANDLED;
1229 }
1230 
1231 static int rtw_pci_io_mapping(struct rtw_dev *rtwdev,
1232                   struct pci_dev *pdev)
1233 {
1234     struct rtw_pci *rtwpci = (struct rtw_pci *)rtwdev->priv;
1235     unsigned long len;
1236     u8 bar_id = 2;
1237     int ret;
1238 
1239     ret = pci_request_regions(pdev, KBUILD_MODNAME);
1240     if (ret) {
1241         rtw_err(rtwdev, "failed to request pci regions\n");
1242         return ret;
1243     }
1244 
1245     len = pci_resource_len(pdev, bar_id);
1246     rtwpci->mmap = pci_iomap(pdev, bar_id, len);
1247     if (!rtwpci->mmap) {
1248         pci_release_regions(pdev);
1249         rtw_err(rtwdev, "failed to map pci memory\n");
1250         return -ENOMEM;
1251     }
1252 
1253     return 0;
1254 }
1255 
1256 static void rtw_pci_io_unmapping(struct rtw_dev *rtwdev,
1257                  struct pci_dev *pdev)
1258 {
1259     struct rtw_pci *rtwpci = (struct rtw_pci *)rtwdev->priv;
1260 
1261     if (rtwpci->mmap) {
1262         pci_iounmap(pdev, rtwpci->mmap);
1263         pci_release_regions(pdev);
1264     }
1265 }
1266 
1267 static void rtw_dbi_write8(struct rtw_dev *rtwdev, u16 addr, u8 data)
1268 {
1269     u16 write_addr;
1270     u16 remainder = addr & ~(BITS_DBI_WREN | BITS_DBI_ADDR_MASK);
1271     u8 flag;
1272     u8 cnt;
1273 
1274     write_addr = addr & BITS_DBI_ADDR_MASK;
1275     write_addr |= u16_encode_bits(BIT(remainder), BITS_DBI_WREN);
1276     rtw_write8(rtwdev, REG_DBI_WDATA_V1 + remainder, data);
1277     rtw_write16(rtwdev, REG_DBI_FLAG_V1, write_addr);
1278     rtw_write8(rtwdev, REG_DBI_FLAG_V1 + 2, BIT_DBI_WFLAG >> 16);
1279 
1280     for (cnt = 0; cnt < RTW_PCI_WR_RETRY_CNT; cnt++) {
1281         flag = rtw_read8(rtwdev, REG_DBI_FLAG_V1 + 2);
1282         if (flag == 0)
1283             return;
1284 
1285         udelay(10);
1286     }
1287 
1288     WARN(flag, "failed to write to DBI register, addr=0x%04x\n", addr);
1289 }
1290 
1291 static int rtw_dbi_read8(struct rtw_dev *rtwdev, u16 addr, u8 *value)
1292 {
1293     u16 read_addr = addr & BITS_DBI_ADDR_MASK;
1294     u8 flag;
1295     u8 cnt;
1296 
1297     rtw_write16(rtwdev, REG_DBI_FLAG_V1, read_addr);
1298     rtw_write8(rtwdev, REG_DBI_FLAG_V1 + 2, BIT_DBI_RFLAG >> 16);
1299 
1300     for (cnt = 0; cnt < RTW_PCI_WR_RETRY_CNT; cnt++) {
1301         flag = rtw_read8(rtwdev, REG_DBI_FLAG_V1 + 2);
1302         if (flag == 0) {
1303             read_addr = REG_DBI_RDATA_V1 + (addr & 3);
1304             *value = rtw_read8(rtwdev, read_addr);
1305             return 0;
1306         }
1307 
1308         udelay(10);
1309     }
1310 
1311     WARN(1, "failed to read DBI register, addr=0x%04x\n", addr);
1312     return -EIO;
1313 }
1314 
1315 static void rtw_mdio_write(struct rtw_dev *rtwdev, u8 addr, u16 data, bool g1)
1316 {
1317     u8 page;
1318     u8 wflag;
1319     u8 cnt;
1320 
1321     rtw_write16(rtwdev, REG_MDIO_V1, data);
1322 
1323     page = addr < RTW_PCI_MDIO_PG_SZ ? 0 : 1;
1324     page += g1 ? RTW_PCI_MDIO_PG_OFFS_G1 : RTW_PCI_MDIO_PG_OFFS_G2;
1325     rtw_write8(rtwdev, REG_PCIE_MIX_CFG, addr & BITS_MDIO_ADDR_MASK);
1326     rtw_write8(rtwdev, REG_PCIE_MIX_CFG + 3, page);
1327     rtw_write32_mask(rtwdev, REG_PCIE_MIX_CFG, BIT_MDIO_WFLAG_V1, 1);
1328 
1329     for (cnt = 0; cnt < RTW_PCI_WR_RETRY_CNT; cnt++) {
1330         wflag = rtw_read32_mask(rtwdev, REG_PCIE_MIX_CFG,
1331                     BIT_MDIO_WFLAG_V1);
1332         if (wflag == 0)
1333             return;
1334 
1335         udelay(10);
1336     }
1337 
1338     WARN(wflag, "failed to write to MDIO register, addr=0x%02x\n", addr);
1339 }
1340 
1341 static void rtw_pci_clkreq_set(struct rtw_dev *rtwdev, bool enable)
1342 {
1343     u8 value;
1344     int ret;
1345 
1346     if (rtw_pci_disable_aspm)
1347         return;
1348 
1349     ret = rtw_dbi_read8(rtwdev, RTK_PCIE_LINK_CFG, &value);
1350     if (ret) {
1351         rtw_err(rtwdev, "failed to read CLKREQ_L1, ret=%d", ret);
1352         return;
1353     }
1354 
1355     if (enable)
1356         value |= BIT_CLKREQ_SW_EN;
1357     else
1358         value &= ~BIT_CLKREQ_SW_EN;
1359 
1360     rtw_dbi_write8(rtwdev, RTK_PCIE_LINK_CFG, value);
1361 }
1362 
1363 static void rtw_pci_clkreq_pad_low(struct rtw_dev *rtwdev, bool enable)
1364 {
1365     u8 value;
1366     int ret;
1367 
1368     ret = rtw_dbi_read8(rtwdev, RTK_PCIE_LINK_CFG, &value);
1369     if (ret) {
1370         rtw_err(rtwdev, "failed to read CLKREQ_L1, ret=%d", ret);
1371         return;
1372     }
1373 
1374     if (enable)
1375         value &= ~BIT_CLKREQ_N_PAD;
1376     else
1377         value |= BIT_CLKREQ_N_PAD;
1378 
1379     rtw_dbi_write8(rtwdev, RTK_PCIE_LINK_CFG, value);
1380 }
1381 
1382 static void rtw_pci_aspm_set(struct rtw_dev *rtwdev, bool enable)
1383 {
1384     u8 value;
1385     int ret;
1386 
1387     if (rtw_pci_disable_aspm)
1388         return;
1389 
1390     ret = rtw_dbi_read8(rtwdev, RTK_PCIE_LINK_CFG, &value);
1391     if (ret) {
1392         rtw_err(rtwdev, "failed to read ASPM, ret=%d", ret);
1393         return;
1394     }
1395 
1396     if (enable)
1397         value |= BIT_L1_SW_EN;
1398     else
1399         value &= ~BIT_L1_SW_EN;
1400 
1401     rtw_dbi_write8(rtwdev, RTK_PCIE_LINK_CFG, value);
1402 }
1403 
1404 static void rtw_pci_link_ps(struct rtw_dev *rtwdev, bool enter)
1405 {
1406     struct rtw_pci *rtwpci = (struct rtw_pci *)rtwdev->priv;
1407 
1408     /* Like CLKREQ, ASPM is also implemented by two HW modules, and can
1409      * only be enabled when host supports it.
1410      *
1411      * And ASPM mechanism should be enabled when driver/firmware enters
1412      * power save mode, without having heavy traffic. Because we've
1413      * experienced some inter-operability issues that the link tends
1414      * to enter L1 state on the fly even when driver is having high
1415      * throughput. This is probably because the ASPM behavior slightly
1416      * varies from different SOC.
1417      */
1418     if (!(rtwpci->link_ctrl & PCI_EXP_LNKCTL_ASPM_L1))
1419         return;
1420 
1421     if ((enter && atomic_dec_if_positive(&rtwpci->link_usage) == 0) ||
1422         (!enter && atomic_inc_return(&rtwpci->link_usage) == 1))
1423         rtw_pci_aspm_set(rtwdev, enter);
1424 }
1425 
1426 static void rtw_pci_link_cfg(struct rtw_dev *rtwdev)
1427 {
1428     struct rtw_chip_info *chip = rtwdev->chip;
1429     struct rtw_pci *rtwpci = (struct rtw_pci *)rtwdev->priv;
1430     struct pci_dev *pdev = rtwpci->pdev;
1431     u16 link_ctrl;
1432     int ret;
1433 
1434     /* RTL8822CE has enabled REFCLK auto calibration, it does not need
1435      * to add clock delay to cover the REFCLK timing gap.
1436      */
1437     if (chip->id == RTW_CHIP_TYPE_8822C)
1438         rtw_dbi_write8(rtwdev, RTK_PCIE_CLKDLY_CTRL, 0);
1439 
1440     /* Though there is standard PCIE configuration space to set the
1441      * link control register, but by Realtek's design, driver should
1442      * check if host supports CLKREQ/ASPM to enable the HW module.
1443      *
1444      * These functions are implemented by two HW modules associated,
1445      * one is responsible to access PCIE configuration space to
1446      * follow the host settings, and another is in charge of doing
1447      * CLKREQ/ASPM mechanisms, it is default disabled. Because sometimes
1448      * the host does not support it, and due to some reasons or wrong
1449      * settings (ex. CLKREQ# not Bi-Direction), it could lead to device
1450      * loss if HW misbehaves on the link.
1451      *
1452      * Hence it's designed that driver should first check the PCIE
1453      * configuration space is sync'ed and enabled, then driver can turn
1454      * on the other module that is actually working on the mechanism.
1455      */
1456     ret = pcie_capability_read_word(pdev, PCI_EXP_LNKCTL, &link_ctrl);
1457     if (ret) {
1458         rtw_err(rtwdev, "failed to read PCI cap, ret=%d\n", ret);
1459         return;
1460     }
1461 
1462     if (link_ctrl & PCI_EXP_LNKCTL_CLKREQ_EN)
1463         rtw_pci_clkreq_set(rtwdev, true);
1464 
1465     rtwpci->link_ctrl = link_ctrl;
1466 }
1467 
1468 static void rtw_pci_interface_cfg(struct rtw_dev *rtwdev)
1469 {
1470     struct rtw_chip_info *chip = rtwdev->chip;
1471 
1472     switch (chip->id) {
1473     case RTW_CHIP_TYPE_8822C:
1474         if (rtwdev->hal.cut_version >= RTW_CHIP_VER_CUT_D)
1475             rtw_write32_mask(rtwdev, REG_HCI_MIX_CFG,
1476                      BIT_PCIE_EMAC_PDN_AUX_TO_FAST_CLK, 1);
1477         break;
1478     default:
1479         break;
1480     }
1481 }
1482 
1483 static void rtw_pci_phy_cfg(struct rtw_dev *rtwdev)
1484 {
1485     struct rtw_pci *rtwpci = (struct rtw_pci *)rtwdev->priv;
1486     struct rtw_chip_info *chip = rtwdev->chip;
1487     struct pci_dev *pdev = rtwpci->pdev;
1488     const struct rtw_intf_phy_para *para;
1489     u16 cut;
1490     u16 value;
1491     u16 offset;
1492     int i;
1493     int ret;
1494 
1495     cut = BIT(0) << rtwdev->hal.cut_version;
1496 
1497     for (i = 0; i < chip->intf_table->n_gen1_para; i++) {
1498         para = &chip->intf_table->gen1_para[i];
1499         if (!(para->cut_mask & cut))
1500             continue;
1501         if (para->offset == 0xffff)
1502             break;
1503         offset = para->offset;
1504         value = para->value;
1505         if (para->ip_sel == RTW_IP_SEL_PHY)
1506             rtw_mdio_write(rtwdev, offset, value, true);
1507         else
1508             rtw_dbi_write8(rtwdev, offset, value);
1509     }
1510 
1511     for (i = 0; i < chip->intf_table->n_gen2_para; i++) {
1512         para = &chip->intf_table->gen2_para[i];
1513         if (!(para->cut_mask & cut))
1514             continue;
1515         if (para->offset == 0xffff)
1516             break;
1517         offset = para->offset;
1518         value = para->value;
1519         if (para->ip_sel == RTW_IP_SEL_PHY)
1520             rtw_mdio_write(rtwdev, offset, value, false);
1521         else
1522             rtw_dbi_write8(rtwdev, offset, value);
1523     }
1524 
1525     rtw_pci_link_cfg(rtwdev);
1526 
1527     /* Disable 8821ce completion timeout by default */
1528     if (chip->id == RTW_CHIP_TYPE_8821C) {
1529         ret = pcie_capability_set_word(pdev, PCI_EXP_DEVCTL2,
1530                            PCI_EXP_DEVCTL2_COMP_TMOUT_DIS);
1531         if (ret)
1532             rtw_err(rtwdev, "failed to set PCI cap, ret = %d\n",
1533                 ret);
1534     }
1535 }
1536 
1537 static int __maybe_unused rtw_pci_suspend(struct device *dev)
1538 {
1539     struct ieee80211_hw *hw = dev_get_drvdata(dev);
1540     struct rtw_dev *rtwdev = hw->priv;
1541     struct rtw_chip_info *chip = rtwdev->chip;
1542     struct rtw_efuse *efuse = &rtwdev->efuse;
1543 
1544     if (chip->id == RTW_CHIP_TYPE_8822C && efuse->rfe_option == 6)
1545         rtw_pci_clkreq_pad_low(rtwdev, true);
1546     return 0;
1547 }
1548 
1549 static int __maybe_unused rtw_pci_resume(struct device *dev)
1550 {
1551     struct ieee80211_hw *hw = dev_get_drvdata(dev);
1552     struct rtw_dev *rtwdev = hw->priv;
1553     struct rtw_chip_info *chip = rtwdev->chip;
1554     struct rtw_efuse *efuse = &rtwdev->efuse;
1555 
1556     if (chip->id == RTW_CHIP_TYPE_8822C && efuse->rfe_option == 6)
1557         rtw_pci_clkreq_pad_low(rtwdev, false);
1558     return 0;
1559 }
1560 
1561 SIMPLE_DEV_PM_OPS(rtw_pm_ops, rtw_pci_suspend, rtw_pci_resume);
1562 EXPORT_SYMBOL(rtw_pm_ops);
1563 
1564 static int rtw_pci_claim(struct rtw_dev *rtwdev, struct pci_dev *pdev)
1565 {
1566     int ret;
1567 
1568     ret = pci_enable_device(pdev);
1569     if (ret) {
1570         rtw_err(rtwdev, "failed to enable pci device\n");
1571         return ret;
1572     }
1573 
1574     pci_set_master(pdev);
1575     pci_set_drvdata(pdev, rtwdev->hw);
1576     SET_IEEE80211_DEV(rtwdev->hw, &pdev->dev);
1577 
1578     return 0;
1579 }
1580 
1581 static void rtw_pci_declaim(struct rtw_dev *rtwdev, struct pci_dev *pdev)
1582 {
1583     pci_clear_master(pdev);
1584     pci_disable_device(pdev);
1585 }
1586 
1587 static int rtw_pci_setup_resource(struct rtw_dev *rtwdev, struct pci_dev *pdev)
1588 {
1589     struct rtw_pci *rtwpci;
1590     int ret;
1591 
1592     rtwpci = (struct rtw_pci *)rtwdev->priv;
1593     rtwpci->pdev = pdev;
1594 
1595     /* after this driver can access to hw registers */
1596     ret = rtw_pci_io_mapping(rtwdev, pdev);
1597     if (ret) {
1598         rtw_err(rtwdev, "failed to request pci io region\n");
1599         goto err_out;
1600     }
1601 
1602     ret = rtw_pci_init(rtwdev);
1603     if (ret) {
1604         rtw_err(rtwdev, "failed to allocate pci resources\n");
1605         goto err_io_unmap;
1606     }
1607 
1608     return 0;
1609 
1610 err_io_unmap:
1611     rtw_pci_io_unmapping(rtwdev, pdev);
1612 
1613 err_out:
1614     return ret;
1615 }
1616 
1617 static void rtw_pci_destroy(struct rtw_dev *rtwdev, struct pci_dev *pdev)
1618 {
1619     rtw_pci_deinit(rtwdev);
1620     rtw_pci_io_unmapping(rtwdev, pdev);
1621 }
1622 
1623 static struct rtw_hci_ops rtw_pci_ops = {
1624     .tx_write = rtw_pci_tx_write,
1625     .tx_kick_off = rtw_pci_tx_kick_off,
1626     .flush_queues = rtw_pci_flush_queues,
1627     .setup = rtw_pci_setup,
1628     .start = rtw_pci_start,
1629     .stop = rtw_pci_stop,
1630     .deep_ps = rtw_pci_deep_ps,
1631     .link_ps = rtw_pci_link_ps,
1632     .interface_cfg = rtw_pci_interface_cfg,
1633 
1634     .read8 = rtw_pci_read8,
1635     .read16 = rtw_pci_read16,
1636     .read32 = rtw_pci_read32,
1637     .write8 = rtw_pci_write8,
1638     .write16 = rtw_pci_write16,
1639     .write32 = rtw_pci_write32,
1640     .write_data_rsvd_page = rtw_pci_write_data_rsvd_page,
1641     .write_data_h2c = rtw_pci_write_data_h2c,
1642 };
1643 
1644 static int rtw_pci_request_irq(struct rtw_dev *rtwdev, struct pci_dev *pdev)
1645 {
1646     unsigned int flags = PCI_IRQ_LEGACY;
1647     int ret;
1648 
1649     if (!rtw_disable_msi)
1650         flags |= PCI_IRQ_MSI;
1651 
1652     ret = pci_alloc_irq_vectors(pdev, 1, 1, flags);
1653     if (ret < 0) {
1654         rtw_err(rtwdev, "failed to alloc PCI irq vectors\n");
1655         return ret;
1656     }
1657 
1658     ret = devm_request_threaded_irq(rtwdev->dev, pdev->irq,
1659                     rtw_pci_interrupt_handler,
1660                     rtw_pci_interrupt_threadfn,
1661                     IRQF_SHARED, KBUILD_MODNAME, rtwdev);
1662     if (ret) {
1663         rtw_err(rtwdev, "failed to request irq %d\n", ret);
1664         pci_free_irq_vectors(pdev);
1665     }
1666 
1667     return ret;
1668 }
1669 
1670 static void rtw_pci_free_irq(struct rtw_dev *rtwdev, struct pci_dev *pdev)
1671 {
1672     devm_free_irq(rtwdev->dev, pdev->irq, rtwdev);
1673     pci_free_irq_vectors(pdev);
1674 }
1675 
1676 static int rtw_pci_napi_poll(struct napi_struct *napi, int budget)
1677 {
1678     struct rtw_pci *rtwpci = container_of(napi, struct rtw_pci, napi);
1679     struct rtw_dev *rtwdev = container_of((void *)rtwpci, struct rtw_dev,
1680                           priv);
1681     int work_done = 0;
1682 
1683     if (rtwpci->rx_no_aspm)
1684         rtw_pci_link_ps(rtwdev, false);
1685 
1686     while (work_done < budget) {
1687         u32 work_done_once;
1688 
1689         work_done_once = rtw_pci_rx_napi(rtwdev, rtwpci, RTW_RX_QUEUE_MPDU,
1690                          budget - work_done);
1691         if (work_done_once == 0)
1692             break;
1693         work_done += work_done_once;
1694     }
1695     if (work_done < budget) {
1696         napi_complete_done(napi, work_done);
1697         spin_lock_bh(&rtwpci->irq_lock);
1698         if (rtwpci->running)
1699             rtw_pci_enable_interrupt(rtwdev, rtwpci, false);
1700         spin_unlock_bh(&rtwpci->irq_lock);
1701         /* When ISR happens during polling and before napi_complete
1702          * while no further data is received. Data on the dma_ring will
1703          * not be processed immediately. Check whether dma ring is
1704          * empty and perform napi_schedule accordingly.
1705          */
1706         if (rtw_pci_get_hw_rx_ring_nr(rtwdev, rtwpci))
1707             napi_schedule(napi);
1708     }
1709     if (rtwpci->rx_no_aspm)
1710         rtw_pci_link_ps(rtwdev, true);
1711 
1712     return work_done;
1713 }
1714 
1715 static void rtw_pci_napi_init(struct rtw_dev *rtwdev)
1716 {
1717     struct rtw_pci *rtwpci = (struct rtw_pci *)rtwdev->priv;
1718 
1719     init_dummy_netdev(&rtwpci->netdev);
1720     netif_napi_add(&rtwpci->netdev, &rtwpci->napi, rtw_pci_napi_poll,
1721                NAPI_POLL_WEIGHT);
1722 }
1723 
1724 static void rtw_pci_napi_deinit(struct rtw_dev *rtwdev)
1725 {
1726     struct rtw_pci *rtwpci = (struct rtw_pci *)rtwdev->priv;
1727 
1728     rtw_pci_napi_stop(rtwdev);
1729     netif_napi_del(&rtwpci->napi);
1730 }
1731 
1732 int rtw_pci_probe(struct pci_dev *pdev,
1733           const struct pci_device_id *id)
1734 {
1735     struct pci_dev *bridge = pci_upstream_bridge(pdev);
1736     struct ieee80211_hw *hw;
1737     struct rtw_dev *rtwdev;
1738     struct rtw_pci *rtwpci;
1739     int drv_data_size;
1740     int ret;
1741 
1742     drv_data_size = sizeof(struct rtw_dev) + sizeof(struct rtw_pci);
1743     hw = ieee80211_alloc_hw(drv_data_size, &rtw_ops);
1744     if (!hw) {
1745         dev_err(&pdev->dev, "failed to allocate hw\n");
1746         return -ENOMEM;
1747     }
1748 
1749     rtwdev = hw->priv;
1750     rtwdev->hw = hw;
1751     rtwdev->dev = &pdev->dev;
1752     rtwdev->chip = (struct rtw_chip_info *)id->driver_data;
1753     rtwdev->hci.ops = &rtw_pci_ops;
1754     rtwdev->hci.type = RTW_HCI_TYPE_PCIE;
1755 
1756     rtwpci = (struct rtw_pci *)rtwdev->priv;
1757     atomic_set(&rtwpci->link_usage, 1);
1758 
1759     ret = rtw_core_init(rtwdev);
1760     if (ret)
1761         goto err_release_hw;
1762 
1763     rtw_dbg(rtwdev, RTW_DBG_PCI,
1764         "rtw88 pci probe: vendor=0x%4.04X device=0x%4.04X rev=%d\n",
1765         pdev->vendor, pdev->device, pdev->revision);
1766 
1767     ret = rtw_pci_claim(rtwdev, pdev);
1768     if (ret) {
1769         rtw_err(rtwdev, "failed to claim pci device\n");
1770         goto err_deinit_core;
1771     }
1772 
1773     ret = rtw_pci_setup_resource(rtwdev, pdev);
1774     if (ret) {
1775         rtw_err(rtwdev, "failed to setup pci resources\n");
1776         goto err_pci_declaim;
1777     }
1778 
1779     rtw_pci_napi_init(rtwdev);
1780 
1781     ret = rtw_chip_info_setup(rtwdev);
1782     if (ret) {
1783         rtw_err(rtwdev, "failed to setup chip information\n");
1784         goto err_destroy_pci;
1785     }
1786 
1787     /* Disable PCIe ASPM L1 while doing NAPI poll for 8821CE */
1788     if (rtwdev->chip->id == RTW_CHIP_TYPE_8821C && bridge->vendor == PCI_VENDOR_ID_INTEL)
1789         rtwpci->rx_no_aspm = true;
1790 
1791     rtw_pci_phy_cfg(rtwdev);
1792 
1793     ret = rtw_register_hw(rtwdev, hw);
1794     if (ret) {
1795         rtw_err(rtwdev, "failed to register hw\n");
1796         goto err_destroy_pci;
1797     }
1798 
1799     ret = rtw_pci_request_irq(rtwdev, pdev);
1800     if (ret) {
1801         ieee80211_unregister_hw(hw);
1802         goto err_destroy_pci;
1803     }
1804 
1805     return 0;
1806 
1807 err_destroy_pci:
1808     rtw_pci_napi_deinit(rtwdev);
1809     rtw_pci_destroy(rtwdev, pdev);
1810 
1811 err_pci_declaim:
1812     rtw_pci_declaim(rtwdev, pdev);
1813 
1814 err_deinit_core:
1815     rtw_core_deinit(rtwdev);
1816 
1817 err_release_hw:
1818     ieee80211_free_hw(hw);
1819 
1820     return ret;
1821 }
1822 EXPORT_SYMBOL(rtw_pci_probe);
1823 
1824 void rtw_pci_remove(struct pci_dev *pdev)
1825 {
1826     struct ieee80211_hw *hw = pci_get_drvdata(pdev);
1827     struct rtw_dev *rtwdev;
1828     struct rtw_pci *rtwpci;
1829 
1830     if (!hw)
1831         return;
1832 
1833     rtwdev = hw->priv;
1834     rtwpci = (struct rtw_pci *)rtwdev->priv;
1835 
1836     rtw_unregister_hw(rtwdev, hw);
1837     rtw_pci_disable_interrupt(rtwdev, rtwpci);
1838     rtw_pci_napi_deinit(rtwdev);
1839     rtw_pci_destroy(rtwdev, pdev);
1840     rtw_pci_declaim(rtwdev, pdev);
1841     rtw_pci_free_irq(rtwdev, pdev);
1842     rtw_core_deinit(rtwdev);
1843     ieee80211_free_hw(hw);
1844 }
1845 EXPORT_SYMBOL(rtw_pci_remove);
1846 
1847 void rtw_pci_shutdown(struct pci_dev *pdev)
1848 {
1849     struct ieee80211_hw *hw = pci_get_drvdata(pdev);
1850     struct rtw_dev *rtwdev;
1851     struct rtw_chip_info *chip;
1852 
1853     if (!hw)
1854         return;
1855 
1856     rtwdev = hw->priv;
1857     chip = rtwdev->chip;
1858 
1859     if (chip->ops->shutdown)
1860         chip->ops->shutdown(rtwdev);
1861 
1862     pci_set_power_state(pdev, PCI_D3hot);
1863 }
1864 EXPORT_SYMBOL(rtw_pci_shutdown);
1865 
1866 MODULE_AUTHOR("Realtek Corporation");
1867 MODULE_DESCRIPTION("Realtek 802.11ac wireless PCI driver");
1868 MODULE_LICENSE("Dual BSD/GPL");
[ Source navigation ] [ Diff markup ] [ Identifier search ] [ general search ]

This page was automatically generated by the 2.1.0 LXR engine.

The LXR team
Valid CSS 2.1! Valid HTML 4.01!