Back to home page

OSCL-LXR
 
 

     

0001 // SPDX-License-Identifier: GPL-2.0-only
0002 /*
0003  * Copyright (c) 2021, MediaTek Inc.
0004  * Copyright (c) 2021-2022, Intel Corporation.
0005  *
0006  * Authors:
0007  *  Amir Hanania <amir.hanania@intel.com>
0008  *  Haijun Liu <haijun.liu@mediatek.com>
0009  *  Moises Veleta <moises.veleta@intel.com>
0010  *  Ricardo Martinez <ricardo.martinez@linux.intel.com>
0011  *
0012  * Contributors:
0013  *  Andy Shevchenko <andriy.shevchenko@linux.intel.com>
0014  *  Chiranjeevi Rapolu <chiranjeevi.rapolu@intel.com>
0015  *  Eliot Lee <eliot.lee@intel.com>
0016  *  Sreehari Kancharla <sreehari.kancharla@intel.com>
0017  */
0018 
0019 #include <linux/bits.h>
0020 #include <linux/bitfield.h>
0021 #include <linux/bitops.h>
0022 #include <linux/delay.h>
0023 #include <linux/dev_printk.h>
0024 #include <linux/io.h>
0025 #include <linux/iopoll.h>
0026 #include <linux/types.h>
0027 
0028 #include "t7xx_dpmaif.h"
0029 #include "t7xx_reg.h"
0030 
0031 #define ioread32_poll_timeout_atomic(addr, val, cond, delay_us, timeout_us) \
0032     readx_poll_timeout_atomic(ioread32, addr, val, cond, delay_us, timeout_us)
0033 
0034 static int t7xx_dpmaif_init_intr(struct dpmaif_hw_info *hw_info)
0035 {
0036     struct dpmaif_isr_en_mask *isr_en_msk = &hw_info->isr_en_mask;
0037     u32 value, ul_intr_enable, dl_intr_enable;
0038     int ret;
0039 
0040     ul_intr_enable = DP_UL_INT_ERR_MSK | DP_UL_INT_QDONE_MSK;
0041     isr_en_msk->ap_ul_l2intr_en_msk = ul_intr_enable;
0042     iowrite32(DPMAIF_AP_ALL_L2TISAR0_MASK, hw_info->pcie_base + DPMAIF_AP_L2TISAR0);
0043 
0044     /* Set interrupt enable mask */
0045     iowrite32(ul_intr_enable, hw_info->pcie_base + DPMAIF_AO_UL_AP_L2TIMCR0);
0046     iowrite32(~ul_intr_enable, hw_info->pcie_base + DPMAIF_AO_UL_AP_L2TIMSR0);
0047 
0048     /* Check mask status */
0049     ret = ioread32_poll_timeout_atomic(hw_info->pcie_base + DPMAIF_AO_UL_AP_L2TIMR0,
0050                        value, (value & ul_intr_enable) != ul_intr_enable, 0,
0051                        DPMAIF_CHECK_INIT_TIMEOUT_US);
0052     if (ret)
0053         return ret;
0054 
0055     dl_intr_enable = DP_DL_INT_PITCNT_LEN_ERR | DP_DL_INT_BATCNT_LEN_ERR;
0056     isr_en_msk->ap_dl_l2intr_err_en_msk = dl_intr_enable;
0057     ul_intr_enable = DPMAIF_DL_INT_DLQ0_QDONE | DPMAIF_DL_INT_DLQ0_PITCNT_LEN |
0058             DPMAIF_DL_INT_DLQ1_QDONE | DPMAIF_DL_INT_DLQ1_PITCNT_LEN;
0059     isr_en_msk->ap_ul_l2intr_en_msk = ul_intr_enable;
0060     iowrite32(DPMAIF_AP_APDL_ALL_L2TISAR0_MASK, hw_info->pcie_base + DPMAIF_AP_APDL_L2TISAR0);
0061 
0062     /* Set DL ISR PD enable mask */
0063     iowrite32(~ul_intr_enable, hw_info->pcie_base + DPMAIF_AO_UL_APDL_L2TIMSR0);
0064     ret = ioread32_poll_timeout_atomic(hw_info->pcie_base + DPMAIF_AO_UL_APDL_L2TIMR0,
0065                        value, (value & ul_intr_enable) != ul_intr_enable, 0,
0066                        DPMAIF_CHECK_INIT_TIMEOUT_US);
0067     if (ret)
0068         return ret;
0069 
0070     isr_en_msk->ap_udl_ip_busy_en_msk = DPMAIF_UDL_IP_BUSY;
0071     iowrite32(DPMAIF_AP_IP_BUSY_MASK, hw_info->pcie_base + DPMAIF_AP_IP_BUSY);
0072     iowrite32(isr_en_msk->ap_udl_ip_busy_en_msk,
0073           hw_info->pcie_base + DPMAIF_AO_AP_DLUL_IP_BUSY_MASK);
0074     value = ioread32(hw_info->pcie_base + DPMAIF_AO_UL_AP_L1TIMR0);
0075     value |= DPMAIF_DL_INT_Q2APTOP | DPMAIF_DL_INT_Q2TOQ1;
0076     iowrite32(value, hw_info->pcie_base + DPMAIF_AO_UL_AP_L1TIMR0);
0077     iowrite32(DPMA_HPC_ALL_INT_MASK, hw_info->pcie_base + DPMAIF_HPC_INTR_MASK);
0078 
0079     return 0;
0080 }
0081 
0082 static void t7xx_dpmaif_mask_ulq_intr(struct dpmaif_hw_info *hw_info, unsigned int q_num)
0083 {
0084     struct dpmaif_isr_en_mask *isr_en_msk;
0085     u32 value, ul_int_que_done;
0086     int ret;
0087 
0088     isr_en_msk = &hw_info->isr_en_mask;
0089     ul_int_que_done = BIT(q_num + DP_UL_INT_DONE_OFFSET) & DP_UL_INT_QDONE_MSK;
0090     isr_en_msk->ap_ul_l2intr_en_msk &= ~ul_int_que_done;
0091     iowrite32(ul_int_que_done, hw_info->pcie_base + DPMAIF_AO_UL_AP_L2TIMSR0);
0092 
0093     ret = ioread32_poll_timeout_atomic(hw_info->pcie_base + DPMAIF_AO_UL_AP_L2TIMR0,
0094                        value, (value & ul_int_que_done) == ul_int_que_done, 0,
0095                        DPMAIF_CHECK_TIMEOUT_US);
0096     if (ret)
0097         dev_err(hw_info->dev,
0098             "Could not mask the UL interrupt. DPMAIF_AO_UL_AP_L2TIMR0 is 0x%x\n",
0099             value);
0100 }
0101 
0102 void t7xx_dpmaif_unmask_ulq_intr(struct dpmaif_hw_info *hw_info, unsigned int q_num)
0103 {
0104     struct dpmaif_isr_en_mask *isr_en_msk;
0105     u32 value, ul_int_que_done;
0106     int ret;
0107 
0108     isr_en_msk = &hw_info->isr_en_mask;
0109     ul_int_que_done = BIT(q_num + DP_UL_INT_DONE_OFFSET) & DP_UL_INT_QDONE_MSK;
0110     isr_en_msk->ap_ul_l2intr_en_msk |= ul_int_que_done;
0111     iowrite32(ul_int_que_done, hw_info->pcie_base + DPMAIF_AO_UL_AP_L2TIMCR0);
0112 
0113     ret = ioread32_poll_timeout_atomic(hw_info->pcie_base + DPMAIF_AO_UL_AP_L2TIMR0,
0114                        value, (value & ul_int_que_done) != ul_int_que_done, 0,
0115                        DPMAIF_CHECK_TIMEOUT_US);
0116     if (ret)
0117         dev_err(hw_info->dev,
0118             "Could not unmask the UL interrupt. DPMAIF_AO_UL_AP_L2TIMR0 is 0x%x\n",
0119             value);
0120 }
0121 
0122 void t7xx_dpmaif_dl_unmask_batcnt_len_err_intr(struct dpmaif_hw_info *hw_info)
0123 {
0124     hw_info->isr_en_mask.ap_dl_l2intr_en_msk |= DP_DL_INT_BATCNT_LEN_ERR;
0125     iowrite32(DP_DL_INT_BATCNT_LEN_ERR, hw_info->pcie_base + DPMAIF_AO_UL_APDL_L2TIMCR0);
0126 }
0127 
0128 void t7xx_dpmaif_dl_unmask_pitcnt_len_err_intr(struct dpmaif_hw_info *hw_info)
0129 {
0130     hw_info->isr_en_mask.ap_dl_l2intr_en_msk |= DP_DL_INT_PITCNT_LEN_ERR;
0131     iowrite32(DP_DL_INT_PITCNT_LEN_ERR, hw_info->pcie_base + DPMAIF_AO_UL_APDL_L2TIMCR0);
0132 }
0133 
0134 static u32 t7xx_update_dlq_intr(struct dpmaif_hw_info *hw_info, u32 q_done)
0135 {
0136     u32 value;
0137 
0138     value = ioread32(hw_info->pcie_base + DPMAIF_AO_UL_AP_L2TIMR0);
0139     iowrite32(q_done, hw_info->pcie_base + DPMAIF_AO_UL_APDL_L2TIMSR0);
0140     return value;
0141 }
0142 
0143 static int t7xx_mask_dlq_intr(struct dpmaif_hw_info *hw_info, unsigned int qno)
0144 {
0145     u32 value, q_done;
0146     int ret;
0147 
0148     q_done = qno == DPF_RX_QNO0 ? DPMAIF_DL_INT_DLQ0_QDONE : DPMAIF_DL_INT_DLQ1_QDONE;
0149     iowrite32(q_done, hw_info->pcie_base + DPMAIF_AO_UL_APDL_L2TIMSR0);
0150 
0151     ret = read_poll_timeout_atomic(t7xx_update_dlq_intr, value, value & q_done,
0152                        0, DPMAIF_CHECK_TIMEOUT_US, false, hw_info, q_done);
0153     if (ret) {
0154         dev_err(hw_info->dev,
0155             "Could not mask the DL interrupt. DPMAIF_AO_UL_AP_L2TIMR0 is 0x%x\n",
0156             value);
0157         return -ETIMEDOUT;
0158     }
0159 
0160     hw_info->isr_en_mask.ap_dl_l2intr_en_msk &= ~q_done;
0161     return 0;
0162 }
0163 
0164 void t7xx_dpmaif_dlq_unmask_rx_done(struct dpmaif_hw_info *hw_info, unsigned int qno)
0165 {
0166     u32 mask;
0167 
0168     mask = qno == DPF_RX_QNO0 ? DPMAIF_DL_INT_DLQ0_QDONE : DPMAIF_DL_INT_DLQ1_QDONE;
0169     iowrite32(mask, hw_info->pcie_base + DPMAIF_AO_UL_APDL_L2TIMCR0);
0170     hw_info->isr_en_mask.ap_dl_l2intr_en_msk |= mask;
0171 }
0172 
0173 void t7xx_dpmaif_clr_ip_busy_sts(struct dpmaif_hw_info *hw_info)
0174 {
0175     u32 ip_busy_sts;
0176 
0177     ip_busy_sts = ioread32(hw_info->pcie_base + DPMAIF_AP_IP_BUSY);
0178     iowrite32(ip_busy_sts, hw_info->pcie_base + DPMAIF_AP_IP_BUSY);
0179 }
0180 
0181 static void t7xx_dpmaif_dlq_mask_rx_pitcnt_len_err_intr(struct dpmaif_hw_info *hw_info,
0182                             unsigned int qno)
0183 {
0184     if (qno == DPF_RX_QNO0)
0185         iowrite32(DPMAIF_DL_INT_DLQ0_PITCNT_LEN,
0186               hw_info->pcie_base + DPMAIF_AO_UL_APDL_L2TIMSR0);
0187     else
0188         iowrite32(DPMAIF_DL_INT_DLQ1_PITCNT_LEN,
0189               hw_info->pcie_base + DPMAIF_AO_UL_APDL_L2TIMSR0);
0190 }
0191 
0192 void t7xx_dpmaif_dlq_unmask_pitcnt_len_err_intr(struct dpmaif_hw_info *hw_info,
0193                         unsigned int qno)
0194 {
0195     if (qno == DPF_RX_QNO0)
0196         iowrite32(DPMAIF_DL_INT_DLQ0_PITCNT_LEN,
0197               hw_info->pcie_base + DPMAIF_AO_UL_APDL_L2TIMCR0);
0198     else
0199         iowrite32(DPMAIF_DL_INT_DLQ1_PITCNT_LEN,
0200               hw_info->pcie_base + DPMAIF_AO_UL_APDL_L2TIMCR0);
0201 }
0202 
0203 void t7xx_dpmaif_ul_clr_all_intr(struct dpmaif_hw_info *hw_info)
0204 {
0205     iowrite32(DPMAIF_AP_ALL_L2TISAR0_MASK, hw_info->pcie_base + DPMAIF_AP_L2TISAR0);
0206 }
0207 
0208 void t7xx_dpmaif_dl_clr_all_intr(struct dpmaif_hw_info *hw_info)
0209 {
0210     iowrite32(DPMAIF_AP_APDL_ALL_L2TISAR0_MASK, hw_info->pcie_base + DPMAIF_AP_APDL_L2TISAR0);
0211 }
0212 
0213 static void t7xx_dpmaif_set_intr_para(struct dpmaif_hw_intr_st_para *para,
0214                       enum dpmaif_hw_intr_type intr_type, unsigned int intr_queue)
0215 {
0216     para->intr_types[para->intr_cnt] = intr_type;
0217     para->intr_queues[para->intr_cnt] = intr_queue;
0218     para->intr_cnt++;
0219 }
0220 
0221 /* The para->intr_cnt counter is set to zero before this function is called.
0222  * It does not check for overflow as there is no risk of overflowing intr_types or intr_queues.
0223  */
0224 static void t7xx_dpmaif_hw_check_tx_intr(struct dpmaif_hw_info *hw_info,
0225                      unsigned int intr_status,
0226                      struct dpmaif_hw_intr_st_para *para)
0227 {
0228     unsigned long value;
0229 
0230     value = FIELD_GET(DP_UL_INT_QDONE_MSK, intr_status);
0231     if (value) {
0232         unsigned int index;
0233 
0234         t7xx_dpmaif_set_intr_para(para, DPF_INTR_UL_DONE, value);
0235 
0236         for_each_set_bit(index, &value, DPMAIF_TXQ_NUM)
0237             t7xx_dpmaif_mask_ulq_intr(hw_info, index);
0238     }
0239 
0240     value = FIELD_GET(DP_UL_INT_EMPTY_MSK, intr_status);
0241     if (value)
0242         t7xx_dpmaif_set_intr_para(para, DPF_INTR_UL_DRB_EMPTY, value);
0243 
0244     value = FIELD_GET(DP_UL_INT_MD_NOTREADY_MSK, intr_status);
0245     if (value)
0246         t7xx_dpmaif_set_intr_para(para, DPF_INTR_UL_MD_NOTREADY, value);
0247 
0248     value = FIELD_GET(DP_UL_INT_MD_PWR_NOTREADY_MSK, intr_status);
0249     if (value)
0250         t7xx_dpmaif_set_intr_para(para, DPF_INTR_UL_MD_PWR_NOTREADY, value);
0251 
0252     value = FIELD_GET(DP_UL_INT_ERR_MSK, intr_status);
0253     if (value)
0254         t7xx_dpmaif_set_intr_para(para, DPF_INTR_UL_LEN_ERR, value);
0255 
0256     /* Clear interrupt status */
0257     iowrite32(intr_status, hw_info->pcie_base + DPMAIF_AP_L2TISAR0);
0258 }
0259 
0260 /* The para->intr_cnt counter is set to zero before this function is called.
0261  * It does not check for overflow as there is no risk of overflowing intr_types or intr_queues.
0262  */
0263 static void t7xx_dpmaif_hw_check_rx_intr(struct dpmaif_hw_info *hw_info,
0264                      unsigned int intr_status,
0265                      struct dpmaif_hw_intr_st_para *para, int qno)
0266 {
0267     if (qno == DPF_RX_QNO_DFT) {
0268         if (intr_status & DP_DL_INT_SKB_LEN_ERR)
0269             t7xx_dpmaif_set_intr_para(para, DPF_INTR_DL_SKB_LEN_ERR, DPF_RX_QNO_DFT);
0270 
0271         if (intr_status & DP_DL_INT_BATCNT_LEN_ERR) {
0272             t7xx_dpmaif_set_intr_para(para, DPF_INTR_DL_BATCNT_LEN_ERR, DPF_RX_QNO_DFT);
0273             hw_info->isr_en_mask.ap_dl_l2intr_en_msk &= ~DP_DL_INT_BATCNT_LEN_ERR;
0274             iowrite32(DP_DL_INT_BATCNT_LEN_ERR,
0275                   hw_info->pcie_base + DPMAIF_AO_UL_APDL_L2TIMSR0);
0276         }
0277 
0278         if (intr_status & DP_DL_INT_PITCNT_LEN_ERR) {
0279             t7xx_dpmaif_set_intr_para(para, DPF_INTR_DL_PITCNT_LEN_ERR, DPF_RX_QNO_DFT);
0280             hw_info->isr_en_mask.ap_dl_l2intr_en_msk &= ~DP_DL_INT_PITCNT_LEN_ERR;
0281             iowrite32(DP_DL_INT_PITCNT_LEN_ERR,
0282                   hw_info->pcie_base + DPMAIF_AO_UL_APDL_L2TIMSR0);
0283         }
0284 
0285         if (intr_status & DP_DL_INT_PKT_EMPTY_MSK)
0286             t7xx_dpmaif_set_intr_para(para, DPF_INTR_DL_PKT_EMPTY_SET, DPF_RX_QNO_DFT);
0287 
0288         if (intr_status & DP_DL_INT_FRG_EMPTY_MSK)
0289             t7xx_dpmaif_set_intr_para(para, DPF_INTR_DL_FRG_EMPTY_SET, DPF_RX_QNO_DFT);
0290 
0291         if (intr_status & DP_DL_INT_MTU_ERR_MSK)
0292             t7xx_dpmaif_set_intr_para(para, DPF_INTR_DL_MTU_ERR, DPF_RX_QNO_DFT);
0293 
0294         if (intr_status & DP_DL_INT_FRG_LEN_ERR_MSK)
0295             t7xx_dpmaif_set_intr_para(para, DPF_INTR_DL_FRGCNT_LEN_ERR, DPF_RX_QNO_DFT);
0296 
0297         if (intr_status & DP_DL_INT_Q0_PITCNT_LEN_ERR) {
0298             t7xx_dpmaif_set_intr_para(para, DPF_INTR_DL_Q0_PITCNT_LEN_ERR, BIT(qno));
0299             t7xx_dpmaif_dlq_mask_rx_pitcnt_len_err_intr(hw_info, qno);
0300         }
0301 
0302         if (intr_status & DP_DL_INT_HPC_ENT_TYPE_ERR)
0303             t7xx_dpmaif_set_intr_para(para, DPF_INTR_DL_HPC_ENT_TYPE_ERR,
0304                           DPF_RX_QNO_DFT);
0305 
0306         if (intr_status & DP_DL_INT_Q0_DONE) {
0307             /* Mask RX done interrupt immediately after it occurs, do not clear
0308              * the interrupt if the mask operation fails.
0309              */
0310             if (!t7xx_mask_dlq_intr(hw_info, qno))
0311                 t7xx_dpmaif_set_intr_para(para, DPF_INTR_DL_Q0_DONE, BIT(qno));
0312             else
0313                 intr_status &= ~DP_DL_INT_Q0_DONE;
0314         }
0315     } else {
0316         if (intr_status & DP_DL_INT_Q1_PITCNT_LEN_ERR) {
0317             t7xx_dpmaif_set_intr_para(para, DPF_INTR_DL_Q1_PITCNT_LEN_ERR, BIT(qno));
0318             t7xx_dpmaif_dlq_mask_rx_pitcnt_len_err_intr(hw_info, qno);
0319         }
0320 
0321         if (intr_status & DP_DL_INT_Q1_DONE) {
0322             if (!t7xx_mask_dlq_intr(hw_info, qno))
0323                 t7xx_dpmaif_set_intr_para(para, DPF_INTR_DL_Q1_DONE, BIT(qno));
0324             else
0325                 intr_status &= ~DP_DL_INT_Q1_DONE;
0326         }
0327     }
0328 
0329     intr_status |= DP_DL_INT_BATCNT_LEN_ERR;
0330     /* Clear interrupt status */
0331     iowrite32(intr_status, hw_info->pcie_base + DPMAIF_AP_APDL_L2TISAR0);
0332 }
0333 
0334 /**
0335  * t7xx_dpmaif_hw_get_intr_cnt() - Reads interrupt status and count from HW.
0336  * @hw_info: Pointer to struct hw_info.
0337  * @para: Pointer to struct dpmaif_hw_intr_st_para.
0338  * @qno: Queue number.
0339  *
0340  * Reads RX/TX interrupt status from HW and clears UL/DL status as needed.
0341  *
0342  * Return: Interrupt count.
0343  */
0344 int t7xx_dpmaif_hw_get_intr_cnt(struct dpmaif_hw_info *hw_info,
0345                 struct dpmaif_hw_intr_st_para *para, int qno)
0346 {
0347     u32 rx_intr_status, tx_intr_status = 0;
0348     u32 rx_intr_qdone, tx_intr_qdone = 0;
0349 
0350     rx_intr_status = ioread32(hw_info->pcie_base + DPMAIF_AP_APDL_L2TISAR0);
0351     rx_intr_qdone = ioread32(hw_info->pcie_base + DPMAIF_AO_UL_APDL_L2TIMR0);
0352 
0353     /* TX interrupt status */
0354     if (qno == DPF_RX_QNO_DFT) {
0355         /* All ULQ and DLQ0 interrupts use the same source no need to check ULQ interrupts
0356          * when a DLQ1 interrupt has occurred.
0357          */
0358         tx_intr_status = ioread32(hw_info->pcie_base + DPMAIF_AP_L2TISAR0);
0359         tx_intr_qdone = ioread32(hw_info->pcie_base + DPMAIF_AO_UL_AP_L2TIMR0);
0360     }
0361 
0362     t7xx_dpmaif_clr_ip_busy_sts(hw_info);
0363 
0364     if (qno == DPF_RX_QNO_DFT) {
0365         /* Do not schedule bottom half again or clear UL interrupt status when we
0366          * have already masked it.
0367          */
0368         tx_intr_status &= ~tx_intr_qdone;
0369         if (tx_intr_status)
0370             t7xx_dpmaif_hw_check_tx_intr(hw_info, tx_intr_status, para);
0371     }
0372 
0373     if (rx_intr_status) {
0374         if (qno == DPF_RX_QNO0) {
0375             rx_intr_status &= DP_DL_Q0_STATUS_MASK;
0376             if (rx_intr_qdone & DPMAIF_DL_INT_DLQ0_QDONE)
0377                 /* Do not schedule bottom half again or clear DL
0378                  * queue done interrupt status when we have already masked it.
0379                  */
0380                 rx_intr_status &= ~DP_DL_INT_Q0_DONE;
0381         } else {
0382             rx_intr_status &= DP_DL_Q1_STATUS_MASK;
0383             if (rx_intr_qdone & DPMAIF_DL_INT_DLQ1_QDONE)
0384                 rx_intr_status &= ~DP_DL_INT_Q1_DONE;
0385         }
0386 
0387         if (rx_intr_status)
0388             t7xx_dpmaif_hw_check_rx_intr(hw_info, rx_intr_status, para, qno);
0389     }
0390 
0391     return para->intr_cnt;
0392 }
0393 
0394 static int t7xx_dpmaif_sram_init(struct dpmaif_hw_info *hw_info)
0395 {
0396     u32 value;
0397 
0398     value = ioread32(hw_info->pcie_base + DPMAIF_AP_MEM_CLR);
0399     value |= DPMAIF_MEM_CLR;
0400     iowrite32(value, hw_info->pcie_base + DPMAIF_AP_MEM_CLR);
0401 
0402     return ioread32_poll_timeout_atomic(hw_info->pcie_base + DPMAIF_AP_MEM_CLR,
0403                         value, !(value & DPMAIF_MEM_CLR), 0,
0404                         DPMAIF_CHECK_INIT_TIMEOUT_US);
0405 }
0406 
0407 static void t7xx_dpmaif_hw_reset(struct dpmaif_hw_info *hw_info)
0408 {
0409     iowrite32(DPMAIF_AP_AO_RST_BIT, hw_info->pcie_base + DPMAIF_AP_AO_RGU_ASSERT);
0410     udelay(2);
0411     iowrite32(DPMAIF_AP_RST_BIT, hw_info->pcie_base + DPMAIF_AP_RGU_ASSERT);
0412     udelay(2);
0413     iowrite32(DPMAIF_AP_AO_RST_BIT, hw_info->pcie_base + DPMAIF_AP_AO_RGU_DEASSERT);
0414     udelay(2);
0415     iowrite32(DPMAIF_AP_RST_BIT, hw_info->pcie_base + DPMAIF_AP_RGU_DEASSERT);
0416     udelay(2);
0417 }
0418 
0419 static int t7xx_dpmaif_hw_config(struct dpmaif_hw_info *hw_info)
0420 {
0421     u32 ap_port_mode;
0422     int ret;
0423 
0424     t7xx_dpmaif_hw_reset(hw_info);
0425 
0426     ret = t7xx_dpmaif_sram_init(hw_info);
0427     if (ret)
0428         return ret;
0429 
0430     ap_port_mode = ioread32(hw_info->pcie_base + DPMAIF_AO_DL_RDY_CHK_THRES);
0431     ap_port_mode |= DPMAIF_PORT_MODE_PCIE;
0432     iowrite32(ap_port_mode, hw_info->pcie_base + DPMAIF_AO_DL_RDY_CHK_THRES);
0433     iowrite32(DPMAIF_CG_EN, hw_info->pcie_base + DPMAIF_AP_CG_EN);
0434     return 0;
0435 }
0436 
0437 static void t7xx_dpmaif_pcie_dpmaif_sign(struct dpmaif_hw_info *hw_info)
0438 {
0439     iowrite32(DPMAIF_PCIE_MODE_SET_VALUE, hw_info->pcie_base + DPMAIF_UL_RESERVE_AO_RW);
0440 }
0441 
0442 static void t7xx_dpmaif_dl_performance(struct dpmaif_hw_info *hw_info)
0443 {
0444     u32 enable_bat_cache, enable_pit_burst;
0445 
0446     enable_bat_cache = ioread32(hw_info->pcie_base + DPMAIF_DL_BAT_INIT_CON1);
0447     enable_bat_cache |= DPMAIF_DL_BAT_CACHE_PRI;
0448     iowrite32(enable_bat_cache, hw_info->pcie_base + DPMAIF_DL_BAT_INIT_CON1);
0449 
0450     enable_pit_burst = ioread32(hw_info->pcie_base + DPMAIF_AO_DL_RDY_CHK_THRES);
0451     enable_pit_burst |= DPMAIF_DL_BURST_PIT_EN;
0452     iowrite32(enable_pit_burst, hw_info->pcie_base + DPMAIF_AO_DL_RDY_CHK_THRES);
0453 }
0454 
0455  /* DPMAIF DL DLQ part HW setting */
0456 
0457 static void t7xx_dpmaif_hw_hpc_cntl_set(struct dpmaif_hw_info *hw_info)
0458 {
0459     unsigned int value;
0460 
0461     value = DPMAIF_HPC_DLQ_PATH_MODE | DPMAIF_HPC_ADD_MODE_DF << 2;
0462     value |= DPMAIF_HASH_PRIME_DF << 4;
0463     value |= DPMAIF_HPC_TOTAL_NUM << 8;
0464     iowrite32(value, hw_info->pcie_base + DPMAIF_AO_DL_HPC_CNTL);
0465 }
0466 
0467 static void t7xx_dpmaif_hw_agg_cfg_set(struct dpmaif_hw_info *hw_info)
0468 {
0469     unsigned int value;
0470 
0471     value = DPMAIF_AGG_MAX_LEN_DF | DPMAIF_AGG_TBL_ENT_NUM_DF << 16;
0472     iowrite32(value, hw_info->pcie_base + DPMAIF_AO_DL_DLQ_AGG_CFG);
0473 }
0474 
0475 static void t7xx_dpmaif_hw_hash_bit_choose_set(struct dpmaif_hw_info *hw_info)
0476 {
0477     iowrite32(DPMAIF_DLQ_HASH_BIT_CHOOSE_DF,
0478           hw_info->pcie_base + DPMAIF_AO_DL_DLQPIT_INIT_CON5);
0479 }
0480 
0481 static void t7xx_dpmaif_hw_mid_pit_timeout_thres_set(struct dpmaif_hw_info *hw_info)
0482 {
0483     iowrite32(DPMAIF_MID_TIMEOUT_THRES_DF, hw_info->pcie_base + DPMAIF_AO_DL_DLQPIT_TIMEOUT0);
0484 }
0485 
0486 static void t7xx_dpmaif_hw_dlq_timeout_thres_set(struct dpmaif_hw_info *hw_info)
0487 {
0488     unsigned int value, i;
0489 
0490     /* Each register holds two DLQ threshold timeout values */
0491     for (i = 0; i < DPMAIF_HPC_MAX_TOTAL_NUM / 2; i++) {
0492         value = FIELD_PREP(DPMAIF_DLQ_LOW_TIMEOUT_THRES_MKS, DPMAIF_DLQ_TIMEOUT_THRES_DF);
0493         value |= FIELD_PREP(DPMAIF_DLQ_HIGH_TIMEOUT_THRES_MSK,
0494                     DPMAIF_DLQ_TIMEOUT_THRES_DF);
0495         iowrite32(value,
0496               hw_info->pcie_base + DPMAIF_AO_DL_DLQPIT_TIMEOUT1 + sizeof(u32) * i);
0497     }
0498 }
0499 
0500 static void t7xx_dpmaif_hw_dlq_start_prs_thres_set(struct dpmaif_hw_info *hw_info)
0501 {
0502     iowrite32(DPMAIF_DLQ_PRS_THRES_DF, hw_info->pcie_base + DPMAIF_AO_DL_DLQPIT_TRIG_THRES);
0503 }
0504 
0505 static void t7xx_dpmaif_dl_dlq_hpc_hw_init(struct dpmaif_hw_info *hw_info)
0506 {
0507     t7xx_dpmaif_hw_hpc_cntl_set(hw_info);
0508     t7xx_dpmaif_hw_agg_cfg_set(hw_info);
0509     t7xx_dpmaif_hw_hash_bit_choose_set(hw_info);
0510     t7xx_dpmaif_hw_mid_pit_timeout_thres_set(hw_info);
0511     t7xx_dpmaif_hw_dlq_timeout_thres_set(hw_info);
0512     t7xx_dpmaif_hw_dlq_start_prs_thres_set(hw_info);
0513 }
0514 
0515 static int t7xx_dpmaif_dl_bat_init_done(struct dpmaif_hw_info *hw_info, bool frg_en)
0516 {
0517     u32 value, dl_bat_init = 0;
0518     int ret;
0519 
0520     if (frg_en)
0521         dl_bat_init = DPMAIF_DL_BAT_FRG_INIT;
0522 
0523     dl_bat_init |= DPMAIF_DL_BAT_INIT_ALLSET;
0524     dl_bat_init |= DPMAIF_DL_BAT_INIT_EN;
0525 
0526     ret = ioread32_poll_timeout_atomic(hw_info->pcie_base + DPMAIF_DL_BAT_INIT,
0527                        value, !(value & DPMAIF_DL_BAT_INIT_NOT_READY), 0,
0528                        DPMAIF_CHECK_INIT_TIMEOUT_US);
0529     if (ret) {
0530         dev_err(hw_info->dev, "Data plane modem DL BAT is not ready\n");
0531         return ret;
0532     }
0533 
0534     iowrite32(dl_bat_init, hw_info->pcie_base + DPMAIF_DL_BAT_INIT);
0535 
0536     ret = ioread32_poll_timeout_atomic(hw_info->pcie_base + DPMAIF_DL_BAT_INIT,
0537                        value, !(value & DPMAIF_DL_BAT_INIT_NOT_READY), 0,
0538                        DPMAIF_CHECK_INIT_TIMEOUT_US);
0539     if (ret)
0540         dev_err(hw_info->dev, "Data plane modem DL BAT initialization failed\n");
0541 
0542     return ret;
0543 }
0544 
0545 static void t7xx_dpmaif_dl_set_bat_base_addr(struct dpmaif_hw_info *hw_info,
0546                          dma_addr_t addr)
0547 {
0548     iowrite32(lower_32_bits(addr), hw_info->pcie_base + DPMAIF_DL_BAT_INIT_CON0);
0549     iowrite32(upper_32_bits(addr), hw_info->pcie_base + DPMAIF_DL_BAT_INIT_CON3);
0550 }
0551 
0552 static void t7xx_dpmaif_dl_set_bat_size(struct dpmaif_hw_info *hw_info, unsigned int size)
0553 {
0554     unsigned int value;
0555 
0556     value = ioread32(hw_info->pcie_base + DPMAIF_DL_BAT_INIT_CON1);
0557     value &= ~DPMAIF_BAT_SIZE_MSK;
0558     value |= size & DPMAIF_BAT_SIZE_MSK;
0559     iowrite32(value, hw_info->pcie_base + DPMAIF_DL_BAT_INIT_CON1);
0560 }
0561 
0562 static void t7xx_dpmaif_dl_bat_en(struct dpmaif_hw_info *hw_info, bool enable)
0563 {
0564     unsigned int value;
0565 
0566     value = ioread32(hw_info->pcie_base + DPMAIF_DL_BAT_INIT_CON1);
0567 
0568     if (enable)
0569         value |= DPMAIF_BAT_EN_MSK;
0570     else
0571         value &= ~DPMAIF_BAT_EN_MSK;
0572 
0573     iowrite32(value, hw_info->pcie_base + DPMAIF_DL_BAT_INIT_CON1);
0574 }
0575 
0576 static void t7xx_dpmaif_dl_set_ao_bid_maxcnt(struct dpmaif_hw_info *hw_info)
0577 {
0578     unsigned int value;
0579 
0580     value = ioread32(hw_info->pcie_base + DPMAIF_AO_DL_PKTINFO_CON0);
0581     value &= ~DPMAIF_BAT_BID_MAXCNT_MSK;
0582     value |= FIELD_PREP(DPMAIF_BAT_BID_MAXCNT_MSK, DPMAIF_HW_PKT_BIDCNT);
0583     iowrite32(value, hw_info->pcie_base + DPMAIF_AO_DL_PKTINFO_CON0);
0584 }
0585 
0586 static void t7xx_dpmaif_dl_set_ao_mtu(struct dpmaif_hw_info *hw_info)
0587 {
0588     iowrite32(DPMAIF_HW_MTU_SIZE, hw_info->pcie_base + DPMAIF_AO_DL_PKTINFO_CON1);
0589 }
0590 
0591 static void t7xx_dpmaif_dl_set_ao_pit_chknum(struct dpmaif_hw_info *hw_info)
0592 {
0593     unsigned int value;
0594 
0595     value = ioread32(hw_info->pcie_base + DPMAIF_AO_DL_PKTINFO_CON2);
0596     value &= ~DPMAIF_PIT_CHK_NUM_MSK;
0597     value |= FIELD_PREP(DPMAIF_PIT_CHK_NUM_MSK, DPMAIF_HW_CHK_PIT_NUM);
0598     iowrite32(value, hw_info->pcie_base + DPMAIF_AO_DL_PKTINFO_CON2);
0599 }
0600 
0601 static void t7xx_dpmaif_dl_set_ao_remain_minsz(struct dpmaif_hw_info *hw_info)
0602 {
0603     unsigned int value;
0604 
0605     value = ioread32(hw_info->pcie_base + DPMAIF_AO_DL_PKTINFO_CON0);
0606     value &= ~DPMAIF_BAT_REMAIN_MINSZ_MSK;
0607     value |= FIELD_PREP(DPMAIF_BAT_REMAIN_MINSZ_MSK,
0608                 DPMAIF_HW_BAT_REMAIN / DPMAIF_BAT_REMAIN_SZ_BASE);
0609     iowrite32(value, hw_info->pcie_base + DPMAIF_AO_DL_PKTINFO_CON0);
0610 }
0611 
0612 static void t7xx_dpmaif_dl_set_ao_bat_bufsz(struct dpmaif_hw_info *hw_info)
0613 {
0614     unsigned int value;
0615 
0616     value = ioread32(hw_info->pcie_base + DPMAIF_AO_DL_PKTINFO_CON2);
0617     value &= ~DPMAIF_BAT_BUF_SZ_MSK;
0618     value |= FIELD_PREP(DPMAIF_BAT_BUF_SZ_MSK,
0619                 DPMAIF_HW_BAT_PKTBUF / DPMAIF_BAT_BUFFER_SZ_BASE);
0620     iowrite32(value, hw_info->pcie_base + DPMAIF_AO_DL_PKTINFO_CON2);
0621 }
0622 
0623 static void t7xx_dpmaif_dl_set_ao_bat_rsv_length(struct dpmaif_hw_info *hw_info)
0624 {
0625     unsigned int value;
0626 
0627     value = ioread32(hw_info->pcie_base + DPMAIF_AO_DL_PKTINFO_CON2);
0628     value &= ~DPMAIF_BAT_RSV_LEN_MSK;
0629     value |= DPMAIF_HW_BAT_RSVLEN;
0630     iowrite32(value, hw_info->pcie_base + DPMAIF_AO_DL_PKTINFO_CON2);
0631 }
0632 
0633 static void t7xx_dpmaif_dl_set_pkt_alignment(struct dpmaif_hw_info *hw_info)
0634 {
0635     unsigned int value;
0636 
0637     value = ioread32(hw_info->pcie_base + DPMAIF_AO_DL_RDY_CHK_THRES);
0638     value &= ~DPMAIF_PKT_ALIGN_MSK;
0639     value |= DPMAIF_PKT_ALIGN_EN;
0640     iowrite32(value, hw_info->pcie_base + DPMAIF_AO_DL_RDY_CHK_THRES);
0641 }
0642 
0643 static void t7xx_dpmaif_dl_set_pkt_checksum(struct dpmaif_hw_info *hw_info)
0644 {
0645     unsigned int value;
0646 
0647     value = ioread32(hw_info->pcie_base + DPMAIF_AO_DL_RDY_CHK_THRES);
0648     value |= DPMAIF_DL_PKT_CHECKSUM_EN;
0649     iowrite32(value, hw_info->pcie_base + DPMAIF_AO_DL_RDY_CHK_THRES);
0650 }
0651 
0652 static void t7xx_dpmaif_dl_set_ao_frg_check_thres(struct dpmaif_hw_info *hw_info)
0653 {
0654     unsigned int value;
0655 
0656     value = ioread32(hw_info->pcie_base + DPMAIF_AO_DL_RDY_CHK_FRG_THRES);
0657     value &= ~DPMAIF_FRG_CHECK_THRES_MSK;
0658     value |= DPMAIF_HW_CHK_FRG_NUM;
0659     iowrite32(value, hw_info->pcie_base + DPMAIF_AO_DL_RDY_CHK_FRG_THRES);
0660 }
0661 
0662 static void t7xx_dpmaif_dl_set_ao_frg_bufsz(struct dpmaif_hw_info *hw_info)
0663 {
0664     unsigned int value;
0665 
0666     value = ioread32(hw_info->pcie_base + DPMAIF_AO_DL_RDY_CHK_FRG_THRES);
0667     value &= ~DPMAIF_FRG_BUF_SZ_MSK;
0668     value |= FIELD_PREP(DPMAIF_FRG_BUF_SZ_MSK,
0669                 DPMAIF_HW_FRG_PKTBUF / DPMAIF_FRG_BUFFER_SZ_BASE);
0670     iowrite32(value, hw_info->pcie_base + DPMAIF_AO_DL_RDY_CHK_FRG_THRES);
0671 }
0672 
0673 static void t7xx_dpmaif_dl_frg_ao_en(struct dpmaif_hw_info *hw_info, bool enable)
0674 {
0675     unsigned int value;
0676 
0677     value = ioread32(hw_info->pcie_base + DPMAIF_AO_DL_RDY_CHK_FRG_THRES);
0678 
0679     if (enable)
0680         value |= DPMAIF_FRG_EN_MSK;
0681     else
0682         value &= ~DPMAIF_FRG_EN_MSK;
0683 
0684     iowrite32(value, hw_info->pcie_base + DPMAIF_AO_DL_RDY_CHK_FRG_THRES);
0685 }
0686 
0687 static void t7xx_dpmaif_dl_set_ao_bat_check_thres(struct dpmaif_hw_info *hw_info)
0688 {
0689     unsigned int value;
0690 
0691     value = ioread32(hw_info->pcie_base + DPMAIF_AO_DL_RDY_CHK_THRES);
0692     value &= ~DPMAIF_BAT_CHECK_THRES_MSK;
0693     value |= FIELD_PREP(DPMAIF_BAT_CHECK_THRES_MSK, DPMAIF_HW_CHK_BAT_NUM);
0694     iowrite32(value, hw_info->pcie_base + DPMAIF_AO_DL_RDY_CHK_THRES);
0695 }
0696 
0697 static void t7xx_dpmaif_dl_set_pit_seqnum(struct dpmaif_hw_info *hw_info)
0698 {
0699     unsigned int value;
0700 
0701     value = ioread32(hw_info->pcie_base + DPMAIF_AO_DL_PIT_SEQ_END);
0702     value &= ~DPMAIF_DL_PIT_SEQ_MSK;
0703     value |= DPMAIF_DL_PIT_SEQ_VALUE;
0704     iowrite32(value, hw_info->pcie_base + DPMAIF_AO_DL_PIT_SEQ_END);
0705 }
0706 
0707 static void t7xx_dpmaif_dl_set_dlq_pit_base_addr(struct dpmaif_hw_info *hw_info,
0708                          dma_addr_t addr)
0709 {
0710     iowrite32(lower_32_bits(addr), hw_info->pcie_base + DPMAIF_DL_DLQPIT_INIT_CON0);
0711     iowrite32(upper_32_bits(addr), hw_info->pcie_base + DPMAIF_DL_DLQPIT_INIT_CON4);
0712 }
0713 
0714 static void t7xx_dpmaif_dl_set_dlq_pit_size(struct dpmaif_hw_info *hw_info, unsigned int size)
0715 {
0716     unsigned int value;
0717 
0718     value = ioread32(hw_info->pcie_base + DPMAIF_DL_DLQPIT_INIT_CON1);
0719     value &= ~DPMAIF_PIT_SIZE_MSK;
0720     value |= size & DPMAIF_PIT_SIZE_MSK;
0721     iowrite32(value, hw_info->pcie_base + DPMAIF_DL_DLQPIT_INIT_CON1);
0722     iowrite32(0, hw_info->pcie_base + DPMAIF_DL_DLQPIT_INIT_CON2);
0723     iowrite32(0, hw_info->pcie_base + DPMAIF_DL_DLQPIT_INIT_CON3);
0724     iowrite32(0, hw_info->pcie_base + DPMAIF_DL_DLQPIT_INIT_CON5);
0725     iowrite32(0, hw_info->pcie_base + DPMAIF_DL_DLQPIT_INIT_CON6);
0726 }
0727 
0728 static void t7xx_dpmaif_dl_dlq_pit_en(struct dpmaif_hw_info *hw_info)
0729 {
0730     unsigned int value;
0731 
0732     value = ioread32(hw_info->pcie_base + DPMAIF_DL_DLQPIT_INIT_CON3);
0733     value |= DPMAIF_DLQPIT_EN_MSK;
0734     iowrite32(value, hw_info->pcie_base + DPMAIF_DL_DLQPIT_INIT_CON3);
0735 }
0736 
0737 static void t7xx_dpmaif_dl_dlq_pit_init_done(struct dpmaif_hw_info *hw_info,
0738                          unsigned int pit_idx)
0739 {
0740     unsigned int dl_pit_init;
0741     int timeout;
0742     u32 value;
0743 
0744     dl_pit_init = DPMAIF_DL_PIT_INIT_ALLSET;
0745     dl_pit_init |= (pit_idx << DPMAIF_DLQPIT_CHAN_OFS);
0746     dl_pit_init |= DPMAIF_DL_PIT_INIT_EN;
0747 
0748     timeout = ioread32_poll_timeout_atomic(hw_info->pcie_base + DPMAIF_DL_DLQPIT_INIT,
0749                            value, !(value & DPMAIF_DL_PIT_INIT_NOT_READY),
0750                            DPMAIF_CHECK_DELAY_US,
0751                            DPMAIF_CHECK_INIT_TIMEOUT_US);
0752     if (timeout) {
0753         dev_err(hw_info->dev, "Data plane modem DL PIT is not ready\n");
0754         return;
0755     }
0756 
0757     iowrite32(dl_pit_init, hw_info->pcie_base + DPMAIF_DL_DLQPIT_INIT);
0758     timeout = ioread32_poll_timeout_atomic(hw_info->pcie_base + DPMAIF_DL_DLQPIT_INIT,
0759                            value, !(value & DPMAIF_DL_PIT_INIT_NOT_READY),
0760                            DPMAIF_CHECK_DELAY_US,
0761                            DPMAIF_CHECK_INIT_TIMEOUT_US);
0762     if (timeout)
0763         dev_err(hw_info->dev, "Data plane modem DL PIT initialization failed\n");
0764 }
0765 
0766 static void t7xx_dpmaif_config_dlq_pit_hw(struct dpmaif_hw_info *hw_info, unsigned int q_num,
0767                       struct dpmaif_dl *dl_que)
0768 {
0769     t7xx_dpmaif_dl_set_dlq_pit_base_addr(hw_info, dl_que->pit_base);
0770     t7xx_dpmaif_dl_set_dlq_pit_size(hw_info, dl_que->pit_size_cnt);
0771     t7xx_dpmaif_dl_dlq_pit_en(hw_info);
0772     t7xx_dpmaif_dl_dlq_pit_init_done(hw_info, q_num);
0773 }
0774 
0775 static void t7xx_dpmaif_config_all_dlq_hw(struct dpmaif_hw_info *hw_info)
0776 {
0777     int i;
0778 
0779     for (i = 0; i < DPMAIF_RXQ_NUM; i++)
0780         t7xx_dpmaif_config_dlq_pit_hw(hw_info, i, &hw_info->dl_que[i]);
0781 }
0782 
0783 static void t7xx_dpmaif_dl_all_q_en(struct dpmaif_hw_info *hw_info, bool enable)
0784 {
0785     u32 dl_bat_init, value;
0786     int timeout;
0787 
0788     value = ioread32(hw_info->pcie_base + DPMAIF_DL_BAT_INIT_CON1);
0789 
0790     if (enable)
0791         value |= DPMAIF_BAT_EN_MSK;
0792     else
0793         value &= ~DPMAIF_BAT_EN_MSK;
0794 
0795     iowrite32(value, hw_info->pcie_base + DPMAIF_DL_BAT_INIT_CON1);
0796     dl_bat_init = DPMAIF_DL_BAT_INIT_ONLY_ENABLE_BIT;
0797     dl_bat_init |= DPMAIF_DL_BAT_INIT_EN;
0798 
0799     timeout = ioread32_poll_timeout_atomic(hw_info->pcie_base + DPMAIF_DL_BAT_INIT,
0800                            value, !(value & DPMAIF_DL_BAT_INIT_NOT_READY), 0,
0801                            DPMAIF_CHECK_TIMEOUT_US);
0802     if (timeout)
0803         dev_err(hw_info->dev, "Timeout updating BAT setting to HW\n");
0804 
0805     iowrite32(dl_bat_init, hw_info->pcie_base + DPMAIF_DL_BAT_INIT);
0806     timeout = ioread32_poll_timeout_atomic(hw_info->pcie_base + DPMAIF_DL_BAT_INIT,
0807                            value, !(value & DPMAIF_DL_BAT_INIT_NOT_READY), 0,
0808                            DPMAIF_CHECK_TIMEOUT_US);
0809     if (timeout)
0810         dev_err(hw_info->dev, "Data plane modem DL BAT is not ready\n");
0811 }
0812 
0813 static int t7xx_dpmaif_config_dlq_hw(struct dpmaif_hw_info *hw_info)
0814 {
0815     struct dpmaif_dl *dl_que;
0816     int ret;
0817 
0818     t7xx_dpmaif_dl_dlq_hpc_hw_init(hw_info);
0819 
0820     dl_que = &hw_info->dl_que[0]; /* All queues share one BAT/frag BAT table */
0821     if (!dl_que->que_started)
0822         return -EBUSY;
0823 
0824     t7xx_dpmaif_dl_set_ao_remain_minsz(hw_info);
0825     t7xx_dpmaif_dl_set_ao_bat_bufsz(hw_info);
0826     t7xx_dpmaif_dl_set_ao_frg_bufsz(hw_info);
0827     t7xx_dpmaif_dl_set_ao_bat_rsv_length(hw_info);
0828     t7xx_dpmaif_dl_set_ao_bid_maxcnt(hw_info);
0829     t7xx_dpmaif_dl_set_pkt_alignment(hw_info);
0830     t7xx_dpmaif_dl_set_pit_seqnum(hw_info);
0831     t7xx_dpmaif_dl_set_ao_mtu(hw_info);
0832     t7xx_dpmaif_dl_set_ao_pit_chknum(hw_info);
0833     t7xx_dpmaif_dl_set_ao_bat_check_thres(hw_info);
0834     t7xx_dpmaif_dl_set_ao_frg_check_thres(hw_info);
0835     t7xx_dpmaif_dl_frg_ao_en(hw_info, true);
0836 
0837     t7xx_dpmaif_dl_set_bat_base_addr(hw_info, dl_que->frg_base);
0838     t7xx_dpmaif_dl_set_bat_size(hw_info, dl_que->frg_size_cnt);
0839     t7xx_dpmaif_dl_bat_en(hw_info, true);
0840 
0841     ret = t7xx_dpmaif_dl_bat_init_done(hw_info, true);
0842     if (ret)
0843         return ret;
0844 
0845     t7xx_dpmaif_dl_set_bat_base_addr(hw_info, dl_que->bat_base);
0846     t7xx_dpmaif_dl_set_bat_size(hw_info, dl_que->bat_size_cnt);
0847     t7xx_dpmaif_dl_bat_en(hw_info, false);
0848 
0849     ret = t7xx_dpmaif_dl_bat_init_done(hw_info, false);
0850     if (ret)
0851         return ret;
0852 
0853     /* Init PIT (two PIT table) */
0854     t7xx_dpmaif_config_all_dlq_hw(hw_info);
0855     t7xx_dpmaif_dl_all_q_en(hw_info, true);
0856     t7xx_dpmaif_dl_set_pkt_checksum(hw_info);
0857     return 0;
0858 }
0859 
0860 static void t7xx_dpmaif_ul_update_drb_size(struct dpmaif_hw_info *hw_info,
0861                        unsigned int q_num, unsigned int size)
0862 {
0863     unsigned int value;
0864 
0865     value = ioread32(hw_info->pcie_base + DPMAIF_UL_DRBSIZE_ADDRH_n(q_num));
0866     value &= ~DPMAIF_DRB_SIZE_MSK;
0867     value |= size & DPMAIF_DRB_SIZE_MSK;
0868     iowrite32(value, hw_info->pcie_base + DPMAIF_UL_DRBSIZE_ADDRH_n(q_num));
0869 }
0870 
0871 static void t7xx_dpmaif_ul_update_drb_base_addr(struct dpmaif_hw_info *hw_info,
0872                         unsigned int q_num, dma_addr_t addr)
0873 {
0874     iowrite32(lower_32_bits(addr), hw_info->pcie_base + DPMAIF_ULQSAR_n(q_num));
0875     iowrite32(upper_32_bits(addr), hw_info->pcie_base + DPMAIF_UL_DRB_ADDRH_n(q_num));
0876 }
0877 
0878 static void t7xx_dpmaif_ul_rdy_en(struct dpmaif_hw_info *hw_info,
0879                   unsigned int q_num, bool ready)
0880 {
0881     u32 value;
0882 
0883     value = ioread32(hw_info->pcie_base + DPMAIF_AO_UL_CHNL_ARB0);
0884 
0885     if (ready)
0886         value |= BIT(q_num);
0887     else
0888         value &= ~BIT(q_num);
0889 
0890     iowrite32(value, hw_info->pcie_base + DPMAIF_AO_UL_CHNL_ARB0);
0891 }
0892 
0893 static void t7xx_dpmaif_ul_arb_en(struct dpmaif_hw_info *hw_info,
0894                   unsigned int q_num, bool enable)
0895 {
0896     u32 value;
0897 
0898     value = ioread32(hw_info->pcie_base + DPMAIF_AO_UL_CHNL_ARB0);
0899 
0900     if (enable)
0901         value |= BIT(q_num + 8);
0902     else
0903         value &= ~BIT(q_num + 8);
0904 
0905     iowrite32(value, hw_info->pcie_base + DPMAIF_AO_UL_CHNL_ARB0);
0906 }
0907 
0908 static void t7xx_dpmaif_config_ulq_hw(struct dpmaif_hw_info *hw_info)
0909 {
0910     struct dpmaif_ul *ul_que;
0911     int i;
0912 
0913     for (i = 0; i < DPMAIF_TXQ_NUM; i++) {
0914         ul_que = &hw_info->ul_que[i];
0915         if (ul_que->que_started) {
0916             t7xx_dpmaif_ul_update_drb_size(hw_info, i, ul_que->drb_size_cnt *
0917                                DPMAIF_UL_DRB_SIZE_WORD);
0918             t7xx_dpmaif_ul_update_drb_base_addr(hw_info, i, ul_que->drb_base);
0919             t7xx_dpmaif_ul_rdy_en(hw_info, i, true);
0920             t7xx_dpmaif_ul_arb_en(hw_info, i, true);
0921         } else {
0922             t7xx_dpmaif_ul_arb_en(hw_info, i, false);
0923         }
0924     }
0925 }
0926 
0927 static int t7xx_dpmaif_hw_init_done(struct dpmaif_hw_info *hw_info)
0928 {
0929     u32 ap_cfg;
0930     int ret;
0931 
0932     ap_cfg = ioread32(hw_info->pcie_base + DPMAIF_AP_OVERWRITE_CFG);
0933     ap_cfg |= DPMAIF_SRAM_SYNC;
0934     iowrite32(ap_cfg, hw_info->pcie_base + DPMAIF_AP_OVERWRITE_CFG);
0935 
0936     ret = ioread32_poll_timeout_atomic(hw_info->pcie_base + DPMAIF_AP_OVERWRITE_CFG,
0937                        ap_cfg, !(ap_cfg & DPMAIF_SRAM_SYNC), 0,
0938                        DPMAIF_CHECK_TIMEOUT_US);
0939     if (ret)
0940         return ret;
0941 
0942     iowrite32(DPMAIF_UL_INIT_DONE, hw_info->pcie_base + DPMAIF_AO_UL_INIT_SET);
0943     iowrite32(DPMAIF_DL_INIT_DONE, hw_info->pcie_base + DPMAIF_AO_DL_INIT_SET);
0944     return 0;
0945 }
0946 
0947 static bool t7xx_dpmaif_dl_idle_check(struct dpmaif_hw_info *hw_info)
0948 {
0949     u32 dpmaif_dl_is_busy = ioread32(hw_info->pcie_base + DPMAIF_DL_CHK_BUSY);
0950 
0951     return !(dpmaif_dl_is_busy & DPMAIF_DL_IDLE_STS);
0952 }
0953 
0954 static void t7xx_dpmaif_ul_all_q_en(struct dpmaif_hw_info *hw_info, bool enable)
0955 {
0956     u32 ul_arb_en = ioread32(hw_info->pcie_base + DPMAIF_AO_UL_CHNL_ARB0);
0957 
0958     if (enable)
0959         ul_arb_en |= DPMAIF_UL_ALL_QUE_ARB_EN;
0960     else
0961         ul_arb_en &= ~DPMAIF_UL_ALL_QUE_ARB_EN;
0962 
0963     iowrite32(ul_arb_en, hw_info->pcie_base + DPMAIF_AO_UL_CHNL_ARB0);
0964 }
0965 
0966 static bool t7xx_dpmaif_ul_idle_check(struct dpmaif_hw_info *hw_info)
0967 {
0968     u32 dpmaif_ul_is_busy = ioread32(hw_info->pcie_base + DPMAIF_UL_CHK_BUSY);
0969 
0970     return !(dpmaif_ul_is_busy & DPMAIF_UL_IDLE_STS);
0971 }
0972 
0973 void t7xx_dpmaif_ul_update_hw_drb_cnt(struct dpmaif_hw_info *hw_info, unsigned int q_num,
0974                       unsigned int drb_entry_cnt)
0975 {
0976     u32 ul_update, value;
0977     int err;
0978 
0979     ul_update = drb_entry_cnt & DPMAIF_UL_ADD_COUNT_MASK;
0980     ul_update |= DPMAIF_UL_ADD_UPDATE;
0981 
0982     err = ioread32_poll_timeout_atomic(hw_info->pcie_base + DPMAIF_ULQ_ADD_DESC_CH_n(q_num),
0983                        value, !(value & DPMAIF_UL_ADD_NOT_READY), 0,
0984                        DPMAIF_CHECK_TIMEOUT_US);
0985     if (err) {
0986         dev_err(hw_info->dev, "UL add is not ready\n");
0987         return;
0988     }
0989 
0990     iowrite32(ul_update, hw_info->pcie_base + DPMAIF_ULQ_ADD_DESC_CH_n(q_num));
0991 
0992     err = ioread32_poll_timeout_atomic(hw_info->pcie_base + DPMAIF_ULQ_ADD_DESC_CH_n(q_num),
0993                        value, !(value & DPMAIF_UL_ADD_NOT_READY), 0,
0994                        DPMAIF_CHECK_TIMEOUT_US);
0995     if (err)
0996         dev_err(hw_info->dev, "Timeout updating UL add\n");
0997 }
0998 
0999 unsigned int t7xx_dpmaif_ul_get_rd_idx(struct dpmaif_hw_info *hw_info, unsigned int q_num)
1000 {
1001     unsigned int value = ioread32(hw_info->pcie_base + DPMAIF_ULQ_STA0_n(q_num));
1002 
1003     return FIELD_GET(DPMAIF_UL_DRB_RIDX_MSK, value) / DPMAIF_UL_DRB_SIZE_WORD;
1004 }
1005 
1006 int t7xx_dpmaif_dlq_add_pit_remain_cnt(struct dpmaif_hw_info *hw_info, unsigned int dlq_pit_idx,
1007                        unsigned int pit_remain_cnt)
1008 {
1009     u32 dl_update, value;
1010     int ret;
1011 
1012     dl_update = pit_remain_cnt & DPMAIF_PIT_REM_CNT_MSK;
1013     dl_update |= DPMAIF_DL_ADD_UPDATE | (dlq_pit_idx << DPMAIF_ADD_DLQ_PIT_CHAN_OFS);
1014 
1015     ret = ioread32_poll_timeout_atomic(hw_info->pcie_base + DPMAIF_DL_DLQPIT_ADD,
1016                        value, !(value & DPMAIF_DL_ADD_NOT_READY), 0,
1017                        DPMAIF_CHECK_TIMEOUT_US);
1018     if (ret) {
1019         dev_err(hw_info->dev, "Data plane modem is not ready to add dlq\n");
1020         return ret;
1021     }
1022 
1023     iowrite32(dl_update, hw_info->pcie_base + DPMAIF_DL_DLQPIT_ADD);
1024 
1025     ret = ioread32_poll_timeout_atomic(hw_info->pcie_base + DPMAIF_DL_DLQPIT_ADD,
1026                        value, !(value & DPMAIF_DL_ADD_NOT_READY), 0,
1027                        DPMAIF_CHECK_TIMEOUT_US);
1028     if (ret) {
1029         dev_err(hw_info->dev, "Data plane modem add dlq failed\n");
1030         return ret;
1031     }
1032 
1033     return 0;
1034 }
1035 
1036 unsigned int t7xx_dpmaif_dl_dlq_pit_get_wr_idx(struct dpmaif_hw_info *hw_info,
1037                            unsigned int dlq_pit_idx)
1038 {
1039     u32 value;
1040 
1041     value = ioread32(hw_info->pcie_base + DPMAIF_AO_DL_DLQ_WR_IDX +
1042              dlq_pit_idx * DLQ_PIT_IDX_SIZE);
1043     return value & DPMAIF_DL_RD_WR_IDX_MSK;
1044 }
1045 
1046 static int t7xx_dl_add_timedout(struct dpmaif_hw_info *hw_info)
1047 {
1048     u32 value;
1049 
1050     return ioread32_poll_timeout_atomic(hw_info->pcie_base + DPMAIF_DL_BAT_ADD,
1051                        value, !(value & DPMAIF_DL_ADD_NOT_READY), 0,
1052                        DPMAIF_CHECK_TIMEOUT_US);
1053 }
1054 
1055 int t7xx_dpmaif_dl_snd_hw_bat_cnt(struct dpmaif_hw_info *hw_info, unsigned int bat_entry_cnt)
1056 {
1057     unsigned int value;
1058 
1059     if (t7xx_dl_add_timedout(hw_info)) {
1060         dev_err(hw_info->dev, "DL add BAT not ready\n");
1061         return -EBUSY;
1062     }
1063 
1064     value = bat_entry_cnt & DPMAIF_DL_ADD_COUNT_MASK;
1065     value |= DPMAIF_DL_ADD_UPDATE;
1066     iowrite32(value, hw_info->pcie_base + DPMAIF_DL_BAT_ADD);
1067 
1068     if (t7xx_dl_add_timedout(hw_info)) {
1069         dev_err(hw_info->dev, "DL add BAT timeout\n");
1070         return -EBUSY;
1071     }
1072 
1073     return 0;
1074 }
1075 
1076 unsigned int t7xx_dpmaif_dl_get_bat_rd_idx(struct dpmaif_hw_info *hw_info, unsigned int q_num)
1077 {
1078     u32 value;
1079 
1080     value = ioread32(hw_info->pcie_base + DPMAIF_AO_DL_BAT_RD_IDX);
1081     return value & DPMAIF_DL_RD_WR_IDX_MSK;
1082 }
1083 
1084 unsigned int t7xx_dpmaif_dl_get_bat_wr_idx(struct dpmaif_hw_info *hw_info, unsigned int q_num)
1085 {
1086     u32 value;
1087 
1088     value = ioread32(hw_info->pcie_base + DPMAIF_AO_DL_BAT_WR_IDX);
1089     return value & DPMAIF_DL_RD_WR_IDX_MSK;
1090 }
1091 
1092 int t7xx_dpmaif_dl_snd_hw_frg_cnt(struct dpmaif_hw_info *hw_info, unsigned int frg_entry_cnt)
1093 {
1094     unsigned int value;
1095 
1096     if (t7xx_dl_add_timedout(hw_info)) {
1097         dev_err(hw_info->dev, "Data plane modem is not ready to add frag DLQ\n");
1098         return -EBUSY;
1099     }
1100 
1101     value = frg_entry_cnt & DPMAIF_DL_ADD_COUNT_MASK;
1102     value |= DPMAIF_DL_FRG_ADD_UPDATE | DPMAIF_DL_ADD_UPDATE;
1103     iowrite32(value, hw_info->pcie_base + DPMAIF_DL_BAT_ADD);
1104 
1105     if (t7xx_dl_add_timedout(hw_info)) {
1106         dev_err(hw_info->dev, "Data plane modem add frag DLQ failed");
1107         return -EBUSY;
1108     }
1109 
1110     return 0;
1111 }
1112 
1113 unsigned int t7xx_dpmaif_dl_get_frg_rd_idx(struct dpmaif_hw_info *hw_info, unsigned int q_num)
1114 {
1115     u32 value;
1116 
1117     value = ioread32(hw_info->pcie_base + DPMAIF_AO_DL_FRGBAT_RD_IDX);
1118     return value & DPMAIF_DL_RD_WR_IDX_MSK;
1119 }
1120 
1121 static void t7xx_dpmaif_set_queue_property(struct dpmaif_hw_info *hw_info,
1122                        struct dpmaif_hw_params *init_para)
1123 {
1124     struct dpmaif_dl *dl_que;
1125     struct dpmaif_ul *ul_que;
1126     int i;
1127 
1128     for (i = 0; i < DPMAIF_RXQ_NUM; i++) {
1129         dl_que = &hw_info->dl_que[i];
1130         dl_que->bat_base = init_para->pkt_bat_base_addr[i];
1131         dl_que->bat_size_cnt = init_para->pkt_bat_size_cnt[i];
1132         dl_que->pit_base = init_para->pit_base_addr[i];
1133         dl_que->pit_size_cnt = init_para->pit_size_cnt[i];
1134         dl_que->frg_base = init_para->frg_bat_base_addr[i];
1135         dl_que->frg_size_cnt = init_para->frg_bat_size_cnt[i];
1136         dl_que->que_started = true;
1137     }
1138 
1139     for (i = 0; i < DPMAIF_TXQ_NUM; i++) {
1140         ul_que = &hw_info->ul_que[i];
1141         ul_que->drb_base = init_para->drb_base_addr[i];
1142         ul_que->drb_size_cnt = init_para->drb_size_cnt[i];
1143         ul_que->que_started = true;
1144     }
1145 }
1146 
1147 /**
1148  * t7xx_dpmaif_hw_stop_all_txq() - Stop all TX queues.
1149  * @hw_info: Pointer to struct hw_info.
1150  *
1151  * Disable HW UL queues. Checks busy UL queues to go to idle
1152  * with an attempt count of 1000000.
1153  *
1154  * Return:
1155  * * 0          - Success
1156  * * -ETIMEDOUT     - Timed out checking busy queues
1157  */
1158 int t7xx_dpmaif_hw_stop_all_txq(struct dpmaif_hw_info *hw_info)
1159 {
1160     int count = 0;
1161 
1162     t7xx_dpmaif_ul_all_q_en(hw_info, false);
1163     while (t7xx_dpmaif_ul_idle_check(hw_info)) {
1164         if (++count >= DPMAIF_MAX_CHECK_COUNT) {
1165             dev_err(hw_info->dev, "Failed to stop TX, status: 0x%x\n",
1166                 ioread32(hw_info->pcie_base + DPMAIF_UL_CHK_BUSY));
1167             return -ETIMEDOUT;
1168         }
1169     }
1170 
1171     return 0;
1172 }
1173 
1174 /**
1175  * t7xx_dpmaif_hw_stop_all_rxq() - Stop all RX queues.
1176  * @hw_info: Pointer to struct hw_info.
1177  *
1178  * Disable HW DL queue. Checks busy UL queues to go to idle
1179  * with an attempt count of 1000000.
1180  * Check that HW PIT write index equals read index with the same
1181  * attempt count.
1182  *
1183  * Return:
1184  * * 0          - Success.
1185  * * -ETIMEDOUT     - Timed out checking busy queues.
1186  */
1187 int t7xx_dpmaif_hw_stop_all_rxq(struct dpmaif_hw_info *hw_info)
1188 {
1189     unsigned int wr_idx, rd_idx;
1190     int count = 0;
1191 
1192     t7xx_dpmaif_dl_all_q_en(hw_info, false);
1193     while (t7xx_dpmaif_dl_idle_check(hw_info)) {
1194         if (++count >= DPMAIF_MAX_CHECK_COUNT) {
1195             dev_err(hw_info->dev, "Failed to stop RX, status: 0x%x\n",
1196                 ioread32(hw_info->pcie_base + DPMAIF_DL_CHK_BUSY));
1197             return -ETIMEDOUT;
1198         }
1199     }
1200 
1201     /* Check middle PIT sync done */
1202     count = 0;
1203     do {
1204         wr_idx = ioread32(hw_info->pcie_base + DPMAIF_AO_DL_PIT_WR_IDX);
1205         wr_idx &= DPMAIF_DL_RD_WR_IDX_MSK;
1206         rd_idx = ioread32(hw_info->pcie_base + DPMAIF_AO_DL_PIT_RD_IDX);
1207         rd_idx &= DPMAIF_DL_RD_WR_IDX_MSK;
1208 
1209         if (wr_idx == rd_idx)
1210             return 0;
1211     } while (++count < DPMAIF_MAX_CHECK_COUNT);
1212 
1213     dev_err(hw_info->dev, "Check middle PIT sync fail\n");
1214     return -ETIMEDOUT;
1215 }
1216 
1217 void t7xx_dpmaif_start_hw(struct dpmaif_hw_info *hw_info)
1218 {
1219     t7xx_dpmaif_ul_all_q_en(hw_info, true);
1220     t7xx_dpmaif_dl_all_q_en(hw_info, true);
1221 }
1222 
1223 /**
1224  * t7xx_dpmaif_hw_init() - Initialize HW data path API.
1225  * @hw_info: Pointer to struct hw_info.
1226  * @init_param: Pointer to struct dpmaif_hw_params.
1227  *
1228  * Configures port mode, clock config, HW interrupt initialization, and HW queue.
1229  *
1230  * Return:
1231  * * 0      - Success.
1232  * * -ERROR - Error code from failure sub-initializations.
1233  */
1234 int t7xx_dpmaif_hw_init(struct dpmaif_hw_info *hw_info, struct dpmaif_hw_params *init_param)
1235 {
1236     int ret;
1237 
1238     ret = t7xx_dpmaif_hw_config(hw_info);
1239     if (ret) {
1240         dev_err(hw_info->dev, "DPMAIF HW config failed\n");
1241         return ret;
1242     }
1243 
1244     ret = t7xx_dpmaif_init_intr(hw_info);
1245     if (ret) {
1246         dev_err(hw_info->dev, "DPMAIF HW interrupts init failed\n");
1247         return ret;
1248     }
1249 
1250     t7xx_dpmaif_set_queue_property(hw_info, init_param);
1251     t7xx_dpmaif_pcie_dpmaif_sign(hw_info);
1252     t7xx_dpmaif_dl_performance(hw_info);
1253 
1254     ret = t7xx_dpmaif_config_dlq_hw(hw_info);
1255     if (ret) {
1256         dev_err(hw_info->dev, "DPMAIF HW dlq config failed\n");
1257         return ret;
1258     }
1259 
1260     t7xx_dpmaif_config_ulq_hw(hw_info);
1261 
1262     ret = t7xx_dpmaif_hw_init_done(hw_info);
1263     if (ret)
1264         dev_err(hw_info->dev, "DPMAIF HW queue init failed\n");
1265 
1266     return ret;
1267 }
1268 
1269 bool t7xx_dpmaif_ul_clr_done(struct dpmaif_hw_info *hw_info, unsigned int qno)
1270 {
1271     u32 intr_status;
1272 
1273     intr_status = ioread32(hw_info->pcie_base + DPMAIF_AP_L2TISAR0);
1274     intr_status &= BIT(DP_UL_INT_DONE_OFFSET + qno);
1275     if (intr_status) {
1276         iowrite32(intr_status, hw_info->pcie_base + DPMAIF_AP_L2TISAR0);
1277         return true;
1278     }
1279 
1280     return false;
1281 }
[ Source navigation ] [ Diff markup ] [ Identifier search ] [ general search ]

This page was automatically generated by the 2.1.0 LXR engine.

The LXR team
Valid CSS 2.1! Valid HTML 4.01!