Back to home page

OSCL-LXR
 
 

     

0001 // SPDX-License-Identifier: GPL-2.0
0002 
0003 /*
0004  * Copyright 2020-2022 HabanaLabs, Ltd.
0005  * All Rights Reserved.
0006  */
0007 
0008 #include "gaudi2P.h"
0009 #include "gaudi2_masks.h"
0010 #include "../include/hw_ip/mmu/mmu_general.h"
0011 #include "../include/hw_ip/mmu/mmu_v2_0.h"
0012 #include "../include/gaudi2/gaudi2_packets.h"
0013 #include "../include/gaudi2/gaudi2_reg_map.h"
0014 #include "../include/gaudi2/gaudi2_async_ids_map_extended.h"
0015 #include "../include/gaudi2/arc/gaudi2_arc_common_packets.h"
0016 
0017 #include <linux/module.h>
0018 #include <linux/pci.h>
0019 #include <linux/hwmon.h>
0020 #include <linux/iommu.h>
0021 
0022 #define GAUDI2_DMA_POOL_BLK_SIZE        SZ_256      /* 256 bytes */
0023 
0024 #define GAUDI2_RESET_TIMEOUT_MSEC       500     /* 500ms */
0025 #define GAUDI2_RESET_POLL_TIMEOUT_USEC      50000       /* 50ms */
0026 #define GAUDI2_PLDM_HRESET_TIMEOUT_MSEC     25000       /* 25s */
0027 #define GAUDI2_PLDM_SRESET_TIMEOUT_MSEC     25000       /* 25s */
0028 #define GAUDI2_PLDM_RESET_POLL_TIMEOUT_USEC 3000000     /* 3s */
0029 #define GAUDI2_RESET_POLL_CNT           3
0030 #define GAUDI2_RESET_WAIT_MSEC          1       /* 1ms */
0031 #define GAUDI2_CPU_RESET_WAIT_MSEC      100     /* 100ms */
0032 #define GAUDI2_PLDM_RESET_WAIT_MSEC     1000        /* 1s */
0033 #define GAUDI2_CB_POOL_CB_CNT           512
0034 #define GAUDI2_CB_POOL_CB_SIZE          SZ_128K     /* 128KB */
0035 #define GAUDI2_MSG_TO_CPU_TIMEOUT_USEC      4000000     /* 4s */
0036 #define GAUDI2_WAIT_FOR_BL_TIMEOUT_USEC     25000000    /* 25s */
0037 #define GAUDI2_TEST_QUEUE_WAIT_USEC     100000      /* 100ms */
0038 #define GAUDI2_PLDM_TEST_QUEUE_WAIT_USEC    1000000     /* 1s */
0039 
0040 #define GAUDI2_ALLOC_CPU_MEM_RETRY_CNT      3
0041 
0042 /*
0043  * since the code already has built-in support for binning of up to MAX_FAULTY_TPCS TPCs
0044  * and the code relies on that value (for array size etc..) we define another value
0045  * for MAX faulty TPCs which reflects the cluster binning requirements
0046  */
0047 #define MAX_CLUSTER_BINNING_FAULTY_TPCS     1
0048 #define MAX_FAULTY_XBARS            1
0049 #define MAX_FAULTY_EDMAS            1
0050 #define MAX_FAULTY_DECODERS         1
0051 
0052 #define GAUDI2_TPC_FULL_MASK            0x1FFFFFF
0053 #define GAUDI2_HIF_HMMU_FULL_MASK       0xFFFF
0054 #define GAUDI2_DECODER_FULL_MASK        0x3FF
0055 
0056 #define GAUDI2_NUM_OF_QM_ERR_CAUSE      18
0057 #define GAUDI2_NUM_OF_QM_LCP_ERR_CAUSE      25
0058 #define GAUDI2_NUM_OF_QM_ARB_ERR_CAUSE      3
0059 #define GAUDI2_NUM_OF_ARC_SEI_ERR_CAUSE     14
0060 #define GAUDI2_NUM_OF_CPU_SEI_ERR_CAUSE     3
0061 #define GAUDI2_NUM_OF_QM_SEI_ERR_CAUSE      2
0062 #define GAUDI2_NUM_OF_ROT_ERR_CAUSE     22
0063 #define GAUDI2_NUM_OF_TPC_INTR_CAUSE        30
0064 #define GAUDI2_NUM_OF_DEC_ERR_CAUSE     25
0065 #define GAUDI2_NUM_OF_MME_ERR_CAUSE     16
0066 #define GAUDI2_NUM_OF_MME_SBTE_ERR_CAUSE    5
0067 #define GAUDI2_NUM_OF_MME_WAP_ERR_CAUSE     7
0068 #define GAUDI2_NUM_OF_DMA_CORE_INTR_CAUSE   8
0069 #define GAUDI2_NUM_OF_MMU_SPI_SEI_CAUSE     19
0070 #define GAUDI2_NUM_OF_HBM_SEI_CAUSE     9
0071 #define GAUDI2_NUM_OF_SM_SEI_ERR_CAUSE      3
0072 #define GAUDI2_NUM_OF_PCIE_ADDR_DEC_ERR_CAUSE   3
0073 #define GAUDI2_NUM_OF_PMMU_FATAL_ERR_CAUSE  2
0074 #define GAUDI2_NUM_OF_HIF_FATAL_ERR_CAUSE   2
0075 #define GAUDI2_NUM_OF_AXI_DRAIN_ERR_CAUSE   2
0076 #define GAUDI2_NUM_OF_HBM_MC_SPI_CAUSE      5
0077 
0078 #define GAUDI2_MMU_CACHE_INV_TIMEOUT_USEC   (MMU_CONFIG_TIMEOUT_USEC * 10)
0079 #define GAUDI2_PLDM_MMU_TIMEOUT_USEC        (MMU_CONFIG_TIMEOUT_USEC * 200)
0080 #define GAUDI2_ARB_WDT_TIMEOUT          (0x1000000)
0081 
0082 #define GAUDI2_VDEC_TIMEOUT_USEC        10000       /* 10ms */
0083 #define GAUDI2_PLDM_VDEC_TIMEOUT_USEC       (GAUDI2_VDEC_TIMEOUT_USEC * 100)
0084 
0085 #define KDMA_TIMEOUT_USEC           USEC_PER_SEC
0086 
0087 #define IS_DMA_IDLE(dma_core_idle_ind_mask) \
0088     (!((dma_core_idle_ind_mask) &       \
0089     ((DCORE0_EDMA0_CORE_IDLE_IND_MASK_DESC_CNT_STS_MASK) | \
0090     (DCORE0_EDMA0_CORE_IDLE_IND_MASK_COMP_MASK))))
0091 
0092 #define IS_MME_IDLE(mme_arch_sts) (((mme_arch_sts) & MME_ARCH_IDLE_MASK) == MME_ARCH_IDLE_MASK)
0093 
0094 #define IS_TPC_IDLE(tpc_cfg_sts) (((tpc_cfg_sts) & (TPC_IDLE_MASK)) == (TPC_IDLE_MASK))
0095 
0096 #define IS_QM_IDLE(qm_glbl_sts0, qm_glbl_sts1, qm_cgm_sts) \
0097     ((((qm_glbl_sts0) & (QM_IDLE_MASK)) == (QM_IDLE_MASK)) && \
0098     (((qm_glbl_sts1) & (QM_ARC_IDLE_MASK)) == (QM_ARC_IDLE_MASK)) && \
0099     (((qm_cgm_sts) & (CGM_IDLE_MASK)) == (CGM_IDLE_MASK)))
0100 
0101 #define PCIE_DEC_EN_MASK            0x300
0102 #define DEC_WORK_STATE_IDLE         0
0103 #define DEC_WORK_STATE_PEND         3
0104 #define IS_DEC_IDLE(dec_swreg15) \
0105     (((dec_swreg15) & DCORE0_DEC0_CMD_SWREG15_SW_WORK_STATE_MASK) == DEC_WORK_STATE_IDLE || \
0106     ((dec_swreg15) & DCORE0_DEC0_CMD_SWREG15_SW_WORK_STATE_MASK) ==  DEC_WORK_STATE_PEND)
0107 
0108 /* HBM MMU address scrambling parameters */
0109 #define GAUDI2_HBM_MMU_SCRM_MEM_SIZE        SZ_8M
0110 #define GAUDI2_HBM_MMU_SCRM_DIV_SHIFT       26
0111 #define GAUDI2_HBM_MMU_SCRM_MOD_SHIFT       0
0112 #define GAUDI2_HBM_MMU_SCRM_ADDRESS_MASK    DRAM_VA_HINT_MASK
0113 #define GAUDI2_COMPENSATE_TLB_PAGE_SIZE_FACTOR  16
0114 #define MMU_RANGE_INV_VA_LSB_SHIFT      12
0115 #define MMU_RANGE_INV_VA_MSB_SHIFT      44
0116 #define MMU_RANGE_INV_EN_SHIFT          0
0117 #define MMU_RANGE_INV_ASID_EN_SHIFT     1
0118 #define MMU_RANGE_INV_ASID_SHIFT        2
0119 
0120 #define GAUDI2_MAX_STRING_LEN           64
0121 
0122 #define GAUDI2_VDEC_MSIX_ENTRIES        (GAUDI2_IRQ_NUM_SHARED_DEC1_ABNRM - \
0123                             GAUDI2_IRQ_NUM_DCORE0_DEC0_NRM + 1)
0124 
0125 enum hl_pmmu_fatal_cause {
0126     LATENCY_RD_OUT_FIFO_OVERRUN,
0127     LATENCY_WR_OUT_FIFO_OVERRUN,
0128 };
0129 
0130 enum hl_pcie_drain_ind_cause {
0131     LBW_AXI_DRAIN_IND,
0132     HBW_AXI_DRAIN_IND
0133 };
0134 
0135 static const u32 cluster_hmmu_hif_enabled_mask[GAUDI2_HBM_NUM] = {
0136     [HBM_ID0] = 0xFFFC,
0137     [HBM_ID1] = 0xFFCF,
0138     [HBM_ID2] = 0xF7F7,
0139     [HBM_ID3] = 0x7F7F,
0140     [HBM_ID4] = 0xFCFF,
0141     [HBM_ID5] = 0xCFFF,
0142 };
0143 
0144 static const u8 xbar_edge_to_hbm_cluster[EDMA_ID_SIZE] = {
0145     [0] = HBM_ID0,
0146     [1] = HBM_ID1,
0147     [2] = HBM_ID4,
0148     [3] = HBM_ID5,
0149 };
0150 
0151 static const u8 edma_to_hbm_cluster[EDMA_ID_SIZE] = {
0152     [EDMA_ID_DCORE0_INSTANCE0] = HBM_ID0,
0153     [EDMA_ID_DCORE0_INSTANCE1] = HBM_ID2,
0154     [EDMA_ID_DCORE1_INSTANCE0] = HBM_ID1,
0155     [EDMA_ID_DCORE1_INSTANCE1] = HBM_ID3,
0156     [EDMA_ID_DCORE2_INSTANCE0] = HBM_ID2,
0157     [EDMA_ID_DCORE2_INSTANCE1] = HBM_ID4,
0158     [EDMA_ID_DCORE3_INSTANCE0] = HBM_ID3,
0159     [EDMA_ID_DCORE3_INSTANCE1] = HBM_ID5,
0160 };
0161 
0162 static const int gaudi2_qman_async_event_id[] = {
0163     [GAUDI2_QUEUE_ID_PDMA_0_0] = GAUDI2_EVENT_PDMA0_QM,
0164     [GAUDI2_QUEUE_ID_PDMA_0_1] = GAUDI2_EVENT_PDMA0_QM,
0165     [GAUDI2_QUEUE_ID_PDMA_0_2] = GAUDI2_EVENT_PDMA0_QM,
0166     [GAUDI2_QUEUE_ID_PDMA_0_3] = GAUDI2_EVENT_PDMA0_QM,
0167     [GAUDI2_QUEUE_ID_PDMA_1_0] = GAUDI2_EVENT_PDMA1_QM,
0168     [GAUDI2_QUEUE_ID_PDMA_1_1] = GAUDI2_EVENT_PDMA1_QM,
0169     [GAUDI2_QUEUE_ID_PDMA_1_2] = GAUDI2_EVENT_PDMA1_QM,
0170     [GAUDI2_QUEUE_ID_PDMA_1_3] = GAUDI2_EVENT_PDMA1_QM,
0171     [GAUDI2_QUEUE_ID_DCORE0_EDMA_0_0] = GAUDI2_EVENT_HDMA0_QM,
0172     [GAUDI2_QUEUE_ID_DCORE0_EDMA_0_1] = GAUDI2_EVENT_HDMA0_QM,
0173     [GAUDI2_QUEUE_ID_DCORE0_EDMA_0_2] = GAUDI2_EVENT_HDMA0_QM,
0174     [GAUDI2_QUEUE_ID_DCORE0_EDMA_0_3] = GAUDI2_EVENT_HDMA0_QM,
0175     [GAUDI2_QUEUE_ID_DCORE0_EDMA_1_0] = GAUDI2_EVENT_HDMA1_QM,
0176     [GAUDI2_QUEUE_ID_DCORE0_EDMA_1_1] = GAUDI2_EVENT_HDMA1_QM,
0177     [GAUDI2_QUEUE_ID_DCORE0_EDMA_1_2] = GAUDI2_EVENT_HDMA1_QM,
0178     [GAUDI2_QUEUE_ID_DCORE0_EDMA_1_3] = GAUDI2_EVENT_HDMA1_QM,
0179     [GAUDI2_QUEUE_ID_DCORE0_MME_0_0] = GAUDI2_EVENT_MME0_QM,
0180     [GAUDI2_QUEUE_ID_DCORE0_MME_0_1] = GAUDI2_EVENT_MME0_QM,
0181     [GAUDI2_QUEUE_ID_DCORE0_MME_0_2] = GAUDI2_EVENT_MME0_QM,
0182     [GAUDI2_QUEUE_ID_DCORE0_MME_0_3] = GAUDI2_EVENT_MME0_QM,
0183     [GAUDI2_QUEUE_ID_DCORE0_TPC_0_0] = GAUDI2_EVENT_TPC0_QM,
0184     [GAUDI2_QUEUE_ID_DCORE0_TPC_0_1] = GAUDI2_EVENT_TPC0_QM,
0185     [GAUDI2_QUEUE_ID_DCORE0_TPC_0_2] = GAUDI2_EVENT_TPC0_QM,
0186     [GAUDI2_QUEUE_ID_DCORE0_TPC_0_3] = GAUDI2_EVENT_TPC0_QM,
0187     [GAUDI2_QUEUE_ID_DCORE0_TPC_1_0] = GAUDI2_EVENT_TPC1_QM,
0188     [GAUDI2_QUEUE_ID_DCORE0_TPC_1_1] = GAUDI2_EVENT_TPC1_QM,
0189     [GAUDI2_QUEUE_ID_DCORE0_TPC_1_2] = GAUDI2_EVENT_TPC1_QM,
0190     [GAUDI2_QUEUE_ID_DCORE0_TPC_1_3] = GAUDI2_EVENT_TPC1_QM,
0191     [GAUDI2_QUEUE_ID_DCORE0_TPC_2_0] = GAUDI2_EVENT_TPC2_QM,
0192     [GAUDI2_QUEUE_ID_DCORE0_TPC_2_1] = GAUDI2_EVENT_TPC2_QM,
0193     [GAUDI2_QUEUE_ID_DCORE0_TPC_2_2] = GAUDI2_EVENT_TPC2_QM,
0194     [GAUDI2_QUEUE_ID_DCORE0_TPC_2_3] = GAUDI2_EVENT_TPC2_QM,
0195     [GAUDI2_QUEUE_ID_DCORE0_TPC_3_0] = GAUDI2_EVENT_TPC3_QM,
0196     [GAUDI2_QUEUE_ID_DCORE0_TPC_3_1] = GAUDI2_EVENT_TPC3_QM,
0197     [GAUDI2_QUEUE_ID_DCORE0_TPC_3_2] = GAUDI2_EVENT_TPC3_QM,
0198     [GAUDI2_QUEUE_ID_DCORE0_TPC_3_3] = GAUDI2_EVENT_TPC3_QM,
0199     [GAUDI2_QUEUE_ID_DCORE0_TPC_4_0] = GAUDI2_EVENT_TPC4_QM,
0200     [GAUDI2_QUEUE_ID_DCORE0_TPC_4_1] = GAUDI2_EVENT_TPC4_QM,
0201     [GAUDI2_QUEUE_ID_DCORE0_TPC_4_2] = GAUDI2_EVENT_TPC4_QM,
0202     [GAUDI2_QUEUE_ID_DCORE0_TPC_4_3] = GAUDI2_EVENT_TPC4_QM,
0203     [GAUDI2_QUEUE_ID_DCORE0_TPC_5_0] = GAUDI2_EVENT_TPC5_QM,
0204     [GAUDI2_QUEUE_ID_DCORE0_TPC_5_1] = GAUDI2_EVENT_TPC5_QM,
0205     [GAUDI2_QUEUE_ID_DCORE0_TPC_5_2] = GAUDI2_EVENT_TPC5_QM,
0206     [GAUDI2_QUEUE_ID_DCORE0_TPC_5_3] = GAUDI2_EVENT_TPC5_QM,
0207     [GAUDI2_QUEUE_ID_DCORE0_TPC_6_0] = GAUDI2_EVENT_TPC24_QM,
0208     [GAUDI2_QUEUE_ID_DCORE0_TPC_6_1] = GAUDI2_EVENT_TPC24_QM,
0209     [GAUDI2_QUEUE_ID_DCORE0_TPC_6_2] = GAUDI2_EVENT_TPC24_QM,
0210     [GAUDI2_QUEUE_ID_DCORE0_TPC_6_3] = GAUDI2_EVENT_TPC24_QM,
0211     [GAUDI2_QUEUE_ID_DCORE1_EDMA_0_0] = GAUDI2_EVENT_HDMA2_QM,
0212     [GAUDI2_QUEUE_ID_DCORE1_EDMA_0_1] = GAUDI2_EVENT_HDMA2_QM,
0213     [GAUDI2_QUEUE_ID_DCORE1_EDMA_0_2] = GAUDI2_EVENT_HDMA2_QM,
0214     [GAUDI2_QUEUE_ID_DCORE1_EDMA_0_3] = GAUDI2_EVENT_HDMA2_QM,
0215     [GAUDI2_QUEUE_ID_DCORE1_EDMA_1_0] = GAUDI2_EVENT_HDMA3_QM,
0216     [GAUDI2_QUEUE_ID_DCORE1_EDMA_1_1] = GAUDI2_EVENT_HDMA3_QM,
0217     [GAUDI2_QUEUE_ID_DCORE1_EDMA_1_2] = GAUDI2_EVENT_HDMA3_QM,
0218     [GAUDI2_QUEUE_ID_DCORE1_EDMA_1_3] = GAUDI2_EVENT_HDMA3_QM,
0219     [GAUDI2_QUEUE_ID_DCORE1_MME_0_0] = GAUDI2_EVENT_MME1_QM,
0220     [GAUDI2_QUEUE_ID_DCORE1_MME_0_1] = GAUDI2_EVENT_MME1_QM,
0221     [GAUDI2_QUEUE_ID_DCORE1_MME_0_2] = GAUDI2_EVENT_MME1_QM,
0222     [GAUDI2_QUEUE_ID_DCORE1_MME_0_3] = GAUDI2_EVENT_MME1_QM,
0223     [GAUDI2_QUEUE_ID_DCORE1_TPC_0_0] = GAUDI2_EVENT_TPC6_QM,
0224     [GAUDI2_QUEUE_ID_DCORE1_TPC_0_1] = GAUDI2_EVENT_TPC6_QM,
0225     [GAUDI2_QUEUE_ID_DCORE1_TPC_0_2] = GAUDI2_EVENT_TPC6_QM,
0226     [GAUDI2_QUEUE_ID_DCORE1_TPC_0_3] = GAUDI2_EVENT_TPC6_QM,
0227     [GAUDI2_QUEUE_ID_DCORE1_TPC_1_0] = GAUDI2_EVENT_TPC7_QM,
0228     [GAUDI2_QUEUE_ID_DCORE1_TPC_1_1] = GAUDI2_EVENT_TPC7_QM,
0229     [GAUDI2_QUEUE_ID_DCORE1_TPC_1_2] = GAUDI2_EVENT_TPC7_QM,
0230     [GAUDI2_QUEUE_ID_DCORE1_TPC_1_3] = GAUDI2_EVENT_TPC7_QM,
0231     [GAUDI2_QUEUE_ID_DCORE1_TPC_2_0] = GAUDI2_EVENT_TPC8_QM,
0232     [GAUDI2_QUEUE_ID_DCORE1_TPC_2_1] = GAUDI2_EVENT_TPC8_QM,
0233     [GAUDI2_QUEUE_ID_DCORE1_TPC_2_2] = GAUDI2_EVENT_TPC8_QM,
0234     [GAUDI2_QUEUE_ID_DCORE1_TPC_2_3] = GAUDI2_EVENT_TPC8_QM,
0235     [GAUDI2_QUEUE_ID_DCORE1_TPC_3_0] = GAUDI2_EVENT_TPC9_QM,
0236     [GAUDI2_QUEUE_ID_DCORE1_TPC_3_1] = GAUDI2_EVENT_TPC9_QM,
0237     [GAUDI2_QUEUE_ID_DCORE1_TPC_3_2] = GAUDI2_EVENT_TPC9_QM,
0238     [GAUDI2_QUEUE_ID_DCORE1_TPC_3_3] = GAUDI2_EVENT_TPC9_QM,
0239     [GAUDI2_QUEUE_ID_DCORE1_TPC_4_0] = GAUDI2_EVENT_TPC10_QM,
0240     [GAUDI2_QUEUE_ID_DCORE1_TPC_4_1] = GAUDI2_EVENT_TPC10_QM,
0241     [GAUDI2_QUEUE_ID_DCORE1_TPC_4_2] = GAUDI2_EVENT_TPC10_QM,
0242     [GAUDI2_QUEUE_ID_DCORE1_TPC_4_3] = GAUDI2_EVENT_TPC10_QM,
0243     [GAUDI2_QUEUE_ID_DCORE1_TPC_5_0] = GAUDI2_EVENT_TPC11_QM,
0244     [GAUDI2_QUEUE_ID_DCORE1_TPC_5_1] = GAUDI2_EVENT_TPC11_QM,
0245     [GAUDI2_QUEUE_ID_DCORE1_TPC_5_2] = GAUDI2_EVENT_TPC11_QM,
0246     [GAUDI2_QUEUE_ID_DCORE1_TPC_5_3] = GAUDI2_EVENT_TPC11_QM,
0247     [GAUDI2_QUEUE_ID_DCORE2_EDMA_0_0] = GAUDI2_EVENT_HDMA4_QM,
0248     [GAUDI2_QUEUE_ID_DCORE2_EDMA_0_1] = GAUDI2_EVENT_HDMA4_QM,
0249     [GAUDI2_QUEUE_ID_DCORE2_EDMA_0_2] = GAUDI2_EVENT_HDMA4_QM,
0250     [GAUDI2_QUEUE_ID_DCORE2_EDMA_0_3] = GAUDI2_EVENT_HDMA4_QM,
0251     [GAUDI2_QUEUE_ID_DCORE2_EDMA_1_0] = GAUDI2_EVENT_HDMA5_QM,
0252     [GAUDI2_QUEUE_ID_DCORE2_EDMA_1_1] = GAUDI2_EVENT_HDMA5_QM,
0253     [GAUDI2_QUEUE_ID_DCORE2_EDMA_1_2] = GAUDI2_EVENT_HDMA5_QM,
0254     [GAUDI2_QUEUE_ID_DCORE2_EDMA_1_3] = GAUDI2_EVENT_HDMA5_QM,
0255     [GAUDI2_QUEUE_ID_DCORE2_MME_0_0] = GAUDI2_EVENT_MME2_QM,
0256     [GAUDI2_QUEUE_ID_DCORE2_MME_0_1] = GAUDI2_EVENT_MME2_QM,
0257     [GAUDI2_QUEUE_ID_DCORE2_MME_0_2] = GAUDI2_EVENT_MME2_QM,
0258     [GAUDI2_QUEUE_ID_DCORE2_MME_0_3] = GAUDI2_EVENT_MME2_QM,
0259     [GAUDI2_QUEUE_ID_DCORE2_TPC_0_0] = GAUDI2_EVENT_TPC12_QM,
0260     [GAUDI2_QUEUE_ID_DCORE2_TPC_0_1] = GAUDI2_EVENT_TPC12_QM,
0261     [GAUDI2_QUEUE_ID_DCORE2_TPC_0_2] = GAUDI2_EVENT_TPC12_QM,
0262     [GAUDI2_QUEUE_ID_DCORE2_TPC_0_3] = GAUDI2_EVENT_TPC12_QM,
0263     [GAUDI2_QUEUE_ID_DCORE2_TPC_1_0] = GAUDI2_EVENT_TPC13_QM,
0264     [GAUDI2_QUEUE_ID_DCORE2_TPC_1_1] = GAUDI2_EVENT_TPC13_QM,
0265     [GAUDI2_QUEUE_ID_DCORE2_TPC_1_2] = GAUDI2_EVENT_TPC13_QM,
0266     [GAUDI2_QUEUE_ID_DCORE2_TPC_1_3] = GAUDI2_EVENT_TPC13_QM,
0267     [GAUDI2_QUEUE_ID_DCORE2_TPC_2_0] = GAUDI2_EVENT_TPC14_QM,
0268     [GAUDI2_QUEUE_ID_DCORE2_TPC_2_1] = GAUDI2_EVENT_TPC14_QM,
0269     [GAUDI2_QUEUE_ID_DCORE2_TPC_2_2] = GAUDI2_EVENT_TPC14_QM,
0270     [GAUDI2_QUEUE_ID_DCORE2_TPC_2_3] = GAUDI2_EVENT_TPC14_QM,
0271     [GAUDI2_QUEUE_ID_DCORE2_TPC_3_0] = GAUDI2_EVENT_TPC15_QM,
0272     [GAUDI2_QUEUE_ID_DCORE2_TPC_3_1] = GAUDI2_EVENT_TPC15_QM,
0273     [GAUDI2_QUEUE_ID_DCORE2_TPC_3_2] = GAUDI2_EVENT_TPC15_QM,
0274     [GAUDI2_QUEUE_ID_DCORE2_TPC_3_3] = GAUDI2_EVENT_TPC15_QM,
0275     [GAUDI2_QUEUE_ID_DCORE2_TPC_4_0] = GAUDI2_EVENT_TPC16_QM,
0276     [GAUDI2_QUEUE_ID_DCORE2_TPC_4_1] = GAUDI2_EVENT_TPC16_QM,
0277     [GAUDI2_QUEUE_ID_DCORE2_TPC_4_2] = GAUDI2_EVENT_TPC16_QM,
0278     [GAUDI2_QUEUE_ID_DCORE2_TPC_4_3] = GAUDI2_EVENT_TPC16_QM,
0279     [GAUDI2_QUEUE_ID_DCORE2_TPC_5_0] = GAUDI2_EVENT_TPC17_QM,
0280     [GAUDI2_QUEUE_ID_DCORE2_TPC_5_1] = GAUDI2_EVENT_TPC17_QM,
0281     [GAUDI2_QUEUE_ID_DCORE2_TPC_5_2] = GAUDI2_EVENT_TPC17_QM,
0282     [GAUDI2_QUEUE_ID_DCORE2_TPC_5_3] = GAUDI2_EVENT_TPC17_QM,
0283     [GAUDI2_QUEUE_ID_DCORE3_EDMA_0_0] = GAUDI2_EVENT_HDMA6_QM,
0284     [GAUDI2_QUEUE_ID_DCORE3_EDMA_0_1] = GAUDI2_EVENT_HDMA6_QM,
0285     [GAUDI2_QUEUE_ID_DCORE3_EDMA_0_2] = GAUDI2_EVENT_HDMA6_QM,
0286     [GAUDI2_QUEUE_ID_DCORE3_EDMA_0_3] = GAUDI2_EVENT_HDMA6_QM,
0287     [GAUDI2_QUEUE_ID_DCORE3_EDMA_1_0] = GAUDI2_EVENT_HDMA7_QM,
0288     [GAUDI2_QUEUE_ID_DCORE3_EDMA_1_1] = GAUDI2_EVENT_HDMA7_QM,
0289     [GAUDI2_QUEUE_ID_DCORE3_EDMA_1_2] = GAUDI2_EVENT_HDMA7_QM,
0290     [GAUDI2_QUEUE_ID_DCORE3_EDMA_1_3] = GAUDI2_EVENT_HDMA7_QM,
0291     [GAUDI2_QUEUE_ID_DCORE3_MME_0_0] = GAUDI2_EVENT_MME3_QM,
0292     [GAUDI2_QUEUE_ID_DCORE3_MME_0_1] = GAUDI2_EVENT_MME3_QM,
0293     [GAUDI2_QUEUE_ID_DCORE3_MME_0_2] = GAUDI2_EVENT_MME3_QM,
0294     [GAUDI2_QUEUE_ID_DCORE3_MME_0_3] = GAUDI2_EVENT_MME3_QM,
0295     [GAUDI2_QUEUE_ID_DCORE3_TPC_0_0] = GAUDI2_EVENT_TPC18_QM,
0296     [GAUDI2_QUEUE_ID_DCORE3_TPC_0_1] = GAUDI2_EVENT_TPC18_QM,
0297     [GAUDI2_QUEUE_ID_DCORE3_TPC_0_2] = GAUDI2_EVENT_TPC18_QM,
0298     [GAUDI2_QUEUE_ID_DCORE3_TPC_0_3] = GAUDI2_EVENT_TPC18_QM,
0299     [GAUDI2_QUEUE_ID_DCORE3_TPC_1_0] = GAUDI2_EVENT_TPC19_QM,
0300     [GAUDI2_QUEUE_ID_DCORE3_TPC_1_1] = GAUDI2_EVENT_TPC19_QM,
0301     [GAUDI2_QUEUE_ID_DCORE3_TPC_1_2] = GAUDI2_EVENT_TPC19_QM,
0302     [GAUDI2_QUEUE_ID_DCORE3_TPC_1_3] = GAUDI2_EVENT_TPC19_QM,
0303     [GAUDI2_QUEUE_ID_DCORE3_TPC_2_0] = GAUDI2_EVENT_TPC20_QM,
0304     [GAUDI2_QUEUE_ID_DCORE3_TPC_2_1] = GAUDI2_EVENT_TPC20_QM,
0305     [GAUDI2_QUEUE_ID_DCORE3_TPC_2_2] = GAUDI2_EVENT_TPC20_QM,
0306     [GAUDI2_QUEUE_ID_DCORE3_TPC_2_3] = GAUDI2_EVENT_TPC20_QM,
0307     [GAUDI2_QUEUE_ID_DCORE3_TPC_3_0] = GAUDI2_EVENT_TPC21_QM,
0308     [GAUDI2_QUEUE_ID_DCORE3_TPC_3_1] = GAUDI2_EVENT_TPC21_QM,
0309     [GAUDI2_QUEUE_ID_DCORE3_TPC_3_2] = GAUDI2_EVENT_TPC21_QM,
0310     [GAUDI2_QUEUE_ID_DCORE3_TPC_3_3] = GAUDI2_EVENT_TPC21_QM,
0311     [GAUDI2_QUEUE_ID_DCORE3_TPC_4_0] = GAUDI2_EVENT_TPC22_QM,
0312     [GAUDI2_QUEUE_ID_DCORE3_TPC_4_1] = GAUDI2_EVENT_TPC22_QM,
0313     [GAUDI2_QUEUE_ID_DCORE3_TPC_4_2] = GAUDI2_EVENT_TPC22_QM,
0314     [GAUDI2_QUEUE_ID_DCORE3_TPC_4_3] = GAUDI2_EVENT_TPC22_QM,
0315     [GAUDI2_QUEUE_ID_DCORE3_TPC_5_0] = GAUDI2_EVENT_TPC23_QM,
0316     [GAUDI2_QUEUE_ID_DCORE3_TPC_5_1] = GAUDI2_EVENT_TPC23_QM,
0317     [GAUDI2_QUEUE_ID_DCORE3_TPC_5_2] = GAUDI2_EVENT_TPC23_QM,
0318     [GAUDI2_QUEUE_ID_DCORE3_TPC_5_3] = GAUDI2_EVENT_TPC23_QM,
0319     [GAUDI2_QUEUE_ID_NIC_0_0] = GAUDI2_EVENT_NIC0_QM0,
0320     [GAUDI2_QUEUE_ID_NIC_0_1] = GAUDI2_EVENT_NIC0_QM0,
0321     [GAUDI2_QUEUE_ID_NIC_0_2] = GAUDI2_EVENT_NIC0_QM0,
0322     [GAUDI2_QUEUE_ID_NIC_0_3] = GAUDI2_EVENT_NIC0_QM0,
0323     [GAUDI2_QUEUE_ID_NIC_1_0] = GAUDI2_EVENT_NIC0_QM1,
0324     [GAUDI2_QUEUE_ID_NIC_1_1] = GAUDI2_EVENT_NIC0_QM1,
0325     [GAUDI2_QUEUE_ID_NIC_1_2] = GAUDI2_EVENT_NIC0_QM1,
0326     [GAUDI2_QUEUE_ID_NIC_1_3] = GAUDI2_EVENT_NIC0_QM1,
0327     [GAUDI2_QUEUE_ID_NIC_2_0] = GAUDI2_EVENT_NIC1_QM0,
0328     [GAUDI2_QUEUE_ID_NIC_2_1] = GAUDI2_EVENT_NIC1_QM0,
0329     [GAUDI2_QUEUE_ID_NIC_2_2] = GAUDI2_EVENT_NIC1_QM0,
0330     [GAUDI2_QUEUE_ID_NIC_2_3] = GAUDI2_EVENT_NIC1_QM0,
0331     [GAUDI2_QUEUE_ID_NIC_3_0] = GAUDI2_EVENT_NIC1_QM1,
0332     [GAUDI2_QUEUE_ID_NIC_3_1] = GAUDI2_EVENT_NIC1_QM1,
0333     [GAUDI2_QUEUE_ID_NIC_3_2] = GAUDI2_EVENT_NIC1_QM1,
0334     [GAUDI2_QUEUE_ID_NIC_3_3] = GAUDI2_EVENT_NIC1_QM1,
0335     [GAUDI2_QUEUE_ID_NIC_4_0] = GAUDI2_EVENT_NIC2_QM0,
0336     [GAUDI2_QUEUE_ID_NIC_4_1] = GAUDI2_EVENT_NIC2_QM0,
0337     [GAUDI2_QUEUE_ID_NIC_4_2] = GAUDI2_EVENT_NIC2_QM0,
0338     [GAUDI2_QUEUE_ID_NIC_4_3] = GAUDI2_EVENT_NIC2_QM0,
0339     [GAUDI2_QUEUE_ID_NIC_5_0] = GAUDI2_EVENT_NIC2_QM1,
0340     [GAUDI2_QUEUE_ID_NIC_5_1] = GAUDI2_EVENT_NIC2_QM1,
0341     [GAUDI2_QUEUE_ID_NIC_5_2] = GAUDI2_EVENT_NIC2_QM1,
0342     [GAUDI2_QUEUE_ID_NIC_5_3] = GAUDI2_EVENT_NIC2_QM1,
0343     [GAUDI2_QUEUE_ID_NIC_6_0] = GAUDI2_EVENT_NIC3_QM0,
0344     [GAUDI2_QUEUE_ID_NIC_6_1] = GAUDI2_EVENT_NIC3_QM0,
0345     [GAUDI2_QUEUE_ID_NIC_6_2] = GAUDI2_EVENT_NIC3_QM0,
0346     [GAUDI2_QUEUE_ID_NIC_6_3] = GAUDI2_EVENT_NIC3_QM0,
0347     [GAUDI2_QUEUE_ID_NIC_7_0] = GAUDI2_EVENT_NIC3_QM1,
0348     [GAUDI2_QUEUE_ID_NIC_7_1] = GAUDI2_EVENT_NIC3_QM1,
0349     [GAUDI2_QUEUE_ID_NIC_7_2] = GAUDI2_EVENT_NIC3_QM1,
0350     [GAUDI2_QUEUE_ID_NIC_7_3] = GAUDI2_EVENT_NIC3_QM1,
0351     [GAUDI2_QUEUE_ID_NIC_8_0] = GAUDI2_EVENT_NIC4_QM0,
0352     [GAUDI2_QUEUE_ID_NIC_8_1] = GAUDI2_EVENT_NIC4_QM0,
0353     [GAUDI2_QUEUE_ID_NIC_8_2] = GAUDI2_EVENT_NIC4_QM0,
0354     [GAUDI2_QUEUE_ID_NIC_8_3] = GAUDI2_EVENT_NIC4_QM0,
0355     [GAUDI2_QUEUE_ID_NIC_9_0] = GAUDI2_EVENT_NIC4_QM1,
0356     [GAUDI2_QUEUE_ID_NIC_9_1] = GAUDI2_EVENT_NIC4_QM1,
0357     [GAUDI2_QUEUE_ID_NIC_9_2] = GAUDI2_EVENT_NIC4_QM1,
0358     [GAUDI2_QUEUE_ID_NIC_9_3] = GAUDI2_EVENT_NIC4_QM1,
0359     [GAUDI2_QUEUE_ID_NIC_10_0] = GAUDI2_EVENT_NIC5_QM0,
0360     [GAUDI2_QUEUE_ID_NIC_10_1] = GAUDI2_EVENT_NIC5_QM0,
0361     [GAUDI2_QUEUE_ID_NIC_10_2] = GAUDI2_EVENT_NIC5_QM0,
0362     [GAUDI2_QUEUE_ID_NIC_10_3] = GAUDI2_EVENT_NIC5_QM0,
0363     [GAUDI2_QUEUE_ID_NIC_11_0] = GAUDI2_EVENT_NIC5_QM1,
0364     [GAUDI2_QUEUE_ID_NIC_11_1] = GAUDI2_EVENT_NIC5_QM1,
0365     [GAUDI2_QUEUE_ID_NIC_11_2] = GAUDI2_EVENT_NIC5_QM1,
0366     [GAUDI2_QUEUE_ID_NIC_11_3] = GAUDI2_EVENT_NIC5_QM1,
0367     [GAUDI2_QUEUE_ID_NIC_12_0] = GAUDI2_EVENT_NIC6_QM0,
0368     [GAUDI2_QUEUE_ID_NIC_12_1] = GAUDI2_EVENT_NIC6_QM0,
0369     [GAUDI2_QUEUE_ID_NIC_12_2] = GAUDI2_EVENT_NIC6_QM0,
0370     [GAUDI2_QUEUE_ID_NIC_12_3] = GAUDI2_EVENT_NIC6_QM0,
0371     [GAUDI2_QUEUE_ID_NIC_13_0] = GAUDI2_EVENT_NIC6_QM1,
0372     [GAUDI2_QUEUE_ID_NIC_13_1] = GAUDI2_EVENT_NIC6_QM1,
0373     [GAUDI2_QUEUE_ID_NIC_13_2] = GAUDI2_EVENT_NIC6_QM1,
0374     [GAUDI2_QUEUE_ID_NIC_13_3] = GAUDI2_EVENT_NIC6_QM1,
0375     [GAUDI2_QUEUE_ID_NIC_14_0] = GAUDI2_EVENT_NIC7_QM0,
0376     [GAUDI2_QUEUE_ID_NIC_14_1] = GAUDI2_EVENT_NIC7_QM0,
0377     [GAUDI2_QUEUE_ID_NIC_14_2] = GAUDI2_EVENT_NIC7_QM0,
0378     [GAUDI2_QUEUE_ID_NIC_14_3] = GAUDI2_EVENT_NIC7_QM0,
0379     [GAUDI2_QUEUE_ID_NIC_15_0] = GAUDI2_EVENT_NIC7_QM1,
0380     [GAUDI2_QUEUE_ID_NIC_15_1] = GAUDI2_EVENT_NIC7_QM1,
0381     [GAUDI2_QUEUE_ID_NIC_15_2] = GAUDI2_EVENT_NIC7_QM1,
0382     [GAUDI2_QUEUE_ID_NIC_15_3] = GAUDI2_EVENT_NIC7_QM1,
0383     [GAUDI2_QUEUE_ID_NIC_16_0] = GAUDI2_EVENT_NIC8_QM0,
0384     [GAUDI2_QUEUE_ID_NIC_16_1] = GAUDI2_EVENT_NIC8_QM0,
0385     [GAUDI2_QUEUE_ID_NIC_16_2] = GAUDI2_EVENT_NIC8_QM0,
0386     [GAUDI2_QUEUE_ID_NIC_16_3] = GAUDI2_EVENT_NIC8_QM0,
0387     [GAUDI2_QUEUE_ID_NIC_17_0] = GAUDI2_EVENT_NIC8_QM1,
0388     [GAUDI2_QUEUE_ID_NIC_17_1] = GAUDI2_EVENT_NIC8_QM1,
0389     [GAUDI2_QUEUE_ID_NIC_17_2] = GAUDI2_EVENT_NIC8_QM1,
0390     [GAUDI2_QUEUE_ID_NIC_17_3] = GAUDI2_EVENT_NIC8_QM1,
0391     [GAUDI2_QUEUE_ID_NIC_18_0] = GAUDI2_EVENT_NIC9_QM0,
0392     [GAUDI2_QUEUE_ID_NIC_18_1] = GAUDI2_EVENT_NIC9_QM0,
0393     [GAUDI2_QUEUE_ID_NIC_18_2] = GAUDI2_EVENT_NIC9_QM0,
0394     [GAUDI2_QUEUE_ID_NIC_18_3] = GAUDI2_EVENT_NIC9_QM0,
0395     [GAUDI2_QUEUE_ID_NIC_19_0] = GAUDI2_EVENT_NIC9_QM1,
0396     [GAUDI2_QUEUE_ID_NIC_19_1] = GAUDI2_EVENT_NIC9_QM1,
0397     [GAUDI2_QUEUE_ID_NIC_19_2] = GAUDI2_EVENT_NIC9_QM1,
0398     [GAUDI2_QUEUE_ID_NIC_19_3] = GAUDI2_EVENT_NIC9_QM1,
0399     [GAUDI2_QUEUE_ID_NIC_20_0] = GAUDI2_EVENT_NIC10_QM0,
0400     [GAUDI2_QUEUE_ID_NIC_20_1] = GAUDI2_EVENT_NIC10_QM0,
0401     [GAUDI2_QUEUE_ID_NIC_20_2] = GAUDI2_EVENT_NIC10_QM0,
0402     [GAUDI2_QUEUE_ID_NIC_20_3] = GAUDI2_EVENT_NIC10_QM0,
0403     [GAUDI2_QUEUE_ID_NIC_21_0] = GAUDI2_EVENT_NIC10_QM1,
0404     [GAUDI2_QUEUE_ID_NIC_21_1] = GAUDI2_EVENT_NIC10_QM1,
0405     [GAUDI2_QUEUE_ID_NIC_21_2] = GAUDI2_EVENT_NIC10_QM1,
0406     [GAUDI2_QUEUE_ID_NIC_21_3] = GAUDI2_EVENT_NIC10_QM1,
0407     [GAUDI2_QUEUE_ID_NIC_22_0] = GAUDI2_EVENT_NIC11_QM0,
0408     [GAUDI2_QUEUE_ID_NIC_22_1] = GAUDI2_EVENT_NIC11_QM0,
0409     [GAUDI2_QUEUE_ID_NIC_22_2] = GAUDI2_EVENT_NIC11_QM0,
0410     [GAUDI2_QUEUE_ID_NIC_22_3] = GAUDI2_EVENT_NIC11_QM0,
0411     [GAUDI2_QUEUE_ID_NIC_23_0] = GAUDI2_EVENT_NIC11_QM1,
0412     [GAUDI2_QUEUE_ID_NIC_23_1] = GAUDI2_EVENT_NIC11_QM1,
0413     [GAUDI2_QUEUE_ID_NIC_23_2] = GAUDI2_EVENT_NIC11_QM1,
0414     [GAUDI2_QUEUE_ID_NIC_23_3] = GAUDI2_EVENT_NIC11_QM1,
0415     [GAUDI2_QUEUE_ID_ROT_0_0] = GAUDI2_EVENT_ROTATOR0_ROT0_QM,
0416     [GAUDI2_QUEUE_ID_ROT_0_1] = GAUDI2_EVENT_ROTATOR0_ROT0_QM,
0417     [GAUDI2_QUEUE_ID_ROT_0_2] = GAUDI2_EVENT_ROTATOR0_ROT0_QM,
0418     [GAUDI2_QUEUE_ID_ROT_0_3] = GAUDI2_EVENT_ROTATOR0_ROT0_QM,
0419     [GAUDI2_QUEUE_ID_ROT_1_0] = GAUDI2_EVENT_ROTATOR1_ROT1_QM,
0420     [GAUDI2_QUEUE_ID_ROT_1_1] = GAUDI2_EVENT_ROTATOR1_ROT1_QM,
0421     [GAUDI2_QUEUE_ID_ROT_1_2] = GAUDI2_EVENT_ROTATOR1_ROT1_QM,
0422     [GAUDI2_QUEUE_ID_ROT_1_3] = GAUDI2_EVENT_ROTATOR1_ROT1_QM
0423 };
0424 
0425 static const int gaudi2_dma_core_async_event_id[] = {
0426     [DMA_CORE_ID_EDMA0] = GAUDI2_EVENT_HDMA0_CORE,
0427     [DMA_CORE_ID_EDMA1] = GAUDI2_EVENT_HDMA1_CORE,
0428     [DMA_CORE_ID_EDMA2] = GAUDI2_EVENT_HDMA2_CORE,
0429     [DMA_CORE_ID_EDMA3] = GAUDI2_EVENT_HDMA3_CORE,
0430     [DMA_CORE_ID_EDMA4] = GAUDI2_EVENT_HDMA4_CORE,
0431     [DMA_CORE_ID_EDMA5] = GAUDI2_EVENT_HDMA5_CORE,
0432     [DMA_CORE_ID_EDMA6] = GAUDI2_EVENT_HDMA6_CORE,
0433     [DMA_CORE_ID_EDMA7] = GAUDI2_EVENT_HDMA7_CORE,
0434     [DMA_CORE_ID_PDMA0] = GAUDI2_EVENT_PDMA0_CORE,
0435     [DMA_CORE_ID_PDMA1] = GAUDI2_EVENT_PDMA1_CORE,
0436     [DMA_CORE_ID_KDMA] = GAUDI2_EVENT_KDMA0_CORE,
0437 };
0438 
0439 static const char * const gaudi2_qm_sei_error_cause[GAUDI2_NUM_OF_QM_SEI_ERR_CAUSE] = {
0440     "qman sei intr",
0441     "arc sei intr"
0442 };
0443 
0444 static const char * const gaudi2_cpu_sei_error_cause[GAUDI2_NUM_OF_CPU_SEI_ERR_CAUSE] = {
0445     "AXI_TERMINATOR WR",
0446     "AXI_TERMINATOR RD",
0447     "AXI SPLIT SEI Status"
0448 };
0449 
0450 static const char * const gaudi2_arc_sei_error_cause[GAUDI2_NUM_OF_ARC_SEI_ERR_CAUSE] = {
0451     "cbu_bresp_sei_intr_cause",
0452     "cbu_rresp_sei_intr_cause",
0453     "lbu_bresp_sei_intr_cause",
0454     "lbu_rresp_sei_intr_cause",
0455     "cbu_axi_split_intr_cause",
0456     "lbu_axi_split_intr_cause",
0457     "arc_ip_excptn_sei_intr_cause",
0458     "dmi_bresp_sei_intr_cause",
0459     "aux2apb_err_sei_intr_cause",
0460     "cfg_lbw_wr_terminated_intr_cause",
0461     "cfg_lbw_rd_terminated_intr_cause",
0462     "cfg_dccm_wr_terminated_intr_cause",
0463     "cfg_dccm_rd_terminated_intr_cause",
0464     "cfg_hbw_rd_terminated_intr_cause"
0465 };
0466 
0467 static const char * const gaudi2_dec_error_cause[GAUDI2_NUM_OF_DEC_ERR_CAUSE] = {
0468     "msix_vcd_hbw_sei",
0469     "msix_l2c_hbw_sei",
0470     "msix_nrm_hbw_sei",
0471     "msix_abnrm_hbw_sei",
0472     "msix_vcd_lbw_sei",
0473     "msix_l2c_lbw_sei",
0474     "msix_nrm_lbw_sei",
0475     "msix_abnrm_lbw_sei",
0476     "apb_vcd_lbw_sei",
0477     "apb_l2c_lbw_sei",
0478     "apb_nrm_lbw_sei",
0479     "apb_abnrm_lbw_sei",
0480     "dec_sei",
0481     "dec_apb_sei",
0482     "trc_apb_sei",
0483     "lbw_mstr_if_sei",
0484     "axi_split_bresp_err_sei",
0485     "hbw_axi_wr_viol_sei",
0486     "hbw_axi_rd_viol_sei",
0487     "lbw_axi_wr_viol_sei",
0488     "lbw_axi_rd_viol_sei",
0489     "vcd_spi",
0490     "l2c_spi",
0491     "nrm_spi",
0492     "abnrm_spi",
0493 };
0494 
0495 static const char * const gaudi2_qman_error_cause[GAUDI2_NUM_OF_QM_ERR_CAUSE] = {
0496     "PQ AXI HBW error",
0497     "CQ AXI HBW error",
0498     "CP AXI HBW error",
0499     "CP error due to undefined OPCODE",
0500     "CP encountered STOP OPCODE",
0501     "CP AXI LBW error",
0502     "CP WRREG32 or WRBULK returned error",
0503     "N/A",
0504     "FENCE 0 inc over max value and clipped",
0505     "FENCE 1 inc over max value and clipped",
0506     "FENCE 2 inc over max value and clipped",
0507     "FENCE 3 inc over max value and clipped",
0508     "FENCE 0 dec under min value and clipped",
0509     "FENCE 1 dec under min value and clipped",
0510     "FENCE 2 dec under min value and clipped",
0511     "FENCE 3 dec under min value and clipped",
0512     "CPDMA Up overflow",
0513     "PQC L2H error"
0514 };
0515 
0516 static const char * const gaudi2_qman_lower_cp_error_cause[GAUDI2_NUM_OF_QM_LCP_ERR_CAUSE] = {
0517     "RSVD0",
0518     "CQ AXI HBW error",
0519     "CP AXI HBW error",
0520     "CP error due to undefined OPCODE",
0521     "CP encountered STOP OPCODE",
0522     "CP AXI LBW error",
0523     "CP WRREG32 or WRBULK returned error",
0524     "N/A",
0525     "FENCE 0 inc over max value and clipped",
0526     "FENCE 1 inc over max value and clipped",
0527     "FENCE 2 inc over max value and clipped",
0528     "FENCE 3 inc over max value and clipped",
0529     "FENCE 0 dec under min value and clipped",
0530     "FENCE 1 dec under min value and clipped",
0531     "FENCE 2 dec under min value and clipped",
0532     "FENCE 3 dec under min value and clipped",
0533     "CPDMA Up overflow",
0534     "RSVD17",
0535     "CQ_WR_IFIFO_CI_ERR",
0536     "CQ_WR_CTL_CI_ERR",
0537     "ARC_CQF_RD_ERR",
0538     "ARC_CQ_WR_IFIFO_CI_ERR",
0539     "ARC_CQ_WR_CTL_CI_ERR",
0540     "ARC_AXI_ERR",
0541     "CP_SWITCH_WDT_ERR"
0542 };
0543 
0544 static const char * const gaudi2_qman_arb_error_cause[GAUDI2_NUM_OF_QM_ARB_ERR_CAUSE] = {
0545     "Choice push while full error",
0546     "Choice Q watchdog error",
0547     "MSG AXI LBW returned with error"
0548 };
0549 
0550 static const char * const guadi2_rot_error_cause[GAUDI2_NUM_OF_ROT_ERR_CAUSE] = {
0551     "qm_axi_err",
0552     "qm_trace_fence_events",
0553     "qm_sw_err",
0554     "qm_cp_sw_stop",
0555     "lbw_mstr_rresp_err",
0556     "lbw_mstr_bresp_err",
0557     "lbw_msg_slverr",
0558     "hbw_msg_slverr",
0559     "wbc_slverr",
0560     "hbw_mstr_rresp_err",
0561     "hbw_mstr_bresp_err",
0562     "sb_resp_intr",
0563     "mrsb_resp_intr",
0564     "core_dw_status_0",
0565     "core_dw_status_1",
0566     "core_dw_status_2",
0567     "core_dw_status_3",
0568     "core_dw_status_4",
0569     "core_dw_status_5",
0570     "core_dw_status_6",
0571     "core_dw_status_7",
0572     "async_arc2cpu_sei_intr",
0573 };
0574 
0575 static const char * const gaudi2_tpc_interrupts_cause[GAUDI2_NUM_OF_TPC_INTR_CAUSE] = {
0576     "tpc_address_exceed_slm",
0577     "tpc_div_by_0",
0578     "tpc_spu_mac_overflow",
0579     "tpc_spu_addsub_overflow",
0580     "tpc_spu_abs_overflow",
0581     "tpc_spu_fma_fp_dst_nan",
0582     "tpc_spu_fma_fp_dst_inf",
0583     "tpc_spu_convert_fp_dst_nan",
0584     "tpc_spu_convert_fp_dst_inf",
0585     "tpc_spu_fp_dst_denorm",
0586     "tpc_vpu_mac_overflow",
0587     "tpc_vpu_addsub_overflow",
0588     "tpc_vpu_abs_overflow",
0589     "tpc_vpu_convert_fp_dst_nan",
0590     "tpc_vpu_convert_fp_dst_inf",
0591     "tpc_vpu_fma_fp_dst_nan",
0592     "tpc_vpu_fma_fp_dst_inf",
0593     "tpc_vpu_fp_dst_denorm",
0594     "tpc_assertions",
0595     "tpc_illegal_instruction",
0596     "tpc_pc_wrap_around",
0597     "tpc_qm_sw_err",
0598     "tpc_hbw_rresp_err",
0599     "tpc_hbw_bresp_err",
0600     "tpc_lbw_rresp_err",
0601     "tpc_lbw_bresp_err",
0602     "st_unlock_already_locked",
0603     "invalid_lock_access",
0604     "LD_L protection violation",
0605     "ST_L protection violation",
0606 };
0607 
0608 static const char * const guadi2_mme_error_cause[GAUDI2_NUM_OF_MME_ERR_CAUSE] = {
0609     "agu_resp_intr",
0610     "qman_axi_err",
0611     "wap sei (wbc axi err)",
0612     "arc sei",
0613     "mme_cfg_unalign_addr",
0614     "qm_sw_err",
0615     "sbte_dbg_intr_0",
0616     "sbte_dbg_intr_1",
0617     "sbte_dbg_intr_2",
0618     "sbte_dbg_intr_3",
0619     "sbte_dbg_intr_4",
0620     "sbte_prtn_intr_0",
0621     "sbte_prtn_intr_1",
0622     "sbte_prtn_intr_2",
0623     "sbte_prtn_intr_3",
0624     "sbte_prtn_intr_4",
0625 };
0626 
0627 static const char * const guadi2_mme_sbte_error_cause[GAUDI2_NUM_OF_MME_SBTE_ERR_CAUSE] = {
0628     "i0",
0629     "i1",
0630     "i2",
0631     "i3",
0632     "i4",
0633 };
0634 
0635 static const char * const guadi2_mme_wap_error_cause[GAUDI2_NUM_OF_MME_WAP_ERR_CAUSE] = {
0636     "WBC ERR RESP_0",
0637     "WBC ERR RESP_1",
0638     "AP SOURCE POS INF",
0639     "AP SOURCE NEG INF",
0640     "AP SOURCE NAN",
0641     "AP RESULT POS INF",
0642     "AP RESULT NEG INF",
0643 };
0644 
0645 static const char * const gaudi2_dma_core_interrupts_cause[GAUDI2_NUM_OF_DMA_CORE_INTR_CAUSE] = {
0646     "HBW Read returned with error RRESP",
0647     "HBW write returned with error BRESP",
0648     "LBW write returned with error BRESP",
0649     "descriptor_fifo_overflow",
0650     "KDMA SB LBW Read returned with error",
0651     "KDMA WBC LBW Write returned with error",
0652     "TRANSPOSE ENGINE DESC FIFO OVERFLOW",
0653     "WRONG CFG FOR COMMIT IN LIN DMA"
0654 };
0655 
0656 static const char * const gaudi2_kdma_core_interrupts_cause[GAUDI2_NUM_OF_DMA_CORE_INTR_CAUSE] = {
0657     "HBW/LBW Read returned with error RRESP",
0658     "HBW/LBW write returned with error BRESP",
0659     "LBW write returned with error BRESP",
0660     "descriptor_fifo_overflow",
0661     "KDMA SB LBW Read returned with error",
0662     "KDMA WBC LBW Write returned with error",
0663     "TRANSPOSE ENGINE DESC FIFO OVERFLOW",
0664     "WRONG CFG FOR COMMIT IN LIN DMA"
0665 };
0666 
0667 struct gaudi2_sm_sei_cause_data {
0668     const char *cause_name;
0669     const char *log_name;
0670     u32 log_mask;
0671 };
0672 
0673 static const struct gaudi2_sm_sei_cause_data
0674 gaudi2_sm_sei_cause[GAUDI2_NUM_OF_SM_SEI_ERR_CAUSE] = {
0675     {"calculated SO value overflow/underflow", "SOB group ID", 0x7FF},
0676     {"payload address of monitor is not aligned to 4B", "monitor addr", 0xFFFF},
0677     {"armed monitor write got BRESP (SLVERR or DECERR)", "AXI id", 0xFFFF},
0678 };
0679 
0680 static const char * const
0681 gaudi2_pmmu_fatal_interrupts_cause[GAUDI2_NUM_OF_PMMU_FATAL_ERR_CAUSE] = {
0682     "LATENCY_RD_OUT_FIFO_OVERRUN",
0683     "LATENCY_WR_OUT_FIFO_OVERRUN",
0684 };
0685 
0686 static const char * const
0687 gaudi2_hif_fatal_interrupts_cause[GAUDI2_NUM_OF_HIF_FATAL_ERR_CAUSE] = {
0688     "LATENCY_RD_OUT_FIFO_OVERRUN",
0689     "LATENCY_WR_OUT_FIFO_OVERRUN",
0690 };
0691 
0692 static const char * const
0693 gaudi2_psoc_axi_drain_interrupts_cause[GAUDI2_NUM_OF_AXI_DRAIN_ERR_CAUSE] = {
0694     "AXI drain HBW",
0695     "AXI drain LBW",
0696 };
0697 
0698 static const char * const
0699 gaudi2_pcie_addr_dec_error_cause[GAUDI2_NUM_OF_PCIE_ADDR_DEC_ERR_CAUSE] = {
0700     "HBW error response",
0701     "LBW error response",
0702     "TLP is blocked by RR"
0703 };
0704 
0705 const u32 gaudi2_qm_blocks_bases[GAUDI2_QUEUE_ID_SIZE] = {
0706     [GAUDI2_QUEUE_ID_PDMA_0_0] = mmPDMA0_QM_BASE,
0707     [GAUDI2_QUEUE_ID_PDMA_0_1] = mmPDMA0_QM_BASE,
0708     [GAUDI2_QUEUE_ID_PDMA_0_2] = mmPDMA0_QM_BASE,
0709     [GAUDI2_QUEUE_ID_PDMA_0_3] = mmPDMA0_QM_BASE,
0710     [GAUDI2_QUEUE_ID_PDMA_1_0] = mmPDMA1_QM_BASE,
0711     [GAUDI2_QUEUE_ID_PDMA_1_1] = mmPDMA1_QM_BASE,
0712     [GAUDI2_QUEUE_ID_PDMA_1_2] = mmPDMA1_QM_BASE,
0713     [GAUDI2_QUEUE_ID_PDMA_1_3] = mmPDMA1_QM_BASE,
0714     [GAUDI2_QUEUE_ID_DCORE0_EDMA_0_0] = mmDCORE0_EDMA0_QM_BASE,
0715     [GAUDI2_QUEUE_ID_DCORE0_EDMA_0_1] = mmDCORE0_EDMA0_QM_BASE,
0716     [GAUDI2_QUEUE_ID_DCORE0_EDMA_0_2] = mmDCORE0_EDMA0_QM_BASE,
0717     [GAUDI2_QUEUE_ID_DCORE0_EDMA_0_3] = mmDCORE0_EDMA0_QM_BASE,
0718     [GAUDI2_QUEUE_ID_DCORE0_EDMA_1_0] = mmDCORE0_EDMA1_QM_BASE,
0719     [GAUDI2_QUEUE_ID_DCORE0_EDMA_1_1] = mmDCORE0_EDMA1_QM_BASE,
0720     [GAUDI2_QUEUE_ID_DCORE0_EDMA_1_2] = mmDCORE0_EDMA1_QM_BASE,
0721     [GAUDI2_QUEUE_ID_DCORE0_EDMA_1_3] = mmDCORE0_EDMA1_QM_BASE,
0722     [GAUDI2_QUEUE_ID_DCORE0_MME_0_0] = mmDCORE0_MME_QM_BASE,
0723     [GAUDI2_QUEUE_ID_DCORE0_MME_0_1] = mmDCORE0_MME_QM_BASE,
0724     [GAUDI2_QUEUE_ID_DCORE0_MME_0_2] = mmDCORE0_MME_QM_BASE,
0725     [GAUDI2_QUEUE_ID_DCORE0_MME_0_3] = mmDCORE0_MME_QM_BASE,
0726     [GAUDI2_QUEUE_ID_DCORE0_TPC_0_0] = mmDCORE0_TPC0_QM_BASE,
0727     [GAUDI2_QUEUE_ID_DCORE0_TPC_0_1] = mmDCORE0_TPC0_QM_BASE,
0728     [GAUDI2_QUEUE_ID_DCORE0_TPC_0_2] = mmDCORE0_TPC0_QM_BASE,
0729     [GAUDI2_QUEUE_ID_DCORE0_TPC_0_3] = mmDCORE0_TPC0_QM_BASE,
0730     [GAUDI2_QUEUE_ID_DCORE0_TPC_1_0] = mmDCORE0_TPC1_QM_BASE,
0731     [GAUDI2_QUEUE_ID_DCORE0_TPC_1_1] = mmDCORE0_TPC1_QM_BASE,
0732     [GAUDI2_QUEUE_ID_DCORE0_TPC_1_2] = mmDCORE0_TPC1_QM_BASE,
0733     [GAUDI2_QUEUE_ID_DCORE0_TPC_1_3] = mmDCORE0_TPC1_QM_BASE,
0734     [GAUDI2_QUEUE_ID_DCORE0_TPC_2_0] = mmDCORE0_TPC2_QM_BASE,
0735     [GAUDI2_QUEUE_ID_DCORE0_TPC_2_1] = mmDCORE0_TPC2_QM_BASE,
0736     [GAUDI2_QUEUE_ID_DCORE0_TPC_2_2] = mmDCORE0_TPC2_QM_BASE,
0737     [GAUDI2_QUEUE_ID_DCORE0_TPC_2_3] = mmDCORE0_TPC2_QM_BASE,
0738     [GAUDI2_QUEUE_ID_DCORE0_TPC_3_0] = mmDCORE0_TPC3_QM_BASE,
0739     [GAUDI2_QUEUE_ID_DCORE0_TPC_3_1] = mmDCORE0_TPC3_QM_BASE,
0740     [GAUDI2_QUEUE_ID_DCORE0_TPC_3_2] = mmDCORE0_TPC3_QM_BASE,
0741     [GAUDI2_QUEUE_ID_DCORE0_TPC_3_3] = mmDCORE0_TPC3_QM_BASE,
0742     [GAUDI2_QUEUE_ID_DCORE0_TPC_4_0] = mmDCORE0_TPC4_QM_BASE,
0743     [GAUDI2_QUEUE_ID_DCORE0_TPC_4_1] = mmDCORE0_TPC4_QM_BASE,
0744     [GAUDI2_QUEUE_ID_DCORE0_TPC_4_2] = mmDCORE0_TPC4_QM_BASE,
0745     [GAUDI2_QUEUE_ID_DCORE0_TPC_4_3] = mmDCORE0_TPC4_QM_BASE,
0746     [GAUDI2_QUEUE_ID_DCORE0_TPC_5_0] = mmDCORE0_TPC5_QM_BASE,
0747     [GAUDI2_QUEUE_ID_DCORE0_TPC_5_1] = mmDCORE0_TPC5_QM_BASE,
0748     [GAUDI2_QUEUE_ID_DCORE0_TPC_5_2] = mmDCORE0_TPC5_QM_BASE,
0749     [GAUDI2_QUEUE_ID_DCORE0_TPC_5_3] = mmDCORE0_TPC5_QM_BASE,
0750     [GAUDI2_QUEUE_ID_DCORE0_TPC_6_0] = mmDCORE0_TPC6_QM_BASE,
0751     [GAUDI2_QUEUE_ID_DCORE0_TPC_6_1] = mmDCORE0_TPC6_QM_BASE,
0752     [GAUDI2_QUEUE_ID_DCORE0_TPC_6_2] = mmDCORE0_TPC6_QM_BASE,
0753     [GAUDI2_QUEUE_ID_DCORE0_TPC_6_3] = mmDCORE0_TPC6_QM_BASE,
0754     [GAUDI2_QUEUE_ID_DCORE1_EDMA_0_0] = mmDCORE1_EDMA0_QM_BASE,
0755     [GAUDI2_QUEUE_ID_DCORE1_EDMA_0_1] = mmDCORE1_EDMA0_QM_BASE,
0756     [GAUDI2_QUEUE_ID_DCORE1_EDMA_0_2] = mmDCORE1_EDMA0_QM_BASE,
0757     [GAUDI2_QUEUE_ID_DCORE1_EDMA_0_3] = mmDCORE1_EDMA0_QM_BASE,
0758     [GAUDI2_QUEUE_ID_DCORE1_EDMA_1_0] = mmDCORE1_EDMA1_QM_BASE,
0759     [GAUDI2_QUEUE_ID_DCORE1_EDMA_1_1] = mmDCORE1_EDMA1_QM_BASE,
0760     [GAUDI2_QUEUE_ID_DCORE1_EDMA_1_2] = mmDCORE1_EDMA1_QM_BASE,
0761     [GAUDI2_QUEUE_ID_DCORE1_EDMA_1_3] = mmDCORE1_EDMA1_QM_BASE,
0762     [GAUDI2_QUEUE_ID_DCORE1_MME_0_0] = mmDCORE1_MME_QM_BASE,
0763     [GAUDI2_QUEUE_ID_DCORE1_MME_0_1] = mmDCORE1_MME_QM_BASE,
0764     [GAUDI2_QUEUE_ID_DCORE1_MME_0_2] = mmDCORE1_MME_QM_BASE,
0765     [GAUDI2_QUEUE_ID_DCORE1_MME_0_3] = mmDCORE1_MME_QM_BASE,
0766     [GAUDI2_QUEUE_ID_DCORE1_TPC_0_0] = mmDCORE1_TPC0_QM_BASE,
0767     [GAUDI2_QUEUE_ID_DCORE1_TPC_0_1] = mmDCORE1_TPC0_QM_BASE,
0768     [GAUDI2_QUEUE_ID_DCORE1_TPC_0_2] = mmDCORE1_TPC0_QM_BASE,
0769     [GAUDI2_QUEUE_ID_DCORE1_TPC_0_3] = mmDCORE1_TPC0_QM_BASE,
0770     [GAUDI2_QUEUE_ID_DCORE1_TPC_1_0] = mmDCORE1_TPC1_QM_BASE,
0771     [GAUDI2_QUEUE_ID_DCORE1_TPC_1_1] = mmDCORE1_TPC1_QM_BASE,
0772     [GAUDI2_QUEUE_ID_DCORE1_TPC_1_2] = mmDCORE1_TPC1_QM_BASE,
0773     [GAUDI2_QUEUE_ID_DCORE1_TPC_1_3] = mmDCORE1_TPC1_QM_BASE,
0774     [GAUDI2_QUEUE_ID_DCORE1_TPC_2_0] = mmDCORE1_TPC2_QM_BASE,
0775     [GAUDI2_QUEUE_ID_DCORE1_TPC_2_1] = mmDCORE1_TPC2_QM_BASE,
0776     [GAUDI2_QUEUE_ID_DCORE1_TPC_2_2] = mmDCORE1_TPC2_QM_BASE,
0777     [GAUDI2_QUEUE_ID_DCORE1_TPC_2_3] = mmDCORE1_TPC2_QM_BASE,
0778     [GAUDI2_QUEUE_ID_DCORE1_TPC_3_0] = mmDCORE1_TPC3_QM_BASE,
0779     [GAUDI2_QUEUE_ID_DCORE1_TPC_3_1] = mmDCORE1_TPC3_QM_BASE,
0780     [GAUDI2_QUEUE_ID_DCORE1_TPC_3_2] = mmDCORE1_TPC3_QM_BASE,
0781     [GAUDI2_QUEUE_ID_DCORE1_TPC_3_3] = mmDCORE1_TPC3_QM_BASE,
0782     [GAUDI2_QUEUE_ID_DCORE1_TPC_4_0] = mmDCORE1_TPC4_QM_BASE,
0783     [GAUDI2_QUEUE_ID_DCORE1_TPC_4_1] = mmDCORE1_TPC4_QM_BASE,
0784     [GAUDI2_QUEUE_ID_DCORE1_TPC_4_2] = mmDCORE1_TPC4_QM_BASE,
0785     [GAUDI2_QUEUE_ID_DCORE1_TPC_4_3] = mmDCORE1_TPC4_QM_BASE,
0786     [GAUDI2_QUEUE_ID_DCORE1_TPC_5_0] = mmDCORE1_TPC5_QM_BASE,
0787     [GAUDI2_QUEUE_ID_DCORE1_TPC_5_1] = mmDCORE1_TPC5_QM_BASE,
0788     [GAUDI2_QUEUE_ID_DCORE1_TPC_5_2] = mmDCORE1_TPC5_QM_BASE,
0789     [GAUDI2_QUEUE_ID_DCORE1_TPC_5_3] = mmDCORE1_TPC5_QM_BASE,
0790     [GAUDI2_QUEUE_ID_DCORE2_EDMA_0_0] = mmDCORE2_EDMA0_QM_BASE,
0791     [GAUDI2_QUEUE_ID_DCORE2_EDMA_0_1] = mmDCORE2_EDMA0_QM_BASE,
0792     [GAUDI2_QUEUE_ID_DCORE2_EDMA_0_2] = mmDCORE2_EDMA0_QM_BASE,
0793     [GAUDI2_QUEUE_ID_DCORE2_EDMA_0_3] = mmDCORE2_EDMA0_QM_BASE,
0794     [GAUDI2_QUEUE_ID_DCORE2_EDMA_1_0] = mmDCORE2_EDMA1_QM_BASE,
0795     [GAUDI2_QUEUE_ID_DCORE2_EDMA_1_1] = mmDCORE2_EDMA1_QM_BASE,
0796     [GAUDI2_QUEUE_ID_DCORE2_EDMA_1_2] = mmDCORE2_EDMA1_QM_BASE,
0797     [GAUDI2_QUEUE_ID_DCORE2_EDMA_1_3] = mmDCORE2_EDMA1_QM_BASE,
0798     [GAUDI2_QUEUE_ID_DCORE2_MME_0_0] = mmDCORE2_MME_QM_BASE,
0799     [GAUDI2_QUEUE_ID_DCORE2_MME_0_1] = mmDCORE2_MME_QM_BASE,
0800     [GAUDI2_QUEUE_ID_DCORE2_MME_0_2] = mmDCORE2_MME_QM_BASE,
0801     [GAUDI2_QUEUE_ID_DCORE2_MME_0_3] = mmDCORE2_MME_QM_BASE,
0802     [GAUDI2_QUEUE_ID_DCORE2_TPC_0_0] = mmDCORE2_TPC0_QM_BASE,
0803     [GAUDI2_QUEUE_ID_DCORE2_TPC_0_1] = mmDCORE2_TPC0_QM_BASE,
0804     [GAUDI2_QUEUE_ID_DCORE2_TPC_0_2] = mmDCORE2_TPC0_QM_BASE,
0805     [GAUDI2_QUEUE_ID_DCORE2_TPC_0_3] = mmDCORE2_TPC0_QM_BASE,
0806     [GAUDI2_QUEUE_ID_DCORE2_TPC_1_0] = mmDCORE2_TPC1_QM_BASE,
0807     [GAUDI2_QUEUE_ID_DCORE2_TPC_1_1] = mmDCORE2_TPC1_QM_BASE,
0808     [GAUDI2_QUEUE_ID_DCORE2_TPC_1_2] = mmDCORE2_TPC1_QM_BASE,
0809     [GAUDI2_QUEUE_ID_DCORE2_TPC_1_3] = mmDCORE2_TPC1_QM_BASE,
0810     [GAUDI2_QUEUE_ID_DCORE2_TPC_2_0] = mmDCORE2_TPC2_QM_BASE,
0811     [GAUDI2_QUEUE_ID_DCORE2_TPC_2_1] = mmDCORE2_TPC2_QM_BASE,
0812     [GAUDI2_QUEUE_ID_DCORE2_TPC_2_2] = mmDCORE2_TPC2_QM_BASE,
0813     [GAUDI2_QUEUE_ID_DCORE2_TPC_2_3] = mmDCORE2_TPC2_QM_BASE,
0814     [GAUDI2_QUEUE_ID_DCORE2_TPC_3_0] = mmDCORE2_TPC3_QM_BASE,
0815     [GAUDI2_QUEUE_ID_DCORE2_TPC_3_1] = mmDCORE2_TPC3_QM_BASE,
0816     [GAUDI2_QUEUE_ID_DCORE2_TPC_3_2] = mmDCORE2_TPC3_QM_BASE,
0817     [GAUDI2_QUEUE_ID_DCORE2_TPC_3_3] = mmDCORE2_TPC3_QM_BASE,
0818     [GAUDI2_QUEUE_ID_DCORE2_TPC_4_0] = mmDCORE2_TPC4_QM_BASE,
0819     [GAUDI2_QUEUE_ID_DCORE2_TPC_4_1] = mmDCORE2_TPC4_QM_BASE,
0820     [GAUDI2_QUEUE_ID_DCORE2_TPC_4_2] = mmDCORE2_TPC4_QM_BASE,
0821     [GAUDI2_QUEUE_ID_DCORE2_TPC_4_3] = mmDCORE2_TPC4_QM_BASE,
0822     [GAUDI2_QUEUE_ID_DCORE2_TPC_5_0] = mmDCORE2_TPC5_QM_BASE,
0823     [GAUDI2_QUEUE_ID_DCORE2_TPC_5_1] = mmDCORE2_TPC5_QM_BASE,
0824     [GAUDI2_QUEUE_ID_DCORE2_TPC_5_2] = mmDCORE2_TPC5_QM_BASE,
0825     [GAUDI2_QUEUE_ID_DCORE2_TPC_5_3] = mmDCORE2_TPC5_QM_BASE,
0826     [GAUDI2_QUEUE_ID_DCORE3_EDMA_0_0] = mmDCORE3_EDMA0_QM_BASE,
0827     [GAUDI2_QUEUE_ID_DCORE3_EDMA_0_1] = mmDCORE3_EDMA0_QM_BASE,
0828     [GAUDI2_QUEUE_ID_DCORE3_EDMA_0_2] = mmDCORE3_EDMA0_QM_BASE,
0829     [GAUDI2_QUEUE_ID_DCORE3_EDMA_0_3] = mmDCORE3_EDMA0_QM_BASE,
0830     [GAUDI2_QUEUE_ID_DCORE3_EDMA_1_0] = mmDCORE3_EDMA1_QM_BASE,
0831     [GAUDI2_QUEUE_ID_DCORE3_EDMA_1_1] = mmDCORE3_EDMA1_QM_BASE,
0832     [GAUDI2_QUEUE_ID_DCORE3_EDMA_1_2] = mmDCORE3_EDMA1_QM_BASE,
0833     [GAUDI2_QUEUE_ID_DCORE3_EDMA_1_3] = mmDCORE3_EDMA1_QM_BASE,
0834     [GAUDI2_QUEUE_ID_DCORE3_MME_0_0] = mmDCORE3_MME_QM_BASE,
0835     [GAUDI2_QUEUE_ID_DCORE3_MME_0_1] = mmDCORE3_MME_QM_BASE,
0836     [GAUDI2_QUEUE_ID_DCORE3_MME_0_2] = mmDCORE3_MME_QM_BASE,
0837     [GAUDI2_QUEUE_ID_DCORE3_MME_0_3] = mmDCORE3_MME_QM_BASE,
0838     [GAUDI2_QUEUE_ID_DCORE3_TPC_0_0] = mmDCORE3_TPC0_QM_BASE,
0839     [GAUDI2_QUEUE_ID_DCORE3_TPC_0_1] = mmDCORE3_TPC0_QM_BASE,
0840     [GAUDI2_QUEUE_ID_DCORE3_TPC_0_2] = mmDCORE3_TPC0_QM_BASE,
0841     [GAUDI2_QUEUE_ID_DCORE3_TPC_0_3] = mmDCORE3_TPC0_QM_BASE,
0842     [GAUDI2_QUEUE_ID_DCORE3_TPC_1_0] = mmDCORE3_TPC1_QM_BASE,
0843     [GAUDI2_QUEUE_ID_DCORE3_TPC_1_1] = mmDCORE3_TPC1_QM_BASE,
0844     [GAUDI2_QUEUE_ID_DCORE3_TPC_1_2] = mmDCORE3_TPC1_QM_BASE,
0845     [GAUDI2_QUEUE_ID_DCORE3_TPC_1_3] = mmDCORE3_TPC1_QM_BASE,
0846     [GAUDI2_QUEUE_ID_DCORE3_TPC_2_0] = mmDCORE3_TPC2_QM_BASE,
0847     [GAUDI2_QUEUE_ID_DCORE3_TPC_2_1] = mmDCORE3_TPC2_QM_BASE,
0848     [GAUDI2_QUEUE_ID_DCORE3_TPC_2_2] = mmDCORE3_TPC2_QM_BASE,
0849     [GAUDI2_QUEUE_ID_DCORE3_TPC_2_3] = mmDCORE3_TPC2_QM_BASE,
0850     [GAUDI2_QUEUE_ID_DCORE3_TPC_3_0] = mmDCORE3_TPC3_QM_BASE,
0851     [GAUDI2_QUEUE_ID_DCORE3_TPC_3_1] = mmDCORE3_TPC3_QM_BASE,
0852     [GAUDI2_QUEUE_ID_DCORE3_TPC_3_2] = mmDCORE3_TPC3_QM_BASE,
0853     [GAUDI2_QUEUE_ID_DCORE3_TPC_3_3] = mmDCORE3_TPC3_QM_BASE,
0854     [GAUDI2_QUEUE_ID_DCORE3_TPC_4_0] = mmDCORE3_TPC4_QM_BASE,
0855     [GAUDI2_QUEUE_ID_DCORE3_TPC_4_1] = mmDCORE3_TPC4_QM_BASE,
0856     [GAUDI2_QUEUE_ID_DCORE3_TPC_4_2] = mmDCORE3_TPC4_QM_BASE,
0857     [GAUDI2_QUEUE_ID_DCORE3_TPC_4_3] = mmDCORE3_TPC4_QM_BASE,
0858     [GAUDI2_QUEUE_ID_DCORE3_TPC_5_0] = mmDCORE3_TPC5_QM_BASE,
0859     [GAUDI2_QUEUE_ID_DCORE3_TPC_5_1] = mmDCORE3_TPC5_QM_BASE,
0860     [GAUDI2_QUEUE_ID_DCORE3_TPC_5_2] = mmDCORE3_TPC5_QM_BASE,
0861     [GAUDI2_QUEUE_ID_DCORE3_TPC_5_3] = mmDCORE3_TPC5_QM_BASE,
0862     [GAUDI2_QUEUE_ID_NIC_0_0] = mmNIC0_QM0_BASE,
0863     [GAUDI2_QUEUE_ID_NIC_0_1] = mmNIC0_QM0_BASE,
0864     [GAUDI2_QUEUE_ID_NIC_0_2] = mmNIC0_QM0_BASE,
0865     [GAUDI2_QUEUE_ID_NIC_0_3] = mmNIC0_QM0_BASE,
0866     [GAUDI2_QUEUE_ID_NIC_1_0] = mmNIC0_QM1_BASE,
0867     [GAUDI2_QUEUE_ID_NIC_1_1] = mmNIC0_QM1_BASE,
0868     [GAUDI2_QUEUE_ID_NIC_1_2] = mmNIC0_QM1_BASE,
0869     [GAUDI2_QUEUE_ID_NIC_1_3] = mmNIC0_QM1_BASE,
0870     [GAUDI2_QUEUE_ID_NIC_2_0] = mmNIC1_QM0_BASE,
0871     [GAUDI2_QUEUE_ID_NIC_2_1] = mmNIC1_QM0_BASE,
0872     [GAUDI2_QUEUE_ID_NIC_2_2] = mmNIC1_QM0_BASE,
0873     [GAUDI2_QUEUE_ID_NIC_2_3] = mmNIC1_QM0_BASE,
0874     [GAUDI2_QUEUE_ID_NIC_3_0] = mmNIC1_QM1_BASE,
0875     [GAUDI2_QUEUE_ID_NIC_3_1] = mmNIC1_QM1_BASE,
0876     [GAUDI2_QUEUE_ID_NIC_3_2] = mmNIC1_QM1_BASE,
0877     [GAUDI2_QUEUE_ID_NIC_3_3] = mmNIC1_QM1_BASE,
0878     [GAUDI2_QUEUE_ID_NIC_4_0] = mmNIC2_QM0_BASE,
0879     [GAUDI2_QUEUE_ID_NIC_4_1] = mmNIC2_QM0_BASE,
0880     [GAUDI2_QUEUE_ID_NIC_4_2] = mmNIC2_QM0_BASE,
0881     [GAUDI2_QUEUE_ID_NIC_4_3] = mmNIC2_QM0_BASE,
0882     [GAUDI2_QUEUE_ID_NIC_5_0] = mmNIC2_QM1_BASE,
0883     [GAUDI2_QUEUE_ID_NIC_5_1] = mmNIC2_QM1_BASE,
0884     [GAUDI2_QUEUE_ID_NIC_5_2] = mmNIC2_QM1_BASE,
0885     [GAUDI2_QUEUE_ID_NIC_5_3] = mmNIC2_QM1_BASE,
0886     [GAUDI2_QUEUE_ID_NIC_6_0] = mmNIC3_QM0_BASE,
0887     [GAUDI2_QUEUE_ID_NIC_6_1] = mmNIC3_QM0_BASE,
0888     [GAUDI2_QUEUE_ID_NIC_6_2] = mmNIC3_QM0_BASE,
0889     [GAUDI2_QUEUE_ID_NIC_6_3] = mmNIC3_QM0_BASE,
0890     [GAUDI2_QUEUE_ID_NIC_7_0] = mmNIC3_QM1_BASE,
0891     [GAUDI2_QUEUE_ID_NIC_7_1] = mmNIC3_QM1_BASE,
0892     [GAUDI2_QUEUE_ID_NIC_7_2] = mmNIC3_QM1_BASE,
0893     [GAUDI2_QUEUE_ID_NIC_7_3] = mmNIC3_QM1_BASE,
0894     [GAUDI2_QUEUE_ID_NIC_8_0] = mmNIC4_QM0_BASE,
0895     [GAUDI2_QUEUE_ID_NIC_8_1] = mmNIC4_QM0_BASE,
0896     [GAUDI2_QUEUE_ID_NIC_8_2] = mmNIC4_QM0_BASE,
0897     [GAUDI2_QUEUE_ID_NIC_8_3] = mmNIC4_QM0_BASE,
0898     [GAUDI2_QUEUE_ID_NIC_9_0] = mmNIC4_QM1_BASE,
0899     [GAUDI2_QUEUE_ID_NIC_9_1] = mmNIC4_QM1_BASE,
0900     [GAUDI2_QUEUE_ID_NIC_9_2] = mmNIC4_QM1_BASE,
0901     [GAUDI2_QUEUE_ID_NIC_9_3] = mmNIC4_QM1_BASE,
0902     [GAUDI2_QUEUE_ID_NIC_10_0] = mmNIC5_QM0_BASE,
0903     [GAUDI2_QUEUE_ID_NIC_10_1] = mmNIC5_QM0_BASE,
0904     [GAUDI2_QUEUE_ID_NIC_10_2] = mmNIC5_QM0_BASE,
0905     [GAUDI2_QUEUE_ID_NIC_10_3] = mmNIC5_QM0_BASE,
0906     [GAUDI2_QUEUE_ID_NIC_11_0] = mmNIC5_QM1_BASE,
0907     [GAUDI2_QUEUE_ID_NIC_11_1] = mmNIC5_QM1_BASE,
0908     [GAUDI2_QUEUE_ID_NIC_11_2] = mmNIC5_QM1_BASE,
0909     [GAUDI2_QUEUE_ID_NIC_11_3] = mmNIC5_QM1_BASE,
0910     [GAUDI2_QUEUE_ID_NIC_12_0] = mmNIC6_QM0_BASE,
0911     [GAUDI2_QUEUE_ID_NIC_12_1] = mmNIC6_QM0_BASE,
0912     [GAUDI2_QUEUE_ID_NIC_12_2] = mmNIC6_QM0_BASE,
0913     [GAUDI2_QUEUE_ID_NIC_12_3] = mmNIC6_QM0_BASE,
0914     [GAUDI2_QUEUE_ID_NIC_13_0] = mmNIC6_QM1_BASE,
0915     [GAUDI2_QUEUE_ID_NIC_13_1] = mmNIC6_QM1_BASE,
0916     [GAUDI2_QUEUE_ID_NIC_13_2] = mmNIC6_QM1_BASE,
0917     [GAUDI2_QUEUE_ID_NIC_13_3] = mmNIC6_QM1_BASE,
0918     [GAUDI2_QUEUE_ID_NIC_14_0] = mmNIC7_QM0_BASE,
0919     [GAUDI2_QUEUE_ID_NIC_14_1] = mmNIC7_QM0_BASE,
0920     [GAUDI2_QUEUE_ID_NIC_14_2] = mmNIC7_QM0_BASE,
0921     [GAUDI2_QUEUE_ID_NIC_14_3] = mmNIC7_QM0_BASE,
0922     [GAUDI2_QUEUE_ID_NIC_15_0] = mmNIC7_QM1_BASE,
0923     [GAUDI2_QUEUE_ID_NIC_15_1] = mmNIC7_QM1_BASE,
0924     [GAUDI2_QUEUE_ID_NIC_15_2] = mmNIC7_QM1_BASE,
0925     [GAUDI2_QUEUE_ID_NIC_15_3] = mmNIC7_QM1_BASE,
0926     [GAUDI2_QUEUE_ID_NIC_16_0] = mmNIC8_QM0_BASE,
0927     [GAUDI2_QUEUE_ID_NIC_16_1] = mmNIC8_QM0_BASE,
0928     [GAUDI2_QUEUE_ID_NIC_16_2] = mmNIC8_QM0_BASE,
0929     [GAUDI2_QUEUE_ID_NIC_16_3] = mmNIC8_QM0_BASE,
0930     [GAUDI2_QUEUE_ID_NIC_17_0] = mmNIC8_QM1_BASE,
0931     [GAUDI2_QUEUE_ID_NIC_17_1] = mmNIC8_QM1_BASE,
0932     [GAUDI2_QUEUE_ID_NIC_17_2] = mmNIC8_QM1_BASE,
0933     [GAUDI2_QUEUE_ID_NIC_17_3] = mmNIC8_QM1_BASE,
0934     [GAUDI2_QUEUE_ID_NIC_18_0] = mmNIC9_QM0_BASE,
0935     [GAUDI2_QUEUE_ID_NIC_18_1] = mmNIC9_QM0_BASE,
0936     [GAUDI2_QUEUE_ID_NIC_18_2] = mmNIC9_QM0_BASE,
0937     [GAUDI2_QUEUE_ID_NIC_18_3] = mmNIC9_QM0_BASE,
0938     [GAUDI2_QUEUE_ID_NIC_19_0] = mmNIC9_QM1_BASE,
0939     [GAUDI2_QUEUE_ID_NIC_19_1] = mmNIC9_QM1_BASE,
0940     [GAUDI2_QUEUE_ID_NIC_19_2] = mmNIC9_QM1_BASE,
0941     [GAUDI2_QUEUE_ID_NIC_19_3] = mmNIC9_QM1_BASE,
0942     [GAUDI2_QUEUE_ID_NIC_20_0] = mmNIC10_QM0_BASE,
0943     [GAUDI2_QUEUE_ID_NIC_20_1] = mmNIC10_QM0_BASE,
0944     [GAUDI2_QUEUE_ID_NIC_20_2] = mmNIC10_QM0_BASE,
0945     [GAUDI2_QUEUE_ID_NIC_20_3] = mmNIC10_QM0_BASE,
0946     [GAUDI2_QUEUE_ID_NIC_21_0] = mmNIC10_QM1_BASE,
0947     [GAUDI2_QUEUE_ID_NIC_21_1] = mmNIC10_QM1_BASE,
0948     [GAUDI2_QUEUE_ID_NIC_21_2] = mmNIC10_QM1_BASE,
0949     [GAUDI2_QUEUE_ID_NIC_21_3] = mmNIC10_QM1_BASE,
0950     [GAUDI2_QUEUE_ID_NIC_22_0] = mmNIC11_QM0_BASE,
0951     [GAUDI2_QUEUE_ID_NIC_22_1] = mmNIC11_QM0_BASE,
0952     [GAUDI2_QUEUE_ID_NIC_22_2] = mmNIC11_QM0_BASE,
0953     [GAUDI2_QUEUE_ID_NIC_22_3] = mmNIC11_QM0_BASE,
0954     [GAUDI2_QUEUE_ID_NIC_23_0] = mmNIC11_QM1_BASE,
0955     [GAUDI2_QUEUE_ID_NIC_23_1] = mmNIC11_QM1_BASE,
0956     [GAUDI2_QUEUE_ID_NIC_23_2] = mmNIC11_QM1_BASE,
0957     [GAUDI2_QUEUE_ID_NIC_23_3] = mmNIC11_QM1_BASE,
0958     [GAUDI2_QUEUE_ID_ROT_0_0] = mmROT0_QM_BASE,
0959     [GAUDI2_QUEUE_ID_ROT_0_1] = mmROT0_QM_BASE,
0960     [GAUDI2_QUEUE_ID_ROT_0_2] = mmROT0_QM_BASE,
0961     [GAUDI2_QUEUE_ID_ROT_0_3] = mmROT0_QM_BASE,
0962     [GAUDI2_QUEUE_ID_ROT_1_0] = mmROT1_QM_BASE,
0963     [GAUDI2_QUEUE_ID_ROT_1_1] = mmROT1_QM_BASE,
0964     [GAUDI2_QUEUE_ID_ROT_1_2] = mmROT1_QM_BASE,
0965     [GAUDI2_QUEUE_ID_ROT_1_3] = mmROT1_QM_BASE
0966 };
0967 
0968 static const u32 gaudi2_arc_blocks_bases[NUM_ARC_CPUS] = {
0969     [CPU_ID_SCHED_ARC0] = mmARC_FARM_ARC0_AUX_BASE,
0970     [CPU_ID_SCHED_ARC1] = mmARC_FARM_ARC1_AUX_BASE,
0971     [CPU_ID_SCHED_ARC2] = mmARC_FARM_ARC2_AUX_BASE,
0972     [CPU_ID_SCHED_ARC3] = mmARC_FARM_ARC3_AUX_BASE,
0973     [CPU_ID_SCHED_ARC4] = mmDCORE1_MME_QM_ARC_AUX_BASE,
0974     [CPU_ID_SCHED_ARC5] = mmDCORE3_MME_QM_ARC_AUX_BASE,
0975     [CPU_ID_TPC_QMAN_ARC0] = mmDCORE0_TPC0_QM_ARC_AUX_BASE,
0976     [CPU_ID_TPC_QMAN_ARC1] = mmDCORE0_TPC1_QM_ARC_AUX_BASE,
0977     [CPU_ID_TPC_QMAN_ARC2] = mmDCORE0_TPC2_QM_ARC_AUX_BASE,
0978     [CPU_ID_TPC_QMAN_ARC3] = mmDCORE0_TPC3_QM_ARC_AUX_BASE,
0979     [CPU_ID_TPC_QMAN_ARC4] = mmDCORE0_TPC4_QM_ARC_AUX_BASE,
0980     [CPU_ID_TPC_QMAN_ARC5] = mmDCORE0_TPC5_QM_ARC_AUX_BASE,
0981     [CPU_ID_TPC_QMAN_ARC6] = mmDCORE1_TPC0_QM_ARC_AUX_BASE,
0982     [CPU_ID_TPC_QMAN_ARC7] = mmDCORE1_TPC1_QM_ARC_AUX_BASE,
0983     [CPU_ID_TPC_QMAN_ARC8] = mmDCORE1_TPC2_QM_ARC_AUX_BASE,
0984     [CPU_ID_TPC_QMAN_ARC9] = mmDCORE1_TPC3_QM_ARC_AUX_BASE,
0985     [CPU_ID_TPC_QMAN_ARC10] = mmDCORE1_TPC4_QM_ARC_AUX_BASE,
0986     [CPU_ID_TPC_QMAN_ARC11] = mmDCORE1_TPC5_QM_ARC_AUX_BASE,
0987     [CPU_ID_TPC_QMAN_ARC12] = mmDCORE2_TPC0_QM_ARC_AUX_BASE,
0988     [CPU_ID_TPC_QMAN_ARC13] = mmDCORE2_TPC1_QM_ARC_AUX_BASE,
0989     [CPU_ID_TPC_QMAN_ARC14] = mmDCORE2_TPC2_QM_ARC_AUX_BASE,
0990     [CPU_ID_TPC_QMAN_ARC15] = mmDCORE2_TPC3_QM_ARC_AUX_BASE,
0991     [CPU_ID_TPC_QMAN_ARC16] = mmDCORE2_TPC4_QM_ARC_AUX_BASE,
0992     [CPU_ID_TPC_QMAN_ARC17] = mmDCORE2_TPC5_QM_ARC_AUX_BASE,
0993     [CPU_ID_TPC_QMAN_ARC18] = mmDCORE3_TPC0_QM_ARC_AUX_BASE,
0994     [CPU_ID_TPC_QMAN_ARC19] = mmDCORE3_TPC1_QM_ARC_AUX_BASE,
0995     [CPU_ID_TPC_QMAN_ARC20] = mmDCORE3_TPC2_QM_ARC_AUX_BASE,
0996     [CPU_ID_TPC_QMAN_ARC21] = mmDCORE3_TPC3_QM_ARC_AUX_BASE,
0997     [CPU_ID_TPC_QMAN_ARC22] = mmDCORE3_TPC4_QM_ARC_AUX_BASE,
0998     [CPU_ID_TPC_QMAN_ARC23] = mmDCORE3_TPC5_QM_ARC_AUX_BASE,
0999     [CPU_ID_TPC_QMAN_ARC24] = mmDCORE0_TPC6_QM_ARC_AUX_BASE,
1000     [CPU_ID_MME_QMAN_ARC0] = mmDCORE0_MME_QM_ARC_AUX_BASE,
1001     [CPU_ID_MME_QMAN_ARC1] = mmDCORE2_MME_QM_ARC_AUX_BASE,
1002     [CPU_ID_EDMA_QMAN_ARC0] = mmDCORE0_EDMA0_QM_ARC_AUX_BASE,
1003     [CPU_ID_EDMA_QMAN_ARC1] = mmDCORE0_EDMA1_QM_ARC_AUX_BASE,
1004     [CPU_ID_EDMA_QMAN_ARC2] = mmDCORE1_EDMA0_QM_ARC_AUX_BASE,
1005     [CPU_ID_EDMA_QMAN_ARC3] = mmDCORE1_EDMA1_QM_ARC_AUX_BASE,
1006     [CPU_ID_EDMA_QMAN_ARC4] = mmDCORE2_EDMA0_QM_ARC_AUX_BASE,
1007     [CPU_ID_EDMA_QMAN_ARC5] = mmDCORE2_EDMA1_QM_ARC_AUX_BASE,
1008     [CPU_ID_EDMA_QMAN_ARC6] = mmDCORE3_EDMA0_QM_ARC_AUX_BASE,
1009     [CPU_ID_EDMA_QMAN_ARC7] = mmDCORE3_EDMA1_QM_ARC_AUX_BASE,
1010     [CPU_ID_PDMA_QMAN_ARC0] = mmPDMA0_QM_ARC_AUX_BASE,
1011     [CPU_ID_PDMA_QMAN_ARC1] = mmPDMA1_QM_ARC_AUX_BASE,
1012     [CPU_ID_ROT_QMAN_ARC0] = mmROT0_QM_ARC_AUX_BASE,
1013     [CPU_ID_ROT_QMAN_ARC1] = mmROT1_QM_ARC_AUX_BASE,
1014     [CPU_ID_NIC_QMAN_ARC0] = mmNIC0_QM_ARC_AUX0_BASE,
1015     [CPU_ID_NIC_QMAN_ARC1] = mmNIC0_QM_ARC_AUX1_BASE,
1016     [CPU_ID_NIC_QMAN_ARC2] = mmNIC1_QM_ARC_AUX0_BASE,
1017     [CPU_ID_NIC_QMAN_ARC3] = mmNIC1_QM_ARC_AUX1_BASE,
1018     [CPU_ID_NIC_QMAN_ARC4] = mmNIC2_QM_ARC_AUX0_BASE,
1019     [CPU_ID_NIC_QMAN_ARC5] = mmNIC2_QM_ARC_AUX1_BASE,
1020     [CPU_ID_NIC_QMAN_ARC6] = mmNIC3_QM_ARC_AUX0_BASE,
1021     [CPU_ID_NIC_QMAN_ARC7] = mmNIC3_QM_ARC_AUX1_BASE,
1022     [CPU_ID_NIC_QMAN_ARC8] = mmNIC4_QM_ARC_AUX0_BASE,
1023     [CPU_ID_NIC_QMAN_ARC9] = mmNIC4_QM_ARC_AUX1_BASE,
1024     [CPU_ID_NIC_QMAN_ARC10] = mmNIC5_QM_ARC_AUX0_BASE,
1025     [CPU_ID_NIC_QMAN_ARC11] = mmNIC5_QM_ARC_AUX1_BASE,
1026     [CPU_ID_NIC_QMAN_ARC12] = mmNIC6_QM_ARC_AUX0_BASE,
1027     [CPU_ID_NIC_QMAN_ARC13] = mmNIC6_QM_ARC_AUX1_BASE,
1028     [CPU_ID_NIC_QMAN_ARC14] = mmNIC7_QM_ARC_AUX0_BASE,
1029     [CPU_ID_NIC_QMAN_ARC15] = mmNIC7_QM_ARC_AUX1_BASE,
1030     [CPU_ID_NIC_QMAN_ARC16] = mmNIC8_QM_ARC_AUX0_BASE,
1031     [CPU_ID_NIC_QMAN_ARC17] = mmNIC8_QM_ARC_AUX1_BASE,
1032     [CPU_ID_NIC_QMAN_ARC18] = mmNIC9_QM_ARC_AUX0_BASE,
1033     [CPU_ID_NIC_QMAN_ARC19] = mmNIC9_QM_ARC_AUX1_BASE,
1034     [CPU_ID_NIC_QMAN_ARC20] = mmNIC10_QM_ARC_AUX0_BASE,
1035     [CPU_ID_NIC_QMAN_ARC21] = mmNIC10_QM_ARC_AUX1_BASE,
1036     [CPU_ID_NIC_QMAN_ARC22] = mmNIC11_QM_ARC_AUX0_BASE,
1037     [CPU_ID_NIC_QMAN_ARC23] = mmNIC11_QM_ARC_AUX1_BASE,
1038 };
1039 
1040 static const u32 gaudi2_arc_dccm_bases[NUM_ARC_CPUS] = {
1041     [CPU_ID_SCHED_ARC0] = mmARC_FARM_ARC0_DCCM0_BASE,
1042     [CPU_ID_SCHED_ARC1] = mmARC_FARM_ARC1_DCCM0_BASE,
1043     [CPU_ID_SCHED_ARC2] = mmARC_FARM_ARC2_DCCM0_BASE,
1044     [CPU_ID_SCHED_ARC3] = mmARC_FARM_ARC3_DCCM0_BASE,
1045     [CPU_ID_SCHED_ARC4] = mmDCORE1_MME_QM_ARC_DCCM_BASE,
1046     [CPU_ID_SCHED_ARC5] = mmDCORE3_MME_QM_ARC_DCCM_BASE,
1047     [CPU_ID_TPC_QMAN_ARC0] = mmDCORE0_TPC0_QM_DCCM_BASE,
1048     [CPU_ID_TPC_QMAN_ARC1] = mmDCORE0_TPC1_QM_DCCM_BASE,
1049     [CPU_ID_TPC_QMAN_ARC2] = mmDCORE0_TPC2_QM_DCCM_BASE,
1050     [CPU_ID_TPC_QMAN_ARC3] = mmDCORE0_TPC3_QM_DCCM_BASE,
1051     [CPU_ID_TPC_QMAN_ARC4] = mmDCORE0_TPC4_QM_DCCM_BASE,
1052     [CPU_ID_TPC_QMAN_ARC5] = mmDCORE0_TPC5_QM_DCCM_BASE,
1053     [CPU_ID_TPC_QMAN_ARC6] = mmDCORE1_TPC0_QM_DCCM_BASE,
1054     [CPU_ID_TPC_QMAN_ARC7] = mmDCORE1_TPC1_QM_DCCM_BASE,
1055     [CPU_ID_TPC_QMAN_ARC8] = mmDCORE1_TPC2_QM_DCCM_BASE,
1056     [CPU_ID_TPC_QMAN_ARC9] = mmDCORE1_TPC3_QM_DCCM_BASE,
1057     [CPU_ID_TPC_QMAN_ARC10] = mmDCORE1_TPC4_QM_DCCM_BASE,
1058     [CPU_ID_TPC_QMAN_ARC11] = mmDCORE1_TPC5_QM_DCCM_BASE,
1059     [CPU_ID_TPC_QMAN_ARC12] = mmDCORE2_TPC0_QM_DCCM_BASE,
1060     [CPU_ID_TPC_QMAN_ARC13] = mmDCORE2_TPC1_QM_DCCM_BASE,
1061     [CPU_ID_TPC_QMAN_ARC14] = mmDCORE2_TPC2_QM_DCCM_BASE,
1062     [CPU_ID_TPC_QMAN_ARC15] = mmDCORE2_TPC3_QM_DCCM_BASE,
1063     [CPU_ID_TPC_QMAN_ARC16] = mmDCORE2_TPC4_QM_DCCM_BASE,
1064     [CPU_ID_TPC_QMAN_ARC17] = mmDCORE2_TPC5_QM_DCCM_BASE,
1065     [CPU_ID_TPC_QMAN_ARC18] = mmDCORE3_TPC0_QM_DCCM_BASE,
1066     [CPU_ID_TPC_QMAN_ARC19] = mmDCORE3_TPC1_QM_DCCM_BASE,
1067     [CPU_ID_TPC_QMAN_ARC20] = mmDCORE3_TPC2_QM_DCCM_BASE,
1068     [CPU_ID_TPC_QMAN_ARC21] = mmDCORE3_TPC3_QM_DCCM_BASE,
1069     [CPU_ID_TPC_QMAN_ARC22] = mmDCORE3_TPC4_QM_DCCM_BASE,
1070     [CPU_ID_TPC_QMAN_ARC23] = mmDCORE3_TPC5_QM_DCCM_BASE,
1071     [CPU_ID_TPC_QMAN_ARC24] = mmDCORE0_TPC6_QM_DCCM_BASE,
1072     [CPU_ID_MME_QMAN_ARC0] = mmDCORE0_MME_QM_ARC_DCCM_BASE,
1073     [CPU_ID_MME_QMAN_ARC1] = mmDCORE2_MME_QM_ARC_DCCM_BASE,
1074     [CPU_ID_EDMA_QMAN_ARC0] = mmDCORE0_EDMA0_QM_DCCM_BASE,
1075     [CPU_ID_EDMA_QMAN_ARC1] = mmDCORE0_EDMA1_QM_DCCM_BASE,
1076     [CPU_ID_EDMA_QMAN_ARC2] = mmDCORE1_EDMA0_QM_DCCM_BASE,
1077     [CPU_ID_EDMA_QMAN_ARC3] = mmDCORE1_EDMA1_QM_DCCM_BASE,
1078     [CPU_ID_EDMA_QMAN_ARC4] = mmDCORE2_EDMA0_QM_DCCM_BASE,
1079     [CPU_ID_EDMA_QMAN_ARC5] = mmDCORE2_EDMA1_QM_DCCM_BASE,
1080     [CPU_ID_EDMA_QMAN_ARC6] = mmDCORE3_EDMA0_QM_DCCM_BASE,
1081     [CPU_ID_EDMA_QMAN_ARC7] = mmDCORE3_EDMA1_QM_DCCM_BASE,
1082     [CPU_ID_PDMA_QMAN_ARC0] = mmPDMA0_QM_ARC_DCCM_BASE,
1083     [CPU_ID_PDMA_QMAN_ARC1] = mmPDMA1_QM_ARC_DCCM_BASE,
1084     [CPU_ID_ROT_QMAN_ARC0] = mmROT0_QM_ARC_DCCM_BASE,
1085     [CPU_ID_ROT_QMAN_ARC1] = mmROT1_QM_ARC_DCCM_BASE,
1086     [CPU_ID_NIC_QMAN_ARC0] = mmNIC0_QM_DCCM0_BASE,
1087     [CPU_ID_NIC_QMAN_ARC1] = mmNIC0_QM_DCCM1_BASE,
1088     [CPU_ID_NIC_QMAN_ARC2] = mmNIC1_QM_DCCM0_BASE,
1089     [CPU_ID_NIC_QMAN_ARC3] = mmNIC1_QM_DCCM1_BASE,
1090     [CPU_ID_NIC_QMAN_ARC4] = mmNIC2_QM_DCCM0_BASE,
1091     [CPU_ID_NIC_QMAN_ARC5] = mmNIC2_QM_DCCM1_BASE,
1092     [CPU_ID_NIC_QMAN_ARC6] = mmNIC3_QM_DCCM0_BASE,
1093     [CPU_ID_NIC_QMAN_ARC7] = mmNIC3_QM_DCCM1_BASE,
1094     [CPU_ID_NIC_QMAN_ARC8] = mmNIC4_QM_DCCM0_BASE,
1095     [CPU_ID_NIC_QMAN_ARC9] = mmNIC4_QM_DCCM1_BASE,
1096     [CPU_ID_NIC_QMAN_ARC10] = mmNIC5_QM_DCCM0_BASE,
1097     [CPU_ID_NIC_QMAN_ARC11] = mmNIC5_QM_DCCM1_BASE,
1098     [CPU_ID_NIC_QMAN_ARC12] = mmNIC6_QM_DCCM0_BASE,
1099     [CPU_ID_NIC_QMAN_ARC13] = mmNIC6_QM_DCCM1_BASE,
1100     [CPU_ID_NIC_QMAN_ARC14] = mmNIC7_QM_DCCM0_BASE,
1101     [CPU_ID_NIC_QMAN_ARC15] = mmNIC7_QM_DCCM1_BASE,
1102     [CPU_ID_NIC_QMAN_ARC16] = mmNIC8_QM_DCCM0_BASE,
1103     [CPU_ID_NIC_QMAN_ARC17] = mmNIC8_QM_DCCM1_BASE,
1104     [CPU_ID_NIC_QMAN_ARC18] = mmNIC9_QM_DCCM0_BASE,
1105     [CPU_ID_NIC_QMAN_ARC19] = mmNIC9_QM_DCCM1_BASE,
1106     [CPU_ID_NIC_QMAN_ARC20] = mmNIC10_QM_DCCM0_BASE,
1107     [CPU_ID_NIC_QMAN_ARC21] = mmNIC10_QM_DCCM1_BASE,
1108     [CPU_ID_NIC_QMAN_ARC22] = mmNIC11_QM_DCCM0_BASE,
1109     [CPU_ID_NIC_QMAN_ARC23] = mmNIC11_QM_DCCM1_BASE,
1110 };
1111 
1112 const u32 gaudi2_mme_ctrl_lo_blocks_bases[MME_ID_SIZE] = {
1113     [MME_ID_DCORE0] = mmDCORE0_MME_CTRL_LO_BASE,
1114     [MME_ID_DCORE1] = mmDCORE1_MME_CTRL_LO_BASE,
1115     [MME_ID_DCORE2] = mmDCORE2_MME_CTRL_LO_BASE,
1116     [MME_ID_DCORE3] = mmDCORE3_MME_CTRL_LO_BASE,
1117 };
1118 
1119 static const u32 gaudi2_queue_id_to_arc_id[GAUDI2_QUEUE_ID_SIZE] = {
1120     [GAUDI2_QUEUE_ID_PDMA_0_0] = CPU_ID_PDMA_QMAN_ARC0,
1121     [GAUDI2_QUEUE_ID_PDMA_0_1] = CPU_ID_PDMA_QMAN_ARC0,
1122     [GAUDI2_QUEUE_ID_PDMA_0_2] = CPU_ID_PDMA_QMAN_ARC0,
1123     [GAUDI2_QUEUE_ID_PDMA_0_3] = CPU_ID_PDMA_QMAN_ARC0,
1124     [GAUDI2_QUEUE_ID_PDMA_1_0] = CPU_ID_PDMA_QMAN_ARC1,
1125     [GAUDI2_QUEUE_ID_PDMA_1_1] = CPU_ID_PDMA_QMAN_ARC1,
1126     [GAUDI2_QUEUE_ID_PDMA_1_2] = CPU_ID_PDMA_QMAN_ARC1,
1127     [GAUDI2_QUEUE_ID_PDMA_1_3] = CPU_ID_PDMA_QMAN_ARC1,
1128     [GAUDI2_QUEUE_ID_DCORE0_EDMA_0_0] = CPU_ID_EDMA_QMAN_ARC0,
1129     [GAUDI2_QUEUE_ID_DCORE0_EDMA_0_1] = CPU_ID_EDMA_QMAN_ARC0,
1130     [GAUDI2_QUEUE_ID_DCORE0_EDMA_0_2] = CPU_ID_EDMA_QMAN_ARC0,
1131     [GAUDI2_QUEUE_ID_DCORE0_EDMA_0_3] = CPU_ID_EDMA_QMAN_ARC0,
1132     [GAUDI2_QUEUE_ID_DCORE0_EDMA_1_0] = CPU_ID_EDMA_QMAN_ARC1,
1133     [GAUDI2_QUEUE_ID_DCORE0_EDMA_1_1] = CPU_ID_EDMA_QMAN_ARC1,
1134     [GAUDI2_QUEUE_ID_DCORE0_EDMA_1_2] = CPU_ID_EDMA_QMAN_ARC1,
1135     [GAUDI2_QUEUE_ID_DCORE0_EDMA_1_3] = CPU_ID_EDMA_QMAN_ARC1,
1136     [GAUDI2_QUEUE_ID_DCORE0_MME_0_0] = CPU_ID_MME_QMAN_ARC0,
1137     [GAUDI2_QUEUE_ID_DCORE0_MME_0_1] = CPU_ID_MME_QMAN_ARC0,
1138     [GAUDI2_QUEUE_ID_DCORE0_MME_0_2] = CPU_ID_MME_QMAN_ARC0,
1139     [GAUDI2_QUEUE_ID_DCORE0_MME_0_3] = CPU_ID_MME_QMAN_ARC0,
1140     [GAUDI2_QUEUE_ID_DCORE0_TPC_0_0] = CPU_ID_TPC_QMAN_ARC0,
1141     [GAUDI2_QUEUE_ID_DCORE0_TPC_0_1] = CPU_ID_TPC_QMAN_ARC0,
1142     [GAUDI2_QUEUE_ID_DCORE0_TPC_0_2] = CPU_ID_TPC_QMAN_ARC0,
1143     [GAUDI2_QUEUE_ID_DCORE0_TPC_0_3] = CPU_ID_TPC_QMAN_ARC0,
1144     [GAUDI2_QUEUE_ID_DCORE0_TPC_1_0] = CPU_ID_TPC_QMAN_ARC1,
1145     [GAUDI2_QUEUE_ID_DCORE0_TPC_1_1] = CPU_ID_TPC_QMAN_ARC1,
1146     [GAUDI2_QUEUE_ID_DCORE0_TPC_1_2] = CPU_ID_TPC_QMAN_ARC1,
1147     [GAUDI2_QUEUE_ID_DCORE0_TPC_1_3] = CPU_ID_TPC_QMAN_ARC1,
1148     [GAUDI2_QUEUE_ID_DCORE0_TPC_2_0] = CPU_ID_TPC_QMAN_ARC2,
1149     [GAUDI2_QUEUE_ID_DCORE0_TPC_2_1] = CPU_ID_TPC_QMAN_ARC2,
1150     [GAUDI2_QUEUE_ID_DCORE0_TPC_2_2] = CPU_ID_TPC_QMAN_ARC2,
1151     [GAUDI2_QUEUE_ID_DCORE0_TPC_2_3] = CPU_ID_TPC_QMAN_ARC2,
1152     [GAUDI2_QUEUE_ID_DCORE0_TPC_3_0] = CPU_ID_TPC_QMAN_ARC3,
1153     [GAUDI2_QUEUE_ID_DCORE0_TPC_3_1] = CPU_ID_TPC_QMAN_ARC3,
1154     [GAUDI2_QUEUE_ID_DCORE0_TPC_3_2] = CPU_ID_TPC_QMAN_ARC3,
1155     [GAUDI2_QUEUE_ID_DCORE0_TPC_3_3] = CPU_ID_TPC_QMAN_ARC3,
1156     [GAUDI2_QUEUE_ID_DCORE0_TPC_4_0] = CPU_ID_TPC_QMAN_ARC4,
1157     [GAUDI2_QUEUE_ID_DCORE0_TPC_4_1] = CPU_ID_TPC_QMAN_ARC4,
1158     [GAUDI2_QUEUE_ID_DCORE0_TPC_4_2] = CPU_ID_TPC_QMAN_ARC4,
1159     [GAUDI2_QUEUE_ID_DCORE0_TPC_4_3] = CPU_ID_TPC_QMAN_ARC4,
1160     [GAUDI2_QUEUE_ID_DCORE0_TPC_5_0] = CPU_ID_TPC_QMAN_ARC5,
1161     [GAUDI2_QUEUE_ID_DCORE0_TPC_5_1] = CPU_ID_TPC_QMAN_ARC5,
1162     [GAUDI2_QUEUE_ID_DCORE0_TPC_5_2] = CPU_ID_TPC_QMAN_ARC5,
1163     [GAUDI2_QUEUE_ID_DCORE0_TPC_5_3] = CPU_ID_TPC_QMAN_ARC5,
1164     [GAUDI2_QUEUE_ID_DCORE0_TPC_6_0] = CPU_ID_TPC_QMAN_ARC24,
1165     [GAUDI2_QUEUE_ID_DCORE0_TPC_6_1] = CPU_ID_TPC_QMAN_ARC24,
1166     [GAUDI2_QUEUE_ID_DCORE0_TPC_6_2] = CPU_ID_TPC_QMAN_ARC24,
1167     [GAUDI2_QUEUE_ID_DCORE0_TPC_6_3] = CPU_ID_TPC_QMAN_ARC24,
1168     [GAUDI2_QUEUE_ID_DCORE1_EDMA_0_0] = CPU_ID_EDMA_QMAN_ARC2,
1169     [GAUDI2_QUEUE_ID_DCORE1_EDMA_0_1] = CPU_ID_EDMA_QMAN_ARC2,
1170     [GAUDI2_QUEUE_ID_DCORE1_EDMA_0_2] = CPU_ID_EDMA_QMAN_ARC2,
1171     [GAUDI2_QUEUE_ID_DCORE1_EDMA_0_3] = CPU_ID_EDMA_QMAN_ARC2,
1172     [GAUDI2_QUEUE_ID_DCORE1_EDMA_1_0] = CPU_ID_EDMA_QMAN_ARC3,
1173     [GAUDI2_QUEUE_ID_DCORE1_EDMA_1_1] = CPU_ID_EDMA_QMAN_ARC3,
1174     [GAUDI2_QUEUE_ID_DCORE1_EDMA_1_2] = CPU_ID_EDMA_QMAN_ARC3,
1175     [GAUDI2_QUEUE_ID_DCORE1_EDMA_1_3] = CPU_ID_EDMA_QMAN_ARC3,
1176     [GAUDI2_QUEUE_ID_DCORE1_MME_0_0] = CPU_ID_SCHED_ARC4,
1177     [GAUDI2_QUEUE_ID_DCORE1_MME_0_1] = CPU_ID_SCHED_ARC4,
1178     [GAUDI2_QUEUE_ID_DCORE1_MME_0_2] = CPU_ID_SCHED_ARC4,
1179     [GAUDI2_QUEUE_ID_DCORE1_MME_0_3] = CPU_ID_SCHED_ARC4,
1180     [GAUDI2_QUEUE_ID_DCORE1_TPC_0_0] = CPU_ID_TPC_QMAN_ARC6,
1181     [GAUDI2_QUEUE_ID_DCORE1_TPC_0_1] = CPU_ID_TPC_QMAN_ARC6,
1182     [GAUDI2_QUEUE_ID_DCORE1_TPC_0_2] = CPU_ID_TPC_QMAN_ARC6,
1183     [GAUDI2_QUEUE_ID_DCORE1_TPC_0_3] = CPU_ID_TPC_QMAN_ARC6,
1184     [GAUDI2_QUEUE_ID_DCORE1_TPC_1_0] = CPU_ID_TPC_QMAN_ARC7,
1185     [GAUDI2_QUEUE_ID_DCORE1_TPC_1_1] = CPU_ID_TPC_QMAN_ARC7,
1186     [GAUDI2_QUEUE_ID_DCORE1_TPC_1_2] = CPU_ID_TPC_QMAN_ARC7,
1187     [GAUDI2_QUEUE_ID_DCORE1_TPC_1_3] = CPU_ID_TPC_QMAN_ARC7,
1188     [GAUDI2_QUEUE_ID_DCORE1_TPC_2_0] = CPU_ID_TPC_QMAN_ARC8,
1189     [GAUDI2_QUEUE_ID_DCORE1_TPC_2_1] = CPU_ID_TPC_QMAN_ARC8,
1190     [GAUDI2_QUEUE_ID_DCORE1_TPC_2_2] = CPU_ID_TPC_QMAN_ARC8,
1191     [GAUDI2_QUEUE_ID_DCORE1_TPC_2_3] = CPU_ID_TPC_QMAN_ARC8,
1192     [GAUDI2_QUEUE_ID_DCORE1_TPC_3_0] = CPU_ID_TPC_QMAN_ARC9,
1193     [GAUDI2_QUEUE_ID_DCORE1_TPC_3_1] = CPU_ID_TPC_QMAN_ARC9,
1194     [GAUDI2_QUEUE_ID_DCORE1_TPC_3_2] = CPU_ID_TPC_QMAN_ARC9,
1195     [GAUDI2_QUEUE_ID_DCORE1_TPC_3_3] = CPU_ID_TPC_QMAN_ARC9,
1196     [GAUDI2_QUEUE_ID_DCORE1_TPC_4_0] = CPU_ID_TPC_QMAN_ARC10,
1197     [GAUDI2_QUEUE_ID_DCORE1_TPC_4_1] = CPU_ID_TPC_QMAN_ARC10,
1198     [GAUDI2_QUEUE_ID_DCORE1_TPC_4_2] = CPU_ID_TPC_QMAN_ARC10,
1199     [GAUDI2_QUEUE_ID_DCORE1_TPC_4_3] = CPU_ID_TPC_QMAN_ARC10,
1200     [GAUDI2_QUEUE_ID_DCORE1_TPC_5_0] = CPU_ID_TPC_QMAN_ARC11,
1201     [GAUDI2_QUEUE_ID_DCORE1_TPC_5_1] = CPU_ID_TPC_QMAN_ARC11,
1202     [GAUDI2_QUEUE_ID_DCORE1_TPC_5_2] = CPU_ID_TPC_QMAN_ARC11,
1203     [GAUDI2_QUEUE_ID_DCORE1_TPC_5_3] = CPU_ID_TPC_QMAN_ARC11,
1204     [GAUDI2_QUEUE_ID_DCORE2_EDMA_0_0] = CPU_ID_EDMA_QMAN_ARC4,
1205     [GAUDI2_QUEUE_ID_DCORE2_EDMA_0_1] = CPU_ID_EDMA_QMAN_ARC4,
1206     [GAUDI2_QUEUE_ID_DCORE2_EDMA_0_2] = CPU_ID_EDMA_QMAN_ARC4,
1207     [GAUDI2_QUEUE_ID_DCORE2_EDMA_0_3] = CPU_ID_EDMA_QMAN_ARC4,
1208     [GAUDI2_QUEUE_ID_DCORE2_EDMA_1_0] = CPU_ID_EDMA_QMAN_ARC5,
1209     [GAUDI2_QUEUE_ID_DCORE2_EDMA_1_1] = CPU_ID_EDMA_QMAN_ARC5,
1210     [GAUDI2_QUEUE_ID_DCORE2_EDMA_1_2] = CPU_ID_EDMA_QMAN_ARC5,
1211     [GAUDI2_QUEUE_ID_DCORE2_EDMA_1_3] = CPU_ID_EDMA_QMAN_ARC5,
1212     [GAUDI2_QUEUE_ID_DCORE2_MME_0_0] = CPU_ID_MME_QMAN_ARC1,
1213     [GAUDI2_QUEUE_ID_DCORE2_MME_0_1] = CPU_ID_MME_QMAN_ARC1,
1214     [GAUDI2_QUEUE_ID_DCORE2_MME_0_2] = CPU_ID_MME_QMAN_ARC1,
1215     [GAUDI2_QUEUE_ID_DCORE2_MME_0_3] = CPU_ID_MME_QMAN_ARC1,
1216     [GAUDI2_QUEUE_ID_DCORE2_TPC_0_0] = CPU_ID_TPC_QMAN_ARC12,
1217     [GAUDI2_QUEUE_ID_DCORE2_TPC_0_1] = CPU_ID_TPC_QMAN_ARC12,
1218     [GAUDI2_QUEUE_ID_DCORE2_TPC_0_2] = CPU_ID_TPC_QMAN_ARC12,
1219     [GAUDI2_QUEUE_ID_DCORE2_TPC_0_3] = CPU_ID_TPC_QMAN_ARC12,
1220     [GAUDI2_QUEUE_ID_DCORE2_TPC_1_0] = CPU_ID_TPC_QMAN_ARC13,
1221     [GAUDI2_QUEUE_ID_DCORE2_TPC_1_1] = CPU_ID_TPC_QMAN_ARC13,
1222     [GAUDI2_QUEUE_ID_DCORE2_TPC_1_2] = CPU_ID_TPC_QMAN_ARC13,
1223     [GAUDI2_QUEUE_ID_DCORE2_TPC_1_3] = CPU_ID_TPC_QMAN_ARC13,
1224     [GAUDI2_QUEUE_ID_DCORE2_TPC_2_0] = CPU_ID_TPC_QMAN_ARC14,
1225     [GAUDI2_QUEUE_ID_DCORE2_TPC_2_1] = CPU_ID_TPC_QMAN_ARC14,
1226     [GAUDI2_QUEUE_ID_DCORE2_TPC_2_2] = CPU_ID_TPC_QMAN_ARC14,
1227     [GAUDI2_QUEUE_ID_DCORE2_TPC_2_3] = CPU_ID_TPC_QMAN_ARC14,
1228     [GAUDI2_QUEUE_ID_DCORE2_TPC_3_0] = CPU_ID_TPC_QMAN_ARC15,
1229     [GAUDI2_QUEUE_ID_DCORE2_TPC_3_1] = CPU_ID_TPC_QMAN_ARC15,
1230     [GAUDI2_QUEUE_ID_DCORE2_TPC_3_2] = CPU_ID_TPC_QMAN_ARC15,
1231     [GAUDI2_QUEUE_ID_DCORE2_TPC_3_3] = CPU_ID_TPC_QMAN_ARC15,
1232     [GAUDI2_QUEUE_ID_DCORE2_TPC_4_0] = CPU_ID_TPC_QMAN_ARC16,
1233     [GAUDI2_QUEUE_ID_DCORE2_TPC_4_1] = CPU_ID_TPC_QMAN_ARC16,
1234     [GAUDI2_QUEUE_ID_DCORE2_TPC_4_2] = CPU_ID_TPC_QMAN_ARC16,
1235     [GAUDI2_QUEUE_ID_DCORE2_TPC_4_3] = CPU_ID_TPC_QMAN_ARC16,
1236     [GAUDI2_QUEUE_ID_DCORE2_TPC_5_0] = CPU_ID_TPC_QMAN_ARC17,
1237     [GAUDI2_QUEUE_ID_DCORE2_TPC_5_1] = CPU_ID_TPC_QMAN_ARC17,
1238     [GAUDI2_QUEUE_ID_DCORE2_TPC_5_2] = CPU_ID_TPC_QMAN_ARC17,
1239     [GAUDI2_QUEUE_ID_DCORE2_TPC_5_3] = CPU_ID_TPC_QMAN_ARC17,
1240     [GAUDI2_QUEUE_ID_DCORE3_EDMA_0_0] = CPU_ID_EDMA_QMAN_ARC6,
1241     [GAUDI2_QUEUE_ID_DCORE3_EDMA_0_1] = CPU_ID_EDMA_QMAN_ARC6,
1242     [GAUDI2_QUEUE_ID_DCORE3_EDMA_0_2] = CPU_ID_EDMA_QMAN_ARC6,
1243     [GAUDI2_QUEUE_ID_DCORE3_EDMA_0_3] = CPU_ID_EDMA_QMAN_ARC6,
1244     [GAUDI2_QUEUE_ID_DCORE3_EDMA_1_0] = CPU_ID_EDMA_QMAN_ARC7,
1245     [GAUDI2_QUEUE_ID_DCORE3_EDMA_1_1] = CPU_ID_EDMA_QMAN_ARC7,
1246     [GAUDI2_QUEUE_ID_DCORE3_EDMA_1_2] = CPU_ID_EDMA_QMAN_ARC7,
1247     [GAUDI2_QUEUE_ID_DCORE3_EDMA_1_3] = CPU_ID_EDMA_QMAN_ARC7,
1248     [GAUDI2_QUEUE_ID_DCORE3_MME_0_0] = CPU_ID_SCHED_ARC5,
1249     [GAUDI2_QUEUE_ID_DCORE3_MME_0_1] = CPU_ID_SCHED_ARC5,
1250     [GAUDI2_QUEUE_ID_DCORE3_MME_0_2] = CPU_ID_SCHED_ARC5,
1251     [GAUDI2_QUEUE_ID_DCORE3_MME_0_3] = CPU_ID_SCHED_ARC5,
1252     [GAUDI2_QUEUE_ID_DCORE3_TPC_0_0] = CPU_ID_TPC_QMAN_ARC18,
1253     [GAUDI2_QUEUE_ID_DCORE3_TPC_0_1] = CPU_ID_TPC_QMAN_ARC18,
1254     [GAUDI2_QUEUE_ID_DCORE3_TPC_0_2] = CPU_ID_TPC_QMAN_ARC18,
1255     [GAUDI2_QUEUE_ID_DCORE3_TPC_0_3] = CPU_ID_TPC_QMAN_ARC18,
1256     [GAUDI2_QUEUE_ID_DCORE3_TPC_1_0] = CPU_ID_TPC_QMAN_ARC19,
1257     [GAUDI2_QUEUE_ID_DCORE3_TPC_1_1] = CPU_ID_TPC_QMAN_ARC19,
1258     [GAUDI2_QUEUE_ID_DCORE3_TPC_1_2] = CPU_ID_TPC_QMAN_ARC19,
1259     [GAUDI2_QUEUE_ID_DCORE3_TPC_1_3] = CPU_ID_TPC_QMAN_ARC19,
1260     [GAUDI2_QUEUE_ID_DCORE3_TPC_2_0] = CPU_ID_TPC_QMAN_ARC20,
1261     [GAUDI2_QUEUE_ID_DCORE3_TPC_2_1] = CPU_ID_TPC_QMAN_ARC20,
1262     [GAUDI2_QUEUE_ID_DCORE3_TPC_2_2] = CPU_ID_TPC_QMAN_ARC20,
1263     [GAUDI2_QUEUE_ID_DCORE3_TPC_2_3] = CPU_ID_TPC_QMAN_ARC20,
1264     [GAUDI2_QUEUE_ID_DCORE3_TPC_3_0] = CPU_ID_TPC_QMAN_ARC21,
1265     [GAUDI2_QUEUE_ID_DCORE3_TPC_3_1] = CPU_ID_TPC_QMAN_ARC21,
1266     [GAUDI2_QUEUE_ID_DCORE3_TPC_3_2] = CPU_ID_TPC_QMAN_ARC21,
1267     [GAUDI2_QUEUE_ID_DCORE3_TPC_3_3] = CPU_ID_TPC_QMAN_ARC21,
1268     [GAUDI2_QUEUE_ID_DCORE3_TPC_4_0] = CPU_ID_TPC_QMAN_ARC22,
1269     [GAUDI2_QUEUE_ID_DCORE3_TPC_4_1] = CPU_ID_TPC_QMAN_ARC22,
1270     [GAUDI2_QUEUE_ID_DCORE3_TPC_4_2] = CPU_ID_TPC_QMAN_ARC22,
1271     [GAUDI2_QUEUE_ID_DCORE3_TPC_4_3] = CPU_ID_TPC_QMAN_ARC22,
1272     [GAUDI2_QUEUE_ID_DCORE3_TPC_5_0] = CPU_ID_TPC_QMAN_ARC23,
1273     [GAUDI2_QUEUE_ID_DCORE3_TPC_5_1] = CPU_ID_TPC_QMAN_ARC23,
1274     [GAUDI2_QUEUE_ID_DCORE3_TPC_5_2] = CPU_ID_TPC_QMAN_ARC23,
1275     [GAUDI2_QUEUE_ID_DCORE3_TPC_5_3] = CPU_ID_TPC_QMAN_ARC23,
1276     [GAUDI2_QUEUE_ID_NIC_0_0] = CPU_ID_NIC_QMAN_ARC0,
1277     [GAUDI2_QUEUE_ID_NIC_0_1] = CPU_ID_NIC_QMAN_ARC0,
1278     [GAUDI2_QUEUE_ID_NIC_0_2] = CPU_ID_NIC_QMAN_ARC0,
1279     [GAUDI2_QUEUE_ID_NIC_0_3] = CPU_ID_NIC_QMAN_ARC0,
1280     [GAUDI2_QUEUE_ID_NIC_1_0] = CPU_ID_NIC_QMAN_ARC1,
1281     [GAUDI2_QUEUE_ID_NIC_1_1] = CPU_ID_NIC_QMAN_ARC1,
1282     [GAUDI2_QUEUE_ID_NIC_1_2] = CPU_ID_NIC_QMAN_ARC1,
1283     [GAUDI2_QUEUE_ID_NIC_1_3] = CPU_ID_NIC_QMAN_ARC1,
1284     [GAUDI2_QUEUE_ID_NIC_2_0] = CPU_ID_NIC_QMAN_ARC2,
1285     [GAUDI2_QUEUE_ID_NIC_2_1] = CPU_ID_NIC_QMAN_ARC2,
1286     [GAUDI2_QUEUE_ID_NIC_2_2] = CPU_ID_NIC_QMAN_ARC2,
1287     [GAUDI2_QUEUE_ID_NIC_2_3] = CPU_ID_NIC_QMAN_ARC2,
1288     [GAUDI2_QUEUE_ID_NIC_3_0] = CPU_ID_NIC_QMAN_ARC3,
1289     [GAUDI2_QUEUE_ID_NIC_3_1] = CPU_ID_NIC_QMAN_ARC3,
1290     [GAUDI2_QUEUE_ID_NIC_3_2] = CPU_ID_NIC_QMAN_ARC3,
1291     [GAUDI2_QUEUE_ID_NIC_3_3] = CPU_ID_NIC_QMAN_ARC3,
1292     [GAUDI2_QUEUE_ID_NIC_4_0] = CPU_ID_NIC_QMAN_ARC4,
1293     [GAUDI2_QUEUE_ID_NIC_4_1] = CPU_ID_NIC_QMAN_ARC4,
1294     [GAUDI2_QUEUE_ID_NIC_4_2] = CPU_ID_NIC_QMAN_ARC4,
1295     [GAUDI2_QUEUE_ID_NIC_4_3] = CPU_ID_NIC_QMAN_ARC4,
1296     [GAUDI2_QUEUE_ID_NIC_5_0] = CPU_ID_NIC_QMAN_ARC5,
1297     [GAUDI2_QUEUE_ID_NIC_5_1] = CPU_ID_NIC_QMAN_ARC5,
1298     [GAUDI2_QUEUE_ID_NIC_5_2] = CPU_ID_NIC_QMAN_ARC5,
1299     [GAUDI2_QUEUE_ID_NIC_5_3] = CPU_ID_NIC_QMAN_ARC5,
1300     [GAUDI2_QUEUE_ID_NIC_6_0] = CPU_ID_NIC_QMAN_ARC6,
1301     [GAUDI2_QUEUE_ID_NIC_6_1] = CPU_ID_NIC_QMAN_ARC6,
1302     [GAUDI2_QUEUE_ID_NIC_6_2] = CPU_ID_NIC_QMAN_ARC6,
1303     [GAUDI2_QUEUE_ID_NIC_6_3] = CPU_ID_NIC_QMAN_ARC6,
1304     [GAUDI2_QUEUE_ID_NIC_7_0] = CPU_ID_NIC_QMAN_ARC7,
1305     [GAUDI2_QUEUE_ID_NIC_7_1] = CPU_ID_NIC_QMAN_ARC7,
1306     [GAUDI2_QUEUE_ID_NIC_7_2] = CPU_ID_NIC_QMAN_ARC7,
1307     [GAUDI2_QUEUE_ID_NIC_7_3] = CPU_ID_NIC_QMAN_ARC7,
1308     [GAUDI2_QUEUE_ID_NIC_8_0] = CPU_ID_NIC_QMAN_ARC8,
1309     [GAUDI2_QUEUE_ID_NIC_8_1] = CPU_ID_NIC_QMAN_ARC8,
1310     [GAUDI2_QUEUE_ID_NIC_8_2] = CPU_ID_NIC_QMAN_ARC8,
1311     [GAUDI2_QUEUE_ID_NIC_8_3] = CPU_ID_NIC_QMAN_ARC8,
1312     [GAUDI2_QUEUE_ID_NIC_9_0] = CPU_ID_NIC_QMAN_ARC9,
1313     [GAUDI2_QUEUE_ID_NIC_9_1] = CPU_ID_NIC_QMAN_ARC9,
1314     [GAUDI2_QUEUE_ID_NIC_9_2] = CPU_ID_NIC_QMAN_ARC9,
1315     [GAUDI2_QUEUE_ID_NIC_9_3] = CPU_ID_NIC_QMAN_ARC9,
1316     [GAUDI2_QUEUE_ID_NIC_10_0] = CPU_ID_NIC_QMAN_ARC10,
1317     [GAUDI2_QUEUE_ID_NIC_10_1] = CPU_ID_NIC_QMAN_ARC10,
1318     [GAUDI2_QUEUE_ID_NIC_10_2] = CPU_ID_NIC_QMAN_ARC10,
1319     [GAUDI2_QUEUE_ID_NIC_10_3] = CPU_ID_NIC_QMAN_ARC10,
1320     [GAUDI2_QUEUE_ID_NIC_11_0] = CPU_ID_NIC_QMAN_ARC11,
1321     [GAUDI2_QUEUE_ID_NIC_11_1] = CPU_ID_NIC_QMAN_ARC11,
1322     [GAUDI2_QUEUE_ID_NIC_11_2] = CPU_ID_NIC_QMAN_ARC11,
1323     [GAUDI2_QUEUE_ID_NIC_11_3] = CPU_ID_NIC_QMAN_ARC11,
1324     [GAUDI2_QUEUE_ID_NIC_12_0] = CPU_ID_NIC_QMAN_ARC12,
1325     [GAUDI2_QUEUE_ID_NIC_12_1] = CPU_ID_NIC_QMAN_ARC12,
1326     [GAUDI2_QUEUE_ID_NIC_12_2] = CPU_ID_NIC_QMAN_ARC12,
1327     [GAUDI2_QUEUE_ID_NIC_12_3] = CPU_ID_NIC_QMAN_ARC12,
1328     [GAUDI2_QUEUE_ID_NIC_13_0] = CPU_ID_NIC_QMAN_ARC13,
1329     [GAUDI2_QUEUE_ID_NIC_13_1] = CPU_ID_NIC_QMAN_ARC13,
1330     [GAUDI2_QUEUE_ID_NIC_13_2] = CPU_ID_NIC_QMAN_ARC13,
1331     [GAUDI2_QUEUE_ID_NIC_13_3] = CPU_ID_NIC_QMAN_ARC13,
1332     [GAUDI2_QUEUE_ID_NIC_14_0] = CPU_ID_NIC_QMAN_ARC14,
1333     [GAUDI2_QUEUE_ID_NIC_14_1] = CPU_ID_NIC_QMAN_ARC14,
1334     [GAUDI2_QUEUE_ID_NIC_14_2] = CPU_ID_NIC_QMAN_ARC14,
1335     [GAUDI2_QUEUE_ID_NIC_14_3] = CPU_ID_NIC_QMAN_ARC14,
1336     [GAUDI2_QUEUE_ID_NIC_15_0] = CPU_ID_NIC_QMAN_ARC15,
1337     [GAUDI2_QUEUE_ID_NIC_15_1] = CPU_ID_NIC_QMAN_ARC15,
1338     [GAUDI2_QUEUE_ID_NIC_15_2] = CPU_ID_NIC_QMAN_ARC15,
1339     [GAUDI2_QUEUE_ID_NIC_15_3] = CPU_ID_NIC_QMAN_ARC15,
1340     [GAUDI2_QUEUE_ID_NIC_16_0] = CPU_ID_NIC_QMAN_ARC16,
1341     [GAUDI2_QUEUE_ID_NIC_16_1] = CPU_ID_NIC_QMAN_ARC16,
1342     [GAUDI2_QUEUE_ID_NIC_16_2] = CPU_ID_NIC_QMAN_ARC16,
1343     [GAUDI2_QUEUE_ID_NIC_16_3] = CPU_ID_NIC_QMAN_ARC16,
1344     [GAUDI2_QUEUE_ID_NIC_17_0] = CPU_ID_NIC_QMAN_ARC17,
1345     [GAUDI2_QUEUE_ID_NIC_17_1] = CPU_ID_NIC_QMAN_ARC17,
1346     [GAUDI2_QUEUE_ID_NIC_17_2] = CPU_ID_NIC_QMAN_ARC17,
1347     [GAUDI2_QUEUE_ID_NIC_17_3] = CPU_ID_NIC_QMAN_ARC17,
1348     [GAUDI2_QUEUE_ID_NIC_18_0] = CPU_ID_NIC_QMAN_ARC18,
1349     [GAUDI2_QUEUE_ID_NIC_18_1] = CPU_ID_NIC_QMAN_ARC18,
1350     [GAUDI2_QUEUE_ID_NIC_18_2] = CPU_ID_NIC_QMAN_ARC18,
1351     [GAUDI2_QUEUE_ID_NIC_18_3] = CPU_ID_NIC_QMAN_ARC18,
1352     [GAUDI2_QUEUE_ID_NIC_19_0] = CPU_ID_NIC_QMAN_ARC19,
1353     [GAUDI2_QUEUE_ID_NIC_19_1] = CPU_ID_NIC_QMAN_ARC19,
1354     [GAUDI2_QUEUE_ID_NIC_19_2] = CPU_ID_NIC_QMAN_ARC19,
1355     [GAUDI2_QUEUE_ID_NIC_19_3] = CPU_ID_NIC_QMAN_ARC19,
1356     [GAUDI2_QUEUE_ID_NIC_20_0] = CPU_ID_NIC_QMAN_ARC20,
1357     [GAUDI2_QUEUE_ID_NIC_20_1] = CPU_ID_NIC_QMAN_ARC20,
1358     [GAUDI2_QUEUE_ID_NIC_20_2] = CPU_ID_NIC_QMAN_ARC20,
1359     [GAUDI2_QUEUE_ID_NIC_20_3] = CPU_ID_NIC_QMAN_ARC20,
1360     [GAUDI2_QUEUE_ID_NIC_21_0] = CPU_ID_NIC_QMAN_ARC21,
1361     [GAUDI2_QUEUE_ID_NIC_21_1] = CPU_ID_NIC_QMAN_ARC21,
1362     [GAUDI2_QUEUE_ID_NIC_21_2] = CPU_ID_NIC_QMAN_ARC21,
1363     [GAUDI2_QUEUE_ID_NIC_21_3] = CPU_ID_NIC_QMAN_ARC21,
1364     [GAUDI2_QUEUE_ID_NIC_22_0] = CPU_ID_NIC_QMAN_ARC22,
1365     [GAUDI2_QUEUE_ID_NIC_22_1] = CPU_ID_NIC_QMAN_ARC22,
1366     [GAUDI2_QUEUE_ID_NIC_22_2] = CPU_ID_NIC_QMAN_ARC22,
1367     [GAUDI2_QUEUE_ID_NIC_22_3] = CPU_ID_NIC_QMAN_ARC22,
1368     [GAUDI2_QUEUE_ID_NIC_23_0] = CPU_ID_NIC_QMAN_ARC23,
1369     [GAUDI2_QUEUE_ID_NIC_23_1] = CPU_ID_NIC_QMAN_ARC23,
1370     [GAUDI2_QUEUE_ID_NIC_23_2] = CPU_ID_NIC_QMAN_ARC23,
1371     [GAUDI2_QUEUE_ID_NIC_23_3] = CPU_ID_NIC_QMAN_ARC23,
1372     [GAUDI2_QUEUE_ID_ROT_0_0] = CPU_ID_ROT_QMAN_ARC0,
1373     [GAUDI2_QUEUE_ID_ROT_0_1] = CPU_ID_ROT_QMAN_ARC0,
1374     [GAUDI2_QUEUE_ID_ROT_0_2] = CPU_ID_ROT_QMAN_ARC0,
1375     [GAUDI2_QUEUE_ID_ROT_0_3] = CPU_ID_ROT_QMAN_ARC0,
1376     [GAUDI2_QUEUE_ID_ROT_1_0] = CPU_ID_ROT_QMAN_ARC1,
1377     [GAUDI2_QUEUE_ID_ROT_1_1] = CPU_ID_ROT_QMAN_ARC1,
1378     [GAUDI2_QUEUE_ID_ROT_1_2] = CPU_ID_ROT_QMAN_ARC1,
1379     [GAUDI2_QUEUE_ID_ROT_1_3] = CPU_ID_ROT_QMAN_ARC1
1380 };
1381 
1382 const u32 gaudi2_dma_core_blocks_bases[DMA_CORE_ID_SIZE] = {
1383     [DMA_CORE_ID_PDMA0] = mmPDMA0_CORE_BASE,
1384     [DMA_CORE_ID_PDMA1] = mmPDMA1_CORE_BASE,
1385     [DMA_CORE_ID_EDMA0] = mmDCORE0_EDMA0_CORE_BASE,
1386     [DMA_CORE_ID_EDMA1] = mmDCORE0_EDMA1_CORE_BASE,
1387     [DMA_CORE_ID_EDMA2] = mmDCORE1_EDMA0_CORE_BASE,
1388     [DMA_CORE_ID_EDMA3] = mmDCORE1_EDMA1_CORE_BASE,
1389     [DMA_CORE_ID_EDMA4] = mmDCORE2_EDMA0_CORE_BASE,
1390     [DMA_CORE_ID_EDMA5] = mmDCORE2_EDMA1_CORE_BASE,
1391     [DMA_CORE_ID_EDMA6] = mmDCORE3_EDMA0_CORE_BASE,
1392     [DMA_CORE_ID_EDMA7] = mmDCORE3_EDMA1_CORE_BASE,
1393     [DMA_CORE_ID_KDMA] = mmARC_FARM_KDMA_BASE
1394 };
1395 
1396 const u32 gaudi2_mme_acc_blocks_bases[MME_ID_SIZE] = {
1397     [MME_ID_DCORE0] = mmDCORE0_MME_ACC_BASE,
1398     [MME_ID_DCORE1] = mmDCORE1_MME_ACC_BASE,
1399     [MME_ID_DCORE2] = mmDCORE2_MME_ACC_BASE,
1400     [MME_ID_DCORE3] = mmDCORE3_MME_ACC_BASE
1401 };
1402 
1403 static const u32 gaudi2_tpc_cfg_blocks_bases[TPC_ID_SIZE] = {
1404     [TPC_ID_DCORE0_TPC0] = mmDCORE0_TPC0_CFG_BASE,
1405     [TPC_ID_DCORE0_TPC1] = mmDCORE0_TPC1_CFG_BASE,
1406     [TPC_ID_DCORE0_TPC2] = mmDCORE0_TPC2_CFG_BASE,
1407     [TPC_ID_DCORE0_TPC3] = mmDCORE0_TPC3_CFG_BASE,
1408     [TPC_ID_DCORE0_TPC4] = mmDCORE0_TPC4_CFG_BASE,
1409     [TPC_ID_DCORE0_TPC5] = mmDCORE0_TPC5_CFG_BASE,
1410     [TPC_ID_DCORE1_TPC0] = mmDCORE1_TPC0_CFG_BASE,
1411     [TPC_ID_DCORE1_TPC1] = mmDCORE1_TPC1_CFG_BASE,
1412     [TPC_ID_DCORE1_TPC2] = mmDCORE1_TPC2_CFG_BASE,
1413     [TPC_ID_DCORE1_TPC3] = mmDCORE1_TPC3_CFG_BASE,
1414     [TPC_ID_DCORE1_TPC4] = mmDCORE1_TPC4_CFG_BASE,
1415     [TPC_ID_DCORE1_TPC5] = mmDCORE1_TPC5_CFG_BASE,
1416     [TPC_ID_DCORE2_TPC0] = mmDCORE2_TPC0_CFG_BASE,
1417     [TPC_ID_DCORE2_TPC1] = mmDCORE2_TPC1_CFG_BASE,
1418     [TPC_ID_DCORE2_TPC2] = mmDCORE2_TPC2_CFG_BASE,
1419     [TPC_ID_DCORE2_TPC3] = mmDCORE2_TPC3_CFG_BASE,
1420     [TPC_ID_DCORE2_TPC4] = mmDCORE2_TPC4_CFG_BASE,
1421     [TPC_ID_DCORE2_TPC5] = mmDCORE2_TPC5_CFG_BASE,
1422     [TPC_ID_DCORE3_TPC0] = mmDCORE3_TPC0_CFG_BASE,
1423     [TPC_ID_DCORE3_TPC1] = mmDCORE3_TPC1_CFG_BASE,
1424     [TPC_ID_DCORE3_TPC2] = mmDCORE3_TPC2_CFG_BASE,
1425     [TPC_ID_DCORE3_TPC3] = mmDCORE3_TPC3_CFG_BASE,
1426     [TPC_ID_DCORE3_TPC4] = mmDCORE3_TPC4_CFG_BASE,
1427     [TPC_ID_DCORE3_TPC5] = mmDCORE3_TPC5_CFG_BASE,
1428     [TPC_ID_DCORE0_TPC6] = mmDCORE0_TPC6_CFG_BASE,
1429 };
1430 
1431 const u32 gaudi2_rot_blocks_bases[ROTATOR_ID_SIZE] = {
1432     [ROTATOR_ID_0] = mmROT0_BASE,
1433     [ROTATOR_ID_1] = mmROT1_BASE
1434 };
1435 
1436 static const u32 gaudi2_tpc_id_to_queue_id[TPC_ID_SIZE] = {
1437     [TPC_ID_DCORE0_TPC0] = GAUDI2_QUEUE_ID_DCORE0_TPC_0_0,
1438     [TPC_ID_DCORE0_TPC1] = GAUDI2_QUEUE_ID_DCORE0_TPC_1_0,
1439     [TPC_ID_DCORE0_TPC2] = GAUDI2_QUEUE_ID_DCORE0_TPC_2_0,
1440     [TPC_ID_DCORE0_TPC3] = GAUDI2_QUEUE_ID_DCORE0_TPC_3_0,
1441     [TPC_ID_DCORE0_TPC4] = GAUDI2_QUEUE_ID_DCORE0_TPC_4_0,
1442     [TPC_ID_DCORE0_TPC5] = GAUDI2_QUEUE_ID_DCORE0_TPC_5_0,
1443     [TPC_ID_DCORE1_TPC0] = GAUDI2_QUEUE_ID_DCORE1_TPC_0_0,
1444     [TPC_ID_DCORE1_TPC1] = GAUDI2_QUEUE_ID_DCORE1_TPC_1_0,
1445     [TPC_ID_DCORE1_TPC2] = GAUDI2_QUEUE_ID_DCORE1_TPC_2_0,
1446     [TPC_ID_DCORE1_TPC3] = GAUDI2_QUEUE_ID_DCORE1_TPC_3_0,
1447     [TPC_ID_DCORE1_TPC4] = GAUDI2_QUEUE_ID_DCORE1_TPC_4_0,
1448     [TPC_ID_DCORE1_TPC5] = GAUDI2_QUEUE_ID_DCORE1_TPC_5_0,
1449     [TPC_ID_DCORE2_TPC0] = GAUDI2_QUEUE_ID_DCORE2_TPC_0_0,
1450     [TPC_ID_DCORE2_TPC1] = GAUDI2_QUEUE_ID_DCORE2_TPC_1_0,
1451     [TPC_ID_DCORE2_TPC2] = GAUDI2_QUEUE_ID_DCORE2_TPC_2_0,
1452     [TPC_ID_DCORE2_TPC3] = GAUDI2_QUEUE_ID_DCORE2_TPC_3_0,
1453     [TPC_ID_DCORE2_TPC4] = GAUDI2_QUEUE_ID_DCORE2_TPC_4_0,
1454     [TPC_ID_DCORE2_TPC5] = GAUDI2_QUEUE_ID_DCORE2_TPC_5_0,
1455     [TPC_ID_DCORE3_TPC0] = GAUDI2_QUEUE_ID_DCORE3_TPC_0_0,
1456     [TPC_ID_DCORE3_TPC1] = GAUDI2_QUEUE_ID_DCORE3_TPC_1_0,
1457     [TPC_ID_DCORE3_TPC2] = GAUDI2_QUEUE_ID_DCORE3_TPC_2_0,
1458     [TPC_ID_DCORE3_TPC3] = GAUDI2_QUEUE_ID_DCORE3_TPC_3_0,
1459     [TPC_ID_DCORE3_TPC4] = GAUDI2_QUEUE_ID_DCORE3_TPC_4_0,
1460     [TPC_ID_DCORE3_TPC5] = GAUDI2_QUEUE_ID_DCORE3_TPC_5_0,
1461     [TPC_ID_DCORE0_TPC6] = GAUDI2_QUEUE_ID_DCORE0_TPC_6_0,
1462 };
1463 
1464 static const u32 gaudi2_rot_id_to_queue_id[ROTATOR_ID_SIZE] = {
1465     [ROTATOR_ID_0] = GAUDI2_QUEUE_ID_ROT_0_0,
1466     [ROTATOR_ID_1] = GAUDI2_QUEUE_ID_ROT_1_0,
1467 };
1468 
1469 const u32 edma_stream_base[NUM_OF_EDMA_PER_DCORE * NUM_OF_DCORES] = {
1470     GAUDI2_QUEUE_ID_DCORE0_EDMA_0_0,
1471     GAUDI2_QUEUE_ID_DCORE0_EDMA_1_0,
1472     GAUDI2_QUEUE_ID_DCORE1_EDMA_0_0,
1473     GAUDI2_QUEUE_ID_DCORE1_EDMA_1_0,
1474     GAUDI2_QUEUE_ID_DCORE2_EDMA_0_0,
1475     GAUDI2_QUEUE_ID_DCORE2_EDMA_1_0,
1476     GAUDI2_QUEUE_ID_DCORE3_EDMA_0_0,
1477     GAUDI2_QUEUE_ID_DCORE3_EDMA_1_0,
1478 };
1479 
1480 static const char gaudi2_vdec_irq_name[GAUDI2_VDEC_MSIX_ENTRIES][GAUDI2_MAX_STRING_LEN] = {
1481     "gaudi2 vdec 0_0", "gaudi2 vdec 0_0 abnormal",
1482     "gaudi2 vdec 0_1", "gaudi2 vdec 0_1 abnormal",
1483     "gaudi2 vdec 1_0", "gaudi2 vdec 1_0 abnormal",
1484     "gaudi2 vdec 1_1", "gaudi2 vdec 1_1 abnormal",
1485     "gaudi2 vdec 2_0", "gaudi2 vdec 2_0 abnormal",
1486     "gaudi2 vdec 2_1", "gaudi2 vdec 2_1 abnormal",
1487     "gaudi2 vdec 3_0", "gaudi2 vdec 3_0 abnormal",
1488     "gaudi2 vdec 3_1", "gaudi2 vdec 3_1 abnormal",
1489     "gaudi2 vdec s_0", "gaudi2 vdec s_0 abnormal",
1490     "gaudi2 vdec s_1", "gaudi2 vdec s_1 abnormal"
1491 };
1492 
1493 static const u32 rtr_coordinates_to_rtr_id[NUM_OF_RTR_PER_DCORE * NUM_OF_DCORES] = {
1494     RTR_ID_X_Y(2, 4),
1495     RTR_ID_X_Y(3, 4),
1496     RTR_ID_X_Y(4, 4),
1497     RTR_ID_X_Y(5, 4),
1498     RTR_ID_X_Y(6, 4),
1499     RTR_ID_X_Y(7, 4),
1500     RTR_ID_X_Y(8, 4),
1501     RTR_ID_X_Y(9, 4),
1502     RTR_ID_X_Y(10, 4),
1503     RTR_ID_X_Y(11, 4),
1504     RTR_ID_X_Y(12, 4),
1505     RTR_ID_X_Y(13, 4),
1506     RTR_ID_X_Y(14, 4),
1507     RTR_ID_X_Y(15, 4),
1508     RTR_ID_X_Y(16, 4),
1509     RTR_ID_X_Y(17, 4),
1510     RTR_ID_X_Y(2, 11),
1511     RTR_ID_X_Y(3, 11),
1512     RTR_ID_X_Y(4, 11),
1513     RTR_ID_X_Y(5, 11),
1514     RTR_ID_X_Y(6, 11),
1515     RTR_ID_X_Y(7, 11),
1516     RTR_ID_X_Y(8, 11),
1517     RTR_ID_X_Y(9, 11),
1518     RTR_ID_X_Y(0, 0),/* 24 no id */
1519     RTR_ID_X_Y(0, 0),/* 25 no id */
1520     RTR_ID_X_Y(0, 0),/* 26 no id */
1521     RTR_ID_X_Y(0, 0),/* 27 no id */
1522     RTR_ID_X_Y(14, 11),
1523     RTR_ID_X_Y(15, 11),
1524     RTR_ID_X_Y(16, 11),
1525     RTR_ID_X_Y(17, 11)
1526 };
1527 
1528 static const u32 gaudi2_tpc_initiator_rtr_id[NUM_OF_TPC_PER_DCORE * NUM_OF_DCORES + 1] = {
1529     1, 1, 2, 2, 3, 3, 14, 14, 13, 13, 12, 12, 19, 19, 18, 18, 17,
1530     17, 28, 28, 29, 29, 30, 30, 0
1531 };
1532 
1533 static const u32 gaudi2_dec_initiator_rtr_id[NUMBER_OF_DEC] = {
1534     0, 0, 15, 15, 16, 16, 31, 31, 0, 0
1535 };
1536 
1537 static const u32 gaudi2_nic_initiator_rtr_id[NIC_NUMBER_OF_MACROS] = {
1538     15, 15, 15, 15, 15, 16, 16, 16, 16, 31, 31, 31
1539 };
1540 
1541 struct sft_info {
1542     u8 interface_id;
1543     u8 dcore_id;
1544 };
1545 
1546 static const struct sft_info gaudi2_edma_initiator_sft_id[NUM_OF_EDMA_PER_DCORE * NUM_OF_DCORES] = {
1547     {0, 0}, {1, 0}, {0, 1}, {1, 1}, {1, 2}, {1, 3}, {0, 2}, {0, 3},
1548 };
1549 
1550 static const u32 gaudi2_pdma_initiator_rtr_id[NUM_OF_PDMA] = {
1551     0, 0
1552 };
1553 
1554 static const u32 gaudi2_rot_initiator_rtr_id[NUM_OF_ROT] = {
1555     16, 31
1556 };
1557 
1558 struct mme_initiators_rtr_id {
1559     u32 wap0;
1560     u32 wap1;
1561     u32 write;
1562     u32 read;
1563     u32 sbte0;
1564     u32 sbte1;
1565     u32 sbte2;
1566     u32 sbte3;
1567     u32 sbte4;
1568 };
1569 
1570 enum mme_initiators {
1571     MME_WAP0 = 0,
1572     MME_WAP1,
1573     MME_WRITE,
1574     MME_READ,
1575     MME_SBTE0,
1576     MME_SBTE1,
1577     MME_SBTE2,
1578     MME_SBTE3,
1579     MME_SBTE4,
1580     MME_INITIATORS_MAX
1581 };
1582 
1583 static const struct mme_initiators_rtr_id
1584 gaudi2_mme_initiator_rtr_id[NUM_OF_MME_PER_DCORE * NUM_OF_DCORES] = {
1585     { .wap0 = 5, .wap1 = 7, .write = 6, .read = 7,
1586     .sbte0 = 7, .sbte1 = 4, .sbte2 = 4, .sbte3 = 5, .sbte4 = 6},
1587     { .wap0 = 10, .wap1 = 8, .write = 9, .read = 8,
1588     .sbte0 = 11, .sbte1 = 11, .sbte2 = 10, .sbte3 = 9, .sbte4 = 8},
1589     { .wap0 = 21, .wap1 = 23, .write = 22, .read = 23,
1590     .sbte0 = 20, .sbte1 = 20, .sbte2 = 21, .sbte3 = 22, .sbte4 = 23},
1591     { .wap0 = 30, .wap1 = 28, .write = 29, .read = 30,
1592     .sbte0 = 31, .sbte1 = 31, .sbte2 = 30, .sbte3 = 29, .sbte4 = 28},
1593 };
1594 
1595 enum razwi_event_sources {
1596     RAZWI_TPC,
1597     RAZWI_MME,
1598     RAZWI_EDMA,
1599     RAZWI_PDMA,
1600     RAZWI_NIC,
1601     RAZWI_DEC,
1602     RAZWI_ROT
1603 };
1604 
1605 struct hbm_mc_error_causes {
1606     u32 mask;
1607     char cause[50];
1608 };
1609 
1610 static struct hbm_mc_error_causes hbm_mc_spi[GAUDI2_NUM_OF_HBM_MC_SPI_CAUSE] = {
1611     {HBM_MC_SPI_TEMP_PIN_CHG_MASK, "temperature pins changed"},
1612     {HBM_MC_SPI_THR_ENG_MASK, "temperature-based throttling engaged"},
1613     {HBM_MC_SPI_THR_DIS_ENG_MASK, "temperature-based throttling disengaged"},
1614     {HBM_MC_SPI_IEEE1500_COMP_MASK, "IEEE1500 op comp"},
1615     {HBM_MC_SPI_IEEE1500_PAUSED_MASK, "IEEE1500 op paused"},
1616 };
1617 
1618 static const char * const hbm_mc_sei_cause[GAUDI2_NUM_OF_HBM_SEI_CAUSE] = {
1619     [HBM_SEI_CMD_PARITY_EVEN] = "SEI C/A parity even",
1620     [HBM_SEI_CMD_PARITY_ODD] = "SEI C/A parity odd",
1621     [HBM_SEI_READ_ERR] = "SEI read data error",
1622     [HBM_SEI_WRITE_DATA_PARITY_ERR] = "SEI write data parity error",
1623     [HBM_SEI_CATTRIP] = "SEI CATTRIP asserted",
1624     [HBM_SEI_MEM_BIST_FAIL] = "SEI memory BIST fail",
1625     [HBM_SEI_DFI] = "SEI DFI error",
1626     [HBM_SEI_INV_TEMP_READ_OUT] = "SEI invalid temp read",
1627     [HBM_SEI_BIST_FAIL] = "SEI BIST fail"
1628 };
1629 
1630 struct mmu_spi_sei_cause {
1631     char cause[50];
1632     int clear_bit;
1633 };
1634 
1635 static const struct mmu_spi_sei_cause gaudi2_mmu_spi_sei[GAUDI2_NUM_OF_MMU_SPI_SEI_CAUSE] = {
1636     {"page fault", 1},      /* INTERRUPT_CLR[1] */
1637     {"page access", 1},     /* INTERRUPT_CLR[1] */
1638     {"bypass ddr", 2},      /* INTERRUPT_CLR[2] */
1639     {"multi hit", 2},       /* INTERRUPT_CLR[2] */
1640     {"mmu rei0", -1},       /* no clear register bit */
1641     {"mmu rei1", -1},       /* no clear register bit */
1642     {"stlb rei0", -1},      /* no clear register bit */
1643     {"stlb rei1", -1},      /* no clear register bit */
1644     {"rr privileged write hit", 2}, /* INTERRUPT_CLR[2] */
1645     {"rr privileged read hit", 2},  /* INTERRUPT_CLR[2] */
1646     {"rr secure write hit", 2}, /* INTERRUPT_CLR[2] */
1647     {"rr secure read hit", 2},  /* INTERRUPT_CLR[2] */
1648     {"bist_fail no use", 2},    /* INTERRUPT_CLR[2] */
1649     {"bist_fail no use", 2},    /* INTERRUPT_CLR[2] */
1650     {"bist_fail no use", 2},    /* INTERRUPT_CLR[2] */
1651     {"bist_fail no use", 2},    /* INTERRUPT_CLR[2] */
1652     {"slave error", 16},        /* INTERRUPT_CLR[16] */
1653     {"dec error", 17},      /* INTERRUPT_CLR[17] */
1654     {"burst fifo full", 2}      /* INTERRUPT_CLR[2] */
1655 };
1656 
1657 struct gaudi2_cache_invld_params {
1658     u64 start_va;
1659     u64 end_va;
1660     u32 inv_start_val;
1661     u32 flags;
1662     bool range_invalidation;
1663 };
1664 
1665 struct gaudi2_tpc_idle_data {
1666     struct seq_file *s;
1667     unsigned long *mask;
1668     bool *is_idle;
1669     const char *tpc_fmt;
1670 };
1671 
1672 struct gaudi2_tpc_mmu_data {
1673     u32 rw_asid;
1674 };
1675 
1676 static s64 gaudi2_state_dump_specs_props[SP_MAX] = {0};
1677 
1678 static int gaudi2_memset_device_memory(struct hl_device *hdev, u64 addr, u64 size, u64 val);
1679 static bool gaudi2_is_queue_enabled(struct hl_device *hdev, u32 hw_queue_id);
1680 static bool gaudi2_is_arc_enabled(struct hl_device *hdev, u64 arc_id);
1681 static void gaudi2_clr_arc_id_cap(struct hl_device *hdev, u64 arc_id);
1682 static void gaudi2_set_arc_id_cap(struct hl_device *hdev, u64 arc_id);
1683 static void gaudi2_memset_device_lbw(struct hl_device *hdev, u32 addr, u32 size, u32 val);
1684 static int gaudi2_send_job_to_kdma(struct hl_device *hdev, u64 src_addr, u64 dst_addr, u32 size,
1685                                         bool is_memset);
1686 static u64 gaudi2_mmu_scramble_addr(struct hl_device *hdev, u64 raw_addr);
1687 
1688 static void gaudi2_init_scrambler_hbm(struct hl_device *hdev)
1689 {
1690 
1691 }
1692 
1693 static u32 gaudi2_get_signal_cb_size(struct hl_device *hdev)
1694 {
1695     return sizeof(struct packet_msg_short);
1696 }
1697 
1698 static u32 gaudi2_get_wait_cb_size(struct hl_device *hdev)
1699 {
1700     return sizeof(struct packet_msg_short) * 4 + sizeof(struct packet_fence);
1701 }
1702 
1703 void gaudi2_iterate_tpcs(struct hl_device *hdev, struct iterate_module_ctx *ctx)
1704 {
1705     struct asic_fixed_properties *prop = &hdev->asic_prop;
1706     int dcore, inst, tpc_seq;
1707     u32 offset;
1708 
1709     for (dcore = 0; dcore < NUM_OF_DCORES; dcore++) {
1710         for (inst = 0; inst < NUM_OF_TPC_PER_DCORE; inst++) {
1711             tpc_seq = dcore * NUM_OF_TPC_PER_DCORE + inst;
1712 
1713             if (!(prop->tpc_enabled_mask & BIT(tpc_seq)))
1714                 continue;
1715 
1716             offset = (DCORE_OFFSET * dcore) + (DCORE_TPC_OFFSET * inst);
1717 
1718             ctx->fn(hdev, dcore, inst, offset, ctx->data);
1719         }
1720     }
1721 
1722     if (!(prop->tpc_enabled_mask & BIT(TPC_ID_DCORE0_TPC6)))
1723         return;
1724 
1725     /* special check for PCI TPC (DCORE0_TPC6) */
1726     offset = DCORE_TPC_OFFSET * (NUM_DCORE0_TPC - 1);
1727     ctx->fn(hdev, 0, NUM_DCORE0_TPC - 1, offset, ctx->data);
1728 }
1729 
1730 static bool gaudi2_host_phys_addr_valid(u64 addr)
1731 {
1732     if ((addr < HOST_PHYS_BASE_0 + HOST_PHYS_SIZE_0) || (addr >= HOST_PHYS_BASE_1))
1733         return true;
1734 
1735     return false;
1736 }
1737 
1738 static int set_number_of_functional_hbms(struct hl_device *hdev)
1739 {
1740     struct asic_fixed_properties *prop = &hdev->asic_prop;
1741     u8 faulty_hbms = hweight64(hdev->dram_binning);
1742 
1743     /* check if all HBMs should be used */
1744     if (!faulty_hbms) {
1745         dev_dbg(hdev->dev, "All HBM are in use (no binning)\n");
1746         prop->num_functional_hbms = GAUDI2_HBM_NUM;
1747         return 0;
1748     }
1749 
1750     /*
1751      * check for error condition in which number of binning
1752      * candidates is higher than the maximum supported by the
1753      * driver (in which case binning mask shall be ignored and driver will
1754      * set the default)
1755      */
1756     if (faulty_hbms > MAX_FAULTY_HBMS) {
1757         dev_err(hdev->dev,
1758             "HBM binning supports max of %d faulty HBMs, supplied mask 0x%llx.\n",
1759             MAX_FAULTY_HBMS, hdev->dram_binning);
1760         return -EINVAL;
1761     }
1762 
1763     /*
1764      * by default, number of functional HBMs in Gaudi2 is always
1765      * GAUDI2_HBM_NUM - 1.
1766      */
1767     prop->num_functional_hbms = GAUDI2_HBM_NUM - faulty_hbms;
1768     return 0;
1769 }
1770 
1771 static int gaudi2_set_dram_properties(struct hl_device *hdev)
1772 {
1773     struct asic_fixed_properties *prop = &hdev->asic_prop;
1774     u32 basic_hbm_page_size;
1775     int rc;
1776 
1777     rc = set_number_of_functional_hbms(hdev);
1778     if (rc)
1779         return -EINVAL;
1780 
1781     /*
1782      * Due to HW bug in which TLB size is x16 smaller than expected we use a workaround
1783      * in which we are using x16 bigger page size to be able to populate the entire
1784      * HBM mappings in the TLB
1785      */
1786     basic_hbm_page_size = prop->num_functional_hbms * SZ_8M;
1787     prop->dram_page_size = GAUDI2_COMPENSATE_TLB_PAGE_SIZE_FACTOR * basic_hbm_page_size;
1788     prop->device_mem_alloc_default_page_size = prop->dram_page_size;
1789     prop->dram_size = prop->num_functional_hbms * SZ_16G;
1790     prop->dram_base_address = DRAM_PHYS_BASE;
1791     prop->dram_end_address = prop->dram_base_address + prop->dram_size;
1792     prop->dram_supports_virtual_memory = true;
1793 
1794     prop->dram_user_base_address = DRAM_PHYS_BASE + prop->dram_page_size;
1795     prop->dram_hints_align_mask = ~GAUDI2_HBM_MMU_SCRM_ADDRESS_MASK;
1796     prop->hints_dram_reserved_va_range.start_addr = RESERVED_VA_RANGE_FOR_ARC_ON_HBM_START;
1797     prop->hints_dram_reserved_va_range.end_addr = RESERVED_VA_RANGE_FOR_ARC_ON_HBM_END;
1798 
1799     /* since DRAM page size differs from DMMU page size we need to allocate
1800      * DRAM memory in units of dram_page size and mapping this memory in
1801      * units of DMMU page size. we overcome this size mismatch using a
1802      * scrambling routine which takes a DRAM page and converts it to a DMMU
1803      * page.
1804      * We therefore:
1805      * 1. partition the virtual address space to DRAM-page (whole) pages.
1806      *    (suppose we get n such pages)
1807      * 2. limit the amount of virtual address space we got from 1 above to
1808      *    a multiple of 64M as we don't want the scrambled address to cross
1809      *    the DRAM virtual address space.
1810      *    ( m = (n * DRAM_page_size) / DMMU_page_size).
1811      * 3. determine the and address accordingly
1812      *    end_addr = start_addr + m * 48M
1813      *
1814      *    the DRAM address MSBs (63:48) are not part of the roundup calculation
1815      */
1816     prop->dmmu.start_addr = prop->dram_base_address +
1817             (prop->dram_page_size *
1818                 DIV_ROUND_UP_SECTOR_T(prop->dram_size, prop->dram_page_size));
1819 
1820     prop->dmmu.end_addr = prop->dmmu.start_addr + prop->dram_page_size *
1821             div_u64((VA_HBM_SPACE_END - prop->dmmu.start_addr), prop->dmmu.page_size);
1822 
1823     return 0;
1824 }
1825 
1826 static int gaudi2_set_fixed_properties(struct hl_device *hdev)
1827 {
1828     struct asic_fixed_properties *prop = &hdev->asic_prop;
1829     struct hw_queue_properties *q_props;
1830     u32 num_sync_stream_queues = 0;
1831     int i;
1832 
1833     prop->max_queues = GAUDI2_QUEUE_ID_SIZE;
1834     prop->hw_queues_props = kcalloc(prop->max_queues, sizeof(struct hw_queue_properties),
1835                     GFP_KERNEL);
1836 
1837     if (!prop->hw_queues_props)
1838         return -ENOMEM;
1839 
1840     q_props = prop->hw_queues_props;
1841 
1842     for (i = 0 ; i < GAUDI2_QUEUE_ID_CPU_PQ ; i++) {
1843         q_props[i].type = QUEUE_TYPE_HW;
1844         q_props[i].driver_only = 0;
1845 
1846         if (i >= GAUDI2_QUEUE_ID_NIC_0_0 && i <= GAUDI2_QUEUE_ID_NIC_23_3) {
1847             q_props[i].supports_sync_stream = 0;
1848         } else {
1849             q_props[i].supports_sync_stream = 1;
1850             num_sync_stream_queues++;
1851         }
1852 
1853         q_props[i].cb_alloc_flags = CB_ALLOC_USER;
1854     }
1855 
1856     q_props[GAUDI2_QUEUE_ID_CPU_PQ].type = QUEUE_TYPE_CPU;
1857     q_props[GAUDI2_QUEUE_ID_CPU_PQ].driver_only = 1;
1858     q_props[GAUDI2_QUEUE_ID_CPU_PQ].cb_alloc_flags = CB_ALLOC_KERNEL;
1859 
1860     prop->cache_line_size = DEVICE_CACHE_LINE_SIZE;
1861     prop->cfg_base_address = CFG_BASE;
1862     prop->device_dma_offset_for_host_access = HOST_PHYS_BASE_0;
1863     prop->host_base_address = HOST_PHYS_BASE_0;
1864     prop->host_end_address = prop->host_base_address + HOST_PHYS_SIZE_0;
1865     prop->max_pending_cs = GAUDI2_MAX_PENDING_CS;
1866     prop->completion_queues_count = GAUDI2_RESERVED_CQ_NUMBER;
1867     prop->user_dec_intr_count = NUMBER_OF_DEC;
1868     prop->user_interrupt_count = GAUDI2_IRQ_NUM_USER_LAST - GAUDI2_IRQ_NUM_USER_FIRST + 1;
1869     prop->completion_mode = HL_COMPLETION_MODE_CS;
1870     prop->sync_stream_first_sob = GAUDI2_RESERVED_SOB_NUMBER;
1871     prop->sync_stream_first_mon = GAUDI2_RESERVED_MON_NUMBER;
1872 
1873     prop->sram_base_address = SRAM_BASE_ADDR;
1874     prop->sram_size = SRAM_SIZE;
1875     prop->sram_end_address = prop->sram_base_address + prop->sram_size;
1876     prop->sram_user_base_address = prop->sram_base_address + SRAM_USER_BASE_OFFSET;
1877 
1878     prop->hints_range_reservation = true;
1879 
1880     if (hdev->pldm)
1881         prop->mmu_pgt_size = 0x800000; /* 8MB */
1882     else
1883         prop->mmu_pgt_size = MMU_PAGE_TABLES_INITIAL_SIZE;
1884 
1885     prop->mmu_pte_size = HL_PTE_SIZE;
1886     prop->mmu_hop_table_size = HOP_TABLE_SIZE_512_PTE;
1887     prop->mmu_hop0_tables_total_size = HOP0_512_PTE_TABLES_TOTAL_SIZE;
1888 
1889     prop->dmmu.hop_shifts[MMU_HOP0] = DHOP0_SHIFT;
1890     prop->dmmu.hop_shifts[MMU_HOP1] = DHOP1_SHIFT;
1891     prop->dmmu.hop_shifts[MMU_HOP2] = DHOP2_SHIFT;
1892     prop->dmmu.hop_shifts[MMU_HOP3] = DHOP3_SHIFT;
1893     prop->dmmu.hop_shifts[MMU_HOP4] = DHOP4_SHIFT;
1894     prop->dmmu.hop_masks[MMU_HOP0] = DHOP0_MASK;
1895     prop->dmmu.hop_masks[MMU_HOP1] = DHOP1_MASK;
1896     prop->dmmu.hop_masks[MMU_HOP2] = DHOP2_MASK;
1897     prop->dmmu.hop_masks[MMU_HOP3] = DHOP3_MASK;
1898     prop->dmmu.hop_masks[MMU_HOP4] = DHOP4_MASK;
1899     prop->dmmu.page_size = PAGE_SIZE_1GB;
1900     prop->dmmu.num_hops = MMU_ARCH_6_HOPS;
1901     prop->dmmu.last_mask = LAST_MASK;
1902     prop->dmmu.host_resident = 1;
1903     /* TODO: will be duplicated until implementing per-MMU props */
1904     prop->dmmu.hop_table_size = prop->mmu_hop_table_size;
1905     prop->dmmu.hop0_tables_total_size = prop->mmu_hop0_tables_total_size;
1906 
1907     /*
1908      * this is done in order to be able to validate FW descriptor (i.e. validating that
1909      * the addresses and allocated space for FW image does not cross memory bounds).
1910      * for this reason we set the DRAM size to the minimum possible and later it will
1911      * be modified according to what reported in the cpucp info packet
1912      */
1913     prop->dram_size = (GAUDI2_HBM_NUM - 1) * SZ_16G;
1914 
1915     hdev->pmmu_huge_range = true;
1916     prop->pmmu.host_resident = 1;
1917     prop->pmmu.num_hops = MMU_ARCH_6_HOPS;
1918     prop->pmmu.last_mask = LAST_MASK;
1919     /* TODO: will be duplicated until implementing per-MMU props */
1920     prop->pmmu.hop_table_size = prop->mmu_hop_table_size;
1921     prop->pmmu.hop0_tables_total_size = prop->mmu_hop0_tables_total_size;
1922 
1923     prop->hints_host_reserved_va_range.start_addr = RESERVED_VA_FOR_VIRTUAL_MSIX_DOORBELL_START;
1924     prop->hints_host_reserved_va_range.end_addr = RESERVED_VA_RANGE_FOR_ARC_ON_HOST_END;
1925     prop->hints_host_hpage_reserved_va_range.start_addr =
1926             RESERVED_VA_RANGE_FOR_ARC_ON_HOST_HPAGE_START;
1927     prop->hints_host_hpage_reserved_va_range.end_addr =
1928             RESERVED_VA_RANGE_FOR_ARC_ON_HOST_HPAGE_END;
1929 
1930     if (PAGE_SIZE == SZ_64K) {
1931         prop->pmmu.hop_shifts[MMU_HOP0] = HOP0_SHIFT_64K;
1932         prop->pmmu.hop_shifts[MMU_HOP1] = HOP1_SHIFT_64K;
1933         prop->pmmu.hop_shifts[MMU_HOP2] = HOP2_SHIFT_64K;
1934         prop->pmmu.hop_shifts[MMU_HOP3] = HOP3_SHIFT_64K;
1935         prop->pmmu.hop_shifts[MMU_HOP4] = HOP4_SHIFT_64K;
1936         prop->pmmu.hop_shifts[MMU_HOP5] = HOP5_SHIFT_64K;
1937         prop->pmmu.hop_masks[MMU_HOP0] = HOP0_MASK_64K;
1938         prop->pmmu.hop_masks[MMU_HOP1] = HOP1_MASK_64K;
1939         prop->pmmu.hop_masks[MMU_HOP2] = HOP2_MASK_64K;
1940         prop->pmmu.hop_masks[MMU_HOP3] = HOP3_MASK_64K;
1941         prop->pmmu.hop_masks[MMU_HOP4] = HOP4_MASK_64K;
1942         prop->pmmu.hop_masks[MMU_HOP5] = HOP5_MASK_64K;
1943         prop->pmmu.start_addr = VA_HOST_SPACE_PAGE_START;
1944         prop->pmmu.end_addr = VA_HOST_SPACE_PAGE_END;
1945         prop->pmmu.page_size = PAGE_SIZE_64KB;
1946 
1947         /* shifts and masks are the same in PMMU and HPMMU */
1948         memcpy(&prop->pmmu_huge, &prop->pmmu, sizeof(prop->pmmu));
1949         prop->pmmu_huge.page_size = PAGE_SIZE_16MB;
1950         prop->pmmu_huge.start_addr = VA_HOST_SPACE_HPAGE_START;
1951         prop->pmmu_huge.end_addr = VA_HOST_SPACE_HPAGE_END;
1952     } else {
1953         prop->pmmu.hop_shifts[MMU_HOP0] = HOP0_SHIFT_4K;
1954         prop->pmmu.hop_shifts[MMU_HOP1] = HOP1_SHIFT_4K;
1955         prop->pmmu.hop_shifts[MMU_HOP2] = HOP2_SHIFT_4K;
1956         prop->pmmu.hop_shifts[MMU_HOP3] = HOP3_SHIFT_4K;
1957         prop->pmmu.hop_shifts[MMU_HOP4] = HOP4_SHIFT_4K;
1958         prop->pmmu.hop_shifts[MMU_HOP5] = HOP5_SHIFT_4K;
1959         prop->pmmu.hop_masks[MMU_HOP0] = HOP0_MASK_4K;
1960         prop->pmmu.hop_masks[MMU_HOP1] = HOP1_MASK_4K;
1961         prop->pmmu.hop_masks[MMU_HOP2] = HOP2_MASK_4K;
1962         prop->pmmu.hop_masks[MMU_HOP3] = HOP3_MASK_4K;
1963         prop->pmmu.hop_masks[MMU_HOP4] = HOP4_MASK_4K;
1964         prop->pmmu.hop_masks[MMU_HOP5] = HOP5_MASK_4K;
1965         prop->pmmu.start_addr = VA_HOST_SPACE_PAGE_START;
1966         prop->pmmu.end_addr = VA_HOST_SPACE_PAGE_END;
1967         prop->pmmu.page_size = PAGE_SIZE_4KB;
1968 
1969         /* shifts and masks are the same in PMMU and HPMMU */
1970         memcpy(&prop->pmmu_huge, &prop->pmmu, sizeof(prop->pmmu));
1971         prop->pmmu_huge.page_size = PAGE_SIZE_2MB;
1972         prop->pmmu_huge.start_addr = VA_HOST_SPACE_HPAGE_START;
1973         prop->pmmu_huge.end_addr = VA_HOST_SPACE_HPAGE_END;
1974     }
1975 
1976     prop->cfg_size = CFG_SIZE;
1977     prop->max_asid = MAX_ASID;
1978     prop->num_of_events = GAUDI2_EVENT_SIZE;
1979 
1980     prop->dc_power_default = DC_POWER_DEFAULT;
1981 
1982     prop->cb_pool_cb_cnt = GAUDI2_CB_POOL_CB_CNT;
1983     prop->cb_pool_cb_size = GAUDI2_CB_POOL_CB_SIZE;
1984     prop->pcie_dbi_base_address = CFG_BASE + mmPCIE_DBI_BASE;
1985     prop->pcie_aux_dbi_reg_addr = CFG_BASE + mmPCIE_AUX_DBI;
1986 
1987     strncpy(prop->cpucp_info.card_name, GAUDI2_DEFAULT_CARD_NAME, CARD_NAME_MAX_LEN);
1988 
1989     prop->mme_master_slave_mode = 1;
1990 
1991     prop->first_available_user_sob[0] = GAUDI2_RESERVED_SOB_NUMBER +
1992                     (num_sync_stream_queues * HL_RSVD_SOBS);
1993 
1994     prop->first_available_user_mon[0] = GAUDI2_RESERVED_MON_NUMBER +
1995                     (num_sync_stream_queues * HL_RSVD_MONS);
1996 
1997     prop->first_available_user_interrupt = GAUDI2_IRQ_NUM_USER_FIRST;
1998 
1999     prop->first_available_cq[0] = GAUDI2_RESERVED_CQ_NUMBER;
2000 
2001     prop->fw_cpu_boot_dev_sts0_valid = false;
2002     prop->fw_cpu_boot_dev_sts1_valid = false;
2003     prop->hard_reset_done_by_fw = false;
2004     prop->gic_interrupts_enable = true;
2005 
2006     prop->server_type = HL_SERVER_TYPE_UNKNOWN;
2007 
2008     prop->cb_va_start_addr = VA_HOST_SPACE_USER_MAPPED_CB_START;
2009     prop->cb_va_end_addr = VA_HOST_SPACE_USER_MAPPED_CB_END;
2010 
2011     prop->max_dec = NUMBER_OF_DEC;
2012 
2013     prop->clk_pll_index = HL_GAUDI2_MME_PLL;
2014 
2015     prop->dma_mask = 64;
2016 
2017     return 0;
2018 }
2019 
2020 static int gaudi2_pci_bars_map(struct hl_device *hdev)
2021 {
2022     static const char * const name[] = {"CFG_SRAM", "MSIX", "DRAM"};
2023     bool is_wc[3] = {false, false, true};
2024     int rc;
2025 
2026     rc = hl_pci_bars_map(hdev, name, is_wc);
2027     if (rc)
2028         return rc;
2029 
2030     hdev->rmmio = hdev->pcie_bar[SRAM_CFG_BAR_ID] + (CFG_BASE - STM_FLASH_BASE_ADDR);
2031 
2032     return 0;
2033 }
2034 
2035 static u64 gaudi2_set_hbm_bar_base(struct hl_device *hdev, u64 addr)
2036 {
2037     struct gaudi2_device *gaudi2 = hdev->asic_specific;
2038     struct hl_inbound_pci_region pci_region;
2039     u64 old_addr = addr;
2040     int rc;
2041 
2042     if ((gaudi2) && (gaudi2->dram_bar_cur_addr == addr))
2043         return old_addr;
2044 
2045     if (hdev->asic_prop.iatu_done_by_fw)
2046         return U64_MAX;
2047 
2048     /* Inbound Region 2 - Bar 4 - Point to DRAM */
2049     pci_region.mode = PCI_BAR_MATCH_MODE;
2050     pci_region.bar = DRAM_BAR_ID;
2051     pci_region.addr = addr;
2052     rc = hl_pci_set_inbound_region(hdev, 2, &pci_region);
2053     if (rc)
2054         return U64_MAX;
2055 
2056     if (gaudi2) {
2057         old_addr = gaudi2->dram_bar_cur_addr;
2058         gaudi2->dram_bar_cur_addr = addr;
2059     }
2060 
2061     return old_addr;
2062 }
2063 
2064 static int gaudi2_init_iatu(struct hl_device *hdev)
2065 {
2066     struct hl_inbound_pci_region inbound_region;
2067     struct hl_outbound_pci_region outbound_region;
2068     u32 bar_addr_low, bar_addr_high;
2069     int rc;
2070 
2071     if (hdev->asic_prop.iatu_done_by_fw)
2072         return 0;
2073 
2074     /* Temporary inbound Region 0 - Bar 0 - Point to CFG
2075      * We must map this region in BAR match mode in order to
2076      * fetch BAR physical base address
2077      */
2078     inbound_region.mode = PCI_BAR_MATCH_MODE;
2079     inbound_region.bar = SRAM_CFG_BAR_ID;
2080     /* Base address must be aligned to Bar size which is 256 MB */
2081     inbound_region.addr = STM_FLASH_BASE_ADDR - STM_FLASH_ALIGNED_OFF;
2082     rc = hl_pci_set_inbound_region(hdev, 0, &inbound_region);
2083     if (rc)
2084         return rc;
2085 
2086     /* Fetch physical BAR address */
2087     bar_addr_high = RREG32(mmPCIE_DBI_BAR1_REG + STM_FLASH_ALIGNED_OFF);
2088     bar_addr_low = RREG32(mmPCIE_DBI_BAR0_REG + STM_FLASH_ALIGNED_OFF) & ~0xF;
2089 
2090     hdev->pcie_bar_phys[SRAM_CFG_BAR_ID] = (u64)bar_addr_high << 32 | bar_addr_low;
2091 
2092     /* Inbound Region 0 - Bar 0 - Point to CFG */
2093     inbound_region.mode = PCI_ADDRESS_MATCH_MODE;
2094     inbound_region.bar = SRAM_CFG_BAR_ID;
2095     inbound_region.offset_in_bar = 0;
2096     inbound_region.addr = STM_FLASH_BASE_ADDR;
2097     inbound_region.size = CFG_REGION_SIZE;
2098     rc = hl_pci_set_inbound_region(hdev, 0, &inbound_region);
2099     if (rc)
2100         return rc;
2101 
2102     /* Inbound Region 1 - Bar 0 - Point to BAR0_RESERVED + SRAM */
2103     inbound_region.mode = PCI_ADDRESS_MATCH_MODE;
2104     inbound_region.bar = SRAM_CFG_BAR_ID;
2105     inbound_region.offset_in_bar = CFG_REGION_SIZE;
2106     inbound_region.addr = BAR0_RSRVD_BASE_ADDR;
2107     inbound_region.size = BAR0_RSRVD_SIZE + SRAM_SIZE;
2108     rc = hl_pci_set_inbound_region(hdev, 1, &inbound_region);
2109     if (rc)
2110         return rc;
2111 
2112     /* Inbound Region 2 - Bar 4 - Point to DRAM */
2113     inbound_region.mode = PCI_BAR_MATCH_MODE;
2114     inbound_region.bar = DRAM_BAR_ID;
2115     inbound_region.addr = DRAM_PHYS_BASE;
2116     rc = hl_pci_set_inbound_region(hdev, 2, &inbound_region);
2117     if (rc)
2118         return rc;
2119 
2120     /* Outbound Region 0 - Point to Host */
2121     outbound_region.addr = HOST_PHYS_BASE_0;
2122     outbound_region.size = HOST_PHYS_SIZE_0;
2123     rc = hl_pci_set_outbound_region(hdev, &outbound_region);
2124 
2125     return rc;
2126 }
2127 
2128 static enum hl_device_hw_state gaudi2_get_hw_state(struct hl_device *hdev)
2129 {
2130     return RREG32(mmHW_STATE);
2131 }
2132 
2133 static int gaudi2_tpc_binning_init_prop(struct hl_device *hdev)
2134 {
2135     struct asic_fixed_properties *prop = &hdev->asic_prop;
2136 
2137     /*
2138      * check for error condition in which number of binning candidates
2139      * is higher than the maximum supported by the driver
2140      */
2141     if (hweight64(hdev->tpc_binning) > MAX_CLUSTER_BINNING_FAULTY_TPCS) {
2142         dev_err(hdev->dev, "TPC binning is supported for max of %d faulty TPCs, provided mask 0x%llx\n",
2143                     MAX_CLUSTER_BINNING_FAULTY_TPCS,
2144                     hdev->tpc_binning);
2145         return -EINVAL;
2146     }
2147 
2148     prop->tpc_binning_mask = hdev->tpc_binning;
2149     prop->tpc_enabled_mask = GAUDI2_TPC_FULL_MASK;
2150 
2151     return 0;
2152 }
2153 
2154 static int gaudi2_set_tpc_binning_masks(struct hl_device *hdev)
2155 {
2156     struct asic_fixed_properties *prop = &hdev->asic_prop;
2157     struct hw_queue_properties *q_props = prop->hw_queues_props;
2158     u64 tpc_binning_mask;
2159     u8 subst_idx = 0;
2160     int i, rc;
2161 
2162     rc = gaudi2_tpc_binning_init_prop(hdev);
2163     if (rc)
2164         return rc;
2165 
2166     tpc_binning_mask = prop->tpc_binning_mask;
2167 
2168     for (i = 0 ; i < MAX_FAULTY_TPCS ; i++) {
2169         u8 subst_seq, binned, qid_base;
2170 
2171         if (tpc_binning_mask == 0)
2172             break;
2173 
2174         if (subst_idx == 0) {
2175             subst_seq = TPC_ID_DCORE0_TPC6;
2176             qid_base = GAUDI2_QUEUE_ID_DCORE0_TPC_6_0;
2177         } else {
2178             subst_seq = TPC_ID_DCORE3_TPC5;
2179             qid_base = GAUDI2_QUEUE_ID_DCORE3_TPC_5_0;
2180         }
2181 
2182 
2183         /* clear bit from mask */
2184         binned = __ffs(tpc_binning_mask);
2185         /*
2186          * Coverity complains about possible out-of-bound access in
2187          * clear_bit
2188          */
2189         if (binned >= TPC_ID_SIZE) {
2190             dev_err(hdev->dev,
2191                 "Invalid binned TPC (binning mask: %llx)\n",
2192                 tpc_binning_mask);
2193             return -EINVAL;
2194         }
2195         clear_bit(binned, (unsigned long *)&tpc_binning_mask);
2196 
2197         /* also clear replacing TPC bit from enabled mask */
2198         clear_bit(subst_seq, (unsigned long *)&prop->tpc_enabled_mask);
2199 
2200         /* bin substite TPC's Qs */
2201         q_props[qid_base].binned = 1;
2202         q_props[qid_base + 1].binned = 1;
2203         q_props[qid_base + 2].binned = 1;
2204         q_props[qid_base + 3].binned = 1;
2205 
2206         subst_idx++;
2207     }
2208 
2209     return 0;
2210 }
2211 
2212 static int gaudi2_set_dec_binning_masks(struct hl_device *hdev)
2213 {
2214     struct asic_fixed_properties *prop = &hdev->asic_prop;
2215     u8 num_faulty;
2216 
2217     num_faulty = hweight32(hdev->decoder_binning);
2218 
2219     /*
2220      * check for error condition in which number of binning candidates
2221      * is higher than the maximum supported by the driver
2222      */
2223     if (num_faulty > MAX_FAULTY_DECODERS) {
2224         dev_err(hdev->dev, "decoder binning is supported for max of single faulty decoder, provided mask 0x%x\n",
2225                         hdev->decoder_binning);
2226         return -EINVAL;
2227     }
2228 
2229     prop->decoder_binning_mask = (hdev->decoder_binning & GAUDI2_DECODER_FULL_MASK);
2230 
2231     if (prop->decoder_binning_mask)
2232         prop->decoder_enabled_mask = (GAUDI2_DECODER_FULL_MASK & ~BIT(DEC_ID_PCIE_VDEC1));
2233     else
2234         prop->decoder_enabled_mask = GAUDI2_DECODER_FULL_MASK;
2235 
2236     return 0;
2237 }
2238 
2239 static void gaudi2_set_dram_binning_masks(struct hl_device *hdev)
2240 {
2241     struct asic_fixed_properties *prop = &hdev->asic_prop;
2242 
2243     /* check if we should override default binning */
2244     if (!hdev->dram_binning) {
2245         prop->dram_binning_mask = 0;
2246         prop->dram_enabled_mask = GAUDI2_DRAM_FULL_MASK;
2247         return;
2248     }
2249 
2250     /* set DRAM binning constraints */
2251     prop->faulty_dram_cluster_map |= hdev->dram_binning;
2252     prop->dram_binning_mask = hdev->dram_binning;
2253     prop->dram_enabled_mask = GAUDI2_DRAM_FULL_MASK & ~BIT(HBM_ID5);
2254 }
2255 
2256 static int gaudi2_set_edma_binning_masks(struct hl_device *hdev)
2257 {
2258     struct asic_fixed_properties *prop = &hdev->asic_prop;
2259     struct hw_queue_properties *q_props;
2260     u8 seq, num_faulty;
2261 
2262     num_faulty = hweight32(hdev->edma_binning);
2263 
2264     /*
2265      * check for error condition in which number of binning candidates
2266      * is higher than the maximum supported by the driver
2267      */
2268     if (num_faulty > MAX_FAULTY_EDMAS) {
2269         dev_err(hdev->dev,
2270             "EDMA binning is supported for max of single faulty EDMA, provided mask 0x%x\n",
2271             hdev->edma_binning);
2272         return -EINVAL;
2273     }
2274 
2275     if (!hdev->edma_binning) {
2276         prop->edma_binning_mask = 0;
2277         prop->edma_enabled_mask = GAUDI2_EDMA_FULL_MASK;
2278         return 0;
2279     }
2280 
2281     seq = __ffs((unsigned long)hdev->edma_binning);
2282 
2283     /* set binning constraints */
2284     prop->faulty_dram_cluster_map |= BIT(edma_to_hbm_cluster[seq]);
2285     prop->edma_binning_mask = hdev->edma_binning;
2286     prop->edma_enabled_mask = GAUDI2_EDMA_FULL_MASK & ~BIT(EDMA_ID_DCORE3_INSTANCE1);
2287 
2288     /* bin substitute EDMA's queue */
2289     q_props = prop->hw_queues_props;
2290     q_props[GAUDI2_QUEUE_ID_DCORE3_EDMA_1_0].binned = 1;
2291     q_props[GAUDI2_QUEUE_ID_DCORE3_EDMA_1_1].binned = 1;
2292     q_props[GAUDI2_QUEUE_ID_DCORE3_EDMA_1_2].binned = 1;
2293     q_props[GAUDI2_QUEUE_ID_DCORE3_EDMA_1_3].binned = 1;
2294 
2295     return 0;
2296 }
2297 
2298 static int gaudi2_set_xbar_edge_enable_mask(struct hl_device *hdev, u32 xbar_edge_iso_mask)
2299 {
2300     struct asic_fixed_properties *prop = &hdev->asic_prop;
2301     u8 num_faulty, seq;
2302 
2303     /* check if we should override default binning */
2304     if (!xbar_edge_iso_mask) {
2305         prop->xbar_edge_enabled_mask = GAUDI2_XBAR_EDGE_FULL_MASK;
2306         return 0;
2307     }
2308 
2309     /*
2310      * note that it can be set to value other than 0 only after cpucp packet (i.e.
2311      * only the FW can set a redundancy value). for user it'll always be 0.
2312      */
2313     num_faulty = hweight32(xbar_edge_iso_mask);
2314 
2315     /*
2316      * check for error condition in which number of binning candidates
2317      * is higher than the maximum supported by the driver
2318      */
2319     if (num_faulty > MAX_FAULTY_XBARS) {
2320         dev_err(hdev->dev, "we cannot have more than %d faulty XBAR EDGE\n",
2321                                     MAX_FAULTY_XBARS);
2322         return -EINVAL;
2323     }
2324 
2325     seq = __ffs((unsigned long)xbar_edge_iso_mask);
2326 
2327     /* set binning constraints */
2328     prop->faulty_dram_cluster_map |= BIT(xbar_edge_to_hbm_cluster[seq]);
2329     prop->xbar_edge_enabled_mask = (~xbar_edge_iso_mask) & GAUDI2_XBAR_EDGE_FULL_MASK;
2330 
2331     return 0;
2332 }
2333 
2334 static int gaudi2_set_cluster_binning_masks_common(struct hl_device *hdev, u8 xbar_edge_iso_mask)
2335 {
2336     int rc;
2337 
2338     /*
2339      * mark all clusters as good, each component will "fail" cluster
2340      * based on eFuse/user values.
2341      * If more than single cluster is faulty- the chip is unusable
2342      */
2343     hdev->asic_prop.faulty_dram_cluster_map = 0;
2344 
2345     gaudi2_set_dram_binning_masks(hdev);
2346 
2347     rc = gaudi2_set_edma_binning_masks(hdev);
2348     if (rc)
2349         return rc;
2350 
2351     rc = gaudi2_set_xbar_edge_enable_mask(hdev, xbar_edge_iso_mask);
2352     if (rc)
2353         return rc;
2354 
2355 
2356     /* always initially set to full mask */
2357     hdev->asic_prop.hmmu_hif_enabled_mask = GAUDI2_HIF_HMMU_FULL_MASK;
2358 
2359     return 0;
2360 }
2361 
2362 static int gaudi2_set_cluster_binning_masks(struct hl_device *hdev)
2363 {
2364     struct asic_fixed_properties *prop = &hdev->asic_prop;
2365     int rc;
2366 
2367     rc = gaudi2_set_cluster_binning_masks_common(hdev, prop->cpucp_info.xbar_binning_mask);
2368     if (rc)
2369         return rc;
2370 
2371     /* if we have DRAM binning reported by FW we should perform cluster config  */
2372     if (prop->faulty_dram_cluster_map) {
2373         u8 cluster_seq = __ffs((unsigned long)prop->faulty_dram_cluster_map);
2374 
2375         prop->hmmu_hif_enabled_mask = cluster_hmmu_hif_enabled_mask[cluster_seq];
2376     }
2377 
2378     return 0;
2379 }
2380 
2381 static int gaudi2_cpucp_info_get(struct hl_device *hdev)
2382 {
2383     struct gaudi2_device *gaudi2 = hdev->asic_specific;
2384     struct asic_fixed_properties *prop = &hdev->asic_prop;
2385     long max_power;
2386     u64 dram_size;
2387     int rc;
2388 
2389     if (!(gaudi2->hw_cap_initialized & HW_CAP_CPU_Q))
2390         return 0;
2391 
2392     /* No point of asking this information again when not doing hard reset, as the device
2393      * CPU hasn't been reset
2394      */
2395     if (hdev->reset_info.in_compute_reset)
2396         return 0;
2397 
2398     rc = hl_fw_cpucp_handshake(hdev, mmCPU_BOOT_DEV_STS0, mmCPU_BOOT_DEV_STS1, mmCPU_BOOT_ERR0,
2399                                         mmCPU_BOOT_ERR1);
2400     if (rc)
2401         return rc;
2402 
2403     dram_size = le64_to_cpu(prop->cpucp_info.dram_size);
2404     if (dram_size) {
2405         /* we can have wither 5 or 6 HBMs. other values are invalid */
2406 
2407         if ((dram_size != ((GAUDI2_HBM_NUM - 1) * SZ_16G)) &&
2408                     (dram_size != (GAUDI2_HBM_NUM * SZ_16G))) {
2409             dev_err(hdev->dev,
2410                 "F/W reported invalid DRAM size %llu. Trying to use default size %llu\n",
2411                 dram_size, prop->dram_size);
2412             dram_size = prop->dram_size;
2413         }
2414 
2415         prop->dram_size = dram_size;
2416         prop->dram_end_address = prop->dram_base_address + dram_size;
2417     }
2418 
2419     if (!strlen(prop->cpucp_info.card_name))
2420         strncpy(prop->cpucp_info.card_name, GAUDI2_DEFAULT_CARD_NAME, CARD_NAME_MAX_LEN);
2421 
2422     /* Overwrite binning masks with the actual binning values from F/W */
2423     hdev->dram_binning = prop->cpucp_info.dram_binning_mask;
2424     hdev->edma_binning = prop->cpucp_info.edma_binning_mask;
2425     hdev->tpc_binning = le64_to_cpu(prop->cpucp_info.tpc_binning_mask);
2426     hdev->decoder_binning = lower_32_bits(le64_to_cpu(prop->cpucp_info.decoder_binning_mask));
2427 
2428     /*
2429      * at this point the DRAM parameters need to be updated according to data obtained
2430      * from the FW
2431      */
2432     rc = gaudi2_set_dram_properties(hdev);
2433     if (rc)
2434         return rc;
2435 
2436     rc = gaudi2_set_cluster_binning_masks(hdev);
2437     if (rc)
2438         return rc;
2439 
2440     rc = gaudi2_set_tpc_binning_masks(hdev);
2441     if (rc)
2442         return rc;
2443 
2444     rc = gaudi2_set_dec_binning_masks(hdev);
2445     if (rc)
2446         return rc;
2447 
2448     max_power = hl_fw_get_max_power(hdev);
2449     if (max_power < 0)
2450         return max_power;
2451 
2452     prop->max_power_default = (u64) max_power;
2453 
2454     return 0;
2455 }
2456 
2457 static int gaudi2_fetch_psoc_frequency(struct hl_device *hdev)
2458 {
2459     struct gaudi2_device *gaudi2 = hdev->asic_specific;
2460     u16 pll_freq_arr[HL_PLL_NUM_OUTPUTS];
2461     int rc;
2462 
2463     if (!(gaudi2->hw_cap_initialized & HW_CAP_CPU_Q))
2464         return 0;
2465 
2466     rc = hl_fw_cpucp_pll_info_get(hdev, HL_GAUDI2_CPU_PLL, pll_freq_arr);
2467     if (rc)
2468         return rc;
2469 
2470     hdev->asic_prop.psoc_timestamp_frequency = pll_freq_arr[3];
2471 
2472     return 0;
2473 }
2474 
2475 static int gaudi2_early_init(struct hl_device *hdev)
2476 {
2477     struct asic_fixed_properties *prop = &hdev->asic_prop;
2478     struct pci_dev *pdev = hdev->pdev;
2479     resource_size_t pci_bar_size;
2480     u32 fw_boot_status;
2481     int rc;
2482 
2483     rc = gaudi2_set_fixed_properties(hdev);
2484     if (rc)
2485         return rc;
2486 
2487     /* Check BAR sizes */
2488     pci_bar_size = pci_resource_len(pdev, SRAM_CFG_BAR_ID);
2489 
2490     if (pci_bar_size != CFG_BAR_SIZE) {
2491         dev_err(hdev->dev, "Not " HL_NAME "? BAR %d size %pa, expecting %llu\n",
2492             SRAM_CFG_BAR_ID, &pci_bar_size, CFG_BAR_SIZE);
2493         rc = -ENODEV;
2494         goto free_queue_props;
2495     }
2496 
2497     pci_bar_size = pci_resource_len(pdev, MSIX_BAR_ID);
2498     if (pci_bar_size != MSIX_BAR_SIZE) {
2499         dev_err(hdev->dev, "Not " HL_NAME "? BAR %d size %pa, expecting %llu\n",
2500             MSIX_BAR_ID, &pci_bar_size, MSIX_BAR_SIZE);
2501         rc = -ENODEV;
2502         goto free_queue_props;
2503     }
2504 
2505     prop->dram_pci_bar_size = pci_resource_len(pdev, DRAM_BAR_ID);
2506     hdev->dram_pci_bar_start = pci_resource_start(pdev, DRAM_BAR_ID);
2507 
2508     /* If FW security is enabled at this point it means no access to ELBI */
2509     if (hdev->asic_prop.fw_security_enabled) {
2510         hdev->asic_prop.iatu_done_by_fw = true;
2511         goto pci_init;
2512     }
2513 
2514     rc = hl_pci_elbi_read(hdev, CFG_BASE + mmCPU_BOOT_DEV_STS0, &fw_boot_status);
2515     if (rc)
2516         goto free_queue_props;
2517 
2518     /* Check whether FW is configuring iATU */
2519     if ((fw_boot_status & CPU_BOOT_DEV_STS0_ENABLED) &&
2520             (fw_boot_status & CPU_BOOT_DEV_STS0_FW_IATU_CONF_EN))
2521         hdev->asic_prop.iatu_done_by_fw = true;
2522 
2523 pci_init:
2524     rc = hl_pci_init(hdev);
2525     if (rc)
2526         goto free_queue_props;
2527 
2528     /* Before continuing in the initialization, we need to read the preboot
2529      * version to determine whether we run with a security-enabled firmware
2530      */
2531     rc = hl_fw_read_preboot_status(hdev);
2532     if (rc) {
2533         if (hdev->reset_on_preboot_fail)
2534             hdev->asic_funcs->hw_fini(hdev, true, false);
2535         goto pci_fini;
2536     }
2537 
2538     if (gaudi2_get_hw_state(hdev) == HL_DEVICE_HW_STATE_DIRTY) {
2539         dev_dbg(hdev->dev, "H/W state is dirty, must reset before initializing\n");
2540         hdev->asic_funcs->hw_fini(hdev, true, false);
2541     }
2542 
2543     return 0;
2544 
2545 pci_fini:
2546     hl_pci_fini(hdev);
2547 free_queue_props:
2548     kfree(hdev->asic_prop.hw_queues_props);
2549     return rc;
2550 }
2551 
2552 static int gaudi2_early_fini(struct hl_device *hdev)
2553 {
2554     kfree(hdev->asic_prop.hw_queues_props);
2555     hl_pci_fini(hdev);
2556 
2557     return 0;
2558 }
2559 
2560 static bool gaudi2_is_arc_nic_owned(u64 arc_id)
2561 {
2562     switch (arc_id) {
2563     case CPU_ID_NIC_QMAN_ARC0...CPU_ID_NIC_QMAN_ARC23:
2564         return true;
2565     default:
2566         return false;
2567     }
2568 }
2569 
2570 static bool gaudi2_is_arc_tpc_owned(u64 arc_id)
2571 {
2572     switch (arc_id) {
2573     case CPU_ID_TPC_QMAN_ARC0...CPU_ID_TPC_QMAN_ARC24:
2574         return true;
2575     default:
2576         return false;
2577     }
2578 }
2579 
2580 static void gaudi2_init_arcs(struct hl_device *hdev)
2581 {
2582     struct gaudi2_device *gaudi2 = hdev->asic_specific;
2583     u64 arc_id;
2584     u32 i;
2585 
2586     for (i = CPU_ID_SCHED_ARC0 ; i <= CPU_ID_SCHED_ARC3 ; i++) {
2587         if (gaudi2_is_arc_enabled(hdev, i))
2588             continue;
2589 
2590         gaudi2_set_arc_id_cap(hdev, i);
2591     }
2592 
2593     for (i = GAUDI2_QUEUE_ID_PDMA_0_0 ; i < GAUDI2_QUEUE_ID_CPU_PQ ; i += 4) {
2594         if (!gaudi2_is_queue_enabled(hdev, i))
2595             continue;
2596 
2597         arc_id = gaudi2_queue_id_to_arc_id[i];
2598         if (gaudi2_is_arc_enabled(hdev, arc_id))
2599             continue;
2600 
2601         if (gaudi2_is_arc_nic_owned(arc_id) &&
2602                 !(hdev->nic_ports_mask & BIT_ULL(arc_id - CPU_ID_NIC_QMAN_ARC0)))
2603             continue;
2604 
2605         if (gaudi2_is_arc_tpc_owned(arc_id) && !(gaudi2->tpc_hw_cap_initialized &
2606                             BIT_ULL(arc_id - CPU_ID_TPC_QMAN_ARC0)))
2607             continue;
2608 
2609         gaudi2_set_arc_id_cap(hdev, arc_id);
2610     }
2611 }
2612 
2613 static int gaudi2_scrub_arc_dccm(struct hl_device *hdev, u32 cpu_id)
2614 {
2615     u32 reg_base, reg_val;
2616     int rc;
2617 
2618     switch (cpu_id) {
2619     case CPU_ID_SCHED_ARC0 ... CPU_ID_SCHED_ARC3:
2620         /* Each ARC scheduler has 2 consecutive DCCM blocks */
2621         rc = gaudi2_send_job_to_kdma(hdev, 0, CFG_BASE + gaudi2_arc_dccm_bases[cpu_id],
2622                         ARC_DCCM_BLOCK_SIZE * 2, true);
2623         if (rc)
2624             return rc;
2625         break;
2626     case CPU_ID_SCHED_ARC4:
2627     case CPU_ID_SCHED_ARC5:
2628     case CPU_ID_MME_QMAN_ARC0:
2629     case CPU_ID_MME_QMAN_ARC1:
2630         reg_base = gaudi2_arc_blocks_bases[cpu_id];
2631 
2632         /* Scrub lower DCCM block */
2633         rc = gaudi2_send_job_to_kdma(hdev, 0, CFG_BASE + gaudi2_arc_dccm_bases[cpu_id],
2634                         ARC_DCCM_BLOCK_SIZE, true);
2635         if (rc)
2636             return rc;
2637 
2638         /* Switch to upper DCCM block */
2639         reg_val = FIELD_PREP(ARC_FARM_ARC0_AUX_MME_ARC_UPPER_DCCM_EN_VAL_MASK, 1);
2640         WREG32(reg_base + ARC_DCCM_UPPER_EN_OFFSET, reg_val);
2641 
2642         /* Scrub upper DCCM block */
2643         rc = gaudi2_send_job_to_kdma(hdev, 0, CFG_BASE + gaudi2_arc_dccm_bases[cpu_id],
2644                         ARC_DCCM_BLOCK_SIZE, true);
2645         if (rc)
2646             return rc;
2647 
2648         /* Switch to lower DCCM block */
2649         reg_val = FIELD_PREP(ARC_FARM_ARC0_AUX_MME_ARC_UPPER_DCCM_EN_VAL_MASK, 0);
2650         WREG32(reg_base + ARC_DCCM_UPPER_EN_OFFSET, reg_val);
2651         break;
2652     default:
2653         rc = gaudi2_send_job_to_kdma(hdev, 0, CFG_BASE + gaudi2_arc_dccm_bases[cpu_id],
2654                         ARC_DCCM_BLOCK_SIZE, true);
2655         if (rc)
2656             return rc;
2657     }
2658 
2659     return 0;
2660 }
2661 
2662 static void gaudi2_scrub_arcs_dccm(struct hl_device *hdev)
2663 {
2664     u16 arc_id;
2665 
2666     for (arc_id = CPU_ID_SCHED_ARC0 ; arc_id < CPU_ID_MAX ; arc_id++) {
2667         if (!gaudi2_is_arc_enabled(hdev, arc_id))
2668             continue;
2669 
2670         gaudi2_scrub_arc_dccm(hdev, arc_id);
2671     }
2672 }
2673 
2674 static int gaudi2_late_init(struct hl_device *hdev)
2675 {
2676     struct gaudi2_device *gaudi2 = hdev->asic_specific;
2677     int rc;
2678 
2679     rc = hl_fw_send_pci_access_msg(hdev, CPUCP_PACKET_ENABLE_PCI_ACCESS,
2680                     gaudi2->virt_msix_db_dma_addr);
2681     if (rc) {
2682         dev_err(hdev->dev, "Failed to enable PCI access from CPU\n");
2683         return rc;
2684     }
2685 
2686     rc = gaudi2_fetch_psoc_frequency(hdev);
2687     if (rc) {
2688         dev_err(hdev->dev, "Failed to fetch psoc frequency\n");
2689         goto disable_pci_access;
2690     }
2691 
2692     gaudi2_init_arcs(hdev);
2693     gaudi2_scrub_arcs_dccm(hdev);
2694     gaudi2_init_security(hdev);
2695 
2696     return 0;
2697 
2698 disable_pci_access:
2699     hl_fw_send_pci_access_msg(hdev, CPUCP_PACKET_DISABLE_PCI_ACCESS, 0x0);
2700 
2701     return rc;
2702 }
2703 
2704 static void gaudi2_late_fini(struct hl_device *hdev)
2705 {
2706     const struct hwmon_channel_info **channel_info_arr;
2707     int i = 0;
2708 
2709     if (!hdev->hl_chip_info->info)
2710         return;
2711 
2712     channel_info_arr = hdev->hl_chip_info->info;
2713 
2714     while (channel_info_arr[i]) {
2715         kfree(channel_info_arr[i]->config);
2716         kfree(channel_info_arr[i]);
2717         i++;
2718     }
2719 
2720     kfree(channel_info_arr);
2721 
2722     hdev->hl_chip_info->info = NULL;
2723 }
2724 
2725 static void gaudi2_user_mapped_dec_init(struct gaudi2_device *gaudi2, u32 start_idx)
2726 {
2727     struct user_mapped_block *blocks = gaudi2->mapped_blocks;
2728 
2729     HL_USR_MAPPED_BLK_INIT(&blocks[start_idx++], mmDCORE0_DEC0_CMD_BASE, HL_BLOCK_SIZE);
2730     HL_USR_MAPPED_BLK_INIT(&blocks[start_idx++], mmDCORE0_DEC1_CMD_BASE, HL_BLOCK_SIZE);
2731     HL_USR_MAPPED_BLK_INIT(&blocks[start_idx++], mmDCORE1_DEC0_CMD_BASE, HL_BLOCK_SIZE);
2732     HL_USR_MAPPED_BLK_INIT(&blocks[start_idx++], mmDCORE1_DEC1_CMD_BASE, HL_BLOCK_SIZE);
2733     HL_USR_MAPPED_BLK_INIT(&blocks[start_idx++], mmDCORE2_DEC0_CMD_BASE, HL_BLOCK_SIZE);
2734     HL_USR_MAPPED_BLK_INIT(&blocks[start_idx++], mmDCORE2_DEC1_CMD_BASE, HL_BLOCK_SIZE);
2735     HL_USR_MAPPED_BLK_INIT(&blocks[start_idx++], mmDCORE3_DEC0_CMD_BASE, HL_BLOCK_SIZE);
2736     HL_USR_MAPPED_BLK_INIT(&blocks[start_idx++], mmDCORE3_DEC1_CMD_BASE, HL_BLOCK_SIZE);
2737     HL_USR_MAPPED_BLK_INIT(&blocks[start_idx++], mmPCIE_DEC0_CMD_BASE, HL_BLOCK_SIZE);
2738     HL_USR_MAPPED_BLK_INIT(&blocks[start_idx], mmPCIE_DEC1_CMD_BASE, HL_BLOCK_SIZE);
2739 }
2740 
2741 static void gaudi2_user_mapped_blocks_init(struct hl_device *hdev)
2742 {
2743     struct gaudi2_device *gaudi2 = hdev->asic_specific;
2744     struct user_mapped_block *blocks = gaudi2->mapped_blocks;
2745     u32 block_size, umr_start_idx, num_umr_blocks;
2746     int i;
2747 
2748     for (i = 0 ; i < NUM_ARC_CPUS ; i++) {
2749         if (i >= CPU_ID_SCHED_ARC0 && i <= CPU_ID_SCHED_ARC3)
2750             block_size = ARC_DCCM_BLOCK_SIZE * 2;
2751         else
2752             block_size = ARC_DCCM_BLOCK_SIZE;
2753 
2754         blocks[i].address = gaudi2_arc_dccm_bases[i];
2755         blocks[i].size = block_size;
2756     }
2757 
2758     blocks[NUM_ARC_CPUS].address = mmARC_FARM_ARC0_ACP_ENG_BASE;
2759     blocks[NUM_ARC_CPUS].size = HL_BLOCK_SIZE;
2760 
2761     blocks[NUM_ARC_CPUS + 1].address = mmARC_FARM_ARC1_ACP_ENG_BASE;
2762     blocks[NUM_ARC_CPUS + 1].size = HL_BLOCK_SIZE;
2763 
2764     blocks[NUM_ARC_CPUS + 2].address = mmARC_FARM_ARC2_ACP_ENG_BASE;
2765     blocks[NUM_ARC_CPUS + 2].size = HL_BLOCK_SIZE;
2766 
2767     blocks[NUM_ARC_CPUS + 3].address = mmARC_FARM_ARC3_ACP_ENG_BASE;
2768     blocks[NUM_ARC_CPUS + 3].size = HL_BLOCK_SIZE;
2769 
2770     blocks[NUM_ARC_CPUS + 4].address = mmDCORE0_MME_QM_ARC_ACP_ENG_BASE;
2771     blocks[NUM_ARC_CPUS + 4].size = HL_BLOCK_SIZE;
2772 
2773     blocks[NUM_ARC_CPUS + 5].address = mmDCORE1_MME_QM_ARC_ACP_ENG_BASE;
2774     blocks[NUM_ARC_CPUS + 5].size = HL_BLOCK_SIZE;
2775 
2776     blocks[NUM_ARC_CPUS + 6].address = mmDCORE2_MME_QM_ARC_ACP_ENG_BASE;
2777     blocks[NUM_ARC_CPUS + 6].size = HL_BLOCK_SIZE;
2778 
2779     blocks[NUM_ARC_CPUS + 7].address = mmDCORE3_MME_QM_ARC_ACP_ENG_BASE;
2780     blocks[NUM_ARC_CPUS + 7].size = HL_BLOCK_SIZE;
2781 
2782     umr_start_idx = NUM_ARC_CPUS + NUM_OF_USER_ACP_BLOCKS;
2783     num_umr_blocks = NIC_NUMBER_OF_ENGINES * NUM_OF_USER_NIC_UMR_BLOCKS;
2784     for (i = 0 ; i < num_umr_blocks ; i++) {
2785         u8 nic_id, umr_block_id;
2786 
2787         nic_id = i / NUM_OF_USER_NIC_UMR_BLOCKS;
2788         umr_block_id = i % NUM_OF_USER_NIC_UMR_BLOCKS;
2789 
2790         blocks[umr_start_idx + i].address =
2791             mmNIC0_UMR0_0_UNSECURE_DOORBELL0_BASE +
2792             (nic_id / NIC_NUMBER_OF_QM_PER_MACRO) * NIC_OFFSET +
2793             (nic_id % NIC_NUMBER_OF_QM_PER_MACRO) * NIC_QM_OFFSET +
2794             umr_block_id * NIC_UMR_OFFSET;
2795         blocks[umr_start_idx + i].size = HL_BLOCK_SIZE;
2796     }
2797 
2798     /* Expose decoder HW configuration block to user */
2799     gaudi2_user_mapped_dec_init(gaudi2, USR_MAPPED_BLK_DEC_START_IDX);
2800 
2801     for (i = 1; i < NUM_OF_DCORES; ++i) {
2802         blocks[USR_MAPPED_BLK_SM_START_IDX + 2 * (i - 1)].size = SM_OBJS_BLOCK_SIZE;
2803         blocks[USR_MAPPED_BLK_SM_START_IDX + 2 * (i - 1) + 1].size = HL_BLOCK_SIZE;
2804 
2805         blocks[USR_MAPPED_BLK_SM_START_IDX + 2 * (i - 1)].address =
2806                         mmDCORE0_SYNC_MNGR_OBJS_BASE + i * DCORE_OFFSET;
2807 
2808         blocks[USR_MAPPED_BLK_SM_START_IDX + 2 * (i - 1) + 1].address =
2809                         mmDCORE0_SYNC_MNGR_GLBL_BASE + i * DCORE_OFFSET;
2810     }
2811 }
2812 
2813 static int gaudi2_alloc_cpu_accessible_dma_mem(struct hl_device *hdev)
2814 {
2815     dma_addr_t dma_addr_arr[GAUDI2_ALLOC_CPU_MEM_RETRY_CNT] = {}, end_addr;
2816     void *virt_addr_arr[GAUDI2_ALLOC_CPU_MEM_RETRY_CNT] = {};
2817     int i, j, rc = 0;
2818 
2819     /* The device ARC works with 32-bits addresses, and because there is a single HW register
2820      * that holds the extension bits (49..28), these bits must be identical in all the allocated
2821      * range.
2822      */
2823 
2824     for (i = 0 ; i < GAUDI2_ALLOC_CPU_MEM_RETRY_CNT ; i++) {
2825         virt_addr_arr[i] = hl_asic_dma_alloc_coherent(hdev, HL_CPU_ACCESSIBLE_MEM_SIZE,
2826                             &dma_addr_arr[i], GFP_KERNEL | __GFP_ZERO);
2827         if (!virt_addr_arr[i]) {
2828             rc = -ENOMEM;
2829             goto free_dma_mem_arr;
2830         }
2831 
2832         end_addr = dma_addr_arr[i] + HL_CPU_ACCESSIBLE_MEM_SIZE - 1;
2833         if (GAUDI2_ARC_PCI_MSB_ADDR(dma_addr_arr[i]) == GAUDI2_ARC_PCI_MSB_ADDR(end_addr))
2834             break;
2835     }
2836 
2837     if (i == GAUDI2_ALLOC_CPU_MEM_RETRY_CNT) {
2838         dev_err(hdev->dev,
2839             "MSB of ARC accessible DMA memory are not identical in all range\n");
2840         rc = -EFAULT;
2841         goto free_dma_mem_arr;
2842     }
2843 
2844     hdev->cpu_accessible_dma_mem = virt_addr_arr[i];
2845     hdev->cpu_accessible_dma_address = dma_addr_arr[i];
2846 
2847 free_dma_mem_arr:
2848     for (j = 0 ; j < i ; j++)
2849         hl_asic_dma_free_coherent(hdev, HL_CPU_ACCESSIBLE_MEM_SIZE, virt_addr_arr[j],
2850                         dma_addr_arr[j]);
2851 
2852     return rc;
2853 }
2854 
2855 static void gaudi2_set_pci_memory_regions(struct hl_device *hdev)
2856 {
2857     struct asic_fixed_properties *prop = &hdev->asic_prop;
2858     struct pci_mem_region *region;
2859 
2860     /* CFG */
2861     region = &hdev->pci_mem_region[PCI_REGION_CFG];
2862     region->region_base = CFG_BASE;
2863     region->region_size = CFG_SIZE;
2864     region->offset_in_bar = CFG_BASE - STM_FLASH_BASE_ADDR;
2865     region->bar_size = CFG_BAR_SIZE;
2866     region->bar_id = SRAM_CFG_BAR_ID;
2867     region->used = 1;
2868 
2869     /* SRAM */
2870     region = &hdev->pci_mem_region[PCI_REGION_SRAM];
2871     region->region_base = SRAM_BASE_ADDR;
2872     region->region_size = SRAM_SIZE;
2873     region->offset_in_bar = CFG_REGION_SIZE + BAR0_RSRVD_SIZE;
2874     region->bar_size = CFG_BAR_SIZE;
2875     region->bar_id = SRAM_CFG_BAR_ID;
2876     region->used = 1;
2877 
2878     /* DRAM */
2879     region = &hdev->pci_mem_region[PCI_REGION_DRAM];
2880     region->region_base = DRAM_PHYS_BASE;
2881     region->region_size = hdev->asic_prop.dram_size;
2882     region->offset_in_bar = 0;
2883     region->bar_size = prop->dram_pci_bar_size;
2884     region->bar_id = DRAM_BAR_ID;
2885     region->used = 1;
2886 }
2887 
2888 static void gaudi2_user_interrupt_setup(struct hl_device *hdev)
2889 {
2890     struct asic_fixed_properties *prop = &hdev->asic_prop;
2891     int i, j, k;
2892 
2893     /* Initialize common user CQ interrupt */
2894     HL_USR_INTR_STRUCT_INIT(hdev->common_user_cq_interrupt, hdev,
2895                 HL_COMMON_USER_CQ_INTERRUPT_ID, false);
2896 
2897     /* Initialize common decoder interrupt */
2898     HL_USR_INTR_STRUCT_INIT(hdev->common_decoder_interrupt, hdev,
2899                 HL_COMMON_DEC_INTERRUPT_ID, true);
2900 
2901     /* User interrupts structure holds both decoder and user interrupts from various engines.
2902      * We first initialize the decoder interrupts and then we add the user interrupts.
2903      * The only limitation is that the last decoder interrupt id must be smaller
2904      * then GAUDI2_IRQ_NUM_USER_FIRST. This is checked at compilation time.
2905      */
2906 
2907     /* Initialize decoder interrupts, expose only normal interrupts,
2908      * error interrupts to be handled by driver
2909      */
2910     for (i = GAUDI2_IRQ_NUM_DCORE0_DEC0_NRM, j = 0 ; i <= GAUDI2_IRQ_NUM_SHARED_DEC1_NRM;
2911                                         i += 2, j++)
2912         HL_USR_INTR_STRUCT_INIT(hdev->user_interrupt[j], hdev, i, true);
2913 
2914     for (i = GAUDI2_IRQ_NUM_USER_FIRST, k = 0 ; k < prop->user_interrupt_count; i++, j++, k++)
2915         HL_USR_INTR_STRUCT_INIT(hdev->user_interrupt[j], hdev, i, false);
2916 }
2917 
2918 static inline int gaudi2_get_non_zero_random_int(void)
2919 {
2920     int rand = get_random_int();
2921 
2922     return rand ? rand : 1;
2923 }
2924 
2925 static int gaudi2_sw_init(struct hl_device *hdev)
2926 {
2927     struct asic_fixed_properties *prop = &hdev->asic_prop;
2928     struct gaudi2_device *gaudi2;
2929     int i, rc;
2930 
2931     /* Allocate device structure */
2932     gaudi2 = kzalloc(sizeof(*gaudi2), GFP_KERNEL);
2933     if (!gaudi2)
2934         return -ENOMEM;
2935 
2936     for (i = 0 ; i < ARRAY_SIZE(gaudi2_irq_map_table) ; i++) {
2937         if (gaudi2_irq_map_table[i].msg || !gaudi2_irq_map_table[i].valid)
2938             continue;
2939 
2940         if (gaudi2->num_of_valid_hw_events == GAUDI2_EVENT_SIZE) {
2941             dev_err(hdev->dev, "H/W events array exceeds the limit of %u events\n",
2942                 GAUDI2_EVENT_SIZE);
2943             rc = -EINVAL;
2944             goto free_gaudi2_device;
2945         }
2946 
2947         gaudi2->hw_events[gaudi2->num_of_valid_hw_events++] = gaudi2_irq_map_table[i].fc_id;
2948     }
2949 
2950     for (i = 0 ; i < MME_NUM_OF_LFSR_SEEDS ; i++)
2951         gaudi2->lfsr_rand_seeds[i] = gaudi2_get_non_zero_random_int();
2952 
2953     gaudi2->cpucp_info_get = gaudi2_cpucp_info_get;
2954 
2955     hdev->asic_specific = gaudi2;
2956 
2957     /* Create DMA pool for small allocations.
2958      * Use DEVICE_CACHE_LINE_SIZE for alignment since the NIC memory-mapped
2959      * PI/CI registers allocated from this pool have this restriction
2960      */
2961     hdev->dma_pool = dma_pool_create(dev_name(hdev->dev), &hdev->pdev->dev,
2962                     GAUDI2_DMA_POOL_BLK_SIZE, DEVICE_CACHE_LINE_SIZE, 0);
2963     if (!hdev->dma_pool) {
2964         dev_err(hdev->dev, "failed to create DMA pool\n");
2965         rc = -ENOMEM;
2966         goto free_gaudi2_device;
2967     }
2968 
2969     rc = gaudi2_alloc_cpu_accessible_dma_mem(hdev);
2970     if (rc)
2971         goto free_dma_pool;
2972 
2973     hdev->cpu_accessible_dma_pool = gen_pool_create(ilog2(32), -1);
2974     if (!hdev->cpu_accessible_dma_pool) {
2975         dev_err(hdev->dev, "Failed to create CPU accessible DMA pool\n");
2976         rc = -ENOMEM;
2977         goto free_cpu_dma_mem;
2978     }
2979 
2980     rc = gen_pool_add(hdev->cpu_accessible_dma_pool, (uintptr_t) hdev->cpu_accessible_dma_mem,
2981                 HL_CPU_ACCESSIBLE_MEM_SIZE, -1);
2982     if (rc) {
2983         dev_err(hdev->dev, "Failed to add memory to CPU accessible DMA pool\n");
2984         rc = -EFAULT;
2985         goto free_cpu_accessible_dma_pool;
2986     }
2987 
2988     gaudi2->virt_msix_db_cpu_addr = hl_cpu_accessible_dma_pool_alloc(hdev, prop->pmmu.page_size,
2989                                 &gaudi2->virt_msix_db_dma_addr);
2990     if (!gaudi2->virt_msix_db_cpu_addr) {
2991         dev_err(hdev->dev, "Failed to allocate DMA memory for virtual MSI-X doorbell\n");
2992         rc = -ENOMEM;
2993         goto free_cpu_accessible_dma_pool;
2994     }
2995 
2996     spin_lock_init(&gaudi2->hw_queues_lock);
2997     spin_lock_init(&gaudi2->kdma_lock);
2998 
2999     gaudi2->scratchpad_kernel_address = hl_asic_dma_alloc_coherent(hdev, PAGE_SIZE,
3000                             &gaudi2->scratchpad_bus_address,
3001                             GFP_KERNEL | __GFP_ZERO);
3002     if (!gaudi2->scratchpad_kernel_address) {
3003         rc = -ENOMEM;
3004         goto free_virt_msix_db_mem;
3005     }
3006 
3007     gaudi2_user_mapped_blocks_init(hdev);
3008 
3009     /* Initialize user interrupts */
3010     gaudi2_user_interrupt_setup(hdev);
3011 
3012     hdev->supports_coresight = true;
3013     hdev->supports_sync_stream = true;
3014     hdev->supports_cb_mapping = true;
3015     hdev->supports_wait_for_multi_cs = false;
3016 
3017     prop->supports_compute_reset = true;
3018 
3019     hdev->asic_funcs->set_pci_memory_regions(hdev);
3020 
3021     return 0;
3022 
3023 free_virt_msix_db_mem:
3024     hl_cpu_accessible_dma_pool_free(hdev, prop->pmmu.page_size, gaudi2->virt_msix_db_cpu_addr);
3025 free_cpu_accessible_dma_pool:
3026     gen_pool_destroy(hdev->cpu_accessible_dma_pool);
3027 free_cpu_dma_mem:
3028     hl_asic_dma_free_coherent(hdev, HL_CPU_ACCESSIBLE_MEM_SIZE, hdev->cpu_accessible_dma_mem,
3029                     hdev->cpu_accessible_dma_address);
3030 free_dma_pool:
3031     dma_pool_destroy(hdev->dma_pool);
3032 free_gaudi2_device:
3033     kfree(gaudi2);
3034     return rc;
3035 }
3036 
3037 static int gaudi2_sw_fini(struct hl_device *hdev)
3038 {
3039     struct asic_fixed_properties *prop = &hdev->asic_prop;
3040     struct gaudi2_device *gaudi2 = hdev->asic_specific;
3041 
3042     hl_cpu_accessible_dma_pool_free(hdev, prop->pmmu.page_size, gaudi2->virt_msix_db_cpu_addr);
3043 
3044     gen_pool_destroy(hdev->cpu_accessible_dma_pool);
3045 
3046     hl_asic_dma_free_coherent(hdev, HL_CPU_ACCESSIBLE_MEM_SIZE, hdev->cpu_accessible_dma_mem,
3047                         hdev->cpu_accessible_dma_address);
3048 
3049     hl_asic_dma_free_coherent(hdev, PAGE_SIZE, gaudi2->scratchpad_kernel_address,
3050                     gaudi2->scratchpad_bus_address);
3051 
3052     dma_pool_destroy(hdev->dma_pool);
3053 
3054     kfree(gaudi2);
3055 
3056     return 0;
3057 }
3058 
3059 static void gaudi2_stop_qman_common(struct hl_device *hdev, u32 reg_base)
3060 {
3061     WREG32(reg_base + QM_GLBL_CFG1_OFFSET, QM_GLBL_CFG1_PQF_STOP |
3062                         QM_GLBL_CFG1_CQF_STOP |
3063                         QM_GLBL_CFG1_CP_STOP);
3064 
3065     /* stop also the ARC */
3066     WREG32(reg_base + QM_GLBL_CFG2_OFFSET, QM_GLBL_CFG2_ARC_CQF_STOP);
3067 }
3068 
3069 static void gaudi2_flush_qman_common(struct hl_device *hdev, u32 reg_base)
3070 {
3071     WREG32(reg_base + QM_GLBL_CFG1_OFFSET, QM_GLBL_CFG1_PQF_FLUSH |
3072                         QM_GLBL_CFG1_CQF_FLUSH |
3073                         QM_GLBL_CFG1_CP_FLUSH);
3074 }
3075 
3076 static void gaudi2_flush_qman_arc_common(struct hl_device *hdev, u32 reg_base)
3077 {
3078     WREG32(reg_base + QM_GLBL_CFG2_OFFSET, QM_GLBL_CFG2_ARC_CQF_FLUSH);
3079 }
3080 
3081 /**
3082  * gaudi2_clear_qm_fence_counters_common - clear QM's fence counters
3083  *
3084  * @hdev: pointer to the habanalabs device structure
3085  * @queue_id: queue to clear fence counters to
3086  * @skip_fence: if true set maximum fence value to all fence counters to avoid
3087  *              getting stuck on any fence value. otherwise set all fence
3088  *              counters to 0 (standard clear of fence counters)
3089  */
3090 static void gaudi2_clear_qm_fence_counters_common(struct hl_device *hdev, u32 queue_id,
3091                         bool skip_fence)
3092 {
3093     u32 size, reg_base;
3094     u32 addr, val;
3095 
3096     reg_base = gaudi2_qm_blocks_bases[queue_id];
3097 
3098     addr = reg_base + QM_CP_FENCE0_CNT_0_OFFSET;
3099     size = mmPDMA0_QM_CP_BARRIER_CFG - mmPDMA0_QM_CP_FENCE0_CNT_0;
3100 
3101     /*
3102      * in case we want to make sure that QM that is stuck on a fence will
3103      * be released we should set the fence counter to a higher value that
3104      * the value the QM waiting for. to comply with any fence counter of
3105      * any value we set maximum fence value to all counters
3106      */
3107     val = skip_fence ? U32_MAX : 0;
3108     gaudi2_memset_device_lbw(hdev, addr, size, val);
3109 }
3110 
3111 static void gaudi2_qman_manual_flush_common(struct hl_device *hdev, u32 queue_id)
3112 {
3113     u32 reg_base = gaudi2_qm_blocks_bases[queue_id];
3114 
3115     gaudi2_clear_qm_fence_counters_common(hdev, queue_id, true);
3116     gaudi2_flush_qman_common(hdev, reg_base);
3117     gaudi2_flush_qman_arc_common(hdev, reg_base);
3118 }
3119 
3120 static void gaudi2_stop_dma_qmans(struct hl_device *hdev)
3121 {
3122     struct gaudi2_device *gaudi2 = hdev->asic_specific;
3123     int dcore, inst;
3124 
3125     if (!(gaudi2->hw_cap_initialized & HW_CAP_PDMA_MASK))
3126         goto stop_edma_qmans;
3127 
3128     /* Stop CPs of PDMA QMANs */
3129     gaudi2_stop_qman_common(hdev, mmPDMA0_QM_BASE);
3130     gaudi2_stop_qman_common(hdev, mmPDMA1_QM_BASE);
3131 
3132 stop_edma_qmans:
3133     if (!(gaudi2->hw_cap_initialized & HW_CAP_EDMA_MASK))
3134         return;
3135 
3136     for (dcore = 0 ; dcore < NUM_OF_DCORES ; dcore++) {
3137         for (inst = 0 ; inst < NUM_OF_EDMA_PER_DCORE ; inst++) {
3138             u8 seq = dcore * NUM_OF_EDMA_PER_DCORE + inst;
3139             u32 qm_base;
3140 
3141             if (!(gaudi2->hw_cap_initialized & BIT_ULL(HW_CAP_EDMA_SHIFT + seq)))
3142                 continue;
3143 
3144             qm_base = mmDCORE0_EDMA0_QM_BASE + dcore * DCORE_OFFSET +
3145                     inst * DCORE_EDMA_OFFSET;
3146 
3147             /* Stop CPs of EDMA QMANs */
3148             gaudi2_stop_qman_common(hdev, qm_base);
3149         }
3150     }
3151 }
3152 
3153 static void gaudi2_stop_mme_qmans(struct hl_device *hdev)
3154 {
3155     struct gaudi2_device *gaudi2 = hdev->asic_specific;
3156     u32 offset, i;
3157 
3158     offset = mmDCORE1_MME_QM_BASE - mmDCORE0_MME_QM_BASE;
3159 
3160     for (i = 0 ; i < NUM_OF_DCORES ; i++) {
3161         if (!(gaudi2->hw_cap_initialized & BIT_ULL(HW_CAP_MME_SHIFT + i)))
3162             continue;
3163 
3164         gaudi2_stop_qman_common(hdev, mmDCORE0_MME_QM_BASE + (i * offset));
3165     }
3166 }
3167 
3168 static void gaudi2_stop_tpc_qmans(struct hl_device *hdev)
3169 {
3170     struct gaudi2_device *gaudi2 = hdev->asic_specific;
3171     u32 reg_base;
3172     int i;
3173 
3174     if (!(gaudi2->tpc_hw_cap_initialized & HW_CAP_TPC_MASK))
3175         return;
3176 
3177     for (i = 0 ; i < TPC_ID_SIZE ; i++) {
3178         if (!(gaudi2->tpc_hw_cap_initialized & BIT_ULL(HW_CAP_TPC_SHIFT + i)))
3179             continue;
3180 
3181         reg_base = gaudi2_qm_blocks_bases[gaudi2_tpc_id_to_queue_id[i]];
3182         gaudi2_stop_qman_common(hdev, reg_base);
3183     }
3184 }
3185 
3186 static void gaudi2_stop_rot_qmans(struct hl_device *hdev)
3187 {
3188     struct gaudi2_device *gaudi2 = hdev->asic_specific;
3189     u32 reg_base;
3190     int i;
3191 
3192     if (!(gaudi2->hw_cap_initialized & HW_CAP_ROT_MASK))
3193         return;
3194 
3195     for (i = 0 ; i < ROTATOR_ID_SIZE ; i++) {
3196         if (!(gaudi2->hw_cap_initialized & BIT_ULL(HW_CAP_ROT_SHIFT + i)))
3197             continue;
3198 
3199         reg_base = gaudi2_qm_blocks_bases[gaudi2_rot_id_to_queue_id[i]];
3200         gaudi2_stop_qman_common(hdev, reg_base);
3201     }
3202 }
3203 
3204 static void gaudi2_stop_nic_qmans(struct hl_device *hdev)
3205 {
3206     struct gaudi2_device *gaudi2 = hdev->asic_specific;
3207     u32 reg_base, queue_id;
3208     int i;
3209 
3210     if (!(gaudi2->nic_hw_cap_initialized & HW_CAP_NIC_MASK))
3211         return;
3212 
3213     queue_id = GAUDI2_QUEUE_ID_NIC_0_0;
3214 
3215     for (i = 0 ; i < NIC_NUMBER_OF_ENGINES ; i++, queue_id += NUM_OF_PQ_PER_QMAN) {
3216         if (!(hdev->nic_ports_mask & BIT(i)))
3217             continue;
3218 
3219         reg_base = gaudi2_qm_blocks_bases[queue_id];
3220         gaudi2_stop_qman_common(hdev, reg_base);
3221     }
3222 }
3223 
3224 static void gaudi2_stall_dma_common(struct hl_device *hdev, u32 reg_base)
3225 {
3226     u32 reg_val;
3227 
3228     reg_val = FIELD_PREP(PDMA0_CORE_CFG_1_HALT_MASK, 0x1);
3229     WREG32(reg_base + DMA_CORE_CFG_1_OFFSET, reg_val);
3230 }
3231 
3232 static void gaudi2_dma_stall(struct hl_device *hdev)
3233 {
3234     struct gaudi2_device *gaudi2 = hdev->asic_specific;
3235     int dcore, inst;
3236 
3237     if (!(gaudi2->hw_cap_initialized & HW_CAP_PDMA_MASK))
3238         goto stall_edma;
3239 
3240     gaudi2_stall_dma_common(hdev, mmPDMA0_CORE_BASE);
3241     gaudi2_stall_dma_common(hdev, mmPDMA1_CORE_BASE);
3242 
3243 stall_edma:
3244     if (!(gaudi2->hw_cap_initialized & HW_CAP_EDMA_MASK))
3245         return;
3246 
3247     for (dcore = 0 ; dcore < NUM_OF_DCORES ; dcore++) {
3248         for (inst = 0 ; inst < NUM_OF_EDMA_PER_DCORE ; inst++) {
3249             u8 seq = dcore * NUM_OF_EDMA_PER_DCORE + inst;
3250             u32 core_base;
3251 
3252             if (!(gaudi2->hw_cap_initialized & BIT_ULL(HW_CAP_EDMA_SHIFT + seq)))
3253                 continue;
3254 
3255             core_base = mmDCORE0_EDMA0_CORE_BASE + dcore * DCORE_OFFSET +
3256                     inst * DCORE_EDMA_OFFSET;
3257 
3258             /* Stall CPs of EDMA QMANs */
3259             gaudi2_stall_dma_common(hdev, core_base);
3260         }
3261     }
3262 }
3263 
3264 static void gaudi2_mme_stall(struct hl_device *hdev)
3265 {
3266     struct gaudi2_device *gaudi2 = hdev->asic_specific;
3267     u32 offset, i;
3268 
3269     offset = mmDCORE1_MME_CTRL_LO_QM_STALL - mmDCORE0_MME_CTRL_LO_QM_STALL;
3270 
3271     for (i = 0 ; i < NUM_OF_DCORES ; i++)
3272         if (gaudi2->hw_cap_initialized & BIT_ULL(HW_CAP_MME_SHIFT + i))
3273             WREG32(mmDCORE0_MME_CTRL_LO_QM_STALL + (i * offset), 1);
3274 }
3275 
3276 static void gaudi2_tpc_stall(struct hl_device *hdev)
3277 {
3278     struct gaudi2_device *gaudi2 = hdev->asic_specific;
3279     u32 reg_base;
3280     int i;
3281 
3282     if (!(gaudi2->tpc_hw_cap_initialized & HW_CAP_TPC_MASK))
3283         return;
3284 
3285     for (i = 0 ; i < TPC_ID_SIZE ; i++) {
3286         if (!(gaudi2->tpc_hw_cap_initialized & BIT_ULL(HW_CAP_TPC_SHIFT + i)))
3287             continue;
3288 
3289         reg_base = gaudi2_tpc_cfg_blocks_bases[i];
3290         WREG32(reg_base + TPC_CFG_STALL_OFFSET, 1);
3291     }
3292 }
3293 
3294 static void gaudi2_rotator_stall(struct hl_device *hdev)
3295 {
3296     struct gaudi2_device *gaudi2 = hdev->asic_specific;
3297     u32 reg_val;
3298     int i;
3299 
3300     if (!(gaudi2->hw_cap_initialized & HW_CAP_ROT_MASK))
3301         return;
3302 
3303     reg_val = FIELD_PREP(ROT_MSS_HALT_WBC_MASK, 0x1) |
3304             FIELD_PREP(ROT_MSS_HALT_RSB_MASK, 0x1) |
3305             FIELD_PREP(ROT_MSS_HALT_MRSB_MASK, 0x1);
3306 
3307     for (i = 0 ; i < ROTATOR_ID_SIZE ; i++) {
3308         if (!(gaudi2->hw_cap_initialized & BIT_ULL(HW_CAP_ROT_SHIFT + i)))
3309             continue;
3310 
3311         WREG32(mmROT0_MSS_HALT + i * ROT_OFFSET, reg_val);
3312     }
3313 }
3314 
3315 static void gaudi2_disable_qman_common(struct hl_device *hdev, u32 reg_base)
3316 {
3317     WREG32(reg_base + QM_GLBL_CFG0_OFFSET, 0);
3318 }
3319 
3320 static void gaudi2_disable_dma_qmans(struct hl_device *hdev)
3321 {
3322     struct gaudi2_device *gaudi2 = hdev->asic_specific;
3323     int dcore, inst;
3324 
3325     if (!(gaudi2->hw_cap_initialized & HW_CAP_PDMA_MASK))
3326         goto stop_edma_qmans;
3327 
3328     gaudi2_disable_qman_common(hdev, mmPDMA0_QM_BASE);
3329     gaudi2_disable_qman_common(hdev, mmPDMA1_QM_BASE);
3330 
3331 stop_edma_qmans:
3332     if (!(gaudi2->hw_cap_initialized & HW_CAP_EDMA_MASK))
3333         return;
3334 
3335     for (dcore = 0 ; dcore < NUM_OF_DCORES ; dcore++) {
3336         for (inst = 0 ; inst < NUM_OF_EDMA_PER_DCORE ; inst++) {
3337             u8 seq = dcore * NUM_OF_EDMA_PER_DCORE + inst;
3338             u32 qm_base;
3339 
3340             if (!(gaudi2->hw_cap_initialized & BIT_ULL(HW_CAP_EDMA_SHIFT + seq)))
3341                 continue;
3342 
3343             qm_base = mmDCORE0_EDMA0_QM_BASE + dcore * DCORE_OFFSET +
3344                     inst * DCORE_EDMA_OFFSET;
3345 
3346             /* Disable CPs of EDMA QMANs */
3347             gaudi2_disable_qman_common(hdev, qm_base);
3348         }
3349     }
3350 }
3351 
3352 static void gaudi2_disable_mme_qmans(struct hl_device *hdev)
3353 {
3354     struct gaudi2_device *gaudi2 = hdev->asic_specific;
3355     u32 offset, i;
3356 
3357     offset = mmDCORE1_MME_QM_BASE - mmDCORE0_MME_QM_BASE;
3358 
3359     for (i = 0 ; i < NUM_OF_DCORES ; i++)
3360         if (gaudi2->hw_cap_initialized & BIT_ULL(HW_CAP_MME_SHIFT + i))
3361             gaudi2_disable_qman_common(hdev, mmDCORE0_MME_QM_BASE + (i * offset));
3362 }
3363 
3364 static void gaudi2_disable_tpc_qmans(struct hl_device *hdev)
3365 {
3366     struct gaudi2_device *gaudi2 = hdev->asic_specific;
3367     u32 reg_base;
3368     int i;
3369 
3370     if (!(gaudi2->tpc_hw_cap_initialized & HW_CAP_TPC_MASK))
3371         return;
3372 
3373     for (i = 0 ; i < TPC_ID_SIZE ; i++) {
3374         if (!(gaudi2->tpc_hw_cap_initialized & BIT_ULL(HW_CAP_TPC_SHIFT + i)))
3375             continue;
3376 
3377         reg_base = gaudi2_qm_blocks_bases[gaudi2_tpc_id_to_queue_id[i]];
3378         gaudi2_disable_qman_common(hdev, reg_base);
3379     }
3380 }
3381 
3382 static void gaudi2_disable_rot_qmans(struct hl_device *hdev)
3383 {
3384     struct gaudi2_device *gaudi2 = hdev->asic_specific;
3385     u32 reg_base;
3386     int i;
3387 
3388     if (!(gaudi2->hw_cap_initialized & HW_CAP_ROT_MASK))
3389         return;
3390 
3391     for (i = 0 ; i < ROTATOR_ID_SIZE ; i++) {
3392         if (!(gaudi2->hw_cap_initialized & BIT_ULL(HW_CAP_ROT_SHIFT + i)))
3393             continue;
3394 
3395         reg_base = gaudi2_qm_blocks_bases[gaudi2_rot_id_to_queue_id[i]];
3396         gaudi2_disable_qman_common(hdev, reg_base);
3397     }
3398 }
3399 
3400 static void gaudi2_disable_nic_qmans(struct hl_device *hdev)
3401 {
3402     struct gaudi2_device *gaudi2 = hdev->asic_specific;
3403     u32 reg_base, queue_id;
3404     int i;
3405 
3406     if (!(gaudi2->nic_hw_cap_initialized & HW_CAP_NIC_MASK))
3407         return;
3408 
3409     queue_id = GAUDI2_QUEUE_ID_NIC_0_0;
3410 
3411     for (i = 0 ; i < NIC_NUMBER_OF_ENGINES ; i++, queue_id += NUM_OF_PQ_PER_QMAN) {
3412         if (!(hdev->nic_ports_mask & BIT(i)))
3413             continue;
3414 
3415         reg_base = gaudi2_qm_blocks_bases[queue_id];
3416         gaudi2_disable_qman_common(hdev, reg_base);
3417     }
3418 }
3419 
3420 static void gaudi2_enable_timestamp(struct hl_device *hdev)
3421 {
3422     /* Disable the timestamp counter */
3423     WREG32(mmPSOC_TIMESTAMP_BASE, 0);
3424 
3425     /* Zero the lower/upper parts of the 64-bit counter */
3426     WREG32(mmPSOC_TIMESTAMP_BASE + 0xC, 0);
3427     WREG32(mmPSOC_TIMESTAMP_BASE + 0x8, 0);
3428 
3429     /* Enable the counter */
3430     WREG32(mmPSOC_TIMESTAMP_BASE, 1);
3431 }
3432 
3433 static void gaudi2_disable_timestamp(struct hl_device *hdev)
3434 {
3435     /* Disable the timestamp counter */
3436     WREG32(mmPSOC_TIMESTAMP_BASE, 0);
3437 }
3438 
3439 static const char *gaudi2_irq_name(u16 irq_number)
3440 {
3441     switch (irq_number) {
3442     case GAUDI2_IRQ_NUM_EVENT_QUEUE:
3443         return "gaudi2 cpu eq";
3444     case GAUDI2_IRQ_NUM_COMPLETION:
3445         return "gaudi2 completion";
3446     case GAUDI2_IRQ_NUM_DCORE0_DEC0_NRM ... GAUDI2_IRQ_NUM_SHARED_DEC1_ABNRM:
3447         return gaudi2_vdec_irq_name[irq_number - GAUDI2_IRQ_NUM_DCORE0_DEC0_NRM];
3448     case GAUDI2_IRQ_NUM_USER_FIRST ... GAUDI2_IRQ_NUM_USER_LAST:
3449         return "gaudi2 user completion";
3450     default:
3451         return "invalid";
3452     }
3453 }
3454 
3455 static void gaudi2_dec_disable_msix(struct hl_device *hdev, u32 max_irq_num)
3456 {
3457     int i, irq, relative_idx;
3458     struct hl_dec *dec;
3459 
3460     for (i = GAUDI2_IRQ_NUM_DCORE0_DEC0_NRM ; i < max_irq_num ; i++) {
3461         irq = pci_irq_vector(hdev->pdev, i);
3462         relative_idx = i - GAUDI2_IRQ_NUM_DCORE0_DEC0_NRM;
3463 
3464         dec = hdev->dec + relative_idx / 2;
3465 
3466         /* We pass different structures depending on the irq handler. For the abnormal
3467          * interrupt we pass hl_dec and for the regular interrupt we pass the relevant
3468          * user_interrupt entry
3469          */
3470         free_irq(irq, ((relative_idx % 2) ?
3471                 (void *) dec :
3472                 (void *) &hdev->user_interrupt[dec->core_id]));
3473     }
3474 }
3475 
3476 static int gaudi2_dec_enable_msix(struct hl_device *hdev)
3477 {
3478     int rc, i, irq_init_cnt, irq, relative_idx;
3479     irq_handler_t irq_handler;
3480     struct hl_dec *dec;
3481 
3482     for (i = GAUDI2_IRQ_NUM_DCORE0_DEC0_NRM, irq_init_cnt = 0;
3483             i <= GAUDI2_IRQ_NUM_SHARED_DEC1_ABNRM;
3484             i++, irq_init_cnt++) {
3485 
3486         irq = pci_irq_vector(hdev->pdev, i);
3487         relative_idx = i - GAUDI2_IRQ_NUM_DCORE0_DEC0_NRM;
3488 
3489         irq_handler = (relative_idx % 2) ?
3490                 hl_irq_handler_dec_abnrm :
3491                 hl_irq_handler_user_interrupt;
3492 
3493         dec = hdev->dec + relative_idx / 2;
3494 
3495         /* We pass different structures depending on the irq handler. For the abnormal
3496          * interrupt we pass hl_dec and for the regular interrupt we pass the relevant
3497          * user_interrupt entry
3498          */
3499         rc = request_irq(irq, irq_handler, 0, gaudi2_irq_name(i),
3500                 ((relative_idx % 2) ?
3501                 (void *) dec :
3502                 (void *) &hdev->user_interrupt[dec->core_id]));
3503         if (rc) {
3504             dev_err(hdev->dev, "Failed to request IRQ %d", irq);
3505             goto free_dec_irqs;
3506         }
3507     }
3508 
3509     return 0;
3510 
3511 free_dec_irqs:
3512     gaudi2_dec_disable_msix(hdev, (GAUDI2_IRQ_NUM_DCORE0_DEC0_NRM + irq_init_cnt));
3513     return rc;
3514 }
3515 
3516 static int gaudi2_enable_msix(struct hl_device *hdev)
3517 {
3518     struct asic_fixed_properties *prop = &hdev->asic_prop;
3519     struct gaudi2_device *gaudi2 = hdev->asic_specific;
3520     int rc, irq, i, j, user_irq_init_cnt;
3521     irq_handler_t irq_handler;
3522     struct hl_cq *cq;
3523 
3524     if (gaudi2->hw_cap_initialized & HW_CAP_MSIX)
3525         return 0;
3526 
3527     rc = pci_alloc_irq_vectors(hdev->pdev, GAUDI2_MSIX_ENTRIES, GAUDI2_MSIX_ENTRIES,
3528                     PCI_IRQ_MSIX);
3529     if (rc < 0) {
3530         dev_err(hdev->dev, "MSI-X: Failed to enable support -- %d/%d\n",
3531             GAUDI2_MSIX_ENTRIES, rc);
3532         return rc;
3533     }
3534 
3535     irq = pci_irq_vector(hdev->pdev, GAUDI2_IRQ_NUM_COMPLETION);
3536     cq = &hdev->completion_queue[GAUDI2_RESERVED_CQ_CS_COMPLETION];
3537     rc = request_irq(irq, hl_irq_handler_cq, 0, gaudi2_irq_name(GAUDI2_IRQ_NUM_COMPLETION), cq);
3538     if (rc) {
3539         dev_err(hdev->dev, "Failed to request IRQ %d", irq);
3540         goto free_irq_vectors;
3541     }
3542 
3543     irq = pci_irq_vector(hdev->pdev, GAUDI2_IRQ_NUM_EVENT_QUEUE);
3544     rc = request_irq(irq, hl_irq_handler_eq, 0, gaudi2_irq_name(GAUDI2_IRQ_NUM_EVENT_QUEUE),
3545             &hdev->event_queue);
3546     if (rc) {
3547         dev_err(hdev->dev, "Failed to request IRQ %d", irq);
3548         goto free_completion_irq;
3549     }
3550 
3551     rc = gaudi2_dec_enable_msix(hdev);
3552     if (rc) {
3553         dev_err(hdev->dev, "Failed to enable decoder IRQ");
3554         goto free_completion_irq;
3555     }
3556 
3557     for (i = GAUDI2_IRQ_NUM_USER_FIRST, j = prop->user_dec_intr_count, user_irq_init_cnt = 0;
3558             user_irq_init_cnt < prop->user_interrupt_count;
3559             i++, j++, user_irq_init_cnt++) {
3560 
3561         irq = pci_irq_vector(hdev->pdev, i);
3562         irq_handler = hl_irq_handler_user_interrupt;
3563 
3564         rc = request_irq(irq, irq_handler, 0, gaudi2_irq_name(i), &hdev->user_interrupt[j]);
3565         if (rc) {
3566             dev_err(hdev->dev, "Failed to request IRQ %d", irq);
3567             goto free_user_irq;
3568         }
3569     }
3570 
3571     gaudi2->hw_cap_initialized |= HW_CAP_MSIX;
3572 
3573     return 0;
3574 
3575 free_user_irq:
3576     for (i = GAUDI2_IRQ_NUM_USER_FIRST, j = prop->user_dec_intr_count;
3577             i < GAUDI2_IRQ_NUM_USER_FIRST + user_irq_init_cnt ; i++, j++) {
3578 
3579         irq = pci_irq_vector(hdev->pdev, i);
3580         free_irq(irq, &hdev->user_interrupt[j]);
3581     }
3582 
3583     gaudi2_dec_disable_msix(hdev, GAUDI2_IRQ_NUM_SHARED_DEC1_ABNRM + 1);
3584 
3585 free_completion_irq:
3586     irq = pci_irq_vector(hdev->pdev, GAUDI2_IRQ_NUM_COMPLETION);
3587     free_irq(irq, cq);
3588 
3589 free_irq_vectors:
3590     pci_free_irq_vectors(hdev->pdev);
3591 
3592     return rc;
3593 }
3594 
3595 static void gaudi2_sync_irqs(struct hl_device *hdev)
3596 {
3597     struct gaudi2_device *gaudi2 = hdev->asic_specific;
3598     int i, j;
3599     int irq;
3600 
3601     if (!(gaudi2->hw_cap_initialized & HW_CAP_MSIX))
3602         return;
3603 
3604     /* Wait for all pending IRQs to be finished */
3605     synchronize_irq(pci_irq_vector(hdev->pdev, GAUDI2_IRQ_NUM_COMPLETION));
3606 
3607     for (i = GAUDI2_IRQ_NUM_DCORE0_DEC0_NRM ; i <= GAUDI2_IRQ_NUM_SHARED_DEC1_ABNRM ; i++) {
3608         irq = pci_irq_vector(hdev->pdev, i);
3609         synchronize_irq(irq);
3610     }
3611 
3612     for (i = GAUDI2_IRQ_NUM_USER_FIRST, j = 0 ; j < hdev->asic_prop.user_interrupt_count;
3613                                         i++, j++) {
3614         irq = pci_irq_vector(hdev->pdev, i);
3615         synchronize_irq(irq);
3616     }
3617 
3618     synchronize_irq(pci_irq_vector(hdev->pdev, GAUDI2_IRQ_NUM_EVENT_QUEUE));
3619 }
3620 
3621 static void gaudi2_disable_msix(struct hl_device *hdev)
3622 {
3623     struct asic_fixed_properties *prop = &hdev->asic_prop;
3624     struct gaudi2_device *gaudi2 = hdev->asic_specific;
3625     struct hl_cq *cq;
3626     int irq, i, j, k;
3627 
3628     if (!(gaudi2->hw_cap_initialized & HW_CAP_MSIX))
3629         return;
3630 
3631     gaudi2_sync_irqs(hdev);
3632 
3633     irq = pci_irq_vector(hdev->pdev, GAUDI2_IRQ_NUM_EVENT_QUEUE);
3634     free_irq(irq, &hdev->event_queue);
3635 
3636     gaudi2_dec_disable_msix(hdev, GAUDI2_IRQ_NUM_SHARED_DEC1_ABNRM + 1);
3637 
3638     for (i = GAUDI2_IRQ_NUM_USER_FIRST, j = prop->user_dec_intr_count, k = 0;
3639             k < hdev->asic_prop.user_interrupt_count ; i++, j++, k++) {
3640 
3641         irq = pci_irq_vector(hdev->pdev, i);
3642         free_irq(irq, &hdev->user_interrupt[j]);
3643     }
3644 
3645     irq = pci_irq_vector(hdev->pdev, GAUDI2_IRQ_NUM_COMPLETION);
3646     cq = &hdev->completion_queue[GAUDI2_RESERVED_CQ_CS_COMPLETION];
3647     free_irq(irq, cq);
3648 
3649     pci_free_irq_vectors(hdev->pdev);
3650 
3651     gaudi2->hw_cap_initialized &= ~HW_CAP_MSIX;
3652 }
3653 
3654 static void gaudi2_stop_dcore_dec(struct hl_device *hdev, int dcore_id)
3655 {
3656     u32 reg_val = FIELD_PREP(DCORE0_VDEC0_BRDG_CTRL_GRACEFUL_STOP_MASK, 0x1);
3657     u32 graceful_pend_mask = DCORE0_VDEC0_BRDG_CTRL_GRACEFUL_PEND_MASK;
3658     u32 timeout_usec, dec_id, dec_bit, offset, graceful;
3659     int rc;
3660 
3661     if (hdev->pldm)
3662         timeout_usec = GAUDI2_PLDM_VDEC_TIMEOUT_USEC;
3663     else
3664         timeout_usec = GAUDI2_VDEC_TIMEOUT_USEC;
3665 
3666     for (dec_id = 0 ; dec_id < NUM_OF_DEC_PER_DCORE ; dec_id++) {
3667         dec_bit = dcore_id * NUM_OF_DEC_PER_DCORE + dec_id;
3668         if (!(hdev->asic_prop.decoder_enabled_mask & BIT(dec_bit)))
3669             continue;
3670 
3671         offset = dcore_id * DCORE_OFFSET + dec_id * DCORE_VDEC_OFFSET;
3672 
3673         WREG32(mmDCORE0_DEC0_CMD_SWREG16 + offset, 0);
3674 
3675         WREG32(mmDCORE0_VDEC0_BRDG_CTRL_GRACEFUL + offset, reg_val);
3676 
3677         /* Wait till all traffic from decoder stops
3678          * before apply core reset.
3679          */
3680         rc = hl_poll_timeout(
3681                 hdev,
3682                 mmDCORE0_VDEC0_BRDG_CTRL_GRACEFUL + offset,
3683                 graceful,
3684                 (graceful & graceful_pend_mask),
3685                 100,
3686                 timeout_usec);
3687         if (rc)
3688             dev_err(hdev->dev,
3689                 "Failed to stop traffic from DCORE%d Decoder %d\n",
3690                 dcore_id, dec_id);
3691     }
3692 }
3693 
3694 static void gaudi2_stop_pcie_dec(struct hl_device *hdev)
3695 {
3696     u32 reg_val = FIELD_PREP(DCORE0_VDEC0_BRDG_CTRL_GRACEFUL_STOP_MASK, 0x1);
3697     u32 graceful_pend_mask = PCIE_VDEC0_BRDG_CTRL_GRACEFUL_PEND_MASK;
3698     u32 timeout_usec, dec_id, dec_bit, offset, graceful;
3699     int rc;
3700 
3701     if (hdev->pldm)
3702         timeout_usec = GAUDI2_PLDM_VDEC_TIMEOUT_USEC;
3703     else
3704         timeout_usec = GAUDI2_VDEC_TIMEOUT_USEC;
3705 
3706     for (dec_id = 0 ; dec_id < NUM_OF_DEC_PER_DCORE ; dec_id++) {
3707         dec_bit = PCIE_DEC_SHIFT + dec_id;
3708         if (!(hdev->asic_prop.decoder_enabled_mask & BIT(dec_bit)))
3709             continue;
3710 
3711         offset = dec_id * PCIE_VDEC_OFFSET;
3712 
3713         WREG32(mmPCIE_DEC0_CMD_SWREG16 + offset, 0);
3714 
3715         WREG32(mmPCIE_VDEC0_BRDG_CTRL_GRACEFUL + offset, reg_val);
3716 
3717         /* Wait till all traffic from decoder stops
3718          * before apply core reset.
3719          */
3720         rc = hl_poll_timeout(
3721                 hdev,
3722                 mmPCIE_VDEC0_BRDG_CTRL_GRACEFUL + offset,
3723                 graceful,
3724                 (graceful & graceful_pend_mask),
3725                 100,
3726                 timeout_usec);
3727         if (rc)
3728             dev_err(hdev->dev,
3729                 "Failed to stop traffic from PCIe Decoder %d\n",
3730                 dec_id);
3731     }
3732 }
3733 
3734 static void gaudi2_stop_dec(struct hl_device *hdev)
3735 {
3736     struct gaudi2_device *gaudi2 = hdev->asic_specific;
3737     int dcore_id;
3738 
3739     if ((gaudi2->dec_hw_cap_initialized & HW_CAP_DEC_MASK) == 0)
3740         return;
3741 
3742     for (dcore_id = 0 ; dcore_id < NUM_OF_DCORES ; dcore_id++)
3743         gaudi2_stop_dcore_dec(hdev, dcore_id);
3744 
3745     gaudi2_stop_pcie_dec(hdev);
3746 }
3747 
3748 static void gaudi2_halt_arc(struct hl_device *hdev, u32 cpu_id)
3749 {
3750     u32 reg_base, reg_val;
3751 
3752     reg_base = gaudi2_arc_blocks_bases[cpu_id];
3753 
3754     /* Halt ARC */
3755     reg_val = FIELD_PREP(ARC_FARM_ARC0_AUX_RUN_HALT_REQ_HALT_REQ_MASK, 1);
3756     WREG32(reg_base + ARC_HALT_REQ_OFFSET, reg_val);
3757 }
3758 
3759 static void gaudi2_halt_arcs(struct hl_device *hdev)
3760 {
3761     u16 arc_id;
3762 
3763     for (arc_id = CPU_ID_SCHED_ARC0; arc_id < CPU_ID_MAX; arc_id++) {
3764         if (gaudi2_is_arc_enabled(hdev, arc_id))
3765             gaudi2_halt_arc(hdev, arc_id);
3766     }
3767 }
3768 
3769 static void gaudi2_reset_arcs(struct hl_device *hdev)
3770 {
3771     struct gaudi2_device *gaudi2 = hdev->asic_specific;
3772     u16 arc_id;
3773 
3774     if (!gaudi2)
3775         return;
3776 
3777     for (arc_id = CPU_ID_SCHED_ARC0; arc_id < CPU_ID_MAX; arc_id++)
3778         if (gaudi2_is_arc_enabled(hdev, arc_id))
3779             gaudi2_clr_arc_id_cap(hdev, arc_id);
3780 }
3781 
3782 static void gaudi2_nic_qmans_manual_flush(struct hl_device *hdev)
3783 {
3784     struct gaudi2_device *gaudi2 = hdev->asic_specific;
3785     u32 queue_id;
3786     int i;
3787 
3788     if (!(gaudi2->nic_hw_cap_initialized & HW_CAP_NIC_MASK))
3789         return;
3790 
3791     queue_id = GAUDI2_QUEUE_ID_NIC_0_0;
3792 
3793     for (i = 0 ; i < NIC_NUMBER_OF_ENGINES ; i++, queue_id += NUM_OF_PQ_PER_QMAN)
3794         gaudi2_qman_manual_flush_common(hdev, queue_id);
3795 }
3796 
3797 static void gaudi2_halt_engines(struct hl_device *hdev, bool hard_reset, bool fw_reset)
3798 {
3799     u32 wait_timeout_ms;
3800 
3801     if (hdev->pldm)
3802         wait_timeout_ms = GAUDI2_PLDM_RESET_WAIT_MSEC;
3803     else
3804         wait_timeout_ms = GAUDI2_RESET_WAIT_MSEC;
3805 
3806     if (fw_reset)
3807         goto skip_engines;
3808 
3809     gaudi2_stop_dma_qmans(hdev);
3810     gaudi2_stop_mme_qmans(hdev);
3811     gaudi2_stop_tpc_qmans(hdev);
3812     gaudi2_stop_rot_qmans(hdev);
3813     gaudi2_stop_nic_qmans(hdev);
3814     msleep(wait_timeout_ms);
3815 
3816     gaudi2_halt_arcs(hdev);
3817     gaudi2_dma_stall(hdev);
3818     gaudi2_mme_stall(hdev);
3819     gaudi2_tpc_stall(hdev);
3820     gaudi2_rotator_stall(hdev);
3821 
3822     msleep(wait_timeout_ms);
3823 
3824     gaudi2_stop_dec(hdev);
3825 
3826     /*
3827      * in case of soft reset do a manual flush for QMANs (currently called
3828      * only for NIC QMANs
3829      */
3830     if (!hard_reset)
3831         gaudi2_nic_qmans_manual_flush(hdev);
3832 
3833     gaudi2_disable_dma_qmans(hdev);
3834     gaudi2_disable_mme_qmans(hdev);
3835     gaudi2_disable_tpc_qmans(hdev);
3836     gaudi2_disable_rot_qmans(hdev);
3837     gaudi2_disable_nic_qmans(hdev);
3838     gaudi2_disable_timestamp(hdev);
3839 
3840 skip_engines:
3841     if (hard_reset) {
3842         gaudi2_disable_msix(hdev);
3843         return;
3844     }
3845 
3846     gaudi2_sync_irqs(hdev);
3847 }
3848 
3849 static void gaudi2_init_firmware_preload_params(struct hl_device *hdev)
3850 {
3851     struct pre_fw_load_props *pre_fw_load = &hdev->fw_loader.pre_fw_load;
3852 
3853     pre_fw_load->cpu_boot_status_reg = mmPSOC_GLOBAL_CONF_CPU_BOOT_STATUS;
3854     pre_fw_load->sts_boot_dev_sts0_reg = mmCPU_BOOT_DEV_STS0;
3855     pre_fw_load->sts_boot_dev_sts1_reg = mmCPU_BOOT_DEV_STS1;
3856     pre_fw_load->boot_err0_reg = mmCPU_BOOT_ERR0;
3857     pre_fw_load->boot_err1_reg = mmCPU_BOOT_ERR1;
3858     pre_fw_load->wait_for_preboot_timeout = GAUDI2_PREBOOT_REQ_TIMEOUT_USEC;
3859 }
3860 
3861 static void gaudi2_init_firmware_loader(struct hl_device *hdev)
3862 {
3863     struct fw_load_mgr *fw_loader = &hdev->fw_loader;
3864     struct dynamic_fw_load_mgr *dynamic_loader;
3865     struct cpu_dyn_regs *dyn_regs;
3866 
3867     /* fill common fields */
3868     fw_loader->fw_comp_loaded = FW_TYPE_NONE;
3869     fw_loader->boot_fit_img.image_name = GAUDI2_BOOT_FIT_FILE;
3870     fw_loader->linux_img.image_name = GAUDI2_LINUX_FW_FILE;
3871     fw_loader->boot_fit_timeout = GAUDI2_BOOT_FIT_REQ_TIMEOUT_USEC;
3872     fw_loader->skip_bmc = false;
3873     fw_loader->sram_bar_id = SRAM_CFG_BAR_ID;
3874     fw_loader->dram_bar_id = DRAM_BAR_ID;
3875 
3876     if (hdev->asic_type == ASIC_GAUDI2 || hdev->asic_type == ASIC_GAUDI2_SEC)
3877         fw_loader->cpu_timeout = GAUDI2_CPU_TIMEOUT_USEC;
3878     else /* ASIC_GAUDI2_FPGA */
3879         fw_loader->cpu_timeout = GAUDI2_FPGA_CPU_TIMEOUT;
3880 
3881     /* here we update initial values for few specific dynamic regs (as
3882      * before reading the first descriptor from FW those value has to be
3883      * hard-coded). in later stages of the protocol those values will be
3884      * updated automatically by reading the FW descriptor so data there
3885      * will always be up-to-date
3886      */
3887     dynamic_loader = &hdev->fw_loader.dynamic_loader;
3888     dyn_regs = &dynamic_loader->comm_desc.cpu_dyn_regs;
3889     dyn_regs->kmd_msg_to_cpu = cpu_to_le32(mmPSOC_GLOBAL_CONF_KMD_MSG_TO_CPU);
3890     dyn_regs->cpu_cmd_status_to_host = cpu_to_le32(mmCPU_CMD_STATUS_TO_HOST);
3891     dynamic_loader->wait_for_bl_timeout = GAUDI2_WAIT_FOR_BL_TIMEOUT_USEC;
3892 }
3893 
3894 static int gaudi2_init_cpu(struct hl_device *hdev)
3895 {
3896     struct gaudi2_device *gaudi2 = hdev->asic_specific;
3897     int rc;
3898 
3899     if (!(hdev->fw_components & FW_TYPE_PREBOOT_CPU))
3900         return 0;
3901 
3902     if (gaudi2->hw_cap_initialized & HW_CAP_CPU)
3903         return 0;
3904 
3905     rc = hl_fw_init_cpu(hdev);
3906     if (rc)
3907         return rc;
3908 
3909     gaudi2->hw_cap_initialized |= HW_CAP_CPU;
3910 
3911     return 0;
3912 }
3913 
3914 static int gaudi2_init_cpu_queues(struct hl_device *hdev, u32 cpu_timeout)
3915 {
3916     struct hl_hw_queue *cpu_pq = &hdev->kernel_queues[GAUDI2_QUEUE_ID_CPU_PQ];
3917     struct asic_fixed_properties *prop = &hdev->asic_prop;
3918     struct gaudi2_device *gaudi2 = hdev->asic_specific;
3919     struct cpu_dyn_regs *dyn_regs;
3920     struct hl_eq *eq;
3921     u32 status;
3922     int err;
3923 
3924     if (!hdev->cpu_queues_enable)
3925         return 0;
3926 
3927     if (gaudi2->hw_cap_initialized & HW_CAP_CPU_Q)
3928         return 0;
3929 
3930     eq = &hdev->event_queue;
3931 
3932     dyn_regs = &hdev->fw_loader.dynamic_loader.comm_desc.cpu_dyn_regs;
3933 
3934     WREG32(mmCPU_IF_PQ_BASE_ADDR_LOW, lower_32_bits(cpu_pq->bus_address));
3935     WREG32(mmCPU_IF_PQ_BASE_ADDR_HIGH, upper_32_bits(cpu_pq->bus_address));
3936 
3937     WREG32(mmCPU_IF_EQ_BASE_ADDR_LOW, lower_32_bits(eq->bus_address));
3938     WREG32(mmCPU_IF_EQ_BASE_ADDR_HIGH, upper_32_bits(eq->bus_address));
3939 
3940     WREG32(mmCPU_IF_CQ_BASE_ADDR_LOW, lower_32_bits(hdev->cpu_accessible_dma_address));
3941     WREG32(mmCPU_IF_CQ_BASE_ADDR_HIGH, upper_32_bits(hdev->cpu_accessible_dma_address));
3942 
3943     WREG32(mmCPU_IF_PQ_LENGTH, HL_QUEUE_SIZE_IN_BYTES);
3944     WREG32(mmCPU_IF_EQ_LENGTH, HL_EQ_SIZE_IN_BYTES);
3945     WREG32(mmCPU_IF_CQ_LENGTH, HL_CPU_ACCESSIBLE_MEM_SIZE);
3946 
3947     /* Used for EQ CI */
3948     WREG32(mmCPU_IF_EQ_RD_OFFS, 0);
3949 
3950     WREG32(mmCPU_IF_PF_PQ_PI, 0);
3951 
3952     WREG32(mmCPU_IF_QUEUE_INIT, PQ_INIT_STATUS_READY_FOR_CP);
3953 
3954     /* Let the ARC know we are ready as it is now handling those queues  */
3955 
3956     WREG32(le32_to_cpu(dyn_regs->gic_host_pi_upd_irq),
3957         gaudi2_irq_map_table[GAUDI2_EVENT_CPU_PI_UPDATE].cpu_id);
3958 
3959     err = hl_poll_timeout(
3960         hdev,
3961         mmCPU_IF_QUEUE_INIT,
3962         status,
3963         (status == PQ_INIT_STATUS_READY_FOR_HOST),
3964         1000,
3965         cpu_timeout);
3966 
3967     if (err) {
3968         dev_err(hdev->dev, "Failed to communicate with device CPU (timeout)\n");
3969         return -EIO;
3970     }
3971 
3972     /* update FW application security bits */
3973     if (prop->fw_cpu_boot_dev_sts0_valid)
3974         prop->fw_app_cpu_boot_dev_sts0 = RREG32(mmCPU_BOOT_DEV_STS0);
3975 
3976     if (prop->fw_cpu_boot_dev_sts1_valid)
3977         prop->fw_app_cpu_boot_dev_sts1 = RREG32(mmCPU_BOOT_DEV_STS1);
3978 
3979     gaudi2->hw_cap_initialized |= HW_CAP_CPU_Q;
3980     return 0;
3981 }
3982 
3983 static void gaudi2_init_qman_pq(struct hl_device *hdev, u32 reg_base,
3984                 u32 queue_id_base)
3985 {
3986     struct hl_hw_queue *q;
3987     u32 pq_id, pq_offset;
3988 
3989     for (pq_id = 0 ; pq_id < NUM_OF_PQ_PER_QMAN ; pq_id++) {
3990         q = &hdev->kernel_queues[queue_id_base + pq_id];
3991         pq_offset = pq_id * 4;
3992 
3993         WREG32(reg_base + QM_PQ_BASE_LO_0_OFFSET + pq_offset,
3994                 lower_32_bits(q->bus_address));
3995         WREG32(reg_base + QM_PQ_BASE_HI_0_OFFSET + pq_offset,
3996                 upper_32_bits(q->bus_address));
3997         WREG32(reg_base + QM_PQ_SIZE_0_OFFSET + pq_offset, ilog2(HL_QUEUE_LENGTH));
3998         WREG32(reg_base + QM_PQ_PI_0_OFFSET + pq_offset, 0);
3999         WREG32(reg_base + QM_PQ_CI_0_OFFSET + pq_offset, 0);
4000     }
4001 }
4002 
4003 static void gaudi2_init_qman_cp(struct hl_device *hdev, u32 reg_base)
4004 {
4005     u32 cp_id, cp_offset, mtr_base_lo, mtr_base_hi, so_base_lo, so_base_hi;
4006 
4007     mtr_base_lo = lower_32_bits(CFG_BASE + mmDCORE0_SYNC_MNGR_OBJS_MON_PAY_ADDRL_0);
4008     mtr_base_hi = upper_32_bits(CFG_BASE + mmDCORE0_SYNC_MNGR_OBJS_MON_PAY_ADDRL_0);
4009     so_base_lo = lower_32_bits(CFG_BASE + mmDCORE0_SYNC_MNGR_OBJS_SOB_OBJ_0);
4010     so_base_hi = upper_32_bits(CFG_BASE + mmDCORE0_SYNC_MNGR_OBJS_SOB_OBJ_0);
4011 
4012     for (cp_id = 0 ; cp_id < NUM_OF_CP_PER_QMAN; cp_id++) {
4013         cp_offset = cp_id * 4;
4014 
4015         WREG32(reg_base + QM_CP_MSG_BASE0_ADDR_LO_0_OFFSET + cp_offset, mtr_base_lo);
4016         WREG32(reg_base + QM_CP_MSG_BASE0_ADDR_HI_0_OFFSET + cp_offset, mtr_base_hi);
4017         WREG32(reg_base + QM_CP_MSG_BASE1_ADDR_LO_0_OFFSET + cp_offset, so_base_lo);
4018         WREG32(reg_base + QM_CP_MSG_BASE1_ADDR_HI_0_OFFSET + cp_offset, so_base_hi);
4019     }
4020 
4021     /* allow QMANs to accept work from ARC CQF */
4022     WREG32(reg_base + QM_CP_CFG_OFFSET, FIELD_PREP(PDMA0_QM_CP_CFG_SWITCH_EN_MASK, 0x1));
4023 }
4024 
4025 static void gaudi2_init_qman_pqc(struct hl_device *hdev, u32 reg_base,
4026                 u32 queue_id_base)
4027 {
4028     struct gaudi2_device *gaudi2 = hdev->asic_specific;
4029     u32 pq_id, pq_offset, so_base_lo, so_base_hi;
4030 
4031     so_base_lo = lower_32_bits(CFG_BASE + mmDCORE0_SYNC_MNGR_OBJS_SOB_OBJ_0);
4032     so_base_hi = upper_32_bits(CFG_BASE + mmDCORE0_SYNC_MNGR_OBJS_SOB_OBJ_0);
4033 
4034     for (pq_id = 0 ; pq_id < NUM_OF_PQ_PER_QMAN ; pq_id++) {
4035         pq_offset = pq_id * 4;
4036 
4037         /* Configure QMAN HBW to scratchpad as it is not needed */
4038         WREG32(reg_base + QM_PQC_HBW_BASE_LO_0_OFFSET + pq_offset,
4039                 lower_32_bits(gaudi2->scratchpad_bus_address));
4040         WREG32(reg_base + QM_PQC_HBW_BASE_HI_0_OFFSET + pq_offset,
4041                 upper_32_bits(gaudi2->scratchpad_bus_address));
4042         WREG32(reg_base + QM_PQC_SIZE_0_OFFSET + pq_offset,
4043                 ilog2(PAGE_SIZE / sizeof(struct hl_cq_entry)));
4044 
4045         WREG32(reg_base + QM_PQC_PI_0_OFFSET + pq_offset, 0);
4046         WREG32(reg_base + QM_PQC_LBW_WDATA_0_OFFSET + pq_offset, QM_PQC_LBW_WDATA);
4047         WREG32(reg_base + QM_PQC_LBW_BASE_LO_0_OFFSET + pq_offset, so_base_lo);
4048         WREG32(reg_base + QM_PQC_LBW_BASE_HI_0_OFFSET + pq_offset, so_base_hi);
4049     }
4050 
4051     /* Enable QMAN H/W completion */
4052     WREG32(reg_base + QM_PQC_CFG_OFFSET, 1 << PDMA0_QM_PQC_CFG_EN_SHIFT);
4053 }
4054 
4055 static u32 gaudi2_get_dyn_sp_reg(struct hl_device *hdev, u32 queue_id_base)
4056 {
4057     struct cpu_dyn_regs *dyn_regs = &hdev->fw_loader.dynamic_loader.comm_desc.cpu_dyn_regs;
4058     u32 sp_reg_addr;
4059 
4060     switch (queue_id_base) {
4061     case GAUDI2_QUEUE_ID_PDMA_0_0...GAUDI2_QUEUE_ID_PDMA_1_3:
4062         fallthrough;
4063     case GAUDI2_QUEUE_ID_DCORE0_EDMA_0_0...GAUDI2_QUEUE_ID_DCORE0_EDMA_1_3:
4064         fallthrough;
4065     case GAUDI2_QUEUE_ID_DCORE1_EDMA_0_0...GAUDI2_QUEUE_ID_DCORE1_EDMA_1_3:
4066         fallthrough;
4067     case GAUDI2_QUEUE_ID_DCORE2_EDMA_0_0...GAUDI2_QUEUE_ID_DCORE2_EDMA_1_3:
4068         fallthrough;
4069     case GAUDI2_QUEUE_ID_DCORE3_EDMA_0_0...GAUDI2_QUEUE_ID_DCORE3_EDMA_1_3:
4070         sp_reg_addr = le32_to_cpu(dyn_regs->gic_dma_qm_irq_ctrl);
4071         break;
4072     case GAUDI2_QUEUE_ID_DCORE0_MME_0_0...GAUDI2_QUEUE_ID_DCORE0_MME_0_3:
4073         fallthrough;
4074     case GAUDI2_QUEUE_ID_DCORE1_MME_0_0...GAUDI2_QUEUE_ID_DCORE1_MME_0_3:
4075         fallthrough;
4076     case GAUDI2_QUEUE_ID_DCORE2_MME_0_0...GAUDI2_QUEUE_ID_DCORE2_MME_0_3:
4077         fallthrough;
4078     case GAUDI2_QUEUE_ID_DCORE3_MME_0_0...GAUDI2_QUEUE_ID_DCORE3_MME_0_3:
4079         sp_reg_addr = le32_to_cpu(dyn_regs->gic_mme_qm_irq_ctrl);
4080         break;
4081     case GAUDI2_QUEUE_ID_DCORE0_TPC_0_0 ... GAUDI2_QUEUE_ID_DCORE0_TPC_6_3:
4082         fallthrough;
4083     case GAUDI2_QUEUE_ID_DCORE1_TPC_0_0 ... GAUDI2_QUEUE_ID_DCORE1_TPC_5_3:
4084         fallthrough;
4085     case GAUDI2_QUEUE_ID_DCORE2_TPC_0_0 ... GAUDI2_QUEUE_ID_DCORE2_TPC_5_3:
4086         fallthrough;
4087     case GAUDI2_QUEUE_ID_DCORE3_TPC_0_0 ... GAUDI2_QUEUE_ID_DCORE3_TPC_5_3:
4088         sp_reg_addr = le32_to_cpu(dyn_regs->gic_tpc_qm_irq_ctrl);
4089         break;
4090     case GAUDI2_QUEUE_ID_ROT_0_0...GAUDI2_QUEUE_ID_ROT_1_3:
4091         sp_reg_addr = le32_to_cpu(dyn_regs->gic_rot_qm_irq_ctrl);
4092         break;
4093     case GAUDI2_QUEUE_ID_NIC_0_0...GAUDI2_QUEUE_ID_NIC_23_3:
4094         sp_reg_addr = le32_to_cpu(dyn_regs->gic_nic_qm_irq_ctrl);
4095         break;
4096     default:
4097         dev_err(hdev->dev, "Unexpected h/w queue %d\n", queue_id_base);
4098         return 0;
4099     }
4100 
4101     return sp_reg_addr;
4102 }
4103 
4104 static void gaudi2_init_qman_common(struct hl_device *hdev, u32 reg_base,
4105                     u32 queue_id_base)
4106 {
4107     u32 glbl_prot = QMAN_MAKE_TRUSTED, irq_handler_offset;
4108     int map_table_entry;
4109 
4110     WREG32(reg_base + QM_GLBL_PROT_OFFSET, glbl_prot);
4111 
4112     irq_handler_offset = gaudi2_get_dyn_sp_reg(hdev, queue_id_base);
4113     WREG32(reg_base + QM_GLBL_ERR_ADDR_LO_OFFSET, lower_32_bits(CFG_BASE + irq_handler_offset));
4114     WREG32(reg_base + QM_GLBL_ERR_ADDR_HI_OFFSET, upper_32_bits(CFG_BASE + irq_handler_offset));
4115 
4116     map_table_entry = gaudi2_qman_async_event_id[queue_id_base];
4117     WREG32(reg_base + QM_GLBL_ERR_WDATA_OFFSET,
4118         gaudi2_irq_map_table[map_table_entry].cpu_id);
4119 
4120     WREG32(reg_base + QM_ARB_ERR_MSG_EN_OFFSET, QM_ARB_ERR_MSG_EN_MASK);
4121 
4122     WREG32(reg_base + QM_ARB_SLV_CHOISE_WDT_OFFSET, GAUDI2_ARB_WDT_TIMEOUT);
4123     WREG32(reg_base + QM_GLBL_CFG1_OFFSET, 0);
4124     WREG32(reg_base + QM_GLBL_CFG2_OFFSET, 0);
4125 
4126     /* Enable the QMAN channel.
4127      * PDMA1 QMAN configuration is different, as we do not allow user to
4128      * access CP2/3, it is reserved for the ARC usage.
4129      */
4130     if (reg_base == gaudi2_qm_blocks_bases[GAUDI2_QUEUE_ID_PDMA_1_0])
4131         WREG32(reg_base + QM_GLBL_CFG0_OFFSET, PDMA1_QMAN_ENABLE);
4132     else
4133         WREG32(reg_base + QM_GLBL_CFG0_OFFSET, QMAN_ENABLE);
4134 }
4135 
4136 static void gaudi2_init_qman(struct hl_device *hdev, u32 reg_base,
4137         u32 queue_id_base)
4138 {
4139     u32 pq_id;
4140 
4141     for (pq_id = 0 ; pq_id < NUM_OF_PQ_PER_QMAN ; pq_id++)
4142         hdev->kernel_queues[queue_id_base + pq_id].cq_id = GAUDI2_RESERVED_CQ_CS_COMPLETION;
4143 
4144     gaudi2_init_qman_pq(hdev, reg_base, queue_id_base);
4145     gaudi2_init_qman_cp(hdev, reg_base);
4146     gaudi2_init_qman_pqc(hdev, reg_base, queue_id_base);
4147     gaudi2_init_qman_common(hdev, reg_base, queue_id_base);
4148 }
4149 
4150 static void gaudi2_init_dma_core(struct hl_device *hdev, u32 reg_base,
4151                 u32 dma_core_id, bool is_secure)
4152 {
4153     u32 prot, irq_handler_offset;
4154     struct cpu_dyn_regs *dyn_regs;
4155     int map_table_entry;
4156 
4157     prot = 1 << ARC_FARM_KDMA_PROT_ERR_VAL_SHIFT;
4158     if (is_secure)
4159         prot |= 1 << ARC_FARM_KDMA_PROT_VAL_SHIFT;
4160 
4161     WREG32(reg_base + DMA_CORE_PROT_OFFSET, prot);
4162 
4163     dyn_regs = &hdev->fw_loader.dynamic_loader.comm_desc.cpu_dyn_regs;
4164     irq_handler_offset = le32_to_cpu(dyn_regs->gic_dma_core_irq_ctrl);
4165 
4166     WREG32(reg_base + DMA_CORE_ERRMSG_ADDR_LO_OFFSET,
4167             lower_32_bits(CFG_BASE + irq_handler_offset));
4168 
4169     WREG32(reg_base + DMA_CORE_ERRMSG_ADDR_HI_OFFSET,
4170             upper_32_bits(CFG_BASE + irq_handler_offset));
4171 
4172     map_table_entry = gaudi2_dma_core_async_event_id[dma_core_id];
4173     WREG32(reg_base + DMA_CORE_ERRMSG_WDATA_OFFSET,
4174         gaudi2_irq_map_table[map_table_entry].cpu_id);
4175 
4176     /* Enable the DMA channel */
4177     WREG32(reg_base + DMA_CORE_CFG_0_OFFSET, 1 << ARC_FARM_KDMA_CFG_0_EN_SHIFT);
4178 }
4179 
4180 static void gaudi2_init_kdma(struct hl_device *hdev)
4181 {
4182     struct gaudi2_device *gaudi2 = hdev->asic_specific;
4183     u32 reg_base;
4184 
4185     if ((gaudi2->hw_cap_initialized & HW_CAP_KDMA) == HW_CAP_KDMA)
4186         return;
4187 
4188     reg_base = gaudi2_dma_core_blocks_bases[DMA_CORE_ID_KDMA];
4189 
4190     gaudi2_init_dma_core(hdev, reg_base, DMA_CORE_ID_KDMA, true);
4191 
4192     gaudi2->hw_cap_initialized |= HW_CAP_KDMA;
4193 }
4194 
4195 static void gaudi2_init_pdma(struct hl_device *hdev)
4196 {
4197     struct gaudi2_device *gaudi2 = hdev->asic_specific;
4198     u32 reg_base;
4199 
4200     if ((gaudi2->hw_cap_initialized & HW_CAP_PDMA_MASK) == HW_CAP_PDMA_MASK)
4201         return;
4202 
4203     reg_base = gaudi2_dma_core_blocks_bases[DMA_CORE_ID_PDMA0];
4204     gaudi2_init_dma_core(hdev, reg_base, DMA_CORE_ID_PDMA0, false);
4205 
4206     reg_base = gaudi2_qm_blocks_bases[GAUDI2_QUEUE_ID_PDMA_0_0];
4207     gaudi2_init_qman(hdev, reg_base, GAUDI2_QUEUE_ID_PDMA_0_0);
4208 
4209     reg_base = gaudi2_dma_core_blocks_bases[DMA_CORE_ID_PDMA1];
4210     gaudi2_init_dma_core(hdev, reg_base, DMA_CORE_ID_PDMA1, false);
4211 
4212     reg_base = gaudi2_qm_blocks_bases[GAUDI2_QUEUE_ID_PDMA_1_0];
4213     gaudi2_init_qman(hdev, reg_base, GAUDI2_QUEUE_ID_PDMA_1_0);
4214 
4215     gaudi2->hw_cap_initialized |= HW_CAP_PDMA_MASK;
4216 }
4217 
4218 static void gaudi2_init_edma_instance(struct hl_device *hdev, u8 seq)
4219 {
4220     u32 reg_base, base_edma_core_id, base_edma_qman_id;
4221 
4222     base_edma_core_id = DMA_CORE_ID_EDMA0 + seq;
4223     base_edma_qman_id = edma_stream_base[seq];
4224 
4225     reg_base = gaudi2_dma_core_blocks_bases[base_edma_core_id];
4226     gaudi2_init_dma_core(hdev, reg_base, base_edma_core_id, false);
4227 
4228     reg_base = gaudi2_qm_blocks_bases[base_edma_qman_id];
4229     gaudi2_init_qman(hdev, reg_base, base_edma_qman_id);
4230 }
4231 
4232 static void gaudi2_init_edma(struct hl_device *hdev)
4233 {
4234     struct asic_fixed_properties *prop = &hdev->asic_prop;
4235     struct gaudi2_device *gaudi2 = hdev->asic_specific;
4236     int dcore, inst;
4237 
4238     if ((gaudi2->hw_cap_initialized & HW_CAP_EDMA_MASK) == HW_CAP_EDMA_MASK)
4239         return;
4240 
4241     for (dcore = 0 ; dcore < NUM_OF_DCORES ; dcore++) {
4242         for (inst = 0 ; inst < NUM_OF_EDMA_PER_DCORE ; inst++) {
4243             u8 seq = dcore * NUM_OF_EDMA_PER_DCORE + inst;
4244 
4245             if (!(prop->edma_enabled_mask & BIT(seq)))
4246                 continue;
4247 
4248             gaudi2_init_edma_instance(hdev, seq);
4249 
4250             gaudi2->hw_cap_initialized |= BIT_ULL(HW_CAP_EDMA_SHIFT + seq);
4251         }
4252     }
4253 }
4254 
4255 /*
4256  * gaudi2_arm_monitors_for_virt_msix_db() - Arm monitors for writing to the virtual MSI-X doorbell.
4257  * @hdev: pointer to habanalabs device structure.
4258  * @sob_id: sync object ID.
4259  * @first_mon_id: ID of first monitor out of 3 consecutive monitors.
4260  * @interrupt_id: interrupt ID.
4261  *
4262  * Some initiators cannot have HBW address in their completion address registers, and thus cannot
4263  * write directly to the HBW host memory of the virtual MSI-X doorbell.
4264  * Instead, they are configured to LBW write to a sync object, and a monitor will do the HBW write.
4265  *
4266  * The mechanism in the sync manager block is composed of a master monitor with 3 messages.
4267  * In addition to the HBW write, the other 2 messages are for preparing the monitor to next
4268  * completion, by decrementing the sync object value and re-arming the monitor.
4269  */
4270 static void gaudi2_arm_monitors_for_virt_msix_db(struct hl_device *hdev, u32 sob_id,
4271                             u32 first_mon_id, u32 interrupt_id)
4272 {
4273     u32 sob_offset, first_mon_offset, mon_offset, payload, sob_group, mode, arm, config;
4274     struct gaudi2_device *gaudi2 = hdev->asic_specific;
4275     u64 addr;
4276     u8 mask;
4277 
4278     /* Reset the SOB value */
4279     sob_offset = sob_id * sizeof(u32);
4280     WREG32(mmDCORE0_SYNC_MNGR_OBJS_SOB_OBJ_0 + sob_offset, 0);
4281 
4282     /* Configure 3 monitors:
4283      * 1. Write interrupt ID to the virtual MSI-X doorbell (master monitor)
4284      * 2. Decrement SOB value by 1.
4285      * 3. Re-arm the master monitor.
4286      */
4287 
4288     first_mon_offset = first_mon_id * sizeof(u32);
4289 
4290     /* 2nd monitor: Decrement SOB value by 1 */
4291     mon_offset = first_mon_offset + sizeof(u32);
4292 
4293     addr = CFG_BASE + mmDCORE0_SYNC_MNGR_OBJS_SOB_OBJ_0 + sob_offset;
4294     WREG32(mmDCORE0_SYNC_MNGR_OBJS_MON_PAY_ADDRL_0 + mon_offset, lower_32_bits(addr));
4295     WREG32(mmDCORE0_SYNC_MNGR_OBJS_MON_PAY_ADDRH_0 + mon_offset, upper_32_bits(addr));
4296 
4297     payload = FIELD_PREP(DCORE0_SYNC_MNGR_OBJS_SOB_OBJ_VAL_MASK, 0x7FFF) | /* "-1" */
4298             FIELD_PREP(DCORE0_SYNC_MNGR_OBJS_SOB_OBJ_SIGN_MASK, 1) |
4299             FIELD_PREP(DCORE0_SYNC_MNGR_OBJS_SOB_OBJ_INC_MASK, 1);
4300     WREG32(mmDCORE0_SYNC_MNGR_OBJS_MON_PAY_DATA_0 + mon_offset, payload);
4301 
4302     /* 3rd monitor: Re-arm the master monitor */
4303     mon_offset = first_mon_offset + 2 * sizeof(u32);
4304 
4305     addr = CFG_BASE + mmDCORE0_SYNC_MNGR_OBJS_MON_ARM_0 + first_mon_offset;
4306     WREG32(mmDCORE0_SYNC_MNGR_OBJS_MON_PAY_ADDRL_0 + mon_offset, lower_32_bits(addr));
4307     WREG32(mmDCORE0_SYNC_MNGR_OBJS_MON_PAY_ADDRH_0 + mon_offset, upper_32_bits(addr));
4308 
4309     sob_group = sob_id / 8;
4310     mask = ~BIT(sob_id & 0x7);
4311     mode = 0; /* comparison mode is "greater than or equal to" */
4312     arm = FIELD_PREP(DCORE0_SYNC_MNGR_OBJS_MON_ARM_SID_MASK, sob_group) |
4313             FIELD_PREP(DCORE0_SYNC_MNGR_OBJS_MON_ARM_MASK_MASK, mask) |
4314             FIELD_PREP(DCORE0_SYNC_MNGR_OBJS_MON_ARM_SOP_MASK, mode) |
4315             FIELD_PREP(DCORE0_SYNC_MNGR_OBJS_MON_ARM_SOD_MASK, 1);
4316 
4317     payload = arm;
4318     WREG32(mmDCORE0_SYNC_MNGR_OBJS_MON_PAY_DATA_0 + mon_offset, payload);
4319 
4320     /* 1st monitor (master): Write interrupt ID to the virtual MSI-X doorbell */
4321     mon_offset = first_mon_offset;
4322 
4323     config = FIELD_PREP(DCORE0_SYNC_MNGR_OBJS_MON_CONFIG_WR_NUM_MASK, 2); /* "2": 3 writes */
4324     WREG32(mmDCORE0_SYNC_MNGR_OBJS_MON_CONFIG_0 + mon_offset, config);
4325 
4326     addr = gaudi2->virt_msix_db_dma_addr;
4327     WREG32(mmDCORE0_SYNC_MNGR_OBJS_MON_PAY_ADDRL_0 + mon_offset, lower_32_bits(addr));
4328     WREG32(mmDCORE0_SYNC_MNGR_OBJS_MON_PAY_ADDRH_0 + mon_offset, upper_32_bits(addr));
4329 
4330     payload = interrupt_id;
4331     WREG32(mmDCORE0_SYNC_MNGR_OBJS_MON_PAY_DATA_0 + mon_offset, payload);
4332 
4333     WREG32(mmDCORE0_SYNC_MNGR_OBJS_MON_ARM_0 + mon_offset, arm);
4334 }
4335 
4336 static void gaudi2_prepare_sm_for_virt_msix_db(struct hl_device *hdev)
4337 {
4338     u32 decoder_id, sob_id, first_mon_id, interrupt_id;
4339     struct asic_fixed_properties *prop = &hdev->asic_prop;
4340 
4341     /* Decoder normal/abnormal interrupts */
4342     for (decoder_id = 0 ; decoder_id < NUMBER_OF_DEC ; ++decoder_id) {
4343         if (!(prop->decoder_enabled_mask & BIT(decoder_id)))
4344             continue;
4345 
4346         sob_id = GAUDI2_RESERVED_SOB_DEC_NRM_FIRST + decoder_id;
4347         first_mon_id = GAUDI2_RESERVED_MON_DEC_NRM_FIRST + 3 * decoder_id;
4348         interrupt_id = GAUDI2_IRQ_NUM_DCORE0_DEC0_NRM + 2 * decoder_id;
4349         gaudi2_arm_monitors_for_virt_msix_db(hdev, sob_id, first_mon_id, interrupt_id);
4350 
4351         sob_id = GAUDI2_RESERVED_SOB_DEC_ABNRM_FIRST + decoder_id;
4352         first_mon_id = GAUDI2_RESERVED_MON_DEC_ABNRM_FIRST + 3 * decoder_id;
4353         interrupt_id += 1;
4354         gaudi2_arm_monitors_for_virt_msix_db(hdev, sob_id, first_mon_id, interrupt_id);
4355     }
4356 }
4357 
4358 static void gaudi2_init_sm(struct hl_device *hdev)
4359 {
4360     struct gaudi2_device *gaudi2 = hdev->asic_specific;
4361     u64 cq_address;
4362     u32 reg_val;
4363     int i;
4364 
4365     /* Enable HBW/LBW CQ for completion monitors */
4366     reg_val = FIELD_PREP(DCORE0_SYNC_MNGR_OBJS_MON_CONFIG_CQ_EN_MASK, 1);
4367     reg_val |= FIELD_PREP(DCORE0_SYNC_MNGR_OBJS_MON_CONFIG_LBW_EN_MASK, 1);
4368 
4369     for (i = 0 ; i < GAUDI2_MAX_PENDING_CS ; i++)
4370         WREG32(mmDCORE0_SYNC_MNGR_OBJS_MON_CONFIG_0 + (4 * i), reg_val);
4371 
4372     /* Enable only HBW CQ for KDMA completion monitor */
4373     reg_val = FIELD_PREP(DCORE0_SYNC_MNGR_OBJS_MON_CONFIG_CQ_EN_MASK, 1);
4374     WREG32(mmDCORE0_SYNC_MNGR_OBJS_MON_CONFIG_0 + (4 * i), reg_val);
4375 
4376     /* Init CQ0 DB */
4377     /* Configure the monitor to trigger MSI-X interrupt */
4378     /* TODO:
4379      * Remove the if statement when virtual MSI-X doorbell is supported in simulator (SW-93022)
4380      * and in F/W (SW-93024).
4381      */
4382     if (!hdev->pdev || hdev->asic_prop.fw_security_enabled) {
4383         u64 msix_db_reg = CFG_BASE + mmPCIE_DBI_MSIX_DOORBELL_OFF;
4384 
4385         WREG32(mmDCORE0_SYNC_MNGR_GLBL_LBW_ADDR_L_0, lower_32_bits(msix_db_reg));
4386         WREG32(mmDCORE0_SYNC_MNGR_GLBL_LBW_ADDR_H_0, upper_32_bits(msix_db_reg));
4387     } else {
4388         WREG32(mmDCORE0_SYNC_MNGR_GLBL_LBW_ADDR_L_0,
4389                 lower_32_bits(gaudi2->virt_msix_db_dma_addr));
4390         WREG32(mmDCORE0_SYNC_MNGR_GLBL_LBW_ADDR_H_0,
4391                 upper_32_bits(gaudi2->virt_msix_db_dma_addr));
4392     }
4393     WREG32(mmDCORE0_SYNC_MNGR_GLBL_LBW_DATA_0, GAUDI2_IRQ_NUM_COMPLETION);
4394 
4395     for (i = 0 ; i < GAUDI2_RESERVED_CQ_NUMBER ; i++) {
4396         cq_address =
4397             hdev->completion_queue[i].bus_address;
4398 
4399         WREG32(mmDCORE0_SYNC_MNGR_GLBL_CQ_BASE_ADDR_L_0 + (4 * i),
4400                             lower_32_bits(cq_address));
4401         WREG32(mmDCORE0_SYNC_MNGR_GLBL_CQ_BASE_ADDR_H_0 + (4 * i),
4402                             upper_32_bits(cq_address));
4403         WREG32(mmDCORE0_SYNC_MNGR_GLBL_CQ_SIZE_LOG2_0 + (4 * i),
4404                             ilog2(HL_CQ_SIZE_IN_BYTES));
4405     }
4406 
4407     /* Configure kernel ASID and MMU BP*/
4408     WREG32(mmDCORE0_SYNC_MNGR_GLBL_ASID_SEC, 0x10000);
4409     WREG32(mmDCORE0_SYNC_MNGR_GLBL_ASID_NONE_SEC_PRIV, 0);
4410 
4411     /* Initialize sync objects and monitors which are used for the virtual MSI-X doorbell */
4412     gaudi2_prepare_sm_for_virt_msix_db(hdev);
4413 }
4414 
4415 static void gaudi2_init_mme_acc(struct hl_device *hdev, u32 reg_base)
4416 {
4417     struct gaudi2_device *gaudi2 = hdev->asic_specific;
4418     u32 reg_val;
4419     int i;
4420 
4421     reg_val = FIELD_PREP(MME_ACC_INTR_MASK_WBC_ERR_RESP_MASK, 0);
4422     reg_val |= FIELD_PREP(MME_ACC_INTR_MASK_AP_SRC_POS_INF_MASK, 1);
4423     reg_val |= FIELD_PREP(MME_ACC_INTR_MASK_AP_SRC_NEG_INF_MASK, 1);
4424     reg_val |= FIELD_PREP(MME_ACC_INTR_MASK_AP_SRC_NAN_MASK, 1);
4425     reg_val |= FIELD_PREP(MME_ACC_INTR_MASK_AP_RESULT_POS_INF_MASK, 1);
4426     reg_val |= FIELD_PREP(MME_ACC_INTR_MASK_AP_RESULT_NEG_INF_MASK, 1);
4427 
4428     WREG32(reg_base + MME_ACC_INTR_MASK_OFFSET, reg_val);
4429     WREG32(reg_base + MME_ACC_AP_LFSR_POLY_OFFSET, 0x80DEADAF);
4430 
4431     for (i = 0 ; i < MME_NUM_OF_LFSR_SEEDS ; i++) {
4432         WREG32(reg_base + MME_ACC_AP_LFSR_SEED_SEL_OFFSET, i);
4433         WREG32(reg_base + MME_ACC_AP_LFSR_SEED_WDATA_OFFSET, gaudi2->lfsr_rand_seeds[i]);
4434     }
4435 }
4436 
4437 static void gaudi2_init_dcore_mme(struct hl_device *hdev, int dcore_id,
4438                             bool config_qman_only)
4439 {
4440     u32 queue_id_base, reg_base, clk_en_addr = 0;
4441 
4442     switch (dcore_id) {
4443     case 0:
4444         queue_id_base = GAUDI2_QUEUE_ID_DCORE0_MME_0_0;
4445         break;
4446     case 1:
4447         queue_id_base = GAUDI2_QUEUE_ID_DCORE1_MME_0_0;
4448         clk_en_addr = mmDCORE1_MME_CTRL_LO_QM_SLV_CLK_EN;
4449         break;
4450     case 2:
4451         queue_id_base = GAUDI2_QUEUE_ID_DCORE2_MME_0_0;
4452         break;
4453     case 3:
4454         queue_id_base = GAUDI2_QUEUE_ID_DCORE3_MME_0_0;
4455         clk_en_addr = mmDCORE3_MME_CTRL_LO_QM_SLV_CLK_EN;
4456         break;
4457     default:
4458         dev_err(hdev->dev, "Invalid dcore id %u\n", dcore_id);
4459         return;
4460     }
4461 
4462     if (clk_en_addr && !(hdev->fw_components & FW_TYPE_BOOT_CPU))
4463         WREG32(clk_en_addr, 0x1);
4464 
4465     if (!config_qman_only) {
4466         reg_base = gaudi2_mme_acc_blocks_bases[dcore_id];
4467         gaudi2_init_mme_acc(hdev, reg_base);
4468     }
4469 
4470     reg_base = gaudi2_qm_blocks_bases[queue_id_base];
4471     gaudi2_init_qman(hdev, reg_base, queue_id_base);
4472 }
4473 
4474 static void gaudi2_init_mme(struct hl_device *hdev)
4475 {
4476     struct gaudi2_device *gaudi2 = hdev->asic_specific;
4477     int i;
4478 
4479     if ((gaudi2->hw_cap_initialized & HW_CAP_MME_MASK) == HW_CAP_MME_MASK)
4480         return;
4481 
4482     for (i = 0 ; i < NUM_OF_DCORES ; i++) {
4483         gaudi2_init_dcore_mme(hdev, i, false);
4484 
4485         gaudi2->hw_cap_initialized |= BIT_ULL(HW_CAP_MME_SHIFT + i);
4486     }
4487 }
4488 
4489 static void gaudi2_init_tpc_cfg(struct hl_device *hdev, u32 reg_base)
4490 {
4491     /* Mask arithmetic and QM interrupts in TPC */
4492     WREG32(reg_base + TPC_CFG_TPC_INTR_MASK_OFFSET, 0x23FFFE);
4493 
4494     /* Set 16 cache lines */
4495     WREG32(reg_base + TPC_CFG_MSS_CONFIG_OFFSET,
4496             2 << DCORE0_TPC0_CFG_MSS_CONFIG_ICACHE_FETCH_LINE_NUM_SHIFT);
4497 }
4498 
4499 struct gaudi2_tpc_init_cfg_data {
4500     enum gaudi2_queue_id dcore_tpc_qid_base[NUM_OF_DCORES];
4501 };
4502 
4503 static void gaudi2_init_tpc_config(struct hl_device *hdev, int dcore, int inst,
4504                             u32 offset, void *data)
4505 {
4506     struct gaudi2_device *gaudi2 = hdev->asic_specific;
4507     struct gaudi2_tpc_init_cfg_data *cfg_data = data;
4508     u32 queue_id_base;
4509     u8 seq;
4510 
4511     queue_id_base = cfg_data->dcore_tpc_qid_base[dcore] + (inst * NUM_OF_PQ_PER_QMAN);
4512 
4513     if (dcore == 0 && inst == (NUM_DCORE0_TPC - 1))
4514         /* gets last sequence number */
4515         seq = NUM_OF_DCORES * NUM_OF_TPC_PER_DCORE;
4516     else
4517         seq = dcore * NUM_OF_TPC_PER_DCORE + inst;
4518 
4519     gaudi2_init_tpc_cfg(hdev, mmDCORE0_TPC0_CFG_BASE + offset);
4520     gaudi2_init_qman(hdev, mmDCORE0_TPC0_QM_BASE + offset, queue_id_base);
4521 
4522     gaudi2->tpc_hw_cap_initialized |= BIT_ULL(HW_CAP_TPC_SHIFT + seq);
4523 }
4524 
4525 static void gaudi2_init_tpc(struct hl_device *hdev)
4526 {
4527     struct gaudi2_device *gaudi2 = hdev->asic_specific;
4528     struct gaudi2_tpc_init_cfg_data init_cfg_data;
4529     struct iterate_module_ctx tpc_iter;
4530 
4531     if (!hdev->asic_prop.tpc_enabled_mask)
4532         return;
4533 
4534     if ((gaudi2->tpc_hw_cap_initialized & HW_CAP_TPC_MASK) == HW_CAP_TPC_MASK)
4535         return;
4536 
4537     init_cfg_data.dcore_tpc_qid_base[0] = GAUDI2_QUEUE_ID_DCORE0_TPC_0_0;
4538     init_cfg_data.dcore_tpc_qid_base[1] = GAUDI2_QUEUE_ID_DCORE1_TPC_0_0;
4539     init_cfg_data.dcore_tpc_qid_base[2] = GAUDI2_QUEUE_ID_DCORE2_TPC_0_0;
4540     init_cfg_data.dcore_tpc_qid_base[3] = GAUDI2_QUEUE_ID_DCORE3_TPC_0_0;
4541     tpc_iter.fn = &gaudi2_init_tpc_config;
4542     tpc_iter.data = &init_cfg_data;
4543     gaudi2_iterate_tpcs(hdev, &tpc_iter);
4544 }
4545 
4546 static void gaudi2_init_rotator(struct hl_device *hdev)
4547 {
4548     struct gaudi2_device *gaudi2 = hdev->asic_specific;
4549     u32 i, reg_base, queue_id;
4550 
4551     queue_id = GAUDI2_QUEUE_ID_ROT_0_0;
4552 
4553     for (i = 0 ; i < NUM_OF_ROT ; i++, queue_id += NUM_OF_PQ_PER_QMAN) {
4554         reg_base = gaudi2_qm_blocks_bases[queue_id];
4555         gaudi2_init_qman(hdev, reg_base, queue_id);
4556 
4557         gaudi2->hw_cap_initialized |= BIT_ULL(HW_CAP_ROT_SHIFT + i);
4558     }
4559 }
4560 
4561 static void gaudi2_init_vdec_brdg_ctrl(struct hl_device *hdev, u64 base_addr, u32 decoder_id)
4562 {
4563     u32 sob_id;
4564 
4565     /* TODO:
4566      * Remove when virtual MSI-X doorbell is supported in simulator (SW-93022) and in F/W
4567      * (SW-93024).
4568      */
4569     if (!hdev->pdev || hdev->asic_prop.fw_security_enabled) {
4570         u32 interrupt_id = GAUDI2_IRQ_NUM_DCORE0_DEC0_NRM + 2 * decoder_id;
4571 
4572         WREG32(base_addr + BRDG_CTRL_NRM_MSIX_LBW_AWADDR, mmPCIE_DBI_MSIX_DOORBELL_OFF);
4573         WREG32(base_addr + BRDG_CTRL_NRM_MSIX_LBW_WDATA, interrupt_id);
4574         WREG32(base_addr + BRDG_CTRL_ABNRM_MSIX_LBW_AWADDR, mmPCIE_DBI_MSIX_DOORBELL_OFF);
4575         WREG32(base_addr + BRDG_CTRL_ABNRM_MSIX_LBW_WDATA, interrupt_id + 1);
4576         return;
4577     }
4578 
4579     /* VCMD normal interrupt */
4580     sob_id = GAUDI2_RESERVED_SOB_DEC_NRM_FIRST + decoder_id;
4581     WREG32(base_addr + BRDG_CTRL_NRM_MSIX_LBW_AWADDR,
4582             mmDCORE0_SYNC_MNGR_OBJS_SOB_OBJ_0 + sob_id * sizeof(u32));
4583     WREG32(base_addr + BRDG_CTRL_NRM_MSIX_LBW_WDATA, GAUDI2_SOB_INCREMENT_BY_ONE);
4584 
4585     /* VCMD abnormal interrupt */
4586     sob_id = GAUDI2_RESERVED_SOB_DEC_ABNRM_FIRST + decoder_id;
4587     WREG32(base_addr + BRDG_CTRL_ABNRM_MSIX_LBW_AWADDR,
4588             mmDCORE0_SYNC_MNGR_OBJS_SOB_OBJ_0 + sob_id * sizeof(u32));
4589     WREG32(base_addr + BRDG_CTRL_ABNRM_MSIX_LBW_WDATA, GAUDI2_SOB_INCREMENT_BY_ONE);
4590 }
4591 
4592 static void gaudi2_init_dec(struct hl_device *hdev)
4593 {
4594     struct gaudi2_device *gaudi2 = hdev->asic_specific;
4595     u32 dcore_id, dec_id, dec_bit;
4596     u64 base_addr;
4597 
4598     if (!hdev->asic_prop.decoder_enabled_mask)
4599         return;
4600 
4601     if ((gaudi2->dec_hw_cap_initialized & HW_CAP_DEC_MASK) == HW_CAP_DEC_MASK)
4602         return;
4603 
4604     for (dcore_id = 0 ; dcore_id < NUM_OF_DCORES ; dcore_id++)
4605         for (dec_id = 0 ; dec_id < NUM_OF_DEC_PER_DCORE ; dec_id++) {
4606             dec_bit = dcore_id * NUM_OF_DEC_PER_DCORE + dec_id;
4607 
4608             if (!(hdev->asic_prop.decoder_enabled_mask & BIT(dec_bit)))
4609                 continue;
4610 
4611             base_addr =  mmDCORE0_DEC0_CMD_BASE +
4612                     BRDG_CTRL_BLOCK_OFFSET +
4613                     dcore_id * DCORE_OFFSET +
4614                     dec_id * DCORE_VDEC_OFFSET;
4615 
4616             gaudi2_init_vdec_brdg_ctrl(hdev, base_addr, dec_bit);
4617 
4618             gaudi2->dec_hw_cap_initialized |= BIT_ULL(HW_CAP_DEC_SHIFT + dec_bit);
4619         }
4620 
4621     for (dec_id = 0 ; dec_id < NUM_OF_PCIE_VDEC ; dec_id++) {
4622         dec_bit = PCIE_DEC_SHIFT + dec_id;
4623         if (!(hdev->asic_prop.decoder_enabled_mask & BIT(dec_bit)))
4624             continue;
4625 
4626         base_addr = mmPCIE_DEC0_CMD_BASE + BRDG_CTRL_BLOCK_OFFSET +
4627                 dec_id * DCORE_VDEC_OFFSET;
4628 
4629         gaudi2_init_vdec_brdg_ctrl(hdev, base_addr, dec_bit);
4630 
4631         gaudi2->dec_hw_cap_initialized |= BIT_ULL(HW_CAP_DEC_SHIFT + dec_bit);
4632     }
4633 }
4634 
4635 static void gaudi2_init_msix_gw_table(struct hl_device *hdev)
4636 {
4637     u32 first_reg_offset, last_reg_offset, msix_gw_table_base;
4638     u8 first_bit, last_bit;
4639     int i;
4640 
4641     msix_gw_table_base = mmPCIE_WRAP_MSIX_GW_TABLE_0;
4642     first_reg_offset = (GAUDI2_IRQ_NUM_USER_FIRST >> 5) << 2;
4643     first_bit = GAUDI2_IRQ_NUM_USER_FIRST % 32;
4644     last_reg_offset = (GAUDI2_IRQ_NUM_USER_LAST >> 5) << 2;
4645     last_bit = GAUDI2_IRQ_NUM_USER_LAST % 32;
4646 
4647     if (first_reg_offset == last_reg_offset) {
4648         WREG32(msix_gw_table_base + first_reg_offset, GENMASK(last_bit, first_bit));
4649         return;
4650     }
4651 
4652     WREG32(msix_gw_table_base + first_reg_offset, GENMASK(31, first_bit));
4653     WREG32(msix_gw_table_base + last_reg_offset, GENMASK(last_bit, 0));
4654 
4655     for (i = first_reg_offset + 4; i < last_reg_offset ; i += 4)
4656         WREG32(msix_gw_table_base + i, 0xFFFFFFFF);
4657 }
4658 
4659 static int gaudi2_mmu_update_asid_hop0_addr(struct hl_device *hdev,
4660                     u32 stlb_base, u32 asid, u64 phys_addr)
4661 {
4662     u32 status, timeout_usec;
4663     int rc;
4664 
4665     if (hdev->pldm || !hdev->pdev)
4666         timeout_usec = GAUDI2_PLDM_MMU_TIMEOUT_USEC;
4667     else
4668         timeout_usec = MMU_CONFIG_TIMEOUT_USEC;
4669 
4670     WREG32(stlb_base + STLB_ASID_OFFSET, asid);
4671     WREG32(stlb_base + STLB_HOP0_PA43_12_OFFSET, phys_addr >> MMU_HOP0_PA43_12_SHIFT);
4672     WREG32(stlb_base + STLB_HOP0_PA63_44_OFFSET, phys_addr >> MMU_HOP0_PA63_44_SHIFT);
4673     WREG32(stlb_base + STLB_BUSY_OFFSET, 0x80000000);
4674 
4675     rc = hl_poll_timeout(
4676         hdev,
4677         stlb_base + STLB_BUSY_OFFSET,
4678         status,
4679         !(status & 0x80000000),
4680         1000,
4681         timeout_usec);
4682 
4683     if (rc) {
4684         dev_err(hdev->dev, "Timeout during MMU hop0 config of asid %d\n", asid);
4685         return rc;
4686     }
4687 
4688     return 0;
4689 }
4690 
4691 static void gaudi2_mmu_send_invalidate_cache_cmd(struct hl_device *hdev, u32 stlb_base,
4692                     u32 start_offset, u32 inv_start_val,
4693                     u32 flags)
4694 {
4695     /* clear PMMU mem line cache (only needed in mmu range invalidation) */
4696     if (flags & MMU_OP_CLEAR_MEMCACHE)
4697         WREG32(mmPMMU_HBW_STLB_MEM_CACHE_INVALIDATION, 0x1);
4698 
4699     if (flags & MMU_OP_SKIP_LOW_CACHE_INV)
4700         return;
4701 
4702     WREG32(stlb_base + start_offset, inv_start_val);
4703 }
4704 
4705 static int gaudi2_mmu_invalidate_cache_status_poll(struct hl_device *hdev, u32 stlb_base,
4706                         struct gaudi2_cache_invld_params *inv_params)
4707 {
4708     u32 status, timeout_usec, start_offset;
4709     int rc;
4710 
4711     timeout_usec = (hdev->pldm) ? GAUDI2_PLDM_MMU_TIMEOUT_USEC :
4712                     GAUDI2_MMU_CACHE_INV_TIMEOUT_USEC;
4713 
4714     /* poll PMMU mem line cache (only needed in mmu range invalidation) */
4715     if (inv_params->flags & MMU_OP_CLEAR_MEMCACHE) {
4716         rc = hl_poll_timeout(
4717             hdev,
4718             mmPMMU_HBW_STLB_MEM_CACHE_INV_STATUS,
4719             status,
4720             status & 0x1,
4721             1000,
4722             timeout_usec);
4723 
4724         if (rc)
4725             return rc;
4726 
4727         /* Need to manually reset the status to 0 */
4728         WREG32(mmPMMU_HBW_STLB_MEM_CACHE_INV_STATUS, 0x0);
4729     }
4730 
4731     /* Lower cache does not work with cache lines, hence we can skip its
4732      * invalidation upon map and invalidate only upon unmap
4733      */
4734     if (inv_params->flags & MMU_OP_SKIP_LOW_CACHE_INV)
4735         return 0;
4736 
4737     start_offset = inv_params->range_invalidation ?
4738             STLB_RANGE_CACHE_INVALIDATION_OFFSET : STLB_INV_ALL_START_OFFSET;
4739 
4740     rc = hl_poll_timeout(
4741         hdev,
4742         stlb_base + start_offset,
4743         status,
4744         !(status & 0x1),
4745         1000,
4746         timeout_usec);
4747 
4748     return rc;
4749 }
4750 
4751 bool gaudi2_is_hmmu_enabled(struct hl_device *hdev, int dcore_id, int hmmu_id)
4752 {
4753     struct gaudi2_device *gaudi2 = hdev->asic_specific;
4754     u32 hw_cap;
4755 
4756     hw_cap = HW_CAP_DCORE0_DMMU0 << (NUM_OF_HMMU_PER_DCORE * dcore_id + hmmu_id);
4757 
4758     if (gaudi2->hw_cap_initialized & hw_cap)
4759         return true;
4760 
4761     return false;
4762 }
4763 
4764 /* this function shall be called only for HMMUs for which capability bit is set */
4765 static inline u32 get_hmmu_stlb_base(int dcore_id, int hmmu_id)
4766 {
4767     u32 offset;
4768 
4769     offset =  (u32) (dcore_id * DCORE_OFFSET + hmmu_id * DCORE_HMMU_OFFSET);
4770     return (u32)(mmDCORE0_HMMU0_STLB_BASE + offset);
4771 }
4772 
4773 static void gaudi2_mmu_invalidate_cache_trigger(struct hl_device *hdev, u32 stlb_base,
4774                         struct gaudi2_cache_invld_params *inv_params)
4775 {
4776     u32 start_offset;
4777 
4778     if (inv_params->range_invalidation) {
4779         /* Set the addresses range
4780          * Note: that the start address we set in register, is not included in
4781          * the range of the invalidation, by design.
4782          * that's why we need to set lower address than the one we actually
4783          * want to be included in the range invalidation.
4784          */
4785         u64 start = inv_params->start_va - 1;
4786 
4787         start_offset = STLB_RANGE_CACHE_INVALIDATION_OFFSET;
4788 
4789         WREG32(stlb_base + STLB_RANGE_INV_START_LSB_OFFSET,
4790                 start >> MMU_RANGE_INV_VA_LSB_SHIFT);
4791 
4792         WREG32(stlb_base + STLB_RANGE_INV_START_MSB_OFFSET,
4793                 start >> MMU_RANGE_INV_VA_MSB_SHIFT);
4794 
4795         WREG32(stlb_base + STLB_RANGE_INV_END_LSB_OFFSET,
4796                 inv_params->end_va >> MMU_RANGE_INV_VA_LSB_SHIFT);
4797 
4798         WREG32(stlb_base + STLB_RANGE_INV_END_MSB_OFFSET,
4799                 inv_params->end_va >> MMU_RANGE_INV_VA_MSB_SHIFT);
4800     } else {
4801         start_offset = STLB_INV_ALL_START_OFFSET;
4802     }
4803 
4804     gaudi2_mmu_send_invalidate_cache_cmd(hdev, stlb_base, start_offset,
4805                         inv_params->inv_start_val, inv_params->flags);
4806 }
4807 
4808 static inline void gaudi2_hmmu_invalidate_cache_trigger(struct hl_device *hdev,
4809                         int dcore_id, int hmmu_id,
4810                         struct gaudi2_cache_invld_params *inv_params)
4811 {
4812     u32 stlb_base = get_hmmu_stlb_base(dcore_id, hmmu_id);
4813 
4814     gaudi2_mmu_invalidate_cache_trigger(hdev, stlb_base, inv_params);
4815 }
4816 
4817 static inline int gaudi2_hmmu_invalidate_cache_status_poll(struct hl_device *hdev,
4818                         int dcore_id, int hmmu_id,
4819                         struct gaudi2_cache_invld_params *inv_params)
4820 {
4821     u32 stlb_base = get_hmmu_stlb_base(dcore_id, hmmu_id);
4822 
4823     return gaudi2_mmu_invalidate_cache_status_poll(hdev, stlb_base, inv_params);
4824 }
4825 
4826 static int gaudi2_hmmus_invalidate_cache(struct hl_device *hdev,
4827                         struct gaudi2_cache_invld_params *inv_params)
4828 {
4829     int dcore_id, hmmu_id;
4830 
4831     /* first send all invalidation commands */
4832     for (dcore_id = 0 ; dcore_id < NUM_OF_DCORES ; dcore_id++) {
4833         for (hmmu_id = 0 ; hmmu_id < NUM_OF_HMMU_PER_DCORE ; hmmu_id++) {
4834             if (!gaudi2_is_hmmu_enabled(hdev, dcore_id, hmmu_id))
4835                 continue;
4836 
4837             gaudi2_hmmu_invalidate_cache_trigger(hdev, dcore_id, hmmu_id, inv_params);
4838         }
4839     }
4840 
4841     /* next, poll all invalidations status */
4842     for (dcore_id = 0 ; dcore_id < NUM_OF_DCORES ; dcore_id++) {
4843         for (hmmu_id = 0 ; hmmu_id < NUM_OF_HMMU_PER_DCORE ; hmmu_id++) {
4844             int rc;
4845 
4846             if (!gaudi2_is_hmmu_enabled(hdev, dcore_id, hmmu_id))
4847                 continue;
4848 
4849             rc = gaudi2_hmmu_invalidate_cache_status_poll(hdev, dcore_id, hmmu_id,
4850                                         inv_params);
4851             if (rc)
4852                 return rc;
4853         }
4854     }
4855 
4856     return 0;
4857 }
4858 
4859 static int gaudi2_mmu_invalidate_cache(struct hl_device *hdev, bool is_hard, u32 flags)
4860 {
4861     struct gaudi2_device *gaudi2 = hdev->asic_specific;
4862     struct gaudi2_cache_invld_params invld_params;
4863     int rc = 0;
4864 
4865     if (hdev->reset_info.hard_reset_pending)
4866         return rc;
4867 
4868     invld_params.range_invalidation = false;
4869     invld_params.inv_start_val = 1;
4870 
4871     if ((flags & MMU_OP_USERPTR) && (gaudi2->hw_cap_initialized & HW_CAP_PMMU)) {
4872         invld_params.flags = flags;
4873         gaudi2_mmu_invalidate_cache_trigger(hdev, mmPMMU_HBW_STLB_BASE, &invld_params);
4874         rc = gaudi2_mmu_invalidate_cache_status_poll(hdev, mmPMMU_HBW_STLB_BASE,
4875                                         &invld_params);
4876     } else if (flags & MMU_OP_PHYS_PACK) {
4877         invld_params.flags = 0;
4878         rc = gaudi2_hmmus_invalidate_cache(hdev, &invld_params);
4879     }
4880 
4881     return rc;
4882 }
4883 
4884 static int gaudi2_mmu_invalidate_cache_range(struct hl_device *hdev, bool is_hard,
4885                 u32 flags, u32 asid, u64 va, u64 size)
4886 {
4887     struct gaudi2_cache_invld_params invld_params = {0};
4888     struct gaudi2_device *gaudi2 = hdev->asic_specific;
4889     u64 start_va, end_va;
4890     u32 inv_start_val;
4891     int rc = 0;
4892 
4893     if (hdev->reset_info.hard_reset_pending)
4894         return 0;
4895 
4896     inv_start_val = (1 << MMU_RANGE_INV_EN_SHIFT |
4897             1 << MMU_RANGE_INV_ASID_EN_SHIFT |
4898             asid << MMU_RANGE_INV_ASID_SHIFT);
4899     start_va = va;
4900     end_va = start_va + size;
4901 
4902     if ((flags & MMU_OP_USERPTR) && (gaudi2->hw_cap_initialized & HW_CAP_PMMU)) {
4903         /* As range invalidation does not support zero address we will
4904          * do full invalidation in this case
4905          */
4906         if (start_va) {
4907             invld_params.range_invalidation = true;
4908             invld_params.start_va = start_va;
4909             invld_params.end_va = end_va;
4910             invld_params.inv_start_val = inv_start_val;
4911             invld_params.flags = flags | MMU_OP_CLEAR_MEMCACHE;
4912         } else {
4913             invld_params.range_invalidation = false;
4914             invld_params.inv_start_val = 1;
4915             invld_params.flags = flags;
4916         }
4917 
4918 
4919         gaudi2_mmu_invalidate_cache_trigger(hdev, mmPMMU_HBW_STLB_BASE, &invld_params);
4920         rc = gaudi2_mmu_invalidate_cache_status_poll(hdev, mmPMMU_HBW_STLB_BASE,
4921                                         &invld_params);
4922         if (rc)
4923             return rc;
4924 
4925     } else if (flags & MMU_OP_PHYS_PACK) {
4926         invld_params.start_va = gaudi2_mmu_scramble_addr(hdev, start_va);
4927         invld_params.end_va = gaudi2_mmu_scramble_addr(hdev, end_va);
4928         invld_params.inv_start_val = inv_start_val;
4929         invld_params.flags = flags;
4930         rc = gaudi2_hmmus_invalidate_cache(hdev, &invld_params);
4931     }
4932 
4933     return rc;
4934 }
4935 
4936 static int gaudi2_mmu_update_hop0_addr(struct hl_device *hdev, u32 stlb_base)
4937 {
4938     struct asic_fixed_properties *prop = &hdev->asic_prop;
4939     u64 hop0_addr;
4940     u32 asid, max_asid = prop->max_asid;
4941     int rc;
4942 
4943     /* it takes too much time to init all of the ASIDs on palladium */
4944     if (hdev->pldm)
4945         max_asid = min((u32) 8, max_asid);
4946 
4947     for (asid = 0 ; asid < max_asid ; asid++) {
4948         hop0_addr = hdev->mmu_priv.hr.mmu_asid_hop0[asid].phys_addr;
4949         rc = gaudi2_mmu_update_asid_hop0_addr(hdev, stlb_base, asid, hop0_addr);
4950         if (rc) {
4951             dev_err(hdev->dev, "failed to set hop0 addr for asid %d\n", asid);
4952             return rc;
4953         }
4954     }
4955 
4956     return 0;
4957 }
4958 
4959 static int gaudi2_mmu_init_common(struct hl_device *hdev, u32 mmu_base,
4960                     u32 stlb_base)
4961 {
4962     u32 status, timeout_usec;
4963     int rc;
4964 
4965     if (hdev->pldm || !hdev->pdev)
4966         timeout_usec = GAUDI2_PLDM_MMU_TIMEOUT_USEC;
4967     else
4968         timeout_usec = GAUDI2_MMU_CACHE_INV_TIMEOUT_USEC;
4969 
4970     WREG32(stlb_base + STLB_INV_ALL_START_OFFSET, 1);
4971 
4972     rc = hl_poll_timeout(
4973         hdev,
4974         stlb_base + STLB_SRAM_INIT_OFFSET,
4975         status,
4976         !status,
4977         1000,
4978         timeout_usec);
4979 
4980     if (rc)
4981         dev_notice_ratelimited(hdev->dev, "Timeout when waiting for MMU SRAM init\n");
4982 
4983     rc = gaudi2_mmu_update_hop0_addr(hdev, stlb_base);
4984     if (rc)
4985         return rc;
4986 
4987     WREG32(mmu_base + MMU_BYPASS_OFFSET, 0);
4988     WREG32(mmu_base + MMU_SPI_SEI_MASK_OFFSET, 0xF);
4989 
4990     rc = hl_poll_timeout(
4991         hdev,
4992         stlb_base + STLB_INV_ALL_START_OFFSET,
4993         status,
4994         !status,
4995         1000,
4996         timeout_usec);
4997 
4998     if (rc)
4999         dev_notice_ratelimited(hdev->dev, "Timeout when waiting for MMU invalidate all\n");
5000 
5001     WREG32(mmu_base + MMU_ENABLE_OFFSET, 1);
5002 
5003     return rc;
5004 }
5005 
5006 static int gaudi2_pci_mmu_init(struct hl_device *hdev)
5007 {
5008     struct gaudi2_device *gaudi2 = hdev->asic_specific;
5009     u32 mmu_base, stlb_base;
5010     int rc;
5011 
5012     if (gaudi2->hw_cap_initialized & HW_CAP_PMMU)
5013         return 0;
5014 
5015     mmu_base = mmPMMU_HBW_MMU_BASE;
5016     stlb_base = mmPMMU_HBW_STLB_BASE;
5017 
5018     RMWREG32(stlb_base + STLB_HOP_CONFIGURATION_OFFSET,
5019         (0 << PMMU_HBW_STLB_HOP_CONFIGURATION_FIRST_HOP_SHIFT) |
5020         (5 << PMMU_HBW_STLB_HOP_CONFIGURATION_FIRST_LOOKUP_HOP_SMALL_P_SHIFT) |
5021         (4 << PMMU_HBW_STLB_HOP_CONFIGURATION_FIRST_LOOKUP_HOP_LARGE_P_SHIFT) |
5022         (5 << PMMU_HBW_STLB_HOP_CONFIGURATION_LAST_HOP_SHIFT) |
5023         (5 << PMMU_HBW_STLB_HOP_CONFIGURATION_FOLLOWER_HOP_SHIFT),
5024         PMMU_HBW_STLB_HOP_CONFIGURATION_FIRST_HOP_MASK |
5025         PMMU_HBW_STLB_HOP_CONFIGURATION_FIRST_LOOKUP_HOP_SMALL_P_MASK |
5026         PMMU_HBW_STLB_HOP_CONFIGURATION_FIRST_LOOKUP_HOP_LARGE_P_MASK |
5027         PMMU_HBW_STLB_HOP_CONFIGURATION_LAST_HOP_MASK |
5028         PMMU_HBW_STLB_HOP_CONFIGURATION_FOLLOWER_HOP_MASK);
5029 
5030     WREG32(stlb_base + STLB_LL_LOOKUP_MASK_63_32_OFFSET, 0);
5031 
5032     if (PAGE_SIZE == SZ_64K) {
5033         /* Set page sizes to 64K on hop5 and 16M on hop4 + enable 8 bit hops */
5034         RMWREG32(mmu_base + MMU_STATIC_MULTI_PAGE_SIZE_OFFSET,
5035             FIELD_PREP(DCORE0_HMMU0_MMU_STATIC_MULTI_PAGE_SIZE_HOP5_PAGE_SIZE_MASK, 4) |
5036             FIELD_PREP(DCORE0_HMMU0_MMU_STATIC_MULTI_PAGE_SIZE_HOP4_PAGE_SIZE_MASK, 3) |
5037             FIELD_PREP(
5038                 DCORE0_HMMU0_MMU_STATIC_MULTI_PAGE_SIZE_CFG_8_BITS_HOP_MODE_EN_MASK,
5039                 1),
5040             DCORE0_HMMU0_MMU_STATIC_MULTI_PAGE_SIZE_HOP5_PAGE_SIZE_MASK |
5041             DCORE0_HMMU0_MMU_STATIC_MULTI_PAGE_SIZE_HOP4_PAGE_SIZE_MASK |
5042             DCORE0_HMMU0_MMU_STATIC_MULTI_PAGE_SIZE_CFG_8_BITS_HOP_MODE_EN_MASK);
5043     }
5044 
5045     rc = gaudi2_mmu_init_common(hdev, mmu_base, stlb_base);
5046     if (rc)
5047         return rc;
5048 
5049     gaudi2->hw_cap_initialized |= HW_CAP_PMMU;
5050 
5051     return 0;
5052 }
5053 
5054 static int gaudi2_dcore_hmmu_init(struct hl_device *hdev, int dcore_id,
5055                 int hmmu_id)
5056 {
5057     struct asic_fixed_properties *prop = &hdev->asic_prop;
5058     struct gaudi2_device *gaudi2 = hdev->asic_specific;
5059     u32 offset, mmu_base, stlb_base, hw_cap;
5060     u8 dmmu_seq;
5061     int rc;
5062 
5063     dmmu_seq = NUM_OF_HMMU_PER_DCORE * dcore_id + hmmu_id;
5064     hw_cap = HW_CAP_DCORE0_DMMU0 << dmmu_seq;
5065 
5066     /*
5067      * return if DMMU is already initialized or if it's not out of
5068      * isolation (due to cluster binning)
5069      */
5070     if ((gaudi2->hw_cap_initialized & hw_cap) || !(prop->hmmu_hif_enabled_mask & BIT(dmmu_seq)))
5071         return 0;
5072 
5073     offset = (u32) (dcore_id * DCORE_OFFSET + hmmu_id * DCORE_HMMU_OFFSET);
5074     mmu_base = mmDCORE0_HMMU0_MMU_BASE + offset;
5075     stlb_base = mmDCORE0_HMMU0_STLB_BASE + offset;
5076 
5077     RMWREG32(mmu_base + MMU_STATIC_MULTI_PAGE_SIZE_OFFSET, 5 /* 64MB */,
5078             MMU_STATIC_MULTI_PAGE_SIZE_HOP4_PAGE_SIZE_MASK);
5079 
5080     RMWREG32(stlb_base + STLB_HOP_CONFIGURATION_OFFSET,
5081         FIELD_PREP(DCORE0_HMMU0_STLB_HOP_CONFIGURATION_FIRST_HOP_MASK, 0) |
5082         FIELD_PREP(DCORE0_HMMU0_STLB_HOP_CONFIGURATION_FIRST_LOOKUP_HOP_SMALL_P_MASK, 3) |
5083         FIELD_PREP(DCORE0_HMMU0_STLB_HOP_CONFIGURATION_FIRST_LOOKUP_HOP_LARGE_P_MASK, 3) |
5084         FIELD_PREP(DCORE0_HMMU0_STLB_HOP_CONFIGURATION_LAST_HOP_MASK, 3) |
5085         FIELD_PREP(DCORE0_HMMU0_STLB_HOP_CONFIGURATION_FOLLOWER_HOP_MASK, 3),
5086             DCORE0_HMMU0_STLB_HOP_CONFIGURATION_FIRST_HOP_MASK |
5087             DCORE0_HMMU0_STLB_HOP_CONFIGURATION_FIRST_LOOKUP_HOP_SMALL_P_MASK |
5088             DCORE0_HMMU0_STLB_HOP_CONFIGURATION_FIRST_LOOKUP_HOP_LARGE_P_MASK |
5089             DCORE0_HMMU0_STLB_HOP_CONFIGURATION_LAST_HOP_MASK |
5090             DCORE0_HMMU0_STLB_HOP_CONFIGURATION_FOLLOWER_HOP_MASK);
5091 
5092     RMWREG32(stlb_base + STLB_HOP_CONFIGURATION_OFFSET, 1,
5093             STLB_HOP_CONFIGURATION_ONLY_LARGE_PAGE_MASK);
5094 
5095     rc = gaudi2_mmu_init_common(hdev, mmu_base, stlb_base);
5096     if (rc)
5097         return rc;
5098 
5099     gaudi2->hw_cap_initialized |= hw_cap;
5100 
5101     return 0;
5102 }
5103 
5104 static int gaudi2_hbm_mmu_init(struct hl_device *hdev)
5105 {
5106     int rc, dcore_id, hmmu_id;
5107 
5108     for (dcore_id = 0 ; dcore_id < NUM_OF_DCORES ; dcore_id++)
5109         for (hmmu_id = 0 ; hmmu_id < NUM_OF_HMMU_PER_DCORE; hmmu_id++) {
5110             rc = gaudi2_dcore_hmmu_init(hdev, dcore_id, hmmu_id);
5111             if (rc)
5112                 return rc;
5113         }
5114 
5115     return 0;
5116 }
5117 
5118 static int gaudi2_mmu_init(struct hl_device *hdev)
5119 {
5120     int rc;
5121 
5122     rc = gaudi2_pci_mmu_init(hdev);
5123     if (rc)
5124         return rc;
5125 
5126     rc = gaudi2_hbm_mmu_init(hdev);
5127     if (rc)
5128         return rc;
5129 
5130     return 0;
5131 }
5132 
5133 static int gaudi2_hw_init(struct hl_device *hdev)
5134 {
5135     struct gaudi2_device *gaudi2 = hdev->asic_specific;
5136     int rc;
5137 
5138     /* Let's mark in the H/W that we have reached this point. We check
5139      * this value in the reset_before_init function to understand whether
5140      * we need to reset the chip before doing H/W init. This register is
5141      * cleared by the H/W upon H/W reset
5142      */
5143     WREG32(mmHW_STATE, HL_DEVICE_HW_STATE_DIRTY);
5144 
5145     /* Perform read from the device to make sure device is up */
5146     RREG32(mmHW_STATE);
5147 
5148     /* If iATU is done by FW, the HBM bar ALWAYS points to DRAM_PHYS_BASE.
5149      * So we set it here and if anyone tries to move it later to
5150      * a different address, there will be an error
5151      */
5152     if (hdev->asic_prop.iatu_done_by_fw)
5153         gaudi2->dram_bar_cur_addr = DRAM_PHYS_BASE;
5154 
5155     /*
5156      * Before pushing u-boot/linux to device, need to set the hbm bar to
5157      * base address of dram
5158      */
5159     if (gaudi2_set_hbm_bar_base(hdev, DRAM_PHYS_BASE) == U64_MAX) {
5160         dev_err(hdev->dev, "failed to map HBM bar to DRAM base address\n");
5161         return -EIO;
5162     }
5163 
5164     rc = gaudi2_init_cpu(hdev);
5165     if (rc) {
5166         dev_err(hdev->dev, "failed to initialize CPU\n");
5167         return rc;
5168     }
5169 
5170     gaudi2_init_msix_gw_table(hdev);
5171 
5172     gaudi2_init_scrambler_hbm(hdev);
5173     gaudi2_init_kdma(hdev);
5174 
5175     rc = gaudi2_init_cpu_queues(hdev, GAUDI2_CPU_TIMEOUT_USEC);
5176     if (rc) {
5177         dev_err(hdev->dev, "failed to initialize CPU H/W queues %d\n", rc);
5178         return rc;
5179     }
5180 
5181     rc = gaudi2->cpucp_info_get(hdev);
5182     if (rc) {
5183         dev_err(hdev->dev, "Failed to get cpucp info\n");
5184         return rc;
5185     }
5186 
5187     rc = gaudi2_mmu_init(hdev);
5188     if (rc)
5189         return rc;
5190 
5191     gaudi2_init_pdma(hdev);
5192     gaudi2_init_edma(hdev);
5193     gaudi2_init_sm(hdev);
5194     gaudi2_init_tpc(hdev);
5195     gaudi2_init_mme(hdev);
5196     gaudi2_init_rotator(hdev);
5197     gaudi2_init_dec(hdev);
5198     gaudi2_enable_timestamp(hdev);
5199 
5200     rc = gaudi2_coresight_init(hdev);
5201     if (rc)
5202         goto disable_queues;
5203 
5204     rc = gaudi2_enable_msix(hdev);
5205     if (rc)
5206         goto disable_queues;
5207 
5208     /* Perform read from the device to flush all configuration */
5209     RREG32(mmHW_STATE);
5210 
5211     return 0;
5212 
5213 disable_queues:
5214     gaudi2_disable_dma_qmans(hdev);
5215     gaudi2_disable_mme_qmans(hdev);
5216     gaudi2_disable_tpc_qmans(hdev);
5217     gaudi2_disable_rot_qmans(hdev);
5218     gaudi2_disable_nic_qmans(hdev);
5219 
5220     gaudi2_disable_timestamp(hdev);
5221 
5222     return rc;
5223 }
5224 
5225 /**
5226  * gaudi2_send_hard_reset_cmd - common function to handle reset
5227  *
5228  * @hdev: pointer to the habanalabs device structure
5229  *
5230  * This function handles the various possible scenarios for reset.
5231  * It considers if reset is handled by driver\FW and what FW components are loaded
5232  */
5233 static void gaudi2_send_hard_reset_cmd(struct hl_device *hdev)
5234 {
5235     struct cpu_dyn_regs *dyn_regs = &hdev->fw_loader.dynamic_loader.comm_desc.cpu_dyn_regs;
5236     bool heartbeat_reset, preboot_only, cpu_initialized = false;
5237     struct gaudi2_device *gaudi2 = hdev->asic_specific;
5238     u32 cpu_boot_status;
5239 
5240     preboot_only = (hdev->fw_loader.fw_comp_loaded == FW_TYPE_PREBOOT_CPU);
5241     heartbeat_reset = (hdev->reset_info.curr_reset_cause == HL_RESET_CAUSE_HEARTBEAT);
5242 
5243     /*
5244      * Handle corner case where failure was at cpu management app load,
5245      * and driver didn't detect any failure while loading the FW,
5246      * then at such scenario driver will send only HALT_MACHINE
5247      * and no one will respond to this request since FW already back to preboot
5248      * and it cannot handle such cmd.
5249      * In this case next time the management app loads it'll check on events register
5250      * which will still have the halt indication, and will reboot the device.
5251      * The solution is to let preboot clear all relevant registers before next boot
5252      * once driver send COMMS_RST_DEV.
5253      */
5254     cpu_boot_status = RREG32(mmPSOC_GLOBAL_CONF_CPU_BOOT_STATUS);
5255 
5256     if (gaudi2 && (gaudi2->hw_cap_initialized & HW_CAP_CPU) &&
5257             (cpu_boot_status == CPU_BOOT_STATUS_SRAM_AVAIL))
5258         cpu_initialized = true;
5259 
5260     /*
5261      * when Linux/Bootfit exist this write to the SP can be interpreted in 2 ways:
5262      * 1. FW reset: FW initiate the reset sequence
5263      * 2. driver reset: FW will start HALT sequence (the preparations for the
5264      *                  reset but not the reset itself as it is not implemented
5265      *                  on their part) and LKD will wait to let FW complete the
5266      *                  sequence before issuing the reset
5267      */
5268     if (!preboot_only && cpu_initialized) {
5269         WREG32(le32_to_cpu(dyn_regs->gic_host_halt_irq),
5270             gaudi2_irq_map_table[GAUDI2_EVENT_CPU_HALT_MACHINE].cpu_id);
5271 
5272         msleep(GAUDI2_CPU_RESET_WAIT_MSEC);
5273     }
5274 
5275     /*
5276      * When working with preboot (without Linux/Boot fit) we can
5277      * communicate only using the COMMS commands to issue halt/reset.
5278      *
5279      * For the case in which we are working with Linux/Bootfit this is a hail-mary
5280      * attempt to revive the card in the small chance that the f/w has
5281      * experienced a watchdog event, which caused it to return back to preboot.
5282      * In that case, triggering reset through GIC won't help. We need to
5283      * trigger the reset as if Linux wasn't loaded.
5284      *
5285      * We do it only if the reset cause was HB, because that would be the
5286      * indication of such an event.
5287      *
5288      * In case watchdog hasn't expired but we still got HB, then this won't
5289      * do any damage.
5290      */
5291 
5292     if (heartbeat_reset || preboot_only || !cpu_initialized) {
5293         if (hdev->asic_prop.hard_reset_done_by_fw)
5294             hl_fw_ask_hard_reset_without_linux(hdev);
5295         else
5296             hl_fw_ask_halt_machine_without_linux(hdev);
5297     }
5298 }
5299 
5300 /**
5301  * gaudi2_execute_hard_reset - execute hard reset by driver/FW
5302  *
5303  * @hdev: pointer to the habanalabs device structure
5304  * @reset_sleep_ms: sleep time in msec after reset
5305  *
5306  * This function executes hard reset based on if driver/FW should do the reset
5307  */
5308 static void gaudi2_execute_hard_reset(struct hl_device *hdev, u32 reset_sleep_ms)
5309 {
5310     if (hdev->asic_prop.hard_reset_done_by_fw) {
5311         gaudi2_send_hard_reset_cmd(hdev);
5312         return;
5313     }
5314 
5315     /* Set device to handle FLR by H/W as we will put the device
5316      * CPU to halt mode
5317      */
5318     WREG32(mmPCIE_AUX_FLR_CTRL,
5319             (PCIE_AUX_FLR_CTRL_HW_CTRL_MASK | PCIE_AUX_FLR_CTRL_INT_MASK_MASK));
5320 
5321     gaudi2_send_hard_reset_cmd(hdev);
5322 
5323     WREG32(mmPSOC_RESET_CONF_SW_ALL_RST, 1);
5324 }
5325 
5326 /**
5327  * gaudi2_execute_soft_reset - execute soft reset by driver/FW
5328  *
5329  * @hdev: pointer to the habanalabs device structure
5330  * @reset_sleep_ms: sleep time in msec after reset
5331  * @driver_performs_reset: true if driver should perform reset instead of f/w.
5332  *
5333  * This function executes soft reset based on if driver/FW should do the reset
5334  */
5335 static void gaudi2_execute_soft_reset(struct hl_device *hdev, u32 reset_sleep_ms,
5336                         bool driver_performs_reset)
5337 {
5338     struct cpu_dyn_regs *dyn_regs = &hdev->fw_loader.dynamic_loader.comm_desc.cpu_dyn_regs;
5339 
5340     if (!driver_performs_reset) {
5341         /* set SP to indicate reset request sent to FW */
5342         WREG32(mmCPU_RST_STATUS_TO_HOST, CPU_RST_STATUS_NA);
5343 
5344         WREG32(le32_to_cpu(dyn_regs->gic_host_soft_rst_irq),
5345             gaudi2_irq_map_table[GAUDI2_EVENT_CPU_SOFT_RESET].cpu_id);
5346         return;
5347     }
5348 
5349     /* Block access to engines, QMANs and SM during reset, these
5350      * RRs will be reconfigured after soft reset.
5351      * PCIE_MSIX is left unsecured to allow NIC packets processing during the reset.
5352      */
5353     gaudi2_write_rr_to_all_lbw_rtrs(hdev, RR_TYPE_LONG, NUM_LONG_LBW_RR - 1,
5354                     mmDCORE0_TPC0_QM_DCCM_BASE, mmPCIE_MSIX_BASE);
5355 
5356     gaudi2_write_rr_to_all_lbw_rtrs(hdev, RR_TYPE_LONG, NUM_LONG_LBW_RR - 2,
5357                 mmPCIE_MSIX_BASE + HL_BLOCK_SIZE,
5358                 mmPCIE_VDEC1_MSTR_IF_RR_SHRD_HBW_BASE + HL_BLOCK_SIZE);
5359 
5360     WREG32(mmPSOC_RESET_CONF_SOFT_RST, 1);
5361 }
5362 
5363 static void gaudi2_poll_btm_indication(struct hl_device *hdev, u32 reset_sleep_ms,
5364                                 u32 poll_timeout_us)
5365 {
5366     int i, rc = 0;
5367     u32 reg_val;
5368 
5369     /* without this sleep reset will not work */
5370     msleep(reset_sleep_ms);
5371 
5372     /* We poll the BTM done indication multiple times after reset due to
5373      * a HW errata 'GAUDI2_0300'
5374      */
5375     for (i = 0 ; i < GAUDI2_RESET_POLL_CNT ; i++)
5376         rc = hl_poll_timeout(
5377             hdev,
5378             mmPSOC_GLOBAL_CONF_BTM_FSM,
5379             reg_val,
5380             reg_val == 0,
5381             1000,
5382             poll_timeout_us);
5383 
5384     if (rc)
5385         dev_err(hdev->dev, "Timeout while waiting for device to reset 0x%x\n", reg_val);
5386 }
5387 
5388 static void gaudi2_get_soft_rst_done_indication(struct hl_device *hdev, u32 poll_timeout_us)
5389 {
5390     int i, rc = 0;
5391     u32 reg_val;
5392 
5393     for (i = 0 ; i < GAUDI2_RESET_POLL_CNT ; i++)
5394         rc = hl_poll_timeout(
5395             hdev,
5396             mmCPU_RST_STATUS_TO_HOST,
5397             reg_val,
5398             reg_val == CPU_RST_STATUS_SOFT_RST_DONE,
5399             1000,
5400             poll_timeout_us);
5401 
5402     if (rc)
5403         dev_err(hdev->dev, "Timeout while waiting for FW to complete soft reset (0x%x)\n",
5404                 reg_val);
5405 }
5406 
5407 static void gaudi2_hw_fini(struct hl_device *hdev, bool hard_reset, bool fw_reset)
5408 {
5409     struct gaudi2_device *gaudi2 = hdev->asic_specific;
5410     u32 poll_timeout_us, reset_sleep_ms;
5411     bool driver_performs_reset = false;
5412 
5413     if (hdev->pldm) {
5414         reset_sleep_ms = hard_reset ? GAUDI2_PLDM_HRESET_TIMEOUT_MSEC :
5415                         GAUDI2_PLDM_SRESET_TIMEOUT_MSEC;
5416         poll_timeout_us = GAUDI2_PLDM_RESET_POLL_TIMEOUT_USEC;
5417     } else {
5418         reset_sleep_ms = GAUDI2_RESET_TIMEOUT_MSEC;
5419         poll_timeout_us = GAUDI2_RESET_POLL_TIMEOUT_USEC;
5420     }
5421 
5422     if (fw_reset)
5423         goto skip_reset;
5424 
5425     gaudi2_reset_arcs(hdev);
5426 
5427     if (hard_reset) {
5428         driver_performs_reset = !hdev->asic_prop.hard_reset_done_by_fw;
5429         gaudi2_execute_hard_reset(hdev, reset_sleep_ms);
5430     } else {
5431         /*
5432          * As we have to support also work with preboot only (which does not supports
5433          * soft reset) we have to make sure that security is disabled before letting driver
5434          * do the reset. user shall control the BFE flags to avoid asking soft reset in
5435          * secured device with preboot only.
5436          */
5437         driver_performs_reset = (hdev->fw_components == FW_TYPE_PREBOOT_CPU &&
5438                             !hdev->asic_prop.fw_security_enabled);
5439         gaudi2_execute_soft_reset(hdev, reset_sleep_ms, driver_performs_reset);
5440     }
5441 
5442 skip_reset:
5443     if (driver_performs_reset || hard_reset)
5444         gaudi2_poll_btm_indication(hdev, reset_sleep_ms, poll_timeout_us);
5445     else
5446         gaudi2_get_soft_rst_done_indication(hdev, poll_timeout_us);
5447 
5448     if (!gaudi2)
5449         return;
5450 
5451     gaudi2->dec_hw_cap_initialized &= ~(HW_CAP_DEC_MASK);
5452     gaudi2->tpc_hw_cap_initialized &= ~(HW_CAP_TPC_MASK);
5453 
5454     /*
5455      * Clear NIC capability mask in order for driver to re-configure
5456      * NIC QMANs. NIC ports will not be re-configured during soft
5457      * reset as we call gaudi2_nic_init only during hard reset
5458      */
5459     gaudi2->nic_hw_cap_initialized &= ~(HW_CAP_NIC_MASK);
5460 
5461     if (hard_reset) {
5462         gaudi2->hw_cap_initialized &=
5463             ~(HW_CAP_DRAM | HW_CAP_CLK_GATE | HW_CAP_HBM_SCRAMBLER_MASK |
5464             HW_CAP_PMMU | HW_CAP_CPU | HW_CAP_CPU_Q |
5465             HW_CAP_SRAM_SCRAMBLER | HW_CAP_DMMU_MASK |
5466             HW_CAP_PDMA_MASK | HW_CAP_EDMA_MASK | HW_CAP_KDMA |
5467             HW_CAP_MME_MASK | HW_CAP_ROT_MASK);
5468 
5469         memset(gaudi2->events_stat, 0, sizeof(gaudi2->events_stat));
5470     } else {
5471         gaudi2->hw_cap_initialized &=
5472             ~(HW_CAP_CLK_GATE | HW_CAP_HBM_SCRAMBLER_SW_RESET |
5473             HW_CAP_PDMA_MASK | HW_CAP_EDMA_MASK | HW_CAP_MME_MASK |
5474             HW_CAP_ROT_MASK);
5475     }
5476 }
5477 
5478 static int gaudi2_suspend(struct hl_device *hdev)
5479 {
5480     int rc;
5481 
5482     rc = hl_fw_send_pci_access_msg(hdev, CPUCP_PACKET_DISABLE_PCI_ACCESS, 0x0);
5483     if (rc)
5484         dev_err(hdev->dev, "Failed to disable PCI access from CPU\n");
5485 
5486     return rc;
5487 }
5488 
5489 static int gaudi2_resume(struct hl_device *hdev)
5490 {
5491     return gaudi2_init_iatu(hdev);
5492 }
5493 
5494 static int gaudi2_mmap(struct hl_device *hdev, struct vm_area_struct *vma,
5495         void *cpu_addr, dma_addr_t dma_addr, size_t size)
5496 {
5497     int rc;
5498 
5499     vma->vm_flags |= VM_IO | VM_PFNMAP | VM_DONTEXPAND | VM_DONTDUMP |
5500             VM_DONTCOPY | VM_NORESERVE;
5501 
5502 #ifdef _HAS_DMA_MMAP_COHERENT
5503 
5504     rc = dma_mmap_coherent(hdev->dev, vma, cpu_addr, dma_addr, size);
5505     if (rc)
5506         dev_err(hdev->dev, "dma_mmap_coherent error %d", rc);
5507 
5508 #else
5509 
5510     rc = remap_pfn_range(vma, vma->vm_start,
5511                 virt_to_phys(cpu_addr) >> PAGE_SHIFT,
5512                 size, vma->vm_page_prot);
5513     if (rc)
5514         dev_err(hdev->dev, "remap_pfn_range error %d", rc);
5515 
5516 #endif
5517 
5518     return rc;
5519 }
5520 
5521 static bool gaudi2_is_queue_enabled(struct hl_device *hdev, u32 hw_queue_id)
5522 {
5523     struct gaudi2_device *gaudi2 = hdev->asic_specific;
5524     u64 hw_cap_mask = 0;
5525     u64 hw_tpc_cap_bit = 0;
5526     u64 hw_nic_cap_bit = 0;
5527     u64 hw_test_cap_bit = 0;
5528 
5529     switch (hw_queue_id) {
5530     case GAUDI2_QUEUE_ID_PDMA_0_0 ... GAUDI2_QUEUE_ID_PDMA_1_1:
5531         hw_cap_mask = HW_CAP_PDMA_MASK;
5532         break;
5533 
5534     case GAUDI2_QUEUE_ID_DCORE0_EDMA_0_0...GAUDI2_QUEUE_ID_DCORE0_EDMA_1_3:
5535         hw_test_cap_bit = HW_CAP_EDMA_SHIFT +
5536             ((hw_queue_id - GAUDI2_QUEUE_ID_DCORE0_EDMA_0_0) >> 2);
5537         break;
5538     case GAUDI2_QUEUE_ID_DCORE1_EDMA_0_0...GAUDI2_QUEUE_ID_DCORE1_EDMA_1_3:
5539         hw_test_cap_bit = HW_CAP_EDMA_SHIFT + NUM_OF_EDMA_PER_DCORE +
5540             ((hw_queue_id - GAUDI2_QUEUE_ID_DCORE1_EDMA_0_0) >> 2);
5541         break;
5542     case GAUDI2_QUEUE_ID_DCORE2_EDMA_0_0...GAUDI2_QUEUE_ID_DCORE2_EDMA_1_3:
5543         hw_test_cap_bit = HW_CAP_EDMA_SHIFT + 2 * NUM_OF_EDMA_PER_DCORE +
5544             ((hw_queue_id - GAUDI2_QUEUE_ID_DCORE2_EDMA_0_0) >> 2);
5545         break;
5546     case GAUDI2_QUEUE_ID_DCORE3_EDMA_0_0...GAUDI2_QUEUE_ID_DCORE3_EDMA_1_3:
5547         hw_test_cap_bit = HW_CAP_EDMA_SHIFT + 3 * NUM_OF_EDMA_PER_DCORE +
5548             ((hw_queue_id - GAUDI2_QUEUE_ID_DCORE3_EDMA_0_0) >> 2);
5549         break;
5550 
5551     case GAUDI2_QUEUE_ID_DCORE0_MME_0_0 ... GAUDI2_QUEUE_ID_DCORE0_MME_0_3:
5552         hw_test_cap_bit = HW_CAP_MME_SHIFT;
5553         break;
5554 
5555     case GAUDI2_QUEUE_ID_DCORE1_MME_0_0 ... GAUDI2_QUEUE_ID_DCORE1_MME_0_3:
5556         hw_test_cap_bit = HW_CAP_MME_SHIFT + 1;
5557         break;
5558 
5559     case GAUDI2_QUEUE_ID_DCORE2_MME_0_0 ... GAUDI2_QUEUE_ID_DCORE2_MME_0_3:
5560         hw_test_cap_bit = HW_CAP_MME_SHIFT + 2;
5561         break;
5562 
5563     case GAUDI2_QUEUE_ID_DCORE3_MME_0_0 ... GAUDI2_QUEUE_ID_DCORE3_MME_0_3:
5564         hw_test_cap_bit = HW_CAP_MME_SHIFT + 3;
5565         break;
5566 
5567     case GAUDI2_QUEUE_ID_DCORE0_TPC_0_0 ... GAUDI2_QUEUE_ID_DCORE0_TPC_5_3:
5568         hw_tpc_cap_bit = HW_CAP_TPC_SHIFT +
5569             ((hw_queue_id - GAUDI2_QUEUE_ID_DCORE0_TPC_0_0) >> 2);
5570 
5571         /* special case where cap bit refers to the first queue id */
5572         if (!hw_tpc_cap_bit)
5573             return !!(gaudi2->tpc_hw_cap_initialized & BIT_ULL(0));
5574         break;
5575 
5576     case GAUDI2_QUEUE_ID_DCORE1_TPC_0_0 ... GAUDI2_QUEUE_ID_DCORE1_TPC_5_3:
5577         hw_tpc_cap_bit = HW_CAP_TPC_SHIFT + NUM_OF_TPC_PER_DCORE +
5578             ((hw_queue_id - GAUDI2_QUEUE_ID_DCORE1_TPC_0_0) >> 2);
5579         break;
5580 
5581     case GAUDI2_QUEUE_ID_DCORE2_TPC_0_0 ... GAUDI2_QUEUE_ID_DCORE2_TPC_5_3:
5582         hw_tpc_cap_bit = HW_CAP_TPC_SHIFT + (2 * NUM_OF_TPC_PER_DCORE) +
5583             ((hw_queue_id - GAUDI2_QUEUE_ID_DCORE2_TPC_0_0) >> 2);
5584         break;
5585 
5586     case GAUDI2_QUEUE_ID_DCORE3_TPC_0_0 ... GAUDI2_QUEUE_ID_DCORE3_TPC_5_3:
5587         hw_tpc_cap_bit = HW_CAP_TPC_SHIFT + (3 * NUM_OF_TPC_PER_DCORE) +
5588             ((hw_queue_id - GAUDI2_QUEUE_ID_DCORE3_TPC_0_0) >> 2);
5589         break;
5590 
5591     case GAUDI2_QUEUE_ID_DCORE0_TPC_6_0 ... GAUDI2_QUEUE_ID_DCORE0_TPC_6_3:
5592         hw_tpc_cap_bit = HW_CAP_TPC_SHIFT + (4 * NUM_OF_TPC_PER_DCORE);
5593         break;
5594 
5595     case GAUDI2_QUEUE_ID_ROT_0_0 ... GAUDI2_QUEUE_ID_ROT_1_3:
5596         hw_test_cap_bit = HW_CAP_ROT_SHIFT + ((hw_queue_id - GAUDI2_QUEUE_ID_ROT_0_0) >> 2);
5597         break;
5598 
5599     case GAUDI2_QUEUE_ID_NIC_0_0 ... GAUDI2_QUEUE_ID_NIC_23_3:
5600         hw_nic_cap_bit = HW_CAP_NIC_SHIFT + ((hw_queue_id - GAUDI2_QUEUE_ID_NIC_0_0) >> 2);
5601 
5602         /* special case where cap bit refers to the first queue id */
5603         if (!hw_nic_cap_bit)
5604             return !!(gaudi2->nic_hw_cap_initialized & BIT_ULL(0));
5605         break;
5606 
5607     case GAUDI2_QUEUE_ID_CPU_PQ:
5608         return !!(gaudi2->hw_cap_initialized & HW_CAP_CPU_Q);
5609 
5610     default:
5611         return false;
5612     }
5613 
5614     if (hw_tpc_cap_bit)
5615         return  !!(gaudi2->tpc_hw_cap_initialized & BIT_ULL(hw_tpc_cap_bit));
5616 
5617     if (hw_nic_cap_bit)
5618         return  !!(gaudi2->nic_hw_cap_initialized & BIT_ULL(hw_nic_cap_bit));
5619 
5620     if (hw_test_cap_bit)
5621         hw_cap_mask = BIT_ULL(hw_test_cap_bit);
5622 
5623     return !!(gaudi2->hw_cap_initialized & hw_cap_mask);
5624 }
5625 
5626 static bool gaudi2_is_arc_enabled(struct hl_device *hdev, u64 arc_id)
5627 {
5628     struct gaudi2_device *gaudi2 = hdev->asic_specific;
5629 
5630     switch (arc_id) {
5631     case CPU_ID_SCHED_ARC0 ... CPU_ID_SCHED_ARC5:
5632     case CPU_ID_MME_QMAN_ARC0...CPU_ID_ROT_QMAN_ARC1:
5633         return !!(gaudi2->active_hw_arc & BIT_ULL(arc_id));
5634 
5635     case CPU_ID_TPC_QMAN_ARC0...CPU_ID_TPC_QMAN_ARC24:
5636         return !!(gaudi2->active_tpc_arc & BIT_ULL(arc_id - CPU_ID_TPC_QMAN_ARC0));
5637 
5638     case CPU_ID_NIC_QMAN_ARC0...CPU_ID_NIC_QMAN_ARC23:
5639         return !!(gaudi2->active_nic_arc & BIT_ULL(arc_id - CPU_ID_NIC_QMAN_ARC0));
5640 
5641     default:
5642         return false;
5643     }
5644 }
5645 
5646 static void gaudi2_clr_arc_id_cap(struct hl_device *hdev, u64 arc_id)
5647 {
5648     struct gaudi2_device *gaudi2 = hdev->asic_specific;
5649 
5650     switch (arc_id) {
5651     case CPU_ID_SCHED_ARC0 ... CPU_ID_SCHED_ARC5:
5652     case CPU_ID_MME_QMAN_ARC0...CPU_ID_ROT_QMAN_ARC1:
5653         gaudi2->active_hw_arc &= ~(BIT_ULL(arc_id));
5654         break;
5655 
5656     case CPU_ID_TPC_QMAN_ARC0...CPU_ID_TPC_QMAN_ARC24:
5657         gaudi2->active_tpc_arc &= ~(BIT_ULL(arc_id - CPU_ID_TPC_QMAN_ARC0));
5658         break;
5659 
5660     case CPU_ID_NIC_QMAN_ARC0...CPU_ID_NIC_QMAN_ARC23:
5661         gaudi2->active_nic_arc &= ~(BIT_ULL(arc_id - CPU_ID_NIC_QMAN_ARC0));
5662         break;
5663 
5664     default:
5665         return;
5666     }
5667 }
5668 
5669 static void gaudi2_set_arc_id_cap(struct hl_device *hdev, u64 arc_id)
5670 {
5671     struct gaudi2_device *gaudi2 = hdev->asic_specific;
5672 
5673     switch (arc_id) {
5674     case CPU_ID_SCHED_ARC0 ... CPU_ID_SCHED_ARC5:
5675     case CPU_ID_MME_QMAN_ARC0...CPU_ID_ROT_QMAN_ARC1:
5676         gaudi2->active_hw_arc |= BIT_ULL(arc_id);
5677         break;
5678 
5679     case CPU_ID_TPC_QMAN_ARC0...CPU_ID_TPC_QMAN_ARC24:
5680         gaudi2->active_tpc_arc |= BIT_ULL(arc_id - CPU_ID_TPC_QMAN_ARC0);
5681         break;
5682 
5683     case CPU_ID_NIC_QMAN_ARC0...CPU_ID_NIC_QMAN_ARC23:
5684         gaudi2->active_nic_arc |= BIT_ULL(arc_id - CPU_ID_NIC_QMAN_ARC0);
5685         break;
5686 
5687     default:
5688         return;
5689     }
5690 }
5691 
5692 static void gaudi2_ring_doorbell(struct hl_device *hdev, u32 hw_queue_id, u32 pi)
5693 {
5694     struct cpu_dyn_regs *dyn_regs = &hdev->fw_loader.dynamic_loader.comm_desc.cpu_dyn_regs;
5695     u32 pq_offset, reg_base, db_reg_offset, db_value;
5696 
5697     if (hw_queue_id != GAUDI2_QUEUE_ID_CPU_PQ) {
5698         /*
5699          * QMAN has 4 successive PQ_PI registers, 1 for each of the QMAN PQs.
5700          * Masking the H/W queue ID with 0x3 extracts the QMAN internal PQ
5701          * number.
5702          */
5703         pq_offset = (hw_queue_id & 0x3) * 4;
5704         reg_base = gaudi2_qm_blocks_bases[hw_queue_id];
5705         db_reg_offset = reg_base + QM_PQ_PI_0_OFFSET + pq_offset;
5706     } else {
5707         db_reg_offset = mmCPU_IF_PF_PQ_PI;
5708     }
5709 
5710     db_value = pi;
5711 
5712     /* ring the doorbell */
5713     WREG32(db_reg_offset, db_value);
5714 
5715     if (hw_queue_id == GAUDI2_QUEUE_ID_CPU_PQ) {
5716         /* make sure device CPU will read latest data from host */
5717         mb();
5718         WREG32(le32_to_cpu(dyn_regs->gic_host_pi_upd_irq),
5719             gaudi2_irq_map_table[GAUDI2_EVENT_CPU_PI_UPDATE].cpu_id);
5720     }
5721 }
5722 
5723 static void gaudi2_pqe_write(struct hl_device *hdev, __le64 *pqe, struct hl_bd *bd)
5724 {
5725     __le64 *pbd = (__le64 *) bd;
5726 
5727     /* The QMANs are on the host memory so a simple copy suffice */
5728     pqe[0] = pbd[0];
5729     pqe[1] = pbd[1];
5730 }
5731 
5732 static void *gaudi2_dma_alloc_coherent(struct hl_device *hdev, size_t size,
5733                 dma_addr_t *dma_handle, gfp_t flags)
5734 {
5735     return dma_alloc_coherent(&hdev->pdev->dev, size, dma_handle, flags);
5736 }
5737 
5738 static void gaudi2_dma_free_coherent(struct hl_device *hdev, size_t size,
5739                 void *cpu_addr, dma_addr_t dma_handle)
5740 {
5741     dma_free_coherent(&hdev->pdev->dev, size, cpu_addr, dma_handle);
5742 }
5743 
5744 static int gaudi2_send_cpu_message(struct hl_device *hdev, u32 *msg, u16 len,
5745                 u32 timeout, u64 *result)
5746 {
5747     struct gaudi2_device *gaudi2 = hdev->asic_specific;
5748 
5749     if (!(gaudi2->hw_cap_initialized & HW_CAP_CPU_Q)) {
5750         if (result)
5751             *result = 0;
5752         return 0;
5753     }
5754 
5755     if (!timeout)
5756         timeout = GAUDI2_MSG_TO_CPU_TIMEOUT_USEC;
5757 
5758     return hl_fw_send_cpu_message(hdev, GAUDI2_QUEUE_ID_CPU_PQ, msg, len, timeout, result);
5759 }
5760 
5761 static void *gaudi2_dma_pool_zalloc(struct hl_device *hdev, size_t size,
5762                 gfp_t mem_flags, dma_addr_t *dma_handle)
5763 {
5764     if (size > GAUDI2_DMA_POOL_BLK_SIZE)
5765         return NULL;
5766 
5767     return dma_pool_zalloc(hdev->dma_pool, mem_flags, dma_handle);
5768 }
5769 
5770 static void gaudi2_dma_pool_free(struct hl_device *hdev, void *vaddr, dma_addr_t dma_addr)
5771 {
5772     dma_pool_free(hdev->dma_pool, vaddr, dma_addr);
5773 }
5774 
5775 static void *gaudi2_cpu_accessible_dma_pool_alloc(struct hl_device *hdev, size_t size,
5776                         dma_addr_t *dma_handle)
5777 {
5778     return hl_fw_cpu_accessible_dma_pool_alloc(hdev, size, dma_handle);
5779 }
5780 
5781 static void gaudi2_cpu_accessible_dma_pool_free(struct hl_device *hdev, size_t size, void *vaddr)
5782 {
5783     hl_fw_cpu_accessible_dma_pool_free(hdev, size, vaddr);
5784 }
5785 
5786 static dma_addr_t gaudi2_dma_map_single(struct hl_device *hdev, void *addr, int len,
5787                     enum dma_data_direction dir)
5788 {
5789     dma_addr_t dma_addr;
5790 
5791     dma_addr = dma_map_single(&hdev->pdev->dev, addr, len, dir);
5792     if (unlikely(dma_mapping_error(&hdev->pdev->dev, dma_addr)))
5793         return 0;
5794 
5795     return dma_addr;
5796 }
5797 
5798 static void gaudi2_dma_unmap_single(struct hl_device *hdev, dma_addr_t addr, int len,
5799                     enum dma_data_direction dir)
5800 {
5801     dma_unmap_single(&hdev->pdev->dev, addr, len, dir);
5802 }
5803 
5804 static int gaudi2_validate_cb_address(struct hl_device *hdev, struct hl_cs_parser *parser)
5805 {
5806     struct asic_fixed_properties *asic_prop = &hdev->asic_prop;
5807     struct gaudi2_device *gaudi2 = hdev->asic_specific;
5808 
5809     if (!gaudi2_is_queue_enabled(hdev, parser->hw_queue_id)) {
5810         dev_err(hdev->dev, "h/w queue %d is disabled\n", parser->hw_queue_id);
5811         return -EINVAL;
5812     }
5813 
5814     /* Just check if CB address is valid */
5815 
5816     if (hl_mem_area_inside_range((u64) (uintptr_t) parser->user_cb,
5817                     parser->user_cb_size,
5818                     asic_prop->sram_user_base_address,
5819                     asic_prop->sram_end_address))
5820         return 0;
5821 
5822     if (hl_mem_area_inside_range((u64) (uintptr_t) parser->user_cb,
5823                     parser->user_cb_size,
5824                     asic_prop->dram_user_base_address,
5825                     asic_prop->dram_end_address))
5826         return 0;
5827 
5828     if ((gaudi2->hw_cap_initialized & HW_CAP_DMMU_MASK) &&
5829         hl_mem_area_inside_range((u64) (uintptr_t) parser->user_cb,
5830                         parser->user_cb_size,
5831                         asic_prop->dmmu.start_addr,
5832                         asic_prop->dmmu.end_addr))
5833         return 0;
5834 
5835     if (gaudi2->hw_cap_initialized & HW_CAP_PMMU) {
5836         if (hl_mem_area_inside_range((u64) (uintptr_t) parser->user_cb,
5837                     parser->user_cb_size,
5838                     asic_prop->pmmu.start_addr,
5839                     asic_prop->pmmu.end_addr) ||
5840             hl_mem_area_inside_range(
5841                     (u64) (uintptr_t) parser->user_cb,
5842                     parser->user_cb_size,
5843                     asic_prop->pmmu_huge.start_addr,
5844                     asic_prop->pmmu_huge.end_addr))
5845             return 0;
5846 
5847     } else if (gaudi2_host_phys_addr_valid((u64) (uintptr_t) parser->user_cb)) {
5848         if (!hdev->pdev)
5849             return 0;
5850 
5851         if (!device_iommu_mapped(&hdev->pdev->dev))
5852             return 0;
5853     }
5854 
5855     dev_err(hdev->dev, "CB address %p + 0x%x for internal QMAN is not valid\n",
5856         parser->user_cb, parser->user_cb_size);
5857 
5858     return -EFAULT;
5859 }
5860 
5861 static int gaudi2_cs_parser(struct hl_device *hdev, struct hl_cs_parser *parser)
5862 {
5863     struct gaudi2_device *gaudi2 = hdev->asic_specific;
5864 
5865     if (!parser->is_kernel_allocated_cb)
5866         return gaudi2_validate_cb_address(hdev, parser);
5867 
5868     if (!(gaudi2->hw_cap_initialized & HW_CAP_PMMU)) {
5869         dev_err(hdev->dev, "PMMU not initialized - Unsupported mode in Gaudi2\n");
5870         return -EINVAL;
5871     }
5872 
5873     return 0;
5874 }
5875 
5876 static int gaudi2_send_heartbeat(struct hl_device *hdev)
5877 {
5878     struct gaudi2_device *gaudi2 = hdev->asic_specific;
5879 
5880     if (!(gaudi2->hw_cap_initialized & HW_CAP_CPU_Q))
5881         return 0;
5882 
5883     return hl_fw_send_heartbeat(hdev);
5884 }
5885 
5886 /* This is an internal helper function, used to update the KDMA mmu props.
5887  * Should be called with a proper kdma lock.
5888  */
5889 static void gaudi2_kdma_set_mmbp_asid(struct hl_device *hdev,
5890                        bool mmu_bypass, u32 asid)
5891 {
5892     u32 rw_asid, rw_mmu_bp;
5893 
5894     rw_asid = (asid << ARC_FARM_KDMA_CTX_AXUSER_HB_ASID_RD_SHIFT) |
5895               (asid << ARC_FARM_KDMA_CTX_AXUSER_HB_ASID_WR_SHIFT);
5896 
5897     rw_mmu_bp = (!!mmu_bypass << ARC_FARM_KDMA_CTX_AXUSER_HB_MMU_BP_RD_SHIFT) |
5898             (!!mmu_bypass << ARC_FARM_KDMA_CTX_AXUSER_HB_MMU_BP_WR_SHIFT);
5899 
5900     WREG32(mmARC_FARM_KDMA_CTX_AXUSER_HB_ASID, rw_asid);
5901     WREG32(mmARC_FARM_KDMA_CTX_AXUSER_HB_MMU_BP, rw_mmu_bp);
5902 }
5903 
5904 static void gaudi2_arm_cq_monitor(struct hl_device *hdev, u32 sob_id, u32 mon_id, u32 cq_id,
5905                         u32 mon_payload, u32 sync_value)
5906 {
5907     u32 sob_offset, mon_offset, sync_group_id, mode, mon_arm;
5908     u8 mask;
5909 
5910     sob_offset = sob_id * 4;
5911     mon_offset = mon_id * 4;
5912 
5913     /* Reset the SOB value */
5914     WREG32(mmDCORE0_SYNC_MNGR_OBJS_SOB_OBJ_0 + sob_offset, 0);
5915 
5916     /* Configure this address with CQ_ID 0 because CQ_EN is set */
5917     WREG32(mmDCORE0_SYNC_MNGR_OBJS_MON_PAY_ADDRL_0 + mon_offset, cq_id);
5918 
5919     /* Configure this address with CS index because CQ_EN is set */
5920     WREG32(mmDCORE0_SYNC_MNGR_OBJS_MON_PAY_DATA_0 + mon_offset, mon_payload);
5921 
5922     sync_group_id = sob_id / 8;
5923     mask = ~(1 << (sob_id & 0x7));
5924     mode = 1; /* comparison mode is "equal to" */
5925 
5926     mon_arm = FIELD_PREP(DCORE0_SYNC_MNGR_OBJS_MON_ARM_SOD_MASK, sync_value);
5927     mon_arm |= FIELD_PREP(DCORE0_SYNC_MNGR_OBJS_MON_ARM_SOP_MASK, mode);
5928     mon_arm |= FIELD_PREP(DCORE0_SYNC_MNGR_OBJS_MON_ARM_MASK_MASK, mask);
5929     mon_arm |= FIELD_PREP(DCORE0_SYNC_MNGR_OBJS_MON_ARM_SID_MASK, sync_group_id);
5930     WREG32(mmDCORE0_SYNC_MNGR_OBJS_MON_ARM_0 + mon_offset, mon_arm);
5931 }
5932 
5933 /* This is an internal helper function used by gaudi2_send_job_to_kdma only */
5934 static int gaudi2_send_job_to_kdma(struct hl_device *hdev,
5935                     u64 src_addr, u64 dst_addr,
5936                     u32 size, bool is_memset)
5937 {
5938     u32 comp_val, commit_mask, *polling_addr, timeout, status = 0;
5939     struct hl_cq_entry *cq_base;
5940     struct hl_cq *cq;
5941     u64 comp_addr;
5942     int rc;
5943 
5944     gaudi2_arm_cq_monitor(hdev, GAUDI2_RESERVED_SOB_KDMA_COMPLETION,
5945                 GAUDI2_RESERVED_MON_KDMA_COMPLETION,
5946                 GAUDI2_RESERVED_CQ_KDMA_COMPLETION, 1, 1);
5947 
5948     comp_addr = CFG_BASE + mmDCORE0_SYNC_MNGR_OBJS_SOB_OBJ_0 +
5949             (GAUDI2_RESERVED_SOB_KDMA_COMPLETION * sizeof(u32));
5950 
5951     comp_val = FIELD_PREP(DCORE0_SYNC_MNGR_OBJS_SOB_OBJ_INC_MASK, 1) |
5952             FIELD_PREP(DCORE0_SYNC_MNGR_OBJS_SOB_OBJ_VAL_MASK, 1);
5953 
5954     WREG32(mmARC_FARM_KDMA_CTX_SRC_BASE_LO, lower_32_bits(src_addr));
5955     WREG32(mmARC_FARM_KDMA_CTX_SRC_BASE_HI, upper_32_bits(src_addr));
5956     WREG32(mmARC_FARM_KDMA_CTX_DST_BASE_LO, lower_32_bits(dst_addr));
5957     WREG32(mmARC_FARM_KDMA_CTX_DST_BASE_HI, upper_32_bits(dst_addr));
5958     WREG32(mmARC_FARM_KDMA_CTX_WR_COMP_ADDR_LO, lower_32_bits(comp_addr));
5959     WREG32(mmARC_FARM_KDMA_CTX_WR_COMP_ADDR_HI, upper_32_bits(comp_addr));
5960     WREG32(mmARC_FARM_KDMA_CTX_WR_COMP_WDATA, comp_val);
5961     WREG32(mmARC_FARM_KDMA_CTX_DST_TSIZE_0, size);
5962 
5963     commit_mask = FIELD_PREP(ARC_FARM_KDMA_CTX_COMMIT_LIN_MASK, 1) |
5964                 FIELD_PREP(ARC_FARM_KDMA_CTX_COMMIT_WR_COMP_EN_MASK, 1);
5965 
5966     if (is_memset)
5967         commit_mask |= FIELD_PREP(ARC_FARM_KDMA_CTX_COMMIT_MEM_SET_MASK, 1);
5968 
5969     WREG32(mmARC_FARM_KDMA_CTX_COMMIT, commit_mask);
5970 
5971     /* Wait for completion */
5972     cq = &hdev->completion_queue[GAUDI2_RESERVED_CQ_KDMA_COMPLETION];
5973     cq_base = cq->kernel_address;
5974     polling_addr = (u32 *)&cq_base[cq->ci];
5975 
5976     if (hdev->pldm)
5977         /* for each 1MB 20 second of timeout */
5978         timeout = ((size / SZ_1M) + 1) * USEC_PER_SEC * 20;
5979     else
5980         timeout = KDMA_TIMEOUT_USEC;
5981 
5982     /* Polling */
5983     rc = hl_poll_timeout_memory(
5984             hdev,
5985             polling_addr,
5986             status,
5987             (status == 1),
5988             1000,
5989             timeout,
5990             true);
5991 
5992     *polling_addr = 0;
5993 
5994     if (rc) {
5995         dev_err(hdev->dev, "Timeout while waiting for KDMA to be idle\n");
5996         WREG32(mmARC_FARM_KDMA_CFG_1, 1 << ARC_FARM_KDMA_CFG_1_HALT_SHIFT);
5997         return rc;
5998     }
5999 
6000     cq->ci = hl_cq_inc_ptr(cq->ci);
6001 
6002     return 0;
6003 }
6004 
6005 static void gaudi2_memset_device_lbw(struct hl_device *hdev, u32 addr, u32 size, u32 val)
6006 {
6007     u32 i;
6008 
6009     for (i = 0 ; i < size ; i += sizeof(u32))
6010         WREG32(addr + i, val);
6011 }
6012 
6013 static void gaudi2_qman_set_test_mode(struct hl_device *hdev, u32 hw_queue_id, bool enable)
6014 {
6015     u32 reg_base = gaudi2_qm_blocks_bases[hw_queue_id];
6016 
6017     if (enable) {
6018         WREG32(reg_base + QM_GLBL_PROT_OFFSET, QMAN_MAKE_TRUSTED_TEST_MODE);
6019         WREG32(reg_base + QM_PQC_CFG_OFFSET, 0);
6020     } else {
6021         WREG32(reg_base + QM_GLBL_PROT_OFFSET, QMAN_MAKE_TRUSTED);
6022         WREG32(reg_base + QM_PQC_CFG_OFFSET, 1 << PDMA0_QM_PQC_CFG_EN_SHIFT);
6023     }
6024 }
6025 
6026 static int gaudi2_test_queue(struct hl_device *hdev, u32 hw_queue_id)
6027 {
6028     u32 sob_offset = hdev->asic_prop.first_available_user_sob[0] * 4;
6029     u32 sob_addr = mmDCORE0_SYNC_MNGR_OBJS_SOB_OBJ_0 + sob_offset;
6030     u32 timeout_usec, tmp, sob_base = 1, sob_val = 0x5a5a;
6031     struct packet_msg_short *msg_short_pkt;
6032     dma_addr_t pkt_dma_addr;
6033     size_t pkt_size;
6034     int rc;
6035 
6036     if (hdev->pldm)
6037         timeout_usec = GAUDI2_PLDM_TEST_QUEUE_WAIT_USEC;
6038     else
6039         timeout_usec = GAUDI2_TEST_QUEUE_WAIT_USEC;
6040 
6041     pkt_size = sizeof(*msg_short_pkt);
6042     msg_short_pkt = hl_asic_dma_pool_zalloc(hdev, pkt_size, GFP_KERNEL, &pkt_dma_addr);
6043     if (!msg_short_pkt) {
6044         dev_err(hdev->dev, "Failed to allocate packet for H/W queue %d testing\n",
6045             hw_queue_id);
6046         return -ENOMEM;
6047     }
6048 
6049     tmp = (PACKET_MSG_SHORT << GAUDI2_PKT_CTL_OPCODE_SHIFT) |
6050         (1 << GAUDI2_PKT_CTL_EB_SHIFT) |
6051         (1 << GAUDI2_PKT_CTL_MB_SHIFT) |
6052         (sob_base << GAUDI2_PKT_SHORT_CTL_BASE_SHIFT) |
6053         (sob_offset << GAUDI2_PKT_SHORT_CTL_ADDR_SHIFT);
6054 
6055     msg_short_pkt->value = cpu_to_le32(sob_val);
6056     msg_short_pkt->ctl = cpu_to_le32(tmp);
6057 
6058     /* Reset the SOB value */
6059     WREG32(sob_addr, 0);
6060 
6061     rc = hl_hw_queue_send_cb_no_cmpl(hdev, hw_queue_id, pkt_size, pkt_dma_addr);
6062     if (rc) {
6063         dev_err(hdev->dev, "Failed to send msg_short packet to H/W queue %d\n",
6064             hw_queue_id);
6065         goto free_pkt;
6066     }
6067 
6068     rc = hl_poll_timeout(
6069             hdev,
6070             sob_addr,
6071             tmp,
6072             (tmp == sob_val),
6073             1000,
6074             timeout_usec);
6075 
6076     if (rc == -ETIMEDOUT) {
6077         dev_err(hdev->dev, "H/W queue %d test failed (SOB_OBJ_0 == 0x%x)\n",
6078             hw_queue_id, tmp);
6079         rc = -EIO;
6080     }
6081 
6082     /* Reset the SOB value */
6083     WREG32(sob_addr, 0);
6084 
6085 free_pkt:
6086     hl_asic_dma_pool_free(hdev, (void *) msg_short_pkt, pkt_dma_addr);
6087     return rc;
6088 }
6089 
6090 static int gaudi2_test_cpu_queue(struct hl_device *hdev)
6091 {
6092     struct gaudi2_device *gaudi2 = hdev->asic_specific;
6093 
6094     /*
6095      * check capability here as send_cpu_message() won't update the result
6096      * value if no capability
6097      */
6098     if (!(gaudi2->hw_cap_initialized & HW_CAP_CPU_Q))
6099         return 0;
6100 
6101     return hl_fw_test_cpu_queue(hdev);
6102 }
6103 
6104 static int gaudi2_test_queues(struct hl_device *hdev)
6105 {
6106     int i, rc, ret_val = 0;
6107 
6108     for (i = GAUDI2_QUEUE_ID_PDMA_0_0 ; i < GAUDI2_QUEUE_ID_CPU_PQ; i++) {
6109         if (!gaudi2_is_queue_enabled(hdev, i))
6110             continue;
6111 
6112         gaudi2_qman_set_test_mode(hdev, i, true);
6113         rc = gaudi2_test_queue(hdev, i);
6114         gaudi2_qman_set_test_mode(hdev, i, false);
6115 
6116         if (rc) {
6117             ret_val = -EINVAL;
6118             goto done;
6119         }
6120     }
6121 
6122     rc = gaudi2_test_cpu_queue(hdev);
6123     if (rc) {
6124         ret_val = -EINVAL;
6125         goto done;
6126     }
6127 
6128 done:
6129     return ret_val;
6130 }
6131 
6132 static int gaudi2_non_hard_reset_late_init(struct hl_device *hdev)
6133 {
6134     struct gaudi2_device *gaudi2 = hdev->asic_specific;
6135     size_t irq_arr_size;
6136 
6137     /* TODO: missing gaudi2_nic_resume.
6138      * Until implemented nic_hw_cap_initialized will remain zeroed
6139      */
6140     gaudi2_init_arcs(hdev);
6141     gaudi2_scrub_arcs_dccm(hdev);
6142     gaudi2_init_security(hdev);
6143 
6144     /* Unmask all IRQs since some could have been received during the soft reset */
6145     irq_arr_size = gaudi2->num_of_valid_hw_events * sizeof(gaudi2->hw_events[0]);
6146     return hl_fw_unmask_irq_arr(hdev, gaudi2->hw_events, irq_arr_size);
6147 }
6148 
6149 static void gaudi2_is_tpc_engine_idle(struct hl_device *hdev, int dcore, int inst, u32 offset,
6150                     void *data)
6151 {
6152     struct gaudi2_tpc_idle_data *idle_data = (struct gaudi2_tpc_idle_data *)data;
6153     u32 tpc_cfg_sts, qm_glbl_sts0, qm_glbl_sts1, qm_cgm_sts;
6154     bool is_eng_idle;
6155     int engine_idx;
6156 
6157     if ((dcore == 0) && (inst == (NUM_DCORE0_TPC - 1)))
6158         engine_idx = GAUDI2_DCORE0_ENGINE_ID_TPC_6;
6159     else
6160         engine_idx = GAUDI2_DCORE0_ENGINE_ID_TPC_0 +
6161                 dcore * GAUDI2_ENGINE_ID_DCORE_OFFSET + inst;
6162 
6163     tpc_cfg_sts = RREG32(mmDCORE0_TPC0_CFG_STATUS + offset);
6164     qm_glbl_sts0 = RREG32(mmDCORE0_TPC0_QM_GLBL_STS0 + offset);
6165     qm_glbl_sts1 = RREG32(mmDCORE0_TPC0_QM_GLBL_STS1 + offset);
6166     qm_cgm_sts = RREG32(mmDCORE0_TPC0_QM_CGM_STS + offset);
6167 
6168     is_eng_idle = IS_QM_IDLE(qm_glbl_sts0, qm_glbl_sts1, qm_cgm_sts) &&
6169                         IS_TPC_IDLE(tpc_cfg_sts);
6170     *(idle_data->is_idle) &= is_eng_idle;
6171 
6172     if (idle_data->mask && !is_eng_idle)
6173         set_bit(engine_idx, idle_data->mask);
6174 
6175     if (idle_data->s)
6176         seq_printf(idle_data->s, idle_data->tpc_fmt, dcore, inst,
6177                     is_eng_idle ? "Y" : "N",
6178                     qm_glbl_sts0, qm_cgm_sts, tpc_cfg_sts);
6179 }
6180 
6181 static bool gaudi2_is_device_idle(struct hl_device *hdev, u64 *mask_arr,
6182                     u8 mask_len, struct seq_file *s)
6183 {
6184     u32 qm_glbl_sts0, qm_glbl_sts1, qm_cgm_sts, dma_core_idle_ind_mask,
6185         mme_arch_sts, dec_swreg15, dec_enabled_bit;
6186     struct asic_fixed_properties *prop = &hdev->asic_prop;
6187     const char *rot_fmt = "%-6d%-5d%-9s%#-14x%#-12x%s\n";
6188     unsigned long *mask = (unsigned long *) mask_arr;
6189     const char *edma_fmt = "%-6d%-6d%-9s%#-14x%#x\n";
6190     const char *mme_fmt = "%-5d%-6s%-9s%#-14x%#x\n";
6191     const char *nic_fmt = "%-5d%-9s%#-14x%#-12x\n";
6192     const char *pdma_fmt = "%-6d%-9s%#-14x%#x\n";
6193     const char *pcie_dec_fmt = "%-10d%-9s%#x\n";
6194     const char *dec_fmt = "%-6d%-5d%-9s%#x\n";
6195     bool is_idle = true, is_eng_idle;
6196     u64 offset;
6197 
6198     struct gaudi2_tpc_idle_data tpc_idle_data = {
6199         .tpc_fmt = "%-6d%-5d%-9s%#-14x%#-12x%#x\n",
6200         .s = s,
6201         .mask = mask,
6202         .is_idle = &is_idle,
6203     };
6204     struct iterate_module_ctx tpc_iter = {
6205         .fn = &gaudi2_is_tpc_engine_idle,
6206         .data = &tpc_idle_data,
6207     };
6208 
6209     int engine_idx, i, j;
6210 
6211     /* EDMA, Two engines per Dcore */
6212     if (s)
6213         seq_puts(s,
6214             "\nCORE  EDMA  is_idle  QM_GLBL_STS0  DMA_CORE_IDLE_IND_MASK\n"
6215             "----  ----  -------  ------------  ----------------------\n");
6216 
6217     for (i = 0; i < NUM_OF_DCORES; i++) {
6218         for (j = 0 ; j < NUM_OF_EDMA_PER_DCORE ; j++) {
6219             int seq = i * NUM_OF_EDMA_PER_DCORE + j;
6220 
6221             if (!(prop->edma_enabled_mask & BIT(seq)))
6222                 continue;
6223 
6224             engine_idx = GAUDI2_DCORE0_ENGINE_ID_EDMA_0 +
6225                     i * GAUDI2_ENGINE_ID_DCORE_OFFSET + j;
6226             offset = i * DCORE_OFFSET + j * DCORE_EDMA_OFFSET;
6227 
6228             dma_core_idle_ind_mask =
6229             RREG32(mmDCORE0_EDMA0_CORE_IDLE_IND_MASK + offset);
6230 
6231             qm_glbl_sts0 = RREG32(mmDCORE0_EDMA0_QM_GLBL_STS0 + offset);
6232             qm_glbl_sts1 = RREG32(mmDCORE0_EDMA0_QM_GLBL_STS1 + offset);
6233             qm_cgm_sts = RREG32(mmDCORE0_EDMA0_QM_CGM_STS + offset);
6234 
6235             is_eng_idle = IS_QM_IDLE(qm_glbl_sts0, qm_glbl_sts1, qm_cgm_sts) &&
6236                     IS_DMA_IDLE(dma_core_idle_ind_mask);
6237             is_idle &= is_eng_idle;
6238 
6239             if (mask && !is_eng_idle)
6240                 set_bit(engine_idx, mask);
6241 
6242             if (s)
6243                 seq_printf(s, edma_fmt, i, j,
6244                         is_eng_idle ? "Y" : "N",
6245                         qm_glbl_sts0,
6246                         dma_core_idle_ind_mask);
6247         }
6248     }
6249 
6250     /* PDMA, Two engines in Full chip */
6251     if (s)
6252         seq_puts(s,
6253             "\nPDMA  is_idle  QM_GLBL_STS0  DMA_CORE_IDLE_IND_MASK\n"
6254             "----  -------  ------------  ----------------------\n");
6255 
6256     for (i = 0 ; i < NUM_OF_PDMA ; i++) {
6257         engine_idx = GAUDI2_ENGINE_ID_PDMA_0 + i;
6258         offset = i * PDMA_OFFSET;
6259         dma_core_idle_ind_mask = RREG32(mmPDMA0_CORE_IDLE_IND_MASK + offset);
6260 
6261         qm_glbl_sts0 = RREG32(mmPDMA0_QM_GLBL_STS0 + offset);
6262         qm_glbl_sts1 = RREG32(mmPDMA0_QM_GLBL_STS1 + offset);
6263         qm_cgm_sts = RREG32(mmPDMA0_QM_CGM_STS + offset);
6264 
6265         is_eng_idle = IS_QM_IDLE(qm_glbl_sts0, qm_glbl_sts1, qm_cgm_sts) &&
6266                 IS_DMA_IDLE(dma_core_idle_ind_mask);
6267         is_idle &= is_eng_idle;
6268 
6269         if (mask && !is_eng_idle)
6270             set_bit(engine_idx, mask);
6271 
6272         if (s)
6273             seq_printf(s, pdma_fmt, i, is_eng_idle ? "Y" : "N", qm_glbl_sts0,
6274                     dma_core_idle_ind_mask);
6275     }
6276 
6277     /* NIC, twelve macros in Full chip */
6278     if (s && hdev->nic_ports_mask)
6279         seq_puts(s,
6280             "\nNIC  is_idle  QM_GLBL_STS0  QM_CGM_STS\n"
6281             "---  -------  ------------  ----------\n");
6282 
6283     for (i = 0 ; i < NIC_NUMBER_OF_ENGINES ; i++) {
6284         if (!(i & 1))
6285             offset = i / 2 * NIC_OFFSET;
6286         else
6287             offset += NIC_QM_OFFSET;
6288 
6289         if (!(hdev->nic_ports_mask & BIT(i)))
6290             continue;
6291 
6292         engine_idx = GAUDI2_ENGINE_ID_NIC0_0 + i;
6293 
6294 
6295         qm_glbl_sts0 = RREG32(mmNIC0_QM0_GLBL_STS0 + offset);
6296         qm_glbl_sts1 = RREG32(mmNIC0_QM0_GLBL_STS1 + offset);
6297         qm_cgm_sts = RREG32(mmNIC0_QM0_CGM_STS + offset);
6298 
6299         is_eng_idle = IS_QM_IDLE(qm_glbl_sts0, qm_glbl_sts1, qm_cgm_sts);
6300         is_idle &= is_eng_idle;
6301 
6302         if (mask && !is_eng_idle)
6303             set_bit(engine_idx, mask);
6304 
6305         if (s)
6306             seq_printf(s, nic_fmt, i, is_eng_idle ? "Y" : "N", qm_glbl_sts0,
6307                     qm_cgm_sts);
6308     }
6309 
6310     if (s)
6311         seq_puts(s,
6312             "\nMME  Stub  is_idle  QM_GLBL_STS0  MME_ARCH_STATUS\n"
6313             "---  ----  -------  ------------  ---------------\n");
6314     /* MME, one per Dcore */
6315     for (i = 0 ; i < NUM_OF_DCORES ; i++) {
6316         engine_idx = GAUDI2_DCORE0_ENGINE_ID_MME + i * GAUDI2_ENGINE_ID_DCORE_OFFSET;
6317         offset = i * DCORE_OFFSET;
6318 
6319         qm_glbl_sts0 = RREG32(mmDCORE0_MME_QM_GLBL_STS0 + offset);
6320         qm_glbl_sts1 = RREG32(mmDCORE0_MME_QM_GLBL_STS1 + offset);
6321         qm_cgm_sts = RREG32(mmDCORE0_MME_QM_CGM_STS + offset);
6322 
6323         is_eng_idle = IS_QM_IDLE(qm_glbl_sts0, qm_glbl_sts1, qm_cgm_sts);
6324         is_idle &= is_eng_idle;
6325 
6326         mme_arch_sts = RREG32(mmDCORE0_MME_CTRL_LO_ARCH_STATUS + offset);
6327         is_eng_idle &= IS_MME_IDLE(mme_arch_sts);
6328         is_idle &= is_eng_idle;
6329 
6330         if (s)
6331             seq_printf(s, mme_fmt, i, "N",
6332                 is_eng_idle ? "Y" : "N",
6333                 qm_glbl_sts0,
6334                 mme_arch_sts);
6335 
6336         if (mask && !is_eng_idle)
6337             set_bit(engine_idx, mask);
6338     }
6339 
6340     /*
6341      * TPC
6342      */
6343     if (s && prop->tpc_enabled_mask)
6344         seq_puts(s,
6345             "\nCORE  TPC   is_idle  QM_GLBL_STS0  QM_CGM_STS  DMA_CORE_IDLE_IND_MASK\n"
6346             "----  ---  --------  ------------  ----------  ----------------------\n");
6347 
6348     gaudi2_iterate_tpcs(hdev, &tpc_iter);
6349 
6350     /* Decoders, two each Dcore and two shared PCIe decoders */
6351     if (s && (prop->decoder_enabled_mask & (~PCIE_DEC_EN_MASK)))
6352         seq_puts(s,
6353             "\nCORE  DEC  is_idle  VSI_CMD_SWREG15\n"
6354             "----  ---  -------  ---------------\n");
6355 
6356     for (i = 0 ; i < NUM_OF_DCORES ; i++) {
6357         for (j = 0 ; j < NUM_OF_DEC_PER_DCORE ; j++) {
6358             dec_enabled_bit = 1 << (i * NUM_OF_DEC_PER_DCORE + j);
6359             if (!(prop->decoder_enabled_mask & dec_enabled_bit))
6360                 continue;
6361 
6362             engine_idx = GAUDI2_DCORE0_ENGINE_ID_DEC_0 +
6363                     i * GAUDI2_ENGINE_ID_DCORE_OFFSET + j;
6364             offset = i * DCORE_OFFSET + j * DCORE_DEC_OFFSET;
6365 
6366             dec_swreg15 = RREG32(mmDCORE0_DEC0_CMD_SWREG15 + offset);
6367             is_eng_idle = IS_DEC_IDLE(dec_swreg15);
6368             is_idle &= is_eng_idle;
6369 
6370             if (mask && !is_eng_idle)
6371                 set_bit(engine_idx, mask);
6372 
6373             if (s)
6374                 seq_printf(s, dec_fmt, i, j, is_eng_idle ? "Y" : "N", dec_swreg15);
6375         }
6376     }
6377 
6378     if (s && (prop->decoder_enabled_mask & PCIE_DEC_EN_MASK))
6379         seq_puts(s,
6380             "\nPCIe DEC  is_idle  VSI_CMD_SWREG15\n"
6381             "--------  -------  ---------------\n");
6382 
6383     /* Check shared(PCIe) decoders */
6384     for (i = 0 ; i < NUM_OF_DEC_PER_DCORE ; i++) {
6385         dec_enabled_bit = PCIE_DEC_SHIFT + i;
6386         if (!(prop->decoder_enabled_mask & BIT(dec_enabled_bit)))
6387             continue;
6388 
6389         engine_idx = GAUDI2_PCIE_ENGINE_ID_DEC_0 + i;
6390         offset = i * DCORE_DEC_OFFSET;
6391         dec_swreg15 = RREG32(mmPCIE_DEC0_CMD_SWREG15 + offset);
6392         is_eng_idle = IS_DEC_IDLE(dec_swreg15);
6393         is_idle &= is_eng_idle;
6394 
6395         if (mask && !is_eng_idle)
6396             set_bit(engine_idx, mask);
6397 
6398         if (s)
6399             seq_printf(s, pcie_dec_fmt, i, is_eng_idle ? "Y" : "N", dec_swreg15);
6400     }
6401 
6402     if (s)
6403         seq_puts(s,
6404             "\nCORE  ROT  is_idle  QM_GLBL_STS0  QM_CGM_STS  DMA_CORE_STS0\n"
6405             "----  ----  -------  ------------  ----------  -------------\n");
6406 
6407     for (i = 0 ; i < NUM_OF_ROT ; i++) {
6408         engine_idx = GAUDI2_ENGINE_ID_ROT_0 + i;
6409 
6410         offset = i * ROT_OFFSET;
6411 
6412         qm_glbl_sts0 = RREG32(mmROT0_QM_GLBL_STS0 + offset);
6413         qm_glbl_sts1 = RREG32(mmROT0_QM_GLBL_STS1 + offset);
6414         qm_cgm_sts = RREG32(mmROT0_QM_CGM_STS + offset);
6415 
6416         is_eng_idle = IS_QM_IDLE(qm_glbl_sts0, qm_glbl_sts1, qm_cgm_sts);
6417         is_idle &= is_eng_idle;
6418 
6419         if (mask && !is_eng_idle)
6420             set_bit(engine_idx, mask);
6421 
6422         if (s)
6423             seq_printf(s, rot_fmt, i, 0, is_eng_idle ? "Y" : "N",
6424                     qm_glbl_sts0, qm_cgm_sts, "-");
6425     }
6426 
6427     return is_idle;
6428 }
6429 
6430 static void gaudi2_hw_queues_lock(struct hl_device *hdev)
6431     __acquires(&gaudi2->hw_queues_lock)
6432 {
6433     struct gaudi2_device *gaudi2 = hdev->asic_specific;
6434 
6435     spin_lock(&gaudi2->hw_queues_lock);
6436 }
6437 
6438 static void gaudi2_hw_queues_unlock(struct hl_device *hdev)
6439     __releases(&gaudi2->hw_queues_lock)
6440 {
6441     struct gaudi2_device *gaudi2 = hdev->asic_specific;
6442 
6443     spin_unlock(&gaudi2->hw_queues_lock);
6444 }
6445 
6446 static void gaudi2_kdma_lock(struct hl_device *hdev, int dcore_id)
6447     __acquires(&gaudi2->kdma_lock)
6448 {
6449     struct gaudi2_device *gaudi2 = hdev->asic_specific;
6450 
6451     spin_lock(&gaudi2->kdma_lock);
6452 }
6453 
6454 static void gaudi2_kdma_unlock(struct hl_device *hdev, int dcore_id)
6455     __releases(&gaudi2->kdma_lock)
6456 {
6457     struct gaudi2_device *gaudi2 = hdev->asic_specific;
6458 
6459     spin_unlock(&gaudi2->kdma_lock);
6460 }
6461 
6462 static u32 gaudi2_get_pci_id(struct hl_device *hdev)
6463 {
6464     return hdev->pdev->device;
6465 }
6466 
6467 static int gaudi2_get_eeprom_data(struct hl_device *hdev, void *data, size_t max_size)
6468 {
6469     struct gaudi2_device *gaudi2 = hdev->asic_specific;
6470 
6471     if (!(gaudi2->hw_cap_initialized & HW_CAP_CPU_Q))
6472         return 0;
6473 
6474     return hl_fw_get_eeprom_data(hdev, data, max_size);
6475 }
6476 
6477 static void gaudi2_update_eq_ci(struct hl_device *hdev, u32 val)
6478 {
6479     WREG32(mmCPU_IF_EQ_RD_OFFS, val);
6480 }
6481 
6482 static void *gaudi2_get_events_stat(struct hl_device *hdev, bool aggregate, u32 *size)
6483 {
6484     struct gaudi2_device *gaudi2 = hdev->asic_specific;
6485 
6486     if (aggregate) {
6487         *size = (u32) sizeof(gaudi2->events_stat_aggregate);
6488         return gaudi2->events_stat_aggregate;
6489     }
6490 
6491     *size = (u32) sizeof(gaudi2->events_stat);
6492     return gaudi2->events_stat;
6493 }
6494 
6495 static void gaudi2_mmu_vdec_dcore_prepare(struct hl_device *hdev, int dcore_id,
6496                 int dcore_vdec_id, u32 rw_asid, u32 rw_mmu_bp)
6497 {
6498     u32 offset = (mmDCORE0_VDEC1_BRDG_CTRL_BASE - mmDCORE0_VDEC0_BRDG_CTRL_BASE) *
6499             dcore_vdec_id + DCORE_OFFSET * dcore_id;
6500 
6501     WREG32(mmDCORE0_VDEC0_BRDG_CTRL_AXUSER_DEC_HB_MMU_BP + offset, rw_mmu_bp);
6502     WREG32(mmDCORE0_VDEC0_BRDG_CTRL_AXUSER_DEC_HB_ASID + offset, rw_asid);
6503 
6504     WREG32(mmDCORE0_VDEC0_BRDG_CTRL_AXUSER_MSIX_ABNRM_HB_MMU_BP + offset, rw_mmu_bp);
6505     WREG32(mmDCORE0_VDEC0_BRDG_CTRL_AXUSER_MSIX_ABNRM_HB_ASID + offset, rw_asid);
6506 
6507     WREG32(mmDCORE0_VDEC0_BRDG_CTRL_AXUSER_MSIX_L2C_HB_MMU_BP + offset, rw_mmu_bp);
6508     WREG32(mmDCORE0_VDEC0_BRDG_CTRL_AXUSER_MSIX_L2C_HB_ASID + offset, rw_asid);
6509 
6510     WREG32(mmDCORE0_VDEC0_BRDG_CTRL_AXUSER_MSIX_NRM_HB_MMU_BP + offset, rw_mmu_bp);
6511     WREG32(mmDCORE0_VDEC0_BRDG_CTRL_AXUSER_MSIX_NRM_HB_ASID + offset, rw_asid);
6512 
6513     WREG32(mmDCORE0_VDEC0_BRDG_CTRL_AXUSER_MSIX_VCD_HB_MMU_BP + offset, rw_mmu_bp);
6514     WREG32(mmDCORE0_VDEC0_BRDG_CTRL_AXUSER_MSIX_VCD_HB_ASID + offset, rw_asid);
6515 }
6516 
6517 static void gaudi2_mmu_dcore_prepare(struct hl_device *hdev, int dcore_id, u32 asid)
6518 {
6519     u32 rw_asid = (asid << ARC_FARM_KDMA_CTX_AXUSER_HB_ASID_RD_SHIFT) |
6520             (asid << ARC_FARM_KDMA_CTX_AXUSER_HB_ASID_WR_SHIFT);
6521     struct asic_fixed_properties *prop = &hdev->asic_prop;
6522     u32 dcore_offset = dcore_id * DCORE_OFFSET;
6523     u32 vdec_id, i, ports_offset, reg_val;
6524     u8 edma_seq_base;
6525 
6526     /* EDMA */
6527     edma_seq_base = dcore_id * NUM_OF_EDMA_PER_DCORE;
6528     if (prop->edma_enabled_mask & BIT(edma_seq_base)) {
6529         WREG32(mmDCORE0_EDMA0_QM_AXUSER_NONSECURED_HB_MMU_BP + dcore_offset, 0);
6530         WREG32(mmDCORE0_EDMA0_QM_AXUSER_NONSECURED_HB_ASID + dcore_offset, rw_asid);
6531         WREG32(mmDCORE0_EDMA0_CORE_CTX_AXUSER_HB_MMU_BP + dcore_offset, 0);
6532         WREG32(mmDCORE0_EDMA0_CORE_CTX_AXUSER_HB_ASID + dcore_offset, rw_asid);
6533     }
6534 
6535     if (prop->edma_enabled_mask & BIT(edma_seq_base + 1)) {
6536         WREG32(mmDCORE0_EDMA1_QM_AXUSER_NONSECURED_HB_MMU_BP + dcore_offset, 0);
6537         WREG32(mmDCORE0_EDMA1_QM_AXUSER_NONSECURED_HB_ASID + dcore_offset, rw_asid);
6538         WREG32(mmDCORE0_EDMA1_CORE_CTX_AXUSER_HB_ASID + dcore_offset, rw_asid);
6539         WREG32(mmDCORE0_EDMA1_CORE_CTX_AXUSER_HB_MMU_BP + dcore_offset, 0);
6540     }
6541 
6542     /* Sync Mngr */
6543     WREG32(mmDCORE0_SYNC_MNGR_GLBL_ASID_NONE_SEC_PRIV + dcore_offset, asid);
6544     /*
6545      * Sync Mngrs on dcores 1 - 3 are exposed to user, so must use user ASID
6546      * for any access type
6547      */
6548     if (dcore_id > 0) {
6549         reg_val = (asid << DCORE0_SYNC_MNGR_MSTR_IF_AXUSER_HB_ASID_RD_SHIFT) |
6550               (asid << DCORE0_SYNC_MNGR_MSTR_IF_AXUSER_HB_ASID_WR_SHIFT);
6551         WREG32(mmDCORE0_SYNC_MNGR_MSTR_IF_AXUSER_HB_ASID + dcore_offset, reg_val);
6552         WREG32(mmDCORE0_SYNC_MNGR_MSTR_IF_AXUSER_HB_MMU_BP + dcore_offset, 0);
6553     }
6554 
6555     WREG32(mmDCORE0_MME_CTRL_LO_MME_AXUSER_HB_MMU_BP + dcore_offset, 0);
6556     WREG32(mmDCORE0_MME_CTRL_LO_MME_AXUSER_HB_ASID + dcore_offset, rw_asid);
6557 
6558     for (i = 0 ; i < NUM_OF_MME_SBTE_PORTS ; i++) {
6559         ports_offset = i * DCORE_MME_SBTE_OFFSET;
6560         WREG32(mmDCORE0_MME_SBTE0_MSTR_IF_AXUSER_HB_MMU_BP +
6561                 dcore_offset + ports_offset, 0);
6562         WREG32(mmDCORE0_MME_SBTE0_MSTR_IF_AXUSER_HB_ASID +
6563                 dcore_offset + ports_offset, rw_asid);
6564     }
6565 
6566     for (i = 0 ; i < NUM_OF_MME_WB_PORTS ; i++) {
6567         ports_offset = i * DCORE_MME_WB_OFFSET;
6568         WREG32(mmDCORE0_MME_WB0_MSTR_IF_AXUSER_HB_MMU_BP +
6569                 dcore_offset + ports_offset, 0);
6570         WREG32(mmDCORE0_MME_WB0_MSTR_IF_AXUSER_HB_ASID +
6571                 dcore_offset + ports_offset, rw_asid);
6572     }
6573 
6574     WREG32(mmDCORE0_MME_QM_AXUSER_NONSECURED_HB_MMU_BP + dcore_offset, 0);
6575     WREG32(mmDCORE0_MME_QM_AXUSER_NONSECURED_HB_ASID + dcore_offset, rw_asid);
6576 
6577     /*
6578      * Decoders
6579      */
6580     for (vdec_id = 0 ; vdec_id < NUM_OF_DEC_PER_DCORE ; vdec_id++) {
6581         if (prop->decoder_enabled_mask & BIT(dcore_id * NUM_OF_DEC_PER_DCORE + vdec_id))
6582             gaudi2_mmu_vdec_dcore_prepare(hdev, dcore_id, vdec_id, rw_asid, 0);
6583     }
6584 }
6585 
6586 static void gudi2_mmu_vdec_shared_prepare(struct hl_device *hdev,
6587                 int shared_vdec_id, u32 rw_asid, u32 rw_mmu_bp)
6588 {
6589     u32 offset = (mmPCIE_VDEC1_BRDG_CTRL_BASE - mmPCIE_VDEC0_BRDG_CTRL_BASE) * shared_vdec_id;
6590 
6591     WREG32(mmPCIE_VDEC0_BRDG_CTRL_AXUSER_DEC_HB_MMU_BP + offset, rw_mmu_bp);
6592     WREG32(mmPCIE_VDEC0_BRDG_CTRL_AXUSER_DEC_HB_ASID + offset, rw_asid);
6593 
6594     WREG32(mmPCIE_VDEC0_BRDG_CTRL_AXUSER_MSIX_ABNRM_HB_MMU_BP + offset, rw_mmu_bp);
6595     WREG32(mmPCIE_VDEC0_BRDG_CTRL_AXUSER_MSIX_ABNRM_HB_ASID + offset, rw_asid);
6596 
6597     WREG32(mmPCIE_VDEC0_BRDG_CTRL_AXUSER_MSIX_L2C_HB_MMU_BP + offset, rw_mmu_bp);
6598     WREG32(mmPCIE_VDEC0_BRDG_CTRL_AXUSER_MSIX_L2C_HB_ASID + offset, rw_asid);
6599 
6600     WREG32(mmPCIE_VDEC0_BRDG_CTRL_AXUSER_MSIX_NRM_HB_MMU_BP + offset, rw_mmu_bp);
6601     WREG32(mmPCIE_VDEC0_BRDG_CTRL_AXUSER_MSIX_NRM_HB_ASID + offset, rw_asid);
6602 
6603     WREG32(mmPCIE_VDEC0_BRDG_CTRL_AXUSER_MSIX_VCD_HB_MMU_BP + offset, rw_mmu_bp);
6604     WREG32(mmPCIE_VDEC0_BRDG_CTRL_AXUSER_MSIX_VCD_HB_ASID + offset, rw_asid);
6605 }
6606 
6607 static void gudi2_mmu_arc_farm_arc_dup_eng_prepare(struct hl_device *hdev, int arc_farm_id,
6608                             u32 rw_asid, u32 rw_mmu_bp)
6609 {
6610     u32 offset = (mmARC_FARM_ARC1_DUP_ENG_BASE - mmARC_FARM_ARC0_DUP_ENG_BASE) * arc_farm_id;
6611 
6612     WREG32(mmARC_FARM_ARC0_DUP_ENG_AXUSER_HB_MMU_BP + offset, rw_mmu_bp);
6613     WREG32(mmARC_FARM_ARC0_DUP_ENG_AXUSER_HB_ASID + offset, rw_asid);
6614 }
6615 
6616 static void gaudi2_arc_mmu_prepare(struct hl_device *hdev, u32 cpu_id, u32 asid)
6617 {
6618     u32 reg_base, reg_offset, reg_val = 0;
6619 
6620     reg_base = gaudi2_arc_blocks_bases[cpu_id];
6621 
6622     /* Enable MMU and configure asid for all relevant ARC regions */
6623     reg_val = FIELD_PREP(ARC_FARM_ARC0_AUX_ARC_REGION_CFG_MMU_BP_MASK, 0);
6624     reg_val |= FIELD_PREP(ARC_FARM_ARC0_AUX_ARC_REGION_CFG_0_ASID_MASK, asid);
6625 
6626     reg_offset = ARC_REGION_CFG_OFFSET(ARC_REGION3_GENERAL);
6627     WREG32(reg_base + reg_offset, reg_val);
6628 
6629     reg_offset = ARC_REGION_CFG_OFFSET(ARC_REGION4_HBM0_FW);
6630     WREG32(reg_base + reg_offset, reg_val);
6631 
6632     reg_offset = ARC_REGION_CFG_OFFSET(ARC_REGION5_HBM1_GC_DATA);
6633     WREG32(reg_base + reg_offset, reg_val);
6634 
6635     reg_offset = ARC_REGION_CFG_OFFSET(ARC_REGION6_HBM2_GC_DATA);
6636     WREG32(reg_base + reg_offset, reg_val);
6637 
6638     reg_offset = ARC_REGION_CFG_OFFSET(ARC_REGION7_HBM3_GC_DATA);
6639     WREG32(reg_base + reg_offset, reg_val);
6640 
6641     reg_offset = ARC_REGION_CFG_OFFSET(ARC_REGION9_PCIE);
6642     WREG32(reg_base + reg_offset, reg_val);
6643 
6644     reg_offset = ARC_REGION_CFG_OFFSET(ARC_REGION10_GENERAL);
6645     WREG32(reg_base + reg_offset, reg_val);
6646 
6647     reg_offset = ARC_REGION_CFG_OFFSET(ARC_REGION11_GENERAL);
6648     WREG32(reg_base + reg_offset, reg_val);
6649 
6650     reg_offset = ARC_REGION_CFG_OFFSET(ARC_REGION12_GENERAL);
6651     WREG32(reg_base + reg_offset, reg_val);
6652 
6653     reg_offset = ARC_REGION_CFG_OFFSET(ARC_REGION13_GENERAL);
6654     WREG32(reg_base + reg_offset, reg_val);
6655 
6656     reg_offset = ARC_REGION_CFG_OFFSET(ARC_REGION14_GENERAL);
6657     WREG32(reg_base + reg_offset, reg_val);
6658 }
6659 
6660 static int gaudi2_arc_mmu_prepare_all(struct hl_device *hdev, u32 asid)
6661 {
6662     int i;
6663 
6664     if (hdev->fw_components & FW_TYPE_BOOT_CPU)
6665         return hl_fw_cpucp_engine_core_asid_set(hdev, asid);
6666 
6667     for (i = CPU_ID_SCHED_ARC0 ; i < NUM_OF_ARC_FARMS_ARC ; i++)
6668         gaudi2_arc_mmu_prepare(hdev, i, asid);
6669 
6670     for (i = GAUDI2_QUEUE_ID_PDMA_0_0 ; i < GAUDI2_QUEUE_ID_CPU_PQ ; i += 4) {
6671         if (!gaudi2_is_queue_enabled(hdev, i))
6672             continue;
6673 
6674         gaudi2_arc_mmu_prepare(hdev, gaudi2_queue_id_to_arc_id[i], asid);
6675     }
6676 
6677     return 0;
6678 }
6679 
6680 static int gaudi2_mmu_shared_prepare(struct hl_device *hdev, u32 asid)
6681 {
6682     struct asic_fixed_properties *prop = &hdev->asic_prop;
6683     u32 rw_asid, offset;
6684     int rc, i;
6685 
6686     rw_asid = FIELD_PREP(ARC_FARM_KDMA_CTX_AXUSER_HB_ASID_RD_MASK, asid) |
6687             FIELD_PREP(ARC_FARM_KDMA_CTX_AXUSER_HB_ASID_WR_MASK, asid);
6688 
6689     WREG32(mmPDMA0_QM_AXUSER_NONSECURED_HB_ASID, rw_asid);
6690     WREG32(mmPDMA0_QM_AXUSER_NONSECURED_HB_MMU_BP, 0);
6691     WREG32(mmPDMA0_CORE_CTX_AXUSER_HB_ASID, rw_asid);
6692     WREG32(mmPDMA0_CORE_CTX_AXUSER_HB_MMU_BP, 0);
6693 
6694     WREG32(mmPDMA1_QM_AXUSER_NONSECURED_HB_ASID, rw_asid);
6695     WREG32(mmPDMA1_QM_AXUSER_NONSECURED_HB_MMU_BP, 0);
6696     WREG32(mmPDMA1_CORE_CTX_AXUSER_HB_ASID, rw_asid);
6697     WREG32(mmPDMA1_CORE_CTX_AXUSER_HB_MMU_BP, 0);
6698 
6699     /* ROT */
6700     for (i = 0 ; i < NUM_OF_ROT ; i++) {
6701         offset = i * ROT_OFFSET;
6702         WREG32(mmROT0_QM_AXUSER_NONSECURED_HB_ASID + offset, rw_asid);
6703         WREG32(mmROT0_QM_AXUSER_NONSECURED_HB_MMU_BP + offset, 0);
6704         RMWREG32(mmROT0_CPL_QUEUE_AWUSER + offset, asid, MMUBP_ASID_MASK);
6705         RMWREG32(mmROT0_DESC_HBW_ARUSER_LO + offset, asid, MMUBP_ASID_MASK);
6706         RMWREG32(mmROT0_DESC_HBW_AWUSER_LO + offset, asid, MMUBP_ASID_MASK);
6707     }
6708 
6709     /* Shared Decoders are the last bits in the decoders mask */
6710     if (prop->decoder_enabled_mask & BIT(NUM_OF_DCORES * NUM_OF_DEC_PER_DCORE + 0))
6711         gudi2_mmu_vdec_shared_prepare(hdev, 0, rw_asid, 0);
6712 
6713     if (prop->decoder_enabled_mask & BIT(NUM_OF_DCORES * NUM_OF_DEC_PER_DCORE + 1))
6714         gudi2_mmu_vdec_shared_prepare(hdev, 1, rw_asid, 0);
6715 
6716     /* arc farm arc dup eng */
6717     for (i = 0 ; i < NUM_OF_ARC_FARMS_ARC ; i++)
6718         gudi2_mmu_arc_farm_arc_dup_eng_prepare(hdev, i, rw_asid, 0);
6719 
6720     rc = gaudi2_arc_mmu_prepare_all(hdev, asid);
6721     if (rc)
6722         return rc;
6723 
6724     return 0;
6725 }
6726 
6727 static void gaudi2_tpc_mmu_prepare(struct hl_device *hdev, int dcore, int inst, u32 offset,
6728                     void *data)
6729 {
6730     struct gaudi2_tpc_mmu_data *mmu_data = (struct gaudi2_tpc_mmu_data *)data;
6731 
6732     WREG32(mmDCORE0_TPC0_CFG_AXUSER_HB_MMU_BP + offset, 0);
6733     WREG32(mmDCORE0_TPC0_CFG_AXUSER_HB_ASID + offset, mmu_data->rw_asid);
6734     WREG32(mmDCORE0_TPC0_QM_AXUSER_NONSECURED_HB_MMU_BP + offset, 0);
6735     WREG32(mmDCORE0_TPC0_QM_AXUSER_NONSECURED_HB_ASID + offset, mmu_data->rw_asid);
6736 }
6737 
6738 /* zero the MMUBP and set the ASID */
6739 static int gaudi2_mmu_prepare(struct hl_device *hdev, u32 asid)
6740 {
6741     struct gaudi2_device *gaudi2 = hdev->asic_specific;
6742     struct gaudi2_tpc_mmu_data tpc_mmu_data;
6743     struct iterate_module_ctx tpc_iter = {
6744         .fn = &gaudi2_tpc_mmu_prepare,
6745         .data = &tpc_mmu_data,
6746     };
6747     int rc, i;
6748 
6749     if (asid & ~DCORE0_HMMU0_STLB_ASID_ASID_MASK) {
6750         dev_crit(hdev->dev, "asid %u is too big\n", asid);
6751         return -EINVAL;
6752     }
6753 
6754     if (!(gaudi2->hw_cap_initialized & HW_CAP_MMU_MASK))
6755         return 0;
6756 
6757     rc = gaudi2_mmu_shared_prepare(hdev, asid);
6758     if (rc)
6759         return rc;
6760 
6761     /* configure DCORE MMUs */
6762     tpc_mmu_data.rw_asid = (asid << ARC_FARM_KDMA_CTX_AXUSER_HB_ASID_RD_SHIFT) |
6763                 (asid << ARC_FARM_KDMA_CTX_AXUSER_HB_ASID_WR_SHIFT);
6764     gaudi2_iterate_tpcs(hdev, &tpc_iter);
6765     for (i = 0 ; i < NUM_OF_DCORES ; i++)
6766         gaudi2_mmu_dcore_prepare(hdev, i, asid);
6767 
6768     return 0;
6769 }
6770 
6771 static inline bool is_info_event(u32 event)
6772 {
6773     switch (event) {
6774     case GAUDI2_EVENT_CPU_CPLD_SHUTDOWN_CAUSE:
6775         return true;
6776     default:
6777         return false;
6778     }
6779 }
6780 
6781 static void gaudi2_print_irq_info(struct hl_device *hdev, u16 event_type)
6782 {
6783     char desc[64] = "";
6784     bool event_valid = false;
6785 
6786     /* return in case of NIC status event - these events are received periodically and not as
6787      * an indication to an error, thus not printed.
6788      */
6789     if (event_type >= GAUDI2_EVENT_CPU0_STATUS_NIC0_ENG0 &&
6790         event_type <= GAUDI2_EVENT_CPU11_STATUS_NIC11_ENG1)
6791         return;
6792 
6793     if (gaudi2_irq_map_table[event_type].valid) {
6794         snprintf(desc, sizeof(desc), gaudi2_irq_map_table[event_type].name);
6795         event_valid = true;
6796     }
6797 
6798     if (!event_valid)
6799         snprintf(desc, sizeof(desc), "N/A");
6800 
6801     if (is_info_event(event_type))
6802         dev_info_ratelimited(hdev->dev, "Received H/W interrupt %d [\"%s\"]\n",
6803                                 event_type, desc);
6804     else
6805         dev_err_ratelimited(hdev->dev, "Received H/W interrupt %d [\"%s\"]\n",
6806                                 event_type, desc);
6807 }
6808 
6809 static bool gaudi2_handle_ecc_event(struct hl_device *hdev, u16 event_type,
6810         struct hl_eq_ecc_data *ecc_data)
6811 {
6812     u64 ecc_address = 0, ecc_syndrom = 0;
6813     u8 memory_wrapper_idx = 0;
6814 
6815     ecc_address = le64_to_cpu(ecc_data->ecc_address);
6816     ecc_syndrom = le64_to_cpu(ecc_data->ecc_syndrom);
6817     memory_wrapper_idx = ecc_data->memory_wrapper_idx;
6818 
6819     dev_err(hdev->dev,
6820         "ECC error detected. address: %#llx. Syndrom: %#llx. block id %u. critical %u.\n",
6821         ecc_address, ecc_syndrom, memory_wrapper_idx, ecc_data->is_critical);
6822 
6823     return !!ecc_data->is_critical;
6824 }
6825 
6826 /*
6827  * gaudi2_queue_idx_dec - decrement queue index (pi/ci) and handle wrap
6828  *
6829  * @idx: the current pi/ci value
6830  * @q_len: the queue length (power of 2)
6831  *
6832  * @return the cyclically decremented index
6833  */
6834 static inline u32 gaudi2_queue_idx_dec(u32 idx, u32 q_len)
6835 {
6836     u32 mask = q_len - 1;
6837 
6838     /*
6839      * modular decrement is equivalent to adding (queue_size -1)
6840      * later we take LSBs to make sure the value is in the
6841      * range [0, queue_len - 1]
6842      */
6843     return (idx + q_len - 1) & mask;
6844 }
6845 
6846 /**
6847  * gaudi2_print_sw_config_stream_data - print SW config stream data
6848  *
6849  * @hdev: pointer to the habanalabs device structure
6850  * @stream: the QMAN's stream
6851  * @qman_base: base address of QMAN registers block
6852  */
6853 static void gaudi2_print_sw_config_stream_data(struct hl_device *hdev,
6854                         u32 stream, u64 qman_base)
6855 {
6856     u64 cq_ptr_lo, cq_ptr_hi, cq_tsize, cq_ptr;
6857     u32 cq_ptr_lo_off, size;
6858 
6859     cq_ptr_lo_off = mmDCORE0_TPC0_QM_CQ_PTR_LO_1 - mmDCORE0_TPC0_QM_CQ_PTR_LO_0;
6860 
6861     cq_ptr_lo = qman_base + (mmDCORE0_TPC0_QM_CQ_PTR_LO_0 - mmDCORE0_TPC0_QM_BASE) +
6862                                     stream * cq_ptr_lo_off;
6863 
6864     cq_ptr_hi = cq_ptr_lo + (mmDCORE0_TPC0_QM_CQ_PTR_HI_0 - mmDCORE0_TPC0_QM_CQ_PTR_LO_0);
6865 
6866     cq_tsize = cq_ptr_lo + (mmDCORE0_TPC0_QM_CQ_TSIZE_0 - mmDCORE0_TPC0_QM_CQ_PTR_LO_0);
6867 
6868     cq_ptr = (((u64) RREG32(cq_ptr_hi)) << 32) | RREG32(cq_ptr_lo);
6869     size = RREG32(cq_tsize);
6870     dev_info(hdev->dev, "stop on err: stream: %u, addr: %#llx, size: %x\n",
6871         stream, cq_ptr, size);
6872 }
6873 
6874 /**
6875  * gaudi2_print_last_pqes_on_err - print last PQEs on error
6876  *
6877  * @hdev: pointer to the habanalabs device structure
6878  * @qid_base: first QID of the QMAN (out of 4 streams)
6879  * @stream: the QMAN's stream
6880  * @qman_base: base address of QMAN registers block
6881  * @pr_sw_conf: if true print the SW config stream data (CQ PTR and SIZE)
6882  */
6883 static void gaudi2_print_last_pqes_on_err(struct hl_device *hdev, u32 qid_base, u32 stream,
6884                         u64 qman_base, bool pr_sw_conf)
6885 {
6886     u32 ci, qm_ci_stream_off;
6887     struct hl_hw_queue *q;
6888     u64 pq_ci;
6889     int i;
6890 
6891     q = &hdev->kernel_queues[qid_base + stream];
6892 
6893     qm_ci_stream_off = mmDCORE0_TPC0_QM_PQ_CI_1 - mmDCORE0_TPC0_QM_PQ_CI_0;
6894     pq_ci = qman_base + (mmDCORE0_TPC0_QM_PQ_CI_0 - mmDCORE0_TPC0_QM_BASE) +
6895                         stream * qm_ci_stream_off;
6896 
6897     hdev->asic_funcs->hw_queues_lock(hdev);
6898 
6899     if (pr_sw_conf)
6900         gaudi2_print_sw_config_stream_data(hdev, stream, qman_base);
6901 
6902     ci = RREG32(pq_ci);
6903 
6904     /* we should start printing form ci -1 */
6905     ci = gaudi2_queue_idx_dec(ci, HL_QUEUE_LENGTH);
6906 
6907     for (i = 0; i < PQ_FETCHER_CACHE_SIZE; i++) {
6908         struct hl_bd *bd;
6909         u64 addr;
6910         u32 len;
6911 
6912         bd = q->kernel_address;
6913         bd += ci;
6914 
6915         len = le32_to_cpu(bd->len);
6916         /* len 0 means uninitialized entry- break */
6917         if (!len)
6918             break;
6919 
6920         addr = le64_to_cpu(bd->ptr);
6921 
6922         dev_info(hdev->dev, "stop on err PQE(stream %u): ci: %u, addr: %#llx, size: %x\n",
6923             stream, ci, addr, len);
6924 
6925         /* get previous ci, wrap if needed */
6926         ci = gaudi2_queue_idx_dec(ci, HL_QUEUE_LENGTH);
6927     }
6928 
6929     hdev->asic_funcs->hw_queues_unlock(hdev);
6930 }
6931 
6932 /**
6933  * print_qman_data_on_err - extract QMAN data on error
6934  *
6935  * @hdev: pointer to the habanalabs device structure
6936  * @qid_base: first QID of the QMAN (out of 4 streams)
6937  * @stream: the QMAN's stream
6938  * @qman_base: base address of QMAN registers block
6939  *
6940  * This function attempt to extract as much data as possible on QMAN error.
6941  * On upper CP print the SW config stream data and last 8 PQEs.
6942  * On lower CP print SW config data and last PQEs of ALL 4 upper CPs
6943  */
6944 static void print_qman_data_on_err(struct hl_device *hdev, u32 qid_base, u32 stream, u64 qman_base)
6945 {
6946     u32 i;
6947 
6948     if (stream != QMAN_STREAMS) {
6949         gaudi2_print_last_pqes_on_err(hdev, qid_base, stream, qman_base, true);
6950         return;
6951     }
6952 
6953     gaudi2_print_sw_config_stream_data(hdev, stream, qman_base);
6954 
6955     for (i = 0 ; i < QMAN_STREAMS ; i++)
6956         gaudi2_print_last_pqes_on_err(hdev, qid_base, i, qman_base, false);
6957 }
6958 
6959 static void gaudi2_handle_qman_err_generic(struct hl_device *hdev, const char *qm_name,
6960                         u64 qman_base, u32 qid_base)
6961 {
6962     u32 i, j, glbl_sts_val, arb_err_val, num_error_causes;
6963     u64 glbl_sts_addr, arb_err_addr;
6964     char reg_desc[32];
6965 
6966     glbl_sts_addr = qman_base + (mmDCORE0_TPC0_QM_GLBL_ERR_STS_0 - mmDCORE0_TPC0_QM_BASE);
6967     arb_err_addr = qman_base + (mmDCORE0_TPC0_QM_ARB_ERR_CAUSE - mmDCORE0_TPC0_QM_BASE);
6968 
6969     /* Iterate through all stream GLBL_ERR_STS registers + Lower CP */
6970     for (i = 0 ; i < QMAN_STREAMS + 1 ; i++) {
6971         glbl_sts_val = RREG32(glbl_sts_addr + 4 * i);
6972 
6973         if (!glbl_sts_val)
6974             continue;
6975 
6976         if (i == QMAN_STREAMS) {
6977             snprintf(reg_desc, ARRAY_SIZE(reg_desc), "LowerCP");
6978             num_error_causes = GAUDI2_NUM_OF_QM_LCP_ERR_CAUSE;
6979         } else {
6980             snprintf(reg_desc, ARRAY_SIZE(reg_desc), "stream%u", i);
6981             num_error_causes = GAUDI2_NUM_OF_QM_ERR_CAUSE;
6982         }
6983 
6984         for (j = 0 ; j < num_error_causes ; j++)
6985             if (glbl_sts_val & BIT(j))
6986                 dev_err_ratelimited(hdev->dev, "%s %s. err cause: %s\n",
6987                         qm_name, reg_desc,
6988                         i == QMAN_STREAMS ?
6989                         gaudi2_qman_lower_cp_error_cause[j] :
6990                         gaudi2_qman_error_cause[j]);
6991 
6992         print_qman_data_on_err(hdev, qid_base, i, qman_base);
6993     }
6994 
6995     arb_err_val = RREG32(arb_err_addr);
6996 
6997     if (!arb_err_val)
6998         return;
6999 
7000     for (j = 0 ; j < GAUDI2_NUM_OF_QM_ARB_ERR_CAUSE ; j++) {
7001         if (arb_err_val & BIT(j))
7002             dev_err_ratelimited(hdev->dev, "%s ARB_ERR. err cause: %s\n",
7003                         qm_name, gaudi2_qman_arb_error_cause[j]);
7004     }
7005 }
7006 
7007 static void gaudi2_razwi_rr_hbw_shared_printf_info(struct hl_device *hdev,
7008             u64 rtr_mstr_if_base_addr, bool is_write, char *name,
7009             bool read_razwi_regs, struct hl_eq_razwi_info *razwi_info)
7010 {
7011     u32 razwi_hi, razwi_lo, razwi_xy;
7012 
7013     if (is_write) {
7014         if (read_razwi_regs) {
7015             razwi_hi = RREG32(rtr_mstr_if_base_addr + RR_SHRD_HBW_AW_RAZWI_HI);
7016             razwi_lo = RREG32(rtr_mstr_if_base_addr + RR_SHRD_HBW_AW_RAZWI_LO);
7017             razwi_xy = RREG32(rtr_mstr_if_base_addr + RR_SHRD_HBW_AW_RAZWI_XY);
7018         } else {
7019             razwi_hi = le32_to_cpu(razwi_info->hbw.rr_aw_razwi_hi_reg);
7020             razwi_lo = le32_to_cpu(razwi_info->hbw.rr_aw_razwi_lo_reg);
7021             razwi_xy = le32_to_cpu(razwi_info->hbw.rr_aw_razwi_id_reg);
7022         }
7023 
7024         dev_err_ratelimited(hdev->dev,
7025             "%s-RAZWI SHARED RR HBW WR error, captured address HI 0x%x LO 0x%x, Initiator coordinates 0x%x\n",
7026             name, razwi_hi, razwi_lo, razwi_xy);
7027     } else {
7028         if (read_razwi_regs) {
7029             razwi_hi = RREG32(rtr_mstr_if_base_addr + RR_SHRD_HBW_AR_RAZWI_HI);
7030             razwi_lo = RREG32(rtr_mstr_if_base_addr + RR_SHRD_HBW_AR_RAZWI_LO);
7031             razwi_xy = RREG32(rtr_mstr_if_base_addr + RR_SHRD_HBW_AR_RAZWI_XY);
7032         } else {
7033             razwi_hi = le32_to_cpu(razwi_info->hbw.rr_ar_razwi_hi_reg);
7034             razwi_lo = le32_to_cpu(razwi_info->hbw.rr_ar_razwi_lo_reg);
7035             razwi_xy = le32_to_cpu(razwi_info->hbw.rr_ar_razwi_id_reg);
7036         }
7037 
7038         dev_err_ratelimited(hdev->dev,
7039             "%s-RAZWI SHARED RR HBW AR error, captured address HI 0x%x LO 0x%x, Initiator coordinates 0x%x\n",
7040             name, razwi_hi, razwi_lo, razwi_xy);
7041     }
7042 }
7043 
7044 static void gaudi2_razwi_rr_lbw_shared_printf_info(struct hl_device *hdev,
7045             u64 rtr_mstr_if_base_addr, bool is_write, char *name,
7046             bool read_razwi_regs, struct hl_eq_razwi_info *razwi_info)
7047 {
7048     u32 razwi_addr, razwi_xy;
7049 
7050     if (is_write) {
7051         if (read_razwi_regs) {
7052             razwi_addr = RREG32(rtr_mstr_if_base_addr + RR_SHRD_LBW_AW_RAZWI);
7053             razwi_xy = RREG32(rtr_mstr_if_base_addr + RR_SHRD_LBW_AW_RAZWI_XY);
7054         } else {
7055             razwi_addr = le32_to_cpu(razwi_info->lbw.rr_aw_razwi_reg);
7056             razwi_xy = le32_to_cpu(razwi_info->lbw.rr_aw_razwi_id_reg);
7057         }
7058 
7059         dev_err_ratelimited(hdev->dev,
7060             "%s-RAZWI SHARED RR LBW WR error, mstr_if 0x%llx, captured address 0x%x, Initiator coordinates 0x%x\n",
7061             name, rtr_mstr_if_base_addr, razwi_addr, razwi_xy);
7062     } else {
7063         if (read_razwi_regs) {
7064             razwi_addr = RREG32(rtr_mstr_if_base_addr + RR_SHRD_LBW_AR_RAZWI);
7065             razwi_xy = RREG32(rtr_mstr_if_base_addr + RR_SHRD_LBW_AR_RAZWI_XY);
7066         } else {
7067             razwi_addr = le32_to_cpu(razwi_info->lbw.rr_ar_razwi_reg);
7068             razwi_xy = le32_to_cpu(razwi_info->lbw.rr_ar_razwi_id_reg);
7069         }
7070 
7071         dev_err_ratelimited(hdev->dev,
7072             "%s-RAZWI SHARED RR LBW AR error, mstr_if 0x%llx, captured address 0x%x Initiator coordinates 0x%x\n",
7073             name, rtr_mstr_if_base_addr, razwi_addr, razwi_xy);
7074     }
7075 }
7076 
7077 /*
7078  * This function handles RR(Range register) hit events.
7079  * raised be initiators not PSOC RAZWI.
7080  */
7081 static void gaudi2_ack_module_razwi_event_handler(struct hl_device *hdev,
7082                 enum razwi_event_sources module, u8 module_idx,
7083                 u8 module_sub_idx, struct hl_eq_razwi_info *razwi_info)
7084 {
7085     bool via_sft = false, read_razwi_regs = false;
7086     u32 rtr_id, dcore_id, dcore_rtr_id, sft_id;
7087     u64 rtr_mstr_if_base_addr;
7088     u32 hbw_shrd_aw = 0, hbw_shrd_ar = 0;
7089     u32 lbw_shrd_aw = 0, lbw_shrd_ar = 0;
7090     char initiator_name[64];
7091 
7092     if (hdev->pldm || !(hdev->fw_components & FW_TYPE_LINUX) || !razwi_info)
7093         read_razwi_regs = true;
7094 
7095     switch (module) {
7096     case RAZWI_TPC:
7097         rtr_id = gaudi2_tpc_initiator_rtr_id[module_idx];
7098         sprintf(initiator_name, "TPC_%u", module_idx);
7099         break;
7100     case RAZWI_MME:
7101         sprintf(initiator_name, "MME_%u", module_idx);
7102         switch (module_sub_idx) {
7103         case MME_WAP0:
7104             rtr_id = gaudi2_mme_initiator_rtr_id[module_idx].wap0;
7105             break;
7106         case MME_WAP1:
7107             rtr_id = gaudi2_mme_initiator_rtr_id[module_idx].wap1;
7108             break;
7109         case MME_WRITE:
7110             rtr_id = gaudi2_mme_initiator_rtr_id[module_idx].write;
7111             break;
7112         case MME_READ:
7113             rtr_id = gaudi2_mme_initiator_rtr_id[module_idx].read;
7114             break;
7115         case MME_SBTE0:
7116             rtr_id = gaudi2_mme_initiator_rtr_id[module_idx].sbte0;
7117             break;
7118         case MME_SBTE1:
7119             rtr_id = gaudi2_mme_initiator_rtr_id[module_idx].sbte1;
7120             break;
7121         case MME_SBTE2:
7122             rtr_id = gaudi2_mme_initiator_rtr_id[module_idx].sbte2;
7123             break;
7124         case MME_SBTE3:
7125             rtr_id = gaudi2_mme_initiator_rtr_id[module_idx].sbte3;
7126             break;
7127         case MME_SBTE4:
7128             rtr_id = gaudi2_mme_initiator_rtr_id[module_idx].sbte4;
7129             break;
7130         default:
7131             return;
7132         }
7133         break;
7134     case RAZWI_EDMA:
7135         sft_id = gaudi2_edma_initiator_sft_id[module_idx].interface_id;
7136         dcore_id = gaudi2_edma_initiator_sft_id[module_idx].dcore_id;
7137         via_sft = true;
7138         sprintf(initiator_name, "EDMA_%u", module_idx);
7139         break;
7140     case RAZWI_PDMA:
7141         rtr_id = gaudi2_pdma_initiator_rtr_id[module_idx];
7142         sprintf(initiator_name, "PDMA_%u", module_idx);
7143         break;
7144     case RAZWI_NIC:
7145         rtr_id = gaudi2_nic_initiator_rtr_id[module_idx];
7146         sprintf(initiator_name, "NIC_%u", module_idx);
7147         break;
7148     case RAZWI_DEC:
7149         rtr_id = gaudi2_dec_initiator_rtr_id[module_idx];
7150         sprintf(initiator_name, "DEC_%u", module_idx);
7151         break;
7152     case RAZWI_ROT:
7153         rtr_id = gaudi2_rot_initiator_rtr_id[module_idx];
7154         sprintf(initiator_name, "ROT_%u", module_idx);
7155         break;
7156     default:
7157         return;
7158     }
7159 
7160     if (!read_razwi_regs) {
7161         if (le32_to_cpu(razwi_info->razwi_happened_mask) & RAZWI_HAPPENED_HBW) {
7162             hbw_shrd_aw = le32_to_cpu(razwi_info->razwi_happened_mask) &
7163                                 RAZWI_HAPPENED_AW;
7164             hbw_shrd_ar = le32_to_cpu(razwi_info->razwi_happened_mask) &
7165                                 RAZWI_HAPPENED_AR;
7166         } else if (le32_to_cpu(razwi_info->razwi_happened_mask) & RAZWI_HAPPENED_LBW) {
7167             lbw_shrd_aw = le32_to_cpu(razwi_info->razwi_happened_mask) &
7168                                 RAZWI_HAPPENED_AW;
7169             lbw_shrd_ar = le32_to_cpu(razwi_info->razwi_happened_mask) &
7170                                 RAZWI_HAPPENED_AR;
7171         }
7172         rtr_mstr_if_base_addr = 0;
7173 
7174         goto dump_info;
7175     }
7176 
7177     /* Find router mstr_if register base */
7178     if (via_sft) {
7179         rtr_mstr_if_base_addr = mmSFT0_HBW_RTR_IF0_RTR_CTRL_BASE +
7180                 dcore_id * SFT_DCORE_OFFSET +
7181                 sft_id * SFT_IF_OFFSET +
7182                 RTR_MSTR_IF_OFFSET;
7183     } else {
7184         dcore_id = rtr_id / NUM_OF_RTR_PER_DCORE;
7185         dcore_rtr_id = rtr_id % NUM_OF_RTR_PER_DCORE;
7186         rtr_mstr_if_base_addr = mmDCORE0_RTR0_CTRL_BASE +
7187                 dcore_id * DCORE_OFFSET +
7188                 dcore_rtr_id * DCORE_RTR_OFFSET +
7189                 RTR_MSTR_IF_OFFSET;
7190     }
7191 
7192     /* Find out event cause by reading "RAZWI_HAPPENED" registers */
7193     hbw_shrd_aw = RREG32(rtr_mstr_if_base_addr + RR_SHRD_HBW_AW_RAZWI_HAPPENED);
7194 
7195     hbw_shrd_ar = RREG32(rtr_mstr_if_base_addr + RR_SHRD_HBW_AR_RAZWI_HAPPENED);
7196 
7197     if (via_sft) {
7198         /* SFT has separate MSTR_IF for LBW, only there we can
7199          * read the LBW razwi related registers
7200          */
7201         u64 base;
7202 
7203         base = mmSFT0_HBW_RTR_IF0_RTR_CTRL_BASE + dcore_id * SFT_DCORE_OFFSET +
7204                 RTR_LBW_MSTR_IF_OFFSET;
7205 
7206         lbw_shrd_aw = RREG32(base + RR_SHRD_LBW_AW_RAZWI_HAPPENED);
7207 
7208         lbw_shrd_ar = RREG32(base + RR_SHRD_LBW_AR_RAZWI_HAPPENED);
7209     } else {
7210         lbw_shrd_aw = RREG32(rtr_mstr_if_base_addr + RR_SHRD_LBW_AW_RAZWI_HAPPENED);
7211 
7212         lbw_shrd_ar = RREG32(rtr_mstr_if_base_addr + RR_SHRD_LBW_AR_RAZWI_HAPPENED);
7213     }
7214 
7215 dump_info:
7216     /* check if there is no RR razwi indication at all */
7217     if (!hbw_shrd_aw && !hbw_shrd_ar && !lbw_shrd_aw && !lbw_shrd_ar)
7218         return;
7219 
7220     if (hbw_shrd_aw) {
7221         gaudi2_razwi_rr_hbw_shared_printf_info(hdev, rtr_mstr_if_base_addr, true,
7222                         initiator_name, read_razwi_regs, razwi_info);
7223 
7224         /* Clear event indication */
7225         if (read_razwi_regs)
7226             WREG32(rtr_mstr_if_base_addr + RR_SHRD_HBW_AW_RAZWI_HAPPENED, hbw_shrd_aw);
7227     }
7228 
7229     if (hbw_shrd_ar) {
7230         gaudi2_razwi_rr_hbw_shared_printf_info(hdev, rtr_mstr_if_base_addr, false,
7231                         initiator_name, read_razwi_regs, razwi_info);
7232 
7233         /* Clear event indication */
7234         if (read_razwi_regs)
7235             WREG32(rtr_mstr_if_base_addr + RR_SHRD_HBW_AR_RAZWI_HAPPENED, hbw_shrd_ar);
7236     }
7237 
7238     if (lbw_shrd_aw) {
7239         gaudi2_razwi_rr_lbw_shared_printf_info(hdev, rtr_mstr_if_base_addr, true,
7240                         initiator_name, read_razwi_regs, razwi_info);
7241 
7242         /* Clear event indication */
7243         if (read_razwi_regs)
7244             WREG32(rtr_mstr_if_base_addr + RR_SHRD_LBW_AW_RAZWI_HAPPENED, lbw_shrd_aw);
7245     }
7246 
7247     if (lbw_shrd_ar) {
7248         gaudi2_razwi_rr_lbw_shared_printf_info(hdev, rtr_mstr_if_base_addr, false,
7249                         initiator_name, read_razwi_regs, razwi_info);
7250 
7251         /* Clear event indication */
7252         if (read_razwi_regs)
7253             WREG32(rtr_mstr_if_base_addr + RR_SHRD_LBW_AR_RAZWI_HAPPENED, lbw_shrd_ar);
7254     }
7255 }
7256 
7257 static void gaudi2_check_if_razwi_happened(struct hl_device *hdev)
7258 {
7259     struct asic_fixed_properties *prop = &hdev->asic_prop;
7260     u8 mod_idx, sub_mod;
7261 
7262     /* check all TPCs */
7263     for (mod_idx = 0 ; mod_idx < (NUM_OF_TPC_PER_DCORE * NUM_OF_DCORES + 1) ; mod_idx++) {
7264         if (prop->tpc_enabled_mask & BIT(mod_idx))
7265             gaudi2_ack_module_razwi_event_handler(hdev, RAZWI_TPC, mod_idx, 0, NULL);
7266     }
7267 
7268     /* check all MMEs */
7269     for (mod_idx = 0 ; mod_idx < (NUM_OF_MME_PER_DCORE * NUM_OF_DCORES) ; mod_idx++)
7270         for (sub_mod = MME_WAP0 ; sub_mod < MME_INITIATORS_MAX ; sub_mod++)
7271             gaudi2_ack_module_razwi_event_handler(hdev, RAZWI_MME, mod_idx,
7272                                 sub_mod, NULL);
7273 
7274     /* check all EDMAs */
7275     for (mod_idx = 0 ; mod_idx < (NUM_OF_EDMA_PER_DCORE * NUM_OF_DCORES) ; mod_idx++)
7276         if (prop->edma_enabled_mask & BIT(mod_idx))
7277             gaudi2_ack_module_razwi_event_handler(hdev, RAZWI_EDMA, mod_idx, 0, NULL);
7278 
7279     /* check all PDMAs */
7280     for (mod_idx = 0 ; mod_idx < NUM_OF_PDMA ; mod_idx++)
7281         gaudi2_ack_module_razwi_event_handler(hdev, RAZWI_PDMA, mod_idx, 0, NULL);
7282 
7283     /* check all NICs */
7284     for (mod_idx = 0 ; mod_idx < NIC_NUMBER_OF_PORTS ; mod_idx++)
7285         if (hdev->nic_ports_mask & BIT(mod_idx))
7286             gaudi2_ack_module_razwi_event_handler(hdev, RAZWI_NIC, mod_idx >> 1, 0,
7287                                 NULL);
7288 
7289     /* check all DECs */
7290     for (mod_idx = 0 ; mod_idx < NUMBER_OF_DEC ; mod_idx++)
7291         if (prop->decoder_enabled_mask & BIT(mod_idx))
7292             gaudi2_ack_module_razwi_event_handler(hdev, RAZWI_DEC, mod_idx, 0, NULL);
7293 
7294     /* check all ROTs */
7295     for (mod_idx = 0 ; mod_idx < NUM_OF_ROT ; mod_idx++)
7296         gaudi2_ack_module_razwi_event_handler(hdev, RAZWI_ROT, mod_idx, 0, NULL);
7297 }
7298 
7299 static void gaudi2_razwi_unmapped_addr_hbw_printf_info(struct hl_device *hdev,
7300                             u64 rtr_ctrl_base_addr, bool is_write)
7301 {
7302     u32 razwi_hi, razwi_lo;
7303 
7304     if (is_write) {
7305         razwi_hi = RREG32(rtr_ctrl_base_addr + DEC_RAZWI_HBW_AW_ADDR_HI);
7306         razwi_lo = RREG32(rtr_ctrl_base_addr + DEC_RAZWI_HBW_AW_ADDR_LO);
7307 
7308         dev_err_ratelimited(hdev->dev,
7309             "RAZWI PSOC unmapped HBW WR error, ctr_base 0x%llx, captured address HI 0x%x, LO 0x%x\n",
7310             rtr_ctrl_base_addr, razwi_hi, razwi_lo);
7311 
7312         /* Clear set indication */
7313         WREG32(rtr_ctrl_base_addr + DEC_RAZWI_HBW_AW_SET, 0x1);
7314     } else {
7315         razwi_hi = RREG32(rtr_ctrl_base_addr + DEC_RAZWI_HBW_AR_ADDR_HI);
7316 
7317         razwi_lo = RREG32(rtr_ctrl_base_addr + DEC_RAZWI_HBW_AR_ADDR_LO);
7318 
7319         dev_err_ratelimited(hdev->dev,
7320             "RAZWI PSOC unmapped HBW AR error, ctr_base 0x%llx, captured address HI 0x%x, LO 0x%x\n",
7321             rtr_ctrl_base_addr, razwi_hi, razwi_lo);
7322 
7323         /* Clear set indication */
7324         WREG32(rtr_ctrl_base_addr + DEC_RAZWI_HBW_AR_SET, 0x1);
7325     }
7326 }
7327 
7328 static void gaudi2_razwi_unmapped_addr_lbw_printf_info(struct hl_device *hdev,
7329                     u64 rtr_ctrl_base_addr, bool is_write)
7330 {
7331     u32 razwi_addr;
7332 
7333     if (is_write) {
7334         razwi_addr = RREG32(rtr_ctrl_base_addr + DEC_RAZWI_LBW_AW_ADDR);
7335 
7336         dev_err_ratelimited(hdev->dev,
7337             "RAZWI PSOC unmapped LBW WR error, ctr_base 0x%llx, captured address 0x%x\n",
7338             rtr_ctrl_base_addr, razwi_addr);
7339 
7340         /* Clear set indication */
7341         WREG32(rtr_ctrl_base_addr + DEC_RAZWI_LBW_AW_SET, 0x1);
7342     } else {
7343         razwi_addr = RREG32(rtr_ctrl_base_addr + DEC_RAZWI_LBW_AR_ADDR);
7344 
7345         dev_err_ratelimited(hdev->dev,
7346             "RAZWI PSOC unmapped LBW AR error, ctr_base 0x%llx, captured address 0x%x\n",
7347             rtr_ctrl_base_addr, razwi_addr);
7348 
7349         /* Clear set indication */
7350         WREG32(rtr_ctrl_base_addr + DEC_RAZWI_LBW_AR_SET, 0x1);
7351     }
7352 }
7353 
7354 /* PSOC RAZWI interrupt occurs only when trying to access a bad address */
7355 static void gaudi2_ack_psoc_razwi_event_handler(struct hl_device *hdev)
7356 {
7357     u32 hbw_aw_set, hbw_ar_set, lbw_aw_set, lbw_ar_set, rtr_id, dcore_id, dcore_rtr_id, xy,
7358                                 razwi_mask_info, razwi_intr = 0;
7359     int rtr_map_arr_len = NUM_OF_RTR_PER_DCORE * NUM_OF_DCORES;
7360     u64 rtr_ctrl_base_addr;
7361 
7362     if (hdev->pldm || !(hdev->fw_components & FW_TYPE_LINUX)) {
7363         razwi_intr = RREG32(mmPSOC_GLOBAL_CONF_RAZWI_INTERRUPT);
7364         if (!razwi_intr)
7365             return;
7366     }
7367 
7368     razwi_mask_info = RREG32(mmPSOC_GLOBAL_CONF_RAZWI_MASK_INFO);
7369 
7370     xy = (razwi_mask_info & PSOC_GLOBAL_CONF_RAZWI_MASK_INFO_AXUSER_L_MASK)
7371         >> PSOC_GLOBAL_CONF_RAZWI_MASK_INFO_AXUSER_L_SHIFT;
7372 
7373     dev_err_ratelimited(hdev->dev,
7374         "PSOC RAZWI interrupt: Mask %d, WAS_AR %d, WAS_AW %d, AXUSER_L 0x%x AXUSER_H 0x%x\n",
7375         (razwi_mask_info & PSOC_GLOBAL_CONF_RAZWI_MASK_INFO_MASK_MASK)
7376             >> PSOC_GLOBAL_CONF_RAZWI_MASK_INFO_MASK_SHIFT,
7377         (razwi_mask_info & PSOC_GLOBAL_CONF_RAZWI_MASK_INFO_WAS_AR_MASK)
7378             >> PSOC_GLOBAL_CONF_RAZWI_MASK_INFO_WAS_AR_SHIFT,
7379         (razwi_mask_info & PSOC_GLOBAL_CONF_RAZWI_MASK_INFO_WAS_AW_MASK)
7380             >> PSOC_GLOBAL_CONF_RAZWI_MASK_INFO_WAS_AW_SHIFT, xy,
7381         (razwi_mask_info &
7382             PSOC_GLOBAL_CONF_RAZWI_MASK_INFO_AXUSER_H_MASK)
7383             >> PSOC_GLOBAL_CONF_RAZWI_MASK_INFO_AXUSER_H_SHIFT);
7384     if (xy == 0) {
7385         dev_err_ratelimited(hdev->dev,
7386                 "PSOC RAZWI interrupt: received event from 0 rtr coordinates\n");
7387         goto clear;
7388     }
7389 
7390     /* Find router id by router coordinates */
7391     for (rtr_id = 0 ; rtr_id < rtr_map_arr_len ; rtr_id++)
7392         if (rtr_coordinates_to_rtr_id[rtr_id] == xy)
7393             break;
7394 
7395     if (rtr_id == rtr_map_arr_len) {
7396         dev_err_ratelimited(hdev->dev,
7397                 "PSOC RAZWI interrupt: invalid rtr coordinates (0x%x)\n", xy);
7398         goto clear;
7399     }
7400 
7401     /* Find router mstr_if register base */
7402     dcore_id = rtr_id / NUM_OF_RTR_PER_DCORE;
7403     dcore_rtr_id = rtr_id % NUM_OF_RTR_PER_DCORE;
7404     rtr_ctrl_base_addr = mmDCORE0_RTR0_CTRL_BASE + dcore_id * DCORE_OFFSET +
7405                 dcore_rtr_id * DCORE_RTR_OFFSET;
7406 
7407     hbw_aw_set = RREG32(rtr_ctrl_base_addr + DEC_RAZWI_HBW_AW_SET);
7408     hbw_ar_set = RREG32(rtr_ctrl_base_addr + DEC_RAZWI_HBW_AR_SET);
7409     lbw_aw_set = RREG32(rtr_ctrl_base_addr + DEC_RAZWI_LBW_AW_SET);
7410     lbw_ar_set = RREG32(rtr_ctrl_base_addr + DEC_RAZWI_LBW_AR_SET);
7411 
7412     if (hbw_aw_set)
7413         gaudi2_razwi_unmapped_addr_hbw_printf_info(hdev, rtr_ctrl_base_addr, true);
7414 
7415     if (hbw_ar_set)
7416         gaudi2_razwi_unmapped_addr_hbw_printf_info(hdev, rtr_ctrl_base_addr, false);
7417 
7418     if (lbw_aw_set)
7419         gaudi2_razwi_unmapped_addr_lbw_printf_info(hdev, rtr_ctrl_base_addr, true);
7420 
7421     if (lbw_ar_set)
7422         gaudi2_razwi_unmapped_addr_lbw_printf_info(hdev, rtr_ctrl_base_addr, false);
7423 
7424 clear:
7425     /* Clear Interrupts only on pldm or if f/w doesn't handle interrupts */
7426     if (hdev->pldm || !(hdev->fw_components & FW_TYPE_LINUX))
7427         WREG32(mmPSOC_GLOBAL_CONF_RAZWI_INTERRUPT, razwi_intr);
7428 }
7429 
7430 static void _gaudi2_handle_qm_sei_err(struct hl_device *hdev, u64 qman_base)
7431 {
7432     u32 i, sts_val, sts_clr_val = 0;
7433 
7434     sts_val = RREG32(qman_base + QM_SEI_STATUS_OFFSET);
7435 
7436     for (i = 0 ; i < GAUDI2_NUM_OF_QM_SEI_ERR_CAUSE ; i++) {
7437         if (sts_val & BIT(i)) {
7438             dev_err_ratelimited(hdev->dev, "QM SEI. err cause: %s\n",
7439                         gaudi2_qm_sei_error_cause[i]);
7440             sts_clr_val |= BIT(i);
7441         }
7442     }
7443 
7444     WREG32(qman_base + QM_SEI_STATUS_OFFSET, sts_clr_val);
7445 }
7446 
7447 static void gaudi2_handle_qm_sei_err(struct hl_device *hdev, u16 event_type,
7448                     struct hl_eq_razwi_info *razwi_info)
7449 {
7450     u64 qman_base;
7451     u8 index;
7452 
7453     switch (event_type) {
7454     case GAUDI2_EVENT_TPC0_AXI_ERR_RSP ... GAUDI2_EVENT_TPC23_AXI_ERR_RSP:
7455         index = event_type - GAUDI2_EVENT_TPC0_AXI_ERR_RSP;
7456         qman_base = mmDCORE0_TPC0_QM_BASE +
7457                 (index / NUM_OF_TPC_PER_DCORE) * DCORE_OFFSET +
7458                 (index % NUM_OF_TPC_PER_DCORE) * DCORE_TPC_OFFSET;
7459         break;
7460     case GAUDI2_EVENT_TPC24_AXI_ERR_RSP:
7461         qman_base = mmDCORE0_TPC6_QM_BASE;
7462         break;
7463     case GAUDI2_EVENT_MME0_CTRL_AXI_ERROR_RESPONSE:
7464     case GAUDI2_EVENT_MME1_CTRL_AXI_ERROR_RESPONSE:
7465     case GAUDI2_EVENT_MME2_CTRL_AXI_ERROR_RESPONSE:
7466     case GAUDI2_EVENT_MME3_CTRL_AXI_ERROR_RESPONSE:
7467         index = (event_type - GAUDI2_EVENT_MME0_CTRL_AXI_ERROR_RESPONSE) /
7468                 (GAUDI2_EVENT_MME1_CTRL_AXI_ERROR_RESPONSE -
7469                         GAUDI2_EVENT_MME0_CTRL_AXI_ERROR_RESPONSE);
7470         qman_base = mmDCORE0_MME_QM_BASE + index * DCORE_OFFSET;
7471         break;
7472     case GAUDI2_EVENT_PDMA_CH0_AXI_ERR_RSP:
7473     case GAUDI2_EVENT_PDMA_CH1_AXI_ERR_RSP:
7474         index = event_type - GAUDI2_EVENT_PDMA_CH0_AXI_ERR_RSP;
7475         qman_base = mmPDMA0_QM_BASE + index * PDMA_OFFSET;
7476         break;
7477     case GAUDI2_EVENT_ROTATOR0_AXI_ERROR_RESPONSE:
7478     case GAUDI2_EVENT_ROTATOR1_AXI_ERROR_RESPONSE:
7479         index = event_type - GAUDI2_EVENT_ROTATOR0_AXI_ERROR_RESPONSE;
7480         qman_base = mmROT0_QM_BASE + index * ROT_OFFSET;
7481         break;
7482     default:
7483         return;
7484     }
7485 
7486     _gaudi2_handle_qm_sei_err(hdev, qman_base);
7487 
7488     /* There is a single event per NIC macro, so should check its both QMAN blocks */
7489     if (event_type >= GAUDI2_EVENT_NIC0_AXI_ERROR_RESPONSE &&
7490             event_type <= GAUDI2_EVENT_NIC11_AXI_ERROR_RESPONSE)
7491         _gaudi2_handle_qm_sei_err(hdev, qman_base + NIC_QM_OFFSET);
7492 
7493     /* check if RAZWI happened */
7494     if (razwi_info)
7495         gaudi2_ack_module_razwi_event_handler(hdev, RAZWI_PDMA, 0, 0, razwi_info);
7496 }
7497 
7498 static void gaudi2_handle_qman_err(struct hl_device *hdev, u16 event_type)
7499 {
7500     u32 qid_base;
7501     u64 qman_base;
7502     char desc[32];
7503     u8 index;
7504 
7505     switch (event_type) {
7506     case GAUDI2_EVENT_TPC0_QM ... GAUDI2_EVENT_TPC5_QM:
7507         index = event_type - GAUDI2_EVENT_TPC0_QM;
7508         qid_base = GAUDI2_QUEUE_ID_DCORE0_TPC_0_0 + index * QMAN_STREAMS;
7509         qman_base = mmDCORE0_TPC0_QM_BASE + index * DCORE_TPC_OFFSET;
7510         snprintf(desc, ARRAY_SIZE(desc), "DCORE0_TPC%d_QM", index);
7511         break;
7512     case GAUDI2_EVENT_TPC6_QM ... GAUDI2_EVENT_TPC11_QM:
7513         index = event_type - GAUDI2_EVENT_TPC6_QM;
7514         qid_base = GAUDI2_QUEUE_ID_DCORE1_TPC_0_0 + index * QMAN_STREAMS;
7515         qman_base = mmDCORE1_TPC0_QM_BASE + index * DCORE_TPC_OFFSET;
7516         snprintf(desc, ARRAY_SIZE(desc), "DCORE1_TPC%d_QM", index);
7517         break;
7518     case GAUDI2_EVENT_TPC12_QM ... GAUDI2_EVENT_TPC17_QM:
7519         index = event_type - GAUDI2_EVENT_TPC12_QM;
7520         qid_base = GAUDI2_QUEUE_ID_DCORE2_TPC_0_0 + index * QMAN_STREAMS;
7521         qman_base = mmDCORE2_TPC0_QM_BASE + index * DCORE_TPC_OFFSET;
7522         snprintf(desc, ARRAY_SIZE(desc), "DCORE2_TPC%d_QM", index);
7523         break;
7524     case GAUDI2_EVENT_TPC18_QM ... GAUDI2_EVENT_TPC23_QM:
7525         index = event_type - GAUDI2_EVENT_TPC18_QM;
7526         qid_base = GAUDI2_QUEUE_ID_DCORE3_TPC_0_0 + index * QMAN_STREAMS;
7527         qman_base = mmDCORE3_TPC0_QM_BASE + index * DCORE_TPC_OFFSET;
7528         snprintf(desc, ARRAY_SIZE(desc), "DCORE3_TPC%d_QM", index);
7529         break;
7530     case GAUDI2_EVENT_TPC24_QM:
7531         qid_base = GAUDI2_QUEUE_ID_DCORE0_TPC_6_0;
7532         qman_base = mmDCORE0_TPC6_QM_BASE;
7533         snprintf(desc, ARRAY_SIZE(desc), "DCORE0_TPC6_QM");
7534         break;
7535     case GAUDI2_EVENT_MME0_QM:
7536         qid_base = GAUDI2_QUEUE_ID_DCORE0_MME_0_0;
7537         qman_base = mmDCORE0_MME_QM_BASE;
7538         snprintf(desc, ARRAY_SIZE(desc), "DCORE0_MME_QM");
7539         break;
7540     case GAUDI2_EVENT_MME1_QM:
7541         qid_base = GAUDI2_QUEUE_ID_DCORE1_MME_0_0;
7542         qman_base = mmDCORE1_MME_QM_BASE;
7543         snprintf(desc, ARRAY_SIZE(desc), "DCORE1_MME_QM");
7544         break;
7545     case GAUDI2_EVENT_MME2_QM:
7546         qid_base = GAUDI2_QUEUE_ID_DCORE2_MME_0_0;
7547         qman_base = mmDCORE2_MME_QM_BASE;
7548         snprintf(desc, ARRAY_SIZE(desc), "DCORE2_MME_QM");
7549         break;
7550     case GAUDI2_EVENT_MME3_QM:
7551         qid_base = GAUDI2_QUEUE_ID_DCORE3_MME_0_0;
7552         qman_base = mmDCORE3_MME_QM_BASE;
7553         snprintf(desc, ARRAY_SIZE(desc), "DCORE3_MME_QM");
7554         break;
7555     case GAUDI2_EVENT_HDMA0_QM:
7556         qid_base = GAUDI2_QUEUE_ID_DCORE0_EDMA_0_0;
7557         qman_base = mmDCORE0_EDMA0_QM_BASE;
7558         snprintf(desc, ARRAY_SIZE(desc), "DCORE0_EDMA0_QM");
7559         break;
7560     case GAUDI2_EVENT_HDMA1_QM:
7561         qid_base = GAUDI2_QUEUE_ID_DCORE0_EDMA_1_0;
7562         qman_base = mmDCORE0_EDMA1_QM_BASE;
7563         snprintf(desc, ARRAY_SIZE(desc), "DCORE0_EDMA1_QM");
7564         break;
7565     case GAUDI2_EVENT_HDMA2_QM:
7566         qid_base = GAUDI2_QUEUE_ID_DCORE1_EDMA_0_0;
7567         qman_base = mmDCORE1_EDMA0_QM_BASE;
7568         snprintf(desc, ARRAY_SIZE(desc), "DCORE1_EDMA0_QM");
7569         break;
7570     case GAUDI2_EVENT_HDMA3_QM:
7571         qid_base = GAUDI2_QUEUE_ID_DCORE1_EDMA_1_0;
7572         qman_base = mmDCORE1_EDMA1_QM_BASE;
7573         snprintf(desc, ARRAY_SIZE(desc), "DCORE1_EDMA1_QM");
7574         break;
7575     case GAUDI2_EVENT_HDMA4_QM:
7576         qid_base = GAUDI2_QUEUE_ID_DCORE2_EDMA_0_0;
7577         qman_base = mmDCORE2_EDMA0_QM_BASE;
7578         snprintf(desc, ARRAY_SIZE(desc), "DCORE2_EDMA0_QM");
7579         break;
7580     case GAUDI2_EVENT_HDMA5_QM:
7581         qid_base = GAUDI2_QUEUE_ID_DCORE2_EDMA_1_0;
7582         qman_base = mmDCORE2_EDMA1_QM_BASE;
7583         snprintf(desc, ARRAY_SIZE(desc), "DCORE2_EDMA1_QM");
7584         break;
7585     case GAUDI2_EVENT_HDMA6_QM:
7586         qid_base = GAUDI2_QUEUE_ID_DCORE3_EDMA_0_0;
7587         qman_base = mmDCORE3_EDMA0_QM_BASE;
7588         snprintf(desc, ARRAY_SIZE(desc), "DCORE3_EDMA0_QM");
7589         break;
7590     case GAUDI2_EVENT_HDMA7_QM:
7591         qid_base = GAUDI2_QUEUE_ID_DCORE3_EDMA_1_0;
7592         qman_base = mmDCORE3_EDMA1_QM_BASE;
7593         snprintf(desc, ARRAY_SIZE(desc), "DCORE3_EDMA1_QM");
7594         break;
7595     case GAUDI2_EVENT_PDMA0_QM:
7596         qid_base = GAUDI2_QUEUE_ID_PDMA_0_0;
7597         qman_base = mmPDMA0_QM_BASE;
7598         snprintf(desc, ARRAY_SIZE(desc), "PDMA0_QM");
7599         break;
7600     case GAUDI2_EVENT_PDMA1_QM:
7601         qid_base = GAUDI2_QUEUE_ID_PDMA_1_0;
7602         qman_base = mmPDMA1_QM_BASE;
7603         snprintf(desc, ARRAY_SIZE(desc), "PDMA1_QM");
7604         break;
7605     case GAUDI2_EVENT_ROTATOR0_ROT0_QM:
7606         qid_base = GAUDI2_QUEUE_ID_ROT_0_0;
7607         qman_base = mmROT0_QM_BASE;
7608         snprintf(desc, ARRAY_SIZE(desc), "ROTATOR0_QM");
7609         break;
7610     case GAUDI2_EVENT_ROTATOR1_ROT1_QM:
7611         qid_base = GAUDI2_QUEUE_ID_ROT_1_0;
7612         qman_base = mmROT1_QM_BASE;
7613         snprintf(desc, ARRAY_SIZE(desc), "ROTATOR1_QM");
7614         break;
7615     default:
7616         return;
7617     }
7618 
7619     gaudi2_handle_qman_err_generic(hdev, desc, qman_base, qid_base);
7620 
7621     /* Handle EDMA QM SEI here because there is no AXI error response event for EDMA */
7622     if (event_type >= GAUDI2_EVENT_HDMA2_QM && event_type <= GAUDI2_EVENT_HDMA5_QM)
7623         _gaudi2_handle_qm_sei_err(hdev, qman_base);
7624 }
7625 
7626 static void gaudi2_handle_arc_farm_sei_err(struct hl_device *hdev)
7627 {
7628     u32 i, sts_val, sts_clr_val = 0;
7629 
7630     sts_val = RREG32(mmARC_FARM_ARC0_AUX_ARC_SEI_INTR_STS);
7631 
7632     for (i = 0 ; i < GAUDI2_NUM_OF_ARC_SEI_ERR_CAUSE ; i++) {
7633         if (sts_val & BIT(i)) {
7634             dev_err_ratelimited(hdev->dev, "ARC SEI. err cause: %s\n",
7635                         gaudi2_arc_sei_error_cause[i]);
7636             sts_clr_val |= BIT(i);
7637         }
7638     }
7639 
7640     WREG32(mmARC_FARM_ARC0_AUX_ARC_SEI_INTR_CLR, sts_clr_val);
7641 }
7642 
7643 static void gaudi2_handle_cpu_sei_err(struct hl_device *hdev)
7644 {
7645     u32 i, sts_val, sts_clr_val = 0;
7646 
7647     sts_val = RREG32(mmCPU_IF_CPU_SEI_INTR_STS);
7648 
7649     for (i = 0 ; i < GAUDI2_NUM_OF_CPU_SEI_ERR_CAUSE ; i++) {
7650         if (sts_val & BIT(i)) {
7651             dev_err_ratelimited(hdev->dev, "CPU SEI. err cause: %s\n",
7652                         gaudi2_cpu_sei_error_cause[i]);
7653             sts_clr_val |= BIT(i);
7654         }
7655     }
7656 
7657     WREG32(mmCPU_IF_CPU_SEI_INTR_CLR, sts_clr_val);
7658 }
7659 
7660 static void gaudi2_handle_rot_err(struct hl_device *hdev, u8 rot_index,
7661                     struct hl_eq_razwi_with_intr_cause *razwi_with_intr_cause)
7662 {
7663     u64 intr_cause_data = le64_to_cpu(razwi_with_intr_cause->intr_cause.intr_cause_data);
7664     int i;
7665 
7666     for (i = 0 ; i < GAUDI2_NUM_OF_ROT_ERR_CAUSE ; i++)
7667         if (intr_cause_data & BIT(i))
7668             dev_err_ratelimited(hdev->dev, "ROT%u. err cause: %s\n",
7669                         rot_index, guadi2_rot_error_cause[i]);
7670 
7671     /* check if RAZWI happened */
7672     gaudi2_ack_module_razwi_event_handler(hdev, RAZWI_ROT, rot_index, 0,
7673                         &razwi_with_intr_cause->razwi_info);
7674 }
7675 
7676 static void gaudi2_tpc_ack_interrupts(struct hl_device *hdev, u8 tpc_index, char *interrupt_name,
7677                     struct hl_eq_razwi_with_intr_cause *razwi_with_intr_cause)
7678 {
7679     u64 intr_cause_data = le64_to_cpu(razwi_with_intr_cause->intr_cause.intr_cause_data);
7680     int i;
7681 
7682     for (i = 0 ; i < GAUDI2_NUM_OF_TPC_INTR_CAUSE ; i++)
7683         if (intr_cause_data & BIT(i))
7684             dev_err_ratelimited(hdev->dev, "TPC%d_%s interrupt cause: %s\n",
7685                     tpc_index, interrupt_name, gaudi2_tpc_interrupts_cause[i]);
7686 
7687     /* check if RAZWI happened */
7688     gaudi2_ack_module_razwi_event_handler(hdev, RAZWI_TPC, tpc_index, 0,
7689                         &razwi_with_intr_cause->razwi_info);
7690 }
7691 
7692 static void gaudi2_handle_dec_err(struct hl_device *hdev, u8 dec_index, const char *interrupt_name,
7693                 struct hl_eq_razwi_info *razwi_info)
7694 {
7695     u32 sts_addr, sts_val, sts_clr_val = 0;
7696     int i;
7697 
7698     if (dec_index < NUM_OF_VDEC_PER_DCORE * NUM_OF_DCORES)
7699         /* DCORE DEC */
7700         sts_addr = mmDCORE0_VDEC0_BRDG_CTRL_CAUSE_INTR +
7701                 DCORE_OFFSET * (dec_index / NUM_OF_DEC_PER_DCORE) +
7702                 DCORE_VDEC_OFFSET * (dec_index % NUM_OF_DEC_PER_DCORE);
7703     else
7704         /* PCIE DEC */
7705         sts_addr = mmPCIE_VDEC0_BRDG_CTRL_CAUSE_INTR + PCIE_VDEC_OFFSET *
7706                 (dec_index - NUM_OF_VDEC_PER_DCORE * NUM_OF_DCORES);
7707 
7708     sts_val = RREG32(sts_addr);
7709 
7710     for (i = 0 ; i < GAUDI2_NUM_OF_DEC_ERR_CAUSE ; i++) {
7711         if (sts_val & BIT(i)) {
7712             dev_err_ratelimited(hdev->dev, "DEC%u_%s err cause: %s\n",
7713                     dec_index, interrupt_name, gaudi2_dec_error_cause[i]);
7714             sts_clr_val |= BIT(i);
7715         }
7716     }
7717 
7718     /* check if RAZWI happened */
7719     gaudi2_ack_module_razwi_event_handler(hdev, RAZWI_DEC, dec_index, 0, razwi_info);
7720 
7721     /* Write 1 clear errors */
7722     WREG32(sts_addr, sts_clr_val);
7723 }
7724 
7725 static void gaudi2_handle_mme_err(struct hl_device *hdev, u8 mme_index, const char *interrupt_name,
7726                 struct hl_eq_razwi_info *razwi_info)
7727 {
7728     u32 sts_addr, sts_val, sts_clr_addr, sts_clr_val = 0;
7729     int i;
7730 
7731     sts_addr = mmDCORE0_MME_CTRL_LO_INTR_CAUSE + DCORE_OFFSET * mme_index;
7732     sts_clr_addr = mmDCORE0_MME_CTRL_LO_INTR_CLEAR + DCORE_OFFSET * mme_index;
7733 
7734     sts_val = RREG32(sts_addr);
7735 
7736     for (i = 0 ; i < GAUDI2_NUM_OF_MME_ERR_CAUSE ; i++) {
7737         if (sts_val & BIT(i)) {
7738             dev_err_ratelimited(hdev->dev, "MME%u_%s err cause: %s\n",
7739                     mme_index, interrupt_name, guadi2_mme_error_cause[i]);
7740             sts_clr_val |= BIT(i);
7741         }
7742     }
7743 
7744     /* check if RAZWI happened */
7745     for (i = MME_WRITE ; i < MME_INITIATORS_MAX ; i++)
7746         gaudi2_ack_module_razwi_event_handler(hdev, RAZWI_MME, mme_index, i, razwi_info);
7747 
7748     WREG32(sts_clr_addr, sts_clr_val);
7749 }
7750 
7751 static void gaudi2_handle_mme_sbte_err(struct hl_device *hdev, u8 mme_index, u8 sbte_index,
7752                     u64 intr_cause_data)
7753 {
7754     int i;
7755 
7756     for (i = 0 ; i < GAUDI2_NUM_OF_MME_SBTE_ERR_CAUSE ; i++)
7757         if (intr_cause_data & BIT(i))
7758             dev_err_ratelimited(hdev->dev, "MME%uSBTE%u_AXI_ERR_RSP err cause: %s\n",
7759                     mme_index, sbte_index, guadi2_mme_sbte_error_cause[i]);
7760 }
7761 
7762 static void gaudi2_handle_mme_wap_err(struct hl_device *hdev, u8 mme_index,
7763                     struct hl_eq_razwi_info *razwi_info)
7764 {
7765     u32 sts_addr, sts_val, sts_clr_addr, sts_clr_val = 0;
7766     int i;
7767 
7768     sts_addr = mmDCORE0_MME_ACC_INTR_CAUSE + DCORE_OFFSET * mme_index;
7769     sts_clr_addr = mmDCORE0_MME_ACC_INTR_CLEAR + DCORE_OFFSET * mme_index;
7770 
7771     sts_val = RREG32(sts_addr);
7772 
7773     for (i = 0 ; i < GAUDI2_NUM_OF_MME_WAP_ERR_CAUSE ; i++) {
7774         if (sts_val & BIT(i)) {
7775             dev_err_ratelimited(hdev->dev,
7776                     "MME%u_WAP_SOURCE_RESULT_INVALID err cause: %s\n",
7777                     mme_index, guadi2_mme_wap_error_cause[i]);
7778             sts_clr_val |= BIT(i);
7779         }
7780     }
7781 
7782     /* check if RAZWI happened on WAP0/1 */
7783     gaudi2_ack_module_razwi_event_handler(hdev, RAZWI_MME, mme_index, MME_WAP0, razwi_info);
7784     gaudi2_ack_module_razwi_event_handler(hdev, RAZWI_MME, mme_index, MME_WAP1, razwi_info);
7785 
7786     WREG32(sts_clr_addr, sts_clr_val);
7787 }
7788 
7789 static void gaudi2_handle_kdma_core_event(struct hl_device *hdev, u64 intr_cause_data)
7790 {
7791     int i;
7792 
7793     /* If an AXI read or write error is received, an error is reported and
7794      * interrupt message is sent. Due to an HW errata, when reading the cause
7795      * register of the KDMA engine, the reported error is always HBW even if
7796      * the actual error caused by a LBW KDMA transaction.
7797      */
7798     for (i = 0 ; i < GAUDI2_NUM_OF_DMA_CORE_INTR_CAUSE ; i++)
7799         if (intr_cause_data & BIT(i))
7800             dev_err_ratelimited(hdev->dev, "kdma core err cause: %s\n",
7801                         gaudi2_kdma_core_interrupts_cause[i]);
7802 }
7803 
7804 static void gaudi2_handle_dma_core_event(struct hl_device *hdev, u64 intr_cause_data)
7805 {
7806     int i;
7807 
7808     for (i = 0 ; i < GAUDI2_NUM_OF_DMA_CORE_INTR_CAUSE ; i++)
7809         if (intr_cause_data & BIT(i))
7810             dev_err_ratelimited(hdev->dev, "dma core err cause: %s\n",
7811                         gaudi2_dma_core_interrupts_cause[i]);
7812 }
7813 
7814 static void gaudi2_print_pcie_addr_dec_info(struct hl_device *hdev, u64 intr_cause_data)
7815 {
7816     int i;
7817 
7818     for (i = 0 ; i < GAUDI2_NUM_OF_PCIE_ADDR_DEC_ERR_CAUSE; i++)
7819         if (intr_cause_data & BIT_ULL(i))
7820             dev_err_ratelimited(hdev->dev, "PCIE ADDR DEC Error: %s\n",
7821                         gaudi2_pcie_addr_dec_error_cause[i]);
7822 }
7823 
7824 static void gaudi2_handle_pif_fatal(struct hl_device *hdev, u64 intr_cause_data)
7825 
7826 {
7827     int i;
7828 
7829     for (i = 0 ; i < GAUDI2_NUM_OF_PMMU_FATAL_ERR_CAUSE ; i++) {
7830         if (intr_cause_data & BIT_ULL(i))
7831             dev_err_ratelimited(hdev->dev, "PMMU PIF err cause: %s\n",
7832                     gaudi2_pmmu_fatal_interrupts_cause[i]);
7833     }
7834 }
7835 
7836 static void gaudi2_handle_hif_fatal(struct hl_device *hdev, u16 event_type, u64 intr_cause_data)
7837 {
7838     u32 dcore_id, hif_id;
7839     int i;
7840 
7841     dcore_id = (event_type - GAUDI2_EVENT_HIF0_FATAL) / 4;
7842     hif_id = (event_type - GAUDI2_EVENT_HIF0_FATAL) % 4;
7843 
7844     for (i = 0 ; i < GAUDI2_NUM_OF_HIF_FATAL_ERR_CAUSE ; i++) {
7845         if (intr_cause_data & BIT_ULL(i))
7846             dev_err_ratelimited(hdev->dev, "DCORE%u_HIF%u: %s\n", dcore_id, hif_id,
7847                     gaudi2_hif_fatal_interrupts_cause[i]);
7848     }
7849 }
7850 
7851 static void gaudi2_handle_page_error(struct hl_device *hdev, u64 mmu_base, bool is_pmmu)
7852 {
7853     u32 valid, val;
7854     u64 addr;
7855 
7856     valid = RREG32(mmu_base + MMU_OFFSET(mmDCORE0_HMMU0_MMU_ACCESS_PAGE_ERROR_VALID));
7857 
7858     if (!(valid & DCORE0_HMMU0_MMU_ACCESS_PAGE_ERROR_VALID_PAGE_ERR_VALID_ENTRY_MASK))
7859         return;
7860 
7861     val = RREG32(mmu_base + MMU_OFFSET(mmDCORE0_HMMU0_MMU_PAGE_ERROR_CAPTURE));
7862     addr = val & DCORE0_HMMU0_MMU_PAGE_ERROR_CAPTURE_VA_63_32_MASK;
7863     addr <<= 32;
7864     addr |= RREG32(mmu_base + MMU_OFFSET(mmDCORE0_HMMU0_MMU_PAGE_ERROR_CAPTURE_VA));
7865 
7866     dev_err_ratelimited(hdev->dev, "%s page fault on va 0x%llx\n",
7867                 is_pmmu ? "PMMU" : "HMMU", addr);
7868 
7869     WREG32(mmu_base + MMU_OFFSET(mmDCORE0_HMMU0_MMU_PAGE_ERROR_CAPTURE), 0);
7870 }
7871 
7872 static void gaudi2_handle_access_error(struct hl_device *hdev, u64 mmu_base, bool is_pmmu)
7873 {
7874     u32 valid, val;
7875     u64 addr;
7876 
7877     valid = RREG32(mmu_base + MMU_OFFSET(mmDCORE0_HMMU0_MMU_ACCESS_PAGE_ERROR_VALID));
7878 
7879     if (!(valid & DCORE0_HMMU0_MMU_ACCESS_PAGE_ERROR_VALID_ACCESS_ERR_VALID_ENTRY_MASK))
7880         return;
7881 
7882     val = RREG32(mmu_base + MMU_OFFSET(mmDCORE0_HMMU0_MMU_ACCESS_ERROR_CAPTURE));
7883     addr = val & DCORE0_HMMU0_MMU_ACCESS_ERROR_CAPTURE_VA_63_32_MASK;
7884     addr <<= 32;
7885     addr |= RREG32(mmu_base + MMU_OFFSET(mmDCORE0_HMMU0_MMU_ACCESS_ERROR_CAPTURE_VA));
7886 
7887     dev_err_ratelimited(hdev->dev, "%s access error on va 0x%llx\n",
7888                 is_pmmu ? "PMMU" : "HMMU", addr);
7889     WREG32(mmu_base + MMU_OFFSET(mmDCORE0_HMMU0_MMU_ACCESS_ERROR_CAPTURE), 0);
7890 }
7891 
7892 static void gaudi2_handle_mmu_spi_sei_generic(struct hl_device *hdev, const char *mmu_name,
7893                         u64 mmu_base, bool is_pmmu)
7894 {
7895     u32 spi_sei_cause, interrupt_clr = 0x0;
7896     int i;
7897 
7898     spi_sei_cause = RREG32(mmu_base + MMU_SPI_SEI_CAUSE_OFFSET);
7899 
7900     for (i = 0 ; i < GAUDI2_NUM_OF_MMU_SPI_SEI_CAUSE ; i++) {
7901         if (spi_sei_cause & BIT(i)) {
7902             dev_err_ratelimited(hdev->dev, "%s SPI_SEI ERR. err cause: %s\n",
7903                         mmu_name, gaudi2_mmu_spi_sei[i].cause);
7904 
7905             if (i == 0)
7906                 gaudi2_handle_page_error(hdev, mmu_base, is_pmmu);
7907             else if (i == 1)
7908                 gaudi2_handle_access_error(hdev, mmu_base, is_pmmu);
7909 
7910             if (gaudi2_mmu_spi_sei[i].clear_bit >= 0)
7911                 interrupt_clr |= BIT(gaudi2_mmu_spi_sei[i].clear_bit);
7912         }
7913     }
7914 
7915     /* Clear cause */
7916     WREG32_AND(mmu_base + MMU_SPI_SEI_CAUSE_OFFSET, ~spi_sei_cause);
7917 
7918     /* Clear interrupt */
7919     WREG32(mmu_base + MMU_INTERRUPT_CLR_OFFSET, interrupt_clr);
7920 }
7921 
7922 static bool gaudi2_handle_sm_err(struct hl_device *hdev, u8 sm_index)
7923 {
7924     u32 sei_cause_addr, sei_cause_val, sei_cause_cause, sei_cause_log;
7925     u32 cq_intr_addr, cq_intr_val, cq_intr_queue_index;
7926     bool reset = true;
7927     int i;
7928 
7929     sei_cause_addr = mmDCORE0_SYNC_MNGR_GLBL_SM_SEI_CAUSE + DCORE_OFFSET * sm_index;
7930     cq_intr_addr = mmDCORE0_SYNC_MNGR_GLBL_CQ_INTR + DCORE_OFFSET * sm_index;
7931 
7932     sei_cause_val = RREG32(sei_cause_addr);
7933     sei_cause_cause = FIELD_GET(DCORE0_SYNC_MNGR_GLBL_SM_SEI_CAUSE_CAUSE_MASK, sei_cause_val);
7934     cq_intr_val = RREG32(cq_intr_addr);
7935 
7936     /* SEI interrupt */
7937     if (sei_cause_cause) {
7938         /* There are corresponding SEI_CAUSE_log bits for every SEI_CAUSE_cause bit */
7939         sei_cause_log = FIELD_GET(DCORE0_SYNC_MNGR_GLBL_SM_SEI_CAUSE_LOG_MASK,
7940                     sei_cause_val);
7941 
7942         for (i = 0 ; i < GAUDI2_NUM_OF_SM_SEI_ERR_CAUSE ; i++) {
7943             if (!(sei_cause_cause & BIT(i)))
7944                 continue;
7945 
7946             dev_err_ratelimited(hdev->dev, "SM%u SEI ERR. err cause: %s. %s: 0x%X\n",
7947                     sm_index,
7948                     gaudi2_sm_sei_cause[i].cause_name,
7949                     gaudi2_sm_sei_cause[i].log_name,
7950                     sei_cause_log & gaudi2_sm_sei_cause[i].log_mask);
7951 
7952             /* Due to a potential H/W issue, do not reset upon BRESP errors */
7953             if (i == 2)
7954                 reset = false;
7955             break;
7956         }
7957 
7958         /* Clear SM_SEI_CAUSE */
7959         WREG32(sei_cause_addr, 0);
7960     }
7961 
7962     /* CQ interrupt */
7963     if (cq_intr_val & DCORE0_SYNC_MNGR_GLBL_CQ_INTR_CQ_SEC_INTR_MASK) {
7964         cq_intr_queue_index =
7965                 FIELD_GET(DCORE0_SYNC_MNGR_GLBL_CQ_INTR_CQ_INTR_QUEUE_INDEX_MASK,
7966                     cq_intr_val);
7967 
7968         dev_err_ratelimited(hdev->dev, "SM%u err. err cause: CQ_INTR. queue index: %u\n",
7969                 sm_index, cq_intr_queue_index);
7970 
7971         /* Clear CQ_INTR */
7972         WREG32(cq_intr_addr, 0);
7973     }
7974 
7975     return reset;
7976 }
7977 
7978 static void gaudi2_handle_mmu_spi_sei_err(struct hl_device *hdev, u16 event_type)
7979 {
7980     bool is_pmmu = false;
7981     char desc[32];
7982     u64 mmu_base;
7983     u8 index;
7984 
7985     switch (event_type) {
7986     case GAUDI2_EVENT_HMMU0_PAGE_FAULT_OR_WR_PERM ... GAUDI2_EVENT_HMMU3_SECURITY_ERROR:
7987         index = (event_type - GAUDI2_EVENT_HMMU0_PAGE_FAULT_OR_WR_PERM) / 3;
7988         mmu_base = mmDCORE0_HMMU0_MMU_BASE + index * DCORE_HMMU_OFFSET;
7989         snprintf(desc, ARRAY_SIZE(desc), "DCORE0_HMMU%d", index);
7990         break;
7991     case GAUDI2_EVENT_HMMU_0_AXI_ERR_RSP ... GAUDI2_EVENT_HMMU_3_AXI_ERR_RSP:
7992         index = (event_type - GAUDI2_EVENT_HMMU_0_AXI_ERR_RSP);
7993         mmu_base = mmDCORE0_HMMU0_MMU_BASE + index * DCORE_HMMU_OFFSET;
7994         snprintf(desc, ARRAY_SIZE(desc), "DCORE0_HMMU%d", index);
7995         break;
7996     case GAUDI2_EVENT_HMMU8_PAGE_FAULT_WR_PERM ... GAUDI2_EVENT_HMMU11_SECURITY_ERROR:
7997         index = (event_type - GAUDI2_EVENT_HMMU8_PAGE_FAULT_WR_PERM) / 3;
7998         mmu_base = mmDCORE1_HMMU0_MMU_BASE + index * DCORE_HMMU_OFFSET;
7999         snprintf(desc, ARRAY_SIZE(desc), "DCORE1_HMMU%d", index);
8000         break;
8001     case GAUDI2_EVENT_HMMU_8_AXI_ERR_RSP ... GAUDI2_EVENT_HMMU_11_AXI_ERR_RSP:
8002         index = (event_type - GAUDI2_EVENT_HMMU_8_AXI_ERR_RSP);
8003         mmu_base = mmDCORE1_HMMU0_MMU_BASE + index * DCORE_HMMU_OFFSET;
8004         snprintf(desc, ARRAY_SIZE(desc), "DCORE1_HMMU%d", index);
8005         break;
8006     case GAUDI2_EVENT_HMMU7_PAGE_FAULT_WR_PERM ... GAUDI2_EVENT_HMMU4_SECURITY_ERROR:
8007         index = (event_type - GAUDI2_EVENT_HMMU7_PAGE_FAULT_WR_PERM) / 3;
8008         mmu_base = mmDCORE2_HMMU0_MMU_BASE + index * DCORE_HMMU_OFFSET;
8009         snprintf(desc, ARRAY_SIZE(desc), "DCORE2_HMMU%d", index);
8010         break;
8011     case GAUDI2_EVENT_HMMU_7_AXI_ERR_RSP ... GAUDI2_EVENT_HMMU_4_AXI_ERR_RSP:
8012         index = (event_type - GAUDI2_EVENT_HMMU_7_AXI_ERR_RSP);
8013         mmu_base = mmDCORE2_HMMU0_MMU_BASE + index * DCORE_HMMU_OFFSET;
8014         snprintf(desc, ARRAY_SIZE(desc), "DCORE2_HMMU%d", index);
8015         break;
8016     case GAUDI2_EVENT_HMMU15_PAGE_FAULT_WR_PERM ... GAUDI2_EVENT_HMMU12_SECURITY_ERROR:
8017         index = (event_type - GAUDI2_EVENT_HMMU15_PAGE_FAULT_WR_PERM) / 3;
8018         mmu_base = mmDCORE3_HMMU0_MMU_BASE + index * DCORE_HMMU_OFFSET;
8019         snprintf(desc, ARRAY_SIZE(desc), "DCORE3_HMMU%d", index);
8020         break;
8021     case GAUDI2_EVENT_HMMU_15_AXI_ERR_RSP ... GAUDI2_EVENT_HMMU_12_AXI_ERR_RSP:
8022         index = (event_type - GAUDI2_EVENT_HMMU_15_AXI_ERR_RSP);
8023         mmu_base = mmDCORE3_HMMU0_MMU_BASE + index * DCORE_HMMU_OFFSET;
8024         snprintf(desc, ARRAY_SIZE(desc), "DCORE3_HMMU%d", index);
8025         break;
8026     case GAUDI2_EVENT_PMMU0_PAGE_FAULT_WR_PERM ... GAUDI2_EVENT_PMMU0_SECURITY_ERROR:
8027     case GAUDI2_EVENT_PMMU_AXI_ERR_RSP_0:
8028         is_pmmu = true;
8029         mmu_base = mmPMMU_HBW_MMU_BASE;
8030         snprintf(desc, ARRAY_SIZE(desc), "PMMU");
8031         break;
8032     default:
8033         return;
8034     }
8035 
8036     gaudi2_handle_mmu_spi_sei_generic(hdev, desc, mmu_base, is_pmmu);
8037 }
8038 
8039 
8040 /* returns true if hard reset is required (ECC DERR or Read parity), false otherwise (ECC SERR) */
8041 static bool gaudi2_hbm_sei_handle_read_err(struct hl_device *hdev,
8042             struct hl_eq_hbm_sei_read_err_intr_info *rd_err_data, u32 err_cnt)
8043 {
8044     u32 addr, beat, beat_shift;
8045     bool rc = false;
8046 
8047     dev_err_ratelimited(hdev->dev,
8048             "READ ERROR count: ECC SERR: %d, ECC DERR: %d, RD_PARITY: %d\n",
8049             FIELD_GET(HBM_ECC_SERR_CNTR_MASK, err_cnt),
8050             FIELD_GET(HBM_ECC_DERR_CNTR_MASK, err_cnt),
8051             FIELD_GET(HBM_RD_PARITY_CNTR_MASK, err_cnt));
8052 
8053     addr = le32_to_cpu(rd_err_data->dbg_rd_err_addr.rd_addr_val);
8054     dev_err_ratelimited(hdev->dev,
8055             "READ ERROR address: sid(%u), bg(%u), ba(%u), col(%u), row(%u)\n",
8056             FIELD_GET(HBM_RD_ADDR_SID_MASK, addr),
8057             FIELD_GET(HBM_RD_ADDR_BG_MASK, addr),
8058             FIELD_GET(HBM_RD_ADDR_BA_MASK, addr),
8059             FIELD_GET(HBM_RD_ADDR_COL_MASK, addr),
8060             FIELD_GET(HBM_RD_ADDR_ROW_MASK, addr));
8061 
8062     /* For each beat (RDQS edge), look for possible errors and print relevant info */
8063     for (beat = 0 ; beat < 4 ; beat++) {
8064         if (le32_to_cpu(rd_err_data->dbg_rd_err_misc) &
8065             (HBM_RD_ERR_SERR_BEAT0_MASK << beat))
8066             dev_err_ratelimited(hdev->dev, "Beat%d ECC SERR: DM: %#x, Syndrome: %#x\n",
8067                         beat,
8068                         le32_to_cpu(rd_err_data->dbg_rd_err_dm),
8069                         le32_to_cpu(rd_err_data->dbg_rd_err_syndrome));
8070 
8071         if (le32_to_cpu(rd_err_data->dbg_rd_err_misc) &
8072             (HBM_RD_ERR_DERR_BEAT0_MASK << beat)) {
8073             dev_err_ratelimited(hdev->dev, "Beat%d ECC DERR: DM: %#x, Syndrome: %#x\n",
8074                         beat,
8075                         le32_to_cpu(rd_err_data->dbg_rd_err_dm),
8076                         le32_to_cpu(rd_err_data->dbg_rd_err_syndrome));
8077             rc |= true;
8078         }
8079 
8080         beat_shift = beat * HBM_RD_ERR_BEAT_SHIFT;
8081         if (le32_to_cpu(rd_err_data->dbg_rd_err_misc) &
8082             (HBM_RD_ERR_PAR_ERR_BEAT0_MASK << beat_shift)) {
8083             dev_err_ratelimited(hdev->dev,
8084                     "Beat%d read PARITY: DM: %#x, PAR data: %#x\n",
8085                     beat,
8086                     le32_to_cpu(rd_err_data->dbg_rd_err_dm),
8087                     (le32_to_cpu(rd_err_data->dbg_rd_err_misc) &
8088                         (HBM_RD_ERR_PAR_DATA_BEAT0_MASK << beat_shift)) >>
8089                         (HBM_RD_ERR_PAR_DATA_BEAT0_SHIFT + beat_shift));
8090             rc |= true;
8091         }
8092 
8093         dev_err_ratelimited(hdev->dev, "Beat%d DQ data:\n", beat);
8094         dev_err_ratelimited(hdev->dev, "\t0x%08x\n",
8095                     le32_to_cpu(rd_err_data->dbg_rd_err_data[beat * 2]));
8096         dev_err_ratelimited(hdev->dev, "\t0x%08x\n",
8097                     le32_to_cpu(rd_err_data->dbg_rd_err_data[beat * 2 + 1]));
8098     }
8099 
8100     return rc;
8101 }
8102 
8103 static void gaudi2_hbm_sei_print_wr_par_info(struct hl_device *hdev,
8104             struct hl_eq_hbm_sei_wr_par_intr_info *wr_par_err_data, u32 err_cnt)
8105 {
8106     struct hbm_sei_wr_cmd_address *wr_cmd_addr = wr_par_err_data->dbg_last_wr_cmds;
8107     u32 i, curr_addr, derr = wr_par_err_data->dbg_derr;
8108 
8109     dev_err_ratelimited(hdev->dev, "WRITE PARITY ERROR count: %d\n", err_cnt);
8110 
8111     dev_err_ratelimited(hdev->dev, "CK-0 DERR: 0x%02x, CK-1 DERR: 0x%02x\n",
8112                 derr & 0x3, derr & 0xc);
8113 
8114     /* JIRA H6-3286 - the following prints may not be valid */
8115     dev_err_ratelimited(hdev->dev, "Last latched write commands addresses:\n");
8116     for (i = 0 ; i < HBM_WR_PAR_CMD_LIFO_LEN ; i++) {
8117         curr_addr = le32_to_cpu(wr_cmd_addr[i].dbg_wr_cmd_addr);
8118         dev_err_ratelimited(hdev->dev,
8119                 "\twrite cmd[%u]: Address: SID(%u) BG(%u) BA(%u) COL(%u).\n",
8120                 i,
8121                 FIELD_GET(WR_PAR_LAST_CMD_SID_MASK, curr_addr),
8122                 FIELD_GET(WR_PAR_LAST_CMD_BG_MASK, curr_addr),
8123                 FIELD_GET(WR_PAR_LAST_CMD_BA_MASK, curr_addr),
8124                 FIELD_GET(WR_PAR_LAST_CMD_COL_MASK, curr_addr));
8125     }
8126 }
8127 
8128 static void gaudi2_hbm_sei_print_ca_par_info(struct hl_device *hdev,
8129         struct hl_eq_hbm_sei_ca_par_intr_info *ca_par_err_data, u32 err_cnt)
8130 {
8131     __le32 *col_cmd = ca_par_err_data->dbg_col;
8132     __le16 *row_cmd = ca_par_err_data->dbg_row;
8133     u32 i;
8134 
8135     dev_err_ratelimited(hdev->dev, "CA ERROR count: %d\n", err_cnt);
8136 
8137     dev_err_ratelimited(hdev->dev, "Last latched C&R bus commands:\n");
8138     for (i = 0 ; i < HBM_CA_ERR_CMD_LIFO_LEN ; i++)
8139         dev_err_ratelimited(hdev->dev, "cmd%u: ROW(0x%04x) COL(0x%05x)\n", i,
8140             le16_to_cpu(row_cmd[i]) & (u16)GENMASK(13, 0),
8141             le32_to_cpu(col_cmd[i]) & (u32)GENMASK(17, 0));
8142 }
8143 
8144 /* Returns true if hard reset is needed or false otherwise */
8145 static bool gaudi2_handle_hbm_mc_sei_err(struct hl_device *hdev, u16 event_type,
8146                     struct hl_eq_hbm_sei_data *sei_data)
8147 {
8148     bool require_hard_reset = false;
8149     u32 hbm_id, mc_id, cause_idx;
8150 
8151     hbm_id = (event_type - GAUDI2_EVENT_HBM0_MC0_SEI_SEVERE) / 4;
8152     mc_id = ((event_type - GAUDI2_EVENT_HBM0_MC0_SEI_SEVERE) / 2) % 2;
8153 
8154     cause_idx = sei_data->hdr.sei_cause;
8155     if (cause_idx > GAUDI2_NUM_OF_HBM_SEI_CAUSE - 1) {
8156         dev_err_ratelimited(hdev->dev, "Invalid HBM SEI event cause (%d) provided by FW\n",
8157                     cause_idx);
8158         return true;
8159     }
8160 
8161     dev_err_ratelimited(hdev->dev,
8162         "System Error Interrupt - HBM(%u) MC(%u) MC_CH(%u) MC_PC(%u). Critical(%u). Error cause: %s\n",
8163         hbm_id, mc_id, sei_data->hdr.mc_channel, sei_data->hdr.mc_pseudo_channel,
8164         sei_data->hdr.is_critical, hbm_mc_sei_cause[cause_idx]);
8165 
8166     /* Print error-specific info */
8167     switch (cause_idx) {
8168     case HBM_SEI_CATTRIP:
8169         require_hard_reset = true;
8170         break;
8171 
8172     case  HBM_SEI_CMD_PARITY_EVEN:
8173         gaudi2_hbm_sei_print_ca_par_info(hdev, &sei_data->ca_parity_even_info,
8174                         le32_to_cpu(sei_data->hdr.cnt));
8175         require_hard_reset = true;
8176         break;
8177 
8178     case  HBM_SEI_CMD_PARITY_ODD:
8179         gaudi2_hbm_sei_print_ca_par_info(hdev, &sei_data->ca_parity_odd_info,
8180                         le32_to_cpu(sei_data->hdr.cnt));
8181         require_hard_reset = true;
8182         break;
8183 
8184     case HBM_SEI_WRITE_DATA_PARITY_ERR:
8185         gaudi2_hbm_sei_print_wr_par_info(hdev, &sei_data->wr_parity_info,
8186                         le32_to_cpu(sei_data->hdr.cnt));
8187         require_hard_reset = true;
8188         break;
8189 
8190     case HBM_SEI_READ_ERR:
8191         /* Unlike other SEI events, read error requires further processing of the
8192          * raw data in order to determine the root cause.
8193          */
8194         require_hard_reset = gaudi2_hbm_sei_handle_read_err(hdev,
8195                                 &sei_data->read_err_info,
8196                                 le32_to_cpu(sei_data->hdr.cnt));
8197         break;
8198 
8199     default:
8200         break;
8201     }
8202 
8203     require_hard_reset |= !!sei_data->hdr.is_critical;
8204 
8205     return require_hard_reset;
8206 }
8207 
8208 static void gaudi2_handle_hbm_cattrip(struct hl_device *hdev, u64 intr_cause_data)
8209 {
8210     dev_err(hdev->dev,
8211         "HBM catastrophic temperature error (CATTRIP) cause %#llx\n",
8212         intr_cause_data);
8213 }
8214 
8215 static void gaudi2_handle_hbm_mc_spi(struct hl_device *hdev, u64 intr_cause_data)
8216 {
8217     u32 i;
8218 
8219     for (i = 0 ; i < GAUDI2_NUM_OF_HBM_MC_SPI_CAUSE ; i++)
8220         if (intr_cause_data & hbm_mc_spi[i].mask)
8221             dev_dbg(hdev->dev, "HBM spi event: notification cause(%s)\n",
8222                 hbm_mc_spi[i].cause);
8223 }
8224 
8225 static void gaudi2_print_clk_change_info(struct hl_device *hdev, u16 event_type)
8226 {
8227     ktime_t zero_time = ktime_set(0, 0);
8228 
8229     mutex_lock(&hdev->clk_throttling.lock);
8230 
8231     switch (event_type) {
8232     case GAUDI2_EVENT_CPU_FIX_POWER_ENV_S:
8233         hdev->clk_throttling.current_reason |= HL_CLK_THROTTLE_POWER;
8234         hdev->clk_throttling.aggregated_reason |= HL_CLK_THROTTLE_POWER;
8235         hdev->clk_throttling.timestamp[HL_CLK_THROTTLE_TYPE_POWER].start = ktime_get();
8236         hdev->clk_throttling.timestamp[HL_CLK_THROTTLE_TYPE_POWER].end = zero_time;
8237         dev_info_ratelimited(hdev->dev, "Clock throttling due to power consumption\n");
8238         break;
8239 
8240     case GAUDI2_EVENT_CPU_FIX_POWER_ENV_E:
8241         hdev->clk_throttling.current_reason &= ~HL_CLK_THROTTLE_POWER;
8242         hdev->clk_throttling.timestamp[HL_CLK_THROTTLE_TYPE_POWER].end = ktime_get();
8243         dev_info_ratelimited(hdev->dev, "Power envelop is safe, back to optimal clock\n");
8244         break;
8245 
8246     case GAUDI2_EVENT_CPU_FIX_THERMAL_ENV_S:
8247         hdev->clk_throttling.current_reason |= HL_CLK_THROTTLE_THERMAL;
8248         hdev->clk_throttling.aggregated_reason |= HL_CLK_THROTTLE_THERMAL;
8249         hdev->clk_throttling.timestamp[HL_CLK_THROTTLE_TYPE_THERMAL].start = ktime_get();
8250         hdev->clk_throttling.timestamp[HL_CLK_THROTTLE_TYPE_THERMAL].end = zero_time;
8251         dev_info_ratelimited(hdev->dev, "Clock throttling due to overheating\n");
8252         break;
8253 
8254     case GAUDI2_EVENT_CPU_FIX_THERMAL_ENV_E:
8255         hdev->clk_throttling.current_reason &= ~HL_CLK_THROTTLE_THERMAL;
8256         hdev->clk_throttling.timestamp[HL_CLK_THROTTLE_TYPE_THERMAL].end = ktime_get();
8257         dev_info_ratelimited(hdev->dev, "Thermal envelop is safe, back to optimal clock\n");
8258         break;
8259 
8260     default:
8261         dev_err(hdev->dev, "Received invalid clock change event %d\n", event_type);
8262         break;
8263     }
8264 
8265     mutex_unlock(&hdev->clk_throttling.lock);
8266 }
8267 
8268 static void gaudi2_print_out_of_sync_info(struct hl_device *hdev,
8269                     struct cpucp_pkt_sync_err *sync_err)
8270 {
8271     struct hl_hw_queue *q = &hdev->kernel_queues[GAUDI2_QUEUE_ID_CPU_PQ];
8272 
8273     dev_err(hdev->dev, "Out of sync with FW, FW: pi=%u, ci=%u, LKD: pi=%u, ci=%u\n",
8274             sync_err->pi, sync_err->ci, q->pi, atomic_read(&q->ci));
8275 }
8276 
8277 static void gaudi2_handle_pcie_p2p_msix(struct hl_device *hdev)
8278 {
8279     u32 p2p_intr, msix_gw_intr;
8280 
8281     p2p_intr = RREG32(mmPCIE_WRAP_P2P_INTR);
8282     msix_gw_intr = RREG32(mmPCIE_WRAP_MSIX_GW_INTR);
8283 
8284     if (p2p_intr) {
8285         dev_err_ratelimited(hdev->dev,
8286             "pcie p2p transaction terminated due to security, req_id(0x%x)\n",
8287             RREG32(mmPCIE_WRAP_P2P_REQ_ID));
8288 
8289         WREG32(mmPCIE_WRAP_P2P_INTR, 0x1);
8290     }
8291 
8292     if (msix_gw_intr) {
8293         dev_err_ratelimited(hdev->dev,
8294             "pcie msi-x gen denied due to vector num check failure, vec(0x%X)\n",
8295             RREG32(mmPCIE_WRAP_MSIX_GW_VEC));
8296 
8297         WREG32(mmPCIE_WRAP_MSIX_GW_INTR, 0x1);
8298     }
8299 }
8300 
8301 static void gaudi2_handle_pcie_drain(struct hl_device *hdev,
8302             struct hl_eq_pcie_drain_ind_data *drain_data)
8303 {
8304     u64 lbw_rd, lbw_wr, hbw_rd, hbw_wr, cause;
8305 
8306     cause = le64_to_cpu(drain_data->intr_cause.intr_cause_data);
8307     lbw_rd = le64_to_cpu(drain_data->drain_rd_addr_lbw);
8308     lbw_wr = le64_to_cpu(drain_data->drain_wr_addr_lbw);
8309     hbw_rd = le64_to_cpu(drain_data->drain_rd_addr_hbw);
8310     hbw_wr = le64_to_cpu(drain_data->drain_wr_addr_hbw);
8311 
8312     if (cause & BIT_ULL(0))
8313         dev_err_ratelimited(hdev->dev,
8314             "PCIE AXI drain LBW completed, read_err %u, write_err %u\n",
8315             !!lbw_rd, !!lbw_wr);
8316 
8317     if (cause & BIT_ULL(1))
8318         dev_err_ratelimited(hdev->dev,
8319             "PCIE AXI drain HBW completed, raddr %#llx, waddr %#llx\n",
8320             hbw_rd, hbw_wr);
8321 }
8322 
8323 static void gaudi2_handle_psoc_drain(struct hl_device *hdev, u64 intr_cause_data)
8324 {
8325     int i;
8326 
8327     for (i = 0 ; i < GAUDI2_NUM_OF_AXI_DRAIN_ERR_CAUSE ; i++) {
8328         if (intr_cause_data & BIT_ULL(i))
8329             dev_err_ratelimited(hdev->dev, "PSOC %s completed\n",
8330                 gaudi2_psoc_axi_drain_interrupts_cause[i]);
8331     }
8332 }
8333 
8334 static void gaudi2_print_cpu_pkt_failure_info(struct hl_device *hdev,
8335                     struct cpucp_pkt_sync_err *sync_err)
8336 {
8337     struct hl_hw_queue *q = &hdev->kernel_queues[GAUDI2_QUEUE_ID_CPU_PQ];
8338 
8339     dev_warn(hdev->dev,
8340         "FW reported sanity check failure, FW: pi=%u, ci=%u, LKD: pi=%u, ci=%u\n",
8341         sync_err->pi, sync_err->ci, q->pi, atomic_read(&q->ci));
8342 }
8343 
8344 static void hl_arc_event_handle(struct hl_device *hdev,
8345                     struct hl_eq_engine_arc_intr_data *data)
8346 {
8347     struct hl_engine_arc_dccm_queue_full_irq *q;
8348     u32 intr_type, engine_id;
8349     u64 payload;
8350 
8351     intr_type = le32_to_cpu(data->intr_type);
8352     engine_id = le32_to_cpu(data->engine_id);
8353     payload = le64_to_cpu(data->payload);
8354 
8355     switch (intr_type) {
8356     case ENGINE_ARC_DCCM_QUEUE_FULL_IRQ:
8357         q = (struct hl_engine_arc_dccm_queue_full_irq *) &payload;
8358 
8359         dev_err_ratelimited(hdev->dev,
8360                 "ARC DCCM Full event: EngId: %u, Intr_type: %u, Qidx: %u\n",
8361                 engine_id, intr_type, q->queue_index);
8362         break;
8363     default:
8364         dev_err_ratelimited(hdev->dev, "Unknown ARC event type\n");
8365     }
8366 }
8367 
8368 static void gaudi2_handle_eqe(struct hl_device *hdev, struct hl_eq_entry *eq_entry)
8369 {
8370     u32 ctl, reset_flags = HL_DRV_RESET_HARD | HL_DRV_RESET_DELAY;
8371     struct gaudi2_device *gaudi2 = hdev->asic_specific;
8372     bool reset_required = false, skip_reset = false;
8373     int index, sbte_index;
8374     u16 event_type;
8375 
8376     ctl = le32_to_cpu(eq_entry->hdr.ctl);
8377     event_type = ((ctl & EQ_CTL_EVENT_TYPE_MASK) >> EQ_CTL_EVENT_TYPE_SHIFT);
8378 
8379     if (event_type >= GAUDI2_EVENT_SIZE) {
8380         dev_err(hdev->dev, "Event type %u exceeds maximum of %u",
8381                 event_type, GAUDI2_EVENT_SIZE - 1);
8382         return;
8383     }
8384 
8385     gaudi2->events_stat[event_type]++;
8386     gaudi2->events_stat_aggregate[event_type]++;
8387 
8388     gaudi2_print_irq_info(hdev, event_type);
8389 
8390     switch (event_type) {
8391     case GAUDI2_EVENT_PCIE_CORE_SERR ... GAUDI2_EVENT_ARC0_ECC_DERR:
8392         fallthrough;
8393     case GAUDI2_EVENT_ROTATOR0_SERR ... GAUDI2_EVENT_ROTATOR1_DERR:
8394         reset_flags |= HL_DRV_RESET_FW_FATAL_ERR;
8395         reset_required = gaudi2_handle_ecc_event(hdev, event_type, &eq_entry->ecc_data);
8396         break;
8397 
8398     case GAUDI2_EVENT_TPC0_QM ... GAUDI2_EVENT_PDMA1_QM:
8399         fallthrough;
8400     case GAUDI2_EVENT_ROTATOR0_ROT0_QM ... GAUDI2_EVENT_ROTATOR1_ROT1_QM:
8401         fallthrough;
8402     case GAUDI2_EVENT_NIC0_QM0 ... GAUDI2_EVENT_NIC11_QM1:
8403         gaudi2_handle_qman_err(hdev, event_type);
8404         break;
8405 
8406     case GAUDI2_EVENT_ARC_AXI_ERROR_RESPONSE_0:
8407         reset_flags |= HL_DRV_RESET_FW_FATAL_ERR;
8408         gaudi2_handle_arc_farm_sei_err(hdev);
8409         break;
8410 
8411     case GAUDI2_EVENT_CPU_AXI_ERR_RSP:
8412         gaudi2_handle_cpu_sei_err(hdev);
8413         break;
8414 
8415     case GAUDI2_EVENT_PDMA_CH0_AXI_ERR_RSP:
8416     case GAUDI2_EVENT_PDMA_CH1_AXI_ERR_RSP:
8417         reset_flags |= HL_DRV_RESET_FW_FATAL_ERR;
8418         gaudi2_handle_qm_sei_err(hdev, event_type, &eq_entry->razwi_info);
8419         break;
8420 
8421     case GAUDI2_EVENT_ROTATOR0_AXI_ERROR_RESPONSE:
8422     case GAUDI2_EVENT_ROTATOR1_AXI_ERROR_RESPONSE:
8423         index = event_type - GAUDI2_EVENT_ROTATOR0_AXI_ERROR_RESPONSE;
8424         gaudi2_handle_rot_err(hdev, index, &eq_entry->razwi_with_intr_cause);
8425         gaudi2_handle_qm_sei_err(hdev, event_type, NULL);
8426         break;
8427 
8428     case GAUDI2_EVENT_TPC0_AXI_ERR_RSP ... GAUDI2_EVENT_TPC24_AXI_ERR_RSP:
8429         index = event_type - GAUDI2_EVENT_TPC0_AXI_ERR_RSP;
8430         gaudi2_tpc_ack_interrupts(hdev, index, "AXI_ERR_RSP",
8431                         &eq_entry->razwi_with_intr_cause);
8432         gaudi2_handle_qm_sei_err(hdev, event_type, NULL);
8433         break;
8434 
8435     case GAUDI2_EVENT_DEC0_AXI_ERR_RSPONSE ... GAUDI2_EVENT_DEC9_AXI_ERR_RSPONSE:
8436         index = event_type - GAUDI2_EVENT_DEC0_AXI_ERR_RSPONSE;
8437         gaudi2_handle_dec_err(hdev, index, "AXI_ERR_RESPONSE", &eq_entry->razwi_info);
8438         break;
8439 
8440     case GAUDI2_EVENT_TPC0_KERNEL_ERR:
8441     case GAUDI2_EVENT_TPC1_KERNEL_ERR:
8442     case GAUDI2_EVENT_TPC2_KERNEL_ERR:
8443     case GAUDI2_EVENT_TPC3_KERNEL_ERR:
8444     case GAUDI2_EVENT_TPC4_KERNEL_ERR:
8445     case GAUDI2_EVENT_TPC5_KERNEL_ERR:
8446     case GAUDI2_EVENT_TPC6_KERNEL_ERR:
8447     case GAUDI2_EVENT_TPC7_KERNEL_ERR:
8448     case GAUDI2_EVENT_TPC8_KERNEL_ERR:
8449     case GAUDI2_EVENT_TPC9_KERNEL_ERR:
8450     case GAUDI2_EVENT_TPC10_KERNEL_ERR:
8451     case GAUDI2_EVENT_TPC11_KERNEL_ERR:
8452     case GAUDI2_EVENT_TPC12_KERNEL_ERR:
8453     case GAUDI2_EVENT_TPC13_KERNEL_ERR:
8454     case GAUDI2_EVENT_TPC14_KERNEL_ERR:
8455     case GAUDI2_EVENT_TPC15_KERNEL_ERR:
8456     case GAUDI2_EVENT_TPC16_KERNEL_ERR:
8457     case GAUDI2_EVENT_TPC17_KERNEL_ERR:
8458     case GAUDI2_EVENT_TPC18_KERNEL_ERR:
8459     case GAUDI2_EVENT_TPC19_KERNEL_ERR:
8460     case GAUDI2_EVENT_TPC20_KERNEL_ERR:
8461     case GAUDI2_EVENT_TPC21_KERNEL_ERR:
8462     case GAUDI2_EVENT_TPC22_KERNEL_ERR:
8463     case GAUDI2_EVENT_TPC23_KERNEL_ERR:
8464     case GAUDI2_EVENT_TPC24_KERNEL_ERR:
8465         index = (event_type - GAUDI2_EVENT_TPC0_KERNEL_ERR) /
8466             (GAUDI2_EVENT_TPC1_KERNEL_ERR - GAUDI2_EVENT_TPC0_KERNEL_ERR);
8467         gaudi2_tpc_ack_interrupts(hdev, index, "KRN_ERR", &eq_entry->razwi_with_intr_cause);
8468         break;
8469 
8470     case GAUDI2_EVENT_DEC0_SPI:
8471     case GAUDI2_EVENT_DEC1_SPI:
8472     case GAUDI2_EVENT_DEC2_SPI:
8473     case GAUDI2_EVENT_DEC3_SPI:
8474     case GAUDI2_EVENT_DEC4_SPI:
8475     case GAUDI2_EVENT_DEC5_SPI:
8476     case GAUDI2_EVENT_DEC6_SPI:
8477     case GAUDI2_EVENT_DEC7_SPI:
8478     case GAUDI2_EVENT_DEC8_SPI:
8479     case GAUDI2_EVENT_DEC9_SPI:
8480         index = (event_type - GAUDI2_EVENT_DEC0_SPI) /
8481                 (GAUDI2_EVENT_DEC1_SPI - GAUDI2_EVENT_DEC0_SPI);
8482         gaudi2_handle_dec_err(hdev, index, "SPI", &eq_entry->razwi_info);
8483         break;
8484 
8485     case GAUDI2_EVENT_MME0_CTRL_AXI_ERROR_RESPONSE:
8486     case GAUDI2_EVENT_MME1_CTRL_AXI_ERROR_RESPONSE:
8487     case GAUDI2_EVENT_MME2_CTRL_AXI_ERROR_RESPONSE:
8488     case GAUDI2_EVENT_MME3_CTRL_AXI_ERROR_RESPONSE:
8489         index = (event_type - GAUDI2_EVENT_MME0_CTRL_AXI_ERROR_RESPONSE) /
8490                 (GAUDI2_EVENT_MME1_CTRL_AXI_ERROR_RESPONSE -
8491                         GAUDI2_EVENT_MME0_CTRL_AXI_ERROR_RESPONSE);
8492         gaudi2_handle_mme_err(hdev, index,
8493                 "CTRL_AXI_ERROR_RESPONSE", &eq_entry->razwi_info);
8494         gaudi2_handle_qm_sei_err(hdev, event_type, NULL);
8495         break;
8496 
8497     case GAUDI2_EVENT_MME0_QMAN_SW_ERROR:
8498     case GAUDI2_EVENT_MME1_QMAN_SW_ERROR:
8499     case GAUDI2_EVENT_MME2_QMAN_SW_ERROR:
8500     case GAUDI2_EVENT_MME3_QMAN_SW_ERROR:
8501         index = (event_type - GAUDI2_EVENT_MME0_QMAN_SW_ERROR) /
8502                 (GAUDI2_EVENT_MME1_QMAN_SW_ERROR -
8503                     GAUDI2_EVENT_MME0_QMAN_SW_ERROR);
8504         gaudi2_handle_mme_err(hdev, index, "QMAN_SW_ERROR", &eq_entry->razwi_info);
8505         break;
8506 
8507     case GAUDI2_EVENT_MME0_WAP_SOURCE_RESULT_INVALID:
8508     case GAUDI2_EVENT_MME1_WAP_SOURCE_RESULT_INVALID:
8509     case GAUDI2_EVENT_MME2_WAP_SOURCE_RESULT_INVALID:
8510     case GAUDI2_EVENT_MME3_WAP_SOURCE_RESULT_INVALID:
8511         index = (event_type - GAUDI2_EVENT_MME0_WAP_SOURCE_RESULT_INVALID) /
8512                 (GAUDI2_EVENT_MME1_WAP_SOURCE_RESULT_INVALID -
8513                     GAUDI2_EVENT_MME0_WAP_SOURCE_RESULT_INVALID);
8514         gaudi2_handle_mme_wap_err(hdev, index, &eq_entry->razwi_info);
8515         break;
8516 
8517     case GAUDI2_EVENT_KDMA_CH0_AXI_ERR_RSP:
8518     case GAUDI2_EVENT_KDMA0_CORE:
8519         gaudi2_handle_kdma_core_event(hdev,
8520                     le64_to_cpu(eq_entry->intr_cause.intr_cause_data));
8521         break;
8522 
8523     case GAUDI2_EVENT_HDMA2_CORE ... GAUDI2_EVENT_PDMA1_CORE:
8524         gaudi2_handle_dma_core_event(hdev,
8525                     le64_to_cpu(eq_entry->intr_cause.intr_cause_data));
8526         break;
8527 
8528     case GAUDI2_EVENT_PCIE_ADDR_DEC_ERR:
8529         gaudi2_print_pcie_addr_dec_info(hdev,
8530                 le64_to_cpu(eq_entry->intr_cause.intr_cause_data));
8531         break;
8532 
8533     case GAUDI2_EVENT_HMMU0_PAGE_FAULT_OR_WR_PERM ... GAUDI2_EVENT_HMMU12_SECURITY_ERROR:
8534     case GAUDI2_EVENT_HMMU_0_AXI_ERR_RSP ... GAUDI2_EVENT_HMMU_12_AXI_ERR_RSP:
8535     case GAUDI2_EVENT_PMMU0_PAGE_FAULT_WR_PERM ... GAUDI2_EVENT_PMMU0_SECURITY_ERROR:
8536     case GAUDI2_EVENT_PMMU_AXI_ERR_RSP_0:
8537         gaudi2_handle_mmu_spi_sei_err(hdev, event_type);
8538         reset_flags |= HL_DRV_RESET_FW_FATAL_ERR;
8539         break;
8540 
8541     case GAUDI2_EVENT_HIF0_FATAL ... GAUDI2_EVENT_HIF12_FATAL:
8542         gaudi2_handle_hif_fatal(hdev, event_type,
8543                 le64_to_cpu(eq_entry->intr_cause.intr_cause_data));
8544         reset_flags |= HL_DRV_RESET_FW_FATAL_ERR;
8545         break;
8546 
8547     case GAUDI2_EVENT_PMMU_FATAL_0:
8548         gaudi2_handle_pif_fatal(hdev,
8549                 le64_to_cpu(eq_entry->intr_cause.intr_cause_data));
8550         reset_flags |= HL_DRV_RESET_FW_FATAL_ERR;
8551         break;
8552 
8553     case GAUDI2_EVENT_PSOC63_RAZWI_OR_PID_MIN_MAX_INTERRUPT:
8554         gaudi2_ack_psoc_razwi_event_handler(hdev);
8555         break;
8556 
8557     case GAUDI2_EVENT_HBM0_MC0_SEI_SEVERE ... GAUDI2_EVENT_HBM5_MC1_SEI_NON_SEVERE:
8558         if (gaudi2_handle_hbm_mc_sei_err(hdev, event_type, &eq_entry->sei_data)) {
8559             reset_flags |= HL_DRV_RESET_FW_FATAL_ERR;
8560             reset_required = true;
8561         }
8562         break;
8563 
8564     case GAUDI2_EVENT_HBM_CATTRIP_0 ... GAUDI2_EVENT_HBM_CATTRIP_5:
8565         gaudi2_handle_hbm_cattrip(hdev, le64_to_cpu(eq_entry->intr_cause.intr_cause_data));
8566         break;
8567 
8568     case GAUDI2_EVENT_HBM0_MC0_SPI ... GAUDI2_EVENT_HBM5_MC1_SPI:
8569         gaudi2_handle_hbm_mc_spi(hdev, le64_to_cpu(eq_entry->intr_cause.intr_cause_data));
8570         break;
8571 
8572     case GAUDI2_EVENT_PCIE_DRAIN_COMPLETE:
8573         gaudi2_handle_pcie_drain(hdev, &eq_entry->pcie_drain_ind_data);
8574         break;
8575 
8576     case GAUDI2_EVENT_PSOC59_RPM_ERROR_OR_DRAIN:
8577         gaudi2_handle_psoc_drain(hdev, le64_to_cpu(eq_entry->intr_cause.intr_cause_data));
8578         break;
8579 
8580     case GAUDI2_EVENT_CPU_AXI_ECC:
8581         reset_flags |= HL_DRV_RESET_FW_FATAL_ERR;
8582         break;
8583     case GAUDI2_EVENT_CPU_L2_RAM_ECC:
8584         reset_flags |= HL_DRV_RESET_FW_FATAL_ERR;
8585         break;
8586     case GAUDI2_EVENT_MME0_SBTE0_AXI_ERR_RSP ... GAUDI2_EVENT_MME0_SBTE4_AXI_ERR_RSP:
8587     case GAUDI2_EVENT_MME1_SBTE0_AXI_ERR_RSP ... GAUDI2_EVENT_MME1_SBTE4_AXI_ERR_RSP:
8588     case GAUDI2_EVENT_MME2_SBTE0_AXI_ERR_RSP ... GAUDI2_EVENT_MME2_SBTE4_AXI_ERR_RSP:
8589     case GAUDI2_EVENT_MME3_SBTE0_AXI_ERR_RSP ... GAUDI2_EVENT_MME3_SBTE4_AXI_ERR_RSP:
8590         index = (event_type - GAUDI2_EVENT_MME0_SBTE0_AXI_ERR_RSP) /
8591                 (GAUDI2_EVENT_MME1_SBTE0_AXI_ERR_RSP -
8592                     GAUDI2_EVENT_MME0_SBTE0_AXI_ERR_RSP);
8593         sbte_index = (event_type - GAUDI2_EVENT_MME0_SBTE0_AXI_ERR_RSP) %
8594                 (GAUDI2_EVENT_MME1_SBTE0_AXI_ERR_RSP -
8595                     GAUDI2_EVENT_MME0_SBTE0_AXI_ERR_RSP);
8596         gaudi2_handle_mme_sbte_err(hdev, index, sbte_index,
8597                         le64_to_cpu(eq_entry->intr_cause.intr_cause_data));
8598         break;
8599     case GAUDI2_EVENT_VM0_ALARM_A ... GAUDI2_EVENT_VM3_ALARM_B:
8600         reset_flags |= HL_DRV_RESET_FW_FATAL_ERR;
8601         break;
8602     case GAUDI2_EVENT_PSOC_AXI_ERR_RSP:
8603     case GAUDI2_EVENT_PSOC_PRSTN_FALL:
8604         break;
8605     case GAUDI2_EVENT_PCIE_APB_TIMEOUT:
8606         reset_flags |= HL_DRV_RESET_FW_FATAL_ERR;
8607         break;
8608     case GAUDI2_EVENT_PCIE_FATAL_ERR:
8609         break;
8610     case GAUDI2_EVENT_TPC0_BMON_SPMU:
8611     case GAUDI2_EVENT_TPC1_BMON_SPMU:
8612     case GAUDI2_EVENT_TPC2_BMON_SPMU:
8613     case GAUDI2_EVENT_TPC3_BMON_SPMU:
8614     case GAUDI2_EVENT_TPC4_BMON_SPMU:
8615     case GAUDI2_EVENT_TPC5_BMON_SPMU:
8616     case GAUDI2_EVENT_TPC6_BMON_SPMU:
8617     case GAUDI2_EVENT_TPC7_BMON_SPMU:
8618     case GAUDI2_EVENT_TPC8_BMON_SPMU:
8619     case GAUDI2_EVENT_TPC9_BMON_SPMU:
8620     case GAUDI2_EVENT_TPC10_BMON_SPMU:
8621     case GAUDI2_EVENT_TPC11_BMON_SPMU:
8622     case GAUDI2_EVENT_TPC12_BMON_SPMU:
8623     case GAUDI2_EVENT_TPC13_BMON_SPMU:
8624     case GAUDI2_EVENT_TPC14_BMON_SPMU:
8625     case GAUDI2_EVENT_TPC15_BMON_SPMU:
8626     case GAUDI2_EVENT_TPC16_BMON_SPMU:
8627     case GAUDI2_EVENT_TPC17_BMON_SPMU:
8628     case GAUDI2_EVENT_TPC18_BMON_SPMU:
8629     case GAUDI2_EVENT_TPC19_BMON_SPMU:
8630     case GAUDI2_EVENT_TPC20_BMON_SPMU:
8631     case GAUDI2_EVENT_TPC21_BMON_SPMU:
8632     case GAUDI2_EVENT_TPC22_BMON_SPMU:
8633     case GAUDI2_EVENT_TPC23_BMON_SPMU:
8634     case GAUDI2_EVENT_TPC24_BMON_SPMU:
8635     case GAUDI2_EVENT_MME0_CTRL_BMON_SPMU:
8636     case GAUDI2_EVENT_MME0_SBTE_BMON_SPMU:
8637     case GAUDI2_EVENT_MME0_WAP_BMON_SPMU:
8638     case GAUDI2_EVENT_MME1_CTRL_BMON_SPMU:
8639     case GAUDI2_EVENT_MME1_SBTE_BMON_SPMU:
8640     case GAUDI2_EVENT_MME1_WAP_BMON_SPMU:
8641     case GAUDI2_EVENT_MME2_CTRL_BMON_SPMU:
8642     case GAUDI2_EVENT_MME2_SBTE_BMON_SPMU:
8643     case GAUDI2_EVENT_MME2_WAP_BMON_SPMU:
8644     case GAUDI2_EVENT_MME3_CTRL_BMON_SPMU:
8645     case GAUDI2_EVENT_MME3_SBTE_BMON_SPMU:
8646     case GAUDI2_EVENT_MME3_WAP_BMON_SPMU:
8647     case GAUDI2_EVENT_HDMA2_BM_SPMU ... GAUDI2_EVENT_PDMA1_BM_SPMU:
8648         fallthrough;
8649     case GAUDI2_EVENT_DEC0_BMON_SPMU:
8650     case GAUDI2_EVENT_DEC1_BMON_SPMU:
8651     case GAUDI2_EVENT_DEC2_BMON_SPMU:
8652     case GAUDI2_EVENT_DEC3_BMON_SPMU:
8653     case GAUDI2_EVENT_DEC4_BMON_SPMU:
8654     case GAUDI2_EVENT_DEC5_BMON_SPMU:
8655     case GAUDI2_EVENT_DEC6_BMON_SPMU:
8656     case GAUDI2_EVENT_DEC7_BMON_SPMU:
8657     case GAUDI2_EVENT_DEC8_BMON_SPMU:
8658     case GAUDI2_EVENT_DEC9_BMON_SPMU:
8659     case GAUDI2_EVENT_ROTATOR0_BMON_SPMU ... GAUDI2_EVENT_SM3_BMON_SPMU:
8660         break;
8661 
8662     case GAUDI2_EVENT_CPU_FIX_POWER_ENV_S:
8663     case GAUDI2_EVENT_CPU_FIX_POWER_ENV_E:
8664     case GAUDI2_EVENT_CPU_FIX_THERMAL_ENV_S:
8665     case GAUDI2_EVENT_CPU_FIX_THERMAL_ENV_E:
8666         gaudi2_print_clk_change_info(hdev, event_type);
8667         break;
8668 
8669     case GAUDI2_EVENT_CPU_PKT_QUEUE_OUT_SYNC:
8670         gaudi2_print_out_of_sync_info(hdev, &eq_entry->pkt_sync_err);
8671         break;
8672 
8673     case GAUDI2_EVENT_PCIE_FLR_REQUESTED:
8674         /* Do nothing- FW will handle it */
8675         break;
8676 
8677     case GAUDI2_EVENT_PCIE_P2P_MSIX:
8678         gaudi2_handle_pcie_p2p_msix(hdev);
8679         break;
8680 
8681     case GAUDI2_EVENT_SM0_AXI_ERROR_RESPONSE ... GAUDI2_EVENT_SM3_AXI_ERROR_RESPONSE:
8682         index = event_type - GAUDI2_EVENT_SM0_AXI_ERROR_RESPONSE;
8683         skip_reset = !gaudi2_handle_sm_err(hdev, index);
8684         break;
8685 
8686     case GAUDI2_EVENT_PSOC_MME_PLL_LOCK_ERR ... GAUDI2_EVENT_DCORE2_HBM_PLL_LOCK_ERR:
8687         break;
8688 
8689     case GAUDI2_EVENT_CPU_CPLD_SHUTDOWN_CAUSE:
8690         dev_info(hdev->dev, "CPLD shutdown cause, reset reason: 0x%llx\n",
8691                         le64_to_cpu(eq_entry->data[0]));
8692         break;
8693     case GAUDI2_EVENT_CPU_CPLD_SHUTDOWN_EVENT:
8694         dev_err(hdev->dev, "CPLD shutdown event, reset reason: 0x%llx\n",
8695                         le64_to_cpu(eq_entry->data[0]));
8696         break;
8697 
8698     case GAUDI2_EVENT_CPU_PKT_SANITY_FAILED:
8699         gaudi2_print_cpu_pkt_failure_info(hdev, &eq_entry->pkt_sync_err);
8700         break;
8701 
8702     case GAUDI2_EVENT_ARC_DCCM_FULL:
8703         hl_arc_event_handle(hdev, &eq_entry->arc_data);
8704         break;
8705 
8706     default:
8707         if (gaudi2_irq_map_table[event_type].valid)
8708             dev_err_ratelimited(hdev->dev, "Cannot find handler for event %d\n",
8709                         event_type);
8710     }
8711 
8712     if ((gaudi2_irq_map_table[event_type].reset || reset_required) && !skip_reset)
8713         goto reset_device;
8714 
8715     /* Send unmask irq only for interrupts not classified as MSG */
8716     if (!gaudi2_irq_map_table[event_type].msg)
8717         hl_fw_unmask_irq(hdev, event_type);
8718 
8719     return;
8720 
8721 reset_device:
8722     if (hdev->hard_reset_on_fw_events) {
8723         hl_device_reset(hdev, reset_flags);
8724     } else {
8725         if (!gaudi2_irq_map_table[event_type].msg)
8726             hl_fw_unmask_irq(hdev, event_type);
8727     }
8728 }
8729 
8730 static int gaudi2_memset_device_memory(struct hl_device *hdev, u64 addr, u64 size, u64 val)
8731 {
8732     struct asic_fixed_properties *prop = &hdev->asic_prop;
8733     u64 comp_addr, cur_addr = addr, end_addr = addr + size;
8734     u32 chunk_size, busy, dcore, edma_idx, sob_offset, sob_addr, comp_val, edma_commit;
8735     u32 old_mmubp, mmubp;
8736     int rc = 0;
8737 
8738     sob_offset = hdev->asic_prop.first_available_user_sob[0] * 4;
8739     sob_addr = mmDCORE0_SYNC_MNGR_OBJS_SOB_OBJ_0 + sob_offset;
8740     comp_addr = CFG_BASE + sob_addr;
8741     comp_val = FIELD_PREP(DCORE0_SYNC_MNGR_OBJS_SOB_OBJ_INC_MASK, 1) |
8742         FIELD_PREP(DCORE0_SYNC_MNGR_OBJS_SOB_OBJ_VAL_MASK, 1);
8743 
8744     edma_commit = FIELD_PREP(ARC_FARM_KDMA_CTX_COMMIT_LIN_MASK, 1) |
8745             FIELD_PREP(ARC_FARM_KDMA_CTX_COMMIT_MEM_SET_MASK, 1) |
8746             FIELD_PREP(ARC_FARM_KDMA_CTX_COMMIT_WR_COMP_EN_MASK, 1);
8747     mmubp = FIELD_PREP(ARC_FARM_KDMA_CTX_AXUSER_HB_MMU_BP_WR_MASK, 1) |
8748         FIELD_PREP(ARC_FARM_KDMA_CTX_AXUSER_HB_MMU_BP_RD_MASK, 1);
8749 
8750     if (prop->edma_enabled_mask == 0) {
8751         dev_info(hdev->dev, "non of the EDMA engines is enabled - skip dram scrubbing\n");
8752         return -EIO;
8753     }
8754 
8755     /*
8756      * set mmu bypass for the scrubbing - all ddmas are configured the same so save
8757      * only the first one to restore later
8758      */
8759     old_mmubp = RREG32(mmDCORE0_EDMA0_CORE_CTX_AXUSER_HB_MMU_BP);
8760     for (dcore = 0 ; dcore < NUM_OF_DCORES ; dcore++) {
8761         for (edma_idx = 0 ; edma_idx < NUM_OF_EDMA_PER_DCORE ; edma_idx++) {
8762             u32 edma_offset = dcore * DCORE_OFFSET + edma_idx * DCORE_EDMA_OFFSET;
8763             u32 edma_bit = dcore * NUM_OF_EDMA_PER_DCORE + edma_idx;
8764 
8765             if (!(prop->edma_enabled_mask & BIT(edma_bit)))
8766                 continue;
8767 
8768             WREG32(mmDCORE0_EDMA0_CORE_CTX_AXUSER_HB_MMU_BP +
8769                     edma_offset, mmubp);
8770         }
8771     }
8772 
8773     while (cur_addr < end_addr) {
8774         int dma_num = 0;
8775 
8776         WREG32(sob_addr, 0);
8777         for (dcore = 0 ; dcore < NUM_OF_DCORES ; dcore++) {
8778             for (edma_idx = 0 ; edma_idx < NUM_OF_EDMA_PER_DCORE ; edma_idx++) {
8779                 u32 edma_offset = dcore * DCORE_OFFSET +
8780                     edma_idx * DCORE_EDMA_OFFSET;
8781                 u32 edma_bit = dcore * NUM_OF_EDMA_PER_DCORE + edma_idx;
8782 
8783                 if (!(prop->edma_enabled_mask & BIT(edma_bit)))
8784                     continue;
8785 
8786                 chunk_size = min_t(u64, SZ_2G, end_addr - cur_addr);
8787 
8788                 WREG32(mmDCORE0_EDMA0_CORE_CTX_SRC_BASE_LO + edma_offset,
8789                         lower_32_bits(val));
8790                 WREG32(mmDCORE0_EDMA0_CORE_CTX_SRC_BASE_HI + edma_offset,
8791                         upper_32_bits(val));
8792 
8793                 WREG32(mmDCORE0_EDMA0_CORE_CTX_DST_BASE_LO + edma_offset,
8794                         lower_32_bits(cur_addr));
8795                 WREG32(mmDCORE0_EDMA0_CORE_CTX_DST_BASE_HI + edma_offset,
8796                         upper_32_bits(cur_addr));
8797 
8798                 WREG32(mmDCORE0_EDMA0_CORE_CTX_WR_COMP_ADDR_LO + edma_offset,
8799                         lower_32_bits(comp_addr));
8800                 WREG32(mmDCORE0_EDMA0_CORE_CTX_WR_COMP_ADDR_HI + edma_offset,
8801                         upper_32_bits(comp_addr));
8802                 WREG32(mmDCORE0_EDMA0_CORE_CTX_WR_COMP_WDATA + edma_offset,
8803                         comp_val);
8804 
8805                 WREG32(mmDCORE0_EDMA0_CORE_CTX_DST_TSIZE_0 + edma_offset,
8806                         chunk_size);
8807                 WREG32(mmDCORE0_EDMA0_CORE_CTX_COMMIT + edma_offset, edma_commit);
8808 
8809                 dma_num++;
8810 
8811                 cur_addr += chunk_size;
8812 
8813                 if (cur_addr == end_addr)
8814                     goto poll;
8815             }
8816         }
8817 poll:
8818         rc = hl_poll_timeout(hdev, sob_addr, busy, (busy == dma_num), 1000, 1000000);
8819         if (rc) {
8820             dev_err(hdev->dev, "DMA Timeout during HBM scrubbing\n");
8821             goto end;
8822         }
8823     }
8824 end:
8825     for (dcore = 0 ; dcore < NUM_OF_DCORES ; dcore++) {
8826         for (edma_idx = 0 ; edma_idx < NUM_OF_EDMA_PER_DCORE ; edma_idx++) {
8827             u32 edma_offset = dcore * DCORE_OFFSET + edma_idx * DCORE_EDMA_OFFSET;
8828             u32 edma_bit = dcore * NUM_OF_EDMA_PER_DCORE + edma_idx;
8829 
8830             if (!(prop->edma_enabled_mask & BIT(edma_bit)))
8831                 continue;
8832 
8833             WREG32(mmDCORE0_EDMA0_CORE_CTX_AXUSER_HB_MMU_BP + edma_offset, old_mmubp);
8834         }
8835     }
8836 
8837     WREG32(sob_addr, 0);
8838     return rc;
8839 }
8840 
8841 static int gaudi2_scrub_device_dram(struct hl_device *hdev, u64 val)
8842 {
8843     int rc;
8844     struct asic_fixed_properties *prop = &hdev->asic_prop;
8845     u64 size = prop->dram_end_address - prop->dram_user_base_address;
8846 
8847     rc = gaudi2_memset_device_memory(hdev, prop->dram_user_base_address, size, val);
8848 
8849     if (rc)
8850         dev_err(hdev->dev, "Failed to scrub dram, address: 0x%llx size: %llu\n",
8851                 prop->dram_user_base_address, size);
8852     return rc;
8853 }
8854 
8855 static int gaudi2_scrub_device_mem(struct hl_device *hdev)
8856 {
8857     int rc;
8858     struct asic_fixed_properties *prop = &hdev->asic_prop;
8859     u64 val = hdev->memory_scrub_val;
8860     u64 addr, size;
8861 
8862     if (!hdev->memory_scrub)
8863         return 0;
8864 
8865     /* scrub SRAM */
8866     addr = prop->sram_user_base_address;
8867     size = hdev->pldm ? 0x10000 : (prop->sram_size - SRAM_USER_BASE_OFFSET);
8868     dev_dbg(hdev->dev, "Scrubbing SRAM: 0x%09llx - 0x%09llx, val: 0x%llx\n",
8869             addr, addr + size, val);
8870     rc = gaudi2_memset_device_memory(hdev, addr, size, val);
8871     if (rc) {
8872         dev_err(hdev->dev, "scrubbing SRAM failed (%d)\n", rc);
8873         return rc;
8874     }
8875 
8876     /* scrub DRAM */
8877     rc = gaudi2_scrub_device_dram(hdev, val);
8878     if (rc) {
8879         dev_err(hdev->dev, "scrubbing DRAM failed (%d)\n", rc);
8880         return rc;
8881     }
8882     return 0;
8883 }
8884 
8885 static void gaudi2_restore_user_sm_registers(struct hl_device *hdev)
8886 {
8887     u64 addr, mon_sts_addr, mon_cfg_addr, cq_lbw_l_addr, cq_lbw_h_addr,
8888         cq_lbw_data_addr, cq_base_l_addr, cq_base_h_addr, cq_size_addr;
8889     u32 val, size, offset;
8890     int dcore_id;
8891 
8892     offset = hdev->asic_prop.first_available_cq[0] * 4;
8893     cq_lbw_l_addr = mmDCORE0_SYNC_MNGR_GLBL_LBW_ADDR_L_0 + offset;
8894     cq_lbw_h_addr = mmDCORE0_SYNC_MNGR_GLBL_LBW_ADDR_H_0 + offset;
8895     cq_lbw_data_addr = mmDCORE0_SYNC_MNGR_GLBL_LBW_DATA_0 + offset;
8896     cq_base_l_addr = mmDCORE0_SYNC_MNGR_GLBL_CQ_BASE_ADDR_L_0 + offset;
8897     cq_base_h_addr = mmDCORE0_SYNC_MNGR_GLBL_CQ_BASE_ADDR_H_0 + offset;
8898     cq_size_addr = mmDCORE0_SYNC_MNGR_GLBL_CQ_SIZE_LOG2_0 + offset;
8899     size = mmDCORE0_SYNC_MNGR_GLBL_LBW_ADDR_H_0 -
8900             (mmDCORE0_SYNC_MNGR_GLBL_LBW_ADDR_L_0 + offset);
8901 
8902     /* memset dcore0 CQ registers */
8903     gaudi2_memset_device_lbw(hdev, cq_lbw_l_addr, size, 0);
8904     gaudi2_memset_device_lbw(hdev, cq_lbw_h_addr, size, 0);
8905     gaudi2_memset_device_lbw(hdev, cq_lbw_data_addr, size, 0);
8906     gaudi2_memset_device_lbw(hdev, cq_base_l_addr, size, 0);
8907     gaudi2_memset_device_lbw(hdev, cq_base_h_addr, size, 0);
8908 
8909     cq_lbw_l_addr = mmDCORE0_SYNC_MNGR_GLBL_LBW_ADDR_L_0 + DCORE_OFFSET;
8910     cq_lbw_h_addr = mmDCORE0_SYNC_MNGR_GLBL_LBW_ADDR_H_0 + DCORE_OFFSET;
8911     cq_lbw_data_addr = mmDCORE0_SYNC_MNGR_GLBL_LBW_DATA_0 + DCORE_OFFSET;
8912     cq_base_l_addr = mmDCORE0_SYNC_MNGR_GLBL_CQ_BASE_ADDR_L_0 + DCORE_OFFSET;
8913     cq_base_h_addr = mmDCORE0_SYNC_MNGR_GLBL_CQ_BASE_ADDR_H_0 + DCORE_OFFSET;
8914     cq_size_addr = mmDCORE0_SYNC_MNGR_GLBL_CQ_SIZE_LOG2_0 + DCORE_OFFSET;
8915     size = mmDCORE0_SYNC_MNGR_GLBL_LBW_ADDR_H_0 - mmDCORE0_SYNC_MNGR_GLBL_LBW_ADDR_L_0;
8916 
8917     for (dcore_id = 1 ; dcore_id < NUM_OF_DCORES ; dcore_id++) {
8918         gaudi2_memset_device_lbw(hdev, cq_lbw_l_addr, size, 0);
8919         gaudi2_memset_device_lbw(hdev, cq_lbw_h_addr, size, 0);
8920         gaudi2_memset_device_lbw(hdev, cq_lbw_data_addr, size, 0);
8921         gaudi2_memset_device_lbw(hdev, cq_base_l_addr, size, 0);
8922         gaudi2_memset_device_lbw(hdev, cq_base_h_addr, size, 0);
8923         gaudi2_memset_device_lbw(hdev, cq_size_addr, size, 0);
8924 
8925         cq_lbw_l_addr += DCORE_OFFSET;
8926         cq_lbw_h_addr += DCORE_OFFSET;
8927         cq_lbw_data_addr += DCORE_OFFSET;
8928         cq_base_l_addr += DCORE_OFFSET;
8929         cq_base_h_addr += DCORE_OFFSET;
8930         cq_size_addr += DCORE_OFFSET;
8931     }
8932 
8933     offset = hdev->asic_prop.first_available_user_mon[0] * 4;
8934     addr = mmDCORE0_SYNC_MNGR_OBJS_MON_STATUS_0 + offset;
8935     val = 1 << DCORE0_SYNC_MNGR_OBJS_MON_STATUS_PROT_SHIFT;
8936     size = mmDCORE0_SYNC_MNGR_OBJS_SM_SEC_0 - (mmDCORE0_SYNC_MNGR_OBJS_MON_STATUS_0 + offset);
8937 
8938     /* memset dcore0 monitors */
8939     gaudi2_memset_device_lbw(hdev, addr, size, val);
8940 
8941     addr = mmDCORE0_SYNC_MNGR_OBJS_MON_CONFIG_0 + offset;
8942     gaudi2_memset_device_lbw(hdev, addr, size, 0);
8943 
8944     mon_sts_addr = mmDCORE0_SYNC_MNGR_OBJS_MON_STATUS_0 + DCORE_OFFSET;
8945     mon_cfg_addr = mmDCORE0_SYNC_MNGR_OBJS_MON_CONFIG_0 + DCORE_OFFSET;
8946     size = mmDCORE0_SYNC_MNGR_OBJS_SM_SEC_0 - mmDCORE0_SYNC_MNGR_OBJS_MON_STATUS_0;
8947 
8948     for (dcore_id = 1 ; dcore_id < NUM_OF_DCORES ; dcore_id++) {
8949         gaudi2_memset_device_lbw(hdev, mon_sts_addr, size, val);
8950         gaudi2_memset_device_lbw(hdev, mon_cfg_addr, size, 0);
8951         mon_sts_addr += DCORE_OFFSET;
8952         mon_cfg_addr += DCORE_OFFSET;
8953     }
8954 
8955     offset = hdev->asic_prop.first_available_user_sob[0] * 4;
8956     addr = mmDCORE0_SYNC_MNGR_OBJS_SOB_OBJ_0 + offset;
8957     val = 0;
8958     size = mmDCORE0_SYNC_MNGR_OBJS_MON_PAY_ADDRL_0 -
8959             (mmDCORE0_SYNC_MNGR_OBJS_SOB_OBJ_0 + offset);
8960 
8961     /* memset dcore0 sobs */
8962     gaudi2_memset_device_lbw(hdev, addr, size, val);
8963 
8964     addr = mmDCORE0_SYNC_MNGR_OBJS_SOB_OBJ_0 + DCORE_OFFSET;
8965     size = mmDCORE0_SYNC_MNGR_OBJS_MON_PAY_ADDRL_0 - mmDCORE0_SYNC_MNGR_OBJS_SOB_OBJ_0;
8966 
8967     for (dcore_id = 1 ; dcore_id < NUM_OF_DCORES ; dcore_id++) {
8968         gaudi2_memset_device_lbw(hdev, addr, size, val);
8969         addr += DCORE_OFFSET;
8970     }
8971 
8972     /* Flush all WREG to prevent race */
8973     val = RREG32(mmDCORE0_SYNC_MNGR_OBJS_SOB_OBJ_0 + offset);
8974 }
8975 
8976 static void gaudi2_restore_user_qm_registers(struct hl_device *hdev)
8977 {
8978     u32 reg_base, hw_queue_id;
8979 
8980     for (hw_queue_id = GAUDI2_QUEUE_ID_PDMA_0_0 ; hw_queue_id <= GAUDI2_QUEUE_ID_ROT_1_0;
8981                             hw_queue_id += NUM_OF_PQ_PER_QMAN) {
8982         if (!gaudi2_is_queue_enabled(hdev, hw_queue_id))
8983             continue;
8984 
8985         gaudi2_clear_qm_fence_counters_common(hdev, hw_queue_id, false);
8986 
8987         reg_base = gaudi2_qm_blocks_bases[hw_queue_id];
8988         WREG32(reg_base + QM_ARB_CFG_0_OFFSET, 0);
8989     }
8990 
8991     /* Flush all WREG to prevent race */
8992     RREG32(mmPDMA0_QM_ARB_CFG_0);
8993 }
8994 
8995 static void gaudi2_restore_nic_qm_registers(struct hl_device *hdev)
8996 {
8997     u32 reg_base, hw_queue_id;
8998 
8999     for (hw_queue_id = GAUDI2_QUEUE_ID_NIC_0_0 ; hw_queue_id <= GAUDI2_QUEUE_ID_NIC_23_3;
9000                             hw_queue_id += NUM_OF_PQ_PER_QMAN) {
9001         if (!gaudi2_is_queue_enabled(hdev, hw_queue_id))
9002             continue;
9003 
9004         gaudi2_clear_qm_fence_counters_common(hdev, hw_queue_id, false);
9005 
9006         reg_base = gaudi2_qm_blocks_bases[hw_queue_id];
9007         WREG32(reg_base + QM_ARB_CFG_0_OFFSET, 0);
9008     }
9009 
9010     /* Flush all WREG to prevent race */
9011     RREG32(mmPDMA0_QM_ARB_CFG_0);
9012 }
9013 
9014 static int gaudi2_context_switch(struct hl_device *hdev, u32 asid)
9015 {
9016     return 0;
9017 }
9018 
9019 static void gaudi2_restore_phase_topology(struct hl_device *hdev)
9020 {
9021 }
9022 
9023 static void gaudi2_init_block_instances(struct hl_device *hdev, u32 block_idx,
9024                         struct dup_block_ctx *cfg_ctx)
9025 {
9026     u64 block_base = cfg_ctx->base + block_idx * cfg_ctx->block_off;
9027     u8 seq;
9028     int i;
9029 
9030     for (i = 0 ; i < cfg_ctx->instances ; i++) {
9031         seq = block_idx * cfg_ctx->instances + i;
9032 
9033         /* skip disabled instance */
9034         if (!(cfg_ctx->enabled_mask & BIT_ULL(seq)))
9035             continue;
9036 
9037         cfg_ctx->instance_cfg_fn(hdev, block_base + i * cfg_ctx->instance_off,
9038                     cfg_ctx->data);
9039     }
9040 }
9041 
9042 static void gaudi2_init_blocks_with_mask(struct hl_device *hdev, struct dup_block_ctx *cfg_ctx,
9043                         u64 mask)
9044 {
9045     int i;
9046 
9047     cfg_ctx->enabled_mask = mask;
9048 
9049     for (i = 0 ; i < cfg_ctx->blocks ; i++)
9050         gaudi2_init_block_instances(hdev, i, cfg_ctx);
9051 }
9052 
9053 void gaudi2_init_blocks(struct hl_device *hdev, struct dup_block_ctx *cfg_ctx)
9054 {
9055     gaudi2_init_blocks_with_mask(hdev, cfg_ctx, U64_MAX);
9056 }
9057 
9058 static int gaudi2_debugfs_read_dma(struct hl_device *hdev, u64 addr, u32 size, void *blob_addr)
9059 {
9060     void *host_mem_virtual_addr;
9061     dma_addr_t host_mem_dma_addr;
9062     u64 reserved_va_base;
9063     u32 pos, size_left, size_to_dma;
9064     struct hl_ctx *ctx;
9065     int rc = 0;
9066 
9067     /* Fetch the ctx */
9068     ctx = hl_get_compute_ctx(hdev);
9069     if (!ctx) {
9070         dev_err(hdev->dev, "No ctx available\n");
9071         return -EINVAL;
9072     }
9073 
9074     /* Allocate buffers for read and for poll */
9075     host_mem_virtual_addr = hl_asic_dma_alloc_coherent(hdev, SZ_2M, &host_mem_dma_addr,
9076                                 GFP_KERNEL | __GFP_ZERO);
9077     if (host_mem_virtual_addr == NULL) {
9078         dev_err(hdev->dev, "Failed to allocate memory for KDMA read\n");
9079         rc = -ENOMEM;
9080         goto put_ctx;
9081     }
9082 
9083     /* Reserve VM region on asic side */
9084     reserved_va_base = hl_reserve_va_block(hdev, ctx, HL_VA_RANGE_TYPE_HOST, SZ_2M,
9085                         HL_MMU_VA_ALIGNMENT_NOT_NEEDED);
9086     if (!reserved_va_base) {
9087         dev_err(hdev->dev, "Failed to reserve vmem on asic\n");
9088         rc = -ENOMEM;
9089         goto free_data_buffer;
9090     }
9091 
9092     /* Create mapping on asic side */
9093     mutex_lock(&ctx->mmu_lock);
9094     rc = hl_mmu_map_contiguous(ctx, reserved_va_base, host_mem_dma_addr, SZ_2M);
9095     hl_mmu_invalidate_cache_range(hdev, false,
9096                       MMU_OP_USERPTR | MMU_OP_SKIP_LOW_CACHE_INV,
9097                       ctx->asid, reserved_va_base, SZ_2M);
9098     mutex_unlock(&ctx->mmu_lock);
9099     if (rc) {
9100         dev_err(hdev->dev, "Failed to create mapping on asic mmu\n");
9101         goto unreserve_va;
9102     }
9103 
9104     hdev->asic_funcs->kdma_lock(hdev, 0);
9105 
9106     /* Enable MMU on KDMA */
9107     gaudi2_kdma_set_mmbp_asid(hdev, false, ctx->asid);
9108 
9109     pos = 0;
9110     size_left = size;
9111     size_to_dma = SZ_2M;
9112 
9113     while (size_left > 0) {
9114         if (size_left < SZ_2M)
9115             size_to_dma = size_left;
9116 
9117         rc = gaudi2_send_job_to_kdma(hdev, addr, reserved_va_base, size_to_dma, false);
9118         if (rc)
9119             break;
9120 
9121         memcpy(blob_addr + pos, host_mem_virtual_addr, size_to_dma);
9122 
9123         if (size_left <= SZ_2M)
9124             break;
9125 
9126         pos += SZ_2M;
9127         addr += SZ_2M;
9128         size_left -= SZ_2M;
9129     }
9130 
9131     gaudi2_kdma_set_mmbp_asid(hdev, true, HL_KERNEL_ASID_ID);
9132 
9133     hdev->asic_funcs->kdma_unlock(hdev, 0);
9134 
9135     mutex_lock(&ctx->mmu_lock);
9136     hl_mmu_unmap_contiguous(ctx, reserved_va_base, SZ_2M);
9137     hl_mmu_invalidate_cache_range(hdev, false, MMU_OP_USERPTR,
9138                       ctx->asid, reserved_va_base, SZ_2M);
9139     mutex_unlock(&ctx->mmu_lock);
9140 unreserve_va:
9141     hl_unreserve_va_block(hdev, ctx, reserved_va_base, SZ_2M);
9142 free_data_buffer:
9143     hl_asic_dma_free_coherent(hdev, SZ_2M, host_mem_virtual_addr, host_mem_dma_addr);
9144 put_ctx:
9145     hl_ctx_put(ctx);
9146 
9147     return rc;
9148 }
9149 
9150 static int gaudi2_internal_cb_pool_init(struct hl_device *hdev, struct hl_ctx *ctx)
9151 {
9152     struct gaudi2_device *gaudi2 = hdev->asic_specific;
9153     int min_alloc_order, rc;
9154 
9155     if (!(gaudi2->hw_cap_initialized & HW_CAP_PMMU))
9156         return 0;
9157 
9158     hdev->internal_cb_pool_virt_addr = hl_asic_dma_alloc_coherent(hdev,
9159                                 HOST_SPACE_INTERNAL_CB_SZ,
9160                                 &hdev->internal_cb_pool_dma_addr,
9161                                 GFP_KERNEL | __GFP_ZERO);
9162 
9163     if (!hdev->internal_cb_pool_virt_addr)
9164         return -ENOMEM;
9165 
9166     min_alloc_order = ilog2(min(gaudi2_get_signal_cb_size(hdev),
9167                     gaudi2_get_wait_cb_size(hdev)));
9168 
9169     hdev->internal_cb_pool = gen_pool_create(min_alloc_order, -1);
9170     if (!hdev->internal_cb_pool) {
9171         dev_err(hdev->dev, "Failed to create internal CB pool\n");
9172         rc = -ENOMEM;
9173         goto free_internal_cb_pool;
9174     }
9175 
9176     rc = gen_pool_add(hdev->internal_cb_pool, (uintptr_t) hdev->internal_cb_pool_virt_addr,
9177                 HOST_SPACE_INTERNAL_CB_SZ, -1);
9178     if (rc) {
9179         dev_err(hdev->dev, "Failed to add memory to internal CB pool\n");
9180         rc = -EFAULT;
9181         goto destroy_internal_cb_pool;
9182     }
9183 
9184     hdev->internal_cb_va_base = hl_reserve_va_block(hdev, ctx, HL_VA_RANGE_TYPE_HOST,
9185                     HOST_SPACE_INTERNAL_CB_SZ, HL_MMU_VA_ALIGNMENT_NOT_NEEDED);
9186 
9187     if (!hdev->internal_cb_va_base) {
9188         rc = -ENOMEM;
9189         goto destroy_internal_cb_pool;
9190     }
9191 
9192     mutex_lock(&ctx->mmu_lock);
9193     rc = hl_mmu_map_contiguous(ctx, hdev->internal_cb_va_base, hdev->internal_cb_pool_dma_addr,
9194                     HOST_SPACE_INTERNAL_CB_SZ);
9195     hl_mmu_invalidate_cache(hdev, false, MMU_OP_USERPTR);
9196     mutex_unlock(&ctx->mmu_lock);
9197 
9198     if (rc)
9199         goto unreserve_internal_cb_pool;
9200 
9201     return 0;
9202 
9203 unreserve_internal_cb_pool:
9204     hl_unreserve_va_block(hdev, ctx, hdev->internal_cb_va_base, HOST_SPACE_INTERNAL_CB_SZ);
9205 destroy_internal_cb_pool:
9206     gen_pool_destroy(hdev->internal_cb_pool);
9207 free_internal_cb_pool:
9208     hl_asic_dma_free_coherent(hdev, HOST_SPACE_INTERNAL_CB_SZ, hdev->internal_cb_pool_virt_addr,
9209                     hdev->internal_cb_pool_dma_addr);
9210 
9211     return rc;
9212 }
9213 
9214 static void gaudi2_internal_cb_pool_fini(struct hl_device *hdev, struct hl_ctx *ctx)
9215 {
9216     struct gaudi2_device *gaudi2 = hdev->asic_specific;
9217 
9218     if (!(gaudi2->hw_cap_initialized & HW_CAP_PMMU))
9219         return;
9220 
9221     mutex_lock(&ctx->mmu_lock);
9222     hl_mmu_unmap_contiguous(ctx, hdev->internal_cb_va_base, HOST_SPACE_INTERNAL_CB_SZ);
9223     hl_unreserve_va_block(hdev, ctx, hdev->internal_cb_va_base, HOST_SPACE_INTERNAL_CB_SZ);
9224     hl_mmu_invalidate_cache(hdev, true, MMU_OP_USERPTR);
9225     mutex_unlock(&ctx->mmu_lock);
9226 
9227     gen_pool_destroy(hdev->internal_cb_pool);
9228 
9229     hl_asic_dma_free_coherent(hdev, HOST_SPACE_INTERNAL_CB_SZ, hdev->internal_cb_pool_virt_addr,
9230                     hdev->internal_cb_pool_dma_addr);
9231 }
9232 
9233 static void gaudi2_restore_user_registers(struct hl_device *hdev)
9234 {
9235     gaudi2_restore_user_sm_registers(hdev);
9236     gaudi2_restore_user_qm_registers(hdev);
9237 }
9238 
9239 static int gaudi2_map_virtual_msix_doorbell_memory(struct hl_ctx *ctx)
9240 {
9241     struct hl_device *hdev = ctx->hdev;
9242     struct asic_fixed_properties *prop = &hdev->asic_prop;
9243     struct gaudi2_device *gaudi2 = hdev->asic_specific;
9244     int rc;
9245 
9246     rc = hl_mmu_map_page(ctx, RESERVED_VA_FOR_VIRTUAL_MSIX_DOORBELL_START,
9247                 gaudi2->virt_msix_db_dma_addr, prop->pmmu.page_size, true);
9248     if (rc)
9249         dev_err(hdev->dev, "Failed to map VA %#llx for virtual MSI-X doorbell memory\n",
9250             RESERVED_VA_FOR_VIRTUAL_MSIX_DOORBELL_START);
9251 
9252     return rc;
9253 }
9254 
9255 static void gaudi2_unmap_virtual_msix_doorbell_memory(struct hl_ctx *ctx)
9256 {
9257     struct hl_device *hdev = ctx->hdev;
9258     struct asic_fixed_properties *prop = &hdev->asic_prop;
9259     int rc;
9260 
9261     rc = hl_mmu_unmap_page(ctx, RESERVED_VA_FOR_VIRTUAL_MSIX_DOORBELL_START,
9262                 prop->pmmu.page_size, true);
9263     if (rc)
9264         dev_err(hdev->dev, "Failed to unmap VA %#llx of virtual MSI-X doorbell memory\n",
9265             RESERVED_VA_FOR_VIRTUAL_MSIX_DOORBELL_START);
9266 }
9267 
9268 static int gaudi2_ctx_init(struct hl_ctx *ctx)
9269 {
9270     int rc;
9271 
9272     rc = gaudi2_mmu_prepare(ctx->hdev, ctx->asid);
9273     if (rc)
9274         return rc;
9275 
9276     /* No need to clear user registers if the device has just
9277      * performed reset, we restore only nic qm registers
9278      */
9279     if (ctx->hdev->reset_upon_device_release)
9280         gaudi2_restore_nic_qm_registers(ctx->hdev);
9281     else
9282         gaudi2_restore_user_registers(ctx->hdev);
9283 
9284     rc = gaudi2_internal_cb_pool_init(ctx->hdev, ctx);
9285     if (rc)
9286         return rc;
9287 
9288     rc = gaudi2_map_virtual_msix_doorbell_memory(ctx);
9289     if (rc)
9290         gaudi2_internal_cb_pool_fini(ctx->hdev, ctx);
9291 
9292     return rc;
9293 }
9294 
9295 static void gaudi2_ctx_fini(struct hl_ctx *ctx)
9296 {
9297     if (ctx->asid == HL_KERNEL_ASID_ID)
9298         return;
9299 
9300     gaudi2_internal_cb_pool_fini(ctx->hdev, ctx);
9301 
9302     gaudi2_unmap_virtual_msix_doorbell_memory(ctx);
9303 }
9304 
9305 static int gaudi2_pre_schedule_cs(struct hl_cs *cs)
9306 {
9307     struct hl_device *hdev = cs->ctx->hdev;
9308     int index = cs->sequence & (hdev->asic_prop.max_pending_cs - 1);
9309     u32 mon_payload, sob_id, mon_id;
9310 
9311     if (!cs_needs_completion(cs))
9312         return 0;
9313 
9314     /*
9315      * First 64 SOB/MON are reserved for driver for QMAN auto completion
9316      * mechanism. Each SOB/MON pair are used for a pending CS with the same
9317      * cyclic index. The SOB value is increased when each of the CS jobs is
9318      * completed. When the SOB reaches the number of CS jobs, the monitor
9319      * generates MSI-X interrupt.
9320      */
9321 
9322     sob_id = mon_id = index;
9323     mon_payload = (1 << CQ_ENTRY_SHADOW_INDEX_VALID_SHIFT) |
9324                 (1 << CQ_ENTRY_READY_SHIFT) | index;
9325 
9326     gaudi2_arm_cq_monitor(hdev, sob_id, mon_id, GAUDI2_RESERVED_CQ_CS_COMPLETION, mon_payload,
9327                 cs->jobs_cnt);
9328 
9329     return 0;
9330 }
9331 
9332 static u32 gaudi2_get_queue_id_for_cq(struct hl_device *hdev, u32 cq_idx)
9333 {
9334     return HL_INVALID_QUEUE;
9335 }
9336 
9337 static u32 gaudi2_gen_signal_cb(struct hl_device *hdev, void *data, u16 sob_id, u32 size, bool eb)
9338 {
9339     struct hl_cb *cb = (struct hl_cb *) data;
9340     struct packet_msg_short *pkt;
9341     u32 value, ctl, pkt_size = sizeof(*pkt);
9342 
9343     pkt = (struct packet_msg_short *) (uintptr_t) (cb->kernel_address + size);
9344     memset(pkt, 0, pkt_size);
9345 
9346     /* Inc by 1, Mode ADD */
9347     value = FIELD_PREP(GAUDI2_PKT_SHORT_VAL_SOB_SYNC_VAL_MASK, 1);
9348     value |= FIELD_PREP(GAUDI2_PKT_SHORT_VAL_SOB_MOD_MASK, 1);
9349 
9350     ctl = FIELD_PREP(GAUDI2_PKT_SHORT_CTL_ADDR_MASK, sob_id * 4);
9351     ctl |= FIELD_PREP(GAUDI2_PKT_SHORT_CTL_BASE_MASK, 1); /* SOB base */
9352     ctl |= FIELD_PREP(GAUDI2_PKT_CTL_OPCODE_MASK, PACKET_MSG_SHORT);
9353     ctl |= FIELD_PREP(GAUDI2_PKT_CTL_EB_MASK, eb);
9354     ctl |= FIELD_PREP(GAUDI2_PKT_CTL_MB_MASK, 1);
9355 
9356     pkt->value = cpu_to_le32(value);
9357     pkt->ctl = cpu_to_le32(ctl);
9358 
9359     return size + pkt_size;
9360 }
9361 
9362 static u32 gaudi2_add_mon_msg_short(struct packet_msg_short *pkt, u32 value, u16 addr)
9363 {
9364     u32 ctl, pkt_size = sizeof(*pkt);
9365 
9366     memset(pkt, 0, pkt_size);
9367 
9368     ctl = FIELD_PREP(GAUDI2_PKT_SHORT_CTL_ADDR_MASK, addr);
9369     ctl |= FIELD_PREP(GAUDI2_PKT_SHORT_CTL_BASE_MASK, 0);  /* MON base */
9370     ctl |= FIELD_PREP(GAUDI2_PKT_CTL_OPCODE_MASK, PACKET_MSG_SHORT);
9371     ctl |= FIELD_PREP(GAUDI2_PKT_CTL_EB_MASK, 0);
9372     ctl |= FIELD_PREP(GAUDI2_PKT_CTL_MB_MASK, 0);
9373 
9374     pkt->value = cpu_to_le32(value);
9375     pkt->ctl = cpu_to_le32(ctl);
9376 
9377     return pkt_size;
9378 }
9379 
9380 static u32 gaudi2_add_arm_monitor_pkt(struct hl_device *hdev, struct packet_msg_short *pkt,
9381                     u16 sob_base, u8 sob_mask, u16 sob_val, u16 addr)
9382 {
9383     u32 ctl, value, pkt_size = sizeof(*pkt);
9384     u8 mask;
9385 
9386     if (hl_gen_sob_mask(sob_base, sob_mask, &mask)) {
9387         dev_err(hdev->dev, "sob_base %u (mask %#x) is not valid\n", sob_base, sob_mask);
9388         return 0;
9389     }
9390 
9391     memset(pkt, 0, pkt_size);
9392 
9393     value = FIELD_PREP(GAUDI2_PKT_SHORT_VAL_MON_SYNC_GID_MASK, sob_base / 8);
9394     value |= FIELD_PREP(GAUDI2_PKT_SHORT_VAL_MON_SYNC_VAL_MASK, sob_val);
9395     value |= FIELD_PREP(GAUDI2_PKT_SHORT_VAL_MON_MODE_MASK, 0); /* GREATER OR EQUAL*/
9396     value |= FIELD_PREP(GAUDI2_PKT_SHORT_VAL_MON_MASK_MASK, mask);
9397 
9398     ctl = FIELD_PREP(GAUDI2_PKT_SHORT_CTL_ADDR_MASK, addr);
9399     ctl |= FIELD_PREP(GAUDI2_PKT_SHORT_CTL_BASE_MASK, 0); /* MON base */
9400     ctl |= FIELD_PREP(GAUDI2_PKT_CTL_OPCODE_MASK, PACKET_MSG_SHORT);
9401     ctl |= FIELD_PREP(GAUDI2_PKT_CTL_EB_MASK, 0);
9402     ctl |= FIELD_PREP(GAUDI2_PKT_CTL_MB_MASK, 1);
9403 
9404     pkt->value = cpu_to_le32(value);
9405     pkt->ctl = cpu_to_le32(ctl);
9406 
9407     return pkt_size;
9408 }
9409 
9410 static u32 gaudi2_add_fence_pkt(struct packet_fence *pkt)
9411 {
9412     u32 ctl, cfg, pkt_size = sizeof(*pkt);
9413 
9414     memset(pkt, 0, pkt_size);
9415 
9416     cfg = FIELD_PREP(GAUDI2_PKT_FENCE_CFG_DEC_VAL_MASK, 1);
9417     cfg |= FIELD_PREP(GAUDI2_PKT_FENCE_CFG_TARGET_VAL_MASK, 1);
9418     cfg |= FIELD_PREP(GAUDI2_PKT_FENCE_CFG_ID_MASK, 2);
9419 
9420     ctl = FIELD_PREP(GAUDI2_PKT_CTL_OPCODE_MASK, PACKET_FENCE);
9421     ctl |= FIELD_PREP(GAUDI2_PKT_CTL_EB_MASK, 0);
9422     ctl |= FIELD_PREP(GAUDI2_PKT_CTL_MB_MASK, 1);
9423 
9424     pkt->cfg = cpu_to_le32(cfg);
9425     pkt->ctl = cpu_to_le32(ctl);
9426 
9427     return pkt_size;
9428 }
9429 
9430 static u32 gaudi2_gen_wait_cb(struct hl_device *hdev, struct hl_gen_wait_properties *prop)
9431 {
9432     struct hl_cb *cb = (struct hl_cb *) prop->data;
9433     void *buf = (void *) (uintptr_t) (cb->kernel_address);
9434 
9435     u64 monitor_base, fence_addr = 0;
9436     u32 stream_index, size = prop->size;
9437     u16 msg_addr_offset;
9438 
9439     stream_index = prop->q_idx % 4;
9440     fence_addr = CFG_BASE + gaudi2_qm_blocks_bases[prop->q_idx] +
9441             QM_FENCE2_OFFSET + stream_index * 4;
9442 
9443     /*
9444      * monitor_base should be the content of the base0 address registers,
9445      * so it will be added to the msg short offsets
9446      */
9447     monitor_base = mmDCORE0_SYNC_MNGR_OBJS_MON_PAY_ADDRL_0;
9448 
9449     /* First monitor config packet: low address of the sync */
9450     msg_addr_offset = (mmDCORE0_SYNC_MNGR_OBJS_MON_PAY_ADDRL_0 + prop->mon_id * 4) -
9451                 monitor_base;
9452 
9453     size += gaudi2_add_mon_msg_short(buf + size, (u32) fence_addr, msg_addr_offset);
9454 
9455     /* Second monitor config packet: high address of the sync */
9456     msg_addr_offset = (mmDCORE0_SYNC_MNGR_OBJS_MON_PAY_ADDRH_0 + prop->mon_id * 4) -
9457                 monitor_base;
9458 
9459     size += gaudi2_add_mon_msg_short(buf + size, (u32) (fence_addr >> 32), msg_addr_offset);
9460 
9461     /*
9462      * Third monitor config packet: the payload, i.e. what to write when the
9463      * sync triggers
9464      */
9465     msg_addr_offset = (mmDCORE0_SYNC_MNGR_OBJS_MON_PAY_DATA_0 + prop->mon_id * 4) -
9466                 monitor_base;
9467 
9468     size += gaudi2_add_mon_msg_short(buf + size, 1, msg_addr_offset);
9469 
9470     /* Fourth monitor config packet: bind the monitor to a sync object */
9471     msg_addr_offset = (mmDCORE0_SYNC_MNGR_OBJS_MON_ARM_0 + prop->mon_id * 4) - monitor_base;
9472 
9473     size += gaudi2_add_arm_monitor_pkt(hdev, buf + size, prop->sob_base, prop->sob_mask,
9474                         prop->sob_val, msg_addr_offset);
9475 
9476     /* Fence packet */
9477     size += gaudi2_add_fence_pkt(buf + size);
9478 
9479     return size;
9480 }
9481 
9482 static void gaudi2_reset_sob(struct hl_device *hdev, void *data)
9483 {
9484     struct hl_hw_sob *hw_sob = (struct hl_hw_sob *) data;
9485 
9486     dev_dbg(hdev->dev, "reset SOB, q_idx: %d, sob_id: %d\n", hw_sob->q_idx, hw_sob->sob_id);
9487 
9488     WREG32(mmDCORE0_SYNC_MNGR_OBJS_SOB_OBJ_0 + hw_sob->sob_id * 4, 0);
9489 
9490     kref_init(&hw_sob->kref);
9491 }
9492 
9493 static void gaudi2_reset_sob_group(struct hl_device *hdev, u16 sob_group)
9494 {
9495 }
9496 
9497 static u64 gaudi2_get_device_time(struct hl_device *hdev)
9498 {
9499     u64 device_time = ((u64) RREG32(mmPSOC_TIMESTAMP_CNTCVU)) << 32;
9500 
9501     return device_time | RREG32(mmPSOC_TIMESTAMP_CNTCVL);
9502 }
9503 
9504 static int gaudi2_collective_wait_init_cs(struct hl_cs *cs)
9505 {
9506     return 0;
9507 }
9508 
9509 static int gaudi2_collective_wait_create_jobs(struct hl_device *hdev, struct hl_ctx *ctx,
9510                     struct hl_cs *cs, u32 wait_queue_id,
9511                     u32 collective_engine_id, u32 encaps_signal_offset)
9512 {
9513     return -EINVAL;
9514 }
9515 
9516 /*
9517  * hl_mmu_scramble - converts a dram (non power of 2) page-size aligned address
9518  *                   to DMMU page-size address (64MB) before mapping it in
9519  *                   the MMU.
9520  * The operation is performed on both the virtual and physical addresses.
9521  * for device with 6 HBMs the scramble is:
9522  * (addr[47:0] / 48M) * 64M + addr % 48M + addr[63:48]
9523  *
9524  * Example:
9525  * =============================================================================
9526  * Allocated DRAM  Reserved VA      scrambled VA for MMU mapping    Scrambled PA
9527  * Phys address                                                     in MMU last
9528  *                                                                    HOP
9529  * =============================================================================
9530  * PA1 0x3000000  VA1 0x9C000000  SVA1= (VA1/48M)*64M 0xD0000000  <- PA1/48M 0x1
9531  * PA2 0x9000000  VA2 0x9F000000  SVA2= (VA2/48M)*64M 0xD4000000  <- PA2/48M 0x3
9532  * =============================================================================
9533  */
9534 static u64 gaudi2_mmu_scramble_addr(struct hl_device *hdev, u64 raw_addr)
9535 {
9536     struct asic_fixed_properties *prop = &hdev->asic_prop;
9537     u32 divisor, mod_va;
9538     u64 div_va;
9539 
9540     /* accept any address in the DRAM address space */
9541     if (hl_mem_area_inside_range(raw_addr, sizeof(raw_addr), DRAM_PHYS_BASE,
9542                                     VA_HBM_SPACE_END)) {
9543 
9544         divisor = prop->num_functional_hbms * GAUDI2_HBM_MMU_SCRM_MEM_SIZE;
9545         div_va = div_u64_rem(raw_addr & GAUDI2_HBM_MMU_SCRM_ADDRESS_MASK, divisor, &mod_va);
9546         return (raw_addr & ~GAUDI2_HBM_MMU_SCRM_ADDRESS_MASK) |
9547             (div_va << GAUDI2_HBM_MMU_SCRM_DIV_SHIFT) |
9548             (mod_va << GAUDI2_HBM_MMU_SCRM_MOD_SHIFT);
9549     }
9550 
9551     return raw_addr;
9552 }
9553 
9554 static u64 gaudi2_mmu_descramble_addr(struct hl_device *hdev, u64 scrambled_addr)
9555 {
9556     struct asic_fixed_properties *prop = &hdev->asic_prop;
9557     u32 divisor, mod_va;
9558     u64 div_va;
9559 
9560     /* accept any address in the DRAM address space */
9561     if (hl_mem_area_inside_range(scrambled_addr, sizeof(scrambled_addr), DRAM_PHYS_BASE,
9562                                     VA_HBM_SPACE_END)) {
9563 
9564         divisor = prop->num_functional_hbms * GAUDI2_HBM_MMU_SCRM_MEM_SIZE;
9565         div_va = div_u64_rem(scrambled_addr & GAUDI2_HBM_MMU_SCRM_ADDRESS_MASK,
9566                     PAGE_SIZE_64MB, &mod_va);
9567 
9568         return ((scrambled_addr & ~GAUDI2_HBM_MMU_SCRM_ADDRESS_MASK) +
9569                     (div_va * divisor + mod_va));
9570     }
9571 
9572     return scrambled_addr;
9573 }
9574 
9575 static u32 gaudi2_get_dec_base_addr(struct hl_device *hdev, u32 core_id)
9576 {
9577     u32 base = 0, dcore_id, dec_id;
9578 
9579     if (core_id >= NUMBER_OF_DEC) {
9580         dev_err(hdev->dev, "Unexpected core number %d for DEC\n", core_id);
9581         goto out;
9582     }
9583 
9584     if (core_id < 8) {
9585         dcore_id = core_id / NUM_OF_DEC_PER_DCORE;
9586         dec_id = core_id % NUM_OF_DEC_PER_DCORE;
9587 
9588         base = mmDCORE0_DEC0_CMD_BASE + dcore_id * DCORE_OFFSET +
9589                 dec_id * DCORE_VDEC_OFFSET;
9590     } else {
9591         /* PCIe Shared Decoder */
9592         base = mmPCIE_DEC0_CMD_BASE + ((core_id % 8) * PCIE_VDEC_OFFSET);
9593     }
9594 out:
9595     return base;
9596 }
9597 
9598 static int gaudi2_get_hw_block_id(struct hl_device *hdev, u64 block_addr,
9599                 u32 *block_size, u32 *block_id)
9600 {
9601     struct gaudi2_device *gaudi2 = hdev->asic_specific;
9602     int i;
9603 
9604     for (i = 0 ; i < NUM_USER_MAPPED_BLOCKS ; i++) {
9605         if (block_addr == CFG_BASE + gaudi2->mapped_blocks[i].address) {
9606             *block_id = i;
9607             if (block_size)
9608                 *block_size = gaudi2->mapped_blocks[i].size;
9609             return 0;
9610         }
9611     }
9612 
9613     dev_err(hdev->dev, "Invalid block address %#llx", block_addr);
9614 
9615     return -EINVAL;
9616 }
9617 
9618 static int gaudi2_block_mmap(struct hl_device *hdev, struct vm_area_struct *vma,
9619             u32 block_id, u32 block_size)
9620 {
9621     struct gaudi2_device *gaudi2 = hdev->asic_specific;
9622     u64 offset_in_bar;
9623     u64 address;
9624     int rc;
9625 
9626     if (block_id >= NUM_USER_MAPPED_BLOCKS) {
9627         dev_err(hdev->dev, "Invalid block id %u", block_id);
9628         return -EINVAL;
9629     }
9630 
9631     /* we allow mapping only an entire block */
9632     if (block_size != gaudi2->mapped_blocks[block_id].size) {
9633         dev_err(hdev->dev, "Invalid block size %u", block_size);
9634         return -EINVAL;
9635     }
9636 
9637     offset_in_bar = CFG_BASE + gaudi2->mapped_blocks[block_id].address - STM_FLASH_BASE_ADDR;
9638 
9639     address = pci_resource_start(hdev->pdev, SRAM_CFG_BAR_ID) + offset_in_bar;
9640 
9641     vma->vm_flags |= VM_IO | VM_PFNMAP | VM_DONTEXPAND | VM_DONTDUMP |
9642             VM_DONTCOPY | VM_NORESERVE;
9643 
9644     rc = remap_pfn_range(vma, vma->vm_start, address >> PAGE_SHIFT,
9645             block_size, vma->vm_page_prot);
9646     if (rc)
9647         dev_err(hdev->dev, "remap_pfn_range error %d", rc);
9648 
9649     return rc;
9650 }
9651 
9652 static void gaudi2_enable_events_from_fw(struct hl_device *hdev)
9653 {
9654     struct gaudi2_device *gaudi2 = hdev->asic_specific;
9655 
9656     struct cpu_dyn_regs *dyn_regs = &hdev->fw_loader.dynamic_loader.comm_desc.cpu_dyn_regs;
9657     u32 irq_handler_offset = le32_to_cpu(dyn_regs->gic_host_ints_irq);
9658 
9659     if (gaudi2->hw_cap_initialized & HW_CAP_CPU_Q)
9660         WREG32(irq_handler_offset,
9661             gaudi2_irq_map_table[GAUDI2_EVENT_CPU_INTS_REGISTER].cpu_id);
9662 }
9663 
9664 static int gaudi2_get_mmu_base(struct hl_device *hdev, u64 mmu_id, u32 *mmu_base)
9665 {
9666     switch (mmu_id) {
9667     case HW_CAP_DCORE0_DMMU0:
9668         *mmu_base = mmDCORE0_HMMU0_MMU_BASE;
9669         break;
9670     case HW_CAP_DCORE0_DMMU1:
9671         *mmu_base = mmDCORE0_HMMU1_MMU_BASE;
9672         break;
9673     case HW_CAP_DCORE0_DMMU2:
9674         *mmu_base = mmDCORE0_HMMU2_MMU_BASE;
9675         break;
9676     case HW_CAP_DCORE0_DMMU3:
9677         *mmu_base = mmDCORE0_HMMU3_MMU_BASE;
9678         break;
9679     case HW_CAP_DCORE1_DMMU0:
9680         *mmu_base = mmDCORE1_HMMU0_MMU_BASE;
9681         break;
9682     case HW_CAP_DCORE1_DMMU1:
9683         *mmu_base = mmDCORE1_HMMU1_MMU_BASE;
9684         break;
9685     case HW_CAP_DCORE1_DMMU2:
9686         *mmu_base = mmDCORE1_HMMU2_MMU_BASE;
9687         break;
9688     case HW_CAP_DCORE1_DMMU3:
9689         *mmu_base = mmDCORE1_HMMU3_MMU_BASE;
9690         break;
9691     case HW_CAP_DCORE2_DMMU0:
9692         *mmu_base = mmDCORE2_HMMU0_MMU_BASE;
9693         break;
9694     case HW_CAP_DCORE2_DMMU1:
9695         *mmu_base = mmDCORE2_HMMU1_MMU_BASE;
9696         break;
9697     case HW_CAP_DCORE2_DMMU2:
9698         *mmu_base = mmDCORE2_HMMU2_MMU_BASE;
9699         break;
9700     case HW_CAP_DCORE2_DMMU3:
9701         *mmu_base = mmDCORE2_HMMU3_MMU_BASE;
9702         break;
9703     case HW_CAP_DCORE3_DMMU0:
9704         *mmu_base = mmDCORE3_HMMU0_MMU_BASE;
9705         break;
9706     case HW_CAP_DCORE3_DMMU1:
9707         *mmu_base = mmDCORE3_HMMU1_MMU_BASE;
9708         break;
9709     case HW_CAP_DCORE3_DMMU2:
9710         *mmu_base = mmDCORE3_HMMU2_MMU_BASE;
9711         break;
9712     case HW_CAP_DCORE3_DMMU3:
9713         *mmu_base = mmDCORE3_HMMU3_MMU_BASE;
9714         break;
9715     case HW_CAP_PMMU:
9716         *mmu_base = mmPMMU_HBW_MMU_BASE;
9717         break;
9718     default:
9719         return -EINVAL;
9720     }
9721 
9722     return 0;
9723 }
9724 
9725 static void gaudi2_ack_mmu_error(struct hl_device *hdev, u64 mmu_id)
9726 {
9727     bool is_pmmu = (mmu_id == HW_CAP_PMMU ? true : false);
9728     struct gaudi2_device *gaudi2 = hdev->asic_specific;
9729     u32 mmu_base;
9730 
9731     if (!(gaudi2->hw_cap_initialized & mmu_id))
9732         return;
9733 
9734     if (gaudi2_get_mmu_base(hdev, mmu_id, &mmu_base))
9735         return;
9736 
9737     gaudi2_handle_page_error(hdev, mmu_base, is_pmmu);
9738     gaudi2_handle_access_error(hdev, mmu_base, is_pmmu);
9739 }
9740 
9741 static int gaudi2_ack_mmu_page_fault_or_access_error(struct hl_device *hdev, u64 mmu_cap_mask)
9742 {
9743     u32 i, mmu_id, num_of_hmmus = NUM_OF_HMMU_PER_DCORE * NUM_OF_DCORES;
9744 
9745     /* check all HMMUs */
9746     for (i = 0 ; i < num_of_hmmus ; i++) {
9747         mmu_id = HW_CAP_DCORE0_DMMU0 << i;
9748 
9749         if (mmu_cap_mask & mmu_id)
9750             gaudi2_ack_mmu_error(hdev, mmu_id);
9751     }
9752 
9753     /* check PMMU */
9754     if (mmu_cap_mask & HW_CAP_PMMU)
9755         gaudi2_ack_mmu_error(hdev, HW_CAP_PMMU);
9756 
9757     return 0;
9758 }
9759 
9760 static void gaudi2_get_msi_info(__le32 *table)
9761 {
9762     table[CPUCP_EVENT_QUEUE_MSI_TYPE] = cpu_to_le32(GAUDI2_EVENT_QUEUE_MSIX_IDX);
9763 }
9764 
9765 static int gaudi2_map_pll_idx_to_fw_idx(u32 pll_idx)
9766 {
9767     switch (pll_idx) {
9768     case HL_GAUDI2_CPU_PLL: return CPU_PLL;
9769     case HL_GAUDI2_PCI_PLL: return PCI_PLL;
9770     case HL_GAUDI2_NIC_PLL: return NIC_PLL;
9771     case HL_GAUDI2_DMA_PLL: return DMA_PLL;
9772     case HL_GAUDI2_MESH_PLL: return MESH_PLL;
9773     case HL_GAUDI2_MME_PLL: return MME_PLL;
9774     case HL_GAUDI2_TPC_PLL: return TPC_PLL;
9775     case HL_GAUDI2_IF_PLL: return IF_PLL;
9776     case HL_GAUDI2_SRAM_PLL: return SRAM_PLL;
9777     case HL_GAUDI2_HBM_PLL: return HBM_PLL;
9778     case HL_GAUDI2_VID_PLL: return VID_PLL;
9779     case HL_GAUDI2_MSS_PLL: return MSS_PLL;
9780     default: return -EINVAL;
9781     }
9782 }
9783 
9784 static int gaudi2_gen_sync_to_engine_map(struct hl_device *hdev, struct hl_sync_to_engine_map *map)
9785 {
9786     /* Not implemented */
9787     return 0;
9788 }
9789 
9790 static int gaudi2_monitor_valid(struct hl_mon_state_dump *mon)
9791 {
9792     /* Not implemented */
9793     return 0;
9794 }
9795 
9796 static int gaudi2_print_single_monitor(char **buf, size_t *size, size_t *offset,
9797                 struct hl_device *hdev, struct hl_mon_state_dump *mon)
9798 {
9799     /* Not implemented */
9800     return 0;
9801 }
9802 
9803 
9804 static int gaudi2_print_fences_single_engine(struct hl_device *hdev, u64 base_offset,
9805                 u64 status_base_offset, enum hl_sync_engine_type engine_type,
9806                 u32 engine_id, char **buf, size_t *size, size_t *offset)
9807 {
9808     /* Not implemented */
9809     return 0;
9810 }
9811 
9812 
9813 static struct hl_state_dump_specs_funcs gaudi2_state_dump_funcs = {
9814     .monitor_valid = gaudi2_monitor_valid,
9815     .print_single_monitor = gaudi2_print_single_monitor,
9816     .gen_sync_to_engine_map = gaudi2_gen_sync_to_engine_map,
9817     .print_fences_single_engine = gaudi2_print_fences_single_engine,
9818 };
9819 
9820 static void gaudi2_state_dump_init(struct hl_device *hdev)
9821 {
9822     /* Not implemented */
9823     hdev->state_dump_specs.props = gaudi2_state_dump_specs_props;
9824     hdev->state_dump_specs.funcs = gaudi2_state_dump_funcs;
9825 }
9826 
9827 static u32 gaudi2_get_sob_addr(struct hl_device *hdev, u32 sob_id)
9828 {
9829     return 0;
9830 }
9831 
9832 static u32 *gaudi2_get_stream_master_qid_arr(void)
9833 {
9834     return NULL;
9835 }
9836 
9837 static void gaudi2_add_device_attr(struct hl_device *hdev, struct attribute_group *dev_clk_attr_grp,
9838                 struct attribute_group *dev_vrm_attr_grp)
9839 {
9840     hl_sysfs_add_dev_clk_attr(hdev, dev_clk_attr_grp);
9841     hl_sysfs_add_dev_vrm_attr(hdev, dev_vrm_attr_grp);
9842 }
9843 
9844 static int gaudi2_mmu_get_real_page_size(struct hl_device *hdev, struct hl_mmu_properties *mmu_prop,
9845                     u32 page_size, u32 *real_page_size, bool is_dram_addr)
9846 {
9847     struct asic_fixed_properties *prop = &hdev->asic_prop;
9848 
9849     /* for host pages the page size must be  */
9850     if (!is_dram_addr) {
9851         if (page_size % mmu_prop->page_size)
9852             goto page_size_err;
9853 
9854         *real_page_size = mmu_prop->page_size;
9855         return 0;
9856     }
9857 
9858     if ((page_size % prop->dram_page_size) || (prop->dram_page_size > mmu_prop->page_size))
9859         goto page_size_err;
9860 
9861     /*
9862      * MMU page size is different from DRAM page size (more precisely, DMMU page is greater
9863      * than DRAM page size).
9864      * for this reason work with the DRAM page size and let the MMU scrambling routine handle
9865      * this mismatch when calculating the address to place in the MMU page table.
9866      * (in that case also make sure that the dram_page_size is not greater than the
9867      * mmu page size)
9868      */
9869     *real_page_size = prop->dram_page_size;
9870 
9871     return 0;
9872 
9873 page_size_err:
9874     dev_err(hdev->dev, "page size of %u is not %uKB aligned, can't map\n",
9875                             page_size, mmu_prop->page_size >> 10);
9876     return -EFAULT;
9877 }
9878 
9879 static int gaudi2_get_monitor_dump(struct hl_device *hdev, void *data)
9880 {
9881     return -EOPNOTSUPP;
9882 }
9883 
9884 static const struct hl_asic_funcs gaudi2_funcs = {
9885     .early_init = gaudi2_early_init,
9886     .early_fini = gaudi2_early_fini,
9887     .late_init = gaudi2_late_init,
9888     .late_fini = gaudi2_late_fini,
9889     .sw_init = gaudi2_sw_init,
9890     .sw_fini = gaudi2_sw_fini,
9891     .hw_init = gaudi2_hw_init,
9892     .hw_fini = gaudi2_hw_fini,
9893     .halt_engines = gaudi2_halt_engines,
9894     .suspend = gaudi2_suspend,
9895     .resume = gaudi2_resume,
9896     .mmap = gaudi2_mmap,
9897     .ring_doorbell = gaudi2_ring_doorbell,
9898     .pqe_write = gaudi2_pqe_write,
9899     .asic_dma_alloc_coherent = gaudi2_dma_alloc_coherent,
9900     .asic_dma_free_coherent = gaudi2_dma_free_coherent,
9901     .scrub_device_mem = gaudi2_scrub_device_mem,
9902     .scrub_device_dram = gaudi2_scrub_device_dram,
9903     .get_int_queue_base = NULL,
9904     .test_queues = gaudi2_test_queues,
9905     .asic_dma_pool_zalloc = gaudi2_dma_pool_zalloc,
9906     .asic_dma_pool_free = gaudi2_dma_pool_free,
9907     .cpu_accessible_dma_pool_alloc = gaudi2_cpu_accessible_dma_pool_alloc,
9908     .cpu_accessible_dma_pool_free = gaudi2_cpu_accessible_dma_pool_free,
9909     .asic_dma_unmap_single = gaudi2_dma_unmap_single,
9910     .asic_dma_map_single = gaudi2_dma_map_single,
9911     .hl_dma_unmap_sgtable = hl_dma_unmap_sgtable,
9912     .cs_parser = gaudi2_cs_parser,
9913     .asic_dma_map_sgtable = hl_dma_map_sgtable,
9914     .add_end_of_cb_packets = NULL,
9915     .update_eq_ci = gaudi2_update_eq_ci,
9916     .context_switch = gaudi2_context_switch,
9917     .restore_phase_topology = gaudi2_restore_phase_topology,
9918     .debugfs_read_dma = gaudi2_debugfs_read_dma,
9919     .add_device_attr = gaudi2_add_device_attr,
9920     .handle_eqe = gaudi2_handle_eqe,
9921     .get_events_stat = gaudi2_get_events_stat,
9922     .read_pte = NULL,
9923     .write_pte = NULL,
9924     .mmu_invalidate_cache = gaudi2_mmu_invalidate_cache,
9925     .mmu_invalidate_cache_range = gaudi2_mmu_invalidate_cache_range,
9926     .mmu_prefetch_cache_range = NULL,
9927     .send_heartbeat = gaudi2_send_heartbeat,
9928     .debug_coresight = gaudi2_debug_coresight,
9929     .is_device_idle = gaudi2_is_device_idle,
9930     .non_hard_reset_late_init = gaudi2_non_hard_reset_late_init,
9931     .hw_queues_lock = gaudi2_hw_queues_lock,
9932     .hw_queues_unlock = gaudi2_hw_queues_unlock,
9933     .kdma_lock = gaudi2_kdma_lock,
9934     .kdma_unlock = gaudi2_kdma_unlock,
9935     .get_pci_id = gaudi2_get_pci_id,
9936     .get_eeprom_data = gaudi2_get_eeprom_data,
9937     .get_monitor_dump = gaudi2_get_monitor_dump,
9938     .send_cpu_message = gaudi2_send_cpu_message,
9939     .pci_bars_map = gaudi2_pci_bars_map,
9940     .init_iatu = gaudi2_init_iatu,
9941     .rreg = hl_rreg,
9942     .wreg = hl_wreg,
9943     .halt_coresight = gaudi2_halt_coresight,
9944     .ctx_init = gaudi2_ctx_init,
9945     .ctx_fini = gaudi2_ctx_fini,
9946     .pre_schedule_cs = gaudi2_pre_schedule_cs,
9947     .get_queue_id_for_cq = gaudi2_get_queue_id_for_cq,
9948     .load_firmware_to_device = NULL,
9949     .load_boot_fit_to_device = NULL,
9950     .get_signal_cb_size = gaudi2_get_signal_cb_size,
9951     .get_wait_cb_size = gaudi2_get_wait_cb_size,
9952     .gen_signal_cb = gaudi2_gen_signal_cb,
9953     .gen_wait_cb = gaudi2_gen_wait_cb,
9954     .reset_sob = gaudi2_reset_sob,
9955     .reset_sob_group = gaudi2_reset_sob_group,
9956     .get_device_time = gaudi2_get_device_time,
9957     .pb_print_security_errors = gaudi2_pb_print_security_errors,
9958     .collective_wait_init_cs = gaudi2_collective_wait_init_cs,
9959     .collective_wait_create_jobs = gaudi2_collective_wait_create_jobs,
9960     .get_dec_base_addr = gaudi2_get_dec_base_addr,
9961     .scramble_addr = gaudi2_mmu_scramble_addr,
9962     .descramble_addr = gaudi2_mmu_descramble_addr,
9963     .ack_protection_bits_errors = gaudi2_ack_protection_bits_errors,
9964     .get_hw_block_id = gaudi2_get_hw_block_id,
9965     .hw_block_mmap = gaudi2_block_mmap,
9966     .enable_events_from_fw = gaudi2_enable_events_from_fw,
9967     .ack_mmu_errors = gaudi2_ack_mmu_page_fault_or_access_error,
9968     .get_msi_info = gaudi2_get_msi_info,
9969     .map_pll_idx_to_fw_idx = gaudi2_map_pll_idx_to_fw_idx,
9970     .init_firmware_preload_params = gaudi2_init_firmware_preload_params,
9971     .init_firmware_loader = gaudi2_init_firmware_loader,
9972     .init_cpu_scrambler_dram = gaudi2_init_scrambler_hbm,
9973     .state_dump_init = gaudi2_state_dump_init,
9974     .get_sob_addr = &gaudi2_get_sob_addr,
9975     .set_pci_memory_regions = gaudi2_set_pci_memory_regions,
9976     .get_stream_master_qid_arr = gaudi2_get_stream_master_qid_arr,
9977     .check_if_razwi_happened = gaudi2_check_if_razwi_happened,
9978     .mmu_get_real_page_size = gaudi2_mmu_get_real_page_size,
9979     .access_dev_mem = hl_access_dev_mem,
9980     .set_dram_bar_base = gaudi2_set_hbm_bar_base,
9981 };
9982 
9983 void gaudi2_set_asic_funcs(struct hl_device *hdev)
9984 {
9985     hdev->asic_funcs = &gaudi2_funcs;
9986 }
[ Source navigation ] [ Diff markup ] [ Identifier search ] [ general search ]

This page was automatically generated by the 2.1.0 LXR engine.

The LXR team
Valid CSS 2.1! Valid HTML 4.01!