amd/amdgpu/si.c

0001 /*
0002  * Copyright 2015 Advanced Micro Devices, Inc.
0003  *
0004  * Permission is hereby granted, free of charge, to any person obtaining a
0005  * copy of this software and associated documentation files (the "Software"),
0006  * to deal in the Software without restriction, including without limitation
0007  * the rights to use, copy, modify, merge, publish, distribute, sublicense,
0008  * and/or sell copies of the Software, and to permit persons to whom the
0009  * Software is furnished to do so, subject to the following conditions:
0010  *
0011  * The above copyright notice and this permission notice shall be included in
0012  * all copies or substantial portions of the Software.
0013  *
0014  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
0015  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
0016  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
0017  * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
0018  * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
0019  * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
0020  * OTHER DEALINGS IN THE SOFTWARE.
0021  *
0022  */
0023
0024 #include <linux/firmware.h>
0025 #include <linux/slab.h>
0026 #include <linux/module.h>
0027 #include <linux/pci.h>
0028
0029 #include <drm/amdgpu_drm.h>
0030
0031 #include "amdgpu.h"
0032 #include "amdgpu_atombios.h"
0033 #include "amdgpu_ih.h"
0034 #include "amdgpu_uvd.h"
0035 #include "amdgpu_vce.h"
0036 #include "atom.h"
0037 #include "amd_pcie.h"
0038 #include "si_dpm.h"
0039 #include "sid.h"
0040 #include "si_ih.h"
0041 #include "gfx_v6_0.h"
0042 #include "gmc_v6_0.h"
0043 #include "si_dma.h"
0044 #include "dce_v6_0.h"
0045 #include "si.h"
0046 #include "uvd_v3_1.h"
0047 #include "amdgpu_vkms.h"
0048 #include "gca/gfx_6_0_d.h"
0049 #include "oss/oss_1_0_d.h"
0050 #include "oss/oss_1_0_sh_mask.h"
0051 #include "gmc/gmc_6_0_d.h"
0052 #include "dce/dce_6_0_d.h"
0053 #include "uvd/uvd_4_0_d.h"
0054 #include "bif/bif_3_0_d.h"
0055 #include "bif/bif_3_0_sh_mask.h"
0056
0057 #include "amdgpu_dm.h"
0058
0059 static const u32 tahiti_golden_registers[] =
0060 {
0061     mmAZALIA_SCLK_CONTROL, 0x00000030, 0x00000011,
0062     mmCB_HW_CONTROL, 0x00010000, 0x00018208,
0063     mmDB_DEBUG, 0xffffffff, 0x00000000,
0064     mmDB_DEBUG2, 0xf00fffff, 0x00000400,
0065     mmDB_DEBUG3, 0x0002021c, 0x00020200,
0066     mmDCI_CLK_CNTL, 0x00000080, 0x00000000,
0067     0x340c, 0x000000c0, 0x00800040,
0068     0x360c, 0x000000c0, 0x00800040,
0069     mmFBC_DEBUG_COMP, 0x000000f0, 0x00000070,
0070     mmFBC_MISC, 0x00200000, 0x50100000,
0071     mmDIG0_HDMI_CONTROL, 0x31000311, 0x00000011,
0072     mmMC_ARB_WTM_CNTL_RD, 0x00000003, 0x000007ff,
0073     mmMC_XPB_P2P_BAR_CFG, 0x000007ff, 0x00000000,
0074     mmPA_CL_ENHANCE, 0xf000001f, 0x00000007,
0075     mmPA_SC_FORCE_EOV_MAX_CNTS, 0xffffffff, 0x00ffffff,
0076     mmPA_SC_LINE_STIPPLE_STATE, 0x0000ff0f, 0x00000000,
0077     mmPA_SC_MODE_CNTL_1, 0x07ffffff, 0x4e000000,
0078     mmPA_SC_RASTER_CONFIG, 0x3f3f3fff, 0x2a00126a,
0079     0x000c, 0xffffffff, 0x0040,
0080     0x000d, 0x00000040, 0x00004040,
0081     mmSPI_CONFIG_CNTL, 0x07ffffff, 0x03000000,
0082     mmSQ_DED_CNT, 0x01ff1f3f, 0x00000000,
0083     mmSQ_SEC_CNT, 0x01ff1f3f, 0x00000000,
0084     mmSX_DEBUG_1, 0x0000007f, 0x00000020,
0085     mmTA_CNTL_AUX, 0x00010000, 0x00010000,
0086     mmTCP_ADDR_CONFIG, 0x00000200, 0x000002fb,
0087     mmTCP_CHAN_STEER_HI, 0xffffffff, 0x0000543b,
0088     mmTCP_CHAN_STEER_LO, 0xffffffff, 0xa9210876,
0089     mmVGT_FIFO_DEPTHS, 0xffffffff, 0x000fff40,
0090     mmVGT_GS_VERTEX_REUSE, 0x0000001f, 0x00000010,
0091     mmVM_CONTEXT0_CNTL, 0x20000000, 0x20fffed8,
0092     mmVM_L2_CG, 0x000c0fc0, 0x000c0400,
0093     mmVM_PRT_APERTURE0_LOW_ADDR, 0x0fffffff, 0xffffffff,
0094     mmVM_PRT_APERTURE1_LOW_ADDR, 0x0fffffff, 0x0fffffff,
0095     mmVM_PRT_APERTURE2_LOW_ADDR, 0x0fffffff, 0x0fffffff,
0096     mmVM_PRT_APERTURE3_LOW_ADDR, 0x0fffffff, 0x0fffffff,
0097 };
0098
0099 static const u32 tahiti_golden_registers2[] =
0100 {
0101     mmMCIF_MEM_CONTROL, 0x00000001, 0x00000001,
0102 };
0103
0104 static const u32 tahiti_golden_rlc_registers[] =
0105 {
0106     mmGB_ADDR_CONFIG, 0xffffffff, 0x12011003,
0107     mmRLC_LB_PARAMS, 0xffffffff, 0x00601005,
0108     0x311f, 0xffffffff, 0x10104040,
0109     0x3122, 0xffffffff, 0x0100000a,
0110     mmRLC_LB_CNTR_MAX, 0xffffffff, 0x00000800,
0111     mmRLC_LB_CNTL, 0xffffffff, 0x800000f4,
0112     mmUVD_CGC_GATE, 0x00000008, 0x00000000,
0113 };
0114
0115 static const u32 pitcairn_golden_registers[] =
0116 {
0117     mmAZALIA_SCLK_CONTROL, 0x00000030, 0x00000011,
0118     mmCB_HW_CONTROL, 0x00010000, 0x00018208,
0119     mmDB_DEBUG, 0xffffffff, 0x00000000,
0120     mmDB_DEBUG2, 0xf00fffff, 0x00000400,
0121     mmDB_DEBUG3, 0x0002021c, 0x00020200,
0122     mmDCI_CLK_CNTL, 0x00000080, 0x00000000,
0123     0x340c, 0x000300c0, 0x00800040,
0124     0x360c, 0x000300c0, 0x00800040,
0125     mmFBC_DEBUG_COMP, 0x000000f0, 0x00000070,
0126     mmFBC_MISC, 0x00200000, 0x50100000,
0127     mmDIG0_HDMI_CONTROL, 0x31000311, 0x00000011,
0128     mmMC_SEQ_PMG_PG_HWCNTL, 0x00073ffe, 0x000022a2,
0129     mmMC_XPB_P2P_BAR_CFG, 0x000007ff, 0x00000000,
0130     mmPA_CL_ENHANCE, 0xf000001f, 0x00000007,
0131     mmPA_SC_FORCE_EOV_MAX_CNTS, 0xffffffff, 0x00ffffff,
0132     mmPA_SC_LINE_STIPPLE_STATE, 0x0000ff0f, 0x00000000,
0133     mmPA_SC_MODE_CNTL_1, 0x07ffffff, 0x4e000000,
0134     mmPA_SC_RASTER_CONFIG, 0x3f3f3fff, 0x2a00126a,
0135     0x000c, 0xffffffff, 0x0040,
0136     0x000d, 0x00000040, 0x00004040,
0137     mmSPI_CONFIG_CNTL, 0x07ffffff, 0x03000000,
0138     mmSX_DEBUG_1, 0x0000007f, 0x00000020,
0139     mmTA_CNTL_AUX, 0x00010000, 0x00010000,
0140     mmTCP_ADDR_CONFIG, 0x000003ff, 0x000000f7,
0141     mmTCP_CHAN_STEER_HI, 0xffffffff, 0x00000000,
0142     mmTCP_CHAN_STEER_LO, 0xffffffff, 0x32761054,
0143     mmVGT_GS_VERTEX_REUSE, 0x0000001f, 0x00000010,
0144     mmVM_L2_CG, 0x000c0fc0, 0x000c0400,
0145     mmVM_PRT_APERTURE0_LOW_ADDR, 0x0fffffff, 0xffffffff,
0146     mmVM_PRT_APERTURE1_LOW_ADDR, 0x0fffffff, 0x0fffffff,
0147     mmVM_PRT_APERTURE2_LOW_ADDR, 0x0fffffff, 0x0fffffff,
0148     mmVM_PRT_APERTURE3_LOW_ADDR, 0x0fffffff, 0x0fffffff,
0149 };
0150
0151 static const u32 pitcairn_golden_rlc_registers[] =
0152 {
0153     mmGB_ADDR_CONFIG, 0xffffffff, 0x12011003,
0154     mmRLC_LB_PARAMS, 0xffffffff, 0x00601004,
0155     0x311f, 0xffffffff, 0x10102020,
0156     0x3122, 0xffffffff, 0x01000020,
0157     mmRLC_LB_CNTR_MAX, 0xffffffff, 0x00000800,
0158     mmRLC_LB_CNTL, 0xffffffff, 0x800000a4,
0159 };
0160
0161 static const u32 verde_pg_init[] =
0162 {
0163     mmGMCON_PGFSM_WRITE, 0xffffffff, 0x40000,
0164     mmGMCON_PGFSM_CONFIG, 0xffffffff, 0x200010ff,
0165     mmGMCON_PGFSM_WRITE, 0xffffffff, 0x0,
0166     mmGMCON_PGFSM_WRITE, 0xffffffff, 0x0,
0167     mmGMCON_PGFSM_WRITE, 0xffffffff, 0x0,
0168     mmGMCON_PGFSM_WRITE, 0xffffffff, 0x0,
0169     mmGMCON_PGFSM_WRITE, 0xffffffff, 0x0,
0170     mmGMCON_PGFSM_WRITE, 0xffffffff, 0x7007,
0171     mmGMCON_PGFSM_CONFIG, 0xffffffff, 0x300010ff,
0172     mmGMCON_PGFSM_WRITE, 0xffffffff, 0x0,
0173     mmGMCON_PGFSM_WRITE, 0xffffffff, 0x0,
0174     mmGMCON_PGFSM_WRITE, 0xffffffff, 0x0,
0175     mmGMCON_PGFSM_WRITE, 0xffffffff, 0x0,
0176     mmGMCON_PGFSM_WRITE, 0xffffffff, 0x0,
0177     mmGMCON_PGFSM_WRITE, 0xffffffff, 0x400000,
0178     mmGMCON_PGFSM_CONFIG, 0xffffffff, 0x100010ff,
0179     mmGMCON_PGFSM_WRITE, 0xffffffff, 0x0,
0180     mmGMCON_PGFSM_WRITE, 0xffffffff, 0x0,
0181     mmGMCON_PGFSM_WRITE, 0xffffffff, 0x0,
0182     mmGMCON_PGFSM_WRITE, 0xffffffff, 0x0,
0183     mmGMCON_PGFSM_WRITE, 0xffffffff, 0x0,
0184     mmGMCON_PGFSM_WRITE, 0xffffffff, 0x120200,
0185     mmGMCON_PGFSM_CONFIG, 0xffffffff, 0x500010ff,
0186     mmGMCON_PGFSM_WRITE, 0xffffffff, 0x0,
0187     mmGMCON_PGFSM_WRITE, 0xffffffff, 0x0,
0188     mmGMCON_PGFSM_WRITE, 0xffffffff, 0x0,
0189     mmGMCON_PGFSM_WRITE, 0xffffffff, 0x0,
0190     mmGMCON_PGFSM_WRITE, 0xffffffff, 0x0,
0191     mmGMCON_PGFSM_WRITE, 0xffffffff, 0x1e1e16,
0192     mmGMCON_PGFSM_CONFIG, 0xffffffff, 0x600010ff,
0193     mmGMCON_PGFSM_WRITE, 0xffffffff, 0x0,
0194     mmGMCON_PGFSM_WRITE, 0xffffffff, 0x0,
0195     mmGMCON_PGFSM_WRITE, 0xffffffff, 0x0,
0196     mmGMCON_PGFSM_WRITE, 0xffffffff, 0x0,
0197     mmGMCON_PGFSM_WRITE, 0xffffffff, 0x0,
0198     mmGMCON_PGFSM_WRITE, 0xffffffff, 0x171f1e,
0199     mmGMCON_PGFSM_CONFIG, 0xffffffff, 0x700010ff,
0200     mmGMCON_PGFSM_WRITE, 0xffffffff, 0x0,
0201     mmGMCON_PGFSM_WRITE, 0xffffffff, 0x0,
0202     mmGMCON_PGFSM_WRITE, 0xffffffff, 0x0,
0203     mmGMCON_PGFSM_WRITE, 0xffffffff, 0x0,
0204     mmGMCON_PGFSM_WRITE, 0xffffffff, 0x0,
0205     mmGMCON_PGFSM_WRITE, 0xffffffff, 0x0,
0206     mmGMCON_PGFSM_CONFIG, 0xffffffff, 0x9ff,
0207     mmGMCON_RENG_RAM_INDEX, 0xffffffff, 0x0,
0208     mmGMCON_RENG_RAM_DATA, 0xffffffff, 0x10000800,
0209     mmGMCON_RENG_RAM_DATA, 0xffffffff, 0xf,
0210     mmGMCON_RENG_RAM_DATA, 0xffffffff, 0xf,
0211     mmGMCON_RENG_RAM_INDEX, 0xffffffff, 0x4,
0212     mmGMCON_RENG_RAM_DATA, 0xffffffff, 0x1000051e,
0213     mmGMCON_RENG_RAM_DATA, 0xffffffff, 0xffff,
0214     mmGMCON_RENG_RAM_DATA, 0xffffffff, 0xffff,
0215     mmGMCON_RENG_RAM_INDEX, 0xffffffff, 0x8,
0216     mmGMCON_RENG_RAM_DATA, 0xffffffff, 0x80500,
0217     mmGMCON_RENG_RAM_INDEX, 0xffffffff, 0x12,
0218     mmGMCON_RENG_RAM_DATA, 0xffffffff, 0x9050c,
0219     mmGMCON_RENG_RAM_INDEX, 0xffffffff, 0x1d,
0220     mmGMCON_RENG_RAM_DATA, 0xffffffff, 0xb052c,
0221     mmGMCON_RENG_RAM_INDEX, 0xffffffff, 0x2a,
0222     mmGMCON_RENG_RAM_DATA, 0xffffffff, 0x1053e,
0223     mmGMCON_RENG_RAM_INDEX, 0xffffffff, 0x2d,
0224     mmGMCON_RENG_RAM_DATA, 0xffffffff, 0x10546,
0225     mmGMCON_RENG_RAM_INDEX, 0xffffffff, 0x30,
0226     mmGMCON_RENG_RAM_DATA, 0xffffffff, 0xa054e,
0227     mmGMCON_RENG_RAM_INDEX, 0xffffffff, 0x3c,
0228     mmGMCON_RENG_RAM_DATA, 0xffffffff, 0x1055f,
0229     mmGMCON_RENG_RAM_INDEX, 0xffffffff, 0x3f,
0230     mmGMCON_RENG_RAM_DATA, 0xffffffff, 0x10567,
0231     mmGMCON_RENG_RAM_INDEX, 0xffffffff, 0x42,
0232     mmGMCON_RENG_RAM_DATA, 0xffffffff, 0x1056f,
0233     mmGMCON_RENG_RAM_INDEX, 0xffffffff, 0x45,
0234     mmGMCON_RENG_RAM_DATA, 0xffffffff, 0x10572,
0235     mmGMCON_RENG_RAM_INDEX, 0xffffffff, 0x48,
0236     mmGMCON_RENG_RAM_DATA, 0xffffffff, 0x20575,
0237     mmGMCON_RENG_RAM_INDEX, 0xffffffff, 0x4c,
0238     mmGMCON_RENG_RAM_DATA, 0xffffffff, 0x190801,
0239     mmGMCON_RENG_RAM_INDEX, 0xffffffff, 0x67,
0240     mmGMCON_RENG_RAM_DATA, 0xffffffff, 0x1082a,
0241     mmGMCON_RENG_RAM_INDEX, 0xffffffff, 0x6a,
0242     mmGMCON_RENG_RAM_DATA, 0xffffffff, 0x1b082d,
0243     mmGMCON_RENG_RAM_INDEX, 0xffffffff, 0x87,
0244     mmGMCON_RENG_RAM_DATA, 0xffffffff, 0x310851,
0245     mmGMCON_RENG_RAM_INDEX, 0xffffffff, 0xba,
0246     mmGMCON_RENG_RAM_DATA, 0xffffffff, 0x891,
0247     mmGMCON_RENG_RAM_INDEX, 0xffffffff, 0xbc,
0248     mmGMCON_RENG_RAM_DATA, 0xffffffff, 0x893,
0249     mmGMCON_RENG_RAM_INDEX, 0xffffffff, 0xbe,
0250     mmGMCON_RENG_RAM_DATA, 0xffffffff, 0x20895,
0251     mmGMCON_RENG_RAM_INDEX, 0xffffffff, 0xc2,
0252     mmGMCON_RENG_RAM_DATA, 0xffffffff, 0x20899,
0253     mmGMCON_RENG_RAM_INDEX, 0xffffffff, 0xc6,
0254     mmGMCON_RENG_RAM_DATA, 0xffffffff, 0x2089d,
0255     mmGMCON_RENG_RAM_INDEX, 0xffffffff, 0xca,
0256     mmGMCON_RENG_RAM_DATA, 0xffffffff, 0x8a1,
0257     mmGMCON_RENG_RAM_INDEX, 0xffffffff, 0xcc,
0258     mmGMCON_RENG_RAM_DATA, 0xffffffff, 0x8a3,
0259     mmGMCON_RENG_RAM_INDEX, 0xffffffff, 0xce,
0260     mmGMCON_RENG_RAM_DATA, 0xffffffff, 0x308a5,
0261     mmGMCON_RENG_RAM_INDEX, 0xffffffff, 0xd3,
0262     mmGMCON_RENG_RAM_DATA, 0xffffffff, 0x6d08cd,
0263     mmGMCON_RENG_RAM_INDEX, 0xffffffff, 0x142,
0264     mmGMCON_RENG_RAM_DATA, 0xffffffff, 0x2000095a,
0265     mmGMCON_RENG_RAM_DATA, 0xffffffff, 0x1,
0266     mmGMCON_RENG_RAM_INDEX, 0xffffffff, 0x144,
0267     mmGMCON_RENG_RAM_DATA, 0xffffffff, 0x301f095b,
0268     mmGMCON_RENG_RAM_INDEX, 0xffffffff, 0x165,
0269     mmGMCON_RENG_RAM_DATA, 0xffffffff, 0xc094d,
0270     mmGMCON_RENG_RAM_INDEX, 0xffffffff, 0x173,
0271     mmGMCON_RENG_RAM_DATA, 0xffffffff, 0xf096d,
0272     mmGMCON_RENG_RAM_INDEX, 0xffffffff, 0x184,
0273     mmGMCON_RENG_RAM_DATA, 0xffffffff, 0x15097f,
0274     mmGMCON_RENG_RAM_INDEX, 0xffffffff, 0x19b,
0275     mmGMCON_RENG_RAM_DATA, 0xffffffff, 0xc0998,
0276     mmGMCON_RENG_RAM_INDEX, 0xffffffff, 0x1a9,
0277     mmGMCON_RENG_RAM_DATA, 0xffffffff, 0x409a7,
0278     mmGMCON_RENG_RAM_INDEX, 0xffffffff, 0x1af,
0279     mmGMCON_RENG_RAM_DATA, 0xffffffff, 0xcdc,
0280     mmGMCON_RENG_RAM_INDEX, 0xffffffff, 0x1b1,
0281     mmGMCON_RENG_RAM_DATA, 0xffffffff, 0x800,
0282     mmGMCON_RENG_EXECUTE, 0xffffffff, 0x6c9b2000,
0283     mmGMCON_MISC2, 0xfc00, 0x2000,
0284     mmGMCON_MISC3, 0xffffffff, 0xfc0,
0285     mmMC_PMG_AUTO_CFG, 0x00000100, 0x100,
0286 };
0287
0288 static const u32 verde_golden_rlc_registers[] =
0289 {
0290     mmGB_ADDR_CONFIG, 0xffffffff, 0x02010002,
0291     mmRLC_LB_PARAMS, 0xffffffff, 0x033f1005,
0292     0x311f, 0xffffffff, 0x10808020,
0293     0x3122, 0xffffffff, 0x00800008,
0294     mmRLC_LB_CNTR_MAX, 0xffffffff, 0x00001000,
0295     mmRLC_LB_CNTL, 0xffffffff, 0x80010014,
0296 };
0297
0298 static const u32 verde_golden_registers[] =
0299 {
0300     mmAZALIA_SCLK_CONTROL, 0x00000030, 0x00000011,
0301     mmCB_HW_CONTROL, 0x00010000, 0x00018208,
0302     mmDB_DEBUG, 0xffffffff, 0x00000000,
0303     mmDB_DEBUG2, 0xf00fffff, 0x00000400,
0304     mmDB_DEBUG3, 0x0002021c, 0x00020200,
0305     mmDCI_CLK_CNTL, 0x00000080, 0x00000000,
0306     0x340c, 0x000300c0, 0x00800040,
0307     0x360c, 0x000300c0, 0x00800040,
0308     mmFBC_DEBUG_COMP, 0x000000f0, 0x00000070,
0309     mmFBC_MISC, 0x00200000, 0x50100000,
0310     mmDIG0_HDMI_CONTROL, 0x31000311, 0x00000011,
0311     mmMC_SEQ_PMG_PG_HWCNTL, 0x00073ffe, 0x000022a2,
0312     mmMC_XPB_P2P_BAR_CFG, 0x000007ff, 0x00000000,
0313     mmPA_CL_ENHANCE, 0xf000001f, 0x00000007,
0314     mmPA_SC_FORCE_EOV_MAX_CNTS, 0xffffffff, 0x00ffffff,
0315     mmPA_SC_LINE_STIPPLE_STATE, 0x0000ff0f, 0x00000000,
0316     mmPA_SC_MODE_CNTL_1, 0x07ffffff, 0x4e000000,
0317     mmPA_SC_RASTER_CONFIG, 0x3f3f3fff, 0x0000124a,
0318     0x000c, 0xffffffff, 0x0040,
0319     0x000d, 0x00000040, 0x00004040,
0320     mmSPI_CONFIG_CNTL, 0x07ffffff, 0x03000000,
0321     mmSQ_DED_CNT, 0x01ff1f3f, 0x00000000,
0322     mmSQ_SEC_CNT, 0x01ff1f3f, 0x00000000,
0323     mmSX_DEBUG_1, 0x0000007f, 0x00000020,
0324     mmTA_CNTL_AUX, 0x00010000, 0x00010000,
0325     mmTCP_ADDR_CONFIG, 0x000003ff, 0x00000003,
0326     mmTCP_CHAN_STEER_HI, 0xffffffff, 0x00000000,
0327     mmTCP_CHAN_STEER_LO, 0xffffffff, 0x00001032,
0328     mmVGT_GS_VERTEX_REUSE, 0x0000001f, 0x00000010,
0329     mmVM_L2_CG, 0x000c0fc0, 0x000c0400,
0330     mmVM_PRT_APERTURE0_LOW_ADDR, 0x0fffffff, 0xffffffff,
0331     mmVM_PRT_APERTURE1_LOW_ADDR, 0x0fffffff, 0x0fffffff,
0332     mmVM_PRT_APERTURE2_LOW_ADDR, 0x0fffffff, 0x0fffffff,
0333     mmVM_PRT_APERTURE3_LOW_ADDR, 0x0fffffff, 0x0fffffff,
0334 };
0335
0336 static const u32 oland_golden_registers[] =
0337 {
0338     mmAZALIA_SCLK_CONTROL, 0x00000030, 0x00000011,
0339     mmCB_HW_CONTROL, 0x00010000, 0x00018208,
0340     mmDB_DEBUG, 0xffffffff, 0x00000000,
0341     mmDB_DEBUG2, 0xf00fffff, 0x00000400,
0342     mmDB_DEBUG3, 0x0002021c, 0x00020200,
0343     mmDCI_CLK_CNTL, 0x00000080, 0x00000000,
0344     0x340c, 0x000300c0, 0x00800040,
0345     0x360c, 0x000300c0, 0x00800040,
0346     mmFBC_DEBUG_COMP, 0x000000f0, 0x00000070,
0347     mmFBC_MISC, 0x00200000, 0x50100000,
0348     mmDIG0_HDMI_CONTROL, 0x31000311, 0x00000011,
0349     mmMC_SEQ_PMG_PG_HWCNTL, 0x00073ffe, 0x000022a2,
0350     mmMC_XPB_P2P_BAR_CFG, 0x000007ff, 0x00000000,
0351     mmPA_CL_ENHANCE, 0xf000001f, 0x00000007,
0352     mmPA_SC_FORCE_EOV_MAX_CNTS, 0xffffffff, 0x00ffffff,
0353     mmPA_SC_LINE_STIPPLE_STATE, 0x0000ff0f, 0x00000000,
0354     mmPA_SC_MODE_CNTL_1, 0x07ffffff, 0x4e000000,
0355     mmPA_SC_RASTER_CONFIG, 0x3f3f3fff, 0x00000082,
0356     0x000c, 0xffffffff, 0x0040,
0357     0x000d, 0x00000040, 0x00004040,
0358     mmSPI_CONFIG_CNTL, 0x07ffffff, 0x03000000,
0359     mmSX_DEBUG_1, 0x0000007f, 0x00000020,
0360     mmTA_CNTL_AUX, 0x00010000, 0x00010000,
0361     mmTCP_ADDR_CONFIG, 0x000003ff, 0x000000f3,
0362     mmTCP_CHAN_STEER_HI, 0xffffffff, 0x00000000,
0363     mmTCP_CHAN_STEER_LO, 0xffffffff, 0x00003210,
0364     mmVGT_GS_VERTEX_REUSE, 0x0000001f, 0x00000010,
0365     mmVM_L2_CG, 0x000c0fc0, 0x000c0400,
0366     mmVM_PRT_APERTURE0_LOW_ADDR, 0x0fffffff, 0xffffffff,
0367     mmVM_PRT_APERTURE1_LOW_ADDR, 0x0fffffff, 0x0fffffff,
0368     mmVM_PRT_APERTURE2_LOW_ADDR, 0x0fffffff, 0x0fffffff,
0369     mmVM_PRT_APERTURE3_LOW_ADDR, 0x0fffffff, 0x0fffffff,
0370
0371 };
0372
0373 static const u32 oland_golden_rlc_registers[] =
0374 {
0375     mmGB_ADDR_CONFIG, 0xffffffff, 0x02010002,
0376     mmRLC_LB_PARAMS, 0xffffffff, 0x00601005,
0377     0x311f, 0xffffffff, 0x10104040,
0378     0x3122, 0xffffffff, 0x0100000a,
0379     mmRLC_LB_CNTR_MAX, 0xffffffff, 0x00000800,
0380     mmRLC_LB_CNTL, 0xffffffff, 0x800000f4,
0381 };
0382
0383 static const u32 hainan_golden_registers[] =
0384 {
0385     0x17bc, 0x00000030, 0x00000011,
0386     mmCB_HW_CONTROL, 0x00010000, 0x00018208,
0387     mmDB_DEBUG, 0xffffffff, 0x00000000,
0388     mmDB_DEBUG2, 0xf00fffff, 0x00000400,
0389     mmDB_DEBUG3, 0x0002021c, 0x00020200,
0390     0x031e, 0x00000080, 0x00000000,
0391     0x3430, 0xff000fff, 0x00000100,
0392     0x340c, 0x000300c0, 0x00800040,
0393     0x3630, 0xff000fff, 0x00000100,
0394     0x360c, 0x000300c0, 0x00800040,
0395     0x16ec, 0x000000f0, 0x00000070,
0396     0x16f0, 0x00200000, 0x50100000,
0397     0x1c0c, 0x31000311, 0x00000011,
0398     mmMC_SEQ_PMG_PG_HWCNTL, 0x00073ffe, 0x000022a2,
0399     mmMC_XPB_P2P_BAR_CFG, 0x000007ff, 0x00000000,
0400     mmPA_CL_ENHANCE, 0xf000001f, 0x00000007,
0401     mmPA_SC_FORCE_EOV_MAX_CNTS, 0xffffffff, 0x00ffffff,
0402     mmPA_SC_LINE_STIPPLE_STATE, 0x0000ff0f, 0x00000000,
0403     mmPA_SC_MODE_CNTL_1, 0x07ffffff, 0x4e000000,
0404     mmPA_SC_RASTER_CONFIG, 0x3f3f3fff, 0x00000000,
0405     0x000c, 0xffffffff, 0x0040,
0406     0x000d, 0x00000040, 0x00004040,
0407     mmSPI_CONFIG_CNTL, 0x03e00000, 0x03600000,
0408     mmSX_DEBUG_1, 0x0000007f, 0x00000020,
0409     mmTA_CNTL_AUX, 0x00010000, 0x00010000,
0410     mmTCP_ADDR_CONFIG, 0x000003ff, 0x000000f1,
0411     mmTCP_CHAN_STEER_HI, 0xffffffff, 0x00000000,
0412     mmTCP_CHAN_STEER_LO, 0xffffffff, 0x00003210,
0413     mmVGT_GS_VERTEX_REUSE, 0x0000001f, 0x00000010,
0414     mmVM_L2_CG, 0x000c0fc0, 0x000c0400,
0415     mmVM_PRT_APERTURE0_LOW_ADDR, 0x0fffffff, 0xffffffff,
0416     mmVM_PRT_APERTURE1_LOW_ADDR, 0x0fffffff, 0x0fffffff,
0417     mmVM_PRT_APERTURE2_LOW_ADDR, 0x0fffffff, 0x0fffffff,
0418     mmVM_PRT_APERTURE3_LOW_ADDR, 0x0fffffff, 0x0fffffff,
0419 };
0420
0421 static const u32 hainan_golden_registers2[] =
0422 {
0423     mmGB_ADDR_CONFIG, 0xffffffff, 0x2011003,
0424 };
0425
0426 static const u32 tahiti_mgcg_cgcg_init[] =
0427 {
0428     mmRLC_CGTT_MGCG_OVERRIDE, 0xffffffff, 0xfffffffc,
0429     mmGRBM_GFX_INDEX, 0xffffffff, 0xe0000000,
0430     mmCB_CGTT_SCLK_CTRL, 0xffffffff, 0x00000100,
0431     mmCGTT_BCI_CLK_CTRL, 0xffffffff, 0x00000100,
0432     mmCGTT_CP_CLK_CTRL, 0xffffffff, 0x00000100,
0433     mmCGTT_GDS_CLK_CTRL, 0xffffffff, 0x00000100,
0434     mmCGTT_IA_CLK_CTRL, 0xffffffff, 0x06000100,
0435     mmCGTT_PA_CLK_CTRL, 0xffffffff, 0x00000100,
0436     mmCGTT_PC_CLK_CTRL, 0xffffffff, 0x00000100,
0437     mmCGTT_RLC_CLK_CTRL, 0xffffffff, 0x00000100,
0438     mmCGTT_SC_CLK_CTRL, 0xffffffff, 0x00000100,
0439     mmCGTT_SPI_CLK_CTRL, 0xffffffff, 0x00000100,
0440     mmCGTT_SQ_CLK_CTRL, 0xffffffff, 0x00000100,
0441     mmCGTT_SQG_CLK_CTRL, 0xffffffff, 0x00000100,
0442     mmCGTT_SX_CLK_CTRL0, 0xffffffff, 0x00000100,
0443     mmCGTT_SX_CLK_CTRL1, 0xffffffff, 0x00000100,
0444     mmCGTT_SX_CLK_CTRL2, 0xffffffff, 0x00000100,
0445     mmCGTT_SX_CLK_CTRL3, 0xffffffff, 0x00000100,
0446     mmCGTT_TCI_CLK_CTRL, 0xffffffff, 0x00000100,
0447     mmCGTT_TCP_CLK_CTRL, 0xffffffff, 0x00000100,
0448     mmCGTT_VGT_CLK_CTRL, 0xffffffff, 0x06000100,
0449     mmDB_CGTT_CLK_CTRL_0, 0xffffffff, 0x00000100,
0450     mmTA_CGTT_CTRL, 0xffffffff, 0x00000100,
0451     mmTCA_CGTT_SCLK_CTRL, 0xffffffff, 0x00000100,
0452     mmTCC_CGTT_SCLK_CTRL, 0xffffffff, 0x00000100,
0453     mmTD_CGTT_CTRL, 0xffffffff, 0x00000100,
0454     mmGRBM_GFX_INDEX, 0xffffffff, 0xe0000000,
0455     0x2458, 0xffffffff, 0x00010000,
0456     0x2459, 0xffffffff, 0x00030002,
0457     0x245a, 0xffffffff, 0x00040007,
0458     0x245b, 0xffffffff, 0x00060005,
0459     0x245c, 0xffffffff, 0x00090008,
0460     0x245d, 0xffffffff, 0x00020001,
0461     0x245e, 0xffffffff, 0x00040003,
0462     0x245f, 0xffffffff, 0x00000007,
0463     0x2460, 0xffffffff, 0x00060005,
0464     0x2461, 0xffffffff, 0x00090008,
0465     0x2462, 0xffffffff, 0x00030002,
0466     0x2463, 0xffffffff, 0x00050004,
0467     0x2464, 0xffffffff, 0x00000008,
0468     0x2465, 0xffffffff, 0x00070006,
0469     0x2466, 0xffffffff, 0x000a0009,
0470     0x2467, 0xffffffff, 0x00040003,
0471     0x2468, 0xffffffff, 0x00060005,
0472     0x2469, 0xffffffff, 0x00000009,
0473     0x246a, 0xffffffff, 0x00080007,
0474     0x246b, 0xffffffff, 0x000b000a,
0475     0x246c, 0xffffffff, 0x00050004,
0476     0x246d, 0xffffffff, 0x00070006,
0477     0x246e, 0xffffffff, 0x0008000b,
0478     0x246f, 0xffffffff, 0x000a0009,
0479     0x2470, 0xffffffff, 0x000d000c,
0480     0x2471, 0xffffffff, 0x00060005,
0481     0x2472, 0xffffffff, 0x00080007,
0482     0x2473, 0xffffffff, 0x0000000b,
0483     0x2474, 0xffffffff, 0x000a0009,
0484     0x2475, 0xffffffff, 0x000d000c,
0485     0x2476, 0xffffffff, 0x00070006,
0486     0x2477, 0xffffffff, 0x00090008,
0487     0x2478, 0xffffffff, 0x0000000c,
0488     0x2479, 0xffffffff, 0x000b000a,
0489     0x247a, 0xffffffff, 0x000e000d,
0490     0x247b, 0xffffffff, 0x00080007,
0491     0x247c, 0xffffffff, 0x000a0009,
0492     0x247d, 0xffffffff, 0x0000000d,
0493     0x247e, 0xffffffff, 0x000c000b,
0494     0x247f, 0xffffffff, 0x000f000e,
0495     0x2480, 0xffffffff, 0x00090008,
0496     0x2481, 0xffffffff, 0x000b000a,
0497     0x2482, 0xffffffff, 0x000c000f,
0498     0x2483, 0xffffffff, 0x000e000d,
0499     0x2484, 0xffffffff, 0x00110010,
0500     0x2485, 0xffffffff, 0x000a0009,
0501     0x2486, 0xffffffff, 0x000c000b,
0502     0x2487, 0xffffffff, 0x0000000f,
0503     0x2488, 0xffffffff, 0x000e000d,
0504     0x2489, 0xffffffff, 0x00110010,
0505     0x248a, 0xffffffff, 0x000b000a,
0506     0x248b, 0xffffffff, 0x000d000c,
0507     0x248c, 0xffffffff, 0x00000010,
0508     0x248d, 0xffffffff, 0x000f000e,
0509     0x248e, 0xffffffff, 0x00120011,
0510     0x248f, 0xffffffff, 0x000c000b,
0511     0x2490, 0xffffffff, 0x000e000d,
0512     0x2491, 0xffffffff, 0x00000011,
0513     0x2492, 0xffffffff, 0x0010000f,
0514     0x2493, 0xffffffff, 0x00130012,
0515     0x2494, 0xffffffff, 0x000d000c,
0516     0x2495, 0xffffffff, 0x000f000e,
0517     0x2496, 0xffffffff, 0x00100013,
0518     0x2497, 0xffffffff, 0x00120011,
0519     0x2498, 0xffffffff, 0x00150014,
0520     0x2499, 0xffffffff, 0x000e000d,
0521     0x249a, 0xffffffff, 0x0010000f,
0522     0x249b, 0xffffffff, 0x00000013,
0523     0x249c, 0xffffffff, 0x00120011,
0524     0x249d, 0xffffffff, 0x00150014,
0525     0x249e, 0xffffffff, 0x000f000e,
0526     0x249f, 0xffffffff, 0x00110010,
0527     0x24a0, 0xffffffff, 0x00000014,
0528     0x24a1, 0xffffffff, 0x00130012,
0529     0x24a2, 0xffffffff, 0x00160015,
0530     0x24a3, 0xffffffff, 0x0010000f,
0531     0x24a4, 0xffffffff, 0x00120011,
0532     0x24a5, 0xffffffff, 0x00000015,
0533     0x24a6, 0xffffffff, 0x00140013,
0534     0x24a7, 0xffffffff, 0x00170016,
0535     mmCGTS_SM_CTRL_REG, 0xffffffff, 0x96940200,
0536     mmCP_RB_WPTR_POLL_CNTL, 0xffffffff, 0x00900100,
0537     mmRLC_GCPM_GENERAL_3, 0xffffffff, 0x00000080,
0538     mmRLC_CGCG_CGLS_CTRL, 0xffffffff, 0x0020003f,
0539     0x000c, 0xffffffff, 0x0000001c,
0540     0x000d, 0x000f0000, 0x000f0000,
0541     0x0583, 0xffffffff, 0x00000100,
0542     mmXDMA_CLOCK_GATING_CNTL, 0xffffffff, 0x00000100,
0543     mmXDMA_MEM_POWER_CNTL, 0x00000101, 0x00000000,
0544     mmMC_MEM_POWER_LS, 0xffffffff, 0x00000104,
0545     mmMC_CITF_MISC_WR_CG, 0x000c0000, 0x000c0000,
0546     mmMC_CITF_MISC_RD_CG, 0x000c0000, 0x000c0000,
0547     mmCGTT_DRM_CLK_CTRL0, 0xff000fff, 0x00000100,
0548     0x157a, 0x00000001, 0x00000001,
0549     mmHDP_MEM_POWER_LS, 0x00000001, 0x00000001,
0550     mmHDP_XDP_CGTT_BLK_CTRL, 0xc0000fff, 0x00000104,
0551     mmCP_MEM_SLP_CNTL, 0x00000001, 0x00000001,
0552     0x3430, 0xfffffff0, 0x00000100,
0553     0x3630, 0xfffffff0, 0x00000100,
0554 };
0555 static const u32 pitcairn_mgcg_cgcg_init[] =
0556 {
0557     mmRLC_CGTT_MGCG_OVERRIDE, 0xffffffff, 0xfffffffc,
0558     mmGRBM_GFX_INDEX, 0xffffffff, 0xe0000000,
0559     mmCB_CGTT_SCLK_CTRL, 0xffffffff, 0x00000100,
0560     mmCGTT_BCI_CLK_CTRL, 0xffffffff, 0x00000100,
0561     mmCGTT_CP_CLK_CTRL, 0xffffffff, 0x00000100,
0562     mmCGTT_GDS_CLK_CTRL, 0xffffffff, 0x00000100,
0563     mmCGTT_IA_CLK_CTRL, 0xffffffff, 0x06000100,
0564     mmCGTT_PA_CLK_CTRL, 0xffffffff, 0x00000100,
0565     mmCGTT_PC_CLK_CTRL, 0xffffffff, 0x00000100,
0566     mmCGTT_RLC_CLK_CTRL, 0xffffffff, 0x00000100,
0567     mmCGTT_SC_CLK_CTRL, 0xffffffff, 0x00000100,
0568     mmCGTT_SPI_CLK_CTRL, 0xffffffff, 0x00000100,
0569     mmCGTT_SQ_CLK_CTRL, 0xffffffff, 0x00000100,
0570     mmCGTT_SQG_CLK_CTRL, 0xffffffff, 0x00000100,
0571     mmCGTT_SX_CLK_CTRL0, 0xffffffff, 0x00000100,
0572     mmCGTT_SX_CLK_CTRL1, 0xffffffff, 0x00000100,
0573     mmCGTT_SX_CLK_CTRL2, 0xffffffff, 0x00000100,
0574     mmCGTT_SX_CLK_CTRL3, 0xffffffff, 0x00000100,
0575     mmCGTT_TCI_CLK_CTRL, 0xffffffff, 0x00000100,
0576     mmCGTT_TCP_CLK_CTRL, 0xffffffff, 0x00000100,
0577     mmCGTT_VGT_CLK_CTRL, 0xffffffff, 0x06000100,
0578     mmDB_CGTT_CLK_CTRL_0, 0xffffffff, 0x00000100,
0579     mmTA_CGTT_CTRL, 0xffffffff, 0x00000100,
0580     mmTCA_CGTT_SCLK_CTRL, 0xffffffff, 0x00000100,
0581     mmTCC_CGTT_SCLK_CTRL, 0xffffffff, 0x00000100,
0582     mmTD_CGTT_CTRL, 0xffffffff, 0x00000100,
0583     mmGRBM_GFX_INDEX, 0xffffffff, 0xe0000000,
0584     0x2458, 0xffffffff, 0x00010000,
0585     0x2459, 0xffffffff, 0x00030002,
0586     0x245a, 0xffffffff, 0x00040007,
0587     0x245b, 0xffffffff, 0x00060005,
0588     0x245c, 0xffffffff, 0x00090008,
0589     0x245d, 0xffffffff, 0x00020001,
0590     0x245e, 0xffffffff, 0x00040003,
0591     0x245f, 0xffffffff, 0x00000007,
0592     0x2460, 0xffffffff, 0x00060005,
0593     0x2461, 0xffffffff, 0x00090008,
0594     0x2462, 0xffffffff, 0x00030002,
0595     0x2463, 0xffffffff, 0x00050004,
0596     0x2464, 0xffffffff, 0x00000008,
0597     0x2465, 0xffffffff, 0x00070006,
0598     0x2466, 0xffffffff, 0x000a0009,
0599     0x2467, 0xffffffff, 0x00040003,
0600     0x2468, 0xffffffff, 0x00060005,
0601     0x2469, 0xffffffff, 0x00000009,
0602     0x246a, 0xffffffff, 0x00080007,
0603     0x246b, 0xffffffff, 0x000b000a,
0604     0x246c, 0xffffffff, 0x00050004,
0605     0x246d, 0xffffffff, 0x00070006,
0606     0x246e, 0xffffffff, 0x0008000b,
0607     0x246f, 0xffffffff, 0x000a0009,
0608     0x2470, 0xffffffff, 0x000d000c,
0609     0x2480, 0xffffffff, 0x00090008,
0610     0x2481, 0xffffffff, 0x000b000a,
0611     0x2482, 0xffffffff, 0x000c000f,
0612     0x2483, 0xffffffff, 0x000e000d,
0613     0x2484, 0xffffffff, 0x00110010,
0614     0x2485, 0xffffffff, 0x000a0009,
0615     0x2486, 0xffffffff, 0x000c000b,
0616     0x2487, 0xffffffff, 0x0000000f,
0617     0x2488, 0xffffffff, 0x000e000d,
0618     0x2489, 0xffffffff, 0x00110010,
0619     0x248a, 0xffffffff, 0x000b000a,
0620     0x248b, 0xffffffff, 0x000d000c,
0621     0x248c, 0xffffffff, 0x00000010,
0622     0x248d, 0xffffffff, 0x000f000e,
0623     0x248e, 0xffffffff, 0x00120011,
0624     0x248f, 0xffffffff, 0x000c000b,
0625     0x2490, 0xffffffff, 0x000e000d,
0626     0x2491, 0xffffffff, 0x00000011,
0627     0x2492, 0xffffffff, 0x0010000f,
0628     0x2493, 0xffffffff, 0x00130012,
0629     0x2494, 0xffffffff, 0x000d000c,
0630     0x2495, 0xffffffff, 0x000f000e,
0631     0x2496, 0xffffffff, 0x00100013,
0632     0x2497, 0xffffffff, 0x00120011,
0633     0x2498, 0xffffffff, 0x00150014,
0634     mmCGTS_SM_CTRL_REG, 0xffffffff, 0x96940200,
0635     mmCP_RB_WPTR_POLL_CNTL, 0xffffffff, 0x00900100,
0636     mmRLC_GCPM_GENERAL_3, 0xffffffff, 0x00000080,
0637     mmRLC_CGCG_CGLS_CTRL, 0xffffffff, 0x0020003f,
0638     0x000c, 0xffffffff, 0x0000001c,
0639     0x000d, 0x000f0000, 0x000f0000,
0640     0x0583, 0xffffffff, 0x00000100,
0641     mmXDMA_CLOCK_GATING_CNTL, 0xffffffff, 0x00000100,
0642     mmXDMA_MEM_POWER_CNTL, 0x00000101, 0x00000000,
0643     mmMC_MEM_POWER_LS, 0xffffffff, 0x00000104,
0644     mmCGTT_DRM_CLK_CTRL0, 0xff000fff, 0x00000100,
0645     0x157a, 0x00000001, 0x00000001,
0646     mmHDP_MEM_POWER_LS, 0x00000001, 0x00000001,
0647     mmHDP_XDP_CGTT_BLK_CTRL, 0xc0000fff, 0x00000104,
0648     mmCP_MEM_SLP_CNTL, 0x00000001, 0x00000001,
0649     0x3430, 0xfffffff0, 0x00000100,
0650     0x3630, 0xfffffff0, 0x00000100,
0651 };
0652
0653 static const u32 verde_mgcg_cgcg_init[] =
0654 {
0655     mmRLC_CGTT_MGCG_OVERRIDE, 0xffffffff, 0xfffffffc,
0656     mmGRBM_GFX_INDEX, 0xffffffff, 0xe0000000,
0657     mmCB_CGTT_SCLK_CTRL, 0xffffffff, 0x00000100,
0658     mmCGTT_BCI_CLK_CTRL, 0xffffffff, 0x00000100,
0659     mmCGTT_CP_CLK_CTRL, 0xffffffff, 0x00000100,
0660     mmCGTT_GDS_CLK_CTRL, 0xffffffff, 0x00000100,
0661     mmCGTT_IA_CLK_CTRL, 0xffffffff, 0x06000100,
0662     mmCGTT_PA_CLK_CTRL, 0xffffffff, 0x00000100,
0663     mmCGTT_PC_CLK_CTRL, 0xffffffff, 0x00000100,
0664     mmCGTT_RLC_CLK_CTRL, 0xffffffff, 0x00000100,
0665     mmCGTT_SC_CLK_CTRL, 0xffffffff, 0x00000100,
0666     mmCGTT_SPI_CLK_CTRL, 0xffffffff, 0x00000100,
0667     mmCGTT_SQ_CLK_CTRL, 0xffffffff, 0x00000100,
0668     mmCGTT_SQG_CLK_CTRL, 0xffffffff, 0x00000100,
0669     mmCGTT_SX_CLK_CTRL0, 0xffffffff, 0x00000100,
0670     mmCGTT_SX_CLK_CTRL1, 0xffffffff, 0x00000100,
0671     mmCGTT_SX_CLK_CTRL2, 0xffffffff, 0x00000100,
0672     mmCGTT_SX_CLK_CTRL3, 0xffffffff, 0x00000100,
0673     mmCGTT_TCI_CLK_CTRL, 0xffffffff, 0x00000100,
0674     mmCGTT_TCP_CLK_CTRL, 0xffffffff, 0x00000100,
0675     mmCGTT_VGT_CLK_CTRL, 0xffffffff, 0x06000100,
0676     mmDB_CGTT_CLK_CTRL_0, 0xffffffff, 0x00000100,
0677     mmTA_CGTT_CTRL, 0xffffffff, 0x00000100,
0678     mmTCA_CGTT_SCLK_CTRL, 0xffffffff, 0x00000100,
0679     mmTCC_CGTT_SCLK_CTRL, 0xffffffff, 0x00000100,
0680     mmTD_CGTT_CTRL, 0xffffffff, 0x00000100,
0681     mmGRBM_GFX_INDEX, 0xffffffff, 0xe0000000,
0682     0x2458, 0xffffffff, 0x00010000,
0683     0x2459, 0xffffffff, 0x00030002,
0684     0x245a, 0xffffffff, 0x00040007,
0685     0x245b, 0xffffffff, 0x00060005,
0686     0x245c, 0xffffffff, 0x00090008,
0687     0x245d, 0xffffffff, 0x00020001,
0688     0x245e, 0xffffffff, 0x00040003,
0689     0x245f, 0xffffffff, 0x00000007,
0690     0x2460, 0xffffffff, 0x00060005,
0691     0x2461, 0xffffffff, 0x00090008,
0692     0x2462, 0xffffffff, 0x00030002,
0693     0x2463, 0xffffffff, 0x00050004,
0694     0x2464, 0xffffffff, 0x00000008,
0695     0x2465, 0xffffffff, 0x00070006,
0696     0x2466, 0xffffffff, 0x000a0009,
0697     0x2467, 0xffffffff, 0x00040003,
0698     0x2468, 0xffffffff, 0x00060005,
0699     0x2469, 0xffffffff, 0x00000009,
0700     0x246a, 0xffffffff, 0x00080007,
0701     0x246b, 0xffffffff, 0x000b000a,
0702     0x246c, 0xffffffff, 0x00050004,
0703     0x246d, 0xffffffff, 0x00070006,
0704     0x246e, 0xffffffff, 0x0008000b,
0705     0x246f, 0xffffffff, 0x000a0009,
0706     0x2470, 0xffffffff, 0x000d000c,
0707     0x2480, 0xffffffff, 0x00090008,
0708     0x2481, 0xffffffff, 0x000b000a,
0709     0x2482, 0xffffffff, 0x000c000f,
0710     0x2483, 0xffffffff, 0x000e000d,
0711     0x2484, 0xffffffff, 0x00110010,
0712     0x2485, 0xffffffff, 0x000a0009,
0713     0x2486, 0xffffffff, 0x000c000b,
0714     0x2487, 0xffffffff, 0x0000000f,
0715     0x2488, 0xffffffff, 0x000e000d,
0716     0x2489, 0xffffffff, 0x00110010,
0717     0x248a, 0xffffffff, 0x000b000a,
0718     0x248b, 0xffffffff, 0x000d000c,
0719     0x248c, 0xffffffff, 0x00000010,
0720     0x248d, 0xffffffff, 0x000f000e,
0721     0x248e, 0xffffffff, 0x00120011,
0722     0x248f, 0xffffffff, 0x000c000b,
0723     0x2490, 0xffffffff, 0x000e000d,
0724     0x2491, 0xffffffff, 0x00000011,
0725     0x2492, 0xffffffff, 0x0010000f,
0726     0x2493, 0xffffffff, 0x00130012,
0727     0x2494, 0xffffffff, 0x000d000c,
0728     0x2495, 0xffffffff, 0x000f000e,
0729     0x2496, 0xffffffff, 0x00100013,
0730     0x2497, 0xffffffff, 0x00120011,
0731     0x2498, 0xffffffff, 0x00150014,
0732     mmCGTS_SM_CTRL_REG, 0xffffffff, 0x96940200,
0733     mmCP_RB_WPTR_POLL_CNTL, 0xffffffff, 0x00900100,
0734     mmRLC_GCPM_GENERAL_3, 0xffffffff, 0x00000080,
0735     mmRLC_CGCG_CGLS_CTRL, 0xffffffff, 0x0020003f,
0736     0x000c, 0xffffffff, 0x0000001c,
0737     0x000d, 0x000f0000, 0x000f0000,
0738     0x0583, 0xffffffff, 0x00000100,
0739     mmXDMA_CLOCK_GATING_CNTL, 0xffffffff, 0x00000100,
0740     mmXDMA_MEM_POWER_CNTL, 0x00000101, 0x00000000,
0741     mmMC_MEM_POWER_LS, 0xffffffff, 0x00000104,
0742     mmMC_CITF_MISC_WR_CG, 0x000c0000, 0x000c0000,
0743     mmMC_CITF_MISC_RD_CG, 0x000c0000, 0x000c0000,
0744     mmCGTT_DRM_CLK_CTRL0, 0xff000fff, 0x00000100,
0745     0x157a, 0x00000001, 0x00000001,
0746     mmHDP_MEM_POWER_LS, 0x00000001, 0x00000001,
0747     mmHDP_XDP_CGTT_BLK_CTRL, 0xc0000fff, 0x00000104,
0748     mmCP_MEM_SLP_CNTL, 0x00000001, 0x00000001,
0749     0x3430, 0xfffffff0, 0x00000100,
0750     0x3630, 0xfffffff0, 0x00000100,
0751 };
0752
0753 static const u32 oland_mgcg_cgcg_init[] =
0754 {
0755     mmRLC_CGTT_MGCG_OVERRIDE, 0xffffffff, 0xfffffffc,
0756     mmGRBM_GFX_INDEX, 0xffffffff, 0xe0000000,
0757     mmCB_CGTT_SCLK_CTRL, 0xffffffff, 0x00000100,
0758     mmCGTT_BCI_CLK_CTRL, 0xffffffff, 0x00000100,
0759     mmCGTT_CP_CLK_CTRL, 0xffffffff, 0x00000100,
0760     mmCGTT_GDS_CLK_CTRL, 0xffffffff, 0x00000100,
0761     mmCGTT_IA_CLK_CTRL, 0xffffffff, 0x06000100,
0762     mmCGTT_PA_CLK_CTRL, 0xffffffff, 0x00000100,
0763     mmCGTT_PC_CLK_CTRL, 0xffffffff, 0x00000100,
0764     mmCGTT_RLC_CLK_CTRL, 0xffffffff, 0x00000100,
0765     mmCGTT_SC_CLK_CTRL, 0xffffffff, 0x00000100,
0766     mmCGTT_SPI_CLK_CTRL, 0xffffffff, 0x00000100,
0767     mmCGTT_SQ_CLK_CTRL, 0xffffffff, 0x00000100,
0768     mmCGTT_SQG_CLK_CTRL, 0xffffffff, 0x00000100,
0769     mmCGTT_SX_CLK_CTRL0, 0xffffffff, 0x00000100,
0770     mmCGTT_SX_CLK_CTRL1, 0xffffffff, 0x00000100,
0771     mmCGTT_SX_CLK_CTRL2, 0xffffffff, 0x00000100,
0772     mmCGTT_SX_CLK_CTRL3, 0xffffffff, 0x00000100,
0773     mmCGTT_TCI_CLK_CTRL, 0xffffffff, 0x00000100,
0774     mmCGTT_TCP_CLK_CTRL, 0xffffffff, 0x00000100,
0775     mmCGTT_VGT_CLK_CTRL, 0xffffffff, 0x06000100,
0776     mmDB_CGTT_CLK_CTRL_0, 0xffffffff, 0x00000100,
0777     mmTA_CGTT_CTRL, 0xffffffff, 0x00000100,
0778     mmTCA_CGTT_SCLK_CTRL, 0xffffffff, 0x00000100,
0779     mmTCC_CGTT_SCLK_CTRL, 0xffffffff, 0x00000100,
0780     mmTD_CGTT_CTRL, 0xffffffff, 0x00000100,
0781     mmGRBM_GFX_INDEX, 0xffffffff, 0xe0000000,
0782     0x2458, 0xffffffff, 0x00010000,
0783     0x2459, 0xffffffff, 0x00030002,
0784     0x245a, 0xffffffff, 0x00040007,
0785     0x245b, 0xffffffff, 0x00060005,
0786     0x245c, 0xffffffff, 0x00090008,
0787     0x245d, 0xffffffff, 0x00020001,
0788     0x245e, 0xffffffff, 0x00040003,
0789     0x245f, 0xffffffff, 0x00000007,
0790     0x2460, 0xffffffff, 0x00060005,
0791     0x2461, 0xffffffff, 0x00090008,
0792     0x2462, 0xffffffff, 0x00030002,
0793     0x2463, 0xffffffff, 0x00050004,
0794     0x2464, 0xffffffff, 0x00000008,
0795     0x2465, 0xffffffff, 0x00070006,
0796     0x2466, 0xffffffff, 0x000a0009,
0797     0x2467, 0xffffffff, 0x00040003,
0798     0x2468, 0xffffffff, 0x00060005,
0799     0x2469, 0xffffffff, 0x00000009,
0800     0x246a, 0xffffffff, 0x00080007,
0801     0x246b, 0xffffffff, 0x000b000a,
0802     0x246c, 0xffffffff, 0x00050004,
0803     0x246d, 0xffffffff, 0x00070006,
0804     0x246e, 0xffffffff, 0x0008000b,
0805     0x246f, 0xffffffff, 0x000a0009,
0806     0x2470, 0xffffffff, 0x000d000c,
0807     0x2471, 0xffffffff, 0x00060005,
0808     0x2472, 0xffffffff, 0x00080007,
0809     0x2473, 0xffffffff, 0x0000000b,
0810     0x2474, 0xffffffff, 0x000a0009,
0811     0x2475, 0xffffffff, 0x000d000c,
0812     mmCGTS_SM_CTRL_REG, 0xffffffff, 0x96940200,
0813     mmCP_RB_WPTR_POLL_CNTL, 0xffffffff, 0x00900100,
0814     mmRLC_GCPM_GENERAL_3, 0xffffffff, 0x00000080,
0815     mmRLC_CGCG_CGLS_CTRL, 0xffffffff, 0x0020003f,
0816     0x000c, 0xffffffff, 0x0000001c,
0817     0x000d, 0x000f0000, 0x000f0000,
0818     0x0583, 0xffffffff, 0x00000100,
0819     mmXDMA_CLOCK_GATING_CNTL, 0xffffffff, 0x00000100,
0820     mmXDMA_MEM_POWER_CNTL, 0x00000101, 0x00000000,
0821     mmMC_MEM_POWER_LS, 0xffffffff, 0x00000104,
0822     mmMC_CITF_MISC_WR_CG, 0x000c0000, 0x000c0000,
0823     mmMC_CITF_MISC_RD_CG, 0x000c0000, 0x000c0000,
0824     mmCGTT_DRM_CLK_CTRL0, 0xff000fff, 0x00000100,
0825     0x157a, 0x00000001, 0x00000001,
0826     mmHDP_MEM_POWER_LS, 0x00000001, 0x00000001,
0827     mmHDP_XDP_CGTT_BLK_CTRL, 0xc0000fff, 0x00000104,
0828     mmCP_MEM_SLP_CNTL, 0x00000001, 0x00000001,
0829     0x3430, 0xfffffff0, 0x00000100,
0830     0x3630, 0xfffffff0, 0x00000100,
0831 };
0832
0833 static const u32 hainan_mgcg_cgcg_init[] =
0834 {
0835     mmRLC_CGTT_MGCG_OVERRIDE, 0xffffffff, 0xfffffffc,
0836     mmGRBM_GFX_INDEX, 0xffffffff, 0xe0000000,
0837     mmCB_CGTT_SCLK_CTRL, 0xffffffff, 0x00000100,
0838     mmCGTT_BCI_CLK_CTRL, 0xffffffff, 0x00000100,
0839     mmCGTT_CP_CLK_CTRL, 0xffffffff, 0x00000100,
0840     mmCGTT_GDS_CLK_CTRL, 0xffffffff, 0x00000100,
0841     mmCGTT_IA_CLK_CTRL, 0xffffffff, 0x06000100,
0842     mmCGTT_PA_CLK_CTRL, 0xffffffff, 0x00000100,
0843     mmCGTT_PC_CLK_CTRL, 0xffffffff, 0x00000100,
0844     mmCGTT_RLC_CLK_CTRL, 0xffffffff, 0x00000100,
0845     mmCGTT_SC_CLK_CTRL, 0xffffffff, 0x00000100,
0846     mmCGTT_SPI_CLK_CTRL, 0xffffffff, 0x00000100,
0847     mmCGTT_SQ_CLK_CTRL, 0xffffffff, 0x00000100,
0848     mmCGTT_SQG_CLK_CTRL, 0xffffffff, 0x00000100,
0849     mmCGTT_SX_CLK_CTRL0, 0xffffffff, 0x00000100,
0850     mmCGTT_SX_CLK_CTRL1, 0xffffffff, 0x00000100,
0851     mmCGTT_SX_CLK_CTRL2, 0xffffffff, 0x00000100,
0852     mmCGTT_SX_CLK_CTRL3, 0xffffffff, 0x00000100,
0853     mmCGTT_TCI_CLK_CTRL, 0xffffffff, 0x00000100,
0854     mmCGTT_TCP_CLK_CTRL, 0xffffffff, 0x00000100,
0855     mmCGTT_VGT_CLK_CTRL, 0xffffffff, 0x06000100,
0856     mmDB_CGTT_CLK_CTRL_0, 0xffffffff, 0x00000100,
0857     mmTA_CGTT_CTRL, 0xffffffff, 0x00000100,
0858     mmTCA_CGTT_SCLK_CTRL, 0xffffffff, 0x00000100,
0859     mmTCC_CGTT_SCLK_CTRL, 0xffffffff, 0x00000100,
0860     mmTD_CGTT_CTRL, 0xffffffff, 0x00000100,
0861     mmGRBM_GFX_INDEX, 0xffffffff, 0xe0000000,
0862     0x2458, 0xffffffff, 0x00010000,
0863     0x2459, 0xffffffff, 0x00030002,
0864     0x245a, 0xffffffff, 0x00040007,
0865     0x245b, 0xffffffff, 0x00060005,
0866     0x245c, 0xffffffff, 0x00090008,
0867     0x245d, 0xffffffff, 0x00020001,
0868     0x245e, 0xffffffff, 0x00040003,
0869     0x245f, 0xffffffff, 0x00000007,
0870     0x2460, 0xffffffff, 0x00060005,
0871     0x2461, 0xffffffff, 0x00090008,
0872     0x2462, 0xffffffff, 0x00030002,
0873     0x2463, 0xffffffff, 0x00050004,
0874     0x2464, 0xffffffff, 0x00000008,
0875     0x2465, 0xffffffff, 0x00070006,
0876     0x2466, 0xffffffff, 0x000a0009,
0877     0x2467, 0xffffffff, 0x00040003,
0878     0x2468, 0xffffffff, 0x00060005,
0879     0x2469, 0xffffffff, 0x00000009,
0880     0x246a, 0xffffffff, 0x00080007,
0881     0x246b, 0xffffffff, 0x000b000a,
0882     0x246c, 0xffffffff, 0x00050004,
0883     0x246d, 0xffffffff, 0x00070006,
0884     0x246e, 0xffffffff, 0x0008000b,
0885     0x246f, 0xffffffff, 0x000a0009,
0886     0x2470, 0xffffffff, 0x000d000c,
0887     0x2471, 0xffffffff, 0x00060005,
0888     0x2472, 0xffffffff, 0x00080007,
0889     0x2473, 0xffffffff, 0x0000000b,
0890     0x2474, 0xffffffff, 0x000a0009,
0891     0x2475, 0xffffffff, 0x000d000c,
0892     mmCGTS_SM_CTRL_REG, 0xffffffff, 0x96940200,
0893     mmCP_RB_WPTR_POLL_CNTL, 0xffffffff, 0x00900100,
0894     mmRLC_GCPM_GENERAL_3, 0xffffffff, 0x00000080,
0895     mmRLC_CGCG_CGLS_CTRL, 0xffffffff, 0x0020003f,
0896     0x000c, 0xffffffff, 0x0000001c,
0897     0x000d, 0x000f0000, 0x000f0000,
0898     0x0583, 0xffffffff, 0x00000100,
0899     0x0409, 0xffffffff, 0x00000100,
0900     mmMC_MEM_POWER_LS, 0xffffffff, 0x00000104,
0901     mmMC_CITF_MISC_WR_CG, 0x000c0000, 0x000c0000,
0902     mmMC_CITF_MISC_RD_CG, 0x000c0000, 0x000c0000,
0903     mmHDP_MEM_POWER_LS, 0x00000001, 0x00000001,
0904     mmHDP_XDP_CGTT_BLK_CTRL, 0xc0000fff, 0x00000104,
0905     mmCP_MEM_SLP_CNTL, 0x00000001, 0x00000001,
0906     0x3430, 0xfffffff0, 0x00000100,
0907     0x3630, 0xfffffff0, 0x00000100,
0908 };
0909
0910 /* XXX: update when we support VCE */
0911 #if 0
0912 /* tahiti, pitcarin, verde */
0913 static const struct amdgpu_video_codec_info tahiti_video_codecs_encode_array[] =
0914 {
0915     {
0916         .codec_type = AMDGPU_INFO_VIDEO_CAPS_CODEC_IDX_MPEG4_AVC,
0917         .max_width = 2048,
0918         .max_height = 1152,
0919         .max_pixels_per_frame = 2048 * 1152,
0920         .max_level = 0,
0921     },
0922 };
0923
0924 static const struct amdgpu_video_codecs tahiti_video_codecs_encode =
0925 {
0926     .codec_count = ARRAY_SIZE(tahiti_video_codecs_encode_array),
0927     .codec_array = tahiti_video_codecs_encode_array,
0928 };
0929 #else
0930 static const struct amdgpu_video_codecs tahiti_video_codecs_encode =
0931 {
0932     .codec_count = 0,
0933     .codec_array = NULL,
0934 };
0935 #endif
0936 /* oland and hainan don't support encode */
0937 static const struct amdgpu_video_codecs hainan_video_codecs_encode =
0938 {
0939     .codec_count = 0,
0940     .codec_array = NULL,
0941 };
0942
0943 /* tahiti, pitcarin, verde, oland */
0944 static const struct amdgpu_video_codec_info tahiti_video_codecs_decode_array[] =
0945 {
0946     {
0947         .codec_type = AMDGPU_INFO_VIDEO_CAPS_CODEC_IDX_MPEG2,
0948         .max_width = 2048,
0949         .max_height = 1152,
0950         .max_pixels_per_frame = 2048 * 1152,
0951         .max_level = 3,
0952     },
0953     {
0954         .codec_type = AMDGPU_INFO_VIDEO_CAPS_CODEC_IDX_MPEG4,
0955         .max_width = 2048,
0956         .max_height = 1152,
0957         .max_pixels_per_frame = 2048 * 1152,
0958         .max_level = 5,
0959     },
0960     {
0961         .codec_type = AMDGPU_INFO_VIDEO_CAPS_CODEC_IDX_MPEG4_AVC,
0962         .max_width = 2048,
0963         .max_height = 1152,
0964         .max_pixels_per_frame = 2048 * 1152,
0965         .max_level = 41,
0966     },
0967     {
0968         .codec_type = AMDGPU_INFO_VIDEO_CAPS_CODEC_IDX_VC1,
0969         .max_width = 2048,
0970         .max_height = 1152,
0971         .max_pixels_per_frame = 2048 * 1152,
0972         .max_level = 4,
0973     },
0974 };
0975
0976 static const struct amdgpu_video_codecs tahiti_video_codecs_decode =
0977 {
0978     .codec_count = ARRAY_SIZE(tahiti_video_codecs_decode_array),
0979     .codec_array = tahiti_video_codecs_decode_array,
0980 };
0981
0982 /* hainan doesn't support decode */
0983 static const struct amdgpu_video_codecs hainan_video_codecs_decode =
0984 {
0985     .codec_count = 0,
0986     .codec_array = NULL,
0987 };
0988
0989 static int si_query_video_codecs(struct amdgpu_device *adev, bool encode,
0990                  const struct amdgpu_video_codecs **codecs)
0991 {
0992     switch (adev->asic_type) {
0993     case CHIP_VERDE:
0994     case CHIP_TAHITI:
0995     case CHIP_PITCAIRN:
0996         if (encode)
0997             *codecs = &tahiti_video_codecs_encode;
0998         else
0999             *codecs = &tahiti_video_codecs_decode;
1000         return 0;
1001     case CHIP_OLAND:
1002         if (encode)
1003             *codecs = &hainan_video_codecs_encode;
1004         else
1005             *codecs = &tahiti_video_codecs_decode;
1006         return 0;
1007     case CHIP_HAINAN:
1008         if (encode)
1009             *codecs = &hainan_video_codecs_encode;
1010         else
1011             *codecs = &hainan_video_codecs_decode;
1012         return 0;
1013     default:
1014         return -EINVAL;
1015     }
1016 }
1017
1018 static u32 si_pcie_rreg(struct amdgpu_device *adev, u32 reg)
1019 {
1020     unsigned long flags;
1021     u32 r;
1022
1023     spin_lock_irqsave(&adev->pcie_idx_lock, flags);
1024     WREG32(AMDGPU_PCIE_INDEX, reg);
1025     (void)RREG32(AMDGPU_PCIE_INDEX);
1026     r = RREG32(AMDGPU_PCIE_DATA);
1027     spin_unlock_irqrestore(&adev->pcie_idx_lock, flags);
1028     return r;
1029 }
1030
1031 static void si_pcie_wreg(struct amdgpu_device *adev, u32 reg, u32 v)
1032 {
1033     unsigned long flags;
1034
1035     spin_lock_irqsave(&adev->pcie_idx_lock, flags);
1036     WREG32(AMDGPU_PCIE_INDEX, reg);
1037     (void)RREG32(AMDGPU_PCIE_INDEX);
1038     WREG32(AMDGPU_PCIE_DATA, v);
1039     (void)RREG32(AMDGPU_PCIE_DATA);
1040     spin_unlock_irqrestore(&adev->pcie_idx_lock, flags);
1041 }
1042
1043 static u32 si_pciep_rreg(struct amdgpu_device *adev, u32 reg)
1044 {
1045     unsigned long flags;
1046     u32 r;
1047
1048     spin_lock_irqsave(&adev->pcie_idx_lock, flags);
1049     WREG32(PCIE_PORT_INDEX, ((reg) & 0xff));
1050     (void)RREG32(PCIE_PORT_INDEX);
1051     r = RREG32(PCIE_PORT_DATA);
1052     spin_unlock_irqrestore(&adev->pcie_idx_lock, flags);
1053     return r;
1054 }
1055
1056 static void si_pciep_wreg(struct amdgpu_device *adev, u32 reg, u32 v)
1057 {
1058     unsigned long flags;
1059
1060     spin_lock_irqsave(&adev->pcie_idx_lock, flags);
1061     WREG32(PCIE_PORT_INDEX, ((reg) & 0xff));
1062     (void)RREG32(PCIE_PORT_INDEX);
1063     WREG32(PCIE_PORT_DATA, (v));
1064     (void)RREG32(PCIE_PORT_DATA);
1065     spin_unlock_irqrestore(&adev->pcie_idx_lock, flags);
1066 }
1067
1068 static u32 si_smc_rreg(struct amdgpu_device *adev, u32 reg)
1069 {
1070     unsigned long flags;
1071     u32 r;
1072
1073     spin_lock_irqsave(&adev->smc_idx_lock, flags);
1074     WREG32(SMC_IND_INDEX_0, (reg));
1075     r = RREG32(SMC_IND_DATA_0);
1076     spin_unlock_irqrestore(&adev->smc_idx_lock, flags);
1077     return r;
1078 }
1079
1080 static void si_smc_wreg(struct amdgpu_device *adev, u32 reg, u32 v)
1081 {
1082     unsigned long flags;
1083
1084     spin_lock_irqsave(&adev->smc_idx_lock, flags);
1085     WREG32(SMC_IND_INDEX_0, (reg));
1086     WREG32(SMC_IND_DATA_0, (v));
1087     spin_unlock_irqrestore(&adev->smc_idx_lock, flags);
1088 }
1089
1090 static u32 si_uvd_ctx_rreg(struct amdgpu_device *adev, u32 reg)
1091 {
1092     unsigned long flags;
1093     u32 r;
1094
1095     spin_lock_irqsave(&adev->uvd_ctx_idx_lock, flags);
1096     WREG32(mmUVD_CTX_INDEX, ((reg) & 0x1ff));
1097     r = RREG32(mmUVD_CTX_DATA);
1098     spin_unlock_irqrestore(&adev->uvd_ctx_idx_lock, flags);
1099     return r;
1100 }
1101
1102 static void si_uvd_ctx_wreg(struct amdgpu_device *adev, u32 reg, u32 v)
1103 {
1104     unsigned long flags;
1105
1106     spin_lock_irqsave(&adev->uvd_ctx_idx_lock, flags);
1107     WREG32(mmUVD_CTX_INDEX, ((reg) & 0x1ff));
1108     WREG32(mmUVD_CTX_DATA, (v));
1109     spin_unlock_irqrestore(&adev->uvd_ctx_idx_lock, flags);
1110 }
1111
1112 static struct amdgpu_allowed_register_entry si_allowed_read_registers[] = {
1113     {GRBM_STATUS},
1114     {mmGRBM_STATUS2},
1115     {mmGRBM_STATUS_SE0},
1116     {mmGRBM_STATUS_SE1},
1117     {mmSRBM_STATUS},
1118     {mmSRBM_STATUS2},
1119     {DMA_STATUS_REG + DMA0_REGISTER_OFFSET},
1120     {DMA_STATUS_REG + DMA1_REGISTER_OFFSET},
1121     {mmCP_STAT},
1122     {mmCP_STALLED_STAT1},
1123     {mmCP_STALLED_STAT2},
1124     {mmCP_STALLED_STAT3},
1125     {GB_ADDR_CONFIG},
1126     {MC_ARB_RAMCFG},
1127     {GB_TILE_MODE0},
1128     {GB_TILE_MODE1},
1129     {GB_TILE_MODE2},
1130     {GB_TILE_MODE3},
1131     {GB_TILE_MODE4},
1132     {GB_TILE_MODE5},
1133     {GB_TILE_MODE6},
1134     {GB_TILE_MODE7},
1135     {GB_TILE_MODE8},
1136     {GB_TILE_MODE9},
1137     {GB_TILE_MODE10},
1138     {GB_TILE_MODE11},
1139     {GB_TILE_MODE12},
1140     {GB_TILE_MODE13},
1141     {GB_TILE_MODE14},
1142     {GB_TILE_MODE15},
1143     {GB_TILE_MODE16},
1144     {GB_TILE_MODE17},
1145     {GB_TILE_MODE18},
1146     {GB_TILE_MODE19},
1147     {GB_TILE_MODE20},
1148     {GB_TILE_MODE21},
1149     {GB_TILE_MODE22},
1150     {GB_TILE_MODE23},
1151     {GB_TILE_MODE24},
1152     {GB_TILE_MODE25},
1153     {GB_TILE_MODE26},
1154     {GB_TILE_MODE27},
1155     {GB_TILE_MODE28},
1156     {GB_TILE_MODE29},
1157     {GB_TILE_MODE30},
1158     {GB_TILE_MODE31},
1159     {CC_RB_BACKEND_DISABLE, true},
1160     {GC_USER_RB_BACKEND_DISABLE, true},
1161     {PA_SC_RASTER_CONFIG, true},
1162 };
1163
1164 static uint32_t si_get_register_value(struct amdgpu_device *adev,
1165                       bool indexed, u32 se_num,
1166                       u32 sh_num, u32 reg_offset)
1167 {
1168     if (indexed) {
1169         uint32_t val;
1170         unsigned se_idx = (se_num == 0xffffffff) ? 0 : se_num;
1171         unsigned sh_idx = (sh_num == 0xffffffff) ? 0 : sh_num;
1172
1173         switch (reg_offset) {
1174         case mmCC_RB_BACKEND_DISABLE:
1175             return adev->gfx.config.rb_config[se_idx][sh_idx].rb_backend_disable;
1176         case mmGC_USER_RB_BACKEND_DISABLE:
1177             return adev->gfx.config.rb_config[se_idx][sh_idx].user_rb_backend_disable;
1178         case mmPA_SC_RASTER_CONFIG:
1179             return adev->gfx.config.rb_config[se_idx][sh_idx].raster_config;
1180         }
1181
1182         mutex_lock(&adev->grbm_idx_mutex);
1183         if (se_num != 0xffffffff || sh_num != 0xffffffff)
1184             amdgpu_gfx_select_se_sh(adev, se_num, sh_num, 0xffffffff);
1185
1186         val = RREG32(reg_offset);
1187
1188         if (se_num != 0xffffffff || sh_num != 0xffffffff)
1189             amdgpu_gfx_select_se_sh(adev, 0xffffffff, 0xffffffff, 0xffffffff);
1190         mutex_unlock(&adev->grbm_idx_mutex);
1191         return val;
1192     } else {
1193         unsigned idx;
1194
1195         switch (reg_offset) {
1196         case mmGB_ADDR_CONFIG:
1197             return adev->gfx.config.gb_addr_config;
1198         case mmMC_ARB_RAMCFG:
1199             return adev->gfx.config.mc_arb_ramcfg;
1200         case mmGB_TILE_MODE0:
1201         case mmGB_TILE_MODE1:
1202         case mmGB_TILE_MODE2:
1203         case mmGB_TILE_MODE3:
1204         case mmGB_TILE_MODE4:
1205         case mmGB_TILE_MODE5:
1206         case mmGB_TILE_MODE6:
1207         case mmGB_TILE_MODE7:
1208         case mmGB_TILE_MODE8:
1209         case mmGB_TILE_MODE9:
1210         case mmGB_TILE_MODE10:
1211         case mmGB_TILE_MODE11:
1212         case mmGB_TILE_MODE12:
1213         case mmGB_TILE_MODE13:
1214         case mmGB_TILE_MODE14:
1215         case mmGB_TILE_MODE15:
1216         case mmGB_TILE_MODE16:
1217         case mmGB_TILE_MODE17:
1218         case mmGB_TILE_MODE18:
1219         case mmGB_TILE_MODE19:
1220         case mmGB_TILE_MODE20:
1221         case mmGB_TILE_MODE21:
1222         case mmGB_TILE_MODE22:
1223         case mmGB_TILE_MODE23:
1224         case mmGB_TILE_MODE24:
1225         case mmGB_TILE_MODE25:
1226         case mmGB_TILE_MODE26:
1227         case mmGB_TILE_MODE27:
1228         case mmGB_TILE_MODE28:
1229         case mmGB_TILE_MODE29:
1230         case mmGB_TILE_MODE30:
1231         case mmGB_TILE_MODE31:
1232             idx = (reg_offset - mmGB_TILE_MODE0);
1233             return adev->gfx.config.tile_mode_array[idx];
1234         default:
1235             return RREG32(reg_offset);
1236         }
1237     }
1238 }
1239 static int si_read_register(struct amdgpu_device *adev, u32 se_num,
1240                  u32 sh_num, u32 reg_offset, u32 *value)
1241 {
1242     uint32_t i;
1243
1244     *value = 0;
1245     for (i = 0; i < ARRAY_SIZE(si_allowed_read_registers); i++) {
1246         bool indexed = si_allowed_read_registers[i].grbm_indexed;
1247
1248         if (reg_offset != si_allowed_read_registers[i].reg_offset)
1249             continue;
1250
1251         *value = si_get_register_value(adev, indexed, se_num, sh_num,
1252                            reg_offset);
1253         return 0;
1254     }
1255     return -EINVAL;
1256 }
1257
1258 static bool si_read_disabled_bios(struct amdgpu_device *adev)
1259 {
1260     u32 bus_cntl;
1261     u32 d1vga_control = 0;
1262     u32 d2vga_control = 0;
1263     u32 vga_render_control = 0;
1264     u32 rom_cntl;
1265     bool r;
1266
1267     bus_cntl = RREG32(R600_BUS_CNTL);
1268     if (adev->mode_info.num_crtc) {
1269         d1vga_control = RREG32(AVIVO_D1VGA_CONTROL);
1270         d2vga_control = RREG32(AVIVO_D2VGA_CONTROL);
1271         vga_render_control = RREG32(VGA_RENDER_CONTROL);
1272     }
1273     rom_cntl = RREG32(R600_ROM_CNTL);
1274
1275     /* enable the rom */
1276     WREG32(R600_BUS_CNTL, (bus_cntl & ~R600_BIOS_ROM_DIS));
1277     if (adev->mode_info.num_crtc) {
1278         /* Disable VGA mode */
1279         WREG32(AVIVO_D1VGA_CONTROL,
1280                (d1vga_control & ~(AVIVO_DVGA_CONTROL_MODE_ENABLE |
1281                       AVIVO_DVGA_CONTROL_TIMING_SELECT)));
1282         WREG32(AVIVO_D2VGA_CONTROL,
1283                (d2vga_control & ~(AVIVO_DVGA_CONTROL_MODE_ENABLE |
1284                       AVIVO_DVGA_CONTROL_TIMING_SELECT)));
1285         WREG32(VGA_RENDER_CONTROL,
1286                (vga_render_control & C_000300_VGA_VSTATUS_CNTL));
1287     }
1288     WREG32(R600_ROM_CNTL, rom_cntl | R600_SCK_OVERWRITE);
1289
1290     r = amdgpu_read_bios(adev);
1291
1292     /* restore regs */
1293     WREG32(R600_BUS_CNTL, bus_cntl);
1294     if (adev->mode_info.num_crtc) {
1295         WREG32(AVIVO_D1VGA_CONTROL, d1vga_control);
1296         WREG32(AVIVO_D2VGA_CONTROL, d2vga_control);
1297         WREG32(VGA_RENDER_CONTROL, vga_render_control);
1298     }
1299     WREG32(R600_ROM_CNTL, rom_cntl);
1300     return r;
1301 }
1302
1303 #define mmROM_INDEX 0x2A
1304 #define mmROM_DATA  0x2B
1305
1306 static bool si_read_bios_from_rom(struct amdgpu_device *adev,
1307                   u8 *bios, u32 length_bytes)
1308 {
1309     u32 *dw_ptr;
1310     u32 i, length_dw;
1311
1312     if (bios == NULL)
1313         return false;
1314     if (length_bytes == 0)
1315         return false;
1316     /* APU vbios image is part of sbios image */
1317     if (adev->flags & AMD_IS_APU)
1318         return false;
1319
1320     dw_ptr = (u32 *)bios;
1321     length_dw = ALIGN(length_bytes, 4) / 4;
1322     /* set rom index to 0 */
1323     WREG32(mmROM_INDEX, 0);
1324     for (i = 0; i < length_dw; i++)
1325         dw_ptr[i] = RREG32(mmROM_DATA);
1326
1327     return true;
1328 }
1329
1330 static void si_set_clk_bypass_mode(struct amdgpu_device *adev)
1331 {
1332     u32 tmp, i;
1333
1334     tmp = RREG32(CG_SPLL_FUNC_CNTL);
1335     tmp |= SPLL_BYPASS_EN;
1336     WREG32(CG_SPLL_FUNC_CNTL, tmp);
1337
1338     tmp = RREG32(CG_SPLL_FUNC_CNTL_2);
1339     tmp |= SPLL_CTLREQ_CHG;
1340     WREG32(CG_SPLL_FUNC_CNTL_2, tmp);
1341
1342     for (i = 0; i < adev->usec_timeout; i++) {
1343         if (RREG32(SPLL_STATUS) & SPLL_CHG_STATUS)
1344             break;
1345         udelay(1);
1346     }
1347
1348     tmp = RREG32(CG_SPLL_FUNC_CNTL_2);
1349     tmp &= ~(SPLL_CTLREQ_CHG | SCLK_MUX_UPDATE);
1350     WREG32(CG_SPLL_FUNC_CNTL_2, tmp);
1351
1352     tmp = RREG32(MPLL_CNTL_MODE);
1353     tmp &= ~MPLL_MCLK_SEL;
1354     WREG32(MPLL_CNTL_MODE, tmp);
1355 }
1356
1357 static void si_spll_powerdown(struct amdgpu_device *adev)
1358 {
1359     u32 tmp;
1360
1361     tmp = RREG32(SPLL_CNTL_MODE);
1362     tmp |= SPLL_SW_DIR_CONTROL;
1363     WREG32(SPLL_CNTL_MODE, tmp);
1364
1365     tmp = RREG32(CG_SPLL_FUNC_CNTL);
1366     tmp |= SPLL_RESET;
1367     WREG32(CG_SPLL_FUNC_CNTL, tmp);
1368
1369     tmp = RREG32(CG_SPLL_FUNC_CNTL);
1370     tmp |= SPLL_SLEEP;
1371     WREG32(CG_SPLL_FUNC_CNTL, tmp);
1372
1373     tmp = RREG32(SPLL_CNTL_MODE);
1374     tmp &= ~SPLL_SW_DIR_CONTROL;
1375     WREG32(SPLL_CNTL_MODE, tmp);
1376 }
1377
1378 static int si_gpu_pci_config_reset(struct amdgpu_device *adev)
1379 {
1380     u32 i;
1381     int r = -EINVAL;
1382
1383     amdgpu_atombios_scratch_regs_engine_hung(adev, true);
1384
1385     /* set mclk/sclk to bypass */
1386     si_set_clk_bypass_mode(adev);
1387     /* powerdown spll */
1388     si_spll_powerdown(adev);
1389     /* disable BM */
1390     pci_clear_master(adev->pdev);
1391     /* reset */
1392     amdgpu_device_pci_config_reset(adev);
1393
1394     udelay(100);
1395
1396     /* wait for asic to come out of reset */
1397     for (i = 0; i < adev->usec_timeout; i++) {
1398         if (RREG32(mmCONFIG_MEMSIZE) != 0xffffffff) {
1399             /* enable BM */
1400             pci_set_master(adev->pdev);
1401             adev->has_hw_reset = true;
1402             r = 0;
1403             break;
1404         }
1405         udelay(1);
1406     }
1407     amdgpu_atombios_scratch_regs_engine_hung(adev, false);
1408
1409     return r;
1410 }
1411
1412 static bool si_asic_supports_baco(struct amdgpu_device *adev)
1413 {
1414     return false;
1415 }
1416
1417 static enum amd_reset_method
1418 si_asic_reset_method(struct amdgpu_device *adev)
1419 {
1420     if (amdgpu_reset_method == AMD_RESET_METHOD_PCI)
1421         return amdgpu_reset_method;
1422     else if (amdgpu_reset_method != AMD_RESET_METHOD_LEGACY &&
1423          amdgpu_reset_method != -1)
1424         dev_warn(adev->dev, "Specified reset method:%d isn't supported, using AUTO instead.\n",
1425              amdgpu_reset_method);
1426
1427     return AMD_RESET_METHOD_LEGACY;
1428 }
1429
1430 static int si_asic_reset(struct amdgpu_device *adev)
1431 {
1432     int r;
1433
1434     switch (si_asic_reset_method(adev)) {
1435     case AMD_RESET_METHOD_PCI:
1436         dev_info(adev->dev, "PCI reset\n");
1437         r = amdgpu_device_pci_reset(adev);
1438         break;
1439     default:
1440         dev_info(adev->dev, "PCI CONFIG reset\n");
1441         r = si_gpu_pci_config_reset(adev);
1442         break;
1443     }
1444
1445     return r;
1446 }
1447
1448 static u32 si_get_config_memsize(struct amdgpu_device *adev)
1449 {
1450     return RREG32(mmCONFIG_MEMSIZE);
1451 }
1452
1453 static void si_vga_set_state(struct amdgpu_device *adev, bool state)
1454 {
1455     uint32_t temp;
1456
1457     temp = RREG32(CONFIG_CNTL);
1458     if (!state) {
1459         temp &= ~(1<<0);
1460         temp |= (1<<1);
1461     } else {
1462         temp &= ~(1<<1);
1463     }
1464     WREG32(CONFIG_CNTL, temp);
1465 }
1466
1467 static u32 si_get_xclk(struct amdgpu_device *adev)
1468 {
1469     u32 reference_clock = adev->clock.spll.reference_freq;
1470     u32 tmp;
1471
1472     tmp = RREG32(CG_CLKPIN_CNTL_2);
1473     if (tmp & MUX_TCLK_TO_XCLK)
1474         return TCLK;
1475
1476     tmp = RREG32(CG_CLKPIN_CNTL);
1477     if (tmp & XTALIN_DIVIDE)
1478         return reference_clock / 4;
1479
1480     return reference_clock;
1481 }
1482
1483 static void si_flush_hdp(struct amdgpu_device *adev, struct amdgpu_ring *ring)
1484 {
1485     if (!ring || !ring->funcs->emit_wreg) {
1486         WREG32(mmHDP_MEM_COHERENCY_FLUSH_CNTL, 1);
1487         RREG32(mmHDP_MEM_COHERENCY_FLUSH_CNTL);
1488     } else {
1489         amdgpu_ring_emit_wreg(ring, mmHDP_MEM_COHERENCY_FLUSH_CNTL, 1);
1490     }
1491 }
1492
1493 static void si_invalidate_hdp(struct amdgpu_device *adev,
1494                   struct amdgpu_ring *ring)
1495 {
1496     if (!ring || !ring->funcs->emit_wreg) {
1497         WREG32(mmHDP_DEBUG0, 1);
1498         RREG32(mmHDP_DEBUG0);
1499     } else {
1500         amdgpu_ring_emit_wreg(ring, mmHDP_DEBUG0, 1);
1501     }
1502 }
1503
1504 static bool si_need_full_reset(struct amdgpu_device *adev)
1505 {
1506     /* change this when we support soft reset */
1507     return true;
1508 }
1509
1510 static bool si_need_reset_on_init(struct amdgpu_device *adev)
1511 {
1512     return false;
1513 }
1514
1515 static int si_get_pcie_lanes(struct amdgpu_device *adev)
1516 {
1517     u32 link_width_cntl;
1518
1519     if (adev->flags & AMD_IS_APU)
1520         return 0;
1521
1522     link_width_cntl = RREG32_PCIE_PORT(PCIE_LC_LINK_WIDTH_CNTL);
1523
1524     switch ((link_width_cntl & LC_LINK_WIDTH_RD_MASK) >> LC_LINK_WIDTH_RD_SHIFT) {
1525     case LC_LINK_WIDTH_X1:
1526         return 1;
1527     case LC_LINK_WIDTH_X2:
1528         return 2;
1529     case LC_LINK_WIDTH_X4:
1530         return 4;
1531     case LC_LINK_WIDTH_X8:
1532         return 8;
1533     case LC_LINK_WIDTH_X0:
1534     case LC_LINK_WIDTH_X16:
1535     default:
1536         return 16;
1537     }
1538 }
1539
1540 static void si_set_pcie_lanes(struct amdgpu_device *adev, int lanes)
1541 {
1542     u32 link_width_cntl, mask;
1543
1544     if (adev->flags & AMD_IS_APU)
1545         return;
1546
1547     switch (lanes) {
1548     case 0:
1549         mask = LC_LINK_WIDTH_X0;
1550         break;
1551     case 1:
1552         mask = LC_LINK_WIDTH_X1;
1553         break;
1554     case 2:
1555         mask = LC_LINK_WIDTH_X2;
1556         break;
1557     case 4:
1558         mask = LC_LINK_WIDTH_X4;
1559         break;
1560     case 8:
1561         mask = LC_LINK_WIDTH_X8;
1562         break;
1563     case 16:
1564         mask = LC_LINK_WIDTH_X16;
1565         break;
1566     default:
1567         DRM_ERROR("invalid pcie lane request: %d\n", lanes);
1568         return;
1569     }
1570
1571     link_width_cntl = RREG32_PCIE_PORT(PCIE_LC_LINK_WIDTH_CNTL);
1572     link_width_cntl &= ~LC_LINK_WIDTH_MASK;
1573     link_width_cntl |= mask << LC_LINK_WIDTH_SHIFT;
1574     link_width_cntl |= (LC_RECONFIG_NOW |
1575                 LC_RECONFIG_ARC_MISSING_ESCAPE);
1576
1577     WREG32_PCIE_PORT(PCIE_LC_LINK_WIDTH_CNTL, link_width_cntl);
1578 }
1579
1580 static void si_get_pcie_usage(struct amdgpu_device *adev, uint64_t *count0,
1581                   uint64_t *count1)
1582 {
1583     uint32_t perfctr = 0;
1584     uint64_t cnt0_of, cnt1_of;
1585     int tmp;
1586
1587     /* This reports 0 on APUs, so return to avoid writing/reading registers
1588      * that may or may not be different from their GPU counterparts
1589      */
1590     if (adev->flags & AMD_IS_APU)
1591         return;
1592
1593     /* Set the 2 events that we wish to watch, defined above */
1594     /* Reg 40 is # received msgs, Reg 104 is # of posted requests sent */
1595     perfctr = REG_SET_FIELD(perfctr, PCIE_PERF_CNTL_TXCLK, EVENT0_SEL, 40);
1596     perfctr = REG_SET_FIELD(perfctr, PCIE_PERF_CNTL_TXCLK, EVENT1_SEL, 104);
1597
1598     /* Write to enable desired perf counters */
1599     WREG32_PCIE(ixPCIE_PERF_CNTL_TXCLK, perfctr);
1600     /* Zero out and enable the perf counters
1601      * Write 0x5:
1602      * Bit 0 = Start all counters(1)
1603      * Bit 2 = Global counter reset enable(1)
1604      */
1605     WREG32_PCIE(ixPCIE_PERF_COUNT_CNTL, 0x00000005);
1606
1607     msleep(1000);
1608
1609     /* Load the shadow and disable the perf counters
1610      * Write 0x2:
1611      * Bit 0 = Stop counters(0)
1612      * Bit 1 = Load the shadow counters(1)
1613      */
1614     WREG32_PCIE(ixPCIE_PERF_COUNT_CNTL, 0x00000002);
1615
1616     /* Read register values to get any >32bit overflow */
1617     tmp = RREG32_PCIE(ixPCIE_PERF_CNTL_TXCLK);
1618     cnt0_of = REG_GET_FIELD(tmp, PCIE_PERF_CNTL_TXCLK, COUNTER0_UPPER);
1619     cnt1_of = REG_GET_FIELD(tmp, PCIE_PERF_CNTL_TXCLK, COUNTER1_UPPER);
1620
1621     /* Get the values and add the overflow */
1622     *count0 = RREG32_PCIE(ixPCIE_PERF_COUNT0_TXCLK) | (cnt0_of << 32);
1623     *count1 = RREG32_PCIE(ixPCIE_PERF_COUNT1_TXCLK) | (cnt1_of << 32);
1624 }
1625
1626 static uint64_t si_get_pcie_replay_count(struct amdgpu_device *adev)
1627 {
1628     uint64_t nak_r, nak_g;
1629
1630     /* Get the number of NAKs received and generated */
1631     nak_r = RREG32_PCIE(ixPCIE_RX_NUM_NAK);
1632     nak_g = RREG32_PCIE(ixPCIE_RX_NUM_NAK_GENERATED);
1633
1634     /* Add the total number of NAKs, i.e the number of replays */
1635     return (nak_r + nak_g);
1636 }
1637
1638 static int si_uvd_send_upll_ctlreq(struct amdgpu_device *adev,
1639                    unsigned cg_upll_func_cntl)
1640 {
1641     unsigned i;
1642
1643     /* Make sure UPLL_CTLREQ is deasserted */
1644     WREG32_P(cg_upll_func_cntl, 0, ~UPLL_CTLREQ_MASK);
1645
1646     mdelay(10);
1647
1648     /* Assert UPLL_CTLREQ */
1649     WREG32_P(cg_upll_func_cntl, UPLL_CTLREQ_MASK, ~UPLL_CTLREQ_MASK);
1650
1651     /* Wait for CTLACK and CTLACK2 to get asserted */
1652     for (i = 0; i < SI_MAX_CTLACKS_ASSERTION_WAIT; ++i) {
1653         uint32_t mask = UPLL_CTLACK_MASK | UPLL_CTLACK2_MASK;
1654
1655         if ((RREG32(cg_upll_func_cntl) & mask) == mask)
1656             break;
1657         mdelay(10);
1658     }
1659
1660     /* Deassert UPLL_CTLREQ */
1661     WREG32_P(cg_upll_func_cntl, 0, ~UPLL_CTLREQ_MASK);
1662
1663     if (i == SI_MAX_CTLACKS_ASSERTION_WAIT) {
1664         DRM_ERROR("Timeout setting UVD clocks!\n");
1665         return -ETIMEDOUT;
1666     }
1667
1668     return 0;
1669 }
1670
1671 static unsigned si_uvd_calc_upll_post_div(unsigned vco_freq,
1672                       unsigned target_freq,
1673                       unsigned pd_min,
1674                       unsigned pd_even)
1675 {
1676     unsigned post_div = vco_freq / target_freq;
1677
1678     /* Adjust to post divider minimum value */
1679     if (post_div < pd_min)
1680         post_div = pd_min;
1681
1682     /* We alway need a frequency less than or equal the target */
1683     if ((vco_freq / post_div) > target_freq)
1684         post_div += 1;
1685
1686     /* Post dividers above a certain value must be even */
1687     if (post_div > pd_even && post_div % 2)
1688         post_div += 1;
1689
1690     return post_div;
1691 }
1692
1693 /**
1694  * si_calc_upll_dividers - calc UPLL clock dividers
1695  *
1696  * @adev: amdgpu_device pointer
1697  * @vclk: wanted VCLK
1698  * @dclk: wanted DCLK
1699  * @vco_min: minimum VCO frequency
1700  * @vco_max: maximum VCO frequency
1701  * @fb_factor: factor to multiply vco freq with
1702  * @fb_mask: limit and bitmask for feedback divider
1703  * @pd_min: post divider minimum
1704  * @pd_max: post divider maximum
1705  * @pd_even: post divider must be even above this value
1706  * @optimal_fb_div: resulting feedback divider
1707  * @optimal_vclk_div: resulting vclk post divider
1708  * @optimal_dclk_div: resulting dclk post divider
1709  *
1710  * Calculate dividers for UVDs UPLL (except APUs).
1711  * Returns zero on success; -EINVAL on error.
1712  */
1713 static int si_calc_upll_dividers(struct amdgpu_device *adev,
1714                  unsigned vclk, unsigned dclk,
1715                  unsigned vco_min, unsigned vco_max,
1716                  unsigned fb_factor, unsigned fb_mask,
1717                  unsigned pd_min, unsigned pd_max,
1718                  unsigned pd_even,
1719                  unsigned *optimal_fb_div,
1720                  unsigned *optimal_vclk_div,
1721                  unsigned *optimal_dclk_div)
1722 {
1723     unsigned vco_freq, ref_freq = adev->clock.spll.reference_freq;
1724
1725     /* Start off with something large */
1726     unsigned optimal_score = ~0;
1727
1728     /* Loop through vco from low to high */
1729     vco_min = max(max(vco_min, vclk), dclk);
1730     for (vco_freq = vco_min; vco_freq <= vco_max; vco_freq += 100) {
1731         uint64_t fb_div = (uint64_t)vco_freq * fb_factor;
1732         unsigned vclk_div, dclk_div, score;
1733
1734         do_div(fb_div, ref_freq);
1735
1736         /* fb div out of range ? */
1737         if (fb_div > fb_mask)
1738             break; /* It can oly get worse */
1739
1740         fb_div &= fb_mask;
1741
1742         /* Calc vclk divider with current vco freq */
1743         vclk_div = si_uvd_calc_upll_post_div(vco_freq, vclk,
1744                              pd_min, pd_even);
1745         if (vclk_div > pd_max)
1746             break; /* vco is too big, it has to stop */
1747
1748         /* Calc dclk divider with current vco freq */
1749         dclk_div = si_uvd_calc_upll_post_div(vco_freq, dclk,
1750                              pd_min, pd_even);
1751         if (dclk_div > pd_max)
1752             break; /* vco is too big, it has to stop */
1753
1754         /* Calc score with current vco freq */
1755         score = vclk - (vco_freq / vclk_div) + dclk - (vco_freq / dclk_div);
1756
1757         /* Determine if this vco setting is better than current optimal settings */
1758         if (score < optimal_score) {
1759             *optimal_fb_div = fb_div;
1760             *optimal_vclk_div = vclk_div;
1761             *optimal_dclk_div = dclk_div;
1762             optimal_score = score;
1763             if (optimal_score == 0)
1764                 break; /* It can't get better than this */
1765         }
1766     }
1767
1768     /* Did we found a valid setup ? */
1769     if (optimal_score == ~0)
1770         return -EINVAL;
1771
1772     return 0;
1773 }
1774
1775 static int si_set_uvd_clocks(struct amdgpu_device *adev, u32 vclk, u32 dclk)
1776 {
1777     unsigned fb_div = 0, vclk_div = 0, dclk_div = 0;
1778     int r;
1779
1780     /* Bypass vclk and dclk with bclk */
1781     WREG32_P(CG_UPLL_FUNC_CNTL_2,
1782          VCLK_SRC_SEL(1) | DCLK_SRC_SEL(1),
1783          ~(VCLK_SRC_SEL_MASK | DCLK_SRC_SEL_MASK));
1784
1785     /* Put PLL in bypass mode */
1786     WREG32_P(CG_UPLL_FUNC_CNTL, UPLL_BYPASS_EN_MASK, ~UPLL_BYPASS_EN_MASK);
1787
1788     if (!vclk || !dclk) {
1789         /* Keep the Bypass mode */
1790         return 0;
1791     }
1792
1793     r = si_calc_upll_dividers(adev, vclk, dclk, 125000, 250000,
1794                   16384, 0x03FFFFFF, 0, 128, 5,
1795                   &fb_div, &vclk_div, &dclk_div);
1796     if (r)
1797         return r;
1798
1799     /* Set RESET_ANTI_MUX to 0 */
1800     WREG32_P(CG_UPLL_FUNC_CNTL_5, 0, ~RESET_ANTI_MUX_MASK);
1801
1802     /* Set VCO_MODE to 1 */
1803     WREG32_P(CG_UPLL_FUNC_CNTL, UPLL_VCO_MODE_MASK, ~UPLL_VCO_MODE_MASK);
1804
1805     /* Disable sleep mode */
1806     WREG32_P(CG_UPLL_FUNC_CNTL, 0, ~UPLL_SLEEP_MASK);
1807
1808     /* Deassert UPLL_RESET */
1809     WREG32_P(CG_UPLL_FUNC_CNTL, 0, ~UPLL_RESET_MASK);
1810
1811     mdelay(1);
1812
1813     r = si_uvd_send_upll_ctlreq(adev, CG_UPLL_FUNC_CNTL);
1814     if (r)
1815         return r;
1816
1817     /* Assert UPLL_RESET again */
1818     WREG32_P(CG_UPLL_FUNC_CNTL, UPLL_RESET_MASK, ~UPLL_RESET_MASK);
1819
1820     /* Disable spread spectrum. */
1821     WREG32_P(CG_UPLL_SPREAD_SPECTRUM, 0, ~SSEN_MASK);
1822
1823     /* Set feedback divider */
1824     WREG32_P(CG_UPLL_FUNC_CNTL_3, UPLL_FB_DIV(fb_div), ~UPLL_FB_DIV_MASK);
1825
1826     /* Set ref divider to 0 */
1827     WREG32_P(CG_UPLL_FUNC_CNTL, 0, ~UPLL_REF_DIV_MASK);
1828
1829     if (fb_div < 307200)
1830         WREG32_P(CG_UPLL_FUNC_CNTL_4, 0, ~UPLL_SPARE_ISPARE9);
1831     else
1832         WREG32_P(CG_UPLL_FUNC_CNTL_4,
1833              UPLL_SPARE_ISPARE9,
1834              ~UPLL_SPARE_ISPARE9);
1835
1836     /* Set PDIV_A and PDIV_B */
1837     WREG32_P(CG_UPLL_FUNC_CNTL_2,
1838          UPLL_PDIV_A(vclk_div) | UPLL_PDIV_B(dclk_div),
1839          ~(UPLL_PDIV_A_MASK | UPLL_PDIV_B_MASK));
1840
1841     /* Give the PLL some time to settle */
1842     mdelay(15);
1843
1844     /* Deassert PLL_RESET */
1845     WREG32_P(CG_UPLL_FUNC_CNTL, 0, ~UPLL_RESET_MASK);
1846
1847     mdelay(15);
1848
1849     /* Switch from bypass mode to normal mode */
1850     WREG32_P(CG_UPLL_FUNC_CNTL, 0, ~UPLL_BYPASS_EN_MASK);
1851
1852     r = si_uvd_send_upll_ctlreq(adev, CG_UPLL_FUNC_CNTL);
1853     if (r)
1854         return r;
1855
1856     /* Switch VCLK and DCLK selection */
1857     WREG32_P(CG_UPLL_FUNC_CNTL_2,
1858          VCLK_SRC_SEL(2) | DCLK_SRC_SEL(2),
1859          ~(VCLK_SRC_SEL_MASK | DCLK_SRC_SEL_MASK));
1860
1861     mdelay(100);
1862
1863     return 0;
1864 }
1865
1866 static int si_vce_send_vcepll_ctlreq(struct amdgpu_device *adev)
1867 {
1868     unsigned i;
1869
1870     /* Make sure VCEPLL_CTLREQ is deasserted */
1871     WREG32_SMC_P(CG_VCEPLL_FUNC_CNTL, 0, ~UPLL_CTLREQ_MASK);
1872
1873     mdelay(10);
1874
1875     /* Assert UPLL_CTLREQ */
1876     WREG32_SMC_P(CG_VCEPLL_FUNC_CNTL, UPLL_CTLREQ_MASK, ~UPLL_CTLREQ_MASK);
1877
1878     /* Wait for CTLACK and CTLACK2 to get asserted */
1879     for (i = 0; i < SI_MAX_CTLACKS_ASSERTION_WAIT; ++i) {
1880         uint32_t mask = UPLL_CTLACK_MASK | UPLL_CTLACK2_MASK;
1881
1882         if ((RREG32_SMC(CG_VCEPLL_FUNC_CNTL) & mask) == mask)
1883             break;
1884         mdelay(10);
1885     }
1886
1887     /* Deassert UPLL_CTLREQ */
1888     WREG32_SMC_P(CG_VCEPLL_FUNC_CNTL, 0, ~UPLL_CTLREQ_MASK);
1889
1890     if (i == SI_MAX_CTLACKS_ASSERTION_WAIT) {
1891         DRM_ERROR("Timeout setting UVD clocks!\n");
1892         return -ETIMEDOUT;
1893     }
1894
1895     return 0;
1896 }
1897
1898 static int si_set_vce_clocks(struct amdgpu_device *adev, u32 evclk, u32 ecclk)
1899 {
1900     unsigned fb_div = 0, evclk_div = 0, ecclk_div = 0;
1901     int r;
1902
1903     /* Bypass evclk and ecclk with bclk */
1904     WREG32_SMC_P(CG_VCEPLL_FUNC_CNTL_2,
1905              EVCLK_SRC_SEL(1) | ECCLK_SRC_SEL(1),
1906              ~(EVCLK_SRC_SEL_MASK | ECCLK_SRC_SEL_MASK));
1907
1908     /* Put PLL in bypass mode */
1909     WREG32_SMC_P(CG_VCEPLL_FUNC_CNTL, VCEPLL_BYPASS_EN_MASK,
1910              ~VCEPLL_BYPASS_EN_MASK);
1911
1912     if (!evclk || !ecclk) {
1913         /* Keep the Bypass mode, put PLL to sleep */
1914         WREG32_SMC_P(CG_VCEPLL_FUNC_CNTL, VCEPLL_SLEEP_MASK,
1915                  ~VCEPLL_SLEEP_MASK);
1916         return 0;
1917     }
1918
1919     r = si_calc_upll_dividers(adev, evclk, ecclk, 125000, 250000,
1920                   16384, 0x03FFFFFF, 0, 128, 5,
1921                   &fb_div, &evclk_div, &ecclk_div);
1922     if (r)
1923         return r;
1924
1925     /* Set RESET_ANTI_MUX to 0 */
1926     WREG32_SMC_P(CG_VCEPLL_FUNC_CNTL_5, 0, ~RESET_ANTI_MUX_MASK);
1927
1928     /* Set VCO_MODE to 1 */
1929     WREG32_SMC_P(CG_VCEPLL_FUNC_CNTL, VCEPLL_VCO_MODE_MASK,
1930              ~VCEPLL_VCO_MODE_MASK);
1931
1932     /* Toggle VCEPLL_SLEEP to 1 then back to 0 */
1933     WREG32_SMC_P(CG_VCEPLL_FUNC_CNTL, VCEPLL_SLEEP_MASK,
1934              ~VCEPLL_SLEEP_MASK);
1935     WREG32_SMC_P(CG_VCEPLL_FUNC_CNTL, 0, ~VCEPLL_SLEEP_MASK);
1936
1937     /* Deassert VCEPLL_RESET */
1938     WREG32_SMC_P(CG_VCEPLL_FUNC_CNTL, 0, ~VCEPLL_RESET_MASK);
1939
1940     mdelay(1);
1941
1942     r = si_vce_send_vcepll_ctlreq(adev);
1943     if (r)
1944         return r;
1945
1946     /* Assert VCEPLL_RESET again */
1947     WREG32_SMC_P(CG_VCEPLL_FUNC_CNTL, VCEPLL_RESET_MASK, ~VCEPLL_RESET_MASK);
1948
1949     /* Disable spread spectrum. */
1950     WREG32_SMC_P(CG_VCEPLL_SPREAD_SPECTRUM, 0, ~SSEN_MASK);
1951
1952     /* Set feedback divider */
1953     WREG32_SMC_P(CG_VCEPLL_FUNC_CNTL_3,
1954              VCEPLL_FB_DIV(fb_div),
1955              ~VCEPLL_FB_DIV_MASK);
1956
1957     /* Set ref divider to 0 */
1958     WREG32_SMC_P(CG_VCEPLL_FUNC_CNTL, 0, ~VCEPLL_REF_DIV_MASK);
1959
1960     /* Set PDIV_A and PDIV_B */
1961     WREG32_SMC_P(CG_VCEPLL_FUNC_CNTL_2,
1962              VCEPLL_PDIV_A(evclk_div) | VCEPLL_PDIV_B(ecclk_div),
1963              ~(VCEPLL_PDIV_A_MASK | VCEPLL_PDIV_B_MASK));
1964
1965     /* Give the PLL some time to settle */
1966     mdelay(15);
1967
1968     /* Deassert PLL_RESET */
1969     WREG32_SMC_P(CG_VCEPLL_FUNC_CNTL, 0, ~VCEPLL_RESET_MASK);
1970
1971     mdelay(15);
1972
1973     /* Switch from bypass mode to normal mode */
1974     WREG32_SMC_P(CG_VCEPLL_FUNC_CNTL, 0, ~VCEPLL_BYPASS_EN_MASK);
1975
1976     r = si_vce_send_vcepll_ctlreq(adev);
1977     if (r)
1978         return r;
1979
1980     /* Switch VCLK and DCLK selection */
1981     WREG32_SMC_P(CG_VCEPLL_FUNC_CNTL_2,
1982              EVCLK_SRC_SEL(16) | ECCLK_SRC_SEL(16),
1983              ~(EVCLK_SRC_SEL_MASK | ECCLK_SRC_SEL_MASK));
1984
1985     mdelay(100);
1986
1987     return 0;
1988 }
1989
1990 static void si_pre_asic_init(struct amdgpu_device *adev)
1991 {
1992 }
1993
1994 static const struct amdgpu_asic_funcs si_asic_funcs =
1995 {
1996     .read_disabled_bios = &si_read_disabled_bios,
1997     .read_bios_from_rom = &si_read_bios_from_rom,
1998     .read_register = &si_read_register,
1999     .reset = &si_asic_reset,
2000     .reset_method = &si_asic_reset_method,
2001     .set_vga_state = &si_vga_set_state,
2002     .get_xclk = &si_get_xclk,
2003     .set_uvd_clocks = &si_set_uvd_clocks,
2004     .set_vce_clocks = &si_set_vce_clocks,
2005     .get_pcie_lanes = &si_get_pcie_lanes,
2006     .set_pcie_lanes = &si_set_pcie_lanes,
2007     .get_config_memsize = &si_get_config_memsize,
2008     .flush_hdp = &si_flush_hdp,
2009     .invalidate_hdp = &si_invalidate_hdp,
2010     .need_full_reset = &si_need_full_reset,
2011     .get_pcie_usage = &si_get_pcie_usage,
2012     .need_reset_on_init = &si_need_reset_on_init,
2013     .get_pcie_replay_count = &si_get_pcie_replay_count,
2014     .supports_baco = &si_asic_supports_baco,
2015     .pre_asic_init = &si_pre_asic_init,
2016     .query_video_codecs = &si_query_video_codecs,
2017 };
2018
2019 static uint32_t si_get_rev_id(struct amdgpu_device *adev)
2020 {
2021     return (RREG32(CC_DRM_ID_STRAPS) & CC_DRM_ID_STRAPS__ATI_REV_ID_MASK)
2022         >> CC_DRM_ID_STRAPS__ATI_REV_ID__SHIFT;
2023 }
2024
2025 static int si_common_early_init(void *handle)
2026 {
2027     struct amdgpu_device *adev = (struct amdgpu_device *)handle;
2028
2029     adev->smc_rreg = &si_smc_rreg;
2030     adev->smc_wreg = &si_smc_wreg;
2031     adev->pcie_rreg = &si_pcie_rreg;
2032     adev->pcie_wreg = &si_pcie_wreg;
2033     adev->pciep_rreg = &si_pciep_rreg;
2034     adev->pciep_wreg = &si_pciep_wreg;
2035     adev->uvd_ctx_rreg = si_uvd_ctx_rreg;
2036     adev->uvd_ctx_wreg = si_uvd_ctx_wreg;
2037     adev->didt_rreg = NULL;
2038     adev->didt_wreg = NULL;
2039
2040     adev->asic_funcs = &si_asic_funcs;
2041
2042     adev->rev_id = si_get_rev_id(adev);
2043     adev->external_rev_id = 0xFF;
2044     switch (adev->asic_type) {
2045     case CHIP_TAHITI:
2046         adev->cg_flags =
2047             AMD_CG_SUPPORT_GFX_MGCG |
2048             AMD_CG_SUPPORT_GFX_MGLS |
2049             /*AMD_CG_SUPPORT_GFX_CGCG |*/
2050             AMD_CG_SUPPORT_GFX_CGLS |
2051             AMD_CG_SUPPORT_GFX_CGTS |
2052             AMD_CG_SUPPORT_GFX_CP_LS |
2053             AMD_CG_SUPPORT_MC_MGCG |
2054             AMD_CG_SUPPORT_SDMA_MGCG |
2055             AMD_CG_SUPPORT_BIF_LS |
2056             AMD_CG_SUPPORT_VCE_MGCG |
2057             AMD_CG_SUPPORT_UVD_MGCG |
2058             AMD_CG_SUPPORT_HDP_LS |
2059             AMD_CG_SUPPORT_HDP_MGCG;
2060         adev->pg_flags = 0;
2061         adev->external_rev_id = (adev->rev_id == 0) ? 1 :
2062                     (adev->rev_id == 1) ? 5 : 6;
2063         break;
2064     case CHIP_PITCAIRN:
2065         adev->cg_flags =
2066             AMD_CG_SUPPORT_GFX_MGCG |
2067             AMD_CG_SUPPORT_GFX_MGLS |
2068             /*AMD_CG_SUPPORT_GFX_CGCG |*/
2069             AMD_CG_SUPPORT_GFX_CGLS |
2070             AMD_CG_SUPPORT_GFX_CGTS |
2071             AMD_CG_SUPPORT_GFX_CP_LS |
2072             AMD_CG_SUPPORT_GFX_RLC_LS |
2073             AMD_CG_SUPPORT_MC_LS |
2074             AMD_CG_SUPPORT_MC_MGCG |
2075             AMD_CG_SUPPORT_SDMA_MGCG |
2076             AMD_CG_SUPPORT_BIF_LS |
2077             AMD_CG_SUPPORT_VCE_MGCG |
2078             AMD_CG_SUPPORT_UVD_MGCG |
2079             AMD_CG_SUPPORT_HDP_LS |
2080             AMD_CG_SUPPORT_HDP_MGCG;
2081         adev->pg_flags = 0;
2082         adev->external_rev_id = adev->rev_id + 20;
2083         break;
2084
2085     case CHIP_VERDE:
2086         adev->cg_flags =
2087             AMD_CG_SUPPORT_GFX_MGCG |
2088             AMD_CG_SUPPORT_GFX_MGLS |
2089             AMD_CG_SUPPORT_GFX_CGLS |
2090             AMD_CG_SUPPORT_GFX_CGTS |
2091             AMD_CG_SUPPORT_GFX_CGTS_LS |
2092             AMD_CG_SUPPORT_GFX_CP_LS |
2093             AMD_CG_SUPPORT_MC_LS |
2094             AMD_CG_SUPPORT_MC_MGCG |
2095             AMD_CG_SUPPORT_SDMA_MGCG |
2096             AMD_CG_SUPPORT_SDMA_LS |
2097             AMD_CG_SUPPORT_BIF_LS |
2098             AMD_CG_SUPPORT_VCE_MGCG |
2099             AMD_CG_SUPPORT_UVD_MGCG |
2100             AMD_CG_SUPPORT_HDP_LS |
2101             AMD_CG_SUPPORT_HDP_MGCG;
2102         adev->pg_flags = 0;
2103         //???
2104         adev->external_rev_id = adev->rev_id + 40;
2105         break;
2106     case CHIP_OLAND:
2107         adev->cg_flags =
2108             AMD_CG_SUPPORT_GFX_MGCG |
2109             AMD_CG_SUPPORT_GFX_MGLS |
2110             /*AMD_CG_SUPPORT_GFX_CGCG |*/
2111             AMD_CG_SUPPORT_GFX_CGLS |
2112             AMD_CG_SUPPORT_GFX_CGTS |
2113             AMD_CG_SUPPORT_GFX_CP_LS |
2114             AMD_CG_SUPPORT_GFX_RLC_LS |
2115             AMD_CG_SUPPORT_MC_LS |
2116             AMD_CG_SUPPORT_MC_MGCG |
2117             AMD_CG_SUPPORT_SDMA_MGCG |
2118             AMD_CG_SUPPORT_BIF_LS |
2119             AMD_CG_SUPPORT_UVD_MGCG |
2120             AMD_CG_SUPPORT_HDP_LS |
2121             AMD_CG_SUPPORT_HDP_MGCG;
2122         adev->pg_flags = 0;
2123         adev->external_rev_id = 60;
2124         break;
2125     case CHIP_HAINAN:
2126         adev->cg_flags =
2127             AMD_CG_SUPPORT_GFX_MGCG |
2128             AMD_CG_SUPPORT_GFX_MGLS |
2129             /*AMD_CG_SUPPORT_GFX_CGCG |*/
2130             AMD_CG_SUPPORT_GFX_CGLS |
2131             AMD_CG_SUPPORT_GFX_CGTS |
2132             AMD_CG_SUPPORT_GFX_CP_LS |
2133             AMD_CG_SUPPORT_GFX_RLC_LS |
2134             AMD_CG_SUPPORT_MC_LS |
2135             AMD_CG_SUPPORT_MC_MGCG |
2136             AMD_CG_SUPPORT_SDMA_MGCG |
2137             AMD_CG_SUPPORT_BIF_LS |
2138             AMD_CG_SUPPORT_HDP_LS |
2139             AMD_CG_SUPPORT_HDP_MGCG;
2140         adev->pg_flags = 0;
2141         adev->external_rev_id = 70;
2142         break;
2143
2144     default:
2145         return -EINVAL;
2146     }
2147
2148     return 0;
2149 }
2150
2151 static int si_common_sw_init(void *handle)
2152 {
2153     return 0;
2154 }
2155
2156 static int si_common_sw_fini(void *handle)
2157 {
2158     return 0;
2159 }
2160
2161
2162 static void si_init_golden_registers(struct amdgpu_device *adev)
2163 {
2164     switch (adev->asic_type) {
2165     case CHIP_TAHITI:
2166         amdgpu_device_program_register_sequence(adev,
2167                             tahiti_golden_registers,
2168                             ARRAY_SIZE(tahiti_golden_registers));
2169         amdgpu_device_program_register_sequence(adev,
2170                             tahiti_golden_rlc_registers,
2171                             ARRAY_SIZE(tahiti_golden_rlc_registers));
2172         amdgpu_device_program_register_sequence(adev,
2173                             tahiti_mgcg_cgcg_init,
2174                             ARRAY_SIZE(tahiti_mgcg_cgcg_init));
2175         amdgpu_device_program_register_sequence(adev,
2176                             tahiti_golden_registers2,
2177                             ARRAY_SIZE(tahiti_golden_registers2));
2178         break;
2179     case CHIP_PITCAIRN:
2180         amdgpu_device_program_register_sequence(adev,
2181                             pitcairn_golden_registers,
2182                             ARRAY_SIZE(pitcairn_golden_registers));
2183         amdgpu_device_program_register_sequence(adev,
2184                             pitcairn_golden_rlc_registers,
2185                             ARRAY_SIZE(pitcairn_golden_rlc_registers));
2186         amdgpu_device_program_register_sequence(adev,
2187                             pitcairn_mgcg_cgcg_init,
2188                             ARRAY_SIZE(pitcairn_mgcg_cgcg_init));
2189         break;
2190     case CHIP_VERDE:
2191         amdgpu_device_program_register_sequence(adev,
2192                             verde_golden_registers,
2193                             ARRAY_SIZE(verde_golden_registers));
2194         amdgpu_device_program_register_sequence(adev,
2195                             verde_golden_rlc_registers,
2196                             ARRAY_SIZE(verde_golden_rlc_registers));
2197         amdgpu_device_program_register_sequence(adev,
2198                             verde_mgcg_cgcg_init,
2199                             ARRAY_SIZE(verde_mgcg_cgcg_init));
2200         amdgpu_device_program_register_sequence(adev,
2201                             verde_pg_init,
2202                             ARRAY_SIZE(verde_pg_init));
2203         break;
2204     case CHIP_OLAND:
2205         amdgpu_device_program_register_sequence(adev,
2206                             oland_golden_registers,
2207                             ARRAY_SIZE(oland_golden_registers));
2208         amdgpu_device_program_register_sequence(adev,
2209                             oland_golden_rlc_registers,
2210                             ARRAY_SIZE(oland_golden_rlc_registers));
2211         amdgpu_device_program_register_sequence(adev,
2212                             oland_mgcg_cgcg_init,
2213                             ARRAY_SIZE(oland_mgcg_cgcg_init));
2214         break;
2215     case CHIP_HAINAN:
2216         amdgpu_device_program_register_sequence(adev,
2217                             hainan_golden_registers,
2218                             ARRAY_SIZE(hainan_golden_registers));
2219         amdgpu_device_program_register_sequence(adev,
2220                             hainan_golden_registers2,
2221                             ARRAY_SIZE(hainan_golden_registers2));
2222         amdgpu_device_program_register_sequence(adev,
2223                             hainan_mgcg_cgcg_init,
2224                             ARRAY_SIZE(hainan_mgcg_cgcg_init));
2225         break;
2226
2227
2228     default:
2229         BUG();
2230     }
2231 }
2232
2233 static void si_pcie_gen3_enable(struct amdgpu_device *adev)
2234 {
2235     struct pci_dev *root = adev->pdev->bus->self;
2236     u32 speed_cntl, current_data_rate;
2237     int i;
2238     u16 tmp16;
2239
2240     if (pci_is_root_bus(adev->pdev->bus))
2241         return;
2242
2243     if (amdgpu_pcie_gen2 == 0)
2244         return;
2245
2246     if (adev->flags & AMD_IS_APU)
2247         return;
2248
2249     if (!(adev->pm.pcie_gen_mask & (CAIL_PCIE_LINK_SPEED_SUPPORT_GEN2 |
2250                     CAIL_PCIE_LINK_SPEED_SUPPORT_GEN3)))
2251         return;
2252
2253     speed_cntl = RREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL);
2254     current_data_rate = (speed_cntl & LC_CURRENT_DATA_RATE_MASK) >>
2255         LC_CURRENT_DATA_RATE_SHIFT;
2256     if (adev->pm.pcie_gen_mask & CAIL_PCIE_LINK_SPEED_SUPPORT_GEN3) {
2257         if (current_data_rate == 2) {
2258             DRM_INFO("PCIE gen 3 link speeds already enabled\n");
2259             return;
2260         }
2261         DRM_INFO("enabling PCIE gen 3 link speeds, disable with amdgpu.pcie_gen2=0\n");
2262     } else if (adev->pm.pcie_gen_mask & CAIL_PCIE_LINK_SPEED_SUPPORT_GEN2) {
2263         if (current_data_rate == 1) {
2264             DRM_INFO("PCIE gen 2 link speeds already enabled\n");
2265             return;
2266         }
2267         DRM_INFO("enabling PCIE gen 2 link speeds, disable with amdgpu.pcie_gen2=0\n");
2268     }
2269
2270     if (!pci_is_pcie(root) || !pci_is_pcie(adev->pdev))
2271         return;
2272
2273     if (adev->pm.pcie_gen_mask & CAIL_PCIE_LINK_SPEED_SUPPORT_GEN3) {
2274         if (current_data_rate != 2) {
2275             u16 bridge_cfg, gpu_cfg;
2276             u16 bridge_cfg2, gpu_cfg2;
2277             u32 max_lw, current_lw, tmp;
2278
2279             pcie_capability_read_word(root, PCI_EXP_LNKCTL,
2280                           &bridge_cfg);
2281             pcie_capability_read_word(adev->pdev, PCI_EXP_LNKCTL,
2282                           &gpu_cfg);
2283
2284             tmp16 = bridge_cfg | PCI_EXP_LNKCTL_HAWD;
2285             pcie_capability_write_word(root, PCI_EXP_LNKCTL, tmp16);
2286
2287             tmp16 = gpu_cfg | PCI_EXP_LNKCTL_HAWD;
2288             pcie_capability_write_word(adev->pdev, PCI_EXP_LNKCTL,
2289                            tmp16);
2290
2291             tmp = RREG32_PCIE(PCIE_LC_STATUS1);
2292             max_lw = (tmp & LC_DETECTED_LINK_WIDTH_MASK) >> LC_DETECTED_LINK_WIDTH_SHIFT;
2293             current_lw = (tmp & LC_OPERATING_LINK_WIDTH_MASK) >> LC_OPERATING_LINK_WIDTH_SHIFT;
2294
2295             if (current_lw < max_lw) {
2296                 tmp = RREG32_PCIE_PORT(PCIE_LC_LINK_WIDTH_CNTL);
2297                 if (tmp & LC_RENEGOTIATION_SUPPORT) {
2298                     tmp &= ~(LC_LINK_WIDTH_MASK | LC_UPCONFIGURE_DIS);
2299                     tmp |= (max_lw << LC_LINK_WIDTH_SHIFT);
2300                     tmp |= LC_UPCONFIGURE_SUPPORT | LC_RENEGOTIATE_EN | LC_RECONFIG_NOW;
2301                     WREG32_PCIE_PORT(PCIE_LC_LINK_WIDTH_CNTL, tmp);
2302                 }
2303             }
2304
2305             for (i = 0; i < 10; i++) {
2306                 pcie_capability_read_word(adev->pdev,
2307                               PCI_EXP_DEVSTA,
2308                               &tmp16);
2309                 if (tmp16 & PCI_EXP_DEVSTA_TRPND)
2310                     break;
2311
2312                 pcie_capability_read_word(root, PCI_EXP_LNKCTL,
2313                               &bridge_cfg);
2314                 pcie_capability_read_word(adev->pdev,
2315                               PCI_EXP_LNKCTL,
2316                               &gpu_cfg);
2317
2318                 pcie_capability_read_word(root, PCI_EXP_LNKCTL2,
2319                               &bridge_cfg2);
2320                 pcie_capability_read_word(adev->pdev,
2321                               PCI_EXP_LNKCTL2,
2322                               &gpu_cfg2);
2323
2324                 tmp = RREG32_PCIE_PORT(PCIE_LC_CNTL4);
2325                 tmp |= LC_SET_QUIESCE;
2326                 WREG32_PCIE_PORT(PCIE_LC_CNTL4, tmp);
2327
2328                 tmp = RREG32_PCIE_PORT(PCIE_LC_CNTL4);
2329                 tmp |= LC_REDO_EQ;
2330                 WREG32_PCIE_PORT(PCIE_LC_CNTL4, tmp);
2331
2332                 mdelay(100);
2333
2334                 pcie_capability_read_word(root, PCI_EXP_LNKCTL,
2335                               &tmp16);
2336                 tmp16 &= ~PCI_EXP_LNKCTL_HAWD;
2337                 tmp16 |= (bridge_cfg & PCI_EXP_LNKCTL_HAWD);
2338                 pcie_capability_write_word(root, PCI_EXP_LNKCTL,
2339                                tmp16);
2340
2341                 pcie_capability_read_word(adev->pdev,
2342                               PCI_EXP_LNKCTL,
2343                               &tmp16);
2344                 tmp16 &= ~PCI_EXP_LNKCTL_HAWD;
2345                 tmp16 |= (gpu_cfg & PCI_EXP_LNKCTL_HAWD);
2346                 pcie_capability_write_word(adev->pdev,
2347                                PCI_EXP_LNKCTL,
2348                                tmp16);
2349
2350                 pcie_capability_read_word(root, PCI_EXP_LNKCTL2,
2351                               &tmp16);
2352                 tmp16 &= ~(PCI_EXP_LNKCTL2_ENTER_COMP |
2353                        PCI_EXP_LNKCTL2_TX_MARGIN);
2354                 tmp16 |= (bridge_cfg2 &
2355                       (PCI_EXP_LNKCTL2_ENTER_COMP |
2356                        PCI_EXP_LNKCTL2_TX_MARGIN));
2357                 pcie_capability_write_word(root,
2358                                PCI_EXP_LNKCTL2,
2359                                tmp16);
2360
2361                 pcie_capability_read_word(adev->pdev,
2362                               PCI_EXP_LNKCTL2,
2363                               &tmp16);
2364                 tmp16 &= ~(PCI_EXP_LNKCTL2_ENTER_COMP |
2365                        PCI_EXP_LNKCTL2_TX_MARGIN);
2366                 tmp16 |= (gpu_cfg2 &
2367                       (PCI_EXP_LNKCTL2_ENTER_COMP |
2368                        PCI_EXP_LNKCTL2_TX_MARGIN));
2369                 pcie_capability_write_word(adev->pdev,
2370                                PCI_EXP_LNKCTL2,
2371                                tmp16);
2372
2373                 tmp = RREG32_PCIE_PORT(PCIE_LC_CNTL4);
2374                 tmp &= ~LC_SET_QUIESCE;
2375                 WREG32_PCIE_PORT(PCIE_LC_CNTL4, tmp);
2376             }
2377         }
2378     }
2379
2380     speed_cntl |= LC_FORCE_EN_SW_SPEED_CHANGE | LC_FORCE_DIS_HW_SPEED_CHANGE;
2381     speed_cntl &= ~LC_FORCE_DIS_SW_SPEED_CHANGE;
2382     WREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL, speed_cntl);
2383
2384     pcie_capability_read_word(adev->pdev, PCI_EXP_LNKCTL2, &tmp16);
2385     tmp16 &= ~PCI_EXP_LNKCTL2_TLS;
2386
2387     if (adev->pm.pcie_gen_mask & CAIL_PCIE_LINK_SPEED_SUPPORT_GEN3)
2388         tmp16 |= PCI_EXP_LNKCTL2_TLS_8_0GT; /* gen3 */
2389     else if (adev->pm.pcie_gen_mask & CAIL_PCIE_LINK_SPEED_SUPPORT_GEN2)
2390         tmp16 |= PCI_EXP_LNKCTL2_TLS_5_0GT; /* gen2 */
2391     else
2392         tmp16 |= PCI_EXP_LNKCTL2_TLS_2_5GT; /* gen1 */
2393     pcie_capability_write_word(adev->pdev, PCI_EXP_LNKCTL2, tmp16);
2394
2395     speed_cntl = RREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL);
2396     speed_cntl |= LC_INITIATE_LINK_SPEED_CHANGE;
2397     WREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL, speed_cntl);
2398
2399     for (i = 0; i < adev->usec_timeout; i++) {
2400         speed_cntl = RREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL);
2401         if ((speed_cntl & LC_INITIATE_LINK_SPEED_CHANGE) == 0)
2402             break;
2403         udelay(1);
2404     }
2405 }
2406
2407 static inline u32 si_pif_phy0_rreg(struct amdgpu_device *adev, u32 reg)
2408 {
2409     unsigned long flags;
2410     u32 r;
2411
2412     spin_lock_irqsave(&adev->pcie_idx_lock, flags);
2413     WREG32(EVERGREEN_PIF_PHY0_INDEX, ((reg) & 0xffff));
2414     r = RREG32(EVERGREEN_PIF_PHY0_DATA);
2415     spin_unlock_irqrestore(&adev->pcie_idx_lock, flags);
2416     return r;
2417 }
2418
2419 static inline void si_pif_phy0_wreg(struct amdgpu_device *adev, u32 reg, u32 v)
2420 {
2421     unsigned long flags;
2422
2423     spin_lock_irqsave(&adev->pcie_idx_lock, flags);
2424     WREG32(EVERGREEN_PIF_PHY0_INDEX, ((reg) & 0xffff));
2425     WREG32(EVERGREEN_PIF_PHY0_DATA, (v));
2426     spin_unlock_irqrestore(&adev->pcie_idx_lock, flags);
2427 }
2428
2429 static inline u32 si_pif_phy1_rreg(struct amdgpu_device *adev, u32 reg)
2430 {
2431     unsigned long flags;
2432     u32 r;
2433
2434     spin_lock_irqsave(&adev->pcie_idx_lock, flags);
2435     WREG32(EVERGREEN_PIF_PHY1_INDEX, ((reg) & 0xffff));
2436     r = RREG32(EVERGREEN_PIF_PHY1_DATA);
2437     spin_unlock_irqrestore(&adev->pcie_idx_lock, flags);
2438     return r;
2439 }
2440
2441 static inline void si_pif_phy1_wreg(struct amdgpu_device *adev, u32 reg, u32 v)
2442 {
2443     unsigned long flags;
2444
2445     spin_lock_irqsave(&adev->pcie_idx_lock, flags);
2446     WREG32(EVERGREEN_PIF_PHY1_INDEX, ((reg) & 0xffff));
2447     WREG32(EVERGREEN_PIF_PHY1_DATA, (v));
2448     spin_unlock_irqrestore(&adev->pcie_idx_lock, flags);
2449 }
2450 static void si_program_aspm(struct amdgpu_device *adev)
2451 {
2452     u32 data, orig;
2453     bool disable_l0s = false, disable_l1 = false, disable_plloff_in_l1 = false;
2454     bool disable_clkreq = false;
2455
2456     if (!amdgpu_device_should_use_aspm(adev))
2457         return;
2458
2459     if (adev->flags & AMD_IS_APU)
2460         return;
2461     orig = data = RREG32_PCIE_PORT(PCIE_LC_N_FTS_CNTL);
2462     data &= ~LC_XMIT_N_FTS_MASK;
2463     data |= LC_XMIT_N_FTS(0x24) | LC_XMIT_N_FTS_OVERRIDE_EN;
2464     if (orig != data)
2465         WREG32_PCIE_PORT(PCIE_LC_N_FTS_CNTL, data);
2466
2467     orig = data = RREG32_PCIE_PORT(PCIE_LC_CNTL3);
2468     data |= LC_GO_TO_RECOVERY;
2469     if (orig != data)
2470         WREG32_PCIE_PORT(PCIE_LC_CNTL3, data);
2471
2472     orig = data = RREG32_PCIE(PCIE_P_CNTL);
2473     data |= P_IGNORE_EDB_ERR;
2474     if (orig != data)
2475         WREG32_PCIE(PCIE_P_CNTL, data);
2476
2477     orig = data = RREG32_PCIE_PORT(PCIE_LC_CNTL);
2478     data &= ~(LC_L0S_INACTIVITY_MASK | LC_L1_INACTIVITY_MASK);
2479     data |= LC_PMI_TO_L1_DIS;
2480     if (!disable_l0s)
2481         data |= LC_L0S_INACTIVITY(7);
2482
2483     if (!disable_l1) {
2484         data |= LC_L1_INACTIVITY(7);
2485         data &= ~LC_PMI_TO_L1_DIS;
2486         if (orig != data)
2487             WREG32_PCIE_PORT(PCIE_LC_CNTL, data);
2488
2489         if (!disable_plloff_in_l1) {
2490             bool clk_req_support;
2491
2492             orig = data = si_pif_phy0_rreg(adev,PB0_PIF_PWRDOWN_0);
2493             data &= ~(PLL_POWER_STATE_IN_OFF_0_MASK | PLL_POWER_STATE_IN_TXS2_0_MASK);
2494             data |= PLL_POWER_STATE_IN_OFF_0(7) | PLL_POWER_STATE_IN_TXS2_0(7);
2495             if (orig != data)
2496                 si_pif_phy0_wreg(adev,PB0_PIF_PWRDOWN_0, data);
2497
2498             orig = data = si_pif_phy0_rreg(adev,PB0_PIF_PWRDOWN_1);
2499             data &= ~(PLL_POWER_STATE_IN_OFF_1_MASK | PLL_POWER_STATE_IN_TXS2_1_MASK);
2500             data |= PLL_POWER_STATE_IN_OFF_1(7) | PLL_POWER_STATE_IN_TXS2_1(7);
2501             if (orig != data)
2502                 si_pif_phy0_wreg(adev,PB0_PIF_PWRDOWN_1, data);
2503
2504             orig = data = si_pif_phy1_rreg(adev,PB1_PIF_PWRDOWN_0);
2505             data &= ~(PLL_POWER_STATE_IN_OFF_0_MASK | PLL_POWER_STATE_IN_TXS2_0_MASK);
2506             data |= PLL_POWER_STATE_IN_OFF_0(7) | PLL_POWER_STATE_IN_TXS2_0(7);
2507             if (orig != data)
2508                 si_pif_phy1_wreg(adev,PB1_PIF_PWRDOWN_0, data);
2509
2510             orig = data = si_pif_phy1_rreg(adev,PB1_PIF_PWRDOWN_1);
2511             data &= ~(PLL_POWER_STATE_IN_OFF_1_MASK | PLL_POWER_STATE_IN_TXS2_1_MASK);
2512             data |= PLL_POWER_STATE_IN_OFF_1(7) | PLL_POWER_STATE_IN_TXS2_1(7);
2513             if (orig != data)
2514                 si_pif_phy1_wreg(adev,PB1_PIF_PWRDOWN_1, data);
2515
2516             if ((adev->asic_type != CHIP_OLAND) && (adev->asic_type != CHIP_HAINAN)) {
2517                 orig = data = si_pif_phy0_rreg(adev,PB0_PIF_PWRDOWN_0);
2518                 data &= ~PLL_RAMP_UP_TIME_0_MASK;
2519                 if (orig != data)
2520                     si_pif_phy0_wreg(adev,PB0_PIF_PWRDOWN_0, data);
2521
2522                 orig = data = si_pif_phy0_rreg(adev,PB0_PIF_PWRDOWN_1);
2523                 data &= ~PLL_RAMP_UP_TIME_1_MASK;
2524                 if (orig != data)
2525                     si_pif_phy0_wreg(adev,PB0_PIF_PWRDOWN_1, data);
2526
2527                 orig = data = si_pif_phy0_rreg(adev,PB0_PIF_PWRDOWN_2);
2528                 data &= ~PLL_RAMP_UP_TIME_2_MASK;
2529                 if (orig != data)
2530                     si_pif_phy0_wreg(adev,PB0_PIF_PWRDOWN_2, data);
2531
2532                 orig = data = si_pif_phy0_rreg(adev,PB0_PIF_PWRDOWN_3);
2533                 data &= ~PLL_RAMP_UP_TIME_3_MASK;
2534                 if (orig != data)
2535                     si_pif_phy0_wreg(adev,PB0_PIF_PWRDOWN_3, data);
2536
2537                 orig = data = si_pif_phy1_rreg(adev,PB1_PIF_PWRDOWN_0);
2538                 data &= ~PLL_RAMP_UP_TIME_0_MASK;
2539                 if (orig != data)
2540                     si_pif_phy1_wreg(adev,PB1_PIF_PWRDOWN_0, data);
2541
2542                 orig = data = si_pif_phy1_rreg(adev,PB1_PIF_PWRDOWN_1);
2543                 data &= ~PLL_RAMP_UP_TIME_1_MASK;
2544                 if (orig != data)
2545                     si_pif_phy1_wreg(adev,PB1_PIF_PWRDOWN_1, data);
2546
2547                 orig = data = si_pif_phy1_rreg(adev,PB1_PIF_PWRDOWN_2);
2548                 data &= ~PLL_RAMP_UP_TIME_2_MASK;
2549                 if (orig != data)
2550                     si_pif_phy1_wreg(adev,PB1_PIF_PWRDOWN_2, data);
2551
2552                 orig = data = si_pif_phy1_rreg(adev,PB1_PIF_PWRDOWN_3);
2553                 data &= ~PLL_RAMP_UP_TIME_3_MASK;
2554                 if (orig != data)
2555                     si_pif_phy1_wreg(adev,PB1_PIF_PWRDOWN_3, data);
2556             }
2557             orig = data = RREG32_PCIE_PORT(PCIE_LC_LINK_WIDTH_CNTL);
2558             data &= ~LC_DYN_LANES_PWR_STATE_MASK;
2559             data |= LC_DYN_LANES_PWR_STATE(3);
2560             if (orig != data)
2561                 WREG32_PCIE_PORT(PCIE_LC_LINK_WIDTH_CNTL, data);
2562
2563             orig = data = si_pif_phy0_rreg(adev,PB0_PIF_CNTL);
2564             data &= ~LS2_EXIT_TIME_MASK;
2565             if ((adev->asic_type == CHIP_OLAND) || (adev->asic_type == CHIP_HAINAN))
2566                 data |= LS2_EXIT_TIME(5);
2567             if (orig != data)
2568                 si_pif_phy0_wreg(adev,PB0_PIF_CNTL, data);
2569
2570             orig = data = si_pif_phy1_rreg(adev,PB1_PIF_CNTL);
2571             data &= ~LS2_EXIT_TIME_MASK;
2572             if ((adev->asic_type == CHIP_OLAND) || (adev->asic_type == CHIP_HAINAN))
2573                 data |= LS2_EXIT_TIME(5);
2574             if (orig != data)
2575                 si_pif_phy1_wreg(adev,PB1_PIF_CNTL, data);
2576
2577             if (!disable_clkreq &&
2578                 !pci_is_root_bus(adev->pdev->bus)) {
2579                 struct pci_dev *root = adev->pdev->bus->self;
2580                 u32 lnkcap;
2581
2582                 clk_req_support = false;
2583                 pcie_capability_read_dword(root, PCI_EXP_LNKCAP, &lnkcap);
2584                 if (lnkcap & PCI_EXP_LNKCAP_CLKPM)
2585                     clk_req_support = true;
2586             } else {
2587                 clk_req_support = false;
2588             }
2589
2590             if (clk_req_support) {
2591                 orig = data = RREG32_PCIE_PORT(PCIE_LC_CNTL2);
2592                 data |= LC_ALLOW_PDWN_IN_L1 | LC_ALLOW_PDWN_IN_L23;
2593                 if (orig != data)
2594                     WREG32_PCIE_PORT(PCIE_LC_CNTL2, data);
2595
2596                 orig = data = RREG32(THM_CLK_CNTL);
2597                 data &= ~(CMON_CLK_SEL_MASK | TMON_CLK_SEL_MASK);
2598                 data |= CMON_CLK_SEL(1) | TMON_CLK_SEL(1);
2599                 if (orig != data)
2600                     WREG32(THM_CLK_CNTL, data);
2601
2602                 orig = data = RREG32(MISC_CLK_CNTL);
2603                 data &= ~(DEEP_SLEEP_CLK_SEL_MASK | ZCLK_SEL_MASK);
2604                 data |= DEEP_SLEEP_CLK_SEL(1) | ZCLK_SEL(1);
2605                 if (orig != data)
2606                     WREG32(MISC_CLK_CNTL, data);
2607
2608                 orig = data = RREG32(CG_CLKPIN_CNTL);
2609                 data &= ~BCLK_AS_XCLK;
2610                 if (orig != data)
2611                     WREG32(CG_CLKPIN_CNTL, data);
2612
2613                 orig = data = RREG32(CG_CLKPIN_CNTL_2);
2614                 data &= ~FORCE_BIF_REFCLK_EN;
2615                 if (orig != data)
2616                     WREG32(CG_CLKPIN_CNTL_2, data);
2617
2618                 orig = data = RREG32(MPLL_BYPASSCLK_SEL);
2619                 data &= ~MPLL_CLKOUT_SEL_MASK;
2620                 data |= MPLL_CLKOUT_SEL(4);
2621                 if (orig != data)
2622                     WREG32(MPLL_BYPASSCLK_SEL, data);
2623
2624                 orig = data = RREG32(SPLL_CNTL_MODE);
2625                 data &= ~SPLL_REFCLK_SEL_MASK;
2626                 if (orig != data)
2627                     WREG32(SPLL_CNTL_MODE, data);
2628             }
2629         }
2630     } else {
2631         if (orig != data)
2632             WREG32_PCIE_PORT(PCIE_LC_CNTL, data);
2633     }
2634
2635     orig = data = RREG32_PCIE(PCIE_CNTL2);
2636     data |= SLV_MEM_LS_EN | MST_MEM_LS_EN | REPLAY_MEM_LS_EN;
2637     if (orig != data)
2638         WREG32_PCIE(PCIE_CNTL2, data);
2639
2640     if (!disable_l0s) {
2641         data = RREG32_PCIE_PORT(PCIE_LC_N_FTS_CNTL);
2642         if((data & LC_N_FTS_MASK) == LC_N_FTS_MASK) {
2643             data = RREG32_PCIE(PCIE_LC_STATUS1);
2644             if ((data & LC_REVERSE_XMIT) && (data & LC_REVERSE_RCVR)) {
2645                 orig = data = RREG32_PCIE_PORT(PCIE_LC_CNTL);
2646                 data &= ~LC_L0S_INACTIVITY_MASK;
2647                 if (orig != data)
2648                     WREG32_PCIE_PORT(PCIE_LC_CNTL, data);
2649             }
2650         }
2651     }
2652 }
2653
2654 static void si_fix_pci_max_read_req_size(struct amdgpu_device *adev)
2655 {
2656     int readrq;
2657     u16 v;
2658
2659     readrq = pcie_get_readrq(adev->pdev);
2660     v = ffs(readrq) - 8;
2661     if ((v == 0) || (v == 6) || (v == 7))
2662         pcie_set_readrq(adev->pdev, 512);
2663 }
2664
2665 static int si_common_hw_init(void *handle)
2666 {
2667     struct amdgpu_device *adev = (struct amdgpu_device *)handle;
2668
2669     si_fix_pci_max_read_req_size(adev);
2670     si_init_golden_registers(adev);
2671     si_pcie_gen3_enable(adev);
2672     si_program_aspm(adev);
2673
2674     return 0;
2675 }
2676
2677 static int si_common_hw_fini(void *handle)
2678 {
2679     return 0;
2680 }
2681
2682 static int si_common_suspend(void *handle)
2683 {
2684     struct amdgpu_device *adev = (struct amdgpu_device *)handle;
2685
2686     return si_common_hw_fini(adev);
2687 }
2688
2689 static int si_common_resume(void *handle)
2690 {
2691     struct amdgpu_device *adev = (struct amdgpu_device *)handle;
2692
2693     return si_common_hw_init(adev);
2694 }
2695
2696 static bool si_common_is_idle(void *handle)
2697 {
2698     return true;
2699 }
2700
2701 static int si_common_wait_for_idle(void *handle)
2702 {
2703     return 0;
2704 }
2705
2706 static int si_common_soft_reset(void *handle)
2707 {
2708     return 0;
2709 }
2710
2711 static int si_common_set_clockgating_state(void *handle,
2712                         enum amd_clockgating_state state)
2713 {
2714     return 0;
2715 }
2716
2717 static int si_common_set_powergating_state(void *handle,
2718                         enum amd_powergating_state state)
2719 {
2720     return 0;
2721 }
2722
2723 static const struct amd_ip_funcs si_common_ip_funcs = {
2724     .name = "si_common",
2725     .early_init = si_common_early_init,
2726     .late_init = NULL,
2727     .sw_init = si_common_sw_init,
2728     .sw_fini = si_common_sw_fini,
2729     .hw_init = si_common_hw_init,
2730     .hw_fini = si_common_hw_fini,
2731     .suspend = si_common_suspend,
2732     .resume = si_common_resume,
2733     .is_idle = si_common_is_idle,
2734     .wait_for_idle = si_common_wait_for_idle,
2735     .soft_reset = si_common_soft_reset,
2736     .set_clockgating_state = si_common_set_clockgating_state,
2737     .set_powergating_state = si_common_set_powergating_state,
2738 };
2739
2740 static const struct amdgpu_ip_block_version si_common_ip_block =
2741 {
2742     .type = AMD_IP_BLOCK_TYPE_COMMON,
2743     .major = 1,
2744     .minor = 0,
2745     .rev = 0,
2746     .funcs = &si_common_ip_funcs,
2747 };
2748
2749 int si_set_ip_blocks(struct amdgpu_device *adev)
2750 {
2751     switch (adev->asic_type) {
2752     case CHIP_VERDE:
2753     case CHIP_TAHITI:
2754     case CHIP_PITCAIRN:
2755         amdgpu_device_ip_block_add(adev, &si_common_ip_block);
2756         amdgpu_device_ip_block_add(adev, &gmc_v6_0_ip_block);
2757         amdgpu_device_ip_block_add(adev, &si_ih_ip_block);
2758         amdgpu_device_ip_block_add(adev, &gfx_v6_0_ip_block);
2759         amdgpu_device_ip_block_add(adev, &si_dma_ip_block);
2760         amdgpu_device_ip_block_add(adev, &si_smu_ip_block);
2761         if (adev->enable_virtual_display)
2762             amdgpu_device_ip_block_add(adev, &amdgpu_vkms_ip_block);
2763 #if defined(CONFIG_DRM_AMD_DC) && defined(CONFIG_DRM_AMD_DC_SI)
2764         else if (amdgpu_device_has_dc_support(adev))
2765             amdgpu_device_ip_block_add(adev, &dm_ip_block);
2766 #endif
2767         else
2768             amdgpu_device_ip_block_add(adev, &dce_v6_0_ip_block);
2769         amdgpu_device_ip_block_add(adev, &uvd_v3_1_ip_block);
2770         /* amdgpu_device_ip_block_add(adev, &vce_v1_0_ip_block); */
2771         break;
2772     case CHIP_OLAND:
2773         amdgpu_device_ip_block_add(adev, &si_common_ip_block);
2774         amdgpu_device_ip_block_add(adev, &gmc_v6_0_ip_block);
2775         amdgpu_device_ip_block_add(adev, &si_ih_ip_block);
2776         amdgpu_device_ip_block_add(adev, &gfx_v6_0_ip_block);
2777         amdgpu_device_ip_block_add(adev, &si_dma_ip_block);
2778         amdgpu_device_ip_block_add(adev, &si_smu_ip_block);
2779         if (adev->enable_virtual_display)
2780             amdgpu_device_ip_block_add(adev, &amdgpu_vkms_ip_block);
2781 #if defined(CONFIG_DRM_AMD_DC) && defined(CONFIG_DRM_AMD_DC_SI)
2782         else if (amdgpu_device_has_dc_support(adev))
2783             amdgpu_device_ip_block_add(adev, &dm_ip_block);
2784 #endif
2785         else
2786             amdgpu_device_ip_block_add(adev, &dce_v6_4_ip_block);
2787         amdgpu_device_ip_block_add(adev, &uvd_v3_1_ip_block);
2788         /* amdgpu_device_ip_block_add(adev, &vce_v1_0_ip_block); */
2789         break;
2790     case CHIP_HAINAN:
2791         amdgpu_device_ip_block_add(adev, &si_common_ip_block);
2792         amdgpu_device_ip_block_add(adev, &gmc_v6_0_ip_block);
2793         amdgpu_device_ip_block_add(adev, &si_ih_ip_block);
2794         amdgpu_device_ip_block_add(adev, &gfx_v6_0_ip_block);
2795         amdgpu_device_ip_block_add(adev, &si_dma_ip_block);
2796         amdgpu_device_ip_block_add(adev, &si_smu_ip_block);
2797         if (adev->enable_virtual_display)
2798             amdgpu_device_ip_block_add(adev, &amdgpu_vkms_ip_block);
2799         break;
2800     default:
2801         BUG();
2802     }
2803     return 0;
2804 }
2805