amd/amdgpu/cik.c

0001 /*
0002  * Copyright 2012 Advanced Micro Devices, Inc.
0003  *
0004  * Permission is hereby granted, free of charge, to any person obtaining a
0005  * copy of this software and associated documentation files (the "Software"),
0006  * to deal in the Software without restriction, including without limitation
0007  * the rights to use, copy, modify, merge, publish, distribute, sublicense,
0008  * and/or sell copies of the Software, and to permit persons to whom the
0009  * Software is furnished to do so, subject to the following conditions:
0010  *
0011  * The above copyright notice and this permission notice shall be included in
0012  * all copies or substantial portions of the Software.
0013  *
0014  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
0015  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
0016  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
0017  * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
0018  * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
0019  * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
0020  * OTHER DEALINGS IN THE SOFTWARE.
0021  *
0022  * Authors: Alex Deucher
0023  */
0024 #include <linux/firmware.h>
0025 #include <linux/slab.h>
0026 #include <linux/module.h>
0027 #include <linux/pci.h>
0028
0029 #include <drm/amdgpu_drm.h>
0030
0031 #include "amdgpu.h"
0032 #include "amdgpu_atombios.h"
0033 #include "amdgpu_ih.h"
0034 #include "amdgpu_uvd.h"
0035 #include "amdgpu_vce.h"
0036 #include "cikd.h"
0037 #include "atom.h"
0038 #include "amd_pcie.h"
0039
0040 #include "cik.h"
0041 #include "gmc_v7_0.h"
0042 #include "cik_ih.h"
0043 #include "dce_v8_0.h"
0044 #include "gfx_v7_0.h"
0045 #include "cik_sdma.h"
0046 #include "uvd_v4_2.h"
0047 #include "vce_v2_0.h"
0048 #include "cik_dpm.h"
0049
0050 #include "uvd/uvd_4_2_d.h"
0051
0052 #include "smu/smu_7_0_1_d.h"
0053 #include "smu/smu_7_0_1_sh_mask.h"
0054
0055 #include "dce/dce_8_0_d.h"
0056 #include "dce/dce_8_0_sh_mask.h"
0057
0058 #include "bif/bif_4_1_d.h"
0059 #include "bif/bif_4_1_sh_mask.h"
0060
0061 #include "gca/gfx_7_2_d.h"
0062 #include "gca/gfx_7_2_enum.h"
0063 #include "gca/gfx_7_2_sh_mask.h"
0064
0065 #include "gmc/gmc_7_1_d.h"
0066 #include "gmc/gmc_7_1_sh_mask.h"
0067
0068 #include "oss/oss_2_0_d.h"
0069 #include "oss/oss_2_0_sh_mask.h"
0070
0071 #include "amdgpu_dm.h"
0072 #include "amdgpu_amdkfd.h"
0073 #include "amdgpu_vkms.h"
0074
0075 static const struct amdgpu_video_codec_info cik_video_codecs_encode_array[] =
0076 {
0077     {
0078         .codec_type = AMDGPU_INFO_VIDEO_CAPS_CODEC_IDX_MPEG4_AVC,
0079         .max_width = 2048,
0080         .max_height = 1152,
0081         .max_pixels_per_frame = 2048 * 1152,
0082         .max_level = 0,
0083     },
0084 };
0085
0086 static const struct amdgpu_video_codecs cik_video_codecs_encode =
0087 {
0088     .codec_count = ARRAY_SIZE(cik_video_codecs_encode_array),
0089     .codec_array = cik_video_codecs_encode_array,
0090 };
0091
0092 static const struct amdgpu_video_codec_info cik_video_codecs_decode_array[] =
0093 {
0094     {
0095         .codec_type = AMDGPU_INFO_VIDEO_CAPS_CODEC_IDX_MPEG2,
0096         .max_width = 2048,
0097         .max_height = 1152,
0098         .max_pixels_per_frame = 2048 * 1152,
0099         .max_level = 3,
0100     },
0101     {
0102         .codec_type = AMDGPU_INFO_VIDEO_CAPS_CODEC_IDX_MPEG4,
0103         .max_width = 2048,
0104         .max_height = 1152,
0105         .max_pixels_per_frame = 2048 * 1152,
0106         .max_level = 5,
0107     },
0108     {
0109         .codec_type = AMDGPU_INFO_VIDEO_CAPS_CODEC_IDX_MPEG4_AVC,
0110         .max_width = 2048,
0111         .max_height = 1152,
0112         .max_pixels_per_frame = 2048 * 1152,
0113         .max_level = 41,
0114     },
0115     {
0116         .codec_type = AMDGPU_INFO_VIDEO_CAPS_CODEC_IDX_VC1,
0117         .max_width = 2048,
0118         .max_height = 1152,
0119         .max_pixels_per_frame = 2048 * 1152,
0120         .max_level = 4,
0121     },
0122 };
0123
0124 static const struct amdgpu_video_codecs cik_video_codecs_decode =
0125 {
0126     .codec_count = ARRAY_SIZE(cik_video_codecs_decode_array),
0127     .codec_array = cik_video_codecs_decode_array,
0128 };
0129
0130 static int cik_query_video_codecs(struct amdgpu_device *adev, bool encode,
0131                   const struct amdgpu_video_codecs **codecs)
0132 {
0133     switch (adev->asic_type) {
0134     case CHIP_BONAIRE:
0135     case CHIP_HAWAII:
0136     case CHIP_KAVERI:
0137     case CHIP_KABINI:
0138     case CHIP_MULLINS:
0139         if (encode)
0140             *codecs = &cik_video_codecs_encode;
0141         else
0142             *codecs = &cik_video_codecs_decode;
0143         return 0;
0144     default:
0145         return -EINVAL;
0146     }
0147 }
0148
0149 /*
0150  * Indirect registers accessor
0151  */
0152 static u32 cik_pcie_rreg(struct amdgpu_device *adev, u32 reg)
0153 {
0154     unsigned long flags;
0155     u32 r;
0156
0157     spin_lock_irqsave(&adev->pcie_idx_lock, flags);
0158     WREG32(mmPCIE_INDEX, reg);
0159     (void)RREG32(mmPCIE_INDEX);
0160     r = RREG32(mmPCIE_DATA);
0161     spin_unlock_irqrestore(&adev->pcie_idx_lock, flags);
0162     return r;
0163 }
0164
0165 static void cik_pcie_wreg(struct amdgpu_device *adev, u32 reg, u32 v)
0166 {
0167     unsigned long flags;
0168
0169     spin_lock_irqsave(&adev->pcie_idx_lock, flags);
0170     WREG32(mmPCIE_INDEX, reg);
0171     (void)RREG32(mmPCIE_INDEX);
0172     WREG32(mmPCIE_DATA, v);
0173     (void)RREG32(mmPCIE_DATA);
0174     spin_unlock_irqrestore(&adev->pcie_idx_lock, flags);
0175 }
0176
0177 static u32 cik_smc_rreg(struct amdgpu_device *adev, u32 reg)
0178 {
0179     unsigned long flags;
0180     u32 r;
0181
0182     spin_lock_irqsave(&adev->smc_idx_lock, flags);
0183     WREG32(mmSMC_IND_INDEX_0, (reg));
0184     r = RREG32(mmSMC_IND_DATA_0);
0185     spin_unlock_irqrestore(&adev->smc_idx_lock, flags);
0186     return r;
0187 }
0188
0189 static void cik_smc_wreg(struct amdgpu_device *adev, u32 reg, u32 v)
0190 {
0191     unsigned long flags;
0192
0193     spin_lock_irqsave(&adev->smc_idx_lock, flags);
0194     WREG32(mmSMC_IND_INDEX_0, (reg));
0195     WREG32(mmSMC_IND_DATA_0, (v));
0196     spin_unlock_irqrestore(&adev->smc_idx_lock, flags);
0197 }
0198
0199 static u32 cik_uvd_ctx_rreg(struct amdgpu_device *adev, u32 reg)
0200 {
0201     unsigned long flags;
0202     u32 r;
0203
0204     spin_lock_irqsave(&adev->uvd_ctx_idx_lock, flags);
0205     WREG32(mmUVD_CTX_INDEX, ((reg) & 0x1ff));
0206     r = RREG32(mmUVD_CTX_DATA);
0207     spin_unlock_irqrestore(&adev->uvd_ctx_idx_lock, flags);
0208     return r;
0209 }
0210
0211 static void cik_uvd_ctx_wreg(struct amdgpu_device *adev, u32 reg, u32 v)
0212 {
0213     unsigned long flags;
0214
0215     spin_lock_irqsave(&adev->uvd_ctx_idx_lock, flags);
0216     WREG32(mmUVD_CTX_INDEX, ((reg) & 0x1ff));
0217     WREG32(mmUVD_CTX_DATA, (v));
0218     spin_unlock_irqrestore(&adev->uvd_ctx_idx_lock, flags);
0219 }
0220
0221 static u32 cik_didt_rreg(struct amdgpu_device *adev, u32 reg)
0222 {
0223     unsigned long flags;
0224     u32 r;
0225
0226     spin_lock_irqsave(&adev->didt_idx_lock, flags);
0227     WREG32(mmDIDT_IND_INDEX, (reg));
0228     r = RREG32(mmDIDT_IND_DATA);
0229     spin_unlock_irqrestore(&adev->didt_idx_lock, flags);
0230     return r;
0231 }
0232
0233 static void cik_didt_wreg(struct amdgpu_device *adev, u32 reg, u32 v)
0234 {
0235     unsigned long flags;
0236
0237     spin_lock_irqsave(&adev->didt_idx_lock, flags);
0238     WREG32(mmDIDT_IND_INDEX, (reg));
0239     WREG32(mmDIDT_IND_DATA, (v));
0240     spin_unlock_irqrestore(&adev->didt_idx_lock, flags);
0241 }
0242
0243 static const u32 bonaire_golden_spm_registers[] =
0244 {
0245     0xc200, 0xe0ffffff, 0xe0000000
0246 };
0247
0248 static const u32 bonaire_golden_common_registers[] =
0249 {
0250     0x31dc, 0xffffffff, 0x00000800,
0251     0x31dd, 0xffffffff, 0x00000800,
0252     0x31e6, 0xffffffff, 0x00007fbf,
0253     0x31e7, 0xffffffff, 0x00007faf
0254 };
0255
0256 static const u32 bonaire_golden_registers[] =
0257 {
0258     0xcd5, 0x00000333, 0x00000333,
0259     0xcd4, 0x000c0fc0, 0x00040200,
0260     0x2684, 0x00010000, 0x00058208,
0261     0xf000, 0xffff1fff, 0x00140000,
0262     0xf080, 0xfdfc0fff, 0x00000100,
0263     0xf08d, 0x40000000, 0x40000200,
0264     0x260c, 0xffffffff, 0x00000000,
0265     0x260d, 0xf00fffff, 0x00000400,
0266     0x260e, 0x0002021c, 0x00020200,
0267     0x31e, 0x00000080, 0x00000000,
0268     0x16ec, 0x000000f0, 0x00000070,
0269     0x16f0, 0xf0311fff, 0x80300000,
0270     0x263e, 0x73773777, 0x12010001,
0271     0xd43, 0x00810000, 0x408af000,
0272     0x1c0c, 0x31000111, 0x00000011,
0273     0xbd2, 0x73773777, 0x12010001,
0274     0x883, 0x00007fb6, 0x0021a1b1,
0275     0x884, 0x00007fb6, 0x002021b1,
0276     0x860, 0x00007fb6, 0x00002191,
0277     0x886, 0x00007fb6, 0x002121b1,
0278     0x887, 0x00007fb6, 0x002021b1,
0279     0x877, 0x00007fb6, 0x00002191,
0280     0x878, 0x00007fb6, 0x00002191,
0281     0xd8a, 0x0000003f, 0x0000000a,
0282     0xd8b, 0x0000003f, 0x0000000a,
0283     0xab9, 0x00073ffe, 0x000022a2,
0284     0x903, 0x000007ff, 0x00000000,
0285     0x2285, 0xf000003f, 0x00000007,
0286     0x22fc, 0x00002001, 0x00000001,
0287     0x22c9, 0xffffffff, 0x00ffffff,
0288     0xc281, 0x0000ff0f, 0x00000000,
0289     0xa293, 0x07ffffff, 0x06000000,
0290     0x136, 0x00000fff, 0x00000100,
0291     0xf9e, 0x00000001, 0x00000002,
0292     0x2440, 0x03000000, 0x0362c688,
0293     0x2300, 0x000000ff, 0x00000001,
0294     0x390, 0x00001fff, 0x00001fff,
0295     0x2418, 0x0000007f, 0x00000020,
0296     0x2542, 0x00010000, 0x00010000,
0297     0x2b05, 0x000003ff, 0x000000f3,
0298     0x2b03, 0xffffffff, 0x00001032
0299 };
0300
0301 static const u32 bonaire_mgcg_cgcg_init[] =
0302 {
0303     0x3108, 0xffffffff, 0xfffffffc,
0304     0xc200, 0xffffffff, 0xe0000000,
0305     0xf0a8, 0xffffffff, 0x00000100,
0306     0xf082, 0xffffffff, 0x00000100,
0307     0xf0b0, 0xffffffff, 0xc0000100,
0308     0xf0b2, 0xffffffff, 0xc0000100,
0309     0xf0b1, 0xffffffff, 0xc0000100,
0310     0x1579, 0xffffffff, 0x00600100,
0311     0xf0a0, 0xffffffff, 0x00000100,
0312     0xf085, 0xffffffff, 0x06000100,
0313     0xf088, 0xffffffff, 0x00000100,
0314     0xf086, 0xffffffff, 0x06000100,
0315     0xf081, 0xffffffff, 0x00000100,
0316     0xf0b8, 0xffffffff, 0x00000100,
0317     0xf089, 0xffffffff, 0x00000100,
0318     0xf080, 0xffffffff, 0x00000100,
0319     0xf08c, 0xffffffff, 0x00000100,
0320     0xf08d, 0xffffffff, 0x00000100,
0321     0xf094, 0xffffffff, 0x00000100,
0322     0xf095, 0xffffffff, 0x00000100,
0323     0xf096, 0xffffffff, 0x00000100,
0324     0xf097, 0xffffffff, 0x00000100,
0325     0xf098, 0xffffffff, 0x00000100,
0326     0xf09f, 0xffffffff, 0x00000100,
0327     0xf09e, 0xffffffff, 0x00000100,
0328     0xf084, 0xffffffff, 0x06000100,
0329     0xf0a4, 0xffffffff, 0x00000100,
0330     0xf09d, 0xffffffff, 0x00000100,
0331     0xf0ad, 0xffffffff, 0x00000100,
0332     0xf0ac, 0xffffffff, 0x00000100,
0333     0xf09c, 0xffffffff, 0x00000100,
0334     0xc200, 0xffffffff, 0xe0000000,
0335     0xf008, 0xffffffff, 0x00010000,
0336     0xf009, 0xffffffff, 0x00030002,
0337     0xf00a, 0xffffffff, 0x00040007,
0338     0xf00b, 0xffffffff, 0x00060005,
0339     0xf00c, 0xffffffff, 0x00090008,
0340     0xf00d, 0xffffffff, 0x00010000,
0341     0xf00e, 0xffffffff, 0x00030002,
0342     0xf00f, 0xffffffff, 0x00040007,
0343     0xf010, 0xffffffff, 0x00060005,
0344     0xf011, 0xffffffff, 0x00090008,
0345     0xf012, 0xffffffff, 0x00010000,
0346     0xf013, 0xffffffff, 0x00030002,
0347     0xf014, 0xffffffff, 0x00040007,
0348     0xf015, 0xffffffff, 0x00060005,
0349     0xf016, 0xffffffff, 0x00090008,
0350     0xf017, 0xffffffff, 0x00010000,
0351     0xf018, 0xffffffff, 0x00030002,
0352     0xf019, 0xffffffff, 0x00040007,
0353     0xf01a, 0xffffffff, 0x00060005,
0354     0xf01b, 0xffffffff, 0x00090008,
0355     0xf01c, 0xffffffff, 0x00010000,
0356     0xf01d, 0xffffffff, 0x00030002,
0357     0xf01e, 0xffffffff, 0x00040007,
0358     0xf01f, 0xffffffff, 0x00060005,
0359     0xf020, 0xffffffff, 0x00090008,
0360     0xf021, 0xffffffff, 0x00010000,
0361     0xf022, 0xffffffff, 0x00030002,
0362     0xf023, 0xffffffff, 0x00040007,
0363     0xf024, 0xffffffff, 0x00060005,
0364     0xf025, 0xffffffff, 0x00090008,
0365     0xf026, 0xffffffff, 0x00010000,
0366     0xf027, 0xffffffff, 0x00030002,
0367     0xf028, 0xffffffff, 0x00040007,
0368     0xf029, 0xffffffff, 0x00060005,
0369     0xf02a, 0xffffffff, 0x00090008,
0370     0xf000, 0xffffffff, 0x96e00200,
0371     0x21c2, 0xffffffff, 0x00900100,
0372     0x3109, 0xffffffff, 0x0020003f,
0373     0xe, 0xffffffff, 0x0140001c,
0374     0xf, 0x000f0000, 0x000f0000,
0375     0x88, 0xffffffff, 0xc060000c,
0376     0x89, 0xc0000fff, 0x00000100,
0377     0x3e4, 0xffffffff, 0x00000100,
0378     0x3e6, 0x00000101, 0x00000000,
0379     0x82a, 0xffffffff, 0x00000104,
0380     0x1579, 0xff000fff, 0x00000100,
0381     0xc33, 0xc0000fff, 0x00000104,
0382     0x3079, 0x00000001, 0x00000001,
0383     0x3403, 0xff000ff0, 0x00000100,
0384     0x3603, 0xff000ff0, 0x00000100
0385 };
0386
0387 static const u32 spectre_golden_spm_registers[] =
0388 {
0389     0xc200, 0xe0ffffff, 0xe0000000
0390 };
0391
0392 static const u32 spectre_golden_common_registers[] =
0393 {
0394     0x31dc, 0xffffffff, 0x00000800,
0395     0x31dd, 0xffffffff, 0x00000800,
0396     0x31e6, 0xffffffff, 0x00007fbf,
0397     0x31e7, 0xffffffff, 0x00007faf
0398 };
0399
0400 static const u32 spectre_golden_registers[] =
0401 {
0402     0xf000, 0xffff1fff, 0x96940200,
0403     0xf003, 0xffff0001, 0xff000000,
0404     0xf080, 0xfffc0fff, 0x00000100,
0405     0x1bb6, 0x00010101, 0x00010000,
0406     0x260d, 0xf00fffff, 0x00000400,
0407     0x260e, 0xfffffffc, 0x00020200,
0408     0x16ec, 0x000000f0, 0x00000070,
0409     0x16f0, 0xf0311fff, 0x80300000,
0410     0x263e, 0x73773777, 0x12010001,
0411     0x26df, 0x00ff0000, 0x00fc0000,
0412     0xbd2, 0x73773777, 0x12010001,
0413     0x2285, 0xf000003f, 0x00000007,
0414     0x22c9, 0xffffffff, 0x00ffffff,
0415     0xa0d4, 0x3f3f3fff, 0x00000082,
0416     0xa0d5, 0x0000003f, 0x00000000,
0417     0xf9e, 0x00000001, 0x00000002,
0418     0x244f, 0xffff03df, 0x00000004,
0419     0x31da, 0x00000008, 0x00000008,
0420     0x2300, 0x000008ff, 0x00000800,
0421     0x2542, 0x00010000, 0x00010000,
0422     0x2b03, 0xffffffff, 0x54763210,
0423     0x853e, 0x01ff01ff, 0x00000002,
0424     0x8526, 0x007ff800, 0x00200000,
0425     0x8057, 0xffffffff, 0x00000f40,
0426     0xc24d, 0xffffffff, 0x00000001
0427 };
0428
0429 static const u32 spectre_mgcg_cgcg_init[] =
0430 {
0431     0x3108, 0xffffffff, 0xfffffffc,
0432     0xc200, 0xffffffff, 0xe0000000,
0433     0xf0a8, 0xffffffff, 0x00000100,
0434     0xf082, 0xffffffff, 0x00000100,
0435     0xf0b0, 0xffffffff, 0x00000100,
0436     0xf0b2, 0xffffffff, 0x00000100,
0437     0xf0b1, 0xffffffff, 0x00000100,
0438     0x1579, 0xffffffff, 0x00600100,
0439     0xf0a0, 0xffffffff, 0x00000100,
0440     0xf085, 0xffffffff, 0x06000100,
0441     0xf088, 0xffffffff, 0x00000100,
0442     0xf086, 0xffffffff, 0x06000100,
0443     0xf081, 0xffffffff, 0x00000100,
0444     0xf0b8, 0xffffffff, 0x00000100,
0445     0xf089, 0xffffffff, 0x00000100,
0446     0xf080, 0xffffffff, 0x00000100,
0447     0xf08c, 0xffffffff, 0x00000100,
0448     0xf08d, 0xffffffff, 0x00000100,
0449     0xf094, 0xffffffff, 0x00000100,
0450     0xf095, 0xffffffff, 0x00000100,
0451     0xf096, 0xffffffff, 0x00000100,
0452     0xf097, 0xffffffff, 0x00000100,
0453     0xf098, 0xffffffff, 0x00000100,
0454     0xf09f, 0xffffffff, 0x00000100,
0455     0xf09e, 0xffffffff, 0x00000100,
0456     0xf084, 0xffffffff, 0x06000100,
0457     0xf0a4, 0xffffffff, 0x00000100,
0458     0xf09d, 0xffffffff, 0x00000100,
0459     0xf0ad, 0xffffffff, 0x00000100,
0460     0xf0ac, 0xffffffff, 0x00000100,
0461     0xf09c, 0xffffffff, 0x00000100,
0462     0xc200, 0xffffffff, 0xe0000000,
0463     0xf008, 0xffffffff, 0x00010000,
0464     0xf009, 0xffffffff, 0x00030002,
0465     0xf00a, 0xffffffff, 0x00040007,
0466     0xf00b, 0xffffffff, 0x00060005,
0467     0xf00c, 0xffffffff, 0x00090008,
0468     0xf00d, 0xffffffff, 0x00010000,
0469     0xf00e, 0xffffffff, 0x00030002,
0470     0xf00f, 0xffffffff, 0x00040007,
0471     0xf010, 0xffffffff, 0x00060005,
0472     0xf011, 0xffffffff, 0x00090008,
0473     0xf012, 0xffffffff, 0x00010000,
0474     0xf013, 0xffffffff, 0x00030002,
0475     0xf014, 0xffffffff, 0x00040007,
0476     0xf015, 0xffffffff, 0x00060005,
0477     0xf016, 0xffffffff, 0x00090008,
0478     0xf017, 0xffffffff, 0x00010000,
0479     0xf018, 0xffffffff, 0x00030002,
0480     0xf019, 0xffffffff, 0x00040007,
0481     0xf01a, 0xffffffff, 0x00060005,
0482     0xf01b, 0xffffffff, 0x00090008,
0483     0xf01c, 0xffffffff, 0x00010000,
0484     0xf01d, 0xffffffff, 0x00030002,
0485     0xf01e, 0xffffffff, 0x00040007,
0486     0xf01f, 0xffffffff, 0x00060005,
0487     0xf020, 0xffffffff, 0x00090008,
0488     0xf021, 0xffffffff, 0x00010000,
0489     0xf022, 0xffffffff, 0x00030002,
0490     0xf023, 0xffffffff, 0x00040007,
0491     0xf024, 0xffffffff, 0x00060005,
0492     0xf025, 0xffffffff, 0x00090008,
0493     0xf026, 0xffffffff, 0x00010000,
0494     0xf027, 0xffffffff, 0x00030002,
0495     0xf028, 0xffffffff, 0x00040007,
0496     0xf029, 0xffffffff, 0x00060005,
0497     0xf02a, 0xffffffff, 0x00090008,
0498     0xf02b, 0xffffffff, 0x00010000,
0499     0xf02c, 0xffffffff, 0x00030002,
0500     0xf02d, 0xffffffff, 0x00040007,
0501     0xf02e, 0xffffffff, 0x00060005,
0502     0xf02f, 0xffffffff, 0x00090008,
0503     0xf000, 0xffffffff, 0x96e00200,
0504     0x21c2, 0xffffffff, 0x00900100,
0505     0x3109, 0xffffffff, 0x0020003f,
0506     0xe, 0xffffffff, 0x0140001c,
0507     0xf, 0x000f0000, 0x000f0000,
0508     0x88, 0xffffffff, 0xc060000c,
0509     0x89, 0xc0000fff, 0x00000100,
0510     0x3e4, 0xffffffff, 0x00000100,
0511     0x3e6, 0x00000101, 0x00000000,
0512     0x82a, 0xffffffff, 0x00000104,
0513     0x1579, 0xff000fff, 0x00000100,
0514     0xc33, 0xc0000fff, 0x00000104,
0515     0x3079, 0x00000001, 0x00000001,
0516     0x3403, 0xff000ff0, 0x00000100,
0517     0x3603, 0xff000ff0, 0x00000100
0518 };
0519
0520 static const u32 kalindi_golden_spm_registers[] =
0521 {
0522     0xc200, 0xe0ffffff, 0xe0000000
0523 };
0524
0525 static const u32 kalindi_golden_common_registers[] =
0526 {
0527     0x31dc, 0xffffffff, 0x00000800,
0528     0x31dd, 0xffffffff, 0x00000800,
0529     0x31e6, 0xffffffff, 0x00007fbf,
0530     0x31e7, 0xffffffff, 0x00007faf
0531 };
0532
0533 static const u32 kalindi_golden_registers[] =
0534 {
0535     0xf000, 0xffffdfff, 0x6e944040,
0536     0x1579, 0xff607fff, 0xfc000100,
0537     0xf088, 0xff000fff, 0x00000100,
0538     0xf089, 0xff000fff, 0x00000100,
0539     0xf080, 0xfffc0fff, 0x00000100,
0540     0x1bb6, 0x00010101, 0x00010000,
0541     0x260c, 0xffffffff, 0x00000000,
0542     0x260d, 0xf00fffff, 0x00000400,
0543     0x16ec, 0x000000f0, 0x00000070,
0544     0x16f0, 0xf0311fff, 0x80300000,
0545     0x263e, 0x73773777, 0x12010001,
0546     0x263f, 0xffffffff, 0x00000010,
0547     0x26df, 0x00ff0000, 0x00fc0000,
0548     0x200c, 0x00001f0f, 0x0000100a,
0549     0xbd2, 0x73773777, 0x12010001,
0550     0x902, 0x000fffff, 0x000c007f,
0551     0x2285, 0xf000003f, 0x00000007,
0552     0x22c9, 0x3fff3fff, 0x00ffcfff,
0553     0xc281, 0x0000ff0f, 0x00000000,
0554     0xa293, 0x07ffffff, 0x06000000,
0555     0x136, 0x00000fff, 0x00000100,
0556     0xf9e, 0x00000001, 0x00000002,
0557     0x31da, 0x00000008, 0x00000008,
0558     0x2300, 0x000000ff, 0x00000003,
0559     0x853e, 0x01ff01ff, 0x00000002,
0560     0x8526, 0x007ff800, 0x00200000,
0561     0x8057, 0xffffffff, 0x00000f40,
0562     0x2231, 0x001f3ae3, 0x00000082,
0563     0x2235, 0x0000001f, 0x00000010,
0564     0xc24d, 0xffffffff, 0x00000000
0565 };
0566
0567 static const u32 kalindi_mgcg_cgcg_init[] =
0568 {
0569     0x3108, 0xffffffff, 0xfffffffc,
0570     0xc200, 0xffffffff, 0xe0000000,
0571     0xf0a8, 0xffffffff, 0x00000100,
0572     0xf082, 0xffffffff, 0x00000100,
0573     0xf0b0, 0xffffffff, 0x00000100,
0574     0xf0b2, 0xffffffff, 0x00000100,
0575     0xf0b1, 0xffffffff, 0x00000100,
0576     0x1579, 0xffffffff, 0x00600100,
0577     0xf0a0, 0xffffffff, 0x00000100,
0578     0xf085, 0xffffffff, 0x06000100,
0579     0xf088, 0xffffffff, 0x00000100,
0580     0xf086, 0xffffffff, 0x06000100,
0581     0xf081, 0xffffffff, 0x00000100,
0582     0xf0b8, 0xffffffff, 0x00000100,
0583     0xf089, 0xffffffff, 0x00000100,
0584     0xf080, 0xffffffff, 0x00000100,
0585     0xf08c, 0xffffffff, 0x00000100,
0586     0xf08d, 0xffffffff, 0x00000100,
0587     0xf094, 0xffffffff, 0x00000100,
0588     0xf095, 0xffffffff, 0x00000100,
0589     0xf096, 0xffffffff, 0x00000100,
0590     0xf097, 0xffffffff, 0x00000100,
0591     0xf098, 0xffffffff, 0x00000100,
0592     0xf09f, 0xffffffff, 0x00000100,
0593     0xf09e, 0xffffffff, 0x00000100,
0594     0xf084, 0xffffffff, 0x06000100,
0595     0xf0a4, 0xffffffff, 0x00000100,
0596     0xf09d, 0xffffffff, 0x00000100,
0597     0xf0ad, 0xffffffff, 0x00000100,
0598     0xf0ac, 0xffffffff, 0x00000100,
0599     0xf09c, 0xffffffff, 0x00000100,
0600     0xc200, 0xffffffff, 0xe0000000,
0601     0xf008, 0xffffffff, 0x00010000,
0602     0xf009, 0xffffffff, 0x00030002,
0603     0xf00a, 0xffffffff, 0x00040007,
0604     0xf00b, 0xffffffff, 0x00060005,
0605     0xf00c, 0xffffffff, 0x00090008,
0606     0xf00d, 0xffffffff, 0x00010000,
0607     0xf00e, 0xffffffff, 0x00030002,
0608     0xf00f, 0xffffffff, 0x00040007,
0609     0xf010, 0xffffffff, 0x00060005,
0610     0xf011, 0xffffffff, 0x00090008,
0611     0xf000, 0xffffffff, 0x96e00200,
0612     0x21c2, 0xffffffff, 0x00900100,
0613     0x3109, 0xffffffff, 0x0020003f,
0614     0xe, 0xffffffff, 0x0140001c,
0615     0xf, 0x000f0000, 0x000f0000,
0616     0x88, 0xffffffff, 0xc060000c,
0617     0x89, 0xc0000fff, 0x00000100,
0618     0x82a, 0xffffffff, 0x00000104,
0619     0x1579, 0xff000fff, 0x00000100,
0620     0xc33, 0xc0000fff, 0x00000104,
0621     0x3079, 0x00000001, 0x00000001,
0622     0x3403, 0xff000ff0, 0x00000100,
0623     0x3603, 0xff000ff0, 0x00000100
0624 };
0625
0626 static const u32 hawaii_golden_spm_registers[] =
0627 {
0628     0xc200, 0xe0ffffff, 0xe0000000
0629 };
0630
0631 static const u32 hawaii_golden_common_registers[] =
0632 {
0633     0xc200, 0xffffffff, 0xe0000000,
0634     0xa0d4, 0xffffffff, 0x3a00161a,
0635     0xa0d5, 0xffffffff, 0x0000002e,
0636     0x2684, 0xffffffff, 0x00018208,
0637     0x263e, 0xffffffff, 0x12011003
0638 };
0639
0640 static const u32 hawaii_golden_registers[] =
0641 {
0642     0xcd5, 0x00000333, 0x00000333,
0643     0x2684, 0x00010000, 0x00058208,
0644     0x260c, 0xffffffff, 0x00000000,
0645     0x260d, 0xf00fffff, 0x00000400,
0646     0x260e, 0x0002021c, 0x00020200,
0647     0x31e, 0x00000080, 0x00000000,
0648     0x16ec, 0x000000f0, 0x00000070,
0649     0x16f0, 0xf0311fff, 0x80300000,
0650     0xd43, 0x00810000, 0x408af000,
0651     0x1c0c, 0x31000111, 0x00000011,
0652     0xbd2, 0x73773777, 0x12010001,
0653     0x848, 0x0000007f, 0x0000001b,
0654     0x877, 0x00007fb6, 0x00002191,
0655     0xd8a, 0x0000003f, 0x0000000a,
0656     0xd8b, 0x0000003f, 0x0000000a,
0657     0xab9, 0x00073ffe, 0x000022a2,
0658     0x903, 0x000007ff, 0x00000000,
0659     0x22fc, 0x00002001, 0x00000001,
0660     0x22c9, 0xffffffff, 0x00ffffff,
0661     0xc281, 0x0000ff0f, 0x00000000,
0662     0xa293, 0x07ffffff, 0x06000000,
0663     0xf9e, 0x00000001, 0x00000002,
0664     0x31da, 0x00000008, 0x00000008,
0665     0x31dc, 0x00000f00, 0x00000800,
0666     0x31dd, 0x00000f00, 0x00000800,
0667     0x31e6, 0x00ffffff, 0x00ff7fbf,
0668     0x31e7, 0x00ffffff, 0x00ff7faf,
0669     0x2300, 0x000000ff, 0x00000800,
0670     0x390, 0x00001fff, 0x00001fff,
0671     0x2418, 0x0000007f, 0x00000020,
0672     0x2542, 0x00010000, 0x00010000,
0673     0x2b80, 0x00100000, 0x000ff07c,
0674     0x2b05, 0x000003ff, 0x0000000f,
0675     0x2b04, 0xffffffff, 0x7564fdec,
0676     0x2b03, 0xffffffff, 0x3120b9a8,
0677     0x2b02, 0x20000000, 0x0f9c0000
0678 };
0679
0680 static const u32 hawaii_mgcg_cgcg_init[] =
0681 {
0682     0x3108, 0xffffffff, 0xfffffffd,
0683     0xc200, 0xffffffff, 0xe0000000,
0684     0xf0a8, 0xffffffff, 0x00000100,
0685     0xf082, 0xffffffff, 0x00000100,
0686     0xf0b0, 0xffffffff, 0x00000100,
0687     0xf0b2, 0xffffffff, 0x00000100,
0688     0xf0b1, 0xffffffff, 0x00000100,
0689     0x1579, 0xffffffff, 0x00200100,
0690     0xf0a0, 0xffffffff, 0x00000100,
0691     0xf085, 0xffffffff, 0x06000100,
0692     0xf088, 0xffffffff, 0x00000100,
0693     0xf086, 0xffffffff, 0x06000100,
0694     0xf081, 0xffffffff, 0x00000100,
0695     0xf0b8, 0xffffffff, 0x00000100,
0696     0xf089, 0xffffffff, 0x00000100,
0697     0xf080, 0xffffffff, 0x00000100,
0698     0xf08c, 0xffffffff, 0x00000100,
0699     0xf08d, 0xffffffff, 0x00000100,
0700     0xf094, 0xffffffff, 0x00000100,
0701     0xf095, 0xffffffff, 0x00000100,
0702     0xf096, 0xffffffff, 0x00000100,
0703     0xf097, 0xffffffff, 0x00000100,
0704     0xf098, 0xffffffff, 0x00000100,
0705     0xf09f, 0xffffffff, 0x00000100,
0706     0xf09e, 0xffffffff, 0x00000100,
0707     0xf084, 0xffffffff, 0x06000100,
0708     0xf0a4, 0xffffffff, 0x00000100,
0709     0xf09d, 0xffffffff, 0x00000100,
0710     0xf0ad, 0xffffffff, 0x00000100,
0711     0xf0ac, 0xffffffff, 0x00000100,
0712     0xf09c, 0xffffffff, 0x00000100,
0713     0xc200, 0xffffffff, 0xe0000000,
0714     0xf008, 0xffffffff, 0x00010000,
0715     0xf009, 0xffffffff, 0x00030002,
0716     0xf00a, 0xffffffff, 0x00040007,
0717     0xf00b, 0xffffffff, 0x00060005,
0718     0xf00c, 0xffffffff, 0x00090008,
0719     0xf00d, 0xffffffff, 0x00010000,
0720     0xf00e, 0xffffffff, 0x00030002,
0721     0xf00f, 0xffffffff, 0x00040007,
0722     0xf010, 0xffffffff, 0x00060005,
0723     0xf011, 0xffffffff, 0x00090008,
0724     0xf012, 0xffffffff, 0x00010000,
0725     0xf013, 0xffffffff, 0x00030002,
0726     0xf014, 0xffffffff, 0x00040007,
0727     0xf015, 0xffffffff, 0x00060005,
0728     0xf016, 0xffffffff, 0x00090008,
0729     0xf017, 0xffffffff, 0x00010000,
0730     0xf018, 0xffffffff, 0x00030002,
0731     0xf019, 0xffffffff, 0x00040007,
0732     0xf01a, 0xffffffff, 0x00060005,
0733     0xf01b, 0xffffffff, 0x00090008,
0734     0xf01c, 0xffffffff, 0x00010000,
0735     0xf01d, 0xffffffff, 0x00030002,
0736     0xf01e, 0xffffffff, 0x00040007,
0737     0xf01f, 0xffffffff, 0x00060005,
0738     0xf020, 0xffffffff, 0x00090008,
0739     0xf021, 0xffffffff, 0x00010000,
0740     0xf022, 0xffffffff, 0x00030002,
0741     0xf023, 0xffffffff, 0x00040007,
0742     0xf024, 0xffffffff, 0x00060005,
0743     0xf025, 0xffffffff, 0x00090008,
0744     0xf026, 0xffffffff, 0x00010000,
0745     0xf027, 0xffffffff, 0x00030002,
0746     0xf028, 0xffffffff, 0x00040007,
0747     0xf029, 0xffffffff, 0x00060005,
0748     0xf02a, 0xffffffff, 0x00090008,
0749     0xf02b, 0xffffffff, 0x00010000,
0750     0xf02c, 0xffffffff, 0x00030002,
0751     0xf02d, 0xffffffff, 0x00040007,
0752     0xf02e, 0xffffffff, 0x00060005,
0753     0xf02f, 0xffffffff, 0x00090008,
0754     0xf030, 0xffffffff, 0x00010000,
0755     0xf031, 0xffffffff, 0x00030002,
0756     0xf032, 0xffffffff, 0x00040007,
0757     0xf033, 0xffffffff, 0x00060005,
0758     0xf034, 0xffffffff, 0x00090008,
0759     0xf035, 0xffffffff, 0x00010000,
0760     0xf036, 0xffffffff, 0x00030002,
0761     0xf037, 0xffffffff, 0x00040007,
0762     0xf038, 0xffffffff, 0x00060005,
0763     0xf039, 0xffffffff, 0x00090008,
0764     0xf03a, 0xffffffff, 0x00010000,
0765     0xf03b, 0xffffffff, 0x00030002,
0766     0xf03c, 0xffffffff, 0x00040007,
0767     0xf03d, 0xffffffff, 0x00060005,
0768     0xf03e, 0xffffffff, 0x00090008,
0769     0x30c6, 0xffffffff, 0x00020200,
0770     0xcd4, 0xffffffff, 0x00000200,
0771     0x570, 0xffffffff, 0x00000400,
0772     0x157a, 0xffffffff, 0x00000000,
0773     0xbd4, 0xffffffff, 0x00000902,
0774     0xf000, 0xffffffff, 0x96940200,
0775     0x21c2, 0xffffffff, 0x00900100,
0776     0x3109, 0xffffffff, 0x0020003f,
0777     0xe, 0xffffffff, 0x0140001c,
0778     0xf, 0x000f0000, 0x000f0000,
0779     0x88, 0xffffffff, 0xc060000c,
0780     0x89, 0xc0000fff, 0x00000100,
0781     0x3e4, 0xffffffff, 0x00000100,
0782     0x3e6, 0x00000101, 0x00000000,
0783     0x82a, 0xffffffff, 0x00000104,
0784     0x1579, 0xff000fff, 0x00000100,
0785     0xc33, 0xc0000fff, 0x00000104,
0786     0x3079, 0x00000001, 0x00000001,
0787     0x3403, 0xff000ff0, 0x00000100,
0788     0x3603, 0xff000ff0, 0x00000100
0789 };
0790
0791 static const u32 godavari_golden_registers[] =
0792 {
0793     0x1579, 0xff607fff, 0xfc000100,
0794     0x1bb6, 0x00010101, 0x00010000,
0795     0x260c, 0xffffffff, 0x00000000,
0796     0x260c0, 0xf00fffff, 0x00000400,
0797     0x184c, 0xffffffff, 0x00010000,
0798     0x16ec, 0x000000f0, 0x00000070,
0799     0x16f0, 0xf0311fff, 0x80300000,
0800     0x263e, 0x73773777, 0x12010001,
0801     0x263f, 0xffffffff, 0x00000010,
0802     0x200c, 0x00001f0f, 0x0000100a,
0803     0xbd2, 0x73773777, 0x12010001,
0804     0x902, 0x000fffff, 0x000c007f,
0805     0x2285, 0xf000003f, 0x00000007,
0806     0x22c9, 0xffffffff, 0x00ff0fff,
0807     0xc281, 0x0000ff0f, 0x00000000,
0808     0xa293, 0x07ffffff, 0x06000000,
0809     0x136, 0x00000fff, 0x00000100,
0810     0x3405, 0x00010000, 0x00810001,
0811     0x3605, 0x00010000, 0x00810001,
0812     0xf9e, 0x00000001, 0x00000002,
0813     0x31da, 0x00000008, 0x00000008,
0814     0x31dc, 0x00000f00, 0x00000800,
0815     0x31dd, 0x00000f00, 0x00000800,
0816     0x31e6, 0x00ffffff, 0x00ff7fbf,
0817     0x31e7, 0x00ffffff, 0x00ff7faf,
0818     0x2300, 0x000000ff, 0x00000001,
0819     0x853e, 0x01ff01ff, 0x00000002,
0820     0x8526, 0x007ff800, 0x00200000,
0821     0x8057, 0xffffffff, 0x00000f40,
0822     0x2231, 0x001f3ae3, 0x00000082,
0823     0x2235, 0x0000001f, 0x00000010,
0824     0xc24d, 0xffffffff, 0x00000000
0825 };
0826
0827 static void cik_init_golden_registers(struct amdgpu_device *adev)
0828 {
0829     /* Some of the registers might be dependent on GRBM_GFX_INDEX */
0830     mutex_lock(&adev->grbm_idx_mutex);
0831
0832     switch (adev->asic_type) {
0833     case CHIP_BONAIRE:
0834         amdgpu_device_program_register_sequence(adev,
0835                             bonaire_mgcg_cgcg_init,
0836                             ARRAY_SIZE(bonaire_mgcg_cgcg_init));
0837         amdgpu_device_program_register_sequence(adev,
0838                             bonaire_golden_registers,
0839                             ARRAY_SIZE(bonaire_golden_registers));
0840         amdgpu_device_program_register_sequence(adev,
0841                             bonaire_golden_common_registers,
0842                             ARRAY_SIZE(bonaire_golden_common_registers));
0843         amdgpu_device_program_register_sequence(adev,
0844                             bonaire_golden_spm_registers,
0845                             ARRAY_SIZE(bonaire_golden_spm_registers));
0846         break;
0847     case CHIP_KABINI:
0848         amdgpu_device_program_register_sequence(adev,
0849                             kalindi_mgcg_cgcg_init,
0850                             ARRAY_SIZE(kalindi_mgcg_cgcg_init));
0851         amdgpu_device_program_register_sequence(adev,
0852                             kalindi_golden_registers,
0853                             ARRAY_SIZE(kalindi_golden_registers));
0854         amdgpu_device_program_register_sequence(adev,
0855                             kalindi_golden_common_registers,
0856                             ARRAY_SIZE(kalindi_golden_common_registers));
0857         amdgpu_device_program_register_sequence(adev,
0858                             kalindi_golden_spm_registers,
0859                             ARRAY_SIZE(kalindi_golden_spm_registers));
0860         break;
0861     case CHIP_MULLINS:
0862         amdgpu_device_program_register_sequence(adev,
0863                             kalindi_mgcg_cgcg_init,
0864                             ARRAY_SIZE(kalindi_mgcg_cgcg_init));
0865         amdgpu_device_program_register_sequence(adev,
0866                             godavari_golden_registers,
0867                             ARRAY_SIZE(godavari_golden_registers));
0868         amdgpu_device_program_register_sequence(adev,
0869                             kalindi_golden_common_registers,
0870                             ARRAY_SIZE(kalindi_golden_common_registers));
0871         amdgpu_device_program_register_sequence(adev,
0872                             kalindi_golden_spm_registers,
0873                             ARRAY_SIZE(kalindi_golden_spm_registers));
0874         break;
0875     case CHIP_KAVERI:
0876         amdgpu_device_program_register_sequence(adev,
0877                             spectre_mgcg_cgcg_init,
0878                             ARRAY_SIZE(spectre_mgcg_cgcg_init));
0879         amdgpu_device_program_register_sequence(adev,
0880                             spectre_golden_registers,
0881                             ARRAY_SIZE(spectre_golden_registers));
0882         amdgpu_device_program_register_sequence(adev,
0883                             spectre_golden_common_registers,
0884                             ARRAY_SIZE(spectre_golden_common_registers));
0885         amdgpu_device_program_register_sequence(adev,
0886                             spectre_golden_spm_registers,
0887                             ARRAY_SIZE(spectre_golden_spm_registers));
0888         break;
0889     case CHIP_HAWAII:
0890         amdgpu_device_program_register_sequence(adev,
0891                             hawaii_mgcg_cgcg_init,
0892                             ARRAY_SIZE(hawaii_mgcg_cgcg_init));
0893         amdgpu_device_program_register_sequence(adev,
0894                             hawaii_golden_registers,
0895                             ARRAY_SIZE(hawaii_golden_registers));
0896         amdgpu_device_program_register_sequence(adev,
0897                             hawaii_golden_common_registers,
0898                             ARRAY_SIZE(hawaii_golden_common_registers));
0899         amdgpu_device_program_register_sequence(adev,
0900                             hawaii_golden_spm_registers,
0901                             ARRAY_SIZE(hawaii_golden_spm_registers));
0902         break;
0903     default:
0904         break;
0905     }
0906     mutex_unlock(&adev->grbm_idx_mutex);
0907 }
0908
0909 /**
0910  * cik_get_xclk - get the xclk
0911  *
0912  * @adev: amdgpu_device pointer
0913  *
0914  * Returns the reference clock used by the gfx engine
0915  * (CIK).
0916  */
0917 static u32 cik_get_xclk(struct amdgpu_device *adev)
0918 {
0919     u32 reference_clock = adev->clock.spll.reference_freq;
0920
0921     if (adev->flags & AMD_IS_APU) {
0922         if (RREG32_SMC(ixGENERAL_PWRMGT) & GENERAL_PWRMGT__GPU_COUNTER_CLK_MASK)
0923             return reference_clock / 2;
0924     } else {
0925         if (RREG32_SMC(ixCG_CLKPIN_CNTL) & CG_CLKPIN_CNTL__XTALIN_DIVIDE_MASK)
0926             return reference_clock / 4;
0927     }
0928     return reference_clock;
0929 }
0930
0931 /**
0932  * cik_srbm_select - select specific register instances
0933  *
0934  * @adev: amdgpu_device pointer
0935  * @me: selected ME (micro engine)
0936  * @pipe: pipe
0937  * @queue: queue
0938  * @vmid: VMID
0939  *
0940  * Switches the currently active registers instances.  Some
0941  * registers are instanced per VMID, others are instanced per
0942  * me/pipe/queue combination.
0943  */
0944 void cik_srbm_select(struct amdgpu_device *adev,
0945              u32 me, u32 pipe, u32 queue, u32 vmid)
0946 {
0947     u32 srbm_gfx_cntl =
0948         (((pipe << SRBM_GFX_CNTL__PIPEID__SHIFT) & SRBM_GFX_CNTL__PIPEID_MASK)|
0949         ((me << SRBM_GFX_CNTL__MEID__SHIFT) & SRBM_GFX_CNTL__MEID_MASK)|
0950         ((vmid << SRBM_GFX_CNTL__VMID__SHIFT) & SRBM_GFX_CNTL__VMID_MASK)|
0951         ((queue << SRBM_GFX_CNTL__QUEUEID__SHIFT) & SRBM_GFX_CNTL__QUEUEID_MASK));
0952     WREG32(mmSRBM_GFX_CNTL, srbm_gfx_cntl);
0953 }
0954
0955 static void cik_vga_set_state(struct amdgpu_device *adev, bool state)
0956 {
0957     uint32_t tmp;
0958
0959     tmp = RREG32(mmCONFIG_CNTL);
0960     if (!state)
0961         tmp |= CONFIG_CNTL__VGA_DIS_MASK;
0962     else
0963         tmp &= ~CONFIG_CNTL__VGA_DIS_MASK;
0964     WREG32(mmCONFIG_CNTL, tmp);
0965 }
0966
0967 static bool cik_read_disabled_bios(struct amdgpu_device *adev)
0968 {
0969     u32 bus_cntl;
0970     u32 d1vga_control = 0;
0971     u32 d2vga_control = 0;
0972     u32 vga_render_control = 0;
0973     u32 rom_cntl;
0974     bool r;
0975
0976     bus_cntl = RREG32(mmBUS_CNTL);
0977     if (adev->mode_info.num_crtc) {
0978         d1vga_control = RREG32(mmD1VGA_CONTROL);
0979         d2vga_control = RREG32(mmD2VGA_CONTROL);
0980         vga_render_control = RREG32(mmVGA_RENDER_CONTROL);
0981     }
0982     rom_cntl = RREG32_SMC(ixROM_CNTL);
0983
0984     /* enable the rom */
0985     WREG32(mmBUS_CNTL, (bus_cntl & ~BUS_CNTL__BIOS_ROM_DIS_MASK));
0986     if (adev->mode_info.num_crtc) {
0987         /* Disable VGA mode */
0988         WREG32(mmD1VGA_CONTROL,
0989                (d1vga_control & ~(D1VGA_CONTROL__D1VGA_MODE_ENABLE_MASK |
0990                       D1VGA_CONTROL__D1VGA_TIMING_SELECT_MASK)));
0991         WREG32(mmD2VGA_CONTROL,
0992                (d2vga_control & ~(D1VGA_CONTROL__D1VGA_MODE_ENABLE_MASK |
0993                       D1VGA_CONTROL__D1VGA_TIMING_SELECT_MASK)));
0994         WREG32(mmVGA_RENDER_CONTROL,
0995                (vga_render_control & ~VGA_RENDER_CONTROL__VGA_VSTATUS_CNTL_MASK));
0996     }
0997     WREG32_SMC(ixROM_CNTL, rom_cntl | ROM_CNTL__SCK_OVERWRITE_MASK);
0998
0999     r = amdgpu_read_bios(adev);
1000
1001     /* restore regs */
1002     WREG32(mmBUS_CNTL, bus_cntl);
1003     if (adev->mode_info.num_crtc) {
1004         WREG32(mmD1VGA_CONTROL, d1vga_control);
1005         WREG32(mmD2VGA_CONTROL, d2vga_control);
1006         WREG32(mmVGA_RENDER_CONTROL, vga_render_control);
1007     }
1008     WREG32_SMC(ixROM_CNTL, rom_cntl);
1009     return r;
1010 }
1011
1012 static bool cik_read_bios_from_rom(struct amdgpu_device *adev,
1013                    u8 *bios, u32 length_bytes)
1014 {
1015     u32 *dw_ptr;
1016     unsigned long flags;
1017     u32 i, length_dw;
1018
1019     if (bios == NULL)
1020         return false;
1021     if (length_bytes == 0)
1022         return false;
1023     /* APU vbios image is part of sbios image */
1024     if (adev->flags & AMD_IS_APU)
1025         return false;
1026
1027     dw_ptr = (u32 *)bios;
1028     length_dw = ALIGN(length_bytes, 4) / 4;
1029     /* take the smc lock since we are using the smc index */
1030     spin_lock_irqsave(&adev->smc_idx_lock, flags);
1031     /* set rom index to 0 */
1032     WREG32(mmSMC_IND_INDEX_0, ixROM_INDEX);
1033     WREG32(mmSMC_IND_DATA_0, 0);
1034     /* set index to data for continous read */
1035     WREG32(mmSMC_IND_INDEX_0, ixROM_DATA);
1036     for (i = 0; i < length_dw; i++)
1037         dw_ptr[i] = RREG32(mmSMC_IND_DATA_0);
1038     spin_unlock_irqrestore(&adev->smc_idx_lock, flags);
1039
1040     return true;
1041 }
1042
1043 static const struct amdgpu_allowed_register_entry cik_allowed_read_registers[] = {
1044     {mmGRBM_STATUS},
1045     {mmGRBM_STATUS2},
1046     {mmGRBM_STATUS_SE0},
1047     {mmGRBM_STATUS_SE1},
1048     {mmGRBM_STATUS_SE2},
1049     {mmGRBM_STATUS_SE3},
1050     {mmSRBM_STATUS},
1051     {mmSRBM_STATUS2},
1052     {mmSDMA0_STATUS_REG + SDMA0_REGISTER_OFFSET},
1053     {mmSDMA0_STATUS_REG + SDMA1_REGISTER_OFFSET},
1054     {mmCP_STAT},
1055     {mmCP_STALLED_STAT1},
1056     {mmCP_STALLED_STAT2},
1057     {mmCP_STALLED_STAT3},
1058     {mmCP_CPF_BUSY_STAT},
1059     {mmCP_CPF_STALLED_STAT1},
1060     {mmCP_CPF_STATUS},
1061     {mmCP_CPC_BUSY_STAT},
1062     {mmCP_CPC_STALLED_STAT1},
1063     {mmCP_CPC_STATUS},
1064     {mmGB_ADDR_CONFIG},
1065     {mmMC_ARB_RAMCFG},
1066     {mmGB_TILE_MODE0},
1067     {mmGB_TILE_MODE1},
1068     {mmGB_TILE_MODE2},
1069     {mmGB_TILE_MODE3},
1070     {mmGB_TILE_MODE4},
1071     {mmGB_TILE_MODE5},
1072     {mmGB_TILE_MODE6},
1073     {mmGB_TILE_MODE7},
1074     {mmGB_TILE_MODE8},
1075     {mmGB_TILE_MODE9},
1076     {mmGB_TILE_MODE10},
1077     {mmGB_TILE_MODE11},
1078     {mmGB_TILE_MODE12},
1079     {mmGB_TILE_MODE13},
1080     {mmGB_TILE_MODE14},
1081     {mmGB_TILE_MODE15},
1082     {mmGB_TILE_MODE16},
1083     {mmGB_TILE_MODE17},
1084     {mmGB_TILE_MODE18},
1085     {mmGB_TILE_MODE19},
1086     {mmGB_TILE_MODE20},
1087     {mmGB_TILE_MODE21},
1088     {mmGB_TILE_MODE22},
1089     {mmGB_TILE_MODE23},
1090     {mmGB_TILE_MODE24},
1091     {mmGB_TILE_MODE25},
1092     {mmGB_TILE_MODE26},
1093     {mmGB_TILE_MODE27},
1094     {mmGB_TILE_MODE28},
1095     {mmGB_TILE_MODE29},
1096     {mmGB_TILE_MODE30},
1097     {mmGB_TILE_MODE31},
1098     {mmGB_MACROTILE_MODE0},
1099     {mmGB_MACROTILE_MODE1},
1100     {mmGB_MACROTILE_MODE2},
1101     {mmGB_MACROTILE_MODE3},
1102     {mmGB_MACROTILE_MODE4},
1103     {mmGB_MACROTILE_MODE5},
1104     {mmGB_MACROTILE_MODE6},
1105     {mmGB_MACROTILE_MODE7},
1106     {mmGB_MACROTILE_MODE8},
1107     {mmGB_MACROTILE_MODE9},
1108     {mmGB_MACROTILE_MODE10},
1109     {mmGB_MACROTILE_MODE11},
1110     {mmGB_MACROTILE_MODE12},
1111     {mmGB_MACROTILE_MODE13},
1112     {mmGB_MACROTILE_MODE14},
1113     {mmGB_MACROTILE_MODE15},
1114     {mmCC_RB_BACKEND_DISABLE, true},
1115     {mmGC_USER_RB_BACKEND_DISABLE, true},
1116     {mmGB_BACKEND_MAP, false},
1117     {mmPA_SC_RASTER_CONFIG, true},
1118     {mmPA_SC_RASTER_CONFIG_1, true},
1119 };
1120
1121
1122 static uint32_t cik_get_register_value(struct amdgpu_device *adev,
1123                        bool indexed, u32 se_num,
1124                        u32 sh_num, u32 reg_offset)
1125 {
1126     if (indexed) {
1127         uint32_t val;
1128         unsigned se_idx = (se_num == 0xffffffff) ? 0 : se_num;
1129         unsigned sh_idx = (sh_num == 0xffffffff) ? 0 : sh_num;
1130
1131         switch (reg_offset) {
1132         case mmCC_RB_BACKEND_DISABLE:
1133             return adev->gfx.config.rb_config[se_idx][sh_idx].rb_backend_disable;
1134         case mmGC_USER_RB_BACKEND_DISABLE:
1135             return adev->gfx.config.rb_config[se_idx][sh_idx].user_rb_backend_disable;
1136         case mmPA_SC_RASTER_CONFIG:
1137             return adev->gfx.config.rb_config[se_idx][sh_idx].raster_config;
1138         case mmPA_SC_RASTER_CONFIG_1:
1139             return adev->gfx.config.rb_config[se_idx][sh_idx].raster_config_1;
1140         }
1141
1142         mutex_lock(&adev->grbm_idx_mutex);
1143         if (se_num != 0xffffffff || sh_num != 0xffffffff)
1144             amdgpu_gfx_select_se_sh(adev, se_num, sh_num, 0xffffffff);
1145
1146         val = RREG32(reg_offset);
1147
1148         if (se_num != 0xffffffff || sh_num != 0xffffffff)
1149             amdgpu_gfx_select_se_sh(adev, 0xffffffff, 0xffffffff, 0xffffffff);
1150         mutex_unlock(&adev->grbm_idx_mutex);
1151         return val;
1152     } else {
1153         unsigned idx;
1154
1155         switch (reg_offset) {
1156         case mmGB_ADDR_CONFIG:
1157             return adev->gfx.config.gb_addr_config;
1158         case mmMC_ARB_RAMCFG:
1159             return adev->gfx.config.mc_arb_ramcfg;
1160         case mmGB_TILE_MODE0:
1161         case mmGB_TILE_MODE1:
1162         case mmGB_TILE_MODE2:
1163         case mmGB_TILE_MODE3:
1164         case mmGB_TILE_MODE4:
1165         case mmGB_TILE_MODE5:
1166         case mmGB_TILE_MODE6:
1167         case mmGB_TILE_MODE7:
1168         case mmGB_TILE_MODE8:
1169         case mmGB_TILE_MODE9:
1170         case mmGB_TILE_MODE10:
1171         case mmGB_TILE_MODE11:
1172         case mmGB_TILE_MODE12:
1173         case mmGB_TILE_MODE13:
1174         case mmGB_TILE_MODE14:
1175         case mmGB_TILE_MODE15:
1176         case mmGB_TILE_MODE16:
1177         case mmGB_TILE_MODE17:
1178         case mmGB_TILE_MODE18:
1179         case mmGB_TILE_MODE19:
1180         case mmGB_TILE_MODE20:
1181         case mmGB_TILE_MODE21:
1182         case mmGB_TILE_MODE22:
1183         case mmGB_TILE_MODE23:
1184         case mmGB_TILE_MODE24:
1185         case mmGB_TILE_MODE25:
1186         case mmGB_TILE_MODE26:
1187         case mmGB_TILE_MODE27:
1188         case mmGB_TILE_MODE28:
1189         case mmGB_TILE_MODE29:
1190         case mmGB_TILE_MODE30:
1191         case mmGB_TILE_MODE31:
1192             idx = (reg_offset - mmGB_TILE_MODE0);
1193             return adev->gfx.config.tile_mode_array[idx];
1194         case mmGB_MACROTILE_MODE0:
1195         case mmGB_MACROTILE_MODE1:
1196         case mmGB_MACROTILE_MODE2:
1197         case mmGB_MACROTILE_MODE3:
1198         case mmGB_MACROTILE_MODE4:
1199         case mmGB_MACROTILE_MODE5:
1200         case mmGB_MACROTILE_MODE6:
1201         case mmGB_MACROTILE_MODE7:
1202         case mmGB_MACROTILE_MODE8:
1203         case mmGB_MACROTILE_MODE9:
1204         case mmGB_MACROTILE_MODE10:
1205         case mmGB_MACROTILE_MODE11:
1206         case mmGB_MACROTILE_MODE12:
1207         case mmGB_MACROTILE_MODE13:
1208         case mmGB_MACROTILE_MODE14:
1209         case mmGB_MACROTILE_MODE15:
1210             idx = (reg_offset - mmGB_MACROTILE_MODE0);
1211             return adev->gfx.config.macrotile_mode_array[idx];
1212         default:
1213             return RREG32(reg_offset);
1214         }
1215     }
1216 }
1217
1218 static int cik_read_register(struct amdgpu_device *adev, u32 se_num,
1219                  u32 sh_num, u32 reg_offset, u32 *value)
1220 {
1221     uint32_t i;
1222
1223     *value = 0;
1224     for (i = 0; i < ARRAY_SIZE(cik_allowed_read_registers); i++) {
1225         bool indexed = cik_allowed_read_registers[i].grbm_indexed;
1226
1227         if (reg_offset != cik_allowed_read_registers[i].reg_offset)
1228             continue;
1229
1230         *value = cik_get_register_value(adev, indexed, se_num, sh_num,
1231                         reg_offset);
1232         return 0;
1233     }
1234     return -EINVAL;
1235 }
1236
1237 struct kv_reset_save_regs {
1238     u32 gmcon_reng_execute;
1239     u32 gmcon_misc;
1240     u32 gmcon_misc3;
1241 };
1242
1243 static void kv_save_regs_for_reset(struct amdgpu_device *adev,
1244                    struct kv_reset_save_regs *save)
1245 {
1246     save->gmcon_reng_execute = RREG32(mmGMCON_RENG_EXECUTE);
1247     save->gmcon_misc = RREG32(mmGMCON_MISC);
1248     save->gmcon_misc3 = RREG32(mmGMCON_MISC3);
1249
1250     WREG32(mmGMCON_RENG_EXECUTE, save->gmcon_reng_execute &
1251         ~GMCON_RENG_EXECUTE__RENG_EXECUTE_ON_PWR_UP_MASK);
1252     WREG32(mmGMCON_MISC, save->gmcon_misc &
1253         ~(GMCON_MISC__RENG_EXECUTE_ON_REG_UPDATE_MASK |
1254             GMCON_MISC__STCTRL_STUTTER_EN_MASK));
1255 }
1256
1257 static void kv_restore_regs_for_reset(struct amdgpu_device *adev,
1258                       struct kv_reset_save_regs *save)
1259 {
1260     int i;
1261
1262     WREG32(mmGMCON_PGFSM_WRITE, 0);
1263     WREG32(mmGMCON_PGFSM_CONFIG, 0x200010ff);
1264
1265     for (i = 0; i < 5; i++)
1266         WREG32(mmGMCON_PGFSM_WRITE, 0);
1267
1268     WREG32(mmGMCON_PGFSM_WRITE, 0);
1269     WREG32(mmGMCON_PGFSM_CONFIG, 0x300010ff);
1270
1271     for (i = 0; i < 5; i++)
1272         WREG32(mmGMCON_PGFSM_WRITE, 0);
1273
1274     WREG32(mmGMCON_PGFSM_WRITE, 0x210000);
1275     WREG32(mmGMCON_PGFSM_CONFIG, 0xa00010ff);
1276
1277     for (i = 0; i < 5; i++)
1278         WREG32(mmGMCON_PGFSM_WRITE, 0);
1279
1280     WREG32(mmGMCON_PGFSM_WRITE, 0x21003);
1281     WREG32(mmGMCON_PGFSM_CONFIG, 0xb00010ff);
1282
1283     for (i = 0; i < 5; i++)
1284         WREG32(mmGMCON_PGFSM_WRITE, 0);
1285
1286     WREG32(mmGMCON_PGFSM_WRITE, 0x2b00);
1287     WREG32(mmGMCON_PGFSM_CONFIG, 0xc00010ff);
1288
1289     for (i = 0; i < 5; i++)
1290         WREG32(mmGMCON_PGFSM_WRITE, 0);
1291
1292     WREG32(mmGMCON_PGFSM_WRITE, 0);
1293     WREG32(mmGMCON_PGFSM_CONFIG, 0xd00010ff);
1294
1295     for (i = 0; i < 5; i++)
1296         WREG32(mmGMCON_PGFSM_WRITE, 0);
1297
1298     WREG32(mmGMCON_PGFSM_WRITE, 0x420000);
1299     WREG32(mmGMCON_PGFSM_CONFIG, 0x100010ff);
1300
1301     for (i = 0; i < 5; i++)
1302         WREG32(mmGMCON_PGFSM_WRITE, 0);
1303
1304     WREG32(mmGMCON_PGFSM_WRITE, 0x120202);
1305     WREG32(mmGMCON_PGFSM_CONFIG, 0x500010ff);
1306
1307     for (i = 0; i < 5; i++)
1308         WREG32(mmGMCON_PGFSM_WRITE, 0);
1309
1310     WREG32(mmGMCON_PGFSM_WRITE, 0x3e3e36);
1311     WREG32(mmGMCON_PGFSM_CONFIG, 0x600010ff);
1312
1313     for (i = 0; i < 5; i++)
1314         WREG32(mmGMCON_PGFSM_WRITE, 0);
1315
1316     WREG32(mmGMCON_PGFSM_WRITE, 0x373f3e);
1317     WREG32(mmGMCON_PGFSM_CONFIG, 0x700010ff);
1318
1319     for (i = 0; i < 5; i++)
1320         WREG32(mmGMCON_PGFSM_WRITE, 0);
1321
1322     WREG32(mmGMCON_PGFSM_WRITE, 0x3e1332);
1323     WREG32(mmGMCON_PGFSM_CONFIG, 0xe00010ff);
1324
1325     WREG32(mmGMCON_MISC3, save->gmcon_misc3);
1326     WREG32(mmGMCON_MISC, save->gmcon_misc);
1327     WREG32(mmGMCON_RENG_EXECUTE, save->gmcon_reng_execute);
1328 }
1329
1330 /**
1331  * cik_asic_pci_config_reset - soft reset GPU
1332  *
1333  * @adev: amdgpu_device pointer
1334  *
1335  * Use PCI Config method to reset the GPU.
1336  *
1337  * Returns 0 for success.
1338  */
1339 static int cik_asic_pci_config_reset(struct amdgpu_device *adev)
1340 {
1341     struct kv_reset_save_regs kv_save = { 0 };
1342     u32 i;
1343     int r = -EINVAL;
1344
1345     amdgpu_atombios_scratch_regs_engine_hung(adev, true);
1346
1347     if (adev->flags & AMD_IS_APU)
1348         kv_save_regs_for_reset(adev, &kv_save);
1349
1350     /* disable BM */
1351     pci_clear_master(adev->pdev);
1352     /* reset */
1353     amdgpu_device_pci_config_reset(adev);
1354
1355     udelay(100);
1356
1357     /* wait for asic to come out of reset */
1358     for (i = 0; i < adev->usec_timeout; i++) {
1359         if (RREG32(mmCONFIG_MEMSIZE) != 0xffffffff) {
1360             /* enable BM */
1361             pci_set_master(adev->pdev);
1362             adev->has_hw_reset = true;
1363             r = 0;
1364             break;
1365         }
1366         udelay(1);
1367     }
1368
1369     /* does asic init need to be run first??? */
1370     if (adev->flags & AMD_IS_APU)
1371         kv_restore_regs_for_reset(adev, &kv_save);
1372
1373     amdgpu_atombios_scratch_regs_engine_hung(adev, false);
1374
1375     return r;
1376 }
1377
1378 static bool cik_asic_supports_baco(struct amdgpu_device *adev)
1379 {
1380     switch (adev->asic_type) {
1381     case CHIP_BONAIRE:
1382     case CHIP_HAWAII:
1383         return amdgpu_dpm_is_baco_supported(adev);
1384     default:
1385         return false;
1386     }
1387 }
1388
1389 static enum amd_reset_method
1390 cik_asic_reset_method(struct amdgpu_device *adev)
1391 {
1392     bool baco_reset;
1393
1394     if (amdgpu_reset_method == AMD_RESET_METHOD_LEGACY ||
1395         amdgpu_reset_method == AMD_RESET_METHOD_BACO)
1396         return amdgpu_reset_method;
1397
1398     if (amdgpu_reset_method != -1)
1399         dev_warn(adev->dev, "Specified reset:%d isn't supported, using AUTO instead.\n",
1400                   amdgpu_reset_method);
1401
1402     switch (adev->asic_type) {
1403     case CHIP_BONAIRE:
1404     case CHIP_HAWAII:
1405         baco_reset = cik_asic_supports_baco(adev);
1406         break;
1407     default:
1408         baco_reset = false;
1409         break;
1410     }
1411
1412     if (baco_reset)
1413         return AMD_RESET_METHOD_BACO;
1414     else
1415         return AMD_RESET_METHOD_LEGACY;
1416 }
1417
1418 /**
1419  * cik_asic_reset - soft reset GPU
1420  *
1421  * @adev: amdgpu_device pointer
1422  *
1423  * Look up which blocks are hung and attempt
1424  * to reset them.
1425  * Returns 0 for success.
1426  */
1427 static int cik_asic_reset(struct amdgpu_device *adev)
1428 {
1429     int r;
1430
1431     /* APUs don't have full asic reset */
1432     if (adev->flags & AMD_IS_APU)
1433         return 0;
1434
1435     if (cik_asic_reset_method(adev) == AMD_RESET_METHOD_BACO) {
1436         dev_info(adev->dev, "BACO reset\n");
1437         r = amdgpu_dpm_baco_reset(adev);
1438     } else {
1439         dev_info(adev->dev, "PCI CONFIG reset\n");
1440         r = cik_asic_pci_config_reset(adev);
1441     }
1442
1443     return r;
1444 }
1445
1446 static u32 cik_get_config_memsize(struct amdgpu_device *adev)
1447 {
1448     return RREG32(mmCONFIG_MEMSIZE);
1449 }
1450
1451 static int cik_set_uvd_clock(struct amdgpu_device *adev, u32 clock,
1452                   u32 cntl_reg, u32 status_reg)
1453 {
1454     int r, i;
1455     struct atom_clock_dividers dividers;
1456     uint32_t tmp;
1457
1458     r = amdgpu_atombios_get_clock_dividers(adev,
1459                            COMPUTE_GPUCLK_INPUT_FLAG_DEFAULT_GPUCLK,
1460                            clock, false, &dividers);
1461     if (r)
1462         return r;
1463
1464     tmp = RREG32_SMC(cntl_reg);
1465     tmp &= ~(CG_DCLK_CNTL__DCLK_DIR_CNTL_EN_MASK |
1466         CG_DCLK_CNTL__DCLK_DIVIDER_MASK);
1467     tmp |= dividers.post_divider;
1468     WREG32_SMC(cntl_reg, tmp);
1469
1470     for (i = 0; i < 100; i++) {
1471         if (RREG32_SMC(status_reg) & CG_DCLK_STATUS__DCLK_STATUS_MASK)
1472             break;
1473         mdelay(10);
1474     }
1475     if (i == 100)
1476         return -ETIMEDOUT;
1477
1478     return 0;
1479 }
1480
1481 static int cik_set_uvd_clocks(struct amdgpu_device *adev, u32 vclk, u32 dclk)
1482 {
1483     int r = 0;
1484
1485     r = cik_set_uvd_clock(adev, vclk, ixCG_VCLK_CNTL, ixCG_VCLK_STATUS);
1486     if (r)
1487         return r;
1488
1489     r = cik_set_uvd_clock(adev, dclk, ixCG_DCLK_CNTL, ixCG_DCLK_STATUS);
1490     return r;
1491 }
1492
1493 static int cik_set_vce_clocks(struct amdgpu_device *adev, u32 evclk, u32 ecclk)
1494 {
1495     int r, i;
1496     struct atom_clock_dividers dividers;
1497     u32 tmp;
1498
1499     r = amdgpu_atombios_get_clock_dividers(adev,
1500                            COMPUTE_GPUCLK_INPUT_FLAG_DEFAULT_GPUCLK,
1501                            ecclk, false, &dividers);
1502     if (r)
1503         return r;
1504
1505     for (i = 0; i < 100; i++) {
1506         if (RREG32_SMC(ixCG_ECLK_STATUS) & CG_ECLK_STATUS__ECLK_STATUS_MASK)
1507             break;
1508         mdelay(10);
1509     }
1510     if (i == 100)
1511         return -ETIMEDOUT;
1512
1513     tmp = RREG32_SMC(ixCG_ECLK_CNTL);
1514     tmp &= ~(CG_ECLK_CNTL__ECLK_DIR_CNTL_EN_MASK |
1515         CG_ECLK_CNTL__ECLK_DIVIDER_MASK);
1516     tmp |= dividers.post_divider;
1517     WREG32_SMC(ixCG_ECLK_CNTL, tmp);
1518
1519     for (i = 0; i < 100; i++) {
1520         if (RREG32_SMC(ixCG_ECLK_STATUS) & CG_ECLK_STATUS__ECLK_STATUS_MASK)
1521             break;
1522         mdelay(10);
1523     }
1524     if (i == 100)
1525         return -ETIMEDOUT;
1526
1527     return 0;
1528 }
1529
1530 static void cik_pcie_gen3_enable(struct amdgpu_device *adev)
1531 {
1532     struct pci_dev *root = adev->pdev->bus->self;
1533     u32 speed_cntl, current_data_rate;
1534     int i;
1535     u16 tmp16;
1536
1537     if (pci_is_root_bus(adev->pdev->bus))
1538         return;
1539
1540     if (amdgpu_pcie_gen2 == 0)
1541         return;
1542
1543     if (adev->flags & AMD_IS_APU)
1544         return;
1545
1546     if (!(adev->pm.pcie_gen_mask & (CAIL_PCIE_LINK_SPEED_SUPPORT_GEN2 |
1547                     CAIL_PCIE_LINK_SPEED_SUPPORT_GEN3)))
1548         return;
1549
1550     speed_cntl = RREG32_PCIE(ixPCIE_LC_SPEED_CNTL);
1551     current_data_rate = (speed_cntl & PCIE_LC_SPEED_CNTL__LC_CURRENT_DATA_RATE_MASK) >>
1552         PCIE_LC_SPEED_CNTL__LC_CURRENT_DATA_RATE__SHIFT;
1553     if (adev->pm.pcie_gen_mask & CAIL_PCIE_LINK_SPEED_SUPPORT_GEN3) {
1554         if (current_data_rate == 2) {
1555             DRM_INFO("PCIE gen 3 link speeds already enabled\n");
1556             return;
1557         }
1558         DRM_INFO("enabling PCIE gen 3 link speeds, disable with amdgpu.pcie_gen2=0\n");
1559     } else if (adev->pm.pcie_gen_mask & CAIL_PCIE_LINK_SPEED_SUPPORT_GEN2) {
1560         if (current_data_rate == 1) {
1561             DRM_INFO("PCIE gen 2 link speeds already enabled\n");
1562             return;
1563         }
1564         DRM_INFO("enabling PCIE gen 2 link speeds, disable with amdgpu.pcie_gen2=0\n");
1565     }
1566
1567     if (!pci_is_pcie(root) || !pci_is_pcie(adev->pdev))
1568         return;
1569
1570     if (adev->pm.pcie_gen_mask & CAIL_PCIE_LINK_SPEED_SUPPORT_GEN3) {
1571         /* re-try equalization if gen3 is not already enabled */
1572         if (current_data_rate != 2) {
1573             u16 bridge_cfg, gpu_cfg;
1574             u16 bridge_cfg2, gpu_cfg2;
1575             u32 max_lw, current_lw, tmp;
1576
1577             pcie_capability_read_word(root, PCI_EXP_LNKCTL,
1578                           &bridge_cfg);
1579             pcie_capability_read_word(adev->pdev, PCI_EXP_LNKCTL,
1580                           &gpu_cfg);
1581
1582             tmp16 = bridge_cfg | PCI_EXP_LNKCTL_HAWD;
1583             pcie_capability_write_word(root, PCI_EXP_LNKCTL, tmp16);
1584
1585             tmp16 = gpu_cfg | PCI_EXP_LNKCTL_HAWD;
1586             pcie_capability_write_word(adev->pdev, PCI_EXP_LNKCTL,
1587                            tmp16);
1588
1589             tmp = RREG32_PCIE(ixPCIE_LC_STATUS1);
1590             max_lw = (tmp & PCIE_LC_STATUS1__LC_DETECTED_LINK_WIDTH_MASK) >>
1591                 PCIE_LC_STATUS1__LC_DETECTED_LINK_WIDTH__SHIFT;
1592             current_lw = (tmp & PCIE_LC_STATUS1__LC_OPERATING_LINK_WIDTH_MASK)
1593                 >> PCIE_LC_STATUS1__LC_OPERATING_LINK_WIDTH__SHIFT;
1594
1595             if (current_lw < max_lw) {
1596                 tmp = RREG32_PCIE(ixPCIE_LC_LINK_WIDTH_CNTL);
1597                 if (tmp & PCIE_LC_LINK_WIDTH_CNTL__LC_RENEGOTIATION_SUPPORT_MASK) {
1598                     tmp &= ~(PCIE_LC_LINK_WIDTH_CNTL__LC_LINK_WIDTH_MASK |
1599                         PCIE_LC_LINK_WIDTH_CNTL__LC_UPCONFIGURE_DIS_MASK);
1600                     tmp |= (max_lw <<
1601                         PCIE_LC_LINK_WIDTH_CNTL__LC_LINK_WIDTH__SHIFT);
1602                     tmp |= PCIE_LC_LINK_WIDTH_CNTL__LC_UPCONFIGURE_SUPPORT_MASK |
1603                     PCIE_LC_LINK_WIDTH_CNTL__LC_RENEGOTIATE_EN_MASK |
1604                     PCIE_LC_LINK_WIDTH_CNTL__LC_RECONFIG_NOW_MASK;
1605                     WREG32_PCIE(ixPCIE_LC_LINK_WIDTH_CNTL, tmp);
1606                 }
1607             }
1608
1609             for (i = 0; i < 10; i++) {
1610                 /* check status */
1611                 pcie_capability_read_word(adev->pdev,
1612                               PCI_EXP_DEVSTA,
1613                               &tmp16);
1614                 if (tmp16 & PCI_EXP_DEVSTA_TRPND)
1615                     break;
1616
1617                 pcie_capability_read_word(root, PCI_EXP_LNKCTL,
1618                               &bridge_cfg);
1619                 pcie_capability_read_word(adev->pdev,
1620                               PCI_EXP_LNKCTL,
1621                               &gpu_cfg);
1622
1623                 pcie_capability_read_word(root, PCI_EXP_LNKCTL2,
1624                               &bridge_cfg2);
1625                 pcie_capability_read_word(adev->pdev,
1626                               PCI_EXP_LNKCTL2,
1627                               &gpu_cfg2);
1628
1629                 tmp = RREG32_PCIE(ixPCIE_LC_CNTL4);
1630                 tmp |= PCIE_LC_CNTL4__LC_SET_QUIESCE_MASK;
1631                 WREG32_PCIE(ixPCIE_LC_CNTL4, tmp);
1632
1633                 tmp = RREG32_PCIE(ixPCIE_LC_CNTL4);
1634                 tmp |= PCIE_LC_CNTL4__LC_REDO_EQ_MASK;
1635                 WREG32_PCIE(ixPCIE_LC_CNTL4, tmp);
1636
1637                 msleep(100);
1638
1639                 /* linkctl */
1640                 pcie_capability_read_word(root, PCI_EXP_LNKCTL,
1641                               &tmp16);
1642                 tmp16 &= ~PCI_EXP_LNKCTL_HAWD;
1643                 tmp16 |= (bridge_cfg & PCI_EXP_LNKCTL_HAWD);
1644                 pcie_capability_write_word(root, PCI_EXP_LNKCTL,
1645                                tmp16);
1646
1647                 pcie_capability_read_word(adev->pdev,
1648                               PCI_EXP_LNKCTL,
1649                               &tmp16);
1650                 tmp16 &= ~PCI_EXP_LNKCTL_HAWD;
1651                 tmp16 |= (gpu_cfg & PCI_EXP_LNKCTL_HAWD);
1652                 pcie_capability_write_word(adev->pdev,
1653                                PCI_EXP_LNKCTL,
1654                                tmp16);
1655
1656                 /* linkctl2 */
1657                 pcie_capability_read_word(root, PCI_EXP_LNKCTL2,
1658                               &tmp16);
1659                 tmp16 &= ~(PCI_EXP_LNKCTL2_ENTER_COMP |
1660                        PCI_EXP_LNKCTL2_TX_MARGIN);
1661                 tmp16 |= (bridge_cfg2 &
1662                       (PCI_EXP_LNKCTL2_ENTER_COMP |
1663                        PCI_EXP_LNKCTL2_TX_MARGIN));
1664                 pcie_capability_write_word(root,
1665                                PCI_EXP_LNKCTL2,
1666                                tmp16);
1667
1668                 pcie_capability_read_word(adev->pdev,
1669                               PCI_EXP_LNKCTL2,
1670                               &tmp16);
1671                 tmp16 &= ~(PCI_EXP_LNKCTL2_ENTER_COMP |
1672                        PCI_EXP_LNKCTL2_TX_MARGIN);
1673                 tmp16 |= (gpu_cfg2 &
1674                       (PCI_EXP_LNKCTL2_ENTER_COMP |
1675                        PCI_EXP_LNKCTL2_TX_MARGIN));
1676                 pcie_capability_write_word(adev->pdev,
1677                                PCI_EXP_LNKCTL2,
1678                                tmp16);
1679
1680                 tmp = RREG32_PCIE(ixPCIE_LC_CNTL4);
1681                 tmp &= ~PCIE_LC_CNTL4__LC_SET_QUIESCE_MASK;
1682                 WREG32_PCIE(ixPCIE_LC_CNTL4, tmp);
1683             }
1684         }
1685     }
1686
1687     /* set the link speed */
1688     speed_cntl |= PCIE_LC_SPEED_CNTL__LC_FORCE_EN_SW_SPEED_CHANGE_MASK |
1689         PCIE_LC_SPEED_CNTL__LC_FORCE_DIS_HW_SPEED_CHANGE_MASK;
1690     speed_cntl &= ~PCIE_LC_SPEED_CNTL__LC_FORCE_DIS_SW_SPEED_CHANGE_MASK;
1691     WREG32_PCIE(ixPCIE_LC_SPEED_CNTL, speed_cntl);
1692
1693     pcie_capability_read_word(adev->pdev, PCI_EXP_LNKCTL2, &tmp16);
1694     tmp16 &= ~PCI_EXP_LNKCTL2_TLS;
1695
1696     if (adev->pm.pcie_gen_mask & CAIL_PCIE_LINK_SPEED_SUPPORT_GEN3)
1697         tmp16 |= PCI_EXP_LNKCTL2_TLS_8_0GT; /* gen3 */
1698     else if (adev->pm.pcie_gen_mask & CAIL_PCIE_LINK_SPEED_SUPPORT_GEN2)
1699         tmp16 |= PCI_EXP_LNKCTL2_TLS_5_0GT; /* gen2 */
1700     else
1701         tmp16 |= PCI_EXP_LNKCTL2_TLS_2_5GT; /* gen1 */
1702     pcie_capability_write_word(adev->pdev, PCI_EXP_LNKCTL2, tmp16);
1703
1704     speed_cntl = RREG32_PCIE(ixPCIE_LC_SPEED_CNTL);
1705     speed_cntl |= PCIE_LC_SPEED_CNTL__LC_INITIATE_LINK_SPEED_CHANGE_MASK;
1706     WREG32_PCIE(ixPCIE_LC_SPEED_CNTL, speed_cntl);
1707
1708     for (i = 0; i < adev->usec_timeout; i++) {
1709         speed_cntl = RREG32_PCIE(ixPCIE_LC_SPEED_CNTL);
1710         if ((speed_cntl & PCIE_LC_SPEED_CNTL__LC_INITIATE_LINK_SPEED_CHANGE_MASK) == 0)
1711             break;
1712         udelay(1);
1713     }
1714 }
1715
1716 static void cik_program_aspm(struct amdgpu_device *adev)
1717 {
1718     u32 data, orig;
1719     bool disable_l0s = false, disable_l1 = false, disable_plloff_in_l1 = false;
1720     bool disable_clkreq = false;
1721
1722     if (!amdgpu_device_should_use_aspm(adev))
1723         return;
1724
1725     if (pci_is_root_bus(adev->pdev->bus))
1726         return;
1727
1728     /* XXX double check APUs */
1729     if (adev->flags & AMD_IS_APU)
1730         return;
1731
1732     orig = data = RREG32_PCIE(ixPCIE_LC_N_FTS_CNTL);
1733     data &= ~PCIE_LC_N_FTS_CNTL__LC_XMIT_N_FTS_MASK;
1734     data |= (0x24 << PCIE_LC_N_FTS_CNTL__LC_XMIT_N_FTS__SHIFT) |
1735         PCIE_LC_N_FTS_CNTL__LC_XMIT_N_FTS_OVERRIDE_EN_MASK;
1736     if (orig != data)
1737         WREG32_PCIE(ixPCIE_LC_N_FTS_CNTL, data);
1738
1739     orig = data = RREG32_PCIE(ixPCIE_LC_CNTL3);
1740     data |= PCIE_LC_CNTL3__LC_GO_TO_RECOVERY_MASK;
1741     if (orig != data)
1742         WREG32_PCIE(ixPCIE_LC_CNTL3, data);
1743
1744     orig = data = RREG32_PCIE(ixPCIE_P_CNTL);
1745     data |= PCIE_P_CNTL__P_IGNORE_EDB_ERR_MASK;
1746     if (orig != data)
1747         WREG32_PCIE(ixPCIE_P_CNTL, data);
1748
1749     orig = data = RREG32_PCIE(ixPCIE_LC_CNTL);
1750     data &= ~(PCIE_LC_CNTL__LC_L0S_INACTIVITY_MASK |
1751         PCIE_LC_CNTL__LC_L1_INACTIVITY_MASK);
1752     data |= PCIE_LC_CNTL__LC_PMI_TO_L1_DIS_MASK;
1753     if (!disable_l0s)
1754         data |= (7 << PCIE_LC_CNTL__LC_L0S_INACTIVITY__SHIFT);
1755
1756     if (!disable_l1) {
1757         data |= (7 << PCIE_LC_CNTL__LC_L1_INACTIVITY__SHIFT);
1758         data &= ~PCIE_LC_CNTL__LC_PMI_TO_L1_DIS_MASK;
1759         if (orig != data)
1760             WREG32_PCIE(ixPCIE_LC_CNTL, data);
1761
1762         if (!disable_plloff_in_l1) {
1763             bool clk_req_support;
1764
1765             orig = data = RREG32_PCIE(ixPB0_PIF_PWRDOWN_0);
1766             data &= ~(PB0_PIF_PWRDOWN_0__PLL_POWER_STATE_IN_OFF_0_MASK |
1767                 PB0_PIF_PWRDOWN_0__PLL_POWER_STATE_IN_TXS2_0_MASK);
1768             data |= (7 << PB0_PIF_PWRDOWN_0__PLL_POWER_STATE_IN_OFF_0__SHIFT) |
1769                 (7 << PB0_PIF_PWRDOWN_0__PLL_POWER_STATE_IN_TXS2_0__SHIFT);
1770             if (orig != data)
1771                 WREG32_PCIE(ixPB0_PIF_PWRDOWN_0, data);
1772
1773             orig = data = RREG32_PCIE(ixPB0_PIF_PWRDOWN_1);
1774             data &= ~(PB0_PIF_PWRDOWN_1__PLL_POWER_STATE_IN_OFF_1_MASK |
1775                 PB0_PIF_PWRDOWN_1__PLL_POWER_STATE_IN_TXS2_1_MASK);
1776             data |= (7 << PB0_PIF_PWRDOWN_1__PLL_POWER_STATE_IN_OFF_1__SHIFT) |
1777                 (7 << PB0_PIF_PWRDOWN_1__PLL_POWER_STATE_IN_TXS2_1__SHIFT);
1778             if (orig != data)
1779                 WREG32_PCIE(ixPB0_PIF_PWRDOWN_1, data);
1780
1781             orig = data = RREG32_PCIE(ixPB1_PIF_PWRDOWN_0);
1782             data &= ~(PB1_PIF_PWRDOWN_0__PLL_POWER_STATE_IN_OFF_0_MASK |
1783                 PB1_PIF_PWRDOWN_0__PLL_POWER_STATE_IN_TXS2_0_MASK);
1784             data |= (7 << PB1_PIF_PWRDOWN_0__PLL_POWER_STATE_IN_OFF_0__SHIFT) |
1785                 (7 << PB1_PIF_PWRDOWN_0__PLL_POWER_STATE_IN_TXS2_0__SHIFT);
1786             if (orig != data)
1787                 WREG32_PCIE(ixPB1_PIF_PWRDOWN_0, data);
1788
1789             orig = data = RREG32_PCIE(ixPB1_PIF_PWRDOWN_1);
1790             data &= ~(PB1_PIF_PWRDOWN_1__PLL_POWER_STATE_IN_OFF_1_MASK |
1791                 PB1_PIF_PWRDOWN_1__PLL_POWER_STATE_IN_TXS2_1_MASK);
1792             data |= (7 << PB1_PIF_PWRDOWN_1__PLL_POWER_STATE_IN_OFF_1__SHIFT) |
1793                 (7 << PB1_PIF_PWRDOWN_1__PLL_POWER_STATE_IN_TXS2_1__SHIFT);
1794             if (orig != data)
1795                 WREG32_PCIE(ixPB1_PIF_PWRDOWN_1, data);
1796
1797             orig = data = RREG32_PCIE(ixPCIE_LC_LINK_WIDTH_CNTL);
1798             data &= ~PCIE_LC_LINK_WIDTH_CNTL__LC_DYN_LANES_PWR_STATE_MASK;
1799             data |= ~(3 << PCIE_LC_LINK_WIDTH_CNTL__LC_DYN_LANES_PWR_STATE__SHIFT);
1800             if (orig != data)
1801                 WREG32_PCIE(ixPCIE_LC_LINK_WIDTH_CNTL, data);
1802
1803             if (!disable_clkreq) {
1804                 struct pci_dev *root = adev->pdev->bus->self;
1805                 u32 lnkcap;
1806
1807                 clk_req_support = false;
1808                 pcie_capability_read_dword(root, PCI_EXP_LNKCAP, &lnkcap);
1809                 if (lnkcap & PCI_EXP_LNKCAP_CLKPM)
1810                     clk_req_support = true;
1811             } else {
1812                 clk_req_support = false;
1813             }
1814
1815             if (clk_req_support) {
1816                 orig = data = RREG32_PCIE(ixPCIE_LC_CNTL2);
1817                 data |= PCIE_LC_CNTL2__LC_ALLOW_PDWN_IN_L1_MASK |
1818                     PCIE_LC_CNTL2__LC_ALLOW_PDWN_IN_L23_MASK;
1819                 if (orig != data)
1820                     WREG32_PCIE(ixPCIE_LC_CNTL2, data);
1821
1822                 orig = data = RREG32_SMC(ixTHM_CLK_CNTL);
1823                 data &= ~(THM_CLK_CNTL__CMON_CLK_SEL_MASK |
1824                     THM_CLK_CNTL__TMON_CLK_SEL_MASK);
1825                 data |= (1 << THM_CLK_CNTL__CMON_CLK_SEL__SHIFT) |
1826                     (1 << THM_CLK_CNTL__TMON_CLK_SEL__SHIFT);
1827                 if (orig != data)
1828                     WREG32_SMC(ixTHM_CLK_CNTL, data);
1829
1830                 orig = data = RREG32_SMC(ixMISC_CLK_CTRL);
1831                 data &= ~(MISC_CLK_CTRL__DEEP_SLEEP_CLK_SEL_MASK |
1832                     MISC_CLK_CTRL__ZCLK_SEL_MASK);
1833                 data |= (1 << MISC_CLK_CTRL__DEEP_SLEEP_CLK_SEL__SHIFT) |
1834                     (1 << MISC_CLK_CTRL__ZCLK_SEL__SHIFT);
1835                 if (orig != data)
1836                     WREG32_SMC(ixMISC_CLK_CTRL, data);
1837
1838                 orig = data = RREG32_SMC(ixCG_CLKPIN_CNTL);
1839                 data &= ~CG_CLKPIN_CNTL__BCLK_AS_XCLK_MASK;
1840                 if (orig != data)
1841                     WREG32_SMC(ixCG_CLKPIN_CNTL, data);
1842
1843                 orig = data = RREG32_SMC(ixCG_CLKPIN_CNTL_2);
1844                 data &= ~CG_CLKPIN_CNTL_2__FORCE_BIF_REFCLK_EN_MASK;
1845                 if (orig != data)
1846                     WREG32_SMC(ixCG_CLKPIN_CNTL_2, data);
1847
1848                 orig = data = RREG32_SMC(ixMPLL_BYPASSCLK_SEL);
1849                 data &= ~MPLL_BYPASSCLK_SEL__MPLL_CLKOUT_SEL_MASK;
1850                 data |= (4 << MPLL_BYPASSCLK_SEL__MPLL_CLKOUT_SEL__SHIFT);
1851                 if (orig != data)
1852                     WREG32_SMC(ixMPLL_BYPASSCLK_SEL, data);
1853             }
1854         }
1855     } else {
1856         if (orig != data)
1857             WREG32_PCIE(ixPCIE_LC_CNTL, data);
1858     }
1859
1860     orig = data = RREG32_PCIE(ixPCIE_CNTL2);
1861     data |= PCIE_CNTL2__SLV_MEM_LS_EN_MASK |
1862         PCIE_CNTL2__MST_MEM_LS_EN_MASK |
1863         PCIE_CNTL2__REPLAY_MEM_LS_EN_MASK;
1864     if (orig != data)
1865         WREG32_PCIE(ixPCIE_CNTL2, data);
1866
1867     if (!disable_l0s) {
1868         data = RREG32_PCIE(ixPCIE_LC_N_FTS_CNTL);
1869         if ((data & PCIE_LC_N_FTS_CNTL__LC_N_FTS_MASK) ==
1870                 PCIE_LC_N_FTS_CNTL__LC_N_FTS_MASK) {
1871             data = RREG32_PCIE(ixPCIE_LC_STATUS1);
1872             if ((data & PCIE_LC_STATUS1__LC_REVERSE_XMIT_MASK) &&
1873             (data & PCIE_LC_STATUS1__LC_REVERSE_RCVR_MASK)) {
1874                 orig = data = RREG32_PCIE(ixPCIE_LC_CNTL);
1875                 data &= ~PCIE_LC_CNTL__LC_L0S_INACTIVITY_MASK;
1876                 if (orig != data)
1877                     WREG32_PCIE(ixPCIE_LC_CNTL, data);
1878             }
1879         }
1880     }
1881 }
1882
1883 static uint32_t cik_get_rev_id(struct amdgpu_device *adev)
1884 {
1885     return (RREG32(mmCC_DRM_ID_STRAPS) & CC_DRM_ID_STRAPS__ATI_REV_ID_MASK)
1886         >> CC_DRM_ID_STRAPS__ATI_REV_ID__SHIFT;
1887 }
1888
1889 static void cik_flush_hdp(struct amdgpu_device *adev, struct amdgpu_ring *ring)
1890 {
1891     if (!ring || !ring->funcs->emit_wreg) {
1892         WREG32(mmHDP_MEM_COHERENCY_FLUSH_CNTL, 1);
1893         RREG32(mmHDP_MEM_COHERENCY_FLUSH_CNTL);
1894     } else {
1895         amdgpu_ring_emit_wreg(ring, mmHDP_MEM_COHERENCY_FLUSH_CNTL, 1);
1896     }
1897 }
1898
1899 static void cik_invalidate_hdp(struct amdgpu_device *adev,
1900                    struct amdgpu_ring *ring)
1901 {
1902     if (!ring || !ring->funcs->emit_wreg) {
1903         WREG32(mmHDP_DEBUG0, 1);
1904         RREG32(mmHDP_DEBUG0);
1905     } else {
1906         amdgpu_ring_emit_wreg(ring, mmHDP_DEBUG0, 1);
1907     }
1908 }
1909
1910 static bool cik_need_full_reset(struct amdgpu_device *adev)
1911 {
1912     /* change this when we support soft reset */
1913     return true;
1914 }
1915
1916 static void cik_get_pcie_usage(struct amdgpu_device *adev, uint64_t *count0,
1917                    uint64_t *count1)
1918 {
1919     uint32_t perfctr = 0;
1920     uint64_t cnt0_of, cnt1_of;
1921     int tmp;
1922
1923     /* This reports 0 on APUs, so return to avoid writing/reading registers
1924      * that may or may not be different from their GPU counterparts
1925      */
1926     if (adev->flags & AMD_IS_APU)
1927         return;
1928
1929     /* Set the 2 events that we wish to watch, defined above */
1930     /* Reg 40 is # received msgs, Reg 104 is # of posted requests sent */
1931     perfctr = REG_SET_FIELD(perfctr, PCIE_PERF_CNTL_TXCLK, EVENT0_SEL, 40);
1932     perfctr = REG_SET_FIELD(perfctr, PCIE_PERF_CNTL_TXCLK, EVENT1_SEL, 104);
1933
1934     /* Write to enable desired perf counters */
1935     WREG32_PCIE(ixPCIE_PERF_CNTL_TXCLK, perfctr);
1936     /* Zero out and enable the perf counters
1937      * Write 0x5:
1938      * Bit 0 = Start all counters(1)
1939      * Bit 2 = Global counter reset enable(1)
1940      */
1941     WREG32_PCIE(ixPCIE_PERF_COUNT_CNTL, 0x00000005);
1942
1943     msleep(1000);
1944
1945     /* Load the shadow and disable the perf counters
1946      * Write 0x2:
1947      * Bit 0 = Stop counters(0)
1948      * Bit 1 = Load the shadow counters(1)
1949      */
1950     WREG32_PCIE(ixPCIE_PERF_COUNT_CNTL, 0x00000002);
1951
1952     /* Read register values to get any >32bit overflow */
1953     tmp = RREG32_PCIE(ixPCIE_PERF_CNTL_TXCLK);
1954     cnt0_of = REG_GET_FIELD(tmp, PCIE_PERF_CNTL_TXCLK, COUNTER0_UPPER);
1955     cnt1_of = REG_GET_FIELD(tmp, PCIE_PERF_CNTL_TXCLK, COUNTER1_UPPER);
1956
1957     /* Get the values and add the overflow */
1958     *count0 = RREG32_PCIE(ixPCIE_PERF_COUNT0_TXCLK) | (cnt0_of << 32);
1959     *count1 = RREG32_PCIE(ixPCIE_PERF_COUNT1_TXCLK) | (cnt1_of << 32);
1960 }
1961
1962 static bool cik_need_reset_on_init(struct amdgpu_device *adev)
1963 {
1964     u32 clock_cntl, pc;
1965
1966     if (adev->flags & AMD_IS_APU)
1967         return false;
1968
1969     /* check if the SMC is already running */
1970     clock_cntl = RREG32_SMC(ixSMC_SYSCON_CLOCK_CNTL_0);
1971     pc = RREG32_SMC(ixSMC_PC_C);
1972     if ((0 == REG_GET_FIELD(clock_cntl, SMC_SYSCON_CLOCK_CNTL_0, ck_disable)) &&
1973         (0x20100 <= pc))
1974         return true;
1975
1976     return false;
1977 }
1978
1979 static uint64_t cik_get_pcie_replay_count(struct amdgpu_device *adev)
1980 {
1981     uint64_t nak_r, nak_g;
1982
1983     /* Get the number of NAKs received and generated */
1984     nak_r = RREG32_PCIE(ixPCIE_RX_NUM_NAK);
1985     nak_g = RREG32_PCIE(ixPCIE_RX_NUM_NAK_GENERATED);
1986
1987     /* Add the total number of NAKs, i.e the number of replays */
1988     return (nak_r + nak_g);
1989 }
1990
1991 static void cik_pre_asic_init(struct amdgpu_device *adev)
1992 {
1993 }
1994
1995 static const struct amdgpu_asic_funcs cik_asic_funcs =
1996 {
1997     .read_disabled_bios = &cik_read_disabled_bios,
1998     .read_bios_from_rom = &cik_read_bios_from_rom,
1999     .read_register = &cik_read_register,
2000     .reset = &cik_asic_reset,
2001     .reset_method = &cik_asic_reset_method,
2002     .set_vga_state = &cik_vga_set_state,
2003     .get_xclk = &cik_get_xclk,
2004     .set_uvd_clocks = &cik_set_uvd_clocks,
2005     .set_vce_clocks = &cik_set_vce_clocks,
2006     .get_config_memsize = &cik_get_config_memsize,
2007     .flush_hdp = &cik_flush_hdp,
2008     .invalidate_hdp = &cik_invalidate_hdp,
2009     .need_full_reset = &cik_need_full_reset,
2010     .init_doorbell_index = &legacy_doorbell_index_init,
2011     .get_pcie_usage = &cik_get_pcie_usage,
2012     .need_reset_on_init = &cik_need_reset_on_init,
2013     .get_pcie_replay_count = &cik_get_pcie_replay_count,
2014     .supports_baco = &cik_asic_supports_baco,
2015     .pre_asic_init = &cik_pre_asic_init,
2016     .query_video_codecs = &cik_query_video_codecs,
2017 };
2018
2019 static int cik_common_early_init(void *handle)
2020 {
2021     struct amdgpu_device *adev = (struct amdgpu_device *)handle;
2022
2023     adev->smc_rreg = &cik_smc_rreg;
2024     adev->smc_wreg = &cik_smc_wreg;
2025     adev->pcie_rreg = &cik_pcie_rreg;
2026     adev->pcie_wreg = &cik_pcie_wreg;
2027     adev->uvd_ctx_rreg = &cik_uvd_ctx_rreg;
2028     adev->uvd_ctx_wreg = &cik_uvd_ctx_wreg;
2029     adev->didt_rreg = &cik_didt_rreg;
2030     adev->didt_wreg = &cik_didt_wreg;
2031
2032     adev->asic_funcs = &cik_asic_funcs;
2033
2034     adev->rev_id = cik_get_rev_id(adev);
2035     adev->external_rev_id = 0xFF;
2036     switch (adev->asic_type) {
2037     case CHIP_BONAIRE:
2038         adev->cg_flags =
2039             AMD_CG_SUPPORT_GFX_MGCG |
2040             AMD_CG_SUPPORT_GFX_MGLS |
2041             /*AMD_CG_SUPPORT_GFX_CGCG |*/
2042             AMD_CG_SUPPORT_GFX_CGLS |
2043             AMD_CG_SUPPORT_GFX_CGTS |
2044             AMD_CG_SUPPORT_GFX_CGTS_LS |
2045             AMD_CG_SUPPORT_GFX_CP_LS |
2046             AMD_CG_SUPPORT_MC_LS |
2047             AMD_CG_SUPPORT_MC_MGCG |
2048             AMD_CG_SUPPORT_SDMA_MGCG |
2049             AMD_CG_SUPPORT_SDMA_LS |
2050             AMD_CG_SUPPORT_BIF_LS |
2051             AMD_CG_SUPPORT_VCE_MGCG |
2052             AMD_CG_SUPPORT_UVD_MGCG |
2053             AMD_CG_SUPPORT_HDP_LS |
2054             AMD_CG_SUPPORT_HDP_MGCG;
2055         adev->pg_flags = 0;
2056         adev->external_rev_id = adev->rev_id + 0x14;
2057         break;
2058     case CHIP_HAWAII:
2059         adev->cg_flags =
2060             AMD_CG_SUPPORT_GFX_MGCG |
2061             AMD_CG_SUPPORT_GFX_MGLS |
2062             /*AMD_CG_SUPPORT_GFX_CGCG |*/
2063             AMD_CG_SUPPORT_GFX_CGLS |
2064             AMD_CG_SUPPORT_GFX_CGTS |
2065             AMD_CG_SUPPORT_GFX_CP_LS |
2066             AMD_CG_SUPPORT_MC_LS |
2067             AMD_CG_SUPPORT_MC_MGCG |
2068             AMD_CG_SUPPORT_SDMA_MGCG |
2069             AMD_CG_SUPPORT_SDMA_LS |
2070             AMD_CG_SUPPORT_BIF_LS |
2071             AMD_CG_SUPPORT_VCE_MGCG |
2072             AMD_CG_SUPPORT_UVD_MGCG |
2073             AMD_CG_SUPPORT_HDP_LS |
2074             AMD_CG_SUPPORT_HDP_MGCG;
2075         adev->pg_flags = 0;
2076         adev->external_rev_id = 0x28;
2077         break;
2078     case CHIP_KAVERI:
2079         adev->cg_flags =
2080             AMD_CG_SUPPORT_GFX_MGCG |
2081             AMD_CG_SUPPORT_GFX_MGLS |
2082             /*AMD_CG_SUPPORT_GFX_CGCG |*/
2083             AMD_CG_SUPPORT_GFX_CGLS |
2084             AMD_CG_SUPPORT_GFX_CGTS |
2085             AMD_CG_SUPPORT_GFX_CGTS_LS |
2086             AMD_CG_SUPPORT_GFX_CP_LS |
2087             AMD_CG_SUPPORT_SDMA_MGCG |
2088             AMD_CG_SUPPORT_SDMA_LS |
2089             AMD_CG_SUPPORT_BIF_LS |
2090             AMD_CG_SUPPORT_VCE_MGCG |
2091             AMD_CG_SUPPORT_UVD_MGCG |
2092             AMD_CG_SUPPORT_HDP_LS |
2093             AMD_CG_SUPPORT_HDP_MGCG;
2094         adev->pg_flags =
2095             /*AMD_PG_SUPPORT_GFX_PG |
2096               AMD_PG_SUPPORT_GFX_SMG |
2097               AMD_PG_SUPPORT_GFX_DMG |*/
2098             AMD_PG_SUPPORT_UVD |
2099             AMD_PG_SUPPORT_VCE |
2100             /*  AMD_PG_SUPPORT_CP |
2101               AMD_PG_SUPPORT_GDS |
2102               AMD_PG_SUPPORT_RLC_SMU_HS |
2103               AMD_PG_SUPPORT_ACP |
2104               AMD_PG_SUPPORT_SAMU |*/
2105             0;
2106         if (adev->pdev->device == 0x1312 ||
2107             adev->pdev->device == 0x1316 ||
2108             adev->pdev->device == 0x1317)
2109             adev->external_rev_id = 0x41;
2110         else
2111             adev->external_rev_id = 0x1;
2112         break;
2113     case CHIP_KABINI:
2114     case CHIP_MULLINS:
2115         adev->cg_flags =
2116             AMD_CG_SUPPORT_GFX_MGCG |
2117             AMD_CG_SUPPORT_GFX_MGLS |
2118             /*AMD_CG_SUPPORT_GFX_CGCG |*/
2119             AMD_CG_SUPPORT_GFX_CGLS |
2120             AMD_CG_SUPPORT_GFX_CGTS |
2121             AMD_CG_SUPPORT_GFX_CGTS_LS |
2122             AMD_CG_SUPPORT_GFX_CP_LS |
2123             AMD_CG_SUPPORT_SDMA_MGCG |
2124             AMD_CG_SUPPORT_SDMA_LS |
2125             AMD_CG_SUPPORT_BIF_LS |
2126             AMD_CG_SUPPORT_VCE_MGCG |
2127             AMD_CG_SUPPORT_UVD_MGCG |
2128             AMD_CG_SUPPORT_HDP_LS |
2129             AMD_CG_SUPPORT_HDP_MGCG;
2130         adev->pg_flags =
2131             /*AMD_PG_SUPPORT_GFX_PG |
2132               AMD_PG_SUPPORT_GFX_SMG | */
2133             AMD_PG_SUPPORT_UVD |
2134             /*AMD_PG_SUPPORT_VCE |
2135               AMD_PG_SUPPORT_CP |
2136               AMD_PG_SUPPORT_GDS |
2137               AMD_PG_SUPPORT_RLC_SMU_HS |
2138               AMD_PG_SUPPORT_SAMU |*/
2139             0;
2140         if (adev->asic_type == CHIP_KABINI) {
2141             if (adev->rev_id == 0)
2142                 adev->external_rev_id = 0x81;
2143             else if (adev->rev_id == 1)
2144                 adev->external_rev_id = 0x82;
2145             else if (adev->rev_id == 2)
2146                 adev->external_rev_id = 0x85;
2147         } else
2148             adev->external_rev_id = adev->rev_id + 0xa1;
2149         break;
2150     default:
2151         /* FIXME: not supported yet */
2152         return -EINVAL;
2153     }
2154
2155     return 0;
2156 }
2157
2158 static int cik_common_sw_init(void *handle)
2159 {
2160     return 0;
2161 }
2162
2163 static int cik_common_sw_fini(void *handle)
2164 {
2165     return 0;
2166 }
2167
2168 static int cik_common_hw_init(void *handle)
2169 {
2170     struct amdgpu_device *adev = (struct amdgpu_device *)handle;
2171
2172     /* move the golden regs per IP block */
2173     cik_init_golden_registers(adev);
2174     /* enable pcie gen2/3 link */
2175     cik_pcie_gen3_enable(adev);
2176     /* enable aspm */
2177     cik_program_aspm(adev);
2178
2179     return 0;
2180 }
2181
2182 static int cik_common_hw_fini(void *handle)
2183 {
2184     return 0;
2185 }
2186
2187 static int cik_common_suspend(void *handle)
2188 {
2189     struct amdgpu_device *adev = (struct amdgpu_device *)handle;
2190
2191     return cik_common_hw_fini(adev);
2192 }
2193
2194 static int cik_common_resume(void *handle)
2195 {
2196     struct amdgpu_device *adev = (struct amdgpu_device *)handle;
2197
2198     return cik_common_hw_init(adev);
2199 }
2200
2201 static bool cik_common_is_idle(void *handle)
2202 {
2203     return true;
2204 }
2205
2206 static int cik_common_wait_for_idle(void *handle)
2207 {
2208     return 0;
2209 }
2210
2211 static int cik_common_soft_reset(void *handle)
2212 {
2213     /* XXX hard reset?? */
2214     return 0;
2215 }
2216
2217 static int cik_common_set_clockgating_state(void *handle,
2218                         enum amd_clockgating_state state)
2219 {
2220     return 0;
2221 }
2222
2223 static int cik_common_set_powergating_state(void *handle,
2224                         enum amd_powergating_state state)
2225 {
2226     return 0;
2227 }
2228
2229 static const struct amd_ip_funcs cik_common_ip_funcs = {
2230     .name = "cik_common",
2231     .early_init = cik_common_early_init,
2232     .late_init = NULL,
2233     .sw_init = cik_common_sw_init,
2234     .sw_fini = cik_common_sw_fini,
2235     .hw_init = cik_common_hw_init,
2236     .hw_fini = cik_common_hw_fini,
2237     .suspend = cik_common_suspend,
2238     .resume = cik_common_resume,
2239     .is_idle = cik_common_is_idle,
2240     .wait_for_idle = cik_common_wait_for_idle,
2241     .soft_reset = cik_common_soft_reset,
2242     .set_clockgating_state = cik_common_set_clockgating_state,
2243     .set_powergating_state = cik_common_set_powergating_state,
2244 };
2245
2246 static const struct amdgpu_ip_block_version cik_common_ip_block =
2247 {
2248     .type = AMD_IP_BLOCK_TYPE_COMMON,
2249     .major = 1,
2250     .minor = 0,
2251     .rev = 0,
2252     .funcs = &cik_common_ip_funcs,
2253 };
2254
2255 int cik_set_ip_blocks(struct amdgpu_device *adev)
2256 {
2257     switch (adev->asic_type) {
2258     case CHIP_BONAIRE:
2259         amdgpu_device_ip_block_add(adev, &cik_common_ip_block);
2260         amdgpu_device_ip_block_add(adev, &gmc_v7_0_ip_block);
2261         amdgpu_device_ip_block_add(adev, &cik_ih_ip_block);
2262         amdgpu_device_ip_block_add(adev, &gfx_v7_2_ip_block);
2263         amdgpu_device_ip_block_add(adev, &cik_sdma_ip_block);
2264         amdgpu_device_ip_block_add(adev, &pp_smu_ip_block);
2265         if (adev->enable_virtual_display)
2266             amdgpu_device_ip_block_add(adev, &amdgpu_vkms_ip_block);
2267 #if defined(CONFIG_DRM_AMD_DC)
2268         else if (amdgpu_device_has_dc_support(adev))
2269             amdgpu_device_ip_block_add(adev, &dm_ip_block);
2270 #endif
2271         else
2272             amdgpu_device_ip_block_add(adev, &dce_v8_2_ip_block);
2273         amdgpu_device_ip_block_add(adev, &uvd_v4_2_ip_block);
2274         amdgpu_device_ip_block_add(adev, &vce_v2_0_ip_block);
2275         break;
2276     case CHIP_HAWAII:
2277         amdgpu_device_ip_block_add(adev, &cik_common_ip_block);
2278         amdgpu_device_ip_block_add(adev, &gmc_v7_0_ip_block);
2279         amdgpu_device_ip_block_add(adev, &cik_ih_ip_block);
2280         amdgpu_device_ip_block_add(adev, &gfx_v7_3_ip_block);
2281         amdgpu_device_ip_block_add(adev, &cik_sdma_ip_block);
2282         amdgpu_device_ip_block_add(adev, &pp_smu_ip_block);
2283         if (adev->enable_virtual_display)
2284             amdgpu_device_ip_block_add(adev, &amdgpu_vkms_ip_block);
2285 #if defined(CONFIG_DRM_AMD_DC)
2286         else if (amdgpu_device_has_dc_support(adev))
2287             amdgpu_device_ip_block_add(adev, &dm_ip_block);
2288 #endif
2289         else
2290             amdgpu_device_ip_block_add(adev, &dce_v8_5_ip_block);
2291         amdgpu_device_ip_block_add(adev, &uvd_v4_2_ip_block);
2292         amdgpu_device_ip_block_add(adev, &vce_v2_0_ip_block);
2293         break;
2294     case CHIP_KAVERI:
2295         amdgpu_device_ip_block_add(adev, &cik_common_ip_block);
2296         amdgpu_device_ip_block_add(adev, &gmc_v7_0_ip_block);
2297         amdgpu_device_ip_block_add(adev, &cik_ih_ip_block);
2298         amdgpu_device_ip_block_add(adev, &gfx_v7_1_ip_block);
2299         amdgpu_device_ip_block_add(adev, &cik_sdma_ip_block);
2300         amdgpu_device_ip_block_add(adev, &kv_smu_ip_block);
2301         if (adev->enable_virtual_display)
2302             amdgpu_device_ip_block_add(adev, &amdgpu_vkms_ip_block);
2303 #if defined(CONFIG_DRM_AMD_DC)
2304         else if (amdgpu_device_has_dc_support(adev))
2305             amdgpu_device_ip_block_add(adev, &dm_ip_block);
2306 #endif
2307         else
2308             amdgpu_device_ip_block_add(adev, &dce_v8_1_ip_block);
2309
2310         amdgpu_device_ip_block_add(adev, &uvd_v4_2_ip_block);
2311         amdgpu_device_ip_block_add(adev, &vce_v2_0_ip_block);
2312         break;
2313     case CHIP_KABINI:
2314     case CHIP_MULLINS:
2315         amdgpu_device_ip_block_add(adev, &cik_common_ip_block);
2316         amdgpu_device_ip_block_add(adev, &gmc_v7_0_ip_block);
2317         amdgpu_device_ip_block_add(adev, &cik_ih_ip_block);
2318         amdgpu_device_ip_block_add(adev, &gfx_v7_2_ip_block);
2319         amdgpu_device_ip_block_add(adev, &cik_sdma_ip_block);
2320         amdgpu_device_ip_block_add(adev, &kv_smu_ip_block);
2321         if (adev->enable_virtual_display)
2322             amdgpu_device_ip_block_add(adev, &amdgpu_vkms_ip_block);
2323 #if defined(CONFIG_DRM_AMD_DC)
2324         else if (amdgpu_device_has_dc_support(adev))
2325             amdgpu_device_ip_block_add(adev, &dm_ip_block);
2326 #endif
2327         else
2328             amdgpu_device_ip_block_add(adev, &dce_v8_3_ip_block);
2329         amdgpu_device_ip_block_add(adev, &uvd_v4_2_ip_block);
2330         amdgpu_device_ip_block_add(adev, &vce_v2_0_ip_block);
2331         break;
2332     default:
2333         /* FIXME: not supported yet */
2334         return -EINVAL;
2335     }
2336     return 0;
2337 }