Back to home page

OSCL-LXR
 
 

     

0001 /*
0002  * Copyright 2012 Advanced Micro Devices, Inc.
0003  *
0004  * Permission is hereby granted, free of charge, to any person obtaining a
0005  * copy of this software and associated documentation files (the "Software"),
0006  * to deal in the Software without restriction, including without limitation
0007  * the rights to use, copy, modify, merge, publish, distribute, sublicense,
0008  * and/or sell copies of the Software, and to permit persons to whom the
0009  * Software is furnished to do so, subject to the following conditions:
0010  *
0011  * The above copyright notice and this permission notice shall be included in
0012  * all copies or substantial portions of the Software.
0013  *
0014  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
0015  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
0016  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
0017  * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
0018  * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
0019  * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
0020  * OTHER DEALINGS IN THE SOFTWARE.
0021  *
0022  * Authors: Alex Deucher
0023  */
0024 
0025 #include <linux/firmware.h>
0026 #include <linux/module.h>
0027 #include <linux/pci.h>
0028 #include <linux/slab.h>
0029 
0030 #include <drm/drm_vblank.h>
0031 
0032 #include "atom.h"
0033 #include "evergreen.h"
0034 #include "cik_blit_shaders.h"
0035 #include "cik.h"
0036 #include "cikd.h"
0037 #include "clearstate_ci.h"
0038 #include "r600.h"
0039 #include "radeon.h"
0040 #include "radeon_asic.h"
0041 #include "radeon_audio.h"
0042 #include "radeon_ucode.h"
0043 #include "si.h"
0044 #include "vce.h"
0045 
0046 #define SH_MEM_CONFIG_GFX_DEFAULT \
0047     ALIGNMENT_MODE(SH_MEM_ALIGNMENT_MODE_UNALIGNED)
0048 
0049 MODULE_FIRMWARE("radeon/BONAIRE_pfp.bin");
0050 MODULE_FIRMWARE("radeon/BONAIRE_me.bin");
0051 MODULE_FIRMWARE("radeon/BONAIRE_ce.bin");
0052 MODULE_FIRMWARE("radeon/BONAIRE_mec.bin");
0053 MODULE_FIRMWARE("radeon/BONAIRE_mc.bin");
0054 MODULE_FIRMWARE("radeon/BONAIRE_mc2.bin");
0055 MODULE_FIRMWARE("radeon/BONAIRE_rlc.bin");
0056 MODULE_FIRMWARE("radeon/BONAIRE_sdma.bin");
0057 MODULE_FIRMWARE("radeon/BONAIRE_smc.bin");
0058 
0059 MODULE_FIRMWARE("radeon/bonaire_pfp.bin");
0060 MODULE_FIRMWARE("radeon/bonaire_me.bin");
0061 MODULE_FIRMWARE("radeon/bonaire_ce.bin");
0062 MODULE_FIRMWARE("radeon/bonaire_mec.bin");
0063 MODULE_FIRMWARE("radeon/bonaire_mc.bin");
0064 MODULE_FIRMWARE("radeon/bonaire_rlc.bin");
0065 MODULE_FIRMWARE("radeon/bonaire_sdma.bin");
0066 MODULE_FIRMWARE("radeon/bonaire_smc.bin");
0067 MODULE_FIRMWARE("radeon/bonaire_k_smc.bin");
0068 
0069 MODULE_FIRMWARE("radeon/HAWAII_pfp.bin");
0070 MODULE_FIRMWARE("radeon/HAWAII_me.bin");
0071 MODULE_FIRMWARE("radeon/HAWAII_ce.bin");
0072 MODULE_FIRMWARE("radeon/HAWAII_mec.bin");
0073 MODULE_FIRMWARE("radeon/HAWAII_mc.bin");
0074 MODULE_FIRMWARE("radeon/HAWAII_mc2.bin");
0075 MODULE_FIRMWARE("radeon/HAWAII_rlc.bin");
0076 MODULE_FIRMWARE("radeon/HAWAII_sdma.bin");
0077 MODULE_FIRMWARE("radeon/HAWAII_smc.bin");
0078 
0079 MODULE_FIRMWARE("radeon/hawaii_pfp.bin");
0080 MODULE_FIRMWARE("radeon/hawaii_me.bin");
0081 MODULE_FIRMWARE("radeon/hawaii_ce.bin");
0082 MODULE_FIRMWARE("radeon/hawaii_mec.bin");
0083 MODULE_FIRMWARE("radeon/hawaii_mc.bin");
0084 MODULE_FIRMWARE("radeon/hawaii_rlc.bin");
0085 MODULE_FIRMWARE("radeon/hawaii_sdma.bin");
0086 MODULE_FIRMWARE("radeon/hawaii_smc.bin");
0087 MODULE_FIRMWARE("radeon/hawaii_k_smc.bin");
0088 
0089 MODULE_FIRMWARE("radeon/KAVERI_pfp.bin");
0090 MODULE_FIRMWARE("radeon/KAVERI_me.bin");
0091 MODULE_FIRMWARE("radeon/KAVERI_ce.bin");
0092 MODULE_FIRMWARE("radeon/KAVERI_mec.bin");
0093 MODULE_FIRMWARE("radeon/KAVERI_rlc.bin");
0094 MODULE_FIRMWARE("radeon/KAVERI_sdma.bin");
0095 
0096 MODULE_FIRMWARE("radeon/kaveri_pfp.bin");
0097 MODULE_FIRMWARE("radeon/kaveri_me.bin");
0098 MODULE_FIRMWARE("radeon/kaveri_ce.bin");
0099 MODULE_FIRMWARE("radeon/kaveri_mec.bin");
0100 MODULE_FIRMWARE("radeon/kaveri_mec2.bin");
0101 MODULE_FIRMWARE("radeon/kaveri_rlc.bin");
0102 MODULE_FIRMWARE("radeon/kaveri_sdma.bin");
0103 
0104 MODULE_FIRMWARE("radeon/KABINI_pfp.bin");
0105 MODULE_FIRMWARE("radeon/KABINI_me.bin");
0106 MODULE_FIRMWARE("radeon/KABINI_ce.bin");
0107 MODULE_FIRMWARE("radeon/KABINI_mec.bin");
0108 MODULE_FIRMWARE("radeon/KABINI_rlc.bin");
0109 MODULE_FIRMWARE("radeon/KABINI_sdma.bin");
0110 
0111 MODULE_FIRMWARE("radeon/kabini_pfp.bin");
0112 MODULE_FIRMWARE("radeon/kabini_me.bin");
0113 MODULE_FIRMWARE("radeon/kabini_ce.bin");
0114 MODULE_FIRMWARE("radeon/kabini_mec.bin");
0115 MODULE_FIRMWARE("radeon/kabini_rlc.bin");
0116 MODULE_FIRMWARE("radeon/kabini_sdma.bin");
0117 
0118 MODULE_FIRMWARE("radeon/MULLINS_pfp.bin");
0119 MODULE_FIRMWARE("radeon/MULLINS_me.bin");
0120 MODULE_FIRMWARE("radeon/MULLINS_ce.bin");
0121 MODULE_FIRMWARE("radeon/MULLINS_mec.bin");
0122 MODULE_FIRMWARE("radeon/MULLINS_rlc.bin");
0123 MODULE_FIRMWARE("radeon/MULLINS_sdma.bin");
0124 
0125 MODULE_FIRMWARE("radeon/mullins_pfp.bin");
0126 MODULE_FIRMWARE("radeon/mullins_me.bin");
0127 MODULE_FIRMWARE("radeon/mullins_ce.bin");
0128 MODULE_FIRMWARE("radeon/mullins_mec.bin");
0129 MODULE_FIRMWARE("radeon/mullins_rlc.bin");
0130 MODULE_FIRMWARE("radeon/mullins_sdma.bin");
0131 
0132 static u32 cik_get_cu_active_bitmap(struct radeon_device *rdev, u32 se, u32 sh);
0133 static void cik_rlc_stop(struct radeon_device *rdev);
0134 static void cik_pcie_gen3_enable(struct radeon_device *rdev);
0135 static void cik_program_aspm(struct radeon_device *rdev);
0136 static void cik_init_pg(struct radeon_device *rdev);
0137 static void cik_init_cg(struct radeon_device *rdev);
0138 static void cik_fini_pg(struct radeon_device *rdev);
0139 static void cik_fini_cg(struct radeon_device *rdev);
0140 static void cik_enable_gui_idle_interrupt(struct radeon_device *rdev,
0141                       bool enable);
0142 
0143 /**
0144  * cik_get_allowed_info_register - fetch the register for the info ioctl
0145  *
0146  * @rdev: radeon_device pointer
0147  * @reg: register offset in bytes
0148  * @val: register value
0149  *
0150  * Returns 0 for success or -EINVAL for an invalid register
0151  *
0152  */
0153 int cik_get_allowed_info_register(struct radeon_device *rdev,
0154                   u32 reg, u32 *val)
0155 {
0156     switch (reg) {
0157     case GRBM_STATUS:
0158     case GRBM_STATUS2:
0159     case GRBM_STATUS_SE0:
0160     case GRBM_STATUS_SE1:
0161     case GRBM_STATUS_SE2:
0162     case GRBM_STATUS_SE3:
0163     case SRBM_STATUS:
0164     case SRBM_STATUS2:
0165     case (SDMA0_STATUS_REG + SDMA0_REGISTER_OFFSET):
0166     case (SDMA0_STATUS_REG + SDMA1_REGISTER_OFFSET):
0167     case UVD_STATUS:
0168     /* TODO VCE */
0169         *val = RREG32(reg);
0170         return 0;
0171     default:
0172         return -EINVAL;
0173     }
0174 }
0175 
0176 /*
0177  * Indirect registers accessor
0178  */
0179 u32 cik_didt_rreg(struct radeon_device *rdev, u32 reg)
0180 {
0181     unsigned long flags;
0182     u32 r;
0183 
0184     spin_lock_irqsave(&rdev->didt_idx_lock, flags);
0185     WREG32(CIK_DIDT_IND_INDEX, (reg));
0186     r = RREG32(CIK_DIDT_IND_DATA);
0187     spin_unlock_irqrestore(&rdev->didt_idx_lock, flags);
0188     return r;
0189 }
0190 
0191 void cik_didt_wreg(struct radeon_device *rdev, u32 reg, u32 v)
0192 {
0193     unsigned long flags;
0194 
0195     spin_lock_irqsave(&rdev->didt_idx_lock, flags);
0196     WREG32(CIK_DIDT_IND_INDEX, (reg));
0197     WREG32(CIK_DIDT_IND_DATA, (v));
0198     spin_unlock_irqrestore(&rdev->didt_idx_lock, flags);
0199 }
0200 
0201 /* get temperature in millidegrees */
0202 int ci_get_temp(struct radeon_device *rdev)
0203 {
0204     u32 temp;
0205     int actual_temp = 0;
0206 
0207     temp = (RREG32_SMC(CG_MULT_THERMAL_STATUS) & CTF_TEMP_MASK) >>
0208         CTF_TEMP_SHIFT;
0209 
0210     if (temp & 0x200)
0211         actual_temp = 255;
0212     else
0213         actual_temp = temp & 0x1ff;
0214 
0215     return actual_temp * 1000;
0216 }
0217 
0218 /* get temperature in millidegrees */
0219 int kv_get_temp(struct radeon_device *rdev)
0220 {
0221     u32 temp;
0222     int actual_temp = 0;
0223 
0224     temp = RREG32_SMC(0xC0300E0C);
0225 
0226     if (temp)
0227         actual_temp = (temp / 8) - 49;
0228     else
0229         actual_temp = 0;
0230 
0231     return actual_temp * 1000;
0232 }
0233 
0234 /*
0235  * Indirect registers accessor
0236  */
0237 u32 cik_pciep_rreg(struct radeon_device *rdev, u32 reg)
0238 {
0239     unsigned long flags;
0240     u32 r;
0241 
0242     spin_lock_irqsave(&rdev->pciep_idx_lock, flags);
0243     WREG32(PCIE_INDEX, reg);
0244     (void)RREG32(PCIE_INDEX);
0245     r = RREG32(PCIE_DATA);
0246     spin_unlock_irqrestore(&rdev->pciep_idx_lock, flags);
0247     return r;
0248 }
0249 
0250 void cik_pciep_wreg(struct radeon_device *rdev, u32 reg, u32 v)
0251 {
0252     unsigned long flags;
0253 
0254     spin_lock_irqsave(&rdev->pciep_idx_lock, flags);
0255     WREG32(PCIE_INDEX, reg);
0256     (void)RREG32(PCIE_INDEX);
0257     WREG32(PCIE_DATA, v);
0258     (void)RREG32(PCIE_DATA);
0259     spin_unlock_irqrestore(&rdev->pciep_idx_lock, flags);
0260 }
0261 
0262 static const u32 spectre_rlc_save_restore_register_list[] =
0263 {
0264     (0x0e00 << 16) | (0xc12c >> 2),
0265     0x00000000,
0266     (0x0e00 << 16) | (0xc140 >> 2),
0267     0x00000000,
0268     (0x0e00 << 16) | (0xc150 >> 2),
0269     0x00000000,
0270     (0x0e00 << 16) | (0xc15c >> 2),
0271     0x00000000,
0272     (0x0e00 << 16) | (0xc168 >> 2),
0273     0x00000000,
0274     (0x0e00 << 16) | (0xc170 >> 2),
0275     0x00000000,
0276     (0x0e00 << 16) | (0xc178 >> 2),
0277     0x00000000,
0278     (0x0e00 << 16) | (0xc204 >> 2),
0279     0x00000000,
0280     (0x0e00 << 16) | (0xc2b4 >> 2),
0281     0x00000000,
0282     (0x0e00 << 16) | (0xc2b8 >> 2),
0283     0x00000000,
0284     (0x0e00 << 16) | (0xc2bc >> 2),
0285     0x00000000,
0286     (0x0e00 << 16) | (0xc2c0 >> 2),
0287     0x00000000,
0288     (0x0e00 << 16) | (0x8228 >> 2),
0289     0x00000000,
0290     (0x0e00 << 16) | (0x829c >> 2),
0291     0x00000000,
0292     (0x0e00 << 16) | (0x869c >> 2),
0293     0x00000000,
0294     (0x0600 << 16) | (0x98f4 >> 2),
0295     0x00000000,
0296     (0x0e00 << 16) | (0x98f8 >> 2),
0297     0x00000000,
0298     (0x0e00 << 16) | (0x9900 >> 2),
0299     0x00000000,
0300     (0x0e00 << 16) | (0xc260 >> 2),
0301     0x00000000,
0302     (0x0e00 << 16) | (0x90e8 >> 2),
0303     0x00000000,
0304     (0x0e00 << 16) | (0x3c000 >> 2),
0305     0x00000000,
0306     (0x0e00 << 16) | (0x3c00c >> 2),
0307     0x00000000,
0308     (0x0e00 << 16) | (0x8c1c >> 2),
0309     0x00000000,
0310     (0x0e00 << 16) | (0x9700 >> 2),
0311     0x00000000,
0312     (0x0e00 << 16) | (0xcd20 >> 2),
0313     0x00000000,
0314     (0x4e00 << 16) | (0xcd20 >> 2),
0315     0x00000000,
0316     (0x5e00 << 16) | (0xcd20 >> 2),
0317     0x00000000,
0318     (0x6e00 << 16) | (0xcd20 >> 2),
0319     0x00000000,
0320     (0x7e00 << 16) | (0xcd20 >> 2),
0321     0x00000000,
0322     (0x8e00 << 16) | (0xcd20 >> 2),
0323     0x00000000,
0324     (0x9e00 << 16) | (0xcd20 >> 2),
0325     0x00000000,
0326     (0xae00 << 16) | (0xcd20 >> 2),
0327     0x00000000,
0328     (0xbe00 << 16) | (0xcd20 >> 2),
0329     0x00000000,
0330     (0x0e00 << 16) | (0x89bc >> 2),
0331     0x00000000,
0332     (0x0e00 << 16) | (0x8900 >> 2),
0333     0x00000000,
0334     0x3,
0335     (0x0e00 << 16) | (0xc130 >> 2),
0336     0x00000000,
0337     (0x0e00 << 16) | (0xc134 >> 2),
0338     0x00000000,
0339     (0x0e00 << 16) | (0xc1fc >> 2),
0340     0x00000000,
0341     (0x0e00 << 16) | (0xc208 >> 2),
0342     0x00000000,
0343     (0x0e00 << 16) | (0xc264 >> 2),
0344     0x00000000,
0345     (0x0e00 << 16) | (0xc268 >> 2),
0346     0x00000000,
0347     (0x0e00 << 16) | (0xc26c >> 2),
0348     0x00000000,
0349     (0x0e00 << 16) | (0xc270 >> 2),
0350     0x00000000,
0351     (0x0e00 << 16) | (0xc274 >> 2),
0352     0x00000000,
0353     (0x0e00 << 16) | (0xc278 >> 2),
0354     0x00000000,
0355     (0x0e00 << 16) | (0xc27c >> 2),
0356     0x00000000,
0357     (0x0e00 << 16) | (0xc280 >> 2),
0358     0x00000000,
0359     (0x0e00 << 16) | (0xc284 >> 2),
0360     0x00000000,
0361     (0x0e00 << 16) | (0xc288 >> 2),
0362     0x00000000,
0363     (0x0e00 << 16) | (0xc28c >> 2),
0364     0x00000000,
0365     (0x0e00 << 16) | (0xc290 >> 2),
0366     0x00000000,
0367     (0x0e00 << 16) | (0xc294 >> 2),
0368     0x00000000,
0369     (0x0e00 << 16) | (0xc298 >> 2),
0370     0x00000000,
0371     (0x0e00 << 16) | (0xc29c >> 2),
0372     0x00000000,
0373     (0x0e00 << 16) | (0xc2a0 >> 2),
0374     0x00000000,
0375     (0x0e00 << 16) | (0xc2a4 >> 2),
0376     0x00000000,
0377     (0x0e00 << 16) | (0xc2a8 >> 2),
0378     0x00000000,
0379     (0x0e00 << 16) | (0xc2ac  >> 2),
0380     0x00000000,
0381     (0x0e00 << 16) | (0xc2b0 >> 2),
0382     0x00000000,
0383     (0x0e00 << 16) | (0x301d0 >> 2),
0384     0x00000000,
0385     (0x0e00 << 16) | (0x30238 >> 2),
0386     0x00000000,
0387     (0x0e00 << 16) | (0x30250 >> 2),
0388     0x00000000,
0389     (0x0e00 << 16) | (0x30254 >> 2),
0390     0x00000000,
0391     (0x0e00 << 16) | (0x30258 >> 2),
0392     0x00000000,
0393     (0x0e00 << 16) | (0x3025c >> 2),
0394     0x00000000,
0395     (0x4e00 << 16) | (0xc900 >> 2),
0396     0x00000000,
0397     (0x5e00 << 16) | (0xc900 >> 2),
0398     0x00000000,
0399     (0x6e00 << 16) | (0xc900 >> 2),
0400     0x00000000,
0401     (0x7e00 << 16) | (0xc900 >> 2),
0402     0x00000000,
0403     (0x8e00 << 16) | (0xc900 >> 2),
0404     0x00000000,
0405     (0x9e00 << 16) | (0xc900 >> 2),
0406     0x00000000,
0407     (0xae00 << 16) | (0xc900 >> 2),
0408     0x00000000,
0409     (0xbe00 << 16) | (0xc900 >> 2),
0410     0x00000000,
0411     (0x4e00 << 16) | (0xc904 >> 2),
0412     0x00000000,
0413     (0x5e00 << 16) | (0xc904 >> 2),
0414     0x00000000,
0415     (0x6e00 << 16) | (0xc904 >> 2),
0416     0x00000000,
0417     (0x7e00 << 16) | (0xc904 >> 2),
0418     0x00000000,
0419     (0x8e00 << 16) | (0xc904 >> 2),
0420     0x00000000,
0421     (0x9e00 << 16) | (0xc904 >> 2),
0422     0x00000000,
0423     (0xae00 << 16) | (0xc904 >> 2),
0424     0x00000000,
0425     (0xbe00 << 16) | (0xc904 >> 2),
0426     0x00000000,
0427     (0x4e00 << 16) | (0xc908 >> 2),
0428     0x00000000,
0429     (0x5e00 << 16) | (0xc908 >> 2),
0430     0x00000000,
0431     (0x6e00 << 16) | (0xc908 >> 2),
0432     0x00000000,
0433     (0x7e00 << 16) | (0xc908 >> 2),
0434     0x00000000,
0435     (0x8e00 << 16) | (0xc908 >> 2),
0436     0x00000000,
0437     (0x9e00 << 16) | (0xc908 >> 2),
0438     0x00000000,
0439     (0xae00 << 16) | (0xc908 >> 2),
0440     0x00000000,
0441     (0xbe00 << 16) | (0xc908 >> 2),
0442     0x00000000,
0443     (0x4e00 << 16) | (0xc90c >> 2),
0444     0x00000000,
0445     (0x5e00 << 16) | (0xc90c >> 2),
0446     0x00000000,
0447     (0x6e00 << 16) | (0xc90c >> 2),
0448     0x00000000,
0449     (0x7e00 << 16) | (0xc90c >> 2),
0450     0x00000000,
0451     (0x8e00 << 16) | (0xc90c >> 2),
0452     0x00000000,
0453     (0x9e00 << 16) | (0xc90c >> 2),
0454     0x00000000,
0455     (0xae00 << 16) | (0xc90c >> 2),
0456     0x00000000,
0457     (0xbe00 << 16) | (0xc90c >> 2),
0458     0x00000000,
0459     (0x4e00 << 16) | (0xc910 >> 2),
0460     0x00000000,
0461     (0x5e00 << 16) | (0xc910 >> 2),
0462     0x00000000,
0463     (0x6e00 << 16) | (0xc910 >> 2),
0464     0x00000000,
0465     (0x7e00 << 16) | (0xc910 >> 2),
0466     0x00000000,
0467     (0x8e00 << 16) | (0xc910 >> 2),
0468     0x00000000,
0469     (0x9e00 << 16) | (0xc910 >> 2),
0470     0x00000000,
0471     (0xae00 << 16) | (0xc910 >> 2),
0472     0x00000000,
0473     (0xbe00 << 16) | (0xc910 >> 2),
0474     0x00000000,
0475     (0x0e00 << 16) | (0xc99c >> 2),
0476     0x00000000,
0477     (0x0e00 << 16) | (0x9834 >> 2),
0478     0x00000000,
0479     (0x0000 << 16) | (0x30f00 >> 2),
0480     0x00000000,
0481     (0x0001 << 16) | (0x30f00 >> 2),
0482     0x00000000,
0483     (0x0000 << 16) | (0x30f04 >> 2),
0484     0x00000000,
0485     (0x0001 << 16) | (0x30f04 >> 2),
0486     0x00000000,
0487     (0x0000 << 16) | (0x30f08 >> 2),
0488     0x00000000,
0489     (0x0001 << 16) | (0x30f08 >> 2),
0490     0x00000000,
0491     (0x0000 << 16) | (0x30f0c >> 2),
0492     0x00000000,
0493     (0x0001 << 16) | (0x30f0c >> 2),
0494     0x00000000,
0495     (0x0600 << 16) | (0x9b7c >> 2),
0496     0x00000000,
0497     (0x0e00 << 16) | (0x8a14 >> 2),
0498     0x00000000,
0499     (0x0e00 << 16) | (0x8a18 >> 2),
0500     0x00000000,
0501     (0x0600 << 16) | (0x30a00 >> 2),
0502     0x00000000,
0503     (0x0e00 << 16) | (0x8bf0 >> 2),
0504     0x00000000,
0505     (0x0e00 << 16) | (0x8bcc >> 2),
0506     0x00000000,
0507     (0x0e00 << 16) | (0x8b24 >> 2),
0508     0x00000000,
0509     (0x0e00 << 16) | (0x30a04 >> 2),
0510     0x00000000,
0511     (0x0600 << 16) | (0x30a10 >> 2),
0512     0x00000000,
0513     (0x0600 << 16) | (0x30a14 >> 2),
0514     0x00000000,
0515     (0x0600 << 16) | (0x30a18 >> 2),
0516     0x00000000,
0517     (0x0600 << 16) | (0x30a2c >> 2),
0518     0x00000000,
0519     (0x0e00 << 16) | (0xc700 >> 2),
0520     0x00000000,
0521     (0x0e00 << 16) | (0xc704 >> 2),
0522     0x00000000,
0523     (0x0e00 << 16) | (0xc708 >> 2),
0524     0x00000000,
0525     (0x0e00 << 16) | (0xc768 >> 2),
0526     0x00000000,
0527     (0x0400 << 16) | (0xc770 >> 2),
0528     0x00000000,
0529     (0x0400 << 16) | (0xc774 >> 2),
0530     0x00000000,
0531     (0x0400 << 16) | (0xc778 >> 2),
0532     0x00000000,
0533     (0x0400 << 16) | (0xc77c >> 2),
0534     0x00000000,
0535     (0x0400 << 16) | (0xc780 >> 2),
0536     0x00000000,
0537     (0x0400 << 16) | (0xc784 >> 2),
0538     0x00000000,
0539     (0x0400 << 16) | (0xc788 >> 2),
0540     0x00000000,
0541     (0x0400 << 16) | (0xc78c >> 2),
0542     0x00000000,
0543     (0x0400 << 16) | (0xc798 >> 2),
0544     0x00000000,
0545     (0x0400 << 16) | (0xc79c >> 2),
0546     0x00000000,
0547     (0x0400 << 16) | (0xc7a0 >> 2),
0548     0x00000000,
0549     (0x0400 << 16) | (0xc7a4 >> 2),
0550     0x00000000,
0551     (0x0400 << 16) | (0xc7a8 >> 2),
0552     0x00000000,
0553     (0x0400 << 16) | (0xc7ac >> 2),
0554     0x00000000,
0555     (0x0400 << 16) | (0xc7b0 >> 2),
0556     0x00000000,
0557     (0x0400 << 16) | (0xc7b4 >> 2),
0558     0x00000000,
0559     (0x0e00 << 16) | (0x9100 >> 2),
0560     0x00000000,
0561     (0x0e00 << 16) | (0x3c010 >> 2),
0562     0x00000000,
0563     (0x0e00 << 16) | (0x92a8 >> 2),
0564     0x00000000,
0565     (0x0e00 << 16) | (0x92ac >> 2),
0566     0x00000000,
0567     (0x0e00 << 16) | (0x92b4 >> 2),
0568     0x00000000,
0569     (0x0e00 << 16) | (0x92b8 >> 2),
0570     0x00000000,
0571     (0x0e00 << 16) | (0x92bc >> 2),
0572     0x00000000,
0573     (0x0e00 << 16) | (0x92c0 >> 2),
0574     0x00000000,
0575     (0x0e00 << 16) | (0x92c4 >> 2),
0576     0x00000000,
0577     (0x0e00 << 16) | (0x92c8 >> 2),
0578     0x00000000,
0579     (0x0e00 << 16) | (0x92cc >> 2),
0580     0x00000000,
0581     (0x0e00 << 16) | (0x92d0 >> 2),
0582     0x00000000,
0583     (0x0e00 << 16) | (0x8c00 >> 2),
0584     0x00000000,
0585     (0x0e00 << 16) | (0x8c04 >> 2),
0586     0x00000000,
0587     (0x0e00 << 16) | (0x8c20 >> 2),
0588     0x00000000,
0589     (0x0e00 << 16) | (0x8c38 >> 2),
0590     0x00000000,
0591     (0x0e00 << 16) | (0x8c3c >> 2),
0592     0x00000000,
0593     (0x0e00 << 16) | (0xae00 >> 2),
0594     0x00000000,
0595     (0x0e00 << 16) | (0x9604 >> 2),
0596     0x00000000,
0597     (0x0e00 << 16) | (0xac08 >> 2),
0598     0x00000000,
0599     (0x0e00 << 16) | (0xac0c >> 2),
0600     0x00000000,
0601     (0x0e00 << 16) | (0xac10 >> 2),
0602     0x00000000,
0603     (0x0e00 << 16) | (0xac14 >> 2),
0604     0x00000000,
0605     (0x0e00 << 16) | (0xac58 >> 2),
0606     0x00000000,
0607     (0x0e00 << 16) | (0xac68 >> 2),
0608     0x00000000,
0609     (0x0e00 << 16) | (0xac6c >> 2),
0610     0x00000000,
0611     (0x0e00 << 16) | (0xac70 >> 2),
0612     0x00000000,
0613     (0x0e00 << 16) | (0xac74 >> 2),
0614     0x00000000,
0615     (0x0e00 << 16) | (0xac78 >> 2),
0616     0x00000000,
0617     (0x0e00 << 16) | (0xac7c >> 2),
0618     0x00000000,
0619     (0x0e00 << 16) | (0xac80 >> 2),
0620     0x00000000,
0621     (0x0e00 << 16) | (0xac84 >> 2),
0622     0x00000000,
0623     (0x0e00 << 16) | (0xac88 >> 2),
0624     0x00000000,
0625     (0x0e00 << 16) | (0xac8c >> 2),
0626     0x00000000,
0627     (0x0e00 << 16) | (0x970c >> 2),
0628     0x00000000,
0629     (0x0e00 << 16) | (0x9714 >> 2),
0630     0x00000000,
0631     (0x0e00 << 16) | (0x9718 >> 2),
0632     0x00000000,
0633     (0x0e00 << 16) | (0x971c >> 2),
0634     0x00000000,
0635     (0x0e00 << 16) | (0x31068 >> 2),
0636     0x00000000,
0637     (0x4e00 << 16) | (0x31068 >> 2),
0638     0x00000000,
0639     (0x5e00 << 16) | (0x31068 >> 2),
0640     0x00000000,
0641     (0x6e00 << 16) | (0x31068 >> 2),
0642     0x00000000,
0643     (0x7e00 << 16) | (0x31068 >> 2),
0644     0x00000000,
0645     (0x8e00 << 16) | (0x31068 >> 2),
0646     0x00000000,
0647     (0x9e00 << 16) | (0x31068 >> 2),
0648     0x00000000,
0649     (0xae00 << 16) | (0x31068 >> 2),
0650     0x00000000,
0651     (0xbe00 << 16) | (0x31068 >> 2),
0652     0x00000000,
0653     (0x0e00 << 16) | (0xcd10 >> 2),
0654     0x00000000,
0655     (0x0e00 << 16) | (0xcd14 >> 2),
0656     0x00000000,
0657     (0x0e00 << 16) | (0x88b0 >> 2),
0658     0x00000000,
0659     (0x0e00 << 16) | (0x88b4 >> 2),
0660     0x00000000,
0661     (0x0e00 << 16) | (0x88b8 >> 2),
0662     0x00000000,
0663     (0x0e00 << 16) | (0x88bc >> 2),
0664     0x00000000,
0665     (0x0400 << 16) | (0x89c0 >> 2),
0666     0x00000000,
0667     (0x0e00 << 16) | (0x88c4 >> 2),
0668     0x00000000,
0669     (0x0e00 << 16) | (0x88c8 >> 2),
0670     0x00000000,
0671     (0x0e00 << 16) | (0x88d0 >> 2),
0672     0x00000000,
0673     (0x0e00 << 16) | (0x88d4 >> 2),
0674     0x00000000,
0675     (0x0e00 << 16) | (0x88d8 >> 2),
0676     0x00000000,
0677     (0x0e00 << 16) | (0x8980 >> 2),
0678     0x00000000,
0679     (0x0e00 << 16) | (0x30938 >> 2),
0680     0x00000000,
0681     (0x0e00 << 16) | (0x3093c >> 2),
0682     0x00000000,
0683     (0x0e00 << 16) | (0x30940 >> 2),
0684     0x00000000,
0685     (0x0e00 << 16) | (0x89a0 >> 2),
0686     0x00000000,
0687     (0x0e00 << 16) | (0x30900 >> 2),
0688     0x00000000,
0689     (0x0e00 << 16) | (0x30904 >> 2),
0690     0x00000000,
0691     (0x0e00 << 16) | (0x89b4 >> 2),
0692     0x00000000,
0693     (0x0e00 << 16) | (0x3c210 >> 2),
0694     0x00000000,
0695     (0x0e00 << 16) | (0x3c214 >> 2),
0696     0x00000000,
0697     (0x0e00 << 16) | (0x3c218 >> 2),
0698     0x00000000,
0699     (0x0e00 << 16) | (0x8904 >> 2),
0700     0x00000000,
0701     0x5,
0702     (0x0e00 << 16) | (0x8c28 >> 2),
0703     (0x0e00 << 16) | (0x8c2c >> 2),
0704     (0x0e00 << 16) | (0x8c30 >> 2),
0705     (0x0e00 << 16) | (0x8c34 >> 2),
0706     (0x0e00 << 16) | (0x9600 >> 2),
0707 };
0708 
0709 static const u32 kalindi_rlc_save_restore_register_list[] =
0710 {
0711     (0x0e00 << 16) | (0xc12c >> 2),
0712     0x00000000,
0713     (0x0e00 << 16) | (0xc140 >> 2),
0714     0x00000000,
0715     (0x0e00 << 16) | (0xc150 >> 2),
0716     0x00000000,
0717     (0x0e00 << 16) | (0xc15c >> 2),
0718     0x00000000,
0719     (0x0e00 << 16) | (0xc168 >> 2),
0720     0x00000000,
0721     (0x0e00 << 16) | (0xc170 >> 2),
0722     0x00000000,
0723     (0x0e00 << 16) | (0xc204 >> 2),
0724     0x00000000,
0725     (0x0e00 << 16) | (0xc2b4 >> 2),
0726     0x00000000,
0727     (0x0e00 << 16) | (0xc2b8 >> 2),
0728     0x00000000,
0729     (0x0e00 << 16) | (0xc2bc >> 2),
0730     0x00000000,
0731     (0x0e00 << 16) | (0xc2c0 >> 2),
0732     0x00000000,
0733     (0x0e00 << 16) | (0x8228 >> 2),
0734     0x00000000,
0735     (0x0e00 << 16) | (0x829c >> 2),
0736     0x00000000,
0737     (0x0e00 << 16) | (0x869c >> 2),
0738     0x00000000,
0739     (0x0600 << 16) | (0x98f4 >> 2),
0740     0x00000000,
0741     (0x0e00 << 16) | (0x98f8 >> 2),
0742     0x00000000,
0743     (0x0e00 << 16) | (0x9900 >> 2),
0744     0x00000000,
0745     (0x0e00 << 16) | (0xc260 >> 2),
0746     0x00000000,
0747     (0x0e00 << 16) | (0x90e8 >> 2),
0748     0x00000000,
0749     (0x0e00 << 16) | (0x3c000 >> 2),
0750     0x00000000,
0751     (0x0e00 << 16) | (0x3c00c >> 2),
0752     0x00000000,
0753     (0x0e00 << 16) | (0x8c1c >> 2),
0754     0x00000000,
0755     (0x0e00 << 16) | (0x9700 >> 2),
0756     0x00000000,
0757     (0x0e00 << 16) | (0xcd20 >> 2),
0758     0x00000000,
0759     (0x4e00 << 16) | (0xcd20 >> 2),
0760     0x00000000,
0761     (0x5e00 << 16) | (0xcd20 >> 2),
0762     0x00000000,
0763     (0x6e00 << 16) | (0xcd20 >> 2),
0764     0x00000000,
0765     (0x7e00 << 16) | (0xcd20 >> 2),
0766     0x00000000,
0767     (0x0e00 << 16) | (0x89bc >> 2),
0768     0x00000000,
0769     (0x0e00 << 16) | (0x8900 >> 2),
0770     0x00000000,
0771     0x3,
0772     (0x0e00 << 16) | (0xc130 >> 2),
0773     0x00000000,
0774     (0x0e00 << 16) | (0xc134 >> 2),
0775     0x00000000,
0776     (0x0e00 << 16) | (0xc1fc >> 2),
0777     0x00000000,
0778     (0x0e00 << 16) | (0xc208 >> 2),
0779     0x00000000,
0780     (0x0e00 << 16) | (0xc264 >> 2),
0781     0x00000000,
0782     (0x0e00 << 16) | (0xc268 >> 2),
0783     0x00000000,
0784     (0x0e00 << 16) | (0xc26c >> 2),
0785     0x00000000,
0786     (0x0e00 << 16) | (0xc270 >> 2),
0787     0x00000000,
0788     (0x0e00 << 16) | (0xc274 >> 2),
0789     0x00000000,
0790     (0x0e00 << 16) | (0xc28c >> 2),
0791     0x00000000,
0792     (0x0e00 << 16) | (0xc290 >> 2),
0793     0x00000000,
0794     (0x0e00 << 16) | (0xc294 >> 2),
0795     0x00000000,
0796     (0x0e00 << 16) | (0xc298 >> 2),
0797     0x00000000,
0798     (0x0e00 << 16) | (0xc2a0 >> 2),
0799     0x00000000,
0800     (0x0e00 << 16) | (0xc2a4 >> 2),
0801     0x00000000,
0802     (0x0e00 << 16) | (0xc2a8 >> 2),
0803     0x00000000,
0804     (0x0e00 << 16) | (0xc2ac >> 2),
0805     0x00000000,
0806     (0x0e00 << 16) | (0x301d0 >> 2),
0807     0x00000000,
0808     (0x0e00 << 16) | (0x30238 >> 2),
0809     0x00000000,
0810     (0x0e00 << 16) | (0x30250 >> 2),
0811     0x00000000,
0812     (0x0e00 << 16) | (0x30254 >> 2),
0813     0x00000000,
0814     (0x0e00 << 16) | (0x30258 >> 2),
0815     0x00000000,
0816     (0x0e00 << 16) | (0x3025c >> 2),
0817     0x00000000,
0818     (0x4e00 << 16) | (0xc900 >> 2),
0819     0x00000000,
0820     (0x5e00 << 16) | (0xc900 >> 2),
0821     0x00000000,
0822     (0x6e00 << 16) | (0xc900 >> 2),
0823     0x00000000,
0824     (0x7e00 << 16) | (0xc900 >> 2),
0825     0x00000000,
0826     (0x4e00 << 16) | (0xc904 >> 2),
0827     0x00000000,
0828     (0x5e00 << 16) | (0xc904 >> 2),
0829     0x00000000,
0830     (0x6e00 << 16) | (0xc904 >> 2),
0831     0x00000000,
0832     (0x7e00 << 16) | (0xc904 >> 2),
0833     0x00000000,
0834     (0x4e00 << 16) | (0xc908 >> 2),
0835     0x00000000,
0836     (0x5e00 << 16) | (0xc908 >> 2),
0837     0x00000000,
0838     (0x6e00 << 16) | (0xc908 >> 2),
0839     0x00000000,
0840     (0x7e00 << 16) | (0xc908 >> 2),
0841     0x00000000,
0842     (0x4e00 << 16) | (0xc90c >> 2),
0843     0x00000000,
0844     (0x5e00 << 16) | (0xc90c >> 2),
0845     0x00000000,
0846     (0x6e00 << 16) | (0xc90c >> 2),
0847     0x00000000,
0848     (0x7e00 << 16) | (0xc90c >> 2),
0849     0x00000000,
0850     (0x4e00 << 16) | (0xc910 >> 2),
0851     0x00000000,
0852     (0x5e00 << 16) | (0xc910 >> 2),
0853     0x00000000,
0854     (0x6e00 << 16) | (0xc910 >> 2),
0855     0x00000000,
0856     (0x7e00 << 16) | (0xc910 >> 2),
0857     0x00000000,
0858     (0x0e00 << 16) | (0xc99c >> 2),
0859     0x00000000,
0860     (0x0e00 << 16) | (0x9834 >> 2),
0861     0x00000000,
0862     (0x0000 << 16) | (0x30f00 >> 2),
0863     0x00000000,
0864     (0x0000 << 16) | (0x30f04 >> 2),
0865     0x00000000,
0866     (0x0000 << 16) | (0x30f08 >> 2),
0867     0x00000000,
0868     (0x0000 << 16) | (0x30f0c >> 2),
0869     0x00000000,
0870     (0x0600 << 16) | (0x9b7c >> 2),
0871     0x00000000,
0872     (0x0e00 << 16) | (0x8a14 >> 2),
0873     0x00000000,
0874     (0x0e00 << 16) | (0x8a18 >> 2),
0875     0x00000000,
0876     (0x0600 << 16) | (0x30a00 >> 2),
0877     0x00000000,
0878     (0x0e00 << 16) | (0x8bf0 >> 2),
0879     0x00000000,
0880     (0x0e00 << 16) | (0x8bcc >> 2),
0881     0x00000000,
0882     (0x0e00 << 16) | (0x8b24 >> 2),
0883     0x00000000,
0884     (0x0e00 << 16) | (0x30a04 >> 2),
0885     0x00000000,
0886     (0x0600 << 16) | (0x30a10 >> 2),
0887     0x00000000,
0888     (0x0600 << 16) | (0x30a14 >> 2),
0889     0x00000000,
0890     (0x0600 << 16) | (0x30a18 >> 2),
0891     0x00000000,
0892     (0x0600 << 16) | (0x30a2c >> 2),
0893     0x00000000,
0894     (0x0e00 << 16) | (0xc700 >> 2),
0895     0x00000000,
0896     (0x0e00 << 16) | (0xc704 >> 2),
0897     0x00000000,
0898     (0x0e00 << 16) | (0xc708 >> 2),
0899     0x00000000,
0900     (0x0e00 << 16) | (0xc768 >> 2),
0901     0x00000000,
0902     (0x0400 << 16) | (0xc770 >> 2),
0903     0x00000000,
0904     (0x0400 << 16) | (0xc774 >> 2),
0905     0x00000000,
0906     (0x0400 << 16) | (0xc798 >> 2),
0907     0x00000000,
0908     (0x0400 << 16) | (0xc79c >> 2),
0909     0x00000000,
0910     (0x0e00 << 16) | (0x9100 >> 2),
0911     0x00000000,
0912     (0x0e00 << 16) | (0x3c010 >> 2),
0913     0x00000000,
0914     (0x0e00 << 16) | (0x8c00 >> 2),
0915     0x00000000,
0916     (0x0e00 << 16) | (0x8c04 >> 2),
0917     0x00000000,
0918     (0x0e00 << 16) | (0x8c20 >> 2),
0919     0x00000000,
0920     (0x0e00 << 16) | (0x8c38 >> 2),
0921     0x00000000,
0922     (0x0e00 << 16) | (0x8c3c >> 2),
0923     0x00000000,
0924     (0x0e00 << 16) | (0xae00 >> 2),
0925     0x00000000,
0926     (0x0e00 << 16) | (0x9604 >> 2),
0927     0x00000000,
0928     (0x0e00 << 16) | (0xac08 >> 2),
0929     0x00000000,
0930     (0x0e00 << 16) | (0xac0c >> 2),
0931     0x00000000,
0932     (0x0e00 << 16) | (0xac10 >> 2),
0933     0x00000000,
0934     (0x0e00 << 16) | (0xac14 >> 2),
0935     0x00000000,
0936     (0x0e00 << 16) | (0xac58 >> 2),
0937     0x00000000,
0938     (0x0e00 << 16) | (0xac68 >> 2),
0939     0x00000000,
0940     (0x0e00 << 16) | (0xac6c >> 2),
0941     0x00000000,
0942     (0x0e00 << 16) | (0xac70 >> 2),
0943     0x00000000,
0944     (0x0e00 << 16) | (0xac74 >> 2),
0945     0x00000000,
0946     (0x0e00 << 16) | (0xac78 >> 2),
0947     0x00000000,
0948     (0x0e00 << 16) | (0xac7c >> 2),
0949     0x00000000,
0950     (0x0e00 << 16) | (0xac80 >> 2),
0951     0x00000000,
0952     (0x0e00 << 16) | (0xac84 >> 2),
0953     0x00000000,
0954     (0x0e00 << 16) | (0xac88 >> 2),
0955     0x00000000,
0956     (0x0e00 << 16) | (0xac8c >> 2),
0957     0x00000000,
0958     (0x0e00 << 16) | (0x970c >> 2),
0959     0x00000000,
0960     (0x0e00 << 16) | (0x9714 >> 2),
0961     0x00000000,
0962     (0x0e00 << 16) | (0x9718 >> 2),
0963     0x00000000,
0964     (0x0e00 << 16) | (0x971c >> 2),
0965     0x00000000,
0966     (0x0e00 << 16) | (0x31068 >> 2),
0967     0x00000000,
0968     (0x4e00 << 16) | (0x31068 >> 2),
0969     0x00000000,
0970     (0x5e00 << 16) | (0x31068 >> 2),
0971     0x00000000,
0972     (0x6e00 << 16) | (0x31068 >> 2),
0973     0x00000000,
0974     (0x7e00 << 16) | (0x31068 >> 2),
0975     0x00000000,
0976     (0x0e00 << 16) | (0xcd10 >> 2),
0977     0x00000000,
0978     (0x0e00 << 16) | (0xcd14 >> 2),
0979     0x00000000,
0980     (0x0e00 << 16) | (0x88b0 >> 2),
0981     0x00000000,
0982     (0x0e00 << 16) | (0x88b4 >> 2),
0983     0x00000000,
0984     (0x0e00 << 16) | (0x88b8 >> 2),
0985     0x00000000,
0986     (0x0e00 << 16) | (0x88bc >> 2),
0987     0x00000000,
0988     (0x0400 << 16) | (0x89c0 >> 2),
0989     0x00000000,
0990     (0x0e00 << 16) | (0x88c4 >> 2),
0991     0x00000000,
0992     (0x0e00 << 16) | (0x88c8 >> 2),
0993     0x00000000,
0994     (0x0e00 << 16) | (0x88d0 >> 2),
0995     0x00000000,
0996     (0x0e00 << 16) | (0x88d4 >> 2),
0997     0x00000000,
0998     (0x0e00 << 16) | (0x88d8 >> 2),
0999     0x00000000,
1000     (0x0e00 << 16) | (0x8980 >> 2),
1001     0x00000000,
1002     (0x0e00 << 16) | (0x30938 >> 2),
1003     0x00000000,
1004     (0x0e00 << 16) | (0x3093c >> 2),
1005     0x00000000,
1006     (0x0e00 << 16) | (0x30940 >> 2),
1007     0x00000000,
1008     (0x0e00 << 16) | (0x89a0 >> 2),
1009     0x00000000,
1010     (0x0e00 << 16) | (0x30900 >> 2),
1011     0x00000000,
1012     (0x0e00 << 16) | (0x30904 >> 2),
1013     0x00000000,
1014     (0x0e00 << 16) | (0x89b4 >> 2),
1015     0x00000000,
1016     (0x0e00 << 16) | (0x3e1fc >> 2),
1017     0x00000000,
1018     (0x0e00 << 16) | (0x3c210 >> 2),
1019     0x00000000,
1020     (0x0e00 << 16) | (0x3c214 >> 2),
1021     0x00000000,
1022     (0x0e00 << 16) | (0x3c218 >> 2),
1023     0x00000000,
1024     (0x0e00 << 16) | (0x8904 >> 2),
1025     0x00000000,
1026     0x5,
1027     (0x0e00 << 16) | (0x8c28 >> 2),
1028     (0x0e00 << 16) | (0x8c2c >> 2),
1029     (0x0e00 << 16) | (0x8c30 >> 2),
1030     (0x0e00 << 16) | (0x8c34 >> 2),
1031     (0x0e00 << 16) | (0x9600 >> 2),
1032 };
1033 
1034 static const u32 bonaire_golden_spm_registers[] =
1035 {
1036     0x30800, 0xe0ffffff, 0xe0000000
1037 };
1038 
1039 static const u32 bonaire_golden_common_registers[] =
1040 {
1041     0xc770, 0xffffffff, 0x00000800,
1042     0xc774, 0xffffffff, 0x00000800,
1043     0xc798, 0xffffffff, 0x00007fbf,
1044     0xc79c, 0xffffffff, 0x00007faf
1045 };
1046 
1047 static const u32 bonaire_golden_registers[] =
1048 {
1049     0x3354, 0x00000333, 0x00000333,
1050     0x3350, 0x000c0fc0, 0x00040200,
1051     0x9a10, 0x00010000, 0x00058208,
1052     0x3c000, 0xffff1fff, 0x00140000,
1053     0x3c200, 0xfdfc0fff, 0x00000100,
1054     0x3c234, 0x40000000, 0x40000200,
1055     0x9830, 0xffffffff, 0x00000000,
1056     0x9834, 0xf00fffff, 0x00000400,
1057     0x9838, 0x0002021c, 0x00020200,
1058     0xc78, 0x00000080, 0x00000000,
1059     0x5bb0, 0x000000f0, 0x00000070,
1060     0x5bc0, 0xf0311fff, 0x80300000,
1061     0x98f8, 0x73773777, 0x12010001,
1062     0x350c, 0x00810000, 0x408af000,
1063     0x7030, 0x31000111, 0x00000011,
1064     0x2f48, 0x73773777, 0x12010001,
1065     0x220c, 0x00007fb6, 0x0021a1b1,
1066     0x2210, 0x00007fb6, 0x002021b1,
1067     0x2180, 0x00007fb6, 0x00002191,
1068     0x2218, 0x00007fb6, 0x002121b1,
1069     0x221c, 0x00007fb6, 0x002021b1,
1070     0x21dc, 0x00007fb6, 0x00002191,
1071     0x21e0, 0x00007fb6, 0x00002191,
1072     0x3628, 0x0000003f, 0x0000000a,
1073     0x362c, 0x0000003f, 0x0000000a,
1074     0x2ae4, 0x00073ffe, 0x000022a2,
1075     0x240c, 0x000007ff, 0x00000000,
1076     0x8a14, 0xf000003f, 0x00000007,
1077     0x8bf0, 0x00002001, 0x00000001,
1078     0x8b24, 0xffffffff, 0x00ffffff,
1079     0x30a04, 0x0000ff0f, 0x00000000,
1080     0x28a4c, 0x07ffffff, 0x06000000,
1081     0x4d8, 0x00000fff, 0x00000100,
1082     0x3e78, 0x00000001, 0x00000002,
1083     0x9100, 0x03000000, 0x0362c688,
1084     0x8c00, 0x000000ff, 0x00000001,
1085     0xe40, 0x00001fff, 0x00001fff,
1086     0x9060, 0x0000007f, 0x00000020,
1087     0x9508, 0x00010000, 0x00010000,
1088     0xac14, 0x000003ff, 0x000000f3,
1089     0xac0c, 0xffffffff, 0x00001032
1090 };
1091 
1092 static const u32 bonaire_mgcg_cgcg_init[] =
1093 {
1094     0xc420, 0xffffffff, 0xfffffffc,
1095     0x30800, 0xffffffff, 0xe0000000,
1096     0x3c2a0, 0xffffffff, 0x00000100,
1097     0x3c208, 0xffffffff, 0x00000100,
1098     0x3c2c0, 0xffffffff, 0xc0000100,
1099     0x3c2c8, 0xffffffff, 0xc0000100,
1100     0x3c2c4, 0xffffffff, 0xc0000100,
1101     0x55e4, 0xffffffff, 0x00600100,
1102     0x3c280, 0xffffffff, 0x00000100,
1103     0x3c214, 0xffffffff, 0x06000100,
1104     0x3c220, 0xffffffff, 0x00000100,
1105     0x3c218, 0xffffffff, 0x06000100,
1106     0x3c204, 0xffffffff, 0x00000100,
1107     0x3c2e0, 0xffffffff, 0x00000100,
1108     0x3c224, 0xffffffff, 0x00000100,
1109     0x3c200, 0xffffffff, 0x00000100,
1110     0x3c230, 0xffffffff, 0x00000100,
1111     0x3c234, 0xffffffff, 0x00000100,
1112     0x3c250, 0xffffffff, 0x00000100,
1113     0x3c254, 0xffffffff, 0x00000100,
1114     0x3c258, 0xffffffff, 0x00000100,
1115     0x3c25c, 0xffffffff, 0x00000100,
1116     0x3c260, 0xffffffff, 0x00000100,
1117     0x3c27c, 0xffffffff, 0x00000100,
1118     0x3c278, 0xffffffff, 0x00000100,
1119     0x3c210, 0xffffffff, 0x06000100,
1120     0x3c290, 0xffffffff, 0x00000100,
1121     0x3c274, 0xffffffff, 0x00000100,
1122     0x3c2b4, 0xffffffff, 0x00000100,
1123     0x3c2b0, 0xffffffff, 0x00000100,
1124     0x3c270, 0xffffffff, 0x00000100,
1125     0x30800, 0xffffffff, 0xe0000000,
1126     0x3c020, 0xffffffff, 0x00010000,
1127     0x3c024, 0xffffffff, 0x00030002,
1128     0x3c028, 0xffffffff, 0x00040007,
1129     0x3c02c, 0xffffffff, 0x00060005,
1130     0x3c030, 0xffffffff, 0x00090008,
1131     0x3c034, 0xffffffff, 0x00010000,
1132     0x3c038, 0xffffffff, 0x00030002,
1133     0x3c03c, 0xffffffff, 0x00040007,
1134     0x3c040, 0xffffffff, 0x00060005,
1135     0x3c044, 0xffffffff, 0x00090008,
1136     0x3c048, 0xffffffff, 0x00010000,
1137     0x3c04c, 0xffffffff, 0x00030002,
1138     0x3c050, 0xffffffff, 0x00040007,
1139     0x3c054, 0xffffffff, 0x00060005,
1140     0x3c058, 0xffffffff, 0x00090008,
1141     0x3c05c, 0xffffffff, 0x00010000,
1142     0x3c060, 0xffffffff, 0x00030002,
1143     0x3c064, 0xffffffff, 0x00040007,
1144     0x3c068, 0xffffffff, 0x00060005,
1145     0x3c06c, 0xffffffff, 0x00090008,
1146     0x3c070, 0xffffffff, 0x00010000,
1147     0x3c074, 0xffffffff, 0x00030002,
1148     0x3c078, 0xffffffff, 0x00040007,
1149     0x3c07c, 0xffffffff, 0x00060005,
1150     0x3c080, 0xffffffff, 0x00090008,
1151     0x3c084, 0xffffffff, 0x00010000,
1152     0x3c088, 0xffffffff, 0x00030002,
1153     0x3c08c, 0xffffffff, 0x00040007,
1154     0x3c090, 0xffffffff, 0x00060005,
1155     0x3c094, 0xffffffff, 0x00090008,
1156     0x3c098, 0xffffffff, 0x00010000,
1157     0x3c09c, 0xffffffff, 0x00030002,
1158     0x3c0a0, 0xffffffff, 0x00040007,
1159     0x3c0a4, 0xffffffff, 0x00060005,
1160     0x3c0a8, 0xffffffff, 0x00090008,
1161     0x3c000, 0xffffffff, 0x96e00200,
1162     0x8708, 0xffffffff, 0x00900100,
1163     0xc424, 0xffffffff, 0x0020003f,
1164     0x38, 0xffffffff, 0x0140001c,
1165     0x3c, 0x000f0000, 0x000f0000,
1166     0x220, 0xffffffff, 0xC060000C,
1167     0x224, 0xc0000fff, 0x00000100,
1168     0xf90, 0xffffffff, 0x00000100,
1169     0xf98, 0x00000101, 0x00000000,
1170     0x20a8, 0xffffffff, 0x00000104,
1171     0x55e4, 0xff000fff, 0x00000100,
1172     0x30cc, 0xc0000fff, 0x00000104,
1173     0xc1e4, 0x00000001, 0x00000001,
1174     0xd00c, 0xff000ff0, 0x00000100,
1175     0xd80c, 0xff000ff0, 0x00000100
1176 };
1177 
1178 static const u32 spectre_golden_spm_registers[] =
1179 {
1180     0x30800, 0xe0ffffff, 0xe0000000
1181 };
1182 
1183 static const u32 spectre_golden_common_registers[] =
1184 {
1185     0xc770, 0xffffffff, 0x00000800,
1186     0xc774, 0xffffffff, 0x00000800,
1187     0xc798, 0xffffffff, 0x00007fbf,
1188     0xc79c, 0xffffffff, 0x00007faf
1189 };
1190 
1191 static const u32 spectre_golden_registers[] =
1192 {
1193     0x3c000, 0xffff1fff, 0x96940200,
1194     0x3c00c, 0xffff0001, 0xff000000,
1195     0x3c200, 0xfffc0fff, 0x00000100,
1196     0x6ed8, 0x00010101, 0x00010000,
1197     0x9834, 0xf00fffff, 0x00000400,
1198     0x9838, 0xfffffffc, 0x00020200,
1199     0x5bb0, 0x000000f0, 0x00000070,
1200     0x5bc0, 0xf0311fff, 0x80300000,
1201     0x98f8, 0x73773777, 0x12010001,
1202     0x9b7c, 0x00ff0000, 0x00fc0000,
1203     0x2f48, 0x73773777, 0x12010001,
1204     0x8a14, 0xf000003f, 0x00000007,
1205     0x8b24, 0xffffffff, 0x00ffffff,
1206     0x28350, 0x3f3f3fff, 0x00000082,
1207     0x28354, 0x0000003f, 0x00000000,
1208     0x3e78, 0x00000001, 0x00000002,
1209     0x913c, 0xffff03df, 0x00000004,
1210     0xc768, 0x00000008, 0x00000008,
1211     0x8c00, 0x000008ff, 0x00000800,
1212     0x9508, 0x00010000, 0x00010000,
1213     0xac0c, 0xffffffff, 0x54763210,
1214     0x214f8, 0x01ff01ff, 0x00000002,
1215     0x21498, 0x007ff800, 0x00200000,
1216     0x2015c, 0xffffffff, 0x00000f40,
1217     0x30934, 0xffffffff, 0x00000001
1218 };
1219 
1220 static const u32 spectre_mgcg_cgcg_init[] =
1221 {
1222     0xc420, 0xffffffff, 0xfffffffc,
1223     0x30800, 0xffffffff, 0xe0000000,
1224     0x3c2a0, 0xffffffff, 0x00000100,
1225     0x3c208, 0xffffffff, 0x00000100,
1226     0x3c2c0, 0xffffffff, 0x00000100,
1227     0x3c2c8, 0xffffffff, 0x00000100,
1228     0x3c2c4, 0xffffffff, 0x00000100,
1229     0x55e4, 0xffffffff, 0x00600100,
1230     0x3c280, 0xffffffff, 0x00000100,
1231     0x3c214, 0xffffffff, 0x06000100,
1232     0x3c220, 0xffffffff, 0x00000100,
1233     0x3c218, 0xffffffff, 0x06000100,
1234     0x3c204, 0xffffffff, 0x00000100,
1235     0x3c2e0, 0xffffffff, 0x00000100,
1236     0x3c224, 0xffffffff, 0x00000100,
1237     0x3c200, 0xffffffff, 0x00000100,
1238     0x3c230, 0xffffffff, 0x00000100,
1239     0x3c234, 0xffffffff, 0x00000100,
1240     0x3c250, 0xffffffff, 0x00000100,
1241     0x3c254, 0xffffffff, 0x00000100,
1242     0x3c258, 0xffffffff, 0x00000100,
1243     0x3c25c, 0xffffffff, 0x00000100,
1244     0x3c260, 0xffffffff, 0x00000100,
1245     0x3c27c, 0xffffffff, 0x00000100,
1246     0x3c278, 0xffffffff, 0x00000100,
1247     0x3c210, 0xffffffff, 0x06000100,
1248     0x3c290, 0xffffffff, 0x00000100,
1249     0x3c274, 0xffffffff, 0x00000100,
1250     0x3c2b4, 0xffffffff, 0x00000100,
1251     0x3c2b0, 0xffffffff, 0x00000100,
1252     0x3c270, 0xffffffff, 0x00000100,
1253     0x30800, 0xffffffff, 0xe0000000,
1254     0x3c020, 0xffffffff, 0x00010000,
1255     0x3c024, 0xffffffff, 0x00030002,
1256     0x3c028, 0xffffffff, 0x00040007,
1257     0x3c02c, 0xffffffff, 0x00060005,
1258     0x3c030, 0xffffffff, 0x00090008,
1259     0x3c034, 0xffffffff, 0x00010000,
1260     0x3c038, 0xffffffff, 0x00030002,
1261     0x3c03c, 0xffffffff, 0x00040007,
1262     0x3c040, 0xffffffff, 0x00060005,
1263     0x3c044, 0xffffffff, 0x00090008,
1264     0x3c048, 0xffffffff, 0x00010000,
1265     0x3c04c, 0xffffffff, 0x00030002,
1266     0x3c050, 0xffffffff, 0x00040007,
1267     0x3c054, 0xffffffff, 0x00060005,
1268     0x3c058, 0xffffffff, 0x00090008,
1269     0x3c05c, 0xffffffff, 0x00010000,
1270     0x3c060, 0xffffffff, 0x00030002,
1271     0x3c064, 0xffffffff, 0x00040007,
1272     0x3c068, 0xffffffff, 0x00060005,
1273     0x3c06c, 0xffffffff, 0x00090008,
1274     0x3c070, 0xffffffff, 0x00010000,
1275     0x3c074, 0xffffffff, 0x00030002,
1276     0x3c078, 0xffffffff, 0x00040007,
1277     0x3c07c, 0xffffffff, 0x00060005,
1278     0x3c080, 0xffffffff, 0x00090008,
1279     0x3c084, 0xffffffff, 0x00010000,
1280     0x3c088, 0xffffffff, 0x00030002,
1281     0x3c08c, 0xffffffff, 0x00040007,
1282     0x3c090, 0xffffffff, 0x00060005,
1283     0x3c094, 0xffffffff, 0x00090008,
1284     0x3c098, 0xffffffff, 0x00010000,
1285     0x3c09c, 0xffffffff, 0x00030002,
1286     0x3c0a0, 0xffffffff, 0x00040007,
1287     0x3c0a4, 0xffffffff, 0x00060005,
1288     0x3c0a8, 0xffffffff, 0x00090008,
1289     0x3c0ac, 0xffffffff, 0x00010000,
1290     0x3c0b0, 0xffffffff, 0x00030002,
1291     0x3c0b4, 0xffffffff, 0x00040007,
1292     0x3c0b8, 0xffffffff, 0x00060005,
1293     0x3c0bc, 0xffffffff, 0x00090008,
1294     0x3c000, 0xffffffff, 0x96e00200,
1295     0x8708, 0xffffffff, 0x00900100,
1296     0xc424, 0xffffffff, 0x0020003f,
1297     0x38, 0xffffffff, 0x0140001c,
1298     0x3c, 0x000f0000, 0x000f0000,
1299     0x220, 0xffffffff, 0xC060000C,
1300     0x224, 0xc0000fff, 0x00000100,
1301     0xf90, 0xffffffff, 0x00000100,
1302     0xf98, 0x00000101, 0x00000000,
1303     0x20a8, 0xffffffff, 0x00000104,
1304     0x55e4, 0xff000fff, 0x00000100,
1305     0x30cc, 0xc0000fff, 0x00000104,
1306     0xc1e4, 0x00000001, 0x00000001,
1307     0xd00c, 0xff000ff0, 0x00000100,
1308     0xd80c, 0xff000ff0, 0x00000100
1309 };
1310 
1311 static const u32 kalindi_golden_spm_registers[] =
1312 {
1313     0x30800, 0xe0ffffff, 0xe0000000
1314 };
1315 
1316 static const u32 kalindi_golden_common_registers[] =
1317 {
1318     0xc770, 0xffffffff, 0x00000800,
1319     0xc774, 0xffffffff, 0x00000800,
1320     0xc798, 0xffffffff, 0x00007fbf,
1321     0xc79c, 0xffffffff, 0x00007faf
1322 };
1323 
1324 static const u32 kalindi_golden_registers[] =
1325 {
1326     0x3c000, 0xffffdfff, 0x6e944040,
1327     0x55e4, 0xff607fff, 0xfc000100,
1328     0x3c220, 0xff000fff, 0x00000100,
1329     0x3c224, 0xff000fff, 0x00000100,
1330     0x3c200, 0xfffc0fff, 0x00000100,
1331     0x6ed8, 0x00010101, 0x00010000,
1332     0x9830, 0xffffffff, 0x00000000,
1333     0x9834, 0xf00fffff, 0x00000400,
1334     0x5bb0, 0x000000f0, 0x00000070,
1335     0x5bc0, 0xf0311fff, 0x80300000,
1336     0x98f8, 0x73773777, 0x12010001,
1337     0x98fc, 0xffffffff, 0x00000010,
1338     0x9b7c, 0x00ff0000, 0x00fc0000,
1339     0x8030, 0x00001f0f, 0x0000100a,
1340     0x2f48, 0x73773777, 0x12010001,
1341     0x2408, 0x000fffff, 0x000c007f,
1342     0x8a14, 0xf000003f, 0x00000007,
1343     0x8b24, 0x3fff3fff, 0x00ffcfff,
1344     0x30a04, 0x0000ff0f, 0x00000000,
1345     0x28a4c, 0x07ffffff, 0x06000000,
1346     0x4d8, 0x00000fff, 0x00000100,
1347     0x3e78, 0x00000001, 0x00000002,
1348     0xc768, 0x00000008, 0x00000008,
1349     0x8c00, 0x000000ff, 0x00000003,
1350     0x214f8, 0x01ff01ff, 0x00000002,
1351     0x21498, 0x007ff800, 0x00200000,
1352     0x2015c, 0xffffffff, 0x00000f40,
1353     0x88c4, 0x001f3ae3, 0x00000082,
1354     0x88d4, 0x0000001f, 0x00000010,
1355     0x30934, 0xffffffff, 0x00000000
1356 };
1357 
1358 static const u32 kalindi_mgcg_cgcg_init[] =
1359 {
1360     0xc420, 0xffffffff, 0xfffffffc,
1361     0x30800, 0xffffffff, 0xe0000000,
1362     0x3c2a0, 0xffffffff, 0x00000100,
1363     0x3c208, 0xffffffff, 0x00000100,
1364     0x3c2c0, 0xffffffff, 0x00000100,
1365     0x3c2c8, 0xffffffff, 0x00000100,
1366     0x3c2c4, 0xffffffff, 0x00000100,
1367     0x55e4, 0xffffffff, 0x00600100,
1368     0x3c280, 0xffffffff, 0x00000100,
1369     0x3c214, 0xffffffff, 0x06000100,
1370     0x3c220, 0xffffffff, 0x00000100,
1371     0x3c218, 0xffffffff, 0x06000100,
1372     0x3c204, 0xffffffff, 0x00000100,
1373     0x3c2e0, 0xffffffff, 0x00000100,
1374     0x3c224, 0xffffffff, 0x00000100,
1375     0x3c200, 0xffffffff, 0x00000100,
1376     0x3c230, 0xffffffff, 0x00000100,
1377     0x3c234, 0xffffffff, 0x00000100,
1378     0x3c250, 0xffffffff, 0x00000100,
1379     0x3c254, 0xffffffff, 0x00000100,
1380     0x3c258, 0xffffffff, 0x00000100,
1381     0x3c25c, 0xffffffff, 0x00000100,
1382     0x3c260, 0xffffffff, 0x00000100,
1383     0x3c27c, 0xffffffff, 0x00000100,
1384     0x3c278, 0xffffffff, 0x00000100,
1385     0x3c210, 0xffffffff, 0x06000100,
1386     0x3c290, 0xffffffff, 0x00000100,
1387     0x3c274, 0xffffffff, 0x00000100,
1388     0x3c2b4, 0xffffffff, 0x00000100,
1389     0x3c2b0, 0xffffffff, 0x00000100,
1390     0x3c270, 0xffffffff, 0x00000100,
1391     0x30800, 0xffffffff, 0xe0000000,
1392     0x3c020, 0xffffffff, 0x00010000,
1393     0x3c024, 0xffffffff, 0x00030002,
1394     0x3c028, 0xffffffff, 0x00040007,
1395     0x3c02c, 0xffffffff, 0x00060005,
1396     0x3c030, 0xffffffff, 0x00090008,
1397     0x3c034, 0xffffffff, 0x00010000,
1398     0x3c038, 0xffffffff, 0x00030002,
1399     0x3c03c, 0xffffffff, 0x00040007,
1400     0x3c040, 0xffffffff, 0x00060005,
1401     0x3c044, 0xffffffff, 0x00090008,
1402     0x3c000, 0xffffffff, 0x96e00200,
1403     0x8708, 0xffffffff, 0x00900100,
1404     0xc424, 0xffffffff, 0x0020003f,
1405     0x38, 0xffffffff, 0x0140001c,
1406     0x3c, 0x000f0000, 0x000f0000,
1407     0x220, 0xffffffff, 0xC060000C,
1408     0x224, 0xc0000fff, 0x00000100,
1409     0x20a8, 0xffffffff, 0x00000104,
1410     0x55e4, 0xff000fff, 0x00000100,
1411     0x30cc, 0xc0000fff, 0x00000104,
1412     0xc1e4, 0x00000001, 0x00000001,
1413     0xd00c, 0xff000ff0, 0x00000100,
1414     0xd80c, 0xff000ff0, 0x00000100
1415 };
1416 
1417 static const u32 hawaii_golden_spm_registers[] =
1418 {
1419     0x30800, 0xe0ffffff, 0xe0000000
1420 };
1421 
1422 static const u32 hawaii_golden_common_registers[] =
1423 {
1424     0x30800, 0xffffffff, 0xe0000000,
1425     0x28350, 0xffffffff, 0x3a00161a,
1426     0x28354, 0xffffffff, 0x0000002e,
1427     0x9a10, 0xffffffff, 0x00018208,
1428     0x98f8, 0xffffffff, 0x12011003
1429 };
1430 
1431 static const u32 hawaii_golden_registers[] =
1432 {
1433     0x3354, 0x00000333, 0x00000333,
1434     0x9a10, 0x00010000, 0x00058208,
1435     0x9830, 0xffffffff, 0x00000000,
1436     0x9834, 0xf00fffff, 0x00000400,
1437     0x9838, 0x0002021c, 0x00020200,
1438     0xc78, 0x00000080, 0x00000000,
1439     0x5bb0, 0x000000f0, 0x00000070,
1440     0x5bc0, 0xf0311fff, 0x80300000,
1441     0x350c, 0x00810000, 0x408af000,
1442     0x7030, 0x31000111, 0x00000011,
1443     0x2f48, 0x73773777, 0x12010001,
1444     0x2120, 0x0000007f, 0x0000001b,
1445     0x21dc, 0x00007fb6, 0x00002191,
1446     0x3628, 0x0000003f, 0x0000000a,
1447     0x362c, 0x0000003f, 0x0000000a,
1448     0x2ae4, 0x00073ffe, 0x000022a2,
1449     0x240c, 0x000007ff, 0x00000000,
1450     0x8bf0, 0x00002001, 0x00000001,
1451     0x8b24, 0xffffffff, 0x00ffffff,
1452     0x30a04, 0x0000ff0f, 0x00000000,
1453     0x28a4c, 0x07ffffff, 0x06000000,
1454     0x3e78, 0x00000001, 0x00000002,
1455     0xc768, 0x00000008, 0x00000008,
1456     0xc770, 0x00000f00, 0x00000800,
1457     0xc774, 0x00000f00, 0x00000800,
1458     0xc798, 0x00ffffff, 0x00ff7fbf,
1459     0xc79c, 0x00ffffff, 0x00ff7faf,
1460     0x8c00, 0x000000ff, 0x00000800,
1461     0xe40, 0x00001fff, 0x00001fff,
1462     0x9060, 0x0000007f, 0x00000020,
1463     0x9508, 0x00010000, 0x00010000,
1464     0xae00, 0x00100000, 0x000ff07c,
1465     0xac14, 0x000003ff, 0x0000000f,
1466     0xac10, 0xffffffff, 0x7564fdec,
1467     0xac0c, 0xffffffff, 0x3120b9a8,
1468     0xac08, 0x20000000, 0x0f9c0000
1469 };
1470 
1471 static const u32 hawaii_mgcg_cgcg_init[] =
1472 {
1473     0xc420, 0xffffffff, 0xfffffffd,
1474     0x30800, 0xffffffff, 0xe0000000,
1475     0x3c2a0, 0xffffffff, 0x00000100,
1476     0x3c208, 0xffffffff, 0x00000100,
1477     0x3c2c0, 0xffffffff, 0x00000100,
1478     0x3c2c8, 0xffffffff, 0x00000100,
1479     0x3c2c4, 0xffffffff, 0x00000100,
1480     0x55e4, 0xffffffff, 0x00200100,
1481     0x3c280, 0xffffffff, 0x00000100,
1482     0x3c214, 0xffffffff, 0x06000100,
1483     0x3c220, 0xffffffff, 0x00000100,
1484     0x3c218, 0xffffffff, 0x06000100,
1485     0x3c204, 0xffffffff, 0x00000100,
1486     0x3c2e0, 0xffffffff, 0x00000100,
1487     0x3c224, 0xffffffff, 0x00000100,
1488     0x3c200, 0xffffffff, 0x00000100,
1489     0x3c230, 0xffffffff, 0x00000100,
1490     0x3c234, 0xffffffff, 0x00000100,
1491     0x3c250, 0xffffffff, 0x00000100,
1492     0x3c254, 0xffffffff, 0x00000100,
1493     0x3c258, 0xffffffff, 0x00000100,
1494     0x3c25c, 0xffffffff, 0x00000100,
1495     0x3c260, 0xffffffff, 0x00000100,
1496     0x3c27c, 0xffffffff, 0x00000100,
1497     0x3c278, 0xffffffff, 0x00000100,
1498     0x3c210, 0xffffffff, 0x06000100,
1499     0x3c290, 0xffffffff, 0x00000100,
1500     0x3c274, 0xffffffff, 0x00000100,
1501     0x3c2b4, 0xffffffff, 0x00000100,
1502     0x3c2b0, 0xffffffff, 0x00000100,
1503     0x3c270, 0xffffffff, 0x00000100,
1504     0x30800, 0xffffffff, 0xe0000000,
1505     0x3c020, 0xffffffff, 0x00010000,
1506     0x3c024, 0xffffffff, 0x00030002,
1507     0x3c028, 0xffffffff, 0x00040007,
1508     0x3c02c, 0xffffffff, 0x00060005,
1509     0x3c030, 0xffffffff, 0x00090008,
1510     0x3c034, 0xffffffff, 0x00010000,
1511     0x3c038, 0xffffffff, 0x00030002,
1512     0x3c03c, 0xffffffff, 0x00040007,
1513     0x3c040, 0xffffffff, 0x00060005,
1514     0x3c044, 0xffffffff, 0x00090008,
1515     0x3c048, 0xffffffff, 0x00010000,
1516     0x3c04c, 0xffffffff, 0x00030002,
1517     0x3c050, 0xffffffff, 0x00040007,
1518     0x3c054, 0xffffffff, 0x00060005,
1519     0x3c058, 0xffffffff, 0x00090008,
1520     0x3c05c, 0xffffffff, 0x00010000,
1521     0x3c060, 0xffffffff, 0x00030002,
1522     0x3c064, 0xffffffff, 0x00040007,
1523     0x3c068, 0xffffffff, 0x00060005,
1524     0x3c06c, 0xffffffff, 0x00090008,
1525     0x3c070, 0xffffffff, 0x00010000,
1526     0x3c074, 0xffffffff, 0x00030002,
1527     0x3c078, 0xffffffff, 0x00040007,
1528     0x3c07c, 0xffffffff, 0x00060005,
1529     0x3c080, 0xffffffff, 0x00090008,
1530     0x3c084, 0xffffffff, 0x00010000,
1531     0x3c088, 0xffffffff, 0x00030002,
1532     0x3c08c, 0xffffffff, 0x00040007,
1533     0x3c090, 0xffffffff, 0x00060005,
1534     0x3c094, 0xffffffff, 0x00090008,
1535     0x3c098, 0xffffffff, 0x00010000,
1536     0x3c09c, 0xffffffff, 0x00030002,
1537     0x3c0a0, 0xffffffff, 0x00040007,
1538     0x3c0a4, 0xffffffff, 0x00060005,
1539     0x3c0a8, 0xffffffff, 0x00090008,
1540     0x3c0ac, 0xffffffff, 0x00010000,
1541     0x3c0b0, 0xffffffff, 0x00030002,
1542     0x3c0b4, 0xffffffff, 0x00040007,
1543     0x3c0b8, 0xffffffff, 0x00060005,
1544     0x3c0bc, 0xffffffff, 0x00090008,
1545     0x3c0c0, 0xffffffff, 0x00010000,
1546     0x3c0c4, 0xffffffff, 0x00030002,
1547     0x3c0c8, 0xffffffff, 0x00040007,
1548     0x3c0cc, 0xffffffff, 0x00060005,
1549     0x3c0d0, 0xffffffff, 0x00090008,
1550     0x3c0d4, 0xffffffff, 0x00010000,
1551     0x3c0d8, 0xffffffff, 0x00030002,
1552     0x3c0dc, 0xffffffff, 0x00040007,
1553     0x3c0e0, 0xffffffff, 0x00060005,
1554     0x3c0e4, 0xffffffff, 0x00090008,
1555     0x3c0e8, 0xffffffff, 0x00010000,
1556     0x3c0ec, 0xffffffff, 0x00030002,
1557     0x3c0f0, 0xffffffff, 0x00040007,
1558     0x3c0f4, 0xffffffff, 0x00060005,
1559     0x3c0f8, 0xffffffff, 0x00090008,
1560     0xc318, 0xffffffff, 0x00020200,
1561     0x3350, 0xffffffff, 0x00000200,
1562     0x15c0, 0xffffffff, 0x00000400,
1563     0x55e8, 0xffffffff, 0x00000000,
1564     0x2f50, 0xffffffff, 0x00000902,
1565     0x3c000, 0xffffffff, 0x96940200,
1566     0x8708, 0xffffffff, 0x00900100,
1567     0xc424, 0xffffffff, 0x0020003f,
1568     0x38, 0xffffffff, 0x0140001c,
1569     0x3c, 0x000f0000, 0x000f0000,
1570     0x220, 0xffffffff, 0xc060000c,
1571     0x224, 0xc0000fff, 0x00000100,
1572     0xf90, 0xffffffff, 0x00000100,
1573     0xf98, 0x00000101, 0x00000000,
1574     0x20a8, 0xffffffff, 0x00000104,
1575     0x55e4, 0xff000fff, 0x00000100,
1576     0x30cc, 0xc0000fff, 0x00000104,
1577     0xc1e4, 0x00000001, 0x00000001,
1578     0xd00c, 0xff000ff0, 0x00000100,
1579     0xd80c, 0xff000ff0, 0x00000100
1580 };
1581 
1582 static const u32 godavari_golden_registers[] =
1583 {
1584     0x55e4, 0xff607fff, 0xfc000100,
1585     0x6ed8, 0x00010101, 0x00010000,
1586     0x9830, 0xffffffff, 0x00000000,
1587     0x98302, 0xf00fffff, 0x00000400,
1588     0x6130, 0xffffffff, 0x00010000,
1589     0x5bb0, 0x000000f0, 0x00000070,
1590     0x5bc0, 0xf0311fff, 0x80300000,
1591     0x98f8, 0x73773777, 0x12010001,
1592     0x98fc, 0xffffffff, 0x00000010,
1593     0x8030, 0x00001f0f, 0x0000100a,
1594     0x2f48, 0x73773777, 0x12010001,
1595     0x2408, 0x000fffff, 0x000c007f,
1596     0x8a14, 0xf000003f, 0x00000007,
1597     0x8b24, 0xffffffff, 0x00ff0fff,
1598     0x30a04, 0x0000ff0f, 0x00000000,
1599     0x28a4c, 0x07ffffff, 0x06000000,
1600     0x4d8, 0x00000fff, 0x00000100,
1601     0xd014, 0x00010000, 0x00810001,
1602     0xd814, 0x00010000, 0x00810001,
1603     0x3e78, 0x00000001, 0x00000002,
1604     0xc768, 0x00000008, 0x00000008,
1605     0xc770, 0x00000f00, 0x00000800,
1606     0xc774, 0x00000f00, 0x00000800,
1607     0xc798, 0x00ffffff, 0x00ff7fbf,
1608     0xc79c, 0x00ffffff, 0x00ff7faf,
1609     0x8c00, 0x000000ff, 0x00000001,
1610     0x214f8, 0x01ff01ff, 0x00000002,
1611     0x21498, 0x007ff800, 0x00200000,
1612     0x2015c, 0xffffffff, 0x00000f40,
1613     0x88c4, 0x001f3ae3, 0x00000082,
1614     0x88d4, 0x0000001f, 0x00000010,
1615     0x30934, 0xffffffff, 0x00000000
1616 };
1617 
1618 
1619 static void cik_init_golden_registers(struct radeon_device *rdev)
1620 {
1621     switch (rdev->family) {
1622     case CHIP_BONAIRE:
1623         radeon_program_register_sequence(rdev,
1624                          bonaire_mgcg_cgcg_init,
1625                          (const u32)ARRAY_SIZE(bonaire_mgcg_cgcg_init));
1626         radeon_program_register_sequence(rdev,
1627                          bonaire_golden_registers,
1628                          (const u32)ARRAY_SIZE(bonaire_golden_registers));
1629         radeon_program_register_sequence(rdev,
1630                          bonaire_golden_common_registers,
1631                          (const u32)ARRAY_SIZE(bonaire_golden_common_registers));
1632         radeon_program_register_sequence(rdev,
1633                          bonaire_golden_spm_registers,
1634                          (const u32)ARRAY_SIZE(bonaire_golden_spm_registers));
1635         break;
1636     case CHIP_KABINI:
1637         radeon_program_register_sequence(rdev,
1638                          kalindi_mgcg_cgcg_init,
1639                          (const u32)ARRAY_SIZE(kalindi_mgcg_cgcg_init));
1640         radeon_program_register_sequence(rdev,
1641                          kalindi_golden_registers,
1642                          (const u32)ARRAY_SIZE(kalindi_golden_registers));
1643         radeon_program_register_sequence(rdev,
1644                          kalindi_golden_common_registers,
1645                          (const u32)ARRAY_SIZE(kalindi_golden_common_registers));
1646         radeon_program_register_sequence(rdev,
1647                          kalindi_golden_spm_registers,
1648                          (const u32)ARRAY_SIZE(kalindi_golden_spm_registers));
1649         break;
1650     case CHIP_MULLINS:
1651         radeon_program_register_sequence(rdev,
1652                          kalindi_mgcg_cgcg_init,
1653                          (const u32)ARRAY_SIZE(kalindi_mgcg_cgcg_init));
1654         radeon_program_register_sequence(rdev,
1655                          godavari_golden_registers,
1656                          (const u32)ARRAY_SIZE(godavari_golden_registers));
1657         radeon_program_register_sequence(rdev,
1658                          kalindi_golden_common_registers,
1659                          (const u32)ARRAY_SIZE(kalindi_golden_common_registers));
1660         radeon_program_register_sequence(rdev,
1661                          kalindi_golden_spm_registers,
1662                          (const u32)ARRAY_SIZE(kalindi_golden_spm_registers));
1663         break;
1664     case CHIP_KAVERI:
1665         radeon_program_register_sequence(rdev,
1666                          spectre_mgcg_cgcg_init,
1667                          (const u32)ARRAY_SIZE(spectre_mgcg_cgcg_init));
1668         radeon_program_register_sequence(rdev,
1669                          spectre_golden_registers,
1670                          (const u32)ARRAY_SIZE(spectre_golden_registers));
1671         radeon_program_register_sequence(rdev,
1672                          spectre_golden_common_registers,
1673                          (const u32)ARRAY_SIZE(spectre_golden_common_registers));
1674         radeon_program_register_sequence(rdev,
1675                          spectre_golden_spm_registers,
1676                          (const u32)ARRAY_SIZE(spectre_golden_spm_registers));
1677         break;
1678     case CHIP_HAWAII:
1679         radeon_program_register_sequence(rdev,
1680                          hawaii_mgcg_cgcg_init,
1681                          (const u32)ARRAY_SIZE(hawaii_mgcg_cgcg_init));
1682         radeon_program_register_sequence(rdev,
1683                          hawaii_golden_registers,
1684                          (const u32)ARRAY_SIZE(hawaii_golden_registers));
1685         radeon_program_register_sequence(rdev,
1686                          hawaii_golden_common_registers,
1687                          (const u32)ARRAY_SIZE(hawaii_golden_common_registers));
1688         radeon_program_register_sequence(rdev,
1689                          hawaii_golden_spm_registers,
1690                          (const u32)ARRAY_SIZE(hawaii_golden_spm_registers));
1691         break;
1692     default:
1693         break;
1694     }
1695 }
1696 
1697 /**
1698  * cik_get_xclk - get the xclk
1699  *
1700  * @rdev: radeon_device pointer
1701  *
1702  * Returns the reference clock used by the gfx engine
1703  * (CIK).
1704  */
1705 u32 cik_get_xclk(struct radeon_device *rdev)
1706 {
1707     u32 reference_clock = rdev->clock.spll.reference_freq;
1708 
1709     if (rdev->flags & RADEON_IS_IGP) {
1710         if (RREG32_SMC(GENERAL_PWRMGT) & GPU_COUNTER_CLK)
1711             return reference_clock / 2;
1712     } else {
1713         if (RREG32_SMC(CG_CLKPIN_CNTL) & XTALIN_DIVIDE)
1714             return reference_clock / 4;
1715     }
1716     return reference_clock;
1717 }
1718 
1719 /**
1720  * cik_mm_rdoorbell - read a doorbell dword
1721  *
1722  * @rdev: radeon_device pointer
1723  * @index: doorbell index
1724  *
1725  * Returns the value in the doorbell aperture at the
1726  * requested doorbell index (CIK).
1727  */
1728 u32 cik_mm_rdoorbell(struct radeon_device *rdev, u32 index)
1729 {
1730     if (index < rdev->doorbell.num_doorbells) {
1731         return readl(rdev->doorbell.ptr + index);
1732     } else {
1733         DRM_ERROR("reading beyond doorbell aperture: 0x%08x!\n", index);
1734         return 0;
1735     }
1736 }
1737 
1738 /**
1739  * cik_mm_wdoorbell - write a doorbell dword
1740  *
1741  * @rdev: radeon_device pointer
1742  * @index: doorbell index
1743  * @v: value to write
1744  *
1745  * Writes @v to the doorbell aperture at the
1746  * requested doorbell index (CIK).
1747  */
1748 void cik_mm_wdoorbell(struct radeon_device *rdev, u32 index, u32 v)
1749 {
1750     if (index < rdev->doorbell.num_doorbells) {
1751         writel(v, rdev->doorbell.ptr + index);
1752     } else {
1753         DRM_ERROR("writing beyond doorbell aperture: 0x%08x!\n", index);
1754     }
1755 }
1756 
1757 #define BONAIRE_IO_MC_REGS_SIZE 36
1758 
1759 static const u32 bonaire_io_mc_regs[BONAIRE_IO_MC_REGS_SIZE][2] =
1760 {
1761     {0x00000070, 0x04400000},
1762     {0x00000071, 0x80c01803},
1763     {0x00000072, 0x00004004},
1764     {0x00000073, 0x00000100},
1765     {0x00000074, 0x00ff0000},
1766     {0x00000075, 0x34000000},
1767     {0x00000076, 0x08000014},
1768     {0x00000077, 0x00cc08ec},
1769     {0x00000078, 0x00000400},
1770     {0x00000079, 0x00000000},
1771     {0x0000007a, 0x04090000},
1772     {0x0000007c, 0x00000000},
1773     {0x0000007e, 0x4408a8e8},
1774     {0x0000007f, 0x00000304},
1775     {0x00000080, 0x00000000},
1776     {0x00000082, 0x00000001},
1777     {0x00000083, 0x00000002},
1778     {0x00000084, 0xf3e4f400},
1779     {0x00000085, 0x052024e3},
1780     {0x00000087, 0x00000000},
1781     {0x00000088, 0x01000000},
1782     {0x0000008a, 0x1c0a0000},
1783     {0x0000008b, 0xff010000},
1784     {0x0000008d, 0xffffefff},
1785     {0x0000008e, 0xfff3efff},
1786     {0x0000008f, 0xfff3efbf},
1787     {0x00000092, 0xf7ffffff},
1788     {0x00000093, 0xffffff7f},
1789     {0x00000095, 0x00101101},
1790     {0x00000096, 0x00000fff},
1791     {0x00000097, 0x00116fff},
1792     {0x00000098, 0x60010000},
1793     {0x00000099, 0x10010000},
1794     {0x0000009a, 0x00006000},
1795     {0x0000009b, 0x00001000},
1796     {0x0000009f, 0x00b48000}
1797 };
1798 
1799 #define HAWAII_IO_MC_REGS_SIZE 22
1800 
1801 static const u32 hawaii_io_mc_regs[HAWAII_IO_MC_REGS_SIZE][2] =
1802 {
1803     {0x0000007d, 0x40000000},
1804     {0x0000007e, 0x40180304},
1805     {0x0000007f, 0x0000ff00},
1806     {0x00000081, 0x00000000},
1807     {0x00000083, 0x00000800},
1808     {0x00000086, 0x00000000},
1809     {0x00000087, 0x00000100},
1810     {0x00000088, 0x00020100},
1811     {0x00000089, 0x00000000},
1812     {0x0000008b, 0x00040000},
1813     {0x0000008c, 0x00000100},
1814     {0x0000008e, 0xff010000},
1815     {0x00000090, 0xffffefff},
1816     {0x00000091, 0xfff3efff},
1817     {0x00000092, 0xfff3efbf},
1818     {0x00000093, 0xf7ffffff},
1819     {0x00000094, 0xffffff7f},
1820     {0x00000095, 0x00000fff},
1821     {0x00000096, 0x00116fff},
1822     {0x00000097, 0x60010000},
1823     {0x00000098, 0x10010000},
1824     {0x0000009f, 0x00c79000}
1825 };
1826 
1827 
1828 /**
1829  * cik_srbm_select - select specific register instances
1830  *
1831  * @rdev: radeon_device pointer
1832  * @me: selected ME (micro engine)
1833  * @pipe: pipe
1834  * @queue: queue
1835  * @vmid: VMID
1836  *
1837  * Switches the currently active registers instances.  Some
1838  * registers are instanced per VMID, others are instanced per
1839  * me/pipe/queue combination.
1840  */
1841 static void cik_srbm_select(struct radeon_device *rdev,
1842                 u32 me, u32 pipe, u32 queue, u32 vmid)
1843 {
1844     u32 srbm_gfx_cntl = (PIPEID(pipe & 0x3) |
1845                  MEID(me & 0x3) |
1846                  VMID(vmid & 0xf) |
1847                  QUEUEID(queue & 0x7));
1848     WREG32(SRBM_GFX_CNTL, srbm_gfx_cntl);
1849 }
1850 
1851 /* ucode loading */
1852 /**
1853  * ci_mc_load_microcode - load MC ucode into the hw
1854  *
1855  * @rdev: radeon_device pointer
1856  *
1857  * Load the GDDR MC ucode into the hw (CIK).
1858  * Returns 0 on success, error on failure.
1859  */
1860 int ci_mc_load_microcode(struct radeon_device *rdev)
1861 {
1862     const __be32 *fw_data = NULL;
1863     const __le32 *new_fw_data = NULL;
1864     u32 running, tmp;
1865     u32 *io_mc_regs = NULL;
1866     const __le32 *new_io_mc_regs = NULL;
1867     int i, regs_size, ucode_size;
1868 
1869     if (!rdev->mc_fw)
1870         return -EINVAL;
1871 
1872     if (rdev->new_fw) {
1873         const struct mc_firmware_header_v1_0 *hdr =
1874             (const struct mc_firmware_header_v1_0 *)rdev->mc_fw->data;
1875 
1876         radeon_ucode_print_mc_hdr(&hdr->header);
1877 
1878         regs_size = le32_to_cpu(hdr->io_debug_size_bytes) / (4 * 2);
1879         new_io_mc_regs = (const __le32 *)
1880             (rdev->mc_fw->data + le32_to_cpu(hdr->io_debug_array_offset_bytes));
1881         ucode_size = le32_to_cpu(hdr->header.ucode_size_bytes) / 4;
1882         new_fw_data = (const __le32 *)
1883             (rdev->mc_fw->data + le32_to_cpu(hdr->header.ucode_array_offset_bytes));
1884     } else {
1885         ucode_size = rdev->mc_fw->size / 4;
1886 
1887         switch (rdev->family) {
1888         case CHIP_BONAIRE:
1889             io_mc_regs = (u32 *)&bonaire_io_mc_regs;
1890             regs_size = BONAIRE_IO_MC_REGS_SIZE;
1891             break;
1892         case CHIP_HAWAII:
1893             io_mc_regs = (u32 *)&hawaii_io_mc_regs;
1894             regs_size = HAWAII_IO_MC_REGS_SIZE;
1895             break;
1896         default:
1897             return -EINVAL;
1898         }
1899         fw_data = (const __be32 *)rdev->mc_fw->data;
1900     }
1901 
1902     running = RREG32(MC_SEQ_SUP_CNTL) & RUN_MASK;
1903 
1904     if (running == 0) {
1905         /* reset the engine and set to writable */
1906         WREG32(MC_SEQ_SUP_CNTL, 0x00000008);
1907         WREG32(MC_SEQ_SUP_CNTL, 0x00000010);
1908 
1909         /* load mc io regs */
1910         for (i = 0; i < regs_size; i++) {
1911             if (rdev->new_fw) {
1912                 WREG32(MC_SEQ_IO_DEBUG_INDEX, le32_to_cpup(new_io_mc_regs++));
1913                 WREG32(MC_SEQ_IO_DEBUG_DATA, le32_to_cpup(new_io_mc_regs++));
1914             } else {
1915                 WREG32(MC_SEQ_IO_DEBUG_INDEX, io_mc_regs[(i << 1)]);
1916                 WREG32(MC_SEQ_IO_DEBUG_DATA, io_mc_regs[(i << 1) + 1]);
1917             }
1918         }
1919 
1920         tmp = RREG32(MC_SEQ_MISC0);
1921         if ((rdev->pdev->device == 0x6649) && ((tmp & 0xff00) == 0x5600)) {
1922             WREG32(MC_SEQ_IO_DEBUG_INDEX, 5);
1923             WREG32(MC_SEQ_IO_DEBUG_DATA, 0x00000023);
1924             WREG32(MC_SEQ_IO_DEBUG_INDEX, 9);
1925             WREG32(MC_SEQ_IO_DEBUG_DATA, 0x000001f0);
1926         }
1927 
1928         /* load the MC ucode */
1929         for (i = 0; i < ucode_size; i++) {
1930             if (rdev->new_fw)
1931                 WREG32(MC_SEQ_SUP_PGM, le32_to_cpup(new_fw_data++));
1932             else
1933                 WREG32(MC_SEQ_SUP_PGM, be32_to_cpup(fw_data++));
1934         }
1935 
1936         /* put the engine back into the active state */
1937         WREG32(MC_SEQ_SUP_CNTL, 0x00000008);
1938         WREG32(MC_SEQ_SUP_CNTL, 0x00000004);
1939         WREG32(MC_SEQ_SUP_CNTL, 0x00000001);
1940 
1941         /* wait for training to complete */
1942         for (i = 0; i < rdev->usec_timeout; i++) {
1943             if (RREG32(MC_SEQ_TRAIN_WAKEUP_CNTL) & TRAIN_DONE_D0)
1944                 break;
1945             udelay(1);
1946         }
1947         for (i = 0; i < rdev->usec_timeout; i++) {
1948             if (RREG32(MC_SEQ_TRAIN_WAKEUP_CNTL) & TRAIN_DONE_D1)
1949                 break;
1950             udelay(1);
1951         }
1952     }
1953 
1954     return 0;
1955 }
1956 
1957 /**
1958  * cik_init_microcode - load ucode images from disk
1959  *
1960  * @rdev: radeon_device pointer
1961  *
1962  * Use the firmware interface to load the ucode images into
1963  * the driver (not loaded into hw).
1964  * Returns 0 on success, error on failure.
1965  */
1966 static int cik_init_microcode(struct radeon_device *rdev)
1967 {
1968     const char *chip_name;
1969     const char *new_chip_name;
1970     size_t pfp_req_size, me_req_size, ce_req_size,
1971         mec_req_size, rlc_req_size, mc_req_size = 0,
1972         sdma_req_size, smc_req_size = 0, mc2_req_size = 0;
1973     char fw_name[30];
1974     int new_fw = 0;
1975     int err;
1976     int num_fw;
1977     bool new_smc = false;
1978 
1979     DRM_DEBUG("\n");
1980 
1981     switch (rdev->family) {
1982     case CHIP_BONAIRE:
1983         chip_name = "BONAIRE";
1984         if ((rdev->pdev->revision == 0x80) ||
1985             (rdev->pdev->revision == 0x81) ||
1986             (rdev->pdev->device == 0x665f))
1987             new_smc = true;
1988         new_chip_name = "bonaire";
1989         pfp_req_size = CIK_PFP_UCODE_SIZE * 4;
1990         me_req_size = CIK_ME_UCODE_SIZE * 4;
1991         ce_req_size = CIK_CE_UCODE_SIZE * 4;
1992         mec_req_size = CIK_MEC_UCODE_SIZE * 4;
1993         rlc_req_size = BONAIRE_RLC_UCODE_SIZE * 4;
1994         mc_req_size = BONAIRE_MC_UCODE_SIZE * 4;
1995         mc2_req_size = BONAIRE_MC2_UCODE_SIZE * 4;
1996         sdma_req_size = CIK_SDMA_UCODE_SIZE * 4;
1997         smc_req_size = ALIGN(BONAIRE_SMC_UCODE_SIZE, 4);
1998         num_fw = 8;
1999         break;
2000     case CHIP_HAWAII:
2001         chip_name = "HAWAII";
2002         if (rdev->pdev->revision == 0x80)
2003             new_smc = true;
2004         new_chip_name = "hawaii";
2005         pfp_req_size = CIK_PFP_UCODE_SIZE * 4;
2006         me_req_size = CIK_ME_UCODE_SIZE * 4;
2007         ce_req_size = CIK_CE_UCODE_SIZE * 4;
2008         mec_req_size = CIK_MEC_UCODE_SIZE * 4;
2009         rlc_req_size = BONAIRE_RLC_UCODE_SIZE * 4;
2010         mc_req_size = HAWAII_MC_UCODE_SIZE * 4;
2011         mc2_req_size = HAWAII_MC2_UCODE_SIZE * 4;
2012         sdma_req_size = CIK_SDMA_UCODE_SIZE * 4;
2013         smc_req_size = ALIGN(HAWAII_SMC_UCODE_SIZE, 4);
2014         num_fw = 8;
2015         break;
2016     case CHIP_KAVERI:
2017         chip_name = "KAVERI";
2018         new_chip_name = "kaveri";
2019         pfp_req_size = CIK_PFP_UCODE_SIZE * 4;
2020         me_req_size = CIK_ME_UCODE_SIZE * 4;
2021         ce_req_size = CIK_CE_UCODE_SIZE * 4;
2022         mec_req_size = CIK_MEC_UCODE_SIZE * 4;
2023         rlc_req_size = KV_RLC_UCODE_SIZE * 4;
2024         sdma_req_size = CIK_SDMA_UCODE_SIZE * 4;
2025         num_fw = 7;
2026         break;
2027     case CHIP_KABINI:
2028         chip_name = "KABINI";
2029         new_chip_name = "kabini";
2030         pfp_req_size = CIK_PFP_UCODE_SIZE * 4;
2031         me_req_size = CIK_ME_UCODE_SIZE * 4;
2032         ce_req_size = CIK_CE_UCODE_SIZE * 4;
2033         mec_req_size = CIK_MEC_UCODE_SIZE * 4;
2034         rlc_req_size = KB_RLC_UCODE_SIZE * 4;
2035         sdma_req_size = CIK_SDMA_UCODE_SIZE * 4;
2036         num_fw = 6;
2037         break;
2038     case CHIP_MULLINS:
2039         chip_name = "MULLINS";
2040         new_chip_name = "mullins";
2041         pfp_req_size = CIK_PFP_UCODE_SIZE * 4;
2042         me_req_size = CIK_ME_UCODE_SIZE * 4;
2043         ce_req_size = CIK_CE_UCODE_SIZE * 4;
2044         mec_req_size = CIK_MEC_UCODE_SIZE * 4;
2045         rlc_req_size = ML_RLC_UCODE_SIZE * 4;
2046         sdma_req_size = CIK_SDMA_UCODE_SIZE * 4;
2047         num_fw = 6;
2048         break;
2049     default: BUG();
2050     }
2051 
2052     DRM_INFO("Loading %s Microcode\n", new_chip_name);
2053 
2054     snprintf(fw_name, sizeof(fw_name), "radeon/%s_pfp.bin", new_chip_name);
2055     err = request_firmware(&rdev->pfp_fw, fw_name, rdev->dev);
2056     if (err) {
2057         snprintf(fw_name, sizeof(fw_name), "radeon/%s_pfp.bin", chip_name);
2058         err = request_firmware(&rdev->pfp_fw, fw_name, rdev->dev);
2059         if (err)
2060             goto out;
2061         if (rdev->pfp_fw->size != pfp_req_size) {
2062             pr_err("cik_cp: Bogus length %zu in firmware \"%s\"\n",
2063                    rdev->pfp_fw->size, fw_name);
2064             err = -EINVAL;
2065             goto out;
2066         }
2067     } else {
2068         err = radeon_ucode_validate(rdev->pfp_fw);
2069         if (err) {
2070             pr_err("cik_fw: validation failed for firmware \"%s\"\n",
2071                    fw_name);
2072             goto out;
2073         } else {
2074             new_fw++;
2075         }
2076     }
2077 
2078     snprintf(fw_name, sizeof(fw_name), "radeon/%s_me.bin", new_chip_name);
2079     err = request_firmware(&rdev->me_fw, fw_name, rdev->dev);
2080     if (err) {
2081         snprintf(fw_name, sizeof(fw_name), "radeon/%s_me.bin", chip_name);
2082         err = request_firmware(&rdev->me_fw, fw_name, rdev->dev);
2083         if (err)
2084             goto out;
2085         if (rdev->me_fw->size != me_req_size) {
2086             pr_err("cik_cp: Bogus length %zu in firmware \"%s\"\n",
2087                    rdev->me_fw->size, fw_name);
2088             err = -EINVAL;
2089         }
2090     } else {
2091         err = radeon_ucode_validate(rdev->me_fw);
2092         if (err) {
2093             pr_err("cik_fw: validation failed for firmware \"%s\"\n",
2094                    fw_name);
2095             goto out;
2096         } else {
2097             new_fw++;
2098         }
2099     }
2100 
2101     snprintf(fw_name, sizeof(fw_name), "radeon/%s_ce.bin", new_chip_name);
2102     err = request_firmware(&rdev->ce_fw, fw_name, rdev->dev);
2103     if (err) {
2104         snprintf(fw_name, sizeof(fw_name), "radeon/%s_ce.bin", chip_name);
2105         err = request_firmware(&rdev->ce_fw, fw_name, rdev->dev);
2106         if (err)
2107             goto out;
2108         if (rdev->ce_fw->size != ce_req_size) {
2109             pr_err("cik_cp: Bogus length %zu in firmware \"%s\"\n",
2110                    rdev->ce_fw->size, fw_name);
2111             err = -EINVAL;
2112         }
2113     } else {
2114         err = radeon_ucode_validate(rdev->ce_fw);
2115         if (err) {
2116             pr_err("cik_fw: validation failed for firmware \"%s\"\n",
2117                    fw_name);
2118             goto out;
2119         } else {
2120             new_fw++;
2121         }
2122     }
2123 
2124     snprintf(fw_name, sizeof(fw_name), "radeon/%s_mec.bin", new_chip_name);
2125     err = request_firmware(&rdev->mec_fw, fw_name, rdev->dev);
2126     if (err) {
2127         snprintf(fw_name, sizeof(fw_name), "radeon/%s_mec.bin", chip_name);
2128         err = request_firmware(&rdev->mec_fw, fw_name, rdev->dev);
2129         if (err)
2130             goto out;
2131         if (rdev->mec_fw->size != mec_req_size) {
2132             pr_err("cik_cp: Bogus length %zu in firmware \"%s\"\n",
2133                    rdev->mec_fw->size, fw_name);
2134             err = -EINVAL;
2135         }
2136     } else {
2137         err = radeon_ucode_validate(rdev->mec_fw);
2138         if (err) {
2139             pr_err("cik_fw: validation failed for firmware \"%s\"\n",
2140                    fw_name);
2141             goto out;
2142         } else {
2143             new_fw++;
2144         }
2145     }
2146 
2147     if (rdev->family == CHIP_KAVERI) {
2148         snprintf(fw_name, sizeof(fw_name), "radeon/%s_mec2.bin", new_chip_name);
2149         err = request_firmware(&rdev->mec2_fw, fw_name, rdev->dev);
2150         if (err) {
2151             goto out;
2152         } else {
2153             err = radeon_ucode_validate(rdev->mec2_fw);
2154             if (err) {
2155                 goto out;
2156             } else {
2157                 new_fw++;
2158             }
2159         }
2160     }
2161 
2162     snprintf(fw_name, sizeof(fw_name), "radeon/%s_rlc.bin", new_chip_name);
2163     err = request_firmware(&rdev->rlc_fw, fw_name, rdev->dev);
2164     if (err) {
2165         snprintf(fw_name, sizeof(fw_name), "radeon/%s_rlc.bin", chip_name);
2166         err = request_firmware(&rdev->rlc_fw, fw_name, rdev->dev);
2167         if (err)
2168             goto out;
2169         if (rdev->rlc_fw->size != rlc_req_size) {
2170             pr_err("cik_rlc: Bogus length %zu in firmware \"%s\"\n",
2171                    rdev->rlc_fw->size, fw_name);
2172             err = -EINVAL;
2173         }
2174     } else {
2175         err = radeon_ucode_validate(rdev->rlc_fw);
2176         if (err) {
2177             pr_err("cik_fw: validation failed for firmware \"%s\"\n",
2178                    fw_name);
2179             goto out;
2180         } else {
2181             new_fw++;
2182         }
2183     }
2184 
2185     snprintf(fw_name, sizeof(fw_name), "radeon/%s_sdma.bin", new_chip_name);
2186     err = request_firmware(&rdev->sdma_fw, fw_name, rdev->dev);
2187     if (err) {
2188         snprintf(fw_name, sizeof(fw_name), "radeon/%s_sdma.bin", chip_name);
2189         err = request_firmware(&rdev->sdma_fw, fw_name, rdev->dev);
2190         if (err)
2191             goto out;
2192         if (rdev->sdma_fw->size != sdma_req_size) {
2193             pr_err("cik_sdma: Bogus length %zu in firmware \"%s\"\n",
2194                    rdev->sdma_fw->size, fw_name);
2195             err = -EINVAL;
2196         }
2197     } else {
2198         err = radeon_ucode_validate(rdev->sdma_fw);
2199         if (err) {
2200             pr_err("cik_fw: validation failed for firmware \"%s\"\n",
2201                    fw_name);
2202             goto out;
2203         } else {
2204             new_fw++;
2205         }
2206     }
2207 
2208     /* No SMC, MC ucode on APUs */
2209     if (!(rdev->flags & RADEON_IS_IGP)) {
2210         snprintf(fw_name, sizeof(fw_name), "radeon/%s_mc.bin", new_chip_name);
2211         err = request_firmware(&rdev->mc_fw, fw_name, rdev->dev);
2212         if (err) {
2213             snprintf(fw_name, sizeof(fw_name), "radeon/%s_mc2.bin", chip_name);
2214             err = request_firmware(&rdev->mc_fw, fw_name, rdev->dev);
2215             if (err) {
2216                 snprintf(fw_name, sizeof(fw_name), "radeon/%s_mc.bin", chip_name);
2217                 err = request_firmware(&rdev->mc_fw, fw_name, rdev->dev);
2218                 if (err)
2219                     goto out;
2220             }
2221             if ((rdev->mc_fw->size != mc_req_size) &&
2222                 (rdev->mc_fw->size != mc2_req_size)){
2223                 pr_err("cik_mc: Bogus length %zu in firmware \"%s\"\n",
2224                        rdev->mc_fw->size, fw_name);
2225                 err = -EINVAL;
2226             }
2227             DRM_INFO("%s: %zu bytes\n", fw_name, rdev->mc_fw->size);
2228         } else {
2229             err = radeon_ucode_validate(rdev->mc_fw);
2230             if (err) {
2231                 pr_err("cik_fw: validation failed for firmware \"%s\"\n",
2232                        fw_name);
2233                 goto out;
2234             } else {
2235                 new_fw++;
2236             }
2237         }
2238 
2239         if (new_smc)
2240             snprintf(fw_name, sizeof(fw_name), "radeon/%s_k_smc.bin", new_chip_name);
2241         else
2242             snprintf(fw_name, sizeof(fw_name), "radeon/%s_smc.bin", new_chip_name);
2243         err = request_firmware(&rdev->smc_fw, fw_name, rdev->dev);
2244         if (err) {
2245             snprintf(fw_name, sizeof(fw_name), "radeon/%s_smc.bin", chip_name);
2246             err = request_firmware(&rdev->smc_fw, fw_name, rdev->dev);
2247             if (err) {
2248                 pr_err("smc: error loading firmware \"%s\"\n",
2249                        fw_name);
2250                 release_firmware(rdev->smc_fw);
2251                 rdev->smc_fw = NULL;
2252                 err = 0;
2253             } else if (rdev->smc_fw->size != smc_req_size) {
2254                 pr_err("cik_smc: Bogus length %zu in firmware \"%s\"\n",
2255                        rdev->smc_fw->size, fw_name);
2256                 err = -EINVAL;
2257             }
2258         } else {
2259             err = radeon_ucode_validate(rdev->smc_fw);
2260             if (err) {
2261                 pr_err("cik_fw: validation failed for firmware \"%s\"\n",
2262                        fw_name);
2263                 goto out;
2264             } else {
2265                 new_fw++;
2266             }
2267         }
2268     }
2269 
2270     if (new_fw == 0) {
2271         rdev->new_fw = false;
2272     } else if (new_fw < num_fw) {
2273         pr_err("ci_fw: mixing new and old firmware!\n");
2274         err = -EINVAL;
2275     } else {
2276         rdev->new_fw = true;
2277     }
2278 
2279 out:
2280     if (err) {
2281         if (err != -EINVAL)
2282             pr_err("cik_cp: Failed to load firmware \"%s\"\n",
2283                    fw_name);
2284         release_firmware(rdev->pfp_fw);
2285         rdev->pfp_fw = NULL;
2286         release_firmware(rdev->me_fw);
2287         rdev->me_fw = NULL;
2288         release_firmware(rdev->ce_fw);
2289         rdev->ce_fw = NULL;
2290         release_firmware(rdev->mec_fw);
2291         rdev->mec_fw = NULL;
2292         release_firmware(rdev->mec2_fw);
2293         rdev->mec2_fw = NULL;
2294         release_firmware(rdev->rlc_fw);
2295         rdev->rlc_fw = NULL;
2296         release_firmware(rdev->sdma_fw);
2297         rdev->sdma_fw = NULL;
2298         release_firmware(rdev->mc_fw);
2299         rdev->mc_fw = NULL;
2300         release_firmware(rdev->smc_fw);
2301         rdev->smc_fw = NULL;
2302     }
2303     return err;
2304 }
2305 
2306 /*
2307  * Core functions
2308  */
2309 /**
2310  * cik_tiling_mode_table_init - init the hw tiling table
2311  *
2312  * @rdev: radeon_device pointer
2313  *
2314  * Starting with SI, the tiling setup is done globally in a
2315  * set of 32 tiling modes.  Rather than selecting each set of
2316  * parameters per surface as on older asics, we just select
2317  * which index in the tiling table we want to use, and the
2318  * surface uses those parameters (CIK).
2319  */
2320 static void cik_tiling_mode_table_init(struct radeon_device *rdev)
2321 {
2322     u32 *tile = rdev->config.cik.tile_mode_array;
2323     u32 *macrotile = rdev->config.cik.macrotile_mode_array;
2324     const u32 num_tile_mode_states =
2325             ARRAY_SIZE(rdev->config.cik.tile_mode_array);
2326     const u32 num_secondary_tile_mode_states =
2327             ARRAY_SIZE(rdev->config.cik.macrotile_mode_array);
2328     u32 reg_offset, split_equal_to_row_size;
2329     u32 num_pipe_configs;
2330     u32 num_rbs = rdev->config.cik.max_backends_per_se *
2331         rdev->config.cik.max_shader_engines;
2332 
2333     switch (rdev->config.cik.mem_row_size_in_kb) {
2334     case 1:
2335         split_equal_to_row_size = ADDR_SURF_TILE_SPLIT_1KB;
2336         break;
2337     case 2:
2338     default:
2339         split_equal_to_row_size = ADDR_SURF_TILE_SPLIT_2KB;
2340         break;
2341     case 4:
2342         split_equal_to_row_size = ADDR_SURF_TILE_SPLIT_4KB;
2343         break;
2344     }
2345 
2346     num_pipe_configs = rdev->config.cik.max_tile_pipes;
2347     if (num_pipe_configs > 8)
2348         num_pipe_configs = 16;
2349 
2350     for (reg_offset = 0; reg_offset < num_tile_mode_states; reg_offset++)
2351         tile[reg_offset] = 0;
2352     for (reg_offset = 0; reg_offset < num_secondary_tile_mode_states; reg_offset++)
2353         macrotile[reg_offset] = 0;
2354 
2355     switch(num_pipe_configs) {
2356     case 16:
2357         tile[0] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2358                MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2359                PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) |
2360                TILE_SPLIT(ADDR_SURF_TILE_SPLIT_64B));
2361         tile[1] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2362                MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2363                PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) |
2364                TILE_SPLIT(ADDR_SURF_TILE_SPLIT_128B));
2365         tile[2] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2366                MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2367                PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) |
2368                TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B));
2369         tile[3] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2370                MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2371                PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) |
2372                TILE_SPLIT(ADDR_SURF_TILE_SPLIT_512B));
2373         tile[4] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2374                MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2375                PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) |
2376                TILE_SPLIT(split_equal_to_row_size));
2377         tile[5] = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2378                PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) |
2379                MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING));
2380         tile[6] = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2381                MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2382                PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) |
2383                TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B));
2384         tile[7] = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2385                MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2386                PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) |
2387                TILE_SPLIT(split_equal_to_row_size));
2388         tile[8] = (ARRAY_MODE(ARRAY_LINEAR_ALIGNED) |
2389                PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16));
2390         tile[9] = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2391                PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) |
2392                MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING));
2393         tile[10] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2394                 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) |
2395                 PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) |
2396                 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2397         tile[11] = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |
2398                 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) |
2399                 PIPE_CONFIG(ADDR_SURF_P16_32x32_8x16) |
2400                 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2401         tile[12] = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2402                 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) |
2403                 PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) |
2404                 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2405         tile[13] = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2406                 PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) |
2407                 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING));
2408         tile[14] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2409                 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
2410                 PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) |
2411                 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2412         tile[16] = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |
2413                 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
2414                 PIPE_CONFIG(ADDR_SURF_P16_32x32_8x16) |
2415                 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2416         tile[17] = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2417                 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
2418                 PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) |
2419                 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2420         tile[27] = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2421                 PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) |
2422                 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING));
2423         tile[28] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2424                 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) |
2425                 PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) |
2426                 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2427         tile[29] = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |
2428                 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) |
2429                 PIPE_CONFIG(ADDR_SURF_P16_32x32_8x16) |
2430                 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2431         tile[30] = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2432                 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) |
2433                 PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) |
2434                 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2435 
2436         macrotile[0] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2437                BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2438                MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
2439                NUM_BANKS(ADDR_SURF_16_BANK));
2440         macrotile[1] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2441                BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
2442                MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
2443                NUM_BANKS(ADDR_SURF_16_BANK));
2444         macrotile[2] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2445                BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2446                MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) |
2447                NUM_BANKS(ADDR_SURF_16_BANK));
2448         macrotile[3] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2449                BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2450                MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) |
2451                NUM_BANKS(ADDR_SURF_16_BANK));
2452         macrotile[4] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2453                BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2454                MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) |
2455                NUM_BANKS(ADDR_SURF_8_BANK));
2456         macrotile[5] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2457                BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2458                MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) |
2459                NUM_BANKS(ADDR_SURF_4_BANK));
2460         macrotile[6] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2461                BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2462                MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) |
2463                NUM_BANKS(ADDR_SURF_2_BANK));
2464         macrotile[8] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2465                BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2466                MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
2467                NUM_BANKS(ADDR_SURF_16_BANK));
2468         macrotile[9] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2469                BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
2470                MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
2471                NUM_BANKS(ADDR_SURF_16_BANK));
2472         macrotile[10] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2473                 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2474                 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) |
2475                 NUM_BANKS(ADDR_SURF_16_BANK));
2476         macrotile[11] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2477                 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2478                 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) |
2479                 NUM_BANKS(ADDR_SURF_8_BANK));
2480         macrotile[12] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2481                 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2482                 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) |
2483                 NUM_BANKS(ADDR_SURF_4_BANK));
2484         macrotile[13] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2485                 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2486                 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) |
2487                 NUM_BANKS(ADDR_SURF_2_BANK));
2488         macrotile[14] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2489                 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2490                 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) |
2491                 NUM_BANKS(ADDR_SURF_2_BANK));
2492 
2493         for (reg_offset = 0; reg_offset < num_tile_mode_states; reg_offset++)
2494             WREG32(GB_TILE_MODE0 + (reg_offset * 4), tile[reg_offset]);
2495         for (reg_offset = 0; reg_offset < num_secondary_tile_mode_states; reg_offset++)
2496             WREG32(GB_MACROTILE_MODE0 + (reg_offset * 4), macrotile[reg_offset]);
2497         break;
2498 
2499     case 8:
2500         tile[0] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2501                MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2502                PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
2503                TILE_SPLIT(ADDR_SURF_TILE_SPLIT_64B));
2504         tile[1] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2505                MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2506                PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
2507                TILE_SPLIT(ADDR_SURF_TILE_SPLIT_128B));
2508         tile[2] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2509                MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2510                PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
2511                TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B));
2512         tile[3] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2513                MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2514                PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
2515                TILE_SPLIT(ADDR_SURF_TILE_SPLIT_512B));
2516         tile[4] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2517                MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2518                PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
2519                TILE_SPLIT(split_equal_to_row_size));
2520         tile[5] = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2521                PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
2522                MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING));
2523         tile[6] = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2524                MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2525                PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
2526                TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B));
2527         tile[7] = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2528                MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2529                PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
2530                TILE_SPLIT(split_equal_to_row_size));
2531         tile[8] = (ARRAY_MODE(ARRAY_LINEAR_ALIGNED) |
2532                PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16));
2533         tile[9] = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2534                PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
2535                MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING));
2536         tile[10] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2537                 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) |
2538                 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
2539                 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2540         tile[11] = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |
2541                 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) |
2542                 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2543                 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2544         tile[12] = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2545                 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) |
2546                 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
2547                 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2548         tile[13] = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2549                 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
2550                 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING));
2551         tile[14] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2552                 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
2553                 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
2554                 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2555         tile[16] = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |
2556                 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
2557                 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2558                 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2559         tile[17] = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2560                 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
2561                 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
2562                 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2563         tile[27] = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2564                 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
2565                 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING));
2566         tile[28] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2567                 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) |
2568                 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
2569                 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2570         tile[29] = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |
2571                 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) |
2572                 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2573                 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2574         tile[30] = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2575                 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) |
2576                 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
2577                 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2578 
2579         macrotile[0] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2580                 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2581                 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
2582                 NUM_BANKS(ADDR_SURF_16_BANK));
2583         macrotile[1] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2584                 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
2585                 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
2586                 NUM_BANKS(ADDR_SURF_16_BANK));
2587         macrotile[2] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2588                 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2589                 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
2590                 NUM_BANKS(ADDR_SURF_16_BANK));
2591         macrotile[3] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2592                 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2593                 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
2594                 NUM_BANKS(ADDR_SURF_16_BANK));
2595         macrotile[4] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2596                 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2597                 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) |
2598                 NUM_BANKS(ADDR_SURF_8_BANK));
2599         macrotile[5] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2600                 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2601                 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) |
2602                 NUM_BANKS(ADDR_SURF_4_BANK));
2603         macrotile[6] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2604                 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2605                 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) |
2606                 NUM_BANKS(ADDR_SURF_2_BANK));
2607         macrotile[8] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2608                 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_8) |
2609                 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
2610                 NUM_BANKS(ADDR_SURF_16_BANK));
2611         macrotile[9] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2612                 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2613                 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
2614                 NUM_BANKS(ADDR_SURF_16_BANK));
2615         macrotile[10] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2616                 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
2617                 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
2618                 NUM_BANKS(ADDR_SURF_16_BANK));
2619         macrotile[11] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2620                 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2621                 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
2622                 NUM_BANKS(ADDR_SURF_16_BANK));
2623         macrotile[12] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2624                 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2625                 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) |
2626                 NUM_BANKS(ADDR_SURF_8_BANK));
2627         macrotile[13] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2628                 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2629                 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) |
2630                 NUM_BANKS(ADDR_SURF_4_BANK));
2631         macrotile[14] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2632                 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2633                 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) |
2634                 NUM_BANKS(ADDR_SURF_2_BANK));
2635 
2636         for (reg_offset = 0; reg_offset < num_tile_mode_states; reg_offset++)
2637             WREG32(GB_TILE_MODE0 + (reg_offset * 4), tile[reg_offset]);
2638         for (reg_offset = 0; reg_offset < num_secondary_tile_mode_states; reg_offset++)
2639             WREG32(GB_MACROTILE_MODE0 + (reg_offset * 4), macrotile[reg_offset]);
2640         break;
2641 
2642     case 4:
2643         if (num_rbs == 4) {
2644         tile[0] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2645                MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2646                PIPE_CONFIG(ADDR_SURF_P4_16x16) |
2647                TILE_SPLIT(ADDR_SURF_TILE_SPLIT_64B));
2648         tile[1] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2649                MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2650                PIPE_CONFIG(ADDR_SURF_P4_16x16) |
2651                TILE_SPLIT(ADDR_SURF_TILE_SPLIT_128B));
2652         tile[2] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2653                MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2654                PIPE_CONFIG(ADDR_SURF_P4_16x16) |
2655                TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B));
2656         tile[3] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2657                MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2658                PIPE_CONFIG(ADDR_SURF_P4_16x16) |
2659                TILE_SPLIT(ADDR_SURF_TILE_SPLIT_512B));
2660         tile[4] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2661                MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2662                PIPE_CONFIG(ADDR_SURF_P4_16x16) |
2663                TILE_SPLIT(split_equal_to_row_size));
2664         tile[5] = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2665                PIPE_CONFIG(ADDR_SURF_P4_16x16) |
2666                MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING));
2667         tile[6] = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2668                MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2669                PIPE_CONFIG(ADDR_SURF_P4_16x16) |
2670                TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B));
2671         tile[7] = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2672                MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2673                PIPE_CONFIG(ADDR_SURF_P4_16x16) |
2674                TILE_SPLIT(split_equal_to_row_size));
2675         tile[8] = (ARRAY_MODE(ARRAY_LINEAR_ALIGNED) |
2676                PIPE_CONFIG(ADDR_SURF_P4_16x16));
2677         tile[9] = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2678                PIPE_CONFIG(ADDR_SURF_P4_16x16) |
2679                MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING));
2680         tile[10] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2681                 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) |
2682                 PIPE_CONFIG(ADDR_SURF_P4_16x16) |
2683                 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2684         tile[11] = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |
2685                 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) |
2686                 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2687                 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2688         tile[12] = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2689                 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) |
2690                 PIPE_CONFIG(ADDR_SURF_P4_16x16) |
2691                 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2692         tile[13] = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2693                 PIPE_CONFIG(ADDR_SURF_P4_16x16) |
2694                 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING));
2695         tile[14] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2696                 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
2697                 PIPE_CONFIG(ADDR_SURF_P4_16x16) |
2698                 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2699         tile[16] = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |
2700                 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
2701                 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2702                 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2703         tile[17] = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2704                 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
2705                 PIPE_CONFIG(ADDR_SURF_P4_16x16) |
2706                 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2707         tile[27] = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2708                 PIPE_CONFIG(ADDR_SURF_P4_16x16) |
2709                 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING));
2710         tile[28] = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2711                 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) |
2712                 PIPE_CONFIG(ADDR_SURF_P4_16x16) |
2713                 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2714         tile[29] = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |
2715                 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) |
2716                 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2717                 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2718         tile[30] = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2719                 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) |
2720                 PIPE_CONFIG(ADDR_SURF_P4_16x16) |
2721                 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2722 
2723         } else if (num_rbs < 4) {
2724         tile[0] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2725                MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2726                PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2727                TILE_SPLIT(ADDR_SURF_TILE_SPLIT_64B));
2728         tile[1] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2729                MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2730                PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2731                TILE_SPLIT(ADDR_SURF_TILE_SPLIT_128B));
2732         tile[2] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2733                MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2734                PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2735                TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B));
2736         tile[3] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2737                MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2738                PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2739                TILE_SPLIT(ADDR_SURF_TILE_SPLIT_512B));
2740         tile[4] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2741                MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2742                PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2743                TILE_SPLIT(split_equal_to_row_size));
2744         tile[5] = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2745                PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2746                MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING));
2747         tile[6] = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2748                MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2749                PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2750                TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B));
2751         tile[7] = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2752                MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2753                PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2754                TILE_SPLIT(split_equal_to_row_size));
2755         tile[8] = (ARRAY_MODE(ARRAY_LINEAR_ALIGNED) |
2756                PIPE_CONFIG(ADDR_SURF_P4_8x16));
2757         tile[9] = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2758                PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2759                MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING));
2760         tile[10] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2761                 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) |
2762                 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2763                 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2764         tile[11] = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |
2765                 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) |
2766                 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2767                 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2768         tile[12] = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2769                 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) |
2770                 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2771                 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2772         tile[13] = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2773                 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2774                 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING));
2775         tile[14] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2776                 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
2777                 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2778                 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2779         tile[16] = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |
2780                 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
2781                 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2782                 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2783         tile[17] = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2784                 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
2785                 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2786                 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2787         tile[27] = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2788                 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2789                 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING));
2790         tile[28] = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2791                 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) |
2792                 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2793                 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2794         tile[29] = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |
2795                 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) |
2796                 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2797                 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2798         tile[30] = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2799                 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) |
2800                 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2801                 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2802         }
2803 
2804         macrotile[0] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2805                 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2806                 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
2807                 NUM_BANKS(ADDR_SURF_16_BANK));
2808         macrotile[1] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2809                 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
2810                 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
2811                 NUM_BANKS(ADDR_SURF_16_BANK));
2812         macrotile[2] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2813                 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2814                 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
2815                 NUM_BANKS(ADDR_SURF_16_BANK));
2816         macrotile[3] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2817                 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2818                 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
2819                 NUM_BANKS(ADDR_SURF_16_BANK));
2820         macrotile[4] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2821                 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2822                 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
2823                 NUM_BANKS(ADDR_SURF_16_BANK));
2824         macrotile[5] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2825                 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2826                 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
2827                 NUM_BANKS(ADDR_SURF_8_BANK));
2828         macrotile[6] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2829                 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2830                 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) |
2831                 NUM_BANKS(ADDR_SURF_4_BANK));
2832         macrotile[8] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_2) |
2833                 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_8) |
2834                 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
2835                 NUM_BANKS(ADDR_SURF_16_BANK));
2836         macrotile[9] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_2) |
2837                 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2838                 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
2839                 NUM_BANKS(ADDR_SURF_16_BANK));
2840         macrotile[10] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2841                 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2842                 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
2843                 NUM_BANKS(ADDR_SURF_16_BANK));
2844         macrotile[11] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2845                 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
2846                 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
2847                 NUM_BANKS(ADDR_SURF_16_BANK));
2848         macrotile[12] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2849                 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2850                 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
2851                 NUM_BANKS(ADDR_SURF_16_BANK));
2852         macrotile[13] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2853                 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2854                 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
2855                 NUM_BANKS(ADDR_SURF_8_BANK));
2856         macrotile[14] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2857                 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2858                 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) |
2859                 NUM_BANKS(ADDR_SURF_4_BANK));
2860 
2861         for (reg_offset = 0; reg_offset < num_tile_mode_states; reg_offset++)
2862             WREG32(GB_TILE_MODE0 + (reg_offset * 4), tile[reg_offset]);
2863         for (reg_offset = 0; reg_offset < num_secondary_tile_mode_states; reg_offset++)
2864             WREG32(GB_MACROTILE_MODE0 + (reg_offset * 4), macrotile[reg_offset]);
2865         break;
2866 
2867     case 2:
2868         tile[0] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2869                MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2870                PIPE_CONFIG(ADDR_SURF_P2) |
2871                TILE_SPLIT(ADDR_SURF_TILE_SPLIT_64B));
2872         tile[1] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2873                MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2874                PIPE_CONFIG(ADDR_SURF_P2) |
2875                TILE_SPLIT(ADDR_SURF_TILE_SPLIT_128B));
2876         tile[2] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2877                MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2878                PIPE_CONFIG(ADDR_SURF_P2) |
2879                TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B));
2880         tile[3] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2881                MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2882                PIPE_CONFIG(ADDR_SURF_P2) |
2883                TILE_SPLIT(ADDR_SURF_TILE_SPLIT_512B));
2884         tile[4] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2885                MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2886                PIPE_CONFIG(ADDR_SURF_P2) |
2887                TILE_SPLIT(split_equal_to_row_size));
2888         tile[5] = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2889                PIPE_CONFIG(ADDR_SURF_P2) |
2890                MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING));
2891         tile[6] = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2892                MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2893                PIPE_CONFIG(ADDR_SURF_P2) |
2894                TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B));
2895         tile[7] = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2896                MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2897                PIPE_CONFIG(ADDR_SURF_P2) |
2898                TILE_SPLIT(split_equal_to_row_size));
2899         tile[8] = ARRAY_MODE(ARRAY_LINEAR_ALIGNED) |
2900                PIPE_CONFIG(ADDR_SURF_P2);
2901         tile[9] = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2902                MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) |
2903                PIPE_CONFIG(ADDR_SURF_P2));
2904         tile[10] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2905                 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) |
2906                 PIPE_CONFIG(ADDR_SURF_P2) |
2907                 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2908         tile[11] = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |
2909                 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) |
2910                 PIPE_CONFIG(ADDR_SURF_P2) |
2911                 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2912         tile[12] = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2913                 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) |
2914                 PIPE_CONFIG(ADDR_SURF_P2) |
2915                 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2916         tile[13] = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2917                 PIPE_CONFIG(ADDR_SURF_P2) |
2918                 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING));
2919         tile[14] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2920                 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
2921                 PIPE_CONFIG(ADDR_SURF_P2) |
2922                 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2923         tile[16] = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |
2924                 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
2925                 PIPE_CONFIG(ADDR_SURF_P2) |
2926                 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2927         tile[17] = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2928                 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
2929                 PIPE_CONFIG(ADDR_SURF_P2) |
2930                 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2931         tile[27] = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2932                 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) |
2933                 PIPE_CONFIG(ADDR_SURF_P2));
2934         tile[28] = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2935                 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) |
2936                 PIPE_CONFIG(ADDR_SURF_P2) |
2937                 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2938         tile[29] = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |
2939                 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) |
2940                 PIPE_CONFIG(ADDR_SURF_P2) |
2941                 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2942         tile[30] = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2943                 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) |
2944                 PIPE_CONFIG(ADDR_SURF_P2) |
2945                 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2946 
2947         macrotile[0] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_2) |
2948                 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2949                 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
2950                 NUM_BANKS(ADDR_SURF_16_BANK));
2951         macrotile[1] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_2) |
2952                 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
2953                 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
2954                 NUM_BANKS(ADDR_SURF_16_BANK));
2955         macrotile[2] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2956                 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
2957                 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
2958                 NUM_BANKS(ADDR_SURF_16_BANK));
2959         macrotile[3] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2960                 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2961                 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
2962                 NUM_BANKS(ADDR_SURF_16_BANK));
2963         macrotile[4] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2964                 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2965                 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
2966                 NUM_BANKS(ADDR_SURF_16_BANK));
2967         macrotile[5] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2968                 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2969                 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
2970                 NUM_BANKS(ADDR_SURF_16_BANK));
2971         macrotile[6] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2972                 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2973                 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
2974                 NUM_BANKS(ADDR_SURF_8_BANK));
2975         macrotile[8] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_4) |
2976                 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_8) |
2977                 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
2978                 NUM_BANKS(ADDR_SURF_16_BANK));
2979         macrotile[9] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_4) |
2980                 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2981                 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
2982                 NUM_BANKS(ADDR_SURF_16_BANK));
2983         macrotile[10] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_2) |
2984                 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2985                 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
2986                 NUM_BANKS(ADDR_SURF_16_BANK));
2987         macrotile[11] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_2) |
2988                 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
2989                 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
2990                 NUM_BANKS(ADDR_SURF_16_BANK));
2991         macrotile[12] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2992                 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
2993                 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
2994                 NUM_BANKS(ADDR_SURF_16_BANK));
2995         macrotile[13] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2996                 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2997                 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
2998                 NUM_BANKS(ADDR_SURF_16_BANK));
2999         macrotile[14] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
3000                 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
3001                 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
3002                 NUM_BANKS(ADDR_SURF_8_BANK));
3003 
3004         for (reg_offset = 0; reg_offset < num_tile_mode_states; reg_offset++)
3005             WREG32(GB_TILE_MODE0 + (reg_offset * 4), tile[reg_offset]);
3006         for (reg_offset = 0; reg_offset < num_secondary_tile_mode_states; reg_offset++)
3007             WREG32(GB_MACROTILE_MODE0 + (reg_offset * 4), macrotile[reg_offset]);
3008         break;
3009 
3010     default:
3011         DRM_ERROR("unknown num pipe config: 0x%x\n", num_pipe_configs);
3012     }
3013 }
3014 
3015 /**
3016  * cik_select_se_sh - select which SE, SH to address
3017  *
3018  * @rdev: radeon_device pointer
3019  * @se_num: shader engine to address
3020  * @sh_num: sh block to address
3021  *
3022  * Select which SE, SH combinations to address. Certain
3023  * registers are instanced per SE or SH.  0xffffffff means
3024  * broadcast to all SEs or SHs (CIK).
3025  */
3026 static void cik_select_se_sh(struct radeon_device *rdev,
3027                  u32 se_num, u32 sh_num)
3028 {
3029     u32 data = INSTANCE_BROADCAST_WRITES;
3030 
3031     if ((se_num == 0xffffffff) && (sh_num == 0xffffffff))
3032         data |= SH_BROADCAST_WRITES | SE_BROADCAST_WRITES;
3033     else if (se_num == 0xffffffff)
3034         data |= SE_BROADCAST_WRITES | SH_INDEX(sh_num);
3035     else if (sh_num == 0xffffffff)
3036         data |= SH_BROADCAST_WRITES | SE_INDEX(se_num);
3037     else
3038         data |= SH_INDEX(sh_num) | SE_INDEX(se_num);
3039     WREG32(GRBM_GFX_INDEX, data);
3040 }
3041 
3042 /**
3043  * cik_create_bitmask - create a bitmask
3044  *
3045  * @bit_width: length of the mask
3046  *
3047  * create a variable length bit mask (CIK).
3048  * Returns the bitmask.
3049  */
3050 static u32 cik_create_bitmask(u32 bit_width)
3051 {
3052     u32 i, mask = 0;
3053 
3054     for (i = 0; i < bit_width; i++) {
3055         mask <<= 1;
3056         mask |= 1;
3057     }
3058     return mask;
3059 }
3060 
3061 /**
3062  * cik_get_rb_disabled - computes the mask of disabled RBs
3063  *
3064  * @rdev: radeon_device pointer
3065  * @max_rb_num_per_se: max RBs (render backends) per SE (shader engine) for the asic
3066  * @sh_per_se: number of SH blocks per SE for the asic
3067  *
3068  * Calculates the bitmask of disabled RBs (CIK).
3069  * Returns the disabled RB bitmask.
3070  */
3071 static u32 cik_get_rb_disabled(struct radeon_device *rdev,
3072                   u32 max_rb_num_per_se,
3073                   u32 sh_per_se)
3074 {
3075     u32 data, mask;
3076 
3077     data = RREG32(CC_RB_BACKEND_DISABLE);
3078     if (data & 1)
3079         data &= BACKEND_DISABLE_MASK;
3080     else
3081         data = 0;
3082     data |= RREG32(GC_USER_RB_BACKEND_DISABLE);
3083 
3084     data >>= BACKEND_DISABLE_SHIFT;
3085 
3086     mask = cik_create_bitmask(max_rb_num_per_se / sh_per_se);
3087 
3088     return data & mask;
3089 }
3090 
3091 /**
3092  * cik_setup_rb - setup the RBs on the asic
3093  *
3094  * @rdev: radeon_device pointer
3095  * @se_num: number of SEs (shader engines) for the asic
3096  * @sh_per_se: number of SH blocks per SE for the asic
3097  * @max_rb_num_per_se: max RBs (render backends) per SE for the asic
3098  *
3099  * Configures per-SE/SH RB registers (CIK).
3100  */
3101 static void cik_setup_rb(struct radeon_device *rdev,
3102              u32 se_num, u32 sh_per_se,
3103              u32 max_rb_num_per_se)
3104 {
3105     int i, j;
3106     u32 data, mask;
3107     u32 disabled_rbs = 0;
3108     u32 enabled_rbs = 0;
3109 
3110     for (i = 0; i < se_num; i++) {
3111         for (j = 0; j < sh_per_se; j++) {
3112             cik_select_se_sh(rdev, i, j);
3113             data = cik_get_rb_disabled(rdev, max_rb_num_per_se, sh_per_se);
3114             if (rdev->family == CHIP_HAWAII)
3115                 disabled_rbs |= data << ((i * sh_per_se + j) * HAWAII_RB_BITMAP_WIDTH_PER_SH);
3116             else
3117                 disabled_rbs |= data << ((i * sh_per_se + j) * CIK_RB_BITMAP_WIDTH_PER_SH);
3118         }
3119     }
3120     cik_select_se_sh(rdev, 0xffffffff, 0xffffffff);
3121 
3122     mask = 1;
3123     for (i = 0; i < max_rb_num_per_se * se_num; i++) {
3124         if (!(disabled_rbs & mask))
3125             enabled_rbs |= mask;
3126         mask <<= 1;
3127     }
3128 
3129     rdev->config.cik.backend_enable_mask = enabled_rbs;
3130 
3131     for (i = 0; i < se_num; i++) {
3132         cik_select_se_sh(rdev, i, 0xffffffff);
3133         data = 0;
3134         for (j = 0; j < sh_per_se; j++) {
3135             switch (enabled_rbs & 3) {
3136             case 0:
3137                 if (j == 0)
3138                     data |= PKR_MAP(RASTER_CONFIG_RB_MAP_3);
3139                 else
3140                     data |= PKR_MAP(RASTER_CONFIG_RB_MAP_0);
3141                 break;
3142             case 1:
3143                 data |= (RASTER_CONFIG_RB_MAP_0 << (i * sh_per_se + j) * 2);
3144                 break;
3145             case 2:
3146                 data |= (RASTER_CONFIG_RB_MAP_3 << (i * sh_per_se + j) * 2);
3147                 break;
3148             case 3:
3149             default:
3150                 data |= (RASTER_CONFIG_RB_MAP_2 << (i * sh_per_se + j) * 2);
3151                 break;
3152             }
3153             enabled_rbs >>= 2;
3154         }
3155         WREG32(PA_SC_RASTER_CONFIG, data);
3156     }
3157     cik_select_se_sh(rdev, 0xffffffff, 0xffffffff);
3158 }
3159 
3160 /**
3161  * cik_gpu_init - setup the 3D engine
3162  *
3163  * @rdev: radeon_device pointer
3164  *
3165  * Configures the 3D engine and tiling configuration
3166  * registers so that the 3D engine is usable.
3167  */
3168 static void cik_gpu_init(struct radeon_device *rdev)
3169 {
3170     u32 gb_addr_config = RREG32(GB_ADDR_CONFIG);
3171     u32 mc_arb_ramcfg;
3172     u32 hdp_host_path_cntl;
3173     u32 tmp;
3174     int i, j;
3175 
3176     switch (rdev->family) {
3177     case CHIP_BONAIRE:
3178         rdev->config.cik.max_shader_engines = 2;
3179         rdev->config.cik.max_tile_pipes = 4;
3180         rdev->config.cik.max_cu_per_sh = 7;
3181         rdev->config.cik.max_sh_per_se = 1;
3182         rdev->config.cik.max_backends_per_se = 2;
3183         rdev->config.cik.max_texture_channel_caches = 4;
3184         rdev->config.cik.max_gprs = 256;
3185         rdev->config.cik.max_gs_threads = 32;
3186         rdev->config.cik.max_hw_contexts = 8;
3187 
3188         rdev->config.cik.sc_prim_fifo_size_frontend = 0x20;
3189         rdev->config.cik.sc_prim_fifo_size_backend = 0x100;
3190         rdev->config.cik.sc_hiz_tile_fifo_size = 0x30;
3191         rdev->config.cik.sc_earlyz_tile_fifo_size = 0x130;
3192         gb_addr_config = BONAIRE_GB_ADDR_CONFIG_GOLDEN;
3193         break;
3194     case CHIP_HAWAII:
3195         rdev->config.cik.max_shader_engines = 4;
3196         rdev->config.cik.max_tile_pipes = 16;
3197         rdev->config.cik.max_cu_per_sh = 11;
3198         rdev->config.cik.max_sh_per_se = 1;
3199         rdev->config.cik.max_backends_per_se = 4;
3200         rdev->config.cik.max_texture_channel_caches = 16;
3201         rdev->config.cik.max_gprs = 256;
3202         rdev->config.cik.max_gs_threads = 32;
3203         rdev->config.cik.max_hw_contexts = 8;
3204 
3205         rdev->config.cik.sc_prim_fifo_size_frontend = 0x20;
3206         rdev->config.cik.sc_prim_fifo_size_backend = 0x100;
3207         rdev->config.cik.sc_hiz_tile_fifo_size = 0x30;
3208         rdev->config.cik.sc_earlyz_tile_fifo_size = 0x130;
3209         gb_addr_config = HAWAII_GB_ADDR_CONFIG_GOLDEN;
3210         break;
3211     case CHIP_KAVERI:
3212         rdev->config.cik.max_shader_engines = 1;
3213         rdev->config.cik.max_tile_pipes = 4;
3214         rdev->config.cik.max_cu_per_sh = 8;
3215         rdev->config.cik.max_backends_per_se = 2;
3216         rdev->config.cik.max_sh_per_se = 1;
3217         rdev->config.cik.max_texture_channel_caches = 4;
3218         rdev->config.cik.max_gprs = 256;
3219         rdev->config.cik.max_gs_threads = 16;
3220         rdev->config.cik.max_hw_contexts = 8;
3221 
3222         rdev->config.cik.sc_prim_fifo_size_frontend = 0x20;
3223         rdev->config.cik.sc_prim_fifo_size_backend = 0x100;
3224         rdev->config.cik.sc_hiz_tile_fifo_size = 0x30;
3225         rdev->config.cik.sc_earlyz_tile_fifo_size = 0x130;
3226         gb_addr_config = BONAIRE_GB_ADDR_CONFIG_GOLDEN;
3227         break;
3228     case CHIP_KABINI:
3229     case CHIP_MULLINS:
3230     default:
3231         rdev->config.cik.max_shader_engines = 1;
3232         rdev->config.cik.max_tile_pipes = 2;
3233         rdev->config.cik.max_cu_per_sh = 2;
3234         rdev->config.cik.max_sh_per_se = 1;
3235         rdev->config.cik.max_backends_per_se = 1;
3236         rdev->config.cik.max_texture_channel_caches = 2;
3237         rdev->config.cik.max_gprs = 256;
3238         rdev->config.cik.max_gs_threads = 16;
3239         rdev->config.cik.max_hw_contexts = 8;
3240 
3241         rdev->config.cik.sc_prim_fifo_size_frontend = 0x20;
3242         rdev->config.cik.sc_prim_fifo_size_backend = 0x100;
3243         rdev->config.cik.sc_hiz_tile_fifo_size = 0x30;
3244         rdev->config.cik.sc_earlyz_tile_fifo_size = 0x130;
3245         gb_addr_config = BONAIRE_GB_ADDR_CONFIG_GOLDEN;
3246         break;
3247     }
3248 
3249     /* Initialize HDP */
3250     for (i = 0, j = 0; i < 32; i++, j += 0x18) {
3251         WREG32((0x2c14 + j), 0x00000000);
3252         WREG32((0x2c18 + j), 0x00000000);
3253         WREG32((0x2c1c + j), 0x00000000);
3254         WREG32((0x2c20 + j), 0x00000000);
3255         WREG32((0x2c24 + j), 0x00000000);
3256     }
3257 
3258     WREG32(GRBM_CNTL, GRBM_READ_TIMEOUT(0xff));
3259     WREG32(SRBM_INT_CNTL, 0x1);
3260     WREG32(SRBM_INT_ACK, 0x1);
3261 
3262     WREG32(BIF_FB_EN, FB_READ_EN | FB_WRITE_EN);
3263 
3264     RREG32(MC_SHARED_CHMAP);
3265     mc_arb_ramcfg = RREG32(MC_ARB_RAMCFG);
3266 
3267     rdev->config.cik.num_tile_pipes = rdev->config.cik.max_tile_pipes;
3268     rdev->config.cik.mem_max_burst_length_bytes = 256;
3269     tmp = (mc_arb_ramcfg & NOOFCOLS_MASK) >> NOOFCOLS_SHIFT;
3270     rdev->config.cik.mem_row_size_in_kb = (4 * (1 << (8 + tmp))) / 1024;
3271     if (rdev->config.cik.mem_row_size_in_kb > 4)
3272         rdev->config.cik.mem_row_size_in_kb = 4;
3273     /* XXX use MC settings? */
3274     rdev->config.cik.shader_engine_tile_size = 32;
3275     rdev->config.cik.num_gpus = 1;
3276     rdev->config.cik.multi_gpu_tile_size = 64;
3277 
3278     /* fix up row size */
3279     gb_addr_config &= ~ROW_SIZE_MASK;
3280     switch (rdev->config.cik.mem_row_size_in_kb) {
3281     case 1:
3282     default:
3283         gb_addr_config |= ROW_SIZE(0);
3284         break;
3285     case 2:
3286         gb_addr_config |= ROW_SIZE(1);
3287         break;
3288     case 4:
3289         gb_addr_config |= ROW_SIZE(2);
3290         break;
3291     }
3292 
3293     /* setup tiling info dword.  gb_addr_config is not adequate since it does
3294      * not have bank info, so create a custom tiling dword.
3295      * bits 3:0   num_pipes
3296      * bits 7:4   num_banks
3297      * bits 11:8  group_size
3298      * bits 15:12 row_size
3299      */
3300     rdev->config.cik.tile_config = 0;
3301     switch (rdev->config.cik.num_tile_pipes) {
3302     case 1:
3303         rdev->config.cik.tile_config |= (0 << 0);
3304         break;
3305     case 2:
3306         rdev->config.cik.tile_config |= (1 << 0);
3307         break;
3308     case 4:
3309         rdev->config.cik.tile_config |= (2 << 0);
3310         break;
3311     case 8:
3312     default:
3313         /* XXX what about 12? */
3314         rdev->config.cik.tile_config |= (3 << 0);
3315         break;
3316     }
3317     rdev->config.cik.tile_config |=
3318         ((mc_arb_ramcfg & NOOFBANK_MASK) >> NOOFBANK_SHIFT) << 4;
3319     rdev->config.cik.tile_config |=
3320         ((gb_addr_config & PIPE_INTERLEAVE_SIZE_MASK) >> PIPE_INTERLEAVE_SIZE_SHIFT) << 8;
3321     rdev->config.cik.tile_config |=
3322         ((gb_addr_config & ROW_SIZE_MASK) >> ROW_SIZE_SHIFT) << 12;
3323 
3324     WREG32(GB_ADDR_CONFIG, gb_addr_config);
3325     WREG32(HDP_ADDR_CONFIG, gb_addr_config);
3326     WREG32(DMIF_ADDR_CALC, gb_addr_config);
3327     WREG32(SDMA0_TILING_CONFIG + SDMA0_REGISTER_OFFSET, gb_addr_config & 0x70);
3328     WREG32(SDMA0_TILING_CONFIG + SDMA1_REGISTER_OFFSET, gb_addr_config & 0x70);
3329     WREG32(UVD_UDEC_ADDR_CONFIG, gb_addr_config);
3330     WREG32(UVD_UDEC_DB_ADDR_CONFIG, gb_addr_config);
3331     WREG32(UVD_UDEC_DBW_ADDR_CONFIG, gb_addr_config);
3332 
3333     cik_tiling_mode_table_init(rdev);
3334 
3335     cik_setup_rb(rdev, rdev->config.cik.max_shader_engines,
3336              rdev->config.cik.max_sh_per_se,
3337              rdev->config.cik.max_backends_per_se);
3338 
3339     rdev->config.cik.active_cus = 0;
3340     for (i = 0; i < rdev->config.cik.max_shader_engines; i++) {
3341         for (j = 0; j < rdev->config.cik.max_sh_per_se; j++) {
3342             rdev->config.cik.active_cus +=
3343                 hweight32(cik_get_cu_active_bitmap(rdev, i, j));
3344         }
3345     }
3346 
3347     /* set HW defaults for 3D engine */
3348     WREG32(CP_MEQ_THRESHOLDS, MEQ1_START(0x30) | MEQ2_START(0x60));
3349 
3350     WREG32(SX_DEBUG_1, 0x20);
3351 
3352     WREG32(TA_CNTL_AUX, 0x00010000);
3353 
3354     tmp = RREG32(SPI_CONFIG_CNTL);
3355     tmp |= 0x03000000;
3356     WREG32(SPI_CONFIG_CNTL, tmp);
3357 
3358     WREG32(SQ_CONFIG, 1);
3359 
3360     WREG32(DB_DEBUG, 0);
3361 
3362     tmp = RREG32(DB_DEBUG2) & ~0xf00fffff;
3363     tmp |= 0x00000400;
3364     WREG32(DB_DEBUG2, tmp);
3365 
3366     tmp = RREG32(DB_DEBUG3) & ~0x0002021c;
3367     tmp |= 0x00020200;
3368     WREG32(DB_DEBUG3, tmp);
3369 
3370     tmp = RREG32(CB_HW_CONTROL) & ~0x00010000;
3371     tmp |= 0x00018208;
3372     WREG32(CB_HW_CONTROL, tmp);
3373 
3374     WREG32(SPI_CONFIG_CNTL_1, VTX_DONE_DELAY(4));
3375 
3376     WREG32(PA_SC_FIFO_SIZE, (SC_FRONTEND_PRIM_FIFO_SIZE(rdev->config.cik.sc_prim_fifo_size_frontend) |
3377                  SC_BACKEND_PRIM_FIFO_SIZE(rdev->config.cik.sc_prim_fifo_size_backend) |
3378                  SC_HIZ_TILE_FIFO_SIZE(rdev->config.cik.sc_hiz_tile_fifo_size) |
3379                  SC_EARLYZ_TILE_FIFO_SIZE(rdev->config.cik.sc_earlyz_tile_fifo_size)));
3380 
3381     WREG32(VGT_NUM_INSTANCES, 1);
3382 
3383     WREG32(CP_PERFMON_CNTL, 0);
3384 
3385     WREG32(SQ_CONFIG, 0);
3386 
3387     WREG32(PA_SC_FORCE_EOV_MAX_CNTS, (FORCE_EOV_MAX_CLK_CNT(4095) |
3388                       FORCE_EOV_MAX_REZ_CNT(255)));
3389 
3390     WREG32(VGT_CACHE_INVALIDATION, CACHE_INVALIDATION(VC_AND_TC) |
3391            AUTO_INVLD_EN(ES_AND_GS_AUTO));
3392 
3393     WREG32(VGT_GS_VERTEX_REUSE, 16);
3394     WREG32(PA_SC_LINE_STIPPLE_STATE, 0);
3395 
3396     tmp = RREG32(HDP_MISC_CNTL);
3397     tmp |= HDP_FLUSH_INVALIDATE_CACHE;
3398     WREG32(HDP_MISC_CNTL, tmp);
3399 
3400     hdp_host_path_cntl = RREG32(HDP_HOST_PATH_CNTL);
3401     WREG32(HDP_HOST_PATH_CNTL, hdp_host_path_cntl);
3402 
3403     WREG32(PA_CL_ENHANCE, CLIP_VTX_REORDER_ENA | NUM_CLIP_SEQ(3));
3404     WREG32(PA_SC_ENHANCE, ENABLE_PA_SC_OUT_OF_ORDER);
3405 
3406     udelay(50);
3407 }
3408 
3409 /*
3410  * GPU scratch registers helpers function.
3411  */
3412 /**
3413  * cik_scratch_init - setup driver info for CP scratch regs
3414  *
3415  * @rdev: radeon_device pointer
3416  *
3417  * Set up the number and offset of the CP scratch registers.
3418  * NOTE: use of CP scratch registers is a legacy inferface and
3419  * is not used by default on newer asics (r6xx+).  On newer asics,
3420  * memory buffers are used for fences rather than scratch regs.
3421  */
3422 static void cik_scratch_init(struct radeon_device *rdev)
3423 {
3424     int i;
3425 
3426     rdev->scratch.num_reg = 7;
3427     rdev->scratch.reg_base = SCRATCH_REG0;
3428     for (i = 0; i < rdev->scratch.num_reg; i++) {
3429         rdev->scratch.free[i] = true;
3430         rdev->scratch.reg[i] = rdev->scratch.reg_base + (i * 4);
3431     }
3432 }
3433 
3434 /**
3435  * cik_ring_test - basic gfx ring test
3436  *
3437  * @rdev: radeon_device pointer
3438  * @ring: radeon_ring structure holding ring information
3439  *
3440  * Allocate a scratch register and write to it using the gfx ring (CIK).
3441  * Provides a basic gfx ring test to verify that the ring is working.
3442  * Used by cik_cp_gfx_resume();
3443  * Returns 0 on success, error on failure.
3444  */
3445 int cik_ring_test(struct radeon_device *rdev, struct radeon_ring *ring)
3446 {
3447     uint32_t scratch;
3448     uint32_t tmp = 0;
3449     unsigned i;
3450     int r;
3451 
3452     r = radeon_scratch_get(rdev, &scratch);
3453     if (r) {
3454         DRM_ERROR("radeon: cp failed to get scratch reg (%d).\n", r);
3455         return r;
3456     }
3457     WREG32(scratch, 0xCAFEDEAD);
3458     r = radeon_ring_lock(rdev, ring, 3);
3459     if (r) {
3460         DRM_ERROR("radeon: cp failed to lock ring %d (%d).\n", ring->idx, r);
3461         radeon_scratch_free(rdev, scratch);
3462         return r;
3463     }
3464     radeon_ring_write(ring, PACKET3(PACKET3_SET_UCONFIG_REG, 1));
3465     radeon_ring_write(ring, ((scratch - PACKET3_SET_UCONFIG_REG_START) >> 2));
3466     radeon_ring_write(ring, 0xDEADBEEF);
3467     radeon_ring_unlock_commit(rdev, ring, false);
3468 
3469     for (i = 0; i < rdev->usec_timeout; i++) {
3470         tmp = RREG32(scratch);
3471         if (tmp == 0xDEADBEEF)
3472             break;
3473         udelay(1);
3474     }
3475     if (i < rdev->usec_timeout) {
3476         DRM_INFO("ring test on %d succeeded in %d usecs\n", ring->idx, i);
3477     } else {
3478         DRM_ERROR("radeon: ring %d test failed (scratch(0x%04X)=0x%08X)\n",
3479               ring->idx, scratch, tmp);
3480         r = -EINVAL;
3481     }
3482     radeon_scratch_free(rdev, scratch);
3483     return r;
3484 }
3485 
3486 /**
3487  * cik_hdp_flush_cp_ring_emit - emit an hdp flush on the cp
3488  *
3489  * @rdev: radeon_device pointer
3490  * @ridx: radeon ring index
3491  *
3492  * Emits an hdp flush on the cp.
3493  */
3494 static void cik_hdp_flush_cp_ring_emit(struct radeon_device *rdev,
3495                        int ridx)
3496 {
3497     struct radeon_ring *ring = &rdev->ring[ridx];
3498     u32 ref_and_mask;
3499 
3500     switch (ring->idx) {
3501     case CAYMAN_RING_TYPE_CP1_INDEX:
3502     case CAYMAN_RING_TYPE_CP2_INDEX:
3503     default:
3504         switch (ring->me) {
3505         case 0:
3506             ref_and_mask = CP2 << ring->pipe;
3507             break;
3508         case 1:
3509             ref_and_mask = CP6 << ring->pipe;
3510             break;
3511         default:
3512             return;
3513         }
3514         break;
3515     case RADEON_RING_TYPE_GFX_INDEX:
3516         ref_and_mask = CP0;
3517         break;
3518     }
3519 
3520     radeon_ring_write(ring, PACKET3(PACKET3_WAIT_REG_MEM, 5));
3521     radeon_ring_write(ring, (WAIT_REG_MEM_OPERATION(1) | /* write, wait, write */
3522                  WAIT_REG_MEM_FUNCTION(3) |  /* == */
3523                  WAIT_REG_MEM_ENGINE(1)));   /* pfp */
3524     radeon_ring_write(ring, GPU_HDP_FLUSH_REQ >> 2);
3525     radeon_ring_write(ring, GPU_HDP_FLUSH_DONE >> 2);
3526     radeon_ring_write(ring, ref_and_mask);
3527     radeon_ring_write(ring, ref_and_mask);
3528     radeon_ring_write(ring, 0x20); /* poll interval */
3529 }
3530 
3531 /**
3532  * cik_fence_gfx_ring_emit - emit a fence on the gfx ring
3533  *
3534  * @rdev: radeon_device pointer
3535  * @fence: radeon fence object
3536  *
3537  * Emits a fence sequnce number on the gfx ring and flushes
3538  * GPU caches.
3539  */
3540 void cik_fence_gfx_ring_emit(struct radeon_device *rdev,
3541                  struct radeon_fence *fence)
3542 {
3543     struct radeon_ring *ring = &rdev->ring[fence->ring];
3544     u64 addr = rdev->fence_drv[fence->ring].gpu_addr;
3545 
3546     /* Workaround for cache flush problems. First send a dummy EOP
3547      * event down the pipe with seq one below.
3548      */
3549     radeon_ring_write(ring, PACKET3(PACKET3_EVENT_WRITE_EOP, 4));
3550     radeon_ring_write(ring, (EOP_TCL1_ACTION_EN |
3551                  EOP_TC_ACTION_EN |
3552                  EVENT_TYPE(CACHE_FLUSH_AND_INV_TS_EVENT) |
3553                  EVENT_INDEX(5)));
3554     radeon_ring_write(ring, addr & 0xfffffffc);
3555     radeon_ring_write(ring, (upper_32_bits(addr) & 0xffff) |
3556                 DATA_SEL(1) | INT_SEL(0));
3557     radeon_ring_write(ring, fence->seq - 1);
3558     radeon_ring_write(ring, 0);
3559 
3560     /* Then send the real EOP event down the pipe. */
3561     radeon_ring_write(ring, PACKET3(PACKET3_EVENT_WRITE_EOP, 4));
3562     radeon_ring_write(ring, (EOP_TCL1_ACTION_EN |
3563                  EOP_TC_ACTION_EN |
3564                  EVENT_TYPE(CACHE_FLUSH_AND_INV_TS_EVENT) |
3565                  EVENT_INDEX(5)));
3566     radeon_ring_write(ring, addr & 0xfffffffc);
3567     radeon_ring_write(ring, (upper_32_bits(addr) & 0xffff) | DATA_SEL(1) | INT_SEL(2));
3568     radeon_ring_write(ring, fence->seq);
3569     radeon_ring_write(ring, 0);
3570 }
3571 
3572 /**
3573  * cik_fence_compute_ring_emit - emit a fence on the compute ring
3574  *
3575  * @rdev: radeon_device pointer
3576  * @fence: radeon fence object
3577  *
3578  * Emits a fence sequnce number on the compute ring and flushes
3579  * GPU caches.
3580  */
3581 void cik_fence_compute_ring_emit(struct radeon_device *rdev,
3582                  struct radeon_fence *fence)
3583 {
3584     struct radeon_ring *ring = &rdev->ring[fence->ring];
3585     u64 addr = rdev->fence_drv[fence->ring].gpu_addr;
3586 
3587     /* RELEASE_MEM - flush caches, send int */
3588     radeon_ring_write(ring, PACKET3(PACKET3_RELEASE_MEM, 5));
3589     radeon_ring_write(ring, (EOP_TCL1_ACTION_EN |
3590                  EOP_TC_ACTION_EN |
3591                  EVENT_TYPE(CACHE_FLUSH_AND_INV_TS_EVENT) |
3592                  EVENT_INDEX(5)));
3593     radeon_ring_write(ring, DATA_SEL(1) | INT_SEL(2));
3594     radeon_ring_write(ring, addr & 0xfffffffc);
3595     radeon_ring_write(ring, upper_32_bits(addr));
3596     radeon_ring_write(ring, fence->seq);
3597     radeon_ring_write(ring, 0);
3598 }
3599 
3600 /**
3601  * cik_semaphore_ring_emit - emit a semaphore on the CP ring
3602  *
3603  * @rdev: radeon_device pointer
3604  * @ring: radeon ring buffer object
3605  * @semaphore: radeon semaphore object
3606  * @emit_wait: Is this a sempahore wait?
3607  *
3608  * Emits a semaphore signal/wait packet to the CP ring and prevents the PFP
3609  * from running ahead of semaphore waits.
3610  */
3611 bool cik_semaphore_ring_emit(struct radeon_device *rdev,
3612                  struct radeon_ring *ring,
3613                  struct radeon_semaphore *semaphore,
3614                  bool emit_wait)
3615 {
3616     uint64_t addr = semaphore->gpu_addr;
3617     unsigned sel = emit_wait ? PACKET3_SEM_SEL_WAIT : PACKET3_SEM_SEL_SIGNAL;
3618 
3619     radeon_ring_write(ring, PACKET3(PACKET3_MEM_SEMAPHORE, 1));
3620     radeon_ring_write(ring, lower_32_bits(addr));
3621     radeon_ring_write(ring, (upper_32_bits(addr) & 0xffff) | sel);
3622 
3623     if (emit_wait && ring->idx == RADEON_RING_TYPE_GFX_INDEX) {
3624         /* Prevent the PFP from running ahead of the semaphore wait */
3625         radeon_ring_write(ring, PACKET3(PACKET3_PFP_SYNC_ME, 0));
3626         radeon_ring_write(ring, 0x0);
3627     }
3628 
3629     return true;
3630 }
3631 
3632 /**
3633  * cik_copy_cpdma - copy pages using the CP DMA engine
3634  *
3635  * @rdev: radeon_device pointer
3636  * @src_offset: src GPU address
3637  * @dst_offset: dst GPU address
3638  * @num_gpu_pages: number of GPU pages to xfer
3639  * @resv: reservation object to sync to
3640  *
3641  * Copy GPU paging using the CP DMA engine (CIK+).
3642  * Used by the radeon ttm implementation to move pages if
3643  * registered as the asic copy callback.
3644  */
3645 struct radeon_fence *cik_copy_cpdma(struct radeon_device *rdev,
3646                     uint64_t src_offset, uint64_t dst_offset,
3647                     unsigned num_gpu_pages,
3648                     struct dma_resv *resv)
3649 {
3650     struct radeon_fence *fence;
3651     struct radeon_sync sync;
3652     int ring_index = rdev->asic->copy.blit_ring_index;
3653     struct radeon_ring *ring = &rdev->ring[ring_index];
3654     u32 size_in_bytes, cur_size_in_bytes, control;
3655     int i, num_loops;
3656     int r = 0;
3657 
3658     radeon_sync_create(&sync);
3659 
3660     size_in_bytes = (num_gpu_pages << RADEON_GPU_PAGE_SHIFT);
3661     num_loops = DIV_ROUND_UP(size_in_bytes, 0x1fffff);
3662     r = radeon_ring_lock(rdev, ring, num_loops * 7 + 18);
3663     if (r) {
3664         DRM_ERROR("radeon: moving bo (%d).\n", r);
3665         radeon_sync_free(rdev, &sync, NULL);
3666         return ERR_PTR(r);
3667     }
3668 
3669     radeon_sync_resv(rdev, &sync, resv, false);
3670     radeon_sync_rings(rdev, &sync, ring->idx);
3671 
3672     for (i = 0; i < num_loops; i++) {
3673         cur_size_in_bytes = size_in_bytes;
3674         if (cur_size_in_bytes > 0x1fffff)
3675             cur_size_in_bytes = 0x1fffff;
3676         size_in_bytes -= cur_size_in_bytes;
3677         control = 0;
3678         if (size_in_bytes == 0)
3679             control |= PACKET3_DMA_DATA_CP_SYNC;
3680         radeon_ring_write(ring, PACKET3(PACKET3_DMA_DATA, 5));
3681         radeon_ring_write(ring, control);
3682         radeon_ring_write(ring, lower_32_bits(src_offset));
3683         radeon_ring_write(ring, upper_32_bits(src_offset));
3684         radeon_ring_write(ring, lower_32_bits(dst_offset));
3685         radeon_ring_write(ring, upper_32_bits(dst_offset));
3686         radeon_ring_write(ring, cur_size_in_bytes);
3687         src_offset += cur_size_in_bytes;
3688         dst_offset += cur_size_in_bytes;
3689     }
3690 
3691     r = radeon_fence_emit(rdev, &fence, ring->idx);
3692     if (r) {
3693         radeon_ring_unlock_undo(rdev, ring);
3694         radeon_sync_free(rdev, &sync, NULL);
3695         return ERR_PTR(r);
3696     }
3697 
3698     radeon_ring_unlock_commit(rdev, ring, false);
3699     radeon_sync_free(rdev, &sync, fence);
3700 
3701     return fence;
3702 }
3703 
3704 /*
3705  * IB stuff
3706  */
3707 /**
3708  * cik_ring_ib_execute - emit an IB (Indirect Buffer) on the gfx ring
3709  *
3710  * @rdev: radeon_device pointer
3711  * @ib: radeon indirect buffer object
3712  *
3713  * Emits a DE (drawing engine) or CE (constant engine) IB
3714  * on the gfx ring.  IBs are usually generated by userspace
3715  * acceleration drivers and submitted to the kernel for
3716  * scheduling on the ring.  This function schedules the IB
3717  * on the gfx ring for execution by the GPU.
3718  */
3719 void cik_ring_ib_execute(struct radeon_device *rdev, struct radeon_ib *ib)
3720 {
3721     struct radeon_ring *ring = &rdev->ring[ib->ring];
3722     unsigned vm_id = ib->vm ? ib->vm->ids[ib->ring].id : 0;
3723     u32 header, control = INDIRECT_BUFFER_VALID;
3724 
3725     if (ib->is_const_ib) {
3726         /* set switch buffer packet before const IB */
3727         radeon_ring_write(ring, PACKET3(PACKET3_SWITCH_BUFFER, 0));
3728         radeon_ring_write(ring, 0);
3729 
3730         header = PACKET3(PACKET3_INDIRECT_BUFFER_CONST, 2);
3731     } else {
3732         u32 next_rptr;
3733         if (ring->rptr_save_reg) {
3734             next_rptr = ring->wptr + 3 + 4;
3735             radeon_ring_write(ring, PACKET3(PACKET3_SET_UCONFIG_REG, 1));
3736             radeon_ring_write(ring, ((ring->rptr_save_reg -
3737                           PACKET3_SET_UCONFIG_REG_START) >> 2));
3738             radeon_ring_write(ring, next_rptr);
3739         } else if (rdev->wb.enabled) {
3740             next_rptr = ring->wptr + 5 + 4;
3741             radeon_ring_write(ring, PACKET3(PACKET3_WRITE_DATA, 3));
3742             radeon_ring_write(ring, WRITE_DATA_DST_SEL(1));
3743             radeon_ring_write(ring, ring->next_rptr_gpu_addr & 0xfffffffc);
3744             radeon_ring_write(ring, upper_32_bits(ring->next_rptr_gpu_addr));
3745             radeon_ring_write(ring, next_rptr);
3746         }
3747 
3748         header = PACKET3(PACKET3_INDIRECT_BUFFER, 2);
3749     }
3750 
3751     control |= ib->length_dw | (vm_id << 24);
3752 
3753     radeon_ring_write(ring, header);
3754     radeon_ring_write(ring, (ib->gpu_addr & 0xFFFFFFFC));
3755     radeon_ring_write(ring, upper_32_bits(ib->gpu_addr) & 0xFFFF);
3756     radeon_ring_write(ring, control);
3757 }
3758 
3759 /**
3760  * cik_ib_test - basic gfx ring IB test
3761  *
3762  * @rdev: radeon_device pointer
3763  * @ring: radeon_ring structure holding ring information
3764  *
3765  * Allocate an IB and execute it on the gfx ring (CIK).
3766  * Provides a basic gfx ring test to verify that IBs are working.
3767  * Returns 0 on success, error on failure.
3768  */
3769 int cik_ib_test(struct radeon_device *rdev, struct radeon_ring *ring)
3770 {
3771     struct radeon_ib ib;
3772     uint32_t scratch;
3773     uint32_t tmp = 0;
3774     unsigned i;
3775     int r;
3776 
3777     r = radeon_scratch_get(rdev, &scratch);
3778     if (r) {
3779         DRM_ERROR("radeon: failed to get scratch reg (%d).\n", r);
3780         return r;
3781     }
3782     WREG32(scratch, 0xCAFEDEAD);
3783     r = radeon_ib_get(rdev, ring->idx, &ib, NULL, 256);
3784     if (r) {
3785         DRM_ERROR("radeon: failed to get ib (%d).\n", r);
3786         radeon_scratch_free(rdev, scratch);
3787         return r;
3788     }
3789     ib.ptr[0] = PACKET3(PACKET3_SET_UCONFIG_REG, 1);
3790     ib.ptr[1] = ((scratch - PACKET3_SET_UCONFIG_REG_START) >> 2);
3791     ib.ptr[2] = 0xDEADBEEF;
3792     ib.length_dw = 3;
3793     r = radeon_ib_schedule(rdev, &ib, NULL, false);
3794     if (r) {
3795         radeon_scratch_free(rdev, scratch);
3796         radeon_ib_free(rdev, &ib);
3797         DRM_ERROR("radeon: failed to schedule ib (%d).\n", r);
3798         return r;
3799     }
3800     r = radeon_fence_wait_timeout(ib.fence, false, usecs_to_jiffies(
3801         RADEON_USEC_IB_TEST_TIMEOUT));
3802     if (r < 0) {
3803         DRM_ERROR("radeon: fence wait failed (%d).\n", r);
3804         radeon_scratch_free(rdev, scratch);
3805         radeon_ib_free(rdev, &ib);
3806         return r;
3807     } else if (r == 0) {
3808         DRM_ERROR("radeon: fence wait timed out.\n");
3809         radeon_scratch_free(rdev, scratch);
3810         radeon_ib_free(rdev, &ib);
3811         return -ETIMEDOUT;
3812     }
3813     r = 0;
3814     for (i = 0; i < rdev->usec_timeout; i++) {
3815         tmp = RREG32(scratch);
3816         if (tmp == 0xDEADBEEF)
3817             break;
3818         udelay(1);
3819     }
3820     if (i < rdev->usec_timeout) {
3821         DRM_INFO("ib test on ring %d succeeded in %u usecs\n", ib.fence->ring, i);
3822     } else {
3823         DRM_ERROR("radeon: ib test failed (scratch(0x%04X)=0x%08X)\n",
3824               scratch, tmp);
3825         r = -EINVAL;
3826     }
3827     radeon_scratch_free(rdev, scratch);
3828     radeon_ib_free(rdev, &ib);
3829     return r;
3830 }
3831 
3832 /*
3833  * CP.
3834  * On CIK, gfx and compute now have independant command processors.
3835  *
3836  * GFX
3837  * Gfx consists of a single ring and can process both gfx jobs and
3838  * compute jobs.  The gfx CP consists of three microengines (ME):
3839  * PFP - Pre-Fetch Parser
3840  * ME - Micro Engine
3841  * CE - Constant Engine
3842  * The PFP and ME make up what is considered the Drawing Engine (DE).
3843  * The CE is an asynchronous engine used for updating buffer desciptors
3844  * used by the DE so that they can be loaded into cache in parallel
3845  * while the DE is processing state update packets.
3846  *
3847  * Compute
3848  * The compute CP consists of two microengines (ME):
3849  * MEC1 - Compute MicroEngine 1
3850  * MEC2 - Compute MicroEngine 2
3851  * Each MEC supports 4 compute pipes and each pipe supports 8 queues.
3852  * The queues are exposed to userspace and are programmed directly
3853  * by the compute runtime.
3854  */
3855 /**
3856  * cik_cp_gfx_enable - enable/disable the gfx CP MEs
3857  *
3858  * @rdev: radeon_device pointer
3859  * @enable: enable or disable the MEs
3860  *
3861  * Halts or unhalts the gfx MEs.
3862  */
3863 static void cik_cp_gfx_enable(struct radeon_device *rdev, bool enable)
3864 {
3865     if (enable)
3866         WREG32(CP_ME_CNTL, 0);
3867     else {
3868         if (rdev->asic->copy.copy_ring_index == RADEON_RING_TYPE_GFX_INDEX)
3869             radeon_ttm_set_active_vram_size(rdev, rdev->mc.visible_vram_size);
3870         WREG32(CP_ME_CNTL, (CP_ME_HALT | CP_PFP_HALT | CP_CE_HALT));
3871         rdev->ring[RADEON_RING_TYPE_GFX_INDEX].ready = false;
3872     }
3873     udelay(50);
3874 }
3875 
3876 /**
3877  * cik_cp_gfx_load_microcode - load the gfx CP ME ucode
3878  *
3879  * @rdev: radeon_device pointer
3880  *
3881  * Loads the gfx PFP, ME, and CE ucode.
3882  * Returns 0 for success, -EINVAL if the ucode is not available.
3883  */
3884 static int cik_cp_gfx_load_microcode(struct radeon_device *rdev)
3885 {
3886     int i;
3887 
3888     if (!rdev->me_fw || !rdev->pfp_fw || !rdev->ce_fw)
3889         return -EINVAL;
3890 
3891     cik_cp_gfx_enable(rdev, false);
3892 
3893     if (rdev->new_fw) {
3894         const struct gfx_firmware_header_v1_0 *pfp_hdr =
3895             (const struct gfx_firmware_header_v1_0 *)rdev->pfp_fw->data;
3896         const struct gfx_firmware_header_v1_0 *ce_hdr =
3897             (const struct gfx_firmware_header_v1_0 *)rdev->ce_fw->data;
3898         const struct gfx_firmware_header_v1_0 *me_hdr =
3899             (const struct gfx_firmware_header_v1_0 *)rdev->me_fw->data;
3900         const __le32 *fw_data;
3901         u32 fw_size;
3902 
3903         radeon_ucode_print_gfx_hdr(&pfp_hdr->header);
3904         radeon_ucode_print_gfx_hdr(&ce_hdr->header);
3905         radeon_ucode_print_gfx_hdr(&me_hdr->header);
3906 
3907         /* PFP */
3908         fw_data = (const __le32 *)
3909             (rdev->pfp_fw->data + le32_to_cpu(pfp_hdr->header.ucode_array_offset_bytes));
3910         fw_size = le32_to_cpu(pfp_hdr->header.ucode_size_bytes) / 4;
3911         WREG32(CP_PFP_UCODE_ADDR, 0);
3912         for (i = 0; i < fw_size; i++)
3913             WREG32(CP_PFP_UCODE_DATA, le32_to_cpup(fw_data++));
3914         WREG32(CP_PFP_UCODE_ADDR, le32_to_cpu(pfp_hdr->header.ucode_version));
3915 
3916         /* CE */
3917         fw_data = (const __le32 *)
3918             (rdev->ce_fw->data + le32_to_cpu(ce_hdr->header.ucode_array_offset_bytes));
3919         fw_size = le32_to_cpu(ce_hdr->header.ucode_size_bytes) / 4;
3920         WREG32(CP_CE_UCODE_ADDR, 0);
3921         for (i = 0; i < fw_size; i++)
3922             WREG32(CP_CE_UCODE_DATA, le32_to_cpup(fw_data++));
3923         WREG32(CP_CE_UCODE_ADDR, le32_to_cpu(ce_hdr->header.ucode_version));
3924 
3925         /* ME */
3926         fw_data = (const __be32 *)
3927             (rdev->me_fw->data + le32_to_cpu(me_hdr->header.ucode_array_offset_bytes));
3928         fw_size = le32_to_cpu(me_hdr->header.ucode_size_bytes) / 4;
3929         WREG32(CP_ME_RAM_WADDR, 0);
3930         for (i = 0; i < fw_size; i++)
3931             WREG32(CP_ME_RAM_DATA, le32_to_cpup(fw_data++));
3932         WREG32(CP_ME_RAM_WADDR, le32_to_cpu(me_hdr->header.ucode_version));
3933         WREG32(CP_ME_RAM_RADDR, le32_to_cpu(me_hdr->header.ucode_version));
3934     } else {
3935         const __be32 *fw_data;
3936 
3937         /* PFP */
3938         fw_data = (const __be32 *)rdev->pfp_fw->data;
3939         WREG32(CP_PFP_UCODE_ADDR, 0);
3940         for (i = 0; i < CIK_PFP_UCODE_SIZE; i++)
3941             WREG32(CP_PFP_UCODE_DATA, be32_to_cpup(fw_data++));
3942         WREG32(CP_PFP_UCODE_ADDR, 0);
3943 
3944         /* CE */
3945         fw_data = (const __be32 *)rdev->ce_fw->data;
3946         WREG32(CP_CE_UCODE_ADDR, 0);
3947         for (i = 0; i < CIK_CE_UCODE_SIZE; i++)
3948             WREG32(CP_CE_UCODE_DATA, be32_to_cpup(fw_data++));
3949         WREG32(CP_CE_UCODE_ADDR, 0);
3950 
3951         /* ME */
3952         fw_data = (const __be32 *)rdev->me_fw->data;
3953         WREG32(CP_ME_RAM_WADDR, 0);
3954         for (i = 0; i < CIK_ME_UCODE_SIZE; i++)
3955             WREG32(CP_ME_RAM_DATA, be32_to_cpup(fw_data++));
3956         WREG32(CP_ME_RAM_WADDR, 0);
3957     }
3958 
3959     return 0;
3960 }
3961 
3962 /**
3963  * cik_cp_gfx_start - start the gfx ring
3964  *
3965  * @rdev: radeon_device pointer
3966  *
3967  * Enables the ring and loads the clear state context and other
3968  * packets required to init the ring.
3969  * Returns 0 for success, error for failure.
3970  */
3971 static int cik_cp_gfx_start(struct radeon_device *rdev)
3972 {
3973     struct radeon_ring *ring = &rdev->ring[RADEON_RING_TYPE_GFX_INDEX];
3974     int r, i;
3975 
3976     /* init the CP */
3977     WREG32(CP_MAX_CONTEXT, rdev->config.cik.max_hw_contexts - 1);
3978     WREG32(CP_ENDIAN_SWAP, 0);
3979     WREG32(CP_DEVICE_ID, 1);
3980 
3981     cik_cp_gfx_enable(rdev, true);
3982 
3983     r = radeon_ring_lock(rdev, ring, cik_default_size + 17);
3984     if (r) {
3985         DRM_ERROR("radeon: cp failed to lock ring (%d).\n", r);
3986         return r;
3987     }
3988 
3989     /* init the CE partitions.  CE only used for gfx on CIK */
3990     radeon_ring_write(ring, PACKET3(PACKET3_SET_BASE, 2));
3991     radeon_ring_write(ring, PACKET3_BASE_INDEX(CE_PARTITION_BASE));
3992     radeon_ring_write(ring, 0x8000);
3993     radeon_ring_write(ring, 0x8000);
3994 
3995     /* setup clear context state */
3996     radeon_ring_write(ring, PACKET3(PACKET3_PREAMBLE_CNTL, 0));
3997     radeon_ring_write(ring, PACKET3_PREAMBLE_BEGIN_CLEAR_STATE);
3998 
3999     radeon_ring_write(ring, PACKET3(PACKET3_CONTEXT_CONTROL, 1));
4000     radeon_ring_write(ring, 0x80000000);
4001     radeon_ring_write(ring, 0x80000000);
4002 
4003     for (i = 0; i < cik_default_size; i++)
4004         radeon_ring_write(ring, cik_default_state[i]);
4005 
4006     radeon_ring_write(ring, PACKET3(PACKET3_PREAMBLE_CNTL, 0));
4007     radeon_ring_write(ring, PACKET3_PREAMBLE_END_CLEAR_STATE);
4008 
4009     /* set clear context state */
4010     radeon_ring_write(ring, PACKET3(PACKET3_CLEAR_STATE, 0));
4011     radeon_ring_write(ring, 0);
4012 
4013     radeon_ring_write(ring, PACKET3(PACKET3_SET_CONTEXT_REG, 2));
4014     radeon_ring_write(ring, 0x00000316);
4015     radeon_ring_write(ring, 0x0000000e); /* VGT_VERTEX_REUSE_BLOCK_CNTL */
4016     radeon_ring_write(ring, 0x00000010); /* VGT_OUT_DEALLOC_CNTL */
4017 
4018     radeon_ring_unlock_commit(rdev, ring, false);
4019 
4020     return 0;
4021 }
4022 
4023 /**
4024  * cik_cp_gfx_fini - stop the gfx ring
4025  *
4026  * @rdev: radeon_device pointer
4027  *
4028  * Stop the gfx ring and tear down the driver ring
4029  * info.
4030  */
4031 static void cik_cp_gfx_fini(struct radeon_device *rdev)
4032 {
4033     cik_cp_gfx_enable(rdev, false);
4034     radeon_ring_fini(rdev, &rdev->ring[RADEON_RING_TYPE_GFX_INDEX]);
4035 }
4036 
4037 /**
4038  * cik_cp_gfx_resume - setup the gfx ring buffer registers
4039  *
4040  * @rdev: radeon_device pointer
4041  *
4042  * Program the location and size of the gfx ring buffer
4043  * and test it to make sure it's working.
4044  * Returns 0 for success, error for failure.
4045  */
4046 static int cik_cp_gfx_resume(struct radeon_device *rdev)
4047 {
4048     struct radeon_ring *ring;
4049     u32 tmp;
4050     u32 rb_bufsz;
4051     u64 rb_addr;
4052     int r;
4053 
4054     WREG32(CP_SEM_WAIT_TIMER, 0x0);
4055     if (rdev->family != CHIP_HAWAII)
4056         WREG32(CP_SEM_INCOMPLETE_TIMER_CNTL, 0x0);
4057 
4058     /* Set the write pointer delay */
4059     WREG32(CP_RB_WPTR_DELAY, 0);
4060 
4061     /* set the RB to use vmid 0 */
4062     WREG32(CP_RB_VMID, 0);
4063 
4064     WREG32(SCRATCH_ADDR, ((rdev->wb.gpu_addr + RADEON_WB_SCRATCH_OFFSET) >> 8) & 0xFFFFFFFF);
4065 
4066     /* ring 0 - compute and gfx */
4067     /* Set ring buffer size */
4068     ring = &rdev->ring[RADEON_RING_TYPE_GFX_INDEX];
4069     rb_bufsz = order_base_2(ring->ring_size / 8);
4070     tmp = (order_base_2(RADEON_GPU_PAGE_SIZE/8) << 8) | rb_bufsz;
4071 #ifdef __BIG_ENDIAN
4072     tmp |= BUF_SWAP_32BIT;
4073 #endif
4074     WREG32(CP_RB0_CNTL, tmp);
4075 
4076     /* Initialize the ring buffer's read and write pointers */
4077     WREG32(CP_RB0_CNTL, tmp | RB_RPTR_WR_ENA);
4078     ring->wptr = 0;
4079     WREG32(CP_RB0_WPTR, ring->wptr);
4080 
4081     /* set the wb address wether it's enabled or not */
4082     WREG32(CP_RB0_RPTR_ADDR, (rdev->wb.gpu_addr + RADEON_WB_CP_RPTR_OFFSET) & 0xFFFFFFFC);
4083     WREG32(CP_RB0_RPTR_ADDR_HI, upper_32_bits(rdev->wb.gpu_addr + RADEON_WB_CP_RPTR_OFFSET) & 0xFF);
4084 
4085     /* scratch register shadowing is no longer supported */
4086     WREG32(SCRATCH_UMSK, 0);
4087 
4088     if (!rdev->wb.enabled)
4089         tmp |= RB_NO_UPDATE;
4090 
4091     mdelay(1);
4092     WREG32(CP_RB0_CNTL, tmp);
4093 
4094     rb_addr = ring->gpu_addr >> 8;
4095     WREG32(CP_RB0_BASE, rb_addr);
4096     WREG32(CP_RB0_BASE_HI, upper_32_bits(rb_addr));
4097 
4098     /* start the ring */
4099     cik_cp_gfx_start(rdev);
4100     rdev->ring[RADEON_RING_TYPE_GFX_INDEX].ready = true;
4101     r = radeon_ring_test(rdev, RADEON_RING_TYPE_GFX_INDEX, &rdev->ring[RADEON_RING_TYPE_GFX_INDEX]);
4102     if (r) {
4103         rdev->ring[RADEON_RING_TYPE_GFX_INDEX].ready = false;
4104         return r;
4105     }
4106 
4107     if (rdev->asic->copy.copy_ring_index == RADEON_RING_TYPE_GFX_INDEX)
4108         radeon_ttm_set_active_vram_size(rdev, rdev->mc.real_vram_size);
4109 
4110     return 0;
4111 }
4112 
4113 u32 cik_gfx_get_rptr(struct radeon_device *rdev,
4114              struct radeon_ring *ring)
4115 {
4116     u32 rptr;
4117 
4118     if (rdev->wb.enabled)
4119         rptr = rdev->wb.wb[ring->rptr_offs/4];
4120     else
4121         rptr = RREG32(CP_RB0_RPTR);
4122 
4123     return rptr;
4124 }
4125 
4126 u32 cik_gfx_get_wptr(struct radeon_device *rdev,
4127              struct radeon_ring *ring)
4128 {
4129     return RREG32(CP_RB0_WPTR);
4130 }
4131 
4132 void cik_gfx_set_wptr(struct radeon_device *rdev,
4133               struct radeon_ring *ring)
4134 {
4135     WREG32(CP_RB0_WPTR, ring->wptr);
4136     (void)RREG32(CP_RB0_WPTR);
4137 }
4138 
4139 u32 cik_compute_get_rptr(struct radeon_device *rdev,
4140              struct radeon_ring *ring)
4141 {
4142     u32 rptr;
4143 
4144     if (rdev->wb.enabled) {
4145         rptr = rdev->wb.wb[ring->rptr_offs/4];
4146     } else {
4147         mutex_lock(&rdev->srbm_mutex);
4148         cik_srbm_select(rdev, ring->me, ring->pipe, ring->queue, 0);
4149         rptr = RREG32(CP_HQD_PQ_RPTR);
4150         cik_srbm_select(rdev, 0, 0, 0, 0);
4151         mutex_unlock(&rdev->srbm_mutex);
4152     }
4153 
4154     return rptr;
4155 }
4156 
4157 u32 cik_compute_get_wptr(struct radeon_device *rdev,
4158              struct radeon_ring *ring)
4159 {
4160     u32 wptr;
4161 
4162     if (rdev->wb.enabled) {
4163         /* XXX check if swapping is necessary on BE */
4164         wptr = rdev->wb.wb[ring->wptr_offs/4];
4165     } else {
4166         mutex_lock(&rdev->srbm_mutex);
4167         cik_srbm_select(rdev, ring->me, ring->pipe, ring->queue, 0);
4168         wptr = RREG32(CP_HQD_PQ_WPTR);
4169         cik_srbm_select(rdev, 0, 0, 0, 0);
4170         mutex_unlock(&rdev->srbm_mutex);
4171     }
4172 
4173     return wptr;
4174 }
4175 
4176 void cik_compute_set_wptr(struct radeon_device *rdev,
4177               struct radeon_ring *ring)
4178 {
4179     /* XXX check if swapping is necessary on BE */
4180     rdev->wb.wb[ring->wptr_offs/4] = ring->wptr;
4181     WDOORBELL32(ring->doorbell_index, ring->wptr);
4182 }
4183 
4184 static void cik_compute_stop(struct radeon_device *rdev,
4185                  struct radeon_ring *ring)
4186 {
4187     u32 j, tmp;
4188 
4189     cik_srbm_select(rdev, ring->me, ring->pipe, ring->queue, 0);
4190     /* Disable wptr polling. */
4191     tmp = RREG32(CP_PQ_WPTR_POLL_CNTL);
4192     tmp &= ~WPTR_POLL_EN;
4193     WREG32(CP_PQ_WPTR_POLL_CNTL, tmp);
4194     /* Disable HQD. */
4195     if (RREG32(CP_HQD_ACTIVE) & 1) {
4196         WREG32(CP_HQD_DEQUEUE_REQUEST, 1);
4197         for (j = 0; j < rdev->usec_timeout; j++) {
4198             if (!(RREG32(CP_HQD_ACTIVE) & 1))
4199                 break;
4200             udelay(1);
4201         }
4202         WREG32(CP_HQD_DEQUEUE_REQUEST, 0);
4203         WREG32(CP_HQD_PQ_RPTR, 0);
4204         WREG32(CP_HQD_PQ_WPTR, 0);
4205     }
4206     cik_srbm_select(rdev, 0, 0, 0, 0);
4207 }
4208 
4209 /**
4210  * cik_cp_compute_enable - enable/disable the compute CP MEs
4211  *
4212  * @rdev: radeon_device pointer
4213  * @enable: enable or disable the MEs
4214  *
4215  * Halts or unhalts the compute MEs.
4216  */
4217 static void cik_cp_compute_enable(struct radeon_device *rdev, bool enable)
4218 {
4219     if (enable)
4220         WREG32(CP_MEC_CNTL, 0);
4221     else {
4222         /*
4223          * To make hibernation reliable we need to clear compute ring
4224          * configuration before halting the compute ring.
4225          */
4226         mutex_lock(&rdev->srbm_mutex);
4227         cik_compute_stop(rdev,&rdev->ring[CAYMAN_RING_TYPE_CP1_INDEX]);
4228         cik_compute_stop(rdev,&rdev->ring[CAYMAN_RING_TYPE_CP2_INDEX]);
4229         mutex_unlock(&rdev->srbm_mutex);
4230 
4231         WREG32(CP_MEC_CNTL, (MEC_ME1_HALT | MEC_ME2_HALT));
4232         rdev->ring[CAYMAN_RING_TYPE_CP1_INDEX].ready = false;
4233         rdev->ring[CAYMAN_RING_TYPE_CP2_INDEX].ready = false;
4234     }
4235     udelay(50);
4236 }
4237 
4238 /**
4239  * cik_cp_compute_load_microcode - load the compute CP ME ucode
4240  *
4241  * @rdev: radeon_device pointer
4242  *
4243  * Loads the compute MEC1&2 ucode.
4244  * Returns 0 for success, -EINVAL if the ucode is not available.
4245  */
4246 static int cik_cp_compute_load_microcode(struct radeon_device *rdev)
4247 {
4248     int i;
4249 
4250     if (!rdev->mec_fw)
4251         return -EINVAL;
4252 
4253     cik_cp_compute_enable(rdev, false);
4254 
4255     if (rdev->new_fw) {
4256         const struct gfx_firmware_header_v1_0 *mec_hdr =
4257             (const struct gfx_firmware_header_v1_0 *)rdev->mec_fw->data;
4258         const __le32 *fw_data;
4259         u32 fw_size;
4260 
4261         radeon_ucode_print_gfx_hdr(&mec_hdr->header);
4262 
4263         /* MEC1 */
4264         fw_data = (const __le32 *)
4265             (rdev->mec_fw->data + le32_to_cpu(mec_hdr->header.ucode_array_offset_bytes));
4266         fw_size = le32_to_cpu(mec_hdr->header.ucode_size_bytes) / 4;
4267         WREG32(CP_MEC_ME1_UCODE_ADDR, 0);
4268         for (i = 0; i < fw_size; i++)
4269             WREG32(CP_MEC_ME1_UCODE_DATA, le32_to_cpup(fw_data++));
4270         WREG32(CP_MEC_ME1_UCODE_ADDR, le32_to_cpu(mec_hdr->header.ucode_version));
4271 
4272         /* MEC2 */
4273         if (rdev->family == CHIP_KAVERI) {
4274             const struct gfx_firmware_header_v1_0 *mec2_hdr =
4275                 (const struct gfx_firmware_header_v1_0 *)rdev->mec2_fw->data;
4276 
4277             fw_data = (const __le32 *)
4278                 (rdev->mec2_fw->data +
4279                  le32_to_cpu(mec2_hdr->header.ucode_array_offset_bytes));
4280             fw_size = le32_to_cpu(mec2_hdr->header.ucode_size_bytes) / 4;
4281             WREG32(CP_MEC_ME2_UCODE_ADDR, 0);
4282             for (i = 0; i < fw_size; i++)
4283                 WREG32(CP_MEC_ME2_UCODE_DATA, le32_to_cpup(fw_data++));
4284             WREG32(CP_MEC_ME2_UCODE_ADDR, le32_to_cpu(mec2_hdr->header.ucode_version));
4285         }
4286     } else {
4287         const __be32 *fw_data;
4288 
4289         /* MEC1 */
4290         fw_data = (const __be32 *)rdev->mec_fw->data;
4291         WREG32(CP_MEC_ME1_UCODE_ADDR, 0);
4292         for (i = 0; i < CIK_MEC_UCODE_SIZE; i++)
4293             WREG32(CP_MEC_ME1_UCODE_DATA, be32_to_cpup(fw_data++));
4294         WREG32(CP_MEC_ME1_UCODE_ADDR, 0);
4295 
4296         if (rdev->family == CHIP_KAVERI) {
4297             /* MEC2 */
4298             fw_data = (const __be32 *)rdev->mec_fw->data;
4299             WREG32(CP_MEC_ME2_UCODE_ADDR, 0);
4300             for (i = 0; i < CIK_MEC_UCODE_SIZE; i++)
4301                 WREG32(CP_MEC_ME2_UCODE_DATA, be32_to_cpup(fw_data++));
4302             WREG32(CP_MEC_ME2_UCODE_ADDR, 0);
4303         }
4304     }
4305 
4306     return 0;
4307 }
4308 
4309 /**
4310  * cik_cp_compute_start - start the compute queues
4311  *
4312  * @rdev: radeon_device pointer
4313  *
4314  * Enable the compute queues.
4315  * Returns 0 for success, error for failure.
4316  */
4317 static int cik_cp_compute_start(struct radeon_device *rdev)
4318 {
4319     cik_cp_compute_enable(rdev, true);
4320 
4321     return 0;
4322 }
4323 
4324 /**
4325  * cik_cp_compute_fini - stop the compute queues
4326  *
4327  * @rdev: radeon_device pointer
4328  *
4329  * Stop the compute queues and tear down the driver queue
4330  * info.
4331  */
4332 static void cik_cp_compute_fini(struct radeon_device *rdev)
4333 {
4334     int i, idx, r;
4335 
4336     cik_cp_compute_enable(rdev, false);
4337 
4338     for (i = 0; i < 2; i++) {
4339         if (i == 0)
4340             idx = CAYMAN_RING_TYPE_CP1_INDEX;
4341         else
4342             idx = CAYMAN_RING_TYPE_CP2_INDEX;
4343 
4344         if (rdev->ring[idx].mqd_obj) {
4345             r = radeon_bo_reserve(rdev->ring[idx].mqd_obj, false);
4346             if (unlikely(r != 0))
4347                 dev_warn(rdev->dev, "(%d) reserve MQD bo failed\n", r);
4348 
4349             radeon_bo_unpin(rdev->ring[idx].mqd_obj);
4350             radeon_bo_unreserve(rdev->ring[idx].mqd_obj);
4351 
4352             radeon_bo_unref(&rdev->ring[idx].mqd_obj);
4353             rdev->ring[idx].mqd_obj = NULL;
4354         }
4355     }
4356 }
4357 
4358 static void cik_mec_fini(struct radeon_device *rdev)
4359 {
4360     int r;
4361 
4362     if (rdev->mec.hpd_eop_obj) {
4363         r = radeon_bo_reserve(rdev->mec.hpd_eop_obj, false);
4364         if (unlikely(r != 0))
4365             dev_warn(rdev->dev, "(%d) reserve HPD EOP bo failed\n", r);
4366         radeon_bo_unpin(rdev->mec.hpd_eop_obj);
4367         radeon_bo_unreserve(rdev->mec.hpd_eop_obj);
4368 
4369         radeon_bo_unref(&rdev->mec.hpd_eop_obj);
4370         rdev->mec.hpd_eop_obj = NULL;
4371     }
4372 }
4373 
4374 #define MEC_HPD_SIZE 2048
4375 
4376 static int cik_mec_init(struct radeon_device *rdev)
4377 {
4378     int r;
4379     u32 *hpd;
4380 
4381     /*
4382      * KV:    2 MEC, 4 Pipes/MEC, 8 Queues/Pipe - 64 Queues total
4383      * CI/KB: 1 MEC, 4 Pipes/MEC, 8 Queues/Pipe - 32 Queues total
4384      */
4385     if (rdev->family == CHIP_KAVERI)
4386         rdev->mec.num_mec = 2;
4387     else
4388         rdev->mec.num_mec = 1;
4389     rdev->mec.num_pipe = 4;
4390     rdev->mec.num_queue = rdev->mec.num_mec * rdev->mec.num_pipe * 8;
4391 
4392     if (rdev->mec.hpd_eop_obj == NULL) {
4393         r = radeon_bo_create(rdev,
4394                      rdev->mec.num_mec *rdev->mec.num_pipe * MEC_HPD_SIZE * 2,
4395                      PAGE_SIZE, true,
4396                      RADEON_GEM_DOMAIN_GTT, 0, NULL, NULL,
4397                      &rdev->mec.hpd_eop_obj);
4398         if (r) {
4399             dev_warn(rdev->dev, "(%d) create HDP EOP bo failed\n", r);
4400             return r;
4401         }
4402     }
4403 
4404     r = radeon_bo_reserve(rdev->mec.hpd_eop_obj, false);
4405     if (unlikely(r != 0)) {
4406         cik_mec_fini(rdev);
4407         return r;
4408     }
4409     r = radeon_bo_pin(rdev->mec.hpd_eop_obj, RADEON_GEM_DOMAIN_GTT,
4410               &rdev->mec.hpd_eop_gpu_addr);
4411     if (r) {
4412         dev_warn(rdev->dev, "(%d) pin HDP EOP bo failed\n", r);
4413         cik_mec_fini(rdev);
4414         return r;
4415     }
4416     r = radeon_bo_kmap(rdev->mec.hpd_eop_obj, (void **)&hpd);
4417     if (r) {
4418         dev_warn(rdev->dev, "(%d) map HDP EOP bo failed\n", r);
4419         cik_mec_fini(rdev);
4420         return r;
4421     }
4422 
4423     /* clear memory.  Not sure if this is required or not */
4424     memset(hpd, 0, rdev->mec.num_mec *rdev->mec.num_pipe * MEC_HPD_SIZE * 2);
4425 
4426     radeon_bo_kunmap(rdev->mec.hpd_eop_obj);
4427     radeon_bo_unreserve(rdev->mec.hpd_eop_obj);
4428 
4429     return 0;
4430 }
4431 
4432 struct hqd_registers
4433 {
4434     u32 cp_mqd_base_addr;
4435     u32 cp_mqd_base_addr_hi;
4436     u32 cp_hqd_active;
4437     u32 cp_hqd_vmid;
4438     u32 cp_hqd_persistent_state;
4439     u32 cp_hqd_pipe_priority;
4440     u32 cp_hqd_queue_priority;
4441     u32 cp_hqd_quantum;
4442     u32 cp_hqd_pq_base;
4443     u32 cp_hqd_pq_base_hi;
4444     u32 cp_hqd_pq_rptr;
4445     u32 cp_hqd_pq_rptr_report_addr;
4446     u32 cp_hqd_pq_rptr_report_addr_hi;
4447     u32 cp_hqd_pq_wptr_poll_addr;
4448     u32 cp_hqd_pq_wptr_poll_addr_hi;
4449     u32 cp_hqd_pq_doorbell_control;
4450     u32 cp_hqd_pq_wptr;
4451     u32 cp_hqd_pq_control;
4452     u32 cp_hqd_ib_base_addr;
4453     u32 cp_hqd_ib_base_addr_hi;
4454     u32 cp_hqd_ib_rptr;
4455     u32 cp_hqd_ib_control;
4456     u32 cp_hqd_iq_timer;
4457     u32 cp_hqd_iq_rptr;
4458     u32 cp_hqd_dequeue_request;
4459     u32 cp_hqd_dma_offload;
4460     u32 cp_hqd_sema_cmd;
4461     u32 cp_hqd_msg_type;
4462     u32 cp_hqd_atomic0_preop_lo;
4463     u32 cp_hqd_atomic0_preop_hi;
4464     u32 cp_hqd_atomic1_preop_lo;
4465     u32 cp_hqd_atomic1_preop_hi;
4466     u32 cp_hqd_hq_scheduler0;
4467     u32 cp_hqd_hq_scheduler1;
4468     u32 cp_mqd_control;
4469 };
4470 
4471 struct bonaire_mqd
4472 {
4473     u32 header;
4474     u32 dispatch_initiator;
4475     u32 dimensions[3];
4476     u32 start_idx[3];
4477     u32 num_threads[3];
4478     u32 pipeline_stat_enable;
4479     u32 perf_counter_enable;
4480     u32 pgm[2];
4481     u32 tba[2];
4482     u32 tma[2];
4483     u32 pgm_rsrc[2];
4484     u32 vmid;
4485     u32 resource_limits;
4486     u32 static_thread_mgmt01[2];
4487     u32 tmp_ring_size;
4488     u32 static_thread_mgmt23[2];
4489     u32 restart[3];
4490     u32 thread_trace_enable;
4491     u32 reserved1;
4492     u32 user_data[16];
4493     u32 vgtcs_invoke_count[2];
4494     struct hqd_registers queue_state;
4495     u32 dequeue_cntr;
4496     u32 interrupt_queue[64];
4497 };
4498 
4499 /**
4500  * cik_cp_compute_resume - setup the compute queue registers
4501  *
4502  * @rdev: radeon_device pointer
4503  *
4504  * Program the compute queues and test them to make sure they
4505  * are working.
4506  * Returns 0 for success, error for failure.
4507  */
4508 static int cik_cp_compute_resume(struct radeon_device *rdev)
4509 {
4510     int r, i, j, idx;
4511     u32 tmp;
4512     bool use_doorbell = true;
4513     u64 hqd_gpu_addr;
4514     u64 mqd_gpu_addr;
4515     u64 eop_gpu_addr;
4516     u64 wb_gpu_addr;
4517     u32 *buf;
4518     struct bonaire_mqd *mqd;
4519 
4520     r = cik_cp_compute_start(rdev);
4521     if (r)
4522         return r;
4523 
4524     /* fix up chicken bits */
4525     tmp = RREG32(CP_CPF_DEBUG);
4526     tmp |= (1 << 23);
4527     WREG32(CP_CPF_DEBUG, tmp);
4528 
4529     /* init the pipes */
4530     mutex_lock(&rdev->srbm_mutex);
4531 
4532     for (i = 0; i < (rdev->mec.num_pipe * rdev->mec.num_mec); ++i) {
4533         int me = (i < 4) ? 1 : 2;
4534         int pipe = (i < 4) ? i : (i - 4);
4535 
4536         cik_srbm_select(rdev, me, pipe, 0, 0);
4537 
4538         eop_gpu_addr = rdev->mec.hpd_eop_gpu_addr + (i * MEC_HPD_SIZE * 2) ;
4539         /* write the EOP addr */
4540         WREG32(CP_HPD_EOP_BASE_ADDR, eop_gpu_addr >> 8);
4541         WREG32(CP_HPD_EOP_BASE_ADDR_HI, upper_32_bits(eop_gpu_addr) >> 8);
4542 
4543         /* set the VMID assigned */
4544         WREG32(CP_HPD_EOP_VMID, 0);
4545 
4546         /* set the EOP size, register value is 2^(EOP_SIZE+1) dwords */
4547         tmp = RREG32(CP_HPD_EOP_CONTROL);
4548         tmp &= ~EOP_SIZE_MASK;
4549         tmp |= order_base_2(MEC_HPD_SIZE / 8);
4550         WREG32(CP_HPD_EOP_CONTROL, tmp);
4551 
4552     }
4553     cik_srbm_select(rdev, 0, 0, 0, 0);
4554     mutex_unlock(&rdev->srbm_mutex);
4555 
4556     /* init the queues.  Just two for now. */
4557     for (i = 0; i < 2; i++) {
4558         if (i == 0)
4559             idx = CAYMAN_RING_TYPE_CP1_INDEX;
4560         else
4561             idx = CAYMAN_RING_TYPE_CP2_INDEX;
4562 
4563         if (rdev->ring[idx].mqd_obj == NULL) {
4564             r = radeon_bo_create(rdev,
4565                          sizeof(struct bonaire_mqd),
4566                          PAGE_SIZE, true,
4567                          RADEON_GEM_DOMAIN_GTT, 0, NULL,
4568                          NULL, &rdev->ring[idx].mqd_obj);
4569             if (r) {
4570                 dev_warn(rdev->dev, "(%d) create MQD bo failed\n", r);
4571                 return r;
4572             }
4573         }
4574 
4575         r = radeon_bo_reserve(rdev->ring[idx].mqd_obj, false);
4576         if (unlikely(r != 0)) {
4577             cik_cp_compute_fini(rdev);
4578             return r;
4579         }
4580         r = radeon_bo_pin(rdev->ring[idx].mqd_obj, RADEON_GEM_DOMAIN_GTT,
4581                   &mqd_gpu_addr);
4582         if (r) {
4583             dev_warn(rdev->dev, "(%d) pin MQD bo failed\n", r);
4584             cik_cp_compute_fini(rdev);
4585             return r;
4586         }
4587         r = radeon_bo_kmap(rdev->ring[idx].mqd_obj, (void **)&buf);
4588         if (r) {
4589             dev_warn(rdev->dev, "(%d) map MQD bo failed\n", r);
4590             cik_cp_compute_fini(rdev);
4591             return r;
4592         }
4593 
4594         /* init the mqd struct */
4595         memset(buf, 0, sizeof(struct bonaire_mqd));
4596 
4597         mqd = (struct bonaire_mqd *)buf;
4598         mqd->header = 0xC0310800;
4599         mqd->static_thread_mgmt01[0] = 0xffffffff;
4600         mqd->static_thread_mgmt01[1] = 0xffffffff;
4601         mqd->static_thread_mgmt23[0] = 0xffffffff;
4602         mqd->static_thread_mgmt23[1] = 0xffffffff;
4603 
4604         mutex_lock(&rdev->srbm_mutex);
4605         cik_srbm_select(rdev, rdev->ring[idx].me,
4606                 rdev->ring[idx].pipe,
4607                 rdev->ring[idx].queue, 0);
4608 
4609         /* disable wptr polling */
4610         tmp = RREG32(CP_PQ_WPTR_POLL_CNTL);
4611         tmp &= ~WPTR_POLL_EN;
4612         WREG32(CP_PQ_WPTR_POLL_CNTL, tmp);
4613 
4614         /* enable doorbell? */
4615         mqd->queue_state.cp_hqd_pq_doorbell_control =
4616             RREG32(CP_HQD_PQ_DOORBELL_CONTROL);
4617         if (use_doorbell)
4618             mqd->queue_state.cp_hqd_pq_doorbell_control |= DOORBELL_EN;
4619         else
4620             mqd->queue_state.cp_hqd_pq_doorbell_control &= ~DOORBELL_EN;
4621         WREG32(CP_HQD_PQ_DOORBELL_CONTROL,
4622                mqd->queue_state.cp_hqd_pq_doorbell_control);
4623 
4624         /* disable the queue if it's active */
4625         mqd->queue_state.cp_hqd_dequeue_request = 0;
4626         mqd->queue_state.cp_hqd_pq_rptr = 0;
4627         mqd->queue_state.cp_hqd_pq_wptr= 0;
4628         if (RREG32(CP_HQD_ACTIVE) & 1) {
4629             WREG32(CP_HQD_DEQUEUE_REQUEST, 1);
4630             for (j = 0; j < rdev->usec_timeout; j++) {
4631                 if (!(RREG32(CP_HQD_ACTIVE) & 1))
4632                     break;
4633                 udelay(1);
4634             }
4635             WREG32(CP_HQD_DEQUEUE_REQUEST, mqd->queue_state.cp_hqd_dequeue_request);
4636             WREG32(CP_HQD_PQ_RPTR, mqd->queue_state.cp_hqd_pq_rptr);
4637             WREG32(CP_HQD_PQ_WPTR, mqd->queue_state.cp_hqd_pq_wptr);
4638         }
4639 
4640         /* set the pointer to the MQD */
4641         mqd->queue_state.cp_mqd_base_addr = mqd_gpu_addr & 0xfffffffc;
4642         mqd->queue_state.cp_mqd_base_addr_hi = upper_32_bits(mqd_gpu_addr);
4643         WREG32(CP_MQD_BASE_ADDR, mqd->queue_state.cp_mqd_base_addr);
4644         WREG32(CP_MQD_BASE_ADDR_HI, mqd->queue_state.cp_mqd_base_addr_hi);
4645         /* set MQD vmid to 0 */
4646         mqd->queue_state.cp_mqd_control = RREG32(CP_MQD_CONTROL);
4647         mqd->queue_state.cp_mqd_control &= ~MQD_VMID_MASK;
4648         WREG32(CP_MQD_CONTROL, mqd->queue_state.cp_mqd_control);
4649 
4650         /* set the pointer to the HQD, this is similar CP_RB0_BASE/_HI */
4651         hqd_gpu_addr = rdev->ring[idx].gpu_addr >> 8;
4652         mqd->queue_state.cp_hqd_pq_base = hqd_gpu_addr;
4653         mqd->queue_state.cp_hqd_pq_base_hi = upper_32_bits(hqd_gpu_addr);
4654         WREG32(CP_HQD_PQ_BASE, mqd->queue_state.cp_hqd_pq_base);
4655         WREG32(CP_HQD_PQ_BASE_HI, mqd->queue_state.cp_hqd_pq_base_hi);
4656 
4657         /* set up the HQD, this is similar to CP_RB0_CNTL */
4658         mqd->queue_state.cp_hqd_pq_control = RREG32(CP_HQD_PQ_CONTROL);
4659         mqd->queue_state.cp_hqd_pq_control &=
4660             ~(QUEUE_SIZE_MASK | RPTR_BLOCK_SIZE_MASK);
4661 
4662         mqd->queue_state.cp_hqd_pq_control |=
4663             order_base_2(rdev->ring[idx].ring_size / 8);
4664         mqd->queue_state.cp_hqd_pq_control |=
4665             (order_base_2(RADEON_GPU_PAGE_SIZE/8) << 8);
4666 #ifdef __BIG_ENDIAN
4667         mqd->queue_state.cp_hqd_pq_control |= BUF_SWAP_32BIT;
4668 #endif
4669         mqd->queue_state.cp_hqd_pq_control &=
4670             ~(UNORD_DISPATCH | ROQ_PQ_IB_FLIP | PQ_VOLATILE);
4671         mqd->queue_state.cp_hqd_pq_control |=
4672             PRIV_STATE | KMD_QUEUE; /* assuming kernel queue control */
4673         WREG32(CP_HQD_PQ_CONTROL, mqd->queue_state.cp_hqd_pq_control);
4674 
4675         /* only used if CP_PQ_WPTR_POLL_CNTL.WPTR_POLL_EN=1 */
4676         if (i == 0)
4677             wb_gpu_addr = rdev->wb.gpu_addr + CIK_WB_CP1_WPTR_OFFSET;
4678         else
4679             wb_gpu_addr = rdev->wb.gpu_addr + CIK_WB_CP2_WPTR_OFFSET;
4680         mqd->queue_state.cp_hqd_pq_wptr_poll_addr = wb_gpu_addr & 0xfffffffc;
4681         mqd->queue_state.cp_hqd_pq_wptr_poll_addr_hi = upper_32_bits(wb_gpu_addr) & 0xffff;
4682         WREG32(CP_HQD_PQ_WPTR_POLL_ADDR, mqd->queue_state.cp_hqd_pq_wptr_poll_addr);
4683         WREG32(CP_HQD_PQ_WPTR_POLL_ADDR_HI,
4684                mqd->queue_state.cp_hqd_pq_wptr_poll_addr_hi);
4685 
4686         /* set the wb address wether it's enabled or not */
4687         if (i == 0)
4688             wb_gpu_addr = rdev->wb.gpu_addr + RADEON_WB_CP1_RPTR_OFFSET;
4689         else
4690             wb_gpu_addr = rdev->wb.gpu_addr + RADEON_WB_CP2_RPTR_OFFSET;
4691         mqd->queue_state.cp_hqd_pq_rptr_report_addr = wb_gpu_addr & 0xfffffffc;
4692         mqd->queue_state.cp_hqd_pq_rptr_report_addr_hi =
4693             upper_32_bits(wb_gpu_addr) & 0xffff;
4694         WREG32(CP_HQD_PQ_RPTR_REPORT_ADDR,
4695                mqd->queue_state.cp_hqd_pq_rptr_report_addr);
4696         WREG32(CP_HQD_PQ_RPTR_REPORT_ADDR_HI,
4697                mqd->queue_state.cp_hqd_pq_rptr_report_addr_hi);
4698 
4699         /* enable the doorbell if requested */
4700         if (use_doorbell) {
4701             mqd->queue_state.cp_hqd_pq_doorbell_control =
4702                 RREG32(CP_HQD_PQ_DOORBELL_CONTROL);
4703             mqd->queue_state.cp_hqd_pq_doorbell_control &= ~DOORBELL_OFFSET_MASK;
4704             mqd->queue_state.cp_hqd_pq_doorbell_control |=
4705                 DOORBELL_OFFSET(rdev->ring[idx].doorbell_index);
4706             mqd->queue_state.cp_hqd_pq_doorbell_control |= DOORBELL_EN;
4707             mqd->queue_state.cp_hqd_pq_doorbell_control &=
4708                 ~(DOORBELL_SOURCE | DOORBELL_HIT);
4709 
4710         } else {
4711             mqd->queue_state.cp_hqd_pq_doorbell_control = 0;
4712         }
4713         WREG32(CP_HQD_PQ_DOORBELL_CONTROL,
4714                mqd->queue_state.cp_hqd_pq_doorbell_control);
4715 
4716         /* read and write pointers, similar to CP_RB0_WPTR/_RPTR */
4717         rdev->ring[idx].wptr = 0;
4718         mqd->queue_state.cp_hqd_pq_wptr = rdev->ring[idx].wptr;
4719         WREG32(CP_HQD_PQ_WPTR, mqd->queue_state.cp_hqd_pq_wptr);
4720         mqd->queue_state.cp_hqd_pq_rptr = RREG32(CP_HQD_PQ_RPTR);
4721 
4722         /* set the vmid for the queue */
4723         mqd->queue_state.cp_hqd_vmid = 0;
4724         WREG32(CP_HQD_VMID, mqd->queue_state.cp_hqd_vmid);
4725 
4726         /* activate the queue */
4727         mqd->queue_state.cp_hqd_active = 1;
4728         WREG32(CP_HQD_ACTIVE, mqd->queue_state.cp_hqd_active);
4729 
4730         cik_srbm_select(rdev, 0, 0, 0, 0);
4731         mutex_unlock(&rdev->srbm_mutex);
4732 
4733         radeon_bo_kunmap(rdev->ring[idx].mqd_obj);
4734         radeon_bo_unreserve(rdev->ring[idx].mqd_obj);
4735 
4736         rdev->ring[idx].ready = true;
4737         r = radeon_ring_test(rdev, idx, &rdev->ring[idx]);
4738         if (r)
4739             rdev->ring[idx].ready = false;
4740     }
4741 
4742     return 0;
4743 }
4744 
4745 static void cik_cp_enable(struct radeon_device *rdev, bool enable)
4746 {
4747     cik_cp_gfx_enable(rdev, enable);
4748     cik_cp_compute_enable(rdev, enable);
4749 }
4750 
4751 static int cik_cp_load_microcode(struct radeon_device *rdev)
4752 {
4753     int r;
4754 
4755     r = cik_cp_gfx_load_microcode(rdev);
4756     if (r)
4757         return r;
4758     r = cik_cp_compute_load_microcode(rdev);
4759     if (r)
4760         return r;
4761 
4762     return 0;
4763 }
4764 
4765 static void cik_cp_fini(struct radeon_device *rdev)
4766 {
4767     cik_cp_gfx_fini(rdev);
4768     cik_cp_compute_fini(rdev);
4769 }
4770 
4771 static int cik_cp_resume(struct radeon_device *rdev)
4772 {
4773     int r;
4774 
4775     cik_enable_gui_idle_interrupt(rdev, false);
4776 
4777     r = cik_cp_load_microcode(rdev);
4778     if (r)
4779         return r;
4780 
4781     r = cik_cp_gfx_resume(rdev);
4782     if (r)
4783         return r;
4784     r = cik_cp_compute_resume(rdev);
4785     if (r)
4786         return r;
4787 
4788     cik_enable_gui_idle_interrupt(rdev, true);
4789 
4790     return 0;
4791 }
4792 
4793 static void cik_print_gpu_status_regs(struct radeon_device *rdev)
4794 {
4795     dev_info(rdev->dev, "  GRBM_STATUS=0x%08X\n",
4796         RREG32(GRBM_STATUS));
4797     dev_info(rdev->dev, "  GRBM_STATUS2=0x%08X\n",
4798         RREG32(GRBM_STATUS2));
4799     dev_info(rdev->dev, "  GRBM_STATUS_SE0=0x%08X\n",
4800         RREG32(GRBM_STATUS_SE0));
4801     dev_info(rdev->dev, "  GRBM_STATUS_SE1=0x%08X\n",
4802         RREG32(GRBM_STATUS_SE1));
4803     dev_info(rdev->dev, "  GRBM_STATUS_SE2=0x%08X\n",
4804         RREG32(GRBM_STATUS_SE2));
4805     dev_info(rdev->dev, "  GRBM_STATUS_SE3=0x%08X\n",
4806         RREG32(GRBM_STATUS_SE3));
4807     dev_info(rdev->dev, "  SRBM_STATUS=0x%08X\n",
4808         RREG32(SRBM_STATUS));
4809     dev_info(rdev->dev, "  SRBM_STATUS2=0x%08X\n",
4810         RREG32(SRBM_STATUS2));
4811     dev_info(rdev->dev, "  SDMA0_STATUS_REG   = 0x%08X\n",
4812         RREG32(SDMA0_STATUS_REG + SDMA0_REGISTER_OFFSET));
4813     dev_info(rdev->dev, "  SDMA1_STATUS_REG   = 0x%08X\n",
4814          RREG32(SDMA0_STATUS_REG + SDMA1_REGISTER_OFFSET));
4815     dev_info(rdev->dev, "  CP_STAT = 0x%08x\n", RREG32(CP_STAT));
4816     dev_info(rdev->dev, "  CP_STALLED_STAT1 = 0x%08x\n",
4817          RREG32(CP_STALLED_STAT1));
4818     dev_info(rdev->dev, "  CP_STALLED_STAT2 = 0x%08x\n",
4819          RREG32(CP_STALLED_STAT2));
4820     dev_info(rdev->dev, "  CP_STALLED_STAT3 = 0x%08x\n",
4821          RREG32(CP_STALLED_STAT3));
4822     dev_info(rdev->dev, "  CP_CPF_BUSY_STAT = 0x%08x\n",
4823          RREG32(CP_CPF_BUSY_STAT));
4824     dev_info(rdev->dev, "  CP_CPF_STALLED_STAT1 = 0x%08x\n",
4825          RREG32(CP_CPF_STALLED_STAT1));
4826     dev_info(rdev->dev, "  CP_CPF_STATUS = 0x%08x\n", RREG32(CP_CPF_STATUS));
4827     dev_info(rdev->dev, "  CP_CPC_BUSY_STAT = 0x%08x\n", RREG32(CP_CPC_BUSY_STAT));
4828     dev_info(rdev->dev, "  CP_CPC_STALLED_STAT1 = 0x%08x\n",
4829          RREG32(CP_CPC_STALLED_STAT1));
4830     dev_info(rdev->dev, "  CP_CPC_STATUS = 0x%08x\n", RREG32(CP_CPC_STATUS));
4831 }
4832 
4833 /**
4834  * cik_gpu_check_soft_reset - check which blocks are busy
4835  *
4836  * @rdev: radeon_device pointer
4837  *
4838  * Check which blocks are busy and return the relevant reset
4839  * mask to be used by cik_gpu_soft_reset().
4840  * Returns a mask of the blocks to be reset.
4841  */
4842 u32 cik_gpu_check_soft_reset(struct radeon_device *rdev)
4843 {
4844     u32 reset_mask = 0;
4845     u32 tmp;
4846 
4847     /* GRBM_STATUS */
4848     tmp = RREG32(GRBM_STATUS);
4849     if (tmp & (PA_BUSY | SC_BUSY |
4850            BCI_BUSY | SX_BUSY |
4851            TA_BUSY | VGT_BUSY |
4852            DB_BUSY | CB_BUSY |
4853            GDS_BUSY | SPI_BUSY |
4854            IA_BUSY | IA_BUSY_NO_DMA))
4855         reset_mask |= RADEON_RESET_GFX;
4856 
4857     if (tmp & (CP_BUSY | CP_COHERENCY_BUSY))
4858         reset_mask |= RADEON_RESET_CP;
4859 
4860     /* GRBM_STATUS2 */
4861     tmp = RREG32(GRBM_STATUS2);
4862     if (tmp & RLC_BUSY)
4863         reset_mask |= RADEON_RESET_RLC;
4864 
4865     /* SDMA0_STATUS_REG */
4866     tmp = RREG32(SDMA0_STATUS_REG + SDMA0_REGISTER_OFFSET);
4867     if (!(tmp & SDMA_IDLE))
4868         reset_mask |= RADEON_RESET_DMA;
4869 
4870     /* SDMA1_STATUS_REG */
4871     tmp = RREG32(SDMA0_STATUS_REG + SDMA1_REGISTER_OFFSET);
4872     if (!(tmp & SDMA_IDLE))
4873         reset_mask |= RADEON_RESET_DMA1;
4874 
4875     /* SRBM_STATUS2 */
4876     tmp = RREG32(SRBM_STATUS2);
4877     if (tmp & SDMA_BUSY)
4878         reset_mask |= RADEON_RESET_DMA;
4879 
4880     if (tmp & SDMA1_BUSY)
4881         reset_mask |= RADEON_RESET_DMA1;
4882 
4883     /* SRBM_STATUS */
4884     tmp = RREG32(SRBM_STATUS);
4885 
4886     if (tmp & IH_BUSY)
4887         reset_mask |= RADEON_RESET_IH;
4888 
4889     if (tmp & SEM_BUSY)
4890         reset_mask |= RADEON_RESET_SEM;
4891 
4892     if (tmp & GRBM_RQ_PENDING)
4893         reset_mask |= RADEON_RESET_GRBM;
4894 
4895     if (tmp & VMC_BUSY)
4896         reset_mask |= RADEON_RESET_VMC;
4897 
4898     if (tmp & (MCB_BUSY | MCB_NON_DISPLAY_BUSY |
4899            MCC_BUSY | MCD_BUSY))
4900         reset_mask |= RADEON_RESET_MC;
4901 
4902     if (evergreen_is_display_hung(rdev))
4903         reset_mask |= RADEON_RESET_DISPLAY;
4904 
4905     /* Skip MC reset as it's mostly likely not hung, just busy */
4906     if (reset_mask & RADEON_RESET_MC) {
4907         DRM_DEBUG("MC busy: 0x%08X, clearing.\n", reset_mask);
4908         reset_mask &= ~RADEON_RESET_MC;
4909     }
4910 
4911     return reset_mask;
4912 }
4913 
4914 /**
4915  * cik_gpu_soft_reset - soft reset GPU
4916  *
4917  * @rdev: radeon_device pointer
4918  * @reset_mask: mask of which blocks to reset
4919  *
4920  * Soft reset the blocks specified in @reset_mask.
4921  */
4922 static void cik_gpu_soft_reset(struct radeon_device *rdev, u32 reset_mask)
4923 {
4924     struct evergreen_mc_save save;
4925     u32 grbm_soft_reset = 0, srbm_soft_reset = 0;
4926     u32 tmp;
4927 
4928     if (reset_mask == 0)
4929         return;
4930 
4931     dev_info(rdev->dev, "GPU softreset: 0x%08X\n", reset_mask);
4932 
4933     cik_print_gpu_status_regs(rdev);
4934     dev_info(rdev->dev, "  VM_CONTEXT1_PROTECTION_FAULT_ADDR   0x%08X\n",
4935          RREG32(VM_CONTEXT1_PROTECTION_FAULT_ADDR));
4936     dev_info(rdev->dev, "  VM_CONTEXT1_PROTECTION_FAULT_STATUS 0x%08X\n",
4937          RREG32(VM_CONTEXT1_PROTECTION_FAULT_STATUS));
4938 
4939     /* disable CG/PG */
4940     cik_fini_pg(rdev);
4941     cik_fini_cg(rdev);
4942 
4943     /* stop the rlc */
4944     cik_rlc_stop(rdev);
4945 
4946     /* Disable GFX parsing/prefetching */
4947     WREG32(CP_ME_CNTL, CP_ME_HALT | CP_PFP_HALT | CP_CE_HALT);
4948 
4949     /* Disable MEC parsing/prefetching */
4950     WREG32(CP_MEC_CNTL, MEC_ME1_HALT | MEC_ME2_HALT);
4951 
4952     if (reset_mask & RADEON_RESET_DMA) {
4953         /* sdma0 */
4954         tmp = RREG32(SDMA0_ME_CNTL + SDMA0_REGISTER_OFFSET);
4955         tmp |= SDMA_HALT;
4956         WREG32(SDMA0_ME_CNTL + SDMA0_REGISTER_OFFSET, tmp);
4957     }
4958     if (reset_mask & RADEON_RESET_DMA1) {
4959         /* sdma1 */
4960         tmp = RREG32(SDMA0_ME_CNTL + SDMA1_REGISTER_OFFSET);
4961         tmp |= SDMA_HALT;
4962         WREG32(SDMA0_ME_CNTL + SDMA1_REGISTER_OFFSET, tmp);
4963     }
4964 
4965     evergreen_mc_stop(rdev, &save);
4966     if (evergreen_mc_wait_for_idle(rdev)) {
4967         dev_warn(rdev->dev, "Wait for MC idle timedout !\n");
4968     }
4969 
4970     if (reset_mask & (RADEON_RESET_GFX | RADEON_RESET_COMPUTE | RADEON_RESET_CP))
4971         grbm_soft_reset = SOFT_RESET_CP | SOFT_RESET_GFX;
4972 
4973     if (reset_mask & RADEON_RESET_CP) {
4974         grbm_soft_reset |= SOFT_RESET_CP;
4975 
4976         srbm_soft_reset |= SOFT_RESET_GRBM;
4977     }
4978 
4979     if (reset_mask & RADEON_RESET_DMA)
4980         srbm_soft_reset |= SOFT_RESET_SDMA;
4981 
4982     if (reset_mask & RADEON_RESET_DMA1)
4983         srbm_soft_reset |= SOFT_RESET_SDMA1;
4984 
4985     if (reset_mask & RADEON_RESET_DISPLAY)
4986         srbm_soft_reset |= SOFT_RESET_DC;
4987 
4988     if (reset_mask & RADEON_RESET_RLC)
4989         grbm_soft_reset |= SOFT_RESET_RLC;
4990 
4991     if (reset_mask & RADEON_RESET_SEM)
4992         srbm_soft_reset |= SOFT_RESET_SEM;
4993 
4994     if (reset_mask & RADEON_RESET_IH)
4995         srbm_soft_reset |= SOFT_RESET_IH;
4996 
4997     if (reset_mask & RADEON_RESET_GRBM)
4998         srbm_soft_reset |= SOFT_RESET_GRBM;
4999 
5000     if (reset_mask & RADEON_RESET_VMC)
5001         srbm_soft_reset |= SOFT_RESET_VMC;
5002 
5003     if (!(rdev->flags & RADEON_IS_IGP)) {
5004         if (reset_mask & RADEON_RESET_MC)
5005             srbm_soft_reset |= SOFT_RESET_MC;
5006     }
5007 
5008     if (grbm_soft_reset) {
5009         tmp = RREG32(GRBM_SOFT_RESET);
5010         tmp |= grbm_soft_reset;
5011         dev_info(rdev->dev, "GRBM_SOFT_RESET=0x%08X\n", tmp);
5012         WREG32(GRBM_SOFT_RESET, tmp);
5013         tmp = RREG32(GRBM_SOFT_RESET);
5014 
5015         udelay(50);
5016 
5017         tmp &= ~grbm_soft_reset;
5018         WREG32(GRBM_SOFT_RESET, tmp);
5019         tmp = RREG32(GRBM_SOFT_RESET);
5020     }
5021 
5022     if (srbm_soft_reset) {
5023         tmp = RREG32(SRBM_SOFT_RESET);
5024         tmp |= srbm_soft_reset;
5025         dev_info(rdev->dev, "SRBM_SOFT_RESET=0x%08X\n", tmp);
5026         WREG32(SRBM_SOFT_RESET, tmp);
5027         tmp = RREG32(SRBM_SOFT_RESET);
5028 
5029         udelay(50);
5030 
5031         tmp &= ~srbm_soft_reset;
5032         WREG32(SRBM_SOFT_RESET, tmp);
5033         tmp = RREG32(SRBM_SOFT_RESET);
5034     }
5035 
5036     /* Wait a little for things to settle down */
5037     udelay(50);
5038 
5039     evergreen_mc_resume(rdev, &save);
5040     udelay(50);
5041 
5042     cik_print_gpu_status_regs(rdev);
5043 }
5044 
5045 struct kv_reset_save_regs {
5046     u32 gmcon_reng_execute;
5047     u32 gmcon_misc;
5048     u32 gmcon_misc3;
5049 };
5050 
5051 static void kv_save_regs_for_reset(struct radeon_device *rdev,
5052                    struct kv_reset_save_regs *save)
5053 {
5054     save->gmcon_reng_execute = RREG32(GMCON_RENG_EXECUTE);
5055     save->gmcon_misc = RREG32(GMCON_MISC);
5056     save->gmcon_misc3 = RREG32(GMCON_MISC3);
5057 
5058     WREG32(GMCON_RENG_EXECUTE, save->gmcon_reng_execute & ~RENG_EXECUTE_ON_PWR_UP);
5059     WREG32(GMCON_MISC, save->gmcon_misc & ~(RENG_EXECUTE_ON_REG_UPDATE |
5060                         STCTRL_STUTTER_EN));
5061 }
5062 
5063 static void kv_restore_regs_for_reset(struct radeon_device *rdev,
5064                       struct kv_reset_save_regs *save)
5065 {
5066     int i;
5067 
5068     WREG32(GMCON_PGFSM_WRITE, 0);
5069     WREG32(GMCON_PGFSM_CONFIG, 0x200010ff);
5070 
5071     for (i = 0; i < 5; i++)
5072         WREG32(GMCON_PGFSM_WRITE, 0);
5073 
5074     WREG32(GMCON_PGFSM_WRITE, 0);
5075     WREG32(GMCON_PGFSM_CONFIG, 0x300010ff);
5076 
5077     for (i = 0; i < 5; i++)
5078         WREG32(GMCON_PGFSM_WRITE, 0);
5079 
5080     WREG32(GMCON_PGFSM_WRITE, 0x210000);
5081     WREG32(GMCON_PGFSM_CONFIG, 0xa00010ff);
5082 
5083     for (i = 0; i < 5; i++)
5084         WREG32(GMCON_PGFSM_WRITE, 0);
5085 
5086     WREG32(GMCON_PGFSM_WRITE, 0x21003);
5087     WREG32(GMCON_PGFSM_CONFIG, 0xb00010ff);
5088 
5089     for (i = 0; i < 5; i++)
5090         WREG32(GMCON_PGFSM_WRITE, 0);
5091 
5092     WREG32(GMCON_PGFSM_WRITE, 0x2b00);
5093     WREG32(GMCON_PGFSM_CONFIG, 0xc00010ff);
5094 
5095     for (i = 0; i < 5; i++)
5096         WREG32(GMCON_PGFSM_WRITE, 0);
5097 
5098     WREG32(GMCON_PGFSM_WRITE, 0);
5099     WREG32(GMCON_PGFSM_CONFIG, 0xd00010ff);
5100 
5101     for (i = 0; i < 5; i++)
5102         WREG32(GMCON_PGFSM_WRITE, 0);
5103 
5104     WREG32(GMCON_PGFSM_WRITE, 0x420000);
5105     WREG32(GMCON_PGFSM_CONFIG, 0x100010ff);
5106 
5107     for (i = 0; i < 5; i++)
5108         WREG32(GMCON_PGFSM_WRITE, 0);
5109 
5110     WREG32(GMCON_PGFSM_WRITE, 0x120202);
5111     WREG32(GMCON_PGFSM_CONFIG, 0x500010ff);
5112 
5113     for (i = 0; i < 5; i++)
5114         WREG32(GMCON_PGFSM_WRITE, 0);
5115 
5116     WREG32(GMCON_PGFSM_WRITE, 0x3e3e36);
5117     WREG32(GMCON_PGFSM_CONFIG, 0x600010ff);
5118 
5119     for (i = 0; i < 5; i++)
5120         WREG32(GMCON_PGFSM_WRITE, 0);
5121 
5122     WREG32(GMCON_PGFSM_WRITE, 0x373f3e);
5123     WREG32(GMCON_PGFSM_CONFIG, 0x700010ff);
5124 
5125     for (i = 0; i < 5; i++)
5126         WREG32(GMCON_PGFSM_WRITE, 0);
5127 
5128     WREG32(GMCON_PGFSM_WRITE, 0x3e1332);
5129     WREG32(GMCON_PGFSM_CONFIG, 0xe00010ff);
5130 
5131     WREG32(GMCON_MISC3, save->gmcon_misc3);
5132     WREG32(GMCON_MISC, save->gmcon_misc);
5133     WREG32(GMCON_RENG_EXECUTE, save->gmcon_reng_execute);
5134 }
5135 
5136 static void cik_gpu_pci_config_reset(struct radeon_device *rdev)
5137 {
5138     struct evergreen_mc_save save;
5139     struct kv_reset_save_regs kv_save = { 0 };
5140     u32 tmp, i;
5141 
5142     dev_info(rdev->dev, "GPU pci config reset\n");
5143 
5144     /* disable dpm? */
5145 
5146     /* disable cg/pg */
5147     cik_fini_pg(rdev);
5148     cik_fini_cg(rdev);
5149 
5150     /* Disable GFX parsing/prefetching */
5151     WREG32(CP_ME_CNTL, CP_ME_HALT | CP_PFP_HALT | CP_CE_HALT);
5152 
5153     /* Disable MEC parsing/prefetching */
5154     WREG32(CP_MEC_CNTL, MEC_ME1_HALT | MEC_ME2_HALT);
5155 
5156     /* sdma0 */
5157     tmp = RREG32(SDMA0_ME_CNTL + SDMA0_REGISTER_OFFSET);
5158     tmp |= SDMA_HALT;
5159     WREG32(SDMA0_ME_CNTL + SDMA0_REGISTER_OFFSET, tmp);
5160     /* sdma1 */
5161     tmp = RREG32(SDMA0_ME_CNTL + SDMA1_REGISTER_OFFSET);
5162     tmp |= SDMA_HALT;
5163     WREG32(SDMA0_ME_CNTL + SDMA1_REGISTER_OFFSET, tmp);
5164     /* XXX other engines? */
5165 
5166     /* halt the rlc, disable cp internal ints */
5167     cik_rlc_stop(rdev);
5168 
5169     udelay(50);
5170 
5171     /* disable mem access */
5172     evergreen_mc_stop(rdev, &save);
5173     if (evergreen_mc_wait_for_idle(rdev)) {
5174         dev_warn(rdev->dev, "Wait for MC idle timed out !\n");
5175     }
5176 
5177     if (rdev->flags & RADEON_IS_IGP)
5178         kv_save_regs_for_reset(rdev, &kv_save);
5179 
5180     /* disable BM */
5181     pci_clear_master(rdev->pdev);
5182     /* reset */
5183     radeon_pci_config_reset(rdev);
5184 
5185     udelay(100);
5186 
5187     /* wait for asic to come out of reset */
5188     for (i = 0; i < rdev->usec_timeout; i++) {
5189         if (RREG32(CONFIG_MEMSIZE) != 0xffffffff)
5190             break;
5191         udelay(1);
5192     }
5193 
5194     /* does asic init need to be run first??? */
5195     if (rdev->flags & RADEON_IS_IGP)
5196         kv_restore_regs_for_reset(rdev, &kv_save);
5197 }
5198 
5199 /**
5200  * cik_asic_reset - soft reset GPU
5201  *
5202  * @rdev: radeon_device pointer
5203  * @hard: force hard reset
5204  *
5205  * Look up which blocks are hung and attempt
5206  * to reset them.
5207  * Returns 0 for success.
5208  */
5209 int cik_asic_reset(struct radeon_device *rdev, bool hard)
5210 {
5211     u32 reset_mask;
5212 
5213     if (hard) {
5214         cik_gpu_pci_config_reset(rdev);
5215         return 0;
5216     }
5217 
5218     reset_mask = cik_gpu_check_soft_reset(rdev);
5219 
5220     if (reset_mask)
5221         r600_set_bios_scratch_engine_hung(rdev, true);
5222 
5223     /* try soft reset */
5224     cik_gpu_soft_reset(rdev, reset_mask);
5225 
5226     reset_mask = cik_gpu_check_soft_reset(rdev);
5227 
5228     /* try pci config reset */
5229     if (reset_mask && radeon_hard_reset)
5230         cik_gpu_pci_config_reset(rdev);
5231 
5232     reset_mask = cik_gpu_check_soft_reset(rdev);
5233 
5234     if (!reset_mask)
5235         r600_set_bios_scratch_engine_hung(rdev, false);
5236 
5237     return 0;
5238 }
5239 
5240 /**
5241  * cik_gfx_is_lockup - check if the 3D engine is locked up
5242  *
5243  * @rdev: radeon_device pointer
5244  * @ring: radeon_ring structure holding ring information
5245  *
5246  * Check if the 3D engine is locked up (CIK).
5247  * Returns true if the engine is locked, false if not.
5248  */
5249 bool cik_gfx_is_lockup(struct radeon_device *rdev, struct radeon_ring *ring)
5250 {
5251     u32 reset_mask = cik_gpu_check_soft_reset(rdev);
5252 
5253     if (!(reset_mask & (RADEON_RESET_GFX |
5254                 RADEON_RESET_COMPUTE |
5255                 RADEON_RESET_CP))) {
5256         radeon_ring_lockup_update(rdev, ring);
5257         return false;
5258     }
5259     return radeon_ring_test_lockup(rdev, ring);
5260 }
5261 
5262 /* MC */
5263 /**
5264  * cik_mc_program - program the GPU memory controller
5265  *
5266  * @rdev: radeon_device pointer
5267  *
5268  * Set the location of vram, gart, and AGP in the GPU's
5269  * physical address space (CIK).
5270  */
5271 static void cik_mc_program(struct radeon_device *rdev)
5272 {
5273     struct evergreen_mc_save save;
5274     u32 tmp;
5275     int i, j;
5276 
5277     /* Initialize HDP */
5278     for (i = 0, j = 0; i < 32; i++, j += 0x18) {
5279         WREG32((0x2c14 + j), 0x00000000);
5280         WREG32((0x2c18 + j), 0x00000000);
5281         WREG32((0x2c1c + j), 0x00000000);
5282         WREG32((0x2c20 + j), 0x00000000);
5283         WREG32((0x2c24 + j), 0x00000000);
5284     }
5285     WREG32(HDP_REG_COHERENCY_FLUSH_CNTL, 0);
5286 
5287     evergreen_mc_stop(rdev, &save);
5288     if (radeon_mc_wait_for_idle(rdev)) {
5289         dev_warn(rdev->dev, "Wait for MC idle timedout !\n");
5290     }
5291     /* Lockout access through VGA aperture*/
5292     WREG32(VGA_HDP_CONTROL, VGA_MEMORY_DISABLE);
5293     /* Update configuration */
5294     WREG32(MC_VM_SYSTEM_APERTURE_LOW_ADDR,
5295            rdev->mc.vram_start >> 12);
5296     WREG32(MC_VM_SYSTEM_APERTURE_HIGH_ADDR,
5297            rdev->mc.vram_end >> 12);
5298     WREG32(MC_VM_SYSTEM_APERTURE_DEFAULT_ADDR,
5299            rdev->vram_scratch.gpu_addr >> 12);
5300     tmp = ((rdev->mc.vram_end >> 24) & 0xFFFF) << 16;
5301     tmp |= ((rdev->mc.vram_start >> 24) & 0xFFFF);
5302     WREG32(MC_VM_FB_LOCATION, tmp);
5303     /* XXX double check these! */
5304     WREG32(HDP_NONSURFACE_BASE, (rdev->mc.vram_start >> 8));
5305     WREG32(HDP_NONSURFACE_INFO, (2 << 7) | (1 << 30));
5306     WREG32(HDP_NONSURFACE_SIZE, 0x3FFFFFFF);
5307     WREG32(MC_VM_AGP_BASE, 0);
5308     WREG32(MC_VM_AGP_TOP, 0x0FFFFFFF);
5309     WREG32(MC_VM_AGP_BOT, 0x0FFFFFFF);
5310     if (radeon_mc_wait_for_idle(rdev)) {
5311         dev_warn(rdev->dev, "Wait for MC idle timedout !\n");
5312     }
5313     evergreen_mc_resume(rdev, &save);
5314     /* we need to own VRAM, so turn off the VGA renderer here
5315      * to stop it overwriting our objects */
5316     rv515_vga_render_disable(rdev);
5317 }
5318 
5319 /**
5320  * cik_mc_init - initialize the memory controller driver params
5321  *
5322  * @rdev: radeon_device pointer
5323  *
5324  * Look up the amount of vram, vram width, and decide how to place
5325  * vram and gart within the GPU's physical address space (CIK).
5326  * Returns 0 for success.
5327  */
5328 static int cik_mc_init(struct radeon_device *rdev)
5329 {
5330     u32 tmp;
5331     int chansize, numchan;
5332 
5333     /* Get VRAM informations */
5334     rdev->mc.vram_is_ddr = true;
5335     tmp = RREG32(MC_ARB_RAMCFG);
5336     if (tmp & CHANSIZE_MASK) {
5337         chansize = 64;
5338     } else {
5339         chansize = 32;
5340     }
5341     tmp = RREG32(MC_SHARED_CHMAP);
5342     switch ((tmp & NOOFCHAN_MASK) >> NOOFCHAN_SHIFT) {
5343     case 0:
5344     default:
5345         numchan = 1;
5346         break;
5347     case 1:
5348         numchan = 2;
5349         break;
5350     case 2:
5351         numchan = 4;
5352         break;
5353     case 3:
5354         numchan = 8;
5355         break;
5356     case 4:
5357         numchan = 3;
5358         break;
5359     case 5:
5360         numchan = 6;
5361         break;
5362     case 6:
5363         numchan = 10;
5364         break;
5365     case 7:
5366         numchan = 12;
5367         break;
5368     case 8:
5369         numchan = 16;
5370         break;
5371     }
5372     rdev->mc.vram_width = numchan * chansize;
5373     /* Could aper size report 0 ? */
5374     rdev->mc.aper_base = pci_resource_start(rdev->pdev, 0);
5375     rdev->mc.aper_size = pci_resource_len(rdev->pdev, 0);
5376     /* size in MB on si */
5377     rdev->mc.mc_vram_size = RREG32(CONFIG_MEMSIZE) * 1024ULL * 1024ULL;
5378     rdev->mc.real_vram_size = RREG32(CONFIG_MEMSIZE) * 1024ULL * 1024ULL;
5379     rdev->mc.visible_vram_size = rdev->mc.aper_size;
5380     si_vram_gtt_location(rdev, &rdev->mc);
5381     radeon_update_bandwidth_info(rdev);
5382 
5383     return 0;
5384 }
5385 
5386 /*
5387  * GART
5388  * VMID 0 is the physical GPU addresses as used by the kernel.
5389  * VMIDs 1-15 are used for userspace clients and are handled
5390  * by the radeon vm/hsa code.
5391  */
5392 /**
5393  * cik_pcie_gart_tlb_flush - gart tlb flush callback
5394  *
5395  * @rdev: radeon_device pointer
5396  *
5397  * Flush the TLB for the VMID 0 page table (CIK).
5398  */
5399 void cik_pcie_gart_tlb_flush(struct radeon_device *rdev)
5400 {
5401     /* flush hdp cache */
5402     WREG32(HDP_MEM_COHERENCY_FLUSH_CNTL, 0);
5403 
5404     /* bits 0-15 are the VM contexts0-15 */
5405     WREG32(VM_INVALIDATE_REQUEST, 0x1);
5406 }
5407 
5408 /**
5409  * cik_pcie_gart_enable - gart enable
5410  *
5411  * @rdev: radeon_device pointer
5412  *
5413  * This sets up the TLBs, programs the page tables for VMID0,
5414  * sets up the hw for VMIDs 1-15 which are allocated on
5415  * demand, and sets up the global locations for the LDS, GDS,
5416  * and GPUVM for FSA64 clients (CIK).
5417  * Returns 0 for success, errors for failure.
5418  */
5419 static int cik_pcie_gart_enable(struct radeon_device *rdev)
5420 {
5421     int r, i;
5422 
5423     if (rdev->gart.robj == NULL) {
5424         dev_err(rdev->dev, "No VRAM object for PCIE GART.\n");
5425         return -EINVAL;
5426     }
5427     r = radeon_gart_table_vram_pin(rdev);
5428     if (r)
5429         return r;
5430     /* Setup TLB control */
5431     WREG32(MC_VM_MX_L1_TLB_CNTL,
5432            (0xA << 7) |
5433            ENABLE_L1_TLB |
5434            ENABLE_L1_FRAGMENT_PROCESSING |
5435            SYSTEM_ACCESS_MODE_NOT_IN_SYS |
5436            ENABLE_ADVANCED_DRIVER_MODEL |
5437            SYSTEM_APERTURE_UNMAPPED_ACCESS_PASS_THRU);
5438     /* Setup L2 cache */
5439     WREG32(VM_L2_CNTL, ENABLE_L2_CACHE |
5440            ENABLE_L2_FRAGMENT_PROCESSING |
5441            ENABLE_L2_PTE_CACHE_LRU_UPDATE_BY_WRITE |
5442            ENABLE_L2_PDE0_CACHE_LRU_UPDATE_BY_WRITE |
5443            EFFECTIVE_L2_QUEUE_SIZE(7) |
5444            CONTEXT1_IDENTITY_ACCESS_MODE(1));
5445     WREG32(VM_L2_CNTL2, INVALIDATE_ALL_L1_TLBS | INVALIDATE_L2_CACHE);
5446     WREG32(VM_L2_CNTL3, L2_CACHE_BIGK_ASSOCIATIVITY |
5447            BANK_SELECT(4) |
5448            L2_CACHE_BIGK_FRAGMENT_SIZE(4));
5449     /* setup context0 */
5450     WREG32(VM_CONTEXT0_PAGE_TABLE_START_ADDR, rdev->mc.gtt_start >> 12);
5451     WREG32(VM_CONTEXT0_PAGE_TABLE_END_ADDR, rdev->mc.gtt_end >> 12);
5452     WREG32(VM_CONTEXT0_PAGE_TABLE_BASE_ADDR, rdev->gart.table_addr >> 12);
5453     WREG32(VM_CONTEXT0_PROTECTION_FAULT_DEFAULT_ADDR,
5454             (u32)(rdev->dummy_page.addr >> 12));
5455     WREG32(VM_CONTEXT0_CNTL2, 0);
5456     WREG32(VM_CONTEXT0_CNTL, (ENABLE_CONTEXT | PAGE_TABLE_DEPTH(0) |
5457                   RANGE_PROTECTION_FAULT_ENABLE_DEFAULT));
5458 
5459     WREG32(0x15D4, 0);
5460     WREG32(0x15D8, 0);
5461     WREG32(0x15DC, 0);
5462 
5463     /* restore context1-15 */
5464     /* set vm size, must be a multiple of 4 */
5465     WREG32(VM_CONTEXT1_PAGE_TABLE_START_ADDR, 0);
5466     WREG32(VM_CONTEXT1_PAGE_TABLE_END_ADDR, rdev->vm_manager.max_pfn - 1);
5467     for (i = 1; i < 16; i++) {
5468         if (i < 8)
5469             WREG32(VM_CONTEXT0_PAGE_TABLE_BASE_ADDR + (i << 2),
5470                    rdev->vm_manager.saved_table_addr[i]);
5471         else
5472             WREG32(VM_CONTEXT8_PAGE_TABLE_BASE_ADDR + ((i - 8) << 2),
5473                    rdev->vm_manager.saved_table_addr[i]);
5474     }
5475 
5476     /* enable context1-15 */
5477     WREG32(VM_CONTEXT1_PROTECTION_FAULT_DEFAULT_ADDR,
5478            (u32)(rdev->dummy_page.addr >> 12));
5479     WREG32(VM_CONTEXT1_CNTL2, 4);
5480     WREG32(VM_CONTEXT1_CNTL, ENABLE_CONTEXT | PAGE_TABLE_DEPTH(1) |
5481                 PAGE_TABLE_BLOCK_SIZE(radeon_vm_block_size - 9) |
5482                 RANGE_PROTECTION_FAULT_ENABLE_INTERRUPT |
5483                 RANGE_PROTECTION_FAULT_ENABLE_DEFAULT |
5484                 DUMMY_PAGE_PROTECTION_FAULT_ENABLE_INTERRUPT |
5485                 DUMMY_PAGE_PROTECTION_FAULT_ENABLE_DEFAULT |
5486                 PDE0_PROTECTION_FAULT_ENABLE_INTERRUPT |
5487                 PDE0_PROTECTION_FAULT_ENABLE_DEFAULT |
5488                 VALID_PROTECTION_FAULT_ENABLE_INTERRUPT |
5489                 VALID_PROTECTION_FAULT_ENABLE_DEFAULT |
5490                 READ_PROTECTION_FAULT_ENABLE_INTERRUPT |
5491                 READ_PROTECTION_FAULT_ENABLE_DEFAULT |
5492                 WRITE_PROTECTION_FAULT_ENABLE_INTERRUPT |
5493                 WRITE_PROTECTION_FAULT_ENABLE_DEFAULT);
5494 
5495     if (rdev->family == CHIP_KAVERI) {
5496         u32 tmp = RREG32(CHUB_CONTROL);
5497         tmp &= ~BYPASS_VM;
5498         WREG32(CHUB_CONTROL, tmp);
5499     }
5500 
5501     /* XXX SH_MEM regs */
5502     /* where to put LDS, scratch, GPUVM in FSA64 space */
5503     mutex_lock(&rdev->srbm_mutex);
5504     for (i = 0; i < 16; i++) {
5505         cik_srbm_select(rdev, 0, 0, 0, i);
5506         /* CP and shaders */
5507         WREG32(SH_MEM_CONFIG, SH_MEM_CONFIG_GFX_DEFAULT);
5508         WREG32(SH_MEM_APE1_BASE, 1);
5509         WREG32(SH_MEM_APE1_LIMIT, 0);
5510         WREG32(SH_MEM_BASES, 0);
5511         /* SDMA GFX */
5512         WREG32(SDMA0_GFX_VIRTUAL_ADDR + SDMA0_REGISTER_OFFSET, 0);
5513         WREG32(SDMA0_GFX_APE1_CNTL + SDMA0_REGISTER_OFFSET, 0);
5514         WREG32(SDMA0_GFX_VIRTUAL_ADDR + SDMA1_REGISTER_OFFSET, 0);
5515         WREG32(SDMA0_GFX_APE1_CNTL + SDMA1_REGISTER_OFFSET, 0);
5516         /* XXX SDMA RLC - todo */
5517     }
5518     cik_srbm_select(rdev, 0, 0, 0, 0);
5519     mutex_unlock(&rdev->srbm_mutex);
5520 
5521     cik_pcie_gart_tlb_flush(rdev);
5522     DRM_INFO("PCIE GART of %uM enabled (table at 0x%016llX).\n",
5523          (unsigned)(rdev->mc.gtt_size >> 20),
5524          (unsigned long long)rdev->gart.table_addr);
5525     rdev->gart.ready = true;
5526     return 0;
5527 }
5528 
5529 /**
5530  * cik_pcie_gart_disable - gart disable
5531  *
5532  * @rdev: radeon_device pointer
5533  *
5534  * This disables all VM page table (CIK).
5535  */
5536 static void cik_pcie_gart_disable(struct radeon_device *rdev)
5537 {
5538     unsigned i;
5539 
5540     for (i = 1; i < 16; ++i) {
5541         uint32_t reg;
5542         if (i < 8)
5543             reg = VM_CONTEXT0_PAGE_TABLE_BASE_ADDR + (i << 2);
5544         else
5545             reg = VM_CONTEXT8_PAGE_TABLE_BASE_ADDR + ((i - 8) << 2);
5546         rdev->vm_manager.saved_table_addr[i] = RREG32(reg);
5547     }
5548 
5549     /* Disable all tables */
5550     WREG32(VM_CONTEXT0_CNTL, 0);
5551     WREG32(VM_CONTEXT1_CNTL, 0);
5552     /* Setup TLB control */
5553     WREG32(MC_VM_MX_L1_TLB_CNTL, SYSTEM_ACCESS_MODE_NOT_IN_SYS |
5554            SYSTEM_APERTURE_UNMAPPED_ACCESS_PASS_THRU);
5555     /* Setup L2 cache */
5556     WREG32(VM_L2_CNTL,
5557            ENABLE_L2_FRAGMENT_PROCESSING |
5558            ENABLE_L2_PTE_CACHE_LRU_UPDATE_BY_WRITE |
5559            ENABLE_L2_PDE0_CACHE_LRU_UPDATE_BY_WRITE |
5560            EFFECTIVE_L2_QUEUE_SIZE(7) |
5561            CONTEXT1_IDENTITY_ACCESS_MODE(1));
5562     WREG32(VM_L2_CNTL2, 0);
5563     WREG32(VM_L2_CNTL3, L2_CACHE_BIGK_ASSOCIATIVITY |
5564            L2_CACHE_BIGK_FRAGMENT_SIZE(6));
5565     radeon_gart_table_vram_unpin(rdev);
5566 }
5567 
5568 /**
5569  * cik_pcie_gart_fini - vm fini callback
5570  *
5571  * @rdev: radeon_device pointer
5572  *
5573  * Tears down the driver GART/VM setup (CIK).
5574  */
5575 static void cik_pcie_gart_fini(struct radeon_device *rdev)
5576 {
5577     cik_pcie_gart_disable(rdev);
5578     radeon_gart_table_vram_free(rdev);
5579     radeon_gart_fini(rdev);
5580 }
5581 
5582 /* vm parser */
5583 /**
5584  * cik_ib_parse - vm ib_parse callback
5585  *
5586  * @rdev: radeon_device pointer
5587  * @ib: indirect buffer pointer
5588  *
5589  * CIK uses hw IB checking so this is a nop (CIK).
5590  */
5591 int cik_ib_parse(struct radeon_device *rdev, struct radeon_ib *ib)
5592 {
5593     return 0;
5594 }
5595 
5596 /*
5597  * vm
5598  * VMID 0 is the physical GPU addresses as used by the kernel.
5599  * VMIDs 1-15 are used for userspace clients and are handled
5600  * by the radeon vm/hsa code.
5601  */
5602 /**
5603  * cik_vm_init - cik vm init callback
5604  *
5605  * @rdev: radeon_device pointer
5606  *
5607  * Inits cik specific vm parameters (number of VMs, base of vram for
5608  * VMIDs 1-15) (CIK).
5609  * Returns 0 for success.
5610  */
5611 int cik_vm_init(struct radeon_device *rdev)
5612 {
5613     /*
5614      * number of VMs
5615      * VMID 0 is reserved for System
5616      * radeon graphics/compute will use VMIDs 1-15
5617      */
5618     rdev->vm_manager.nvm = 16;
5619     /* base offset of vram pages */
5620     if (rdev->flags & RADEON_IS_IGP) {
5621         u64 tmp = RREG32(MC_VM_FB_OFFSET);
5622         tmp <<= 22;
5623         rdev->vm_manager.vram_base_offset = tmp;
5624     } else
5625         rdev->vm_manager.vram_base_offset = 0;
5626 
5627     return 0;
5628 }
5629 
5630 /**
5631  * cik_vm_fini - cik vm fini callback
5632  *
5633  * @rdev: radeon_device pointer
5634  *
5635  * Tear down any asic specific VM setup (CIK).
5636  */
5637 void cik_vm_fini(struct radeon_device *rdev)
5638 {
5639 }
5640 
5641 /**
5642  * cik_vm_decode_fault - print human readable fault info
5643  *
5644  * @rdev: radeon_device pointer
5645  * @status: VM_CONTEXT1_PROTECTION_FAULT_STATUS register value
5646  * @addr: VM_CONTEXT1_PROTECTION_FAULT_ADDR register value
5647  * @mc_client: VM_CONTEXT1_PROTECTION_FAULT_MCCLIENT register value
5648  *
5649  * Print human readable fault information (CIK).
5650  */
5651 static void cik_vm_decode_fault(struct radeon_device *rdev,
5652                 u32 status, u32 addr, u32 mc_client)
5653 {
5654     u32 mc_id;
5655     u32 vmid = (status & FAULT_VMID_MASK) >> FAULT_VMID_SHIFT;
5656     u32 protections = (status & PROTECTIONS_MASK) >> PROTECTIONS_SHIFT;
5657     char block[5] = { mc_client >> 24, (mc_client >> 16) & 0xff,
5658         (mc_client >> 8) & 0xff, mc_client & 0xff, 0 };
5659 
5660     if (rdev->family == CHIP_HAWAII)
5661         mc_id = (status & HAWAII_MEMORY_CLIENT_ID_MASK) >> MEMORY_CLIENT_ID_SHIFT;
5662     else
5663         mc_id = (status & MEMORY_CLIENT_ID_MASK) >> MEMORY_CLIENT_ID_SHIFT;
5664 
5665     printk("VM fault (0x%02x, vmid %d) at page %u, %s from '%s' (0x%08x) (%d)\n",
5666            protections, vmid, addr,
5667            (status & MEMORY_CLIENT_RW_MASK) ? "write" : "read",
5668            block, mc_client, mc_id);
5669 }
5670 
5671 /*
5672  * cik_vm_flush - cik vm flush using the CP
5673  *
5674  * Update the page table base and flush the VM TLB
5675  * using the CP (CIK).
5676  */
5677 void cik_vm_flush(struct radeon_device *rdev, struct radeon_ring *ring,
5678           unsigned vm_id, uint64_t pd_addr)
5679 {
5680     int usepfp = (ring->idx == RADEON_RING_TYPE_GFX_INDEX);
5681 
5682     radeon_ring_write(ring, PACKET3(PACKET3_WRITE_DATA, 3));
5683     radeon_ring_write(ring, (WRITE_DATA_ENGINE_SEL(usepfp) |
5684                  WRITE_DATA_DST_SEL(0)));
5685     if (vm_id < 8) {
5686         radeon_ring_write(ring,
5687                   (VM_CONTEXT0_PAGE_TABLE_BASE_ADDR + (vm_id << 2)) >> 2);
5688     } else {
5689         radeon_ring_write(ring,
5690                   (VM_CONTEXT8_PAGE_TABLE_BASE_ADDR + ((vm_id - 8) << 2)) >> 2);
5691     }
5692     radeon_ring_write(ring, 0);
5693     radeon_ring_write(ring, pd_addr >> 12);
5694 
5695     /* update SH_MEM_* regs */
5696     radeon_ring_write(ring, PACKET3(PACKET3_WRITE_DATA, 3));
5697     radeon_ring_write(ring, (WRITE_DATA_ENGINE_SEL(usepfp) |
5698                  WRITE_DATA_DST_SEL(0)));
5699     radeon_ring_write(ring, SRBM_GFX_CNTL >> 2);
5700     radeon_ring_write(ring, 0);
5701     radeon_ring_write(ring, VMID(vm_id));
5702 
5703     radeon_ring_write(ring, PACKET3(PACKET3_WRITE_DATA, 6));
5704     radeon_ring_write(ring, (WRITE_DATA_ENGINE_SEL(usepfp) |
5705                  WRITE_DATA_DST_SEL(0)));
5706     radeon_ring_write(ring, SH_MEM_BASES >> 2);
5707     radeon_ring_write(ring, 0);
5708 
5709     radeon_ring_write(ring, 0); /* SH_MEM_BASES */
5710     radeon_ring_write(ring, SH_MEM_CONFIG_GFX_DEFAULT); /* SH_MEM_CONFIG */
5711     radeon_ring_write(ring, 1); /* SH_MEM_APE1_BASE */
5712     radeon_ring_write(ring, 0); /* SH_MEM_APE1_LIMIT */
5713 
5714     radeon_ring_write(ring, PACKET3(PACKET3_WRITE_DATA, 3));
5715     radeon_ring_write(ring, (WRITE_DATA_ENGINE_SEL(usepfp) |
5716                  WRITE_DATA_DST_SEL(0)));
5717     radeon_ring_write(ring, SRBM_GFX_CNTL >> 2);
5718     radeon_ring_write(ring, 0);
5719     radeon_ring_write(ring, VMID(0));
5720 
5721     /* HDP flush */
5722     cik_hdp_flush_cp_ring_emit(rdev, ring->idx);
5723 
5724     /* bits 0-15 are the VM contexts0-15 */
5725     radeon_ring_write(ring, PACKET3(PACKET3_WRITE_DATA, 3));
5726     radeon_ring_write(ring, (WRITE_DATA_ENGINE_SEL(usepfp) |
5727                  WRITE_DATA_DST_SEL(0)));
5728     radeon_ring_write(ring, VM_INVALIDATE_REQUEST >> 2);
5729     radeon_ring_write(ring, 0);
5730     radeon_ring_write(ring, 1 << vm_id);
5731 
5732     /* wait for the invalidate to complete */
5733     radeon_ring_write(ring, PACKET3(PACKET3_WAIT_REG_MEM, 5));
5734     radeon_ring_write(ring, (WAIT_REG_MEM_OPERATION(0) | /* wait */
5735                  WAIT_REG_MEM_FUNCTION(0) |  /* always */
5736                  WAIT_REG_MEM_ENGINE(0))); /* me */
5737     radeon_ring_write(ring, VM_INVALIDATE_REQUEST >> 2);
5738     radeon_ring_write(ring, 0);
5739     radeon_ring_write(ring, 0); /* ref */
5740     radeon_ring_write(ring, 0); /* mask */
5741     radeon_ring_write(ring, 0x20); /* poll interval */
5742 
5743     /* compute doesn't have PFP */
5744     if (usepfp) {
5745         /* sync PFP to ME, otherwise we might get invalid PFP reads */
5746         radeon_ring_write(ring, PACKET3(PACKET3_PFP_SYNC_ME, 0));
5747         radeon_ring_write(ring, 0x0);
5748     }
5749 }
5750 
5751 /*
5752  * RLC
5753  * The RLC is a multi-purpose microengine that handles a
5754  * variety of functions, the most important of which is
5755  * the interrupt controller.
5756  */
5757 static void cik_enable_gui_idle_interrupt(struct radeon_device *rdev,
5758                       bool enable)
5759 {
5760     u32 tmp = RREG32(CP_INT_CNTL_RING0);
5761 
5762     if (enable)
5763         tmp |= (CNTX_BUSY_INT_ENABLE | CNTX_EMPTY_INT_ENABLE);
5764     else
5765         tmp &= ~(CNTX_BUSY_INT_ENABLE | CNTX_EMPTY_INT_ENABLE);
5766     WREG32(CP_INT_CNTL_RING0, tmp);
5767 }
5768 
5769 static void cik_enable_lbpw(struct radeon_device *rdev, bool enable)
5770 {
5771     u32 tmp;
5772 
5773     tmp = RREG32(RLC_LB_CNTL);
5774     if (enable)
5775         tmp |= LOAD_BALANCE_ENABLE;
5776     else
5777         tmp &= ~LOAD_BALANCE_ENABLE;
5778     WREG32(RLC_LB_CNTL, tmp);
5779 }
5780 
5781 static void cik_wait_for_rlc_serdes(struct radeon_device *rdev)
5782 {
5783     u32 i, j, k;
5784     u32 mask;
5785 
5786     for (i = 0; i < rdev->config.cik.max_shader_engines; i++) {
5787         for (j = 0; j < rdev->config.cik.max_sh_per_se; j++) {
5788             cik_select_se_sh(rdev, i, j);
5789             for (k = 0; k < rdev->usec_timeout; k++) {
5790                 if (RREG32(RLC_SERDES_CU_MASTER_BUSY) == 0)
5791                     break;
5792                 udelay(1);
5793             }
5794         }
5795     }
5796     cik_select_se_sh(rdev, 0xffffffff, 0xffffffff);
5797 
5798     mask = SE_MASTER_BUSY_MASK | GC_MASTER_BUSY | TC0_MASTER_BUSY | TC1_MASTER_BUSY;
5799     for (k = 0; k < rdev->usec_timeout; k++) {
5800         if ((RREG32(RLC_SERDES_NONCU_MASTER_BUSY) & mask) == 0)
5801             break;
5802         udelay(1);
5803     }
5804 }
5805 
5806 static void cik_update_rlc(struct radeon_device *rdev, u32 rlc)
5807 {
5808     u32 tmp;
5809 
5810     tmp = RREG32(RLC_CNTL);
5811     if (tmp != rlc)
5812         WREG32(RLC_CNTL, rlc);
5813 }
5814 
5815 static u32 cik_halt_rlc(struct radeon_device *rdev)
5816 {
5817     u32 data, orig;
5818 
5819     orig = data = RREG32(RLC_CNTL);
5820 
5821     if (data & RLC_ENABLE) {
5822         u32 i;
5823 
5824         data &= ~RLC_ENABLE;
5825         WREG32(RLC_CNTL, data);
5826 
5827         for (i = 0; i < rdev->usec_timeout; i++) {
5828             if ((RREG32(RLC_GPM_STAT) & RLC_GPM_BUSY) == 0)
5829                 break;
5830             udelay(1);
5831         }
5832 
5833         cik_wait_for_rlc_serdes(rdev);
5834     }
5835 
5836     return orig;
5837 }
5838 
5839 void cik_enter_rlc_safe_mode(struct radeon_device *rdev)
5840 {
5841     u32 tmp, i, mask;
5842 
5843     tmp = REQ | MESSAGE(MSG_ENTER_RLC_SAFE_MODE);
5844     WREG32(RLC_GPR_REG2, tmp);
5845 
5846     mask = GFX_POWER_STATUS | GFX_CLOCK_STATUS;
5847     for (i = 0; i < rdev->usec_timeout; i++) {
5848         if ((RREG32(RLC_GPM_STAT) & mask) == mask)
5849             break;
5850         udelay(1);
5851     }
5852 
5853     for (i = 0; i < rdev->usec_timeout; i++) {
5854         if ((RREG32(RLC_GPR_REG2) & REQ) == 0)
5855             break;
5856         udelay(1);
5857     }
5858 }
5859 
5860 void cik_exit_rlc_safe_mode(struct radeon_device *rdev)
5861 {
5862     u32 tmp;
5863 
5864     tmp = REQ | MESSAGE(MSG_EXIT_RLC_SAFE_MODE);
5865     WREG32(RLC_GPR_REG2, tmp);
5866 }
5867 
5868 /**
5869  * cik_rlc_stop - stop the RLC ME
5870  *
5871  * @rdev: radeon_device pointer
5872  *
5873  * Halt the RLC ME (MicroEngine) (CIK).
5874  */
5875 static void cik_rlc_stop(struct radeon_device *rdev)
5876 {
5877     WREG32(RLC_CNTL, 0);
5878 
5879     cik_enable_gui_idle_interrupt(rdev, false);
5880 
5881     cik_wait_for_rlc_serdes(rdev);
5882 }
5883 
5884 /**
5885  * cik_rlc_start - start the RLC ME
5886  *
5887  * @rdev: radeon_device pointer
5888  *
5889  * Unhalt the RLC ME (MicroEngine) (CIK).
5890  */
5891 static void cik_rlc_start(struct radeon_device *rdev)
5892 {
5893     WREG32(RLC_CNTL, RLC_ENABLE);
5894 
5895     cik_enable_gui_idle_interrupt(rdev, true);
5896 
5897     udelay(50);
5898 }
5899 
5900 /**
5901  * cik_rlc_resume - setup the RLC hw
5902  *
5903  * @rdev: radeon_device pointer
5904  *
5905  * Initialize the RLC registers, load the ucode,
5906  * and start the RLC (CIK).
5907  * Returns 0 for success, -EINVAL if the ucode is not available.
5908  */
5909 static int cik_rlc_resume(struct radeon_device *rdev)
5910 {
5911     u32 i, size, tmp;
5912 
5913     if (!rdev->rlc_fw)
5914         return -EINVAL;
5915 
5916     cik_rlc_stop(rdev);
5917 
5918     /* disable CG */
5919     tmp = RREG32(RLC_CGCG_CGLS_CTRL) & 0xfffffffc;
5920     WREG32(RLC_CGCG_CGLS_CTRL, tmp);
5921 
5922     si_rlc_reset(rdev);
5923 
5924     cik_init_pg(rdev);
5925 
5926     cik_init_cg(rdev);
5927 
5928     WREG32(RLC_LB_CNTR_INIT, 0);
5929     WREG32(RLC_LB_CNTR_MAX, 0x00008000);
5930 
5931     cik_select_se_sh(rdev, 0xffffffff, 0xffffffff);
5932     WREG32(RLC_LB_INIT_CU_MASK, 0xffffffff);
5933     WREG32(RLC_LB_PARAMS, 0x00600408);
5934     WREG32(RLC_LB_CNTL, 0x80000004);
5935 
5936     WREG32(RLC_MC_CNTL, 0);
5937     WREG32(RLC_UCODE_CNTL, 0);
5938 
5939     if (rdev->new_fw) {
5940         const struct rlc_firmware_header_v1_0 *hdr =
5941             (const struct rlc_firmware_header_v1_0 *)rdev->rlc_fw->data;
5942         const __le32 *fw_data = (const __le32 *)
5943             (rdev->rlc_fw->data + le32_to_cpu(hdr->header.ucode_array_offset_bytes));
5944 
5945         radeon_ucode_print_rlc_hdr(&hdr->header);
5946 
5947         size = le32_to_cpu(hdr->header.ucode_size_bytes) / 4;
5948         WREG32(RLC_GPM_UCODE_ADDR, 0);
5949         for (i = 0; i < size; i++)
5950             WREG32(RLC_GPM_UCODE_DATA, le32_to_cpup(fw_data++));
5951         WREG32(RLC_GPM_UCODE_ADDR, le32_to_cpu(hdr->header.ucode_version));
5952     } else {
5953         const __be32 *fw_data;
5954 
5955         switch (rdev->family) {
5956         case CHIP_BONAIRE:
5957         case CHIP_HAWAII:
5958         default:
5959             size = BONAIRE_RLC_UCODE_SIZE;
5960             break;
5961         case CHIP_KAVERI:
5962             size = KV_RLC_UCODE_SIZE;
5963             break;
5964         case CHIP_KABINI:
5965             size = KB_RLC_UCODE_SIZE;
5966             break;
5967         case CHIP_MULLINS:
5968             size = ML_RLC_UCODE_SIZE;
5969             break;
5970         }
5971 
5972         fw_data = (const __be32 *)rdev->rlc_fw->data;
5973         WREG32(RLC_GPM_UCODE_ADDR, 0);
5974         for (i = 0; i < size; i++)
5975             WREG32(RLC_GPM_UCODE_DATA, be32_to_cpup(fw_data++));
5976         WREG32(RLC_GPM_UCODE_ADDR, 0);
5977     }
5978 
5979     /* XXX - find out what chips support lbpw */
5980     cik_enable_lbpw(rdev, false);
5981 
5982     if (rdev->family == CHIP_BONAIRE)
5983         WREG32(RLC_DRIVER_DMA_STATUS, 0);
5984 
5985     cik_rlc_start(rdev);
5986 
5987     return 0;
5988 }
5989 
5990 static void cik_enable_cgcg(struct radeon_device *rdev, bool enable)
5991 {
5992     u32 data, orig, tmp, tmp2;
5993 
5994     orig = data = RREG32(RLC_CGCG_CGLS_CTRL);
5995 
5996     if (enable && (rdev->cg_flags & RADEON_CG_SUPPORT_GFX_CGCG)) {
5997         cik_enable_gui_idle_interrupt(rdev, true);
5998 
5999         tmp = cik_halt_rlc(rdev);
6000 
6001         cik_select_se_sh(rdev, 0xffffffff, 0xffffffff);
6002         WREG32(RLC_SERDES_WR_CU_MASTER_MASK, 0xffffffff);
6003         WREG32(RLC_SERDES_WR_NONCU_MASTER_MASK, 0xffffffff);
6004         tmp2 = BPM_ADDR_MASK | CGCG_OVERRIDE_0 | CGLS_ENABLE;
6005         WREG32(RLC_SERDES_WR_CTRL, tmp2);
6006 
6007         cik_update_rlc(rdev, tmp);
6008 
6009         data |= CGCG_EN | CGLS_EN;
6010     } else {
6011         cik_enable_gui_idle_interrupt(rdev, false);
6012 
6013         RREG32(CB_CGTT_SCLK_CTRL);
6014         RREG32(CB_CGTT_SCLK_CTRL);
6015         RREG32(CB_CGTT_SCLK_CTRL);
6016         RREG32(CB_CGTT_SCLK_CTRL);
6017 
6018         data &= ~(CGCG_EN | CGLS_EN);
6019     }
6020 
6021     if (orig != data)
6022         WREG32(RLC_CGCG_CGLS_CTRL, data);
6023 
6024 }
6025 
6026 static void cik_enable_mgcg(struct radeon_device *rdev, bool enable)
6027 {
6028     u32 data, orig, tmp = 0;
6029 
6030     if (enable && (rdev->cg_flags & RADEON_CG_SUPPORT_GFX_MGCG)) {
6031         if (rdev->cg_flags & RADEON_CG_SUPPORT_GFX_MGLS) {
6032             if (rdev->cg_flags & RADEON_CG_SUPPORT_GFX_CP_LS) {
6033                 orig = data = RREG32(CP_MEM_SLP_CNTL);
6034                 data |= CP_MEM_LS_EN;
6035                 if (orig != data)
6036                     WREG32(CP_MEM_SLP_CNTL, data);
6037             }
6038         }
6039 
6040         orig = data = RREG32(RLC_CGTT_MGCG_OVERRIDE);
6041         data |= 0x00000001;
6042         data &= 0xfffffffd;
6043         if (orig != data)
6044             WREG32(RLC_CGTT_MGCG_OVERRIDE, data);
6045 
6046         tmp = cik_halt_rlc(rdev);
6047 
6048         cik_select_se_sh(rdev, 0xffffffff, 0xffffffff);
6049         WREG32(RLC_SERDES_WR_CU_MASTER_MASK, 0xffffffff);
6050         WREG32(RLC_SERDES_WR_NONCU_MASTER_MASK, 0xffffffff);
6051         data = BPM_ADDR_MASK | MGCG_OVERRIDE_0;
6052         WREG32(RLC_SERDES_WR_CTRL, data);
6053 
6054         cik_update_rlc(rdev, tmp);
6055 
6056         if (rdev->cg_flags & RADEON_CG_SUPPORT_GFX_CGTS) {
6057             orig = data = RREG32(CGTS_SM_CTRL_REG);
6058             data &= ~SM_MODE_MASK;
6059             data |= SM_MODE(0x2);
6060             data |= SM_MODE_ENABLE;
6061             data &= ~CGTS_OVERRIDE;
6062             if ((rdev->cg_flags & RADEON_CG_SUPPORT_GFX_MGLS) &&
6063                 (rdev->cg_flags & RADEON_CG_SUPPORT_GFX_CGTS_LS))
6064                 data &= ~CGTS_LS_OVERRIDE;
6065             data &= ~ON_MONITOR_ADD_MASK;
6066             data |= ON_MONITOR_ADD_EN;
6067             data |= ON_MONITOR_ADD(0x96);
6068             if (orig != data)
6069                 WREG32(CGTS_SM_CTRL_REG, data);
6070         }
6071     } else {
6072         orig = data = RREG32(RLC_CGTT_MGCG_OVERRIDE);
6073         data |= 0x00000003;
6074         if (orig != data)
6075             WREG32(RLC_CGTT_MGCG_OVERRIDE, data);
6076 
6077         data = RREG32(RLC_MEM_SLP_CNTL);
6078         if (data & RLC_MEM_LS_EN) {
6079             data &= ~RLC_MEM_LS_EN;
6080             WREG32(RLC_MEM_SLP_CNTL, data);
6081         }
6082 
6083         data = RREG32(CP_MEM_SLP_CNTL);
6084         if (data & CP_MEM_LS_EN) {
6085             data &= ~CP_MEM_LS_EN;
6086             WREG32(CP_MEM_SLP_CNTL, data);
6087         }
6088 
6089         orig = data = RREG32(CGTS_SM_CTRL_REG);
6090         data |= CGTS_OVERRIDE | CGTS_LS_OVERRIDE;
6091         if (orig != data)
6092             WREG32(CGTS_SM_CTRL_REG, data);
6093 
6094         tmp = cik_halt_rlc(rdev);
6095 
6096         cik_select_se_sh(rdev, 0xffffffff, 0xffffffff);
6097         WREG32(RLC_SERDES_WR_CU_MASTER_MASK, 0xffffffff);
6098         WREG32(RLC_SERDES_WR_NONCU_MASTER_MASK, 0xffffffff);
6099         data = BPM_ADDR_MASK | MGCG_OVERRIDE_1;
6100         WREG32(RLC_SERDES_WR_CTRL, data);
6101 
6102         cik_update_rlc(rdev, tmp);
6103     }
6104 }
6105 
6106 static const u32 mc_cg_registers[] =
6107 {
6108     MC_HUB_MISC_HUB_CG,
6109     MC_HUB_MISC_SIP_CG,
6110     MC_HUB_MISC_VM_CG,
6111     MC_XPB_CLK_GAT,
6112     ATC_MISC_CG,
6113     MC_CITF_MISC_WR_CG,
6114     MC_CITF_MISC_RD_CG,
6115     MC_CITF_MISC_VM_CG,
6116     VM_L2_CG,
6117 };
6118 
6119 static void cik_enable_mc_ls(struct radeon_device *rdev,
6120                  bool enable)
6121 {
6122     int i;
6123     u32 orig, data;
6124 
6125     for (i = 0; i < ARRAY_SIZE(mc_cg_registers); i++) {
6126         orig = data = RREG32(mc_cg_registers[i]);
6127         if (enable && (rdev->cg_flags & RADEON_CG_SUPPORT_MC_LS))
6128             data |= MC_LS_ENABLE;
6129         else
6130             data &= ~MC_LS_ENABLE;
6131         if (data != orig)
6132             WREG32(mc_cg_registers[i], data);
6133     }
6134 }
6135 
6136 static void cik_enable_mc_mgcg(struct radeon_device *rdev,
6137                    bool enable)
6138 {
6139     int i;
6140     u32 orig, data;
6141 
6142     for (i = 0; i < ARRAY_SIZE(mc_cg_registers); i++) {
6143         orig = data = RREG32(mc_cg_registers[i]);
6144         if (enable && (rdev->cg_flags & RADEON_CG_SUPPORT_MC_MGCG))
6145             data |= MC_CG_ENABLE;
6146         else
6147             data &= ~MC_CG_ENABLE;
6148         if (data != orig)
6149             WREG32(mc_cg_registers[i], data);
6150     }
6151 }
6152 
6153 static void cik_enable_sdma_mgcg(struct radeon_device *rdev,
6154                  bool enable)
6155 {
6156     u32 orig, data;
6157 
6158     if (enable && (rdev->cg_flags & RADEON_CG_SUPPORT_SDMA_MGCG)) {
6159         WREG32(SDMA0_CLK_CTRL + SDMA0_REGISTER_OFFSET, 0x00000100);
6160         WREG32(SDMA0_CLK_CTRL + SDMA1_REGISTER_OFFSET, 0x00000100);
6161     } else {
6162         orig = data = RREG32(SDMA0_CLK_CTRL + SDMA0_REGISTER_OFFSET);
6163         data |= 0xff000000;
6164         if (data != orig)
6165             WREG32(SDMA0_CLK_CTRL + SDMA0_REGISTER_OFFSET, data);
6166 
6167         orig = data = RREG32(SDMA0_CLK_CTRL + SDMA1_REGISTER_OFFSET);
6168         data |= 0xff000000;
6169         if (data != orig)
6170             WREG32(SDMA0_CLK_CTRL + SDMA1_REGISTER_OFFSET, data);
6171     }
6172 }
6173 
6174 static void cik_enable_sdma_mgls(struct radeon_device *rdev,
6175                  bool enable)
6176 {
6177     u32 orig, data;
6178 
6179     if (enable && (rdev->cg_flags & RADEON_CG_SUPPORT_SDMA_LS)) {
6180         orig = data = RREG32(SDMA0_POWER_CNTL + SDMA0_REGISTER_OFFSET);
6181         data |= 0x100;
6182         if (orig != data)
6183             WREG32(SDMA0_POWER_CNTL + SDMA0_REGISTER_OFFSET, data);
6184 
6185         orig = data = RREG32(SDMA0_POWER_CNTL + SDMA1_REGISTER_OFFSET);
6186         data |= 0x100;
6187         if (orig != data)
6188             WREG32(SDMA0_POWER_CNTL + SDMA1_REGISTER_OFFSET, data);
6189     } else {
6190         orig = data = RREG32(SDMA0_POWER_CNTL + SDMA0_REGISTER_OFFSET);
6191         data &= ~0x100;
6192         if (orig != data)
6193             WREG32(SDMA0_POWER_CNTL + SDMA0_REGISTER_OFFSET, data);
6194 
6195         orig = data = RREG32(SDMA0_POWER_CNTL + SDMA1_REGISTER_OFFSET);
6196         data &= ~0x100;
6197         if (orig != data)
6198             WREG32(SDMA0_POWER_CNTL + SDMA1_REGISTER_OFFSET, data);
6199     }
6200 }
6201 
6202 static void cik_enable_uvd_mgcg(struct radeon_device *rdev,
6203                 bool enable)
6204 {
6205     u32 orig, data;
6206 
6207     if (enable && (rdev->cg_flags & RADEON_CG_SUPPORT_UVD_MGCG)) {
6208         data = RREG32_UVD_CTX(UVD_CGC_MEM_CTRL);
6209         data = 0xfff;
6210         WREG32_UVD_CTX(UVD_CGC_MEM_CTRL, data);
6211 
6212         orig = data = RREG32(UVD_CGC_CTRL);
6213         data |= DCM;
6214         if (orig != data)
6215             WREG32(UVD_CGC_CTRL, data);
6216     } else {
6217         data = RREG32_UVD_CTX(UVD_CGC_MEM_CTRL);
6218         data &= ~0xfff;
6219         WREG32_UVD_CTX(UVD_CGC_MEM_CTRL, data);
6220 
6221         orig = data = RREG32(UVD_CGC_CTRL);
6222         data &= ~DCM;
6223         if (orig != data)
6224             WREG32(UVD_CGC_CTRL, data);
6225     }
6226 }
6227 
6228 static void cik_enable_bif_mgls(struct radeon_device *rdev,
6229                    bool enable)
6230 {
6231     u32 orig, data;
6232 
6233     orig = data = RREG32_PCIE_PORT(PCIE_CNTL2);
6234 
6235     if (enable && (rdev->cg_flags & RADEON_CG_SUPPORT_BIF_LS))
6236         data |= SLV_MEM_LS_EN | MST_MEM_LS_EN |
6237             REPLAY_MEM_LS_EN | SLV_MEM_AGGRESSIVE_LS_EN;
6238     else
6239         data &= ~(SLV_MEM_LS_EN | MST_MEM_LS_EN |
6240               REPLAY_MEM_LS_EN | SLV_MEM_AGGRESSIVE_LS_EN);
6241 
6242     if (orig != data)
6243         WREG32_PCIE_PORT(PCIE_CNTL2, data);
6244 }
6245 
6246 static void cik_enable_hdp_mgcg(struct radeon_device *rdev,
6247                 bool enable)
6248 {
6249     u32 orig, data;
6250 
6251     orig = data = RREG32(HDP_HOST_PATH_CNTL);
6252 
6253     if (enable && (rdev->cg_flags & RADEON_CG_SUPPORT_HDP_MGCG))
6254         data &= ~CLOCK_GATING_DIS;
6255     else
6256         data |= CLOCK_GATING_DIS;
6257 
6258     if (orig != data)
6259         WREG32(HDP_HOST_PATH_CNTL, data);
6260 }
6261 
6262 static void cik_enable_hdp_ls(struct radeon_device *rdev,
6263                   bool enable)
6264 {
6265     u32 orig, data;
6266 
6267     orig = data = RREG32(HDP_MEM_POWER_LS);
6268 
6269     if (enable && (rdev->cg_flags & RADEON_CG_SUPPORT_HDP_LS))
6270         data |= HDP_LS_ENABLE;
6271     else
6272         data &= ~HDP_LS_ENABLE;
6273 
6274     if (orig != data)
6275         WREG32(HDP_MEM_POWER_LS, data);
6276 }
6277 
6278 void cik_update_cg(struct radeon_device *rdev,
6279            u32 block, bool enable)
6280 {
6281 
6282     if (block & RADEON_CG_BLOCK_GFX) {
6283         cik_enable_gui_idle_interrupt(rdev, false);
6284         /* order matters! */
6285         if (enable) {
6286             cik_enable_mgcg(rdev, true);
6287             cik_enable_cgcg(rdev, true);
6288         } else {
6289             cik_enable_cgcg(rdev, false);
6290             cik_enable_mgcg(rdev, false);
6291         }
6292         cik_enable_gui_idle_interrupt(rdev, true);
6293     }
6294 
6295     if (block & RADEON_CG_BLOCK_MC) {
6296         if (!(rdev->flags & RADEON_IS_IGP)) {
6297             cik_enable_mc_mgcg(rdev, enable);
6298             cik_enable_mc_ls(rdev, enable);
6299         }
6300     }
6301 
6302     if (block & RADEON_CG_BLOCK_SDMA) {
6303         cik_enable_sdma_mgcg(rdev, enable);
6304         cik_enable_sdma_mgls(rdev, enable);
6305     }
6306 
6307     if (block & RADEON_CG_BLOCK_BIF) {
6308         cik_enable_bif_mgls(rdev, enable);
6309     }
6310 
6311     if (block & RADEON_CG_BLOCK_UVD) {
6312         if (rdev->has_uvd)
6313             cik_enable_uvd_mgcg(rdev, enable);
6314     }
6315 
6316     if (block & RADEON_CG_BLOCK_HDP) {
6317         cik_enable_hdp_mgcg(rdev, enable);
6318         cik_enable_hdp_ls(rdev, enable);
6319     }
6320 
6321     if (block & RADEON_CG_BLOCK_VCE) {
6322         vce_v2_0_enable_mgcg(rdev, enable);
6323     }
6324 }
6325 
6326 static void cik_init_cg(struct radeon_device *rdev)
6327 {
6328 
6329     cik_update_cg(rdev, RADEON_CG_BLOCK_GFX, true);
6330 
6331     if (rdev->has_uvd)
6332         si_init_uvd_internal_cg(rdev);
6333 
6334     cik_update_cg(rdev, (RADEON_CG_BLOCK_MC |
6335                  RADEON_CG_BLOCK_SDMA |
6336                  RADEON_CG_BLOCK_BIF |
6337                  RADEON_CG_BLOCK_UVD |
6338                  RADEON_CG_BLOCK_HDP), true);
6339 }
6340 
6341 static void cik_fini_cg(struct radeon_device *rdev)
6342 {
6343     cik_update_cg(rdev, (RADEON_CG_BLOCK_MC |
6344                  RADEON_CG_BLOCK_SDMA |
6345                  RADEON_CG_BLOCK_BIF |
6346                  RADEON_CG_BLOCK_UVD |
6347                  RADEON_CG_BLOCK_HDP), false);
6348 
6349     cik_update_cg(rdev, RADEON_CG_BLOCK_GFX, false);
6350 }
6351 
6352 static void cik_enable_sck_slowdown_on_pu(struct radeon_device *rdev,
6353                       bool enable)
6354 {
6355     u32 data, orig;
6356 
6357     orig = data = RREG32(RLC_PG_CNTL);
6358     if (enable && (rdev->pg_flags & RADEON_PG_SUPPORT_RLC_SMU_HS))
6359         data |= SMU_CLK_SLOWDOWN_ON_PU_ENABLE;
6360     else
6361         data &= ~SMU_CLK_SLOWDOWN_ON_PU_ENABLE;
6362     if (orig != data)
6363         WREG32(RLC_PG_CNTL, data);
6364 }
6365 
6366 static void cik_enable_sck_slowdown_on_pd(struct radeon_device *rdev,
6367                       bool enable)
6368 {
6369     u32 data, orig;
6370 
6371     orig = data = RREG32(RLC_PG_CNTL);
6372     if (enable && (rdev->pg_flags & RADEON_PG_SUPPORT_RLC_SMU_HS))
6373         data |= SMU_CLK_SLOWDOWN_ON_PD_ENABLE;
6374     else
6375         data &= ~SMU_CLK_SLOWDOWN_ON_PD_ENABLE;
6376     if (orig != data)
6377         WREG32(RLC_PG_CNTL, data);
6378 }
6379 
6380 static void cik_enable_cp_pg(struct radeon_device *rdev, bool enable)
6381 {
6382     u32 data, orig;
6383 
6384     orig = data = RREG32(RLC_PG_CNTL);
6385     if (enable && (rdev->pg_flags & RADEON_PG_SUPPORT_CP))
6386         data &= ~DISABLE_CP_PG;
6387     else
6388         data |= DISABLE_CP_PG;
6389     if (orig != data)
6390         WREG32(RLC_PG_CNTL, data);
6391 }
6392 
6393 static void cik_enable_gds_pg(struct radeon_device *rdev, bool enable)
6394 {
6395     u32 data, orig;
6396 
6397     orig = data = RREG32(RLC_PG_CNTL);
6398     if (enable && (rdev->pg_flags & RADEON_PG_SUPPORT_GDS))
6399         data &= ~DISABLE_GDS_PG;
6400     else
6401         data |= DISABLE_GDS_PG;
6402     if (orig != data)
6403         WREG32(RLC_PG_CNTL, data);
6404 }
6405 
6406 #define CP_ME_TABLE_SIZE    96
6407 #define CP_ME_TABLE_OFFSET  2048
6408 #define CP_MEC_TABLE_OFFSET 4096
6409 
6410 void cik_init_cp_pg_table(struct radeon_device *rdev)
6411 {
6412     volatile u32 *dst_ptr;
6413     int me, i, max_me = 4;
6414     u32 bo_offset = 0;
6415     u32 table_offset, table_size;
6416 
6417     if (rdev->family == CHIP_KAVERI)
6418         max_me = 5;
6419 
6420     if (rdev->rlc.cp_table_ptr == NULL)
6421         return;
6422 
6423     /* write the cp table buffer */
6424     dst_ptr = rdev->rlc.cp_table_ptr;
6425     for (me = 0; me < max_me; me++) {
6426         if (rdev->new_fw) {
6427             const __le32 *fw_data;
6428             const struct gfx_firmware_header_v1_0 *hdr;
6429 
6430             if (me == 0) {
6431                 hdr = (const struct gfx_firmware_header_v1_0 *)rdev->ce_fw->data;
6432                 fw_data = (const __le32 *)
6433                     (rdev->ce_fw->data + le32_to_cpu(hdr->header.ucode_array_offset_bytes));
6434                 table_offset = le32_to_cpu(hdr->jt_offset);
6435                 table_size = le32_to_cpu(hdr->jt_size);
6436             } else if (me == 1) {
6437                 hdr = (const struct gfx_firmware_header_v1_0 *)rdev->pfp_fw->data;
6438                 fw_data = (const __le32 *)
6439                     (rdev->pfp_fw->data + le32_to_cpu(hdr->header.ucode_array_offset_bytes));
6440                 table_offset = le32_to_cpu(hdr->jt_offset);
6441                 table_size = le32_to_cpu(hdr->jt_size);
6442             } else if (me == 2) {
6443                 hdr = (const struct gfx_firmware_header_v1_0 *)rdev->me_fw->data;
6444                 fw_data = (const __le32 *)
6445                     (rdev->me_fw->data + le32_to_cpu(hdr->header.ucode_array_offset_bytes));
6446                 table_offset = le32_to_cpu(hdr->jt_offset);
6447                 table_size = le32_to_cpu(hdr->jt_size);
6448             } else if (me == 3) {
6449                 hdr = (const struct gfx_firmware_header_v1_0 *)rdev->mec_fw->data;
6450                 fw_data = (const __le32 *)
6451                     (rdev->mec_fw->data + le32_to_cpu(hdr->header.ucode_array_offset_bytes));
6452                 table_offset = le32_to_cpu(hdr->jt_offset);
6453                 table_size = le32_to_cpu(hdr->jt_size);
6454             } else {
6455                 hdr = (const struct gfx_firmware_header_v1_0 *)rdev->mec2_fw->data;
6456                 fw_data = (const __le32 *)
6457                     (rdev->mec2_fw->data + le32_to_cpu(hdr->header.ucode_array_offset_bytes));
6458                 table_offset = le32_to_cpu(hdr->jt_offset);
6459                 table_size = le32_to_cpu(hdr->jt_size);
6460             }
6461 
6462             for (i = 0; i < table_size; i ++) {
6463                 dst_ptr[bo_offset + i] =
6464                     cpu_to_le32(le32_to_cpu(fw_data[table_offset + i]));
6465             }
6466             bo_offset += table_size;
6467         } else {
6468             const __be32 *fw_data;
6469             table_size = CP_ME_TABLE_SIZE;
6470 
6471             if (me == 0) {
6472                 fw_data = (const __be32 *)rdev->ce_fw->data;
6473                 table_offset = CP_ME_TABLE_OFFSET;
6474             } else if (me == 1) {
6475                 fw_data = (const __be32 *)rdev->pfp_fw->data;
6476                 table_offset = CP_ME_TABLE_OFFSET;
6477             } else if (me == 2) {
6478                 fw_data = (const __be32 *)rdev->me_fw->data;
6479                 table_offset = CP_ME_TABLE_OFFSET;
6480             } else {
6481                 fw_data = (const __be32 *)rdev->mec_fw->data;
6482                 table_offset = CP_MEC_TABLE_OFFSET;
6483             }
6484 
6485             for (i = 0; i < table_size; i ++) {
6486                 dst_ptr[bo_offset + i] =
6487                     cpu_to_le32(be32_to_cpu(fw_data[table_offset + i]));
6488             }
6489             bo_offset += table_size;
6490         }
6491     }
6492 }
6493 
6494 static void cik_enable_gfx_cgpg(struct radeon_device *rdev,
6495                 bool enable)
6496 {
6497     u32 data, orig;
6498 
6499     if (enable && (rdev->pg_flags & RADEON_PG_SUPPORT_GFX_PG)) {
6500         orig = data = RREG32(RLC_PG_CNTL);
6501         data |= GFX_PG_ENABLE;
6502         if (orig != data)
6503             WREG32(RLC_PG_CNTL, data);
6504 
6505         orig = data = RREG32(RLC_AUTO_PG_CTRL);
6506         data |= AUTO_PG_EN;
6507         if (orig != data)
6508             WREG32(RLC_AUTO_PG_CTRL, data);
6509     } else {
6510         orig = data = RREG32(RLC_PG_CNTL);
6511         data &= ~GFX_PG_ENABLE;
6512         if (orig != data)
6513             WREG32(RLC_PG_CNTL, data);
6514 
6515         orig = data = RREG32(RLC_AUTO_PG_CTRL);
6516         data &= ~AUTO_PG_EN;
6517         if (orig != data)
6518             WREG32(RLC_AUTO_PG_CTRL, data);
6519 
6520         data = RREG32(DB_RENDER_CONTROL);
6521     }
6522 }
6523 
6524 static u32 cik_get_cu_active_bitmap(struct radeon_device *rdev, u32 se, u32 sh)
6525 {
6526     u32 mask = 0, tmp, tmp1;
6527     int i;
6528 
6529     cik_select_se_sh(rdev, se, sh);
6530     tmp = RREG32(CC_GC_SHADER_ARRAY_CONFIG);
6531     tmp1 = RREG32(GC_USER_SHADER_ARRAY_CONFIG);
6532     cik_select_se_sh(rdev, 0xffffffff, 0xffffffff);
6533 
6534     tmp &= 0xffff0000;
6535 
6536     tmp |= tmp1;
6537     tmp >>= 16;
6538 
6539     for (i = 0; i < rdev->config.cik.max_cu_per_sh; i ++) {
6540         mask <<= 1;
6541         mask |= 1;
6542     }
6543 
6544     return (~tmp) & mask;
6545 }
6546 
6547 static void cik_init_ao_cu_mask(struct radeon_device *rdev)
6548 {
6549     u32 i, j, k, active_cu_number = 0;
6550     u32 mask, counter, cu_bitmap;
6551     u32 tmp = 0;
6552 
6553     for (i = 0; i < rdev->config.cik.max_shader_engines; i++) {
6554         for (j = 0; j < rdev->config.cik.max_sh_per_se; j++) {
6555             mask = 1;
6556             cu_bitmap = 0;
6557             counter = 0;
6558             for (k = 0; k < rdev->config.cik.max_cu_per_sh; k ++) {
6559                 if (cik_get_cu_active_bitmap(rdev, i, j) & mask) {
6560                     if (counter < 2)
6561                         cu_bitmap |= mask;
6562                     counter ++;
6563                 }
6564                 mask <<= 1;
6565             }
6566 
6567             active_cu_number += counter;
6568             tmp |= (cu_bitmap << (i * 16 + j * 8));
6569         }
6570     }
6571 
6572     WREG32(RLC_PG_AO_CU_MASK, tmp);
6573 
6574     tmp = RREG32(RLC_MAX_PG_CU);
6575     tmp &= ~MAX_PU_CU_MASK;
6576     tmp |= MAX_PU_CU(active_cu_number);
6577     WREG32(RLC_MAX_PG_CU, tmp);
6578 }
6579 
6580 static void cik_enable_gfx_static_mgpg(struct radeon_device *rdev,
6581                        bool enable)
6582 {
6583     u32 data, orig;
6584 
6585     orig = data = RREG32(RLC_PG_CNTL);
6586     if (enable && (rdev->pg_flags & RADEON_PG_SUPPORT_GFX_SMG))
6587         data |= STATIC_PER_CU_PG_ENABLE;
6588     else
6589         data &= ~STATIC_PER_CU_PG_ENABLE;
6590     if (orig != data)
6591         WREG32(RLC_PG_CNTL, data);
6592 }
6593 
6594 static void cik_enable_gfx_dynamic_mgpg(struct radeon_device *rdev,
6595                     bool enable)
6596 {
6597     u32 data, orig;
6598 
6599     orig = data = RREG32(RLC_PG_CNTL);
6600     if (enable && (rdev->pg_flags & RADEON_PG_SUPPORT_GFX_DMG))
6601         data |= DYN_PER_CU_PG_ENABLE;
6602     else
6603         data &= ~DYN_PER_CU_PG_ENABLE;
6604     if (orig != data)
6605         WREG32(RLC_PG_CNTL, data);
6606 }
6607 
6608 #define RLC_SAVE_AND_RESTORE_STARTING_OFFSET 0x90
6609 #define RLC_CLEAR_STATE_DESCRIPTOR_OFFSET    0x3D
6610 
6611 static void cik_init_gfx_cgpg(struct radeon_device *rdev)
6612 {
6613     u32 data, orig;
6614     u32 i;
6615 
6616     if (rdev->rlc.cs_data) {
6617         WREG32(RLC_GPM_SCRATCH_ADDR, RLC_CLEAR_STATE_DESCRIPTOR_OFFSET);
6618         WREG32(RLC_GPM_SCRATCH_DATA, upper_32_bits(rdev->rlc.clear_state_gpu_addr));
6619         WREG32(RLC_GPM_SCRATCH_DATA, lower_32_bits(rdev->rlc.clear_state_gpu_addr));
6620         WREG32(RLC_GPM_SCRATCH_DATA, rdev->rlc.clear_state_size);
6621     } else {
6622         WREG32(RLC_GPM_SCRATCH_ADDR, RLC_CLEAR_STATE_DESCRIPTOR_OFFSET);
6623         for (i = 0; i < 3; i++)
6624             WREG32(RLC_GPM_SCRATCH_DATA, 0);
6625     }
6626     if (rdev->rlc.reg_list) {
6627         WREG32(RLC_GPM_SCRATCH_ADDR, RLC_SAVE_AND_RESTORE_STARTING_OFFSET);
6628         for (i = 0; i < rdev->rlc.reg_list_size; i++)
6629             WREG32(RLC_GPM_SCRATCH_DATA, rdev->rlc.reg_list[i]);
6630     }
6631 
6632     orig = data = RREG32(RLC_PG_CNTL);
6633     data |= GFX_PG_SRC;
6634     if (orig != data)
6635         WREG32(RLC_PG_CNTL, data);
6636 
6637     WREG32(RLC_SAVE_AND_RESTORE_BASE, rdev->rlc.save_restore_gpu_addr >> 8);
6638     WREG32(RLC_CP_TABLE_RESTORE, rdev->rlc.cp_table_gpu_addr >> 8);
6639 
6640     data = RREG32(CP_RB_WPTR_POLL_CNTL);
6641     data &= ~IDLE_POLL_COUNT_MASK;
6642     data |= IDLE_POLL_COUNT(0x60);
6643     WREG32(CP_RB_WPTR_POLL_CNTL, data);
6644 
6645     data = 0x10101010;
6646     WREG32(RLC_PG_DELAY, data);
6647 
6648     data = RREG32(RLC_PG_DELAY_2);
6649     data &= ~0xff;
6650     data |= 0x3;
6651     WREG32(RLC_PG_DELAY_2, data);
6652 
6653     data = RREG32(RLC_AUTO_PG_CTRL);
6654     data &= ~GRBM_REG_SGIT_MASK;
6655     data |= GRBM_REG_SGIT(0x700);
6656     WREG32(RLC_AUTO_PG_CTRL, data);
6657 
6658 }
6659 
6660 static void cik_update_gfx_pg(struct radeon_device *rdev, bool enable)
6661 {
6662     cik_enable_gfx_cgpg(rdev, enable);
6663     cik_enable_gfx_static_mgpg(rdev, enable);
6664     cik_enable_gfx_dynamic_mgpg(rdev, enable);
6665 }
6666 
6667 u32 cik_get_csb_size(struct radeon_device *rdev)
6668 {
6669     u32 count = 0;
6670     const struct cs_section_def *sect = NULL;
6671     const struct cs_extent_def *ext = NULL;
6672 
6673     if (rdev->rlc.cs_data == NULL)
6674         return 0;
6675 
6676     /* begin clear state */
6677     count += 2;
6678     /* context control state */
6679     count += 3;
6680 
6681     for (sect = rdev->rlc.cs_data; sect->section != NULL; ++sect) {
6682         for (ext = sect->section; ext->extent != NULL; ++ext) {
6683             if (sect->id == SECT_CONTEXT)
6684                 count += 2 + ext->reg_count;
6685             else
6686                 return 0;
6687         }
6688     }
6689     /* pa_sc_raster_config/pa_sc_raster_config1 */
6690     count += 4;
6691     /* end clear state */
6692     count += 2;
6693     /* clear state */
6694     count += 2;
6695 
6696     return count;
6697 }
6698 
6699 void cik_get_csb_buffer(struct radeon_device *rdev, volatile u32 *buffer)
6700 {
6701     u32 count = 0, i;
6702     const struct cs_section_def *sect = NULL;
6703     const struct cs_extent_def *ext = NULL;
6704 
6705     if (rdev->rlc.cs_data == NULL)
6706         return;
6707     if (buffer == NULL)
6708         return;
6709 
6710     buffer[count++] = cpu_to_le32(PACKET3(PACKET3_PREAMBLE_CNTL, 0));
6711     buffer[count++] = cpu_to_le32(PACKET3_PREAMBLE_BEGIN_CLEAR_STATE);
6712 
6713     buffer[count++] = cpu_to_le32(PACKET3(PACKET3_CONTEXT_CONTROL, 1));
6714     buffer[count++] = cpu_to_le32(0x80000000);
6715     buffer[count++] = cpu_to_le32(0x80000000);
6716 
6717     for (sect = rdev->rlc.cs_data; sect->section != NULL; ++sect) {
6718         for (ext = sect->section; ext->extent != NULL; ++ext) {
6719             if (sect->id == SECT_CONTEXT) {
6720                 buffer[count++] =
6721                     cpu_to_le32(PACKET3(PACKET3_SET_CONTEXT_REG, ext->reg_count));
6722                 buffer[count++] = cpu_to_le32(ext->reg_index - 0xa000);
6723                 for (i = 0; i < ext->reg_count; i++)
6724                     buffer[count++] = cpu_to_le32(ext->extent[i]);
6725             } else {
6726                 return;
6727             }
6728         }
6729     }
6730 
6731     buffer[count++] = cpu_to_le32(PACKET3(PACKET3_SET_CONTEXT_REG, 2));
6732     buffer[count++] = cpu_to_le32(PA_SC_RASTER_CONFIG - PACKET3_SET_CONTEXT_REG_START);
6733     switch (rdev->family) {
6734     case CHIP_BONAIRE:
6735         buffer[count++] = cpu_to_le32(0x16000012);
6736         buffer[count++] = cpu_to_le32(0x00000000);
6737         break;
6738     case CHIP_KAVERI:
6739         buffer[count++] = cpu_to_le32(0x00000000); /* XXX */
6740         buffer[count++] = cpu_to_le32(0x00000000);
6741         break;
6742     case CHIP_KABINI:
6743     case CHIP_MULLINS:
6744         buffer[count++] = cpu_to_le32(0x00000000); /* XXX */
6745         buffer[count++] = cpu_to_le32(0x00000000);
6746         break;
6747     case CHIP_HAWAII:
6748         buffer[count++] = cpu_to_le32(0x3a00161a);
6749         buffer[count++] = cpu_to_le32(0x0000002e);
6750         break;
6751     default:
6752         buffer[count++] = cpu_to_le32(0x00000000);
6753         buffer[count++] = cpu_to_le32(0x00000000);
6754         break;
6755     }
6756 
6757     buffer[count++] = cpu_to_le32(PACKET3(PACKET3_PREAMBLE_CNTL, 0));
6758     buffer[count++] = cpu_to_le32(PACKET3_PREAMBLE_END_CLEAR_STATE);
6759 
6760     buffer[count++] = cpu_to_le32(PACKET3(PACKET3_CLEAR_STATE, 0));
6761     buffer[count++] = cpu_to_le32(0);
6762 }
6763 
6764 static void cik_init_pg(struct radeon_device *rdev)
6765 {
6766     if (rdev->pg_flags) {
6767         cik_enable_sck_slowdown_on_pu(rdev, true);
6768         cik_enable_sck_slowdown_on_pd(rdev, true);
6769         if (rdev->pg_flags & RADEON_PG_SUPPORT_GFX_PG) {
6770             cik_init_gfx_cgpg(rdev);
6771             cik_enable_cp_pg(rdev, true);
6772             cik_enable_gds_pg(rdev, true);
6773         }
6774         cik_init_ao_cu_mask(rdev);
6775         cik_update_gfx_pg(rdev, true);
6776     }
6777 }
6778 
6779 static void cik_fini_pg(struct radeon_device *rdev)
6780 {
6781     if (rdev->pg_flags) {
6782         cik_update_gfx_pg(rdev, false);
6783         if (rdev->pg_flags & RADEON_PG_SUPPORT_GFX_PG) {
6784             cik_enable_cp_pg(rdev, false);
6785             cik_enable_gds_pg(rdev, false);
6786         }
6787     }
6788 }
6789 
6790 /*
6791  * Interrupts
6792  * Starting with r6xx, interrupts are handled via a ring buffer.
6793  * Ring buffers are areas of GPU accessible memory that the GPU
6794  * writes interrupt vectors into and the host reads vectors out of.
6795  * There is a rptr (read pointer) that determines where the
6796  * host is currently reading, and a wptr (write pointer)
6797  * which determines where the GPU has written.  When the
6798  * pointers are equal, the ring is idle.  When the GPU
6799  * writes vectors to the ring buffer, it increments the
6800  * wptr.  When there is an interrupt, the host then starts
6801  * fetching commands and processing them until the pointers are
6802  * equal again at which point it updates the rptr.
6803  */
6804 
6805 /**
6806  * cik_enable_interrupts - Enable the interrupt ring buffer
6807  *
6808  * @rdev: radeon_device pointer
6809  *
6810  * Enable the interrupt ring buffer (CIK).
6811  */
6812 static void cik_enable_interrupts(struct radeon_device *rdev)
6813 {
6814     u32 ih_cntl = RREG32(IH_CNTL);
6815     u32 ih_rb_cntl = RREG32(IH_RB_CNTL);
6816 
6817     ih_cntl |= ENABLE_INTR;
6818     ih_rb_cntl |= IH_RB_ENABLE;
6819     WREG32(IH_CNTL, ih_cntl);
6820     WREG32(IH_RB_CNTL, ih_rb_cntl);
6821     rdev->ih.enabled = true;
6822 }
6823 
6824 /**
6825  * cik_disable_interrupts - Disable the interrupt ring buffer
6826  *
6827  * @rdev: radeon_device pointer
6828  *
6829  * Disable the interrupt ring buffer (CIK).
6830  */
6831 static void cik_disable_interrupts(struct radeon_device *rdev)
6832 {
6833     u32 ih_rb_cntl = RREG32(IH_RB_CNTL);
6834     u32 ih_cntl = RREG32(IH_CNTL);
6835 
6836     ih_rb_cntl &= ~IH_RB_ENABLE;
6837     ih_cntl &= ~ENABLE_INTR;
6838     WREG32(IH_RB_CNTL, ih_rb_cntl);
6839     WREG32(IH_CNTL, ih_cntl);
6840     /* set rptr, wptr to 0 */
6841     WREG32(IH_RB_RPTR, 0);
6842     WREG32(IH_RB_WPTR, 0);
6843     rdev->ih.enabled = false;
6844     rdev->ih.rptr = 0;
6845 }
6846 
6847 /**
6848  * cik_disable_interrupt_state - Disable all interrupt sources
6849  *
6850  * @rdev: radeon_device pointer
6851  *
6852  * Clear all interrupt enable bits used by the driver (CIK).
6853  */
6854 static void cik_disable_interrupt_state(struct radeon_device *rdev)
6855 {
6856     u32 tmp;
6857 
6858     /* gfx ring */
6859     tmp = RREG32(CP_INT_CNTL_RING0) &
6860         (CNTX_BUSY_INT_ENABLE | CNTX_EMPTY_INT_ENABLE);
6861     WREG32(CP_INT_CNTL_RING0, tmp);
6862     /* sdma */
6863     tmp = RREG32(SDMA0_CNTL + SDMA0_REGISTER_OFFSET) & ~TRAP_ENABLE;
6864     WREG32(SDMA0_CNTL + SDMA0_REGISTER_OFFSET, tmp);
6865     tmp = RREG32(SDMA0_CNTL + SDMA1_REGISTER_OFFSET) & ~TRAP_ENABLE;
6866     WREG32(SDMA0_CNTL + SDMA1_REGISTER_OFFSET, tmp);
6867     /* compute queues */
6868     WREG32(CP_ME1_PIPE0_INT_CNTL, 0);
6869     WREG32(CP_ME1_PIPE1_INT_CNTL, 0);
6870     WREG32(CP_ME1_PIPE2_INT_CNTL, 0);
6871     WREG32(CP_ME1_PIPE3_INT_CNTL, 0);
6872     WREG32(CP_ME2_PIPE0_INT_CNTL, 0);
6873     WREG32(CP_ME2_PIPE1_INT_CNTL, 0);
6874     WREG32(CP_ME2_PIPE2_INT_CNTL, 0);
6875     WREG32(CP_ME2_PIPE3_INT_CNTL, 0);
6876     /* grbm */
6877     WREG32(GRBM_INT_CNTL, 0);
6878     /* SRBM */
6879     WREG32(SRBM_INT_CNTL, 0);
6880     /* vline/vblank, etc. */
6881     WREG32(LB_INTERRUPT_MASK + EVERGREEN_CRTC0_REGISTER_OFFSET, 0);
6882     WREG32(LB_INTERRUPT_MASK + EVERGREEN_CRTC1_REGISTER_OFFSET, 0);
6883     if (rdev->num_crtc >= 4) {
6884         WREG32(LB_INTERRUPT_MASK + EVERGREEN_CRTC2_REGISTER_OFFSET, 0);
6885         WREG32(LB_INTERRUPT_MASK + EVERGREEN_CRTC3_REGISTER_OFFSET, 0);
6886     }
6887     if (rdev->num_crtc >= 6) {
6888         WREG32(LB_INTERRUPT_MASK + EVERGREEN_CRTC4_REGISTER_OFFSET, 0);
6889         WREG32(LB_INTERRUPT_MASK + EVERGREEN_CRTC5_REGISTER_OFFSET, 0);
6890     }
6891     /* pflip */
6892     if (rdev->num_crtc >= 2) {
6893         WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC0_REGISTER_OFFSET, 0);
6894         WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC1_REGISTER_OFFSET, 0);
6895     }
6896     if (rdev->num_crtc >= 4) {
6897         WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC2_REGISTER_OFFSET, 0);
6898         WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC3_REGISTER_OFFSET, 0);
6899     }
6900     if (rdev->num_crtc >= 6) {
6901         WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC4_REGISTER_OFFSET, 0);
6902         WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC5_REGISTER_OFFSET, 0);
6903     }
6904 
6905     /* dac hotplug */
6906     WREG32(DAC_AUTODETECT_INT_CONTROL, 0);
6907 
6908     /* digital hotplug */
6909     tmp = RREG32(DC_HPD1_INT_CONTROL) & DC_HPDx_INT_POLARITY;
6910     WREG32(DC_HPD1_INT_CONTROL, tmp);
6911     tmp = RREG32(DC_HPD2_INT_CONTROL) & DC_HPDx_INT_POLARITY;
6912     WREG32(DC_HPD2_INT_CONTROL, tmp);
6913     tmp = RREG32(DC_HPD3_INT_CONTROL) & DC_HPDx_INT_POLARITY;
6914     WREG32(DC_HPD3_INT_CONTROL, tmp);
6915     tmp = RREG32(DC_HPD4_INT_CONTROL) & DC_HPDx_INT_POLARITY;
6916     WREG32(DC_HPD4_INT_CONTROL, tmp);
6917     tmp = RREG32(DC_HPD5_INT_CONTROL) & DC_HPDx_INT_POLARITY;
6918     WREG32(DC_HPD5_INT_CONTROL, tmp);
6919     tmp = RREG32(DC_HPD6_INT_CONTROL) & DC_HPDx_INT_POLARITY;
6920     WREG32(DC_HPD6_INT_CONTROL, tmp);
6921 
6922 }
6923 
6924 /**
6925  * cik_irq_init - init and enable the interrupt ring
6926  *
6927  * @rdev: radeon_device pointer
6928  *
6929  * Allocate a ring buffer for the interrupt controller,
6930  * enable the RLC, disable interrupts, enable the IH
6931  * ring buffer and enable it (CIK).
6932  * Called at device load and reume.
6933  * Returns 0 for success, errors for failure.
6934  */
6935 static int cik_irq_init(struct radeon_device *rdev)
6936 {
6937     int ret = 0;
6938     int rb_bufsz;
6939     u32 interrupt_cntl, ih_cntl, ih_rb_cntl;
6940 
6941     /* allocate ring */
6942     ret = r600_ih_ring_alloc(rdev);
6943     if (ret)
6944         return ret;
6945 
6946     /* disable irqs */
6947     cik_disable_interrupts(rdev);
6948 
6949     /* init rlc */
6950     ret = cik_rlc_resume(rdev);
6951     if (ret) {
6952         r600_ih_ring_fini(rdev);
6953         return ret;
6954     }
6955 
6956     /* setup interrupt control */
6957     /* set dummy read address to dummy page address */
6958     WREG32(INTERRUPT_CNTL2, rdev->dummy_page.addr >> 8);
6959     interrupt_cntl = RREG32(INTERRUPT_CNTL);
6960     /* IH_DUMMY_RD_OVERRIDE=0 - dummy read disabled with msi, enabled without msi
6961      * IH_DUMMY_RD_OVERRIDE=1 - dummy read controlled by IH_DUMMY_RD_EN
6962      */
6963     interrupt_cntl &= ~IH_DUMMY_RD_OVERRIDE;
6964     /* IH_REQ_NONSNOOP_EN=1 if ring is in non-cacheable memory, e.g., vram */
6965     interrupt_cntl &= ~IH_REQ_NONSNOOP_EN;
6966     WREG32(INTERRUPT_CNTL, interrupt_cntl);
6967 
6968     WREG32(IH_RB_BASE, rdev->ih.gpu_addr >> 8);
6969     rb_bufsz = order_base_2(rdev->ih.ring_size / 4);
6970 
6971     ih_rb_cntl = (IH_WPTR_OVERFLOW_ENABLE |
6972               IH_WPTR_OVERFLOW_CLEAR |
6973               (rb_bufsz << 1));
6974 
6975     if (rdev->wb.enabled)
6976         ih_rb_cntl |= IH_WPTR_WRITEBACK_ENABLE;
6977 
6978     /* set the writeback address whether it's enabled or not */
6979     WREG32(IH_RB_WPTR_ADDR_LO, (rdev->wb.gpu_addr + R600_WB_IH_WPTR_OFFSET) & 0xFFFFFFFC);
6980     WREG32(IH_RB_WPTR_ADDR_HI, upper_32_bits(rdev->wb.gpu_addr + R600_WB_IH_WPTR_OFFSET) & 0xFF);
6981 
6982     WREG32(IH_RB_CNTL, ih_rb_cntl);
6983 
6984     /* set rptr, wptr to 0 */
6985     WREG32(IH_RB_RPTR, 0);
6986     WREG32(IH_RB_WPTR, 0);
6987 
6988     /* Default settings for IH_CNTL (disabled at first) */
6989     ih_cntl = MC_WRREQ_CREDIT(0x10) | MC_WR_CLEAN_CNT(0x10) | MC_VMID(0);
6990     /* RPTR_REARM only works if msi's are enabled */
6991     if (rdev->msi_enabled)
6992         ih_cntl |= RPTR_REARM;
6993     WREG32(IH_CNTL, ih_cntl);
6994 
6995     /* force the active interrupt state to all disabled */
6996     cik_disable_interrupt_state(rdev);
6997 
6998     pci_set_master(rdev->pdev);
6999 
7000     /* enable irqs */
7001     cik_enable_interrupts(rdev);
7002 
7003     return ret;
7004 }
7005 
7006 /**
7007  * cik_irq_set - enable/disable interrupt sources
7008  *
7009  * @rdev: radeon_device pointer
7010  *
7011  * Enable interrupt sources on the GPU (vblanks, hpd,
7012  * etc.) (CIK).
7013  * Returns 0 for success, errors for failure.
7014  */
7015 int cik_irq_set(struct radeon_device *rdev)
7016 {
7017     u32 cp_int_cntl;
7018     u32 cp_m1p0, cp_m1p1, cp_m1p2, cp_m1p3;
7019     u32 cp_m2p0, cp_m2p1, cp_m2p2, cp_m2p3;
7020     u32 crtc1 = 0, crtc2 = 0, crtc3 = 0, crtc4 = 0, crtc5 = 0, crtc6 = 0;
7021     u32 hpd1, hpd2, hpd3, hpd4, hpd5, hpd6;
7022     u32 grbm_int_cntl = 0;
7023     u32 dma_cntl, dma_cntl1;
7024 
7025     if (!rdev->irq.installed) {
7026         WARN(1, "Can't enable IRQ/MSI because no handler is installed\n");
7027         return -EINVAL;
7028     }
7029     /* don't enable anything if the ih is disabled */
7030     if (!rdev->ih.enabled) {
7031         cik_disable_interrupts(rdev);
7032         /* force the active interrupt state to all disabled */
7033         cik_disable_interrupt_state(rdev);
7034         return 0;
7035     }
7036 
7037     cp_int_cntl = RREG32(CP_INT_CNTL_RING0) &
7038         (CNTX_BUSY_INT_ENABLE | CNTX_EMPTY_INT_ENABLE);
7039     cp_int_cntl |= PRIV_INSTR_INT_ENABLE | PRIV_REG_INT_ENABLE;
7040 
7041     hpd1 = RREG32(DC_HPD1_INT_CONTROL) & ~(DC_HPDx_INT_EN | DC_HPDx_RX_INT_EN);
7042     hpd2 = RREG32(DC_HPD2_INT_CONTROL) & ~(DC_HPDx_INT_EN | DC_HPDx_RX_INT_EN);
7043     hpd3 = RREG32(DC_HPD3_INT_CONTROL) & ~(DC_HPDx_INT_EN | DC_HPDx_RX_INT_EN);
7044     hpd4 = RREG32(DC_HPD4_INT_CONTROL) & ~(DC_HPDx_INT_EN | DC_HPDx_RX_INT_EN);
7045     hpd5 = RREG32(DC_HPD5_INT_CONTROL) & ~(DC_HPDx_INT_EN | DC_HPDx_RX_INT_EN);
7046     hpd6 = RREG32(DC_HPD6_INT_CONTROL) & ~(DC_HPDx_INT_EN | DC_HPDx_RX_INT_EN);
7047 
7048     dma_cntl = RREG32(SDMA0_CNTL + SDMA0_REGISTER_OFFSET) & ~TRAP_ENABLE;
7049     dma_cntl1 = RREG32(SDMA0_CNTL + SDMA1_REGISTER_OFFSET) & ~TRAP_ENABLE;
7050 
7051     cp_m1p0 = RREG32(CP_ME1_PIPE0_INT_CNTL) & ~TIME_STAMP_INT_ENABLE;
7052     cp_m1p1 = RREG32(CP_ME1_PIPE1_INT_CNTL) & ~TIME_STAMP_INT_ENABLE;
7053     cp_m1p2 = RREG32(CP_ME1_PIPE2_INT_CNTL) & ~TIME_STAMP_INT_ENABLE;
7054     cp_m1p3 = RREG32(CP_ME1_PIPE3_INT_CNTL) & ~TIME_STAMP_INT_ENABLE;
7055     cp_m2p0 = RREG32(CP_ME2_PIPE0_INT_CNTL) & ~TIME_STAMP_INT_ENABLE;
7056     cp_m2p1 = RREG32(CP_ME2_PIPE1_INT_CNTL) & ~TIME_STAMP_INT_ENABLE;
7057     cp_m2p2 = RREG32(CP_ME2_PIPE2_INT_CNTL) & ~TIME_STAMP_INT_ENABLE;
7058     cp_m2p3 = RREG32(CP_ME2_PIPE3_INT_CNTL) & ~TIME_STAMP_INT_ENABLE;
7059 
7060     /* enable CP interrupts on all rings */
7061     if (atomic_read(&rdev->irq.ring_int[RADEON_RING_TYPE_GFX_INDEX])) {
7062         DRM_DEBUG("cik_irq_set: sw int gfx\n");
7063         cp_int_cntl |= TIME_STAMP_INT_ENABLE;
7064     }
7065     if (atomic_read(&rdev->irq.ring_int[CAYMAN_RING_TYPE_CP1_INDEX])) {
7066         struct radeon_ring *ring = &rdev->ring[CAYMAN_RING_TYPE_CP1_INDEX];
7067         DRM_DEBUG("si_irq_set: sw int cp1\n");
7068         if (ring->me == 1) {
7069             switch (ring->pipe) {
7070             case 0:
7071                 cp_m1p0 |= TIME_STAMP_INT_ENABLE;
7072                 break;
7073             case 1:
7074                 cp_m1p1 |= TIME_STAMP_INT_ENABLE;
7075                 break;
7076             case 2:
7077                 cp_m1p2 |= TIME_STAMP_INT_ENABLE;
7078                 break;
7079             case 3:
7080                 cp_m1p2 |= TIME_STAMP_INT_ENABLE;
7081                 break;
7082             default:
7083                 DRM_DEBUG("si_irq_set: sw int cp1 invalid pipe %d\n", ring->pipe);
7084                 break;
7085             }
7086         } else if (ring->me == 2) {
7087             switch (ring->pipe) {
7088             case 0:
7089                 cp_m2p0 |= TIME_STAMP_INT_ENABLE;
7090                 break;
7091             case 1:
7092                 cp_m2p1 |= TIME_STAMP_INT_ENABLE;
7093                 break;
7094             case 2:
7095                 cp_m2p2 |= TIME_STAMP_INT_ENABLE;
7096                 break;
7097             case 3:
7098                 cp_m2p2 |= TIME_STAMP_INT_ENABLE;
7099                 break;
7100             default:
7101                 DRM_DEBUG("si_irq_set: sw int cp1 invalid pipe %d\n", ring->pipe);
7102                 break;
7103             }
7104         } else {
7105             DRM_DEBUG("si_irq_set: sw int cp1 invalid me %d\n", ring->me);
7106         }
7107     }
7108     if (atomic_read(&rdev->irq.ring_int[CAYMAN_RING_TYPE_CP2_INDEX])) {
7109         struct radeon_ring *ring = &rdev->ring[CAYMAN_RING_TYPE_CP2_INDEX];
7110         DRM_DEBUG("si_irq_set: sw int cp2\n");
7111         if (ring->me == 1) {
7112             switch (ring->pipe) {
7113             case 0:
7114                 cp_m1p0 |= TIME_STAMP_INT_ENABLE;
7115                 break;
7116             case 1:
7117                 cp_m1p1 |= TIME_STAMP_INT_ENABLE;
7118                 break;
7119             case 2:
7120                 cp_m1p2 |= TIME_STAMP_INT_ENABLE;
7121                 break;
7122             case 3:
7123                 cp_m1p2 |= TIME_STAMP_INT_ENABLE;
7124                 break;
7125             default:
7126                 DRM_DEBUG("si_irq_set: sw int cp2 invalid pipe %d\n", ring->pipe);
7127                 break;
7128             }
7129         } else if (ring->me == 2) {
7130             switch (ring->pipe) {
7131             case 0:
7132                 cp_m2p0 |= TIME_STAMP_INT_ENABLE;
7133                 break;
7134             case 1:
7135                 cp_m2p1 |= TIME_STAMP_INT_ENABLE;
7136                 break;
7137             case 2:
7138                 cp_m2p2 |= TIME_STAMP_INT_ENABLE;
7139                 break;
7140             case 3:
7141                 cp_m2p2 |= TIME_STAMP_INT_ENABLE;
7142                 break;
7143             default:
7144                 DRM_DEBUG("si_irq_set: sw int cp2 invalid pipe %d\n", ring->pipe);
7145                 break;
7146             }
7147         } else {
7148             DRM_DEBUG("si_irq_set: sw int cp2 invalid me %d\n", ring->me);
7149         }
7150     }
7151 
7152     if (atomic_read(&rdev->irq.ring_int[R600_RING_TYPE_DMA_INDEX])) {
7153         DRM_DEBUG("cik_irq_set: sw int dma\n");
7154         dma_cntl |= TRAP_ENABLE;
7155     }
7156 
7157     if (atomic_read(&rdev->irq.ring_int[CAYMAN_RING_TYPE_DMA1_INDEX])) {
7158         DRM_DEBUG("cik_irq_set: sw int dma1\n");
7159         dma_cntl1 |= TRAP_ENABLE;
7160     }
7161 
7162     if (rdev->irq.crtc_vblank_int[0] ||
7163         atomic_read(&rdev->irq.pflip[0])) {
7164         DRM_DEBUG("cik_irq_set: vblank 0\n");
7165         crtc1 |= VBLANK_INTERRUPT_MASK;
7166     }
7167     if (rdev->irq.crtc_vblank_int[1] ||
7168         atomic_read(&rdev->irq.pflip[1])) {
7169         DRM_DEBUG("cik_irq_set: vblank 1\n");
7170         crtc2 |= VBLANK_INTERRUPT_MASK;
7171     }
7172     if (rdev->irq.crtc_vblank_int[2] ||
7173         atomic_read(&rdev->irq.pflip[2])) {
7174         DRM_DEBUG("cik_irq_set: vblank 2\n");
7175         crtc3 |= VBLANK_INTERRUPT_MASK;
7176     }
7177     if (rdev->irq.crtc_vblank_int[3] ||
7178         atomic_read(&rdev->irq.pflip[3])) {
7179         DRM_DEBUG("cik_irq_set: vblank 3\n");
7180         crtc4 |= VBLANK_INTERRUPT_MASK;
7181     }
7182     if (rdev->irq.crtc_vblank_int[4] ||
7183         atomic_read(&rdev->irq.pflip[4])) {
7184         DRM_DEBUG("cik_irq_set: vblank 4\n");
7185         crtc5 |= VBLANK_INTERRUPT_MASK;
7186     }
7187     if (rdev->irq.crtc_vblank_int[5] ||
7188         atomic_read(&rdev->irq.pflip[5])) {
7189         DRM_DEBUG("cik_irq_set: vblank 5\n");
7190         crtc6 |= VBLANK_INTERRUPT_MASK;
7191     }
7192     if (rdev->irq.hpd[0]) {
7193         DRM_DEBUG("cik_irq_set: hpd 1\n");
7194         hpd1 |= DC_HPDx_INT_EN | DC_HPDx_RX_INT_EN;
7195     }
7196     if (rdev->irq.hpd[1]) {
7197         DRM_DEBUG("cik_irq_set: hpd 2\n");
7198         hpd2 |= DC_HPDx_INT_EN | DC_HPDx_RX_INT_EN;
7199     }
7200     if (rdev->irq.hpd[2]) {
7201         DRM_DEBUG("cik_irq_set: hpd 3\n");
7202         hpd3 |= DC_HPDx_INT_EN | DC_HPDx_RX_INT_EN;
7203     }
7204     if (rdev->irq.hpd[3]) {
7205         DRM_DEBUG("cik_irq_set: hpd 4\n");
7206         hpd4 |= DC_HPDx_INT_EN | DC_HPDx_RX_INT_EN;
7207     }
7208     if (rdev->irq.hpd[4]) {
7209         DRM_DEBUG("cik_irq_set: hpd 5\n");
7210         hpd5 |= DC_HPDx_INT_EN | DC_HPDx_RX_INT_EN;
7211     }
7212     if (rdev->irq.hpd[5]) {
7213         DRM_DEBUG("cik_irq_set: hpd 6\n");
7214         hpd6 |= DC_HPDx_INT_EN | DC_HPDx_RX_INT_EN;
7215     }
7216 
7217     WREG32(CP_INT_CNTL_RING0, cp_int_cntl);
7218 
7219     WREG32(SDMA0_CNTL + SDMA0_REGISTER_OFFSET, dma_cntl);
7220     WREG32(SDMA0_CNTL + SDMA1_REGISTER_OFFSET, dma_cntl1);
7221 
7222     WREG32(CP_ME1_PIPE0_INT_CNTL, cp_m1p0);
7223     WREG32(CP_ME1_PIPE1_INT_CNTL, cp_m1p1);
7224     WREG32(CP_ME1_PIPE2_INT_CNTL, cp_m1p2);
7225     WREG32(CP_ME1_PIPE3_INT_CNTL, cp_m1p3);
7226     WREG32(CP_ME2_PIPE0_INT_CNTL, cp_m2p0);
7227     WREG32(CP_ME2_PIPE1_INT_CNTL, cp_m2p1);
7228     WREG32(CP_ME2_PIPE2_INT_CNTL, cp_m2p2);
7229     WREG32(CP_ME2_PIPE3_INT_CNTL, cp_m2p3);
7230 
7231     WREG32(GRBM_INT_CNTL, grbm_int_cntl);
7232 
7233     WREG32(LB_INTERRUPT_MASK + EVERGREEN_CRTC0_REGISTER_OFFSET, crtc1);
7234     WREG32(LB_INTERRUPT_MASK + EVERGREEN_CRTC1_REGISTER_OFFSET, crtc2);
7235     if (rdev->num_crtc >= 4) {
7236         WREG32(LB_INTERRUPT_MASK + EVERGREEN_CRTC2_REGISTER_OFFSET, crtc3);
7237         WREG32(LB_INTERRUPT_MASK + EVERGREEN_CRTC3_REGISTER_OFFSET, crtc4);
7238     }
7239     if (rdev->num_crtc >= 6) {
7240         WREG32(LB_INTERRUPT_MASK + EVERGREEN_CRTC4_REGISTER_OFFSET, crtc5);
7241         WREG32(LB_INTERRUPT_MASK + EVERGREEN_CRTC5_REGISTER_OFFSET, crtc6);
7242     }
7243 
7244     if (rdev->num_crtc >= 2) {
7245         WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC0_REGISTER_OFFSET,
7246                GRPH_PFLIP_INT_MASK);
7247         WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC1_REGISTER_OFFSET,
7248                GRPH_PFLIP_INT_MASK);
7249     }
7250     if (rdev->num_crtc >= 4) {
7251         WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC2_REGISTER_OFFSET,
7252                GRPH_PFLIP_INT_MASK);
7253         WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC3_REGISTER_OFFSET,
7254                GRPH_PFLIP_INT_MASK);
7255     }
7256     if (rdev->num_crtc >= 6) {
7257         WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC4_REGISTER_OFFSET,
7258                GRPH_PFLIP_INT_MASK);
7259         WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC5_REGISTER_OFFSET,
7260                GRPH_PFLIP_INT_MASK);
7261     }
7262 
7263     WREG32(DC_HPD1_INT_CONTROL, hpd1);
7264     WREG32(DC_HPD2_INT_CONTROL, hpd2);
7265     WREG32(DC_HPD3_INT_CONTROL, hpd3);
7266     WREG32(DC_HPD4_INT_CONTROL, hpd4);
7267     WREG32(DC_HPD5_INT_CONTROL, hpd5);
7268     WREG32(DC_HPD6_INT_CONTROL, hpd6);
7269 
7270     /* posting read */
7271     RREG32(SRBM_STATUS);
7272 
7273     return 0;
7274 }
7275 
7276 /**
7277  * cik_irq_ack - ack interrupt sources
7278  *
7279  * @rdev: radeon_device pointer
7280  *
7281  * Ack interrupt sources on the GPU (vblanks, hpd,
7282  * etc.) (CIK).  Certain interrupts sources are sw
7283  * generated and do not require an explicit ack.
7284  */
7285 static inline void cik_irq_ack(struct radeon_device *rdev)
7286 {
7287     u32 tmp;
7288 
7289     rdev->irq.stat_regs.cik.disp_int = RREG32(DISP_INTERRUPT_STATUS);
7290     rdev->irq.stat_regs.cik.disp_int_cont = RREG32(DISP_INTERRUPT_STATUS_CONTINUE);
7291     rdev->irq.stat_regs.cik.disp_int_cont2 = RREG32(DISP_INTERRUPT_STATUS_CONTINUE2);
7292     rdev->irq.stat_regs.cik.disp_int_cont3 = RREG32(DISP_INTERRUPT_STATUS_CONTINUE3);
7293     rdev->irq.stat_regs.cik.disp_int_cont4 = RREG32(DISP_INTERRUPT_STATUS_CONTINUE4);
7294     rdev->irq.stat_regs.cik.disp_int_cont5 = RREG32(DISP_INTERRUPT_STATUS_CONTINUE5);
7295     rdev->irq.stat_regs.cik.disp_int_cont6 = RREG32(DISP_INTERRUPT_STATUS_CONTINUE6);
7296 
7297     rdev->irq.stat_regs.cik.d1grph_int = RREG32(GRPH_INT_STATUS +
7298         EVERGREEN_CRTC0_REGISTER_OFFSET);
7299     rdev->irq.stat_regs.cik.d2grph_int = RREG32(GRPH_INT_STATUS +
7300         EVERGREEN_CRTC1_REGISTER_OFFSET);
7301     if (rdev->num_crtc >= 4) {
7302         rdev->irq.stat_regs.cik.d3grph_int = RREG32(GRPH_INT_STATUS +
7303             EVERGREEN_CRTC2_REGISTER_OFFSET);
7304         rdev->irq.stat_regs.cik.d4grph_int = RREG32(GRPH_INT_STATUS +
7305             EVERGREEN_CRTC3_REGISTER_OFFSET);
7306     }
7307     if (rdev->num_crtc >= 6) {
7308         rdev->irq.stat_regs.cik.d5grph_int = RREG32(GRPH_INT_STATUS +
7309             EVERGREEN_CRTC4_REGISTER_OFFSET);
7310         rdev->irq.stat_regs.cik.d6grph_int = RREG32(GRPH_INT_STATUS +
7311             EVERGREEN_CRTC5_REGISTER_OFFSET);
7312     }
7313 
7314     if (rdev->irq.stat_regs.cik.d1grph_int & GRPH_PFLIP_INT_OCCURRED)
7315         WREG32(GRPH_INT_STATUS + EVERGREEN_CRTC0_REGISTER_OFFSET,
7316                GRPH_PFLIP_INT_CLEAR);
7317     if (rdev->irq.stat_regs.cik.d2grph_int & GRPH_PFLIP_INT_OCCURRED)
7318         WREG32(GRPH_INT_STATUS + EVERGREEN_CRTC1_REGISTER_OFFSET,
7319                GRPH_PFLIP_INT_CLEAR);
7320     if (rdev->irq.stat_regs.cik.disp_int & LB_D1_VBLANK_INTERRUPT)
7321         WREG32(LB_VBLANK_STATUS + EVERGREEN_CRTC0_REGISTER_OFFSET, VBLANK_ACK);
7322     if (rdev->irq.stat_regs.cik.disp_int & LB_D1_VLINE_INTERRUPT)
7323         WREG32(LB_VLINE_STATUS + EVERGREEN_CRTC0_REGISTER_OFFSET, VLINE_ACK);
7324     if (rdev->irq.stat_regs.cik.disp_int_cont & LB_D2_VBLANK_INTERRUPT)
7325         WREG32(LB_VBLANK_STATUS + EVERGREEN_CRTC1_REGISTER_OFFSET, VBLANK_ACK);
7326     if (rdev->irq.stat_regs.cik.disp_int_cont & LB_D2_VLINE_INTERRUPT)
7327         WREG32(LB_VLINE_STATUS + EVERGREEN_CRTC1_REGISTER_OFFSET, VLINE_ACK);
7328 
7329     if (rdev->num_crtc >= 4) {
7330         if (rdev->irq.stat_regs.cik.d3grph_int & GRPH_PFLIP_INT_OCCURRED)
7331             WREG32(GRPH_INT_STATUS + EVERGREEN_CRTC2_REGISTER_OFFSET,
7332                    GRPH_PFLIP_INT_CLEAR);
7333         if (rdev->irq.stat_regs.cik.d4grph_int & GRPH_PFLIP_INT_OCCURRED)
7334             WREG32(GRPH_INT_STATUS + EVERGREEN_CRTC3_REGISTER_OFFSET,
7335                    GRPH_PFLIP_INT_CLEAR);
7336         if (rdev->irq.stat_regs.cik.disp_int_cont2 & LB_D3_VBLANK_INTERRUPT)
7337             WREG32(LB_VBLANK_STATUS + EVERGREEN_CRTC2_REGISTER_OFFSET, VBLANK_ACK);
7338         if (rdev->irq.stat_regs.cik.disp_int_cont2 & LB_D3_VLINE_INTERRUPT)
7339             WREG32(LB_VLINE_STATUS + EVERGREEN_CRTC2_REGISTER_OFFSET, VLINE_ACK);
7340         if (rdev->irq.stat_regs.cik.disp_int_cont3 & LB_D4_VBLANK_INTERRUPT)
7341             WREG32(LB_VBLANK_STATUS + EVERGREEN_CRTC3_REGISTER_OFFSET, VBLANK_ACK);
7342         if (rdev->irq.stat_regs.cik.disp_int_cont3 & LB_D4_VLINE_INTERRUPT)
7343             WREG32(LB_VLINE_STATUS + EVERGREEN_CRTC3_REGISTER_OFFSET, VLINE_ACK);
7344     }
7345 
7346     if (rdev->num_crtc >= 6) {
7347         if (rdev->irq.stat_regs.cik.d5grph_int & GRPH_PFLIP_INT_OCCURRED)
7348             WREG32(GRPH_INT_STATUS + EVERGREEN_CRTC4_REGISTER_OFFSET,
7349                    GRPH_PFLIP_INT_CLEAR);
7350         if (rdev->irq.stat_regs.cik.d6grph_int & GRPH_PFLIP_INT_OCCURRED)
7351             WREG32(GRPH_INT_STATUS + EVERGREEN_CRTC5_REGISTER_OFFSET,
7352                    GRPH_PFLIP_INT_CLEAR);
7353         if (rdev->irq.stat_regs.cik.disp_int_cont4 & LB_D5_VBLANK_INTERRUPT)
7354             WREG32(LB_VBLANK_STATUS + EVERGREEN_CRTC4_REGISTER_OFFSET, VBLANK_ACK);
7355         if (rdev->irq.stat_regs.cik.disp_int_cont4 & LB_D5_VLINE_INTERRUPT)
7356             WREG32(LB_VLINE_STATUS + EVERGREEN_CRTC4_REGISTER_OFFSET, VLINE_ACK);
7357         if (rdev->irq.stat_regs.cik.disp_int_cont5 & LB_D6_VBLANK_INTERRUPT)
7358             WREG32(LB_VBLANK_STATUS + EVERGREEN_CRTC5_REGISTER_OFFSET, VBLANK_ACK);
7359         if (rdev->irq.stat_regs.cik.disp_int_cont5 & LB_D6_VLINE_INTERRUPT)
7360             WREG32(LB_VLINE_STATUS + EVERGREEN_CRTC5_REGISTER_OFFSET, VLINE_ACK);
7361     }
7362 
7363     if (rdev->irq.stat_regs.cik.disp_int & DC_HPD1_INTERRUPT) {
7364         tmp = RREG32(DC_HPD1_INT_CONTROL);
7365         tmp |= DC_HPDx_INT_ACK;
7366         WREG32(DC_HPD1_INT_CONTROL, tmp);
7367     }
7368     if (rdev->irq.stat_regs.cik.disp_int_cont & DC_HPD2_INTERRUPT) {
7369         tmp = RREG32(DC_HPD2_INT_CONTROL);
7370         tmp |= DC_HPDx_INT_ACK;
7371         WREG32(DC_HPD2_INT_CONTROL, tmp);
7372     }
7373     if (rdev->irq.stat_regs.cik.disp_int_cont2 & DC_HPD3_INTERRUPT) {
7374         tmp = RREG32(DC_HPD3_INT_CONTROL);
7375         tmp |= DC_HPDx_INT_ACK;
7376         WREG32(DC_HPD3_INT_CONTROL, tmp);
7377     }
7378     if (rdev->irq.stat_regs.cik.disp_int_cont3 & DC_HPD4_INTERRUPT) {
7379         tmp = RREG32(DC_HPD4_INT_CONTROL);
7380         tmp |= DC_HPDx_INT_ACK;
7381         WREG32(DC_HPD4_INT_CONTROL, tmp);
7382     }
7383     if (rdev->irq.stat_regs.cik.disp_int_cont4 & DC_HPD5_INTERRUPT) {
7384         tmp = RREG32(DC_HPD5_INT_CONTROL);
7385         tmp |= DC_HPDx_INT_ACK;
7386         WREG32(DC_HPD5_INT_CONTROL, tmp);
7387     }
7388     if (rdev->irq.stat_regs.cik.disp_int_cont5 & DC_HPD6_INTERRUPT) {
7389         tmp = RREG32(DC_HPD6_INT_CONTROL);
7390         tmp |= DC_HPDx_INT_ACK;
7391         WREG32(DC_HPD6_INT_CONTROL, tmp);
7392     }
7393     if (rdev->irq.stat_regs.cik.disp_int & DC_HPD1_RX_INTERRUPT) {
7394         tmp = RREG32(DC_HPD1_INT_CONTROL);
7395         tmp |= DC_HPDx_RX_INT_ACK;
7396         WREG32(DC_HPD1_INT_CONTROL, tmp);
7397     }
7398     if (rdev->irq.stat_regs.cik.disp_int_cont & DC_HPD2_RX_INTERRUPT) {
7399         tmp = RREG32(DC_HPD2_INT_CONTROL);
7400         tmp |= DC_HPDx_RX_INT_ACK;
7401         WREG32(DC_HPD2_INT_CONTROL, tmp);
7402     }
7403     if (rdev->irq.stat_regs.cik.disp_int_cont2 & DC_HPD3_RX_INTERRUPT) {
7404         tmp = RREG32(DC_HPD3_INT_CONTROL);
7405         tmp |= DC_HPDx_RX_INT_ACK;
7406         WREG32(DC_HPD3_INT_CONTROL, tmp);
7407     }
7408     if (rdev->irq.stat_regs.cik.disp_int_cont3 & DC_HPD4_RX_INTERRUPT) {
7409         tmp = RREG32(DC_HPD4_INT_CONTROL);
7410         tmp |= DC_HPDx_RX_INT_ACK;
7411         WREG32(DC_HPD4_INT_CONTROL, tmp);
7412     }
7413     if (rdev->irq.stat_regs.cik.disp_int_cont4 & DC_HPD5_RX_INTERRUPT) {
7414         tmp = RREG32(DC_HPD5_INT_CONTROL);
7415         tmp |= DC_HPDx_RX_INT_ACK;
7416         WREG32(DC_HPD5_INT_CONTROL, tmp);
7417     }
7418     if (rdev->irq.stat_regs.cik.disp_int_cont5 & DC_HPD6_RX_INTERRUPT) {
7419         tmp = RREG32(DC_HPD6_INT_CONTROL);
7420         tmp |= DC_HPDx_RX_INT_ACK;
7421         WREG32(DC_HPD6_INT_CONTROL, tmp);
7422     }
7423 }
7424 
7425 /**
7426  * cik_irq_disable - disable interrupts
7427  *
7428  * @rdev: radeon_device pointer
7429  *
7430  * Disable interrupts on the hw (CIK).
7431  */
7432 static void cik_irq_disable(struct radeon_device *rdev)
7433 {
7434     cik_disable_interrupts(rdev);
7435     /* Wait and acknowledge irq */
7436     mdelay(1);
7437     cik_irq_ack(rdev);
7438     cik_disable_interrupt_state(rdev);
7439 }
7440 
7441 /**
7442  * cik_irq_suspend - disable interrupts for suspend
7443  *
7444  * @rdev: radeon_device pointer
7445  *
7446  * Disable interrupts and stop the RLC (CIK).
7447  * Used for suspend.
7448  */
7449 static void cik_irq_suspend(struct radeon_device *rdev)
7450 {
7451     cik_irq_disable(rdev);
7452     cik_rlc_stop(rdev);
7453 }
7454 
7455 /**
7456  * cik_irq_fini - tear down interrupt support
7457  *
7458  * @rdev: radeon_device pointer
7459  *
7460  * Disable interrupts on the hw and free the IH ring
7461  * buffer (CIK).
7462  * Used for driver unload.
7463  */
7464 static void cik_irq_fini(struct radeon_device *rdev)
7465 {
7466     cik_irq_suspend(rdev);
7467     r600_ih_ring_fini(rdev);
7468 }
7469 
7470 /**
7471  * cik_get_ih_wptr - get the IH ring buffer wptr
7472  *
7473  * @rdev: radeon_device pointer
7474  *
7475  * Get the IH ring buffer wptr from either the register
7476  * or the writeback memory buffer (CIK).  Also check for
7477  * ring buffer overflow and deal with it.
7478  * Used by cik_irq_process().
7479  * Returns the value of the wptr.
7480  */
7481 static inline u32 cik_get_ih_wptr(struct radeon_device *rdev)
7482 {
7483     u32 wptr, tmp;
7484 
7485     if (rdev->wb.enabled)
7486         wptr = le32_to_cpu(rdev->wb.wb[R600_WB_IH_WPTR_OFFSET/4]);
7487     else
7488         wptr = RREG32(IH_RB_WPTR);
7489 
7490     if (wptr & RB_OVERFLOW) {
7491         wptr &= ~RB_OVERFLOW;
7492         /* When a ring buffer overflow happen start parsing interrupt
7493          * from the last not overwritten vector (wptr + 16). Hopefully
7494          * this should allow us to catchup.
7495          */
7496         dev_warn(rdev->dev, "IH ring buffer overflow (0x%08X, 0x%08X, 0x%08X)\n",
7497              wptr, rdev->ih.rptr, (wptr + 16) & rdev->ih.ptr_mask);
7498         rdev->ih.rptr = (wptr + 16) & rdev->ih.ptr_mask;
7499         tmp = RREG32(IH_RB_CNTL);
7500         tmp |= IH_WPTR_OVERFLOW_CLEAR;
7501         WREG32(IH_RB_CNTL, tmp);
7502     }
7503     return (wptr & rdev->ih.ptr_mask);
7504 }
7505 
7506 /*        CIK IV Ring
7507  * Each IV ring entry is 128 bits:
7508  * [7:0]    - interrupt source id
7509  * [31:8]   - reserved
7510  * [59:32]  - interrupt source data
7511  * [63:60]  - reserved
7512  * [71:64]  - RINGID
7513  *            CP:
7514  *            ME_ID [1:0], PIPE_ID[1:0], QUEUE_ID[2:0]
7515  *            QUEUE_ID - for compute, which of the 8 queues owned by the dispatcher
7516  *                     - for gfx, hw shader state (0=PS...5=LS, 6=CS)
7517  *            ME_ID - 0 = gfx, 1 = first 4 CS pipes, 2 = second 4 CS pipes
7518  *            PIPE_ID - ME0 0=3D
7519  *                    - ME1&2 compute dispatcher (4 pipes each)
7520  *            SDMA:
7521  *            INSTANCE_ID [1:0], QUEUE_ID[1:0]
7522  *            INSTANCE_ID - 0 = sdma0, 1 = sdma1
7523  *            QUEUE_ID - 0 = gfx, 1 = rlc0, 2 = rlc1
7524  * [79:72]  - VMID
7525  * [95:80]  - PASID
7526  * [127:96] - reserved
7527  */
7528 /**
7529  * cik_irq_process - interrupt handler
7530  *
7531  * @rdev: radeon_device pointer
7532  *
7533  * Interrupt hander (CIK).  Walk the IH ring,
7534  * ack interrupts and schedule work to handle
7535  * interrupt events.
7536  * Returns irq process return code.
7537  */
7538 int cik_irq_process(struct radeon_device *rdev)
7539 {
7540     struct radeon_ring *cp1_ring = &rdev->ring[CAYMAN_RING_TYPE_CP1_INDEX];
7541     struct radeon_ring *cp2_ring = &rdev->ring[CAYMAN_RING_TYPE_CP2_INDEX];
7542     u32 wptr;
7543     u32 rptr;
7544     u32 src_id, src_data, ring_id;
7545     u8 me_id, pipe_id, queue_id;
7546     u32 ring_index;
7547     bool queue_hotplug = false;
7548     bool queue_dp = false;
7549     bool queue_reset = false;
7550     u32 addr, status, mc_client;
7551     bool queue_thermal = false;
7552 
7553     if (!rdev->ih.enabled || rdev->shutdown)
7554         return IRQ_NONE;
7555 
7556     wptr = cik_get_ih_wptr(rdev);
7557 
7558 restart_ih:
7559     /* is somebody else already processing irqs? */
7560     if (atomic_xchg(&rdev->ih.lock, 1))
7561         return IRQ_NONE;
7562 
7563     rptr = rdev->ih.rptr;
7564     DRM_DEBUG("cik_irq_process start: rptr %d, wptr %d\n", rptr, wptr);
7565 
7566     /* Order reading of wptr vs. reading of IH ring data */
7567     rmb();
7568 
7569     /* display interrupts */
7570     cik_irq_ack(rdev);
7571 
7572     while (rptr != wptr) {
7573         /* wptr/rptr are in bytes! */
7574         ring_index = rptr / 4;
7575 
7576         src_id =  le32_to_cpu(rdev->ih.ring[ring_index]) & 0xff;
7577         src_data = le32_to_cpu(rdev->ih.ring[ring_index + 1]) & 0xfffffff;
7578         ring_id = le32_to_cpu(rdev->ih.ring[ring_index + 2]) & 0xff;
7579 
7580         switch (src_id) {
7581         case 1: /* D1 vblank/vline */
7582             switch (src_data) {
7583             case 0: /* D1 vblank */
7584                 if (!(rdev->irq.stat_regs.cik.disp_int & LB_D1_VBLANK_INTERRUPT))
7585                     DRM_DEBUG("IH: IH event w/o asserted irq bit?\n");
7586 
7587                 if (rdev->irq.crtc_vblank_int[0]) {
7588                     drm_handle_vblank(rdev->ddev, 0);
7589                     rdev->pm.vblank_sync = true;
7590                     wake_up(&rdev->irq.vblank_queue);
7591                 }
7592                 if (atomic_read(&rdev->irq.pflip[0]))
7593                     radeon_crtc_handle_vblank(rdev, 0);
7594                 rdev->irq.stat_regs.cik.disp_int &= ~LB_D1_VBLANK_INTERRUPT;
7595                 DRM_DEBUG("IH: D1 vblank\n");
7596 
7597                 break;
7598             case 1: /* D1 vline */
7599                 if (!(rdev->irq.stat_regs.cik.disp_int & LB_D1_VLINE_INTERRUPT))
7600                     DRM_DEBUG("IH: IH event w/o asserted irq bit?\n");
7601 
7602                 rdev->irq.stat_regs.cik.disp_int &= ~LB_D1_VLINE_INTERRUPT;
7603                 DRM_DEBUG("IH: D1 vline\n");
7604 
7605                 break;
7606             default:
7607                 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
7608                 break;
7609             }
7610             break;
7611         case 2: /* D2 vblank/vline */
7612             switch (src_data) {
7613             case 0: /* D2 vblank */
7614                 if (!(rdev->irq.stat_regs.cik.disp_int_cont & LB_D2_VBLANK_INTERRUPT))
7615                     DRM_DEBUG("IH: IH event w/o asserted irq bit?\n");
7616 
7617                 if (rdev->irq.crtc_vblank_int[1]) {
7618                     drm_handle_vblank(rdev->ddev, 1);
7619                     rdev->pm.vblank_sync = true;
7620                     wake_up(&rdev->irq.vblank_queue);
7621                 }
7622                 if (atomic_read(&rdev->irq.pflip[1]))
7623                     radeon_crtc_handle_vblank(rdev, 1);
7624                 rdev->irq.stat_regs.cik.disp_int_cont &= ~LB_D2_VBLANK_INTERRUPT;
7625                 DRM_DEBUG("IH: D2 vblank\n");
7626 
7627                 break;
7628             case 1: /* D2 vline */
7629                 if (!(rdev->irq.stat_regs.cik.disp_int_cont & LB_D2_VLINE_INTERRUPT))
7630                     DRM_DEBUG("IH: IH event w/o asserted irq bit?\n");
7631 
7632                 rdev->irq.stat_regs.cik.disp_int_cont &= ~LB_D2_VLINE_INTERRUPT;
7633                 DRM_DEBUG("IH: D2 vline\n");
7634 
7635                 break;
7636             default:
7637                 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
7638                 break;
7639             }
7640             break;
7641         case 3: /* D3 vblank/vline */
7642             switch (src_data) {
7643             case 0: /* D3 vblank */
7644                 if (!(rdev->irq.stat_regs.cik.disp_int_cont2 & LB_D3_VBLANK_INTERRUPT))
7645                     DRM_DEBUG("IH: IH event w/o asserted irq bit?\n");
7646 
7647                 if (rdev->irq.crtc_vblank_int[2]) {
7648                     drm_handle_vblank(rdev->ddev, 2);
7649                     rdev->pm.vblank_sync = true;
7650                     wake_up(&rdev->irq.vblank_queue);
7651                 }
7652                 if (atomic_read(&rdev->irq.pflip[2]))
7653                     radeon_crtc_handle_vblank(rdev, 2);
7654                 rdev->irq.stat_regs.cik.disp_int_cont2 &= ~LB_D3_VBLANK_INTERRUPT;
7655                 DRM_DEBUG("IH: D3 vblank\n");
7656 
7657                 break;
7658             case 1: /* D3 vline */
7659                 if (!(rdev->irq.stat_regs.cik.disp_int_cont2 & LB_D3_VLINE_INTERRUPT))
7660                     DRM_DEBUG("IH: IH event w/o asserted irq bit?\n");
7661 
7662                 rdev->irq.stat_regs.cik.disp_int_cont2 &= ~LB_D3_VLINE_INTERRUPT;
7663                 DRM_DEBUG("IH: D3 vline\n");
7664 
7665                 break;
7666             default:
7667                 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
7668                 break;
7669             }
7670             break;
7671         case 4: /* D4 vblank/vline */
7672             switch (src_data) {
7673             case 0: /* D4 vblank */
7674                 if (!(rdev->irq.stat_regs.cik.disp_int_cont3 & LB_D4_VBLANK_INTERRUPT))
7675                     DRM_DEBUG("IH: IH event w/o asserted irq bit?\n");
7676 
7677                 if (rdev->irq.crtc_vblank_int[3]) {
7678                     drm_handle_vblank(rdev->ddev, 3);
7679                     rdev->pm.vblank_sync = true;
7680                     wake_up(&rdev->irq.vblank_queue);
7681                 }
7682                 if (atomic_read(&rdev->irq.pflip[3]))
7683                     radeon_crtc_handle_vblank(rdev, 3);
7684                 rdev->irq.stat_regs.cik.disp_int_cont3 &= ~LB_D4_VBLANK_INTERRUPT;
7685                 DRM_DEBUG("IH: D4 vblank\n");
7686 
7687                 break;
7688             case 1: /* D4 vline */
7689                 if (!(rdev->irq.stat_regs.cik.disp_int_cont3 & LB_D4_VLINE_INTERRUPT))
7690                     DRM_DEBUG("IH: IH event w/o asserted irq bit?\n");
7691 
7692                 rdev->irq.stat_regs.cik.disp_int_cont3 &= ~LB_D4_VLINE_INTERRUPT;
7693                 DRM_DEBUG("IH: D4 vline\n");
7694 
7695                 break;
7696             default:
7697                 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
7698                 break;
7699             }
7700             break;
7701         case 5: /* D5 vblank/vline */
7702             switch (src_data) {
7703             case 0: /* D5 vblank */
7704                 if (!(rdev->irq.stat_regs.cik.disp_int_cont4 & LB_D5_VBLANK_INTERRUPT))
7705                     DRM_DEBUG("IH: IH event w/o asserted irq bit?\n");
7706 
7707                 if (rdev->irq.crtc_vblank_int[4]) {
7708                     drm_handle_vblank(rdev->ddev, 4);
7709                     rdev->pm.vblank_sync = true;
7710                     wake_up(&rdev->irq.vblank_queue);
7711                 }
7712                 if (atomic_read(&rdev->irq.pflip[4]))
7713                     radeon_crtc_handle_vblank(rdev, 4);
7714                 rdev->irq.stat_regs.cik.disp_int_cont4 &= ~LB_D5_VBLANK_INTERRUPT;
7715                 DRM_DEBUG("IH: D5 vblank\n");
7716 
7717                 break;
7718             case 1: /* D5 vline */
7719                 if (!(rdev->irq.stat_regs.cik.disp_int_cont4 & LB_D5_VLINE_INTERRUPT))
7720                     DRM_DEBUG("IH: IH event w/o asserted irq bit?\n");
7721 
7722                 rdev->irq.stat_regs.cik.disp_int_cont4 &= ~LB_D5_VLINE_INTERRUPT;
7723                 DRM_DEBUG("IH: D5 vline\n");
7724 
7725                 break;
7726             default:
7727                 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
7728                 break;
7729             }
7730             break;
7731         case 6: /* D6 vblank/vline */
7732             switch (src_data) {
7733             case 0: /* D6 vblank */
7734                 if (!(rdev->irq.stat_regs.cik.disp_int_cont5 & LB_D6_VBLANK_INTERRUPT))
7735                     DRM_DEBUG("IH: IH event w/o asserted irq bit?\n");
7736 
7737                 if (rdev->irq.crtc_vblank_int[5]) {
7738                     drm_handle_vblank(rdev->ddev, 5);
7739                     rdev->pm.vblank_sync = true;
7740                     wake_up(&rdev->irq.vblank_queue);
7741                 }
7742                 if (atomic_read(&rdev->irq.pflip[5]))
7743                     radeon_crtc_handle_vblank(rdev, 5);
7744                 rdev->irq.stat_regs.cik.disp_int_cont5 &= ~LB_D6_VBLANK_INTERRUPT;
7745                 DRM_DEBUG("IH: D6 vblank\n");
7746 
7747                 break;
7748             case 1: /* D6 vline */
7749                 if (!(rdev->irq.stat_regs.cik.disp_int_cont5 & LB_D6_VLINE_INTERRUPT))
7750                     DRM_DEBUG("IH: IH event w/o asserted irq bit?\n");
7751 
7752                 rdev->irq.stat_regs.cik.disp_int_cont5 &= ~LB_D6_VLINE_INTERRUPT;
7753                 DRM_DEBUG("IH: D6 vline\n");
7754 
7755                 break;
7756             default:
7757                 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
7758                 break;
7759             }
7760             break;
7761         case 8: /* D1 page flip */
7762         case 10: /* D2 page flip */
7763         case 12: /* D3 page flip */
7764         case 14: /* D4 page flip */
7765         case 16: /* D5 page flip */
7766         case 18: /* D6 page flip */
7767             DRM_DEBUG("IH: D%d flip\n", ((src_id - 8) >> 1) + 1);
7768             if (radeon_use_pflipirq > 0)
7769                 radeon_crtc_handle_flip(rdev, (src_id - 8) >> 1);
7770             break;
7771         case 42: /* HPD hotplug */
7772             switch (src_data) {
7773             case 0:
7774                 if (!(rdev->irq.stat_regs.cik.disp_int & DC_HPD1_INTERRUPT))
7775                     DRM_DEBUG("IH: IH event w/o asserted irq bit?\n");
7776 
7777                 rdev->irq.stat_regs.cik.disp_int &= ~DC_HPD1_INTERRUPT;
7778                 queue_hotplug = true;
7779                 DRM_DEBUG("IH: HPD1\n");
7780 
7781                 break;
7782             case 1:
7783                 if (!(rdev->irq.stat_regs.cik.disp_int_cont & DC_HPD2_INTERRUPT))
7784                     DRM_DEBUG("IH: IH event w/o asserted irq bit?\n");
7785 
7786                 rdev->irq.stat_regs.cik.disp_int_cont &= ~DC_HPD2_INTERRUPT;
7787                 queue_hotplug = true;
7788                 DRM_DEBUG("IH: HPD2\n");
7789 
7790                 break;
7791             case 2:
7792                 if (!(rdev->irq.stat_regs.cik.disp_int_cont2 & DC_HPD3_INTERRUPT))
7793                     DRM_DEBUG("IH: IH event w/o asserted irq bit?\n");
7794 
7795                 rdev->irq.stat_regs.cik.disp_int_cont2 &= ~DC_HPD3_INTERRUPT;
7796                 queue_hotplug = true;
7797                 DRM_DEBUG("IH: HPD3\n");
7798 
7799                 break;
7800             case 3:
7801                 if (!(rdev->irq.stat_regs.cik.disp_int_cont3 & DC_HPD4_INTERRUPT))
7802                     DRM_DEBUG("IH: IH event w/o asserted irq bit?\n");
7803 
7804                 rdev->irq.stat_regs.cik.disp_int_cont3 &= ~DC_HPD4_INTERRUPT;
7805                 queue_hotplug = true;
7806                 DRM_DEBUG("IH: HPD4\n");
7807 
7808                 break;
7809             case 4:
7810                 if (!(rdev->irq.stat_regs.cik.disp_int_cont4 & DC_HPD5_INTERRUPT))
7811                     DRM_DEBUG("IH: IH event w/o asserted irq bit?\n");
7812 
7813                 rdev->irq.stat_regs.cik.disp_int_cont4 &= ~DC_HPD5_INTERRUPT;
7814                 queue_hotplug = true;
7815                 DRM_DEBUG("IH: HPD5\n");
7816 
7817                 break;
7818             case 5:
7819                 if (!(rdev->irq.stat_regs.cik.disp_int_cont5 & DC_HPD6_INTERRUPT))
7820                     DRM_DEBUG("IH: IH event w/o asserted irq bit?\n");
7821 
7822                 rdev->irq.stat_regs.cik.disp_int_cont5 &= ~DC_HPD6_INTERRUPT;
7823                 queue_hotplug = true;
7824                 DRM_DEBUG("IH: HPD6\n");
7825 
7826                 break;
7827             case 6:
7828                 if (!(rdev->irq.stat_regs.cik.disp_int & DC_HPD1_RX_INTERRUPT))
7829                     DRM_DEBUG("IH: IH event w/o asserted irq bit?\n");
7830 
7831                 rdev->irq.stat_regs.cik.disp_int &= ~DC_HPD1_RX_INTERRUPT;
7832                 queue_dp = true;
7833                 DRM_DEBUG("IH: HPD_RX 1\n");
7834 
7835                 break;
7836             case 7:
7837                 if (!(rdev->irq.stat_regs.cik.disp_int_cont & DC_HPD2_RX_INTERRUPT))
7838                     DRM_DEBUG("IH: IH event w/o asserted irq bit?\n");
7839 
7840                 rdev->irq.stat_regs.cik.disp_int_cont &= ~DC_HPD2_RX_INTERRUPT;
7841                 queue_dp = true;
7842                 DRM_DEBUG("IH: HPD_RX 2\n");
7843 
7844                 break;
7845             case 8:
7846                 if (!(rdev->irq.stat_regs.cik.disp_int_cont2 & DC_HPD3_RX_INTERRUPT))
7847                     DRM_DEBUG("IH: IH event w/o asserted irq bit?\n");
7848 
7849                 rdev->irq.stat_regs.cik.disp_int_cont2 &= ~DC_HPD3_RX_INTERRUPT;
7850                 queue_dp = true;
7851                 DRM_DEBUG("IH: HPD_RX 3\n");
7852 
7853                 break;
7854             case 9:
7855                 if (!(rdev->irq.stat_regs.cik.disp_int_cont3 & DC_HPD4_RX_INTERRUPT))
7856                     DRM_DEBUG("IH: IH event w/o asserted irq bit?\n");
7857 
7858                 rdev->irq.stat_regs.cik.disp_int_cont3 &= ~DC_HPD4_RX_INTERRUPT;
7859                 queue_dp = true;
7860                 DRM_DEBUG("IH: HPD_RX 4\n");
7861 
7862                 break;
7863             case 10:
7864                 if (!(rdev->irq.stat_regs.cik.disp_int_cont4 & DC_HPD5_RX_INTERRUPT))
7865                     DRM_DEBUG("IH: IH event w/o asserted irq bit?\n");
7866 
7867                 rdev->irq.stat_regs.cik.disp_int_cont4 &= ~DC_HPD5_RX_INTERRUPT;
7868                 queue_dp = true;
7869                 DRM_DEBUG("IH: HPD_RX 5\n");
7870 
7871                 break;
7872             case 11:
7873                 if (!(rdev->irq.stat_regs.cik.disp_int_cont5 & DC_HPD6_RX_INTERRUPT))
7874                     DRM_DEBUG("IH: IH event w/o asserted irq bit?\n");
7875 
7876                 rdev->irq.stat_regs.cik.disp_int_cont5 &= ~DC_HPD6_RX_INTERRUPT;
7877                 queue_dp = true;
7878                 DRM_DEBUG("IH: HPD_RX 6\n");
7879 
7880                 break;
7881             default:
7882                 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
7883                 break;
7884             }
7885             break;
7886         case 96:
7887             DRM_ERROR("SRBM_READ_ERROR: 0x%x\n", RREG32(SRBM_READ_ERROR));
7888             WREG32(SRBM_INT_ACK, 0x1);
7889             break;
7890         case 124: /* UVD */
7891             DRM_DEBUG("IH: UVD int: 0x%08x\n", src_data);
7892             radeon_fence_process(rdev, R600_RING_TYPE_UVD_INDEX);
7893             break;
7894         case 146:
7895         case 147:
7896             addr = RREG32(VM_CONTEXT1_PROTECTION_FAULT_ADDR);
7897             status = RREG32(VM_CONTEXT1_PROTECTION_FAULT_STATUS);
7898             mc_client = RREG32(VM_CONTEXT1_PROTECTION_FAULT_MCCLIENT);
7899             /* reset addr and status */
7900             WREG32_P(VM_CONTEXT1_CNTL2, 1, ~1);
7901             if (addr == 0x0 && status == 0x0)
7902                 break;
7903             dev_err(rdev->dev, "GPU fault detected: %d 0x%08x\n", src_id, src_data);
7904             dev_err(rdev->dev, "  VM_CONTEXT1_PROTECTION_FAULT_ADDR   0x%08X\n",
7905                 addr);
7906             dev_err(rdev->dev, "  VM_CONTEXT1_PROTECTION_FAULT_STATUS 0x%08X\n",
7907                 status);
7908             cik_vm_decode_fault(rdev, status, addr, mc_client);
7909             break;
7910         case 167: /* VCE */
7911             DRM_DEBUG("IH: VCE int: 0x%08x\n", src_data);
7912             switch (src_data) {
7913             case 0:
7914                 radeon_fence_process(rdev, TN_RING_TYPE_VCE1_INDEX);
7915                 break;
7916             case 1:
7917                 radeon_fence_process(rdev, TN_RING_TYPE_VCE2_INDEX);
7918                 break;
7919             default:
7920                 DRM_ERROR("Unhandled interrupt: %d %d\n", src_id, src_data);
7921                 break;
7922             }
7923             break;
7924         case 176: /* GFX RB CP_INT */
7925         case 177: /* GFX IB CP_INT */
7926             radeon_fence_process(rdev, RADEON_RING_TYPE_GFX_INDEX);
7927             break;
7928         case 181: /* CP EOP event */
7929             DRM_DEBUG("IH: CP EOP\n");
7930             /* XXX check the bitfield order! */
7931             me_id = (ring_id & 0x60) >> 5;
7932             pipe_id = (ring_id & 0x18) >> 3;
7933             queue_id = (ring_id & 0x7) >> 0;
7934             switch (me_id) {
7935             case 0:
7936                 radeon_fence_process(rdev, RADEON_RING_TYPE_GFX_INDEX);
7937                 break;
7938             case 1:
7939             case 2:
7940                 if ((cp1_ring->me == me_id) & (cp1_ring->pipe == pipe_id))
7941                     radeon_fence_process(rdev, CAYMAN_RING_TYPE_CP1_INDEX);
7942                 if ((cp2_ring->me == me_id) & (cp2_ring->pipe == pipe_id))
7943                     radeon_fence_process(rdev, CAYMAN_RING_TYPE_CP2_INDEX);
7944                 break;
7945             }
7946             break;
7947         case 184: /* CP Privileged reg access */
7948             DRM_ERROR("Illegal register access in command stream\n");
7949             /* XXX check the bitfield order! */
7950             me_id = (ring_id & 0x60) >> 5;
7951             switch (me_id) {
7952             case 0:
7953                 /* This results in a full GPU reset, but all we need to do is soft
7954                  * reset the CP for gfx
7955                  */
7956                 queue_reset = true;
7957                 break;
7958             case 1:
7959                 /* XXX compute */
7960                 queue_reset = true;
7961                 break;
7962             case 2:
7963                 /* XXX compute */
7964                 queue_reset = true;
7965                 break;
7966             }
7967             break;
7968         case 185: /* CP Privileged inst */
7969             DRM_ERROR("Illegal instruction in command stream\n");
7970             /* XXX check the bitfield order! */
7971             me_id = (ring_id & 0x60) >> 5;
7972             switch (me_id) {
7973             case 0:
7974                 /* This results in a full GPU reset, but all we need to do is soft
7975                  * reset the CP for gfx
7976                  */
7977                 queue_reset = true;
7978                 break;
7979             case 1:
7980                 /* XXX compute */
7981                 queue_reset = true;
7982                 break;
7983             case 2:
7984                 /* XXX compute */
7985                 queue_reset = true;
7986                 break;
7987             }
7988             break;
7989         case 224: /* SDMA trap event */
7990             /* XXX check the bitfield order! */
7991             me_id = (ring_id & 0x3) >> 0;
7992             queue_id = (ring_id & 0xc) >> 2;
7993             DRM_DEBUG("IH: SDMA trap\n");
7994             switch (me_id) {
7995             case 0:
7996                 switch (queue_id) {
7997                 case 0:
7998                     radeon_fence_process(rdev, R600_RING_TYPE_DMA_INDEX);
7999                     break;
8000                 case 1:
8001                     /* XXX compute */
8002                     break;
8003                 case 2:
8004                     /* XXX compute */
8005                     break;
8006                 }
8007                 break;
8008             case 1:
8009                 switch (queue_id) {
8010                 case 0:
8011                     radeon_fence_process(rdev, CAYMAN_RING_TYPE_DMA1_INDEX);
8012                     break;
8013                 case 1:
8014                     /* XXX compute */
8015                     break;
8016                 case 2:
8017                     /* XXX compute */
8018                     break;
8019                 }
8020                 break;
8021             }
8022             break;
8023         case 230: /* thermal low to high */
8024             DRM_DEBUG("IH: thermal low to high\n");
8025             rdev->pm.dpm.thermal.high_to_low = false;
8026             queue_thermal = true;
8027             break;
8028         case 231: /* thermal high to low */
8029             DRM_DEBUG("IH: thermal high to low\n");
8030             rdev->pm.dpm.thermal.high_to_low = true;
8031             queue_thermal = true;
8032             break;
8033         case 233: /* GUI IDLE */
8034             DRM_DEBUG("IH: GUI idle\n");
8035             break;
8036         case 241: /* SDMA Privileged inst */
8037         case 247: /* SDMA Privileged inst */
8038             DRM_ERROR("Illegal instruction in SDMA command stream\n");
8039             /* XXX check the bitfield order! */
8040             me_id = (ring_id & 0x3) >> 0;
8041             queue_id = (ring_id & 0xc) >> 2;
8042             switch (me_id) {
8043             case 0:
8044                 switch (queue_id) {
8045                 case 0:
8046                     queue_reset = true;
8047                     break;
8048                 case 1:
8049                     /* XXX compute */
8050                     queue_reset = true;
8051                     break;
8052                 case 2:
8053                     /* XXX compute */
8054                     queue_reset = true;
8055                     break;
8056                 }
8057                 break;
8058             case 1:
8059                 switch (queue_id) {
8060                 case 0:
8061                     queue_reset = true;
8062                     break;
8063                 case 1:
8064                     /* XXX compute */
8065                     queue_reset = true;
8066                     break;
8067                 case 2:
8068                     /* XXX compute */
8069                     queue_reset = true;
8070                     break;
8071                 }
8072                 break;
8073             }
8074             break;
8075         default:
8076             DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
8077             break;
8078         }
8079 
8080         /* wptr/rptr are in bytes! */
8081         rptr += 16;
8082         rptr &= rdev->ih.ptr_mask;
8083         WREG32(IH_RB_RPTR, rptr);
8084     }
8085     if (queue_dp)
8086         schedule_work(&rdev->dp_work);
8087     if (queue_hotplug)
8088         schedule_delayed_work(&rdev->hotplug_work, 0);
8089     if (queue_reset) {
8090         rdev->needs_reset = true;
8091         wake_up_all(&rdev->fence_queue);
8092     }
8093     if (queue_thermal)
8094         schedule_work(&rdev->pm.dpm.thermal.work);
8095     rdev->ih.rptr = rptr;
8096     atomic_set(&rdev->ih.lock, 0);
8097 
8098     /* make sure wptr hasn't changed while processing */
8099     wptr = cik_get_ih_wptr(rdev);
8100     if (wptr != rptr)
8101         goto restart_ih;
8102 
8103     return IRQ_HANDLED;
8104 }
8105 
8106 /*
8107  * startup/shutdown callbacks
8108  */
8109 static void cik_uvd_init(struct radeon_device *rdev)
8110 {
8111     int r;
8112 
8113     if (!rdev->has_uvd)
8114         return;
8115 
8116     r = radeon_uvd_init(rdev);
8117     if (r) {
8118         dev_err(rdev->dev, "failed UVD (%d) init.\n", r);
8119         /*
8120          * At this point rdev->uvd.vcpu_bo is NULL which trickles down
8121          * to early fails cik_uvd_start() and thus nothing happens
8122          * there. So it is pointless to try to go through that code
8123          * hence why we disable uvd here.
8124          */
8125         rdev->has_uvd = false;
8126         return;
8127     }
8128     rdev->ring[R600_RING_TYPE_UVD_INDEX].ring_obj = NULL;
8129     r600_ring_init(rdev, &rdev->ring[R600_RING_TYPE_UVD_INDEX], 4096);
8130 }
8131 
8132 static void cik_uvd_start(struct radeon_device *rdev)
8133 {
8134     int r;
8135 
8136     if (!rdev->has_uvd)
8137         return;
8138 
8139     r = radeon_uvd_resume(rdev);
8140     if (r) {
8141         dev_err(rdev->dev, "failed UVD resume (%d).\n", r);
8142         goto error;
8143     }
8144     r = uvd_v4_2_resume(rdev);
8145     if (r) {
8146         dev_err(rdev->dev, "failed UVD 4.2 resume (%d).\n", r);
8147         goto error;
8148     }
8149     r = radeon_fence_driver_start_ring(rdev, R600_RING_TYPE_UVD_INDEX);
8150     if (r) {
8151         dev_err(rdev->dev, "failed initializing UVD fences (%d).\n", r);
8152         goto error;
8153     }
8154     return;
8155 
8156 error:
8157     rdev->ring[R600_RING_TYPE_UVD_INDEX].ring_size = 0;
8158 }
8159 
8160 static void cik_uvd_resume(struct radeon_device *rdev)
8161 {
8162     struct radeon_ring *ring;
8163     int r;
8164 
8165     if (!rdev->has_uvd || !rdev->ring[R600_RING_TYPE_UVD_INDEX].ring_size)
8166         return;
8167 
8168     ring = &rdev->ring[R600_RING_TYPE_UVD_INDEX];
8169     r = radeon_ring_init(rdev, ring, ring->ring_size, 0, PACKET0(UVD_NO_OP, 0));
8170     if (r) {
8171         dev_err(rdev->dev, "failed initializing UVD ring (%d).\n", r);
8172         return;
8173     }
8174     r = uvd_v1_0_init(rdev);
8175     if (r) {
8176         dev_err(rdev->dev, "failed initializing UVD (%d).\n", r);
8177         return;
8178     }
8179 }
8180 
8181 static void cik_vce_init(struct radeon_device *rdev)
8182 {
8183     int r;
8184 
8185     if (!rdev->has_vce)
8186         return;
8187 
8188     r = radeon_vce_init(rdev);
8189     if (r) {
8190         dev_err(rdev->dev, "failed VCE (%d) init.\n", r);
8191         /*
8192          * At this point rdev->vce.vcpu_bo is NULL which trickles down
8193          * to early fails cik_vce_start() and thus nothing happens
8194          * there. So it is pointless to try to go through that code
8195          * hence why we disable vce here.
8196          */
8197         rdev->has_vce = false;
8198         return;
8199     }
8200     rdev->ring[TN_RING_TYPE_VCE1_INDEX].ring_obj = NULL;
8201     r600_ring_init(rdev, &rdev->ring[TN_RING_TYPE_VCE1_INDEX], 4096);
8202     rdev->ring[TN_RING_TYPE_VCE2_INDEX].ring_obj = NULL;
8203     r600_ring_init(rdev, &rdev->ring[TN_RING_TYPE_VCE2_INDEX], 4096);
8204 }
8205 
8206 static void cik_vce_start(struct radeon_device *rdev)
8207 {
8208     int r;
8209 
8210     if (!rdev->has_vce)
8211         return;
8212 
8213     r = radeon_vce_resume(rdev);
8214     if (r) {
8215         dev_err(rdev->dev, "failed VCE resume (%d).\n", r);
8216         goto error;
8217     }
8218     r = vce_v2_0_resume(rdev);
8219     if (r) {
8220         dev_err(rdev->dev, "failed VCE resume (%d).\n", r);
8221         goto error;
8222     }
8223     r = radeon_fence_driver_start_ring(rdev, TN_RING_TYPE_VCE1_INDEX);
8224     if (r) {
8225         dev_err(rdev->dev, "failed initializing VCE1 fences (%d).\n", r);
8226         goto error;
8227     }
8228     r = radeon_fence_driver_start_ring(rdev, TN_RING_TYPE_VCE2_INDEX);
8229     if (r) {
8230         dev_err(rdev->dev, "failed initializing VCE2 fences (%d).\n", r);
8231         goto error;
8232     }
8233     return;
8234 
8235 error:
8236     rdev->ring[TN_RING_TYPE_VCE1_INDEX].ring_size = 0;
8237     rdev->ring[TN_RING_TYPE_VCE2_INDEX].ring_size = 0;
8238 }
8239 
8240 static void cik_vce_resume(struct radeon_device *rdev)
8241 {
8242     struct radeon_ring *ring;
8243     int r;
8244 
8245     if (!rdev->has_vce || !rdev->ring[TN_RING_TYPE_VCE1_INDEX].ring_size)
8246         return;
8247 
8248     ring = &rdev->ring[TN_RING_TYPE_VCE1_INDEX];
8249     r = radeon_ring_init(rdev, ring, ring->ring_size, 0, VCE_CMD_NO_OP);
8250     if (r) {
8251         dev_err(rdev->dev, "failed initializing VCE1 ring (%d).\n", r);
8252         return;
8253     }
8254     ring = &rdev->ring[TN_RING_TYPE_VCE2_INDEX];
8255     r = radeon_ring_init(rdev, ring, ring->ring_size, 0, VCE_CMD_NO_OP);
8256     if (r) {
8257         dev_err(rdev->dev, "failed initializing VCE1 ring (%d).\n", r);
8258         return;
8259     }
8260     r = vce_v1_0_init(rdev);
8261     if (r) {
8262         dev_err(rdev->dev, "failed initializing VCE (%d).\n", r);
8263         return;
8264     }
8265 }
8266 
8267 /**
8268  * cik_startup - program the asic to a functional state
8269  *
8270  * @rdev: radeon_device pointer
8271  *
8272  * Programs the asic to a functional state (CIK).
8273  * Called by cik_init() and cik_resume().
8274  * Returns 0 for success, error for failure.
8275  */
8276 static int cik_startup(struct radeon_device *rdev)
8277 {
8278     struct radeon_ring *ring;
8279     u32 nop;
8280     int r;
8281 
8282     /* enable pcie gen2/3 link */
8283     cik_pcie_gen3_enable(rdev);
8284     /* enable aspm */
8285     cik_program_aspm(rdev);
8286 
8287     /* scratch needs to be initialized before MC */
8288     r = r600_vram_scratch_init(rdev);
8289     if (r)
8290         return r;
8291 
8292     cik_mc_program(rdev);
8293 
8294     if (!(rdev->flags & RADEON_IS_IGP) && !rdev->pm.dpm_enabled) {
8295         r = ci_mc_load_microcode(rdev);
8296         if (r) {
8297             DRM_ERROR("Failed to load MC firmware!\n");
8298             return r;
8299         }
8300     }
8301 
8302     r = cik_pcie_gart_enable(rdev);
8303     if (r)
8304         return r;
8305     cik_gpu_init(rdev);
8306 
8307     /* allocate rlc buffers */
8308     if (rdev->flags & RADEON_IS_IGP) {
8309         if (rdev->family == CHIP_KAVERI) {
8310             rdev->rlc.reg_list = spectre_rlc_save_restore_register_list;
8311             rdev->rlc.reg_list_size =
8312                 (u32)ARRAY_SIZE(spectre_rlc_save_restore_register_list);
8313         } else {
8314             rdev->rlc.reg_list = kalindi_rlc_save_restore_register_list;
8315             rdev->rlc.reg_list_size =
8316                 (u32)ARRAY_SIZE(kalindi_rlc_save_restore_register_list);
8317         }
8318     }
8319     rdev->rlc.cs_data = ci_cs_data;
8320     rdev->rlc.cp_table_size = ALIGN(CP_ME_TABLE_SIZE * 5 * 4, 2048); /* CP JT */
8321     rdev->rlc.cp_table_size += 64 * 1024; /* GDS */
8322     r = sumo_rlc_init(rdev);
8323     if (r) {
8324         DRM_ERROR("Failed to init rlc BOs!\n");
8325         return r;
8326     }
8327 
8328     /* allocate wb buffer */
8329     r = radeon_wb_init(rdev);
8330     if (r)
8331         return r;
8332 
8333     /* allocate mec buffers */
8334     r = cik_mec_init(rdev);
8335     if (r) {
8336         DRM_ERROR("Failed to init MEC BOs!\n");
8337         return r;
8338     }
8339 
8340     r = radeon_fence_driver_start_ring(rdev, RADEON_RING_TYPE_GFX_INDEX);
8341     if (r) {
8342         dev_err(rdev->dev, "failed initializing CP fences (%d).\n", r);
8343         return r;
8344     }
8345 
8346     r = radeon_fence_driver_start_ring(rdev, CAYMAN_RING_TYPE_CP1_INDEX);
8347     if (r) {
8348         dev_err(rdev->dev, "failed initializing CP fences (%d).\n", r);
8349         return r;
8350     }
8351 
8352     r = radeon_fence_driver_start_ring(rdev, CAYMAN_RING_TYPE_CP2_INDEX);
8353     if (r) {
8354         dev_err(rdev->dev, "failed initializing CP fences (%d).\n", r);
8355         return r;
8356     }
8357 
8358     r = radeon_fence_driver_start_ring(rdev, R600_RING_TYPE_DMA_INDEX);
8359     if (r) {
8360         dev_err(rdev->dev, "failed initializing DMA fences (%d).\n", r);
8361         return r;
8362     }
8363 
8364     r = radeon_fence_driver_start_ring(rdev, CAYMAN_RING_TYPE_DMA1_INDEX);
8365     if (r) {
8366         dev_err(rdev->dev, "failed initializing DMA fences (%d).\n", r);
8367         return r;
8368     }
8369 
8370     cik_uvd_start(rdev);
8371     cik_vce_start(rdev);
8372 
8373     /* Enable IRQ */
8374     if (!rdev->irq.installed) {
8375         r = radeon_irq_kms_init(rdev);
8376         if (r)
8377             return r;
8378     }
8379 
8380     r = cik_irq_init(rdev);
8381     if (r) {
8382         DRM_ERROR("radeon: IH init failed (%d).\n", r);
8383         radeon_irq_kms_fini(rdev);
8384         return r;
8385     }
8386     cik_irq_set(rdev);
8387 
8388     if (rdev->family == CHIP_HAWAII) {
8389         if (rdev->new_fw)
8390             nop = PACKET3(PACKET3_NOP, 0x3FFF);
8391         else
8392             nop = RADEON_CP_PACKET2;
8393     } else {
8394         nop = PACKET3(PACKET3_NOP, 0x3FFF);
8395     }
8396 
8397     ring = &rdev->ring[RADEON_RING_TYPE_GFX_INDEX];
8398     r = radeon_ring_init(rdev, ring, ring->ring_size, RADEON_WB_CP_RPTR_OFFSET,
8399                  nop);
8400     if (r)
8401         return r;
8402 
8403     /* set up the compute queues */
8404     /* type-2 packets are deprecated on MEC, use type-3 instead */
8405     ring = &rdev->ring[CAYMAN_RING_TYPE_CP1_INDEX];
8406     r = radeon_ring_init(rdev, ring, ring->ring_size, RADEON_WB_CP1_RPTR_OFFSET,
8407                  nop);
8408     if (r)
8409         return r;
8410     ring->me = 1; /* first MEC */
8411     ring->pipe = 0; /* first pipe */
8412     ring->queue = 0; /* first queue */
8413     ring->wptr_offs = CIK_WB_CP1_WPTR_OFFSET;
8414 
8415     /* type-2 packets are deprecated on MEC, use type-3 instead */
8416     ring = &rdev->ring[CAYMAN_RING_TYPE_CP2_INDEX];
8417     r = radeon_ring_init(rdev, ring, ring->ring_size, RADEON_WB_CP2_RPTR_OFFSET,
8418                  nop);
8419     if (r)
8420         return r;
8421     /* dGPU only have 1 MEC */
8422     ring->me = 1; /* first MEC */
8423     ring->pipe = 0; /* first pipe */
8424     ring->queue = 1; /* second queue */
8425     ring->wptr_offs = CIK_WB_CP2_WPTR_OFFSET;
8426 
8427     ring = &rdev->ring[R600_RING_TYPE_DMA_INDEX];
8428     r = radeon_ring_init(rdev, ring, ring->ring_size, R600_WB_DMA_RPTR_OFFSET,
8429                  SDMA_PACKET(SDMA_OPCODE_NOP, 0, 0));
8430     if (r)
8431         return r;
8432 
8433     ring = &rdev->ring[CAYMAN_RING_TYPE_DMA1_INDEX];
8434     r = radeon_ring_init(rdev, ring, ring->ring_size, CAYMAN_WB_DMA1_RPTR_OFFSET,
8435                  SDMA_PACKET(SDMA_OPCODE_NOP, 0, 0));
8436     if (r)
8437         return r;
8438 
8439     r = cik_cp_resume(rdev);
8440     if (r)
8441         return r;
8442 
8443     r = cik_sdma_resume(rdev);
8444     if (r)
8445         return r;
8446 
8447     cik_uvd_resume(rdev);
8448     cik_vce_resume(rdev);
8449 
8450     r = radeon_ib_pool_init(rdev);
8451     if (r) {
8452         dev_err(rdev->dev, "IB initialization failed (%d).\n", r);
8453         return r;
8454     }
8455 
8456     r = radeon_vm_manager_init(rdev);
8457     if (r) {
8458         dev_err(rdev->dev, "vm manager initialization failed (%d).\n", r);
8459         return r;
8460     }
8461 
8462     r = radeon_audio_init(rdev);
8463     if (r)
8464         return r;
8465 
8466     return 0;
8467 }
8468 
8469 /**
8470  * cik_resume - resume the asic to a functional state
8471  *
8472  * @rdev: radeon_device pointer
8473  *
8474  * Programs the asic to a functional state (CIK).
8475  * Called at resume.
8476  * Returns 0 for success, error for failure.
8477  */
8478 int cik_resume(struct radeon_device *rdev)
8479 {
8480     int r;
8481 
8482     /* post card */
8483     atom_asic_init(rdev->mode_info.atom_context);
8484 
8485     /* init golden registers */
8486     cik_init_golden_registers(rdev);
8487 
8488     if (rdev->pm.pm_method == PM_METHOD_DPM)
8489         radeon_pm_resume(rdev);
8490 
8491     rdev->accel_working = true;
8492     r = cik_startup(rdev);
8493     if (r) {
8494         DRM_ERROR("cik startup failed on resume\n");
8495         rdev->accel_working = false;
8496         return r;
8497     }
8498 
8499     return r;
8500 
8501 }
8502 
8503 /**
8504  * cik_suspend - suspend the asic
8505  *
8506  * @rdev: radeon_device pointer
8507  *
8508  * Bring the chip into a state suitable for suspend (CIK).
8509  * Called at suspend.
8510  * Returns 0 for success.
8511  */
8512 int cik_suspend(struct radeon_device *rdev)
8513 {
8514     radeon_pm_suspend(rdev);
8515     radeon_audio_fini(rdev);
8516     radeon_vm_manager_fini(rdev);
8517     cik_cp_enable(rdev, false);
8518     cik_sdma_enable(rdev, false);
8519     if (rdev->has_uvd) {
8520         radeon_uvd_suspend(rdev);
8521         uvd_v1_0_fini(rdev);
8522     }
8523     if (rdev->has_vce)
8524         radeon_vce_suspend(rdev);
8525     cik_fini_pg(rdev);
8526     cik_fini_cg(rdev);
8527     cik_irq_suspend(rdev);
8528     radeon_wb_disable(rdev);
8529     cik_pcie_gart_disable(rdev);
8530     return 0;
8531 }
8532 
8533 /* Plan is to move initialization in that function and use
8534  * helper function so that radeon_device_init pretty much
8535  * do nothing more than calling asic specific function. This
8536  * should also allow to remove a bunch of callback function
8537  * like vram_info.
8538  */
8539 /**
8540  * cik_init - asic specific driver and hw init
8541  *
8542  * @rdev: radeon_device pointer
8543  *
8544  * Setup asic specific driver variables and program the hw
8545  * to a functional state (CIK).
8546  * Called at driver startup.
8547  * Returns 0 for success, errors for failure.
8548  */
8549 int cik_init(struct radeon_device *rdev)
8550 {
8551     struct radeon_ring *ring;
8552     int r;
8553 
8554     /* Read BIOS */
8555     if (!radeon_get_bios(rdev)) {
8556         if (ASIC_IS_AVIVO(rdev))
8557             return -EINVAL;
8558     }
8559     /* Must be an ATOMBIOS */
8560     if (!rdev->is_atom_bios) {
8561         dev_err(rdev->dev, "Expecting atombios for cayman GPU\n");
8562         return -EINVAL;
8563     }
8564     r = radeon_atombios_init(rdev);
8565     if (r)
8566         return r;
8567 
8568     /* Post card if necessary */
8569     if (!radeon_card_posted(rdev)) {
8570         if (!rdev->bios) {
8571             dev_err(rdev->dev, "Card not posted and no BIOS - ignoring\n");
8572             return -EINVAL;
8573         }
8574         DRM_INFO("GPU not posted. posting now...\n");
8575         atom_asic_init(rdev->mode_info.atom_context);
8576     }
8577     /* init golden registers */
8578     cik_init_golden_registers(rdev);
8579     /* Initialize scratch registers */
8580     cik_scratch_init(rdev);
8581     /* Initialize surface registers */
8582     radeon_surface_init(rdev);
8583     /* Initialize clocks */
8584     radeon_get_clock_info(rdev->ddev);
8585 
8586     /* Fence driver */
8587     radeon_fence_driver_init(rdev);
8588 
8589     /* initialize memory controller */
8590     r = cik_mc_init(rdev);
8591     if (r)
8592         return r;
8593     /* Memory manager */
8594     r = radeon_bo_init(rdev);
8595     if (r)
8596         return r;
8597 
8598     if (rdev->flags & RADEON_IS_IGP) {
8599         if (!rdev->me_fw || !rdev->pfp_fw || !rdev->ce_fw ||
8600             !rdev->mec_fw || !rdev->sdma_fw || !rdev->rlc_fw) {
8601             r = cik_init_microcode(rdev);
8602             if (r) {
8603                 DRM_ERROR("Failed to load firmware!\n");
8604                 return r;
8605             }
8606         }
8607     } else {
8608         if (!rdev->me_fw || !rdev->pfp_fw || !rdev->ce_fw ||
8609             !rdev->mec_fw || !rdev->sdma_fw || !rdev->rlc_fw ||
8610             !rdev->mc_fw) {
8611             r = cik_init_microcode(rdev);
8612             if (r) {
8613                 DRM_ERROR("Failed to load firmware!\n");
8614                 return r;
8615             }
8616         }
8617     }
8618 
8619     /* Initialize power management */
8620     radeon_pm_init(rdev);
8621 
8622     ring = &rdev->ring[RADEON_RING_TYPE_GFX_INDEX];
8623     ring->ring_obj = NULL;
8624     r600_ring_init(rdev, ring, 1024 * 1024);
8625 
8626     ring = &rdev->ring[CAYMAN_RING_TYPE_CP1_INDEX];
8627     ring->ring_obj = NULL;
8628     r600_ring_init(rdev, ring, 1024 * 1024);
8629     r = radeon_doorbell_get(rdev, &ring->doorbell_index);
8630     if (r)
8631         return r;
8632 
8633     ring = &rdev->ring[CAYMAN_RING_TYPE_CP2_INDEX];
8634     ring->ring_obj = NULL;
8635     r600_ring_init(rdev, ring, 1024 * 1024);
8636     r = radeon_doorbell_get(rdev, &ring->doorbell_index);
8637     if (r)
8638         return r;
8639 
8640     ring = &rdev->ring[R600_RING_TYPE_DMA_INDEX];
8641     ring->ring_obj = NULL;
8642     r600_ring_init(rdev, ring, 256 * 1024);
8643 
8644     ring = &rdev->ring[CAYMAN_RING_TYPE_DMA1_INDEX];
8645     ring->ring_obj = NULL;
8646     r600_ring_init(rdev, ring, 256 * 1024);
8647 
8648     cik_uvd_init(rdev);
8649     cik_vce_init(rdev);
8650 
8651     rdev->ih.ring_obj = NULL;
8652     r600_ih_ring_init(rdev, 64 * 1024);
8653 
8654     r = r600_pcie_gart_init(rdev);
8655     if (r)
8656         return r;
8657 
8658     rdev->accel_working = true;
8659     r = cik_startup(rdev);
8660     if (r) {
8661         dev_err(rdev->dev, "disabling GPU acceleration\n");
8662         cik_cp_fini(rdev);
8663         cik_sdma_fini(rdev);
8664         cik_irq_fini(rdev);
8665         sumo_rlc_fini(rdev);
8666         cik_mec_fini(rdev);
8667         radeon_wb_fini(rdev);
8668         radeon_ib_pool_fini(rdev);
8669         radeon_vm_manager_fini(rdev);
8670         radeon_irq_kms_fini(rdev);
8671         cik_pcie_gart_fini(rdev);
8672         rdev->accel_working = false;
8673     }
8674 
8675     /* Don't start up if the MC ucode is missing.
8676      * The default clocks and voltages before the MC ucode
8677      * is loaded are not suffient for advanced operations.
8678      */
8679     if (!rdev->mc_fw && !(rdev->flags & RADEON_IS_IGP)) {
8680         DRM_ERROR("radeon: MC ucode required for NI+.\n");
8681         return -EINVAL;
8682     }
8683 
8684     return 0;
8685 }
8686 
8687 /**
8688  * cik_fini - asic specific driver and hw fini
8689  *
8690  * @rdev: radeon_device pointer
8691  *
8692  * Tear down the asic specific driver variables and program the hw
8693  * to an idle state (CIK).
8694  * Called at driver unload.
8695  */
8696 void cik_fini(struct radeon_device *rdev)
8697 {
8698     radeon_pm_fini(rdev);
8699     cik_cp_fini(rdev);
8700     cik_sdma_fini(rdev);
8701     cik_fini_pg(rdev);
8702     cik_fini_cg(rdev);
8703     cik_irq_fini(rdev);
8704     sumo_rlc_fini(rdev);
8705     cik_mec_fini(rdev);
8706     radeon_wb_fini(rdev);
8707     radeon_vm_manager_fini(rdev);
8708     radeon_ib_pool_fini(rdev);
8709     radeon_irq_kms_fini(rdev);
8710     uvd_v1_0_fini(rdev);
8711     radeon_uvd_fini(rdev);
8712     radeon_vce_fini(rdev);
8713     cik_pcie_gart_fini(rdev);
8714     r600_vram_scratch_fini(rdev);
8715     radeon_gem_fini(rdev);
8716     radeon_fence_driver_fini(rdev);
8717     radeon_bo_fini(rdev);
8718     radeon_atombios_fini(rdev);
8719     kfree(rdev->bios);
8720     rdev->bios = NULL;
8721 }
8722 
8723 void dce8_program_fmt(struct drm_encoder *encoder)
8724 {
8725     struct drm_device *dev = encoder->dev;
8726     struct radeon_device *rdev = dev->dev_private;
8727     struct radeon_encoder *radeon_encoder = to_radeon_encoder(encoder);
8728     struct radeon_crtc *radeon_crtc = to_radeon_crtc(encoder->crtc);
8729     struct drm_connector *connector = radeon_get_connector_for_encoder(encoder);
8730     int bpc = 0;
8731     u32 tmp = 0;
8732     enum radeon_connector_dither dither = RADEON_FMT_DITHER_DISABLE;
8733 
8734     if (connector) {
8735         struct radeon_connector *radeon_connector = to_radeon_connector(connector);
8736         bpc = radeon_get_monitor_bpc(connector);
8737         dither = radeon_connector->dither;
8738     }
8739 
8740     /* LVDS/eDP FMT is set up by atom */
8741     if (radeon_encoder->devices & ATOM_DEVICE_LCD_SUPPORT)
8742         return;
8743 
8744     /* not needed for analog */
8745     if ((radeon_encoder->encoder_id == ENCODER_OBJECT_ID_INTERNAL_KLDSCP_DAC1) ||
8746         (radeon_encoder->encoder_id == ENCODER_OBJECT_ID_INTERNAL_KLDSCP_DAC2))
8747         return;
8748 
8749     if (bpc == 0)
8750         return;
8751 
8752     switch (bpc) {
8753     case 6:
8754         if (dither == RADEON_FMT_DITHER_ENABLE)
8755             /* XXX sort out optimal dither settings */
8756             tmp |= (FMT_FRAME_RANDOM_ENABLE | FMT_HIGHPASS_RANDOM_ENABLE |
8757                 FMT_SPATIAL_DITHER_EN | FMT_SPATIAL_DITHER_DEPTH(0));
8758         else
8759             tmp |= (FMT_TRUNCATE_EN | FMT_TRUNCATE_DEPTH(0));
8760         break;
8761     case 8:
8762         if (dither == RADEON_FMT_DITHER_ENABLE)
8763             /* XXX sort out optimal dither settings */
8764             tmp |= (FMT_FRAME_RANDOM_ENABLE | FMT_HIGHPASS_RANDOM_ENABLE |
8765                 FMT_RGB_RANDOM_ENABLE |
8766                 FMT_SPATIAL_DITHER_EN | FMT_SPATIAL_DITHER_DEPTH(1));
8767         else
8768             tmp |= (FMT_TRUNCATE_EN | FMT_TRUNCATE_DEPTH(1));
8769         break;
8770     case 10:
8771         if (dither == RADEON_FMT_DITHER_ENABLE)
8772             /* XXX sort out optimal dither settings */
8773             tmp |= (FMT_FRAME_RANDOM_ENABLE | FMT_HIGHPASS_RANDOM_ENABLE |
8774                 FMT_RGB_RANDOM_ENABLE |
8775                 FMT_SPATIAL_DITHER_EN | FMT_SPATIAL_DITHER_DEPTH(2));
8776         else
8777             tmp |= (FMT_TRUNCATE_EN | FMT_TRUNCATE_DEPTH(2));
8778         break;
8779     default:
8780         /* not needed */
8781         break;
8782     }
8783 
8784     WREG32(FMT_BIT_DEPTH_CONTROL + radeon_crtc->crtc_offset, tmp);
8785 }
8786 
8787 /* display watermark setup */
8788 /**
8789  * dce8_line_buffer_adjust - Set up the line buffer
8790  *
8791  * @rdev: radeon_device pointer
8792  * @radeon_crtc: the selected display controller
8793  * @mode: the current display mode on the selected display
8794  * controller
8795  *
8796  * Setup up the line buffer allocation for
8797  * the selected display controller (CIK).
8798  * Returns the line buffer size in pixels.
8799  */
8800 static u32 dce8_line_buffer_adjust(struct radeon_device *rdev,
8801                    struct radeon_crtc *radeon_crtc,
8802                    struct drm_display_mode *mode)
8803 {
8804     u32 tmp, buffer_alloc, i;
8805     u32 pipe_offset = radeon_crtc->crtc_id * 0x20;
8806     /*
8807      * Line Buffer Setup
8808      * There are 6 line buffers, one for each display controllers.
8809      * There are 3 partitions per LB. Select the number of partitions
8810      * to enable based on the display width.  For display widths larger
8811      * than 4096, you need use to use 2 display controllers and combine
8812      * them using the stereo blender.
8813      */
8814     if (radeon_crtc->base.enabled && mode) {
8815         if (mode->crtc_hdisplay < 1920) {
8816             tmp = 1;
8817             buffer_alloc = 2;
8818         } else if (mode->crtc_hdisplay < 2560) {
8819             tmp = 2;
8820             buffer_alloc = 2;
8821         } else if (mode->crtc_hdisplay < 4096) {
8822             tmp = 0;
8823             buffer_alloc = (rdev->flags & RADEON_IS_IGP) ? 2 : 4;
8824         } else {
8825             DRM_DEBUG_KMS("Mode too big for LB!\n");
8826             tmp = 0;
8827             buffer_alloc = (rdev->flags & RADEON_IS_IGP) ? 2 : 4;
8828         }
8829     } else {
8830         tmp = 1;
8831         buffer_alloc = 0;
8832     }
8833 
8834     WREG32(LB_MEMORY_CTRL + radeon_crtc->crtc_offset,
8835            LB_MEMORY_CONFIG(tmp) | LB_MEMORY_SIZE(0x6B0));
8836 
8837     WREG32(PIPE0_DMIF_BUFFER_CONTROL + pipe_offset,
8838            DMIF_BUFFERS_ALLOCATED(buffer_alloc));
8839     for (i = 0; i < rdev->usec_timeout; i++) {
8840         if (RREG32(PIPE0_DMIF_BUFFER_CONTROL + pipe_offset) &
8841             DMIF_BUFFERS_ALLOCATED_COMPLETED)
8842             break;
8843         udelay(1);
8844     }
8845 
8846     if (radeon_crtc->base.enabled && mode) {
8847         switch (tmp) {
8848         case 0:
8849         default:
8850             return 4096 * 2;
8851         case 1:
8852             return 1920 * 2;
8853         case 2:
8854             return 2560 * 2;
8855         }
8856     }
8857 
8858     /* controller not enabled, so no lb used */
8859     return 0;
8860 }
8861 
8862 /**
8863  * cik_get_number_of_dram_channels - get the number of dram channels
8864  *
8865  * @rdev: radeon_device pointer
8866  *
8867  * Look up the number of video ram channels (CIK).
8868  * Used for display watermark bandwidth calculations
8869  * Returns the number of dram channels
8870  */
8871 static u32 cik_get_number_of_dram_channels(struct radeon_device *rdev)
8872 {
8873     u32 tmp = RREG32(MC_SHARED_CHMAP);
8874 
8875     switch ((tmp & NOOFCHAN_MASK) >> NOOFCHAN_SHIFT) {
8876     case 0:
8877     default:
8878         return 1;
8879     case 1:
8880         return 2;
8881     case 2:
8882         return 4;
8883     case 3:
8884         return 8;
8885     case 4:
8886         return 3;
8887     case 5:
8888         return 6;
8889     case 6:
8890         return 10;
8891     case 7:
8892         return 12;
8893     case 8:
8894         return 16;
8895     }
8896 }
8897 
8898 struct dce8_wm_params {
8899     u32 dram_channels; /* number of dram channels */
8900     u32 yclk;          /* bandwidth per dram data pin in kHz */
8901     u32 sclk;          /* engine clock in kHz */
8902     u32 disp_clk;      /* display clock in kHz */
8903     u32 src_width;     /* viewport width */
8904     u32 active_time;   /* active display time in ns */
8905     u32 blank_time;    /* blank time in ns */
8906     bool interlaced;    /* mode is interlaced */
8907     fixed20_12 vsc;    /* vertical scale ratio */
8908     u32 num_heads;     /* number of active crtcs */
8909     u32 bytes_per_pixel; /* bytes per pixel display + overlay */
8910     u32 lb_size;       /* line buffer allocated to pipe */
8911     u32 vtaps;         /* vertical scaler taps */
8912 };
8913 
8914 /**
8915  * dce8_dram_bandwidth - get the dram bandwidth
8916  *
8917  * @wm: watermark calculation data
8918  *
8919  * Calculate the raw dram bandwidth (CIK).
8920  * Used for display watermark bandwidth calculations
8921  * Returns the dram bandwidth in MBytes/s
8922  */
8923 static u32 dce8_dram_bandwidth(struct dce8_wm_params *wm)
8924 {
8925     /* Calculate raw DRAM Bandwidth */
8926     fixed20_12 dram_efficiency; /* 0.7 */
8927     fixed20_12 yclk, dram_channels, bandwidth;
8928     fixed20_12 a;
8929 
8930     a.full = dfixed_const(1000);
8931     yclk.full = dfixed_const(wm->yclk);
8932     yclk.full = dfixed_div(yclk, a);
8933     dram_channels.full = dfixed_const(wm->dram_channels * 4);
8934     a.full = dfixed_const(10);
8935     dram_efficiency.full = dfixed_const(7);
8936     dram_efficiency.full = dfixed_div(dram_efficiency, a);
8937     bandwidth.full = dfixed_mul(dram_channels, yclk);
8938     bandwidth.full = dfixed_mul(bandwidth, dram_efficiency);
8939 
8940     return dfixed_trunc(bandwidth);
8941 }
8942 
8943 /**
8944  * dce8_dram_bandwidth_for_display - get the dram bandwidth for display
8945  *
8946  * @wm: watermark calculation data
8947  *
8948  * Calculate the dram bandwidth used for display (CIK).
8949  * Used for display watermark bandwidth calculations
8950  * Returns the dram bandwidth for display in MBytes/s
8951  */
8952 static u32 dce8_dram_bandwidth_for_display(struct dce8_wm_params *wm)
8953 {
8954     /* Calculate DRAM Bandwidth and the part allocated to display. */
8955     fixed20_12 disp_dram_allocation; /* 0.3 to 0.7 */
8956     fixed20_12 yclk, dram_channels, bandwidth;
8957     fixed20_12 a;
8958 
8959     a.full = dfixed_const(1000);
8960     yclk.full = dfixed_const(wm->yclk);
8961     yclk.full = dfixed_div(yclk, a);
8962     dram_channels.full = dfixed_const(wm->dram_channels * 4);
8963     a.full = dfixed_const(10);
8964     disp_dram_allocation.full = dfixed_const(3); /* XXX worse case value 0.3 */
8965     disp_dram_allocation.full = dfixed_div(disp_dram_allocation, a);
8966     bandwidth.full = dfixed_mul(dram_channels, yclk);
8967     bandwidth.full = dfixed_mul(bandwidth, disp_dram_allocation);
8968 
8969     return dfixed_trunc(bandwidth);
8970 }
8971 
8972 /**
8973  * dce8_data_return_bandwidth - get the data return bandwidth
8974  *
8975  * @wm: watermark calculation data
8976  *
8977  * Calculate the data return bandwidth used for display (CIK).
8978  * Used for display watermark bandwidth calculations
8979  * Returns the data return bandwidth in MBytes/s
8980  */
8981 static u32 dce8_data_return_bandwidth(struct dce8_wm_params *wm)
8982 {
8983     /* Calculate the display Data return Bandwidth */
8984     fixed20_12 return_efficiency; /* 0.8 */
8985     fixed20_12 sclk, bandwidth;
8986     fixed20_12 a;
8987 
8988     a.full = dfixed_const(1000);
8989     sclk.full = dfixed_const(wm->sclk);
8990     sclk.full = dfixed_div(sclk, a);
8991     a.full = dfixed_const(10);
8992     return_efficiency.full = dfixed_const(8);
8993     return_efficiency.full = dfixed_div(return_efficiency, a);
8994     a.full = dfixed_const(32);
8995     bandwidth.full = dfixed_mul(a, sclk);
8996     bandwidth.full = dfixed_mul(bandwidth, return_efficiency);
8997 
8998     return dfixed_trunc(bandwidth);
8999 }
9000 
9001 /**
9002  * dce8_dmif_request_bandwidth - get the dmif bandwidth
9003  *
9004  * @wm: watermark calculation data
9005  *
9006  * Calculate the dmif bandwidth used for display (CIK).
9007  * Used for display watermark bandwidth calculations
9008  * Returns the dmif bandwidth in MBytes/s
9009  */
9010 static u32 dce8_dmif_request_bandwidth(struct dce8_wm_params *wm)
9011 {
9012     /* Calculate the DMIF Request Bandwidth */
9013     fixed20_12 disp_clk_request_efficiency; /* 0.8 */
9014     fixed20_12 disp_clk, bandwidth;
9015     fixed20_12 a, b;
9016 
9017     a.full = dfixed_const(1000);
9018     disp_clk.full = dfixed_const(wm->disp_clk);
9019     disp_clk.full = dfixed_div(disp_clk, a);
9020     a.full = dfixed_const(32);
9021     b.full = dfixed_mul(a, disp_clk);
9022 
9023     a.full = dfixed_const(10);
9024     disp_clk_request_efficiency.full = dfixed_const(8);
9025     disp_clk_request_efficiency.full = dfixed_div(disp_clk_request_efficiency, a);
9026 
9027     bandwidth.full = dfixed_mul(b, disp_clk_request_efficiency);
9028 
9029     return dfixed_trunc(bandwidth);
9030 }
9031 
9032 /**
9033  * dce8_available_bandwidth - get the min available bandwidth
9034  *
9035  * @wm: watermark calculation data
9036  *
9037  * Calculate the min available bandwidth used for display (CIK).
9038  * Used for display watermark bandwidth calculations
9039  * Returns the min available bandwidth in MBytes/s
9040  */
9041 static u32 dce8_available_bandwidth(struct dce8_wm_params *wm)
9042 {
9043     /* Calculate the Available bandwidth. Display can use this temporarily but not in average. */
9044     u32 dram_bandwidth = dce8_dram_bandwidth(wm);
9045     u32 data_return_bandwidth = dce8_data_return_bandwidth(wm);
9046     u32 dmif_req_bandwidth = dce8_dmif_request_bandwidth(wm);
9047 
9048     return min(dram_bandwidth, min(data_return_bandwidth, dmif_req_bandwidth));
9049 }
9050 
9051 /**
9052  * dce8_average_bandwidth - get the average available bandwidth
9053  *
9054  * @wm: watermark calculation data
9055  *
9056  * Calculate the average available bandwidth used for display (CIK).
9057  * Used for display watermark bandwidth calculations
9058  * Returns the average available bandwidth in MBytes/s
9059  */
9060 static u32 dce8_average_bandwidth(struct dce8_wm_params *wm)
9061 {
9062     /* Calculate the display mode Average Bandwidth
9063      * DisplayMode should contain the source and destination dimensions,
9064      * timing, etc.
9065      */
9066     fixed20_12 bpp;
9067     fixed20_12 line_time;
9068     fixed20_12 src_width;
9069     fixed20_12 bandwidth;
9070     fixed20_12 a;
9071 
9072     a.full = dfixed_const(1000);
9073     line_time.full = dfixed_const(wm->active_time + wm->blank_time);
9074     line_time.full = dfixed_div(line_time, a);
9075     bpp.full = dfixed_const(wm->bytes_per_pixel);
9076     src_width.full = dfixed_const(wm->src_width);
9077     bandwidth.full = dfixed_mul(src_width, bpp);
9078     bandwidth.full = dfixed_mul(bandwidth, wm->vsc);
9079     bandwidth.full = dfixed_div(bandwidth, line_time);
9080 
9081     return dfixed_trunc(bandwidth);
9082 }
9083 
9084 /**
9085  * dce8_latency_watermark - get the latency watermark
9086  *
9087  * @wm: watermark calculation data
9088  *
9089  * Calculate the latency watermark (CIK).
9090  * Used for display watermark bandwidth calculations
9091  * Returns the latency watermark in ns
9092  */
9093 static u32 dce8_latency_watermark(struct dce8_wm_params *wm)
9094 {
9095     /* First calculate the latency in ns */
9096     u32 mc_latency = 2000; /* 2000 ns. */
9097     u32 available_bandwidth = dce8_available_bandwidth(wm);
9098     u32 worst_chunk_return_time = (512 * 8 * 1000) / available_bandwidth;
9099     u32 cursor_line_pair_return_time = (128 * 4 * 1000) / available_bandwidth;
9100     u32 dc_latency = 40000000 / wm->disp_clk; /* dc pipe latency */
9101     u32 other_heads_data_return_time = ((wm->num_heads + 1) * worst_chunk_return_time) +
9102         (wm->num_heads * cursor_line_pair_return_time);
9103     u32 latency = mc_latency + other_heads_data_return_time + dc_latency;
9104     u32 max_src_lines_per_dst_line, lb_fill_bw, line_fill_time;
9105     u32 tmp, dmif_size = 12288;
9106     fixed20_12 a, b, c;
9107 
9108     if (wm->num_heads == 0)
9109         return 0;
9110 
9111     a.full = dfixed_const(2);
9112     b.full = dfixed_const(1);
9113     if ((wm->vsc.full > a.full) ||
9114         ((wm->vsc.full > b.full) && (wm->vtaps >= 3)) ||
9115         (wm->vtaps >= 5) ||
9116         ((wm->vsc.full >= a.full) && wm->interlaced))
9117         max_src_lines_per_dst_line = 4;
9118     else
9119         max_src_lines_per_dst_line = 2;
9120 
9121     a.full = dfixed_const(available_bandwidth);
9122     b.full = dfixed_const(wm->num_heads);
9123     a.full = dfixed_div(a, b);
9124     tmp = div_u64((u64) dmif_size * (u64) wm->disp_clk, mc_latency + 512);
9125     tmp = min(dfixed_trunc(a), tmp);
9126 
9127     lb_fill_bw = min(tmp, wm->disp_clk * wm->bytes_per_pixel / 1000);
9128 
9129     a.full = dfixed_const(max_src_lines_per_dst_line * wm->src_width * wm->bytes_per_pixel);
9130     b.full = dfixed_const(1000);
9131     c.full = dfixed_const(lb_fill_bw);
9132     b.full = dfixed_div(c, b);
9133     a.full = dfixed_div(a, b);
9134     line_fill_time = dfixed_trunc(a);
9135 
9136     if (line_fill_time < wm->active_time)
9137         return latency;
9138     else
9139         return latency + (line_fill_time - wm->active_time);
9140 
9141 }
9142 
9143 /**
9144  * dce8_average_bandwidth_vs_dram_bandwidth_for_display - check
9145  * average and available dram bandwidth
9146  *
9147  * @wm: watermark calculation data
9148  *
9149  * Check if the display average bandwidth fits in the display
9150  * dram bandwidth (CIK).
9151  * Used for display watermark bandwidth calculations
9152  * Returns true if the display fits, false if not.
9153  */
9154 static bool dce8_average_bandwidth_vs_dram_bandwidth_for_display(struct dce8_wm_params *wm)
9155 {
9156     if (dce8_average_bandwidth(wm) <=
9157         (dce8_dram_bandwidth_for_display(wm) / wm->num_heads))
9158         return true;
9159     else
9160         return false;
9161 }
9162 
9163 /**
9164  * dce8_average_bandwidth_vs_available_bandwidth - check
9165  * average and available bandwidth
9166  *
9167  * @wm: watermark calculation data
9168  *
9169  * Check if the display average bandwidth fits in the display
9170  * available bandwidth (CIK).
9171  * Used for display watermark bandwidth calculations
9172  * Returns true if the display fits, false if not.
9173  */
9174 static bool dce8_average_bandwidth_vs_available_bandwidth(struct dce8_wm_params *wm)
9175 {
9176     if (dce8_average_bandwidth(wm) <=
9177         (dce8_available_bandwidth(wm) / wm->num_heads))
9178         return true;
9179     else
9180         return false;
9181 }
9182 
9183 /**
9184  * dce8_check_latency_hiding - check latency hiding
9185  *
9186  * @wm: watermark calculation data
9187  *
9188  * Check latency hiding (CIK).
9189  * Used for display watermark bandwidth calculations
9190  * Returns true if the display fits, false if not.
9191  */
9192 static bool dce8_check_latency_hiding(struct dce8_wm_params *wm)
9193 {
9194     u32 lb_partitions = wm->lb_size / wm->src_width;
9195     u32 line_time = wm->active_time + wm->blank_time;
9196     u32 latency_tolerant_lines;
9197     u32 latency_hiding;
9198     fixed20_12 a;
9199 
9200     a.full = dfixed_const(1);
9201     if (wm->vsc.full > a.full)
9202         latency_tolerant_lines = 1;
9203     else {
9204         if (lb_partitions <= (wm->vtaps + 1))
9205             latency_tolerant_lines = 1;
9206         else
9207             latency_tolerant_lines = 2;
9208     }
9209 
9210     latency_hiding = (latency_tolerant_lines * line_time + wm->blank_time);
9211 
9212     if (dce8_latency_watermark(wm) <= latency_hiding)
9213         return true;
9214     else
9215         return false;
9216 }
9217 
9218 /**
9219  * dce8_program_watermarks - program display watermarks
9220  *
9221  * @rdev: radeon_device pointer
9222  * @radeon_crtc: the selected display controller
9223  * @lb_size: line buffer size
9224  * @num_heads: number of display controllers in use
9225  *
9226  * Calculate and program the display watermarks for the
9227  * selected display controller (CIK).
9228  */
9229 static void dce8_program_watermarks(struct radeon_device *rdev,
9230                     struct radeon_crtc *radeon_crtc,
9231                     u32 lb_size, u32 num_heads)
9232 {
9233     struct drm_display_mode *mode = &radeon_crtc->base.mode;
9234     struct dce8_wm_params wm_low, wm_high;
9235     u32 active_time;
9236     u32 line_time = 0;
9237     u32 latency_watermark_a = 0, latency_watermark_b = 0;
9238     u32 tmp, wm_mask;
9239 
9240     if (radeon_crtc->base.enabled && num_heads && mode) {
9241         active_time = (u32) div_u64((u64)mode->crtc_hdisplay * 1000000,
9242                         (u32)mode->clock);
9243         line_time = (u32) div_u64((u64)mode->crtc_htotal * 1000000,
9244                       (u32)mode->clock);
9245         line_time = min(line_time, (u32)65535);
9246 
9247         /* watermark for high clocks */
9248         if ((rdev->pm.pm_method == PM_METHOD_DPM) &&
9249             rdev->pm.dpm_enabled) {
9250             wm_high.yclk =
9251                 radeon_dpm_get_mclk(rdev, false) * 10;
9252             wm_high.sclk =
9253                 radeon_dpm_get_sclk(rdev, false) * 10;
9254         } else {
9255             wm_high.yclk = rdev->pm.current_mclk * 10;
9256             wm_high.sclk = rdev->pm.current_sclk * 10;
9257         }
9258 
9259         wm_high.disp_clk = mode->clock;
9260         wm_high.src_width = mode->crtc_hdisplay;
9261         wm_high.active_time = active_time;
9262         wm_high.blank_time = line_time - wm_high.active_time;
9263         wm_high.interlaced = false;
9264         if (mode->flags & DRM_MODE_FLAG_INTERLACE)
9265             wm_high.interlaced = true;
9266         wm_high.vsc = radeon_crtc->vsc;
9267         wm_high.vtaps = 1;
9268         if (radeon_crtc->rmx_type != RMX_OFF)
9269             wm_high.vtaps = 2;
9270         wm_high.bytes_per_pixel = 4; /* XXX: get this from fb config */
9271         wm_high.lb_size = lb_size;
9272         wm_high.dram_channels = cik_get_number_of_dram_channels(rdev);
9273         wm_high.num_heads = num_heads;
9274 
9275         /* set for high clocks */
9276         latency_watermark_a = min(dce8_latency_watermark(&wm_high), (u32)65535);
9277 
9278         /* possibly force display priority to high */
9279         /* should really do this at mode validation time... */
9280         if (!dce8_average_bandwidth_vs_dram_bandwidth_for_display(&wm_high) ||
9281             !dce8_average_bandwidth_vs_available_bandwidth(&wm_high) ||
9282             !dce8_check_latency_hiding(&wm_high) ||
9283             (rdev->disp_priority == 2)) {
9284             DRM_DEBUG_KMS("force priority to high\n");
9285         }
9286 
9287         /* watermark for low clocks */
9288         if ((rdev->pm.pm_method == PM_METHOD_DPM) &&
9289             rdev->pm.dpm_enabled) {
9290             wm_low.yclk =
9291                 radeon_dpm_get_mclk(rdev, true) * 10;
9292             wm_low.sclk =
9293                 radeon_dpm_get_sclk(rdev, true) * 10;
9294         } else {
9295             wm_low.yclk = rdev->pm.current_mclk * 10;
9296             wm_low.sclk = rdev->pm.current_sclk * 10;
9297         }
9298 
9299         wm_low.disp_clk = mode->clock;
9300         wm_low.src_width = mode->crtc_hdisplay;
9301         wm_low.active_time = active_time;
9302         wm_low.blank_time = line_time - wm_low.active_time;
9303         wm_low.interlaced = false;
9304         if (mode->flags & DRM_MODE_FLAG_INTERLACE)
9305             wm_low.interlaced = true;
9306         wm_low.vsc = radeon_crtc->vsc;
9307         wm_low.vtaps = 1;
9308         if (radeon_crtc->rmx_type != RMX_OFF)
9309             wm_low.vtaps = 2;
9310         wm_low.bytes_per_pixel = 4; /* XXX: get this from fb config */
9311         wm_low.lb_size = lb_size;
9312         wm_low.dram_channels = cik_get_number_of_dram_channels(rdev);
9313         wm_low.num_heads = num_heads;
9314 
9315         /* set for low clocks */
9316         latency_watermark_b = min(dce8_latency_watermark(&wm_low), (u32)65535);
9317 
9318         /* possibly force display priority to high */
9319         /* should really do this at mode validation time... */
9320         if (!dce8_average_bandwidth_vs_dram_bandwidth_for_display(&wm_low) ||
9321             !dce8_average_bandwidth_vs_available_bandwidth(&wm_low) ||
9322             !dce8_check_latency_hiding(&wm_low) ||
9323             (rdev->disp_priority == 2)) {
9324             DRM_DEBUG_KMS("force priority to high\n");
9325         }
9326 
9327         /* Save number of lines the linebuffer leads before the scanout */
9328         radeon_crtc->lb_vblank_lead_lines = DIV_ROUND_UP(lb_size, mode->crtc_hdisplay);
9329     }
9330 
9331     /* select wm A */
9332     wm_mask = RREG32(DPG_WATERMARK_MASK_CONTROL + radeon_crtc->crtc_offset);
9333     tmp = wm_mask;
9334     tmp &= ~LATENCY_WATERMARK_MASK(3);
9335     tmp |= LATENCY_WATERMARK_MASK(1);
9336     WREG32(DPG_WATERMARK_MASK_CONTROL + radeon_crtc->crtc_offset, tmp);
9337     WREG32(DPG_PIPE_LATENCY_CONTROL + radeon_crtc->crtc_offset,
9338            (LATENCY_LOW_WATERMARK(latency_watermark_a) |
9339         LATENCY_HIGH_WATERMARK(line_time)));
9340     /* select wm B */
9341     tmp = RREG32(DPG_WATERMARK_MASK_CONTROL + radeon_crtc->crtc_offset);
9342     tmp &= ~LATENCY_WATERMARK_MASK(3);
9343     tmp |= LATENCY_WATERMARK_MASK(2);
9344     WREG32(DPG_WATERMARK_MASK_CONTROL + radeon_crtc->crtc_offset, tmp);
9345     WREG32(DPG_PIPE_LATENCY_CONTROL + radeon_crtc->crtc_offset,
9346            (LATENCY_LOW_WATERMARK(latency_watermark_b) |
9347         LATENCY_HIGH_WATERMARK(line_time)));
9348     /* restore original selection */
9349     WREG32(DPG_WATERMARK_MASK_CONTROL + radeon_crtc->crtc_offset, wm_mask);
9350 
9351     /* save values for DPM */
9352     radeon_crtc->line_time = line_time;
9353     radeon_crtc->wm_high = latency_watermark_a;
9354     radeon_crtc->wm_low = latency_watermark_b;
9355 }
9356 
9357 /**
9358  * dce8_bandwidth_update - program display watermarks
9359  *
9360  * @rdev: radeon_device pointer
9361  *
9362  * Calculate and program the display watermarks and line
9363  * buffer allocation (CIK).
9364  */
9365 void dce8_bandwidth_update(struct radeon_device *rdev)
9366 {
9367     struct drm_display_mode *mode = NULL;
9368     u32 num_heads = 0, lb_size;
9369     int i;
9370 
9371     if (!rdev->mode_info.mode_config_initialized)
9372         return;
9373 
9374     radeon_update_display_priority(rdev);
9375 
9376     for (i = 0; i < rdev->num_crtc; i++) {
9377         if (rdev->mode_info.crtcs[i]->base.enabled)
9378             num_heads++;
9379     }
9380     for (i = 0; i < rdev->num_crtc; i++) {
9381         mode = &rdev->mode_info.crtcs[i]->base.mode;
9382         lb_size = dce8_line_buffer_adjust(rdev, rdev->mode_info.crtcs[i], mode);
9383         dce8_program_watermarks(rdev, rdev->mode_info.crtcs[i], lb_size, num_heads);
9384     }
9385 }
9386 
9387 /**
9388  * cik_get_gpu_clock_counter - return GPU clock counter snapshot
9389  *
9390  * @rdev: radeon_device pointer
9391  *
9392  * Fetches a GPU clock counter snapshot (SI).
9393  * Returns the 64 bit clock counter snapshot.
9394  */
9395 uint64_t cik_get_gpu_clock_counter(struct radeon_device *rdev)
9396 {
9397     uint64_t clock;
9398 
9399     mutex_lock(&rdev->gpu_clock_mutex);
9400     WREG32(RLC_CAPTURE_GPU_CLOCK_COUNT, 1);
9401     clock = (uint64_t)RREG32(RLC_GPU_CLOCK_COUNT_LSB) |
9402         ((uint64_t)RREG32(RLC_GPU_CLOCK_COUNT_MSB) << 32ULL);
9403     mutex_unlock(&rdev->gpu_clock_mutex);
9404     return clock;
9405 }
9406 
9407 static int cik_set_uvd_clock(struct radeon_device *rdev, u32 clock,
9408                  u32 cntl_reg, u32 status_reg)
9409 {
9410     int r, i;
9411     struct atom_clock_dividers dividers;
9412     uint32_t tmp;
9413 
9414     r = radeon_atom_get_clock_dividers(rdev, COMPUTE_GPUCLK_INPUT_FLAG_DEFAULT_GPUCLK,
9415                        clock, false, &dividers);
9416     if (r)
9417         return r;
9418 
9419     tmp = RREG32_SMC(cntl_reg);
9420     tmp &= ~(DCLK_DIR_CNTL_EN|DCLK_DIVIDER_MASK);
9421     tmp |= dividers.post_divider;
9422     WREG32_SMC(cntl_reg, tmp);
9423 
9424     for (i = 0; i < 100; i++) {
9425         if (RREG32_SMC(status_reg) & DCLK_STATUS)
9426             break;
9427         mdelay(10);
9428     }
9429     if (i == 100)
9430         return -ETIMEDOUT;
9431 
9432     return 0;
9433 }
9434 
9435 int cik_set_uvd_clocks(struct radeon_device *rdev, u32 vclk, u32 dclk)
9436 {
9437     int r = 0;
9438 
9439     r = cik_set_uvd_clock(rdev, vclk, CG_VCLK_CNTL, CG_VCLK_STATUS);
9440     if (r)
9441         return r;
9442 
9443     r = cik_set_uvd_clock(rdev, dclk, CG_DCLK_CNTL, CG_DCLK_STATUS);
9444     return r;
9445 }
9446 
9447 int cik_set_vce_clocks(struct radeon_device *rdev, u32 evclk, u32 ecclk)
9448 {
9449     int r, i;
9450     struct atom_clock_dividers dividers;
9451     u32 tmp;
9452 
9453     r = radeon_atom_get_clock_dividers(rdev, COMPUTE_GPUCLK_INPUT_FLAG_DEFAULT_GPUCLK,
9454                        ecclk, false, &dividers);
9455     if (r)
9456         return r;
9457 
9458     for (i = 0; i < 100; i++) {
9459         if (RREG32_SMC(CG_ECLK_STATUS) & ECLK_STATUS)
9460             break;
9461         mdelay(10);
9462     }
9463     if (i == 100)
9464         return -ETIMEDOUT;
9465 
9466     tmp = RREG32_SMC(CG_ECLK_CNTL);
9467     tmp &= ~(ECLK_DIR_CNTL_EN|ECLK_DIVIDER_MASK);
9468     tmp |= dividers.post_divider;
9469     WREG32_SMC(CG_ECLK_CNTL, tmp);
9470 
9471     for (i = 0; i < 100; i++) {
9472         if (RREG32_SMC(CG_ECLK_STATUS) & ECLK_STATUS)
9473             break;
9474         mdelay(10);
9475     }
9476     if (i == 100)
9477         return -ETIMEDOUT;
9478 
9479     return 0;
9480 }
9481 
9482 static void cik_pcie_gen3_enable(struct radeon_device *rdev)
9483 {
9484     struct pci_dev *root = rdev->pdev->bus->self;
9485     enum pci_bus_speed speed_cap;
9486     u32 speed_cntl, current_data_rate;
9487     int i;
9488     u16 tmp16;
9489 
9490     if (pci_is_root_bus(rdev->pdev->bus))
9491         return;
9492 
9493     if (radeon_pcie_gen2 == 0)
9494         return;
9495 
9496     if (rdev->flags & RADEON_IS_IGP)
9497         return;
9498 
9499     if (!(rdev->flags & RADEON_IS_PCIE))
9500         return;
9501 
9502     speed_cap = pcie_get_speed_cap(root);
9503     if (speed_cap == PCI_SPEED_UNKNOWN)
9504         return;
9505 
9506     if ((speed_cap != PCIE_SPEED_8_0GT) &&
9507         (speed_cap != PCIE_SPEED_5_0GT))
9508         return;
9509 
9510     speed_cntl = RREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL);
9511     current_data_rate = (speed_cntl & LC_CURRENT_DATA_RATE_MASK) >>
9512         LC_CURRENT_DATA_RATE_SHIFT;
9513     if (speed_cap == PCIE_SPEED_8_0GT) {
9514         if (current_data_rate == 2) {
9515             DRM_INFO("PCIE gen 3 link speeds already enabled\n");
9516             return;
9517         }
9518         DRM_INFO("enabling PCIE gen 3 link speeds, disable with radeon.pcie_gen2=0\n");
9519     } else if (speed_cap == PCIE_SPEED_5_0GT) {
9520         if (current_data_rate == 1) {
9521             DRM_INFO("PCIE gen 2 link speeds already enabled\n");
9522             return;
9523         }
9524         DRM_INFO("enabling PCIE gen 2 link speeds, disable with radeon.pcie_gen2=0\n");
9525     }
9526 
9527     if (!pci_is_pcie(root) || !pci_is_pcie(rdev->pdev))
9528         return;
9529 
9530     if (speed_cap == PCIE_SPEED_8_0GT) {
9531         /* re-try equalization if gen3 is not already enabled */
9532         if (current_data_rate != 2) {
9533             u16 bridge_cfg, gpu_cfg;
9534             u16 bridge_cfg2, gpu_cfg2;
9535             u32 max_lw, current_lw, tmp;
9536 
9537             pcie_capability_read_word(root, PCI_EXP_LNKCTL,
9538                           &bridge_cfg);
9539             pcie_capability_read_word(rdev->pdev, PCI_EXP_LNKCTL,
9540                           &gpu_cfg);
9541 
9542             tmp16 = bridge_cfg | PCI_EXP_LNKCTL_HAWD;
9543             pcie_capability_write_word(root, PCI_EXP_LNKCTL, tmp16);
9544 
9545             tmp16 = gpu_cfg | PCI_EXP_LNKCTL_HAWD;
9546             pcie_capability_write_word(rdev->pdev, PCI_EXP_LNKCTL,
9547                            tmp16);
9548 
9549             tmp = RREG32_PCIE_PORT(PCIE_LC_STATUS1);
9550             max_lw = (tmp & LC_DETECTED_LINK_WIDTH_MASK) >> LC_DETECTED_LINK_WIDTH_SHIFT;
9551             current_lw = (tmp & LC_OPERATING_LINK_WIDTH_MASK) >> LC_OPERATING_LINK_WIDTH_SHIFT;
9552 
9553             if (current_lw < max_lw) {
9554                 tmp = RREG32_PCIE_PORT(PCIE_LC_LINK_WIDTH_CNTL);
9555                 if (tmp & LC_RENEGOTIATION_SUPPORT) {
9556                     tmp &= ~(LC_LINK_WIDTH_MASK | LC_UPCONFIGURE_DIS);
9557                     tmp |= (max_lw << LC_LINK_WIDTH_SHIFT);
9558                     tmp |= LC_UPCONFIGURE_SUPPORT | LC_RENEGOTIATE_EN | LC_RECONFIG_NOW;
9559                     WREG32_PCIE_PORT(PCIE_LC_LINK_WIDTH_CNTL, tmp);
9560                 }
9561             }
9562 
9563             for (i = 0; i < 10; i++) {
9564                 /* check status */
9565                 pcie_capability_read_word(rdev->pdev,
9566                               PCI_EXP_DEVSTA,
9567                               &tmp16);
9568                 if (tmp16 & PCI_EXP_DEVSTA_TRPND)
9569                     break;
9570 
9571                 pcie_capability_read_word(root, PCI_EXP_LNKCTL,
9572                               &bridge_cfg);
9573                 pcie_capability_read_word(rdev->pdev,
9574                               PCI_EXP_LNKCTL,
9575                               &gpu_cfg);
9576 
9577                 pcie_capability_read_word(root, PCI_EXP_LNKCTL2,
9578                               &bridge_cfg2);
9579                 pcie_capability_read_word(rdev->pdev,
9580                               PCI_EXP_LNKCTL2,
9581                               &gpu_cfg2);
9582 
9583                 tmp = RREG32_PCIE_PORT(PCIE_LC_CNTL4);
9584                 tmp |= LC_SET_QUIESCE;
9585                 WREG32_PCIE_PORT(PCIE_LC_CNTL4, tmp);
9586 
9587                 tmp = RREG32_PCIE_PORT(PCIE_LC_CNTL4);
9588                 tmp |= LC_REDO_EQ;
9589                 WREG32_PCIE_PORT(PCIE_LC_CNTL4, tmp);
9590 
9591                 msleep(100);
9592 
9593                 /* linkctl */
9594                 pcie_capability_read_word(root, PCI_EXP_LNKCTL,
9595                               &tmp16);
9596                 tmp16 &= ~PCI_EXP_LNKCTL_HAWD;
9597                 tmp16 |= (bridge_cfg & PCI_EXP_LNKCTL_HAWD);
9598                 pcie_capability_write_word(root, PCI_EXP_LNKCTL,
9599                                tmp16);
9600 
9601                 pcie_capability_read_word(rdev->pdev,
9602                               PCI_EXP_LNKCTL,
9603                               &tmp16);
9604                 tmp16 &= ~PCI_EXP_LNKCTL_HAWD;
9605                 tmp16 |= (gpu_cfg & PCI_EXP_LNKCTL_HAWD);
9606                 pcie_capability_write_word(rdev->pdev,
9607                                PCI_EXP_LNKCTL,
9608                                tmp16);
9609 
9610                 /* linkctl2 */
9611                 pcie_capability_read_word(root, PCI_EXP_LNKCTL2,
9612                               &tmp16);
9613                 tmp16 &= ~(PCI_EXP_LNKCTL2_ENTER_COMP |
9614                        PCI_EXP_LNKCTL2_TX_MARGIN);
9615                 tmp16 |= (bridge_cfg2 &
9616                       (PCI_EXP_LNKCTL2_ENTER_COMP |
9617                        PCI_EXP_LNKCTL2_TX_MARGIN));
9618                 pcie_capability_write_word(root,
9619                                PCI_EXP_LNKCTL2,
9620                                tmp16);
9621 
9622                 pcie_capability_read_word(rdev->pdev,
9623                               PCI_EXP_LNKCTL2,
9624                               &tmp16);
9625                 tmp16 &= ~(PCI_EXP_LNKCTL2_ENTER_COMP |
9626                        PCI_EXP_LNKCTL2_TX_MARGIN);
9627                 tmp16 |= (gpu_cfg2 &
9628                       (PCI_EXP_LNKCTL2_ENTER_COMP |
9629                        PCI_EXP_LNKCTL2_TX_MARGIN));
9630                 pcie_capability_write_word(rdev->pdev,
9631                                PCI_EXP_LNKCTL2,
9632                                tmp16);
9633 
9634                 tmp = RREG32_PCIE_PORT(PCIE_LC_CNTL4);
9635                 tmp &= ~LC_SET_QUIESCE;
9636                 WREG32_PCIE_PORT(PCIE_LC_CNTL4, tmp);
9637             }
9638         }
9639     }
9640 
9641     /* set the link speed */
9642     speed_cntl |= LC_FORCE_EN_SW_SPEED_CHANGE | LC_FORCE_DIS_HW_SPEED_CHANGE;
9643     speed_cntl &= ~LC_FORCE_DIS_SW_SPEED_CHANGE;
9644     WREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL, speed_cntl);
9645 
9646     pcie_capability_read_word(rdev->pdev, PCI_EXP_LNKCTL2, &tmp16);
9647     tmp16 &= ~PCI_EXP_LNKCTL2_TLS;
9648     if (speed_cap == PCIE_SPEED_8_0GT)
9649         tmp16 |= PCI_EXP_LNKCTL2_TLS_8_0GT; /* gen3 */
9650     else if (speed_cap == PCIE_SPEED_5_0GT)
9651         tmp16 |= PCI_EXP_LNKCTL2_TLS_5_0GT; /* gen2 */
9652     else
9653         tmp16 |= PCI_EXP_LNKCTL2_TLS_2_5GT; /* gen1 */
9654     pcie_capability_write_word(rdev->pdev, PCI_EXP_LNKCTL2, tmp16);
9655 
9656     speed_cntl = RREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL);
9657     speed_cntl |= LC_INITIATE_LINK_SPEED_CHANGE;
9658     WREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL, speed_cntl);
9659 
9660     for (i = 0; i < rdev->usec_timeout; i++) {
9661         speed_cntl = RREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL);
9662         if ((speed_cntl & LC_INITIATE_LINK_SPEED_CHANGE) == 0)
9663             break;
9664         udelay(1);
9665     }
9666 }
9667 
9668 static void cik_program_aspm(struct radeon_device *rdev)
9669 {
9670     u32 data, orig;
9671     bool disable_l0s = false, disable_l1 = false, disable_plloff_in_l1 = false;
9672     bool disable_clkreq = false;
9673 
9674     if (radeon_aspm == 0)
9675         return;
9676 
9677     /* XXX double check IGPs */
9678     if (rdev->flags & RADEON_IS_IGP)
9679         return;
9680 
9681     if (!(rdev->flags & RADEON_IS_PCIE))
9682         return;
9683 
9684     orig = data = RREG32_PCIE_PORT(PCIE_LC_N_FTS_CNTL);
9685     data &= ~LC_XMIT_N_FTS_MASK;
9686     data |= LC_XMIT_N_FTS(0x24) | LC_XMIT_N_FTS_OVERRIDE_EN;
9687     if (orig != data)
9688         WREG32_PCIE_PORT(PCIE_LC_N_FTS_CNTL, data);
9689 
9690     orig = data = RREG32_PCIE_PORT(PCIE_LC_CNTL3);
9691     data |= LC_GO_TO_RECOVERY;
9692     if (orig != data)
9693         WREG32_PCIE_PORT(PCIE_LC_CNTL3, data);
9694 
9695     orig = data = RREG32_PCIE_PORT(PCIE_P_CNTL);
9696     data |= P_IGNORE_EDB_ERR;
9697     if (orig != data)
9698         WREG32_PCIE_PORT(PCIE_P_CNTL, data);
9699 
9700     orig = data = RREG32_PCIE_PORT(PCIE_LC_CNTL);
9701     data &= ~(LC_L0S_INACTIVITY_MASK | LC_L1_INACTIVITY_MASK);
9702     data |= LC_PMI_TO_L1_DIS;
9703     if (!disable_l0s)
9704         data |= LC_L0S_INACTIVITY(7);
9705 
9706     if (!disable_l1) {
9707         data |= LC_L1_INACTIVITY(7);
9708         data &= ~LC_PMI_TO_L1_DIS;
9709         if (orig != data)
9710             WREG32_PCIE_PORT(PCIE_LC_CNTL, data);
9711 
9712         if (!disable_plloff_in_l1) {
9713             bool clk_req_support;
9714 
9715             orig = data = RREG32_PCIE_PORT(PB0_PIF_PWRDOWN_0);
9716             data &= ~(PLL_POWER_STATE_IN_OFF_0_MASK | PLL_POWER_STATE_IN_TXS2_0_MASK);
9717             data |= PLL_POWER_STATE_IN_OFF_0(7) | PLL_POWER_STATE_IN_TXS2_0(7);
9718             if (orig != data)
9719                 WREG32_PCIE_PORT(PB0_PIF_PWRDOWN_0, data);
9720 
9721             orig = data = RREG32_PCIE_PORT(PB0_PIF_PWRDOWN_1);
9722             data &= ~(PLL_POWER_STATE_IN_OFF_1_MASK | PLL_POWER_STATE_IN_TXS2_1_MASK);
9723             data |= PLL_POWER_STATE_IN_OFF_1(7) | PLL_POWER_STATE_IN_TXS2_1(7);
9724             if (orig != data)
9725                 WREG32_PCIE_PORT(PB0_PIF_PWRDOWN_1, data);
9726 
9727             orig = data = RREG32_PCIE_PORT(PB1_PIF_PWRDOWN_0);
9728             data &= ~(PLL_POWER_STATE_IN_OFF_0_MASK | PLL_POWER_STATE_IN_TXS2_0_MASK);
9729             data |= PLL_POWER_STATE_IN_OFF_0(7) | PLL_POWER_STATE_IN_TXS2_0(7);
9730             if (orig != data)
9731                 WREG32_PCIE_PORT(PB1_PIF_PWRDOWN_0, data);
9732 
9733             orig = data = RREG32_PCIE_PORT(PB1_PIF_PWRDOWN_1);
9734             data &= ~(PLL_POWER_STATE_IN_OFF_1_MASK | PLL_POWER_STATE_IN_TXS2_1_MASK);
9735             data |= PLL_POWER_STATE_IN_OFF_1(7) | PLL_POWER_STATE_IN_TXS2_1(7);
9736             if (orig != data)
9737                 WREG32_PCIE_PORT(PB1_PIF_PWRDOWN_1, data);
9738 
9739             orig = data = RREG32_PCIE_PORT(PCIE_LC_LINK_WIDTH_CNTL);
9740             data &= ~LC_DYN_LANES_PWR_STATE_MASK;
9741             data |= LC_DYN_LANES_PWR_STATE(3);
9742             if (orig != data)
9743                 WREG32_PCIE_PORT(PCIE_LC_LINK_WIDTH_CNTL, data);
9744 
9745             if (!disable_clkreq &&
9746                 !pci_is_root_bus(rdev->pdev->bus)) {
9747                 struct pci_dev *root = rdev->pdev->bus->self;
9748                 u32 lnkcap;
9749 
9750                 clk_req_support = false;
9751                 pcie_capability_read_dword(root, PCI_EXP_LNKCAP, &lnkcap);
9752                 if (lnkcap & PCI_EXP_LNKCAP_CLKPM)
9753                     clk_req_support = true;
9754             } else {
9755                 clk_req_support = false;
9756             }
9757 
9758             if (clk_req_support) {
9759                 orig = data = RREG32_PCIE_PORT(PCIE_LC_CNTL2);
9760                 data |= LC_ALLOW_PDWN_IN_L1 | LC_ALLOW_PDWN_IN_L23;
9761                 if (orig != data)
9762                     WREG32_PCIE_PORT(PCIE_LC_CNTL2, data);
9763 
9764                 orig = data = RREG32_SMC(THM_CLK_CNTL);
9765                 data &= ~(CMON_CLK_SEL_MASK | TMON_CLK_SEL_MASK);
9766                 data |= CMON_CLK_SEL(1) | TMON_CLK_SEL(1);
9767                 if (orig != data)
9768                     WREG32_SMC(THM_CLK_CNTL, data);
9769 
9770                 orig = data = RREG32_SMC(MISC_CLK_CTRL);
9771                 data &= ~(DEEP_SLEEP_CLK_SEL_MASK | ZCLK_SEL_MASK);
9772                 data |= DEEP_SLEEP_CLK_SEL(1) | ZCLK_SEL(1);
9773                 if (orig != data)
9774                     WREG32_SMC(MISC_CLK_CTRL, data);
9775 
9776                 orig = data = RREG32_SMC(CG_CLKPIN_CNTL);
9777                 data &= ~BCLK_AS_XCLK;
9778                 if (orig != data)
9779                     WREG32_SMC(CG_CLKPIN_CNTL, data);
9780 
9781                 orig = data = RREG32_SMC(CG_CLKPIN_CNTL_2);
9782                 data &= ~FORCE_BIF_REFCLK_EN;
9783                 if (orig != data)
9784                     WREG32_SMC(CG_CLKPIN_CNTL_2, data);
9785 
9786                 orig = data = RREG32_SMC(MPLL_BYPASSCLK_SEL);
9787                 data &= ~MPLL_CLKOUT_SEL_MASK;
9788                 data |= MPLL_CLKOUT_SEL(4);
9789                 if (orig != data)
9790                     WREG32_SMC(MPLL_BYPASSCLK_SEL, data);
9791             }
9792         }
9793     } else {
9794         if (orig != data)
9795             WREG32_PCIE_PORT(PCIE_LC_CNTL, data);
9796     }
9797 
9798     orig = data = RREG32_PCIE_PORT(PCIE_CNTL2);
9799     data |= SLV_MEM_LS_EN | MST_MEM_LS_EN | REPLAY_MEM_LS_EN;
9800     if (orig != data)
9801         WREG32_PCIE_PORT(PCIE_CNTL2, data);
9802 
9803     if (!disable_l0s) {
9804         data = RREG32_PCIE_PORT(PCIE_LC_N_FTS_CNTL);
9805         if((data & LC_N_FTS_MASK) == LC_N_FTS_MASK) {
9806             data = RREG32_PCIE_PORT(PCIE_LC_STATUS1);
9807             if ((data & LC_REVERSE_XMIT) && (data & LC_REVERSE_RCVR)) {
9808                 orig = data = RREG32_PCIE_PORT(PCIE_LC_CNTL);
9809                 data &= ~LC_L0S_INACTIVITY_MASK;
9810                 if (orig != data)
9811                     WREG32_PCIE_PORT(PCIE_LC_CNTL, data);
9812             }
9813         }
9814     }
9815 }
[ Source navigation ] [ Diff markup ] [ Identifier search ] [ general search ]

This page was automatically generated by the 2.1.0 LXR engine.

The LXR team
Valid CSS 2.1! Valid HTML 4.01!