drm/radeon/ni_dpm.c

0001 /*
0002  * Copyright 2012 Advanced Micro Devices, Inc.
0003  *
0004  * Permission is hereby granted, free of charge, to any person obtaining a
0005  * copy of this software and associated documentation files (the "Software"),
0006  * to deal in the Software without restriction, including without limitation
0007  * the rights to use, copy, modify, merge, publish, distribute, sublicense,
0008  * and/or sell copies of the Software, and to permit persons to whom the
0009  * Software is furnished to do so, subject to the following conditions:
0010  *
0011  * The above copyright notice and this permission notice shall be included in
0012  * all copies or substantial portions of the Software.
0013  *
0014  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
0015  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
0016  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
0017  * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
0018  * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
0019  * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
0020  * OTHER DEALINGS IN THE SOFTWARE.
0021  *
0022  */
0023
0024 #include <linux/math64.h>
0025 #include <linux/pci.h>
0026 #include <linux/seq_file.h>
0027
0028 #include "atom.h"
0029 #include "evergreen.h"
0030 #include "ni_dpm.h"
0031 #include "nid.h"
0032 #include "r600_dpm.h"
0033 #include "rv770.h"
0034 #include "radeon.h"
0035 #include "radeon_asic.h"
0036
0037 #define MC_CG_ARB_FREQ_F0           0x0a
0038 #define MC_CG_ARB_FREQ_F1           0x0b
0039 #define MC_CG_ARB_FREQ_F2           0x0c
0040 #define MC_CG_ARB_FREQ_F3           0x0d
0041
0042 #define SMC_RAM_END 0xC000
0043
0044 static const struct ni_cac_weights cac_weights_cayman_xt =
0045 {
0046     0x15,
0047     0x2,
0048     0x19,
0049     0x2,
0050     0x8,
0051     0x14,
0052     0x2,
0053     0x16,
0054     0xE,
0055     0x17,
0056     0x13,
0057     0x2B,
0058     0x10,
0059     0x7,
0060     0x5,
0061     0x5,
0062     0x5,
0063     0x2,
0064     0x3,
0065     0x9,
0066     0x10,
0067     0x10,
0068     0x2B,
0069     0xA,
0070     0x9,
0071     0x4,
0072     0xD,
0073     0xD,
0074     0x3E,
0075     0x18,
0076     0x14,
0077     0,
0078     0x3,
0079     0x3,
0080     0x5,
0081     0,
0082     0x2,
0083     0,
0084     0,
0085     0,
0086     0,
0087     0,
0088     0,
0089     0,
0090     0,
0091     0,
0092     0x1CC,
0093     0,
0094     0x164,
0095     1,
0096     1,
0097     1,
0098     1,
0099     12,
0100     12,
0101     12,
0102     0x12,
0103     0x1F,
0104     132,
0105     5,
0106     7,
0107     0,
0108     { 0, 0, 0, 0, 0, 0, 0, 0 },
0109     { 0, 0, 0, 0 },
0110     true
0111 };
0112
0113 static const struct ni_cac_weights cac_weights_cayman_pro =
0114 {
0115     0x16,
0116     0x4,
0117     0x10,
0118     0x2,
0119     0xA,
0120     0x16,
0121     0x2,
0122     0x18,
0123     0x10,
0124     0x1A,
0125     0x16,
0126     0x2D,
0127     0x12,
0128     0xA,
0129     0x6,
0130     0x6,
0131     0x6,
0132     0x2,
0133     0x4,
0134     0xB,
0135     0x11,
0136     0x11,
0137     0x2D,
0138     0xC,
0139     0xC,
0140     0x7,
0141     0x10,
0142     0x10,
0143     0x3F,
0144     0x1A,
0145     0x16,
0146     0,
0147     0x7,
0148     0x4,
0149     0x6,
0150     1,
0151     0x2,
0152     0x1,
0153     0,
0154     0,
0155     0,
0156     0,
0157     0,
0158     0,
0159     0x30,
0160     0,
0161     0x1CF,
0162     0,
0163     0x166,
0164     1,
0165     1,
0166     1,
0167     1,
0168     12,
0169     12,
0170     12,
0171     0x15,
0172     0x1F,
0173     132,
0174     6,
0175     6,
0176     0,
0177     { 0, 0, 0, 0, 0, 0, 0, 0 },
0178     { 0, 0, 0, 0 },
0179     true
0180 };
0181
0182 static const struct ni_cac_weights cac_weights_cayman_le =
0183 {
0184     0x7,
0185     0xE,
0186     0x1,
0187     0xA,
0188     0x1,
0189     0x3F,
0190     0x2,
0191     0x18,
0192     0x10,
0193     0x1A,
0194     0x1,
0195     0x3F,
0196     0x1,
0197     0xE,
0198     0x6,
0199     0x6,
0200     0x6,
0201     0x2,
0202     0x4,
0203     0x9,
0204     0x1A,
0205     0x1A,
0206     0x2C,
0207     0xA,
0208     0x11,
0209     0x8,
0210     0x19,
0211     0x19,
0212     0x1,
0213     0x1,
0214     0x1A,
0215     0,
0216     0x8,
0217     0x5,
0218     0x8,
0219     0x1,
0220     0x3,
0221     0x1,
0222     0,
0223     0,
0224     0,
0225     0,
0226     0,
0227     0,
0228     0x38,
0229     0x38,
0230     0x239,
0231     0x3,
0232     0x18A,
0233     1,
0234     1,
0235     1,
0236     1,
0237     12,
0238     12,
0239     12,
0240     0x15,
0241     0x22,
0242     132,
0243     6,
0244     6,
0245     0,
0246     { 0, 0, 0, 0, 0, 0, 0, 0 },
0247     { 0, 0, 0, 0 },
0248     true
0249 };
0250
0251 #define NISLANDS_MGCG_SEQUENCE  300
0252
0253 static const u32 cayman_cgcg_cgls_default[] =
0254 {
0255     0x000008f8, 0x00000010, 0xffffffff,
0256     0x000008fc, 0x00000000, 0xffffffff,
0257     0x000008f8, 0x00000011, 0xffffffff,
0258     0x000008fc, 0x00000000, 0xffffffff,
0259     0x000008f8, 0x00000012, 0xffffffff,
0260     0x000008fc, 0x00000000, 0xffffffff,
0261     0x000008f8, 0x00000013, 0xffffffff,
0262     0x000008fc, 0x00000000, 0xffffffff,
0263     0x000008f8, 0x00000014, 0xffffffff,
0264     0x000008fc, 0x00000000, 0xffffffff,
0265     0x000008f8, 0x00000015, 0xffffffff,
0266     0x000008fc, 0x00000000, 0xffffffff,
0267     0x000008f8, 0x00000016, 0xffffffff,
0268     0x000008fc, 0x00000000, 0xffffffff,
0269     0x000008f8, 0x00000017, 0xffffffff,
0270     0x000008fc, 0x00000000, 0xffffffff,
0271     0x000008f8, 0x00000018, 0xffffffff,
0272     0x000008fc, 0x00000000, 0xffffffff,
0273     0x000008f8, 0x00000019, 0xffffffff,
0274     0x000008fc, 0x00000000, 0xffffffff,
0275     0x000008f8, 0x0000001a, 0xffffffff,
0276     0x000008fc, 0x00000000, 0xffffffff,
0277     0x000008f8, 0x0000001b, 0xffffffff,
0278     0x000008fc, 0x00000000, 0xffffffff,
0279     0x000008f8, 0x00000020, 0xffffffff,
0280     0x000008fc, 0x00000000, 0xffffffff,
0281     0x000008f8, 0x00000021, 0xffffffff,
0282     0x000008fc, 0x00000000, 0xffffffff,
0283     0x000008f8, 0x00000022, 0xffffffff,
0284     0x000008fc, 0x00000000, 0xffffffff,
0285     0x000008f8, 0x00000023, 0xffffffff,
0286     0x000008fc, 0x00000000, 0xffffffff,
0287     0x000008f8, 0x00000024, 0xffffffff,
0288     0x000008fc, 0x00000000, 0xffffffff,
0289     0x000008f8, 0x00000025, 0xffffffff,
0290     0x000008fc, 0x00000000, 0xffffffff,
0291     0x000008f8, 0x00000026, 0xffffffff,
0292     0x000008fc, 0x00000000, 0xffffffff,
0293     0x000008f8, 0x00000027, 0xffffffff,
0294     0x000008fc, 0x00000000, 0xffffffff,
0295     0x000008f8, 0x00000028, 0xffffffff,
0296     0x000008fc, 0x00000000, 0xffffffff,
0297     0x000008f8, 0x00000029, 0xffffffff,
0298     0x000008fc, 0x00000000, 0xffffffff,
0299     0x000008f8, 0x0000002a, 0xffffffff,
0300     0x000008fc, 0x00000000, 0xffffffff,
0301     0x000008f8, 0x0000002b, 0xffffffff,
0302     0x000008fc, 0x00000000, 0xffffffff
0303 };
0304 #define CAYMAN_CGCG_CGLS_DEFAULT_LENGTH sizeof(cayman_cgcg_cgls_default) / (3 * sizeof(u32))
0305
0306 static const u32 cayman_cgcg_cgls_disable[] =
0307 {
0308     0x000008f8, 0x00000010, 0xffffffff,
0309     0x000008fc, 0xffffffff, 0xffffffff,
0310     0x000008f8, 0x00000011, 0xffffffff,
0311     0x000008fc, 0xffffffff, 0xffffffff,
0312     0x000008f8, 0x00000012, 0xffffffff,
0313     0x000008fc, 0xffffffff, 0xffffffff,
0314     0x000008f8, 0x00000013, 0xffffffff,
0315     0x000008fc, 0xffffffff, 0xffffffff,
0316     0x000008f8, 0x00000014, 0xffffffff,
0317     0x000008fc, 0xffffffff, 0xffffffff,
0318     0x000008f8, 0x00000015, 0xffffffff,
0319     0x000008fc, 0xffffffff, 0xffffffff,
0320     0x000008f8, 0x00000016, 0xffffffff,
0321     0x000008fc, 0xffffffff, 0xffffffff,
0322     0x000008f8, 0x00000017, 0xffffffff,
0323     0x000008fc, 0xffffffff, 0xffffffff,
0324     0x000008f8, 0x00000018, 0xffffffff,
0325     0x000008fc, 0xffffffff, 0xffffffff,
0326     0x000008f8, 0x00000019, 0xffffffff,
0327     0x000008fc, 0xffffffff, 0xffffffff,
0328     0x000008f8, 0x0000001a, 0xffffffff,
0329     0x000008fc, 0xffffffff, 0xffffffff,
0330     0x000008f8, 0x0000001b, 0xffffffff,
0331     0x000008fc, 0xffffffff, 0xffffffff,
0332     0x000008f8, 0x00000020, 0xffffffff,
0333     0x000008fc, 0x00000000, 0xffffffff,
0334     0x000008f8, 0x00000021, 0xffffffff,
0335     0x000008fc, 0x00000000, 0xffffffff,
0336     0x000008f8, 0x00000022, 0xffffffff,
0337     0x000008fc, 0x00000000, 0xffffffff,
0338     0x000008f8, 0x00000023, 0xffffffff,
0339     0x000008fc, 0x00000000, 0xffffffff,
0340     0x000008f8, 0x00000024, 0xffffffff,
0341     0x000008fc, 0x00000000, 0xffffffff,
0342     0x000008f8, 0x00000025, 0xffffffff,
0343     0x000008fc, 0x00000000, 0xffffffff,
0344     0x000008f8, 0x00000026, 0xffffffff,
0345     0x000008fc, 0x00000000, 0xffffffff,
0346     0x000008f8, 0x00000027, 0xffffffff,
0347     0x000008fc, 0x00000000, 0xffffffff,
0348     0x000008f8, 0x00000028, 0xffffffff,
0349     0x000008fc, 0x00000000, 0xffffffff,
0350     0x000008f8, 0x00000029, 0xffffffff,
0351     0x000008fc, 0x00000000, 0xffffffff,
0352     0x000008f8, 0x0000002a, 0xffffffff,
0353     0x000008fc, 0x00000000, 0xffffffff,
0354     0x000008f8, 0x0000002b, 0xffffffff,
0355     0x000008fc, 0x00000000, 0xffffffff,
0356     0x00000644, 0x000f7902, 0x001f4180,
0357     0x00000644, 0x000f3802, 0x001f4180
0358 };
0359 #define CAYMAN_CGCG_CGLS_DISABLE_LENGTH sizeof(cayman_cgcg_cgls_disable) / (3 * sizeof(u32))
0360
0361 static const u32 cayman_cgcg_cgls_enable[] =
0362 {
0363     0x00000644, 0x000f7882, 0x001f4080,
0364     0x000008f8, 0x00000010, 0xffffffff,
0365     0x000008fc, 0x00000000, 0xffffffff,
0366     0x000008f8, 0x00000011, 0xffffffff,
0367     0x000008fc, 0x00000000, 0xffffffff,
0368     0x000008f8, 0x00000012, 0xffffffff,
0369     0x000008fc, 0x00000000, 0xffffffff,
0370     0x000008f8, 0x00000013, 0xffffffff,
0371     0x000008fc, 0x00000000, 0xffffffff,
0372     0x000008f8, 0x00000014, 0xffffffff,
0373     0x000008fc, 0x00000000, 0xffffffff,
0374     0x000008f8, 0x00000015, 0xffffffff,
0375     0x000008fc, 0x00000000, 0xffffffff,
0376     0x000008f8, 0x00000016, 0xffffffff,
0377     0x000008fc, 0x00000000, 0xffffffff,
0378     0x000008f8, 0x00000017, 0xffffffff,
0379     0x000008fc, 0x00000000, 0xffffffff,
0380     0x000008f8, 0x00000018, 0xffffffff,
0381     0x000008fc, 0x00000000, 0xffffffff,
0382     0x000008f8, 0x00000019, 0xffffffff,
0383     0x000008fc, 0x00000000, 0xffffffff,
0384     0x000008f8, 0x0000001a, 0xffffffff,
0385     0x000008fc, 0x00000000, 0xffffffff,
0386     0x000008f8, 0x0000001b, 0xffffffff,
0387     0x000008fc, 0x00000000, 0xffffffff,
0388     0x000008f8, 0x00000020, 0xffffffff,
0389     0x000008fc, 0xffffffff, 0xffffffff,
0390     0x000008f8, 0x00000021, 0xffffffff,
0391     0x000008fc, 0xffffffff, 0xffffffff,
0392     0x000008f8, 0x00000022, 0xffffffff,
0393     0x000008fc, 0xffffffff, 0xffffffff,
0394     0x000008f8, 0x00000023, 0xffffffff,
0395     0x000008fc, 0xffffffff, 0xffffffff,
0396     0x000008f8, 0x00000024, 0xffffffff,
0397     0x000008fc, 0xffffffff, 0xffffffff,
0398     0x000008f8, 0x00000025, 0xffffffff,
0399     0x000008fc, 0xffffffff, 0xffffffff,
0400     0x000008f8, 0x00000026, 0xffffffff,
0401     0x000008fc, 0xffffffff, 0xffffffff,
0402     0x000008f8, 0x00000027, 0xffffffff,
0403     0x000008fc, 0xffffffff, 0xffffffff,
0404     0x000008f8, 0x00000028, 0xffffffff,
0405     0x000008fc, 0xffffffff, 0xffffffff,
0406     0x000008f8, 0x00000029, 0xffffffff,
0407     0x000008fc, 0xffffffff, 0xffffffff,
0408     0x000008f8, 0x0000002a, 0xffffffff,
0409     0x000008fc, 0xffffffff, 0xffffffff,
0410     0x000008f8, 0x0000002b, 0xffffffff,
0411     0x000008fc, 0xffffffff, 0xffffffff
0412 };
0413 #define CAYMAN_CGCG_CGLS_ENABLE_LENGTH  sizeof(cayman_cgcg_cgls_enable) / (3 * sizeof(u32))
0414
0415 static const u32 cayman_mgcg_default[] =
0416 {
0417     0x0000802c, 0xc0000000, 0xffffffff,
0418     0x00003fc4, 0xc0000000, 0xffffffff,
0419     0x00005448, 0x00000100, 0xffffffff,
0420     0x000055e4, 0x00000100, 0xffffffff,
0421     0x0000160c, 0x00000100, 0xffffffff,
0422     0x00008984, 0x06000100, 0xffffffff,
0423     0x0000c164, 0x00000100, 0xffffffff,
0424     0x00008a18, 0x00000100, 0xffffffff,
0425     0x0000897c, 0x06000100, 0xffffffff,
0426     0x00008b28, 0x00000100, 0xffffffff,
0427     0x00009144, 0x00800200, 0xffffffff,
0428     0x00009a60, 0x00000100, 0xffffffff,
0429     0x00009868, 0x00000100, 0xffffffff,
0430     0x00008d58, 0x00000100, 0xffffffff,
0431     0x00009510, 0x00000100, 0xffffffff,
0432     0x0000949c, 0x00000100, 0xffffffff,
0433     0x00009654, 0x00000100, 0xffffffff,
0434     0x00009030, 0x00000100, 0xffffffff,
0435     0x00009034, 0x00000100, 0xffffffff,
0436     0x00009038, 0x00000100, 0xffffffff,
0437     0x0000903c, 0x00000100, 0xffffffff,
0438     0x00009040, 0x00000100, 0xffffffff,
0439     0x0000a200, 0x00000100, 0xffffffff,
0440     0x0000a204, 0x00000100, 0xffffffff,
0441     0x0000a208, 0x00000100, 0xffffffff,
0442     0x0000a20c, 0x00000100, 0xffffffff,
0443     0x00009744, 0x00000100, 0xffffffff,
0444     0x00003f80, 0x00000100, 0xffffffff,
0445     0x0000a210, 0x00000100, 0xffffffff,
0446     0x0000a214, 0x00000100, 0xffffffff,
0447     0x000004d8, 0x00000100, 0xffffffff,
0448     0x00009664, 0x00000100, 0xffffffff,
0449     0x00009698, 0x00000100, 0xffffffff,
0450     0x000004d4, 0x00000200, 0xffffffff,
0451     0x000004d0, 0x00000000, 0xffffffff,
0452     0x000030cc, 0x00000104, 0xffffffff,
0453     0x0000d0c0, 0x00000100, 0xffffffff,
0454     0x0000d8c0, 0x00000100, 0xffffffff,
0455     0x0000802c, 0x40000000, 0xffffffff,
0456     0x00003fc4, 0x40000000, 0xffffffff,
0457     0x0000915c, 0x00010000, 0xffffffff,
0458     0x00009160, 0x00030002, 0xffffffff,
0459     0x00009164, 0x00050004, 0xffffffff,
0460     0x00009168, 0x00070006, 0xffffffff,
0461     0x00009178, 0x00070000, 0xffffffff,
0462     0x0000917c, 0x00030002, 0xffffffff,
0463     0x00009180, 0x00050004, 0xffffffff,
0464     0x0000918c, 0x00010006, 0xffffffff,
0465     0x00009190, 0x00090008, 0xffffffff,
0466     0x00009194, 0x00070000, 0xffffffff,
0467     0x00009198, 0x00030002, 0xffffffff,
0468     0x0000919c, 0x00050004, 0xffffffff,
0469     0x000091a8, 0x00010006, 0xffffffff,
0470     0x000091ac, 0x00090008, 0xffffffff,
0471     0x000091b0, 0x00070000, 0xffffffff,
0472     0x000091b4, 0x00030002, 0xffffffff,
0473     0x000091b8, 0x00050004, 0xffffffff,
0474     0x000091c4, 0x00010006, 0xffffffff,
0475     0x000091c8, 0x00090008, 0xffffffff,
0476     0x000091cc, 0x00070000, 0xffffffff,
0477     0x000091d0, 0x00030002, 0xffffffff,
0478     0x000091d4, 0x00050004, 0xffffffff,
0479     0x000091e0, 0x00010006, 0xffffffff,
0480     0x000091e4, 0x00090008, 0xffffffff,
0481     0x000091e8, 0x00000000, 0xffffffff,
0482     0x000091ec, 0x00070000, 0xffffffff,
0483     0x000091f0, 0x00030002, 0xffffffff,
0484     0x000091f4, 0x00050004, 0xffffffff,
0485     0x00009200, 0x00010006, 0xffffffff,
0486     0x00009204, 0x00090008, 0xffffffff,
0487     0x00009208, 0x00070000, 0xffffffff,
0488     0x0000920c, 0x00030002, 0xffffffff,
0489     0x00009210, 0x00050004, 0xffffffff,
0490     0x0000921c, 0x00010006, 0xffffffff,
0491     0x00009220, 0x00090008, 0xffffffff,
0492     0x00009224, 0x00070000, 0xffffffff,
0493     0x00009228, 0x00030002, 0xffffffff,
0494     0x0000922c, 0x00050004, 0xffffffff,
0495     0x00009238, 0x00010006, 0xffffffff,
0496     0x0000923c, 0x00090008, 0xffffffff,
0497     0x00009240, 0x00070000, 0xffffffff,
0498     0x00009244, 0x00030002, 0xffffffff,
0499     0x00009248, 0x00050004, 0xffffffff,
0500     0x00009254, 0x00010006, 0xffffffff,
0501     0x00009258, 0x00090008, 0xffffffff,
0502     0x0000925c, 0x00070000, 0xffffffff,
0503     0x00009260, 0x00030002, 0xffffffff,
0504     0x00009264, 0x00050004, 0xffffffff,
0505     0x00009270, 0x00010006, 0xffffffff,
0506     0x00009274, 0x00090008, 0xffffffff,
0507     0x00009278, 0x00070000, 0xffffffff,
0508     0x0000927c, 0x00030002, 0xffffffff,
0509     0x00009280, 0x00050004, 0xffffffff,
0510     0x0000928c, 0x00010006, 0xffffffff,
0511     0x00009290, 0x00090008, 0xffffffff,
0512     0x000092a8, 0x00070000, 0xffffffff,
0513     0x000092ac, 0x00030002, 0xffffffff,
0514     0x000092b0, 0x00050004, 0xffffffff,
0515     0x000092bc, 0x00010006, 0xffffffff,
0516     0x000092c0, 0x00090008, 0xffffffff,
0517     0x000092c4, 0x00070000, 0xffffffff,
0518     0x000092c8, 0x00030002, 0xffffffff,
0519     0x000092cc, 0x00050004, 0xffffffff,
0520     0x000092d8, 0x00010006, 0xffffffff,
0521     0x000092dc, 0x00090008, 0xffffffff,
0522     0x00009294, 0x00000000, 0xffffffff,
0523     0x0000802c, 0x40010000, 0xffffffff,
0524     0x00003fc4, 0x40010000, 0xffffffff,
0525     0x0000915c, 0x00010000, 0xffffffff,
0526     0x00009160, 0x00030002, 0xffffffff,
0527     0x00009164, 0x00050004, 0xffffffff,
0528     0x00009168, 0x00070006, 0xffffffff,
0529     0x00009178, 0x00070000, 0xffffffff,
0530     0x0000917c, 0x00030002, 0xffffffff,
0531     0x00009180, 0x00050004, 0xffffffff,
0532     0x0000918c, 0x00010006, 0xffffffff,
0533     0x00009190, 0x00090008, 0xffffffff,
0534     0x00009194, 0x00070000, 0xffffffff,
0535     0x00009198, 0x00030002, 0xffffffff,
0536     0x0000919c, 0x00050004, 0xffffffff,
0537     0x000091a8, 0x00010006, 0xffffffff,
0538     0x000091ac, 0x00090008, 0xffffffff,
0539     0x000091b0, 0x00070000, 0xffffffff,
0540     0x000091b4, 0x00030002, 0xffffffff,
0541     0x000091b8, 0x00050004, 0xffffffff,
0542     0x000091c4, 0x00010006, 0xffffffff,
0543     0x000091c8, 0x00090008, 0xffffffff,
0544     0x000091cc, 0x00070000, 0xffffffff,
0545     0x000091d0, 0x00030002, 0xffffffff,
0546     0x000091d4, 0x00050004, 0xffffffff,
0547     0x000091e0, 0x00010006, 0xffffffff,
0548     0x000091e4, 0x00090008, 0xffffffff,
0549     0x000091e8, 0x00000000, 0xffffffff,
0550     0x000091ec, 0x00070000, 0xffffffff,
0551     0x000091f0, 0x00030002, 0xffffffff,
0552     0x000091f4, 0x00050004, 0xffffffff,
0553     0x00009200, 0x00010006, 0xffffffff,
0554     0x00009204, 0x00090008, 0xffffffff,
0555     0x00009208, 0x00070000, 0xffffffff,
0556     0x0000920c, 0x00030002, 0xffffffff,
0557     0x00009210, 0x00050004, 0xffffffff,
0558     0x0000921c, 0x00010006, 0xffffffff,
0559     0x00009220, 0x00090008, 0xffffffff,
0560     0x00009224, 0x00070000, 0xffffffff,
0561     0x00009228, 0x00030002, 0xffffffff,
0562     0x0000922c, 0x00050004, 0xffffffff,
0563     0x00009238, 0x00010006, 0xffffffff,
0564     0x0000923c, 0x00090008, 0xffffffff,
0565     0x00009240, 0x00070000, 0xffffffff,
0566     0x00009244, 0x00030002, 0xffffffff,
0567     0x00009248, 0x00050004, 0xffffffff,
0568     0x00009254, 0x00010006, 0xffffffff,
0569     0x00009258, 0x00090008, 0xffffffff,
0570     0x0000925c, 0x00070000, 0xffffffff,
0571     0x00009260, 0x00030002, 0xffffffff,
0572     0x00009264, 0x00050004, 0xffffffff,
0573     0x00009270, 0x00010006, 0xffffffff,
0574     0x00009274, 0x00090008, 0xffffffff,
0575     0x00009278, 0x00070000, 0xffffffff,
0576     0x0000927c, 0x00030002, 0xffffffff,
0577     0x00009280, 0x00050004, 0xffffffff,
0578     0x0000928c, 0x00010006, 0xffffffff,
0579     0x00009290, 0x00090008, 0xffffffff,
0580     0x000092a8, 0x00070000, 0xffffffff,
0581     0x000092ac, 0x00030002, 0xffffffff,
0582     0x000092b0, 0x00050004, 0xffffffff,
0583     0x000092bc, 0x00010006, 0xffffffff,
0584     0x000092c0, 0x00090008, 0xffffffff,
0585     0x000092c4, 0x00070000, 0xffffffff,
0586     0x000092c8, 0x00030002, 0xffffffff,
0587     0x000092cc, 0x00050004, 0xffffffff,
0588     0x000092d8, 0x00010006, 0xffffffff,
0589     0x000092dc, 0x00090008, 0xffffffff,
0590     0x00009294, 0x00000000, 0xffffffff,
0591     0x0000802c, 0xc0000000, 0xffffffff,
0592     0x00003fc4, 0xc0000000, 0xffffffff,
0593     0x000008f8, 0x00000010, 0xffffffff,
0594     0x000008fc, 0x00000000, 0xffffffff,
0595     0x000008f8, 0x00000011, 0xffffffff,
0596     0x000008fc, 0x00000000, 0xffffffff,
0597     0x000008f8, 0x00000012, 0xffffffff,
0598     0x000008fc, 0x00000000, 0xffffffff,
0599     0x000008f8, 0x00000013, 0xffffffff,
0600     0x000008fc, 0x00000000, 0xffffffff,
0601     0x000008f8, 0x00000014, 0xffffffff,
0602     0x000008fc, 0x00000000, 0xffffffff,
0603     0x000008f8, 0x00000015, 0xffffffff,
0604     0x000008fc, 0x00000000, 0xffffffff,
0605     0x000008f8, 0x00000016, 0xffffffff,
0606     0x000008fc, 0x00000000, 0xffffffff,
0607     0x000008f8, 0x00000017, 0xffffffff,
0608     0x000008fc, 0x00000000, 0xffffffff,
0609     0x000008f8, 0x00000018, 0xffffffff,
0610     0x000008fc, 0x00000000, 0xffffffff,
0611     0x000008f8, 0x00000019, 0xffffffff,
0612     0x000008fc, 0x00000000, 0xffffffff,
0613     0x000008f8, 0x0000001a, 0xffffffff,
0614     0x000008fc, 0x00000000, 0xffffffff,
0615     0x000008f8, 0x0000001b, 0xffffffff,
0616     0x000008fc, 0x00000000, 0xffffffff
0617 };
0618 #define CAYMAN_MGCG_DEFAULT_LENGTH sizeof(cayman_mgcg_default) / (3 * sizeof(u32))
0619
0620 static const u32 cayman_mgcg_disable[] =
0621 {
0622     0x0000802c, 0xc0000000, 0xffffffff,
0623     0x000008f8, 0x00000000, 0xffffffff,
0624     0x000008fc, 0xffffffff, 0xffffffff,
0625     0x000008f8, 0x00000001, 0xffffffff,
0626     0x000008fc, 0xffffffff, 0xffffffff,
0627     0x000008f8, 0x00000002, 0xffffffff,
0628     0x000008fc, 0xffffffff, 0xffffffff,
0629     0x000008f8, 0x00000003, 0xffffffff,
0630     0x000008fc, 0xffffffff, 0xffffffff,
0631     0x00009150, 0x00600000, 0xffffffff
0632 };
0633 #define CAYMAN_MGCG_DISABLE_LENGTH   sizeof(cayman_mgcg_disable) / (3 * sizeof(u32))
0634
0635 static const u32 cayman_mgcg_enable[] =
0636 {
0637     0x0000802c, 0xc0000000, 0xffffffff,
0638     0x000008f8, 0x00000000, 0xffffffff,
0639     0x000008fc, 0x00000000, 0xffffffff,
0640     0x000008f8, 0x00000001, 0xffffffff,
0641     0x000008fc, 0x00000000, 0xffffffff,
0642     0x000008f8, 0x00000002, 0xffffffff,
0643     0x000008fc, 0x00600000, 0xffffffff,
0644     0x000008f8, 0x00000003, 0xffffffff,
0645     0x000008fc, 0x00000000, 0xffffffff,
0646     0x00009150, 0x96944200, 0xffffffff
0647 };
0648
0649 #define CAYMAN_MGCG_ENABLE_LENGTH   sizeof(cayman_mgcg_enable) / (3 * sizeof(u32))
0650
0651 #define NISLANDS_SYSLS_SEQUENCE  100
0652
0653 static const u32 cayman_sysls_default[] =
0654 {
0655     /* Register,   Value,     Mask bits */
0656     0x000055e8, 0x00000000, 0xffffffff,
0657     0x0000d0bc, 0x00000000, 0xffffffff,
0658     0x0000d8bc, 0x00000000, 0xffffffff,
0659     0x000015c0, 0x000c1401, 0xffffffff,
0660     0x0000264c, 0x000c0400, 0xffffffff,
0661     0x00002648, 0x000c0400, 0xffffffff,
0662     0x00002650, 0x000c0400, 0xffffffff,
0663     0x000020b8, 0x000c0400, 0xffffffff,
0664     0x000020bc, 0x000c0400, 0xffffffff,
0665     0x000020c0, 0x000c0c80, 0xffffffff,
0666     0x0000f4a0, 0x000000c0, 0xffffffff,
0667     0x0000f4a4, 0x00680fff, 0xffffffff,
0668     0x00002f50, 0x00000404, 0xffffffff,
0669     0x000004c8, 0x00000001, 0xffffffff,
0670     0x000064ec, 0x00000000, 0xffffffff,
0671     0x00000c7c, 0x00000000, 0xffffffff,
0672     0x00008dfc, 0x00000000, 0xffffffff
0673 };
0674 #define CAYMAN_SYSLS_DEFAULT_LENGTH sizeof(cayman_sysls_default) / (3 * sizeof(u32))
0675
0676 static const u32 cayman_sysls_disable[] =
0677 {
0678     /* Register,   Value,     Mask bits */
0679     0x0000d0c0, 0x00000000, 0xffffffff,
0680     0x0000d8c0, 0x00000000, 0xffffffff,
0681     0x000055e8, 0x00000000, 0xffffffff,
0682     0x0000d0bc, 0x00000000, 0xffffffff,
0683     0x0000d8bc, 0x00000000, 0xffffffff,
0684     0x000015c0, 0x00041401, 0xffffffff,
0685     0x0000264c, 0x00040400, 0xffffffff,
0686     0x00002648, 0x00040400, 0xffffffff,
0687     0x00002650, 0x00040400, 0xffffffff,
0688     0x000020b8, 0x00040400, 0xffffffff,
0689     0x000020bc, 0x00040400, 0xffffffff,
0690     0x000020c0, 0x00040c80, 0xffffffff,
0691     0x0000f4a0, 0x000000c0, 0xffffffff,
0692     0x0000f4a4, 0x00680000, 0xffffffff,
0693     0x00002f50, 0x00000404, 0xffffffff,
0694     0x000004c8, 0x00000001, 0xffffffff,
0695     0x000064ec, 0x00007ffd, 0xffffffff,
0696     0x00000c7c, 0x0000ff00, 0xffffffff,
0697     0x00008dfc, 0x0000007f, 0xffffffff
0698 };
0699 #define CAYMAN_SYSLS_DISABLE_LENGTH sizeof(cayman_sysls_disable) / (3 * sizeof(u32))
0700
0701 static const u32 cayman_sysls_enable[] =
0702 {
0703     /* Register,   Value,     Mask bits */
0704     0x000055e8, 0x00000001, 0xffffffff,
0705     0x0000d0bc, 0x00000100, 0xffffffff,
0706     0x0000d8bc, 0x00000100, 0xffffffff,
0707     0x000015c0, 0x000c1401, 0xffffffff,
0708     0x0000264c, 0x000c0400, 0xffffffff,
0709     0x00002648, 0x000c0400, 0xffffffff,
0710     0x00002650, 0x000c0400, 0xffffffff,
0711     0x000020b8, 0x000c0400, 0xffffffff,
0712     0x000020bc, 0x000c0400, 0xffffffff,
0713     0x000020c0, 0x000c0c80, 0xffffffff,
0714     0x0000f4a0, 0x000000c0, 0xffffffff,
0715     0x0000f4a4, 0x00680fff, 0xffffffff,
0716     0x00002f50, 0x00000903, 0xffffffff,
0717     0x000004c8, 0x00000000, 0xffffffff,
0718     0x000064ec, 0x00000000, 0xffffffff,
0719     0x00000c7c, 0x00000000, 0xffffffff,
0720     0x00008dfc, 0x00000000, 0xffffffff
0721 };
0722 #define CAYMAN_SYSLS_ENABLE_LENGTH sizeof(cayman_sysls_enable) / (3 * sizeof(u32))
0723
0724 extern int ni_mc_load_microcode(struct radeon_device *rdev);
0725
0726 struct ni_power_info *ni_get_pi(struct radeon_device *rdev)
0727 {
0728     struct ni_power_info *pi = rdev->pm.dpm.priv;
0729
0730     return pi;
0731 }
0732
0733 struct ni_ps *ni_get_ps(struct radeon_ps *rps)
0734 {
0735     struct ni_ps *ps = rps->ps_priv;
0736
0737     return ps;
0738 }
0739
0740 static void ni_calculate_leakage_for_v_and_t_formula(const struct ni_leakage_coeffients *coeff,
0741                              u16 v, s32 t,
0742                              u32 ileakage,
0743                              u32 *leakage)
0744 {
0745     s64 kt, kv, leakage_w, i_leakage, vddc, temperature;
0746
0747     i_leakage = div64_s64(drm_int2fixp(ileakage), 1000);
0748     vddc = div64_s64(drm_int2fixp(v), 1000);
0749     temperature = div64_s64(drm_int2fixp(t), 1000);
0750
0751     kt = drm_fixp_mul(div64_s64(drm_int2fixp(coeff->at), 1000),
0752               drm_fixp_exp(drm_fixp_mul(div64_s64(drm_int2fixp(coeff->bt), 1000), temperature)));
0753     kv = drm_fixp_mul(div64_s64(drm_int2fixp(coeff->av), 1000),
0754               drm_fixp_exp(drm_fixp_mul(div64_s64(drm_int2fixp(coeff->bv), 1000), vddc)));
0755
0756     leakage_w = drm_fixp_mul(drm_fixp_mul(drm_fixp_mul(i_leakage, kt), kv), vddc);
0757
0758     *leakage = drm_fixp2int(leakage_w * 1000);
0759 }
0760
0761 static void ni_calculate_leakage_for_v_and_t(struct radeon_device *rdev,
0762                          const struct ni_leakage_coeffients *coeff,
0763                          u16 v,
0764                          s32 t,
0765                          u32 i_leakage,
0766                          u32 *leakage)
0767 {
0768     ni_calculate_leakage_for_v_and_t_formula(coeff, v, t, i_leakage, leakage);
0769 }
0770
0771 bool ni_dpm_vblank_too_short(struct radeon_device *rdev)
0772 {
0773     struct rv7xx_power_info *pi = rv770_get_pi(rdev);
0774     u32 vblank_time = r600_dpm_get_vblank_time(rdev);
0775     /* we never hit the non-gddr5 limit so disable it */
0776     u32 switch_limit = pi->mem_gddr5 ? 450 : 0;
0777
0778     if (vblank_time < switch_limit)
0779         return true;
0780     else
0781         return false;
0782
0783 }
0784
0785 static void ni_apply_state_adjust_rules(struct radeon_device *rdev,
0786                     struct radeon_ps *rps)
0787 {
0788     struct ni_ps *ps = ni_get_ps(rps);
0789     struct radeon_clock_and_voltage_limits *max_limits;
0790     bool disable_mclk_switching;
0791     u32 mclk;
0792     u16 vddci;
0793     int i;
0794
0795     if ((rdev->pm.dpm.new_active_crtc_count > 1) ||
0796         ni_dpm_vblank_too_short(rdev))
0797         disable_mclk_switching = true;
0798     else
0799         disable_mclk_switching = false;
0800
0801     if (rdev->pm.dpm.ac_power)
0802         max_limits = &rdev->pm.dpm.dyn_state.max_clock_voltage_on_ac;
0803     else
0804         max_limits = &rdev->pm.dpm.dyn_state.max_clock_voltage_on_dc;
0805
0806     if (rdev->pm.dpm.ac_power == false) {
0807         for (i = 0; i < ps->performance_level_count; i++) {
0808             if (ps->performance_levels[i].mclk > max_limits->mclk)
0809                 ps->performance_levels[i].mclk = max_limits->mclk;
0810             if (ps->performance_levels[i].sclk > max_limits->sclk)
0811                 ps->performance_levels[i].sclk = max_limits->sclk;
0812             if (ps->performance_levels[i].vddc > max_limits->vddc)
0813                 ps->performance_levels[i].vddc = max_limits->vddc;
0814             if (ps->performance_levels[i].vddci > max_limits->vddci)
0815                 ps->performance_levels[i].vddci = max_limits->vddci;
0816         }
0817     }
0818
0819     /* XXX validate the min clocks required for display */
0820
0821     /* adjust low state */
0822     if (disable_mclk_switching) {
0823         ps->performance_levels[0].mclk =
0824             ps->performance_levels[ps->performance_level_count - 1].mclk;
0825         ps->performance_levels[0].vddci =
0826             ps->performance_levels[ps->performance_level_count - 1].vddci;
0827     }
0828
0829     btc_skip_blacklist_clocks(rdev, max_limits->sclk, max_limits->mclk,
0830                   &ps->performance_levels[0].sclk,
0831                   &ps->performance_levels[0].mclk);
0832
0833     for (i = 1; i < ps->performance_level_count; i++) {
0834         if (ps->performance_levels[i].sclk < ps->performance_levels[i - 1].sclk)
0835             ps->performance_levels[i].sclk = ps->performance_levels[i - 1].sclk;
0836         if (ps->performance_levels[i].vddc < ps->performance_levels[i - 1].vddc)
0837             ps->performance_levels[i].vddc = ps->performance_levels[i - 1].vddc;
0838     }
0839
0840     /* adjust remaining states */
0841     if (disable_mclk_switching) {
0842         mclk = ps->performance_levels[0].mclk;
0843         vddci = ps->performance_levels[0].vddci;
0844         for (i = 1; i < ps->performance_level_count; i++) {
0845             if (mclk < ps->performance_levels[i].mclk)
0846                 mclk = ps->performance_levels[i].mclk;
0847             if (vddci < ps->performance_levels[i].vddci)
0848                 vddci = ps->performance_levels[i].vddci;
0849         }
0850         for (i = 0; i < ps->performance_level_count; i++) {
0851             ps->performance_levels[i].mclk = mclk;
0852             ps->performance_levels[i].vddci = vddci;
0853         }
0854     } else {
0855         for (i = 1; i < ps->performance_level_count; i++) {
0856             if (ps->performance_levels[i].mclk < ps->performance_levels[i - 1].mclk)
0857                 ps->performance_levels[i].mclk = ps->performance_levels[i - 1].mclk;
0858             if (ps->performance_levels[i].vddci < ps->performance_levels[i - 1].vddci)
0859                 ps->performance_levels[i].vddci = ps->performance_levels[i - 1].vddci;
0860         }
0861     }
0862
0863     for (i = 1; i < ps->performance_level_count; i++)
0864         btc_skip_blacklist_clocks(rdev, max_limits->sclk, max_limits->mclk,
0865                       &ps->performance_levels[i].sclk,
0866                       &ps->performance_levels[i].mclk);
0867
0868     for (i = 0; i < ps->performance_level_count; i++)
0869         btc_adjust_clock_combinations(rdev, max_limits,
0870                           &ps->performance_levels[i]);
0871
0872     for (i = 0; i < ps->performance_level_count; i++) {
0873         btc_apply_voltage_dependency_rules(&rdev->pm.dpm.dyn_state.vddc_dependency_on_sclk,
0874                            ps->performance_levels[i].sclk,
0875                            max_limits->vddc,  &ps->performance_levels[i].vddc);
0876         btc_apply_voltage_dependency_rules(&rdev->pm.dpm.dyn_state.vddci_dependency_on_mclk,
0877                            ps->performance_levels[i].mclk,
0878                            max_limits->vddci, &ps->performance_levels[i].vddci);
0879         btc_apply_voltage_dependency_rules(&rdev->pm.dpm.dyn_state.vddc_dependency_on_mclk,
0880                            ps->performance_levels[i].mclk,
0881                            max_limits->vddc,  &ps->performance_levels[i].vddc);
0882         btc_apply_voltage_dependency_rules(&rdev->pm.dpm.dyn_state.vddc_dependency_on_dispclk,
0883                            rdev->clock.current_dispclk,
0884                            max_limits->vddc,  &ps->performance_levels[i].vddc);
0885     }
0886
0887     for (i = 0; i < ps->performance_level_count; i++) {
0888         btc_apply_voltage_delta_rules(rdev,
0889                           max_limits->vddc, max_limits->vddci,
0890                           &ps->performance_levels[i].vddc,
0891                           &ps->performance_levels[i].vddci);
0892     }
0893
0894     ps->dc_compatible = true;
0895     for (i = 0; i < ps->performance_level_count; i++) {
0896         if (ps->performance_levels[i].vddc > rdev->pm.dpm.dyn_state.max_clock_voltage_on_dc.vddc)
0897             ps->dc_compatible = false;
0898
0899         if (ps->performance_levels[i].vddc < rdev->pm.dpm.dyn_state.min_vddc_for_pcie_gen2)
0900             ps->performance_levels[i].flags &= ~ATOM_PPLIB_R600_FLAGS_PCIEGEN2;
0901     }
0902 }
0903
0904 static void ni_cg_clockgating_default(struct radeon_device *rdev)
0905 {
0906     u32 count;
0907     const u32 *ps = NULL;
0908
0909     ps = (const u32 *)&cayman_cgcg_cgls_default;
0910     count = CAYMAN_CGCG_CGLS_DEFAULT_LENGTH;
0911
0912     btc_program_mgcg_hw_sequence(rdev, ps, count);
0913 }
0914
0915 static void ni_gfx_clockgating_enable(struct radeon_device *rdev,
0916                       bool enable)
0917 {
0918     u32 count;
0919     const u32 *ps = NULL;
0920
0921     if (enable) {
0922         ps = (const u32 *)&cayman_cgcg_cgls_enable;
0923         count = CAYMAN_CGCG_CGLS_ENABLE_LENGTH;
0924     } else {
0925         ps = (const u32 *)&cayman_cgcg_cgls_disable;
0926         count = CAYMAN_CGCG_CGLS_DISABLE_LENGTH;
0927     }
0928
0929     btc_program_mgcg_hw_sequence(rdev, ps, count);
0930 }
0931
0932 static void ni_mg_clockgating_default(struct radeon_device *rdev)
0933 {
0934     u32 count;
0935     const u32 *ps = NULL;
0936
0937     ps = (const u32 *)&cayman_mgcg_default;
0938     count = CAYMAN_MGCG_DEFAULT_LENGTH;
0939
0940     btc_program_mgcg_hw_sequence(rdev, ps, count);
0941 }
0942
0943 static void ni_mg_clockgating_enable(struct radeon_device *rdev,
0944                      bool enable)
0945 {
0946     u32 count;
0947     const u32 *ps = NULL;
0948
0949     if (enable) {
0950         ps = (const u32 *)&cayman_mgcg_enable;
0951         count = CAYMAN_MGCG_ENABLE_LENGTH;
0952     } else {
0953         ps = (const u32 *)&cayman_mgcg_disable;
0954         count = CAYMAN_MGCG_DISABLE_LENGTH;
0955     }
0956
0957     btc_program_mgcg_hw_sequence(rdev, ps, count);
0958 }
0959
0960 static void ni_ls_clockgating_default(struct radeon_device *rdev)
0961 {
0962     u32 count;
0963     const u32 *ps = NULL;
0964
0965     ps = (const u32 *)&cayman_sysls_default;
0966     count = CAYMAN_SYSLS_DEFAULT_LENGTH;
0967
0968     btc_program_mgcg_hw_sequence(rdev, ps, count);
0969 }
0970
0971 static void ni_ls_clockgating_enable(struct radeon_device *rdev,
0972                      bool enable)
0973 {
0974     u32 count;
0975     const u32 *ps = NULL;
0976
0977     if (enable) {
0978         ps = (const u32 *)&cayman_sysls_enable;
0979         count = CAYMAN_SYSLS_ENABLE_LENGTH;
0980     } else {
0981         ps = (const u32 *)&cayman_sysls_disable;
0982         count = CAYMAN_SYSLS_DISABLE_LENGTH;
0983     }
0984
0985     btc_program_mgcg_hw_sequence(rdev, ps, count);
0986
0987 }
0988
0989 static int ni_patch_single_dependency_table_based_on_leakage(struct radeon_device *rdev,
0990                                  struct radeon_clock_voltage_dependency_table *table)
0991 {
0992     struct rv7xx_power_info *pi = rv770_get_pi(rdev);
0993     u32 i;
0994
0995     if (table) {
0996         for (i = 0; i < table->count; i++) {
0997             if (0xff01 == table->entries[i].v) {
0998                 if (pi->max_vddc == 0)
0999                     return -EINVAL;
1000                 table->entries[i].v = pi->max_vddc;
1001             }
1002         }
1003     }
1004     return 0;
1005 }
1006
1007 static int ni_patch_dependency_tables_based_on_leakage(struct radeon_device *rdev)
1008 {
1009     int ret = 0;
1010
1011     ret = ni_patch_single_dependency_table_based_on_leakage(rdev,
1012                                 &rdev->pm.dpm.dyn_state.vddc_dependency_on_sclk);
1013
1014     ret = ni_patch_single_dependency_table_based_on_leakage(rdev,
1015                                 &rdev->pm.dpm.dyn_state.vddc_dependency_on_mclk);
1016     return ret;
1017 }
1018
1019 static void ni_stop_dpm(struct radeon_device *rdev)
1020 {
1021     WREG32_P(GENERAL_PWRMGT, 0, ~GLOBAL_PWRMGT_EN);
1022 }
1023
1024 #if 0
1025 static int ni_notify_hw_of_power_source(struct radeon_device *rdev,
1026                     bool ac_power)
1027 {
1028     if (ac_power)
1029         return (rv770_send_msg_to_smc(rdev, PPSMC_MSG_RunningOnAC) == PPSMC_Result_OK) ?
1030             0 : -EINVAL;
1031
1032     return 0;
1033 }
1034 #endif
1035
1036 static PPSMC_Result ni_send_msg_to_smc_with_parameter(struct radeon_device *rdev,
1037                               PPSMC_Msg msg, u32 parameter)
1038 {
1039     WREG32(SMC_SCRATCH0, parameter);
1040     return rv770_send_msg_to_smc(rdev, msg);
1041 }
1042
1043 static int ni_restrict_performance_levels_before_switch(struct radeon_device *rdev)
1044 {
1045     if (rv770_send_msg_to_smc(rdev, PPSMC_MSG_NoForcedLevel) != PPSMC_Result_OK)
1046         return -EINVAL;
1047
1048     return (ni_send_msg_to_smc_with_parameter(rdev, PPSMC_MSG_SetEnabledLevels, 1) == PPSMC_Result_OK) ?
1049         0 : -EINVAL;
1050 }
1051
1052 int ni_dpm_force_performance_level(struct radeon_device *rdev,
1053                    enum radeon_dpm_forced_level level)
1054 {
1055     if (level == RADEON_DPM_FORCED_LEVEL_HIGH) {
1056         if (ni_send_msg_to_smc_with_parameter(rdev, PPSMC_MSG_SetEnabledLevels, 0) != PPSMC_Result_OK)
1057             return -EINVAL;
1058
1059         if (ni_send_msg_to_smc_with_parameter(rdev, PPSMC_MSG_SetForcedLevels, 1) != PPSMC_Result_OK)
1060             return -EINVAL;
1061     } else if (level == RADEON_DPM_FORCED_LEVEL_LOW) {
1062         if (ni_send_msg_to_smc_with_parameter(rdev, PPSMC_MSG_SetForcedLevels, 0) != PPSMC_Result_OK)
1063             return -EINVAL;
1064
1065         if (ni_send_msg_to_smc_with_parameter(rdev, PPSMC_MSG_SetEnabledLevels, 1) != PPSMC_Result_OK)
1066             return -EINVAL;
1067     } else if (level == RADEON_DPM_FORCED_LEVEL_AUTO) {
1068         if (ni_send_msg_to_smc_with_parameter(rdev, PPSMC_MSG_SetForcedLevels, 0) != PPSMC_Result_OK)
1069             return -EINVAL;
1070
1071         if (ni_send_msg_to_smc_with_parameter(rdev, PPSMC_MSG_SetEnabledLevels, 0) != PPSMC_Result_OK)
1072             return -EINVAL;
1073     }
1074
1075     rdev->pm.dpm.forced_level = level;
1076
1077     return 0;
1078 }
1079
1080 static void ni_stop_smc(struct radeon_device *rdev)
1081 {
1082     u32 tmp;
1083     int i;
1084
1085     for (i = 0; i < rdev->usec_timeout; i++) {
1086         tmp = RREG32(LB_SYNC_RESET_SEL) & LB_SYNC_RESET_SEL_MASK;
1087         if (tmp != 1)
1088             break;
1089         udelay(1);
1090     }
1091
1092     udelay(100);
1093
1094     r7xx_stop_smc(rdev);
1095 }
1096
1097 static int ni_process_firmware_header(struct radeon_device *rdev)
1098 {
1099     struct rv7xx_power_info *pi = rv770_get_pi(rdev);
1100     struct evergreen_power_info *eg_pi = evergreen_get_pi(rdev);
1101     struct ni_power_info *ni_pi = ni_get_pi(rdev);
1102     u32 tmp;
1103     int ret;
1104
1105     ret = rv770_read_smc_sram_dword(rdev,
1106                     NISLANDS_SMC_FIRMWARE_HEADER_LOCATION +
1107                     NISLANDS_SMC_FIRMWARE_HEADER_stateTable,
1108                     &tmp, pi->sram_end);
1109
1110     if (ret)
1111         return ret;
1112
1113     pi->state_table_start = (u16)tmp;
1114
1115     ret = rv770_read_smc_sram_dword(rdev,
1116                     NISLANDS_SMC_FIRMWARE_HEADER_LOCATION +
1117                     NISLANDS_SMC_FIRMWARE_HEADER_softRegisters,
1118                     &tmp, pi->sram_end);
1119
1120     if (ret)
1121         return ret;
1122
1123     pi->soft_regs_start = (u16)tmp;
1124
1125     ret = rv770_read_smc_sram_dword(rdev,
1126                     NISLANDS_SMC_FIRMWARE_HEADER_LOCATION +
1127                     NISLANDS_SMC_FIRMWARE_HEADER_mcRegisterTable,
1128                     &tmp, pi->sram_end);
1129
1130     if (ret)
1131         return ret;
1132
1133     eg_pi->mc_reg_table_start = (u16)tmp;
1134
1135     ret = rv770_read_smc_sram_dword(rdev,
1136                     NISLANDS_SMC_FIRMWARE_HEADER_LOCATION +
1137                     NISLANDS_SMC_FIRMWARE_HEADER_fanTable,
1138                     &tmp, pi->sram_end);
1139
1140     if (ret)
1141         return ret;
1142
1143     ni_pi->fan_table_start = (u16)tmp;
1144
1145     ret = rv770_read_smc_sram_dword(rdev,
1146                     NISLANDS_SMC_FIRMWARE_HEADER_LOCATION +
1147                     NISLANDS_SMC_FIRMWARE_HEADER_mcArbDramAutoRefreshTable,
1148                     &tmp, pi->sram_end);
1149
1150     if (ret)
1151         return ret;
1152
1153     ni_pi->arb_table_start = (u16)tmp;
1154
1155     ret = rv770_read_smc_sram_dword(rdev,
1156                     NISLANDS_SMC_FIRMWARE_HEADER_LOCATION +
1157                     NISLANDS_SMC_FIRMWARE_HEADER_cacTable,
1158                     &tmp, pi->sram_end);
1159
1160     if (ret)
1161         return ret;
1162
1163     ni_pi->cac_table_start = (u16)tmp;
1164
1165     ret = rv770_read_smc_sram_dword(rdev,
1166                     NISLANDS_SMC_FIRMWARE_HEADER_LOCATION +
1167                     NISLANDS_SMC_FIRMWARE_HEADER_spllTable,
1168                     &tmp, pi->sram_end);
1169
1170     if (ret)
1171         return ret;
1172
1173     ni_pi->spll_table_start = (u16)tmp;
1174
1175
1176     return ret;
1177 }
1178
1179 static void ni_read_clock_registers(struct radeon_device *rdev)
1180 {
1181     struct ni_power_info *ni_pi = ni_get_pi(rdev);
1182
1183     ni_pi->clock_registers.cg_spll_func_cntl = RREG32(CG_SPLL_FUNC_CNTL);
1184     ni_pi->clock_registers.cg_spll_func_cntl_2 = RREG32(CG_SPLL_FUNC_CNTL_2);
1185     ni_pi->clock_registers.cg_spll_func_cntl_3 = RREG32(CG_SPLL_FUNC_CNTL_3);
1186     ni_pi->clock_registers.cg_spll_func_cntl_4 = RREG32(CG_SPLL_FUNC_CNTL_4);
1187     ni_pi->clock_registers.cg_spll_spread_spectrum = RREG32(CG_SPLL_SPREAD_SPECTRUM);
1188     ni_pi->clock_registers.cg_spll_spread_spectrum_2 = RREG32(CG_SPLL_SPREAD_SPECTRUM_2);
1189     ni_pi->clock_registers.mpll_ad_func_cntl = RREG32(MPLL_AD_FUNC_CNTL);
1190     ni_pi->clock_registers.mpll_ad_func_cntl_2 = RREG32(MPLL_AD_FUNC_CNTL_2);
1191     ni_pi->clock_registers.mpll_dq_func_cntl = RREG32(MPLL_DQ_FUNC_CNTL);
1192     ni_pi->clock_registers.mpll_dq_func_cntl_2 = RREG32(MPLL_DQ_FUNC_CNTL_2);
1193     ni_pi->clock_registers.mclk_pwrmgt_cntl = RREG32(MCLK_PWRMGT_CNTL);
1194     ni_pi->clock_registers.dll_cntl = RREG32(DLL_CNTL);
1195     ni_pi->clock_registers.mpll_ss1 = RREG32(MPLL_SS1);
1196     ni_pi->clock_registers.mpll_ss2 = RREG32(MPLL_SS2);
1197 }
1198
1199 #if 0
1200 static int ni_enter_ulp_state(struct radeon_device *rdev)
1201 {
1202     struct rv7xx_power_info *pi = rv770_get_pi(rdev);
1203
1204     if (pi->gfx_clock_gating) {
1205         WREG32_P(SCLK_PWRMGT_CNTL, 0, ~DYN_GFX_CLK_OFF_EN);
1206         WREG32_P(SCLK_PWRMGT_CNTL, GFX_CLK_FORCE_ON, ~GFX_CLK_FORCE_ON);
1207         WREG32_P(SCLK_PWRMGT_CNTL, 0, ~GFX_CLK_FORCE_ON);
1208         RREG32(GB_ADDR_CONFIG);
1209     }
1210
1211     WREG32_P(SMC_MSG, HOST_SMC_MSG(PPSMC_MSG_SwitchToMinimumPower),
1212          ~HOST_SMC_MSG_MASK);
1213
1214     udelay(25000);
1215
1216     return 0;
1217 }
1218 #endif
1219
1220 static void ni_program_response_times(struct radeon_device *rdev)
1221 {
1222     u32 voltage_response_time, backbias_response_time, acpi_delay_time, vbi_time_out;
1223     u32 vddc_dly, bb_dly, acpi_dly, vbi_dly, mclk_switch_limit;
1224     u32 reference_clock;
1225
1226     rv770_write_smc_soft_register(rdev, NI_SMC_SOFT_REGISTER_mvdd_chg_time, 1);
1227
1228     voltage_response_time = (u32)rdev->pm.dpm.voltage_response_time;
1229     backbias_response_time = (u32)rdev->pm.dpm.backbias_response_time;
1230
1231     if (voltage_response_time == 0)
1232         voltage_response_time = 1000;
1233
1234     if (backbias_response_time == 0)
1235         backbias_response_time = 1000;
1236
1237     acpi_delay_time = 15000;
1238     vbi_time_out = 100000;
1239
1240     reference_clock = radeon_get_xclk(rdev);
1241
1242     vddc_dly = (voltage_response_time  * reference_clock) / 1600;
1243     bb_dly   = (backbias_response_time * reference_clock) / 1600;
1244     acpi_dly = (acpi_delay_time * reference_clock) / 1600;
1245     vbi_dly  = (vbi_time_out * reference_clock) / 1600;
1246
1247     mclk_switch_limit = (460 * reference_clock) / 100;
1248
1249     rv770_write_smc_soft_register(rdev, NI_SMC_SOFT_REGISTER_delay_vreg,  vddc_dly);
1250     rv770_write_smc_soft_register(rdev, NI_SMC_SOFT_REGISTER_delay_bbias, bb_dly);
1251     rv770_write_smc_soft_register(rdev, NI_SMC_SOFT_REGISTER_delay_acpi,  acpi_dly);
1252     rv770_write_smc_soft_register(rdev, NI_SMC_SOFT_REGISTER_mclk_chg_timeout, vbi_dly);
1253     rv770_write_smc_soft_register(rdev, NI_SMC_SOFT_REGISTER_mc_block_delay, 0xAA);
1254     rv770_write_smc_soft_register(rdev, NI_SMC_SOFT_REGISTER_mclk_switch_lim, mclk_switch_limit);
1255 }
1256
1257 static void ni_populate_smc_voltage_table(struct radeon_device *rdev,
1258                       struct atom_voltage_table *voltage_table,
1259                       NISLANDS_SMC_STATETABLE *table)
1260 {
1261     unsigned int i;
1262
1263     for (i = 0; i < voltage_table->count; i++) {
1264         table->highSMIO[i] = 0;
1265         table->lowSMIO[i] |= cpu_to_be32(voltage_table->entries[i].smio_low);
1266     }
1267 }
1268
1269 static void ni_populate_smc_voltage_tables(struct radeon_device *rdev,
1270                        NISLANDS_SMC_STATETABLE *table)
1271 {
1272     struct rv7xx_power_info *pi = rv770_get_pi(rdev);
1273     struct evergreen_power_info *eg_pi = evergreen_get_pi(rdev);
1274     unsigned char i;
1275
1276     if (eg_pi->vddc_voltage_table.count) {
1277         ni_populate_smc_voltage_table(rdev, &eg_pi->vddc_voltage_table, table);
1278         table->voltageMaskTable.highMask[NISLANDS_SMC_VOLTAGEMASK_VDDC] = 0;
1279         table->voltageMaskTable.lowMask[NISLANDS_SMC_VOLTAGEMASK_VDDC] =
1280             cpu_to_be32(eg_pi->vddc_voltage_table.mask_low);
1281
1282         for (i = 0; i < eg_pi->vddc_voltage_table.count; i++) {
1283             if (pi->max_vddc_in_table <= eg_pi->vddc_voltage_table.entries[i].value) {
1284                 table->maxVDDCIndexInPPTable = i;
1285                 break;
1286             }
1287         }
1288     }
1289
1290     if (eg_pi->vddci_voltage_table.count) {
1291         ni_populate_smc_voltage_table(rdev, &eg_pi->vddci_voltage_table, table);
1292
1293         table->voltageMaskTable.highMask[NISLANDS_SMC_VOLTAGEMASK_VDDCI] = 0;
1294         table->voltageMaskTable.lowMask[NISLANDS_SMC_VOLTAGEMASK_VDDCI] =
1295             cpu_to_be32(eg_pi->vddci_voltage_table.mask_low);
1296     }
1297 }
1298
1299 static int ni_populate_voltage_value(struct radeon_device *rdev,
1300                      struct atom_voltage_table *table,
1301                      u16 value,
1302                      NISLANDS_SMC_VOLTAGE_VALUE *voltage)
1303 {
1304     unsigned int i;
1305
1306     for (i = 0; i < table->count; i++) {
1307         if (value <= table->entries[i].value) {
1308             voltage->index = (u8)i;
1309             voltage->value = cpu_to_be16(table->entries[i].value);
1310             break;
1311         }
1312     }
1313
1314     if (i >= table->count)
1315         return -EINVAL;
1316
1317     return 0;
1318 }
1319
1320 static void ni_populate_mvdd_value(struct radeon_device *rdev,
1321                    u32 mclk,
1322                    NISLANDS_SMC_VOLTAGE_VALUE *voltage)
1323 {
1324     struct rv7xx_power_info *pi = rv770_get_pi(rdev);
1325     struct evergreen_power_info *eg_pi = evergreen_get_pi(rdev);
1326
1327     if (!pi->mvdd_control) {
1328         voltage->index = eg_pi->mvdd_high_index;
1329         voltage->value = cpu_to_be16(MVDD_HIGH_VALUE);
1330         return;
1331     }
1332
1333     if (mclk <= pi->mvdd_split_frequency) {
1334         voltage->index = eg_pi->mvdd_low_index;
1335         voltage->value = cpu_to_be16(MVDD_LOW_VALUE);
1336     } else {
1337         voltage->index = eg_pi->mvdd_high_index;
1338         voltage->value = cpu_to_be16(MVDD_HIGH_VALUE);
1339     }
1340 }
1341
1342 static int ni_get_std_voltage_value(struct radeon_device *rdev,
1343                     NISLANDS_SMC_VOLTAGE_VALUE *voltage,
1344                     u16 *std_voltage)
1345 {
1346     if (rdev->pm.dpm.dyn_state.cac_leakage_table.entries &&
1347         ((u32)voltage->index < rdev->pm.dpm.dyn_state.cac_leakage_table.count))
1348         *std_voltage = rdev->pm.dpm.dyn_state.cac_leakage_table.entries[voltage->index].vddc;
1349     else
1350         *std_voltage = be16_to_cpu(voltage->value);
1351
1352     return 0;
1353 }
1354
1355 static void ni_populate_std_voltage_value(struct radeon_device *rdev,
1356                       u16 value, u8 index,
1357                       NISLANDS_SMC_VOLTAGE_VALUE *voltage)
1358 {
1359     voltage->index = index;
1360     voltage->value = cpu_to_be16(value);
1361 }
1362
1363 static u32 ni_get_smc_power_scaling_factor(struct radeon_device *rdev)
1364 {
1365     u32 xclk_period;
1366     u32 xclk = radeon_get_xclk(rdev);
1367     u32 tmp = RREG32(CG_CAC_CTRL) & TID_CNT_MASK;
1368
1369     xclk_period = (1000000000UL / xclk);
1370     xclk_period /= 10000UL;
1371
1372     return tmp * xclk_period;
1373 }
1374
1375 static u32 ni_scale_power_for_smc(u32 power_in_watts, u32 scaling_factor)
1376 {
1377     return (power_in_watts * scaling_factor) << 2;
1378 }
1379
1380 static u32 ni_calculate_power_boost_limit(struct radeon_device *rdev,
1381                       struct radeon_ps *radeon_state,
1382                       u32 near_tdp_limit)
1383 {
1384     struct ni_ps *state = ni_get_ps(radeon_state);
1385     struct evergreen_power_info *eg_pi = evergreen_get_pi(rdev);
1386     struct ni_power_info *ni_pi = ni_get_pi(rdev);
1387     u32 power_boost_limit = 0;
1388     int ret;
1389
1390     if (ni_pi->enable_power_containment &&
1391         ni_pi->use_power_boost_limit) {
1392         NISLANDS_SMC_VOLTAGE_VALUE vddc;
1393         u16 std_vddc_med;
1394         u16 std_vddc_high;
1395         u64 tmp, n, d;
1396
1397         if (state->performance_level_count < 3)
1398             return 0;
1399
1400         ret = ni_populate_voltage_value(rdev, &eg_pi->vddc_voltage_table,
1401                         state->performance_levels[state->performance_level_count - 2].vddc,
1402                         &vddc);
1403         if (ret)
1404             return 0;
1405
1406         ret = ni_get_std_voltage_value(rdev, &vddc, &std_vddc_med);
1407         if (ret)
1408             return 0;
1409
1410         ret = ni_populate_voltage_value(rdev, &eg_pi->vddc_voltage_table,
1411                         state->performance_levels[state->performance_level_count - 1].vddc,
1412                         &vddc);
1413         if (ret)
1414             return 0;
1415
1416         ret = ni_get_std_voltage_value(rdev, &vddc, &std_vddc_high);
1417         if (ret)
1418             return 0;
1419
1420         n = ((u64)near_tdp_limit * ((u64)std_vddc_med * (u64)std_vddc_med) * 90);
1421         d = ((u64)std_vddc_high * (u64)std_vddc_high * 100);
1422         tmp = div64_u64(n, d);
1423
1424         if (tmp >> 32)
1425             return 0;
1426         power_boost_limit = (u32)tmp;
1427     }
1428
1429     return power_boost_limit;
1430 }
1431
1432 static int ni_calculate_adjusted_tdp_limits(struct radeon_device *rdev,
1433                         bool adjust_polarity,
1434                         u32 tdp_adjustment,
1435                         u32 *tdp_limit,
1436                         u32 *near_tdp_limit)
1437 {
1438     if (tdp_adjustment > (u32)rdev->pm.dpm.tdp_od_limit)
1439         return -EINVAL;
1440
1441     if (adjust_polarity) {
1442         *tdp_limit = ((100 + tdp_adjustment) * rdev->pm.dpm.tdp_limit) / 100;
1443         *near_tdp_limit = rdev->pm.dpm.near_tdp_limit + (*tdp_limit - rdev->pm.dpm.tdp_limit);
1444     } else {
1445         *tdp_limit = ((100 - tdp_adjustment) * rdev->pm.dpm.tdp_limit) / 100;
1446         *near_tdp_limit = rdev->pm.dpm.near_tdp_limit - (rdev->pm.dpm.tdp_limit - *tdp_limit);
1447     }
1448
1449     return 0;
1450 }
1451
1452 static int ni_populate_smc_tdp_limits(struct radeon_device *rdev,
1453                       struct radeon_ps *radeon_state)
1454 {
1455     struct rv7xx_power_info *pi = rv770_get_pi(rdev);
1456     struct ni_power_info *ni_pi = ni_get_pi(rdev);
1457
1458     if (ni_pi->enable_power_containment) {
1459         NISLANDS_SMC_STATETABLE *smc_table = &ni_pi->smc_statetable;
1460         u32 scaling_factor = ni_get_smc_power_scaling_factor(rdev);
1461         u32 tdp_limit;
1462         u32 near_tdp_limit;
1463         u32 power_boost_limit;
1464         int ret;
1465
1466         if (scaling_factor == 0)
1467             return -EINVAL;
1468
1469         memset(smc_table, 0, sizeof(NISLANDS_SMC_STATETABLE));
1470
1471         ret = ni_calculate_adjusted_tdp_limits(rdev,
1472                                false, /* ??? */
1473                                rdev->pm.dpm.tdp_adjustment,
1474                                &tdp_limit,
1475                                &near_tdp_limit);
1476         if (ret)
1477             return ret;
1478
1479         power_boost_limit = ni_calculate_power_boost_limit(rdev, radeon_state,
1480                                    near_tdp_limit);
1481
1482         smc_table->dpm2Params.TDPLimit =
1483             cpu_to_be32(ni_scale_power_for_smc(tdp_limit, scaling_factor));
1484         smc_table->dpm2Params.NearTDPLimit =
1485             cpu_to_be32(ni_scale_power_for_smc(near_tdp_limit, scaling_factor));
1486         smc_table->dpm2Params.SafePowerLimit =
1487             cpu_to_be32(ni_scale_power_for_smc((near_tdp_limit * NISLANDS_DPM2_TDP_SAFE_LIMIT_PERCENT) / 100,
1488                                scaling_factor));
1489         smc_table->dpm2Params.PowerBoostLimit =
1490             cpu_to_be32(ni_scale_power_for_smc(power_boost_limit, scaling_factor));
1491
1492         ret = rv770_copy_bytes_to_smc(rdev,
1493                           (u16)(pi->state_table_start + offsetof(NISLANDS_SMC_STATETABLE, dpm2Params) +
1494                             offsetof(PP_NIslands_DPM2Parameters, TDPLimit)),
1495                           (u8 *)(&smc_table->dpm2Params.TDPLimit),
1496                           sizeof(u32) * 4, pi->sram_end);
1497         if (ret)
1498             return ret;
1499     }
1500
1501     return 0;
1502 }
1503
1504 int ni_copy_and_switch_arb_sets(struct radeon_device *rdev,
1505                 u32 arb_freq_src, u32 arb_freq_dest)
1506 {
1507     u32 mc_arb_dram_timing;
1508     u32 mc_arb_dram_timing2;
1509     u32 burst_time;
1510     u32 mc_cg_config;
1511
1512     switch (arb_freq_src) {
1513     case MC_CG_ARB_FREQ_F0:
1514         mc_arb_dram_timing  = RREG32(MC_ARB_DRAM_TIMING);
1515         mc_arb_dram_timing2 = RREG32(MC_ARB_DRAM_TIMING2);
1516         burst_time = (RREG32(MC_ARB_BURST_TIME) & STATE0_MASK) >> STATE0_SHIFT;
1517         break;
1518     case MC_CG_ARB_FREQ_F1:
1519         mc_arb_dram_timing  = RREG32(MC_ARB_DRAM_TIMING_1);
1520         mc_arb_dram_timing2 = RREG32(MC_ARB_DRAM_TIMING2_1);
1521         burst_time = (RREG32(MC_ARB_BURST_TIME) & STATE1_MASK) >> STATE1_SHIFT;
1522         break;
1523     case MC_CG_ARB_FREQ_F2:
1524         mc_arb_dram_timing  = RREG32(MC_ARB_DRAM_TIMING_2);
1525         mc_arb_dram_timing2 = RREG32(MC_ARB_DRAM_TIMING2_2);
1526         burst_time = (RREG32(MC_ARB_BURST_TIME) & STATE2_MASK) >> STATE2_SHIFT;
1527         break;
1528     case MC_CG_ARB_FREQ_F3:
1529         mc_arb_dram_timing  = RREG32(MC_ARB_DRAM_TIMING_3);
1530         mc_arb_dram_timing2 = RREG32(MC_ARB_DRAM_TIMING2_3);
1531         burst_time = (RREG32(MC_ARB_BURST_TIME) & STATE3_MASK) >> STATE3_SHIFT;
1532         break;
1533     default:
1534         return -EINVAL;
1535     }
1536
1537     switch (arb_freq_dest) {
1538     case MC_CG_ARB_FREQ_F0:
1539         WREG32(MC_ARB_DRAM_TIMING, mc_arb_dram_timing);
1540         WREG32(MC_ARB_DRAM_TIMING2, mc_arb_dram_timing2);
1541         WREG32_P(MC_ARB_BURST_TIME, STATE0(burst_time), ~STATE0_MASK);
1542         break;
1543     case MC_CG_ARB_FREQ_F1:
1544         WREG32(MC_ARB_DRAM_TIMING_1, mc_arb_dram_timing);
1545         WREG32(MC_ARB_DRAM_TIMING2_1, mc_arb_dram_timing2);
1546         WREG32_P(MC_ARB_BURST_TIME, STATE1(burst_time), ~STATE1_MASK);
1547         break;
1548     case MC_CG_ARB_FREQ_F2:
1549         WREG32(MC_ARB_DRAM_TIMING_2, mc_arb_dram_timing);
1550         WREG32(MC_ARB_DRAM_TIMING2_2, mc_arb_dram_timing2);
1551         WREG32_P(MC_ARB_BURST_TIME, STATE2(burst_time), ~STATE2_MASK);
1552         break;
1553     case MC_CG_ARB_FREQ_F3:
1554         WREG32(MC_ARB_DRAM_TIMING_3, mc_arb_dram_timing);
1555         WREG32(MC_ARB_DRAM_TIMING2_3, mc_arb_dram_timing2);
1556         WREG32_P(MC_ARB_BURST_TIME, STATE3(burst_time), ~STATE3_MASK);
1557         break;
1558     default:
1559         return -EINVAL;
1560     }
1561
1562     mc_cg_config = RREG32(MC_CG_CONFIG) | 0x0000000F;
1563     WREG32(MC_CG_CONFIG, mc_cg_config);
1564     WREG32_P(MC_ARB_CG, CG_ARB_REQ(arb_freq_dest), ~CG_ARB_REQ_MASK);
1565
1566     return 0;
1567 }
1568
1569 static int ni_init_arb_table_index(struct radeon_device *rdev)
1570 {
1571     struct rv7xx_power_info *pi = rv770_get_pi(rdev);
1572     struct ni_power_info *ni_pi = ni_get_pi(rdev);
1573     u32 tmp;
1574     int ret;
1575
1576     ret = rv770_read_smc_sram_dword(rdev, ni_pi->arb_table_start,
1577                     &tmp, pi->sram_end);
1578     if (ret)
1579         return ret;
1580
1581     tmp &= 0x00FFFFFF;
1582     tmp |= ((u32)MC_CG_ARB_FREQ_F1) << 24;
1583
1584     return rv770_write_smc_sram_dword(rdev, ni_pi->arb_table_start,
1585                       tmp, pi->sram_end);
1586 }
1587
1588 static int ni_initial_switch_from_arb_f0_to_f1(struct radeon_device *rdev)
1589 {
1590     return ni_copy_and_switch_arb_sets(rdev, MC_CG_ARB_FREQ_F0, MC_CG_ARB_FREQ_F1);
1591 }
1592
1593 static int ni_force_switch_to_arb_f0(struct radeon_device *rdev)
1594 {
1595     struct rv7xx_power_info *pi = rv770_get_pi(rdev);
1596     struct ni_power_info *ni_pi = ni_get_pi(rdev);
1597     u32 tmp;
1598     int ret;
1599
1600     ret = rv770_read_smc_sram_dword(rdev, ni_pi->arb_table_start,
1601                     &tmp, pi->sram_end);
1602     if (ret)
1603         return ret;
1604
1605     tmp = (tmp >> 24) & 0xff;
1606
1607     if (tmp == MC_CG_ARB_FREQ_F0)
1608         return 0;
1609
1610     return ni_copy_and_switch_arb_sets(rdev, tmp, MC_CG_ARB_FREQ_F0);
1611 }
1612
1613 static int ni_populate_memory_timing_parameters(struct radeon_device *rdev,
1614                         struct rv7xx_pl *pl,
1615                         SMC_NIslands_MCArbDramTimingRegisterSet *arb_regs)
1616 {
1617     u32 dram_timing;
1618     u32 dram_timing2;
1619
1620     arb_regs->mc_arb_rfsh_rate =
1621         (u8)rv770_calculate_memory_refresh_rate(rdev, pl->sclk);
1622
1623
1624     radeon_atom_set_engine_dram_timings(rdev, pl->sclk, pl->mclk);
1625
1626     dram_timing = RREG32(MC_ARB_DRAM_TIMING);
1627     dram_timing2 = RREG32(MC_ARB_DRAM_TIMING2);
1628
1629     arb_regs->mc_arb_dram_timing  = cpu_to_be32(dram_timing);
1630     arb_regs->mc_arb_dram_timing2 = cpu_to_be32(dram_timing2);
1631
1632     return 0;
1633 }
1634
1635 static int ni_do_program_memory_timing_parameters(struct radeon_device *rdev,
1636                           struct radeon_ps *radeon_state,
1637                           unsigned int first_arb_set)
1638 {
1639     struct rv7xx_power_info *pi = rv770_get_pi(rdev);
1640     struct ni_power_info *ni_pi = ni_get_pi(rdev);
1641     struct ni_ps *state = ni_get_ps(radeon_state);
1642     SMC_NIslands_MCArbDramTimingRegisterSet arb_regs = { 0 };
1643     int i, ret = 0;
1644
1645     for (i = 0; i < state->performance_level_count; i++) {
1646         ret = ni_populate_memory_timing_parameters(rdev, &state->performance_levels[i], &arb_regs);
1647         if (ret)
1648             break;
1649
1650         ret = rv770_copy_bytes_to_smc(rdev,
1651                           (u16)(ni_pi->arb_table_start +
1652                             offsetof(SMC_NIslands_MCArbDramTimingRegisters, data) +
1653                             sizeof(SMC_NIslands_MCArbDramTimingRegisterSet) * (first_arb_set + i)),
1654                           (u8 *)&arb_regs,
1655                           (u16)sizeof(SMC_NIslands_MCArbDramTimingRegisterSet),
1656                           pi->sram_end);
1657         if (ret)
1658             break;
1659     }
1660     return ret;
1661 }
1662
1663 static int ni_program_memory_timing_parameters(struct radeon_device *rdev,
1664                            struct radeon_ps *radeon_new_state)
1665 {
1666     return ni_do_program_memory_timing_parameters(rdev, radeon_new_state,
1667                               NISLANDS_DRIVER_STATE_ARB_INDEX);
1668 }
1669
1670 static void ni_populate_initial_mvdd_value(struct radeon_device *rdev,
1671                        struct NISLANDS_SMC_VOLTAGE_VALUE *voltage)
1672 {
1673     struct evergreen_power_info *eg_pi = evergreen_get_pi(rdev);
1674
1675     voltage->index = eg_pi->mvdd_high_index;
1676     voltage->value = cpu_to_be16(MVDD_HIGH_VALUE);
1677 }
1678
1679 static int ni_populate_smc_initial_state(struct radeon_device *rdev,
1680                      struct radeon_ps *radeon_initial_state,
1681                      NISLANDS_SMC_STATETABLE *table)
1682 {
1683     struct ni_ps *initial_state = ni_get_ps(radeon_initial_state);
1684     struct rv7xx_power_info *pi = rv770_get_pi(rdev);
1685     struct evergreen_power_info *eg_pi = evergreen_get_pi(rdev);
1686     struct ni_power_info *ni_pi = ni_get_pi(rdev);
1687     u32 reg;
1688     int ret;
1689
1690     table->initialState.level.mclk.vMPLL_AD_FUNC_CNTL =
1691         cpu_to_be32(ni_pi->clock_registers.mpll_ad_func_cntl);
1692     table->initialState.level.mclk.vMPLL_AD_FUNC_CNTL_2 =
1693         cpu_to_be32(ni_pi->clock_registers.mpll_ad_func_cntl_2);
1694     table->initialState.level.mclk.vMPLL_DQ_FUNC_CNTL =
1695         cpu_to_be32(ni_pi->clock_registers.mpll_dq_func_cntl);
1696     table->initialState.level.mclk.vMPLL_DQ_FUNC_CNTL_2 =
1697         cpu_to_be32(ni_pi->clock_registers.mpll_dq_func_cntl_2);
1698     table->initialState.level.mclk.vMCLK_PWRMGT_CNTL =
1699         cpu_to_be32(ni_pi->clock_registers.mclk_pwrmgt_cntl);
1700     table->initialState.level.mclk.vDLL_CNTL =
1701         cpu_to_be32(ni_pi->clock_registers.dll_cntl);
1702     table->initialState.level.mclk.vMPLL_SS =
1703         cpu_to_be32(ni_pi->clock_registers.mpll_ss1);
1704     table->initialState.level.mclk.vMPLL_SS2 =
1705         cpu_to_be32(ni_pi->clock_registers.mpll_ss2);
1706     table->initialState.level.mclk.mclk_value =
1707         cpu_to_be32(initial_state->performance_levels[0].mclk);
1708
1709     table->initialState.level.sclk.vCG_SPLL_FUNC_CNTL =
1710         cpu_to_be32(ni_pi->clock_registers.cg_spll_func_cntl);
1711     table->initialState.level.sclk.vCG_SPLL_FUNC_CNTL_2 =
1712         cpu_to_be32(ni_pi->clock_registers.cg_spll_func_cntl_2);
1713     table->initialState.level.sclk.vCG_SPLL_FUNC_CNTL_3 =
1714         cpu_to_be32(ni_pi->clock_registers.cg_spll_func_cntl_3);
1715     table->initialState.level.sclk.vCG_SPLL_FUNC_CNTL_4 =
1716         cpu_to_be32(ni_pi->clock_registers.cg_spll_func_cntl_4);
1717     table->initialState.level.sclk.vCG_SPLL_SPREAD_SPECTRUM =
1718         cpu_to_be32(ni_pi->clock_registers.cg_spll_spread_spectrum);
1719     table->initialState.level.sclk.vCG_SPLL_SPREAD_SPECTRUM_2 =
1720         cpu_to_be32(ni_pi->clock_registers.cg_spll_spread_spectrum_2);
1721     table->initialState.level.sclk.sclk_value =
1722         cpu_to_be32(initial_state->performance_levels[0].sclk);
1723     table->initialState.level.arbRefreshState =
1724         NISLANDS_INITIAL_STATE_ARB_INDEX;
1725
1726     table->initialState.level.ACIndex = 0;
1727
1728     ret = ni_populate_voltage_value(rdev, &eg_pi->vddc_voltage_table,
1729                     initial_state->performance_levels[0].vddc,
1730                     &table->initialState.level.vddc);
1731     if (!ret) {
1732         u16 std_vddc;
1733
1734         ret = ni_get_std_voltage_value(rdev,
1735                            &table->initialState.level.vddc,
1736                            &std_vddc);
1737         if (!ret)
1738             ni_populate_std_voltage_value(rdev, std_vddc,
1739                               table->initialState.level.vddc.index,
1740                               &table->initialState.level.std_vddc);
1741     }
1742
1743     if (eg_pi->vddci_control)
1744         ni_populate_voltage_value(rdev,
1745                       &eg_pi->vddci_voltage_table,
1746                       initial_state->performance_levels[0].vddci,
1747                       &table->initialState.level.vddci);
1748
1749     ni_populate_initial_mvdd_value(rdev, &table->initialState.level.mvdd);
1750
1751     reg = CG_R(0xffff) | CG_L(0);
1752     table->initialState.level.aT = cpu_to_be32(reg);
1753
1754     table->initialState.level.bSP = cpu_to_be32(pi->dsp);
1755
1756     if (pi->boot_in_gen2)
1757         table->initialState.level.gen2PCIE = 1;
1758     else
1759         table->initialState.level.gen2PCIE = 0;
1760
1761     if (pi->mem_gddr5) {
1762         table->initialState.level.strobeMode =
1763             cypress_get_strobe_mode_settings(rdev,
1764                              initial_state->performance_levels[0].mclk);
1765
1766         if (initial_state->performance_levels[0].mclk > pi->mclk_edc_enable_threshold)
1767             table->initialState.level.mcFlags = NISLANDS_SMC_MC_EDC_RD_FLAG | NISLANDS_SMC_MC_EDC_WR_FLAG;
1768         else
1769             table->initialState.level.mcFlags =  0;
1770     }
1771
1772     table->initialState.levelCount = 1;
1773
1774     table->initialState.flags |= PPSMC_SWSTATE_FLAG_DC;
1775
1776     table->initialState.level.dpm2.MaxPS = 0;
1777     table->initialState.level.dpm2.NearTDPDec = 0;
1778     table->initialState.level.dpm2.AboveSafeInc = 0;
1779     table->initialState.level.dpm2.BelowSafeInc = 0;
1780
1781     reg = MIN_POWER_MASK | MAX_POWER_MASK;
1782     table->initialState.level.SQPowerThrottle = cpu_to_be32(reg);
1783
1784     reg = MAX_POWER_DELTA_MASK | STI_SIZE_MASK | LTI_RATIO_MASK;
1785     table->initialState.level.SQPowerThrottle_2 = cpu_to_be32(reg);
1786
1787     return 0;
1788 }
1789
1790 static int ni_populate_smc_acpi_state(struct radeon_device *rdev,
1791                       NISLANDS_SMC_STATETABLE *table)
1792 {
1793     struct rv7xx_power_info *pi = rv770_get_pi(rdev);
1794     struct evergreen_power_info *eg_pi = evergreen_get_pi(rdev);
1795     struct ni_power_info *ni_pi = ni_get_pi(rdev);
1796     u32 mpll_ad_func_cntl   = ni_pi->clock_registers.mpll_ad_func_cntl;
1797     u32 mpll_ad_func_cntl_2 = ni_pi->clock_registers.mpll_ad_func_cntl_2;
1798     u32 mpll_dq_func_cntl   = ni_pi->clock_registers.mpll_dq_func_cntl;
1799     u32 mpll_dq_func_cntl_2 = ni_pi->clock_registers.mpll_dq_func_cntl_2;
1800     u32 spll_func_cntl      = ni_pi->clock_registers.cg_spll_func_cntl;
1801     u32 spll_func_cntl_2    = ni_pi->clock_registers.cg_spll_func_cntl_2;
1802     u32 spll_func_cntl_3    = ni_pi->clock_registers.cg_spll_func_cntl_3;
1803     u32 spll_func_cntl_4    = ni_pi->clock_registers.cg_spll_func_cntl_4;
1804     u32 mclk_pwrmgt_cntl    = ni_pi->clock_registers.mclk_pwrmgt_cntl;
1805     u32 dll_cntl            = ni_pi->clock_registers.dll_cntl;
1806     u32 reg;
1807     int ret;
1808
1809     table->ACPIState = table->initialState;
1810
1811     table->ACPIState.flags &= ~PPSMC_SWSTATE_FLAG_DC;
1812
1813     if (pi->acpi_vddc) {
1814         ret = ni_populate_voltage_value(rdev,
1815                         &eg_pi->vddc_voltage_table,
1816                         pi->acpi_vddc, &table->ACPIState.level.vddc);
1817         if (!ret) {
1818             u16 std_vddc;
1819
1820             ret = ni_get_std_voltage_value(rdev,
1821                                &table->ACPIState.level.vddc, &std_vddc);
1822             if (!ret)
1823                 ni_populate_std_voltage_value(rdev, std_vddc,
1824                                   table->ACPIState.level.vddc.index,
1825                                   &table->ACPIState.level.std_vddc);
1826         }
1827
1828         if (pi->pcie_gen2) {
1829             if (pi->acpi_pcie_gen2)
1830                 table->ACPIState.level.gen2PCIE = 1;
1831             else
1832                 table->ACPIState.level.gen2PCIE = 0;
1833         } else {
1834             table->ACPIState.level.gen2PCIE = 0;
1835         }
1836     } else {
1837         ret = ni_populate_voltage_value(rdev,
1838                         &eg_pi->vddc_voltage_table,
1839                         pi->min_vddc_in_table,
1840                         &table->ACPIState.level.vddc);
1841         if (!ret) {
1842             u16 std_vddc;
1843
1844             ret = ni_get_std_voltage_value(rdev,
1845                                &table->ACPIState.level.vddc,
1846                                &std_vddc);
1847             if (!ret)
1848                 ni_populate_std_voltage_value(rdev, std_vddc,
1849                                   table->ACPIState.level.vddc.index,
1850                                   &table->ACPIState.level.std_vddc);
1851         }
1852         table->ACPIState.level.gen2PCIE = 0;
1853     }
1854
1855     if (eg_pi->acpi_vddci) {
1856         if (eg_pi->vddci_control)
1857             ni_populate_voltage_value(rdev,
1858                           &eg_pi->vddci_voltage_table,
1859                           eg_pi->acpi_vddci,
1860                           &table->ACPIState.level.vddci);
1861     }
1862
1863
1864     mpll_ad_func_cntl &= ~PDNB;
1865
1866     mpll_ad_func_cntl_2 |= BIAS_GEN_PDNB | RESET_EN;
1867
1868     if (pi->mem_gddr5)
1869         mpll_dq_func_cntl &= ~PDNB;
1870     mpll_dq_func_cntl_2 |= BIAS_GEN_PDNB | RESET_EN | BYPASS;
1871
1872
1873     mclk_pwrmgt_cntl |= (MRDCKA0_RESET |
1874                  MRDCKA1_RESET |
1875                  MRDCKB0_RESET |
1876                  MRDCKB1_RESET |
1877                  MRDCKC0_RESET |
1878                  MRDCKC1_RESET |
1879                  MRDCKD0_RESET |
1880                  MRDCKD1_RESET);
1881
1882     mclk_pwrmgt_cntl &= ~(MRDCKA0_PDNB |
1883                   MRDCKA1_PDNB |
1884                   MRDCKB0_PDNB |
1885                   MRDCKB1_PDNB |
1886                   MRDCKC0_PDNB |
1887                   MRDCKC1_PDNB |
1888                   MRDCKD0_PDNB |
1889                   MRDCKD1_PDNB);
1890
1891     dll_cntl |= (MRDCKA0_BYPASS |
1892              MRDCKA1_BYPASS |
1893              MRDCKB0_BYPASS |
1894              MRDCKB1_BYPASS |
1895              MRDCKC0_BYPASS |
1896              MRDCKC1_BYPASS |
1897              MRDCKD0_BYPASS |
1898              MRDCKD1_BYPASS);
1899
1900     spll_func_cntl_2 &= ~SCLK_MUX_SEL_MASK;
1901     spll_func_cntl_2 |= SCLK_MUX_SEL(4);
1902
1903     table->ACPIState.level.mclk.vMPLL_AD_FUNC_CNTL = cpu_to_be32(mpll_ad_func_cntl);
1904     table->ACPIState.level.mclk.vMPLL_AD_FUNC_CNTL_2 = cpu_to_be32(mpll_ad_func_cntl_2);
1905     table->ACPIState.level.mclk.vMPLL_DQ_FUNC_CNTL = cpu_to_be32(mpll_dq_func_cntl);
1906     table->ACPIState.level.mclk.vMPLL_DQ_FUNC_CNTL_2 = cpu_to_be32(mpll_dq_func_cntl_2);
1907     table->ACPIState.level.mclk.vMCLK_PWRMGT_CNTL = cpu_to_be32(mclk_pwrmgt_cntl);
1908     table->ACPIState.level.mclk.vDLL_CNTL = cpu_to_be32(dll_cntl);
1909
1910     table->ACPIState.level.mclk.mclk_value = 0;
1911
1912     table->ACPIState.level.sclk.vCG_SPLL_FUNC_CNTL = cpu_to_be32(spll_func_cntl);
1913     table->ACPIState.level.sclk.vCG_SPLL_FUNC_CNTL_2 = cpu_to_be32(spll_func_cntl_2);
1914     table->ACPIState.level.sclk.vCG_SPLL_FUNC_CNTL_3 = cpu_to_be32(spll_func_cntl_3);
1915     table->ACPIState.level.sclk.vCG_SPLL_FUNC_CNTL_4 = cpu_to_be32(spll_func_cntl_4);
1916
1917     table->ACPIState.level.sclk.sclk_value = 0;
1918
1919     ni_populate_mvdd_value(rdev, 0, &table->ACPIState.level.mvdd);
1920
1921     if (eg_pi->dynamic_ac_timing)
1922         table->ACPIState.level.ACIndex = 1;
1923
1924     table->ACPIState.level.dpm2.MaxPS = 0;
1925     table->ACPIState.level.dpm2.NearTDPDec = 0;
1926     table->ACPIState.level.dpm2.AboveSafeInc = 0;
1927     table->ACPIState.level.dpm2.BelowSafeInc = 0;
1928
1929     reg = MIN_POWER_MASK | MAX_POWER_MASK;
1930     table->ACPIState.level.SQPowerThrottle = cpu_to_be32(reg);
1931
1932     reg = MAX_POWER_DELTA_MASK | STI_SIZE_MASK | LTI_RATIO_MASK;
1933     table->ACPIState.level.SQPowerThrottle_2 = cpu_to_be32(reg);
1934
1935     return 0;
1936 }
1937
1938 static int ni_init_smc_table(struct radeon_device *rdev)
1939 {
1940     struct rv7xx_power_info *pi = rv770_get_pi(rdev);
1941     struct ni_power_info *ni_pi = ni_get_pi(rdev);
1942     int ret;
1943     struct radeon_ps *radeon_boot_state = rdev->pm.dpm.boot_ps;
1944     NISLANDS_SMC_STATETABLE *table = &ni_pi->smc_statetable;
1945
1946     memset(table, 0, sizeof(NISLANDS_SMC_STATETABLE));
1947
1948     ni_populate_smc_voltage_tables(rdev, table);
1949
1950     switch (rdev->pm.int_thermal_type) {
1951     case THERMAL_TYPE_NI:
1952     case THERMAL_TYPE_EMC2103_WITH_INTERNAL:
1953         table->thermalProtectType = PPSMC_THERMAL_PROTECT_TYPE_INTERNAL;
1954         break;
1955     case THERMAL_TYPE_NONE:
1956         table->thermalProtectType = PPSMC_THERMAL_PROTECT_TYPE_NONE;
1957         break;
1958     default:
1959         table->thermalProtectType = PPSMC_THERMAL_PROTECT_TYPE_EXTERNAL;
1960         break;
1961     }
1962
1963     if (rdev->pm.dpm.platform_caps & ATOM_PP_PLATFORM_CAP_HARDWAREDC)
1964         table->systemFlags |= PPSMC_SYSTEMFLAG_GPIO_DC;
1965
1966     if (rdev->pm.dpm.platform_caps & ATOM_PP_PLATFORM_CAP_REGULATOR_HOT)
1967         table->systemFlags |= PPSMC_SYSTEMFLAG_REGULATOR_HOT;
1968
1969     if (rdev->pm.dpm.platform_caps & ATOM_PP_PLATFORM_CAP_STEPVDDC)
1970         table->systemFlags |= PPSMC_SYSTEMFLAG_STEPVDDC;
1971
1972     if (pi->mem_gddr5)
1973         table->systemFlags |= PPSMC_SYSTEMFLAG_GDDR5;
1974
1975     ret = ni_populate_smc_initial_state(rdev, radeon_boot_state, table);
1976     if (ret)
1977         return ret;
1978
1979     ret = ni_populate_smc_acpi_state(rdev, table);
1980     if (ret)
1981         return ret;
1982
1983     table->driverState.flags = table->initialState.flags;
1984     table->driverState.levelCount = table->initialState.levelCount;
1985     table->driverState.levels[0] = table->initialState.level;
1986
1987     table->ULVState = table->initialState;
1988
1989     ret = ni_do_program_memory_timing_parameters(rdev, radeon_boot_state,
1990                              NISLANDS_INITIAL_STATE_ARB_INDEX);
1991     if (ret)
1992         return ret;
1993
1994     return rv770_copy_bytes_to_smc(rdev, pi->state_table_start, (u8 *)table,
1995                        sizeof(NISLANDS_SMC_STATETABLE), pi->sram_end);
1996 }
1997
1998 static int ni_calculate_sclk_params(struct radeon_device *rdev,
1999                     u32 engine_clock,
2000                     NISLANDS_SMC_SCLK_VALUE *sclk)
2001 {
2002     struct rv7xx_power_info *pi = rv770_get_pi(rdev);
2003     struct ni_power_info *ni_pi = ni_get_pi(rdev);
2004     struct atom_clock_dividers dividers;
2005     u32 spll_func_cntl = ni_pi->clock_registers.cg_spll_func_cntl;
2006     u32 spll_func_cntl_2 = ni_pi->clock_registers.cg_spll_func_cntl_2;
2007     u32 spll_func_cntl_3 = ni_pi->clock_registers.cg_spll_func_cntl_3;
2008     u32 spll_func_cntl_4 = ni_pi->clock_registers.cg_spll_func_cntl_4;
2009     u32 cg_spll_spread_spectrum = ni_pi->clock_registers.cg_spll_spread_spectrum;
2010     u32 cg_spll_spread_spectrum_2 = ni_pi->clock_registers.cg_spll_spread_spectrum_2;
2011     u64 tmp;
2012     u32 reference_clock = rdev->clock.spll.reference_freq;
2013     u32 reference_divider;
2014     u32 fbdiv;
2015     int ret;
2016
2017     ret = radeon_atom_get_clock_dividers(rdev, COMPUTE_ENGINE_PLL_PARAM,
2018                          engine_clock, false, &dividers);
2019     if (ret)
2020         return ret;
2021
2022     reference_divider = 1 + dividers.ref_div;
2023
2024
2025     tmp = (u64) engine_clock * reference_divider * dividers.post_div * 16834;
2026     do_div(tmp, reference_clock);
2027     fbdiv = (u32) tmp;
2028
2029     spll_func_cntl &= ~(SPLL_PDIV_A_MASK | SPLL_REF_DIV_MASK);
2030     spll_func_cntl |= SPLL_REF_DIV(dividers.ref_div);
2031     spll_func_cntl |= SPLL_PDIV_A(dividers.post_div);
2032
2033     spll_func_cntl_2 &= ~SCLK_MUX_SEL_MASK;
2034     spll_func_cntl_2 |= SCLK_MUX_SEL(2);
2035
2036     spll_func_cntl_3 &= ~SPLL_FB_DIV_MASK;
2037     spll_func_cntl_3 |= SPLL_FB_DIV(fbdiv);
2038     spll_func_cntl_3 |= SPLL_DITHEN;
2039
2040     if (pi->sclk_ss) {
2041         struct radeon_atom_ss ss;
2042         u32 vco_freq = engine_clock * dividers.post_div;
2043
2044         if (radeon_atombios_get_asic_ss_info(rdev, &ss,
2045                              ASIC_INTERNAL_ENGINE_SS, vco_freq)) {
2046             u32 clk_s = reference_clock * 5 / (reference_divider * ss.rate);
2047             u32 clk_v = 4 * ss.percentage * fbdiv / (clk_s * 10000);
2048
2049             cg_spll_spread_spectrum &= ~CLK_S_MASK;
2050             cg_spll_spread_spectrum |= CLK_S(clk_s);
2051             cg_spll_spread_spectrum |= SSEN;
2052
2053             cg_spll_spread_spectrum_2 &= ~CLK_V_MASK;
2054             cg_spll_spread_spectrum_2 |= CLK_V(clk_v);
2055         }
2056     }
2057
2058     sclk->sclk_value = engine_clock;
2059     sclk->vCG_SPLL_FUNC_CNTL = spll_func_cntl;
2060     sclk->vCG_SPLL_FUNC_CNTL_2 = spll_func_cntl_2;
2061     sclk->vCG_SPLL_FUNC_CNTL_3 = spll_func_cntl_3;
2062     sclk->vCG_SPLL_FUNC_CNTL_4 = spll_func_cntl_4;
2063     sclk->vCG_SPLL_SPREAD_SPECTRUM = cg_spll_spread_spectrum;
2064     sclk->vCG_SPLL_SPREAD_SPECTRUM_2 = cg_spll_spread_spectrum_2;
2065
2066     return 0;
2067 }
2068
2069 static int ni_populate_sclk_value(struct radeon_device *rdev,
2070                   u32 engine_clock,
2071                   NISLANDS_SMC_SCLK_VALUE *sclk)
2072 {
2073     NISLANDS_SMC_SCLK_VALUE sclk_tmp;
2074     int ret;
2075
2076     ret = ni_calculate_sclk_params(rdev, engine_clock, &sclk_tmp);
2077     if (!ret) {
2078         sclk->sclk_value = cpu_to_be32(sclk_tmp.sclk_value);
2079         sclk->vCG_SPLL_FUNC_CNTL = cpu_to_be32(sclk_tmp.vCG_SPLL_FUNC_CNTL);
2080         sclk->vCG_SPLL_FUNC_CNTL_2 = cpu_to_be32(sclk_tmp.vCG_SPLL_FUNC_CNTL_2);
2081         sclk->vCG_SPLL_FUNC_CNTL_3 = cpu_to_be32(sclk_tmp.vCG_SPLL_FUNC_CNTL_3);
2082         sclk->vCG_SPLL_FUNC_CNTL_4 = cpu_to_be32(sclk_tmp.vCG_SPLL_FUNC_CNTL_4);
2083         sclk->vCG_SPLL_SPREAD_SPECTRUM = cpu_to_be32(sclk_tmp.vCG_SPLL_SPREAD_SPECTRUM);
2084         sclk->vCG_SPLL_SPREAD_SPECTRUM_2 = cpu_to_be32(sclk_tmp.vCG_SPLL_SPREAD_SPECTRUM_2);
2085     }
2086
2087     return ret;
2088 }
2089
2090 static int ni_init_smc_spll_table(struct radeon_device *rdev)
2091 {
2092     struct rv7xx_power_info *pi = rv770_get_pi(rdev);
2093     struct ni_power_info *ni_pi = ni_get_pi(rdev);
2094     SMC_NISLANDS_SPLL_DIV_TABLE *spll_table;
2095     NISLANDS_SMC_SCLK_VALUE sclk_params;
2096     u32 fb_div;
2097     u32 p_div;
2098     u32 clk_s;
2099     u32 clk_v;
2100     u32 sclk = 0;
2101     int i, ret;
2102     u32 tmp;
2103
2104     if (ni_pi->spll_table_start == 0)
2105         return -EINVAL;
2106
2107     spll_table = kzalloc(sizeof(SMC_NISLANDS_SPLL_DIV_TABLE), GFP_KERNEL);
2108     if (spll_table == NULL)
2109         return -ENOMEM;
2110
2111     for (i = 0; i < 256; i++) {
2112         ret = ni_calculate_sclk_params(rdev, sclk, &sclk_params);
2113         if (ret)
2114             break;
2115
2116         p_div = (sclk_params.vCG_SPLL_FUNC_CNTL & SPLL_PDIV_A_MASK) >> SPLL_PDIV_A_SHIFT;
2117         fb_div = (sclk_params.vCG_SPLL_FUNC_CNTL_3 & SPLL_FB_DIV_MASK) >> SPLL_FB_DIV_SHIFT;
2118         clk_s = (sclk_params.vCG_SPLL_SPREAD_SPECTRUM & CLK_S_MASK) >> CLK_S_SHIFT;
2119         clk_v = (sclk_params.vCG_SPLL_SPREAD_SPECTRUM_2 & CLK_V_MASK) >> CLK_V_SHIFT;
2120
2121         fb_div &= ~0x00001FFF;
2122         fb_div >>= 1;
2123         clk_v >>= 6;
2124
2125         if (p_div & ~(SMC_NISLANDS_SPLL_DIV_TABLE_PDIV_MASK >> SMC_NISLANDS_SPLL_DIV_TABLE_PDIV_SHIFT))
2126             ret = -EINVAL;
2127
2128         if (clk_s & ~(SMC_NISLANDS_SPLL_DIV_TABLE_CLKS_MASK >> SMC_NISLANDS_SPLL_DIV_TABLE_CLKS_SHIFT))
2129             ret = -EINVAL;
2130
2131         if (fb_div & ~(SMC_NISLANDS_SPLL_DIV_TABLE_FBDIV_MASK >> SMC_NISLANDS_SPLL_DIV_TABLE_FBDIV_SHIFT))
2132             ret = -EINVAL;
2133
2134         if (clk_v & ~(SMC_NISLANDS_SPLL_DIV_TABLE_CLKV_MASK >> SMC_NISLANDS_SPLL_DIV_TABLE_CLKV_SHIFT))
2135             ret = -EINVAL;
2136
2137         if (ret)
2138             break;
2139
2140         tmp = ((fb_div << SMC_NISLANDS_SPLL_DIV_TABLE_FBDIV_SHIFT) & SMC_NISLANDS_SPLL_DIV_TABLE_FBDIV_MASK) |
2141             ((p_div << SMC_NISLANDS_SPLL_DIV_TABLE_PDIV_SHIFT) & SMC_NISLANDS_SPLL_DIV_TABLE_PDIV_MASK);
2142         spll_table->freq[i] = cpu_to_be32(tmp);
2143
2144         tmp = ((clk_v << SMC_NISLANDS_SPLL_DIV_TABLE_CLKV_SHIFT) & SMC_NISLANDS_SPLL_DIV_TABLE_CLKV_MASK) |
2145             ((clk_s << SMC_NISLANDS_SPLL_DIV_TABLE_CLKS_SHIFT) & SMC_NISLANDS_SPLL_DIV_TABLE_CLKS_MASK);
2146         spll_table->ss[i] = cpu_to_be32(tmp);
2147
2148         sclk += 512;
2149     }
2150
2151     if (!ret)
2152         ret = rv770_copy_bytes_to_smc(rdev, ni_pi->spll_table_start, (u8 *)spll_table,
2153                           sizeof(SMC_NISLANDS_SPLL_DIV_TABLE), pi->sram_end);
2154
2155     kfree(spll_table);
2156
2157     return ret;
2158 }
2159
2160 static int ni_populate_mclk_value(struct radeon_device *rdev,
2161                   u32 engine_clock,
2162                   u32 memory_clock,
2163                   NISLANDS_SMC_MCLK_VALUE *mclk,
2164                   bool strobe_mode,
2165                   bool dll_state_on)
2166 {
2167     struct rv7xx_power_info *pi = rv770_get_pi(rdev);
2168     struct ni_power_info *ni_pi = ni_get_pi(rdev);
2169     u32 mpll_ad_func_cntl = ni_pi->clock_registers.mpll_ad_func_cntl;
2170     u32 mpll_ad_func_cntl_2 = ni_pi->clock_registers.mpll_ad_func_cntl_2;
2171     u32 mpll_dq_func_cntl = ni_pi->clock_registers.mpll_dq_func_cntl;
2172     u32 mpll_dq_func_cntl_2 = ni_pi->clock_registers.mpll_dq_func_cntl_2;
2173     u32 mclk_pwrmgt_cntl = ni_pi->clock_registers.mclk_pwrmgt_cntl;
2174     u32 dll_cntl = ni_pi->clock_registers.dll_cntl;
2175     u32 mpll_ss1 = ni_pi->clock_registers.mpll_ss1;
2176     u32 mpll_ss2 = ni_pi->clock_registers.mpll_ss2;
2177     struct atom_clock_dividers dividers;
2178     u32 ibias;
2179     u32 dll_speed;
2180     int ret;
2181     u32 mc_seq_misc7;
2182
2183     ret = radeon_atom_get_clock_dividers(rdev, COMPUTE_MEMORY_PLL_PARAM,
2184                          memory_clock, strobe_mode, &dividers);
2185     if (ret)
2186         return ret;
2187
2188     if (!strobe_mode) {
2189         mc_seq_misc7 = RREG32(MC_SEQ_MISC7);
2190
2191         if (mc_seq_misc7 & 0x8000000)
2192             dividers.post_div = 1;
2193     }
2194
2195     ibias = cypress_map_clkf_to_ibias(rdev, dividers.whole_fb_div);
2196
2197     mpll_ad_func_cntl &= ~(CLKR_MASK |
2198                    YCLK_POST_DIV_MASK |
2199                    CLKF_MASK |
2200                    CLKFRAC_MASK |
2201                    IBIAS_MASK);
2202     mpll_ad_func_cntl |= CLKR(dividers.ref_div);
2203     mpll_ad_func_cntl |= YCLK_POST_DIV(dividers.post_div);
2204     mpll_ad_func_cntl |= CLKF(dividers.whole_fb_div);
2205     mpll_ad_func_cntl |= CLKFRAC(dividers.frac_fb_div);
2206     mpll_ad_func_cntl |= IBIAS(ibias);
2207
2208     if (dividers.vco_mode)
2209         mpll_ad_func_cntl_2 |= VCO_MODE;
2210     else
2211         mpll_ad_func_cntl_2 &= ~VCO_MODE;
2212
2213     if (pi->mem_gddr5) {
2214         mpll_dq_func_cntl &= ~(CLKR_MASK |
2215                        YCLK_POST_DIV_MASK |
2216                        CLKF_MASK |
2217                        CLKFRAC_MASK |
2218                        IBIAS_MASK);
2219         mpll_dq_func_cntl |= CLKR(dividers.ref_div);
2220         mpll_dq_func_cntl |= YCLK_POST_DIV(dividers.post_div);
2221         mpll_dq_func_cntl |= CLKF(dividers.whole_fb_div);
2222         mpll_dq_func_cntl |= CLKFRAC(dividers.frac_fb_div);
2223         mpll_dq_func_cntl |= IBIAS(ibias);
2224
2225         if (strobe_mode)
2226             mpll_dq_func_cntl &= ~PDNB;
2227         else
2228             mpll_dq_func_cntl |= PDNB;
2229
2230         if (dividers.vco_mode)
2231             mpll_dq_func_cntl_2 |= VCO_MODE;
2232         else
2233             mpll_dq_func_cntl_2 &= ~VCO_MODE;
2234     }
2235
2236     if (pi->mclk_ss) {
2237         struct radeon_atom_ss ss;
2238         u32 vco_freq = memory_clock * dividers.post_div;
2239
2240         if (radeon_atombios_get_asic_ss_info(rdev, &ss,
2241                              ASIC_INTERNAL_MEMORY_SS, vco_freq)) {
2242             u32 reference_clock = rdev->clock.mpll.reference_freq;
2243             u32 decoded_ref = rv740_get_decoded_reference_divider(dividers.ref_div);
2244             u32 clk_s = reference_clock * 5 / (decoded_ref * ss.rate);
2245             u32 clk_v = ss.percentage *
2246                 (0x4000 * dividers.whole_fb_div + 0x800 * dividers.frac_fb_div) / (clk_s * 625);
2247
2248             mpll_ss1 &= ~CLKV_MASK;
2249             mpll_ss1 |= CLKV(clk_v);
2250
2251             mpll_ss2 &= ~CLKS_MASK;
2252             mpll_ss2 |= CLKS(clk_s);
2253         }
2254     }
2255
2256     dll_speed = rv740_get_dll_speed(pi->mem_gddr5,
2257                     memory_clock);
2258
2259     mclk_pwrmgt_cntl &= ~DLL_SPEED_MASK;
2260     mclk_pwrmgt_cntl |= DLL_SPEED(dll_speed);
2261     if (dll_state_on)
2262         mclk_pwrmgt_cntl |= (MRDCKA0_PDNB |
2263                      MRDCKA1_PDNB |
2264                      MRDCKB0_PDNB |
2265                      MRDCKB1_PDNB |
2266                      MRDCKC0_PDNB |
2267                      MRDCKC1_PDNB |
2268                      MRDCKD0_PDNB |
2269                      MRDCKD1_PDNB);
2270     else
2271         mclk_pwrmgt_cntl &= ~(MRDCKA0_PDNB |
2272                       MRDCKA1_PDNB |
2273                       MRDCKB0_PDNB |
2274                       MRDCKB1_PDNB |
2275                       MRDCKC0_PDNB |
2276                       MRDCKC1_PDNB |
2277                       MRDCKD0_PDNB |
2278                       MRDCKD1_PDNB);
2279
2280
2281     mclk->mclk_value = cpu_to_be32(memory_clock);
2282     mclk->vMPLL_AD_FUNC_CNTL = cpu_to_be32(mpll_ad_func_cntl);
2283     mclk->vMPLL_AD_FUNC_CNTL_2 = cpu_to_be32(mpll_ad_func_cntl_2);
2284     mclk->vMPLL_DQ_FUNC_CNTL = cpu_to_be32(mpll_dq_func_cntl);
2285     mclk->vMPLL_DQ_FUNC_CNTL_2 = cpu_to_be32(mpll_dq_func_cntl_2);
2286     mclk->vMCLK_PWRMGT_CNTL = cpu_to_be32(mclk_pwrmgt_cntl);
2287     mclk->vDLL_CNTL = cpu_to_be32(dll_cntl);
2288     mclk->vMPLL_SS = cpu_to_be32(mpll_ss1);
2289     mclk->vMPLL_SS2 = cpu_to_be32(mpll_ss2);
2290
2291     return 0;
2292 }
2293
2294 static void ni_populate_smc_sp(struct radeon_device *rdev,
2295                    struct radeon_ps *radeon_state,
2296                    NISLANDS_SMC_SWSTATE *smc_state)
2297 {
2298     struct ni_ps *ps = ni_get_ps(radeon_state);
2299     struct rv7xx_power_info *pi = rv770_get_pi(rdev);
2300     int i;
2301
2302     for (i = 0; i < ps->performance_level_count - 1; i++)
2303         smc_state->levels[i].bSP = cpu_to_be32(pi->dsp);
2304
2305     smc_state->levels[ps->performance_level_count - 1].bSP =
2306         cpu_to_be32(pi->psp);
2307 }
2308
2309 static int ni_convert_power_level_to_smc(struct radeon_device *rdev,
2310                      struct rv7xx_pl *pl,
2311                      NISLANDS_SMC_HW_PERFORMANCE_LEVEL *level)
2312 {
2313     struct rv7xx_power_info *pi = rv770_get_pi(rdev);
2314     struct evergreen_power_info *eg_pi = evergreen_get_pi(rdev);
2315     struct ni_power_info *ni_pi = ni_get_pi(rdev);
2316     int ret;
2317     bool dll_state_on;
2318     u16 std_vddc;
2319     u32 tmp = RREG32(DC_STUTTER_CNTL);
2320
2321     level->gen2PCIE = pi->pcie_gen2 ?
2322         ((pl->flags & ATOM_PPLIB_R600_FLAGS_PCIEGEN2) ? 1 : 0) : 0;
2323
2324     ret = ni_populate_sclk_value(rdev, pl->sclk, &level->sclk);
2325     if (ret)
2326         return ret;
2327
2328     level->mcFlags =  0;
2329     if (pi->mclk_stutter_mode_threshold &&
2330         (pl->mclk <= pi->mclk_stutter_mode_threshold) &&
2331         !eg_pi->uvd_enabled &&
2332         (tmp & DC_STUTTER_ENABLE_A) &&
2333         (tmp & DC_STUTTER_ENABLE_B))
2334         level->mcFlags |= NISLANDS_SMC_MC_STUTTER_EN;
2335
2336     if (pi->mem_gddr5) {
2337         if (pl->mclk > pi->mclk_edc_enable_threshold)
2338             level->mcFlags |= NISLANDS_SMC_MC_EDC_RD_FLAG;
2339         if (pl->mclk > eg_pi->mclk_edc_wr_enable_threshold)
2340             level->mcFlags |= NISLANDS_SMC_MC_EDC_WR_FLAG;
2341
2342         level->strobeMode = cypress_get_strobe_mode_settings(rdev, pl->mclk);
2343
2344         if (level->strobeMode & NISLANDS_SMC_STROBE_ENABLE) {
2345             if (cypress_get_mclk_frequency_ratio(rdev, pl->mclk, true) >=
2346                 ((RREG32(MC_SEQ_MISC7) >> 16) & 0xf))
2347                 dll_state_on = ((RREG32(MC_SEQ_MISC5) >> 1) & 0x1) ? true : false;
2348             else
2349                 dll_state_on = ((RREG32(MC_SEQ_MISC6) >> 1) & 0x1) ? true : false;
2350         } else {
2351             dll_state_on = false;
2352             if (pl->mclk > ni_pi->mclk_rtt_mode_threshold)
2353                 level->mcFlags |= NISLANDS_SMC_MC_RTT_ENABLE;
2354         }
2355
2356         ret = ni_populate_mclk_value(rdev, pl->sclk, pl->mclk,
2357                          &level->mclk,
2358                          (level->strobeMode & NISLANDS_SMC_STROBE_ENABLE) != 0,
2359                          dll_state_on);
2360     } else
2361         ret = ni_populate_mclk_value(rdev, pl->sclk, pl->mclk, &level->mclk, 1, 1);
2362
2363     if (ret)
2364         return ret;
2365
2366     ret = ni_populate_voltage_value(rdev, &eg_pi->vddc_voltage_table,
2367                     pl->vddc, &level->vddc);
2368     if (ret)
2369         return ret;
2370
2371     ret = ni_get_std_voltage_value(rdev, &level->vddc, &std_vddc);
2372     if (ret)
2373         return ret;
2374
2375     ni_populate_std_voltage_value(rdev, std_vddc,
2376                       level->vddc.index, &level->std_vddc);
2377
2378     if (eg_pi->vddci_control) {
2379         ret = ni_populate_voltage_value(rdev, &eg_pi->vddci_voltage_table,
2380                         pl->vddci, &level->vddci);
2381         if (ret)
2382             return ret;
2383     }
2384
2385     ni_populate_mvdd_value(rdev, pl->mclk, &level->mvdd);
2386
2387     return ret;
2388 }
2389
2390 static int ni_populate_smc_t(struct radeon_device *rdev,
2391                  struct radeon_ps *radeon_state,
2392                  NISLANDS_SMC_SWSTATE *smc_state)
2393 {
2394     struct rv7xx_power_info *pi = rv770_get_pi(rdev);
2395     struct evergreen_power_info *eg_pi = evergreen_get_pi(rdev);
2396     struct ni_ps *state = ni_get_ps(radeon_state);
2397     u32 a_t;
2398     u32 t_l, t_h;
2399     u32 high_bsp;
2400     int i, ret;
2401
2402     if (state->performance_level_count >= 9)
2403         return -EINVAL;
2404
2405     if (state->performance_level_count < 2) {
2406         a_t = CG_R(0xffff) | CG_L(0);
2407         smc_state->levels[0].aT = cpu_to_be32(a_t);
2408         return 0;
2409     }
2410
2411     smc_state->levels[0].aT = cpu_to_be32(0);
2412
2413     for (i = 0; i <= state->performance_level_count - 2; i++) {
2414         if (eg_pi->uvd_enabled)
2415             ret = r600_calculate_at(
2416                 1000 * (i * (eg_pi->smu_uvd_hs ? 2 : 8) + 2),
2417                 100 * R600_AH_DFLT,
2418                 state->performance_levels[i + 1].sclk,
2419                 state->performance_levels[i].sclk,
2420                 &t_l,
2421                 &t_h);
2422         else
2423             ret = r600_calculate_at(
2424                 1000 * (i + 1),
2425                 100 * R600_AH_DFLT,
2426                 state->performance_levels[i + 1].sclk,
2427                 state->performance_levels[i].sclk,
2428                 &t_l,
2429                 &t_h);
2430
2431         if (ret) {
2432             t_h = (i + 1) * 1000 - 50 * R600_AH_DFLT;
2433             t_l = (i + 1) * 1000 + 50 * R600_AH_DFLT;
2434         }
2435
2436         a_t = be32_to_cpu(smc_state->levels[i].aT) & ~CG_R_MASK;
2437         a_t |= CG_R(t_l * pi->bsp / 20000);
2438         smc_state->levels[i].aT = cpu_to_be32(a_t);
2439
2440         high_bsp = (i == state->performance_level_count - 2) ?
2441             pi->pbsp : pi->bsp;
2442
2443         a_t = CG_R(0xffff) | CG_L(t_h * high_bsp / 20000);
2444         smc_state->levels[i + 1].aT = cpu_to_be32(a_t);
2445     }
2446
2447     return 0;
2448 }
2449
2450 static int ni_populate_power_containment_values(struct radeon_device *rdev,
2451                         struct radeon_ps *radeon_state,
2452                         NISLANDS_SMC_SWSTATE *smc_state)
2453 {
2454     struct rv7xx_power_info *pi = rv770_get_pi(rdev);
2455     struct evergreen_power_info *eg_pi = evergreen_get_pi(rdev);
2456     struct ni_power_info *ni_pi = ni_get_pi(rdev);
2457     struct ni_ps *state = ni_get_ps(radeon_state);
2458     u32 prev_sclk;
2459     u32 max_sclk;
2460     u32 min_sclk;
2461     int i, ret;
2462     u32 tdp_limit;
2463     u32 near_tdp_limit;
2464     u32 power_boost_limit;
2465     u8 max_ps_percent;
2466
2467     if (ni_pi->enable_power_containment == false)
2468         return 0;
2469
2470     if (state->performance_level_count == 0)
2471         return -EINVAL;
2472
2473     if (smc_state->levelCount != state->performance_level_count)
2474         return -EINVAL;
2475
2476     ret = ni_calculate_adjusted_tdp_limits(rdev,
2477                            false, /* ??? */
2478                            rdev->pm.dpm.tdp_adjustment,
2479                            &tdp_limit,
2480                            &near_tdp_limit);
2481     if (ret)
2482         return ret;
2483
2484     power_boost_limit = ni_calculate_power_boost_limit(rdev, radeon_state, near_tdp_limit);
2485
2486     ret = rv770_write_smc_sram_dword(rdev,
2487                      pi->state_table_start +
2488                      offsetof(NISLANDS_SMC_STATETABLE, dpm2Params) +
2489                      offsetof(PP_NIslands_DPM2Parameters, PowerBoostLimit),
2490                      ni_scale_power_for_smc(power_boost_limit, ni_get_smc_power_scaling_factor(rdev)),
2491                      pi->sram_end);
2492     if (ret)
2493         power_boost_limit = 0;
2494
2495     smc_state->levels[0].dpm2.MaxPS = 0;
2496     smc_state->levels[0].dpm2.NearTDPDec = 0;
2497     smc_state->levels[0].dpm2.AboveSafeInc = 0;
2498     smc_state->levels[0].dpm2.BelowSafeInc = 0;
2499     smc_state->levels[0].stateFlags |= power_boost_limit ? PPSMC_STATEFLAG_POWERBOOST : 0;
2500
2501     for (i = 1; i < state->performance_level_count; i++) {
2502         prev_sclk = state->performance_levels[i-1].sclk;
2503         max_sclk  = state->performance_levels[i].sclk;
2504         max_ps_percent = (i != (state->performance_level_count - 1)) ?
2505             NISLANDS_DPM2_MAXPS_PERCENT_M : NISLANDS_DPM2_MAXPS_PERCENT_H;
2506
2507         if (max_sclk < prev_sclk)
2508             return -EINVAL;
2509
2510         if ((max_ps_percent == 0) || (prev_sclk == max_sclk) || eg_pi->uvd_enabled)
2511             min_sclk = max_sclk;
2512         else if (1 == i)
2513             min_sclk = prev_sclk;
2514         else
2515             min_sclk = (prev_sclk * (u32)max_ps_percent) / 100;
2516
2517         if (min_sclk < state->performance_levels[0].sclk)
2518             min_sclk = state->performance_levels[0].sclk;
2519
2520         if (min_sclk == 0)
2521             return -EINVAL;
2522
2523         smc_state->levels[i].dpm2.MaxPS =
2524             (u8)((NISLANDS_DPM2_MAX_PULSE_SKIP * (max_sclk - min_sclk)) / max_sclk);
2525         smc_state->levels[i].dpm2.NearTDPDec = NISLANDS_DPM2_NEAR_TDP_DEC;
2526         smc_state->levels[i].dpm2.AboveSafeInc = NISLANDS_DPM2_ABOVE_SAFE_INC;
2527         smc_state->levels[i].dpm2.BelowSafeInc = NISLANDS_DPM2_BELOW_SAFE_INC;
2528         smc_state->levels[i].stateFlags |=
2529             ((i != (state->performance_level_count - 1)) && power_boost_limit) ?
2530             PPSMC_STATEFLAG_POWERBOOST : 0;
2531     }
2532
2533     return 0;
2534 }
2535
2536 static int ni_populate_sq_ramping_values(struct radeon_device *rdev,
2537                      struct radeon_ps *radeon_state,
2538                      NISLANDS_SMC_SWSTATE *smc_state)
2539 {
2540     struct ni_power_info *ni_pi = ni_get_pi(rdev);
2541     struct ni_ps *state = ni_get_ps(radeon_state);
2542     u32 sq_power_throttle;
2543     u32 sq_power_throttle2;
2544     bool enable_sq_ramping = ni_pi->enable_sq_ramping;
2545     int i;
2546
2547     if (state->performance_level_count == 0)
2548         return -EINVAL;
2549
2550     if (smc_state->levelCount != state->performance_level_count)
2551         return -EINVAL;
2552
2553     if (rdev->pm.dpm.sq_ramping_threshold == 0)
2554         return -EINVAL;
2555
2556     if (NISLANDS_DPM2_SQ_RAMP_MAX_POWER > (MAX_POWER_MASK >> MAX_POWER_SHIFT))
2557         enable_sq_ramping = false;
2558
2559     if (NISLANDS_DPM2_SQ_RAMP_MIN_POWER > (MIN_POWER_MASK >> MIN_POWER_SHIFT))
2560         enable_sq_ramping = false;
2561
2562     if (NISLANDS_DPM2_SQ_RAMP_MAX_POWER_DELTA > (MAX_POWER_DELTA_MASK >> MAX_POWER_DELTA_SHIFT))
2563         enable_sq_ramping = false;
2564
2565     if (NISLANDS_DPM2_SQ_RAMP_STI_SIZE > (STI_SIZE_MASK >> STI_SIZE_SHIFT))
2566         enable_sq_ramping = false;
2567
2568     if (NISLANDS_DPM2_SQ_RAMP_LTI_RATIO > (LTI_RATIO_MASK >> LTI_RATIO_SHIFT))
2569         enable_sq_ramping = false;
2570
2571     for (i = 0; i < state->performance_level_count; i++) {
2572         sq_power_throttle  = 0;
2573         sq_power_throttle2 = 0;
2574
2575         if ((state->performance_levels[i].sclk >= rdev->pm.dpm.sq_ramping_threshold) &&
2576             enable_sq_ramping) {
2577             sq_power_throttle |= MAX_POWER(NISLANDS_DPM2_SQ_RAMP_MAX_POWER);
2578             sq_power_throttle |= MIN_POWER(NISLANDS_DPM2_SQ_RAMP_MIN_POWER);
2579             sq_power_throttle2 |= MAX_POWER_DELTA(NISLANDS_DPM2_SQ_RAMP_MAX_POWER_DELTA);
2580             sq_power_throttle2 |= STI_SIZE(NISLANDS_DPM2_SQ_RAMP_STI_SIZE);
2581             sq_power_throttle2 |= LTI_RATIO(NISLANDS_DPM2_SQ_RAMP_LTI_RATIO);
2582         } else {
2583             sq_power_throttle |= MAX_POWER_MASK | MIN_POWER_MASK;
2584             sq_power_throttle2 |= MAX_POWER_DELTA_MASK | STI_SIZE_MASK | LTI_RATIO_MASK;
2585         }
2586
2587         smc_state->levels[i].SQPowerThrottle   = cpu_to_be32(sq_power_throttle);
2588         smc_state->levels[i].SQPowerThrottle_2 = cpu_to_be32(sq_power_throttle2);
2589     }
2590
2591     return 0;
2592 }
2593
2594 static int ni_enable_power_containment(struct radeon_device *rdev,
2595                        struct radeon_ps *radeon_new_state,
2596                        bool enable)
2597 {
2598     struct ni_power_info *ni_pi = ni_get_pi(rdev);
2599     PPSMC_Result smc_result;
2600     int ret = 0;
2601
2602     if (ni_pi->enable_power_containment) {
2603         if (enable) {
2604             if (!r600_is_uvd_state(radeon_new_state->class, radeon_new_state->class2)) {
2605                 smc_result = rv770_send_msg_to_smc(rdev, PPSMC_TDPClampingActive);
2606                 if (smc_result != PPSMC_Result_OK) {
2607                     ret = -EINVAL;
2608                     ni_pi->pc_enabled = false;
2609                 } else {
2610                     ni_pi->pc_enabled = true;
2611                 }
2612             }
2613         } else {
2614             smc_result = rv770_send_msg_to_smc(rdev, PPSMC_TDPClampingInactive);
2615             if (smc_result != PPSMC_Result_OK)
2616                 ret = -EINVAL;
2617             ni_pi->pc_enabled = false;
2618         }
2619     }
2620
2621     return ret;
2622 }
2623
2624 static int ni_convert_power_state_to_smc(struct radeon_device *rdev,
2625                      struct radeon_ps *radeon_state,
2626                      NISLANDS_SMC_SWSTATE *smc_state)
2627 {
2628     struct evergreen_power_info *eg_pi = evergreen_get_pi(rdev);
2629     struct ni_power_info *ni_pi = ni_get_pi(rdev);
2630     struct ni_ps *state = ni_get_ps(radeon_state);
2631     int i, ret;
2632     u32 threshold = state->performance_levels[state->performance_level_count - 1].sclk * 100 / 100;
2633
2634     if (!(radeon_state->caps & ATOM_PPLIB_DISALLOW_ON_DC))
2635         smc_state->flags |= PPSMC_SWSTATE_FLAG_DC;
2636
2637     smc_state->levelCount = 0;
2638
2639     if (state->performance_level_count > NISLANDS_MAX_SMC_PERFORMANCE_LEVELS_PER_SWSTATE)
2640         return -EINVAL;
2641
2642     for (i = 0; i < state->performance_level_count; i++) {
2643         ret = ni_convert_power_level_to_smc(rdev, &state->performance_levels[i],
2644                             &smc_state->levels[i]);
2645         smc_state->levels[i].arbRefreshState =
2646             (u8)(NISLANDS_DRIVER_STATE_ARB_INDEX + i);
2647
2648         if (ret)
2649             return ret;
2650
2651         if (ni_pi->enable_power_containment)
2652             smc_state->levels[i].displayWatermark =
2653                 (state->performance_levels[i].sclk < threshold) ?
2654                 PPSMC_DISPLAY_WATERMARK_LOW : PPSMC_DISPLAY_WATERMARK_HIGH;
2655         else
2656             smc_state->levels[i].displayWatermark = (i < 2) ?
2657                 PPSMC_DISPLAY_WATERMARK_LOW : PPSMC_DISPLAY_WATERMARK_HIGH;
2658
2659         if (eg_pi->dynamic_ac_timing)
2660             smc_state->levels[i].ACIndex = NISLANDS_MCREGISTERTABLE_FIRST_DRIVERSTATE_SLOT + i;
2661         else
2662             smc_state->levels[i].ACIndex = 0;
2663
2664         smc_state->levelCount++;
2665     }
2666
2667     rv770_write_smc_soft_register(rdev, NI_SMC_SOFT_REGISTER_watermark_threshold,
2668                       cpu_to_be32(threshold / 512));
2669
2670     ni_populate_smc_sp(rdev, radeon_state, smc_state);
2671
2672     ret = ni_populate_power_containment_values(rdev, radeon_state, smc_state);
2673     if (ret)
2674         ni_pi->enable_power_containment = false;
2675
2676     ret = ni_populate_sq_ramping_values(rdev, radeon_state, smc_state);
2677     if (ret)
2678         ni_pi->enable_sq_ramping = false;
2679
2680     return ni_populate_smc_t(rdev, radeon_state, smc_state);
2681 }
2682
2683 static int ni_upload_sw_state(struct radeon_device *rdev,
2684                   struct radeon_ps *radeon_new_state)
2685 {
2686     struct rv7xx_power_info *pi = rv770_get_pi(rdev);
2687     u16 address = pi->state_table_start +
2688         offsetof(NISLANDS_SMC_STATETABLE, driverState);
2689     NISLANDS_SMC_SWSTATE *smc_state;
2690     size_t state_size = struct_size(smc_state, levels,
2691             NISLANDS_MAX_SMC_PERFORMANCE_LEVELS_PER_SWSTATE);
2692     int ret;
2693
2694     smc_state = kzalloc(state_size, GFP_KERNEL);
2695     if (smc_state == NULL)
2696         return -ENOMEM;
2697
2698     ret = ni_convert_power_state_to_smc(rdev, radeon_new_state, smc_state);
2699     if (ret)
2700         goto done;
2701
2702     ret = rv770_copy_bytes_to_smc(rdev, address, (u8 *)smc_state, state_size, pi->sram_end);
2703
2704 done:
2705     kfree(smc_state);
2706
2707     return ret;
2708 }
2709
2710 static int ni_set_mc_special_registers(struct radeon_device *rdev,
2711                        struct ni_mc_reg_table *table)
2712 {
2713     struct rv7xx_power_info *pi = rv770_get_pi(rdev);
2714     u8 i, j, k;
2715     u32 temp_reg;
2716
2717     for (i = 0, j = table->last; i < table->last; i++) {
2718         switch (table->mc_reg_address[i].s1) {
2719         case MC_SEQ_MISC1 >> 2:
2720             if (j >= SMC_NISLANDS_MC_REGISTER_ARRAY_SIZE)
2721                 return -EINVAL;
2722             temp_reg = RREG32(MC_PMG_CMD_EMRS);
2723             table->mc_reg_address[j].s1 = MC_PMG_CMD_EMRS >> 2;
2724             table->mc_reg_address[j].s0 = MC_SEQ_PMG_CMD_EMRS_LP >> 2;
2725             for (k = 0; k < table->num_entries; k++)
2726                 table->mc_reg_table_entry[k].mc_data[j] =
2727                     ((temp_reg & 0xffff0000)) |
2728                     ((table->mc_reg_table_entry[k].mc_data[i] & 0xffff0000) >> 16);
2729             j++;
2730             if (j >= SMC_NISLANDS_MC_REGISTER_ARRAY_SIZE)
2731                 return -EINVAL;
2732
2733             temp_reg = RREG32(MC_PMG_CMD_MRS);
2734             table->mc_reg_address[j].s1 = MC_PMG_CMD_MRS >> 2;
2735             table->mc_reg_address[j].s0 = MC_SEQ_PMG_CMD_MRS_LP >> 2;
2736             for(k = 0; k < table->num_entries; k++) {
2737                 table->mc_reg_table_entry[k].mc_data[j] =
2738                     (temp_reg & 0xffff0000) |
2739                     (table->mc_reg_table_entry[k].mc_data[i] & 0x0000ffff);
2740                 if (!pi->mem_gddr5)
2741                     table->mc_reg_table_entry[k].mc_data[j] |= 0x100;
2742             }
2743             j++;
2744             break;
2745         case MC_SEQ_RESERVE_M >> 2:
2746             if (j >= SMC_NISLANDS_MC_REGISTER_ARRAY_SIZE)
2747                 return -EINVAL;
2748             temp_reg = RREG32(MC_PMG_CMD_MRS1);
2749             table->mc_reg_address[j].s1 = MC_PMG_CMD_MRS1 >> 2;
2750             table->mc_reg_address[j].s0 = MC_SEQ_PMG_CMD_MRS1_LP >> 2;
2751             for (k = 0; k < table->num_entries; k++)
2752                 table->mc_reg_table_entry[k].mc_data[j] =
2753                     (temp_reg & 0xffff0000) |
2754                     (table->mc_reg_table_entry[k].mc_data[i] & 0x0000ffff);
2755             j++;
2756             break;
2757         default:
2758             break;
2759         }
2760     }
2761
2762     table->last = j;
2763
2764     return 0;
2765 }
2766
2767 static bool ni_check_s0_mc_reg_index(u16 in_reg, u16 *out_reg)
2768 {
2769     bool result = true;
2770
2771     switch (in_reg) {
2772     case  MC_SEQ_RAS_TIMING >> 2:
2773         *out_reg = MC_SEQ_RAS_TIMING_LP >> 2;
2774         break;
2775     case MC_SEQ_CAS_TIMING >> 2:
2776         *out_reg = MC_SEQ_CAS_TIMING_LP >> 2;
2777         break;
2778     case MC_SEQ_MISC_TIMING >> 2:
2779         *out_reg = MC_SEQ_MISC_TIMING_LP >> 2;
2780         break;
2781     case MC_SEQ_MISC_TIMING2 >> 2:
2782         *out_reg = MC_SEQ_MISC_TIMING2_LP >> 2;
2783         break;
2784     case MC_SEQ_RD_CTL_D0 >> 2:
2785         *out_reg = MC_SEQ_RD_CTL_D0_LP >> 2;
2786         break;
2787     case MC_SEQ_RD_CTL_D1 >> 2:
2788         *out_reg = MC_SEQ_RD_CTL_D1_LP >> 2;
2789         break;
2790     case MC_SEQ_WR_CTL_D0 >> 2:
2791         *out_reg = MC_SEQ_WR_CTL_D0_LP >> 2;
2792         break;
2793     case MC_SEQ_WR_CTL_D1 >> 2:
2794         *out_reg = MC_SEQ_WR_CTL_D1_LP >> 2;
2795         break;
2796     case MC_PMG_CMD_EMRS >> 2:
2797         *out_reg = MC_SEQ_PMG_CMD_EMRS_LP >> 2;
2798         break;
2799     case MC_PMG_CMD_MRS >> 2:
2800         *out_reg = MC_SEQ_PMG_CMD_MRS_LP >> 2;
2801         break;
2802     case MC_PMG_CMD_MRS1 >> 2:
2803         *out_reg = MC_SEQ_PMG_CMD_MRS1_LP >> 2;
2804         break;
2805     case MC_SEQ_PMG_TIMING >> 2:
2806         *out_reg = MC_SEQ_PMG_TIMING_LP >> 2;
2807         break;
2808     case MC_PMG_CMD_MRS2 >> 2:
2809         *out_reg = MC_SEQ_PMG_CMD_MRS2_LP >> 2;
2810         break;
2811     default:
2812         result = false;
2813         break;
2814     }
2815
2816     return result;
2817 }
2818
2819 static void ni_set_valid_flag(struct ni_mc_reg_table *table)
2820 {
2821     u8 i, j;
2822
2823     for (i = 0; i < table->last; i++) {
2824         for (j = 1; j < table->num_entries; j++) {
2825             if (table->mc_reg_table_entry[j-1].mc_data[i] != table->mc_reg_table_entry[j].mc_data[i]) {
2826                 table->valid_flag |= 1 << i;
2827                 break;
2828             }
2829         }
2830     }
2831 }
2832
2833 static void ni_set_s0_mc_reg_index(struct ni_mc_reg_table *table)
2834 {
2835     u32 i;
2836     u16 address;
2837
2838     for (i = 0; i < table->last; i++)
2839         table->mc_reg_address[i].s0 =
2840             ni_check_s0_mc_reg_index(table->mc_reg_address[i].s1, &address) ?
2841             address : table->mc_reg_address[i].s1;
2842 }
2843
2844 static int ni_copy_vbios_mc_reg_table(struct atom_mc_reg_table *table,
2845                       struct ni_mc_reg_table *ni_table)
2846 {
2847     u8 i, j;
2848
2849     if (table->last > SMC_NISLANDS_MC_REGISTER_ARRAY_SIZE)
2850         return -EINVAL;
2851     if (table->num_entries > MAX_AC_TIMING_ENTRIES)
2852         return -EINVAL;
2853
2854     for (i = 0; i < table->last; i++)
2855         ni_table->mc_reg_address[i].s1 = table->mc_reg_address[i].s1;
2856     ni_table->last = table->last;
2857
2858     for (i = 0; i < table->num_entries; i++) {
2859         ni_table->mc_reg_table_entry[i].mclk_max =
2860             table->mc_reg_table_entry[i].mclk_max;
2861         for (j = 0; j < table->last; j++)
2862             ni_table->mc_reg_table_entry[i].mc_data[j] =
2863                 table->mc_reg_table_entry[i].mc_data[j];
2864     }
2865     ni_table->num_entries = table->num_entries;
2866
2867     return 0;
2868 }
2869
2870 static int ni_initialize_mc_reg_table(struct radeon_device *rdev)
2871 {
2872     struct ni_power_info *ni_pi = ni_get_pi(rdev);
2873     int ret;
2874     struct atom_mc_reg_table *table;
2875     struct ni_mc_reg_table *ni_table = &ni_pi->mc_reg_table;
2876     u8 module_index = rv770_get_memory_module_index(rdev);
2877
2878     table = kzalloc(sizeof(struct atom_mc_reg_table), GFP_KERNEL);
2879     if (!table)
2880         return -ENOMEM;
2881
2882     WREG32(MC_SEQ_RAS_TIMING_LP, RREG32(MC_SEQ_RAS_TIMING));
2883     WREG32(MC_SEQ_CAS_TIMING_LP, RREG32(MC_SEQ_CAS_TIMING));
2884     WREG32(MC_SEQ_MISC_TIMING_LP, RREG32(MC_SEQ_MISC_TIMING));
2885     WREG32(MC_SEQ_MISC_TIMING2_LP, RREG32(MC_SEQ_MISC_TIMING2));
2886     WREG32(MC_SEQ_PMG_CMD_EMRS_LP, RREG32(MC_PMG_CMD_EMRS));
2887     WREG32(MC_SEQ_PMG_CMD_MRS_LP, RREG32(MC_PMG_CMD_MRS));
2888     WREG32(MC_SEQ_PMG_CMD_MRS1_LP, RREG32(MC_PMG_CMD_MRS1));
2889     WREG32(MC_SEQ_WR_CTL_D0_LP, RREG32(MC_SEQ_WR_CTL_D0));
2890     WREG32(MC_SEQ_WR_CTL_D1_LP, RREG32(MC_SEQ_WR_CTL_D1));
2891     WREG32(MC_SEQ_RD_CTL_D0_LP, RREG32(MC_SEQ_RD_CTL_D0));
2892     WREG32(MC_SEQ_RD_CTL_D1_LP, RREG32(MC_SEQ_RD_CTL_D1));
2893     WREG32(MC_SEQ_PMG_TIMING_LP, RREG32(MC_SEQ_PMG_TIMING));
2894     WREG32(MC_SEQ_PMG_CMD_MRS2_LP, RREG32(MC_PMG_CMD_MRS2));
2895
2896     ret = radeon_atom_init_mc_reg_table(rdev, module_index, table);
2897
2898     if (ret)
2899         goto init_mc_done;
2900
2901     ret = ni_copy_vbios_mc_reg_table(table, ni_table);
2902
2903     if (ret)
2904         goto init_mc_done;
2905
2906     ni_set_s0_mc_reg_index(ni_table);
2907
2908     ret = ni_set_mc_special_registers(rdev, ni_table);
2909
2910     if (ret)
2911         goto init_mc_done;
2912
2913     ni_set_valid_flag(ni_table);
2914
2915 init_mc_done:
2916     kfree(table);
2917
2918     return ret;
2919 }
2920
2921 static void ni_populate_mc_reg_addresses(struct radeon_device *rdev,
2922                      SMC_NIslands_MCRegisters *mc_reg_table)
2923 {
2924     struct ni_power_info *ni_pi = ni_get_pi(rdev);
2925     u32 i, j;
2926
2927     for (i = 0, j = 0; j < ni_pi->mc_reg_table.last; j++) {
2928         if (ni_pi->mc_reg_table.valid_flag & (1 << j)) {
2929             if (i >= SMC_NISLANDS_MC_REGISTER_ARRAY_SIZE)
2930                 break;
2931             mc_reg_table->address[i].s0 =
2932                 cpu_to_be16(ni_pi->mc_reg_table.mc_reg_address[j].s0);
2933             mc_reg_table->address[i].s1 =
2934                 cpu_to_be16(ni_pi->mc_reg_table.mc_reg_address[j].s1);
2935             i++;
2936         }
2937     }
2938     mc_reg_table->last = (u8)i;
2939 }
2940
2941
2942 static void ni_convert_mc_registers(struct ni_mc_reg_entry *entry,
2943                     SMC_NIslands_MCRegisterSet *data,
2944                     u32 num_entries, u32 valid_flag)
2945 {
2946     u32 i, j;
2947
2948     for (i = 0, j = 0; j < num_entries; j++) {
2949         if (valid_flag & (1 << j)) {
2950             data->value[i] = cpu_to_be32(entry->mc_data[j]);
2951             i++;
2952         }
2953     }
2954 }
2955
2956 static void ni_convert_mc_reg_table_entry_to_smc(struct radeon_device *rdev,
2957                          struct rv7xx_pl *pl,
2958                          SMC_NIslands_MCRegisterSet *mc_reg_table_data)
2959 {
2960     struct ni_power_info *ni_pi = ni_get_pi(rdev);
2961     u32 i = 0;
2962
2963     for (i = 0; i < ni_pi->mc_reg_table.num_entries; i++) {
2964         if (pl->mclk <= ni_pi->mc_reg_table.mc_reg_table_entry[i].mclk_max)
2965             break;
2966     }
2967
2968     if ((i == ni_pi->mc_reg_table.num_entries) && (i > 0))
2969         --i;
2970
2971     ni_convert_mc_registers(&ni_pi->mc_reg_table.mc_reg_table_entry[i],
2972                 mc_reg_table_data,
2973                 ni_pi->mc_reg_table.last,
2974                 ni_pi->mc_reg_table.valid_flag);
2975 }
2976
2977 static void ni_convert_mc_reg_table_to_smc(struct radeon_device *rdev,
2978                        struct radeon_ps *radeon_state,
2979                        SMC_NIslands_MCRegisters *mc_reg_table)
2980 {
2981     struct ni_ps *state = ni_get_ps(radeon_state);
2982     int i;
2983
2984     for (i = 0; i < state->performance_level_count; i++) {
2985         ni_convert_mc_reg_table_entry_to_smc(rdev,
2986                              &state->performance_levels[i],
2987                              &mc_reg_table->data[NISLANDS_MCREGISTERTABLE_FIRST_DRIVERSTATE_SLOT + i]);
2988     }
2989 }
2990
2991 static int ni_populate_mc_reg_table(struct radeon_device *rdev,
2992                     struct radeon_ps *radeon_boot_state)
2993 {
2994     struct rv7xx_power_info *pi = rv770_get_pi(rdev);
2995     struct evergreen_power_info *eg_pi = evergreen_get_pi(rdev);
2996     struct ni_power_info *ni_pi = ni_get_pi(rdev);
2997     struct ni_ps *boot_state = ni_get_ps(radeon_boot_state);
2998     SMC_NIslands_MCRegisters *mc_reg_table = &ni_pi->smc_mc_reg_table;
2999
3000     memset(mc_reg_table, 0, sizeof(SMC_NIslands_MCRegisters));
3001
3002     rv770_write_smc_soft_register(rdev, NI_SMC_SOFT_REGISTER_seq_index, 1);
3003
3004     ni_populate_mc_reg_addresses(rdev, mc_reg_table);
3005
3006     ni_convert_mc_reg_table_entry_to_smc(rdev, &boot_state->performance_levels[0],
3007                          &mc_reg_table->data[0]);
3008
3009     ni_convert_mc_registers(&ni_pi->mc_reg_table.mc_reg_table_entry[0],
3010                 &mc_reg_table->data[1],
3011                 ni_pi->mc_reg_table.last,
3012                 ni_pi->mc_reg_table.valid_flag);
3013
3014     ni_convert_mc_reg_table_to_smc(rdev, radeon_boot_state, mc_reg_table);
3015
3016     return rv770_copy_bytes_to_smc(rdev, eg_pi->mc_reg_table_start,
3017                        (u8 *)mc_reg_table,
3018                        sizeof(SMC_NIslands_MCRegisters),
3019                        pi->sram_end);
3020 }
3021
3022 static int ni_upload_mc_reg_table(struct radeon_device *rdev,
3023                   struct radeon_ps *radeon_new_state)
3024 {
3025     struct rv7xx_power_info *pi = rv770_get_pi(rdev);
3026     struct evergreen_power_info *eg_pi = evergreen_get_pi(rdev);
3027     struct ni_power_info *ni_pi = ni_get_pi(rdev);
3028     struct ni_ps *ni_new_state = ni_get_ps(radeon_new_state);
3029     SMC_NIslands_MCRegisters *mc_reg_table = &ni_pi->smc_mc_reg_table;
3030     u16 address;
3031
3032     memset(mc_reg_table, 0, sizeof(SMC_NIslands_MCRegisters));
3033
3034     ni_convert_mc_reg_table_to_smc(rdev, radeon_new_state, mc_reg_table);
3035
3036     address = eg_pi->mc_reg_table_start +
3037         (u16)offsetof(SMC_NIslands_MCRegisters, data[NISLANDS_MCREGISTERTABLE_FIRST_DRIVERSTATE_SLOT]);
3038
3039     return rv770_copy_bytes_to_smc(rdev, address,
3040                        (u8 *)&mc_reg_table->data[NISLANDS_MCREGISTERTABLE_FIRST_DRIVERSTATE_SLOT],
3041                        sizeof(SMC_NIslands_MCRegisterSet) * ni_new_state->performance_level_count,
3042                        pi->sram_end);
3043 }
3044
3045 static int ni_init_driver_calculated_leakage_table(struct radeon_device *rdev,
3046                            PP_NIslands_CACTABLES *cac_tables)
3047 {
3048     struct ni_power_info *ni_pi = ni_get_pi(rdev);
3049     struct evergreen_power_info *eg_pi = evergreen_get_pi(rdev);
3050     u32 leakage = 0;
3051     unsigned int i, j, table_size;
3052     s32 t;
3053     u32 smc_leakage, max_leakage = 0;
3054     u32 scaling_factor;
3055
3056     table_size = eg_pi->vddc_voltage_table.count;
3057
3058     if (SMC_NISLANDS_LKGE_LUT_NUM_OF_VOLT_ENTRIES < table_size)
3059         table_size = SMC_NISLANDS_LKGE_LUT_NUM_OF_VOLT_ENTRIES;
3060
3061     scaling_factor = ni_get_smc_power_scaling_factor(rdev);
3062
3063     for (i = 0; i < SMC_NISLANDS_LKGE_LUT_NUM_OF_TEMP_ENTRIES; i++) {
3064         for (j = 0; j < table_size; j++) {
3065             t = (1000 * ((i + 1) * 8));
3066
3067             if (t < ni_pi->cac_data.leakage_minimum_temperature)
3068                 t = ni_pi->cac_data.leakage_minimum_temperature;
3069
3070             ni_calculate_leakage_for_v_and_t(rdev,
3071                              &ni_pi->cac_data.leakage_coefficients,
3072                              eg_pi->vddc_voltage_table.entries[j].value,
3073                              t,
3074                              ni_pi->cac_data.i_leakage,
3075                              &leakage);
3076
3077             smc_leakage = ni_scale_power_for_smc(leakage, scaling_factor) / 1000;
3078             if (smc_leakage > max_leakage)
3079                 max_leakage = smc_leakage;
3080
3081             cac_tables->cac_lkge_lut[i][j] = cpu_to_be32(smc_leakage);
3082         }
3083     }
3084
3085     for (j = table_size; j < SMC_NISLANDS_LKGE_LUT_NUM_OF_VOLT_ENTRIES; j++) {
3086         for (i = 0; i < SMC_NISLANDS_LKGE_LUT_NUM_OF_TEMP_ENTRIES; i++)
3087             cac_tables->cac_lkge_lut[i][j] = cpu_to_be32(max_leakage);
3088     }
3089     return 0;
3090 }
3091
3092 static int ni_init_simplified_leakage_table(struct radeon_device *rdev,
3093                         PP_NIslands_CACTABLES *cac_tables)
3094 {
3095     struct evergreen_power_info *eg_pi = evergreen_get_pi(rdev);
3096     struct radeon_cac_leakage_table *leakage_table =
3097         &rdev->pm.dpm.dyn_state.cac_leakage_table;
3098     u32 i, j, table_size;
3099     u32 smc_leakage, max_leakage = 0;
3100     u32 scaling_factor;
3101
3102     if (!leakage_table)
3103         return -EINVAL;
3104
3105     table_size = leakage_table->count;
3106
3107     if (eg_pi->vddc_voltage_table.count != table_size)
3108         table_size = (eg_pi->vddc_voltage_table.count < leakage_table->count) ?
3109             eg_pi->vddc_voltage_table.count : leakage_table->count;
3110
3111     if (SMC_NISLANDS_LKGE_LUT_NUM_OF_VOLT_ENTRIES < table_size)
3112         table_size = SMC_NISLANDS_LKGE_LUT_NUM_OF_VOLT_ENTRIES;
3113
3114     if (table_size == 0)
3115         return -EINVAL;
3116
3117     scaling_factor = ni_get_smc_power_scaling_factor(rdev);
3118
3119     for (j = 0; j < table_size; j++) {
3120         smc_leakage = leakage_table->entries[j].leakage;
3121
3122         if (smc_leakage > max_leakage)
3123             max_leakage = smc_leakage;
3124
3125         for (i = 0; i < SMC_NISLANDS_LKGE_LUT_NUM_OF_TEMP_ENTRIES; i++)
3126             cac_tables->cac_lkge_lut[i][j] =
3127                 cpu_to_be32(ni_scale_power_for_smc(smc_leakage, scaling_factor));
3128     }
3129
3130     for (j = table_size; j < SMC_NISLANDS_LKGE_LUT_NUM_OF_VOLT_ENTRIES; j++) {
3131         for (i = 0; i < SMC_NISLANDS_LKGE_LUT_NUM_OF_TEMP_ENTRIES; i++)
3132             cac_tables->cac_lkge_lut[i][j] =
3133                 cpu_to_be32(ni_scale_power_for_smc(max_leakage, scaling_factor));
3134     }
3135     return 0;
3136 }
3137
3138 static int ni_initialize_smc_cac_tables(struct radeon_device *rdev)
3139 {
3140     struct rv7xx_power_info *pi = rv770_get_pi(rdev);
3141     struct ni_power_info *ni_pi = ni_get_pi(rdev);
3142     PP_NIslands_CACTABLES *cac_tables = NULL;
3143     int i, ret;
3144     u32 reg;
3145
3146     if (ni_pi->enable_cac == false)
3147         return 0;
3148
3149     cac_tables = kzalloc(sizeof(PP_NIslands_CACTABLES), GFP_KERNEL);
3150     if (!cac_tables)
3151         return -ENOMEM;
3152
3153     reg = RREG32(CG_CAC_CTRL) & ~(TID_CNT_MASK | TID_UNIT_MASK);
3154     reg |= (TID_CNT(ni_pi->cac_weights->tid_cnt) |
3155         TID_UNIT(ni_pi->cac_weights->tid_unit));
3156     WREG32(CG_CAC_CTRL, reg);
3157
3158     for (i = 0; i < NISLANDS_DCCAC_MAX_LEVELS; i++)
3159         ni_pi->dc_cac_table[i] = ni_pi->cac_weights->dc_cac[i];
3160
3161     for (i = 0; i < SMC_NISLANDS_BIF_LUT_NUM_OF_ENTRIES; i++)
3162         cac_tables->cac_bif_lut[i] = ni_pi->cac_weights->pcie_cac[i];
3163
3164     ni_pi->cac_data.i_leakage = rdev->pm.dpm.cac_leakage;
3165     ni_pi->cac_data.pwr_const = 0;
3166     ni_pi->cac_data.dc_cac_value = ni_pi->dc_cac_table[NISLANDS_DCCAC_LEVEL_0];
3167     ni_pi->cac_data.bif_cac_value = 0;
3168     ni_pi->cac_data.mc_wr_weight = ni_pi->cac_weights->mc_write_weight;
3169     ni_pi->cac_data.mc_rd_weight = ni_pi->cac_weights->mc_read_weight;
3170     ni_pi->cac_data.allow_ovrflw = 0;
3171     ni_pi->cac_data.l2num_win_tdp = ni_pi->lta_window_size;
3172     ni_pi->cac_data.num_win_tdp = 0;
3173     ni_pi->cac_data.lts_truncate_n = ni_pi->lts_truncate;
3174
3175     if (ni_pi->driver_calculate_cac_leakage)
3176         ret = ni_init_driver_calculated_leakage_table(rdev, cac_tables);
3177     else
3178         ret = ni_init_simplified_leakage_table(rdev, cac_tables);
3179
3180     if (ret)
3181         goto done_free;
3182
3183     cac_tables->pwr_const      = cpu_to_be32(ni_pi->cac_data.pwr_const);
3184     cac_tables->dc_cacValue    = cpu_to_be32(ni_pi->cac_data.dc_cac_value);
3185     cac_tables->bif_cacValue   = cpu_to_be32(ni_pi->cac_data.bif_cac_value);
3186     cac_tables->AllowOvrflw    = ni_pi->cac_data.allow_ovrflw;
3187     cac_tables->MCWrWeight     = ni_pi->cac_data.mc_wr_weight;
3188     cac_tables->MCRdWeight     = ni_pi->cac_data.mc_rd_weight;
3189     cac_tables->numWin_TDP     = ni_pi->cac_data.num_win_tdp;
3190     cac_tables->l2numWin_TDP   = ni_pi->cac_data.l2num_win_tdp;
3191     cac_tables->lts_truncate_n = ni_pi->cac_data.lts_truncate_n;
3192
3193     ret = rv770_copy_bytes_to_smc(rdev, ni_pi->cac_table_start, (u8 *)cac_tables,
3194                       sizeof(PP_NIslands_CACTABLES), pi->sram_end);
3195
3196 done_free:
3197     if (ret) {
3198         ni_pi->enable_cac = false;
3199         ni_pi->enable_power_containment = false;
3200     }
3201
3202     kfree(cac_tables);
3203
3204     return 0;
3205 }
3206
3207 static int ni_initialize_hardware_cac_manager(struct radeon_device *rdev)
3208 {
3209     struct ni_power_info *ni_pi = ni_get_pi(rdev);
3210     u32 reg;
3211
3212     if (!ni_pi->enable_cac ||
3213         !ni_pi->cac_configuration_required)
3214         return 0;
3215
3216     if (ni_pi->cac_weights == NULL)
3217         return -EINVAL;
3218
3219     reg = RREG32_CG(CG_CAC_REGION_1_WEIGHT_0) & ~(WEIGHT_TCP_SIG0_MASK |
3220                               WEIGHT_TCP_SIG1_MASK |
3221                               WEIGHT_TA_SIG_MASK);
3222     reg |= (WEIGHT_TCP_SIG0(ni_pi->cac_weights->weight_tcp_sig0) |
3223         WEIGHT_TCP_SIG1(ni_pi->cac_weights->weight_tcp_sig1) |
3224         WEIGHT_TA_SIG(ni_pi->cac_weights->weight_ta_sig));
3225     WREG32_CG(CG_CAC_REGION_1_WEIGHT_0, reg);
3226
3227     reg = RREG32_CG(CG_CAC_REGION_1_WEIGHT_1) & ~(WEIGHT_TCC_EN0_MASK |
3228                               WEIGHT_TCC_EN1_MASK |
3229                               WEIGHT_TCC_EN2_MASK);
3230     reg |= (WEIGHT_TCC_EN0(ni_pi->cac_weights->weight_tcc_en0) |
3231         WEIGHT_TCC_EN1(ni_pi->cac_weights->weight_tcc_en1) |
3232         WEIGHT_TCC_EN2(ni_pi->cac_weights->weight_tcc_en2));
3233     WREG32_CG(CG_CAC_REGION_1_WEIGHT_1, reg);
3234
3235     reg = RREG32_CG(CG_CAC_REGION_2_WEIGHT_0) & ~(WEIGHT_CB_EN0_MASK |
3236                               WEIGHT_CB_EN1_MASK |
3237                               WEIGHT_CB_EN2_MASK |
3238                               WEIGHT_CB_EN3_MASK);
3239     reg |= (WEIGHT_CB_EN0(ni_pi->cac_weights->weight_cb_en0) |
3240         WEIGHT_CB_EN1(ni_pi->cac_weights->weight_cb_en1) |
3241         WEIGHT_CB_EN2(ni_pi->cac_weights->weight_cb_en2) |
3242         WEIGHT_CB_EN3(ni_pi->cac_weights->weight_cb_en3));
3243     WREG32_CG(CG_CAC_REGION_2_WEIGHT_0, reg);
3244
3245     reg = RREG32_CG(CG_CAC_REGION_2_WEIGHT_1) & ~(WEIGHT_DB_SIG0_MASK |
3246                               WEIGHT_DB_SIG1_MASK |
3247                               WEIGHT_DB_SIG2_MASK |
3248                               WEIGHT_DB_SIG3_MASK);
3249     reg |= (WEIGHT_DB_SIG0(ni_pi->cac_weights->weight_db_sig0) |
3250         WEIGHT_DB_SIG1(ni_pi->cac_weights->weight_db_sig1) |
3251         WEIGHT_DB_SIG2(ni_pi->cac_weights->weight_db_sig2) |
3252         WEIGHT_DB_SIG3(ni_pi->cac_weights->weight_db_sig3));
3253     WREG32_CG(CG_CAC_REGION_2_WEIGHT_1, reg);
3254
3255     reg = RREG32_CG(CG_CAC_REGION_2_WEIGHT_2) & ~(WEIGHT_SXM_SIG0_MASK |
3256                               WEIGHT_SXM_SIG1_MASK |
3257                               WEIGHT_SXM_SIG2_MASK |
3258                               WEIGHT_SXS_SIG0_MASK |
3259                               WEIGHT_SXS_SIG1_MASK);
3260     reg |= (WEIGHT_SXM_SIG0(ni_pi->cac_weights->weight_sxm_sig0) |
3261         WEIGHT_SXM_SIG1(ni_pi->cac_weights->weight_sxm_sig1) |
3262         WEIGHT_SXM_SIG2(ni_pi->cac_weights->weight_sxm_sig2) |
3263         WEIGHT_SXS_SIG0(ni_pi->cac_weights->weight_sxs_sig0) |
3264         WEIGHT_SXS_SIG1(ni_pi->cac_weights->weight_sxs_sig1));
3265     WREG32_CG(CG_CAC_REGION_2_WEIGHT_2, reg);
3266
3267     reg = RREG32_CG(CG_CAC_REGION_3_WEIGHT_0) & ~(WEIGHT_XBR_0_MASK |
3268                               WEIGHT_XBR_1_MASK |
3269                               WEIGHT_XBR_2_MASK |
3270                               WEIGHT_SPI_SIG0_MASK);
3271     reg |= (WEIGHT_XBR_0(ni_pi->cac_weights->weight_xbr_0) |
3272         WEIGHT_XBR_1(ni_pi->cac_weights->weight_xbr_1) |
3273         WEIGHT_XBR_2(ni_pi->cac_weights->weight_xbr_2) |
3274         WEIGHT_SPI_SIG0(ni_pi->cac_weights->weight_spi_sig0));
3275     WREG32_CG(CG_CAC_REGION_3_WEIGHT_0, reg);
3276
3277     reg = RREG32_CG(CG_CAC_REGION_3_WEIGHT_1) & ~(WEIGHT_SPI_SIG1_MASK |
3278                               WEIGHT_SPI_SIG2_MASK |
3279                               WEIGHT_SPI_SIG3_MASK |
3280                               WEIGHT_SPI_SIG4_MASK |
3281                               WEIGHT_SPI_SIG5_MASK);
3282     reg |= (WEIGHT_SPI_SIG1(ni_pi->cac_weights->weight_spi_sig1) |
3283         WEIGHT_SPI_SIG2(ni_pi->cac_weights->weight_spi_sig2) |
3284         WEIGHT_SPI_SIG3(ni_pi->cac_weights->weight_spi_sig3) |
3285         WEIGHT_SPI_SIG4(ni_pi->cac_weights->weight_spi_sig4) |
3286         WEIGHT_SPI_SIG5(ni_pi->cac_weights->weight_spi_sig5));
3287     WREG32_CG(CG_CAC_REGION_3_WEIGHT_1, reg);
3288
3289     reg = RREG32_CG(CG_CAC_REGION_4_WEIGHT_0) & ~(WEIGHT_LDS_SIG0_MASK |
3290                               WEIGHT_LDS_SIG1_MASK |
3291                               WEIGHT_SC_MASK);
3292     reg |= (WEIGHT_LDS_SIG0(ni_pi->cac_weights->weight_lds_sig0) |
3293         WEIGHT_LDS_SIG1(ni_pi->cac_weights->weight_lds_sig1) |
3294         WEIGHT_SC(ni_pi->cac_weights->weight_sc));
3295     WREG32_CG(CG_CAC_REGION_4_WEIGHT_0, reg);
3296
3297     reg = RREG32_CG(CG_CAC_REGION_4_WEIGHT_1) & ~(WEIGHT_BIF_MASK |
3298                               WEIGHT_CP_MASK |
3299                               WEIGHT_PA_SIG0_MASK |
3300                               WEIGHT_PA_SIG1_MASK |
3301                               WEIGHT_VGT_SIG0_MASK);
3302     reg |= (WEIGHT_BIF(ni_pi->cac_weights->weight_bif) |
3303         WEIGHT_CP(ni_pi->cac_weights->weight_cp) |
3304         WEIGHT_PA_SIG0(ni_pi->cac_weights->weight_pa_sig0) |
3305         WEIGHT_PA_SIG1(ni_pi->cac_weights->weight_pa_sig1) |
3306         WEIGHT_VGT_SIG0(ni_pi->cac_weights->weight_vgt_sig0));
3307     WREG32_CG(CG_CAC_REGION_4_WEIGHT_1, reg);
3308
3309     reg = RREG32_CG(CG_CAC_REGION_4_WEIGHT_2) & ~(WEIGHT_VGT_SIG1_MASK |
3310                               WEIGHT_VGT_SIG2_MASK |
3311                               WEIGHT_DC_SIG0_MASK |
3312                               WEIGHT_DC_SIG1_MASK |
3313                               WEIGHT_DC_SIG2_MASK);
3314     reg |= (WEIGHT_VGT_SIG1(ni_pi->cac_weights->weight_vgt_sig1) |
3315         WEIGHT_VGT_SIG2(ni_pi->cac_weights->weight_vgt_sig2) |
3316         WEIGHT_DC_SIG0(ni_pi->cac_weights->weight_dc_sig0) |
3317         WEIGHT_DC_SIG1(ni_pi->cac_weights->weight_dc_sig1) |
3318         WEIGHT_DC_SIG2(ni_pi->cac_weights->weight_dc_sig2));
3319     WREG32_CG(CG_CAC_REGION_4_WEIGHT_2, reg);
3320
3321     reg = RREG32_CG(CG_CAC_REGION_4_WEIGHT_3) & ~(WEIGHT_DC_SIG3_MASK |
3322                               WEIGHT_UVD_SIG0_MASK |
3323                               WEIGHT_UVD_SIG1_MASK |
3324                               WEIGHT_SPARE0_MASK |
3325                               WEIGHT_SPARE1_MASK);
3326     reg |= (WEIGHT_DC_SIG3(ni_pi->cac_weights->weight_dc_sig3) |
3327         WEIGHT_UVD_SIG0(ni_pi->cac_weights->weight_uvd_sig0) |
3328         WEIGHT_UVD_SIG1(ni_pi->cac_weights->weight_uvd_sig1) |
3329         WEIGHT_SPARE0(ni_pi->cac_weights->weight_spare0) |
3330         WEIGHT_SPARE1(ni_pi->cac_weights->weight_spare1));
3331     WREG32_CG(CG_CAC_REGION_4_WEIGHT_3, reg);
3332
3333     reg = RREG32_CG(CG_CAC_REGION_5_WEIGHT_0) & ~(WEIGHT_SQ_VSP_MASK |
3334                               WEIGHT_SQ_VSP0_MASK);
3335     reg |= (WEIGHT_SQ_VSP(ni_pi->cac_weights->weight_sq_vsp) |
3336         WEIGHT_SQ_VSP0(ni_pi->cac_weights->weight_sq_vsp0));
3337     WREG32_CG(CG_CAC_REGION_5_WEIGHT_0, reg);
3338
3339     reg = RREG32_CG(CG_CAC_REGION_5_WEIGHT_1) & ~(WEIGHT_SQ_GPR_MASK);
3340     reg |= WEIGHT_SQ_GPR(ni_pi->cac_weights->weight_sq_gpr);
3341     WREG32_CG(CG_CAC_REGION_5_WEIGHT_1, reg);
3342
3343     reg = RREG32_CG(CG_CAC_REGION_4_OVERRIDE_4) & ~(OVR_MODE_SPARE_0_MASK |
3344                             OVR_VAL_SPARE_0_MASK |
3345                             OVR_MODE_SPARE_1_MASK |
3346                             OVR_VAL_SPARE_1_MASK);
3347     reg |= (OVR_MODE_SPARE_0(ni_pi->cac_weights->ovr_mode_spare_0) |
3348         OVR_VAL_SPARE_0(ni_pi->cac_weights->ovr_val_spare_0) |
3349         OVR_MODE_SPARE_1(ni_pi->cac_weights->ovr_mode_spare_1) |
3350         OVR_VAL_SPARE_1(ni_pi->cac_weights->ovr_val_spare_1));
3351     WREG32_CG(CG_CAC_REGION_4_OVERRIDE_4, reg);
3352
3353     reg = RREG32(SQ_CAC_THRESHOLD) & ~(VSP_MASK |
3354                        VSP0_MASK |
3355                        GPR_MASK);
3356     reg |= (VSP(ni_pi->cac_weights->vsp) |
3357         VSP0(ni_pi->cac_weights->vsp0) |
3358         GPR(ni_pi->cac_weights->gpr));
3359     WREG32(SQ_CAC_THRESHOLD, reg);
3360
3361     reg = (MCDW_WR_ENABLE |
3362            MCDX_WR_ENABLE |
3363            MCDY_WR_ENABLE |
3364            MCDZ_WR_ENABLE |
3365            INDEX(0x09D4));
3366     WREG32(MC_CG_CONFIG, reg);
3367
3368     reg = (READ_WEIGHT(ni_pi->cac_weights->mc_read_weight) |
3369            WRITE_WEIGHT(ni_pi->cac_weights->mc_write_weight) |
3370            ALLOW_OVERFLOW);
3371     WREG32(MC_CG_DATAPORT, reg);
3372
3373     return 0;
3374 }
3375
3376 static int ni_enable_smc_cac(struct radeon_device *rdev,
3377                  struct radeon_ps *radeon_new_state,
3378                  bool enable)
3379 {
3380     struct ni_power_info *ni_pi = ni_get_pi(rdev);
3381     int ret = 0;
3382     PPSMC_Result smc_result;
3383
3384     if (ni_pi->enable_cac) {
3385         if (enable) {
3386             if (!r600_is_uvd_state(radeon_new_state->class, radeon_new_state->class2)) {
3387                 smc_result = rv770_send_msg_to_smc(rdev, PPSMC_MSG_CollectCAC_PowerCorreln);
3388
3389                 if (ni_pi->support_cac_long_term_average) {
3390                     smc_result = rv770_send_msg_to_smc(rdev, PPSMC_CACLongTermAvgEnable);
3391                     if (PPSMC_Result_OK != smc_result)
3392                         ni_pi->support_cac_long_term_average = false;
3393                 }
3394
3395                 smc_result = rv770_send_msg_to_smc(rdev, PPSMC_MSG_EnableCac);
3396                 if (PPSMC_Result_OK != smc_result)
3397                     ret = -EINVAL;
3398
3399                 ni_pi->cac_enabled = (PPSMC_Result_OK == smc_result) ? true : false;
3400             }
3401         } else if (ni_pi->cac_enabled) {
3402             smc_result = rv770_send_msg_to_smc(rdev, PPSMC_MSG_DisableCac);
3403
3404             ni_pi->cac_enabled = false;
3405
3406             if (ni_pi->support_cac_long_term_average) {
3407                 smc_result = rv770_send_msg_to_smc(rdev, PPSMC_CACLongTermAvgDisable);
3408                 if (PPSMC_Result_OK != smc_result)
3409                     ni_pi->support_cac_long_term_average = false;
3410             }
3411         }
3412     }
3413
3414     return ret;
3415 }
3416
3417 static int ni_pcie_performance_request(struct radeon_device *rdev,
3418                        u8 perf_req, bool advertise)
3419 {
3420 #if defined(CONFIG_ACPI)
3421     struct evergreen_power_info *eg_pi = evergreen_get_pi(rdev);
3422
3423     if ((perf_req == PCIE_PERF_REQ_PECI_GEN1) ||
3424         (perf_req == PCIE_PERF_REQ_PECI_GEN2)) {
3425         if (eg_pi->pcie_performance_request_registered == false)
3426             radeon_acpi_pcie_notify_device_ready(rdev);
3427         eg_pi->pcie_performance_request_registered = true;
3428         return radeon_acpi_pcie_performance_request(rdev, perf_req, advertise);
3429     } else if ((perf_req == PCIE_PERF_REQ_REMOVE_REGISTRY) &&
3430             eg_pi->pcie_performance_request_registered) {
3431         eg_pi->pcie_performance_request_registered = false;
3432         return radeon_acpi_pcie_performance_request(rdev, perf_req, advertise);
3433     }
3434 #endif
3435     return 0;
3436 }
3437
3438 static int ni_advertise_gen2_capability(struct radeon_device *rdev)
3439 {
3440     struct rv7xx_power_info *pi = rv770_get_pi(rdev);
3441     u32 tmp;
3442
3443     tmp = RREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL);
3444
3445     if ((tmp & LC_OTHER_SIDE_EVER_SENT_GEN2) &&
3446         (tmp & LC_OTHER_SIDE_SUPPORTS_GEN2))
3447         pi->pcie_gen2 = true;
3448     else
3449         pi->pcie_gen2 = false;
3450
3451     if (!pi->pcie_gen2)
3452         ni_pcie_performance_request(rdev, PCIE_PERF_REQ_PECI_GEN2, true);
3453
3454     return 0;
3455 }
3456
3457 static void ni_enable_bif_dynamic_pcie_gen2(struct radeon_device *rdev,
3458                         bool enable)
3459 {
3460     struct rv7xx_power_info *pi = rv770_get_pi(rdev);
3461     u32 tmp, bif;
3462
3463     tmp = RREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL);
3464
3465     if ((tmp & LC_OTHER_SIDE_EVER_SENT_GEN2) &&
3466         (tmp & LC_OTHER_SIDE_SUPPORTS_GEN2)) {
3467         if (enable) {
3468             if (!pi->boot_in_gen2) {
3469                 bif = RREG32(CG_BIF_REQ_AND_RSP) & ~CG_CLIENT_REQ_MASK;
3470                 bif |= CG_CLIENT_REQ(0xd);
3471                 WREG32(CG_BIF_REQ_AND_RSP, bif);
3472             }
3473             tmp &= ~LC_HW_VOLTAGE_IF_CONTROL_MASK;
3474             tmp |= LC_HW_VOLTAGE_IF_CONTROL(1);
3475             tmp |= LC_GEN2_EN_STRAP;
3476
3477             tmp |= LC_CLR_FAILED_SPD_CHANGE_CNT;
3478             WREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL, tmp);
3479             udelay(10);
3480             tmp &= ~LC_CLR_FAILED_SPD_CHANGE_CNT;
3481             WREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL, tmp);
3482         } else {
3483             if (!pi->boot_in_gen2) {
3484                 bif = RREG32(CG_BIF_REQ_AND_RSP) & ~CG_CLIENT_REQ_MASK;
3485                 bif |= CG_CLIENT_REQ(0xd);
3486                 WREG32(CG_BIF_REQ_AND_RSP, bif);
3487
3488                 tmp &= ~LC_HW_VOLTAGE_IF_CONTROL_MASK;
3489                 tmp &= ~LC_GEN2_EN_STRAP;
3490             }
3491             WREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL, tmp);
3492         }
3493     }
3494 }
3495
3496 static void ni_enable_dynamic_pcie_gen2(struct radeon_device *rdev,
3497                     bool enable)
3498 {
3499     ni_enable_bif_dynamic_pcie_gen2(rdev, enable);
3500
3501     if (enable)
3502         WREG32_P(GENERAL_PWRMGT, ENABLE_GEN2PCIE, ~ENABLE_GEN2PCIE);
3503     else
3504         WREG32_P(GENERAL_PWRMGT, 0, ~ENABLE_GEN2PCIE);
3505 }
3506
3507 void ni_set_uvd_clock_before_set_eng_clock(struct radeon_device *rdev,
3508                        struct radeon_ps *new_ps,
3509                        struct radeon_ps *old_ps)
3510 {
3511     struct ni_ps *new_state = ni_get_ps(new_ps);
3512     struct ni_ps *current_state = ni_get_ps(old_ps);
3513
3514     if ((new_ps->vclk == old_ps->vclk) &&
3515         (new_ps->dclk == old_ps->dclk))
3516         return;
3517
3518     if (new_state->performance_levels[new_state->performance_level_count - 1].sclk >=
3519         current_state->performance_levels[current_state->performance_level_count - 1].sclk)
3520         return;
3521
3522     radeon_set_uvd_clocks(rdev, new_ps->vclk, new_ps->dclk);
3523 }
3524
3525 void ni_set_uvd_clock_after_set_eng_clock(struct radeon_device *rdev,
3526                       struct radeon_ps *new_ps,
3527                       struct radeon_ps *old_ps)
3528 {
3529     struct ni_ps *new_state = ni_get_ps(new_ps);
3530     struct ni_ps *current_state = ni_get_ps(old_ps);
3531
3532     if ((new_ps->vclk == old_ps->vclk) &&
3533         (new_ps->dclk == old_ps->dclk))
3534         return;
3535
3536     if (new_state->performance_levels[new_state->performance_level_count - 1].sclk <
3537         current_state->performance_levels[current_state->performance_level_count - 1].sclk)
3538         return;
3539
3540     radeon_set_uvd_clocks(rdev, new_ps->vclk, new_ps->dclk);
3541 }
3542
3543 void ni_dpm_setup_asic(struct radeon_device *rdev)
3544 {
3545     struct evergreen_power_info *eg_pi = evergreen_get_pi(rdev);
3546     int r;
3547
3548     r = ni_mc_load_microcode(rdev);
3549     if (r)
3550         DRM_ERROR("Failed to load MC firmware!\n");
3551     ni_read_clock_registers(rdev);
3552     btc_read_arb_registers(rdev);
3553     rv770_get_memory_type(rdev);
3554     if (eg_pi->pcie_performance_request)
3555         ni_advertise_gen2_capability(rdev);
3556     rv770_get_pcie_gen2_status(rdev);
3557     rv770_enable_acpi_pm(rdev);
3558 }
3559
3560 void ni_update_current_ps(struct radeon_device *rdev,
3561               struct radeon_ps *rps)
3562 {
3563     struct ni_ps *new_ps = ni_get_ps(rps);
3564     struct evergreen_power_info *eg_pi = evergreen_get_pi(rdev);
3565     struct ni_power_info *ni_pi = ni_get_pi(rdev);
3566
3567     eg_pi->current_rps = *rps;
3568     ni_pi->current_ps = *new_ps;
3569     eg_pi->current_rps.ps_priv = &ni_pi->current_ps;
3570 }
3571
3572 void ni_update_requested_ps(struct radeon_device *rdev,
3573                 struct radeon_ps *rps)
3574 {
3575     struct ni_ps *new_ps = ni_get_ps(rps);
3576     struct evergreen_power_info *eg_pi = evergreen_get_pi(rdev);
3577     struct ni_power_info *ni_pi = ni_get_pi(rdev);
3578
3579     eg_pi->requested_rps = *rps;
3580     ni_pi->requested_ps = *new_ps;
3581     eg_pi->requested_rps.ps_priv = &ni_pi->requested_ps;
3582 }
3583
3584 int ni_dpm_enable(struct radeon_device *rdev)
3585 {
3586     struct rv7xx_power_info *pi = rv770_get_pi(rdev);
3587     struct evergreen_power_info *eg_pi = evergreen_get_pi(rdev);
3588     struct radeon_ps *boot_ps = rdev->pm.dpm.boot_ps;
3589     int ret;
3590
3591     if (pi->gfx_clock_gating)
3592         ni_cg_clockgating_default(rdev);
3593     if (btc_dpm_enabled(rdev))
3594         return -EINVAL;
3595     if (pi->mg_clock_gating)
3596         ni_mg_clockgating_default(rdev);
3597     if (eg_pi->ls_clock_gating)
3598         ni_ls_clockgating_default(rdev);
3599     if (pi->voltage_control) {
3600         rv770_enable_voltage_control(rdev, true);
3601         ret = cypress_construct_voltage_tables(rdev);
3602         if (ret) {
3603             DRM_ERROR("cypress_construct_voltage_tables failed\n");
3604             return ret;
3605         }
3606     }
3607     if (eg_pi->dynamic_ac_timing) {
3608         ret = ni_initialize_mc_reg_table(rdev);
3609         if (ret)
3610             eg_pi->dynamic_ac_timing = false;
3611     }
3612     if (pi->dynamic_ss)
3613         cypress_enable_spread_spectrum(rdev, true);
3614     if (pi->thermal_protection)
3615         rv770_enable_thermal_protection(rdev, true);
3616     rv770_setup_bsp(rdev);
3617     rv770_program_git(rdev);
3618     rv770_program_tp(rdev);
3619     rv770_program_tpp(rdev);
3620     rv770_program_sstp(rdev);
3621     cypress_enable_display_gap(rdev);
3622     rv770_program_vc(rdev);
3623     if (pi->dynamic_pcie_gen2)
3624         ni_enable_dynamic_pcie_gen2(rdev, true);
3625     ret = rv770_upload_firmware(rdev);
3626     if (ret) {
3627         DRM_ERROR("rv770_upload_firmware failed\n");
3628         return ret;
3629     }
3630     ret = ni_process_firmware_header(rdev);
3631     if (ret) {
3632         DRM_ERROR("ni_process_firmware_header failed\n");
3633         return ret;
3634     }
3635     ret = ni_initial_switch_from_arb_f0_to_f1(rdev);
3636     if (ret) {
3637         DRM_ERROR("ni_initial_switch_from_arb_f0_to_f1 failed\n");
3638         return ret;
3639     }
3640     ret = ni_init_smc_table(rdev);
3641     if (ret) {
3642         DRM_ERROR("ni_init_smc_table failed\n");
3643         return ret;
3644     }
3645     ret = ni_init_smc_spll_table(rdev);
3646     if (ret) {
3647         DRM_ERROR("ni_init_smc_spll_table failed\n");
3648         return ret;
3649     }
3650     ret = ni_init_arb_table_index(rdev);
3651     if (ret) {
3652         DRM_ERROR("ni_init_arb_table_index failed\n");
3653         return ret;
3654     }
3655     if (eg_pi->dynamic_ac_timing) {
3656         ret = ni_populate_mc_reg_table(rdev, boot_ps);
3657         if (ret) {
3658             DRM_ERROR("ni_populate_mc_reg_table failed\n");
3659             return ret;
3660         }
3661     }
3662     ret = ni_initialize_smc_cac_tables(rdev);
3663     if (ret) {
3664         DRM_ERROR("ni_initialize_smc_cac_tables failed\n");
3665         return ret;
3666     }
3667     ret = ni_initialize_hardware_cac_manager(rdev);
3668     if (ret) {
3669         DRM_ERROR("ni_initialize_hardware_cac_manager failed\n");
3670         return ret;
3671     }
3672     ret = ni_populate_smc_tdp_limits(rdev, boot_ps);
3673     if (ret) {
3674         DRM_ERROR("ni_populate_smc_tdp_limits failed\n");
3675         return ret;
3676     }
3677     ni_program_response_times(rdev);
3678     r7xx_start_smc(rdev);
3679     ret = cypress_notify_smc_display_change(rdev, false);
3680     if (ret) {
3681         DRM_ERROR("cypress_notify_smc_display_change failed\n");
3682         return ret;
3683     }
3684     cypress_enable_sclk_control(rdev, true);
3685     if (eg_pi->memory_transition)
3686         cypress_enable_mclk_control(rdev, true);
3687     cypress_start_dpm(rdev);
3688     if (pi->gfx_clock_gating)
3689         ni_gfx_clockgating_enable(rdev, true);
3690     if (pi->mg_clock_gating)
3691         ni_mg_clockgating_enable(rdev, true);
3692     if (eg_pi->ls_clock_gating)
3693         ni_ls_clockgating_enable(rdev, true);
3694
3695     rv770_enable_auto_throttle_source(rdev, RADEON_DPM_AUTO_THROTTLE_SRC_THERMAL, true);
3696
3697     ni_update_current_ps(rdev, boot_ps);
3698
3699     return 0;
3700 }
3701
3702 void ni_dpm_disable(struct radeon_device *rdev)
3703 {
3704     struct rv7xx_power_info *pi = rv770_get_pi(rdev);
3705     struct evergreen_power_info *eg_pi = evergreen_get_pi(rdev);
3706     struct radeon_ps *boot_ps = rdev->pm.dpm.boot_ps;
3707
3708     if (!btc_dpm_enabled(rdev))
3709         return;
3710     rv770_clear_vc(rdev);
3711     if (pi->thermal_protection)
3712         rv770_enable_thermal_protection(rdev, false);
3713     ni_enable_power_containment(rdev, boot_ps, false);
3714     ni_enable_smc_cac(rdev, boot_ps, false);
3715     cypress_enable_spread_spectrum(rdev, false);
3716     rv770_enable_auto_throttle_source(rdev, RADEON_DPM_AUTO_THROTTLE_SRC_THERMAL, false);
3717     if (pi->dynamic_pcie_gen2)
3718         ni_enable_dynamic_pcie_gen2(rdev, false);
3719
3720     if (rdev->irq.installed &&
3721         r600_is_internal_thermal_sensor(rdev->pm.int_thermal_type)) {
3722         rdev->irq.dpm_thermal = false;
3723         radeon_irq_set(rdev);
3724     }
3725
3726     if (pi->gfx_clock_gating)
3727         ni_gfx_clockgating_enable(rdev, false);
3728     if (pi->mg_clock_gating)
3729         ni_mg_clockgating_enable(rdev, false);
3730     if (eg_pi->ls_clock_gating)
3731         ni_ls_clockgating_enable(rdev, false);
3732     ni_stop_dpm(rdev);
3733     btc_reset_to_default(rdev);
3734     ni_stop_smc(rdev);
3735     ni_force_switch_to_arb_f0(rdev);
3736
3737     ni_update_current_ps(rdev, boot_ps);
3738 }
3739
3740 static int ni_power_control_set_level(struct radeon_device *rdev)
3741 {
3742     struct radeon_ps *new_ps = rdev->pm.dpm.requested_ps;
3743     int ret;
3744
3745     ret = ni_restrict_performance_levels_before_switch(rdev);
3746     if (ret)
3747         return ret;
3748     ret = rv770_halt_smc(rdev);
3749     if (ret)
3750         return ret;
3751     ret = ni_populate_smc_tdp_limits(rdev, new_ps);
3752     if (ret)
3753         return ret;
3754     ret = rv770_resume_smc(rdev);
3755     if (ret)
3756         return ret;
3757     ret = rv770_set_sw_state(rdev);
3758     if (ret)
3759         return ret;
3760
3761     return 0;
3762 }
3763
3764 int ni_dpm_pre_set_power_state(struct radeon_device *rdev)
3765 {
3766     struct evergreen_power_info *eg_pi = evergreen_get_pi(rdev);
3767     struct radeon_ps requested_ps = *rdev->pm.dpm.requested_ps;
3768     struct radeon_ps *new_ps = &requested_ps;
3769
3770     ni_update_requested_ps(rdev, new_ps);
3771
3772     ni_apply_state_adjust_rules(rdev, &eg_pi->requested_rps);
3773
3774     return 0;
3775 }
3776
3777 int ni_dpm_set_power_state(struct radeon_device *rdev)
3778 {
3779     struct evergreen_power_info *eg_pi = evergreen_get_pi(rdev);
3780     struct radeon_ps *new_ps = &eg_pi->requested_rps;
3781     struct radeon_ps *old_ps = &eg_pi->current_rps;
3782     int ret;
3783
3784     ret = ni_restrict_performance_levels_before_switch(rdev);
3785     if (ret) {
3786         DRM_ERROR("ni_restrict_performance_levels_before_switch failed\n");
3787         return ret;
3788     }
3789     ni_set_uvd_clock_before_set_eng_clock(rdev, new_ps, old_ps);
3790     ret = ni_enable_power_containment(rdev, new_ps, false);
3791     if (ret) {
3792         DRM_ERROR("ni_enable_power_containment failed\n");
3793         return ret;
3794     }
3795     ret = ni_enable_smc_cac(rdev, new_ps, false);
3796     if (ret) {
3797         DRM_ERROR("ni_enable_smc_cac failed\n");
3798         return ret;
3799     }
3800     ret = rv770_halt_smc(rdev);
3801     if (ret) {
3802         DRM_ERROR("rv770_halt_smc failed\n");
3803         return ret;
3804     }
3805     if (eg_pi->smu_uvd_hs)
3806         btc_notify_uvd_to_smc(rdev, new_ps);
3807     ret = ni_upload_sw_state(rdev, new_ps);
3808     if (ret) {
3809         DRM_ERROR("ni_upload_sw_state failed\n");
3810         return ret;
3811     }
3812     if (eg_pi->dynamic_ac_timing) {
3813         ret = ni_upload_mc_reg_table(rdev, new_ps);
3814         if (ret) {
3815             DRM_ERROR("ni_upload_mc_reg_table failed\n");
3816             return ret;
3817         }
3818     }
3819     ret = ni_program_memory_timing_parameters(rdev, new_ps);
3820     if (ret) {
3821         DRM_ERROR("ni_program_memory_timing_parameters failed\n");
3822         return ret;
3823     }
3824     ret = rv770_resume_smc(rdev);
3825     if (ret) {
3826         DRM_ERROR("rv770_resume_smc failed\n");
3827         return ret;
3828     }
3829     ret = rv770_set_sw_state(rdev);
3830     if (ret) {
3831         DRM_ERROR("rv770_set_sw_state failed\n");
3832         return ret;
3833     }
3834     ni_set_uvd_clock_after_set_eng_clock(rdev, new_ps, old_ps);
3835     ret = ni_enable_smc_cac(rdev, new_ps, true);
3836     if (ret) {
3837         DRM_ERROR("ni_enable_smc_cac failed\n");
3838         return ret;
3839     }
3840     ret = ni_enable_power_containment(rdev, new_ps, true);
3841     if (ret) {
3842         DRM_ERROR("ni_enable_power_containment failed\n");
3843         return ret;
3844     }
3845
3846     /* update tdp */
3847     ret = ni_power_control_set_level(rdev);
3848     if (ret) {
3849         DRM_ERROR("ni_power_control_set_level failed\n");
3850         return ret;
3851     }
3852
3853     return 0;
3854 }
3855
3856 void ni_dpm_post_set_power_state(struct radeon_device *rdev)
3857 {
3858     struct evergreen_power_info *eg_pi = evergreen_get_pi(rdev);
3859     struct radeon_ps *new_ps = &eg_pi->requested_rps;
3860
3861     ni_update_current_ps(rdev, new_ps);
3862 }
3863
3864 #if 0
3865 void ni_dpm_reset_asic(struct radeon_device *rdev)
3866 {
3867     ni_restrict_performance_levels_before_switch(rdev);
3868     rv770_set_boot_state(rdev);
3869 }
3870 #endif
3871
3872 union power_info {
3873     struct _ATOM_POWERPLAY_INFO info;
3874     struct _ATOM_POWERPLAY_INFO_V2 info_2;
3875     struct _ATOM_POWERPLAY_INFO_V3 info_3;
3876     struct _ATOM_PPLIB_POWERPLAYTABLE pplib;
3877     struct _ATOM_PPLIB_POWERPLAYTABLE2 pplib2;
3878     struct _ATOM_PPLIB_POWERPLAYTABLE3 pplib3;
3879 };
3880
3881 union pplib_clock_info {
3882     struct _ATOM_PPLIB_R600_CLOCK_INFO r600;
3883     struct _ATOM_PPLIB_RS780_CLOCK_INFO rs780;
3884     struct _ATOM_PPLIB_EVERGREEN_CLOCK_INFO evergreen;
3885     struct _ATOM_PPLIB_SUMO_CLOCK_INFO sumo;
3886 };
3887
3888 union pplib_power_state {
3889     struct _ATOM_PPLIB_STATE v1;
3890     struct _ATOM_PPLIB_STATE_V2 v2;
3891 };
3892
3893 static void ni_parse_pplib_non_clock_info(struct radeon_device *rdev,
3894                       struct radeon_ps *rps,
3895                       struct _ATOM_PPLIB_NONCLOCK_INFO *non_clock_info,
3896                       u8 table_rev)
3897 {
3898     rps->caps = le32_to_cpu(non_clock_info->ulCapsAndSettings);
3899     rps->class = le16_to_cpu(non_clock_info->usClassification);
3900     rps->class2 = le16_to_cpu(non_clock_info->usClassification2);
3901
3902     if (ATOM_PPLIB_NONCLOCKINFO_VER1 < table_rev) {
3903         rps->vclk = le32_to_cpu(non_clock_info->ulVCLK);
3904         rps->dclk = le32_to_cpu(non_clock_info->ulDCLK);
3905     } else if (r600_is_uvd_state(rps->class, rps->class2)) {
3906         rps->vclk = RV770_DEFAULT_VCLK_FREQ;
3907         rps->dclk = RV770_DEFAULT_DCLK_FREQ;
3908     } else {
3909         rps->vclk = 0;
3910         rps->dclk = 0;
3911     }
3912
3913     if (rps->class & ATOM_PPLIB_CLASSIFICATION_BOOT)
3914         rdev->pm.dpm.boot_ps = rps;
3915     if (rps->class & ATOM_PPLIB_CLASSIFICATION_UVDSTATE)
3916         rdev->pm.dpm.uvd_ps = rps;
3917 }
3918
3919 static void ni_parse_pplib_clock_info(struct radeon_device *rdev,
3920                       struct radeon_ps *rps, int index,
3921                       union pplib_clock_info *clock_info)
3922 {
3923     struct rv7xx_power_info *pi = rv770_get_pi(rdev);
3924     struct evergreen_power_info *eg_pi = evergreen_get_pi(rdev);
3925     struct ni_ps *ps = ni_get_ps(rps);
3926     struct rv7xx_pl *pl = &ps->performance_levels[index];
3927
3928     ps->performance_level_count = index + 1;
3929
3930     pl->sclk = le16_to_cpu(clock_info->evergreen.usEngineClockLow);
3931     pl->sclk |= clock_info->evergreen.ucEngineClockHigh << 16;
3932     pl->mclk = le16_to_cpu(clock_info->evergreen.usMemoryClockLow);
3933     pl->mclk |= clock_info->evergreen.ucMemoryClockHigh << 16;
3934
3935     pl->vddc = le16_to_cpu(clock_info->evergreen.usVDDC);
3936     pl->vddci = le16_to_cpu(clock_info->evergreen.usVDDCI);
3937     pl->flags = le32_to_cpu(clock_info->evergreen.ulFlags);
3938
3939     /* patch up vddc if necessary */
3940     if (pl->vddc == 0xff01) {
3941         if (pi->max_vddc)
3942             pl->vddc = pi->max_vddc;
3943     }
3944
3945     if (rps->class & ATOM_PPLIB_CLASSIFICATION_ACPI) {
3946         pi->acpi_vddc = pl->vddc;
3947         eg_pi->acpi_vddci = pl->vddci;
3948         if (ps->performance_levels[0].flags & ATOM_PPLIB_R600_FLAGS_PCIEGEN2)
3949             pi->acpi_pcie_gen2 = true;
3950         else
3951             pi->acpi_pcie_gen2 = false;
3952     }
3953
3954     if (rps->class2 & ATOM_PPLIB_CLASSIFICATION2_ULV) {
3955         eg_pi->ulv.supported = true;
3956         eg_pi->ulv.pl = pl;
3957     }
3958
3959     if (pi->min_vddc_in_table > pl->vddc)
3960         pi->min_vddc_in_table = pl->vddc;
3961
3962     if (pi->max_vddc_in_table < pl->vddc)
3963         pi->max_vddc_in_table = pl->vddc;
3964
3965     /* patch up boot state */
3966     if (rps->class & ATOM_PPLIB_CLASSIFICATION_BOOT) {
3967         u16 vddc, vddci, mvdd;
3968         radeon_atombios_get_default_voltages(rdev, &vddc, &vddci, &mvdd);
3969         pl->mclk = rdev->clock.default_mclk;
3970         pl->sclk = rdev->clock.default_sclk;
3971         pl->vddc = vddc;
3972         pl->vddci = vddci;
3973     }
3974
3975     if ((rps->class & ATOM_PPLIB_CLASSIFICATION_UI_MASK) ==
3976         ATOM_PPLIB_CLASSIFICATION_UI_PERFORMANCE) {
3977         rdev->pm.dpm.dyn_state.max_clock_voltage_on_ac.sclk = pl->sclk;
3978         rdev->pm.dpm.dyn_state.max_clock_voltage_on_ac.mclk = pl->mclk;
3979         rdev->pm.dpm.dyn_state.max_clock_voltage_on_ac.vddc = pl->vddc;
3980         rdev->pm.dpm.dyn_state.max_clock_voltage_on_ac.vddci = pl->vddci;
3981     }
3982 }
3983
3984 static int ni_parse_power_table(struct radeon_device *rdev)
3985 {
3986     struct radeon_mode_info *mode_info = &rdev->mode_info;
3987     struct _ATOM_PPLIB_NONCLOCK_INFO *non_clock_info;
3988     union pplib_power_state *power_state;
3989     int i, j;
3990     union pplib_clock_info *clock_info;
3991     union power_info *power_info;
3992     int index = GetIndexIntoMasterTable(DATA, PowerPlayInfo);
3993     u16 data_offset;
3994     u8 frev, crev;
3995     struct ni_ps *ps;
3996
3997     if (!atom_parse_data_header(mode_info->atom_context, index, NULL,
3998                    &frev, &crev, &data_offset))
3999         return -EINVAL;
4000     power_info = (union power_info *)(mode_info->atom_context->bios + data_offset);
4001
4002     rdev->pm.dpm.ps = kcalloc(power_info->pplib.ucNumStates,
4003                   sizeof(struct radeon_ps),
4004                   GFP_KERNEL);
4005     if (!rdev->pm.dpm.ps)
4006         return -ENOMEM;
4007
4008     for (i = 0; i < power_info->pplib.ucNumStates; i++) {
4009         power_state = (union pplib_power_state *)
4010             (mode_info->atom_context->bios + data_offset +
4011              le16_to_cpu(power_info->pplib.usStateArrayOffset) +
4012              i * power_info->pplib.ucStateEntrySize);
4013         non_clock_info = (struct _ATOM_PPLIB_NONCLOCK_INFO *)
4014             (mode_info->atom_context->bios + data_offset +
4015              le16_to_cpu(power_info->pplib.usNonClockInfoArrayOffset) +
4016              (power_state->v1.ucNonClockStateIndex *
4017               power_info->pplib.ucNonClockSize));
4018         if (power_info->pplib.ucStateEntrySize - 1) {
4019             u8 *idx;
4020             ps = kzalloc(sizeof(struct ni_ps), GFP_KERNEL);
4021             if (ps == NULL) {
4022                 kfree(rdev->pm.dpm.ps);
4023                 return -ENOMEM;
4024             }
4025             rdev->pm.dpm.ps[i].ps_priv = ps;
4026             ni_parse_pplib_non_clock_info(rdev, &rdev->pm.dpm.ps[i],
4027                              non_clock_info,
4028                              power_info->pplib.ucNonClockSize);
4029             idx = (u8 *)&power_state->v1.ucClockStateIndices[0];
4030             for (j = 0; j < (power_info->pplib.ucStateEntrySize - 1); j++) {
4031                 clock_info = (union pplib_clock_info *)
4032                     (mode_info->atom_context->bios + data_offset +
4033                      le16_to_cpu(power_info->pplib.usClockInfoArrayOffset) +
4034                      (idx[j] * power_info->pplib.ucClockInfoSize));
4035                 ni_parse_pplib_clock_info(rdev,
4036                               &rdev->pm.dpm.ps[i], j,
4037                               clock_info);
4038             }
4039         }
4040     }
4041     rdev->pm.dpm.num_ps = power_info->pplib.ucNumStates;
4042     return 0;
4043 }
4044
4045 int ni_dpm_init(struct radeon_device *rdev)
4046 {
4047     struct rv7xx_power_info *pi;
4048     struct evergreen_power_info *eg_pi;
4049     struct ni_power_info *ni_pi;
4050     struct atom_clock_dividers dividers;
4051     int ret;
4052
4053     ni_pi = kzalloc(sizeof(struct ni_power_info), GFP_KERNEL);
4054     if (ni_pi == NULL)
4055         return -ENOMEM;
4056     rdev->pm.dpm.priv = ni_pi;
4057     eg_pi = &ni_pi->eg;
4058     pi = &eg_pi->rv7xx;
4059
4060     rv770_get_max_vddc(rdev);
4061
4062     eg_pi->ulv.supported = false;
4063     pi->acpi_vddc = 0;
4064     eg_pi->acpi_vddci = 0;
4065     pi->min_vddc_in_table = 0;
4066     pi->max_vddc_in_table = 0;
4067
4068     ret = r600_get_platform_caps(rdev);
4069     if (ret)
4070         return ret;
4071
4072     ret = ni_parse_power_table(rdev);
4073     if (ret)
4074         return ret;
4075     ret = r600_parse_extended_power_table(rdev);
4076     if (ret)
4077         return ret;
4078
4079     rdev->pm.dpm.dyn_state.vddc_dependency_on_dispclk.entries =
4080         kcalloc(4,
4081             sizeof(struct radeon_clock_voltage_dependency_entry),
4082             GFP_KERNEL);
4083     if (!rdev->pm.dpm.dyn_state.vddc_dependency_on_dispclk.entries) {
4084         r600_free_extended_power_table(rdev);
4085         return -ENOMEM;
4086     }
4087     rdev->pm.dpm.dyn_state.vddc_dependency_on_dispclk.count = 4;
4088     rdev->pm.dpm.dyn_state.vddc_dependency_on_dispclk.entries[0].clk = 0;
4089     rdev->pm.dpm.dyn_state.vddc_dependency_on_dispclk.entries[0].v = 0;
4090     rdev->pm.dpm.dyn_state.vddc_dependency_on_dispclk.entries[1].clk = 36000;
4091     rdev->pm.dpm.dyn_state.vddc_dependency_on_dispclk.entries[1].v = 720;
4092     rdev->pm.dpm.dyn_state.vddc_dependency_on_dispclk.entries[2].clk = 54000;
4093     rdev->pm.dpm.dyn_state.vddc_dependency_on_dispclk.entries[2].v = 810;
4094     rdev->pm.dpm.dyn_state.vddc_dependency_on_dispclk.entries[3].clk = 72000;
4095     rdev->pm.dpm.dyn_state.vddc_dependency_on_dispclk.entries[3].v = 900;
4096
4097     ni_patch_dependency_tables_based_on_leakage(rdev);
4098
4099     if (rdev->pm.dpm.voltage_response_time == 0)
4100         rdev->pm.dpm.voltage_response_time = R600_VOLTAGERESPONSETIME_DFLT;
4101     if (rdev->pm.dpm.backbias_response_time == 0)
4102         rdev->pm.dpm.backbias_response_time = R600_BACKBIASRESPONSETIME_DFLT;
4103
4104     ret = radeon_atom_get_clock_dividers(rdev, COMPUTE_ENGINE_PLL_PARAM,
4105                          0, false, &dividers);
4106     if (ret)
4107         pi->ref_div = dividers.ref_div + 1;
4108     else
4109         pi->ref_div = R600_REFERENCEDIVIDER_DFLT;
4110
4111     pi->rlp = RV770_RLP_DFLT;
4112     pi->rmp = RV770_RMP_DFLT;
4113     pi->lhp = RV770_LHP_DFLT;
4114     pi->lmp = RV770_LMP_DFLT;
4115
4116     eg_pi->ats[0].rlp = RV770_RLP_DFLT;
4117     eg_pi->ats[0].rmp = RV770_RMP_DFLT;
4118     eg_pi->ats[0].lhp = RV770_LHP_DFLT;
4119     eg_pi->ats[0].lmp = RV770_LMP_DFLT;
4120
4121     eg_pi->ats[1].rlp = BTC_RLP_UVD_DFLT;
4122     eg_pi->ats[1].rmp = BTC_RMP_UVD_DFLT;
4123     eg_pi->ats[1].lhp = BTC_LHP_UVD_DFLT;
4124     eg_pi->ats[1].lmp = BTC_LMP_UVD_DFLT;
4125
4126     eg_pi->smu_uvd_hs = true;
4127
4128     if (rdev->pdev->device == 0x6707) {
4129         pi->mclk_strobe_mode_threshold = 55000;
4130         pi->mclk_edc_enable_threshold = 55000;
4131         eg_pi->mclk_edc_wr_enable_threshold = 55000;
4132     } else {
4133         pi->mclk_strobe_mode_threshold = 40000;
4134         pi->mclk_edc_enable_threshold = 40000;
4135         eg_pi->mclk_edc_wr_enable_threshold = 40000;
4136     }
4137     ni_pi->mclk_rtt_mode_threshold = eg_pi->mclk_edc_wr_enable_threshold;
4138
4139     pi->voltage_control =
4140         radeon_atom_is_voltage_gpio(rdev, SET_VOLTAGE_TYPE_ASIC_VDDC, 0);
4141
4142     pi->mvdd_control =
4143         radeon_atom_is_voltage_gpio(rdev, SET_VOLTAGE_TYPE_ASIC_MVDDC, 0);
4144
4145     eg_pi->vddci_control =
4146         radeon_atom_is_voltage_gpio(rdev, SET_VOLTAGE_TYPE_ASIC_VDDCI, 0);
4147
4148     rv770_get_engine_memory_ss(rdev);
4149
4150     pi->asi = RV770_ASI_DFLT;
4151     pi->pasi = CYPRESS_HASI_DFLT;
4152     pi->vrc = CYPRESS_VRC_DFLT;
4153
4154     pi->power_gating = false;
4155
4156     pi->gfx_clock_gating = true;
4157
4158     pi->mg_clock_gating = true;
4159     pi->mgcgtssm = true;
4160     eg_pi->ls_clock_gating = false;
4161     eg_pi->sclk_deep_sleep = false;
4162
4163     pi->dynamic_pcie_gen2 = true;
4164
4165     if (rdev->pm.int_thermal_type != THERMAL_TYPE_NONE)
4166         pi->thermal_protection = true;
4167     else
4168         pi->thermal_protection = false;
4169
4170     pi->display_gap = true;
4171
4172     pi->dcodt = true;
4173
4174     pi->ulps = true;
4175
4176     eg_pi->dynamic_ac_timing = true;
4177     eg_pi->abm = true;
4178     eg_pi->mcls = true;
4179     eg_pi->light_sleep = true;
4180     eg_pi->memory_transition = true;
4181 #if defined(CONFIG_ACPI)
4182     eg_pi->pcie_performance_request =
4183         radeon_acpi_is_pcie_performance_request_supported(rdev);
4184 #else
4185     eg_pi->pcie_performance_request = false;
4186 #endif
4187
4188     eg_pi->dll_default_on = false;
4189
4190     eg_pi->sclk_deep_sleep = false;
4191
4192     pi->mclk_stutter_mode_threshold = 0;
4193
4194     pi->sram_end = SMC_RAM_END;
4195
4196     rdev->pm.dpm.dyn_state.mclk_sclk_ratio = 3;
4197     rdev->pm.dpm.dyn_state.vddc_vddci_delta = 200;
4198     rdev->pm.dpm.dyn_state.min_vddc_for_pcie_gen2 = 900;
4199     rdev->pm.dpm.dyn_state.valid_sclk_values.count = ARRAY_SIZE(btc_valid_sclk);
4200     rdev->pm.dpm.dyn_state.valid_sclk_values.values = btc_valid_sclk;
4201     rdev->pm.dpm.dyn_state.valid_mclk_values.count = 0;
4202     rdev->pm.dpm.dyn_state.valid_mclk_values.values = NULL;
4203     rdev->pm.dpm.dyn_state.sclk_mclk_delta = 12500;
4204
4205     ni_pi->cac_data.leakage_coefficients.at = 516;
4206     ni_pi->cac_data.leakage_coefficients.bt = 18;
4207     ni_pi->cac_data.leakage_coefficients.av = 51;
4208     ni_pi->cac_data.leakage_coefficients.bv = 2957;
4209
4210     switch (rdev->pdev->device) {
4211     case 0x6700:
4212     case 0x6701:
4213     case 0x6702:
4214     case 0x6703:
4215     case 0x6718:
4216         ni_pi->cac_weights = &cac_weights_cayman_xt;
4217         break;
4218     case 0x6705:
4219     case 0x6719:
4220     case 0x671D:
4221     case 0x671C:
4222     default:
4223         ni_pi->cac_weights = &cac_weights_cayman_pro;
4224         break;
4225     case 0x6704:
4226     case 0x6706:
4227     case 0x6707:
4228     case 0x6708:
4229     case 0x6709:
4230         ni_pi->cac_weights = &cac_weights_cayman_le;
4231         break;
4232     }
4233
4234     if (ni_pi->cac_weights->enable_power_containment_by_default) {
4235         ni_pi->enable_power_containment = true;
4236         ni_pi->enable_cac = true;
4237         ni_pi->enable_sq_ramping = true;
4238     } else {
4239         ni_pi->enable_power_containment = false;
4240         ni_pi->enable_cac = false;
4241         ni_pi->enable_sq_ramping = false;
4242     }
4243
4244     ni_pi->driver_calculate_cac_leakage = false;
4245     ni_pi->cac_configuration_required = true;
4246
4247     if (ni_pi->cac_configuration_required) {
4248         ni_pi->support_cac_long_term_average = true;
4249         ni_pi->lta_window_size = ni_pi->cac_weights->l2_lta_window_size;
4250         ni_pi->lts_truncate = ni_pi->cac_weights->lts_truncate;
4251     } else {
4252         ni_pi->support_cac_long_term_average = false;
4253         ni_pi->lta_window_size = 0;
4254         ni_pi->lts_truncate = 0;
4255     }
4256
4257     ni_pi->use_power_boost_limit = true;
4258
4259     /* make sure dc limits are valid */
4260     if ((rdev->pm.dpm.dyn_state.max_clock_voltage_on_dc.sclk == 0) ||
4261         (rdev->pm.dpm.dyn_state.max_clock_voltage_on_dc.mclk == 0))
4262         rdev->pm.dpm.dyn_state.max_clock_voltage_on_dc =
4263             rdev->pm.dpm.dyn_state.max_clock_voltage_on_ac;
4264
4265     return 0;
4266 }
4267
4268 void ni_dpm_fini(struct radeon_device *rdev)
4269 {
4270     int i;
4271
4272     for (i = 0; i < rdev->pm.dpm.num_ps; i++) {
4273         kfree(rdev->pm.dpm.ps[i].ps_priv);
4274     }
4275     kfree(rdev->pm.dpm.ps);
4276     kfree(rdev->pm.dpm.priv);
4277     kfree(rdev->pm.dpm.dyn_state.vddc_dependency_on_dispclk.entries);
4278     r600_free_extended_power_table(rdev);
4279 }
4280
4281 void ni_dpm_print_power_state(struct radeon_device *rdev,
4282                   struct radeon_ps *rps)
4283 {
4284     struct ni_ps *ps = ni_get_ps(rps);
4285     struct rv7xx_pl *pl;
4286     int i;
4287
4288     r600_dpm_print_class_info(rps->class, rps->class2);
4289     r600_dpm_print_cap_info(rps->caps);
4290     printk("\tuvd    vclk: %d dclk: %d\n", rps->vclk, rps->dclk);
4291     for (i = 0; i < ps->performance_level_count; i++) {
4292         pl = &ps->performance_levels[i];
4293         if (rdev->family >= CHIP_TAHITI)
4294             printk("\t\tpower level %d    sclk: %u mclk: %u vddc: %u vddci: %u pcie gen: %u\n",
4295                    i, pl->sclk, pl->mclk, pl->vddc, pl->vddci, pl->pcie_gen + 1);
4296         else
4297             printk("\t\tpower level %d    sclk: %u mclk: %u vddc: %u vddci: %u\n",
4298                    i, pl->sclk, pl->mclk, pl->vddc, pl->vddci);
4299     }
4300     r600_dpm_print_ps_status(rdev, rps);
4301 }
4302
4303 void ni_dpm_debugfs_print_current_performance_level(struct radeon_device *rdev,
4304                             struct seq_file *m)
4305 {
4306     struct evergreen_power_info *eg_pi = evergreen_get_pi(rdev);
4307     struct radeon_ps *rps = &eg_pi->current_rps;
4308     struct ni_ps *ps = ni_get_ps(rps);
4309     struct rv7xx_pl *pl;
4310     u32 current_index =
4311         (RREG32(TARGET_AND_CURRENT_PROFILE_INDEX) & CURRENT_STATE_INDEX_MASK) >>
4312         CURRENT_STATE_INDEX_SHIFT;
4313
4314     if (current_index >= ps->performance_level_count) {
4315         seq_printf(m, "invalid dpm profile %d\n", current_index);
4316     } else {
4317         pl = &ps->performance_levels[current_index];
4318         seq_printf(m, "uvd    vclk: %d dclk: %d\n", rps->vclk, rps->dclk);
4319         seq_printf(m, "power level %d    sclk: %u mclk: %u vddc: %u vddci: %u\n",
4320                current_index, pl->sclk, pl->mclk, pl->vddc, pl->vddci);
4321     }
4322 }
4323
4324 u32 ni_dpm_get_current_sclk(struct radeon_device *rdev)
4325 {
4326     struct evergreen_power_info *eg_pi = evergreen_get_pi(rdev);
4327     struct radeon_ps *rps = &eg_pi->current_rps;
4328     struct ni_ps *ps = ni_get_ps(rps);
4329     struct rv7xx_pl *pl;
4330     u32 current_index =
4331         (RREG32(TARGET_AND_CURRENT_PROFILE_INDEX) & CURRENT_STATE_INDEX_MASK) >>
4332         CURRENT_STATE_INDEX_SHIFT;
4333
4334     if (current_index >= ps->performance_level_count) {
4335         return 0;
4336     } else {
4337         pl = &ps->performance_levels[current_index];
4338         return pl->sclk;
4339     }
4340 }
4341
4342 u32 ni_dpm_get_current_mclk(struct radeon_device *rdev)
4343 {
4344     struct evergreen_power_info *eg_pi = evergreen_get_pi(rdev);
4345     struct radeon_ps *rps = &eg_pi->current_rps;
4346     struct ni_ps *ps = ni_get_ps(rps);
4347     struct rv7xx_pl *pl;
4348     u32 current_index =
4349         (RREG32(TARGET_AND_CURRENT_PROFILE_INDEX) & CURRENT_STATE_INDEX_MASK) >>
4350         CURRENT_STATE_INDEX_SHIFT;
4351
4352     if (current_index >= ps->performance_level_count) {
4353         return 0;
4354     } else {
4355         pl = &ps->performance_levels[current_index];
4356         return pl->mclk;
4357     }
4358 }
4359
4360 u32 ni_dpm_get_sclk(struct radeon_device *rdev, bool low)
4361 {
4362     struct evergreen_power_info *eg_pi = evergreen_get_pi(rdev);
4363     struct ni_ps *requested_state = ni_get_ps(&eg_pi->requested_rps);
4364
4365     if (low)
4366         return requested_state->performance_levels[0].sclk;
4367     else
4368         return requested_state->performance_levels[requested_state->performance_level_count - 1].sclk;
4369 }
4370
4371 u32 ni_dpm_get_mclk(struct radeon_device *rdev, bool low)
4372 {
4373     struct evergreen_power_info *eg_pi = evergreen_get_pi(rdev);
4374     struct ni_ps *requested_state = ni_get_ps(&eg_pi->requested_rps);
4375
4376     if (low)
4377         return requested_state->performance_levels[0].mclk;
4378     else
4379         return requested_state->performance_levels[requested_state->performance_level_count - 1].mclk;
4380 }
4381