Back to home page

OSCL-LXR
 
 

     

0001 // SPDX-License-Identifier: GPL-2.0
0002 /*
0003  * Copyright (c) 2015-2020, NVIDIA CORPORATION.  All rights reserved.
0004  */
0005 
0006 #include <linux/bitfield.h>
0007 #include <linux/clk.h>
0008 #include <linux/clk/tegra.h>
0009 #include <linux/debugfs.h>
0010 #include <linux/delay.h>
0011 #include <linux/kernel.h>
0012 #include <linux/module.h>
0013 #include <linux/of_address.h>
0014 #include <linux/of_platform.h>
0015 #include <linux/of_reserved_mem.h>
0016 #include <linux/slab.h>
0017 #include <linux/thermal.h>
0018 #include <soc/tegra/fuse.h>
0019 #include <soc/tegra/mc.h>
0020 
0021 #include "tegra210-emc.h"
0022 #include "tegra210-mc.h"
0023 
0024 /* CLK_RST_CONTROLLER_CLK_SOURCE_EMC */
0025 #define EMC_CLK_EMC_2X_CLK_SRC_SHIFT            29
0026 #define EMC_CLK_EMC_2X_CLK_SRC_MASK         \
0027     (0x7 << EMC_CLK_EMC_2X_CLK_SRC_SHIFT)
0028 #define EMC_CLK_SOURCE_PLLM_LJ              0x4
0029 #define EMC_CLK_SOURCE_PLLMB_LJ             0x5
0030 #define EMC_CLK_FORCE_CC_TRIGGER            BIT(27)
0031 #define EMC_CLK_MC_EMC_SAME_FREQ            BIT(16)
0032 #define EMC_CLK_EMC_2X_CLK_DIVISOR_SHIFT        0
0033 #define EMC_CLK_EMC_2X_CLK_DIVISOR_MASK         \
0034     (0xff << EMC_CLK_EMC_2X_CLK_DIVISOR_SHIFT)
0035 
0036 /* CLK_RST_CONTROLLER_CLK_SOURCE_EMC_DLL */
0037 #define DLL_CLK_EMC_DLL_CLK_SRC_SHIFT           29
0038 #define DLL_CLK_EMC_DLL_CLK_SRC_MASK            \
0039     (0x7 << DLL_CLK_EMC_DLL_CLK_SRC_SHIFT)
0040 #define DLL_CLK_EMC_DLL_DDLL_CLK_SEL_SHIFT      10
0041 #define DLL_CLK_EMC_DLL_DDLL_CLK_SEL_MASK       \
0042     (0x3 << DLL_CLK_EMC_DLL_DDLL_CLK_SEL_SHIFT)
0043 #define PLLM_VCOA                   0
0044 #define PLLM_VCOB                   1
0045 #define EMC_DLL_SWITCH_OUT              2
0046 #define DLL_CLK_EMC_DLL_CLK_DIVISOR_SHIFT       0
0047 #define DLL_CLK_EMC_DLL_CLK_DIVISOR_MASK        \
0048     (0xff << DLL_CLK_EMC_DLL_CLK_DIVISOR_SHIFT)
0049 
0050 /* MC_EMEM_ARB_MISC0 */
0051 #define MC_EMEM_ARB_MISC0_EMC_SAME_FREQ         BIT(27)
0052 
0053 /* EMC_DATA_BRLSHFT_X */
0054 #define EMC0_EMC_DATA_BRLSHFT_0_INDEX   2
0055 #define EMC1_EMC_DATA_BRLSHFT_0_INDEX   3
0056 #define EMC0_EMC_DATA_BRLSHFT_1_INDEX   4
0057 #define EMC1_EMC_DATA_BRLSHFT_1_INDEX   5
0058 
0059 #define TRIM_REG(chan, rank, reg, byte)                 \
0060     (((EMC_PMACRO_OB_DDLL_LONG_DQ_RANK ## rank ## _ ## reg ##   \
0061        _OB_DDLL_LONG_DQ_RANK ## rank ## _BYTE ## byte ## _MASK &    \
0062        next->trim_regs[EMC_PMACRO_OB_DDLL_LONG_DQ_RANK ##       \
0063                  rank ## _ ## reg ## _INDEX]) >>    \
0064       EMC_PMACRO_OB_DDLL_LONG_DQ_RANK ## rank ## _ ## reg ##    \
0065       _OB_DDLL_LONG_DQ_RANK ## rank ## _BYTE ## byte ## _SHIFT) \
0066      +                              \
0067      (((EMC_DATA_BRLSHFT_ ## rank ## _RANK ## rank ## _BYTE ##  \
0068         byte ## _DATA_BRLSHFT_MASK &                \
0069         next->trim_perch_regs[EMC ## chan ##            \
0070                   _EMC_DATA_BRLSHFT_ ## rank ## _INDEX]) >> \
0071        EMC_DATA_BRLSHFT_ ## rank ## _RANK ## rank ## _BYTE ##   \
0072        byte ## _DATA_BRLSHFT_SHIFT) * 64))
0073 
0074 #define CALC_TEMP(rank, reg, byte1, byte2, n)               \
0075     (((new[n] << EMC_PMACRO_OB_DDLL_LONG_DQ_RANK ## rank ## _ ##    \
0076        reg ## _OB_DDLL_LONG_DQ_RANK ## rank ## _BYTE ## byte1 ## _SHIFT) & \
0077       EMC_PMACRO_OB_DDLL_LONG_DQ_RANK ## rank ## _ ## reg ##    \
0078       _OB_DDLL_LONG_DQ_RANK ## rank ## _BYTE ## byte1 ## _MASK) \
0079      |                              \
0080      ((new[n + 1] << EMC_PMACRO_OB_DDLL_LONG_DQ_RANK ## rank ## _ ##\
0081        reg ## _OB_DDLL_LONG_DQ_RANK ## rank ## _BYTE ## byte2 ## _SHIFT) & \
0082       EMC_PMACRO_OB_DDLL_LONG_DQ_RANK ## rank ## _ ## reg ##    \
0083       _OB_DDLL_LONG_DQ_RANK ## rank ## _BYTE ## byte2 ## _MASK))
0084 
0085 #define REFRESH_SPEEDUP(value, speedup) \
0086         (((value) & 0xffff0000) | ((value) & 0xffff) * (speedup))
0087 
0088 #define LPDDR2_MR4_SRR GENMASK(2, 0)
0089 
0090 static const struct tegra210_emc_sequence *tegra210_emc_sequences[] = {
0091     &tegra210_emc_r21021,
0092 };
0093 
0094 static const struct tegra210_emc_table_register_offsets
0095 tegra210_emc_table_register_offsets = {
0096     .burst = {
0097         EMC_RC,
0098         EMC_RFC,
0099         EMC_RFCPB,
0100         EMC_REFCTRL2,
0101         EMC_RFC_SLR,
0102         EMC_RAS,
0103         EMC_RP,
0104         EMC_R2W,
0105         EMC_W2R,
0106         EMC_R2P,
0107         EMC_W2P,
0108         EMC_R2R,
0109         EMC_TPPD,
0110         EMC_CCDMW,
0111         EMC_RD_RCD,
0112         EMC_WR_RCD,
0113         EMC_RRD,
0114         EMC_REXT,
0115         EMC_WEXT,
0116         EMC_WDV_CHK,
0117         EMC_WDV,
0118         EMC_WSV,
0119         EMC_WEV,
0120         EMC_WDV_MASK,
0121         EMC_WS_DURATION,
0122         EMC_WE_DURATION,
0123         EMC_QUSE,
0124         EMC_QUSE_WIDTH,
0125         EMC_IBDLY,
0126         EMC_OBDLY,
0127         EMC_EINPUT,
0128         EMC_MRW6,
0129         EMC_EINPUT_DURATION,
0130         EMC_PUTERM_EXTRA,
0131         EMC_PUTERM_WIDTH,
0132         EMC_QRST,
0133         EMC_QSAFE,
0134         EMC_RDV,
0135         EMC_RDV_MASK,
0136         EMC_RDV_EARLY,
0137         EMC_RDV_EARLY_MASK,
0138         EMC_REFRESH,
0139         EMC_BURST_REFRESH_NUM,
0140         EMC_PRE_REFRESH_REQ_CNT,
0141         EMC_PDEX2WR,
0142         EMC_PDEX2RD,
0143         EMC_PCHG2PDEN,
0144         EMC_ACT2PDEN,
0145         EMC_AR2PDEN,
0146         EMC_RW2PDEN,
0147         EMC_CKE2PDEN,
0148         EMC_PDEX2CKE,
0149         EMC_PDEX2MRR,
0150         EMC_TXSR,
0151         EMC_TXSRDLL,
0152         EMC_TCKE,
0153         EMC_TCKESR,
0154         EMC_TPD,
0155         EMC_TFAW,
0156         EMC_TRPAB,
0157         EMC_TCLKSTABLE,
0158         EMC_TCLKSTOP,
0159         EMC_MRW7,
0160         EMC_TREFBW,
0161         EMC_ODT_WRITE,
0162         EMC_FBIO_CFG5,
0163         EMC_FBIO_CFG7,
0164         EMC_CFG_DIG_DLL,
0165         EMC_CFG_DIG_DLL_PERIOD,
0166         EMC_PMACRO_IB_RXRT,
0167         EMC_CFG_PIPE_1,
0168         EMC_CFG_PIPE_2,
0169         EMC_PMACRO_QUSE_DDLL_RANK0_4,
0170         EMC_PMACRO_QUSE_DDLL_RANK0_5,
0171         EMC_PMACRO_QUSE_DDLL_RANK1_4,
0172         EMC_PMACRO_QUSE_DDLL_RANK1_5,
0173         EMC_MRW8,
0174         EMC_PMACRO_OB_DDLL_LONG_DQ_RANK1_4,
0175         EMC_PMACRO_OB_DDLL_LONG_DQ_RANK1_5,
0176         EMC_PMACRO_OB_DDLL_LONG_DQS_RANK0_0,
0177         EMC_PMACRO_OB_DDLL_LONG_DQS_RANK0_1,
0178         EMC_PMACRO_OB_DDLL_LONG_DQS_RANK0_2,
0179         EMC_PMACRO_OB_DDLL_LONG_DQS_RANK0_3,
0180         EMC_PMACRO_OB_DDLL_LONG_DQS_RANK0_4,
0181         EMC_PMACRO_OB_DDLL_LONG_DQS_RANK0_5,
0182         EMC_PMACRO_OB_DDLL_LONG_DQS_RANK1_0,
0183         EMC_PMACRO_OB_DDLL_LONG_DQS_RANK1_1,
0184         EMC_PMACRO_OB_DDLL_LONG_DQS_RANK1_2,
0185         EMC_PMACRO_OB_DDLL_LONG_DQS_RANK1_3,
0186         EMC_PMACRO_OB_DDLL_LONG_DQS_RANK1_4,
0187         EMC_PMACRO_OB_DDLL_LONG_DQS_RANK1_5,
0188         EMC_PMACRO_DDLL_LONG_CMD_0,
0189         EMC_PMACRO_DDLL_LONG_CMD_1,
0190         EMC_PMACRO_DDLL_LONG_CMD_2,
0191         EMC_PMACRO_DDLL_LONG_CMD_3,
0192         EMC_PMACRO_DDLL_LONG_CMD_4,
0193         EMC_PMACRO_DDLL_SHORT_CMD_0,
0194         EMC_PMACRO_DDLL_SHORT_CMD_1,
0195         EMC_PMACRO_DDLL_SHORT_CMD_2,
0196         EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK0_BYTE0_3,
0197         EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK0_BYTE1_3,
0198         EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK0_BYTE2_3,
0199         EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK0_BYTE3_3,
0200         EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK0_BYTE4_3,
0201         EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK0_BYTE5_3,
0202         EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK0_BYTE6_3,
0203         EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK0_BYTE7_3,
0204         EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK0_CMD0_3,
0205         EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK0_CMD1_3,
0206         EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK0_CMD2_3,
0207         EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK0_CMD3_3,
0208         EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK1_BYTE0_3,
0209         EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK1_BYTE1_3,
0210         EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK1_BYTE2_3,
0211         EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK1_BYTE3_3,
0212         EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK1_BYTE4_3,
0213         EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK1_BYTE5_3,
0214         EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK1_BYTE6_3,
0215         EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK1_BYTE7_3,
0216         EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK1_CMD0_0,
0217         EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK1_CMD0_1,
0218         EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK1_CMD0_2,
0219         EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK1_CMD0_3,
0220         EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK1_CMD1_0,
0221         EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK1_CMD1_1,
0222         EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK1_CMD1_2,
0223         EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK1_CMD1_3,
0224         EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK1_CMD2_0,
0225         EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK1_CMD2_1,
0226         EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK1_CMD2_2,
0227         EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK1_CMD2_3,
0228         EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK1_CMD3_0,
0229         EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK1_CMD3_1,
0230         EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK1_CMD3_2,
0231         EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK1_CMD3_3,
0232         EMC_TXDSRVTTGEN,
0233         EMC_FDPD_CTRL_DQ,
0234         EMC_FDPD_CTRL_CMD,
0235         EMC_FBIO_SPARE,
0236         EMC_ZCAL_INTERVAL,
0237         EMC_ZCAL_WAIT_CNT,
0238         EMC_MRS_WAIT_CNT,
0239         EMC_MRS_WAIT_CNT2,
0240         EMC_AUTO_CAL_CHANNEL,
0241         EMC_DLL_CFG_0,
0242         EMC_DLL_CFG_1,
0243         EMC_PMACRO_AUTOCAL_CFG_COMMON,
0244         EMC_PMACRO_ZCTRL,
0245         EMC_CFG,
0246         EMC_CFG_PIPE,
0247         EMC_DYN_SELF_REF_CONTROL,
0248         EMC_QPOP,
0249         EMC_DQS_BRLSHFT_0,
0250         EMC_DQS_BRLSHFT_1,
0251         EMC_CMD_BRLSHFT_2,
0252         EMC_CMD_BRLSHFT_3,
0253         EMC_PMACRO_PAD_CFG_CTRL,
0254         EMC_PMACRO_DATA_PAD_RX_CTRL,
0255         EMC_PMACRO_CMD_PAD_RX_CTRL,
0256         EMC_PMACRO_DATA_RX_TERM_MODE,
0257         EMC_PMACRO_CMD_RX_TERM_MODE,
0258         EMC_PMACRO_CMD_PAD_TX_CTRL,
0259         EMC_PMACRO_DATA_PAD_TX_CTRL,
0260         EMC_PMACRO_COMMON_PAD_TX_CTRL,
0261         EMC_PMACRO_VTTGEN_CTRL_0,
0262         EMC_PMACRO_VTTGEN_CTRL_1,
0263         EMC_PMACRO_VTTGEN_CTRL_2,
0264         EMC_PMACRO_BRICK_CTRL_RFU1,
0265         EMC_PMACRO_CMD_BRICK_CTRL_FDPD,
0266         EMC_PMACRO_BRICK_CTRL_RFU2,
0267         EMC_PMACRO_DATA_BRICK_CTRL_FDPD,
0268         EMC_PMACRO_BG_BIAS_CTRL_0,
0269         EMC_CFG_3,
0270         EMC_PMACRO_TX_PWRD_0,
0271         EMC_PMACRO_TX_PWRD_1,
0272         EMC_PMACRO_TX_PWRD_2,
0273         EMC_PMACRO_TX_PWRD_3,
0274         EMC_PMACRO_TX_PWRD_4,
0275         EMC_PMACRO_TX_PWRD_5,
0276         EMC_CONFIG_SAMPLE_DELAY,
0277         EMC_PMACRO_TX_SEL_CLK_SRC_0,
0278         EMC_PMACRO_TX_SEL_CLK_SRC_1,
0279         EMC_PMACRO_TX_SEL_CLK_SRC_2,
0280         EMC_PMACRO_TX_SEL_CLK_SRC_3,
0281         EMC_PMACRO_TX_SEL_CLK_SRC_4,
0282         EMC_PMACRO_TX_SEL_CLK_SRC_5,
0283         EMC_PMACRO_DDLL_BYPASS,
0284         EMC_PMACRO_DDLL_PWRD_0,
0285         EMC_PMACRO_DDLL_PWRD_1,
0286         EMC_PMACRO_DDLL_PWRD_2,
0287         EMC_PMACRO_CMD_CTRL_0,
0288         EMC_PMACRO_CMD_CTRL_1,
0289         EMC_PMACRO_CMD_CTRL_2,
0290         EMC_TR_TIMING_0,
0291         EMC_TR_DVFS,
0292         EMC_TR_CTRL_1,
0293         EMC_TR_RDV,
0294         EMC_TR_QPOP,
0295         EMC_TR_RDV_MASK,
0296         EMC_MRW14,
0297         EMC_TR_QSAFE,
0298         EMC_TR_QRST,
0299         EMC_TRAINING_CTRL,
0300         EMC_TRAINING_SETTLE,
0301         EMC_TRAINING_VREF_SETTLE,
0302         EMC_TRAINING_CA_FINE_CTRL,
0303         EMC_TRAINING_CA_CTRL_MISC,
0304         EMC_TRAINING_CA_CTRL_MISC1,
0305         EMC_TRAINING_CA_VREF_CTRL,
0306         EMC_TRAINING_QUSE_CORS_CTRL,
0307         EMC_TRAINING_QUSE_FINE_CTRL,
0308         EMC_TRAINING_QUSE_CTRL_MISC,
0309         EMC_TRAINING_QUSE_VREF_CTRL,
0310         EMC_TRAINING_READ_FINE_CTRL,
0311         EMC_TRAINING_READ_CTRL_MISC,
0312         EMC_TRAINING_READ_VREF_CTRL,
0313         EMC_TRAINING_WRITE_FINE_CTRL,
0314         EMC_TRAINING_WRITE_CTRL_MISC,
0315         EMC_TRAINING_WRITE_VREF_CTRL,
0316         EMC_TRAINING_MPC,
0317         EMC_MRW15,
0318     },
0319     .trim = {
0320         EMC_PMACRO_IB_DDLL_LONG_DQS_RANK0_0,
0321         EMC_PMACRO_IB_DDLL_LONG_DQS_RANK0_1,
0322         EMC_PMACRO_IB_DDLL_LONG_DQS_RANK0_2,
0323         EMC_PMACRO_IB_DDLL_LONG_DQS_RANK0_3,
0324         EMC_PMACRO_IB_DDLL_LONG_DQS_RANK1_0,
0325         EMC_PMACRO_IB_DDLL_LONG_DQS_RANK1_1,
0326         EMC_PMACRO_IB_DDLL_LONG_DQS_RANK1_2,
0327         EMC_PMACRO_IB_DDLL_LONG_DQS_RANK1_3,
0328         EMC_PMACRO_IB_DDLL_SHORT_DQ_RANK0_BYTE0_0,
0329         EMC_PMACRO_IB_DDLL_SHORT_DQ_RANK0_BYTE0_1,
0330         EMC_PMACRO_IB_DDLL_SHORT_DQ_RANK0_BYTE0_2,
0331         EMC_PMACRO_IB_DDLL_SHORT_DQ_RANK0_BYTE1_0,
0332         EMC_PMACRO_IB_DDLL_SHORT_DQ_RANK0_BYTE1_1,
0333         EMC_PMACRO_IB_DDLL_SHORT_DQ_RANK0_BYTE1_2,
0334         EMC_PMACRO_IB_DDLL_SHORT_DQ_RANK0_BYTE2_0,
0335         EMC_PMACRO_IB_DDLL_SHORT_DQ_RANK0_BYTE2_1,
0336         EMC_PMACRO_IB_DDLL_SHORT_DQ_RANK0_BYTE2_2,
0337         EMC_PMACRO_IB_DDLL_SHORT_DQ_RANK0_BYTE3_0,
0338         EMC_PMACRO_IB_DDLL_SHORT_DQ_RANK0_BYTE3_1,
0339         EMC_PMACRO_IB_DDLL_SHORT_DQ_RANK0_BYTE3_2,
0340         EMC_PMACRO_IB_DDLL_SHORT_DQ_RANK0_BYTE4_0,
0341         EMC_PMACRO_IB_DDLL_SHORT_DQ_RANK0_BYTE4_1,
0342         EMC_PMACRO_IB_DDLL_SHORT_DQ_RANK0_BYTE4_2,
0343         EMC_PMACRO_IB_DDLL_SHORT_DQ_RANK0_BYTE5_0,
0344         EMC_PMACRO_IB_DDLL_SHORT_DQ_RANK0_BYTE5_1,
0345         EMC_PMACRO_IB_DDLL_SHORT_DQ_RANK0_BYTE5_2,
0346         EMC_PMACRO_IB_DDLL_SHORT_DQ_RANK0_BYTE6_0,
0347         EMC_PMACRO_IB_DDLL_SHORT_DQ_RANK0_BYTE6_1,
0348         EMC_PMACRO_IB_DDLL_SHORT_DQ_RANK0_BYTE6_2,
0349         EMC_PMACRO_IB_DDLL_SHORT_DQ_RANK0_BYTE7_0,
0350         EMC_PMACRO_IB_DDLL_SHORT_DQ_RANK0_BYTE7_1,
0351         EMC_PMACRO_IB_DDLL_SHORT_DQ_RANK0_BYTE7_2,
0352         EMC_PMACRO_IB_DDLL_SHORT_DQ_RANK1_BYTE0_0,
0353         EMC_PMACRO_IB_DDLL_SHORT_DQ_RANK1_BYTE0_1,
0354         EMC_PMACRO_IB_DDLL_SHORT_DQ_RANK1_BYTE0_2,
0355         EMC_PMACRO_IB_DDLL_SHORT_DQ_RANK1_BYTE1_0,
0356         EMC_PMACRO_IB_DDLL_SHORT_DQ_RANK1_BYTE1_1,
0357         EMC_PMACRO_IB_DDLL_SHORT_DQ_RANK1_BYTE1_2,
0358         EMC_PMACRO_IB_DDLL_SHORT_DQ_RANK1_BYTE2_0,
0359         EMC_PMACRO_IB_DDLL_SHORT_DQ_RANK1_BYTE2_1,
0360         EMC_PMACRO_IB_DDLL_SHORT_DQ_RANK1_BYTE2_2,
0361         EMC_PMACRO_IB_DDLL_SHORT_DQ_RANK1_BYTE3_0,
0362         EMC_PMACRO_IB_DDLL_SHORT_DQ_RANK1_BYTE3_1,
0363         EMC_PMACRO_IB_DDLL_SHORT_DQ_RANK1_BYTE3_2,
0364         EMC_PMACRO_IB_DDLL_SHORT_DQ_RANK1_BYTE4_0,
0365         EMC_PMACRO_IB_DDLL_SHORT_DQ_RANK1_BYTE4_1,
0366         EMC_PMACRO_IB_DDLL_SHORT_DQ_RANK1_BYTE4_2,
0367         EMC_PMACRO_IB_DDLL_SHORT_DQ_RANK1_BYTE5_0,
0368         EMC_PMACRO_IB_DDLL_SHORT_DQ_RANK1_BYTE5_1,
0369         EMC_PMACRO_IB_DDLL_SHORT_DQ_RANK1_BYTE5_2,
0370         EMC_PMACRO_IB_DDLL_SHORT_DQ_RANK1_BYTE6_0,
0371         EMC_PMACRO_IB_DDLL_SHORT_DQ_RANK1_BYTE6_1,
0372         EMC_PMACRO_IB_DDLL_SHORT_DQ_RANK1_BYTE6_2,
0373         EMC_PMACRO_IB_DDLL_SHORT_DQ_RANK1_BYTE7_0,
0374         EMC_PMACRO_IB_DDLL_SHORT_DQ_RANK1_BYTE7_1,
0375         EMC_PMACRO_IB_DDLL_SHORT_DQ_RANK1_BYTE7_2,
0376         EMC_PMACRO_IB_VREF_DQS_0,
0377         EMC_PMACRO_IB_VREF_DQS_1,
0378         EMC_PMACRO_IB_VREF_DQ_0,
0379         EMC_PMACRO_IB_VREF_DQ_1,
0380         EMC_PMACRO_OB_DDLL_LONG_DQ_RANK0_0,
0381         EMC_PMACRO_OB_DDLL_LONG_DQ_RANK0_1,
0382         EMC_PMACRO_OB_DDLL_LONG_DQ_RANK0_2,
0383         EMC_PMACRO_OB_DDLL_LONG_DQ_RANK0_3,
0384         EMC_PMACRO_OB_DDLL_LONG_DQ_RANK0_4,
0385         EMC_PMACRO_OB_DDLL_LONG_DQ_RANK0_5,
0386         EMC_PMACRO_OB_DDLL_LONG_DQ_RANK1_0,
0387         EMC_PMACRO_OB_DDLL_LONG_DQ_RANK1_1,
0388         EMC_PMACRO_OB_DDLL_LONG_DQ_RANK1_2,
0389         EMC_PMACRO_OB_DDLL_LONG_DQ_RANK1_3,
0390         EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK0_BYTE0_0,
0391         EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK0_BYTE0_1,
0392         EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK0_BYTE0_2,
0393         EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK0_BYTE1_0,
0394         EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK0_BYTE1_1,
0395         EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK0_BYTE1_2,
0396         EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK0_BYTE2_0,
0397         EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK0_BYTE2_1,
0398         EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK0_BYTE2_2,
0399         EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK0_BYTE3_0,
0400         EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK0_BYTE3_1,
0401         EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK0_BYTE3_2,
0402         EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK0_BYTE4_0,
0403         EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK0_BYTE4_1,
0404         EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK0_BYTE4_2,
0405         EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK0_BYTE5_0,
0406         EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK0_BYTE5_1,
0407         EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK0_BYTE5_2,
0408         EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK0_BYTE6_0,
0409         EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK0_BYTE6_1,
0410         EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK0_BYTE6_2,
0411         EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK0_BYTE7_0,
0412         EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK0_BYTE7_1,
0413         EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK0_BYTE7_2,
0414         EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK0_CMD0_0,
0415         EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK0_CMD0_1,
0416         EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK0_CMD0_2,
0417         EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK0_CMD1_0,
0418         EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK0_CMD1_1,
0419         EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK0_CMD1_2,
0420         EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK0_CMD2_0,
0421         EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK0_CMD2_1,
0422         EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK0_CMD2_2,
0423         EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK0_CMD3_0,
0424         EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK0_CMD3_1,
0425         EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK0_CMD3_2,
0426         EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK1_BYTE0_0,
0427         EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK1_BYTE0_1,
0428         EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK1_BYTE0_2,
0429         EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK1_BYTE1_0,
0430         EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK1_BYTE1_1,
0431         EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK1_BYTE1_2,
0432         EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK1_BYTE2_0,
0433         EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK1_BYTE2_1,
0434         EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK1_BYTE2_2,
0435         EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK1_BYTE3_0,
0436         EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK1_BYTE3_1,
0437         EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK1_BYTE3_2,
0438         EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK1_BYTE4_0,
0439         EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK1_BYTE4_1,
0440         EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK1_BYTE4_2,
0441         EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK1_BYTE5_0,
0442         EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK1_BYTE5_1,
0443         EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK1_BYTE5_2,
0444         EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK1_BYTE6_0,
0445         EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK1_BYTE6_1,
0446         EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK1_BYTE6_2,
0447         EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK1_BYTE7_0,
0448         EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK1_BYTE7_1,
0449         EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK1_BYTE7_2,
0450         EMC_PMACRO_QUSE_DDLL_RANK0_0,
0451         EMC_PMACRO_QUSE_DDLL_RANK0_1,
0452         EMC_PMACRO_QUSE_DDLL_RANK0_2,
0453         EMC_PMACRO_QUSE_DDLL_RANK0_3,
0454         EMC_PMACRO_QUSE_DDLL_RANK1_0,
0455         EMC_PMACRO_QUSE_DDLL_RANK1_1,
0456         EMC_PMACRO_QUSE_DDLL_RANK1_2,
0457         EMC_PMACRO_QUSE_DDLL_RANK1_3
0458     },
0459     .burst_mc = {
0460         MC_EMEM_ARB_CFG,
0461         MC_EMEM_ARB_OUTSTANDING_REQ,
0462         MC_EMEM_ARB_REFPB_HP_CTRL,
0463         MC_EMEM_ARB_REFPB_BANK_CTRL,
0464         MC_EMEM_ARB_TIMING_RCD,
0465         MC_EMEM_ARB_TIMING_RP,
0466         MC_EMEM_ARB_TIMING_RC,
0467         MC_EMEM_ARB_TIMING_RAS,
0468         MC_EMEM_ARB_TIMING_FAW,
0469         MC_EMEM_ARB_TIMING_RRD,
0470         MC_EMEM_ARB_TIMING_RAP2PRE,
0471         MC_EMEM_ARB_TIMING_WAP2PRE,
0472         MC_EMEM_ARB_TIMING_R2R,
0473         MC_EMEM_ARB_TIMING_W2W,
0474         MC_EMEM_ARB_TIMING_R2W,
0475         MC_EMEM_ARB_TIMING_CCDMW,
0476         MC_EMEM_ARB_TIMING_W2R,
0477         MC_EMEM_ARB_TIMING_RFCPB,
0478         MC_EMEM_ARB_DA_TURNS,
0479         MC_EMEM_ARB_DA_COVERS,
0480         MC_EMEM_ARB_MISC0,
0481         MC_EMEM_ARB_MISC1,
0482         MC_EMEM_ARB_MISC2,
0483         MC_EMEM_ARB_RING1_THROTTLE,
0484         MC_EMEM_ARB_DHYST_CTRL,
0485         MC_EMEM_ARB_DHYST_TIMEOUT_UTIL_0,
0486         MC_EMEM_ARB_DHYST_TIMEOUT_UTIL_1,
0487         MC_EMEM_ARB_DHYST_TIMEOUT_UTIL_2,
0488         MC_EMEM_ARB_DHYST_TIMEOUT_UTIL_3,
0489         MC_EMEM_ARB_DHYST_TIMEOUT_UTIL_4,
0490         MC_EMEM_ARB_DHYST_TIMEOUT_UTIL_5,
0491         MC_EMEM_ARB_DHYST_TIMEOUT_UTIL_6,
0492         MC_EMEM_ARB_DHYST_TIMEOUT_UTIL_7,
0493     },
0494     .la_scale = {
0495         MC_MLL_MPCORER_PTSA_RATE,
0496         MC_FTOP_PTSA_RATE,
0497         MC_PTSA_GRANT_DECREMENT,
0498         MC_LATENCY_ALLOWANCE_XUSB_0,
0499         MC_LATENCY_ALLOWANCE_XUSB_1,
0500         MC_LATENCY_ALLOWANCE_TSEC_0,
0501         MC_LATENCY_ALLOWANCE_SDMMCA_0,
0502         MC_LATENCY_ALLOWANCE_SDMMCAA_0,
0503         MC_LATENCY_ALLOWANCE_SDMMC_0,
0504         MC_LATENCY_ALLOWANCE_SDMMCAB_0,
0505         MC_LATENCY_ALLOWANCE_PPCS_0,
0506         MC_LATENCY_ALLOWANCE_PPCS_1,
0507         MC_LATENCY_ALLOWANCE_MPCORE_0,
0508         MC_LATENCY_ALLOWANCE_HC_0,
0509         MC_LATENCY_ALLOWANCE_HC_1,
0510         MC_LATENCY_ALLOWANCE_AVPC_0,
0511         MC_LATENCY_ALLOWANCE_GPU_0,
0512         MC_LATENCY_ALLOWANCE_GPU2_0,
0513         MC_LATENCY_ALLOWANCE_NVENC_0,
0514         MC_LATENCY_ALLOWANCE_NVDEC_0,
0515         MC_LATENCY_ALLOWANCE_VIC_0,
0516         MC_LATENCY_ALLOWANCE_VI2_0,
0517         MC_LATENCY_ALLOWANCE_ISP2_0,
0518         MC_LATENCY_ALLOWANCE_ISP2_1,
0519     },
0520     .burst_per_channel = {
0521         { .bank = 0, .offset = EMC_MRW10, },
0522         { .bank = 1, .offset = EMC_MRW10, },
0523         { .bank = 0, .offset = EMC_MRW11, },
0524         { .bank = 1, .offset = EMC_MRW11, },
0525         { .bank = 0, .offset = EMC_MRW12, },
0526         { .bank = 1, .offset = EMC_MRW12, },
0527         { .bank = 0, .offset = EMC_MRW13, },
0528         { .bank = 1, .offset = EMC_MRW13, },
0529     },
0530     .trim_per_channel = {
0531         { .bank = 0, .offset = EMC_CMD_BRLSHFT_0, },
0532         { .bank = 1, .offset = EMC_CMD_BRLSHFT_1, },
0533         { .bank = 0, .offset = EMC_DATA_BRLSHFT_0, },
0534         { .bank = 1, .offset = EMC_DATA_BRLSHFT_0, },
0535         { .bank = 0, .offset = EMC_DATA_BRLSHFT_1, },
0536         { .bank = 1, .offset = EMC_DATA_BRLSHFT_1, },
0537         { .bank = 0, .offset = EMC_QUSE_BRLSHFT_0, },
0538         { .bank = 1, .offset = EMC_QUSE_BRLSHFT_1, },
0539         { .bank = 0, .offset = EMC_QUSE_BRLSHFT_2, },
0540         { .bank = 1, .offset = EMC_QUSE_BRLSHFT_3, },
0541     },
0542     .vref_per_channel = {
0543         {
0544             .bank = 0,
0545             .offset = EMC_TRAINING_OPT_DQS_IB_VREF_RANK0,
0546         }, {
0547             .bank = 1,
0548             .offset = EMC_TRAINING_OPT_DQS_IB_VREF_RANK0,
0549         }, {
0550             .bank = 0,
0551             .offset = EMC_TRAINING_OPT_DQS_IB_VREF_RANK1,
0552         }, {
0553             .bank = 1,
0554             .offset = EMC_TRAINING_OPT_DQS_IB_VREF_RANK1,
0555         },
0556     },
0557 };
0558 
0559 static void tegra210_emc_train(struct timer_list *timer)
0560 {
0561     struct tegra210_emc *emc = from_timer(emc, timer, training);
0562     unsigned long flags;
0563 
0564     if (!emc->last)
0565         return;
0566 
0567     spin_lock_irqsave(&emc->lock, flags);
0568 
0569     if (emc->sequence->periodic_compensation)
0570         emc->sequence->periodic_compensation(emc);
0571 
0572     spin_unlock_irqrestore(&emc->lock, flags);
0573 
0574     mod_timer(&emc->training,
0575           jiffies + msecs_to_jiffies(emc->training_interval));
0576 }
0577 
0578 static void tegra210_emc_training_start(struct tegra210_emc *emc)
0579 {
0580     mod_timer(&emc->training,
0581           jiffies + msecs_to_jiffies(emc->training_interval));
0582 }
0583 
0584 static void tegra210_emc_training_stop(struct tegra210_emc *emc)
0585 {
0586     del_timer(&emc->training);
0587 }
0588 
0589 static unsigned int tegra210_emc_get_temperature(struct tegra210_emc *emc)
0590 {
0591     unsigned long flags;
0592     u32 value, max = 0;
0593     unsigned int i;
0594 
0595     spin_lock_irqsave(&emc->lock, flags);
0596 
0597     for (i = 0; i < emc->num_devices; i++) {
0598         value = tegra210_emc_mrr_read(emc, i, 4);
0599 
0600         if (value & BIT(7))
0601             dev_dbg(emc->dev,
0602                 "sensor reading changed for device %u: %08x\n",
0603                 i, value);
0604 
0605         value = FIELD_GET(LPDDR2_MR4_SRR, value);
0606         if (value > max)
0607             max = value;
0608     }
0609 
0610     spin_unlock_irqrestore(&emc->lock, flags);
0611 
0612     return max;
0613 }
0614 
0615 static void tegra210_emc_poll_refresh(struct timer_list *timer)
0616 {
0617     struct tegra210_emc *emc = from_timer(emc, timer, refresh_timer);
0618     unsigned int temperature;
0619 
0620     if (!emc->debugfs.temperature)
0621         temperature = tegra210_emc_get_temperature(emc);
0622     else
0623         temperature = emc->debugfs.temperature;
0624 
0625     if (temperature == emc->temperature)
0626         goto reset;
0627 
0628     switch (temperature) {
0629     case 0 ... 3:
0630         /* temperature is fine, using regular refresh */
0631         dev_dbg(emc->dev, "switching to nominal refresh...\n");
0632         tegra210_emc_set_refresh(emc, TEGRA210_EMC_REFRESH_NOMINAL);
0633         break;
0634 
0635     case 4:
0636         dev_dbg(emc->dev, "switching to 2x refresh...\n");
0637         tegra210_emc_set_refresh(emc, TEGRA210_EMC_REFRESH_2X);
0638         break;
0639 
0640     case 5:
0641         dev_dbg(emc->dev, "switching to 4x refresh...\n");
0642         tegra210_emc_set_refresh(emc, TEGRA210_EMC_REFRESH_4X);
0643         break;
0644 
0645     case 6 ... 7:
0646         dev_dbg(emc->dev, "switching to throttle refresh...\n");
0647         tegra210_emc_set_refresh(emc, TEGRA210_EMC_REFRESH_THROTTLE);
0648         break;
0649 
0650     default:
0651         WARN(1, "invalid DRAM temperature state %u\n", temperature);
0652         return;
0653     }
0654 
0655     emc->temperature = temperature;
0656 
0657 reset:
0658     if (atomic_read(&emc->refresh_poll) > 0) {
0659         unsigned int interval = emc->refresh_poll_interval;
0660         unsigned int timeout = msecs_to_jiffies(interval);
0661 
0662         mod_timer(&emc->refresh_timer, jiffies + timeout);
0663     }
0664 }
0665 
0666 static void tegra210_emc_poll_refresh_stop(struct tegra210_emc *emc)
0667 {
0668     atomic_set(&emc->refresh_poll, 0);
0669     del_timer_sync(&emc->refresh_timer);
0670 }
0671 
0672 static void tegra210_emc_poll_refresh_start(struct tegra210_emc *emc)
0673 {
0674     atomic_set(&emc->refresh_poll, 1);
0675 
0676     mod_timer(&emc->refresh_timer,
0677           jiffies + msecs_to_jiffies(emc->refresh_poll_interval));
0678 }
0679 
0680 static int tegra210_emc_cd_max_state(struct thermal_cooling_device *cd,
0681                      unsigned long *state)
0682 {
0683     *state = 1;
0684 
0685     return 0;
0686 }
0687 
0688 static int tegra210_emc_cd_get_state(struct thermal_cooling_device *cd,
0689                      unsigned long *state)
0690 {
0691     struct tegra210_emc *emc = cd->devdata;
0692 
0693     *state = atomic_read(&emc->refresh_poll);
0694 
0695     return 0;
0696 }
0697 
0698 static int tegra210_emc_cd_set_state(struct thermal_cooling_device *cd,
0699                      unsigned long state)
0700 {
0701     struct tegra210_emc *emc = cd->devdata;
0702 
0703     if (state == atomic_read(&emc->refresh_poll))
0704         return 0;
0705 
0706     if (state)
0707         tegra210_emc_poll_refresh_start(emc);
0708     else
0709         tegra210_emc_poll_refresh_stop(emc);
0710 
0711     return 0;
0712 }
0713 
0714 static const struct thermal_cooling_device_ops tegra210_emc_cd_ops = {
0715     .get_max_state = tegra210_emc_cd_max_state,
0716     .get_cur_state = tegra210_emc_cd_get_state,
0717     .set_cur_state = tegra210_emc_cd_set_state,
0718 };
0719 
0720 static void tegra210_emc_set_clock(struct tegra210_emc *emc, u32 clksrc)
0721 {
0722     emc->sequence->set_clock(emc, clksrc);
0723 
0724     if (emc->next->periodic_training)
0725         tegra210_emc_training_start(emc);
0726     else
0727         tegra210_emc_training_stop(emc);
0728 }
0729 
0730 static void tegra210_change_dll_src(struct tegra210_emc *emc,
0731                     u32 clksrc)
0732 {
0733     u32 dll_setting = emc->next->dll_clk_src;
0734     u32 emc_clk_src;
0735     u32 emc_clk_div;
0736 
0737     emc_clk_src = (clksrc & EMC_CLK_EMC_2X_CLK_SRC_MASK) >>
0738                EMC_CLK_EMC_2X_CLK_SRC_SHIFT;
0739     emc_clk_div = (clksrc & EMC_CLK_EMC_2X_CLK_DIVISOR_MASK) >>
0740                EMC_CLK_EMC_2X_CLK_DIVISOR_SHIFT;
0741 
0742     dll_setting &= ~(DLL_CLK_EMC_DLL_CLK_SRC_MASK |
0743              DLL_CLK_EMC_DLL_CLK_DIVISOR_MASK);
0744     dll_setting |= emc_clk_src << DLL_CLK_EMC_DLL_CLK_SRC_SHIFT;
0745     dll_setting |= emc_clk_div << DLL_CLK_EMC_DLL_CLK_DIVISOR_SHIFT;
0746 
0747     dll_setting &= ~DLL_CLK_EMC_DLL_DDLL_CLK_SEL_MASK;
0748     if (emc_clk_src == EMC_CLK_SOURCE_PLLMB_LJ)
0749         dll_setting |= (PLLM_VCOB <<
0750                 DLL_CLK_EMC_DLL_DDLL_CLK_SEL_SHIFT);
0751     else if (emc_clk_src == EMC_CLK_SOURCE_PLLM_LJ)
0752         dll_setting |= (PLLM_VCOA <<
0753                 DLL_CLK_EMC_DLL_DDLL_CLK_SEL_SHIFT);
0754     else
0755         dll_setting |= (EMC_DLL_SWITCH_OUT <<
0756                 DLL_CLK_EMC_DLL_DDLL_CLK_SEL_SHIFT);
0757 
0758     tegra210_clk_emc_dll_update_setting(dll_setting);
0759 
0760     if (emc->next->clk_out_enb_x_0_clk_enb_emc_dll)
0761         tegra210_clk_emc_dll_enable(true);
0762     else
0763         tegra210_clk_emc_dll_enable(false);
0764 }
0765 
0766 int tegra210_emc_set_refresh(struct tegra210_emc *emc,
0767                  enum tegra210_emc_refresh refresh)
0768 {
0769     struct tegra210_emc_timing *timings;
0770     unsigned long flags;
0771 
0772     if ((emc->dram_type != DRAM_TYPE_LPDDR2 &&
0773          emc->dram_type != DRAM_TYPE_LPDDR4) ||
0774         !emc->last)
0775         return -ENODEV;
0776 
0777     if (refresh > TEGRA210_EMC_REFRESH_THROTTLE)
0778         return -EINVAL;
0779 
0780     if (refresh == emc->refresh)
0781         return 0;
0782 
0783     spin_lock_irqsave(&emc->lock, flags);
0784 
0785     if (refresh == TEGRA210_EMC_REFRESH_THROTTLE && emc->derated)
0786         timings = emc->derated;
0787     else
0788         timings = emc->nominal;
0789 
0790     if (timings != emc->timings) {
0791         unsigned int index = emc->last - emc->timings;
0792         u32 clksrc;
0793 
0794         clksrc = emc->provider.configs[index].value |
0795              EMC_CLK_FORCE_CC_TRIGGER;
0796 
0797         emc->next = &timings[index];
0798         emc->timings = timings;
0799 
0800         tegra210_emc_set_clock(emc, clksrc);
0801     } else {
0802         tegra210_emc_adjust_timing(emc, emc->last);
0803         tegra210_emc_timing_update(emc);
0804 
0805         if (refresh != TEGRA210_EMC_REFRESH_NOMINAL)
0806             emc_writel(emc, EMC_REF_REF_CMD, EMC_REF);
0807     }
0808 
0809     spin_unlock_irqrestore(&emc->lock, flags);
0810 
0811     return 0;
0812 }
0813 
0814 u32 tegra210_emc_mrr_read(struct tegra210_emc *emc, unsigned int chip,
0815               unsigned int address)
0816 {
0817     u32 value, ret = 0;
0818     unsigned int i;
0819 
0820     value = (chip & EMC_MRR_DEV_SEL_MASK) << EMC_MRR_DEV_SEL_SHIFT |
0821         (address & EMC_MRR_MA_MASK) << EMC_MRR_MA_SHIFT;
0822     emc_writel(emc, value, EMC_MRR);
0823 
0824     for (i = 0; i < emc->num_channels; i++)
0825         WARN(tegra210_emc_wait_for_update(emc, i, EMC_EMC_STATUS,
0826                           EMC_EMC_STATUS_MRR_DIVLD, 1),
0827              "Timed out waiting for MRR %u (ch=%u)\n", address, i);
0828 
0829     for (i = 0; i < emc->num_channels; i++) {
0830         value = emc_channel_readl(emc, i, EMC_MRR);
0831         value &= EMC_MRR_DATA_MASK;
0832 
0833         ret = (ret << 16) | value;
0834     }
0835 
0836     return ret;
0837 }
0838 
0839 void tegra210_emc_do_clock_change(struct tegra210_emc *emc, u32 clksrc)
0840 {
0841     int err;
0842 
0843     mc_readl(emc->mc, MC_EMEM_ADR_CFG);
0844     emc_readl(emc, EMC_INTSTATUS);
0845 
0846     tegra210_clk_emc_update_setting(clksrc);
0847 
0848     err = tegra210_emc_wait_for_update(emc, 0, EMC_INTSTATUS,
0849                        EMC_INTSTATUS_CLKCHANGE_COMPLETE,
0850                        true);
0851     if (err)
0852         dev_warn(emc->dev, "clock change completion error: %d\n", err);
0853 }
0854 
0855 struct tegra210_emc_timing *tegra210_emc_find_timing(struct tegra210_emc *emc,
0856                              unsigned long rate)
0857 {
0858     unsigned int i;
0859 
0860     for (i = 0; i < emc->num_timings; i++)
0861         if (emc->timings[i].rate * 1000UL == rate)
0862             return &emc->timings[i];
0863 
0864     return NULL;
0865 }
0866 
0867 int tegra210_emc_wait_for_update(struct tegra210_emc *emc, unsigned int channel,
0868                  unsigned int offset, u32 bit_mask, bool state)
0869 {
0870     unsigned int i;
0871     u32 value;
0872 
0873     for (i = 0; i < EMC_STATUS_UPDATE_TIMEOUT; i++) {
0874         value = emc_channel_readl(emc, channel, offset);
0875         if (!!(value & bit_mask) == state)
0876             return 0;
0877 
0878         udelay(1);
0879     }
0880 
0881     return -ETIMEDOUT;
0882 }
0883 
0884 void tegra210_emc_set_shadow_bypass(struct tegra210_emc *emc, int set)
0885 {
0886     u32 emc_dbg = emc_readl(emc, EMC_DBG);
0887 
0888     if (set)
0889         emc_writel(emc, emc_dbg | EMC_DBG_WRITE_MUX_ACTIVE, EMC_DBG);
0890     else
0891         emc_writel(emc, emc_dbg & ~EMC_DBG_WRITE_MUX_ACTIVE, EMC_DBG);
0892 }
0893 
0894 u32 tegra210_emc_get_dll_state(struct tegra210_emc_timing *next)
0895 {
0896     if (next->emc_emrs & 0x1)
0897         return 0;
0898 
0899     return 1;
0900 }
0901 
0902 void tegra210_emc_timing_update(struct tegra210_emc *emc)
0903 {
0904     unsigned int i;
0905     int err = 0;
0906 
0907     emc_writel(emc, 0x1, EMC_TIMING_CONTROL);
0908 
0909     for (i = 0; i < emc->num_channels; i++) {
0910         err |= tegra210_emc_wait_for_update(emc, i, EMC_EMC_STATUS,
0911                             EMC_EMC_STATUS_TIMING_UPDATE_STALLED,
0912                             false);
0913     }
0914 
0915     if (err)
0916         dev_warn(emc->dev, "timing update error: %d\n", err);
0917 }
0918 
0919 unsigned long tegra210_emc_actual_osc_clocks(u32 in)
0920 {
0921     if (in < 0x40)
0922         return in * 16;
0923     else if (in < 0x80)
0924         return 2048;
0925     else if (in < 0xc0)
0926         return 4096;
0927     else
0928         return 8192;
0929 }
0930 
0931 void tegra210_emc_start_periodic_compensation(struct tegra210_emc *emc)
0932 {
0933     u32 mpc_req = 0x4b;
0934 
0935     emc_writel(emc, mpc_req, EMC_MPC);
0936     mpc_req = emc_readl(emc, EMC_MPC);
0937 }
0938 
0939 u32 tegra210_emc_compensate(struct tegra210_emc_timing *next, u32 offset)
0940 {
0941     u32 temp = 0, rate = next->rate / 1000;
0942     s32 delta[4], delta_taps[4];
0943     s32 new[] = {
0944         TRIM_REG(0, 0, 0, 0),
0945         TRIM_REG(0, 0, 0, 1),
0946         TRIM_REG(0, 0, 1, 2),
0947         TRIM_REG(0, 0, 1, 3),
0948 
0949         TRIM_REG(1, 0, 2, 4),
0950         TRIM_REG(1, 0, 2, 5),
0951         TRIM_REG(1, 0, 3, 6),
0952         TRIM_REG(1, 0, 3, 7),
0953 
0954         TRIM_REG(0, 1, 0, 0),
0955         TRIM_REG(0, 1, 0, 1),
0956         TRIM_REG(0, 1, 1, 2),
0957         TRIM_REG(0, 1, 1, 3),
0958 
0959         TRIM_REG(1, 1, 2, 4),
0960         TRIM_REG(1, 1, 2, 5),
0961         TRIM_REG(1, 1, 3, 6),
0962         TRIM_REG(1, 1, 3, 7)
0963     };
0964     unsigned i;
0965 
0966     switch (offset) {
0967     case EMC_PMACRO_OB_DDLL_LONG_DQ_RANK0_0:
0968     case EMC_PMACRO_OB_DDLL_LONG_DQ_RANK0_1:
0969     case EMC_PMACRO_OB_DDLL_LONG_DQ_RANK0_2:
0970     case EMC_PMACRO_OB_DDLL_LONG_DQ_RANK0_3:
0971     case EMC_DATA_BRLSHFT_0:
0972         delta[0] = 128 * (next->current_dram_clktree[C0D0U0] -
0973                   next->trained_dram_clktree[C0D0U0]);
0974         delta[1] = 128 * (next->current_dram_clktree[C0D0U1] -
0975                   next->trained_dram_clktree[C0D0U1]);
0976         delta[2] = 128 * (next->current_dram_clktree[C1D0U0] -
0977                   next->trained_dram_clktree[C1D0U0]);
0978         delta[3] = 128 * (next->current_dram_clktree[C1D0U1] -
0979                   next->trained_dram_clktree[C1D0U1]);
0980 
0981         delta_taps[0] = (delta[0] * (s32)rate) / 1000000;
0982         delta_taps[1] = (delta[1] * (s32)rate) / 1000000;
0983         delta_taps[2] = (delta[2] * (s32)rate) / 1000000;
0984         delta_taps[3] = (delta[3] * (s32)rate) / 1000000;
0985 
0986         for (i = 0; i < 4; i++) {
0987             if ((delta_taps[i] > next->tree_margin) ||
0988                 (delta_taps[i] < (-1 * next->tree_margin))) {
0989                 new[i * 2] = new[i * 2] + delta_taps[i];
0990                 new[i * 2 + 1] = new[i * 2 + 1] +
0991                             delta_taps[i];
0992             }
0993         }
0994 
0995         if (offset == EMC_DATA_BRLSHFT_0) {
0996             for (i = 0; i < 8; i++)
0997                 new[i] = new[i] / 64;
0998         } else {
0999             for (i = 0; i < 8; i++)
1000                 new[i] = new[i] % 64;
1001         }
1002 
1003         break;
1004 
1005     case EMC_PMACRO_OB_DDLL_LONG_DQ_RANK1_0:
1006     case EMC_PMACRO_OB_DDLL_LONG_DQ_RANK1_1:
1007     case EMC_PMACRO_OB_DDLL_LONG_DQ_RANK1_2:
1008     case EMC_PMACRO_OB_DDLL_LONG_DQ_RANK1_3:
1009     case EMC_DATA_BRLSHFT_1:
1010         delta[0] = 128 * (next->current_dram_clktree[C0D1U0] -
1011                   next->trained_dram_clktree[C0D1U0]);
1012         delta[1] = 128 * (next->current_dram_clktree[C0D1U1] -
1013                   next->trained_dram_clktree[C0D1U1]);
1014         delta[2] = 128 * (next->current_dram_clktree[C1D1U0] -
1015                   next->trained_dram_clktree[C1D1U0]);
1016         delta[3] = 128 * (next->current_dram_clktree[C1D1U1] -
1017                   next->trained_dram_clktree[C1D1U1]);
1018 
1019         delta_taps[0] = (delta[0] * (s32)rate) / 1000000;
1020         delta_taps[1] = (delta[1] * (s32)rate) / 1000000;
1021         delta_taps[2] = (delta[2] * (s32)rate) / 1000000;
1022         delta_taps[3] = (delta[3] * (s32)rate) / 1000000;
1023 
1024         for (i = 0; i < 4; i++) {
1025             if ((delta_taps[i] > next->tree_margin) ||
1026                 (delta_taps[i] < (-1 * next->tree_margin))) {
1027                 new[8 + i * 2] = new[8 + i * 2] +
1028                             delta_taps[i];
1029                 new[8 + i * 2 + 1] = new[8 + i * 2 + 1] +
1030                             delta_taps[i];
1031             }
1032         }
1033 
1034         if (offset == EMC_DATA_BRLSHFT_1) {
1035             for (i = 0; i < 8; i++)
1036                 new[i + 8] = new[i + 8] / 64;
1037         } else {
1038             for (i = 0; i < 8; i++)
1039                 new[i + 8] = new[i + 8] % 64;
1040         }
1041 
1042         break;
1043     }
1044 
1045     switch (offset) {
1046     case EMC_PMACRO_OB_DDLL_LONG_DQ_RANK0_0:
1047         temp = CALC_TEMP(0, 0, 0, 1, 0);
1048         break;
1049 
1050     case EMC_PMACRO_OB_DDLL_LONG_DQ_RANK0_1:
1051         temp = CALC_TEMP(0, 1, 2, 3, 2);
1052         break;
1053 
1054     case EMC_PMACRO_OB_DDLL_LONG_DQ_RANK0_2:
1055         temp = CALC_TEMP(0, 2, 4, 5, 4);
1056         break;
1057 
1058     case EMC_PMACRO_OB_DDLL_LONG_DQ_RANK0_3:
1059         temp = CALC_TEMP(0, 3, 6, 7, 6);
1060         break;
1061 
1062     case EMC_PMACRO_OB_DDLL_LONG_DQ_RANK1_0:
1063         temp = CALC_TEMP(1, 0, 0, 1, 8);
1064         break;
1065 
1066     case EMC_PMACRO_OB_DDLL_LONG_DQ_RANK1_1:
1067         temp = CALC_TEMP(1, 1, 2, 3, 10);
1068         break;
1069 
1070     case EMC_PMACRO_OB_DDLL_LONG_DQ_RANK1_2:
1071         temp = CALC_TEMP(1, 2, 4, 5, 12);
1072         break;
1073 
1074     case EMC_PMACRO_OB_DDLL_LONG_DQ_RANK1_3:
1075         temp = CALC_TEMP(1, 3, 6, 7, 14);
1076         break;
1077 
1078     case EMC_DATA_BRLSHFT_0:
1079         temp = ((new[0] <<
1080              EMC_DATA_BRLSHFT_0_RANK0_BYTE0_DATA_BRLSHFT_SHIFT) &
1081              EMC_DATA_BRLSHFT_0_RANK0_BYTE0_DATA_BRLSHFT_MASK) |
1082                ((new[1] <<
1083              EMC_DATA_BRLSHFT_0_RANK0_BYTE1_DATA_BRLSHFT_SHIFT) &
1084              EMC_DATA_BRLSHFT_0_RANK0_BYTE1_DATA_BRLSHFT_MASK) |
1085                ((new[2] <<
1086              EMC_DATA_BRLSHFT_0_RANK0_BYTE2_DATA_BRLSHFT_SHIFT) &
1087              EMC_DATA_BRLSHFT_0_RANK0_BYTE2_DATA_BRLSHFT_MASK) |
1088                ((new[3] <<
1089              EMC_DATA_BRLSHFT_0_RANK0_BYTE3_DATA_BRLSHFT_SHIFT) &
1090              EMC_DATA_BRLSHFT_0_RANK0_BYTE3_DATA_BRLSHFT_MASK) |
1091                ((new[4] <<
1092              EMC_DATA_BRLSHFT_0_RANK0_BYTE4_DATA_BRLSHFT_SHIFT) &
1093              EMC_DATA_BRLSHFT_0_RANK0_BYTE4_DATA_BRLSHFT_MASK) |
1094                ((new[5] <<
1095              EMC_DATA_BRLSHFT_0_RANK0_BYTE5_DATA_BRLSHFT_SHIFT) &
1096              EMC_DATA_BRLSHFT_0_RANK0_BYTE5_DATA_BRLSHFT_MASK) |
1097                ((new[6] <<
1098              EMC_DATA_BRLSHFT_0_RANK0_BYTE6_DATA_BRLSHFT_SHIFT) &
1099              EMC_DATA_BRLSHFT_0_RANK0_BYTE6_DATA_BRLSHFT_MASK) |
1100                ((new[7] <<
1101              EMC_DATA_BRLSHFT_0_RANK0_BYTE7_DATA_BRLSHFT_SHIFT) &
1102              EMC_DATA_BRLSHFT_0_RANK0_BYTE7_DATA_BRLSHFT_MASK);
1103         break;
1104 
1105     case EMC_DATA_BRLSHFT_1:
1106         temp = ((new[8] <<
1107              EMC_DATA_BRLSHFT_1_RANK1_BYTE0_DATA_BRLSHFT_SHIFT) &
1108              EMC_DATA_BRLSHFT_1_RANK1_BYTE0_DATA_BRLSHFT_MASK) |
1109                ((new[9] <<
1110              EMC_DATA_BRLSHFT_1_RANK1_BYTE1_DATA_BRLSHFT_SHIFT) &
1111              EMC_DATA_BRLSHFT_1_RANK1_BYTE1_DATA_BRLSHFT_MASK) |
1112                ((new[10] <<
1113              EMC_DATA_BRLSHFT_1_RANK1_BYTE2_DATA_BRLSHFT_SHIFT) &
1114              EMC_DATA_BRLSHFT_1_RANK1_BYTE2_DATA_BRLSHFT_MASK) |
1115                ((new[11] <<
1116              EMC_DATA_BRLSHFT_1_RANK1_BYTE3_DATA_BRLSHFT_SHIFT) &
1117              EMC_DATA_BRLSHFT_1_RANK1_BYTE3_DATA_BRLSHFT_MASK) |
1118                ((new[12] <<
1119              EMC_DATA_BRLSHFT_1_RANK1_BYTE4_DATA_BRLSHFT_SHIFT) &
1120              EMC_DATA_BRLSHFT_1_RANK1_BYTE4_DATA_BRLSHFT_MASK) |
1121                ((new[13] <<
1122              EMC_DATA_BRLSHFT_1_RANK1_BYTE5_DATA_BRLSHFT_SHIFT) &
1123              EMC_DATA_BRLSHFT_1_RANK1_BYTE5_DATA_BRLSHFT_MASK) |
1124                ((new[14] <<
1125              EMC_DATA_BRLSHFT_1_RANK1_BYTE6_DATA_BRLSHFT_SHIFT) &
1126              EMC_DATA_BRLSHFT_1_RANK1_BYTE6_DATA_BRLSHFT_MASK) |
1127                ((new[15] <<
1128              EMC_DATA_BRLSHFT_1_RANK1_BYTE7_DATA_BRLSHFT_SHIFT) &
1129              EMC_DATA_BRLSHFT_1_RANK1_BYTE7_DATA_BRLSHFT_MASK);
1130         break;
1131 
1132     default:
1133         break;
1134     }
1135 
1136     return temp;
1137 }
1138 
1139 u32 tegra210_emc_dll_prelock(struct tegra210_emc *emc, u32 clksrc)
1140 {
1141     unsigned int i;
1142     u32 value;
1143 
1144     value = emc_readl(emc, EMC_CFG_DIG_DLL);
1145     value &= ~EMC_CFG_DIG_DLL_CFG_DLL_LOCK_LIMIT_MASK;
1146     value |= (3 << EMC_CFG_DIG_DLL_CFG_DLL_LOCK_LIMIT_SHIFT);
1147     value &= ~EMC_CFG_DIG_DLL_CFG_DLL_EN;
1148     value &= ~EMC_CFG_DIG_DLL_CFG_DLL_MODE_MASK;
1149     value |= (3 << EMC_CFG_DIG_DLL_CFG_DLL_MODE_SHIFT);
1150     value |= EMC_CFG_DIG_DLL_CFG_DLL_STALL_ALL_TRAFFIC;
1151     value &= ~EMC_CFG_DIG_DLL_CFG_DLL_STALL_RW_UNTIL_LOCK;
1152     value &= ~EMC_CFG_DIG_DLL_CFG_DLL_STALL_ALL_UNTIL_LOCK;
1153     emc_writel(emc, value, EMC_CFG_DIG_DLL);
1154     emc_writel(emc, 1, EMC_TIMING_CONTROL);
1155 
1156     for (i = 0; i < emc->num_channels; i++)
1157         tegra210_emc_wait_for_update(emc, i, EMC_EMC_STATUS,
1158                          EMC_EMC_STATUS_TIMING_UPDATE_STALLED,
1159                          0);
1160 
1161     for (i = 0; i < emc->num_channels; i++) {
1162         while (true) {
1163             value = emc_channel_readl(emc, i, EMC_CFG_DIG_DLL);
1164             if ((value & EMC_CFG_DIG_DLL_CFG_DLL_EN) == 0)
1165                 break;
1166         }
1167     }
1168 
1169     value = emc->next->burst_regs[EMC_DLL_CFG_0_INDEX];
1170     emc_writel(emc, value, EMC_DLL_CFG_0);
1171 
1172     value = emc_readl(emc, EMC_DLL_CFG_1);
1173     value &= EMC_DLL_CFG_1_DDLLCAL_CTRL_START_TRIM_MASK;
1174 
1175     if (emc->next->rate >= 400000 && emc->next->rate < 600000)
1176         value |= 150;
1177     else if (emc->next->rate >= 600000 && emc->next->rate < 800000)
1178         value |= 100;
1179     else if (emc->next->rate >= 800000 && emc->next->rate < 1000000)
1180         value |= 70;
1181     else if (emc->next->rate >= 1000000 && emc->next->rate < 1200000)
1182         value |= 30;
1183     else
1184         value |= 20;
1185 
1186     emc_writel(emc, value, EMC_DLL_CFG_1);
1187 
1188     tegra210_change_dll_src(emc, clksrc);
1189 
1190     value = emc_readl(emc, EMC_CFG_DIG_DLL);
1191     value |= EMC_CFG_DIG_DLL_CFG_DLL_EN;
1192     emc_writel(emc, value, EMC_CFG_DIG_DLL);
1193 
1194     tegra210_emc_timing_update(emc);
1195 
1196     for (i = 0; i < emc->num_channels; i++) {
1197         while (true) {
1198             value = emc_channel_readl(emc, 0, EMC_CFG_DIG_DLL);
1199             if (value & EMC_CFG_DIG_DLL_CFG_DLL_EN)
1200                 break;
1201         }
1202     }
1203 
1204     while (true) {
1205         value = emc_readl(emc, EMC_DIG_DLL_STATUS);
1206 
1207         if ((value & EMC_DIG_DLL_STATUS_DLL_PRIV_UPDATED) == 0)
1208             continue;
1209 
1210         if ((value & EMC_DIG_DLL_STATUS_DLL_LOCK) == 0)
1211             continue;
1212 
1213         break;
1214     }
1215 
1216     value = emc_readl(emc, EMC_DIG_DLL_STATUS);
1217 
1218     return value & EMC_DIG_DLL_STATUS_DLL_OUT_MASK;
1219 }
1220 
1221 u32 tegra210_emc_dvfs_power_ramp_up(struct tegra210_emc *emc, u32 clk,
1222                     bool flip_backward)
1223 {
1224     u32 cmd_pad, dq_pad, rfu1, cfg5, common_tx, ramp_up_wait = 0;
1225     const struct tegra210_emc_timing *timing;
1226 
1227     if (flip_backward)
1228         timing = emc->last;
1229     else
1230         timing = emc->next;
1231 
1232     cmd_pad = timing->burst_regs[EMC_PMACRO_CMD_PAD_TX_CTRL_INDEX];
1233     dq_pad = timing->burst_regs[EMC_PMACRO_DATA_PAD_TX_CTRL_INDEX];
1234     rfu1 = timing->burst_regs[EMC_PMACRO_BRICK_CTRL_RFU1_INDEX];
1235     cfg5 = timing->burst_regs[EMC_FBIO_CFG5_INDEX];
1236     common_tx = timing->burst_regs[EMC_PMACRO_COMMON_PAD_TX_CTRL_INDEX];
1237 
1238     cmd_pad |= EMC_PMACRO_CMD_PAD_TX_CTRL_CMD_DQ_TX_DRVFORCEON;
1239 
1240     if (clk < 1000000 / DVFS_FGCG_MID_SPEED_THRESHOLD) {
1241         ccfifo_writel(emc, common_tx & 0xa,
1242                   EMC_PMACRO_COMMON_PAD_TX_CTRL, 0);
1243         ccfifo_writel(emc, common_tx & 0xf,
1244                   EMC_PMACRO_COMMON_PAD_TX_CTRL,
1245                   (100000 / clk) + 1);
1246         ramp_up_wait += 100000;
1247     } else {
1248         ccfifo_writel(emc, common_tx | 0x8,
1249                   EMC_PMACRO_COMMON_PAD_TX_CTRL, 0);
1250     }
1251 
1252     if (clk < 1000000 / DVFS_FGCG_HIGH_SPEED_THRESHOLD) {
1253         if (clk < 1000000 / IOBRICK_DCC_THRESHOLD) {
1254             cmd_pad |=
1255                 EMC_PMACRO_CMD_PAD_TX_CTRL_CMD_DQSP_TX_E_DCC |
1256                 EMC_PMACRO_CMD_PAD_TX_CTRL_CMD_DQSN_TX_E_DCC;
1257             cmd_pad &=
1258                 ~(EMC_PMACRO_CMD_PAD_TX_CTRL_CMD_DQ_TX_E_DCC |
1259                   EMC_PMACRO_CMD_PAD_TX_CTRL_CMD_CMD_TX_E_DCC);
1260             ccfifo_writel(emc, cmd_pad,
1261                       EMC_PMACRO_CMD_PAD_TX_CTRL,
1262                       (100000 / clk) + 1);
1263             ramp_up_wait += 100000;
1264 
1265             dq_pad |=
1266                 EMC_PMACRO_DATA_PAD_TX_CTRL_DATA_DQSP_TX_E_DCC |
1267                 EMC_PMACRO_DATA_PAD_TX_CTRL_DATA_DQSN_TX_E_DCC;
1268             dq_pad &=
1269                    ~(EMC_PMACRO_DATA_PAD_TX_CTRL_DATA_DQ_TX_E_DCC |
1270                  EMC_PMACRO_DATA_PAD_TX_CTRL_DATA_CMD_TX_E_DCC);
1271             ccfifo_writel(emc, dq_pad,
1272                       EMC_PMACRO_DATA_PAD_TX_CTRL, 0);
1273             ccfifo_writel(emc, rfu1 & 0xfe40fe40,
1274                       EMC_PMACRO_BRICK_CTRL_RFU1, 0);
1275         } else {
1276             ccfifo_writel(emc, rfu1 & 0xfe40fe40,
1277                       EMC_PMACRO_BRICK_CTRL_RFU1,
1278                       (100000 / clk) + 1);
1279             ramp_up_wait += 100000;
1280         }
1281 
1282         ccfifo_writel(emc, rfu1 & 0xfeedfeed,
1283                   EMC_PMACRO_BRICK_CTRL_RFU1, (100000 / clk) + 1);
1284         ramp_up_wait += 100000;
1285 
1286         if (clk < 1000000 / IOBRICK_DCC_THRESHOLD) {
1287             cmd_pad |=
1288                 EMC_PMACRO_CMD_PAD_TX_CTRL_CMD_DQSP_TX_E_DCC |
1289                 EMC_PMACRO_CMD_PAD_TX_CTRL_CMD_DQSN_TX_E_DCC |
1290                 EMC_PMACRO_CMD_PAD_TX_CTRL_CMD_DQ_TX_E_DCC |
1291                 EMC_PMACRO_CMD_PAD_TX_CTRL_CMD_CMD_TX_E_DCC;
1292             ccfifo_writel(emc, cmd_pad,
1293                       EMC_PMACRO_CMD_PAD_TX_CTRL,
1294                       (100000 / clk) + 1);
1295             ramp_up_wait += 100000;
1296 
1297             dq_pad |=
1298                 EMC_PMACRO_DATA_PAD_TX_CTRL_DATA_DQSP_TX_E_DCC |
1299                 EMC_PMACRO_DATA_PAD_TX_CTRL_DATA_DQSN_TX_E_DCC |
1300                 EMC_PMACRO_DATA_PAD_TX_CTRL_DATA_DQ_TX_E_DCC |
1301                 EMC_PMACRO_DATA_PAD_TX_CTRL_DATA_CMD_TX_E_DCC;
1302             ccfifo_writel(emc, dq_pad,
1303                       EMC_PMACRO_DATA_PAD_TX_CTRL, 0);
1304             ccfifo_writel(emc, rfu1,
1305                       EMC_PMACRO_BRICK_CTRL_RFU1, 0);
1306         } else {
1307             ccfifo_writel(emc, rfu1,
1308                       EMC_PMACRO_BRICK_CTRL_RFU1,
1309                       (100000 / clk) + 1);
1310             ramp_up_wait += 100000;
1311         }
1312 
1313         ccfifo_writel(emc, cfg5 & ~EMC_FBIO_CFG5_CMD_TX_DIS,
1314                   EMC_FBIO_CFG5, (100000 / clk) + 10);
1315         ramp_up_wait += 100000 + (10 * clk);
1316     } else if (clk < 1000000 / DVFS_FGCG_MID_SPEED_THRESHOLD) {
1317         ccfifo_writel(emc, rfu1 | 0x06000600,
1318                   EMC_PMACRO_BRICK_CTRL_RFU1, (100000 / clk) + 1);
1319         ccfifo_writel(emc, cfg5 & ~EMC_FBIO_CFG5_CMD_TX_DIS,
1320                   EMC_FBIO_CFG5, (100000 / clk) + 10);
1321         ramp_up_wait += 100000 + 10 * clk;
1322     } else {
1323         ccfifo_writel(emc, rfu1 | 0x00000600,
1324                   EMC_PMACRO_BRICK_CTRL_RFU1, 0);
1325         ccfifo_writel(emc, cfg5 & ~EMC_FBIO_CFG5_CMD_TX_DIS,
1326                   EMC_FBIO_CFG5, 12);
1327         ramp_up_wait += 12 * clk;
1328     }
1329 
1330     cmd_pad &= ~EMC_PMACRO_CMD_PAD_TX_CTRL_CMD_DQ_TX_DRVFORCEON;
1331     ccfifo_writel(emc, cmd_pad, EMC_PMACRO_CMD_PAD_TX_CTRL, 5);
1332 
1333     return ramp_up_wait;
1334 }
1335 
1336 u32 tegra210_emc_dvfs_power_ramp_down(struct tegra210_emc *emc, u32 clk,
1337                       bool flip_backward)
1338 {
1339     u32 ramp_down_wait = 0, cmd_pad, dq_pad, rfu1, cfg5, common_tx;
1340     const struct tegra210_emc_timing *entry;
1341     u32 seq_wait;
1342 
1343     if (flip_backward)
1344         entry = emc->next;
1345     else
1346         entry = emc->last;
1347 
1348     cmd_pad = entry->burst_regs[EMC_PMACRO_CMD_PAD_TX_CTRL_INDEX];
1349     dq_pad = entry->burst_regs[EMC_PMACRO_DATA_PAD_TX_CTRL_INDEX];
1350     rfu1 = entry->burst_regs[EMC_PMACRO_BRICK_CTRL_RFU1_INDEX];
1351     cfg5 = entry->burst_regs[EMC_FBIO_CFG5_INDEX];
1352     common_tx = entry->burst_regs[EMC_PMACRO_COMMON_PAD_TX_CTRL_INDEX];
1353 
1354     cmd_pad |= EMC_PMACRO_CMD_PAD_TX_CTRL_CMD_DQ_TX_DRVFORCEON;
1355 
1356     ccfifo_writel(emc, cmd_pad, EMC_PMACRO_CMD_PAD_TX_CTRL, 0);
1357     ccfifo_writel(emc, cfg5 | EMC_FBIO_CFG5_CMD_TX_DIS,
1358               EMC_FBIO_CFG5, 12);
1359     ramp_down_wait = 12 * clk;
1360 
1361     seq_wait = (100000 / clk) + 1;
1362 
1363     if (clk < (1000000 / DVFS_FGCG_HIGH_SPEED_THRESHOLD)) {
1364         if (clk < (1000000 / IOBRICK_DCC_THRESHOLD)) {
1365             cmd_pad &=
1366                 ~(EMC_PMACRO_CMD_PAD_TX_CTRL_CMD_DQ_TX_E_DCC |
1367                   EMC_PMACRO_CMD_PAD_TX_CTRL_CMD_CMD_TX_E_DCC);
1368             cmd_pad |=
1369                 EMC_PMACRO_CMD_PAD_TX_CTRL_CMD_DQSP_TX_E_DCC |
1370                 EMC_PMACRO_CMD_PAD_TX_CTRL_CMD_DQSN_TX_E_DCC;
1371             ccfifo_writel(emc, cmd_pad,
1372                       EMC_PMACRO_CMD_PAD_TX_CTRL, seq_wait);
1373             ramp_down_wait += 100000;
1374 
1375             dq_pad &=
1376                   ~(EMC_PMACRO_DATA_PAD_TX_CTRL_DATA_DQ_TX_E_DCC |
1377                 EMC_PMACRO_DATA_PAD_TX_CTRL_DATA_CMD_TX_E_DCC);
1378             dq_pad |=
1379                 EMC_PMACRO_DATA_PAD_TX_CTRL_DATA_DQSP_TX_E_DCC |
1380                 EMC_PMACRO_DATA_PAD_TX_CTRL_DATA_DQSN_TX_E_DCC;
1381             ccfifo_writel(emc, dq_pad,
1382                       EMC_PMACRO_DATA_PAD_TX_CTRL, 0);
1383             ccfifo_writel(emc, rfu1 & ~0x01120112,
1384                       EMC_PMACRO_BRICK_CTRL_RFU1, 0);
1385         } else {
1386             ccfifo_writel(emc, rfu1 & ~0x01120112,
1387                       EMC_PMACRO_BRICK_CTRL_RFU1, seq_wait);
1388             ramp_down_wait += 100000;
1389         }
1390 
1391         ccfifo_writel(emc, rfu1 & ~0x01bf01bf,
1392                   EMC_PMACRO_BRICK_CTRL_RFU1, seq_wait);
1393         ramp_down_wait += 100000;
1394 
1395         if (clk < (1000000 / IOBRICK_DCC_THRESHOLD)) {
1396             cmd_pad &=
1397                 ~(EMC_PMACRO_CMD_PAD_TX_CTRL_CMD_DQ_TX_E_DCC |
1398                   EMC_PMACRO_CMD_PAD_TX_CTRL_CMD_CMD_TX_E_DCC |
1399                   EMC_PMACRO_CMD_PAD_TX_CTRL_CMD_DQSP_TX_E_DCC |
1400                   EMC_PMACRO_CMD_PAD_TX_CTRL_CMD_DQSN_TX_E_DCC);
1401             ccfifo_writel(emc, cmd_pad,
1402                       EMC_PMACRO_CMD_PAD_TX_CTRL, seq_wait);
1403             ramp_down_wait += 100000;
1404 
1405             dq_pad &=
1406                   ~(EMC_PMACRO_DATA_PAD_TX_CTRL_DATA_DQ_TX_E_DCC |
1407                 EMC_PMACRO_DATA_PAD_TX_CTRL_DATA_CMD_TX_E_DCC |
1408                 EMC_PMACRO_DATA_PAD_TX_CTRL_DATA_DQSP_TX_E_DCC |
1409                 EMC_PMACRO_DATA_PAD_TX_CTRL_DATA_DQSN_TX_E_DCC);
1410             ccfifo_writel(emc, dq_pad,
1411                       EMC_PMACRO_DATA_PAD_TX_CTRL, 0);
1412             ccfifo_writel(emc, rfu1 & ~0x07ff07ff,
1413                       EMC_PMACRO_BRICK_CTRL_RFU1, 0);
1414         } else {
1415             ccfifo_writel(emc, rfu1 & ~0x07ff07ff,
1416                       EMC_PMACRO_BRICK_CTRL_RFU1, seq_wait);
1417             ramp_down_wait += 100000;
1418         }
1419     } else {
1420         ccfifo_writel(emc, rfu1 & ~0xffff07ff,
1421                   EMC_PMACRO_BRICK_CTRL_RFU1, seq_wait + 19);
1422         ramp_down_wait += 100000 + (20 * clk);
1423     }
1424 
1425     if (clk < (1000000 / DVFS_FGCG_MID_SPEED_THRESHOLD)) {
1426         ramp_down_wait += 100000;
1427         ccfifo_writel(emc, common_tx & ~0x5,
1428                   EMC_PMACRO_COMMON_PAD_TX_CTRL, seq_wait);
1429         ramp_down_wait += 100000;
1430         ccfifo_writel(emc, common_tx & ~0xf,
1431                   EMC_PMACRO_COMMON_PAD_TX_CTRL, seq_wait);
1432         ramp_down_wait += 100000;
1433         ccfifo_writel(emc, 0, 0, seq_wait);
1434         ramp_down_wait += 100000;
1435     } else {
1436         ccfifo_writel(emc, common_tx & ~0xf,
1437                   EMC_PMACRO_COMMON_PAD_TX_CTRL, seq_wait);
1438     }
1439 
1440     return ramp_down_wait;
1441 }
1442 
1443 void tegra210_emc_reset_dram_clktree_values(struct tegra210_emc_timing *timing)
1444 {
1445     timing->current_dram_clktree[C0D0U0] =
1446         timing->trained_dram_clktree[C0D0U0];
1447     timing->current_dram_clktree[C0D0U1] =
1448         timing->trained_dram_clktree[C0D0U1];
1449     timing->current_dram_clktree[C1D0U0] =
1450         timing->trained_dram_clktree[C1D0U0];
1451     timing->current_dram_clktree[C1D0U1] =
1452         timing->trained_dram_clktree[C1D0U1];
1453     timing->current_dram_clktree[C1D1U0] =
1454         timing->trained_dram_clktree[C1D1U0];
1455     timing->current_dram_clktree[C1D1U1] =
1456         timing->trained_dram_clktree[C1D1U1];
1457 }
1458 
1459 static void update_dll_control(struct tegra210_emc *emc, u32 value, bool state)
1460 {
1461     unsigned int i;
1462 
1463     emc_writel(emc, value, EMC_CFG_DIG_DLL);
1464     tegra210_emc_timing_update(emc);
1465 
1466     for (i = 0; i < emc->num_channels; i++)
1467         tegra210_emc_wait_for_update(emc, i, EMC_CFG_DIG_DLL,
1468                          EMC_CFG_DIG_DLL_CFG_DLL_EN,
1469                          state);
1470 }
1471 
1472 void tegra210_emc_dll_disable(struct tegra210_emc *emc)
1473 {
1474     u32 value;
1475 
1476     value = emc_readl(emc, EMC_CFG_DIG_DLL);
1477     value &= ~EMC_CFG_DIG_DLL_CFG_DLL_EN;
1478 
1479     update_dll_control(emc, value, false);
1480 }
1481 
1482 void tegra210_emc_dll_enable(struct tegra210_emc *emc)
1483 {
1484     u32 value;
1485 
1486     value = emc_readl(emc, EMC_CFG_DIG_DLL);
1487     value |= EMC_CFG_DIG_DLL_CFG_DLL_EN;
1488 
1489     update_dll_control(emc, value, true);
1490 }
1491 
1492 void tegra210_emc_adjust_timing(struct tegra210_emc *emc,
1493                 struct tegra210_emc_timing *timing)
1494 {
1495     u32 dsr_cntrl = timing->burst_regs[EMC_DYN_SELF_REF_CONTROL_INDEX];
1496     u32 pre_ref = timing->burst_regs[EMC_PRE_REFRESH_REQ_CNT_INDEX];
1497     u32 ref = timing->burst_regs[EMC_REFRESH_INDEX];
1498 
1499     switch (emc->refresh) {
1500     case TEGRA210_EMC_REFRESH_NOMINAL:
1501     case TEGRA210_EMC_REFRESH_THROTTLE:
1502         break;
1503 
1504     case TEGRA210_EMC_REFRESH_2X:
1505         ref = REFRESH_SPEEDUP(ref, 2);
1506         pre_ref = REFRESH_SPEEDUP(pre_ref, 2);
1507         dsr_cntrl = REFRESH_SPEEDUP(dsr_cntrl, 2);
1508         break;
1509 
1510     case TEGRA210_EMC_REFRESH_4X:
1511         ref = REFRESH_SPEEDUP(ref, 4);
1512         pre_ref = REFRESH_SPEEDUP(pre_ref, 4);
1513         dsr_cntrl = REFRESH_SPEEDUP(dsr_cntrl, 4);
1514         break;
1515 
1516     default:
1517         dev_warn(emc->dev, "failed to set refresh: %d\n", emc->refresh);
1518         return;
1519     }
1520 
1521     emc_writel(emc, ref, emc->offsets->burst[EMC_REFRESH_INDEX]);
1522     emc_writel(emc, pre_ref,
1523            emc->offsets->burst[EMC_PRE_REFRESH_REQ_CNT_INDEX]);
1524     emc_writel(emc, dsr_cntrl,
1525            emc->offsets->burst[EMC_DYN_SELF_REF_CONTROL_INDEX]);
1526 }
1527 
1528 static int tegra210_emc_set_rate(struct device *dev,
1529                  const struct tegra210_clk_emc_config *config)
1530 {
1531     struct tegra210_emc *emc = dev_get_drvdata(dev);
1532     struct tegra210_emc_timing *timing = NULL;
1533     unsigned long rate = config->rate;
1534     s64 last_change_delay;
1535     unsigned long flags;
1536     unsigned int i;
1537 
1538     if (rate == emc->last->rate * 1000UL)
1539         return 0;
1540 
1541     for (i = 0; i < emc->num_timings; i++) {
1542         if (emc->timings[i].rate * 1000UL == rate) {
1543             timing = &emc->timings[i];
1544             break;
1545         }
1546     }
1547 
1548     if (!timing)
1549         return -EINVAL;
1550 
1551     if (rate > 204000000 && !timing->trained)
1552         return -EINVAL;
1553 
1554     emc->next = timing;
1555     last_change_delay = ktime_us_delta(ktime_get(), emc->clkchange_time);
1556 
1557     /* XXX use non-busy-looping sleep? */
1558     if ((last_change_delay >= 0) &&
1559         (last_change_delay < emc->clkchange_delay))
1560         udelay(emc->clkchange_delay - (int)last_change_delay);
1561 
1562     spin_lock_irqsave(&emc->lock, flags);
1563     tegra210_emc_set_clock(emc, config->value);
1564     emc->clkchange_time = ktime_get();
1565     emc->last = timing;
1566     spin_unlock_irqrestore(&emc->lock, flags);
1567 
1568     return 0;
1569 }
1570 
1571 /*
1572  * debugfs interface
1573  *
1574  * The memory controller driver exposes some files in debugfs that can be used
1575  * to control the EMC frequency. The top-level directory can be found here:
1576  *
1577  *   /sys/kernel/debug/emc
1578  *
1579  * It contains the following files:
1580  *
1581  *   - available_rates: This file contains a list of valid, space-separated
1582  *     EMC frequencies.
1583  *
1584  *   - min_rate: Writing a value to this file sets the given frequency as the
1585  *       floor of the permitted range. If this is higher than the currently
1586  *       configured EMC frequency, this will cause the frequency to be
1587  *       increased so that it stays within the valid range.
1588  *
1589  *   - max_rate: Similarily to the min_rate file, writing a value to this file
1590  *       sets the given frequency as the ceiling of the permitted range. If
1591  *       the value is lower than the currently configured EMC frequency, this
1592  *       will cause the frequency to be decreased so that it stays within the
1593  *       valid range.
1594  */
1595 
1596 static bool tegra210_emc_validate_rate(struct tegra210_emc *emc,
1597                        unsigned long rate)
1598 {
1599     unsigned int i;
1600 
1601     for (i = 0; i < emc->num_timings; i++)
1602         if (rate == emc->timings[i].rate * 1000UL)
1603             return true;
1604 
1605     return false;
1606 }
1607 
1608 static int tegra210_emc_debug_available_rates_show(struct seq_file *s,
1609                            void *data)
1610 {
1611     struct tegra210_emc *emc = s->private;
1612     const char *prefix = "";
1613     unsigned int i;
1614 
1615     for (i = 0; i < emc->num_timings; i++) {
1616         seq_printf(s, "%s%u", prefix, emc->timings[i].rate * 1000);
1617         prefix = " ";
1618     }
1619 
1620     seq_puts(s, "\n");
1621 
1622     return 0;
1623 }
1624 
1625 static int tegra210_emc_debug_available_rates_open(struct inode *inode,
1626                            struct file *file)
1627 {
1628     return single_open(file, tegra210_emc_debug_available_rates_show,
1629                inode->i_private);
1630 }
1631 
1632 static const struct file_operations tegra210_emc_debug_available_rates_fops = {
1633     .open = tegra210_emc_debug_available_rates_open,
1634     .read = seq_read,
1635     .llseek = seq_lseek,
1636     .release = single_release,
1637 };
1638 
1639 static int tegra210_emc_debug_min_rate_get(void *data, u64 *rate)
1640 {
1641     struct tegra210_emc *emc = data;
1642 
1643     *rate = emc->debugfs.min_rate;
1644 
1645     return 0;
1646 }
1647 
1648 static int tegra210_emc_debug_min_rate_set(void *data, u64 rate)
1649 {
1650     struct tegra210_emc *emc = data;
1651     int err;
1652 
1653     if (!tegra210_emc_validate_rate(emc, rate))
1654         return -EINVAL;
1655 
1656     err = clk_set_min_rate(emc->clk, rate);
1657     if (err < 0)
1658         return err;
1659 
1660     emc->debugfs.min_rate = rate;
1661 
1662     return 0;
1663 }
1664 
1665 DEFINE_DEBUGFS_ATTRIBUTE(tegra210_emc_debug_min_rate_fops,
1666             tegra210_emc_debug_min_rate_get,
1667             tegra210_emc_debug_min_rate_set, "%llu\n");
1668 
1669 static int tegra210_emc_debug_max_rate_get(void *data, u64 *rate)
1670 {
1671     struct tegra210_emc *emc = data;
1672 
1673     *rate = emc->debugfs.max_rate;
1674 
1675     return 0;
1676 }
1677 
1678 static int tegra210_emc_debug_max_rate_set(void *data, u64 rate)
1679 {
1680     struct tegra210_emc *emc = data;
1681     int err;
1682 
1683     if (!tegra210_emc_validate_rate(emc, rate))
1684         return -EINVAL;
1685 
1686     err = clk_set_max_rate(emc->clk, rate);
1687     if (err < 0)
1688         return err;
1689 
1690     emc->debugfs.max_rate = rate;
1691 
1692     return 0;
1693 }
1694 
1695 DEFINE_DEBUGFS_ATTRIBUTE(tegra210_emc_debug_max_rate_fops,
1696             tegra210_emc_debug_max_rate_get,
1697             tegra210_emc_debug_max_rate_set, "%llu\n");
1698 
1699 static int tegra210_emc_debug_temperature_get(void *data, u64 *temperature)
1700 {
1701     struct tegra210_emc *emc = data;
1702     unsigned int value;
1703 
1704     if (!emc->debugfs.temperature)
1705         value = tegra210_emc_get_temperature(emc);
1706     else
1707         value = emc->debugfs.temperature;
1708 
1709     *temperature = value;
1710 
1711     return 0;
1712 }
1713 
1714 static int tegra210_emc_debug_temperature_set(void *data, u64 temperature)
1715 {
1716     struct tegra210_emc *emc = data;
1717 
1718     if (temperature > 7)
1719         return -EINVAL;
1720 
1721     emc->debugfs.temperature = temperature;
1722 
1723     return 0;
1724 }
1725 
1726 DEFINE_DEBUGFS_ATTRIBUTE(tegra210_emc_debug_temperature_fops,
1727             tegra210_emc_debug_temperature_get,
1728             tegra210_emc_debug_temperature_set, "%llu\n");
1729 
1730 static void tegra210_emc_debugfs_init(struct tegra210_emc *emc)
1731 {
1732     struct device *dev = emc->dev;
1733     unsigned int i;
1734     int err;
1735 
1736     emc->debugfs.min_rate = ULONG_MAX;
1737     emc->debugfs.max_rate = 0;
1738 
1739     for (i = 0; i < emc->num_timings; i++) {
1740         if (emc->timings[i].rate * 1000UL < emc->debugfs.min_rate)
1741             emc->debugfs.min_rate = emc->timings[i].rate * 1000UL;
1742 
1743         if (emc->timings[i].rate * 1000UL > emc->debugfs.max_rate)
1744             emc->debugfs.max_rate = emc->timings[i].rate * 1000UL;
1745     }
1746 
1747     if (!emc->num_timings) {
1748         emc->debugfs.min_rate = clk_get_rate(emc->clk);
1749         emc->debugfs.max_rate = emc->debugfs.min_rate;
1750     }
1751 
1752     err = clk_set_rate_range(emc->clk, emc->debugfs.min_rate,
1753                  emc->debugfs.max_rate);
1754     if (err < 0) {
1755         dev_err(dev, "failed to set rate range [%lu-%lu] for %pC\n",
1756             emc->debugfs.min_rate, emc->debugfs.max_rate,
1757             emc->clk);
1758         return;
1759     }
1760 
1761     emc->debugfs.root = debugfs_create_dir("emc", NULL);
1762 
1763     debugfs_create_file("available_rates", 0444, emc->debugfs.root, emc,
1764                 &tegra210_emc_debug_available_rates_fops);
1765     debugfs_create_file("min_rate", 0644, emc->debugfs.root, emc,
1766                 &tegra210_emc_debug_min_rate_fops);
1767     debugfs_create_file("max_rate", 0644, emc->debugfs.root, emc,
1768                 &tegra210_emc_debug_max_rate_fops);
1769     debugfs_create_file("temperature", 0644, emc->debugfs.root, emc,
1770                 &tegra210_emc_debug_temperature_fops);
1771 }
1772 
1773 static void tegra210_emc_detect(struct tegra210_emc *emc)
1774 {
1775     u32 value;
1776 
1777     /* probe the number of connected DRAM devices */
1778     value = mc_readl(emc->mc, MC_EMEM_ADR_CFG);
1779 
1780     if (value & MC_EMEM_ADR_CFG_EMEM_NUMDEV)
1781         emc->num_devices = 2;
1782     else
1783         emc->num_devices = 1;
1784 
1785     /* probe the type of DRAM */
1786     value = emc_readl(emc, EMC_FBIO_CFG5);
1787     emc->dram_type = value & 0x3;
1788 
1789     /* probe the number of channels */
1790     value = emc_readl(emc, EMC_FBIO_CFG7);
1791 
1792     if ((value & EMC_FBIO_CFG7_CH1_ENABLE) &&
1793         (value & EMC_FBIO_CFG7_CH0_ENABLE))
1794         emc->num_channels = 2;
1795     else
1796         emc->num_channels = 1;
1797 }
1798 
1799 static int tegra210_emc_validate_timings(struct tegra210_emc *emc,
1800                      struct tegra210_emc_timing *timings,
1801                      unsigned int num_timings)
1802 {
1803     unsigned int i;
1804 
1805     for (i = 0; i < num_timings; i++) {
1806         u32 min_volt = timings[i].min_volt;
1807         u32 rate = timings[i].rate;
1808 
1809         if (!rate)
1810             return -EINVAL;
1811 
1812         if ((i > 0) && ((rate <= timings[i - 1].rate) ||
1813             (min_volt < timings[i - 1].min_volt)))
1814             return -EINVAL;
1815 
1816         if (timings[i].revision != timings[0].revision)
1817             continue;
1818     }
1819 
1820     return 0;
1821 }
1822 
1823 static int tegra210_emc_probe(struct platform_device *pdev)
1824 {
1825     struct thermal_cooling_device *cd;
1826     unsigned long current_rate;
1827     struct tegra210_emc *emc;
1828     struct device_node *np;
1829     unsigned int i;
1830     int err;
1831 
1832     emc = devm_kzalloc(&pdev->dev, sizeof(*emc), GFP_KERNEL);
1833     if (!emc)
1834         return -ENOMEM;
1835 
1836     emc->clk = devm_clk_get(&pdev->dev, "emc");
1837     if (IS_ERR(emc->clk))
1838         return PTR_ERR(emc->clk);
1839 
1840     platform_set_drvdata(pdev, emc);
1841     spin_lock_init(&emc->lock);
1842     emc->dev = &pdev->dev;
1843 
1844     emc->mc = devm_tegra_memory_controller_get(&pdev->dev);
1845     if (IS_ERR(emc->mc))
1846         return PTR_ERR(emc->mc);
1847 
1848     emc->regs = devm_platform_ioremap_resource(pdev, 0);
1849     if (IS_ERR(emc->regs))
1850         return PTR_ERR(emc->regs);
1851 
1852     for (i = 0; i < 2; i++) {
1853         emc->channel[i] = devm_platform_ioremap_resource(pdev, 1 + i);
1854         if (IS_ERR(emc->channel[i]))
1855             return PTR_ERR(emc->channel[i]);
1856 
1857     }
1858 
1859     tegra210_emc_detect(emc);
1860     np = pdev->dev.of_node;
1861 
1862     /* attach to the nominal and (optional) derated tables */
1863     err = of_reserved_mem_device_init_by_name(emc->dev, np, "nominal");
1864     if (err < 0) {
1865         dev_err(emc->dev, "failed to get nominal EMC table: %d\n", err);
1866         return err;
1867     }
1868 
1869     err = of_reserved_mem_device_init_by_name(emc->dev, np, "derated");
1870     if (err < 0 && err != -ENODEV) {
1871         dev_err(emc->dev, "failed to get derated EMC table: %d\n", err);
1872         goto release;
1873     }
1874 
1875     /* validate the tables */
1876     if (emc->nominal) {
1877         err = tegra210_emc_validate_timings(emc, emc->nominal,
1878                             emc->num_timings);
1879         if (err < 0)
1880             goto release;
1881     }
1882 
1883     if (emc->derated) {
1884         err = tegra210_emc_validate_timings(emc, emc->derated,
1885                             emc->num_timings);
1886         if (err < 0)
1887             goto release;
1888     }
1889 
1890     /* default to the nominal table */
1891     emc->timings = emc->nominal;
1892 
1893     /* pick the current timing based on the current EMC clock rate */
1894     current_rate = clk_get_rate(emc->clk) / 1000;
1895 
1896     for (i = 0; i < emc->num_timings; i++) {
1897         if (emc->timings[i].rate == current_rate) {
1898             emc->last = &emc->timings[i];
1899             break;
1900         }
1901     }
1902 
1903     if (i == emc->num_timings) {
1904         dev_err(emc->dev, "no EMC table entry found for %lu kHz\n",
1905             current_rate);
1906         err = -ENOENT;
1907         goto release;
1908     }
1909 
1910     /* pick a compatible clock change sequence for the EMC table */
1911     for (i = 0; i < ARRAY_SIZE(tegra210_emc_sequences); i++) {
1912         const struct tegra210_emc_sequence *sequence =
1913                 tegra210_emc_sequences[i];
1914 
1915         if (emc->timings[0].revision == sequence->revision) {
1916             emc->sequence = sequence;
1917             break;
1918         }
1919     }
1920 
1921     if (!emc->sequence) {
1922         dev_err(&pdev->dev, "sequence %u not supported\n",
1923             emc->timings[0].revision);
1924         err = -ENOTSUPP;
1925         goto release;
1926     }
1927 
1928     emc->offsets = &tegra210_emc_table_register_offsets;
1929     emc->refresh = TEGRA210_EMC_REFRESH_NOMINAL;
1930 
1931     emc->provider.owner = THIS_MODULE;
1932     emc->provider.dev = &pdev->dev;
1933     emc->provider.set_rate = tegra210_emc_set_rate;
1934 
1935     emc->provider.configs = devm_kcalloc(&pdev->dev, emc->num_timings,
1936                          sizeof(*emc->provider.configs),
1937                          GFP_KERNEL);
1938     if (!emc->provider.configs) {
1939         err = -ENOMEM;
1940         goto release;
1941     }
1942 
1943     emc->provider.num_configs = emc->num_timings;
1944 
1945     for (i = 0; i < emc->provider.num_configs; i++) {
1946         struct tegra210_emc_timing *timing = &emc->timings[i];
1947         struct tegra210_clk_emc_config *config =
1948                 &emc->provider.configs[i];
1949         u32 value;
1950 
1951         config->rate = timing->rate * 1000UL;
1952         config->value = timing->clk_src_emc;
1953 
1954         value = timing->burst_mc_regs[MC_EMEM_ARB_MISC0_INDEX];
1955 
1956         if ((value & MC_EMEM_ARB_MISC0_EMC_SAME_FREQ) == 0)
1957             config->same_freq = false;
1958         else
1959             config->same_freq = true;
1960     }
1961 
1962     err = tegra210_clk_emc_attach(emc->clk, &emc->provider);
1963     if (err < 0) {
1964         dev_err(&pdev->dev, "failed to attach to EMC clock: %d\n", err);
1965         goto release;
1966     }
1967 
1968     emc->clkchange_delay = 100;
1969     emc->training_interval = 100;
1970     dev_set_drvdata(emc->dev, emc);
1971 
1972     timer_setup(&emc->refresh_timer, tegra210_emc_poll_refresh,
1973             TIMER_DEFERRABLE);
1974     atomic_set(&emc->refresh_poll, 0);
1975     emc->refresh_poll_interval = 1000;
1976 
1977     timer_setup(&emc->training, tegra210_emc_train, 0);
1978 
1979     tegra210_emc_debugfs_init(emc);
1980 
1981     cd = devm_thermal_of_cooling_device_register(emc->dev, np, "emc", emc,
1982                              &tegra210_emc_cd_ops);
1983     if (IS_ERR(cd)) {
1984         err = PTR_ERR(cd);
1985         dev_err(emc->dev, "failed to register cooling device: %d\n",
1986             err);
1987         goto detach;
1988     }
1989 
1990     return 0;
1991 
1992 detach:
1993     debugfs_remove_recursive(emc->debugfs.root);
1994     tegra210_clk_emc_detach(emc->clk);
1995 release:
1996     of_reserved_mem_device_release(emc->dev);
1997 
1998     return err;
1999 }
2000 
2001 static int tegra210_emc_remove(struct platform_device *pdev)
2002 {
2003     struct tegra210_emc *emc = platform_get_drvdata(pdev);
2004 
2005     debugfs_remove_recursive(emc->debugfs.root);
2006     tegra210_clk_emc_detach(emc->clk);
2007     of_reserved_mem_device_release(emc->dev);
2008 
2009     return 0;
2010 }
2011 
2012 static int __maybe_unused tegra210_emc_suspend(struct device *dev)
2013 {
2014     struct tegra210_emc *emc = dev_get_drvdata(dev);
2015     int err;
2016 
2017     err = clk_rate_exclusive_get(emc->clk);
2018     if (err < 0) {
2019         dev_err(emc->dev, "failed to acquire clock: %d\n", err);
2020         return err;
2021     }
2022 
2023     emc->resume_rate = clk_get_rate(emc->clk);
2024 
2025     clk_set_rate(emc->clk, 204000000);
2026     tegra210_clk_emc_detach(emc->clk);
2027 
2028     dev_dbg(dev, "suspending at %lu Hz\n", clk_get_rate(emc->clk));
2029 
2030     return 0;
2031 }
2032 
2033 static int __maybe_unused tegra210_emc_resume(struct device *dev)
2034 {
2035     struct tegra210_emc *emc = dev_get_drvdata(dev);
2036     int err;
2037 
2038     err = tegra210_clk_emc_attach(emc->clk, &emc->provider);
2039     if (err < 0) {
2040         dev_err(dev, "failed to attach to EMC clock: %d\n", err);
2041         return err;
2042     }
2043 
2044     clk_set_rate(emc->clk, emc->resume_rate);
2045     clk_rate_exclusive_put(emc->clk);
2046 
2047     dev_dbg(dev, "resuming at %lu Hz\n", clk_get_rate(emc->clk));
2048 
2049     return 0;
2050 }
2051 
2052 static const struct dev_pm_ops tegra210_emc_pm_ops = {
2053     SET_SYSTEM_SLEEP_PM_OPS(tegra210_emc_suspend, tegra210_emc_resume)
2054 };
2055 
2056 static const struct of_device_id tegra210_emc_of_match[] = {
2057     { .compatible = "nvidia,tegra210-emc", },
2058     { },
2059 };
2060 MODULE_DEVICE_TABLE(of, tegra210_emc_of_match);
2061 
2062 static struct platform_driver tegra210_emc_driver = {
2063     .driver = {
2064         .name = "tegra210-emc",
2065         .of_match_table = tegra210_emc_of_match,
2066         .pm = &tegra210_emc_pm_ops,
2067     },
2068     .probe = tegra210_emc_probe,
2069     .remove = tegra210_emc_remove,
2070 };
2071 
2072 module_platform_driver(tegra210_emc_driver);
2073 
2074 MODULE_AUTHOR("Thierry Reding <treding@nvidia.com>");
2075 MODULE_AUTHOR("Joseph Lo <josephl@nvidia.com>");
2076 MODULE_DESCRIPTION("NVIDIA Tegra210 EMC driver");
2077 MODULE_LICENSE("GPL v2");
[ Source navigation ] [ Diff markup ] [ Identifier search ] [ general search ]

This page was automatically generated by the 2.1.0 LXR engine.

The LXR team
Valid CSS 2.1! Valid HTML 4.01!