drivers/memory/ti-emif-sram-pm.S

0001 /* SPDX-License-Identifier: GPL-2.0-only */
0002 /*
0003  * Low level PM code for TI EMIF
0004  *
0005  * Copyright (C) 2016-2017 Texas Instruments Incorporated - http://www.ti.com/
0006  *  Dave Gerlach
0007  */
0008
0009 #include <linux/linkage.h>
0010 #include <asm/assembler.h>
0011 #include <asm/memory.h>
0012
0013 #include "emif.h"
0014 #include "ti-emif-asm-offsets.h"
0015
0016 #define EMIF_POWER_MGMT_WAIT_SELF_REFRESH_8192_CYCLES   0x00a0
0017 #define EMIF_POWER_MGMT_SR_TIMER_MASK           0x00f0
0018 #define EMIF_POWER_MGMT_SELF_REFRESH_MODE       0x0200
0019 #define EMIF_POWER_MGMT_SELF_REFRESH_MODE_MASK      0x0700
0020
0021 #define EMIF_SDCFG_TYPE_DDR2                0x2 << SDRAM_TYPE_SHIFT
0022 #define EMIF_SDCFG_TYPE_DDR3                0x3 << SDRAM_TYPE_SHIFT
0023 #define EMIF_STATUS_READY               0x4
0024
0025 #define AM43XX_EMIF_PHY_CTRL_REG_COUNT                  0x120
0026
0027 #define EMIF_AM437X_REGISTERS               0x1
0028
0029     .arm
0030     .align 3
0031
0032 ENTRY(ti_emif_sram)
0033
0034 /*
0035  * void ti_emif_save_context(void)
0036  *
0037  * Used during suspend to save the context of all required EMIF registers
0038  * to local memory if the EMIF is going to lose context during the sleep
0039  * transition. Operates on the VIRTUAL address of the EMIF.
0040  */
0041 ENTRY(ti_emif_save_context)
0042     stmfd   sp!, {r4 - r11, lr}     @ save registers on stack
0043
0044     adr r4, ti_emif_pm_sram_data
0045     ldr r0, [r4, #EMIF_PM_BASE_ADDR_VIRT_OFFSET]
0046     ldr r2, [r4, #EMIF_PM_REGS_VIRT_OFFSET]
0047
0048     /* Save EMIF configuration */
0049     ldr r1, [r0, #EMIF_SDRAM_CONFIG]
0050     str r1, [r2, #EMIF_SDCFG_VAL_OFFSET]
0051
0052     ldr r1, [r0, #EMIF_SDRAM_REFRESH_CONTROL]
0053     str r1, [r2, #EMIF_REF_CTRL_VAL_OFFSET]
0054
0055     ldr r1, [r0, #EMIF_SDRAM_TIMING_1]
0056     str     r1, [r2, #EMIF_TIMING1_VAL_OFFSET]
0057
0058     ldr r1, [r0, #EMIF_SDRAM_TIMING_2]
0059     str     r1, [r2, #EMIF_TIMING2_VAL_OFFSET]
0060
0061     ldr r1, [r0, #EMIF_SDRAM_TIMING_3]
0062     str     r1, [r2, #EMIF_TIMING3_VAL_OFFSET]
0063
0064     ldr r1, [r0, #EMIF_POWER_MANAGEMENT_CONTROL]
0065     str     r1, [r2, #EMIF_PMCR_VAL_OFFSET]
0066
0067     ldr r1, [r0, #EMIF_POWER_MANAGEMENT_CTRL_SHDW]
0068     str     r1, [r2, #EMIF_PMCR_SHDW_VAL_OFFSET]
0069
0070     ldr r1, [r0, #EMIF_SDRAM_OUTPUT_IMPEDANCE_CALIBRATION_CONFIG]
0071     str     r1, [r2, #EMIF_ZQCFG_VAL_OFFSET]
0072
0073     ldr r1, [r0, #EMIF_DDR_PHY_CTRL_1]
0074     str     r1, [r2, #EMIF_DDR_PHY_CTLR_1_OFFSET]
0075
0076     ldr r1, [r0, #EMIF_COS_CONFIG]
0077     str     r1, [r2, #EMIF_COS_CONFIG_OFFSET]
0078
0079     ldr r1, [r0, #EMIF_PRIORITY_TO_CLASS_OF_SERVICE_MAPPING]
0080     str     r1, [r2, #EMIF_PRIORITY_TO_COS_MAPPING_OFFSET]
0081
0082     ldr r1, [r0, #EMIF_CONNECTION_ID_TO_CLASS_OF_SERVICE_1_MAPPING]
0083     str     r1, [r2, #EMIF_CONNECT_ID_SERV_1_MAP_OFFSET]
0084
0085     ldr r1, [r0, #EMIF_CONNECTION_ID_TO_CLASS_OF_SERVICE_2_MAPPING]
0086     str     r1, [r2, #EMIF_CONNECT_ID_SERV_2_MAP_OFFSET]
0087
0088     ldr r1, [r0, #EMIF_OCP_CONFIG]
0089     str     r1, [r2, #EMIF_OCP_CONFIG_VAL_OFFSET]
0090
0091     ldr r5, [r4, #EMIF_PM_CONFIG_OFFSET]
0092     cmp r5, #EMIF_SRAM_AM43_REG_LAYOUT
0093     bne emif_skip_save_extra_regs
0094
0095     ldr r1, [r0, #EMIF_READ_WRITE_LEVELING_RAMP_CONTROL]
0096     str     r1, [r2, #EMIF_RD_WR_LEVEL_RAMP_CTRL_OFFSET]
0097
0098     ldr r1, [r0, #EMIF_READ_WRITE_EXECUTION_THRESHOLD]
0099     str     r1, [r2, #EMIF_RD_WR_EXEC_THRESH_OFFSET]
0100
0101     ldr r1, [r0, #EMIF_LPDDR2_NVM_TIMING]
0102     str     r1, [r2, #EMIF_LPDDR2_NVM_TIM_OFFSET]
0103
0104     ldr r1, [r0, #EMIF_LPDDR2_NVM_TIMING_SHDW]
0105     str     r1, [r2, #EMIF_LPDDR2_NVM_TIM_SHDW_OFFSET]
0106
0107     ldr r1, [r0, #EMIF_DLL_CALIB_CTRL]
0108     str     r1, [r2, #EMIF_DLL_CALIB_CTRL_VAL_OFFSET]
0109
0110     ldr r1, [r0, #EMIF_DLL_CALIB_CTRL_SHDW]
0111     str     r1, [r2, #EMIF_DLL_CALIB_CTRL_VAL_SHDW_OFFSET]
0112
0113     /* Loop and save entire block of emif phy regs */
0114     mov r5, #0x0
0115     add r4, r2, #EMIF_EXT_PHY_CTRL_VALS_OFFSET
0116     add r3, r0, #EMIF_EXT_PHY_CTRL_1
0117 ddr_phy_ctrl_save:
0118     ldr r1, [r3, r5]
0119     str r1, [r4, r5]
0120     add r5, r5, #0x4
0121     cmp r5, #AM43XX_EMIF_PHY_CTRL_REG_COUNT
0122     bne ddr_phy_ctrl_save
0123
0124 emif_skip_save_extra_regs:
0125     ldmfd   sp!, {r4 - r11, pc} @ restore regs and return
0126 ENDPROC(ti_emif_save_context)
0127
0128 /*
0129  * void ti_emif_restore_context(void)
0130  *
0131  * Used during resume to restore the context of all required EMIF registers
0132  * from local memory after the EMIF has lost context during a sleep transition.
0133  * Operates on the PHYSICAL address of the EMIF.
0134  */
0135 ENTRY(ti_emif_restore_context)
0136     adr r4, ti_emif_pm_sram_data
0137     ldr r0, [r4, #EMIF_PM_BASE_ADDR_PHYS_OFFSET]
0138     ldr r2, [r4, #EMIF_PM_REGS_PHYS_OFFSET]
0139
0140     /* Config EMIF Timings */
0141     ldr     r1, [r2, #EMIF_DDR_PHY_CTLR_1_OFFSET]
0142     str r1, [r0, #EMIF_DDR_PHY_CTRL_1]
0143     str r1, [r0, #EMIF_DDR_PHY_CTRL_1_SHDW]
0144
0145     ldr     r1, [r2, #EMIF_TIMING1_VAL_OFFSET]
0146     str r1, [r0, #EMIF_SDRAM_TIMING_1]
0147     str r1, [r0, #EMIF_SDRAM_TIMING_1_SHDW]
0148
0149     ldr     r1, [r2, #EMIF_TIMING2_VAL_OFFSET]
0150     str r1, [r0, #EMIF_SDRAM_TIMING_2]
0151     str r1, [r0, #EMIF_SDRAM_TIMING_2_SHDW]
0152
0153     ldr     r1, [r2, #EMIF_TIMING3_VAL_OFFSET]
0154     str r1, [r0, #EMIF_SDRAM_TIMING_3]
0155     str r1, [r0, #EMIF_SDRAM_TIMING_3_SHDW]
0156
0157     ldr     r1, [r2, #EMIF_REF_CTRL_VAL_OFFSET]
0158     str r1, [r0, #EMIF_SDRAM_REFRESH_CONTROL]
0159     str r1, [r0, #EMIF_SDRAM_REFRESH_CTRL_SHDW]
0160
0161     ldr     r1, [r2, #EMIF_PMCR_VAL_OFFSET]
0162     str r1, [r0, #EMIF_POWER_MANAGEMENT_CONTROL]
0163
0164     ldr     r1, [r2, #EMIF_PMCR_SHDW_VAL_OFFSET]
0165     str r1, [r0, #EMIF_POWER_MANAGEMENT_CTRL_SHDW]
0166
0167     ldr     r1, [r2, #EMIF_COS_CONFIG_OFFSET]
0168     str r1, [r0, #EMIF_COS_CONFIG]
0169
0170     ldr     r1, [r2, #EMIF_PRIORITY_TO_COS_MAPPING_OFFSET]
0171     str r1, [r0, #EMIF_PRIORITY_TO_CLASS_OF_SERVICE_MAPPING]
0172
0173     ldr r1, [r2, #EMIF_CONNECT_ID_SERV_1_MAP_OFFSET]
0174     str r1, [r0, #EMIF_CONNECTION_ID_TO_CLASS_OF_SERVICE_1_MAPPING]
0175
0176     ldr     r1, [r2, #EMIF_CONNECT_ID_SERV_2_MAP_OFFSET]
0177     str r1, [r0, #EMIF_CONNECTION_ID_TO_CLASS_OF_SERVICE_2_MAPPING]
0178
0179     ldr     r1, [r2, #EMIF_OCP_CONFIG_VAL_OFFSET]
0180     str r1, [r0, #EMIF_OCP_CONFIG]
0181
0182     ldr r5, [r4, #EMIF_PM_CONFIG_OFFSET]
0183     cmp r5, #EMIF_SRAM_AM43_REG_LAYOUT
0184     bne emif_skip_restore_extra_regs
0185
0186     ldr     r1, [r2, #EMIF_RD_WR_LEVEL_RAMP_CTRL_OFFSET]
0187     str r1, [r0, #EMIF_READ_WRITE_LEVELING_RAMP_CONTROL]
0188
0189     ldr     r1, [r2, #EMIF_RD_WR_EXEC_THRESH_OFFSET]
0190     str r1, [r0, #EMIF_READ_WRITE_EXECUTION_THRESHOLD]
0191
0192     ldr     r1, [r2, #EMIF_LPDDR2_NVM_TIM_OFFSET]
0193     str r1, [r0, #EMIF_LPDDR2_NVM_TIMING]
0194
0195     ldr     r1, [r2, #EMIF_LPDDR2_NVM_TIM_SHDW_OFFSET]
0196     str r1, [r0, #EMIF_LPDDR2_NVM_TIMING_SHDW]
0197
0198     ldr     r1, [r2, #EMIF_DLL_CALIB_CTRL_VAL_OFFSET]
0199     str r1, [r0, #EMIF_DLL_CALIB_CTRL]
0200
0201     ldr     r1, [r2, #EMIF_DLL_CALIB_CTRL_VAL_SHDW_OFFSET]
0202     str r1, [r0, #EMIF_DLL_CALIB_CTRL_SHDW]
0203
0204     ldr     r1, [r2, #EMIF_ZQCFG_VAL_OFFSET]
0205     str r1, [r0, #EMIF_SDRAM_OUTPUT_IMPEDANCE_CALIBRATION_CONFIG]
0206
0207     /* Loop and restore entire block of emif phy regs */
0208     mov r5, #0x0
0209     /* Load ti_emif_regs_amx3 + EMIF_EXT_PHY_CTRL_VALS_OFFSET for address
0210      * to phy register save space
0211      */
0212     add r3, r2, #EMIF_EXT_PHY_CTRL_VALS_OFFSET
0213     add r4, r0, #EMIF_EXT_PHY_CTRL_1
0214 ddr_phy_ctrl_restore:
0215     ldr r1, [r3, r5]
0216     str r1, [r4, r5]
0217     add r5, r5, #0x4
0218     cmp r5, #AM43XX_EMIF_PHY_CTRL_REG_COUNT
0219     bne ddr_phy_ctrl_restore
0220
0221 emif_skip_restore_extra_regs:
0222     /*
0223      * Output impedence calib needed only for DDR3
0224      * but since the initial state of this will be
0225      * disabled for DDR2 no harm in restoring the
0226      * old configuration
0227      */
0228     ldr     r1, [r2, #EMIF_ZQCFG_VAL_OFFSET]
0229     str r1, [r0, #EMIF_SDRAM_OUTPUT_IMPEDANCE_CALIBRATION_CONFIG]
0230
0231     /* Write to sdcfg last for DDR2 only */
0232     ldr r1, [r2, #EMIF_SDCFG_VAL_OFFSET]
0233     and r2, r1, #SDRAM_TYPE_MASK
0234     cmp r2, #EMIF_SDCFG_TYPE_DDR2
0235     streq   r1, [r0, #EMIF_SDRAM_CONFIG]
0236
0237     mov pc, lr
0238 ENDPROC(ti_emif_restore_context)
0239
0240 /*
0241  * void ti_emif_run_hw_leveling(void)
0242  *
0243  * Used during resume to run hardware leveling again and restore the
0244  * configuration of the EMIF PHY, only for DDR3.
0245  */
0246 ENTRY(ti_emif_run_hw_leveling)
0247     adr r4, ti_emif_pm_sram_data
0248     ldr r0, [r4, #EMIF_PM_BASE_ADDR_PHYS_OFFSET]
0249
0250     ldr r3, [r0, #EMIF_READ_WRITE_LEVELING_CONTROL]
0251     orr r3, r3, #RDWRLVLFULL_START
0252     ldr r2, [r0, #EMIF_SDRAM_CONFIG]
0253     and r2, r2, #SDRAM_TYPE_MASK
0254     cmp r2, #EMIF_SDCFG_TYPE_DDR3
0255     bne skip_hwlvl
0256
0257     str r3, [r0, #EMIF_READ_WRITE_LEVELING_CONTROL]
0258
0259     /*
0260      * If EMIF registers are touched during initial stage of HW
0261      * leveling sequence there will be an L3 NOC timeout error issued
0262      * as the EMIF will not respond, which is not fatal, but it is
0263      * avoidable. This small wait loop is enough time for this condition
0264      * to clear, even at worst case of CPU running at max speed of 1Ghz.
0265      */
0266     mov r2, #0x2000
0267 1:
0268     subs    r2, r2, #0x1
0269     bne 1b
0270
0271     /* Bit clears when operation is complete */
0272 2:  ldr     r1, [r0, #EMIF_READ_WRITE_LEVELING_CONTROL]
0273     tst     r1, #RDWRLVLFULL_START
0274     bne     2b
0275
0276 skip_hwlvl:
0277     mov pc, lr
0278 ENDPROC(ti_emif_run_hw_leveling)
0279
0280 /*
0281  * void ti_emif_enter_sr(void)
0282  *
0283  * Programs the EMIF to tell the SDRAM to enter into self-refresh
0284  * mode during a sleep transition. Operates on the VIRTUAL address
0285  * of the EMIF.
0286  */
0287 ENTRY(ti_emif_enter_sr)
0288     stmfd   sp!, {r4 - r11, lr}     @ save registers on stack
0289
0290     adr r4, ti_emif_pm_sram_data
0291     ldr r0, [r4, #EMIF_PM_BASE_ADDR_VIRT_OFFSET]
0292     ldr r2, [r4, #EMIF_PM_REGS_VIRT_OFFSET]
0293
0294     ldr r1, [r0, #EMIF_POWER_MANAGEMENT_CONTROL]
0295     bic r1, r1, #EMIF_POWER_MGMT_SELF_REFRESH_MODE_MASK
0296     orr r1, r1, #EMIF_POWER_MGMT_SELF_REFRESH_MODE
0297     str r1, [r0, #EMIF_POWER_MANAGEMENT_CONTROL]
0298
0299     ldmfd   sp!, {r4 - r11, pc} @ restore regs and return
0300 ENDPROC(ti_emif_enter_sr)
0301
0302 /*
0303  * void ti_emif_exit_sr(void)
0304  *
0305  * Programs the EMIF to tell the SDRAM to exit self-refresh mode
0306  * after a sleep transition. Operates on the PHYSICAL address of
0307  * the EMIF.
0308  */
0309 ENTRY(ti_emif_exit_sr)
0310     adr r4, ti_emif_pm_sram_data
0311     ldr r0, [r4, #EMIF_PM_BASE_ADDR_PHYS_OFFSET]
0312     ldr r2, [r4, #EMIF_PM_REGS_PHYS_OFFSET]
0313
0314     /*
0315      * Toggle EMIF to exit refresh mode:
0316      * if EMIF lost context, PWR_MGT_CTRL is currently 0, writing disable
0317      *   (0x0), wont do diddly squat! so do a toggle from SR(0x2) to disable
0318      *   (0x0) here.
0319      * *If* EMIF did not lose context, nothing broken as we write the same
0320      *   value(0x2) to reg before we write a disable (0x0).
0321      */
0322     ldr r1, [r2, #EMIF_PMCR_VAL_OFFSET]
0323     bic r1, r1, #EMIF_POWER_MGMT_SELF_REFRESH_MODE_MASK
0324     orr r1, r1, #EMIF_POWER_MGMT_SELF_REFRESH_MODE
0325     str r1, [r0, #EMIF_POWER_MANAGEMENT_CONTROL]
0326     bic r1, r1, #EMIF_POWER_MGMT_SELF_REFRESH_MODE_MASK
0327     str r1, [r0, #EMIF_POWER_MANAGEMENT_CONTROL]
0328
0329         /* Wait for EMIF to become ready */
0330 1:  ldr     r1, [r0, #EMIF_STATUS]
0331     tst     r1, #EMIF_STATUS_READY
0332     beq     1b
0333
0334     mov pc, lr
0335 ENDPROC(ti_emif_exit_sr)
0336
0337 /*
0338  * void ti_emif_abort_sr(void)
0339  *
0340  * Disables self-refresh after a failed transition to a low-power
0341  * state so the kernel can jump back to DDR and follow abort path.
0342  * Operates on the VIRTUAL address of the EMIF.
0343  */
0344 ENTRY(ti_emif_abort_sr)
0345     stmfd   sp!, {r4 - r11, lr}     @ save registers on stack
0346
0347     adr r4, ti_emif_pm_sram_data
0348     ldr r0, [r4, #EMIF_PM_BASE_ADDR_VIRT_OFFSET]
0349     ldr r2, [r4, #EMIF_PM_REGS_VIRT_OFFSET]
0350
0351     ldr r1, [r2, #EMIF_PMCR_VAL_OFFSET]
0352     bic r1, r1, #EMIF_POWER_MGMT_SELF_REFRESH_MODE_MASK
0353     str r1, [r0, #EMIF_POWER_MANAGEMENT_CONTROL]
0354
0355     /* Wait for EMIF to become ready */
0356 1:  ldr     r1, [r0, #EMIF_STATUS]
0357     tst     r1, #EMIF_STATUS_READY
0358     beq     1b
0359
0360     ldmfd   sp!, {r4 - r11, pc} @ restore regs and return
0361 ENDPROC(ti_emif_abort_sr)
0362
0363     .align 3
0364 ENTRY(ti_emif_pm_sram_data)
0365     .space EMIF_PM_DATA_SIZE
0366 ENTRY(ti_emif_sram_sz)
0367         .word   . - ti_emif_save_context