drivers/devfreq/sun8i-a33-mbus.c

0001 // SPDX-License-Identifier: GPL-2.0-only
0002 //
0003 // Copyright (C) 2020-2021 Samuel Holland <samuel@sholland.org>
0004 //
0005
0006 #include <linux/clk.h>
0007 #include <linux/devfreq.h>
0008 #include <linux/err.h>
0009 #include <linux/io.h>
0010 #include <linux/iopoll.h>
0011 #include <linux/module.h>
0012 #include <linux/of.h>
0013 #include <linux/platform_device.h>
0014 #include <linux/property.h>
0015
0016 #define MBUS_CR             0x0000
0017 #define MBUS_CR_GET_DRAM_TYPE(x)    (((x) >> 16) & 0x7)
0018 #define MBUS_CR_DRAM_TYPE_DDR2      2
0019 #define MBUS_CR_DRAM_TYPE_DDR3      3
0020 #define MBUS_CR_DRAM_TYPE_DDR4      4
0021 #define MBUS_CR_DRAM_TYPE_LPDDR2    6
0022 #define MBUS_CR_DRAM_TYPE_LPDDR3    7
0023
0024 #define MBUS_TMR            0x000c
0025 #define MBUS_TMR_PERIOD(x)      ((x) - 1)
0026
0027 #define MBUS_PMU_CFG            0x009c
0028 #define MBUS_PMU_CFG_PERIOD(x)      (((x) - 1) << 16)
0029 #define MBUS_PMU_CFG_UNIT       (0x3 << 1)
0030 #define MBUS_PMU_CFG_UNIT_B     (0x0 << 1)
0031 #define MBUS_PMU_CFG_UNIT_KB        (0x1 << 1)
0032 #define MBUS_PMU_CFG_UNIT_MB        (0x2 << 1)
0033 #define MBUS_PMU_CFG_ENABLE     (0x1 << 0)
0034
0035 #define MBUS_PMU_BWCR(n)        (0x00a0 + (0x04 * (n)))
0036
0037 #define MBUS_TOTAL_BWCR         MBUS_PMU_BWCR(5)
0038 #define MBUS_TOTAL_BWCR_H616        MBUS_PMU_BWCR(13)
0039
0040 #define MBUS_MDFSCR         0x0100
0041 #define MBUS_MDFSCR_BUFFER_TIMING   (0x1 << 15)
0042 #define MBUS_MDFSCR_PAD_HOLD        (0x1 << 13)
0043 #define MBUS_MDFSCR_BYPASS      (0x1 << 4)
0044 #define MBUS_MDFSCR_MODE        (0x1 << 1)
0045 #define MBUS_MDFSCR_MODE_DFS        (0x0 << 1)
0046 #define MBUS_MDFSCR_MODE_CFS        (0x1 << 1)
0047 #define MBUS_MDFSCR_START       (0x1 << 0)
0048
0049 #define MBUS_MDFSMRMR           0x0108
0050
0051 #define DRAM_PWRCTL         0x0004
0052 #define DRAM_PWRCTL_SELFREF_EN      (0x1 << 0)
0053
0054 #define DRAM_RFSHTMG            0x0090
0055 #define DRAM_RFSHTMG_TREFI(x)       ((x) << 16)
0056 #define DRAM_RFSHTMG_TRFC(x)        ((x) << 0)
0057
0058 #define DRAM_VTFCR          0x00b8
0059 #define DRAM_VTFCR_VTF_ENABLE       (0x3 << 8)
0060
0061 #define DRAM_ODTMAP         0x0120
0062
0063 #define DRAM_DX_MAX         4
0064
0065 #define DRAM_DXnGCR0(n)         (0x0344 + 0x80 * (n))
0066 #define DRAM_DXnGCR0_DXODT      (0x3 << 4)
0067 #define DRAM_DXnGCR0_DXODT_DYNAMIC  (0x0 << 4)
0068 #define DRAM_DXnGCR0_DXODT_ENABLED  (0x1 << 4)
0069 #define DRAM_DXnGCR0_DXODT_DISABLED (0x2 << 4)
0070 #define DRAM_DXnGCR0_DXEN       (0x1 << 0)
0071
0072 struct sun8i_a33_mbus_variant {
0073     u32                 min_dram_divider;
0074     u32                 max_dram_divider;
0075     u32                 odt_freq_mhz;
0076 };
0077
0078 struct sun8i_a33_mbus {
0079     const struct sun8i_a33_mbus_variant *variant;
0080     void __iomem                *reg_dram;
0081     void __iomem                *reg_mbus;
0082     struct clk              *clk_bus;
0083     struct clk              *clk_dram;
0084     struct clk              *clk_mbus;
0085     struct devfreq              *devfreq_dram;
0086     struct devfreq_simple_ondemand_data gov_data;
0087     struct devfreq_dev_profile      profile;
0088     u32                 data_width;
0089     u32                 nominal_bw;
0090     u32                 odtmap;
0091     u32                 tREFI_ns;
0092     u32                 tRFC_ns;
0093     unsigned long               freq_table[];
0094 };
0095
0096 /*
0097  * The unit for this value is (MBUS clock cycles / MBUS_TMR_PERIOD). When
0098  * MBUS_TMR_PERIOD is programmed to match the MBUS clock frequency in MHz, as
0099  * it is during DRAM init and during probe, the resulting unit is microseconds.
0100  */
0101 static int pmu_period = 50000;
0102 module_param(pmu_period, int, 0644);
0103 MODULE_PARM_DESC(pmu_period, "Bandwidth measurement period (microseconds)");
0104
0105 static u32 sun8i_a33_mbus_get_peak_bw(struct sun8i_a33_mbus *priv)
0106 {
0107     /* Returns the peak transfer (in KiB) during any single PMU period. */
0108     return readl_relaxed(priv->reg_mbus + MBUS_TOTAL_BWCR);
0109 }
0110
0111 static void sun8i_a33_mbus_restart_pmu_counters(struct sun8i_a33_mbus *priv)
0112 {
0113     u32 pmu_cfg = MBUS_PMU_CFG_PERIOD(pmu_period) | MBUS_PMU_CFG_UNIT_KB;
0114
0115     /* All PMU counters are cleared on a disable->enable transition. */
0116     writel_relaxed(pmu_cfg,
0117                priv->reg_mbus + MBUS_PMU_CFG);
0118     writel_relaxed(pmu_cfg | MBUS_PMU_CFG_ENABLE,
0119                priv->reg_mbus + MBUS_PMU_CFG);
0120
0121 }
0122
0123 static void sun8i_a33_mbus_update_nominal_bw(struct sun8i_a33_mbus *priv,
0124                          u32 ddr_freq_mhz)
0125 {
0126     /*
0127      * Nominal bandwidth (KiB per PMU period):
0128      *
0129      *   DDR transfers   microseconds     KiB
0130      *   ------------- * ------------ * --------
0131      *    microsecond     PMU period    transfer
0132      */
0133     priv->nominal_bw = ddr_freq_mhz * pmu_period * priv->data_width / 1024;
0134 }
0135
0136 static int sun8i_a33_mbus_set_dram_freq(struct sun8i_a33_mbus *priv,
0137                     unsigned long freq)
0138 {
0139     u32  ddr_freq_mhz = freq / USEC_PER_SEC; /* DDR */
0140     u32 dram_freq_mhz =    ddr_freq_mhz / 2; /* SDR */
0141     u32 mctl_freq_mhz =   dram_freq_mhz / 2; /* HDR */
0142     u32 dxodt, mdfscr, pwrctl, vtfcr;
0143     u32 i, tREFI_32ck, tRFC_ck;
0144     int ret;
0145
0146     /* The rate change is not effective until the MDFS process runs. */
0147     ret = clk_set_rate(priv->clk_dram, freq);
0148     if (ret)
0149         return ret;
0150
0151     /* Disable automatic self-refesh and VTF before starting MDFS. */
0152     pwrctl = readl_relaxed(priv->reg_dram + DRAM_PWRCTL) &
0153          ~DRAM_PWRCTL_SELFREF_EN;
0154     writel_relaxed(pwrctl, priv->reg_dram + DRAM_PWRCTL);
0155     vtfcr = readl_relaxed(priv->reg_dram + DRAM_VTFCR);
0156     writel_relaxed(vtfcr & ~DRAM_VTFCR_VTF_ENABLE,
0157                priv->reg_dram + DRAM_VTFCR);
0158
0159     /* Set up MDFS and enable double buffering for timing registers. */
0160     mdfscr = MBUS_MDFSCR_MODE_DFS |
0161          MBUS_MDFSCR_BYPASS |
0162          MBUS_MDFSCR_PAD_HOLD |
0163          MBUS_MDFSCR_BUFFER_TIMING;
0164     writel(mdfscr, priv->reg_mbus + MBUS_MDFSCR);
0165
0166     /* Update the buffered copy of RFSHTMG. */
0167     tREFI_32ck = priv->tREFI_ns * mctl_freq_mhz / 1000 / 32;
0168     tRFC_ck = DIV_ROUND_UP(priv->tRFC_ns * mctl_freq_mhz, 1000);
0169     writel(DRAM_RFSHTMG_TREFI(tREFI_32ck) | DRAM_RFSHTMG_TRFC(tRFC_ck),
0170            priv->reg_dram + DRAM_RFSHTMG);
0171
0172     /* Enable ODT if needed, or disable it to save power. */
0173     if (priv->odtmap && dram_freq_mhz > priv->variant->odt_freq_mhz) {
0174         dxodt = DRAM_DXnGCR0_DXODT_DYNAMIC;
0175         writel(priv->odtmap, priv->reg_dram + DRAM_ODTMAP);
0176     } else {
0177         dxodt = DRAM_DXnGCR0_DXODT_DISABLED;
0178         writel(0, priv->reg_dram + DRAM_ODTMAP);
0179     }
0180     for (i = 0; i < DRAM_DX_MAX; ++i) {
0181         void __iomem *reg = priv->reg_dram + DRAM_DXnGCR0(i);
0182
0183         writel((readl(reg) & ~DRAM_DXnGCR0_DXODT) | dxodt, reg);
0184     }
0185
0186     dev_dbg(priv->devfreq_dram->dev.parent,
0187         "Setting DRAM to %u MHz, tREFI=%u, tRFC=%u, ODT=%s\n",
0188         dram_freq_mhz, tREFI_32ck, tRFC_ck,
0189         dxodt == DRAM_DXnGCR0_DXODT_DYNAMIC ? "dynamic" : "disabled");
0190
0191     /* Trigger hardware MDFS. */
0192     writel(mdfscr | MBUS_MDFSCR_START, priv->reg_mbus + MBUS_MDFSCR);
0193     ret = readl_poll_timeout_atomic(priv->reg_mbus + MBUS_MDFSCR, mdfscr,
0194                     !(mdfscr & MBUS_MDFSCR_START), 10, 1000);
0195     if (ret)
0196         return ret;
0197
0198     /* Disable double buffering. */
0199     writel(0, priv->reg_mbus + MBUS_MDFSCR);
0200
0201     /* Restore VTF configuration. */
0202     writel_relaxed(vtfcr, priv->reg_dram + DRAM_VTFCR);
0203
0204     /* Enable automatic self-refresh at the lowest frequency only. */
0205     if (freq == priv->freq_table[0])
0206         pwrctl |= DRAM_PWRCTL_SELFREF_EN;
0207     writel_relaxed(pwrctl, priv->reg_dram + DRAM_PWRCTL);
0208
0209     sun8i_a33_mbus_restart_pmu_counters(priv);
0210     sun8i_a33_mbus_update_nominal_bw(priv, ddr_freq_mhz);
0211
0212     return 0;
0213 }
0214
0215 static int sun8i_a33_mbus_set_dram_target(struct device *dev,
0216                       unsigned long *freq, u32 flags)
0217 {
0218     struct sun8i_a33_mbus *priv = dev_get_drvdata(dev);
0219     struct devfreq *devfreq = priv->devfreq_dram;
0220     struct dev_pm_opp *opp;
0221     int ret;
0222
0223     opp = devfreq_recommended_opp(dev, freq, flags);
0224     if (IS_ERR(opp))
0225         return PTR_ERR(opp);
0226
0227     dev_pm_opp_put(opp);
0228
0229     if (*freq == devfreq->previous_freq)
0230         return 0;
0231
0232     ret = sun8i_a33_mbus_set_dram_freq(priv, *freq);
0233     if (ret) {
0234         dev_warn(dev, "failed to set DRAM frequency: %d\n", ret);
0235         *freq = devfreq->previous_freq;
0236     }
0237
0238     return ret;
0239 }
0240
0241 static int sun8i_a33_mbus_get_dram_status(struct device *dev,
0242                       struct devfreq_dev_status *stat)
0243 {
0244     struct sun8i_a33_mbus *priv = dev_get_drvdata(dev);
0245
0246     stat->busy_time     = sun8i_a33_mbus_get_peak_bw(priv);
0247     stat->total_time    = priv->nominal_bw;
0248     stat->current_frequency = priv->devfreq_dram->previous_freq;
0249
0250     sun8i_a33_mbus_restart_pmu_counters(priv);
0251
0252     dev_dbg(dev, "Using %lu/%lu (%lu%%) at %lu MHz\n",
0253         stat->busy_time, stat->total_time,
0254         DIV_ROUND_CLOSEST(stat->busy_time * 100, stat->total_time),
0255         stat->current_frequency / USEC_PER_SEC);
0256
0257     return 0;
0258 }
0259
0260 static int sun8i_a33_mbus_hw_init(struct device *dev,
0261                   struct sun8i_a33_mbus *priv,
0262                   unsigned long ddr_freq)
0263 {
0264     u32 i, mbus_cr, mbus_freq_mhz;
0265
0266     /* Choose tREFI and tRFC to match the configured DRAM type. */
0267     mbus_cr = readl_relaxed(priv->reg_mbus + MBUS_CR);
0268     switch (MBUS_CR_GET_DRAM_TYPE(mbus_cr)) {
0269     case MBUS_CR_DRAM_TYPE_DDR2:
0270     case MBUS_CR_DRAM_TYPE_DDR3:
0271     case MBUS_CR_DRAM_TYPE_DDR4:
0272         priv->tREFI_ns = 7800;
0273         priv->tRFC_ns = 350;
0274         break;
0275     case MBUS_CR_DRAM_TYPE_LPDDR2:
0276     case MBUS_CR_DRAM_TYPE_LPDDR3:
0277         priv->tREFI_ns = 3900;
0278         priv->tRFC_ns = 210;
0279         break;
0280     default:
0281         return -EINVAL;
0282     }
0283
0284     /* Save ODTMAP so it can be restored when raising the frequency. */
0285     priv->odtmap = readl_relaxed(priv->reg_dram + DRAM_ODTMAP);
0286
0287     /* Compute the DRAM data bus width by counting enabled DATx8 blocks. */
0288     for (i = 0; i < DRAM_DX_MAX; ++i) {
0289         void __iomem *reg = priv->reg_dram + DRAM_DXnGCR0(i);
0290
0291         if (!(readl_relaxed(reg) & DRAM_DXnGCR0_DXEN))
0292             break;
0293     }
0294     priv->data_width = i;
0295
0296     dev_dbg(dev, "Detected %u-bit %sDDRx with%s ODT\n",
0297         priv->data_width * 8,
0298         MBUS_CR_GET_DRAM_TYPE(mbus_cr) > 4 ? "LP" : "",
0299         priv->odtmap ? "" : "out");
0300
0301     /* Program MBUS_TMR such that the PMU period unit is microseconds. */
0302     mbus_freq_mhz = clk_get_rate(priv->clk_mbus) / USEC_PER_SEC;
0303     writel_relaxed(MBUS_TMR_PERIOD(mbus_freq_mhz),
0304                priv->reg_mbus + MBUS_TMR);
0305
0306     /* "Master Ready Mask Register" bits must be set or MDFS will block. */
0307     writel_relaxed(0xffffffff, priv->reg_mbus + MBUS_MDFSMRMR);
0308
0309     sun8i_a33_mbus_restart_pmu_counters(priv);
0310     sun8i_a33_mbus_update_nominal_bw(priv, ddr_freq / USEC_PER_SEC);
0311
0312     return 0;
0313 }
0314
0315 static int __maybe_unused sun8i_a33_mbus_suspend(struct device *dev)
0316 {
0317     struct sun8i_a33_mbus *priv = dev_get_drvdata(dev);
0318
0319     clk_disable_unprepare(priv->clk_bus);
0320
0321     return 0;
0322 }
0323
0324 static int __maybe_unused sun8i_a33_mbus_resume(struct device *dev)
0325 {
0326     struct sun8i_a33_mbus *priv = dev_get_drvdata(dev);
0327
0328     return clk_prepare_enable(priv->clk_bus);
0329 }
0330
0331 static int sun8i_a33_mbus_probe(struct platform_device *pdev)
0332 {
0333     const struct sun8i_a33_mbus_variant *variant;
0334     struct device *dev = &pdev->dev;
0335     struct sun8i_a33_mbus *priv;
0336     unsigned long base_freq;
0337     unsigned int max_state;
0338     const char *err;
0339     int i, ret;
0340
0341     variant = device_get_match_data(dev);
0342     if (!variant)
0343         return -EINVAL;
0344
0345     max_state = variant->max_dram_divider - variant->min_dram_divider + 1;
0346
0347     priv = devm_kzalloc(dev, struct_size(priv, freq_table, max_state), GFP_KERNEL);
0348     if (!priv)
0349         return -ENOMEM;
0350
0351     platform_set_drvdata(pdev, priv);
0352
0353     priv->variant = variant;
0354
0355     priv->reg_dram = devm_platform_ioremap_resource_byname(pdev, "dram");
0356     if (IS_ERR(priv->reg_dram))
0357         return PTR_ERR(priv->reg_dram);
0358
0359     priv->reg_mbus = devm_platform_ioremap_resource_byname(pdev, "mbus");
0360     if (IS_ERR(priv->reg_mbus))
0361         return PTR_ERR(priv->reg_mbus);
0362
0363     priv->clk_bus = devm_clk_get(dev, "bus");
0364     if (IS_ERR(priv->clk_bus))
0365         return dev_err_probe(dev, PTR_ERR(priv->clk_bus),
0366                      "failed to get bus clock\n");
0367
0368     priv->clk_dram = devm_clk_get(dev, "dram");
0369     if (IS_ERR(priv->clk_dram))
0370         return dev_err_probe(dev, PTR_ERR(priv->clk_dram),
0371                      "failed to get dram clock\n");
0372
0373     priv->clk_mbus = devm_clk_get(dev, "mbus");
0374     if (IS_ERR(priv->clk_mbus))
0375         return dev_err_probe(dev, PTR_ERR(priv->clk_mbus),
0376                      "failed to get mbus clock\n");
0377
0378     ret = clk_prepare_enable(priv->clk_bus);
0379     if (ret)
0380         return dev_err_probe(dev, ret,
0381                      "failed to enable bus clock\n");
0382
0383     /* Lock the DRAM clock rate to keep priv->nominal_bw in sync. */
0384     ret = clk_rate_exclusive_get(priv->clk_dram);
0385     if (ret) {
0386         err = "failed to lock dram clock rate\n";
0387         goto err_disable_bus;
0388     }
0389
0390     /* Lock the MBUS clock rate to keep MBUS_TMR_PERIOD in sync. */
0391     ret = clk_rate_exclusive_get(priv->clk_mbus);
0392     if (ret) {
0393         err = "failed to lock mbus clock rate\n";
0394         goto err_unlock_dram;
0395     }
0396
0397     priv->gov_data.upthreshold  = 10;
0398     priv->gov_data.downdifferential =  5;
0399
0400     priv->profile.initial_freq  = clk_get_rate(priv->clk_dram);
0401     priv->profile.polling_ms    = 1000;
0402     priv->profile.target        = sun8i_a33_mbus_set_dram_target;
0403     priv->profile.get_dev_status    = sun8i_a33_mbus_get_dram_status;
0404     priv->profile.freq_table    = priv->freq_table;
0405     priv->profile.max_state     = max_state;
0406
0407     ret = devm_pm_opp_set_clkname(dev, "dram");
0408     if (ret) {
0409         err = "failed to add OPP table\n";
0410         goto err_unlock_mbus;
0411     }
0412
0413     base_freq = clk_get_rate(clk_get_parent(priv->clk_dram));
0414     for (i = 0; i < max_state; ++i) {
0415         unsigned int div = variant->max_dram_divider - i;
0416
0417         priv->freq_table[i] = base_freq / div;
0418
0419         ret = dev_pm_opp_add(dev, priv->freq_table[i], 0);
0420         if (ret) {
0421             err = "failed to add OPPs\n";
0422             goto err_remove_opps;
0423         }
0424     }
0425
0426     ret = sun8i_a33_mbus_hw_init(dev, priv, priv->profile.initial_freq);
0427     if (ret) {
0428         err = "failed to init hardware\n";
0429         goto err_remove_opps;
0430     }
0431
0432     priv->devfreq_dram = devfreq_add_device(dev, &priv->profile,
0433                         DEVFREQ_GOV_SIMPLE_ONDEMAND,
0434                         &priv->gov_data);
0435     if (IS_ERR(priv->devfreq_dram)) {
0436         ret = PTR_ERR(priv->devfreq_dram);
0437         err = "failed to add devfreq device\n";
0438         goto err_remove_opps;
0439     }
0440
0441     /*
0442      * This must be set manually after registering the devfreq device,
0443      * because there is no way to select a dynamic OPP as the suspend OPP.
0444      */
0445     priv->devfreq_dram->suspend_freq = priv->freq_table[0];
0446
0447     return 0;
0448
0449 err_remove_opps:
0450     dev_pm_opp_remove_all_dynamic(dev);
0451 err_unlock_mbus:
0452     clk_rate_exclusive_put(priv->clk_mbus);
0453 err_unlock_dram:
0454     clk_rate_exclusive_put(priv->clk_dram);
0455 err_disable_bus:
0456     clk_disable_unprepare(priv->clk_bus);
0457
0458     return dev_err_probe(dev, ret, err);
0459 }
0460
0461 static int sun8i_a33_mbus_remove(struct platform_device *pdev)
0462 {
0463     struct sun8i_a33_mbus *priv = platform_get_drvdata(pdev);
0464     unsigned long initial_freq = priv->profile.initial_freq;
0465     struct device *dev = &pdev->dev;
0466     int ret;
0467
0468     devfreq_remove_device(priv->devfreq_dram);
0469
0470     ret = sun8i_a33_mbus_set_dram_freq(priv, initial_freq);
0471     if (ret)
0472         dev_warn(dev, "failed to restore DRAM frequency: %d\n", ret);
0473
0474     dev_pm_opp_remove_all_dynamic(dev);
0475     clk_rate_exclusive_put(priv->clk_mbus);
0476     clk_rate_exclusive_put(priv->clk_dram);
0477     clk_disable_unprepare(priv->clk_bus);
0478
0479     return 0;
0480 }
0481
0482 static const struct sun8i_a33_mbus_variant sun50i_a64_mbus = {
0483     .min_dram_divider   = 1,
0484     .max_dram_divider   = 4,
0485     .odt_freq_mhz       = 400,
0486 };
0487
0488 static const struct of_device_id sun8i_a33_mbus_of_match[] = {
0489     { .compatible = "allwinner,sun50i-a64-mbus", .data = &sun50i_a64_mbus },
0490     { .compatible = "allwinner,sun50i-h5-mbus", .data = &sun50i_a64_mbus },
0491     { },
0492 };
0493 MODULE_DEVICE_TABLE(of, sun8i_a33_mbus_of_match);
0494
0495 static SIMPLE_DEV_PM_OPS(sun8i_a33_mbus_pm_ops,
0496              sun8i_a33_mbus_suspend, sun8i_a33_mbus_resume);
0497
0498 static struct platform_driver sun8i_a33_mbus_driver = {
0499     .probe  = sun8i_a33_mbus_probe,
0500     .remove = sun8i_a33_mbus_remove,
0501     .driver = {
0502         .name       = "sun8i-a33-mbus",
0503         .of_match_table = sun8i_a33_mbus_of_match,
0504         .pm     = pm_ptr(&sun8i_a33_mbus_pm_ops),
0505     },
0506 };
0507 module_platform_driver(sun8i_a33_mbus_driver);
0508
0509 MODULE_AUTHOR("Samuel Holland <samuel@sholland.org>");
0510 MODULE_DESCRIPTION("Allwinner sun8i/sun50i MBUS DEVFREQ Driver");
0511 MODULE_LICENSE("GPL v2");