drivers/devfreq/imx8m-ddrc.c

0001 // SPDX-License-Identifier: GPL-2.0
0002 /*
0003  * Copyright 2019 NXP
0004  */
0005
0006 #include <linux/module.h>
0007 #include <linux/device.h>
0008 #include <linux/of_device.h>
0009 #include <linux/platform_device.h>
0010 #include <linux/devfreq.h>
0011 #include <linux/pm_opp.h>
0012 #include <linux/clk.h>
0013 #include <linux/clk-provider.h>
0014 #include <linux/arm-smccc.h>
0015
0016 #define IMX_SIP_DDR_DVFS            0xc2000004
0017
0018 /* Query available frequencies. */
0019 #define IMX_SIP_DDR_DVFS_GET_FREQ_COUNT     0x10
0020 #define IMX_SIP_DDR_DVFS_GET_FREQ_INFO      0x11
0021
0022 /*
0023  * This should be in a 1:1 mapping with devicetree OPPs but
0024  * firmware provides additional info.
0025  */
0026 struct imx8m_ddrc_freq {
0027     unsigned long rate;
0028     unsigned long smcarg;
0029     int dram_core_parent_index;
0030     int dram_alt_parent_index;
0031     int dram_apb_parent_index;
0032 };
0033
0034 /* Hardware limitation */
0035 #define IMX8M_DDRC_MAX_FREQ_COUNT 4
0036
0037 /*
0038  * i.MX8M DRAM Controller clocks have the following structure (abridged):
0039  *
0040  * +----------+       |\            +------+
0041  * | dram_pll |-------|M| dram_core |      |
0042  * +----------+       |U|---------->| D    |
0043  *                 /--|X|           |  D   |
0044  *   dram_alt_root |  |/            |   R  |
0045  *                 |                |    C |
0046  *            +---------+           |      |
0047  *            |FIX DIV/4|           |      |
0048  *            +---------+           |      |
0049  *  composite:     |                |      |
0050  * +----------+    |                |      |
0051  * | dram_alt |----/                |      |
0052  * +----------+                     |      |
0053  * | dram_apb |-------------------->|      |
0054  * +----------+                     +------+
0055  *
0056  * The dram_pll is used for higher rates and dram_alt is used for lower rates.
0057  *
0058  * Frequency switching is implemented in TF-A (via SMC call) and can change the
0059  * configuration of the clocks, including mux parents. The dram_alt and
0060  * dram_apb clocks are "imx composite" and their parent can change too.
0061  *
0062  * We need to prepare/enable the new mux parents head of switching and update
0063  * their information afterwards.
0064  */
0065 struct imx8m_ddrc {
0066     struct devfreq_dev_profile profile;
0067     struct devfreq *devfreq;
0068
0069     /* For frequency switching: */
0070     struct clk *dram_core;
0071     struct clk *dram_pll;
0072     struct clk *dram_alt;
0073     struct clk *dram_apb;
0074
0075     int freq_count;
0076     struct imx8m_ddrc_freq freq_table[IMX8M_DDRC_MAX_FREQ_COUNT];
0077 };
0078
0079 static struct imx8m_ddrc_freq *imx8m_ddrc_find_freq(struct imx8m_ddrc *priv,
0080                             unsigned long rate)
0081 {
0082     struct imx8m_ddrc_freq *freq;
0083     int i;
0084
0085     /*
0086      * Firmware reports values in MT/s, so we round-down from Hz
0087      * Rounding is extra generous to ensure a match.
0088      */
0089     rate = DIV_ROUND_CLOSEST(rate, 250000);
0090     for (i = 0; i < priv->freq_count; ++i) {
0091         freq = &priv->freq_table[i];
0092         if (freq->rate == rate ||
0093                 freq->rate + 1 == rate ||
0094                 freq->rate - 1 == rate)
0095             return freq;
0096     }
0097
0098     return NULL;
0099 }
0100
0101 static void imx8m_ddrc_smc_set_freq(int target_freq)
0102 {
0103     struct arm_smccc_res res;
0104     u32 online_cpus = 0;
0105     int cpu;
0106
0107     local_irq_disable();
0108
0109     for_each_online_cpu(cpu)
0110         online_cpus |= (1 << (cpu * 8));
0111
0112     /* change the ddr freqency */
0113     arm_smccc_smc(IMX_SIP_DDR_DVFS, target_freq, online_cpus,
0114             0, 0, 0, 0, 0, &res);
0115
0116     local_irq_enable();
0117 }
0118
0119 static struct clk *clk_get_parent_by_index(struct clk *clk, int index)
0120 {
0121     struct clk_hw *hw;
0122
0123     hw = clk_hw_get_parent_by_index(__clk_get_hw(clk), index);
0124
0125     return hw ? hw->clk : NULL;
0126 }
0127
0128 static int imx8m_ddrc_set_freq(struct device *dev, struct imx8m_ddrc_freq *freq)
0129 {
0130     struct imx8m_ddrc *priv = dev_get_drvdata(dev);
0131     struct clk *new_dram_core_parent;
0132     struct clk *new_dram_alt_parent;
0133     struct clk *new_dram_apb_parent;
0134     int ret;
0135
0136     /*
0137      * Fetch new parents
0138      *
0139      * new_dram_alt_parent and new_dram_apb_parent are optional but
0140      * new_dram_core_parent is not.
0141      */
0142     new_dram_core_parent = clk_get_parent_by_index(
0143             priv->dram_core, freq->dram_core_parent_index - 1);
0144     if (!new_dram_core_parent) {
0145         dev_err(dev, "failed to fetch new dram_core parent\n");
0146         return -EINVAL;
0147     }
0148     if (freq->dram_alt_parent_index) {
0149         new_dram_alt_parent = clk_get_parent_by_index(
0150                 priv->dram_alt,
0151                 freq->dram_alt_parent_index - 1);
0152         if (!new_dram_alt_parent) {
0153             dev_err(dev, "failed to fetch new dram_alt parent\n");
0154             return -EINVAL;
0155         }
0156     } else
0157         new_dram_alt_parent = NULL;
0158
0159     if (freq->dram_apb_parent_index) {
0160         new_dram_apb_parent = clk_get_parent_by_index(
0161                 priv->dram_apb,
0162                 freq->dram_apb_parent_index - 1);
0163         if (!new_dram_apb_parent) {
0164             dev_err(dev, "failed to fetch new dram_apb parent\n");
0165             return -EINVAL;
0166         }
0167     } else
0168         new_dram_apb_parent = NULL;
0169
0170     /* increase reference counts and ensure clks are ON before switch */
0171     ret = clk_prepare_enable(new_dram_core_parent);
0172     if (ret) {
0173         dev_err(dev, "failed to enable new dram_core parent: %d\n",
0174             ret);
0175         goto out;
0176     }
0177     ret = clk_prepare_enable(new_dram_alt_parent);
0178     if (ret) {
0179         dev_err(dev, "failed to enable new dram_alt parent: %d\n",
0180             ret);
0181         goto out_disable_core_parent;
0182     }
0183     ret = clk_prepare_enable(new_dram_apb_parent);
0184     if (ret) {
0185         dev_err(dev, "failed to enable new dram_apb parent: %d\n",
0186             ret);
0187         goto out_disable_alt_parent;
0188     }
0189
0190     imx8m_ddrc_smc_set_freq(freq->smcarg);
0191
0192     /* update parents in clk tree after switch. */
0193     ret = clk_set_parent(priv->dram_core, new_dram_core_parent);
0194     if (ret)
0195         dev_warn(dev, "failed to set dram_core parent: %d\n", ret);
0196     if (new_dram_alt_parent) {
0197         ret = clk_set_parent(priv->dram_alt, new_dram_alt_parent);
0198         if (ret)
0199             dev_warn(dev, "failed to set dram_alt parent: %d\n",
0200                  ret);
0201     }
0202     if (new_dram_apb_parent) {
0203         ret = clk_set_parent(priv->dram_apb, new_dram_apb_parent);
0204         if (ret)
0205             dev_warn(dev, "failed to set dram_apb parent: %d\n",
0206                  ret);
0207     }
0208
0209     /*
0210      * Explicitly refresh dram PLL rate.
0211      *
0212      * Even if it's marked with CLK_GET_RATE_NOCACHE the rate will not be
0213      * automatically refreshed when clk_get_rate is called on children.
0214      */
0215     clk_get_rate(priv->dram_pll);
0216
0217     /*
0218      * clk_set_parent transfer the reference count from old parent.
0219      * now we drop extra reference counts used during the switch
0220      */
0221     clk_disable_unprepare(new_dram_apb_parent);
0222 out_disable_alt_parent:
0223     clk_disable_unprepare(new_dram_alt_parent);
0224 out_disable_core_parent:
0225     clk_disable_unprepare(new_dram_core_parent);
0226 out:
0227     return ret;
0228 }
0229
0230 static int imx8m_ddrc_target(struct device *dev, unsigned long *freq, u32 flags)
0231 {
0232     struct imx8m_ddrc *priv = dev_get_drvdata(dev);
0233     struct imx8m_ddrc_freq *freq_info;
0234     struct dev_pm_opp *new_opp;
0235     unsigned long old_freq, new_freq;
0236     int ret;
0237
0238     new_opp = devfreq_recommended_opp(dev, freq, flags);
0239     if (IS_ERR(new_opp)) {
0240         ret = PTR_ERR(new_opp);
0241         dev_err(dev, "failed to get recommended opp: %d\n", ret);
0242         return ret;
0243     }
0244     dev_pm_opp_put(new_opp);
0245
0246     old_freq = clk_get_rate(priv->dram_core);
0247     if (*freq == old_freq)
0248         return 0;
0249
0250     freq_info = imx8m_ddrc_find_freq(priv, *freq);
0251     if (!freq_info)
0252         return -EINVAL;
0253
0254     /*
0255      * Read back the clk rate to verify switch was correct and so that
0256      * we can report it on all error paths.
0257      */
0258     ret = imx8m_ddrc_set_freq(dev, freq_info);
0259
0260     new_freq = clk_get_rate(priv->dram_core);
0261     if (ret)
0262         dev_err(dev, "ddrc failed freq switch to %lu from %lu: error %d. now at %lu\n",
0263             *freq, old_freq, ret, new_freq);
0264     else if (*freq != new_freq)
0265         dev_err(dev, "ddrc failed freq update to %lu from %lu, now at %lu\n",
0266             *freq, old_freq, new_freq);
0267     else
0268         dev_dbg(dev, "ddrc freq set to %lu (was %lu)\n",
0269             *freq, old_freq);
0270
0271     return ret;
0272 }
0273
0274 static int imx8m_ddrc_get_cur_freq(struct device *dev, unsigned long *freq)
0275 {
0276     struct imx8m_ddrc *priv = dev_get_drvdata(dev);
0277
0278     *freq = clk_get_rate(priv->dram_core);
0279
0280     return 0;
0281 }
0282
0283 static int imx8m_ddrc_init_freq_info(struct device *dev)
0284 {
0285     struct imx8m_ddrc *priv = dev_get_drvdata(dev);
0286     struct arm_smccc_res res;
0287     int index;
0288
0289     /* An error here means DDR DVFS API not supported by firmware */
0290     arm_smccc_smc(IMX_SIP_DDR_DVFS, IMX_SIP_DDR_DVFS_GET_FREQ_COUNT,
0291             0, 0, 0, 0, 0, 0, &res);
0292     priv->freq_count = res.a0;
0293     if (priv->freq_count <= 0 ||
0294             priv->freq_count > IMX8M_DDRC_MAX_FREQ_COUNT)
0295         return -ENODEV;
0296
0297     for (index = 0; index < priv->freq_count; ++index) {
0298         struct imx8m_ddrc_freq *freq = &priv->freq_table[index];
0299
0300         arm_smccc_smc(IMX_SIP_DDR_DVFS, IMX_SIP_DDR_DVFS_GET_FREQ_INFO,
0301                   index, 0, 0, 0, 0, 0, &res);
0302         /* Result should be strictly positive */
0303         if ((long)res.a0 <= 0)
0304             return -ENODEV;
0305
0306         freq->rate = res.a0;
0307         freq->smcarg = index;
0308         freq->dram_core_parent_index = res.a1;
0309         freq->dram_alt_parent_index = res.a2;
0310         freq->dram_apb_parent_index = res.a3;
0311
0312         /* dram_core has 2 options: dram_pll or dram_alt_root */
0313         if (freq->dram_core_parent_index != 1 &&
0314                 freq->dram_core_parent_index != 2)
0315             return -ENODEV;
0316         /* dram_apb and dram_alt have exactly 8 possible parents */
0317         if (freq->dram_alt_parent_index > 8 ||
0318                 freq->dram_apb_parent_index > 8)
0319             return -ENODEV;
0320         /* dram_core from alt requires explicit dram_alt parent */
0321         if (freq->dram_core_parent_index == 2 &&
0322                 freq->dram_alt_parent_index == 0)
0323             return -ENODEV;
0324     }
0325
0326     return 0;
0327 }
0328
0329 static int imx8m_ddrc_check_opps(struct device *dev)
0330 {
0331     struct imx8m_ddrc *priv = dev_get_drvdata(dev);
0332     struct imx8m_ddrc_freq *freq_info;
0333     struct dev_pm_opp *opp;
0334     unsigned long freq;
0335     int i, opp_count;
0336
0337     /* Enumerate DT OPPs and disable those not supported by firmware */
0338     opp_count = dev_pm_opp_get_opp_count(dev);
0339     if (opp_count < 0)
0340         return opp_count;
0341     for (i = 0, freq = 0; i < opp_count; ++i, ++freq) {
0342         opp = dev_pm_opp_find_freq_ceil(dev, &freq);
0343         if (IS_ERR(opp)) {
0344             dev_err(dev, "Failed enumerating OPPs: %ld\n",
0345                 PTR_ERR(opp));
0346             return PTR_ERR(opp);
0347         }
0348         dev_pm_opp_put(opp);
0349
0350         freq_info = imx8m_ddrc_find_freq(priv, freq);
0351         if (!freq_info) {
0352             dev_info(dev, "Disable unsupported OPP %luHz %luMT/s\n",
0353                     freq, DIV_ROUND_CLOSEST(freq, 250000));
0354             dev_pm_opp_disable(dev, freq);
0355         }
0356     }
0357
0358     return 0;
0359 }
0360
0361 static void imx8m_ddrc_exit(struct device *dev)
0362 {
0363     dev_pm_opp_of_remove_table(dev);
0364 }
0365
0366 static int imx8m_ddrc_probe(struct platform_device *pdev)
0367 {
0368     struct device *dev = &pdev->dev;
0369     struct imx8m_ddrc *priv;
0370     const char *gov = DEVFREQ_GOV_USERSPACE;
0371     int ret;
0372
0373     priv = devm_kzalloc(dev, sizeof(*priv), GFP_KERNEL);
0374     if (!priv)
0375         return -ENOMEM;
0376
0377     platform_set_drvdata(pdev, priv);
0378
0379     ret = imx8m_ddrc_init_freq_info(dev);
0380     if (ret) {
0381         dev_err(dev, "failed to init firmware freq info: %d\n", ret);
0382         return ret;
0383     }
0384
0385     priv->dram_core = devm_clk_get(dev, "core");
0386     if (IS_ERR(priv->dram_core)) {
0387         ret = PTR_ERR(priv->dram_core);
0388         dev_err(dev, "failed to fetch core clock: %d\n", ret);
0389         return ret;
0390     }
0391     priv->dram_pll = devm_clk_get(dev, "pll");
0392     if (IS_ERR(priv->dram_pll)) {
0393         ret = PTR_ERR(priv->dram_pll);
0394         dev_err(dev, "failed to fetch pll clock: %d\n", ret);
0395         return ret;
0396     }
0397     priv->dram_alt = devm_clk_get(dev, "alt");
0398     if (IS_ERR(priv->dram_alt)) {
0399         ret = PTR_ERR(priv->dram_alt);
0400         dev_err(dev, "failed to fetch alt clock: %d\n", ret);
0401         return ret;
0402     }
0403     priv->dram_apb = devm_clk_get(dev, "apb");
0404     if (IS_ERR(priv->dram_apb)) {
0405         ret = PTR_ERR(priv->dram_apb);
0406         dev_err(dev, "failed to fetch apb clock: %d\n", ret);
0407         return ret;
0408     }
0409
0410     ret = dev_pm_opp_of_add_table(dev);
0411     if (ret < 0) {
0412         dev_err(dev, "failed to get OPP table\n");
0413         return ret;
0414     }
0415
0416     ret = imx8m_ddrc_check_opps(dev);
0417     if (ret < 0)
0418         goto err;
0419
0420     priv->profile.target = imx8m_ddrc_target;
0421     priv->profile.exit = imx8m_ddrc_exit;
0422     priv->profile.get_cur_freq = imx8m_ddrc_get_cur_freq;
0423     priv->profile.initial_freq = clk_get_rate(priv->dram_core);
0424
0425     priv->devfreq = devm_devfreq_add_device(dev, &priv->profile,
0426                         gov, NULL);
0427     if (IS_ERR(priv->devfreq)) {
0428         ret = PTR_ERR(priv->devfreq);
0429         dev_err(dev, "failed to add devfreq device: %d\n", ret);
0430         goto err;
0431     }
0432
0433     return 0;
0434
0435 err:
0436     dev_pm_opp_of_remove_table(dev);
0437     return ret;
0438 }
0439
0440 static const struct of_device_id imx8m_ddrc_of_match[] = {
0441     { .compatible = "fsl,imx8m-ddrc", },
0442     { /* sentinel */ },
0443 };
0444 MODULE_DEVICE_TABLE(of, imx8m_ddrc_of_match);
0445
0446 static struct platform_driver imx8m_ddrc_platdrv = {
0447     .probe      = imx8m_ddrc_probe,
0448     .driver = {
0449         .name   = "imx8m-ddrc-devfreq",
0450         .of_match_table = imx8m_ddrc_of_match,
0451     },
0452 };
0453 module_platform_driver(imx8m_ddrc_platdrv);
0454
0455 MODULE_DESCRIPTION("i.MX8M DDR Controller frequency driver");
0456 MODULE_AUTHOR("Leonard Crestez <leonard.crestez@nxp.com>");
0457 MODULE_LICENSE("GPL v2");