bcm/brcmstb/biuctrl.c

0001 // SPDX-License-Identifier: GPL-2.0-only
0002 /*
0003  * Broadcom STB SoCs Bus Unit Interface controls
0004  *
0005  * Copyright (C) 2015, Broadcom Corporation
0006  */
0007
0008 #define pr_fmt(fmt) "brcmstb: " KBUILD_MODNAME ": " fmt
0009
0010 #include <linux/kernel.h>
0011 #include <linux/io.h>
0012 #include <linux/of_address.h>
0013 #include <linux/syscore_ops.h>
0014 #include <linux/soc/brcmstb/brcmstb.h>
0015
0016 #define RACENPREF_MASK          0x3
0017 #define RACPREFINST_SHIFT       0
0018 #define RACENINST_SHIFT         2
0019 #define RACPREFDATA_SHIFT       4
0020 #define RACENDATA_SHIFT         6
0021 #define RAC_CPU_SHIFT           8
0022 #define RACCFG_MASK         0xff
0023 #define DPREF_LINE_2_SHIFT      24
0024 #define DPREF_LINE_2_MASK       0xff
0025
0026 /* Bitmask to enable instruction and data prefetching with a 256-bytes stride */
0027 #define RAC_DATA_INST_EN_MASK       (1 << RACPREFINST_SHIFT | \
0028                      RACENPREF_MASK << RACENINST_SHIFT | \
0029                      1 << RACPREFDATA_SHIFT | \
0030                      RACENPREF_MASK << RACENDATA_SHIFT)
0031
0032 #define  CPU_CREDIT_REG_MCPx_WR_PAIRING_EN_MASK 0x70000000
0033 #define CPU_CREDIT_REG_MCPx_READ_CRED_MASK  0xf
0034 #define CPU_CREDIT_REG_MCPx_WRITE_CRED_MASK 0xf
0035 #define CPU_CREDIT_REG_MCPx_READ_CRED_SHIFT(x)  ((x) * 8)
0036 #define CPU_CREDIT_REG_MCPx_WRITE_CRED_SHIFT(x) (((x) * 8) + 4)
0037
0038 #define CPU_MCP_FLOW_REG_MCPx_RDBUFF_CRED_SHIFT(x)  ((x) * 8)
0039 #define CPU_MCP_FLOW_REG_MCPx_RDBUFF_CRED_MASK      0xff
0040
0041 #define CPU_WRITEBACK_CTRL_REG_WB_THROTTLE_THRESHOLD_MASK   0xf
0042 #define CPU_WRITEBACK_CTRL_REG_WB_THROTTLE_TIMEOUT_MASK     0xf
0043 #define CPU_WRITEBACK_CTRL_REG_WB_THROTTLE_TIMEOUT_SHIFT    4
0044 #define CPU_WRITEBACK_CTRL_REG_WB_THROTTLE_ENABLE       BIT(8)
0045
0046 static void __iomem *cpubiuctrl_base;
0047 static bool mcp_wr_pairing_en;
0048 static const int *cpubiuctrl_regs;
0049
0050 enum cpubiuctrl_regs {
0051     CPU_CREDIT_REG = 0,
0052     CPU_MCP_FLOW_REG,
0053     CPU_WRITEBACK_CTRL_REG,
0054     RAC_CONFIG0_REG,
0055     RAC_CONFIG1_REG,
0056     NUM_CPU_BIUCTRL_REGS,
0057 };
0058
0059 static inline u32 cbc_readl(int reg)
0060 {
0061     int offset = cpubiuctrl_regs[reg];
0062
0063     if (offset == -1 ||
0064         (IS_ENABLED(CONFIG_CACHE_B15_RAC) && reg >= RAC_CONFIG0_REG))
0065         return (u32)-1;
0066
0067     return readl_relaxed(cpubiuctrl_base + offset);
0068 }
0069
0070 static inline void cbc_writel(u32 val, int reg)
0071 {
0072     int offset = cpubiuctrl_regs[reg];
0073
0074     if (offset == -1 ||
0075         (IS_ENABLED(CONFIG_CACHE_B15_RAC) && reg >= RAC_CONFIG0_REG))
0076         return;
0077
0078     writel(val, cpubiuctrl_base + offset);
0079 }
0080
0081 static const int b15_cpubiuctrl_regs[] = {
0082     [CPU_CREDIT_REG] = 0x184,
0083     [CPU_MCP_FLOW_REG] = -1,
0084     [CPU_WRITEBACK_CTRL_REG] = -1,
0085     [RAC_CONFIG0_REG] = -1,
0086     [RAC_CONFIG1_REG] = -1,
0087 };
0088
0089 /* Odd cases, e.g: 7260A0 */
0090 static const int b53_cpubiuctrl_no_wb_regs[] = {
0091     [CPU_CREDIT_REG] = 0x0b0,
0092     [CPU_MCP_FLOW_REG] = 0x0b4,
0093     [CPU_WRITEBACK_CTRL_REG] = -1,
0094     [RAC_CONFIG0_REG] = 0x78,
0095     [RAC_CONFIG1_REG] = 0x7c,
0096 };
0097
0098 static const int b53_cpubiuctrl_regs[] = {
0099     [CPU_CREDIT_REG] = 0x0b0,
0100     [CPU_MCP_FLOW_REG] = 0x0b4,
0101     [CPU_WRITEBACK_CTRL_REG] = 0x22c,
0102     [RAC_CONFIG0_REG] = 0x78,
0103     [RAC_CONFIG1_REG] = 0x7c,
0104 };
0105
0106 static const int a72_cpubiuctrl_regs[] = {
0107     [CPU_CREDIT_REG] = 0x18,
0108     [CPU_MCP_FLOW_REG] = 0x1c,
0109     [CPU_WRITEBACK_CTRL_REG] = 0x20,
0110     [RAC_CONFIG0_REG] = 0x08,
0111     [RAC_CONFIG1_REG] = 0x0c,
0112 };
0113
0114 static int __init mcp_write_pairing_set(void)
0115 {
0116     u32 creds = 0;
0117
0118     if (!cpubiuctrl_base)
0119         return -1;
0120
0121     creds = cbc_readl(CPU_CREDIT_REG);
0122     if (mcp_wr_pairing_en) {
0123         pr_info("MCP: Enabling write pairing\n");
0124         cbc_writel(creds | CPU_CREDIT_REG_MCPx_WR_PAIRING_EN_MASK,
0125                CPU_CREDIT_REG);
0126     } else if (creds & CPU_CREDIT_REG_MCPx_WR_PAIRING_EN_MASK) {
0127         pr_info("MCP: Disabling write pairing\n");
0128         cbc_writel(creds & ~CPU_CREDIT_REG_MCPx_WR_PAIRING_EN_MASK,
0129                CPU_CREDIT_REG);
0130     } else {
0131         pr_info("MCP: Write pairing already disabled\n");
0132     }
0133
0134     return 0;
0135 }
0136
0137 static const u32 a72_b53_mach_compat[] = {
0138     0x7211,
0139     0x72113,
0140     0x72116,
0141     0x7216,
0142     0x72164,
0143     0x72165,
0144     0x7255,
0145     0x7260,
0146     0x7268,
0147     0x7271,
0148     0x7278,
0149 };
0150
0151 /* The read-ahead cache present in the Brahma-B53 CPU is a special piece of
0152  * hardware after the integrated L2 cache of the B53 CPU complex whose purpose
0153  * is to prefetch instruction and/or data with a line size of either 64 bytes
0154  * or 256 bytes. The rationale is that the data-bus of the CPU interface is
0155  * optimized for 256-byte transactions, and enabling the read-ahead cache
0156  * provides a significant performance boost (typically twice the performance
0157  * for a memcpy benchmark application).
0158  *
0159  * The read-ahead cache is transparent for Virtual Address cache maintenance
0160  * operations: IC IVAU, DC IVAC, DC CVAC, DC CVAU and DC CIVAC.  So no special
0161  * handling is needed for the DMA API above and beyond what is included in the
0162  * arm64 implementation.
0163  *
0164  * In addition, since the Point of Unification is typically between L1 and L2
0165  * for the Brahma-B53 processor no special read-ahead cache handling is needed
0166  * for the IC IALLU and IC IALLUIS cache maintenance operations.
0167  *
0168  * However, it is not possible to specify the cache level (L3) for the cache
0169  * maintenance instructions operating by set/way to operate on the read-ahead
0170  * cache.  The read-ahead cache will maintain coherency when inner cache lines
0171  * are cleaned by set/way, but if it is necessary to invalidate inner cache
0172  * lines by set/way to maintain coherency with system masters operating on
0173  * shared memory that does not have hardware support for coherency, then it
0174  * will also be necessary to explicitly invalidate the read-ahead cache.
0175  */
0176 static void __init a72_b53_rac_enable_all(struct device_node *np)
0177 {
0178     unsigned int cpu;
0179     u32 enable = 0, pref_dist, shift;
0180
0181     if (IS_ENABLED(CONFIG_CACHE_B15_RAC))
0182         return;
0183
0184     if (WARN(num_possible_cpus() > 4, "RAC only supports 4 CPUs\n"))
0185         return;
0186
0187     pref_dist = cbc_readl(RAC_CONFIG1_REG);
0188     for_each_possible_cpu(cpu) {
0189         shift = cpu * RAC_CPU_SHIFT + RACPREFDATA_SHIFT;
0190         enable |= RAC_DATA_INST_EN_MASK << (cpu * RAC_CPU_SHIFT);
0191         if (cpubiuctrl_regs == a72_cpubiuctrl_regs) {
0192             enable &= ~(RACENPREF_MASK << shift);
0193             enable |= 3 << shift;
0194             pref_dist |= 1 << (cpu + DPREF_LINE_2_SHIFT);
0195         }
0196     }
0197
0198     cbc_writel(enable, RAC_CONFIG0_REG);
0199     cbc_writel(pref_dist, RAC_CONFIG1_REG);
0200
0201     pr_info("%pOF: Broadcom %s read-ahead cache\n",
0202         np, cpubiuctrl_regs == a72_cpubiuctrl_regs ?
0203         "Cortex-A72" : "Brahma-B53");
0204 }
0205
0206 static void __init mcp_a72_b53_set(void)
0207 {
0208     unsigned int i;
0209     u32 reg;
0210
0211     reg = brcmstb_get_family_id();
0212
0213     for (i = 0; i < ARRAY_SIZE(a72_b53_mach_compat); i++) {
0214         if (BRCM_ID(reg) == a72_b53_mach_compat[i])
0215             break;
0216     }
0217
0218     if (i == ARRAY_SIZE(a72_b53_mach_compat))
0219         return;
0220
0221     /* Set all 3 MCP interfaces to 8 credits */
0222     reg = cbc_readl(CPU_CREDIT_REG);
0223     for (i = 0; i < 3; i++) {
0224         reg &= ~(CPU_CREDIT_REG_MCPx_WRITE_CRED_MASK <<
0225              CPU_CREDIT_REG_MCPx_WRITE_CRED_SHIFT(i));
0226         reg &= ~(CPU_CREDIT_REG_MCPx_READ_CRED_MASK <<
0227              CPU_CREDIT_REG_MCPx_READ_CRED_SHIFT(i));
0228         reg |= 8 << CPU_CREDIT_REG_MCPx_WRITE_CRED_SHIFT(i);
0229         reg |= 8 << CPU_CREDIT_REG_MCPx_READ_CRED_SHIFT(i);
0230     }
0231     cbc_writel(reg, CPU_CREDIT_REG);
0232
0233     /* Max out the number of in-flight Jwords reads on the MCP interface */
0234     reg = cbc_readl(CPU_MCP_FLOW_REG);
0235     for (i = 0; i < 3; i++)
0236         reg |= CPU_MCP_FLOW_REG_MCPx_RDBUFF_CRED_MASK <<
0237             CPU_MCP_FLOW_REG_MCPx_RDBUFF_CRED_SHIFT(i);
0238     cbc_writel(reg, CPU_MCP_FLOW_REG);
0239
0240     /* Enable writeback throttling, set timeout to 128 cycles, 256 cycles
0241      * threshold
0242      */
0243     reg = cbc_readl(CPU_WRITEBACK_CTRL_REG);
0244     reg |= CPU_WRITEBACK_CTRL_REG_WB_THROTTLE_ENABLE;
0245     reg &= ~CPU_WRITEBACK_CTRL_REG_WB_THROTTLE_THRESHOLD_MASK;
0246     reg &= ~(CPU_WRITEBACK_CTRL_REG_WB_THROTTLE_TIMEOUT_MASK <<
0247          CPU_WRITEBACK_CTRL_REG_WB_THROTTLE_TIMEOUT_SHIFT);
0248     reg |= 8;
0249     reg |= 7 << CPU_WRITEBACK_CTRL_REG_WB_THROTTLE_TIMEOUT_SHIFT;
0250     cbc_writel(reg, CPU_WRITEBACK_CTRL_REG);
0251 }
0252
0253 static int __init setup_hifcpubiuctrl_regs(struct device_node *np)
0254 {
0255     struct device_node *cpu_dn;
0256     u32 family_id;
0257     int ret = 0;
0258
0259     cpubiuctrl_base = of_iomap(np, 0);
0260     if (!cpubiuctrl_base) {
0261         pr_err("failed to remap BIU control base\n");
0262         ret = -ENOMEM;
0263         goto out;
0264     }
0265
0266     mcp_wr_pairing_en = of_property_read_bool(np, "brcm,write-pairing");
0267
0268     cpu_dn = of_get_cpu_node(0, NULL);
0269     if (!cpu_dn) {
0270         pr_err("failed to obtain CPU device node\n");
0271         ret = -ENODEV;
0272         goto out;
0273     }
0274
0275     if (of_device_is_compatible(cpu_dn, "brcm,brahma-b15"))
0276         cpubiuctrl_regs = b15_cpubiuctrl_regs;
0277     else if (of_device_is_compatible(cpu_dn, "brcm,brahma-b53"))
0278         cpubiuctrl_regs = b53_cpubiuctrl_regs;
0279     else if (of_device_is_compatible(cpu_dn, "arm,cortex-a72"))
0280         cpubiuctrl_regs = a72_cpubiuctrl_regs;
0281     else {
0282         pr_err("unsupported CPU\n");
0283         ret = -EINVAL;
0284     }
0285     of_node_put(cpu_dn);
0286
0287     family_id = brcmstb_get_family_id();
0288     if (BRCM_ID(family_id) == 0x7260 && BRCM_REV(family_id) == 0)
0289         cpubiuctrl_regs = b53_cpubiuctrl_no_wb_regs;
0290 out:
0291     return ret;
0292 }
0293
0294 #ifdef CONFIG_PM_SLEEP
0295 static u32 cpubiuctrl_reg_save[NUM_CPU_BIUCTRL_REGS];
0296
0297 static int brcmstb_cpu_credit_reg_suspend(void)
0298 {
0299     unsigned int i;
0300
0301     if (!cpubiuctrl_base)
0302         return 0;
0303
0304     for (i = 0; i < NUM_CPU_BIUCTRL_REGS; i++)
0305         cpubiuctrl_reg_save[i] = cbc_readl(i);
0306
0307     return 0;
0308 }
0309
0310 static void brcmstb_cpu_credit_reg_resume(void)
0311 {
0312     unsigned int i;
0313
0314     if (!cpubiuctrl_base)
0315         return;
0316
0317     for (i = 0; i < NUM_CPU_BIUCTRL_REGS; i++)
0318         cbc_writel(cpubiuctrl_reg_save[i], i);
0319 }
0320
0321 static struct syscore_ops brcmstb_cpu_credit_syscore_ops = {
0322     .suspend = brcmstb_cpu_credit_reg_suspend,
0323     .resume = brcmstb_cpu_credit_reg_resume,
0324 };
0325 #endif
0326
0327
0328 static int __init brcmstb_biuctrl_init(void)
0329 {
0330     struct device_node *np;
0331     int ret;
0332
0333     /* We might be running on a multi-platform kernel, don't make this a
0334      * fatal error, just bail out early
0335      */
0336     np = of_find_compatible_node(NULL, NULL, "brcm,brcmstb-cpu-biu-ctrl");
0337     if (!np)
0338         return 0;
0339
0340     ret = setup_hifcpubiuctrl_regs(np);
0341     if (ret)
0342         goto out_put;
0343
0344     ret = mcp_write_pairing_set();
0345     if (ret) {
0346         pr_err("MCP: Unable to disable write pairing!\n");
0347         goto out_put;
0348     }
0349
0350     a72_b53_rac_enable_all(np);
0351     mcp_a72_b53_set();
0352 #ifdef CONFIG_PM_SLEEP
0353     register_syscore_ops(&brcmstb_cpu_credit_syscore_ops);
0354 #endif
0355     ret = 0;
0356 out_put:
0357     of_node_put(np);
0358     return ret;
0359 }
0360 early_initcall(brcmstb_biuctrl_init);