arm/mm/cache-b15-rac.c

0001 // SPDX-License-Identifier: GPL-2.0-only
0002 /*
0003  * Broadcom Brahma-B15 CPU read-ahead cache management functions
0004  *
0005  * Copyright (C) 2015-2016 Broadcom
0006  */
0007
0008 #include <linux/err.h>
0009 #include <linux/spinlock.h>
0010 #include <linux/io.h>
0011 #include <linux/bitops.h>
0012 #include <linux/of_address.h>
0013 #include <linux/notifier.h>
0014 #include <linux/cpu.h>
0015 #include <linux/syscore_ops.h>
0016 #include <linux/reboot.h>
0017
0018 #include <asm/cacheflush.h>
0019 #include <asm/hardware/cache-b15-rac.h>
0020
0021 extern void v7_flush_kern_cache_all(void);
0022
0023 /* RAC register offsets, relative to the HIF_CPU_BIUCTRL register base */
0024 #define RAC_CONFIG0_REG         (0x78)
0025 #define  RACENPREF_MASK         (0x3)
0026 #define  RACPREFINST_SHIFT      (0)
0027 #define  RACENINST_SHIFT        (2)
0028 #define  RACPREFDATA_SHIFT      (4)
0029 #define  RACENDATA_SHIFT        (6)
0030 #define  RAC_CPU_SHIFT          (8)
0031 #define  RACCFG_MASK            (0xff)
0032 #define RAC_CONFIG1_REG         (0x7c)
0033 /* Brahma-B15 is a quad-core only design */
0034 #define B15_RAC_FLUSH_REG       (0x80)
0035 /* Brahma-B53 is an octo-core design */
0036 #define B53_RAC_FLUSH_REG       (0x84)
0037 #define  FLUSH_RAC          (1 << 0)
0038
0039 /* Bitmask to enable instruction and data prefetching with a 256-bytes stride */
0040 #define RAC_DATA_INST_EN_MASK       (1 << RACPREFINST_SHIFT | \
0041                      RACENPREF_MASK << RACENINST_SHIFT | \
0042                      1 << RACPREFDATA_SHIFT | \
0043                      RACENPREF_MASK << RACENDATA_SHIFT)
0044
0045 #define RAC_ENABLED         0
0046 /* Special state where we want to bypass the spinlock and call directly
0047  * into the v7 cache maintenance operations during suspend/resume
0048  */
0049 #define RAC_SUSPENDED           1
0050
0051 static void __iomem *b15_rac_base;
0052 static DEFINE_SPINLOCK(rac_lock);
0053
0054 static u32 rac_config0_reg;
0055 static u32 rac_flush_offset;
0056
0057 /* Initialization flag to avoid checking for b15_rac_base, and to prevent
0058  * multi-platform kernels from crashing here as well.
0059  */
0060 static unsigned long b15_rac_flags;
0061
0062 static inline u32 __b15_rac_disable(void)
0063 {
0064     u32 val = __raw_readl(b15_rac_base + RAC_CONFIG0_REG);
0065     __raw_writel(0, b15_rac_base + RAC_CONFIG0_REG);
0066     dmb();
0067     return val;
0068 }
0069
0070 static inline void __b15_rac_flush(void)
0071 {
0072     u32 reg;
0073
0074     __raw_writel(FLUSH_RAC, b15_rac_base + rac_flush_offset);
0075     do {
0076         /* This dmb() is required to force the Bus Interface Unit
0077          * to clean outstanding writes, and forces an idle cycle
0078          * to be inserted.
0079          */
0080         dmb();
0081         reg = __raw_readl(b15_rac_base + rac_flush_offset);
0082     } while (reg & FLUSH_RAC);
0083 }
0084
0085 static inline u32 b15_rac_disable_and_flush(void)
0086 {
0087     u32 reg;
0088
0089     reg = __b15_rac_disable();
0090     __b15_rac_flush();
0091     return reg;
0092 }
0093
0094 static inline void __b15_rac_enable(u32 val)
0095 {
0096     __raw_writel(val, b15_rac_base + RAC_CONFIG0_REG);
0097     /* dsb() is required here to be consistent with __flush_icache_all() */
0098     dsb();
0099 }
0100
0101 #define BUILD_RAC_CACHE_OP(name, bar)               \
0102 void b15_flush_##name(void)                 \
0103 {                               \
0104     unsigned int do_flush;                  \
0105     u32 val = 0;                        \
0106                                 \
0107     if (test_bit(RAC_SUSPENDED, &b15_rac_flags)) {      \
0108         v7_flush_##name();              \
0109         bar;                        \
0110         return;                     \
0111     }                           \
0112                                 \
0113     spin_lock(&rac_lock);                   \
0114     do_flush = test_bit(RAC_ENABLED, &b15_rac_flags);   \
0115     if (do_flush)                       \
0116         val = b15_rac_disable_and_flush();      \
0117     v7_flush_##name();                  \
0118     if (!do_flush)                      \
0119         bar;                        \
0120     else                            \
0121         __b15_rac_enable(val);              \
0122     spin_unlock(&rac_lock);                 \
0123 }
0124
0125 #define nobarrier
0126
0127 /* The readahead cache present in the Brahma-B15 CPU is a special piece of
0128  * hardware after the integrated L2 cache of the B15 CPU complex whose purpose
0129  * is to prefetch instruction and/or data with a line size of either 64 bytes
0130  * or 256 bytes. The rationale is that the data-bus of the CPU interface is
0131  * optimized for 256-bytes transactions, and enabling the readahead cache
0132  * provides a significant performance boost we want it enabled (typically
0133  * twice the performance for a memcpy benchmark application).
0134  *
0135  * The readahead cache is transparent for Modified Virtual Addresses
0136  * cache maintenance operations: ICIMVAU, DCIMVAC, DCCMVAC, DCCMVAU and
0137  * DCCIMVAC.
0138  *
0139  * It is however not transparent for the following cache maintenance
0140  * operations: DCISW, DCCSW, DCCISW, ICIALLUIS and ICIALLU which is precisely
0141  * what we are patching here with our BUILD_RAC_CACHE_OP here.
0142  */
0143 BUILD_RAC_CACHE_OP(kern_cache_all, nobarrier);
0144
0145 static void b15_rac_enable(void)
0146 {
0147     unsigned int cpu;
0148     u32 enable = 0;
0149
0150     for_each_possible_cpu(cpu)
0151         enable |= (RAC_DATA_INST_EN_MASK << (cpu * RAC_CPU_SHIFT));
0152
0153     b15_rac_disable_and_flush();
0154     __b15_rac_enable(enable);
0155 }
0156
0157 static int b15_rac_reboot_notifier(struct notifier_block *nb,
0158                    unsigned long action,
0159                    void *data)
0160 {
0161     /* During kexec, we are not yet migrated on the boot CPU, so we need to
0162      * make sure we are SMP safe here. Once the RAC is disabled, flag it as
0163      * suspended such that the hotplug notifier returns early.
0164      */
0165     if (action == SYS_RESTART) {
0166         spin_lock(&rac_lock);
0167         b15_rac_disable_and_flush();
0168         clear_bit(RAC_ENABLED, &b15_rac_flags);
0169         set_bit(RAC_SUSPENDED, &b15_rac_flags);
0170         spin_unlock(&rac_lock);
0171     }
0172
0173     return NOTIFY_DONE;
0174 }
0175
0176 static struct notifier_block b15_rac_reboot_nb = {
0177     .notifier_call  = b15_rac_reboot_notifier,
0178 };
0179
0180 /* The CPU hotplug case is the most interesting one, we basically need to make
0181  * sure that the RAC is disabled for the entire system prior to having a CPU
0182  * die, in particular prior to this dying CPU having exited the coherency
0183  * domain.
0184  *
0185  * Once this CPU is marked dead, we can safely re-enable the RAC for the
0186  * remaining CPUs in the system which are still online.
0187  *
0188  * Offlining a CPU is the problematic case, onlining a CPU is not much of an
0189  * issue since the CPU and its cache-level hierarchy will start filling with
0190  * the RAC disabled, so L1 and L2 only.
0191  *
0192  * In this function, we should NOT have to verify any unsafe setting/condition
0193  * b15_rac_base:
0194  *
0195  *   It is protected by the RAC_ENABLED flag which is cleared by default, and
0196  *   being cleared when initial procedure is done. b15_rac_base had been set at
0197  *   that time.
0198  *
0199  * RAC_ENABLED:
0200  *   There is a small timing windows, in b15_rac_init(), between
0201  *      cpuhp_setup_state_*()
0202  *      ...
0203  *      set RAC_ENABLED
0204  *   However, there is no hotplug activity based on the Linux booting procedure.
0205  *
0206  * Since we have to disable RAC for all cores, we keep RAC on as long as as
0207  * possible (disable it as late as possible) to gain the cache benefit.
0208  *
0209  * Thus, dying/dead states are chosen here
0210  *
0211  * We are choosing not do disable the RAC on a per-CPU basis, here, if we did
0212  * we would want to consider disabling it as early as possible to benefit the
0213  * other active CPUs.
0214  */
0215
0216 /* Running on the dying CPU */
0217 static int b15_rac_dying_cpu(unsigned int cpu)
0218 {
0219     /* During kexec/reboot, the RAC is disabled via the reboot notifier
0220      * return early here.
0221      */
0222     if (test_bit(RAC_SUSPENDED, &b15_rac_flags))
0223         return 0;
0224
0225     spin_lock(&rac_lock);
0226
0227     /* Indicate that we are starting a hotplug procedure */
0228     __clear_bit(RAC_ENABLED, &b15_rac_flags);
0229
0230     /* Disable the readahead cache and save its value to a global */
0231     rac_config0_reg = b15_rac_disable_and_flush();
0232
0233     spin_unlock(&rac_lock);
0234
0235     return 0;
0236 }
0237
0238 /* Running on a non-dying CPU */
0239 static int b15_rac_dead_cpu(unsigned int cpu)
0240 {
0241     /* During kexec/reboot, the RAC is disabled via the reboot notifier
0242      * return early here.
0243      */
0244     if (test_bit(RAC_SUSPENDED, &b15_rac_flags))
0245         return 0;
0246
0247     spin_lock(&rac_lock);
0248
0249     /* And enable it */
0250     __b15_rac_enable(rac_config0_reg);
0251     __set_bit(RAC_ENABLED, &b15_rac_flags);
0252
0253     spin_unlock(&rac_lock);
0254
0255     return 0;
0256 }
0257
0258 static int b15_rac_suspend(void)
0259 {
0260     /* Suspend the read-ahead cache oeprations, forcing our cache
0261      * implementation to fallback to the regular ARMv7 calls.
0262      *
0263      * We are guaranteed to be running on the boot CPU at this point and
0264      * with every other CPU quiesced, so setting RAC_SUSPENDED is not racy
0265      * here.
0266      */
0267     rac_config0_reg = b15_rac_disable_and_flush();
0268     set_bit(RAC_SUSPENDED, &b15_rac_flags);
0269
0270     return 0;
0271 }
0272
0273 static void b15_rac_resume(void)
0274 {
0275     /* Coming out of a S3 suspend/resume cycle, the read-ahead cache
0276      * register RAC_CONFIG0_REG will be restored to its default value, make
0277      * sure we re-enable it and set the enable flag, we are also guaranteed
0278      * to run on the boot CPU, so not racy again.
0279      */
0280     __b15_rac_enable(rac_config0_reg);
0281     clear_bit(RAC_SUSPENDED, &b15_rac_flags);
0282 }
0283
0284 static struct syscore_ops b15_rac_syscore_ops = {
0285     .suspend    = b15_rac_suspend,
0286     .resume     = b15_rac_resume,
0287 };
0288
0289 static int __init b15_rac_init(void)
0290 {
0291     struct device_node *dn, *cpu_dn;
0292     int ret = 0, cpu;
0293     u32 reg, en_mask = 0;
0294
0295     dn = of_find_compatible_node(NULL, NULL, "brcm,brcmstb-cpu-biu-ctrl");
0296     if (!dn)
0297         return -ENODEV;
0298
0299     if (WARN(num_possible_cpus() > 4, "RAC only supports 4 CPUs\n"))
0300         goto out;
0301
0302     b15_rac_base = of_iomap(dn, 0);
0303     if (!b15_rac_base) {
0304         pr_err("failed to remap BIU control base\n");
0305         ret = -ENOMEM;
0306         goto out;
0307     }
0308
0309     cpu_dn = of_get_cpu_node(0, NULL);
0310     if (!cpu_dn) {
0311         ret = -ENODEV;
0312         goto out;
0313     }
0314
0315     if (of_device_is_compatible(cpu_dn, "brcm,brahma-b15"))
0316         rac_flush_offset = B15_RAC_FLUSH_REG;
0317     else if (of_device_is_compatible(cpu_dn, "brcm,brahma-b53"))
0318         rac_flush_offset = B53_RAC_FLUSH_REG;
0319     else {
0320         pr_err("Unsupported CPU\n");
0321         of_node_put(cpu_dn);
0322         ret = -EINVAL;
0323         goto out;
0324     }
0325     of_node_put(cpu_dn);
0326
0327     ret = register_reboot_notifier(&b15_rac_reboot_nb);
0328     if (ret) {
0329         pr_err("failed to register reboot notifier\n");
0330         iounmap(b15_rac_base);
0331         goto out;
0332     }
0333
0334     if (IS_ENABLED(CONFIG_HOTPLUG_CPU)) {
0335         ret = cpuhp_setup_state_nocalls(CPUHP_AP_ARM_CACHE_B15_RAC_DEAD,
0336                     "arm/cache-b15-rac:dead",
0337                     NULL, b15_rac_dead_cpu);
0338         if (ret)
0339             goto out_unmap;
0340
0341         ret = cpuhp_setup_state_nocalls(CPUHP_AP_ARM_CACHE_B15_RAC_DYING,
0342                     "arm/cache-b15-rac:dying",
0343                     NULL, b15_rac_dying_cpu);
0344         if (ret)
0345             goto out_cpu_dead;
0346     }
0347
0348     if (IS_ENABLED(CONFIG_PM_SLEEP))
0349         register_syscore_ops(&b15_rac_syscore_ops);
0350
0351     spin_lock(&rac_lock);
0352     reg = __raw_readl(b15_rac_base + RAC_CONFIG0_REG);
0353     for_each_possible_cpu(cpu)
0354         en_mask |= ((1 << RACPREFDATA_SHIFT) << (cpu * RAC_CPU_SHIFT));
0355     WARN(reg & en_mask, "Read-ahead cache not previously disabled\n");
0356
0357     b15_rac_enable();
0358     set_bit(RAC_ENABLED, &b15_rac_flags);
0359     spin_unlock(&rac_lock);
0360
0361     pr_info("%pOF: Broadcom Brahma-B15 readahead cache\n", dn);
0362
0363     goto out;
0364
0365 out_cpu_dead:
0366     cpuhp_remove_state_nocalls(CPUHP_AP_ARM_CACHE_B15_RAC_DYING);
0367 out_unmap:
0368     unregister_reboot_notifier(&b15_rac_reboot_nb);
0369     iounmap(b15_rac_base);
0370 out:
0371     of_node_put(dn);
0372     return ret;
0373 }
0374 arch_initcall(b15_rac_init);