arm64/kernel/topology.c

0001 /*
0002  * arch/arm64/kernel/topology.c
0003  *
0004  * Copyright (C) 2011,2013,2014 Linaro Limited.
0005  *
0006  * Based on the arm32 version written by Vincent Guittot in turn based on
0007  * arch/sh/kernel/topology.c
0008  *
0009  * This file is subject to the terms and conditions of the GNU General Public
0010  * License.  See the file "COPYING" in the main directory of this archive
0011  * for more details.
0012  */
0013
0014 #include <linux/acpi.h>
0015 #include <linux/arch_topology.h>
0016 #include <linux/cacheinfo.h>
0017 #include <linux/cpufreq.h>
0018 #include <linux/init.h>
0019 #include <linux/percpu.h>
0020
0021 #include <asm/cpu.h>
0022 #include <asm/cputype.h>
0023 #include <asm/topology.h>
0024
0025 void store_cpu_topology(unsigned int cpuid)
0026 {
0027     struct cpu_topology *cpuid_topo = &cpu_topology[cpuid];
0028     u64 mpidr;
0029
0030     if (cpuid_topo->package_id != -1)
0031         goto topology_populated;
0032
0033     mpidr = read_cpuid_mpidr();
0034
0035     /* Uniprocessor systems can rely on default topology values */
0036     if (mpidr & MPIDR_UP_BITMASK)
0037         return;
0038
0039     /*
0040      * This would be the place to create cpu topology based on MPIDR.
0041      *
0042      * However, it cannot be trusted to depict the actual topology; some
0043      * pieces of the architecture enforce an artificial cap on Aff0 values
0044      * (e.g. GICv3's ICC_SGI1R_EL1 limits it to 15), leading to an
0045      * artificial cycling of Aff1, Aff2 and Aff3 values. IOW, these end up
0046      * having absolutely no relationship to the actual underlying system
0047      * topology, and cannot be reasonably used as core / package ID.
0048      *
0049      * If the MT bit is set, Aff0 *could* be used to define a thread ID, but
0050      * we still wouldn't be able to obtain a sane core ID. This means we
0051      * need to entirely ignore MPIDR for any topology deduction.
0052      */
0053     cpuid_topo->thread_id  = -1;
0054     cpuid_topo->core_id    = cpuid;
0055     cpuid_topo->package_id = cpu_to_node(cpuid);
0056
0057     pr_debug("CPU%u: cluster %d core %d thread %d mpidr %#016llx\n",
0058          cpuid, cpuid_topo->package_id, cpuid_topo->core_id,
0059          cpuid_topo->thread_id, mpidr);
0060
0061 topology_populated:
0062     update_siblings_masks(cpuid);
0063 }
0064
0065 #ifdef CONFIG_ACPI
0066 static bool __init acpi_cpu_is_threaded(int cpu)
0067 {
0068     int is_threaded = acpi_pptt_cpu_is_thread(cpu);
0069
0070     /*
0071      * if the PPTT doesn't have thread information, assume a homogeneous
0072      * machine and return the current CPU's thread state.
0073      */
0074     if (is_threaded < 0)
0075         is_threaded = read_cpuid_mpidr() & MPIDR_MT_BITMASK;
0076
0077     return !!is_threaded;
0078 }
0079
0080 /*
0081  * Propagate the topology information of the processor_topology_node tree to the
0082  * cpu_topology array.
0083  */
0084 int __init parse_acpi_topology(void)
0085 {
0086     int cpu, topology_id;
0087
0088     if (acpi_disabled)
0089         return 0;
0090
0091     for_each_possible_cpu(cpu) {
0092         topology_id = find_acpi_cpu_topology(cpu, 0);
0093         if (topology_id < 0)
0094             return topology_id;
0095
0096         if (acpi_cpu_is_threaded(cpu)) {
0097             cpu_topology[cpu].thread_id = topology_id;
0098             topology_id = find_acpi_cpu_topology(cpu, 1);
0099             cpu_topology[cpu].core_id   = topology_id;
0100         } else {
0101             cpu_topology[cpu].thread_id  = -1;
0102             cpu_topology[cpu].core_id    = topology_id;
0103         }
0104         topology_id = find_acpi_cpu_topology_cluster(cpu);
0105         cpu_topology[cpu].cluster_id = topology_id;
0106         topology_id = find_acpi_cpu_topology_package(cpu);
0107         cpu_topology[cpu].package_id = topology_id;
0108     }
0109
0110     return 0;
0111 }
0112 #endif
0113
0114 #ifdef CONFIG_ARM64_AMU_EXTN
0115 #define read_corecnt()  read_sysreg_s(SYS_AMEVCNTR0_CORE_EL0)
0116 #define read_constcnt() read_sysreg_s(SYS_AMEVCNTR0_CONST_EL0)
0117 #else
0118 #define read_corecnt()  (0UL)
0119 #define read_constcnt() (0UL)
0120 #endif
0121
0122 #undef pr_fmt
0123 #define pr_fmt(fmt) "AMU: " fmt
0124
0125 static DEFINE_PER_CPU_READ_MOSTLY(unsigned long, arch_max_freq_scale);
0126 static DEFINE_PER_CPU(u64, arch_const_cycles_prev);
0127 static DEFINE_PER_CPU(u64, arch_core_cycles_prev);
0128 static cpumask_var_t amu_fie_cpus;
0129
0130 void update_freq_counters_refs(void)
0131 {
0132     this_cpu_write(arch_core_cycles_prev, read_corecnt());
0133     this_cpu_write(arch_const_cycles_prev, read_constcnt());
0134 }
0135
0136 static inline bool freq_counters_valid(int cpu)
0137 {
0138     if ((cpu >= nr_cpu_ids) || !cpumask_test_cpu(cpu, cpu_present_mask))
0139         return false;
0140
0141     if (!cpu_has_amu_feat(cpu)) {
0142         pr_debug("CPU%d: counters are not supported.\n", cpu);
0143         return false;
0144     }
0145
0146     if (unlikely(!per_cpu(arch_const_cycles_prev, cpu) ||
0147              !per_cpu(arch_core_cycles_prev, cpu))) {
0148         pr_debug("CPU%d: cycle counters are not enabled.\n", cpu);
0149         return false;
0150     }
0151
0152     return true;
0153 }
0154
0155 static int freq_inv_set_max_ratio(int cpu, u64 max_rate, u64 ref_rate)
0156 {
0157     u64 ratio;
0158
0159     if (unlikely(!max_rate || !ref_rate)) {
0160         pr_debug("CPU%d: invalid maximum or reference frequency.\n",
0161              cpu);
0162         return -EINVAL;
0163     }
0164
0165     /*
0166      * Pre-compute the fixed ratio between the frequency of the constant
0167      * reference counter and the maximum frequency of the CPU.
0168      *
0169      *              ref_rate
0170      * arch_max_freq_scale =   ---------- * SCHED_CAPACITY_SCALE²
0171      *              max_rate
0172      *
0173      * We use a factor of 2 * SCHED_CAPACITY_SHIFT -> SCHED_CAPACITY_SCALE²
0174      * in order to ensure a good resolution for arch_max_freq_scale for
0175      * very low reference frequencies (down to the KHz range which should
0176      * be unlikely).
0177      */
0178     ratio = ref_rate << (2 * SCHED_CAPACITY_SHIFT);
0179     ratio = div64_u64(ratio, max_rate);
0180     if (!ratio) {
0181         WARN_ONCE(1, "Reference frequency too low.\n");
0182         return -EINVAL;
0183     }
0184
0185     per_cpu(arch_max_freq_scale, cpu) = (unsigned long)ratio;
0186
0187     return 0;
0188 }
0189
0190 static void amu_scale_freq_tick(void)
0191 {
0192     u64 prev_core_cnt, prev_const_cnt;
0193     u64 core_cnt, const_cnt, scale;
0194
0195     prev_const_cnt = this_cpu_read(arch_const_cycles_prev);
0196     prev_core_cnt = this_cpu_read(arch_core_cycles_prev);
0197
0198     update_freq_counters_refs();
0199
0200     const_cnt = this_cpu_read(arch_const_cycles_prev);
0201     core_cnt = this_cpu_read(arch_core_cycles_prev);
0202
0203     if (unlikely(core_cnt <= prev_core_cnt ||
0204              const_cnt <= prev_const_cnt))
0205         return;
0206
0207     /*
0208      *      /\core    arch_max_freq_scale
0209      * scale =  ------- * --------------------
0210      *      /\const   SCHED_CAPACITY_SCALE
0211      *
0212      * See validate_cpu_freq_invariance_counters() for details on
0213      * arch_max_freq_scale and the use of SCHED_CAPACITY_SHIFT.
0214      */
0215     scale = core_cnt - prev_core_cnt;
0216     scale *= this_cpu_read(arch_max_freq_scale);
0217     scale = div64_u64(scale >> SCHED_CAPACITY_SHIFT,
0218               const_cnt - prev_const_cnt);
0219
0220     scale = min_t(unsigned long, scale, SCHED_CAPACITY_SCALE);
0221     this_cpu_write(arch_freq_scale, (unsigned long)scale);
0222 }
0223
0224 static struct scale_freq_data amu_sfd = {
0225     .source = SCALE_FREQ_SOURCE_ARCH,
0226     .set_freq_scale = amu_scale_freq_tick,
0227 };
0228
0229 static void amu_fie_setup(const struct cpumask *cpus)
0230 {
0231     int cpu;
0232
0233     /* We are already set since the last insmod of cpufreq driver */
0234     if (unlikely(cpumask_subset(cpus, amu_fie_cpus)))
0235         return;
0236
0237     for_each_cpu(cpu, cpus) {
0238         if (!freq_counters_valid(cpu) ||
0239             freq_inv_set_max_ratio(cpu,
0240                        cpufreq_get_hw_max_freq(cpu) * 1000ULL,
0241                        arch_timer_get_rate()))
0242             return;
0243     }
0244
0245     cpumask_or(amu_fie_cpus, amu_fie_cpus, cpus);
0246
0247     topology_set_scale_freq_source(&amu_sfd, amu_fie_cpus);
0248
0249     pr_debug("CPUs[%*pbl]: counters will be used for FIE.",
0250          cpumask_pr_args(cpus));
0251 }
0252
0253 static int init_amu_fie_callback(struct notifier_block *nb, unsigned long val,
0254                  void *data)
0255 {
0256     struct cpufreq_policy *policy = data;
0257
0258     if (val == CPUFREQ_CREATE_POLICY)
0259         amu_fie_setup(policy->related_cpus);
0260
0261     /*
0262      * We don't need to handle CPUFREQ_REMOVE_POLICY event as the AMU
0263      * counters don't have any dependency on cpufreq driver once we have
0264      * initialized AMU support and enabled invariance. The AMU counters will
0265      * keep on working just fine in the absence of the cpufreq driver, and
0266      * for the CPUs for which there are no counters available, the last set
0267      * value of arch_freq_scale will remain valid as that is the frequency
0268      * those CPUs are running at.
0269      */
0270
0271     return 0;
0272 }
0273
0274 static struct notifier_block init_amu_fie_notifier = {
0275     .notifier_call = init_amu_fie_callback,
0276 };
0277
0278 static int __init init_amu_fie(void)
0279 {
0280     int ret;
0281
0282     if (!zalloc_cpumask_var(&amu_fie_cpus, GFP_KERNEL))
0283         return -ENOMEM;
0284
0285     ret = cpufreq_register_notifier(&init_amu_fie_notifier,
0286                     CPUFREQ_POLICY_NOTIFIER);
0287     if (ret)
0288         free_cpumask_var(amu_fie_cpus);
0289
0290     return ret;
0291 }
0292 core_initcall(init_amu_fie);
0293
0294 #ifdef CONFIG_ACPI_CPPC_LIB
0295 #include <acpi/cppc_acpi.h>
0296
0297 static void cpu_read_corecnt(void *val)
0298 {
0299     /*
0300      * A value of 0 can be returned if the current CPU does not support AMUs
0301      * or if the counter is disabled for this CPU. A return value of 0 at
0302      * counter read is properly handled as an error case by the users of the
0303      * counter.
0304      */
0305     *(u64 *)val = read_corecnt();
0306 }
0307
0308 static void cpu_read_constcnt(void *val)
0309 {
0310     /*
0311      * Return 0 if the current CPU is affected by erratum 2457168. A value
0312      * of 0 is also returned if the current CPU does not support AMUs or if
0313      * the counter is disabled. A return value of 0 at counter read is
0314      * properly handled as an error case by the users of the counter.
0315      */
0316     *(u64 *)val = this_cpu_has_cap(ARM64_WORKAROUND_2457168) ?
0317               0UL : read_constcnt();
0318 }
0319
0320 static inline
0321 int counters_read_on_cpu(int cpu, smp_call_func_t func, u64 *val)
0322 {
0323     /*
0324      * Abort call on counterless CPU or when interrupts are
0325      * disabled - can lead to deadlock in smp sync call.
0326      */
0327     if (!cpu_has_amu_feat(cpu))
0328         return -EOPNOTSUPP;
0329
0330     if (WARN_ON_ONCE(irqs_disabled()))
0331         return -EPERM;
0332
0333     smp_call_function_single(cpu, func, val, 1);
0334
0335     return 0;
0336 }
0337
0338 /*
0339  * Refer to drivers/acpi/cppc_acpi.c for the description of the functions
0340  * below.
0341  */
0342 bool cpc_ffh_supported(void)
0343 {
0344     int cpu = get_cpu_with_amu_feat();
0345
0346     /*
0347      * FFH is considered supported if there is at least one present CPU that
0348      * supports AMUs. Using FFH to read core and reference counters for CPUs
0349      * that do not support AMUs, have counters disabled or that are affected
0350      * by errata, will result in a return value of 0.
0351      *
0352      * This is done to allow any enabled and valid counters to be read
0353      * through FFH, knowing that potentially returning 0 as counter value is
0354      * properly handled by the users of these counters.
0355      */
0356     if ((cpu >= nr_cpu_ids) || !cpumask_test_cpu(cpu, cpu_present_mask))
0357         return false;
0358
0359     return true;
0360 }
0361
0362 int cpc_read_ffh(int cpu, struct cpc_reg *reg, u64 *val)
0363 {
0364     int ret = -EOPNOTSUPP;
0365
0366     switch ((u64)reg->address) {
0367     case 0x0:
0368         ret = counters_read_on_cpu(cpu, cpu_read_corecnt, val);
0369         break;
0370     case 0x1:
0371         ret = counters_read_on_cpu(cpu, cpu_read_constcnt, val);
0372         break;
0373     }
0374
0375     if (!ret) {
0376         *val &= GENMASK_ULL(reg->bit_offset + reg->bit_width - 1,
0377                     reg->bit_offset);
0378         *val >>= reg->bit_offset;
0379     }
0380
0381     return ret;
0382 }
0383
0384 int cpc_write_ffh(int cpunum, struct cpc_reg *reg, u64 val)
0385 {
0386     return -EOPNOTSUPP;
0387 }
0388 #endif /* CONFIG_ACPI_CPPC_LIB */