intel/uncore-frequency/uncore-frequency.c

0001 // SPDX-License-Identifier: GPL-2.0
0002 /*
0003  * Intel Uncore Frequency Setting
0004  * Copyright (c) 2022, Intel Corporation.
0005  * All rights reserved.
0006  *
0007  * Provide interface to set MSR 620 at a granularity of per die. On CPU online,
0008  * one control CPU is identified per die to read/write limit. This control CPU
0009  * is changed, if the CPU state is changed to offline. When the last CPU is
0010  * offline in a die then remove the sysfs object for that die.
0011  * The majority of actual code is related to sysfs create and read/write
0012  * attributes.
0013  *
0014  * Author: Srinivas Pandruvada <srinivas.pandruvada@linux.intel.com>
0015  */
0016
0017 #include <linux/cpu.h>
0018 #include <linux/module.h>
0019 #include <linux/slab.h>
0020 #include <linux/suspend.h>
0021 #include <asm/cpu_device_id.h>
0022 #include <asm/intel-family.h>
0023
0024 #include "uncore-frequency-common.h"
0025
0026 /* Max instances for uncore data, one for each die */
0027 static int uncore_max_entries __read_mostly;
0028 /* Storage for uncore data for all instances */
0029 static struct uncore_data *uncore_instances;
0030 /* Stores the CPU mask of the target CPUs to use during uncore read/write */
0031 static cpumask_t uncore_cpu_mask;
0032 /* CPU online callback register instance */
0033 static enum cpuhp_state uncore_hp_state __read_mostly;
0034
0035 #define MSR_UNCORE_RATIO_LIMIT  0x620
0036 #define MSR_UNCORE_PERF_STATUS  0x621
0037 #define UNCORE_FREQ_KHZ_MULTIPLIER  100000
0038
0039 static int uncore_read_control_freq(struct uncore_data *data, unsigned int *min,
0040                     unsigned int *max)
0041 {
0042     u64 cap;
0043     int ret;
0044
0045     if (data->control_cpu < 0)
0046         return -ENXIO;
0047
0048     ret = rdmsrl_on_cpu(data->control_cpu, MSR_UNCORE_RATIO_LIMIT, &cap);
0049     if (ret)
0050         return ret;
0051
0052     *max = (cap & 0x7F) * UNCORE_FREQ_KHZ_MULTIPLIER;
0053     *min = ((cap & GENMASK(14, 8)) >> 8) * UNCORE_FREQ_KHZ_MULTIPLIER;
0054
0055     return 0;
0056 }
0057
0058 static int uncore_write_control_freq(struct uncore_data *data, unsigned int input,
0059                      unsigned int min_max)
0060 {
0061     int ret;
0062     u64 cap;
0063
0064     input /= UNCORE_FREQ_KHZ_MULTIPLIER;
0065     if (!input || input > 0x7F)
0066         return -EINVAL;
0067
0068     if (data->control_cpu < 0)
0069         return -ENXIO;
0070
0071     ret = rdmsrl_on_cpu(data->control_cpu, MSR_UNCORE_RATIO_LIMIT, &cap);
0072     if (ret)
0073         return ret;
0074
0075     if (min_max) {
0076         cap &= ~0x7F;
0077         cap |= input;
0078     } else  {
0079         cap &= ~GENMASK(14, 8);
0080         cap |= (input << 8);
0081     }
0082
0083     ret = wrmsrl_on_cpu(data->control_cpu, MSR_UNCORE_RATIO_LIMIT, cap);
0084     if (ret)
0085         return ret;
0086
0087     data->stored_uncore_data = cap;
0088
0089     return 0;
0090 }
0091
0092 static int uncore_read_freq(struct uncore_data *data, unsigned int *freq)
0093 {
0094     u64 ratio;
0095     int ret;
0096
0097     if (data->control_cpu < 0)
0098         return -ENXIO;
0099
0100     ret = rdmsrl_on_cpu(data->control_cpu, MSR_UNCORE_PERF_STATUS, &ratio);
0101     if (ret)
0102         return ret;
0103
0104     *freq = (ratio & 0x7F) * UNCORE_FREQ_KHZ_MULTIPLIER;
0105
0106     return 0;
0107 }
0108
0109 /* Caller provides protection */
0110 static struct uncore_data *uncore_get_instance(unsigned int cpu)
0111 {
0112     int id = topology_logical_die_id(cpu);
0113
0114     if (id >= 0 && id < uncore_max_entries)
0115         return &uncore_instances[id];
0116
0117     return NULL;
0118 }
0119
0120 static int uncore_event_cpu_online(unsigned int cpu)
0121 {
0122     struct uncore_data *data;
0123     int target;
0124
0125     /* Check if there is an online cpu in the package for uncore MSR */
0126     target = cpumask_any_and(&uncore_cpu_mask, topology_die_cpumask(cpu));
0127     if (target < nr_cpu_ids)
0128         return 0;
0129
0130     /* Use this CPU on this die as a control CPU */
0131     cpumask_set_cpu(cpu, &uncore_cpu_mask);
0132
0133     data = uncore_get_instance(cpu);
0134     if (!data)
0135         return 0;
0136
0137     data->package_id = topology_physical_package_id(cpu);
0138     data->die_id = topology_die_id(cpu);
0139
0140     return uncore_freq_add_entry(data, cpu);
0141 }
0142
0143 static int uncore_event_cpu_offline(unsigned int cpu)
0144 {
0145     struct uncore_data *data;
0146     int target;
0147
0148     data = uncore_get_instance(cpu);
0149     if (!data)
0150         return 0;
0151
0152     /* Check if existing cpu is used for uncore MSRs */
0153     if (!cpumask_test_and_clear_cpu(cpu, &uncore_cpu_mask))
0154         return 0;
0155
0156     /* Find a new cpu to set uncore MSR */
0157     target = cpumask_any_but(topology_die_cpumask(cpu), cpu);
0158
0159     if (target < nr_cpu_ids) {
0160         cpumask_set_cpu(target, &uncore_cpu_mask);
0161         uncore_freq_add_entry(data, target);
0162     } else {
0163         uncore_freq_remove_die_entry(data);
0164     }
0165
0166     return 0;
0167 }
0168
0169 static int uncore_pm_notify(struct notifier_block *nb, unsigned long mode,
0170                 void *_unused)
0171 {
0172     int i;
0173
0174     switch (mode) {
0175     case PM_POST_HIBERNATION:
0176     case PM_POST_RESTORE:
0177     case PM_POST_SUSPEND:
0178         for (i = 0; i < uncore_max_entries; ++i) {
0179             struct uncore_data *data = &uncore_instances[i];
0180
0181             if (!data || !data->valid || !data->stored_uncore_data)
0182                 return 0;
0183
0184             wrmsrl_on_cpu(data->control_cpu, MSR_UNCORE_RATIO_LIMIT,
0185                       data->stored_uncore_data);
0186         }
0187         break;
0188     default:
0189         break;
0190     }
0191     return 0;
0192 }
0193
0194 static struct notifier_block uncore_pm_nb = {
0195     .notifier_call = uncore_pm_notify,
0196 };
0197
0198 static const struct x86_cpu_id intel_uncore_cpu_ids[] = {
0199     X86_MATCH_INTEL_FAM6_MODEL(BROADWELL_G, NULL),
0200     X86_MATCH_INTEL_FAM6_MODEL(BROADWELL_X, NULL),
0201     X86_MATCH_INTEL_FAM6_MODEL(BROADWELL_D, NULL),
0202     X86_MATCH_INTEL_FAM6_MODEL(SKYLAKE_X,   NULL),
0203     X86_MATCH_INTEL_FAM6_MODEL(ICELAKE_X,   NULL),
0204     X86_MATCH_INTEL_FAM6_MODEL(ICELAKE_D,   NULL),
0205     X86_MATCH_INTEL_FAM6_MODEL(SAPPHIRERAPIDS_X, NULL),
0206     {}
0207 };
0208 MODULE_DEVICE_TABLE(x86cpu, intel_uncore_cpu_ids);
0209
0210 static int __init intel_uncore_init(void)
0211 {
0212     const struct x86_cpu_id *id;
0213     int ret;
0214
0215     if (cpu_feature_enabled(X86_FEATURE_HYPERVISOR))
0216         return -ENODEV;
0217
0218     id = x86_match_cpu(intel_uncore_cpu_ids);
0219     if (!id)
0220         return -ENODEV;
0221
0222     uncore_max_entries = topology_max_packages() *
0223                     topology_max_die_per_package();
0224     uncore_instances = kcalloc(uncore_max_entries,
0225                    sizeof(*uncore_instances), GFP_KERNEL);
0226     if (!uncore_instances)
0227         return -ENOMEM;
0228
0229     ret = uncore_freq_common_init(uncore_read_control_freq, uncore_write_control_freq,
0230                       uncore_read_freq);
0231     if (ret)
0232         goto err_free;
0233
0234     ret = cpuhp_setup_state(CPUHP_AP_ONLINE_DYN,
0235                 "platform/x86/uncore-freq:online",
0236                 uncore_event_cpu_online,
0237                 uncore_event_cpu_offline);
0238     if (ret < 0)
0239         goto err_rem_kobj;
0240
0241     uncore_hp_state = ret;
0242
0243     ret = register_pm_notifier(&uncore_pm_nb);
0244     if (ret)
0245         goto err_rem_state;
0246
0247     return 0;
0248
0249 err_rem_state:
0250     cpuhp_remove_state(uncore_hp_state);
0251 err_rem_kobj:
0252     uncore_freq_common_exit();
0253 err_free:
0254     kfree(uncore_instances);
0255
0256     return ret;
0257 }
0258 module_init(intel_uncore_init)
0259
0260 static void __exit intel_uncore_exit(void)
0261 {
0262     int i;
0263
0264     unregister_pm_notifier(&uncore_pm_nb);
0265     cpuhp_remove_state(uncore_hp_state);
0266     for (i = 0; i < uncore_max_entries; ++i)
0267         uncore_freq_remove_die_entry(&uncore_instances[i]);
0268     uncore_freq_common_exit();
0269     kfree(uncore_instances);
0270 }
0271 module_exit(intel_uncore_exit)
0272
0273 MODULE_IMPORT_NS(INTEL_UNCORE_FREQUENCY);
0274 MODULE_LICENSE("GPL v2");
0275 MODULE_DESCRIPTION("Intel Uncore Frequency Limits Driver");