kernel/cpu/umwait.c

0001 // SPDX-License-Identifier: GPL-2.0
0002 #include <linux/syscore_ops.h>
0003 #include <linux/suspend.h>
0004 #include <linux/cpu.h>
0005
0006 #include <asm/msr.h>
0007 #include <asm/mwait.h>
0008
0009 #define UMWAIT_C02_ENABLE   0
0010
0011 #define UMWAIT_CTRL_VAL(max_time, c02_disable)              \
0012     (((max_time) & MSR_IA32_UMWAIT_CONTROL_TIME_MASK) |     \
0013     ((c02_disable) & MSR_IA32_UMWAIT_CONTROL_C02_DISABLE))
0014
0015 /*
0016  * Cache IA32_UMWAIT_CONTROL MSR. This is a systemwide control. By default,
0017  * umwait max time is 100000 in TSC-quanta and C0.2 is enabled
0018  */
0019 static u32 umwait_control_cached = UMWAIT_CTRL_VAL(100000, UMWAIT_C02_ENABLE);
0020
0021 /*
0022  * Cache the original IA32_UMWAIT_CONTROL MSR value which is configured by
0023  * hardware or BIOS before kernel boot.
0024  */
0025 static u32 orig_umwait_control_cached __ro_after_init;
0026
0027 /*
0028  * Serialize access to umwait_control_cached and IA32_UMWAIT_CONTROL MSR in
0029  * the sysfs write functions.
0030  */
0031 static DEFINE_MUTEX(umwait_lock);
0032
0033 static void umwait_update_control_msr(void * unused)
0034 {
0035     lockdep_assert_irqs_disabled();
0036     wrmsr(MSR_IA32_UMWAIT_CONTROL, READ_ONCE(umwait_control_cached), 0);
0037 }
0038
0039 /*
0040  * The CPU hotplug callback sets the control MSR to the global control
0041  * value.
0042  *
0043  * Disable interrupts so the read of umwait_control_cached and the WRMSR
0044  * are protected against a concurrent sysfs write. Otherwise the sysfs
0045  * write could update the cached value after it had been read on this CPU
0046  * and issue the IPI before the old value had been written. The IPI would
0047  * interrupt, write the new value and after return from IPI the previous
0048  * value would be written by this CPU.
0049  *
0050  * With interrupts disabled the upcoming CPU either sees the new control
0051  * value or the IPI is updating this CPU to the new control value after
0052  * interrupts have been reenabled.
0053  */
0054 static int umwait_cpu_online(unsigned int cpu)
0055 {
0056     local_irq_disable();
0057     umwait_update_control_msr(NULL);
0058     local_irq_enable();
0059     return 0;
0060 }
0061
0062 /*
0063  * The CPU hotplug callback sets the control MSR to the original control
0064  * value.
0065  */
0066 static int umwait_cpu_offline(unsigned int cpu)
0067 {
0068     /*
0069      * This code is protected by the CPU hotplug already and
0070      * orig_umwait_control_cached is never changed after it caches
0071      * the original control MSR value in umwait_init(). So there
0072      * is no race condition here.
0073      */
0074     wrmsr(MSR_IA32_UMWAIT_CONTROL, orig_umwait_control_cached, 0);
0075
0076     return 0;
0077 }
0078
0079 /*
0080  * On resume, restore IA32_UMWAIT_CONTROL MSR on the boot processor which
0081  * is the only active CPU at this time. The MSR is set up on the APs via the
0082  * CPU hotplug callback.
0083  *
0084  * This function is invoked on resume from suspend and hibernation. On
0085  * resume from suspend the restore should be not required, but we neither
0086  * trust the firmware nor does it matter if the same value is written
0087  * again.
0088  */
0089 static void umwait_syscore_resume(void)
0090 {
0091     umwait_update_control_msr(NULL);
0092 }
0093
0094 static struct syscore_ops umwait_syscore_ops = {
0095     .resume = umwait_syscore_resume,
0096 };
0097
0098 /* sysfs interface */
0099
0100 /*
0101  * When bit 0 in IA32_UMWAIT_CONTROL MSR is 1, C0.2 is disabled.
0102  * Otherwise, C0.2 is enabled.
0103  */
0104 static inline bool umwait_ctrl_c02_enabled(u32 ctrl)
0105 {
0106     return !(ctrl & MSR_IA32_UMWAIT_CONTROL_C02_DISABLE);
0107 }
0108
0109 static inline u32 umwait_ctrl_max_time(u32 ctrl)
0110 {
0111     return ctrl & MSR_IA32_UMWAIT_CONTROL_TIME_MASK;
0112 }
0113
0114 static inline void umwait_update_control(u32 maxtime, bool c02_enable)
0115 {
0116     u32 ctrl = maxtime & MSR_IA32_UMWAIT_CONTROL_TIME_MASK;
0117
0118     if (!c02_enable)
0119         ctrl |= MSR_IA32_UMWAIT_CONTROL_C02_DISABLE;
0120
0121     WRITE_ONCE(umwait_control_cached, ctrl);
0122     /* Propagate to all CPUs */
0123     on_each_cpu(umwait_update_control_msr, NULL, 1);
0124 }
0125
0126 static ssize_t
0127 enable_c02_show(struct device *dev, struct device_attribute *attr, char *buf)
0128 {
0129     u32 ctrl = READ_ONCE(umwait_control_cached);
0130
0131     return sprintf(buf, "%d\n", umwait_ctrl_c02_enabled(ctrl));
0132 }
0133
0134 static ssize_t enable_c02_store(struct device *dev,
0135                 struct device_attribute *attr,
0136                 const char *buf, size_t count)
0137 {
0138     bool c02_enable;
0139     u32 ctrl;
0140     int ret;
0141
0142     ret = kstrtobool(buf, &c02_enable);
0143     if (ret)
0144         return ret;
0145
0146     mutex_lock(&umwait_lock);
0147
0148     ctrl = READ_ONCE(umwait_control_cached);
0149     if (c02_enable != umwait_ctrl_c02_enabled(ctrl))
0150         umwait_update_control(ctrl, c02_enable);
0151
0152     mutex_unlock(&umwait_lock);
0153
0154     return count;
0155 }
0156 static DEVICE_ATTR_RW(enable_c02);
0157
0158 static ssize_t
0159 max_time_show(struct device *kobj, struct device_attribute *attr, char *buf)
0160 {
0161     u32 ctrl = READ_ONCE(umwait_control_cached);
0162
0163     return sprintf(buf, "%u\n", umwait_ctrl_max_time(ctrl));
0164 }
0165
0166 static ssize_t max_time_store(struct device *kobj,
0167                   struct device_attribute *attr,
0168                   const char *buf, size_t count)
0169 {
0170     u32 max_time, ctrl;
0171     int ret;
0172
0173     ret = kstrtou32(buf, 0, &max_time);
0174     if (ret)
0175         return ret;
0176
0177     /* bits[1:0] must be zero */
0178     if (max_time & ~MSR_IA32_UMWAIT_CONTROL_TIME_MASK)
0179         return -EINVAL;
0180
0181     mutex_lock(&umwait_lock);
0182
0183     ctrl = READ_ONCE(umwait_control_cached);
0184     if (max_time != umwait_ctrl_max_time(ctrl))
0185         umwait_update_control(max_time, umwait_ctrl_c02_enabled(ctrl));
0186
0187     mutex_unlock(&umwait_lock);
0188
0189     return count;
0190 }
0191 static DEVICE_ATTR_RW(max_time);
0192
0193 static struct attribute *umwait_attrs[] = {
0194     &dev_attr_enable_c02.attr,
0195     &dev_attr_max_time.attr,
0196     NULL
0197 };
0198
0199 static struct attribute_group umwait_attr_group = {
0200     .attrs = umwait_attrs,
0201     .name = "umwait_control",
0202 };
0203
0204 static int __init umwait_init(void)
0205 {
0206     struct device *dev;
0207     int ret;
0208
0209     if (!boot_cpu_has(X86_FEATURE_WAITPKG))
0210         return -ENODEV;
0211
0212     /*
0213      * Cache the original control MSR value before the control MSR is
0214      * changed. This is the only place where orig_umwait_control_cached
0215      * is modified.
0216      */
0217     rdmsrl(MSR_IA32_UMWAIT_CONTROL, orig_umwait_control_cached);
0218
0219     ret = cpuhp_setup_state(CPUHP_AP_ONLINE_DYN, "umwait:online",
0220                 umwait_cpu_online, umwait_cpu_offline);
0221     if (ret < 0) {
0222         /*
0223          * On failure, the control MSR on all CPUs has the
0224          * original control value.
0225          */
0226         return ret;
0227     }
0228
0229     register_syscore_ops(&umwait_syscore_ops);
0230
0231     /*
0232      * Add umwait control interface. Ignore failure, so at least the
0233      * default values are set up in case the machine manages to boot.
0234      */
0235     dev = cpu_subsys.dev_root;
0236     return sysfs_create_group(&dev->kobj, &umwait_attr_group);
0237 }
0238 device_initcall(umwait_init);