thermal/intel/x86_pkg_temp_thermal.c

0001 // SPDX-License-Identifier: GPL-2.0-only
0002 /*
0003  * x86_pkg_temp_thermal driver
0004  * Copyright (c) 2013, Intel Corporation.
0005  */
0006 #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
0007
0008 #include <linux/module.h>
0009 #include <linux/init.h>
0010 #include <linux/err.h>
0011 #include <linux/param.h>
0012 #include <linux/device.h>
0013 #include <linux/platform_device.h>
0014 #include <linux/cpu.h>
0015 #include <linux/smp.h>
0016 #include <linux/slab.h>
0017 #include <linux/pm.h>
0018 #include <linux/thermal.h>
0019 #include <linux/debugfs.h>
0020
0021 #include <asm/cpu_device_id.h>
0022
0023 #include "thermal_interrupt.h"
0024
0025 /*
0026 * Rate control delay: Idea is to introduce denounce effect
0027 * This should be long enough to avoid reduce events, when
0028 * threshold is set to a temperature, which is constantly
0029 * violated, but at the short enough to take any action.
0030 * The action can be remove threshold or change it to next
0031 * interesting setting. Based on experiments, in around
0032 * every 5 seconds under load will give us a significant
0033 * temperature change.
0034 */
0035 #define PKG_TEMP_THERMAL_NOTIFY_DELAY   5000
0036 static int notify_delay_ms = PKG_TEMP_THERMAL_NOTIFY_DELAY;
0037 module_param(notify_delay_ms, int, 0644);
0038 MODULE_PARM_DESC(notify_delay_ms,
0039     "User space notification delay in milli seconds.");
0040
0041 /* Number of trip points in thermal zone. Currently it can't
0042 * be more than 2. MSR can allow setting and getting notifications
0043 * for only 2 thresholds. This define enforces this, if there
0044 * is some wrong values returned by cpuid for number of thresholds.
0045 */
0046 #define MAX_NUMBER_OF_TRIPS 2
0047
0048 struct zone_device {
0049     int             cpu;
0050     bool                work_scheduled;
0051     u32             tj_max;
0052     u32             msr_pkg_therm_low;
0053     u32             msr_pkg_therm_high;
0054     struct delayed_work     work;
0055     struct thermal_zone_device  *tzone;
0056     struct cpumask          cpumask;
0057 };
0058
0059 static struct thermal_zone_params pkg_temp_tz_params = {
0060     .no_hwmon   = true,
0061 };
0062
0063 /* Keep track of how many zone pointers we allocated in init() */
0064 static int max_id __read_mostly;
0065 /* Array of zone pointers */
0066 static struct zone_device **zones;
0067 /* Serializes interrupt notification, work and hotplug */
0068 static DEFINE_RAW_SPINLOCK(pkg_temp_lock);
0069 /* Protects zone operation in the work function against hotplug removal */
0070 static DEFINE_MUTEX(thermal_zone_mutex);
0071
0072 /* The dynamically assigned cpu hotplug state for module_exit() */
0073 static enum cpuhp_state pkg_thermal_hp_state __read_mostly;
0074
0075 /* Debug counters to show using debugfs */
0076 static struct dentry *debugfs;
0077 static unsigned int pkg_interrupt_cnt;
0078 static unsigned int pkg_work_cnt;
0079
0080 static void pkg_temp_debugfs_init(void)
0081 {
0082     debugfs = debugfs_create_dir("pkg_temp_thermal", NULL);
0083
0084     debugfs_create_u32("pkg_thres_interrupt", S_IRUGO, debugfs,
0085                &pkg_interrupt_cnt);
0086     debugfs_create_u32("pkg_thres_work", S_IRUGO, debugfs,
0087                &pkg_work_cnt);
0088 }
0089
0090 /*
0091  * Protection:
0092  *
0093  * - cpu hotplug: Read serialized by cpu hotplug lock
0094  *        Write must hold pkg_temp_lock
0095  *
0096  * - Other callsites: Must hold pkg_temp_lock
0097  */
0098 static struct zone_device *pkg_temp_thermal_get_dev(unsigned int cpu)
0099 {
0100     int id = topology_logical_die_id(cpu);
0101
0102     if (id >= 0 && id < max_id)
0103         return zones[id];
0104     return NULL;
0105 }
0106
0107 /*
0108 * tj-max is interesting because threshold is set relative to this
0109 * temperature.
0110 */
0111 static int get_tj_max(int cpu, u32 *tj_max)
0112 {
0113     u32 eax, edx, val;
0114     int err;
0115
0116     err = rdmsr_safe_on_cpu(cpu, MSR_IA32_TEMPERATURE_TARGET, &eax, &edx);
0117     if (err)
0118         return err;
0119
0120     val = (eax >> 16) & 0xff;
0121     *tj_max = val * 1000;
0122
0123     return val ? 0 : -EINVAL;
0124 }
0125
0126 static int sys_get_curr_temp(struct thermal_zone_device *tzd, int *temp)
0127 {
0128     struct zone_device *zonedev = tzd->devdata;
0129     u32 eax, edx;
0130
0131     rdmsr_on_cpu(zonedev->cpu, MSR_IA32_PACKAGE_THERM_STATUS,
0132             &eax, &edx);
0133     if (eax & 0x80000000) {
0134         *temp = zonedev->tj_max - ((eax >> 16) & 0x7f) * 1000;
0135         pr_debug("sys_get_curr_temp %d\n", *temp);
0136         return 0;
0137     }
0138     return -EINVAL;
0139 }
0140
0141 static int sys_get_trip_temp(struct thermal_zone_device *tzd,
0142                  int trip, int *temp)
0143 {
0144     struct zone_device *zonedev = tzd->devdata;
0145     unsigned long thres_reg_value;
0146     u32 mask, shift, eax, edx;
0147     int ret;
0148
0149     if (trip >= MAX_NUMBER_OF_TRIPS)
0150         return -EINVAL;
0151
0152     if (trip) {
0153         mask = THERM_MASK_THRESHOLD1;
0154         shift = THERM_SHIFT_THRESHOLD1;
0155     } else {
0156         mask = THERM_MASK_THRESHOLD0;
0157         shift = THERM_SHIFT_THRESHOLD0;
0158     }
0159
0160     ret = rdmsr_on_cpu(zonedev->cpu, MSR_IA32_PACKAGE_THERM_INTERRUPT,
0161                &eax, &edx);
0162     if (ret < 0)
0163         return ret;
0164
0165     thres_reg_value = (eax & mask) >> shift;
0166     if (thres_reg_value)
0167         *temp = zonedev->tj_max - thres_reg_value * 1000;
0168     else
0169         *temp = THERMAL_TEMP_INVALID;
0170     pr_debug("sys_get_trip_temp %d\n", *temp);
0171
0172     return 0;
0173 }
0174
0175 static int
0176 sys_set_trip_temp(struct thermal_zone_device *tzd, int trip, int temp)
0177 {
0178     struct zone_device *zonedev = tzd->devdata;
0179     u32 l, h, mask, shift, intr;
0180     int ret;
0181
0182     if (trip >= MAX_NUMBER_OF_TRIPS || temp >= zonedev->tj_max)
0183         return -EINVAL;
0184
0185     ret = rdmsr_on_cpu(zonedev->cpu, MSR_IA32_PACKAGE_THERM_INTERRUPT,
0186                &l, &h);
0187     if (ret < 0)
0188         return ret;
0189
0190     if (trip) {
0191         mask = THERM_MASK_THRESHOLD1;
0192         shift = THERM_SHIFT_THRESHOLD1;
0193         intr = THERM_INT_THRESHOLD1_ENABLE;
0194     } else {
0195         mask = THERM_MASK_THRESHOLD0;
0196         shift = THERM_SHIFT_THRESHOLD0;
0197         intr = THERM_INT_THRESHOLD0_ENABLE;
0198     }
0199     l &= ~mask;
0200     /*
0201     * When users space sets a trip temperature == 0, which is indication
0202     * that, it is no longer interested in receiving notifications.
0203     */
0204     if (!temp) {
0205         l &= ~intr;
0206     } else {
0207         l |= (zonedev->tj_max - temp)/1000 << shift;
0208         l |= intr;
0209     }
0210
0211     return wrmsr_on_cpu(zonedev->cpu, MSR_IA32_PACKAGE_THERM_INTERRUPT,
0212             l, h);
0213 }
0214
0215 static int sys_get_trip_type(struct thermal_zone_device *thermal, int trip,
0216                  enum thermal_trip_type *type)
0217 {
0218     *type = THERMAL_TRIP_PASSIVE;
0219     return 0;
0220 }
0221
0222 /* Thermal zone callback registry */
0223 static struct thermal_zone_device_ops tzone_ops = {
0224     .get_temp = sys_get_curr_temp,
0225     .get_trip_temp = sys_get_trip_temp,
0226     .get_trip_type = sys_get_trip_type,
0227     .set_trip_temp = sys_set_trip_temp,
0228 };
0229
0230 static bool pkg_thermal_rate_control(void)
0231 {
0232     return true;
0233 }
0234
0235 /* Enable threshold interrupt on local package/cpu */
0236 static inline void enable_pkg_thres_interrupt(void)
0237 {
0238     u8 thres_0, thres_1;
0239     u32 l, h;
0240
0241     rdmsr(MSR_IA32_PACKAGE_THERM_INTERRUPT, l, h);
0242     /* only enable/disable if it had valid threshold value */
0243     thres_0 = (l & THERM_MASK_THRESHOLD0) >> THERM_SHIFT_THRESHOLD0;
0244     thres_1 = (l & THERM_MASK_THRESHOLD1) >> THERM_SHIFT_THRESHOLD1;
0245     if (thres_0)
0246         l |= THERM_INT_THRESHOLD0_ENABLE;
0247     if (thres_1)
0248         l |= THERM_INT_THRESHOLD1_ENABLE;
0249     wrmsr(MSR_IA32_PACKAGE_THERM_INTERRUPT, l, h);
0250 }
0251
0252 /* Disable threshold interrupt on local package/cpu */
0253 static inline void disable_pkg_thres_interrupt(void)
0254 {
0255     u32 l, h;
0256
0257     rdmsr(MSR_IA32_PACKAGE_THERM_INTERRUPT, l, h);
0258
0259     l &= ~(THERM_INT_THRESHOLD0_ENABLE | THERM_INT_THRESHOLD1_ENABLE);
0260     wrmsr(MSR_IA32_PACKAGE_THERM_INTERRUPT, l, h);
0261 }
0262
0263 static void pkg_temp_thermal_threshold_work_fn(struct work_struct *work)
0264 {
0265     struct thermal_zone_device *tzone = NULL;
0266     int cpu = smp_processor_id();
0267     struct zone_device *zonedev;
0268     u64 msr_val, wr_val;
0269
0270     mutex_lock(&thermal_zone_mutex);
0271     raw_spin_lock_irq(&pkg_temp_lock);
0272     ++pkg_work_cnt;
0273
0274     zonedev = pkg_temp_thermal_get_dev(cpu);
0275     if (!zonedev) {
0276         raw_spin_unlock_irq(&pkg_temp_lock);
0277         mutex_unlock(&thermal_zone_mutex);
0278         return;
0279     }
0280     zonedev->work_scheduled = false;
0281
0282     rdmsrl(MSR_IA32_PACKAGE_THERM_STATUS, msr_val);
0283     wr_val = msr_val & ~(THERM_LOG_THRESHOLD0 | THERM_LOG_THRESHOLD1);
0284     if (wr_val != msr_val) {
0285         wrmsrl(MSR_IA32_PACKAGE_THERM_STATUS, wr_val);
0286         tzone = zonedev->tzone;
0287     }
0288
0289     enable_pkg_thres_interrupt();
0290     raw_spin_unlock_irq(&pkg_temp_lock);
0291
0292     /*
0293      * If tzone is not NULL, then thermal_zone_mutex will prevent the
0294      * concurrent removal in the cpu offline callback.
0295      */
0296     if (tzone)
0297         thermal_zone_device_update(tzone, THERMAL_EVENT_UNSPECIFIED);
0298
0299     mutex_unlock(&thermal_zone_mutex);
0300 }
0301
0302 static void pkg_thermal_schedule_work(int cpu, struct delayed_work *work)
0303 {
0304     unsigned long ms = msecs_to_jiffies(notify_delay_ms);
0305
0306     schedule_delayed_work_on(cpu, work, ms);
0307 }
0308
0309 static int pkg_thermal_notify(u64 msr_val)
0310 {
0311     int cpu = smp_processor_id();
0312     struct zone_device *zonedev;
0313     unsigned long flags;
0314
0315     raw_spin_lock_irqsave(&pkg_temp_lock, flags);
0316     ++pkg_interrupt_cnt;
0317
0318     disable_pkg_thres_interrupt();
0319
0320     /* Work is per package, so scheduling it once is enough. */
0321     zonedev = pkg_temp_thermal_get_dev(cpu);
0322     if (zonedev && !zonedev->work_scheduled) {
0323         zonedev->work_scheduled = true;
0324         pkg_thermal_schedule_work(zonedev->cpu, &zonedev->work);
0325     }
0326
0327     raw_spin_unlock_irqrestore(&pkg_temp_lock, flags);
0328     return 0;
0329 }
0330
0331 static int pkg_temp_thermal_device_add(unsigned int cpu)
0332 {
0333     int id = topology_logical_die_id(cpu);
0334     u32 tj_max, eax, ebx, ecx, edx;
0335     struct zone_device *zonedev;
0336     int thres_count, err;
0337
0338     if (id >= max_id)
0339         return -ENOMEM;
0340
0341     cpuid(6, &eax, &ebx, &ecx, &edx);
0342     thres_count = ebx & 0x07;
0343     if (!thres_count)
0344         return -ENODEV;
0345
0346     thres_count = clamp_val(thres_count, 0, MAX_NUMBER_OF_TRIPS);
0347
0348     err = get_tj_max(cpu, &tj_max);
0349     if (err)
0350         return err;
0351
0352     zonedev = kzalloc(sizeof(*zonedev), GFP_KERNEL);
0353     if (!zonedev)
0354         return -ENOMEM;
0355
0356     INIT_DELAYED_WORK(&zonedev->work, pkg_temp_thermal_threshold_work_fn);
0357     zonedev->cpu = cpu;
0358     zonedev->tj_max = tj_max;
0359     zonedev->tzone = thermal_zone_device_register("x86_pkg_temp",
0360             thres_count,
0361             (thres_count == MAX_NUMBER_OF_TRIPS) ? 0x03 : 0x01,
0362             zonedev, &tzone_ops, &pkg_temp_tz_params, 0, 0);
0363     if (IS_ERR(zonedev->tzone)) {
0364         err = PTR_ERR(zonedev->tzone);
0365         kfree(zonedev);
0366         return err;
0367     }
0368     err = thermal_zone_device_enable(zonedev->tzone);
0369     if (err) {
0370         thermal_zone_device_unregister(zonedev->tzone);
0371         kfree(zonedev);
0372         return err;
0373     }
0374     /* Store MSR value for package thermal interrupt, to restore at exit */
0375     rdmsr(MSR_IA32_PACKAGE_THERM_INTERRUPT, zonedev->msr_pkg_therm_low,
0376           zonedev->msr_pkg_therm_high);
0377
0378     cpumask_set_cpu(cpu, &zonedev->cpumask);
0379     raw_spin_lock_irq(&pkg_temp_lock);
0380     zones[id] = zonedev;
0381     raw_spin_unlock_irq(&pkg_temp_lock);
0382     return 0;
0383 }
0384
0385 static int pkg_thermal_cpu_offline(unsigned int cpu)
0386 {
0387     struct zone_device *zonedev = pkg_temp_thermal_get_dev(cpu);
0388     bool lastcpu, was_target;
0389     int target;
0390
0391     if (!zonedev)
0392         return 0;
0393
0394     target = cpumask_any_but(&zonedev->cpumask, cpu);
0395     cpumask_clear_cpu(cpu, &zonedev->cpumask);
0396     lastcpu = target >= nr_cpu_ids;
0397     /*
0398      * Remove the sysfs files, if this is the last cpu in the package
0399      * before doing further cleanups.
0400      */
0401     if (lastcpu) {
0402         struct thermal_zone_device *tzone = zonedev->tzone;
0403
0404         /*
0405          * We must protect against a work function calling
0406          * thermal_zone_update, after/while unregister. We null out
0407          * the pointer under the zone mutex, so the worker function
0408          * won't try to call.
0409          */
0410         mutex_lock(&thermal_zone_mutex);
0411         zonedev->tzone = NULL;
0412         mutex_unlock(&thermal_zone_mutex);
0413
0414         thermal_zone_device_unregister(tzone);
0415     }
0416
0417     /* Protect against work and interrupts */
0418     raw_spin_lock_irq(&pkg_temp_lock);
0419
0420     /*
0421      * Check whether this cpu was the current target and store the new
0422      * one. When we drop the lock, then the interrupt notify function
0423      * will see the new target.
0424      */
0425     was_target = zonedev->cpu == cpu;
0426     zonedev->cpu = target;
0427
0428     /*
0429      * If this is the last CPU in the package remove the package
0430      * reference from the array and restore the interrupt MSR. When we
0431      * drop the lock neither the interrupt notify function nor the
0432      * worker will see the package anymore.
0433      */
0434     if (lastcpu) {
0435         zones[topology_logical_die_id(cpu)] = NULL;
0436         /* After this point nothing touches the MSR anymore. */
0437         wrmsr(MSR_IA32_PACKAGE_THERM_INTERRUPT,
0438               zonedev->msr_pkg_therm_low, zonedev->msr_pkg_therm_high);
0439     }
0440
0441     /*
0442      * Check whether there is work scheduled and whether the work is
0443      * targeted at the outgoing CPU.
0444      */
0445     if (zonedev->work_scheduled && was_target) {
0446         /*
0447          * To cancel the work we need to drop the lock, otherwise
0448          * we might deadlock if the work needs to be flushed.
0449          */
0450         raw_spin_unlock_irq(&pkg_temp_lock);
0451         cancel_delayed_work_sync(&zonedev->work);
0452         raw_spin_lock_irq(&pkg_temp_lock);
0453         /*
0454          * If this is not the last cpu in the package and the work
0455          * did not run after we dropped the lock above, then we
0456          * need to reschedule the work, otherwise the interrupt
0457          * stays disabled forever.
0458          */
0459         if (!lastcpu && zonedev->work_scheduled)
0460             pkg_thermal_schedule_work(target, &zonedev->work);
0461     }
0462
0463     raw_spin_unlock_irq(&pkg_temp_lock);
0464
0465     /* Final cleanup if this is the last cpu */
0466     if (lastcpu)
0467         kfree(zonedev);
0468     return 0;
0469 }
0470
0471 static int pkg_thermal_cpu_online(unsigned int cpu)
0472 {
0473     struct zone_device *zonedev = pkg_temp_thermal_get_dev(cpu);
0474     struct cpuinfo_x86 *c = &cpu_data(cpu);
0475
0476     /* Paranoia check */
0477     if (!cpu_has(c, X86_FEATURE_DTHERM) || !cpu_has(c, X86_FEATURE_PTS))
0478         return -ENODEV;
0479
0480     /* If the package exists, nothing to do */
0481     if (zonedev) {
0482         cpumask_set_cpu(cpu, &zonedev->cpumask);
0483         return 0;
0484     }
0485     return pkg_temp_thermal_device_add(cpu);
0486 }
0487
0488 static const struct x86_cpu_id __initconst pkg_temp_thermal_ids[] = {
0489     X86_MATCH_VENDOR_FEATURE(INTEL, X86_FEATURE_PTS, NULL),
0490     {}
0491 };
0492 MODULE_DEVICE_TABLE(x86cpu, pkg_temp_thermal_ids);
0493
0494 static int __init pkg_temp_thermal_init(void)
0495 {
0496     int ret;
0497
0498     if (!x86_match_cpu(pkg_temp_thermal_ids))
0499         return -ENODEV;
0500
0501     max_id = topology_max_packages() * topology_max_die_per_package();
0502     zones = kcalloc(max_id, sizeof(struct zone_device *),
0503                GFP_KERNEL);
0504     if (!zones)
0505         return -ENOMEM;
0506
0507     ret = cpuhp_setup_state(CPUHP_AP_ONLINE_DYN, "thermal/x86_pkg:online",
0508                 pkg_thermal_cpu_online, pkg_thermal_cpu_offline);
0509     if (ret < 0)
0510         goto err;
0511
0512     /* Store the state for module exit */
0513     pkg_thermal_hp_state = ret;
0514
0515     platform_thermal_package_notify = pkg_thermal_notify;
0516     platform_thermal_package_rate_control = pkg_thermal_rate_control;
0517
0518      /* Don't care if it fails */
0519     pkg_temp_debugfs_init();
0520     return 0;
0521
0522 err:
0523     kfree(zones);
0524     return ret;
0525 }
0526 module_init(pkg_temp_thermal_init)
0527
0528 static void __exit pkg_temp_thermal_exit(void)
0529 {
0530     platform_thermal_package_notify = NULL;
0531     platform_thermal_package_rate_control = NULL;
0532
0533     cpuhp_remove_state(pkg_thermal_hp_state);
0534     debugfs_remove_recursive(debugfs);
0535     kfree(zones);
0536 }
0537 module_exit(pkg_temp_thermal_exit)
0538
0539 MODULE_DESCRIPTION("X86 PKG TEMP Thermal Driver");
0540 MODULE_AUTHOR("Srinivas Pandruvada <srinivas.pandruvada@linux.intel.com>");
0541 MODULE_LICENSE("GPL v2");