drivers/acpi/acpi_pad.c

0001 // SPDX-License-Identifier: GPL-2.0-only
0002 /*
0003  * acpi_pad.c ACPI Processor Aggregator Driver
0004  *
0005  * Copyright (c) 2009, Intel Corporation.
0006  */
0007
0008 #include <linux/kernel.h>
0009 #include <linux/cpumask.h>
0010 #include <linux/module.h>
0011 #include <linux/init.h>
0012 #include <linux/types.h>
0013 #include <linux/kthread.h>
0014 #include <uapi/linux/sched/types.h>
0015 #include <linux/freezer.h>
0016 #include <linux/cpu.h>
0017 #include <linux/tick.h>
0018 #include <linux/slab.h>
0019 #include <linux/acpi.h>
0020 #include <linux/perf_event.h>
0021 #include <asm/mwait.h>
0022 #include <xen/xen.h>
0023
0024 #define ACPI_PROCESSOR_AGGREGATOR_CLASS "acpi_pad"
0025 #define ACPI_PROCESSOR_AGGREGATOR_DEVICE_NAME "Processor Aggregator"
0026 #define ACPI_PROCESSOR_AGGREGATOR_NOTIFY 0x80
0027 static DEFINE_MUTEX(isolated_cpus_lock);
0028 static DEFINE_MUTEX(round_robin_lock);
0029
0030 static unsigned long power_saving_mwait_eax;
0031
0032 static unsigned char tsc_detected_unstable;
0033 static unsigned char tsc_marked_unstable;
0034
0035 static void power_saving_mwait_init(void)
0036 {
0037     unsigned int eax, ebx, ecx, edx;
0038     unsigned int highest_cstate = 0;
0039     unsigned int highest_subcstate = 0;
0040     int i;
0041
0042     if (!boot_cpu_has(X86_FEATURE_MWAIT))
0043         return;
0044     if (boot_cpu_data.cpuid_level < CPUID_MWAIT_LEAF)
0045         return;
0046
0047     cpuid(CPUID_MWAIT_LEAF, &eax, &ebx, &ecx, &edx);
0048
0049     if (!(ecx & CPUID5_ECX_EXTENSIONS_SUPPORTED) ||
0050         !(ecx & CPUID5_ECX_INTERRUPT_BREAK))
0051         return;
0052
0053     edx >>= MWAIT_SUBSTATE_SIZE;
0054     for (i = 0; i < 7 && edx; i++, edx >>= MWAIT_SUBSTATE_SIZE) {
0055         if (edx & MWAIT_SUBSTATE_MASK) {
0056             highest_cstate = i;
0057             highest_subcstate = edx & MWAIT_SUBSTATE_MASK;
0058         }
0059     }
0060     power_saving_mwait_eax = (highest_cstate << MWAIT_SUBSTATE_SIZE) |
0061         (highest_subcstate - 1);
0062
0063 #if defined(CONFIG_X86)
0064     switch (boot_cpu_data.x86_vendor) {
0065     case X86_VENDOR_HYGON:
0066     case X86_VENDOR_AMD:
0067     case X86_VENDOR_INTEL:
0068     case X86_VENDOR_ZHAOXIN:
0069         /*
0070          * AMD Fam10h TSC will tick in all
0071          * C/P/S0/S1 states when this bit is set.
0072          */
0073         if (!boot_cpu_has(X86_FEATURE_NONSTOP_TSC))
0074             tsc_detected_unstable = 1;
0075         break;
0076     default:
0077         /* TSC could halt in idle */
0078         tsc_detected_unstable = 1;
0079     }
0080 #endif
0081 }
0082
0083 static unsigned long cpu_weight[NR_CPUS];
0084 static int tsk_in_cpu[NR_CPUS] = {[0 ... NR_CPUS-1] = -1};
0085 static DECLARE_BITMAP(pad_busy_cpus_bits, NR_CPUS);
0086 static void round_robin_cpu(unsigned int tsk_index)
0087 {
0088     struct cpumask *pad_busy_cpus = to_cpumask(pad_busy_cpus_bits);
0089     cpumask_var_t tmp;
0090     int cpu;
0091     unsigned long min_weight = -1;
0092     unsigned long preferred_cpu;
0093
0094     if (!alloc_cpumask_var(&tmp, GFP_KERNEL))
0095         return;
0096
0097     mutex_lock(&round_robin_lock);
0098     cpumask_clear(tmp);
0099     for_each_cpu(cpu, pad_busy_cpus)
0100         cpumask_or(tmp, tmp, topology_sibling_cpumask(cpu));
0101     cpumask_andnot(tmp, cpu_online_mask, tmp);
0102     /* avoid HT sibilings if possible */
0103     if (cpumask_empty(tmp))
0104         cpumask_andnot(tmp, cpu_online_mask, pad_busy_cpus);
0105     if (cpumask_empty(tmp)) {
0106         mutex_unlock(&round_robin_lock);
0107         free_cpumask_var(tmp);
0108         return;
0109     }
0110     for_each_cpu(cpu, tmp) {
0111         if (cpu_weight[cpu] < min_weight) {
0112             min_weight = cpu_weight[cpu];
0113             preferred_cpu = cpu;
0114         }
0115     }
0116
0117     if (tsk_in_cpu[tsk_index] != -1)
0118         cpumask_clear_cpu(tsk_in_cpu[tsk_index], pad_busy_cpus);
0119     tsk_in_cpu[tsk_index] = preferred_cpu;
0120     cpumask_set_cpu(preferred_cpu, pad_busy_cpus);
0121     cpu_weight[preferred_cpu]++;
0122     mutex_unlock(&round_robin_lock);
0123
0124     set_cpus_allowed_ptr(current, cpumask_of(preferred_cpu));
0125
0126     free_cpumask_var(tmp);
0127 }
0128
0129 static void exit_round_robin(unsigned int tsk_index)
0130 {
0131     struct cpumask *pad_busy_cpus = to_cpumask(pad_busy_cpus_bits);
0132
0133     cpumask_clear_cpu(tsk_in_cpu[tsk_index], pad_busy_cpus);
0134     tsk_in_cpu[tsk_index] = -1;
0135 }
0136
0137 static unsigned int idle_pct = 5; /* percentage */
0138 static unsigned int round_robin_time = 1; /* second */
0139 static int power_saving_thread(void *data)
0140 {
0141     int do_sleep;
0142     unsigned int tsk_index = (unsigned long)data;
0143     u64 last_jiffies = 0;
0144
0145     sched_set_fifo_low(current);
0146
0147     while (!kthread_should_stop()) {
0148         unsigned long expire_time;
0149
0150         /* round robin to cpus */
0151         expire_time = last_jiffies + round_robin_time * HZ;
0152         if (time_before(expire_time, jiffies)) {
0153             last_jiffies = jiffies;
0154             round_robin_cpu(tsk_index);
0155         }
0156
0157         do_sleep = 0;
0158
0159         expire_time = jiffies + HZ * (100 - idle_pct) / 100;
0160
0161         while (!need_resched()) {
0162             if (tsc_detected_unstable && !tsc_marked_unstable) {
0163                 /* TSC could halt in idle, so notify users */
0164                 mark_tsc_unstable("TSC halts in idle");
0165                 tsc_marked_unstable = 1;
0166             }
0167             local_irq_disable();
0168
0169             perf_lopwr_cb(true);
0170
0171             tick_broadcast_enable();
0172             tick_broadcast_enter();
0173             stop_critical_timings();
0174
0175             mwait_idle_with_hints(power_saving_mwait_eax, 1);
0176
0177             start_critical_timings();
0178             tick_broadcast_exit();
0179
0180             perf_lopwr_cb(false);
0181
0182             local_irq_enable();
0183
0184             if (time_before(expire_time, jiffies)) {
0185                 do_sleep = 1;
0186                 break;
0187             }
0188         }
0189
0190         /*
0191          * current sched_rt has threshold for rt task running time.
0192          * When a rt task uses 95% CPU time, the rt thread will be
0193          * scheduled out for 5% CPU time to not starve other tasks. But
0194          * the mechanism only works when all CPUs have RT task running,
0195          * as if one CPU hasn't RT task, RT task from other CPUs will
0196          * borrow CPU time from this CPU and cause RT task use > 95%
0197          * CPU time. To make 'avoid starvation' work, takes a nap here.
0198          */
0199         if (unlikely(do_sleep))
0200             schedule_timeout_killable(HZ * idle_pct / 100);
0201
0202         /* If an external event has set the need_resched flag, then
0203          * we need to deal with it, or this loop will continue to
0204          * spin without calling __mwait().
0205          */
0206         if (unlikely(need_resched()))
0207             schedule();
0208     }
0209
0210     exit_round_robin(tsk_index);
0211     return 0;
0212 }
0213
0214 static struct task_struct *ps_tsks[NR_CPUS];
0215 static unsigned int ps_tsk_num;
0216 static int create_power_saving_task(void)
0217 {
0218     int rc;
0219
0220     ps_tsks[ps_tsk_num] = kthread_run(power_saving_thread,
0221         (void *)(unsigned long)ps_tsk_num,
0222         "acpi_pad/%d", ps_tsk_num);
0223
0224     if (IS_ERR(ps_tsks[ps_tsk_num])) {
0225         rc = PTR_ERR(ps_tsks[ps_tsk_num]);
0226         ps_tsks[ps_tsk_num] = NULL;
0227     } else {
0228         rc = 0;
0229         ps_tsk_num++;
0230     }
0231
0232     return rc;
0233 }
0234
0235 static void destroy_power_saving_task(void)
0236 {
0237     if (ps_tsk_num > 0) {
0238         ps_tsk_num--;
0239         kthread_stop(ps_tsks[ps_tsk_num]);
0240         ps_tsks[ps_tsk_num] = NULL;
0241     }
0242 }
0243
0244 static void set_power_saving_task_num(unsigned int num)
0245 {
0246     if (num > ps_tsk_num) {
0247         while (ps_tsk_num < num) {
0248             if (create_power_saving_task())
0249                 return;
0250         }
0251     } else if (num < ps_tsk_num) {
0252         while (ps_tsk_num > num)
0253             destroy_power_saving_task();
0254     }
0255 }
0256
0257 static void acpi_pad_idle_cpus(unsigned int num_cpus)
0258 {
0259     cpus_read_lock();
0260
0261     num_cpus = min_t(unsigned int, num_cpus, num_online_cpus());
0262     set_power_saving_task_num(num_cpus);
0263
0264     cpus_read_unlock();
0265 }
0266
0267 static uint32_t acpi_pad_idle_cpus_num(void)
0268 {
0269     return ps_tsk_num;
0270 }
0271
0272 static ssize_t rrtime_store(struct device *dev,
0273     struct device_attribute *attr, const char *buf, size_t count)
0274 {
0275     unsigned long num;
0276
0277     if (kstrtoul(buf, 0, &num))
0278         return -EINVAL;
0279     if (num < 1 || num >= 100)
0280         return -EINVAL;
0281     mutex_lock(&isolated_cpus_lock);
0282     round_robin_time = num;
0283     mutex_unlock(&isolated_cpus_lock);
0284     return count;
0285 }
0286
0287 static ssize_t rrtime_show(struct device *dev,
0288     struct device_attribute *attr, char *buf)
0289 {
0290     return scnprintf(buf, PAGE_SIZE, "%d\n", round_robin_time);
0291 }
0292 static DEVICE_ATTR_RW(rrtime);
0293
0294 static ssize_t idlepct_store(struct device *dev,
0295     struct device_attribute *attr, const char *buf, size_t count)
0296 {
0297     unsigned long num;
0298
0299     if (kstrtoul(buf, 0, &num))
0300         return -EINVAL;
0301     if (num < 1 || num >= 100)
0302         return -EINVAL;
0303     mutex_lock(&isolated_cpus_lock);
0304     idle_pct = num;
0305     mutex_unlock(&isolated_cpus_lock);
0306     return count;
0307 }
0308
0309 static ssize_t idlepct_show(struct device *dev,
0310     struct device_attribute *attr, char *buf)
0311 {
0312     return scnprintf(buf, PAGE_SIZE, "%d\n", idle_pct);
0313 }
0314 static DEVICE_ATTR_RW(idlepct);
0315
0316 static ssize_t idlecpus_store(struct device *dev,
0317     struct device_attribute *attr, const char *buf, size_t count)
0318 {
0319     unsigned long num;
0320
0321     if (kstrtoul(buf, 0, &num))
0322         return -EINVAL;
0323     mutex_lock(&isolated_cpus_lock);
0324     acpi_pad_idle_cpus(num);
0325     mutex_unlock(&isolated_cpus_lock);
0326     return count;
0327 }
0328
0329 static ssize_t idlecpus_show(struct device *dev,
0330     struct device_attribute *attr, char *buf)
0331 {
0332     return cpumap_print_to_pagebuf(false, buf,
0333                        to_cpumask(pad_busy_cpus_bits));
0334 }
0335
0336 static DEVICE_ATTR_RW(idlecpus);
0337
0338 static int acpi_pad_add_sysfs(struct acpi_device *device)
0339 {
0340     int result;
0341
0342     result = device_create_file(&device->dev, &dev_attr_idlecpus);
0343     if (result)
0344         return -ENODEV;
0345     result = device_create_file(&device->dev, &dev_attr_idlepct);
0346     if (result) {
0347         device_remove_file(&device->dev, &dev_attr_idlecpus);
0348         return -ENODEV;
0349     }
0350     result = device_create_file(&device->dev, &dev_attr_rrtime);
0351     if (result) {
0352         device_remove_file(&device->dev, &dev_attr_idlecpus);
0353         device_remove_file(&device->dev, &dev_attr_idlepct);
0354         return -ENODEV;
0355     }
0356     return 0;
0357 }
0358
0359 static void acpi_pad_remove_sysfs(struct acpi_device *device)
0360 {
0361     device_remove_file(&device->dev, &dev_attr_idlecpus);
0362     device_remove_file(&device->dev, &dev_attr_idlepct);
0363     device_remove_file(&device->dev, &dev_attr_rrtime);
0364 }
0365
0366 /*
0367  * Query firmware how many CPUs should be idle
0368  * return -1 on failure
0369  */
0370 static int acpi_pad_pur(acpi_handle handle)
0371 {
0372     struct acpi_buffer buffer = {ACPI_ALLOCATE_BUFFER, NULL};
0373     union acpi_object *package;
0374     int num = -1;
0375
0376     if (ACPI_FAILURE(acpi_evaluate_object(handle, "_PUR", NULL, &buffer)))
0377         return num;
0378
0379     if (!buffer.length || !buffer.pointer)
0380         return num;
0381
0382     package = buffer.pointer;
0383
0384     if (package->type == ACPI_TYPE_PACKAGE &&
0385         package->package.count == 2 &&
0386         package->package.elements[0].integer.value == 1) /* rev 1 */
0387
0388         num = package->package.elements[1].integer.value;
0389
0390     kfree(buffer.pointer);
0391     return num;
0392 }
0393
0394 static void acpi_pad_handle_notify(acpi_handle handle)
0395 {
0396     int num_cpus;
0397     uint32_t idle_cpus;
0398     struct acpi_buffer param = {
0399         .length = 4,
0400         .pointer = (void *)&idle_cpus,
0401     };
0402
0403     mutex_lock(&isolated_cpus_lock);
0404     num_cpus = acpi_pad_pur(handle);
0405     if (num_cpus < 0) {
0406         mutex_unlock(&isolated_cpus_lock);
0407         return;
0408     }
0409     acpi_pad_idle_cpus(num_cpus);
0410     idle_cpus = acpi_pad_idle_cpus_num();
0411     acpi_evaluate_ost(handle, ACPI_PROCESSOR_AGGREGATOR_NOTIFY, 0, &param);
0412     mutex_unlock(&isolated_cpus_lock);
0413 }
0414
0415 static void acpi_pad_notify(acpi_handle handle, u32 event,
0416     void *data)
0417 {
0418     struct acpi_device *device = data;
0419
0420     switch (event) {
0421     case ACPI_PROCESSOR_AGGREGATOR_NOTIFY:
0422         acpi_pad_handle_notify(handle);
0423         acpi_bus_generate_netlink_event(device->pnp.device_class,
0424             dev_name(&device->dev), event, 0);
0425         break;
0426     default:
0427         pr_warn("Unsupported event [0x%x]\n", event);
0428         break;
0429     }
0430 }
0431
0432 static int acpi_pad_add(struct acpi_device *device)
0433 {
0434     acpi_status status;
0435
0436     strcpy(acpi_device_name(device), ACPI_PROCESSOR_AGGREGATOR_DEVICE_NAME);
0437     strcpy(acpi_device_class(device), ACPI_PROCESSOR_AGGREGATOR_CLASS);
0438
0439     if (acpi_pad_add_sysfs(device))
0440         return -ENODEV;
0441
0442     status = acpi_install_notify_handler(device->handle,
0443         ACPI_DEVICE_NOTIFY, acpi_pad_notify, device);
0444     if (ACPI_FAILURE(status)) {
0445         acpi_pad_remove_sysfs(device);
0446         return -ENODEV;
0447     }
0448
0449     return 0;
0450 }
0451
0452 static int acpi_pad_remove(struct acpi_device *device)
0453 {
0454     mutex_lock(&isolated_cpus_lock);
0455     acpi_pad_idle_cpus(0);
0456     mutex_unlock(&isolated_cpus_lock);
0457
0458     acpi_remove_notify_handler(device->handle,
0459         ACPI_DEVICE_NOTIFY, acpi_pad_notify);
0460     acpi_pad_remove_sysfs(device);
0461     return 0;
0462 }
0463
0464 static const struct acpi_device_id pad_device_ids[] = {
0465     {"ACPI000C", 0},
0466     {"", 0},
0467 };
0468 MODULE_DEVICE_TABLE(acpi, pad_device_ids);
0469
0470 static struct acpi_driver acpi_pad_driver = {
0471     .name = "processor_aggregator",
0472     .class = ACPI_PROCESSOR_AGGREGATOR_CLASS,
0473     .ids = pad_device_ids,
0474     .ops = {
0475         .add = acpi_pad_add,
0476         .remove = acpi_pad_remove,
0477     },
0478 };
0479
0480 static int __init acpi_pad_init(void)
0481 {
0482     /* Xen ACPI PAD is used when running as Xen Dom0. */
0483     if (xen_initial_domain())
0484         return -ENODEV;
0485
0486     power_saving_mwait_init();
0487     if (power_saving_mwait_eax == 0)
0488         return -EINVAL;
0489
0490     return acpi_bus_register_driver(&acpi_pad_driver);
0491 }
0492
0493 static void __exit acpi_pad_exit(void)
0494 {
0495     acpi_bus_unregister_driver(&acpi_pad_driver);
0496 }
0497
0498 module_init(acpi_pad_init);
0499 module_exit(acpi_pad_exit);
0500 MODULE_AUTHOR("Shaohua Li<shaohua.li@intel.com>");
0501 MODULE_DESCRIPTION("ACPI Processor Aggregator Driver");
0502 MODULE_LICENSE("GPL");