drivers/base/cpu.c

0001 // SPDX-License-Identifier: GPL-2.0
0002 /*
0003  * CPU subsystem support
0004  */
0005
0006 #include <linux/kernel.h>
0007 #include <linux/module.h>
0008 #include <linux/init.h>
0009 #include <linux/sched.h>
0010 #include <linux/cpu.h>
0011 #include <linux/topology.h>
0012 #include <linux/device.h>
0013 #include <linux/node.h>
0014 #include <linux/gfp.h>
0015 #include <linux/slab.h>
0016 #include <linux/percpu.h>
0017 #include <linux/acpi.h>
0018 #include <linux/of.h>
0019 #include <linux/cpufeature.h>
0020 #include <linux/tick.h>
0021 #include <linux/pm_qos.h>
0022 #include <linux/sched/isolation.h>
0023
0024 #include "base.h"
0025
0026 static DEFINE_PER_CPU(struct device *, cpu_sys_devices);
0027
0028 static int cpu_subsys_match(struct device *dev, struct device_driver *drv)
0029 {
0030     /* ACPI style match is the only one that may succeed. */
0031     if (acpi_driver_match_device(dev, drv))
0032         return 1;
0033
0034     return 0;
0035 }
0036
0037 #ifdef CONFIG_HOTPLUG_CPU
0038 static void change_cpu_under_node(struct cpu *cpu,
0039             unsigned int from_nid, unsigned int to_nid)
0040 {
0041     int cpuid = cpu->dev.id;
0042     unregister_cpu_under_node(cpuid, from_nid);
0043     register_cpu_under_node(cpuid, to_nid);
0044     cpu->node_id = to_nid;
0045 }
0046
0047 static int cpu_subsys_online(struct device *dev)
0048 {
0049     struct cpu *cpu = container_of(dev, struct cpu, dev);
0050     int cpuid = dev->id;
0051     int from_nid, to_nid;
0052     int ret;
0053
0054     from_nid = cpu_to_node(cpuid);
0055     if (from_nid == NUMA_NO_NODE)
0056         return -ENODEV;
0057
0058     ret = cpu_device_up(dev);
0059     /*
0060      * When hot adding memory to memoryless node and enabling a cpu
0061      * on the node, node number of the cpu may internally change.
0062      */
0063     to_nid = cpu_to_node(cpuid);
0064     if (from_nid != to_nid)
0065         change_cpu_under_node(cpu, from_nid, to_nid);
0066
0067     return ret;
0068 }
0069
0070 static int cpu_subsys_offline(struct device *dev)
0071 {
0072     return cpu_device_down(dev);
0073 }
0074
0075 void unregister_cpu(struct cpu *cpu)
0076 {
0077     int logical_cpu = cpu->dev.id;
0078
0079     unregister_cpu_under_node(logical_cpu, cpu_to_node(logical_cpu));
0080
0081     device_unregister(&cpu->dev);
0082     per_cpu(cpu_sys_devices, logical_cpu) = NULL;
0083     return;
0084 }
0085
0086 #ifdef CONFIG_ARCH_CPU_PROBE_RELEASE
0087 static ssize_t cpu_probe_store(struct device *dev,
0088                    struct device_attribute *attr,
0089                    const char *buf,
0090                    size_t count)
0091 {
0092     ssize_t cnt;
0093     int ret;
0094
0095     ret = lock_device_hotplug_sysfs();
0096     if (ret)
0097         return ret;
0098
0099     cnt = arch_cpu_probe(buf, count);
0100
0101     unlock_device_hotplug();
0102     return cnt;
0103 }
0104
0105 static ssize_t cpu_release_store(struct device *dev,
0106                  struct device_attribute *attr,
0107                  const char *buf,
0108                  size_t count)
0109 {
0110     ssize_t cnt;
0111     int ret;
0112
0113     ret = lock_device_hotplug_sysfs();
0114     if (ret)
0115         return ret;
0116
0117     cnt = arch_cpu_release(buf, count);
0118
0119     unlock_device_hotplug();
0120     return cnt;
0121 }
0122
0123 static DEVICE_ATTR(probe, S_IWUSR, NULL, cpu_probe_store);
0124 static DEVICE_ATTR(release, S_IWUSR, NULL, cpu_release_store);
0125 #endif /* CONFIG_ARCH_CPU_PROBE_RELEASE */
0126 #endif /* CONFIG_HOTPLUG_CPU */
0127
0128 struct bus_type cpu_subsys = {
0129     .name = "cpu",
0130     .dev_name = "cpu",
0131     .match = cpu_subsys_match,
0132 #ifdef CONFIG_HOTPLUG_CPU
0133     .online = cpu_subsys_online,
0134     .offline = cpu_subsys_offline,
0135 #endif
0136 };
0137 EXPORT_SYMBOL_GPL(cpu_subsys);
0138
0139 #ifdef CONFIG_KEXEC
0140 #include <linux/kexec.h>
0141
0142 static ssize_t crash_notes_show(struct device *dev,
0143                 struct device_attribute *attr,
0144                 char *buf)
0145 {
0146     struct cpu *cpu = container_of(dev, struct cpu, dev);
0147     unsigned long long addr;
0148     int cpunum;
0149
0150     cpunum = cpu->dev.id;
0151
0152     /*
0153      * Might be reading other cpu's data based on which cpu read thread
0154      * has been scheduled. But cpu data (memory) is allocated once during
0155      * boot up and this data does not change there after. Hence this
0156      * operation should be safe. No locking required.
0157      */
0158     addr = per_cpu_ptr_to_phys(per_cpu_ptr(crash_notes, cpunum));
0159
0160     return sysfs_emit(buf, "%llx\n", addr);
0161 }
0162 static DEVICE_ATTR_ADMIN_RO(crash_notes);
0163
0164 static ssize_t crash_notes_size_show(struct device *dev,
0165                      struct device_attribute *attr,
0166                      char *buf)
0167 {
0168     return sysfs_emit(buf, "%zu\n", sizeof(note_buf_t));
0169 }
0170 static DEVICE_ATTR_ADMIN_RO(crash_notes_size);
0171
0172 static struct attribute *crash_note_cpu_attrs[] = {
0173     &dev_attr_crash_notes.attr,
0174     &dev_attr_crash_notes_size.attr,
0175     NULL
0176 };
0177
0178 static const struct attribute_group crash_note_cpu_attr_group = {
0179     .attrs = crash_note_cpu_attrs,
0180 };
0181 #endif
0182
0183 static const struct attribute_group *common_cpu_attr_groups[] = {
0184 #ifdef CONFIG_KEXEC
0185     &crash_note_cpu_attr_group,
0186 #endif
0187     NULL
0188 };
0189
0190 static const struct attribute_group *hotplugable_cpu_attr_groups[] = {
0191 #ifdef CONFIG_KEXEC
0192     &crash_note_cpu_attr_group,
0193 #endif
0194     NULL
0195 };
0196
0197 /*
0198  * Print cpu online, possible, present, and system maps
0199  */
0200
0201 struct cpu_attr {
0202     struct device_attribute attr;
0203     const struct cpumask *const map;
0204 };
0205
0206 static ssize_t show_cpus_attr(struct device *dev,
0207                   struct device_attribute *attr,
0208                   char *buf)
0209 {
0210     struct cpu_attr *ca = container_of(attr, struct cpu_attr, attr);
0211
0212     return cpumap_print_to_pagebuf(true, buf, ca->map);
0213 }
0214
0215 #define _CPU_ATTR(name, map) \
0216     { __ATTR(name, 0444, show_cpus_attr, NULL), map }
0217
0218 /* Keep in sync with cpu_subsys_attrs */
0219 static struct cpu_attr cpu_attrs[] = {
0220     _CPU_ATTR(online, &__cpu_online_mask),
0221     _CPU_ATTR(possible, &__cpu_possible_mask),
0222     _CPU_ATTR(present, &__cpu_present_mask),
0223 };
0224
0225 /*
0226  * Print values for NR_CPUS and offlined cpus
0227  */
0228 static ssize_t print_cpus_kernel_max(struct device *dev,
0229                      struct device_attribute *attr, char *buf)
0230 {
0231     return sysfs_emit(buf, "%d\n", NR_CPUS - 1);
0232 }
0233 static DEVICE_ATTR(kernel_max, 0444, print_cpus_kernel_max, NULL);
0234
0235 /* arch-optional setting to enable display of offline cpus >= nr_cpu_ids */
0236 unsigned int total_cpus;
0237
0238 static ssize_t print_cpus_offline(struct device *dev,
0239                   struct device_attribute *attr, char *buf)
0240 {
0241     int len = 0;
0242     cpumask_var_t offline;
0243
0244     /* display offline cpus < nr_cpu_ids */
0245     if (!alloc_cpumask_var(&offline, GFP_KERNEL))
0246         return -ENOMEM;
0247     cpumask_andnot(offline, cpu_possible_mask, cpu_online_mask);
0248     len += sysfs_emit_at(buf, len, "%*pbl", cpumask_pr_args(offline));
0249     free_cpumask_var(offline);
0250
0251     /* display offline cpus >= nr_cpu_ids */
0252     if (total_cpus && nr_cpu_ids < total_cpus) {
0253         len += sysfs_emit_at(buf, len, ",");
0254
0255         if (nr_cpu_ids == total_cpus-1)
0256             len += sysfs_emit_at(buf, len, "%u", nr_cpu_ids);
0257         else
0258             len += sysfs_emit_at(buf, len, "%u-%d",
0259                          nr_cpu_ids, total_cpus - 1);
0260     }
0261
0262     len += sysfs_emit_at(buf, len, "\n");
0263
0264     return len;
0265 }
0266 static DEVICE_ATTR(offline, 0444, print_cpus_offline, NULL);
0267
0268 static ssize_t print_cpus_isolated(struct device *dev,
0269                   struct device_attribute *attr, char *buf)
0270 {
0271     int len;
0272     cpumask_var_t isolated;
0273
0274     if (!alloc_cpumask_var(&isolated, GFP_KERNEL))
0275         return -ENOMEM;
0276
0277     cpumask_andnot(isolated, cpu_possible_mask,
0278                housekeeping_cpumask(HK_TYPE_DOMAIN));
0279     len = sysfs_emit(buf, "%*pbl\n", cpumask_pr_args(isolated));
0280
0281     free_cpumask_var(isolated);
0282
0283     return len;
0284 }
0285 static DEVICE_ATTR(isolated, 0444, print_cpus_isolated, NULL);
0286
0287 #ifdef CONFIG_NO_HZ_FULL
0288 static ssize_t print_cpus_nohz_full(struct device *dev,
0289                     struct device_attribute *attr, char *buf)
0290 {
0291     return sysfs_emit(buf, "%*pbl\n", cpumask_pr_args(tick_nohz_full_mask));
0292 }
0293 static DEVICE_ATTR(nohz_full, 0444, print_cpus_nohz_full, NULL);
0294 #endif
0295
0296 static void cpu_device_release(struct device *dev)
0297 {
0298     /*
0299      * This is an empty function to prevent the driver core from spitting a
0300      * warning at us.  Yes, I know this is directly opposite of what the
0301      * documentation for the driver core and kobjects say, and the author
0302      * of this code has already been publically ridiculed for doing
0303      * something as foolish as this.  However, at this point in time, it is
0304      * the only way to handle the issue of statically allocated cpu
0305      * devices.  The different architectures will have their cpu device
0306      * code reworked to properly handle this in the near future, so this
0307      * function will then be changed to correctly free up the memory held
0308      * by the cpu device.
0309      *
0310      * Never copy this way of doing things, or you too will be made fun of
0311      * on the linux-kernel list, you have been warned.
0312      */
0313 }
0314
0315 #ifdef CONFIG_GENERIC_CPU_AUTOPROBE
0316 static ssize_t print_cpu_modalias(struct device *dev,
0317                   struct device_attribute *attr,
0318                   char *buf)
0319 {
0320     int len = 0;
0321     u32 i;
0322
0323     len += sysfs_emit_at(buf, len,
0324                  "cpu:type:" CPU_FEATURE_TYPEFMT ":feature:",
0325                  CPU_FEATURE_TYPEVAL);
0326
0327     for (i = 0; i < MAX_CPU_FEATURES; i++)
0328         if (cpu_have_feature(i)) {
0329             if (len + sizeof(",XXXX\n") >= PAGE_SIZE) {
0330                 WARN(1, "CPU features overflow page\n");
0331                 break;
0332             }
0333             len += sysfs_emit_at(buf, len, ",%04X", i);
0334         }
0335     len += sysfs_emit_at(buf, len, "\n");
0336     return len;
0337 }
0338
0339 static int cpu_uevent(struct device *dev, struct kobj_uevent_env *env)
0340 {
0341     char *buf = kzalloc(PAGE_SIZE, GFP_KERNEL);
0342     if (buf) {
0343         print_cpu_modalias(NULL, NULL, buf);
0344         add_uevent_var(env, "MODALIAS=%s", buf);
0345         kfree(buf);
0346     }
0347     return 0;
0348 }
0349 #endif
0350
0351 /*
0352  * register_cpu - Setup a sysfs device for a CPU.
0353  * @cpu - cpu->hotpluggable field set to 1 will generate a control file in
0354  *    sysfs for this CPU.
0355  * @num - CPU number to use when creating the device.
0356  *
0357  * Initialize and register the CPU device.
0358  */
0359 int register_cpu(struct cpu *cpu, int num)
0360 {
0361     int error;
0362
0363     cpu->node_id = cpu_to_node(num);
0364     memset(&cpu->dev, 0x00, sizeof(struct device));
0365     cpu->dev.id = num;
0366     cpu->dev.bus = &cpu_subsys;
0367     cpu->dev.release = cpu_device_release;
0368     cpu->dev.offline_disabled = !cpu->hotpluggable;
0369     cpu->dev.offline = !cpu_online(num);
0370     cpu->dev.of_node = of_get_cpu_node(num, NULL);
0371 #ifdef CONFIG_GENERIC_CPU_AUTOPROBE
0372     cpu->dev.bus->uevent = cpu_uevent;
0373 #endif
0374     cpu->dev.groups = common_cpu_attr_groups;
0375     if (cpu->hotpluggable)
0376         cpu->dev.groups = hotplugable_cpu_attr_groups;
0377     error = device_register(&cpu->dev);
0378     if (error) {
0379         put_device(&cpu->dev);
0380         return error;
0381     }
0382
0383     per_cpu(cpu_sys_devices, num) = &cpu->dev;
0384     register_cpu_under_node(num, cpu_to_node(num));
0385     dev_pm_qos_expose_latency_limit(&cpu->dev,
0386                     PM_QOS_RESUME_LATENCY_NO_CONSTRAINT);
0387
0388     return 0;
0389 }
0390
0391 struct device *get_cpu_device(unsigned int cpu)
0392 {
0393     if (cpu < nr_cpu_ids && cpu_possible(cpu))
0394         return per_cpu(cpu_sys_devices, cpu);
0395     else
0396         return NULL;
0397 }
0398 EXPORT_SYMBOL_GPL(get_cpu_device);
0399
0400 static void device_create_release(struct device *dev)
0401 {
0402     kfree(dev);
0403 }
0404
0405 __printf(4, 0)
0406 static struct device *
0407 __cpu_device_create(struct device *parent, void *drvdata,
0408             const struct attribute_group **groups,
0409             const char *fmt, va_list args)
0410 {
0411     struct device *dev = NULL;
0412     int retval = -ENOMEM;
0413
0414     dev = kzalloc(sizeof(*dev), GFP_KERNEL);
0415     if (!dev)
0416         goto error;
0417
0418     device_initialize(dev);
0419     dev->parent = parent;
0420     dev->groups = groups;
0421     dev->release = device_create_release;
0422     device_set_pm_not_required(dev);
0423     dev_set_drvdata(dev, drvdata);
0424
0425     retval = kobject_set_name_vargs(&dev->kobj, fmt, args);
0426     if (retval)
0427         goto error;
0428
0429     retval = device_add(dev);
0430     if (retval)
0431         goto error;
0432
0433     return dev;
0434
0435 error:
0436     put_device(dev);
0437     return ERR_PTR(retval);
0438 }
0439
0440 struct device *cpu_device_create(struct device *parent, void *drvdata,
0441                  const struct attribute_group **groups,
0442                  const char *fmt, ...)
0443 {
0444     va_list vargs;
0445     struct device *dev;
0446
0447     va_start(vargs, fmt);
0448     dev = __cpu_device_create(parent, drvdata, groups, fmt, vargs);
0449     va_end(vargs);
0450     return dev;
0451 }
0452 EXPORT_SYMBOL_GPL(cpu_device_create);
0453
0454 #ifdef CONFIG_GENERIC_CPU_AUTOPROBE
0455 static DEVICE_ATTR(modalias, 0444, print_cpu_modalias, NULL);
0456 #endif
0457
0458 static struct attribute *cpu_root_attrs[] = {
0459 #ifdef CONFIG_ARCH_CPU_PROBE_RELEASE
0460     &dev_attr_probe.attr,
0461     &dev_attr_release.attr,
0462 #endif
0463     &cpu_attrs[0].attr.attr,
0464     &cpu_attrs[1].attr.attr,
0465     &cpu_attrs[2].attr.attr,
0466     &dev_attr_kernel_max.attr,
0467     &dev_attr_offline.attr,
0468     &dev_attr_isolated.attr,
0469 #ifdef CONFIG_NO_HZ_FULL
0470     &dev_attr_nohz_full.attr,
0471 #endif
0472 #ifdef CONFIG_GENERIC_CPU_AUTOPROBE
0473     &dev_attr_modalias.attr,
0474 #endif
0475     NULL
0476 };
0477
0478 static const struct attribute_group cpu_root_attr_group = {
0479     .attrs = cpu_root_attrs,
0480 };
0481
0482 static const struct attribute_group *cpu_root_attr_groups[] = {
0483     &cpu_root_attr_group,
0484     NULL,
0485 };
0486
0487 bool cpu_is_hotpluggable(unsigned int cpu)
0488 {
0489     struct device *dev = get_cpu_device(cpu);
0490     return dev && container_of(dev, struct cpu, dev)->hotpluggable;
0491 }
0492 EXPORT_SYMBOL_GPL(cpu_is_hotpluggable);
0493
0494 #ifdef CONFIG_GENERIC_CPU_DEVICES
0495 static DEFINE_PER_CPU(struct cpu, cpu_devices);
0496 #endif
0497
0498 static void __init cpu_dev_register_generic(void)
0499 {
0500 #ifdef CONFIG_GENERIC_CPU_DEVICES
0501     int i;
0502
0503     for_each_possible_cpu(i) {
0504         if (register_cpu(&per_cpu(cpu_devices, i), i))
0505             panic("Failed to register CPU device");
0506     }
0507 #endif
0508 }
0509
0510 #ifdef CONFIG_GENERIC_CPU_VULNERABILITIES
0511
0512 ssize_t __weak cpu_show_meltdown(struct device *dev,
0513                  struct device_attribute *attr, char *buf)
0514 {
0515     return sysfs_emit(buf, "Not affected\n");
0516 }
0517
0518 ssize_t __weak cpu_show_spectre_v1(struct device *dev,
0519                    struct device_attribute *attr, char *buf)
0520 {
0521     return sysfs_emit(buf, "Not affected\n");
0522 }
0523
0524 ssize_t __weak cpu_show_spectre_v2(struct device *dev,
0525                    struct device_attribute *attr, char *buf)
0526 {
0527     return sysfs_emit(buf, "Not affected\n");
0528 }
0529
0530 ssize_t __weak cpu_show_spec_store_bypass(struct device *dev,
0531                       struct device_attribute *attr, char *buf)
0532 {
0533     return sysfs_emit(buf, "Not affected\n");
0534 }
0535
0536 ssize_t __weak cpu_show_l1tf(struct device *dev,
0537                  struct device_attribute *attr, char *buf)
0538 {
0539     return sysfs_emit(buf, "Not affected\n");
0540 }
0541
0542 ssize_t __weak cpu_show_mds(struct device *dev,
0543                 struct device_attribute *attr, char *buf)
0544 {
0545     return sysfs_emit(buf, "Not affected\n");
0546 }
0547
0548 ssize_t __weak cpu_show_tsx_async_abort(struct device *dev,
0549                     struct device_attribute *attr,
0550                     char *buf)
0551 {
0552     return sysfs_emit(buf, "Not affected\n");
0553 }
0554
0555 ssize_t __weak cpu_show_itlb_multihit(struct device *dev,
0556                       struct device_attribute *attr, char *buf)
0557 {
0558     return sysfs_emit(buf, "Not affected\n");
0559 }
0560
0561 ssize_t __weak cpu_show_srbds(struct device *dev,
0562                   struct device_attribute *attr, char *buf)
0563 {
0564     return sysfs_emit(buf, "Not affected\n");
0565 }
0566
0567 ssize_t __weak cpu_show_mmio_stale_data(struct device *dev,
0568                     struct device_attribute *attr, char *buf)
0569 {
0570     return sysfs_emit(buf, "Not affected\n");
0571 }
0572
0573 ssize_t __weak cpu_show_retbleed(struct device *dev,
0574                  struct device_attribute *attr, char *buf)
0575 {
0576     return sysfs_emit(buf, "Not affected\n");
0577 }
0578
0579 static DEVICE_ATTR(meltdown, 0444, cpu_show_meltdown, NULL);
0580 static DEVICE_ATTR(spectre_v1, 0444, cpu_show_spectre_v1, NULL);
0581 static DEVICE_ATTR(spectre_v2, 0444, cpu_show_spectre_v2, NULL);
0582 static DEVICE_ATTR(spec_store_bypass, 0444, cpu_show_spec_store_bypass, NULL);
0583 static DEVICE_ATTR(l1tf, 0444, cpu_show_l1tf, NULL);
0584 static DEVICE_ATTR(mds, 0444, cpu_show_mds, NULL);
0585 static DEVICE_ATTR(tsx_async_abort, 0444, cpu_show_tsx_async_abort, NULL);
0586 static DEVICE_ATTR(itlb_multihit, 0444, cpu_show_itlb_multihit, NULL);
0587 static DEVICE_ATTR(srbds, 0444, cpu_show_srbds, NULL);
0588 static DEVICE_ATTR(mmio_stale_data, 0444, cpu_show_mmio_stale_data, NULL);
0589 static DEVICE_ATTR(retbleed, 0444, cpu_show_retbleed, NULL);
0590
0591 static struct attribute *cpu_root_vulnerabilities_attrs[] = {
0592     &dev_attr_meltdown.attr,
0593     &dev_attr_spectre_v1.attr,
0594     &dev_attr_spectre_v2.attr,
0595     &dev_attr_spec_store_bypass.attr,
0596     &dev_attr_l1tf.attr,
0597     &dev_attr_mds.attr,
0598     &dev_attr_tsx_async_abort.attr,
0599     &dev_attr_itlb_multihit.attr,
0600     &dev_attr_srbds.attr,
0601     &dev_attr_mmio_stale_data.attr,
0602     &dev_attr_retbleed.attr,
0603     NULL
0604 };
0605
0606 static const struct attribute_group cpu_root_vulnerabilities_group = {
0607     .name  = "vulnerabilities",
0608     .attrs = cpu_root_vulnerabilities_attrs,
0609 };
0610
0611 static void __init cpu_register_vulnerabilities(void)
0612 {
0613     if (sysfs_create_group(&cpu_subsys.dev_root->kobj,
0614                    &cpu_root_vulnerabilities_group))
0615         pr_err("Unable to register CPU vulnerabilities\n");
0616 }
0617
0618 #else
0619 static inline void cpu_register_vulnerabilities(void) { }
0620 #endif
0621
0622 void __init cpu_dev_init(void)
0623 {
0624     if (subsys_system_register(&cpu_subsys, cpu_root_attr_groups))
0625         panic("Failed to register CPU subsystem");
0626
0627     cpu_dev_register_generic();
0628     cpu_register_vulnerabilities();
0629 }