x86/power/cpu.c

0001 // SPDX-License-Identifier: GPL-2.0-only
0002 /*
0003  * Suspend support specific for i386/x86-64.
0004  *
0005  * Copyright (c) 2007 Rafael J. Wysocki <rjw@sisk.pl>
0006  * Copyright (c) 2002 Pavel Machek <pavel@ucw.cz>
0007  * Copyright (c) 2001 Patrick Mochel <mochel@osdl.org>
0008  */
0009
0010 #include <linux/suspend.h>
0011 #include <linux/export.h>
0012 #include <linux/smp.h>
0013 #include <linux/perf_event.h>
0014 #include <linux/tboot.h>
0015 #include <linux/dmi.h>
0016 #include <linux/pgtable.h>
0017
0018 #include <asm/proto.h>
0019 #include <asm/mtrr.h>
0020 #include <asm/page.h>
0021 #include <asm/mce.h>
0022 #include <asm/suspend.h>
0023 #include <asm/fpu/api.h>
0024 #include <asm/debugreg.h>
0025 #include <asm/cpu.h>
0026 #include <asm/mmu_context.h>
0027 #include <asm/cpu_device_id.h>
0028 #include <asm/microcode.h>
0029
0030 #ifdef CONFIG_X86_32
0031 __visible unsigned long saved_context_ebx;
0032 __visible unsigned long saved_context_esp, saved_context_ebp;
0033 __visible unsigned long saved_context_esi, saved_context_edi;
0034 __visible unsigned long saved_context_eflags;
0035 #endif
0036 struct saved_context saved_context;
0037
0038 static void msr_save_context(struct saved_context *ctxt)
0039 {
0040     struct saved_msr *msr = ctxt->saved_msrs.array;
0041     struct saved_msr *end = msr + ctxt->saved_msrs.num;
0042
0043     while (msr < end) {
0044         if (msr->valid)
0045             rdmsrl(msr->info.msr_no, msr->info.reg.q);
0046         msr++;
0047     }
0048 }
0049
0050 static void msr_restore_context(struct saved_context *ctxt)
0051 {
0052     struct saved_msr *msr = ctxt->saved_msrs.array;
0053     struct saved_msr *end = msr + ctxt->saved_msrs.num;
0054
0055     while (msr < end) {
0056         if (msr->valid)
0057             wrmsrl(msr->info.msr_no, msr->info.reg.q);
0058         msr++;
0059     }
0060 }
0061
0062 /**
0063  * __save_processor_state() - Save CPU registers before creating a
0064  *                             hibernation image and before restoring
0065  *                             the memory state from it
0066  * @ctxt: Structure to store the registers contents in.
0067  *
0068  * NOTE: If there is a CPU register the modification of which by the
0069  * boot kernel (ie. the kernel used for loading the hibernation image)
0070  * might affect the operations of the restored target kernel (ie. the one
0071  * saved in the hibernation image), then its contents must be saved by this
0072  * function.  In other words, if kernel A is hibernated and different
0073  * kernel B is used for loading the hibernation image into memory, the
0074  * kernel A's __save_processor_state() function must save all registers
0075  * needed by kernel A, so that it can operate correctly after the resume
0076  * regardless of what kernel B does in the meantime.
0077  */
0078 static void __save_processor_state(struct saved_context *ctxt)
0079 {
0080 #ifdef CONFIG_X86_32
0081     mtrr_save_fixed_ranges(NULL);
0082 #endif
0083     kernel_fpu_begin();
0084
0085     /*
0086      * descriptor tables
0087      */
0088     store_idt(&ctxt->idt);
0089
0090     /*
0091      * We save it here, but restore it only in the hibernate case.
0092      * For ACPI S3 resume, this is loaded via 'early_gdt_desc' in 64-bit
0093      * mode in "secondary_startup_64". In 32-bit mode it is done via
0094      * 'pmode_gdt' in wakeup_start.
0095      */
0096     ctxt->gdt_desc.size = GDT_SIZE - 1;
0097     ctxt->gdt_desc.address = (unsigned long)get_cpu_gdt_rw(smp_processor_id());
0098
0099     store_tr(ctxt->tr);
0100
0101     /* XMM0..XMM15 should be handled by kernel_fpu_begin(). */
0102     /*
0103      * segment registers
0104      */
0105     savesegment(gs, ctxt->gs);
0106 #ifdef CONFIG_X86_64
0107     savesegment(fs, ctxt->fs);
0108     savesegment(ds, ctxt->ds);
0109     savesegment(es, ctxt->es);
0110
0111     rdmsrl(MSR_FS_BASE, ctxt->fs_base);
0112     rdmsrl(MSR_GS_BASE, ctxt->kernelmode_gs_base);
0113     rdmsrl(MSR_KERNEL_GS_BASE, ctxt->usermode_gs_base);
0114     mtrr_save_fixed_ranges(NULL);
0115
0116     rdmsrl(MSR_EFER, ctxt->efer);
0117 #endif
0118
0119     /*
0120      * control registers
0121      */
0122     ctxt->cr0 = read_cr0();
0123     ctxt->cr2 = read_cr2();
0124     ctxt->cr3 = __read_cr3();
0125     ctxt->cr4 = __read_cr4();
0126     ctxt->misc_enable_saved = !rdmsrl_safe(MSR_IA32_MISC_ENABLE,
0127                            &ctxt->misc_enable);
0128     msr_save_context(ctxt);
0129 }
0130
0131 /* Needed by apm.c */
0132 void save_processor_state(void)
0133 {
0134     __save_processor_state(&saved_context);
0135     x86_platform.save_sched_clock_state();
0136 }
0137 #ifdef CONFIG_X86_32
0138 EXPORT_SYMBOL(save_processor_state);
0139 #endif
0140
0141 static void do_fpu_end(void)
0142 {
0143     /*
0144      * Restore FPU regs if necessary.
0145      */
0146     kernel_fpu_end();
0147 }
0148
0149 static void fix_processor_context(void)
0150 {
0151     int cpu = smp_processor_id();
0152 #ifdef CONFIG_X86_64
0153     struct desc_struct *desc = get_cpu_gdt_rw(cpu);
0154     tss_desc tss;
0155 #endif
0156
0157     /*
0158      * We need to reload TR, which requires that we change the
0159      * GDT entry to indicate "available" first.
0160      *
0161      * XXX: This could probably all be replaced by a call to
0162      * force_reload_TR().
0163      */
0164     set_tss_desc(cpu, &get_cpu_entry_area(cpu)->tss.x86_tss);
0165
0166 #ifdef CONFIG_X86_64
0167     memcpy(&tss, &desc[GDT_ENTRY_TSS], sizeof(tss_desc));
0168     tss.type = 0x9; /* The available 64-bit TSS (see AMD vol 2, pg 91 */
0169     write_gdt_entry(desc, GDT_ENTRY_TSS, &tss, DESC_TSS);
0170
0171     syscall_init();             /* This sets MSR_*STAR and related */
0172 #else
0173     if (boot_cpu_has(X86_FEATURE_SEP))
0174         enable_sep_cpu();
0175 #endif
0176     load_TR_desc();             /* This does ltr */
0177     load_mm_ldt(current->active_mm);    /* This does lldt */
0178     initialize_tlbstate_and_flush();
0179
0180     fpu__resume_cpu();
0181
0182     /* The processor is back on the direct GDT, load back the fixmap */
0183     load_fixmap_gdt(cpu);
0184 }
0185
0186 /**
0187  * __restore_processor_state() - Restore the contents of CPU registers saved
0188  *                               by __save_processor_state()
0189  * @ctxt: Structure to load the registers contents from.
0190  *
0191  * The asm code that gets us here will have restored a usable GDT, although
0192  * it will be pointing to the wrong alias.
0193  */
0194 static void notrace __restore_processor_state(struct saved_context *ctxt)
0195 {
0196     struct cpuinfo_x86 *c;
0197
0198     if (ctxt->misc_enable_saved)
0199         wrmsrl(MSR_IA32_MISC_ENABLE, ctxt->misc_enable);
0200     /*
0201      * control registers
0202      */
0203     /* cr4 was introduced in the Pentium CPU */
0204 #ifdef CONFIG_X86_32
0205     if (ctxt->cr4)
0206         __write_cr4(ctxt->cr4);
0207 #else
0208 /* CONFIG X86_64 */
0209     wrmsrl(MSR_EFER, ctxt->efer);
0210     __write_cr4(ctxt->cr4);
0211 #endif
0212     write_cr3(ctxt->cr3);
0213     write_cr2(ctxt->cr2);
0214     write_cr0(ctxt->cr0);
0215
0216     /* Restore the IDT. */
0217     load_idt(&ctxt->idt);
0218
0219     /*
0220      * Just in case the asm code got us here with the SS, DS, or ES
0221      * out of sync with the GDT, update them.
0222      */
0223     loadsegment(ss, __KERNEL_DS);
0224     loadsegment(ds, __USER_DS);
0225     loadsegment(es, __USER_DS);
0226
0227     /*
0228      * Restore percpu access.  Percpu access can happen in exception
0229      * handlers or in complicated helpers like load_gs_index().
0230      */
0231 #ifdef CONFIG_X86_64
0232     wrmsrl(MSR_GS_BASE, ctxt->kernelmode_gs_base);
0233 #else
0234     loadsegment(fs, __KERNEL_PERCPU);
0235 #endif
0236
0237     /* Restore the TSS, RO GDT, LDT, and usermode-relevant MSRs. */
0238     fix_processor_context();
0239
0240     /*
0241      * Now that we have descriptor tables fully restored and working
0242      * exception handling, restore the usermode segments.
0243      */
0244 #ifdef CONFIG_X86_64
0245     loadsegment(ds, ctxt->es);
0246     loadsegment(es, ctxt->es);
0247     loadsegment(fs, ctxt->fs);
0248     load_gs_index(ctxt->gs);
0249
0250     /*
0251      * Restore FSBASE and GSBASE after restoring the selectors, since
0252      * restoring the selectors clobbers the bases.  Keep in mind
0253      * that MSR_KERNEL_GS_BASE is horribly misnamed.
0254      */
0255     wrmsrl(MSR_FS_BASE, ctxt->fs_base);
0256     wrmsrl(MSR_KERNEL_GS_BASE, ctxt->usermode_gs_base);
0257 #else
0258     loadsegment(gs, ctxt->gs);
0259 #endif
0260
0261     do_fpu_end();
0262     tsc_verify_tsc_adjust(true);
0263     x86_platform.restore_sched_clock_state();
0264     mtrr_bp_restore();
0265     perf_restore_debug_store();
0266
0267     c = &cpu_data(smp_processor_id());
0268     if (cpu_has(c, X86_FEATURE_MSR_IA32_FEAT_CTL))
0269         init_ia32_feat_ctl(c);
0270
0271     microcode_bsp_resume();
0272
0273     /*
0274      * This needs to happen after the microcode has been updated upon resume
0275      * because some of the MSRs are "emulated" in microcode.
0276      */
0277     msr_restore_context(ctxt);
0278 }
0279
0280 /* Needed by apm.c */
0281 void notrace restore_processor_state(void)
0282 {
0283     __restore_processor_state(&saved_context);
0284 }
0285 #ifdef CONFIG_X86_32
0286 EXPORT_SYMBOL(restore_processor_state);
0287 #endif
0288
0289 #if defined(CONFIG_HIBERNATION) && defined(CONFIG_HOTPLUG_CPU)
0290 static void resume_play_dead(void)
0291 {
0292     play_dead_common();
0293     tboot_shutdown(TB_SHUTDOWN_WFS);
0294     hlt_play_dead();
0295 }
0296
0297 int hibernate_resume_nonboot_cpu_disable(void)
0298 {
0299     void (*play_dead)(void) = smp_ops.play_dead;
0300     int ret;
0301
0302     /*
0303      * Ensure that MONITOR/MWAIT will not be used in the "play dead" loop
0304      * during hibernate image restoration, because it is likely that the
0305      * monitored address will be actually written to at that time and then
0306      * the "dead" CPU will attempt to execute instructions again, but the
0307      * address in its instruction pointer may not be possible to resolve
0308      * any more at that point (the page tables used by it previously may
0309      * have been overwritten by hibernate image data).
0310      *
0311      * First, make sure that we wake up all the potentially disabled SMT
0312      * threads which have been initially brought up and then put into
0313      * mwait/cpuidle sleep.
0314      * Those will be put to proper (not interfering with hibernation
0315      * resume) sleep afterwards, and the resumed kernel will decide itself
0316      * what to do with them.
0317      */
0318     ret = cpuhp_smt_enable();
0319     if (ret)
0320         return ret;
0321     smp_ops.play_dead = resume_play_dead;
0322     ret = freeze_secondary_cpus(0);
0323     smp_ops.play_dead = play_dead;
0324     return ret;
0325 }
0326 #endif
0327
0328 /*
0329  * When bsp_check() is called in hibernate and suspend, cpu hotplug
0330  * is disabled already. So it's unnecessary to handle race condition between
0331  * cpumask query and cpu hotplug.
0332  */
0333 static int bsp_check(void)
0334 {
0335     if (cpumask_first(cpu_online_mask) != 0) {
0336         pr_warn("CPU0 is offline.\n");
0337         return -ENODEV;
0338     }
0339
0340     return 0;
0341 }
0342
0343 static int bsp_pm_callback(struct notifier_block *nb, unsigned long action,
0344                void *ptr)
0345 {
0346     int ret = 0;
0347
0348     switch (action) {
0349     case PM_SUSPEND_PREPARE:
0350     case PM_HIBERNATION_PREPARE:
0351         ret = bsp_check();
0352         break;
0353 #ifdef CONFIG_DEBUG_HOTPLUG_CPU0
0354     case PM_RESTORE_PREPARE:
0355         /*
0356          * When system resumes from hibernation, online CPU0 because
0357          * 1. it's required for resume and
0358          * 2. the CPU was online before hibernation
0359          */
0360         if (!cpu_online(0))
0361             _debug_hotplug_cpu(0, 1);
0362         break;
0363     case PM_POST_RESTORE:
0364         /*
0365          * When a resume really happens, this code won't be called.
0366          *
0367          * This code is called only when user space hibernation software
0368          * prepares for snapshot device during boot time. So we just
0369          * call _debug_hotplug_cpu() to restore to CPU0's state prior to
0370          * preparing the snapshot device.
0371          *
0372          * This works for normal boot case in our CPU0 hotplug debug
0373          * mode, i.e. CPU0 is offline and user mode hibernation
0374          * software initializes during boot time.
0375          *
0376          * If CPU0 is online and user application accesses snapshot
0377          * device after boot time, this will offline CPU0 and user may
0378          * see different CPU0 state before and after accessing
0379          * the snapshot device. But hopefully this is not a case when
0380          * user debugging CPU0 hotplug. Even if users hit this case,
0381          * they can easily online CPU0 back.
0382          *
0383          * To simplify this debug code, we only consider normal boot
0384          * case. Otherwise we need to remember CPU0's state and restore
0385          * to that state and resolve racy conditions etc.
0386          */
0387         _debug_hotplug_cpu(0, 0);
0388         break;
0389 #endif
0390     default:
0391         break;
0392     }
0393     return notifier_from_errno(ret);
0394 }
0395
0396 static int __init bsp_pm_check_init(void)
0397 {
0398     /*
0399      * Set this bsp_pm_callback as lower priority than
0400      * cpu_hotplug_pm_callback. So cpu_hotplug_pm_callback will be called
0401      * earlier to disable cpu hotplug before bsp online check.
0402      */
0403     pm_notifier(bsp_pm_callback, -INT_MAX);
0404     return 0;
0405 }
0406
0407 core_initcall(bsp_pm_check_init);
0408
0409 static int msr_build_context(const u32 *msr_id, const int num)
0410 {
0411     struct saved_msrs *saved_msrs = &saved_context.saved_msrs;
0412     struct saved_msr *msr_array;
0413     int total_num;
0414     int i, j;
0415
0416     total_num = saved_msrs->num + num;
0417
0418     msr_array = kmalloc_array(total_num, sizeof(struct saved_msr), GFP_KERNEL);
0419     if (!msr_array) {
0420         pr_err("x86/pm: Can not allocate memory to save/restore MSRs during suspend.\n");
0421         return -ENOMEM;
0422     }
0423
0424     if (saved_msrs->array) {
0425         /*
0426          * Multiple callbacks can invoke this function, so copy any
0427          * MSR save requests from previous invocations.
0428          */
0429         memcpy(msr_array, saved_msrs->array,
0430                sizeof(struct saved_msr) * saved_msrs->num);
0431
0432         kfree(saved_msrs->array);
0433     }
0434
0435     for (i = saved_msrs->num, j = 0; i < total_num; i++, j++) {
0436         u64 dummy;
0437
0438         msr_array[i].info.msr_no    = msr_id[j];
0439         msr_array[i].valid      = !rdmsrl_safe(msr_id[j], &dummy);
0440         msr_array[i].info.reg.q     = 0;
0441     }
0442     saved_msrs->num   = total_num;
0443     saved_msrs->array = msr_array;
0444
0445     return 0;
0446 }
0447
0448 /*
0449  * The following sections are a quirk framework for problematic BIOSen:
0450  * Sometimes MSRs are modified by the BIOSen after suspended to
0451  * RAM, this might cause unexpected behavior after wakeup.
0452  * Thus we save/restore these specified MSRs across suspend/resume
0453  * in order to work around it.
0454  *
0455  * For any further problematic BIOSen/platforms,
0456  * please add your own function similar to msr_initialize_bdw.
0457  */
0458 static int msr_initialize_bdw(const struct dmi_system_id *d)
0459 {
0460     /* Add any extra MSR ids into this array. */
0461     u32 bdw_msr_id[] = { MSR_IA32_THERM_CONTROL };
0462
0463     pr_info("x86/pm: %s detected, MSR saving is needed during suspending.\n", d->ident);
0464     return msr_build_context(bdw_msr_id, ARRAY_SIZE(bdw_msr_id));
0465 }
0466
0467 static const struct dmi_system_id msr_save_dmi_table[] = {
0468     {
0469      .callback = msr_initialize_bdw,
0470      .ident = "BROADWELL BDX_EP",
0471      .matches = {
0472         DMI_MATCH(DMI_PRODUCT_NAME, "GRANTLEY"),
0473         DMI_MATCH(DMI_PRODUCT_VERSION, "E63448-400"),
0474         },
0475     },
0476     {}
0477 };
0478
0479 static int msr_save_cpuid_features(const struct x86_cpu_id *c)
0480 {
0481     u32 cpuid_msr_id[] = {
0482         MSR_AMD64_CPUID_FN_1,
0483     };
0484
0485     pr_info("x86/pm: family %#hx cpu detected, MSR saving is needed during suspending.\n",
0486         c->family);
0487
0488     return msr_build_context(cpuid_msr_id, ARRAY_SIZE(cpuid_msr_id));
0489 }
0490
0491 static const struct x86_cpu_id msr_save_cpu_table[] = {
0492     X86_MATCH_VENDOR_FAM(AMD, 0x15, &msr_save_cpuid_features),
0493     X86_MATCH_VENDOR_FAM(AMD, 0x16, &msr_save_cpuid_features),
0494     {}
0495 };
0496
0497 typedef int (*pm_cpu_match_t)(const struct x86_cpu_id *);
0498 static int pm_cpu_check(const struct x86_cpu_id *c)
0499 {
0500     const struct x86_cpu_id *m;
0501     int ret = 0;
0502
0503     m = x86_match_cpu(msr_save_cpu_table);
0504     if (m) {
0505         pm_cpu_match_t fn;
0506
0507         fn = (pm_cpu_match_t)m->driver_data;
0508         ret = fn(m);
0509     }
0510
0511     return ret;
0512 }
0513
0514 static void pm_save_spec_msr(void)
0515 {
0516     u32 spec_msr_id[] = {
0517         MSR_IA32_SPEC_CTRL,
0518         MSR_IA32_TSX_CTRL,
0519         MSR_TSX_FORCE_ABORT,
0520         MSR_IA32_MCU_OPT_CTRL,
0521         MSR_AMD64_LS_CFG,
0522     };
0523
0524     msr_build_context(spec_msr_id, ARRAY_SIZE(spec_msr_id));
0525 }
0526
0527 static int pm_check_save_msr(void)
0528 {
0529     dmi_check_system(msr_save_dmi_table);
0530     pm_cpu_check(msr_save_cpu_table);
0531     pm_save_spec_msr();
0532
0533     return 0;
0534 }
0535
0536 device_initcall(pm_check_save_msr);