kernel/fpu/init.c

0001 // SPDX-License-Identifier: GPL-2.0-only
0002 /*
0003  * x86 FPU boot time init code:
0004  */
0005 #include <asm/fpu/api.h>
0006 #include <asm/tlbflush.h>
0007 #include <asm/setup.h>
0008
0009 #include <linux/sched.h>
0010 #include <linux/sched/task.h>
0011 #include <linux/init.h>
0012
0013 #include "internal.h"
0014 #include "legacy.h"
0015 #include "xstate.h"
0016
0017 /*
0018  * Initialize the registers found in all CPUs, CR0 and CR4:
0019  */
0020 static void fpu__init_cpu_generic(void)
0021 {
0022     unsigned long cr0;
0023     unsigned long cr4_mask = 0;
0024
0025     if (boot_cpu_has(X86_FEATURE_FXSR))
0026         cr4_mask |= X86_CR4_OSFXSR;
0027     if (boot_cpu_has(X86_FEATURE_XMM))
0028         cr4_mask |= X86_CR4_OSXMMEXCPT;
0029     if (cr4_mask)
0030         cr4_set_bits(cr4_mask);
0031
0032     cr0 = read_cr0();
0033     cr0 &= ~(X86_CR0_TS|X86_CR0_EM); /* clear TS and EM */
0034     if (!boot_cpu_has(X86_FEATURE_FPU))
0035         cr0 |= X86_CR0_EM;
0036     write_cr0(cr0);
0037
0038     /* Flush out any pending x87 state: */
0039 #ifdef CONFIG_MATH_EMULATION
0040     if (!boot_cpu_has(X86_FEATURE_FPU))
0041         fpstate_init_soft(&current->thread.fpu.fpstate->regs.soft);
0042     else
0043 #endif
0044         asm volatile ("fninit");
0045 }
0046
0047 /*
0048  * Enable all supported FPU features. Called when a CPU is brought online:
0049  */
0050 void fpu__init_cpu(void)
0051 {
0052     fpu__init_cpu_generic();
0053     fpu__init_cpu_xstate();
0054 }
0055
0056 static bool fpu__probe_without_cpuid(void)
0057 {
0058     unsigned long cr0;
0059     u16 fsw, fcw;
0060
0061     fsw = fcw = 0xffff;
0062
0063     cr0 = read_cr0();
0064     cr0 &= ~(X86_CR0_TS | X86_CR0_EM);
0065     write_cr0(cr0);
0066
0067     asm volatile("fninit ; fnstsw %0 ; fnstcw %1" : "+m" (fsw), "+m" (fcw));
0068
0069     pr_info("x86/fpu: Probing for FPU: FSW=0x%04hx FCW=0x%04hx\n", fsw, fcw);
0070
0071     return fsw == 0 && (fcw & 0x103f) == 0x003f;
0072 }
0073
0074 static void fpu__init_system_early_generic(struct cpuinfo_x86 *c)
0075 {
0076     if (!boot_cpu_has(X86_FEATURE_CPUID) &&
0077         !test_bit(X86_FEATURE_FPU, (unsigned long *)cpu_caps_cleared)) {
0078         if (fpu__probe_without_cpuid())
0079             setup_force_cpu_cap(X86_FEATURE_FPU);
0080         else
0081             setup_clear_cpu_cap(X86_FEATURE_FPU);
0082     }
0083
0084 #ifndef CONFIG_MATH_EMULATION
0085     if (!test_cpu_cap(&boot_cpu_data, X86_FEATURE_FPU)) {
0086         pr_emerg("x86/fpu: Giving up, no FPU found and no math emulation present\n");
0087         for (;;)
0088             asm volatile("hlt");
0089     }
0090 #endif
0091 }
0092
0093 /*
0094  * Boot time FPU feature detection code:
0095  */
0096 unsigned int mxcsr_feature_mask __ro_after_init = 0xffffffffu;
0097 EXPORT_SYMBOL_GPL(mxcsr_feature_mask);
0098
0099 static void __init fpu__init_system_mxcsr(void)
0100 {
0101     unsigned int mask = 0;
0102
0103     if (boot_cpu_has(X86_FEATURE_FXSR)) {
0104         /* Static because GCC does not get 16-byte stack alignment right: */
0105         static struct fxregs_state fxregs __initdata;
0106
0107         asm volatile("fxsave %0" : "+m" (fxregs));
0108
0109         mask = fxregs.mxcsr_mask;
0110
0111         /*
0112          * If zero then use the default features mask,
0113          * which has all features set, except the
0114          * denormals-are-zero feature bit:
0115          */
0116         if (mask == 0)
0117             mask = 0x0000ffbf;
0118     }
0119     mxcsr_feature_mask &= mask;
0120 }
0121
0122 /*
0123  * Once per bootup FPU initialization sequences that will run on most x86 CPUs:
0124  */
0125 static void __init fpu__init_system_generic(void)
0126 {
0127     /*
0128      * Set up the legacy init FPU context. Will be updated when the
0129      * CPU supports XSAVE[S].
0130      */
0131     fpstate_init_user(&init_fpstate);
0132
0133     fpu__init_system_mxcsr();
0134 }
0135
0136 /* Get alignment of the TYPE. */
0137 #define TYPE_ALIGN(TYPE) offsetof(struct { char x; TYPE test; }, test)
0138
0139 /*
0140  * Enforce that 'MEMBER' is the last field of 'TYPE'.
0141  *
0142  * Align the computed size with alignment of the TYPE,
0143  * because that's how C aligns structs.
0144  */
0145 #define CHECK_MEMBER_AT_END_OF(TYPE, MEMBER) \
0146     BUILD_BUG_ON(sizeof(TYPE) != ALIGN(offsetofend(TYPE, MEMBER), \
0147                        TYPE_ALIGN(TYPE)))
0148
0149 /*
0150  * We append the 'struct fpu' to the task_struct:
0151  */
0152 static void __init fpu__init_task_struct_size(void)
0153 {
0154     int task_size = sizeof(struct task_struct);
0155
0156     /*
0157      * Subtract off the static size of the register state.
0158      * It potentially has a bunch of padding.
0159      */
0160     task_size -= sizeof(current->thread.fpu.__fpstate.regs);
0161
0162     /*
0163      * Add back the dynamically-calculated register state
0164      * size.
0165      */
0166     task_size += fpu_kernel_cfg.default_size;
0167
0168     /*
0169      * We dynamically size 'struct fpu', so we require that
0170      * it be at the end of 'thread_struct' and that
0171      * 'thread_struct' be at the end of 'task_struct'.  If
0172      * you hit a compile error here, check the structure to
0173      * see if something got added to the end.
0174      */
0175     CHECK_MEMBER_AT_END_OF(struct fpu, __fpstate);
0176     CHECK_MEMBER_AT_END_OF(struct thread_struct, fpu);
0177     CHECK_MEMBER_AT_END_OF(struct task_struct, thread);
0178
0179     arch_task_struct_size = task_size;
0180 }
0181
0182 /*
0183  * Set up the user and kernel xstate sizes based on the legacy FPU context size.
0184  *
0185  * We set this up first, and later it will be overwritten by
0186  * fpu__init_system_xstate() if the CPU knows about xstates.
0187  */
0188 static void __init fpu__init_system_xstate_size_legacy(void)
0189 {
0190     unsigned int size;
0191
0192     /*
0193      * Note that the size configuration might be overwritten later
0194      * during fpu__init_system_xstate().
0195      */
0196     if (!cpu_feature_enabled(X86_FEATURE_FPU)) {
0197         size = sizeof(struct swregs_state);
0198     } else if (cpu_feature_enabled(X86_FEATURE_FXSR)) {
0199         size = sizeof(struct fxregs_state);
0200         fpu_user_cfg.legacy_features = XFEATURE_MASK_FPSSE;
0201     } else {
0202         size = sizeof(struct fregs_state);
0203         fpu_user_cfg.legacy_features = XFEATURE_MASK_FP;
0204     }
0205
0206     fpu_kernel_cfg.max_size = size;
0207     fpu_kernel_cfg.default_size = size;
0208     fpu_user_cfg.max_size = size;
0209     fpu_user_cfg.default_size = size;
0210     fpstate_reset(&current->thread.fpu);
0211 }
0212
0213 static void __init fpu__init_init_fpstate(void)
0214 {
0215     /* Bring init_fpstate size and features up to date */
0216     init_fpstate.size       = fpu_kernel_cfg.max_size;
0217     init_fpstate.xfeatures      = fpu_kernel_cfg.max_features;
0218 }
0219
0220 /*
0221  * Called on the boot CPU once per system bootup, to set up the initial
0222  * FPU state that is later cloned into all processes:
0223  */
0224 void __init fpu__init_system(struct cpuinfo_x86 *c)
0225 {
0226     fpstate_reset(&current->thread.fpu);
0227     fpu__init_system_early_generic(c);
0228
0229     /*
0230      * The FPU has to be operational for some of the
0231      * later FPU init activities:
0232      */
0233     fpu__init_cpu();
0234
0235     fpu__init_system_generic();
0236     fpu__init_system_xstate_size_legacy();
0237     fpu__init_system_xstate(fpu_kernel_cfg.max_size);
0238     fpu__init_task_struct_size();
0239     fpu__init_init_fpstate();
0240 }