kernel/fpu/signal.c

0001 // SPDX-License-Identifier: GPL-2.0
0002 /*
0003  * FPU signal frame handling routines.
0004  */
0005
0006 #include <linux/compat.h>
0007 #include <linux/cpu.h>
0008 #include <linux/pagemap.h>
0009
0010 #include <asm/fpu/signal.h>
0011 #include <asm/fpu/regset.h>
0012 #include <asm/fpu/xstate.h>
0013
0014 #include <asm/sigframe.h>
0015 #include <asm/trapnr.h>
0016 #include <asm/trace/fpu.h>
0017
0018 #include "context.h"
0019 #include "internal.h"
0020 #include "legacy.h"
0021 #include "xstate.h"
0022
0023 /*
0024  * Check for the presence of extended state information in the
0025  * user fpstate pointer in the sigcontext.
0026  */
0027 static inline bool check_xstate_in_sigframe(struct fxregs_state __user *fxbuf,
0028                         struct _fpx_sw_bytes *fx_sw)
0029 {
0030     int min_xstate_size = sizeof(struct fxregs_state) +
0031                   sizeof(struct xstate_header);
0032     void __user *fpstate = fxbuf;
0033     unsigned int magic2;
0034
0035     if (__copy_from_user(fx_sw, &fxbuf->sw_reserved[0], sizeof(*fx_sw)))
0036         return false;
0037
0038     /* Check for the first magic field and other error scenarios. */
0039     if (fx_sw->magic1 != FP_XSTATE_MAGIC1 ||
0040         fx_sw->xstate_size < min_xstate_size ||
0041         fx_sw->xstate_size > current->thread.fpu.fpstate->user_size ||
0042         fx_sw->xstate_size > fx_sw->extended_size)
0043         goto setfx;
0044
0045     /*
0046      * Check for the presence of second magic word at the end of memory
0047      * layout. This detects the case where the user just copied the legacy
0048      * fpstate layout with out copying the extended state information
0049      * in the memory layout.
0050      */
0051     if (__get_user(magic2, (__u32 __user *)(fpstate + fx_sw->xstate_size)))
0052         return false;
0053
0054     if (likely(magic2 == FP_XSTATE_MAGIC2))
0055         return true;
0056 setfx:
0057     trace_x86_fpu_xstate_check_failed(&current->thread.fpu);
0058
0059     /* Set the parameters for fx only state */
0060     fx_sw->magic1 = 0;
0061     fx_sw->xstate_size = sizeof(struct fxregs_state);
0062     fx_sw->xfeatures = XFEATURE_MASK_FPSSE;
0063     return true;
0064 }
0065
0066 /*
0067  * Signal frame handlers.
0068  */
0069 static inline bool save_fsave_header(struct task_struct *tsk, void __user *buf)
0070 {
0071     if (use_fxsr()) {
0072         struct xregs_state *xsave = &tsk->thread.fpu.fpstate->regs.xsave;
0073         struct user_i387_ia32_struct env;
0074         struct _fpstate_32 __user *fp = buf;
0075
0076         fpregs_lock();
0077         if (!test_thread_flag(TIF_NEED_FPU_LOAD))
0078             fxsave(&tsk->thread.fpu.fpstate->regs.fxsave);
0079         fpregs_unlock();
0080
0081         convert_from_fxsr(&env, tsk);
0082
0083         if (__copy_to_user(buf, &env, sizeof(env)) ||
0084             __put_user(xsave->i387.swd, &fp->status) ||
0085             __put_user(X86_FXSR_MAGIC, &fp->magic))
0086             return false;
0087     } else {
0088         struct fregs_state __user *fp = buf;
0089         u32 swd;
0090
0091         if (__get_user(swd, &fp->swd) || __put_user(swd, &fp->status))
0092             return false;
0093     }
0094
0095     return true;
0096 }
0097
0098 /*
0099  * Prepare the SW reserved portion of the fxsave memory layout, indicating
0100  * the presence of the extended state information in the memory layout
0101  * pointed to by the fpstate pointer in the sigcontext.
0102  * This is saved when ever the FP and extended state context is
0103  * saved on the user stack during the signal handler delivery to the user.
0104  */
0105 static inline void save_sw_bytes(struct _fpx_sw_bytes *sw_bytes, bool ia32_frame,
0106                  struct fpstate *fpstate)
0107 {
0108     sw_bytes->magic1 = FP_XSTATE_MAGIC1;
0109     sw_bytes->extended_size = fpstate->user_size + FP_XSTATE_MAGIC2_SIZE;
0110     sw_bytes->xfeatures = fpstate->user_xfeatures;
0111     sw_bytes->xstate_size = fpstate->user_size;
0112
0113     if (ia32_frame)
0114         sw_bytes->extended_size += sizeof(struct fregs_state);
0115 }
0116
0117 static inline bool save_xstate_epilog(void __user *buf, int ia32_frame,
0118                       struct fpstate *fpstate)
0119 {
0120     struct xregs_state __user *x = buf;
0121     struct _fpx_sw_bytes sw_bytes = {};
0122     u32 xfeatures;
0123     int err;
0124
0125     /* Setup the bytes not touched by the [f]xsave and reserved for SW. */
0126     save_sw_bytes(&sw_bytes, ia32_frame, fpstate);
0127     err = __copy_to_user(&x->i387.sw_reserved, &sw_bytes, sizeof(sw_bytes));
0128
0129     if (!use_xsave())
0130         return !err;
0131
0132     err |= __put_user(FP_XSTATE_MAGIC2,
0133               (__u32 __user *)(buf + fpstate->user_size));
0134
0135     /*
0136      * Read the xfeatures which we copied (directly from the cpu or
0137      * from the state in task struct) to the user buffers.
0138      */
0139     err |= __get_user(xfeatures, (__u32 __user *)&x->header.xfeatures);
0140
0141     /*
0142      * For legacy compatible, we always set FP/SSE bits in the bit
0143      * vector while saving the state to the user context. This will
0144      * enable us capturing any changes(during sigreturn) to
0145      * the FP/SSE bits by the legacy applications which don't touch
0146      * xfeatures in the xsave header.
0147      *
0148      * xsave aware apps can change the xfeatures in the xsave
0149      * header as well as change any contents in the memory layout.
0150      * xrestore as part of sigreturn will capture all the changes.
0151      */
0152     xfeatures |= XFEATURE_MASK_FPSSE;
0153
0154     err |= __put_user(xfeatures, (__u32 __user *)&x->header.xfeatures);
0155
0156     return !err;
0157 }
0158
0159 static inline int copy_fpregs_to_sigframe(struct xregs_state __user *buf)
0160 {
0161     if (use_xsave())
0162         return xsave_to_user_sigframe(buf);
0163     if (use_fxsr())
0164         return fxsave_to_user_sigframe((struct fxregs_state __user *) buf);
0165     else
0166         return fnsave_to_user_sigframe((struct fregs_state __user *) buf);
0167 }
0168
0169 /*
0170  * Save the fpu, extended register state to the user signal frame.
0171  *
0172  * 'buf_fx' is the 64-byte aligned pointer at which the [f|fx|x]save
0173  *  state is copied.
0174  *  'buf' points to the 'buf_fx' or to the fsave header followed by 'buf_fx'.
0175  *
0176  *  buf == buf_fx for 64-bit frames and 32-bit fsave frame.
0177  *  buf != buf_fx for 32-bit frames with fxstate.
0178  *
0179  * Save it directly to the user frame with disabled page fault handler. If
0180  * that faults, try to clear the frame which handles the page fault.
0181  *
0182  * If this is a 32-bit frame with fxstate, put a fsave header before
0183  * the aligned state at 'buf_fx'.
0184  *
0185  * For [f]xsave state, update the SW reserved fields in the [f]xsave frame
0186  * indicating the absence/presence of the extended state to the user.
0187  */
0188 bool copy_fpstate_to_sigframe(void __user *buf, void __user *buf_fx, int size)
0189 {
0190     struct task_struct *tsk = current;
0191     struct fpstate *fpstate = tsk->thread.fpu.fpstate;
0192     bool ia32_fxstate = (buf != buf_fx);
0193     int ret;
0194
0195     ia32_fxstate &= (IS_ENABLED(CONFIG_X86_32) ||
0196              IS_ENABLED(CONFIG_IA32_EMULATION));
0197
0198     if (!static_cpu_has(X86_FEATURE_FPU)) {
0199         struct user_i387_ia32_struct fp;
0200
0201         fpregs_soft_get(current, NULL, (struct membuf){.p = &fp,
0202                         .left = sizeof(fp)});
0203         return !copy_to_user(buf, &fp, sizeof(fp));
0204     }
0205
0206     if (!access_ok(buf, size))
0207         return false;
0208
0209     if (use_xsave()) {
0210         struct xregs_state __user *xbuf = buf_fx;
0211
0212         /*
0213          * Clear the xsave header first, so that reserved fields are
0214          * initialized to zero.
0215          */
0216         if (__clear_user(&xbuf->header, sizeof(xbuf->header)))
0217             return false;
0218     }
0219 retry:
0220     /*
0221      * Load the FPU registers if they are not valid for the current task.
0222      * With a valid FPU state we can attempt to save the state directly to
0223      * userland's stack frame which will likely succeed. If it does not,
0224      * resolve the fault in the user memory and try again.
0225      */
0226     fpregs_lock();
0227     if (test_thread_flag(TIF_NEED_FPU_LOAD))
0228         fpregs_restore_userregs();
0229
0230     pagefault_disable();
0231     ret = copy_fpregs_to_sigframe(buf_fx);
0232     pagefault_enable();
0233     fpregs_unlock();
0234
0235     if (ret) {
0236         if (!__clear_user(buf_fx, fpstate->user_size))
0237             goto retry;
0238         return false;
0239     }
0240
0241     /* Save the fsave header for the 32-bit frames. */
0242     if ((ia32_fxstate || !use_fxsr()) && !save_fsave_header(tsk, buf))
0243         return false;
0244
0245     if (use_fxsr() && !save_xstate_epilog(buf_fx, ia32_fxstate, fpstate))
0246         return false;
0247
0248     return true;
0249 }
0250
0251 static int __restore_fpregs_from_user(void __user *buf, u64 ufeatures,
0252                       u64 xrestore, bool fx_only)
0253 {
0254     if (use_xsave()) {
0255         u64 init_bv = ufeatures & ~xrestore;
0256         int ret;
0257
0258         if (likely(!fx_only))
0259             ret = xrstor_from_user_sigframe(buf, xrestore);
0260         else
0261             ret = fxrstor_from_user_sigframe(buf);
0262
0263         if (!ret && unlikely(init_bv))
0264             os_xrstor(&init_fpstate, init_bv);
0265         return ret;
0266     } else if (use_fxsr()) {
0267         return fxrstor_from_user_sigframe(buf);
0268     } else {
0269         return frstor_from_user_sigframe(buf);
0270     }
0271 }
0272
0273 /*
0274  * Attempt to restore the FPU registers directly from user memory.
0275  * Pagefaults are handled and any errors returned are fatal.
0276  */
0277 static bool restore_fpregs_from_user(void __user *buf, u64 xrestore,
0278                      bool fx_only, unsigned int size)
0279 {
0280     struct fpu *fpu = &current->thread.fpu;
0281     int ret;
0282
0283 retry:
0284     fpregs_lock();
0285     /* Ensure that XFD is up to date */
0286     xfd_update_state(fpu->fpstate);
0287     pagefault_disable();
0288     ret = __restore_fpregs_from_user(buf, fpu->fpstate->user_xfeatures,
0289                      xrestore, fx_only);
0290     pagefault_enable();
0291
0292     if (unlikely(ret)) {
0293         /*
0294          * The above did an FPU restore operation, restricted to
0295          * the user portion of the registers, and failed, but the
0296          * microcode might have modified the FPU registers
0297          * nevertheless.
0298          *
0299          * If the FPU registers do not belong to current, then
0300          * invalidate the FPU register state otherwise the task
0301          * might preempt current and return to user space with
0302          * corrupted FPU registers.
0303          */
0304         if (test_thread_flag(TIF_NEED_FPU_LOAD))
0305             __cpu_invalidate_fpregs_state();
0306         fpregs_unlock();
0307
0308         /* Try to handle #PF, but anything else is fatal. */
0309         if (ret != X86_TRAP_PF)
0310             return false;
0311
0312         if (!fault_in_readable(buf, size))
0313             goto retry;
0314         return false;
0315     }
0316
0317     /*
0318      * Restore supervisor states: previous context switch etc has done
0319      * XSAVES and saved the supervisor states in the kernel buffer from
0320      * which they can be restored now.
0321      *
0322      * It would be optimal to handle this with a single XRSTORS, but
0323      * this does not work because the rest of the FPU registers have
0324      * been restored from a user buffer directly.
0325      */
0326     if (test_thread_flag(TIF_NEED_FPU_LOAD) && xfeatures_mask_supervisor())
0327         os_xrstor_supervisor(fpu->fpstate);
0328
0329     fpregs_mark_activate();
0330     fpregs_unlock();
0331     return true;
0332 }
0333
0334 static bool __fpu_restore_sig(void __user *buf, void __user *buf_fx,
0335                   bool ia32_fxstate)
0336 {
0337     struct task_struct *tsk = current;
0338     struct fpu *fpu = &tsk->thread.fpu;
0339     struct user_i387_ia32_struct env;
0340     bool success, fx_only = false;
0341     union fpregs_state *fpregs;
0342     unsigned int state_size;
0343     u64 user_xfeatures = 0;
0344
0345     if (use_xsave()) {
0346         struct _fpx_sw_bytes fx_sw_user;
0347
0348         if (!check_xstate_in_sigframe(buf_fx, &fx_sw_user))
0349             return false;
0350
0351         fx_only = !fx_sw_user.magic1;
0352         state_size = fx_sw_user.xstate_size;
0353         user_xfeatures = fx_sw_user.xfeatures;
0354     } else {
0355         user_xfeatures = XFEATURE_MASK_FPSSE;
0356         state_size = fpu->fpstate->user_size;
0357     }
0358
0359     if (likely(!ia32_fxstate)) {
0360         /* Restore the FPU registers directly from user memory. */
0361         return restore_fpregs_from_user(buf_fx, user_xfeatures, fx_only,
0362                         state_size);
0363     }
0364
0365     /*
0366      * Copy the legacy state because the FP portion of the FX frame has
0367      * to be ignored for histerical raisins. The legacy state is folded
0368      * in once the larger state has been copied.
0369      */
0370     if (__copy_from_user(&env, buf, sizeof(env)))
0371         return false;
0372
0373     /*
0374      * By setting TIF_NEED_FPU_LOAD it is ensured that our xstate is
0375      * not modified on context switch and that the xstate is considered
0376      * to be loaded again on return to userland (overriding last_cpu avoids
0377      * the optimisation).
0378      */
0379     fpregs_lock();
0380     if (!test_thread_flag(TIF_NEED_FPU_LOAD)) {
0381         /*
0382          * If supervisor states are available then save the
0383          * hardware state in current's fpstate so that the
0384          * supervisor state is preserved. Save the full state for
0385          * simplicity. There is no point in optimizing this by only
0386          * saving the supervisor states and then shuffle them to
0387          * the right place in memory. It's ia32 mode. Shrug.
0388          */
0389         if (xfeatures_mask_supervisor())
0390             os_xsave(fpu->fpstate);
0391         set_thread_flag(TIF_NEED_FPU_LOAD);
0392     }
0393     __fpu_invalidate_fpregs_state(fpu);
0394     __cpu_invalidate_fpregs_state();
0395     fpregs_unlock();
0396
0397     fpregs = &fpu->fpstate->regs;
0398     if (use_xsave() && !fx_only) {
0399         if (copy_sigframe_from_user_to_xstate(fpu->fpstate, buf_fx))
0400             return false;
0401     } else {
0402         if (__copy_from_user(&fpregs->fxsave, buf_fx,
0403                      sizeof(fpregs->fxsave)))
0404             return false;
0405
0406         if (IS_ENABLED(CONFIG_X86_64)) {
0407             /* Reject invalid MXCSR values. */
0408             if (fpregs->fxsave.mxcsr & ~mxcsr_feature_mask)
0409                 return false;
0410         } else {
0411             /* Mask invalid bits out for historical reasons (broken hardware). */
0412             fpregs->fxsave.mxcsr &= mxcsr_feature_mask;
0413         }
0414
0415         /* Enforce XFEATURE_MASK_FPSSE when XSAVE is enabled */
0416         if (use_xsave())
0417             fpregs->xsave.header.xfeatures |= XFEATURE_MASK_FPSSE;
0418     }
0419
0420     /* Fold the legacy FP storage */
0421     convert_to_fxsr(&fpregs->fxsave, &env);
0422
0423     fpregs_lock();
0424     if (use_xsave()) {
0425         /*
0426          * Remove all UABI feature bits not set in user_xfeatures
0427          * from the memory xstate header which makes the full
0428          * restore below bring them into init state. This works for
0429          * fx_only mode as well because that has only FP and SSE
0430          * set in user_xfeatures.
0431          *
0432          * Preserve supervisor states!
0433          */
0434         u64 mask = user_xfeatures | xfeatures_mask_supervisor();
0435
0436         fpregs->xsave.header.xfeatures &= mask;
0437         success = !os_xrstor_safe(fpu->fpstate,
0438                       fpu_kernel_cfg.max_features);
0439     } else {
0440         success = !fxrstor_safe(&fpregs->fxsave);
0441     }
0442
0443     if (likely(success))
0444         fpregs_mark_activate();
0445
0446     fpregs_unlock();
0447     return success;
0448 }
0449
0450 static inline unsigned int xstate_sigframe_size(struct fpstate *fpstate)
0451 {
0452     unsigned int size = fpstate->user_size;
0453
0454     return use_xsave() ? size + FP_XSTATE_MAGIC2_SIZE : size;
0455 }
0456
0457 /*
0458  * Restore FPU state from a sigframe:
0459  */
0460 bool fpu__restore_sig(void __user *buf, int ia32_frame)
0461 {
0462     struct fpu *fpu = &current->thread.fpu;
0463     void __user *buf_fx = buf;
0464     bool ia32_fxstate = false;
0465     bool success = false;
0466     unsigned int size;
0467
0468     if (unlikely(!buf)) {
0469         fpu__clear_user_states(fpu);
0470         return true;
0471     }
0472
0473     size = xstate_sigframe_size(fpu->fpstate);
0474
0475     ia32_frame &= (IS_ENABLED(CONFIG_X86_32) ||
0476                IS_ENABLED(CONFIG_IA32_EMULATION));
0477
0478     /*
0479      * Only FXSR enabled systems need the FX state quirk.
0480      * FRSTOR does not need it and can use the fast path.
0481      */
0482     if (ia32_frame && use_fxsr()) {
0483         buf_fx = buf + sizeof(struct fregs_state);
0484         size += sizeof(struct fregs_state);
0485         ia32_fxstate = true;
0486     }
0487
0488     if (!access_ok(buf, size))
0489         goto out;
0490
0491     if (!IS_ENABLED(CONFIG_X86_64) && !cpu_feature_enabled(X86_FEATURE_FPU)) {
0492         success = !fpregs_soft_set(current, NULL, 0,
0493                        sizeof(struct user_i387_ia32_struct),
0494                        NULL, buf);
0495     } else {
0496         success = __fpu_restore_sig(buf, buf_fx, ia32_fxstate);
0497     }
0498
0499 out:
0500     if (unlikely(!success))
0501         fpu__clear_user_states(fpu);
0502     return success;
0503 }
0504
0505 unsigned long
0506 fpu__alloc_mathframe(unsigned long sp, int ia32_frame,
0507              unsigned long *buf_fx, unsigned long *size)
0508 {
0509     unsigned long frame_size = xstate_sigframe_size(current->thread.fpu.fpstate);
0510
0511     *buf_fx = sp = round_down(sp - frame_size, 64);
0512     if (ia32_frame && use_fxsr()) {
0513         frame_size += sizeof(struct fregs_state);
0514         sp -= sizeof(struct fregs_state);
0515     }
0516
0517     *size = frame_size;
0518
0519     return sp;
0520 }
0521
0522 unsigned long __init fpu__get_fpstate_size(void)
0523 {
0524     unsigned long ret = fpu_user_cfg.max_size;
0525
0526     if (use_xsave())
0527         ret += FP_XSTATE_MAGIC2_SIZE;
0528
0529     /*
0530      * This space is needed on (most) 32-bit kernels, or when a 32-bit
0531      * app is running on a 64-bit kernel. To keep things simple, just
0532      * assume the worst case and always include space for 'freg_state',
0533      * even for 64-bit apps on 64-bit kernels. This wastes a bit of
0534      * space, but keeps the code simple.
0535      */
0536     if ((IS_ENABLED(CONFIG_IA32_EMULATION) ||
0537          IS_ENABLED(CONFIG_X86_32)) && use_fxsr())
0538         ret += sizeof(struct fregs_state);
0539
0540     return ret;
0541 }
0542