kernel/fpu/regset.c

0001 // SPDX-License-Identifier: GPL-2.0
0002 /*
0003  * FPU register's regset abstraction, for ptrace, core dumps, etc.
0004  */
0005 #include <linux/sched/task_stack.h>
0006 #include <linux/vmalloc.h>
0007
0008 #include <asm/fpu/api.h>
0009 #include <asm/fpu/signal.h>
0010 #include <asm/fpu/regset.h>
0011
0012 #include "context.h"
0013 #include "internal.h"
0014 #include "legacy.h"
0015 #include "xstate.h"
0016
0017 /*
0018  * The xstateregs_active() routine is the same as the regset_fpregs_active() routine,
0019  * as the "regset->n" for the xstate regset will be updated based on the feature
0020  * capabilities supported by the xsave.
0021  */
0022 int regset_fpregs_active(struct task_struct *target, const struct user_regset *regset)
0023 {
0024     return regset->n;
0025 }
0026
0027 int regset_xregset_fpregs_active(struct task_struct *target, const struct user_regset *regset)
0028 {
0029     if (boot_cpu_has(X86_FEATURE_FXSR))
0030         return regset->n;
0031     else
0032         return 0;
0033 }
0034
0035 /*
0036  * The regset get() functions are invoked from:
0037  *
0038  *   - coredump to dump the current task's fpstate. If the current task
0039  *     owns the FPU then the memory state has to be synchronized and the
0040  *     FPU register state preserved. Otherwise fpstate is already in sync.
0041  *
0042  *   - ptrace to dump fpstate of a stopped task, in which case the registers
0043  *     have already been saved to fpstate on context switch.
0044  */
0045 static void sync_fpstate(struct fpu *fpu)
0046 {
0047     if (fpu == &current->thread.fpu)
0048         fpu_sync_fpstate(fpu);
0049 }
0050
0051 /*
0052  * Invalidate cached FPU registers before modifying the stopped target
0053  * task's fpstate.
0054  *
0055  * This forces the target task on resume to restore the FPU registers from
0056  * modified fpstate. Otherwise the task might skip the restore and operate
0057  * with the cached FPU registers which discards the modifications.
0058  */
0059 static void fpu_force_restore(struct fpu *fpu)
0060 {
0061     /*
0062      * Only stopped child tasks can be used to modify the FPU
0063      * state in the fpstate buffer:
0064      */
0065     WARN_ON_FPU(fpu == &current->thread.fpu);
0066
0067     __fpu_invalidate_fpregs_state(fpu);
0068 }
0069
0070 int xfpregs_get(struct task_struct *target, const struct user_regset *regset,
0071         struct membuf to)
0072 {
0073     struct fpu *fpu = &target->thread.fpu;
0074
0075     if (!cpu_feature_enabled(X86_FEATURE_FXSR))
0076         return -ENODEV;
0077
0078     sync_fpstate(fpu);
0079
0080     if (!use_xsave()) {
0081         return membuf_write(&to, &fpu->fpstate->regs.fxsave,
0082                     sizeof(fpu->fpstate->regs.fxsave));
0083     }
0084
0085     copy_xstate_to_uabi_buf(to, target, XSTATE_COPY_FX);
0086     return 0;
0087 }
0088
0089 int xfpregs_set(struct task_struct *target, const struct user_regset *regset,
0090         unsigned int pos, unsigned int count,
0091         const void *kbuf, const void __user *ubuf)
0092 {
0093     struct fpu *fpu = &target->thread.fpu;
0094     struct fxregs_state newstate;
0095     int ret;
0096
0097     if (!cpu_feature_enabled(X86_FEATURE_FXSR))
0098         return -ENODEV;
0099
0100     /* No funny business with partial or oversized writes is permitted. */
0101     if (pos != 0 || count != sizeof(newstate))
0102         return -EINVAL;
0103
0104     ret = user_regset_copyin(&pos, &count, &kbuf, &ubuf, &newstate, 0, -1);
0105     if (ret)
0106         return ret;
0107
0108     /* Do not allow an invalid MXCSR value. */
0109     if (newstate.mxcsr & ~mxcsr_feature_mask)
0110         return -EINVAL;
0111
0112     fpu_force_restore(fpu);
0113
0114     /* Copy the state  */
0115     memcpy(&fpu->fpstate->regs.fxsave, &newstate, sizeof(newstate));
0116
0117     /* Clear xmm8..15 for 32-bit callers */
0118     BUILD_BUG_ON(sizeof(fpu->__fpstate.regs.fxsave.xmm_space) != 16 * 16);
0119     if (in_ia32_syscall())
0120         memset(&fpu->fpstate->regs.fxsave.xmm_space[8*4], 0, 8 * 16);
0121
0122     /* Mark FP and SSE as in use when XSAVE is enabled */
0123     if (use_xsave())
0124         fpu->fpstate->regs.xsave.header.xfeatures |= XFEATURE_MASK_FPSSE;
0125
0126     return 0;
0127 }
0128
0129 int xstateregs_get(struct task_struct *target, const struct user_regset *regset,
0130         struct membuf to)
0131 {
0132     if (!cpu_feature_enabled(X86_FEATURE_XSAVE))
0133         return -ENODEV;
0134
0135     sync_fpstate(&target->thread.fpu);
0136
0137     copy_xstate_to_uabi_buf(to, target, XSTATE_COPY_XSAVE);
0138     return 0;
0139 }
0140
0141 int xstateregs_set(struct task_struct *target, const struct user_regset *regset,
0142           unsigned int pos, unsigned int count,
0143           const void *kbuf, const void __user *ubuf)
0144 {
0145     struct fpu *fpu = &target->thread.fpu;
0146     struct xregs_state *tmpbuf = NULL;
0147     int ret;
0148
0149     if (!cpu_feature_enabled(X86_FEATURE_XSAVE))
0150         return -ENODEV;
0151
0152     /*
0153      * A whole standard-format XSAVE buffer is needed:
0154      */
0155     if (pos != 0 || count != fpu_user_cfg.max_size)
0156         return -EFAULT;
0157
0158     if (!kbuf) {
0159         tmpbuf = vmalloc(count);
0160         if (!tmpbuf)
0161             return -ENOMEM;
0162
0163         if (copy_from_user(tmpbuf, ubuf, count)) {
0164             ret = -EFAULT;
0165             goto out;
0166         }
0167     }
0168
0169     fpu_force_restore(fpu);
0170     ret = copy_uabi_from_kernel_to_xstate(fpu->fpstate, kbuf ?: tmpbuf);
0171
0172 out:
0173     vfree(tmpbuf);
0174     return ret;
0175 }
0176
0177 #if defined CONFIG_X86_32 || defined CONFIG_IA32_EMULATION
0178
0179 /*
0180  * FPU tag word conversions.
0181  */
0182
0183 static inline unsigned short twd_i387_to_fxsr(unsigned short twd)
0184 {
0185     unsigned int tmp; /* to avoid 16 bit prefixes in the code */
0186
0187     /* Transform each pair of bits into 01 (valid) or 00 (empty) */
0188     tmp = ~twd;
0189     tmp = (tmp | (tmp>>1)) & 0x5555; /* 0V0V0V0V0V0V0V0V */
0190     /* and move the valid bits to the lower byte. */
0191     tmp = (tmp | (tmp >> 1)) & 0x3333; /* 00VV00VV00VV00VV */
0192     tmp = (tmp | (tmp >> 2)) & 0x0f0f; /* 0000VVVV0000VVVV */
0193     tmp = (tmp | (tmp >> 4)) & 0x00ff; /* 00000000VVVVVVVV */
0194
0195     return tmp;
0196 }
0197
0198 #define FPREG_ADDR(f, n)    ((void *)&(f)->st_space + (n) * 16)
0199 #define FP_EXP_TAG_VALID    0
0200 #define FP_EXP_TAG_ZERO     1
0201 #define FP_EXP_TAG_SPECIAL  2
0202 #define FP_EXP_TAG_EMPTY    3
0203
0204 static inline u32 twd_fxsr_to_i387(struct fxregs_state *fxsave)
0205 {
0206     struct _fpxreg *st;
0207     u32 tos = (fxsave->swd >> 11) & 7;
0208     u32 twd = (unsigned long) fxsave->twd;
0209     u32 tag;
0210     u32 ret = 0xffff0000u;
0211     int i;
0212
0213     for (i = 0; i < 8; i++, twd >>= 1) {
0214         if (twd & 0x1) {
0215             st = FPREG_ADDR(fxsave, (i - tos) & 7);
0216
0217             switch (st->exponent & 0x7fff) {
0218             case 0x7fff:
0219                 tag = FP_EXP_TAG_SPECIAL;
0220                 break;
0221             case 0x0000:
0222                 if (!st->significand[0] &&
0223                     !st->significand[1] &&
0224                     !st->significand[2] &&
0225                     !st->significand[3])
0226                     tag = FP_EXP_TAG_ZERO;
0227                 else
0228                     tag = FP_EXP_TAG_SPECIAL;
0229                 break;
0230             default:
0231                 if (st->significand[3] & 0x8000)
0232                     tag = FP_EXP_TAG_VALID;
0233                 else
0234                     tag = FP_EXP_TAG_SPECIAL;
0235                 break;
0236             }
0237         } else {
0238             tag = FP_EXP_TAG_EMPTY;
0239         }
0240         ret |= tag << (2 * i);
0241     }
0242     return ret;
0243 }
0244
0245 /*
0246  * FXSR floating point environment conversions.
0247  */
0248
0249 static void __convert_from_fxsr(struct user_i387_ia32_struct *env,
0250                 struct task_struct *tsk,
0251                 struct fxregs_state *fxsave)
0252 {
0253     struct _fpreg *to = (struct _fpreg *) &env->st_space[0];
0254     struct _fpxreg *from = (struct _fpxreg *) &fxsave->st_space[0];
0255     int i;
0256
0257     env->cwd = fxsave->cwd | 0xffff0000u;
0258     env->swd = fxsave->swd | 0xffff0000u;
0259     env->twd = twd_fxsr_to_i387(fxsave);
0260
0261 #ifdef CONFIG_X86_64
0262     env->fip = fxsave->rip;
0263     env->foo = fxsave->rdp;
0264     /*
0265      * should be actually ds/cs at fpu exception time, but
0266      * that information is not available in 64bit mode.
0267      */
0268     env->fcs = task_pt_regs(tsk)->cs;
0269     if (tsk == current) {
0270         savesegment(ds, env->fos);
0271     } else {
0272         env->fos = tsk->thread.ds;
0273     }
0274     env->fos |= 0xffff0000;
0275 #else
0276     env->fip = fxsave->fip;
0277     env->fcs = (u16) fxsave->fcs | ((u32) fxsave->fop << 16);
0278     env->foo = fxsave->foo;
0279     env->fos = fxsave->fos;
0280 #endif
0281
0282     for (i = 0; i < 8; ++i)
0283         memcpy(&to[i], &from[i], sizeof(to[0]));
0284 }
0285
0286 void
0287 convert_from_fxsr(struct user_i387_ia32_struct *env, struct task_struct *tsk)
0288 {
0289     __convert_from_fxsr(env, tsk, &tsk->thread.fpu.fpstate->regs.fxsave);
0290 }
0291
0292 void convert_to_fxsr(struct fxregs_state *fxsave,
0293              const struct user_i387_ia32_struct *env)
0294
0295 {
0296     struct _fpreg *from = (struct _fpreg *) &env->st_space[0];
0297     struct _fpxreg *to = (struct _fpxreg *) &fxsave->st_space[0];
0298     int i;
0299
0300     fxsave->cwd = env->cwd;
0301     fxsave->swd = env->swd;
0302     fxsave->twd = twd_i387_to_fxsr(env->twd);
0303     fxsave->fop = (u16) ((u32) env->fcs >> 16);
0304 #ifdef CONFIG_X86_64
0305     fxsave->rip = env->fip;
0306     fxsave->rdp = env->foo;
0307     /* cs and ds ignored */
0308 #else
0309     fxsave->fip = env->fip;
0310     fxsave->fcs = (env->fcs & 0xffff);
0311     fxsave->foo = env->foo;
0312     fxsave->fos = env->fos;
0313 #endif
0314
0315     for (i = 0; i < 8; ++i)
0316         memcpy(&to[i], &from[i], sizeof(from[0]));
0317 }
0318
0319 int fpregs_get(struct task_struct *target, const struct user_regset *regset,
0320            struct membuf to)
0321 {
0322     struct fpu *fpu = &target->thread.fpu;
0323     struct user_i387_ia32_struct env;
0324     struct fxregs_state fxsave, *fx;
0325
0326     sync_fpstate(fpu);
0327
0328     if (!cpu_feature_enabled(X86_FEATURE_FPU))
0329         return fpregs_soft_get(target, regset, to);
0330
0331     if (!cpu_feature_enabled(X86_FEATURE_FXSR)) {
0332         return membuf_write(&to, &fpu->fpstate->regs.fsave,
0333                     sizeof(struct fregs_state));
0334     }
0335
0336     if (use_xsave()) {
0337         struct membuf mb = { .p = &fxsave, .left = sizeof(fxsave) };
0338
0339         /* Handle init state optimized xstate correctly */
0340         copy_xstate_to_uabi_buf(mb, target, XSTATE_COPY_FP);
0341         fx = &fxsave;
0342     } else {
0343         fx = &fpu->fpstate->regs.fxsave;
0344     }
0345
0346     __convert_from_fxsr(&env, target, fx);
0347     return membuf_write(&to, &env, sizeof(env));
0348 }
0349
0350 int fpregs_set(struct task_struct *target, const struct user_regset *regset,
0351            unsigned int pos, unsigned int count,
0352            const void *kbuf, const void __user *ubuf)
0353 {
0354     struct fpu *fpu = &target->thread.fpu;
0355     struct user_i387_ia32_struct env;
0356     int ret;
0357
0358     /* No funny business with partial or oversized writes is permitted. */
0359     if (pos != 0 || count != sizeof(struct user_i387_ia32_struct))
0360         return -EINVAL;
0361
0362     if (!cpu_feature_enabled(X86_FEATURE_FPU))
0363         return fpregs_soft_set(target, regset, pos, count, kbuf, ubuf);
0364
0365     ret = user_regset_copyin(&pos, &count, &kbuf, &ubuf, &env, 0, -1);
0366     if (ret)
0367         return ret;
0368
0369     fpu_force_restore(fpu);
0370
0371     if (cpu_feature_enabled(X86_FEATURE_FXSR))
0372         convert_to_fxsr(&fpu->fpstate->regs.fxsave, &env);
0373     else
0374         memcpy(&fpu->fpstate->regs.fsave, &env, sizeof(env));
0375
0376     /*
0377      * Update the header bit in the xsave header, indicating the
0378      * presence of FP.
0379      */
0380     if (cpu_feature_enabled(X86_FEATURE_XSAVE))
0381         fpu->fpstate->regs.xsave.header.xfeatures |= XFEATURE_MASK_FP;
0382
0383     return 0;
0384 }
0385
0386 #endif  /* CONFIG_X86_32 || CONFIG_IA32_EMULATION */