Back to home page

OSCL-LXR
 
 

     

0001 // SPDX-License-Identifier: GPL-2.0
0002 /*
0003  * In-kernel vector facility support functions
0004  *
0005  * Copyright IBM Corp. 2015
0006  * Author(s): Hendrik Brueckner <brueckner@linux.vnet.ibm.com>
0007  */
0008 #include <linux/kernel.h>
0009 #include <linux/cpu.h>
0010 #include <linux/sched.h>
0011 #include <asm/fpu/types.h>
0012 #include <asm/fpu/api.h>
0013 
0014 asm(".include \"asm/vx-insn.h\"\n");
0015 
0016 void __kernel_fpu_begin(struct kernel_fpu *state, u32 flags)
0017 {
0018     /*
0019      * Limit the save to the FPU/vector registers already
0020      * in use by the previous context
0021      */
0022     flags &= state->mask;
0023 
0024     if (flags & KERNEL_FPC)
0025         /* Save floating point control */
0026         asm volatile("stfpc %0" : "=Q" (state->fpc));
0027 
0028     if (!MACHINE_HAS_VX) {
0029         if (flags & KERNEL_VXR_V0V7) {
0030             /* Save floating-point registers */
0031             asm volatile("std 0,%0" : "=Q" (state->fprs[0]));
0032             asm volatile("std 1,%0" : "=Q" (state->fprs[1]));
0033             asm volatile("std 2,%0" : "=Q" (state->fprs[2]));
0034             asm volatile("std 3,%0" : "=Q" (state->fprs[3]));
0035             asm volatile("std 4,%0" : "=Q" (state->fprs[4]));
0036             asm volatile("std 5,%0" : "=Q" (state->fprs[5]));
0037             asm volatile("std 6,%0" : "=Q" (state->fprs[6]));
0038             asm volatile("std 7,%0" : "=Q" (state->fprs[7]));
0039             asm volatile("std 8,%0" : "=Q" (state->fprs[8]));
0040             asm volatile("std 9,%0" : "=Q" (state->fprs[9]));
0041             asm volatile("std 10,%0" : "=Q" (state->fprs[10]));
0042             asm volatile("std 11,%0" : "=Q" (state->fprs[11]));
0043             asm volatile("std 12,%0" : "=Q" (state->fprs[12]));
0044             asm volatile("std 13,%0" : "=Q" (state->fprs[13]));
0045             asm volatile("std 14,%0" : "=Q" (state->fprs[14]));
0046             asm volatile("std 15,%0" : "=Q" (state->fprs[15]));
0047         }
0048         return;
0049     }
0050 
0051     /* Test and save vector registers */
0052     asm volatile (
0053         /*
0054          * Test if any vector register must be saved and, if so,
0055          * test if all register can be saved.
0056          */
0057         "   la  1,%[vxrs]\n"    /* load save area */
0058         "   tmll    %[m],30\n"  /* KERNEL_VXR */
0059         "   jz  7f\n"       /* no work -> done */
0060         "   jo  5f\n"       /* -> save V0..V31 */
0061         /*
0062          * Test for special case KERNEL_FPU_MID only. In this
0063          * case a vstm V8..V23 is the best instruction
0064          */
0065         "   chi %[m],12\n"  /* KERNEL_VXR_MID */
0066         "   jne 0f\n"       /* -> save V8..V23 */
0067         "   VSTM    8,23,128,1\n"   /* vstm %v8,%v23,128(%r1) */
0068         "   j   7f\n"
0069         /* Test and save the first half of 16 vector registers */
0070         "0: tmll    %[m],6\n"   /* KERNEL_VXR_LOW */
0071         "   jz  3f\n"       /* -> KERNEL_VXR_HIGH */
0072         "   jo  2f\n"       /* 11 -> save V0..V15 */
0073         "   brc 2,1f\n"     /* 10 -> save V8..V15 */
0074         "   VSTM    0,7,0,1\n"  /* vstm %v0,%v7,0(%r1) */
0075         "   j   3f\n"
0076         "1: VSTM    8,15,128,1\n"   /* vstm %v8,%v15,128(%r1) */
0077         "   j   3f\n"
0078         "2: VSTM    0,15,0,1\n" /* vstm %v0,%v15,0(%r1) */
0079         /* Test and save the second half of 16 vector registers */
0080         "3: tmll    %[m],24\n"  /* KERNEL_VXR_HIGH */
0081         "   jz  7f\n"
0082         "   jo  6f\n"       /* 11 -> save V16..V31 */
0083         "   brc 2,4f\n"     /* 10 -> save V24..V31 */
0084         "   VSTM    16,23,256,1\n"  /* vstm %v16,%v23,256(%r1) */
0085         "   j   7f\n"
0086         "4: VSTM    24,31,384,1\n"  /* vstm %v24,%v31,384(%r1) */
0087         "   j   7f\n"
0088         "5: VSTM    0,15,0,1\n" /* vstm %v0,%v15,0(%r1) */
0089         "6: VSTM    16,31,256,1\n"  /* vstm %v16,%v31,256(%r1) */
0090         "7:"
0091         : [vxrs] "=Q" (*(struct vx_array *) &state->vxrs)
0092         : [m] "d" (flags)
0093         : "1", "cc");
0094 }
0095 EXPORT_SYMBOL(__kernel_fpu_begin);
0096 
0097 void __kernel_fpu_end(struct kernel_fpu *state, u32 flags)
0098 {
0099     /*
0100      * Limit the restore to the FPU/vector registers of the
0101      * previous context that have been overwritte by the
0102      * current context
0103      */
0104     flags &= state->mask;
0105 
0106     if (flags & KERNEL_FPC)
0107         /* Restore floating-point controls */
0108         asm volatile("lfpc %0" : : "Q" (state->fpc));
0109 
0110     if (!MACHINE_HAS_VX) {
0111         if (flags & KERNEL_VXR_V0V7) {
0112             /* Restore floating-point registers */
0113             asm volatile("ld 0,%0" : : "Q" (state->fprs[0]));
0114             asm volatile("ld 1,%0" : : "Q" (state->fprs[1]));
0115             asm volatile("ld 2,%0" : : "Q" (state->fprs[2]));
0116             asm volatile("ld 3,%0" : : "Q" (state->fprs[3]));
0117             asm volatile("ld 4,%0" : : "Q" (state->fprs[4]));
0118             asm volatile("ld 5,%0" : : "Q" (state->fprs[5]));
0119             asm volatile("ld 6,%0" : : "Q" (state->fprs[6]));
0120             asm volatile("ld 7,%0" : : "Q" (state->fprs[7]));
0121             asm volatile("ld 8,%0" : : "Q" (state->fprs[8]));
0122             asm volatile("ld 9,%0" : : "Q" (state->fprs[9]));
0123             asm volatile("ld 10,%0" : : "Q" (state->fprs[10]));
0124             asm volatile("ld 11,%0" : : "Q" (state->fprs[11]));
0125             asm volatile("ld 12,%0" : : "Q" (state->fprs[12]));
0126             asm volatile("ld 13,%0" : : "Q" (state->fprs[13]));
0127             asm volatile("ld 14,%0" : : "Q" (state->fprs[14]));
0128             asm volatile("ld 15,%0" : : "Q" (state->fprs[15]));
0129         }
0130         return;
0131     }
0132 
0133     /* Test and restore (load) vector registers */
0134     asm volatile (
0135         /*
0136          * Test if any vector register must be loaded and, if so,
0137          * test if all registers can be loaded at once.
0138          */
0139         "   la  1,%[vxrs]\n"    /* load restore area */
0140         "   tmll    %[m],30\n"  /* KERNEL_VXR */
0141         "   jz  7f\n"       /* no work -> done */
0142         "   jo  5f\n"       /* -> restore V0..V31 */
0143         /*
0144          * Test for special case KERNEL_FPU_MID only. In this
0145          * case a vlm V8..V23 is the best instruction
0146          */
0147         "   chi %[m],12\n"  /* KERNEL_VXR_MID */
0148         "   jne 0f\n"       /* -> restore V8..V23 */
0149         "   VLM 8,23,128,1\n"   /* vlm %v8,%v23,128(%r1) */
0150         "   j   7f\n"
0151         /* Test and restore the first half of 16 vector registers */
0152         "0: tmll    %[m],6\n"   /* KERNEL_VXR_LOW */
0153         "   jz  3f\n"       /* -> KERNEL_VXR_HIGH */
0154         "   jo  2f\n"       /* 11 -> restore V0..V15 */
0155         "   brc 2,1f\n"     /* 10 -> restore V8..V15 */
0156         "   VLM 0,7,0,1\n"  /* vlm %v0,%v7,0(%r1) */
0157         "   j   3f\n"
0158         "1: VLM 8,15,128,1\n"   /* vlm %v8,%v15,128(%r1) */
0159         "   j   3f\n"
0160         "2: VLM 0,15,0,1\n" /* vlm %v0,%v15,0(%r1) */
0161         /* Test and restore the second half of 16 vector registers */
0162         "3: tmll    %[m],24\n"  /* KERNEL_VXR_HIGH */
0163         "   jz  7f\n"
0164         "   jo  6f\n"       /* 11 -> restore V16..V31 */
0165         "   brc 2,4f\n"     /* 10 -> restore V24..V31 */
0166         "   VLM 16,23,256,1\n"  /* vlm %v16,%v23,256(%r1) */
0167         "   j   7f\n"
0168         "4: VLM 24,31,384,1\n"  /* vlm %v24,%v31,384(%r1) */
0169         "   j   7f\n"
0170         "5: VLM 0,15,0,1\n" /* vlm %v0,%v15,0(%r1) */
0171         "6: VLM 16,31,256,1\n"  /* vlm %v16,%v31,256(%r1) */
0172         "7:"
0173         : [vxrs] "=Q" (*(struct vx_array *) &state->vxrs)
0174         : [m] "d" (flags)
0175         : "1", "cc");
0176 }
0177 EXPORT_SYMBOL(__kernel_fpu_end);
0178 
0179 void __load_fpu_regs(void)
0180 {
0181     struct fpu *state = &current->thread.fpu;
0182     unsigned long *regs = current->thread.fpu.regs;
0183 
0184     asm volatile("lfpc %0" : : "Q" (state->fpc));
0185     if (likely(MACHINE_HAS_VX)) {
0186         asm volatile("lgr   1,%0\n"
0187                  "VLM   0,15,0,1\n"
0188                  "VLM   16,31,256,1\n"
0189                  :
0190                  : "d" (regs)
0191                  : "1", "cc", "memory");
0192     } else {
0193         asm volatile("ld 0,%0" : : "Q" (regs[0]));
0194         asm volatile("ld 1,%0" : : "Q" (regs[1]));
0195         asm volatile("ld 2,%0" : : "Q" (regs[2]));
0196         asm volatile("ld 3,%0" : : "Q" (regs[3]));
0197         asm volatile("ld 4,%0" : : "Q" (regs[4]));
0198         asm volatile("ld 5,%0" : : "Q" (regs[5]));
0199         asm volatile("ld 6,%0" : : "Q" (regs[6]));
0200         asm volatile("ld 7,%0" : : "Q" (regs[7]));
0201         asm volatile("ld 8,%0" : : "Q" (regs[8]));
0202         asm volatile("ld 9,%0" : : "Q" (regs[9]));
0203         asm volatile("ld 10,%0" : : "Q" (regs[10]));
0204         asm volatile("ld 11,%0" : : "Q" (regs[11]));
0205         asm volatile("ld 12,%0" : : "Q" (regs[12]));
0206         asm volatile("ld 13,%0" : : "Q" (regs[13]));
0207         asm volatile("ld 14,%0" : : "Q" (regs[14]));
0208         asm volatile("ld 15,%0" : : "Q" (regs[15]));
0209     }
0210     clear_cpu_flag(CIF_FPU);
0211 }
0212 EXPORT_SYMBOL(__load_fpu_regs);
0213 
0214 void load_fpu_regs(void)
0215 {
0216     raw_local_irq_disable();
0217     __load_fpu_regs();
0218     raw_local_irq_enable();
0219 }
0220 EXPORT_SYMBOL(load_fpu_regs);
0221 
0222 void save_fpu_regs(void)
0223 {
0224     unsigned long flags, *regs;
0225     struct fpu *state;
0226 
0227     local_irq_save(flags);
0228 
0229     if (test_cpu_flag(CIF_FPU))
0230         goto out;
0231 
0232     state = &current->thread.fpu;
0233     regs = current->thread.fpu.regs;
0234 
0235     asm volatile("stfpc %0" : "=Q" (state->fpc));
0236     if (likely(MACHINE_HAS_VX)) {
0237         asm volatile("lgr   1,%0\n"
0238                  "VSTM  0,15,0,1\n"
0239                  "VSTM  16,31,256,1\n"
0240                  :
0241                  : "d" (regs)
0242                  : "1", "cc", "memory");
0243     } else {
0244         asm volatile("std 0,%0" : "=Q" (regs[0]));
0245         asm volatile("std 1,%0" : "=Q" (regs[1]));
0246         asm volatile("std 2,%0" : "=Q" (regs[2]));
0247         asm volatile("std 3,%0" : "=Q" (regs[3]));
0248         asm volatile("std 4,%0" : "=Q" (regs[4]));
0249         asm volatile("std 5,%0" : "=Q" (regs[5]));
0250         asm volatile("std 6,%0" : "=Q" (regs[6]));
0251         asm volatile("std 7,%0" : "=Q" (regs[7]));
0252         asm volatile("std 8,%0" : "=Q" (regs[8]));
0253         asm volatile("std 9,%0" : "=Q" (regs[9]));
0254         asm volatile("std 10,%0" : "=Q" (regs[10]));
0255         asm volatile("std 11,%0" : "=Q" (regs[11]));
0256         asm volatile("std 12,%0" : "=Q" (regs[12]));
0257         asm volatile("std 13,%0" : "=Q" (regs[13]));
0258         asm volatile("std 14,%0" : "=Q" (regs[14]));
0259         asm volatile("std 15,%0" : "=Q" (regs[15]));
0260     }
0261     set_cpu_flag(CIF_FPU);
0262 out:
0263     local_irq_restore(flags);
0264 }
0265 EXPORT_SYMBOL(save_fpu_regs);
[ Source navigation ] [ Diff markup ] [ Identifier search ] [ general search ]

This page was automatically generated by the 2.1.0 LXR engine.

The LXR team
Valid CSS 2.1! Valid HTML 4.01!