Back to home page

OSCL-LXR
 
 

     

0001 /* SPDX-License-Identifier: GPL-2.0 */
0002 /* U1memcpy.S: UltraSPARC-I/II/IIi/IIe optimized memcpy.
0003  *
0004  * Copyright (C) 1997, 2004 David S. Miller (davem@redhat.com)
0005  * Copyright (C) 1996, 1997, 1998, 1999 Jakub Jelinek (jj@ultra.linux.cz)
0006  */
0007 
0008 #ifdef __KERNEL__
0009 #include <linux/linkage.h>
0010 #include <asm/visasm.h>
0011 #include <asm/asi.h>
0012 #include <asm/export.h>
0013 #define GLOBAL_SPARE    g7
0014 #else
0015 #define GLOBAL_SPARE    g5
0016 #define ASI_BLK_P 0xf0
0017 #define FPRS_FEF  0x04
0018 #ifdef MEMCPY_DEBUG
0019 #define VISEntry rd %fprs, %o5; wr %g0, FPRS_FEF, %fprs; \
0020          clr %g1; clr %g2; clr %g3; subcc %g0, %g0, %g0;
0021 #define VISExit and %o5, FPRS_FEF, %o5; wr %o5, 0x0, %fprs
0022 #else
0023 #define VISEntry rd %fprs, %o5; wr %g0, FPRS_FEF, %fprs
0024 #define VISExit and %o5, FPRS_FEF, %o5; wr %o5, 0x0, %fprs
0025 #endif
0026 #endif
0027 
0028 #ifndef EX_LD
0029 #define EX_LD(x,y)  x
0030 #endif
0031 #ifndef EX_LD_FP
0032 #define EX_LD_FP(x,y)   x
0033 #endif
0034 
0035 #ifndef EX_ST
0036 #define EX_ST(x,y)  x
0037 #endif
0038 #ifndef EX_ST_FP
0039 #define EX_ST_FP(x,y)   x
0040 #endif
0041 
0042 #ifndef LOAD
0043 #define LOAD(type,addr,dest)    type [addr], dest
0044 #endif
0045 
0046 #ifndef LOAD_BLK
0047 #define LOAD_BLK(addr,dest) ldda [addr] ASI_BLK_P, dest
0048 #endif
0049 
0050 #ifndef STORE
0051 #define STORE(type,src,addr)    type src, [addr]
0052 #endif
0053 
0054 #ifndef STORE_BLK
0055 #define STORE_BLK(src,addr) stda src, [addr] ASI_BLK_P
0056 #endif
0057 
0058 #ifndef FUNC_NAME
0059 #define FUNC_NAME   memcpy
0060 #endif
0061 
0062 #ifndef PREAMBLE
0063 #define PREAMBLE
0064 #endif
0065 
0066 #ifndef XCC
0067 #define XCC xcc
0068 #endif
0069 
0070 #define FREG_FROB(f1, f2, f3, f4, f5, f6, f7, f8, f9)       \
0071     faligndata      %f1, %f2, %f48;         \
0072     faligndata      %f2, %f3, %f50;         \
0073     faligndata      %f3, %f4, %f52;         \
0074     faligndata      %f4, %f5, %f54;         \
0075     faligndata      %f5, %f6, %f56;         \
0076     faligndata      %f6, %f7, %f58;         \
0077     faligndata      %f7, %f8, %f60;         \
0078     faligndata      %f8, %f9, %f62;
0079 
0080 #define MAIN_LOOP_CHUNK(src, dest, fdest, fsrc, jmptgt)         \
0081     EX_LD_FP(LOAD_BLK(%src, %fdest), U1_gs_80_fp);          \
0082     EX_ST_FP(STORE_BLK(%fsrc, %dest), U1_gs_80_fp);         \
0083     add         %src, 0x40, %src;           \
0084     subcc           %GLOBAL_SPARE, 0x40, %GLOBAL_SPARE; \
0085     be,pn           %xcc, jmptgt;               \
0086      add            %dest, 0x40, %dest;         \
0087 
0088 #define LOOP_CHUNK1(src, dest, branch_dest)     \
0089     MAIN_LOOP_CHUNK(src, dest, f0,  f48, branch_dest)
0090 #define LOOP_CHUNK2(src, dest, branch_dest)     \
0091     MAIN_LOOP_CHUNK(src, dest, f16, f48, branch_dest)
0092 #define LOOP_CHUNK3(src, dest, branch_dest)     \
0093     MAIN_LOOP_CHUNK(src, dest, f32, f48, branch_dest)
0094 
0095 #define DO_SYNC         membar  #Sync;
0096 #define STORE_SYNC(dest, fsrc)              \
0097     EX_ST_FP(STORE_BLK(%fsrc, %dest), U1_gs_80_fp); \
0098     add         %dest, 0x40, %dest; \
0099     DO_SYNC
0100 
0101 #define STORE_JUMP(dest, fsrc, target)          \
0102     EX_ST_FP(STORE_BLK(%fsrc, %dest), U1_gs_40_fp); \
0103     add         %dest, 0x40, %dest; \
0104     ba,pt           %xcc, target;       \
0105      nop;
0106 
0107 #define FINISH_VISCHUNK(dest, f0, f1)           \
0108     subcc           %g3, 8, %g3;        \
0109     bl,pn           %xcc, 95f;      \
0110      faligndata     %f0, %f1, %f48;     \
0111     EX_ST_FP(STORE(std, %f48, %dest), U1_g3_8_fp);  \
0112     add         %dest, 8, %dest;
0113 
0114 #define UNEVEN_VISCHUNK_LAST(dest, f0, f1)  \
0115     subcc           %g3, 8, %g3;    \
0116     bl,pn           %xcc, 95f;  \
0117      fsrc2          %f0, %f1;
0118 
0119 #define UNEVEN_VISCHUNK(dest, f0, f1)       \
0120     UNEVEN_VISCHUNK_LAST(dest, f0, f1)  \
0121     ba,a,pt         %xcc, 93f;
0122 
0123     .register   %g2,#scratch
0124     .register   %g3,#scratch
0125 
0126     .text
0127 #ifndef EX_RETVAL
0128 #define EX_RETVAL(x)    x
0129 ENTRY(U1_g1_1_fp)
0130     VISExitHalf
0131     add     %g1, 1, %g1
0132     add     %g1, %g2, %g1
0133     retl
0134      add        %g1, %o2, %o0
0135 ENDPROC(U1_g1_1_fp)
0136 ENTRY(U1_g2_0_fp)
0137     VISExitHalf
0138     retl
0139      add        %g2, %o2, %o0
0140 ENDPROC(U1_g2_0_fp)
0141 ENTRY(U1_g2_8_fp)
0142     VISExitHalf
0143     add     %g2, 8, %g2
0144     retl
0145      add        %g2, %o2, %o0
0146 ENDPROC(U1_g2_8_fp)
0147 ENTRY(U1_gs_0_fp)
0148     VISExitHalf
0149     add     %GLOBAL_SPARE, %g3, %o0
0150     retl
0151      add        %o0, %o2, %o0
0152 ENDPROC(U1_gs_0_fp)
0153 ENTRY(U1_gs_80_fp)
0154     VISExitHalf
0155     add     %GLOBAL_SPARE, 0x80, %GLOBAL_SPARE
0156     add     %GLOBAL_SPARE, %g3, %o0
0157     retl
0158      add        %o0, %o2, %o0
0159 ENDPROC(U1_gs_80_fp)
0160 ENTRY(U1_gs_40_fp)
0161     VISExitHalf
0162     add     %GLOBAL_SPARE, 0x40, %GLOBAL_SPARE
0163     add     %GLOBAL_SPARE, %g3, %o0
0164     retl
0165      add        %o0, %o2, %o0
0166 ENDPROC(U1_gs_40_fp)
0167 ENTRY(U1_g3_0_fp)
0168     VISExitHalf
0169     retl
0170      add        %g3, %o2, %o0
0171 ENDPROC(U1_g3_0_fp)
0172 ENTRY(U1_g3_8_fp)
0173     VISExitHalf
0174     add     %g3, 8, %g3
0175     retl
0176      add        %g3, %o2, %o0
0177 ENDPROC(U1_g3_8_fp)
0178 ENTRY(U1_o2_0_fp)
0179     VISExitHalf
0180     retl
0181      mov        %o2, %o0
0182 ENDPROC(U1_o2_0_fp)
0183 ENTRY(U1_o2_1_fp)
0184     VISExitHalf
0185     retl
0186      add        %o2, 1, %o0
0187 ENDPROC(U1_o2_1_fp)
0188 ENTRY(U1_gs_0)
0189     VISExitHalf
0190     retl
0191      add        %GLOBAL_SPARE, %o2, %o0
0192 ENDPROC(U1_gs_0)
0193 ENTRY(U1_gs_8)
0194     VISExitHalf
0195     add     %GLOBAL_SPARE, %o2, %GLOBAL_SPARE
0196     retl
0197      add        %GLOBAL_SPARE, 0x8, %o0
0198 ENDPROC(U1_gs_8)
0199 ENTRY(U1_gs_10)
0200     VISExitHalf
0201     add     %GLOBAL_SPARE, %o2, %GLOBAL_SPARE
0202     retl
0203      add        %GLOBAL_SPARE, 0x10, %o0
0204 ENDPROC(U1_gs_10)
0205 ENTRY(U1_o2_0)
0206     retl
0207      mov        %o2, %o0
0208 ENDPROC(U1_o2_0)
0209 ENTRY(U1_o2_8)
0210     retl
0211      add        %o2, 8, %o0
0212 ENDPROC(U1_o2_8)
0213 ENTRY(U1_o2_4)
0214     retl
0215      add        %o2, 4, %o0
0216 ENDPROC(U1_o2_4)
0217 ENTRY(U1_o2_1)
0218     retl
0219      add        %o2, 1, %o0
0220 ENDPROC(U1_o2_1)
0221 ENTRY(U1_g1_0)
0222     retl
0223      add        %g1, %o2, %o0
0224 ENDPROC(U1_g1_0)
0225 ENTRY(U1_g1_1)
0226     add     %g1, 1, %g1
0227     retl
0228      add        %g1, %o2, %o0
0229 ENDPROC(U1_g1_1)
0230 ENTRY(U1_gs_0_o2_adj)
0231     and     %o2, 7, %o2
0232     retl
0233      add        %GLOBAL_SPARE, %o2, %o0
0234 ENDPROC(U1_gs_0_o2_adj)
0235 ENTRY(U1_gs_8_o2_adj)
0236     and     %o2, 7, %o2
0237     add     %GLOBAL_SPARE, 8, %GLOBAL_SPARE
0238     retl
0239      add        %GLOBAL_SPARE, %o2, %o0
0240 ENDPROC(U1_gs_8_o2_adj)
0241 #endif
0242 
0243     .align      64
0244 
0245     .globl      FUNC_NAME
0246     .type       FUNC_NAME,#function
0247 FUNC_NAME:      /* %o0=dst, %o1=src, %o2=len */
0248     srlx        %o2, 31, %g2
0249     cmp     %g2, 0
0250     tne     %xcc, 5
0251     PREAMBLE
0252     mov     %o0, %o4
0253     cmp     %o2, 0
0254     be,pn       %XCC, 85f
0255      or     %o0, %o1, %o3
0256     cmp     %o2, 16
0257     blu,a,pn    %XCC, 80f
0258      or     %o3, %o2, %o3
0259 
0260     cmp     %o2, (5 * 64)
0261     blu,pt      %XCC, 70f
0262      andcc      %o3, 0x7, %g0
0263 
0264     /* Clobbers o5/g1/g2/g3/g7/icc/xcc.  */
0265     VISEntry
0266 
0267     /* Is 'dst' already aligned on an 64-byte boundary? */
0268     andcc       %o0, 0x3f, %g2
0269     be,pt       %XCC, 2f
0270 
0271     /* Compute abs((dst & 0x3f) - 0x40) into %g2.  This is the number
0272      * of bytes to copy to make 'dst' 64-byte aligned.  We pre-
0273      * subtract this from 'len'.
0274      */
0275      sub        %o0, %o1, %GLOBAL_SPARE
0276     sub     %g2, 0x40, %g2
0277     sub     %g0, %g2, %g2
0278     sub     %o2, %g2, %o2
0279     andcc       %g2, 0x7, %g1
0280     be,pt       %icc, 2f
0281      and        %g2, 0x38, %g2
0282 
0283 1:  subcc       %g1, 0x1, %g1
0284     EX_LD_FP(LOAD(ldub, %o1 + 0x00, %o3), U1_g1_1_fp)
0285     EX_ST_FP(STORE(stb, %o3, %o1 + %GLOBAL_SPARE), U1_g1_1_fp)
0286     bgu,pt      %XCC, 1b
0287      add        %o1, 0x1, %o1
0288 
0289     add     %o1, %GLOBAL_SPARE, %o0
0290 
0291 2:  cmp     %g2, 0x0
0292     and     %o1, 0x7, %g1
0293     be,pt       %icc, 3f
0294      alignaddr  %o1, %g0, %o1
0295 
0296     EX_LD_FP(LOAD(ldd, %o1, %f4), U1_g2_0_fp)
0297 1:  EX_LD_FP(LOAD(ldd, %o1 + 0x8, %f6), U1_g2_0_fp)
0298     add     %o1, 0x8, %o1
0299     subcc       %g2, 0x8, %g2
0300     faligndata  %f4, %f6, %f0
0301     EX_ST_FP(STORE(std, %f0, %o0), U1_g2_8_fp)
0302     be,pn       %icc, 3f
0303      add        %o0, 0x8, %o0
0304 
0305     EX_LD_FP(LOAD(ldd, %o1 + 0x8, %f4), U1_g2_0_fp)
0306     add     %o1, 0x8, %o1
0307     subcc       %g2, 0x8, %g2
0308     faligndata  %f6, %f4, %f0
0309     EX_ST_FP(STORE(std, %f0, %o0), U1_g2_8_fp)
0310     bne,pt      %icc, 1b
0311      add        %o0, 0x8, %o0
0312 
0313     /* Destination is 64-byte aligned.  */
0314 3:  
0315     membar        #LoadStore | #StoreStore | #StoreLoad
0316 
0317     subcc       %o2, 0x40, %GLOBAL_SPARE
0318     add     %o1, %g1, %g1
0319     andncc      %GLOBAL_SPARE, (0x40 - 1), %GLOBAL_SPARE
0320     srl     %g1, 3, %g2
0321     sub     %o2, %GLOBAL_SPARE, %g3
0322     andn        %o1, (0x40 - 1), %o1
0323     and     %g2, 7, %g2
0324     andncc      %g3, 0x7, %g3
0325     fsrc2       %f0, %f2
0326     sub     %g3, 0x8, %g3
0327     sub     %o2, %GLOBAL_SPARE, %o2
0328 
0329     add     %g1, %GLOBAL_SPARE, %g1
0330     subcc       %o2, %g3, %o2
0331 
0332     EX_LD_FP(LOAD_BLK(%o1, %f0), U1_gs_0_fp)
0333     add     %o1, 0x40, %o1
0334     add     %g1, %g3, %g1
0335     EX_LD_FP(LOAD_BLK(%o1, %f16), U1_gs_0_fp)
0336     add     %o1, 0x40, %o1
0337     sub     %GLOBAL_SPARE, 0x80, %GLOBAL_SPARE
0338     EX_LD_FP(LOAD_BLK(%o1, %f32), U1_gs_80_fp)
0339     add     %o1, 0x40, %o1
0340 
0341     /* There are 8 instances of the unrolled loop,
0342      * one for each possible alignment of the
0343      * source buffer.  Each loop instance is 452
0344      * bytes.
0345      */
0346     sll     %g2, 3, %o3
0347     sub     %o3, %g2, %o3
0348     sllx        %o3, 4, %o3
0349     add     %o3, %g2, %o3
0350     sllx        %o3, 2, %g2
0351 1:  rd      %pc, %o3
0352     add     %o3, %lo(1f - 1b), %o3
0353     jmpl        %o3 + %g2, %g0
0354      nop
0355 
0356     .align      64
0357 1:  FREG_FROB(f0, f2, f4, f6, f8, f10,f12,f14,f16)
0358     LOOP_CHUNK1(o1, o0, 1f)
0359     FREG_FROB(f16,f18,f20,f22,f24,f26,f28,f30,f32)
0360     LOOP_CHUNK2(o1, o0, 2f)
0361     FREG_FROB(f32,f34,f36,f38,f40,f42,f44,f46,f0)
0362     LOOP_CHUNK3(o1, o0, 3f)
0363     ba,pt       %xcc, 1b+4
0364      faligndata %f0, %f2, %f48
0365 1:  FREG_FROB(f16,f18,f20,f22,f24,f26,f28,f30,f32)
0366     STORE_SYNC(o0, f48)
0367     FREG_FROB(f32,f34,f36,f38,f40,f42,f44,f46,f0)
0368     STORE_JUMP(o0, f48, 40f)
0369 2:  FREG_FROB(f32,f34,f36,f38,f40,f42,f44,f46,f0)
0370     STORE_SYNC(o0, f48)
0371     FREG_FROB(f0, f2, f4, f6, f8, f10,f12,f14,f16)
0372     STORE_JUMP(o0, f48, 48f)
0373 3:  FREG_FROB(f0, f2, f4, f6, f8, f10,f12,f14,f16)
0374     STORE_SYNC(o0, f48)
0375     FREG_FROB(f16,f18,f20,f22,f24,f26,f28,f30,f32)
0376     STORE_JUMP(o0, f48, 56f)
0377 
0378 1:  FREG_FROB(f2, f4, f6, f8, f10,f12,f14,f16,f18)
0379     LOOP_CHUNK1(o1, o0, 1f)
0380     FREG_FROB(f18,f20,f22,f24,f26,f28,f30,f32,f34)
0381     LOOP_CHUNK2(o1, o0, 2f)
0382     FREG_FROB(f34,f36,f38,f40,f42,f44,f46,f0, f2)
0383     LOOP_CHUNK3(o1, o0, 3f)
0384     ba,pt       %xcc, 1b+4
0385      faligndata %f2, %f4, %f48
0386 1:  FREG_FROB(f18,f20,f22,f24,f26,f28,f30,f32,f34)
0387     STORE_SYNC(o0, f48)
0388     FREG_FROB(f34,f36,f38,f40,f42,f44,f46,f0, f2)
0389     STORE_JUMP(o0, f48, 41f)
0390 2:  FREG_FROB(f34,f36,f38,f40,f42,f44,f46,f0, f2)
0391     STORE_SYNC(o0, f48)
0392     FREG_FROB(f2, f4, f6, f8, f10,f12,f14,f16,f18)
0393     STORE_JUMP(o0, f48, 49f)
0394 3:  FREG_FROB(f2, f4, f6, f8, f10,f12,f14,f16,f18)
0395     STORE_SYNC(o0, f48)
0396     FREG_FROB(f18,f20,f22,f24,f26,f28,f30,f32,f34)
0397     STORE_JUMP(o0, f48, 57f)
0398 
0399 1:  FREG_FROB(f4, f6, f8, f10,f12,f14,f16,f18,f20)
0400     LOOP_CHUNK1(o1, o0, 1f)
0401     FREG_FROB(f20,f22,f24,f26,f28,f30,f32,f34,f36)
0402     LOOP_CHUNK2(o1, o0, 2f)
0403     FREG_FROB(f36,f38,f40,f42,f44,f46,f0, f2, f4)
0404     LOOP_CHUNK3(o1, o0, 3f)
0405     ba,pt       %xcc, 1b+4
0406      faligndata %f4, %f6, %f48
0407 1:  FREG_FROB(f20,f22,f24,f26,f28,f30,f32,f34,f36)
0408     STORE_SYNC(o0, f48)
0409     FREG_FROB(f36,f38,f40,f42,f44,f46,f0, f2, f4)
0410     STORE_JUMP(o0, f48, 42f)
0411 2:  FREG_FROB(f36,f38,f40,f42,f44,f46,f0, f2, f4)
0412     STORE_SYNC(o0, f48)
0413     FREG_FROB(f4, f6, f8, f10,f12,f14,f16,f18,f20)
0414     STORE_JUMP(o0, f48, 50f)
0415 3:  FREG_FROB(f4, f6, f8, f10,f12,f14,f16,f18,f20)
0416     STORE_SYNC(o0, f48)
0417     FREG_FROB(f20,f22,f24,f26,f28,f30,f32,f34,f36)
0418     STORE_JUMP(o0, f48, 58f)
0419 
0420 1:  FREG_FROB(f6, f8, f10,f12,f14,f16,f18,f20,f22)
0421     LOOP_CHUNK1(o1, o0, 1f)
0422     FREG_FROB(f22,f24,f26,f28,f30,f32,f34,f36,f38)
0423     LOOP_CHUNK2(o1, o0, 2f)
0424     FREG_FROB(f38,f40,f42,f44,f46,f0, f2, f4, f6) 
0425     LOOP_CHUNK3(o1, o0, 3f)
0426     ba,pt       %xcc, 1b+4
0427      faligndata %f6, %f8, %f48
0428 1:  FREG_FROB(f22,f24,f26,f28,f30,f32,f34,f36,f38)
0429     STORE_SYNC(o0, f48)
0430     FREG_FROB(f38,f40,f42,f44,f46,f0, f2, f4, f6)
0431     STORE_JUMP(o0, f48, 43f)
0432 2:  FREG_FROB(f38,f40,f42,f44,f46,f0, f2, f4, f6)
0433     STORE_SYNC(o0, f48)
0434     FREG_FROB(f6, f8, f10,f12,f14,f16,f18,f20,f22)
0435     STORE_JUMP(o0, f48, 51f)
0436 3:  FREG_FROB(f6, f8, f10,f12,f14,f16,f18,f20,f22)
0437     STORE_SYNC(o0, f48)
0438     FREG_FROB(f22,f24,f26,f28,f30,f32,f34,f36,f38)
0439     STORE_JUMP(o0, f48, 59f)
0440 
0441 1:  FREG_FROB(f8, f10,f12,f14,f16,f18,f20,f22,f24)
0442     LOOP_CHUNK1(o1, o0, 1f)
0443     FREG_FROB(f24,f26,f28,f30,f32,f34,f36,f38,f40)
0444     LOOP_CHUNK2(o1, o0, 2f)
0445     FREG_FROB(f40,f42,f44,f46,f0, f2, f4, f6, f8)
0446     LOOP_CHUNK3(o1, o0, 3f)
0447     ba,pt       %xcc, 1b+4
0448      faligndata %f8, %f10, %f48
0449 1:  FREG_FROB(f24,f26,f28,f30,f32,f34,f36,f38,f40)
0450     STORE_SYNC(o0, f48)
0451     FREG_FROB(f40,f42,f44,f46,f0, f2, f4, f6, f8)
0452     STORE_JUMP(o0, f48, 44f)
0453 2:  FREG_FROB(f40,f42,f44,f46,f0, f2, f4, f6, f8)
0454     STORE_SYNC(o0, f48)
0455     FREG_FROB(f8, f10,f12,f14,f16,f18,f20,f22,f24)
0456     STORE_JUMP(o0, f48, 52f)
0457 3:  FREG_FROB(f8, f10,f12,f14,f16,f18,f20,f22,f24)
0458     STORE_SYNC(o0, f48)
0459     FREG_FROB(f24,f26,f28,f30,f32,f34,f36,f38,f40)
0460     STORE_JUMP(o0, f48, 60f)
0461 
0462 1:  FREG_FROB(f10,f12,f14,f16,f18,f20,f22,f24,f26)
0463     LOOP_CHUNK1(o1, o0, 1f)
0464     FREG_FROB(f26,f28,f30,f32,f34,f36,f38,f40,f42)
0465     LOOP_CHUNK2(o1, o0, 2f)
0466     FREG_FROB(f42,f44,f46,f0, f2, f4, f6, f8, f10)
0467     LOOP_CHUNK3(o1, o0, 3f)
0468     ba,pt       %xcc, 1b+4
0469      faligndata %f10, %f12, %f48
0470 1:  FREG_FROB(f26,f28,f30,f32,f34,f36,f38,f40,f42)
0471     STORE_SYNC(o0, f48)
0472     FREG_FROB(f42,f44,f46,f0, f2, f4, f6, f8, f10)
0473     STORE_JUMP(o0, f48, 45f)
0474 2:  FREG_FROB(f42,f44,f46,f0, f2, f4, f6, f8, f10)
0475     STORE_SYNC(o0, f48)
0476     FREG_FROB(f10,f12,f14,f16,f18,f20,f22,f24,f26)
0477     STORE_JUMP(o0, f48, 53f)
0478 3:  FREG_FROB(f10,f12,f14,f16,f18,f20,f22,f24,f26)
0479     STORE_SYNC(o0, f48)
0480     FREG_FROB(f26,f28,f30,f32,f34,f36,f38,f40,f42)
0481     STORE_JUMP(o0, f48, 61f)
0482 
0483 1:  FREG_FROB(f12,f14,f16,f18,f20,f22,f24,f26,f28)
0484     LOOP_CHUNK1(o1, o0, 1f)
0485     FREG_FROB(f28,f30,f32,f34,f36,f38,f40,f42,f44)
0486     LOOP_CHUNK2(o1, o0, 2f)
0487     FREG_FROB(f44,f46,f0, f2, f4, f6, f8, f10,f12)
0488     LOOP_CHUNK3(o1, o0, 3f)
0489     ba,pt       %xcc, 1b+4
0490      faligndata %f12, %f14, %f48
0491 1:  FREG_FROB(f28,f30,f32,f34,f36,f38,f40,f42,f44)
0492     STORE_SYNC(o0, f48)
0493     FREG_FROB(f44,f46,f0, f2, f4, f6, f8, f10,f12)
0494     STORE_JUMP(o0, f48, 46f)
0495 2:  FREG_FROB(f44,f46,f0, f2, f4, f6, f8, f10,f12)
0496     STORE_SYNC(o0, f48)
0497     FREG_FROB(f12,f14,f16,f18,f20,f22,f24,f26,f28)
0498     STORE_JUMP(o0, f48, 54f)
0499 3:  FREG_FROB(f12,f14,f16,f18,f20,f22,f24,f26,f28)
0500     STORE_SYNC(o0, f48)
0501     FREG_FROB(f28,f30,f32,f34,f36,f38,f40,f42,f44)
0502     STORE_JUMP(o0, f48, 62f)
0503 
0504 1:  FREG_FROB(f14,f16,f18,f20,f22,f24,f26,f28,f30)
0505     LOOP_CHUNK1(o1, o0, 1f)
0506     FREG_FROB(f30,f32,f34,f36,f38,f40,f42,f44,f46)
0507     LOOP_CHUNK2(o1, o0, 2f)
0508     FREG_FROB(f46,f0, f2, f4, f6, f8, f10,f12,f14)
0509     LOOP_CHUNK3(o1, o0, 3f)
0510     ba,pt       %xcc, 1b+4
0511      faligndata %f14, %f16, %f48
0512 1:  FREG_FROB(f30,f32,f34,f36,f38,f40,f42,f44,f46)
0513     STORE_SYNC(o0, f48)
0514     FREG_FROB(f46,f0, f2, f4, f6, f8, f10,f12,f14)
0515     STORE_JUMP(o0, f48, 47f)
0516 2:  FREG_FROB(f46,f0, f2, f4, f6, f8, f10,f12,f14)
0517     STORE_SYNC(o0, f48)
0518     FREG_FROB(f14,f16,f18,f20,f22,f24,f26,f28,f30)
0519     STORE_JUMP(o0, f48, 55f)
0520 3:  FREG_FROB(f14,f16,f18,f20,f22,f24,f26,f28,f30)
0521     STORE_SYNC(o0, f48)
0522     FREG_FROB(f30,f32,f34,f36,f38,f40,f42,f44,f46)
0523     STORE_JUMP(o0, f48, 63f)
0524 
0525 40: FINISH_VISCHUNK(o0, f0,  f2)
0526 41: FINISH_VISCHUNK(o0, f2,  f4)
0527 42: FINISH_VISCHUNK(o0, f4,  f6)
0528 43: FINISH_VISCHUNK(o0, f6,  f8)
0529 44: FINISH_VISCHUNK(o0, f8,  f10)
0530 45: FINISH_VISCHUNK(o0, f10, f12)
0531 46: FINISH_VISCHUNK(o0, f12, f14)
0532 47: UNEVEN_VISCHUNK(o0, f14, f0)
0533 48: FINISH_VISCHUNK(o0, f16, f18)
0534 49: FINISH_VISCHUNK(o0, f18, f20)
0535 50: FINISH_VISCHUNK(o0, f20, f22)
0536 51: FINISH_VISCHUNK(o0, f22, f24)
0537 52: FINISH_VISCHUNK(o0, f24, f26)
0538 53: FINISH_VISCHUNK(o0, f26, f28)
0539 54: FINISH_VISCHUNK(o0, f28, f30)
0540 55: UNEVEN_VISCHUNK(o0, f30, f0)
0541 56: FINISH_VISCHUNK(o0, f32, f34)
0542 57: FINISH_VISCHUNK(o0, f34, f36)
0543 58: FINISH_VISCHUNK(o0, f36, f38)
0544 59: FINISH_VISCHUNK(o0, f38, f40)
0545 60: FINISH_VISCHUNK(o0, f40, f42)
0546 61: FINISH_VISCHUNK(o0, f42, f44)
0547 62: FINISH_VISCHUNK(o0, f44, f46)
0548 63: UNEVEN_VISCHUNK_LAST(o0, f46, f0)
0549 
0550 93: EX_LD_FP(LOAD(ldd, %o1, %f2), U1_g3_0_fp)
0551     add     %o1, 8, %o1
0552     subcc       %g3, 8, %g3
0553     faligndata  %f0, %f2, %f8
0554     EX_ST_FP(STORE(std, %f8, %o0), U1_g3_8_fp)
0555     bl,pn       %xcc, 95f
0556      add        %o0, 8, %o0
0557     EX_LD_FP(LOAD(ldd, %o1, %f0), U1_g3_0_fp)
0558     add     %o1, 8, %o1
0559     subcc       %g3, 8, %g3
0560     faligndata  %f2, %f0, %f8
0561     EX_ST_FP(STORE(std, %f8, %o0), U1_g3_8_fp)
0562     bge,pt      %xcc, 93b
0563      add        %o0, 8, %o0
0564 
0565 95: brz,pt      %o2, 2f
0566      mov        %g1, %o1
0567 
0568 1:  EX_LD_FP(LOAD(ldub, %o1, %o3), U1_o2_0_fp)
0569     add     %o1, 1, %o1
0570     subcc       %o2, 1, %o2
0571     EX_ST_FP(STORE(stb, %o3, %o0), U1_o2_1_fp)
0572     bne,pt      %xcc, 1b
0573      add        %o0, 1, %o0
0574 
0575 2:  membar      #StoreLoad | #StoreStore
0576     VISExit
0577     retl
0578      mov        EX_RETVAL(%o4), %o0
0579 
0580     .align      64
0581 70: /* 16 < len <= (5 * 64) */
0582     bne,pn      %XCC, 75f
0583      sub        %o0, %o1, %o3
0584 
0585 72: andn        %o2, 0xf, %GLOBAL_SPARE
0586     and     %o2, 0xf, %o2
0587 1:  EX_LD(LOAD(ldx, %o1 + 0x00, %o5), U1_gs_0)
0588     EX_LD(LOAD(ldx, %o1 + 0x08, %g1), U1_gs_0)
0589     subcc       %GLOBAL_SPARE, 0x10, %GLOBAL_SPARE
0590     EX_ST(STORE(stx, %o5, %o1 + %o3), U1_gs_10)
0591     add     %o1, 0x8, %o1
0592     EX_ST(STORE(stx, %g1, %o1 + %o3), U1_gs_8)
0593     bgu,pt      %XCC, 1b
0594      add        %o1, 0x8, %o1
0595 73: andcc       %o2, 0x8, %g0
0596     be,pt       %XCC, 1f
0597      nop
0598     EX_LD(LOAD(ldx, %o1, %o5), U1_o2_0)
0599     sub     %o2, 0x8, %o2
0600     EX_ST(STORE(stx, %o5, %o1 + %o3), U1_o2_8)
0601     add     %o1, 0x8, %o1
0602 1:  andcc       %o2, 0x4, %g0
0603     be,pt       %XCC, 1f
0604      nop
0605     EX_LD(LOAD(lduw, %o1, %o5), U1_o2_0)
0606     sub     %o2, 0x4, %o2
0607     EX_ST(STORE(stw, %o5, %o1 + %o3), U1_o2_4)
0608     add     %o1, 0x4, %o1
0609 1:  cmp     %o2, 0
0610     be,pt       %XCC, 85f
0611      nop
0612     ba,pt       %xcc, 90f
0613      nop
0614 
0615 75: andcc       %o0, 0x7, %g1
0616     sub     %g1, 0x8, %g1
0617     be,pn       %icc, 2f
0618      sub        %g0, %g1, %g1
0619     sub     %o2, %g1, %o2
0620 
0621 1:  EX_LD(LOAD(ldub, %o1, %o5), U1_g1_0)
0622     subcc       %g1, 1, %g1
0623     EX_ST(STORE(stb, %o5, %o1 + %o3), U1_g1_1)
0624     bgu,pt      %icc, 1b
0625      add        %o1, 1, %o1
0626 
0627 2:  add     %o1, %o3, %o0
0628     andcc       %o1, 0x7, %g1
0629     bne,pt      %icc, 8f
0630      sll        %g1, 3, %g1
0631 
0632     cmp     %o2, 16
0633     bgeu,pt     %icc, 72b
0634      nop
0635     ba,a,pt     %xcc, 73b
0636 
0637 8:  mov     64, %o3
0638     andn        %o1, 0x7, %o1
0639     EX_LD(LOAD(ldx, %o1, %g2), U1_o2_0)
0640     sub     %o3, %g1, %o3
0641     andn        %o2, 0x7, %GLOBAL_SPARE
0642     sllx        %g2, %g1, %g2
0643 1:  EX_LD(LOAD(ldx, %o1 + 0x8, %g3), U1_gs_0_o2_adj)
0644     subcc       %GLOBAL_SPARE, 0x8, %GLOBAL_SPARE
0645     add     %o1, 0x8, %o1
0646     srlx        %g3, %o3, %o5
0647     or      %o5, %g2, %o5
0648     EX_ST(STORE(stx, %o5, %o0), U1_gs_8_o2_adj)
0649     add     %o0, 0x8, %o0
0650     bgu,pt      %icc, 1b
0651      sllx       %g3, %g1, %g2
0652 
0653     srl     %g1, 3, %g1
0654     andcc       %o2, 0x7, %o2
0655     be,pn       %icc, 85f
0656      add        %o1, %g1, %o1
0657     ba,pt       %xcc, 90f
0658      sub        %o0, %o1, %o3
0659 
0660     .align      64
0661 80: /* 0 < len <= 16 */
0662     andcc       %o3, 0x3, %g0
0663     bne,pn      %XCC, 90f
0664      sub        %o0, %o1, %o3
0665 
0666 1:  EX_LD(LOAD(lduw, %o1, %g1), U1_o2_0)
0667     subcc       %o2, 4, %o2
0668     EX_ST(STORE(stw, %g1, %o1 + %o3), U1_o2_4)
0669     bgu,pt      %XCC, 1b
0670      add        %o1, 4, %o1
0671 
0672 85: retl
0673      mov        EX_RETVAL(%o4), %o0
0674 
0675     .align      32
0676 90: EX_LD(LOAD(ldub, %o1, %g1), U1_o2_0)
0677     subcc       %o2, 1, %o2
0678     EX_ST(STORE(stb, %g1, %o1 + %o3), U1_o2_1)
0679     bgu,pt      %XCC, 90b
0680      add        %o1, 1, %o1
0681     retl
0682      mov        EX_RETVAL(%o4), %o0
0683 
0684     .size       FUNC_NAME, .-FUNC_NAME
0685 EXPORT_SYMBOL(FUNC_NAME)
[ Source navigation ] [ Diff markup ] [ Identifier search ] [ general search ]

This page was automatically generated by the 2.1.0 LXR engine.

The LXR team
Valid CSS 2.1! Valid HTML 4.01!