sparc/lib/copy_user.S

0001 /* SPDX-License-Identifier: GPL-2.0 */
0002 /* copy_user.S: Sparc optimized copy_from_user and copy_to_user code.
0003  *
0004  *  Copyright(C) 1995 Linus Torvalds
0005  *  Copyright(C) 1996 David S. Miller
0006  *  Copyright(C) 1996 Eddie C. Dost
0007  *  Copyright(C) 1996,1998 Jakub Jelinek
0008  *
0009  * derived from:
0010  *  e-mail between David and Eddie.
0011  *
0012  * Returns 0 if successful, otherwise count of bytes not copied yet
0013  */
0014
0015 #include <asm/ptrace.h>
0016 #include <asm/asmmacro.h>
0017 #include <asm/page.h>
0018 #include <asm/thread_info.h>
0019 #include <asm/export.h>
0020
0021 /* Work around cpp -rob */
0022 #define ALLOC #alloc
0023 #define EXECINSTR #execinstr
0024
0025 #define EX_ENTRY(l1, l2)            \
0026     .section __ex_table,ALLOC;      \
0027     .align  4;              \
0028     .word   l1, l2;             \
0029     .text;
0030
0031 #define EX(x,y,a,b)                 \
0032 98:     x,y;                    \
0033     .section .fixup,ALLOC,EXECINSTR;    \
0034     .align  4;              \
0035 99: retl;                   \
0036      a, b, %o0;             \
0037     EX_ENTRY(98b, 99b)
0038
0039 #define EX2(x,y,c,d,e,a,b)          \
0040 98:     x,y;                    \
0041     .section .fixup,ALLOC,EXECINSTR;    \
0042     .align  4;              \
0043 99: c, d, e;                \
0044     retl;                   \
0045      a, b, %o0;             \
0046     EX_ENTRY(98b, 99b)
0047
0048 #define EXO2(x,y)               \
0049 98:     x, y;                   \
0050     EX_ENTRY(98b, 97f)
0051
0052 #define LD(insn, src, offset, reg, label)   \
0053 98: insn [%src + (offset)], %reg;       \
0054     .section .fixup,ALLOC,EXECINSTR;    \
0055 99: ba  label;              \
0056      mov    offset, %g5;            \
0057     EX_ENTRY(98b, 99b)
0058
0059 #define ST(insn, dst, offset, reg, label)   \
0060 98: insn %reg, [%dst + (offset)];       \
0061     .section .fixup,ALLOC,EXECINSTR;    \
0062 99: ba  label;              \
0063      mov    offset, %g5;            \
0064     EX_ENTRY(98b, 99b)
0065
0066 /* Both these macros have to start with exactly the same insn */
0067 /* left: g7 + (g1 % 128) - offset */
0068 #define MOVE_BIGCHUNK(src, dst, offset, t0, t1, t2, t3, t4, t5, t6, t7) \
0069     LD(ldd, src, offset + 0x00, t0, bigchunk_fault) \
0070     LD(ldd, src, offset + 0x08, t2, bigchunk_fault) \
0071     LD(ldd, src, offset + 0x10, t4, bigchunk_fault) \
0072     LD(ldd, src, offset + 0x18, t6, bigchunk_fault) \
0073     ST(st, dst, offset + 0x00, t0, bigchunk_fault)  \
0074     ST(st, dst, offset + 0x04, t1, bigchunk_fault)  \
0075     ST(st, dst, offset + 0x08, t2, bigchunk_fault)  \
0076     ST(st, dst, offset + 0x0c, t3, bigchunk_fault)  \
0077     ST(st, dst, offset + 0x10, t4, bigchunk_fault)  \
0078     ST(st, dst, offset + 0x14, t5, bigchunk_fault)  \
0079     ST(st, dst, offset + 0x18, t6, bigchunk_fault)  \
0080     ST(st, dst, offset + 0x1c, t7, bigchunk_fault)
0081
0082 /* left: g7 + (g1 % 128) - offset */
0083 #define MOVE_BIGALIGNCHUNK(src, dst, offset, t0, t1, t2, t3, t4, t5, t6, t7) \
0084     LD(ldd, src, offset + 0x00, t0, bigchunk_fault) \
0085     LD(ldd, src, offset + 0x08, t2, bigchunk_fault) \
0086     LD(ldd, src, offset + 0x10, t4, bigchunk_fault) \
0087     LD(ldd, src, offset + 0x18, t6, bigchunk_fault) \
0088     ST(std, dst, offset + 0x00, t0, bigchunk_fault) \
0089     ST(std, dst, offset + 0x08, t2, bigchunk_fault) \
0090     ST(std, dst, offset + 0x10, t4, bigchunk_fault) \
0091     ST(std, dst, offset + 0x18, t6, bigchunk_fault)
0092
0093     .section .fixup,#alloc,#execinstr
0094 bigchunk_fault:
0095     sub %g7, %g5, %o0
0096     and %g1, 127, %g1
0097     retl
0098      add    %o0, %g1, %o0
0099
0100 /* left: offset + 16 + (g1 % 16) */
0101 #define MOVE_LASTCHUNK(src, dst, offset, t0, t1, t2, t3) \
0102     LD(ldd, src, -(offset + 0x10), t0, lastchunk_fault) \
0103     LD(ldd, src, -(offset + 0x08), t2, lastchunk_fault) \
0104     ST(st, dst, -(offset + 0x10), t0, lastchunk_fault)  \
0105     ST(st, dst, -(offset + 0x0c), t1, lastchunk_fault)  \
0106     ST(st, dst, -(offset + 0x08), t2, lastchunk_fault)  \
0107     ST(st, dst, -(offset + 0x04), t3, lastchunk_fault)
0108
0109     .section .fixup,#alloc,#execinstr
0110 lastchunk_fault:
0111     and %g1, 15, %g1
0112     retl
0113      sub    %g1, %g5, %o0
0114
0115 /* left: o3 + (o2 % 16) - offset */
0116 #define MOVE_HALFCHUNK(src, dst, offset, t0, t1, t2, t3) \
0117     LD(lduh, src, offset + 0x00, t0, halfchunk_fault)   \
0118     LD(lduh, src, offset + 0x02, t1, halfchunk_fault)   \
0119     LD(lduh, src, offset + 0x04, t2, halfchunk_fault)   \
0120     LD(lduh, src, offset + 0x06, t3, halfchunk_fault)   \
0121     ST(sth, dst, offset + 0x00, t0, halfchunk_fault)    \
0122     ST(sth, dst, offset + 0x02, t1, halfchunk_fault)    \
0123     ST(sth, dst, offset + 0x04, t2, halfchunk_fault)    \
0124     ST(sth, dst, offset + 0x06, t3, halfchunk_fault)
0125
0126 /* left: o3 + (o2 % 16) + offset + 2 */
0127 #define MOVE_SHORTCHUNK(src, dst, offset, t0, t1) \
0128     LD(ldub, src, -(offset + 0x02), t0, halfchunk_fault)    \
0129     LD(ldub, src, -(offset + 0x01), t1, halfchunk_fault)    \
0130     ST(stb, dst, -(offset + 0x02), t0, halfchunk_fault) \
0131     ST(stb, dst, -(offset + 0x01), t1, halfchunk_fault)
0132
0133     .section .fixup,#alloc,#execinstr
0134 halfchunk_fault:
0135     and %o2, 15, %o2
0136     sub %o3, %g5, %o3
0137     retl
0138      add    %o2, %o3, %o0
0139
0140 /* left: offset + 2 + (o2 % 2) */
0141 #define MOVE_LAST_SHORTCHUNK(src, dst, offset, t0, t1) \
0142     LD(ldub, src, -(offset + 0x02), t0, last_shortchunk_fault)  \
0143     LD(ldub, src, -(offset + 0x01), t1, last_shortchunk_fault)  \
0144     ST(stb, dst, -(offset + 0x02), t0, last_shortchunk_fault)   \
0145     ST(stb, dst, -(offset + 0x01), t1, last_shortchunk_fault)
0146
0147     .section .fixup,#alloc,#execinstr
0148 last_shortchunk_fault:
0149     and %o2, 1, %o2
0150     retl
0151      sub    %o2, %g5, %o0
0152
0153     .text
0154     .align  4
0155
0156     .globl  __copy_user_begin
0157 __copy_user_begin:
0158
0159     .globl  __copy_user
0160     EXPORT_SYMBOL(__copy_user)
0161 dword_align:
0162     andcc   %o1, 1, %g0
0163     be  4f
0164      andcc  %o1, 2, %g0
0165
0166     EXO2(ldub [%o1], %g2)
0167     add %o1, 1, %o1
0168     EXO2(stb %g2, [%o0])
0169     sub %o2, 1, %o2
0170     bne 3f
0171      add    %o0, 1, %o0
0172
0173     EXO2(lduh [%o1], %g2)
0174     add %o1, 2, %o1
0175     EXO2(sth %g2, [%o0])
0176     sub %o2, 2, %o2
0177     b   3f
0178      add    %o0, 2, %o0
0179 4:
0180     EXO2(lduh [%o1], %g2)
0181     add %o1, 2, %o1
0182     EXO2(sth %g2, [%o0])
0183     sub %o2, 2, %o2
0184     b   3f
0185      add    %o0, 2, %o0
0186
0187 __copy_user:    /* %o0=dst %o1=src %o2=len */
0188     xor %o0, %o1, %o4
0189 1:
0190     andcc   %o4, 3, %o5
0191 2:
0192     bne cannot_optimize
0193      cmp    %o2, 15
0194
0195     bleu    short_aligned_end
0196      andcc  %o1, 3, %g0
0197
0198     bne dword_align
0199 3:
0200      andcc  %o1, 4, %g0
0201
0202     be  2f
0203      mov    %o2, %g1
0204
0205     EXO2(ld [%o1], %o4)
0206     sub %g1, 4, %g1
0207     EXO2(st %o4, [%o0])
0208     add %o1, 4, %o1
0209     add %o0, 4, %o0
0210 2:
0211     andcc   %g1, 0xffffff80, %g7
0212     be  3f
0213      andcc  %o0, 4, %g0
0214
0215     be  ldd_std + 4
0216 5:
0217     MOVE_BIGCHUNK(o1, o0, 0x00, o2, o3, o4, o5, g2, g3, g4, g5)
0218     MOVE_BIGCHUNK(o1, o0, 0x20, o2, o3, o4, o5, g2, g3, g4, g5)
0219     MOVE_BIGCHUNK(o1, o0, 0x40, o2, o3, o4, o5, g2, g3, g4, g5)
0220     MOVE_BIGCHUNK(o1, o0, 0x60, o2, o3, o4, o5, g2, g3, g4, g5)
0221     subcc   %g7, 128, %g7
0222     add %o1, 128, %o1
0223     bne 5b
0224      add    %o0, 128, %o0
0225 3:
0226     andcc   %g1, 0x70, %g7
0227     be  copy_user_table_end
0228      andcc  %g1, 8, %g0
0229
0230     sethi   %hi(copy_user_table_end), %o5
0231     srl %g7, 1, %o4
0232     add %g7, %o4, %o4
0233     add %o1, %g7, %o1
0234     sub %o5, %o4, %o5
0235     jmpl    %o5 + %lo(copy_user_table_end), %g0
0236      add    %o0, %g7, %o0
0237
0238     MOVE_LASTCHUNK(o1, o0, 0x60, g2, g3, g4, g5)
0239     MOVE_LASTCHUNK(o1, o0, 0x50, g2, g3, g4, g5)
0240     MOVE_LASTCHUNK(o1, o0, 0x40, g2, g3, g4, g5)
0241     MOVE_LASTCHUNK(o1, o0, 0x30, g2, g3, g4, g5)
0242     MOVE_LASTCHUNK(o1, o0, 0x20, g2, g3, g4, g5)
0243     MOVE_LASTCHUNK(o1, o0, 0x10, g2, g3, g4, g5)
0244     MOVE_LASTCHUNK(o1, o0, 0x00, g2, g3, g4, g5)
0245 copy_user_table_end:
0246     be  copy_user_last7
0247      andcc  %g1, 4, %g0
0248
0249     EX(ldd  [%o1], %g2, and %g1, 0xf)
0250     add %o0, 8, %o0
0251     add %o1, 8, %o1
0252     EX(st   %g2, [%o0 - 0x08], and %g1, 0xf)
0253     EX2(st  %g3, [%o0 - 0x04], and %g1, 0xf, %g1, sub %g1, 4)
0254 copy_user_last7:
0255     be  1f
0256      andcc  %g1, 2, %g0
0257
0258     EX(ld   [%o1], %g2, and %g1, 7)
0259     add %o1, 4, %o1
0260     EX(st   %g2, [%o0], and %g1, 7)
0261     add %o0, 4, %o0
0262 1:
0263     be  1f
0264      andcc  %g1, 1, %g0
0265
0266     EX(lduh [%o1], %g2, and %g1, 3)
0267     add %o1, 2, %o1
0268     EX(sth  %g2, [%o0], and %g1, 3)
0269     add %o0, 2, %o0
0270 1:
0271     be  1f
0272      nop
0273
0274     EX(ldub [%o1], %g2, add %g0, 1)
0275     EX(stb  %g2, [%o0], add %g0, 1)
0276 1:
0277     retl
0278      clr    %o0
0279
0280 ldd_std:
0281     MOVE_BIGALIGNCHUNK(o1, o0, 0x00, o2, o3, o4, o5, g2, g3, g4, g5)
0282     MOVE_BIGALIGNCHUNK(o1, o0, 0x20, o2, o3, o4, o5, g2, g3, g4, g5)
0283     MOVE_BIGALIGNCHUNK(o1, o0, 0x40, o2, o3, o4, o5, g2, g3, g4, g5)
0284     MOVE_BIGALIGNCHUNK(o1, o0, 0x60, o2, o3, o4, o5, g2, g3, g4, g5)
0285     subcc   %g7, 128, %g7
0286     add %o1, 128, %o1
0287     bne ldd_std
0288      add    %o0, 128, %o0
0289
0290     andcc   %g1, 0x70, %g7
0291     be  copy_user_table_end
0292      andcc  %g1, 8, %g0
0293
0294     sethi   %hi(copy_user_table_end), %o5
0295     srl %g7, 1, %o4
0296     add %g7, %o4, %o4
0297     add %o1, %g7, %o1
0298     sub %o5, %o4, %o5
0299     jmpl    %o5 + %lo(copy_user_table_end), %g0
0300      add    %o0, %g7, %o0
0301
0302 cannot_optimize:
0303     bleu    short_end
0304      cmp    %o5, 2
0305
0306     bne byte_chunk
0307      and    %o2, 0xfffffff0, %o3
0308
0309     andcc   %o1, 1, %g0
0310     be  10f
0311      nop
0312
0313     EXO2(ldub [%o1], %g2)
0314     add %o1, 1, %o1
0315     EXO2(stb %g2, [%o0])
0316     sub %o2, 1, %o2
0317     andcc   %o2, 0xfffffff0, %o3
0318     be  short_end
0319      add    %o0, 1, %o0
0320 10:
0321     MOVE_HALFCHUNK(o1, o0, 0x00, g2, g3, g4, g5)
0322     MOVE_HALFCHUNK(o1, o0, 0x08, g2, g3, g4, g5)
0323     subcc   %o3, 0x10, %o3
0324     add %o1, 0x10, %o1
0325     bne 10b
0326      add    %o0, 0x10, %o0
0327     b   2f
0328      and    %o2, 0xe, %o3
0329
0330 byte_chunk:
0331     MOVE_SHORTCHUNK(o1, o0, -0x02, g2, g3)
0332     MOVE_SHORTCHUNK(o1, o0, -0x04, g2, g3)
0333     MOVE_SHORTCHUNK(o1, o0, -0x06, g2, g3)
0334     MOVE_SHORTCHUNK(o1, o0, -0x08, g2, g3)
0335     MOVE_SHORTCHUNK(o1, o0, -0x0a, g2, g3)
0336     MOVE_SHORTCHUNK(o1, o0, -0x0c, g2, g3)
0337     MOVE_SHORTCHUNK(o1, o0, -0x0e, g2, g3)
0338     MOVE_SHORTCHUNK(o1, o0, -0x10, g2, g3)
0339     subcc   %o3, 0x10, %o3
0340     add %o1, 0x10, %o1
0341     bne byte_chunk
0342      add    %o0, 0x10, %o0
0343
0344 short_end:
0345     and %o2, 0xe, %o3
0346 2:
0347     sethi   %hi(short_table_end), %o5
0348     sll %o3, 3, %o4
0349     add %o0, %o3, %o0
0350     sub %o5, %o4, %o5
0351     add %o1, %o3, %o1
0352     jmpl    %o5 + %lo(short_table_end), %g0
0353      andcc  %o2, 1, %g0
0354     MOVE_LAST_SHORTCHUNK(o1, o0, 0x0c, g2, g3)
0355     MOVE_LAST_SHORTCHUNK(o1, o0, 0x0a, g2, g3)
0356     MOVE_LAST_SHORTCHUNK(o1, o0, 0x08, g2, g3)
0357     MOVE_LAST_SHORTCHUNK(o1, o0, 0x06, g2, g3)
0358     MOVE_LAST_SHORTCHUNK(o1, o0, 0x04, g2, g3)
0359     MOVE_LAST_SHORTCHUNK(o1, o0, 0x02, g2, g3)
0360     MOVE_LAST_SHORTCHUNK(o1, o0, 0x00, g2, g3)
0361 short_table_end:
0362     be  1f
0363      nop
0364     EX(ldub [%o1], %g2, add %g0, 1)
0365     EX(stb  %g2, [%o0], add %g0, 1)
0366 1:
0367     retl
0368      clr    %o0
0369
0370 short_aligned_end:
0371     bne short_end
0372      andcc  %o2, 8, %g0
0373
0374     be  1f
0375      andcc  %o2, 4, %g0
0376
0377     EXO2(ld [%o1 + 0x00], %g2)
0378     EXO2(ld [%o1 + 0x04], %g3)
0379     add %o1, 8, %o1
0380     EXO2(st %g2, [%o0 + 0x00])
0381     EX(st   %g3, [%o0 + 0x04], sub %o2, 4)
0382     add %o0, 8, %o0
0383 1:
0384     b   copy_user_last7
0385      mov    %o2, %g1
0386
0387     .section .fixup,#alloc,#execinstr
0388     .align  4
0389 97:
0390     retl
0391      mov    %o2, %o0
0392
0393     .globl  __copy_user_end
0394 __copy_user_end: