mips/lib/memcpy.S

0001 /*
0002  * This file is subject to the terms and conditions of the GNU General Public
0003  * License.  See the file "COPYING" in the main directory of this archive
0004  * for more details.
0005  *
0006  * Unified implementation of memcpy, memmove and the __copy_user backend.
0007  *
0008  * Copyright (C) 1998, 99, 2000, 01, 2002 Ralf Baechle (ralf@gnu.org)
0009  * Copyright (C) 1999, 2000, 01, 2002 Silicon Graphics, Inc.
0010  * Copyright (C) 2002 Broadcom, Inc.
0011  *   memcpy/copy_user author: Mark Vandevoorde
0012  * Copyright (C) 2007  Maciej W. Rozycki
0013  * Copyright (C) 2014 Imagination Technologies Ltd.
0014  *
0015  * Mnemonic names for arguments to memcpy/__copy_user
0016  */
0017
0018 /*
0019  * Hack to resolve longstanding prefetch issue
0020  *
0021  * Prefetching may be fatal on some systems if we're prefetching beyond the
0022  * end of memory on some systems.  It's also a seriously bad idea on non
0023  * dma-coherent systems.
0024  */
0025 #ifdef CONFIG_DMA_NONCOHERENT
0026 #undef CONFIG_CPU_HAS_PREFETCH
0027 #endif
0028 #ifdef CONFIG_MIPS_MALTA
0029 #undef CONFIG_CPU_HAS_PREFETCH
0030 #endif
0031 #ifdef CONFIG_CPU_MIPSR6
0032 #undef CONFIG_CPU_HAS_PREFETCH
0033 #endif
0034
0035 #include <asm/asm.h>
0036 #include <asm/asm-offsets.h>
0037 #include <asm/export.h>
0038 #include <asm/regdef.h>
0039
0040 #define dst a0
0041 #define src a1
0042 #define len a2
0043
0044 /*
0045  * Spec
0046  *
0047  * memcpy copies len bytes from src to dst and sets v0 to dst.
0048  * It assumes that
0049  *   - src and dst don't overlap
0050  *   - src is readable
0051  *   - dst is writable
0052  * memcpy uses the standard calling convention
0053  *
0054  * __copy_user copies up to len bytes from src to dst and sets a2 (len) to
0055  * the number of uncopied bytes due to an exception caused by a read or write.
0056  * __copy_user assumes that src and dst don't overlap, and that the call is
0057  * implementing one of the following:
0058  *   copy_to_user
0059  *     - src is readable  (no exceptions when reading src)
0060  *   copy_from_user
0061  *     - dst is writable  (no exceptions when writing dst)
0062  * __copy_user uses a non-standard calling convention; see
0063  * include/asm-mips/uaccess.h
0064  *
0065  * When an exception happens on a load, the handler must
0066  # ensure that all of the destination buffer is overwritten to prevent
0067  * leaking information to user mode programs.
0068  */
0069
0070 /*
0071  * Implementation
0072  */
0073
0074 /*
0075  * The exception handler for loads requires that:
0076  *  1- AT contain the address of the byte just past the end of the source
0077  *     of the copy,
0078  *  2- src_entry <= src < AT, and
0079  *  3- (dst - src) == (dst_entry - src_entry),
0080  * The _entry suffix denotes values when __copy_user was called.
0081  *
0082  * (1) is set up up by uaccess.h and maintained by not writing AT in copy_user
0083  * (2) is met by incrementing src by the number of bytes copied
0084  * (3) is met by not doing loads between a pair of increments of dst and src
0085  *
0086  * The exception handlers for stores adjust len (if necessary) and return.
0087  * These handlers do not need to overwrite any data.
0088  *
0089  * For __rmemcpy and memmove an exception is always a kernel bug, therefore
0090  * they're not protected.
0091  */
0092
0093 /* Instruction type */
0094 #define LD_INSN 1
0095 #define ST_INSN 2
0096 /* Pretech type */
0097 #define SRC_PREFETCH 1
0098 #define DST_PREFETCH 2
0099 #define LEGACY_MODE 1
0100 #define EVA_MODE    2
0101 #define USEROP   1
0102 #define KERNELOP 2
0103
0104 /*
0105  * Wrapper to add an entry in the exception table
0106  * in case the insn causes a memory exception.
0107  * Arguments:
0108  * insn    : Load/store instruction
0109  * type    : Instruction type
0110  * reg     : Register
0111  * addr    : Address
0112  * handler : Exception handler
0113  */
0114
0115 #define EXC(insn, type, reg, addr, handler)         \
0116     .if \mode == LEGACY_MODE;               \
0117 9:      insn reg, addr;                 \
0118         .section __ex_table,"a";            \
0119         PTR_WD  9b, handler;                \
0120         .previous;                  \
0121     /* This is assembled in EVA mode */         \
0122     .else;                          \
0123         /* If loading from user or storing to user */   \
0124         .if ((\from == USEROP) && (type == LD_INSN)) || \
0125             ((\to == USEROP) && (type == ST_INSN)); \
0126 9:          __BUILD_EVA_INSN(insn##e, reg, addr);   \
0127             .section __ex_table,"a";        \
0128             PTR_WD  9b, handler;            \
0129             .previous;              \
0130         .else;                      \
0131             /*                  \
0132              *  Still in EVA, but no need for   \
0133              * exception handler or EVA insn    \
0134              */                 \
0135             insn reg, addr;             \
0136         .endif;                     \
0137     .endif
0138
0139 /*
0140  * Only on the 64-bit kernel we can made use of 64-bit registers.
0141  */
0142 #ifdef CONFIG_64BIT
0143 #define USE_DOUBLE
0144 #endif
0145
0146 #ifdef USE_DOUBLE
0147
0148 #define LOADK ld /* No exception */
0149 #define LOAD(reg, addr, handler)    EXC(ld, LD_INSN, reg, addr, handler)
0150 #define LOADL(reg, addr, handler)   EXC(ldl, LD_INSN, reg, addr, handler)
0151 #define LOADR(reg, addr, handler)   EXC(ldr, LD_INSN, reg, addr, handler)
0152 #define STOREL(reg, addr, handler)  EXC(sdl, ST_INSN, reg, addr, handler)
0153 #define STORER(reg, addr, handler)  EXC(sdr, ST_INSN, reg, addr, handler)
0154 #define STORE(reg, addr, handler)   EXC(sd, ST_INSN, reg, addr, handler)
0155 #define ADD    daddu
0156 #define SUB    dsubu
0157 #define SRL    dsrl
0158 #define SRA    dsra
0159 #define SLL    dsll
0160 #define SLLV   dsllv
0161 #define SRLV   dsrlv
0162 #define NBYTES 8
0163 #define LOG_NBYTES 3
0164
0165 /*
0166  * As we are sharing code base with the mips32 tree (which use the o32 ABI
0167  * register definitions). We need to redefine the register definitions from
0168  * the n64 ABI register naming to the o32 ABI register naming.
0169  */
0170 #undef t0
0171 #undef t1
0172 #undef t2
0173 #undef t3
0174 #define t0  $8
0175 #define t1  $9
0176 #define t2  $10
0177 #define t3  $11
0178 #define t4  $12
0179 #define t5  $13
0180 #define t6  $14
0181 #define t7  $15
0182
0183 #else
0184
0185 #define LOADK lw /* No exception */
0186 #define LOAD(reg, addr, handler)    EXC(lw, LD_INSN, reg, addr, handler)
0187 #define LOADL(reg, addr, handler)   EXC(lwl, LD_INSN, reg, addr, handler)
0188 #define LOADR(reg, addr, handler)   EXC(lwr, LD_INSN, reg, addr, handler)
0189 #define STOREL(reg, addr, handler)  EXC(swl, ST_INSN, reg, addr, handler)
0190 #define STORER(reg, addr, handler)  EXC(swr, ST_INSN, reg, addr, handler)
0191 #define STORE(reg, addr, handler)   EXC(sw, ST_INSN, reg, addr, handler)
0192 #define ADD    addu
0193 #define SUB    subu
0194 #define SRL    srl
0195 #define SLL    sll
0196 #define SRA    sra
0197 #define SLLV   sllv
0198 #define SRLV   srlv
0199 #define NBYTES 4
0200 #define LOG_NBYTES 2
0201
0202 #endif /* USE_DOUBLE */
0203
0204 #define LOADB(reg, addr, handler)   EXC(lb, LD_INSN, reg, addr, handler)
0205 #define STOREB(reg, addr, handler)  EXC(sb, ST_INSN, reg, addr, handler)
0206
0207 #ifdef CONFIG_CPU_HAS_PREFETCH
0208 # define _PREF(hint, addr, type)                    \
0209     .if \mode == LEGACY_MODE;                   \
0210         kernel_pref(hint, addr);                \
0211     .else;                              \
0212         .if ((\from == USEROP) && (type == SRC_PREFETCH)) ||    \
0213             ((\to == USEROP) && (type == DST_PREFETCH));    \
0214             /*                      \
0215              * PREFE has only 9 bits for the offset     \
0216              * compared to PREF which has 16, so it may \
0217              * need to use the $at register but this    \
0218              * register should remain intact because it's   \
0219              * used later on. Therefore use $v1.        \
0220              */                     \
0221             .set at=v1;                 \
0222             user_pref(hint, addr);              \
0223             .set noat;                  \
0224         .else;                          \
0225             kernel_pref(hint, addr);            \
0226         .endif;                         \
0227     .endif
0228 #else
0229 # define _PREF(hint, addr, type)
0230 #endif
0231
0232 #define PREFS(hint, addr) _PREF(hint, addr, SRC_PREFETCH)
0233 #define PREFD(hint, addr) _PREF(hint, addr, DST_PREFETCH)
0234
0235 #ifdef CONFIG_CPU_LITTLE_ENDIAN
0236 #define LDFIRST LOADR
0237 #define LDREST  LOADL
0238 #define STFIRST STORER
0239 #define STREST  STOREL
0240 #define SHIFT_DISCARD SLLV
0241 #else
0242 #define LDFIRST LOADL
0243 #define LDREST  LOADR
0244 #define STFIRST STOREL
0245 #define STREST  STORER
0246 #define SHIFT_DISCARD SRLV
0247 #endif
0248
0249 #define FIRST(unit) ((unit)*NBYTES)
0250 #define REST(unit)  (FIRST(unit)+NBYTES-1)
0251 #define UNIT(unit)  FIRST(unit)
0252
0253 #define ADDRMASK (NBYTES-1)
0254
0255     .text
0256     .set    noreorder
0257 #ifndef CONFIG_CPU_DADDI_WORKAROUNDS
0258     .set    noat
0259 #else
0260     .set    at=v1
0261 #endif
0262
0263     .align  5
0264
0265     /*
0266      * Macro to build the __copy_user common code
0267      * Arguments:
0268      * mode : LEGACY_MODE or EVA_MODE
0269      * from : Source operand. USEROP or KERNELOP
0270      * to   : Destination operand. USEROP or KERNELOP
0271      */
0272     .macro __BUILD_COPY_USER mode, from, to
0273
0274     /* initialize __memcpy if this the first time we execute this macro */
0275     .ifnotdef __memcpy
0276     .set __memcpy, 1
0277     .hidden __memcpy /* make sure it does not leak */
0278     .endif
0279
0280     /*
0281      * Note: dst & src may be unaligned, len may be 0
0282      * Temps
0283      */
0284 #define rem t8
0285
0286     R10KCBARRIER(0(ra))
0287     /*
0288      * The "issue break"s below are very approximate.
0289      * Issue delays for dcache fills will perturb the schedule, as will
0290      * load queue full replay traps, etc.
0291      *
0292      * If len < NBYTES use byte operations.
0293      */
0294     PREFS(  0, 0(src) )
0295     PREFD(  1, 0(dst) )
0296     sltu    t2, len, NBYTES
0297     and t1, dst, ADDRMASK
0298     PREFS(  0, 1*32(src) )
0299     PREFD(  1, 1*32(dst) )
0300     bnez    t2, .Lcopy_bytes_checklen\@
0301      and    t0, src, ADDRMASK
0302     PREFS(  0, 2*32(src) )
0303     PREFD(  1, 2*32(dst) )
0304 #ifndef CONFIG_CPU_NO_LOAD_STORE_LR
0305     bnez    t1, .Ldst_unaligned\@
0306      nop
0307     bnez    t0, .Lsrc_unaligned_dst_aligned\@
0308 #else /* CONFIG_CPU_NO_LOAD_STORE_LR */
0309     or  t0, t0, t1
0310     bnez    t0, .Lcopy_unaligned_bytes\@
0311 #endif /* CONFIG_CPU_NO_LOAD_STORE_LR */
0312     /*
0313      * use delay slot for fall-through
0314      * src and dst are aligned; need to compute rem
0315      */
0316 .Lboth_aligned\@:
0317      SRL    t0, len, LOG_NBYTES+3    # +3 for 8 units/iter
0318     beqz    t0, .Lcleanup_both_aligned\@ # len < 8*NBYTES
0319      and    rem, len, (8*NBYTES-1)   # rem = len % (8*NBYTES)
0320     PREFS(  0, 3*32(src) )
0321     PREFD(  1, 3*32(dst) )
0322     .align  4
0323 1:
0324     R10KCBARRIER(0(ra))
0325     LOAD(t0, UNIT(0)(src), .Ll_exc\@)
0326     LOAD(t1, UNIT(1)(src), .Ll_exc_copy\@)
0327     LOAD(t2, UNIT(2)(src), .Ll_exc_copy\@)
0328     LOAD(t3, UNIT(3)(src), .Ll_exc_copy\@)
0329     SUB len, len, 8*NBYTES
0330     LOAD(t4, UNIT(4)(src), .Ll_exc_copy\@)
0331     LOAD(t7, UNIT(5)(src), .Ll_exc_copy\@)
0332     STORE(t0, UNIT(0)(dst), .Ls_exc_p8u\@)
0333     STORE(t1, UNIT(1)(dst), .Ls_exc_p7u\@)
0334     LOAD(t0, UNIT(6)(src), .Ll_exc_copy\@)
0335     LOAD(t1, UNIT(7)(src), .Ll_exc_copy\@)
0336     ADD src, src, 8*NBYTES
0337     ADD dst, dst, 8*NBYTES
0338     STORE(t2, UNIT(-6)(dst), .Ls_exc_p6u\@)
0339     STORE(t3, UNIT(-5)(dst), .Ls_exc_p5u\@)
0340     STORE(t4, UNIT(-4)(dst), .Ls_exc_p4u\@)
0341     STORE(t7, UNIT(-3)(dst), .Ls_exc_p3u\@)
0342     STORE(t0, UNIT(-2)(dst), .Ls_exc_p2u\@)
0343     STORE(t1, UNIT(-1)(dst), .Ls_exc_p1u\@)
0344     PREFS(  0, 8*32(src) )
0345     PREFD(  1, 8*32(dst) )
0346     bne len, rem, 1b
0347      nop
0348
0349     /*
0350      * len == rem == the number of bytes left to copy < 8*NBYTES
0351      */
0352 .Lcleanup_both_aligned\@:
0353     beqz    len, .Ldone\@
0354      sltu   t0, len, 4*NBYTES
0355     bnez    t0, .Lless_than_4units\@
0356      and    rem, len, (NBYTES-1)    # rem = len % NBYTES
0357     /*
0358      * len >= 4*NBYTES
0359      */
0360     LOAD( t0, UNIT(0)(src), .Ll_exc\@)
0361     LOAD( t1, UNIT(1)(src), .Ll_exc_copy\@)
0362     LOAD( t2, UNIT(2)(src), .Ll_exc_copy\@)
0363     LOAD( t3, UNIT(3)(src), .Ll_exc_copy\@)
0364     SUB len, len, 4*NBYTES
0365     ADD src, src, 4*NBYTES
0366     R10KCBARRIER(0(ra))
0367     STORE(t0, UNIT(0)(dst), .Ls_exc_p4u\@)
0368     STORE(t1, UNIT(1)(dst), .Ls_exc_p3u\@)
0369     STORE(t2, UNIT(2)(dst), .Ls_exc_p2u\@)
0370     STORE(t3, UNIT(3)(dst), .Ls_exc_p1u\@)
0371     .set    reorder             /* DADDI_WAR */
0372     ADD dst, dst, 4*NBYTES
0373     beqz    len, .Ldone\@
0374     .set    noreorder
0375 .Lless_than_4units\@:
0376     /*
0377      * rem = len % NBYTES
0378      */
0379     beq rem, len, .Lcopy_bytes\@
0380      nop
0381 1:
0382     R10KCBARRIER(0(ra))
0383     LOAD(t0, 0(src), .Ll_exc\@)
0384     ADD src, src, NBYTES
0385     SUB len, len, NBYTES
0386     STORE(t0, 0(dst), .Ls_exc_p1u\@)
0387     .set    reorder             /* DADDI_WAR */
0388     ADD dst, dst, NBYTES
0389     bne rem, len, 1b
0390     .set    noreorder
0391
0392 #ifndef CONFIG_CPU_NO_LOAD_STORE_LR
0393     /*
0394      * src and dst are aligned, need to copy rem bytes (rem < NBYTES)
0395      * A loop would do only a byte at a time with possible branch
0396      * mispredicts.  Can't do an explicit LOAD dst,mask,or,STORE
0397      * because can't assume read-access to dst.  Instead, use
0398      * STREST dst, which doesn't require read access to dst.
0399      *
0400      * This code should perform better than a simple loop on modern,
0401      * wide-issue mips processors because the code has fewer branches and
0402      * more instruction-level parallelism.
0403      */
0404 #define bits t2
0405     beqz    len, .Ldone\@
0406      ADD    t1, dst, len    # t1 is just past last byte of dst
0407     li  bits, 8*NBYTES
0408     SLL rem, len, 3 # rem = number of bits to keep
0409     LOAD(t0, 0(src), .Ll_exc\@)
0410     SUB bits, bits, rem # bits = number of bits to discard
0411     SHIFT_DISCARD t0, t0, bits
0412     STREST(t0, -1(t1), .Ls_exc\@)
0413     jr  ra
0414      move   len, zero
0415 .Ldst_unaligned\@:
0416     /*
0417      * dst is unaligned
0418      * t0 = src & ADDRMASK
0419      * t1 = dst & ADDRMASK; T1 > 0
0420      * len >= NBYTES
0421      *
0422      * Copy enough bytes to align dst
0423      * Set match = (src and dst have same alignment)
0424      */
0425 #define match rem
0426     LDFIRST(t3, FIRST(0)(src), .Ll_exc\@)
0427     ADD t2, zero, NBYTES
0428     LDREST(t3, REST(0)(src), .Ll_exc_copy\@)
0429     SUB t2, t2, t1  # t2 = number of bytes copied
0430     xor match, t0, t1
0431     R10KCBARRIER(0(ra))
0432     STFIRST(t3, FIRST(0)(dst), .Ls_exc\@)
0433     beq len, t2, .Ldone\@
0434      SUB    len, len, t2
0435     ADD dst, dst, t2
0436     beqz    match, .Lboth_aligned\@
0437      ADD    src, src, t2
0438
0439 .Lsrc_unaligned_dst_aligned\@:
0440     SRL t0, len, LOG_NBYTES+2    # +2 for 4 units/iter
0441     PREFS(  0, 3*32(src) )
0442     beqz    t0, .Lcleanup_src_unaligned\@
0443      and    rem, len, (4*NBYTES-1)   # rem = len % 4*NBYTES
0444     PREFD(  1, 3*32(dst) )
0445 1:
0446 /*
0447  * Avoid consecutive LD*'s to the same register since some mips
0448  * implementations can't issue them in the same cycle.
0449  * It's OK to load FIRST(N+1) before REST(N) because the two addresses
0450  * are to the same unit (unless src is aligned, but it's not).
0451  */
0452     R10KCBARRIER(0(ra))
0453     LDFIRST(t0, FIRST(0)(src), .Ll_exc\@)
0454     LDFIRST(t1, FIRST(1)(src), .Ll_exc_copy\@)
0455     SUB len, len, 4*NBYTES
0456     LDREST(t0, REST(0)(src), .Ll_exc_copy\@)
0457     LDREST(t1, REST(1)(src), .Ll_exc_copy\@)
0458     LDFIRST(t2, FIRST(2)(src), .Ll_exc_copy\@)
0459     LDFIRST(t3, FIRST(3)(src), .Ll_exc_copy\@)
0460     LDREST(t2, REST(2)(src), .Ll_exc_copy\@)
0461     LDREST(t3, REST(3)(src), .Ll_exc_copy\@)
0462     PREFS(  0, 9*32(src) )      # 0 is PREF_LOAD  (not streamed)
0463     ADD src, src, 4*NBYTES
0464 #ifdef CONFIG_CPU_SB1
0465     nop             # improves slotting
0466 #endif
0467     STORE(t0, UNIT(0)(dst), .Ls_exc_p4u\@)
0468     STORE(t1, UNIT(1)(dst), .Ls_exc_p3u\@)
0469     STORE(t2, UNIT(2)(dst), .Ls_exc_p2u\@)
0470     STORE(t3, UNIT(3)(dst), .Ls_exc_p1u\@)
0471     PREFD(  1, 9*32(dst) )      # 1 is PREF_STORE (not streamed)
0472     .set    reorder             /* DADDI_WAR */
0473     ADD dst, dst, 4*NBYTES
0474     bne len, rem, 1b
0475     .set    noreorder
0476
0477 .Lcleanup_src_unaligned\@:
0478     beqz    len, .Ldone\@
0479      and    rem, len, NBYTES-1  # rem = len % NBYTES
0480     beq rem, len, .Lcopy_bytes\@
0481      nop
0482 1:
0483     R10KCBARRIER(0(ra))
0484     LDFIRST(t0, FIRST(0)(src), .Ll_exc\@)
0485     LDREST(t0, REST(0)(src), .Ll_exc_copy\@)
0486     ADD src, src, NBYTES
0487     SUB len, len, NBYTES
0488     STORE(t0, 0(dst), .Ls_exc_p1u\@)
0489     .set    reorder             /* DADDI_WAR */
0490     ADD dst, dst, NBYTES
0491     bne len, rem, 1b
0492     .set    noreorder
0493
0494 #endif /* !CONFIG_CPU_NO_LOAD_STORE_LR */
0495 .Lcopy_bytes_checklen\@:
0496     beqz    len, .Ldone\@
0497      nop
0498 .Lcopy_bytes\@:
0499     /* 0 < len < NBYTES  */
0500     R10KCBARRIER(0(ra))
0501 #define COPY_BYTE(N)            \
0502     LOADB(t0, N(src), .Ll_exc\@);   \
0503     SUB len, len, 1;        \
0504     beqz    len, .Ldone\@;      \
0505     STOREB(t0, N(dst), .Ls_exc_p1\@)
0506
0507     COPY_BYTE(0)
0508     COPY_BYTE(1)
0509 #ifdef USE_DOUBLE
0510     COPY_BYTE(2)
0511     COPY_BYTE(3)
0512     COPY_BYTE(4)
0513     COPY_BYTE(5)
0514 #endif
0515     LOADB(t0, NBYTES-2(src), .Ll_exc\@)
0516     SUB len, len, 1
0517     jr  ra
0518     STOREB(t0, NBYTES-2(dst), .Ls_exc_p1\@)
0519 .Ldone\@:
0520     jr  ra
0521      nop
0522
0523 #ifdef CONFIG_CPU_NO_LOAD_STORE_LR
0524 .Lcopy_unaligned_bytes\@:
0525 1:
0526     COPY_BYTE(0)
0527     COPY_BYTE(1)
0528     COPY_BYTE(2)
0529     COPY_BYTE(3)
0530     COPY_BYTE(4)
0531     COPY_BYTE(5)
0532     COPY_BYTE(6)
0533     COPY_BYTE(7)
0534     ADD src, src, 8
0535     b   1b
0536      ADD    dst, dst, 8
0537 #endif /* CONFIG_CPU_NO_LOAD_STORE_LR */
0538     .if __memcpy == 1
0539     END(memcpy)
0540     .set __memcpy, 0
0541     .hidden __memcpy
0542     .endif
0543
0544 .Ll_exc_copy\@:
0545     /*
0546      * Copy bytes from src until faulting load address (or until a
0547      * lb faults)
0548      *
0549      * When reached by a faulting LDFIRST/LDREST, THREAD_BUADDR($28)
0550      * may be more than a byte beyond the last address.
0551      * Hence, the lb below may get an exception.
0552      *
0553      * Assumes src < THREAD_BUADDR($28)
0554      */
0555     LOADK   t0, TI_TASK($28)
0556      nop
0557     LOADK   t0, THREAD_BUADDR(t0)
0558 1:
0559     LOADB(t1, 0(src), .Ll_exc\@)
0560     ADD src, src, 1
0561     sb  t1, 0(dst)  # can't fault -- we're copy_from_user
0562     .set    reorder             /* DADDI_WAR */
0563     ADD dst, dst, 1
0564     bne src, t0, 1b
0565     .set    noreorder
0566 .Ll_exc\@:
0567     LOADK   t0, TI_TASK($28)
0568      nop
0569     LOADK   t0, THREAD_BUADDR(t0)   # t0 is just past last good address
0570      nop
0571     SUB len, AT, t0     # len number of uncopied bytes
0572     jr  ra
0573      nop
0574
0575 #define SEXC(n)                         \
0576     .set    reorder;            /* DADDI_WAR */ \
0577 .Ls_exc_p ## n ## u\@:                      \
0578     ADD len, len, n*NBYTES;             \
0579     jr  ra;                     \
0580     .set    noreorder
0581
0582 SEXC(8)
0583 SEXC(7)
0584 SEXC(6)
0585 SEXC(5)
0586 SEXC(4)
0587 SEXC(3)
0588 SEXC(2)
0589 SEXC(1)
0590
0591 .Ls_exc_p1\@:
0592     .set    reorder             /* DADDI_WAR */
0593     ADD len, len, 1
0594     jr  ra
0595     .set    noreorder
0596 .Ls_exc\@:
0597     jr  ra
0598      nop
0599     .endm
0600
0601 #ifndef CONFIG_HAVE_PLAT_MEMCPY
0602     .align  5
0603 LEAF(memmove)
0604 EXPORT_SYMBOL(memmove)
0605     ADD t0, a0, a2
0606     ADD t1, a1, a2
0607     sltu    t0, a1, t0          # dst + len <= src -> memcpy
0608     sltu    t1, a0, t1          # dst >= src + len -> memcpy
0609     and t0, t1
0610     beqz    t0, .L__memcpy
0611      move   v0, a0              /* return value */
0612     beqz    a2, .Lr_out
0613     END(memmove)
0614
0615     /* fall through to __rmemcpy */
0616 LEAF(__rmemcpy)                 /* a0=dst a1=src a2=len */
0617      sltu   t0, a1, a0
0618     beqz    t0, .Lr_end_bytes_up        # src >= dst
0619      nop
0620     ADD a0, a2              # dst = dst + len
0621     ADD a1, a2              # src = src + len
0622
0623 .Lr_end_bytes:
0624     R10KCBARRIER(0(ra))
0625     lb  t0, -1(a1)
0626     SUB a2, a2, 0x1
0627     sb  t0, -1(a0)
0628     SUB a1, a1, 0x1
0629     .set    reorder             /* DADDI_WAR */
0630     SUB a0, a0, 0x1
0631     bnez    a2, .Lr_end_bytes
0632     .set    noreorder
0633
0634 .Lr_out:
0635     jr  ra
0636      move   a2, zero
0637
0638 .Lr_end_bytes_up:
0639     R10KCBARRIER(0(ra))
0640     lb  t0, (a1)
0641     SUB a2, a2, 0x1
0642     sb  t0, (a0)
0643     ADD a1, a1, 0x1
0644     .set    reorder             /* DADDI_WAR */
0645     ADD a0, a0, 0x1
0646     bnez    a2, .Lr_end_bytes_up
0647     .set    noreorder
0648
0649     jr  ra
0650      move   a2, zero
0651     END(__rmemcpy)
0652
0653 /*
0654  * A combined memcpy/__copy_user
0655  * __copy_user sets len to 0 for success; else to an upper bound of
0656  * the number of uncopied bytes.
0657  * memcpy sets v0 to dst.
0658  */
0659     .align  5
0660 LEAF(memcpy)                    /* a0=dst a1=src a2=len */
0661 EXPORT_SYMBOL(memcpy)
0662     move    v0, dst             /* return value */
0663 .L__memcpy:
0664 #ifndef CONFIG_EVA
0665 FEXPORT(__raw_copy_from_user)
0666 EXPORT_SYMBOL(__raw_copy_from_user)
0667 FEXPORT(__raw_copy_to_user)
0668 EXPORT_SYMBOL(__raw_copy_to_user)
0669 #endif
0670     /* Legacy Mode, user <-> user */
0671     __BUILD_COPY_USER LEGACY_MODE USEROP USEROP
0672
0673 #endif
0674
0675 #ifdef CONFIG_EVA
0676
0677 /*
0678  * For EVA we need distinct symbols for reading and writing to user space.
0679  * This is because we need to use specific EVA instructions to perform the
0680  * virtual <-> physical translation when a virtual address is actually in user
0681  * space
0682  */
0683
0684 /*
0685  * __copy_from_user (EVA)
0686  */
0687
0688 LEAF(__raw_copy_from_user)
0689 EXPORT_SYMBOL(__raw_copy_from_user)
0690     __BUILD_COPY_USER EVA_MODE USEROP KERNELOP
0691 END(__raw_copy_from_user)
0692
0693
0694
0695 /*
0696  * __copy_to_user (EVA)
0697  */
0698
0699 LEAF(__raw_copy_to_user)
0700 EXPORT_SYMBOL(__raw_copy_to_user)
0701 __BUILD_COPY_USER EVA_MODE KERNELOP USEROP
0702 END(__raw_copy_to_user)
0703
0704 #endif