arm/lib/memmove.S

0001 /* SPDX-License-Identifier: GPL-2.0-only */
0002 /*
0003  *  linux/arch/arm/lib/memmove.S
0004  *
0005  *  Author: Nicolas Pitre
0006  *  Created:    Sep 28, 2005
0007  *  Copyright:  (C) MontaVista Software Inc.
0008  */
0009
0010 #include <linux/linkage.h>
0011 #include <asm/assembler.h>
0012 #include <asm/unwind.h>
0013
0014         .text
0015
0016 /*
0017  * Prototype: void *memmove(void *dest, const void *src, size_t n);
0018  *
0019  * Note:
0020  *
0021  * If the memory regions don't overlap, we simply branch to memcpy which is
0022  * normally a bit faster. Otherwise the copy is done going downwards.  This
0023  * is a transposition of the code from copy_template.S but with the copy
0024  * occurring in the opposite direction.
0025  */
0026
0027 ENTRY(__memmove)
0028 WEAK(memmove)
0029     UNWIND( .fnstart            )
0030
0031         subs    ip, r0, r1
0032         cmphi   r2, ip
0033         bls __memcpy
0034     UNWIND( .fnend              )
0035
0036     UNWIND( .fnstart            )
0037     UNWIND( .save   {r0, r4, fpreg, lr} )
0038         stmfd   sp!, {r0, r4, UNWIND(fpreg,) lr}
0039     UNWIND( .setfp  fpreg, sp       )
0040     UNWIND( mov fpreg, sp       )
0041         add r1, r1, r2
0042         add r0, r0, r2
0043         subs    r2, r2, #4
0044         blt 8f
0045         ands    ip, r0, #3
0046     PLD(    pld [r1, #-4]       )
0047         bne 9f
0048         ands    ip, r1, #3
0049         bne 10f
0050
0051 1:      subs    r2, r2, #(28)
0052         stmfd   sp!, {r5, r6, r8, r9}
0053         blt 5f
0054
0055     CALGN(  ands    ip, r0, #31     )
0056     CALGN(  sbcsne  r4, ip, r2      )  @ C is always set here
0057     CALGN(  bcs 2f          )
0058     CALGN(  adr r4, 6f          )
0059     CALGN(  subs    r2, r2, ip      )  @ C is set here
0060     CALGN(  rsb ip, ip, #32     )
0061     CALGN(  add pc, r4, ip      )
0062
0063     PLD(    pld [r1, #-4]       )
0064 2:  PLD(    subs    r2, r2, #96     )
0065     PLD(    pld [r1, #-32]      )
0066     PLD(    blt 4f          )
0067     PLD(    pld [r1, #-64]      )
0068     PLD(    pld [r1, #-96]      )
0069
0070 3:  PLD(    pld [r1, #-128]     )
0071 4:      ldmdb   r1!, {r3, r4, r5, r6, r8, r9, ip, lr}
0072         subs    r2, r2, #32
0073         stmdb   r0!, {r3, r4, r5, r6, r8, r9, ip, lr}
0074         bge 3b
0075     PLD(    cmn r2, #96         )
0076     PLD(    bge 4b          )
0077
0078 5:      ands    ip, r2, #28
0079         rsb ip, ip, #32
0080         addne   pc, pc, ip      @ C is always clear here
0081         b   7f
0082 6:      W(nop)
0083         W(ldr)  r3, [r1, #-4]!
0084         W(ldr)  r4, [r1, #-4]!
0085         W(ldr)  r5, [r1, #-4]!
0086         W(ldr)  r6, [r1, #-4]!
0087         W(ldr)  r8, [r1, #-4]!
0088         W(ldr)  r9, [r1, #-4]!
0089         W(ldr)  lr, [r1, #-4]!
0090
0091         add pc, pc, ip
0092         nop
0093         W(nop)
0094         W(str)  r3, [r0, #-4]!
0095         W(str)  r4, [r0, #-4]!
0096         W(str)  r5, [r0, #-4]!
0097         W(str)  r6, [r0, #-4]!
0098         W(str)  r8, [r0, #-4]!
0099         W(str)  r9, [r0, #-4]!
0100         W(str)  lr, [r0, #-4]!
0101
0102     CALGN(  bcs 2b          )
0103
0104 7:      ldmfd   sp!, {r5, r6, r8, r9}
0105
0106 8:      movs    r2, r2, lsl #31
0107         ldrbne  r3, [r1, #-1]!
0108         ldrbcs  r4, [r1, #-1]!
0109         ldrbcs  ip, [r1, #-1]
0110         strbne  r3, [r0, #-1]!
0111         strbcs  r4, [r0, #-1]!
0112         strbcs  ip, [r0, #-1]
0113         ldmfd   sp!, {r0, r4, UNWIND(fpreg,) pc}
0114
0115 9:      cmp ip, #2
0116         ldrbgt  r3, [r1, #-1]!
0117         ldrbge  r4, [r1, #-1]!
0118         ldrb    lr, [r1, #-1]!
0119         strbgt  r3, [r0, #-1]!
0120         strbge  r4, [r0, #-1]!
0121         subs    r2, r2, ip
0122         strb    lr, [r0, #-1]!
0123         blt 8b
0124         ands    ip, r1, #3
0125         beq 1b
0126
0127 10:     bic r1, r1, #3
0128         cmp ip, #2
0129         ldr r3, [r1, #0]
0130         beq 17f
0131         blt 18f
0132
0133
0134         .macro  backward_copy_shift push pull
0135
0136         subs    r2, r2, #28
0137         blt 14f
0138
0139     CALGN(  ands    ip, r0, #31     )
0140     CALGN(  sbcsne  r4, ip, r2      )  @ C is always set here
0141     CALGN(  subcc   r2, r2, ip      )
0142     CALGN(  bcc 15f         )
0143
0144 11:     stmfd   sp!, {r5, r6, r8 - r10}
0145
0146     PLD(    pld [r1, #-4]       )
0147     PLD(    subs    r2, r2, #96     )
0148     PLD(    pld [r1, #-32]      )
0149     PLD(    blt 13f         )
0150     PLD(    pld [r1, #-64]      )
0151     PLD(    pld [r1, #-96]      )
0152
0153 12: PLD(    pld [r1, #-128]     )
0154 13:     ldmdb   r1!, {r8, r9, r10, ip}
0155         mov     lr, r3, lspush #\push
0156         subs    r2, r2, #32
0157         ldmdb   r1!, {r3, r4, r5, r6}
0158         orr     lr, lr, ip, lspull #\pull
0159         mov     ip, ip, lspush #\push
0160         orr     ip, ip, r10, lspull #\pull
0161         mov     r10, r10, lspush #\push
0162         orr     r10, r10, r9, lspull #\pull
0163         mov     r9, r9, lspush #\push
0164         orr     r9, r9, r8, lspull #\pull
0165         mov     r8, r8, lspush #\push
0166         orr     r8, r8, r6, lspull #\pull
0167         mov     r6, r6, lspush #\push
0168         orr     r6, r6, r5, lspull #\pull
0169         mov     r5, r5, lspush #\push
0170         orr     r5, r5, r4, lspull #\pull
0171         mov     r4, r4, lspush #\push
0172         orr     r4, r4, r3, lspull #\pull
0173         stmdb   r0!, {r4 - r6, r8 - r10, ip, lr}
0174         bge 12b
0175     PLD(    cmn r2, #96         )
0176     PLD(    bge 13b         )
0177
0178         ldmfd   sp!, {r5, r6, r8 - r10}
0179
0180 14:     ands    ip, r2, #28
0181         beq 16f
0182
0183 15:     mov     lr, r3, lspush #\push
0184         ldr r3, [r1, #-4]!
0185         subs    ip, ip, #4
0186         orr lr, lr, r3, lspull #\pull
0187         str lr, [r0, #-4]!
0188         bgt 15b
0189     CALGN(  cmp r2, #0          )
0190     CALGN(  bge 11b         )
0191
0192 16:     add r1, r1, #(\pull / 8)
0193         b   8b
0194
0195         .endm
0196
0197
0198         backward_copy_shift push=8  pull=24
0199
0200 17:     backward_copy_shift push=16 pull=16
0201
0202 18:     backward_copy_shift push=24 pull=8
0203
0204     UNWIND( .fnend              )
0205 ENDPROC(memmove)
0206 ENDPROC(__memmove)