arm/lib/copy_template.S

0001 /* SPDX-License-Identifier: GPL-2.0-only */
0002 /*
0003  *  linux/arch/arm/lib/copy_template.s
0004  *
0005  *  Code template for optimized memory copy functions
0006  *
0007  *  Author: Nicolas Pitre
0008  *  Created:    Sep 28, 2005
0009  *  Copyright:  MontaVista Software, Inc.
0010  */
0011
0012 /*
0013  * Theory of operation
0014  * -------------------
0015  *
0016  * This file provides the core code for a forward memory copy used in
0017  * the implementation of memcopy(), copy_to_user() and copy_from_user().
0018  *
0019  * The including file must define the following accessor macros
0020  * according to the need of the given function:
0021  *
0022  * ldr1w ptr reg abort
0023  *
0024  *  This loads one word from 'ptr', stores it in 'reg' and increments
0025  *  'ptr' to the next word. The 'abort' argument is used for fixup tables.
0026  *
0027  * ldr4w ptr reg1 reg2 reg3 reg4 abort
0028  * ldr8w ptr, reg1 reg2 reg3 reg4 reg5 reg6 reg7 reg8 abort
0029  *
0030  *  This loads four or eight words starting from 'ptr', stores them
0031  *  in provided registers and increments 'ptr' past those words.
0032  *  The'abort' argument is used for fixup tables.
0033  *
0034  * ldr1b ptr reg cond abort
0035  *
0036  *  Similar to ldr1w, but it loads a byte and increments 'ptr' one byte.
0037  *  It also must apply the condition code if provided, otherwise the
0038  *  "al" condition is assumed by default.
0039  *
0040  * str1w ptr reg abort
0041  * str8w ptr reg1 reg2 reg3 reg4 reg5 reg6 reg7 reg8 abort
0042  * str1b ptr reg cond abort
0043  *
0044  *  Same as their ldr* counterparts, but data is stored to 'ptr' location
0045  *  rather than being loaded.
0046  *
0047  * enter reg1 reg2
0048  *
0049  *  Preserve the provided registers on the stack plus any additional
0050  *  data as needed by the implementation including this code. Called
0051  *  upon code entry.
0052  *
0053  * usave reg1 reg2
0054  *
0055  *  Unwind annotation macro is corresponding for 'enter' macro.
0056  *  It tell unwinder that preserved some provided registers on the stack
0057  *  and additional data by a prior 'enter' macro.
0058  *
0059  * exit reg1 reg2
0060  *
0061  *  Restore registers with the values previously saved with the
0062  *  'preserv' macro. Called upon code termination.
0063  *
0064  * LDR1W_SHIFT
0065  * STR1W_SHIFT
0066  *
0067  *  Correction to be applied to the "ip" register when branching into
0068  *  the ldr1w or str1w instructions (some of these macros may expand to
0069  *  than one 32bit instruction in Thumb-2)
0070  */
0071
0072     UNWIND( .fnstart            )
0073         enter   r4, UNWIND(fpreg,) lr
0074     UNWIND( .setfp  fpreg, sp       )
0075     UNWIND( mov fpreg, sp       )
0076
0077         subs    r2, r2, #4
0078         blt 8f
0079         ands    ip, r0, #3
0080     PLD(    pld [r1, #0]        )
0081         bne 9f
0082         ands    ip, r1, #3
0083         bne 10f
0084
0085 1:      subs    r2, r2, #(28)
0086         stmfd   sp!, {r5, r6, r8, r9}
0087         blt 5f
0088
0089     CALGN(  ands    ip, r0, #31     )
0090     CALGN(  rsb r3, ip, #32     )
0091     CALGN(  sbcsne  r4, r3, r2      )  @ C is always set here
0092     CALGN(  bcs 2f          )
0093     CALGN(  adr r4, 6f          )
0094     CALGN(  subs    r2, r2, r3      )  @ C gets set
0095     CALGN(  add pc, r4, ip      )
0096
0097     PLD(    pld [r1, #0]        )
0098 2:  PLD(    subs    r2, r2, #96     )
0099     PLD(    pld [r1, #28]       )
0100     PLD(    blt 4f          )
0101     PLD(    pld [r1, #60]       )
0102     PLD(    pld [r1, #92]       )
0103
0104 3:  PLD(    pld [r1, #124]      )
0105 4:      ldr8w   r1, r3, r4, r5, r6, r8, r9, ip, lr, abort=20f
0106         subs    r2, r2, #32
0107         str8w   r0, r3, r4, r5, r6, r8, r9, ip, lr, abort=20f
0108         bge 3b
0109     PLD(    cmn r2, #96         )
0110     PLD(    bge 4b          )
0111
0112 5:      ands    ip, r2, #28
0113         rsb ip, ip, #32
0114 #if LDR1W_SHIFT > 0
0115         lsl ip, ip, #LDR1W_SHIFT
0116 #endif
0117         addne   pc, pc, ip      @ C is always clear here
0118         b   7f
0119 6:
0120         .rept   (1 << LDR1W_SHIFT)
0121         W(nop)
0122         .endr
0123         ldr1w   r1, r3, abort=20f
0124         ldr1w   r1, r4, abort=20f
0125         ldr1w   r1, r5, abort=20f
0126         ldr1w   r1, r6, abort=20f
0127         ldr1w   r1, r8, abort=20f
0128         ldr1w   r1, r9, abort=20f
0129         ldr1w   r1, lr, abort=20f
0130
0131 #if LDR1W_SHIFT < STR1W_SHIFT
0132         lsl ip, ip, #STR1W_SHIFT - LDR1W_SHIFT
0133 #elif LDR1W_SHIFT > STR1W_SHIFT
0134         lsr ip, ip, #LDR1W_SHIFT - STR1W_SHIFT
0135 #endif
0136         add pc, pc, ip
0137         nop
0138         .rept   (1 << STR1W_SHIFT)
0139         W(nop)
0140         .endr
0141         str1w   r0, r3, abort=20f
0142         str1w   r0, r4, abort=20f
0143         str1w   r0, r5, abort=20f
0144         str1w   r0, r6, abort=20f
0145         str1w   r0, r8, abort=20f
0146         str1w   r0, r9, abort=20f
0147         str1w   r0, lr, abort=20f
0148
0149     CALGN(  bcs 2b          )
0150
0151 7:      ldmfd   sp!, {r5, r6, r8, r9}
0152
0153 8:      movs    r2, r2, lsl #31
0154         ldr1b   r1, r3, ne, abort=21f
0155         ldr1b   r1, r4, cs, abort=21f
0156         ldr1b   r1, ip, cs, abort=21f
0157         str1b   r0, r3, ne, abort=21f
0158         str1b   r0, r4, cs, abort=21f
0159         str1b   r0, ip, cs, abort=21f
0160
0161         exit    r4, UNWIND(fpreg,) pc
0162
0163 9:      rsb ip, ip, #4
0164         cmp ip, #2
0165         ldr1b   r1, r3, gt, abort=21f
0166         ldr1b   r1, r4, ge, abort=21f
0167         ldr1b   r1, lr, abort=21f
0168         str1b   r0, r3, gt, abort=21f
0169         str1b   r0, r4, ge, abort=21f
0170         subs    r2, r2, ip
0171         str1b   r0, lr, abort=21f
0172         blt 8b
0173         ands    ip, r1, #3
0174         beq 1b
0175
0176 10:     bic r1, r1, #3
0177         cmp ip, #2
0178         ldr1w   r1, lr, abort=21f
0179         beq 17f
0180         bgt 18f
0181
0182
0183         .macro  forward_copy_shift pull push
0184
0185         subs    r2, r2, #28
0186         blt 14f
0187
0188     CALGN(  ands    ip, r0, #31     )
0189     CALGN(  rsb ip, ip, #32     )
0190     CALGN(  sbcsne  r4, ip, r2      )  @ C is always set here
0191     CALGN(  subcc   r2, r2, ip      )
0192     CALGN(  bcc 15f         )
0193
0194 11:     stmfd   sp!, {r5, r6, r8 - r10}
0195
0196     PLD(    pld [r1, #0]        )
0197     PLD(    subs    r2, r2, #96     )
0198     PLD(    pld [r1, #28]       )
0199     PLD(    blt 13f         )
0200     PLD(    pld [r1, #60]       )
0201     PLD(    pld [r1, #92]       )
0202
0203 12: PLD(    pld [r1, #124]      )
0204 13:     ldr4w   r1, r4, r5, r6, r8, abort=19f
0205         mov r3, lr, lspull #\pull
0206         subs    r2, r2, #32
0207         ldr4w   r1, r9, r10, ip, lr, abort=19f
0208         orr r3, r3, r4, lspush #\push
0209         mov r4, r4, lspull #\pull
0210         orr r4, r4, r5, lspush #\push
0211         mov r5, r5, lspull #\pull
0212         orr r5, r5, r6, lspush #\push
0213         mov r6, r6, lspull #\pull
0214         orr r6, r6, r8, lspush #\push
0215         mov r8, r8, lspull #\pull
0216         orr r8, r8, r9, lspush #\push
0217         mov r9, r9, lspull #\pull
0218         orr r9, r9, r10, lspush #\push
0219         mov r10, r10, lspull #\pull
0220         orr r10, r10, ip, lspush #\push
0221         mov ip, ip, lspull #\pull
0222         orr ip, ip, lr, lspush #\push
0223         str8w   r0, r3, r4, r5, r6, r8, r9, r10, ip, abort=19f
0224         bge 12b
0225     PLD(    cmn r2, #96         )
0226     PLD(    bge 13b         )
0227
0228         ldmfd   sp!, {r5, r6, r8 - r10}
0229
0230 14:     ands    ip, r2, #28
0231         beq 16f
0232
0233 15:     mov r3, lr, lspull #\pull
0234         ldr1w   r1, lr, abort=21f
0235         subs    ip, ip, #4
0236         orr r3, r3, lr, lspush #\push
0237         str1w   r0, r3, abort=21f
0238         bgt 15b
0239     CALGN(  cmp r2, #0          )
0240     CALGN(  bge 11b         )
0241
0242 16:     sub r1, r1, #(\push / 8)
0243         b   8b
0244
0245         .endm
0246
0247
0248         forward_copy_shift  pull=8  push=24
0249
0250 17:     forward_copy_shift  pull=16 push=16
0251
0252 18:     forward_copy_shift  pull=24 push=8
0253
0254     UNWIND( .fnend              )
0255
0256 /*
0257  * Abort preamble and completion macros.
0258  * If a fixup handler is required then those macros must surround it.
0259  * It is assumed that the fixup code will handle the private part of
0260  * the exit macro.
0261  */
0262
0263     .macro  copy_abort_preamble
0264 19: ldmfd   sp!, {r5, r6, r8 - r10}
0265     b   21f
0266 20: ldmfd   sp!, {r5, r6, r8, r9}
0267 21:
0268     .endm
0269
0270     .macro  copy_abort_end
0271     ldmfd   sp!, {r4, UNWIND(fpreg,) pc}
0272     .endm
0273