powerpc/lib/mem_64.S

0001 /* SPDX-License-Identifier: GPL-2.0-or-later */
0002 /*
0003  * String handling functions for PowerPC.
0004  *
0005  * Copyright (C) 1996 Paul Mackerras.
0006  */
0007 #include <asm/processor.h>
0008 #include <asm/errno.h>
0009 #include <asm/ppc_asm.h>
0010 #include <asm/export.h>
0011 #include <asm/kasan.h>
0012
0013 #ifndef CONFIG_KASAN
0014 _GLOBAL(__memset16)
0015     rlwimi  r4,r4,16,0,15
0016     /* fall through */
0017
0018 _GLOBAL(__memset32)
0019     rldimi  r4,r4,32,0
0020     /* fall through */
0021
0022 _GLOBAL(__memset64)
0023     neg r0,r3
0024     andi.   r0,r0,7
0025     cmplw   cr1,r5,r0
0026     b   .Lms
0027 EXPORT_SYMBOL(__memset16)
0028 EXPORT_SYMBOL(__memset32)
0029 EXPORT_SYMBOL(__memset64)
0030 #endif
0031
0032 _GLOBAL_KASAN(memset)
0033     neg r0,r3
0034     rlwimi  r4,r4,8,16,23
0035     andi.   r0,r0,7         /* # bytes to be 8-byte aligned */
0036     rlwimi  r4,r4,16,0,15
0037     cmplw   cr1,r5,r0       /* do we get that far? */
0038     rldimi  r4,r4,32,0
0039 .Lms:   PPC_MTOCRF(1,r0)
0040     mr  r6,r3
0041     blt cr1,8f
0042     beq 3f          /* if already 8-byte aligned */
0043     subf    r5,r0,r5
0044     bf  31,1f
0045     stb r4,0(r6)
0046     addi    r6,r6,1
0047 1:  bf  30,2f
0048     sth r4,0(r6)
0049     addi    r6,r6,2
0050 2:  bf  29,3f
0051     stw r4,0(r6)
0052     addi    r6,r6,4
0053 3:  srdi.   r0,r5,6
0054     clrldi  r5,r5,58
0055     mtctr   r0
0056     beq 5f
0057     .balign 16
0058 4:  std r4,0(r6)
0059     std r4,8(r6)
0060     std r4,16(r6)
0061     std r4,24(r6)
0062     std r4,32(r6)
0063     std r4,40(r6)
0064     std r4,48(r6)
0065     std r4,56(r6)
0066     addi    r6,r6,64
0067     bdnz    4b
0068 5:  srwi.   r0,r5,3
0069     clrlwi  r5,r5,29
0070     PPC_MTOCRF(1,r0)
0071     beq 8f
0072     bf  29,6f
0073     std r4,0(r6)
0074     std r4,8(r6)
0075     std r4,16(r6)
0076     std r4,24(r6)
0077     addi    r6,r6,32
0078 6:  bf  30,7f
0079     std r4,0(r6)
0080     std r4,8(r6)
0081     addi    r6,r6,16
0082 7:  bf  31,8f
0083     std r4,0(r6)
0084     addi    r6,r6,8
0085 8:  cmpwi   r5,0
0086     PPC_MTOCRF(1,r5)
0087     beqlr
0088     bf  29,9f
0089     stw r4,0(r6)
0090     addi    r6,r6,4
0091 9:  bf  30,10f
0092     sth r4,0(r6)
0093     addi    r6,r6,2
0094 10: bflr    31
0095     stb r4,0(r6)
0096     blr
0097 EXPORT_SYMBOL(memset)
0098 EXPORT_SYMBOL_KASAN(memset)
0099
0100 _GLOBAL_TOC_KASAN(memmove)
0101     cmplw   0,r3,r4
0102     bgt backwards_memcpy
0103     b   memcpy
0104
0105 _GLOBAL(backwards_memcpy)
0106     rlwinm. r7,r5,32-3,3,31     /* r0 = r5 >> 3 */
0107     add r6,r3,r5
0108     add r4,r4,r5
0109     beq 2f
0110     andi.   r0,r6,3
0111     mtctr   r7
0112     bne 5f
0113     .balign 16
0114 1:  lwz r7,-4(r4)
0115     lwzu    r8,-8(r4)
0116     stw r7,-4(r6)
0117     stwu    r8,-8(r6)
0118     bdnz    1b
0119     andi.   r5,r5,7
0120 2:  cmplwi  0,r5,4
0121     blt 3f
0122     lwzu    r0,-4(r4)
0123     subi    r5,r5,4
0124     stwu    r0,-4(r6)
0125 3:  cmpwi   0,r5,0
0126     beqlr
0127     mtctr   r5
0128 4:  lbzu    r0,-1(r4)
0129     stbu    r0,-1(r6)
0130     bdnz    4b
0131     blr
0132 5:  mtctr   r0
0133 6:  lbzu    r7,-1(r4)
0134     stbu    r7,-1(r6)
0135     bdnz    6b
0136     subf    r5,r0,r5
0137     rlwinm. r7,r5,32-3,3,31
0138     beq 2b
0139     mtctr   r7
0140     b   1b
0141 EXPORT_SYMBOL(memmove)
0142 EXPORT_SYMBOL_KASAN(memmove)