powerpc/lib/string_64.S

0001 /* SPDX-License-Identifier: GPL-2.0-or-later */
0002 /*
0003  *
0004  * Copyright (C) IBM Corporation, 2012
0005  *
0006  * Author: Anton Blanchard <anton@au.ibm.com>
0007  */
0008
0009 #include <asm/ppc_asm.h>
0010 #include <asm/linkage.h>
0011 #include <asm/asm-offsets.h>
0012 #include <asm/export.h>
0013
0014     .section    ".toc","aw"
0015 PPC64_CACHES:
0016     .tc     ppc64_caches[TC],ppc64_caches
0017     .section    ".text"
0018
0019 /**
0020  * __arch_clear_user: - Zero a block of memory in user space, with less checking.
0021  * @to:   Destination address, in user space.
0022  * @n:    Number of bytes to zero.
0023  *
0024  * Zero a block of memory in user space.  Caller must check
0025  * the specified block with access_ok() before calling this function.
0026  *
0027  * Returns number of bytes that could not be cleared.
0028  * On success, this will be zero.
0029  */
0030
0031     .macro err1
0032 100:
0033     EX_TABLE(100b,.Ldo_err1)
0034     .endm
0035
0036     .macro err2
0037 200:
0038     EX_TABLE(200b,.Ldo_err2)
0039     .endm
0040
0041     .macro err3
0042 300:
0043     EX_TABLE(300b,.Ldo_err3)
0044     .endm
0045
0046 .Ldo_err1:
0047     mr  r3,r8
0048
0049 .Ldo_err2:
0050     mtctr   r4
0051 1:
0052 err3;   stb r0,0(r3)
0053     addi    r3,r3,1
0054     addi    r4,r4,-1
0055     bdnz    1b
0056
0057 .Ldo_err3:
0058     mr  r3,r4
0059     blr
0060
0061 _GLOBAL_TOC(__arch_clear_user)
0062     cmpdi   r4,32
0063     neg r6,r3
0064     li  r0,0
0065     blt .Lshort_clear
0066     mr  r8,r3
0067     mtocrf  0x01,r6
0068     clrldi  r6,r6,(64-3)
0069
0070     /* Get the destination 8 byte aligned */
0071     bf  cr7*4+3,1f
0072 err1;   stb r0,0(r3)
0073     addi    r3,r3,1
0074
0075 1:  bf  cr7*4+2,2f
0076 err1;   sth r0,0(r3)
0077     addi    r3,r3,2
0078
0079 2:  bf  cr7*4+1,3f
0080 err1;   stw r0,0(r3)
0081     addi    r3,r3,4
0082
0083 3:  sub r4,r4,r6
0084
0085     cmpdi   r4,32
0086     cmpdi   cr1,r4,512
0087     blt .Lshort_clear
0088     bgt cr1,.Llong_clear
0089
0090 .Lmedium_clear:
0091     srdi    r6,r4,5
0092     mtctr   r6
0093
0094     /* Do 32 byte chunks */
0095 4:
0096 err2;   std r0,0(r3)
0097 err2;   std r0,8(r3)
0098 err2;   std r0,16(r3)
0099 err2;   std r0,24(r3)
0100     addi    r3,r3,32
0101     addi    r4,r4,-32
0102     bdnz    4b
0103
0104 .Lshort_clear:
0105     /* up to 31 bytes to go */
0106     cmpdi   r4,16
0107     blt 6f
0108 err2;   std r0,0(r3)
0109 err2;   std r0,8(r3)
0110     addi    r3,r3,16
0111     addi    r4,r4,-16
0112
0113     /* Up to 15 bytes to go */
0114 6:  mr  r8,r3
0115     clrldi  r4,r4,(64-4)
0116     mtocrf  0x01,r4
0117     bf  cr7*4+0,7f
0118 err1;   std r0,0(r3)
0119     addi    r3,r3,8
0120
0121 7:  bf  cr7*4+1,8f
0122 err1;   stw r0,0(r3)
0123     addi    r3,r3,4
0124
0125 8:  bf  cr7*4+2,9f
0126 err1;   sth r0,0(r3)
0127     addi    r3,r3,2
0128
0129 9:  bf  cr7*4+3,10f
0130 err1;   stb r0,0(r3)
0131
0132 10: li  r3,0
0133     blr
0134
0135 .Llong_clear:
0136     ld  r5,PPC64_CACHES@toc(r2)
0137
0138     bf  cr7*4+0,11f
0139 err2;   std r0,0(r3)
0140     addi    r3,r3,8
0141     addi    r4,r4,-8
0142
0143     /* Destination is 16 byte aligned, need to get it cache block aligned */
0144 11: lwz r7,DCACHEL1LOGBLOCKSIZE(r5)
0145     lwz r9,DCACHEL1BLOCKSIZE(r5)
0146
0147     /*
0148      * With worst case alignment the long clear loop takes a minimum
0149      * of 1 byte less than 2 cachelines.
0150      */
0151     sldi    r10,r9,2
0152     cmpd    r4,r10
0153     blt .Lmedium_clear
0154
0155     neg r6,r3
0156     addi    r10,r9,-1
0157     and.    r5,r6,r10
0158     beq 13f
0159
0160     srdi    r6,r5,4
0161     mtctr   r6
0162     mr  r8,r3
0163 12:
0164 err1;   std r0,0(r3)
0165 err1;   std r0,8(r3)
0166     addi    r3,r3,16
0167     bdnz    12b
0168
0169     sub r4,r4,r5
0170
0171 13: srd r6,r4,r7
0172     mtctr   r6
0173     mr  r8,r3
0174 14:
0175 err1;   dcbz    0,r3
0176     add r3,r3,r9
0177     bdnz    14b
0178
0179     and r4,r4,r10
0180
0181     cmpdi   r4,32
0182     blt .Lshort_clear
0183     b   .Lmedium_clear
0184 EXPORT_SYMBOL(__arch_clear_user)