s390/lib/mem.S

0001 /* SPDX-License-Identifier: GPL-2.0 */
0002 /*
0003  * String handling functions.
0004  *
0005  * Copyright IBM Corp. 2012
0006  */
0007
0008 #include <linux/linkage.h>
0009 #include <asm/export.h>
0010 #include <asm/nospec-insn.h>
0011
0012     GEN_BR_THUNK %r14
0013
0014 /*
0015  * void *memmove(void *dest, const void *src, size_t n)
0016  */
0017 WEAK(memmove)
0018 ENTRY(__memmove)
0019     ltgr    %r4,%r4
0020     lgr %r1,%r2
0021     jz  .Lmemmove_exit
0022     aghi    %r4,-1
0023     clgr    %r2,%r3
0024     jnh .Lmemmove_forward
0025     la  %r5,1(%r4,%r3)
0026     clgr    %r2,%r5
0027     jl  .Lmemmove_reverse
0028 .Lmemmove_forward:
0029     srlg    %r0,%r4,8
0030     ltgr    %r0,%r0
0031     jz  .Lmemmove_forward_remainder
0032 .Lmemmove_forward_loop:
0033     mvc 0(256,%r1),0(%r3)
0034     la  %r1,256(%r1)
0035     la  %r3,256(%r3)
0036     brctg   %r0,.Lmemmove_forward_loop
0037 .Lmemmove_forward_remainder:
0038     larl    %r5,.Lmemmove_mvc
0039     ex  %r4,0(%r5)
0040 .Lmemmove_exit:
0041     BR_EX   %r14
0042 .Lmemmove_reverse:
0043     ic  %r0,0(%r4,%r3)
0044     stc %r0,0(%r4,%r1)
0045     brctg   %r4,.Lmemmove_reverse
0046     ic  %r0,0(%r4,%r3)
0047     stc %r0,0(%r4,%r1)
0048     BR_EX   %r14
0049 .Lmemmove_mvc:
0050     mvc 0(1,%r1),0(%r3)
0051 ENDPROC(__memmove)
0052 EXPORT_SYMBOL(memmove)
0053
0054 /*
0055  * memset implementation
0056  *
0057  * This code corresponds to the C construct below. We do distinguish
0058  * between clearing (c == 0) and setting a memory array (c != 0) simply
0059  * because nearly all memset invocations in the kernel clear memory and
0060  * the xc instruction is preferred in such cases.
0061  *
0062  * void *memset(void *s, int c, size_t n)
0063  * {
0064  *  if (likely(c == 0))
0065  *      return __builtin_memset(s, 0, n);
0066  *  return __builtin_memset(s, c, n);
0067  * }
0068  */
0069 WEAK(memset)
0070 ENTRY(__memset)
0071     ltgr    %r4,%r4
0072     jz  .Lmemset_exit
0073     ltgr    %r3,%r3
0074     jnz .Lmemset_fill
0075     aghi    %r4,-1
0076     srlg    %r3,%r4,8
0077     ltgr    %r3,%r3
0078     lgr %r1,%r2
0079     jz  .Lmemset_clear_remainder
0080 .Lmemset_clear_loop:
0081     xc  0(256,%r1),0(%r1)
0082     la  %r1,256(%r1)
0083     brctg   %r3,.Lmemset_clear_loop
0084 .Lmemset_clear_remainder:
0085     larl    %r3,.Lmemset_xc
0086     ex  %r4,0(%r3)
0087 .Lmemset_exit:
0088     BR_EX   %r14
0089 .Lmemset_fill:
0090     cghi    %r4,1
0091     lgr %r1,%r2
0092     je  .Lmemset_fill_exit
0093     aghi    %r4,-2
0094     srlg    %r5,%r4,8
0095     ltgr    %r5,%r5
0096     jz  .Lmemset_fill_remainder
0097 .Lmemset_fill_loop:
0098     stc %r3,0(%r1)
0099     mvc 1(255,%r1),0(%r1)
0100     la  %r1,256(%r1)
0101     brctg   %r5,.Lmemset_fill_loop
0102 .Lmemset_fill_remainder:
0103     stc %r3,0(%r1)
0104     larl    %r5,.Lmemset_mvc
0105     ex  %r4,0(%r5)
0106     BR_EX   %r14
0107 .Lmemset_fill_exit:
0108     stc %r3,0(%r1)
0109     BR_EX   %r14
0110 .Lmemset_xc:
0111     xc  0(1,%r1),0(%r1)
0112 .Lmemset_mvc:
0113     mvc 1(1,%r1),0(%r1)
0114 ENDPROC(__memset)
0115 EXPORT_SYMBOL(memset)
0116
0117 /*
0118  * memcpy implementation
0119  *
0120  * void *memcpy(void *dest, const void *src, size_t n)
0121  */
0122 WEAK(memcpy)
0123 ENTRY(__memcpy)
0124     ltgr    %r4,%r4
0125     jz  .Lmemcpy_exit
0126     aghi    %r4,-1
0127     srlg    %r5,%r4,8
0128     ltgr    %r5,%r5
0129     lgr %r1,%r2
0130     jnz .Lmemcpy_loop
0131 .Lmemcpy_remainder:
0132     larl    %r5,.Lmemcpy_mvc
0133     ex  %r4,0(%r5)
0134 .Lmemcpy_exit:
0135     BR_EX   %r14
0136 .Lmemcpy_loop:
0137     mvc 0(256,%r1),0(%r3)
0138     la  %r1,256(%r1)
0139     la  %r3,256(%r3)
0140     brctg   %r5,.Lmemcpy_loop
0141     j   .Lmemcpy_remainder
0142 .Lmemcpy_mvc:
0143     mvc 0(1,%r1),0(%r3)
0144 ENDPROC(__memcpy)
0145 EXPORT_SYMBOL(memcpy)
0146
0147 /*
0148  * __memset16/32/64
0149  *
0150  * void *__memset16(uint16_t *s, uint16_t v, size_t count)
0151  * void *__memset32(uint32_t *s, uint32_t v, size_t count)
0152  * void *__memset64(uint64_t *s, uint64_t v, size_t count)
0153  */
0154 .macro __MEMSET bits,bytes,insn
0155 ENTRY(__memset\bits)
0156     ltgr    %r4,%r4
0157     jz  .L__memset_exit\bits
0158     cghi    %r4,\bytes
0159     je  .L__memset_store\bits
0160     aghi    %r4,-(\bytes+1)
0161     srlg    %r5,%r4,8
0162     ltgr    %r5,%r5
0163     lgr %r1,%r2
0164     jz  .L__memset_remainder\bits
0165 .L__memset_loop\bits:
0166     \insn   %r3,0(%r1)
0167     mvc \bytes(256-\bytes,%r1),0(%r1)
0168     la  %r1,256(%r1)
0169     brctg   %r5,.L__memset_loop\bits
0170 .L__memset_remainder\bits:
0171     \insn   %r3,0(%r1)
0172     larl    %r5,.L__memset_mvc\bits
0173     ex  %r4,0(%r5)
0174     BR_EX   %r14
0175 .L__memset_store\bits:
0176     \insn   %r3,0(%r2)
0177 .L__memset_exit\bits:
0178     BR_EX   %r14
0179 .L__memset_mvc\bits:
0180     mvc \bytes(1,%r1),0(%r1)
0181 ENDPROC(__memset\bits)
0182 .endm
0183
0184 __MEMSET 16,2,sth
0185 EXPORT_SYMBOL(__memset16)
0186
0187 __MEMSET 32,4,st
0188 EXPORT_SYMBOL(__memset32)
0189
0190 __MEMSET 64,8,stg
0191 EXPORT_SYMBOL(__memset64)