x86/lib/copy_mc_64.S

0001 /* SPDX-License-Identifier: GPL-2.0-only */
0002 /* Copyright(c) 2016-2020 Intel Corporation. All rights reserved. */
0003
0004 #include <linux/linkage.h>
0005 #include <asm/asm.h>
0006
0007 #ifndef CONFIG_UML
0008
0009 #ifdef CONFIG_X86_MCE
0010
0011 /*
0012  * copy_mc_fragile - copy memory with indication if an exception / fault happened
0013  *
0014  * The 'fragile' version is opted into by platform quirks and takes
0015  * pains to avoid unrecoverable corner cases like 'fast-string'
0016  * instruction sequences, and consuming poison across a cacheline
0017  * boundary. The non-fragile version is equivalent to memcpy()
0018  * regardless of CPU machine-check-recovery capability.
0019  */
0020 SYM_FUNC_START(copy_mc_fragile)
0021     cmpl $8, %edx
0022     /* Less than 8 bytes? Go to byte copy loop */
0023     jb .L_no_whole_words
0024
0025     /* Check for bad alignment of source */
0026     testl $7, %esi
0027     /* Already aligned */
0028     jz .L_8byte_aligned
0029
0030     /* Copy one byte at a time until source is 8-byte aligned */
0031     movl %esi, %ecx
0032     andl $7, %ecx
0033     subl $8, %ecx
0034     negl %ecx
0035     subl %ecx, %edx
0036 .L_read_leading_bytes:
0037     movb (%rsi), %al
0038 .L_write_leading_bytes:
0039     movb %al, (%rdi)
0040     incq %rsi
0041     incq %rdi
0042     decl %ecx
0043     jnz .L_read_leading_bytes
0044
0045 .L_8byte_aligned:
0046     movl %edx, %ecx
0047     andl $7, %edx
0048     shrl $3, %ecx
0049     jz .L_no_whole_words
0050
0051 .L_read_words:
0052     movq (%rsi), %r8
0053 .L_write_words:
0054     movq %r8, (%rdi)
0055     addq $8, %rsi
0056     addq $8, %rdi
0057     decl %ecx
0058     jnz .L_read_words
0059
0060     /* Any trailing bytes? */
0061 .L_no_whole_words:
0062     andl %edx, %edx
0063     jz .L_done_memcpy_trap
0064
0065     /* Copy trailing bytes */
0066     movl %edx, %ecx
0067 .L_read_trailing_bytes:
0068     movb (%rsi), %al
0069 .L_write_trailing_bytes:
0070     movb %al, (%rdi)
0071     incq %rsi
0072     incq %rdi
0073     decl %ecx
0074     jnz .L_read_trailing_bytes
0075
0076     /* Copy successful. Return zero */
0077 .L_done_memcpy_trap:
0078     xorl %eax, %eax
0079 .L_done:
0080     RET
0081
0082     /*
0083      * Return number of bytes not copied for any failure. Note that
0084      * there is no "tail" handling since the source buffer is 8-byte
0085      * aligned and poison is cacheline aligned.
0086      */
0087 .E_read_words:
0088     shll    $3, %ecx
0089 .E_leading_bytes:
0090     addl    %edx, %ecx
0091 .E_trailing_bytes:
0092     mov %ecx, %eax
0093     jmp .L_done
0094
0095     /*
0096      * For write fault handling, given the destination is unaligned,
0097      * we handle faults on multi-byte writes with a byte-by-byte
0098      * copy up to the write-protected page.
0099      */
0100 .E_write_words:
0101     shll    $3, %ecx
0102     addl    %edx, %ecx
0103     movl    %ecx, %edx
0104     jmp copy_mc_fragile_handle_tail
0105
0106     _ASM_EXTABLE_TYPE(.L_read_leading_bytes, .E_leading_bytes, EX_TYPE_DEFAULT_MCE_SAFE)
0107     _ASM_EXTABLE_TYPE(.L_read_words, .E_read_words, EX_TYPE_DEFAULT_MCE_SAFE)
0108     _ASM_EXTABLE_TYPE(.L_read_trailing_bytes, .E_trailing_bytes, EX_TYPE_DEFAULT_MCE_SAFE)
0109     _ASM_EXTABLE(.L_write_leading_bytes, .E_leading_bytes)
0110     _ASM_EXTABLE(.L_write_words, .E_write_words)
0111     _ASM_EXTABLE(.L_write_trailing_bytes, .E_trailing_bytes)
0112
0113 SYM_FUNC_END(copy_mc_fragile)
0114 #endif /* CONFIG_X86_MCE */
0115
0116 /*
0117  * copy_mc_enhanced_fast_string - memory copy with exception handling
0118  *
0119  * Fast string copy + fault / exception handling. If the CPU does
0120  * support machine check exception recovery, but does not support
0121  * recovering from fast-string exceptions then this CPU needs to be
0122  * added to the copy_mc_fragile_key set of quirks. Otherwise, absent any
0123  * machine check recovery support this version should be no slower than
0124  * standard memcpy.
0125  */
0126 SYM_FUNC_START(copy_mc_enhanced_fast_string)
0127     movq %rdi, %rax
0128     movq %rdx, %rcx
0129 .L_copy:
0130     rep movsb
0131     /* Copy successful. Return zero */
0132     xorl %eax, %eax
0133     RET
0134
0135 .E_copy:
0136     /*
0137      * On fault %rcx is updated such that the copy instruction could
0138      * optionally be restarted at the fault position, i.e. it
0139      * contains 'bytes remaining'. A non-zero return indicates error
0140      * to copy_mc_generic() users, or indicate short transfers to
0141      * user-copy routines.
0142      */
0143     movq %rcx, %rax
0144     RET
0145
0146     _ASM_EXTABLE_TYPE(.L_copy, .E_copy, EX_TYPE_DEFAULT_MCE_SAFE)
0147
0148 SYM_FUNC_END(copy_mc_enhanced_fast_string)
0149 #endif /* !CONFIG_UML */