x86/lib/copy_mc.c

0001 // SPDX-License-Identifier: GPL-2.0
0002 /* Copyright(c) 2016-2020 Intel Corporation. All rights reserved. */
0003
0004 #include <linux/jump_label.h>
0005 #include <linux/uaccess.h>
0006 #include <linux/export.h>
0007 #include <linux/string.h>
0008 #include <linux/types.h>
0009
0010 #include <asm/mce.h>
0011
0012 #ifdef CONFIG_X86_MCE
0013 static DEFINE_STATIC_KEY_FALSE(copy_mc_fragile_key);
0014
0015 void enable_copy_mc_fragile(void)
0016 {
0017     static_branch_inc(&copy_mc_fragile_key);
0018 }
0019 #define copy_mc_fragile_enabled (static_branch_unlikely(&copy_mc_fragile_key))
0020
0021 /*
0022  * Similar to copy_user_handle_tail, probe for the write fault point, or
0023  * source exception point.
0024  */
0025 __visible notrace unsigned long
0026 copy_mc_fragile_handle_tail(char *to, char *from, unsigned len)
0027 {
0028     for (; len; --len, to++, from++)
0029         if (copy_mc_fragile(to, from, 1))
0030             break;
0031     return len;
0032 }
0033 #else
0034 /*
0035  * No point in doing careful copying, or consulting a static key when
0036  * there is no #MC handler in the CONFIG_X86_MCE=n case.
0037  */
0038 void enable_copy_mc_fragile(void)
0039 {
0040 }
0041 #define copy_mc_fragile_enabled (0)
0042 #endif
0043
0044 unsigned long copy_mc_enhanced_fast_string(void *dst, const void *src, unsigned len);
0045
0046 /**
0047  * copy_mc_to_kernel - memory copy that handles source exceptions
0048  *
0049  * @dst:    destination address
0050  * @src:    source address
0051  * @len:    number of bytes to copy
0052  *
0053  * Call into the 'fragile' version on systems that benefit from avoiding
0054  * corner case poison consumption scenarios, For example, accessing
0055  * poison across 2 cachelines with a single instruction. Almost all
0056  * other uses case can use copy_mc_enhanced_fast_string() for a fast
0057  * recoverable copy, or fallback to plain memcpy.
0058  *
0059  * Return 0 for success, or number of bytes not copied if there was an
0060  * exception.
0061  */
0062 unsigned long __must_check copy_mc_to_kernel(void *dst, const void *src, unsigned len)
0063 {
0064     if (copy_mc_fragile_enabled)
0065         return copy_mc_fragile(dst, src, len);
0066     if (static_cpu_has(X86_FEATURE_ERMS))
0067         return copy_mc_enhanced_fast_string(dst, src, len);
0068     memcpy(dst, src, len);
0069     return 0;
0070 }
0071 EXPORT_SYMBOL_GPL(copy_mc_to_kernel);
0072
0073 unsigned long __must_check copy_mc_to_user(void *dst, const void *src, unsigned len)
0074 {
0075     unsigned long ret;
0076
0077     if (copy_mc_fragile_enabled) {
0078         __uaccess_begin();
0079         ret = copy_mc_fragile(dst, src, len);
0080         __uaccess_end();
0081         return ret;
0082     }
0083
0084     if (static_cpu_has(X86_FEATURE_ERMS)) {
0085         __uaccess_begin();
0086         ret = copy_mc_enhanced_fast_string(dst, src, len);
0087         __uaccess_end();
0088         return ret;
0089     }
0090
0091     return copy_user_generic(dst, src, len);
0092 }