include/asm/string_64.h

0001 /* SPDX-License-Identifier: GPL-2.0 */
0002 #ifndef _ASM_X86_STRING_64_H
0003 #define _ASM_X86_STRING_64_H
0004
0005 #ifdef __KERNEL__
0006 #include <linux/jump_label.h>
0007
0008 /* Written 2002 by Andi Kleen */
0009
0010 /* Even with __builtin_ the compiler may decide to use the out of line
0011    function. */
0012
0013 #define __HAVE_ARCH_MEMCPY 1
0014 extern void *memcpy(void *to, const void *from, size_t len);
0015 extern void *__memcpy(void *to, const void *from, size_t len);
0016
0017 #define __HAVE_ARCH_MEMSET
0018 void *memset(void *s, int c, size_t n);
0019 void *__memset(void *s, int c, size_t n);
0020
0021 #define __HAVE_ARCH_MEMSET16
0022 static inline void *memset16(uint16_t *s, uint16_t v, size_t n)
0023 {
0024     long d0, d1;
0025     asm volatile("rep\n\t"
0026              "stosw"
0027              : "=&c" (d0), "=&D" (d1)
0028              : "a" (v), "1" (s), "0" (n)
0029              : "memory");
0030     return s;
0031 }
0032
0033 #define __HAVE_ARCH_MEMSET32
0034 static inline void *memset32(uint32_t *s, uint32_t v, size_t n)
0035 {
0036     long d0, d1;
0037     asm volatile("rep\n\t"
0038              "stosl"
0039              : "=&c" (d0), "=&D" (d1)
0040              : "a" (v), "1" (s), "0" (n)
0041              : "memory");
0042     return s;
0043 }
0044
0045 #define __HAVE_ARCH_MEMSET64
0046 static inline void *memset64(uint64_t *s, uint64_t v, size_t n)
0047 {
0048     long d0, d1;
0049     asm volatile("rep\n\t"
0050              "stosq"
0051              : "=&c" (d0), "=&D" (d1)
0052              : "a" (v), "1" (s), "0" (n)
0053              : "memory");
0054     return s;
0055 }
0056
0057 #define __HAVE_ARCH_MEMMOVE
0058 void *memmove(void *dest, const void *src, size_t count);
0059 void *__memmove(void *dest, const void *src, size_t count);
0060
0061 int memcmp(const void *cs, const void *ct, size_t count);
0062 size_t strlen(const char *s);
0063 char *strcpy(char *dest, const char *src);
0064 char *strcat(char *dest, const char *src);
0065 int strcmp(const char *cs, const char *ct);
0066
0067 #if defined(CONFIG_KASAN) && !defined(__SANITIZE_ADDRESS__)
0068
0069 /*
0070  * For files that not instrumented (e.g. mm/slub.c) we
0071  * should use not instrumented version of mem* functions.
0072  */
0073
0074 #undef memcpy
0075 #define memcpy(dst, src, len) __memcpy(dst, src, len)
0076 #define memmove(dst, src, len) __memmove(dst, src, len)
0077 #define memset(s, c, n) __memset(s, c, n)
0078
0079 #ifndef __NO_FORTIFY
0080 #define __NO_FORTIFY /* FORTIFY_SOURCE uses __builtin_memcpy, etc. */
0081 #endif
0082
0083 #endif
0084
0085 #ifdef CONFIG_ARCH_HAS_UACCESS_FLUSHCACHE
0086 #define __HAVE_ARCH_MEMCPY_FLUSHCACHE 1
0087 void __memcpy_flushcache(void *dst, const void *src, size_t cnt);
0088 static __always_inline void memcpy_flushcache(void *dst, const void *src, size_t cnt)
0089 {
0090     if (__builtin_constant_p(cnt)) {
0091         switch (cnt) {
0092             case 4:
0093                 asm ("movntil %1, %0" : "=m"(*(u32 *)dst) : "r"(*(u32 *)src));
0094                 return;
0095             case 8:
0096                 asm ("movntiq %1, %0" : "=m"(*(u64 *)dst) : "r"(*(u64 *)src));
0097                 return;
0098             case 16:
0099                 asm ("movntiq %1, %0" : "=m"(*(u64 *)dst) : "r"(*(u64 *)src));
0100                 asm ("movntiq %1, %0" : "=m"(*(u64 *)(dst + 8)) : "r"(*(u64 *)(src + 8)));
0101                 return;
0102         }
0103     }
0104     __memcpy_flushcache(dst, src, cnt);
0105 }
0106 #endif
0107
0108 #endif /* __KERNEL__ */
0109
0110 #endif /* _ASM_X86_STRING_64_H */