include/asm/rwonce.h

0001 /* SPDX-License-Identifier: GPL-2.0 */
0002 /*
0003  * Copyright (C) 2020 Google LLC.
0004  */
0005 #ifndef __ASM_RWONCE_H
0006 #define __ASM_RWONCE_H
0007
0008 #if defined(CONFIG_LTO) && !defined(__ASSEMBLY__)
0009
0010 #include <linux/compiler_types.h>
0011 #include <asm/alternative-macros.h>
0012
0013 #ifndef BUILD_VDSO
0014
0015 #ifdef CONFIG_AS_HAS_LDAPR
0016 #define __LOAD_RCPC(sfx, regs...)                   \
0017     ALTERNATIVE(                            \
0018         "ldar"  #sfx "\t" #regs,                \
0019         ".arch_extension rcpc\n"                \
0020         "ldapr" #sfx "\t" #regs,                \
0021     ARM64_HAS_LDAPR)
0022 #else
0023 #define __LOAD_RCPC(sfx, regs...)   "ldar" #sfx "\t" #regs
0024 #endif /* CONFIG_AS_HAS_LDAPR */
0025
0026 /*
0027  * When building with LTO, there is an increased risk of the compiler
0028  * converting an address dependency headed by a READ_ONCE() invocation
0029  * into a control dependency and consequently allowing for harmful
0030  * reordering by the CPU.
0031  *
0032  * Ensure that such transformations are harmless by overriding the generic
0033  * READ_ONCE() definition with one that provides RCpc acquire semantics
0034  * when building with LTO.
0035  */
0036 #define __READ_ONCE(x)                          \
0037 ({                                  \
0038     typeof(&(x)) __x = &(x);                    \
0039     int atomic = 1;                         \
0040     union { __unqual_scalar_typeof(*__x) __val; char __c[1]; } __u; \
0041     switch (sizeof(x)) {                        \
0042     case 1:                             \
0043         asm volatile(__LOAD_RCPC(b, %w0, %1)            \
0044             : "=r" (*(__u8 *)__u.__c)           \
0045             : "Q" (*__x) : "memory");           \
0046         break;                          \
0047     case 2:                             \
0048         asm volatile(__LOAD_RCPC(h, %w0, %1)            \
0049             : "=r" (*(__u16 *)__u.__c)          \
0050             : "Q" (*__x) : "memory");           \
0051         break;                          \
0052     case 4:                             \
0053         asm volatile(__LOAD_RCPC(, %w0, %1)         \
0054             : "=r" (*(__u32 *)__u.__c)          \
0055             : "Q" (*__x) : "memory");           \
0056         break;                          \
0057     case 8:                             \
0058         asm volatile(__LOAD_RCPC(, %0, %1)          \
0059             : "=r" (*(__u64 *)__u.__c)          \
0060             : "Q" (*__x) : "memory");           \
0061         break;                          \
0062     default:                            \
0063         atomic = 0;                     \
0064     }                               \
0065     atomic ? (typeof(*__x))__u.__val : (*(volatile typeof(__x))__x);\
0066 })
0067
0068 #endif  /* !BUILD_VDSO */
0069 #endif  /* CONFIG_LTO && !__ASSEMBLY__ */
0070
0071 #include <asm-generic/rwonce.h>
0072
0073 #endif  /* __ASM_RWONCE_H */