include/asm/synch.h

0001 /* SPDX-License-Identifier: GPL-2.0 */
0002 #ifndef _ASM_POWERPC_SYNCH_H
0003 #define _ASM_POWERPC_SYNCH_H
0004 #ifdef __KERNEL__
0005
0006 #include <asm/cputable.h>
0007 #include <asm/feature-fixups.h>
0008 #include <asm/ppc-opcode.h>
0009
0010 #ifndef __ASSEMBLY__
0011 extern unsigned int __start___lwsync_fixup, __stop___lwsync_fixup;
0012 extern void do_lwsync_fixups(unsigned long value, void *fixup_start,
0013                  void *fixup_end);
0014
0015 static inline void eieio(void)
0016 {
0017     if (IS_ENABLED(CONFIG_BOOKE))
0018         __asm__ __volatile__ ("mbar" : : : "memory");
0019     else
0020         __asm__ __volatile__ ("eieio" : : : "memory");
0021 }
0022
0023 static inline void isync(void)
0024 {
0025     __asm__ __volatile__ ("isync" : : : "memory");
0026 }
0027
0028 static inline void ppc_after_tlbiel_barrier(void)
0029 {
0030     asm volatile("ptesync": : :"memory");
0031     /*
0032      * POWER9, POWER10 need a cp_abort after tlbiel to ensure the copy is
0033      * invalidated correctly. If this is not done, the paste can take data
0034      * from the physical address that was translated at copy time.
0035      *
0036      * POWER9 in practice does not need this, because address spaces with
0037      * accelerators mapped will use tlbie (which does invalidate the copy)
0038      * to invalidate translations. It's not possible to limit POWER10 this
0039      * way due to local copy-paste.
0040      */
0041     asm volatile(ASM_FTR_IFSET(PPC_CP_ABORT, "", %0) : : "i" (CPU_FTR_ARCH_31) : "memory");
0042 }
0043 #endif /* __ASSEMBLY__ */
0044
0045 #if defined(__powerpc64__)
0046 #    define LWSYNC  lwsync
0047 #elif defined(CONFIG_E500)
0048 #    define LWSYNC                  \
0049     START_LWSYNC_SECTION(96);           \
0050     sync;                       \
0051     MAKE_LWSYNC_SECTION_ENTRY(96, __lwsync_fixup);
0052 #else
0053 #    define LWSYNC  sync
0054 #endif
0055
0056 #ifdef CONFIG_SMP
0057 #define __PPC_ACQUIRE_BARRIER               \
0058     START_LWSYNC_SECTION(97);           \
0059     isync;                      \
0060     MAKE_LWSYNC_SECTION_ENTRY(97, __lwsync_fixup);
0061 #define PPC_ACQUIRE_BARRIER  "\n" stringify_in_c(__PPC_ACQUIRE_BARRIER)
0062 #define PPC_RELEASE_BARRIER  stringify_in_c(LWSYNC) "\n"
0063 #define PPC_ATOMIC_ENTRY_BARRIER "\n" stringify_in_c(sync) "\n"
0064 #define PPC_ATOMIC_EXIT_BARRIER  "\n" stringify_in_c(sync) "\n"
0065 #else
0066 #define PPC_ACQUIRE_BARRIER
0067 #define PPC_RELEASE_BARRIER
0068 #define PPC_ATOMIC_ENTRY_BARRIER
0069 #define PPC_ATOMIC_EXIT_BARRIER
0070 #endif
0071
0072 #endif /* __KERNEL__ */
0073 #endif  /* _ASM_POWERPC_SYNCH_H */