include/asm/mwait.h

0001 /* SPDX-License-Identifier: GPL-2.0 */
0002 #ifndef _ASM_X86_MWAIT_H
0003 #define _ASM_X86_MWAIT_H
0004
0005 #include <linux/sched.h>
0006 #include <linux/sched/idle.h>
0007
0008 #include <asm/cpufeature.h>
0009 #include <asm/nospec-branch.h>
0010
0011 #define MWAIT_SUBSTATE_MASK     0xf
0012 #define MWAIT_CSTATE_MASK       0xf
0013 #define MWAIT_SUBSTATE_SIZE     4
0014 #define MWAIT_HINT2CSTATE(hint)     (((hint) >> MWAIT_SUBSTATE_SIZE) & MWAIT_CSTATE_MASK)
0015 #define MWAIT_HINT2SUBSTATE(hint)   ((hint) & MWAIT_CSTATE_MASK)
0016 #define MWAIT_C1_SUBSTATE_MASK  0xf0
0017
0018 #define CPUID_MWAIT_LEAF        5
0019 #define CPUID5_ECX_EXTENSIONS_SUPPORTED 0x1
0020 #define CPUID5_ECX_INTERRUPT_BREAK  0x2
0021
0022 #define MWAIT_ECX_INTERRUPT_BREAK   0x1
0023 #define MWAITX_ECX_TIMER_ENABLE     BIT(1)
0024 #define MWAITX_MAX_WAIT_CYCLES      UINT_MAX
0025 #define MWAITX_DISABLE_CSTATES      0xf0
0026 #define TPAUSE_C01_STATE        1
0027 #define TPAUSE_C02_STATE        0
0028
0029 static inline void __monitor(const void *eax, unsigned long ecx,
0030                  unsigned long edx)
0031 {
0032     /* "monitor %eax, %ecx, %edx;" */
0033     asm volatile(".byte 0x0f, 0x01, 0xc8;"
0034              :: "a" (eax), "c" (ecx), "d"(edx));
0035 }
0036
0037 static inline void __monitorx(const void *eax, unsigned long ecx,
0038                   unsigned long edx)
0039 {
0040     /* "monitorx %eax, %ecx, %edx;" */
0041     asm volatile(".byte 0x0f, 0x01, 0xfa;"
0042              :: "a" (eax), "c" (ecx), "d"(edx));
0043 }
0044
0045 static inline void __mwait(unsigned long eax, unsigned long ecx)
0046 {
0047     mds_idle_clear_cpu_buffers();
0048
0049     /* "mwait %eax, %ecx;" */
0050     asm volatile(".byte 0x0f, 0x01, 0xc9;"
0051              :: "a" (eax), "c" (ecx));
0052 }
0053
0054 /*
0055  * MWAITX allows for a timer expiration to get the core out a wait state in
0056  * addition to the default MWAIT exit condition of a store appearing at a
0057  * monitored virtual address.
0058  *
0059  * Registers:
0060  *
0061  * MWAITX ECX[1]: enable timer if set
0062  * MWAITX EBX[31:0]: max wait time expressed in SW P0 clocks. The software P0
0063  * frequency is the same as the TSC frequency.
0064  *
0065  * Below is a comparison between MWAIT and MWAITX on AMD processors:
0066  *
0067  *                 MWAIT                           MWAITX
0068  * opcode          0f 01 c9           |            0f 01 fb
0069  * ECX[0]                  value of RFLAGS.IF seen by instruction
0070  * ECX[1]          unused/#GP if set  |            enable timer if set
0071  * ECX[31:2]                     unused/#GP if set
0072  * EAX                           unused (reserve for hint)
0073  * EBX[31:0]       unused             |            max wait time (P0 clocks)
0074  *
0075  *                 MONITOR                         MONITORX
0076  * opcode          0f 01 c8           |            0f 01 fa
0077  * EAX                     (logical) address to monitor
0078  * ECX                     #GP if not zero
0079  */
0080 static inline void __mwaitx(unsigned long eax, unsigned long ebx,
0081                 unsigned long ecx)
0082 {
0083     /* No MDS buffer clear as this is AMD/HYGON only */
0084
0085     /* "mwaitx %eax, %ebx, %ecx;" */
0086     asm volatile(".byte 0x0f, 0x01, 0xfb;"
0087              :: "a" (eax), "b" (ebx), "c" (ecx));
0088 }
0089
0090 static inline void __sti_mwait(unsigned long eax, unsigned long ecx)
0091 {
0092     mds_idle_clear_cpu_buffers();
0093     /* "mwait %eax, %ecx;" */
0094     asm volatile("sti; .byte 0x0f, 0x01, 0xc9;"
0095              :: "a" (eax), "c" (ecx));
0096 }
0097
0098 /*
0099  * This uses new MONITOR/MWAIT instructions on P4 processors with PNI,
0100  * which can obviate IPI to trigger checking of need_resched.
0101  * We execute MONITOR against need_resched and enter optimized wait state
0102  * through MWAIT. Whenever someone changes need_resched, we would be woken
0103  * up from MWAIT (without an IPI).
0104  *
0105  * New with Core Duo processors, MWAIT can take some hints based on CPU
0106  * capability.
0107  */
0108 static inline void mwait_idle_with_hints(unsigned long eax, unsigned long ecx)
0109 {
0110     if (static_cpu_has_bug(X86_BUG_MONITOR) || !current_set_polling_and_test()) {
0111         if (static_cpu_has_bug(X86_BUG_CLFLUSH_MONITOR)) {
0112             mb();
0113             clflush((void *)&current_thread_info()->flags);
0114             mb();
0115         }
0116
0117         __monitor((void *)&current_thread_info()->flags, 0, 0);
0118         if (!need_resched())
0119             __mwait(eax, ecx);
0120     }
0121     current_clr_polling();
0122 }
0123
0124 /*
0125  * Caller can specify whether to enter C0.1 (low latency, less
0126  * power saving) or C0.2 state (saves more power, but longer wakeup
0127  * latency). This may be overridden by the IA32_UMWAIT_CONTROL MSR
0128  * which can force requests for C0.2 to be downgraded to C0.1.
0129  */
0130 static inline void __tpause(u32 ecx, u32 edx, u32 eax)
0131 {
0132     /* "tpause %ecx, %edx, %eax;" */
0133     #ifdef CONFIG_AS_TPAUSE
0134     asm volatile("tpause %%ecx\n"
0135              :
0136              : "c"(ecx), "d"(edx), "a"(eax));
0137     #else
0138     asm volatile(".byte 0x66, 0x0f, 0xae, 0xf1\t\n"
0139              :
0140              : "c"(ecx), "d"(edx), "a"(eax));
0141     #endif
0142 }
0143
0144 #endif /* _ASM_X86_MWAIT_H */