include/asm/qspinlock_paravirt.h

0001 /* SPDX-License-Identifier: GPL-2.0 */
0002 #ifndef __ASM_QSPINLOCK_PARAVIRT_H
0003 #define __ASM_QSPINLOCK_PARAVIRT_H
0004
0005 #include <asm/ibt.h>
0006
0007 /*
0008  * For x86-64, PV_CALLEE_SAVE_REGS_THUNK() saves and restores 8 64-bit
0009  * registers. For i386, however, only 1 32-bit register needs to be saved
0010  * and restored. So an optimized version of __pv_queued_spin_unlock() is
0011  * hand-coded for 64-bit, but it isn't worthwhile to do it for 32-bit.
0012  */
0013 #ifdef CONFIG_64BIT
0014
0015 PV_CALLEE_SAVE_REGS_THUNK(__pv_queued_spin_unlock_slowpath);
0016 #define __pv_queued_spin_unlock __pv_queued_spin_unlock
0017 #define PV_UNLOCK       "__raw_callee_save___pv_queued_spin_unlock"
0018 #define PV_UNLOCK_SLOWPATH  "__raw_callee_save___pv_queued_spin_unlock_slowpath"
0019
0020 /*
0021  * Optimized assembly version of __raw_callee_save___pv_queued_spin_unlock
0022  * which combines the registers saving trunk and the body of the following
0023  * C code:
0024  *
0025  * void __pv_queued_spin_unlock(struct qspinlock *lock)
0026  * {
0027  *  u8 lockval = cmpxchg(&lock->locked, _Q_LOCKED_VAL, 0);
0028  *
0029  *  if (likely(lockval == _Q_LOCKED_VAL))
0030  *      return;
0031  *  pv_queued_spin_unlock_slowpath(lock, lockval);
0032  * }
0033  *
0034  * For x86-64,
0035  *   rdi = lock              (first argument)
0036  *   rsi = lockval           (second argument)
0037  *   rdx = internal variable (set to 0)
0038  */
0039 asm    (".pushsection .text;"
0040     ".globl " PV_UNLOCK ";"
0041     ".type " PV_UNLOCK ", @function;"
0042     ".align 4,0x90;"
0043     PV_UNLOCK ": "
0044     ASM_ENDBR
0045     FRAME_BEGIN
0046     "push  %rdx;"
0047     "mov   $0x1,%eax;"
0048     "xor   %edx,%edx;"
0049     LOCK_PREFIX "cmpxchg %dl,(%rdi);"
0050     "cmp   $0x1,%al;"
0051     "jne   .slowpath;"
0052     "pop   %rdx;"
0053     FRAME_END
0054     ASM_RET
0055     ".slowpath: "
0056     "push   %rsi;"
0057     "movzbl %al,%esi;"
0058     "call " PV_UNLOCK_SLOWPATH ";"
0059     "pop    %rsi;"
0060     "pop    %rdx;"
0061     FRAME_END
0062     ASM_RET
0063     ".size " PV_UNLOCK ", .-" PV_UNLOCK ";"
0064     ".popsection");
0065
0066 #else /* CONFIG_64BIT */
0067
0068 extern void __pv_queued_spin_unlock(struct qspinlock *lock);
0069 PV_CALLEE_SAVE_REGS_THUNK(__pv_queued_spin_unlock);
0070
0071 #endif /* CONFIG_64BIT */
0072 #endif