x86/lib/atomic64_cx8_32.S

0001 /* SPDX-License-Identifier: GPL-2.0-or-later */
0002 /*
0003  * atomic64_t for 586+
0004  *
0005  * Copyright © 2010  Luca Barbieri
0006  */
0007
0008 #include <linux/linkage.h>
0009 #include <asm/alternative.h>
0010
0011 .macro read64 reg
0012     movl %ebx, %eax
0013     movl %ecx, %edx
0014 /* we need LOCK_PREFIX since otherwise cmpxchg8b always does the write */
0015     LOCK_PREFIX
0016     cmpxchg8b (\reg)
0017 .endm
0018
0019 SYM_FUNC_START(atomic64_read_cx8)
0020     read64 %ecx
0021     RET
0022 SYM_FUNC_END(atomic64_read_cx8)
0023
0024 SYM_FUNC_START(atomic64_set_cx8)
0025 1:
0026 /* we don't need LOCK_PREFIX since aligned 64-bit writes
0027  * are atomic on 586 and newer */
0028     cmpxchg8b (%esi)
0029     jne 1b
0030
0031     RET
0032 SYM_FUNC_END(atomic64_set_cx8)
0033
0034 SYM_FUNC_START(atomic64_xchg_cx8)
0035 1:
0036     LOCK_PREFIX
0037     cmpxchg8b (%esi)
0038     jne 1b
0039
0040     RET
0041 SYM_FUNC_END(atomic64_xchg_cx8)
0042
0043 .macro addsub_return func ins insc
0044 SYM_FUNC_START(atomic64_\func\()_return_cx8)
0045     pushl %ebp
0046     pushl %ebx
0047     pushl %esi
0048     pushl %edi
0049
0050     movl %eax, %esi
0051     movl %edx, %edi
0052     movl %ecx, %ebp
0053
0054     read64 %ecx
0055 1:
0056     movl %eax, %ebx
0057     movl %edx, %ecx
0058     \ins\()l %esi, %ebx
0059     \insc\()l %edi, %ecx
0060     LOCK_PREFIX
0061     cmpxchg8b (%ebp)
0062     jne 1b
0063
0064 10:
0065     movl %ebx, %eax
0066     movl %ecx, %edx
0067     popl %edi
0068     popl %esi
0069     popl %ebx
0070     popl %ebp
0071     RET
0072 SYM_FUNC_END(atomic64_\func\()_return_cx8)
0073 .endm
0074
0075 addsub_return add add adc
0076 addsub_return sub sub sbb
0077
0078 .macro incdec_return func ins insc
0079 SYM_FUNC_START(atomic64_\func\()_return_cx8)
0080     pushl %ebx
0081
0082     read64 %esi
0083 1:
0084     movl %eax, %ebx
0085     movl %edx, %ecx
0086     \ins\()l $1, %ebx
0087     \insc\()l $0, %ecx
0088     LOCK_PREFIX
0089     cmpxchg8b (%esi)
0090     jne 1b
0091
0092 10:
0093     movl %ebx, %eax
0094     movl %ecx, %edx
0095     popl %ebx
0096     RET
0097 SYM_FUNC_END(atomic64_\func\()_return_cx8)
0098 .endm
0099
0100 incdec_return inc add adc
0101 incdec_return dec sub sbb
0102
0103 SYM_FUNC_START(atomic64_dec_if_positive_cx8)
0104     pushl %ebx
0105
0106     read64 %esi
0107 1:
0108     movl %eax, %ebx
0109     movl %edx, %ecx
0110     subl $1, %ebx
0111     sbb $0, %ecx
0112     js 2f
0113     LOCK_PREFIX
0114     cmpxchg8b (%esi)
0115     jne 1b
0116
0117 2:
0118     movl %ebx, %eax
0119     movl %ecx, %edx
0120     popl %ebx
0121     RET
0122 SYM_FUNC_END(atomic64_dec_if_positive_cx8)
0123
0124 SYM_FUNC_START(atomic64_add_unless_cx8)
0125     pushl %ebp
0126     pushl %ebx
0127 /* these just push these two parameters on the stack */
0128     pushl %edi
0129     pushl %ecx
0130
0131     movl %eax, %ebp
0132     movl %edx, %edi
0133
0134     read64 %esi
0135 1:
0136     cmpl %eax, 0(%esp)
0137     je 4f
0138 2:
0139     movl %eax, %ebx
0140     movl %edx, %ecx
0141     addl %ebp, %ebx
0142     adcl %edi, %ecx
0143     LOCK_PREFIX
0144     cmpxchg8b (%esi)
0145     jne 1b
0146
0147     movl $1, %eax
0148 3:
0149     addl $8, %esp
0150     popl %ebx
0151     popl %ebp
0152     RET
0153 4:
0154     cmpl %edx, 4(%esp)
0155     jne 2b
0156     xorl %eax, %eax
0157     jmp 3b
0158 SYM_FUNC_END(atomic64_add_unless_cx8)
0159
0160 SYM_FUNC_START(atomic64_inc_not_zero_cx8)
0161     pushl %ebx
0162
0163     read64 %esi
0164 1:
0165     movl %eax, %ecx
0166     orl %edx, %ecx
0167     jz 3f
0168     movl %eax, %ebx
0169     xorl %ecx, %ecx
0170     addl $1, %ebx
0171     adcl %edx, %ecx
0172     LOCK_PREFIX
0173     cmpxchg8b (%esi)
0174     jne 1b
0175
0176     movl $1, %eax
0177 3:
0178     popl %ebx
0179     RET
0180 SYM_FUNC_END(atomic64_inc_not_zero_cx8)