sparc/lib/atomic_64.S

0001 /* SPDX-License-Identifier: GPL-2.0 */
0002 /* atomic.S: These things are too big to do inline.
0003  *
0004  * Copyright (C) 1999, 2007 2012 David S. Miller (davem@davemloft.net)
0005  */
0006
0007 #include <linux/linkage.h>
0008 #include <asm/asi.h>
0009 #include <asm/backoff.h>
0010 #include <asm/export.h>
0011
0012     .text
0013
0014     /* Three versions of the atomic routines, one that
0015      * does not return a value and does not perform
0016      * memory barriers, and a two which return
0017      * a value, the new and old value resp. and does the
0018      * barriers.
0019      */
0020
0021 #define ATOMIC_OP(op)                           \
0022 ENTRY(arch_atomic_##op) /* %o0 = increment, %o1 = atomic_ptr */     \
0023     BACKOFF_SETUP(%o2);                     \
0024 1:  lduw    [%o1], %g1;                     \
0025     op  %g1, %o0, %g7;                      \
0026     cas [%o1], %g1, %g7;                    \
0027     cmp %g1, %g7;                       \
0028     bne,pn  %icc, BACKOFF_LABEL(2f, 1b);                \
0029      nop;                               \
0030     retl;                               \
0031      nop;                               \
0032 2:  BACKOFF_SPIN(%o2, %o3, 1b);                 \
0033 ENDPROC(arch_atomic_##op);                      \
0034 EXPORT_SYMBOL(arch_atomic_##op);
0035
0036 #define ATOMIC_OP_RETURN(op)                        \
0037 ENTRY(arch_atomic_##op##_return) /* %o0 = increment, %o1 = atomic_ptr */\
0038     BACKOFF_SETUP(%o2);                     \
0039 1:  lduw    [%o1], %g1;                     \
0040     op  %g1, %o0, %g7;                      \
0041     cas [%o1], %g1, %g7;                    \
0042     cmp %g1, %g7;                       \
0043     bne,pn  %icc, BACKOFF_LABEL(2f, 1b);                \
0044      op %g1, %o0, %g1;                      \
0045     retl;                               \
0046      sra    %g1, 0, %o0;                        \
0047 2:  BACKOFF_SPIN(%o2, %o3, 1b);                 \
0048 ENDPROC(arch_atomic_##op##_return);                 \
0049 EXPORT_SYMBOL(arch_atomic_##op##_return);
0050
0051 #define ATOMIC_FETCH_OP(op)                     \
0052 ENTRY(arch_atomic_fetch_##op) /* %o0 = increment, %o1 = atomic_ptr */   \
0053     BACKOFF_SETUP(%o2);                     \
0054 1:  lduw    [%o1], %g1;                     \
0055     op  %g1, %o0, %g7;                      \
0056     cas [%o1], %g1, %g7;                    \
0057     cmp %g1, %g7;                       \
0058     bne,pn  %icc, BACKOFF_LABEL(2f, 1b);                \
0059      nop;                               \
0060     retl;                               \
0061      sra    %g1, 0, %o0;                        \
0062 2:  BACKOFF_SPIN(%o2, %o3, 1b);                 \
0063 ENDPROC(arch_atomic_fetch_##op);                    \
0064 EXPORT_SYMBOL(arch_atomic_fetch_##op);
0065
0066 ATOMIC_OP(add)
0067 ATOMIC_OP_RETURN(add)
0068 ATOMIC_FETCH_OP(add)
0069
0070 ATOMIC_OP(sub)
0071 ATOMIC_OP_RETURN(sub)
0072 ATOMIC_FETCH_OP(sub)
0073
0074 ATOMIC_OP(and)
0075 ATOMIC_FETCH_OP(and)
0076
0077 ATOMIC_OP(or)
0078 ATOMIC_FETCH_OP(or)
0079
0080 ATOMIC_OP(xor)
0081 ATOMIC_FETCH_OP(xor)
0082
0083 #undef ATOMIC_FETCH_OP
0084 #undef ATOMIC_OP_RETURN
0085 #undef ATOMIC_OP
0086
0087 #define ATOMIC64_OP(op)                         \
0088 ENTRY(arch_atomic64_##op) /* %o0 = increment, %o1 = atomic_ptr */   \
0089     BACKOFF_SETUP(%o2);                     \
0090 1:  ldx [%o1], %g1;                     \
0091     op  %g1, %o0, %g7;                      \
0092     casx    [%o1], %g1, %g7;                    \
0093     cmp %g1, %g7;                       \
0094     bne,pn  %xcc, BACKOFF_LABEL(2f, 1b);                \
0095      nop;                               \
0096     retl;                               \
0097      nop;                               \
0098 2:  BACKOFF_SPIN(%o2, %o3, 1b);                 \
0099 ENDPROC(arch_atomic64_##op);                        \
0100 EXPORT_SYMBOL(arch_atomic64_##op);
0101
0102 #define ATOMIC64_OP_RETURN(op)                      \
0103 ENTRY(arch_atomic64_##op##_return) /* %o0 = increment, %o1 = atomic_ptr */  \
0104     BACKOFF_SETUP(%o2);                     \
0105 1:  ldx [%o1], %g1;                     \
0106     op  %g1, %o0, %g7;                      \
0107     casx    [%o1], %g1, %g7;                    \
0108     cmp %g1, %g7;                       \
0109     bne,pn  %xcc, BACKOFF_LABEL(2f, 1b);                \
0110      nop;                               \
0111     retl;                               \
0112      op %g1, %o0, %o0;                      \
0113 2:  BACKOFF_SPIN(%o2, %o3, 1b);                 \
0114 ENDPROC(arch_atomic64_##op##_return);                   \
0115 EXPORT_SYMBOL(arch_atomic64_##op##_return);
0116
0117 #define ATOMIC64_FETCH_OP(op)                       \
0118 ENTRY(arch_atomic64_fetch_##op) /* %o0 = increment, %o1 = atomic_ptr */ \
0119     BACKOFF_SETUP(%o2);                     \
0120 1:  ldx [%o1], %g1;                     \
0121     op  %g1, %o0, %g7;                      \
0122     casx    [%o1], %g1, %g7;                    \
0123     cmp %g1, %g7;                       \
0124     bne,pn  %xcc, BACKOFF_LABEL(2f, 1b);                \
0125      nop;                               \
0126     retl;                               \
0127      mov    %g1, %o0;                       \
0128 2:  BACKOFF_SPIN(%o2, %o3, 1b);                 \
0129 ENDPROC(arch_atomic64_fetch_##op);                  \
0130 EXPORT_SYMBOL(arch_atomic64_fetch_##op);
0131
0132 ATOMIC64_OP(add)
0133 ATOMIC64_OP_RETURN(add)
0134 ATOMIC64_FETCH_OP(add)
0135
0136 ATOMIC64_OP(sub)
0137 ATOMIC64_OP_RETURN(sub)
0138 ATOMIC64_FETCH_OP(sub)
0139
0140 ATOMIC64_OP(and)
0141 ATOMIC64_FETCH_OP(and)
0142
0143 ATOMIC64_OP(or)
0144 ATOMIC64_FETCH_OP(or)
0145
0146 ATOMIC64_OP(xor)
0147 ATOMIC64_FETCH_OP(xor)
0148
0149 #undef ATOMIC64_FETCH_OP
0150 #undef ATOMIC64_OP_RETURN
0151 #undef ATOMIC64_OP
0152
0153 ENTRY(arch_atomic64_dec_if_positive) /* %o0 = atomic_ptr */
0154     BACKOFF_SETUP(%o2)
0155 1:  ldx [%o0], %g1
0156     brlez,pn %g1, 3f
0157      sub    %g1, 1, %g7
0158     casx    [%o0], %g1, %g7
0159     cmp %g1, %g7
0160     bne,pn  %xcc, BACKOFF_LABEL(2f, 1b)
0161      nop
0162 3:  retl
0163      sub    %g1, 1, %o0
0164 2:  BACKOFF_SPIN(%o2, %o3, 1b)
0165 ENDPROC(arch_atomic64_dec_if_positive)
0166 EXPORT_SYMBOL(arch_atomic64_dec_if_positive)