include/asm/atomic64-arcv2.h

0001 /* SPDX-License-Identifier: GPL-2.0-only */
0002
0003 /*
0004  * ARCv2 supports 64-bit exclusive load (LLOCKD) / store (SCONDD)
0005  *  - The address HAS to be 64-bit aligned
0006  */
0007
0008 #ifndef _ASM_ARC_ATOMIC64_ARCV2_H
0009 #define _ASM_ARC_ATOMIC64_ARCV2_H
0010
0011 typedef struct {
0012     s64 __aligned(8) counter;
0013 } atomic64_t;
0014
0015 #define ATOMIC64_INIT(a) { (a) }
0016
0017 static inline s64 arch_atomic64_read(const atomic64_t *v)
0018 {
0019     s64 val;
0020
0021     __asm__ __volatile__(
0022     "   ldd   %0, [%1]  \n"
0023     : "=r"(val)
0024     : "r"(&v->counter));
0025
0026     return val;
0027 }
0028
0029 static inline void arch_atomic64_set(atomic64_t *v, s64 a)
0030 {
0031     /*
0032      * This could have been a simple assignment in "C" but would need
0033      * explicit volatile. Otherwise gcc optimizers could elide the store
0034      * which borked atomic64 self-test
0035      * In the inline asm version, memory clobber needed for exact same
0036      * reason, to tell gcc about the store.
0037      *
0038      * This however is not needed for sibling atomic64_add() etc since both
0039      * load/store are explicitly done in inline asm. As long as API is used
0040      * for each access, gcc has no way to optimize away any load/store
0041      */
0042     __asm__ __volatile__(
0043     "   std   %0, [%1]  \n"
0044     :
0045     : "r"(a), "r"(&v->counter)
0046     : "memory");
0047 }
0048
0049 #define ATOMIC64_OP(op, op1, op2)                   \
0050 static inline void arch_atomic64_##op(s64 a, atomic64_t *v)     \
0051 {                                   \
0052     s64 val;                            \
0053                                     \
0054     __asm__ __volatile__(                       \
0055     "1:             \n"             \
0056     "   llockd  %0, [%1]    \n"             \
0057     "   " #op1 " %L0, %L0, %L2  \n"             \
0058     "   " #op2 " %H0, %H0, %H2  \n"             \
0059     "   scondd   %0, [%1]   \n"             \
0060     "   bnz     1b      \n"             \
0061     : "=&r"(val)                            \
0062     : "r"(&v->counter), "ir"(a)                 \
0063     : "cc");                            \
0064 }                                   \
0065
0066 #define ATOMIC64_OP_RETURN(op, op1, op2)                    \
0067 static inline s64 arch_atomic64_##op##_return_relaxed(s64 a, atomic64_t *v) \
0068 {                                   \
0069     s64 val;                            \
0070                                     \
0071     __asm__ __volatile__(                       \
0072     "1:             \n"             \
0073     "   llockd   %0, [%1]   \n"             \
0074     "   " #op1 " %L0, %L0, %L2  \n"             \
0075     "   " #op2 " %H0, %H0, %H2  \n"             \
0076     "   scondd   %0, [%1]   \n"             \
0077     "   bnz     1b      \n"             \
0078     : [val] "=&r"(val)                      \
0079     : "r"(&v->counter), "ir"(a)                 \
0080     : "cc");    /* memory clobber comes from smp_mb() */    \
0081                                     \
0082     return val;                         \
0083 }
0084
0085 #define arch_atomic64_add_return_relaxed    arch_atomic64_add_return_relaxed
0086 #define arch_atomic64_sub_return_relaxed    arch_atomic64_sub_return_relaxed
0087
0088 #define ATOMIC64_FETCH_OP(op, op1, op2)                     \
0089 static inline s64 arch_atomic64_fetch_##op##_relaxed(s64 a, atomic64_t *v)  \
0090 {                                   \
0091     s64 val, orig;                          \
0092                                     \
0093     __asm__ __volatile__(                       \
0094     "1:             \n"             \
0095     "   llockd   %0, [%2]   \n"             \
0096     "   " #op1 " %L1, %L0, %L3  \n"             \
0097     "   " #op2 " %H1, %H0, %H3  \n"             \
0098     "   scondd   %1, [%2]   \n"             \
0099     "   bnz     1b      \n"             \
0100     : "=&r"(orig), "=&r"(val)                   \
0101     : "r"(&v->counter), "ir"(a)                 \
0102     : "cc");    /* memory clobber comes from smp_mb() */    \
0103                                     \
0104     return orig;                            \
0105 }
0106
0107 #define arch_atomic64_fetch_add_relaxed     arch_atomic64_fetch_add_relaxed
0108 #define arch_atomic64_fetch_sub_relaxed     arch_atomic64_fetch_sub_relaxed
0109
0110 #define arch_atomic64_fetch_and_relaxed     arch_atomic64_fetch_and_relaxed
0111 #define arch_atomic64_fetch_andnot_relaxed  arch_atomic64_fetch_andnot_relaxed
0112 #define arch_atomic64_fetch_or_relaxed      arch_atomic64_fetch_or_relaxed
0113 #define arch_atomic64_fetch_xor_relaxed     arch_atomic64_fetch_xor_relaxed
0114
0115 #define ATOMIC64_OPS(op, op1, op2)                  \
0116     ATOMIC64_OP(op, op1, op2)                   \
0117     ATOMIC64_OP_RETURN(op, op1, op2)                \
0118     ATOMIC64_FETCH_OP(op, op1, op2)
0119
0120 ATOMIC64_OPS(add, add.f, adc)
0121 ATOMIC64_OPS(sub, sub.f, sbc)
0122
0123 #undef ATOMIC64_OPS
0124 #define ATOMIC64_OPS(op, op1, op2)                  \
0125     ATOMIC64_OP(op, op1, op2)                   \
0126     ATOMIC64_FETCH_OP(op, op1, op2)
0127
0128 ATOMIC64_OPS(and, and, and)
0129 ATOMIC64_OPS(andnot, bic, bic)
0130 ATOMIC64_OPS(or, or, or)
0131 ATOMIC64_OPS(xor, xor, xor)
0132
0133 #define arch_atomic64_andnot        arch_atomic64_andnot
0134
0135 #undef ATOMIC64_OPS
0136 #undef ATOMIC64_FETCH_OP
0137 #undef ATOMIC64_OP_RETURN
0138 #undef ATOMIC64_OP
0139
0140 static inline s64
0141 arch_atomic64_cmpxchg(atomic64_t *ptr, s64 expected, s64 new)
0142 {
0143     s64 prev;
0144
0145     smp_mb();
0146
0147     __asm__ __volatile__(
0148     "1: llockd  %0, [%1]    \n"
0149     "   brne    %L0, %L2, 2f    \n"
0150     "   brne    %H0, %H2, 2f    \n"
0151     "   scondd  %3, [%1]    \n"
0152     "   bnz     1b      \n"
0153     "2:             \n"
0154     : "=&r"(prev)
0155     : "r"(ptr), "ir"(expected), "r"(new)
0156     : "cc");    /* memory clobber comes from smp_mb() */
0157
0158     smp_mb();
0159
0160     return prev;
0161 }
0162
0163 static inline s64 arch_atomic64_xchg(atomic64_t *ptr, s64 new)
0164 {
0165     s64 prev;
0166
0167     smp_mb();
0168
0169     __asm__ __volatile__(
0170     "1: llockd  %0, [%1]    \n"
0171     "   scondd  %2, [%1]    \n"
0172     "   bnz     1b      \n"
0173     "2:             \n"
0174     : "=&r"(prev)
0175     : "r"(ptr), "r"(new)
0176     : "cc");    /* memory clobber comes from smp_mb() */
0177
0178     smp_mb();
0179
0180     return prev;
0181 }
0182
0183 /**
0184  * arch_atomic64_dec_if_positive - decrement by 1 if old value positive
0185  * @v: pointer of type atomic64_t
0186  *
0187  * The function returns the old value of *v minus 1, even if
0188  * the atomic variable, v, was not decremented.
0189  */
0190
0191 static inline s64 arch_atomic64_dec_if_positive(atomic64_t *v)
0192 {
0193     s64 val;
0194
0195     smp_mb();
0196
0197     __asm__ __volatile__(
0198     "1: llockd  %0, [%1]    \n"
0199     "   sub.f   %L0, %L0, 1 # w0 - 1, set C on borrow\n"
0200     "   sub.c   %H0, %H0, 1 # if C set, w1 - 1\n"
0201     "   brlt    %H0, 0, 2f  \n"
0202     "   scondd  %0, [%1]    \n"
0203     "   bnz     1b      \n"
0204     "2:             \n"
0205     : "=&r"(val)
0206     : "r"(&v->counter)
0207     : "cc");    /* memory clobber comes from smp_mb() */
0208
0209     smp_mb();
0210
0211     return val;
0212 }
0213 #define arch_atomic64_dec_if_positive arch_atomic64_dec_if_positive
0214
0215 /**
0216  * arch_atomic64_fetch_add_unless - add unless the number is a given value
0217  * @v: pointer of type atomic64_t
0218  * @a: the amount to add to v...
0219  * @u: ...unless v is equal to u.
0220  *
0221  * Atomically adds @a to @v, if it was not @u.
0222  * Returns the old value of @v
0223  */
0224 static inline s64 arch_atomic64_fetch_add_unless(atomic64_t *v, s64 a, s64 u)
0225 {
0226     s64 old, temp;
0227
0228     smp_mb();
0229
0230     __asm__ __volatile__(
0231     "1: llockd  %0, [%2]    \n"
0232     "   brne    %L0, %L4, 2f    # continue to add since v != u \n"
0233     "   breq.d  %H0, %H4, 3f    # return since v == u \n"
0234     "2:             \n"
0235     "   add.f   %L1, %L0, %L3   \n"
0236     "   adc     %H1, %H0, %H3   \n"
0237     "   scondd  %1, [%2]    \n"
0238     "   bnz     1b      \n"
0239     "3:             \n"
0240     : "=&r"(old), "=&r" (temp)
0241     : "r"(&v->counter), "r"(a), "r"(u)
0242     : "cc");    /* memory clobber comes from smp_mb() */
0243
0244     smp_mb();
0245
0246     return old;
0247 }
0248 #define arch_atomic64_fetch_add_unless arch_atomic64_fetch_add_unless
0249
0250 #endif