include/asm/atomic.h

0001 /* SPDX-License-Identifier: GPL-2.0-only */
0002 /*
0003  * Based on arch/arm/include/asm/atomic.h
0004  *
0005  * Copyright (C) 1996 Russell King.
0006  * Copyright (C) 2002 Deep Blue Solutions Ltd.
0007  * Copyright (C) 2012 ARM Ltd.
0008  */
0009 #ifndef __ASM_ATOMIC_H
0010 #define __ASM_ATOMIC_H
0011
0012 #include <linux/compiler.h>
0013 #include <linux/types.h>
0014
0015 #include <asm/barrier.h>
0016 #include <asm/cmpxchg.h>
0017 #include <asm/lse.h>
0018
0019 #define ATOMIC_OP(op)                           \
0020 static __always_inline void arch_##op(int i, atomic_t *v)       \
0021 {                                   \
0022     __lse_ll_sc_body(op, i, v);                 \
0023 }
0024
0025 ATOMIC_OP(atomic_andnot)
0026 ATOMIC_OP(atomic_or)
0027 ATOMIC_OP(atomic_xor)
0028 ATOMIC_OP(atomic_add)
0029 ATOMIC_OP(atomic_and)
0030 ATOMIC_OP(atomic_sub)
0031
0032 #undef ATOMIC_OP
0033
0034 #define ATOMIC_FETCH_OP(name, op)                   \
0035 static __always_inline int arch_##op##name(int i, atomic_t *v)      \
0036 {                                   \
0037     return __lse_ll_sc_body(op##name, i, v);            \
0038 }
0039
0040 #define ATOMIC_FETCH_OPS(op)                        \
0041     ATOMIC_FETCH_OP(_relaxed, op)                   \
0042     ATOMIC_FETCH_OP(_acquire, op)                   \
0043     ATOMIC_FETCH_OP(_release, op)                   \
0044     ATOMIC_FETCH_OP(        , op)
0045
0046 ATOMIC_FETCH_OPS(atomic_fetch_andnot)
0047 ATOMIC_FETCH_OPS(atomic_fetch_or)
0048 ATOMIC_FETCH_OPS(atomic_fetch_xor)
0049 ATOMIC_FETCH_OPS(atomic_fetch_add)
0050 ATOMIC_FETCH_OPS(atomic_fetch_and)
0051 ATOMIC_FETCH_OPS(atomic_fetch_sub)
0052 ATOMIC_FETCH_OPS(atomic_add_return)
0053 ATOMIC_FETCH_OPS(atomic_sub_return)
0054
0055 #undef ATOMIC_FETCH_OP
0056 #undef ATOMIC_FETCH_OPS
0057
0058 #define ATOMIC64_OP(op)                         \
0059 static __always_inline void arch_##op(long i, atomic64_t *v)        \
0060 {                                   \
0061     __lse_ll_sc_body(op, i, v);                 \
0062 }
0063
0064 ATOMIC64_OP(atomic64_andnot)
0065 ATOMIC64_OP(atomic64_or)
0066 ATOMIC64_OP(atomic64_xor)
0067 ATOMIC64_OP(atomic64_add)
0068 ATOMIC64_OP(atomic64_and)
0069 ATOMIC64_OP(atomic64_sub)
0070
0071 #undef ATOMIC64_OP
0072
0073 #define ATOMIC64_FETCH_OP(name, op)                 \
0074 static __always_inline long arch_##op##name(long i, atomic64_t *v)  \
0075 {                                   \
0076     return __lse_ll_sc_body(op##name, i, v);            \
0077 }
0078
0079 #define ATOMIC64_FETCH_OPS(op)                      \
0080     ATOMIC64_FETCH_OP(_relaxed, op)                 \
0081     ATOMIC64_FETCH_OP(_acquire, op)                 \
0082     ATOMIC64_FETCH_OP(_release, op)                 \
0083     ATOMIC64_FETCH_OP(        , op)
0084
0085 ATOMIC64_FETCH_OPS(atomic64_fetch_andnot)
0086 ATOMIC64_FETCH_OPS(atomic64_fetch_or)
0087 ATOMIC64_FETCH_OPS(atomic64_fetch_xor)
0088 ATOMIC64_FETCH_OPS(atomic64_fetch_add)
0089 ATOMIC64_FETCH_OPS(atomic64_fetch_and)
0090 ATOMIC64_FETCH_OPS(atomic64_fetch_sub)
0091 ATOMIC64_FETCH_OPS(atomic64_add_return)
0092 ATOMIC64_FETCH_OPS(atomic64_sub_return)
0093
0094 #undef ATOMIC64_FETCH_OP
0095 #undef ATOMIC64_FETCH_OPS
0096
0097 static __always_inline long arch_atomic64_dec_if_positive(atomic64_t *v)
0098 {
0099     return __lse_ll_sc_body(atomic64_dec_if_positive, v);
0100 }
0101
0102 #define arch_atomic_read(v)         __READ_ONCE((v)->counter)
0103 #define arch_atomic_set(v, i)           __WRITE_ONCE(((v)->counter), (i))
0104
0105 #define arch_atomic_add_return_relaxed      arch_atomic_add_return_relaxed
0106 #define arch_atomic_add_return_acquire      arch_atomic_add_return_acquire
0107 #define arch_atomic_add_return_release      arch_atomic_add_return_release
0108 #define arch_atomic_add_return          arch_atomic_add_return
0109
0110 #define arch_atomic_sub_return_relaxed      arch_atomic_sub_return_relaxed
0111 #define arch_atomic_sub_return_acquire      arch_atomic_sub_return_acquire
0112 #define arch_atomic_sub_return_release      arch_atomic_sub_return_release
0113 #define arch_atomic_sub_return          arch_atomic_sub_return
0114
0115 #define arch_atomic_fetch_add_relaxed       arch_atomic_fetch_add_relaxed
0116 #define arch_atomic_fetch_add_acquire       arch_atomic_fetch_add_acquire
0117 #define arch_atomic_fetch_add_release       arch_atomic_fetch_add_release
0118 #define arch_atomic_fetch_add           arch_atomic_fetch_add
0119
0120 #define arch_atomic_fetch_sub_relaxed       arch_atomic_fetch_sub_relaxed
0121 #define arch_atomic_fetch_sub_acquire       arch_atomic_fetch_sub_acquire
0122 #define arch_atomic_fetch_sub_release       arch_atomic_fetch_sub_release
0123 #define arch_atomic_fetch_sub           arch_atomic_fetch_sub
0124
0125 #define arch_atomic_fetch_and_relaxed       arch_atomic_fetch_and_relaxed
0126 #define arch_atomic_fetch_and_acquire       arch_atomic_fetch_and_acquire
0127 #define arch_atomic_fetch_and_release       arch_atomic_fetch_and_release
0128 #define arch_atomic_fetch_and           arch_atomic_fetch_and
0129
0130 #define arch_atomic_fetch_andnot_relaxed    arch_atomic_fetch_andnot_relaxed
0131 #define arch_atomic_fetch_andnot_acquire    arch_atomic_fetch_andnot_acquire
0132 #define arch_atomic_fetch_andnot_release    arch_atomic_fetch_andnot_release
0133 #define arch_atomic_fetch_andnot        arch_atomic_fetch_andnot
0134
0135 #define arch_atomic_fetch_or_relaxed        arch_atomic_fetch_or_relaxed
0136 #define arch_atomic_fetch_or_acquire        arch_atomic_fetch_or_acquire
0137 #define arch_atomic_fetch_or_release        arch_atomic_fetch_or_release
0138 #define arch_atomic_fetch_or            arch_atomic_fetch_or
0139
0140 #define arch_atomic_fetch_xor_relaxed       arch_atomic_fetch_xor_relaxed
0141 #define arch_atomic_fetch_xor_acquire       arch_atomic_fetch_xor_acquire
0142 #define arch_atomic_fetch_xor_release       arch_atomic_fetch_xor_release
0143 #define arch_atomic_fetch_xor           arch_atomic_fetch_xor
0144
0145 #define arch_atomic_xchg_relaxed(v, new) \
0146     arch_xchg_relaxed(&((v)->counter), (new))
0147 #define arch_atomic_xchg_acquire(v, new) \
0148     arch_xchg_acquire(&((v)->counter), (new))
0149 #define arch_atomic_xchg_release(v, new) \
0150     arch_xchg_release(&((v)->counter), (new))
0151 #define arch_atomic_xchg(v, new) \
0152     arch_xchg(&((v)->counter), (new))
0153
0154 #define arch_atomic_cmpxchg_relaxed(v, old, new) \
0155     arch_cmpxchg_relaxed(&((v)->counter), (old), (new))
0156 #define arch_atomic_cmpxchg_acquire(v, old, new) \
0157     arch_cmpxchg_acquire(&((v)->counter), (old), (new))
0158 #define arch_atomic_cmpxchg_release(v, old, new) \
0159     arch_cmpxchg_release(&((v)->counter), (old), (new))
0160 #define arch_atomic_cmpxchg(v, old, new) \
0161     arch_cmpxchg(&((v)->counter), (old), (new))
0162
0163 #define arch_atomic_andnot          arch_atomic_andnot
0164
0165 /*
0166  * 64-bit arch_atomic operations.
0167  */
0168 #define ATOMIC64_INIT               ATOMIC_INIT
0169 #define arch_atomic64_read          arch_atomic_read
0170 #define arch_atomic64_set           arch_atomic_set
0171
0172 #define arch_atomic64_add_return_relaxed    arch_atomic64_add_return_relaxed
0173 #define arch_atomic64_add_return_acquire    arch_atomic64_add_return_acquire
0174 #define arch_atomic64_add_return_release    arch_atomic64_add_return_release
0175 #define arch_atomic64_add_return        arch_atomic64_add_return
0176
0177 #define arch_atomic64_sub_return_relaxed    arch_atomic64_sub_return_relaxed
0178 #define arch_atomic64_sub_return_acquire    arch_atomic64_sub_return_acquire
0179 #define arch_atomic64_sub_return_release    arch_atomic64_sub_return_release
0180 #define arch_atomic64_sub_return        arch_atomic64_sub_return
0181
0182 #define arch_atomic64_fetch_add_relaxed     arch_atomic64_fetch_add_relaxed
0183 #define arch_atomic64_fetch_add_acquire     arch_atomic64_fetch_add_acquire
0184 #define arch_atomic64_fetch_add_release     arch_atomic64_fetch_add_release
0185 #define arch_atomic64_fetch_add         arch_atomic64_fetch_add
0186
0187 #define arch_atomic64_fetch_sub_relaxed     arch_atomic64_fetch_sub_relaxed
0188 #define arch_atomic64_fetch_sub_acquire     arch_atomic64_fetch_sub_acquire
0189 #define arch_atomic64_fetch_sub_release     arch_atomic64_fetch_sub_release
0190 #define arch_atomic64_fetch_sub         arch_atomic64_fetch_sub
0191
0192 #define arch_atomic64_fetch_and_relaxed     arch_atomic64_fetch_and_relaxed
0193 #define arch_atomic64_fetch_and_acquire     arch_atomic64_fetch_and_acquire
0194 #define arch_atomic64_fetch_and_release     arch_atomic64_fetch_and_release
0195 #define arch_atomic64_fetch_and         arch_atomic64_fetch_and
0196
0197 #define arch_atomic64_fetch_andnot_relaxed  arch_atomic64_fetch_andnot_relaxed
0198 #define arch_atomic64_fetch_andnot_acquire  arch_atomic64_fetch_andnot_acquire
0199 #define arch_atomic64_fetch_andnot_release  arch_atomic64_fetch_andnot_release
0200 #define arch_atomic64_fetch_andnot      arch_atomic64_fetch_andnot
0201
0202 #define arch_atomic64_fetch_or_relaxed      arch_atomic64_fetch_or_relaxed
0203 #define arch_atomic64_fetch_or_acquire      arch_atomic64_fetch_or_acquire
0204 #define arch_atomic64_fetch_or_release      arch_atomic64_fetch_or_release
0205 #define arch_atomic64_fetch_or          arch_atomic64_fetch_or
0206
0207 #define arch_atomic64_fetch_xor_relaxed     arch_atomic64_fetch_xor_relaxed
0208 #define arch_atomic64_fetch_xor_acquire     arch_atomic64_fetch_xor_acquire
0209 #define arch_atomic64_fetch_xor_release     arch_atomic64_fetch_xor_release
0210 #define arch_atomic64_fetch_xor         arch_atomic64_fetch_xor
0211
0212 #define arch_atomic64_xchg_relaxed      arch_atomic_xchg_relaxed
0213 #define arch_atomic64_xchg_acquire      arch_atomic_xchg_acquire
0214 #define arch_atomic64_xchg_release      arch_atomic_xchg_release
0215 #define arch_atomic64_xchg          arch_atomic_xchg
0216
0217 #define arch_atomic64_cmpxchg_relaxed       arch_atomic_cmpxchg_relaxed
0218 #define arch_atomic64_cmpxchg_acquire       arch_atomic_cmpxchg_acquire
0219 #define arch_atomic64_cmpxchg_release       arch_atomic_cmpxchg_release
0220 #define arch_atomic64_cmpxchg           arch_atomic_cmpxchg
0221
0222 #define arch_atomic64_andnot            arch_atomic64_andnot
0223
0224 #define arch_atomic64_dec_if_positive       arch_atomic64_dec_if_positive
0225
0226 #endif /* __ASM_ATOMIC_H */