0001 // SPDX-License-Identifier: GPL-2.0+
0002 //
0003 // An earlier version of this file appeared in the companion webpage for
0004 // "Frightening small children and disconcerting grown-ups: Concurrency
0005 // in the Linux kernel" by Alglave, Maranget, McKenney, Parri, and Stern,
0006 // which appeared in ASPLOS 2018.
0007
0008 // ONCE
0009 READ_ONCE(X) __load{once}(X)
0010 WRITE_ONCE(X,V) { __store{once}(X,V); }
0011
0012 // Release Acquire and friends
0013 smp_store_release(X,V) { __store{release}(*X,V); }
0014 smp_load_acquire(X) __load{acquire}(*X)
0015 rcu_assign_pointer(X,V) { __store{release}(X,V); }
0016 rcu_dereference(X) __load{once}(X)
0017 smp_store_mb(X,V) { __store{once}(X,V); __fence{mb}; }
0018
0019 // Fences
0020 smp_mb() { __fence{mb}; }
0021 smp_rmb() { __fence{rmb}; }
0022 smp_wmb() { __fence{wmb}; }
0023 smp_mb__before_atomic() { __fence{before-atomic}; }
0024 smp_mb__after_atomic() { __fence{after-atomic}; }
0025 smp_mb__after_spinlock() { __fence{after-spinlock}; }
0026 smp_mb__after_unlock_lock() { __fence{after-unlock-lock}; }
0027 barrier() { __fence{barrier}; }
0028
0029 // Exchange
0030 xchg(X,V) __xchg{mb}(X,V)
0031 xchg_relaxed(X,V) __xchg{once}(X,V)
0032 xchg_release(X,V) __xchg{release}(X,V)
0033 xchg_acquire(X,V) __xchg{acquire}(X,V)
0034 cmpxchg(X,V,W) __cmpxchg{mb}(X,V,W)
0035 cmpxchg_relaxed(X,V,W) __cmpxchg{once}(X,V,W)
0036 cmpxchg_acquire(X,V,W) __cmpxchg{acquire}(X,V,W)
0037 cmpxchg_release(X,V,W) __cmpxchg{release}(X,V,W)
0038
0039 // Spinlocks
0040 spin_lock(X) { __lock(X); }
0041 spin_unlock(X) { __unlock(X); }
0042 spin_trylock(X) __trylock(X)
0043 spin_is_locked(X) __islocked(X)
0044
0045 // RCU
0046 rcu_read_lock() { __fence{rcu-lock}; }
0047 rcu_read_unlock() { __fence{rcu-unlock}; }
0048 synchronize_rcu() { __fence{sync-rcu}; }
0049 synchronize_rcu_expedited() { __fence{sync-rcu}; }
0050
0051 // SRCU
0052 srcu_read_lock(X) __srcu{srcu-lock}(X)
0053 srcu_read_unlock(X,Y) { __srcu{srcu-unlock}(X,Y); }
0054 synchronize_srcu(X) { __srcu{sync-srcu}(X); }
0055 synchronize_srcu_expedited(X) { __srcu{sync-srcu}(X); }
0056
0057 // Atomic
0058 atomic_read(X) READ_ONCE(*X)
0059 atomic_set(X,V) { WRITE_ONCE(*X,V); }
0060 atomic_read_acquire(X) smp_load_acquire(X)
0061 atomic_set_release(X,V) { smp_store_release(X,V); }
0062
0063 atomic_add(V,X) { __atomic_op(X,+,V); }
0064 atomic_sub(V,X) { __atomic_op(X,-,V); }
0065 atomic_inc(X) { __atomic_op(X,+,1); }
0066 atomic_dec(X) { __atomic_op(X,-,1); }
0067
0068 atomic_add_return(V,X) __atomic_op_return{mb}(X,+,V)
0069 atomic_add_return_relaxed(V,X) __atomic_op_return{once}(X,+,V)
0070 atomic_add_return_acquire(V,X) __atomic_op_return{acquire}(X,+,V)
0071 atomic_add_return_release(V,X) __atomic_op_return{release}(X,+,V)
0072 atomic_fetch_add(V,X) __atomic_fetch_op{mb}(X,+,V)
0073 atomic_fetch_add_relaxed(V,X) __atomic_fetch_op{once}(X,+,V)
0074 atomic_fetch_add_acquire(V,X) __atomic_fetch_op{acquire}(X,+,V)
0075 atomic_fetch_add_release(V,X) __atomic_fetch_op{release}(X,+,V)
0076
0077 atomic_inc_return(X) __atomic_op_return{mb}(X,+,1)
0078 atomic_inc_return_relaxed(X) __atomic_op_return{once}(X,+,1)
0079 atomic_inc_return_acquire(X) __atomic_op_return{acquire}(X,+,1)
0080 atomic_inc_return_release(X) __atomic_op_return{release}(X,+,1)
0081 atomic_fetch_inc(X) __atomic_fetch_op{mb}(X,+,1)
0082 atomic_fetch_inc_relaxed(X) __atomic_fetch_op{once}(X,+,1)
0083 atomic_fetch_inc_acquire(X) __atomic_fetch_op{acquire}(X,+,1)
0084 atomic_fetch_inc_release(X) __atomic_fetch_op{release}(X,+,1)
0085
0086 atomic_sub_return(V,X) __atomic_op_return{mb}(X,-,V)
0087 atomic_sub_return_relaxed(V,X) __atomic_op_return{once}(X,-,V)
0088 atomic_sub_return_acquire(V,X) __atomic_op_return{acquire}(X,-,V)
0089 atomic_sub_return_release(V,X) __atomic_op_return{release}(X,-,V)
0090 atomic_fetch_sub(V,X) __atomic_fetch_op{mb}(X,-,V)
0091 atomic_fetch_sub_relaxed(V,X) __atomic_fetch_op{once}(X,-,V)
0092 atomic_fetch_sub_acquire(V,X) __atomic_fetch_op{acquire}(X,-,V)
0093 atomic_fetch_sub_release(V,X) __atomic_fetch_op{release}(X,-,V)
0094
0095 atomic_dec_return(X) __atomic_op_return{mb}(X,-,1)
0096 atomic_dec_return_relaxed(X) __atomic_op_return{once}(X,-,1)
0097 atomic_dec_return_acquire(X) __atomic_op_return{acquire}(X,-,1)
0098 atomic_dec_return_release(X) __atomic_op_return{release}(X,-,1)
0099 atomic_fetch_dec(X) __atomic_fetch_op{mb}(X,-,1)
0100 atomic_fetch_dec_relaxed(X) __atomic_fetch_op{once}(X,-,1)
0101 atomic_fetch_dec_acquire(X) __atomic_fetch_op{acquire}(X,-,1)
0102 atomic_fetch_dec_release(X) __atomic_fetch_op{release}(X,-,1)
0103
0104 atomic_xchg(X,V) __xchg{mb}(X,V)
0105 atomic_xchg_relaxed(X,V) __xchg{once}(X,V)
0106 atomic_xchg_release(X,V) __xchg{release}(X,V)
0107 atomic_xchg_acquire(X,V) __xchg{acquire}(X,V)
0108 atomic_cmpxchg(X,V,W) __cmpxchg{mb}(X,V,W)
0109 atomic_cmpxchg_relaxed(X,V,W) __cmpxchg{once}(X,V,W)
0110 atomic_cmpxchg_acquire(X,V,W) __cmpxchg{acquire}(X,V,W)
0111 atomic_cmpxchg_release(X,V,W) __cmpxchg{release}(X,V,W)
0112
0113 atomic_sub_and_test(V,X) __atomic_op_return{mb}(X,-,V) == 0
0114 atomic_dec_and_test(X) __atomic_op_return{mb}(X,-,1) == 0
0115 atomic_inc_and_test(X) __atomic_op_return{mb}(X,+,1) == 0
0116 atomic_add_negative(V,X) __atomic_op_return{mb}(X,+,V) < 0
