include/asm/spinlock.h

0001 /* SPDX-License-Identifier: GPL-2.0 */
0002 #ifndef __ASM_SPINLOCK_H
0003 #define __ASM_SPINLOCK_H
0004
0005 #if __LINUX_ARM_ARCH__ < 6
0006 #error SMP not supported on pre-ARMv6 CPUs
0007 #endif
0008
0009 #include <linux/prefetch.h>
0010 #include <asm/barrier.h>
0011 #include <asm/processor.h>
0012
0013 /*
0014  * sev and wfe are ARMv6K extensions.  Uniprocessor ARMv6 may not have the K
0015  * extensions, so when running on UP, we have to patch these instructions away.
0016  */
0017 #ifdef CONFIG_THUMB2_KERNEL
0018 /*
0019  * For Thumb-2, special care is needed to ensure that the conditional WFE
0020  * instruction really does assemble to exactly 4 bytes (as required by
0021  * the SMP_ON_UP fixup code).   By itself "wfene" might cause the
0022  * assembler to insert a extra (16-bit) IT instruction, depending on the
0023  * presence or absence of neighbouring conditional instructions.
0024  *
0025  * To avoid this unpredictability, an appropriate IT is inserted explicitly:
0026  * the assembler won't change IT instructions which are explicitly present
0027  * in the input.
0028  */
0029 #define WFE(cond)   __ALT_SMP_ASM(      \
0030     "it " cond "\n\t"           \
0031     "wfe" cond ".n",            \
0032                         \
0033     "nop.w"                 \
0034 )
0035 #else
0036 #define WFE(cond)   __ALT_SMP_ASM("wfe" cond, "nop")
0037 #endif
0038
0039 #define SEV     __ALT_SMP_ASM(WASM(sev), WASM(nop))
0040
0041 static inline void dsb_sev(void)
0042 {
0043
0044     dsb(ishst);
0045     __asm__(SEV);
0046 }
0047
0048 /*
0049  * ARMv6 ticket-based spin-locking.
0050  *
0051  * A memory barrier is required after we get a lock, and before we
0052  * release it, because V6 CPUs are assumed to have weakly ordered
0053  * memory.
0054  */
0055
0056 static inline void arch_spin_lock(arch_spinlock_t *lock)
0057 {
0058     unsigned long tmp;
0059     u32 newval;
0060     arch_spinlock_t lockval;
0061
0062     prefetchw(&lock->slock);
0063     __asm__ __volatile__(
0064 "1: ldrex   %0, [%3]\n"
0065 "   add %1, %0, %4\n"
0066 "   strex   %2, %1, [%3]\n"
0067 "   teq %2, #0\n"
0068 "   bne 1b"
0069     : "=&r" (lockval), "=&r" (newval), "=&r" (tmp)
0070     : "r" (&lock->slock), "I" (1 << TICKET_SHIFT)
0071     : "cc");
0072
0073     while (lockval.tickets.next != lockval.tickets.owner) {
0074         wfe();
0075         lockval.tickets.owner = READ_ONCE(lock->tickets.owner);
0076     }
0077
0078     smp_mb();
0079 }
0080
0081 static inline int arch_spin_trylock(arch_spinlock_t *lock)
0082 {
0083     unsigned long contended, res;
0084     u32 slock;
0085
0086     prefetchw(&lock->slock);
0087     do {
0088         __asm__ __volatile__(
0089         "   ldrex   %0, [%3]\n"
0090         "   mov %2, #0\n"
0091         "   subs    %1, %0, %0, ror #16\n"
0092         "   addeq   %0, %0, %4\n"
0093         "   strexeq %2, %0, [%3]"
0094         : "=&r" (slock), "=&r" (contended), "=&r" (res)
0095         : "r" (&lock->slock), "I" (1 << TICKET_SHIFT)
0096         : "cc");
0097     } while (res);
0098
0099     if (!contended) {
0100         smp_mb();
0101         return 1;
0102     } else {
0103         return 0;
0104     }
0105 }
0106
0107 static inline void arch_spin_unlock(arch_spinlock_t *lock)
0108 {
0109     smp_mb();
0110     lock->tickets.owner++;
0111     dsb_sev();
0112 }
0113
0114 static inline int arch_spin_value_unlocked(arch_spinlock_t lock)
0115 {
0116     return lock.tickets.owner == lock.tickets.next;
0117 }
0118
0119 static inline int arch_spin_is_locked(arch_spinlock_t *lock)
0120 {
0121     return !arch_spin_value_unlocked(READ_ONCE(*lock));
0122 }
0123
0124 static inline int arch_spin_is_contended(arch_spinlock_t *lock)
0125 {
0126     struct __raw_tickets tickets = READ_ONCE(lock->tickets);
0127     return (tickets.next - tickets.owner) > 1;
0128 }
0129 #define arch_spin_is_contended  arch_spin_is_contended
0130
0131 /*
0132  * RWLOCKS
0133  *
0134  *
0135  * Write locks are easy - we just set bit 31.  When unlocking, we can
0136  * just write zero since the lock is exclusively held.
0137  */
0138
0139 static inline void arch_write_lock(arch_rwlock_t *rw)
0140 {
0141     unsigned long tmp;
0142
0143     prefetchw(&rw->lock);
0144     __asm__ __volatile__(
0145 "1: ldrex   %0, [%1]\n"
0146 "   teq %0, #0\n"
0147     WFE("ne")
0148 "   strexeq %0, %2, [%1]\n"
0149 "   teq %0, #0\n"
0150 "   bne 1b"
0151     : "=&r" (tmp)
0152     : "r" (&rw->lock), "r" (0x80000000)
0153     : "cc");
0154
0155     smp_mb();
0156 }
0157
0158 static inline int arch_write_trylock(arch_rwlock_t *rw)
0159 {
0160     unsigned long contended, res;
0161
0162     prefetchw(&rw->lock);
0163     do {
0164         __asm__ __volatile__(
0165         "   ldrex   %0, [%2]\n"
0166         "   mov %1, #0\n"
0167         "   teq %0, #0\n"
0168         "   strexeq %1, %3, [%2]"
0169         : "=&r" (contended), "=&r" (res)
0170         : "r" (&rw->lock), "r" (0x80000000)
0171         : "cc");
0172     } while (res);
0173
0174     if (!contended) {
0175         smp_mb();
0176         return 1;
0177     } else {
0178         return 0;
0179     }
0180 }
0181
0182 static inline void arch_write_unlock(arch_rwlock_t *rw)
0183 {
0184     smp_mb();
0185
0186     __asm__ __volatile__(
0187     "str    %1, [%0]\n"
0188     :
0189     : "r" (&rw->lock), "r" (0)
0190     : "cc");
0191
0192     dsb_sev();
0193 }
0194
0195 /*
0196  * Read locks are a bit more hairy:
0197  *  - Exclusively load the lock value.
0198  *  - Increment it.
0199  *  - Store new lock value if positive, and we still own this location.
0200  *    If the value is negative, we've already failed.
0201  *  - If we failed to store the value, we want a negative result.
0202  *  - If we failed, try again.
0203  * Unlocking is similarly hairy.  We may have multiple read locks
0204  * currently active.  However, we know we won't have any write
0205  * locks.
0206  */
0207 static inline void arch_read_lock(arch_rwlock_t *rw)
0208 {
0209     unsigned long tmp, tmp2;
0210
0211     prefetchw(&rw->lock);
0212     __asm__ __volatile__(
0213 "   .syntax unified\n"
0214 "1: ldrex   %0, [%2]\n"
0215 "   adds    %0, %0, #1\n"
0216 "   strexpl %1, %0, [%2]\n"
0217     WFE("mi")
0218 "   rsbspl  %0, %1, #0\n"
0219 "   bmi 1b"
0220     : "=&r" (tmp), "=&r" (tmp2)
0221     : "r" (&rw->lock)
0222     : "cc");
0223
0224     smp_mb();
0225 }
0226
0227 static inline void arch_read_unlock(arch_rwlock_t *rw)
0228 {
0229     unsigned long tmp, tmp2;
0230
0231     smp_mb();
0232
0233     prefetchw(&rw->lock);
0234     __asm__ __volatile__(
0235 "1: ldrex   %0, [%2]\n"
0236 "   sub %0, %0, #1\n"
0237 "   strex   %1, %0, [%2]\n"
0238 "   teq %1, #0\n"
0239 "   bne 1b"
0240     : "=&r" (tmp), "=&r" (tmp2)
0241     : "r" (&rw->lock)
0242     : "cc");
0243
0244     if (tmp == 0)
0245         dsb_sev();
0246 }
0247
0248 static inline int arch_read_trylock(arch_rwlock_t *rw)
0249 {
0250     unsigned long contended, res;
0251
0252     prefetchw(&rw->lock);
0253     do {
0254         __asm__ __volatile__(
0255         "   ldrex   %0, [%2]\n"
0256         "   mov %1, #0\n"
0257         "   adds    %0, %0, #1\n"
0258         "   strexpl %1, %0, [%2]"
0259         : "=&r" (contended), "=&r" (res)
0260         : "r" (&rw->lock)
0261         : "cc");
0262     } while (res);
0263
0264     /* If the lock is negative, then it is already held for write. */
0265     if (contended < 0x80000000) {
0266         smp_mb();
0267         return 1;
0268     } else {
0269         return 0;
0270     }
0271 }
0272
0273 #endif /* __ASM_SPINLOCK_H */