s390/lib/spinlock.c

0001 // SPDX-License-Identifier: GPL-2.0
0002 /*
0003  *    Out of line spinlock code.
0004  *
0005  *    Copyright IBM Corp. 2004, 2006
0006  *    Author(s): Martin Schwidefsky (schwidefsky@de.ibm.com)
0007  */
0008
0009 #include <linux/types.h>
0010 #include <linux/export.h>
0011 #include <linux/spinlock.h>
0012 #include <linux/jiffies.h>
0013 #include <linux/init.h>
0014 #include <linux/smp.h>
0015 #include <linux/percpu.h>
0016 #include <asm/alternative.h>
0017 #include <asm/io.h>
0018
0019 int spin_retry = -1;
0020
0021 static int __init spin_retry_init(void)
0022 {
0023     if (spin_retry < 0)
0024         spin_retry = 1000;
0025     return 0;
0026 }
0027 early_initcall(spin_retry_init);
0028
0029 /*
0030  * spin_retry= parameter
0031  */
0032 static int __init spin_retry_setup(char *str)
0033 {
0034     spin_retry = simple_strtoul(str, &str, 0);
0035     return 1;
0036 }
0037 __setup("spin_retry=", spin_retry_setup);
0038
0039 struct spin_wait {
0040     struct spin_wait *next, *prev;
0041     int node_id;
0042 } __aligned(32);
0043
0044 static DEFINE_PER_CPU_ALIGNED(struct spin_wait, spin_wait[4]);
0045
0046 #define _Q_LOCK_CPU_OFFSET  0
0047 #define _Q_LOCK_STEAL_OFFSET    16
0048 #define _Q_TAIL_IDX_OFFSET  18
0049 #define _Q_TAIL_CPU_OFFSET  20
0050
0051 #define _Q_LOCK_CPU_MASK    0x0000ffff
0052 #define _Q_LOCK_STEAL_ADD   0x00010000
0053 #define _Q_LOCK_STEAL_MASK  0x00030000
0054 #define _Q_TAIL_IDX_MASK    0x000c0000
0055 #define _Q_TAIL_CPU_MASK    0xfff00000
0056
0057 #define _Q_LOCK_MASK        (_Q_LOCK_CPU_MASK | _Q_LOCK_STEAL_MASK)
0058 #define _Q_TAIL_MASK        (_Q_TAIL_IDX_MASK | _Q_TAIL_CPU_MASK)
0059
0060 void arch_spin_lock_setup(int cpu)
0061 {
0062     struct spin_wait *node;
0063     int ix;
0064
0065     node = per_cpu_ptr(&spin_wait[0], cpu);
0066     for (ix = 0; ix < 4; ix++, node++) {
0067         memset(node, 0, sizeof(*node));
0068         node->node_id = ((cpu + 1) << _Q_TAIL_CPU_OFFSET) +
0069             (ix << _Q_TAIL_IDX_OFFSET);
0070     }
0071 }
0072
0073 static inline int arch_load_niai4(int *lock)
0074 {
0075     int owner;
0076
0077     asm_inline volatile(
0078         ALTERNATIVE("nop", ".insn rre,0xb2fa0000,4,0", 49) /* NIAI 4 */
0079         "   l   %0,%1\n"
0080         : "=d" (owner) : "Q" (*lock) : "memory");
0081     return owner;
0082 }
0083
0084 static inline int arch_cmpxchg_niai8(int *lock, int old, int new)
0085 {
0086     int expected = old;
0087
0088     asm_inline volatile(
0089         ALTERNATIVE("nop", ".insn rre,0xb2fa0000,8,0", 49) /* NIAI 8 */
0090         "   cs  %0,%3,%1\n"
0091         : "=d" (old), "=Q" (*lock)
0092         : "0" (old), "d" (new), "Q" (*lock)
0093         : "cc", "memory");
0094     return expected == old;
0095 }
0096
0097 static inline struct spin_wait *arch_spin_decode_tail(int lock)
0098 {
0099     int ix, cpu;
0100
0101     ix = (lock & _Q_TAIL_IDX_MASK) >> _Q_TAIL_IDX_OFFSET;
0102     cpu = (lock & _Q_TAIL_CPU_MASK) >> _Q_TAIL_CPU_OFFSET;
0103     return per_cpu_ptr(&spin_wait[ix], cpu - 1);
0104 }
0105
0106 static inline int arch_spin_yield_target(int lock, struct spin_wait *node)
0107 {
0108     if (lock & _Q_LOCK_CPU_MASK)
0109         return lock & _Q_LOCK_CPU_MASK;
0110     if (node == NULL || node->prev == NULL)
0111         return 0;   /* 0 -> no target cpu */
0112     while (node->prev)
0113         node = node->prev;
0114     return node->node_id >> _Q_TAIL_CPU_OFFSET;
0115 }
0116
0117 static inline void arch_spin_lock_queued(arch_spinlock_t *lp)
0118 {
0119     struct spin_wait *node, *next;
0120     int lockval, ix, node_id, tail_id, old, new, owner, count;
0121
0122     ix = S390_lowcore.spinlock_index++;
0123     barrier();
0124     lockval = SPINLOCK_LOCKVAL; /* cpu + 1 */
0125     node = this_cpu_ptr(&spin_wait[ix]);
0126     node->prev = node->next = NULL;
0127     node_id = node->node_id;
0128
0129     /* Enqueue the node for this CPU in the spinlock wait queue */
0130     while (1) {
0131         old = READ_ONCE(lp->lock);
0132         if ((old & _Q_LOCK_CPU_MASK) == 0 &&
0133             (old & _Q_LOCK_STEAL_MASK) != _Q_LOCK_STEAL_MASK) {
0134             /*
0135              * The lock is free but there may be waiters.
0136              * With no waiters simply take the lock, if there
0137              * are waiters try to steal the lock. The lock may
0138              * be stolen three times before the next queued
0139              * waiter will get the lock.
0140              */
0141             new = (old ? (old + _Q_LOCK_STEAL_ADD) : 0) | lockval;
0142             if (__atomic_cmpxchg_bool(&lp->lock, old, new))
0143                 /* Got the lock */
0144                 goto out;
0145             /* lock passing in progress */
0146             continue;
0147         }
0148         /* Make the node of this CPU the new tail. */
0149         new = node_id | (old & _Q_LOCK_MASK);
0150         if (__atomic_cmpxchg_bool(&lp->lock, old, new))
0151             break;
0152     }
0153     /* Set the 'next' pointer of the tail node in the queue */
0154     tail_id = old & _Q_TAIL_MASK;
0155     if (tail_id != 0) {
0156         node->prev = arch_spin_decode_tail(tail_id);
0157         WRITE_ONCE(node->prev->next, node);
0158     }
0159
0160     /* Pass the virtual CPU to the lock holder if it is not running */
0161     owner = arch_spin_yield_target(old, node);
0162     if (owner && arch_vcpu_is_preempted(owner - 1))
0163         smp_yield_cpu(owner - 1);
0164
0165     /* Spin on the CPU local node->prev pointer */
0166     if (tail_id != 0) {
0167         count = spin_retry;
0168         while (READ_ONCE(node->prev) != NULL) {
0169             if (count-- >= 0)
0170                 continue;
0171             count = spin_retry;
0172             /* Query running state of lock holder again. */
0173             owner = arch_spin_yield_target(old, node);
0174             if (owner && arch_vcpu_is_preempted(owner - 1))
0175                 smp_yield_cpu(owner - 1);
0176         }
0177     }
0178
0179     /* Spin on the lock value in the spinlock_t */
0180     count = spin_retry;
0181     while (1) {
0182         old = READ_ONCE(lp->lock);
0183         owner = old & _Q_LOCK_CPU_MASK;
0184         if (!owner) {
0185             tail_id = old & _Q_TAIL_MASK;
0186             new = ((tail_id != node_id) ? tail_id : 0) | lockval;
0187             if (__atomic_cmpxchg_bool(&lp->lock, old, new))
0188                 /* Got the lock */
0189                 break;
0190             continue;
0191         }
0192         if (count-- >= 0)
0193             continue;
0194         count = spin_retry;
0195         if (!MACHINE_IS_LPAR || arch_vcpu_is_preempted(owner - 1))
0196             smp_yield_cpu(owner - 1);
0197     }
0198
0199     /* Pass lock_spin job to next CPU in the queue */
0200     if (node_id && tail_id != node_id) {
0201         /* Wait until the next CPU has set up the 'next' pointer */
0202         while ((next = READ_ONCE(node->next)) == NULL)
0203             ;
0204         next->prev = NULL;
0205     }
0206
0207  out:
0208     S390_lowcore.spinlock_index--;
0209 }
0210
0211 static inline void arch_spin_lock_classic(arch_spinlock_t *lp)
0212 {
0213     int lockval, old, new, owner, count;
0214
0215     lockval = SPINLOCK_LOCKVAL; /* cpu + 1 */
0216
0217     /* Pass the virtual CPU to the lock holder if it is not running */
0218     owner = arch_spin_yield_target(READ_ONCE(lp->lock), NULL);
0219     if (owner && arch_vcpu_is_preempted(owner - 1))
0220         smp_yield_cpu(owner - 1);
0221
0222     count = spin_retry;
0223     while (1) {
0224         old = arch_load_niai4(&lp->lock);
0225         owner = old & _Q_LOCK_CPU_MASK;
0226         /* Try to get the lock if it is free. */
0227         if (!owner) {
0228             new = (old & _Q_TAIL_MASK) | lockval;
0229             if (arch_cmpxchg_niai8(&lp->lock, old, new)) {
0230                 /* Got the lock */
0231                 return;
0232             }
0233             continue;
0234         }
0235         if (count-- >= 0)
0236             continue;
0237         count = spin_retry;
0238         if (!MACHINE_IS_LPAR || arch_vcpu_is_preempted(owner - 1))
0239             smp_yield_cpu(owner - 1);
0240     }
0241 }
0242
0243 void arch_spin_lock_wait(arch_spinlock_t *lp)
0244 {
0245     if (test_cpu_flag(CIF_DEDICATED_CPU))
0246         arch_spin_lock_queued(lp);
0247     else
0248         arch_spin_lock_classic(lp);
0249 }
0250 EXPORT_SYMBOL(arch_spin_lock_wait);
0251
0252 int arch_spin_trylock_retry(arch_spinlock_t *lp)
0253 {
0254     int cpu = SPINLOCK_LOCKVAL;
0255     int owner, count;
0256
0257     for (count = spin_retry; count > 0; count--) {
0258         owner = READ_ONCE(lp->lock);
0259         /* Try to get the lock if it is free. */
0260         if (!owner) {
0261             if (__atomic_cmpxchg_bool(&lp->lock, 0, cpu))
0262                 return 1;
0263         }
0264     }
0265     return 0;
0266 }
0267 EXPORT_SYMBOL(arch_spin_trylock_retry);
0268
0269 void arch_read_lock_wait(arch_rwlock_t *rw)
0270 {
0271     if (unlikely(in_interrupt())) {
0272         while (READ_ONCE(rw->cnts) & 0x10000)
0273             barrier();
0274         return;
0275     }
0276
0277     /* Remove this reader again to allow recursive read locking */
0278     __atomic_add_const(-1, &rw->cnts);
0279     /* Put the reader into the wait queue */
0280     arch_spin_lock(&rw->wait);
0281     /* Now add this reader to the count value again */
0282     __atomic_add_const(1, &rw->cnts);
0283     /* Loop until the writer is done */
0284     while (READ_ONCE(rw->cnts) & 0x10000)
0285         barrier();
0286     arch_spin_unlock(&rw->wait);
0287 }
0288 EXPORT_SYMBOL(arch_read_lock_wait);
0289
0290 void arch_write_lock_wait(arch_rwlock_t *rw)
0291 {
0292     int old;
0293
0294     /* Add this CPU to the write waiters */
0295     __atomic_add(0x20000, &rw->cnts);
0296
0297     /* Put the writer into the wait queue */
0298     arch_spin_lock(&rw->wait);
0299
0300     while (1) {
0301         old = READ_ONCE(rw->cnts);
0302         if ((old & 0x1ffff) == 0 &&
0303             __atomic_cmpxchg_bool(&rw->cnts, old, old | 0x10000))
0304             /* Got the lock */
0305             break;
0306         barrier();
0307     }
0308
0309     arch_spin_unlock(&rw->wait);
0310 }
0311 EXPORT_SYMBOL(arch_write_lock_wait);
0312
0313 void arch_spin_relax(arch_spinlock_t *lp)
0314 {
0315     int cpu;
0316
0317     cpu = READ_ONCE(lp->lock) & _Q_LOCK_CPU_MASK;
0318     if (!cpu)
0319         return;
0320     if (MACHINE_IS_LPAR && !arch_vcpu_is_preempted(cpu - 1))
0321         return;
0322     smp_yield_cpu(cpu - 1);
0323 }
0324 EXPORT_SYMBOL(arch_spin_relax);