kernel/futex/futex.h

0001 /* SPDX-License-Identifier: GPL-2.0 */
0002 #ifndef _FUTEX_H
0003 #define _FUTEX_H
0004
0005 #include <linux/futex.h>
0006 #include <linux/rtmutex.h>
0007 #include <linux/sched/wake_q.h>
0008
0009 #ifdef CONFIG_PREEMPT_RT
0010 #include <linux/rcuwait.h>
0011 #endif
0012
0013 #include <asm/futex.h>
0014
0015 /*
0016  * Futex flags used to encode options to functions and preserve them across
0017  * restarts.
0018  */
0019 #ifdef CONFIG_MMU
0020 # define FLAGS_SHARED       0x01
0021 #else
0022 /*
0023  * NOMMU does not have per process address space. Let the compiler optimize
0024  * code away.
0025  */
0026 # define FLAGS_SHARED       0x00
0027 #endif
0028 #define FLAGS_CLOCKRT       0x02
0029 #define FLAGS_HAS_TIMEOUT   0x04
0030
0031 #ifdef CONFIG_FAIL_FUTEX
0032 extern bool should_fail_futex(bool fshared);
0033 #else
0034 static inline bool should_fail_futex(bool fshared)
0035 {
0036     return false;
0037 }
0038 #endif
0039
0040 /*
0041  * Hash buckets are shared by all the futex_keys that hash to the same
0042  * location.  Each key may have multiple futex_q structures, one for each task
0043  * waiting on a futex.
0044  */
0045 struct futex_hash_bucket {
0046     atomic_t waiters;
0047     spinlock_t lock;
0048     struct plist_head chain;
0049 } ____cacheline_aligned_in_smp;
0050
0051 /*
0052  * Priority Inheritance state:
0053  */
0054 struct futex_pi_state {
0055     /*
0056      * list of 'owned' pi_state instances - these have to be
0057      * cleaned up in do_exit() if the task exits prematurely:
0058      */
0059     struct list_head list;
0060
0061     /*
0062      * The PI object:
0063      */
0064     struct rt_mutex_base pi_mutex;
0065
0066     struct task_struct *owner;
0067     refcount_t refcount;
0068
0069     union futex_key key;
0070 } __randomize_layout;
0071
0072 /**
0073  * struct futex_q - The hashed futex queue entry, one per waiting task
0074  * @list:       priority-sorted list of tasks waiting on this futex
0075  * @task:       the task waiting on the futex
0076  * @lock_ptr:       the hash bucket lock
0077  * @key:        the key the futex is hashed on
0078  * @pi_state:       optional priority inheritance state
0079  * @rt_waiter:      rt_waiter storage for use with requeue_pi
0080  * @requeue_pi_key: the requeue_pi target futex key
0081  * @bitset:     bitset for the optional bitmasked wakeup
0082  * @requeue_state:  State field for futex_requeue_pi()
0083  * @requeue_wait:   RCU wait for futex_requeue_pi() (RT only)
0084  *
0085  * We use this hashed waitqueue, instead of a normal wait_queue_entry_t, so
0086  * we can wake only the relevant ones (hashed queues may be shared).
0087  *
0088  * A futex_q has a woken state, just like tasks have TASK_RUNNING.
0089  * It is considered woken when plist_node_empty(&q->list) || q->lock_ptr == 0.
0090  * The order of wakeup is always to make the first condition true, then
0091  * the second.
0092  *
0093  * PI futexes are typically woken before they are removed from the hash list via
0094  * the rt_mutex code. See futex_unqueue_pi().
0095  */
0096 struct futex_q {
0097     struct plist_node list;
0098
0099     struct task_struct *task;
0100     spinlock_t *lock_ptr;
0101     union futex_key key;
0102     struct futex_pi_state *pi_state;
0103     struct rt_mutex_waiter *rt_waiter;
0104     union futex_key *requeue_pi_key;
0105     u32 bitset;
0106     atomic_t requeue_state;
0107 #ifdef CONFIG_PREEMPT_RT
0108     struct rcuwait requeue_wait;
0109 #endif
0110 } __randomize_layout;
0111
0112 extern const struct futex_q futex_q_init;
0113
0114 enum futex_access {
0115     FUTEX_READ,
0116     FUTEX_WRITE
0117 };
0118
0119 extern int get_futex_key(u32 __user *uaddr, bool fshared, union futex_key *key,
0120              enum futex_access rw);
0121
0122 extern struct hrtimer_sleeper *
0123 futex_setup_timer(ktime_t *time, struct hrtimer_sleeper *timeout,
0124           int flags, u64 range_ns);
0125
0126 extern struct futex_hash_bucket *futex_hash(union futex_key *key);
0127
0128 /**
0129  * futex_match - Check whether two futex keys are equal
0130  * @key1:   Pointer to key1
0131  * @key2:   Pointer to key2
0132  *
0133  * Return 1 if two futex_keys are equal, 0 otherwise.
0134  */
0135 static inline int futex_match(union futex_key *key1, union futex_key *key2)
0136 {
0137     return (key1 && key2
0138         && key1->both.word == key2->both.word
0139         && key1->both.ptr == key2->both.ptr
0140         && key1->both.offset == key2->both.offset);
0141 }
0142
0143 extern int futex_wait_setup(u32 __user *uaddr, u32 val, unsigned int flags,
0144                 struct futex_q *q, struct futex_hash_bucket **hb);
0145 extern void futex_wait_queue(struct futex_hash_bucket *hb, struct futex_q *q,
0146                    struct hrtimer_sleeper *timeout);
0147 extern void futex_wake_mark(struct wake_q_head *wake_q, struct futex_q *q);
0148
0149 extern int fault_in_user_writeable(u32 __user *uaddr);
0150 extern int futex_cmpxchg_value_locked(u32 *curval, u32 __user *uaddr, u32 uval, u32 newval);
0151 extern int futex_get_value_locked(u32 *dest, u32 __user *from);
0152 extern struct futex_q *futex_top_waiter(struct futex_hash_bucket *hb, union futex_key *key);
0153
0154 extern void __futex_unqueue(struct futex_q *q);
0155 extern void __futex_queue(struct futex_q *q, struct futex_hash_bucket *hb);
0156 extern int futex_unqueue(struct futex_q *q);
0157
0158 /**
0159  * futex_queue() - Enqueue the futex_q on the futex_hash_bucket
0160  * @q:  The futex_q to enqueue
0161  * @hb: The destination hash bucket
0162  *
0163  * The hb->lock must be held by the caller, and is released here. A call to
0164  * futex_queue() is typically paired with exactly one call to futex_unqueue().  The
0165  * exceptions involve the PI related operations, which may use futex_unqueue_pi()
0166  * or nothing if the unqueue is done as part of the wake process and the unqueue
0167  * state is implicit in the state of woken task (see futex_wait_requeue_pi() for
0168  * an example).
0169  */
0170 static inline void futex_queue(struct futex_q *q, struct futex_hash_bucket *hb)
0171     __releases(&hb->lock)
0172 {
0173     __futex_queue(q, hb);
0174     spin_unlock(&hb->lock);
0175 }
0176
0177 extern void futex_unqueue_pi(struct futex_q *q);
0178
0179 extern void wait_for_owner_exiting(int ret, struct task_struct *exiting);
0180
0181 /*
0182  * Reflects a new waiter being added to the waitqueue.
0183  */
0184 static inline void futex_hb_waiters_inc(struct futex_hash_bucket *hb)
0185 {
0186 #ifdef CONFIG_SMP
0187     atomic_inc(&hb->waiters);
0188     /*
0189      * Full barrier (A), see the ordering comment above.
0190      */
0191     smp_mb__after_atomic();
0192 #endif
0193 }
0194
0195 /*
0196  * Reflects a waiter being removed from the waitqueue by wakeup
0197  * paths.
0198  */
0199 static inline void futex_hb_waiters_dec(struct futex_hash_bucket *hb)
0200 {
0201 #ifdef CONFIG_SMP
0202     atomic_dec(&hb->waiters);
0203 #endif
0204 }
0205
0206 static inline int futex_hb_waiters_pending(struct futex_hash_bucket *hb)
0207 {
0208 #ifdef CONFIG_SMP
0209     /*
0210      * Full barrier (B), see the ordering comment above.
0211      */
0212     smp_mb();
0213     return atomic_read(&hb->waiters);
0214 #else
0215     return 1;
0216 #endif
0217 }
0218
0219 extern struct futex_hash_bucket *futex_q_lock(struct futex_q *q);
0220 extern void futex_q_unlock(struct futex_hash_bucket *hb);
0221
0222
0223 extern int futex_lock_pi_atomic(u32 __user *uaddr, struct futex_hash_bucket *hb,
0224                 union futex_key *key,
0225                 struct futex_pi_state **ps,
0226                 struct task_struct *task,
0227                 struct task_struct **exiting,
0228                 int set_waiters);
0229
0230 extern int refill_pi_state_cache(void);
0231 extern void get_pi_state(struct futex_pi_state *pi_state);
0232 extern void put_pi_state(struct futex_pi_state *pi_state);
0233 extern int fixup_pi_owner(u32 __user *uaddr, struct futex_q *q, int locked);
0234
0235 /*
0236  * Express the locking dependencies for lockdep:
0237  */
0238 static inline void
0239 double_lock_hb(struct futex_hash_bucket *hb1, struct futex_hash_bucket *hb2)
0240 {
0241     if (hb1 > hb2)
0242         swap(hb1, hb2);
0243
0244     spin_lock(&hb1->lock);
0245     if (hb1 != hb2)
0246         spin_lock_nested(&hb2->lock, SINGLE_DEPTH_NESTING);
0247 }
0248
0249 static inline void
0250 double_unlock_hb(struct futex_hash_bucket *hb1, struct futex_hash_bucket *hb2)
0251 {
0252     spin_unlock(&hb1->lock);
0253     if (hb1 != hb2)
0254         spin_unlock(&hb2->lock);
0255 }
0256
0257 /* syscalls */
0258
0259 extern int futex_wait_requeue_pi(u32 __user *uaddr, unsigned int flags, u32
0260                  val, ktime_t *abs_time, u32 bitset, u32 __user
0261                  *uaddr2);
0262
0263 extern int futex_requeue(u32 __user *uaddr1, unsigned int flags,
0264              u32 __user *uaddr2, int nr_wake, int nr_requeue,
0265              u32 *cmpval, int requeue_pi);
0266
0267 extern int futex_wait(u32 __user *uaddr, unsigned int flags, u32 val,
0268               ktime_t *abs_time, u32 bitset);
0269
0270 /**
0271  * struct futex_vector - Auxiliary struct for futex_waitv()
0272  * @w: Userspace provided data
0273  * @q: Kernel side data
0274  *
0275  * Struct used to build an array with all data need for futex_waitv()
0276  */
0277 struct futex_vector {
0278     struct futex_waitv w;
0279     struct futex_q q;
0280 };
0281
0282 extern int futex_wait_multiple(struct futex_vector *vs, unsigned int count,
0283                    struct hrtimer_sleeper *to);
0284
0285 extern int futex_wake(u32 __user *uaddr, unsigned int flags, int nr_wake, u32 bitset);
0286
0287 extern int futex_wake_op(u32 __user *uaddr1, unsigned int flags,
0288              u32 __user *uaddr2, int nr_wake, int nr_wake2, int op);
0289
0290 extern int futex_unlock_pi(u32 __user *uaddr, unsigned int flags);
0291
0292 extern int futex_lock_pi(u32 __user *uaddr, unsigned int flags, ktime_t *time, int trylock);
0293
0294 #endif /* _FUTEX_H */