kernel/futex/syscalls.c

0001 // SPDX-License-Identifier: GPL-2.0-or-later
0002
0003 #include <linux/compat.h>
0004 #include <linux/syscalls.h>
0005 #include <linux/time_namespace.h>
0006
0007 #include "futex.h"
0008
0009 /*
0010  * Support for robust futexes: the kernel cleans up held futexes at
0011  * thread exit time.
0012  *
0013  * Implementation: user-space maintains a per-thread list of locks it
0014  * is holding. Upon do_exit(), the kernel carefully walks this list,
0015  * and marks all locks that are owned by this thread with the
0016  * FUTEX_OWNER_DIED bit, and wakes up a waiter (if any). The list is
0017  * always manipulated with the lock held, so the list is private and
0018  * per-thread. Userspace also maintains a per-thread 'list_op_pending'
0019  * field, to allow the kernel to clean up if the thread dies after
0020  * acquiring the lock, but just before it could have added itself to
0021  * the list. There can only be one such pending lock.
0022  */
0023
0024 /**
0025  * sys_set_robust_list() - Set the robust-futex list head of a task
0026  * @head:   pointer to the list-head
0027  * @len:    length of the list-head, as userspace expects
0028  */
0029 SYSCALL_DEFINE2(set_robust_list, struct robust_list_head __user *, head,
0030         size_t, len)
0031 {
0032     /*
0033      * The kernel knows only one size for now:
0034      */
0035     if (unlikely(len != sizeof(*head)))
0036         return -EINVAL;
0037
0038     current->robust_list = head;
0039
0040     return 0;
0041 }
0042
0043 /**
0044  * sys_get_robust_list() - Get the robust-futex list head of a task
0045  * @pid:    pid of the process [zero for current task]
0046  * @head_ptr:   pointer to a list-head pointer, the kernel fills it in
0047  * @len_ptr:    pointer to a length field, the kernel fills in the header size
0048  */
0049 SYSCALL_DEFINE3(get_robust_list, int, pid,
0050         struct robust_list_head __user * __user *, head_ptr,
0051         size_t __user *, len_ptr)
0052 {
0053     struct robust_list_head __user *head;
0054     unsigned long ret;
0055     struct task_struct *p;
0056
0057     rcu_read_lock();
0058
0059     ret = -ESRCH;
0060     if (!pid)
0061         p = current;
0062     else {
0063         p = find_task_by_vpid(pid);
0064         if (!p)
0065             goto err_unlock;
0066     }
0067
0068     ret = -EPERM;
0069     if (!ptrace_may_access(p, PTRACE_MODE_READ_REALCREDS))
0070         goto err_unlock;
0071
0072     head = p->robust_list;
0073     rcu_read_unlock();
0074
0075     if (put_user(sizeof(*head), len_ptr))
0076         return -EFAULT;
0077     return put_user(head, head_ptr);
0078
0079 err_unlock:
0080     rcu_read_unlock();
0081
0082     return ret;
0083 }
0084
0085 long do_futex(u32 __user *uaddr, int op, u32 val, ktime_t *timeout,
0086         u32 __user *uaddr2, u32 val2, u32 val3)
0087 {
0088     int cmd = op & FUTEX_CMD_MASK;
0089     unsigned int flags = 0;
0090
0091     if (!(op & FUTEX_PRIVATE_FLAG))
0092         flags |= FLAGS_SHARED;
0093
0094     if (op & FUTEX_CLOCK_REALTIME) {
0095         flags |= FLAGS_CLOCKRT;
0096         if (cmd != FUTEX_WAIT_BITSET && cmd != FUTEX_WAIT_REQUEUE_PI &&
0097             cmd != FUTEX_LOCK_PI2)
0098             return -ENOSYS;
0099     }
0100
0101     switch (cmd) {
0102     case FUTEX_WAIT:
0103         val3 = FUTEX_BITSET_MATCH_ANY;
0104         fallthrough;
0105     case FUTEX_WAIT_BITSET:
0106         return futex_wait(uaddr, flags, val, timeout, val3);
0107     case FUTEX_WAKE:
0108         val3 = FUTEX_BITSET_MATCH_ANY;
0109         fallthrough;
0110     case FUTEX_WAKE_BITSET:
0111         return futex_wake(uaddr, flags, val, val3);
0112     case FUTEX_REQUEUE:
0113         return futex_requeue(uaddr, flags, uaddr2, val, val2, NULL, 0);
0114     case FUTEX_CMP_REQUEUE:
0115         return futex_requeue(uaddr, flags, uaddr2, val, val2, &val3, 0);
0116     case FUTEX_WAKE_OP:
0117         return futex_wake_op(uaddr, flags, uaddr2, val, val2, val3);
0118     case FUTEX_LOCK_PI:
0119         flags |= FLAGS_CLOCKRT;
0120         fallthrough;
0121     case FUTEX_LOCK_PI2:
0122         return futex_lock_pi(uaddr, flags, timeout, 0);
0123     case FUTEX_UNLOCK_PI:
0124         return futex_unlock_pi(uaddr, flags);
0125     case FUTEX_TRYLOCK_PI:
0126         return futex_lock_pi(uaddr, flags, NULL, 1);
0127     case FUTEX_WAIT_REQUEUE_PI:
0128         val3 = FUTEX_BITSET_MATCH_ANY;
0129         return futex_wait_requeue_pi(uaddr, flags, val, timeout, val3,
0130                          uaddr2);
0131     case FUTEX_CMP_REQUEUE_PI:
0132         return futex_requeue(uaddr, flags, uaddr2, val, val2, &val3, 1);
0133     }
0134     return -ENOSYS;
0135 }
0136
0137 static __always_inline bool futex_cmd_has_timeout(u32 cmd)
0138 {
0139     switch (cmd) {
0140     case FUTEX_WAIT:
0141     case FUTEX_LOCK_PI:
0142     case FUTEX_LOCK_PI2:
0143     case FUTEX_WAIT_BITSET:
0144     case FUTEX_WAIT_REQUEUE_PI:
0145         return true;
0146     }
0147     return false;
0148 }
0149
0150 static __always_inline int
0151 futex_init_timeout(u32 cmd, u32 op, struct timespec64 *ts, ktime_t *t)
0152 {
0153     if (!timespec64_valid(ts))
0154         return -EINVAL;
0155
0156     *t = timespec64_to_ktime(*ts);
0157     if (cmd == FUTEX_WAIT)
0158         *t = ktime_add_safe(ktime_get(), *t);
0159     else if (cmd != FUTEX_LOCK_PI && !(op & FUTEX_CLOCK_REALTIME))
0160         *t = timens_ktime_to_host(CLOCK_MONOTONIC, *t);
0161     return 0;
0162 }
0163
0164 SYSCALL_DEFINE6(futex, u32 __user *, uaddr, int, op, u32, val,
0165         const struct __kernel_timespec __user *, utime,
0166         u32 __user *, uaddr2, u32, val3)
0167 {
0168     int ret, cmd = op & FUTEX_CMD_MASK;
0169     ktime_t t, *tp = NULL;
0170     struct timespec64 ts;
0171
0172     if (utime && futex_cmd_has_timeout(cmd)) {
0173         if (unlikely(should_fail_futex(!(op & FUTEX_PRIVATE_FLAG))))
0174             return -EFAULT;
0175         if (get_timespec64(&ts, utime))
0176             return -EFAULT;
0177         ret = futex_init_timeout(cmd, op, &ts, &t);
0178         if (ret)
0179             return ret;
0180         tp = &t;
0181     }
0182
0183     return do_futex(uaddr, op, val, tp, uaddr2, (unsigned long)utime, val3);
0184 }
0185
0186 /* Mask of available flags for each futex in futex_waitv list */
0187 #define FUTEXV_WAITER_MASK (FUTEX_32 | FUTEX_PRIVATE_FLAG)
0188
0189 /**
0190  * futex_parse_waitv - Parse a waitv array from userspace
0191  * @futexv: Kernel side list of waiters to be filled
0192  * @uwaitv:     Userspace list to be parsed
0193  * @nr_futexes: Length of futexv
0194  *
0195  * Return: Error code on failure, 0 on success
0196  */
0197 static int futex_parse_waitv(struct futex_vector *futexv,
0198                  struct futex_waitv __user *uwaitv,
0199                  unsigned int nr_futexes)
0200 {
0201     struct futex_waitv aux;
0202     unsigned int i;
0203
0204     for (i = 0; i < nr_futexes; i++) {
0205         if (copy_from_user(&aux, &uwaitv[i], sizeof(aux)))
0206             return -EFAULT;
0207
0208         if ((aux.flags & ~FUTEXV_WAITER_MASK) || aux.__reserved)
0209             return -EINVAL;
0210
0211         if (!(aux.flags & FUTEX_32))
0212             return -EINVAL;
0213
0214         futexv[i].w.flags = aux.flags;
0215         futexv[i].w.val = aux.val;
0216         futexv[i].w.uaddr = aux.uaddr;
0217         futexv[i].q = futex_q_init;
0218     }
0219
0220     return 0;
0221 }
0222
0223 /**
0224  * sys_futex_waitv - Wait on a list of futexes
0225  * @waiters:    List of futexes to wait on
0226  * @nr_futexes: Length of futexv
0227  * @flags:      Flag for timeout (monotonic/realtime)
0228  * @timeout:    Optional absolute timeout.
0229  * @clockid:    Clock to be used for the timeout, realtime or monotonic.
0230  *
0231  * Given an array of `struct futex_waitv`, wait on each uaddr. The thread wakes
0232  * if a futex_wake() is performed at any uaddr. The syscall returns immediately
0233  * if any waiter has *uaddr != val. *timeout is an optional timeout value for
0234  * the operation. Each waiter has individual flags. The `flags` argument for
0235  * the syscall should be used solely for specifying the timeout as realtime, if
0236  * needed. Flags for private futexes, sizes, etc. should be used on the
0237  * individual flags of each waiter.
0238  *
0239  * Returns the array index of one of the woken futexes. No further information
0240  * is provided: any number of other futexes may also have been woken by the
0241  * same event, and if more than one futex was woken, the retrned index may
0242  * refer to any one of them. (It is not necessaryily the futex with the
0243  * smallest index, nor the one most recently woken, nor...)
0244  */
0245
0246 SYSCALL_DEFINE5(futex_waitv, struct futex_waitv __user *, waiters,
0247         unsigned int, nr_futexes, unsigned int, flags,
0248         struct __kernel_timespec __user *, timeout, clockid_t, clockid)
0249 {
0250     struct hrtimer_sleeper to;
0251     struct futex_vector *futexv;
0252     struct timespec64 ts;
0253     ktime_t time;
0254     int ret;
0255
0256     /* This syscall supports no flags for now */
0257     if (flags)
0258         return -EINVAL;
0259
0260     if (!nr_futexes || nr_futexes > FUTEX_WAITV_MAX || !waiters)
0261         return -EINVAL;
0262
0263     if (timeout) {
0264         int flag_clkid = 0, flag_init = 0;
0265
0266         if (clockid == CLOCK_REALTIME) {
0267             flag_clkid = FLAGS_CLOCKRT;
0268             flag_init = FUTEX_CLOCK_REALTIME;
0269         }
0270
0271         if (clockid != CLOCK_REALTIME && clockid != CLOCK_MONOTONIC)
0272             return -EINVAL;
0273
0274         if (get_timespec64(&ts, timeout))
0275             return -EFAULT;
0276
0277         /*
0278          * Since there's no opcode for futex_waitv, use
0279          * FUTEX_WAIT_BITSET that uses absolute timeout as well
0280          */
0281         ret = futex_init_timeout(FUTEX_WAIT_BITSET, flag_init, &ts, &time);
0282         if (ret)
0283             return ret;
0284
0285         futex_setup_timer(&time, &to, flag_clkid, 0);
0286     }
0287
0288     futexv = kcalloc(nr_futexes, sizeof(*futexv), GFP_KERNEL);
0289     if (!futexv)
0290         return -ENOMEM;
0291
0292     ret = futex_parse_waitv(futexv, waiters, nr_futexes);
0293     if (!ret)
0294         ret = futex_wait_multiple(futexv, nr_futexes, timeout ? &to : NULL);
0295
0296     if (timeout) {
0297         hrtimer_cancel(&to.timer);
0298         destroy_hrtimer_on_stack(&to.timer);
0299     }
0300
0301     kfree(futexv);
0302     return ret;
0303 }
0304
0305 #ifdef CONFIG_COMPAT
0306 COMPAT_SYSCALL_DEFINE2(set_robust_list,
0307         struct compat_robust_list_head __user *, head,
0308         compat_size_t, len)
0309 {
0310     if (unlikely(len != sizeof(*head)))
0311         return -EINVAL;
0312
0313     current->compat_robust_list = head;
0314
0315     return 0;
0316 }
0317
0318 COMPAT_SYSCALL_DEFINE3(get_robust_list, int, pid,
0319             compat_uptr_t __user *, head_ptr,
0320             compat_size_t __user *, len_ptr)
0321 {
0322     struct compat_robust_list_head __user *head;
0323     unsigned long ret;
0324     struct task_struct *p;
0325
0326     rcu_read_lock();
0327
0328     ret = -ESRCH;
0329     if (!pid)
0330         p = current;
0331     else {
0332         p = find_task_by_vpid(pid);
0333         if (!p)
0334             goto err_unlock;
0335     }
0336
0337     ret = -EPERM;
0338     if (!ptrace_may_access(p, PTRACE_MODE_READ_REALCREDS))
0339         goto err_unlock;
0340
0341     head = p->compat_robust_list;
0342     rcu_read_unlock();
0343
0344     if (put_user(sizeof(*head), len_ptr))
0345         return -EFAULT;
0346     return put_user(ptr_to_compat(head), head_ptr);
0347
0348 err_unlock:
0349     rcu_read_unlock();
0350
0351     return ret;
0352 }
0353 #endif /* CONFIG_COMPAT */
0354
0355 #ifdef CONFIG_COMPAT_32BIT_TIME
0356 SYSCALL_DEFINE6(futex_time32, u32 __user *, uaddr, int, op, u32, val,
0357         const struct old_timespec32 __user *, utime, u32 __user *, uaddr2,
0358         u32, val3)
0359 {
0360     int ret, cmd = op & FUTEX_CMD_MASK;
0361     ktime_t t, *tp = NULL;
0362     struct timespec64 ts;
0363
0364     if (utime && futex_cmd_has_timeout(cmd)) {
0365         if (get_old_timespec32(&ts, utime))
0366             return -EFAULT;
0367         ret = futex_init_timeout(cmd, op, &ts, &t);
0368         if (ret)
0369             return ret;
0370         tp = &t;
0371     }
0372
0373     return do_futex(uaddr, op, val, tp, uaddr2, (unsigned long)utime, val3);
0374 }
0375 #endif /* CONFIG_COMPAT_32BIT_TIME */
0376