Back to home page

LXR

 
 

    


0001 /*
0002  * linux/ipc/sem.c
0003  * Copyright (C) 1992 Krishna Balasubramanian
0004  * Copyright (C) 1995 Eric Schenk, Bruno Haible
0005  *
0006  * /proc/sysvipc/sem support (c) 1999 Dragos Acostachioaie <dragos@iname.com>
0007  *
0008  * SMP-threaded, sysctl's added
0009  * (c) 1999 Manfred Spraul <manfred@colorfullife.com>
0010  * Enforced range limit on SEM_UNDO
0011  * (c) 2001 Red Hat Inc
0012  * Lockless wakeup
0013  * (c) 2003 Manfred Spraul <manfred@colorfullife.com>
0014  * (c) 2016 Davidlohr Bueso <dave@stgolabs.net>
0015  * Further wakeup optimizations, documentation
0016  * (c) 2010 Manfred Spraul <manfred@colorfullife.com>
0017  *
0018  * support for audit of ipc object properties and permission changes
0019  * Dustin Kirkland <dustin.kirkland@us.ibm.com>
0020  *
0021  * namespaces support
0022  * OpenVZ, SWsoft Inc.
0023  * Pavel Emelianov <xemul@openvz.org>
0024  *
0025  * Implementation notes: (May 2010)
0026  * This file implements System V semaphores.
0027  *
0028  * User space visible behavior:
0029  * - FIFO ordering for semop() operations (just FIFO, not starvation
0030  *   protection)
0031  * - multiple semaphore operations that alter the same semaphore in
0032  *   one semop() are handled.
0033  * - sem_ctime (time of last semctl()) is updated in the IPC_SET, SETVAL and
0034  *   SETALL calls.
0035  * - two Linux specific semctl() commands: SEM_STAT, SEM_INFO.
0036  * - undo adjustments at process exit are limited to 0..SEMVMX.
0037  * - namespace are supported.
0038  * - SEMMSL, SEMMNS, SEMOPM and SEMMNI can be configured at runtine by writing
0039  *   to /proc/sys/kernel/sem.
0040  * - statistics about the usage are reported in /proc/sysvipc/sem.
0041  *
0042  * Internals:
0043  * - scalability:
0044  *   - all global variables are read-mostly.
0045  *   - semop() calls and semctl(RMID) are synchronized by RCU.
0046  *   - most operations do write operations (actually: spin_lock calls) to
0047  *     the per-semaphore array structure.
0048  *   Thus: Perfect SMP scaling between independent semaphore arrays.
0049  *         If multiple semaphores in one array are used, then cache line
0050  *         trashing on the semaphore array spinlock will limit the scaling.
0051  * - semncnt and semzcnt are calculated on demand in count_semcnt()
0052  * - the task that performs a successful semop() scans the list of all
0053  *   sleeping tasks and completes any pending operations that can be fulfilled.
0054  *   Semaphores are actively given to waiting tasks (necessary for FIFO).
0055  *   (see update_queue())
0056  * - To improve the scalability, the actual wake-up calls are performed after
0057  *   dropping all locks. (see wake_up_sem_queue_prepare())
0058  * - All work is done by the waker, the woken up task does not have to do
0059  *   anything - not even acquiring a lock or dropping a refcount.
0060  * - A woken up task may not even touch the semaphore array anymore, it may
0061  *   have been destroyed already by a semctl(RMID).
0062  * - UNDO values are stored in an array (one per process and per
0063  *   semaphore array, lazily allocated). For backwards compatibility, multiple
0064  *   modes for the UNDO variables are supported (per process, per thread)
0065  *   (see copy_semundo, CLONE_SYSVSEM)
0066  * - There are two lists of the pending operations: a per-array list
0067  *   and per-semaphore list (stored in the array). This allows to achieve FIFO
0068  *   ordering without always scanning all pending operations.
0069  *   The worst-case behavior is nevertheless O(N^2) for N wakeups.
0070  */
0071 
0072 #include <linux/slab.h>
0073 #include <linux/spinlock.h>
0074 #include <linux/init.h>
0075 #include <linux/proc_fs.h>
0076 #include <linux/time.h>
0077 #include <linux/security.h>
0078 #include <linux/syscalls.h>
0079 #include <linux/audit.h>
0080 #include <linux/capability.h>
0081 #include <linux/seq_file.h>
0082 #include <linux/rwsem.h>
0083 #include <linux/nsproxy.h>
0084 #include <linux/ipc_namespace.h>
0085 
0086 #include <linux/uaccess.h>
0087 #include "util.h"
0088 
0089 /* One semaphore structure for each semaphore in the system. */
0090 struct sem {
0091     int semval;     /* current value */
0092     /*
0093      * PID of the process that last modified the semaphore. For
0094      * Linux, specifically these are:
0095      *  - semop
0096      *  - semctl, via SETVAL and SETALL.
0097      *  - at task exit when performing undo adjustments (see exit_sem).
0098      */
0099     int sempid;
0100     spinlock_t  lock;   /* spinlock for fine-grained semtimedop */
0101     struct list_head pending_alter; /* pending single-sop operations */
0102                     /* that alter the semaphore */
0103     struct list_head pending_const; /* pending single-sop operations */
0104                     /* that do not alter the semaphore*/
0105     time_t  sem_otime;  /* candidate for sem_otime */
0106 } ____cacheline_aligned_in_smp;
0107 
0108 /* One queue for each sleeping process in the system. */
0109 struct sem_queue {
0110     struct list_head    list;    /* queue of pending operations */
0111     struct task_struct  *sleeper; /* this process */
0112     struct sem_undo     *undo;   /* undo structure */
0113     int         pid;     /* process id of requesting process */
0114     int         status;  /* completion status of operation */
0115     struct sembuf       *sops;   /* array of pending operations */
0116     struct sembuf       *blocking; /* the operation that blocked */
0117     int         nsops;   /* number of operations */
0118     bool            alter;   /* does *sops alter the array? */
0119     bool                    dupsop;  /* sops on more than one sem_num */
0120 };
0121 
0122 /* Each task has a list of undo requests. They are executed automatically
0123  * when the process exits.
0124  */
0125 struct sem_undo {
0126     struct list_head    list_proc;  /* per-process list: *
0127                          * all undos from one process
0128                          * rcu protected */
0129     struct rcu_head     rcu;        /* rcu struct for sem_undo */
0130     struct sem_undo_list    *ulp;       /* back ptr to sem_undo_list */
0131     struct list_head    list_id;    /* per semaphore array list:
0132                          * all undos for one array */
0133     int         semid;      /* semaphore set identifier */
0134     short           *semadj;    /* array of adjustments */
0135                         /* one per semaphore */
0136 };
0137 
0138 /* sem_undo_list controls shared access to the list of sem_undo structures
0139  * that may be shared among all a CLONE_SYSVSEM task group.
0140  */
0141 struct sem_undo_list {
0142     atomic_t        refcnt;
0143     spinlock_t      lock;
0144     struct list_head    list_proc;
0145 };
0146 
0147 
0148 #define sem_ids(ns) ((ns)->ids[IPC_SEM_IDS])
0149 
0150 #define sem_checkid(sma, semid) ipc_checkid(&sma->sem_perm, semid)
0151 
0152 static int newary(struct ipc_namespace *, struct ipc_params *);
0153 static void freeary(struct ipc_namespace *, struct kern_ipc_perm *);
0154 #ifdef CONFIG_PROC_FS
0155 static int sysvipc_sem_proc_show(struct seq_file *s, void *it);
0156 #endif
0157 
0158 #define SEMMSL_FAST 256 /* 512 bytes on stack */
0159 #define SEMOPM_FAST 64  /* ~ 372 bytes on stack */
0160 
0161 /*
0162  * Locking:
0163  * a) global sem_lock() for read/write
0164  *  sem_undo.id_next,
0165  *  sem_array.complex_count,
0166  *  sem_array.complex_mode
0167  *  sem_array.pending{_alter,_const},
0168  *  sem_array.sem_undo
0169  *
0170  * b) global or semaphore sem_lock() for read/write:
0171  *  sem_array.sem_base[i].pending_{const,alter}:
0172  *  sem_array.complex_mode (for read)
0173  *
0174  * c) special:
0175  *  sem_undo_list.list_proc:
0176  *  * undo_list->lock for write
0177  *  * rcu for read
0178  */
0179 
0180 #define sc_semmsl   sem_ctls[0]
0181 #define sc_semmns   sem_ctls[1]
0182 #define sc_semopm   sem_ctls[2]
0183 #define sc_semmni   sem_ctls[3]
0184 
0185 void sem_init_ns(struct ipc_namespace *ns)
0186 {
0187     ns->sc_semmsl = SEMMSL;
0188     ns->sc_semmns = SEMMNS;
0189     ns->sc_semopm = SEMOPM;
0190     ns->sc_semmni = SEMMNI;
0191     ns->used_sems = 0;
0192     ipc_init_ids(&ns->ids[IPC_SEM_IDS]);
0193 }
0194 
0195 #ifdef CONFIG_IPC_NS
0196 void sem_exit_ns(struct ipc_namespace *ns)
0197 {
0198     free_ipcs(ns, &sem_ids(ns), freeary);
0199     idr_destroy(&ns->ids[IPC_SEM_IDS].ipcs_idr);
0200 }
0201 #endif
0202 
0203 void __init sem_init(void)
0204 {
0205     sem_init_ns(&init_ipc_ns);
0206     ipc_init_proc_interface("sysvipc/sem",
0207                 "       key      semid perms      nsems   uid   gid  cuid  cgid      otime      ctime\n",
0208                 IPC_SEM_IDS, sysvipc_sem_proc_show);
0209 }
0210 
0211 /**
0212  * unmerge_queues - unmerge queues, if possible.
0213  * @sma: semaphore array
0214  *
0215  * The function unmerges the wait queues if complex_count is 0.
0216  * It must be called prior to dropping the global semaphore array lock.
0217  */
0218 static void unmerge_queues(struct sem_array *sma)
0219 {
0220     struct sem_queue *q, *tq;
0221 
0222     /* complex operations still around? */
0223     if (sma->complex_count)
0224         return;
0225     /*
0226      * We will switch back to simple mode.
0227      * Move all pending operation back into the per-semaphore
0228      * queues.
0229      */
0230     list_for_each_entry_safe(q, tq, &sma->pending_alter, list) {
0231         struct sem *curr;
0232         curr = &sma->sem_base[q->sops[0].sem_num];
0233 
0234         list_add_tail(&q->list, &curr->pending_alter);
0235     }
0236     INIT_LIST_HEAD(&sma->pending_alter);
0237 }
0238 
0239 /**
0240  * merge_queues - merge single semop queues into global queue
0241  * @sma: semaphore array
0242  *
0243  * This function merges all per-semaphore queues into the global queue.
0244  * It is necessary to achieve FIFO ordering for the pending single-sop
0245  * operations when a multi-semop operation must sleep.
0246  * Only the alter operations must be moved, the const operations can stay.
0247  */
0248 static void merge_queues(struct sem_array *sma)
0249 {
0250     int i;
0251     for (i = 0; i < sma->sem_nsems; i++) {
0252         struct sem *sem = sma->sem_base + i;
0253 
0254         list_splice_init(&sem->pending_alter, &sma->pending_alter);
0255     }
0256 }
0257 
0258 static void sem_rcu_free(struct rcu_head *head)
0259 {
0260     struct ipc_rcu *p = container_of(head, struct ipc_rcu, rcu);
0261     struct sem_array *sma = ipc_rcu_to_struct(p);
0262 
0263     security_sem_free(sma);
0264     ipc_rcu_free(head);
0265 }
0266 
0267 /*
0268  * Enter the mode suitable for non-simple operations:
0269  * Caller must own sem_perm.lock.
0270  */
0271 static void complexmode_enter(struct sem_array *sma)
0272 {
0273     int i;
0274     struct sem *sem;
0275 
0276     if (sma->complex_mode)  {
0277         /* We are already in complex_mode. Nothing to do */
0278         return;
0279     }
0280 
0281     /* We need a full barrier after seting complex_mode:
0282      * The write to complex_mode must be visible
0283      * before we read the first sem->lock spinlock state.
0284      */
0285     smp_store_mb(sma->complex_mode, true);
0286 
0287     for (i = 0; i < sma->sem_nsems; i++) {
0288         sem = sma->sem_base + i;
0289         spin_unlock_wait(&sem->lock);
0290     }
0291     /*
0292      * spin_unlock_wait() is not a memory barriers, it is only a
0293      * control barrier. The code must pair with spin_unlock(&sem->lock),
0294      * thus just the control barrier is insufficient.
0295      *
0296      * smp_rmb() is sufficient, as writes cannot pass the control barrier.
0297      */
0298     smp_rmb();
0299 }
0300 
0301 /*
0302  * Try to leave the mode that disallows simple operations:
0303  * Caller must own sem_perm.lock.
0304  */
0305 static void complexmode_tryleave(struct sem_array *sma)
0306 {
0307     if (sma->complex_count)  {
0308         /* Complex ops are sleeping.
0309          * We must stay in complex mode
0310          */
0311         return;
0312     }
0313     /*
0314      * Immediately after setting complex_mode to false,
0315      * a simple op can start. Thus: all memory writes
0316      * performed by the current operation must be visible
0317      * before we set complex_mode to false.
0318      */
0319     smp_store_release(&sma->complex_mode, false);
0320 }
0321 
0322 #define SEM_GLOBAL_LOCK (-1)
0323 /*
0324  * If the request contains only one semaphore operation, and there are
0325  * no complex transactions pending, lock only the semaphore involved.
0326  * Otherwise, lock the entire semaphore array, since we either have
0327  * multiple semaphores in our own semops, or we need to look at
0328  * semaphores from other pending complex operations.
0329  */
0330 static inline int sem_lock(struct sem_array *sma, struct sembuf *sops,
0331                   int nsops)
0332 {
0333     struct sem *sem;
0334 
0335     if (nsops != 1) {
0336         /* Complex operation - acquire a full lock */
0337         ipc_lock_object(&sma->sem_perm);
0338 
0339         /* Prevent parallel simple ops */
0340         complexmode_enter(sma);
0341         return SEM_GLOBAL_LOCK;
0342     }
0343 
0344     /*
0345      * Only one semaphore affected - try to optimize locking.
0346      * Optimized locking is possible if no complex operation
0347      * is either enqueued or processed right now.
0348      *
0349      * Both facts are tracked by complex_mode.
0350      */
0351     sem = sma->sem_base + sops->sem_num;
0352 
0353     /*
0354      * Initial check for complex_mode. Just an optimization,
0355      * no locking, no memory barrier.
0356      */
0357     if (!sma->complex_mode) {
0358         /*
0359          * It appears that no complex operation is around.
0360          * Acquire the per-semaphore lock.
0361          */
0362         spin_lock(&sem->lock);
0363 
0364         /*
0365          * See 51d7d5205d33
0366          * ("powerpc: Add smp_mb() to arch_spin_is_locked()"):
0367          * A full barrier is required: the write of sem->lock
0368          * must be visible before the read is executed
0369          */
0370         smp_mb();
0371 
0372         if (!smp_load_acquire(&sma->complex_mode)) {
0373             /* fast path successful! */
0374             return sops->sem_num;
0375         }
0376         spin_unlock(&sem->lock);
0377     }
0378 
0379     /* slow path: acquire the full lock */
0380     ipc_lock_object(&sma->sem_perm);
0381 
0382     if (sma->complex_count == 0) {
0383         /* False alarm:
0384          * There is no complex operation, thus we can switch
0385          * back to the fast path.
0386          */
0387         spin_lock(&sem->lock);
0388         ipc_unlock_object(&sma->sem_perm);
0389         return sops->sem_num;
0390     } else {
0391         /* Not a false alarm, thus complete the sequence for a
0392          * full lock.
0393          */
0394         complexmode_enter(sma);
0395         return SEM_GLOBAL_LOCK;
0396     }
0397 }
0398 
0399 static inline void sem_unlock(struct sem_array *sma, int locknum)
0400 {
0401     if (locknum == SEM_GLOBAL_LOCK) {
0402         unmerge_queues(sma);
0403         complexmode_tryleave(sma);
0404         ipc_unlock_object(&sma->sem_perm);
0405     } else {
0406         struct sem *sem = sma->sem_base + locknum;
0407         spin_unlock(&sem->lock);
0408     }
0409 }
0410 
0411 /*
0412  * sem_lock_(check_) routines are called in the paths where the rwsem
0413  * is not held.
0414  *
0415  * The caller holds the RCU read lock.
0416  */
0417 static inline struct sem_array *sem_obtain_object(struct ipc_namespace *ns, int id)
0418 {
0419     struct kern_ipc_perm *ipcp = ipc_obtain_object_idr(&sem_ids(ns), id);
0420 
0421     if (IS_ERR(ipcp))
0422         return ERR_CAST(ipcp);
0423 
0424     return container_of(ipcp, struct sem_array, sem_perm);
0425 }
0426 
0427 static inline struct sem_array *sem_obtain_object_check(struct ipc_namespace *ns,
0428                             int id)
0429 {
0430     struct kern_ipc_perm *ipcp = ipc_obtain_object_check(&sem_ids(ns), id);
0431 
0432     if (IS_ERR(ipcp))
0433         return ERR_CAST(ipcp);
0434 
0435     return container_of(ipcp, struct sem_array, sem_perm);
0436 }
0437 
0438 static inline void sem_lock_and_putref(struct sem_array *sma)
0439 {
0440     sem_lock(sma, NULL, -1);
0441     ipc_rcu_putref(sma, sem_rcu_free);
0442 }
0443 
0444 static inline void sem_rmid(struct ipc_namespace *ns, struct sem_array *s)
0445 {
0446     ipc_rmid(&sem_ids(ns), &s->sem_perm);
0447 }
0448 
0449 /**
0450  * newary - Create a new semaphore set
0451  * @ns: namespace
0452  * @params: ptr to the structure that contains key, semflg and nsems
0453  *
0454  * Called with sem_ids.rwsem held (as a writer)
0455  */
0456 static int newary(struct ipc_namespace *ns, struct ipc_params *params)
0457 {
0458     int id;
0459     int retval;
0460     struct sem_array *sma;
0461     int size;
0462     key_t key = params->key;
0463     int nsems = params->u.nsems;
0464     int semflg = params->flg;
0465     int i;
0466 
0467     if (!nsems)
0468         return -EINVAL;
0469     if (ns->used_sems + nsems > ns->sc_semmns)
0470         return -ENOSPC;
0471 
0472     size = sizeof(*sma) + nsems * sizeof(struct sem);
0473     sma = ipc_rcu_alloc(size);
0474     if (!sma)
0475         return -ENOMEM;
0476 
0477     memset(sma, 0, size);
0478 
0479     sma->sem_perm.mode = (semflg & S_IRWXUGO);
0480     sma->sem_perm.key = key;
0481 
0482     sma->sem_perm.security = NULL;
0483     retval = security_sem_alloc(sma);
0484     if (retval) {
0485         ipc_rcu_putref(sma, ipc_rcu_free);
0486         return retval;
0487     }
0488 
0489     sma->sem_base = (struct sem *) &sma[1];
0490 
0491     for (i = 0; i < nsems; i++) {
0492         INIT_LIST_HEAD(&sma->sem_base[i].pending_alter);
0493         INIT_LIST_HEAD(&sma->sem_base[i].pending_const);
0494         spin_lock_init(&sma->sem_base[i].lock);
0495     }
0496 
0497     sma->complex_count = 0;
0498     sma->complex_mode = true; /* dropped by sem_unlock below */
0499     INIT_LIST_HEAD(&sma->pending_alter);
0500     INIT_LIST_HEAD(&sma->pending_const);
0501     INIT_LIST_HEAD(&sma->list_id);
0502     sma->sem_nsems = nsems;
0503     sma->sem_ctime = get_seconds();
0504 
0505     id = ipc_addid(&sem_ids(ns), &sma->sem_perm, ns->sc_semmni);
0506     if (id < 0) {
0507         ipc_rcu_putref(sma, sem_rcu_free);
0508         return id;
0509     }
0510     ns->used_sems += nsems;
0511 
0512     sem_unlock(sma, -1);
0513     rcu_read_unlock();
0514 
0515     return sma->sem_perm.id;
0516 }
0517 
0518 
0519 /*
0520  * Called with sem_ids.rwsem and ipcp locked.
0521  */
0522 static inline int sem_security(struct kern_ipc_perm *ipcp, int semflg)
0523 {
0524     struct sem_array *sma;
0525 
0526     sma = container_of(ipcp, struct sem_array, sem_perm);
0527     return security_sem_associate(sma, semflg);
0528 }
0529 
0530 /*
0531  * Called with sem_ids.rwsem and ipcp locked.
0532  */
0533 static inline int sem_more_checks(struct kern_ipc_perm *ipcp,
0534                 struct ipc_params *params)
0535 {
0536     struct sem_array *sma;
0537 
0538     sma = container_of(ipcp, struct sem_array, sem_perm);
0539     if (params->u.nsems > sma->sem_nsems)
0540         return -EINVAL;
0541 
0542     return 0;
0543 }
0544 
0545 SYSCALL_DEFINE3(semget, key_t, key, int, nsems, int, semflg)
0546 {
0547     struct ipc_namespace *ns;
0548     static const struct ipc_ops sem_ops = {
0549         .getnew = newary,
0550         .associate = sem_security,
0551         .more_checks = sem_more_checks,
0552     };
0553     struct ipc_params sem_params;
0554 
0555     ns = current->nsproxy->ipc_ns;
0556 
0557     if (nsems < 0 || nsems > ns->sc_semmsl)
0558         return -EINVAL;
0559 
0560     sem_params.key = key;
0561     sem_params.flg = semflg;
0562     sem_params.u.nsems = nsems;
0563 
0564     return ipcget(ns, &sem_ids(ns), &sem_ops, &sem_params);
0565 }
0566 
0567 /**
0568  * perform_atomic_semop[_slow] - Attempt to perform semaphore
0569  *                               operations on a given array.
0570  * @sma: semaphore array
0571  * @q: struct sem_queue that describes the operation
0572  *
0573  * Caller blocking are as follows, based the value
0574  * indicated by the semaphore operation (sem_op):
0575  *
0576  *  (1) >0 never blocks.
0577  *  (2)  0 (wait-for-zero operation): semval is non-zero.
0578  *  (3) <0 attempting to decrement semval to a value smaller than zero.
0579  *
0580  * Returns 0 if the operation was possible.
0581  * Returns 1 if the operation is impossible, the caller must sleep.
0582  * Returns <0 for error codes.
0583  */
0584 static int perform_atomic_semop_slow(struct sem_array *sma, struct sem_queue *q)
0585 {
0586     int result, sem_op, nsops, pid;
0587     struct sembuf *sop;
0588     struct sem *curr;
0589     struct sembuf *sops;
0590     struct sem_undo *un;
0591 
0592     sops = q->sops;
0593     nsops = q->nsops;
0594     un = q->undo;
0595 
0596     for (sop = sops; sop < sops + nsops; sop++) {
0597         curr = sma->sem_base + sop->sem_num;
0598         sem_op = sop->sem_op;
0599         result = curr->semval;
0600 
0601         if (!sem_op && result)
0602             goto would_block;
0603 
0604         result += sem_op;
0605         if (result < 0)
0606             goto would_block;
0607         if (result > SEMVMX)
0608             goto out_of_range;
0609 
0610         if (sop->sem_flg & SEM_UNDO) {
0611             int undo = un->semadj[sop->sem_num] - sem_op;
0612             /* Exceeding the undo range is an error. */
0613             if (undo < (-SEMAEM - 1) || undo > SEMAEM)
0614                 goto out_of_range;
0615             un->semadj[sop->sem_num] = undo;
0616         }
0617 
0618         curr->semval = result;
0619     }
0620 
0621     sop--;
0622     pid = q->pid;
0623     while (sop >= sops) {
0624         sma->sem_base[sop->sem_num].sempid = pid;
0625         sop--;
0626     }
0627 
0628     return 0;
0629 
0630 out_of_range:
0631     result = -ERANGE;
0632     goto undo;
0633 
0634 would_block:
0635     q->blocking = sop;
0636 
0637     if (sop->sem_flg & IPC_NOWAIT)
0638         result = -EAGAIN;
0639     else
0640         result = 1;
0641 
0642 undo:
0643     sop--;
0644     while (sop >= sops) {
0645         sem_op = sop->sem_op;
0646         sma->sem_base[sop->sem_num].semval -= sem_op;
0647         if (sop->sem_flg & SEM_UNDO)
0648             un->semadj[sop->sem_num] += sem_op;
0649         sop--;
0650     }
0651 
0652     return result;
0653 }
0654 
0655 static int perform_atomic_semop(struct sem_array *sma, struct sem_queue *q)
0656 {
0657     int result, sem_op, nsops;
0658     struct sembuf *sop;
0659     struct sem *curr;
0660     struct sembuf *sops;
0661     struct sem_undo *un;
0662 
0663     sops = q->sops;
0664     nsops = q->nsops;
0665     un = q->undo;
0666 
0667     if (unlikely(q->dupsop))
0668         return perform_atomic_semop_slow(sma, q);
0669 
0670     /*
0671      * We scan the semaphore set twice, first to ensure that the entire
0672      * operation can succeed, therefore avoiding any pointless writes
0673      * to shared memory and having to undo such changes in order to block
0674      * until the operations can go through.
0675      */
0676     for (sop = sops; sop < sops + nsops; sop++) {
0677         curr = sma->sem_base + sop->sem_num;
0678         sem_op = sop->sem_op;
0679         result = curr->semval;
0680 
0681         if (!sem_op && result)
0682             goto would_block; /* wait-for-zero */
0683 
0684         result += sem_op;
0685         if (result < 0)
0686             goto would_block;
0687 
0688         if (result > SEMVMX)
0689             return -ERANGE;
0690 
0691         if (sop->sem_flg & SEM_UNDO) {
0692             int undo = un->semadj[sop->sem_num] - sem_op;
0693 
0694             /* Exceeding the undo range is an error. */
0695             if (undo < (-SEMAEM - 1) || undo > SEMAEM)
0696                 return -ERANGE;
0697         }
0698     }
0699 
0700     for (sop = sops; sop < sops + nsops; sop++) {
0701         curr = sma->sem_base + sop->sem_num;
0702         sem_op = sop->sem_op;
0703         result = curr->semval;
0704 
0705         if (sop->sem_flg & SEM_UNDO) {
0706             int undo = un->semadj[sop->sem_num] - sem_op;
0707 
0708             un->semadj[sop->sem_num] = undo;
0709         }
0710         curr->semval += sem_op;
0711         curr->sempid = q->pid;
0712     }
0713 
0714     return 0;
0715 
0716 would_block:
0717     q->blocking = sop;
0718     return sop->sem_flg & IPC_NOWAIT ? -EAGAIN : 1;
0719 }
0720 
0721 static inline void wake_up_sem_queue_prepare(struct sem_queue *q, int error,
0722                          struct wake_q_head *wake_q)
0723 {
0724     wake_q_add(wake_q, q->sleeper);
0725     /*
0726      * Rely on the above implicit barrier, such that we can
0727      * ensure that we hold reference to the task before setting
0728      * q->status. Otherwise we could race with do_exit if the
0729      * task is awoken by an external event before calling
0730      * wake_up_process().
0731      */
0732     WRITE_ONCE(q->status, error);
0733 }
0734 
0735 static void unlink_queue(struct sem_array *sma, struct sem_queue *q)
0736 {
0737     list_del(&q->list);
0738     if (q->nsops > 1)
0739         sma->complex_count--;
0740 }
0741 
0742 /** check_restart(sma, q)
0743  * @sma: semaphore array
0744  * @q: the operation that just completed
0745  *
0746  * update_queue is O(N^2) when it restarts scanning the whole queue of
0747  * waiting operations. Therefore this function checks if the restart is
0748  * really necessary. It is called after a previously waiting operation
0749  * modified the array.
0750  * Note that wait-for-zero operations are handled without restart.
0751  */
0752 static inline int check_restart(struct sem_array *sma, struct sem_queue *q)
0753 {
0754     /* pending complex alter operations are too difficult to analyse */
0755     if (!list_empty(&sma->pending_alter))
0756         return 1;
0757 
0758     /* we were a sleeping complex operation. Too difficult */
0759     if (q->nsops > 1)
0760         return 1;
0761 
0762     /* It is impossible that someone waits for the new value:
0763      * - complex operations always restart.
0764      * - wait-for-zero are handled seperately.
0765      * - q is a previously sleeping simple operation that
0766      *   altered the array. It must be a decrement, because
0767      *   simple increments never sleep.
0768      * - If there are older (higher priority) decrements
0769      *   in the queue, then they have observed the original
0770      *   semval value and couldn't proceed. The operation
0771      *   decremented to value - thus they won't proceed either.
0772      */
0773     return 0;
0774 }
0775 
0776 /**
0777  * wake_const_ops - wake up non-alter tasks
0778  * @sma: semaphore array.
0779  * @semnum: semaphore that was modified.
0780  * @wake_q: lockless wake-queue head.
0781  *
0782  * wake_const_ops must be called after a semaphore in a semaphore array
0783  * was set to 0. If complex const operations are pending, wake_const_ops must
0784  * be called with semnum = -1, as well as with the number of each modified
0785  * semaphore.
0786  * The tasks that must be woken up are added to @wake_q. The return code
0787  * is stored in q->pid.
0788  * The function returns 1 if at least one operation was completed successfully.
0789  */
0790 static int wake_const_ops(struct sem_array *sma, int semnum,
0791               struct wake_q_head *wake_q)
0792 {
0793     struct sem_queue *q, *tmp;
0794     struct list_head *pending_list;
0795     int semop_completed = 0;
0796 
0797     if (semnum == -1)
0798         pending_list = &sma->pending_const;
0799     else
0800         pending_list = &sma->sem_base[semnum].pending_const;
0801 
0802     list_for_each_entry_safe(q, tmp, pending_list, list) {
0803         int error = perform_atomic_semop(sma, q);
0804 
0805         if (error > 0)
0806             continue;
0807         /* operation completed, remove from queue & wakeup */
0808         unlink_queue(sma, q);
0809 
0810         wake_up_sem_queue_prepare(q, error, wake_q);
0811         if (error == 0)
0812             semop_completed = 1;
0813     }
0814 
0815     return semop_completed;
0816 }
0817 
0818 /**
0819  * do_smart_wakeup_zero - wakeup all wait for zero tasks
0820  * @sma: semaphore array
0821  * @sops: operations that were performed
0822  * @nsops: number of operations
0823  * @wake_q: lockless wake-queue head
0824  *
0825  * Checks all required queue for wait-for-zero operations, based
0826  * on the actual changes that were performed on the semaphore array.
0827  * The function returns 1 if at least one operation was completed successfully.
0828  */
0829 static int do_smart_wakeup_zero(struct sem_array *sma, struct sembuf *sops,
0830                 int nsops, struct wake_q_head *wake_q)
0831 {
0832     int i;
0833     int semop_completed = 0;
0834     int got_zero = 0;
0835 
0836     /* first: the per-semaphore queues, if known */
0837     if (sops) {
0838         for (i = 0; i < nsops; i++) {
0839             int num = sops[i].sem_num;
0840 
0841             if (sma->sem_base[num].semval == 0) {
0842                 got_zero = 1;
0843                 semop_completed |= wake_const_ops(sma, num, wake_q);
0844             }
0845         }
0846     } else {
0847         /*
0848          * No sops means modified semaphores not known.
0849          * Assume all were changed.
0850          */
0851         for (i = 0; i < sma->sem_nsems; i++) {
0852             if (sma->sem_base[i].semval == 0) {
0853                 got_zero = 1;
0854                 semop_completed |= wake_const_ops(sma, i, wake_q);
0855             }
0856         }
0857     }
0858     /*
0859      * If one of the modified semaphores got 0,
0860      * then check the global queue, too.
0861      */
0862     if (got_zero)
0863         semop_completed |= wake_const_ops(sma, -1, wake_q);
0864 
0865     return semop_completed;
0866 }
0867 
0868 
0869 /**
0870  * update_queue - look for tasks that can be completed.
0871  * @sma: semaphore array.
0872  * @semnum: semaphore that was modified.
0873  * @wake_q: lockless wake-queue head.
0874  *
0875  * update_queue must be called after a semaphore in a semaphore array
0876  * was modified. If multiple semaphores were modified, update_queue must
0877  * be called with semnum = -1, as well as with the number of each modified
0878  * semaphore.
0879  * The tasks that must be woken up are added to @wake_q. The return code
0880  * is stored in q->pid.
0881  * The function internally checks if const operations can now succeed.
0882  *
0883  * The function return 1 if at least one semop was completed successfully.
0884  */
0885 static int update_queue(struct sem_array *sma, int semnum, struct wake_q_head *wake_q)
0886 {
0887     struct sem_queue *q, *tmp;
0888     struct list_head *pending_list;
0889     int semop_completed = 0;
0890 
0891     if (semnum == -1)
0892         pending_list = &sma->pending_alter;
0893     else
0894         pending_list = &sma->sem_base[semnum].pending_alter;
0895 
0896 again:
0897     list_for_each_entry_safe(q, tmp, pending_list, list) {
0898         int error, restart;
0899 
0900         /* If we are scanning the single sop, per-semaphore list of
0901          * one semaphore and that semaphore is 0, then it is not
0902          * necessary to scan further: simple increments
0903          * that affect only one entry succeed immediately and cannot
0904          * be in the  per semaphore pending queue, and decrements
0905          * cannot be successful if the value is already 0.
0906          */
0907         if (semnum != -1 && sma->sem_base[semnum].semval == 0)
0908             break;
0909 
0910         error = perform_atomic_semop(sma, q);
0911 
0912         /* Does q->sleeper still need to sleep? */
0913         if (error > 0)
0914             continue;
0915 
0916         unlink_queue(sma, q);
0917 
0918         if (error) {
0919             restart = 0;
0920         } else {
0921             semop_completed = 1;
0922             do_smart_wakeup_zero(sma, q->sops, q->nsops, wake_q);
0923             restart = check_restart(sma, q);
0924         }
0925 
0926         wake_up_sem_queue_prepare(q, error, wake_q);
0927         if (restart)
0928             goto again;
0929     }
0930     return semop_completed;
0931 }
0932 
0933 /**
0934  * set_semotime - set sem_otime
0935  * @sma: semaphore array
0936  * @sops: operations that modified the array, may be NULL
0937  *
0938  * sem_otime is replicated to avoid cache line trashing.
0939  * This function sets one instance to the current time.
0940  */
0941 static void set_semotime(struct sem_array *sma, struct sembuf *sops)
0942 {
0943     if (sops == NULL) {
0944         sma->sem_base[0].sem_otime = get_seconds();
0945     } else {
0946         sma->sem_base[sops[0].sem_num].sem_otime =
0947                             get_seconds();
0948     }
0949 }
0950 
0951 /**
0952  * do_smart_update - optimized update_queue
0953  * @sma: semaphore array
0954  * @sops: operations that were performed
0955  * @nsops: number of operations
0956  * @otime: force setting otime
0957  * @wake_q: lockless wake-queue head
0958  *
0959  * do_smart_update() does the required calls to update_queue and wakeup_zero,
0960  * based on the actual changes that were performed on the semaphore array.
0961  * Note that the function does not do the actual wake-up: the caller is
0962  * responsible for calling wake_up_q().
0963  * It is safe to perform this call after dropping all locks.
0964  */
0965 static void do_smart_update(struct sem_array *sma, struct sembuf *sops, int nsops,
0966                 int otime, struct wake_q_head *wake_q)
0967 {
0968     int i;
0969 
0970     otime |= do_smart_wakeup_zero(sma, sops, nsops, wake_q);
0971 
0972     if (!list_empty(&sma->pending_alter)) {
0973         /* semaphore array uses the global queue - just process it. */
0974         otime |= update_queue(sma, -1, wake_q);
0975     } else {
0976         if (!sops) {
0977             /*
0978              * No sops, thus the modified semaphores are not
0979              * known. Check all.
0980              */
0981             for (i = 0; i < sma->sem_nsems; i++)
0982                 otime |= update_queue(sma, i, wake_q);
0983         } else {
0984             /*
0985              * Check the semaphores that were increased:
0986              * - No complex ops, thus all sleeping ops are
0987              *   decrease.
0988              * - if we decreased the value, then any sleeping
0989              *   semaphore ops wont be able to run: If the
0990              *   previous value was too small, then the new
0991              *   value will be too small, too.
0992              */
0993             for (i = 0; i < nsops; i++) {
0994                 if (sops[i].sem_op > 0) {
0995                     otime |= update_queue(sma,
0996                                   sops[i].sem_num, wake_q);
0997                 }
0998             }
0999         }
1000     }
1001     if (otime)
1002         set_semotime(sma, sops);
1003 }
1004 
1005 /*
1006  * check_qop: Test if a queued operation sleeps on the semaphore semnum
1007  */
1008 static int check_qop(struct sem_array *sma, int semnum, struct sem_queue *q,
1009             bool count_zero)
1010 {
1011     struct sembuf *sop = q->blocking;
1012 
1013     /*
1014      * Linux always (since 0.99.10) reported a task as sleeping on all
1015      * semaphores. This violates SUS, therefore it was changed to the
1016      * standard compliant behavior.
1017      * Give the administrators a chance to notice that an application
1018      * might misbehave because it relies on the Linux behavior.
1019      */
1020     pr_info_once("semctl(GETNCNT/GETZCNT) is since 3.16 Single Unix Specification compliant.\n"
1021             "The task %s (%d) triggered the difference, watch for misbehavior.\n",
1022             current->comm, task_pid_nr(current));
1023 
1024     if (sop->sem_num != semnum)
1025         return 0;
1026 
1027     if (count_zero && sop->sem_op == 0)
1028         return 1;
1029     if (!count_zero && sop->sem_op < 0)
1030         return 1;
1031 
1032     return 0;
1033 }
1034 
1035 /* The following counts are associated to each semaphore:
1036  *   semncnt        number of tasks waiting on semval being nonzero
1037  *   semzcnt        number of tasks waiting on semval being zero
1038  *
1039  * Per definition, a task waits only on the semaphore of the first semop
1040  * that cannot proceed, even if additional operation would block, too.
1041  */
1042 static int count_semcnt(struct sem_array *sma, ushort semnum,
1043             bool count_zero)
1044 {
1045     struct list_head *l;
1046     struct sem_queue *q;
1047     int semcnt;
1048 
1049     semcnt = 0;
1050     /* First: check the simple operations. They are easy to evaluate */
1051     if (count_zero)
1052         l = &sma->sem_base[semnum].pending_const;
1053     else
1054         l = &sma->sem_base[semnum].pending_alter;
1055 
1056     list_for_each_entry(q, l, list) {
1057         /* all task on a per-semaphore list sleep on exactly
1058          * that semaphore
1059          */
1060         semcnt++;
1061     }
1062 
1063     /* Then: check the complex operations. */
1064     list_for_each_entry(q, &sma->pending_alter, list) {
1065         semcnt += check_qop(sma, semnum, q, count_zero);
1066     }
1067     if (count_zero) {
1068         list_for_each_entry(q, &sma->pending_const, list) {
1069             semcnt += check_qop(sma, semnum, q, count_zero);
1070         }
1071     }
1072     return semcnt;
1073 }
1074 
1075 /* Free a semaphore set. freeary() is called with sem_ids.rwsem locked
1076  * as a writer and the spinlock for this semaphore set hold. sem_ids.rwsem
1077  * remains locked on exit.
1078  */
1079 static void freeary(struct ipc_namespace *ns, struct kern_ipc_perm *ipcp)
1080 {
1081     struct sem_undo *un, *tu;
1082     struct sem_queue *q, *tq;
1083     struct sem_array *sma = container_of(ipcp, struct sem_array, sem_perm);
1084     int i;
1085     DEFINE_WAKE_Q(wake_q);
1086 
1087     /* Free the existing undo structures for this semaphore set.  */
1088     ipc_assert_locked_object(&sma->sem_perm);
1089     list_for_each_entry_safe(un, tu, &sma->list_id, list_id) {
1090         list_del(&un->list_id);
1091         spin_lock(&un->ulp->lock);
1092         un->semid = -1;
1093         list_del_rcu(&un->list_proc);
1094         spin_unlock(&un->ulp->lock);
1095         kfree_rcu(un, rcu);
1096     }
1097 
1098     /* Wake up all pending processes and let them fail with EIDRM. */
1099     list_for_each_entry_safe(q, tq, &sma->pending_const, list) {
1100         unlink_queue(sma, q);
1101         wake_up_sem_queue_prepare(q, -EIDRM, &wake_q);
1102     }
1103 
1104     list_for_each_entry_safe(q, tq, &sma->pending_alter, list) {
1105         unlink_queue(sma, q);
1106         wake_up_sem_queue_prepare(q, -EIDRM, &wake_q);
1107     }
1108     for (i = 0; i < sma->sem_nsems; i++) {
1109         struct sem *sem = sma->sem_base + i;
1110         list_for_each_entry_safe(q, tq, &sem->pending_const, list) {
1111             unlink_queue(sma, q);
1112             wake_up_sem_queue_prepare(q, -EIDRM, &wake_q);
1113         }
1114         list_for_each_entry_safe(q, tq, &sem->pending_alter, list) {
1115             unlink_queue(sma, q);
1116             wake_up_sem_queue_prepare(q, -EIDRM, &wake_q);
1117         }
1118     }
1119 
1120     /* Remove the semaphore set from the IDR */
1121     sem_rmid(ns, sma);
1122     sem_unlock(sma, -1);
1123     rcu_read_unlock();
1124 
1125     wake_up_q(&wake_q);
1126     ns->used_sems -= sma->sem_nsems;
1127     ipc_rcu_putref(sma, sem_rcu_free);
1128 }
1129 
1130 static unsigned long copy_semid_to_user(void __user *buf, struct semid64_ds *in, int version)
1131 {
1132     switch (version) {
1133     case IPC_64:
1134         return copy_to_user(buf, in, sizeof(*in));
1135     case IPC_OLD:
1136         {
1137         struct semid_ds out;
1138 
1139         memset(&out, 0, sizeof(out));
1140 
1141         ipc64_perm_to_ipc_perm(&in->sem_perm, &out.sem_perm);
1142 
1143         out.sem_otime   = in->sem_otime;
1144         out.sem_ctime   = in->sem_ctime;
1145         out.sem_nsems   = in->sem_nsems;
1146 
1147         return copy_to_user(buf, &out, sizeof(out));
1148         }
1149     default:
1150         return -EINVAL;
1151     }
1152 }
1153 
1154 static time_t get_semotime(struct sem_array *sma)
1155 {
1156     int i;
1157     time_t res;
1158 
1159     res = sma->sem_base[0].sem_otime;
1160     for (i = 1; i < sma->sem_nsems; i++) {
1161         time_t to = sma->sem_base[i].sem_otime;
1162 
1163         if (to > res)
1164             res = to;
1165     }
1166     return res;
1167 }
1168 
1169 static int semctl_nolock(struct ipc_namespace *ns, int semid,
1170              int cmd, int version, void __user *p)
1171 {
1172     int err;
1173     struct sem_array *sma;
1174 
1175     switch (cmd) {
1176     case IPC_INFO:
1177     case SEM_INFO:
1178     {
1179         struct seminfo seminfo;
1180         int max_id;
1181 
1182         err = security_sem_semctl(NULL, cmd);
1183         if (err)
1184             return err;
1185 
1186         memset(&seminfo, 0, sizeof(seminfo));
1187         seminfo.semmni = ns->sc_semmni;
1188         seminfo.semmns = ns->sc_semmns;
1189         seminfo.semmsl = ns->sc_semmsl;
1190         seminfo.semopm = ns->sc_semopm;
1191         seminfo.semvmx = SEMVMX;
1192         seminfo.semmnu = SEMMNU;
1193         seminfo.semmap = SEMMAP;
1194         seminfo.semume = SEMUME;
1195         down_read(&sem_ids(ns).rwsem);
1196         if (cmd == SEM_INFO) {
1197             seminfo.semusz = sem_ids(ns).in_use;
1198             seminfo.semaem = ns->used_sems;
1199         } else {
1200             seminfo.semusz = SEMUSZ;
1201             seminfo.semaem = SEMAEM;
1202         }
1203         max_id = ipc_get_maxid(&sem_ids(ns));
1204         up_read(&sem_ids(ns).rwsem);
1205         if (copy_to_user(p, &seminfo, sizeof(struct seminfo)))
1206             return -EFAULT;
1207         return (max_id < 0) ? 0 : max_id;
1208     }
1209     case IPC_STAT:
1210     case SEM_STAT:
1211     {
1212         struct semid64_ds tbuf;
1213         int id = 0;
1214 
1215         memset(&tbuf, 0, sizeof(tbuf));
1216 
1217         rcu_read_lock();
1218         if (cmd == SEM_STAT) {
1219             sma = sem_obtain_object(ns, semid);
1220             if (IS_ERR(sma)) {
1221                 err = PTR_ERR(sma);
1222                 goto out_unlock;
1223             }
1224             id = sma->sem_perm.id;
1225         } else {
1226             sma = sem_obtain_object_check(ns, semid);
1227             if (IS_ERR(sma)) {
1228                 err = PTR_ERR(sma);
1229                 goto out_unlock;
1230             }
1231         }
1232 
1233         err = -EACCES;
1234         if (ipcperms(ns, &sma->sem_perm, S_IRUGO))
1235             goto out_unlock;
1236 
1237         err = security_sem_semctl(sma, cmd);
1238         if (err)
1239             goto out_unlock;
1240 
1241         kernel_to_ipc64_perm(&sma->sem_perm, &tbuf.sem_perm);
1242         tbuf.sem_otime = get_semotime(sma);
1243         tbuf.sem_ctime = sma->sem_ctime;
1244         tbuf.sem_nsems = sma->sem_nsems;
1245         rcu_read_unlock();
1246         if (copy_semid_to_user(p, &tbuf, version))
1247             return -EFAULT;
1248         return id;
1249     }
1250     default:
1251         return -EINVAL;
1252     }
1253 out_unlock:
1254     rcu_read_unlock();
1255     return err;
1256 }
1257 
1258 static int semctl_setval(struct ipc_namespace *ns, int semid, int semnum,
1259         unsigned long arg)
1260 {
1261     struct sem_undo *un;
1262     struct sem_array *sma;
1263     struct sem *curr;
1264     int err, val;
1265     DEFINE_WAKE_Q(wake_q);
1266 
1267 #if defined(CONFIG_64BIT) && defined(__BIG_ENDIAN)
1268     /* big-endian 64bit */
1269     val = arg >> 32;
1270 #else
1271     /* 32bit or little-endian 64bit */
1272     val = arg;
1273 #endif
1274 
1275     if (val > SEMVMX || val < 0)
1276         return -ERANGE;
1277 
1278     rcu_read_lock();
1279     sma = sem_obtain_object_check(ns, semid);
1280     if (IS_ERR(sma)) {
1281         rcu_read_unlock();
1282         return PTR_ERR(sma);
1283     }
1284 
1285     if (semnum < 0 || semnum >= sma->sem_nsems) {
1286         rcu_read_unlock();
1287         return -EINVAL;
1288     }
1289 
1290 
1291     if (ipcperms(ns, &sma->sem_perm, S_IWUGO)) {
1292         rcu_read_unlock();
1293         return -EACCES;
1294     }
1295 
1296     err = security_sem_semctl(sma, SETVAL);
1297     if (err) {
1298         rcu_read_unlock();
1299         return -EACCES;
1300     }
1301 
1302     sem_lock(sma, NULL, -1);
1303 
1304     if (!ipc_valid_object(&sma->sem_perm)) {
1305         sem_unlock(sma, -1);
1306         rcu_read_unlock();
1307         return -EIDRM;
1308     }
1309 
1310     curr = &sma->sem_base[semnum];
1311 
1312     ipc_assert_locked_object(&sma->sem_perm);
1313     list_for_each_entry(un, &sma->list_id, list_id)
1314         un->semadj[semnum] = 0;
1315 
1316     curr->semval = val;
1317     curr->sempid = task_tgid_vnr(current);
1318     sma->sem_ctime = get_seconds();
1319     /* maybe some queued-up processes were waiting for this */
1320     do_smart_update(sma, NULL, 0, 0, &wake_q);
1321     sem_unlock(sma, -1);
1322     rcu_read_unlock();
1323     wake_up_q(&wake_q);
1324     return 0;
1325 }
1326 
1327 static int semctl_main(struct ipc_namespace *ns, int semid, int semnum,
1328         int cmd, void __user *p)
1329 {
1330     struct sem_array *sma;
1331     struct sem *curr;
1332     int err, nsems;
1333     ushort fast_sem_io[SEMMSL_FAST];
1334     ushort *sem_io = fast_sem_io;
1335     DEFINE_WAKE_Q(wake_q);
1336 
1337     rcu_read_lock();
1338     sma = sem_obtain_object_check(ns, semid);
1339     if (IS_ERR(sma)) {
1340         rcu_read_unlock();
1341         return PTR_ERR(sma);
1342     }
1343 
1344     nsems = sma->sem_nsems;
1345 
1346     err = -EACCES;
1347     if (ipcperms(ns, &sma->sem_perm, cmd == SETALL ? S_IWUGO : S_IRUGO))
1348         goto out_rcu_wakeup;
1349 
1350     err = security_sem_semctl(sma, cmd);
1351     if (err)
1352         goto out_rcu_wakeup;
1353 
1354     err = -EACCES;
1355     switch (cmd) {
1356     case GETALL:
1357     {
1358         ushort __user *array = p;
1359         int i;
1360 
1361         sem_lock(sma, NULL, -1);
1362         if (!ipc_valid_object(&sma->sem_perm)) {
1363             err = -EIDRM;
1364             goto out_unlock;
1365         }
1366         if (nsems > SEMMSL_FAST) {
1367             if (!ipc_rcu_getref(sma)) {
1368                 err = -EIDRM;
1369                 goto out_unlock;
1370             }
1371             sem_unlock(sma, -1);
1372             rcu_read_unlock();
1373             sem_io = ipc_alloc(sizeof(ushort)*nsems);
1374             if (sem_io == NULL) {
1375                 ipc_rcu_putref(sma, sem_rcu_free);
1376                 return -ENOMEM;
1377             }
1378 
1379             rcu_read_lock();
1380             sem_lock_and_putref(sma);
1381             if (!ipc_valid_object(&sma->sem_perm)) {
1382                 err = -EIDRM;
1383                 goto out_unlock;
1384             }
1385         }
1386         for (i = 0; i < sma->sem_nsems; i++)
1387             sem_io[i] = sma->sem_base[i].semval;
1388         sem_unlock(sma, -1);
1389         rcu_read_unlock();
1390         err = 0;
1391         if (copy_to_user(array, sem_io, nsems*sizeof(ushort)))
1392             err = -EFAULT;
1393         goto out_free;
1394     }
1395     case SETALL:
1396     {
1397         int i;
1398         struct sem_undo *un;
1399 
1400         if (!ipc_rcu_getref(sma)) {
1401             err = -EIDRM;
1402             goto out_rcu_wakeup;
1403         }
1404         rcu_read_unlock();
1405 
1406         if (nsems > SEMMSL_FAST) {
1407             sem_io = ipc_alloc(sizeof(ushort)*nsems);
1408             if (sem_io == NULL) {
1409                 ipc_rcu_putref(sma, sem_rcu_free);
1410                 return -ENOMEM;
1411             }
1412         }
1413 
1414         if (copy_from_user(sem_io, p, nsems*sizeof(ushort))) {
1415             ipc_rcu_putref(sma, sem_rcu_free);
1416             err = -EFAULT;
1417             goto out_free;
1418         }
1419 
1420         for (i = 0; i < nsems; i++) {
1421             if (sem_io[i] > SEMVMX) {
1422                 ipc_rcu_putref(sma, sem_rcu_free);
1423                 err = -ERANGE;
1424                 goto out_free;
1425             }
1426         }
1427         rcu_read_lock();
1428         sem_lock_and_putref(sma);
1429         if (!ipc_valid_object(&sma->sem_perm)) {
1430             err = -EIDRM;
1431             goto out_unlock;
1432         }
1433 
1434         for (i = 0; i < nsems; i++) {
1435             sma->sem_base[i].semval = sem_io[i];
1436             sma->sem_base[i].sempid = task_tgid_vnr(current);
1437         }
1438 
1439         ipc_assert_locked_object(&sma->sem_perm);
1440         list_for_each_entry(un, &sma->list_id, list_id) {
1441             for (i = 0; i < nsems; i++)
1442                 un->semadj[i] = 0;
1443         }
1444         sma->sem_ctime = get_seconds();
1445         /* maybe some queued-up processes were waiting for this */
1446         do_smart_update(sma, NULL, 0, 0, &wake_q);
1447         err = 0;
1448         goto out_unlock;
1449     }
1450     /* GETVAL, GETPID, GETNCTN, GETZCNT: fall-through */
1451     }
1452     err = -EINVAL;
1453     if (semnum < 0 || semnum >= nsems)
1454         goto out_rcu_wakeup;
1455 
1456     sem_lock(sma, NULL, -1);
1457     if (!ipc_valid_object(&sma->sem_perm)) {
1458         err = -EIDRM;
1459         goto out_unlock;
1460     }
1461     curr = &sma->sem_base[semnum];
1462 
1463     switch (cmd) {
1464     case GETVAL:
1465         err = curr->semval;
1466         goto out_unlock;
1467     case GETPID:
1468         err = curr->sempid;
1469         goto out_unlock;
1470     case GETNCNT:
1471         err = count_semcnt(sma, semnum, 0);
1472         goto out_unlock;
1473     case GETZCNT:
1474         err = count_semcnt(sma, semnum, 1);
1475         goto out_unlock;
1476     }
1477 
1478 out_unlock:
1479     sem_unlock(sma, -1);
1480 out_rcu_wakeup:
1481     rcu_read_unlock();
1482     wake_up_q(&wake_q);
1483 out_free:
1484     if (sem_io != fast_sem_io)
1485         ipc_free(sem_io);
1486     return err;
1487 }
1488 
1489 static inline unsigned long
1490 copy_semid_from_user(struct semid64_ds *out, void __user *buf, int version)
1491 {
1492     switch (version) {
1493     case IPC_64:
1494         if (copy_from_user(out, buf, sizeof(*out)))
1495             return -EFAULT;
1496         return 0;
1497     case IPC_OLD:
1498         {
1499         struct semid_ds tbuf_old;
1500 
1501         if (copy_from_user(&tbuf_old, buf, sizeof(tbuf_old)))
1502             return -EFAULT;
1503 
1504         out->sem_perm.uid   = tbuf_old.sem_perm.uid;
1505         out->sem_perm.gid   = tbuf_old.sem_perm.gid;
1506         out->sem_perm.mode  = tbuf_old.sem_perm.mode;
1507 
1508         return 0;
1509         }
1510     default:
1511         return -EINVAL;
1512     }
1513 }
1514 
1515 /*
1516  * This function handles some semctl commands which require the rwsem
1517  * to be held in write mode.
1518  * NOTE: no locks must be held, the rwsem is taken inside this function.
1519  */
1520 static int semctl_down(struct ipc_namespace *ns, int semid,
1521                int cmd, int version, void __user *p)
1522 {
1523     struct sem_array *sma;
1524     int err;
1525     struct semid64_ds semid64;
1526     struct kern_ipc_perm *ipcp;
1527 
1528     if (cmd == IPC_SET) {
1529         if (copy_semid_from_user(&semid64, p, version))
1530             return -EFAULT;
1531     }
1532 
1533     down_write(&sem_ids(ns).rwsem);
1534     rcu_read_lock();
1535 
1536     ipcp = ipcctl_pre_down_nolock(ns, &sem_ids(ns), semid, cmd,
1537                       &semid64.sem_perm, 0);
1538     if (IS_ERR(ipcp)) {
1539         err = PTR_ERR(ipcp);
1540         goto out_unlock1;
1541     }
1542 
1543     sma = container_of(ipcp, struct sem_array, sem_perm);
1544 
1545     err = security_sem_semctl(sma, cmd);
1546     if (err)
1547         goto out_unlock1;
1548 
1549     switch (cmd) {
1550     case IPC_RMID:
1551         sem_lock(sma, NULL, -1);
1552         /* freeary unlocks the ipc object and rcu */
1553         freeary(ns, ipcp);
1554         goto out_up;
1555     case IPC_SET:
1556         sem_lock(sma, NULL, -1);
1557         err = ipc_update_perm(&semid64.sem_perm, ipcp);
1558         if (err)
1559             goto out_unlock0;
1560         sma->sem_ctime = get_seconds();
1561         break;
1562     default:
1563         err = -EINVAL;
1564         goto out_unlock1;
1565     }
1566 
1567 out_unlock0:
1568     sem_unlock(sma, -1);
1569 out_unlock1:
1570     rcu_read_unlock();
1571 out_up:
1572     up_write(&sem_ids(ns).rwsem);
1573     return err;
1574 }
1575 
1576 SYSCALL_DEFINE4(semctl, int, semid, int, semnum, int, cmd, unsigned long, arg)
1577 {
1578     int version;
1579     struct ipc_namespace *ns;
1580     void __user *p = (void __user *)arg;
1581 
1582     if (semid < 0)
1583         return -EINVAL;
1584 
1585     version = ipc_parse_version(&cmd);
1586     ns = current->nsproxy->ipc_ns;
1587 
1588     switch (cmd) {
1589     case IPC_INFO:
1590     case SEM_INFO:
1591     case IPC_STAT:
1592     case SEM_STAT:
1593         return semctl_nolock(ns, semid, cmd, version, p);
1594     case GETALL:
1595     case GETVAL:
1596     case GETPID:
1597     case GETNCNT:
1598     case GETZCNT:
1599     case SETALL:
1600         return semctl_main(ns, semid, semnum, cmd, p);
1601     case SETVAL:
1602         return semctl_setval(ns, semid, semnum, arg);
1603     case IPC_RMID:
1604     case IPC_SET:
1605         return semctl_down(ns, semid, cmd, version, p);
1606     default:
1607         return -EINVAL;
1608     }
1609 }
1610 
1611 /* If the task doesn't already have a undo_list, then allocate one
1612  * here.  We guarantee there is only one thread using this undo list,
1613  * and current is THE ONE
1614  *
1615  * If this allocation and assignment succeeds, but later
1616  * portions of this code fail, there is no need to free the sem_undo_list.
1617  * Just let it stay associated with the task, and it'll be freed later
1618  * at exit time.
1619  *
1620  * This can block, so callers must hold no locks.
1621  */
1622 static inline int get_undo_list(struct sem_undo_list **undo_listp)
1623 {
1624     struct sem_undo_list *undo_list;
1625 
1626     undo_list = current->sysvsem.undo_list;
1627     if (!undo_list) {
1628         undo_list = kzalloc(sizeof(*undo_list), GFP_KERNEL);
1629         if (undo_list == NULL)
1630             return -ENOMEM;
1631         spin_lock_init(&undo_list->lock);
1632         atomic_set(&undo_list->refcnt, 1);
1633         INIT_LIST_HEAD(&undo_list->list_proc);
1634 
1635         current->sysvsem.undo_list = undo_list;
1636     }
1637     *undo_listp = undo_list;
1638     return 0;
1639 }
1640 
1641 static struct sem_undo *__lookup_undo(struct sem_undo_list *ulp, int semid)
1642 {
1643     struct sem_undo *un;
1644 
1645     list_for_each_entry_rcu(un, &ulp->list_proc, list_proc) {
1646         if (un->semid == semid)
1647             return un;
1648     }
1649     return NULL;
1650 }
1651 
1652 static struct sem_undo *lookup_undo(struct sem_undo_list *ulp, int semid)
1653 {
1654     struct sem_undo *un;
1655 
1656     assert_spin_locked(&ulp->lock);
1657 
1658     un = __lookup_undo(ulp, semid);
1659     if (un) {
1660         list_del_rcu(&un->list_proc);
1661         list_add_rcu(&un->list_proc, &ulp->list_proc);
1662     }
1663     return un;
1664 }
1665 
1666 /**
1667  * find_alloc_undo - lookup (and if not present create) undo array
1668  * @ns: namespace
1669  * @semid: semaphore array id
1670  *
1671  * The function looks up (and if not present creates) the undo structure.
1672  * The size of the undo structure depends on the size of the semaphore
1673  * array, thus the alloc path is not that straightforward.
1674  * Lifetime-rules: sem_undo is rcu-protected, on success, the function
1675  * performs a rcu_read_lock().
1676  */
1677 static struct sem_undo *find_alloc_undo(struct ipc_namespace *ns, int semid)
1678 {
1679     struct sem_array *sma;
1680     struct sem_undo_list *ulp;
1681     struct sem_undo *un, *new;
1682     int nsems, error;
1683 
1684     error = get_undo_list(&ulp);
1685     if (error)
1686         return ERR_PTR(error);
1687 
1688     rcu_read_lock();
1689     spin_lock(&ulp->lock);
1690     un = lookup_undo(ulp, semid);
1691     spin_unlock(&ulp->lock);
1692     if (likely(un != NULL))
1693         goto out;
1694 
1695     /* no undo structure around - allocate one. */
1696     /* step 1: figure out the size of the semaphore array */
1697     sma = sem_obtain_object_check(ns, semid);
1698     if (IS_ERR(sma)) {
1699         rcu_read_unlock();
1700         return ERR_CAST(sma);
1701     }
1702 
1703     nsems = sma->sem_nsems;
1704     if (!ipc_rcu_getref(sma)) {
1705         rcu_read_unlock();
1706         un = ERR_PTR(-EIDRM);
1707         goto out;
1708     }
1709     rcu_read_unlock();
1710 
1711     /* step 2: allocate new undo structure */
1712     new = kzalloc(sizeof(struct sem_undo) + sizeof(short)*nsems, GFP_KERNEL);
1713     if (!new) {
1714         ipc_rcu_putref(sma, sem_rcu_free);
1715         return ERR_PTR(-ENOMEM);
1716     }
1717 
1718     /* step 3: Acquire the lock on semaphore array */
1719     rcu_read_lock();
1720     sem_lock_and_putref(sma);
1721     if (!ipc_valid_object(&sma->sem_perm)) {
1722         sem_unlock(sma, -1);
1723         rcu_read_unlock();
1724         kfree(new);
1725         un = ERR_PTR(-EIDRM);
1726         goto out;
1727     }
1728     spin_lock(&ulp->lock);
1729 
1730     /*
1731      * step 4: check for races: did someone else allocate the undo struct?
1732      */
1733     un = lookup_undo(ulp, semid);
1734     if (un) {
1735         kfree(new);
1736         goto success;
1737     }
1738     /* step 5: initialize & link new undo structure */
1739     new->semadj = (short *) &new[1];
1740     new->ulp = ulp;
1741     new->semid = semid;
1742     assert_spin_locked(&ulp->lock);
1743     list_add_rcu(&new->list_proc, &ulp->list_proc);
1744     ipc_assert_locked_object(&sma->sem_perm);
1745     list_add(&new->list_id, &sma->list_id);
1746     un = new;
1747 
1748 success:
1749     spin_unlock(&ulp->lock);
1750     sem_unlock(sma, -1);
1751 out:
1752     return un;
1753 }
1754 
1755 SYSCALL_DEFINE4(semtimedop, int, semid, struct sembuf __user *, tsops,
1756         unsigned, nsops, const struct timespec __user *, timeout)
1757 {
1758     int error = -EINVAL;
1759     struct sem_array *sma;
1760     struct sembuf fast_sops[SEMOPM_FAST];
1761     struct sembuf *sops = fast_sops, *sop;
1762     struct sem_undo *un;
1763     int max, locknum;
1764     bool undos = false, alter = false, dupsop = false;
1765     struct sem_queue queue;
1766     unsigned long dup = 0, jiffies_left = 0;
1767     struct ipc_namespace *ns;
1768 
1769     ns = current->nsproxy->ipc_ns;
1770 
1771     if (nsops < 1 || semid < 0)
1772         return -EINVAL;
1773     if (nsops > ns->sc_semopm)
1774         return -E2BIG;
1775     if (nsops > SEMOPM_FAST) {
1776         sops = kmalloc(sizeof(*sops)*nsops, GFP_KERNEL);
1777         if (sops == NULL)
1778             return -ENOMEM;
1779     }
1780 
1781     if (copy_from_user(sops, tsops, nsops * sizeof(*tsops))) {
1782         error =  -EFAULT;
1783         goto out_free;
1784     }
1785 
1786     if (timeout) {
1787         struct timespec _timeout;
1788         if (copy_from_user(&_timeout, timeout, sizeof(*timeout))) {
1789             error = -EFAULT;
1790             goto out_free;
1791         }
1792         if (_timeout.tv_sec < 0 || _timeout.tv_nsec < 0 ||
1793             _timeout.tv_nsec >= 1000000000L) {
1794             error = -EINVAL;
1795             goto out_free;
1796         }
1797         jiffies_left = timespec_to_jiffies(&_timeout);
1798     }
1799 
1800     max = 0;
1801     for (sop = sops; sop < sops + nsops; sop++) {
1802         unsigned long mask = 1ULL << ((sop->sem_num) % BITS_PER_LONG);
1803 
1804         if (sop->sem_num >= max)
1805             max = sop->sem_num;
1806         if (sop->sem_flg & SEM_UNDO)
1807             undos = true;
1808         if (dup & mask) {
1809             /*
1810              * There was a previous alter access that appears
1811              * to have accessed the same semaphore, thus use
1812              * the dupsop logic. "appears", because the detection
1813              * can only check % BITS_PER_LONG.
1814              */
1815             dupsop = true;
1816         }
1817         if (sop->sem_op != 0) {
1818             alter = true;
1819             dup |= mask;
1820         }
1821     }
1822 
1823     if (undos) {
1824         /* On success, find_alloc_undo takes the rcu_read_lock */
1825         un = find_alloc_undo(ns, semid);
1826         if (IS_ERR(un)) {
1827             error = PTR_ERR(un);
1828             goto out_free;
1829         }
1830     } else {
1831         un = NULL;
1832         rcu_read_lock();
1833     }
1834 
1835     sma = sem_obtain_object_check(ns, semid);
1836     if (IS_ERR(sma)) {
1837         rcu_read_unlock();
1838         error = PTR_ERR(sma);
1839         goto out_free;
1840     }
1841 
1842     error = -EFBIG;
1843     if (max >= sma->sem_nsems) {
1844         rcu_read_unlock();
1845         goto out_free;
1846     }
1847 
1848     error = -EACCES;
1849     if (ipcperms(ns, &sma->sem_perm, alter ? S_IWUGO : S_IRUGO)) {
1850         rcu_read_unlock();
1851         goto out_free;
1852     }
1853 
1854     error = security_sem_semop(sma, sops, nsops, alter);
1855     if (error) {
1856         rcu_read_unlock();
1857         goto out_free;
1858     }
1859 
1860     error = -EIDRM;
1861     locknum = sem_lock(sma, sops, nsops);
1862     /*
1863      * We eventually might perform the following check in a lockless
1864      * fashion, considering ipc_valid_object() locking constraints.
1865      * If nsops == 1 and there is no contention for sem_perm.lock, then
1866      * only a per-semaphore lock is held and it's OK to proceed with the
1867      * check below. More details on the fine grained locking scheme
1868      * entangled here and why it's RMID race safe on comments at sem_lock()
1869      */
1870     if (!ipc_valid_object(&sma->sem_perm))
1871         goto out_unlock_free;
1872     /*
1873      * semid identifiers are not unique - find_alloc_undo may have
1874      * allocated an undo structure, it was invalidated by an RMID
1875      * and now a new array with received the same id. Check and fail.
1876      * This case can be detected checking un->semid. The existence of
1877      * "un" itself is guaranteed by rcu.
1878      */
1879     if (un && un->semid == -1)
1880         goto out_unlock_free;
1881 
1882     queue.sops = sops;
1883     queue.nsops = nsops;
1884     queue.undo = un;
1885     queue.pid = task_tgid_vnr(current);
1886     queue.alter = alter;
1887     queue.dupsop = dupsop;
1888 
1889     error = perform_atomic_semop(sma, &queue);
1890     if (error == 0) { /* non-blocking succesfull path */
1891         DEFINE_WAKE_Q(wake_q);
1892 
1893         /*
1894          * If the operation was successful, then do
1895          * the required updates.
1896          */
1897         if (alter)
1898             do_smart_update(sma, sops, nsops, 1, &wake_q);
1899         else
1900             set_semotime(sma, sops);
1901 
1902         sem_unlock(sma, locknum);
1903         rcu_read_unlock();
1904         wake_up_q(&wake_q);
1905 
1906         goto out_free;
1907     }
1908     if (error < 0) /* non-blocking error path */
1909         goto out_unlock_free;
1910 
1911     /*
1912      * We need to sleep on this operation, so we put the current
1913      * task into the pending queue and go to sleep.
1914      */
1915     if (nsops == 1) {
1916         struct sem *curr;
1917         curr = &sma->sem_base[sops->sem_num];
1918 
1919         if (alter) {
1920             if (sma->complex_count) {
1921                 list_add_tail(&queue.list,
1922                         &sma->pending_alter);
1923             } else {
1924 
1925                 list_add_tail(&queue.list,
1926                         &curr->pending_alter);
1927             }
1928         } else {
1929             list_add_tail(&queue.list, &curr->pending_const);
1930         }
1931     } else {
1932         if (!sma->complex_count)
1933             merge_queues(sma);
1934 
1935         if (alter)
1936             list_add_tail(&queue.list, &sma->pending_alter);
1937         else
1938             list_add_tail(&queue.list, &sma->pending_const);
1939 
1940         sma->complex_count++;
1941     }
1942 
1943     do {
1944         queue.status = -EINTR;
1945         queue.sleeper = current;
1946 
1947         __set_current_state(TASK_INTERRUPTIBLE);
1948         sem_unlock(sma, locknum);
1949         rcu_read_unlock();
1950 
1951         if (timeout)
1952             jiffies_left = schedule_timeout(jiffies_left);
1953         else
1954             schedule();
1955 
1956         /*
1957          * fastpath: the semop has completed, either successfully or
1958          * not, from the syscall pov, is quite irrelevant to us at this
1959          * point; we're done.
1960          *
1961          * We _do_ care, nonetheless, about being awoken by a signal or
1962          * spuriously.  The queue.status is checked again in the
1963          * slowpath (aka after taking sem_lock), such that we can detect
1964          * scenarios where we were awakened externally, during the
1965          * window between wake_q_add() and wake_up_q().
1966          */
1967         error = READ_ONCE(queue.status);
1968         if (error != -EINTR) {
1969             /*
1970              * User space could assume that semop() is a memory
1971              * barrier: Without the mb(), the cpu could
1972              * speculatively read in userspace stale data that was
1973              * overwritten by the previous owner of the semaphore.
1974              */
1975             smp_mb();
1976             goto out_free;
1977         }
1978 
1979         rcu_read_lock();
1980         locknum = sem_lock(sma, sops, nsops);
1981 
1982         if (!ipc_valid_object(&sma->sem_perm))
1983             goto out_unlock_free;
1984 
1985         error = READ_ONCE(queue.status);
1986 
1987         /*
1988          * If queue.status != -EINTR we are woken up by another process.
1989          * Leave without unlink_queue(), but with sem_unlock().
1990          */
1991         if (error != -EINTR)
1992             goto out_unlock_free;
1993 
1994         /*
1995          * If an interrupt occurred we have to clean up the queue.
1996          */
1997         if (timeout && jiffies_left == 0)
1998             error = -EAGAIN;
1999     } while (error == -EINTR && !signal_pending(current)); /* spurious */
2000 
2001     unlink_queue(sma, &queue);
2002 
2003 out_unlock_free:
2004     sem_unlock(sma, locknum);
2005     rcu_read_unlock();
2006 out_free:
2007     if (sops != fast_sops)
2008         kfree(sops);
2009     return error;
2010 }
2011 
2012 SYSCALL_DEFINE3(semop, int, semid, struct sembuf __user *, tsops,
2013         unsigned, nsops)
2014 {
2015     return sys_semtimedop(semid, tsops, nsops, NULL);
2016 }
2017 
2018 /* If CLONE_SYSVSEM is set, establish sharing of SEM_UNDO state between
2019  * parent and child tasks.
2020  */
2021 
2022 int copy_semundo(unsigned long clone_flags, struct task_struct *tsk)
2023 {
2024     struct sem_undo_list *undo_list;
2025     int error;
2026 
2027     if (clone_flags & CLONE_SYSVSEM) {
2028         error = get_undo_list(&undo_list);
2029         if (error)
2030             return error;
2031         atomic_inc(&undo_list->refcnt);
2032         tsk->sysvsem.undo_list = undo_list;
2033     } else
2034         tsk->sysvsem.undo_list = NULL;
2035 
2036     return 0;
2037 }
2038 
2039 /*
2040  * add semadj values to semaphores, free undo structures.
2041  * undo structures are not freed when semaphore arrays are destroyed
2042  * so some of them may be out of date.
2043  * IMPLEMENTATION NOTE: There is some confusion over whether the
2044  * set of adjustments that needs to be done should be done in an atomic
2045  * manner or not. That is, if we are attempting to decrement the semval
2046  * should we queue up and wait until we can do so legally?
2047  * The original implementation attempted to do this (queue and wait).
2048  * The current implementation does not do so. The POSIX standard
2049  * and SVID should be consulted to determine what behavior is mandated.
2050  */
2051 void exit_sem(struct task_struct *tsk)
2052 {
2053     struct sem_undo_list *ulp;
2054 
2055     ulp = tsk->sysvsem.undo_list;
2056     if (!ulp)
2057         return;
2058     tsk->sysvsem.undo_list = NULL;
2059 
2060     if (!atomic_dec_and_test(&ulp->refcnt))
2061         return;
2062 
2063     for (;;) {
2064         struct sem_array *sma;
2065         struct sem_undo *un;
2066         int semid, i;
2067         DEFINE_WAKE_Q(wake_q);
2068 
2069         cond_resched();
2070 
2071         rcu_read_lock();
2072         un = list_entry_rcu(ulp->list_proc.next,
2073                     struct sem_undo, list_proc);
2074         if (&un->list_proc == &ulp->list_proc) {
2075             /*
2076              * We must wait for freeary() before freeing this ulp,
2077              * in case we raced with last sem_undo. There is a small
2078              * possibility where we exit while freeary() didn't
2079              * finish unlocking sem_undo_list.
2080              */
2081             spin_unlock_wait(&ulp->lock);
2082             rcu_read_unlock();
2083             break;
2084         }
2085         spin_lock(&ulp->lock);
2086         semid = un->semid;
2087         spin_unlock(&ulp->lock);
2088 
2089         /* exit_sem raced with IPC_RMID, nothing to do */
2090         if (semid == -1) {
2091             rcu_read_unlock();
2092             continue;
2093         }
2094 
2095         sma = sem_obtain_object_check(tsk->nsproxy->ipc_ns, semid);
2096         /* exit_sem raced with IPC_RMID, nothing to do */
2097         if (IS_ERR(sma)) {
2098             rcu_read_unlock();
2099             continue;
2100         }
2101 
2102         sem_lock(sma, NULL, -1);
2103         /* exit_sem raced with IPC_RMID, nothing to do */
2104         if (!ipc_valid_object(&sma->sem_perm)) {
2105             sem_unlock(sma, -1);
2106             rcu_read_unlock();
2107             continue;
2108         }
2109         un = __lookup_undo(ulp, semid);
2110         if (un == NULL) {
2111             /* exit_sem raced with IPC_RMID+semget() that created
2112              * exactly the same semid. Nothing to do.
2113              */
2114             sem_unlock(sma, -1);
2115             rcu_read_unlock();
2116             continue;
2117         }
2118 
2119         /* remove un from the linked lists */
2120         ipc_assert_locked_object(&sma->sem_perm);
2121         list_del(&un->list_id);
2122 
2123         /* we are the last process using this ulp, acquiring ulp->lock
2124          * isn't required. Besides that, we are also protected against
2125          * IPC_RMID as we hold sma->sem_perm lock now
2126          */
2127         list_del_rcu(&un->list_proc);
2128 
2129         /* perform adjustments registered in un */
2130         for (i = 0; i < sma->sem_nsems; i++) {
2131             struct sem *semaphore = &sma->sem_base[i];
2132             if (un->semadj[i]) {
2133                 semaphore->semval += un->semadj[i];
2134                 /*
2135                  * Range checks of the new semaphore value,
2136                  * not defined by sus:
2137                  * - Some unices ignore the undo entirely
2138                  *   (e.g. HP UX 11i 11.22, Tru64 V5.1)
2139                  * - some cap the value (e.g. FreeBSD caps
2140                  *   at 0, but doesn't enforce SEMVMX)
2141                  *
2142                  * Linux caps the semaphore value, both at 0
2143                  * and at SEMVMX.
2144                  *
2145                  *  Manfred <manfred@colorfullife.com>
2146                  */
2147                 if (semaphore->semval < 0)
2148                     semaphore->semval = 0;
2149                 if (semaphore->semval > SEMVMX)
2150                     semaphore->semval = SEMVMX;
2151                 semaphore->sempid = task_tgid_vnr(current);
2152             }
2153         }
2154         /* maybe some queued-up processes were waiting for this */
2155         do_smart_update(sma, NULL, 0, 1, &wake_q);
2156         sem_unlock(sma, -1);
2157         rcu_read_unlock();
2158         wake_up_q(&wake_q);
2159 
2160         kfree_rcu(un, rcu);
2161     }
2162     kfree(ulp);
2163 }
2164 
2165 #ifdef CONFIG_PROC_FS
2166 static int sysvipc_sem_proc_show(struct seq_file *s, void *it)
2167 {
2168     struct user_namespace *user_ns = seq_user_ns(s);
2169     struct sem_array *sma = it;
2170     time_t sem_otime;
2171 
2172     /*
2173      * The proc interface isn't aware of sem_lock(), it calls
2174      * ipc_lock_object() directly (in sysvipc_find_ipc).
2175      * In order to stay compatible with sem_lock(), we must
2176      * enter / leave complex_mode.
2177      */
2178     complexmode_enter(sma);
2179 
2180     sem_otime = get_semotime(sma);
2181 
2182     seq_printf(s,
2183            "%10d %10d  %4o %10u %5u %5u %5u %5u %10lu %10lu\n",
2184            sma->sem_perm.key,
2185            sma->sem_perm.id,
2186            sma->sem_perm.mode,
2187            sma->sem_nsems,
2188            from_kuid_munged(user_ns, sma->sem_perm.uid),
2189            from_kgid_munged(user_ns, sma->sem_perm.gid),
2190            from_kuid_munged(user_ns, sma->sem_perm.cuid),
2191            from_kgid_munged(user_ns, sma->sem_perm.cgid),
2192            sem_otime,
2193            sma->sem_ctime);
2194 
2195     complexmode_tryleave(sma);
2196 
2197     return 0;
2198 }
2199 #endif