kernel/sched/core_sched.c

0001 // SPDX-License-Identifier: GPL-2.0-only
0002
0003 /*
0004  * A simple wrapper around refcount. An allocated sched_core_cookie's
0005  * address is used to compute the cookie of the task.
0006  */
0007 struct sched_core_cookie {
0008     refcount_t refcnt;
0009 };
0010
0011 static unsigned long sched_core_alloc_cookie(void)
0012 {
0013     struct sched_core_cookie *ck = kmalloc(sizeof(*ck), GFP_KERNEL);
0014     if (!ck)
0015         return 0;
0016
0017     refcount_set(&ck->refcnt, 1);
0018     sched_core_get();
0019
0020     return (unsigned long)ck;
0021 }
0022
0023 static void sched_core_put_cookie(unsigned long cookie)
0024 {
0025     struct sched_core_cookie *ptr = (void *)cookie;
0026
0027     if (ptr && refcount_dec_and_test(&ptr->refcnt)) {
0028         kfree(ptr);
0029         sched_core_put();
0030     }
0031 }
0032
0033 static unsigned long sched_core_get_cookie(unsigned long cookie)
0034 {
0035     struct sched_core_cookie *ptr = (void *)cookie;
0036
0037     if (ptr)
0038         refcount_inc(&ptr->refcnt);
0039
0040     return cookie;
0041 }
0042
0043 /*
0044  * sched_core_update_cookie - replace the cookie on a task
0045  * @p: the task to update
0046  * @cookie: the new cookie
0047  *
0048  * Effectively exchange the task cookie; caller is responsible for lifetimes on
0049  * both ends.
0050  *
0051  * Returns: the old cookie
0052  */
0053 static unsigned long sched_core_update_cookie(struct task_struct *p,
0054                           unsigned long cookie)
0055 {
0056     unsigned long old_cookie;
0057     struct rq_flags rf;
0058     struct rq *rq;
0059
0060     rq = task_rq_lock(p, &rf);
0061
0062     /*
0063      * Since creating a cookie implies sched_core_get(), and we cannot set
0064      * a cookie until after we've created it, similarly, we cannot destroy
0065      * a cookie until after we've removed it, we must have core scheduling
0066      * enabled here.
0067      */
0068     SCHED_WARN_ON((p->core_cookie || cookie) && !sched_core_enabled(rq));
0069
0070     if (sched_core_enqueued(p))
0071         sched_core_dequeue(rq, p, DEQUEUE_SAVE);
0072
0073     old_cookie = p->core_cookie;
0074     p->core_cookie = cookie;
0075
0076     /*
0077      * Consider the cases: !prev_cookie and !cookie.
0078      */
0079     if (cookie && task_on_rq_queued(p))
0080         sched_core_enqueue(rq, p);
0081
0082     /*
0083      * If task is currently running, it may not be compatible anymore after
0084      * the cookie change, so enter the scheduler on its CPU to schedule it
0085      * away.
0086      *
0087      * Note that it is possible that as a result of this cookie change, the
0088      * core has now entered/left forced idle state. Defer accounting to the
0089      * next scheduling edge, rather than always forcing a reschedule here.
0090      */
0091     if (task_running(rq, p))
0092         resched_curr(rq);
0093
0094     task_rq_unlock(rq, p, &rf);
0095
0096     return old_cookie;
0097 }
0098
0099 static unsigned long sched_core_clone_cookie(struct task_struct *p)
0100 {
0101     unsigned long cookie, flags;
0102
0103     raw_spin_lock_irqsave(&p->pi_lock, flags);
0104     cookie = sched_core_get_cookie(p->core_cookie);
0105     raw_spin_unlock_irqrestore(&p->pi_lock, flags);
0106
0107     return cookie;
0108 }
0109
0110 void sched_core_fork(struct task_struct *p)
0111 {
0112     RB_CLEAR_NODE(&p->core_node);
0113     p->core_cookie = sched_core_clone_cookie(current);
0114 }
0115
0116 void sched_core_free(struct task_struct *p)
0117 {
0118     sched_core_put_cookie(p->core_cookie);
0119 }
0120
0121 static void __sched_core_set(struct task_struct *p, unsigned long cookie)
0122 {
0123     cookie = sched_core_get_cookie(cookie);
0124     cookie = sched_core_update_cookie(p, cookie);
0125     sched_core_put_cookie(cookie);
0126 }
0127
0128 /* Called from prctl interface: PR_SCHED_CORE */
0129 int sched_core_share_pid(unsigned int cmd, pid_t pid, enum pid_type type,
0130              unsigned long uaddr)
0131 {
0132     unsigned long cookie = 0, id = 0;
0133     struct task_struct *task, *p;
0134     struct pid *grp;
0135     int err = 0;
0136
0137     if (!static_branch_likely(&sched_smt_present))
0138         return -ENODEV;
0139
0140     BUILD_BUG_ON(PR_SCHED_CORE_SCOPE_THREAD != PIDTYPE_PID);
0141     BUILD_BUG_ON(PR_SCHED_CORE_SCOPE_THREAD_GROUP != PIDTYPE_TGID);
0142     BUILD_BUG_ON(PR_SCHED_CORE_SCOPE_PROCESS_GROUP != PIDTYPE_PGID);
0143
0144     if (type > PIDTYPE_PGID || cmd >= PR_SCHED_CORE_MAX || pid < 0 ||
0145         (cmd != PR_SCHED_CORE_GET && uaddr))
0146         return -EINVAL;
0147
0148     rcu_read_lock();
0149     if (pid == 0) {
0150         task = current;
0151     } else {
0152         task = find_task_by_vpid(pid);
0153         if (!task) {
0154             rcu_read_unlock();
0155             return -ESRCH;
0156         }
0157     }
0158     get_task_struct(task);
0159     rcu_read_unlock();
0160
0161     /*
0162      * Check if this process has the right to modify the specified
0163      * process. Use the regular "ptrace_may_access()" checks.
0164      */
0165     if (!ptrace_may_access(task, PTRACE_MODE_READ_REALCREDS)) {
0166         err = -EPERM;
0167         goto out;
0168     }
0169
0170     switch (cmd) {
0171     case PR_SCHED_CORE_GET:
0172         if (type != PIDTYPE_PID || uaddr & 7) {
0173             err = -EINVAL;
0174             goto out;
0175         }
0176         cookie = sched_core_clone_cookie(task);
0177         if (cookie) {
0178             /* XXX improve ? */
0179             ptr_to_hashval((void *)cookie, &id);
0180         }
0181         err = put_user(id, (u64 __user *)uaddr);
0182         goto out;
0183
0184     case PR_SCHED_CORE_CREATE:
0185         cookie = sched_core_alloc_cookie();
0186         if (!cookie) {
0187             err = -ENOMEM;
0188             goto out;
0189         }
0190         break;
0191
0192     case PR_SCHED_CORE_SHARE_TO:
0193         cookie = sched_core_clone_cookie(current);
0194         break;
0195
0196     case PR_SCHED_CORE_SHARE_FROM:
0197         if (type != PIDTYPE_PID) {
0198             err = -EINVAL;
0199             goto out;
0200         }
0201         cookie = sched_core_clone_cookie(task);
0202         __sched_core_set(current, cookie);
0203         goto out;
0204
0205     default:
0206         err = -EINVAL;
0207         goto out;
0208     };
0209
0210     if (type == PIDTYPE_PID) {
0211         __sched_core_set(task, cookie);
0212         goto out;
0213     }
0214
0215     read_lock(&tasklist_lock);
0216     grp = task_pid_type(task, type);
0217
0218     do_each_pid_thread(grp, type, p) {
0219         if (!ptrace_may_access(p, PTRACE_MODE_READ_REALCREDS)) {
0220             err = -EPERM;
0221             goto out_tasklist;
0222         }
0223     } while_each_pid_thread(grp, type, p);
0224
0225     do_each_pid_thread(grp, type, p) {
0226         __sched_core_set(p, cookie);
0227     } while_each_pid_thread(grp, type, p);
0228 out_tasklist:
0229     read_unlock(&tasklist_lock);
0230
0231 out:
0232     sched_core_put_cookie(cookie);
0233     put_task_struct(task);
0234     return err;
0235 }
0236
0237 #ifdef CONFIG_SCHEDSTATS
0238
0239 /* REQUIRES: rq->core's clock recently updated. */
0240 void __sched_core_account_forceidle(struct rq *rq)
0241 {
0242     const struct cpumask *smt_mask = cpu_smt_mask(cpu_of(rq));
0243     u64 delta, now = rq_clock(rq->core);
0244     struct rq *rq_i;
0245     struct task_struct *p;
0246     int i;
0247
0248     lockdep_assert_rq_held(rq);
0249
0250     WARN_ON_ONCE(!rq->core->core_forceidle_count);
0251
0252     if (rq->core->core_forceidle_start == 0)
0253         return;
0254
0255     delta = now - rq->core->core_forceidle_start;
0256     if (unlikely((s64)delta <= 0))
0257         return;
0258
0259     rq->core->core_forceidle_start = now;
0260
0261     if (WARN_ON_ONCE(!rq->core->core_forceidle_occupation)) {
0262         /* can't be forced idle without a running task */
0263     } else if (rq->core->core_forceidle_count > 1 ||
0264            rq->core->core_forceidle_occupation > 1) {
0265         /*
0266          * For larger SMT configurations, we need to scale the charged
0267          * forced idle amount since there can be more than one forced
0268          * idle sibling and more than one running cookied task.
0269          */
0270         delta *= rq->core->core_forceidle_count;
0271         delta = div_u64(delta, rq->core->core_forceidle_occupation);
0272     }
0273
0274     for_each_cpu(i, smt_mask) {
0275         rq_i = cpu_rq(i);
0276         p = rq_i->core_pick ?: rq_i->curr;
0277
0278         if (p == rq_i->idle)
0279             continue;
0280
0281         /*
0282          * Note: this will account forceidle to the current cpu, even
0283          * if it comes from our SMT sibling.
0284          */
0285         __account_forceidle_time(p, delta);
0286     }
0287 }
0288
0289 void __sched_core_tick(struct rq *rq)
0290 {
0291     if (!rq->core->core_forceidle_count)
0292         return;
0293
0294     if (rq != rq->core)
0295         update_rq_clock(rq->core);
0296
0297     __sched_core_account_forceidle(rq);
0298 }
0299
0300 #endif /* CONFIG_SCHEDSTATS */