the-tree/kernel/async.c

0001 // SPDX-License-Identifier: GPL-2.0-only
0002 /*
0003  * async.c: Asynchronous function calls for boot performance
0004  *
0005  * (C) Copyright 2009 Intel Corporation
0006  * Author: Arjan van de Ven <arjan@linux.intel.com>
0007  */
0008
0009
0010 /*
0011
0012 Goals and Theory of Operation
0013
0014 The primary goal of this feature is to reduce the kernel boot time,
0015 by doing various independent hardware delays and discovery operations
0016 decoupled and not strictly serialized.
0017
0018 More specifically, the asynchronous function call concept allows
0019 certain operations (primarily during system boot) to happen
0020 asynchronously, out of order, while these operations still
0021 have their externally visible parts happen sequentially and in-order.
0022 (not unlike how out-of-order CPUs retire their instructions in order)
0023
0024 Key to the asynchronous function call implementation is the concept of
0025 a "sequence cookie" (which, although it has an abstracted type, can be
0026 thought of as a monotonically incrementing number).
0027
0028 The async core will assign each scheduled event such a sequence cookie and
0029 pass this to the called functions.
0030
0031 The asynchronously called function should before doing a globally visible
0032 operation, such as registering device numbers, call the
0033 async_synchronize_cookie() function and pass in its own cookie. The
0034 async_synchronize_cookie() function will make sure that all asynchronous
0035 operations that were scheduled prior to the operation corresponding with the
0036 cookie have completed.
0037
0038 Subsystem/driver initialization code that scheduled asynchronous probe
0039 functions, but which shares global resources with other drivers/subsystems
0040 that do not use the asynchronous call feature, need to do a full
0041 synchronization with the async_synchronize_full() function, before returning
0042 from their init function. This is to maintain strict ordering between the
0043 asynchronous and synchronous parts of the kernel.
0044
0045 */
0046
0047 #include <linux/async.h>
0048 #include <linux/atomic.h>
0049 #include <linux/ktime.h>
0050 #include <linux/export.h>
0051 #include <linux/wait.h>
0052 #include <linux/sched.h>
0053 #include <linux/slab.h>
0054 #include <linux/workqueue.h>
0055
0056 #include "workqueue_internal.h"
0057
0058 static async_cookie_t next_cookie = 1;
0059
0060 #define MAX_WORK        32768
0061 #define ASYNC_COOKIE_MAX    ULLONG_MAX  /* infinity cookie */
0062
0063 static LIST_HEAD(async_global_pending); /* pending from all registered doms */
0064 static ASYNC_DOMAIN(async_dfl_domain);
0065 static DEFINE_SPINLOCK(async_lock);
0066
0067 struct async_entry {
0068     struct list_head    domain_list;
0069     struct list_head    global_list;
0070     struct work_struct  work;
0071     async_cookie_t      cookie;
0072     async_func_t        func;
0073     void            *data;
0074     struct async_domain *domain;
0075 };
0076
0077 static DECLARE_WAIT_QUEUE_HEAD(async_done);
0078
0079 static atomic_t entry_count;
0080
0081 static long long microseconds_since(ktime_t start)
0082 {
0083     ktime_t now = ktime_get();
0084     return ktime_to_ns(ktime_sub(now, start)) >> 10;
0085 }
0086
0087 static async_cookie_t lowest_in_progress(struct async_domain *domain)
0088 {
0089     struct async_entry *first = NULL;
0090     async_cookie_t ret = ASYNC_COOKIE_MAX;
0091     unsigned long flags;
0092
0093     spin_lock_irqsave(&async_lock, flags);
0094
0095     if (domain) {
0096         if (!list_empty(&domain->pending))
0097             first = list_first_entry(&domain->pending,
0098                     struct async_entry, domain_list);
0099     } else {
0100         if (!list_empty(&async_global_pending))
0101             first = list_first_entry(&async_global_pending,
0102                     struct async_entry, global_list);
0103     }
0104
0105     if (first)
0106         ret = first->cookie;
0107
0108     spin_unlock_irqrestore(&async_lock, flags);
0109     return ret;
0110 }
0111
0112 /*
0113  * pick the first pending entry and run it
0114  */
0115 static void async_run_entry_fn(struct work_struct *work)
0116 {
0117     struct async_entry *entry =
0118         container_of(work, struct async_entry, work);
0119     unsigned long flags;
0120     ktime_t calltime;
0121
0122     /* 1) run (and print duration) */
0123     pr_debug("calling  %lli_%pS @ %i\n", (long long)entry->cookie,
0124          entry->func, task_pid_nr(current));
0125     calltime = ktime_get();
0126
0127     entry->func(entry->data, entry->cookie);
0128
0129     pr_debug("initcall %lli_%pS returned after %lld usecs\n",
0130          (long long)entry->cookie, entry->func,
0131          microseconds_since(calltime));
0132
0133     /* 2) remove self from the pending queues */
0134     spin_lock_irqsave(&async_lock, flags);
0135     list_del_init(&entry->domain_list);
0136     list_del_init(&entry->global_list);
0137
0138     /* 3) free the entry */
0139     kfree(entry);
0140     atomic_dec(&entry_count);
0141
0142     spin_unlock_irqrestore(&async_lock, flags);
0143
0144     /* 4) wake up any waiters */
0145     wake_up(&async_done);
0146 }
0147
0148 /**
0149  * async_schedule_node_domain - NUMA specific version of async_schedule_domain
0150  * @func: function to execute asynchronously
0151  * @data: data pointer to pass to the function
0152  * @node: NUMA node that we want to schedule this on or close to
0153  * @domain: the domain
0154  *
0155  * Returns an async_cookie_t that may be used for checkpointing later.
0156  * @domain may be used in the async_synchronize_*_domain() functions to
0157  * wait within a certain synchronization domain rather than globally.
0158  *
0159  * Note: This function may be called from atomic or non-atomic contexts.
0160  *
0161  * The node requested will be honored on a best effort basis. If the node
0162  * has no CPUs associated with it then the work is distributed among all
0163  * available CPUs.
0164  */
0165 async_cookie_t async_schedule_node_domain(async_func_t func, void *data,
0166                       int node, struct async_domain *domain)
0167 {
0168     struct async_entry *entry;
0169     unsigned long flags;
0170     async_cookie_t newcookie;
0171
0172     /* allow irq-off callers */
0173     entry = kzalloc(sizeof(struct async_entry), GFP_ATOMIC);
0174
0175     /*
0176      * If we're out of memory or if there's too much work
0177      * pending already, we execute synchronously.
0178      */
0179     if (!entry || atomic_read(&entry_count) > MAX_WORK) {
0180         kfree(entry);
0181         spin_lock_irqsave(&async_lock, flags);
0182         newcookie = next_cookie++;
0183         spin_unlock_irqrestore(&async_lock, flags);
0184
0185         /* low on memory.. run synchronously */
0186         func(data, newcookie);
0187         return newcookie;
0188     }
0189     INIT_LIST_HEAD(&entry->domain_list);
0190     INIT_LIST_HEAD(&entry->global_list);
0191     INIT_WORK(&entry->work, async_run_entry_fn);
0192     entry->func = func;
0193     entry->data = data;
0194     entry->domain = domain;
0195
0196     spin_lock_irqsave(&async_lock, flags);
0197
0198     /* allocate cookie and queue */
0199     newcookie = entry->cookie = next_cookie++;
0200
0201     list_add_tail(&entry->domain_list, &domain->pending);
0202     if (domain->registered)
0203         list_add_tail(&entry->global_list, &async_global_pending);
0204
0205     atomic_inc(&entry_count);
0206     spin_unlock_irqrestore(&async_lock, flags);
0207
0208     /* schedule for execution */
0209     queue_work_node(node, system_unbound_wq, &entry->work);
0210
0211     return newcookie;
0212 }
0213 EXPORT_SYMBOL_GPL(async_schedule_node_domain);
0214
0215 /**
0216  * async_schedule_node - NUMA specific version of async_schedule
0217  * @func: function to execute asynchronously
0218  * @data: data pointer to pass to the function
0219  * @node: NUMA node that we want to schedule this on or close to
0220  *
0221  * Returns an async_cookie_t that may be used for checkpointing later.
0222  * Note: This function may be called from atomic or non-atomic contexts.
0223  *
0224  * The node requested will be honored on a best effort basis. If the node
0225  * has no CPUs associated with it then the work is distributed among all
0226  * available CPUs.
0227  */
0228 async_cookie_t async_schedule_node(async_func_t func, void *data, int node)
0229 {
0230     return async_schedule_node_domain(func, data, node, &async_dfl_domain);
0231 }
0232 EXPORT_SYMBOL_GPL(async_schedule_node);
0233
0234 /**
0235  * async_synchronize_full - synchronize all asynchronous function calls
0236  *
0237  * This function waits until all asynchronous function calls have been done.
0238  */
0239 void async_synchronize_full(void)
0240 {
0241     async_synchronize_full_domain(NULL);
0242 }
0243 EXPORT_SYMBOL_GPL(async_synchronize_full);
0244
0245 /**
0246  * async_synchronize_full_domain - synchronize all asynchronous function within a certain domain
0247  * @domain: the domain to synchronize
0248  *
0249  * This function waits until all asynchronous function calls for the
0250  * synchronization domain specified by @domain have been done.
0251  */
0252 void async_synchronize_full_domain(struct async_domain *domain)
0253 {
0254     async_synchronize_cookie_domain(ASYNC_COOKIE_MAX, domain);
0255 }
0256 EXPORT_SYMBOL_GPL(async_synchronize_full_domain);
0257
0258 /**
0259  * async_synchronize_cookie_domain - synchronize asynchronous function calls within a certain domain with cookie checkpointing
0260  * @cookie: async_cookie_t to use as checkpoint
0261  * @domain: the domain to synchronize (%NULL for all registered domains)
0262  *
0263  * This function waits until all asynchronous function calls for the
0264  * synchronization domain specified by @domain submitted prior to @cookie
0265  * have been done.
0266  */
0267 void async_synchronize_cookie_domain(async_cookie_t cookie, struct async_domain *domain)
0268 {
0269     ktime_t starttime;
0270
0271     pr_debug("async_waiting @ %i\n", task_pid_nr(current));
0272     starttime = ktime_get();
0273
0274     wait_event(async_done, lowest_in_progress(domain) >= cookie);
0275
0276     pr_debug("async_continuing @ %i after %lli usec\n", task_pid_nr(current),
0277          microseconds_since(starttime));
0278 }
0279 EXPORT_SYMBOL_GPL(async_synchronize_cookie_domain);
0280
0281 /**
0282  * async_synchronize_cookie - synchronize asynchronous function calls with cookie checkpointing
0283  * @cookie: async_cookie_t to use as checkpoint
0284  *
0285  * This function waits until all asynchronous function calls prior to @cookie
0286  * have been done.
0287  */
0288 void async_synchronize_cookie(async_cookie_t cookie)
0289 {
0290     async_synchronize_cookie_domain(cookie, &async_dfl_domain);
0291 }
0292 EXPORT_SYMBOL_GPL(async_synchronize_cookie);
0293
0294 /**
0295  * current_is_async - is %current an async worker task?
0296  *
0297  * Returns %true if %current is an async worker task.
0298  */
0299 bool current_is_async(void)
0300 {
0301     struct worker *worker = current_wq_worker();
0302
0303     return worker && worker->current_func == async_run_entry_fn;
0304 }
0305 EXPORT_SYMBOL_GPL(current_is_async);