OSCL-LXR |
|
|
|
|||
0001 /* SPDX-License-Identifier: GPL-2.0+ */ 0002 /* 0003 * RCU segmented callback lists 0004 * 0005 * This seemingly RCU-private file must be available to SRCU users 0006 * because the size of the TREE SRCU srcu_struct structure depends 0007 * on these definitions. 0008 * 0009 * Copyright IBM Corporation, 2017 0010 * 0011 * Authors: Paul E. McKenney <paulmck@linux.net.ibm.com> 0012 */ 0013 0014 #ifndef __INCLUDE_LINUX_RCU_SEGCBLIST_H 0015 #define __INCLUDE_LINUX_RCU_SEGCBLIST_H 0016 0017 #include <linux/types.h> 0018 #include <linux/atomic.h> 0019 0020 /* Simple unsegmented callback lists. */ 0021 struct rcu_cblist { 0022 struct rcu_head *head; 0023 struct rcu_head **tail; 0024 long len; 0025 }; 0026 0027 #define RCU_CBLIST_INITIALIZER(n) { .head = NULL, .tail = &n.head } 0028 0029 /* Complicated segmented callback lists. ;-) */ 0030 0031 /* 0032 * Index values for segments in rcu_segcblist structure. 0033 * 0034 * The segments are as follows: 0035 * 0036 * [head, *tails[RCU_DONE_TAIL]): 0037 * Callbacks whose grace period has elapsed, and thus can be invoked. 0038 * [*tails[RCU_DONE_TAIL], *tails[RCU_WAIT_TAIL]): 0039 * Callbacks waiting for the current GP from the current CPU's viewpoint. 0040 * [*tails[RCU_WAIT_TAIL], *tails[RCU_NEXT_READY_TAIL]): 0041 * Callbacks that arrived before the next GP started, again from 0042 * the current CPU's viewpoint. These can be handled by the next GP. 0043 * [*tails[RCU_NEXT_READY_TAIL], *tails[RCU_NEXT_TAIL]): 0044 * Callbacks that might have arrived after the next GP started. 0045 * There is some uncertainty as to when a given GP starts and 0046 * ends, but a CPU knows the exact times if it is the one starting 0047 * or ending the GP. Other CPUs know that the previous GP ends 0048 * before the next one starts. 0049 * 0050 * Note that RCU_WAIT_TAIL cannot be empty unless RCU_NEXT_READY_TAIL is also 0051 * empty. 0052 * 0053 * The ->gp_seq[] array contains the grace-period number at which the 0054 * corresponding segment of callbacks will be ready to invoke. A given 0055 * element of this array is meaningful only when the corresponding segment 0056 * is non-empty, and it is never valid for RCU_DONE_TAIL (whose callbacks 0057 * are already ready to invoke) or for RCU_NEXT_TAIL (whose callbacks have 0058 * not yet been assigned a grace-period number). 0059 */ 0060 #define RCU_DONE_TAIL 0 /* Also RCU_WAIT head. */ 0061 #define RCU_WAIT_TAIL 1 /* Also RCU_NEXT_READY head. */ 0062 #define RCU_NEXT_READY_TAIL 2 /* Also RCU_NEXT head. */ 0063 #define RCU_NEXT_TAIL 3 0064 #define RCU_CBLIST_NSEGS 4 0065 0066 0067 /* 0068 * ==NOCB Offloading state machine== 0069 * 0070 * 0071 * ---------------------------------------------------------------------------- 0072 * | SEGCBLIST_RCU_CORE | 0073 * | | 0074 * | Callbacks processed by rcu_core() from softirqs or local | 0075 * | rcuc kthread, without holding nocb_lock. | 0076 * ---------------------------------------------------------------------------- 0077 * | 0078 * v 0079 * ---------------------------------------------------------------------------- 0080 * | SEGCBLIST_RCU_CORE | SEGCBLIST_LOCKING | SEGCBLIST_OFFLOADED | 0081 * | | 0082 * | Callbacks processed by rcu_core() from softirqs or local | 0083 * | rcuc kthread, while holding nocb_lock. Waking up CB and GP kthreads, | 0084 * | allowing nocb_timer to be armed. | 0085 * ---------------------------------------------------------------------------- 0086 * | 0087 * v 0088 * ----------------------------------- 0089 * | | 0090 * v v 0091 * --------------------------------------- ----------------------------------| 0092 * | SEGCBLIST_RCU_CORE | | | SEGCBLIST_RCU_CORE | | 0093 * | SEGCBLIST_LOCKING | | | SEGCBLIST_LOCKING | | 0094 * | SEGCBLIST_OFFLOADED | | | SEGCBLIST_OFFLOADED | | 0095 * | SEGCBLIST_KTHREAD_CB | | SEGCBLIST_KTHREAD_GP | 0096 * | | | | 0097 * | | | | 0098 * | CB kthread woke up and | | GP kthread woke up and | 0099 * | acknowledged SEGCBLIST_OFFLOADED. | | acknowledged SEGCBLIST_OFFLOADED| 0100 * | Processes callbacks concurrently | | | 0101 * | with rcu_core(), holding | | | 0102 * | nocb_lock. | | | 0103 * --------------------------------------- ----------------------------------- 0104 * | | 0105 * ----------------------------------- 0106 * | 0107 * v 0108 * |--------------------------------------------------------------------------| 0109 * | SEGCBLIST_LOCKING | | 0110 * | SEGCBLIST_OFFLOADED | | 0111 * | SEGCBLIST_KTHREAD_GP | | 0112 * | SEGCBLIST_KTHREAD_CB | 0113 * | | 0114 * | Kthreads handle callbacks holding nocb_lock, local rcu_core() stops | 0115 * | handling callbacks. Enable bypass queueing. | 0116 * ---------------------------------------------------------------------------- 0117 */ 0118 0119 0120 0121 /* 0122 * ==NOCB De-Offloading state machine== 0123 * 0124 * 0125 * |--------------------------------------------------------------------------| 0126 * | SEGCBLIST_LOCKING | | 0127 * | SEGCBLIST_OFFLOADED | | 0128 * | SEGCBLIST_KTHREAD_CB | | 0129 * | SEGCBLIST_KTHREAD_GP | 0130 * | | 0131 * | CB/GP kthreads handle callbacks holding nocb_lock, local rcu_core() | 0132 * | ignores callbacks. Bypass enqueue is enabled. | 0133 * ---------------------------------------------------------------------------- 0134 * | 0135 * v 0136 * |--------------------------------------------------------------------------| 0137 * | SEGCBLIST_RCU_CORE | | 0138 * | SEGCBLIST_LOCKING | | 0139 * | SEGCBLIST_OFFLOADED | | 0140 * | SEGCBLIST_KTHREAD_CB | | 0141 * | SEGCBLIST_KTHREAD_GP | 0142 * | | 0143 * | CB/GP kthreads handle callbacks holding nocb_lock, local rcu_core() | 0144 * | handles callbacks concurrently. Bypass enqueue is enabled. | 0145 * | Invoke RCU core so we make sure not to preempt it in the middle with | 0146 * | leaving some urgent work unattended within a jiffy. | 0147 * ---------------------------------------------------------------------------- 0148 * | 0149 * v 0150 * |--------------------------------------------------------------------------| 0151 * | SEGCBLIST_RCU_CORE | | 0152 * | SEGCBLIST_LOCKING | | 0153 * | SEGCBLIST_KTHREAD_CB | | 0154 * | SEGCBLIST_KTHREAD_GP | 0155 * | | 0156 * | CB/GP kthreads and local rcu_core() handle callbacks concurrently | 0157 * | holding nocb_lock. Wake up CB and GP kthreads if necessary. Disable | 0158 * | bypass enqueue. | 0159 * ---------------------------------------------------------------------------- 0160 * | 0161 * v 0162 * ----------------------------------- 0163 * | | 0164 * v v 0165 * ---------------------------------------------------------------------------| 0166 * | | | 0167 * | SEGCBLIST_RCU_CORE | | SEGCBLIST_RCU_CORE | | 0168 * | SEGCBLIST_LOCKING | | SEGCBLIST_LOCKING | | 0169 * | SEGCBLIST_KTHREAD_CB | SEGCBLIST_KTHREAD_GP | 0170 * | | | 0171 * | GP kthread woke up and | CB kthread woke up and | 0172 * | acknowledged the fact that | acknowledged the fact that | 0173 * | SEGCBLIST_OFFLOADED got cleared. | SEGCBLIST_OFFLOADED got cleared. | 0174 * | | The CB kthread goes to sleep | 0175 * | The callbacks from the target CPU | until it ever gets re-offloaded. | 0176 * | will be ignored from the GP kthread | | 0177 * | loop. | | 0178 * ---------------------------------------------------------------------------- 0179 * | | 0180 * ----------------------------------- 0181 * | 0182 * v 0183 * ---------------------------------------------------------------------------- 0184 * | SEGCBLIST_RCU_CORE | SEGCBLIST_LOCKING | 0185 * | | 0186 * | Callbacks processed by rcu_core() from softirqs or local | 0187 * | rcuc kthread, while holding nocb_lock. Forbid nocb_timer to be armed. | 0188 * | Flush pending nocb_timer. Flush nocb bypass callbacks. | 0189 * ---------------------------------------------------------------------------- 0190 * | 0191 * v 0192 * ---------------------------------------------------------------------------- 0193 * | SEGCBLIST_RCU_CORE | 0194 * | | 0195 * | Callbacks processed by rcu_core() from softirqs or local | 0196 * | rcuc kthread, without holding nocb_lock. | 0197 * ---------------------------------------------------------------------------- 0198 */ 0199 #define SEGCBLIST_ENABLED BIT(0) 0200 #define SEGCBLIST_RCU_CORE BIT(1) 0201 #define SEGCBLIST_LOCKING BIT(2) 0202 #define SEGCBLIST_KTHREAD_CB BIT(3) 0203 #define SEGCBLIST_KTHREAD_GP BIT(4) 0204 #define SEGCBLIST_OFFLOADED BIT(5) 0205 0206 struct rcu_segcblist { 0207 struct rcu_head *head; 0208 struct rcu_head **tails[RCU_CBLIST_NSEGS]; 0209 unsigned long gp_seq[RCU_CBLIST_NSEGS]; 0210 #ifdef CONFIG_RCU_NOCB_CPU 0211 atomic_long_t len; 0212 #else 0213 long len; 0214 #endif 0215 long seglen[RCU_CBLIST_NSEGS]; 0216 u8 flags; 0217 }; 0218 0219 #define RCU_SEGCBLIST_INITIALIZER(n) \ 0220 { \ 0221 .head = NULL, \ 0222 .tails[RCU_DONE_TAIL] = &n.head, \ 0223 .tails[RCU_WAIT_TAIL] = &n.head, \ 0224 .tails[RCU_NEXT_READY_TAIL] = &n.head, \ 0225 .tails[RCU_NEXT_TAIL] = &n.head, \ 0226 } 0227 0228 #endif /* __INCLUDE_LINUX_RCU_SEGCBLIST_H */
| [ Source navigation ] | [ Diff markup ] | [ Identifier search ] | [ general search ] |
|
This page was automatically generated by the 2.1.0 LXR engine. The LXR team |
|