kernel/rcu/rcu_segcblist.h

0001 /* SPDX-License-Identifier: GPL-2.0+ */
0002 /*
0003  * RCU segmented callback lists, internal-to-rcu header file
0004  *
0005  * Copyright IBM Corporation, 2017
0006  *
0007  * Authors: Paul E. McKenney <paulmck@linux.ibm.com>
0008  */
0009
0010 #include <linux/rcu_segcblist.h>
0011
0012 /* Return number of callbacks in the specified callback list. */
0013 static inline long rcu_cblist_n_cbs(struct rcu_cblist *rclp)
0014 {
0015     return READ_ONCE(rclp->len);
0016 }
0017
0018 /* Return number of callbacks in segmented callback list by summing seglen. */
0019 long rcu_segcblist_n_segment_cbs(struct rcu_segcblist *rsclp);
0020
0021 void rcu_cblist_init(struct rcu_cblist *rclp);
0022 void rcu_cblist_enqueue(struct rcu_cblist *rclp, struct rcu_head *rhp);
0023 void rcu_cblist_flush_enqueue(struct rcu_cblist *drclp,
0024                   struct rcu_cblist *srclp,
0025                   struct rcu_head *rhp);
0026 struct rcu_head *rcu_cblist_dequeue(struct rcu_cblist *rclp);
0027
0028 /*
0029  * Is the specified rcu_segcblist structure empty?
0030  *
0031  * But careful!  The fact that the ->head field is NULL does not
0032  * necessarily imply that there are no callbacks associated with
0033  * this structure.  When callbacks are being invoked, they are
0034  * removed as a group.  If callback invocation must be preempted,
0035  * the remaining callbacks will be added back to the list.  Either
0036  * way, the counts are updated later.
0037  *
0038  * So it is often the case that rcu_segcblist_n_cbs() should be used
0039  * instead.
0040  */
0041 static inline bool rcu_segcblist_empty(struct rcu_segcblist *rsclp)
0042 {
0043     return !READ_ONCE(rsclp->head);
0044 }
0045
0046 /* Return number of callbacks in segmented callback list. */
0047 static inline long rcu_segcblist_n_cbs(struct rcu_segcblist *rsclp)
0048 {
0049 #ifdef CONFIG_RCU_NOCB_CPU
0050     return atomic_long_read(&rsclp->len);
0051 #else
0052     return READ_ONCE(rsclp->len);
0053 #endif
0054 }
0055
0056 static inline void rcu_segcblist_set_flags(struct rcu_segcblist *rsclp,
0057                        int flags)
0058 {
0059     WRITE_ONCE(rsclp->flags, rsclp->flags | flags);
0060 }
0061
0062 static inline void rcu_segcblist_clear_flags(struct rcu_segcblist *rsclp,
0063                          int flags)
0064 {
0065     WRITE_ONCE(rsclp->flags, rsclp->flags & ~flags);
0066 }
0067
0068 static inline bool rcu_segcblist_test_flags(struct rcu_segcblist *rsclp,
0069                         int flags)
0070 {
0071     return READ_ONCE(rsclp->flags) & flags;
0072 }
0073
0074 /*
0075  * Is the specified rcu_segcblist enabled, for example, not corresponding
0076  * to an offline CPU?
0077  */
0078 static inline bool rcu_segcblist_is_enabled(struct rcu_segcblist *rsclp)
0079 {
0080     return rcu_segcblist_test_flags(rsclp, SEGCBLIST_ENABLED);
0081 }
0082
0083 /*
0084  * Is the specified rcu_segcblist NOCB offloaded (or in the middle of the
0085  * [de]offloading process)?
0086  */
0087 static inline bool rcu_segcblist_is_offloaded(struct rcu_segcblist *rsclp)
0088 {
0089     if (IS_ENABLED(CONFIG_RCU_NOCB_CPU) &&
0090         rcu_segcblist_test_flags(rsclp, SEGCBLIST_LOCKING))
0091         return true;
0092
0093     return false;
0094 }
0095
0096 static inline bool rcu_segcblist_completely_offloaded(struct rcu_segcblist *rsclp)
0097 {
0098     if (IS_ENABLED(CONFIG_RCU_NOCB_CPU) &&
0099         !rcu_segcblist_test_flags(rsclp, SEGCBLIST_RCU_CORE))
0100         return true;
0101
0102     return false;
0103 }
0104
0105 /*
0106  * Are all segments following the specified segment of the specified
0107  * rcu_segcblist structure empty of callbacks?  (The specified
0108  * segment might well contain callbacks.)
0109  */
0110 static inline bool rcu_segcblist_restempty(struct rcu_segcblist *rsclp, int seg)
0111 {
0112     return !READ_ONCE(*READ_ONCE(rsclp->tails[seg]));
0113 }
0114
0115 /*
0116  * Is the specified segment of the specified rcu_segcblist structure
0117  * empty of callbacks?
0118  */
0119 static inline bool rcu_segcblist_segempty(struct rcu_segcblist *rsclp, int seg)
0120 {
0121     if (seg == RCU_DONE_TAIL)
0122         return &rsclp->head == rsclp->tails[RCU_DONE_TAIL];
0123     return rsclp->tails[seg - 1] == rsclp->tails[seg];
0124 }
0125
0126 void rcu_segcblist_inc_len(struct rcu_segcblist *rsclp);
0127 void rcu_segcblist_add_len(struct rcu_segcblist *rsclp, long v);
0128 void rcu_segcblist_init(struct rcu_segcblist *rsclp);
0129 void rcu_segcblist_disable(struct rcu_segcblist *rsclp);
0130 void rcu_segcblist_offload(struct rcu_segcblist *rsclp, bool offload);
0131 bool rcu_segcblist_ready_cbs(struct rcu_segcblist *rsclp);
0132 bool rcu_segcblist_pend_cbs(struct rcu_segcblist *rsclp);
0133 struct rcu_head *rcu_segcblist_first_cb(struct rcu_segcblist *rsclp);
0134 struct rcu_head *rcu_segcblist_first_pend_cb(struct rcu_segcblist *rsclp);
0135 bool rcu_segcblist_nextgp(struct rcu_segcblist *rsclp, unsigned long *lp);
0136 void rcu_segcblist_enqueue(struct rcu_segcblist *rsclp,
0137                struct rcu_head *rhp);
0138 bool rcu_segcblist_entrain(struct rcu_segcblist *rsclp,
0139                struct rcu_head *rhp);
0140 void rcu_segcblist_extract_done_cbs(struct rcu_segcblist *rsclp,
0141                     struct rcu_cblist *rclp);
0142 void rcu_segcblist_extract_pend_cbs(struct rcu_segcblist *rsclp,
0143                     struct rcu_cblist *rclp);
0144 void rcu_segcblist_insert_count(struct rcu_segcblist *rsclp,
0145                 struct rcu_cblist *rclp);
0146 void rcu_segcblist_insert_done_cbs(struct rcu_segcblist *rsclp,
0147                    struct rcu_cblist *rclp);
0148 void rcu_segcblist_insert_pend_cbs(struct rcu_segcblist *rsclp,
0149                    struct rcu_cblist *rclp);
0150 void rcu_segcblist_advance(struct rcu_segcblist *rsclp, unsigned long seq);
0151 bool rcu_segcblist_accelerate(struct rcu_segcblist *rsclp, unsigned long seq);
0152 void rcu_segcblist_merge(struct rcu_segcblist *dst_rsclp,
0153              struct rcu_segcblist *src_rsclp);