include/linux/srcutree.h

0001 /* SPDX-License-Identifier: GPL-2.0+ */
0002 /*
0003  * Sleepable Read-Copy Update mechanism for mutual exclusion,
0004  *  tree variant.
0005  *
0006  * Copyright (C) IBM Corporation, 2017
0007  *
0008  * Author: Paul McKenney <paulmck@linux.ibm.com>
0009  */
0010
0011 #ifndef _LINUX_SRCU_TREE_H
0012 #define _LINUX_SRCU_TREE_H
0013
0014 #include <linux/rcu_node_tree.h>
0015 #include <linux/completion.h>
0016
0017 struct srcu_node;
0018 struct srcu_struct;
0019
0020 /*
0021  * Per-CPU structure feeding into leaf srcu_node, similar in function
0022  * to rcu_node.
0023  */
0024 struct srcu_data {
0025     /* Read-side state. */
0026     unsigned long srcu_lock_count[2];   /* Locks per CPU. */
0027     unsigned long srcu_unlock_count[2]; /* Unlocks per CPU. */
0028
0029     /* Update-side state. */
0030     spinlock_t __private lock ____cacheline_internodealigned_in_smp;
0031     struct rcu_segcblist srcu_cblist;   /* List of callbacks.*/
0032     unsigned long srcu_gp_seq_needed;   /* Furthest future GP needed. */
0033     unsigned long srcu_gp_seq_needed_exp;   /* Furthest future exp GP. */
0034     bool srcu_cblist_invoking;      /* Invoking these CBs? */
0035     struct timer_list delay_work;       /* Delay for CB invoking */
0036     struct work_struct work;        /* Context for CB invoking. */
0037     struct rcu_head srcu_barrier_head;  /* For srcu_barrier() use. */
0038     struct srcu_node *mynode;       /* Leaf srcu_node. */
0039     unsigned long grpmask;          /* Mask for leaf srcu_node */
0040                         /*  ->srcu_data_have_cbs[]. */
0041     int cpu;
0042     struct srcu_struct *ssp;
0043 };
0044
0045 /*
0046  * Node in SRCU combining tree, similar in function to rcu_data.
0047  */
0048 struct srcu_node {
0049     spinlock_t __private lock;
0050     unsigned long srcu_have_cbs[4];     /* GP seq for children having CBs, but only */
0051                         /*  if greater than ->srcu_gq_seq. */
0052     unsigned long srcu_data_have_cbs[4];    /* Which srcu_data structs have CBs for given GP? */
0053     unsigned long srcu_gp_seq_needed_exp;   /* Furthest future exp GP. */
0054     struct srcu_node *srcu_parent;      /* Next up in tree. */
0055     int grplo;              /* Least CPU for node. */
0056     int grphi;              /* Biggest CPU for node. */
0057 };
0058
0059 /*
0060  * Per-SRCU-domain structure, similar in function to rcu_state.
0061  */
0062 struct srcu_struct {
0063     struct srcu_node *node;         /* Combining tree. */
0064     struct srcu_node *level[RCU_NUM_LVLS + 1];
0065                         /* First node at each level. */
0066     int srcu_size_state;            /* Small-to-big transition state. */
0067     struct mutex srcu_cb_mutex;     /* Serialize CB preparation. */
0068     spinlock_t __private lock;      /* Protect counters and size state. */
0069     struct mutex srcu_gp_mutex;     /* Serialize GP work. */
0070     unsigned int srcu_idx;          /* Current rdr array element. */
0071     unsigned long srcu_gp_seq;      /* Grace-period seq #. */
0072     unsigned long srcu_gp_seq_needed;   /* Latest gp_seq needed. */
0073     unsigned long srcu_gp_seq_needed_exp;   /* Furthest future exp GP. */
0074     unsigned long srcu_gp_start;        /* Last GP start timestamp (jiffies) */
0075     unsigned long srcu_last_gp_end;     /* Last GP end timestamp (ns) */
0076     unsigned long srcu_size_jiffies;    /* Current contention-measurement interval. */
0077     unsigned long srcu_n_lock_retries;  /* Contention events in current interval. */
0078     unsigned long srcu_n_exp_nodelay;   /* # expedited no-delays in current GP phase. */
0079     struct srcu_data __percpu *sda;     /* Per-CPU srcu_data array. */
0080     bool sda_is_static;         /* May ->sda be passed to free_percpu()? */
0081     unsigned long srcu_barrier_seq;     /* srcu_barrier seq #. */
0082     struct mutex srcu_barrier_mutex;    /* Serialize barrier ops. */
0083     struct completion srcu_barrier_completion;
0084                         /* Awaken barrier rq at end. */
0085     atomic_t srcu_barrier_cpu_cnt;      /* # CPUs not yet posting a */
0086                         /*  callback for the barrier */
0087                         /*  operation. */
0088     unsigned long reschedule_jiffies;
0089     unsigned long reschedule_count;
0090     struct delayed_work work;
0091     struct lockdep_map dep_map;
0092 };
0093
0094 /* Values for size state variable (->srcu_size_state). */
0095 #define SRCU_SIZE_SMALL     0
0096 #define SRCU_SIZE_ALLOC     1
0097 #define SRCU_SIZE_WAIT_BARRIER  2
0098 #define SRCU_SIZE_WAIT_CALL 3
0099 #define SRCU_SIZE_WAIT_CBS1 4
0100 #define SRCU_SIZE_WAIT_CBS2 5
0101 #define SRCU_SIZE_WAIT_CBS3 6
0102 #define SRCU_SIZE_WAIT_CBS4 7
0103 #define SRCU_SIZE_BIG       8
0104
0105 /* Values for state variable (bottom bits of ->srcu_gp_seq). */
0106 #define SRCU_STATE_IDLE     0
0107 #define SRCU_STATE_SCAN1    1
0108 #define SRCU_STATE_SCAN2    2
0109
0110 #define __SRCU_STRUCT_INIT(name, pcpu_name)             \
0111 {                                   \
0112     .sda = &pcpu_name,                      \
0113     .lock = __SPIN_LOCK_UNLOCKED(name.lock),            \
0114     .srcu_gp_seq_needed = -1UL,                 \
0115     .work = __DELAYED_WORK_INITIALIZER(name.work, NULL, 0),     \
0116     __SRCU_DEP_MAP_INIT(name)                   \
0117 }
0118
0119 /*
0120  * Define and initialize a srcu struct at build time.
0121  * Do -not- call init_srcu_struct() nor cleanup_srcu_struct() on it.
0122  *
0123  * Note that although DEFINE_STATIC_SRCU() hides the name from other
0124  * files, the per-CPU variable rules nevertheless require that the
0125  * chosen name be globally unique.  These rules also prohibit use of
0126  * DEFINE_STATIC_SRCU() within a function.  If these rules are too
0127  * restrictive, declare the srcu_struct manually.  For example, in
0128  * each file:
0129  *
0130  *  static struct srcu_struct my_srcu;
0131  *
0132  * Then, before the first use of each my_srcu, manually initialize it:
0133  *
0134  *  init_srcu_struct(&my_srcu);
0135  *
0136  * See include/linux/percpu-defs.h for the rules on per-CPU variables.
0137  */
0138 #ifdef MODULE
0139 # define __DEFINE_SRCU(name, is_static)                 \
0140     is_static struct srcu_struct name;              \
0141     extern struct srcu_struct * const __srcu_struct_##name;     \
0142     struct srcu_struct * const __srcu_struct_##name         \
0143         __section("___srcu_struct_ptrs") = &name
0144 #else
0145 # define __DEFINE_SRCU(name, is_static)                 \
0146     static DEFINE_PER_CPU(struct srcu_data, name##_srcu_data);  \
0147     is_static struct srcu_struct name =             \
0148         __SRCU_STRUCT_INIT(name, name##_srcu_data)
0149 #endif
0150 #define DEFINE_SRCU(name)       __DEFINE_SRCU(name, /* not static */)
0151 #define DEFINE_STATIC_SRCU(name)    __DEFINE_SRCU(name, static)
0152
0153 void synchronize_srcu_expedited(struct srcu_struct *ssp);
0154 void srcu_barrier(struct srcu_struct *ssp);
0155 void srcu_torture_stats_print(struct srcu_struct *ssp, char *tt, char *tf);
0156
0157 #endif