the-tree/lib/percpu_counter.c

0001 // SPDX-License-Identifier: GPL-2.0
0002 /*
0003  * Fast batching percpu counters.
0004  */
0005
0006 #include <linux/percpu_counter.h>
0007 #include <linux/mutex.h>
0008 #include <linux/init.h>
0009 #include <linux/cpu.h>
0010 #include <linux/module.h>
0011 #include <linux/debugobjects.h>
0012
0013 #ifdef CONFIG_HOTPLUG_CPU
0014 static LIST_HEAD(percpu_counters);
0015 static DEFINE_SPINLOCK(percpu_counters_lock);
0016 #endif
0017
0018 #ifdef CONFIG_DEBUG_OBJECTS_PERCPU_COUNTER
0019
0020 static const struct debug_obj_descr percpu_counter_debug_descr;
0021
0022 static bool percpu_counter_fixup_free(void *addr, enum debug_obj_state state)
0023 {
0024     struct percpu_counter *fbc = addr;
0025
0026     switch (state) {
0027     case ODEBUG_STATE_ACTIVE:
0028         percpu_counter_destroy(fbc);
0029         debug_object_free(fbc, &percpu_counter_debug_descr);
0030         return true;
0031     default:
0032         return false;
0033     }
0034 }
0035
0036 static const struct debug_obj_descr percpu_counter_debug_descr = {
0037     .name       = "percpu_counter",
0038     .fixup_free = percpu_counter_fixup_free,
0039 };
0040
0041 static inline void debug_percpu_counter_activate(struct percpu_counter *fbc)
0042 {
0043     debug_object_init(fbc, &percpu_counter_debug_descr);
0044     debug_object_activate(fbc, &percpu_counter_debug_descr);
0045 }
0046
0047 static inline void debug_percpu_counter_deactivate(struct percpu_counter *fbc)
0048 {
0049     debug_object_deactivate(fbc, &percpu_counter_debug_descr);
0050     debug_object_free(fbc, &percpu_counter_debug_descr);
0051 }
0052
0053 #else   /* CONFIG_DEBUG_OBJECTS_PERCPU_COUNTER */
0054 static inline void debug_percpu_counter_activate(struct percpu_counter *fbc)
0055 { }
0056 static inline void debug_percpu_counter_deactivate(struct percpu_counter *fbc)
0057 { }
0058 #endif  /* CONFIG_DEBUG_OBJECTS_PERCPU_COUNTER */
0059
0060 void percpu_counter_set(struct percpu_counter *fbc, s64 amount)
0061 {
0062     int cpu;
0063     unsigned long flags;
0064
0065     raw_spin_lock_irqsave(&fbc->lock, flags);
0066     for_each_possible_cpu(cpu) {
0067         s32 *pcount = per_cpu_ptr(fbc->counters, cpu);
0068         *pcount = 0;
0069     }
0070     fbc->count = amount;
0071     raw_spin_unlock_irqrestore(&fbc->lock, flags);
0072 }
0073 EXPORT_SYMBOL(percpu_counter_set);
0074
0075 /*
0076  * This function is both preempt and irq safe. The former is due to explicit
0077  * preemption disable. The latter is guaranteed by the fact that the slow path
0078  * is explicitly protected by an irq-safe spinlock whereas the fast patch uses
0079  * this_cpu_add which is irq-safe by definition. Hence there is no need muck
0080  * with irq state before calling this one
0081  */
0082 void percpu_counter_add_batch(struct percpu_counter *fbc, s64 amount, s32 batch)
0083 {
0084     s64 count;
0085
0086     preempt_disable();
0087     count = __this_cpu_read(*fbc->counters) + amount;
0088     if (abs(count) >= batch) {
0089         unsigned long flags;
0090         raw_spin_lock_irqsave(&fbc->lock, flags);
0091         fbc->count += count;
0092         __this_cpu_sub(*fbc->counters, count - amount);
0093         raw_spin_unlock_irqrestore(&fbc->lock, flags);
0094     } else {
0095         this_cpu_add(*fbc->counters, amount);
0096     }
0097     preempt_enable();
0098 }
0099 EXPORT_SYMBOL(percpu_counter_add_batch);
0100
0101 /*
0102  * For percpu_counter with a big batch, the devication of its count could
0103  * be big, and there is requirement to reduce the deviation, like when the
0104  * counter's batch could be runtime decreased to get a better accuracy,
0105  * which can be achieved by running this sync function on each CPU.
0106  */
0107 void percpu_counter_sync(struct percpu_counter *fbc)
0108 {
0109     unsigned long flags;
0110     s64 count;
0111
0112     raw_spin_lock_irqsave(&fbc->lock, flags);
0113     count = __this_cpu_read(*fbc->counters);
0114     fbc->count += count;
0115     __this_cpu_sub(*fbc->counters, count);
0116     raw_spin_unlock_irqrestore(&fbc->lock, flags);
0117 }
0118 EXPORT_SYMBOL(percpu_counter_sync);
0119
0120 /*
0121  * Add up all the per-cpu counts, return the result.  This is a more accurate
0122  * but much slower version of percpu_counter_read_positive()
0123  */
0124 s64 __percpu_counter_sum(struct percpu_counter *fbc)
0125 {
0126     s64 ret;
0127     int cpu;
0128     unsigned long flags;
0129
0130     raw_spin_lock_irqsave(&fbc->lock, flags);
0131     ret = fbc->count;
0132     for_each_online_cpu(cpu) {
0133         s32 *pcount = per_cpu_ptr(fbc->counters, cpu);
0134         ret += *pcount;
0135     }
0136     raw_spin_unlock_irqrestore(&fbc->lock, flags);
0137     return ret;
0138 }
0139 EXPORT_SYMBOL(__percpu_counter_sum);
0140
0141 int __percpu_counter_init(struct percpu_counter *fbc, s64 amount, gfp_t gfp,
0142               struct lock_class_key *key)
0143 {
0144     unsigned long flags __maybe_unused;
0145
0146     raw_spin_lock_init(&fbc->lock);
0147     lockdep_set_class(&fbc->lock, key);
0148     fbc->count = amount;
0149     fbc->counters = alloc_percpu_gfp(s32, gfp);
0150     if (!fbc->counters)
0151         return -ENOMEM;
0152
0153     debug_percpu_counter_activate(fbc);
0154
0155 #ifdef CONFIG_HOTPLUG_CPU
0156     INIT_LIST_HEAD(&fbc->list);
0157     spin_lock_irqsave(&percpu_counters_lock, flags);
0158     list_add(&fbc->list, &percpu_counters);
0159     spin_unlock_irqrestore(&percpu_counters_lock, flags);
0160 #endif
0161     return 0;
0162 }
0163 EXPORT_SYMBOL(__percpu_counter_init);
0164
0165 void percpu_counter_destroy(struct percpu_counter *fbc)
0166 {
0167     unsigned long flags __maybe_unused;
0168
0169     if (!fbc->counters)
0170         return;
0171
0172     debug_percpu_counter_deactivate(fbc);
0173
0174 #ifdef CONFIG_HOTPLUG_CPU
0175     spin_lock_irqsave(&percpu_counters_lock, flags);
0176     list_del(&fbc->list);
0177     spin_unlock_irqrestore(&percpu_counters_lock, flags);
0178 #endif
0179     free_percpu(fbc->counters);
0180     fbc->counters = NULL;
0181 }
0182 EXPORT_SYMBOL(percpu_counter_destroy);
0183
0184 int percpu_counter_batch __read_mostly = 32;
0185 EXPORT_SYMBOL(percpu_counter_batch);
0186
0187 static int compute_batch_value(unsigned int cpu)
0188 {
0189     int nr = num_online_cpus();
0190
0191     percpu_counter_batch = max(32, nr*2);
0192     return 0;
0193 }
0194
0195 static int percpu_counter_cpu_dead(unsigned int cpu)
0196 {
0197 #ifdef CONFIG_HOTPLUG_CPU
0198     struct percpu_counter *fbc;
0199
0200     compute_batch_value(cpu);
0201
0202     spin_lock_irq(&percpu_counters_lock);
0203     list_for_each_entry(fbc, &percpu_counters, list) {
0204         s32 *pcount;
0205
0206         raw_spin_lock(&fbc->lock);
0207         pcount = per_cpu_ptr(fbc->counters, cpu);
0208         fbc->count += *pcount;
0209         *pcount = 0;
0210         raw_spin_unlock(&fbc->lock);
0211     }
0212     spin_unlock_irq(&percpu_counters_lock);
0213 #endif
0214     return 0;
0215 }
0216
0217 /*
0218  * Compare counter against given value.
0219  * Return 1 if greater, 0 if equal and -1 if less
0220  */
0221 int __percpu_counter_compare(struct percpu_counter *fbc, s64 rhs, s32 batch)
0222 {
0223     s64 count;
0224
0225     count = percpu_counter_read(fbc);
0226     /* Check to see if rough count will be sufficient for comparison */
0227     if (abs(count - rhs) > (batch * num_online_cpus())) {
0228         if (count > rhs)
0229             return 1;
0230         else
0231             return -1;
0232     }
0233     /* Need to use precise count */
0234     count = percpu_counter_sum(fbc);
0235     if (count > rhs)
0236         return 1;
0237     else if (count < rhs)
0238         return -1;
0239     else
0240         return 0;
0241 }
0242 EXPORT_SYMBOL(__percpu_counter_compare);
0243
0244 static int __init percpu_counter_startup(void)
0245 {
0246     int ret;
0247
0248     ret = cpuhp_setup_state(CPUHP_AP_ONLINE_DYN, "lib/percpu_cnt:online",
0249                 compute_batch_value, NULL);
0250     WARN_ON(ret < 0);
0251     ret = cpuhp_setup_state_nocalls(CPUHP_PERCPU_CNT_DEAD,
0252                     "lib/percpu_cnt:dead", NULL,
0253                     percpu_counter_cpu_dead);
0254     WARN_ON(ret < 0);
0255     return 0;
0256 }
0257 module_init(percpu_counter_startup);