include/linux/cookie.h

0001 /* SPDX-License-Identifier: GPL-2.0 */
0002 #ifndef __LINUX_COOKIE_H
0003 #define __LINUX_COOKIE_H
0004
0005 #include <linux/atomic.h>
0006 #include <linux/percpu.h>
0007 #include <asm/local.h>
0008
0009 struct pcpu_gen_cookie {
0010     local_t nesting;
0011     u64 last;
0012 } __aligned(16);
0013
0014 struct gen_cookie {
0015     struct pcpu_gen_cookie __percpu *local;
0016     atomic64_t forward_last ____cacheline_aligned_in_smp;
0017     atomic64_t reverse_last;
0018 };
0019
0020 #define COOKIE_LOCAL_BATCH  4096
0021
0022 #define DEFINE_COOKIE(name)                     \
0023     static DEFINE_PER_CPU(struct pcpu_gen_cookie, __##name);    \
0024     static struct gen_cookie name = {               \
0025         .local      = &__##name,                \
0026         .forward_last   = ATOMIC64_INIT(0),         \
0027         .reverse_last   = ATOMIC64_INIT(0),         \
0028     }
0029
0030 static __always_inline u64 gen_cookie_next(struct gen_cookie *gc)
0031 {
0032     struct pcpu_gen_cookie *local = this_cpu_ptr(gc->local);
0033     u64 val;
0034
0035     if (likely(local_inc_return(&local->nesting) == 1)) {
0036         val = local->last;
0037         if (__is_defined(CONFIG_SMP) &&
0038             unlikely((val & (COOKIE_LOCAL_BATCH - 1)) == 0)) {
0039             s64 next = atomic64_add_return(COOKIE_LOCAL_BATCH,
0040                                &gc->forward_last);
0041             val = next - COOKIE_LOCAL_BATCH;
0042         }
0043         local->last = ++val;
0044     } else {
0045         val = atomic64_dec_return(&gc->reverse_last);
0046     }
0047     local_dec(&local->nesting);
0048     return val;
0049 }
0050
0051 #endif /* __LINUX_COOKIE_H */