lib/bpf/bpf_helpers.h

0001 /* SPDX-License-Identifier: (LGPL-2.1 OR BSD-2-Clause) */
0002 #ifndef __BPF_HELPERS__
0003 #define __BPF_HELPERS__
0004
0005 /*
0006  * Note that bpf programs need to include either
0007  * vmlinux.h (auto-generated from BTF) or linux/types.h
0008  * in advance since bpf_helper_defs.h uses such types
0009  * as __u64.
0010  */
0011 #include "bpf_helper_defs.h"
0012
0013 #define __uint(name, val) int (*name)[val]
0014 #define __type(name, val) typeof(val) *name
0015 #define __array(name, val) typeof(val) *name[]
0016
0017 /*
0018  * Helper macro to place programs, maps, license in
0019  * different sections in elf_bpf file. Section names
0020  * are interpreted by libbpf depending on the context (BPF programs, BPF maps,
0021  * extern variables, etc).
0022  * To allow use of SEC() with externs (e.g., for extern .maps declarations),
0023  * make sure __attribute__((unused)) doesn't trigger compilation warning.
0024  */
0025 #if __GNUC__ && !__clang__
0026
0027 /*
0028  * Pragma macros are broken on GCC
0029  * https://gcc.gnu.org/bugzilla/show_bug.cgi?id=55578
0030  * https://gcc.gnu.org/bugzilla/show_bug.cgi?id=90400
0031  */
0032 #define SEC(name) __attribute__((section(name), used))
0033
0034 #else
0035
0036 #define SEC(name) \
0037     _Pragma("GCC diagnostic push")                      \
0038     _Pragma("GCC diagnostic ignored \"-Wignored-attributes\"")      \
0039     __attribute__((section(name), used))                    \
0040     _Pragma("GCC diagnostic pop")                       \
0041
0042 #endif
0043
0044 /* Avoid 'linux/stddef.h' definition of '__always_inline'. */
0045 #undef __always_inline
0046 #define __always_inline inline __attribute__((always_inline))
0047
0048 #ifndef __noinline
0049 #define __noinline __attribute__((noinline))
0050 #endif
0051 #ifndef __weak
0052 #define __weak __attribute__((weak))
0053 #endif
0054
0055 /*
0056  * Use __hidden attribute to mark a non-static BPF subprogram effectively
0057  * static for BPF verifier's verification algorithm purposes, allowing more
0058  * extensive and permissive BPF verification process, taking into account
0059  * subprogram's caller context.
0060  */
0061 #define __hidden __attribute__((visibility("hidden")))
0062
0063 /* When utilizing vmlinux.h with BPF CO-RE, user BPF programs can't include
0064  * any system-level headers (such as stddef.h, linux/version.h, etc), and
0065  * commonly-used macros like NULL and KERNEL_VERSION aren't available through
0066  * vmlinux.h. This just adds unnecessary hurdles and forces users to re-define
0067  * them on their own. So as a convenience, provide such definitions here.
0068  */
0069 #ifndef NULL
0070 #define NULL ((void *)0)
0071 #endif
0072
0073 #ifndef KERNEL_VERSION
0074 #define KERNEL_VERSION(a, b, c) (((a) << 16) + ((b) << 8) + ((c) > 255 ? 255 : (c)))
0075 #endif
0076
0077 /*
0078  * Helper macros to manipulate data structures
0079  */
0080 #ifndef offsetof
0081 #define offsetof(TYPE, MEMBER)  ((unsigned long)&((TYPE *)0)->MEMBER)
0082 #endif
0083 #ifndef container_of
0084 #define container_of(ptr, type, member)             \
0085     ({                          \
0086         void *__mptr = (void *)(ptr);           \
0087         ((type *)(__mptr - offsetof(type, member)));    \
0088     })
0089 #endif
0090
0091 /*
0092  * Compiler (optimization) barrier.
0093  */
0094 #ifndef barrier
0095 #define barrier() asm volatile("" ::: "memory")
0096 #endif
0097
0098 /* Variable-specific compiler (optimization) barrier. It's a no-op which makes
0099  * compiler believe that there is some black box modification of a given
0100  * variable and thus prevents compiler from making extra assumption about its
0101  * value and potential simplifications and optimizations on this variable.
0102  *
0103  * E.g., compiler might often delay or even omit 32-bit to 64-bit casting of
0104  * a variable, making some code patterns unverifiable. Putting barrier_var()
0105  * in place will ensure that cast is performed before the barrier_var()
0106  * invocation, because compiler has to pessimistically assume that embedded
0107  * asm section might perform some extra operations on that variable.
0108  *
0109  * This is a variable-specific variant of more global barrier().
0110  */
0111 #ifndef barrier_var
0112 #define barrier_var(var) asm volatile("" : "=r"(var) : "0"(var))
0113 #endif
0114
0115 /*
0116  * Helper macro to throw a compilation error if __bpf_unreachable() gets
0117  * built into the resulting code. This works given BPF back end does not
0118  * implement __builtin_trap(). This is useful to assert that certain paths
0119  * of the program code are never used and hence eliminated by the compiler.
0120  *
0121  * For example, consider a switch statement that covers known cases used by
0122  * the program. __bpf_unreachable() can then reside in the default case. If
0123  * the program gets extended such that a case is not covered in the switch
0124  * statement, then it will throw a build error due to the default case not
0125  * being compiled out.
0126  */
0127 #ifndef __bpf_unreachable
0128 # define __bpf_unreachable()    __builtin_trap()
0129 #endif
0130
0131 /*
0132  * Helper function to perform a tail call with a constant/immediate map slot.
0133  */
0134 #if __clang_major__ >= 8 && defined(__bpf__)
0135 static __always_inline void
0136 bpf_tail_call_static(void *ctx, const void *map, const __u32 slot)
0137 {
0138     if (!__builtin_constant_p(slot))
0139         __bpf_unreachable();
0140
0141     /*
0142      * Provide a hard guarantee that LLVM won't optimize setting r2 (map
0143      * pointer) and r3 (constant map index) from _different paths_ ending
0144      * up at the _same_ call insn as otherwise we won't be able to use the
0145      * jmpq/nopl retpoline-free patching by the x86-64 JIT in the kernel
0146      * given they mismatch. See also d2e4c1e6c294 ("bpf: Constant map key
0147      * tracking for prog array pokes") for details on verifier tracking.
0148      *
0149      * Note on clobber list: we need to stay in-line with BPF calling
0150      * convention, so even if we don't end up using r0, r4, r5, we need
0151      * to mark them as clobber so that LLVM doesn't end up using them
0152      * before / after the call.
0153      */
0154     asm volatile("r1 = %[ctx]\n\t"
0155              "r2 = %[map]\n\t"
0156              "r3 = %[slot]\n\t"
0157              "call 12"
0158              :: [ctx]"r"(ctx), [map]"r"(map), [slot]"i"(slot)
0159              : "r0", "r1", "r2", "r3", "r4", "r5");
0160 }
0161 #endif
0162
0163 /*
0164  * Helper structure used by eBPF C program
0165  * to describe BPF map attributes to libbpf loader
0166  */
0167 struct bpf_map_def {
0168     unsigned int type;
0169     unsigned int key_size;
0170     unsigned int value_size;
0171     unsigned int max_entries;
0172     unsigned int map_flags;
0173 } __attribute__((deprecated("use BTF-defined maps in .maps section")));
0174
0175 enum libbpf_pin_type {
0176     LIBBPF_PIN_NONE,
0177     /* PIN_BY_NAME: pin maps by name (in /sys/fs/bpf by default) */
0178     LIBBPF_PIN_BY_NAME,
0179 };
0180
0181 enum libbpf_tristate {
0182     TRI_NO = 0,
0183     TRI_YES = 1,
0184     TRI_MODULE = 2,
0185 };
0186
0187 #define __kconfig __attribute__((section(".kconfig")))
0188 #define __ksym __attribute__((section(".ksyms")))
0189 #define __kptr __attribute__((btf_type_tag("kptr")))
0190 #define __kptr_ref __attribute__((btf_type_tag("kptr_ref")))
0191
0192 #ifndef ___bpf_concat
0193 #define ___bpf_concat(a, b) a ## b
0194 #endif
0195 #ifndef ___bpf_apply
0196 #define ___bpf_apply(fn, n) ___bpf_concat(fn, n)
0197 #endif
0198 #ifndef ___bpf_nth
0199 #define ___bpf_nth(_, _1, _2, _3, _4, _5, _6, _7, _8, _9, _a, _b, _c, N, ...) N
0200 #endif
0201 #ifndef ___bpf_narg
0202 #define ___bpf_narg(...) \
0203     ___bpf_nth(_, ##__VA_ARGS__, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0)
0204 #endif
0205
0206 #define ___bpf_fill0(arr, p, x) do {} while (0)
0207 #define ___bpf_fill1(arr, p, x) arr[p] = x
0208 #define ___bpf_fill2(arr, p, x, args...) arr[p] = x; ___bpf_fill1(arr, p + 1, args)
0209 #define ___bpf_fill3(arr, p, x, args...) arr[p] = x; ___bpf_fill2(arr, p + 1, args)
0210 #define ___bpf_fill4(arr, p, x, args...) arr[p] = x; ___bpf_fill3(arr, p + 1, args)
0211 #define ___bpf_fill5(arr, p, x, args...) arr[p] = x; ___bpf_fill4(arr, p + 1, args)
0212 #define ___bpf_fill6(arr, p, x, args...) arr[p] = x; ___bpf_fill5(arr, p + 1, args)
0213 #define ___bpf_fill7(arr, p, x, args...) arr[p] = x; ___bpf_fill6(arr, p + 1, args)
0214 #define ___bpf_fill8(arr, p, x, args...) arr[p] = x; ___bpf_fill7(arr, p + 1, args)
0215 #define ___bpf_fill9(arr, p, x, args...) arr[p] = x; ___bpf_fill8(arr, p + 1, args)
0216 #define ___bpf_fill10(arr, p, x, args...) arr[p] = x; ___bpf_fill9(arr, p + 1, args)
0217 #define ___bpf_fill11(arr, p, x, args...) arr[p] = x; ___bpf_fill10(arr, p + 1, args)
0218 #define ___bpf_fill12(arr, p, x, args...) arr[p] = x; ___bpf_fill11(arr, p + 1, args)
0219 #define ___bpf_fill(arr, args...) \
0220     ___bpf_apply(___bpf_fill, ___bpf_narg(args))(arr, 0, args)
0221
0222 /*
0223  * BPF_SEQ_PRINTF to wrap bpf_seq_printf to-be-printed values
0224  * in a structure.
0225  */
0226 #define BPF_SEQ_PRINTF(seq, fmt, args...)           \
0227 ({                              \
0228     static const char ___fmt[] = fmt;           \
0229     unsigned long long ___param[___bpf_narg(args)];     \
0230                                 \
0231     _Pragma("GCC diagnostic push")              \
0232     _Pragma("GCC diagnostic ignored \"-Wint-conversion\"")  \
0233     ___bpf_fill(___param, args);                \
0234     _Pragma("GCC diagnostic pop")               \
0235                                 \
0236     bpf_seq_printf(seq, ___fmt, sizeof(___fmt),     \
0237                ___param, sizeof(___param));     \
0238 })
0239
0240 /*
0241  * BPF_SNPRINTF wraps the bpf_snprintf helper with variadic arguments instead of
0242  * an array of u64.
0243  */
0244 #define BPF_SNPRINTF(out, out_size, fmt, args...)       \
0245 ({                              \
0246     static const char ___fmt[] = fmt;           \
0247     unsigned long long ___param[___bpf_narg(args)];     \
0248                                 \
0249     _Pragma("GCC diagnostic push")              \
0250     _Pragma("GCC diagnostic ignored \"-Wint-conversion\"")  \
0251     ___bpf_fill(___param, args);                \
0252     _Pragma("GCC diagnostic pop")               \
0253                                 \
0254     bpf_snprintf(out, out_size, ___fmt,         \
0255              ___param, sizeof(___param));       \
0256 })
0257
0258 #ifdef BPF_NO_GLOBAL_DATA
0259 #define BPF_PRINTK_FMT_MOD
0260 #else
0261 #define BPF_PRINTK_FMT_MOD static const
0262 #endif
0263
0264 #define __bpf_printk(fmt, ...)              \
0265 ({                          \
0266     BPF_PRINTK_FMT_MOD char ____fmt[] = fmt;    \
0267     bpf_trace_printk(____fmt, sizeof(____fmt),  \
0268              ##__VA_ARGS__);        \
0269 })
0270
0271 /*
0272  * __bpf_vprintk wraps the bpf_trace_vprintk helper with variadic arguments
0273  * instead of an array of u64.
0274  */
0275 #define __bpf_vprintk(fmt, args...)             \
0276 ({                              \
0277     static const char ___fmt[] = fmt;           \
0278     unsigned long long ___param[___bpf_narg(args)];     \
0279                                 \
0280     _Pragma("GCC diagnostic push")              \
0281     _Pragma("GCC diagnostic ignored \"-Wint-conversion\"")  \
0282     ___bpf_fill(___param, args);                \
0283     _Pragma("GCC diagnostic pop")               \
0284                                 \
0285     bpf_trace_vprintk(___fmt, sizeof(___fmt),       \
0286               ___param, sizeof(___param));      \
0287 })
0288
0289 /* Use __bpf_printk when bpf_printk call has 3 or fewer fmt args
0290  * Otherwise use __bpf_vprintk
0291  */
0292 #define ___bpf_pick_printk(...) \
0293     ___bpf_nth(_, ##__VA_ARGS__, __bpf_vprintk, __bpf_vprintk, __bpf_vprintk,   \
0294            __bpf_vprintk, __bpf_vprintk, __bpf_vprintk, __bpf_vprintk,      \
0295            __bpf_vprintk, __bpf_vprintk, __bpf_printk /*3*/, __bpf_printk /*2*/,\
0296            __bpf_printk /*1*/, __bpf_printk /*0*/)
0297
0298 /* Helper macro to print out debug messages */
0299 #define bpf_printk(fmt, args...) ___bpf_pick_printk(args)(fmt, ##args)
0300
0301 #endif