lib/bpf/libbpf_internal.h

0001 /* SPDX-License-Identifier: (LGPL-2.1 OR BSD-2-Clause) */
0002
0003 /*
0004  * Internal libbpf helpers.
0005  *
0006  * Copyright (c) 2019 Facebook
0007  */
0008
0009 #ifndef __LIBBPF_LIBBPF_INTERNAL_H
0010 #define __LIBBPF_LIBBPF_INTERNAL_H
0011
0012 #include <stdlib.h>
0013 #include <limits.h>
0014 #include <errno.h>
0015 #include <linux/err.h>
0016 #include <fcntl.h>
0017 #include <unistd.h>
0018 #include "relo_core.h"
0019
0020 /* make sure libbpf doesn't use kernel-only integer typedefs */
0021 #pragma GCC poison u8 u16 u32 u64 s8 s16 s32 s64
0022
0023 /* prevent accidental re-addition of reallocarray() */
0024 #pragma GCC poison reallocarray
0025
0026 #include "libbpf.h"
0027 #include "btf.h"
0028
0029 #ifndef EM_BPF
0030 #define EM_BPF 247
0031 #endif
0032
0033 #ifndef R_BPF_64_64
0034 #define R_BPF_64_64 1
0035 #endif
0036 #ifndef R_BPF_64_ABS64
0037 #define R_BPF_64_ABS64 2
0038 #endif
0039 #ifndef R_BPF_64_ABS32
0040 #define R_BPF_64_ABS32 3
0041 #endif
0042 #ifndef R_BPF_64_32
0043 #define R_BPF_64_32 10
0044 #endif
0045
0046 #ifndef SHT_LLVM_ADDRSIG
0047 #define SHT_LLVM_ADDRSIG 0x6FFF4C03
0048 #endif
0049
0050 /* if libelf is old and doesn't support mmap(), fall back to read() */
0051 #ifndef ELF_C_READ_MMAP
0052 #define ELF_C_READ_MMAP ELF_C_READ
0053 #endif
0054
0055 /* Older libelf all end up in this expression, for both 32 and 64 bit */
0056 #ifndef ELF64_ST_VISIBILITY
0057 #define ELF64_ST_VISIBILITY(o) ((o) & 0x03)
0058 #endif
0059
0060 #define BTF_INFO_ENC(kind, kind_flag, vlen) \
0061     ((!!(kind_flag) << 31) | ((kind) << 24) | ((vlen) & BTF_MAX_VLEN))
0062 #define BTF_TYPE_ENC(name, info, size_or_type) (name), (info), (size_or_type)
0063 #define BTF_INT_ENC(encoding, bits_offset, nr_bits) \
0064     ((encoding) << 24 | (bits_offset) << 16 | (nr_bits))
0065 #define BTF_TYPE_INT_ENC(name, encoding, bits_offset, bits, sz) \
0066     BTF_TYPE_ENC(name, BTF_INFO_ENC(BTF_KIND_INT, 0, 0), sz), \
0067     BTF_INT_ENC(encoding, bits_offset, bits)
0068 #define BTF_MEMBER_ENC(name, type, bits_offset) (name), (type), (bits_offset)
0069 #define BTF_PARAM_ENC(name, type) (name), (type)
0070 #define BTF_VAR_SECINFO_ENC(type, offset, size) (type), (offset), (size)
0071 #define BTF_TYPE_FLOAT_ENC(name, sz) \
0072     BTF_TYPE_ENC(name, BTF_INFO_ENC(BTF_KIND_FLOAT, 0, 0), sz)
0073 #define BTF_TYPE_DECL_TAG_ENC(value, type, component_idx) \
0074     BTF_TYPE_ENC(value, BTF_INFO_ENC(BTF_KIND_DECL_TAG, 0, 0), type), (component_idx)
0075 #define BTF_TYPE_TYPE_TAG_ENC(value, type) \
0076     BTF_TYPE_ENC(value, BTF_INFO_ENC(BTF_KIND_TYPE_TAG, 0, 0), type)
0077
0078 #ifndef likely
0079 #define likely(x) __builtin_expect(!!(x), 1)
0080 #endif
0081 #ifndef unlikely
0082 #define unlikely(x) __builtin_expect(!!(x), 0)
0083 #endif
0084 #ifndef min
0085 # define min(x, y) ((x) < (y) ? (x) : (y))
0086 #endif
0087 #ifndef max
0088 # define max(x, y) ((x) < (y) ? (y) : (x))
0089 #endif
0090 #ifndef offsetofend
0091 # define offsetofend(TYPE, FIELD) \
0092     (offsetof(TYPE, FIELD) + sizeof(((TYPE *)0)->FIELD))
0093 #endif
0094 #ifndef __alias
0095 #define __alias(symbol) __attribute__((alias(#symbol)))
0096 #endif
0097
0098 /* Check whether a string `str` has prefix `pfx`, regardless if `pfx` is
0099  * a string literal known at compilation time or char * pointer known only at
0100  * runtime.
0101  */
0102 #define str_has_pfx(str, pfx) \
0103     (strncmp(str, pfx, __builtin_constant_p(pfx) ? sizeof(pfx) - 1 : strlen(pfx)) == 0)
0104
0105 /* suffix check */
0106 static inline bool str_has_sfx(const char *str, const char *sfx)
0107 {
0108     size_t str_len = strlen(str);
0109     size_t sfx_len = strlen(sfx);
0110
0111     if (sfx_len > str_len)
0112         return false;
0113     return strcmp(str + str_len - sfx_len, sfx) == 0;
0114 }
0115
0116 /* Symbol versioning is different between static and shared library.
0117  * Properly versioned symbols are needed for shared library, but
0118  * only the symbol of the new version is needed for static library.
0119  * Starting with GNU C 10, use symver attribute instead of .symver assembler
0120  * directive, which works better with GCC LTO builds.
0121  */
0122 #if defined(SHARED) && defined(__GNUC__) && __GNUC__ >= 10
0123
0124 #define DEFAULT_VERSION(internal_name, api_name, version) \
0125     __attribute__((symver(#api_name "@@" #version)))
0126 #define COMPAT_VERSION(internal_name, api_name, version) \
0127     __attribute__((symver(#api_name "@" #version)))
0128
0129 #elif defined(SHARED)
0130
0131 #define COMPAT_VERSION(internal_name, api_name, version) \
0132     asm(".symver " #internal_name "," #api_name "@" #version);
0133 #define DEFAULT_VERSION(internal_name, api_name, version) \
0134     asm(".symver " #internal_name "," #api_name "@@" #version);
0135
0136 #else /* !SHARED */
0137
0138 #define COMPAT_VERSION(internal_name, api_name, version)
0139 #define DEFAULT_VERSION(internal_name, api_name, version) \
0140     extern typeof(internal_name) api_name \
0141     __attribute__((alias(#internal_name)));
0142
0143 #endif
0144
0145 extern void libbpf_print(enum libbpf_print_level level,
0146              const char *format, ...)
0147     __attribute__((format(printf, 2, 3)));
0148
0149 #define __pr(level, fmt, ...)   \
0150 do {                \
0151     libbpf_print(level, "libbpf: " fmt, ##__VA_ARGS__); \
0152 } while (0)
0153
0154 #define pr_warn(fmt, ...)   __pr(LIBBPF_WARN, fmt, ##__VA_ARGS__)
0155 #define pr_info(fmt, ...)   __pr(LIBBPF_INFO, fmt, ##__VA_ARGS__)
0156 #define pr_debug(fmt, ...)  __pr(LIBBPF_DEBUG, fmt, ##__VA_ARGS__)
0157
0158 #ifndef __has_builtin
0159 #define __has_builtin(x) 0
0160 #endif
0161
0162 struct bpf_link {
0163     int (*detach)(struct bpf_link *link);
0164     void (*dealloc)(struct bpf_link *link);
0165     char *pin_path;     /* NULL, if not pinned */
0166     int fd;         /* hook FD, -1 if not applicable */
0167     bool disconnected;
0168 };
0169
0170 /*
0171  * Re-implement glibc's reallocarray() for libbpf internal-only use.
0172  * reallocarray(), unfortunately, is not available in all versions of glibc,
0173  * so requires extra feature detection and using reallocarray() stub from
0174  * <tools/libc_compat.h> and COMPAT_NEED_REALLOCARRAY. All this complicates
0175  * build of libbpf unnecessarily and is just a maintenance burden. Instead,
0176  * it's trivial to implement libbpf-specific internal version and use it
0177  * throughout libbpf.
0178  */
0179 static inline void *libbpf_reallocarray(void *ptr, size_t nmemb, size_t size)
0180 {
0181     size_t total;
0182
0183 #if __has_builtin(__builtin_mul_overflow)
0184     if (unlikely(__builtin_mul_overflow(nmemb, size, &total)))
0185         return NULL;
0186 #else
0187     if (size == 0 || nmemb > ULONG_MAX / size)
0188         return NULL;
0189     total = nmemb * size;
0190 #endif
0191     return realloc(ptr, total);
0192 }
0193
0194 /* Copy up to sz - 1 bytes from zero-terminated src string and ensure that dst
0195  * is zero-terminated string no matter what (unless sz == 0, in which case
0196  * it's a no-op). It's conceptually close to FreeBSD's strlcpy(), but differs
0197  * in what is returned. Given this is internal helper, it's trivial to extend
0198  * this, when necessary. Use this instead of strncpy inside libbpf source code.
0199  */
0200 static inline void libbpf_strlcpy(char *dst, const char *src, size_t sz)
0201 {
0202     size_t i;
0203
0204     if (sz == 0)
0205         return;
0206
0207     sz--;
0208     for (i = 0; i < sz && src[i]; i++)
0209         dst[i] = src[i];
0210     dst[i] = '\0';
0211 }
0212
0213 __u32 get_kernel_version(void);
0214
0215 struct btf;
0216 struct btf_type;
0217
0218 struct btf_type *btf_type_by_id(const struct btf *btf, __u32 type_id);
0219 const char *btf_kind_str(const struct btf_type *t);
0220 const struct btf_type *skip_mods_and_typedefs(const struct btf *btf, __u32 id, __u32 *res_id);
0221
0222 static inline enum btf_func_linkage btf_func_linkage(const struct btf_type *t)
0223 {
0224     return (enum btf_func_linkage)(int)btf_vlen(t);
0225 }
0226
0227 static inline __u32 btf_type_info(int kind, int vlen, int kflag)
0228 {
0229     return (kflag << 31) | (kind << 24) | vlen;
0230 }
0231
0232 enum map_def_parts {
0233     MAP_DEF_MAP_TYPE    = 0x001,
0234     MAP_DEF_KEY_TYPE    = 0x002,
0235     MAP_DEF_KEY_SIZE    = 0x004,
0236     MAP_DEF_VALUE_TYPE  = 0x008,
0237     MAP_DEF_VALUE_SIZE  = 0x010,
0238     MAP_DEF_MAX_ENTRIES = 0x020,
0239     MAP_DEF_MAP_FLAGS   = 0x040,
0240     MAP_DEF_NUMA_NODE   = 0x080,
0241     MAP_DEF_PINNING     = 0x100,
0242     MAP_DEF_INNER_MAP   = 0x200,
0243     MAP_DEF_MAP_EXTRA   = 0x400,
0244
0245     MAP_DEF_ALL     = 0x7ff, /* combination of all above */
0246 };
0247
0248 struct btf_map_def {
0249     enum map_def_parts parts;
0250     __u32 map_type;
0251     __u32 key_type_id;
0252     __u32 key_size;
0253     __u32 value_type_id;
0254     __u32 value_size;
0255     __u32 max_entries;
0256     __u32 map_flags;
0257     __u32 numa_node;
0258     __u32 pinning;
0259     __u64 map_extra;
0260 };
0261
0262 int parse_btf_map_def(const char *map_name, struct btf *btf,
0263               const struct btf_type *def_t, bool strict,
0264               struct btf_map_def *map_def, struct btf_map_def *inner_def);
0265
0266 void *libbpf_add_mem(void **data, size_t *cap_cnt, size_t elem_sz,
0267              size_t cur_cnt, size_t max_cnt, size_t add_cnt);
0268 int libbpf_ensure_mem(void **data, size_t *cap_cnt, size_t elem_sz, size_t need_cnt);
0269
0270 static inline bool libbpf_is_mem_zeroed(const char *p, ssize_t len)
0271 {
0272     while (len > 0) {
0273         if (*p)
0274             return false;
0275         p++;
0276         len--;
0277     }
0278     return true;
0279 }
0280
0281 static inline bool libbpf_validate_opts(const char *opts,
0282                     size_t opts_sz, size_t user_sz,
0283                     const char *type_name)
0284 {
0285     if (user_sz < sizeof(size_t)) {
0286         pr_warn("%s size (%zu) is too small\n", type_name, user_sz);
0287         return false;
0288     }
0289     if (!libbpf_is_mem_zeroed(opts + opts_sz, (ssize_t)user_sz - opts_sz)) {
0290         pr_warn("%s has non-zero extra bytes\n", type_name);
0291         return false;
0292     }
0293     return true;
0294 }
0295
0296 #define OPTS_VALID(opts, type)                            \
0297     (!(opts) || libbpf_validate_opts((const char *)opts,              \
0298                      offsetofend(struct type,         \
0299                              type##__last_field),     \
0300                      (opts)->sz, #type))
0301 #define OPTS_HAS(opts, field) \
0302     ((opts) && opts->sz >= offsetofend(typeof(*(opts)), field))
0303 #define OPTS_GET(opts, field, fallback_value) \
0304     (OPTS_HAS(opts, field) ? (opts)->field : fallback_value)
0305 #define OPTS_SET(opts, field, value)        \
0306     do {                    \
0307         if (OPTS_HAS(opts, field))  \
0308             (opts)->field = value;  \
0309     } while (0)
0310
0311 #define OPTS_ZEROED(opts, last_nonzero_field)                     \
0312 ({                                        \
0313     ssize_t __off = offsetofend(typeof(*(opts)), last_nonzero_field);     \
0314     !(opts) || libbpf_is_mem_zeroed((const void *)opts + __off,       \
0315                     (opts)->sz - __off);              \
0316 })
0317
0318 enum kern_feature_id {
0319     /* v4.14: kernel support for program & map names. */
0320     FEAT_PROG_NAME,
0321     /* v5.2: kernel support for global data sections. */
0322     FEAT_GLOBAL_DATA,
0323     /* BTF support */
0324     FEAT_BTF,
0325     /* BTF_KIND_FUNC and BTF_KIND_FUNC_PROTO support */
0326     FEAT_BTF_FUNC,
0327     /* BTF_KIND_VAR and BTF_KIND_DATASEC support */
0328     FEAT_BTF_DATASEC,
0329     /* BTF_FUNC_GLOBAL is supported */
0330     FEAT_BTF_GLOBAL_FUNC,
0331     /* BPF_F_MMAPABLE is supported for arrays */
0332     FEAT_ARRAY_MMAP,
0333     /* kernel support for expected_attach_type in BPF_PROG_LOAD */
0334     FEAT_EXP_ATTACH_TYPE,
0335     /* bpf_probe_read_{kernel,user}[_str] helpers */
0336     FEAT_PROBE_READ_KERN,
0337     /* BPF_PROG_BIND_MAP is supported */
0338     FEAT_PROG_BIND_MAP,
0339     /* Kernel support for module BTFs */
0340     FEAT_MODULE_BTF,
0341     /* BTF_KIND_FLOAT support */
0342     FEAT_BTF_FLOAT,
0343     /* BPF perf link support */
0344     FEAT_PERF_LINK,
0345     /* BTF_KIND_DECL_TAG support */
0346     FEAT_BTF_DECL_TAG,
0347     /* BTF_KIND_TYPE_TAG support */
0348     FEAT_BTF_TYPE_TAG,
0349     /* memcg-based accounting for BPF maps and progs */
0350     FEAT_MEMCG_ACCOUNT,
0351     /* BPF cookie (bpf_get_attach_cookie() BPF helper) support */
0352     FEAT_BPF_COOKIE,
0353     /* BTF_KIND_ENUM64 support and BTF_KIND_ENUM kflag support */
0354     FEAT_BTF_ENUM64,
0355     /* Kernel uses syscall wrapper (CONFIG_ARCH_HAS_SYSCALL_WRAPPER) */
0356     FEAT_SYSCALL_WRAPPER,
0357     __FEAT_CNT,
0358 };
0359
0360 int probe_memcg_account(void);
0361 bool kernel_supports(const struct bpf_object *obj, enum kern_feature_id feat_id);
0362 int bump_rlimit_memlock(void);
0363
0364 int parse_cpu_mask_str(const char *s, bool **mask, int *mask_sz);
0365 int parse_cpu_mask_file(const char *fcpu, bool **mask, int *mask_sz);
0366 int libbpf__load_raw_btf(const char *raw_types, size_t types_len,
0367              const char *str_sec, size_t str_len);
0368 int btf_load_into_kernel(struct btf *btf, char *log_buf, size_t log_sz, __u32 log_level);
0369
0370 struct btf *btf_get_from_fd(int btf_fd, struct btf *base_btf);
0371 void btf_get_kernel_prefix_kind(enum bpf_attach_type attach_type,
0372                 const char **prefix, int *kind);
0373
0374 struct btf_ext_info {
0375     /*
0376      * info points to the individual info section (e.g. func_info and
0377      * line_info) from the .BTF.ext. It does not include the __u32 rec_size.
0378      */
0379     void *info;
0380     __u32 rec_size;
0381     __u32 len;
0382     /* optional (maintained internally by libbpf) mapping between .BTF.ext
0383      * section and corresponding ELF section. This is used to join
0384      * information like CO-RE relocation records with corresponding BPF
0385      * programs defined in ELF sections
0386      */
0387     __u32 *sec_idxs;
0388     int sec_cnt;
0389 };
0390
0391 #define for_each_btf_ext_sec(seg, sec)                  \
0392     for (sec = (seg)->info;                     \
0393          (void *)sec < (seg)->info + (seg)->len;            \
0394          sec = (void *)sec + sizeof(struct btf_ext_info_sec) +  \
0395            (seg)->rec_size * sec->num_info)
0396
0397 #define for_each_btf_ext_rec(seg, sec, i, rec)              \
0398     for (i = 0, rec = (void *)&(sec)->data;             \
0399          i < (sec)->num_info;                   \
0400          i++, rec = (void *)rec + (seg)->rec_size)
0401
0402 /*
0403  * The .BTF.ext ELF section layout defined as
0404  *   struct btf_ext_header
0405  *   func_info subsection
0406  *
0407  * The func_info subsection layout:
0408  *   record size for struct bpf_func_info in the func_info subsection
0409  *   struct btf_sec_func_info for section #1
0410  *   a list of bpf_func_info records for section #1
0411  *     where struct bpf_func_info mimics one in include/uapi/linux/bpf.h
0412  *     but may not be identical
0413  *   struct btf_sec_func_info for section #2
0414  *   a list of bpf_func_info records for section #2
0415  *   ......
0416  *
0417  * Note that the bpf_func_info record size in .BTF.ext may not
0418  * be the same as the one defined in include/uapi/linux/bpf.h.
0419  * The loader should ensure that record_size meets minimum
0420  * requirement and pass the record as is to the kernel. The
0421  * kernel will handle the func_info properly based on its contents.
0422  */
0423 struct btf_ext_header {
0424     __u16   magic;
0425     __u8    version;
0426     __u8    flags;
0427     __u32   hdr_len;
0428
0429     /* All offsets are in bytes relative to the end of this header */
0430     __u32   func_info_off;
0431     __u32   func_info_len;
0432     __u32   line_info_off;
0433     __u32   line_info_len;
0434
0435     /* optional part of .BTF.ext header */
0436     __u32   core_relo_off;
0437     __u32   core_relo_len;
0438 };
0439
0440 struct btf_ext {
0441     union {
0442         struct btf_ext_header *hdr;
0443         void *data;
0444     };
0445     struct btf_ext_info func_info;
0446     struct btf_ext_info line_info;
0447     struct btf_ext_info core_relo_info;
0448     __u32 data_size;
0449 };
0450
0451 struct btf_ext_info_sec {
0452     __u32   sec_name_off;
0453     __u32   num_info;
0454     /* Followed by num_info * record_size number of bytes */
0455     __u8    data[];
0456 };
0457
0458 /* The minimum bpf_func_info checked by the loader */
0459 struct bpf_func_info_min {
0460     __u32   insn_off;
0461     __u32   type_id;
0462 };
0463
0464 /* The minimum bpf_line_info checked by the loader */
0465 struct bpf_line_info_min {
0466     __u32   insn_off;
0467     __u32   file_name_off;
0468     __u32   line_off;
0469     __u32   line_col;
0470 };
0471
0472
0473 typedef int (*type_id_visit_fn)(__u32 *type_id, void *ctx);
0474 typedef int (*str_off_visit_fn)(__u32 *str_off, void *ctx);
0475 int btf_type_visit_type_ids(struct btf_type *t, type_id_visit_fn visit, void *ctx);
0476 int btf_type_visit_str_offs(struct btf_type *t, str_off_visit_fn visit, void *ctx);
0477 int btf_ext_visit_type_ids(struct btf_ext *btf_ext, type_id_visit_fn visit, void *ctx);
0478 int btf_ext_visit_str_offs(struct btf_ext *btf_ext, str_off_visit_fn visit, void *ctx);
0479 __s32 btf__find_by_name_kind_own(const struct btf *btf, const char *type_name,
0480                  __u32 kind);
0481
0482 typedef int (*kallsyms_cb_t)(unsigned long long sym_addr, char sym_type,
0483                  const char *sym_name, void *ctx);
0484
0485 int libbpf_kallsyms_parse(kallsyms_cb_t cb, void *arg);
0486
0487 /* handle direct returned errors */
0488 static inline int libbpf_err(int ret)
0489 {
0490     if (ret < 0)
0491         errno = -ret;
0492     return ret;
0493 }
0494
0495 /* handle errno-based (e.g., syscall or libc) errors according to libbpf's
0496  * strict mode settings
0497  */
0498 static inline int libbpf_err_errno(int ret)
0499 {
0500     /* errno is already assumed to be set on error */
0501     return ret < 0 ? -errno : ret;
0502 }
0503
0504 /* handle error for pointer-returning APIs, err is assumed to be < 0 always */
0505 static inline void *libbpf_err_ptr(int err)
0506 {
0507     /* set errno on error, this doesn't break anything */
0508     errno = -err;
0509     return NULL;
0510 }
0511
0512 /* handle pointer-returning APIs' error handling */
0513 static inline void *libbpf_ptr(void *ret)
0514 {
0515     /* set errno on error, this doesn't break anything */
0516     if (IS_ERR(ret))
0517         errno = -PTR_ERR(ret);
0518
0519     return IS_ERR(ret) ? NULL : ret;
0520 }
0521
0522 static inline bool str_is_empty(const char *s)
0523 {
0524     return !s || !s[0];
0525 }
0526
0527 static inline bool is_ldimm64_insn(struct bpf_insn *insn)
0528 {
0529     return insn->code == (BPF_LD | BPF_IMM | BPF_DW);
0530 }
0531
0532 /* if fd is stdin, stdout, or stderr, dup to a fd greater than 2
0533  * Takes ownership of the fd passed in, and closes it if calling
0534  * fcntl(fd, F_DUPFD_CLOEXEC, 3).
0535  */
0536 static inline int ensure_good_fd(int fd)
0537 {
0538     int old_fd = fd, saved_errno;
0539
0540     if (fd < 0)
0541         return fd;
0542     if (fd < 3) {
0543         fd = fcntl(fd, F_DUPFD_CLOEXEC, 3);
0544         saved_errno = errno;
0545         close(old_fd);
0546         if (fd < 0) {
0547             pr_warn("failed to dup FD %d to FD > 2: %d\n", old_fd, -saved_errno);
0548             errno = saved_errno;
0549         }
0550     }
0551     return fd;
0552 }
0553
0554 /* The following two functions are exposed to bpftool */
0555 int bpf_core_add_cands(struct bpf_core_cand *local_cand,
0556                size_t local_essent_len,
0557                const struct btf *targ_btf,
0558                const char *targ_btf_name,
0559                int targ_start_id,
0560                struct bpf_core_cand_list *cands);
0561 void bpf_core_free_cands(struct bpf_core_cand_list *cands);
0562
0563 struct usdt_manager *usdt_manager_new(struct bpf_object *obj);
0564 void usdt_manager_free(struct usdt_manager *man);
0565 struct bpf_link * usdt_manager_attach_usdt(struct usdt_manager *man,
0566                        const struct bpf_program *prog,
0567                        pid_t pid, const char *path,
0568                        const char *usdt_provider, const char *usdt_name,
0569                        __u64 usdt_cookie);
0570
0571 static inline bool is_pow_of_2(size_t x)
0572 {
0573     return x && (x & (x - 1)) == 0;
0574 }
0575
0576 #endif /* __LIBBPF_LIBBPF_INTERNAL_H */