lib/bpf/usdt.bpf.h

0001 /* SPDX-License-Identifier: (LGPL-2.1 OR BSD-2-Clause) */
0002 /* Copyright (c) 2022 Meta Platforms, Inc. and affiliates. */
0003 #ifndef __USDT_BPF_H__
0004 #define __USDT_BPF_H__
0005
0006 #include <linux/errno.h>
0007 #include <bpf/bpf_helpers.h>
0008 #include <bpf/bpf_tracing.h>
0009
0010 /* Below types and maps are internal implementation details of libbpf's USDT
0011  * support and are subjects to change. Also, bpf_usdt_xxx() API helpers should
0012  * be considered an unstable API as well and might be adjusted based on user
0013  * feedback from using libbpf's USDT support in production.
0014  */
0015
0016 /* User can override BPF_USDT_MAX_SPEC_CNT to change default size of internal
0017  * map that keeps track of USDT argument specifications. This might be
0018  * necessary if there are a lot of USDT attachments.
0019  */
0020 #ifndef BPF_USDT_MAX_SPEC_CNT
0021 #define BPF_USDT_MAX_SPEC_CNT 256
0022 #endif
0023 /* User can override BPF_USDT_MAX_IP_CNT to change default size of internal
0024  * map that keeps track of IP (memory address) mapping to USDT argument
0025  * specification.
0026  * Note, if kernel supports BPF cookies, this map is not used and could be
0027  * resized all the way to 1 to save a bit of memory.
0028  */
0029 #ifndef BPF_USDT_MAX_IP_CNT
0030 #define BPF_USDT_MAX_IP_CNT (4 * BPF_USDT_MAX_SPEC_CNT)
0031 #endif
0032
0033 enum __bpf_usdt_arg_type {
0034     BPF_USDT_ARG_CONST,
0035     BPF_USDT_ARG_REG,
0036     BPF_USDT_ARG_REG_DEREF,
0037 };
0038
0039 struct __bpf_usdt_arg_spec {
0040     /* u64 scalar interpreted depending on arg_type, see below */
0041     __u64 val_off;
0042     /* arg location case, see bpf_udst_arg() for details */
0043     enum __bpf_usdt_arg_type arg_type;
0044     /* offset of referenced register within struct pt_regs */
0045     short reg_off;
0046     /* whether arg should be interpreted as signed value */
0047     bool arg_signed;
0048     /* number of bits that need to be cleared and, optionally,
0049      * sign-extended to cast arguments that are 1, 2, or 4 bytes
0050      * long into final 8-byte u64/s64 value returned to user
0051      */
0052     char arg_bitshift;
0053 };
0054
0055 /* should match USDT_MAX_ARG_CNT in usdt.c exactly */
0056 #define BPF_USDT_MAX_ARG_CNT 12
0057 struct __bpf_usdt_spec {
0058     struct __bpf_usdt_arg_spec args[BPF_USDT_MAX_ARG_CNT];
0059     __u64 usdt_cookie;
0060     short arg_cnt;
0061 };
0062
0063 struct {
0064     __uint(type, BPF_MAP_TYPE_ARRAY);
0065     __uint(max_entries, BPF_USDT_MAX_SPEC_CNT);
0066     __type(key, int);
0067     __type(value, struct __bpf_usdt_spec);
0068 } __bpf_usdt_specs SEC(".maps") __weak;
0069
0070 struct {
0071     __uint(type, BPF_MAP_TYPE_HASH);
0072     __uint(max_entries, BPF_USDT_MAX_IP_CNT);
0073     __type(key, long);
0074     __type(value, __u32);
0075 } __bpf_usdt_ip_to_spec_id SEC(".maps") __weak;
0076
0077 extern const _Bool LINUX_HAS_BPF_COOKIE __kconfig;
0078
0079 static __always_inline
0080 int __bpf_usdt_spec_id(struct pt_regs *ctx)
0081 {
0082     if (!LINUX_HAS_BPF_COOKIE) {
0083         long ip = PT_REGS_IP(ctx);
0084         int *spec_id_ptr;
0085
0086         spec_id_ptr = bpf_map_lookup_elem(&__bpf_usdt_ip_to_spec_id, &ip);
0087         return spec_id_ptr ? *spec_id_ptr : -ESRCH;
0088     }
0089
0090     return bpf_get_attach_cookie(ctx);
0091 }
0092
0093 /* Return number of USDT arguments defined for currently traced USDT. */
0094 __weak __hidden
0095 int bpf_usdt_arg_cnt(struct pt_regs *ctx)
0096 {
0097     struct __bpf_usdt_spec *spec;
0098     int spec_id;
0099
0100     spec_id = __bpf_usdt_spec_id(ctx);
0101     if (spec_id < 0)
0102         return -ESRCH;
0103
0104     spec = bpf_map_lookup_elem(&__bpf_usdt_specs, &spec_id);
0105     if (!spec)
0106         return -ESRCH;
0107
0108     return spec->arg_cnt;
0109 }
0110
0111 /* Fetch USDT argument #*arg_num* (zero-indexed) and put its value into *res.
0112  * Returns 0 on success; negative error, otherwise.
0113  * On error *res is guaranteed to be set to zero.
0114  */
0115 __weak __hidden
0116 int bpf_usdt_arg(struct pt_regs *ctx, __u64 arg_num, long *res)
0117 {
0118     struct __bpf_usdt_spec *spec;
0119     struct __bpf_usdt_arg_spec *arg_spec;
0120     unsigned long val;
0121     int err, spec_id;
0122
0123     *res = 0;
0124
0125     spec_id = __bpf_usdt_spec_id(ctx);
0126     if (spec_id < 0)
0127         return -ESRCH;
0128
0129     spec = bpf_map_lookup_elem(&__bpf_usdt_specs, &spec_id);
0130     if (!spec)
0131         return -ESRCH;
0132
0133     if (arg_num >= BPF_USDT_MAX_ARG_CNT || arg_num >= spec->arg_cnt)
0134         return -ENOENT;
0135
0136     arg_spec = &spec->args[arg_num];
0137     switch (arg_spec->arg_type) {
0138     case BPF_USDT_ARG_CONST:
0139         /* Arg is just a constant ("-4@$-9" in USDT arg spec).
0140          * value is recorded in arg_spec->val_off directly.
0141          */
0142         val = arg_spec->val_off;
0143         break;
0144     case BPF_USDT_ARG_REG:
0145         /* Arg is in a register (e.g, "8@%rax" in USDT arg spec),
0146          * so we read the contents of that register directly from
0147          * struct pt_regs. To keep things simple user-space parts
0148          * record offsetof(struct pt_regs, <regname>) in arg_spec->reg_off.
0149          */
0150         err = bpf_probe_read_kernel(&val, sizeof(val), (void *)ctx + arg_spec->reg_off);
0151         if (err)
0152             return err;
0153         break;
0154     case BPF_USDT_ARG_REG_DEREF:
0155         /* Arg is in memory addressed by register, plus some offset
0156          * (e.g., "-4@-1204(%rbp)" in USDT arg spec). Register is
0157          * identified like with BPF_USDT_ARG_REG case, and the offset
0158          * is in arg_spec->val_off. We first fetch register contents
0159          * from pt_regs, then do another user-space probe read to
0160          * fetch argument value itself.
0161          */
0162         err = bpf_probe_read_kernel(&val, sizeof(val), (void *)ctx + arg_spec->reg_off);
0163         if (err)
0164             return err;
0165         err = bpf_probe_read_user(&val, sizeof(val), (void *)val + arg_spec->val_off);
0166         if (err)
0167             return err;
0168 #if __BYTE_ORDER__ == __ORDER_BIG_ENDIAN__
0169         val >>= arg_spec->arg_bitshift;
0170 #endif
0171         break;
0172     default:
0173         return -EINVAL;
0174     }
0175
0176     /* cast arg from 1, 2, or 4 bytes to final 8 byte size clearing
0177      * necessary upper arg_bitshift bits, with sign extension if argument
0178      * is signed
0179      */
0180     val <<= arg_spec->arg_bitshift;
0181     if (arg_spec->arg_signed)
0182         val = ((long)val) >> arg_spec->arg_bitshift;
0183     else
0184         val = val >> arg_spec->arg_bitshift;
0185     *res = val;
0186     return 0;
0187 }
0188
0189 /* Retrieve user-specified cookie value provided during attach as
0190  * bpf_usdt_opts.usdt_cookie. This serves the same purpose as BPF cookie
0191  * returned by bpf_get_attach_cookie(). Libbpf's support for USDT is itself
0192  * utilizing BPF cookies internally, so user can't use BPF cookie directly
0193  * for USDT programs and has to use bpf_usdt_cookie() API instead.
0194  */
0195 __weak __hidden
0196 long bpf_usdt_cookie(struct pt_regs *ctx)
0197 {
0198     struct __bpf_usdt_spec *spec;
0199     int spec_id;
0200
0201     spec_id = __bpf_usdt_spec_id(ctx);
0202     if (spec_id < 0)
0203         return 0;
0204
0205     spec = bpf_map_lookup_elem(&__bpf_usdt_specs, &spec_id);
0206     if (!spec)
0207         return 0;
0208
0209     return spec->usdt_cookie;
0210 }
0211
0212 /* we rely on ___bpf_apply() and ___bpf_narg() macros already defined in bpf_tracing.h */
0213 #define ___bpf_usdt_args0() ctx
0214 #define ___bpf_usdt_args1(x) ___bpf_usdt_args0(), ({ long _x; bpf_usdt_arg(ctx, 0, &_x); (void *)_x; })
0215 #define ___bpf_usdt_args2(x, args...) ___bpf_usdt_args1(args), ({ long _x; bpf_usdt_arg(ctx, 1, &_x); (void *)_x; })
0216 #define ___bpf_usdt_args3(x, args...) ___bpf_usdt_args2(args), ({ long _x; bpf_usdt_arg(ctx, 2, &_x); (void *)_x; })
0217 #define ___bpf_usdt_args4(x, args...) ___bpf_usdt_args3(args), ({ long _x; bpf_usdt_arg(ctx, 3, &_x); (void *)_x; })
0218 #define ___bpf_usdt_args5(x, args...) ___bpf_usdt_args4(args), ({ long _x; bpf_usdt_arg(ctx, 4, &_x); (void *)_x; })
0219 #define ___bpf_usdt_args6(x, args...) ___bpf_usdt_args5(args), ({ long _x; bpf_usdt_arg(ctx, 5, &_x); (void *)_x; })
0220 #define ___bpf_usdt_args7(x, args...) ___bpf_usdt_args6(args), ({ long _x; bpf_usdt_arg(ctx, 6, &_x); (void *)_x; })
0221 #define ___bpf_usdt_args8(x, args...) ___bpf_usdt_args7(args), ({ long _x; bpf_usdt_arg(ctx, 7, &_x); (void *)_x; })
0222 #define ___bpf_usdt_args9(x, args...) ___bpf_usdt_args8(args), ({ long _x; bpf_usdt_arg(ctx, 8, &_x); (void *)_x; })
0223 #define ___bpf_usdt_args10(x, args...) ___bpf_usdt_args9(args), ({ long _x; bpf_usdt_arg(ctx, 9, &_x); (void *)_x; })
0224 #define ___bpf_usdt_args11(x, args...) ___bpf_usdt_args10(args), ({ long _x; bpf_usdt_arg(ctx, 10, &_x); (void *)_x; })
0225 #define ___bpf_usdt_args12(x, args...) ___bpf_usdt_args11(args), ({ long _x; bpf_usdt_arg(ctx, 11, &_x); (void *)_x; })
0226 #define ___bpf_usdt_args(args...) ___bpf_apply(___bpf_usdt_args, ___bpf_narg(args))(args)
0227
0228 /*
0229  * BPF_USDT serves the same purpose for USDT handlers as BPF_PROG for
0230  * tp_btf/fentry/fexit BPF programs and BPF_KPROBE for kprobes.
0231  * Original struct pt_regs * context is preserved as 'ctx' argument.
0232  */
0233 #define BPF_USDT(name, args...)                         \
0234 name(struct pt_regs *ctx);                          \
0235 static __attribute__((always_inline)) typeof(name(0))               \
0236 ____##name(struct pt_regs *ctx, ##args);                    \
0237 typeof(name(0)) name(struct pt_regs *ctx)                   \
0238 {                                       \
0239         _Pragma("GCC diagnostic push")                      \
0240         _Pragma("GCC diagnostic ignored \"-Wint-conversion\"")          \
0241         return ____##name(___bpf_usdt_args(args));              \
0242         _Pragma("GCC diagnostic pop")                       \
0243 }                                       \
0244 static __attribute__((always_inline)) typeof(name(0))               \
0245 ____##name(struct pt_regs *ctx, ##args)
0246
0247 #endif /* __USDT_BPF_H__ */