Back to home page

OSCL-LXR

 
 

    


0001 // SPDX-License-Identifier: GPL-2.0
0002 #include <errno.h>
0003 #include <stdlib.h>
0004 #include <bpf/bpf.h>
0005 #include <bpf/btf.h>
0006 #include <bpf/libbpf.h>
0007 #include <linux/btf.h>
0008 #include <linux/err.h>
0009 #include <linux/string.h>
0010 #include <internal/lib.h>
0011 #include <symbol/kallsyms.h>
0012 #include "bpf-event.h"
0013 #include "bpf-utils.h"
0014 #include "debug.h"
0015 #include "dso.h"
0016 #include "symbol.h"
0017 #include "machine.h"
0018 #include "env.h"
0019 #include "session.h"
0020 #include "map.h"
0021 #include "evlist.h"
0022 #include "record.h"
0023 #include "util/synthetic-events.h"
0024 
0025 #ifndef HAVE_LIBBPF_BTF__LOAD_FROM_KERNEL_BY_ID
0026 struct btf *btf__load_from_kernel_by_id(__u32 id)
0027 {
0028        struct btf *btf;
0029 #pragma GCC diagnostic push
0030 #pragma GCC diagnostic ignored "-Wdeprecated-declarations"
0031        int err = btf__get_from_id(id, &btf);
0032 #pragma GCC diagnostic pop
0033 
0034        return err ? ERR_PTR(err) : btf;
0035 }
0036 #endif
0037 
0038 #ifndef HAVE_LIBBPF_BPF_PROG_LOAD
0039 int bpf_prog_load(enum bpf_prog_type prog_type,
0040           const char *prog_name __maybe_unused,
0041           const char *license,
0042           const struct bpf_insn *insns, size_t insn_cnt,
0043           const struct bpf_prog_load_opts *opts)
0044 {
0045 #pragma GCC diagnostic push
0046 #pragma GCC diagnostic ignored "-Wdeprecated-declarations"
0047     return bpf_load_program(prog_type, insns, insn_cnt, license,
0048                 opts->kern_version, opts->log_buf, opts->log_size);
0049 #pragma GCC diagnostic pop
0050 }
0051 #endif
0052 
0053 #ifndef HAVE_LIBBPF_BPF_OBJECT__NEXT_PROGRAM
0054 struct bpf_program *
0055 bpf_object__next_program(const struct bpf_object *obj, struct bpf_program *prev)
0056 {
0057 #pragma GCC diagnostic push
0058 #pragma GCC diagnostic ignored "-Wdeprecated-declarations"
0059     return bpf_program__next(prev, obj);
0060 #pragma GCC diagnostic pop
0061 }
0062 #endif
0063 
0064 #ifndef HAVE_LIBBPF_BPF_OBJECT__NEXT_MAP
0065 struct bpf_map *
0066 bpf_object__next_map(const struct bpf_object *obj, const struct bpf_map *prev)
0067 {
0068 #pragma GCC diagnostic push
0069 #pragma GCC diagnostic ignored "-Wdeprecated-declarations"
0070     return bpf_map__next(prev, obj);
0071 #pragma GCC diagnostic pop
0072 }
0073 #endif
0074 
0075 #ifndef HAVE_LIBBPF_BTF__RAW_DATA
0076 const void *
0077 btf__raw_data(const struct btf *btf_ro, __u32 *size)
0078 {
0079 #pragma GCC diagnostic push
0080 #pragma GCC diagnostic ignored "-Wdeprecated-declarations"
0081     return btf__get_raw_data(btf_ro, size);
0082 #pragma GCC diagnostic pop
0083 }
0084 #endif
0085 
0086 static int snprintf_hex(char *buf, size_t size, unsigned char *data, size_t len)
0087 {
0088     int ret = 0;
0089     size_t i;
0090 
0091     for (i = 0; i < len; i++)
0092         ret += snprintf(buf + ret, size - ret, "%02x", data[i]);
0093     return ret;
0094 }
0095 
0096 static int machine__process_bpf_event_load(struct machine *machine,
0097                        union perf_event *event,
0098                        struct perf_sample *sample __maybe_unused)
0099 {
0100     struct bpf_prog_info_node *info_node;
0101     struct perf_env *env = machine->env;
0102     struct perf_bpil *info_linear;
0103     int id = event->bpf.id;
0104     unsigned int i;
0105 
0106     /* perf-record, no need to handle bpf-event */
0107     if (env == NULL)
0108         return 0;
0109 
0110     info_node = perf_env__find_bpf_prog_info(env, id);
0111     if (!info_node)
0112         return 0;
0113     info_linear = info_node->info_linear;
0114 
0115     for (i = 0; i < info_linear->info.nr_jited_ksyms; i++) {
0116         u64 *addrs = (u64 *)(uintptr_t)(info_linear->info.jited_ksyms);
0117         u64 addr = addrs[i];
0118         struct map *map = maps__find(machine__kernel_maps(machine), addr);
0119 
0120         if (map) {
0121             map->dso->binary_type = DSO_BINARY_TYPE__BPF_PROG_INFO;
0122             map->dso->bpf_prog.id = id;
0123             map->dso->bpf_prog.sub_id = i;
0124             map->dso->bpf_prog.env = env;
0125         }
0126     }
0127     return 0;
0128 }
0129 
0130 int machine__process_bpf(struct machine *machine, union perf_event *event,
0131              struct perf_sample *sample)
0132 {
0133     if (dump_trace)
0134         perf_event__fprintf_bpf(event, stdout);
0135 
0136     switch (event->bpf.type) {
0137     case PERF_BPF_EVENT_PROG_LOAD:
0138         return machine__process_bpf_event_load(machine, event, sample);
0139 
0140     case PERF_BPF_EVENT_PROG_UNLOAD:
0141         /*
0142          * Do not free bpf_prog_info and btf of the program here,
0143          * as annotation still need them. They will be freed at
0144          * the end of the session.
0145          */
0146         break;
0147     default:
0148         pr_debug("unexpected bpf event type of %d\n", event->bpf.type);
0149         break;
0150     }
0151     return 0;
0152 }
0153 
0154 static int perf_env__fetch_btf(struct perf_env *env,
0155                    u32 btf_id,
0156                    struct btf *btf)
0157 {
0158     struct btf_node *node;
0159     u32 data_size;
0160     const void *data;
0161 
0162     data = btf__raw_data(btf, &data_size);
0163 
0164     node = malloc(data_size + sizeof(struct btf_node));
0165     if (!node)
0166         return -1;
0167 
0168     node->id = btf_id;
0169     node->data_size = data_size;
0170     memcpy(node->data, data, data_size);
0171 
0172     if (!perf_env__insert_btf(env, node)) {
0173         /* Insertion failed because of a duplicate. */
0174         free(node);
0175         return -1;
0176     }
0177     return 0;
0178 }
0179 
0180 static int synthesize_bpf_prog_name(char *buf, int size,
0181                     struct bpf_prog_info *info,
0182                     struct btf *btf,
0183                     u32 sub_id)
0184 {
0185     u8 (*prog_tags)[BPF_TAG_SIZE] = (void *)(uintptr_t)(info->prog_tags);
0186     void *func_infos = (void *)(uintptr_t)(info->func_info);
0187     u32 sub_prog_cnt = info->nr_jited_ksyms;
0188     const struct bpf_func_info *finfo;
0189     const char *short_name = NULL;
0190     const struct btf_type *t;
0191     int name_len;
0192 
0193     name_len = snprintf(buf, size, "bpf_prog_");
0194     name_len += snprintf_hex(buf + name_len, size - name_len,
0195                  prog_tags[sub_id], BPF_TAG_SIZE);
0196     if (btf) {
0197         finfo = func_infos + sub_id * info->func_info_rec_size;
0198         t = btf__type_by_id(btf, finfo->type_id);
0199         short_name = btf__name_by_offset(btf, t->name_off);
0200     } else if (sub_id == 0 && sub_prog_cnt == 1) {
0201         /* no subprog */
0202         if (info->name[0])
0203             short_name = info->name;
0204     } else
0205         short_name = "F";
0206     if (short_name)
0207         name_len += snprintf(buf + name_len, size - name_len,
0208                      "_%s", short_name);
0209     return name_len;
0210 }
0211 
0212 /*
0213  * Synthesize PERF_RECORD_KSYMBOL and PERF_RECORD_BPF_EVENT for one bpf
0214  * program. One PERF_RECORD_BPF_EVENT is generated for the program. And
0215  * one PERF_RECORD_KSYMBOL is generated for each sub program.
0216  *
0217  * Returns:
0218  *    0 for success;
0219  *   -1 for failures;
0220  *   -2 for lack of kernel support.
0221  */
0222 static int perf_event__synthesize_one_bpf_prog(struct perf_session *session,
0223                            perf_event__handler_t process,
0224                            struct machine *machine,
0225                            int fd,
0226                            union perf_event *event,
0227                            struct record_opts *opts)
0228 {
0229     struct perf_record_ksymbol *ksymbol_event = &event->ksymbol;
0230     struct perf_record_bpf_event *bpf_event = &event->bpf;
0231     struct perf_tool *tool = session->tool;
0232     struct bpf_prog_info_node *info_node;
0233     struct perf_bpil *info_linear;
0234     struct bpf_prog_info *info;
0235     struct btf *btf = NULL;
0236     struct perf_env *env;
0237     u32 sub_prog_cnt, i;
0238     int err = 0;
0239     u64 arrays;
0240 
0241     /*
0242      * for perf-record and perf-report use header.env;
0243      * otherwise, use global perf_env.
0244      */
0245     env = session->data ? &session->header.env : &perf_env;
0246 
0247     arrays = 1UL << PERF_BPIL_JITED_KSYMS;
0248     arrays |= 1UL << PERF_BPIL_JITED_FUNC_LENS;
0249     arrays |= 1UL << PERF_BPIL_FUNC_INFO;
0250     arrays |= 1UL << PERF_BPIL_PROG_TAGS;
0251     arrays |= 1UL << PERF_BPIL_JITED_INSNS;
0252     arrays |= 1UL << PERF_BPIL_LINE_INFO;
0253     arrays |= 1UL << PERF_BPIL_JITED_LINE_INFO;
0254 
0255     info_linear = get_bpf_prog_info_linear(fd, arrays);
0256     if (IS_ERR_OR_NULL(info_linear)) {
0257         info_linear = NULL;
0258         pr_debug("%s: failed to get BPF program info. aborting\n", __func__);
0259         return -1;
0260     }
0261 
0262     if (info_linear->info_len < offsetof(struct bpf_prog_info, prog_tags)) {
0263         free(info_linear);
0264         pr_debug("%s: the kernel is too old, aborting\n", __func__);
0265         return -2;
0266     }
0267 
0268     info = &info_linear->info;
0269     if (!info->jited_ksyms) {
0270         free(info_linear);
0271         return -1;
0272     }
0273 
0274     /* number of ksyms, func_lengths, and tags should match */
0275     sub_prog_cnt = info->nr_jited_ksyms;
0276     if (sub_prog_cnt != info->nr_prog_tags ||
0277         sub_prog_cnt != info->nr_jited_func_lens) {
0278         free(info_linear);
0279         return -1;
0280     }
0281 
0282     /* check BTF func info support */
0283     if (info->btf_id && info->nr_func_info && info->func_info_rec_size) {
0284         /* btf func info number should be same as sub_prog_cnt */
0285         if (sub_prog_cnt != info->nr_func_info) {
0286             pr_debug("%s: mismatch in BPF sub program count and BTF function info count, aborting\n", __func__);
0287             free(info_linear);
0288             return -1;
0289         }
0290         btf = btf__load_from_kernel_by_id(info->btf_id);
0291         if (libbpf_get_error(btf)) {
0292             pr_debug("%s: failed to get BTF of id %u, aborting\n", __func__, info->btf_id);
0293             err = -1;
0294             goto out;
0295         }
0296         perf_env__fetch_btf(env, info->btf_id, btf);
0297     }
0298 
0299     /* Synthesize PERF_RECORD_KSYMBOL */
0300     for (i = 0; i < sub_prog_cnt; i++) {
0301         __u32 *prog_lens = (__u32 *)(uintptr_t)(info->jited_func_lens);
0302         __u64 *prog_addrs = (__u64 *)(uintptr_t)(info->jited_ksyms);
0303         int name_len;
0304 
0305         *ksymbol_event = (struct perf_record_ksymbol) {
0306             .header = {
0307                 .type = PERF_RECORD_KSYMBOL,
0308                 .size = offsetof(struct perf_record_ksymbol, name),
0309             },
0310             .addr = prog_addrs[i],
0311             .len = prog_lens[i],
0312             .ksym_type = PERF_RECORD_KSYMBOL_TYPE_BPF,
0313             .flags = 0,
0314         };
0315 
0316         name_len = synthesize_bpf_prog_name(ksymbol_event->name,
0317                             KSYM_NAME_LEN, info, btf, i);
0318         ksymbol_event->header.size += PERF_ALIGN(name_len + 1,
0319                              sizeof(u64));
0320 
0321         memset((void *)event + event->header.size, 0, machine->id_hdr_size);
0322         event->header.size += machine->id_hdr_size;
0323         err = perf_tool__process_synth_event(tool, event,
0324                              machine, process);
0325     }
0326 
0327     if (!opts->no_bpf_event) {
0328         /* Synthesize PERF_RECORD_BPF_EVENT */
0329         *bpf_event = (struct perf_record_bpf_event) {
0330             .header = {
0331                 .type = PERF_RECORD_BPF_EVENT,
0332                 .size = sizeof(struct perf_record_bpf_event),
0333             },
0334             .type = PERF_BPF_EVENT_PROG_LOAD,
0335             .flags = 0,
0336             .id = info->id,
0337         };
0338         memcpy(bpf_event->tag, info->tag, BPF_TAG_SIZE);
0339         memset((void *)event + event->header.size, 0, machine->id_hdr_size);
0340         event->header.size += machine->id_hdr_size;
0341 
0342         /* save bpf_prog_info to env */
0343         info_node = malloc(sizeof(struct bpf_prog_info_node));
0344         if (!info_node) {
0345             err = -1;
0346             goto out;
0347         }
0348 
0349         info_node->info_linear = info_linear;
0350         perf_env__insert_bpf_prog_info(env, info_node);
0351         info_linear = NULL;
0352 
0353         /*
0354          * process after saving bpf_prog_info to env, so that
0355          * required information is ready for look up
0356          */
0357         err = perf_tool__process_synth_event(tool, event,
0358                              machine, process);
0359     }
0360 
0361 out:
0362     free(info_linear);
0363     btf__free(btf);
0364     return err ? -1 : 0;
0365 }
0366 
0367 struct kallsyms_parse {
0368     union perf_event    *event;
0369     perf_event__handler_t    process;
0370     struct machine      *machine;
0371     struct perf_tool    *tool;
0372 };
0373 
0374 static int
0375 process_bpf_image(char *name, u64 addr, struct kallsyms_parse *data)
0376 {
0377     struct machine *machine = data->machine;
0378     union perf_event *event = data->event;
0379     struct perf_record_ksymbol *ksymbol;
0380     int len;
0381 
0382     ksymbol = &event->ksymbol;
0383 
0384     *ksymbol = (struct perf_record_ksymbol) {
0385         .header = {
0386             .type = PERF_RECORD_KSYMBOL,
0387             .size = offsetof(struct perf_record_ksymbol, name),
0388         },
0389         .addr      = addr,
0390         .len       = page_size,
0391         .ksym_type = PERF_RECORD_KSYMBOL_TYPE_BPF,
0392         .flags     = 0,
0393     };
0394 
0395     len = scnprintf(ksymbol->name, KSYM_NAME_LEN, "%s", name);
0396     ksymbol->header.size += PERF_ALIGN(len + 1, sizeof(u64));
0397     memset((void *) event + event->header.size, 0, machine->id_hdr_size);
0398     event->header.size += machine->id_hdr_size;
0399 
0400     return perf_tool__process_synth_event(data->tool, event, machine,
0401                           data->process);
0402 }
0403 
0404 static int
0405 kallsyms_process_symbol(void *data, const char *_name,
0406             char type __maybe_unused, u64 start)
0407 {
0408     char disp[KSYM_NAME_LEN];
0409     char *module, *name;
0410     unsigned long id;
0411     int err = 0;
0412 
0413     module = strchr(_name, '\t');
0414     if (!module)
0415         return 0;
0416 
0417     /* We are going after [bpf] module ... */
0418     if (strcmp(module + 1, "[bpf]"))
0419         return 0;
0420 
0421     name = memdup(_name, (module - _name) + 1);
0422     if (!name)
0423         return -ENOMEM;
0424 
0425     name[module - _name] = 0;
0426 
0427     /* .. and only for trampolines and dispatchers */
0428     if ((sscanf(name, "bpf_trampoline_%lu", &id) == 1) ||
0429         (sscanf(name, "bpf_dispatcher_%s", disp) == 1))
0430         err = process_bpf_image(name, start, data);
0431 
0432     free(name);
0433     return err;
0434 }
0435 
0436 int perf_event__synthesize_bpf_events(struct perf_session *session,
0437                       perf_event__handler_t process,
0438                       struct machine *machine,
0439                       struct record_opts *opts)
0440 {
0441     const char *kallsyms_filename = "/proc/kallsyms";
0442     struct kallsyms_parse arg;
0443     union perf_event *event;
0444     __u32 id = 0;
0445     int err;
0446     int fd;
0447 
0448     event = malloc(sizeof(event->bpf) + KSYM_NAME_LEN + machine->id_hdr_size);
0449     if (!event)
0450         return -1;
0451 
0452     /* Synthesize all the bpf programs in system. */
0453     while (true) {
0454         err = bpf_prog_get_next_id(id, &id);
0455         if (err) {
0456             if (errno == ENOENT) {
0457                 err = 0;
0458                 break;
0459             }
0460             pr_debug("%s: can't get next program: %s%s\n",
0461                  __func__, strerror(errno),
0462                  errno == EINVAL ? " -- kernel too old?" : "");
0463             /* don't report error on old kernel or EPERM  */
0464             err = (errno == EINVAL || errno == EPERM) ? 0 : -1;
0465             break;
0466         }
0467         fd = bpf_prog_get_fd_by_id(id);
0468         if (fd < 0) {
0469             pr_debug("%s: failed to get fd for prog_id %u\n",
0470                  __func__, id);
0471             continue;
0472         }
0473 
0474         err = perf_event__synthesize_one_bpf_prog(session, process,
0475                               machine, fd,
0476                               event, opts);
0477         close(fd);
0478         if (err) {
0479             /* do not return error for old kernel */
0480             if (err == -2)
0481                 err = 0;
0482             break;
0483         }
0484     }
0485 
0486     /* Synthesize all the bpf images - trampolines/dispatchers. */
0487     if (symbol_conf.kallsyms_name != NULL)
0488         kallsyms_filename = symbol_conf.kallsyms_name;
0489 
0490     arg = (struct kallsyms_parse) {
0491         .event   = event,
0492         .process = process,
0493         .machine = machine,
0494         .tool    = session->tool,
0495     };
0496 
0497     if (kallsyms__parse(kallsyms_filename, &arg, kallsyms_process_symbol)) {
0498         pr_err("%s: failed to synthesize bpf images: %s\n",
0499                __func__, strerror(errno));
0500     }
0501 
0502     free(event);
0503     return err;
0504 }
0505 
0506 static void perf_env__add_bpf_info(struct perf_env *env, u32 id)
0507 {
0508     struct bpf_prog_info_node *info_node;
0509     struct perf_bpil *info_linear;
0510     struct btf *btf = NULL;
0511     u64 arrays;
0512     u32 btf_id;
0513     int fd;
0514 
0515     fd = bpf_prog_get_fd_by_id(id);
0516     if (fd < 0)
0517         return;
0518 
0519     arrays = 1UL << PERF_BPIL_JITED_KSYMS;
0520     arrays |= 1UL << PERF_BPIL_JITED_FUNC_LENS;
0521     arrays |= 1UL << PERF_BPIL_FUNC_INFO;
0522     arrays |= 1UL << PERF_BPIL_PROG_TAGS;
0523     arrays |= 1UL << PERF_BPIL_JITED_INSNS;
0524     arrays |= 1UL << PERF_BPIL_LINE_INFO;
0525     arrays |= 1UL << PERF_BPIL_JITED_LINE_INFO;
0526 
0527     info_linear = get_bpf_prog_info_linear(fd, arrays);
0528     if (IS_ERR_OR_NULL(info_linear)) {
0529         pr_debug("%s: failed to get BPF program info. aborting\n", __func__);
0530         goto out;
0531     }
0532 
0533     btf_id = info_linear->info.btf_id;
0534 
0535     info_node = malloc(sizeof(struct bpf_prog_info_node));
0536     if (info_node) {
0537         info_node->info_linear = info_linear;
0538         perf_env__insert_bpf_prog_info(env, info_node);
0539     } else
0540         free(info_linear);
0541 
0542     if (btf_id == 0)
0543         goto out;
0544 
0545     btf = btf__load_from_kernel_by_id(btf_id);
0546     if (libbpf_get_error(btf)) {
0547         pr_debug("%s: failed to get BTF of id %u, aborting\n",
0548              __func__, btf_id);
0549         goto out;
0550     }
0551     perf_env__fetch_btf(env, btf_id, btf);
0552 
0553 out:
0554     btf__free(btf);
0555     close(fd);
0556 }
0557 
0558 static int bpf_event__sb_cb(union perf_event *event, void *data)
0559 {
0560     struct perf_env *env = data;
0561 
0562     if (event->header.type != PERF_RECORD_BPF_EVENT)
0563         return -1;
0564 
0565     switch (event->bpf.type) {
0566     case PERF_BPF_EVENT_PROG_LOAD:
0567         perf_env__add_bpf_info(env, event->bpf.id);
0568 
0569     case PERF_BPF_EVENT_PROG_UNLOAD:
0570         /*
0571          * Do not free bpf_prog_info and btf of the program here,
0572          * as annotation still need them. They will be freed at
0573          * the end of the session.
0574          */
0575         break;
0576     default:
0577         pr_debug("unexpected bpf event type of %d\n", event->bpf.type);
0578         break;
0579     }
0580 
0581     return 0;
0582 }
0583 
0584 int evlist__add_bpf_sb_event(struct evlist *evlist, struct perf_env *env)
0585 {
0586     struct perf_event_attr attr = {
0587         .type             = PERF_TYPE_SOFTWARE,
0588         .config           = PERF_COUNT_SW_DUMMY,
0589         .sample_id_all    = 1,
0590         .watermark        = 1,
0591         .bpf_event        = 1,
0592         .size      = sizeof(attr), /* to capture ABI version */
0593     };
0594 
0595     /*
0596      * Older gcc versions don't support designated initializers, like above,
0597      * for unnamed union members, such as the following:
0598      */
0599     attr.wakeup_watermark = 1;
0600 
0601     return evlist__add_sb_event(evlist, &attr, bpf_event__sb_cb, env);
0602 }
0603 
0604 void bpf_event__print_bpf_prog_info(struct bpf_prog_info *info,
0605                     struct perf_env *env,
0606                     FILE *fp)
0607 {
0608     __u32 *prog_lens = (__u32 *)(uintptr_t)(info->jited_func_lens);
0609     __u64 *prog_addrs = (__u64 *)(uintptr_t)(info->jited_ksyms);
0610     char name[KSYM_NAME_LEN];
0611     struct btf *btf = NULL;
0612     u32 sub_prog_cnt, i;
0613 
0614     sub_prog_cnt = info->nr_jited_ksyms;
0615     if (sub_prog_cnt != info->nr_prog_tags ||
0616         sub_prog_cnt != info->nr_jited_func_lens)
0617         return;
0618 
0619     if (info->btf_id) {
0620         struct btf_node *node;
0621 
0622         node = perf_env__find_btf(env, info->btf_id);
0623         if (node)
0624             btf = btf__new((__u8 *)(node->data),
0625                        node->data_size);
0626     }
0627 
0628     if (sub_prog_cnt == 1) {
0629         synthesize_bpf_prog_name(name, KSYM_NAME_LEN, info, btf, 0);
0630         fprintf(fp, "# bpf_prog_info %u: %s addr 0x%llx size %u\n",
0631             info->id, name, prog_addrs[0], prog_lens[0]);
0632         goto out;
0633     }
0634 
0635     fprintf(fp, "# bpf_prog_info %u:\n", info->id);
0636     for (i = 0; i < sub_prog_cnt; i++) {
0637         synthesize_bpf_prog_name(name, KSYM_NAME_LEN, info, btf, i);
0638 
0639         fprintf(fp, "# \tsub_prog %u: %s addr 0x%llx size %u\n",
0640             i, name, prog_addrs[i], prog_lens[i]);
0641     }
0642 out:
0643     btf__free(btf);
0644 }