bpf/progs/pyperf.h

0001 // SPDX-License-Identifier: GPL-2.0
0002 // Copyright (c) 2019 Facebook
0003 #include <linux/sched.h>
0004 #include <linux/ptrace.h>
0005 #include <stdint.h>
0006 #include <stddef.h>
0007 #include <stdbool.h>
0008 #include <linux/bpf.h>
0009 #include <bpf/bpf_helpers.h>
0010
0011 #define FUNCTION_NAME_LEN 64
0012 #define FILE_NAME_LEN 128
0013 #define TASK_COMM_LEN 16
0014
0015 typedef struct {
0016     int PyThreadState_frame;
0017     int PyThreadState_thread;
0018     int PyFrameObject_back;
0019     int PyFrameObject_code;
0020     int PyFrameObject_lineno;
0021     int PyCodeObject_filename;
0022     int PyCodeObject_name;
0023     int String_data;
0024     int String_size;
0025 } OffsetConfig;
0026
0027 typedef struct {
0028     uintptr_t current_state_addr;
0029     uintptr_t tls_key_addr;
0030     OffsetConfig offsets;
0031     bool use_tls;
0032 } PidData;
0033
0034 typedef struct {
0035     uint32_t success;
0036 } Stats;
0037
0038 typedef struct {
0039     char name[FUNCTION_NAME_LEN];
0040     char file[FILE_NAME_LEN];
0041 } Symbol;
0042
0043 typedef struct {
0044     uint32_t pid;
0045     uint32_t tid;
0046     char comm[TASK_COMM_LEN];
0047     int32_t kernel_stack_id;
0048     int32_t user_stack_id;
0049     bool thread_current;
0050     bool pthread_match;
0051     bool stack_complete;
0052     int16_t stack_len;
0053     int32_t stack[STACK_MAX_LEN];
0054
0055     int has_meta;
0056     int metadata;
0057     char dummy_safeguard;
0058 } Event;
0059
0060
0061 typedef int pid_t;
0062
0063 typedef struct {
0064     void* f_back; // PyFrameObject.f_back, previous frame
0065     void* f_code; // PyFrameObject.f_code, pointer to PyCodeObject
0066     void* co_filename; // PyCodeObject.co_filename
0067     void* co_name; // PyCodeObject.co_name
0068 } FrameData;
0069
0070 #ifdef SUBPROGS
0071 __noinline
0072 #else
0073 __always_inline
0074 #endif
0075 static void *get_thread_state(void *tls_base, PidData *pidData)
0076 {
0077     void* thread_state;
0078     int key;
0079
0080     bpf_probe_read_user(&key, sizeof(key), (void*)(long)pidData->tls_key_addr);
0081     bpf_probe_read_user(&thread_state, sizeof(thread_state),
0082                 tls_base + 0x310 + key * 0x10 + 0x08);
0083     return thread_state;
0084 }
0085
0086 static __always_inline bool get_frame_data(void *frame_ptr, PidData *pidData,
0087                        FrameData *frame, Symbol *symbol)
0088 {
0089     // read data from PyFrameObject
0090     bpf_probe_read_user(&frame->f_back,
0091                 sizeof(frame->f_back),
0092                 frame_ptr + pidData->offsets.PyFrameObject_back);
0093     bpf_probe_read_user(&frame->f_code,
0094                 sizeof(frame->f_code),
0095                 frame_ptr + pidData->offsets.PyFrameObject_code);
0096
0097     // read data from PyCodeObject
0098     if (!frame->f_code)
0099         return false;
0100     bpf_probe_read_user(&frame->co_filename,
0101                 sizeof(frame->co_filename),
0102                 frame->f_code + pidData->offsets.PyCodeObject_filename);
0103     bpf_probe_read_user(&frame->co_name,
0104                 sizeof(frame->co_name),
0105                 frame->f_code + pidData->offsets.PyCodeObject_name);
0106     // read actual names into symbol
0107     if (frame->co_filename)
0108         bpf_probe_read_user_str(&symbol->file,
0109                     sizeof(symbol->file),
0110                     frame->co_filename +
0111                     pidData->offsets.String_data);
0112     if (frame->co_name)
0113         bpf_probe_read_user_str(&symbol->name,
0114                     sizeof(symbol->name),
0115                     frame->co_name +
0116                     pidData->offsets.String_data);
0117     return true;
0118 }
0119
0120 struct {
0121     __uint(type, BPF_MAP_TYPE_HASH);
0122     __uint(max_entries, 1);
0123     __type(key, int);
0124     __type(value, PidData);
0125 } pidmap SEC(".maps");
0126
0127 struct {
0128     __uint(type, BPF_MAP_TYPE_HASH);
0129     __uint(max_entries, 1);
0130     __type(key, int);
0131     __type(value, Event);
0132 } eventmap SEC(".maps");
0133
0134 struct {
0135     __uint(type, BPF_MAP_TYPE_HASH);
0136     __uint(max_entries, 1);
0137     __type(key, Symbol);
0138     __type(value, int);
0139 } symbolmap SEC(".maps");
0140
0141 struct {
0142     __uint(type, BPF_MAP_TYPE_ARRAY);
0143     __uint(max_entries, 1);
0144     __type(key, int);
0145     __type(value, Stats);
0146 } statsmap SEC(".maps");
0147
0148 struct {
0149     __uint(type, BPF_MAP_TYPE_PERF_EVENT_ARRAY);
0150     __uint(max_entries, 32);
0151     __uint(key_size, sizeof(int));
0152     __uint(value_size, sizeof(int));
0153 } perfmap SEC(".maps");
0154
0155 struct {
0156     __uint(type, BPF_MAP_TYPE_STACK_TRACE);
0157     __uint(max_entries, 1000);
0158     __uint(key_size, sizeof(int));
0159     __uint(value_size, sizeof(long long) * 127);
0160 } stackmap SEC(".maps");
0161
0162 #ifdef USE_BPF_LOOP
0163 struct process_frame_ctx {
0164     int cur_cpu;
0165     int32_t *symbol_counter;
0166     void *frame_ptr;
0167     FrameData *frame;
0168     PidData *pidData;
0169     Symbol *sym;
0170     Event *event;
0171     bool done;
0172 };
0173
0174 static int process_frame_callback(__u32 i, struct process_frame_ctx *ctx)
0175 {
0176     int zero = 0;
0177     void *frame_ptr = ctx->frame_ptr;
0178     PidData *pidData = ctx->pidData;
0179     FrameData *frame = ctx->frame;
0180     int32_t *symbol_counter = ctx->symbol_counter;
0181     int cur_cpu = ctx->cur_cpu;
0182     Event *event = ctx->event;
0183     Symbol *sym = ctx->sym;
0184
0185     if (frame_ptr && get_frame_data(frame_ptr, pidData, frame, sym)) {
0186         int32_t new_symbol_id = *symbol_counter * 64 + cur_cpu;
0187         int32_t *symbol_id = bpf_map_lookup_elem(&symbolmap, sym);
0188
0189         if (!symbol_id) {
0190             bpf_map_update_elem(&symbolmap, sym, &zero, 0);
0191             symbol_id = bpf_map_lookup_elem(&symbolmap, sym);
0192             if (!symbol_id) {
0193                 ctx->done = true;
0194                 return 1;
0195             }
0196         }
0197         if (*symbol_id == new_symbol_id)
0198             (*symbol_counter)++;
0199
0200         barrier_var(i);
0201         if (i >= STACK_MAX_LEN)
0202             return 1;
0203
0204         event->stack[i] = *symbol_id;
0205
0206         event->stack_len = i + 1;
0207         frame_ptr = frame->f_back;
0208     }
0209     return 0;
0210 }
0211 #endif /* USE_BPF_LOOP */
0212
0213 #ifdef GLOBAL_FUNC
0214 __noinline
0215 #elif defined(SUBPROGS)
0216 static __noinline
0217 #else
0218 static __always_inline
0219 #endif
0220 int __on_event(struct bpf_raw_tracepoint_args *ctx)
0221 {
0222     uint64_t pid_tgid = bpf_get_current_pid_tgid();
0223     pid_t pid = (pid_t)(pid_tgid >> 32);
0224     PidData* pidData = bpf_map_lookup_elem(&pidmap, &pid);
0225     if (!pidData)
0226         return 0;
0227
0228     int zero = 0;
0229     Event* event = bpf_map_lookup_elem(&eventmap, &zero);
0230     if (!event)
0231         return 0;
0232
0233     event->pid = pid;
0234
0235     event->tid = (pid_t)pid_tgid;
0236     bpf_get_current_comm(&event->comm, sizeof(event->comm));
0237
0238     event->user_stack_id = bpf_get_stackid(ctx, &stackmap, BPF_F_USER_STACK);
0239     event->kernel_stack_id = bpf_get_stackid(ctx, &stackmap, 0);
0240
0241     void* thread_state_current = (void*)0;
0242     bpf_probe_read_user(&thread_state_current,
0243                 sizeof(thread_state_current),
0244                 (void*)(long)pidData->current_state_addr);
0245
0246     struct task_struct* task = (struct task_struct*)bpf_get_current_task();
0247     void* tls_base = (void*)task;
0248
0249     void* thread_state = pidData->use_tls ? get_thread_state(tls_base, pidData)
0250         : thread_state_current;
0251     event->thread_current = thread_state == thread_state_current;
0252
0253     if (pidData->use_tls) {
0254         uint64_t pthread_created;
0255         uint64_t pthread_self;
0256         bpf_probe_read_user(&pthread_self, sizeof(pthread_self),
0257                     tls_base + 0x10);
0258
0259         bpf_probe_read_user(&pthread_created,
0260                     sizeof(pthread_created),
0261                     thread_state +
0262                     pidData->offsets.PyThreadState_thread);
0263         event->pthread_match = pthread_created == pthread_self;
0264     } else {
0265         event->pthread_match = 1;
0266     }
0267
0268     if (event->pthread_match || !pidData->use_tls) {
0269         void* frame_ptr;
0270         FrameData frame;
0271         Symbol sym = {};
0272         int cur_cpu = bpf_get_smp_processor_id();
0273
0274         bpf_probe_read_user(&frame_ptr,
0275                     sizeof(frame_ptr),
0276                     thread_state +
0277                     pidData->offsets.PyThreadState_frame);
0278
0279         int32_t* symbol_counter = bpf_map_lookup_elem(&symbolmap, &sym);
0280         if (symbol_counter == NULL)
0281             return 0;
0282 #ifdef USE_BPF_LOOP
0283     struct process_frame_ctx ctx = {
0284         .cur_cpu = cur_cpu,
0285         .symbol_counter = symbol_counter,
0286         .frame_ptr = frame_ptr,
0287         .frame = &frame,
0288         .pidData = pidData,
0289         .sym = &sym,
0290         .event = event,
0291     };
0292
0293     bpf_loop(STACK_MAX_LEN, process_frame_callback, &ctx, 0);
0294     if (ctx.done)
0295         return 0;
0296 #else
0297 #ifdef NO_UNROLL
0298 #pragma clang loop unroll(disable)
0299 #else
0300 #ifdef UNROLL_COUNT
0301 #pragma clang loop unroll_count(UNROLL_COUNT)
0302 #else
0303 #pragma clang loop unroll(full)
0304 #endif
0305 #endif /* NO_UNROLL */
0306         /* Unwind python stack */
0307         for (int i = 0; i < STACK_MAX_LEN; ++i) {
0308             if (frame_ptr && get_frame_data(frame_ptr, pidData, &frame, &sym)) {
0309                 int32_t new_symbol_id = *symbol_counter * 64 + cur_cpu;
0310                 int32_t *symbol_id = bpf_map_lookup_elem(&symbolmap, &sym);
0311                 if (!symbol_id) {
0312                     bpf_map_update_elem(&symbolmap, &sym, &zero, 0);
0313                     symbol_id = bpf_map_lookup_elem(&symbolmap, &sym);
0314                     if (!symbol_id)
0315                         return 0;
0316                 }
0317                 if (*symbol_id == new_symbol_id)
0318                     (*symbol_counter)++;
0319                 event->stack[i] = *symbol_id;
0320                 event->stack_len = i + 1;
0321                 frame_ptr = frame.f_back;
0322             }
0323         }
0324 #endif /* USE_BPF_LOOP */
0325         event->stack_complete = frame_ptr == NULL;
0326     } else {
0327         event->stack_complete = 1;
0328     }
0329
0330     Stats* stats = bpf_map_lookup_elem(&statsmap, &zero);
0331     if (stats)
0332         stats->success++;
0333
0334     event->has_meta = 0;
0335     bpf_perf_event_output(ctx, &perfmap, 0, event, offsetof(Event, metadata));
0336     return 0;
0337 }
0338
0339 SEC("raw_tracepoint/kfree_skb")
0340 int on_event(struct bpf_raw_tracepoint_args* ctx)
0341 {
0342     int i, ret = 0;
0343     ret |= __on_event(ctx);
0344     ret |= __on_event(ctx);
0345     ret |= __on_event(ctx);
0346     ret |= __on_event(ctx);
0347     ret |= __on_event(ctx);
0348     return ret;
0349 }
0350
0351 char _license[] SEC("license") = "GPL";