bpf/runqslower/runqslower.bpf.c

0001 // SPDX-License-Identifier: GPL-2.0
0002 // Copyright (c) 2019 Facebook
0003 #include "vmlinux.h"
0004 #include <bpf/bpf_helpers.h>
0005 #include "runqslower.h"
0006
0007 #define TASK_RUNNING 0
0008 #define BPF_F_CURRENT_CPU 0xffffffffULL
0009
0010 const volatile __u64 min_us = 0;
0011 const volatile pid_t targ_pid = 0;
0012
0013 struct {
0014     __uint(type, BPF_MAP_TYPE_TASK_STORAGE);
0015     __uint(map_flags, BPF_F_NO_PREALLOC);
0016     __type(key, int);
0017     __type(value, u64);
0018 } start SEC(".maps");
0019
0020 struct {
0021     __uint(type, BPF_MAP_TYPE_PERF_EVENT_ARRAY);
0022     __uint(key_size, sizeof(u32));
0023     __uint(value_size, sizeof(u32));
0024 } events SEC(".maps");
0025
0026 /* record enqueue timestamp */
0027 __always_inline
0028 static int trace_enqueue(struct task_struct *t)
0029 {
0030     u32 pid = t->pid;
0031     u64 *ptr;
0032
0033     if (!pid || (targ_pid && targ_pid != pid))
0034         return 0;
0035
0036     ptr = bpf_task_storage_get(&start, t, 0,
0037                    BPF_LOCAL_STORAGE_GET_F_CREATE);
0038     if (!ptr)
0039         return 0;
0040
0041     *ptr = bpf_ktime_get_ns();
0042     return 0;
0043 }
0044
0045 SEC("tp_btf/sched_wakeup")
0046 int handle__sched_wakeup(u64 *ctx)
0047 {
0048     /* TP_PROTO(struct task_struct *p) */
0049     struct task_struct *p = (void *)ctx[0];
0050
0051     return trace_enqueue(p);
0052 }
0053
0054 SEC("tp_btf/sched_wakeup_new")
0055 int handle__sched_wakeup_new(u64 *ctx)
0056 {
0057     /* TP_PROTO(struct task_struct *p) */
0058     struct task_struct *p = (void *)ctx[0];
0059
0060     return trace_enqueue(p);
0061 }
0062
0063 SEC("tp_btf/sched_switch")
0064 int handle__sched_switch(u64 *ctx)
0065 {
0066     /* TP_PROTO(bool preempt, struct task_struct *prev,
0067      *      struct task_struct *next)
0068      */
0069     struct task_struct *prev = (struct task_struct *)ctx[1];
0070     struct task_struct *next = (struct task_struct *)ctx[2];
0071     struct runq_event event = {};
0072     u64 *tsp, delta_us;
0073     long state;
0074     u32 pid;
0075
0076     /* ivcsw: treat like an enqueue event and store timestamp */
0077     if (prev->__state == TASK_RUNNING)
0078         trace_enqueue(prev);
0079
0080     pid = next->pid;
0081
0082     /* For pid mismatch, save a bpf_task_storage_get */
0083     if (!pid || (targ_pid && targ_pid != pid))
0084         return 0;
0085
0086     /* fetch timestamp and calculate delta */
0087     tsp = bpf_task_storage_get(&start, next, 0, 0);
0088     if (!tsp)
0089         return 0;   /* missed enqueue */
0090
0091     delta_us = (bpf_ktime_get_ns() - *tsp) / 1000;
0092     if (min_us && delta_us <= min_us)
0093         return 0;
0094
0095     event.pid = pid;
0096     event.delta_us = delta_us;
0097     bpf_get_current_comm(&event.task, sizeof(event.task));
0098
0099     /* output */
0100     bpf_perf_event_output(ctx, &events, BPF_F_CURRENT_CPU,
0101                   &event, sizeof(event));
0102
0103     bpf_task_storage_delete(&start, next);
0104     return 0;
0105 }
0106
0107 char LICENSE[] SEC("license") = "GPL";