kernel/trace/trace_branch.c

0001 // SPDX-License-Identifier: GPL-2.0
0002 /*
0003  * unlikely profiler
0004  *
0005  * Copyright (C) 2008 Steven Rostedt <srostedt@redhat.com>
0006  */
0007 #include <linux/kallsyms.h>
0008 #include <linux/seq_file.h>
0009 #include <linux/spinlock.h>
0010 #include <linux/irqflags.h>
0011 #include <linux/uaccess.h>
0012 #include <linux/module.h>
0013 #include <linux/ftrace.h>
0014 #include <linux/hash.h>
0015 #include <linux/fs.h>
0016 #include <asm/local.h>
0017
0018 #include "trace.h"
0019 #include "trace_stat.h"
0020 #include "trace_output.h"
0021
0022 #ifdef CONFIG_BRANCH_TRACER
0023
0024 static struct tracer branch_trace;
0025 static int branch_tracing_enabled __read_mostly;
0026 static DEFINE_MUTEX(branch_tracing_mutex);
0027
0028 static struct trace_array *branch_tracer;
0029
0030 static void
0031 probe_likely_condition(struct ftrace_likely_data *f, int val, int expect)
0032 {
0033     struct trace_event_call *call = &event_branch;
0034     struct trace_array *tr = branch_tracer;
0035     struct trace_buffer *buffer;
0036     struct trace_array_cpu *data;
0037     struct ring_buffer_event *event;
0038     struct trace_branch *entry;
0039     unsigned long flags;
0040     unsigned int trace_ctx;
0041     const char *p;
0042
0043     if (current->trace_recursion & TRACE_BRANCH_BIT)
0044         return;
0045
0046     /*
0047      * I would love to save just the ftrace_likely_data pointer, but
0048      * this code can also be used by modules. Ugly things can happen
0049      * if the module is unloaded, and then we go and read the
0050      * pointer.  This is slower, but much safer.
0051      */
0052
0053     if (unlikely(!tr))
0054         return;
0055
0056     raw_local_irq_save(flags);
0057     current->trace_recursion |= TRACE_BRANCH_BIT;
0058     data = this_cpu_ptr(tr->array_buffer.data);
0059     if (atomic_read(&data->disabled))
0060         goto out;
0061
0062     trace_ctx = tracing_gen_ctx_flags(flags);
0063     buffer = tr->array_buffer.buffer;
0064     event = trace_buffer_lock_reserve(buffer, TRACE_BRANCH,
0065                       sizeof(*entry), trace_ctx);
0066     if (!event)
0067         goto out;
0068
0069     entry   = ring_buffer_event_data(event);
0070
0071     /* Strip off the path, only save the file */
0072     p = f->data.file + strlen(f->data.file);
0073     while (p >= f->data.file && *p != '/')
0074         p--;
0075     p++;
0076
0077     strncpy(entry->func, f->data.func, TRACE_FUNC_SIZE);
0078     strncpy(entry->file, p, TRACE_FILE_SIZE);
0079     entry->func[TRACE_FUNC_SIZE] = 0;
0080     entry->file[TRACE_FILE_SIZE] = 0;
0081     entry->constant = f->constant;
0082     entry->line = f->data.line;
0083     entry->correct = val == expect;
0084
0085     if (!call_filter_check_discard(call, entry, buffer, event))
0086         trace_buffer_unlock_commit_nostack(buffer, event);
0087
0088  out:
0089     current->trace_recursion &= ~TRACE_BRANCH_BIT;
0090     raw_local_irq_restore(flags);
0091 }
0092
0093 static inline
0094 void trace_likely_condition(struct ftrace_likely_data *f, int val, int expect)
0095 {
0096     if (!branch_tracing_enabled)
0097         return;
0098
0099     probe_likely_condition(f, val, expect);
0100 }
0101
0102 int enable_branch_tracing(struct trace_array *tr)
0103 {
0104     mutex_lock(&branch_tracing_mutex);
0105     branch_tracer = tr;
0106     /*
0107      * Must be seen before enabling. The reader is a condition
0108      * where we do not need a matching rmb()
0109      */
0110     smp_wmb();
0111     branch_tracing_enabled++;
0112     mutex_unlock(&branch_tracing_mutex);
0113
0114     return 0;
0115 }
0116
0117 void disable_branch_tracing(void)
0118 {
0119     mutex_lock(&branch_tracing_mutex);
0120
0121     if (!branch_tracing_enabled)
0122         goto out_unlock;
0123
0124     branch_tracing_enabled--;
0125
0126  out_unlock:
0127     mutex_unlock(&branch_tracing_mutex);
0128 }
0129
0130 static int branch_trace_init(struct trace_array *tr)
0131 {
0132     return enable_branch_tracing(tr);
0133 }
0134
0135 static void branch_trace_reset(struct trace_array *tr)
0136 {
0137     disable_branch_tracing();
0138 }
0139
0140 static enum print_line_t trace_branch_print(struct trace_iterator *iter,
0141                         int flags, struct trace_event *event)
0142 {
0143     struct trace_branch *field;
0144
0145     trace_assign_type(field, iter->ent);
0146
0147     trace_seq_printf(&iter->seq, "[%s] %s:%s:%d\n",
0148              field->correct ? "  ok  " : " MISS ",
0149              field->func,
0150              field->file,
0151              field->line);
0152
0153     return trace_handle_return(&iter->seq);
0154 }
0155
0156 static void branch_print_header(struct seq_file *s)
0157 {
0158     seq_puts(s, "#           TASK-PID    CPU#    TIMESTAMP  CORRECT"
0159             "  FUNC:FILE:LINE\n"
0160             "#              | |       |          |         |   "
0161             "    |\n");
0162 }
0163
0164 static struct trace_event_functions trace_branch_funcs = {
0165     .trace      = trace_branch_print,
0166 };
0167
0168 static struct trace_event trace_branch_event = {
0169     .type       = TRACE_BRANCH,
0170     .funcs      = &trace_branch_funcs,
0171 };
0172
0173 static struct tracer branch_trace __read_mostly =
0174 {
0175     .name       = "branch",
0176     .init       = branch_trace_init,
0177     .reset      = branch_trace_reset,
0178 #ifdef CONFIG_FTRACE_SELFTEST
0179     .selftest   = trace_selftest_startup_branch,
0180 #endif /* CONFIG_FTRACE_SELFTEST */
0181     .print_header   = branch_print_header,
0182 };
0183
0184 __init static int init_branch_tracer(void)
0185 {
0186     int ret;
0187
0188     ret = register_trace_event(&trace_branch_event);
0189     if (!ret) {
0190         printk(KERN_WARNING "Warning: could not register "
0191                     "branch events\n");
0192         return 1;
0193     }
0194     return register_tracer(&branch_trace);
0195 }
0196 core_initcall(init_branch_tracer);
0197
0198 #else
0199 static inline
0200 void trace_likely_condition(struct ftrace_likely_data *f, int val, int expect)
0201 {
0202 }
0203 #endif /* CONFIG_BRANCH_TRACER */
0204
0205 void ftrace_likely_update(struct ftrace_likely_data *f, int val,
0206               int expect, int is_constant)
0207 {
0208     unsigned long flags = user_access_save();
0209
0210     /* A constant is always correct */
0211     if (is_constant) {
0212         f->constant++;
0213         val = expect;
0214     }
0215     /*
0216      * I would love to have a trace point here instead, but the
0217      * trace point code is so inundated with unlikely and likely
0218      * conditions that the recursive nightmare that exists is too
0219      * much to try to get working. At least for now.
0220      */
0221     trace_likely_condition(f, val, expect);
0222
0223     /* FIXME: Make this atomic! */
0224     if (val == expect)
0225         f->data.correct++;
0226     else
0227         f->data.incorrect++;
0228
0229     user_access_restore(flags);
0230 }
0231 EXPORT_SYMBOL(ftrace_likely_update);
0232
0233 extern unsigned long __start_annotated_branch_profile[];
0234 extern unsigned long __stop_annotated_branch_profile[];
0235
0236 static int annotated_branch_stat_headers(struct seq_file *m)
0237 {
0238     seq_puts(m, " correct incorrect  % "
0239             "       Function                "
0240             "  File              Line\n"
0241             " ------- ---------  - "
0242             "       --------                "
0243             "  ----              ----\n");
0244     return 0;
0245 }
0246
0247 static inline long get_incorrect_percent(const struct ftrace_branch_data *p)
0248 {
0249     long percent;
0250
0251     if (p->correct) {
0252         percent = p->incorrect * 100;
0253         percent /= p->correct + p->incorrect;
0254     } else
0255         percent = p->incorrect ? 100 : -1;
0256
0257     return percent;
0258 }
0259
0260 static const char *branch_stat_process_file(struct ftrace_branch_data *p)
0261 {
0262     const char *f;
0263
0264     /* Only print the file, not the path */
0265     f = p->file + strlen(p->file);
0266     while (f >= p->file && *f != '/')
0267         f--;
0268     return ++f;
0269 }
0270
0271 static void branch_stat_show(struct seq_file *m,
0272                  struct ftrace_branch_data *p, const char *f)
0273 {
0274     long percent;
0275
0276     /*
0277      * The miss is overlayed on correct, and hit on incorrect.
0278      */
0279     percent = get_incorrect_percent(p);
0280
0281     if (percent < 0)
0282         seq_puts(m, "  X ");
0283     else
0284         seq_printf(m, "%3ld ", percent);
0285
0286     seq_printf(m, "%-30.30s %-20.20s %d\n", p->func, f, p->line);
0287 }
0288
0289 static int branch_stat_show_normal(struct seq_file *m,
0290                    struct ftrace_branch_data *p, const char *f)
0291 {
0292     seq_printf(m, "%8lu %8lu ",  p->correct, p->incorrect);
0293     branch_stat_show(m, p, f);
0294     return 0;
0295 }
0296
0297 static int annotate_branch_stat_show(struct seq_file *m, void *v)
0298 {
0299     struct ftrace_likely_data *p = v;
0300     const char *f;
0301     int l;
0302
0303     f = branch_stat_process_file(&p->data);
0304
0305     if (!p->constant)
0306         return branch_stat_show_normal(m, &p->data, f);
0307
0308     l = snprintf(NULL, 0, "/%lu", p->constant);
0309     l = l > 8 ? 0 : 8 - l;
0310
0311     seq_printf(m, "%8lu/%lu %*lu ",
0312            p->data.correct, p->constant, l, p->data.incorrect);
0313     branch_stat_show(m, &p->data, f);
0314     return 0;
0315 }
0316
0317 static void *annotated_branch_stat_start(struct tracer_stat *trace)
0318 {
0319     return __start_annotated_branch_profile;
0320 }
0321
0322 static void *
0323 annotated_branch_stat_next(void *v, int idx)
0324 {
0325     struct ftrace_likely_data *p = v;
0326
0327     ++p;
0328
0329     if ((void *)p >= (void *)__stop_annotated_branch_profile)
0330         return NULL;
0331
0332     return p;
0333 }
0334
0335 static int annotated_branch_stat_cmp(const void *p1, const void *p2)
0336 {
0337     const struct ftrace_branch_data *a = p1;
0338     const struct ftrace_branch_data *b = p2;
0339
0340     long percent_a, percent_b;
0341
0342     percent_a = get_incorrect_percent(a);
0343     percent_b = get_incorrect_percent(b);
0344
0345     if (percent_a < percent_b)
0346         return -1;
0347     if (percent_a > percent_b)
0348         return 1;
0349
0350     if (a->incorrect < b->incorrect)
0351         return -1;
0352     if (a->incorrect > b->incorrect)
0353         return 1;
0354
0355     /*
0356      * Since the above shows worse (incorrect) cases
0357      * first, we continue that by showing best (correct)
0358      * cases last.
0359      */
0360     if (a->correct > b->correct)
0361         return -1;
0362     if (a->correct < b->correct)
0363         return 1;
0364
0365     return 0;
0366 }
0367
0368 static struct tracer_stat annotated_branch_stats = {
0369     .name = "branch_annotated",
0370     .stat_start = annotated_branch_stat_start,
0371     .stat_next = annotated_branch_stat_next,
0372     .stat_cmp = annotated_branch_stat_cmp,
0373     .stat_headers = annotated_branch_stat_headers,
0374     .stat_show = annotate_branch_stat_show
0375 };
0376
0377 __init static int init_annotated_branch_stats(void)
0378 {
0379     int ret;
0380
0381     ret = register_stat_tracer(&annotated_branch_stats);
0382     if (!ret) {
0383         printk(KERN_WARNING "Warning: could not register "
0384                     "annotated branches stats\n");
0385         return 1;
0386     }
0387     return 0;
0388 }
0389 fs_initcall(init_annotated_branch_stats);
0390
0391 #ifdef CONFIG_PROFILE_ALL_BRANCHES
0392
0393 extern unsigned long __start_branch_profile[];
0394 extern unsigned long __stop_branch_profile[];
0395
0396 static int all_branch_stat_headers(struct seq_file *m)
0397 {
0398     seq_puts(m, "   miss      hit    % "
0399             "       Function                "
0400             "  File              Line\n"
0401             " ------- ---------  - "
0402             "       --------                "
0403             "  ----              ----\n");
0404     return 0;
0405 }
0406
0407 static void *all_branch_stat_start(struct tracer_stat *trace)
0408 {
0409     return __start_branch_profile;
0410 }
0411
0412 static void *
0413 all_branch_stat_next(void *v, int idx)
0414 {
0415     struct ftrace_branch_data *p = v;
0416
0417     ++p;
0418
0419     if ((void *)p >= (void *)__stop_branch_profile)
0420         return NULL;
0421
0422     return p;
0423 }
0424
0425 static int all_branch_stat_show(struct seq_file *m, void *v)
0426 {
0427     struct ftrace_branch_data *p = v;
0428     const char *f;
0429
0430     f = branch_stat_process_file(p);
0431     return branch_stat_show_normal(m, p, f);
0432 }
0433
0434 static struct tracer_stat all_branch_stats = {
0435     .name = "branch_all",
0436     .stat_start = all_branch_stat_start,
0437     .stat_next = all_branch_stat_next,
0438     .stat_headers = all_branch_stat_headers,
0439     .stat_show = all_branch_stat_show
0440 };
0441
0442 __init static int all_annotated_branch_stats(void)
0443 {
0444     int ret;
0445
0446     ret = register_stat_tracer(&all_branch_stats);
0447     if (!ret) {
0448         printk(KERN_WARNING "Warning: could not register "
0449                     "all branches stats\n");
0450         return 1;
0451     }
0452     return 0;
0453 }
0454 fs_initcall(all_annotated_branch_stats);
0455 #endif /* CONFIG_PROFILE_ALL_BRANCHES */