perf/tests/builtin-test.c

0001 // SPDX-License-Identifier: GPL-2.0
0002 /*
0003  * builtin-test.c
0004  *
0005  * Builtin regression testing command: ever growing number of sanity tests
0006  */
0007 #include <fcntl.h>
0008 #include <errno.h>
0009 #include <unistd.h>
0010 #include <string.h>
0011 #include <stdlib.h>
0012 #include <sys/types.h>
0013 #include <dirent.h>
0014 #include <sys/wait.h>
0015 #include <sys/stat.h>
0016 #include "builtin.h"
0017 #include "hist.h"
0018 #include "intlist.h"
0019 #include "tests.h"
0020 #include "debug.h"
0021 #include "color.h"
0022 #include <subcmd/parse-options.h>
0023 #include "string2.h"
0024 #include "symbol.h"
0025 #include "util/rlimit.h"
0026 #include <linux/kernel.h>
0027 #include <linux/string.h>
0028 #include <subcmd/exec-cmd.h>
0029 #include <linux/zalloc.h>
0030
0031 #include "builtin-test-list.h"
0032
0033 static bool dont_fork;
0034
0035 struct test_suite *__weak arch_tests[] = {
0036     NULL,
0037 };
0038
0039 static struct test_suite *generic_tests[] = {
0040     &suite__vmlinux_matches_kallsyms,
0041     &suite__openat_syscall_event,
0042     &suite__openat_syscall_event_on_all_cpus,
0043     &suite__basic_mmap,
0044     &suite__mem,
0045     &suite__parse_events,
0046     &suite__expr,
0047     &suite__PERF_RECORD,
0048     &suite__pmu,
0049     &suite__pmu_events,
0050     &suite__dso_data,
0051     &suite__dso_data_cache,
0052     &suite__dso_data_reopen,
0053     &suite__perf_evsel__roundtrip_name_test,
0054     &suite__perf_evsel__tp_sched_test,
0055     &suite__syscall_openat_tp_fields,
0056     &suite__attr,
0057     &suite__hists_link,
0058     &suite__python_use,
0059     &suite__bp_signal,
0060     &suite__bp_signal_overflow,
0061     &suite__bp_accounting,
0062     &suite__wp,
0063     &suite__task_exit,
0064     &suite__sw_clock_freq,
0065     &suite__code_reading,
0066     &suite__sample_parsing,
0067     &suite__keep_tracking,
0068     &suite__parse_no_sample_id_all,
0069     &suite__hists_filter,
0070     &suite__mmap_thread_lookup,
0071     &suite__thread_maps_share,
0072     &suite__hists_output,
0073     &suite__hists_cumulate,
0074     &suite__switch_tracking,
0075     &suite__fdarray__filter,
0076     &suite__fdarray__add,
0077     &suite__kmod_path__parse,
0078     &suite__thread_map,
0079     &suite__llvm,
0080     &suite__session_topology,
0081     &suite__bpf,
0082     &suite__thread_map_synthesize,
0083     &suite__thread_map_remove,
0084     &suite__cpu_map_synthesize,
0085     &suite__synthesize_stat_config,
0086     &suite__synthesize_stat,
0087     &suite__synthesize_stat_round,
0088     &suite__event_update,
0089     &suite__event_times,
0090     &suite__backward_ring_buffer,
0091     &suite__cpu_map_print,
0092     &suite__cpu_map_merge,
0093     &suite__sdt_event,
0094     &suite__is_printable_array,
0095     &suite__bitmap_print,
0096     &suite__perf_hooks,
0097     &suite__clang,
0098     &suite__unit_number__scnprint,
0099     &suite__mem2node,
0100     &suite__time_utils,
0101     &suite__jit_write_elf,
0102     &suite__pfm,
0103     &suite__api_io,
0104     &suite__maps__merge_in,
0105     &suite__demangle_java,
0106     &suite__demangle_ocaml,
0107     &suite__parse_metric,
0108     &suite__pe_file_parsing,
0109     &suite__expand_cgroup_events,
0110     &suite__perf_time_to_tsc,
0111     &suite__dlfilter,
0112     &suite__sigtrap,
0113     NULL,
0114 };
0115
0116 static struct test_suite **tests[] = {
0117     generic_tests,
0118     arch_tests,
0119 };
0120
0121 static int num_subtests(const struct test_suite *t)
0122 {
0123     int num;
0124
0125     if (!t->test_cases)
0126         return 0;
0127
0128     num = 0;
0129     while (t->test_cases[num].name)
0130         num++;
0131
0132     return num;
0133 }
0134
0135 static bool has_subtests(const struct test_suite *t)
0136 {
0137     return num_subtests(t) > 1;
0138 }
0139
0140 static const char *skip_reason(const struct test_suite *t, int subtest)
0141 {
0142     if (!t->test_cases)
0143         return NULL;
0144
0145     return t->test_cases[subtest >= 0 ? subtest : 0].skip_reason;
0146 }
0147
0148 static const char *test_description(const struct test_suite *t, int subtest)
0149 {
0150     if (t->test_cases && subtest >= 0)
0151         return t->test_cases[subtest].desc;
0152
0153     return t->desc;
0154 }
0155
0156 static test_fnptr test_function(const struct test_suite *t, int subtest)
0157 {
0158     if (subtest <= 0)
0159         return t->test_cases[0].run_case;
0160
0161     return t->test_cases[subtest].run_case;
0162 }
0163
0164 static bool perf_test__matches(const char *desc, int curr, int argc, const char *argv[])
0165 {
0166     int i;
0167
0168     if (argc == 0)
0169         return true;
0170
0171     for (i = 0; i < argc; ++i) {
0172         char *end;
0173         long nr = strtoul(argv[i], &end, 10);
0174
0175         if (*end == '\0') {
0176             if (nr == curr + 1)
0177                 return true;
0178             continue;
0179         }
0180
0181         if (strcasestr(desc, argv[i]))
0182             return true;
0183     }
0184
0185     return false;
0186 }
0187
0188 static int run_test(struct test_suite *test, int subtest)
0189 {
0190     int status, err = -1, child = dont_fork ? 0 : fork();
0191     char sbuf[STRERR_BUFSIZE];
0192
0193     if (child < 0) {
0194         pr_err("failed to fork test: %s\n",
0195             str_error_r(errno, sbuf, sizeof(sbuf)));
0196         return -1;
0197     }
0198
0199     if (!child) {
0200         if (!dont_fork) {
0201             pr_debug("test child forked, pid %d\n", getpid());
0202
0203             if (verbose <= 0) {
0204                 int nullfd = open("/dev/null", O_WRONLY);
0205
0206                 if (nullfd >= 0) {
0207                     close(STDERR_FILENO);
0208                     close(STDOUT_FILENO);
0209
0210                     dup2(nullfd, STDOUT_FILENO);
0211                     dup2(STDOUT_FILENO, STDERR_FILENO);
0212                     close(nullfd);
0213                 }
0214             } else {
0215                 signal(SIGSEGV, sighandler_dump_stack);
0216                 signal(SIGFPE, sighandler_dump_stack);
0217             }
0218         }
0219
0220         err = test_function(test, subtest)(test, subtest);
0221         if (!dont_fork)
0222             exit(err);
0223     }
0224
0225     if (!dont_fork) {
0226         wait(&status);
0227
0228         if (WIFEXITED(status)) {
0229             err = (signed char)WEXITSTATUS(status);
0230             pr_debug("test child finished with %d\n", err);
0231         } else if (WIFSIGNALED(status)) {
0232             err = -1;
0233             pr_debug("test child interrupted\n");
0234         }
0235     }
0236
0237     return err;
0238 }
0239
0240 #define for_each_test(j, k, t)          \
0241     for (j = 0; j < ARRAY_SIZE(tests); j++) \
0242         for (k = 0, t = tests[j][k]; tests[j][k]; k++, t = tests[j][k])
0243
0244 static int test_and_print(struct test_suite *t, int subtest)
0245 {
0246     int err;
0247
0248     pr_debug("\n--- start ---\n");
0249     err = run_test(t, subtest);
0250     pr_debug("---- end ----\n");
0251
0252     if (!has_subtests(t))
0253         pr_debug("%s:", t->desc);
0254     else
0255         pr_debug("%s subtest %d:", t->desc, subtest + 1);
0256
0257     switch (err) {
0258     case TEST_OK:
0259         pr_info(" Ok\n");
0260         break;
0261     case TEST_SKIP: {
0262         const char *reason = skip_reason(t, subtest);
0263
0264         if (reason)
0265             color_fprintf(stderr, PERF_COLOR_YELLOW, " Skip (%s)\n", reason);
0266         else
0267             color_fprintf(stderr, PERF_COLOR_YELLOW, " Skip\n");
0268     }
0269         break;
0270     case TEST_FAIL:
0271     default:
0272         color_fprintf(stderr, PERF_COLOR_RED, " FAILED!\n");
0273         break;
0274     }
0275
0276     return err;
0277 }
0278
0279 struct shell_test {
0280     const char *dir;
0281     const char *file;
0282 };
0283
0284 static int shell_test__run(struct test_suite *test, int subdir __maybe_unused)
0285 {
0286     int err;
0287     char script[PATH_MAX];
0288     struct shell_test *st = test->priv;
0289
0290     path__join(script, sizeof(script) - 3, st->dir, st->file);
0291
0292     if (verbose)
0293         strncat(script, " -v", sizeof(script) - strlen(script) - 1);
0294
0295     err = system(script);
0296     if (!err)
0297         return TEST_OK;
0298
0299     return WEXITSTATUS(err) == 2 ? TEST_SKIP : TEST_FAIL;
0300 }
0301
0302 static int run_shell_tests(int argc, const char *argv[], int i, int width,
0303                 struct intlist *skiplist)
0304 {
0305     struct shell_test st;
0306     const struct script_file *files, *file;
0307
0308     files = list_script_files();
0309     if (!files)
0310         return 0;
0311     for (file = files; file->dir; file++) {
0312         int curr = i++;
0313         struct test_case test_cases[] = {
0314             {
0315                 .desc = file->desc,
0316                 .run_case = shell_test__run,
0317             },
0318             { .name = NULL, }
0319         };
0320         struct test_suite test_suite = {
0321             .desc = test_cases[0].desc,
0322             .test_cases = test_cases,
0323             .priv = &st,
0324         };
0325         st.dir = file->dir;
0326
0327         if (test_suite.desc == NULL ||
0328             !perf_test__matches(test_suite.desc, curr, argc, argv))
0329             continue;
0330
0331         st.file = file->file;
0332         pr_info("%3d: %-*s:", i, width, test_suite.desc);
0333
0334         if (intlist__find(skiplist, i)) {
0335             color_fprintf(stderr, PERF_COLOR_YELLOW, " Skip (user override)\n");
0336             continue;
0337         }
0338
0339         test_and_print(&test_suite, 0);
0340     }
0341     return 0;
0342 }
0343
0344 static int __cmd_test(int argc, const char *argv[], struct intlist *skiplist)
0345 {
0346     struct test_suite *t;
0347     unsigned int j, k;
0348     int i = 0;
0349     int width = list_script_max_width();
0350
0351     for_each_test(j, k, t) {
0352         int len = strlen(test_description(t, -1));
0353
0354         if (width < len)
0355             width = len;
0356     }
0357
0358     for_each_test(j, k, t) {
0359         int curr = i++;
0360         int subi;
0361
0362         if (!perf_test__matches(test_description(t, -1), curr, argc, argv)) {
0363             bool skip = true;
0364             int subn;
0365
0366             subn = num_subtests(t);
0367
0368             for (subi = 0; subi < subn; subi++) {
0369                 if (perf_test__matches(test_description(t, subi),
0370                             curr, argc, argv))
0371                     skip = false;
0372             }
0373
0374             if (skip)
0375                 continue;
0376         }
0377
0378         pr_info("%3d: %-*s:", i, width, test_description(t, -1));
0379
0380         if (intlist__find(skiplist, i)) {
0381             color_fprintf(stderr, PERF_COLOR_YELLOW, " Skip (user override)\n");
0382             continue;
0383         }
0384
0385         if (!has_subtests(t)) {
0386             test_and_print(t, -1);
0387         } else {
0388             int subn = num_subtests(t);
0389             /*
0390              * minus 2 to align with normal testcases.
0391              * For subtest we print additional '.x' in number.
0392              * for example:
0393              *
0394              * 35: Test LLVM searching and compiling                        :
0395              * 35.1: Basic BPF llvm compiling test                          : Ok
0396              */
0397             int subw = width > 2 ? width - 2 : width;
0398
0399             if (subn <= 0) {
0400                 color_fprintf(stderr, PERF_COLOR_YELLOW,
0401                           " Skip (not compiled in)\n");
0402                 continue;
0403             }
0404             pr_info("\n");
0405
0406             for (subi = 0; subi < subn; subi++) {
0407                 int len = strlen(test_description(t, subi));
0408
0409                 if (subw < len)
0410                     subw = len;
0411             }
0412
0413             for (subi = 0; subi < subn; subi++) {
0414                 if (!perf_test__matches(test_description(t, subi),
0415                             curr, argc, argv))
0416                     continue;
0417
0418                 pr_info("%3d.%1d: %-*s:", i, subi + 1, subw,
0419                     test_description(t, subi));
0420                 test_and_print(t, subi);
0421             }
0422         }
0423     }
0424
0425     return run_shell_tests(argc, argv, i, width, skiplist);
0426 }
0427
0428 static int perf_test__list_shell(int argc, const char **argv, int i)
0429 {
0430     const struct script_file *files, *file;
0431
0432     files = list_script_files();
0433     if (!files)
0434         return 0;
0435     for (file = files; file->dir; file++) {
0436         int curr = i++;
0437         struct test_suite t = {
0438             .desc = file->desc
0439         };
0440
0441         if (!perf_test__matches(t.desc, curr, argc, argv))
0442             continue;
0443
0444         pr_info("%3d: %s\n", i, t.desc);
0445     }
0446     return 0;
0447 }
0448
0449 static int perf_test__list(int argc, const char **argv)
0450 {
0451     unsigned int j, k;
0452     struct test_suite *t;
0453     int i = 0;
0454
0455     for_each_test(j, k, t) {
0456         int curr = i++;
0457
0458         if (!perf_test__matches(test_description(t, -1), curr, argc, argv))
0459             continue;
0460
0461         pr_info("%3d: %s\n", i, test_description(t, -1));
0462
0463         if (has_subtests(t)) {
0464             int subn = num_subtests(t);
0465             int subi;
0466
0467             for (subi = 0; subi < subn; subi++)
0468                 pr_info("%3d:%1d: %s\n", i, subi + 1,
0469                     test_description(t, subi));
0470         }
0471     }
0472
0473     perf_test__list_shell(argc, argv, i);
0474
0475     return 0;
0476 }
0477
0478 int cmd_test(int argc, const char **argv)
0479 {
0480     const char *test_usage[] = {
0481     "perf test [<options>] [{list <test-name-fragment>|[<test-name-fragments>|<test-numbers>]}]",
0482     NULL,
0483     };
0484     const char *skip = NULL;
0485     const struct option test_options[] = {
0486     OPT_STRING('s', "skip", &skip, "tests", "tests to skip"),
0487     OPT_INCR('v', "verbose", &verbose,
0488             "be more verbose (show symbol address, etc)"),
0489     OPT_BOOLEAN('F', "dont-fork", &dont_fork,
0490             "Do not fork for testcase"),
0491     OPT_END()
0492     };
0493     const char * const test_subcommands[] = { "list", NULL };
0494     struct intlist *skiplist = NULL;
0495         int ret = hists__init();
0496
0497         if (ret < 0)
0498                 return ret;
0499
0500     /* Unbuffered output */
0501     setvbuf(stdout, NULL, _IONBF, 0);
0502
0503     argc = parse_options_subcommand(argc, argv, test_options, test_subcommands, test_usage, 0);
0504     if (argc >= 1 && !strcmp(argv[0], "list"))
0505         return perf_test__list(argc - 1, argv + 1);
0506
0507     symbol_conf.priv_size = sizeof(int);
0508     symbol_conf.sort_by_name = true;
0509     symbol_conf.try_vmlinux_path = true;
0510
0511     if (symbol__init(NULL) < 0)
0512         return -1;
0513
0514     if (skip != NULL)
0515         skiplist = intlist__new(skip);
0516     /*
0517      * Tests that create BPF maps, for instance, need more than the 64K
0518      * default:
0519      */
0520     rlimit__bump_memlock();
0521
0522     return __cmd_test(argc, argv, skiplist);
0523 }