samples/bpf/map_perf_test_user.c

0001 // SPDX-License-Identifier: GPL-2.0-only
0002 /* Copyright (c) 2016 Facebook
0003  */
0004 #define _GNU_SOURCE
0005 #include <sched.h>
0006 #include <stdio.h>
0007 #include <sys/types.h>
0008 #include <asm/unistd.h>
0009 #include <unistd.h>
0010 #include <assert.h>
0011 #include <sys/wait.h>
0012 #include <stdlib.h>
0013 #include <signal.h>
0014 #include <string.h>
0015 #include <time.h>
0016 #include <arpa/inet.h>
0017 #include <errno.h>
0018
0019 #include <bpf/bpf.h>
0020 #include <bpf/libbpf.h>
0021
0022 #define TEST_BIT(t) (1U << (t))
0023 #define MAX_NR_CPUS 1024
0024
0025 static __u64 time_get_ns(void)
0026 {
0027     struct timespec ts;
0028
0029     clock_gettime(CLOCK_MONOTONIC, &ts);
0030     return ts.tv_sec * 1000000000ull + ts.tv_nsec;
0031 }
0032
0033 enum test_type {
0034     HASH_PREALLOC,
0035     PERCPU_HASH_PREALLOC,
0036     HASH_KMALLOC,
0037     PERCPU_HASH_KMALLOC,
0038     LRU_HASH_PREALLOC,
0039     NOCOMMON_LRU_HASH_PREALLOC,
0040     LPM_KMALLOC,
0041     HASH_LOOKUP,
0042     ARRAY_LOOKUP,
0043     INNER_LRU_HASH_PREALLOC,
0044     LRU_HASH_LOOKUP,
0045     NR_TESTS,
0046 };
0047
0048 const char *test_map_names[NR_TESTS] = {
0049     [HASH_PREALLOC] = "hash_map",
0050     [PERCPU_HASH_PREALLOC] = "percpu_hash_map",
0051     [HASH_KMALLOC] = "hash_map_alloc",
0052     [PERCPU_HASH_KMALLOC] = "percpu_hash_map_alloc",
0053     [LRU_HASH_PREALLOC] = "lru_hash_map",
0054     [NOCOMMON_LRU_HASH_PREALLOC] = "nocommon_lru_hash_map",
0055     [LPM_KMALLOC] = "lpm_trie_map_alloc",
0056     [HASH_LOOKUP] = "hash_map",
0057     [ARRAY_LOOKUP] = "array_map",
0058     [INNER_LRU_HASH_PREALLOC] = "inner_lru_hash_map",
0059     [LRU_HASH_LOOKUP] = "lru_hash_lookup_map",
0060 };
0061
0062 enum map_idx {
0063     array_of_lru_hashs_idx,
0064     hash_map_alloc_idx,
0065     lru_hash_lookup_idx,
0066     NR_IDXES,
0067 };
0068
0069 static int map_fd[NR_IDXES];
0070
0071 static int test_flags = ~0;
0072 static uint32_t num_map_entries;
0073 static uint32_t inner_lru_hash_size;
0074 static int lru_hash_lookup_test_entries = 32;
0075 static uint32_t max_cnt = 1000000;
0076
0077 static int check_test_flags(enum test_type t)
0078 {
0079     return test_flags & TEST_BIT(t);
0080 }
0081
0082 static void test_hash_prealloc(int cpu)
0083 {
0084     __u64 start_time;
0085     int i;
0086
0087     start_time = time_get_ns();
0088     for (i = 0; i < max_cnt; i++)
0089         syscall(__NR_getuid);
0090     printf("%d:hash_map_perf pre-alloc %lld events per sec\n",
0091            cpu, max_cnt * 1000000000ll / (time_get_ns() - start_time));
0092 }
0093
0094 static int pre_test_lru_hash_lookup(int tasks)
0095 {
0096     int fd = map_fd[lru_hash_lookup_idx];
0097     uint32_t key;
0098     long val = 1;
0099     int ret;
0100
0101     if (num_map_entries > lru_hash_lookup_test_entries)
0102         lru_hash_lookup_test_entries = num_map_entries;
0103
0104     /* Populate the lru_hash_map for LRU_HASH_LOOKUP perf test.
0105      *
0106      * It is fine that the user requests for a map with
0107      * num_map_entries < 32 and some of the later lru hash lookup
0108      * may return not found.  For LRU map, we are not interested
0109      * in such small map performance.
0110      */
0111     for (key = 0; key < lru_hash_lookup_test_entries; key++) {
0112         ret = bpf_map_update_elem(fd, &key, &val, BPF_NOEXIST);
0113         if (ret)
0114             return ret;
0115     }
0116
0117     return 0;
0118 }
0119
0120 static void do_test_lru(enum test_type test, int cpu)
0121 {
0122     static int inner_lru_map_fds[MAX_NR_CPUS];
0123
0124     struct sockaddr_in6 in6 = { .sin6_family = AF_INET6 };
0125     const char *test_name;
0126     __u64 start_time;
0127     int i, ret;
0128
0129     if (test == INNER_LRU_HASH_PREALLOC && cpu) {
0130         /* If CPU is not 0, create inner_lru hash map and insert the fd
0131          * value into the array_of_lru_hash map. In case of CPU 0,
0132          * 'inner_lru_hash_map' was statically inserted on the map init
0133          */
0134         int outer_fd = map_fd[array_of_lru_hashs_idx];
0135         unsigned int mycpu, mynode;
0136         LIBBPF_OPTS(bpf_map_create_opts, opts,
0137             .map_flags = BPF_F_NUMA_NODE,
0138         );
0139
0140         assert(cpu < MAX_NR_CPUS);
0141
0142         ret = syscall(__NR_getcpu, &mycpu, &mynode, NULL);
0143         assert(!ret);
0144
0145         opts.numa_node = mynode;
0146         inner_lru_map_fds[cpu] =
0147             bpf_map_create(BPF_MAP_TYPE_LRU_HASH,
0148                        test_map_names[INNER_LRU_HASH_PREALLOC],
0149                        sizeof(uint32_t),
0150                        sizeof(long),
0151                        inner_lru_hash_size, &opts);
0152         if (inner_lru_map_fds[cpu] == -1) {
0153             printf("cannot create BPF_MAP_TYPE_LRU_HASH %s(%d)\n",
0154                    strerror(errno), errno);
0155             exit(1);
0156         }
0157
0158         ret = bpf_map_update_elem(outer_fd, &cpu,
0159                       &inner_lru_map_fds[cpu],
0160                       BPF_ANY);
0161         if (ret) {
0162             printf("cannot update ARRAY_OF_LRU_HASHS with key:%u. %s(%d)\n",
0163                    cpu, strerror(errno), errno);
0164             exit(1);
0165         }
0166     }
0167
0168     in6.sin6_addr.s6_addr16[0] = 0xdead;
0169     in6.sin6_addr.s6_addr16[1] = 0xbeef;
0170
0171     if (test == LRU_HASH_PREALLOC) {
0172         test_name = "lru_hash_map_perf";
0173         in6.sin6_addr.s6_addr16[2] = 0;
0174     } else if (test == NOCOMMON_LRU_HASH_PREALLOC) {
0175         test_name = "nocommon_lru_hash_map_perf";
0176         in6.sin6_addr.s6_addr16[2] = 1;
0177     } else if (test == INNER_LRU_HASH_PREALLOC) {
0178         test_name = "inner_lru_hash_map_perf";
0179         in6.sin6_addr.s6_addr16[2] = 2;
0180     } else if (test == LRU_HASH_LOOKUP) {
0181         test_name = "lru_hash_lookup_perf";
0182         in6.sin6_addr.s6_addr16[2] = 3;
0183         in6.sin6_addr.s6_addr32[3] = 0;
0184     } else {
0185         assert(0);
0186     }
0187
0188     start_time = time_get_ns();
0189     for (i = 0; i < max_cnt; i++) {
0190         ret = connect(-1, (const struct sockaddr *)&in6, sizeof(in6));
0191         assert(ret == -1 && errno == EBADF);
0192         if (in6.sin6_addr.s6_addr32[3] <
0193             lru_hash_lookup_test_entries - 32)
0194             in6.sin6_addr.s6_addr32[3] += 32;
0195         else
0196             in6.sin6_addr.s6_addr32[3] = 0;
0197     }
0198     printf("%d:%s pre-alloc %lld events per sec\n",
0199            cpu, test_name,
0200            max_cnt * 1000000000ll / (time_get_ns() - start_time));
0201 }
0202
0203 static void test_lru_hash_prealloc(int cpu)
0204 {
0205     do_test_lru(LRU_HASH_PREALLOC, cpu);
0206 }
0207
0208 static void test_nocommon_lru_hash_prealloc(int cpu)
0209 {
0210     do_test_lru(NOCOMMON_LRU_HASH_PREALLOC, cpu);
0211 }
0212
0213 static void test_inner_lru_hash_prealloc(int cpu)
0214 {
0215     do_test_lru(INNER_LRU_HASH_PREALLOC, cpu);
0216 }
0217
0218 static void test_lru_hash_lookup(int cpu)
0219 {
0220     do_test_lru(LRU_HASH_LOOKUP, cpu);
0221 }
0222
0223 static void test_percpu_hash_prealloc(int cpu)
0224 {
0225     __u64 start_time;
0226     int i;
0227
0228     start_time = time_get_ns();
0229     for (i = 0; i < max_cnt; i++)
0230         syscall(__NR_geteuid);
0231     printf("%d:percpu_hash_map_perf pre-alloc %lld events per sec\n",
0232            cpu, max_cnt * 1000000000ll / (time_get_ns() - start_time));
0233 }
0234
0235 static void test_hash_kmalloc(int cpu)
0236 {
0237     __u64 start_time;
0238     int i;
0239
0240     start_time = time_get_ns();
0241     for (i = 0; i < max_cnt; i++)
0242         syscall(__NR_getgid);
0243     printf("%d:hash_map_perf kmalloc %lld events per sec\n",
0244            cpu, max_cnt * 1000000000ll / (time_get_ns() - start_time));
0245 }
0246
0247 static void test_percpu_hash_kmalloc(int cpu)
0248 {
0249     __u64 start_time;
0250     int i;
0251
0252     start_time = time_get_ns();
0253     for (i = 0; i < max_cnt; i++)
0254         syscall(__NR_getegid);
0255     printf("%d:percpu_hash_map_perf kmalloc %lld events per sec\n",
0256            cpu, max_cnt * 1000000000ll / (time_get_ns() - start_time));
0257 }
0258
0259 static void test_lpm_kmalloc(int cpu)
0260 {
0261     __u64 start_time;
0262     int i;
0263
0264     start_time = time_get_ns();
0265     for (i = 0; i < max_cnt; i++)
0266         syscall(__NR_gettid);
0267     printf("%d:lpm_perf kmalloc %lld events per sec\n",
0268            cpu, max_cnt * 1000000000ll / (time_get_ns() - start_time));
0269 }
0270
0271 static void test_hash_lookup(int cpu)
0272 {
0273     __u64 start_time;
0274     int i;
0275
0276     start_time = time_get_ns();
0277     for (i = 0; i < max_cnt; i++)
0278         syscall(__NR_getpgid, 0);
0279     printf("%d:hash_lookup %lld lookups per sec\n",
0280            cpu, max_cnt * 1000000000ll * 64 / (time_get_ns() - start_time));
0281 }
0282
0283 static void test_array_lookup(int cpu)
0284 {
0285     __u64 start_time;
0286     int i;
0287
0288     start_time = time_get_ns();
0289     for (i = 0; i < max_cnt; i++)
0290         syscall(__NR_getppid, 0);
0291     printf("%d:array_lookup %lld lookups per sec\n",
0292            cpu, max_cnt * 1000000000ll * 64 / (time_get_ns() - start_time));
0293 }
0294
0295 typedef int (*pre_test_func)(int tasks);
0296 const pre_test_func pre_test_funcs[] = {
0297     [LRU_HASH_LOOKUP] = pre_test_lru_hash_lookup,
0298 };
0299
0300 typedef void (*test_func)(int cpu);
0301 const test_func test_funcs[] = {
0302     [HASH_PREALLOC] = test_hash_prealloc,
0303     [PERCPU_HASH_PREALLOC] = test_percpu_hash_prealloc,
0304     [HASH_KMALLOC] = test_hash_kmalloc,
0305     [PERCPU_HASH_KMALLOC] = test_percpu_hash_kmalloc,
0306     [LRU_HASH_PREALLOC] = test_lru_hash_prealloc,
0307     [NOCOMMON_LRU_HASH_PREALLOC] = test_nocommon_lru_hash_prealloc,
0308     [LPM_KMALLOC] = test_lpm_kmalloc,
0309     [HASH_LOOKUP] = test_hash_lookup,
0310     [ARRAY_LOOKUP] = test_array_lookup,
0311     [INNER_LRU_HASH_PREALLOC] = test_inner_lru_hash_prealloc,
0312     [LRU_HASH_LOOKUP] = test_lru_hash_lookup,
0313 };
0314
0315 static int pre_test(int tasks)
0316 {
0317     int i;
0318
0319     for (i = 0; i < NR_TESTS; i++) {
0320         if (pre_test_funcs[i] && check_test_flags(i)) {
0321             int ret = pre_test_funcs[i](tasks);
0322
0323             if (ret)
0324                 return ret;
0325         }
0326     }
0327
0328     return 0;
0329 }
0330
0331 static void loop(int cpu)
0332 {
0333     cpu_set_t cpuset;
0334     int i;
0335
0336     CPU_ZERO(&cpuset);
0337     CPU_SET(cpu, &cpuset);
0338     sched_setaffinity(0, sizeof(cpuset), &cpuset);
0339
0340     for (i = 0; i < NR_TESTS; i++) {
0341         if (check_test_flags(i))
0342             test_funcs[i](cpu);
0343     }
0344 }
0345
0346 static void run_perf_test(int tasks)
0347 {
0348     pid_t pid[tasks];
0349     int i;
0350
0351     assert(!pre_test(tasks));
0352
0353     for (i = 0; i < tasks; i++) {
0354         pid[i] = fork();
0355         if (pid[i] == 0) {
0356             loop(i);
0357             exit(0);
0358         } else if (pid[i] == -1) {
0359             printf("couldn't spawn #%d process\n", i);
0360             exit(1);
0361         }
0362     }
0363     for (i = 0; i < tasks; i++) {
0364         int status;
0365
0366         assert(waitpid(pid[i], &status, 0) == pid[i]);
0367         assert(status == 0);
0368     }
0369 }
0370
0371 static void fill_lpm_trie(void)
0372 {
0373     struct bpf_lpm_trie_key *key;
0374     unsigned long value = 0;
0375     unsigned int i;
0376     int r;
0377
0378     key = alloca(sizeof(*key) + 4);
0379     key->prefixlen = 32;
0380
0381     for (i = 0; i < 512; ++i) {
0382         key->prefixlen = rand() % 33;
0383         key->data[0] = rand() & 0xff;
0384         key->data[1] = rand() & 0xff;
0385         key->data[2] = rand() & 0xff;
0386         key->data[3] = rand() & 0xff;
0387         r = bpf_map_update_elem(map_fd[hash_map_alloc_idx],
0388                     key, &value, 0);
0389         assert(!r);
0390     }
0391
0392     key->prefixlen = 32;
0393     key->data[0] = 192;
0394     key->data[1] = 168;
0395     key->data[2] = 0;
0396     key->data[3] = 1;
0397     value = 128;
0398
0399     r = bpf_map_update_elem(map_fd[hash_map_alloc_idx], key, &value, 0);
0400     assert(!r);
0401 }
0402
0403 static void fixup_map(struct bpf_object *obj)
0404 {
0405     struct bpf_map *map;
0406     int i;
0407
0408     bpf_object__for_each_map(map, obj) {
0409         const char *name = bpf_map__name(map);
0410
0411         /* Only change the max_entries for the enabled test(s) */
0412         for (i = 0; i < NR_TESTS; i++) {
0413             if (!strcmp(test_map_names[i], name) &&
0414                 (check_test_flags(i))) {
0415                 bpf_map__set_max_entries(map, num_map_entries);
0416                 continue;
0417             }
0418         }
0419     }
0420
0421     inner_lru_hash_size = num_map_entries;
0422 }
0423
0424 int main(int argc, char **argv)
0425 {
0426     int nr_cpus = sysconf(_SC_NPROCESSORS_ONLN);
0427     struct bpf_link *links[8];
0428     struct bpf_program *prog;
0429     struct bpf_object *obj;
0430     struct bpf_map *map;
0431     char filename[256];
0432     int i = 0;
0433
0434     if (argc > 1)
0435         test_flags = atoi(argv[1]) ? : test_flags;
0436
0437     if (argc > 2)
0438         nr_cpus = atoi(argv[2]) ? : nr_cpus;
0439
0440     if (argc > 3)
0441         num_map_entries = atoi(argv[3]);
0442
0443     if (argc > 4)
0444         max_cnt = atoi(argv[4]);
0445
0446     snprintf(filename, sizeof(filename), "%s_kern.o", argv[0]);
0447     obj = bpf_object__open_file(filename, NULL);
0448     if (libbpf_get_error(obj)) {
0449         fprintf(stderr, "ERROR: opening BPF object file failed\n");
0450         return 0;
0451     }
0452
0453     map = bpf_object__find_map_by_name(obj, "inner_lru_hash_map");
0454     if (libbpf_get_error(map)) {
0455         fprintf(stderr, "ERROR: finding a map in obj file failed\n");
0456         goto cleanup;
0457     }
0458
0459     inner_lru_hash_size = bpf_map__max_entries(map);
0460     if (!inner_lru_hash_size) {
0461         fprintf(stderr, "ERROR: failed to get map attribute\n");
0462         goto cleanup;
0463     }
0464
0465     /* resize BPF map prior to loading */
0466     if (num_map_entries > 0)
0467         fixup_map(obj);
0468
0469     /* load BPF program */
0470     if (bpf_object__load(obj)) {
0471         fprintf(stderr, "ERROR: loading BPF object file failed\n");
0472         goto cleanup;
0473     }
0474
0475     map_fd[0] = bpf_object__find_map_fd_by_name(obj, "array_of_lru_hashs");
0476     map_fd[1] = bpf_object__find_map_fd_by_name(obj, "hash_map_alloc");
0477     map_fd[2] = bpf_object__find_map_fd_by_name(obj, "lru_hash_lookup_map");
0478     if (map_fd[0] < 0 || map_fd[1] < 0 || map_fd[2] < 0) {
0479         fprintf(stderr, "ERROR: finding a map in obj file failed\n");
0480         goto cleanup;
0481     }
0482
0483     bpf_object__for_each_program(prog, obj) {
0484         links[i] = bpf_program__attach(prog);
0485         if (libbpf_get_error(links[i])) {
0486             fprintf(stderr, "ERROR: bpf_program__attach failed\n");
0487             links[i] = NULL;
0488             goto cleanup;
0489         }
0490         i++;
0491     }
0492
0493     fill_lpm_trie();
0494
0495     run_perf_test(nr_cpus);
0496
0497 cleanup:
0498     for (i--; i >= 0; i--)
0499         bpf_link__destroy(links[i]);
0500
0501     bpf_object__close(obj);
0502     return 0;
0503 }