bpf/benchs/bench_local_storage.c

0001 // SPDX-License-Identifier: GPL-2.0
0002 /* Copyright (c) 2022 Meta Platforms, Inc. and affiliates. */
0003
0004 #include <argp.h>
0005 #include <linux/btf.h>
0006
0007 #include "local_storage_bench.skel.h"
0008 #include "bench.h"
0009
0010 #include <test_btf.h>
0011
0012 static struct {
0013     __u32 nr_maps;
0014     __u32 hashmap_nr_keys_used;
0015 } args = {
0016     .nr_maps = 1000,
0017     .hashmap_nr_keys_used = 1000,
0018 };
0019
0020 enum {
0021     ARG_NR_MAPS = 6000,
0022     ARG_HASHMAP_NR_KEYS_USED = 6001,
0023 };
0024
0025 static const struct argp_option opts[] = {
0026     { "nr_maps", ARG_NR_MAPS, "NR_MAPS", 0,
0027         "Set number of local_storage maps"},
0028     { "hashmap_nr_keys_used", ARG_HASHMAP_NR_KEYS_USED, "NR_KEYS",
0029         0, "When doing hashmap test, set number of hashmap keys test uses"},
0030     {},
0031 };
0032
0033 static error_t parse_arg(int key, char *arg, struct argp_state *state)
0034 {
0035     long ret;
0036
0037     switch (key) {
0038     case ARG_NR_MAPS:
0039         ret = strtol(arg, NULL, 10);
0040         if (ret < 1 || ret > UINT_MAX) {
0041             fprintf(stderr, "invalid nr_maps");
0042             argp_usage(state);
0043         }
0044         args.nr_maps = ret;
0045         break;
0046     case ARG_HASHMAP_NR_KEYS_USED:
0047         ret = strtol(arg, NULL, 10);
0048         if (ret < 1 || ret > UINT_MAX) {
0049             fprintf(stderr, "invalid hashmap_nr_keys_used");
0050             argp_usage(state);
0051         }
0052         args.hashmap_nr_keys_used = ret;
0053         break;
0054     default:
0055         return ARGP_ERR_UNKNOWN;
0056     }
0057
0058     return 0;
0059 }
0060
0061 const struct argp bench_local_storage_argp = {
0062     .options = opts,
0063     .parser = parse_arg,
0064 };
0065
0066 /* Keep in sync w/ array of maps in bpf */
0067 #define MAX_NR_MAPS 1000
0068 /* keep in sync w/ same define in bpf */
0069 #define HASHMAP_SZ 4194304
0070
0071 static void validate(void)
0072 {
0073     if (env.producer_cnt != 1) {
0074         fprintf(stderr, "benchmark doesn't support multi-producer!\n");
0075         exit(1);
0076     }
0077     if (env.consumer_cnt != 1) {
0078         fprintf(stderr, "benchmark doesn't support multi-consumer!\n");
0079         exit(1);
0080     }
0081
0082     if (args.nr_maps > MAX_NR_MAPS) {
0083         fprintf(stderr, "nr_maps must be <= 1000\n");
0084         exit(1);
0085     }
0086
0087     if (args.hashmap_nr_keys_used > HASHMAP_SZ) {
0088         fprintf(stderr, "hashmap_nr_keys_used must be <= %u\n", HASHMAP_SZ);
0089         exit(1);
0090     }
0091 }
0092
0093 static struct {
0094     struct local_storage_bench *skel;
0095     void *bpf_obj;
0096     struct bpf_map *array_of_maps;
0097 } ctx;
0098
0099 static void prepopulate_hashmap(int fd)
0100 {
0101     int i, key, val;
0102
0103     /* local_storage gets will have BPF_LOCAL_STORAGE_GET_F_CREATE flag set, so
0104      * populate the hashmap for a similar comparison
0105      */
0106     for (i = 0; i < HASHMAP_SZ; i++) {
0107         key = val = i;
0108         if (bpf_map_update_elem(fd, &key, &val, 0)) {
0109             fprintf(stderr, "Error prepopulating hashmap (key %d)\n", key);
0110             exit(1);
0111         }
0112     }
0113 }
0114
0115 static void __setup(struct bpf_program *prog, bool hashmap)
0116 {
0117     struct bpf_map *inner_map;
0118     int i, fd, mim_fd, err;
0119
0120     LIBBPF_OPTS(bpf_map_create_opts, create_opts);
0121
0122     if (!hashmap)
0123         create_opts.map_flags = BPF_F_NO_PREALLOC;
0124
0125     ctx.skel->rodata->num_maps = args.nr_maps;
0126     ctx.skel->rodata->hashmap_num_keys = args.hashmap_nr_keys_used;
0127     inner_map = bpf_map__inner_map(ctx.array_of_maps);
0128     create_opts.btf_key_type_id = bpf_map__btf_key_type_id(inner_map);
0129     create_opts.btf_value_type_id = bpf_map__btf_value_type_id(inner_map);
0130
0131     err = local_storage_bench__load(ctx.skel);
0132     if (err) {
0133         fprintf(stderr, "Error loading skeleton\n");
0134         goto err_out;
0135     }
0136
0137     create_opts.btf_fd = bpf_object__btf_fd(ctx.skel->obj);
0138
0139     mim_fd = bpf_map__fd(ctx.array_of_maps);
0140     if (mim_fd < 0) {
0141         fprintf(stderr, "Error getting map_in_map fd\n");
0142         goto err_out;
0143     }
0144
0145     for (i = 0; i < args.nr_maps; i++) {
0146         if (hashmap)
0147             fd = bpf_map_create(BPF_MAP_TYPE_HASH, NULL, sizeof(int),
0148                         sizeof(int), HASHMAP_SZ, &create_opts);
0149         else
0150             fd = bpf_map_create(BPF_MAP_TYPE_TASK_STORAGE, NULL, sizeof(int),
0151                         sizeof(int), 0, &create_opts);
0152         if (fd < 0) {
0153             fprintf(stderr, "Error creating map %d: %d\n", i, fd);
0154             goto err_out;
0155         }
0156
0157         if (hashmap)
0158             prepopulate_hashmap(fd);
0159
0160         err = bpf_map_update_elem(mim_fd, &i, &fd, 0);
0161         if (err) {
0162             fprintf(stderr, "Error updating array-of-maps w/ map %d\n", i);
0163             goto err_out;
0164         }
0165     }
0166
0167     if (!bpf_program__attach(prog)) {
0168         fprintf(stderr, "Error attaching bpf program\n");
0169         goto err_out;
0170     }
0171
0172     return;
0173 err_out:
0174     exit(1);
0175 }
0176
0177 static void hashmap_setup(void)
0178 {
0179     struct local_storage_bench *skel;
0180
0181     setup_libbpf();
0182
0183     skel = local_storage_bench__open();
0184     ctx.skel = skel;
0185     ctx.array_of_maps = skel->maps.array_of_hash_maps;
0186     skel->rodata->use_hashmap = 1;
0187     skel->rodata->interleave = 0;
0188
0189     __setup(skel->progs.get_local, true);
0190 }
0191
0192 static void local_storage_cache_get_setup(void)
0193 {
0194     struct local_storage_bench *skel;
0195
0196     setup_libbpf();
0197
0198     skel = local_storage_bench__open();
0199     ctx.skel = skel;
0200     ctx.array_of_maps = skel->maps.array_of_local_storage_maps;
0201     skel->rodata->use_hashmap = 0;
0202     skel->rodata->interleave = 0;
0203
0204     __setup(skel->progs.get_local, false);
0205 }
0206
0207 static void local_storage_cache_get_interleaved_setup(void)
0208 {
0209     struct local_storage_bench *skel;
0210
0211     setup_libbpf();
0212
0213     skel = local_storage_bench__open();
0214     ctx.skel = skel;
0215     ctx.array_of_maps = skel->maps.array_of_local_storage_maps;
0216     skel->rodata->use_hashmap = 0;
0217     skel->rodata->interleave = 1;
0218
0219     __setup(skel->progs.get_local, false);
0220 }
0221
0222 static void measure(struct bench_res *res)
0223 {
0224     res->hits = atomic_swap(&ctx.skel->bss->hits, 0);
0225     res->important_hits = atomic_swap(&ctx.skel->bss->important_hits, 0);
0226 }
0227
0228 static inline void trigger_bpf_program(void)
0229 {
0230     syscall(__NR_getpgid);
0231 }
0232
0233 static void *consumer(void *input)
0234 {
0235     return NULL;
0236 }
0237
0238 static void *producer(void *input)
0239 {
0240     while (true)
0241         trigger_bpf_program();
0242
0243     return NULL;
0244 }
0245
0246 /* cache sequential and interleaved get benchs test local_storage get
0247  * performance, specifically they demonstrate performance cliff of
0248  * current list-plus-cache local_storage model.
0249  *
0250  * cache sequential get: call bpf_task_storage_get on n maps in order
0251  * cache interleaved get: like "sequential get", but interleave 4 calls to the
0252  *  'important' map (idx 0 in array_of_maps) for every 10 calls. Goal
0253  *  is to mimic environment where many progs are accessing their local_storage
0254  *  maps, with 'our' prog needing to access its map more often than others
0255  */
0256 const struct bench bench_local_storage_cache_seq_get = {
0257     .name = "local-storage-cache-seq-get",
0258     .validate = validate,
0259     .setup = local_storage_cache_get_setup,
0260     .producer_thread = producer,
0261     .consumer_thread = consumer,
0262     .measure = measure,
0263     .report_progress = local_storage_report_progress,
0264     .report_final = local_storage_report_final,
0265 };
0266
0267 const struct bench bench_local_storage_cache_interleaved_get = {
0268     .name = "local-storage-cache-int-get",
0269     .validate = validate,
0270     .setup = local_storage_cache_get_interleaved_setup,
0271     .producer_thread = producer,
0272     .consumer_thread = consumer,
0273     .measure = measure,
0274     .report_progress = local_storage_report_progress,
0275     .report_final = local_storage_report_final,
0276 };
0277
0278 const struct bench bench_local_storage_cache_hashmap_control = {
0279     .name = "local-storage-cache-hashmap-control",
0280     .validate = validate,
0281     .setup = hashmap_setup,
0282     .producer_thread = producer,
0283     .consumer_thread = consumer,
0284     .measure = measure,
0285     .report_progress = local_storage_report_progress,
0286     .report_final = local_storage_report_final,
0287 };