samples/bpf/xdp_redirect_cpu_user.c

0001 // SPDX-License-Identifier: GPL-2.0-only
0002 /* Copyright(c) 2017 Jesper Dangaard Brouer, Red Hat, Inc.
0003  */
0004 static const char *__doc__ =
0005 "XDP CPU redirect tool, using BPF_MAP_TYPE_CPUMAP\n"
0006 "Usage: xdp_redirect_cpu -d <IFINDEX|IFNAME> -c 0 ... -c N\n"
0007 "Valid specification for CPUMAP BPF program:\n"
0008 "  --mprog-name/-e pass (use built-in XDP_PASS program)\n"
0009 "  --mprog-name/-e drop (use built-in XDP_DROP program)\n"
0010 "  --redirect-device/-r <ifindex|ifname> (use built-in DEVMAP redirect program)\n"
0011 "  Custom CPUMAP BPF program:\n"
0012 "    --mprog-filename/-f <filename> --mprog-name/-e <program>\n"
0013 "    Optionally, also pass --redirect-map/-m and --redirect-device/-r together\n"
0014 "    to configure DEVMAP in BPF object <filename>\n";
0015
0016 #include <errno.h>
0017 #include <signal.h>
0018 #include <stdio.h>
0019 #include <stdlib.h>
0020 #include <stdbool.h>
0021 #include <string.h>
0022 #include <unistd.h>
0023 #include <locale.h>
0024 #include <sys/sysinfo.h>
0025 #include <getopt.h>
0026 #include <net/if.h>
0027 #include <time.h>
0028 #include <linux/limits.h>
0029 #include <arpa/inet.h>
0030 #include <linux/if_link.h>
0031 #include <bpf/bpf.h>
0032 #include <bpf/libbpf.h>
0033 #include "bpf_util.h"
0034 #include "xdp_sample_user.h"
0035 #include "xdp_redirect_cpu.skel.h"
0036
0037 static int map_fd;
0038 static int avail_fd;
0039 static int count_fd;
0040
0041 static int mask = SAMPLE_RX_CNT | SAMPLE_REDIRECT_ERR_MAP_CNT |
0042           SAMPLE_CPUMAP_ENQUEUE_CNT | SAMPLE_CPUMAP_KTHREAD_CNT |
0043           SAMPLE_EXCEPTION_CNT;
0044
0045 DEFINE_SAMPLE_INIT(xdp_redirect_cpu);
0046
0047 static const struct option long_options[] = {
0048     { "help", no_argument, NULL, 'h' },
0049     { "dev", required_argument, NULL, 'd' },
0050     { "skb-mode", no_argument, NULL, 'S' },
0051     { "progname", required_argument, NULL, 'p' },
0052     { "qsize", required_argument, NULL, 'q' },
0053     { "cpu", required_argument, NULL, 'c' },
0054     { "stress-mode", no_argument, NULL, 'x' },
0055     { "force", no_argument, NULL, 'F' },
0056     { "interval", required_argument, NULL, 'i' },
0057     { "verbose", no_argument, NULL, 'v' },
0058     { "stats", no_argument, NULL, 's' },
0059     { "mprog-name", required_argument, NULL, 'e' },
0060     { "mprog-filename", required_argument, NULL, 'f' },
0061     { "redirect-device", required_argument, NULL, 'r' },
0062     { "redirect-map", required_argument, NULL, 'm' },
0063     {}
0064 };
0065
0066 static void print_avail_progs(struct bpf_object *obj)
0067 {
0068     struct bpf_program *pos;
0069
0070     printf(" Programs to be used for -p/--progname:\n");
0071     bpf_object__for_each_program(pos, obj) {
0072         if (bpf_program__type(pos) == BPF_PROG_TYPE_XDP) {
0073             if (!strncmp(bpf_program__name(pos), "xdp_prognum",
0074                      sizeof("xdp_prognum") - 1))
0075                 printf(" %s\n", bpf_program__name(pos));
0076         }
0077     }
0078 }
0079
0080 static void usage(char *argv[], const struct option *long_options,
0081           const char *doc, int mask, bool error, struct bpf_object *obj)
0082 {
0083     sample_usage(argv, long_options, doc, mask, error);
0084     print_avail_progs(obj);
0085 }
0086
0087 static int create_cpu_entry(__u32 cpu, struct bpf_cpumap_val *value,
0088                 __u32 avail_idx, bool new)
0089 {
0090     __u32 curr_cpus_count = 0;
0091     __u32 key = 0;
0092     int ret;
0093
0094     /* Add a CPU entry to cpumap, as this allocate a cpu entry in
0095      * the kernel for the cpu.
0096      */
0097     ret = bpf_map_update_elem(map_fd, &cpu, value, 0);
0098     if (ret < 0) {
0099         fprintf(stderr, "Create CPU entry failed: %s\n", strerror(errno));
0100         return ret;
0101     }
0102
0103     /* Inform bpf_prog's that a new CPU is available to select
0104      * from via some control maps.
0105      */
0106     ret = bpf_map_update_elem(avail_fd, &avail_idx, &cpu, 0);
0107     if (ret < 0) {
0108         fprintf(stderr, "Add to avail CPUs failed: %s\n", strerror(errno));
0109         return ret;
0110     }
0111
0112     /* When not replacing/updating existing entry, bump the count */
0113     ret = bpf_map_lookup_elem(count_fd, &key, &curr_cpus_count);
0114     if (ret < 0) {
0115         fprintf(stderr, "Failed reading curr cpus_count: %s\n",
0116             strerror(errno));
0117         return ret;
0118     }
0119     if (new) {
0120         curr_cpus_count++;
0121         ret = bpf_map_update_elem(count_fd, &key,
0122                       &curr_cpus_count, 0);
0123         if (ret < 0) {
0124             fprintf(stderr, "Failed write curr cpus_count: %s\n",
0125                 strerror(errno));
0126             return ret;
0127         }
0128     }
0129
0130     printf("%s CPU: %u as idx: %u qsize: %d cpumap_prog_fd: %d (cpus_count: %u)\n",
0131            new ? "Add new" : "Replace", cpu, avail_idx,
0132            value->qsize, value->bpf_prog.fd, curr_cpus_count);
0133
0134     return 0;
0135 }
0136
0137 /* CPUs are zero-indexed. Thus, add a special sentinel default value
0138  * in map cpus_available to mark CPU index'es not configured
0139  */
0140 static int mark_cpus_unavailable(void)
0141 {
0142     int ret, i, n_cpus = libbpf_num_possible_cpus();
0143     __u32 invalid_cpu = n_cpus;
0144
0145     for (i = 0; i < n_cpus; i++) {
0146         ret = bpf_map_update_elem(avail_fd, &i,
0147                       &invalid_cpu, 0);
0148         if (ret < 0) {
0149             fprintf(stderr, "Failed marking CPU unavailable: %s\n",
0150                 strerror(errno));
0151             return ret;
0152         }
0153     }
0154
0155     return 0;
0156 }
0157
0158 /* Stress cpumap management code by concurrently changing underlying cpumap */
0159 static void stress_cpumap(void *ctx)
0160 {
0161     struct bpf_cpumap_val *value = ctx;
0162
0163     /* Changing qsize will cause kernel to free and alloc a new
0164      * bpf_cpu_map_entry, with an associated/complicated tear-down
0165      * procedure.
0166      */
0167     value->qsize = 1024;
0168     create_cpu_entry(1, value, 0, false);
0169     value->qsize = 8;
0170     create_cpu_entry(1, value, 0, false);
0171     value->qsize = 16000;
0172     create_cpu_entry(1, value, 0, false);
0173 }
0174
0175 static int set_cpumap_prog(struct xdp_redirect_cpu *skel,
0176                const char *redir_interface, const char *redir_map,
0177                const char *mprog_filename, const char *mprog_name)
0178 {
0179     if (mprog_filename) {
0180         struct bpf_program *prog;
0181         struct bpf_object *obj;
0182         int ret;
0183
0184         if (!mprog_name) {
0185             fprintf(stderr, "BPF program not specified for file %s\n",
0186                 mprog_filename);
0187             goto end;
0188         }
0189         if ((redir_interface && !redir_map) || (!redir_interface && redir_map)) {
0190             fprintf(stderr, "--redirect-%s specified but --redirect-%s not specified\n",
0191                 redir_interface ? "device" : "map", redir_interface ? "map" : "device");
0192             goto end;
0193         }
0194
0195         /* Custom BPF program */
0196         obj = bpf_object__open_file(mprog_filename, NULL);
0197         if (!obj) {
0198             ret = -errno;
0199             fprintf(stderr, "Failed to bpf_prog_load_xattr: %s\n",
0200                 strerror(errno));
0201             return ret;
0202         }
0203
0204         ret = bpf_object__load(obj);
0205         if (ret < 0) {
0206             ret = -errno;
0207             fprintf(stderr, "Failed to bpf_object__load: %s\n",
0208                 strerror(errno));
0209             return ret;
0210         }
0211
0212         if (redir_map) {
0213             int err, redir_map_fd, ifindex_out, key = 0;
0214
0215             redir_map_fd = bpf_object__find_map_fd_by_name(obj, redir_map);
0216             if (redir_map_fd < 0) {
0217                 fprintf(stderr, "Failed to bpf_object__find_map_fd_by_name: %s\n",
0218                     strerror(errno));
0219                 return redir_map_fd;
0220             }
0221
0222             ifindex_out = if_nametoindex(redir_interface);
0223             if (!ifindex_out)
0224                 ifindex_out = strtoul(redir_interface, NULL, 0);
0225             if (!ifindex_out) {
0226                 fprintf(stderr, "Bad interface name or index\n");
0227                 return -EINVAL;
0228             }
0229
0230             err = bpf_map_update_elem(redir_map_fd, &key, &ifindex_out, 0);
0231             if (err < 0)
0232                 return err;
0233         }
0234
0235         prog = bpf_object__find_program_by_name(obj, mprog_name);
0236         if (!prog) {
0237             ret = -errno;
0238             fprintf(stderr, "Failed to bpf_object__find_program_by_name: %s\n",
0239                 strerror(errno));
0240             return ret;
0241         }
0242
0243         return bpf_program__fd(prog);
0244     } else {
0245         if (mprog_name) {
0246             if (redir_interface || redir_map) {
0247                 fprintf(stderr, "Need to specify --mprog-filename/-f\n");
0248                 goto end;
0249             }
0250             if (!strcmp(mprog_name, "pass") || !strcmp(mprog_name, "drop")) {
0251                 /* Use built-in pass/drop programs */
0252                 return *mprog_name == 'p' ? bpf_program__fd(skel->progs.xdp_redirect_cpu_pass)
0253                     : bpf_program__fd(skel->progs.xdp_redirect_cpu_drop);
0254             } else {
0255                 fprintf(stderr, "Unknown name \"%s\" for built-in BPF program\n",
0256                     mprog_name);
0257                 goto end;
0258             }
0259         } else {
0260             if (redir_map) {
0261                 fprintf(stderr, "Need to specify --mprog-filename, --mprog-name and"
0262                     " --redirect-device with --redirect-map\n");
0263                 goto end;
0264             }
0265             if (redir_interface) {
0266                 /* Use built-in devmap redirect */
0267                 struct bpf_devmap_val val = {};
0268                 int ifindex_out, err;
0269                 __u32 key = 0;
0270
0271                 if (!redir_interface)
0272                     return 0;
0273
0274                 ifindex_out = if_nametoindex(redir_interface);
0275                 if (!ifindex_out)
0276                     ifindex_out = strtoul(redir_interface, NULL, 0);
0277                 if (!ifindex_out) {
0278                     fprintf(stderr, "Bad interface name or index\n");
0279                     return -EINVAL;
0280                 }
0281
0282                 if (get_mac_addr(ifindex_out, skel->bss->tx_mac_addr) < 0) {
0283                     printf("Get interface %d mac failed\n", ifindex_out);
0284                     return -EINVAL;
0285                 }
0286
0287                 val.ifindex = ifindex_out;
0288                 val.bpf_prog.fd = bpf_program__fd(skel->progs.xdp_redirect_egress_prog);
0289                 err = bpf_map_update_elem(bpf_map__fd(skel->maps.tx_port), &key, &val, 0);
0290                 if (err < 0)
0291                     return -errno;
0292
0293                 return bpf_program__fd(skel->progs.xdp_redirect_cpu_devmap);
0294             }
0295         }
0296     }
0297
0298     /* Disabled */
0299     return 0;
0300 end:
0301     fprintf(stderr, "Invalid options for CPUMAP BPF program\n");
0302     return -EINVAL;
0303 }
0304
0305 int main(int argc, char **argv)
0306 {
0307     const char *redir_interface = NULL, *redir_map = NULL;
0308     const char *mprog_filename = NULL, *mprog_name = NULL;
0309     struct xdp_redirect_cpu *skel;
0310     struct bpf_map_info info = {};
0311     struct bpf_cpumap_val value;
0312     __u32 infosz = sizeof(info);
0313     int ret = EXIT_FAIL_OPTION;
0314     unsigned long interval = 2;
0315     bool stress_mode = false;
0316     struct bpf_program *prog;
0317     const char *prog_name;
0318     bool generic = false;
0319     bool force = false;
0320     int added_cpus = 0;
0321     bool error = true;
0322     int longindex = 0;
0323     int add_cpu = -1;
0324     int ifindex = -1;
0325     int *cpu, i, opt;
0326     __u32 qsize;
0327     int n_cpus;
0328
0329     n_cpus = libbpf_num_possible_cpus();
0330
0331     /* Notice: Choosing the queue size is very important when CPU is
0332      * configured with power-saving states.
0333      *
0334      * If deepest state take 133 usec to wakeup from (133/10^6). When link
0335      * speed is 10Gbit/s ((10*10^9/8) in bytes/sec). How many bytes can
0336      * arrive with in 133 usec at this speed: (10*10^9/8)*(133/10^6) =
0337      * 166250 bytes. With MTU size packets this is 110 packets, and with
0338      * minimum Ethernet (MAC-preamble + intergap) 84 bytes is 1979 packets.
0339      *
0340      * Setting default cpumap queue to 2048 as worst-case (small packet)
0341      * should be +64 packet due kthread wakeup call (due to xdp_do_flush)
0342      * worst-case is 2043 packets.
0343      *
0344      * Sysadm can configured system to avoid deep-sleep via:
0345      *   tuned-adm profile network-latency
0346      */
0347     qsize = 2048;
0348
0349     skel = xdp_redirect_cpu__open();
0350     if (!skel) {
0351         fprintf(stderr, "Failed to xdp_redirect_cpu__open: %s\n",
0352             strerror(errno));
0353         ret = EXIT_FAIL_BPF;
0354         goto end;
0355     }
0356
0357     ret = sample_init_pre_load(skel);
0358     if (ret < 0) {
0359         fprintf(stderr, "Failed to sample_init_pre_load: %s\n", strerror(-ret));
0360         ret = EXIT_FAIL_BPF;
0361         goto end_destroy;
0362     }
0363
0364     if (bpf_map__set_max_entries(skel->maps.cpu_map, n_cpus) < 0) {
0365         fprintf(stderr, "Failed to set max entries for cpu_map map: %s",
0366             strerror(errno));
0367         ret = EXIT_FAIL_BPF;
0368         goto end_destroy;
0369     }
0370
0371     if (bpf_map__set_max_entries(skel->maps.cpus_available, n_cpus) < 0) {
0372         fprintf(stderr, "Failed to set max entries for cpus_available map: %s",
0373             strerror(errno));
0374         ret = EXIT_FAIL_BPF;
0375         goto end_destroy;
0376     }
0377
0378     cpu = calloc(n_cpus, sizeof(int));
0379     if (!cpu) {
0380         fprintf(stderr, "Failed to allocate cpu array\n");
0381         goto end_destroy;
0382     }
0383
0384     prog = skel->progs.xdp_prognum5_lb_hash_ip_pairs;
0385     while ((opt = getopt_long(argc, argv, "d:si:Sxp:f:e:r:m:c:q:Fvh",
0386                   long_options, &longindex)) != -1) {
0387         switch (opt) {
0388         case 'd':
0389             if (strlen(optarg) >= IF_NAMESIZE) {
0390                 fprintf(stderr, "-d/--dev name too long\n");
0391                 usage(argv, long_options, __doc__, mask, true, skel->obj);
0392                 goto end_cpu;
0393             }
0394             ifindex = if_nametoindex(optarg);
0395             if (!ifindex)
0396                 ifindex = strtoul(optarg, NULL, 0);
0397             if (!ifindex) {
0398                 fprintf(stderr, "Bad interface index or name (%d): %s\n",
0399                     errno, strerror(errno));
0400                 usage(argv, long_options, __doc__, mask, true, skel->obj);
0401                 goto end_cpu;
0402             }
0403             break;
0404         case 's':
0405             mask |= SAMPLE_REDIRECT_MAP_CNT;
0406             break;
0407         case 'i':
0408             interval = strtoul(optarg, NULL, 0);
0409             break;
0410         case 'S':
0411             generic = true;
0412             break;
0413         case 'x':
0414             stress_mode = true;
0415             break;
0416         case 'p':
0417             /* Selecting eBPF prog to load */
0418             prog_name = optarg;
0419             prog = bpf_object__find_program_by_name(skel->obj,
0420                                 prog_name);
0421             if (!prog) {
0422                 fprintf(stderr,
0423                     "Failed to find program %s specified by"
0424                     " option -p/--progname\n",
0425                     prog_name);
0426                 print_avail_progs(skel->obj);
0427                 goto end_cpu;
0428             }
0429             break;
0430         case 'f':
0431             mprog_filename = optarg;
0432             break;
0433         case 'e':
0434             mprog_name = optarg;
0435             break;
0436         case 'r':
0437             redir_interface = optarg;
0438             mask |= SAMPLE_DEVMAP_XMIT_CNT_MULTI;
0439             break;
0440         case 'm':
0441             redir_map = optarg;
0442             break;
0443         case 'c':
0444             /* Add multiple CPUs */
0445             add_cpu = strtoul(optarg, NULL, 0);
0446             if (add_cpu >= n_cpus) {
0447                 fprintf(stderr,
0448                 "--cpu nr too large for cpumap err (%d):%s\n",
0449                     errno, strerror(errno));
0450                 usage(argv, long_options, __doc__, mask, true, skel->obj);
0451                 goto end_cpu;
0452             }
0453             cpu[added_cpus++] = add_cpu;
0454             break;
0455         case 'q':
0456             qsize = strtoul(optarg, NULL, 0);
0457             break;
0458         case 'F':
0459             force = true;
0460             break;
0461         case 'v':
0462             sample_switch_mode();
0463             break;
0464         case 'h':
0465             error = false;
0466         default:
0467             usage(argv, long_options, __doc__, mask, error, skel->obj);
0468             goto end_cpu;
0469         }
0470     }
0471
0472     ret = EXIT_FAIL_OPTION;
0473     if (ifindex == -1) {
0474         fprintf(stderr, "Required option --dev missing\n");
0475         usage(argv, long_options, __doc__, mask, true, skel->obj);
0476         goto end_cpu;
0477     }
0478
0479     if (add_cpu == -1) {
0480         fprintf(stderr, "Required option --cpu missing\n"
0481                 "Specify multiple --cpu option to add more\n");
0482         usage(argv, long_options, __doc__, mask, true, skel->obj);
0483         goto end_cpu;
0484     }
0485
0486     skel->rodata->from_match[0] = ifindex;
0487     if (redir_interface)
0488         skel->rodata->to_match[0] = if_nametoindex(redir_interface);
0489
0490     ret = xdp_redirect_cpu__load(skel);
0491     if (ret < 0) {
0492         fprintf(stderr, "Failed to xdp_redirect_cpu__load: %s\n",
0493             strerror(errno));
0494         goto end_cpu;
0495     }
0496
0497     ret = bpf_obj_get_info_by_fd(bpf_map__fd(skel->maps.cpu_map), &info, &infosz);
0498     if (ret < 0) {
0499         fprintf(stderr, "Failed bpf_obj_get_info_by_fd for cpumap: %s\n",
0500             strerror(errno));
0501         goto end_cpu;
0502     }
0503
0504     skel->bss->cpumap_map_id = info.id;
0505
0506     map_fd = bpf_map__fd(skel->maps.cpu_map);
0507     avail_fd = bpf_map__fd(skel->maps.cpus_available);
0508     count_fd = bpf_map__fd(skel->maps.cpus_count);
0509
0510     ret = mark_cpus_unavailable();
0511     if (ret < 0) {
0512         fprintf(stderr, "Unable to mark CPUs as unavailable\n");
0513         goto end_cpu;
0514     }
0515
0516     ret = sample_init(skel, mask);
0517     if (ret < 0) {
0518         fprintf(stderr, "Failed to initialize sample: %s\n", strerror(-ret));
0519         ret = EXIT_FAIL;
0520         goto end_cpu;
0521     }
0522
0523     value.bpf_prog.fd = set_cpumap_prog(skel, redir_interface, redir_map,
0524                         mprog_filename, mprog_name);
0525     if (value.bpf_prog.fd < 0) {
0526         fprintf(stderr, "Failed to set CPUMAP BPF program: %s\n",
0527             strerror(-value.bpf_prog.fd));
0528         usage(argv, long_options, __doc__, mask, true, skel->obj);
0529         ret = EXIT_FAIL_BPF;
0530         goto end_cpu;
0531     }
0532     value.qsize = qsize;
0533
0534     for (i = 0; i < added_cpus; i++) {
0535         if (create_cpu_entry(cpu[i], &value, i, true) < 0) {
0536             fprintf(stderr, "Cannot proceed, exiting\n");
0537             usage(argv, long_options, __doc__, mask, true, skel->obj);
0538             goto end_cpu;
0539         }
0540     }
0541
0542     ret = EXIT_FAIL_XDP;
0543     if (sample_install_xdp(prog, ifindex, generic, force) < 0)
0544         goto end_cpu;
0545
0546     ret = sample_run(interval, stress_mode ? stress_cpumap : NULL, &value);
0547     if (ret < 0) {
0548         fprintf(stderr, "Failed during sample run: %s\n", strerror(-ret));
0549         ret = EXIT_FAIL;
0550         goto end_cpu;
0551     }
0552     ret = EXIT_OK;
0553 end_cpu:
0554     free(cpu);
0555 end_destroy:
0556     xdp_redirect_cpu__destroy(skel);
0557 end:
0558     sample_exit(ret);
0559 }