0001
0002 #define _GNU_SOURCE
0003
0004 #include <arpa/inet.h>
0005 #include <bpf/bpf.h>
0006 #include <bpf/libbpf.h>
0007 #include <errno.h>
0008 #include <fcntl.h>
0009 #include <getopt.h>
0010 #include <linux/ethtool.h>
0011 #include <linux/hashtable.h>
0012 #include <linux/if_link.h>
0013 #include <linux/jhash.h>
0014 #include <linux/limits.h>
0015 #include <linux/list.h>
0016 #include <linux/sockios.h>
0017 #include <locale.h>
0018 #include <math.h>
0019 #include <net/if.h>
0020 #include <poll.h>
0021 #include <signal.h>
0022 #include <stdbool.h>
0023 #include <stdio.h>
0024 #include <stdlib.h>
0025 #include <string.h>
0026 #include <sys/ioctl.h>
0027 #include <sys/mman.h>
0028 #include <sys/signalfd.h>
0029 #include <sys/sysinfo.h>
0030 #include <sys/timerfd.h>
0031 #include <sys/utsname.h>
0032 #include <time.h>
0033 #include <unistd.h>
0034
0035 #include "bpf_util.h"
0036 #include "xdp_sample_user.h"
0037
0038 #define __sample_print(fmt, cond, ...) \
0039 ({ \
0040 if (cond) \
0041 printf(fmt, ##__VA_ARGS__); \
0042 })
0043
0044 #define print_always(fmt, ...) __sample_print(fmt, 1, ##__VA_ARGS__)
0045 #define print_default(fmt, ...) \
0046 __sample_print(fmt, sample_log_level & LL_DEFAULT, ##__VA_ARGS__)
0047 #define __print_err(err, fmt, ...) \
0048 ({ \
0049 __sample_print(fmt, err > 0 || sample_log_level & LL_DEFAULT, \
0050 ##__VA_ARGS__); \
0051 sample_err_exp = sample_err_exp ? true : err > 0; \
0052 })
0053 #define print_err(err, fmt, ...) __print_err(err, fmt, ##__VA_ARGS__)
0054
0055 #define __COLUMN(x) "%'10" x " %-13s"
0056 #define FMT_COLUMNf __COLUMN(".0f")
0057 #define FMT_COLUMNd __COLUMN("d")
0058 #define FMT_COLUMNl __COLUMN("llu")
0059 #define RX(rx) rx, "rx/s"
0060 #define PPS(pps) pps, "pkt/s"
0061 #define DROP(drop) drop, "drop/s"
0062 #define ERR(err) err, "error/s"
0063 #define HITS(hits) hits, "hit/s"
0064 #define XMIT(xmit) xmit, "xmit/s"
0065 #define PASS(pass) pass, "pass/s"
0066 #define REDIR(redir) redir, "redir/s"
0067 #define NANOSEC_PER_SEC 1000000000
0068
0069 #define XDP_UNKNOWN (XDP_REDIRECT + 1)
0070 #define XDP_ACTION_MAX (XDP_UNKNOWN + 1)
0071 #define XDP_REDIRECT_ERR_MAX 7
0072
0073 enum map_type {
0074 MAP_RX,
0075 MAP_REDIRECT_ERR,
0076 MAP_CPUMAP_ENQUEUE,
0077 MAP_CPUMAP_KTHREAD,
0078 MAP_EXCEPTION,
0079 MAP_DEVMAP_XMIT,
0080 MAP_DEVMAP_XMIT_MULTI,
0081 NUM_MAP,
0082 };
0083
0084 enum log_level {
0085 LL_DEFAULT = 1U << 0,
0086 LL_SIMPLE = 1U << 1,
0087 LL_DEBUG = 1U << 2,
0088 };
0089
0090 struct record {
0091 __u64 timestamp;
0092 struct datarec total;
0093 struct datarec *cpu;
0094 };
0095
0096 struct map_entry {
0097 struct hlist_node node;
0098 __u64 pair;
0099 struct record val;
0100 };
0101
0102 struct stats_record {
0103 struct record rx_cnt;
0104 struct record redir_err[XDP_REDIRECT_ERR_MAX];
0105 struct record kthread;
0106 struct record exception[XDP_ACTION_MAX];
0107 struct record devmap_xmit;
0108 DECLARE_HASHTABLE(xmit_map, 5);
0109 struct record enq[];
0110 };
0111
0112 struct sample_output {
0113 struct {
0114 __u64 rx;
0115 __u64 redir;
0116 __u64 drop;
0117 __u64 drop_xmit;
0118 __u64 err;
0119 __u64 xmit;
0120 } totals;
0121 struct {
0122 union {
0123 __u64 pps;
0124 __u64 num;
0125 };
0126 __u64 drop;
0127 __u64 err;
0128 } rx_cnt;
0129 struct {
0130 __u64 suc;
0131 __u64 err;
0132 } redir_cnt;
0133 struct {
0134 __u64 hits;
0135 } except_cnt;
0136 struct {
0137 __u64 pps;
0138 __u64 drop;
0139 __u64 err;
0140 double bavg;
0141 } xmit_cnt;
0142 };
0143
0144 struct xdp_desc {
0145 int ifindex;
0146 __u32 prog_id;
0147 int flags;
0148 } sample_xdp_progs[32];
0149
0150 struct datarec *sample_mmap[NUM_MAP];
0151 struct bpf_map *sample_map[NUM_MAP];
0152 size_t sample_map_count[NUM_MAP];
0153 enum log_level sample_log_level;
0154 struct sample_output sample_out;
0155 unsigned long sample_interval;
0156 bool sample_err_exp;
0157 int sample_xdp_cnt;
0158 int sample_n_cpus;
0159 int sample_sig_fd;
0160 int sample_mask;
0161
0162 static const char *xdp_redirect_err_names[XDP_REDIRECT_ERR_MAX] = {
0163
0164 "Success",
0165 "Unknown",
0166 "EINVAL",
0167 "ENETDOWN",
0168 "EMSGSIZE",
0169 "EOPNOTSUPP",
0170 "ENOSPC",
0171 };
0172
0173
0174 static const char *xdp_redirect_err_help[XDP_REDIRECT_ERR_MAX - 1] = {
0175 "Unknown error",
0176 "Invalid redirection",
0177 "Device being redirected to is down",
0178 "Packet length too large for device",
0179 "Operation not supported",
0180 "No space in ptr_ring of cpumap kthread",
0181 };
0182
0183 static const char *xdp_action_names[XDP_ACTION_MAX] = {
0184 [XDP_ABORTED] = "XDP_ABORTED",
0185 [XDP_DROP] = "XDP_DROP",
0186 [XDP_PASS] = "XDP_PASS",
0187 [XDP_TX] = "XDP_TX",
0188 [XDP_REDIRECT] = "XDP_REDIRECT",
0189 [XDP_UNKNOWN] = "XDP_UNKNOWN",
0190 };
0191
0192 static __u64 gettime(void)
0193 {
0194 struct timespec t;
0195 int res;
0196
0197 res = clock_gettime(CLOCK_MONOTONIC, &t);
0198 if (res < 0) {
0199 fprintf(stderr, "Error with gettimeofday! (%i)\n", res);
0200 return UINT64_MAX;
0201 }
0202 return (__u64)t.tv_sec * NANOSEC_PER_SEC + t.tv_nsec;
0203 }
0204
0205 static const char *action2str(int action)
0206 {
0207 if (action < XDP_ACTION_MAX)
0208 return xdp_action_names[action];
0209 return NULL;
0210 }
0211
0212 static void sample_print_help(int mask)
0213 {
0214 printf("Output format description\n\n"
0215 "By default, redirect success statistics are disabled, use -s to enable.\n"
0216 "The terse output mode is default, verbose mode can be activated using -v\n"
0217 "Use SIGQUIT (Ctrl + \\) to switch the mode dynamically at runtime\n\n"
0218 "Terse mode displays at most the following fields:\n"
0219 " rx/s Number of packets received per second\n"
0220 " redir/s Number of packets successfully redirected per second\n"
0221 " err,drop/s Aggregated count of errors per second (including dropped packets)\n"
0222 " xmit/s Number of packets transmitted on the output device per second\n\n"
0223 "Output description for verbose mode:\n"
0224 " FIELD DESCRIPTION\n");
0225
0226 if (mask & SAMPLE_RX_CNT) {
0227 printf(" receive\t\tDisplays the number of packets received & errors encountered\n"
0228 " \t\t\tWhenever an error or packet drop occurs, details of per CPU error\n"
0229 " \t\t\tand drop statistics will be expanded inline in terse mode.\n"
0230 " \t\t\t\tpkt/s - Packets received per second\n"
0231 " \t\t\t\tdrop/s - Packets dropped per second\n"
0232 " \t\t\t\terror/s - Errors encountered per second\n\n");
0233 }
0234 if (mask & (SAMPLE_REDIRECT_CNT | SAMPLE_REDIRECT_ERR_CNT)) {
0235 printf(" redirect\t\tDisplays the number of packets successfully redirected\n"
0236 " \t\t\tErrors encountered are expanded under redirect_err field\n"
0237 " \t\t\tNote that passing -s to enable it has a per packet overhead\n"
0238 " \t\t\t\tredir/s - Packets redirected successfully per second\n\n"
0239 " redirect_err\t\tDisplays the number of packets that failed redirection\n"
0240 " \t\t\tThe errno is expanded under this field with per CPU count\n"
0241 " \t\t\tThe recognized errors are:\n");
0242
0243 for (int i = 2; i < XDP_REDIRECT_ERR_MAX; i++)
0244 printf("\t\t\t %s: %s\n", xdp_redirect_err_names[i],
0245 xdp_redirect_err_help[i - 1]);
0246
0247 printf(" \n\t\t\t\terror/s - Packets that failed redirection per second\n\n");
0248 }
0249
0250 if (mask & SAMPLE_CPUMAP_ENQUEUE_CNT) {
0251 printf(" enqueue to cpu N\tDisplays the number of packets enqueued to bulk queue of CPU N\n"
0252 " \t\t\tExpands to cpu:FROM->N to display enqueue stats for each CPU enqueuing to CPU N\n"
0253 " \t\t\tReceived packets can be associated with the CPU redirect program is enqueuing \n"
0254 " \t\t\tpackets to.\n"
0255 " \t\t\t\tpkt/s - Packets enqueued per second from other CPU to CPU N\n"
0256 " \t\t\t\tdrop/s - Packets dropped when trying to enqueue to CPU N\n"
0257 " \t\t\t\tbulk-avg - Average number of packets processed for each event\n\n");
0258 }
0259
0260 if (mask & SAMPLE_CPUMAP_KTHREAD_CNT) {
0261 printf(" kthread\t\tDisplays the number of packets processed in CPUMAP kthread for each CPU\n"
0262 " \t\t\tPackets consumed from ptr_ring in kthread, and its xdp_stats (after calling \n"
0263 " \t\t\tCPUMAP bpf prog) are expanded below this. xdp_stats are expanded as a total and\n"
0264 " \t\t\tthen per-CPU to associate it to each CPU's pinned CPUMAP kthread.\n"
0265 " \t\t\t\tpkt/s - Packets consumed per second from ptr_ring\n"
0266 " \t\t\t\tdrop/s - Packets dropped per second in kthread\n"
0267 " \t\t\t\tsched - Number of times kthread called schedule()\n\n"
0268 " \t\t\txdp_stats (also expands to per-CPU counts)\n"
0269 " \t\t\t\tpass/s - XDP_PASS count for CPUMAP program execution\n"
0270 " \t\t\t\tdrop/s - XDP_DROP count for CPUMAP program execution\n"
0271 " \t\t\t\tredir/s - XDP_REDIRECT count for CPUMAP program execution\n\n");
0272 }
0273
0274 if (mask & SAMPLE_EXCEPTION_CNT) {
0275 printf(" xdp_exception\t\tDisplays xdp_exception tracepoint events\n"
0276 " \t\t\tThis can occur due to internal driver errors, unrecognized\n"
0277 " \t\t\tXDP actions and due to explicit user trigger by use of XDP_ABORTED\n"
0278 " \t\t\tEach action is expanded below this field with its count\n"
0279 " \t\t\t\thit/s - Number of times the tracepoint was hit per second\n\n");
0280 }
0281
0282 if (mask & SAMPLE_DEVMAP_XMIT_CNT) {
0283 printf(" devmap_xmit\t\tDisplays devmap_xmit tracepoint events\n"
0284 " \t\t\tThis tracepoint is invoked for successful transmissions on output\n"
0285 " \t\t\tdevice but these statistics are not available for generic XDP mode,\n"
0286 " \t\t\thence they will be omitted from the output when using SKB mode\n"
0287 " \t\t\t\txmit/s - Number of packets that were transmitted per second\n"
0288 " \t\t\t\tdrop/s - Number of packets that failed transmissions per second\n"
0289 " \t\t\t\tdrv_err/s - Number of internal driver errors per second\n"
0290 " \t\t\t\tbulk-avg - Average number of packets processed for each event\n\n");
0291 }
0292 }
0293
0294 void sample_usage(char *argv[], const struct option *long_options,
0295 const char *doc, int mask, bool error)
0296 {
0297 int i;
0298
0299 if (!error)
0300 sample_print_help(mask);
0301
0302 printf("\n%s\nOption for %s:\n", doc, argv[0]);
0303 for (i = 0; long_options[i].name != 0; i++) {
0304 printf(" --%-15s", long_options[i].name);
0305 if (long_options[i].flag != NULL)
0306 printf(" flag (internal value: %d)",
0307 *long_options[i].flag);
0308 else
0309 printf("\t short-option: -%c", long_options[i].val);
0310 printf("\n");
0311 }
0312 printf("\n");
0313 }
0314
0315 static struct datarec *alloc_record_per_cpu(void)
0316 {
0317 unsigned int nr_cpus = libbpf_num_possible_cpus();
0318 struct datarec *array;
0319
0320 array = calloc(nr_cpus, sizeof(*array));
0321 if (!array) {
0322 fprintf(stderr, "Failed to allocate memory (nr_cpus: %u)\n",
0323 nr_cpus);
0324 return NULL;
0325 }
0326 return array;
0327 }
0328
0329 static int map_entry_init(struct map_entry *e, __u64 pair)
0330 {
0331 e->pair = pair;
0332 INIT_HLIST_NODE(&e->node);
0333 e->val.timestamp = gettime();
0334 e->val.cpu = alloc_record_per_cpu();
0335 if (!e->val.cpu)
0336 return -ENOMEM;
0337 return 0;
0338 }
0339
0340 static void map_collect_percpu(struct datarec *values, struct record *rec)
0341 {
0342
0343 unsigned int nr_cpus = libbpf_num_possible_cpus();
0344 __u64 sum_xdp_redirect = 0;
0345 __u64 sum_processed = 0;
0346 __u64 sum_xdp_pass = 0;
0347 __u64 sum_xdp_drop = 0;
0348 __u64 sum_dropped = 0;
0349 __u64 sum_issue = 0;
0350 int i;
0351
0352
0353 rec->timestamp = gettime();
0354
0355
0356 for (i = 0; i < nr_cpus; i++) {
0357 rec->cpu[i].processed = READ_ONCE(values[i].processed);
0358 rec->cpu[i].dropped = READ_ONCE(values[i].dropped);
0359 rec->cpu[i].issue = READ_ONCE(values[i].issue);
0360 rec->cpu[i].xdp_pass = READ_ONCE(values[i].xdp_pass);
0361 rec->cpu[i].xdp_drop = READ_ONCE(values[i].xdp_drop);
0362 rec->cpu[i].xdp_redirect = READ_ONCE(values[i].xdp_redirect);
0363
0364 sum_processed += rec->cpu[i].processed;
0365 sum_dropped += rec->cpu[i].dropped;
0366 sum_issue += rec->cpu[i].issue;
0367 sum_xdp_pass += rec->cpu[i].xdp_pass;
0368 sum_xdp_drop += rec->cpu[i].xdp_drop;
0369 sum_xdp_redirect += rec->cpu[i].xdp_redirect;
0370 }
0371
0372 rec->total.processed = sum_processed;
0373 rec->total.dropped = sum_dropped;
0374 rec->total.issue = sum_issue;
0375 rec->total.xdp_pass = sum_xdp_pass;
0376 rec->total.xdp_drop = sum_xdp_drop;
0377 rec->total.xdp_redirect = sum_xdp_redirect;
0378 }
0379
0380 static int map_collect_percpu_devmap(int map_fd, struct stats_record *rec)
0381 {
0382 unsigned int nr_cpus = bpf_num_possible_cpus();
0383 __u32 batch, count = 32;
0384 struct datarec *values;
0385 bool init = false;
0386 __u64 *keys;
0387 int i, ret;
0388
0389 keys = calloc(count, sizeof(__u64));
0390 if (!keys)
0391 return -ENOMEM;
0392 values = calloc(count * nr_cpus, sizeof(struct datarec));
0393 if (!values) {
0394 free(keys);
0395 return -ENOMEM;
0396 }
0397
0398 for (;;) {
0399 bool exit = false;
0400
0401 ret = bpf_map_lookup_batch(map_fd, init ? &batch : NULL, &batch,
0402 keys, values, &count, NULL);
0403 if (ret < 0 && errno != ENOENT)
0404 break;
0405 if (errno == ENOENT)
0406 exit = true;
0407
0408 init = true;
0409 for (i = 0; i < count; i++) {
0410 struct map_entry *e, *x = NULL;
0411 __u64 pair = keys[i];
0412 struct datarec *arr;
0413
0414 arr = &values[i * nr_cpus];
0415 hash_for_each_possible(rec->xmit_map, e, node, pair) {
0416 if (e->pair == pair) {
0417 x = e;
0418 break;
0419 }
0420 }
0421 if (!x) {
0422 x = calloc(1, sizeof(*x));
0423 if (!x)
0424 goto cleanup;
0425 if (map_entry_init(x, pair) < 0) {
0426 free(x);
0427 goto cleanup;
0428 }
0429 hash_add(rec->xmit_map, &x->node, pair);
0430 }
0431 map_collect_percpu(arr, &x->val);
0432 }
0433
0434 if (exit)
0435 break;
0436 count = 32;
0437 }
0438
0439 free(values);
0440 free(keys);
0441 return 0;
0442 cleanup:
0443 free(values);
0444 free(keys);
0445 return -ENOMEM;
0446 }
0447
0448 static struct stats_record *alloc_stats_record(void)
0449 {
0450 struct stats_record *rec;
0451 int i;
0452
0453 rec = calloc(1, sizeof(*rec) + sample_n_cpus * sizeof(struct record));
0454 if (!rec) {
0455 fprintf(stderr, "Failed to allocate memory\n");
0456 return NULL;
0457 }
0458
0459 if (sample_mask & SAMPLE_RX_CNT) {
0460 rec->rx_cnt.cpu = alloc_record_per_cpu();
0461 if (!rec->rx_cnt.cpu) {
0462 fprintf(stderr,
0463 "Failed to allocate rx_cnt per-CPU array\n");
0464 goto end_rec;
0465 }
0466 }
0467 if (sample_mask & (SAMPLE_REDIRECT_CNT | SAMPLE_REDIRECT_ERR_CNT)) {
0468 for (i = 0; i < XDP_REDIRECT_ERR_MAX; i++) {
0469 rec->redir_err[i].cpu = alloc_record_per_cpu();
0470 if (!rec->redir_err[i].cpu) {
0471 fprintf(stderr,
0472 "Failed to allocate redir_err per-CPU array for "
0473 "\"%s\" case\n",
0474 xdp_redirect_err_names[i]);
0475 while (i--)
0476 free(rec->redir_err[i].cpu);
0477 goto end_rx_cnt;
0478 }
0479 }
0480 }
0481 if (sample_mask & SAMPLE_CPUMAP_KTHREAD_CNT) {
0482 rec->kthread.cpu = alloc_record_per_cpu();
0483 if (!rec->kthread.cpu) {
0484 fprintf(stderr,
0485 "Failed to allocate kthread per-CPU array\n");
0486 goto end_redir;
0487 }
0488 }
0489 if (sample_mask & SAMPLE_EXCEPTION_CNT) {
0490 for (i = 0; i < XDP_ACTION_MAX; i++) {
0491 rec->exception[i].cpu = alloc_record_per_cpu();
0492 if (!rec->exception[i].cpu) {
0493 fprintf(stderr,
0494 "Failed to allocate exception per-CPU array for "
0495 "\"%s\" case\n",
0496 action2str(i));
0497 while (i--)
0498 free(rec->exception[i].cpu);
0499 goto end_kthread;
0500 }
0501 }
0502 }
0503 if (sample_mask & SAMPLE_DEVMAP_XMIT_CNT) {
0504 rec->devmap_xmit.cpu = alloc_record_per_cpu();
0505 if (!rec->devmap_xmit.cpu) {
0506 fprintf(stderr,
0507 "Failed to allocate devmap_xmit per-CPU array\n");
0508 goto end_exception;
0509 }
0510 }
0511 if (sample_mask & SAMPLE_DEVMAP_XMIT_CNT_MULTI)
0512 hash_init(rec->xmit_map);
0513 if (sample_mask & SAMPLE_CPUMAP_ENQUEUE_CNT) {
0514 for (i = 0; i < sample_n_cpus; i++) {
0515 rec->enq[i].cpu = alloc_record_per_cpu();
0516 if (!rec->enq[i].cpu) {
0517 fprintf(stderr,
0518 "Failed to allocate enqueue per-CPU array for "
0519 "CPU %d\n",
0520 i);
0521 while (i--)
0522 free(rec->enq[i].cpu);
0523 goto end_devmap_xmit;
0524 }
0525 }
0526 }
0527
0528 return rec;
0529
0530 end_devmap_xmit:
0531 free(rec->devmap_xmit.cpu);
0532 end_exception:
0533 for (i = 0; i < XDP_ACTION_MAX; i++)
0534 free(rec->exception[i].cpu);
0535 end_kthread:
0536 free(rec->kthread.cpu);
0537 end_redir:
0538 for (i = 0; i < XDP_REDIRECT_ERR_MAX; i++)
0539 free(rec->redir_err[i].cpu);
0540 end_rx_cnt:
0541 free(rec->rx_cnt.cpu);
0542 end_rec:
0543 free(rec);
0544 return NULL;
0545 }
0546
0547 static void free_stats_record(struct stats_record *r)
0548 {
0549 struct hlist_node *tmp;
0550 struct map_entry *e;
0551 int i;
0552
0553 for (i = 0; i < sample_n_cpus; i++)
0554 free(r->enq[i].cpu);
0555 hash_for_each_safe(r->xmit_map, i, tmp, e, node) {
0556 hash_del(&e->node);
0557 free(e->val.cpu);
0558 free(e);
0559 }
0560 free(r->devmap_xmit.cpu);
0561 for (i = 0; i < XDP_ACTION_MAX; i++)
0562 free(r->exception[i].cpu);
0563 free(r->kthread.cpu);
0564 for (i = 0; i < XDP_REDIRECT_ERR_MAX; i++)
0565 free(r->redir_err[i].cpu);
0566 free(r->rx_cnt.cpu);
0567 free(r);
0568 }
0569
0570 static double calc_period(struct record *r, struct record *p)
0571 {
0572 double period_ = 0;
0573 __u64 period = 0;
0574
0575 period = r->timestamp - p->timestamp;
0576 if (period > 0)
0577 period_ = ((double)period / NANOSEC_PER_SEC);
0578
0579 return period_;
0580 }
0581
0582 static double sample_round(double val)
0583 {
0584 if (val - floor(val) < 0.5)
0585 return floor(val);
0586 return ceil(val);
0587 }
0588
0589 static __u64 calc_pps(struct datarec *r, struct datarec *p, double period_)
0590 {
0591 __u64 packets = 0;
0592 __u64 pps = 0;
0593
0594 if (period_ > 0) {
0595 packets = r->processed - p->processed;
0596 pps = sample_round(packets / period_);
0597 }
0598 return pps;
0599 }
0600
0601 static __u64 calc_drop_pps(struct datarec *r, struct datarec *p, double period_)
0602 {
0603 __u64 packets = 0;
0604 __u64 pps = 0;
0605
0606 if (period_ > 0) {
0607 packets = r->dropped - p->dropped;
0608 pps = sample_round(packets / period_);
0609 }
0610 return pps;
0611 }
0612
0613 static __u64 calc_errs_pps(struct datarec *r, struct datarec *p, double period_)
0614 {
0615 __u64 packets = 0;
0616 __u64 pps = 0;
0617
0618 if (period_ > 0) {
0619 packets = r->issue - p->issue;
0620 pps = sample_round(packets / period_);
0621 }
0622 return pps;
0623 }
0624
0625 static __u64 calc_info_pps(struct datarec *r, struct datarec *p, double period_)
0626 {
0627 __u64 packets = 0;
0628 __u64 pps = 0;
0629
0630 if (period_ > 0) {
0631 packets = r->info - p->info;
0632 pps = sample_round(packets / period_);
0633 }
0634 return pps;
0635 }
0636
0637 static void calc_xdp_pps(struct datarec *r, struct datarec *p, double *xdp_pass,
0638 double *xdp_drop, double *xdp_redirect, double period_)
0639 {
0640 *xdp_pass = 0, *xdp_drop = 0, *xdp_redirect = 0;
0641 if (period_ > 0) {
0642 *xdp_redirect = (r->xdp_redirect - p->xdp_redirect) / period_;
0643 *xdp_pass = (r->xdp_pass - p->xdp_pass) / period_;
0644 *xdp_drop = (r->xdp_drop - p->xdp_drop) / period_;
0645 }
0646 }
0647
0648 static void stats_get_rx_cnt(struct stats_record *stats_rec,
0649 struct stats_record *stats_prev,
0650 unsigned int nr_cpus, struct sample_output *out)
0651 {
0652 struct record *rec, *prev;
0653 double t, pps, drop, err;
0654 int i;
0655
0656 rec = &stats_rec->rx_cnt;
0657 prev = &stats_prev->rx_cnt;
0658 t = calc_period(rec, prev);
0659
0660 for (i = 0; i < nr_cpus; i++) {
0661 struct datarec *r = &rec->cpu[i];
0662 struct datarec *p = &prev->cpu[i];
0663 char str[64];
0664
0665 pps = calc_pps(r, p, t);
0666 drop = calc_drop_pps(r, p, t);
0667 err = calc_errs_pps(r, p, t);
0668 if (!pps && !drop && !err)
0669 continue;
0670
0671 snprintf(str, sizeof(str), "cpu:%d", i);
0672 print_default(" %-18s " FMT_COLUMNf FMT_COLUMNf FMT_COLUMNf
0673 "\n",
0674 str, PPS(pps), DROP(drop), ERR(err));
0675 }
0676
0677 if (out) {
0678 pps = calc_pps(&rec->total, &prev->total, t);
0679 drop = calc_drop_pps(&rec->total, &prev->total, t);
0680 err = calc_errs_pps(&rec->total, &prev->total, t);
0681
0682 out->rx_cnt.pps = pps;
0683 out->rx_cnt.drop = drop;
0684 out->rx_cnt.err = err;
0685 out->totals.rx += pps;
0686 out->totals.drop += drop;
0687 out->totals.err += err;
0688 }
0689 }
0690
0691 static void stats_get_cpumap_enqueue(struct stats_record *stats_rec,
0692 struct stats_record *stats_prev,
0693 unsigned int nr_cpus)
0694 {
0695 struct record *rec, *prev;
0696 double t, pps, drop, err;
0697 int i, to_cpu;
0698
0699
0700 for (to_cpu = 0; to_cpu < sample_n_cpus; to_cpu++) {
0701 rec = &stats_rec->enq[to_cpu];
0702 prev = &stats_prev->enq[to_cpu];
0703 t = calc_period(rec, prev);
0704
0705 pps = calc_pps(&rec->total, &prev->total, t);
0706 drop = calc_drop_pps(&rec->total, &prev->total, t);
0707 err = calc_errs_pps(&rec->total, &prev->total, t);
0708
0709 if (pps > 0 || drop > 0) {
0710 char str[64];
0711
0712 snprintf(str, sizeof(str), "enqueue to cpu %d", to_cpu);
0713
0714 if (err > 0)
0715 err = pps / err;
0716
0717 print_err(drop,
0718 " %-20s " FMT_COLUMNf FMT_COLUMNf __COLUMN(
0719 ".2f") "\n",
0720 str, PPS(pps), DROP(drop), err, "bulk-avg");
0721 }
0722
0723 for (i = 0; i < nr_cpus; i++) {
0724 struct datarec *r = &rec->cpu[i];
0725 struct datarec *p = &prev->cpu[i];
0726 char str[64];
0727
0728 pps = calc_pps(r, p, t);
0729 drop = calc_drop_pps(r, p, t);
0730 err = calc_errs_pps(r, p, t);
0731 if (!pps && !drop && !err)
0732 continue;
0733
0734 snprintf(str, sizeof(str), "cpu:%d->%d", i, to_cpu);
0735 if (err > 0)
0736 err = pps / err;
0737 print_default(
0738 " %-18s " FMT_COLUMNf FMT_COLUMNf __COLUMN(
0739 ".2f") "\n",
0740 str, PPS(pps), DROP(drop), err, "bulk-avg");
0741 }
0742 }
0743 }
0744
0745 static void stats_get_cpumap_remote(struct stats_record *stats_rec,
0746 struct stats_record *stats_prev,
0747 unsigned int nr_cpus)
0748 {
0749 double xdp_pass, xdp_drop, xdp_redirect;
0750 struct record *rec, *prev;
0751 double t;
0752 int i;
0753
0754 rec = &stats_rec->kthread;
0755 prev = &stats_prev->kthread;
0756 t = calc_period(rec, prev);
0757
0758 calc_xdp_pps(&rec->total, &prev->total, &xdp_pass, &xdp_drop,
0759 &xdp_redirect, t);
0760 if (xdp_pass || xdp_drop || xdp_redirect) {
0761 print_err(xdp_drop,
0762 " %-18s " FMT_COLUMNf FMT_COLUMNf FMT_COLUMNf "\n",
0763 "xdp_stats", PASS(xdp_pass), DROP(xdp_drop),
0764 REDIR(xdp_redirect));
0765 }
0766
0767 for (i = 0; i < nr_cpus; i++) {
0768 struct datarec *r = &rec->cpu[i];
0769 struct datarec *p = &prev->cpu[i];
0770 char str[64];
0771
0772 calc_xdp_pps(r, p, &xdp_pass, &xdp_drop, &xdp_redirect, t);
0773 if (!xdp_pass && !xdp_drop && !xdp_redirect)
0774 continue;
0775
0776 snprintf(str, sizeof(str), "cpu:%d", i);
0777 print_default(" %-16s " FMT_COLUMNf FMT_COLUMNf FMT_COLUMNf
0778 "\n",
0779 str, PASS(xdp_pass), DROP(xdp_drop),
0780 REDIR(xdp_redirect));
0781 }
0782 }
0783
0784 static void stats_get_cpumap_kthread(struct stats_record *stats_rec,
0785 struct stats_record *stats_prev,
0786 unsigned int nr_cpus)
0787 {
0788 struct record *rec, *prev;
0789 double t, pps, drop, err;
0790 int i;
0791
0792 rec = &stats_rec->kthread;
0793 prev = &stats_prev->kthread;
0794 t = calc_period(rec, prev);
0795
0796 pps = calc_pps(&rec->total, &prev->total, t);
0797 drop = calc_drop_pps(&rec->total, &prev->total, t);
0798 err = calc_errs_pps(&rec->total, &prev->total, t);
0799
0800 print_err(drop, " %-20s " FMT_COLUMNf FMT_COLUMNf FMT_COLUMNf "\n",
0801 pps ? "kthread total" : "kthread", PPS(pps), DROP(drop), err,
0802 "sched");
0803
0804 for (i = 0; i < nr_cpus; i++) {
0805 struct datarec *r = &rec->cpu[i];
0806 struct datarec *p = &prev->cpu[i];
0807 char str[64];
0808
0809 pps = calc_pps(r, p, t);
0810 drop = calc_drop_pps(r, p, t);
0811 err = calc_errs_pps(r, p, t);
0812 if (!pps && !drop && !err)
0813 continue;
0814
0815 snprintf(str, sizeof(str), "cpu:%d", i);
0816 print_default(" %-18s " FMT_COLUMNf FMT_COLUMNf FMT_COLUMNf
0817 "\n",
0818 str, PPS(pps), DROP(drop), err, "sched");
0819 }
0820 }
0821
0822 static void stats_get_redirect_cnt(struct stats_record *stats_rec,
0823 struct stats_record *stats_prev,
0824 unsigned int nr_cpus,
0825 struct sample_output *out)
0826 {
0827 struct record *rec, *prev;
0828 double t, pps;
0829 int i;
0830
0831 rec = &stats_rec->redir_err[0];
0832 prev = &stats_prev->redir_err[0];
0833 t = calc_period(rec, prev);
0834 for (i = 0; i < nr_cpus; i++) {
0835 struct datarec *r = &rec->cpu[i];
0836 struct datarec *p = &prev->cpu[i];
0837 char str[64];
0838
0839 pps = calc_pps(r, p, t);
0840 if (!pps)
0841 continue;
0842
0843 snprintf(str, sizeof(str), "cpu:%d", i);
0844 print_default(" %-18s " FMT_COLUMNf "\n", str, REDIR(pps));
0845 }
0846
0847 if (out) {
0848 pps = calc_pps(&rec->total, &prev->total, t);
0849 out->redir_cnt.suc = pps;
0850 out->totals.redir += pps;
0851 }
0852 }
0853
0854 static void stats_get_redirect_err_cnt(struct stats_record *stats_rec,
0855 struct stats_record *stats_prev,
0856 unsigned int nr_cpus,
0857 struct sample_output *out)
0858 {
0859 struct record *rec, *prev;
0860 double t, drop, sum = 0;
0861 int rec_i, i;
0862
0863 for (rec_i = 1; rec_i < XDP_REDIRECT_ERR_MAX; rec_i++) {
0864 char str[64];
0865
0866 rec = &stats_rec->redir_err[rec_i];
0867 prev = &stats_prev->redir_err[rec_i];
0868 t = calc_period(rec, prev);
0869
0870 drop = calc_drop_pps(&rec->total, &prev->total, t);
0871 if (drop > 0 && !out) {
0872 snprintf(str, sizeof(str),
0873 sample_log_level & LL_DEFAULT ? "%s total" :
0874 "%s",
0875 xdp_redirect_err_names[rec_i]);
0876 print_err(drop, " %-18s " FMT_COLUMNf "\n", str,
0877 ERR(drop));
0878 }
0879
0880 for (i = 0; i < nr_cpus; i++) {
0881 struct datarec *r = &rec->cpu[i];
0882 struct datarec *p = &prev->cpu[i];
0883 double drop;
0884
0885 drop = calc_drop_pps(r, p, t);
0886 if (!drop)
0887 continue;
0888
0889 snprintf(str, sizeof(str), "cpu:%d", i);
0890 print_default(" %-16s" FMT_COLUMNf "\n", str,
0891 ERR(drop));
0892 }
0893
0894 sum += drop;
0895 }
0896
0897 if (out) {
0898 out->redir_cnt.err = sum;
0899 out->totals.err += sum;
0900 }
0901 }
0902
0903 static void stats_get_exception_cnt(struct stats_record *stats_rec,
0904 struct stats_record *stats_prev,
0905 unsigned int nr_cpus,
0906 struct sample_output *out)
0907 {
0908 double t, drop, sum = 0;
0909 struct record *rec, *prev;
0910 int rec_i, i;
0911
0912 for (rec_i = 0; rec_i < XDP_ACTION_MAX; rec_i++) {
0913 rec = &stats_rec->exception[rec_i];
0914 prev = &stats_prev->exception[rec_i];
0915 t = calc_period(rec, prev);
0916
0917 drop = calc_drop_pps(&rec->total, &prev->total, t);
0918
0919 sum += drop;
0920
0921 if (drop > 0 && !out) {
0922 print_always(" %-18s " FMT_COLUMNf "\n",
0923 action2str(rec_i), ERR(drop));
0924
0925 for (i = 0; i < nr_cpus; i++) {
0926 struct datarec *r = &rec->cpu[i];
0927 struct datarec *p = &prev->cpu[i];
0928 char str[64];
0929 double drop;
0930
0931 drop = calc_drop_pps(r, p, t);
0932 if (!drop)
0933 continue;
0934
0935 snprintf(str, sizeof(str), "cpu:%d", i);
0936 print_default(" %-16s" FMT_COLUMNf "\n",
0937 str, ERR(drop));
0938 }
0939 }
0940 }
0941
0942 if (out) {
0943 out->except_cnt.hits = sum;
0944 out->totals.err += sum;
0945 }
0946 }
0947
0948 static void stats_get_devmap_xmit(struct stats_record *stats_rec,
0949 struct stats_record *stats_prev,
0950 unsigned int nr_cpus,
0951 struct sample_output *out)
0952 {
0953 double pps, drop, info, err;
0954 struct record *rec, *prev;
0955 double t;
0956 int i;
0957
0958 rec = &stats_rec->devmap_xmit;
0959 prev = &stats_prev->devmap_xmit;
0960 t = calc_period(rec, prev);
0961 for (i = 0; i < nr_cpus; i++) {
0962 struct datarec *r = &rec->cpu[i];
0963 struct datarec *p = &prev->cpu[i];
0964 char str[64];
0965
0966 pps = calc_pps(r, p, t);
0967 drop = calc_drop_pps(r, p, t);
0968 err = calc_errs_pps(r, p, t);
0969
0970 if (!pps && !drop && !err)
0971 continue;
0972
0973 snprintf(str, sizeof(str), "cpu:%d", i);
0974 info = calc_info_pps(r, p, t);
0975 if (info > 0)
0976 info = (pps + drop) / info;
0977 print_default(" %-18s" FMT_COLUMNf FMT_COLUMNf FMT_COLUMNf
0978 __COLUMN(".2f") "\n",
0979 str, XMIT(pps), DROP(drop), err, "drv_err/s",
0980 info, "bulk-avg");
0981 }
0982 if (out) {
0983 pps = calc_pps(&rec->total, &prev->total, t);
0984 drop = calc_drop_pps(&rec->total, &prev->total, t);
0985 info = calc_info_pps(&rec->total, &prev->total, t);
0986 if (info > 0)
0987 info = (pps + drop) / info;
0988 err = calc_errs_pps(&rec->total, &prev->total, t);
0989
0990 out->xmit_cnt.pps = pps;
0991 out->xmit_cnt.drop = drop;
0992 out->xmit_cnt.bavg = info;
0993 out->xmit_cnt.err = err;
0994 out->totals.xmit += pps;
0995 out->totals.drop_xmit += drop;
0996 out->totals.err += err;
0997 }
0998 }
0999
1000 static void stats_get_devmap_xmit_multi(struct stats_record *stats_rec,
1001 struct stats_record *stats_prev,
1002 unsigned int nr_cpus,
1003 struct sample_output *out,
1004 bool xmit_total)
1005 {
1006 double pps, drop, info, err;
1007 struct map_entry *entry;
1008 struct record *r, *p;
1009 double t;
1010 int bkt;
1011
1012 hash_for_each(stats_rec->xmit_map, bkt, entry, node) {
1013 struct map_entry *e, *x = NULL;
1014 char ifname_from[IFNAMSIZ];
1015 char ifname_to[IFNAMSIZ];
1016 const char *fstr, *tstr;
1017 unsigned long prev_time;
1018 struct record beg = {};
1019 __u32 from_idx, to_idx;
1020 char str[128];
1021 __u64 pair;
1022 int i;
1023
1024 prev_time = sample_interval * NANOSEC_PER_SEC;
1025
1026 pair = entry->pair;
1027 from_idx = pair >> 32;
1028 to_idx = pair & 0xFFFFFFFF;
1029
1030 r = &entry->val;
1031 beg.timestamp = r->timestamp - prev_time;
1032
1033
1034 hash_for_each_possible(stats_prev->xmit_map, e, node, pair) {
1035 if (e->pair == pair) {
1036 x = e;
1037 break;
1038 }
1039 }
1040 if (x)
1041 p = &x->val;
1042 else
1043 p = &beg;
1044 t = calc_period(r, p);
1045 pps = calc_pps(&r->total, &p->total, t);
1046 drop = calc_drop_pps(&r->total, &p->total, t);
1047 info = calc_info_pps(&r->total, &p->total, t);
1048 if (info > 0)
1049 info = (pps + drop) / info;
1050 err = calc_errs_pps(&r->total, &p->total, t);
1051
1052 if (out) {
1053
1054 out->totals.xmit += pps;
1055 out->totals.drop_xmit += drop;
1056 out->totals.err += err;
1057 continue;
1058 }
1059
1060 fstr = tstr = NULL;
1061 if (if_indextoname(from_idx, ifname_from))
1062 fstr = ifname_from;
1063 if (if_indextoname(to_idx, ifname_to))
1064 tstr = ifname_to;
1065
1066 snprintf(str, sizeof(str), "xmit %s->%s", fstr ?: "?",
1067 tstr ?: "?");
1068
1069 if (pps || drop || err) {
1070 print_err(drop,
1071 " %-20s " FMT_COLUMNf FMT_COLUMNf FMT_COLUMNf
1072 __COLUMN(".2f") "\n", str, XMIT(pps), DROP(drop),
1073 err, "drv_err/s", info, "bulk-avg");
1074 }
1075
1076 for (i = 0; i < nr_cpus; i++) {
1077 struct datarec *rc = &r->cpu[i];
1078 struct datarec *pc, p_beg = {};
1079 char str[64];
1080
1081 pc = p == &beg ? &p_beg : &p->cpu[i];
1082
1083 pps = calc_pps(rc, pc, t);
1084 drop = calc_drop_pps(rc, pc, t);
1085 err = calc_errs_pps(rc, pc, t);
1086
1087 if (!pps && !drop && !err)
1088 continue;
1089
1090 snprintf(str, sizeof(str), "cpu:%d", i);
1091 info = calc_info_pps(rc, pc, t);
1092 if (info > 0)
1093 info = (pps + drop) / info;
1094
1095 print_default(" %-18s" FMT_COLUMNf FMT_COLUMNf FMT_COLUMNf
1096 __COLUMN(".2f") "\n", str, XMIT(pps),
1097 DROP(drop), err, "drv_err/s", info, "bulk-avg");
1098 }
1099 }
1100 }
1101
1102 static void stats_print(const char *prefix, int mask, struct stats_record *r,
1103 struct stats_record *p, struct sample_output *out)
1104 {
1105 int nr_cpus = libbpf_num_possible_cpus();
1106 const char *str;
1107
1108 print_always("%-23s", prefix ?: "Summary");
1109 if (mask & SAMPLE_RX_CNT)
1110 print_always(FMT_COLUMNl, RX(out->totals.rx));
1111 if (mask & SAMPLE_REDIRECT_CNT)
1112 print_always(FMT_COLUMNl, REDIR(out->totals.redir));
1113 printf(FMT_COLUMNl,
1114 out->totals.err + out->totals.drop + out->totals.drop_xmit,
1115 "err,drop/s");
1116 if (mask & SAMPLE_DEVMAP_XMIT_CNT ||
1117 mask & SAMPLE_DEVMAP_XMIT_CNT_MULTI)
1118 printf(FMT_COLUMNl, XMIT(out->totals.xmit));
1119 printf("\n");
1120
1121 if (mask & SAMPLE_RX_CNT) {
1122 str = (sample_log_level & LL_DEFAULT) && out->rx_cnt.pps ?
1123 "receive total" :
1124 "receive";
1125 print_err((out->rx_cnt.err || out->rx_cnt.drop),
1126 " %-20s " FMT_COLUMNl FMT_COLUMNl FMT_COLUMNl "\n",
1127 str, PPS(out->rx_cnt.pps), DROP(out->rx_cnt.drop),
1128 ERR(out->rx_cnt.err));
1129
1130 stats_get_rx_cnt(r, p, nr_cpus, NULL);
1131 }
1132
1133 if (mask & SAMPLE_CPUMAP_ENQUEUE_CNT)
1134 stats_get_cpumap_enqueue(r, p, nr_cpus);
1135
1136 if (mask & SAMPLE_CPUMAP_KTHREAD_CNT) {
1137 stats_get_cpumap_kthread(r, p, nr_cpus);
1138 stats_get_cpumap_remote(r, p, nr_cpus);
1139 }
1140
1141 if (mask & SAMPLE_REDIRECT_CNT) {
1142 str = out->redir_cnt.suc ? "redirect total" : "redirect";
1143 print_default(" %-20s " FMT_COLUMNl "\n", str,
1144 REDIR(out->redir_cnt.suc));
1145
1146 stats_get_redirect_cnt(r, p, nr_cpus, NULL);
1147 }
1148
1149 if (mask & SAMPLE_REDIRECT_ERR_CNT) {
1150 str = (sample_log_level & LL_DEFAULT) && out->redir_cnt.err ?
1151 "redirect_err total" :
1152 "redirect_err";
1153 print_err(out->redir_cnt.err, " %-20s " FMT_COLUMNl "\n", str,
1154 ERR(out->redir_cnt.err));
1155
1156 stats_get_redirect_err_cnt(r, p, nr_cpus, NULL);
1157 }
1158
1159 if (mask & SAMPLE_EXCEPTION_CNT) {
1160 str = out->except_cnt.hits ? "xdp_exception total" :
1161 "xdp_exception";
1162
1163 print_err(out->except_cnt.hits, " %-20s " FMT_COLUMNl "\n", str,
1164 HITS(out->except_cnt.hits));
1165
1166 stats_get_exception_cnt(r, p, nr_cpus, NULL);
1167 }
1168
1169 if (mask & SAMPLE_DEVMAP_XMIT_CNT) {
1170 str = (sample_log_level & LL_DEFAULT) && out->xmit_cnt.pps ?
1171 "devmap_xmit total" :
1172 "devmap_xmit";
1173
1174 print_err(out->xmit_cnt.err || out->xmit_cnt.drop,
1175 " %-20s " FMT_COLUMNl FMT_COLUMNl FMT_COLUMNl
1176 __COLUMN(".2f") "\n",
1177 str, XMIT(out->xmit_cnt.pps),
1178 DROP(out->xmit_cnt.drop), out->xmit_cnt.err,
1179 "drv_err/s", out->xmit_cnt.bavg, "bulk-avg");
1180
1181 stats_get_devmap_xmit(r, p, nr_cpus, NULL);
1182 }
1183
1184 if (mask & SAMPLE_DEVMAP_XMIT_CNT_MULTI)
1185 stats_get_devmap_xmit_multi(r, p, nr_cpus, NULL,
1186 mask & SAMPLE_DEVMAP_XMIT_CNT);
1187
1188 if (sample_log_level & LL_DEFAULT ||
1189 ((sample_log_level & LL_SIMPLE) && sample_err_exp)) {
1190 sample_err_exp = false;
1191 printf("\n");
1192 }
1193 }
1194
1195 int sample_setup_maps(struct bpf_map **maps)
1196 {
1197 sample_n_cpus = libbpf_num_possible_cpus();
1198
1199 for (int i = 0; i < MAP_DEVMAP_XMIT_MULTI; i++) {
1200 sample_map[i] = maps[i];
1201
1202 switch (i) {
1203 case MAP_RX:
1204 case MAP_CPUMAP_KTHREAD:
1205 case MAP_DEVMAP_XMIT:
1206 sample_map_count[i] = sample_n_cpus;
1207 break;
1208 case MAP_REDIRECT_ERR:
1209 sample_map_count[i] =
1210 XDP_REDIRECT_ERR_MAX * sample_n_cpus;
1211 break;
1212 case MAP_EXCEPTION:
1213 sample_map_count[i] = XDP_ACTION_MAX * sample_n_cpus;
1214 case MAP_CPUMAP_ENQUEUE:
1215 sample_map_count[i] = sample_n_cpus * sample_n_cpus;
1216 break;
1217 default:
1218 return -EINVAL;
1219 }
1220 if (bpf_map__set_max_entries(sample_map[i], sample_map_count[i]) < 0)
1221 return -errno;
1222 }
1223 sample_map[MAP_DEVMAP_XMIT_MULTI] = maps[MAP_DEVMAP_XMIT_MULTI];
1224 return 0;
1225 }
1226
1227 static int sample_setup_maps_mappings(void)
1228 {
1229 for (int i = 0; i < MAP_DEVMAP_XMIT_MULTI; i++) {
1230 size_t size = sample_map_count[i] * sizeof(struct datarec);
1231
1232 sample_mmap[i] = mmap(NULL, size, PROT_READ | PROT_WRITE,
1233 MAP_SHARED, bpf_map__fd(sample_map[i]), 0);
1234 if (sample_mmap[i] == MAP_FAILED)
1235 return -errno;
1236 }
1237 return 0;
1238 }
1239
1240 int __sample_init(int mask)
1241 {
1242 sigset_t st;
1243
1244 sigemptyset(&st);
1245 sigaddset(&st, SIGQUIT);
1246 sigaddset(&st, SIGINT);
1247 sigaddset(&st, SIGTERM);
1248
1249 if (sigprocmask(SIG_BLOCK, &st, NULL) < 0)
1250 return -errno;
1251
1252 sample_sig_fd = signalfd(-1, &st, SFD_CLOEXEC | SFD_NONBLOCK);
1253 if (sample_sig_fd < 0)
1254 return -errno;
1255
1256 sample_mask = mask;
1257
1258 return sample_setup_maps_mappings();
1259 }
1260
1261 static int __sample_remove_xdp(int ifindex, __u32 prog_id, int xdp_flags)
1262 {
1263 __u32 cur_prog_id = 0;
1264 int ret;
1265
1266 if (prog_id) {
1267 ret = bpf_xdp_query_id(ifindex, xdp_flags, &cur_prog_id);
1268 if (ret < 0)
1269 return -errno;
1270
1271 if (prog_id != cur_prog_id) {
1272 print_always(
1273 "Program on ifindex %d does not match installed "
1274 "program, skipping unload\n",
1275 ifindex);
1276 return -ENOENT;
1277 }
1278 }
1279
1280 return bpf_xdp_detach(ifindex, xdp_flags, NULL);
1281 }
1282
1283 int sample_install_xdp(struct bpf_program *xdp_prog, int ifindex, bool generic,
1284 bool force)
1285 {
1286 int ret, xdp_flags = 0;
1287 __u32 prog_id = 0;
1288
1289 if (sample_xdp_cnt == 32) {
1290 fprintf(stderr,
1291 "Total limit for installed XDP programs in a sample reached\n");
1292 return -ENOTSUP;
1293 }
1294
1295 xdp_flags |= !force ? XDP_FLAGS_UPDATE_IF_NOEXIST : 0;
1296 xdp_flags |= generic ? XDP_FLAGS_SKB_MODE : XDP_FLAGS_DRV_MODE;
1297 ret = bpf_xdp_attach(ifindex, bpf_program__fd(xdp_prog), xdp_flags, NULL);
1298 if (ret < 0) {
1299 ret = -errno;
1300 fprintf(stderr,
1301 "Failed to install program \"%s\" on ifindex %d, mode = %s, "
1302 "force = %s: %s\n",
1303 bpf_program__name(xdp_prog), ifindex,
1304 generic ? "skb" : "native", force ? "true" : "false",
1305 strerror(-ret));
1306 return ret;
1307 }
1308
1309 ret = bpf_xdp_query_id(ifindex, xdp_flags, &prog_id);
1310 if (ret < 0) {
1311 ret = -errno;
1312 fprintf(stderr,
1313 "Failed to get XDP program id for ifindex %d, removing program: %s\n",
1314 ifindex, strerror(errno));
1315 __sample_remove_xdp(ifindex, 0, xdp_flags);
1316 return ret;
1317 }
1318 sample_xdp_progs[sample_xdp_cnt++] =
1319 (struct xdp_desc){ ifindex, prog_id, xdp_flags };
1320
1321 return 0;
1322 }
1323
1324 static void sample_summary_print(void)
1325 {
1326 double num = sample_out.rx_cnt.num;
1327
1328 if (sample_out.totals.rx) {
1329 double pkts = sample_out.totals.rx;
1330
1331 print_always(" Packets received : %'-10llu\n",
1332 sample_out.totals.rx);
1333 print_always(" Average packets/s : %'-10.0f\n",
1334 sample_round(pkts / num));
1335 }
1336 if (sample_out.totals.redir) {
1337 double pkts = sample_out.totals.redir;
1338
1339 print_always(" Packets redirected : %'-10llu\n",
1340 sample_out.totals.redir);
1341 print_always(" Average redir/s : %'-10.0f\n",
1342 sample_round(pkts / num));
1343 }
1344 if (sample_out.totals.drop)
1345 print_always(" Rx dropped : %'-10llu\n",
1346 sample_out.totals.drop);
1347 if (sample_out.totals.drop_xmit)
1348 print_always(" Tx dropped : %'-10llu\n",
1349 sample_out.totals.drop_xmit);
1350 if (sample_out.totals.err)
1351 print_always(" Errors recorded : %'-10llu\n",
1352 sample_out.totals.err);
1353 if (sample_out.totals.xmit) {
1354 double pkts = sample_out.totals.xmit;
1355
1356 print_always(" Packets transmitted : %'-10llu\n",
1357 sample_out.totals.xmit);
1358 print_always(" Average transmit/s : %'-10.0f\n",
1359 sample_round(pkts / num));
1360 }
1361 }
1362
1363 void sample_exit(int status)
1364 {
1365 size_t size;
1366
1367 for (int i = 0; i < NUM_MAP; i++) {
1368 size = sample_map_count[i] * sizeof(**sample_mmap);
1369 munmap(sample_mmap[i], size);
1370 }
1371 while (sample_xdp_cnt--) {
1372 int i = sample_xdp_cnt, ifindex, xdp_flags;
1373 __u32 prog_id;
1374
1375 prog_id = sample_xdp_progs[i].prog_id;
1376 ifindex = sample_xdp_progs[i].ifindex;
1377 xdp_flags = sample_xdp_progs[i].flags;
1378
1379 __sample_remove_xdp(ifindex, prog_id, xdp_flags);
1380 }
1381 sample_summary_print();
1382 close(sample_sig_fd);
1383 exit(status);
1384 }
1385
1386 static int sample_stats_collect(struct stats_record *rec)
1387 {
1388 int i;
1389
1390 if (sample_mask & SAMPLE_RX_CNT)
1391 map_collect_percpu(sample_mmap[MAP_RX], &rec->rx_cnt);
1392
1393 if (sample_mask & SAMPLE_REDIRECT_CNT)
1394 map_collect_percpu(sample_mmap[MAP_REDIRECT_ERR], &rec->redir_err[0]);
1395
1396 if (sample_mask & SAMPLE_REDIRECT_ERR_CNT) {
1397 for (i = 1; i < XDP_REDIRECT_ERR_MAX; i++)
1398 map_collect_percpu(&sample_mmap[MAP_REDIRECT_ERR][i * sample_n_cpus],
1399 &rec->redir_err[i]);
1400 }
1401
1402 if (sample_mask & SAMPLE_CPUMAP_ENQUEUE_CNT)
1403 for (i = 0; i < sample_n_cpus; i++)
1404 map_collect_percpu(&sample_mmap[MAP_CPUMAP_ENQUEUE][i * sample_n_cpus],
1405 &rec->enq[i]);
1406
1407 if (sample_mask & SAMPLE_CPUMAP_KTHREAD_CNT)
1408 map_collect_percpu(sample_mmap[MAP_CPUMAP_KTHREAD],
1409 &rec->kthread);
1410
1411 if (sample_mask & SAMPLE_EXCEPTION_CNT)
1412 for (i = 0; i < XDP_ACTION_MAX; i++)
1413 map_collect_percpu(&sample_mmap[MAP_EXCEPTION][i * sample_n_cpus],
1414 &rec->exception[i]);
1415
1416 if (sample_mask & SAMPLE_DEVMAP_XMIT_CNT)
1417 map_collect_percpu(sample_mmap[MAP_DEVMAP_XMIT], &rec->devmap_xmit);
1418
1419 if (sample_mask & SAMPLE_DEVMAP_XMIT_CNT_MULTI) {
1420 if (map_collect_percpu_devmap(bpf_map__fd(sample_map[MAP_DEVMAP_XMIT_MULTI]), rec) < 0)
1421 return -EINVAL;
1422 }
1423 return 0;
1424 }
1425
1426 static void sample_summary_update(struct sample_output *out)
1427 {
1428 sample_out.totals.rx += out->totals.rx;
1429 sample_out.totals.redir += out->totals.redir;
1430 sample_out.totals.drop += out->totals.drop;
1431 sample_out.totals.drop_xmit += out->totals.drop_xmit;
1432 sample_out.totals.err += out->totals.err;
1433 sample_out.totals.xmit += out->totals.xmit;
1434 sample_out.rx_cnt.num++;
1435 }
1436
1437 static void sample_stats_print(int mask, struct stats_record *cur,
1438 struct stats_record *prev, char *prog_name)
1439 {
1440 struct sample_output out = {};
1441
1442 if (mask & SAMPLE_RX_CNT)
1443 stats_get_rx_cnt(cur, prev, 0, &out);
1444 if (mask & SAMPLE_REDIRECT_CNT)
1445 stats_get_redirect_cnt(cur, prev, 0, &out);
1446 if (mask & SAMPLE_REDIRECT_ERR_CNT)
1447 stats_get_redirect_err_cnt(cur, prev, 0, &out);
1448 if (mask & SAMPLE_EXCEPTION_CNT)
1449 stats_get_exception_cnt(cur, prev, 0, &out);
1450 if (mask & SAMPLE_DEVMAP_XMIT_CNT)
1451 stats_get_devmap_xmit(cur, prev, 0, &out);
1452 else if (mask & SAMPLE_DEVMAP_XMIT_CNT_MULTI)
1453 stats_get_devmap_xmit_multi(cur, prev, 0, &out,
1454 mask & SAMPLE_DEVMAP_XMIT_CNT);
1455 sample_summary_update(&out);
1456
1457 stats_print(prog_name, mask, cur, prev, &out);
1458 }
1459
1460 void sample_switch_mode(void)
1461 {
1462 sample_log_level ^= LL_DEBUG - 1;
1463 }
1464
1465 static int sample_signal_cb(void)
1466 {
1467 struct signalfd_siginfo si;
1468 int r;
1469
1470 r = read(sample_sig_fd, &si, sizeof(si));
1471 if (r < 0)
1472 return -errno;
1473
1474 switch (si.ssi_signo) {
1475 case SIGQUIT:
1476 sample_switch_mode();
1477 printf("\n");
1478 break;
1479 default:
1480 printf("\n");
1481 return 1;
1482 }
1483
1484 return 0;
1485 }
1486
1487
1488 static void swap(struct stats_record **a, struct stats_record **b)
1489 {
1490 struct stats_record *tmp;
1491
1492 tmp = *a;
1493 *a = *b;
1494 *b = tmp;
1495 }
1496
1497 static int sample_timer_cb(int timerfd, struct stats_record **rec,
1498 struct stats_record **prev)
1499 {
1500 char line[64] = "Summary";
1501 int ret;
1502 __u64 t;
1503
1504 ret = read(timerfd, &t, sizeof(t));
1505 if (ret < 0)
1506 return -errno;
1507
1508 swap(prev, rec);
1509 ret = sample_stats_collect(*rec);
1510 if (ret < 0)
1511 return ret;
1512
1513 if (sample_xdp_cnt == 2 && !(sample_mask & SAMPLE_SKIP_HEADING)) {
1514 char fi[IFNAMSIZ];
1515 char to[IFNAMSIZ];
1516 const char *f, *t;
1517
1518 f = t = NULL;
1519 if (if_indextoname(sample_xdp_progs[0].ifindex, fi))
1520 f = fi;
1521 if (if_indextoname(sample_xdp_progs[1].ifindex, to))
1522 t = to;
1523
1524 snprintf(line, sizeof(line), "%s->%s", f ?: "?", t ?: "?");
1525 }
1526
1527 sample_stats_print(sample_mask, *rec, *prev, line);
1528 return 0;
1529 }
1530
1531 int sample_run(int interval, void (*post_cb)(void *), void *ctx)
1532 {
1533 struct timespec ts = { interval, 0 };
1534 struct itimerspec its = { ts, ts };
1535 struct stats_record *rec, *prev;
1536 struct pollfd pfd[2] = {};
1537 int timerfd, ret;
1538
1539 if (!interval) {
1540 fprintf(stderr, "Incorrect interval 0\n");
1541 return -EINVAL;
1542 }
1543 sample_interval = interval;
1544
1545 setlocale(LC_NUMERIC, "en_US.UTF-8");
1546
1547 timerfd = timerfd_create(CLOCK_MONOTONIC, TFD_CLOEXEC | TFD_NONBLOCK);
1548 if (timerfd < 0)
1549 return -errno;
1550 timerfd_settime(timerfd, 0, &its, NULL);
1551
1552 pfd[0].fd = sample_sig_fd;
1553 pfd[0].events = POLLIN;
1554
1555 pfd[1].fd = timerfd;
1556 pfd[1].events = POLLIN;
1557
1558 ret = -ENOMEM;
1559 rec = alloc_stats_record();
1560 if (!rec)
1561 goto end;
1562 prev = alloc_stats_record();
1563 if (!prev)
1564 goto end_rec;
1565
1566 ret = sample_stats_collect(rec);
1567 if (ret < 0)
1568 goto end_rec_prev;
1569
1570 for (;;) {
1571 ret = poll(pfd, 2, -1);
1572 if (ret < 0) {
1573 if (errno == EINTR)
1574 continue;
1575 else
1576 break;
1577 }
1578
1579 if (pfd[0].revents & POLLIN)
1580 ret = sample_signal_cb();
1581 else if (pfd[1].revents & POLLIN)
1582 ret = sample_timer_cb(timerfd, &rec, &prev);
1583
1584 if (ret)
1585 break;
1586
1587 if (post_cb)
1588 post_cb(ctx);
1589 }
1590
1591 end_rec_prev:
1592 free_stats_record(prev);
1593 end_rec:
1594 free_stats_record(rec);
1595 end:
1596 close(timerfd);
1597
1598 return ret;
1599 }
1600
1601 const char *get_driver_name(int ifindex)
1602 {
1603 struct ethtool_drvinfo drv = {};
1604 char ifname[IF_NAMESIZE];
1605 static char drvname[32];
1606 struct ifreq ifr = {};
1607 int fd, r = 0;
1608
1609 fd = socket(AF_INET, SOCK_DGRAM, 0);
1610 if (fd < 0)
1611 return "[error]";
1612
1613 if (!if_indextoname(ifindex, ifname))
1614 goto end;
1615
1616 drv.cmd = ETHTOOL_GDRVINFO;
1617 safe_strncpy(ifr.ifr_name, ifname, sizeof(ifr.ifr_name));
1618 ifr.ifr_data = (void *)&drv;
1619
1620 r = ioctl(fd, SIOCETHTOOL, &ifr);
1621 if (r)
1622 goto end;
1623
1624 safe_strncpy(drvname, drv.driver, sizeof(drvname));
1625
1626 close(fd);
1627 return drvname;
1628
1629 end:
1630 r = errno;
1631 close(fd);
1632 return r == EOPNOTSUPP ? "loopback" : "[error]";
1633 }
1634
1635 int get_mac_addr(int ifindex, void *mac_addr)
1636 {
1637 char ifname[IF_NAMESIZE];
1638 struct ifreq ifr = {};
1639 int fd, r;
1640
1641 fd = socket(AF_INET, SOCK_DGRAM, 0);
1642 if (fd < 0)
1643 return -errno;
1644
1645 if (!if_indextoname(ifindex, ifname)) {
1646 r = -errno;
1647 goto end;
1648 }
1649
1650 safe_strncpy(ifr.ifr_name, ifname, sizeof(ifr.ifr_name));
1651
1652 r = ioctl(fd, SIOCGIFHWADDR, &ifr);
1653 if (r) {
1654 r = -errno;
1655 goto end;
1656 }
1657
1658 memcpy(mac_addr, ifr.ifr_hwaddr.sa_data, 6 * sizeof(char));
1659
1660 end:
1661 close(fd);
1662 return r;
1663 }
1664
1665 __attribute__((constructor)) static void sample_ctor(void)
1666 {
1667 if (libbpf_set_strict_mode(LIBBPF_STRICT_ALL) < 0) {
1668 fprintf(stderr, "Failed to set libbpf strict mode: %s\n",
1669 strerror(errno));
1670
1671 exit(EXIT_FAIL_BPF);
1672 }
1673 }
