Back to home page

OSCL-LXR
 
 

     

0001 // SPDX-License-Identifier: GPL-2.0
0002 // Copyright (c) 2017-2018 Covalent IO, Inc. http://covalent.io
0003 #include <stdio.h>
0004 #include <stdlib.h>
0005 #include <sys/socket.h>
0006 #include <sys/ioctl.h>
0007 #include <sys/select.h>
0008 #include <netinet/in.h>
0009 #include <arpa/inet.h>
0010 #include <unistd.h>
0011 #include <string.h>
0012 #include <errno.h>
0013 #include <stdbool.h>
0014 #include <signal.h>
0015 #include <fcntl.h>
0016 #include <sys/wait.h>
0017 #include <time.h>
0018 #include <sched.h>
0019 
0020 #include <sys/time.h>
0021 #include <sys/types.h>
0022 #include <sys/sendfile.h>
0023 
0024 #include <linux/netlink.h>
0025 #include <linux/socket.h>
0026 #include <linux/sock_diag.h>
0027 #include <linux/bpf.h>
0028 #include <linux/if_link.h>
0029 #include <linux/tls.h>
0030 #include <assert.h>
0031 #include <libgen.h>
0032 
0033 #include <getopt.h>
0034 
0035 #include <bpf/bpf.h>
0036 #include <bpf/libbpf.h>
0037 
0038 #include "bpf_util.h"
0039 #include "cgroup_helpers.h"
0040 
0041 int running;
0042 static void running_handler(int a);
0043 
0044 #ifndef TCP_ULP
0045 # define TCP_ULP 31
0046 #endif
0047 #ifndef SOL_TLS
0048 # define SOL_TLS 282
0049 #endif
0050 
0051 /* randomly selected ports for testing on lo */
0052 #define S1_PORT 10000
0053 #define S2_PORT 10001
0054 
0055 #define BPF_SOCKMAP_FILENAME  "test_sockmap_kern.o"
0056 #define BPF_SOCKHASH_FILENAME "test_sockhash_kern.o"
0057 #define CG_PATH "/sockmap"
0058 
0059 /* global sockets */
0060 int s1, s2, c1, c2, p1, p2;
0061 int test_cnt;
0062 int passed;
0063 int failed;
0064 int map_fd[9];
0065 struct bpf_map *maps[9];
0066 int prog_fd[11];
0067 
0068 int txmsg_pass;
0069 int txmsg_redir;
0070 int txmsg_drop;
0071 int txmsg_apply;
0072 int txmsg_cork;
0073 int txmsg_start;
0074 int txmsg_end;
0075 int txmsg_start_push;
0076 int txmsg_end_push;
0077 int txmsg_start_pop;
0078 int txmsg_pop;
0079 int txmsg_ingress;
0080 int txmsg_redir_skb;
0081 int txmsg_ktls_skb;
0082 int txmsg_ktls_skb_drop;
0083 int txmsg_ktls_skb_redir;
0084 int ktls;
0085 int peek_flag;
0086 int skb_use_parser;
0087 int txmsg_omit_skb_parser;
0088 
0089 static const struct option long_options[] = {
0090     {"help",    no_argument,        NULL, 'h' },
0091     {"cgroup",  required_argument,  NULL, 'c' },
0092     {"rate",    required_argument,  NULL, 'r' },
0093     {"verbose", optional_argument,  NULL, 'v' },
0094     {"iov_count",   required_argument,  NULL, 'i' },
0095     {"length",  required_argument,  NULL, 'l' },
0096     {"test",    required_argument,  NULL, 't' },
0097     {"data_test",   no_argument,        NULL, 'd' },
0098     {"txmsg",       no_argument,    &txmsg_pass,  1  },
0099     {"txmsg_redir",     no_argument,    &txmsg_redir, 1  },
0100     {"txmsg_drop",      no_argument,    &txmsg_drop, 1 },
0101     {"txmsg_apply", required_argument,  NULL, 'a'},
0102     {"txmsg_cork",  required_argument,  NULL, 'k'},
0103     {"txmsg_start", required_argument,  NULL, 's'},
0104     {"txmsg_end",   required_argument,  NULL, 'e'},
0105     {"txmsg_start_push", required_argument, NULL, 'p'},
0106     {"txmsg_end_push",   required_argument, NULL, 'q'},
0107     {"txmsg_start_pop",  required_argument, NULL, 'w'},
0108     {"txmsg_pop",        required_argument, NULL, 'x'},
0109     {"txmsg_ingress", no_argument,      &txmsg_ingress, 1 },
0110     {"txmsg_redir_skb", no_argument,    &txmsg_redir_skb, 1 },
0111     {"ktls", no_argument,           &ktls, 1 },
0112     {"peek", no_argument,           &peek_flag, 1 },
0113     {"txmsg_omit_skb_parser", no_argument,      &txmsg_omit_skb_parser, 1},
0114     {"whitelist", required_argument,    NULL, 'n' },
0115     {"blacklist", required_argument,    NULL, 'b' },
0116     {0, 0, NULL, 0 }
0117 };
0118 
0119 struct test_env {
0120     const char *type;
0121     const char *subtest;
0122     const char *prepend;
0123 
0124     int test_num;
0125     int subtest_num;
0126 
0127     int succ_cnt;
0128     int fail_cnt;
0129     int fail_last;
0130 };
0131 
0132 struct test_env env;
0133 
0134 struct sockmap_options {
0135     int verbose;
0136     bool base;
0137     bool sendpage;
0138     bool data_test;
0139     bool drop_expected;
0140     bool check_recved_len;
0141     int iov_count;
0142     int iov_length;
0143     int rate;
0144     char *map;
0145     char *whitelist;
0146     char *blacklist;
0147     char *prepend;
0148 };
0149 
0150 struct _test {
0151     char *title;
0152     void (*tester)(int cg_fd, struct sockmap_options *opt);
0153 };
0154 
0155 static void test_start(void)
0156 {
0157     env.subtest_num++;
0158 }
0159 
0160 static void test_fail(void)
0161 {
0162     env.fail_cnt++;
0163 }
0164 
0165 static void test_pass(void)
0166 {
0167     env.succ_cnt++;
0168 }
0169 
0170 static void test_reset(void)
0171 {
0172     txmsg_start = txmsg_end = 0;
0173     txmsg_start_pop = txmsg_pop = 0;
0174     txmsg_start_push = txmsg_end_push = 0;
0175     txmsg_pass = txmsg_drop = txmsg_redir = 0;
0176     txmsg_apply = txmsg_cork = 0;
0177     txmsg_ingress = txmsg_redir_skb = 0;
0178     txmsg_ktls_skb = txmsg_ktls_skb_drop = txmsg_ktls_skb_redir = 0;
0179     txmsg_omit_skb_parser = 0;
0180     skb_use_parser = 0;
0181 }
0182 
0183 static int test_start_subtest(const struct _test *t, struct sockmap_options *o)
0184 {
0185     env.type = o->map;
0186     env.subtest = t->title;
0187     env.prepend = o->prepend;
0188     env.test_num++;
0189     env.subtest_num = 0;
0190     env.fail_last = env.fail_cnt;
0191     test_reset();
0192     return 0;
0193 }
0194 
0195 static void test_end_subtest(void)
0196 {
0197     int error = env.fail_cnt - env.fail_last;
0198     int type = strcmp(env.type, BPF_SOCKMAP_FILENAME);
0199 
0200     if (!error)
0201         test_pass();
0202 
0203     fprintf(stdout, "#%2d/%2d %8s:%s:%s:%s\n",
0204         env.test_num, env.subtest_num,
0205         !type ? "sockmap" : "sockhash",
0206         env.prepend ? : "",
0207         env.subtest, error ? "FAIL" : "OK");
0208 }
0209 
0210 static void test_print_results(void)
0211 {
0212     fprintf(stdout, "Pass: %d Fail: %d\n",
0213         env.succ_cnt, env.fail_cnt);
0214 }
0215 
0216 static void usage(char *argv[])
0217 {
0218     int i;
0219 
0220     printf(" Usage: %s --cgroup <cgroup_path>\n", argv[0]);
0221     printf(" options:\n");
0222     for (i = 0; long_options[i].name != 0; i++) {
0223         printf(" --%-12s", long_options[i].name);
0224         if (long_options[i].flag != NULL)
0225             printf(" flag (internal value:%d)\n",
0226                 *long_options[i].flag);
0227         else
0228             printf(" -%c\n", long_options[i].val);
0229     }
0230     printf("\n");
0231 }
0232 
0233 char *sock_to_string(int s)
0234 {
0235     if (s == c1)
0236         return "client1";
0237     else if (s == c2)
0238         return "client2";
0239     else if (s == s1)
0240         return "server1";
0241     else if (s == s2)
0242         return "server2";
0243     else if (s == p1)
0244         return "peer1";
0245     else if (s == p2)
0246         return "peer2";
0247     else
0248         return "unknown";
0249 }
0250 
0251 static int sockmap_init_ktls(int verbose, int s)
0252 {
0253     struct tls12_crypto_info_aes_gcm_128 tls_tx = {
0254         .info = {
0255             .version     = TLS_1_2_VERSION,
0256             .cipher_type = TLS_CIPHER_AES_GCM_128,
0257         },
0258     };
0259     struct tls12_crypto_info_aes_gcm_128 tls_rx = {
0260         .info = {
0261             .version     = TLS_1_2_VERSION,
0262             .cipher_type = TLS_CIPHER_AES_GCM_128,
0263         },
0264     };
0265     int so_buf = 6553500;
0266     int err;
0267 
0268     err = setsockopt(s, 6, TCP_ULP, "tls", sizeof("tls"));
0269     if (err) {
0270         fprintf(stderr, "setsockopt: TCP_ULP(%s) failed with error %i\n", sock_to_string(s), err);
0271         return -EINVAL;
0272     }
0273     err = setsockopt(s, SOL_TLS, TLS_TX, (void *)&tls_tx, sizeof(tls_tx));
0274     if (err) {
0275         fprintf(stderr, "setsockopt: TLS_TX(%s) failed with error %i\n", sock_to_string(s), err);
0276         return -EINVAL;
0277     }
0278     err = setsockopt(s, SOL_TLS, TLS_RX, (void *)&tls_rx, sizeof(tls_rx));
0279     if (err) {
0280         fprintf(stderr, "setsockopt: TLS_RX(%s) failed with error %i\n", sock_to_string(s), err);
0281         return -EINVAL;
0282     }
0283     err = setsockopt(s, SOL_SOCKET, SO_SNDBUF, &so_buf, sizeof(so_buf));
0284     if (err) {
0285         fprintf(stderr, "setsockopt: (%s) failed sndbuf with error %i\n", sock_to_string(s), err);
0286         return -EINVAL;
0287     }
0288     err = setsockopt(s, SOL_SOCKET, SO_RCVBUF, &so_buf, sizeof(so_buf));
0289     if (err) {
0290         fprintf(stderr, "setsockopt: (%s) failed rcvbuf with error %i\n", sock_to_string(s), err);
0291         return -EINVAL;
0292     }
0293 
0294     if (verbose)
0295         fprintf(stdout, "socket(%s) kTLS enabled\n", sock_to_string(s));
0296     return 0;
0297 }
0298 static int sockmap_init_sockets(int verbose)
0299 {
0300     int i, err, one = 1;
0301     struct sockaddr_in addr;
0302     int *fds[4] = {&s1, &s2, &c1, &c2};
0303 
0304     s1 = s2 = p1 = p2 = c1 = c2 = 0;
0305 
0306     /* Init sockets */
0307     for (i = 0; i < 4; i++) {
0308         *fds[i] = socket(AF_INET, SOCK_STREAM, 0);
0309         if (*fds[i] < 0) {
0310             perror("socket s1 failed()");
0311             return errno;
0312         }
0313     }
0314 
0315     /* Allow reuse */
0316     for (i = 0; i < 2; i++) {
0317         err = setsockopt(*fds[i], SOL_SOCKET, SO_REUSEADDR,
0318                  (char *)&one, sizeof(one));
0319         if (err) {
0320             perror("setsockopt failed()");
0321             return errno;
0322         }
0323     }
0324 
0325     /* Non-blocking sockets */
0326     for (i = 0; i < 2; i++) {
0327         err = ioctl(*fds[i], FIONBIO, (char *)&one);
0328         if (err < 0) {
0329             perror("ioctl s1 failed()");
0330             return errno;
0331         }
0332     }
0333 
0334     /* Bind server sockets */
0335     memset(&addr, 0, sizeof(struct sockaddr_in));
0336     addr.sin_family = AF_INET;
0337     addr.sin_addr.s_addr = inet_addr("127.0.0.1");
0338 
0339     addr.sin_port = htons(S1_PORT);
0340     err = bind(s1, (struct sockaddr *)&addr, sizeof(addr));
0341     if (err < 0) {
0342         perror("bind s1 failed()");
0343         return errno;
0344     }
0345 
0346     addr.sin_port = htons(S2_PORT);
0347     err = bind(s2, (struct sockaddr *)&addr, sizeof(addr));
0348     if (err < 0) {
0349         perror("bind s2 failed()");
0350         return errno;
0351     }
0352 
0353     /* Listen server sockets */
0354     addr.sin_port = htons(S1_PORT);
0355     err = listen(s1, 32);
0356     if (err < 0) {
0357         perror("listen s1 failed()");
0358         return errno;
0359     }
0360 
0361     addr.sin_port = htons(S2_PORT);
0362     err = listen(s2, 32);
0363     if (err < 0) {
0364         perror("listen s1 failed()");
0365         return errno;
0366     }
0367 
0368     /* Initiate Connect */
0369     addr.sin_port = htons(S1_PORT);
0370     err = connect(c1, (struct sockaddr *)&addr, sizeof(addr));
0371     if (err < 0 && errno != EINPROGRESS) {
0372         perror("connect c1 failed()");
0373         return errno;
0374     }
0375 
0376     addr.sin_port = htons(S2_PORT);
0377     err = connect(c2, (struct sockaddr *)&addr, sizeof(addr));
0378     if (err < 0 && errno != EINPROGRESS) {
0379         perror("connect c2 failed()");
0380         return errno;
0381     } else if (err < 0) {
0382         err = 0;
0383     }
0384 
0385     /* Accept Connecrtions */
0386     p1 = accept(s1, NULL, NULL);
0387     if (p1 < 0) {
0388         perror("accept s1 failed()");
0389         return errno;
0390     }
0391 
0392     p2 = accept(s2, NULL, NULL);
0393     if (p2 < 0) {
0394         perror("accept s1 failed()");
0395         return errno;
0396     }
0397 
0398     if (verbose > 1) {
0399         printf("connected sockets: c1 <-> p1, c2 <-> p2\n");
0400         printf("cgroups binding: c1(%i) <-> s1(%i) - - - c2(%i) <-> s2(%i)\n",
0401             c1, s1, c2, s2);
0402     }
0403     return 0;
0404 }
0405 
0406 struct msg_stats {
0407     size_t bytes_sent;
0408     size_t bytes_recvd;
0409     struct timespec start;
0410     struct timespec end;
0411 };
0412 
0413 static int msg_loop_sendpage(int fd, int iov_length, int cnt,
0414                  struct msg_stats *s,
0415                  struct sockmap_options *opt)
0416 {
0417     bool drop = opt->drop_expected;
0418     unsigned char k = 0;
0419     FILE *file;
0420     int i, fp;
0421 
0422     file = tmpfile();
0423     if (!file) {
0424         perror("create file for sendpage");
0425         return 1;
0426     }
0427     for (i = 0; i < iov_length * cnt; i++, k++)
0428         fwrite(&k, sizeof(char), 1, file);
0429     fflush(file);
0430     fseek(file, 0, SEEK_SET);
0431 
0432     fp = fileno(file);
0433 
0434     clock_gettime(CLOCK_MONOTONIC, &s->start);
0435     for (i = 0; i < cnt; i++) {
0436         int sent;
0437 
0438         errno = 0;
0439         sent = sendfile(fd, fp, NULL, iov_length);
0440 
0441         if (!drop && sent < 0) {
0442             perror("sendpage loop error");
0443             fclose(file);
0444             return sent;
0445         } else if (drop && sent >= 0) {
0446             printf("sendpage loop error expected: %i errno %i\n",
0447                    sent, errno);
0448             fclose(file);
0449             return -EIO;
0450         }
0451 
0452         if (sent > 0)
0453             s->bytes_sent += sent;
0454     }
0455     clock_gettime(CLOCK_MONOTONIC, &s->end);
0456     fclose(file);
0457     return 0;
0458 }
0459 
0460 static void msg_free_iov(struct msghdr *msg)
0461 {
0462     int i;
0463 
0464     for (i = 0; i < msg->msg_iovlen; i++)
0465         free(msg->msg_iov[i].iov_base);
0466     free(msg->msg_iov);
0467     msg->msg_iov = NULL;
0468     msg->msg_iovlen = 0;
0469 }
0470 
0471 static int msg_alloc_iov(struct msghdr *msg,
0472              int iov_count, int iov_length,
0473              bool data, bool xmit)
0474 {
0475     unsigned char k = 0;
0476     struct iovec *iov;
0477     int i;
0478 
0479     iov = calloc(iov_count, sizeof(struct iovec));
0480     if (!iov)
0481         return errno;
0482 
0483     for (i = 0; i < iov_count; i++) {
0484         unsigned char *d = calloc(iov_length, sizeof(char));
0485 
0486         if (!d) {
0487             fprintf(stderr, "iov_count %i/%i OOM\n", i, iov_count);
0488             goto unwind_iov;
0489         }
0490         iov[i].iov_base = d;
0491         iov[i].iov_len = iov_length;
0492 
0493         if (data && xmit) {
0494             int j;
0495 
0496             for (j = 0; j < iov_length; j++)
0497                 d[j] = k++;
0498         }
0499     }
0500 
0501     msg->msg_iov = iov;
0502     msg->msg_iovlen = iov_count;
0503 
0504     return 0;
0505 unwind_iov:
0506     for (i--; i >= 0 ; i--)
0507         free(msg->msg_iov[i].iov_base);
0508     return -ENOMEM;
0509 }
0510 
0511 static int msg_verify_data(struct msghdr *msg, int size, int chunk_sz)
0512 {
0513     int i, j = 0, bytes_cnt = 0;
0514     unsigned char k = 0;
0515 
0516     for (i = 0; i < msg->msg_iovlen; i++) {
0517         unsigned char *d = msg->msg_iov[i].iov_base;
0518 
0519         /* Special case test for skb ingress + ktls */
0520         if (i == 0 && txmsg_ktls_skb) {
0521             if (msg->msg_iov[i].iov_len < 4)
0522                 return -EIO;
0523             if (memcmp(d, "PASS", 4) != 0) {
0524                 fprintf(stderr,
0525                     "detected skb data error with skb ingress update @iov[%i]:%i \"%02x %02x %02x %02x\" != \"PASS\"\n",
0526                     i, 0, d[0], d[1], d[2], d[3]);
0527                 return -EIO;
0528             }
0529             j = 4; /* advance index past PASS header */
0530         }
0531 
0532         for (; j < msg->msg_iov[i].iov_len && size; j++) {
0533             if (d[j] != k++) {
0534                 fprintf(stderr,
0535                     "detected data corruption @iov[%i]:%i %02x != %02x, %02x ?= %02x\n",
0536                     i, j, d[j], k - 1, d[j+1], k);
0537                 return -EIO;
0538             }
0539             bytes_cnt++;
0540             if (bytes_cnt == chunk_sz) {
0541                 k = 0;
0542                 bytes_cnt = 0;
0543             }
0544             size--;
0545         }
0546     }
0547     return 0;
0548 }
0549 
0550 static int msg_loop(int fd, int iov_count, int iov_length, int cnt,
0551             struct msg_stats *s, bool tx,
0552             struct sockmap_options *opt)
0553 {
0554     struct msghdr msg = {0}, msg_peek = {0};
0555     int err, i, flags = MSG_NOSIGNAL;
0556     bool drop = opt->drop_expected;
0557     bool data = opt->data_test;
0558     int iov_alloc_length = iov_length;
0559 
0560     if (!tx && opt->check_recved_len)
0561         iov_alloc_length *= 2;
0562 
0563     err = msg_alloc_iov(&msg, iov_count, iov_alloc_length, data, tx);
0564     if (err)
0565         goto out_errno;
0566     if (peek_flag) {
0567         err = msg_alloc_iov(&msg_peek, iov_count, iov_length, data, tx);
0568         if (err)
0569             goto out_errno;
0570     }
0571 
0572     if (tx) {
0573         clock_gettime(CLOCK_MONOTONIC, &s->start);
0574         for (i = 0; i < cnt; i++) {
0575             int sent;
0576 
0577             errno = 0;
0578             sent = sendmsg(fd, &msg, flags);
0579 
0580             if (!drop && sent < 0) {
0581                 perror("sendmsg loop error");
0582                 goto out_errno;
0583             } else if (drop && sent >= 0) {
0584                 fprintf(stderr,
0585                     "sendmsg loop error expected: %i errno %i\n",
0586                     sent, errno);
0587                 errno = -EIO;
0588                 goto out_errno;
0589             }
0590             if (sent > 0)
0591                 s->bytes_sent += sent;
0592         }
0593         clock_gettime(CLOCK_MONOTONIC, &s->end);
0594     } else {
0595         int slct, recvp = 0, recv, max_fd = fd;
0596         float total_bytes, txmsg_pop_total;
0597         int fd_flags = O_NONBLOCK;
0598         struct timeval timeout;
0599         fd_set w;
0600 
0601         fcntl(fd, fd_flags);
0602         /* Account for pop bytes noting each iteration of apply will
0603          * call msg_pop_data helper so we need to account for this
0604          * by calculating the number of apply iterations. Note user
0605          * of the tool can create cases where no data is sent by
0606          * manipulating pop/push/pull/etc. For example txmsg_apply 1
0607          * with txmsg_pop 1 will try to apply 1B at a time but each
0608          * iteration will then pop 1B so no data will ever be sent.
0609          * This is really only useful for testing edge cases in code
0610          * paths.
0611          */
0612         total_bytes = (float)iov_count * (float)iov_length * (float)cnt;
0613         if (txmsg_apply)
0614             txmsg_pop_total = txmsg_pop * (total_bytes / txmsg_apply);
0615         else
0616             txmsg_pop_total = txmsg_pop * cnt;
0617         total_bytes -= txmsg_pop_total;
0618         err = clock_gettime(CLOCK_MONOTONIC, &s->start);
0619         if (err < 0)
0620             perror("recv start time");
0621         while (s->bytes_recvd < total_bytes) {
0622             if (txmsg_cork) {
0623                 timeout.tv_sec = 0;
0624                 timeout.tv_usec = 300000;
0625             } else {
0626                 timeout.tv_sec = 3;
0627                 timeout.tv_usec = 0;
0628             }
0629 
0630             /* FD sets */
0631             FD_ZERO(&w);
0632             FD_SET(fd, &w);
0633 
0634             slct = select(max_fd + 1, &w, NULL, NULL, &timeout);
0635             if (slct == -1) {
0636                 perror("select()");
0637                 clock_gettime(CLOCK_MONOTONIC, &s->end);
0638                 goto out_errno;
0639             } else if (!slct) {
0640                 if (opt->verbose)
0641                     fprintf(stderr, "unexpected timeout: recved %zu/%f pop_total %f\n", s->bytes_recvd, total_bytes, txmsg_pop_total);
0642                 errno = -EIO;
0643                 clock_gettime(CLOCK_MONOTONIC, &s->end);
0644                 goto out_errno;
0645             }
0646 
0647             errno = 0;
0648             if (peek_flag) {
0649                 flags |= MSG_PEEK;
0650                 recvp = recvmsg(fd, &msg_peek, flags);
0651                 if (recvp < 0) {
0652                     if (errno != EWOULDBLOCK) {
0653                         clock_gettime(CLOCK_MONOTONIC, &s->end);
0654                         goto out_errno;
0655                     }
0656                 }
0657                 flags = 0;
0658             }
0659 
0660             recv = recvmsg(fd, &msg, flags);
0661             if (recv < 0) {
0662                 if (errno != EWOULDBLOCK) {
0663                     clock_gettime(CLOCK_MONOTONIC, &s->end);
0664                     perror("recv failed()");
0665                     goto out_errno;
0666                 }
0667             }
0668 
0669             s->bytes_recvd += recv;
0670 
0671             if (opt->check_recved_len && s->bytes_recvd > total_bytes) {
0672                 errno = EMSGSIZE;
0673                 fprintf(stderr, "recv failed(), bytes_recvd:%zd, total_bytes:%f\n",
0674                         s->bytes_recvd, total_bytes);
0675                 goto out_errno;
0676             }
0677 
0678             if (data) {
0679                 int chunk_sz = opt->sendpage ?
0680                         iov_length * cnt :
0681                         iov_length * iov_count;
0682 
0683                 errno = msg_verify_data(&msg, recv, chunk_sz);
0684                 if (errno) {
0685                     perror("data verify msg failed");
0686                     goto out_errno;
0687                 }
0688                 if (recvp) {
0689                     errno = msg_verify_data(&msg_peek,
0690                                 recvp,
0691                                 chunk_sz);
0692                     if (errno) {
0693                         perror("data verify msg_peek failed");
0694                         goto out_errno;
0695                     }
0696                 }
0697             }
0698         }
0699         clock_gettime(CLOCK_MONOTONIC, &s->end);
0700     }
0701 
0702     msg_free_iov(&msg);
0703     msg_free_iov(&msg_peek);
0704     return err;
0705 out_errno:
0706     msg_free_iov(&msg);
0707     msg_free_iov(&msg_peek);
0708     return errno;
0709 }
0710 
0711 static float giga = 1000000000;
0712 
0713 static inline float sentBps(struct msg_stats s)
0714 {
0715     return s.bytes_sent / (s.end.tv_sec - s.start.tv_sec);
0716 }
0717 
0718 static inline float recvdBps(struct msg_stats s)
0719 {
0720     return s.bytes_recvd / (s.end.tv_sec - s.start.tv_sec);
0721 }
0722 
0723 static int sendmsg_test(struct sockmap_options *opt)
0724 {
0725     float sent_Bps = 0, recvd_Bps = 0;
0726     int rx_fd, txpid, rxpid, err = 0;
0727     struct msg_stats s = {0};
0728     int iov_count = opt->iov_count;
0729     int iov_buf = opt->iov_length;
0730     int rx_status, tx_status;
0731     int cnt = opt->rate;
0732 
0733     errno = 0;
0734 
0735     if (opt->base)
0736         rx_fd = p1;
0737     else
0738         rx_fd = p2;
0739 
0740     if (ktls) {
0741         /* Redirecting into non-TLS socket which sends into a TLS
0742          * socket is not a valid test. So in this case lets not
0743          * enable kTLS but still run the test.
0744          */
0745         if (!txmsg_redir || txmsg_ingress) {
0746             err = sockmap_init_ktls(opt->verbose, rx_fd);
0747             if (err)
0748                 return err;
0749         }
0750         err = sockmap_init_ktls(opt->verbose, c1);
0751         if (err)
0752             return err;
0753     }
0754 
0755     rxpid = fork();
0756     if (rxpid == 0) {
0757         if (txmsg_pop || txmsg_start_pop)
0758             iov_buf -= (txmsg_pop - txmsg_start_pop + 1);
0759         if (opt->drop_expected || txmsg_ktls_skb_drop)
0760             _exit(0);
0761 
0762         if (!iov_buf) /* zero bytes sent case */
0763             _exit(0);
0764 
0765         if (opt->sendpage)
0766             iov_count = 1;
0767         err = msg_loop(rx_fd, iov_count, iov_buf,
0768                    cnt, &s, false, opt);
0769         if (opt->verbose > 1)
0770             fprintf(stderr,
0771                 "msg_loop_rx: iov_count %i iov_buf %i cnt %i err %i\n",
0772                 iov_count, iov_buf, cnt, err);
0773         if (s.end.tv_sec - s.start.tv_sec) {
0774             sent_Bps = sentBps(s);
0775             recvd_Bps = recvdBps(s);
0776         }
0777         if (opt->verbose > 1)
0778             fprintf(stdout,
0779                 "rx_sendmsg: TX: %zuB %fB/s %fGB/s RX: %zuB %fB/s %fGB/s %s\n",
0780                 s.bytes_sent, sent_Bps, sent_Bps/giga,
0781                 s.bytes_recvd, recvd_Bps, recvd_Bps/giga,
0782                 peek_flag ? "(peek_msg)" : "");
0783         if (err && txmsg_cork)
0784             err = 0;
0785         exit(err ? 1 : 0);
0786     } else if (rxpid == -1) {
0787         perror("msg_loop_rx");
0788         return errno;
0789     }
0790 
0791     txpid = fork();
0792     if (txpid == 0) {
0793         if (opt->sendpage)
0794             err = msg_loop_sendpage(c1, iov_buf, cnt, &s, opt);
0795         else
0796             err = msg_loop(c1, iov_count, iov_buf,
0797                        cnt, &s, true, opt);
0798 
0799         if (err)
0800             fprintf(stderr,
0801                 "msg_loop_tx: iov_count %i iov_buf %i cnt %i err %i\n",
0802                 iov_count, iov_buf, cnt, err);
0803         if (s.end.tv_sec - s.start.tv_sec) {
0804             sent_Bps = sentBps(s);
0805             recvd_Bps = recvdBps(s);
0806         }
0807         if (opt->verbose > 1)
0808             fprintf(stdout,
0809                 "tx_sendmsg: TX: %zuB %fB/s %f GB/s RX: %zuB %fB/s %fGB/s\n",
0810                 s.bytes_sent, sent_Bps, sent_Bps/giga,
0811                 s.bytes_recvd, recvd_Bps, recvd_Bps/giga);
0812         exit(err ? 1 : 0);
0813     } else if (txpid == -1) {
0814         perror("msg_loop_tx");
0815         return errno;
0816     }
0817 
0818     assert(waitpid(rxpid, &rx_status, 0) == rxpid);
0819     assert(waitpid(txpid, &tx_status, 0) == txpid);
0820     if (WIFEXITED(rx_status)) {
0821         err = WEXITSTATUS(rx_status);
0822         if (err) {
0823             fprintf(stderr, "rx thread exited with err %d.\n", err);
0824             goto out;
0825         }
0826     }
0827     if (WIFEXITED(tx_status)) {
0828         err = WEXITSTATUS(tx_status);
0829         if (err)
0830             fprintf(stderr, "tx thread exited with err %d.\n", err);
0831     }
0832 out:
0833     return err;
0834 }
0835 
0836 static int forever_ping_pong(int rate, struct sockmap_options *opt)
0837 {
0838     struct timeval timeout;
0839     char buf[1024] = {0};
0840     int sc;
0841 
0842     timeout.tv_sec = 10;
0843     timeout.tv_usec = 0;
0844 
0845     /* Ping/Pong data from client to server */
0846     sc = send(c1, buf, sizeof(buf), 0);
0847     if (sc < 0) {
0848         perror("send failed()");
0849         return sc;
0850     }
0851 
0852     do {
0853         int s, rc, i, max_fd = p2;
0854         fd_set w;
0855 
0856         /* FD sets */
0857         FD_ZERO(&w);
0858         FD_SET(c1, &w);
0859         FD_SET(c2, &w);
0860         FD_SET(p1, &w);
0861         FD_SET(p2, &w);
0862 
0863         s = select(max_fd + 1, &w, NULL, NULL, &timeout);
0864         if (s == -1) {
0865             perror("select()");
0866             break;
0867         } else if (!s) {
0868             fprintf(stderr, "unexpected timeout\n");
0869             break;
0870         }
0871 
0872         for (i = 0; i <= max_fd && s > 0; ++i) {
0873             if (!FD_ISSET(i, &w))
0874                 continue;
0875 
0876             s--;
0877 
0878             rc = recv(i, buf, sizeof(buf), 0);
0879             if (rc < 0) {
0880                 if (errno != EWOULDBLOCK) {
0881                     perror("recv failed()");
0882                     return rc;
0883                 }
0884             }
0885 
0886             if (rc == 0) {
0887                 close(i);
0888                 break;
0889             }
0890 
0891             sc = send(i, buf, rc, 0);
0892             if (sc < 0) {
0893                 perror("send failed()");
0894                 return sc;
0895             }
0896         }
0897 
0898         if (rate)
0899             sleep(rate);
0900 
0901         if (opt->verbose) {
0902             printf(".");
0903             fflush(stdout);
0904 
0905         }
0906     } while (running);
0907 
0908     return 0;
0909 }
0910 
0911 enum {
0912     SELFTESTS,
0913     PING_PONG,
0914     SENDMSG,
0915     BASE,
0916     BASE_SENDPAGE,
0917     SENDPAGE,
0918 };
0919 
0920 static int run_options(struct sockmap_options *options, int cg_fd,  int test)
0921 {
0922     int i, key, next_key, err, tx_prog_fd = -1, zero = 0;
0923 
0924     /* If base test skip BPF setup */
0925     if (test == BASE || test == BASE_SENDPAGE)
0926         goto run;
0927 
0928     /* Attach programs to sockmap */
0929     if (!txmsg_omit_skb_parser) {
0930         err = bpf_prog_attach(prog_fd[0], map_fd[0],
0931                       BPF_SK_SKB_STREAM_PARSER, 0);
0932         if (err) {
0933             fprintf(stderr,
0934                 "ERROR: bpf_prog_attach (sockmap %i->%i): %d (%s)\n",
0935                 prog_fd[0], map_fd[0], err, strerror(errno));
0936             return err;
0937         }
0938     }
0939 
0940     err = bpf_prog_attach(prog_fd[1], map_fd[0],
0941                 BPF_SK_SKB_STREAM_VERDICT, 0);
0942     if (err) {
0943         fprintf(stderr, "ERROR: bpf_prog_attach (sockmap): %d (%s)\n",
0944             err, strerror(errno));
0945         return err;
0946     }
0947 
0948     /* Attach programs to TLS sockmap */
0949     if (txmsg_ktls_skb) {
0950         if (!txmsg_omit_skb_parser) {
0951             err = bpf_prog_attach(prog_fd[0], map_fd[8],
0952                           BPF_SK_SKB_STREAM_PARSER, 0);
0953             if (err) {
0954                 fprintf(stderr,
0955                     "ERROR: bpf_prog_attach (TLS sockmap %i->%i): %d (%s)\n",
0956                     prog_fd[0], map_fd[8], err, strerror(errno));
0957                 return err;
0958             }
0959         }
0960 
0961         err = bpf_prog_attach(prog_fd[2], map_fd[8],
0962                       BPF_SK_SKB_STREAM_VERDICT, 0);
0963         if (err) {
0964             fprintf(stderr, "ERROR: bpf_prog_attach (TLS sockmap): %d (%s)\n",
0965                 err, strerror(errno));
0966             return err;
0967         }
0968     }
0969 
0970     /* Attach to cgroups */
0971     err = bpf_prog_attach(prog_fd[3], cg_fd, BPF_CGROUP_SOCK_OPS, 0);
0972     if (err) {
0973         fprintf(stderr, "ERROR: bpf_prog_attach (groups): %d (%s)\n",
0974             err, strerror(errno));
0975         return err;
0976     }
0977 
0978 run:
0979     err = sockmap_init_sockets(options->verbose);
0980     if (err) {
0981         fprintf(stderr, "ERROR: test socket failed: %d\n", err);
0982         goto out;
0983     }
0984 
0985     /* Attach txmsg program to sockmap */
0986     if (txmsg_pass)
0987         tx_prog_fd = prog_fd[4];
0988     else if (txmsg_redir)
0989         tx_prog_fd = prog_fd[5];
0990     else if (txmsg_apply)
0991         tx_prog_fd = prog_fd[6];
0992     else if (txmsg_cork)
0993         tx_prog_fd = prog_fd[7];
0994     else if (txmsg_drop)
0995         tx_prog_fd = prog_fd[8];
0996     else
0997         tx_prog_fd = 0;
0998 
0999     if (tx_prog_fd) {
1000         int redir_fd, i = 0;
1001 
1002         err = bpf_prog_attach(tx_prog_fd,
1003                       map_fd[1], BPF_SK_MSG_VERDICT, 0);
1004         if (err) {
1005             fprintf(stderr,
1006                 "ERROR: bpf_prog_attach (txmsg): %d (%s)\n",
1007                 err, strerror(errno));
1008             goto out;
1009         }
1010 
1011         err = bpf_map_update_elem(map_fd[1], &i, &c1, BPF_ANY);
1012         if (err) {
1013             fprintf(stderr,
1014                 "ERROR: bpf_map_update_elem (txmsg):  %d (%s\n",
1015                 err, strerror(errno));
1016             goto out;
1017         }
1018 
1019         if (txmsg_redir)
1020             redir_fd = c2;
1021         else
1022             redir_fd = c1;
1023 
1024         err = bpf_map_update_elem(map_fd[2], &i, &redir_fd, BPF_ANY);
1025         if (err) {
1026             fprintf(stderr,
1027                 "ERROR: bpf_map_update_elem (txmsg):  %d (%s\n",
1028                 err, strerror(errno));
1029             goto out;
1030         }
1031 
1032         if (txmsg_apply) {
1033             err = bpf_map_update_elem(map_fd[3],
1034                           &i, &txmsg_apply, BPF_ANY);
1035             if (err) {
1036                 fprintf(stderr,
1037                     "ERROR: bpf_map_update_elem (apply_bytes):  %d (%s\n",
1038                     err, strerror(errno));
1039                 goto out;
1040             }
1041         }
1042 
1043         if (txmsg_cork) {
1044             err = bpf_map_update_elem(map_fd[4],
1045                           &i, &txmsg_cork, BPF_ANY);
1046             if (err) {
1047                 fprintf(stderr,
1048                     "ERROR: bpf_map_update_elem (cork_bytes):  %d (%s\n",
1049                     err, strerror(errno));
1050                 goto out;
1051             }
1052         }
1053 
1054         if (txmsg_start) {
1055             err = bpf_map_update_elem(map_fd[5],
1056                           &i, &txmsg_start, BPF_ANY);
1057             if (err) {
1058                 fprintf(stderr,
1059                     "ERROR: bpf_map_update_elem (txmsg_start):  %d (%s)\n",
1060                     err, strerror(errno));
1061                 goto out;
1062             }
1063         }
1064 
1065         if (txmsg_end) {
1066             i = 1;
1067             err = bpf_map_update_elem(map_fd[5],
1068                           &i, &txmsg_end, BPF_ANY);
1069             if (err) {
1070                 fprintf(stderr,
1071                     "ERROR: bpf_map_update_elem (txmsg_end):  %d (%s)\n",
1072                     err, strerror(errno));
1073                 goto out;
1074             }
1075         }
1076 
1077         if (txmsg_start_push) {
1078             i = 2;
1079             err = bpf_map_update_elem(map_fd[5],
1080                           &i, &txmsg_start_push, BPF_ANY);
1081             if (err) {
1082                 fprintf(stderr,
1083                     "ERROR: bpf_map_update_elem (txmsg_start_push):  %d (%s)\n",
1084                     err, strerror(errno));
1085                 goto out;
1086             }
1087         }
1088 
1089         if (txmsg_end_push) {
1090             i = 3;
1091             err = bpf_map_update_elem(map_fd[5],
1092                           &i, &txmsg_end_push, BPF_ANY);
1093             if (err) {
1094                 fprintf(stderr,
1095                     "ERROR: bpf_map_update_elem %i@%i (txmsg_end_push):  %d (%s)\n",
1096                     txmsg_end_push, i, err, strerror(errno));
1097                 goto out;
1098             }
1099         }
1100 
1101         if (txmsg_start_pop) {
1102             i = 4;
1103             err = bpf_map_update_elem(map_fd[5],
1104                           &i, &txmsg_start_pop, BPF_ANY);
1105             if (err) {
1106                 fprintf(stderr,
1107                     "ERROR: bpf_map_update_elem %i@%i (txmsg_start_pop):  %d (%s)\n",
1108                     txmsg_start_pop, i, err, strerror(errno));
1109                 goto out;
1110             }
1111         } else {
1112             i = 4;
1113             bpf_map_update_elem(map_fd[5],
1114                           &i, &txmsg_start_pop, BPF_ANY);
1115         }
1116 
1117         if (txmsg_pop) {
1118             i = 5;
1119             err = bpf_map_update_elem(map_fd[5],
1120                           &i, &txmsg_pop, BPF_ANY);
1121             if (err) {
1122                 fprintf(stderr,
1123                     "ERROR: bpf_map_update_elem %i@%i (txmsg_pop):  %d (%s)\n",
1124                     txmsg_pop, i, err, strerror(errno));
1125                 goto out;
1126             }
1127         } else {
1128             i = 5;
1129             bpf_map_update_elem(map_fd[5],
1130                         &i, &txmsg_pop, BPF_ANY);
1131 
1132         }
1133 
1134         if (txmsg_ingress) {
1135             int in = BPF_F_INGRESS;
1136 
1137             i = 0;
1138             err = bpf_map_update_elem(map_fd[6], &i, &in, BPF_ANY);
1139             if (err) {
1140                 fprintf(stderr,
1141                     "ERROR: bpf_map_update_elem (txmsg_ingress): %d (%s)\n",
1142                     err, strerror(errno));
1143             }
1144             i = 1;
1145             err = bpf_map_update_elem(map_fd[1], &i, &p1, BPF_ANY);
1146             if (err) {
1147                 fprintf(stderr,
1148                     "ERROR: bpf_map_update_elem (p1 txmsg): %d (%s)\n",
1149                     err, strerror(errno));
1150             }
1151             err = bpf_map_update_elem(map_fd[2], &i, &p1, BPF_ANY);
1152             if (err) {
1153                 fprintf(stderr,
1154                     "ERROR: bpf_map_update_elem (p1 redir): %d (%s)\n",
1155                     err, strerror(errno));
1156             }
1157 
1158             i = 2;
1159             err = bpf_map_update_elem(map_fd[2], &i, &p2, BPF_ANY);
1160             if (err) {
1161                 fprintf(stderr,
1162                     "ERROR: bpf_map_update_elem (p2 txmsg): %d (%s)\n",
1163                     err, strerror(errno));
1164             }
1165         }
1166 
1167         if (txmsg_ktls_skb) {
1168             int ingress = BPF_F_INGRESS;
1169 
1170             i = 0;
1171             err = bpf_map_update_elem(map_fd[8], &i, &p2, BPF_ANY);
1172             if (err) {
1173                 fprintf(stderr,
1174                     "ERROR: bpf_map_update_elem (c1 sockmap): %d (%s)\n",
1175                     err, strerror(errno));
1176             }
1177 
1178             if (txmsg_ktls_skb_redir) {
1179                 i = 1;
1180                 err = bpf_map_update_elem(map_fd[7],
1181                               &i, &ingress, BPF_ANY);
1182                 if (err) {
1183                     fprintf(stderr,
1184                         "ERROR: bpf_map_update_elem (txmsg_ingress): %d (%s)\n",
1185                         err, strerror(errno));
1186                 }
1187             }
1188 
1189             if (txmsg_ktls_skb_drop) {
1190                 i = 1;
1191                 err = bpf_map_update_elem(map_fd[7], &i, &i, BPF_ANY);
1192             }
1193         }
1194 
1195         if (txmsg_redir_skb) {
1196             int skb_fd = (test == SENDMSG || test == SENDPAGE) ?
1197                     p2 : p1;
1198             int ingress = BPF_F_INGRESS;
1199 
1200             i = 0;
1201             err = bpf_map_update_elem(map_fd[7],
1202                           &i, &ingress, BPF_ANY);
1203             if (err) {
1204                 fprintf(stderr,
1205                     "ERROR: bpf_map_update_elem (txmsg_ingress): %d (%s)\n",
1206                     err, strerror(errno));
1207             }
1208 
1209             i = 3;
1210             err = bpf_map_update_elem(map_fd[0], &i, &skb_fd, BPF_ANY);
1211             if (err) {
1212                 fprintf(stderr,
1213                     "ERROR: bpf_map_update_elem (c1 sockmap): %d (%s)\n",
1214                     err, strerror(errno));
1215             }
1216         }
1217     }
1218 
1219     if (skb_use_parser) {
1220         i = 2;
1221         err = bpf_map_update_elem(map_fd[7], &i, &skb_use_parser, BPF_ANY);
1222     }
1223 
1224     if (txmsg_drop)
1225         options->drop_expected = true;
1226 
1227     if (test == PING_PONG)
1228         err = forever_ping_pong(options->rate, options);
1229     else if (test == SENDMSG) {
1230         options->base = false;
1231         options->sendpage = false;
1232         err = sendmsg_test(options);
1233     } else if (test == SENDPAGE) {
1234         options->base = false;
1235         options->sendpage = true;
1236         err = sendmsg_test(options);
1237     } else if (test == BASE) {
1238         options->base = true;
1239         options->sendpage = false;
1240         err = sendmsg_test(options);
1241     } else if (test == BASE_SENDPAGE) {
1242         options->base = true;
1243         options->sendpage = true;
1244         err = sendmsg_test(options);
1245     } else
1246         fprintf(stderr, "unknown test\n");
1247 out:
1248     /* Detatch and zero all the maps */
1249     bpf_prog_detach2(prog_fd[3], cg_fd, BPF_CGROUP_SOCK_OPS);
1250     bpf_prog_detach2(prog_fd[0], map_fd[0], BPF_SK_SKB_STREAM_PARSER);
1251     bpf_prog_detach2(prog_fd[1], map_fd[0], BPF_SK_SKB_STREAM_VERDICT);
1252     bpf_prog_detach2(prog_fd[0], map_fd[8], BPF_SK_SKB_STREAM_PARSER);
1253     bpf_prog_detach2(prog_fd[2], map_fd[8], BPF_SK_SKB_STREAM_VERDICT);
1254 
1255     if (tx_prog_fd >= 0)
1256         bpf_prog_detach2(tx_prog_fd, map_fd[1], BPF_SK_MSG_VERDICT);
1257 
1258     for (i = 0; i < 8; i++) {
1259         key = next_key = 0;
1260         bpf_map_update_elem(map_fd[i], &key, &zero, BPF_ANY);
1261         while (bpf_map_get_next_key(map_fd[i], &key, &next_key) == 0) {
1262             bpf_map_update_elem(map_fd[i], &key, &zero, BPF_ANY);
1263             key = next_key;
1264         }
1265     }
1266 
1267     close(s1);
1268     close(s2);
1269     close(p1);
1270     close(p2);
1271     close(c1);
1272     close(c2);
1273     return err;
1274 }
1275 
1276 static char *test_to_str(int test)
1277 {
1278     switch (test) {
1279     case SENDMSG:
1280         return "sendmsg";
1281     case SENDPAGE:
1282         return "sendpage";
1283     }
1284     return "unknown";
1285 }
1286 
1287 static void append_str(char *dst, const char *src, size_t dst_cap)
1288 {
1289     size_t avail = dst_cap - strlen(dst);
1290 
1291     if (avail <= 1) /* just zero byte could be written */
1292         return;
1293 
1294     strncat(dst, src, avail - 1); /* strncat() adds + 1 for zero byte */
1295 }
1296 
1297 #define OPTSTRING 60
1298 static void test_options(char *options)
1299 {
1300     char tstr[OPTSTRING];
1301 
1302     memset(options, 0, OPTSTRING);
1303 
1304     if (txmsg_pass)
1305         append_str(options, "pass,", OPTSTRING);
1306     if (txmsg_redir)
1307         append_str(options, "redir,", OPTSTRING);
1308     if (txmsg_drop)
1309         append_str(options, "drop,", OPTSTRING);
1310     if (txmsg_apply) {
1311         snprintf(tstr, OPTSTRING, "apply %d,", txmsg_apply);
1312         append_str(options, tstr, OPTSTRING);
1313     }
1314     if (txmsg_cork) {
1315         snprintf(tstr, OPTSTRING, "cork %d,", txmsg_cork);
1316         append_str(options, tstr, OPTSTRING);
1317     }
1318     if (txmsg_start) {
1319         snprintf(tstr, OPTSTRING, "start %d,", txmsg_start);
1320         append_str(options, tstr, OPTSTRING);
1321     }
1322     if (txmsg_end) {
1323         snprintf(tstr, OPTSTRING, "end %d,", txmsg_end);
1324         append_str(options, tstr, OPTSTRING);
1325     }
1326     if (txmsg_start_pop) {
1327         snprintf(tstr, OPTSTRING, "pop (%d,%d),",
1328              txmsg_start_pop, txmsg_start_pop + txmsg_pop);
1329         append_str(options, tstr, OPTSTRING);
1330     }
1331     if (txmsg_ingress)
1332         append_str(options, "ingress,", OPTSTRING);
1333     if (txmsg_redir_skb)
1334         append_str(options, "redir_skb,", OPTSTRING);
1335     if (txmsg_ktls_skb)
1336         append_str(options, "ktls_skb,", OPTSTRING);
1337     if (ktls)
1338         append_str(options, "ktls,", OPTSTRING);
1339     if (peek_flag)
1340         append_str(options, "peek,", OPTSTRING);
1341 }
1342 
1343 static int __test_exec(int cgrp, int test, struct sockmap_options *opt)
1344 {
1345     char *options = calloc(OPTSTRING, sizeof(char));
1346     int err;
1347 
1348     if (test == SENDPAGE)
1349         opt->sendpage = true;
1350     else
1351         opt->sendpage = false;
1352 
1353     if (txmsg_drop)
1354         opt->drop_expected = true;
1355     else
1356         opt->drop_expected = false;
1357 
1358     test_options(options);
1359 
1360     if (opt->verbose) {
1361         fprintf(stdout,
1362             " [TEST %i]: (%i, %i, %i, %s, %s): ",
1363             test_cnt, opt->rate, opt->iov_count, opt->iov_length,
1364             test_to_str(test), options);
1365         fflush(stdout);
1366     }
1367     err = run_options(opt, cgrp, test);
1368     if (opt->verbose)
1369         fprintf(stdout, " %s\n", !err ? "PASS" : "FAILED");
1370     test_cnt++;
1371     !err ? passed++ : failed++;
1372     free(options);
1373     return err;
1374 }
1375 
1376 static void test_exec(int cgrp, struct sockmap_options *opt)
1377 {
1378     int type = strcmp(opt->map, BPF_SOCKMAP_FILENAME);
1379     int err;
1380 
1381     if (type == 0) {
1382         test_start();
1383         err = __test_exec(cgrp, SENDMSG, opt);
1384         if (err)
1385             test_fail();
1386     } else {
1387         test_start();
1388         err = __test_exec(cgrp, SENDPAGE, opt);
1389         if (err)
1390             test_fail();
1391     }
1392 }
1393 
1394 static void test_send_one(struct sockmap_options *opt, int cgrp)
1395 {
1396     opt->iov_length = 1;
1397     opt->iov_count = 1;
1398     opt->rate = 1;
1399     test_exec(cgrp, opt);
1400 
1401     opt->iov_length = 1;
1402     opt->iov_count = 1024;
1403     opt->rate = 1;
1404     test_exec(cgrp, opt);
1405 
1406     opt->iov_length = 1024;
1407     opt->iov_count = 1;
1408     opt->rate = 1;
1409     test_exec(cgrp, opt);
1410 
1411 }
1412 
1413 static void test_send_many(struct sockmap_options *opt, int cgrp)
1414 {
1415     opt->iov_length = 3;
1416     opt->iov_count = 1;
1417     opt->rate = 512;
1418     test_exec(cgrp, opt);
1419 
1420     opt->rate = 100;
1421     opt->iov_count = 1;
1422     opt->iov_length = 5;
1423     test_exec(cgrp, opt);
1424 }
1425 
1426 static void test_send_large(struct sockmap_options *opt, int cgrp)
1427 {
1428     opt->iov_length = 256;
1429     opt->iov_count = 1024;
1430     opt->rate = 2;
1431     test_exec(cgrp, opt);
1432 }
1433 
1434 static void test_send(struct sockmap_options *opt, int cgrp)
1435 {
1436     test_send_one(opt, cgrp);
1437     test_send_many(opt, cgrp);
1438     test_send_large(opt, cgrp);
1439     sched_yield();
1440 }
1441 
1442 static void test_txmsg_pass(int cgrp, struct sockmap_options *opt)
1443 {
1444     /* Test small and large iov_count values with pass/redir/apply/cork */
1445     txmsg_pass = 1;
1446     test_send(opt, cgrp);
1447 }
1448 
1449 static void test_txmsg_redir(int cgrp, struct sockmap_options *opt)
1450 {
1451     txmsg_redir = 1;
1452     test_send(opt, cgrp);
1453 }
1454 
1455 static void test_txmsg_drop(int cgrp, struct sockmap_options *opt)
1456 {
1457     txmsg_drop = 1;
1458     test_send(opt, cgrp);
1459 }
1460 
1461 static void test_txmsg_ingress_redir(int cgrp, struct sockmap_options *opt)
1462 {
1463     txmsg_pass = txmsg_drop = 0;
1464     txmsg_ingress = txmsg_redir = 1;
1465     test_send(opt, cgrp);
1466 }
1467 
1468 static void test_txmsg_skb(int cgrp, struct sockmap_options *opt)
1469 {
1470     bool data = opt->data_test;
1471     int k = ktls;
1472 
1473     opt->data_test = true;
1474     ktls = 1;
1475 
1476     txmsg_pass = txmsg_drop = 0;
1477     txmsg_ingress = txmsg_redir = 0;
1478     txmsg_ktls_skb = 1;
1479     txmsg_pass = 1;
1480 
1481     /* Using data verification so ensure iov layout is
1482      * expected from test receiver side. e.g. has enough
1483      * bytes to write test code.
1484      */
1485     opt->iov_length = 100;
1486     opt->iov_count = 1;
1487     opt->rate = 1;
1488     test_exec(cgrp, opt);
1489 
1490     txmsg_ktls_skb_drop = 1;
1491     test_exec(cgrp, opt);
1492 
1493     txmsg_ktls_skb_drop = 0;
1494     txmsg_ktls_skb_redir = 1;
1495     test_exec(cgrp, opt);
1496     txmsg_ktls_skb_redir = 0;
1497 
1498     /* Tests that omit skb_parser */
1499     txmsg_omit_skb_parser = 1;
1500     ktls = 0;
1501     txmsg_ktls_skb = 0;
1502     test_exec(cgrp, opt);
1503 
1504     txmsg_ktls_skb_drop = 1;
1505     test_exec(cgrp, opt);
1506     txmsg_ktls_skb_drop = 0;
1507 
1508     txmsg_ktls_skb_redir = 1;
1509     test_exec(cgrp, opt);
1510 
1511     ktls = 1;
1512     test_exec(cgrp, opt);
1513     txmsg_omit_skb_parser = 0;
1514 
1515     opt->data_test = data;
1516     ktls = k;
1517 }
1518 
1519 /* Test cork with hung data. This tests poor usage patterns where
1520  * cork can leave data on the ring if user program is buggy and
1521  * doesn't flush them somehow. They do take some time however
1522  * because they wait for a timeout. Test pass, redir and cork with
1523  * apply logic. Use cork size of 4097 with send_large to avoid
1524  * aligning cork size with send size.
1525  */
1526 static void test_txmsg_cork_hangs(int cgrp, struct sockmap_options *opt)
1527 {
1528     txmsg_pass = 1;
1529     txmsg_redir = 0;
1530     txmsg_cork = 4097;
1531     txmsg_apply = 4097;
1532     test_send_large(opt, cgrp);
1533 
1534     txmsg_pass = 0;
1535     txmsg_redir = 1;
1536     txmsg_apply = 0;
1537     txmsg_cork = 4097;
1538     test_send_large(opt, cgrp);
1539 
1540     txmsg_pass = 0;
1541     txmsg_redir = 1;
1542     txmsg_apply = 4097;
1543     txmsg_cork = 4097;
1544     test_send_large(opt, cgrp);
1545 }
1546 
1547 static void test_txmsg_pull(int cgrp, struct sockmap_options *opt)
1548 {
1549     /* Test basic start/end */
1550     txmsg_start = 1;
1551     txmsg_end = 2;
1552     test_send(opt, cgrp);
1553 
1554     /* Test >4k pull */
1555     txmsg_start = 4096;
1556     txmsg_end = 9182;
1557     test_send_large(opt, cgrp);
1558 
1559     /* Test pull + redirect */
1560     txmsg_redir = 0;
1561     txmsg_start = 1;
1562     txmsg_end = 2;
1563     test_send(opt, cgrp);
1564 
1565     /* Test pull + cork */
1566     txmsg_redir = 0;
1567     txmsg_cork = 512;
1568     txmsg_start = 1;
1569     txmsg_end = 2;
1570     test_send_many(opt, cgrp);
1571 
1572     /* Test pull + cork + redirect */
1573     txmsg_redir = 1;
1574     txmsg_cork = 512;
1575     txmsg_start = 1;
1576     txmsg_end = 2;
1577     test_send_many(opt, cgrp);
1578 }
1579 
1580 static void test_txmsg_pop(int cgrp, struct sockmap_options *opt)
1581 {
1582     /* Test basic pop */
1583     txmsg_start_pop = 1;
1584     txmsg_pop = 2;
1585     test_send_many(opt, cgrp);
1586 
1587     /* Test pop with >4k */
1588     txmsg_start_pop = 4096;
1589     txmsg_pop = 4096;
1590     test_send_large(opt, cgrp);
1591 
1592     /* Test pop + redirect */
1593     txmsg_redir = 1;
1594     txmsg_start_pop = 1;
1595     txmsg_pop = 2;
1596     test_send_many(opt, cgrp);
1597 
1598     /* Test pop + cork */
1599     txmsg_redir = 0;
1600     txmsg_cork = 512;
1601     txmsg_start_pop = 1;
1602     txmsg_pop = 2;
1603     test_send_many(opt, cgrp);
1604 
1605     /* Test pop + redirect + cork */
1606     txmsg_redir = 1;
1607     txmsg_cork = 4;
1608     txmsg_start_pop = 1;
1609     txmsg_pop = 2;
1610     test_send_many(opt, cgrp);
1611 }
1612 
1613 static void test_txmsg_push(int cgrp, struct sockmap_options *opt)
1614 {
1615     /* Test basic push */
1616     txmsg_start_push = 1;
1617     txmsg_end_push = 1;
1618     test_send(opt, cgrp);
1619 
1620     /* Test push 4kB >4k */
1621     txmsg_start_push = 4096;
1622     txmsg_end_push = 4096;
1623     test_send_large(opt, cgrp);
1624 
1625     /* Test push + redirect */
1626     txmsg_redir = 1;
1627     txmsg_start_push = 1;
1628     txmsg_end_push = 2;
1629     test_send_many(opt, cgrp);
1630 
1631     /* Test push + cork */
1632     txmsg_redir = 0;
1633     txmsg_cork = 512;
1634     txmsg_start_push = 1;
1635     txmsg_end_push = 2;
1636     test_send_many(opt, cgrp);
1637 }
1638 
1639 static void test_txmsg_push_pop(int cgrp, struct sockmap_options *opt)
1640 {
1641     txmsg_start_push = 1;
1642     txmsg_end_push = 10;
1643     txmsg_start_pop = 5;
1644     txmsg_pop = 4;
1645     test_send_large(opt, cgrp);
1646 }
1647 
1648 static void test_txmsg_apply(int cgrp, struct sockmap_options *opt)
1649 {
1650     txmsg_pass = 1;
1651     txmsg_redir = 0;
1652     txmsg_apply = 1;
1653     txmsg_cork = 0;
1654     test_send_one(opt, cgrp);
1655 
1656     txmsg_pass = 0;
1657     txmsg_redir = 1;
1658     txmsg_apply = 1;
1659     txmsg_cork = 0;
1660     test_send_one(opt, cgrp);
1661 
1662     txmsg_pass = 1;
1663     txmsg_redir = 0;
1664     txmsg_apply = 1024;
1665     txmsg_cork = 0;
1666     test_send_large(opt, cgrp);
1667 
1668     txmsg_pass = 0;
1669     txmsg_redir = 1;
1670     txmsg_apply = 1024;
1671     txmsg_cork = 0;
1672     test_send_large(opt, cgrp);
1673 }
1674 
1675 static void test_txmsg_cork(int cgrp, struct sockmap_options *opt)
1676 {
1677     txmsg_pass = 1;
1678     txmsg_redir = 0;
1679     txmsg_apply = 0;
1680     txmsg_cork = 1;
1681     test_send(opt, cgrp);
1682 
1683     txmsg_pass = 1;
1684     txmsg_redir = 0;
1685     txmsg_apply = 1;
1686     txmsg_cork = 1;
1687     test_send(opt, cgrp);
1688 }
1689 
1690 static void test_txmsg_ingress_parser(int cgrp, struct sockmap_options *opt)
1691 {
1692     txmsg_pass = 1;
1693     skb_use_parser = 512;
1694     if (ktls == 1)
1695         skb_use_parser = 570;
1696     opt->iov_length = 256;
1697     opt->iov_count = 1;
1698     opt->rate = 2;
1699     test_exec(cgrp, opt);
1700 }
1701 
1702 static void test_txmsg_ingress_parser2(int cgrp, struct sockmap_options *opt)
1703 {
1704     if (ktls == 1)
1705         return;
1706     skb_use_parser = 10;
1707     opt->iov_length = 20;
1708     opt->iov_count = 1;
1709     opt->rate = 1;
1710     opt->check_recved_len = true;
1711     test_exec(cgrp, opt);
1712     opt->check_recved_len = false;
1713 }
1714 
1715 char *map_names[] = {
1716     "sock_map",
1717     "sock_map_txmsg",
1718     "sock_map_redir",
1719     "sock_apply_bytes",
1720     "sock_cork_bytes",
1721     "sock_bytes",
1722     "sock_redir_flags",
1723     "sock_skb_opts",
1724     "tls_sock_map",
1725 };
1726 
1727 int prog_attach_type[] = {
1728     BPF_SK_SKB_STREAM_PARSER,
1729     BPF_SK_SKB_STREAM_VERDICT,
1730     BPF_SK_SKB_STREAM_VERDICT,
1731     BPF_CGROUP_SOCK_OPS,
1732     BPF_SK_MSG_VERDICT,
1733     BPF_SK_MSG_VERDICT,
1734     BPF_SK_MSG_VERDICT,
1735     BPF_SK_MSG_VERDICT,
1736     BPF_SK_MSG_VERDICT,
1737     BPF_SK_MSG_VERDICT,
1738     BPF_SK_MSG_VERDICT,
1739 };
1740 
1741 int prog_type[] = {
1742     BPF_PROG_TYPE_SK_SKB,
1743     BPF_PROG_TYPE_SK_SKB,
1744     BPF_PROG_TYPE_SK_SKB,
1745     BPF_PROG_TYPE_SOCK_OPS,
1746     BPF_PROG_TYPE_SK_MSG,
1747     BPF_PROG_TYPE_SK_MSG,
1748     BPF_PROG_TYPE_SK_MSG,
1749     BPF_PROG_TYPE_SK_MSG,
1750     BPF_PROG_TYPE_SK_MSG,
1751     BPF_PROG_TYPE_SK_MSG,
1752     BPF_PROG_TYPE_SK_MSG,
1753 };
1754 
1755 static int populate_progs(char *bpf_file)
1756 {
1757     struct bpf_program *prog;
1758     struct bpf_object *obj;
1759     int i = 0;
1760     long err;
1761 
1762     obj = bpf_object__open(bpf_file);
1763     err = libbpf_get_error(obj);
1764     if (err) {
1765         char err_buf[256];
1766 
1767         libbpf_strerror(err, err_buf, sizeof(err_buf));
1768         printf("Unable to load eBPF objects in file '%s' : %s\n",
1769                bpf_file, err_buf);
1770         return -1;
1771     }
1772 
1773     bpf_object__for_each_program(prog, obj) {
1774         bpf_program__set_type(prog, prog_type[i]);
1775         bpf_program__set_expected_attach_type(prog,
1776                               prog_attach_type[i]);
1777         i++;
1778     }
1779 
1780     i = bpf_object__load(obj);
1781     i = 0;
1782     bpf_object__for_each_program(prog, obj) {
1783         prog_fd[i] = bpf_program__fd(prog);
1784         i++;
1785     }
1786 
1787     for (i = 0; i < ARRAY_SIZE(map_fd); i++) {
1788         maps[i] = bpf_object__find_map_by_name(obj, map_names[i]);
1789         map_fd[i] = bpf_map__fd(maps[i]);
1790         if (map_fd[i] < 0) {
1791             fprintf(stderr, "load_bpf_file: (%i) %s\n",
1792                 map_fd[i], strerror(errno));
1793             return -1;
1794         }
1795     }
1796 
1797     return 0;
1798 }
1799 
1800 struct _test test[] = {
1801     {"txmsg test passthrough", test_txmsg_pass},
1802     {"txmsg test redirect", test_txmsg_redir},
1803     {"txmsg test drop", test_txmsg_drop},
1804     {"txmsg test ingress redirect", test_txmsg_ingress_redir},
1805     {"txmsg test skb", test_txmsg_skb},
1806     {"txmsg test apply", test_txmsg_apply},
1807     {"txmsg test cork", test_txmsg_cork},
1808     {"txmsg test hanging corks", test_txmsg_cork_hangs},
1809     {"txmsg test push_data", test_txmsg_push},
1810     {"txmsg test pull-data", test_txmsg_pull},
1811     {"txmsg test pop-data", test_txmsg_pop},
1812     {"txmsg test push/pop data", test_txmsg_push_pop},
1813     {"txmsg test ingress parser", test_txmsg_ingress_parser},
1814     {"txmsg test ingress parser2", test_txmsg_ingress_parser2},
1815 };
1816 
1817 static int check_whitelist(struct _test *t, struct sockmap_options *opt)
1818 {
1819     char *entry, *ptr;
1820 
1821     if (!opt->whitelist)
1822         return 0;
1823     ptr = strdup(opt->whitelist);
1824     if (!ptr)
1825         return -ENOMEM;
1826     entry = strtok(ptr, ",");
1827     while (entry) {
1828         if ((opt->prepend && strstr(opt->prepend, entry) != 0) ||
1829             strstr(opt->map, entry) != 0 ||
1830             strstr(t->title, entry) != 0)
1831             return 0;
1832         entry = strtok(NULL, ",");
1833     }
1834     return -EINVAL;
1835 }
1836 
1837 static int check_blacklist(struct _test *t, struct sockmap_options *opt)
1838 {
1839     char *entry, *ptr;
1840 
1841     if (!opt->blacklist)
1842         return -EINVAL;
1843     ptr = strdup(opt->blacklist);
1844     if (!ptr)
1845         return -ENOMEM;
1846     entry = strtok(ptr, ",");
1847     while (entry) {
1848         if ((opt->prepend && strstr(opt->prepend, entry) != 0) ||
1849             strstr(opt->map, entry) != 0 ||
1850             strstr(t->title, entry) != 0)
1851             return 0;
1852         entry = strtok(NULL, ",");
1853     }
1854     return -EINVAL;
1855 }
1856 
1857 static int __test_selftests(int cg_fd, struct sockmap_options *opt)
1858 {
1859     int i, err;
1860 
1861     err = populate_progs(opt->map);
1862     if (err < 0) {
1863         fprintf(stderr, "ERROR: (%i) load bpf failed\n", err);
1864         return err;
1865     }
1866 
1867     /* Tests basic commands and APIs */
1868     for (i = 0; i < ARRAY_SIZE(test); i++) {
1869         struct _test t = test[i];
1870 
1871         if (check_whitelist(&t, opt) != 0)
1872             continue;
1873         if (check_blacklist(&t, opt) == 0)
1874             continue;
1875 
1876         test_start_subtest(&t, opt);
1877         t.tester(cg_fd, opt);
1878         test_end_subtest();
1879     }
1880 
1881     return err;
1882 }
1883 
1884 static void test_selftests_sockmap(int cg_fd, struct sockmap_options *opt)
1885 {
1886     opt->map = BPF_SOCKMAP_FILENAME;
1887     __test_selftests(cg_fd, opt);
1888 }
1889 
1890 static void test_selftests_sockhash(int cg_fd, struct sockmap_options *opt)
1891 {
1892     opt->map = BPF_SOCKHASH_FILENAME;
1893     __test_selftests(cg_fd, opt);
1894 }
1895 
1896 static void test_selftests_ktls(int cg_fd, struct sockmap_options *opt)
1897 {
1898     opt->map = BPF_SOCKHASH_FILENAME;
1899     opt->prepend = "ktls";
1900     ktls = 1;
1901     __test_selftests(cg_fd, opt);
1902     ktls = 0;
1903 }
1904 
1905 static int test_selftest(int cg_fd, struct sockmap_options *opt)
1906 {
1907 
1908     test_selftests_sockmap(cg_fd, opt);
1909     test_selftests_sockhash(cg_fd, opt);
1910     test_selftests_ktls(cg_fd, opt);
1911     test_print_results();
1912     return 0;
1913 }
1914 
1915 int main(int argc, char **argv)
1916 {
1917     int iov_count = 1, length = 1024, rate = 1;
1918     struct sockmap_options options = {0};
1919     int opt, longindex, err, cg_fd = 0;
1920     char *bpf_file = BPF_SOCKMAP_FILENAME;
1921     int test = SELFTESTS;
1922     bool cg_created = 0;
1923 
1924     while ((opt = getopt_long(argc, argv, ":dhv:c:r:i:l:t:p:q:n:b:",
1925                   long_options, &longindex)) != -1) {
1926         switch (opt) {
1927         case 's':
1928             txmsg_start = atoi(optarg);
1929             break;
1930         case 'e':
1931             txmsg_end = atoi(optarg);
1932             break;
1933         case 'p':
1934             txmsg_start_push = atoi(optarg);
1935             break;
1936         case 'q':
1937             txmsg_end_push = atoi(optarg);
1938             break;
1939         case 'w':
1940             txmsg_start_pop = atoi(optarg);
1941             break;
1942         case 'x':
1943             txmsg_pop = atoi(optarg);
1944             break;
1945         case 'a':
1946             txmsg_apply = atoi(optarg);
1947             break;
1948         case 'k':
1949             txmsg_cork = atoi(optarg);
1950             break;
1951         case 'c':
1952             cg_fd = open(optarg, O_DIRECTORY, O_RDONLY);
1953             if (cg_fd < 0) {
1954                 fprintf(stderr,
1955                     "ERROR: (%i) open cg path failed: %s\n",
1956                     cg_fd, optarg);
1957                 return cg_fd;
1958             }
1959             break;
1960         case 'r':
1961             rate = atoi(optarg);
1962             break;
1963         case 'v':
1964             options.verbose = 1;
1965             if (optarg)
1966                 options.verbose = atoi(optarg);
1967             break;
1968         case 'i':
1969             iov_count = atoi(optarg);
1970             break;
1971         case 'l':
1972             length = atoi(optarg);
1973             break;
1974         case 'd':
1975             options.data_test = true;
1976             break;
1977         case 't':
1978             if (strcmp(optarg, "ping") == 0) {
1979                 test = PING_PONG;
1980             } else if (strcmp(optarg, "sendmsg") == 0) {
1981                 test = SENDMSG;
1982             } else if (strcmp(optarg, "base") == 0) {
1983                 test = BASE;
1984             } else if (strcmp(optarg, "base_sendpage") == 0) {
1985                 test = BASE_SENDPAGE;
1986             } else if (strcmp(optarg, "sendpage") == 0) {
1987                 test = SENDPAGE;
1988             } else {
1989                 usage(argv);
1990                 return -1;
1991             }
1992             break;
1993         case 'n':
1994             options.whitelist = strdup(optarg);
1995             if (!options.whitelist)
1996                 return -ENOMEM;
1997             break;
1998         case 'b':
1999             options.blacklist = strdup(optarg);
2000             if (!options.blacklist)
2001                 return -ENOMEM;
2002         case 0:
2003             break;
2004         case 'h':
2005         default:
2006             usage(argv);
2007             return -1;
2008         }
2009     }
2010 
2011     if (!cg_fd) {
2012         cg_fd = cgroup_setup_and_join(CG_PATH);
2013         if (cg_fd < 0)
2014             return cg_fd;
2015         cg_created = 1;
2016     }
2017 
2018     /* Use libbpf 1.0 API mode */
2019     libbpf_set_strict_mode(LIBBPF_STRICT_ALL);
2020 
2021     if (test == SELFTESTS) {
2022         err = test_selftest(cg_fd, &options);
2023         goto out;
2024     }
2025 
2026     err = populate_progs(bpf_file);
2027     if (err) {
2028         fprintf(stderr, "populate program: (%s) %s\n",
2029             bpf_file, strerror(errno));
2030         return 1;
2031     }
2032     running = 1;
2033 
2034     /* catch SIGINT */
2035     signal(SIGINT, running_handler);
2036 
2037     options.iov_count = iov_count;
2038     options.iov_length = length;
2039     options.rate = rate;
2040 
2041     err = run_options(&options, cg_fd, test);
2042 out:
2043     if (options.whitelist)
2044         free(options.whitelist);
2045     if (options.blacklist)
2046         free(options.blacklist);
2047     if (cg_created)
2048         cleanup_cgroup_environment();
2049     close(cg_fd);
2050     return err;
2051 }
2052 
2053 void running_handler(int a)
2054 {
2055     running = 0;
2056 }
[ Source navigation ] [ Diff markup ] [ Identifier search ] [ general search ]

This page was automatically generated by the 2.1.0 LXR engine.

The LXR team
Valid CSS 2.1! Valid HTML 4.01!