Back to home page

OSCL-LXR
 
 

     

0001 // SPDX-License-Identifier: GPL-2.0
0002 /* Copyright(c) 2020 Intel Corporation. */
0003 
0004 /*
0005  * Some functions in this program are taken from
0006  * Linux kernel samples/bpf/xdpsock* and modified
0007  * for use.
0008  *
0009  * See test_xsk.sh for detailed information on test topology
0010  * and prerequisite network setup.
0011  *
0012  * This test program contains two threads, each thread is single socket with
0013  * a unique UMEM. It validates in-order packet delivery and packet content
0014  * by sending packets to each other.
0015  *
0016  * Tests Information:
0017  * ------------------
0018  * These selftests test AF_XDP SKB and Native/DRV modes using veth
0019  * Virtual Ethernet interfaces.
0020  *
0021  * For each mode, the following tests are run:
0022  *    a. nopoll - soft-irq processing in run-to-completion mode
0023  *    b. poll - using poll() syscall
0024  *    c. Socket Teardown
0025  *       Create a Tx and a Rx socket, Tx from one socket, Rx on another. Destroy
0026  *       both sockets, then repeat multiple times. Only nopoll mode is used
0027  *    d. Bi-directional sockets
0028  *       Configure sockets as bi-directional tx/rx sockets, sets up fill and
0029  *       completion rings on each socket, tx/rx in both directions. Only nopoll
0030  *       mode is used
0031  *    e. Statistics
0032  *       Trigger some error conditions and ensure that the appropriate statistics
0033  *       are incremented. Within this test, the following statistics are tested:
0034  *       i.   rx dropped
0035  *            Increase the UMEM frame headroom to a value which results in
0036  *            insufficient space in the rx buffer for both the packet and the headroom.
0037  *       ii.  tx invalid
0038  *            Set the 'len' field of tx descriptors to an invalid value (umem frame
0039  *            size + 1).
0040  *       iii. rx ring full
0041  *            Reduce the size of the RX ring to a fraction of the fill ring size.
0042  *       iv.  fill queue empty
0043  *            Do not populate the fill queue and then try to receive pkts.
0044  *    f. bpf_link resource persistence
0045  *       Configure sockets at indexes 0 and 1, run a traffic on queue ids 0,
0046  *       then remove xsk sockets from queue 0 on both veth interfaces and
0047  *       finally run a traffic on queues ids 1
0048  *    g. unaligned mode
0049  *    h. tests for invalid and corner case Tx descriptors so that the correct ones
0050  *       are discarded and let through, respectively.
0051  *    i. 2K frame size tests
0052  *
0053  * Total tests: 12
0054  *
0055  * Flow:
0056  * -----
0057  * - Single process spawns two threads: Tx and Rx
0058  * - Each of these two threads attach to a veth interface within their assigned
0059  *   namespaces
0060  * - Each thread Creates one AF_XDP socket connected to a unique umem for each
0061  *   veth interface
0062  * - Tx thread Transmits 10k packets from veth<xxxx> to veth<yyyy>
0063  * - Rx thread verifies if all 10k packets were received and delivered in-order,
0064  *   and have the right content
0065  *
0066  * Enable/disable packet dump mode:
0067  * --------------------------
0068  * To enable L2 - L4 headers and payload dump of each packet on STDOUT, add
0069  * parameter -D to params array in test_xsk.sh, i.e. params=("-S" "-D")
0070  */
0071 
0072 #define _GNU_SOURCE
0073 #include <fcntl.h>
0074 #include <errno.h>
0075 #include <getopt.h>
0076 #include <asm/barrier.h>
0077 #include <linux/if_link.h>
0078 #include <linux/if_ether.h>
0079 #include <linux/ip.h>
0080 #include <linux/udp.h>
0081 #include <arpa/inet.h>
0082 #include <net/if.h>
0083 #include <locale.h>
0084 #include <poll.h>
0085 #include <pthread.h>
0086 #include <signal.h>
0087 #include <stdbool.h>
0088 #include <stdio.h>
0089 #include <stdlib.h>
0090 #include <string.h>
0091 #include <stddef.h>
0092 #include <sys/mman.h>
0093 #include <sys/socket.h>
0094 #include <sys/time.h>
0095 #include <sys/types.h>
0096 #include <sys/queue.h>
0097 #include <time.h>
0098 #include <unistd.h>
0099 #include <stdatomic.h>
0100 #include "xsk.h"
0101 #include "xskxceiver.h"
0102 #include "../kselftest.h"
0103 
0104 /* AF_XDP APIs were moved into libxdp and marked as deprecated in libbpf.
0105  * Until xskxceiver is either moved or re-writed into libxdp, suppress
0106  * deprecation warnings in this file
0107  */
0108 #pragma GCC diagnostic ignored "-Wdeprecated-declarations"
0109 
0110 static const char *MAC1 = "\x00\x0A\x56\x9E\xEE\x62";
0111 static const char *MAC2 = "\x00\x0A\x56\x9E\xEE\x61";
0112 static const char *IP1 = "192.168.100.162";
0113 static const char *IP2 = "192.168.100.161";
0114 static const u16 UDP_PORT1 = 2020;
0115 static const u16 UDP_PORT2 = 2121;
0116 
0117 static void __exit_with_error(int error, const char *file, const char *func, int line)
0118 {
0119     ksft_test_result_fail("[%s:%s:%i]: ERROR: %d/\"%s\"\n", file, func, line, error,
0120                   strerror(error));
0121     ksft_exit_xfail();
0122 }
0123 
0124 #define exit_with_error(error) __exit_with_error(error, __FILE__, __func__, __LINE__)
0125 
0126 #define mode_string(test) (test)->ifobj_tx->xdp_flags & XDP_FLAGS_SKB_MODE ? "SKB" : "DRV"
0127 #define busy_poll_string(test) (test)->ifobj_tx->busy_poll ? "BUSY-POLL " : ""
0128 
0129 static void report_failure(struct test_spec *test)
0130 {
0131     if (test->fail)
0132         return;
0133 
0134     ksft_test_result_fail("FAIL: %s %s%s\n", mode_string(test), busy_poll_string(test),
0135                   test->name);
0136     test->fail = true;
0137 }
0138 
0139 static void memset32_htonl(void *dest, u32 val, u32 size)
0140 {
0141     u32 *ptr = (u32 *)dest;
0142     int i;
0143 
0144     val = htonl(val);
0145 
0146     for (i = 0; i < (size & (~0x3)); i += 4)
0147         ptr[i >> 2] = val;
0148 }
0149 
0150 /*
0151  * Fold a partial checksum
0152  * This function code has been taken from
0153  * Linux kernel include/asm-generic/checksum.h
0154  */
0155 static __u16 csum_fold(__u32 csum)
0156 {
0157     u32 sum = (__force u32)csum;
0158 
0159     sum = (sum & 0xffff) + (sum >> 16);
0160     sum = (sum & 0xffff) + (sum >> 16);
0161     return (__force __u16)~sum;
0162 }
0163 
0164 /*
0165  * This function code has been taken from
0166  * Linux kernel lib/checksum.c
0167  */
0168 static u32 from64to32(u64 x)
0169 {
0170     /* add up 32-bit and 32-bit for 32+c bit */
0171     x = (x & 0xffffffff) + (x >> 32);
0172     /* add up carry.. */
0173     x = (x & 0xffffffff) + (x >> 32);
0174     return (u32)x;
0175 }
0176 
0177 /*
0178  * This function code has been taken from
0179  * Linux kernel lib/checksum.c
0180  */
0181 static __u32 csum_tcpudp_nofold(__be32 saddr, __be32 daddr, __u32 len, __u8 proto, __u32 sum)
0182 {
0183     unsigned long long s = (__force u32)sum;
0184 
0185     s += (__force u32)saddr;
0186     s += (__force u32)daddr;
0187 #ifdef __BIG_ENDIAN__
0188     s += proto + len;
0189 #else
0190     s += (proto + len) << 8;
0191 #endif
0192     return (__force __u32)from64to32(s);
0193 }
0194 
0195 /*
0196  * This function has been taken from
0197  * Linux kernel include/asm-generic/checksum.h
0198  */
0199 static __u16 csum_tcpudp_magic(__be32 saddr, __be32 daddr, __u32 len, __u8 proto, __u32 sum)
0200 {
0201     return csum_fold(csum_tcpudp_nofold(saddr, daddr, len, proto, sum));
0202 }
0203 
0204 static u16 udp_csum(u32 saddr, u32 daddr, u32 len, u8 proto, u16 *udp_pkt)
0205 {
0206     u32 csum = 0;
0207     u32 cnt = 0;
0208 
0209     /* udp hdr and data */
0210     for (; cnt < len; cnt += 2)
0211         csum += udp_pkt[cnt >> 1];
0212 
0213     return csum_tcpudp_magic(saddr, daddr, len, proto, csum);
0214 }
0215 
0216 static void gen_eth_hdr(struct ifobject *ifobject, struct ethhdr *eth_hdr)
0217 {
0218     memcpy(eth_hdr->h_dest, ifobject->dst_mac, ETH_ALEN);
0219     memcpy(eth_hdr->h_source, ifobject->src_mac, ETH_ALEN);
0220     eth_hdr->h_proto = htons(ETH_P_IP);
0221 }
0222 
0223 static void gen_ip_hdr(struct ifobject *ifobject, struct iphdr *ip_hdr)
0224 {
0225     ip_hdr->version = IP_PKT_VER;
0226     ip_hdr->ihl = 0x5;
0227     ip_hdr->tos = IP_PKT_TOS;
0228     ip_hdr->tot_len = htons(IP_PKT_SIZE);
0229     ip_hdr->id = 0;
0230     ip_hdr->frag_off = 0;
0231     ip_hdr->ttl = IPDEFTTL;
0232     ip_hdr->protocol = IPPROTO_UDP;
0233     ip_hdr->saddr = ifobject->src_ip;
0234     ip_hdr->daddr = ifobject->dst_ip;
0235     ip_hdr->check = 0;
0236 }
0237 
0238 static void gen_udp_hdr(u32 payload, void *pkt, struct ifobject *ifobject,
0239             struct udphdr *udp_hdr)
0240 {
0241     udp_hdr->source = htons(ifobject->src_port);
0242     udp_hdr->dest = htons(ifobject->dst_port);
0243     udp_hdr->len = htons(UDP_PKT_SIZE);
0244     memset32_htonl(pkt + PKT_HDR_SIZE, payload, UDP_PKT_DATA_SIZE);
0245 }
0246 
0247 static void gen_udp_csum(struct udphdr *udp_hdr, struct iphdr *ip_hdr)
0248 {
0249     udp_hdr->check = 0;
0250     udp_hdr->check =
0251         udp_csum(ip_hdr->saddr, ip_hdr->daddr, UDP_PKT_SIZE, IPPROTO_UDP, (u16 *)udp_hdr);
0252 }
0253 
0254 static int xsk_configure_umem(struct xsk_umem_info *umem, void *buffer, u64 size)
0255 {
0256     struct xsk_umem_config cfg = {
0257         .fill_size = XSK_RING_PROD__DEFAULT_NUM_DESCS,
0258         .comp_size = XSK_RING_CONS__DEFAULT_NUM_DESCS,
0259         .frame_size = umem->frame_size,
0260         .frame_headroom = umem->frame_headroom,
0261         .flags = XSK_UMEM__DEFAULT_FLAGS
0262     };
0263     int ret;
0264 
0265     if (umem->unaligned_mode)
0266         cfg.flags |= XDP_UMEM_UNALIGNED_CHUNK_FLAG;
0267 
0268     ret = xsk_umem__create(&umem->umem, buffer, size,
0269                    &umem->fq, &umem->cq, &cfg);
0270     if (ret)
0271         return ret;
0272 
0273     umem->buffer = buffer;
0274     return 0;
0275 }
0276 
0277 static void enable_busy_poll(struct xsk_socket_info *xsk)
0278 {
0279     int sock_opt;
0280 
0281     sock_opt = 1;
0282     if (setsockopt(xsk_socket__fd(xsk->xsk), SOL_SOCKET, SO_PREFER_BUSY_POLL,
0283                (void *)&sock_opt, sizeof(sock_opt)) < 0)
0284         exit_with_error(errno);
0285 
0286     sock_opt = 20;
0287     if (setsockopt(xsk_socket__fd(xsk->xsk), SOL_SOCKET, SO_BUSY_POLL,
0288                (void *)&sock_opt, sizeof(sock_opt)) < 0)
0289         exit_with_error(errno);
0290 
0291     sock_opt = BATCH_SIZE;
0292     if (setsockopt(xsk_socket__fd(xsk->xsk), SOL_SOCKET, SO_BUSY_POLL_BUDGET,
0293                (void *)&sock_opt, sizeof(sock_opt)) < 0)
0294         exit_with_error(errno);
0295 }
0296 
0297 static int xsk_configure_socket(struct xsk_socket_info *xsk, struct xsk_umem_info *umem,
0298                 struct ifobject *ifobject, bool shared)
0299 {
0300     struct xsk_socket_config cfg = {};
0301     struct xsk_ring_cons *rxr;
0302     struct xsk_ring_prod *txr;
0303 
0304     xsk->umem = umem;
0305     cfg.rx_size = xsk->rxqsize;
0306     cfg.tx_size = XSK_RING_PROD__DEFAULT_NUM_DESCS;
0307     cfg.libbpf_flags = XSK_LIBBPF_FLAGS__INHIBIT_PROG_LOAD;
0308     cfg.xdp_flags = ifobject->xdp_flags;
0309     cfg.bind_flags = ifobject->bind_flags;
0310     if (shared)
0311         cfg.bind_flags |= XDP_SHARED_UMEM;
0312 
0313     txr = ifobject->tx_on ? &xsk->tx : NULL;
0314     rxr = ifobject->rx_on ? &xsk->rx : NULL;
0315     return xsk_socket__create(&xsk->xsk, ifobject->ifname, 0, umem->umem, rxr, txr, &cfg);
0316 }
0317 
0318 static struct option long_options[] = {
0319     {"interface", required_argument, 0, 'i'},
0320     {"busy-poll", no_argument, 0, 'b'},
0321     {"dump-pkts", no_argument, 0, 'D'},
0322     {"verbose", no_argument, 0, 'v'},
0323     {0, 0, 0, 0}
0324 };
0325 
0326 static void usage(const char *prog)
0327 {
0328     const char *str =
0329         "  Usage: %s [OPTIONS]\n"
0330         "  Options:\n"
0331         "  -i, --interface      Use interface\n"
0332         "  -D, --dump-pkts      Dump packets L2 - L5\n"
0333         "  -v, --verbose        Verbose output\n"
0334         "  -b, --busy-poll      Enable busy poll\n";
0335 
0336     ksft_print_msg(str, prog);
0337 }
0338 
0339 static int switch_namespace(const char *nsname)
0340 {
0341     char fqns[26] = "/var/run/netns/";
0342     int nsfd;
0343 
0344     if (!nsname || strlen(nsname) == 0)
0345         return -1;
0346 
0347     strncat(fqns, nsname, sizeof(fqns) - strlen(fqns) - 1);
0348     nsfd = open(fqns, O_RDONLY);
0349 
0350     if (nsfd == -1)
0351         exit_with_error(errno);
0352 
0353     if (setns(nsfd, 0) == -1)
0354         exit_with_error(errno);
0355 
0356     print_verbose("NS switched: %s\n", nsname);
0357 
0358     return nsfd;
0359 }
0360 
0361 static bool validate_interface(struct ifobject *ifobj)
0362 {
0363     if (!strcmp(ifobj->ifname, ""))
0364         return false;
0365     return true;
0366 }
0367 
0368 static void parse_command_line(struct ifobject *ifobj_tx, struct ifobject *ifobj_rx, int argc,
0369                    char **argv)
0370 {
0371     struct ifobject *ifobj;
0372     u32 interface_nb = 0;
0373     int option_index, c;
0374 
0375     opterr = 0;
0376 
0377     for (;;) {
0378         char *sptr, *token;
0379 
0380         c = getopt_long(argc, argv, "i:Dvb", long_options, &option_index);
0381         if (c == -1)
0382             break;
0383 
0384         switch (c) {
0385         case 'i':
0386             if (interface_nb == 0)
0387                 ifobj = ifobj_tx;
0388             else if (interface_nb == 1)
0389                 ifobj = ifobj_rx;
0390             else
0391                 break;
0392 
0393             sptr = strndupa(optarg, strlen(optarg));
0394             memcpy(ifobj->ifname, strsep(&sptr, ","), MAX_INTERFACE_NAME_CHARS);
0395             token = strsep(&sptr, ",");
0396             if (token)
0397                 memcpy(ifobj->nsname, token, MAX_INTERFACES_NAMESPACE_CHARS);
0398             interface_nb++;
0399             break;
0400         case 'D':
0401             opt_pkt_dump = true;
0402             break;
0403         case 'v':
0404             opt_verbose = true;
0405             break;
0406         case 'b':
0407             ifobj_tx->busy_poll = true;
0408             ifobj_rx->busy_poll = true;
0409             break;
0410         default:
0411             usage(basename(argv[0]));
0412             ksft_exit_xfail();
0413         }
0414     }
0415 }
0416 
0417 static void __test_spec_init(struct test_spec *test, struct ifobject *ifobj_tx,
0418                  struct ifobject *ifobj_rx)
0419 {
0420     u32 i, j;
0421 
0422     for (i = 0; i < MAX_INTERFACES; i++) {
0423         struct ifobject *ifobj = i ? ifobj_rx : ifobj_tx;
0424 
0425         ifobj->xsk = &ifobj->xsk_arr[0];
0426         ifobj->use_poll = false;
0427         ifobj->use_fill_ring = true;
0428         ifobj->release_rx = true;
0429         ifobj->pkt_stream = test->pkt_stream_default;
0430         ifobj->validation_func = NULL;
0431 
0432         if (i == 0) {
0433             ifobj->rx_on = false;
0434             ifobj->tx_on = true;
0435         } else {
0436             ifobj->rx_on = true;
0437             ifobj->tx_on = false;
0438         }
0439 
0440         memset(ifobj->umem, 0, sizeof(*ifobj->umem));
0441         ifobj->umem->num_frames = DEFAULT_UMEM_BUFFERS;
0442         ifobj->umem->frame_size = XSK_UMEM__DEFAULT_FRAME_SIZE;
0443 
0444         for (j = 0; j < MAX_SOCKETS; j++) {
0445             memset(&ifobj->xsk_arr[j], 0, sizeof(ifobj->xsk_arr[j]));
0446             ifobj->xsk_arr[j].rxqsize = XSK_RING_CONS__DEFAULT_NUM_DESCS;
0447         }
0448     }
0449 
0450     test->ifobj_tx = ifobj_tx;
0451     test->ifobj_rx = ifobj_rx;
0452     test->current_step = 0;
0453     test->total_steps = 1;
0454     test->nb_sockets = 1;
0455     test->fail = false;
0456 }
0457 
0458 static void test_spec_init(struct test_spec *test, struct ifobject *ifobj_tx,
0459                struct ifobject *ifobj_rx, enum test_mode mode)
0460 {
0461     struct pkt_stream *pkt_stream;
0462     u32 i;
0463 
0464     pkt_stream = test->pkt_stream_default;
0465     memset(test, 0, sizeof(*test));
0466     test->pkt_stream_default = pkt_stream;
0467 
0468     for (i = 0; i < MAX_INTERFACES; i++) {
0469         struct ifobject *ifobj = i ? ifobj_rx : ifobj_tx;
0470 
0471         ifobj->xdp_flags = XDP_FLAGS_UPDATE_IF_NOEXIST;
0472         if (mode == TEST_MODE_SKB)
0473             ifobj->xdp_flags |= XDP_FLAGS_SKB_MODE;
0474         else
0475             ifobj->xdp_flags |= XDP_FLAGS_DRV_MODE;
0476 
0477         ifobj->bind_flags = XDP_USE_NEED_WAKEUP | XDP_COPY;
0478     }
0479 
0480     __test_spec_init(test, ifobj_tx, ifobj_rx);
0481 }
0482 
0483 static void test_spec_reset(struct test_spec *test)
0484 {
0485     __test_spec_init(test, test->ifobj_tx, test->ifobj_rx);
0486 }
0487 
0488 static void test_spec_set_name(struct test_spec *test, const char *name)
0489 {
0490     strncpy(test->name, name, MAX_TEST_NAME_SIZE);
0491 }
0492 
0493 static void pkt_stream_reset(struct pkt_stream *pkt_stream)
0494 {
0495     if (pkt_stream)
0496         pkt_stream->rx_pkt_nb = 0;
0497 }
0498 
0499 static struct pkt *pkt_stream_get_pkt(struct pkt_stream *pkt_stream, u32 pkt_nb)
0500 {
0501     if (pkt_nb >= pkt_stream->nb_pkts)
0502         return NULL;
0503 
0504     return &pkt_stream->pkts[pkt_nb];
0505 }
0506 
0507 static struct pkt *pkt_stream_get_next_rx_pkt(struct pkt_stream *pkt_stream, u32 *pkts_sent)
0508 {
0509     while (pkt_stream->rx_pkt_nb < pkt_stream->nb_pkts) {
0510         (*pkts_sent)++;
0511         if (pkt_stream->pkts[pkt_stream->rx_pkt_nb].valid)
0512             return &pkt_stream->pkts[pkt_stream->rx_pkt_nb++];
0513         pkt_stream->rx_pkt_nb++;
0514     }
0515     return NULL;
0516 }
0517 
0518 static void pkt_stream_delete(struct pkt_stream *pkt_stream)
0519 {
0520     free(pkt_stream->pkts);
0521     free(pkt_stream);
0522 }
0523 
0524 static void pkt_stream_restore_default(struct test_spec *test)
0525 {
0526     struct pkt_stream *tx_pkt_stream = test->ifobj_tx->pkt_stream;
0527 
0528     if (tx_pkt_stream != test->pkt_stream_default) {
0529         pkt_stream_delete(test->ifobj_tx->pkt_stream);
0530         test->ifobj_tx->pkt_stream = test->pkt_stream_default;
0531     }
0532 
0533     if (test->ifobj_rx->pkt_stream != test->pkt_stream_default &&
0534         test->ifobj_rx->pkt_stream != tx_pkt_stream)
0535         pkt_stream_delete(test->ifobj_rx->pkt_stream);
0536     test->ifobj_rx->pkt_stream = test->pkt_stream_default;
0537 }
0538 
0539 static struct pkt_stream *__pkt_stream_alloc(u32 nb_pkts)
0540 {
0541     struct pkt_stream *pkt_stream;
0542 
0543     pkt_stream = calloc(1, sizeof(*pkt_stream));
0544     if (!pkt_stream)
0545         return NULL;
0546 
0547     pkt_stream->pkts = calloc(nb_pkts, sizeof(*pkt_stream->pkts));
0548     if (!pkt_stream->pkts) {
0549         free(pkt_stream);
0550         return NULL;
0551     }
0552 
0553     pkt_stream->nb_pkts = nb_pkts;
0554     return pkt_stream;
0555 }
0556 
0557 static void pkt_set(struct xsk_umem_info *umem, struct pkt *pkt, u64 addr, u32 len)
0558 {
0559     pkt->addr = addr;
0560     pkt->len = len;
0561     if (len > umem->frame_size - XDP_PACKET_HEADROOM - MIN_PKT_SIZE * 2 - umem->frame_headroom)
0562         pkt->valid = false;
0563     else
0564         pkt->valid = true;
0565 }
0566 
0567 static struct pkt_stream *pkt_stream_generate(struct xsk_umem_info *umem, u32 nb_pkts, u32 pkt_len)
0568 {
0569     struct pkt_stream *pkt_stream;
0570     u32 i;
0571 
0572     pkt_stream = __pkt_stream_alloc(nb_pkts);
0573     if (!pkt_stream)
0574         exit_with_error(ENOMEM);
0575 
0576     pkt_stream->nb_pkts = nb_pkts;
0577     for (i = 0; i < nb_pkts; i++) {
0578         pkt_set(umem, &pkt_stream->pkts[i], (i % umem->num_frames) * umem->frame_size,
0579             pkt_len);
0580         pkt_stream->pkts[i].payload = i;
0581     }
0582 
0583     return pkt_stream;
0584 }
0585 
0586 static struct pkt_stream *pkt_stream_clone(struct xsk_umem_info *umem,
0587                        struct pkt_stream *pkt_stream)
0588 {
0589     return pkt_stream_generate(umem, pkt_stream->nb_pkts, pkt_stream->pkts[0].len);
0590 }
0591 
0592 static void pkt_stream_replace(struct test_spec *test, u32 nb_pkts, u32 pkt_len)
0593 {
0594     struct pkt_stream *pkt_stream;
0595 
0596     pkt_stream = pkt_stream_generate(test->ifobj_tx->umem, nb_pkts, pkt_len);
0597     test->ifobj_tx->pkt_stream = pkt_stream;
0598     test->ifobj_rx->pkt_stream = pkt_stream;
0599 }
0600 
0601 static void pkt_stream_replace_half(struct test_spec *test, u32 pkt_len, int offset)
0602 {
0603     struct xsk_umem_info *umem = test->ifobj_tx->umem;
0604     struct pkt_stream *pkt_stream;
0605     u32 i;
0606 
0607     pkt_stream = pkt_stream_clone(umem, test->pkt_stream_default);
0608     for (i = 1; i < test->pkt_stream_default->nb_pkts; i += 2)
0609         pkt_set(umem, &pkt_stream->pkts[i],
0610             (i % umem->num_frames) * umem->frame_size + offset, pkt_len);
0611 
0612     test->ifobj_tx->pkt_stream = pkt_stream;
0613     test->ifobj_rx->pkt_stream = pkt_stream;
0614 }
0615 
0616 static void pkt_stream_receive_half(struct test_spec *test)
0617 {
0618     struct xsk_umem_info *umem = test->ifobj_rx->umem;
0619     struct pkt_stream *pkt_stream = test->ifobj_tx->pkt_stream;
0620     u32 i;
0621 
0622     test->ifobj_rx->pkt_stream = pkt_stream_generate(umem, pkt_stream->nb_pkts,
0623                              pkt_stream->pkts[0].len);
0624     pkt_stream = test->ifobj_rx->pkt_stream;
0625     for (i = 1; i < pkt_stream->nb_pkts; i += 2)
0626         pkt_stream->pkts[i].valid = false;
0627 }
0628 
0629 static struct pkt *pkt_generate(struct ifobject *ifobject, u32 pkt_nb)
0630 {
0631     struct pkt *pkt = pkt_stream_get_pkt(ifobject->pkt_stream, pkt_nb);
0632     struct udphdr *udp_hdr;
0633     struct ethhdr *eth_hdr;
0634     struct iphdr *ip_hdr;
0635     void *data;
0636 
0637     if (!pkt)
0638         return NULL;
0639     if (!pkt->valid || pkt->len < MIN_PKT_SIZE)
0640         return pkt;
0641 
0642     data = xsk_umem__get_data(ifobject->umem->buffer, pkt->addr);
0643     udp_hdr = (struct udphdr *)(data + sizeof(struct ethhdr) + sizeof(struct iphdr));
0644     ip_hdr = (struct iphdr *)(data + sizeof(struct ethhdr));
0645     eth_hdr = (struct ethhdr *)data;
0646 
0647     gen_udp_hdr(pkt_nb, data, ifobject, udp_hdr);
0648     gen_ip_hdr(ifobject, ip_hdr);
0649     gen_udp_csum(udp_hdr, ip_hdr);
0650     gen_eth_hdr(ifobject, eth_hdr);
0651 
0652     return pkt;
0653 }
0654 
0655 static void pkt_stream_generate_custom(struct test_spec *test, struct pkt *pkts, u32 nb_pkts)
0656 {
0657     struct pkt_stream *pkt_stream;
0658     u32 i;
0659 
0660     pkt_stream = __pkt_stream_alloc(nb_pkts);
0661     if (!pkt_stream)
0662         exit_with_error(ENOMEM);
0663 
0664     test->ifobj_tx->pkt_stream = pkt_stream;
0665     test->ifobj_rx->pkt_stream = pkt_stream;
0666 
0667     for (i = 0; i < nb_pkts; i++) {
0668         pkt_stream->pkts[i].addr = pkts[i].addr;
0669         pkt_stream->pkts[i].len = pkts[i].len;
0670         pkt_stream->pkts[i].payload = i;
0671         pkt_stream->pkts[i].valid = pkts[i].valid;
0672     }
0673 }
0674 
0675 static void pkt_dump(void *pkt, u32 len)
0676 {
0677     char s[INET_ADDRSTRLEN];
0678     struct ethhdr *ethhdr;
0679     struct udphdr *udphdr;
0680     struct iphdr *iphdr;
0681     int payload, i;
0682 
0683     ethhdr = pkt;
0684     iphdr = pkt + sizeof(*ethhdr);
0685     udphdr = pkt + sizeof(*ethhdr) + sizeof(*iphdr);
0686 
0687     /*extract L2 frame */
0688     fprintf(stdout, "DEBUG>> L2: dst mac: ");
0689     for (i = 0; i < ETH_ALEN; i++)
0690         fprintf(stdout, "%02X", ethhdr->h_dest[i]);
0691 
0692     fprintf(stdout, "\nDEBUG>> L2: src mac: ");
0693     for (i = 0; i < ETH_ALEN; i++)
0694         fprintf(stdout, "%02X", ethhdr->h_source[i]);
0695 
0696     /*extract L3 frame */
0697     fprintf(stdout, "\nDEBUG>> L3: ip_hdr->ihl: %02X\n", iphdr->ihl);
0698     fprintf(stdout, "DEBUG>> L3: ip_hdr->saddr: %s\n",
0699         inet_ntop(AF_INET, &iphdr->saddr, s, sizeof(s)));
0700     fprintf(stdout, "DEBUG>> L3: ip_hdr->daddr: %s\n",
0701         inet_ntop(AF_INET, &iphdr->daddr, s, sizeof(s)));
0702     /*extract L4 frame */
0703     fprintf(stdout, "DEBUG>> L4: udp_hdr->src: %d\n", ntohs(udphdr->source));
0704     fprintf(stdout, "DEBUG>> L4: udp_hdr->dst: %d\n", ntohs(udphdr->dest));
0705     /*extract L5 frame */
0706     payload = *((uint32_t *)(pkt + PKT_HDR_SIZE));
0707 
0708     fprintf(stdout, "DEBUG>> L5: payload: %d\n", payload);
0709     fprintf(stdout, "---------------------------------------\n");
0710 }
0711 
0712 static bool is_offset_correct(struct xsk_umem_info *umem, struct pkt_stream *pkt_stream, u64 addr,
0713                   u64 pkt_stream_addr)
0714 {
0715     u32 headroom = umem->unaligned_mode ? 0 : umem->frame_headroom;
0716     u32 offset = addr % umem->frame_size, expected_offset = 0;
0717 
0718     if (!pkt_stream->use_addr_for_fill)
0719         pkt_stream_addr = 0;
0720 
0721     expected_offset += (pkt_stream_addr + headroom + XDP_PACKET_HEADROOM) % umem->frame_size;
0722 
0723     if (offset == expected_offset)
0724         return true;
0725 
0726     ksft_print_msg("[%s] expected [%u], got [%u]\n", __func__, expected_offset, offset);
0727     return false;
0728 }
0729 
0730 static bool is_pkt_valid(struct pkt *pkt, void *buffer, u64 addr, u32 len)
0731 {
0732     void *data = xsk_umem__get_data(buffer, addr);
0733     struct iphdr *iphdr = (struct iphdr *)(data + sizeof(struct ethhdr));
0734 
0735     if (!pkt) {
0736         ksft_print_msg("[%s] too many packets received\n", __func__);
0737         return false;
0738     }
0739 
0740     if (len < MIN_PKT_SIZE || pkt->len < MIN_PKT_SIZE) {
0741         /* Do not try to verify packets that are smaller than minimum size. */
0742         return true;
0743     }
0744 
0745     if (pkt->len != len) {
0746         ksft_print_msg("[%s] expected length [%d], got length [%d]\n",
0747                    __func__, pkt->len, len);
0748         return false;
0749     }
0750 
0751     if (iphdr->version == IP_PKT_VER && iphdr->tos == IP_PKT_TOS) {
0752         u32 seqnum = ntohl(*((u32 *)(data + PKT_HDR_SIZE)));
0753 
0754         if (opt_pkt_dump)
0755             pkt_dump(data, PKT_SIZE);
0756 
0757         if (pkt->payload != seqnum) {
0758             ksft_print_msg("[%s] expected seqnum [%d], got seqnum [%d]\n",
0759                        __func__, pkt->payload, seqnum);
0760             return false;
0761         }
0762     } else {
0763         ksft_print_msg("Invalid frame received: ");
0764         ksft_print_msg("[IP_PKT_VER: %02X], [IP_PKT_TOS: %02X]\n", iphdr->version,
0765                    iphdr->tos);
0766         return false;
0767     }
0768 
0769     return true;
0770 }
0771 
0772 static void kick_tx(struct xsk_socket_info *xsk)
0773 {
0774     int ret;
0775 
0776     ret = sendto(xsk_socket__fd(xsk->xsk), NULL, 0, MSG_DONTWAIT, NULL, 0);
0777     if (ret >= 0)
0778         return;
0779     if (errno == ENOBUFS || errno == EAGAIN || errno == EBUSY || errno == ENETDOWN) {
0780         usleep(100);
0781         return;
0782     }
0783     exit_with_error(errno);
0784 }
0785 
0786 static void kick_rx(struct xsk_socket_info *xsk)
0787 {
0788     int ret;
0789 
0790     ret = recvfrom(xsk_socket__fd(xsk->xsk), NULL, 0, MSG_DONTWAIT, NULL, NULL);
0791     if (ret < 0)
0792         exit_with_error(errno);
0793 }
0794 
0795 static int complete_pkts(struct xsk_socket_info *xsk, int batch_size)
0796 {
0797     unsigned int rcvd;
0798     u32 idx;
0799 
0800     if (xsk_ring_prod__needs_wakeup(&xsk->tx))
0801         kick_tx(xsk);
0802 
0803     rcvd = xsk_ring_cons__peek(&xsk->umem->cq, batch_size, &idx);
0804     if (rcvd) {
0805         if (rcvd > xsk->outstanding_tx) {
0806             u64 addr = *xsk_ring_cons__comp_addr(&xsk->umem->cq, idx + rcvd - 1);
0807 
0808             ksft_print_msg("[%s] Too many packets completed\n", __func__);
0809             ksft_print_msg("Last completion address: %llx\n", addr);
0810             return TEST_FAILURE;
0811         }
0812 
0813         xsk_ring_cons__release(&xsk->umem->cq, rcvd);
0814         xsk->outstanding_tx -= rcvd;
0815     }
0816 
0817     return TEST_PASS;
0818 }
0819 
0820 static int receive_pkts(struct ifobject *ifobj, struct pollfd *fds)
0821 {
0822     struct timeval tv_end, tv_now, tv_timeout = {RECV_TMOUT, 0};
0823     u32 idx_rx = 0, idx_fq = 0, rcvd, i, pkts_sent = 0;
0824     struct pkt_stream *pkt_stream = ifobj->pkt_stream;
0825     struct xsk_socket_info *xsk = ifobj->xsk;
0826     struct xsk_umem_info *umem = xsk->umem;
0827     struct pkt *pkt;
0828     int ret;
0829 
0830     ret = gettimeofday(&tv_now, NULL);
0831     if (ret)
0832         exit_with_error(errno);
0833     timeradd(&tv_now, &tv_timeout, &tv_end);
0834 
0835     pkt = pkt_stream_get_next_rx_pkt(pkt_stream, &pkts_sent);
0836     while (pkt) {
0837         ret = gettimeofday(&tv_now, NULL);
0838         if (ret)
0839             exit_with_error(errno);
0840         if (timercmp(&tv_now, &tv_end, >)) {
0841             ksft_print_msg("ERROR: [%s] Receive loop timed out\n", __func__);
0842             return TEST_FAILURE;
0843         }
0844 
0845         kick_rx(xsk);
0846 
0847         rcvd = xsk_ring_cons__peek(&xsk->rx, BATCH_SIZE, &idx_rx);
0848         if (!rcvd) {
0849             if (xsk_ring_prod__needs_wakeup(&umem->fq)) {
0850                 ret = poll(fds, 1, POLL_TMOUT);
0851                 if (ret < 0)
0852                     exit_with_error(-ret);
0853             }
0854             continue;
0855         }
0856 
0857         if (ifobj->use_fill_ring) {
0858             ret = xsk_ring_prod__reserve(&umem->fq, rcvd, &idx_fq);
0859             while (ret != rcvd) {
0860                 if (ret < 0)
0861                     exit_with_error(-ret);
0862                 if (xsk_ring_prod__needs_wakeup(&umem->fq)) {
0863                     ret = poll(fds, 1, POLL_TMOUT);
0864                     if (ret < 0)
0865                         exit_with_error(-ret);
0866                 }
0867                 ret = xsk_ring_prod__reserve(&umem->fq, rcvd, &idx_fq);
0868             }
0869         }
0870 
0871         for (i = 0; i < rcvd; i++) {
0872             const struct xdp_desc *desc = xsk_ring_cons__rx_desc(&xsk->rx, idx_rx++);
0873             u64 addr = desc->addr, orig;
0874 
0875             orig = xsk_umem__extract_addr(addr);
0876             addr = xsk_umem__add_offset_to_addr(addr);
0877 
0878             if (!is_pkt_valid(pkt, umem->buffer, addr, desc->len) ||
0879                 !is_offset_correct(umem, pkt_stream, addr, pkt->addr))
0880                 return TEST_FAILURE;
0881 
0882             if (ifobj->use_fill_ring)
0883                 *xsk_ring_prod__fill_addr(&umem->fq, idx_fq++) = orig;
0884             pkt = pkt_stream_get_next_rx_pkt(pkt_stream, &pkts_sent);
0885         }
0886 
0887         if (ifobj->use_fill_ring)
0888             xsk_ring_prod__submit(&umem->fq, rcvd);
0889         if (ifobj->release_rx)
0890             xsk_ring_cons__release(&xsk->rx, rcvd);
0891 
0892         pthread_mutex_lock(&pacing_mutex);
0893         pkts_in_flight -= pkts_sent;
0894         if (pkts_in_flight < umem->num_frames)
0895             pthread_cond_signal(&pacing_cond);
0896         pthread_mutex_unlock(&pacing_mutex);
0897         pkts_sent = 0;
0898     }
0899 
0900     return TEST_PASS;
0901 }
0902 
0903 static int __send_pkts(struct ifobject *ifobject, u32 *pkt_nb)
0904 {
0905     struct xsk_socket_info *xsk = ifobject->xsk;
0906     u32 i, idx, valid_pkts = 0;
0907 
0908     while (xsk_ring_prod__reserve(&xsk->tx, BATCH_SIZE, &idx) < BATCH_SIZE)
0909         complete_pkts(xsk, BATCH_SIZE);
0910 
0911     for (i = 0; i < BATCH_SIZE; i++) {
0912         struct xdp_desc *tx_desc = xsk_ring_prod__tx_desc(&xsk->tx, idx + i);
0913         struct pkt *pkt = pkt_generate(ifobject, *pkt_nb);
0914 
0915         if (!pkt)
0916             break;
0917 
0918         tx_desc->addr = pkt->addr;
0919         tx_desc->len = pkt->len;
0920         (*pkt_nb)++;
0921         if (pkt->valid)
0922             valid_pkts++;
0923     }
0924 
0925     pthread_mutex_lock(&pacing_mutex);
0926     pkts_in_flight += valid_pkts;
0927     /* pkts_in_flight might be negative if many invalid packets are sent */
0928     if (pkts_in_flight >= (int)(ifobject->umem->num_frames - BATCH_SIZE)) {
0929         kick_tx(xsk);
0930         pthread_cond_wait(&pacing_cond, &pacing_mutex);
0931     }
0932     pthread_mutex_unlock(&pacing_mutex);
0933 
0934     xsk_ring_prod__submit(&xsk->tx, i);
0935     xsk->outstanding_tx += valid_pkts;
0936     if (complete_pkts(xsk, i))
0937         return TEST_FAILURE;
0938 
0939     usleep(10);
0940     return TEST_PASS;
0941 }
0942 
0943 static void wait_for_tx_completion(struct xsk_socket_info *xsk)
0944 {
0945     while (xsk->outstanding_tx)
0946         complete_pkts(xsk, BATCH_SIZE);
0947 }
0948 
0949 static int send_pkts(struct test_spec *test, struct ifobject *ifobject)
0950 {
0951     struct pollfd fds = { };
0952     u32 pkt_cnt = 0;
0953 
0954     fds.fd = xsk_socket__fd(ifobject->xsk->xsk);
0955     fds.events = POLLOUT;
0956 
0957     while (pkt_cnt < ifobject->pkt_stream->nb_pkts) {
0958         int err;
0959 
0960         if (ifobject->use_poll) {
0961             int ret;
0962 
0963             ret = poll(&fds, 1, POLL_TMOUT);
0964             if (ret <= 0)
0965                 continue;
0966 
0967             if (!(fds.revents & POLLOUT))
0968                 continue;
0969         }
0970 
0971         err = __send_pkts(ifobject, &pkt_cnt);
0972         if (err || test->fail)
0973             return TEST_FAILURE;
0974     }
0975 
0976     wait_for_tx_completion(ifobject->xsk);
0977     return TEST_PASS;
0978 }
0979 
0980 static int get_xsk_stats(struct xsk_socket *xsk, struct xdp_statistics *stats)
0981 {
0982     int fd = xsk_socket__fd(xsk), err;
0983     socklen_t optlen, expected_len;
0984 
0985     optlen = sizeof(*stats);
0986     err = getsockopt(fd, SOL_XDP, XDP_STATISTICS, stats, &optlen);
0987     if (err) {
0988         ksft_print_msg("[%s] getsockopt(XDP_STATISTICS) error %u %s\n",
0989                    __func__, -err, strerror(-err));
0990         return TEST_FAILURE;
0991     }
0992 
0993     expected_len = sizeof(struct xdp_statistics);
0994     if (optlen != expected_len) {
0995         ksft_print_msg("[%s] getsockopt optlen error. Expected: %u got: %u\n",
0996                    __func__, expected_len, optlen);
0997         return TEST_FAILURE;
0998     }
0999 
1000     return TEST_PASS;
1001 }
1002 
1003 static int validate_rx_dropped(struct ifobject *ifobject)
1004 {
1005     struct xsk_socket *xsk = ifobject->xsk->xsk;
1006     struct xdp_statistics stats;
1007     int err;
1008 
1009     kick_rx(ifobject->xsk);
1010 
1011     err = get_xsk_stats(xsk, &stats);
1012     if (err)
1013         return TEST_FAILURE;
1014 
1015     if (stats.rx_dropped == ifobject->pkt_stream->nb_pkts / 2)
1016         return TEST_PASS;
1017 
1018     return TEST_FAILURE;
1019 }
1020 
1021 static int validate_rx_full(struct ifobject *ifobject)
1022 {
1023     struct xsk_socket *xsk = ifobject->xsk->xsk;
1024     struct xdp_statistics stats;
1025     int err;
1026 
1027     usleep(1000);
1028     kick_rx(ifobject->xsk);
1029 
1030     err = get_xsk_stats(xsk, &stats);
1031     if (err)
1032         return TEST_FAILURE;
1033 
1034     if (stats.rx_ring_full)
1035         return TEST_PASS;
1036 
1037     return TEST_FAILURE;
1038 }
1039 
1040 static int validate_fill_empty(struct ifobject *ifobject)
1041 {
1042     struct xsk_socket *xsk = ifobject->xsk->xsk;
1043     struct xdp_statistics stats;
1044     int err;
1045 
1046     usleep(1000);
1047     kick_rx(ifobject->xsk);
1048 
1049     err = get_xsk_stats(xsk, &stats);
1050     if (err)
1051         return TEST_FAILURE;
1052 
1053     if (stats.rx_fill_ring_empty_descs)
1054         return TEST_PASS;
1055 
1056     return TEST_FAILURE;
1057 }
1058 
1059 static int validate_tx_invalid_descs(struct ifobject *ifobject)
1060 {
1061     struct xsk_socket *xsk = ifobject->xsk->xsk;
1062     int fd = xsk_socket__fd(xsk);
1063     struct xdp_statistics stats;
1064     socklen_t optlen;
1065     int err;
1066 
1067     optlen = sizeof(stats);
1068     err = getsockopt(fd, SOL_XDP, XDP_STATISTICS, &stats, &optlen);
1069     if (err) {
1070         ksft_print_msg("[%s] getsockopt(XDP_STATISTICS) error %u %s\n",
1071                    __func__, -err, strerror(-err));
1072         return TEST_FAILURE;
1073     }
1074 
1075     if (stats.tx_invalid_descs != ifobject->pkt_stream->nb_pkts / 2) {
1076         ksft_print_msg("[%s] tx_invalid_descs incorrect. Got [%u] expected [%u]\n",
1077                    __func__, stats.tx_invalid_descs, ifobject->pkt_stream->nb_pkts);
1078         return TEST_FAILURE;
1079     }
1080 
1081     return TEST_PASS;
1082 }
1083 
1084 static void thread_common_ops(struct test_spec *test, struct ifobject *ifobject)
1085 {
1086     u64 umem_sz = ifobject->umem->num_frames * ifobject->umem->frame_size;
1087     int mmap_flags = MAP_PRIVATE | MAP_ANONYMOUS | MAP_NORESERVE;
1088     LIBBPF_OPTS(bpf_xdp_query_opts, opts);
1089     int ret, ifindex;
1090     void *bufs;
1091     u32 i;
1092 
1093     ifobject->ns_fd = switch_namespace(ifobject->nsname);
1094 
1095     if (ifobject->umem->unaligned_mode)
1096         mmap_flags |= MAP_HUGETLB;
1097 
1098     bufs = mmap(NULL, umem_sz, PROT_READ | PROT_WRITE, mmap_flags, -1, 0);
1099     if (bufs == MAP_FAILED)
1100         exit_with_error(errno);
1101 
1102     ret = xsk_configure_umem(ifobject->umem, bufs, umem_sz);
1103     if (ret)
1104         exit_with_error(-ret);
1105 
1106     for (i = 0; i < test->nb_sockets; i++) {
1107         u32 ctr = 0;
1108 
1109         while (ctr++ < SOCK_RECONF_CTR) {
1110             ret = xsk_configure_socket(&ifobject->xsk_arr[i], ifobject->umem,
1111                            ifobject, !!i);
1112             if (!ret)
1113                 break;
1114 
1115             /* Retry if it fails as xsk_socket__create() is asynchronous */
1116             if (ctr >= SOCK_RECONF_CTR)
1117                 exit_with_error(-ret);
1118             usleep(USLEEP_MAX);
1119         }
1120 
1121         if (ifobject->busy_poll)
1122             enable_busy_poll(&ifobject->xsk_arr[i]);
1123     }
1124 
1125     ifobject->xsk = &ifobject->xsk_arr[0];
1126 
1127     if (!ifobject->rx_on)
1128         return;
1129 
1130     ifindex = if_nametoindex(ifobject->ifname);
1131     if (!ifindex)
1132         exit_with_error(errno);
1133 
1134     ret = xsk_setup_xdp_prog_xsk(ifobject->xsk->xsk, &ifobject->xsk_map_fd);
1135     if (ret)
1136         exit_with_error(-ret);
1137 
1138     ret = bpf_xdp_query(ifindex, ifobject->xdp_flags, &opts);
1139     if (ret)
1140         exit_with_error(-ret);
1141 
1142     if (ifobject->xdp_flags & XDP_FLAGS_SKB_MODE) {
1143         if (opts.attach_mode != XDP_ATTACHED_SKB) {
1144             ksft_print_msg("ERROR: [%s] XDP prog not in SKB mode\n");
1145             exit_with_error(-EINVAL);
1146         }
1147     } else if (ifobject->xdp_flags & XDP_FLAGS_DRV_MODE) {
1148         if (opts.attach_mode != XDP_ATTACHED_DRV) {
1149             ksft_print_msg("ERROR: [%s] XDP prog not in DRV mode\n");
1150             exit_with_error(-EINVAL);
1151         }
1152     }
1153 
1154     ret = xsk_socket__update_xskmap(ifobject->xsk->xsk, ifobject->xsk_map_fd);
1155     if (ret)
1156         exit_with_error(-ret);
1157 }
1158 
1159 static void testapp_cleanup_xsk_res(struct ifobject *ifobj)
1160 {
1161     print_verbose("Destroying socket\n");
1162     xsk_socket__delete(ifobj->xsk->xsk);
1163     munmap(ifobj->umem->buffer, ifobj->umem->num_frames * ifobj->umem->frame_size);
1164     xsk_umem__delete(ifobj->umem->umem);
1165 }
1166 
1167 static void *worker_testapp_validate_tx(void *arg)
1168 {
1169     struct test_spec *test = (struct test_spec *)arg;
1170     struct ifobject *ifobject = test->ifobj_tx;
1171     int err;
1172 
1173     if (test->current_step == 1)
1174         thread_common_ops(test, ifobject);
1175 
1176     print_verbose("Sending %d packets on interface %s\n", ifobject->pkt_stream->nb_pkts,
1177               ifobject->ifname);
1178     err = send_pkts(test, ifobject);
1179 
1180     if (!err && ifobject->validation_func)
1181         err = ifobject->validation_func(ifobject);
1182     if (err)
1183         report_failure(test);
1184 
1185     if (test->total_steps == test->current_step || err)
1186         testapp_cleanup_xsk_res(ifobject);
1187     pthread_exit(NULL);
1188 }
1189 
1190 static void xsk_populate_fill_ring(struct xsk_umem_info *umem, struct pkt_stream *pkt_stream)
1191 {
1192     u32 idx = 0, i, buffers_to_fill;
1193     int ret;
1194 
1195     if (umem->num_frames < XSK_RING_PROD__DEFAULT_NUM_DESCS)
1196         buffers_to_fill = umem->num_frames;
1197     else
1198         buffers_to_fill = XSK_RING_PROD__DEFAULT_NUM_DESCS;
1199 
1200     ret = xsk_ring_prod__reserve(&umem->fq, buffers_to_fill, &idx);
1201     if (ret != buffers_to_fill)
1202         exit_with_error(ENOSPC);
1203     for (i = 0; i < buffers_to_fill; i++) {
1204         u64 addr;
1205 
1206         if (pkt_stream->use_addr_for_fill) {
1207             struct pkt *pkt = pkt_stream_get_pkt(pkt_stream, i);
1208 
1209             if (!pkt)
1210                 break;
1211             addr = pkt->addr;
1212         } else {
1213             addr = i * umem->frame_size;
1214         }
1215 
1216         *xsk_ring_prod__fill_addr(&umem->fq, idx++) = addr;
1217     }
1218     xsk_ring_prod__submit(&umem->fq, buffers_to_fill);
1219 }
1220 
1221 static void *worker_testapp_validate_rx(void *arg)
1222 {
1223     struct test_spec *test = (struct test_spec *)arg;
1224     struct ifobject *ifobject = test->ifobj_rx;
1225     struct pollfd fds = { };
1226     int err;
1227 
1228     if (test->current_step == 1)
1229         thread_common_ops(test, ifobject);
1230 
1231     xsk_populate_fill_ring(ifobject->umem, ifobject->pkt_stream);
1232 
1233     fds.fd = xsk_socket__fd(ifobject->xsk->xsk);
1234     fds.events = POLLIN;
1235 
1236     pthread_barrier_wait(&barr);
1237 
1238     err = receive_pkts(ifobject, &fds);
1239 
1240     if (!err && ifobject->validation_func)
1241         err = ifobject->validation_func(ifobject);
1242     if (err) {
1243         report_failure(test);
1244         pthread_mutex_lock(&pacing_mutex);
1245         pthread_cond_signal(&pacing_cond);
1246         pthread_mutex_unlock(&pacing_mutex);
1247     }
1248 
1249     if (test->total_steps == test->current_step || err)
1250         testapp_cleanup_xsk_res(ifobject);
1251     pthread_exit(NULL);
1252 }
1253 
1254 static int testapp_validate_traffic(struct test_spec *test)
1255 {
1256     struct ifobject *ifobj_tx = test->ifobj_tx;
1257     struct ifobject *ifobj_rx = test->ifobj_rx;
1258     pthread_t t0, t1;
1259 
1260     if (pthread_barrier_init(&barr, NULL, 2))
1261         exit_with_error(errno);
1262 
1263     test->current_step++;
1264     pkt_stream_reset(ifobj_rx->pkt_stream);
1265     pkts_in_flight = 0;
1266 
1267     /*Spawn RX thread */
1268     pthread_create(&t0, NULL, ifobj_rx->func_ptr, test);
1269 
1270     pthread_barrier_wait(&barr);
1271     if (pthread_barrier_destroy(&barr))
1272         exit_with_error(errno);
1273 
1274     /*Spawn TX thread */
1275     pthread_create(&t1, NULL, ifobj_tx->func_ptr, test);
1276 
1277     pthread_join(t1, NULL);
1278     pthread_join(t0, NULL);
1279 
1280     return !!test->fail;
1281 }
1282 
1283 static void testapp_teardown(struct test_spec *test)
1284 {
1285     int i;
1286 
1287     test_spec_set_name(test, "TEARDOWN");
1288     for (i = 0; i < MAX_TEARDOWN_ITER; i++) {
1289         if (testapp_validate_traffic(test))
1290             return;
1291         test_spec_reset(test);
1292     }
1293 }
1294 
1295 static void swap_directions(struct ifobject **ifobj1, struct ifobject **ifobj2)
1296 {
1297     thread_func_t tmp_func_ptr = (*ifobj1)->func_ptr;
1298     struct ifobject *tmp_ifobj = (*ifobj1);
1299 
1300     (*ifobj1)->func_ptr = (*ifobj2)->func_ptr;
1301     (*ifobj2)->func_ptr = tmp_func_ptr;
1302 
1303     *ifobj1 = *ifobj2;
1304     *ifobj2 = tmp_ifobj;
1305 }
1306 
1307 static void testapp_bidi(struct test_spec *test)
1308 {
1309     test_spec_set_name(test, "BIDIRECTIONAL");
1310     test->ifobj_tx->rx_on = true;
1311     test->ifobj_rx->tx_on = true;
1312     test->total_steps = 2;
1313     if (testapp_validate_traffic(test))
1314         return;
1315 
1316     print_verbose("Switching Tx/Rx vectors\n");
1317     swap_directions(&test->ifobj_rx, &test->ifobj_tx);
1318     testapp_validate_traffic(test);
1319 
1320     swap_directions(&test->ifobj_rx, &test->ifobj_tx);
1321 }
1322 
1323 static void swap_xsk_resources(struct ifobject *ifobj_tx, struct ifobject *ifobj_rx)
1324 {
1325     int ret;
1326 
1327     xsk_socket__delete(ifobj_tx->xsk->xsk);
1328     xsk_socket__delete(ifobj_rx->xsk->xsk);
1329     ifobj_tx->xsk = &ifobj_tx->xsk_arr[1];
1330     ifobj_rx->xsk = &ifobj_rx->xsk_arr[1];
1331 
1332     ret = xsk_socket__update_xskmap(ifobj_rx->xsk->xsk, ifobj_rx->xsk_map_fd);
1333     if (ret)
1334         exit_with_error(-ret);
1335 }
1336 
1337 static void testapp_bpf_res(struct test_spec *test)
1338 {
1339     test_spec_set_name(test, "BPF_RES");
1340     test->total_steps = 2;
1341     test->nb_sockets = 2;
1342     if (testapp_validate_traffic(test))
1343         return;
1344 
1345     swap_xsk_resources(test->ifobj_tx, test->ifobj_rx);
1346     testapp_validate_traffic(test);
1347 }
1348 
1349 static void testapp_headroom(struct test_spec *test)
1350 {
1351     test_spec_set_name(test, "UMEM_HEADROOM");
1352     test->ifobj_rx->umem->frame_headroom = UMEM_HEADROOM_TEST_SIZE;
1353     testapp_validate_traffic(test);
1354 }
1355 
1356 static void testapp_stats_rx_dropped(struct test_spec *test)
1357 {
1358     test_spec_set_name(test, "STAT_RX_DROPPED");
1359     test->ifobj_rx->umem->frame_headroom = test->ifobj_rx->umem->frame_size -
1360         XDP_PACKET_HEADROOM - MIN_PKT_SIZE * 3;
1361     pkt_stream_replace_half(test, MIN_PKT_SIZE * 4, 0);
1362     pkt_stream_receive_half(test);
1363     test->ifobj_rx->validation_func = validate_rx_dropped;
1364     testapp_validate_traffic(test);
1365 }
1366 
1367 static void testapp_stats_tx_invalid_descs(struct test_spec *test)
1368 {
1369     test_spec_set_name(test, "STAT_TX_INVALID");
1370     pkt_stream_replace_half(test, XSK_UMEM__INVALID_FRAME_SIZE, 0);
1371     test->ifobj_tx->validation_func = validate_tx_invalid_descs;
1372     testapp_validate_traffic(test);
1373 
1374     pkt_stream_restore_default(test);
1375 }
1376 
1377 static void testapp_stats_rx_full(struct test_spec *test)
1378 {
1379     test_spec_set_name(test, "STAT_RX_FULL");
1380     pkt_stream_replace(test, DEFAULT_UMEM_BUFFERS + DEFAULT_UMEM_BUFFERS / 2, PKT_SIZE);
1381     test->ifobj_rx->pkt_stream = pkt_stream_generate(test->ifobj_rx->umem,
1382                              DEFAULT_UMEM_BUFFERS, PKT_SIZE);
1383     if (!test->ifobj_rx->pkt_stream)
1384         exit_with_error(ENOMEM);
1385 
1386     test->ifobj_rx->xsk->rxqsize = DEFAULT_UMEM_BUFFERS;
1387     test->ifobj_rx->release_rx = false;
1388     test->ifobj_rx->validation_func = validate_rx_full;
1389     testapp_validate_traffic(test);
1390 
1391     pkt_stream_restore_default(test);
1392 }
1393 
1394 static void testapp_stats_fill_empty(struct test_spec *test)
1395 {
1396     test_spec_set_name(test, "STAT_RX_FILL_EMPTY");
1397     pkt_stream_replace(test, DEFAULT_UMEM_BUFFERS + DEFAULT_UMEM_BUFFERS / 2, PKT_SIZE);
1398     test->ifobj_rx->pkt_stream = pkt_stream_generate(test->ifobj_rx->umem,
1399                              DEFAULT_UMEM_BUFFERS, PKT_SIZE);
1400     if (!test->ifobj_rx->pkt_stream)
1401         exit_with_error(ENOMEM);
1402 
1403     test->ifobj_rx->use_fill_ring = false;
1404     test->ifobj_rx->validation_func = validate_fill_empty;
1405     testapp_validate_traffic(test);
1406 
1407     pkt_stream_restore_default(test);
1408 }
1409 
1410 /* Simple test */
1411 static bool hugepages_present(struct ifobject *ifobject)
1412 {
1413     const size_t mmap_sz = 2 * ifobject->umem->num_frames * ifobject->umem->frame_size;
1414     void *bufs;
1415 
1416     bufs = mmap(NULL, mmap_sz, PROT_READ | PROT_WRITE,
1417             MAP_PRIVATE | MAP_ANONYMOUS | MAP_HUGETLB, -1, 0);
1418     if (bufs == MAP_FAILED)
1419         return false;
1420 
1421     munmap(bufs, mmap_sz);
1422     return true;
1423 }
1424 
1425 static bool testapp_unaligned(struct test_spec *test)
1426 {
1427     if (!hugepages_present(test->ifobj_tx)) {
1428         ksft_test_result_skip("No 2M huge pages present.\n");
1429         return false;
1430     }
1431 
1432     test_spec_set_name(test, "UNALIGNED_MODE");
1433     test->ifobj_tx->umem->unaligned_mode = true;
1434     test->ifobj_rx->umem->unaligned_mode = true;
1435     /* Let half of the packets straddle a buffer boundrary */
1436     pkt_stream_replace_half(test, PKT_SIZE, -PKT_SIZE / 2);
1437     test->ifobj_rx->pkt_stream->use_addr_for_fill = true;
1438     testapp_validate_traffic(test);
1439 
1440     pkt_stream_restore_default(test);
1441     return true;
1442 }
1443 
1444 static void testapp_single_pkt(struct test_spec *test)
1445 {
1446     struct pkt pkts[] = {{0x1000, PKT_SIZE, 0, true}};
1447 
1448     pkt_stream_generate_custom(test, pkts, ARRAY_SIZE(pkts));
1449     testapp_validate_traffic(test);
1450     pkt_stream_restore_default(test);
1451 }
1452 
1453 static void testapp_invalid_desc(struct test_spec *test)
1454 {
1455     struct pkt pkts[] = {
1456         /* Zero packet address allowed */
1457         {0, PKT_SIZE, 0, true},
1458         /* Allowed packet */
1459         {0x1000, PKT_SIZE, 0, true},
1460         /* Straddling the start of umem */
1461         {-2, PKT_SIZE, 0, false},
1462         /* Packet too large */
1463         {0x2000, XSK_UMEM__INVALID_FRAME_SIZE, 0, false},
1464         /* After umem ends */
1465         {UMEM_SIZE, PKT_SIZE, 0, false},
1466         /* Straddle the end of umem */
1467         {UMEM_SIZE - PKT_SIZE / 2, PKT_SIZE, 0, false},
1468         /* Straddle a page boundrary */
1469         {0x3000 - PKT_SIZE / 2, PKT_SIZE, 0, false},
1470         /* Straddle a 2K boundrary */
1471         {0x3800 - PKT_SIZE / 2, PKT_SIZE, 0, true},
1472         /* Valid packet for synch so that something is received */
1473         {0x4000, PKT_SIZE, 0, true}};
1474 
1475     if (test->ifobj_tx->umem->unaligned_mode) {
1476         /* Crossing a page boundrary allowed */
1477         pkts[6].valid = true;
1478     }
1479     if (test->ifobj_tx->umem->frame_size == XSK_UMEM__DEFAULT_FRAME_SIZE / 2) {
1480         /* Crossing a 2K frame size boundrary not allowed */
1481         pkts[7].valid = false;
1482     }
1483 
1484     pkt_stream_generate_custom(test, pkts, ARRAY_SIZE(pkts));
1485     testapp_validate_traffic(test);
1486     pkt_stream_restore_default(test);
1487 }
1488 
1489 static void init_iface(struct ifobject *ifobj, const char *dst_mac, const char *src_mac,
1490                const char *dst_ip, const char *src_ip, const u16 dst_port,
1491                const u16 src_port, thread_func_t func_ptr)
1492 {
1493     struct in_addr ip;
1494 
1495     memcpy(ifobj->dst_mac, dst_mac, ETH_ALEN);
1496     memcpy(ifobj->src_mac, src_mac, ETH_ALEN);
1497 
1498     inet_aton(dst_ip, &ip);
1499     ifobj->dst_ip = ip.s_addr;
1500 
1501     inet_aton(src_ip, &ip);
1502     ifobj->src_ip = ip.s_addr;
1503 
1504     ifobj->dst_port = dst_port;
1505     ifobj->src_port = src_port;
1506 
1507     ifobj->func_ptr = func_ptr;
1508 }
1509 
1510 static void run_pkt_test(struct test_spec *test, enum test_mode mode, enum test_type type)
1511 {
1512     switch (type) {
1513     case TEST_TYPE_STATS_RX_DROPPED:
1514         testapp_stats_rx_dropped(test);
1515         break;
1516     case TEST_TYPE_STATS_TX_INVALID_DESCS:
1517         testapp_stats_tx_invalid_descs(test);
1518         break;
1519     case TEST_TYPE_STATS_RX_FULL:
1520         testapp_stats_rx_full(test);
1521         break;
1522     case TEST_TYPE_STATS_FILL_EMPTY:
1523         testapp_stats_fill_empty(test);
1524         break;
1525     case TEST_TYPE_TEARDOWN:
1526         testapp_teardown(test);
1527         break;
1528     case TEST_TYPE_BIDI:
1529         testapp_bidi(test);
1530         break;
1531     case TEST_TYPE_BPF_RES:
1532         testapp_bpf_res(test);
1533         break;
1534     case TEST_TYPE_RUN_TO_COMPLETION:
1535         test_spec_set_name(test, "RUN_TO_COMPLETION");
1536         testapp_validate_traffic(test);
1537         break;
1538     case TEST_TYPE_RUN_TO_COMPLETION_SINGLE_PKT:
1539         test_spec_set_name(test, "RUN_TO_COMPLETION_SINGLE_PKT");
1540         testapp_single_pkt(test);
1541         break;
1542     case TEST_TYPE_RUN_TO_COMPLETION_2K_FRAME:
1543         test_spec_set_name(test, "RUN_TO_COMPLETION_2K_FRAME_SIZE");
1544         test->ifobj_tx->umem->frame_size = 2048;
1545         test->ifobj_rx->umem->frame_size = 2048;
1546         pkt_stream_replace(test, DEFAULT_PKT_CNT, PKT_SIZE);
1547         testapp_validate_traffic(test);
1548 
1549         pkt_stream_restore_default(test);
1550         break;
1551     case TEST_TYPE_POLL:
1552         test->ifobj_tx->use_poll = true;
1553         test->ifobj_rx->use_poll = true;
1554         test_spec_set_name(test, "POLL");
1555         testapp_validate_traffic(test);
1556         break;
1557     case TEST_TYPE_ALIGNED_INV_DESC:
1558         test_spec_set_name(test, "ALIGNED_INV_DESC");
1559         testapp_invalid_desc(test);
1560         break;
1561     case TEST_TYPE_ALIGNED_INV_DESC_2K_FRAME:
1562         test_spec_set_name(test, "ALIGNED_INV_DESC_2K_FRAME_SIZE");
1563         test->ifobj_tx->umem->frame_size = 2048;
1564         test->ifobj_rx->umem->frame_size = 2048;
1565         testapp_invalid_desc(test);
1566         break;
1567     case TEST_TYPE_UNALIGNED_INV_DESC:
1568         if (!hugepages_present(test->ifobj_tx)) {
1569             ksft_test_result_skip("No 2M huge pages present.\n");
1570             return;
1571         }
1572         test_spec_set_name(test, "UNALIGNED_INV_DESC");
1573         test->ifobj_tx->umem->unaligned_mode = true;
1574         test->ifobj_rx->umem->unaligned_mode = true;
1575         testapp_invalid_desc(test);
1576         break;
1577     case TEST_TYPE_UNALIGNED:
1578         if (!testapp_unaligned(test))
1579             return;
1580         break;
1581     case TEST_TYPE_HEADROOM:
1582         testapp_headroom(test);
1583         break;
1584     default:
1585         break;
1586     }
1587 
1588     if (!test->fail)
1589         ksft_test_result_pass("PASS: %s %s%s\n", mode_string(test), busy_poll_string(test),
1590                       test->name);
1591 }
1592 
1593 static struct ifobject *ifobject_create(void)
1594 {
1595     struct ifobject *ifobj;
1596 
1597     ifobj = calloc(1, sizeof(struct ifobject));
1598     if (!ifobj)
1599         return NULL;
1600 
1601     ifobj->xsk_arr = calloc(MAX_SOCKETS, sizeof(*ifobj->xsk_arr));
1602     if (!ifobj->xsk_arr)
1603         goto out_xsk_arr;
1604 
1605     ifobj->umem = calloc(1, sizeof(*ifobj->umem));
1606     if (!ifobj->umem)
1607         goto out_umem;
1608 
1609     return ifobj;
1610 
1611 out_umem:
1612     free(ifobj->xsk_arr);
1613 out_xsk_arr:
1614     free(ifobj);
1615     return NULL;
1616 }
1617 
1618 static void ifobject_delete(struct ifobject *ifobj)
1619 {
1620     free(ifobj->umem);
1621     free(ifobj->xsk_arr);
1622     free(ifobj);
1623 }
1624 
1625 int main(int argc, char **argv)
1626 {
1627     struct pkt_stream *pkt_stream_default;
1628     struct ifobject *ifobj_tx, *ifobj_rx;
1629     u32 i, j, failed_tests = 0;
1630     struct test_spec test;
1631 
1632     /* Use libbpf 1.0 API mode */
1633     libbpf_set_strict_mode(LIBBPF_STRICT_ALL);
1634 
1635     ifobj_tx = ifobject_create();
1636     if (!ifobj_tx)
1637         exit_with_error(ENOMEM);
1638     ifobj_rx = ifobject_create();
1639     if (!ifobj_rx)
1640         exit_with_error(ENOMEM);
1641 
1642     setlocale(LC_ALL, "");
1643 
1644     parse_command_line(ifobj_tx, ifobj_rx, argc, argv);
1645 
1646     if (!validate_interface(ifobj_tx) || !validate_interface(ifobj_rx)) {
1647         usage(basename(argv[0]));
1648         ksft_exit_xfail();
1649     }
1650 
1651     init_iface(ifobj_tx, MAC1, MAC2, IP1, IP2, UDP_PORT1, UDP_PORT2,
1652            worker_testapp_validate_tx);
1653     init_iface(ifobj_rx, MAC2, MAC1, IP2, IP1, UDP_PORT2, UDP_PORT1,
1654            worker_testapp_validate_rx);
1655 
1656     test_spec_init(&test, ifobj_tx, ifobj_rx, 0);
1657     pkt_stream_default = pkt_stream_generate(ifobj_tx->umem, DEFAULT_PKT_CNT, PKT_SIZE);
1658     if (!pkt_stream_default)
1659         exit_with_error(ENOMEM);
1660     test.pkt_stream_default = pkt_stream_default;
1661 
1662     ksft_set_plan(TEST_MODE_MAX * TEST_TYPE_MAX);
1663 
1664     for (i = 0; i < TEST_MODE_MAX; i++)
1665         for (j = 0; j < TEST_TYPE_MAX; j++) {
1666             test_spec_init(&test, ifobj_tx, ifobj_rx, i);
1667             run_pkt_test(&test, i, j);
1668             usleep(USLEEP_MAX);
1669 
1670             if (test.fail)
1671                 failed_tests++;
1672         }
1673 
1674     pkt_stream_delete(pkt_stream_default);
1675     ifobject_delete(ifobj_tx);
1676     ifobject_delete(ifobj_rx);
1677 
1678     if (failed_tests)
1679         ksft_exit_fail();
1680     else
1681         ksft_exit_pass();
1682 }
[ Source navigation ] [ Diff markup ] [ Identifier search ] [ general search ]

This page was automatically generated by the 2.1.0 LXR engine.

The LXR team
Valid CSS 2.1! Valid HTML 4.01!