Back to home page

OSCL-LXR
 
 

     

0001 // SPDX-License-Identifier: GPL-2.0
0002 /* Toeplitz test
0003  *
0004  * 1. Read packets and their rx_hash using PF_PACKET/TPACKET_V3
0005  * 2. Compute the rx_hash in software based on the packet contents
0006  * 3. Compare the two
0007  *
0008  * Optionally, either '-C $rx_irq_cpu_list' or '-r $rps_bitmap' may be given.
0009  *
0010  * If '-C $rx_irq_cpu_list' is given, also
0011  *
0012  * 4. Identify the cpu on which the packet arrived with PACKET_FANOUT_CPU
0013  * 5. Compute the rxqueue that RSS would select based on this rx_hash
0014  * 6. Using the $rx_irq_cpu_list map, identify the arriving cpu based on rxq irq
0015  * 7. Compare the cpus from 4 and 6
0016  *
0017  * Else if '-r $rps_bitmap' is given, also
0018  *
0019  * 4. Identify the cpu on which the packet arrived with PACKET_FANOUT_CPU
0020  * 5. Compute the cpu that RPS should select based on rx_hash and $rps_bitmap
0021  * 6. Compare the cpus from 4 and 5
0022  */
0023 
0024 #define _GNU_SOURCE
0025 
0026 #include <arpa/inet.h>
0027 #include <errno.h>
0028 #include <error.h>
0029 #include <fcntl.h>
0030 #include <getopt.h>
0031 #include <linux/filter.h>
0032 #include <linux/if_ether.h>
0033 #include <linux/if_packet.h>
0034 #include <net/if.h>
0035 #include <netdb.h>
0036 #include <netinet/ip.h>
0037 #include <netinet/ip6.h>
0038 #include <netinet/tcp.h>
0039 #include <netinet/udp.h>
0040 #include <poll.h>
0041 #include <stdbool.h>
0042 #include <stddef.h>
0043 #include <stdint.h>
0044 #include <stdio.h>
0045 #include <stdlib.h>
0046 #include <string.h>
0047 #include <sys/mman.h>
0048 #include <sys/socket.h>
0049 #include <sys/stat.h>
0050 #include <sys/sysinfo.h>
0051 #include <sys/time.h>
0052 #include <sys/types.h>
0053 #include <unistd.h>
0054 
0055 #include "../kselftest.h"
0056 
0057 #define TOEPLITZ_KEY_MIN_LEN    40
0058 #define TOEPLITZ_KEY_MAX_LEN    60
0059 
0060 #define TOEPLITZ_STR_LEN(K) (((K) * 3) - 1) /* hex encoded: AA:BB:CC:...:ZZ */
0061 #define TOEPLITZ_STR_MIN_LEN    TOEPLITZ_STR_LEN(TOEPLITZ_KEY_MIN_LEN)
0062 #define TOEPLITZ_STR_MAX_LEN    TOEPLITZ_STR_LEN(TOEPLITZ_KEY_MAX_LEN)
0063 
0064 #define FOUR_TUPLE_MAX_LEN  ((sizeof(struct in6_addr) * 2) + (sizeof(uint16_t) * 2))
0065 
0066 #define RSS_MAX_CPUS (1 << 16)  /* real constraint is PACKET_FANOUT_MAX */
0067 
0068 #define RPS_MAX_CPUS 16UL   /* must be a power of 2 */
0069 
0070 /* configuration options (cmdline arguments) */
0071 static uint16_t cfg_dport = 8000;
0072 static int cfg_family =     AF_INET6;
0073 static char *cfg_ifname =   "eth0";
0074 static int cfg_num_queues;
0075 static int cfg_num_rps_cpus;
0076 static bool cfg_sink;
0077 static int cfg_type =       SOCK_STREAM;
0078 static int cfg_timeout_msec =   1000;
0079 static bool cfg_verbose;
0080 
0081 /* global vars */
0082 static int num_cpus;
0083 static int ring_block_nr;
0084 static int ring_block_sz;
0085 
0086 /* stats */
0087 static int frames_received;
0088 static int frames_nohash;
0089 static int frames_error;
0090 
0091 #define log_verbose(args...)    do { if (cfg_verbose) fprintf(stderr, args); } while (0)
0092 
0093 /* tpacket ring */
0094 struct ring_state {
0095     int fd;
0096     char *mmap;
0097     int idx;
0098     int cpu;
0099 };
0100 
0101 static unsigned int rx_irq_cpus[RSS_MAX_CPUS];  /* map from rxq to cpu */
0102 static int rps_silo_to_cpu[RPS_MAX_CPUS];
0103 static unsigned char toeplitz_key[TOEPLITZ_KEY_MAX_LEN];
0104 static struct ring_state rings[RSS_MAX_CPUS];
0105 
0106 static inline uint32_t toeplitz(const unsigned char *four_tuple,
0107                 const unsigned char *key)
0108 {
0109     int i, bit, ret = 0;
0110     uint32_t key32;
0111 
0112     key32 = ntohl(*((uint32_t *)key));
0113     key += 4;
0114 
0115     for (i = 0; i < FOUR_TUPLE_MAX_LEN; i++) {
0116         for (bit = 7; bit >= 0; bit--) {
0117             if (four_tuple[i] & (1 << bit))
0118                 ret ^= key32;
0119 
0120             key32 <<= 1;
0121             key32 |= !!(key[0] & (1 << bit));
0122         }
0123         key++;
0124     }
0125 
0126     return ret;
0127 }
0128 
0129 /* Compare computed cpu with arrival cpu from packet_fanout_cpu */
0130 static void verify_rss(uint32_t rx_hash, int cpu)
0131 {
0132     int queue = rx_hash % cfg_num_queues;
0133 
0134     log_verbose(" rxq %d (cpu %d)", queue, rx_irq_cpus[queue]);
0135     if (rx_irq_cpus[queue] != cpu) {
0136         log_verbose(". error: rss cpu mismatch (%d)", cpu);
0137         frames_error++;
0138     }
0139 }
0140 
0141 static void verify_rps(uint64_t rx_hash, int cpu)
0142 {
0143     int silo = (rx_hash * cfg_num_rps_cpus) >> 32;
0144 
0145     log_verbose(" silo %d (cpu %d)", silo, rps_silo_to_cpu[silo]);
0146     if (rps_silo_to_cpu[silo] != cpu) {
0147         log_verbose(". error: rps cpu mismatch (%d)", cpu);
0148         frames_error++;
0149     }
0150 }
0151 
0152 static void log_rxhash(int cpu, uint32_t rx_hash,
0153                const char *addrs, int addr_len)
0154 {
0155     char saddr[INET6_ADDRSTRLEN], daddr[INET6_ADDRSTRLEN];
0156     uint16_t *ports;
0157 
0158     if (!inet_ntop(cfg_family, addrs, saddr, sizeof(saddr)) ||
0159         !inet_ntop(cfg_family, addrs + addr_len, daddr, sizeof(daddr)))
0160         error(1, 0, "address parse error");
0161 
0162     ports = (void *)addrs + (addr_len * 2);
0163     log_verbose("cpu %d: rx_hash 0x%08x [saddr %s daddr %s sport %02hu dport %02hu]",
0164             cpu, rx_hash, saddr, daddr,
0165             ntohs(ports[0]), ntohs(ports[1]));
0166 }
0167 
0168 /* Compare computed rxhash with rxhash received from tpacket_v3 */
0169 static void verify_rxhash(const char *pkt, uint32_t rx_hash, int cpu)
0170 {
0171     unsigned char four_tuple[FOUR_TUPLE_MAX_LEN] = {0};
0172     uint32_t rx_hash_sw;
0173     const char *addrs;
0174     int addr_len;
0175 
0176     if (cfg_family == AF_INET) {
0177         addr_len = sizeof(struct in_addr);
0178         addrs = pkt + offsetof(struct iphdr, saddr);
0179     } else {
0180         addr_len = sizeof(struct in6_addr);
0181         addrs = pkt + offsetof(struct ip6_hdr, ip6_src);
0182     }
0183 
0184     memcpy(four_tuple, addrs, (addr_len * 2) + (sizeof(uint16_t) * 2));
0185     rx_hash_sw = toeplitz(four_tuple, toeplitz_key);
0186 
0187     if (cfg_verbose)
0188         log_rxhash(cpu, rx_hash, addrs, addr_len);
0189 
0190     if (rx_hash != rx_hash_sw) {
0191         log_verbose(" != expected 0x%x\n", rx_hash_sw);
0192         frames_error++;
0193         return;
0194     }
0195 
0196     log_verbose(" OK");
0197     if (cfg_num_queues)
0198         verify_rss(rx_hash, cpu);
0199     else if (cfg_num_rps_cpus)
0200         verify_rps(rx_hash, cpu);
0201     log_verbose("\n");
0202 }
0203 
0204 static char *recv_frame(const struct ring_state *ring, char *frame)
0205 {
0206     struct tpacket3_hdr *hdr = (void *)frame;
0207 
0208     if (hdr->hv1.tp_rxhash)
0209         verify_rxhash(frame + hdr->tp_net, hdr->hv1.tp_rxhash,
0210                   ring->cpu);
0211     else
0212         frames_nohash++;
0213 
0214     return frame + hdr->tp_next_offset;
0215 }
0216 
0217 /* A single TPACKET_V3 block can hold multiple frames */
0218 static void recv_block(struct ring_state *ring)
0219 {
0220     struct tpacket_block_desc *block;
0221     char *frame;
0222     int i;
0223 
0224     block = (void *)(ring->mmap + ring->idx * ring_block_sz);
0225     if (!(block->hdr.bh1.block_status & TP_STATUS_USER))
0226         return;
0227 
0228     frame = (char *)block;
0229     frame += block->hdr.bh1.offset_to_first_pkt;
0230 
0231     for (i = 0; i < block->hdr.bh1.num_pkts; i++) {
0232         frame = recv_frame(ring, frame);
0233         frames_received++;
0234     }
0235 
0236     block->hdr.bh1.block_status = TP_STATUS_KERNEL;
0237     ring->idx = (ring->idx + 1) % ring_block_nr;
0238 }
0239 
0240 /* simple test: sleep once unconditionally and then process all rings */
0241 static void process_rings(void)
0242 {
0243     int i;
0244 
0245     usleep(1000 * cfg_timeout_msec);
0246 
0247     for (i = 0; i < num_cpus; i++)
0248         recv_block(&rings[i]);
0249 
0250     fprintf(stderr, "count: pass=%u nohash=%u fail=%u\n",
0251         frames_received - frames_nohash - frames_error,
0252         frames_nohash, frames_error);
0253 }
0254 
0255 static char *setup_ring(int fd)
0256 {
0257     struct tpacket_req3 req3 = {0};
0258     void *ring;
0259 
0260     req3.tp_retire_blk_tov = cfg_timeout_msec;
0261     req3.tp_feature_req_word = TP_FT_REQ_FILL_RXHASH;
0262 
0263     req3.tp_frame_size = 2048;
0264     req3.tp_frame_nr = 1 << 10;
0265     req3.tp_block_nr = 2;
0266 
0267     req3.tp_block_size = req3.tp_frame_size * req3.tp_frame_nr;
0268     req3.tp_block_size /= req3.tp_block_nr;
0269 
0270     if (setsockopt(fd, SOL_PACKET, PACKET_RX_RING, &req3, sizeof(req3)))
0271         error(1, errno, "setsockopt PACKET_RX_RING");
0272 
0273     ring_block_sz = req3.tp_block_size;
0274     ring_block_nr = req3.tp_block_nr;
0275 
0276     ring = mmap(0, req3.tp_block_size * req3.tp_block_nr,
0277             PROT_READ | PROT_WRITE,
0278             MAP_SHARED | MAP_LOCKED | MAP_POPULATE, fd, 0);
0279     if (ring == MAP_FAILED)
0280         error(1, 0, "mmap failed");
0281 
0282     return ring;
0283 }
0284 
0285 static void __set_filter(int fd, int off_proto, uint8_t proto, int off_dport)
0286 {
0287     struct sock_filter filter[] = {
0288         BPF_STMT(BPF_LD  + BPF_B   + BPF_ABS, SKF_AD_OFF + SKF_AD_PKTTYPE),
0289         BPF_JUMP(BPF_JMP + BPF_JEQ + BPF_K, PACKET_HOST, 0, 4),
0290         BPF_STMT(BPF_LD  + BPF_B   + BPF_ABS, off_proto),
0291         BPF_JUMP(BPF_JMP + BPF_JEQ + BPF_K, proto, 0, 2),
0292         BPF_STMT(BPF_LD  + BPF_H   + BPF_ABS, off_dport),
0293         BPF_JUMP(BPF_JMP + BPF_JEQ + BPF_K, cfg_dport, 1, 0),
0294         BPF_STMT(BPF_RET + BPF_K, 0),
0295         BPF_STMT(BPF_RET + BPF_K, 0xFFFF),
0296     };
0297     struct sock_fprog prog = {};
0298 
0299     prog.filter = filter;
0300     prog.len = ARRAY_SIZE(filter);
0301     if (setsockopt(fd, SOL_SOCKET, SO_ATTACH_FILTER, &prog, sizeof(prog)))
0302         error(1, errno, "setsockopt filter");
0303 }
0304 
0305 /* filter on transport protocol and destination port */
0306 static void set_filter(int fd)
0307 {
0308     const int off_dport = offsetof(struct tcphdr, dest);    /* same for udp */
0309     uint8_t proto;
0310 
0311     proto = cfg_type == SOCK_STREAM ? IPPROTO_TCP : IPPROTO_UDP;
0312     if (cfg_family == AF_INET)
0313         __set_filter(fd, offsetof(struct iphdr, protocol), proto,
0314                  sizeof(struct iphdr) + off_dport);
0315     else
0316         __set_filter(fd, offsetof(struct ip6_hdr, ip6_nxt), proto,
0317                  sizeof(struct ip6_hdr) + off_dport);
0318 }
0319 
0320 /* drop everything: used temporarily during setup */
0321 static void set_filter_null(int fd)
0322 {
0323     struct sock_filter filter[] = {
0324         BPF_STMT(BPF_RET + BPF_K, 0),
0325     };
0326     struct sock_fprog prog = {};
0327 
0328     prog.filter = filter;
0329     prog.len = ARRAY_SIZE(filter);
0330     if (setsockopt(fd, SOL_SOCKET, SO_ATTACH_FILTER, &prog, sizeof(prog)))
0331         error(1, errno, "setsockopt filter");
0332 }
0333 
0334 static int create_ring(char **ring)
0335 {
0336     struct fanout_args args = {
0337         .id = 1,
0338         .type_flags = PACKET_FANOUT_CPU,
0339         .max_num_members = RSS_MAX_CPUS
0340     };
0341     struct sockaddr_ll ll = { 0 };
0342     int fd, val;
0343 
0344     fd = socket(PF_PACKET, SOCK_DGRAM, 0);
0345     if (fd == -1)
0346         error(1, errno, "socket creation failed");
0347 
0348     val = TPACKET_V3;
0349     if (setsockopt(fd, SOL_PACKET, PACKET_VERSION, &val, sizeof(val)))
0350         error(1, errno, "setsockopt PACKET_VERSION");
0351     *ring = setup_ring(fd);
0352 
0353     /* block packets until all rings are added to the fanout group:
0354      * else packets can arrive during setup and get misclassified
0355      */
0356     set_filter_null(fd);
0357 
0358     ll.sll_family = AF_PACKET;
0359     ll.sll_ifindex = if_nametoindex(cfg_ifname);
0360     ll.sll_protocol = cfg_family == AF_INET ? htons(ETH_P_IP) :
0361                           htons(ETH_P_IPV6);
0362     if (bind(fd, (void *)&ll, sizeof(ll)))
0363         error(1, errno, "bind");
0364 
0365     /* must come after bind: verifies all programs in group match */
0366     if (setsockopt(fd, SOL_PACKET, PACKET_FANOUT, &args, sizeof(args))) {
0367         /* on failure, retry using old API if that is sufficient:
0368          * it has a hard limit of 256 sockets, so only try if
0369          * (a) only testing rxhash, not RSS or (b) <= 256 cpus.
0370          * in this API, the third argument is left implicit.
0371          */
0372         if (cfg_num_queues || num_cpus > 256 ||
0373             setsockopt(fd, SOL_PACKET, PACKET_FANOUT,
0374                    &args, sizeof(uint32_t)))
0375             error(1, errno, "setsockopt PACKET_FANOUT cpu");
0376     }
0377 
0378     return fd;
0379 }
0380 
0381 /* setup inet(6) socket to blackhole the test traffic, if arg '-s' */
0382 static int setup_sink(void)
0383 {
0384     int fd, val;
0385 
0386     fd = socket(cfg_family, cfg_type, 0);
0387     if (fd == -1)
0388         error(1, errno, "socket %d.%d", cfg_family, cfg_type);
0389 
0390     val = 1 << 20;
0391     if (setsockopt(fd, SOL_SOCKET, SO_RCVBUFFORCE, &val, sizeof(val)))
0392         error(1, errno, "setsockopt rcvbuf");
0393 
0394     return fd;
0395 }
0396 
0397 static void setup_rings(void)
0398 {
0399     int i;
0400 
0401     for (i = 0; i < num_cpus; i++) {
0402         rings[i].cpu = i;
0403         rings[i].fd = create_ring(&rings[i].mmap);
0404     }
0405 
0406     /* accept packets once all rings in the fanout group are up */
0407     for (i = 0; i < num_cpus; i++)
0408         set_filter(rings[i].fd);
0409 }
0410 
0411 static void cleanup_rings(void)
0412 {
0413     int i;
0414 
0415     for (i = 0; i < num_cpus; i++) {
0416         if (munmap(rings[i].mmap, ring_block_nr * ring_block_sz))
0417             error(1, errno, "munmap");
0418         if (close(rings[i].fd))
0419             error(1, errno, "close");
0420     }
0421 }
0422 
0423 static void parse_cpulist(const char *arg)
0424 {
0425     do {
0426         rx_irq_cpus[cfg_num_queues++] = strtol(arg, NULL, 10);
0427 
0428         arg = strchr(arg, ',');
0429         if (!arg)
0430             break;
0431         arg++;          // skip ','
0432     } while (1);
0433 }
0434 
0435 static void show_cpulist(void)
0436 {
0437     int i;
0438 
0439     for (i = 0; i < cfg_num_queues; i++)
0440         fprintf(stderr, "rxq %d: cpu %d\n", i, rx_irq_cpus[i]);
0441 }
0442 
0443 static void show_silos(void)
0444 {
0445     int i;
0446 
0447     for (i = 0; i < cfg_num_rps_cpus; i++)
0448         fprintf(stderr, "silo %d: cpu %d\n", i, rps_silo_to_cpu[i]);
0449 }
0450 
0451 static void parse_toeplitz_key(const char *str, int slen, unsigned char *key)
0452 {
0453     int i, ret, off;
0454 
0455     if (slen < TOEPLITZ_STR_MIN_LEN ||
0456         slen > TOEPLITZ_STR_MAX_LEN + 1)
0457         error(1, 0, "invalid toeplitz key");
0458 
0459     for (i = 0, off = 0; off < slen; i++, off += 3) {
0460         ret = sscanf(str + off, "%hhx", &key[i]);
0461         if (ret != 1)
0462             error(1, 0, "key parse error at %d off %d len %d",
0463                   i, off, slen);
0464     }
0465 }
0466 
0467 static void parse_rps_bitmap(const char *arg)
0468 {
0469     unsigned long bitmap;
0470     int i;
0471 
0472     bitmap = strtoul(arg, NULL, 0);
0473 
0474     if (bitmap & ~(RPS_MAX_CPUS - 1))
0475         error(1, 0, "rps bitmap 0x%lx out of bounds 0..%lu",
0476               bitmap, RPS_MAX_CPUS - 1);
0477 
0478     for (i = 0; i < RPS_MAX_CPUS; i++)
0479         if (bitmap & 1UL << i)
0480             rps_silo_to_cpu[cfg_num_rps_cpus++] = i;
0481 }
0482 
0483 static void parse_opts(int argc, char **argv)
0484 {
0485     static struct option long_options[] = {
0486         {"dport",   required_argument, 0, 'd'},
0487         {"cpus",    required_argument, 0, 'C'},
0488         {"key", required_argument, 0, 'k'},
0489         {"iface",   required_argument, 0, 'i'},
0490         {"ipv4",    no_argument, 0, '4'},
0491         {"ipv6",    no_argument, 0, '6'},
0492         {"sink",    no_argument, 0, 's'},
0493         {"tcp", no_argument, 0, 't'},
0494         {"timeout", required_argument, 0, 'T'},
0495         {"udp", no_argument, 0, 'u'},
0496         {"verbose", no_argument, 0, 'v'},
0497         {"rps", required_argument, 0, 'r'},
0498         {0, 0, 0, 0}
0499     };
0500     bool have_toeplitz = false;
0501     int index, c;
0502 
0503     while ((c = getopt_long(argc, argv, "46C:d:i:k:r:stT:uv", long_options, &index)) != -1) {
0504         switch (c) {
0505         case '4':
0506             cfg_family = AF_INET;
0507             break;
0508         case '6':
0509             cfg_family = AF_INET6;
0510             break;
0511         case 'C':
0512             parse_cpulist(optarg);
0513             break;
0514         case 'd':
0515             cfg_dport = strtol(optarg, NULL, 0);
0516             break;
0517         case 'i':
0518             cfg_ifname = optarg;
0519             break;
0520         case 'k':
0521             parse_toeplitz_key(optarg, strlen(optarg),
0522                        toeplitz_key);
0523             have_toeplitz = true;
0524             break;
0525         case 'r':
0526             parse_rps_bitmap(optarg);
0527             break;
0528         case 's':
0529             cfg_sink = true;
0530             break;
0531         case 't':
0532             cfg_type = SOCK_STREAM;
0533             break;
0534         case 'T':
0535             cfg_timeout_msec = strtol(optarg, NULL, 0);
0536             break;
0537         case 'u':
0538             cfg_type = SOCK_DGRAM;
0539             break;
0540         case 'v':
0541             cfg_verbose = true;
0542             break;
0543 
0544         default:
0545             error(1, 0, "unknown option %c", optopt);
0546             break;
0547         }
0548     }
0549 
0550     if (!have_toeplitz)
0551         error(1, 0, "Must supply rss key ('-k')");
0552 
0553     num_cpus = get_nprocs();
0554     if (num_cpus > RSS_MAX_CPUS)
0555         error(1, 0, "increase RSS_MAX_CPUS");
0556 
0557     if (cfg_num_queues && cfg_num_rps_cpus)
0558         error(1, 0,
0559               "Can't supply both RSS cpus ('-C') and RPS map ('-r')");
0560     if (cfg_verbose) {
0561         show_cpulist();
0562         show_silos();
0563     }
0564 }
0565 
0566 int main(int argc, char **argv)
0567 {
0568     const int min_tests = 10;
0569     int fd_sink = -1;
0570 
0571     parse_opts(argc, argv);
0572 
0573     if (cfg_sink)
0574         fd_sink = setup_sink();
0575 
0576     setup_rings();
0577     process_rings();
0578     cleanup_rings();
0579 
0580     if (cfg_sink && close(fd_sink))
0581         error(1, errno, "close sink");
0582 
0583     if (frames_received - frames_nohash < min_tests)
0584         error(1, 0, "too few frames for verification");
0585 
0586     return frames_error;
0587 }
[ Source navigation ] [ Diff markup ] [ Identifier search ] [ general search ]

This page was automatically generated by the 2.1.0 LXR engine.

The LXR team
Valid CSS 2.1! Valid HTML 4.01!