bpf/progs/test_l4lb_noinline.c

0001 // SPDX-License-Identifier: GPL-2.0
0002 // Copyright (c) 2017 Facebook
0003 #include <stddef.h>
0004 #include <stdbool.h>
0005 #include <string.h>
0006 #include <linux/pkt_cls.h>
0007 #include <linux/bpf.h>
0008 #include <linux/in.h>
0009 #include <linux/if_ether.h>
0010 #include <linux/ip.h>
0011 #include <linux/ipv6.h>
0012 #include <linux/icmp.h>
0013 #include <linux/icmpv6.h>
0014 #include <linux/tcp.h>
0015 #include <linux/udp.h>
0016 #include <bpf/bpf_helpers.h>
0017 #include "test_iptunnel_common.h"
0018 #include <bpf/bpf_endian.h>
0019
0020 static __always_inline __u32 rol32(__u32 word, unsigned int shift)
0021 {
0022     return (word << shift) | (word >> ((-shift) & 31));
0023 }
0024
0025 /* copy paste of jhash from kernel sources to make sure llvm
0026  * can compile it into valid sequence of bpf instructions
0027  */
0028 #define __jhash_mix(a, b, c)            \
0029 {                       \
0030     a -= c;  a ^= rol32(c, 4);  c += b; \
0031     b -= a;  b ^= rol32(a, 6);  a += c; \
0032     c -= b;  c ^= rol32(b, 8);  b += a; \
0033     a -= c;  a ^= rol32(c, 16); c += b; \
0034     b -= a;  b ^= rol32(a, 19); a += c; \
0035     c -= b;  c ^= rol32(b, 4);  b += a; \
0036 }
0037
0038 #define __jhash_final(a, b, c)          \
0039 {                       \
0040     c ^= b; c -= rol32(b, 14);      \
0041     a ^= c; a -= rol32(c, 11);      \
0042     b ^= a; b -= rol32(a, 25);      \
0043     c ^= b; c -= rol32(b, 16);      \
0044     a ^= c; a -= rol32(c, 4);       \
0045     b ^= a; b -= rol32(a, 14);      \
0046     c ^= b; c -= rol32(b, 24);      \
0047 }
0048
0049 #define JHASH_INITVAL       0xdeadbeef
0050
0051 typedef unsigned int u32;
0052
0053 static __noinline u32 jhash(const void *key, u32 length, u32 initval)
0054 {
0055     u32 a, b, c;
0056     const unsigned char *k = key;
0057
0058     a = b = c = JHASH_INITVAL + length + initval;
0059
0060     while (length > 12) {
0061         a += *(u32 *)(k);
0062         b += *(u32 *)(k + 4);
0063         c += *(u32 *)(k + 8);
0064         __jhash_mix(a, b, c);
0065         length -= 12;
0066         k += 12;
0067     }
0068     switch (length) {
0069     case 12: c += (u32)k[11]<<24;
0070     case 11: c += (u32)k[10]<<16;
0071     case 10: c += (u32)k[9]<<8;
0072     case 9:  c += k[8];
0073     case 8:  b += (u32)k[7]<<24;
0074     case 7:  b += (u32)k[6]<<16;
0075     case 6:  b += (u32)k[5]<<8;
0076     case 5:  b += k[4];
0077     case 4:  a += (u32)k[3]<<24;
0078     case 3:  a += (u32)k[2]<<16;
0079     case 2:  a += (u32)k[1]<<8;
0080     case 1:  a += k[0];
0081          __jhash_final(a, b, c);
0082     case 0: /* Nothing left to add */
0083         break;
0084     }
0085
0086     return c;
0087 }
0088
0089 static __noinline u32 __jhash_nwords(u32 a, u32 b, u32 c, u32 initval)
0090 {
0091     a += initval;
0092     b += initval;
0093     c += initval;
0094     __jhash_final(a, b, c);
0095     return c;
0096 }
0097
0098 static __noinline u32 jhash_2words(u32 a, u32 b, u32 initval)
0099 {
0100     return __jhash_nwords(a, b, 0, initval + JHASH_INITVAL + (2 << 2));
0101 }
0102
0103 #define PCKT_FRAGMENTED 65343
0104 #define IPV4_HDR_LEN_NO_OPT 20
0105 #define IPV4_PLUS_ICMP_HDR 28
0106 #define IPV6_PLUS_ICMP_HDR 48
0107 #define RING_SIZE 2
0108 #define MAX_VIPS 12
0109 #define MAX_REALS 5
0110 #define CTL_MAP_SIZE 16
0111 #define CH_RINGS_SIZE (MAX_VIPS * RING_SIZE)
0112 #define F_IPV6 (1 << 0)
0113 #define F_HASH_NO_SRC_PORT (1 << 0)
0114 #define F_ICMP (1 << 0)
0115 #define F_SYN_SET (1 << 1)
0116
0117 struct packet_description {
0118     union {
0119         __be32 src;
0120         __be32 srcv6[4];
0121     };
0122     union {
0123         __be32 dst;
0124         __be32 dstv6[4];
0125     };
0126     union {
0127         __u32 ports;
0128         __u16 port16[2];
0129     };
0130     __u8 proto;
0131     __u8 flags;
0132 };
0133
0134 struct ctl_value {
0135     union {
0136         __u64 value;
0137         __u32 ifindex;
0138         __u8 mac[6];
0139     };
0140 };
0141
0142 struct vip_meta {
0143     __u32 flags;
0144     __u32 vip_num;
0145 };
0146
0147 struct real_definition {
0148     union {
0149         __be32 dst;
0150         __be32 dstv6[4];
0151     };
0152     __u8 flags;
0153 };
0154
0155 struct vip_stats {
0156     __u64 bytes;
0157     __u64 pkts;
0158 };
0159
0160 struct eth_hdr {
0161     unsigned char eth_dest[ETH_ALEN];
0162     unsigned char eth_source[ETH_ALEN];
0163     unsigned short eth_proto;
0164 };
0165
0166 struct {
0167     __uint(type, BPF_MAP_TYPE_HASH);
0168     __uint(max_entries, MAX_VIPS);
0169     __type(key, struct vip);
0170     __type(value, struct vip_meta);
0171 } vip_map SEC(".maps");
0172
0173 struct {
0174     __uint(type, BPF_MAP_TYPE_ARRAY);
0175     __uint(max_entries, CH_RINGS_SIZE);
0176     __type(key, __u32);
0177     __type(value, __u32);
0178 } ch_rings SEC(".maps");
0179
0180 struct {
0181     __uint(type, BPF_MAP_TYPE_ARRAY);
0182     __uint(max_entries, MAX_REALS);
0183     __type(key, __u32);
0184     __type(value, struct real_definition);
0185 } reals SEC(".maps");
0186
0187 struct {
0188     __uint(type, BPF_MAP_TYPE_PERCPU_ARRAY);
0189     __uint(max_entries, MAX_VIPS);
0190     __type(key, __u32);
0191     __type(value, struct vip_stats);
0192 } stats SEC(".maps");
0193
0194 struct {
0195     __uint(type, BPF_MAP_TYPE_ARRAY);
0196     __uint(max_entries, CTL_MAP_SIZE);
0197     __type(key, __u32);
0198     __type(value, struct ctl_value);
0199 } ctl_array SEC(".maps");
0200
0201 static __noinline __u32 get_packet_hash(struct packet_description *pckt, bool ipv6)
0202 {
0203     if (ipv6)
0204         return jhash_2words(jhash(pckt->srcv6, 16, MAX_VIPS),
0205                     pckt->ports, CH_RINGS_SIZE);
0206     else
0207         return jhash_2words(pckt->src, pckt->ports, CH_RINGS_SIZE);
0208 }
0209
0210 static __noinline bool get_packet_dst(struct real_definition **real,
0211                       struct packet_description *pckt,
0212                       struct vip_meta *vip_info,
0213                       bool is_ipv6)
0214 {
0215     __u32 hash = get_packet_hash(pckt, is_ipv6);
0216     __u32 key = RING_SIZE * vip_info->vip_num + hash % RING_SIZE;
0217     __u32 *real_pos;
0218
0219     if (hash != 0x358459b7 /* jhash of ipv4 packet */  &&
0220         hash != 0x2f4bc6bb /* jhash of ipv6 packet */)
0221         return false;
0222
0223     real_pos = bpf_map_lookup_elem(&ch_rings, &key);
0224     if (!real_pos)
0225         return false;
0226     key = *real_pos;
0227     *real = bpf_map_lookup_elem(&reals, &key);
0228     if (!(*real))
0229         return false;
0230     return true;
0231 }
0232
0233 static __noinline int parse_icmpv6(void *data, void *data_end, __u64 off,
0234                    struct packet_description *pckt)
0235 {
0236     struct icmp6hdr *icmp_hdr;
0237     struct ipv6hdr *ip6h;
0238
0239     icmp_hdr = data + off;
0240     if (icmp_hdr + 1 > data_end)
0241         return TC_ACT_SHOT;
0242     if (icmp_hdr->icmp6_type != ICMPV6_PKT_TOOBIG)
0243         return TC_ACT_OK;
0244     off += sizeof(struct icmp6hdr);
0245     ip6h = data + off;
0246     if (ip6h + 1 > data_end)
0247         return TC_ACT_SHOT;
0248     pckt->proto = ip6h->nexthdr;
0249     pckt->flags |= F_ICMP;
0250     memcpy(pckt->srcv6, ip6h->daddr.s6_addr32, 16);
0251     memcpy(pckt->dstv6, ip6h->saddr.s6_addr32, 16);
0252     return TC_ACT_UNSPEC;
0253 }
0254
0255 static __noinline int parse_icmp(void *data, void *data_end, __u64 off,
0256                  struct packet_description *pckt)
0257 {
0258     struct icmphdr *icmp_hdr;
0259     struct iphdr *iph;
0260
0261     icmp_hdr = data + off;
0262     if (icmp_hdr + 1 > data_end)
0263         return TC_ACT_SHOT;
0264     if (icmp_hdr->type != ICMP_DEST_UNREACH ||
0265         icmp_hdr->code != ICMP_FRAG_NEEDED)
0266         return TC_ACT_OK;
0267     off += sizeof(struct icmphdr);
0268     iph = data + off;
0269     if (iph + 1 > data_end)
0270         return TC_ACT_SHOT;
0271     if (iph->ihl != 5)
0272         return TC_ACT_SHOT;
0273     pckt->proto = iph->protocol;
0274     pckt->flags |= F_ICMP;
0275     pckt->src = iph->daddr;
0276     pckt->dst = iph->saddr;
0277     return TC_ACT_UNSPEC;
0278 }
0279
0280 static __noinline bool parse_udp(void *data, __u64 off, void *data_end,
0281                  struct packet_description *pckt)
0282 {
0283     struct udphdr *udp;
0284     udp = data + off;
0285
0286     if (udp + 1 > data_end)
0287         return false;
0288
0289     if (!(pckt->flags & F_ICMP)) {
0290         pckt->port16[0] = udp->source;
0291         pckt->port16[1] = udp->dest;
0292     } else {
0293         pckt->port16[0] = udp->dest;
0294         pckt->port16[1] = udp->source;
0295     }
0296     return true;
0297 }
0298
0299 static __noinline bool parse_tcp(void *data, __u64 off, void *data_end,
0300                  struct packet_description *pckt)
0301 {
0302     struct tcphdr *tcp;
0303
0304     tcp = data + off;
0305     if (tcp + 1 > data_end)
0306         return false;
0307
0308     if (tcp->syn)
0309         pckt->flags |= F_SYN_SET;
0310
0311     if (!(pckt->flags & F_ICMP)) {
0312         pckt->port16[0] = tcp->source;
0313         pckt->port16[1] = tcp->dest;
0314     } else {
0315         pckt->port16[0] = tcp->dest;
0316         pckt->port16[1] = tcp->source;
0317     }
0318     return true;
0319 }
0320
0321 static __noinline int process_packet(void *data, __u64 off, void *data_end,
0322                      bool is_ipv6, struct __sk_buff *skb)
0323 {
0324     void *pkt_start = (void *)(long)skb->data;
0325     struct packet_description pckt = {};
0326     struct eth_hdr *eth = pkt_start;
0327     struct bpf_tunnel_key tkey = {};
0328     struct vip_stats *data_stats;
0329     struct real_definition *dst;
0330     struct vip_meta *vip_info;
0331     struct ctl_value *cval;
0332     __u32 v4_intf_pos = 1;
0333     __u32 v6_intf_pos = 2;
0334     struct ipv6hdr *ip6h;
0335     struct vip vip = {};
0336     struct iphdr *iph;
0337     int tun_flag = 0;
0338     __u16 pkt_bytes;
0339     __u64 iph_len;
0340     __u32 ifindex;
0341     __u8 protocol;
0342     __u32 vip_num;
0343     int action;
0344
0345     tkey.tunnel_ttl = 64;
0346     if (is_ipv6) {
0347         ip6h = data + off;
0348         if (ip6h + 1 > data_end)
0349             return TC_ACT_SHOT;
0350
0351         iph_len = sizeof(struct ipv6hdr);
0352         protocol = ip6h->nexthdr;
0353         pckt.proto = protocol;
0354         pkt_bytes = bpf_ntohs(ip6h->payload_len);
0355         off += iph_len;
0356         if (protocol == IPPROTO_FRAGMENT) {
0357             return TC_ACT_SHOT;
0358         } else if (protocol == IPPROTO_ICMPV6) {
0359             action = parse_icmpv6(data, data_end, off, &pckt);
0360             if (action >= 0)
0361                 return action;
0362             off += IPV6_PLUS_ICMP_HDR;
0363         } else {
0364             memcpy(pckt.srcv6, ip6h->saddr.s6_addr32, 16);
0365             memcpy(pckt.dstv6, ip6h->daddr.s6_addr32, 16);
0366         }
0367     } else {
0368         iph = data + off;
0369         if (iph + 1 > data_end)
0370             return TC_ACT_SHOT;
0371         if (iph->ihl != 5)
0372             return TC_ACT_SHOT;
0373
0374         protocol = iph->protocol;
0375         pckt.proto = protocol;
0376         pkt_bytes = bpf_ntohs(iph->tot_len);
0377         off += IPV4_HDR_LEN_NO_OPT;
0378
0379         if (iph->frag_off & PCKT_FRAGMENTED)
0380             return TC_ACT_SHOT;
0381         if (protocol == IPPROTO_ICMP) {
0382             action = parse_icmp(data, data_end, off, &pckt);
0383             if (action >= 0)
0384                 return action;
0385             off += IPV4_PLUS_ICMP_HDR;
0386         } else {
0387             pckt.src = iph->saddr;
0388             pckt.dst = iph->daddr;
0389         }
0390     }
0391     protocol = pckt.proto;
0392
0393     if (protocol == IPPROTO_TCP) {
0394         if (!parse_tcp(data, off, data_end, &pckt))
0395             return TC_ACT_SHOT;
0396     } else if (protocol == IPPROTO_UDP) {
0397         if (!parse_udp(data, off, data_end, &pckt))
0398             return TC_ACT_SHOT;
0399     } else {
0400         return TC_ACT_SHOT;
0401     }
0402
0403     if (is_ipv6)
0404         memcpy(vip.daddr.v6, pckt.dstv6, 16);
0405     else
0406         vip.daddr.v4 = pckt.dst;
0407
0408     vip.dport = pckt.port16[1];
0409     vip.protocol = pckt.proto;
0410     vip_info = bpf_map_lookup_elem(&vip_map, &vip);
0411     if (!vip_info) {
0412         vip.dport = 0;
0413         vip_info = bpf_map_lookup_elem(&vip_map, &vip);
0414         if (!vip_info)
0415             return TC_ACT_SHOT;
0416         pckt.port16[1] = 0;
0417     }
0418
0419     if (vip_info->flags & F_HASH_NO_SRC_PORT)
0420         pckt.port16[0] = 0;
0421
0422     if (!get_packet_dst(&dst, &pckt, vip_info, is_ipv6))
0423         return TC_ACT_SHOT;
0424
0425     if (dst->flags & F_IPV6) {
0426         cval = bpf_map_lookup_elem(&ctl_array, &v6_intf_pos);
0427         if (!cval)
0428             return TC_ACT_SHOT;
0429         ifindex = cval->ifindex;
0430         memcpy(tkey.remote_ipv6, dst->dstv6, 16);
0431         tun_flag = BPF_F_TUNINFO_IPV6;
0432     } else {
0433         cval = bpf_map_lookup_elem(&ctl_array, &v4_intf_pos);
0434         if (!cval)
0435             return TC_ACT_SHOT;
0436         ifindex = cval->ifindex;
0437         tkey.remote_ipv4 = dst->dst;
0438     }
0439     vip_num = vip_info->vip_num;
0440     data_stats = bpf_map_lookup_elem(&stats, &vip_num);
0441     if (!data_stats)
0442         return TC_ACT_SHOT;
0443     data_stats->pkts++;
0444     data_stats->bytes += pkt_bytes;
0445     bpf_skb_set_tunnel_key(skb, &tkey, sizeof(tkey), tun_flag);
0446     *(u32 *)eth->eth_dest = tkey.remote_ipv4;
0447     return bpf_redirect(ifindex, 0);
0448 }
0449
0450 SEC("tc")
0451 int balancer_ingress(struct __sk_buff *ctx)
0452 {
0453     void *data_end = (void *)(long)ctx->data_end;
0454     void *data = (void *)(long)ctx->data;
0455     struct eth_hdr *eth = data;
0456     __u32 eth_proto;
0457     __u32 nh_off;
0458
0459     nh_off = sizeof(struct eth_hdr);
0460     if (data + nh_off > data_end)
0461         return TC_ACT_SHOT;
0462     eth_proto = eth->eth_proto;
0463     if (eth_proto == bpf_htons(ETH_P_IP))
0464         return process_packet(data, nh_off, data_end, false, ctx);
0465     else if (eth_proto == bpf_htons(ETH_P_IPV6))
0466         return process_packet(data, nh_off, data_end, true, ctx);
0467     else
0468         return TC_ACT_SHOT;
0469 }
0470 char _license[] SEC("license") = "GPL";