x86/net/bpf_jit_comp32.c

0001 // SPDX-License-Identifier: GPL-2.0
0002 /*
0003  * Just-In-Time compiler for eBPF filters on IA32 (32bit x86)
0004  *
0005  * Author: Wang YanQing (udknight@gmail.com)
0006  * The code based on code and ideas from:
0007  * Eric Dumazet (eric.dumazet@gmail.com)
0008  * and from:
0009  * Shubham Bansal <illusionist.neo@gmail.com>
0010  */
0011
0012 #include <linux/netdevice.h>
0013 #include <linux/filter.h>
0014 #include <linux/if_vlan.h>
0015 #include <asm/cacheflush.h>
0016 #include <asm/set_memory.h>
0017 #include <asm/nospec-branch.h>
0018 #include <asm/asm-prototypes.h>
0019 #include <linux/bpf.h>
0020
0021 /*
0022  * eBPF prog stack layout:
0023  *
0024  *                         high
0025  * original ESP =>        +-----+
0026  *                        |     | callee saved registers
0027  *                        +-----+
0028  *                        | ... | eBPF JIT scratch space
0029  * BPF_FP,IA32_EBP  =>    +-----+
0030  *                        | ... | eBPF prog stack
0031  *                        +-----+
0032  *                        |RSVD | JIT scratchpad
0033  * current ESP =>         +-----+
0034  *                        |     |
0035  *                        | ... | Function call stack
0036  *                        |     |
0037  *                        +-----+
0038  *                          low
0039  *
0040  * The callee saved registers:
0041  *
0042  *                                high
0043  * original ESP =>        +------------------+ \
0044  *                        |        ebp       | |
0045  * current EBP =>         +------------------+ } callee saved registers
0046  *                        |    ebx,esi,edi   | |
0047  *                        +------------------+ /
0048  *                                low
0049  */
0050
0051 static u8 *emit_code(u8 *ptr, u32 bytes, unsigned int len)
0052 {
0053     if (len == 1)
0054         *ptr = bytes;
0055     else if (len == 2)
0056         *(u16 *)ptr = bytes;
0057     else {
0058         *(u32 *)ptr = bytes;
0059         barrier();
0060     }
0061     return ptr + len;
0062 }
0063
0064 #define EMIT(bytes, len) \
0065     do { prog = emit_code(prog, bytes, len); cnt += len; } while (0)
0066
0067 #define EMIT1(b1)       EMIT(b1, 1)
0068 #define EMIT2(b1, b2)       EMIT((b1) + ((b2) << 8), 2)
0069 #define EMIT3(b1, b2, b3)   EMIT((b1) + ((b2) << 8) + ((b3) << 16), 3)
0070 #define EMIT4(b1, b2, b3, b4)   \
0071     EMIT((b1) + ((b2) << 8) + ((b3) << 16) + ((b4) << 24), 4)
0072
0073 #define EMIT1_off32(b1, off) \
0074     do { EMIT1(b1); EMIT(off, 4); } while (0)
0075 #define EMIT2_off32(b1, b2, off) \
0076     do { EMIT2(b1, b2); EMIT(off, 4); } while (0)
0077 #define EMIT3_off32(b1, b2, b3, off) \
0078     do { EMIT3(b1, b2, b3); EMIT(off, 4); } while (0)
0079 #define EMIT4_off32(b1, b2, b3, b4, off) \
0080     do { EMIT4(b1, b2, b3, b4); EMIT(off, 4); } while (0)
0081
0082 #define jmp_label(label, jmp_insn_len) (label - cnt - jmp_insn_len)
0083
0084 static bool is_imm8(int value)
0085 {
0086     return value <= 127 && value >= -128;
0087 }
0088
0089 static bool is_simm32(s64 value)
0090 {
0091     return value == (s64) (s32) value;
0092 }
0093
0094 #define STACK_OFFSET(k) (k)
0095 #define TCALL_CNT   (MAX_BPF_JIT_REG + 0)   /* Tail Call Count */
0096
0097 #define IA32_EAX    (0x0)
0098 #define IA32_EBX    (0x3)
0099 #define IA32_ECX    (0x1)
0100 #define IA32_EDX    (0x2)
0101 #define IA32_ESI    (0x6)
0102 #define IA32_EDI    (0x7)
0103 #define IA32_EBP    (0x5)
0104 #define IA32_ESP    (0x4)
0105
0106 /*
0107  * List of x86 cond jumps opcodes (. + s8)
0108  * Add 0x10 (and an extra 0x0f) to generate far jumps (. + s32)
0109  */
0110 #define IA32_JB  0x72
0111 #define IA32_JAE 0x73
0112 #define IA32_JE  0x74
0113 #define IA32_JNE 0x75
0114 #define IA32_JBE 0x76
0115 #define IA32_JA  0x77
0116 #define IA32_JL  0x7C
0117 #define IA32_JGE 0x7D
0118 #define IA32_JLE 0x7E
0119 #define IA32_JG  0x7F
0120
0121 #define COND_JMP_OPCODE_INVALID (0xFF)
0122
0123 /*
0124  * Map eBPF registers to IA32 32bit registers or stack scratch space.
0125  *
0126  * 1. All the registers, R0-R10, are mapped to scratch space on stack.
0127  * 2. We need two 64 bit temp registers to do complex operations on eBPF
0128  *    registers.
0129  * 3. For performance reason, the BPF_REG_AX for blinding constant, is
0130  *    mapped to real hardware register pair, IA32_ESI and IA32_EDI.
0131  *
0132  * As the eBPF registers are all 64 bit registers and IA32 has only 32 bit
0133  * registers, we have to map each eBPF registers with two IA32 32 bit regs
0134  * or scratch memory space and we have to build eBPF 64 bit register from those.
0135  *
0136  * We use IA32_EAX, IA32_EDX, IA32_ECX, IA32_EBX as temporary registers.
0137  */
0138 static const u8 bpf2ia32[][2] = {
0139     /* Return value from in-kernel function, and exit value from eBPF */
0140     [BPF_REG_0] = {STACK_OFFSET(0), STACK_OFFSET(4)},
0141
0142     /* The arguments from eBPF program to in-kernel function */
0143     /* Stored on stack scratch space */
0144     [BPF_REG_1] = {STACK_OFFSET(8), STACK_OFFSET(12)},
0145     [BPF_REG_2] = {STACK_OFFSET(16), STACK_OFFSET(20)},
0146     [BPF_REG_3] = {STACK_OFFSET(24), STACK_OFFSET(28)},
0147     [BPF_REG_4] = {STACK_OFFSET(32), STACK_OFFSET(36)},
0148     [BPF_REG_5] = {STACK_OFFSET(40), STACK_OFFSET(44)},
0149
0150     /* Callee saved registers that in-kernel function will preserve */
0151     /* Stored on stack scratch space */
0152     [BPF_REG_6] = {STACK_OFFSET(48), STACK_OFFSET(52)},
0153     [BPF_REG_7] = {STACK_OFFSET(56), STACK_OFFSET(60)},
0154     [BPF_REG_8] = {STACK_OFFSET(64), STACK_OFFSET(68)},
0155     [BPF_REG_9] = {STACK_OFFSET(72), STACK_OFFSET(76)},
0156
0157     /* Read only Frame Pointer to access Stack */
0158     [BPF_REG_FP] = {STACK_OFFSET(80), STACK_OFFSET(84)},
0159
0160     /* Temporary register for blinding constants. */
0161     [BPF_REG_AX] = {IA32_ESI, IA32_EDI},
0162
0163     /* Tail call count. Stored on stack scratch space. */
0164     [TCALL_CNT] = {STACK_OFFSET(88), STACK_OFFSET(92)},
0165 };
0166
0167 #define dst_lo  dst[0]
0168 #define dst_hi  dst[1]
0169 #define src_lo  src[0]
0170 #define src_hi  src[1]
0171
0172 #define STACK_ALIGNMENT 8
0173 /*
0174  * Stack space for BPF_REG_1, BPF_REG_2, BPF_REG_3, BPF_REG_4,
0175  * BPF_REG_5, BPF_REG_6, BPF_REG_7, BPF_REG_8, BPF_REG_9,
0176  * BPF_REG_FP, BPF_REG_AX and Tail call counts.
0177  */
0178 #define SCRATCH_SIZE 96
0179
0180 /* Total stack size used in JITed code */
0181 #define _STACK_SIZE (stack_depth + SCRATCH_SIZE)
0182
0183 #define STACK_SIZE ALIGN(_STACK_SIZE, STACK_ALIGNMENT)
0184
0185 /* Get the offset of eBPF REGISTERs stored on scratch space. */
0186 #define STACK_VAR(off) (off)
0187
0188 /* Encode 'dst_reg' register into IA32 opcode 'byte' */
0189 static u8 add_1reg(u8 byte, u32 dst_reg)
0190 {
0191     return byte + dst_reg;
0192 }
0193
0194 /* Encode 'dst_reg' and 'src_reg' registers into IA32 opcode 'byte' */
0195 static u8 add_2reg(u8 byte, u32 dst_reg, u32 src_reg)
0196 {
0197     return byte + dst_reg + (src_reg << 3);
0198 }
0199
0200 static void jit_fill_hole(void *area, unsigned int size)
0201 {
0202     /* Fill whole space with int3 instructions */
0203     memset(area, 0xcc, size);
0204 }
0205
0206 static inline void emit_ia32_mov_i(const u8 dst, const u32 val, bool dstk,
0207                    u8 **pprog)
0208 {
0209     u8 *prog = *pprog;
0210     int cnt = 0;
0211
0212     if (dstk) {
0213         if (val == 0) {
0214             /* xor eax,eax */
0215             EMIT2(0x33, add_2reg(0xC0, IA32_EAX, IA32_EAX));
0216             /* mov dword ptr [ebp+off],eax */
0217             EMIT3(0x89, add_2reg(0x40, IA32_EBP, IA32_EAX),
0218                   STACK_VAR(dst));
0219         } else {
0220             EMIT3_off32(0xC7, add_1reg(0x40, IA32_EBP),
0221                     STACK_VAR(dst), val);
0222         }
0223     } else {
0224         if (val == 0)
0225             EMIT2(0x33, add_2reg(0xC0, dst, dst));
0226         else
0227             EMIT2_off32(0xC7, add_1reg(0xC0, dst),
0228                     val);
0229     }
0230     *pprog = prog;
0231 }
0232
0233 /* dst = imm (4 bytes)*/
0234 static inline void emit_ia32_mov_r(const u8 dst, const u8 src, bool dstk,
0235                    bool sstk, u8 **pprog)
0236 {
0237     u8 *prog = *pprog;
0238     int cnt = 0;
0239     u8 sreg = sstk ? IA32_EAX : src;
0240
0241     if (sstk)
0242         /* mov eax,dword ptr [ebp+off] */
0243         EMIT3(0x8B, add_2reg(0x40, IA32_EBP, IA32_EAX), STACK_VAR(src));
0244     if (dstk)
0245         /* mov dword ptr [ebp+off],eax */
0246         EMIT3(0x89, add_2reg(0x40, IA32_EBP, sreg), STACK_VAR(dst));
0247     else
0248         /* mov dst,sreg */
0249         EMIT2(0x89, add_2reg(0xC0, dst, sreg));
0250
0251     *pprog = prog;
0252 }
0253
0254 /* dst = src */
0255 static inline void emit_ia32_mov_r64(const bool is64, const u8 dst[],
0256                      const u8 src[], bool dstk,
0257                      bool sstk, u8 **pprog,
0258                      const struct bpf_prog_aux *aux)
0259 {
0260     emit_ia32_mov_r(dst_lo, src_lo, dstk, sstk, pprog);
0261     if (is64)
0262         /* complete 8 byte move */
0263         emit_ia32_mov_r(dst_hi, src_hi, dstk, sstk, pprog);
0264     else if (!aux->verifier_zext)
0265         /* zero out high 4 bytes */
0266         emit_ia32_mov_i(dst_hi, 0, dstk, pprog);
0267 }
0268
0269 /* Sign extended move */
0270 static inline void emit_ia32_mov_i64(const bool is64, const u8 dst[],
0271                      const u32 val, bool dstk, u8 **pprog)
0272 {
0273     u32 hi = 0;
0274
0275     if (is64 && (val & (1<<31)))
0276         hi = (u32)~0;
0277     emit_ia32_mov_i(dst_lo, val, dstk, pprog);
0278     emit_ia32_mov_i(dst_hi, hi, dstk, pprog);
0279 }
0280
0281 /*
0282  * ALU operation (32 bit)
0283  * dst = dst * src
0284  */
0285 static inline void emit_ia32_mul_r(const u8 dst, const u8 src, bool dstk,
0286                    bool sstk, u8 **pprog)
0287 {
0288     u8 *prog = *pprog;
0289     int cnt = 0;
0290     u8 sreg = sstk ? IA32_ECX : src;
0291
0292     if (sstk)
0293         /* mov ecx,dword ptr [ebp+off] */
0294         EMIT3(0x8B, add_2reg(0x40, IA32_EBP, IA32_ECX), STACK_VAR(src));
0295
0296     if (dstk)
0297         /* mov eax,dword ptr [ebp+off] */
0298         EMIT3(0x8B, add_2reg(0x40, IA32_EBP, IA32_EAX), STACK_VAR(dst));
0299     else
0300         /* mov eax,dst */
0301         EMIT2(0x8B, add_2reg(0xC0, dst, IA32_EAX));
0302
0303
0304     EMIT2(0xF7, add_1reg(0xE0, sreg));
0305
0306     if (dstk)
0307         /* mov dword ptr [ebp+off],eax */
0308         EMIT3(0x89, add_2reg(0x40, IA32_EBP, IA32_EAX),
0309               STACK_VAR(dst));
0310     else
0311         /* mov dst,eax */
0312         EMIT2(0x89, add_2reg(0xC0, dst, IA32_EAX));
0313
0314     *pprog = prog;
0315 }
0316
0317 static inline void emit_ia32_to_le_r64(const u8 dst[], s32 val,
0318                      bool dstk, u8 **pprog,
0319                      const struct bpf_prog_aux *aux)
0320 {
0321     u8 *prog = *pprog;
0322     int cnt = 0;
0323     u8 dreg_lo = dstk ? IA32_EAX : dst_lo;
0324     u8 dreg_hi = dstk ? IA32_EDX : dst_hi;
0325
0326     if (dstk && val != 64) {
0327         EMIT3(0x8B, add_2reg(0x40, IA32_EBP, IA32_EAX),
0328               STACK_VAR(dst_lo));
0329         EMIT3(0x8B, add_2reg(0x40, IA32_EBP, IA32_EDX),
0330               STACK_VAR(dst_hi));
0331     }
0332     switch (val) {
0333     case 16:
0334         /*
0335          * Emit 'movzwl eax,ax' to zero extend 16-bit
0336          * into 64 bit
0337          */
0338         EMIT2(0x0F, 0xB7);
0339         EMIT1(add_2reg(0xC0, dreg_lo, dreg_lo));
0340         if (!aux->verifier_zext)
0341             /* xor dreg_hi,dreg_hi */
0342             EMIT2(0x33, add_2reg(0xC0, dreg_hi, dreg_hi));
0343         break;
0344     case 32:
0345         if (!aux->verifier_zext)
0346             /* xor dreg_hi,dreg_hi */
0347             EMIT2(0x33, add_2reg(0xC0, dreg_hi, dreg_hi));
0348         break;
0349     case 64:
0350         /* nop */
0351         break;
0352     }
0353
0354     if (dstk && val != 64) {
0355         /* mov dword ptr [ebp+off],dreg_lo */
0356         EMIT3(0x89, add_2reg(0x40, IA32_EBP, dreg_lo),
0357               STACK_VAR(dst_lo));
0358         /* mov dword ptr [ebp+off],dreg_hi */
0359         EMIT3(0x89, add_2reg(0x40, IA32_EBP, dreg_hi),
0360               STACK_VAR(dst_hi));
0361     }
0362     *pprog = prog;
0363 }
0364
0365 static inline void emit_ia32_to_be_r64(const u8 dst[], s32 val,
0366                        bool dstk, u8 **pprog,
0367                        const struct bpf_prog_aux *aux)
0368 {
0369     u8 *prog = *pprog;
0370     int cnt = 0;
0371     u8 dreg_lo = dstk ? IA32_EAX : dst_lo;
0372     u8 dreg_hi = dstk ? IA32_EDX : dst_hi;
0373
0374     if (dstk) {
0375         EMIT3(0x8B, add_2reg(0x40, IA32_EBP, IA32_EAX),
0376               STACK_VAR(dst_lo));
0377         EMIT3(0x8B, add_2reg(0x40, IA32_EBP, IA32_EDX),
0378               STACK_VAR(dst_hi));
0379     }
0380     switch (val) {
0381     case 16:
0382         /* Emit 'ror %ax, 8' to swap lower 2 bytes */
0383         EMIT1(0x66);
0384         EMIT3(0xC1, add_1reg(0xC8, dreg_lo), 8);
0385
0386         EMIT2(0x0F, 0xB7);
0387         EMIT1(add_2reg(0xC0, dreg_lo, dreg_lo));
0388
0389         if (!aux->verifier_zext)
0390             /* xor dreg_hi,dreg_hi */
0391             EMIT2(0x33, add_2reg(0xC0, dreg_hi, dreg_hi));
0392         break;
0393     case 32:
0394         /* Emit 'bswap eax' to swap lower 4 bytes */
0395         EMIT1(0x0F);
0396         EMIT1(add_1reg(0xC8, dreg_lo));
0397
0398         if (!aux->verifier_zext)
0399             /* xor dreg_hi,dreg_hi */
0400             EMIT2(0x33, add_2reg(0xC0, dreg_hi, dreg_hi));
0401         break;
0402     case 64:
0403         /* Emit 'bswap eax' to swap lower 4 bytes */
0404         EMIT1(0x0F);
0405         EMIT1(add_1reg(0xC8, dreg_lo));
0406
0407         /* Emit 'bswap edx' to swap lower 4 bytes */
0408         EMIT1(0x0F);
0409         EMIT1(add_1reg(0xC8, dreg_hi));
0410
0411         /* mov ecx,dreg_hi */
0412         EMIT2(0x89, add_2reg(0xC0, IA32_ECX, dreg_hi));
0413         /* mov dreg_hi,dreg_lo */
0414         EMIT2(0x89, add_2reg(0xC0, dreg_hi, dreg_lo));
0415         /* mov dreg_lo,ecx */
0416         EMIT2(0x89, add_2reg(0xC0, dreg_lo, IA32_ECX));
0417
0418         break;
0419     }
0420     if (dstk) {
0421         /* mov dword ptr [ebp+off],dreg_lo */
0422         EMIT3(0x89, add_2reg(0x40, IA32_EBP, dreg_lo),
0423               STACK_VAR(dst_lo));
0424         /* mov dword ptr [ebp+off],dreg_hi */
0425         EMIT3(0x89, add_2reg(0x40, IA32_EBP, dreg_hi),
0426               STACK_VAR(dst_hi));
0427     }
0428     *pprog = prog;
0429 }
0430
0431 /*
0432  * ALU operation (32 bit)
0433  * dst = dst (div|mod) src
0434  */
0435 static inline void emit_ia32_div_mod_r(const u8 op, const u8 dst, const u8 src,
0436                        bool dstk, bool sstk, u8 **pprog)
0437 {
0438     u8 *prog = *pprog;
0439     int cnt = 0;
0440
0441     if (sstk)
0442         /* mov ecx,dword ptr [ebp+off] */
0443         EMIT3(0x8B, add_2reg(0x40, IA32_EBP, IA32_ECX),
0444               STACK_VAR(src));
0445     else if (src != IA32_ECX)
0446         /* mov ecx,src */
0447         EMIT2(0x8B, add_2reg(0xC0, src, IA32_ECX));
0448
0449     if (dstk)
0450         /* mov eax,dword ptr [ebp+off] */
0451         EMIT3(0x8B, add_2reg(0x40, IA32_EBP, IA32_EAX),
0452               STACK_VAR(dst));
0453     else
0454         /* mov eax,dst */
0455         EMIT2(0x8B, add_2reg(0xC0, dst, IA32_EAX));
0456
0457     /* xor edx,edx */
0458     EMIT2(0x31, add_2reg(0xC0, IA32_EDX, IA32_EDX));
0459     /* div ecx */
0460     EMIT2(0xF7, add_1reg(0xF0, IA32_ECX));
0461
0462     if (op == BPF_MOD) {
0463         if (dstk)
0464             EMIT3(0x89, add_2reg(0x40, IA32_EBP, IA32_EDX),
0465                   STACK_VAR(dst));
0466         else
0467             EMIT2(0x89, add_2reg(0xC0, dst, IA32_EDX));
0468     } else {
0469         if (dstk)
0470             EMIT3(0x89, add_2reg(0x40, IA32_EBP, IA32_EAX),
0471                   STACK_VAR(dst));
0472         else
0473             EMIT2(0x89, add_2reg(0xC0, dst, IA32_EAX));
0474     }
0475     *pprog = prog;
0476 }
0477
0478 /*
0479  * ALU operation (32 bit)
0480  * dst = dst (shift) src
0481  */
0482 static inline void emit_ia32_shift_r(const u8 op, const u8 dst, const u8 src,
0483                      bool dstk, bool sstk, u8 **pprog)
0484 {
0485     u8 *prog = *pprog;
0486     int cnt = 0;
0487     u8 dreg = dstk ? IA32_EAX : dst;
0488     u8 b2;
0489
0490     if (dstk)
0491         /* mov eax,dword ptr [ebp+off] */
0492         EMIT3(0x8B, add_2reg(0x40, IA32_EBP, IA32_EAX), STACK_VAR(dst));
0493
0494     if (sstk)
0495         /* mov ecx,dword ptr [ebp+off] */
0496         EMIT3(0x8B, add_2reg(0x40, IA32_EBP, IA32_ECX), STACK_VAR(src));
0497     else if (src != IA32_ECX)
0498         /* mov ecx,src */
0499         EMIT2(0x8B, add_2reg(0xC0, src, IA32_ECX));
0500
0501     switch (op) {
0502     case BPF_LSH:
0503         b2 = 0xE0; break;
0504     case BPF_RSH:
0505         b2 = 0xE8; break;
0506     case BPF_ARSH:
0507         b2 = 0xF8; break;
0508     default:
0509         return;
0510     }
0511     EMIT2(0xD3, add_1reg(b2, dreg));
0512
0513     if (dstk)
0514         /* mov dword ptr [ebp+off],dreg */
0515         EMIT3(0x89, add_2reg(0x40, IA32_EBP, dreg), STACK_VAR(dst));
0516     *pprog = prog;
0517 }
0518
0519 /*
0520  * ALU operation (32 bit)
0521  * dst = dst (op) src
0522  */
0523 static inline void emit_ia32_alu_r(const bool is64, const bool hi, const u8 op,
0524                    const u8 dst, const u8 src, bool dstk,
0525                    bool sstk, u8 **pprog)
0526 {
0527     u8 *prog = *pprog;
0528     int cnt = 0;
0529     u8 sreg = sstk ? IA32_EAX : src;
0530     u8 dreg = dstk ? IA32_EDX : dst;
0531
0532     if (sstk)
0533         /* mov eax,dword ptr [ebp+off] */
0534         EMIT3(0x8B, add_2reg(0x40, IA32_EBP, IA32_EAX), STACK_VAR(src));
0535
0536     if (dstk)
0537         /* mov eax,dword ptr [ebp+off] */
0538         EMIT3(0x8B, add_2reg(0x40, IA32_EBP, IA32_EDX), STACK_VAR(dst));
0539
0540     switch (BPF_OP(op)) {
0541     /* dst = dst + src */
0542     case BPF_ADD:
0543         if (hi && is64)
0544             EMIT2(0x11, add_2reg(0xC0, dreg, sreg));
0545         else
0546             EMIT2(0x01, add_2reg(0xC0, dreg, sreg));
0547         break;
0548     /* dst = dst - src */
0549     case BPF_SUB:
0550         if (hi && is64)
0551             EMIT2(0x19, add_2reg(0xC0, dreg, sreg));
0552         else
0553             EMIT2(0x29, add_2reg(0xC0, dreg, sreg));
0554         break;
0555     /* dst = dst | src */
0556     case BPF_OR:
0557         EMIT2(0x09, add_2reg(0xC0, dreg, sreg));
0558         break;
0559     /* dst = dst & src */
0560     case BPF_AND:
0561         EMIT2(0x21, add_2reg(0xC0, dreg, sreg));
0562         break;
0563     /* dst = dst ^ src */
0564     case BPF_XOR:
0565         EMIT2(0x31, add_2reg(0xC0, dreg, sreg));
0566         break;
0567     }
0568
0569     if (dstk)
0570         /* mov dword ptr [ebp+off],dreg */
0571         EMIT3(0x89, add_2reg(0x40, IA32_EBP, dreg),
0572               STACK_VAR(dst));
0573     *pprog = prog;
0574 }
0575
0576 /* ALU operation (64 bit) */
0577 static inline void emit_ia32_alu_r64(const bool is64, const u8 op,
0578                      const u8 dst[], const u8 src[],
0579                      bool dstk,  bool sstk,
0580                      u8 **pprog, const struct bpf_prog_aux *aux)
0581 {
0582     u8 *prog = *pprog;
0583
0584     emit_ia32_alu_r(is64, false, op, dst_lo, src_lo, dstk, sstk, &prog);
0585     if (is64)
0586         emit_ia32_alu_r(is64, true, op, dst_hi, src_hi, dstk, sstk,
0587                 &prog);
0588     else if (!aux->verifier_zext)
0589         emit_ia32_mov_i(dst_hi, 0, dstk, &prog);
0590     *pprog = prog;
0591 }
0592
0593 /*
0594  * ALU operation (32 bit)
0595  * dst = dst (op) val
0596  */
0597 static inline void emit_ia32_alu_i(const bool is64, const bool hi, const u8 op,
0598                    const u8 dst, const s32 val, bool dstk,
0599                    u8 **pprog)
0600 {
0601     u8 *prog = *pprog;
0602     int cnt = 0;
0603     u8 dreg = dstk ? IA32_EAX : dst;
0604     u8 sreg = IA32_EDX;
0605
0606     if (dstk)
0607         /* mov eax,dword ptr [ebp+off] */
0608         EMIT3(0x8B, add_2reg(0x40, IA32_EBP, IA32_EAX), STACK_VAR(dst));
0609
0610     if (!is_imm8(val))
0611         /* mov edx,imm32*/
0612         EMIT2_off32(0xC7, add_1reg(0xC0, IA32_EDX), val);
0613
0614     switch (op) {
0615     /* dst = dst + val */
0616     case BPF_ADD:
0617         if (hi && is64) {
0618             if (is_imm8(val))
0619                 EMIT3(0x83, add_1reg(0xD0, dreg), val);
0620             else
0621                 EMIT2(0x11, add_2reg(0xC0, dreg, sreg));
0622         } else {
0623             if (is_imm8(val))
0624                 EMIT3(0x83, add_1reg(0xC0, dreg), val);
0625             else
0626                 EMIT2(0x01, add_2reg(0xC0, dreg, sreg));
0627         }
0628         break;
0629     /* dst = dst - val */
0630     case BPF_SUB:
0631         if (hi && is64) {
0632             if (is_imm8(val))
0633                 EMIT3(0x83, add_1reg(0xD8, dreg), val);
0634             else
0635                 EMIT2(0x19, add_2reg(0xC0, dreg, sreg));
0636         } else {
0637             if (is_imm8(val))
0638                 EMIT3(0x83, add_1reg(0xE8, dreg), val);
0639             else
0640                 EMIT2(0x29, add_2reg(0xC0, dreg, sreg));
0641         }
0642         break;
0643     /* dst = dst | val */
0644     case BPF_OR:
0645         if (is_imm8(val))
0646             EMIT3(0x83, add_1reg(0xC8, dreg), val);
0647         else
0648             EMIT2(0x09, add_2reg(0xC0, dreg, sreg));
0649         break;
0650     /* dst = dst & val */
0651     case BPF_AND:
0652         if (is_imm8(val))
0653             EMIT3(0x83, add_1reg(0xE0, dreg), val);
0654         else
0655             EMIT2(0x21, add_2reg(0xC0, dreg, sreg));
0656         break;
0657     /* dst = dst ^ val */
0658     case BPF_XOR:
0659         if (is_imm8(val))
0660             EMIT3(0x83, add_1reg(0xF0, dreg), val);
0661         else
0662             EMIT2(0x31, add_2reg(0xC0, dreg, sreg));
0663         break;
0664     case BPF_NEG:
0665         EMIT2(0xF7, add_1reg(0xD8, dreg));
0666         break;
0667     }
0668
0669     if (dstk)
0670         /* mov dword ptr [ebp+off],dreg */
0671         EMIT3(0x89, add_2reg(0x40, IA32_EBP, dreg),
0672               STACK_VAR(dst));
0673     *pprog = prog;
0674 }
0675
0676 /* ALU operation (64 bit) */
0677 static inline void emit_ia32_alu_i64(const bool is64, const u8 op,
0678                      const u8 dst[], const u32 val,
0679                      bool dstk, u8 **pprog,
0680                      const struct bpf_prog_aux *aux)
0681 {
0682     u8 *prog = *pprog;
0683     u32 hi = 0;
0684
0685     if (is64 && (val & (1<<31)))
0686         hi = (u32)~0;
0687
0688     emit_ia32_alu_i(is64, false, op, dst_lo, val, dstk, &prog);
0689     if (is64)
0690         emit_ia32_alu_i(is64, true, op, dst_hi, hi, dstk, &prog);
0691     else if (!aux->verifier_zext)
0692         emit_ia32_mov_i(dst_hi, 0, dstk, &prog);
0693
0694     *pprog = prog;
0695 }
0696
0697 /* dst = ~dst (64 bit) */
0698 static inline void emit_ia32_neg64(const u8 dst[], bool dstk, u8 **pprog)
0699 {
0700     u8 *prog = *pprog;
0701     int cnt = 0;
0702     u8 dreg_lo = dstk ? IA32_EAX : dst_lo;
0703     u8 dreg_hi = dstk ? IA32_EDX : dst_hi;
0704
0705     if (dstk) {
0706         EMIT3(0x8B, add_2reg(0x40, IA32_EBP, IA32_EAX),
0707               STACK_VAR(dst_lo));
0708         EMIT3(0x8B, add_2reg(0x40, IA32_EBP, IA32_EDX),
0709               STACK_VAR(dst_hi));
0710     }
0711
0712     /* neg dreg_lo */
0713     EMIT2(0xF7, add_1reg(0xD8, dreg_lo));
0714     /* adc dreg_hi,0x0 */
0715     EMIT3(0x83, add_1reg(0xD0, dreg_hi), 0x00);
0716     /* neg dreg_hi */
0717     EMIT2(0xF7, add_1reg(0xD8, dreg_hi));
0718
0719     if (dstk) {
0720         /* mov dword ptr [ebp+off],dreg_lo */
0721         EMIT3(0x89, add_2reg(0x40, IA32_EBP, dreg_lo),
0722               STACK_VAR(dst_lo));
0723         /* mov dword ptr [ebp+off],dreg_hi */
0724         EMIT3(0x89, add_2reg(0x40, IA32_EBP, dreg_hi),
0725               STACK_VAR(dst_hi));
0726     }
0727     *pprog = prog;
0728 }
0729
0730 /* dst = dst << src */
0731 static inline void emit_ia32_lsh_r64(const u8 dst[], const u8 src[],
0732                      bool dstk, bool sstk, u8 **pprog)
0733 {
0734     u8 *prog = *pprog;
0735     int cnt = 0;
0736     u8 dreg_lo = dstk ? IA32_EAX : dst_lo;
0737     u8 dreg_hi = dstk ? IA32_EDX : dst_hi;
0738
0739     if (dstk) {
0740         EMIT3(0x8B, add_2reg(0x40, IA32_EBP, IA32_EAX),
0741               STACK_VAR(dst_lo));
0742         EMIT3(0x8B, add_2reg(0x40, IA32_EBP, IA32_EDX),
0743               STACK_VAR(dst_hi));
0744     }
0745
0746     if (sstk)
0747         /* mov ecx,dword ptr [ebp+off] */
0748         EMIT3(0x8B, add_2reg(0x40, IA32_EBP, IA32_ECX),
0749               STACK_VAR(src_lo));
0750     else
0751         /* mov ecx,src_lo */
0752         EMIT2(0x8B, add_2reg(0xC0, src_lo, IA32_ECX));
0753
0754     /* shld dreg_hi,dreg_lo,cl */
0755     EMIT3(0x0F, 0xA5, add_2reg(0xC0, dreg_hi, dreg_lo));
0756     /* shl dreg_lo,cl */
0757     EMIT2(0xD3, add_1reg(0xE0, dreg_lo));
0758
0759     /* if ecx >= 32, mov dreg_lo into dreg_hi and clear dreg_lo */
0760
0761     /* cmp ecx,32 */
0762     EMIT3(0x83, add_1reg(0xF8, IA32_ECX), 32);
0763     /* skip the next two instructions (4 bytes) when < 32 */
0764     EMIT2(IA32_JB, 4);
0765
0766     /* mov dreg_hi,dreg_lo */
0767     EMIT2(0x89, add_2reg(0xC0, dreg_hi, dreg_lo));
0768     /* xor dreg_lo,dreg_lo */
0769     EMIT2(0x33, add_2reg(0xC0, dreg_lo, dreg_lo));
0770
0771     if (dstk) {
0772         /* mov dword ptr [ebp+off],dreg_lo */
0773         EMIT3(0x89, add_2reg(0x40, IA32_EBP, dreg_lo),
0774               STACK_VAR(dst_lo));
0775         /* mov dword ptr [ebp+off],dreg_hi */
0776         EMIT3(0x89, add_2reg(0x40, IA32_EBP, dreg_hi),
0777               STACK_VAR(dst_hi));
0778     }
0779     /* out: */
0780     *pprog = prog;
0781 }
0782
0783 /* dst = dst >> src (signed)*/
0784 static inline void emit_ia32_arsh_r64(const u8 dst[], const u8 src[],
0785                       bool dstk, bool sstk, u8 **pprog)
0786 {
0787     u8 *prog = *pprog;
0788     int cnt = 0;
0789     u8 dreg_lo = dstk ? IA32_EAX : dst_lo;
0790     u8 dreg_hi = dstk ? IA32_EDX : dst_hi;
0791
0792     if (dstk) {
0793         EMIT3(0x8B, add_2reg(0x40, IA32_EBP, IA32_EAX),
0794               STACK_VAR(dst_lo));
0795         EMIT3(0x8B, add_2reg(0x40, IA32_EBP, IA32_EDX),
0796               STACK_VAR(dst_hi));
0797     }
0798
0799     if (sstk)
0800         /* mov ecx,dword ptr [ebp+off] */
0801         EMIT3(0x8B, add_2reg(0x40, IA32_EBP, IA32_ECX),
0802               STACK_VAR(src_lo));
0803     else
0804         /* mov ecx,src_lo */
0805         EMIT2(0x8B, add_2reg(0xC0, src_lo, IA32_ECX));
0806
0807     /* shrd dreg_lo,dreg_hi,cl */
0808     EMIT3(0x0F, 0xAD, add_2reg(0xC0, dreg_lo, dreg_hi));
0809     /* sar dreg_hi,cl */
0810     EMIT2(0xD3, add_1reg(0xF8, dreg_hi));
0811
0812     /* if ecx >= 32, mov dreg_hi to dreg_lo and set/clear dreg_hi depending on sign */
0813
0814     /* cmp ecx,32 */
0815     EMIT3(0x83, add_1reg(0xF8, IA32_ECX), 32);
0816     /* skip the next two instructions (5 bytes) when < 32 */
0817     EMIT2(IA32_JB, 5);
0818
0819     /* mov dreg_lo,dreg_hi */
0820     EMIT2(0x89, add_2reg(0xC0, dreg_lo, dreg_hi));
0821     /* sar dreg_hi,31 */
0822     EMIT3(0xC1, add_1reg(0xF8, dreg_hi), 31);
0823
0824     if (dstk) {
0825         /* mov dword ptr [ebp+off],dreg_lo */
0826         EMIT3(0x89, add_2reg(0x40, IA32_EBP, dreg_lo),
0827               STACK_VAR(dst_lo));
0828         /* mov dword ptr [ebp+off],dreg_hi */
0829         EMIT3(0x89, add_2reg(0x40, IA32_EBP, dreg_hi),
0830               STACK_VAR(dst_hi));
0831     }
0832     /* out: */
0833     *pprog = prog;
0834 }
0835
0836 /* dst = dst >> src */
0837 static inline void emit_ia32_rsh_r64(const u8 dst[], const u8 src[], bool dstk,
0838                      bool sstk, u8 **pprog)
0839 {
0840     u8 *prog = *pprog;
0841     int cnt = 0;
0842     u8 dreg_lo = dstk ? IA32_EAX : dst_lo;
0843     u8 dreg_hi = dstk ? IA32_EDX : dst_hi;
0844
0845     if (dstk) {
0846         EMIT3(0x8B, add_2reg(0x40, IA32_EBP, IA32_EAX),
0847               STACK_VAR(dst_lo));
0848         EMIT3(0x8B, add_2reg(0x40, IA32_EBP, IA32_EDX),
0849               STACK_VAR(dst_hi));
0850     }
0851
0852     if (sstk)
0853         /* mov ecx,dword ptr [ebp+off] */
0854         EMIT3(0x8B, add_2reg(0x40, IA32_EBP, IA32_ECX),
0855               STACK_VAR(src_lo));
0856     else
0857         /* mov ecx,src_lo */
0858         EMIT2(0x8B, add_2reg(0xC0, src_lo, IA32_ECX));
0859
0860     /* shrd dreg_lo,dreg_hi,cl */
0861     EMIT3(0x0F, 0xAD, add_2reg(0xC0, dreg_lo, dreg_hi));
0862     /* shr dreg_hi,cl */
0863     EMIT2(0xD3, add_1reg(0xE8, dreg_hi));
0864
0865     /* if ecx >= 32, mov dreg_hi to dreg_lo and clear dreg_hi */
0866
0867     /* cmp ecx,32 */
0868     EMIT3(0x83, add_1reg(0xF8, IA32_ECX), 32);
0869     /* skip the next two instructions (4 bytes) when < 32 */
0870     EMIT2(IA32_JB, 4);
0871
0872     /* mov dreg_lo,dreg_hi */
0873     EMIT2(0x89, add_2reg(0xC0, dreg_lo, dreg_hi));
0874     /* xor dreg_hi,dreg_hi */
0875     EMIT2(0x33, add_2reg(0xC0, dreg_hi, dreg_hi));
0876
0877     if (dstk) {
0878         /* mov dword ptr [ebp+off],dreg_lo */
0879         EMIT3(0x89, add_2reg(0x40, IA32_EBP, dreg_lo),
0880               STACK_VAR(dst_lo));
0881         /* mov dword ptr [ebp+off],dreg_hi */
0882         EMIT3(0x89, add_2reg(0x40, IA32_EBP, dreg_hi),
0883               STACK_VAR(dst_hi));
0884     }
0885     /* out: */
0886     *pprog = prog;
0887 }
0888
0889 /* dst = dst << val */
0890 static inline void emit_ia32_lsh_i64(const u8 dst[], const u32 val,
0891                      bool dstk, u8 **pprog)
0892 {
0893     u8 *prog = *pprog;
0894     int cnt = 0;
0895     u8 dreg_lo = dstk ? IA32_EAX : dst_lo;
0896     u8 dreg_hi = dstk ? IA32_EDX : dst_hi;
0897
0898     if (dstk) {
0899         EMIT3(0x8B, add_2reg(0x40, IA32_EBP, IA32_EAX),
0900               STACK_VAR(dst_lo));
0901         EMIT3(0x8B, add_2reg(0x40, IA32_EBP, IA32_EDX),
0902               STACK_VAR(dst_hi));
0903     }
0904     /* Do LSH operation */
0905     if (val < 32) {
0906         /* shld dreg_hi,dreg_lo,imm8 */
0907         EMIT4(0x0F, 0xA4, add_2reg(0xC0, dreg_hi, dreg_lo), val);
0908         /* shl dreg_lo,imm8 */
0909         EMIT3(0xC1, add_1reg(0xE0, dreg_lo), val);
0910     } else if (val >= 32 && val < 64) {
0911         u32 value = val - 32;
0912
0913         /* shl dreg_lo,imm8 */
0914         EMIT3(0xC1, add_1reg(0xE0, dreg_lo), value);
0915         /* mov dreg_hi,dreg_lo */
0916         EMIT2(0x89, add_2reg(0xC0, dreg_hi, dreg_lo));
0917         /* xor dreg_lo,dreg_lo */
0918         EMIT2(0x33, add_2reg(0xC0, dreg_lo, dreg_lo));
0919     } else {
0920         /* xor dreg_lo,dreg_lo */
0921         EMIT2(0x33, add_2reg(0xC0, dreg_lo, dreg_lo));
0922         /* xor dreg_hi,dreg_hi */
0923         EMIT2(0x33, add_2reg(0xC0, dreg_hi, dreg_hi));
0924     }
0925
0926     if (dstk) {
0927         /* mov dword ptr [ebp+off],dreg_lo */
0928         EMIT3(0x89, add_2reg(0x40, IA32_EBP, dreg_lo),
0929               STACK_VAR(dst_lo));
0930         /* mov dword ptr [ebp+off],dreg_hi */
0931         EMIT3(0x89, add_2reg(0x40, IA32_EBP, dreg_hi),
0932               STACK_VAR(dst_hi));
0933     }
0934     *pprog = prog;
0935 }
0936
0937 /* dst = dst >> val */
0938 static inline void emit_ia32_rsh_i64(const u8 dst[], const u32 val,
0939                      bool dstk, u8 **pprog)
0940 {
0941     u8 *prog = *pprog;
0942     int cnt = 0;
0943     u8 dreg_lo = dstk ? IA32_EAX : dst_lo;
0944     u8 dreg_hi = dstk ? IA32_EDX : dst_hi;
0945
0946     if (dstk) {
0947         EMIT3(0x8B, add_2reg(0x40, IA32_EBP, IA32_EAX),
0948               STACK_VAR(dst_lo));
0949         EMIT3(0x8B, add_2reg(0x40, IA32_EBP, IA32_EDX),
0950               STACK_VAR(dst_hi));
0951     }
0952
0953     /* Do RSH operation */
0954     if (val < 32) {
0955         /* shrd dreg_lo,dreg_hi,imm8 */
0956         EMIT4(0x0F, 0xAC, add_2reg(0xC0, dreg_lo, dreg_hi), val);
0957         /* shr dreg_hi,imm8 */
0958         EMIT3(0xC1, add_1reg(0xE8, dreg_hi), val);
0959     } else if (val >= 32 && val < 64) {
0960         u32 value = val - 32;
0961
0962         /* shr dreg_hi,imm8 */
0963         EMIT3(0xC1, add_1reg(0xE8, dreg_hi), value);
0964         /* mov dreg_lo,dreg_hi */
0965         EMIT2(0x89, add_2reg(0xC0, dreg_lo, dreg_hi));
0966         /* xor dreg_hi,dreg_hi */
0967         EMIT2(0x33, add_2reg(0xC0, dreg_hi, dreg_hi));
0968     } else {
0969         /* xor dreg_lo,dreg_lo */
0970         EMIT2(0x33, add_2reg(0xC0, dreg_lo, dreg_lo));
0971         /* xor dreg_hi,dreg_hi */
0972         EMIT2(0x33, add_2reg(0xC0, dreg_hi, dreg_hi));
0973     }
0974
0975     if (dstk) {
0976         /* mov dword ptr [ebp+off],dreg_lo */
0977         EMIT3(0x89, add_2reg(0x40, IA32_EBP, dreg_lo),
0978               STACK_VAR(dst_lo));
0979         /* mov dword ptr [ebp+off],dreg_hi */
0980         EMIT3(0x89, add_2reg(0x40, IA32_EBP, dreg_hi),
0981               STACK_VAR(dst_hi));
0982     }
0983     *pprog = prog;
0984 }
0985
0986 /* dst = dst >> val (signed) */
0987 static inline void emit_ia32_arsh_i64(const u8 dst[], const u32 val,
0988                       bool dstk, u8 **pprog)
0989 {
0990     u8 *prog = *pprog;
0991     int cnt = 0;
0992     u8 dreg_lo = dstk ? IA32_EAX : dst_lo;
0993     u8 dreg_hi = dstk ? IA32_EDX : dst_hi;
0994
0995     if (dstk) {
0996         EMIT3(0x8B, add_2reg(0x40, IA32_EBP, IA32_EAX),
0997               STACK_VAR(dst_lo));
0998         EMIT3(0x8B, add_2reg(0x40, IA32_EBP, IA32_EDX),
0999               STACK_VAR(dst_hi));
1000     }
1001     /* Do RSH operation */
1002     if (val < 32) {
1003         /* shrd dreg_lo,dreg_hi,imm8 */
1004         EMIT4(0x0F, 0xAC, add_2reg(0xC0, dreg_lo, dreg_hi), val);
1005         /* ashr dreg_hi,imm8 */
1006         EMIT3(0xC1, add_1reg(0xF8, dreg_hi), val);
1007     } else if (val >= 32 && val < 64) {
1008         u32 value = val - 32;
1009
1010         /* ashr dreg_hi,imm8 */
1011         EMIT3(0xC1, add_1reg(0xF8, dreg_hi), value);
1012         /* mov dreg_lo,dreg_hi */
1013         EMIT2(0x89, add_2reg(0xC0, dreg_lo, dreg_hi));
1014
1015         /* ashr dreg_hi,imm8 */
1016         EMIT3(0xC1, add_1reg(0xF8, dreg_hi), 31);
1017     } else {
1018         /* ashr dreg_hi,imm8 */
1019         EMIT3(0xC1, add_1reg(0xF8, dreg_hi), 31);
1020         /* mov dreg_lo,dreg_hi */
1021         EMIT2(0x89, add_2reg(0xC0, dreg_lo, dreg_hi));
1022     }
1023
1024     if (dstk) {
1025         /* mov dword ptr [ebp+off],dreg_lo */
1026         EMIT3(0x89, add_2reg(0x40, IA32_EBP, dreg_lo),
1027               STACK_VAR(dst_lo));
1028         /* mov dword ptr [ebp+off],dreg_hi */
1029         EMIT3(0x89, add_2reg(0x40, IA32_EBP, dreg_hi),
1030               STACK_VAR(dst_hi));
1031     }
1032     *pprog = prog;
1033 }
1034
1035 static inline void emit_ia32_mul_r64(const u8 dst[], const u8 src[], bool dstk,
1036                      bool sstk, u8 **pprog)
1037 {
1038     u8 *prog = *pprog;
1039     int cnt = 0;
1040
1041     if (dstk)
1042         /* mov eax,dword ptr [ebp+off] */
1043         EMIT3(0x8B, add_2reg(0x40, IA32_EBP, IA32_EAX),
1044               STACK_VAR(dst_hi));
1045     else
1046         /* mov eax,dst_hi */
1047         EMIT2(0x8B, add_2reg(0xC0, dst_hi, IA32_EAX));
1048
1049     if (sstk)
1050         /* mul dword ptr [ebp+off] */
1051         EMIT3(0xF7, add_1reg(0x60, IA32_EBP), STACK_VAR(src_lo));
1052     else
1053         /* mul src_lo */
1054         EMIT2(0xF7, add_1reg(0xE0, src_lo));
1055
1056     /* mov ecx,eax */
1057     EMIT2(0x89, add_2reg(0xC0, IA32_ECX, IA32_EAX));
1058
1059     if (dstk)
1060         /* mov eax,dword ptr [ebp+off] */
1061         EMIT3(0x8B, add_2reg(0x40, IA32_EBP, IA32_EAX),
1062               STACK_VAR(dst_lo));
1063     else
1064         /* mov eax,dst_lo */
1065         EMIT2(0x8B, add_2reg(0xC0, dst_lo, IA32_EAX));
1066
1067     if (sstk)
1068         /* mul dword ptr [ebp+off] */
1069         EMIT3(0xF7, add_1reg(0x60, IA32_EBP), STACK_VAR(src_hi));
1070     else
1071         /* mul src_hi */
1072         EMIT2(0xF7, add_1reg(0xE0, src_hi));
1073
1074     /* add eax,eax */
1075     EMIT2(0x01, add_2reg(0xC0, IA32_ECX, IA32_EAX));
1076
1077     if (dstk)
1078         /* mov eax,dword ptr [ebp+off] */
1079         EMIT3(0x8B, add_2reg(0x40, IA32_EBP, IA32_EAX),
1080               STACK_VAR(dst_lo));
1081     else
1082         /* mov eax,dst_lo */
1083         EMIT2(0x8B, add_2reg(0xC0, dst_lo, IA32_EAX));
1084
1085     if (sstk)
1086         /* mul dword ptr [ebp+off] */
1087         EMIT3(0xF7, add_1reg(0x60, IA32_EBP), STACK_VAR(src_lo));
1088     else
1089         /* mul src_lo */
1090         EMIT2(0xF7, add_1reg(0xE0, src_lo));
1091
1092     /* add ecx,edx */
1093     EMIT2(0x01, add_2reg(0xC0, IA32_ECX, IA32_EDX));
1094
1095     if (dstk) {
1096         /* mov dword ptr [ebp+off],eax */
1097         EMIT3(0x89, add_2reg(0x40, IA32_EBP, IA32_EAX),
1098               STACK_VAR(dst_lo));
1099         /* mov dword ptr [ebp+off],ecx */
1100         EMIT3(0x89, add_2reg(0x40, IA32_EBP, IA32_ECX),
1101               STACK_VAR(dst_hi));
1102     } else {
1103         /* mov dst_lo,eax */
1104         EMIT2(0x89, add_2reg(0xC0, dst_lo, IA32_EAX));
1105         /* mov dst_hi,ecx */
1106         EMIT2(0x89, add_2reg(0xC0, dst_hi, IA32_ECX));
1107     }
1108
1109     *pprog = prog;
1110 }
1111
1112 static inline void emit_ia32_mul_i64(const u8 dst[], const u32 val,
1113                      bool dstk, u8 **pprog)
1114 {
1115     u8 *prog = *pprog;
1116     int cnt = 0;
1117     u32 hi;
1118
1119     hi = val & (1<<31) ? (u32)~0 : 0;
1120     /* movl eax,imm32 */
1121     EMIT2_off32(0xC7, add_1reg(0xC0, IA32_EAX), val);
1122     if (dstk)
1123         /* mul dword ptr [ebp+off] */
1124         EMIT3(0xF7, add_1reg(0x60, IA32_EBP), STACK_VAR(dst_hi));
1125     else
1126         /* mul dst_hi */
1127         EMIT2(0xF7, add_1reg(0xE0, dst_hi));
1128
1129     /* mov ecx,eax */
1130     EMIT2(0x89, add_2reg(0xC0, IA32_ECX, IA32_EAX));
1131
1132     /* movl eax,imm32 */
1133     EMIT2_off32(0xC7, add_1reg(0xC0, IA32_EAX), hi);
1134     if (dstk)
1135         /* mul dword ptr [ebp+off] */
1136         EMIT3(0xF7, add_1reg(0x60, IA32_EBP), STACK_VAR(dst_lo));
1137     else
1138         /* mul dst_lo */
1139         EMIT2(0xF7, add_1reg(0xE0, dst_lo));
1140     /* add ecx,eax */
1141     EMIT2(0x01, add_2reg(0xC0, IA32_ECX, IA32_EAX));
1142
1143     /* movl eax,imm32 */
1144     EMIT2_off32(0xC7, add_1reg(0xC0, IA32_EAX), val);
1145     if (dstk)
1146         /* mul dword ptr [ebp+off] */
1147         EMIT3(0xF7, add_1reg(0x60, IA32_EBP), STACK_VAR(dst_lo));
1148     else
1149         /* mul dst_lo */
1150         EMIT2(0xF7, add_1reg(0xE0, dst_lo));
1151
1152     /* add ecx,edx */
1153     EMIT2(0x01, add_2reg(0xC0, IA32_ECX, IA32_EDX));
1154
1155     if (dstk) {
1156         /* mov dword ptr [ebp+off],eax */
1157         EMIT3(0x89, add_2reg(0x40, IA32_EBP, IA32_EAX),
1158               STACK_VAR(dst_lo));
1159         /* mov dword ptr [ebp+off],ecx */
1160         EMIT3(0x89, add_2reg(0x40, IA32_EBP, IA32_ECX),
1161               STACK_VAR(dst_hi));
1162     } else {
1163         /* mov dword ptr [ebp+off],eax */
1164         EMIT2(0x89, add_2reg(0xC0, dst_lo, IA32_EAX));
1165         /* mov dword ptr [ebp+off],ecx */
1166         EMIT2(0x89, add_2reg(0xC0, dst_hi, IA32_ECX));
1167     }
1168
1169     *pprog = prog;
1170 }
1171
1172 static int bpf_size_to_x86_bytes(int bpf_size)
1173 {
1174     if (bpf_size == BPF_W)
1175         return 4;
1176     else if (bpf_size == BPF_H)
1177         return 2;
1178     else if (bpf_size == BPF_B)
1179         return 1;
1180     else if (bpf_size == BPF_DW)
1181         return 4; /* imm32 */
1182     else
1183         return 0;
1184 }
1185
1186 struct jit_context {
1187     int cleanup_addr; /* Epilogue code offset */
1188 };
1189
1190 /* Maximum number of bytes emitted while JITing one eBPF insn */
1191 #define BPF_MAX_INSN_SIZE   128
1192 #define BPF_INSN_SAFETY     64
1193
1194 #define PROLOGUE_SIZE 35
1195
1196 /*
1197  * Emit prologue code for BPF program and check it's size.
1198  * bpf_tail_call helper will skip it while jumping into another program.
1199  */
1200 static void emit_prologue(u8 **pprog, u32 stack_depth)
1201 {
1202     u8 *prog = *pprog;
1203     int cnt = 0;
1204     const u8 *r1 = bpf2ia32[BPF_REG_1];
1205     const u8 fplo = bpf2ia32[BPF_REG_FP][0];
1206     const u8 fphi = bpf2ia32[BPF_REG_FP][1];
1207     const u8 *tcc = bpf2ia32[TCALL_CNT];
1208
1209     /* push ebp */
1210     EMIT1(0x55);
1211     /* mov ebp,esp */
1212     EMIT2(0x89, 0xE5);
1213     /* push edi */
1214     EMIT1(0x57);
1215     /* push esi */
1216     EMIT1(0x56);
1217     /* push ebx */
1218     EMIT1(0x53);
1219
1220     /* sub esp,STACK_SIZE */
1221     EMIT2_off32(0x81, 0xEC, STACK_SIZE);
1222     /* sub ebp,SCRATCH_SIZE+12*/
1223     EMIT3(0x83, add_1reg(0xE8, IA32_EBP), SCRATCH_SIZE + 12);
1224     /* xor ebx,ebx */
1225     EMIT2(0x31, add_2reg(0xC0, IA32_EBX, IA32_EBX));
1226
1227     /* Set up BPF prog stack base register */
1228     EMIT3(0x89, add_2reg(0x40, IA32_EBP, IA32_EBP), STACK_VAR(fplo));
1229     EMIT3(0x89, add_2reg(0x40, IA32_EBP, IA32_EBX), STACK_VAR(fphi));
1230
1231     /* Move BPF_CTX (EAX) to BPF_REG_R1 */
1232     /* mov dword ptr [ebp+off],eax */
1233     EMIT3(0x89, add_2reg(0x40, IA32_EBP, IA32_EAX), STACK_VAR(r1[0]));
1234     EMIT3(0x89, add_2reg(0x40, IA32_EBP, IA32_EBX), STACK_VAR(r1[1]));
1235
1236     /* Initialize Tail Count */
1237     EMIT3(0x89, add_2reg(0x40, IA32_EBP, IA32_EBX), STACK_VAR(tcc[0]));
1238     EMIT3(0x89, add_2reg(0x40, IA32_EBP, IA32_EBX), STACK_VAR(tcc[1]));
1239
1240     BUILD_BUG_ON(cnt != PROLOGUE_SIZE);
1241     *pprog = prog;
1242 }
1243
1244 /* Emit epilogue code for BPF program */
1245 static void emit_epilogue(u8 **pprog, u32 stack_depth)
1246 {
1247     u8 *prog = *pprog;
1248     const u8 *r0 = bpf2ia32[BPF_REG_0];
1249     int cnt = 0;
1250
1251     /* mov eax,dword ptr [ebp+off]*/
1252     EMIT3(0x8B, add_2reg(0x40, IA32_EBP, IA32_EAX), STACK_VAR(r0[0]));
1253     /* mov edx,dword ptr [ebp+off]*/
1254     EMIT3(0x8B, add_2reg(0x40, IA32_EBP, IA32_EDX), STACK_VAR(r0[1]));
1255
1256     /* add ebp,SCRATCH_SIZE+12*/
1257     EMIT3(0x83, add_1reg(0xC0, IA32_EBP), SCRATCH_SIZE + 12);
1258
1259     /* mov ebx,dword ptr [ebp-12]*/
1260     EMIT3(0x8B, add_2reg(0x40, IA32_EBP, IA32_EBX), -12);
1261     /* mov esi,dword ptr [ebp-8]*/
1262     EMIT3(0x8B, add_2reg(0x40, IA32_EBP, IA32_ESI), -8);
1263     /* mov edi,dword ptr [ebp-4]*/
1264     EMIT3(0x8B, add_2reg(0x40, IA32_EBP, IA32_EDI), -4);
1265
1266     EMIT1(0xC9); /* leave */
1267     EMIT1(0xC3); /* ret */
1268     *pprog = prog;
1269 }
1270
1271 static int emit_jmp_edx(u8 **pprog, u8 *ip)
1272 {
1273     u8 *prog = *pprog;
1274     int cnt = 0;
1275
1276 #ifdef CONFIG_RETPOLINE
1277     EMIT1_off32(0xE9, (u8 *)__x86_indirect_thunk_edx - (ip + 5));
1278 #else
1279     EMIT2(0xFF, 0xE2);
1280 #endif
1281     *pprog = prog;
1282
1283     return cnt;
1284 }
1285
1286 /*
1287  * Generate the following code:
1288  * ... bpf_tail_call(void *ctx, struct bpf_array *array, u64 index) ...
1289  *   if (index >= array->map.max_entries)
1290  *     goto out;
1291  *   if (++tail_call_cnt > MAX_TAIL_CALL_CNT)
1292  *     goto out;
1293  *   prog = array->ptrs[index];
1294  *   if (prog == NULL)
1295  *     goto out;
1296  *   goto *(prog->bpf_func + prologue_size);
1297  * out:
1298  */
1299 static void emit_bpf_tail_call(u8 **pprog, u8 *ip)
1300 {
1301     u8 *prog = *pprog;
1302     int cnt = 0;
1303     const u8 *r1 = bpf2ia32[BPF_REG_1];
1304     const u8 *r2 = bpf2ia32[BPF_REG_2];
1305     const u8 *r3 = bpf2ia32[BPF_REG_3];
1306     const u8 *tcc = bpf2ia32[TCALL_CNT];
1307     u32 lo, hi;
1308     static int jmp_label1 = -1;
1309
1310     /*
1311      * if (index >= array->map.max_entries)
1312      *     goto out;
1313      */
1314     /* mov eax,dword ptr [ebp+off] */
1315     EMIT3(0x8B, add_2reg(0x40, IA32_EBP, IA32_EAX), STACK_VAR(r2[0]));
1316     /* mov edx,dword ptr [ebp+off] */
1317     EMIT3(0x8B, add_2reg(0x40, IA32_EBP, IA32_EDX), STACK_VAR(r3[0]));
1318
1319     /* cmp dword ptr [eax+off],edx */
1320     EMIT3(0x39, add_2reg(0x40, IA32_EAX, IA32_EDX),
1321           offsetof(struct bpf_array, map.max_entries));
1322     /* jbe out */
1323     EMIT2(IA32_JBE, jmp_label(jmp_label1, 2));
1324
1325     /*
1326      * if (tail_call_cnt++ >= MAX_TAIL_CALL_CNT)
1327      *     goto out;
1328      */
1329     lo = (u32)MAX_TAIL_CALL_CNT;
1330     hi = (u32)((u64)MAX_TAIL_CALL_CNT >> 32);
1331     EMIT3(0x8B, add_2reg(0x40, IA32_EBP, IA32_ECX), STACK_VAR(tcc[0]));
1332     EMIT3(0x8B, add_2reg(0x40, IA32_EBP, IA32_EBX), STACK_VAR(tcc[1]));
1333
1334     /* cmp edx,hi */
1335     EMIT3(0x83, add_1reg(0xF8, IA32_EBX), hi);
1336     EMIT2(IA32_JNE, 3);
1337     /* cmp ecx,lo */
1338     EMIT3(0x83, add_1reg(0xF8, IA32_ECX), lo);
1339
1340     /* jae out */
1341     EMIT2(IA32_JAE, jmp_label(jmp_label1, 2));
1342
1343     /* add eax,0x1 */
1344     EMIT3(0x83, add_1reg(0xC0, IA32_ECX), 0x01);
1345     /* adc ebx,0x0 */
1346     EMIT3(0x83, add_1reg(0xD0, IA32_EBX), 0x00);
1347
1348     /* mov dword ptr [ebp+off],eax */
1349     EMIT3(0x89, add_2reg(0x40, IA32_EBP, IA32_ECX), STACK_VAR(tcc[0]));
1350     /* mov dword ptr [ebp+off],edx */
1351     EMIT3(0x89, add_2reg(0x40, IA32_EBP, IA32_EBX), STACK_VAR(tcc[1]));
1352
1353     /* prog = array->ptrs[index]; */
1354     /* mov edx, [eax + edx * 4 + offsetof(...)] */
1355     EMIT3_off32(0x8B, 0x94, 0x90, offsetof(struct bpf_array, ptrs));
1356
1357     /*
1358      * if (prog == NULL)
1359      *     goto out;
1360      */
1361     /* test edx,edx */
1362     EMIT2(0x85, add_2reg(0xC0, IA32_EDX, IA32_EDX));
1363     /* je out */
1364     EMIT2(IA32_JE, jmp_label(jmp_label1, 2));
1365
1366     /* goto *(prog->bpf_func + prologue_size); */
1367     /* mov edx, dword ptr [edx + 32] */
1368     EMIT3(0x8B, add_2reg(0x40, IA32_EDX, IA32_EDX),
1369           offsetof(struct bpf_prog, bpf_func));
1370     /* add edx,prologue_size */
1371     EMIT3(0x83, add_1reg(0xC0, IA32_EDX), PROLOGUE_SIZE);
1372
1373     /* mov eax,dword ptr [ebp+off] */
1374     EMIT3(0x8B, add_2reg(0x40, IA32_EBP, IA32_EAX), STACK_VAR(r1[0]));
1375
1376     /*
1377      * Now we're ready to jump into next BPF program:
1378      * eax == ctx (1st arg)
1379      * edx == prog->bpf_func + prologue_size
1380      */
1381     cnt += emit_jmp_edx(&prog, ip + cnt);
1382
1383     if (jmp_label1 == -1)
1384         jmp_label1 = cnt;
1385
1386     /* out: */
1387     *pprog = prog;
1388 }
1389
1390 /* Push the scratch stack register on top of the stack. */
1391 static inline void emit_push_r64(const u8 src[], u8 **pprog)
1392 {
1393     u8 *prog = *pprog;
1394     int cnt = 0;
1395
1396     /* mov ecx,dword ptr [ebp+off] */
1397     EMIT3(0x8B, add_2reg(0x40, IA32_EBP, IA32_ECX), STACK_VAR(src_hi));
1398     /* push ecx */
1399     EMIT1(0x51);
1400
1401     /* mov ecx,dword ptr [ebp+off] */
1402     EMIT3(0x8B, add_2reg(0x40, IA32_EBP, IA32_ECX), STACK_VAR(src_lo));
1403     /* push ecx */
1404     EMIT1(0x51);
1405
1406     *pprog = prog;
1407 }
1408
1409 static void emit_push_r32(const u8 src[], u8 **pprog)
1410 {
1411     u8 *prog = *pprog;
1412     int cnt = 0;
1413
1414     /* mov ecx,dword ptr [ebp+off] */
1415     EMIT3(0x8B, add_2reg(0x40, IA32_EBP, IA32_ECX), STACK_VAR(src_lo));
1416     /* push ecx */
1417     EMIT1(0x51);
1418
1419     *pprog = prog;
1420 }
1421
1422 static u8 get_cond_jmp_opcode(const u8 op, bool is_cmp_lo)
1423 {
1424     u8 jmp_cond;
1425
1426     /* Convert BPF opcode to x86 */
1427     switch (op) {
1428     case BPF_JEQ:
1429         jmp_cond = IA32_JE;
1430         break;
1431     case BPF_JSET:
1432     case BPF_JNE:
1433         jmp_cond = IA32_JNE;
1434         break;
1435     case BPF_JGT:
1436         /* GT is unsigned '>', JA in x86 */
1437         jmp_cond = IA32_JA;
1438         break;
1439     case BPF_JLT:
1440         /* LT is unsigned '<', JB in x86 */
1441         jmp_cond = IA32_JB;
1442         break;
1443     case BPF_JGE:
1444         /* GE is unsigned '>=', JAE in x86 */
1445         jmp_cond = IA32_JAE;
1446         break;
1447     case BPF_JLE:
1448         /* LE is unsigned '<=', JBE in x86 */
1449         jmp_cond = IA32_JBE;
1450         break;
1451     case BPF_JSGT:
1452         if (!is_cmp_lo)
1453             /* Signed '>', GT in x86 */
1454             jmp_cond = IA32_JG;
1455         else
1456             /* GT is unsigned '>', JA in x86 */
1457             jmp_cond = IA32_JA;
1458         break;
1459     case BPF_JSLT:
1460         if (!is_cmp_lo)
1461             /* Signed '<', LT in x86 */
1462             jmp_cond = IA32_JL;
1463         else
1464             /* LT is unsigned '<', JB in x86 */
1465             jmp_cond = IA32_JB;
1466         break;
1467     case BPF_JSGE:
1468         if (!is_cmp_lo)
1469             /* Signed '>=', GE in x86 */
1470             jmp_cond = IA32_JGE;
1471         else
1472             /* GE is unsigned '>=', JAE in x86 */
1473             jmp_cond = IA32_JAE;
1474         break;
1475     case BPF_JSLE:
1476         if (!is_cmp_lo)
1477             /* Signed '<=', LE in x86 */
1478             jmp_cond = IA32_JLE;
1479         else
1480             /* LE is unsigned '<=', JBE in x86 */
1481             jmp_cond = IA32_JBE;
1482         break;
1483     default: /* to silence GCC warning */
1484         jmp_cond = COND_JMP_OPCODE_INVALID;
1485         break;
1486     }
1487
1488     return jmp_cond;
1489 }
1490
1491 /* i386 kernel compiles with "-mregparm=3".  From gcc document:
1492  *
1493  * ==== snippet ====
1494  * regparm (number)
1495  *  On x86-32 targets, the regparm attribute causes the compiler
1496  *  to pass arguments number one to (number) if they are of integral
1497  *  type in registers EAX, EDX, and ECX instead of on the stack.
1498  *  Functions that take a variable number of arguments continue
1499  *  to be passed all of their arguments on the stack.
1500  * ==== snippet ====
1501  *
1502  * The first three args of a function will be considered for
1503  * putting into the 32bit register EAX, EDX, and ECX.
1504  *
1505  * Two 32bit registers are used to pass a 64bit arg.
1506  *
1507  * For example,
1508  * void foo(u32 a, u32 b, u32 c, u32 d):
1509  *  u32 a: EAX
1510  *  u32 b: EDX
1511  *  u32 c: ECX
1512  *  u32 d: stack
1513  *
1514  * void foo(u64 a, u32 b, u32 c):
1515  *  u64 a: EAX (lo32) EDX (hi32)
1516  *  u32 b: ECX
1517  *  u32 c: stack
1518  *
1519  * void foo(u32 a, u64 b, u32 c):
1520  *  u32 a: EAX
1521  *  u64 b: EDX (lo32) ECX (hi32)
1522  *  u32 c: stack
1523  *
1524  * void foo(u32 a, u32 b, u64 c):
1525  *  u32 a: EAX
1526  *  u32 b: EDX
1527  *  u64 c: stack
1528  *
1529  * The return value will be stored in the EAX (and EDX for 64bit value).
1530  *
1531  * For example,
1532  * u32 foo(u32 a, u32 b, u32 c):
1533  *  return value: EAX
1534  *
1535  * u64 foo(u32 a, u32 b, u32 c):
1536  *  return value: EAX (lo32) EDX (hi32)
1537  *
1538  * Notes:
1539  *  The verifier only accepts function having integer and pointers
1540  *  as its args and return value, so it does not have
1541  *  struct-by-value.
1542  *
1543  * emit_kfunc_call() finds out the btf_func_model by calling
1544  * bpf_jit_find_kfunc_model().  A btf_func_model
1545  * has the details about the number of args, size of each arg,
1546  * and the size of the return value.
1547  *
1548  * It first decides how many args can be passed by EAX, EDX, and ECX.
1549  * That will decide what args should be pushed to the stack:
1550  * [first_stack_regno, last_stack_regno] are the bpf regnos
1551  * that should be pushed to the stack.
1552  *
1553  * It will first push all args to the stack because the push
1554  * will need to use ECX.  Then, it moves
1555  * [BPF_REG_1, first_stack_regno) to EAX, EDX, and ECX.
1556  *
1557  * When emitting a call (0xE8), it needs to figure out
1558  * the jmp_offset relative to the jit-insn address immediately
1559  * following the call (0xE8) instruction.  At this point, it knows
1560  * the end of the jit-insn address after completely translated the
1561  * current (BPF_JMP | BPF_CALL) bpf-insn.  It is passed as "end_addr"
1562  * to the emit_kfunc_call().  Thus, it can learn the "immediate-follow-call"
1563  * address by figuring out how many jit-insn is generated between
1564  * the call (0xE8) and the end_addr:
1565  *  - 0-1 jit-insn (3 bytes each) to restore the esp pointer if there
1566  *    is arg pushed to the stack.
1567  *  - 0-2 jit-insns (3 bytes each) to handle the return value.
1568  */
1569 static int emit_kfunc_call(const struct bpf_prog *bpf_prog, u8 *end_addr,
1570                const struct bpf_insn *insn, u8 **pprog)
1571 {
1572     const u8 arg_regs[] = { IA32_EAX, IA32_EDX, IA32_ECX };
1573     int i, cnt = 0, first_stack_regno, last_stack_regno;
1574     int free_arg_regs = ARRAY_SIZE(arg_regs);
1575     const struct btf_func_model *fm;
1576     int bytes_in_stack = 0;
1577     const u8 *cur_arg_reg;
1578     u8 *prog = *pprog;
1579     s64 jmp_offset;
1580
1581     fm = bpf_jit_find_kfunc_model(bpf_prog, insn);
1582     if (!fm)
1583         return -EINVAL;
1584
1585     first_stack_regno = BPF_REG_1;
1586     for (i = 0; i < fm->nr_args; i++) {
1587         int regs_needed = fm->arg_size[i] > sizeof(u32) ? 2 : 1;
1588
1589         if (regs_needed > free_arg_regs)
1590             break;
1591
1592         free_arg_regs -= regs_needed;
1593         first_stack_regno++;
1594     }
1595
1596     /* Push the args to the stack */
1597     last_stack_regno = BPF_REG_0 + fm->nr_args;
1598     for (i = last_stack_regno; i >= first_stack_regno; i--) {
1599         if (fm->arg_size[i - 1] > sizeof(u32)) {
1600             emit_push_r64(bpf2ia32[i], &prog);
1601             bytes_in_stack += 8;
1602         } else {
1603             emit_push_r32(bpf2ia32[i], &prog);
1604             bytes_in_stack += 4;
1605         }
1606     }
1607
1608     cur_arg_reg = &arg_regs[0];
1609     for (i = BPF_REG_1; i < first_stack_regno; i++) {
1610         /* mov e[adc]x,dword ptr [ebp+off] */
1611         EMIT3(0x8B, add_2reg(0x40, IA32_EBP, *cur_arg_reg++),
1612               STACK_VAR(bpf2ia32[i][0]));
1613         if (fm->arg_size[i - 1] > sizeof(u32))
1614             /* mov e[adc]x,dword ptr [ebp+off] */
1615             EMIT3(0x8B, add_2reg(0x40, IA32_EBP, *cur_arg_reg++),
1616                   STACK_VAR(bpf2ia32[i][1]));
1617     }
1618
1619     if (bytes_in_stack)
1620         /* add esp,"bytes_in_stack" */
1621         end_addr -= 3;
1622
1623     /* mov dword ptr [ebp+off],edx */
1624     if (fm->ret_size > sizeof(u32))
1625         end_addr -= 3;
1626
1627     /* mov dword ptr [ebp+off],eax */
1628     if (fm->ret_size)
1629         end_addr -= 3;
1630
1631     jmp_offset = (u8 *)__bpf_call_base + insn->imm - end_addr;
1632     if (!is_simm32(jmp_offset)) {
1633         pr_err("unsupported BPF kernel function jmp_offset:%lld\n",
1634                jmp_offset);
1635         return -EINVAL;
1636     }
1637
1638     EMIT1_off32(0xE8, jmp_offset);
1639
1640     if (fm->ret_size)
1641         /* mov dword ptr [ebp+off],eax */
1642         EMIT3(0x89, add_2reg(0x40, IA32_EBP, IA32_EAX),
1643               STACK_VAR(bpf2ia32[BPF_REG_0][0]));
1644
1645     if (fm->ret_size > sizeof(u32))
1646         /* mov dword ptr [ebp+off],edx */
1647         EMIT3(0x89, add_2reg(0x40, IA32_EBP, IA32_EDX),
1648               STACK_VAR(bpf2ia32[BPF_REG_0][1]));
1649
1650     if (bytes_in_stack)
1651         /* add esp,"bytes_in_stack" */
1652         EMIT3(0x83, add_1reg(0xC0, IA32_ESP), bytes_in_stack);
1653
1654     *pprog = prog;
1655
1656     return 0;
1657 }
1658
1659 static int do_jit(struct bpf_prog *bpf_prog, int *addrs, u8 *image,
1660           int oldproglen, struct jit_context *ctx)
1661 {
1662     struct bpf_insn *insn = bpf_prog->insnsi;
1663     int insn_cnt = bpf_prog->len;
1664     bool seen_exit = false;
1665     u8 temp[BPF_MAX_INSN_SIZE + BPF_INSN_SAFETY];
1666     int i, cnt = 0;
1667     int proglen = 0;
1668     u8 *prog = temp;
1669
1670     emit_prologue(&prog, bpf_prog->aux->stack_depth);
1671
1672     for (i = 0; i < insn_cnt; i++, insn++) {
1673         const s32 imm32 = insn->imm;
1674         const bool is64 = BPF_CLASS(insn->code) == BPF_ALU64;
1675         const bool dstk = insn->dst_reg != BPF_REG_AX;
1676         const bool sstk = insn->src_reg != BPF_REG_AX;
1677         const u8 code = insn->code;
1678         const u8 *dst = bpf2ia32[insn->dst_reg];
1679         const u8 *src = bpf2ia32[insn->src_reg];
1680         const u8 *r0 = bpf2ia32[BPF_REG_0];
1681         s64 jmp_offset;
1682         u8 jmp_cond;
1683         int ilen;
1684         u8 *func;
1685
1686         switch (code) {
1687         /* ALU operations */
1688         /* dst = src */
1689         case BPF_ALU | BPF_MOV | BPF_K:
1690         case BPF_ALU | BPF_MOV | BPF_X:
1691         case BPF_ALU64 | BPF_MOV | BPF_K:
1692         case BPF_ALU64 | BPF_MOV | BPF_X:
1693             switch (BPF_SRC(code)) {
1694             case BPF_X:
1695                 if (imm32 == 1) {
1696                     /* Special mov32 for zext. */
1697                     emit_ia32_mov_i(dst_hi, 0, dstk, &prog);
1698                     break;
1699                 }
1700                 emit_ia32_mov_r64(is64, dst, src, dstk, sstk,
1701                           &prog, bpf_prog->aux);
1702                 break;
1703             case BPF_K:
1704                 /* Sign-extend immediate value to dst reg */
1705                 emit_ia32_mov_i64(is64, dst, imm32,
1706                           dstk, &prog);
1707                 break;
1708             }
1709             break;
1710         /* dst = dst + src/imm */
1711         /* dst = dst - src/imm */
1712         /* dst = dst | src/imm */
1713         /* dst = dst & src/imm */
1714         /* dst = dst ^ src/imm */
1715         /* dst = dst * src/imm */
1716         /* dst = dst << src */
1717         /* dst = dst >> src */
1718         case BPF_ALU | BPF_ADD | BPF_K:
1719         case BPF_ALU | BPF_ADD | BPF_X:
1720         case BPF_ALU | BPF_SUB | BPF_K:
1721         case BPF_ALU | BPF_SUB | BPF_X:
1722         case BPF_ALU | BPF_OR | BPF_K:
1723         case BPF_ALU | BPF_OR | BPF_X:
1724         case BPF_ALU | BPF_AND | BPF_K:
1725         case BPF_ALU | BPF_AND | BPF_X:
1726         case BPF_ALU | BPF_XOR | BPF_K:
1727         case BPF_ALU | BPF_XOR | BPF_X:
1728         case BPF_ALU64 | BPF_ADD | BPF_K:
1729         case BPF_ALU64 | BPF_ADD | BPF_X:
1730         case BPF_ALU64 | BPF_SUB | BPF_K:
1731         case BPF_ALU64 | BPF_SUB | BPF_X:
1732         case BPF_ALU64 | BPF_OR | BPF_K:
1733         case BPF_ALU64 | BPF_OR | BPF_X:
1734         case BPF_ALU64 | BPF_AND | BPF_K:
1735         case BPF_ALU64 | BPF_AND | BPF_X:
1736         case BPF_ALU64 | BPF_XOR | BPF_K:
1737         case BPF_ALU64 | BPF_XOR | BPF_X:
1738             switch (BPF_SRC(code)) {
1739             case BPF_X:
1740                 emit_ia32_alu_r64(is64, BPF_OP(code), dst,
1741                           src, dstk, sstk, &prog,
1742                           bpf_prog->aux);
1743                 break;
1744             case BPF_K:
1745                 emit_ia32_alu_i64(is64, BPF_OP(code), dst,
1746                           imm32, dstk, &prog,
1747                           bpf_prog->aux);
1748                 break;
1749             }
1750             break;
1751         case BPF_ALU | BPF_MUL | BPF_K:
1752         case BPF_ALU | BPF_MUL | BPF_X:
1753             switch (BPF_SRC(code)) {
1754             case BPF_X:
1755                 emit_ia32_mul_r(dst_lo, src_lo, dstk,
1756                         sstk, &prog);
1757                 break;
1758             case BPF_K:
1759                 /* mov ecx,imm32*/
1760                 EMIT2_off32(0xC7, add_1reg(0xC0, IA32_ECX),
1761                         imm32);
1762                 emit_ia32_mul_r(dst_lo, IA32_ECX, dstk,
1763                         false, &prog);
1764                 break;
1765             }
1766             if (!bpf_prog->aux->verifier_zext)
1767                 emit_ia32_mov_i(dst_hi, 0, dstk, &prog);
1768             break;
1769         case BPF_ALU | BPF_LSH | BPF_X:
1770         case BPF_ALU | BPF_RSH | BPF_X:
1771         case BPF_ALU | BPF_ARSH | BPF_K:
1772         case BPF_ALU | BPF_ARSH | BPF_X:
1773             switch (BPF_SRC(code)) {
1774             case BPF_X:
1775                 emit_ia32_shift_r(BPF_OP(code), dst_lo, src_lo,
1776                           dstk, sstk, &prog);
1777                 break;
1778             case BPF_K:
1779                 /* mov ecx,imm32*/
1780                 EMIT2_off32(0xC7, add_1reg(0xC0, IA32_ECX),
1781                         imm32);
1782                 emit_ia32_shift_r(BPF_OP(code), dst_lo,
1783                           IA32_ECX, dstk, false,
1784                           &prog);
1785                 break;
1786             }
1787             if (!bpf_prog->aux->verifier_zext)
1788                 emit_ia32_mov_i(dst_hi, 0, dstk, &prog);
1789             break;
1790         /* dst = dst / src(imm) */
1791         /* dst = dst % src(imm) */
1792         case BPF_ALU | BPF_DIV | BPF_K:
1793         case BPF_ALU | BPF_DIV | BPF_X:
1794         case BPF_ALU | BPF_MOD | BPF_K:
1795         case BPF_ALU | BPF_MOD | BPF_X:
1796             switch (BPF_SRC(code)) {
1797             case BPF_X:
1798                 emit_ia32_div_mod_r(BPF_OP(code), dst_lo,
1799                             src_lo, dstk, sstk, &prog);
1800                 break;
1801             case BPF_K:
1802                 /* mov ecx,imm32*/
1803                 EMIT2_off32(0xC7, add_1reg(0xC0, IA32_ECX),
1804                         imm32);
1805                 emit_ia32_div_mod_r(BPF_OP(code), dst_lo,
1806                             IA32_ECX, dstk, false,
1807                             &prog);
1808                 break;
1809             }
1810             if (!bpf_prog->aux->verifier_zext)
1811                 emit_ia32_mov_i(dst_hi, 0, dstk, &prog);
1812             break;
1813         case BPF_ALU64 | BPF_DIV | BPF_K:
1814         case BPF_ALU64 | BPF_DIV | BPF_X:
1815         case BPF_ALU64 | BPF_MOD | BPF_K:
1816         case BPF_ALU64 | BPF_MOD | BPF_X:
1817             goto notyet;
1818         /* dst = dst >> imm */
1819         /* dst = dst << imm */
1820         case BPF_ALU | BPF_RSH | BPF_K:
1821         case BPF_ALU | BPF_LSH | BPF_K:
1822             if (unlikely(imm32 > 31))
1823                 return -EINVAL;
1824             /* mov ecx,imm32*/
1825             EMIT2_off32(0xC7, add_1reg(0xC0, IA32_ECX), imm32);
1826             emit_ia32_shift_r(BPF_OP(code), dst_lo, IA32_ECX, dstk,
1827                       false, &prog);
1828             if (!bpf_prog->aux->verifier_zext)
1829                 emit_ia32_mov_i(dst_hi, 0, dstk, &prog);
1830             break;
1831         /* dst = dst << imm */
1832         case BPF_ALU64 | BPF_LSH | BPF_K:
1833             if (unlikely(imm32 > 63))
1834                 return -EINVAL;
1835             emit_ia32_lsh_i64(dst, imm32, dstk, &prog);
1836             break;
1837         /* dst = dst >> imm */
1838         case BPF_ALU64 | BPF_RSH | BPF_K:
1839             if (unlikely(imm32 > 63))
1840                 return -EINVAL;
1841             emit_ia32_rsh_i64(dst, imm32, dstk, &prog);
1842             break;
1843         /* dst = dst << src */
1844         case BPF_ALU64 | BPF_LSH | BPF_X:
1845             emit_ia32_lsh_r64(dst, src, dstk, sstk, &prog);
1846             break;
1847         /* dst = dst >> src */
1848         case BPF_ALU64 | BPF_RSH | BPF_X:
1849             emit_ia32_rsh_r64(dst, src, dstk, sstk, &prog);
1850             break;
1851         /* dst = dst >> src (signed) */
1852         case BPF_ALU64 | BPF_ARSH | BPF_X:
1853             emit_ia32_arsh_r64(dst, src, dstk, sstk, &prog);
1854             break;
1855         /* dst = dst >> imm (signed) */
1856         case BPF_ALU64 | BPF_ARSH | BPF_K:
1857             if (unlikely(imm32 > 63))
1858                 return -EINVAL;
1859             emit_ia32_arsh_i64(dst, imm32, dstk, &prog);
1860             break;
1861         /* dst = ~dst */
1862         case BPF_ALU | BPF_NEG:
1863             emit_ia32_alu_i(is64, false, BPF_OP(code),
1864                     dst_lo, 0, dstk, &prog);
1865             if (!bpf_prog->aux->verifier_zext)
1866                 emit_ia32_mov_i(dst_hi, 0, dstk, &prog);
1867             break;
1868         /* dst = ~dst (64 bit) */
1869         case BPF_ALU64 | BPF_NEG:
1870             emit_ia32_neg64(dst, dstk, &prog);
1871             break;
1872         /* dst = dst * src/imm */
1873         case BPF_ALU64 | BPF_MUL | BPF_X:
1874         case BPF_ALU64 | BPF_MUL | BPF_K:
1875             switch (BPF_SRC(code)) {
1876             case BPF_X:
1877                 emit_ia32_mul_r64(dst, src, dstk, sstk, &prog);
1878                 break;
1879             case BPF_K:
1880                 emit_ia32_mul_i64(dst, imm32, dstk, &prog);
1881                 break;
1882             }
1883             break;
1884         /* dst = htole(dst) */
1885         case BPF_ALU | BPF_END | BPF_FROM_LE:
1886             emit_ia32_to_le_r64(dst, imm32, dstk, &prog,
1887                         bpf_prog->aux);
1888             break;
1889         /* dst = htobe(dst) */
1890         case BPF_ALU | BPF_END | BPF_FROM_BE:
1891             emit_ia32_to_be_r64(dst, imm32, dstk, &prog,
1892                         bpf_prog->aux);
1893             break;
1894         /* dst = imm64 */
1895         case BPF_LD | BPF_IMM | BPF_DW: {
1896             s32 hi, lo = imm32;
1897
1898             hi = insn[1].imm;
1899             emit_ia32_mov_i(dst_lo, lo, dstk, &prog);
1900             emit_ia32_mov_i(dst_hi, hi, dstk, &prog);
1901             insn++;
1902             i++;
1903             break;
1904         }
1905         /* speculation barrier */
1906         case BPF_ST | BPF_NOSPEC:
1907             if (boot_cpu_has(X86_FEATURE_XMM2))
1908                 /* Emit 'lfence' */
1909                 EMIT3(0x0F, 0xAE, 0xE8);
1910             break;
1911         /* ST: *(u8*)(dst_reg + off) = imm */
1912         case BPF_ST | BPF_MEM | BPF_H:
1913         case BPF_ST | BPF_MEM | BPF_B:
1914         case BPF_ST | BPF_MEM | BPF_W:
1915         case BPF_ST | BPF_MEM | BPF_DW:
1916             if (dstk)
1917                 /* mov eax,dword ptr [ebp+off] */
1918                 EMIT3(0x8B, add_2reg(0x40, IA32_EBP, IA32_EAX),
1919                       STACK_VAR(dst_lo));
1920             else
1921                 /* mov eax,dst_lo */
1922                 EMIT2(0x8B, add_2reg(0xC0, dst_lo, IA32_EAX));
1923
1924             switch (BPF_SIZE(code)) {
1925             case BPF_B:
1926                 EMIT(0xC6, 1); break;
1927             case BPF_H:
1928                 EMIT2(0x66, 0xC7); break;
1929             case BPF_W:
1930             case BPF_DW:
1931                 EMIT(0xC7, 1); break;
1932             }
1933
1934             if (is_imm8(insn->off))
1935                 EMIT2(add_1reg(0x40, IA32_EAX), insn->off);
1936             else
1937                 EMIT1_off32(add_1reg(0x80, IA32_EAX),
1938                         insn->off);
1939             EMIT(imm32, bpf_size_to_x86_bytes(BPF_SIZE(code)));
1940
1941             if (BPF_SIZE(code) == BPF_DW) {
1942                 u32 hi;
1943
1944                 hi = imm32 & (1<<31) ? (u32)~0 : 0;
1945                 EMIT2_off32(0xC7, add_1reg(0x80, IA32_EAX),
1946                         insn->off + 4);
1947                 EMIT(hi, 4);
1948             }
1949             break;
1950
1951         /* STX: *(u8*)(dst_reg + off) = src_reg */
1952         case BPF_STX | BPF_MEM | BPF_B:
1953         case BPF_STX | BPF_MEM | BPF_H:
1954         case BPF_STX | BPF_MEM | BPF_W:
1955         case BPF_STX | BPF_MEM | BPF_DW:
1956             if (dstk)
1957                 /* mov eax,dword ptr [ebp+off] */
1958                 EMIT3(0x8B, add_2reg(0x40, IA32_EBP, IA32_EAX),
1959                       STACK_VAR(dst_lo));
1960             else
1961                 /* mov eax,dst_lo */
1962                 EMIT2(0x8B, add_2reg(0xC0, dst_lo, IA32_EAX));
1963
1964             if (sstk)
1965                 /* mov edx,dword ptr [ebp+off] */
1966                 EMIT3(0x8B, add_2reg(0x40, IA32_EBP, IA32_EDX),
1967                       STACK_VAR(src_lo));
1968             else
1969                 /* mov edx,src_lo */
1970                 EMIT2(0x8B, add_2reg(0xC0, src_lo, IA32_EDX));
1971
1972             switch (BPF_SIZE(code)) {
1973             case BPF_B:
1974                 EMIT(0x88, 1); break;
1975             case BPF_H:
1976                 EMIT2(0x66, 0x89); break;
1977             case BPF_W:
1978             case BPF_DW:
1979                 EMIT(0x89, 1); break;
1980             }
1981
1982             if (is_imm8(insn->off))
1983                 EMIT2(add_2reg(0x40, IA32_EAX, IA32_EDX),
1984                       insn->off);
1985             else
1986                 EMIT1_off32(add_2reg(0x80, IA32_EAX, IA32_EDX),
1987                         insn->off);
1988
1989             if (BPF_SIZE(code) == BPF_DW) {
1990                 if (sstk)
1991                     /* mov edi,dword ptr [ebp+off] */
1992                     EMIT3(0x8B, add_2reg(0x40, IA32_EBP,
1993                                  IA32_EDX),
1994                           STACK_VAR(src_hi));
1995                 else
1996                     /* mov edi,src_hi */
1997                     EMIT2(0x8B, add_2reg(0xC0, src_hi,
1998                                  IA32_EDX));
1999                 EMIT1(0x89);
2000                 if (is_imm8(insn->off + 4)) {
2001                     EMIT2(add_2reg(0x40, IA32_EAX,
2002                                IA32_EDX),
2003                           insn->off + 4);
2004                 } else {
2005                     EMIT1(add_2reg(0x80, IA32_EAX,
2006                                IA32_EDX));
2007                     EMIT(insn->off + 4, 4);
2008                 }
2009             }
2010             break;
2011
2012         /* LDX: dst_reg = *(u8*)(src_reg + off) */
2013         case BPF_LDX | BPF_MEM | BPF_B:
2014         case BPF_LDX | BPF_MEM | BPF_H:
2015         case BPF_LDX | BPF_MEM | BPF_W:
2016         case BPF_LDX | BPF_MEM | BPF_DW:
2017             if (sstk)
2018                 /* mov eax,dword ptr [ebp+off] */
2019                 EMIT3(0x8B, add_2reg(0x40, IA32_EBP, IA32_EAX),
2020                       STACK_VAR(src_lo));
2021             else
2022                 /* mov eax,dword ptr [ebp+off] */
2023                 EMIT2(0x8B, add_2reg(0xC0, src_lo, IA32_EAX));
2024
2025             switch (BPF_SIZE(code)) {
2026             case BPF_B:
2027                 EMIT2(0x0F, 0xB6); break;
2028             case BPF_H:
2029                 EMIT2(0x0F, 0xB7); break;
2030             case BPF_W:
2031             case BPF_DW:
2032                 EMIT(0x8B, 1); break;
2033             }
2034
2035             if (is_imm8(insn->off))
2036                 EMIT2(add_2reg(0x40, IA32_EAX, IA32_EDX),
2037                       insn->off);
2038             else
2039                 EMIT1_off32(add_2reg(0x80, IA32_EAX, IA32_EDX),
2040                         insn->off);
2041
2042             if (dstk)
2043                 /* mov dword ptr [ebp+off],edx */
2044                 EMIT3(0x89, add_2reg(0x40, IA32_EBP, IA32_EDX),
2045                       STACK_VAR(dst_lo));
2046             else
2047                 /* mov dst_lo,edx */
2048                 EMIT2(0x89, add_2reg(0xC0, dst_lo, IA32_EDX));
2049             switch (BPF_SIZE(code)) {
2050             case BPF_B:
2051             case BPF_H:
2052             case BPF_W:
2053                 if (bpf_prog->aux->verifier_zext)
2054                     break;
2055                 if (dstk) {
2056                     EMIT3(0xC7, add_1reg(0x40, IA32_EBP),
2057                           STACK_VAR(dst_hi));
2058                     EMIT(0x0, 4);
2059                 } else {
2060                     /* xor dst_hi,dst_hi */
2061                     EMIT2(0x33,
2062                           add_2reg(0xC0, dst_hi, dst_hi));
2063                 }
2064                 break;
2065             case BPF_DW:
2066                 EMIT2_off32(0x8B,
2067                         add_2reg(0x80, IA32_EAX, IA32_EDX),
2068                         insn->off + 4);
2069                 if (dstk)
2070                     EMIT3(0x89,
2071                           add_2reg(0x40, IA32_EBP,
2072                                IA32_EDX),
2073                           STACK_VAR(dst_hi));
2074                 else
2075                     EMIT2(0x89,
2076                           add_2reg(0xC0, dst_hi, IA32_EDX));
2077                 break;
2078             default:
2079                 break;
2080             }
2081             break;
2082         /* call */
2083         case BPF_JMP | BPF_CALL:
2084         {
2085             const u8 *r1 = bpf2ia32[BPF_REG_1];
2086             const u8 *r2 = bpf2ia32[BPF_REG_2];
2087             const u8 *r3 = bpf2ia32[BPF_REG_3];
2088             const u8 *r4 = bpf2ia32[BPF_REG_4];
2089             const u8 *r5 = bpf2ia32[BPF_REG_5];
2090
2091             if (insn->src_reg == BPF_PSEUDO_CALL)
2092                 goto notyet;
2093
2094             if (insn->src_reg == BPF_PSEUDO_KFUNC_CALL) {
2095                 int err;
2096
2097                 err = emit_kfunc_call(bpf_prog,
2098                               image + addrs[i],
2099                               insn, &prog);
2100
2101                 if (err)
2102                     return err;
2103                 break;
2104             }
2105
2106             func = (u8 *) __bpf_call_base + imm32;
2107             jmp_offset = func - (image + addrs[i]);
2108
2109             if (!imm32 || !is_simm32(jmp_offset)) {
2110                 pr_err("unsupported BPF func %d addr %p image %p\n",
2111                        imm32, func, image);
2112                 return -EINVAL;
2113             }
2114
2115             /* mov eax,dword ptr [ebp+off] */
2116             EMIT3(0x8B, add_2reg(0x40, IA32_EBP, IA32_EAX),
2117                   STACK_VAR(r1[0]));
2118             /* mov edx,dword ptr [ebp+off] */
2119             EMIT3(0x8B, add_2reg(0x40, IA32_EBP, IA32_EDX),
2120                   STACK_VAR(r1[1]));
2121
2122             emit_push_r64(r5, &prog);
2123             emit_push_r64(r4, &prog);
2124             emit_push_r64(r3, &prog);
2125             emit_push_r64(r2, &prog);
2126
2127             EMIT1_off32(0xE8, jmp_offset + 9);
2128
2129             /* mov dword ptr [ebp+off],eax */
2130             EMIT3(0x89, add_2reg(0x40, IA32_EBP, IA32_EAX),
2131                   STACK_VAR(r0[0]));
2132             /* mov dword ptr [ebp+off],edx */
2133             EMIT3(0x89, add_2reg(0x40, IA32_EBP, IA32_EDX),
2134                   STACK_VAR(r0[1]));
2135
2136             /* add esp,32 */
2137             EMIT3(0x83, add_1reg(0xC0, IA32_ESP), 32);
2138             break;
2139         }
2140         case BPF_JMP | BPF_TAIL_CALL:
2141             emit_bpf_tail_call(&prog, image + addrs[i - 1]);
2142             break;
2143
2144         /* cond jump */
2145         case BPF_JMP | BPF_JEQ | BPF_X:
2146         case BPF_JMP | BPF_JNE | BPF_X:
2147         case BPF_JMP | BPF_JGT | BPF_X:
2148         case BPF_JMP | BPF_JLT | BPF_X:
2149         case BPF_JMP | BPF_JGE | BPF_X:
2150         case BPF_JMP | BPF_JLE | BPF_X:
2151         case BPF_JMP32 | BPF_JEQ | BPF_X:
2152         case BPF_JMP32 | BPF_JNE | BPF_X:
2153         case BPF_JMP32 | BPF_JGT | BPF_X:
2154         case BPF_JMP32 | BPF_JLT | BPF_X:
2155         case BPF_JMP32 | BPF_JGE | BPF_X:
2156         case BPF_JMP32 | BPF_JLE | BPF_X:
2157         case BPF_JMP32 | BPF_JSGT | BPF_X:
2158         case BPF_JMP32 | BPF_JSLE | BPF_X:
2159         case BPF_JMP32 | BPF_JSLT | BPF_X:
2160         case BPF_JMP32 | BPF_JSGE | BPF_X: {
2161             bool is_jmp64 = BPF_CLASS(insn->code) == BPF_JMP;
2162             u8 dreg_lo = dstk ? IA32_EAX : dst_lo;
2163             u8 dreg_hi = dstk ? IA32_EDX : dst_hi;
2164             u8 sreg_lo = sstk ? IA32_ECX : src_lo;
2165             u8 sreg_hi = sstk ? IA32_EBX : src_hi;
2166
2167             if (dstk) {
2168                 EMIT3(0x8B, add_2reg(0x40, IA32_EBP, IA32_EAX),
2169                       STACK_VAR(dst_lo));
2170                 if (is_jmp64)
2171                     EMIT3(0x8B,
2172                           add_2reg(0x40, IA32_EBP,
2173                                IA32_EDX),
2174                           STACK_VAR(dst_hi));
2175             }
2176
2177             if (sstk) {
2178                 EMIT3(0x8B, add_2reg(0x40, IA32_EBP, IA32_ECX),
2179                       STACK_VAR(src_lo));
2180                 if (is_jmp64)
2181                     EMIT3(0x8B,
2182                           add_2reg(0x40, IA32_EBP,
2183                                IA32_EBX),
2184                           STACK_VAR(src_hi));
2185             }
2186
2187             if (is_jmp64) {
2188                 /* cmp dreg_hi,sreg_hi */
2189                 EMIT2(0x39, add_2reg(0xC0, dreg_hi, sreg_hi));
2190                 EMIT2(IA32_JNE, 2);
2191             }
2192             /* cmp dreg_lo,sreg_lo */
2193             EMIT2(0x39, add_2reg(0xC0, dreg_lo, sreg_lo));
2194             goto emit_cond_jmp;
2195         }
2196         case BPF_JMP | BPF_JSGT | BPF_X:
2197         case BPF_JMP | BPF_JSLE | BPF_X:
2198         case BPF_JMP | BPF_JSLT | BPF_X:
2199         case BPF_JMP | BPF_JSGE | BPF_X: {
2200             u8 dreg_lo = dstk ? IA32_EAX : dst_lo;
2201             u8 dreg_hi = dstk ? IA32_EDX : dst_hi;
2202             u8 sreg_lo = sstk ? IA32_ECX : src_lo;
2203             u8 sreg_hi = sstk ? IA32_EBX : src_hi;
2204
2205             if (dstk) {
2206                 EMIT3(0x8B, add_2reg(0x40, IA32_EBP, IA32_EAX),
2207                       STACK_VAR(dst_lo));
2208                 EMIT3(0x8B,
2209                       add_2reg(0x40, IA32_EBP,
2210                            IA32_EDX),
2211                       STACK_VAR(dst_hi));
2212             }
2213
2214             if (sstk) {
2215                 EMIT3(0x8B, add_2reg(0x40, IA32_EBP, IA32_ECX),
2216                       STACK_VAR(src_lo));
2217                 EMIT3(0x8B,
2218                       add_2reg(0x40, IA32_EBP,
2219                            IA32_EBX),
2220                       STACK_VAR(src_hi));
2221             }
2222
2223             /* cmp dreg_hi,sreg_hi */
2224             EMIT2(0x39, add_2reg(0xC0, dreg_hi, sreg_hi));
2225             EMIT2(IA32_JNE, 10);
2226             /* cmp dreg_lo,sreg_lo */
2227             EMIT2(0x39, add_2reg(0xC0, dreg_lo, sreg_lo));
2228             goto emit_cond_jmp_signed;
2229         }
2230         case BPF_JMP | BPF_JSET | BPF_X:
2231         case BPF_JMP32 | BPF_JSET | BPF_X: {
2232             bool is_jmp64 = BPF_CLASS(insn->code) == BPF_JMP;
2233             u8 dreg_lo = IA32_EAX;
2234             u8 dreg_hi = IA32_EDX;
2235             u8 sreg_lo = sstk ? IA32_ECX : src_lo;
2236             u8 sreg_hi = sstk ? IA32_EBX : src_hi;
2237
2238             if (dstk) {
2239                 EMIT3(0x8B, add_2reg(0x40, IA32_EBP, IA32_EAX),
2240                       STACK_VAR(dst_lo));
2241                 if (is_jmp64)
2242                     EMIT3(0x8B,
2243                           add_2reg(0x40, IA32_EBP,
2244                                IA32_EDX),
2245                           STACK_VAR(dst_hi));
2246             } else {
2247                 /* mov dreg_lo,dst_lo */
2248                 EMIT2(0x89, add_2reg(0xC0, dreg_lo, dst_lo));
2249                 if (is_jmp64)
2250                     /* mov dreg_hi,dst_hi */
2251                     EMIT2(0x89,
2252                           add_2reg(0xC0, dreg_hi, dst_hi));
2253             }
2254
2255             if (sstk) {
2256                 EMIT3(0x8B, add_2reg(0x40, IA32_EBP, IA32_ECX),
2257                       STACK_VAR(src_lo));
2258                 if (is_jmp64)
2259                     EMIT3(0x8B,
2260                           add_2reg(0x40, IA32_EBP,
2261                                IA32_EBX),
2262                           STACK_VAR(src_hi));
2263             }
2264             /* and dreg_lo,sreg_lo */
2265             EMIT2(0x23, add_2reg(0xC0, sreg_lo, dreg_lo));
2266             if (is_jmp64) {
2267                 /* and dreg_hi,sreg_hi */
2268                 EMIT2(0x23, add_2reg(0xC0, sreg_hi, dreg_hi));
2269                 /* or dreg_lo,dreg_hi */
2270                 EMIT2(0x09, add_2reg(0xC0, dreg_lo, dreg_hi));
2271             }
2272             goto emit_cond_jmp;
2273         }
2274         case BPF_JMP | BPF_JSET | BPF_K:
2275         case BPF_JMP32 | BPF_JSET | BPF_K: {
2276             bool is_jmp64 = BPF_CLASS(insn->code) == BPF_JMP;
2277             u8 dreg_lo = IA32_EAX;
2278             u8 dreg_hi = IA32_EDX;
2279             u8 sreg_lo = IA32_ECX;
2280             u8 sreg_hi = IA32_EBX;
2281             u32 hi;
2282
2283             if (dstk) {
2284                 EMIT3(0x8B, add_2reg(0x40, IA32_EBP, IA32_EAX),
2285                       STACK_VAR(dst_lo));
2286                 if (is_jmp64)
2287                     EMIT3(0x8B,
2288                           add_2reg(0x40, IA32_EBP,
2289                                IA32_EDX),
2290                           STACK_VAR(dst_hi));
2291             } else {
2292                 /* mov dreg_lo,dst_lo */
2293                 EMIT2(0x89, add_2reg(0xC0, dreg_lo, dst_lo));
2294                 if (is_jmp64)
2295                     /* mov dreg_hi,dst_hi */
2296                     EMIT2(0x89,
2297                           add_2reg(0xC0, dreg_hi, dst_hi));
2298             }
2299
2300             /* mov ecx,imm32 */
2301             EMIT2_off32(0xC7, add_1reg(0xC0, sreg_lo), imm32);
2302
2303             /* and dreg_lo,sreg_lo */
2304             EMIT2(0x23, add_2reg(0xC0, sreg_lo, dreg_lo));
2305             if (is_jmp64) {
2306                 hi = imm32 & (1 << 31) ? (u32)~0 : 0;
2307                 /* mov ebx,imm32 */
2308                 EMIT2_off32(0xC7, add_1reg(0xC0, sreg_hi), hi);
2309                 /* and dreg_hi,sreg_hi */
2310                 EMIT2(0x23, add_2reg(0xC0, sreg_hi, dreg_hi));
2311                 /* or dreg_lo,dreg_hi */
2312                 EMIT2(0x09, add_2reg(0xC0, dreg_lo, dreg_hi));
2313             }
2314             goto emit_cond_jmp;
2315         }
2316         case BPF_JMP | BPF_JEQ | BPF_K:
2317         case BPF_JMP | BPF_JNE | BPF_K:
2318         case BPF_JMP | BPF_JGT | BPF_K:
2319         case BPF_JMP | BPF_JLT | BPF_K:
2320         case BPF_JMP | BPF_JGE | BPF_K:
2321         case BPF_JMP | BPF_JLE | BPF_K:
2322         case BPF_JMP32 | BPF_JEQ | BPF_K:
2323         case BPF_JMP32 | BPF_JNE | BPF_K:
2324         case BPF_JMP32 | BPF_JGT | BPF_K:
2325         case BPF_JMP32 | BPF_JLT | BPF_K:
2326         case BPF_JMP32 | BPF_JGE | BPF_K:
2327         case BPF_JMP32 | BPF_JLE | BPF_K:
2328         case BPF_JMP32 | BPF_JSGT | BPF_K:
2329         case BPF_JMP32 | BPF_JSLE | BPF_K:
2330         case BPF_JMP32 | BPF_JSLT | BPF_K:
2331         case BPF_JMP32 | BPF_JSGE | BPF_K: {
2332             bool is_jmp64 = BPF_CLASS(insn->code) == BPF_JMP;
2333             u8 dreg_lo = dstk ? IA32_EAX : dst_lo;
2334             u8 dreg_hi = dstk ? IA32_EDX : dst_hi;
2335             u8 sreg_lo = IA32_ECX;
2336             u8 sreg_hi = IA32_EBX;
2337             u32 hi;
2338
2339             if (dstk) {
2340                 EMIT3(0x8B, add_2reg(0x40, IA32_EBP, IA32_EAX),
2341                       STACK_VAR(dst_lo));
2342                 if (is_jmp64)
2343                     EMIT3(0x8B,
2344                           add_2reg(0x40, IA32_EBP,
2345                                IA32_EDX),
2346                           STACK_VAR(dst_hi));
2347             }
2348
2349             /* mov ecx,imm32 */
2350             EMIT2_off32(0xC7, add_1reg(0xC0, IA32_ECX), imm32);
2351             if (is_jmp64) {
2352                 hi = imm32 & (1 << 31) ? (u32)~0 : 0;
2353                 /* mov ebx,imm32 */
2354                 EMIT2_off32(0xC7, add_1reg(0xC0, IA32_EBX), hi);
2355                 /* cmp dreg_hi,sreg_hi */
2356                 EMIT2(0x39, add_2reg(0xC0, dreg_hi, sreg_hi));
2357                 EMIT2(IA32_JNE, 2);
2358             }
2359             /* cmp dreg_lo,sreg_lo */
2360             EMIT2(0x39, add_2reg(0xC0, dreg_lo, sreg_lo));
2361
2362 emit_cond_jmp:      jmp_cond = get_cond_jmp_opcode(BPF_OP(code), false);
2363             if (jmp_cond == COND_JMP_OPCODE_INVALID)
2364                 return -EFAULT;
2365             jmp_offset = addrs[i + insn->off] - addrs[i];
2366             if (is_imm8(jmp_offset)) {
2367                 EMIT2(jmp_cond, jmp_offset);
2368             } else if (is_simm32(jmp_offset)) {
2369                 EMIT2_off32(0x0F, jmp_cond + 0x10, jmp_offset);
2370             } else {
2371                 pr_err("cond_jmp gen bug %llx\n", jmp_offset);
2372                 return -EFAULT;
2373             }
2374             break;
2375         }
2376         case BPF_JMP | BPF_JSGT | BPF_K:
2377         case BPF_JMP | BPF_JSLE | BPF_K:
2378         case BPF_JMP | BPF_JSLT | BPF_K:
2379         case BPF_JMP | BPF_JSGE | BPF_K: {
2380             u8 dreg_lo = dstk ? IA32_EAX : dst_lo;
2381             u8 dreg_hi = dstk ? IA32_EDX : dst_hi;
2382             u8 sreg_lo = IA32_ECX;
2383             u8 sreg_hi = IA32_EBX;
2384             u32 hi;
2385
2386             if (dstk) {
2387                 EMIT3(0x8B, add_2reg(0x40, IA32_EBP, IA32_EAX),
2388                       STACK_VAR(dst_lo));
2389                 EMIT3(0x8B,
2390                       add_2reg(0x40, IA32_EBP,
2391                            IA32_EDX),
2392                       STACK_VAR(dst_hi));
2393             }
2394
2395             /* mov ecx,imm32 */
2396             EMIT2_off32(0xC7, add_1reg(0xC0, IA32_ECX), imm32);
2397             hi = imm32 & (1 << 31) ? (u32)~0 : 0;
2398             /* mov ebx,imm32 */
2399             EMIT2_off32(0xC7, add_1reg(0xC0, IA32_EBX), hi);
2400             /* cmp dreg_hi,sreg_hi */
2401             EMIT2(0x39, add_2reg(0xC0, dreg_hi, sreg_hi));
2402             EMIT2(IA32_JNE, 10);
2403             /* cmp dreg_lo,sreg_lo */
2404             EMIT2(0x39, add_2reg(0xC0, dreg_lo, sreg_lo));
2405
2406             /*
2407              * For simplicity of branch offset computation,
2408              * let's use fixed jump coding here.
2409              */
2410 emit_cond_jmp_signed:   /* Check the condition for low 32-bit comparison */
2411             jmp_cond = get_cond_jmp_opcode(BPF_OP(code), true);
2412             if (jmp_cond == COND_JMP_OPCODE_INVALID)
2413                 return -EFAULT;
2414             jmp_offset = addrs[i + insn->off] - addrs[i] + 8;
2415             if (is_simm32(jmp_offset)) {
2416                 EMIT2_off32(0x0F, jmp_cond + 0x10, jmp_offset);
2417             } else {
2418                 pr_err("cond_jmp gen bug %llx\n", jmp_offset);
2419                 return -EFAULT;
2420             }
2421             EMIT2(0xEB, 6);
2422
2423             /* Check the condition for high 32-bit comparison */
2424             jmp_cond = get_cond_jmp_opcode(BPF_OP(code), false);
2425             if (jmp_cond == COND_JMP_OPCODE_INVALID)
2426                 return -EFAULT;
2427             jmp_offset = addrs[i + insn->off] - addrs[i];
2428             if (is_simm32(jmp_offset)) {
2429                 EMIT2_off32(0x0F, jmp_cond + 0x10, jmp_offset);
2430             } else {
2431                 pr_err("cond_jmp gen bug %llx\n", jmp_offset);
2432                 return -EFAULT;
2433             }
2434             break;
2435         }
2436         case BPF_JMP | BPF_JA:
2437             if (insn->off == -1)
2438                 /* -1 jmp instructions will always jump
2439                  * backwards two bytes. Explicitly handling
2440                  * this case avoids wasting too many passes
2441                  * when there are long sequences of replaced
2442                  * dead code.
2443                  */
2444                 jmp_offset = -2;
2445             else
2446                 jmp_offset = addrs[i + insn->off] - addrs[i];
2447
2448             if (!jmp_offset)
2449                 /* Optimize out nop jumps */
2450                 break;
2451 emit_jmp:
2452             if (is_imm8(jmp_offset)) {
2453                 EMIT2(0xEB, jmp_offset);
2454             } else if (is_simm32(jmp_offset)) {
2455                 EMIT1_off32(0xE9, jmp_offset);
2456             } else {
2457                 pr_err("jmp gen bug %llx\n", jmp_offset);
2458                 return -EFAULT;
2459             }
2460             break;
2461         case BPF_STX | BPF_ATOMIC | BPF_W:
2462         case BPF_STX | BPF_ATOMIC | BPF_DW:
2463             goto notyet;
2464         case BPF_JMP | BPF_EXIT:
2465             if (seen_exit) {
2466                 jmp_offset = ctx->cleanup_addr - addrs[i];
2467                 goto emit_jmp;
2468             }
2469             seen_exit = true;
2470             /* Update cleanup_addr */
2471             ctx->cleanup_addr = proglen;
2472             emit_epilogue(&prog, bpf_prog->aux->stack_depth);
2473             break;
2474 notyet:
2475             pr_info_once("*** NOT YET: opcode %02x ***\n", code);
2476             return -EFAULT;
2477         default:
2478             /*
2479              * This error will be seen if new instruction was added
2480              * to interpreter, but not to JIT or if there is junk in
2481              * bpf_prog
2482              */
2483             pr_err("bpf_jit: unknown opcode %02x\n", code);
2484             return -EINVAL;
2485         }
2486
2487         ilen = prog - temp;
2488         if (ilen > BPF_MAX_INSN_SIZE) {
2489             pr_err("bpf_jit: fatal insn size error\n");
2490             return -EFAULT;
2491         }
2492
2493         if (image) {
2494             /*
2495              * When populating the image, assert that:
2496              *
2497              *  i) We do not write beyond the allocated space, and
2498              * ii) addrs[i] did not change from the prior run, in order
2499              *     to validate assumptions made for computing branch
2500              *     displacements.
2501              */
2502             if (unlikely(proglen + ilen > oldproglen ||
2503                      proglen + ilen != addrs[i])) {
2504                 pr_err("bpf_jit: fatal error\n");
2505                 return -EFAULT;
2506             }
2507             memcpy(image + proglen, temp, ilen);
2508         }
2509         proglen += ilen;
2510         addrs[i] = proglen;
2511         prog = temp;
2512     }
2513     return proglen;
2514 }
2515
2516 bool bpf_jit_needs_zext(void)
2517 {
2518     return true;
2519 }
2520
2521 struct bpf_prog *bpf_int_jit_compile(struct bpf_prog *prog)
2522 {
2523     struct bpf_binary_header *header = NULL;
2524     struct bpf_prog *tmp, *orig_prog = prog;
2525     int proglen, oldproglen = 0;
2526     struct jit_context ctx = {};
2527     bool tmp_blinded = false;
2528     u8 *image = NULL;
2529     int *addrs;
2530     int pass;
2531     int i;
2532
2533     if (!prog->jit_requested)
2534         return orig_prog;
2535
2536     tmp = bpf_jit_blind_constants(prog);
2537     /*
2538      * If blinding was requested and we failed during blinding,
2539      * we must fall back to the interpreter.
2540      */
2541     if (IS_ERR(tmp))
2542         return orig_prog;
2543     if (tmp != prog) {
2544         tmp_blinded = true;
2545         prog = tmp;
2546     }
2547
2548     addrs = kmalloc_array(prog->len, sizeof(*addrs), GFP_KERNEL);
2549     if (!addrs) {
2550         prog = orig_prog;
2551         goto out;
2552     }
2553
2554     /*
2555      * Before first pass, make a rough estimation of addrs[]
2556      * each BPF instruction is translated to less than 64 bytes
2557      */
2558     for (proglen = 0, i = 0; i < prog->len; i++) {
2559         proglen += 64;
2560         addrs[i] = proglen;
2561     }
2562     ctx.cleanup_addr = proglen;
2563
2564     /*
2565      * JITed image shrinks with every pass and the loop iterates
2566      * until the image stops shrinking. Very large BPF programs
2567      * may converge on the last pass. In such case do one more
2568      * pass to emit the final image.
2569      */
2570     for (pass = 0; pass < 20 || image; pass++) {
2571         proglen = do_jit(prog, addrs, image, oldproglen, &ctx);
2572         if (proglen <= 0) {
2573 out_image:
2574             image = NULL;
2575             if (header)
2576                 bpf_jit_binary_free(header);
2577             prog = orig_prog;
2578             goto out_addrs;
2579         }
2580         if (image) {
2581             if (proglen != oldproglen) {
2582                 pr_err("bpf_jit: proglen=%d != oldproglen=%d\n",
2583                        proglen, oldproglen);
2584                 goto out_image;
2585             }
2586             break;
2587         }
2588         if (proglen == oldproglen) {
2589             header = bpf_jit_binary_alloc(proglen, &image,
2590                               1, jit_fill_hole);
2591             if (!header) {
2592                 prog = orig_prog;
2593                 goto out_addrs;
2594             }
2595         }
2596         oldproglen = proglen;
2597         cond_resched();
2598     }
2599
2600     if (bpf_jit_enable > 1)
2601         bpf_jit_dump(prog->len, proglen, pass + 1, image);
2602
2603     if (image) {
2604         bpf_jit_binary_lock_ro(header);
2605         prog->bpf_func = (void *)image;
2606         prog->jited = 1;
2607         prog->jited_len = proglen;
2608     } else {
2609         prog = orig_prog;
2610     }
2611
2612 out_addrs:
2613     kfree(addrs);
2614 out:
2615     if (tmp_blinded)
2616         bpf_jit_prog_release_other(prog, prog == orig_prog ?
2617                        tmp : orig_prog);
2618     return prog;
2619 }
2620
2621 bool bpf_jit_supports_kfunc_call(void)
2622 {
2623     return true;
2624 }