OSCL-LXR linux-6.0.1/​arch/​sparc/​net/​bpf_jit_comp_64.c	Source navigation Diff markup Identifier search General search
	Version: linux-6.0.1 spark-3.0.0 hadoop-3.2.0-src hadoop-3.3.0-src spark-2.4.7 linux-4.15.13 hadoop-2.7.4-src Revert   Change

 // SPDX-License-Identifier: GPL-2.0
 #include <linux/moduleloader.h>
 #include <linux/workqueue.h>
 #include <linux/netdevice.h>
 #include <linux/filter.h>
 #include <linux/bpf.h>
 #include <linux/cache.h>
 #include <linux/if_vlan.h>
 
 #include <asm/cacheflush.h>
 #include <asm/ptrace.h>
 
 #include "bpf_jit_64.h"
 
 static inline bool is_simm13(unsigned int value)
 {
     return value + 0x1000 < 0x2000;
 }
 
 static inline bool is_simm10(unsigned int value)
 {
     return value + 0x200 < 0x400;
 }
 
 static inline bool is_simm5(unsigned int value)
 {
     return value + 0x10 < 0x20;
 }
 
 static inline bool is_sethi(unsigned int value)
 {
     return (value & ~0x3fffff) == 0;
 }
 
 static void bpf_flush_icache(void *start_, void *end_)
 {
     /* Cheetah's I-cache is fully coherent.  */
     if (tlb_type == spitfire) {
         unsigned long start = (unsigned long) start_;
         unsigned long end = (unsigned long) end_;
 
         start &= ~7UL;
         end = (end + 7UL) & ~7UL;
         while (start < end) {
             flushi(start);
             start += 32;
         }
     }
 }
 
 #define S13(X)      ((X) & 0x1fff)
 #define S5(X)       ((X) & 0x1f)
 #define IMMED       0x00002000
 #define RD(X)       ((X) << 25)
 #define RS1(X)      ((X) << 14)
 #define RS2(X)      ((X))
 #define OP(X)       ((X) << 30)
 #define OP2(X)      ((X) << 22)
 #define OP3(X)      ((X) << 19)
 #define COND(X)     (((X) & 0xf) << 25)
 #define CBCOND(X)   (((X) & 0x1f) << 25)
 #define F1(X)       OP(X)
 #define F2(X, Y)    (OP(X) | OP2(Y))
 #define F3(X, Y)    (OP(X) | OP3(Y))
 #define ASI(X)      (((X) & 0xff) << 5)
 
 #define CONDN       COND(0x0)
 #define CONDE       COND(0x1)
 #define CONDLE      COND(0x2)
 #define CONDL       COND(0x3)
 #define CONDLEU     COND(0x4)
 #define CONDCS      COND(0x5)
 #define CONDNEG     COND(0x6)
 #define CONDVC      COND(0x7)
 #define CONDA       COND(0x8)
 #define CONDNE      COND(0x9)
 #define CONDG       COND(0xa)
 #define CONDGE      COND(0xb)
 #define CONDGU      COND(0xc)
 #define CONDCC      COND(0xd)
 #define CONDPOS     COND(0xe)
 #define CONDVS      COND(0xf)
 
 #define CONDGEU     CONDCC
 #define CONDLU      CONDCS
 
 #define WDISP22(X)  (((X) >> 2) & 0x3fffff)
 #define WDISP19(X)  (((X) >> 2) & 0x7ffff)
 
 /* The 10-bit branch displacement for CBCOND is split into two fields */
 static u32 WDISP10(u32 off)
 {
     u32 ret = ((off >> 2) & 0xff) << 5;
 
     ret |= ((off >> (2 + 8)) & 0x03) << 19;
 
     return ret;
 }
 
 #define CBCONDE     CBCOND(0x09)
 #define CBCONDLE    CBCOND(0x0a)
 #define CBCONDL     CBCOND(0x0b)
 #define CBCONDLEU   CBCOND(0x0c)
 #define CBCONDCS    CBCOND(0x0d)
 #define CBCONDN     CBCOND(0x0e)
 #define CBCONDVS    CBCOND(0x0f)
 #define CBCONDNE    CBCOND(0x19)
 #define CBCONDG     CBCOND(0x1a)
 #define CBCONDGE    CBCOND(0x1b)
 #define CBCONDGU    CBCOND(0x1c)
 #define CBCONDCC    CBCOND(0x1d)
 #define CBCONDPOS   CBCOND(0x1e)
 #define CBCONDVC    CBCOND(0x1f)
 
 #define CBCONDGEU   CBCONDCC
 #define CBCONDLU    CBCONDCS
 
 #define ANNUL       (1 << 29)
 #define XCC     (1 << 21)
 
 #define BRANCH      (F2(0, 1) | XCC)
 #define CBCOND_OP   (F2(0, 3) | XCC)
 
 #define BA      (BRANCH | CONDA)
 #define BG      (BRANCH | CONDG)
 #define BL      (BRANCH | CONDL)
 #define BLE     (BRANCH | CONDLE)
 #define BGU     (BRANCH | CONDGU)
 #define BLEU        (BRANCH | CONDLEU)
 #define BGE     (BRANCH | CONDGE)
 #define BGEU        (BRANCH | CONDGEU)
 #define BLU     (BRANCH | CONDLU)
 #define BE      (BRANCH | CONDE)
 #define BNE     (BRANCH | CONDNE)
 
 #define SETHI(K, REG)   \
     (F2(0, 0x4) | RD(REG) | (((K) >> 10) & 0x3fffff))
 #define OR_LO(K, REG)   \
     (F3(2, 0x02) | IMMED | RS1(REG) | ((K) & 0x3ff) | RD(REG))
 
 #define ADD     F3(2, 0x00)
 #define AND     F3(2, 0x01)
 #define ANDCC       F3(2, 0x11)
 #define OR      F3(2, 0x02)
 #define XOR     F3(2, 0x03)
 #define SUB     F3(2, 0x04)
 #define SUBCC       F3(2, 0x14)
 #define MUL     F3(2, 0x0a)
 #define MULX        F3(2, 0x09)
 #define UDIVX       F3(2, 0x0d)
 #define DIV     F3(2, 0x0e)
 #define SLL     F3(2, 0x25)
 #define SLLX        (F3(2, 0x25)|(1<<12))
 #define SRA     F3(2, 0x27)
 #define SRAX        (F3(2, 0x27)|(1<<12))
 #define SRL     F3(2, 0x26)
 #define SRLX        (F3(2, 0x26)|(1<<12))
 #define JMPL        F3(2, 0x38)
 #define SAVE        F3(2, 0x3c)
 #define RESTORE     F3(2, 0x3d)
 #define CALL        F1(1)
 #define BR      F2(0, 0x01)
 #define RD_Y        F3(2, 0x28)
 #define WR_Y        F3(2, 0x30)
 
 #define LD32        F3(3, 0x00)
 #define LD8     F3(3, 0x01)
 #define LD16        F3(3, 0x02)
 #define LD64        F3(3, 0x0b)
 #define LD64A       F3(3, 0x1b)
 #define ST8     F3(3, 0x05)
 #define ST16        F3(3, 0x06)
 #define ST32        F3(3, 0x04)
 #define ST64        F3(3, 0x0e)
 
 #define CAS     F3(3, 0x3c)
 #define CASX        F3(3, 0x3e)
 
 #define LDPTR       LD64
 #define BASE_STACKFRAME 176
 
 #define LD32I       (LD32 | IMMED)
 #define LD8I        (LD8 | IMMED)
 #define LD16I       (LD16 | IMMED)
 #define LD64I       (LD64 | IMMED)
 #define LDPTRI      (LDPTR | IMMED)
 #define ST32I       (ST32 | IMMED)
 
 struct jit_ctx {
     struct bpf_prog     *prog;
     unsigned int        *offset;
     int         idx;
     int         epilogue_offset;
     bool            tmp_1_used;
     bool            tmp_2_used;
     bool            tmp_3_used;
     bool            saw_frame_pointer;
     bool            saw_call;
     bool            saw_tail_call;
     u32         *image;
 };
 
 #define TMP_REG_1   (MAX_BPF_JIT_REG + 0)
 #define TMP_REG_2   (MAX_BPF_JIT_REG + 1)
 #define TMP_REG_3   (MAX_BPF_JIT_REG + 2)
 
 /* Map BPF registers to SPARC registers */
 static const int bpf2sparc[] = {
     /* return value from in-kernel function, and exit value from eBPF */
     [BPF_REG_0] = O5,
 
     /* arguments from eBPF program to in-kernel function */
     [BPF_REG_1] = O0,
     [BPF_REG_2] = O1,
     [BPF_REG_3] = O2,
     [BPF_REG_4] = O3,
     [BPF_REG_5] = O4,
 
     /* callee saved registers that in-kernel function will preserve */
     [BPF_REG_6] = L0,
     [BPF_REG_7] = L1,
     [BPF_REG_8] = L2,
     [BPF_REG_9] = L3,
 
     /* read-only frame pointer to access stack */
     [BPF_REG_FP] = L6,
 
     [BPF_REG_AX] = G7,
 
     /* temporary register for BPF JIT */
     [TMP_REG_1] = G1,
     [TMP_REG_2] = G2,
     [TMP_REG_3] = G3,
 };
 
 static void emit(const u32 insn, struct jit_ctx *ctx)
 {
     if (ctx->image != NULL)
         ctx->image[ctx->idx] = insn;
 
     ctx->idx++;
 }
 
 static void emit_call(u32 *func, struct jit_ctx *ctx)
 {
     if (ctx->image != NULL) {
         void *here = &ctx->image[ctx->idx];
         unsigned int off;
 
         off = (void *)func - here;
         ctx->image[ctx->idx] = CALL | ((off >> 2) & 0x3fffffff);
     }
     ctx->idx++;
 }
 
 static void emit_nop(struct jit_ctx *ctx)
 {
     emit(SETHI(0, G0), ctx);
 }
 
 static void emit_reg_move(u32 from, u32 to, struct jit_ctx *ctx)
 {
     emit(OR | RS1(G0) | RS2(from) | RD(to), ctx);
 }
 
 /* Emit 32-bit constant, zero extended. */
 static void emit_set_const(s32 K, u32 reg, struct jit_ctx *ctx)
 {
     emit(SETHI(K, reg), ctx);
     emit(OR_LO(K, reg), ctx);
 }
 
 /* Emit 32-bit constant, sign extended. */
 static void emit_set_const_sext(s32 K, u32 reg, struct jit_ctx *ctx)
 {
     if (K >= 0) {
         emit(SETHI(K, reg), ctx);
         emit(OR_LO(K, reg), ctx);
     } else {
         u32 hbits = ~(u32) K;
         u32 lbits = -0x400 | (u32) K;
 
         emit(SETHI(hbits, reg), ctx);
         emit(XOR | IMMED | RS1(reg) | S13(lbits) | RD(reg), ctx);
     }
 }
 
 static void emit_alu(u32 opcode, u32 src, u32 dst, struct jit_ctx *ctx)
 {
     emit(opcode | RS1(dst) | RS2(src) | RD(dst), ctx);
 }
 
 static void emit_alu3(u32 opcode, u32 a, u32 b, u32 c, struct jit_ctx *ctx)
 {
     emit(opcode | RS1(a) | RS2(b) | RD(c), ctx);
 }
 
 static void emit_alu_K(unsigned int opcode, unsigned int dst, unsigned int imm,
                struct jit_ctx *ctx)
 {
     bool small_immed = is_simm13(imm);
     unsigned int insn = opcode;
 
     insn |= RS1(dst) | RD(dst);
     if (small_immed) {
         emit(insn | IMMED | S13(imm), ctx);
     } else {
         unsigned int tmp = bpf2sparc[TMP_REG_1];
 
         ctx->tmp_1_used = true;
 
         emit_set_const_sext(imm, tmp, ctx);
         emit(insn | RS2(tmp), ctx);
     }
 }
 
 static void emit_alu3_K(unsigned int opcode, unsigned int src, unsigned int imm,
             unsigned int dst, struct jit_ctx *ctx)
 {
     bool small_immed = is_simm13(imm);
     unsigned int insn = opcode;
 
     insn |= RS1(src) | RD(dst);
     if (small_immed) {
         emit(insn | IMMED | S13(imm), ctx);
     } else {
         unsigned int tmp = bpf2sparc[TMP_REG_1];
 
         ctx->tmp_1_used = true;
 
         emit_set_const_sext(imm, tmp, ctx);
         emit(insn | RS2(tmp), ctx);
     }
 }
 
 static void emit_loadimm32(s32 K, unsigned int dest, struct jit_ctx *ctx)
 {
     if (K >= 0 && is_simm13(K)) {
         /* or %g0, K, DEST */
         emit(OR | IMMED | RS1(G0) | S13(K) | RD(dest), ctx);
     } else {
         emit_set_const(K, dest, ctx);
     }
 }
 
 static void emit_loadimm(s32 K, unsigned int dest, struct jit_ctx *ctx)
 {
     if (is_simm13(K)) {
         /* or %g0, K, DEST */
         emit(OR | IMMED | RS1(G0) | S13(K) | RD(dest), ctx);
     } else {
         emit_set_const(K, dest, ctx);
     }
 }
 
 static void emit_loadimm_sext(s32 K, unsigned int dest, struct jit_ctx *ctx)
 {
     if (is_simm13(K)) {
         /* or %g0, K, DEST */
         emit(OR | IMMED | RS1(G0) | S13(K) | RD(dest), ctx);
     } else {
         emit_set_const_sext(K, dest, ctx);
     }
 }
 
 static void analyze_64bit_constant(u32 high_bits, u32 low_bits,
                    int *hbsp, int *lbsp, int *abbasp)
 {
     int lowest_bit_set, highest_bit_set, all_bits_between_are_set;
     int i;
 
     lowest_bit_set = highest_bit_set = -1;
     i = 0;
     do {
         if ((lowest_bit_set == -1) && ((low_bits >> i) & 1))
             lowest_bit_set = i;
         if ((highest_bit_set == -1) && ((high_bits >> (32 - i - 1)) & 1))
             highest_bit_set = (64 - i - 1);
     }  while (++i < 32 && (highest_bit_set == -1 ||
                    lowest_bit_set == -1));
     if (i == 32) {
         i = 0;
         do {
             if (lowest_bit_set == -1 && ((high_bits >> i) & 1))
                 lowest_bit_set = i + 32;
             if (highest_bit_set == -1 &&
                 ((low_bits >> (32 - i - 1)) & 1))
                 highest_bit_set = 32 - i - 1;
         } while (++i < 32 && (highest_bit_set == -1 ||
                       lowest_bit_set == -1));
     }
 
     all_bits_between_are_set = 1;
     for (i = lowest_bit_set; i <= highest_bit_set; i++) {
         if (i < 32) {
             if ((low_bits & (1 << i)) != 0)
                 continue;
         } else {
             if ((high_bits & (1 << (i - 32))) != 0)
                 continue;
         }
         all_bits_between_are_set = 0;
         break;
     }
     *hbsp = highest_bit_set;
     *lbsp = lowest_bit_set;
     *abbasp = all_bits_between_are_set;
 }
 
 static unsigned long create_simple_focus_bits(unsigned long high_bits,
                           unsigned long low_bits,
                           int lowest_bit_set, int shift)
 {
     long hi, lo;
 
     if (lowest_bit_set < 32) {
         lo = (low_bits >> lowest_bit_set) << shift;
         hi = ((high_bits << (32 - lowest_bit_set)) << shift);
     } else {
         lo = 0;
         hi = ((high_bits >> (lowest_bit_set - 32)) << shift);
     }
     return hi | lo;
 }
 
 static bool const64_is_2insns(unsigned long high_bits,
                   unsigned long low_bits)
 {
     int highest_bit_set, lowest_bit_set, all_bits_between_are_set;
 
     if (high_bits == 0 || high_bits == 0xffffffff)
         return true;
 
     analyze_64bit_constant(high_bits, low_bits,
                    &highest_bit_set, &lowest_bit_set,
                    &all_bits_between_are_set);
 
     if ((highest_bit_set == 63 || lowest_bit_set == 0) &&
         all_bits_between_are_set != 0)
         return true;
 
     if (highest_bit_set - lowest_bit_set < 21)
         return true;
 
     return false;
 }
 
 static void sparc_emit_set_const64_quick2(unsigned long high_bits,
                       unsigned long low_imm,
                       unsigned int dest,
                       int shift_count, struct jit_ctx *ctx)
 {
     emit_loadimm32(high_bits, dest, ctx);
 
     /* Now shift it up into place.  */
     emit_alu_K(SLLX, dest, shift_count, ctx);
 
     /* If there is a low immediate part piece, finish up by
      * putting that in as well.
      */
     if (low_imm != 0)
         emit(OR | IMMED | RS1(dest) | S13(low_imm) | RD(dest), ctx);
 }
 
 static void emit_loadimm64(u64 K, unsigned int dest, struct jit_ctx *ctx)
 {
     int all_bits_between_are_set, lowest_bit_set, highest_bit_set;
     unsigned int tmp = bpf2sparc[TMP_REG_1];
     u32 low_bits = (K & 0xffffffff);
     u32 high_bits = (K >> 32);
 
     /* These two tests also take care of all of the one
      * instruction cases.
      */
     if (high_bits == 0xffffffff && (low_bits & 0x80000000))
         return emit_loadimm_sext(K, dest, ctx);
     if (high_bits == 0x00000000)
         return emit_loadimm32(K, dest, ctx);
 
     analyze_64bit_constant(high_bits, low_bits, &highest_bit_set,
                    &lowest_bit_set, &all_bits_between_are_set);
 
     /* 1) mov   -1, %reg
      *    sllx  %reg, shift, %reg
      * 2) mov   -1, %reg
      *    srlx  %reg, shift, %reg
      * 3) mov   some_small_const, %reg
      *    sllx  %reg, shift, %reg
      */
     if (((highest_bit_set == 63 || lowest_bit_set == 0) &&
          all_bits_between_are_set != 0) ||
         ((highest_bit_set - lowest_bit_set) < 12)) {
         int shift = lowest_bit_set;
         long the_const = -1;
 
         if ((highest_bit_set != 63 && lowest_bit_set != 0) ||
             all_bits_between_are_set == 0) {
             the_const =
                 create_simple_focus_bits(high_bits, low_bits,
                              lowest_bit_set, 0);
         } else if (lowest_bit_set == 0)
             shift = -(63 - highest_bit_set);
 
         emit(OR | IMMED | RS1(G0) | S13(the_const) | RD(dest), ctx);
         if (shift > 0)
             emit_alu_K(SLLX, dest, shift, ctx);
         else if (shift < 0)
             emit_alu_K(SRLX, dest, -shift, ctx);
 
         return;
     }
 
     /* Now a range of 22 or less bits set somewhere.
      * 1) sethi %hi(focus_bits), %reg
      *    sllx  %reg, shift, %reg
      * 2) sethi %hi(focus_bits), %reg
      *    srlx  %reg, shift, %reg
      */
     if ((highest_bit_set - lowest_bit_set) < 21) {
         unsigned long focus_bits =
             create_simple_focus_bits(high_bits, low_bits,
                          lowest_bit_set, 10);
 
         emit(SETHI(focus_bits, dest), ctx);
 
         /* If lowest_bit_set == 10 then a sethi alone could
          * have done it.
          */
         if (lowest_bit_set < 10)
             emit_alu_K(SRLX, dest, 10 - lowest_bit_set, ctx);
         else if (lowest_bit_set > 10)
             emit_alu_K(SLLX, dest, lowest_bit_set - 10, ctx);
         return;
     }
 
     /* Ok, now 3 instruction sequences.  */
     if (low_bits == 0) {
         emit_loadimm32(high_bits, dest, ctx);
         emit_alu_K(SLLX, dest, 32, ctx);
         return;
     }
 
     /* We may be able to do something quick
      * when the constant is negated, so try that.
      */
     if (const64_is_2insns((~high_bits) & 0xffffffff,
                   (~low_bits) & 0xfffffc00)) {
         /* NOTE: The trailing bits get XOR'd so we need the
          * non-negated bits, not the negated ones.
          */
         unsigned long trailing_bits = low_bits & 0x3ff;
 
         if ((((~high_bits) & 0xffffffff) == 0 &&
              ((~low_bits) & 0x80000000) == 0) ||
             (((~high_bits) & 0xffffffff) == 0xffffffff &&
              ((~low_bits) & 0x80000000) != 0)) {
             unsigned long fast_int = (~low_bits & 0xffffffff);
 
             if ((is_sethi(fast_int) &&
                  (~high_bits & 0xffffffff) == 0)) {
                 emit(SETHI(fast_int, dest), ctx);
             } else if (is_simm13(fast_int)) {
                 emit(OR | IMMED | RS1(G0) | S13(fast_int) | RD(dest), ctx);
             } else {
                 emit_loadimm64(fast_int, dest, ctx);
             }
         } else {
             u64 n = ((~low_bits) & 0xfffffc00) |
                 (((unsigned long)((~high_bits) & 0xffffffff))<<32);
             emit_loadimm64(n, dest, ctx);
         }
 
         low_bits = -0x400 | trailing_bits;
 
         emit(XOR | IMMED | RS1(dest) | S13(low_bits) | RD(dest), ctx);
         return;
     }
 
     /* 1) sethi %hi(xxx), %reg
      *    or    %reg, %lo(xxx), %reg
      *    sllx  %reg, yyy, %reg
      */
     if ((highest_bit_set - lowest_bit_set) < 32) {
         unsigned long focus_bits =
             create_simple_focus_bits(high_bits, low_bits,
                          lowest_bit_set, 0);
 
         /* So what we know is that the set bits straddle the
          * middle of the 64-bit word.
          */
         sparc_emit_set_const64_quick2(focus_bits, 0, dest,
                           lowest_bit_set, ctx);
         return;
     }
 
     /* 1) sethi %hi(high_bits), %reg
      *    or    %reg, %lo(high_bits), %reg
      *    sllx  %reg, 32, %reg
      *    or    %reg, low_bits, %reg
      */
     if (is_simm13(low_bits) && ((int)low_bits > 0)) {
         sparc_emit_set_const64_quick2(high_bits, low_bits,
                           dest, 32, ctx);
         return;
     }
 
     /* Oh well, we tried... Do a full 64-bit decomposition.  */
     ctx->tmp_1_used = true;
 
     emit_loadimm32(high_bits, tmp, ctx);
     emit_loadimm32(low_bits, dest, ctx);
     emit_alu_K(SLLX, tmp, 32, ctx);
     emit(OR | RS1(dest) | RS2(tmp) | RD(dest), ctx);
 }
 
 static void emit_branch(unsigned int br_opc, unsigned int from_idx, unsigned int to_idx,
             struct jit_ctx *ctx)
 {
     unsigned int off = to_idx - from_idx;
 
     if (br_opc & XCC)
         emit(br_opc | WDISP19(off << 2), ctx);
     else
         emit(br_opc | WDISP22(off << 2), ctx);
 }
 
 static void emit_cbcond(unsigned int cb_opc, unsigned int from_idx, unsigned int to_idx,
             const u8 dst, const u8 src, struct jit_ctx *ctx)
 {
     unsigned int off = to_idx - from_idx;
 
     emit(cb_opc | WDISP10(off << 2) | RS1(dst) | RS2(src), ctx);
 }
 
 static void emit_cbcondi(unsigned int cb_opc, unsigned int from_idx, unsigned int to_idx,
              const u8 dst, s32 imm, struct jit_ctx *ctx)
 {
     unsigned int off = to_idx - from_idx;
 
     emit(cb_opc | IMMED | WDISP10(off << 2) | RS1(dst) | S5(imm), ctx);
 }
 
 #define emit_read_y(REG, CTX)   emit(RD_Y | RD(REG), CTX)
 #define emit_write_y(REG, CTX)  emit(WR_Y | IMMED | RS1(REG) | S13(0), CTX)
 
 #define emit_cmp(R1, R2, CTX)               \
     emit(SUBCC | RS1(R1) | RS2(R2) | RD(G0), CTX)
 
 #define emit_cmpi(R1, IMM, CTX)             \
     emit(SUBCC | IMMED | RS1(R1) | S13(IMM) | RD(G0), CTX)
 
 #define emit_btst(R1, R2, CTX)              \
     emit(ANDCC | RS1(R1) | RS2(R2) | RD(G0), CTX)
 
 #define emit_btsti(R1, IMM, CTX)            \
     emit(ANDCC | IMMED | RS1(R1) | S13(IMM) | RD(G0), CTX)
 
 static int emit_compare_and_branch(const u8 code, const u8 dst, u8 src,
                    const s32 imm, bool is_imm, int branch_dst,
                    struct jit_ctx *ctx)
 {
     bool use_cbcond = (sparc64_elf_hwcap & AV_SPARC_CBCOND) != 0;
     const u8 tmp = bpf2sparc[TMP_REG_1];
 
     branch_dst = ctx->offset[branch_dst];
 
     if (!is_simm10(branch_dst - ctx->idx) ||
         BPF_OP(code) == BPF_JSET)
         use_cbcond = false;
 
     if (is_imm) {
         bool fits = true;
 
         if (use_cbcond) {
             if (!is_simm5(imm))
                 fits = false;
         } else if (!is_simm13(imm)) {
             fits = false;
         }
         if (!fits) {
             ctx->tmp_1_used = true;
             emit_loadimm_sext(imm, tmp, ctx);
             src = tmp;
             is_imm = false;
         }
     }
 
     if (!use_cbcond) {
         u32 br_opcode;
 
         if (BPF_OP(code) == BPF_JSET) {
             if (is_imm)
                 emit_btsti(dst, imm, ctx);
             else
                 emit_btst(dst, src, ctx);
         } else {
             if (is_imm)
                 emit_cmpi(dst, imm, ctx);
             else
                 emit_cmp(dst, src, ctx);
         }
         switch (BPF_OP(code)) {
         case BPF_JEQ:
             br_opcode = BE;
             break;
         case BPF_JGT:
             br_opcode = BGU;
             break;
         case BPF_JLT:
             br_opcode = BLU;
             break;
         case BPF_JGE:
             br_opcode = BGEU;
             break;
         case BPF_JLE:
             br_opcode = BLEU;
             break;
         case BPF_JSET:
         case BPF_JNE:
             br_opcode = BNE;
             break;
         case BPF_JSGT:
             br_opcode = BG;
             break;
         case BPF_JSLT:
             br_opcode = BL;
             break;
         case BPF_JSGE:
             br_opcode = BGE;
             break;
         case BPF_JSLE:
             br_opcode = BLE;
             break;
         default:
             /* Make sure we dont leak kernel information to the
              * user.
              */
             return -EFAULT;
         }
         emit_branch(br_opcode, ctx->idx, branch_dst, ctx);
         emit_nop(ctx);
     } else {
         u32 cbcond_opcode;
 
         switch (BPF_OP(code)) {
         case BPF_JEQ:
             cbcond_opcode = CBCONDE;
             break;
         case BPF_JGT:
             cbcond_opcode = CBCONDGU;
             break;
         case BPF_JLT:
             cbcond_opcode = CBCONDLU;
             break;
         case BPF_JGE:
             cbcond_opcode = CBCONDGEU;
             break;
         case BPF_JLE:
             cbcond_opcode = CBCONDLEU;
             break;
         case BPF_JNE:
             cbcond_opcode = CBCONDNE;
             break;
         case BPF_JSGT:
             cbcond_opcode = CBCONDG;
             break;
         case BPF_JSLT:
             cbcond_opcode = CBCONDL;
             break;
         case BPF_JSGE:
             cbcond_opcode = CBCONDGE;
             break;
         case BPF_JSLE:
             cbcond_opcode = CBCONDLE;
             break;
         default:
             /* Make sure we dont leak kernel information to the
              * user.
              */
             return -EFAULT;
         }
         cbcond_opcode |= CBCOND_OP;
         if (is_imm)
             emit_cbcondi(cbcond_opcode, ctx->idx, branch_dst,
                      dst, imm, ctx);
         else
             emit_cbcond(cbcond_opcode, ctx->idx, branch_dst,
                     dst, src, ctx);
     }
     return 0;
 }
 
 /* Just skip the save instruction and the ctx register move.  */
 #define BPF_TAILCALL_PROLOGUE_SKIP  32
 #define BPF_TAILCALL_CNT_SP_OFF     (STACK_BIAS + 128)
 
 static void build_prologue(struct jit_ctx *ctx)
 {
     s32 stack_needed = BASE_STACKFRAME;
 
     if (ctx->saw_frame_pointer || ctx->saw_tail_call) {
         struct bpf_prog *prog = ctx->prog;
         u32 stack_depth;
 
         stack_depth = prog->aux->stack_depth;
         stack_needed += round_up(stack_depth, 16);
     }
 
     if (ctx->saw_tail_call)
         stack_needed += 8;
 
     /* save %sp, -176, %sp */
     emit(SAVE | IMMED | RS1(SP) | S13(-stack_needed) | RD(SP), ctx);
 
     /* tail_call_cnt = 0 */
     if (ctx->saw_tail_call) {
         u32 off = BPF_TAILCALL_CNT_SP_OFF;
 
         emit(ST32 | IMMED | RS1(SP) | S13(off) | RD(G0), ctx);
     } else {
         emit_nop(ctx);
     }
     if (ctx->saw_frame_pointer) {
         const u8 vfp = bpf2sparc[BPF_REG_FP];
 
         emit(ADD | IMMED | RS1(FP) | S13(STACK_BIAS) | RD(vfp), ctx);
     } else {
         emit_nop(ctx);
     }
 
     emit_reg_move(I0, O0, ctx);
     emit_reg_move(I1, O1, ctx);
     emit_reg_move(I2, O2, ctx);
     emit_reg_move(I3, O3, ctx);
     emit_reg_move(I4, O4, ctx);
     /* If you add anything here, adjust BPF_TAILCALL_PROLOGUE_SKIP above. */
 }
 
 static void build_epilogue(struct jit_ctx *ctx)
 {
     ctx->epilogue_offset = ctx->idx;
 
     /* ret (jmpl %i7 + 8, %g0) */
     emit(JMPL | IMMED | RS1(I7) | S13(8) | RD(G0), ctx);
 
     /* restore %i5, %g0, %o0 */
     emit(RESTORE | RS1(bpf2sparc[BPF_REG_0]) | RS2(G0) | RD(O0), ctx);
 }
 
 static void emit_tail_call(struct jit_ctx *ctx)
 {
     const u8 bpf_array = bpf2sparc[BPF_REG_2];
     const u8 bpf_index = bpf2sparc[BPF_REG_3];
     const u8 tmp = bpf2sparc[TMP_REG_1];
     u32 off;
 
     ctx->saw_tail_call = true;
 
     off = offsetof(struct bpf_array, map.max_entries);
     emit(LD32 | IMMED | RS1(bpf_array) | S13(off) | RD(tmp), ctx);
     emit_cmp(bpf_index, tmp, ctx);
 #define OFFSET1 17
     emit_branch(BGEU, ctx->idx, ctx->idx + OFFSET1, ctx);
     emit_nop(ctx);
 
     off = BPF_TAILCALL_CNT_SP_OFF;
     emit(LD32 | IMMED | RS1(SP) | S13(off) | RD(tmp), ctx);
     emit_cmpi(tmp, MAX_TAIL_CALL_CNT, ctx);
 #define OFFSET2 13
     emit_branch(BGEU, ctx->idx, ctx->idx + OFFSET2, ctx);
     emit_nop(ctx);
 
     emit_alu_K(ADD, tmp, 1, ctx);
     off = BPF_TAILCALL_CNT_SP_OFF;
     emit(ST32 | IMMED | RS1(SP) | S13(off) | RD(tmp), ctx);
 
     emit_alu3_K(SLL, bpf_index, 3, tmp, ctx);
     emit_alu(ADD, bpf_array, tmp, ctx);
     off = offsetof(struct bpf_array, ptrs);
     emit(LD64 | IMMED | RS1(tmp) | S13(off) | RD(tmp), ctx);
 
     emit_cmpi(tmp, 0, ctx);
 #define OFFSET3 5
     emit_branch(BE, ctx->idx, ctx->idx + OFFSET3, ctx);
     emit_nop(ctx);
 
     off = offsetof(struct bpf_prog, bpf_func);
     emit(LD64 | IMMED | RS1(tmp) | S13(off) | RD(tmp), ctx);
 
     off = BPF_TAILCALL_PROLOGUE_SKIP;
     emit(JMPL | IMMED | RS1(tmp) | S13(off) | RD(G0), ctx);
     emit_nop(ctx);
 }
 
 static int build_insn(const struct bpf_insn *insn, struct jit_ctx *ctx)
 {
     const u8 code = insn->code;
     const u8 dst = bpf2sparc[insn->dst_reg];
     const u8 src = bpf2sparc[insn->src_reg];
     const int i = insn - ctx->prog->insnsi;
     const s16 off = insn->off;
     const s32 imm = insn->imm;
 
     if (insn->src_reg == BPF_REG_FP)
         ctx->saw_frame_pointer = true;
 
     switch (code) {
     /* dst = src */
     case BPF_ALU | BPF_MOV | BPF_X:
         emit_alu3_K(SRL, src, 0, dst, ctx);
         if (insn_is_zext(&insn[1]))
             return 1;
         break;
     case BPF_ALU64 | BPF_MOV | BPF_X:
         emit_reg_move(src, dst, ctx);
         break;
     /* dst = dst OP src */
     case BPF_ALU | BPF_ADD | BPF_X:
     case BPF_ALU64 | BPF_ADD | BPF_X:
         emit_alu(ADD, src, dst, ctx);
         goto do_alu32_trunc;
     case BPF_ALU | BPF_SUB | BPF_X:
     case BPF_ALU64 | BPF_SUB | BPF_X:
         emit_alu(SUB, src, dst, ctx);
         goto do_alu32_trunc;
     case BPF_ALU | BPF_AND | BPF_X:
     case BPF_ALU64 | BPF_AND | BPF_X:
         emit_alu(AND, src, dst, ctx);
         goto do_alu32_trunc;
     case BPF_ALU | BPF_OR | BPF_X:
     case BPF_ALU64 | BPF_OR | BPF_X:
         emit_alu(OR, src, dst, ctx);
         goto do_alu32_trunc;
     case BPF_ALU | BPF_XOR | BPF_X:
     case BPF_ALU64 | BPF_XOR | BPF_X:
         emit_alu(XOR, src, dst, ctx);
         goto do_alu32_trunc;
     case BPF_ALU | BPF_MUL | BPF_X:
         emit_alu(MUL, src, dst, ctx);
         goto do_alu32_trunc;
     case BPF_ALU64 | BPF_MUL | BPF_X:
         emit_alu(MULX, src, dst, ctx);
         break;
     case BPF_ALU | BPF_DIV | BPF_X:
         emit_write_y(G0, ctx);
         emit_alu(DIV, src, dst, ctx);
         if (insn_is_zext(&insn[1]))
             return 1;
         break;
     case BPF_ALU64 | BPF_DIV | BPF_X:
         emit_alu(UDIVX, src, dst, ctx);
         break;
     case BPF_ALU | BPF_MOD | BPF_X: {
         const u8 tmp = bpf2sparc[TMP_REG_1];
 
         ctx->tmp_1_used = true;
 
         emit_write_y(G0, ctx);
         emit_alu3(DIV, dst, src, tmp, ctx);
         emit_alu3(MULX, tmp, src, tmp, ctx);
         emit_alu3(SUB, dst, tmp, dst, ctx);
         goto do_alu32_trunc;
     }
     case BPF_ALU64 | BPF_MOD | BPF_X: {
         const u8 tmp = bpf2sparc[TMP_REG_1];
 
         ctx->tmp_1_used = true;
 
         emit_alu3(UDIVX, dst, src, tmp, ctx);
         emit_alu3(MULX, tmp, src, tmp, ctx);
         emit_alu3(SUB, dst, tmp, dst, ctx);
         break;
     }
     case BPF_ALU | BPF_LSH | BPF_X:
         emit_alu(SLL, src, dst, ctx);
         goto do_alu32_trunc;
     case BPF_ALU64 | BPF_LSH | BPF_X:
         emit_alu(SLLX, src, dst, ctx);
         break;
     case BPF_ALU | BPF_RSH | BPF_X:
         emit_alu(SRL, src, dst, ctx);
         if (insn_is_zext(&insn[1]))
             return 1;
         break;
     case BPF_ALU64 | BPF_RSH | BPF_X:
         emit_alu(SRLX, src, dst, ctx);
         break;
     case BPF_ALU | BPF_ARSH | BPF_X:
         emit_alu(SRA, src, dst, ctx);
         goto do_alu32_trunc;
     case BPF_ALU64 | BPF_ARSH | BPF_X:
         emit_alu(SRAX, src, dst, ctx);
         break;
 
     /* dst = -dst */
     case BPF_ALU | BPF_NEG:
     case BPF_ALU64 | BPF_NEG:
         emit(SUB | RS1(0) | RS2(dst) | RD(dst), ctx);
         goto do_alu32_trunc;
 
     case BPF_ALU | BPF_END | BPF_FROM_BE:
         switch (imm) {
         case 16:
             emit_alu_K(SLL, dst, 16, ctx);
             emit_alu_K(SRL, dst, 16, ctx);
             if (insn_is_zext(&insn[1]))
                 return 1;
             break;
         case 32:
             if (!ctx->prog->aux->verifier_zext)
                 emit_alu_K(SRL, dst, 0, ctx);
             break;
         case 64:
             /* nop */
             break;
 
         }
         break;
 
     /* dst = BSWAP##imm(dst) */
     case BPF_ALU | BPF_END | BPF_FROM_LE: {
         const u8 tmp = bpf2sparc[TMP_REG_1];
         const u8 tmp2 = bpf2sparc[TMP_REG_2];
 
         ctx->tmp_1_used = true;
         switch (imm) {
         case 16:
             emit_alu3_K(AND, dst, 0xff, tmp, ctx);
             emit_alu3_K(SRL, dst, 8, dst, ctx);
             emit_alu3_K(AND, dst, 0xff, dst, ctx);
             emit_alu3_K(SLL, tmp, 8, tmp, ctx);
             emit_alu(OR, tmp, dst, ctx);
             if (insn_is_zext(&insn[1]))
                 return 1;
             break;
 
         case 32:
             ctx->tmp_2_used = true;
             emit_alu3_K(SRL, dst, 24, tmp, ctx);    /* tmp  = dst >> 24 */
             emit_alu3_K(SRL, dst, 16, tmp2, ctx);   /* tmp2 = dst >> 16 */
             emit_alu3_K(AND, tmp2, 0xff, tmp2, ctx);/* tmp2 = tmp2 & 0xff */
             emit_alu3_K(SLL, tmp2, 8, tmp2, ctx);   /* tmp2 = tmp2 << 8 */
             emit_alu(OR, tmp2, tmp, ctx);       /* tmp  = tmp | tmp2 */
             emit_alu3_K(SRL, dst, 8, tmp2, ctx);    /* tmp2 = dst >> 8 */
             emit_alu3_K(AND, tmp2, 0xff, tmp2, ctx);/* tmp2 = tmp2 & 0xff */
             emit_alu3_K(SLL, tmp2, 16, tmp2, ctx);  /* tmp2 = tmp2 << 16 */
             emit_alu(OR, tmp2, tmp, ctx);       /* tmp  = tmp | tmp2 */
             emit_alu3_K(AND, dst, 0xff, dst, ctx);  /* dst  = dst & 0xff */
             emit_alu3_K(SLL, dst, 24, dst, ctx);    /* dst  = dst << 24 */
             emit_alu(OR, tmp, dst, ctx);        /* dst  = dst | tmp */
             if (insn_is_zext(&insn[1]))
                 return 1;
             break;
 
         case 64:
             emit_alu3_K(ADD, SP, STACK_BIAS + 128, tmp, ctx);
             emit(ST64 | RS1(tmp) | RS2(G0) | RD(dst), ctx);
             emit(LD64A | ASI(ASI_PL) | RS1(tmp) | RS2(G0) | RD(dst), ctx);
             break;
         }
         break;
     }
     /* dst = imm */
     case BPF_ALU | BPF_MOV | BPF_K:
         emit_loadimm32(imm, dst, ctx);
         if (insn_is_zext(&insn[1]))
             return 1;
         break;
     case BPF_ALU64 | BPF_MOV | BPF_K:
         emit_loadimm_sext(imm, dst, ctx);
         break;
     /* dst = dst OP imm */
     case BPF_ALU | BPF_ADD | BPF_K:
     case BPF_ALU64 | BPF_ADD | BPF_K:
         emit_alu_K(ADD, dst, imm, ctx);
         goto do_alu32_trunc;
     case BPF_ALU | BPF_SUB | BPF_K:
     case BPF_ALU64 | BPF_SUB | BPF_K:
         emit_alu_K(SUB, dst, imm, ctx);
         goto do_alu32_trunc;
     case BPF_ALU | BPF_AND | BPF_K:
     case BPF_ALU64 | BPF_AND | BPF_K:
         emit_alu_K(AND, dst, imm, ctx);
         goto do_alu32_trunc;
     case BPF_ALU | BPF_OR | BPF_K:
     case BPF_ALU64 | BPF_OR | BPF_K:
         emit_alu_K(OR, dst, imm, ctx);
         goto do_alu32_trunc;
     case BPF_ALU | BPF_XOR | BPF_K:
     case BPF_ALU64 | BPF_XOR | BPF_K:
         emit_alu_K(XOR, dst, imm, ctx);
         goto do_alu32_trunc;
     case BPF_ALU | BPF_MUL | BPF_K:
         emit_alu_K(MUL, dst, imm, ctx);
         goto do_alu32_trunc;
     case BPF_ALU64 | BPF_MUL | BPF_K:
         emit_alu_K(MULX, dst, imm, ctx);
         break;
     case BPF_ALU | BPF_DIV | BPF_K:
         if (imm == 0)
             return -EINVAL;
 
         emit_write_y(G0, ctx);
         emit_alu_K(DIV, dst, imm, ctx);
         goto do_alu32_trunc;
     case BPF_ALU64 | BPF_DIV | BPF_K:
         if (imm == 0)
             return -EINVAL;
 
         emit_alu_K(UDIVX, dst, imm, ctx);
         break;
     case BPF_ALU64 | BPF_MOD | BPF_K:
     case BPF_ALU | BPF_MOD | BPF_K: {
         const u8 tmp = bpf2sparc[TMP_REG_2];
         unsigned int div;
 
         if (imm == 0)
             return -EINVAL;
 
         div = (BPF_CLASS(code) == BPF_ALU64) ? UDIVX : DIV;
 
         ctx->tmp_2_used = true;
 
         if (BPF_CLASS(code) != BPF_ALU64)
             emit_write_y(G0, ctx);
         if (is_simm13(imm)) {
             emit(div | IMMED | RS1(dst) | S13(imm) | RD(tmp), ctx);
             emit(MULX | IMMED | RS1(tmp) | S13(imm) | RD(tmp), ctx);
             emit(SUB | RS1(dst) | RS2(tmp) | RD(dst), ctx);
         } else {
             const u8 tmp1 = bpf2sparc[TMP_REG_1];
 
             ctx->tmp_1_used = true;
 
             emit_set_const_sext(imm, tmp1, ctx);
             emit(div | RS1(dst) | RS2(tmp1) | RD(tmp), ctx);
             emit(MULX | RS1(tmp) | RS2(tmp1) | RD(tmp), ctx);
             emit(SUB | RS1(dst) | RS2(tmp) | RD(dst), ctx);
         }
         goto do_alu32_trunc;
     }
     case BPF_ALU | BPF_LSH | BPF_K:
         emit_alu_K(SLL, dst, imm, ctx);
         goto do_alu32_trunc;
     case BPF_ALU64 | BPF_LSH | BPF_K:
         emit_alu_K(SLLX, dst, imm, ctx);
         break;
     case BPF_ALU | BPF_RSH | BPF_K:
         emit_alu_K(SRL, dst, imm, ctx);
         if (insn_is_zext(&insn[1]))
             return 1;
         break;
     case BPF_ALU64 | BPF_RSH | BPF_K:
         emit_alu_K(SRLX, dst, imm, ctx);
         break;
     case BPF_ALU | BPF_ARSH | BPF_K:
         emit_alu_K(SRA, dst, imm, ctx);
         goto do_alu32_trunc;
     case BPF_ALU64 | BPF_ARSH | BPF_K:
         emit_alu_K(SRAX, dst, imm, ctx);
         break;
 
     do_alu32_trunc:
         if (BPF_CLASS(code) == BPF_ALU &&
             !ctx->prog->aux->verifier_zext)
             emit_alu_K(SRL, dst, 0, ctx);
         break;
 
     /* JUMP off */
     case BPF_JMP | BPF_JA:
         emit_branch(BA, ctx->idx, ctx->offset[i + off], ctx);
         emit_nop(ctx);
         break;
     /* IF (dst COND src) JUMP off */
     case BPF_JMP | BPF_JEQ | BPF_X:
     case BPF_JMP | BPF_JGT | BPF_X:
     case BPF_JMP | BPF_JLT | BPF_X:
     case BPF_JMP | BPF_JGE | BPF_X:
     case BPF_JMP | BPF_JLE | BPF_X:
     case BPF_JMP | BPF_JNE | BPF_X:
     case BPF_JMP | BPF_JSGT | BPF_X:
     case BPF_JMP | BPF_JSLT | BPF_X:
     case BPF_JMP | BPF_JSGE | BPF_X:
     case BPF_JMP | BPF_JSLE | BPF_X:
     case BPF_JMP | BPF_JSET | BPF_X: {
         int err;
 
         err = emit_compare_and_branch(code, dst, src, 0, false, i + off, ctx);
         if (err)
             return err;
         break;
     }
     /* IF (dst COND imm) JUMP off */
     case BPF_JMP | BPF_JEQ | BPF_K:
     case BPF_JMP | BPF_JGT | BPF_K:
     case BPF_JMP | BPF_JLT | BPF_K:
     case BPF_JMP | BPF_JGE | BPF_K:
     case BPF_JMP | BPF_JLE | BPF_K:
     case BPF_JMP | BPF_JNE | BPF_K:
     case BPF_JMP | BPF_JSGT | BPF_K:
     case BPF_JMP | BPF_JSLT | BPF_K:
     case BPF_JMP | BPF_JSGE | BPF_K:
     case BPF_JMP | BPF_JSLE | BPF_K:
     case BPF_JMP | BPF_JSET | BPF_K: {
         int err;
 
         err = emit_compare_and_branch(code, dst, 0, imm, true, i + off, ctx);
         if (err)
             return err;
         break;
     }
 
     /* function call */
     case BPF_JMP | BPF_CALL:
     {
         u8 *func = ((u8 *)__bpf_call_base) + imm;
 
         ctx->saw_call = true;
 
         emit_call((u32 *)func, ctx);
         emit_nop(ctx);
 
         emit_reg_move(O0, bpf2sparc[BPF_REG_0], ctx);
         break;
     }
 
     /* tail call */
     case BPF_JMP | BPF_TAIL_CALL:
         emit_tail_call(ctx);
         break;
 
     /* function return */
     case BPF_JMP | BPF_EXIT:
         /* Optimization: when last instruction is EXIT,
            simply fallthrough to epilogue. */
         if (i == ctx->prog->len - 1)
             break;
         emit_branch(BA, ctx->idx, ctx->epilogue_offset, ctx);
         emit_nop(ctx);
         break;
 
     /* dst = imm64 */
     case BPF_LD | BPF_IMM | BPF_DW:
     {
         const struct bpf_insn insn1 = insn[1];
         u64 imm64;
 
         imm64 = (u64)insn1.imm << 32 | (u32)imm;
         emit_loadimm64(imm64, dst, ctx);
 
         return 1;
     }
 
     /* LDX: dst = *(size *)(src + off) */
     case BPF_LDX | BPF_MEM | BPF_W:
     case BPF_LDX | BPF_MEM | BPF_H:
     case BPF_LDX | BPF_MEM | BPF_B:
     case BPF_LDX | BPF_MEM | BPF_DW: {
         const u8 tmp = bpf2sparc[TMP_REG_1];
         u32 opcode = 0, rs2;
 
         ctx->tmp_1_used = true;
         switch (BPF_SIZE(code)) {
         case BPF_W:
             opcode = LD32;
             break;
         case BPF_H:
             opcode = LD16;
             break;
         case BPF_B:
             opcode = LD8;
             break;
         case BPF_DW:
             opcode = LD64;
             break;
         }
 
         if (is_simm13(off)) {
             opcode |= IMMED;
             rs2 = S13(off);
         } else {
             emit_loadimm(off, tmp, ctx);
             rs2 = RS2(tmp);
         }
         emit(opcode | RS1(src) | rs2 | RD(dst), ctx);
         if (opcode != LD64 && insn_is_zext(&insn[1]))
             return 1;
         break;
     }
     /* speculation barrier */
     case BPF_ST | BPF_NOSPEC:
         break;
     /* ST: *(size *)(dst + off) = imm */
     case BPF_ST | BPF_MEM | BPF_W:
     case BPF_ST | BPF_MEM | BPF_H:
     case BPF_ST | BPF_MEM | BPF_B:
     case BPF_ST | BPF_MEM | BPF_DW: {
         const u8 tmp = bpf2sparc[TMP_REG_1];
         const u8 tmp2 = bpf2sparc[TMP_REG_2];
         u32 opcode = 0, rs2;
 
         if (insn->dst_reg == BPF_REG_FP)
             ctx->saw_frame_pointer = true;
 
         ctx->tmp_2_used = true;
         emit_loadimm(imm, tmp2, ctx);
 
         switch (BPF_SIZE(code)) {
         case BPF_W:
             opcode = ST32;
             break;
         case BPF_H:
             opcode = ST16;
             break;
         case BPF_B:
             opcode = ST8;
             break;
         case BPF_DW:
             opcode = ST64;
             break;
         }
 
         if (is_simm13(off)) {
             opcode |= IMMED;
             rs2 = S13(off);
         } else {
             ctx->tmp_1_used = true;
             emit_loadimm(off, tmp, ctx);
             rs2 = RS2(tmp);
         }
         emit(opcode | RS1(dst) | rs2 | RD(tmp2), ctx);
         break;
     }
 
     /* STX: *(size *)(dst + off) = src */
     case BPF_STX | BPF_MEM | BPF_W:
     case BPF_STX | BPF_MEM | BPF_H:
     case BPF_STX | BPF_MEM | BPF_B:
     case BPF_STX | BPF_MEM | BPF_DW: {
         const u8 tmp = bpf2sparc[TMP_REG_1];
         u32 opcode = 0, rs2;
 
         if (insn->dst_reg == BPF_REG_FP)
             ctx->saw_frame_pointer = true;
 
         switch (BPF_SIZE(code)) {
         case BPF_W:
             opcode = ST32;
             break;
         case BPF_H:
             opcode = ST16;
             break;
         case BPF_B:
             opcode = ST8;
             break;
         case BPF_DW:
             opcode = ST64;
             break;
         }
         if (is_simm13(off)) {
             opcode |= IMMED;
             rs2 = S13(off);
         } else {
             ctx->tmp_1_used = true;
             emit_loadimm(off, tmp, ctx);
             rs2 = RS2(tmp);
         }
         emit(opcode | RS1(dst) | rs2 | RD(src), ctx);
         break;
     }
 
     case BPF_STX | BPF_ATOMIC | BPF_W: {
         const u8 tmp = bpf2sparc[TMP_REG_1];
         const u8 tmp2 = bpf2sparc[TMP_REG_2];
         const u8 tmp3 = bpf2sparc[TMP_REG_3];
 
         if (insn->imm != BPF_ADD) {
             pr_err_once("unknown atomic op %02x\n", insn->imm);
             return -EINVAL;
         }
 
         /* lock *(u32 *)(dst + off) += src */
 
         if (insn->dst_reg == BPF_REG_FP)
             ctx->saw_frame_pointer = true;
 
         ctx->tmp_1_used = true;
         ctx->tmp_2_used = true;
         ctx->tmp_3_used = true;
         emit_loadimm(off, tmp, ctx);
         emit_alu3(ADD, dst, tmp, tmp, ctx);
 
         emit(LD32 | RS1(tmp) | RS2(G0) | RD(tmp2), ctx);
         emit_alu3(ADD, tmp2, src, tmp3, ctx);
         emit(CAS | ASI(ASI_P) | RS1(tmp) | RS2(tmp2) | RD(tmp3), ctx);
         emit_cmp(tmp2, tmp3, ctx);
         emit_branch(BNE, 4, 0, ctx);
         emit_nop(ctx);
         break;
     }
     /* STX XADD: lock *(u64 *)(dst + off) += src */
     case BPF_STX | BPF_ATOMIC | BPF_DW: {
         const u8 tmp = bpf2sparc[TMP_REG_1];
         const u8 tmp2 = bpf2sparc[TMP_REG_2];
         const u8 tmp3 = bpf2sparc[TMP_REG_3];
 
         if (insn->imm != BPF_ADD) {
             pr_err_once("unknown atomic op %02x\n", insn->imm);
             return -EINVAL;
         }
 
         if (insn->dst_reg == BPF_REG_FP)
             ctx->saw_frame_pointer = true;
 
         ctx->tmp_1_used = true;
         ctx->tmp_2_used = true;
         ctx->tmp_3_used = true;
         emit_loadimm(off, tmp, ctx);
         emit_alu3(ADD, dst, tmp, tmp, ctx);
 
         emit(LD64 | RS1(tmp) | RS2(G0) | RD(tmp2), ctx);
         emit_alu3(ADD, tmp2, src, tmp3, ctx);
         emit(CASX | ASI(ASI_P) | RS1(tmp) | RS2(tmp2) | RD(tmp3), ctx);
         emit_cmp(tmp2, tmp3, ctx);
         emit_branch(BNE, 4, 0, ctx);
         emit_nop(ctx);
         break;
     }
 
     default:
         pr_err_once("unknown opcode %02x\n", code);
         return -EINVAL;
     }
 
     return 0;
 }
 
 static int build_body(struct jit_ctx *ctx)
 {
     const struct bpf_prog *prog = ctx->prog;
     int i;
 
     for (i = 0; i < prog->len; i++) {
         const struct bpf_insn *insn = &prog->insnsi[i];
         int ret;
 
         ret = build_insn(insn, ctx);
 
         if (ret > 0) {
             i++;
             ctx->offset[i] = ctx->idx;
             continue;
         }
         ctx->offset[i] = ctx->idx;
         if (ret)
             return ret;
     }
     return 0;
 }
 
 static void jit_fill_hole(void *area, unsigned int size)
 {
     u32 *ptr;
     /* We are guaranteed to have aligned memory. */
     for (ptr = area; size >= sizeof(u32); size -= sizeof(u32))
         *ptr++ = 0x91d02005; /* ta 5 */
 }
 
 bool bpf_jit_needs_zext(void)
 {
     return true;
 }
 
 struct sparc64_jit_data {
     struct bpf_binary_header *header;
     u8 *image;
     struct jit_ctx ctx;
 };
 
 struct bpf_prog *bpf_int_jit_compile(struct bpf_prog *prog)
 {
     struct bpf_prog *tmp, *orig_prog = prog;
     struct sparc64_jit_data *jit_data;
     struct bpf_binary_header *header;
     u32 prev_image_size, image_size;
     bool tmp_blinded = false;
     bool extra_pass = false;
     struct jit_ctx ctx;
     u8 *image_ptr;
     int pass, i;
 
     if (!prog->jit_requested)
         return orig_prog;
 
     tmp = bpf_jit_blind_constants(prog);
     /* If blinding was requested and we failed during blinding,
      * we must fall back to the interpreter.
      */
     if (IS_ERR(tmp))
         return orig_prog;
     if (tmp != prog) {
         tmp_blinded = true;
         prog = tmp;
     }
 
     jit_data = prog->aux->jit_data;
     if (!jit_data) {
         jit_data = kzalloc(sizeof(*jit_data), GFP_KERNEL);
         if (!jit_data) {
             prog = orig_prog;
             goto out;
         }
         prog->aux->jit_data = jit_data;
     }
     if (jit_data->ctx.offset) {
         ctx = jit_data->ctx;
         image_ptr = jit_data->image;
         header = jit_data->header;
         extra_pass = true;
         image_size = sizeof(u32) * ctx.idx;
         prev_image_size = image_size;
         pass = 1;
         goto skip_init_ctx;
     }
 
     memset(&ctx, 0, sizeof(ctx));
     ctx.prog = prog;
 
     ctx.offset = kmalloc_array(prog->len, sizeof(unsigned int), GFP_KERNEL);
     if (ctx.offset == NULL) {
         prog = orig_prog;
         goto out_off;
     }
 
     /* Longest sequence emitted is for bswap32, 12 instructions.  Pre-cook
      * the offset array so that we converge faster.
      */
     for (i = 0; i < prog->len; i++)
         ctx.offset[i] = i * (12 * 4);
 
     prev_image_size = ~0U;
     for (pass = 1; pass < 40; pass++) {
         ctx.idx = 0;
 
         build_prologue(&ctx);
         if (build_body(&ctx)) {
             prog = orig_prog;
             goto out_off;
         }
         build_epilogue(&ctx);
 
         if (bpf_jit_enable > 1)
             pr_info("Pass %d: size = %u, seen = [%c%c%c%c%c%c]\n", pass,
                 ctx.idx * 4,
                 ctx.tmp_1_used ? '1' : ' ',
                 ctx.tmp_2_used ? '2' : ' ',
                 ctx.tmp_3_used ? '3' : ' ',
                 ctx.saw_frame_pointer ? 'F' : ' ',
                 ctx.saw_call ? 'C' : ' ',
                 ctx.saw_tail_call ? 'T' : ' ');
 
         if (ctx.idx * 4 == prev_image_size)
             break;
         prev_image_size = ctx.idx * 4;
         cond_resched();
     }
 
     /* Now we know the actual image size. */
     image_size = sizeof(u32) * ctx.idx;
     header = bpf_jit_binary_alloc(image_size, &image_ptr,
                       sizeof(u32), jit_fill_hole);
     if (header == NULL) {
         prog = orig_prog;
         goto out_off;
     }
 
     ctx.image = (u32 *)image_ptr;
 skip_init_ctx:
     ctx.idx = 0;
 
     build_prologue(&ctx);
 
     if (build_body(&ctx)) {
         bpf_jit_binary_free(header);
         prog = orig_prog;
         goto out_off;
     }
 
     build_epilogue(&ctx);
 
     if (ctx.idx * 4 != prev_image_size) {
         pr_err("bpf_jit: Failed to converge, prev_size=%u size=%d\n",
                prev_image_size, ctx.idx * 4);
         bpf_jit_binary_free(header);
         prog = orig_prog;
         goto out_off;
     }
 
     if (bpf_jit_enable > 1)
         bpf_jit_dump(prog->len, image_size, pass, ctx.image);
 
     bpf_flush_icache(header, (u8 *)header + header->size);
 
     if (!prog->is_func || extra_pass) {
         bpf_jit_binary_lock_ro(header);
     } else {
         jit_data->ctx = ctx;
         jit_data->image = image_ptr;
         jit_data->header = header;
     }
 
     prog->bpf_func = (void *)ctx.image;
     prog->jited = 1;
     prog->jited_len = image_size;
 
     if (!prog->is_func || extra_pass) {
         bpf_prog_fill_jited_linfo(prog, ctx.offset);
 out_off:
         kfree(ctx.offset);
         kfree(jit_data);
         prog->aux->jit_data = NULL;
     }
 out:
     if (tmp_blinded)
         bpf_jit_prog_release_other(prog, prog == orig_prog ?
                        tmp : orig_prog);
     return prog;
 }

This page was automatically generated by the 2.1.0 LXR engine. The LXR team