mips/net/bpf_jit_comp.h

0001 /* SPDX-License-Identifier: GPL-2.0-only */
0002 /*
0003  * Just-In-Time compiler for eBPF bytecode on 32-bit and 64-bit MIPS.
0004  *
0005  * Copyright (c) 2021 Anyfi Networks AB.
0006  * Author: Johan Almbladh <johan.almbladh@gmail.com>
0007  *
0008  * Based on code and ideas from
0009  * Copyright (c) 2017 Cavium, Inc.
0010  * Copyright (c) 2017 Shubham Bansal <illusionist.neo@gmail.com>
0011  * Copyright (c) 2011 Mircea Gherzan <mgherzan@gmail.com>
0012  */
0013
0014 #ifndef _BPF_JIT_COMP_H
0015 #define _BPF_JIT_COMP_H
0016
0017 /* MIPS registers */
0018 #define MIPS_R_ZERO 0   /* Const zero */
0019 #define MIPS_R_AT   1   /* Asm temp   */
0020 #define MIPS_R_V0   2   /* Result     */
0021 #define MIPS_R_V1   3   /* Result     */
0022 #define MIPS_R_A0   4   /* Argument   */
0023 #define MIPS_R_A1   5   /* Argument   */
0024 #define MIPS_R_A2   6   /* Argument   */
0025 #define MIPS_R_A3   7   /* Argument   */
0026 #define MIPS_R_A4   8   /* Arg (n64)  */
0027 #define MIPS_R_A5   9   /* Arg (n64)  */
0028 #define MIPS_R_A6   10  /* Arg (n64)  */
0029 #define MIPS_R_A7   11  /* Arg (n64)  */
0030 #define MIPS_R_T0   8   /* Temp (o32) */
0031 #define MIPS_R_T1   9   /* Temp (o32) */
0032 #define MIPS_R_T2   10  /* Temp (o32) */
0033 #define MIPS_R_T3   11  /* Temp (o32) */
0034 #define MIPS_R_T4   12  /* Temporary  */
0035 #define MIPS_R_T5   13  /* Temporary  */
0036 #define MIPS_R_T6   14  /* Temporary  */
0037 #define MIPS_R_T7   15  /* Temporary  */
0038 #define MIPS_R_S0   16  /* Saved      */
0039 #define MIPS_R_S1   17  /* Saved      */
0040 #define MIPS_R_S2   18  /* Saved      */
0041 #define MIPS_R_S3   19  /* Saved      */
0042 #define MIPS_R_S4   20  /* Saved      */
0043 #define MIPS_R_S5   21  /* Saved      */
0044 #define MIPS_R_S6   22  /* Saved      */
0045 #define MIPS_R_S7   23  /* Saved      */
0046 #define MIPS_R_T8   24  /* Temporary  */
0047 #define MIPS_R_T9   25  /* Temporary  */
0048 /*      MIPS_R_K0   26     Reserved   */
0049 /*      MIPS_R_K1   27     Reserved   */
0050 #define MIPS_R_GP   28  /* Global ptr */
0051 #define MIPS_R_SP   29  /* Stack ptr  */
0052 #define MIPS_R_FP   30  /* Frame ptr  */
0053 #define MIPS_R_RA   31  /* Return     */
0054
0055 /*
0056  * Jump address mask for immediate jumps. The four most significant bits
0057  * must be equal to PC.
0058  */
0059 #define MIPS_JMP_MASK   0x0fffffffUL
0060
0061 /* Maximum number of iterations in offset table computation */
0062 #define JIT_MAX_ITERATIONS  8
0063
0064 /*
0065  * Jump pseudo-instructions used internally
0066  * for branch conversion and branch optimization.
0067  */
0068 #define JIT_JNSET   0xe0
0069 #define JIT_JNOP    0xf0
0070
0071 /* Descriptor flag for PC-relative branch conversion */
0072 #define JIT_DESC_CONVERT    BIT(31)
0073
0074 /* JIT context for an eBPF program */
0075 struct jit_context {
0076     struct bpf_prog *program;     /* The eBPF program being JITed        */
0077     u32 *descriptors;             /* eBPF to JITed CPU insn descriptors  */
0078     u32 *target;                  /* JITed code buffer                   */
0079     u32 bpf_index;                /* Index of current BPF program insn   */
0080     u32 jit_index;                /* Index of current JIT target insn    */
0081     u32 changes;                  /* Number of PC-relative branch conv   */
0082     u32 accessed;                 /* Bit mask of read eBPF registers     */
0083     u32 clobbered;                /* Bit mask of modified CPU registers  */
0084     u32 stack_size;               /* Total allocated stack size in bytes */
0085     u32 saved_size;               /* Size of callee-saved registers      */
0086     u32 stack_used;               /* Stack size used for function calls  */
0087 };
0088
0089 /* Emit the instruction if the JIT memory space has been allocated */
0090 #define __emit(ctx, func, ...)                  \
0091 do {                                \
0092     if ((ctx)->target != NULL) {                \
0093         u32 *p = &(ctx)->target[ctx->jit_index];    \
0094         uasm_i_##func(&p, ##__VA_ARGS__);       \
0095     }                           \
0096     (ctx)->jit_index++;                 \
0097 } while (0)
0098 #define emit(...) __emit(__VA_ARGS__)
0099
0100 /* Workaround for R10000 ll/sc errata */
0101 #ifdef CONFIG_WAR_R10000_LLSC
0102 #define LLSC_beqz   beqzl
0103 #else
0104 #define LLSC_beqz   beqz
0105 #endif
0106
0107 /* Workaround for Loongson-3 ll/sc errata */
0108 #ifdef CONFIG_CPU_LOONGSON3_WORKAROUNDS
0109 #define LLSC_sync(ctx)  emit(ctx, sync, 0)
0110 #define LLSC_offset 4
0111 #else
0112 #define LLSC_sync(ctx)
0113 #define LLSC_offset 0
0114 #endif
0115
0116 /* Workaround for Loongson-2F jump errata */
0117 #ifdef CONFIG_CPU_JUMP_WORKAROUNDS
0118 #define JALR_MASK   0xffffffffcfffffffULL
0119 #else
0120 #define JALR_MASK   (~0ULL)
0121 #endif
0122
0123 /*
0124  * Mark a BPF register as accessed, it needs to be
0125  * initialized by the program if expected, e.g. FP.
0126  */
0127 static inline void access_reg(struct jit_context *ctx, u8 reg)
0128 {
0129     ctx->accessed |= BIT(reg);
0130 }
0131
0132 /*
0133  * Mark a CPU register as clobbered, it needs to be
0134  * saved/restored by the program if callee-saved.
0135  */
0136 static inline void clobber_reg(struct jit_context *ctx, u8 reg)
0137 {
0138     ctx->clobbered |= BIT(reg);
0139 }
0140
0141 /*
0142  * Push registers on the stack, starting at a given depth from the stack
0143  * pointer and increasing. The next depth to be written is returned.
0144  */
0145 int push_regs(struct jit_context *ctx, u32 mask, u32 excl, int depth);
0146
0147 /*
0148  * Pop registers from the stack, starting at a given depth from the stack
0149  * pointer and increasing. The next depth to be read is returned.
0150  */
0151 int pop_regs(struct jit_context *ctx, u32 mask, u32 excl, int depth);
0152
0153 /* Compute the 28-bit jump target address from a BPF program location */
0154 int get_target(struct jit_context *ctx, u32 loc);
0155
0156 /* Compute the PC-relative offset to relative BPF program offset */
0157 int get_offset(const struct jit_context *ctx, int off);
0158
0159 /* dst = imm (32-bit) */
0160 void emit_mov_i(struct jit_context *ctx, u8 dst, s32 imm);
0161
0162 /* dst = src (32-bit) */
0163 void emit_mov_r(struct jit_context *ctx, u8 dst, u8 src);
0164
0165 /* Validate ALU/ALU64 immediate range */
0166 bool valid_alu_i(u8 op, s32 imm);
0167
0168 /* Rewrite ALU/ALU64 immediate operation */
0169 bool rewrite_alu_i(u8 op, s32 imm, u8 *alu, s32 *val);
0170
0171 /* ALU immediate operation (32-bit) */
0172 void emit_alu_i(struct jit_context *ctx, u8 dst, s32 imm, u8 op);
0173
0174 /* ALU register operation (32-bit) */
0175 void emit_alu_r(struct jit_context *ctx, u8 dst, u8 src, u8 op);
0176
0177 /* Atomic read-modify-write (32-bit) */
0178 void emit_atomic_r(struct jit_context *ctx, u8 dst, u8 src, s16 off, u8 code);
0179
0180 /* Atomic compare-and-exchange (32-bit) */
0181 void emit_cmpxchg_r(struct jit_context *ctx, u8 dst, u8 src, u8 res, s16 off);
0182
0183 /* Swap bytes and truncate a register word or half word */
0184 void emit_bswap_r(struct jit_context *ctx, u8 dst, u32 width);
0185
0186 /* Validate JMP/JMP32 immediate range */
0187 bool valid_jmp_i(u8 op, s32 imm);
0188
0189 /* Prepare a PC-relative jump operation with immediate conditional */
0190 void setup_jmp_i(struct jit_context *ctx, s32 imm, u8 width,
0191          u8 bpf_op, s16 bpf_off, u8 *jit_op, s32 *jit_off);
0192
0193 /* Prepare a PC-relative jump operation with register conditional */
0194 void setup_jmp_r(struct jit_context *ctx, bool same_reg,
0195          u8 bpf_op, s16 bpf_off, u8 *jit_op, s32 *jit_off);
0196
0197 /* Finish a PC-relative jump operation */
0198 int finish_jmp(struct jit_context *ctx, u8 jit_op, s16 bpf_off);
0199
0200 /* Conditional JMP/JMP32 immediate */
0201 void emit_jmp_i(struct jit_context *ctx, u8 dst, s32 imm, s32 off, u8 op);
0202
0203 /* Conditional JMP/JMP32 register */
0204 void emit_jmp_r(struct jit_context *ctx, u8 dst, u8 src, s32 off, u8 op);
0205
0206 /* Jump always */
0207 int emit_ja(struct jit_context *ctx, s16 off);
0208
0209 /* Jump to epilogue */
0210 int emit_exit(struct jit_context *ctx);
0211
0212 /*
0213  * Build program prologue to set up the stack and registers.
0214  * This function is implemented separately for 32-bit and 64-bit JITs.
0215  */
0216 void build_prologue(struct jit_context *ctx);
0217
0218 /*
0219  * Build the program epilogue to restore the stack and registers.
0220  * This function is implemented separately for 32-bit and 64-bit JITs.
0221  */
0222 void build_epilogue(struct jit_context *ctx, int dest_reg);
0223
0224 /*
0225  * Convert an eBPF instruction to native instruction, i.e
0226  * JITs an eBPF instruction.
0227  * Returns :
0228  *  0  - Successfully JITed an 8-byte eBPF instruction
0229  *  >0 - Successfully JITed a 16-byte eBPF instruction
0230  *  <0 - Failed to JIT.
0231  * This function is implemented separately for 32-bit and 64-bit JITs.
0232  */
0233 int build_insn(const struct bpf_insn *insn, struct jit_context *ctx);
0234
0235 #endif /* _BPF_JIT_COMP_H */