Back to home page

OSCL-LXR
 
 

     

0001 // SPDX-License-Identifier: GPL-2.0-only
0002 /*
0003  * bpf_jit_comp64.c: eBPF JIT compiler
0004  *
0005  * Copyright 2016 Naveen N. Rao <naveen.n.rao@linux.vnet.ibm.com>
0006  *        IBM Corporation
0007  *
0008  * Based on the powerpc classic BPF JIT compiler by Matt Evans
0009  */
0010 #include <linux/moduleloader.h>
0011 #include <asm/cacheflush.h>
0012 #include <asm/asm-compat.h>
0013 #include <linux/netdevice.h>
0014 #include <linux/filter.h>
0015 #include <linux/if_vlan.h>
0016 #include <asm/kprobes.h>
0017 #include <linux/bpf.h>
0018 #include <asm/security_features.h>
0019 
0020 #include "bpf_jit.h"
0021 
0022 /*
0023  * Stack layout:
0024  * Ensure the top half (upto local_tmp_var) stays consistent
0025  * with our redzone usage.
0026  *
0027  *      [   prev sp     ] <-------------
0028  *      [   nv gpr save area    ] 5*8       |
0029  *      [    tail_call_cnt  ] 8     |
0030  *      [    local_tmp_var  ] 16        |
0031  * fp (r31) --> [   ebpf stack space    ] upto 512  |
0032  *      [     frame header  ] 32/112    |
0033  * sp (r1) ---> [    stack pointer  ] --------------
0034  */
0035 
0036 /* for gpr non volatile registers BPG_REG_6 to 10 */
0037 #define BPF_PPC_STACK_SAVE  (5*8)
0038 /* for bpf JIT code internal usage */
0039 #define BPF_PPC_STACK_LOCALS    24
0040 /* stack frame excluding BPF stack, ensure this is quadword aligned */
0041 #define BPF_PPC_STACKFRAME  (STACK_FRAME_MIN_SIZE + \
0042                  BPF_PPC_STACK_LOCALS + BPF_PPC_STACK_SAVE)
0043 
0044 /* BPF register usage */
0045 #define TMP_REG_1   (MAX_BPF_JIT_REG + 0)
0046 #define TMP_REG_2   (MAX_BPF_JIT_REG + 1)
0047 
0048 /* BPF to ppc register mappings */
0049 void bpf_jit_init_reg_mapping(struct codegen_context *ctx)
0050 {
0051     /* function return value */
0052     ctx->b2p[BPF_REG_0] = _R8;
0053     /* function arguments */
0054     ctx->b2p[BPF_REG_1] = _R3;
0055     ctx->b2p[BPF_REG_2] = _R4;
0056     ctx->b2p[BPF_REG_3] = _R5;
0057     ctx->b2p[BPF_REG_4] = _R6;
0058     ctx->b2p[BPF_REG_5] = _R7;
0059     /* non volatile registers */
0060     ctx->b2p[BPF_REG_6] = _R27;
0061     ctx->b2p[BPF_REG_7] = _R28;
0062     ctx->b2p[BPF_REG_8] = _R29;
0063     ctx->b2p[BPF_REG_9] = _R30;
0064     /* frame pointer aka BPF_REG_10 */
0065     ctx->b2p[BPF_REG_FP] = _R31;
0066     /* eBPF jit internal registers */
0067     ctx->b2p[BPF_REG_AX] = _R12;
0068     ctx->b2p[TMP_REG_1] = _R9;
0069     ctx->b2p[TMP_REG_2] = _R10;
0070 }
0071 
0072 /* PPC NVR range -- update this if we ever use NVRs below r27 */
0073 #define BPF_PPC_NVR_MIN     _R27
0074 
0075 static inline bool bpf_has_stack_frame(struct codegen_context *ctx)
0076 {
0077     /*
0078      * We only need a stack frame if:
0079      * - we call other functions (kernel helpers), or
0080      * - the bpf program uses its stack area
0081      * The latter condition is deduced from the usage of BPF_REG_FP
0082      */
0083     return ctx->seen & SEEN_FUNC || bpf_is_seen_register(ctx, bpf_to_ppc(BPF_REG_FP));
0084 }
0085 
0086 /*
0087  * When not setting up our own stackframe, the redzone usage is:
0088  *
0089  *      [   prev sp     ] <-------------
0090  *      [     ...           ]       |
0091  * sp (r1) ---> [    stack pointer  ] --------------
0092  *      [   nv gpr save area    ] 5*8
0093  *      [    tail_call_cnt  ] 8
0094  *      [    local_tmp_var  ] 16
0095  *      [   unused red zone ] 208 bytes protected
0096  */
0097 static int bpf_jit_stack_local(struct codegen_context *ctx)
0098 {
0099     if (bpf_has_stack_frame(ctx))
0100         return STACK_FRAME_MIN_SIZE + ctx->stack_size;
0101     else
0102         return -(BPF_PPC_STACK_SAVE + 24);
0103 }
0104 
0105 static int bpf_jit_stack_tailcallcnt(struct codegen_context *ctx)
0106 {
0107     return bpf_jit_stack_local(ctx) + 16;
0108 }
0109 
0110 static int bpf_jit_stack_offsetof(struct codegen_context *ctx, int reg)
0111 {
0112     if (reg >= BPF_PPC_NVR_MIN && reg < 32)
0113         return (bpf_has_stack_frame(ctx) ?
0114             (BPF_PPC_STACKFRAME + ctx->stack_size) : 0)
0115                 - (8 * (32 - reg));
0116 
0117     pr_err("BPF JIT is asking about unknown registers");
0118     BUG();
0119 }
0120 
0121 void bpf_jit_realloc_regs(struct codegen_context *ctx)
0122 {
0123 }
0124 
0125 void bpf_jit_build_prologue(u32 *image, struct codegen_context *ctx)
0126 {
0127     int i;
0128 
0129     if (IS_ENABLED(CONFIG_PPC64_ELF_ABI_V2))
0130         EMIT(PPC_RAW_LD(_R2, _R13, offsetof(struct paca_struct, kernel_toc)));
0131 
0132     /*
0133      * Initialize tail_call_cnt if we do tail calls.
0134      * Otherwise, put in NOPs so that it can be skipped when we are
0135      * invoked through a tail call.
0136      */
0137     if (ctx->seen & SEEN_TAILCALL) {
0138         EMIT(PPC_RAW_LI(bpf_to_ppc(TMP_REG_1), 0));
0139         /* this goes in the redzone */
0140         EMIT(PPC_RAW_STD(bpf_to_ppc(TMP_REG_1), _R1, -(BPF_PPC_STACK_SAVE + 8)));
0141     } else {
0142         EMIT(PPC_RAW_NOP());
0143         EMIT(PPC_RAW_NOP());
0144     }
0145 
0146     if (bpf_has_stack_frame(ctx)) {
0147         /*
0148          * We need a stack frame, but we don't necessarily need to
0149          * save/restore LR unless we call other functions
0150          */
0151         if (ctx->seen & SEEN_FUNC) {
0152             EMIT(PPC_RAW_MFLR(_R0));
0153             EMIT(PPC_RAW_STD(_R0, _R1, PPC_LR_STKOFF));
0154         }
0155 
0156         EMIT(PPC_RAW_STDU(_R1, _R1, -(BPF_PPC_STACKFRAME + ctx->stack_size)));
0157     }
0158 
0159     /*
0160      * Back up non-volatile regs -- BPF registers 6-10
0161      * If we haven't created our own stack frame, we save these
0162      * in the protected zone below the previous stack frame
0163      */
0164     for (i = BPF_REG_6; i <= BPF_REG_10; i++)
0165         if (bpf_is_seen_register(ctx, bpf_to_ppc(i)))
0166             EMIT(PPC_RAW_STD(bpf_to_ppc(i), _R1, bpf_jit_stack_offsetof(ctx, bpf_to_ppc(i))));
0167 
0168     /* Setup frame pointer to point to the bpf stack area */
0169     if (bpf_is_seen_register(ctx, bpf_to_ppc(BPF_REG_FP)))
0170         EMIT(PPC_RAW_ADDI(bpf_to_ppc(BPF_REG_FP), _R1,
0171                 STACK_FRAME_MIN_SIZE + ctx->stack_size));
0172 }
0173 
0174 static void bpf_jit_emit_common_epilogue(u32 *image, struct codegen_context *ctx)
0175 {
0176     int i;
0177 
0178     /* Restore NVRs */
0179     for (i = BPF_REG_6; i <= BPF_REG_10; i++)
0180         if (bpf_is_seen_register(ctx, bpf_to_ppc(i)))
0181             EMIT(PPC_RAW_LD(bpf_to_ppc(i), _R1, bpf_jit_stack_offsetof(ctx, bpf_to_ppc(i))));
0182 
0183     /* Tear down our stack frame */
0184     if (bpf_has_stack_frame(ctx)) {
0185         EMIT(PPC_RAW_ADDI(_R1, _R1, BPF_PPC_STACKFRAME + ctx->stack_size));
0186         if (ctx->seen & SEEN_FUNC) {
0187             EMIT(PPC_RAW_LD(_R0, _R1, PPC_LR_STKOFF));
0188             EMIT(PPC_RAW_MTLR(_R0));
0189         }
0190     }
0191 }
0192 
0193 void bpf_jit_build_epilogue(u32 *image, struct codegen_context *ctx)
0194 {
0195     bpf_jit_emit_common_epilogue(image, ctx);
0196 
0197     /* Move result to r3 */
0198     EMIT(PPC_RAW_MR(_R3, bpf_to_ppc(BPF_REG_0)));
0199 
0200     EMIT(PPC_RAW_BLR());
0201 }
0202 
0203 static int bpf_jit_emit_func_call_hlp(u32 *image, struct codegen_context *ctx, u64 func)
0204 {
0205     unsigned long func_addr = func ? ppc_function_entry((void *)func) : 0;
0206     long reladdr;
0207 
0208     if (WARN_ON_ONCE(!core_kernel_text(func_addr)))
0209         return -EINVAL;
0210 
0211     reladdr = func_addr - kernel_toc_addr();
0212     if (reladdr > 0x7FFFFFFF || reladdr < -(0x80000000L)) {
0213         pr_err("eBPF: address of %ps out of range of kernel_toc.\n", (void *)func);
0214         return -ERANGE;
0215     }
0216 
0217     EMIT(PPC_RAW_ADDIS(_R12, _R2, PPC_HA(reladdr)));
0218     EMIT(PPC_RAW_ADDI(_R12, _R12, PPC_LO(reladdr)));
0219     EMIT(PPC_RAW_MTCTR(_R12));
0220     EMIT(PPC_RAW_BCTRL());
0221 
0222     return 0;
0223 }
0224 
0225 int bpf_jit_emit_func_call_rel(u32 *image, struct codegen_context *ctx, u64 func)
0226 {
0227     unsigned int i, ctx_idx = ctx->idx;
0228 
0229     if (WARN_ON_ONCE(func && is_module_text_address(func)))
0230         return -EINVAL;
0231 
0232     /* skip past descriptor if elf v1 */
0233     func += FUNCTION_DESCR_SIZE;
0234 
0235     /* Load function address into r12 */
0236     PPC_LI64(_R12, func);
0237 
0238     /* For bpf-to-bpf function calls, the callee's address is unknown
0239      * until the last extra pass. As seen above, we use PPC_LI64() to
0240      * load the callee's address, but this may optimize the number of
0241      * instructions required based on the nature of the address.
0242      *
0243      * Since we don't want the number of instructions emitted to change,
0244      * we pad the optimized PPC_LI64() call with NOPs to guarantee that
0245      * we always have a five-instruction sequence, which is the maximum
0246      * that PPC_LI64() can emit.
0247      */
0248     for (i = ctx->idx - ctx_idx; i < 5; i++)
0249         EMIT(PPC_RAW_NOP());
0250 
0251     EMIT(PPC_RAW_MTCTR(_R12));
0252     EMIT(PPC_RAW_BCTRL());
0253 
0254     return 0;
0255 }
0256 
0257 static int bpf_jit_emit_tail_call(u32 *image, struct codegen_context *ctx, u32 out)
0258 {
0259     /*
0260      * By now, the eBPF program has already setup parameters in r3, r4 and r5
0261      * r3/BPF_REG_1 - pointer to ctx -- passed as is to the next bpf program
0262      * r4/BPF_REG_2 - pointer to bpf_array
0263      * r5/BPF_REG_3 - index in bpf_array
0264      */
0265     int b2p_bpf_array = bpf_to_ppc(BPF_REG_2);
0266     int b2p_index = bpf_to_ppc(BPF_REG_3);
0267     int bpf_tailcall_prologue_size = 8;
0268 
0269     if (IS_ENABLED(CONFIG_PPC64_ELF_ABI_V2))
0270         bpf_tailcall_prologue_size += 4; /* skip past the toc load */
0271 
0272     /*
0273      * if (index >= array->map.max_entries)
0274      *   goto out;
0275      */
0276     EMIT(PPC_RAW_LWZ(bpf_to_ppc(TMP_REG_1), b2p_bpf_array, offsetof(struct bpf_array, map.max_entries)));
0277     EMIT(PPC_RAW_RLWINM(b2p_index, b2p_index, 0, 0, 31));
0278     EMIT(PPC_RAW_CMPLW(b2p_index, bpf_to_ppc(TMP_REG_1)));
0279     PPC_BCC_SHORT(COND_GE, out);
0280 
0281     /*
0282      * if (tail_call_cnt >= MAX_TAIL_CALL_CNT)
0283      *   goto out;
0284      */
0285     EMIT(PPC_RAW_LD(bpf_to_ppc(TMP_REG_1), _R1, bpf_jit_stack_tailcallcnt(ctx)));
0286     EMIT(PPC_RAW_CMPLWI(bpf_to_ppc(TMP_REG_1), MAX_TAIL_CALL_CNT));
0287     PPC_BCC_SHORT(COND_GE, out);
0288 
0289     /*
0290      * tail_call_cnt++;
0291      */
0292     EMIT(PPC_RAW_ADDI(bpf_to_ppc(TMP_REG_1), bpf_to_ppc(TMP_REG_1), 1));
0293     EMIT(PPC_RAW_STD(bpf_to_ppc(TMP_REG_1), _R1, bpf_jit_stack_tailcallcnt(ctx)));
0294 
0295     /* prog = array->ptrs[index]; */
0296     EMIT(PPC_RAW_MULI(bpf_to_ppc(TMP_REG_1), b2p_index, 8));
0297     EMIT(PPC_RAW_ADD(bpf_to_ppc(TMP_REG_1), bpf_to_ppc(TMP_REG_1), b2p_bpf_array));
0298     EMIT(PPC_RAW_LD(bpf_to_ppc(TMP_REG_1), bpf_to_ppc(TMP_REG_1), offsetof(struct bpf_array, ptrs)));
0299 
0300     /*
0301      * if (prog == NULL)
0302      *   goto out;
0303      */
0304     EMIT(PPC_RAW_CMPLDI(bpf_to_ppc(TMP_REG_1), 0));
0305     PPC_BCC_SHORT(COND_EQ, out);
0306 
0307     /* goto *(prog->bpf_func + prologue_size); */
0308     EMIT(PPC_RAW_LD(bpf_to_ppc(TMP_REG_1), bpf_to_ppc(TMP_REG_1), offsetof(struct bpf_prog, bpf_func)));
0309     EMIT(PPC_RAW_ADDI(bpf_to_ppc(TMP_REG_1), bpf_to_ppc(TMP_REG_1),
0310             FUNCTION_DESCR_SIZE + bpf_tailcall_prologue_size));
0311     EMIT(PPC_RAW_MTCTR(bpf_to_ppc(TMP_REG_1)));
0312 
0313     /* tear down stack, restore NVRs, ... */
0314     bpf_jit_emit_common_epilogue(image, ctx);
0315 
0316     EMIT(PPC_RAW_BCTR());
0317 
0318     /* out: */
0319     return 0;
0320 }
0321 
0322 /*
0323  * We spill into the redzone always, even if the bpf program has its own stackframe.
0324  * Offsets hardcoded based on BPF_PPC_STACK_SAVE -- see bpf_jit_stack_local()
0325  */
0326 void bpf_stf_barrier(void);
0327 
0328 asm (
0329 "       .global bpf_stf_barrier     ;"
0330 "   bpf_stf_barrier:            ;"
0331 "       std 21,-64(1)       ;"
0332 "       std 22,-56(1)       ;"
0333 "       sync                ;"
0334 "       ld  21,-64(1)       ;"
0335 "       ld  22,-56(1)       ;"
0336 "       ori 31,31,0         ;"
0337 "       .rept 14            ;"
0338 "       b   1f          ;"
0339 "   1:                  ;"
0340 "       .endr               ;"
0341 "       blr             ;"
0342 );
0343 
0344 /* Assemble the body code between the prologue & epilogue */
0345 int bpf_jit_build_body(struct bpf_prog *fp, u32 *image, struct codegen_context *ctx,
0346                u32 *addrs, int pass)
0347 {
0348     enum stf_barrier_type stf_barrier = stf_barrier_type_get();
0349     const struct bpf_insn *insn = fp->insnsi;
0350     int flen = fp->len;
0351     int i, ret;
0352 
0353     /* Start of epilogue code - will only be valid 2nd pass onwards */
0354     u32 exit_addr = addrs[flen];
0355 
0356     for (i = 0; i < flen; i++) {
0357         u32 code = insn[i].code;
0358         u32 dst_reg = bpf_to_ppc(insn[i].dst_reg);
0359         u32 src_reg = bpf_to_ppc(insn[i].src_reg);
0360         u32 size = BPF_SIZE(code);
0361         u32 tmp1_reg = bpf_to_ppc(TMP_REG_1);
0362         u32 tmp2_reg = bpf_to_ppc(TMP_REG_2);
0363         u32 save_reg, ret_reg;
0364         s16 off = insn[i].off;
0365         s32 imm = insn[i].imm;
0366         bool func_addr_fixed;
0367         u64 func_addr;
0368         u64 imm64;
0369         u32 true_cond;
0370         u32 tmp_idx;
0371         int j;
0372 
0373         /*
0374          * addrs[] maps a BPF bytecode address into a real offset from
0375          * the start of the body code.
0376          */
0377         addrs[i] = ctx->idx * 4;
0378 
0379         /*
0380          * As an optimization, we note down which non-volatile registers
0381          * are used so that we can only save/restore those in our
0382          * prologue and epilogue. We do this here regardless of whether
0383          * the actual BPF instruction uses src/dst registers or not
0384          * (for instance, BPF_CALL does not use them). The expectation
0385          * is that those instructions will have src_reg/dst_reg set to
0386          * 0. Even otherwise, we just lose some prologue/epilogue
0387          * optimization but everything else should work without
0388          * any issues.
0389          */
0390         if (dst_reg >= BPF_PPC_NVR_MIN && dst_reg < 32)
0391             bpf_set_seen_register(ctx, dst_reg);
0392         if (src_reg >= BPF_PPC_NVR_MIN && src_reg < 32)
0393             bpf_set_seen_register(ctx, src_reg);
0394 
0395         switch (code) {
0396         /*
0397          * Arithmetic operations: ADD/SUB/MUL/DIV/MOD/NEG
0398          */
0399         case BPF_ALU | BPF_ADD | BPF_X: /* (u32) dst += (u32) src */
0400         case BPF_ALU64 | BPF_ADD | BPF_X: /* dst += src */
0401             EMIT(PPC_RAW_ADD(dst_reg, dst_reg, src_reg));
0402             goto bpf_alu32_trunc;
0403         case BPF_ALU | BPF_SUB | BPF_X: /* (u32) dst -= (u32) src */
0404         case BPF_ALU64 | BPF_SUB | BPF_X: /* dst -= src */
0405             EMIT(PPC_RAW_SUB(dst_reg, dst_reg, src_reg));
0406             goto bpf_alu32_trunc;
0407         case BPF_ALU | BPF_ADD | BPF_K: /* (u32) dst += (u32) imm */
0408         case BPF_ALU64 | BPF_ADD | BPF_K: /* dst += imm */
0409             if (!imm) {
0410                 goto bpf_alu32_trunc;
0411             } else if (imm >= -32768 && imm < 32768) {
0412                 EMIT(PPC_RAW_ADDI(dst_reg, dst_reg, IMM_L(imm)));
0413             } else {
0414                 PPC_LI32(tmp1_reg, imm);
0415                 EMIT(PPC_RAW_ADD(dst_reg, dst_reg, tmp1_reg));
0416             }
0417             goto bpf_alu32_trunc;
0418         case BPF_ALU | BPF_SUB | BPF_K: /* (u32) dst -= (u32) imm */
0419         case BPF_ALU64 | BPF_SUB | BPF_K: /* dst -= imm */
0420             if (!imm) {
0421                 goto bpf_alu32_trunc;
0422             } else if (imm > -32768 && imm <= 32768) {
0423                 EMIT(PPC_RAW_ADDI(dst_reg, dst_reg, IMM_L(-imm)));
0424             } else {
0425                 PPC_LI32(tmp1_reg, imm);
0426                 EMIT(PPC_RAW_SUB(dst_reg, dst_reg, tmp1_reg));
0427             }
0428             goto bpf_alu32_trunc;
0429         case BPF_ALU | BPF_MUL | BPF_X: /* (u32) dst *= (u32) src */
0430         case BPF_ALU64 | BPF_MUL | BPF_X: /* dst *= src */
0431             if (BPF_CLASS(code) == BPF_ALU)
0432                 EMIT(PPC_RAW_MULW(dst_reg, dst_reg, src_reg));
0433             else
0434                 EMIT(PPC_RAW_MULD(dst_reg, dst_reg, src_reg));
0435             goto bpf_alu32_trunc;
0436         case BPF_ALU | BPF_MUL | BPF_K: /* (u32) dst *= (u32) imm */
0437         case BPF_ALU64 | BPF_MUL | BPF_K: /* dst *= imm */
0438             if (imm >= -32768 && imm < 32768)
0439                 EMIT(PPC_RAW_MULI(dst_reg, dst_reg, IMM_L(imm)));
0440             else {
0441                 PPC_LI32(tmp1_reg, imm);
0442                 if (BPF_CLASS(code) == BPF_ALU)
0443                     EMIT(PPC_RAW_MULW(dst_reg, dst_reg, tmp1_reg));
0444                 else
0445                     EMIT(PPC_RAW_MULD(dst_reg, dst_reg, tmp1_reg));
0446             }
0447             goto bpf_alu32_trunc;
0448         case BPF_ALU | BPF_DIV | BPF_X: /* (u32) dst /= (u32) src */
0449         case BPF_ALU | BPF_MOD | BPF_X: /* (u32) dst %= (u32) src */
0450             if (BPF_OP(code) == BPF_MOD) {
0451                 EMIT(PPC_RAW_DIVWU(tmp1_reg, dst_reg, src_reg));
0452                 EMIT(PPC_RAW_MULW(tmp1_reg, src_reg, tmp1_reg));
0453                 EMIT(PPC_RAW_SUB(dst_reg, dst_reg, tmp1_reg));
0454             } else
0455                 EMIT(PPC_RAW_DIVWU(dst_reg, dst_reg, src_reg));
0456             goto bpf_alu32_trunc;
0457         case BPF_ALU64 | BPF_DIV | BPF_X: /* dst /= src */
0458         case BPF_ALU64 | BPF_MOD | BPF_X: /* dst %= src */
0459             if (BPF_OP(code) == BPF_MOD) {
0460                 EMIT(PPC_RAW_DIVDU(tmp1_reg, dst_reg, src_reg));
0461                 EMIT(PPC_RAW_MULD(tmp1_reg, src_reg, tmp1_reg));
0462                 EMIT(PPC_RAW_SUB(dst_reg, dst_reg, tmp1_reg));
0463             } else
0464                 EMIT(PPC_RAW_DIVDU(dst_reg, dst_reg, src_reg));
0465             break;
0466         case BPF_ALU | BPF_MOD | BPF_K: /* (u32) dst %= (u32) imm */
0467         case BPF_ALU | BPF_DIV | BPF_K: /* (u32) dst /= (u32) imm */
0468         case BPF_ALU64 | BPF_MOD | BPF_K: /* dst %= imm */
0469         case BPF_ALU64 | BPF_DIV | BPF_K: /* dst /= imm */
0470             if (imm == 0)
0471                 return -EINVAL;
0472             if (imm == 1) {
0473                 if (BPF_OP(code) == BPF_DIV) {
0474                     goto bpf_alu32_trunc;
0475                 } else {
0476                     EMIT(PPC_RAW_LI(dst_reg, 0));
0477                     break;
0478                 }
0479             }
0480 
0481             PPC_LI32(tmp1_reg, imm);
0482             switch (BPF_CLASS(code)) {
0483             case BPF_ALU:
0484                 if (BPF_OP(code) == BPF_MOD) {
0485                     EMIT(PPC_RAW_DIVWU(tmp2_reg, dst_reg, tmp1_reg));
0486                     EMIT(PPC_RAW_MULW(tmp1_reg, tmp1_reg, tmp2_reg));
0487                     EMIT(PPC_RAW_SUB(dst_reg, dst_reg, tmp1_reg));
0488                 } else
0489                     EMIT(PPC_RAW_DIVWU(dst_reg, dst_reg, tmp1_reg));
0490                 break;
0491             case BPF_ALU64:
0492                 if (BPF_OP(code) == BPF_MOD) {
0493                     EMIT(PPC_RAW_DIVDU(tmp2_reg, dst_reg, tmp1_reg));
0494                     EMIT(PPC_RAW_MULD(tmp1_reg, tmp1_reg, tmp2_reg));
0495                     EMIT(PPC_RAW_SUB(dst_reg, dst_reg, tmp1_reg));
0496                 } else
0497                     EMIT(PPC_RAW_DIVDU(dst_reg, dst_reg, tmp1_reg));
0498                 break;
0499             }
0500             goto bpf_alu32_trunc;
0501         case BPF_ALU | BPF_NEG: /* (u32) dst = -dst */
0502         case BPF_ALU64 | BPF_NEG: /* dst = -dst */
0503             EMIT(PPC_RAW_NEG(dst_reg, dst_reg));
0504             goto bpf_alu32_trunc;
0505 
0506         /*
0507          * Logical operations: AND/OR/XOR/[A]LSH/[A]RSH
0508          */
0509         case BPF_ALU | BPF_AND | BPF_X: /* (u32) dst = dst & src */
0510         case BPF_ALU64 | BPF_AND | BPF_X: /* dst = dst & src */
0511             EMIT(PPC_RAW_AND(dst_reg, dst_reg, src_reg));
0512             goto bpf_alu32_trunc;
0513         case BPF_ALU | BPF_AND | BPF_K: /* (u32) dst = dst & imm */
0514         case BPF_ALU64 | BPF_AND | BPF_K: /* dst = dst & imm */
0515             if (!IMM_H(imm))
0516                 EMIT(PPC_RAW_ANDI(dst_reg, dst_reg, IMM_L(imm)));
0517             else {
0518                 /* Sign-extended */
0519                 PPC_LI32(tmp1_reg, imm);
0520                 EMIT(PPC_RAW_AND(dst_reg, dst_reg, tmp1_reg));
0521             }
0522             goto bpf_alu32_trunc;
0523         case BPF_ALU | BPF_OR | BPF_X: /* dst = (u32) dst | (u32) src */
0524         case BPF_ALU64 | BPF_OR | BPF_X: /* dst = dst | src */
0525             EMIT(PPC_RAW_OR(dst_reg, dst_reg, src_reg));
0526             goto bpf_alu32_trunc;
0527         case BPF_ALU | BPF_OR | BPF_K:/* dst = (u32) dst | (u32) imm */
0528         case BPF_ALU64 | BPF_OR | BPF_K:/* dst = dst | imm */
0529             if (imm < 0 && BPF_CLASS(code) == BPF_ALU64) {
0530                 /* Sign-extended */
0531                 PPC_LI32(tmp1_reg, imm);
0532                 EMIT(PPC_RAW_OR(dst_reg, dst_reg, tmp1_reg));
0533             } else {
0534                 if (IMM_L(imm))
0535                     EMIT(PPC_RAW_ORI(dst_reg, dst_reg, IMM_L(imm)));
0536                 if (IMM_H(imm))
0537                     EMIT(PPC_RAW_ORIS(dst_reg, dst_reg, IMM_H(imm)));
0538             }
0539             goto bpf_alu32_trunc;
0540         case BPF_ALU | BPF_XOR | BPF_X: /* (u32) dst ^= src */
0541         case BPF_ALU64 | BPF_XOR | BPF_X: /* dst ^= src */
0542             EMIT(PPC_RAW_XOR(dst_reg, dst_reg, src_reg));
0543             goto bpf_alu32_trunc;
0544         case BPF_ALU | BPF_XOR | BPF_K: /* (u32) dst ^= (u32) imm */
0545         case BPF_ALU64 | BPF_XOR | BPF_K: /* dst ^= imm */
0546             if (imm < 0 && BPF_CLASS(code) == BPF_ALU64) {
0547                 /* Sign-extended */
0548                 PPC_LI32(tmp1_reg, imm);
0549                 EMIT(PPC_RAW_XOR(dst_reg, dst_reg, tmp1_reg));
0550             } else {
0551                 if (IMM_L(imm))
0552                     EMIT(PPC_RAW_XORI(dst_reg, dst_reg, IMM_L(imm)));
0553                 if (IMM_H(imm))
0554                     EMIT(PPC_RAW_XORIS(dst_reg, dst_reg, IMM_H(imm)));
0555             }
0556             goto bpf_alu32_trunc;
0557         case BPF_ALU | BPF_LSH | BPF_X: /* (u32) dst <<= (u32) src */
0558             /* slw clears top 32 bits */
0559             EMIT(PPC_RAW_SLW(dst_reg, dst_reg, src_reg));
0560             /* skip zero extension move, but set address map. */
0561             if (insn_is_zext(&insn[i + 1]))
0562                 addrs[++i] = ctx->idx * 4;
0563             break;
0564         case BPF_ALU64 | BPF_LSH | BPF_X: /* dst <<= src; */
0565             EMIT(PPC_RAW_SLD(dst_reg, dst_reg, src_reg));
0566             break;
0567         case BPF_ALU | BPF_LSH | BPF_K: /* (u32) dst <<== (u32) imm */
0568             /* with imm 0, we still need to clear top 32 bits */
0569             EMIT(PPC_RAW_SLWI(dst_reg, dst_reg, imm));
0570             if (insn_is_zext(&insn[i + 1]))
0571                 addrs[++i] = ctx->idx * 4;
0572             break;
0573         case BPF_ALU64 | BPF_LSH | BPF_K: /* dst <<== imm */
0574             if (imm != 0)
0575                 EMIT(PPC_RAW_SLDI(dst_reg, dst_reg, imm));
0576             break;
0577         case BPF_ALU | BPF_RSH | BPF_X: /* (u32) dst >>= (u32) src */
0578             EMIT(PPC_RAW_SRW(dst_reg, dst_reg, src_reg));
0579             if (insn_is_zext(&insn[i + 1]))
0580                 addrs[++i] = ctx->idx * 4;
0581             break;
0582         case BPF_ALU64 | BPF_RSH | BPF_X: /* dst >>= src */
0583             EMIT(PPC_RAW_SRD(dst_reg, dst_reg, src_reg));
0584             break;
0585         case BPF_ALU | BPF_RSH | BPF_K: /* (u32) dst >>= (u32) imm */
0586             EMIT(PPC_RAW_SRWI(dst_reg, dst_reg, imm));
0587             if (insn_is_zext(&insn[i + 1]))
0588                 addrs[++i] = ctx->idx * 4;
0589             break;
0590         case BPF_ALU64 | BPF_RSH | BPF_K: /* dst >>= imm */
0591             if (imm != 0)
0592                 EMIT(PPC_RAW_SRDI(dst_reg, dst_reg, imm));
0593             break;
0594         case BPF_ALU | BPF_ARSH | BPF_X: /* (s32) dst >>= src */
0595             EMIT(PPC_RAW_SRAW(dst_reg, dst_reg, src_reg));
0596             goto bpf_alu32_trunc;
0597         case BPF_ALU64 | BPF_ARSH | BPF_X: /* (s64) dst >>= src */
0598             EMIT(PPC_RAW_SRAD(dst_reg, dst_reg, src_reg));
0599             break;
0600         case BPF_ALU | BPF_ARSH | BPF_K: /* (s32) dst >>= imm */
0601             EMIT(PPC_RAW_SRAWI(dst_reg, dst_reg, imm));
0602             goto bpf_alu32_trunc;
0603         case BPF_ALU64 | BPF_ARSH | BPF_K: /* (s64) dst >>= imm */
0604             if (imm != 0)
0605                 EMIT(PPC_RAW_SRADI(dst_reg, dst_reg, imm));
0606             break;
0607 
0608         /*
0609          * MOV
0610          */
0611         case BPF_ALU | BPF_MOV | BPF_X: /* (u32) dst = src */
0612         case BPF_ALU64 | BPF_MOV | BPF_X: /* dst = src */
0613             if (imm == 1) {
0614                 /* special mov32 for zext */
0615                 EMIT(PPC_RAW_RLWINM(dst_reg, dst_reg, 0, 0, 31));
0616                 break;
0617             }
0618             EMIT(PPC_RAW_MR(dst_reg, src_reg));
0619             goto bpf_alu32_trunc;
0620         case BPF_ALU | BPF_MOV | BPF_K: /* (u32) dst = imm */
0621         case BPF_ALU64 | BPF_MOV | BPF_K: /* dst = (s64) imm */
0622             PPC_LI32(dst_reg, imm);
0623             if (imm < 0)
0624                 goto bpf_alu32_trunc;
0625             else if (insn_is_zext(&insn[i + 1]))
0626                 addrs[++i] = ctx->idx * 4;
0627             break;
0628 
0629 bpf_alu32_trunc:
0630         /* Truncate to 32-bits */
0631         if (BPF_CLASS(code) == BPF_ALU && !fp->aux->verifier_zext)
0632             EMIT(PPC_RAW_RLWINM(dst_reg, dst_reg, 0, 0, 31));
0633         break;
0634 
0635         /*
0636          * BPF_FROM_BE/LE
0637          */
0638         case BPF_ALU | BPF_END | BPF_FROM_LE:
0639         case BPF_ALU | BPF_END | BPF_FROM_BE:
0640 #ifdef __BIG_ENDIAN__
0641             if (BPF_SRC(code) == BPF_FROM_BE)
0642                 goto emit_clear;
0643 #else /* !__BIG_ENDIAN__ */
0644             if (BPF_SRC(code) == BPF_FROM_LE)
0645                 goto emit_clear;
0646 #endif
0647             switch (imm) {
0648             case 16:
0649                 /* Rotate 8 bits left & mask with 0x0000ff00 */
0650                 EMIT(PPC_RAW_RLWINM(tmp1_reg, dst_reg, 8, 16, 23));
0651                 /* Rotate 8 bits right & insert LSB to reg */
0652                 EMIT(PPC_RAW_RLWIMI(tmp1_reg, dst_reg, 24, 24, 31));
0653                 /* Move result back to dst_reg */
0654                 EMIT(PPC_RAW_MR(dst_reg, tmp1_reg));
0655                 break;
0656             case 32:
0657                 /*
0658                  * Rotate word left by 8 bits:
0659                  * 2 bytes are already in their final position
0660                  * -- byte 2 and 4 (of bytes 1, 2, 3 and 4)
0661                  */
0662                 EMIT(PPC_RAW_RLWINM(tmp1_reg, dst_reg, 8, 0, 31));
0663                 /* Rotate 24 bits and insert byte 1 */
0664                 EMIT(PPC_RAW_RLWIMI(tmp1_reg, dst_reg, 24, 0, 7));
0665                 /* Rotate 24 bits and insert byte 3 */
0666                 EMIT(PPC_RAW_RLWIMI(tmp1_reg, dst_reg, 24, 16, 23));
0667                 EMIT(PPC_RAW_MR(dst_reg, tmp1_reg));
0668                 break;
0669             case 64:
0670                 /* Store the value to stack and then use byte-reverse loads */
0671                 EMIT(PPC_RAW_STD(dst_reg, _R1, bpf_jit_stack_local(ctx)));
0672                 EMIT(PPC_RAW_ADDI(tmp1_reg, _R1, bpf_jit_stack_local(ctx)));
0673                 if (cpu_has_feature(CPU_FTR_ARCH_206)) {
0674                     EMIT(PPC_RAW_LDBRX(dst_reg, 0, tmp1_reg));
0675                 } else {
0676                     EMIT(PPC_RAW_LWBRX(dst_reg, 0, tmp1_reg));
0677                     if (IS_ENABLED(CONFIG_CPU_LITTLE_ENDIAN))
0678                         EMIT(PPC_RAW_SLDI(dst_reg, dst_reg, 32));
0679                     EMIT(PPC_RAW_LI(tmp2_reg, 4));
0680                     EMIT(PPC_RAW_LWBRX(tmp2_reg, tmp2_reg, tmp1_reg));
0681                     if (IS_ENABLED(CONFIG_CPU_BIG_ENDIAN))
0682                         EMIT(PPC_RAW_SLDI(tmp2_reg, tmp2_reg, 32));
0683                     EMIT(PPC_RAW_OR(dst_reg, dst_reg, tmp2_reg));
0684                 }
0685                 break;
0686             }
0687             break;
0688 
0689 emit_clear:
0690             switch (imm) {
0691             case 16:
0692                 /* zero-extend 16 bits into 64 bits */
0693                 EMIT(PPC_RAW_RLDICL(dst_reg, dst_reg, 0, 48));
0694                 if (insn_is_zext(&insn[i + 1]))
0695                     addrs[++i] = ctx->idx * 4;
0696                 break;
0697             case 32:
0698                 if (!fp->aux->verifier_zext)
0699                     /* zero-extend 32 bits into 64 bits */
0700                     EMIT(PPC_RAW_RLDICL(dst_reg, dst_reg, 0, 32));
0701                 break;
0702             case 64:
0703                 /* nop */
0704                 break;
0705             }
0706             break;
0707 
0708         /*
0709          * BPF_ST NOSPEC (speculation barrier)
0710          */
0711         case BPF_ST | BPF_NOSPEC:
0712             if (!security_ftr_enabled(SEC_FTR_FAVOUR_SECURITY) ||
0713                     !security_ftr_enabled(SEC_FTR_STF_BARRIER))
0714                 break;
0715 
0716             switch (stf_barrier) {
0717             case STF_BARRIER_EIEIO:
0718                 EMIT(PPC_RAW_EIEIO() | 0x02000000);
0719                 break;
0720             case STF_BARRIER_SYNC_ORI:
0721                 EMIT(PPC_RAW_SYNC());
0722                 EMIT(PPC_RAW_LD(tmp1_reg, _R13, 0));
0723                 EMIT(PPC_RAW_ORI(_R31, _R31, 0));
0724                 break;
0725             case STF_BARRIER_FALLBACK:
0726                 ctx->seen |= SEEN_FUNC;
0727                 PPC_LI64(_R12, dereference_kernel_function_descriptor(bpf_stf_barrier));
0728                 EMIT(PPC_RAW_MTCTR(_R12));
0729                 EMIT(PPC_RAW_BCTRL());
0730                 break;
0731             case STF_BARRIER_NONE:
0732                 break;
0733             }
0734             break;
0735 
0736         /*
0737          * BPF_ST(X)
0738          */
0739         case BPF_STX | BPF_MEM | BPF_B: /* *(u8 *)(dst + off) = src */
0740         case BPF_ST | BPF_MEM | BPF_B: /* *(u8 *)(dst + off) = imm */
0741             if (BPF_CLASS(code) == BPF_ST) {
0742                 EMIT(PPC_RAW_LI(tmp1_reg, imm));
0743                 src_reg = tmp1_reg;
0744             }
0745             EMIT(PPC_RAW_STB(src_reg, dst_reg, off));
0746             break;
0747         case BPF_STX | BPF_MEM | BPF_H: /* (u16 *)(dst + off) = src */
0748         case BPF_ST | BPF_MEM | BPF_H: /* (u16 *)(dst + off) = imm */
0749             if (BPF_CLASS(code) == BPF_ST) {
0750                 EMIT(PPC_RAW_LI(tmp1_reg, imm));
0751                 src_reg = tmp1_reg;
0752             }
0753             EMIT(PPC_RAW_STH(src_reg, dst_reg, off));
0754             break;
0755         case BPF_STX | BPF_MEM | BPF_W: /* *(u32 *)(dst + off) = src */
0756         case BPF_ST | BPF_MEM | BPF_W: /* *(u32 *)(dst + off) = imm */
0757             if (BPF_CLASS(code) == BPF_ST) {
0758                 PPC_LI32(tmp1_reg, imm);
0759                 src_reg = tmp1_reg;
0760             }
0761             EMIT(PPC_RAW_STW(src_reg, dst_reg, off));
0762             break;
0763         case BPF_STX | BPF_MEM | BPF_DW: /* (u64 *)(dst + off) = src */
0764         case BPF_ST | BPF_MEM | BPF_DW: /* *(u64 *)(dst + off) = imm */
0765             if (BPF_CLASS(code) == BPF_ST) {
0766                 PPC_LI32(tmp1_reg, imm);
0767                 src_reg = tmp1_reg;
0768             }
0769             if (off % 4) {
0770                 EMIT(PPC_RAW_LI(tmp2_reg, off));
0771                 EMIT(PPC_RAW_STDX(src_reg, dst_reg, tmp2_reg));
0772             } else {
0773                 EMIT(PPC_RAW_STD(src_reg, dst_reg, off));
0774             }
0775             break;
0776 
0777         /*
0778          * BPF_STX ATOMIC (atomic ops)
0779          */
0780         case BPF_STX | BPF_ATOMIC | BPF_W:
0781         case BPF_STX | BPF_ATOMIC | BPF_DW:
0782             save_reg = tmp2_reg;
0783             ret_reg = src_reg;
0784 
0785             /* Get offset into TMP_REG_1 */
0786             EMIT(PPC_RAW_LI(tmp1_reg, off));
0787             tmp_idx = ctx->idx * 4;
0788             /* load value from memory into TMP_REG_2 */
0789             if (size == BPF_DW)
0790                 EMIT(PPC_RAW_LDARX(tmp2_reg, tmp1_reg, dst_reg, 0));
0791             else
0792                 EMIT(PPC_RAW_LWARX(tmp2_reg, tmp1_reg, dst_reg, 0));
0793 
0794             /* Save old value in _R0 */
0795             if (imm & BPF_FETCH)
0796                 EMIT(PPC_RAW_MR(_R0, tmp2_reg));
0797 
0798             switch (imm) {
0799             case BPF_ADD:
0800             case BPF_ADD | BPF_FETCH:
0801                 EMIT(PPC_RAW_ADD(tmp2_reg, tmp2_reg, src_reg));
0802                 break;
0803             case BPF_AND:
0804             case BPF_AND | BPF_FETCH:
0805                 EMIT(PPC_RAW_AND(tmp2_reg, tmp2_reg, src_reg));
0806                 break;
0807             case BPF_OR:
0808             case BPF_OR | BPF_FETCH:
0809                 EMIT(PPC_RAW_OR(tmp2_reg, tmp2_reg, src_reg));
0810                 break;
0811             case BPF_XOR:
0812             case BPF_XOR | BPF_FETCH:
0813                 EMIT(PPC_RAW_XOR(tmp2_reg, tmp2_reg, src_reg));
0814                 break;
0815             case BPF_CMPXCHG:
0816                 /*
0817                  * Return old value in BPF_REG_0 for BPF_CMPXCHG &
0818                  * in src_reg for other cases.
0819                  */
0820                 ret_reg = bpf_to_ppc(BPF_REG_0);
0821 
0822                 /* Compare with old value in BPF_R0 */
0823                 if (size == BPF_DW)
0824                     EMIT(PPC_RAW_CMPD(bpf_to_ppc(BPF_REG_0), tmp2_reg));
0825                 else
0826                     EMIT(PPC_RAW_CMPW(bpf_to_ppc(BPF_REG_0), tmp2_reg));
0827                 /* Don't set if different from old value */
0828                 PPC_BCC_SHORT(COND_NE, (ctx->idx + 3) * 4);
0829                 fallthrough;
0830             case BPF_XCHG:
0831                 save_reg = src_reg;
0832                 break;
0833             default:
0834                 pr_err_ratelimited(
0835                     "eBPF filter atomic op code %02x (@%d) unsupported\n",
0836                     code, i);
0837                 return -EOPNOTSUPP;
0838             }
0839 
0840             /* store new value */
0841             if (size == BPF_DW)
0842                 EMIT(PPC_RAW_STDCX(save_reg, tmp1_reg, dst_reg));
0843             else
0844                 EMIT(PPC_RAW_STWCX(save_reg, tmp1_reg, dst_reg));
0845             /* we're done if this succeeded */
0846             PPC_BCC_SHORT(COND_NE, tmp_idx);
0847 
0848             if (imm & BPF_FETCH) {
0849                 EMIT(PPC_RAW_MR(ret_reg, _R0));
0850                 /*
0851                  * Skip unnecessary zero-extension for 32-bit cmpxchg.
0852                  * For context, see commit 39491867ace5.
0853                  */
0854                 if (size != BPF_DW && imm == BPF_CMPXCHG &&
0855                     insn_is_zext(&insn[i + 1]))
0856                     addrs[++i] = ctx->idx * 4;
0857             }
0858             break;
0859 
0860         /*
0861          * BPF_LDX
0862          */
0863         /* dst = *(u8 *)(ul) (src + off) */
0864         case BPF_LDX | BPF_MEM | BPF_B:
0865         case BPF_LDX | BPF_PROBE_MEM | BPF_B:
0866         /* dst = *(u16 *)(ul) (src + off) */
0867         case BPF_LDX | BPF_MEM | BPF_H:
0868         case BPF_LDX | BPF_PROBE_MEM | BPF_H:
0869         /* dst = *(u32 *)(ul) (src + off) */
0870         case BPF_LDX | BPF_MEM | BPF_W:
0871         case BPF_LDX | BPF_PROBE_MEM | BPF_W:
0872         /* dst = *(u64 *)(ul) (src + off) */
0873         case BPF_LDX | BPF_MEM | BPF_DW:
0874         case BPF_LDX | BPF_PROBE_MEM | BPF_DW:
0875             /*
0876              * As PTR_TO_BTF_ID that uses BPF_PROBE_MEM mode could either be a valid
0877              * kernel pointer or NULL but not a userspace address, execute BPF_PROBE_MEM
0878              * load only if addr is kernel address (see is_kernel_addr()), otherwise
0879              * set dst_reg=0 and move on.
0880              */
0881             if (BPF_MODE(code) == BPF_PROBE_MEM) {
0882                 EMIT(PPC_RAW_ADDI(tmp1_reg, src_reg, off));
0883                 if (IS_ENABLED(CONFIG_PPC_BOOK3E_64))
0884                     PPC_LI64(tmp2_reg, 0x8000000000000000ul);
0885                 else /* BOOK3S_64 */
0886                     PPC_LI64(tmp2_reg, PAGE_OFFSET);
0887                 EMIT(PPC_RAW_CMPLD(tmp1_reg, tmp2_reg));
0888                 PPC_BCC_SHORT(COND_GT, (ctx->idx + 3) * 4);
0889                 EMIT(PPC_RAW_LI(dst_reg, 0));
0890                 /*
0891                  * Check if 'off' is word aligned for BPF_DW, because
0892                  * we might generate two instructions.
0893                  */
0894                 if (BPF_SIZE(code) == BPF_DW && (off & 3))
0895                     PPC_JMP((ctx->idx + 3) * 4);
0896                 else
0897                     PPC_JMP((ctx->idx + 2) * 4);
0898             }
0899 
0900             switch (size) {
0901             case BPF_B:
0902                 EMIT(PPC_RAW_LBZ(dst_reg, src_reg, off));
0903                 break;
0904             case BPF_H:
0905                 EMIT(PPC_RAW_LHZ(dst_reg, src_reg, off));
0906                 break;
0907             case BPF_W:
0908                 EMIT(PPC_RAW_LWZ(dst_reg, src_reg, off));
0909                 break;
0910             case BPF_DW:
0911                 if (off % 4) {
0912                     EMIT(PPC_RAW_LI(tmp1_reg, off));
0913                     EMIT(PPC_RAW_LDX(dst_reg, src_reg, tmp1_reg));
0914                 } else {
0915                     EMIT(PPC_RAW_LD(dst_reg, src_reg, off));
0916                 }
0917                 break;
0918             }
0919 
0920             if (size != BPF_DW && insn_is_zext(&insn[i + 1]))
0921                 addrs[++i] = ctx->idx * 4;
0922 
0923             if (BPF_MODE(code) == BPF_PROBE_MEM) {
0924                 ret = bpf_add_extable_entry(fp, image, pass, ctx, ctx->idx - 1,
0925                                 4, dst_reg);
0926                 if (ret)
0927                     return ret;
0928             }
0929             break;
0930 
0931         /*
0932          * Doubleword load
0933          * 16 byte instruction that uses two 'struct bpf_insn'
0934          */
0935         case BPF_LD | BPF_IMM | BPF_DW: /* dst = (u64) imm */
0936             imm64 = ((u64)(u32) insn[i].imm) |
0937                     (((u64)(u32) insn[i+1].imm) << 32);
0938             tmp_idx = ctx->idx;
0939             PPC_LI64(dst_reg, imm64);
0940             /* padding to allow full 5 instructions for later patching */
0941             for (j = ctx->idx - tmp_idx; j < 5; j++)
0942                 EMIT(PPC_RAW_NOP());
0943             /* Adjust for two bpf instructions */
0944             addrs[++i] = ctx->idx * 4;
0945             break;
0946 
0947         /*
0948          * Return/Exit
0949          */
0950         case BPF_JMP | BPF_EXIT:
0951             /*
0952              * If this isn't the very last instruction, branch to
0953              * the epilogue. If we _are_ the last instruction,
0954              * we'll just fall through to the epilogue.
0955              */
0956             if (i != flen - 1) {
0957                 ret = bpf_jit_emit_exit_insn(image, ctx, tmp1_reg, exit_addr);
0958                 if (ret)
0959                     return ret;
0960             }
0961             /* else fall through to the epilogue */
0962             break;
0963 
0964         /*
0965          * Call kernel helper or bpf function
0966          */
0967         case BPF_JMP | BPF_CALL:
0968             ctx->seen |= SEEN_FUNC;
0969 
0970             ret = bpf_jit_get_func_addr(fp, &insn[i], false,
0971                             &func_addr, &func_addr_fixed);
0972             if (ret < 0)
0973                 return ret;
0974 
0975             if (func_addr_fixed)
0976                 ret = bpf_jit_emit_func_call_hlp(image, ctx, func_addr);
0977             else
0978                 ret = bpf_jit_emit_func_call_rel(image, ctx, func_addr);
0979 
0980             if (ret)
0981                 return ret;
0982 
0983             /* move return value from r3 to BPF_REG_0 */
0984             EMIT(PPC_RAW_MR(bpf_to_ppc(BPF_REG_0), _R3));
0985             break;
0986 
0987         /*
0988          * Jumps and branches
0989          */
0990         case BPF_JMP | BPF_JA:
0991             PPC_JMP(addrs[i + 1 + off]);
0992             break;
0993 
0994         case BPF_JMP | BPF_JGT | BPF_K:
0995         case BPF_JMP | BPF_JGT | BPF_X:
0996         case BPF_JMP | BPF_JSGT | BPF_K:
0997         case BPF_JMP | BPF_JSGT | BPF_X:
0998         case BPF_JMP32 | BPF_JGT | BPF_K:
0999         case BPF_JMP32 | BPF_JGT | BPF_X:
1000         case BPF_JMP32 | BPF_JSGT | BPF_K:
1001         case BPF_JMP32 | BPF_JSGT | BPF_X:
1002             true_cond = COND_GT;
1003             goto cond_branch;
1004         case BPF_JMP | BPF_JLT | BPF_K:
1005         case BPF_JMP | BPF_JLT | BPF_X:
1006         case BPF_JMP | BPF_JSLT | BPF_K:
1007         case BPF_JMP | BPF_JSLT | BPF_X:
1008         case BPF_JMP32 | BPF_JLT | BPF_K:
1009         case BPF_JMP32 | BPF_JLT | BPF_X:
1010         case BPF_JMP32 | BPF_JSLT | BPF_K:
1011         case BPF_JMP32 | BPF_JSLT | BPF_X:
1012             true_cond = COND_LT;
1013             goto cond_branch;
1014         case BPF_JMP | BPF_JGE | BPF_K:
1015         case BPF_JMP | BPF_JGE | BPF_X:
1016         case BPF_JMP | BPF_JSGE | BPF_K:
1017         case BPF_JMP | BPF_JSGE | BPF_X:
1018         case BPF_JMP32 | BPF_JGE | BPF_K:
1019         case BPF_JMP32 | BPF_JGE | BPF_X:
1020         case BPF_JMP32 | BPF_JSGE | BPF_K:
1021         case BPF_JMP32 | BPF_JSGE | BPF_X:
1022             true_cond = COND_GE;
1023             goto cond_branch;
1024         case BPF_JMP | BPF_JLE | BPF_K:
1025         case BPF_JMP | BPF_JLE | BPF_X:
1026         case BPF_JMP | BPF_JSLE | BPF_K:
1027         case BPF_JMP | BPF_JSLE | BPF_X:
1028         case BPF_JMP32 | BPF_JLE | BPF_K:
1029         case BPF_JMP32 | BPF_JLE | BPF_X:
1030         case BPF_JMP32 | BPF_JSLE | BPF_K:
1031         case BPF_JMP32 | BPF_JSLE | BPF_X:
1032             true_cond = COND_LE;
1033             goto cond_branch;
1034         case BPF_JMP | BPF_JEQ | BPF_K:
1035         case BPF_JMP | BPF_JEQ | BPF_X:
1036         case BPF_JMP32 | BPF_JEQ | BPF_K:
1037         case BPF_JMP32 | BPF_JEQ | BPF_X:
1038             true_cond = COND_EQ;
1039             goto cond_branch;
1040         case BPF_JMP | BPF_JNE | BPF_K:
1041         case BPF_JMP | BPF_JNE | BPF_X:
1042         case BPF_JMP32 | BPF_JNE | BPF_K:
1043         case BPF_JMP32 | BPF_JNE | BPF_X:
1044             true_cond = COND_NE;
1045             goto cond_branch;
1046         case BPF_JMP | BPF_JSET | BPF_K:
1047         case BPF_JMP | BPF_JSET | BPF_X:
1048         case BPF_JMP32 | BPF_JSET | BPF_K:
1049         case BPF_JMP32 | BPF_JSET | BPF_X:
1050             true_cond = COND_NE;
1051             /* Fall through */
1052 
1053 cond_branch:
1054             switch (code) {
1055             case BPF_JMP | BPF_JGT | BPF_X:
1056             case BPF_JMP | BPF_JLT | BPF_X:
1057             case BPF_JMP | BPF_JGE | BPF_X:
1058             case BPF_JMP | BPF_JLE | BPF_X:
1059             case BPF_JMP | BPF_JEQ | BPF_X:
1060             case BPF_JMP | BPF_JNE | BPF_X:
1061             case BPF_JMP32 | BPF_JGT | BPF_X:
1062             case BPF_JMP32 | BPF_JLT | BPF_X:
1063             case BPF_JMP32 | BPF_JGE | BPF_X:
1064             case BPF_JMP32 | BPF_JLE | BPF_X:
1065             case BPF_JMP32 | BPF_JEQ | BPF_X:
1066             case BPF_JMP32 | BPF_JNE | BPF_X:
1067                 /* unsigned comparison */
1068                 if (BPF_CLASS(code) == BPF_JMP32)
1069                     EMIT(PPC_RAW_CMPLW(dst_reg, src_reg));
1070                 else
1071                     EMIT(PPC_RAW_CMPLD(dst_reg, src_reg));
1072                 break;
1073             case BPF_JMP | BPF_JSGT | BPF_X:
1074             case BPF_JMP | BPF_JSLT | BPF_X:
1075             case BPF_JMP | BPF_JSGE | BPF_X:
1076             case BPF_JMP | BPF_JSLE | BPF_X:
1077             case BPF_JMP32 | BPF_JSGT | BPF_X:
1078             case BPF_JMP32 | BPF_JSLT | BPF_X:
1079             case BPF_JMP32 | BPF_JSGE | BPF_X:
1080             case BPF_JMP32 | BPF_JSLE | BPF_X:
1081                 /* signed comparison */
1082                 if (BPF_CLASS(code) == BPF_JMP32)
1083                     EMIT(PPC_RAW_CMPW(dst_reg, src_reg));
1084                 else
1085                     EMIT(PPC_RAW_CMPD(dst_reg, src_reg));
1086                 break;
1087             case BPF_JMP | BPF_JSET | BPF_X:
1088             case BPF_JMP32 | BPF_JSET | BPF_X:
1089                 if (BPF_CLASS(code) == BPF_JMP) {
1090                     EMIT(PPC_RAW_AND_DOT(tmp1_reg, dst_reg, src_reg));
1091                 } else {
1092                     EMIT(PPC_RAW_AND(tmp1_reg, dst_reg, src_reg));
1093                     EMIT(PPC_RAW_RLWINM_DOT(tmp1_reg, tmp1_reg, 0, 0, 31));
1094                 }
1095                 break;
1096             case BPF_JMP | BPF_JNE | BPF_K:
1097             case BPF_JMP | BPF_JEQ | BPF_K:
1098             case BPF_JMP | BPF_JGT | BPF_K:
1099             case BPF_JMP | BPF_JLT | BPF_K:
1100             case BPF_JMP | BPF_JGE | BPF_K:
1101             case BPF_JMP | BPF_JLE | BPF_K:
1102             case BPF_JMP32 | BPF_JNE | BPF_K:
1103             case BPF_JMP32 | BPF_JEQ | BPF_K:
1104             case BPF_JMP32 | BPF_JGT | BPF_K:
1105             case BPF_JMP32 | BPF_JLT | BPF_K:
1106             case BPF_JMP32 | BPF_JGE | BPF_K:
1107             case BPF_JMP32 | BPF_JLE | BPF_K:
1108             {
1109                 bool is_jmp32 = BPF_CLASS(code) == BPF_JMP32;
1110 
1111                 /*
1112                  * Need sign-extended load, so only positive
1113                  * values can be used as imm in cmpldi
1114                  */
1115                 if (imm >= 0 && imm < 32768) {
1116                     if (is_jmp32)
1117                         EMIT(PPC_RAW_CMPLWI(dst_reg, imm));
1118                     else
1119                         EMIT(PPC_RAW_CMPLDI(dst_reg, imm));
1120                 } else {
1121                     /* sign-extending load */
1122                     PPC_LI32(tmp1_reg, imm);
1123                     /* ... but unsigned comparison */
1124                     if (is_jmp32)
1125                         EMIT(PPC_RAW_CMPLW(dst_reg, tmp1_reg));
1126                     else
1127                         EMIT(PPC_RAW_CMPLD(dst_reg, tmp1_reg));
1128                 }
1129                 break;
1130             }
1131             case BPF_JMP | BPF_JSGT | BPF_K:
1132             case BPF_JMP | BPF_JSLT | BPF_K:
1133             case BPF_JMP | BPF_JSGE | BPF_K:
1134             case BPF_JMP | BPF_JSLE | BPF_K:
1135             case BPF_JMP32 | BPF_JSGT | BPF_K:
1136             case BPF_JMP32 | BPF_JSLT | BPF_K:
1137             case BPF_JMP32 | BPF_JSGE | BPF_K:
1138             case BPF_JMP32 | BPF_JSLE | BPF_K:
1139             {
1140                 bool is_jmp32 = BPF_CLASS(code) == BPF_JMP32;
1141 
1142                 /*
1143                  * signed comparison, so any 16-bit value
1144                  * can be used in cmpdi
1145                  */
1146                 if (imm >= -32768 && imm < 32768) {
1147                     if (is_jmp32)
1148                         EMIT(PPC_RAW_CMPWI(dst_reg, imm));
1149                     else
1150                         EMIT(PPC_RAW_CMPDI(dst_reg, imm));
1151                 } else {
1152                     PPC_LI32(tmp1_reg, imm);
1153                     if (is_jmp32)
1154                         EMIT(PPC_RAW_CMPW(dst_reg, tmp1_reg));
1155                     else
1156                         EMIT(PPC_RAW_CMPD(dst_reg, tmp1_reg));
1157                 }
1158                 break;
1159             }
1160             case BPF_JMP | BPF_JSET | BPF_K:
1161             case BPF_JMP32 | BPF_JSET | BPF_K:
1162                 /* andi does not sign-extend the immediate */
1163                 if (imm >= 0 && imm < 32768)
1164                     /* PPC_ANDI is _only/always_ dot-form */
1165                     EMIT(PPC_RAW_ANDI(tmp1_reg, dst_reg, imm));
1166                 else {
1167                     PPC_LI32(tmp1_reg, imm);
1168                     if (BPF_CLASS(code) == BPF_JMP) {
1169                         EMIT(PPC_RAW_AND_DOT(tmp1_reg, dst_reg,
1170                                      tmp1_reg));
1171                     } else {
1172                         EMIT(PPC_RAW_AND(tmp1_reg, dst_reg, tmp1_reg));
1173                         EMIT(PPC_RAW_RLWINM_DOT(tmp1_reg, tmp1_reg,
1174                                     0, 0, 31));
1175                     }
1176                 }
1177                 break;
1178             }
1179             PPC_BCC(true_cond, addrs[i + 1 + off]);
1180             break;
1181 
1182         /*
1183          * Tail call
1184          */
1185         case BPF_JMP | BPF_TAIL_CALL:
1186             ctx->seen |= SEEN_TAILCALL;
1187             ret = bpf_jit_emit_tail_call(image, ctx, addrs[i + 1]);
1188             if (ret < 0)
1189                 return ret;
1190             break;
1191 
1192         default:
1193             /*
1194              * The filter contains something cruel & unusual.
1195              * We don't handle it, but also there shouldn't be
1196              * anything missing from our list.
1197              */
1198             pr_err_ratelimited("eBPF filter opcode %04x (@%d) unsupported\n",
1199                     code, i);
1200             return -ENOTSUPP;
1201         }
1202     }
1203 
1204     /* Set end-of-body-code address for exit. */
1205     addrs[i] = ctx->idx * 4;
1206 
1207     return 0;
1208 }
[ Source navigation ] [ Diff markup ] [ Identifier search ] [ general search ]

This page was automatically generated by the 2.1.0 LXR engine.

The LXR team
Valid CSS 2.1! Valid HTML 4.01!