Back to home page

OSCL-LXR
 
 

     

0001 // SPDX-License-Identifier: GPL-2.0-only
0002 /*
0003  * eBPF JIT compiler for PPC32
0004  *
0005  * Copyright 2020 Christophe Leroy <christophe.leroy@csgroup.eu>
0006  *        CS GROUP France
0007  *
0008  * Based on PPC64 eBPF JIT compiler by Naveen N. Rao
0009  */
0010 #include <linux/moduleloader.h>
0011 #include <asm/cacheflush.h>
0012 #include <asm/asm-compat.h>
0013 #include <linux/netdevice.h>
0014 #include <linux/filter.h>
0015 #include <linux/if_vlan.h>
0016 #include <asm/kprobes.h>
0017 #include <linux/bpf.h>
0018 
0019 #include "bpf_jit.h"
0020 
0021 /*
0022  * Stack layout:
0023  *
0024  *      [   prev sp     ] <-------------
0025  *      [   nv gpr save area    ] 16 * 4    |
0026  * fp (r31) --> [   ebpf stack space    ] upto 512  |
0027  *      [     frame header  ] 16        |
0028  * sp (r1) ---> [    stack pointer  ] --------------
0029  */
0030 
0031 /* for gpr non volatile registers r17 to r31 (14) + tail call */
0032 #define BPF_PPC_STACK_SAVE  (15 * 4 + 4)
0033 /* stack frame, ensure this is quadword aligned */
0034 #define BPF_PPC_STACKFRAME(ctx) (STACK_FRAME_MIN_SIZE + BPF_PPC_STACK_SAVE + (ctx)->stack_size)
0035 
0036 #define PPC_EX32(r, i)      EMIT(PPC_RAW_LI((r), (i) < 0 ? -1 : 0))
0037 
0038 /* PPC NVR range -- update this if we ever use NVRs below r17 */
0039 #define BPF_PPC_NVR_MIN     _R17
0040 #define BPF_PPC_TC      _R16
0041 
0042 /* BPF register usage */
0043 #define TMP_REG         (MAX_BPF_JIT_REG + 0)
0044 
0045 /* BPF to ppc register mappings */
0046 void bpf_jit_init_reg_mapping(struct codegen_context *ctx)
0047 {
0048     /* function return value */
0049     ctx->b2p[BPF_REG_0] = _R12;
0050     /* function arguments */
0051     ctx->b2p[BPF_REG_1] = _R4;
0052     ctx->b2p[BPF_REG_2] = _R6;
0053     ctx->b2p[BPF_REG_3] = _R8;
0054     ctx->b2p[BPF_REG_4] = _R10;
0055     ctx->b2p[BPF_REG_5] = _R22;
0056     /* non volatile registers */
0057     ctx->b2p[BPF_REG_6] = _R24;
0058     ctx->b2p[BPF_REG_7] = _R26;
0059     ctx->b2p[BPF_REG_8] = _R28;
0060     ctx->b2p[BPF_REG_9] = _R30;
0061     /* frame pointer aka BPF_REG_10 */
0062     ctx->b2p[BPF_REG_FP] = _R18;
0063     /* eBPF jit internal registers */
0064     ctx->b2p[BPF_REG_AX] = _R20;
0065     ctx->b2p[TMP_REG] = _R31;       /* 32 bits */
0066 }
0067 
0068 static int bpf_jit_stack_offsetof(struct codegen_context *ctx, int reg)
0069 {
0070     if ((reg >= BPF_PPC_NVR_MIN && reg < 32) || reg == BPF_PPC_TC)
0071         return BPF_PPC_STACKFRAME(ctx) - 4 * (32 - reg);
0072 
0073     WARN(true, "BPF JIT is asking about unknown registers, will crash the stack");
0074     /* Use the hole we have left for alignment */
0075     return BPF_PPC_STACKFRAME(ctx) - 4;
0076 }
0077 
0078 #define SEEN_VREG_MASK      0x1ff80000 /* Volatile registers r3-r12 */
0079 #define SEEN_NVREG_FULL_MASK    0x0003ffff /* Non volatile registers r14-r31 */
0080 #define SEEN_NVREG_TEMP_MASK    0x00001e01 /* BPF_REG_5, BPF_REG_AX, TMP_REG */
0081 
0082 void bpf_jit_realloc_regs(struct codegen_context *ctx)
0083 {
0084     unsigned int nvreg_mask;
0085 
0086     if (ctx->seen & SEEN_FUNC)
0087         nvreg_mask = SEEN_NVREG_TEMP_MASK;
0088     else
0089         nvreg_mask = SEEN_NVREG_FULL_MASK;
0090 
0091     while (ctx->seen & nvreg_mask &&
0092           (ctx->seen & SEEN_VREG_MASK) != SEEN_VREG_MASK) {
0093         int old = 32 - fls(ctx->seen & (nvreg_mask & 0xaaaaaaab));
0094         int new = 32 - fls(~ctx->seen & (SEEN_VREG_MASK & 0xaaaaaaaa));
0095         int i;
0096 
0097         for (i = BPF_REG_0; i <= TMP_REG; i++) {
0098             if (ctx->b2p[i] != old)
0099                 continue;
0100             ctx->b2p[i] = new;
0101             bpf_set_seen_register(ctx, new);
0102             bpf_clear_seen_register(ctx, old);
0103             if (i != TMP_REG) {
0104                 bpf_set_seen_register(ctx, new - 1);
0105                 bpf_clear_seen_register(ctx, old - 1);
0106             }
0107             break;
0108         }
0109     }
0110 }
0111 
0112 void bpf_jit_build_prologue(u32 *image, struct codegen_context *ctx)
0113 {
0114     int i;
0115 
0116     /* First arg comes in as a 32 bits pointer. */
0117     EMIT(PPC_RAW_MR(bpf_to_ppc(BPF_REG_1), _R3));
0118     EMIT(PPC_RAW_LI(bpf_to_ppc(BPF_REG_1) - 1, 0));
0119     EMIT(PPC_RAW_STWU(_R1, _R1, -BPF_PPC_STACKFRAME(ctx)));
0120 
0121     /*
0122      * Initialize tail_call_cnt in stack frame if we do tail calls.
0123      * Otherwise, put in NOPs so that it can be skipped when we are
0124      * invoked through a tail call.
0125      */
0126     if (ctx->seen & SEEN_TAILCALL)
0127         EMIT(PPC_RAW_STW(bpf_to_ppc(BPF_REG_1) - 1, _R1,
0128                  bpf_jit_stack_offsetof(ctx, BPF_PPC_TC)));
0129     else
0130         EMIT(PPC_RAW_NOP());
0131 
0132 #define BPF_TAILCALL_PROLOGUE_SIZE  16
0133 
0134     /*
0135      * We need a stack frame, but we don't necessarily need to
0136      * save/restore LR unless we call other functions
0137      */
0138     if (ctx->seen & SEEN_FUNC)
0139         EMIT(PPC_RAW_MFLR(_R0));
0140 
0141     /*
0142      * Back up non-volatile regs -- registers r18-r31
0143      */
0144     for (i = BPF_PPC_NVR_MIN; i <= 31; i++)
0145         if (bpf_is_seen_register(ctx, i))
0146             EMIT(PPC_RAW_STW(i, _R1, bpf_jit_stack_offsetof(ctx, i)));
0147 
0148     /* If needed retrieve arguments 9 and 10, ie 5th 64 bits arg.*/
0149     if (bpf_is_seen_register(ctx, bpf_to_ppc(BPF_REG_5))) {
0150         EMIT(PPC_RAW_LWZ(bpf_to_ppc(BPF_REG_5) - 1, _R1, BPF_PPC_STACKFRAME(ctx)) + 8);
0151         EMIT(PPC_RAW_LWZ(bpf_to_ppc(BPF_REG_5), _R1, BPF_PPC_STACKFRAME(ctx)) + 12);
0152     }
0153 
0154     /* Setup frame pointer to point to the bpf stack area */
0155     if (bpf_is_seen_register(ctx, bpf_to_ppc(BPF_REG_FP))) {
0156         EMIT(PPC_RAW_LI(bpf_to_ppc(BPF_REG_FP) - 1, 0));
0157         EMIT(PPC_RAW_ADDI(bpf_to_ppc(BPF_REG_FP), _R1,
0158                   STACK_FRAME_MIN_SIZE + ctx->stack_size));
0159     }
0160 
0161     if (ctx->seen & SEEN_FUNC)
0162         EMIT(PPC_RAW_STW(_R0, _R1, BPF_PPC_STACKFRAME(ctx) + PPC_LR_STKOFF));
0163 }
0164 
0165 static void bpf_jit_emit_common_epilogue(u32 *image, struct codegen_context *ctx)
0166 {
0167     int i;
0168 
0169     /* Restore NVRs */
0170     for (i = BPF_PPC_NVR_MIN; i <= 31; i++)
0171         if (bpf_is_seen_register(ctx, i))
0172             EMIT(PPC_RAW_LWZ(i, _R1, bpf_jit_stack_offsetof(ctx, i)));
0173 }
0174 
0175 void bpf_jit_build_epilogue(u32 *image, struct codegen_context *ctx)
0176 {
0177     EMIT(PPC_RAW_MR(_R3, bpf_to_ppc(BPF_REG_0)));
0178 
0179     bpf_jit_emit_common_epilogue(image, ctx);
0180 
0181     /* Tear down our stack frame */
0182 
0183     if (ctx->seen & SEEN_FUNC)
0184         EMIT(PPC_RAW_LWZ(_R0, _R1, BPF_PPC_STACKFRAME(ctx) + PPC_LR_STKOFF));
0185 
0186     EMIT(PPC_RAW_ADDI(_R1, _R1, BPF_PPC_STACKFRAME(ctx)));
0187 
0188     if (ctx->seen & SEEN_FUNC)
0189         EMIT(PPC_RAW_MTLR(_R0));
0190 
0191     EMIT(PPC_RAW_BLR());
0192 }
0193 
0194 int bpf_jit_emit_func_call_rel(u32 *image, struct codegen_context *ctx, u64 func)
0195 {
0196     s32 rel = (s32)func - (s32)(image + ctx->idx);
0197 
0198     if (image && rel < 0x2000000 && rel >= -0x2000000) {
0199         PPC_BL(func);
0200         EMIT(PPC_RAW_NOP());
0201         EMIT(PPC_RAW_NOP());
0202         EMIT(PPC_RAW_NOP());
0203     } else {
0204         /* Load function address into r0 */
0205         EMIT(PPC_RAW_LIS(_R0, IMM_H(func)));
0206         EMIT(PPC_RAW_ORI(_R0, _R0, IMM_L(func)));
0207         EMIT(PPC_RAW_MTCTR(_R0));
0208         EMIT(PPC_RAW_BCTRL());
0209     }
0210 
0211     return 0;
0212 }
0213 
0214 static int bpf_jit_emit_tail_call(u32 *image, struct codegen_context *ctx, u32 out)
0215 {
0216     /*
0217      * By now, the eBPF program has already setup parameters in r3-r6
0218      * r3-r4/BPF_REG_1 - pointer to ctx -- passed as is to the next bpf program
0219      * r5-r6/BPF_REG_2 - pointer to bpf_array
0220      * r7-r8/BPF_REG_3 - index in bpf_array
0221      */
0222     int b2p_bpf_array = bpf_to_ppc(BPF_REG_2);
0223     int b2p_index = bpf_to_ppc(BPF_REG_3);
0224 
0225     /*
0226      * if (index >= array->map.max_entries)
0227      *   goto out;
0228      */
0229     EMIT(PPC_RAW_LWZ(_R0, b2p_bpf_array, offsetof(struct bpf_array, map.max_entries)));
0230     EMIT(PPC_RAW_CMPLW(b2p_index, _R0));
0231     EMIT(PPC_RAW_LWZ(_R0, _R1, bpf_jit_stack_offsetof(ctx, BPF_PPC_TC)));
0232     PPC_BCC_SHORT(COND_GE, out);
0233 
0234     /*
0235      * if (tail_call_cnt >= MAX_TAIL_CALL_CNT)
0236      *   goto out;
0237      */
0238     EMIT(PPC_RAW_CMPLWI(_R0, MAX_TAIL_CALL_CNT));
0239     /* tail_call_cnt++; */
0240     EMIT(PPC_RAW_ADDIC(_R0, _R0, 1));
0241     PPC_BCC_SHORT(COND_GE, out);
0242 
0243     /* prog = array->ptrs[index]; */
0244     EMIT(PPC_RAW_RLWINM(_R3, b2p_index, 2, 0, 29));
0245     EMIT(PPC_RAW_ADD(_R3, _R3, b2p_bpf_array));
0246     EMIT(PPC_RAW_LWZ(_R3, _R3, offsetof(struct bpf_array, ptrs)));
0247     EMIT(PPC_RAW_STW(_R0, _R1, bpf_jit_stack_offsetof(ctx, BPF_PPC_TC)));
0248 
0249     /*
0250      * if (prog == NULL)
0251      *   goto out;
0252      */
0253     EMIT(PPC_RAW_CMPLWI(_R3, 0));
0254     PPC_BCC_SHORT(COND_EQ, out);
0255 
0256     /* goto *(prog->bpf_func + prologue_size); */
0257     EMIT(PPC_RAW_LWZ(_R3, _R3, offsetof(struct bpf_prog, bpf_func)));
0258 
0259     if (ctx->seen & SEEN_FUNC)
0260         EMIT(PPC_RAW_LWZ(_R0, _R1, BPF_PPC_STACKFRAME(ctx) + PPC_LR_STKOFF));
0261 
0262     EMIT(PPC_RAW_ADDIC(_R3, _R3, BPF_TAILCALL_PROLOGUE_SIZE));
0263 
0264     if (ctx->seen & SEEN_FUNC)
0265         EMIT(PPC_RAW_MTLR(_R0));
0266 
0267     EMIT(PPC_RAW_MTCTR(_R3));
0268 
0269     EMIT(PPC_RAW_MR(_R3, bpf_to_ppc(BPF_REG_1)));
0270 
0271     /* tear restore NVRs, ... */
0272     bpf_jit_emit_common_epilogue(image, ctx);
0273 
0274     EMIT(PPC_RAW_BCTR());
0275 
0276     /* out: */
0277     return 0;
0278 }
0279 
0280 /* Assemble the body code between the prologue & epilogue */
0281 int bpf_jit_build_body(struct bpf_prog *fp, u32 *image, struct codegen_context *ctx,
0282                u32 *addrs, int pass)
0283 {
0284     const struct bpf_insn *insn = fp->insnsi;
0285     int flen = fp->len;
0286     int i, ret;
0287 
0288     /* Start of epilogue code - will only be valid 2nd pass onwards */
0289     u32 exit_addr = addrs[flen];
0290 
0291     for (i = 0; i < flen; i++) {
0292         u32 code = insn[i].code;
0293         u32 dst_reg = bpf_to_ppc(insn[i].dst_reg);
0294         u32 dst_reg_h = dst_reg - 1;
0295         u32 src_reg = bpf_to_ppc(insn[i].src_reg);
0296         u32 src_reg_h = src_reg - 1;
0297         u32 ax_reg = bpf_to_ppc(BPF_REG_AX);
0298         u32 tmp_reg = bpf_to_ppc(TMP_REG);
0299         u32 size = BPF_SIZE(code);
0300         u32 save_reg, ret_reg;
0301         s16 off = insn[i].off;
0302         s32 imm = insn[i].imm;
0303         bool func_addr_fixed;
0304         u64 func_addr;
0305         u32 true_cond;
0306         u32 tmp_idx;
0307         int j;
0308 
0309         /*
0310          * addrs[] maps a BPF bytecode address into a real offset from
0311          * the start of the body code.
0312          */
0313         addrs[i] = ctx->idx * 4;
0314 
0315         /*
0316          * As an optimization, we note down which registers
0317          * are used so that we can only save/restore those in our
0318          * prologue and epilogue. We do this here regardless of whether
0319          * the actual BPF instruction uses src/dst registers or not
0320          * (for instance, BPF_CALL does not use them). The expectation
0321          * is that those instructions will have src_reg/dst_reg set to
0322          * 0. Even otherwise, we just lose some prologue/epilogue
0323          * optimization but everything else should work without
0324          * any issues.
0325          */
0326         if (dst_reg >= 3 && dst_reg < 32) {
0327             bpf_set_seen_register(ctx, dst_reg);
0328             bpf_set_seen_register(ctx, dst_reg_h);
0329         }
0330 
0331         if (src_reg >= 3 && src_reg < 32) {
0332             bpf_set_seen_register(ctx, src_reg);
0333             bpf_set_seen_register(ctx, src_reg_h);
0334         }
0335 
0336         switch (code) {
0337         /*
0338          * Arithmetic operations: ADD/SUB/MUL/DIV/MOD/NEG
0339          */
0340         case BPF_ALU | BPF_ADD | BPF_X: /* (u32) dst += (u32) src */
0341             EMIT(PPC_RAW_ADD(dst_reg, dst_reg, src_reg));
0342             break;
0343         case BPF_ALU64 | BPF_ADD | BPF_X: /* dst += src */
0344             EMIT(PPC_RAW_ADDC(dst_reg, dst_reg, src_reg));
0345             EMIT(PPC_RAW_ADDE(dst_reg_h, dst_reg_h, src_reg_h));
0346             break;
0347         case BPF_ALU | BPF_SUB | BPF_X: /* (u32) dst -= (u32) src */
0348             EMIT(PPC_RAW_SUB(dst_reg, dst_reg, src_reg));
0349             break;
0350         case BPF_ALU64 | BPF_SUB | BPF_X: /* dst -= src */
0351             EMIT(PPC_RAW_SUBFC(dst_reg, src_reg, dst_reg));
0352             EMIT(PPC_RAW_SUBFE(dst_reg_h, src_reg_h, dst_reg_h));
0353             break;
0354         case BPF_ALU | BPF_SUB | BPF_K: /* (u32) dst -= (u32) imm */
0355             imm = -imm;
0356             fallthrough;
0357         case BPF_ALU | BPF_ADD | BPF_K: /* (u32) dst += (u32) imm */
0358             if (IMM_HA(imm) & 0xffff)
0359                 EMIT(PPC_RAW_ADDIS(dst_reg, dst_reg, IMM_HA(imm)));
0360             if (IMM_L(imm))
0361                 EMIT(PPC_RAW_ADDI(dst_reg, dst_reg, IMM_L(imm)));
0362             break;
0363         case BPF_ALU64 | BPF_SUB | BPF_K: /* dst -= imm */
0364             imm = -imm;
0365             fallthrough;
0366         case BPF_ALU64 | BPF_ADD | BPF_K: /* dst += imm */
0367             if (!imm)
0368                 break;
0369 
0370             if (imm >= -32768 && imm < 32768) {
0371                 EMIT(PPC_RAW_ADDIC(dst_reg, dst_reg, imm));
0372             } else {
0373                 PPC_LI32(_R0, imm);
0374                 EMIT(PPC_RAW_ADDC(dst_reg, dst_reg, _R0));
0375             }
0376             if (imm >= 0 || (BPF_OP(code) == BPF_SUB && imm == 0x80000000))
0377                 EMIT(PPC_RAW_ADDZE(dst_reg_h, dst_reg_h));
0378             else
0379                 EMIT(PPC_RAW_ADDME(dst_reg_h, dst_reg_h));
0380             break;
0381         case BPF_ALU64 | BPF_MUL | BPF_X: /* dst *= src */
0382             bpf_set_seen_register(ctx, tmp_reg);
0383             EMIT(PPC_RAW_MULW(_R0, dst_reg, src_reg_h));
0384             EMIT(PPC_RAW_MULW(dst_reg_h, dst_reg_h, src_reg));
0385             EMIT(PPC_RAW_MULHWU(tmp_reg, dst_reg, src_reg));
0386             EMIT(PPC_RAW_MULW(dst_reg, dst_reg, src_reg));
0387             EMIT(PPC_RAW_ADD(dst_reg_h, dst_reg_h, _R0));
0388             EMIT(PPC_RAW_ADD(dst_reg_h, dst_reg_h, tmp_reg));
0389             break;
0390         case BPF_ALU | BPF_MUL | BPF_X: /* (u32) dst *= (u32) src */
0391             EMIT(PPC_RAW_MULW(dst_reg, dst_reg, src_reg));
0392             break;
0393         case BPF_ALU | BPF_MUL | BPF_K: /* (u32) dst *= (u32) imm */
0394             if (imm >= -32768 && imm < 32768) {
0395                 EMIT(PPC_RAW_MULI(dst_reg, dst_reg, imm));
0396             } else {
0397                 PPC_LI32(_R0, imm);
0398                 EMIT(PPC_RAW_MULW(dst_reg, dst_reg, _R0));
0399             }
0400             break;
0401         case BPF_ALU64 | BPF_MUL | BPF_K: /* dst *= imm */
0402             if (!imm) {
0403                 PPC_LI32(dst_reg, 0);
0404                 PPC_LI32(dst_reg_h, 0);
0405                 break;
0406             }
0407             if (imm == 1)
0408                 break;
0409             if (imm == -1) {
0410                 EMIT(PPC_RAW_SUBFIC(dst_reg, dst_reg, 0));
0411                 EMIT(PPC_RAW_SUBFZE(dst_reg_h, dst_reg_h));
0412                 break;
0413             }
0414             bpf_set_seen_register(ctx, tmp_reg);
0415             PPC_LI32(tmp_reg, imm);
0416             EMIT(PPC_RAW_MULW(dst_reg_h, dst_reg_h, tmp_reg));
0417             if (imm < 0)
0418                 EMIT(PPC_RAW_SUB(dst_reg_h, dst_reg_h, dst_reg));
0419             EMIT(PPC_RAW_MULHWU(_R0, dst_reg, tmp_reg));
0420             EMIT(PPC_RAW_MULW(dst_reg, dst_reg, tmp_reg));
0421             EMIT(PPC_RAW_ADD(dst_reg_h, dst_reg_h, _R0));
0422             break;
0423         case BPF_ALU | BPF_DIV | BPF_X: /* (u32) dst /= (u32) src */
0424             EMIT(PPC_RAW_DIVWU(dst_reg, dst_reg, src_reg));
0425             break;
0426         case BPF_ALU | BPF_MOD | BPF_X: /* (u32) dst %= (u32) src */
0427             EMIT(PPC_RAW_DIVWU(_R0, dst_reg, src_reg));
0428             EMIT(PPC_RAW_MULW(_R0, src_reg, _R0));
0429             EMIT(PPC_RAW_SUB(dst_reg, dst_reg, _R0));
0430             break;
0431         case BPF_ALU64 | BPF_DIV | BPF_X: /* dst /= src */
0432             return -EOPNOTSUPP;
0433         case BPF_ALU64 | BPF_MOD | BPF_X: /* dst %= src */
0434             return -EOPNOTSUPP;
0435         case BPF_ALU | BPF_DIV | BPF_K: /* (u32) dst /= (u32) imm */
0436             if (!imm)
0437                 return -EINVAL;
0438             if (imm == 1)
0439                 break;
0440 
0441             PPC_LI32(_R0, imm);
0442             EMIT(PPC_RAW_DIVWU(dst_reg, dst_reg, _R0));
0443             break;
0444         case BPF_ALU | BPF_MOD | BPF_K: /* (u32) dst %= (u32) imm */
0445             if (!imm)
0446                 return -EINVAL;
0447 
0448             if (!is_power_of_2((u32)imm)) {
0449                 bpf_set_seen_register(ctx, tmp_reg);
0450                 PPC_LI32(tmp_reg, imm);
0451                 EMIT(PPC_RAW_DIVWU(_R0, dst_reg, tmp_reg));
0452                 EMIT(PPC_RAW_MULW(_R0, tmp_reg, _R0));
0453                 EMIT(PPC_RAW_SUB(dst_reg, dst_reg, _R0));
0454                 break;
0455             }
0456             if (imm == 1)
0457                 EMIT(PPC_RAW_LI(dst_reg, 0));
0458             else
0459                 EMIT(PPC_RAW_RLWINM(dst_reg, dst_reg, 0, 32 - ilog2((u32)imm), 31));
0460 
0461             break;
0462         case BPF_ALU64 | BPF_MOD | BPF_K: /* dst %= imm */
0463             if (!imm)
0464                 return -EINVAL;
0465             if (imm < 0)
0466                 imm = -imm;
0467             if (!is_power_of_2(imm))
0468                 return -EOPNOTSUPP;
0469             if (imm == 1)
0470                 EMIT(PPC_RAW_LI(dst_reg, 0));
0471             else
0472                 EMIT(PPC_RAW_RLWINM(dst_reg, dst_reg, 0, 32 - ilog2(imm), 31));
0473             EMIT(PPC_RAW_LI(dst_reg_h, 0));
0474             break;
0475         case BPF_ALU64 | BPF_DIV | BPF_K: /* dst /= imm */
0476             if (!imm)
0477                 return -EINVAL;
0478             if (!is_power_of_2(abs(imm)))
0479                 return -EOPNOTSUPP;
0480 
0481             if (imm < 0) {
0482                 EMIT(PPC_RAW_SUBFIC(dst_reg, dst_reg, 0));
0483                 EMIT(PPC_RAW_SUBFZE(dst_reg_h, dst_reg_h));
0484                 imm = -imm;
0485             }
0486             if (imm == 1)
0487                 break;
0488             imm = ilog2(imm);
0489             EMIT(PPC_RAW_RLWINM(dst_reg, dst_reg, 32 - imm, imm, 31));
0490             EMIT(PPC_RAW_RLWIMI(dst_reg, dst_reg_h, 32 - imm, 0, imm - 1));
0491             EMIT(PPC_RAW_SRAWI(dst_reg_h, dst_reg_h, imm));
0492             break;
0493         case BPF_ALU | BPF_NEG: /* (u32) dst = -dst */
0494             EMIT(PPC_RAW_NEG(dst_reg, dst_reg));
0495             break;
0496         case BPF_ALU64 | BPF_NEG: /* dst = -dst */
0497             EMIT(PPC_RAW_SUBFIC(dst_reg, dst_reg, 0));
0498             EMIT(PPC_RAW_SUBFZE(dst_reg_h, dst_reg_h));
0499             break;
0500 
0501         /*
0502          * Logical operations: AND/OR/XOR/[A]LSH/[A]RSH
0503          */
0504         case BPF_ALU64 | BPF_AND | BPF_X: /* dst = dst & src */
0505             EMIT(PPC_RAW_AND(dst_reg, dst_reg, src_reg));
0506             EMIT(PPC_RAW_AND(dst_reg_h, dst_reg_h, src_reg_h));
0507             break;
0508         case BPF_ALU | BPF_AND | BPF_X: /* (u32) dst = dst & src */
0509             EMIT(PPC_RAW_AND(dst_reg, dst_reg, src_reg));
0510             break;
0511         case BPF_ALU64 | BPF_AND | BPF_K: /* dst = dst & imm */
0512             if (imm >= 0)
0513                 EMIT(PPC_RAW_LI(dst_reg_h, 0));
0514             fallthrough;
0515         case BPF_ALU | BPF_AND | BPF_K: /* (u32) dst = dst & imm */
0516             if (!IMM_H(imm)) {
0517                 EMIT(PPC_RAW_ANDI(dst_reg, dst_reg, IMM_L(imm)));
0518             } else if (!IMM_L(imm)) {
0519                 EMIT(PPC_RAW_ANDIS(dst_reg, dst_reg, IMM_H(imm)));
0520             } else if (imm == (((1 << fls(imm)) - 1) ^ ((1 << (ffs(i) - 1)) - 1))) {
0521                 EMIT(PPC_RAW_RLWINM(dst_reg, dst_reg, 0,
0522                             32 - fls(imm), 32 - ffs(imm)));
0523             } else {
0524                 PPC_LI32(_R0, imm);
0525                 EMIT(PPC_RAW_AND(dst_reg, dst_reg, _R0));
0526             }
0527             break;
0528         case BPF_ALU64 | BPF_OR | BPF_X: /* dst = dst | src */
0529             EMIT(PPC_RAW_OR(dst_reg, dst_reg, src_reg));
0530             EMIT(PPC_RAW_OR(dst_reg_h, dst_reg_h, src_reg_h));
0531             break;
0532         case BPF_ALU | BPF_OR | BPF_X: /* dst = (u32) dst | (u32) src */
0533             EMIT(PPC_RAW_OR(dst_reg, dst_reg, src_reg));
0534             break;
0535         case BPF_ALU64 | BPF_OR | BPF_K:/* dst = dst | imm */
0536             /* Sign-extended */
0537             if (imm < 0)
0538                 EMIT(PPC_RAW_LI(dst_reg_h, -1));
0539             fallthrough;
0540         case BPF_ALU | BPF_OR | BPF_K:/* dst = (u32) dst | (u32) imm */
0541             if (IMM_L(imm))
0542                 EMIT(PPC_RAW_ORI(dst_reg, dst_reg, IMM_L(imm)));
0543             if (IMM_H(imm))
0544                 EMIT(PPC_RAW_ORIS(dst_reg, dst_reg, IMM_H(imm)));
0545             break;
0546         case BPF_ALU64 | BPF_XOR | BPF_X: /* dst ^= src */
0547             if (dst_reg == src_reg) {
0548                 EMIT(PPC_RAW_LI(dst_reg, 0));
0549                 EMIT(PPC_RAW_LI(dst_reg_h, 0));
0550             } else {
0551                 EMIT(PPC_RAW_XOR(dst_reg, dst_reg, src_reg));
0552                 EMIT(PPC_RAW_XOR(dst_reg_h, dst_reg_h, src_reg_h));
0553             }
0554             break;
0555         case BPF_ALU | BPF_XOR | BPF_X: /* (u32) dst ^= src */
0556             if (dst_reg == src_reg)
0557                 EMIT(PPC_RAW_LI(dst_reg, 0));
0558             else
0559                 EMIT(PPC_RAW_XOR(dst_reg, dst_reg, src_reg));
0560             break;
0561         case BPF_ALU64 | BPF_XOR | BPF_K: /* dst ^= imm */
0562             if (imm < 0)
0563                 EMIT(PPC_RAW_NOR(dst_reg_h, dst_reg_h, dst_reg_h));
0564             fallthrough;
0565         case BPF_ALU | BPF_XOR | BPF_K: /* (u32) dst ^= (u32) imm */
0566             if (IMM_L(imm))
0567                 EMIT(PPC_RAW_XORI(dst_reg, dst_reg, IMM_L(imm)));
0568             if (IMM_H(imm))
0569                 EMIT(PPC_RAW_XORIS(dst_reg, dst_reg, IMM_H(imm)));
0570             break;
0571         case BPF_ALU | BPF_LSH | BPF_X: /* (u32) dst <<= (u32) src */
0572             EMIT(PPC_RAW_SLW(dst_reg, dst_reg, src_reg));
0573             break;
0574         case BPF_ALU64 | BPF_LSH | BPF_X: /* dst <<= src; */
0575             bpf_set_seen_register(ctx, tmp_reg);
0576             EMIT(PPC_RAW_SUBFIC(_R0, src_reg, 32));
0577             EMIT(PPC_RAW_SLW(dst_reg_h, dst_reg_h, src_reg));
0578             EMIT(PPC_RAW_ADDI(tmp_reg, src_reg, 32));
0579             EMIT(PPC_RAW_SRW(_R0, dst_reg, _R0));
0580             EMIT(PPC_RAW_SLW(tmp_reg, dst_reg, tmp_reg));
0581             EMIT(PPC_RAW_OR(dst_reg_h, dst_reg_h, _R0));
0582             EMIT(PPC_RAW_SLW(dst_reg, dst_reg, src_reg));
0583             EMIT(PPC_RAW_OR(dst_reg_h, dst_reg_h, tmp_reg));
0584             break;
0585         case BPF_ALU | BPF_LSH | BPF_K: /* (u32) dst <<= (u32) imm */
0586             if (!imm)
0587                 break;
0588             EMIT(PPC_RAW_SLWI(dst_reg, dst_reg, imm));
0589             break;
0590         case BPF_ALU64 | BPF_LSH | BPF_K: /* dst <<= imm */
0591             if (imm < 0)
0592                 return -EINVAL;
0593             if (!imm)
0594                 break;
0595             if (imm < 32) {
0596                 EMIT(PPC_RAW_RLWINM(dst_reg_h, dst_reg_h, imm, 0, 31 - imm));
0597                 EMIT(PPC_RAW_RLWIMI(dst_reg_h, dst_reg, imm, 32 - imm, 31));
0598                 EMIT(PPC_RAW_RLWINM(dst_reg, dst_reg, imm, 0, 31 - imm));
0599                 break;
0600             }
0601             if (imm < 64)
0602                 EMIT(PPC_RAW_RLWINM(dst_reg_h, dst_reg, imm, 0, 31 - imm));
0603             else
0604                 EMIT(PPC_RAW_LI(dst_reg_h, 0));
0605             EMIT(PPC_RAW_LI(dst_reg, 0));
0606             break;
0607         case BPF_ALU | BPF_RSH | BPF_X: /* (u32) dst >>= (u32) src */
0608             EMIT(PPC_RAW_SRW(dst_reg, dst_reg, src_reg));
0609             break;
0610         case BPF_ALU64 | BPF_RSH | BPF_X: /* dst >>= src */
0611             bpf_set_seen_register(ctx, tmp_reg);
0612             EMIT(PPC_RAW_SUBFIC(_R0, src_reg, 32));
0613             EMIT(PPC_RAW_SRW(dst_reg, dst_reg, src_reg));
0614             EMIT(PPC_RAW_ADDI(tmp_reg, src_reg, 32));
0615             EMIT(PPC_RAW_SLW(_R0, dst_reg_h, _R0));
0616             EMIT(PPC_RAW_SRW(tmp_reg, dst_reg_h, tmp_reg));
0617             EMIT(PPC_RAW_OR(dst_reg, dst_reg, _R0));
0618             EMIT(PPC_RAW_SRW(dst_reg_h, dst_reg_h, src_reg));
0619             EMIT(PPC_RAW_OR(dst_reg, dst_reg, tmp_reg));
0620             break;
0621         case BPF_ALU | BPF_RSH | BPF_K: /* (u32) dst >>= (u32) imm */
0622             if (!imm)
0623                 break;
0624             EMIT(PPC_RAW_SRWI(dst_reg, dst_reg, imm));
0625             break;
0626         case BPF_ALU64 | BPF_RSH | BPF_K: /* dst >>= imm */
0627             if (imm < 0)
0628                 return -EINVAL;
0629             if (!imm)
0630                 break;
0631             if (imm < 32) {
0632                 EMIT(PPC_RAW_RLWINM(dst_reg, dst_reg, 32 - imm, imm, 31));
0633                 EMIT(PPC_RAW_RLWIMI(dst_reg, dst_reg_h, 32 - imm, 0, imm - 1));
0634                 EMIT(PPC_RAW_RLWINM(dst_reg_h, dst_reg_h, 32 - imm, imm, 31));
0635                 break;
0636             }
0637             if (imm < 64)
0638                 EMIT(PPC_RAW_RLWINM(dst_reg, dst_reg_h, 64 - imm, imm - 32, 31));
0639             else
0640                 EMIT(PPC_RAW_LI(dst_reg, 0));
0641             EMIT(PPC_RAW_LI(dst_reg_h, 0));
0642             break;
0643         case BPF_ALU | BPF_ARSH | BPF_X: /* (s32) dst >>= src */
0644             EMIT(PPC_RAW_SRAW(dst_reg, dst_reg, src_reg));
0645             break;
0646         case BPF_ALU64 | BPF_ARSH | BPF_X: /* (s64) dst >>= src */
0647             bpf_set_seen_register(ctx, tmp_reg);
0648             EMIT(PPC_RAW_SUBFIC(_R0, src_reg, 32));
0649             EMIT(PPC_RAW_SRW(dst_reg, dst_reg, src_reg));
0650             EMIT(PPC_RAW_SLW(_R0, dst_reg_h, _R0));
0651             EMIT(PPC_RAW_ADDI(tmp_reg, src_reg, 32));
0652             EMIT(PPC_RAW_OR(dst_reg, dst_reg, _R0));
0653             EMIT(PPC_RAW_RLWINM(_R0, tmp_reg, 0, 26, 26));
0654             EMIT(PPC_RAW_SRAW(tmp_reg, dst_reg_h, tmp_reg));
0655             EMIT(PPC_RAW_SRAW(dst_reg_h, dst_reg_h, src_reg));
0656             EMIT(PPC_RAW_SLW(tmp_reg, tmp_reg, _R0));
0657             EMIT(PPC_RAW_OR(dst_reg, dst_reg, tmp_reg));
0658             break;
0659         case BPF_ALU | BPF_ARSH | BPF_K: /* (s32) dst >>= imm */
0660             if (!imm)
0661                 break;
0662             EMIT(PPC_RAW_SRAWI(dst_reg, dst_reg, imm));
0663             break;
0664         case BPF_ALU64 | BPF_ARSH | BPF_K: /* (s64) dst >>= imm */
0665             if (imm < 0)
0666                 return -EINVAL;
0667             if (!imm)
0668                 break;
0669             if (imm < 32) {
0670                 EMIT(PPC_RAW_RLWINM(dst_reg, dst_reg, 32 - imm, imm, 31));
0671                 EMIT(PPC_RAW_RLWIMI(dst_reg, dst_reg_h, 32 - imm, 0, imm - 1));
0672                 EMIT(PPC_RAW_SRAWI(dst_reg_h, dst_reg_h, imm));
0673                 break;
0674             }
0675             if (imm < 64)
0676                 EMIT(PPC_RAW_SRAWI(dst_reg, dst_reg_h, imm - 32));
0677             else
0678                 EMIT(PPC_RAW_SRAWI(dst_reg, dst_reg_h, 31));
0679             EMIT(PPC_RAW_SRAWI(dst_reg_h, dst_reg_h, 31));
0680             break;
0681 
0682         /*
0683          * MOV
0684          */
0685         case BPF_ALU64 | BPF_MOV | BPF_X: /* dst = src */
0686             if (dst_reg == src_reg)
0687                 break;
0688             EMIT(PPC_RAW_MR(dst_reg, src_reg));
0689             EMIT(PPC_RAW_MR(dst_reg_h, src_reg_h));
0690             break;
0691         case BPF_ALU | BPF_MOV | BPF_X: /* (u32) dst = src */
0692             /* special mov32 for zext */
0693             if (imm == 1)
0694                 EMIT(PPC_RAW_LI(dst_reg_h, 0));
0695             else if (dst_reg != src_reg)
0696                 EMIT(PPC_RAW_MR(dst_reg, src_reg));
0697             break;
0698         case BPF_ALU64 | BPF_MOV | BPF_K: /* dst = (s64) imm */
0699             PPC_LI32(dst_reg, imm);
0700             PPC_EX32(dst_reg_h, imm);
0701             break;
0702         case BPF_ALU | BPF_MOV | BPF_K: /* (u32) dst = imm */
0703             PPC_LI32(dst_reg, imm);
0704             break;
0705 
0706         /*
0707          * BPF_FROM_BE/LE
0708          */
0709         case BPF_ALU | BPF_END | BPF_FROM_LE:
0710             switch (imm) {
0711             case 16:
0712                 /* Copy 16 bits to upper part */
0713                 EMIT(PPC_RAW_RLWIMI(dst_reg, dst_reg, 16, 0, 15));
0714                 /* Rotate 8 bits right & mask */
0715                 EMIT(PPC_RAW_RLWINM(dst_reg, dst_reg, 24, 16, 31));
0716                 break;
0717             case 32:
0718                 /*
0719                  * Rotate word left by 8 bits:
0720                  * 2 bytes are already in their final position
0721                  * -- byte 2 and 4 (of bytes 1, 2, 3 and 4)
0722                  */
0723                 EMIT(PPC_RAW_RLWINM(_R0, dst_reg, 8, 0, 31));
0724                 /* Rotate 24 bits and insert byte 1 */
0725                 EMIT(PPC_RAW_RLWIMI(_R0, dst_reg, 24, 0, 7));
0726                 /* Rotate 24 bits and insert byte 3 */
0727                 EMIT(PPC_RAW_RLWIMI(_R0, dst_reg, 24, 16, 23));
0728                 EMIT(PPC_RAW_MR(dst_reg, _R0));
0729                 break;
0730             case 64:
0731                 bpf_set_seen_register(ctx, tmp_reg);
0732                 EMIT(PPC_RAW_RLWINM(tmp_reg, dst_reg, 8, 0, 31));
0733                 EMIT(PPC_RAW_RLWINM(_R0, dst_reg_h, 8, 0, 31));
0734                 /* Rotate 24 bits and insert byte 1 */
0735                 EMIT(PPC_RAW_RLWIMI(tmp_reg, dst_reg, 24, 0, 7));
0736                 EMIT(PPC_RAW_RLWIMI(_R0, dst_reg_h, 24, 0, 7));
0737                 /* Rotate 24 bits and insert byte 3 */
0738                 EMIT(PPC_RAW_RLWIMI(tmp_reg, dst_reg, 24, 16, 23));
0739                 EMIT(PPC_RAW_RLWIMI(_R0, dst_reg_h, 24, 16, 23));
0740                 EMIT(PPC_RAW_MR(dst_reg, _R0));
0741                 EMIT(PPC_RAW_MR(dst_reg_h, tmp_reg));
0742                 break;
0743             }
0744             break;
0745         case BPF_ALU | BPF_END | BPF_FROM_BE:
0746             switch (imm) {
0747             case 16:
0748                 /* zero-extend 16 bits into 32 bits */
0749                 EMIT(PPC_RAW_RLWINM(dst_reg, dst_reg, 0, 16, 31));
0750                 break;
0751             case 32:
0752             case 64:
0753                 /* nop */
0754                 break;
0755             }
0756             break;
0757 
0758         /*
0759          * BPF_ST NOSPEC (speculation barrier)
0760          */
0761         case BPF_ST | BPF_NOSPEC:
0762             break;
0763 
0764         /*
0765          * BPF_ST(X)
0766          */
0767         case BPF_STX | BPF_MEM | BPF_B: /* *(u8 *)(dst + off) = src */
0768             EMIT(PPC_RAW_STB(src_reg, dst_reg, off));
0769             break;
0770         case BPF_ST | BPF_MEM | BPF_B: /* *(u8 *)(dst + off) = imm */
0771             PPC_LI32(_R0, imm);
0772             EMIT(PPC_RAW_STB(_R0, dst_reg, off));
0773             break;
0774         case BPF_STX | BPF_MEM | BPF_H: /* (u16 *)(dst + off) = src */
0775             EMIT(PPC_RAW_STH(src_reg, dst_reg, off));
0776             break;
0777         case BPF_ST | BPF_MEM | BPF_H: /* (u16 *)(dst + off) = imm */
0778             PPC_LI32(_R0, imm);
0779             EMIT(PPC_RAW_STH(_R0, dst_reg, off));
0780             break;
0781         case BPF_STX | BPF_MEM | BPF_W: /* *(u32 *)(dst + off) = src */
0782             EMIT(PPC_RAW_STW(src_reg, dst_reg, off));
0783             break;
0784         case BPF_ST | BPF_MEM | BPF_W: /* *(u32 *)(dst + off) = imm */
0785             PPC_LI32(_R0, imm);
0786             EMIT(PPC_RAW_STW(_R0, dst_reg, off));
0787             break;
0788         case BPF_STX | BPF_MEM | BPF_DW: /* (u64 *)(dst + off) = src */
0789             EMIT(PPC_RAW_STW(src_reg_h, dst_reg, off));
0790             EMIT(PPC_RAW_STW(src_reg, dst_reg, off + 4));
0791             break;
0792         case BPF_ST | BPF_MEM | BPF_DW: /* *(u64 *)(dst + off) = imm */
0793             PPC_LI32(_R0, imm);
0794             EMIT(PPC_RAW_STW(_R0, dst_reg, off + 4));
0795             PPC_EX32(_R0, imm);
0796             EMIT(PPC_RAW_STW(_R0, dst_reg, off));
0797             break;
0798 
0799         /*
0800          * BPF_STX ATOMIC (atomic ops)
0801          */
0802         case BPF_STX | BPF_ATOMIC | BPF_W:
0803             save_reg = _R0;
0804             ret_reg = src_reg;
0805 
0806             bpf_set_seen_register(ctx, tmp_reg);
0807             bpf_set_seen_register(ctx, ax_reg);
0808 
0809             /* Get offset into TMP_REG */
0810             EMIT(PPC_RAW_LI(tmp_reg, off));
0811             tmp_idx = ctx->idx * 4;
0812             /* load value from memory into r0 */
0813             EMIT(PPC_RAW_LWARX(_R0, tmp_reg, dst_reg, 0));
0814 
0815             /* Save old value in BPF_REG_AX */
0816             if (imm & BPF_FETCH)
0817                 EMIT(PPC_RAW_MR(ax_reg, _R0));
0818 
0819             switch (imm) {
0820             case BPF_ADD:
0821             case BPF_ADD | BPF_FETCH:
0822                 EMIT(PPC_RAW_ADD(_R0, _R0, src_reg));
0823                 break;
0824             case BPF_AND:
0825             case BPF_AND | BPF_FETCH:
0826                 EMIT(PPC_RAW_AND(_R0, _R0, src_reg));
0827                 break;
0828             case BPF_OR:
0829             case BPF_OR | BPF_FETCH:
0830                 EMIT(PPC_RAW_OR(_R0, _R0, src_reg));
0831                 break;
0832             case BPF_XOR:
0833             case BPF_XOR | BPF_FETCH:
0834                 EMIT(PPC_RAW_XOR(_R0, _R0, src_reg));
0835                 break;
0836             case BPF_CMPXCHG:
0837                 /*
0838                  * Return old value in BPF_REG_0 for BPF_CMPXCHG &
0839                  * in src_reg for other cases.
0840                  */
0841                 ret_reg = bpf_to_ppc(BPF_REG_0);
0842 
0843                 /* Compare with old value in BPF_REG_0 */
0844                 EMIT(PPC_RAW_CMPW(bpf_to_ppc(BPF_REG_0), _R0));
0845                 /* Don't set if different from old value */
0846                 PPC_BCC_SHORT(COND_NE, (ctx->idx + 3) * 4);
0847                 fallthrough;
0848             case BPF_XCHG:
0849                 save_reg = src_reg;
0850                 break;
0851             default:
0852                 pr_err_ratelimited("eBPF filter atomic op code %02x (@%d) unsupported\n",
0853                            code, i);
0854                 return -EOPNOTSUPP;
0855             }
0856 
0857             /* store new value */
0858             EMIT(PPC_RAW_STWCX(save_reg, tmp_reg, dst_reg));
0859             /* we're done if this succeeded */
0860             PPC_BCC_SHORT(COND_NE, tmp_idx);
0861 
0862             /* For the BPF_FETCH variant, get old data into src_reg */
0863             if (imm & BPF_FETCH) {
0864                 EMIT(PPC_RAW_MR(ret_reg, ax_reg));
0865                 if (!fp->aux->verifier_zext)
0866                     EMIT(PPC_RAW_LI(ret_reg - 1, 0)); /* higher 32-bit */
0867             }
0868             break;
0869 
0870         case BPF_STX | BPF_ATOMIC | BPF_DW: /* *(u64 *)(dst + off) += src */
0871             return -EOPNOTSUPP;
0872 
0873         /*
0874          * BPF_LDX
0875          */
0876         case BPF_LDX | BPF_MEM | BPF_B: /* dst = *(u8 *)(ul) (src + off) */
0877         case BPF_LDX | BPF_PROBE_MEM | BPF_B:
0878         case BPF_LDX | BPF_MEM | BPF_H: /* dst = *(u16 *)(ul) (src + off) */
0879         case BPF_LDX | BPF_PROBE_MEM | BPF_H:
0880         case BPF_LDX | BPF_MEM | BPF_W: /* dst = *(u32 *)(ul) (src + off) */
0881         case BPF_LDX | BPF_PROBE_MEM | BPF_W:
0882         case BPF_LDX | BPF_MEM | BPF_DW: /* dst = *(u64 *)(ul) (src + off) */
0883         case BPF_LDX | BPF_PROBE_MEM | BPF_DW:
0884             /*
0885              * As PTR_TO_BTF_ID that uses BPF_PROBE_MEM mode could either be a valid
0886              * kernel pointer or NULL but not a userspace address, execute BPF_PROBE_MEM
0887              * load only if addr is kernel address (see is_kernel_addr()), otherwise
0888              * set dst_reg=0 and move on.
0889              */
0890             if (BPF_MODE(code) == BPF_PROBE_MEM) {
0891                 PPC_LI32(_R0, TASK_SIZE - off);
0892                 EMIT(PPC_RAW_CMPLW(src_reg, _R0));
0893                 PPC_BCC_SHORT(COND_GT, (ctx->idx + 4) * 4);
0894                 EMIT(PPC_RAW_LI(dst_reg, 0));
0895                 /*
0896                  * For BPF_DW case, "li reg_h,0" would be needed when
0897                  * !fp->aux->verifier_zext. Emit NOP otherwise.
0898                  *
0899                  * Note that "li reg_h,0" is emitted for BPF_B/H/W case,
0900                  * if necessary. So, jump there insted of emitting an
0901                  * additional "li reg_h,0" instruction.
0902                  */
0903                 if (size == BPF_DW && !fp->aux->verifier_zext)
0904                     EMIT(PPC_RAW_LI(dst_reg_h, 0));
0905                 else
0906                     EMIT(PPC_RAW_NOP());
0907                 /*
0908                  * Need to jump two instructions instead of one for BPF_DW case
0909                  * as there are two load instructions for dst_reg_h & dst_reg
0910                  * respectively.
0911                  */
0912                 if (size == BPF_DW)
0913                     PPC_JMP((ctx->idx + 3) * 4);
0914                 else
0915                     PPC_JMP((ctx->idx + 2) * 4);
0916             }
0917 
0918             switch (size) {
0919             case BPF_B:
0920                 EMIT(PPC_RAW_LBZ(dst_reg, src_reg, off));
0921                 break;
0922             case BPF_H:
0923                 EMIT(PPC_RAW_LHZ(dst_reg, src_reg, off));
0924                 break;
0925             case BPF_W:
0926                 EMIT(PPC_RAW_LWZ(dst_reg, src_reg, off));
0927                 break;
0928             case BPF_DW:
0929                 EMIT(PPC_RAW_LWZ(dst_reg_h, src_reg, off));
0930                 EMIT(PPC_RAW_LWZ(dst_reg, src_reg, off + 4));
0931                 break;
0932             }
0933 
0934             if (size != BPF_DW && !fp->aux->verifier_zext)
0935                 EMIT(PPC_RAW_LI(dst_reg_h, 0));
0936 
0937             if (BPF_MODE(code) == BPF_PROBE_MEM) {
0938                 int insn_idx = ctx->idx - 1;
0939                 int jmp_off = 4;
0940 
0941                 /*
0942                  * In case of BPF_DW, two lwz instructions are emitted, one
0943                  * for higher 32-bit and another for lower 32-bit. So, set
0944                  * ex->insn to the first of the two and jump over both
0945                  * instructions in fixup.
0946                  *
0947                  * Similarly, with !verifier_zext, two instructions are
0948                  * emitted for BPF_B/H/W case. So, set ex->insn to the
0949                  * instruction that could fault and skip over both
0950                  * instructions.
0951                  */
0952                 if (size == BPF_DW || !fp->aux->verifier_zext) {
0953                     insn_idx -= 1;
0954                     jmp_off += 4;
0955                 }
0956 
0957                 ret = bpf_add_extable_entry(fp, image, pass, ctx, insn_idx,
0958                                 jmp_off, dst_reg);
0959                 if (ret)
0960                     return ret;
0961             }
0962             break;
0963 
0964         /*
0965          * Doubleword load
0966          * 16 byte instruction that uses two 'struct bpf_insn'
0967          */
0968         case BPF_LD | BPF_IMM | BPF_DW: /* dst = (u64) imm */
0969             tmp_idx = ctx->idx;
0970             PPC_LI32(dst_reg_h, (u32)insn[i + 1].imm);
0971             PPC_LI32(dst_reg, (u32)insn[i].imm);
0972             /* padding to allow full 4 instructions for later patching */
0973             for (j = ctx->idx - tmp_idx; j < 4; j++)
0974                 EMIT(PPC_RAW_NOP());
0975             /* Adjust for two bpf instructions */
0976             addrs[++i] = ctx->idx * 4;
0977             break;
0978 
0979         /*
0980          * Return/Exit
0981          */
0982         case BPF_JMP | BPF_EXIT:
0983             /*
0984              * If this isn't the very last instruction, branch to
0985              * the epilogue. If we _are_ the last instruction,
0986              * we'll just fall through to the epilogue.
0987              */
0988             if (i != flen - 1) {
0989                 ret = bpf_jit_emit_exit_insn(image, ctx, _R0, exit_addr);
0990                 if (ret)
0991                     return ret;
0992             }
0993             /* else fall through to the epilogue */
0994             break;
0995 
0996         /*
0997          * Call kernel helper or bpf function
0998          */
0999         case BPF_JMP | BPF_CALL:
1000             ctx->seen |= SEEN_FUNC;
1001 
1002             ret = bpf_jit_get_func_addr(fp, &insn[i], false,
1003                             &func_addr, &func_addr_fixed);
1004             if (ret < 0)
1005                 return ret;
1006 
1007             if (bpf_is_seen_register(ctx, bpf_to_ppc(BPF_REG_5))) {
1008                 EMIT(PPC_RAW_STW(bpf_to_ppc(BPF_REG_5) - 1, _R1, 8));
1009                 EMIT(PPC_RAW_STW(bpf_to_ppc(BPF_REG_5), _R1, 12));
1010             }
1011 
1012             ret = bpf_jit_emit_func_call_rel(image, ctx, func_addr);
1013             if (ret)
1014                 return ret;
1015 
1016             EMIT(PPC_RAW_MR(bpf_to_ppc(BPF_REG_0) - 1, _R3));
1017             EMIT(PPC_RAW_MR(bpf_to_ppc(BPF_REG_0), _R4));
1018             break;
1019 
1020         /*
1021          * Jumps and branches
1022          */
1023         case BPF_JMP | BPF_JA:
1024             PPC_JMP(addrs[i + 1 + off]);
1025             break;
1026 
1027         case BPF_JMP | BPF_JGT | BPF_K:
1028         case BPF_JMP | BPF_JGT | BPF_X:
1029         case BPF_JMP | BPF_JSGT | BPF_K:
1030         case BPF_JMP | BPF_JSGT | BPF_X:
1031         case BPF_JMP32 | BPF_JGT | BPF_K:
1032         case BPF_JMP32 | BPF_JGT | BPF_X:
1033         case BPF_JMP32 | BPF_JSGT | BPF_K:
1034         case BPF_JMP32 | BPF_JSGT | BPF_X:
1035             true_cond = COND_GT;
1036             goto cond_branch;
1037         case BPF_JMP | BPF_JLT | BPF_K:
1038         case BPF_JMP | BPF_JLT | BPF_X:
1039         case BPF_JMP | BPF_JSLT | BPF_K:
1040         case BPF_JMP | BPF_JSLT | BPF_X:
1041         case BPF_JMP32 | BPF_JLT | BPF_K:
1042         case BPF_JMP32 | BPF_JLT | BPF_X:
1043         case BPF_JMP32 | BPF_JSLT | BPF_K:
1044         case BPF_JMP32 | BPF_JSLT | BPF_X:
1045             true_cond = COND_LT;
1046             goto cond_branch;
1047         case BPF_JMP | BPF_JGE | BPF_K:
1048         case BPF_JMP | BPF_JGE | BPF_X:
1049         case BPF_JMP | BPF_JSGE | BPF_K:
1050         case BPF_JMP | BPF_JSGE | BPF_X:
1051         case BPF_JMP32 | BPF_JGE | BPF_K:
1052         case BPF_JMP32 | BPF_JGE | BPF_X:
1053         case BPF_JMP32 | BPF_JSGE | BPF_K:
1054         case BPF_JMP32 | BPF_JSGE | BPF_X:
1055             true_cond = COND_GE;
1056             goto cond_branch;
1057         case BPF_JMP | BPF_JLE | BPF_K:
1058         case BPF_JMP | BPF_JLE | BPF_X:
1059         case BPF_JMP | BPF_JSLE | BPF_K:
1060         case BPF_JMP | BPF_JSLE | BPF_X:
1061         case BPF_JMP32 | BPF_JLE | BPF_K:
1062         case BPF_JMP32 | BPF_JLE | BPF_X:
1063         case BPF_JMP32 | BPF_JSLE | BPF_K:
1064         case BPF_JMP32 | BPF_JSLE | BPF_X:
1065             true_cond = COND_LE;
1066             goto cond_branch;
1067         case BPF_JMP | BPF_JEQ | BPF_K:
1068         case BPF_JMP | BPF_JEQ | BPF_X:
1069         case BPF_JMP32 | BPF_JEQ | BPF_K:
1070         case BPF_JMP32 | BPF_JEQ | BPF_X:
1071             true_cond = COND_EQ;
1072             goto cond_branch;
1073         case BPF_JMP | BPF_JNE | BPF_K:
1074         case BPF_JMP | BPF_JNE | BPF_X:
1075         case BPF_JMP32 | BPF_JNE | BPF_K:
1076         case BPF_JMP32 | BPF_JNE | BPF_X:
1077             true_cond = COND_NE;
1078             goto cond_branch;
1079         case BPF_JMP | BPF_JSET | BPF_K:
1080         case BPF_JMP | BPF_JSET | BPF_X:
1081         case BPF_JMP32 | BPF_JSET | BPF_K:
1082         case BPF_JMP32 | BPF_JSET | BPF_X:
1083             true_cond = COND_NE;
1084             /* fallthrough; */
1085 
1086 cond_branch:
1087             switch (code) {
1088             case BPF_JMP | BPF_JGT | BPF_X:
1089             case BPF_JMP | BPF_JLT | BPF_X:
1090             case BPF_JMP | BPF_JGE | BPF_X:
1091             case BPF_JMP | BPF_JLE | BPF_X:
1092             case BPF_JMP | BPF_JEQ | BPF_X:
1093             case BPF_JMP | BPF_JNE | BPF_X:
1094                 /* unsigned comparison */
1095                 EMIT(PPC_RAW_CMPLW(dst_reg_h, src_reg_h));
1096                 PPC_BCC_SHORT(COND_NE, (ctx->idx + 2) * 4);
1097                 EMIT(PPC_RAW_CMPLW(dst_reg, src_reg));
1098                 break;
1099             case BPF_JMP32 | BPF_JGT | BPF_X:
1100             case BPF_JMP32 | BPF_JLT | BPF_X:
1101             case BPF_JMP32 | BPF_JGE | BPF_X:
1102             case BPF_JMP32 | BPF_JLE | BPF_X:
1103             case BPF_JMP32 | BPF_JEQ | BPF_X:
1104             case BPF_JMP32 | BPF_JNE | BPF_X:
1105                 /* unsigned comparison */
1106                 EMIT(PPC_RAW_CMPLW(dst_reg, src_reg));
1107                 break;
1108             case BPF_JMP | BPF_JSGT | BPF_X:
1109             case BPF_JMP | BPF_JSLT | BPF_X:
1110             case BPF_JMP | BPF_JSGE | BPF_X:
1111             case BPF_JMP | BPF_JSLE | BPF_X:
1112                 /* signed comparison */
1113                 EMIT(PPC_RAW_CMPW(dst_reg_h, src_reg_h));
1114                 PPC_BCC_SHORT(COND_NE, (ctx->idx + 2) * 4);
1115                 EMIT(PPC_RAW_CMPLW(dst_reg, src_reg));
1116                 break;
1117             case BPF_JMP32 | BPF_JSGT | BPF_X:
1118             case BPF_JMP32 | BPF_JSLT | BPF_X:
1119             case BPF_JMP32 | BPF_JSGE | BPF_X:
1120             case BPF_JMP32 | BPF_JSLE | BPF_X:
1121                 /* signed comparison */
1122                 EMIT(PPC_RAW_CMPW(dst_reg, src_reg));
1123                 break;
1124             case BPF_JMP | BPF_JSET | BPF_X:
1125                 EMIT(PPC_RAW_AND_DOT(_R0, dst_reg_h, src_reg_h));
1126                 PPC_BCC_SHORT(COND_NE, (ctx->idx + 2) * 4);
1127                 EMIT(PPC_RAW_AND_DOT(_R0, dst_reg, src_reg));
1128                 break;
1129             case BPF_JMP32 | BPF_JSET | BPF_X: {
1130                 EMIT(PPC_RAW_AND_DOT(_R0, dst_reg, src_reg));
1131                 break;
1132             case BPF_JMP | BPF_JNE | BPF_K:
1133             case BPF_JMP | BPF_JEQ | BPF_K:
1134             case BPF_JMP | BPF_JGT | BPF_K:
1135             case BPF_JMP | BPF_JLT | BPF_K:
1136             case BPF_JMP | BPF_JGE | BPF_K:
1137             case BPF_JMP | BPF_JLE | BPF_K:
1138                 /*
1139                  * Need sign-extended load, so only positive
1140                  * values can be used as imm in cmplwi
1141                  */
1142                 if (imm >= 0 && imm < 32768) {
1143                     EMIT(PPC_RAW_CMPLWI(dst_reg_h, 0));
1144                     PPC_BCC_SHORT(COND_NE, (ctx->idx + 2) * 4);
1145                     EMIT(PPC_RAW_CMPLWI(dst_reg, imm));
1146                 } else {
1147                     /* sign-extending load ... but unsigned comparison */
1148                     PPC_EX32(_R0, imm);
1149                     EMIT(PPC_RAW_CMPLW(dst_reg_h, _R0));
1150                     PPC_LI32(_R0, imm);
1151                     PPC_BCC_SHORT(COND_NE, (ctx->idx + 2) * 4);
1152                     EMIT(PPC_RAW_CMPLW(dst_reg, _R0));
1153                 }
1154                 break;
1155             case BPF_JMP32 | BPF_JNE | BPF_K:
1156             case BPF_JMP32 | BPF_JEQ | BPF_K:
1157             case BPF_JMP32 | BPF_JGT | BPF_K:
1158             case BPF_JMP32 | BPF_JLT | BPF_K:
1159             case BPF_JMP32 | BPF_JGE | BPF_K:
1160             case BPF_JMP32 | BPF_JLE | BPF_K:
1161                 if (imm >= 0 && imm < 65536) {
1162                     EMIT(PPC_RAW_CMPLWI(dst_reg, imm));
1163                 } else {
1164                     PPC_LI32(_R0, imm);
1165                     EMIT(PPC_RAW_CMPLW(dst_reg, _R0));
1166                 }
1167                 break;
1168             }
1169             case BPF_JMP | BPF_JSGT | BPF_K:
1170             case BPF_JMP | BPF_JSLT | BPF_K:
1171             case BPF_JMP | BPF_JSGE | BPF_K:
1172             case BPF_JMP | BPF_JSLE | BPF_K:
1173                 if (imm >= 0 && imm < 65536) {
1174                     EMIT(PPC_RAW_CMPWI(dst_reg_h, imm < 0 ? -1 : 0));
1175                     PPC_BCC_SHORT(COND_NE, (ctx->idx + 2) * 4);
1176                     EMIT(PPC_RAW_CMPLWI(dst_reg, imm));
1177                 } else {
1178                     /* sign-extending load */
1179                     EMIT(PPC_RAW_CMPWI(dst_reg_h, imm < 0 ? -1 : 0));
1180                     PPC_LI32(_R0, imm);
1181                     PPC_BCC_SHORT(COND_NE, (ctx->idx + 2) * 4);
1182                     EMIT(PPC_RAW_CMPLW(dst_reg, _R0));
1183                 }
1184                 break;
1185             case BPF_JMP32 | BPF_JSGT | BPF_K:
1186             case BPF_JMP32 | BPF_JSLT | BPF_K:
1187             case BPF_JMP32 | BPF_JSGE | BPF_K:
1188             case BPF_JMP32 | BPF_JSLE | BPF_K:
1189                 /*
1190                  * signed comparison, so any 16-bit value
1191                  * can be used in cmpwi
1192                  */
1193                 if (imm >= -32768 && imm < 32768) {
1194                     EMIT(PPC_RAW_CMPWI(dst_reg, imm));
1195                 } else {
1196                     /* sign-extending load */
1197                     PPC_LI32(_R0, imm);
1198                     EMIT(PPC_RAW_CMPW(dst_reg, _R0));
1199                 }
1200                 break;
1201             case BPF_JMP | BPF_JSET | BPF_K:
1202                 /* andi does not sign-extend the immediate */
1203                 if (imm >= 0 && imm < 32768) {
1204                     /* PPC_ANDI is _only/always_ dot-form */
1205                     EMIT(PPC_RAW_ANDI(_R0, dst_reg, imm));
1206                 } else {
1207                     PPC_LI32(_R0, imm);
1208                     if (imm < 0) {
1209                         EMIT(PPC_RAW_CMPWI(dst_reg_h, 0));
1210                         PPC_BCC_SHORT(COND_NE, (ctx->idx + 2) * 4);
1211                     }
1212                     EMIT(PPC_RAW_AND_DOT(_R0, dst_reg, _R0));
1213                 }
1214                 break;
1215             case BPF_JMP32 | BPF_JSET | BPF_K:
1216                 /* andi does not sign-extend the immediate */
1217                 if (imm >= 0 && imm < 32768) {
1218                     /* PPC_ANDI is _only/always_ dot-form */
1219                     EMIT(PPC_RAW_ANDI(_R0, dst_reg, imm));
1220                 } else {
1221                     PPC_LI32(_R0, imm);
1222                     EMIT(PPC_RAW_AND_DOT(_R0, dst_reg, _R0));
1223                 }
1224                 break;
1225             }
1226             PPC_BCC(true_cond, addrs[i + 1 + off]);
1227             break;
1228 
1229         /*
1230          * Tail call
1231          */
1232         case BPF_JMP | BPF_TAIL_CALL:
1233             ctx->seen |= SEEN_TAILCALL;
1234             ret = bpf_jit_emit_tail_call(image, ctx, addrs[i + 1]);
1235             if (ret < 0)
1236                 return ret;
1237             break;
1238 
1239         default:
1240             /*
1241              * The filter contains something cruel & unusual.
1242              * We don't handle it, but also there shouldn't be
1243              * anything missing from our list.
1244              */
1245             pr_err_ratelimited("eBPF filter opcode %04x (@%d) unsupported\n", code, i);
1246             return -EOPNOTSUPP;
1247         }
1248         if (BPF_CLASS(code) == BPF_ALU && !fp->aux->verifier_zext &&
1249             !insn_is_zext(&insn[i + 1]) && !(BPF_OP(code) == BPF_END && imm == 64))
1250             EMIT(PPC_RAW_LI(dst_reg_h, 0));
1251     }
1252 
1253     /* Set end-of-body-code address for exit. */
1254     addrs[i] = ctx->idx * 4;
1255 
1256     return 0;
1257 }
[ Source navigation ] [ Diff markup ] [ Identifier search ] [ general search ]

This page was automatically generated by the 2.1.0 LXR engine.

The LXR team
Valid CSS 2.1! Valid HTML 4.01!