powerpc/net/bpf_jit_comp.c

0001 // SPDX-License-Identifier: GPL-2.0-only
0002 /*
0003  * eBPF JIT compiler
0004  *
0005  * Copyright 2016 Naveen N. Rao <naveen.n.rao@linux.vnet.ibm.com>
0006  *        IBM Corporation
0007  *
0008  * Based on the powerpc classic BPF JIT compiler by Matt Evans
0009  */
0010 #include <linux/moduleloader.h>
0011 #include <asm/cacheflush.h>
0012 #include <asm/asm-compat.h>
0013 #include <linux/netdevice.h>
0014 #include <linux/filter.h>
0015 #include <linux/if_vlan.h>
0016 #include <asm/kprobes.h>
0017 #include <linux/bpf.h>
0018
0019 #include "bpf_jit.h"
0020
0021 static void bpf_jit_fill_ill_insns(void *area, unsigned int size)
0022 {
0023     memset32(area, BREAKPOINT_INSTRUCTION, size / 4);
0024 }
0025
0026 /* Fix updated addresses (for subprog calls, ldimm64, et al) during extra pass */
0027 static int bpf_jit_fixup_addresses(struct bpf_prog *fp, u32 *image,
0028                    struct codegen_context *ctx, u32 *addrs)
0029 {
0030     const struct bpf_insn *insn = fp->insnsi;
0031     bool func_addr_fixed;
0032     u64 func_addr;
0033     u32 tmp_idx;
0034     int i, j, ret;
0035
0036     for (i = 0; i < fp->len; i++) {
0037         /*
0038          * During the extra pass, only the branch target addresses for
0039          * the subprog calls need to be fixed. All other instructions
0040          * can left untouched.
0041          *
0042          * The JITed image length does not change because we already
0043          * ensure that the JITed instruction sequence for these calls
0044          * are of fixed length by padding them with NOPs.
0045          */
0046         if (insn[i].code == (BPF_JMP | BPF_CALL) &&
0047             insn[i].src_reg == BPF_PSEUDO_CALL) {
0048             ret = bpf_jit_get_func_addr(fp, &insn[i], true,
0049                             &func_addr,
0050                             &func_addr_fixed);
0051             if (ret < 0)
0052                 return ret;
0053
0054             /*
0055              * Save ctx->idx as this would currently point to the
0056              * end of the JITed image and set it to the offset of
0057              * the instruction sequence corresponding to the
0058              * subprog call temporarily.
0059              */
0060             tmp_idx = ctx->idx;
0061             ctx->idx = addrs[i] / 4;
0062             ret = bpf_jit_emit_func_call_rel(image, ctx, func_addr);
0063             if (ret)
0064                 return ret;
0065
0066             /*
0067              * Restore ctx->idx here. This is safe as the length
0068              * of the JITed sequence remains unchanged.
0069              */
0070             ctx->idx = tmp_idx;
0071         } else if (insn[i].code == (BPF_LD | BPF_IMM | BPF_DW)) {
0072             tmp_idx = ctx->idx;
0073             ctx->idx = addrs[i] / 4;
0074 #ifdef CONFIG_PPC32
0075             PPC_LI32(bpf_to_ppc(insn[i].dst_reg) - 1, (u32)insn[i + 1].imm);
0076             PPC_LI32(bpf_to_ppc(insn[i].dst_reg), (u32)insn[i].imm);
0077             for (j = ctx->idx - addrs[i] / 4; j < 4; j++)
0078                 EMIT(PPC_RAW_NOP());
0079 #else
0080             func_addr = ((u64)(u32)insn[i].imm) | (((u64)(u32)insn[i + 1].imm) << 32);
0081             PPC_LI64(bpf_to_ppc(insn[i].dst_reg), func_addr);
0082             /* overwrite rest with nops */
0083             for (j = ctx->idx - addrs[i] / 4; j < 5; j++)
0084                 EMIT(PPC_RAW_NOP());
0085 #endif
0086             ctx->idx = tmp_idx;
0087             i++;
0088         }
0089     }
0090
0091     return 0;
0092 }
0093
0094 int bpf_jit_emit_exit_insn(u32 *image, struct codegen_context *ctx, int tmp_reg, long exit_addr)
0095 {
0096     if (!exit_addr || is_offset_in_branch_range(exit_addr - (ctx->idx * 4))) {
0097         PPC_JMP(exit_addr);
0098     } else if (ctx->alt_exit_addr) {
0099         if (WARN_ON(!is_offset_in_branch_range((long)ctx->alt_exit_addr - (ctx->idx * 4))))
0100             return -1;
0101         PPC_JMP(ctx->alt_exit_addr);
0102     } else {
0103         ctx->alt_exit_addr = ctx->idx * 4;
0104         bpf_jit_build_epilogue(image, ctx);
0105     }
0106
0107     return 0;
0108 }
0109
0110 struct powerpc64_jit_data {
0111     struct bpf_binary_header *header;
0112     u32 *addrs;
0113     u8 *image;
0114     u32 proglen;
0115     struct codegen_context ctx;
0116 };
0117
0118 bool bpf_jit_needs_zext(void)
0119 {
0120     return true;
0121 }
0122
0123 struct bpf_prog *bpf_int_jit_compile(struct bpf_prog *fp)
0124 {
0125     u32 proglen;
0126     u32 alloclen;
0127     u8 *image = NULL;
0128     u32 *code_base;
0129     u32 *addrs;
0130     struct powerpc64_jit_data *jit_data;
0131     struct codegen_context cgctx;
0132     int pass;
0133     int flen;
0134     struct bpf_binary_header *bpf_hdr;
0135     struct bpf_prog *org_fp = fp;
0136     struct bpf_prog *tmp_fp;
0137     bool bpf_blinded = false;
0138     bool extra_pass = false;
0139     u32 extable_len;
0140     u32 fixup_len;
0141
0142     if (!fp->jit_requested)
0143         return org_fp;
0144
0145     tmp_fp = bpf_jit_blind_constants(org_fp);
0146     if (IS_ERR(tmp_fp))
0147         return org_fp;
0148
0149     if (tmp_fp != org_fp) {
0150         bpf_blinded = true;
0151         fp = tmp_fp;
0152     }
0153
0154     jit_data = fp->aux->jit_data;
0155     if (!jit_data) {
0156         jit_data = kzalloc(sizeof(*jit_data), GFP_KERNEL);
0157         if (!jit_data) {
0158             fp = org_fp;
0159             goto out;
0160         }
0161         fp->aux->jit_data = jit_data;
0162     }
0163
0164     flen = fp->len;
0165     addrs = jit_data->addrs;
0166     if (addrs) {
0167         cgctx = jit_data->ctx;
0168         image = jit_data->image;
0169         bpf_hdr = jit_data->header;
0170         proglen = jit_data->proglen;
0171         extra_pass = true;
0172         goto skip_init_ctx;
0173     }
0174
0175     addrs = kcalloc(flen + 1, sizeof(*addrs), GFP_KERNEL);
0176     if (addrs == NULL) {
0177         fp = org_fp;
0178         goto out_addrs;
0179     }
0180
0181     memset(&cgctx, 0, sizeof(struct codegen_context));
0182     bpf_jit_init_reg_mapping(&cgctx);
0183
0184     /* Make sure that the stack is quadword aligned. */
0185     cgctx.stack_size = round_up(fp->aux->stack_depth, 16);
0186
0187     /* Scouting faux-generate pass 0 */
0188     if (bpf_jit_build_body(fp, 0, &cgctx, addrs, 0)) {
0189         /* We hit something illegal or unsupported. */
0190         fp = org_fp;
0191         goto out_addrs;
0192     }
0193
0194     /*
0195      * If we have seen a tail call, we need a second pass.
0196      * This is because bpf_jit_emit_common_epilogue() is called
0197      * from bpf_jit_emit_tail_call() with a not yet stable ctx->seen.
0198      * We also need a second pass if we ended up with too large
0199      * a program so as to ensure BPF_EXIT branches are in range.
0200      */
0201     if (cgctx.seen & SEEN_TAILCALL || !is_offset_in_branch_range((long)cgctx.idx * 4)) {
0202         cgctx.idx = 0;
0203         if (bpf_jit_build_body(fp, 0, &cgctx, addrs, 0)) {
0204             fp = org_fp;
0205             goto out_addrs;
0206         }
0207     }
0208
0209     bpf_jit_realloc_regs(&cgctx);
0210     /*
0211      * Pretend to build prologue, given the features we've seen.  This will
0212      * update ctgtx.idx as it pretends to output instructions, then we can
0213      * calculate total size from idx.
0214      */
0215     bpf_jit_build_prologue(0, &cgctx);
0216     addrs[fp->len] = cgctx.idx * 4;
0217     bpf_jit_build_epilogue(0, &cgctx);
0218
0219     fixup_len = fp->aux->num_exentries * BPF_FIXUP_LEN * 4;
0220     extable_len = fp->aux->num_exentries * sizeof(struct exception_table_entry);
0221
0222     proglen = cgctx.idx * 4;
0223     alloclen = proglen + FUNCTION_DESCR_SIZE + fixup_len + extable_len;
0224
0225     bpf_hdr = bpf_jit_binary_alloc(alloclen, &image, 4, bpf_jit_fill_ill_insns);
0226     if (!bpf_hdr) {
0227         fp = org_fp;
0228         goto out_addrs;
0229     }
0230
0231     if (extable_len)
0232         fp->aux->extable = (void *)image + FUNCTION_DESCR_SIZE + proglen + fixup_len;
0233
0234 skip_init_ctx:
0235     code_base = (u32 *)(image + FUNCTION_DESCR_SIZE);
0236
0237     if (extra_pass) {
0238         /*
0239          * Do not touch the prologue and epilogue as they will remain
0240          * unchanged. Only fix the branch target address for subprog
0241          * calls in the body, and ldimm64 instructions.
0242          *
0243          * This does not change the offsets and lengths of the subprog
0244          * call instruction sequences and hence, the size of the JITed
0245          * image as well.
0246          */
0247         bpf_jit_fixup_addresses(fp, code_base, &cgctx, addrs);
0248
0249         /* There is no need to perform the usual passes. */
0250         goto skip_codegen_passes;
0251     }
0252
0253     /* Code generation passes 1-2 */
0254     for (pass = 1; pass < 3; pass++) {
0255         /* Now build the prologue, body code & epilogue for real. */
0256         cgctx.idx = 0;
0257         cgctx.alt_exit_addr = 0;
0258         bpf_jit_build_prologue(code_base, &cgctx);
0259         if (bpf_jit_build_body(fp, code_base, &cgctx, addrs, pass)) {
0260             bpf_jit_binary_free(bpf_hdr);
0261             fp = org_fp;
0262             goto out_addrs;
0263         }
0264         bpf_jit_build_epilogue(code_base, &cgctx);
0265
0266         if (bpf_jit_enable > 1)
0267             pr_info("Pass %d: shrink = %d, seen = 0x%x\n", pass,
0268                 proglen - (cgctx.idx * 4), cgctx.seen);
0269     }
0270
0271 skip_codegen_passes:
0272     if (bpf_jit_enable > 1)
0273         /*
0274          * Note that we output the base address of the code_base
0275          * rather than image, since opcodes are in code_base.
0276          */
0277         bpf_jit_dump(flen, proglen, pass, code_base);
0278
0279 #ifdef CONFIG_PPC64_ELF_ABI_V1
0280     /* Function descriptor nastiness: Address + TOC */
0281     ((u64 *)image)[0] = (u64)code_base;
0282     ((u64 *)image)[1] = local_paca->kernel_toc;
0283 #endif
0284
0285     fp->bpf_func = (void *)image;
0286     fp->jited = 1;
0287     fp->jited_len = proglen + FUNCTION_DESCR_SIZE;
0288
0289     bpf_flush_icache(bpf_hdr, (u8 *)bpf_hdr + bpf_hdr->size);
0290     if (!fp->is_func || extra_pass) {
0291         bpf_jit_binary_lock_ro(bpf_hdr);
0292         bpf_prog_fill_jited_linfo(fp, addrs);
0293 out_addrs:
0294         kfree(addrs);
0295         kfree(jit_data);
0296         fp->aux->jit_data = NULL;
0297     } else {
0298         jit_data->addrs = addrs;
0299         jit_data->ctx = cgctx;
0300         jit_data->proglen = proglen;
0301         jit_data->image = image;
0302         jit_data->header = bpf_hdr;
0303     }
0304
0305 out:
0306     if (bpf_blinded)
0307         bpf_jit_prog_release_other(fp, fp == org_fp ? tmp_fp : org_fp);
0308
0309     return fp;
0310 }
0311
0312 /*
0313  * The caller should check for (BPF_MODE(code) == BPF_PROBE_MEM) before calling
0314  * this function, as this only applies to BPF_PROBE_MEM, for now.
0315  */
0316 int bpf_add_extable_entry(struct bpf_prog *fp, u32 *image, int pass, struct codegen_context *ctx,
0317               int insn_idx, int jmp_off, int dst_reg)
0318 {
0319     off_t offset;
0320     unsigned long pc;
0321     struct exception_table_entry *ex;
0322     u32 *fixup;
0323
0324     /* Populate extable entries only in the last pass */
0325     if (pass != 2)
0326         return 0;
0327
0328     if (!fp->aux->extable ||
0329         WARN_ON_ONCE(ctx->exentry_idx >= fp->aux->num_exentries))
0330         return -EINVAL;
0331
0332     pc = (unsigned long)&image[insn_idx];
0333
0334     fixup = (void *)fp->aux->extable -
0335         (fp->aux->num_exentries * BPF_FIXUP_LEN * 4) +
0336         (ctx->exentry_idx * BPF_FIXUP_LEN * 4);
0337
0338     fixup[0] = PPC_RAW_LI(dst_reg, 0);
0339     if (IS_ENABLED(CONFIG_PPC32))
0340         fixup[1] = PPC_RAW_LI(dst_reg - 1, 0); /* clear higher 32-bit register too */
0341
0342     fixup[BPF_FIXUP_LEN - 1] =
0343         PPC_RAW_BRANCH((long)(pc + jmp_off) - (long)&fixup[BPF_FIXUP_LEN - 1]);
0344
0345     ex = &fp->aux->extable[ctx->exentry_idx];
0346
0347     offset = pc - (long)&ex->insn;
0348     if (WARN_ON_ONCE(offset >= 0 || offset < INT_MIN))
0349         return -ERANGE;
0350     ex->insn = offset;
0351
0352     offset = (long)fixup - (long)&ex->fixup;
0353     if (WARN_ON_ONCE(offset >= 0 || offset < INT_MIN))
0354         return -ERANGE;
0355     ex->fixup = offset;
0356
0357     ctx->exentry_idx++;
0358     return 0;
0359 }