kernel/bpf/dispatcher.c

0001 // SPDX-License-Identifier: GPL-2.0-only
0002 /* Copyright(c) 2019 Intel Corporation. */
0003
0004 #include <linux/hash.h>
0005 #include <linux/bpf.h>
0006 #include <linux/filter.h>
0007
0008 /* The BPF dispatcher is a multiway branch code generator. The
0009  * dispatcher is a mechanism to avoid the performance penalty of an
0010  * indirect call, which is expensive when retpolines are enabled. A
0011  * dispatch client registers a BPF program into the dispatcher, and if
0012  * there is available room in the dispatcher a direct call to the BPF
0013  * program will be generated. All calls to the BPF programs called via
0014  * the dispatcher will then be a direct call, instead of an
0015  * indirect. The dispatcher hijacks a trampoline function it via the
0016  * __fentry__ of the trampoline. The trampoline function has the
0017  * following signature:
0018  *
0019  * unsigned int trampoline(const void *ctx, const struct bpf_insn *insnsi,
0020  *                         unsigned int (*bpf_func)(const void *,
0021  *                                                  const struct bpf_insn *));
0022  */
0023
0024 static struct bpf_dispatcher_prog *bpf_dispatcher_find_prog(
0025     struct bpf_dispatcher *d, struct bpf_prog *prog)
0026 {
0027     int i;
0028
0029     for (i = 0; i < BPF_DISPATCHER_MAX; i++) {
0030         if (prog == d->progs[i].prog)
0031             return &d->progs[i];
0032     }
0033     return NULL;
0034 }
0035
0036 static struct bpf_dispatcher_prog *bpf_dispatcher_find_free(
0037     struct bpf_dispatcher *d)
0038 {
0039     return bpf_dispatcher_find_prog(d, NULL);
0040 }
0041
0042 static bool bpf_dispatcher_add_prog(struct bpf_dispatcher *d,
0043                     struct bpf_prog *prog)
0044 {
0045     struct bpf_dispatcher_prog *entry;
0046
0047     if (!prog)
0048         return false;
0049
0050     entry = bpf_dispatcher_find_prog(d, prog);
0051     if (entry) {
0052         refcount_inc(&entry->users);
0053         return false;
0054     }
0055
0056     entry = bpf_dispatcher_find_free(d);
0057     if (!entry)
0058         return false;
0059
0060     bpf_prog_inc(prog);
0061     entry->prog = prog;
0062     refcount_set(&entry->users, 1);
0063     d->num_progs++;
0064     return true;
0065 }
0066
0067 static bool bpf_dispatcher_remove_prog(struct bpf_dispatcher *d,
0068                        struct bpf_prog *prog)
0069 {
0070     struct bpf_dispatcher_prog *entry;
0071
0072     if (!prog)
0073         return false;
0074
0075     entry = bpf_dispatcher_find_prog(d, prog);
0076     if (!entry)
0077         return false;
0078
0079     if (refcount_dec_and_test(&entry->users)) {
0080         entry->prog = NULL;
0081         bpf_prog_put(prog);
0082         d->num_progs--;
0083         return true;
0084     }
0085     return false;
0086 }
0087
0088 int __weak arch_prepare_bpf_dispatcher(void *image, s64 *funcs, int num_funcs)
0089 {
0090     return -ENOTSUPP;
0091 }
0092
0093 static int bpf_dispatcher_prepare(struct bpf_dispatcher *d, void *image)
0094 {
0095     s64 ips[BPF_DISPATCHER_MAX] = {}, *ipsp = &ips[0];
0096     int i;
0097
0098     for (i = 0; i < BPF_DISPATCHER_MAX; i++) {
0099         if (d->progs[i].prog)
0100             *ipsp++ = (s64)(uintptr_t)d->progs[i].prog->bpf_func;
0101     }
0102     return arch_prepare_bpf_dispatcher(image, &ips[0], d->num_progs);
0103 }
0104
0105 static void bpf_dispatcher_update(struct bpf_dispatcher *d, int prev_num_progs)
0106 {
0107     void *old, *new;
0108     u32 noff;
0109     int err;
0110
0111     if (!prev_num_progs) {
0112         old = NULL;
0113         noff = 0;
0114     } else {
0115         old = d->image + d->image_off;
0116         noff = d->image_off ^ (PAGE_SIZE / 2);
0117     }
0118
0119     new = d->num_progs ? d->image + noff : NULL;
0120     if (new) {
0121         if (bpf_dispatcher_prepare(d, new))
0122             return;
0123     }
0124
0125     err = bpf_arch_text_poke(d->func, BPF_MOD_JUMP, old, new);
0126     if (err || !new)
0127         return;
0128
0129     d->image_off = noff;
0130 }
0131
0132 void bpf_dispatcher_change_prog(struct bpf_dispatcher *d, struct bpf_prog *from,
0133                 struct bpf_prog *to)
0134 {
0135     bool changed = false;
0136     int prev_num_progs;
0137
0138     if (from == to)
0139         return;
0140
0141     mutex_lock(&d->mutex);
0142     if (!d->image) {
0143         d->image = bpf_jit_alloc_exec_page();
0144         if (!d->image)
0145             goto out;
0146         bpf_image_ksym_add(d->image, &d->ksym);
0147     }
0148
0149     prev_num_progs = d->num_progs;
0150     changed |= bpf_dispatcher_remove_prog(d, from);
0151     changed |= bpf_dispatcher_add_prog(d, to);
0152
0153     if (!changed)
0154         goto out;
0155
0156     bpf_dispatcher_update(d, prev_num_progs);
0157 out:
0158     mutex_unlock(&d->mutex);
0159 }