0001
0002
0003
0004
0005
0006
0007
0008
0009
0010
0011
0012
0013
0014
0015
0016 #include <linux/bpf.h>
0017 #include <linux/bpf_verifier.h>
0018 #include <linux/debugfs.h>
0019 #include <linux/kernel.h>
0020 #include <linux/mutex.h>
0021 #include <linux/rtnetlink.h>
0022 #include <net/pkt_cls.h>
0023
0024 #include "netdevsim.h"
0025
0026 #define pr_vlog(env, fmt, ...) \
0027 bpf_verifier_log_write(env, "[netdevsim] " fmt, ##__VA_ARGS__)
0028
0029 struct nsim_bpf_bound_prog {
0030 struct nsim_dev *nsim_dev;
0031 struct bpf_prog *prog;
0032 struct dentry *ddir;
0033 const char *state;
0034 bool is_loaded;
0035 struct list_head l;
0036 };
0037
0038 #define NSIM_BPF_MAX_KEYS 2
0039
0040 struct nsim_bpf_bound_map {
0041 struct netdevsim *ns;
0042 struct bpf_offloaded_map *map;
0043 struct mutex mutex;
0044 struct nsim_map_entry {
0045 void *key;
0046 void *value;
0047 } entry[NSIM_BPF_MAX_KEYS];
0048 struct list_head l;
0049 };
0050
0051 static int nsim_bpf_string_show(struct seq_file *file, void *data)
0052 {
0053 const char **str = file->private;
0054
0055 if (*str)
0056 seq_printf(file, "%s\n", *str);
0057
0058 return 0;
0059 }
0060 DEFINE_SHOW_ATTRIBUTE(nsim_bpf_string);
0061
0062 static int
0063 nsim_bpf_verify_insn(struct bpf_verifier_env *env, int insn_idx, int prev_insn)
0064 {
0065 struct nsim_bpf_bound_prog *state;
0066 int ret = 0;
0067
0068 state = env->prog->aux->offload->dev_priv;
0069 if (state->nsim_dev->bpf_bind_verifier_delay && !insn_idx)
0070 msleep(state->nsim_dev->bpf_bind_verifier_delay);
0071
0072 if (insn_idx == env->prog->len - 1) {
0073 pr_vlog(env, "Hello from netdevsim!\n");
0074
0075 if (!state->nsim_dev->bpf_bind_verifier_accept)
0076 ret = -EOPNOTSUPP;
0077 }
0078
0079 return ret;
0080 }
0081
0082 static int nsim_bpf_finalize(struct bpf_verifier_env *env)
0083 {
0084 return 0;
0085 }
0086
0087 static bool nsim_xdp_offload_active(struct netdevsim *ns)
0088 {
0089 return ns->xdp_hw.prog;
0090 }
0091
0092 static void nsim_prog_set_loaded(struct bpf_prog *prog, bool loaded)
0093 {
0094 struct nsim_bpf_bound_prog *state;
0095
0096 if (!prog || !prog->aux->offload)
0097 return;
0098
0099 state = prog->aux->offload->dev_priv;
0100 state->is_loaded = loaded;
0101 }
0102
0103 static int
0104 nsim_bpf_offload(struct netdevsim *ns, struct bpf_prog *prog, bool oldprog)
0105 {
0106 nsim_prog_set_loaded(ns->bpf_offloaded, false);
0107
0108 WARN(!!ns->bpf_offloaded != oldprog,
0109 "bad offload state, expected offload %sto be active",
0110 oldprog ? "" : "not ");
0111 ns->bpf_offloaded = prog;
0112 ns->bpf_offloaded_id = prog ? prog->aux->id : 0;
0113 nsim_prog_set_loaded(prog, true);
0114
0115 return 0;
0116 }
0117
0118 int nsim_bpf_setup_tc_block_cb(enum tc_setup_type type,
0119 void *type_data, void *cb_priv)
0120 {
0121 struct tc_cls_bpf_offload *cls_bpf = type_data;
0122 struct bpf_prog *prog = cls_bpf->prog;
0123 struct netdevsim *ns = cb_priv;
0124 struct bpf_prog *oldprog;
0125
0126 if (type != TC_SETUP_CLSBPF) {
0127 NSIM_EA(cls_bpf->common.extack,
0128 "only offload of BPF classifiers supported");
0129 return -EOPNOTSUPP;
0130 }
0131
0132 if (!tc_cls_can_offload_and_chain0(ns->netdev, &cls_bpf->common))
0133 return -EOPNOTSUPP;
0134
0135 if (cls_bpf->common.protocol != htons(ETH_P_ALL)) {
0136 NSIM_EA(cls_bpf->common.extack,
0137 "only ETH_P_ALL supported as filter protocol");
0138 return -EOPNOTSUPP;
0139 }
0140
0141 if (!ns->bpf_tc_accept) {
0142 NSIM_EA(cls_bpf->common.extack,
0143 "netdevsim configured to reject BPF TC offload");
0144 return -EOPNOTSUPP;
0145 }
0146
0147 if (prog && !prog->aux->offload && !ns->bpf_tc_non_bound_accept) {
0148 NSIM_EA(cls_bpf->common.extack,
0149 "netdevsim configured to reject unbound programs");
0150 return -EOPNOTSUPP;
0151 }
0152
0153 if (cls_bpf->command != TC_CLSBPF_OFFLOAD)
0154 return -EOPNOTSUPP;
0155
0156 oldprog = cls_bpf->oldprog;
0157
0158
0159 if (ns->bpf_offloaded != oldprog) {
0160 oldprog = NULL;
0161 if (!cls_bpf->prog)
0162 return 0;
0163 if (ns->bpf_offloaded) {
0164 NSIM_EA(cls_bpf->common.extack,
0165 "driver and netdev offload states mismatch");
0166 return -EBUSY;
0167 }
0168 }
0169
0170 return nsim_bpf_offload(ns, cls_bpf->prog, oldprog);
0171 }
0172
0173 int nsim_bpf_disable_tc(struct netdevsim *ns)
0174 {
0175 if (ns->bpf_offloaded && !nsim_xdp_offload_active(ns))
0176 return -EBUSY;
0177 return 0;
0178 }
0179
0180 static int nsim_xdp_offload_prog(struct netdevsim *ns, struct netdev_bpf *bpf)
0181 {
0182 if (!nsim_xdp_offload_active(ns) && !bpf->prog)
0183 return 0;
0184 if (!nsim_xdp_offload_active(ns) && bpf->prog && ns->bpf_offloaded) {
0185 NSIM_EA(bpf->extack, "TC program is already loaded");
0186 return -EBUSY;
0187 }
0188
0189 return nsim_bpf_offload(ns, bpf->prog, nsim_xdp_offload_active(ns));
0190 }
0191
0192 static int
0193 nsim_xdp_set_prog(struct netdevsim *ns, struct netdev_bpf *bpf,
0194 struct xdp_attachment_info *xdp)
0195 {
0196 int err;
0197
0198 if (bpf->command == XDP_SETUP_PROG && !ns->bpf_xdpdrv_accept) {
0199 NSIM_EA(bpf->extack, "driver XDP disabled in DebugFS");
0200 return -EOPNOTSUPP;
0201 }
0202 if (bpf->command == XDP_SETUP_PROG_HW && !ns->bpf_xdpoffload_accept) {
0203 NSIM_EA(bpf->extack, "XDP offload disabled in DebugFS");
0204 return -EOPNOTSUPP;
0205 }
0206
0207 if (bpf->command == XDP_SETUP_PROG_HW) {
0208 err = nsim_xdp_offload_prog(ns, bpf);
0209 if (err)
0210 return err;
0211 }
0212
0213 xdp_attachment_setup(xdp, bpf);
0214
0215 return 0;
0216 }
0217
0218 static int nsim_bpf_create_prog(struct nsim_dev *nsim_dev,
0219 struct bpf_prog *prog)
0220 {
0221 struct nsim_bpf_bound_prog *state;
0222 char name[16];
0223 int ret;
0224
0225 state = kzalloc(sizeof(*state), GFP_KERNEL);
0226 if (!state)
0227 return -ENOMEM;
0228
0229 state->nsim_dev = nsim_dev;
0230 state->prog = prog;
0231 state->state = "verify";
0232
0233
0234 sprintf(name, "%u", nsim_dev->prog_id_gen++);
0235 state->ddir = debugfs_create_dir(name, nsim_dev->ddir_bpf_bound_progs);
0236 if (IS_ERR(state->ddir)) {
0237 ret = PTR_ERR(state->ddir);
0238 kfree(state);
0239 return ret;
0240 }
0241
0242 debugfs_create_u32("id", 0400, state->ddir, &prog->aux->id);
0243 debugfs_create_file("state", 0400, state->ddir,
0244 &state->state, &nsim_bpf_string_fops);
0245 debugfs_create_bool("loaded", 0400, state->ddir, &state->is_loaded);
0246
0247 list_add_tail(&state->l, &nsim_dev->bpf_bound_progs);
0248
0249 prog->aux->offload->dev_priv = state;
0250
0251 return 0;
0252 }
0253
0254 static int nsim_bpf_verifier_prep(struct bpf_prog *prog)
0255 {
0256 struct nsim_dev *nsim_dev =
0257 bpf_offload_dev_priv(prog->aux->offload->offdev);
0258
0259 if (!nsim_dev->bpf_bind_accept)
0260 return -EOPNOTSUPP;
0261
0262 return nsim_bpf_create_prog(nsim_dev, prog);
0263 }
0264
0265 static int nsim_bpf_translate(struct bpf_prog *prog)
0266 {
0267 struct nsim_bpf_bound_prog *state = prog->aux->offload->dev_priv;
0268
0269 state->state = "xlated";
0270 return 0;
0271 }
0272
0273 static void nsim_bpf_destroy_prog(struct bpf_prog *prog)
0274 {
0275 struct nsim_bpf_bound_prog *state;
0276
0277 state = prog->aux->offload->dev_priv;
0278 WARN(state->is_loaded,
0279 "offload state destroyed while program still bound");
0280 debugfs_remove_recursive(state->ddir);
0281 list_del(&state->l);
0282 kfree(state);
0283 }
0284
0285 static const struct bpf_prog_offload_ops nsim_bpf_dev_ops = {
0286 .insn_hook = nsim_bpf_verify_insn,
0287 .finalize = nsim_bpf_finalize,
0288 .prepare = nsim_bpf_verifier_prep,
0289 .translate = nsim_bpf_translate,
0290 .destroy = nsim_bpf_destroy_prog,
0291 };
0292
0293 static int nsim_setup_prog_checks(struct netdevsim *ns, struct netdev_bpf *bpf)
0294 {
0295 if (bpf->prog && bpf->prog->aux->offload) {
0296 NSIM_EA(bpf->extack, "attempt to load offloaded prog to drv");
0297 return -EINVAL;
0298 }
0299 if (ns->netdev->mtu > NSIM_XDP_MAX_MTU) {
0300 NSIM_EA(bpf->extack, "MTU too large w/ XDP enabled");
0301 return -EINVAL;
0302 }
0303 return 0;
0304 }
0305
0306 static int
0307 nsim_setup_prog_hw_checks(struct netdevsim *ns, struct netdev_bpf *bpf)
0308 {
0309 struct nsim_bpf_bound_prog *state;
0310
0311 if (!bpf->prog)
0312 return 0;
0313
0314 if (!bpf->prog->aux->offload) {
0315 NSIM_EA(bpf->extack, "xdpoffload of non-bound program");
0316 return -EINVAL;
0317 }
0318 if (!bpf_offload_dev_match(bpf->prog, ns->netdev)) {
0319 NSIM_EA(bpf->extack, "program bound to different dev");
0320 return -EINVAL;
0321 }
0322
0323 state = bpf->prog->aux->offload->dev_priv;
0324 if (WARN_ON(strcmp(state->state, "xlated"))) {
0325 NSIM_EA(bpf->extack, "offloading program in bad state");
0326 return -EINVAL;
0327 }
0328 return 0;
0329 }
0330
0331 static bool
0332 nsim_map_key_match(struct bpf_map *map, struct nsim_map_entry *e, void *key)
0333 {
0334 return e->key && !memcmp(key, e->key, map->key_size);
0335 }
0336
0337 static int nsim_map_key_find(struct bpf_offloaded_map *offmap, void *key)
0338 {
0339 struct nsim_bpf_bound_map *nmap = offmap->dev_priv;
0340 unsigned int i;
0341
0342 for (i = 0; i < ARRAY_SIZE(nmap->entry); i++)
0343 if (nsim_map_key_match(&offmap->map, &nmap->entry[i], key))
0344 return i;
0345
0346 return -ENOENT;
0347 }
0348
0349 static int
0350 nsim_map_alloc_elem(struct bpf_offloaded_map *offmap, unsigned int idx)
0351 {
0352 struct nsim_bpf_bound_map *nmap = offmap->dev_priv;
0353
0354 nmap->entry[idx].key = kmalloc(offmap->map.key_size,
0355 GFP_KERNEL_ACCOUNT | __GFP_NOWARN);
0356 if (!nmap->entry[idx].key)
0357 return -ENOMEM;
0358 nmap->entry[idx].value = kmalloc(offmap->map.value_size,
0359 GFP_KERNEL_ACCOUNT | __GFP_NOWARN);
0360 if (!nmap->entry[idx].value) {
0361 kfree(nmap->entry[idx].key);
0362 nmap->entry[idx].key = NULL;
0363 return -ENOMEM;
0364 }
0365
0366 return 0;
0367 }
0368
0369 static int
0370 nsim_map_get_next_key(struct bpf_offloaded_map *offmap,
0371 void *key, void *next_key)
0372 {
0373 struct nsim_bpf_bound_map *nmap = offmap->dev_priv;
0374 int idx = -ENOENT;
0375
0376 mutex_lock(&nmap->mutex);
0377
0378 if (key)
0379 idx = nsim_map_key_find(offmap, key);
0380 if (idx == -ENOENT)
0381 idx = 0;
0382 else
0383 idx++;
0384
0385 for (; idx < ARRAY_SIZE(nmap->entry); idx++) {
0386 if (nmap->entry[idx].key) {
0387 memcpy(next_key, nmap->entry[idx].key,
0388 offmap->map.key_size);
0389 break;
0390 }
0391 }
0392
0393 mutex_unlock(&nmap->mutex);
0394
0395 if (idx == ARRAY_SIZE(nmap->entry))
0396 return -ENOENT;
0397 return 0;
0398 }
0399
0400 static int
0401 nsim_map_lookup_elem(struct bpf_offloaded_map *offmap, void *key, void *value)
0402 {
0403 struct nsim_bpf_bound_map *nmap = offmap->dev_priv;
0404 int idx;
0405
0406 mutex_lock(&nmap->mutex);
0407
0408 idx = nsim_map_key_find(offmap, key);
0409 if (idx >= 0)
0410 memcpy(value, nmap->entry[idx].value, offmap->map.value_size);
0411
0412 mutex_unlock(&nmap->mutex);
0413
0414 return idx < 0 ? idx : 0;
0415 }
0416
0417 static int
0418 nsim_map_update_elem(struct bpf_offloaded_map *offmap,
0419 void *key, void *value, u64 flags)
0420 {
0421 struct nsim_bpf_bound_map *nmap = offmap->dev_priv;
0422 int idx, err = 0;
0423
0424 mutex_lock(&nmap->mutex);
0425
0426 idx = nsim_map_key_find(offmap, key);
0427 if (idx < 0 && flags == BPF_EXIST) {
0428 err = idx;
0429 goto exit_unlock;
0430 }
0431 if (idx >= 0 && flags == BPF_NOEXIST) {
0432 err = -EEXIST;
0433 goto exit_unlock;
0434 }
0435
0436 if (idx < 0) {
0437 for (idx = 0; idx < ARRAY_SIZE(nmap->entry); idx++)
0438 if (!nmap->entry[idx].key)
0439 break;
0440 if (idx == ARRAY_SIZE(nmap->entry)) {
0441 err = -E2BIG;
0442 goto exit_unlock;
0443 }
0444
0445 err = nsim_map_alloc_elem(offmap, idx);
0446 if (err)
0447 goto exit_unlock;
0448 }
0449
0450 memcpy(nmap->entry[idx].key, key, offmap->map.key_size);
0451 memcpy(nmap->entry[idx].value, value, offmap->map.value_size);
0452 exit_unlock:
0453 mutex_unlock(&nmap->mutex);
0454
0455 return err;
0456 }
0457
0458 static int nsim_map_delete_elem(struct bpf_offloaded_map *offmap, void *key)
0459 {
0460 struct nsim_bpf_bound_map *nmap = offmap->dev_priv;
0461 int idx;
0462
0463 if (offmap->map.map_type == BPF_MAP_TYPE_ARRAY)
0464 return -EINVAL;
0465
0466 mutex_lock(&nmap->mutex);
0467
0468 idx = nsim_map_key_find(offmap, key);
0469 if (idx >= 0) {
0470 kfree(nmap->entry[idx].key);
0471 kfree(nmap->entry[idx].value);
0472 memset(&nmap->entry[idx], 0, sizeof(nmap->entry[idx]));
0473 }
0474
0475 mutex_unlock(&nmap->mutex);
0476
0477 return idx < 0 ? idx : 0;
0478 }
0479
0480 static const struct bpf_map_dev_ops nsim_bpf_map_ops = {
0481 .map_get_next_key = nsim_map_get_next_key,
0482 .map_lookup_elem = nsim_map_lookup_elem,
0483 .map_update_elem = nsim_map_update_elem,
0484 .map_delete_elem = nsim_map_delete_elem,
0485 };
0486
0487 static int
0488 nsim_bpf_map_alloc(struct netdevsim *ns, struct bpf_offloaded_map *offmap)
0489 {
0490 struct nsim_bpf_bound_map *nmap;
0491 int i, err;
0492
0493 if (WARN_ON(offmap->map.map_type != BPF_MAP_TYPE_ARRAY &&
0494 offmap->map.map_type != BPF_MAP_TYPE_HASH))
0495 return -EINVAL;
0496 if (offmap->map.max_entries > NSIM_BPF_MAX_KEYS)
0497 return -ENOMEM;
0498 if (offmap->map.map_flags)
0499 return -EINVAL;
0500
0501 nmap = kzalloc(sizeof(*nmap), GFP_KERNEL_ACCOUNT);
0502 if (!nmap)
0503 return -ENOMEM;
0504
0505 offmap->dev_priv = nmap;
0506 nmap->ns = ns;
0507 nmap->map = offmap;
0508 mutex_init(&nmap->mutex);
0509
0510 if (offmap->map.map_type == BPF_MAP_TYPE_ARRAY) {
0511 for (i = 0; i < ARRAY_SIZE(nmap->entry); i++) {
0512 u32 *key;
0513
0514 err = nsim_map_alloc_elem(offmap, i);
0515 if (err)
0516 goto err_free;
0517 key = nmap->entry[i].key;
0518 *key = i;
0519 memset(nmap->entry[i].value, 0, offmap->map.value_size);
0520 }
0521 }
0522
0523 offmap->dev_ops = &nsim_bpf_map_ops;
0524 list_add_tail(&nmap->l, &ns->nsim_dev->bpf_bound_maps);
0525
0526 return 0;
0527
0528 err_free:
0529 while (--i >= 0) {
0530 kfree(nmap->entry[i].key);
0531 kfree(nmap->entry[i].value);
0532 }
0533 kfree(nmap);
0534 return err;
0535 }
0536
0537 static void nsim_bpf_map_free(struct bpf_offloaded_map *offmap)
0538 {
0539 struct nsim_bpf_bound_map *nmap = offmap->dev_priv;
0540 unsigned int i;
0541
0542 for (i = 0; i < ARRAY_SIZE(nmap->entry); i++) {
0543 kfree(nmap->entry[i].key);
0544 kfree(nmap->entry[i].value);
0545 }
0546 list_del_init(&nmap->l);
0547 mutex_destroy(&nmap->mutex);
0548 kfree(nmap);
0549 }
0550
0551 int nsim_bpf(struct net_device *dev, struct netdev_bpf *bpf)
0552 {
0553 struct netdevsim *ns = netdev_priv(dev);
0554 int err;
0555
0556 ASSERT_RTNL();
0557
0558 switch (bpf->command) {
0559 case XDP_SETUP_PROG:
0560 err = nsim_setup_prog_checks(ns, bpf);
0561 if (err)
0562 return err;
0563
0564 return nsim_xdp_set_prog(ns, bpf, &ns->xdp);
0565 case XDP_SETUP_PROG_HW:
0566 err = nsim_setup_prog_hw_checks(ns, bpf);
0567 if (err)
0568 return err;
0569
0570 return nsim_xdp_set_prog(ns, bpf, &ns->xdp_hw);
0571 case BPF_OFFLOAD_MAP_ALLOC:
0572 if (!ns->bpf_map_accept)
0573 return -EOPNOTSUPP;
0574
0575 return nsim_bpf_map_alloc(ns, bpf->offmap);
0576 case BPF_OFFLOAD_MAP_FREE:
0577 nsim_bpf_map_free(bpf->offmap);
0578 return 0;
0579 default:
0580 return -EINVAL;
0581 }
0582 }
0583
0584 int nsim_bpf_dev_init(struct nsim_dev *nsim_dev)
0585 {
0586 int err;
0587
0588 INIT_LIST_HEAD(&nsim_dev->bpf_bound_progs);
0589 INIT_LIST_HEAD(&nsim_dev->bpf_bound_maps);
0590
0591 nsim_dev->ddir_bpf_bound_progs = debugfs_create_dir("bpf_bound_progs",
0592 nsim_dev->ddir);
0593 if (IS_ERR(nsim_dev->ddir_bpf_bound_progs))
0594 return PTR_ERR(nsim_dev->ddir_bpf_bound_progs);
0595
0596 nsim_dev->bpf_dev = bpf_offload_dev_create(&nsim_bpf_dev_ops, nsim_dev);
0597 err = PTR_ERR_OR_ZERO(nsim_dev->bpf_dev);
0598 if (err)
0599 return err;
0600
0601 nsim_dev->bpf_bind_accept = true;
0602 debugfs_create_bool("bpf_bind_accept", 0600, nsim_dev->ddir,
0603 &nsim_dev->bpf_bind_accept);
0604 debugfs_create_u32("bpf_bind_verifier_delay", 0600, nsim_dev->ddir,
0605 &nsim_dev->bpf_bind_verifier_delay);
0606 nsim_dev->bpf_bind_verifier_accept = true;
0607 debugfs_create_bool("bpf_bind_verifier_accept", 0600, nsim_dev->ddir,
0608 &nsim_dev->bpf_bind_verifier_accept);
0609 return 0;
0610 }
0611
0612 void nsim_bpf_dev_exit(struct nsim_dev *nsim_dev)
0613 {
0614 WARN_ON(!list_empty(&nsim_dev->bpf_bound_progs));
0615 WARN_ON(!list_empty(&nsim_dev->bpf_bound_maps));
0616 bpf_offload_dev_destroy(nsim_dev->bpf_dev);
0617 }
0618
0619 int nsim_bpf_init(struct netdevsim *ns)
0620 {
0621 struct dentry *ddir = ns->nsim_dev_port->ddir;
0622 int err;
0623
0624 err = bpf_offload_dev_netdev_register(ns->nsim_dev->bpf_dev,
0625 ns->netdev);
0626 if (err)
0627 return err;
0628
0629 debugfs_create_u32("bpf_offloaded_id", 0400, ddir,
0630 &ns->bpf_offloaded_id);
0631
0632 ns->bpf_tc_accept = true;
0633 debugfs_create_bool("bpf_tc_accept", 0600, ddir,
0634 &ns->bpf_tc_accept);
0635 debugfs_create_bool("bpf_tc_non_bound_accept", 0600, ddir,
0636 &ns->bpf_tc_non_bound_accept);
0637 ns->bpf_xdpdrv_accept = true;
0638 debugfs_create_bool("bpf_xdpdrv_accept", 0600, ddir,
0639 &ns->bpf_xdpdrv_accept);
0640 ns->bpf_xdpoffload_accept = true;
0641 debugfs_create_bool("bpf_xdpoffload_accept", 0600, ddir,
0642 &ns->bpf_xdpoffload_accept);
0643
0644 ns->bpf_map_accept = true;
0645 debugfs_create_bool("bpf_map_accept", 0600, ddir,
0646 &ns->bpf_map_accept);
0647
0648 return 0;
0649 }
0650
0651 void nsim_bpf_uninit(struct netdevsim *ns)
0652 {
0653 WARN_ON(ns->xdp.prog);
0654 WARN_ON(ns->xdp_hw.prog);
0655 WARN_ON(ns->bpf_offloaded);
0656 bpf_offload_dev_netdev_unregister(ns->nsim_dev->bpf_dev, ns->netdev);
0657 }