Back to home page

OSCL-LXR
 
 

     

0001 // SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause)
0002 /* Copyright (C) 2017-2018 Netronome Systems, Inc. */
0003 
0004 #include <net/pkt_cls.h>
0005 
0006 #include "../nfpcore/nfp_cpp.h"
0007 #include "../nfpcore/nfp_nffw.h"
0008 #include "../nfpcore/nfp_nsp.h"
0009 #include "../nfp_app.h"
0010 #include "../nfp_main.h"
0011 #include "../nfp_net.h"
0012 #include "../nfp_port.h"
0013 #include "fw.h"
0014 #include "main.h"
0015 
0016 const struct rhashtable_params nfp_bpf_maps_neutral_params = {
0017     .nelem_hint     = 4,
0018     .key_len        = sizeof_field(struct bpf_map, id),
0019     .key_offset     = offsetof(struct nfp_bpf_neutral_map, map_id),
0020     .head_offset        = offsetof(struct nfp_bpf_neutral_map, l),
0021     .automatic_shrinking    = true,
0022 };
0023 
0024 static bool nfp_net_ebpf_capable(struct nfp_net *nn)
0025 {
0026 #ifdef __LITTLE_ENDIAN
0027     struct nfp_app_bpf *bpf = nn->app->priv;
0028 
0029     return nn->cap & NFP_NET_CFG_CTRL_BPF &&
0030            bpf->abi_version &&
0031            nn_readb(nn, NFP_NET_CFG_BPF_ABI) == bpf->abi_version;
0032 #else
0033     return false;
0034 #endif
0035 }
0036 
0037 static int
0038 nfp_bpf_xdp_offload(struct nfp_app *app, struct nfp_net *nn,
0039             struct bpf_prog *prog, struct netlink_ext_ack *extack)
0040 {
0041     bool running, xdp_running;
0042 
0043     if (!nfp_net_ebpf_capable(nn))
0044         return -EINVAL;
0045 
0046     running = nn->dp.ctrl & NFP_NET_CFG_CTRL_BPF;
0047     xdp_running = running && nn->xdp_hw.prog;
0048 
0049     if (!prog && !xdp_running)
0050         return 0;
0051     if (prog && running && !xdp_running)
0052         return -EBUSY;
0053 
0054     return nfp_net_bpf_offload(nn, prog, running, extack);
0055 }
0056 
0057 static const char *nfp_bpf_extra_cap(struct nfp_app *app, struct nfp_net *nn)
0058 {
0059     return nfp_net_ebpf_capable(nn) ? "BPF" : "";
0060 }
0061 
0062 static int
0063 nfp_bpf_vnic_alloc(struct nfp_app *app, struct nfp_net *nn, unsigned int id)
0064 {
0065     struct nfp_pf *pf = app->pf;
0066     struct nfp_bpf_vnic *bv;
0067     int err;
0068 
0069     if (!pf->eth_tbl) {
0070         nfp_err(pf->cpp, "No ETH table\n");
0071         return -EINVAL;
0072     }
0073     if (pf->max_data_vnics != pf->eth_tbl->count) {
0074         nfp_err(pf->cpp, "ETH entries don't match vNICs (%d vs %d)\n",
0075             pf->max_data_vnics, pf->eth_tbl->count);
0076         return -EINVAL;
0077     }
0078 
0079     bv = kzalloc(sizeof(*bv), GFP_KERNEL);
0080     if (!bv)
0081         return -ENOMEM;
0082     nn->app_priv = bv;
0083 
0084     err = nfp_app_nic_vnic_alloc(app, nn, id);
0085     if (err)
0086         goto err_free_priv;
0087 
0088     bv->start_off = nn_readw(nn, NFP_NET_CFG_BPF_START);
0089     bv->tgt_done = nn_readw(nn, NFP_NET_CFG_BPF_DONE);
0090 
0091     return 0;
0092 err_free_priv:
0093     kfree(nn->app_priv);
0094     return err;
0095 }
0096 
0097 static void nfp_bpf_vnic_free(struct nfp_app *app, struct nfp_net *nn)
0098 {
0099     struct nfp_bpf_vnic *bv = nn->app_priv;
0100 
0101     WARN_ON(bv->tc_prog);
0102     kfree(bv);
0103 }
0104 
0105 static int nfp_bpf_setup_tc_block_cb(enum tc_setup_type type,
0106                      void *type_data, void *cb_priv)
0107 {
0108     struct tc_cls_bpf_offload *cls_bpf = type_data;
0109     struct nfp_net *nn = cb_priv;
0110     struct bpf_prog *oldprog;
0111     struct nfp_bpf_vnic *bv;
0112     int err;
0113 
0114     if (type != TC_SETUP_CLSBPF) {
0115         NL_SET_ERR_MSG_MOD(cls_bpf->common.extack,
0116                    "only offload of BPF classifiers supported");
0117         return -EOPNOTSUPP;
0118     }
0119     if (!tc_cls_can_offload_and_chain0(nn->dp.netdev, &cls_bpf->common))
0120         return -EOPNOTSUPP;
0121     if (!nfp_net_ebpf_capable(nn)) {
0122         NL_SET_ERR_MSG_MOD(cls_bpf->common.extack,
0123                    "NFP firmware does not support eBPF offload");
0124         return -EOPNOTSUPP;
0125     }
0126     if (cls_bpf->common.protocol != htons(ETH_P_ALL)) {
0127         NL_SET_ERR_MSG_MOD(cls_bpf->common.extack,
0128                    "only ETH_P_ALL supported as filter protocol");
0129         return -EOPNOTSUPP;
0130     }
0131 
0132     /* Only support TC direct action */
0133     if (!cls_bpf->exts_integrated ||
0134         tcf_exts_has_actions(cls_bpf->exts)) {
0135         NL_SET_ERR_MSG_MOD(cls_bpf->common.extack,
0136                    "only direct action with no legacy actions supported");
0137         return -EOPNOTSUPP;
0138     }
0139 
0140     if (cls_bpf->command != TC_CLSBPF_OFFLOAD)
0141         return -EOPNOTSUPP;
0142 
0143     bv = nn->app_priv;
0144     oldprog = cls_bpf->oldprog;
0145 
0146     /* Don't remove if oldprog doesn't match driver's state */
0147     if (bv->tc_prog != oldprog) {
0148         oldprog = NULL;
0149         if (!cls_bpf->prog)
0150             return 0;
0151     }
0152 
0153     err = nfp_net_bpf_offload(nn, cls_bpf->prog, oldprog,
0154                   cls_bpf->common.extack);
0155     if (err)
0156         return err;
0157 
0158     bv->tc_prog = cls_bpf->prog;
0159     nn->port->tc_offload_cnt = !!bv->tc_prog;
0160     return 0;
0161 }
0162 
0163 static LIST_HEAD(nfp_bpf_block_cb_list);
0164 
0165 static int nfp_bpf_setup_tc(struct nfp_app *app, struct net_device *netdev,
0166                 enum tc_setup_type type, void *type_data)
0167 {
0168     struct nfp_net *nn = netdev_priv(netdev);
0169 
0170     switch (type) {
0171     case TC_SETUP_BLOCK:
0172         return flow_block_cb_setup_simple(type_data,
0173                           &nfp_bpf_block_cb_list,
0174                           nfp_bpf_setup_tc_block_cb,
0175                           nn, nn, true);
0176     default:
0177         return -EOPNOTSUPP;
0178     }
0179 }
0180 
0181 static int
0182 nfp_bpf_check_mtu(struct nfp_app *app, struct net_device *netdev, int new_mtu)
0183 {
0184     struct nfp_net *nn = netdev_priv(netdev);
0185     struct nfp_bpf_vnic *bv;
0186     struct bpf_prog *prog;
0187 
0188     if (~nn->dp.ctrl & NFP_NET_CFG_CTRL_BPF)
0189         return 0;
0190 
0191     if (nn->xdp_hw.prog) {
0192         prog = nn->xdp_hw.prog;
0193     } else {
0194         bv = nn->app_priv;
0195         prog = bv->tc_prog;
0196     }
0197 
0198     if (nfp_bpf_offload_check_mtu(nn, prog, new_mtu)) {
0199         nn_info(nn, "BPF offload active, potential packet access beyond hardware packet boundary");
0200         return -EBUSY;
0201     }
0202     return 0;
0203 }
0204 
0205 static int
0206 nfp_bpf_parse_cap_adjust_head(struct nfp_app_bpf *bpf, void __iomem *value,
0207                   u32 length)
0208 {
0209     struct nfp_bpf_cap_tlv_adjust_head __iomem *cap = value;
0210     struct nfp_cpp *cpp = bpf->app->pf->cpp;
0211 
0212     if (length < sizeof(*cap)) {
0213         nfp_err(cpp, "truncated adjust_head TLV: %d\n", length);
0214         return -EINVAL;
0215     }
0216 
0217     bpf->adjust_head.flags = readl(&cap->flags);
0218     bpf->adjust_head.off_min = readl(&cap->off_min);
0219     bpf->adjust_head.off_max = readl(&cap->off_max);
0220     bpf->adjust_head.guaranteed_sub = readl(&cap->guaranteed_sub);
0221     bpf->adjust_head.guaranteed_add = readl(&cap->guaranteed_add);
0222 
0223     if (bpf->adjust_head.off_min > bpf->adjust_head.off_max) {
0224         nfp_err(cpp, "invalid adjust_head TLV: min > max\n");
0225         return -EINVAL;
0226     }
0227     if (!FIELD_FIT(UR_REG_IMM_MAX, bpf->adjust_head.off_min) ||
0228         !FIELD_FIT(UR_REG_IMM_MAX, bpf->adjust_head.off_max)) {
0229         nfp_warn(cpp, "disabling adjust_head - driver expects min/max to fit in as immediates\n");
0230         memset(&bpf->adjust_head, 0, sizeof(bpf->adjust_head));
0231         return 0;
0232     }
0233 
0234     return 0;
0235 }
0236 
0237 static int
0238 nfp_bpf_parse_cap_func(struct nfp_app_bpf *bpf, void __iomem *value, u32 length)
0239 {
0240     struct nfp_bpf_cap_tlv_func __iomem *cap = value;
0241 
0242     if (length < sizeof(*cap)) {
0243         nfp_err(bpf->app->cpp, "truncated function TLV: %d\n", length);
0244         return -EINVAL;
0245     }
0246 
0247     switch (readl(&cap->func_id)) {
0248     case BPF_FUNC_map_lookup_elem:
0249         bpf->helpers.map_lookup = readl(&cap->func_addr);
0250         break;
0251     case BPF_FUNC_map_update_elem:
0252         bpf->helpers.map_update = readl(&cap->func_addr);
0253         break;
0254     case BPF_FUNC_map_delete_elem:
0255         bpf->helpers.map_delete = readl(&cap->func_addr);
0256         break;
0257     case BPF_FUNC_perf_event_output:
0258         bpf->helpers.perf_event_output = readl(&cap->func_addr);
0259         break;
0260     }
0261 
0262     return 0;
0263 }
0264 
0265 static int
0266 nfp_bpf_parse_cap_maps(struct nfp_app_bpf *bpf, void __iomem *value, u32 length)
0267 {
0268     struct nfp_bpf_cap_tlv_maps __iomem *cap = value;
0269 
0270     if (length < sizeof(*cap)) {
0271         nfp_err(bpf->app->cpp, "truncated maps TLV: %d\n", length);
0272         return -EINVAL;
0273     }
0274 
0275     bpf->maps.types = readl(&cap->types);
0276     bpf->maps.max_maps = readl(&cap->max_maps);
0277     bpf->maps.max_elems = readl(&cap->max_elems);
0278     bpf->maps.max_key_sz = readl(&cap->max_key_sz);
0279     bpf->maps.max_val_sz = readl(&cap->max_val_sz);
0280     bpf->maps.max_elem_sz = readl(&cap->max_elem_sz);
0281 
0282     return 0;
0283 }
0284 
0285 static int
0286 nfp_bpf_parse_cap_random(struct nfp_app_bpf *bpf, void __iomem *value,
0287              u32 length)
0288 {
0289     bpf->pseudo_random = true;
0290     return 0;
0291 }
0292 
0293 static int
0294 nfp_bpf_parse_cap_qsel(struct nfp_app_bpf *bpf, void __iomem *value, u32 length)
0295 {
0296     bpf->queue_select = true;
0297     return 0;
0298 }
0299 
0300 static int
0301 nfp_bpf_parse_cap_adjust_tail(struct nfp_app_bpf *bpf, void __iomem *value,
0302                   u32 length)
0303 {
0304     bpf->adjust_tail = true;
0305     return 0;
0306 }
0307 
0308 static int
0309 nfp_bpf_parse_cap_cmsg_multi_ent(struct nfp_app_bpf *bpf, void __iomem *value,
0310                  u32 length)
0311 {
0312     bpf->cmsg_multi_ent = true;
0313     return 0;
0314 }
0315 
0316 static int
0317 nfp_bpf_parse_cap_abi_version(struct nfp_app_bpf *bpf, void __iomem *value,
0318                   u32 length)
0319 {
0320     if (length < 4) {
0321         nfp_err(bpf->app->cpp, "truncated ABI version TLV: %d\n",
0322             length);
0323         return -EINVAL;
0324     }
0325 
0326     bpf->abi_version = readl(value);
0327     if (bpf->abi_version < 2 || bpf->abi_version > 3) {
0328         nfp_warn(bpf->app->cpp, "unsupported BPF ABI version: %d\n",
0329              bpf->abi_version);
0330         bpf->abi_version = 0;
0331     }
0332 
0333     return 0;
0334 }
0335 
0336 static int nfp_bpf_parse_capabilities(struct nfp_app *app)
0337 {
0338     struct nfp_cpp *cpp = app->pf->cpp;
0339     struct nfp_cpp_area *area;
0340     u8 __iomem *mem, *start;
0341 
0342     mem = nfp_rtsym_map(app->pf->rtbl, "_abi_bpf_capabilities", "bpf.cap",
0343                 8, &area);
0344     if (IS_ERR(mem))
0345         return PTR_ERR(mem) == -ENOENT ? 0 : PTR_ERR(mem);
0346 
0347     start = mem;
0348     while (mem - start + 8 <= nfp_cpp_area_size(area)) {
0349         u8 __iomem *value;
0350         u32 type, length;
0351 
0352         type = readl(mem);
0353         length = readl(mem + 4);
0354         value = mem + 8;
0355 
0356         mem += 8 + length;
0357         if (mem - start > nfp_cpp_area_size(area))
0358             goto err_release_free;
0359 
0360         switch (type) {
0361         case NFP_BPF_CAP_TYPE_FUNC:
0362             if (nfp_bpf_parse_cap_func(app->priv, value, length))
0363                 goto err_release_free;
0364             break;
0365         case NFP_BPF_CAP_TYPE_ADJUST_HEAD:
0366             if (nfp_bpf_parse_cap_adjust_head(app->priv, value,
0367                               length))
0368                 goto err_release_free;
0369             break;
0370         case NFP_BPF_CAP_TYPE_MAPS:
0371             if (nfp_bpf_parse_cap_maps(app->priv, value, length))
0372                 goto err_release_free;
0373             break;
0374         case NFP_BPF_CAP_TYPE_RANDOM:
0375             if (nfp_bpf_parse_cap_random(app->priv, value, length))
0376                 goto err_release_free;
0377             break;
0378         case NFP_BPF_CAP_TYPE_QUEUE_SELECT:
0379             if (nfp_bpf_parse_cap_qsel(app->priv, value, length))
0380                 goto err_release_free;
0381             break;
0382         case NFP_BPF_CAP_TYPE_ADJUST_TAIL:
0383             if (nfp_bpf_parse_cap_adjust_tail(app->priv, value,
0384                               length))
0385                 goto err_release_free;
0386             break;
0387         case NFP_BPF_CAP_TYPE_ABI_VERSION:
0388             if (nfp_bpf_parse_cap_abi_version(app->priv, value,
0389                               length))
0390                 goto err_release_free;
0391             break;
0392         case NFP_BPF_CAP_TYPE_CMSG_MULTI_ENT:
0393             if (nfp_bpf_parse_cap_cmsg_multi_ent(app->priv, value,
0394                                  length))
0395                 goto err_release_free;
0396             break;
0397         default:
0398             nfp_dbg(cpp, "unknown BPF capability: %d\n", type);
0399             break;
0400         }
0401     }
0402     if (mem - start != nfp_cpp_area_size(area)) {
0403         nfp_err(cpp, "BPF capabilities left after parsing, parsed:%zd total length:%zu\n",
0404             mem - start, nfp_cpp_area_size(area));
0405         goto err_release_free;
0406     }
0407 
0408     nfp_cpp_area_release_free(area);
0409 
0410     return 0;
0411 
0412 err_release_free:
0413     nfp_err(cpp, "invalid BPF capabilities at offset:%zd\n", mem - start);
0414     nfp_cpp_area_release_free(area);
0415     return -EINVAL;
0416 }
0417 
0418 static void nfp_bpf_init_capabilities(struct nfp_app_bpf *bpf)
0419 {
0420     bpf->abi_version = 2; /* Original BPF ABI version */
0421 }
0422 
0423 static int nfp_bpf_ndo_init(struct nfp_app *app, struct net_device *netdev)
0424 {
0425     struct nfp_app_bpf *bpf = app->priv;
0426 
0427     return bpf_offload_dev_netdev_register(bpf->bpf_dev, netdev);
0428 }
0429 
0430 static void nfp_bpf_ndo_uninit(struct nfp_app *app, struct net_device *netdev)
0431 {
0432     struct nfp_app_bpf *bpf = app->priv;
0433 
0434     bpf_offload_dev_netdev_unregister(bpf->bpf_dev, netdev);
0435 }
0436 
0437 static int nfp_bpf_start(struct nfp_app *app)
0438 {
0439     struct nfp_app_bpf *bpf = app->priv;
0440 
0441     if (app->ctrl->dp.mtu < nfp_bpf_ctrl_cmsg_min_mtu(bpf)) {
0442         nfp_err(bpf->app->cpp,
0443             "ctrl channel MTU below min required %u < %u\n",
0444             app->ctrl->dp.mtu, nfp_bpf_ctrl_cmsg_min_mtu(bpf));
0445         return -EINVAL;
0446     }
0447 
0448     if (bpf->cmsg_multi_ent)
0449         bpf->cmsg_cache_cnt = nfp_bpf_ctrl_cmsg_cache_cnt(bpf);
0450     else
0451         bpf->cmsg_cache_cnt = 1;
0452 
0453     return 0;
0454 }
0455 
0456 static int nfp_bpf_init(struct nfp_app *app)
0457 {
0458     struct nfp_app_bpf *bpf;
0459     int err;
0460 
0461     bpf = kzalloc(sizeof(*bpf), GFP_KERNEL);
0462     if (!bpf)
0463         return -ENOMEM;
0464     bpf->app = app;
0465     app->priv = bpf;
0466 
0467     INIT_LIST_HEAD(&bpf->map_list);
0468 
0469     err = nfp_ccm_init(&bpf->ccm, app);
0470     if (err)
0471         goto err_free_bpf;
0472 
0473     err = rhashtable_init(&bpf->maps_neutral, &nfp_bpf_maps_neutral_params);
0474     if (err)
0475         goto err_clean_ccm;
0476 
0477     nfp_bpf_init_capabilities(bpf);
0478 
0479     err = nfp_bpf_parse_capabilities(app);
0480     if (err)
0481         goto err_free_neutral_maps;
0482 
0483     if (bpf->abi_version < 3) {
0484         bpf->cmsg_key_sz = CMSG_MAP_KEY_LW * 4;
0485         bpf->cmsg_val_sz = CMSG_MAP_VALUE_LW * 4;
0486     } else {
0487         bpf->cmsg_key_sz = bpf->maps.max_key_sz;
0488         bpf->cmsg_val_sz = bpf->maps.max_val_sz;
0489         app->ctrl_mtu = nfp_bpf_ctrl_cmsg_mtu(bpf);
0490     }
0491 
0492     bpf->bpf_dev = bpf_offload_dev_create(&nfp_bpf_dev_ops, bpf);
0493     err = PTR_ERR_OR_ZERO(bpf->bpf_dev);
0494     if (err)
0495         goto err_free_neutral_maps;
0496 
0497     return 0;
0498 
0499 err_free_neutral_maps:
0500     rhashtable_destroy(&bpf->maps_neutral);
0501 err_clean_ccm:
0502     nfp_ccm_clean(&bpf->ccm);
0503 err_free_bpf:
0504     kfree(bpf);
0505     return err;
0506 }
0507 
0508 static void nfp_bpf_clean(struct nfp_app *app)
0509 {
0510     struct nfp_app_bpf *bpf = app->priv;
0511 
0512     bpf_offload_dev_destroy(bpf->bpf_dev);
0513     nfp_ccm_clean(&bpf->ccm);
0514     WARN_ON(!list_empty(&bpf->map_list));
0515     WARN_ON(bpf->maps_in_use || bpf->map_elems_in_use);
0516     rhashtable_free_and_destroy(&bpf->maps_neutral,
0517                     nfp_check_rhashtable_empty, NULL);
0518     kfree(bpf);
0519 }
0520 
0521 const struct nfp_app_type app_bpf = {
0522     .id     = NFP_APP_BPF_NIC,
0523     .name       = "ebpf",
0524 
0525     .ctrl_cap_mask  = 0,
0526 
0527     .init       = nfp_bpf_init,
0528     .clean      = nfp_bpf_clean,
0529     .start      = nfp_bpf_start,
0530 
0531     .check_mtu  = nfp_bpf_check_mtu,
0532 
0533     .extra_cap  = nfp_bpf_extra_cap,
0534 
0535     .ndo_init   = nfp_bpf_ndo_init,
0536     .ndo_uninit = nfp_bpf_ndo_uninit,
0537 
0538     .vnic_alloc = nfp_bpf_vnic_alloc,
0539     .vnic_free  = nfp_bpf_vnic_free,
0540 
0541     .ctrl_msg_rx    = nfp_bpf_ctrl_msg_rx,
0542     .ctrl_msg_rx_raw    = nfp_bpf_ctrl_msg_rx_raw,
0543 
0544     .setup_tc   = nfp_bpf_setup_tc,
0545     .bpf        = nfp_ndo_bpf,
0546     .xdp_offload    = nfp_bpf_xdp_offload,
0547 };
[ Source navigation ] [ Diff markup ] [ Identifier search ] [ general search ]

This page was automatically generated by the 2.1.0 LXR engine.

The LXR team
Valid CSS 2.1! Valid HTML 4.01!