nfp/abm/ctrl.c

0001 // SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause)
0002 /* Copyright (C) 2018 Netronome Systems, Inc. */
0003
0004 #include <linux/bitops.h>
0005 #include <linux/kernel.h>
0006 #include <linux/log2.h>
0007
0008 #include "../nfpcore/nfp_cpp.h"
0009 #include "../nfpcore/nfp_nffw.h"
0010 #include "../nfp_app.h"
0011 #include "../nfp_abi.h"
0012 #include "../nfp_main.h"
0013 #include "../nfp_net.h"
0014 #include "main.h"
0015
0016 #define NFP_NUM_PRIOS_SYM_NAME  "_abi_pci_dscp_num_prio_%u"
0017 #define NFP_NUM_BANDS_SYM_NAME  "_abi_pci_dscp_num_band_%u"
0018 #define NFP_ACT_MASK_SYM_NAME   "_abi_nfd_out_q_actions_%u"
0019
0020 #define NFP_RED_SUPPORT_SYM_NAME    "_abi_nfd_out_red_offload_%u"
0021
0022 #define NFP_QLVL_SYM_NAME   "_abi_nfd_out_q_lvls_%u%s"
0023 #define NFP_QLVL_STRIDE     16
0024 #define NFP_QLVL_BLOG_BYTES 0
0025 #define NFP_QLVL_BLOG_PKTS  4
0026 #define NFP_QLVL_THRS       8
0027 #define NFP_QLVL_ACT        12
0028
0029 #define NFP_QMSTAT_SYM_NAME "_abi_nfdqm%u_stats%s"
0030 #define NFP_QMSTAT_STRIDE   32
0031 #define NFP_QMSTAT_NON_STO  0
0032 #define NFP_QMSTAT_STO      8
0033 #define NFP_QMSTAT_DROP     16
0034 #define NFP_QMSTAT_ECN      24
0035
0036 #define NFP_Q_STAT_SYM_NAME "_abi_nfd_rxq_stats%u%s"
0037 #define NFP_Q_STAT_STRIDE   16
0038 #define NFP_Q_STAT_PKTS     0
0039 #define NFP_Q_STAT_BYTES    8
0040
0041 #define NFP_NET_ABM_MBOX_CMD        NFP_NET_CFG_MBOX_SIMPLE_CMD
0042 #define NFP_NET_ABM_MBOX_RET        NFP_NET_CFG_MBOX_SIMPLE_RET
0043 #define NFP_NET_ABM_MBOX_DATALEN    NFP_NET_CFG_MBOX_SIMPLE_VAL
0044 #define NFP_NET_ABM_MBOX_RESERVED   (NFP_NET_CFG_MBOX_SIMPLE_VAL + 4)
0045 #define NFP_NET_ABM_MBOX_DATA       (NFP_NET_CFG_MBOX_SIMPLE_VAL + 8)
0046
0047 static int
0048 nfp_abm_ctrl_stat(struct nfp_abm_link *alink, const struct nfp_rtsym *sym,
0049           unsigned int stride, unsigned int offset, unsigned int band,
0050           unsigned int queue, bool is_u64, u64 *res)
0051 {
0052     struct nfp_cpp *cpp = alink->abm->app->cpp;
0053     u64 val, sym_offset;
0054     unsigned int qid;
0055     u32 val32;
0056     int err;
0057
0058     qid = band * NFP_NET_MAX_RX_RINGS + alink->queue_base + queue;
0059
0060     sym_offset = qid * stride + offset;
0061     if (is_u64)
0062         err = __nfp_rtsym_readq(cpp, sym, 3, 0, sym_offset, &val);
0063     else
0064         err = __nfp_rtsym_readl(cpp, sym, 3, 0, sym_offset, &val32);
0065     if (err) {
0066         nfp_err(cpp, "RED offload reading stat failed on vNIC %d band %d queue %d (+ %d)\n",
0067             alink->id, band, queue, alink->queue_base);
0068         return err;
0069     }
0070
0071     *res = is_u64 ? val : val32;
0072     return 0;
0073 }
0074
0075 int __nfp_abm_ctrl_set_q_lvl(struct nfp_abm *abm, unsigned int id, u32 val)
0076 {
0077     struct nfp_cpp *cpp = abm->app->cpp;
0078     u64 sym_offset;
0079     int err;
0080
0081     __clear_bit(id, abm->threshold_undef);
0082     if (abm->thresholds[id] == val)
0083         return 0;
0084
0085     sym_offset = id * NFP_QLVL_STRIDE + NFP_QLVL_THRS;
0086     err = __nfp_rtsym_writel(cpp, abm->q_lvls, 4, 0, sym_offset, val);
0087     if (err) {
0088         nfp_err(cpp,
0089             "RED offload setting level failed on subqueue %d\n",
0090             id);
0091         return err;
0092     }
0093
0094     abm->thresholds[id] = val;
0095     return 0;
0096 }
0097
0098 int nfp_abm_ctrl_set_q_lvl(struct nfp_abm_link *alink, unsigned int band,
0099                unsigned int queue, u32 val)
0100 {
0101     unsigned int threshold;
0102
0103     threshold = band * NFP_NET_MAX_RX_RINGS + alink->queue_base + queue;
0104
0105     return __nfp_abm_ctrl_set_q_lvl(alink->abm, threshold, val);
0106 }
0107
0108 int __nfp_abm_ctrl_set_q_act(struct nfp_abm *abm, unsigned int id,
0109                  enum nfp_abm_q_action act)
0110 {
0111     struct nfp_cpp *cpp = abm->app->cpp;
0112     u64 sym_offset;
0113     int err;
0114
0115     if (abm->actions[id] == act)
0116         return 0;
0117
0118     sym_offset = id * NFP_QLVL_STRIDE + NFP_QLVL_ACT;
0119     err = __nfp_rtsym_writel(cpp, abm->q_lvls, 4, 0, sym_offset, act);
0120     if (err) {
0121         nfp_err(cpp,
0122             "RED offload setting action failed on subqueue %d\n",
0123             id);
0124         return err;
0125     }
0126
0127     abm->actions[id] = act;
0128     return 0;
0129 }
0130
0131 int nfp_abm_ctrl_set_q_act(struct nfp_abm_link *alink, unsigned int band,
0132                unsigned int queue, enum nfp_abm_q_action act)
0133 {
0134     unsigned int qid;
0135
0136     qid = band * NFP_NET_MAX_RX_RINGS + alink->queue_base + queue;
0137
0138     return __nfp_abm_ctrl_set_q_act(alink->abm, qid, act);
0139 }
0140
0141 u64 nfp_abm_ctrl_stat_non_sto(struct nfp_abm_link *alink, unsigned int queue)
0142 {
0143     unsigned int band;
0144     u64 val, sum = 0;
0145
0146     for (band = 0; band < alink->abm->num_bands; band++) {
0147         if (nfp_abm_ctrl_stat(alink, alink->abm->qm_stats,
0148                       NFP_QMSTAT_STRIDE, NFP_QMSTAT_NON_STO,
0149                       band, queue, true, &val))
0150             return 0;
0151         sum += val;
0152     }
0153
0154     return sum;
0155 }
0156
0157 u64 nfp_abm_ctrl_stat_sto(struct nfp_abm_link *alink, unsigned int queue)
0158 {
0159     unsigned int band;
0160     u64 val, sum = 0;
0161
0162     for (band = 0; band < alink->abm->num_bands; band++) {
0163         if (nfp_abm_ctrl_stat(alink, alink->abm->qm_stats,
0164                       NFP_QMSTAT_STRIDE, NFP_QMSTAT_STO,
0165                       band, queue, true, &val))
0166             return 0;
0167         sum += val;
0168     }
0169
0170     return sum;
0171 }
0172
0173 static int
0174 nfp_abm_ctrl_stat_basic(struct nfp_abm_link *alink, unsigned int band,
0175             unsigned int queue, unsigned int off, u64 *val)
0176 {
0177     if (!nfp_abm_has_prio(alink->abm)) {
0178         if (!band) {
0179             unsigned int id = alink->queue_base + queue;
0180
0181             *val = nn_readq(alink->vnic,
0182                     NFP_NET_CFG_RXR_STATS(id) + off);
0183         } else {
0184             *val = 0;
0185         }
0186
0187         return 0;
0188     } else {
0189         return nfp_abm_ctrl_stat(alink, alink->abm->q_stats,
0190                      NFP_Q_STAT_STRIDE, off, band, queue,
0191                      true, val);
0192     }
0193 }
0194
0195 int nfp_abm_ctrl_read_q_stats(struct nfp_abm_link *alink, unsigned int band,
0196                   unsigned int queue, struct nfp_alink_stats *stats)
0197 {
0198     int err;
0199
0200     err = nfp_abm_ctrl_stat_basic(alink, band, queue, NFP_Q_STAT_PKTS,
0201                       &stats->tx_pkts);
0202     if (err)
0203         return err;
0204
0205     err = nfp_abm_ctrl_stat_basic(alink, band, queue, NFP_Q_STAT_BYTES,
0206                       &stats->tx_bytes);
0207     if (err)
0208         return err;
0209
0210     err = nfp_abm_ctrl_stat(alink, alink->abm->q_lvls, NFP_QLVL_STRIDE,
0211                 NFP_QLVL_BLOG_BYTES, band, queue, false,
0212                 &stats->backlog_bytes);
0213     if (err)
0214         return err;
0215
0216     err = nfp_abm_ctrl_stat(alink, alink->abm->q_lvls,
0217                 NFP_QLVL_STRIDE, NFP_QLVL_BLOG_PKTS,
0218                 band, queue, false, &stats->backlog_pkts);
0219     if (err)
0220         return err;
0221
0222     err = nfp_abm_ctrl_stat(alink, alink->abm->qm_stats,
0223                 NFP_QMSTAT_STRIDE, NFP_QMSTAT_DROP,
0224                 band, queue, true, &stats->drops);
0225     if (err)
0226         return err;
0227
0228     return nfp_abm_ctrl_stat(alink, alink->abm->qm_stats,
0229                  NFP_QMSTAT_STRIDE, NFP_QMSTAT_ECN,
0230                  band, queue, true, &stats->overlimits);
0231 }
0232
0233 int nfp_abm_ctrl_read_q_xstats(struct nfp_abm_link *alink,
0234                    unsigned int band, unsigned int queue,
0235                    struct nfp_alink_xstats *xstats)
0236 {
0237     int err;
0238
0239     err = nfp_abm_ctrl_stat(alink, alink->abm->qm_stats,
0240                 NFP_QMSTAT_STRIDE, NFP_QMSTAT_DROP,
0241                 band, queue, true, &xstats->pdrop);
0242     if (err)
0243         return err;
0244
0245     return nfp_abm_ctrl_stat(alink, alink->abm->qm_stats,
0246                  NFP_QMSTAT_STRIDE, NFP_QMSTAT_ECN,
0247                  band, queue, true, &xstats->ecn_marked);
0248 }
0249
0250 int nfp_abm_ctrl_qm_enable(struct nfp_abm *abm)
0251 {
0252     return nfp_mbox_cmd(abm->app->pf, NFP_MBOX_PCIE_ABM_ENABLE,
0253                 NULL, 0, NULL, 0);
0254 }
0255
0256 int nfp_abm_ctrl_qm_disable(struct nfp_abm *abm)
0257 {
0258     return nfp_mbox_cmd(abm->app->pf, NFP_MBOX_PCIE_ABM_DISABLE,
0259                 NULL, 0, NULL, 0);
0260 }
0261
0262 int nfp_abm_ctrl_prio_map_update(struct nfp_abm_link *alink, u32 *packed)
0263 {
0264     const u32 cmd = NFP_NET_CFG_MBOX_CMD_PCI_DSCP_PRIOMAP_SET;
0265     struct nfp_net *nn = alink->vnic;
0266     unsigned int i;
0267     int err;
0268
0269     err = nfp_net_mbox_lock(nn, alink->abm->prio_map_len);
0270     if (err)
0271         return err;
0272
0273     /* Write data_len and wipe reserved */
0274     nn_writeq(nn, nn->tlv_caps.mbox_off + NFP_NET_ABM_MBOX_DATALEN,
0275           alink->abm->prio_map_len);
0276
0277     for (i = 0; i < alink->abm->prio_map_len; i += sizeof(u32))
0278         nn_writel(nn, nn->tlv_caps.mbox_off + NFP_NET_ABM_MBOX_DATA + i,
0279               packed[i / sizeof(u32)]);
0280
0281     err = nfp_net_mbox_reconfig_and_unlock(nn, cmd);
0282     if (err)
0283         nfp_err(alink->abm->app->cpp,
0284             "setting DSCP -> VQ map failed with error %d\n", err);
0285     return err;
0286 }
0287
0288 static int nfp_abm_ctrl_prio_check_params(struct nfp_abm_link *alink)
0289 {
0290     struct nfp_abm *abm = alink->abm;
0291     struct nfp_net *nn = alink->vnic;
0292     unsigned int min_mbox_sz;
0293
0294     if (!nfp_abm_has_prio(alink->abm))
0295         return 0;
0296
0297     min_mbox_sz = NFP_NET_ABM_MBOX_DATA + alink->abm->prio_map_len;
0298     if (nn->tlv_caps.mbox_len < min_mbox_sz) {
0299         nfp_err(abm->app->pf->cpp, "vNIC mailbox too small for prio offload: %u, need: %u\n",
0300             nn->tlv_caps.mbox_len,  min_mbox_sz);
0301         return -EINVAL;
0302     }
0303
0304     return 0;
0305 }
0306
0307 int nfp_abm_ctrl_read_params(struct nfp_abm_link *alink)
0308 {
0309     alink->queue_base = nn_readl(alink->vnic, NFP_NET_CFG_START_RXQ);
0310     alink->queue_base /= alink->vnic->stride_rx;
0311
0312     return nfp_abm_ctrl_prio_check_params(alink);
0313 }
0314
0315 static unsigned int nfp_abm_ctrl_prio_map_size(struct nfp_abm *abm)
0316 {
0317     unsigned int size;
0318
0319     size = roundup_pow_of_two(order_base_2(abm->num_bands));
0320     size = DIV_ROUND_UP(size * abm->num_prios, BITS_PER_BYTE);
0321     size = round_up(size, sizeof(u32));
0322
0323     return size;
0324 }
0325
0326 static const struct nfp_rtsym *
0327 nfp_abm_ctrl_find_rtsym(struct nfp_pf *pf, const char *name, unsigned int size)
0328 {
0329     const struct nfp_rtsym *sym;
0330
0331     sym = nfp_rtsym_lookup(pf->rtbl, name);
0332     if (!sym) {
0333         nfp_err(pf->cpp, "Symbol '%s' not found\n", name);
0334         return ERR_PTR(-ENOENT);
0335     }
0336     if (nfp_rtsym_size(sym) != size) {
0337         nfp_err(pf->cpp,
0338             "Symbol '%s' wrong size: expected %u got %llu\n",
0339             name, size, nfp_rtsym_size(sym));
0340         return ERR_PTR(-EINVAL);
0341     }
0342
0343     return sym;
0344 }
0345
0346 static const struct nfp_rtsym *
0347 nfp_abm_ctrl_find_q_rtsym(struct nfp_abm *abm, const char *name_fmt,
0348               size_t size)
0349 {
0350     char pf_symbol[64];
0351
0352     size = array3_size(size, abm->num_bands, NFP_NET_MAX_RX_RINGS);
0353     snprintf(pf_symbol, sizeof(pf_symbol), name_fmt,
0354          abm->pf_id, nfp_abm_has_prio(abm) ? "_per_band" : "");
0355
0356     return nfp_abm_ctrl_find_rtsym(abm->app->pf, pf_symbol, size);
0357 }
0358
0359 int nfp_abm_ctrl_find_addrs(struct nfp_abm *abm)
0360 {
0361     struct nfp_pf *pf = abm->app->pf;
0362     const struct nfp_rtsym *sym;
0363     int res;
0364
0365     abm->pf_id = nfp_cppcore_pcie_unit(pf->cpp);
0366
0367     /* Check if Qdisc offloads are supported */
0368     res = nfp_pf_rtsym_read_optional(pf, NFP_RED_SUPPORT_SYM_NAME, 1);
0369     if (res < 0)
0370         return res;
0371     abm->red_support = res;
0372
0373     /* Read count of prios and prio bands */
0374     res = nfp_pf_rtsym_read_optional(pf, NFP_NUM_BANDS_SYM_NAME, 1);
0375     if (res < 0)
0376         return res;
0377     abm->num_bands = res;
0378
0379     res = nfp_pf_rtsym_read_optional(pf, NFP_NUM_PRIOS_SYM_NAME, 1);
0380     if (res < 0)
0381         return res;
0382     abm->num_prios = res;
0383
0384     /* Read available actions */
0385     res = nfp_pf_rtsym_read_optional(pf, NFP_ACT_MASK_SYM_NAME,
0386                      BIT(NFP_ABM_ACT_MARK_DROP));
0387     if (res < 0)
0388         return res;
0389     abm->action_mask = res;
0390
0391     abm->prio_map_len = nfp_abm_ctrl_prio_map_size(abm);
0392     abm->dscp_mask = GENMASK(7, 8 - order_base_2(abm->num_prios));
0393
0394     /* Check values are sane, U16_MAX is arbitrarily chosen as max */
0395     if (!is_power_of_2(abm->num_bands) || !is_power_of_2(abm->num_prios) ||
0396         abm->num_bands > U16_MAX || abm->num_prios > U16_MAX ||
0397         (abm->num_bands == 1) != (abm->num_prios == 1)) {
0398         nfp_err(pf->cpp,
0399             "invalid priomap description num bands: %u and num prios: %u\n",
0400             abm->num_bands, abm->num_prios);
0401         return -EINVAL;
0402     }
0403
0404     /* Find level and stat symbols */
0405     if (!abm->red_support)
0406         return 0;
0407
0408     sym = nfp_abm_ctrl_find_q_rtsym(abm, NFP_QLVL_SYM_NAME,
0409                     NFP_QLVL_STRIDE);
0410     if (IS_ERR(sym))
0411         return PTR_ERR(sym);
0412     abm->q_lvls = sym;
0413
0414     sym = nfp_abm_ctrl_find_q_rtsym(abm, NFP_QMSTAT_SYM_NAME,
0415                     NFP_QMSTAT_STRIDE);
0416     if (IS_ERR(sym))
0417         return PTR_ERR(sym);
0418     abm->qm_stats = sym;
0419
0420     if (nfp_abm_has_prio(abm)) {
0421         sym = nfp_abm_ctrl_find_q_rtsym(abm, NFP_Q_STAT_SYM_NAME,
0422                         NFP_Q_STAT_STRIDE);
0423         if (IS_ERR(sym))
0424             return PTR_ERR(sym);
0425         abm->q_stats = sym;
0426     }
0427
0428     return 0;
0429 }