core/en/qos.c

0001 // SPDX-License-Identifier: GPL-2.0 OR Linux-OpenIB
0002 /* Copyright (c) 2020, Mellanox Technologies inc. All rights reserved. */
0003 #include <net/sch_generic.h>
0004
0005 #include <net/pkt_cls.h>
0006 #include "en.h"
0007 #include "params.h"
0008 #include "../qos.h"
0009 #include "en/htb.h"
0010
0011 struct qos_sq_callback_params {
0012     struct mlx5e_priv *priv;
0013     struct mlx5e_channels *chs;
0014 };
0015
0016 int mlx5e_qos_bytes_rate_check(struct mlx5_core_dev *mdev, u64 nbytes)
0017 {
0018     if (nbytes < BYTES_IN_MBIT) {
0019         qos_warn(mdev, "Input rate (%llu Bytes/sec) below minimum supported (%u Bytes/sec)\n",
0020              nbytes, BYTES_IN_MBIT);
0021         return -EINVAL;
0022     }
0023     return 0;
0024 }
0025
0026 static u32 mlx5e_qos_bytes2mbits(struct mlx5_core_dev *mdev, u64 nbytes)
0027 {
0028     return div_u64(nbytes, BYTES_IN_MBIT);
0029 }
0030
0031 int mlx5e_qos_max_leaf_nodes(struct mlx5_core_dev *mdev)
0032 {
0033     return min(MLX5E_QOS_MAX_LEAF_NODES, mlx5_qos_max_leaf_nodes(mdev));
0034 }
0035
0036 /* TX datapath API */
0037
0038 u16 mlx5e_qid_from_qos(struct mlx5e_channels *chs, u16 qid)
0039 {
0040     /* These channel params are safe to access from the datapath, because:
0041      * 1. This function is called only after checking selq->htb_maj_id != 0,
0042      *    and the number of queues can't change while HTB offload is active.
0043      * 2. When selq->htb_maj_id becomes 0, synchronize_rcu waits for
0044      *    mlx5e_select_queue to finish while holding priv->state_lock,
0045      *    preventing other code from changing the number of queues.
0046      */
0047     bool is_ptp = MLX5E_GET_PFLAG(&chs->params, MLX5E_PFLAG_TX_PORT_TS);
0048
0049     return (chs->params.num_channels + is_ptp) * mlx5e_get_dcb_num_tc(&chs->params) + qid;
0050 }
0051
0052 /* SQ lifecycle */
0053
0054 static struct mlx5e_txqsq *mlx5e_get_qos_sq(struct mlx5e_priv *priv, int qid)
0055 {
0056     struct mlx5e_params *params = &priv->channels.params;
0057     struct mlx5e_txqsq __rcu **qos_sqs;
0058     struct mlx5e_channel *c;
0059     int ix;
0060
0061     ix = qid % params->num_channels;
0062     qid /= params->num_channels;
0063     c = priv->channels.c[ix];
0064
0065     qos_sqs = mlx5e_state_dereference(priv, c->qos_sqs);
0066     return mlx5e_state_dereference(priv, qos_sqs[qid]);
0067 }
0068
0069 int mlx5e_open_qos_sq(struct mlx5e_priv *priv, struct mlx5e_channels *chs,
0070               u16 node_qid, u32 hw_id)
0071 {
0072     struct mlx5e_create_cq_param ccp = {};
0073     struct mlx5e_txqsq __rcu **qos_sqs;
0074     struct mlx5e_sq_param param_sq;
0075     struct mlx5e_cq_param param_cq;
0076     int txq_ix, ix, qid, err = 0;
0077     struct mlx5e_params *params;
0078     struct mlx5e_channel *c;
0079     struct mlx5e_txqsq *sq;
0080
0081     params = &chs->params;
0082
0083     txq_ix = mlx5e_qid_from_qos(chs, node_qid);
0084
0085     WARN_ON(node_qid > priv->htb_max_qos_sqs);
0086     if (node_qid == priv->htb_max_qos_sqs) {
0087         struct mlx5e_sq_stats *stats, **stats_list = NULL;
0088
0089         if (priv->htb_max_qos_sqs == 0) {
0090             stats_list = kvcalloc(mlx5e_qos_max_leaf_nodes(priv->mdev),
0091                           sizeof(*stats_list),
0092                           GFP_KERNEL);
0093             if (!stats_list)
0094                 return -ENOMEM;
0095         }
0096         stats = kzalloc(sizeof(*stats), GFP_KERNEL);
0097         if (!stats) {
0098             kvfree(stats_list);
0099             return -ENOMEM;
0100         }
0101         if (stats_list)
0102             WRITE_ONCE(priv->htb_qos_sq_stats, stats_list);
0103         WRITE_ONCE(priv->htb_qos_sq_stats[node_qid], stats);
0104         /* Order htb_max_qos_sqs increment after writing the array pointer.
0105          * Pairs with smp_load_acquire in en_stats.c.
0106          */
0107         smp_store_release(&priv->htb_max_qos_sqs, priv->htb_max_qos_sqs + 1);
0108     }
0109
0110     ix = node_qid % params->num_channels;
0111     qid = node_qid / params->num_channels;
0112     c = chs->c[ix];
0113
0114     qos_sqs = mlx5e_state_dereference(priv, c->qos_sqs);
0115     sq = kzalloc(sizeof(*sq), GFP_KERNEL);
0116
0117     if (!sq)
0118         return -ENOMEM;
0119
0120     mlx5e_build_create_cq_param(&ccp, c);
0121
0122     memset(&param_sq, 0, sizeof(param_sq));
0123     memset(&param_cq, 0, sizeof(param_cq));
0124     mlx5e_build_sq_param(priv->mdev, params, &param_sq);
0125     mlx5e_build_tx_cq_param(priv->mdev, params, &param_cq);
0126     err = mlx5e_open_cq(priv, params->tx_cq_moderation, &param_cq, &ccp, &sq->cq);
0127     if (err)
0128         goto err_free_sq;
0129     err = mlx5e_open_txqsq(c, priv->tisn[c->lag_port][0], txq_ix, params,
0130                    &param_sq, sq, 0, hw_id,
0131                    priv->htb_qos_sq_stats[node_qid]);
0132     if (err)
0133         goto err_close_cq;
0134
0135     rcu_assign_pointer(qos_sqs[qid], sq);
0136
0137     return 0;
0138
0139 err_close_cq:
0140     mlx5e_close_cq(&sq->cq);
0141 err_free_sq:
0142     kfree(sq);
0143     return err;
0144 }
0145
0146 static int mlx5e_open_qos_sq_cb_wrapper(void *data, u16 node_qid, u32 hw_id)
0147 {
0148     struct qos_sq_callback_params *cb_params = data;
0149
0150     return mlx5e_open_qos_sq(cb_params->priv, cb_params->chs, node_qid, hw_id);
0151 }
0152
0153 int mlx5e_activate_qos_sq(void *data, u16 node_qid, u32 hw_id)
0154 {
0155     struct mlx5e_priv *priv = data;
0156     struct mlx5e_txqsq *sq;
0157     u16 qid;
0158
0159     sq = mlx5e_get_qos_sq(priv, node_qid);
0160
0161     qid = mlx5e_qid_from_qos(&priv->channels, node_qid);
0162
0163     /* If it's a new queue, it will be marked as started at this point.
0164      * Stop it before updating txq2sq.
0165      */
0166     mlx5e_tx_disable_queue(netdev_get_tx_queue(priv->netdev, qid));
0167
0168     priv->txq2sq[qid] = sq;
0169
0170     /* Make the change to txq2sq visible before the queue is started.
0171      * As mlx5e_xmit runs under a spinlock, there is an implicit ACQUIRE,
0172      * which pairs with this barrier.
0173      */
0174     smp_wmb();
0175
0176     qos_dbg(priv->mdev, "Activate QoS SQ qid %u\n", node_qid);
0177     mlx5e_activate_txqsq(sq);
0178
0179     return 0;
0180 }
0181
0182 void mlx5e_deactivate_qos_sq(struct mlx5e_priv *priv, u16 qid)
0183 {
0184     struct mlx5e_txqsq *sq;
0185
0186     sq = mlx5e_get_qos_sq(priv, qid);
0187     if (!sq) /* Handle the case when the SQ failed to open. */
0188         return;
0189
0190     qos_dbg(priv->mdev, "Deactivate QoS SQ qid %u\n", qid);
0191     mlx5e_deactivate_txqsq(sq);
0192
0193     priv->txq2sq[mlx5e_qid_from_qos(&priv->channels, qid)] = NULL;
0194
0195     /* Make the change to txq2sq visible before the queue is started again.
0196      * As mlx5e_xmit runs under a spinlock, there is an implicit ACQUIRE,
0197      * which pairs with this barrier.
0198      */
0199     smp_wmb();
0200 }
0201
0202 void mlx5e_close_qos_sq(struct mlx5e_priv *priv, u16 qid)
0203 {
0204     struct mlx5e_txqsq __rcu **qos_sqs;
0205     struct mlx5e_params *params;
0206     struct mlx5e_channel *c;
0207     struct mlx5e_txqsq *sq;
0208     int ix;
0209
0210     params = &priv->channels.params;
0211
0212     ix = qid % params->num_channels;
0213     qid /= params->num_channels;
0214     c = priv->channels.c[ix];
0215     qos_sqs = mlx5e_state_dereference(priv, c->qos_sqs);
0216     sq = rcu_replace_pointer(qos_sqs[qid], NULL, lockdep_is_held(&priv->state_lock));
0217     if (!sq) /* Handle the case when the SQ failed to open. */
0218         return;
0219
0220     synchronize_rcu(); /* Sync with NAPI. */
0221
0222     mlx5e_close_txqsq(sq);
0223     mlx5e_close_cq(&sq->cq);
0224     kfree(sq);
0225 }
0226
0227 void mlx5e_qos_close_queues(struct mlx5e_channel *c)
0228 {
0229     struct mlx5e_txqsq __rcu **qos_sqs;
0230     int i;
0231
0232     qos_sqs = rcu_replace_pointer(c->qos_sqs, NULL, lockdep_is_held(&c->priv->state_lock));
0233     if (!qos_sqs)
0234         return;
0235     synchronize_rcu(); /* Sync with NAPI. */
0236
0237     for (i = 0; i < c->qos_sqs_size; i++) {
0238         struct mlx5e_txqsq *sq;
0239
0240         sq = mlx5e_state_dereference(c->priv, qos_sqs[i]);
0241         if (!sq) /* Handle the case when the SQ failed to open. */
0242             continue;
0243
0244         mlx5e_close_txqsq(sq);
0245         mlx5e_close_cq(&sq->cq);
0246         kfree(sq);
0247     }
0248
0249     kvfree(qos_sqs);
0250 }
0251
0252 void mlx5e_qos_close_all_queues(struct mlx5e_channels *chs)
0253 {
0254     int i;
0255
0256     for (i = 0; i < chs->num; i++)
0257         mlx5e_qos_close_queues(chs->c[i]);
0258 }
0259
0260 int mlx5e_qos_alloc_queues(struct mlx5e_priv *priv, struct mlx5e_channels *chs)
0261 {
0262     u16 qos_sqs_size;
0263     int i;
0264
0265     qos_sqs_size = DIV_ROUND_UP(mlx5e_qos_max_leaf_nodes(priv->mdev), chs->num);
0266
0267     for (i = 0; i < chs->num; i++) {
0268         struct mlx5e_txqsq **sqs;
0269
0270         sqs = kvcalloc(qos_sqs_size, sizeof(struct mlx5e_txqsq *), GFP_KERNEL);
0271         if (!sqs)
0272             goto err_free;
0273
0274         WRITE_ONCE(chs->c[i]->qos_sqs_size, qos_sqs_size);
0275         smp_wmb(); /* Pairs with mlx5e_napi_poll. */
0276         rcu_assign_pointer(chs->c[i]->qos_sqs, sqs);
0277     }
0278
0279     return 0;
0280
0281 err_free:
0282     while (--i >= 0) {
0283         struct mlx5e_txqsq **sqs;
0284
0285         sqs = rcu_replace_pointer(chs->c[i]->qos_sqs, NULL,
0286                       lockdep_is_held(&priv->state_lock));
0287
0288         synchronize_rcu(); /* Sync with NAPI. */
0289         kvfree(sqs);
0290     }
0291     return -ENOMEM;
0292 }
0293
0294 int mlx5e_qos_open_queues(struct mlx5e_priv *priv, struct mlx5e_channels *chs)
0295 {
0296     struct qos_sq_callback_params callback_params;
0297     int err;
0298
0299     err = mlx5e_qos_alloc_queues(priv, chs);
0300     if (err)
0301         return err;
0302
0303     callback_params.priv = priv;
0304     callback_params.chs = chs;
0305
0306     err = mlx5e_htb_enumerate_leaves(priv->htb, mlx5e_open_qos_sq_cb_wrapper, &callback_params);
0307     if (err) {
0308         mlx5e_qos_close_all_queues(chs);
0309         return err;
0310     }
0311
0312     return 0;
0313 }
0314
0315 void mlx5e_qos_activate_queues(struct mlx5e_priv *priv)
0316 {
0317     mlx5e_htb_enumerate_leaves(priv->htb, mlx5e_activate_qos_sq, priv);
0318 }
0319
0320 void mlx5e_qos_deactivate_queues(struct mlx5e_channel *c)
0321 {
0322     struct mlx5e_params *params = &c->priv->channels.params;
0323     struct mlx5e_txqsq __rcu **qos_sqs;
0324     int i;
0325
0326     qos_sqs = mlx5e_state_dereference(c->priv, c->qos_sqs);
0327     if (!qos_sqs)
0328         return;
0329
0330     for (i = 0; i < c->qos_sqs_size; i++) {
0331         u16 qid = params->num_channels * i + c->ix;
0332         struct mlx5e_txqsq *sq;
0333
0334         sq = mlx5e_state_dereference(c->priv, qos_sqs[i]);
0335         if (!sq) /* Handle the case when the SQ failed to open. */
0336             continue;
0337
0338         qos_dbg(c->mdev, "Deactivate QoS SQ qid %u\n", qid);
0339         mlx5e_deactivate_txqsq(sq);
0340
0341         /* The queue is disabled, no synchronization with datapath is needed. */
0342         c->priv->txq2sq[mlx5e_qid_from_qos(&c->priv->channels, qid)] = NULL;
0343     }
0344 }
0345
0346 void mlx5e_qos_deactivate_all_queues(struct mlx5e_channels *chs)
0347 {
0348     int i;
0349
0350     for (i = 0; i < chs->num; i++)
0351         mlx5e_qos_deactivate_queues(chs->c[i]);
0352 }
0353
0354 void mlx5e_reactivate_qos_sq(struct mlx5e_priv *priv, u16 qid, struct netdev_queue *txq)
0355 {
0356     qos_dbg(priv->mdev, "Reactivate QoS SQ qid %u\n", qid);
0357     netdev_tx_reset_queue(txq);
0358     netif_tx_start_queue(txq);
0359 }
0360
0361 void mlx5e_reset_qdisc(struct net_device *dev, u16 qid)
0362 {
0363     struct netdev_queue *dev_queue = netdev_get_tx_queue(dev, qid);
0364     struct Qdisc *qdisc = dev_queue->qdisc_sleeping;
0365
0366     if (!qdisc)
0367         return;
0368
0369     spin_lock_bh(qdisc_lock(qdisc));
0370     qdisc_reset(qdisc);
0371     spin_unlock_bh(qdisc_lock(qdisc));
0372 }
0373
0374 int mlx5e_htb_setup_tc(struct mlx5e_priv *priv, struct tc_htb_qopt_offload *htb_qopt)
0375 {
0376     struct mlx5e_htb *htb = priv->htb;
0377     int res;
0378
0379     if (!htb && htb_qopt->command != TC_HTB_CREATE)
0380         return -EINVAL;
0381
0382     switch (htb_qopt->command) {
0383     case TC_HTB_CREATE:
0384         if (!mlx5_qos_is_supported(priv->mdev)) {
0385             NL_SET_ERR_MSG_MOD(htb_qopt->extack,
0386                        "Missing QoS capabilities. Try disabling SRIOV or use a supported device.");
0387             return -EOPNOTSUPP;
0388         }
0389         priv->htb = mlx5e_htb_alloc();
0390         htb = priv->htb;
0391         if (!htb)
0392             return -ENOMEM;
0393         res = mlx5e_htb_init(htb, htb_qopt, priv->netdev, priv->mdev, &priv->selq, priv);
0394         if (res) {
0395             mlx5e_htb_free(htb);
0396             priv->htb = NULL;
0397         }
0398         return res;
0399     case TC_HTB_DESTROY:
0400         mlx5e_htb_cleanup(htb);
0401         mlx5e_htb_free(htb);
0402         priv->htb = NULL;
0403         return 0;
0404     case TC_HTB_LEAF_ALLOC_QUEUE:
0405         res = mlx5e_htb_leaf_alloc_queue(htb, htb_qopt->classid, htb_qopt->parent_classid,
0406                          htb_qopt->rate, htb_qopt->ceil, htb_qopt->extack);
0407         if (res < 0)
0408             return res;
0409         htb_qopt->qid = res;
0410         return 0;
0411     case TC_HTB_LEAF_TO_INNER:
0412         return mlx5e_htb_leaf_to_inner(htb, htb_qopt->parent_classid, htb_qopt->classid,
0413                            htb_qopt->rate, htb_qopt->ceil, htb_qopt->extack);
0414     case TC_HTB_LEAF_DEL:
0415         return mlx5e_htb_leaf_del(htb, &htb_qopt->classid, htb_qopt->extack);
0416     case TC_HTB_LEAF_DEL_LAST:
0417     case TC_HTB_LEAF_DEL_LAST_FORCE:
0418         return mlx5e_htb_leaf_del_last(htb, htb_qopt->classid,
0419                            htb_qopt->command == TC_HTB_LEAF_DEL_LAST_FORCE,
0420                            htb_qopt->extack);
0421     case TC_HTB_NODE_MODIFY:
0422         return mlx5e_htb_node_modify(htb, htb_qopt->classid, htb_qopt->rate, htb_qopt->ceil,
0423                          htb_qopt->extack);
0424     case TC_HTB_LEAF_QUERY_QUEUE:
0425         res = mlx5e_htb_get_txq_by_classid(htb, htb_qopt->classid);
0426         if (res < 0)
0427             return res;
0428         htb_qopt->qid = res;
0429         return 0;
0430     default:
0431         return -EOPNOTSUPP;
0432     }
0433 }
0434
0435 struct mlx5e_mqprio_rl {
0436     struct mlx5_core_dev *mdev;
0437     u32 root_id;
0438     u32 *leaves_id;
0439     u8 num_tc;
0440 };
0441
0442 struct mlx5e_mqprio_rl *mlx5e_mqprio_rl_alloc(void)
0443 {
0444     return kvzalloc(sizeof(struct mlx5e_mqprio_rl), GFP_KERNEL);
0445 }
0446
0447 void mlx5e_mqprio_rl_free(struct mlx5e_mqprio_rl *rl)
0448 {
0449     kvfree(rl);
0450 }
0451
0452 int mlx5e_mqprio_rl_init(struct mlx5e_mqprio_rl *rl, struct mlx5_core_dev *mdev, u8 num_tc,
0453              u64 max_rate[])
0454 {
0455     int err;
0456     int tc;
0457
0458     if (!mlx5_qos_is_supported(mdev)) {
0459         qos_warn(mdev, "Missing QoS capabilities. Try disabling SRIOV or use a supported device.");
0460         return -EOPNOTSUPP;
0461     }
0462     if (num_tc > mlx5e_qos_max_leaf_nodes(mdev))
0463         return -EINVAL;
0464
0465     rl->mdev = mdev;
0466     rl->num_tc = num_tc;
0467     rl->leaves_id = kvcalloc(num_tc, sizeof(*rl->leaves_id), GFP_KERNEL);
0468     if (!rl->leaves_id)
0469         return -ENOMEM;
0470
0471     err = mlx5_qos_create_root_node(mdev, &rl->root_id);
0472     if (err)
0473         goto err_free_leaves;
0474
0475     qos_dbg(mdev, "Root created, id %#x\n", rl->root_id);
0476
0477     for (tc = 0; tc < num_tc; tc++) {
0478         u32 max_average_bw;
0479
0480         max_average_bw = mlx5e_qos_bytes2mbits(mdev, max_rate[tc]);
0481         err = mlx5_qos_create_leaf_node(mdev, rl->root_id, 0, max_average_bw,
0482                         &rl->leaves_id[tc]);
0483         if (err)
0484             goto err_destroy_leaves;
0485
0486         qos_dbg(mdev, "Leaf[%d] created, id %#x, max average bw %u Mbits/sec\n",
0487             tc, rl->leaves_id[tc], max_average_bw);
0488     }
0489     return 0;
0490
0491 err_destroy_leaves:
0492     while (--tc >= 0)
0493         mlx5_qos_destroy_node(mdev, rl->leaves_id[tc]);
0494     mlx5_qos_destroy_node(mdev, rl->root_id);
0495 err_free_leaves:
0496     kvfree(rl->leaves_id);
0497     return err;
0498 }
0499
0500 void mlx5e_mqprio_rl_cleanup(struct mlx5e_mqprio_rl *rl)
0501 {
0502     int tc;
0503
0504     for (tc = 0; tc < rl->num_tc; tc++)
0505         mlx5_qos_destroy_node(rl->mdev, rl->leaves_id[tc]);
0506     mlx5_qos_destroy_node(rl->mdev, rl->root_id);
0507     kvfree(rl->leaves_id);
0508 }
0509
0510 int mlx5e_mqprio_rl_get_node_hw_id(struct mlx5e_mqprio_rl *rl, int tc, u32 *hw_id)
0511 {
0512     if (tc >= rl->num_tc)
0513         return -EINVAL;
0514
0515     *hw_id = rl->leaves_id[tc];
0516     return 0;
0517 }
0518