net/sched/sch_multiq.c

0001 // SPDX-License-Identifier: GPL-2.0-only
0002 /*
0003  * Copyright (c) 2008, Intel Corporation.
0004  *
0005  * Author: Alexander Duyck <alexander.h.duyck@intel.com>
0006  */
0007
0008 #include <linux/module.h>
0009 #include <linux/slab.h>
0010 #include <linux/types.h>
0011 #include <linux/kernel.h>
0012 #include <linux/string.h>
0013 #include <linux/errno.h>
0014 #include <linux/skbuff.h>
0015 #include <net/netlink.h>
0016 #include <net/pkt_sched.h>
0017 #include <net/pkt_cls.h>
0018
0019 struct multiq_sched_data {
0020     u16 bands;
0021     u16 max_bands;
0022     u16 curband;
0023     struct tcf_proto __rcu *filter_list;
0024     struct tcf_block *block;
0025     struct Qdisc **queues;
0026 };
0027
0028
0029 static struct Qdisc *
0030 multiq_classify(struct sk_buff *skb, struct Qdisc *sch, int *qerr)
0031 {
0032     struct multiq_sched_data *q = qdisc_priv(sch);
0033     u32 band;
0034     struct tcf_result res;
0035     struct tcf_proto *fl = rcu_dereference_bh(q->filter_list);
0036     int err;
0037
0038     *qerr = NET_XMIT_SUCCESS | __NET_XMIT_BYPASS;
0039     err = tcf_classify(skb, NULL, fl, &res, false);
0040 #ifdef CONFIG_NET_CLS_ACT
0041     switch (err) {
0042     case TC_ACT_STOLEN:
0043     case TC_ACT_QUEUED:
0044     case TC_ACT_TRAP:
0045         *qerr = NET_XMIT_SUCCESS | __NET_XMIT_STOLEN;
0046         fallthrough;
0047     case TC_ACT_SHOT:
0048         return NULL;
0049     }
0050 #endif
0051     band = skb_get_queue_mapping(skb);
0052
0053     if (band >= q->bands)
0054         return q->queues[0];
0055
0056     return q->queues[band];
0057 }
0058
0059 static int
0060 multiq_enqueue(struct sk_buff *skb, struct Qdisc *sch,
0061            struct sk_buff **to_free)
0062 {
0063     struct Qdisc *qdisc;
0064     int ret;
0065
0066     qdisc = multiq_classify(skb, sch, &ret);
0067 #ifdef CONFIG_NET_CLS_ACT
0068     if (qdisc == NULL) {
0069
0070         if (ret & __NET_XMIT_BYPASS)
0071             qdisc_qstats_drop(sch);
0072         __qdisc_drop(skb, to_free);
0073         return ret;
0074     }
0075 #endif
0076
0077     ret = qdisc_enqueue(skb, qdisc, to_free);
0078     if (ret == NET_XMIT_SUCCESS) {
0079         sch->q.qlen++;
0080         return NET_XMIT_SUCCESS;
0081     }
0082     if (net_xmit_drop_count(ret))
0083         qdisc_qstats_drop(sch);
0084     return ret;
0085 }
0086
0087 static struct sk_buff *multiq_dequeue(struct Qdisc *sch)
0088 {
0089     struct multiq_sched_data *q = qdisc_priv(sch);
0090     struct Qdisc *qdisc;
0091     struct sk_buff *skb;
0092     int band;
0093
0094     for (band = 0; band < q->bands; band++) {
0095         /* cycle through bands to ensure fairness */
0096         q->curband++;
0097         if (q->curband >= q->bands)
0098             q->curband = 0;
0099
0100         /* Check that target subqueue is available before
0101          * pulling an skb to avoid head-of-line blocking.
0102          */
0103         if (!netif_xmit_stopped(
0104             netdev_get_tx_queue(qdisc_dev(sch), q->curband))) {
0105             qdisc = q->queues[q->curband];
0106             skb = qdisc->dequeue(qdisc);
0107             if (skb) {
0108                 qdisc_bstats_update(sch, skb);
0109                 sch->q.qlen--;
0110                 return skb;
0111             }
0112         }
0113     }
0114     return NULL;
0115
0116 }
0117
0118 static struct sk_buff *multiq_peek(struct Qdisc *sch)
0119 {
0120     struct multiq_sched_data *q = qdisc_priv(sch);
0121     unsigned int curband = q->curband;
0122     struct Qdisc *qdisc;
0123     struct sk_buff *skb;
0124     int band;
0125
0126     for (band = 0; band < q->bands; band++) {
0127         /* cycle through bands to ensure fairness */
0128         curband++;
0129         if (curband >= q->bands)
0130             curband = 0;
0131
0132         /* Check that target subqueue is available before
0133          * pulling an skb to avoid head-of-line blocking.
0134          */
0135         if (!netif_xmit_stopped(
0136             netdev_get_tx_queue(qdisc_dev(sch), curband))) {
0137             qdisc = q->queues[curband];
0138             skb = qdisc->ops->peek(qdisc);
0139             if (skb)
0140                 return skb;
0141         }
0142     }
0143     return NULL;
0144
0145 }
0146
0147 static void
0148 multiq_reset(struct Qdisc *sch)
0149 {
0150     u16 band;
0151     struct multiq_sched_data *q = qdisc_priv(sch);
0152
0153     for (band = 0; band < q->bands; band++)
0154         qdisc_reset(q->queues[band]);
0155     sch->q.qlen = 0;
0156     q->curband = 0;
0157 }
0158
0159 static void
0160 multiq_destroy(struct Qdisc *sch)
0161 {
0162     int band;
0163     struct multiq_sched_data *q = qdisc_priv(sch);
0164
0165     tcf_block_put(q->block);
0166     for (band = 0; band < q->bands; band++)
0167         qdisc_put(q->queues[band]);
0168
0169     kfree(q->queues);
0170 }
0171
0172 static int multiq_tune(struct Qdisc *sch, struct nlattr *opt,
0173                struct netlink_ext_ack *extack)
0174 {
0175     struct multiq_sched_data *q = qdisc_priv(sch);
0176     struct tc_multiq_qopt *qopt;
0177     struct Qdisc **removed;
0178     int i, n_removed = 0;
0179
0180     if (!netif_is_multiqueue(qdisc_dev(sch)))
0181         return -EOPNOTSUPP;
0182     if (nla_len(opt) < sizeof(*qopt))
0183         return -EINVAL;
0184
0185     qopt = nla_data(opt);
0186
0187     qopt->bands = qdisc_dev(sch)->real_num_tx_queues;
0188
0189     removed = kmalloc(sizeof(*removed) * (q->max_bands - q->bands),
0190               GFP_KERNEL);
0191     if (!removed)
0192         return -ENOMEM;
0193
0194     sch_tree_lock(sch);
0195     q->bands = qopt->bands;
0196     for (i = q->bands; i < q->max_bands; i++) {
0197         if (q->queues[i] != &noop_qdisc) {
0198             struct Qdisc *child = q->queues[i];
0199
0200             q->queues[i] = &noop_qdisc;
0201             qdisc_purge_queue(child);
0202             removed[n_removed++] = child;
0203         }
0204     }
0205
0206     sch_tree_unlock(sch);
0207
0208     for (i = 0; i < n_removed; i++)
0209         qdisc_put(removed[i]);
0210     kfree(removed);
0211
0212     for (i = 0; i < q->bands; i++) {
0213         if (q->queues[i] == &noop_qdisc) {
0214             struct Qdisc *child, *old;
0215             child = qdisc_create_dflt(sch->dev_queue,
0216                           &pfifo_qdisc_ops,
0217                           TC_H_MAKE(sch->handle,
0218                                 i + 1), extack);
0219             if (child) {
0220                 sch_tree_lock(sch);
0221                 old = q->queues[i];
0222                 q->queues[i] = child;
0223                 if (child != &noop_qdisc)
0224                     qdisc_hash_add(child, true);
0225
0226                 if (old != &noop_qdisc)
0227                     qdisc_purge_queue(old);
0228                 sch_tree_unlock(sch);
0229                 qdisc_put(old);
0230             }
0231         }
0232     }
0233     return 0;
0234 }
0235
0236 static int multiq_init(struct Qdisc *sch, struct nlattr *opt,
0237                struct netlink_ext_ack *extack)
0238 {
0239     struct multiq_sched_data *q = qdisc_priv(sch);
0240     int i, err;
0241
0242     q->queues = NULL;
0243
0244     if (!opt)
0245         return -EINVAL;
0246
0247     err = tcf_block_get(&q->block, &q->filter_list, sch, extack);
0248     if (err)
0249         return err;
0250
0251     q->max_bands = qdisc_dev(sch)->num_tx_queues;
0252
0253     q->queues = kcalloc(q->max_bands, sizeof(struct Qdisc *), GFP_KERNEL);
0254     if (!q->queues)
0255         return -ENOBUFS;
0256     for (i = 0; i < q->max_bands; i++)
0257         q->queues[i] = &noop_qdisc;
0258
0259     return multiq_tune(sch, opt, extack);
0260 }
0261
0262 static int multiq_dump(struct Qdisc *sch, struct sk_buff *skb)
0263 {
0264     struct multiq_sched_data *q = qdisc_priv(sch);
0265     unsigned char *b = skb_tail_pointer(skb);
0266     struct tc_multiq_qopt opt;
0267
0268     opt.bands = q->bands;
0269     opt.max_bands = q->max_bands;
0270
0271     if (nla_put(skb, TCA_OPTIONS, sizeof(opt), &opt))
0272         goto nla_put_failure;
0273
0274     return skb->len;
0275
0276 nla_put_failure:
0277     nlmsg_trim(skb, b);
0278     return -1;
0279 }
0280
0281 static int multiq_graft(struct Qdisc *sch, unsigned long arg, struct Qdisc *new,
0282             struct Qdisc **old, struct netlink_ext_ack *extack)
0283 {
0284     struct multiq_sched_data *q = qdisc_priv(sch);
0285     unsigned long band = arg - 1;
0286
0287     if (new == NULL)
0288         new = &noop_qdisc;
0289
0290     *old = qdisc_replace(sch, new, &q->queues[band]);
0291     return 0;
0292 }
0293
0294 static struct Qdisc *
0295 multiq_leaf(struct Qdisc *sch, unsigned long arg)
0296 {
0297     struct multiq_sched_data *q = qdisc_priv(sch);
0298     unsigned long band = arg - 1;
0299
0300     return q->queues[band];
0301 }
0302
0303 static unsigned long multiq_find(struct Qdisc *sch, u32 classid)
0304 {
0305     struct multiq_sched_data *q = qdisc_priv(sch);
0306     unsigned long band = TC_H_MIN(classid);
0307
0308     if (band - 1 >= q->bands)
0309         return 0;
0310     return band;
0311 }
0312
0313 static unsigned long multiq_bind(struct Qdisc *sch, unsigned long parent,
0314                  u32 classid)
0315 {
0316     return multiq_find(sch, classid);
0317 }
0318
0319
0320 static void multiq_unbind(struct Qdisc *q, unsigned long cl)
0321 {
0322 }
0323
0324 static int multiq_dump_class(struct Qdisc *sch, unsigned long cl,
0325                  struct sk_buff *skb, struct tcmsg *tcm)
0326 {
0327     struct multiq_sched_data *q = qdisc_priv(sch);
0328
0329     tcm->tcm_handle |= TC_H_MIN(cl);
0330     tcm->tcm_info = q->queues[cl - 1]->handle;
0331     return 0;
0332 }
0333
0334 static int multiq_dump_class_stats(struct Qdisc *sch, unsigned long cl,
0335                  struct gnet_dump *d)
0336 {
0337     struct multiq_sched_data *q = qdisc_priv(sch);
0338     struct Qdisc *cl_q;
0339
0340     cl_q = q->queues[cl - 1];
0341     if (gnet_stats_copy_basic(d, cl_q->cpu_bstats, &cl_q->bstats, true) < 0 ||
0342         qdisc_qstats_copy(d, cl_q) < 0)
0343         return -1;
0344
0345     return 0;
0346 }
0347
0348 static void multiq_walk(struct Qdisc *sch, struct qdisc_walker *arg)
0349 {
0350     struct multiq_sched_data *q = qdisc_priv(sch);
0351     int band;
0352
0353     if (arg->stop)
0354         return;
0355
0356     for (band = 0; band < q->bands; band++) {
0357         if (arg->count < arg->skip) {
0358             arg->count++;
0359             continue;
0360         }
0361         if (arg->fn(sch, band + 1, arg) < 0) {
0362             arg->stop = 1;
0363             break;
0364         }
0365         arg->count++;
0366     }
0367 }
0368
0369 static struct tcf_block *multiq_tcf_block(struct Qdisc *sch, unsigned long cl,
0370                       struct netlink_ext_ack *extack)
0371 {
0372     struct multiq_sched_data *q = qdisc_priv(sch);
0373
0374     if (cl)
0375         return NULL;
0376     return q->block;
0377 }
0378
0379 static const struct Qdisc_class_ops multiq_class_ops = {
0380     .graft      =   multiq_graft,
0381     .leaf       =   multiq_leaf,
0382     .find       =   multiq_find,
0383     .walk       =   multiq_walk,
0384     .tcf_block  =   multiq_tcf_block,
0385     .bind_tcf   =   multiq_bind,
0386     .unbind_tcf =   multiq_unbind,
0387     .dump       =   multiq_dump_class,
0388     .dump_stats =   multiq_dump_class_stats,
0389 };
0390
0391 static struct Qdisc_ops multiq_qdisc_ops __read_mostly = {
0392     .next       =   NULL,
0393     .cl_ops     =   &multiq_class_ops,
0394     .id     =   "multiq",
0395     .priv_size  =   sizeof(struct multiq_sched_data),
0396     .enqueue    =   multiq_enqueue,
0397     .dequeue    =   multiq_dequeue,
0398     .peek       =   multiq_peek,
0399     .init       =   multiq_init,
0400     .reset      =   multiq_reset,
0401     .destroy    =   multiq_destroy,
0402     .change     =   multiq_tune,
0403     .dump       =   multiq_dump,
0404     .owner      =   THIS_MODULE,
0405 };
0406
0407 static int __init multiq_module_init(void)
0408 {
0409     return register_qdisc(&multiq_qdisc_ops);
0410 }
0411
0412 static void __exit multiq_module_exit(void)
0413 {
0414     unregister_qdisc(&multiq_qdisc_ops);
0415 }
0416
0417 module_init(multiq_module_init)
0418 module_exit(multiq_module_exit)
0419
0420 MODULE_LICENSE("GPL");