0001
0002
0003
0004
0005
0006
0007
0008
0009
0010
0011
0012
0013
0014
0015
0016
0017
0018
0019
0020
0021 #ifndef _LINUX_NETDEVICE_H
0022 #define _LINUX_NETDEVICE_H
0023
0024 #include <linux/timer.h>
0025 #include <linux/bug.h>
0026 #include <linux/delay.h>
0027 #include <linux/atomic.h>
0028 #include <linux/prefetch.h>
0029 #include <asm/cache.h>
0030 #include <asm/byteorder.h>
0031 #include <asm/local.h>
0032
0033 #include <linux/percpu.h>
0034 #include <linux/rculist.h>
0035 #include <linux/workqueue.h>
0036 #include <linux/dynamic_queue_limits.h>
0037
0038 #include <net/net_namespace.h>
0039 #ifdef CONFIG_DCB
0040 #include <net/dcbnl.h>
0041 #endif
0042 #include <net/netprio_cgroup.h>
0043 #include <net/xdp.h>
0044
0045 #include <linux/netdev_features.h>
0046 #include <linux/neighbour.h>
0047 #include <uapi/linux/netdevice.h>
0048 #include <uapi/linux/if_bonding.h>
0049 #include <uapi/linux/pkt_cls.h>
0050 #include <linux/hashtable.h>
0051 #include <linux/rbtree.h>
0052 #include <net/net_trackers.h>
0053 #include <net/net_debug.h>
0054
0055 struct netpoll_info;
0056 struct device;
0057 struct ethtool_ops;
0058 struct phy_device;
0059 struct dsa_port;
0060 struct ip_tunnel_parm;
0061 struct macsec_context;
0062 struct macsec_ops;
0063 struct netdev_name_node;
0064 struct sd_flow_limit;
0065 struct sfp_bus;
0066
0067 struct wireless_dev;
0068
0069 struct wpan_dev;
0070 struct mpls_dev;
0071
0072 struct udp_tunnel_info;
0073 struct udp_tunnel_nic_info;
0074 struct udp_tunnel_nic;
0075 struct bpf_prog;
0076 struct xdp_buff;
0077
0078 void synchronize_net(void);
0079 void netdev_set_default_ethtool_ops(struct net_device *dev,
0080 const struct ethtool_ops *ops);
0081
0082
0083 #define NET_RX_SUCCESS 0
0084 #define NET_RX_DROP 1
0085
0086 #define MAX_NEST_DEV 8
0087
0088
0089
0090
0091
0092
0093
0094
0095
0096
0097
0098
0099
0100
0101
0102
0103
0104
0105
0106 #define NET_XMIT_SUCCESS 0x00
0107 #define NET_XMIT_DROP 0x01
0108 #define NET_XMIT_CN 0x02
0109 #define NET_XMIT_MASK 0x0f
0110
0111
0112
0113
0114 #define net_xmit_eval(e) ((e) == NET_XMIT_CN ? 0 : (e))
0115 #define net_xmit_errno(e) ((e) != NET_XMIT_CN ? -ENOBUFS : 0)
0116
0117
0118 #define NETDEV_TX_MASK 0xf0
0119
0120 enum netdev_tx {
0121 __NETDEV_TX_MIN = INT_MIN,
0122 NETDEV_TX_OK = 0x00,
0123 NETDEV_TX_BUSY = 0x10,
0124 };
0125 typedef enum netdev_tx netdev_tx_t;
0126
0127
0128
0129
0130
0131 static inline bool dev_xmit_complete(int rc)
0132 {
0133
0134
0135
0136
0137
0138
0139 if (likely(rc < NET_XMIT_MASK))
0140 return true;
0141
0142 return false;
0143 }
0144
0145
0146
0147
0148
0149
0150 #if defined(CONFIG_HYPERV_NET)
0151 # define LL_MAX_HEADER 128
0152 #elif defined(CONFIG_WLAN) || IS_ENABLED(CONFIG_AX25)
0153 # if defined(CONFIG_MAC80211_MESH)
0154 # define LL_MAX_HEADER 128
0155 # else
0156 # define LL_MAX_HEADER 96
0157 # endif
0158 #else
0159 # define LL_MAX_HEADER 32
0160 #endif
0161
0162 #if !IS_ENABLED(CONFIG_NET_IPIP) && !IS_ENABLED(CONFIG_NET_IPGRE) && \
0163 !IS_ENABLED(CONFIG_IPV6_SIT) && !IS_ENABLED(CONFIG_IPV6_TUNNEL)
0164 #define MAX_HEADER LL_MAX_HEADER
0165 #else
0166 #define MAX_HEADER (LL_MAX_HEADER + 48)
0167 #endif
0168
0169
0170
0171
0172
0173
0174 struct net_device_stats {
0175 unsigned long rx_packets;
0176 unsigned long tx_packets;
0177 unsigned long rx_bytes;
0178 unsigned long tx_bytes;
0179 unsigned long rx_errors;
0180 unsigned long tx_errors;
0181 unsigned long rx_dropped;
0182 unsigned long tx_dropped;
0183 unsigned long multicast;
0184 unsigned long collisions;
0185 unsigned long rx_length_errors;
0186 unsigned long rx_over_errors;
0187 unsigned long rx_crc_errors;
0188 unsigned long rx_frame_errors;
0189 unsigned long rx_fifo_errors;
0190 unsigned long rx_missed_errors;
0191 unsigned long tx_aborted_errors;
0192 unsigned long tx_carrier_errors;
0193 unsigned long tx_fifo_errors;
0194 unsigned long tx_heartbeat_errors;
0195 unsigned long tx_window_errors;
0196 unsigned long rx_compressed;
0197 unsigned long tx_compressed;
0198 };
0199
0200
0201
0202
0203 struct net_device_core_stats {
0204 unsigned long rx_dropped;
0205 unsigned long tx_dropped;
0206 unsigned long rx_nohandler;
0207 unsigned long rx_otherhost_dropped;
0208 } __aligned(4 * sizeof(unsigned long));
0209
0210 #include <linux/cache.h>
0211 #include <linux/skbuff.h>
0212
0213 #ifdef CONFIG_RPS
0214 #include <linux/static_key.h>
0215 extern struct static_key_false rps_needed;
0216 extern struct static_key_false rfs_needed;
0217 #endif
0218
0219 struct neighbour;
0220 struct neigh_parms;
0221 struct sk_buff;
0222
0223 struct netdev_hw_addr {
0224 struct list_head list;
0225 struct rb_node node;
0226 unsigned char addr[MAX_ADDR_LEN];
0227 unsigned char type;
0228 #define NETDEV_HW_ADDR_T_LAN 1
0229 #define NETDEV_HW_ADDR_T_SAN 2
0230 #define NETDEV_HW_ADDR_T_UNICAST 3
0231 #define NETDEV_HW_ADDR_T_MULTICAST 4
0232 bool global_use;
0233 int sync_cnt;
0234 int refcount;
0235 int synced;
0236 struct rcu_head rcu_head;
0237 };
0238
0239 struct netdev_hw_addr_list {
0240 struct list_head list;
0241 int count;
0242
0243
0244 struct rb_root tree;
0245 };
0246
0247 #define netdev_hw_addr_list_count(l) ((l)->count)
0248 #define netdev_hw_addr_list_empty(l) (netdev_hw_addr_list_count(l) == 0)
0249 #define netdev_hw_addr_list_for_each(ha, l) \
0250 list_for_each_entry(ha, &(l)->list, list)
0251
0252 #define netdev_uc_count(dev) netdev_hw_addr_list_count(&(dev)->uc)
0253 #define netdev_uc_empty(dev) netdev_hw_addr_list_empty(&(dev)->uc)
0254 #define netdev_for_each_uc_addr(ha, dev) \
0255 netdev_hw_addr_list_for_each(ha, &(dev)->uc)
0256
0257 #define netdev_mc_count(dev) netdev_hw_addr_list_count(&(dev)->mc)
0258 #define netdev_mc_empty(dev) netdev_hw_addr_list_empty(&(dev)->mc)
0259 #define netdev_for_each_mc_addr(ha, dev) \
0260 netdev_hw_addr_list_for_each(ha, &(dev)->mc)
0261
0262 struct hh_cache {
0263 unsigned int hh_len;
0264 seqlock_t hh_lock;
0265
0266
0267 #define HH_DATA_MOD 16
0268 #define HH_DATA_OFF(__len) \
0269 (HH_DATA_MOD - (((__len - 1) & (HH_DATA_MOD - 1)) + 1))
0270 #define HH_DATA_ALIGN(__len) \
0271 (((__len)+(HH_DATA_MOD-1))&~(HH_DATA_MOD - 1))
0272 unsigned long hh_data[HH_DATA_ALIGN(LL_MAX_HEADER) / sizeof(long)];
0273 };
0274
0275
0276
0277
0278
0279
0280
0281
0282
0283 #define LL_RESERVED_SPACE(dev) \
0284 ((((dev)->hard_header_len+(dev)->needed_headroom)&~(HH_DATA_MOD - 1)) + HH_DATA_MOD)
0285 #define LL_RESERVED_SPACE_EXTRA(dev,extra) \
0286 ((((dev)->hard_header_len+(dev)->needed_headroom+(extra))&~(HH_DATA_MOD - 1)) + HH_DATA_MOD)
0287
0288 struct header_ops {
0289 int (*create) (struct sk_buff *skb, struct net_device *dev,
0290 unsigned short type, const void *daddr,
0291 const void *saddr, unsigned int len);
0292 int (*parse)(const struct sk_buff *skb, unsigned char *haddr);
0293 int (*cache)(const struct neighbour *neigh, struct hh_cache *hh, __be16 type);
0294 void (*cache_update)(struct hh_cache *hh,
0295 const struct net_device *dev,
0296 const unsigned char *haddr);
0297 bool (*validate)(const char *ll_header, unsigned int len);
0298 __be16 (*parse_protocol)(const struct sk_buff *skb);
0299 };
0300
0301
0302
0303
0304
0305
0306 enum netdev_state_t {
0307 __LINK_STATE_START,
0308 __LINK_STATE_PRESENT,
0309 __LINK_STATE_NOCARRIER,
0310 __LINK_STATE_LINKWATCH_PENDING,
0311 __LINK_STATE_DORMANT,
0312 __LINK_STATE_TESTING,
0313 };
0314
0315 struct gro_list {
0316 struct list_head list;
0317 int count;
0318 };
0319
0320
0321
0322
0323
0324 #define GRO_HASH_BUCKETS 8
0325
0326
0327
0328
0329 struct napi_struct {
0330
0331
0332
0333
0334
0335
0336 struct list_head poll_list;
0337
0338 unsigned long state;
0339 int weight;
0340 int defer_hard_irqs_count;
0341 unsigned long gro_bitmask;
0342 int (*poll)(struct napi_struct *, int);
0343 #ifdef CONFIG_NETPOLL
0344 int poll_owner;
0345 #endif
0346 struct net_device *dev;
0347 struct gro_list gro_hash[GRO_HASH_BUCKETS];
0348 struct sk_buff *skb;
0349 struct list_head rx_list;
0350 int rx_count;
0351 struct hrtimer timer;
0352 struct list_head dev_list;
0353 struct hlist_node napi_hash_node;
0354 unsigned int napi_id;
0355 struct task_struct *thread;
0356 };
0357
0358 enum {
0359 NAPI_STATE_SCHED,
0360 NAPI_STATE_MISSED,
0361 NAPI_STATE_DISABLE,
0362 NAPI_STATE_NPSVC,
0363 NAPI_STATE_LISTED,
0364 NAPI_STATE_NO_BUSY_POLL,
0365 NAPI_STATE_IN_BUSY_POLL,
0366 NAPI_STATE_PREFER_BUSY_POLL,
0367 NAPI_STATE_THREADED,
0368 NAPI_STATE_SCHED_THREADED,
0369 };
0370
0371 enum {
0372 NAPIF_STATE_SCHED = BIT(NAPI_STATE_SCHED),
0373 NAPIF_STATE_MISSED = BIT(NAPI_STATE_MISSED),
0374 NAPIF_STATE_DISABLE = BIT(NAPI_STATE_DISABLE),
0375 NAPIF_STATE_NPSVC = BIT(NAPI_STATE_NPSVC),
0376 NAPIF_STATE_LISTED = BIT(NAPI_STATE_LISTED),
0377 NAPIF_STATE_NO_BUSY_POLL = BIT(NAPI_STATE_NO_BUSY_POLL),
0378 NAPIF_STATE_IN_BUSY_POLL = BIT(NAPI_STATE_IN_BUSY_POLL),
0379 NAPIF_STATE_PREFER_BUSY_POLL = BIT(NAPI_STATE_PREFER_BUSY_POLL),
0380 NAPIF_STATE_THREADED = BIT(NAPI_STATE_THREADED),
0381 NAPIF_STATE_SCHED_THREADED = BIT(NAPI_STATE_SCHED_THREADED),
0382 };
0383
0384 enum gro_result {
0385 GRO_MERGED,
0386 GRO_MERGED_FREE,
0387 GRO_HELD,
0388 GRO_NORMAL,
0389 GRO_CONSUMED,
0390 };
0391 typedef enum gro_result gro_result_t;
0392
0393
0394
0395
0396
0397
0398
0399
0400
0401
0402
0403
0404
0405
0406
0407
0408
0409
0410
0411
0412
0413
0414
0415
0416
0417
0418
0419
0420
0421
0422
0423
0424
0425
0426
0427
0428
0429
0430
0431
0432
0433
0434 enum rx_handler_result {
0435 RX_HANDLER_CONSUMED,
0436 RX_HANDLER_ANOTHER,
0437 RX_HANDLER_EXACT,
0438 RX_HANDLER_PASS,
0439 };
0440 typedef enum rx_handler_result rx_handler_result_t;
0441 typedef rx_handler_result_t rx_handler_func_t(struct sk_buff **pskb);
0442
0443 void __napi_schedule(struct napi_struct *n);
0444 void __napi_schedule_irqoff(struct napi_struct *n);
0445
0446 static inline bool napi_disable_pending(struct napi_struct *n)
0447 {
0448 return test_bit(NAPI_STATE_DISABLE, &n->state);
0449 }
0450
0451 static inline bool napi_prefer_busy_poll(struct napi_struct *n)
0452 {
0453 return test_bit(NAPI_STATE_PREFER_BUSY_POLL, &n->state);
0454 }
0455
0456 bool napi_schedule_prep(struct napi_struct *n);
0457
0458
0459
0460
0461
0462
0463
0464
0465 static inline void napi_schedule(struct napi_struct *n)
0466 {
0467 if (napi_schedule_prep(n))
0468 __napi_schedule(n);
0469 }
0470
0471
0472
0473
0474
0475
0476
0477 static inline void napi_schedule_irqoff(struct napi_struct *n)
0478 {
0479 if (napi_schedule_prep(n))
0480 __napi_schedule_irqoff(n);
0481 }
0482
0483
0484 static inline bool napi_reschedule(struct napi_struct *napi)
0485 {
0486 if (napi_schedule_prep(napi)) {
0487 __napi_schedule(napi);
0488 return true;
0489 }
0490 return false;
0491 }
0492
0493 bool napi_complete_done(struct napi_struct *n, int work_done);
0494
0495
0496
0497
0498
0499
0500
0501
0502 static inline bool napi_complete(struct napi_struct *n)
0503 {
0504 return napi_complete_done(n, 0);
0505 }
0506
0507 int dev_set_threaded(struct net_device *dev, bool threaded);
0508
0509
0510
0511
0512
0513
0514
0515
0516 void napi_disable(struct napi_struct *n);
0517
0518 void napi_enable(struct napi_struct *n);
0519
0520
0521
0522
0523
0524
0525
0526
0527
0528 static inline void napi_synchronize(const struct napi_struct *n)
0529 {
0530 if (IS_ENABLED(CONFIG_SMP))
0531 while (test_bit(NAPI_STATE_SCHED, &n->state))
0532 msleep(1);
0533 else
0534 barrier();
0535 }
0536
0537
0538
0539
0540
0541
0542
0543
0544
0545 static inline bool napi_if_scheduled_mark_missed(struct napi_struct *n)
0546 {
0547 unsigned long val, new;
0548
0549 do {
0550 val = READ_ONCE(n->state);
0551 if (val & NAPIF_STATE_DISABLE)
0552 return true;
0553
0554 if (!(val & NAPIF_STATE_SCHED))
0555 return false;
0556
0557 new = val | NAPIF_STATE_MISSED;
0558 } while (cmpxchg(&n->state, val, new) != val);
0559
0560 return true;
0561 }
0562
0563 enum netdev_queue_state_t {
0564 __QUEUE_STATE_DRV_XOFF,
0565 __QUEUE_STATE_STACK_XOFF,
0566 __QUEUE_STATE_FROZEN,
0567 };
0568
0569 #define QUEUE_STATE_DRV_XOFF (1 << __QUEUE_STATE_DRV_XOFF)
0570 #define QUEUE_STATE_STACK_XOFF (1 << __QUEUE_STATE_STACK_XOFF)
0571 #define QUEUE_STATE_FROZEN (1 << __QUEUE_STATE_FROZEN)
0572
0573 #define QUEUE_STATE_ANY_XOFF (QUEUE_STATE_DRV_XOFF | QUEUE_STATE_STACK_XOFF)
0574 #define QUEUE_STATE_ANY_XOFF_OR_FROZEN (QUEUE_STATE_ANY_XOFF | \
0575 QUEUE_STATE_FROZEN)
0576 #define QUEUE_STATE_DRV_XOFF_OR_FROZEN (QUEUE_STATE_DRV_XOFF | \
0577 QUEUE_STATE_FROZEN)
0578
0579
0580
0581
0582
0583
0584
0585
0586
0587
0588
0589 struct netdev_queue {
0590
0591
0592
0593 struct net_device *dev;
0594 netdevice_tracker dev_tracker;
0595
0596 struct Qdisc __rcu *qdisc;
0597 struct Qdisc *qdisc_sleeping;
0598 #ifdef CONFIG_SYSFS
0599 struct kobject kobj;
0600 #endif
0601 #if defined(CONFIG_XPS) && defined(CONFIG_NUMA)
0602 int numa_node;
0603 #endif
0604 unsigned long tx_maxrate;
0605
0606
0607
0608
0609 atomic_long_t trans_timeout;
0610
0611
0612 struct net_device *sb_dev;
0613 #ifdef CONFIG_XDP_SOCKETS
0614 struct xsk_buff_pool *pool;
0615 #endif
0616
0617
0618
0619 spinlock_t _xmit_lock ____cacheline_aligned_in_smp;
0620 int xmit_lock_owner;
0621
0622
0623
0624 unsigned long trans_start;
0625
0626 unsigned long state;
0627
0628 #ifdef CONFIG_BQL
0629 struct dql dql;
0630 #endif
0631 } ____cacheline_aligned_in_smp;
0632
0633 extern int sysctl_fb_tunnels_only_for_init_net;
0634 extern int sysctl_devconf_inherit_init_net;
0635
0636
0637
0638
0639
0640
0641 static inline bool net_has_fallback_tunnels(const struct net *net)
0642 {
0643 #if IS_ENABLED(CONFIG_SYSCTL)
0644 int fb_tunnels_only_for_init_net = READ_ONCE(sysctl_fb_tunnels_only_for_init_net);
0645
0646 return !fb_tunnels_only_for_init_net ||
0647 (net_eq(net, &init_net) && fb_tunnels_only_for_init_net == 1);
0648 #else
0649 return true;
0650 #endif
0651 }
0652
0653 static inline int net_inherit_devconf(void)
0654 {
0655 #if IS_ENABLED(CONFIG_SYSCTL)
0656 return READ_ONCE(sysctl_devconf_inherit_init_net);
0657 #else
0658 return 0;
0659 #endif
0660 }
0661
0662 static inline int netdev_queue_numa_node_read(const struct netdev_queue *q)
0663 {
0664 #if defined(CONFIG_XPS) && defined(CONFIG_NUMA)
0665 return q->numa_node;
0666 #else
0667 return NUMA_NO_NODE;
0668 #endif
0669 }
0670
0671 static inline void netdev_queue_numa_node_write(struct netdev_queue *q, int node)
0672 {
0673 #if defined(CONFIG_XPS) && defined(CONFIG_NUMA)
0674 q->numa_node = node;
0675 #endif
0676 }
0677
0678 #ifdef CONFIG_RPS
0679
0680
0681
0682
0683 struct rps_map {
0684 unsigned int len;
0685 struct rcu_head rcu;
0686 u16 cpus[];
0687 };
0688 #define RPS_MAP_SIZE(_num) (sizeof(struct rps_map) + ((_num) * sizeof(u16)))
0689
0690
0691
0692
0693
0694
0695 struct rps_dev_flow {
0696 u16 cpu;
0697 u16 filter;
0698 unsigned int last_qtail;
0699 };
0700 #define RPS_NO_FILTER 0xffff
0701
0702
0703
0704
0705 struct rps_dev_flow_table {
0706 unsigned int mask;
0707 struct rcu_head rcu;
0708 struct rps_dev_flow flows[];
0709 };
0710 #define RPS_DEV_FLOW_TABLE_SIZE(_num) (sizeof(struct rps_dev_flow_table) + \
0711 ((_num) * sizeof(struct rps_dev_flow)))
0712
0713
0714
0715
0716
0717
0718
0719
0720
0721
0722
0723 struct rps_sock_flow_table {
0724 u32 mask;
0725
0726 u32 ents[] ____cacheline_aligned_in_smp;
0727 };
0728 #define RPS_SOCK_FLOW_TABLE_SIZE(_num) (offsetof(struct rps_sock_flow_table, ents[_num]))
0729
0730 #define RPS_NO_CPU 0xffff
0731
0732 extern u32 rps_cpu_mask;
0733 extern struct rps_sock_flow_table __rcu *rps_sock_flow_table;
0734
0735 static inline void rps_record_sock_flow(struct rps_sock_flow_table *table,
0736 u32 hash)
0737 {
0738 if (table && hash) {
0739 unsigned int index = hash & table->mask;
0740 u32 val = hash & ~rps_cpu_mask;
0741
0742
0743 val |= raw_smp_processor_id();
0744
0745 if (table->ents[index] != val)
0746 table->ents[index] = val;
0747 }
0748 }
0749
0750 #ifdef CONFIG_RFS_ACCEL
0751 bool rps_may_expire_flow(struct net_device *dev, u16 rxq_index, u32 flow_id,
0752 u16 filter_id);
0753 #endif
0754 #endif
0755
0756
0757 struct netdev_rx_queue {
0758 struct xdp_rxq_info xdp_rxq;
0759 #ifdef CONFIG_RPS
0760 struct rps_map __rcu *rps_map;
0761 struct rps_dev_flow_table __rcu *rps_flow_table;
0762 #endif
0763 struct kobject kobj;
0764 struct net_device *dev;
0765 netdevice_tracker dev_tracker;
0766
0767 #ifdef CONFIG_XDP_SOCKETS
0768 struct xsk_buff_pool *pool;
0769 #endif
0770 } ____cacheline_aligned_in_smp;
0771
0772
0773
0774
0775 struct rx_queue_attribute {
0776 struct attribute attr;
0777 ssize_t (*show)(struct netdev_rx_queue *queue, char *buf);
0778 ssize_t (*store)(struct netdev_rx_queue *queue,
0779 const char *buf, size_t len);
0780 };
0781
0782
0783 enum xps_map_type {
0784 XPS_CPUS = 0,
0785 XPS_RXQS,
0786 XPS_MAPS_MAX,
0787 };
0788
0789 #ifdef CONFIG_XPS
0790
0791
0792
0793
0794 struct xps_map {
0795 unsigned int len;
0796 unsigned int alloc_len;
0797 struct rcu_head rcu;
0798 u16 queues[];
0799 };
0800 #define XPS_MAP_SIZE(_num) (sizeof(struct xps_map) + ((_num) * sizeof(u16)))
0801 #define XPS_MIN_MAP_ALLOC ((L1_CACHE_ALIGN(offsetof(struct xps_map, queues[1])) \
0802 - sizeof(struct xps_map)) / sizeof(u16))
0803
0804
0805
0806
0807
0808
0809
0810
0811
0812
0813
0814
0815 struct xps_dev_maps {
0816 struct rcu_head rcu;
0817 unsigned int nr_ids;
0818 s16 num_tc;
0819 struct xps_map __rcu *attr_map[];
0820 };
0821
0822 #define XPS_CPU_DEV_MAPS_SIZE(_tcs) (sizeof(struct xps_dev_maps) + \
0823 (nr_cpu_ids * (_tcs) * sizeof(struct xps_map *)))
0824
0825 #define XPS_RXQ_DEV_MAPS_SIZE(_tcs, _rxqs) (sizeof(struct xps_dev_maps) +\
0826 (_rxqs * (_tcs) * sizeof(struct xps_map *)))
0827
0828 #endif
0829
0830 #define TC_MAX_QUEUE 16
0831 #define TC_BITMASK 15
0832
0833 struct netdev_tc_txq {
0834 u16 count;
0835 u16 offset;
0836 };
0837
0838 #if defined(CONFIG_FCOE) || defined(CONFIG_FCOE_MODULE)
0839
0840
0841
0842
0843 struct netdev_fcoe_hbainfo {
0844 char manufacturer[64];
0845 char serial_number[64];
0846 char hardware_version[64];
0847 char driver_version[64];
0848 char optionrom_version[64];
0849 char firmware_version[64];
0850 char model[256];
0851 char model_description[256];
0852 };
0853 #endif
0854
0855 #define MAX_PHYS_ITEM_ID_LEN 32
0856
0857
0858
0859
0860 struct netdev_phys_item_id {
0861 unsigned char id[MAX_PHYS_ITEM_ID_LEN];
0862 unsigned char id_len;
0863 };
0864
0865 static inline bool netdev_phys_item_id_same(struct netdev_phys_item_id *a,
0866 struct netdev_phys_item_id *b)
0867 {
0868 return a->id_len == b->id_len &&
0869 memcmp(a->id, b->id, a->id_len) == 0;
0870 }
0871
0872 typedef u16 (*select_queue_fallback_t)(struct net_device *dev,
0873 struct sk_buff *skb,
0874 struct net_device *sb_dev);
0875
0876 enum net_device_path_type {
0877 DEV_PATH_ETHERNET = 0,
0878 DEV_PATH_VLAN,
0879 DEV_PATH_BRIDGE,
0880 DEV_PATH_PPPOE,
0881 DEV_PATH_DSA,
0882 DEV_PATH_MTK_WDMA,
0883 };
0884
0885 struct net_device_path {
0886 enum net_device_path_type type;
0887 const struct net_device *dev;
0888 union {
0889 struct {
0890 u16 id;
0891 __be16 proto;
0892 u8 h_dest[ETH_ALEN];
0893 } encap;
0894 struct {
0895 enum {
0896 DEV_PATH_BR_VLAN_KEEP,
0897 DEV_PATH_BR_VLAN_TAG,
0898 DEV_PATH_BR_VLAN_UNTAG,
0899 DEV_PATH_BR_VLAN_UNTAG_HW,
0900 } vlan_mode;
0901 u16 vlan_id;
0902 __be16 vlan_proto;
0903 } bridge;
0904 struct {
0905 int port;
0906 u16 proto;
0907 } dsa;
0908 struct {
0909 u8 wdma_idx;
0910 u8 queue;
0911 u16 wcid;
0912 u8 bss;
0913 } mtk_wdma;
0914 };
0915 };
0916
0917 #define NET_DEVICE_PATH_STACK_MAX 5
0918 #define NET_DEVICE_PATH_VLAN_MAX 2
0919
0920 struct net_device_path_stack {
0921 int num_paths;
0922 struct net_device_path path[NET_DEVICE_PATH_STACK_MAX];
0923 };
0924
0925 struct net_device_path_ctx {
0926 const struct net_device *dev;
0927 u8 daddr[ETH_ALEN];
0928
0929 int num_vlans;
0930 struct {
0931 u16 id;
0932 __be16 proto;
0933 } vlan[NET_DEVICE_PATH_VLAN_MAX];
0934 };
0935
0936 enum tc_setup_type {
0937 TC_SETUP_QDISC_MQPRIO,
0938 TC_SETUP_CLSU32,
0939 TC_SETUP_CLSFLOWER,
0940 TC_SETUP_CLSMATCHALL,
0941 TC_SETUP_CLSBPF,
0942 TC_SETUP_BLOCK,
0943 TC_SETUP_QDISC_CBS,
0944 TC_SETUP_QDISC_RED,
0945 TC_SETUP_QDISC_PRIO,
0946 TC_SETUP_QDISC_MQ,
0947 TC_SETUP_QDISC_ETF,
0948 TC_SETUP_ROOT_QDISC,
0949 TC_SETUP_QDISC_GRED,
0950 TC_SETUP_QDISC_TAPRIO,
0951 TC_SETUP_FT,
0952 TC_SETUP_QDISC_ETS,
0953 TC_SETUP_QDISC_TBF,
0954 TC_SETUP_QDISC_FIFO,
0955 TC_SETUP_QDISC_HTB,
0956 TC_SETUP_ACT,
0957 };
0958
0959
0960
0961
0962 enum bpf_netdev_command {
0963
0964
0965
0966
0967
0968
0969
0970 XDP_SETUP_PROG,
0971 XDP_SETUP_PROG_HW,
0972
0973 BPF_OFFLOAD_MAP_ALLOC,
0974 BPF_OFFLOAD_MAP_FREE,
0975 XDP_SETUP_XSK_POOL,
0976 };
0977
0978 struct bpf_prog_offload_ops;
0979 struct netlink_ext_ack;
0980 struct xdp_umem;
0981 struct xdp_dev_bulk_queue;
0982 struct bpf_xdp_link;
0983
0984 enum bpf_xdp_mode {
0985 XDP_MODE_SKB = 0,
0986 XDP_MODE_DRV = 1,
0987 XDP_MODE_HW = 2,
0988 __MAX_XDP_MODE
0989 };
0990
0991 struct bpf_xdp_entity {
0992 struct bpf_prog *prog;
0993 struct bpf_xdp_link *link;
0994 };
0995
0996 struct netdev_bpf {
0997 enum bpf_netdev_command command;
0998 union {
0999
1000 struct {
1001 u32 flags;
1002 struct bpf_prog *prog;
1003 struct netlink_ext_ack *extack;
1004 };
1005
1006 struct {
1007 struct bpf_offloaded_map *offmap;
1008 };
1009
1010 struct {
1011 struct xsk_buff_pool *pool;
1012 u16 queue_id;
1013 } xsk;
1014 };
1015 };
1016
1017
1018 #define XDP_WAKEUP_RX (1 << 0)
1019 #define XDP_WAKEUP_TX (1 << 1)
1020
1021 #ifdef CONFIG_XFRM_OFFLOAD
1022 struct xfrmdev_ops {
1023 int (*xdo_dev_state_add) (struct xfrm_state *x);
1024 void (*xdo_dev_state_delete) (struct xfrm_state *x);
1025 void (*xdo_dev_state_free) (struct xfrm_state *x);
1026 bool (*xdo_dev_offload_ok) (struct sk_buff *skb,
1027 struct xfrm_state *x);
1028 void (*xdo_dev_state_advance_esn) (struct xfrm_state *x);
1029 };
1030 #endif
1031
1032 struct dev_ifalias {
1033 struct rcu_head rcuhead;
1034 char ifalias[];
1035 };
1036
1037 struct devlink;
1038 struct tlsdev_ops;
1039
1040 struct netdev_net_notifier {
1041 struct list_head list;
1042 struct notifier_block *nb;
1043 };
1044
1045
1046
1047
1048
1049
1050
1051
1052
1053
1054
1055
1056
1057
1058
1059
1060
1061
1062
1063
1064
1065
1066
1067
1068
1069
1070
1071
1072
1073
1074
1075
1076
1077
1078
1079
1080
1081
1082
1083
1084
1085
1086
1087
1088
1089
1090
1091
1092
1093
1094
1095
1096
1097
1098
1099
1100
1101
1102
1103
1104
1105
1106
1107
1108
1109
1110
1111
1112
1113
1114
1115
1116
1117
1118
1119
1120
1121
1122
1123
1124
1125
1126
1127
1128
1129
1130
1131
1132
1133
1134
1135
1136
1137
1138
1139
1140
1141
1142
1143
1144
1145
1146
1147
1148
1149
1150
1151
1152
1153
1154
1155
1156
1157
1158
1159
1160
1161
1162
1163
1164
1165
1166
1167
1168
1169
1170
1171
1172
1173
1174
1175
1176
1177
1178
1179
1180
1181
1182
1183
1184
1185
1186
1187
1188
1189
1190
1191
1192
1193
1194
1195
1196
1197
1198
1199
1200
1201
1202
1203
1204
1205
1206
1207
1208
1209
1210
1211
1212
1213
1214
1215
1216
1217
1218
1219
1220
1221
1222
1223
1224
1225
1226
1227
1228
1229
1230
1231
1232
1233
1234
1235
1236
1237
1238
1239
1240
1241
1242
1243
1244
1245
1246
1247
1248
1249
1250
1251
1252
1253
1254
1255
1256
1257
1258
1259
1260
1261
1262
1263
1264
1265
1266
1267
1268
1269
1270
1271
1272
1273
1274
1275
1276
1277
1278
1279
1280
1281
1282
1283
1284
1285
1286
1287
1288
1289
1290
1291
1292
1293
1294
1295
1296
1297
1298
1299
1300
1301
1302
1303
1304
1305
1306
1307
1308
1309
1310
1311
1312
1313
1314
1315
1316
1317
1318
1319
1320
1321
1322
1323
1324
1325
1326
1327
1328
1329
1330
1331
1332
1333
1334
1335
1336
1337
1338
1339
1340
1341
1342
1343
1344
1345
1346
1347
1348
1349
1350
1351
1352
1353
1354
1355
1356
1357
1358
1359
1360
1361
1362
1363
1364
1365
1366
1367
1368
1369
1370
1371
1372
1373
1374
1375
1376
1377
1378
1379
1380
1381 struct net_device_ops {
1382 int (*ndo_init)(struct net_device *dev);
1383 void (*ndo_uninit)(struct net_device *dev);
1384 int (*ndo_open)(struct net_device *dev);
1385 int (*ndo_stop)(struct net_device *dev);
1386 netdev_tx_t (*ndo_start_xmit)(struct sk_buff *skb,
1387 struct net_device *dev);
1388 netdev_features_t (*ndo_features_check)(struct sk_buff *skb,
1389 struct net_device *dev,
1390 netdev_features_t features);
1391 u16 (*ndo_select_queue)(struct net_device *dev,
1392 struct sk_buff *skb,
1393 struct net_device *sb_dev);
1394 void (*ndo_change_rx_flags)(struct net_device *dev,
1395 int flags);
1396 void (*ndo_set_rx_mode)(struct net_device *dev);
1397 int (*ndo_set_mac_address)(struct net_device *dev,
1398 void *addr);
1399 int (*ndo_validate_addr)(struct net_device *dev);
1400 int (*ndo_do_ioctl)(struct net_device *dev,
1401 struct ifreq *ifr, int cmd);
1402 int (*ndo_eth_ioctl)(struct net_device *dev,
1403 struct ifreq *ifr, int cmd);
1404 int (*ndo_siocbond)(struct net_device *dev,
1405 struct ifreq *ifr, int cmd);
1406 int (*ndo_siocwandev)(struct net_device *dev,
1407 struct if_settings *ifs);
1408 int (*ndo_siocdevprivate)(struct net_device *dev,
1409 struct ifreq *ifr,
1410 void __user *data, int cmd);
1411 int (*ndo_set_config)(struct net_device *dev,
1412 struct ifmap *map);
1413 int (*ndo_change_mtu)(struct net_device *dev,
1414 int new_mtu);
1415 int (*ndo_neigh_setup)(struct net_device *dev,
1416 struct neigh_parms *);
1417 void (*ndo_tx_timeout) (struct net_device *dev,
1418 unsigned int txqueue);
1419
1420 void (*ndo_get_stats64)(struct net_device *dev,
1421 struct rtnl_link_stats64 *storage);
1422 bool (*ndo_has_offload_stats)(const struct net_device *dev, int attr_id);
1423 int (*ndo_get_offload_stats)(int attr_id,
1424 const struct net_device *dev,
1425 void *attr_data);
1426 struct net_device_stats* (*ndo_get_stats)(struct net_device *dev);
1427
1428 int (*ndo_vlan_rx_add_vid)(struct net_device *dev,
1429 __be16 proto, u16 vid);
1430 int (*ndo_vlan_rx_kill_vid)(struct net_device *dev,
1431 __be16 proto, u16 vid);
1432 #ifdef CONFIG_NET_POLL_CONTROLLER
1433 void (*ndo_poll_controller)(struct net_device *dev);
1434 int (*ndo_netpoll_setup)(struct net_device *dev,
1435 struct netpoll_info *info);
1436 void (*ndo_netpoll_cleanup)(struct net_device *dev);
1437 #endif
1438 int (*ndo_set_vf_mac)(struct net_device *dev,
1439 int queue, u8 *mac);
1440 int (*ndo_set_vf_vlan)(struct net_device *dev,
1441 int queue, u16 vlan,
1442 u8 qos, __be16 proto);
1443 int (*ndo_set_vf_rate)(struct net_device *dev,
1444 int vf, int min_tx_rate,
1445 int max_tx_rate);
1446 int (*ndo_set_vf_spoofchk)(struct net_device *dev,
1447 int vf, bool setting);
1448 int (*ndo_set_vf_trust)(struct net_device *dev,
1449 int vf, bool setting);
1450 int (*ndo_get_vf_config)(struct net_device *dev,
1451 int vf,
1452 struct ifla_vf_info *ivf);
1453 int (*ndo_set_vf_link_state)(struct net_device *dev,
1454 int vf, int link_state);
1455 int (*ndo_get_vf_stats)(struct net_device *dev,
1456 int vf,
1457 struct ifla_vf_stats
1458 *vf_stats);
1459 int (*ndo_set_vf_port)(struct net_device *dev,
1460 int vf,
1461 struct nlattr *port[]);
1462 int (*ndo_get_vf_port)(struct net_device *dev,
1463 int vf, struct sk_buff *skb);
1464 int (*ndo_get_vf_guid)(struct net_device *dev,
1465 int vf,
1466 struct ifla_vf_guid *node_guid,
1467 struct ifla_vf_guid *port_guid);
1468 int (*ndo_set_vf_guid)(struct net_device *dev,
1469 int vf, u64 guid,
1470 int guid_type);
1471 int (*ndo_set_vf_rss_query_en)(
1472 struct net_device *dev,
1473 int vf, bool setting);
1474 int (*ndo_setup_tc)(struct net_device *dev,
1475 enum tc_setup_type type,
1476 void *type_data);
1477 #if IS_ENABLED(CONFIG_FCOE)
1478 int (*ndo_fcoe_enable)(struct net_device *dev);
1479 int (*ndo_fcoe_disable)(struct net_device *dev);
1480 int (*ndo_fcoe_ddp_setup)(struct net_device *dev,
1481 u16 xid,
1482 struct scatterlist *sgl,
1483 unsigned int sgc);
1484 int (*ndo_fcoe_ddp_done)(struct net_device *dev,
1485 u16 xid);
1486 int (*ndo_fcoe_ddp_target)(struct net_device *dev,
1487 u16 xid,
1488 struct scatterlist *sgl,
1489 unsigned int sgc);
1490 int (*ndo_fcoe_get_hbainfo)(struct net_device *dev,
1491 struct netdev_fcoe_hbainfo *hbainfo);
1492 #endif
1493
1494 #if IS_ENABLED(CONFIG_LIBFCOE)
1495 #define NETDEV_FCOE_WWNN 0
1496 #define NETDEV_FCOE_WWPN 1
1497 int (*ndo_fcoe_get_wwn)(struct net_device *dev,
1498 u64 *wwn, int type);
1499 #endif
1500
1501 #ifdef CONFIG_RFS_ACCEL
1502 int (*ndo_rx_flow_steer)(struct net_device *dev,
1503 const struct sk_buff *skb,
1504 u16 rxq_index,
1505 u32 flow_id);
1506 #endif
1507 int (*ndo_add_slave)(struct net_device *dev,
1508 struct net_device *slave_dev,
1509 struct netlink_ext_ack *extack);
1510 int (*ndo_del_slave)(struct net_device *dev,
1511 struct net_device *slave_dev);
1512 struct net_device* (*ndo_get_xmit_slave)(struct net_device *dev,
1513 struct sk_buff *skb,
1514 bool all_slaves);
1515 struct net_device* (*ndo_sk_get_lower_dev)(struct net_device *dev,
1516 struct sock *sk);
1517 netdev_features_t (*ndo_fix_features)(struct net_device *dev,
1518 netdev_features_t features);
1519 int (*ndo_set_features)(struct net_device *dev,
1520 netdev_features_t features);
1521 int (*ndo_neigh_construct)(struct net_device *dev,
1522 struct neighbour *n);
1523 void (*ndo_neigh_destroy)(struct net_device *dev,
1524 struct neighbour *n);
1525
1526 int (*ndo_fdb_add)(struct ndmsg *ndm,
1527 struct nlattr *tb[],
1528 struct net_device *dev,
1529 const unsigned char *addr,
1530 u16 vid,
1531 u16 flags,
1532 struct netlink_ext_ack *extack);
1533 int (*ndo_fdb_del)(struct ndmsg *ndm,
1534 struct nlattr *tb[],
1535 struct net_device *dev,
1536 const unsigned char *addr,
1537 u16 vid, struct netlink_ext_ack *extack);
1538 int (*ndo_fdb_del_bulk)(struct ndmsg *ndm,
1539 struct nlattr *tb[],
1540 struct net_device *dev,
1541 u16 vid,
1542 struct netlink_ext_ack *extack);
1543 int (*ndo_fdb_dump)(struct sk_buff *skb,
1544 struct netlink_callback *cb,
1545 struct net_device *dev,
1546 struct net_device *filter_dev,
1547 int *idx);
1548 int (*ndo_fdb_get)(struct sk_buff *skb,
1549 struct nlattr *tb[],
1550 struct net_device *dev,
1551 const unsigned char *addr,
1552 u16 vid, u32 portid, u32 seq,
1553 struct netlink_ext_ack *extack);
1554 int (*ndo_bridge_setlink)(struct net_device *dev,
1555 struct nlmsghdr *nlh,
1556 u16 flags,
1557 struct netlink_ext_ack *extack);
1558 int (*ndo_bridge_getlink)(struct sk_buff *skb,
1559 u32 pid, u32 seq,
1560 struct net_device *dev,
1561 u32 filter_mask,
1562 int nlflags);
1563 int (*ndo_bridge_dellink)(struct net_device *dev,
1564 struct nlmsghdr *nlh,
1565 u16 flags);
1566 int (*ndo_change_carrier)(struct net_device *dev,
1567 bool new_carrier);
1568 int (*ndo_get_phys_port_id)(struct net_device *dev,
1569 struct netdev_phys_item_id *ppid);
1570 int (*ndo_get_port_parent_id)(struct net_device *dev,
1571 struct netdev_phys_item_id *ppid);
1572 int (*ndo_get_phys_port_name)(struct net_device *dev,
1573 char *name, size_t len);
1574 void* (*ndo_dfwd_add_station)(struct net_device *pdev,
1575 struct net_device *dev);
1576 void (*ndo_dfwd_del_station)(struct net_device *pdev,
1577 void *priv);
1578
1579 int (*ndo_set_tx_maxrate)(struct net_device *dev,
1580 int queue_index,
1581 u32 maxrate);
1582 int (*ndo_get_iflink)(const struct net_device *dev);
1583 int (*ndo_fill_metadata_dst)(struct net_device *dev,
1584 struct sk_buff *skb);
1585 void (*ndo_set_rx_headroom)(struct net_device *dev,
1586 int needed_headroom);
1587 int (*ndo_bpf)(struct net_device *dev,
1588 struct netdev_bpf *bpf);
1589 int (*ndo_xdp_xmit)(struct net_device *dev, int n,
1590 struct xdp_frame **xdp,
1591 u32 flags);
1592 struct net_device * (*ndo_xdp_get_xmit_slave)(struct net_device *dev,
1593 struct xdp_buff *xdp);
1594 int (*ndo_xsk_wakeup)(struct net_device *dev,
1595 u32 queue_id, u32 flags);
1596 struct devlink_port * (*ndo_get_devlink_port)(struct net_device *dev);
1597 int (*ndo_tunnel_ctl)(struct net_device *dev,
1598 struct ip_tunnel_parm *p, int cmd);
1599 struct net_device * (*ndo_get_peer_dev)(struct net_device *dev);
1600 int (*ndo_fill_forward_path)(struct net_device_path_ctx *ctx,
1601 struct net_device_path *path);
1602 ktime_t (*ndo_get_tstamp)(struct net_device *dev,
1603 const struct skb_shared_hwtstamps *hwtstamps,
1604 bool cycles);
1605 };
1606
1607
1608
1609
1610
1611
1612
1613
1614
1615
1616
1617
1618
1619
1620
1621
1622
1623
1624
1625
1626
1627
1628
1629
1630
1631
1632
1633
1634
1635
1636
1637
1638
1639
1640
1641
1642
1643
1644
1645
1646
1647
1648
1649
1650
1651
1652
1653
1654
1655
1656 enum netdev_priv_flags {
1657 IFF_802_1Q_VLAN = 1<<0,
1658 IFF_EBRIDGE = 1<<1,
1659 IFF_BONDING = 1<<2,
1660 IFF_ISATAP = 1<<3,
1661 IFF_WAN_HDLC = 1<<4,
1662 IFF_XMIT_DST_RELEASE = 1<<5,
1663 IFF_DONT_BRIDGE = 1<<6,
1664 IFF_DISABLE_NETPOLL = 1<<7,
1665 IFF_MACVLAN_PORT = 1<<8,
1666 IFF_BRIDGE_PORT = 1<<9,
1667 IFF_OVS_DATAPATH = 1<<10,
1668 IFF_TX_SKB_SHARING = 1<<11,
1669 IFF_UNICAST_FLT = 1<<12,
1670 IFF_TEAM_PORT = 1<<13,
1671 IFF_SUPP_NOFCS = 1<<14,
1672 IFF_LIVE_ADDR_CHANGE = 1<<15,
1673 IFF_MACVLAN = 1<<16,
1674 IFF_XMIT_DST_RELEASE_PERM = 1<<17,
1675 IFF_L3MDEV_MASTER = 1<<18,
1676 IFF_NO_QUEUE = 1<<19,
1677 IFF_OPENVSWITCH = 1<<20,
1678 IFF_L3MDEV_SLAVE = 1<<21,
1679 IFF_TEAM = 1<<22,
1680 IFF_RXFH_CONFIGURED = 1<<23,
1681 IFF_PHONY_HEADROOM = 1<<24,
1682 IFF_MACSEC = 1<<25,
1683 IFF_NO_RX_HANDLER = 1<<26,
1684 IFF_FAILOVER = 1<<27,
1685 IFF_FAILOVER_SLAVE = 1<<28,
1686 IFF_L3MDEV_RX_HANDLER = 1<<29,
1687 IFF_LIVE_RENAME_OK = 1<<30,
1688 IFF_TX_SKB_NO_LINEAR = BIT_ULL(31),
1689 IFF_CHANGE_PROTO_DOWN = BIT_ULL(32),
1690 };
1691
1692 #define IFF_802_1Q_VLAN IFF_802_1Q_VLAN
1693 #define IFF_EBRIDGE IFF_EBRIDGE
1694 #define IFF_BONDING IFF_BONDING
1695 #define IFF_ISATAP IFF_ISATAP
1696 #define IFF_WAN_HDLC IFF_WAN_HDLC
1697 #define IFF_XMIT_DST_RELEASE IFF_XMIT_DST_RELEASE
1698 #define IFF_DONT_BRIDGE IFF_DONT_BRIDGE
1699 #define IFF_DISABLE_NETPOLL IFF_DISABLE_NETPOLL
1700 #define IFF_MACVLAN_PORT IFF_MACVLAN_PORT
1701 #define IFF_BRIDGE_PORT IFF_BRIDGE_PORT
1702 #define IFF_OVS_DATAPATH IFF_OVS_DATAPATH
1703 #define IFF_TX_SKB_SHARING IFF_TX_SKB_SHARING
1704 #define IFF_UNICAST_FLT IFF_UNICAST_FLT
1705 #define IFF_TEAM_PORT IFF_TEAM_PORT
1706 #define IFF_SUPP_NOFCS IFF_SUPP_NOFCS
1707 #define IFF_LIVE_ADDR_CHANGE IFF_LIVE_ADDR_CHANGE
1708 #define IFF_MACVLAN IFF_MACVLAN
1709 #define IFF_XMIT_DST_RELEASE_PERM IFF_XMIT_DST_RELEASE_PERM
1710 #define IFF_L3MDEV_MASTER IFF_L3MDEV_MASTER
1711 #define IFF_NO_QUEUE IFF_NO_QUEUE
1712 #define IFF_OPENVSWITCH IFF_OPENVSWITCH
1713 #define IFF_L3MDEV_SLAVE IFF_L3MDEV_SLAVE
1714 #define IFF_TEAM IFF_TEAM
1715 #define IFF_RXFH_CONFIGURED IFF_RXFH_CONFIGURED
1716 #define IFF_PHONY_HEADROOM IFF_PHONY_HEADROOM
1717 #define IFF_MACSEC IFF_MACSEC
1718 #define IFF_NO_RX_HANDLER IFF_NO_RX_HANDLER
1719 #define IFF_FAILOVER IFF_FAILOVER
1720 #define IFF_FAILOVER_SLAVE IFF_FAILOVER_SLAVE
1721 #define IFF_L3MDEV_RX_HANDLER IFF_L3MDEV_RX_HANDLER
1722 #define IFF_LIVE_RENAME_OK IFF_LIVE_RENAME_OK
1723 #define IFF_TX_SKB_NO_LINEAR IFF_TX_SKB_NO_LINEAR
1724
1725
1726 enum netdev_ml_priv_type {
1727 ML_PRIV_NONE,
1728 ML_PRIV_CAN,
1729 };
1730
1731
1732
1733
1734
1735
1736
1737
1738
1739
1740
1741
1742
1743
1744
1745
1746
1747
1748
1749
1750
1751
1752
1753
1754
1755
1756
1757
1758
1759
1760
1761
1762
1763
1764
1765
1766
1767
1768
1769
1770
1771
1772
1773
1774
1775
1776
1777
1778
1779
1780
1781
1782
1783
1784
1785
1786
1787
1788
1789
1790
1791
1792
1793
1794
1795
1796
1797
1798
1799
1800
1801
1802
1803
1804
1805
1806
1807
1808
1809
1810
1811
1812
1813
1814
1815
1816
1817
1818
1819
1820
1821
1822
1823
1824
1825
1826
1827
1828
1829
1830
1831
1832
1833
1834
1835
1836
1837
1838
1839
1840
1841
1842
1843
1844
1845
1846
1847
1848
1849
1850
1851
1852
1853
1854
1855
1856
1857
1858
1859
1860
1861
1862
1863
1864
1865
1866
1867
1868
1869
1870
1871
1872
1873
1874
1875
1876
1877
1878
1879
1880
1881
1882
1883
1884
1885
1886
1887
1888
1889
1890
1891
1892
1893
1894
1895
1896
1897
1898
1899
1900
1901
1902
1903
1904
1905
1906
1907
1908
1909
1910
1911
1912
1913
1914
1915
1916
1917
1918
1919
1920
1921
1922
1923
1924
1925
1926
1927
1928
1929
1930
1931
1932
1933
1934
1935
1936
1937
1938
1939
1940
1941
1942
1943
1944
1945
1946
1947
1948
1949
1950
1951
1952
1953
1954
1955
1956
1957
1958
1959
1960
1961
1962
1963
1964
1965
1966
1967
1968
1969
1970
1971
1972
1973
1974
1975
1976
1977
1978
1979
1980
1981
1982
1983
1984
1985
1986
1987
1988
1989
1990
1991
1992
1993
1994
1995
1996
1997
1998
1999
2000 struct net_device {
2001 char name[IFNAMSIZ];
2002 struct netdev_name_node *name_node;
2003 struct dev_ifalias __rcu *ifalias;
2004
2005
2006
2007
2008 unsigned long mem_end;
2009 unsigned long mem_start;
2010 unsigned long base_addr;
2011
2012
2013
2014
2015
2016
2017
2018 unsigned long state;
2019
2020 struct list_head dev_list;
2021 struct list_head napi_list;
2022 struct list_head unreg_list;
2023 struct list_head close_list;
2024 struct list_head ptype_all;
2025 struct list_head ptype_specific;
2026
2027 struct {
2028 struct list_head upper;
2029 struct list_head lower;
2030 } adj_list;
2031
2032
2033 unsigned int flags;
2034 unsigned long long priv_flags;
2035 const struct net_device_ops *netdev_ops;
2036 int ifindex;
2037 unsigned short gflags;
2038 unsigned short hard_header_len;
2039
2040
2041
2042
2043
2044
2045 unsigned int mtu;
2046 unsigned short needed_headroom;
2047 unsigned short needed_tailroom;
2048
2049 netdev_features_t features;
2050 netdev_features_t hw_features;
2051 netdev_features_t wanted_features;
2052 netdev_features_t vlan_features;
2053 netdev_features_t hw_enc_features;
2054 netdev_features_t mpls_features;
2055 netdev_features_t gso_partial_features;
2056
2057 unsigned int min_mtu;
2058 unsigned int max_mtu;
2059 unsigned short type;
2060 unsigned char min_header_len;
2061 unsigned char name_assign_type;
2062
2063 int group;
2064
2065 struct net_device_stats stats;
2066
2067 struct net_device_core_stats __percpu *core_stats;
2068
2069
2070 atomic_t carrier_up_count;
2071 atomic_t carrier_down_count;
2072
2073 #ifdef CONFIG_WIRELESS_EXT
2074 const struct iw_handler_def *wireless_handlers;
2075 struct iw_public_data *wireless_data;
2076 #endif
2077 const struct ethtool_ops *ethtool_ops;
2078 #ifdef CONFIG_NET_L3_MASTER_DEV
2079 const struct l3mdev_ops *l3mdev_ops;
2080 #endif
2081 #if IS_ENABLED(CONFIG_IPV6)
2082 const struct ndisc_ops *ndisc_ops;
2083 #endif
2084
2085 #ifdef CONFIG_XFRM_OFFLOAD
2086 const struct xfrmdev_ops *xfrmdev_ops;
2087 #endif
2088
2089 #if IS_ENABLED(CONFIG_TLS_DEVICE)
2090 const struct tlsdev_ops *tlsdev_ops;
2091 #endif
2092
2093 const struct header_ops *header_ops;
2094
2095 unsigned char operstate;
2096 unsigned char link_mode;
2097
2098 unsigned char if_port;
2099 unsigned char dma;
2100
2101
2102 unsigned char perm_addr[MAX_ADDR_LEN];
2103 unsigned char addr_assign_type;
2104 unsigned char addr_len;
2105 unsigned char upper_level;
2106 unsigned char lower_level;
2107
2108 unsigned short neigh_priv_len;
2109 unsigned short dev_id;
2110 unsigned short dev_port;
2111 unsigned short padded;
2112
2113 spinlock_t addr_list_lock;
2114 int irq;
2115
2116 struct netdev_hw_addr_list uc;
2117 struct netdev_hw_addr_list mc;
2118 struct netdev_hw_addr_list dev_addrs;
2119
2120 #ifdef CONFIG_SYSFS
2121 struct kset *queues_kset;
2122 #endif
2123 #ifdef CONFIG_LOCKDEP
2124 struct list_head unlink_list;
2125 #endif
2126 unsigned int promiscuity;
2127 unsigned int allmulti;
2128 bool uc_promisc;
2129 #ifdef CONFIG_LOCKDEP
2130 unsigned char nested_level;
2131 #endif
2132
2133
2134
2135
2136 struct in_device __rcu *ip_ptr;
2137 struct inet6_dev __rcu *ip6_ptr;
2138 #if IS_ENABLED(CONFIG_VLAN_8021Q)
2139 struct vlan_info __rcu *vlan_info;
2140 #endif
2141 #if IS_ENABLED(CONFIG_NET_DSA)
2142 struct dsa_port *dsa_ptr;
2143 #endif
2144 #if IS_ENABLED(CONFIG_TIPC)
2145 struct tipc_bearer __rcu *tipc_ptr;
2146 #endif
2147 #if IS_ENABLED(CONFIG_ATALK)
2148 void *atalk_ptr;
2149 #endif
2150 #if IS_ENABLED(CONFIG_DECNET)
2151 struct dn_dev __rcu *dn_ptr;
2152 #endif
2153 #if IS_ENABLED(CONFIG_AX25)
2154 void *ax25_ptr;
2155 #endif
2156 #if IS_ENABLED(CONFIG_CFG80211)
2157 struct wireless_dev *ieee80211_ptr;
2158 #endif
2159 #if IS_ENABLED(CONFIG_IEEE802154) || IS_ENABLED(CONFIG_6LOWPAN)
2160 struct wpan_dev *ieee802154_ptr;
2161 #endif
2162 #if IS_ENABLED(CONFIG_MPLS_ROUTING)
2163 struct mpls_dev __rcu *mpls_ptr;
2164 #endif
2165 #if IS_ENABLED(CONFIG_MCTP)
2166 struct mctp_dev __rcu *mctp_ptr;
2167 #endif
2168
2169
2170
2171
2172
2173 const unsigned char *dev_addr;
2174
2175 struct netdev_rx_queue *_rx;
2176 unsigned int num_rx_queues;
2177 unsigned int real_num_rx_queues;
2178
2179 struct bpf_prog __rcu *xdp_prog;
2180 unsigned long gro_flush_timeout;
2181 int napi_defer_hard_irqs;
2182 #define GRO_LEGACY_MAX_SIZE 65536u
2183
2184
2185
2186 #define GRO_MAX_SIZE (8 * 65535u)
2187 unsigned int gro_max_size;
2188 rx_handler_func_t __rcu *rx_handler;
2189 void __rcu *rx_handler_data;
2190
2191 #ifdef CONFIG_NET_CLS_ACT
2192 struct mini_Qdisc __rcu *miniq_ingress;
2193 #endif
2194 struct netdev_queue __rcu *ingress_queue;
2195 #ifdef CONFIG_NETFILTER_INGRESS
2196 struct nf_hook_entries __rcu *nf_hooks_ingress;
2197 #endif
2198
2199 unsigned char broadcast[MAX_ADDR_LEN];
2200 #ifdef CONFIG_RFS_ACCEL
2201 struct cpu_rmap *rx_cpu_rmap;
2202 #endif
2203 struct hlist_node index_hlist;
2204
2205
2206
2207
2208 struct netdev_queue *_tx ____cacheline_aligned_in_smp;
2209 unsigned int num_tx_queues;
2210 unsigned int real_num_tx_queues;
2211 struct Qdisc __rcu *qdisc;
2212 unsigned int tx_queue_len;
2213 spinlock_t tx_global_lock;
2214
2215 struct xdp_dev_bulk_queue __percpu *xdp_bulkq;
2216
2217 #ifdef CONFIG_XPS
2218 struct xps_dev_maps __rcu *xps_maps[XPS_MAPS_MAX];
2219 #endif
2220 #ifdef CONFIG_NET_CLS_ACT
2221 struct mini_Qdisc __rcu *miniq_egress;
2222 #endif
2223 #ifdef CONFIG_NETFILTER_EGRESS
2224 struct nf_hook_entries __rcu *nf_hooks_egress;
2225 #endif
2226
2227 #ifdef CONFIG_NET_SCHED
2228 DECLARE_HASHTABLE (qdisc_hash, 4);
2229 #endif
2230
2231 struct timer_list watchdog_timer;
2232 int watchdog_timeo;
2233
2234 u32 proto_down_reason;
2235
2236 struct list_head todo_list;
2237
2238 #ifdef CONFIG_PCPU_DEV_REFCNT
2239 int __percpu *pcpu_refcnt;
2240 #else
2241 refcount_t dev_refcnt;
2242 #endif
2243 struct ref_tracker_dir refcnt_tracker;
2244
2245 struct list_head link_watch_list;
2246
2247 enum { NETREG_UNINITIALIZED=0,
2248 NETREG_REGISTERED,
2249 NETREG_UNREGISTERING,
2250 NETREG_UNREGISTERED,
2251 NETREG_RELEASED,
2252 NETREG_DUMMY,
2253 } reg_state:8;
2254
2255 bool dismantle;
2256
2257 enum {
2258 RTNL_LINK_INITIALIZED,
2259 RTNL_LINK_INITIALIZING,
2260 } rtnl_link_state:16;
2261
2262 bool needs_free_netdev;
2263 void (*priv_destructor)(struct net_device *dev);
2264
2265 #ifdef CONFIG_NETPOLL
2266 struct netpoll_info __rcu *npinfo;
2267 #endif
2268
2269 possible_net_t nd_net;
2270
2271
2272 void *ml_priv;
2273 enum netdev_ml_priv_type ml_priv_type;
2274
2275 union {
2276 struct pcpu_lstats __percpu *lstats;
2277 struct pcpu_sw_netstats __percpu *tstats;
2278 struct pcpu_dstats __percpu *dstats;
2279 };
2280
2281 #if IS_ENABLED(CONFIG_GARP)
2282 struct garp_port __rcu *garp_port;
2283 #endif
2284 #if IS_ENABLED(CONFIG_MRP)
2285 struct mrp_port __rcu *mrp_port;
2286 #endif
2287 #if IS_ENABLED(CONFIG_NET_DROP_MONITOR)
2288 struct dm_hw_stat_delta __rcu *dm_private;
2289 #endif
2290 struct device dev;
2291 const struct attribute_group *sysfs_groups[4];
2292 const struct attribute_group *sysfs_rx_queue_group;
2293
2294 const struct rtnl_link_ops *rtnl_link_ops;
2295
2296
2297 #define GSO_MAX_SEGS 65535u
2298 #define GSO_LEGACY_MAX_SIZE 65536u
2299
2300
2301
2302 #define GSO_MAX_SIZE (8 * GSO_MAX_SEGS)
2303
2304 unsigned int gso_max_size;
2305 #define TSO_LEGACY_MAX_SIZE 65536
2306 #define TSO_MAX_SIZE UINT_MAX
2307 unsigned int tso_max_size;
2308 u16 gso_max_segs;
2309 #define TSO_MAX_SEGS U16_MAX
2310 u16 tso_max_segs;
2311
2312 #ifdef CONFIG_DCB
2313 const struct dcbnl_rtnl_ops *dcbnl_ops;
2314 #endif
2315 s16 num_tc;
2316 struct netdev_tc_txq tc_to_txq[TC_MAX_QUEUE];
2317 u8 prio_tc_map[TC_BITMASK + 1];
2318
2319 #if IS_ENABLED(CONFIG_FCOE)
2320 unsigned int fcoe_ddp_xid;
2321 #endif
2322 #if IS_ENABLED(CONFIG_CGROUP_NET_PRIO)
2323 struct netprio_map __rcu *priomap;
2324 #endif
2325 struct phy_device *phydev;
2326 struct sfp_bus *sfp_bus;
2327 struct lock_class_key *qdisc_tx_busylock;
2328 bool proto_down;
2329 unsigned wol_enabled:1;
2330 unsigned threaded:1;
2331
2332 struct list_head net_notifier_list;
2333
2334 #if IS_ENABLED(CONFIG_MACSEC)
2335
2336 const struct macsec_ops *macsec_ops;
2337 #endif
2338 const struct udp_tunnel_nic_info *udp_tunnel_nic_info;
2339 struct udp_tunnel_nic *udp_tunnel_nic;
2340
2341
2342 struct bpf_xdp_entity xdp_state[__MAX_XDP_MODE];
2343
2344 u8 dev_addr_shadow[MAX_ADDR_LEN];
2345 netdevice_tracker linkwatch_dev_tracker;
2346 netdevice_tracker watchdog_dev_tracker;
2347 netdevice_tracker dev_registered_tracker;
2348 struct rtnl_hw_stats64 *offload_xstats_l3;
2349 };
2350 #define to_net_dev(d) container_of(d, struct net_device, dev)
2351
2352 static inline bool netif_elide_gro(const struct net_device *dev)
2353 {
2354 if (!(dev->features & NETIF_F_GRO) || dev->xdp_prog)
2355 return true;
2356 return false;
2357 }
2358
2359 #define NETDEV_ALIGN 32
2360
2361 static inline
2362 int netdev_get_prio_tc_map(const struct net_device *dev, u32 prio)
2363 {
2364 return dev->prio_tc_map[prio & TC_BITMASK];
2365 }
2366
2367 static inline
2368 int netdev_set_prio_tc_map(struct net_device *dev, u8 prio, u8 tc)
2369 {
2370 if (tc >= dev->num_tc)
2371 return -EINVAL;
2372
2373 dev->prio_tc_map[prio & TC_BITMASK] = tc & TC_BITMASK;
2374 return 0;
2375 }
2376
2377 int netdev_txq_to_tc(struct net_device *dev, unsigned int txq);
2378 void netdev_reset_tc(struct net_device *dev);
2379 int netdev_set_tc_queue(struct net_device *dev, u8 tc, u16 count, u16 offset);
2380 int netdev_set_num_tc(struct net_device *dev, u8 num_tc);
2381
2382 static inline
2383 int netdev_get_num_tc(struct net_device *dev)
2384 {
2385 return dev->num_tc;
2386 }
2387
2388 static inline void net_prefetch(void *p)
2389 {
2390 prefetch(p);
2391 #if L1_CACHE_BYTES < 128
2392 prefetch((u8 *)p + L1_CACHE_BYTES);
2393 #endif
2394 }
2395
2396 static inline void net_prefetchw(void *p)
2397 {
2398 prefetchw(p);
2399 #if L1_CACHE_BYTES < 128
2400 prefetchw((u8 *)p + L1_CACHE_BYTES);
2401 #endif
2402 }
2403
2404 void netdev_unbind_sb_channel(struct net_device *dev,
2405 struct net_device *sb_dev);
2406 int netdev_bind_sb_channel_queue(struct net_device *dev,
2407 struct net_device *sb_dev,
2408 u8 tc, u16 count, u16 offset);
2409 int netdev_set_sb_channel(struct net_device *dev, u16 channel);
2410 static inline int netdev_get_sb_channel(struct net_device *dev)
2411 {
2412 return max_t(int, -dev->num_tc, 0);
2413 }
2414
2415 static inline
2416 struct netdev_queue *netdev_get_tx_queue(const struct net_device *dev,
2417 unsigned int index)
2418 {
2419 return &dev->_tx[index];
2420 }
2421
2422 static inline struct netdev_queue *skb_get_tx_queue(const struct net_device *dev,
2423 const struct sk_buff *skb)
2424 {
2425 return netdev_get_tx_queue(dev, skb_get_queue_mapping(skb));
2426 }
2427
2428 static inline void netdev_for_each_tx_queue(struct net_device *dev,
2429 void (*f)(struct net_device *,
2430 struct netdev_queue *,
2431 void *),
2432 void *arg)
2433 {
2434 unsigned int i;
2435
2436 for (i = 0; i < dev->num_tx_queues; i++)
2437 f(dev, &dev->_tx[i], arg);
2438 }
2439
2440 #define netdev_lockdep_set_classes(dev) \
2441 { \
2442 static struct lock_class_key qdisc_tx_busylock_key; \
2443 static struct lock_class_key qdisc_xmit_lock_key; \
2444 static struct lock_class_key dev_addr_list_lock_key; \
2445 unsigned int i; \
2446 \
2447 (dev)->qdisc_tx_busylock = &qdisc_tx_busylock_key; \
2448 lockdep_set_class(&(dev)->addr_list_lock, \
2449 &dev_addr_list_lock_key); \
2450 for (i = 0; i < (dev)->num_tx_queues; i++) \
2451 lockdep_set_class(&(dev)->_tx[i]._xmit_lock, \
2452 &qdisc_xmit_lock_key); \
2453 }
2454
2455 u16 netdev_pick_tx(struct net_device *dev, struct sk_buff *skb,
2456 struct net_device *sb_dev);
2457 struct netdev_queue *netdev_core_pick_tx(struct net_device *dev,
2458 struct sk_buff *skb,
2459 struct net_device *sb_dev);
2460
2461
2462
2463
2464 static inline unsigned netdev_get_fwd_headroom(struct net_device *dev)
2465 {
2466 return dev->priv_flags & IFF_PHONY_HEADROOM ? 0 : dev->needed_headroom;
2467 }
2468
2469 static inline void netdev_set_rx_headroom(struct net_device *dev, int new_hr)
2470 {
2471 if (dev->netdev_ops->ndo_set_rx_headroom)
2472 dev->netdev_ops->ndo_set_rx_headroom(dev, new_hr);
2473 }
2474
2475
2476 static inline void netdev_reset_rx_headroom(struct net_device *dev)
2477 {
2478 netdev_set_rx_headroom(dev, -1);
2479 }
2480
2481 static inline void *netdev_get_ml_priv(struct net_device *dev,
2482 enum netdev_ml_priv_type type)
2483 {
2484 if (dev->ml_priv_type != type)
2485 return NULL;
2486
2487 return dev->ml_priv;
2488 }
2489
2490 static inline void netdev_set_ml_priv(struct net_device *dev,
2491 void *ml_priv,
2492 enum netdev_ml_priv_type type)
2493 {
2494 WARN(dev->ml_priv_type && dev->ml_priv_type != type,
2495 "Overwriting already set ml_priv_type (%u) with different ml_priv_type (%u)!\n",
2496 dev->ml_priv_type, type);
2497 WARN(!dev->ml_priv_type && dev->ml_priv,
2498 "Overwriting already set ml_priv and ml_priv_type is ML_PRIV_NONE!\n");
2499
2500 dev->ml_priv = ml_priv;
2501 dev->ml_priv_type = type;
2502 }
2503
2504
2505
2506
2507 static inline
2508 struct net *dev_net(const struct net_device *dev)
2509 {
2510 return read_pnet(&dev->nd_net);
2511 }
2512
2513 static inline
2514 void dev_net_set(struct net_device *dev, struct net *net)
2515 {
2516 write_pnet(&dev->nd_net, net);
2517 }
2518
2519
2520
2521
2522
2523
2524
2525 static inline void *netdev_priv(const struct net_device *dev)
2526 {
2527 return (char *)dev + ALIGN(sizeof(struct net_device), NETDEV_ALIGN);
2528 }
2529
2530
2531
2532
2533 #define SET_NETDEV_DEV(net, pdev) ((net)->dev.parent = (pdev))
2534
2535
2536
2537
2538
2539 #define SET_NETDEV_DEVTYPE(net, devtype) ((net)->dev.type = (devtype))
2540
2541
2542
2543
2544 #define NAPI_POLL_WEIGHT 64
2545
2546 void netif_napi_add_weight(struct net_device *dev, struct napi_struct *napi,
2547 int (*poll)(struct napi_struct *, int), int weight);
2548
2549
2550
2551
2552
2553
2554
2555
2556
2557
2558
2559 static inline void
2560 netif_napi_add(struct net_device *dev, struct napi_struct *napi,
2561 int (*poll)(struct napi_struct *, int), int weight)
2562 {
2563 netif_napi_add_weight(dev, napi, poll, weight);
2564 }
2565
2566 static inline void
2567 netif_napi_add_tx_weight(struct net_device *dev,
2568 struct napi_struct *napi,
2569 int (*poll)(struct napi_struct *, int),
2570 int weight)
2571 {
2572 set_bit(NAPI_STATE_NO_BUSY_POLL, &napi->state);
2573 netif_napi_add_weight(dev, napi, poll, weight);
2574 }
2575
2576 #define netif_tx_napi_add netif_napi_add_tx_weight
2577
2578
2579
2580
2581
2582
2583
2584
2585
2586
2587
2588 static inline void netif_napi_add_tx(struct net_device *dev,
2589 struct napi_struct *napi,
2590 int (*poll)(struct napi_struct *, int))
2591 {
2592 netif_napi_add_tx_weight(dev, napi, poll, NAPI_POLL_WEIGHT);
2593 }
2594
2595
2596
2597
2598
2599
2600
2601
2602
2603 void __netif_napi_del(struct napi_struct *napi);
2604
2605
2606
2607
2608
2609
2610
2611 static inline void netif_napi_del(struct napi_struct *napi)
2612 {
2613 __netif_napi_del(napi);
2614 synchronize_net();
2615 }
2616
2617 struct packet_type {
2618 __be16 type;
2619 bool ignore_outgoing;
2620 struct net_device *dev;
2621 netdevice_tracker dev_tracker;
2622 int (*func) (struct sk_buff *,
2623 struct net_device *,
2624 struct packet_type *,
2625 struct net_device *);
2626 void (*list_func) (struct list_head *,
2627 struct packet_type *,
2628 struct net_device *);
2629 bool (*id_match)(struct packet_type *ptype,
2630 struct sock *sk);
2631 struct net *af_packet_net;
2632 void *af_packet_priv;
2633 struct list_head list;
2634 };
2635
2636 struct offload_callbacks {
2637 struct sk_buff *(*gso_segment)(struct sk_buff *skb,
2638 netdev_features_t features);
2639 struct sk_buff *(*gro_receive)(struct list_head *head,
2640 struct sk_buff *skb);
2641 int (*gro_complete)(struct sk_buff *skb, int nhoff);
2642 };
2643
2644 struct packet_offload {
2645 __be16 type;
2646 u16 priority;
2647 struct offload_callbacks callbacks;
2648 struct list_head list;
2649 };
2650
2651
2652 struct pcpu_sw_netstats {
2653 u64_stats_t rx_packets;
2654 u64_stats_t rx_bytes;
2655 u64_stats_t tx_packets;
2656 u64_stats_t tx_bytes;
2657 struct u64_stats_sync syncp;
2658 } __aligned(4 * sizeof(u64));
2659
2660 struct pcpu_lstats {
2661 u64_stats_t packets;
2662 u64_stats_t bytes;
2663 struct u64_stats_sync syncp;
2664 } __aligned(2 * sizeof(u64));
2665
2666 void dev_lstats_read(struct net_device *dev, u64 *packets, u64 *bytes);
2667
2668 static inline void dev_sw_netstats_rx_add(struct net_device *dev, unsigned int len)
2669 {
2670 struct pcpu_sw_netstats *tstats = this_cpu_ptr(dev->tstats);
2671
2672 u64_stats_update_begin(&tstats->syncp);
2673 u64_stats_add(&tstats->rx_bytes, len);
2674 u64_stats_inc(&tstats->rx_packets);
2675 u64_stats_update_end(&tstats->syncp);
2676 }
2677
2678 static inline void dev_sw_netstats_tx_add(struct net_device *dev,
2679 unsigned int packets,
2680 unsigned int len)
2681 {
2682 struct pcpu_sw_netstats *tstats = this_cpu_ptr(dev->tstats);
2683
2684 u64_stats_update_begin(&tstats->syncp);
2685 u64_stats_add(&tstats->tx_bytes, len);
2686 u64_stats_add(&tstats->tx_packets, packets);
2687 u64_stats_update_end(&tstats->syncp);
2688 }
2689
2690 static inline void dev_lstats_add(struct net_device *dev, unsigned int len)
2691 {
2692 struct pcpu_lstats *lstats = this_cpu_ptr(dev->lstats);
2693
2694 u64_stats_update_begin(&lstats->syncp);
2695 u64_stats_add(&lstats->bytes, len);
2696 u64_stats_inc(&lstats->packets);
2697 u64_stats_update_end(&lstats->syncp);
2698 }
2699
2700 #define __netdev_alloc_pcpu_stats(type, gfp) \
2701 ({ \
2702 typeof(type) __percpu *pcpu_stats = alloc_percpu_gfp(type, gfp);\
2703 if (pcpu_stats) { \
2704 int __cpu; \
2705 for_each_possible_cpu(__cpu) { \
2706 typeof(type) *stat; \
2707 stat = per_cpu_ptr(pcpu_stats, __cpu); \
2708 u64_stats_init(&stat->syncp); \
2709 } \
2710 } \
2711 pcpu_stats; \
2712 })
2713
2714 #define netdev_alloc_pcpu_stats(type) \
2715 __netdev_alloc_pcpu_stats(type, GFP_KERNEL)
2716
2717 #define devm_netdev_alloc_pcpu_stats(dev, type) \
2718 ({ \
2719 typeof(type) __percpu *pcpu_stats = devm_alloc_percpu(dev, type);\
2720 if (pcpu_stats) { \
2721 int __cpu; \
2722 for_each_possible_cpu(__cpu) { \
2723 typeof(type) *stat; \
2724 stat = per_cpu_ptr(pcpu_stats, __cpu); \
2725 u64_stats_init(&stat->syncp); \
2726 } \
2727 } \
2728 pcpu_stats; \
2729 })
2730
2731 enum netdev_lag_tx_type {
2732 NETDEV_LAG_TX_TYPE_UNKNOWN,
2733 NETDEV_LAG_TX_TYPE_RANDOM,
2734 NETDEV_LAG_TX_TYPE_BROADCAST,
2735 NETDEV_LAG_TX_TYPE_ROUNDROBIN,
2736 NETDEV_LAG_TX_TYPE_ACTIVEBACKUP,
2737 NETDEV_LAG_TX_TYPE_HASH,
2738 };
2739
2740 enum netdev_lag_hash {
2741 NETDEV_LAG_HASH_NONE,
2742 NETDEV_LAG_HASH_L2,
2743 NETDEV_LAG_HASH_L34,
2744 NETDEV_LAG_HASH_L23,
2745 NETDEV_LAG_HASH_E23,
2746 NETDEV_LAG_HASH_E34,
2747 NETDEV_LAG_HASH_VLAN_SRCMAC,
2748 NETDEV_LAG_HASH_UNKNOWN,
2749 };
2750
2751 struct netdev_lag_upper_info {
2752 enum netdev_lag_tx_type tx_type;
2753 enum netdev_lag_hash hash_type;
2754 };
2755
2756 struct netdev_lag_lower_state_info {
2757 u8 link_up : 1,
2758 tx_enabled : 1;
2759 };
2760
2761 #include <linux/notifier.h>
2762
2763
2764
2765
2766
2767 enum netdev_cmd {
2768 NETDEV_UP = 1,
2769 NETDEV_DOWN,
2770 NETDEV_REBOOT,
2771
2772
2773
2774 NETDEV_CHANGE,
2775 NETDEV_REGISTER,
2776 NETDEV_UNREGISTER,
2777 NETDEV_CHANGEMTU,
2778 NETDEV_CHANGEADDR,
2779 NETDEV_PRE_CHANGEADDR,
2780 NETDEV_GOING_DOWN,
2781 NETDEV_CHANGENAME,
2782 NETDEV_FEAT_CHANGE,
2783 NETDEV_BONDING_FAILOVER,
2784 NETDEV_PRE_UP,
2785 NETDEV_PRE_TYPE_CHANGE,
2786 NETDEV_POST_TYPE_CHANGE,
2787 NETDEV_POST_INIT,
2788 NETDEV_RELEASE,
2789 NETDEV_NOTIFY_PEERS,
2790 NETDEV_JOIN,
2791 NETDEV_CHANGEUPPER,
2792 NETDEV_RESEND_IGMP,
2793 NETDEV_PRECHANGEMTU,
2794 NETDEV_CHANGEINFODATA,
2795 NETDEV_BONDING_INFO,
2796 NETDEV_PRECHANGEUPPER,
2797 NETDEV_CHANGELOWERSTATE,
2798 NETDEV_UDP_TUNNEL_PUSH_INFO,
2799 NETDEV_UDP_TUNNEL_DROP_INFO,
2800 NETDEV_CHANGE_TX_QUEUE_LEN,
2801 NETDEV_CVLAN_FILTER_PUSH_INFO,
2802 NETDEV_CVLAN_FILTER_DROP_INFO,
2803 NETDEV_SVLAN_FILTER_PUSH_INFO,
2804 NETDEV_SVLAN_FILTER_DROP_INFO,
2805 NETDEV_OFFLOAD_XSTATS_ENABLE,
2806 NETDEV_OFFLOAD_XSTATS_DISABLE,
2807 NETDEV_OFFLOAD_XSTATS_REPORT_USED,
2808 NETDEV_OFFLOAD_XSTATS_REPORT_DELTA,
2809 };
2810 const char *netdev_cmd_to_name(enum netdev_cmd cmd);
2811
2812 int register_netdevice_notifier(struct notifier_block *nb);
2813 int unregister_netdevice_notifier(struct notifier_block *nb);
2814 int register_netdevice_notifier_net(struct net *net, struct notifier_block *nb);
2815 int unregister_netdevice_notifier_net(struct net *net,
2816 struct notifier_block *nb);
2817 int register_netdevice_notifier_dev_net(struct net_device *dev,
2818 struct notifier_block *nb,
2819 struct netdev_net_notifier *nn);
2820 int unregister_netdevice_notifier_dev_net(struct net_device *dev,
2821 struct notifier_block *nb,
2822 struct netdev_net_notifier *nn);
2823
2824 struct netdev_notifier_info {
2825 struct net_device *dev;
2826 struct netlink_ext_ack *extack;
2827 };
2828
2829 struct netdev_notifier_info_ext {
2830 struct netdev_notifier_info info;
2831 union {
2832 u32 mtu;
2833 } ext;
2834 };
2835
2836 struct netdev_notifier_change_info {
2837 struct netdev_notifier_info info;
2838 unsigned int flags_changed;
2839 };
2840
2841 struct netdev_notifier_changeupper_info {
2842 struct netdev_notifier_info info;
2843 struct net_device *upper_dev;
2844 bool master;
2845 bool linking;
2846 void *upper_info;
2847 };
2848
2849 struct netdev_notifier_changelowerstate_info {
2850 struct netdev_notifier_info info;
2851 void *lower_state_info;
2852 };
2853
2854 struct netdev_notifier_pre_changeaddr_info {
2855 struct netdev_notifier_info info;
2856 const unsigned char *dev_addr;
2857 };
2858
2859 enum netdev_offload_xstats_type {
2860 NETDEV_OFFLOAD_XSTATS_TYPE_L3 = 1,
2861 };
2862
2863 struct netdev_notifier_offload_xstats_info {
2864 struct netdev_notifier_info info;
2865 enum netdev_offload_xstats_type type;
2866
2867 union {
2868
2869 struct netdev_notifier_offload_xstats_rd *report_delta;
2870
2871 struct netdev_notifier_offload_xstats_ru *report_used;
2872 };
2873 };
2874
2875 int netdev_offload_xstats_enable(struct net_device *dev,
2876 enum netdev_offload_xstats_type type,
2877 struct netlink_ext_ack *extack);
2878 int netdev_offload_xstats_disable(struct net_device *dev,
2879 enum netdev_offload_xstats_type type);
2880 bool netdev_offload_xstats_enabled(const struct net_device *dev,
2881 enum netdev_offload_xstats_type type);
2882 int netdev_offload_xstats_get(struct net_device *dev,
2883 enum netdev_offload_xstats_type type,
2884 struct rtnl_hw_stats64 *stats, bool *used,
2885 struct netlink_ext_ack *extack);
2886 void
2887 netdev_offload_xstats_report_delta(struct netdev_notifier_offload_xstats_rd *rd,
2888 const struct rtnl_hw_stats64 *stats);
2889 void
2890 netdev_offload_xstats_report_used(struct netdev_notifier_offload_xstats_ru *ru);
2891 void netdev_offload_xstats_push_delta(struct net_device *dev,
2892 enum netdev_offload_xstats_type type,
2893 const struct rtnl_hw_stats64 *stats);
2894
2895 static inline void netdev_notifier_info_init(struct netdev_notifier_info *info,
2896 struct net_device *dev)
2897 {
2898 info->dev = dev;
2899 info->extack = NULL;
2900 }
2901
2902 static inline struct net_device *
2903 netdev_notifier_info_to_dev(const struct netdev_notifier_info *info)
2904 {
2905 return info->dev;
2906 }
2907
2908 static inline struct netlink_ext_ack *
2909 netdev_notifier_info_to_extack(const struct netdev_notifier_info *info)
2910 {
2911 return info->extack;
2912 }
2913
2914 int call_netdevice_notifiers(unsigned long val, struct net_device *dev);
2915
2916
2917 extern rwlock_t dev_base_lock;
2918
2919 #define for_each_netdev(net, d) \
2920 list_for_each_entry(d, &(net)->dev_base_head, dev_list)
2921 #define for_each_netdev_reverse(net, d) \
2922 list_for_each_entry_reverse(d, &(net)->dev_base_head, dev_list)
2923 #define for_each_netdev_rcu(net, d) \
2924 list_for_each_entry_rcu(d, &(net)->dev_base_head, dev_list)
2925 #define for_each_netdev_safe(net, d, n) \
2926 list_for_each_entry_safe(d, n, &(net)->dev_base_head, dev_list)
2927 #define for_each_netdev_continue(net, d) \
2928 list_for_each_entry_continue(d, &(net)->dev_base_head, dev_list)
2929 #define for_each_netdev_continue_reverse(net, d) \
2930 list_for_each_entry_continue_reverse(d, &(net)->dev_base_head, \
2931 dev_list)
2932 #define for_each_netdev_continue_rcu(net, d) \
2933 list_for_each_entry_continue_rcu(d, &(net)->dev_base_head, dev_list)
2934 #define for_each_netdev_in_bond_rcu(bond, slave) \
2935 for_each_netdev_rcu(&init_net, slave) \
2936 if (netdev_master_upper_dev_get_rcu(slave) == (bond))
2937 #define net_device_entry(lh) list_entry(lh, struct net_device, dev_list)
2938
2939 static inline struct net_device *next_net_device(struct net_device *dev)
2940 {
2941 struct list_head *lh;
2942 struct net *net;
2943
2944 net = dev_net(dev);
2945 lh = dev->dev_list.next;
2946 return lh == &net->dev_base_head ? NULL : net_device_entry(lh);
2947 }
2948
2949 static inline struct net_device *next_net_device_rcu(struct net_device *dev)
2950 {
2951 struct list_head *lh;
2952 struct net *net;
2953
2954 net = dev_net(dev);
2955 lh = rcu_dereference(list_next_rcu(&dev->dev_list));
2956 return lh == &net->dev_base_head ? NULL : net_device_entry(lh);
2957 }
2958
2959 static inline struct net_device *first_net_device(struct net *net)
2960 {
2961 return list_empty(&net->dev_base_head) ? NULL :
2962 net_device_entry(net->dev_base_head.next);
2963 }
2964
2965 static inline struct net_device *first_net_device_rcu(struct net *net)
2966 {
2967 struct list_head *lh = rcu_dereference(list_next_rcu(&net->dev_base_head));
2968
2969 return lh == &net->dev_base_head ? NULL : net_device_entry(lh);
2970 }
2971
2972 int netdev_boot_setup_check(struct net_device *dev);
2973 struct net_device *dev_getbyhwaddr_rcu(struct net *net, unsigned short type,
2974 const char *hwaddr);
2975 struct net_device *dev_getfirstbyhwtype(struct net *net, unsigned short type);
2976 void dev_add_pack(struct packet_type *pt);
2977 void dev_remove_pack(struct packet_type *pt);
2978 void __dev_remove_pack(struct packet_type *pt);
2979 void dev_add_offload(struct packet_offload *po);
2980 void dev_remove_offload(struct packet_offload *po);
2981
2982 int dev_get_iflink(const struct net_device *dev);
2983 int dev_fill_metadata_dst(struct net_device *dev, struct sk_buff *skb);
2984 int dev_fill_forward_path(const struct net_device *dev, const u8 *daddr,
2985 struct net_device_path_stack *stack);
2986 struct net_device *__dev_get_by_flags(struct net *net, unsigned short flags,
2987 unsigned short mask);
2988 struct net_device *dev_get_by_name(struct net *net, const char *name);
2989 struct net_device *dev_get_by_name_rcu(struct net *net, const char *name);
2990 struct net_device *__dev_get_by_name(struct net *net, const char *name);
2991 bool netdev_name_in_use(struct net *net, const char *name);
2992 int dev_alloc_name(struct net_device *dev, const char *name);
2993 int dev_open(struct net_device *dev, struct netlink_ext_ack *extack);
2994 void dev_close(struct net_device *dev);
2995 void dev_close_many(struct list_head *head, bool unlink);
2996 void dev_disable_lro(struct net_device *dev);
2997 int dev_loopback_xmit(struct net *net, struct sock *sk, struct sk_buff *newskb);
2998 u16 dev_pick_tx_zero(struct net_device *dev, struct sk_buff *skb,
2999 struct net_device *sb_dev);
3000 u16 dev_pick_tx_cpu_id(struct net_device *dev, struct sk_buff *skb,
3001 struct net_device *sb_dev);
3002
3003 int __dev_queue_xmit(struct sk_buff *skb, struct net_device *sb_dev);
3004 int __dev_direct_xmit(struct sk_buff *skb, u16 queue_id);
3005
3006 static inline int dev_queue_xmit(struct sk_buff *skb)
3007 {
3008 return __dev_queue_xmit(skb, NULL);
3009 }
3010
3011 static inline int dev_queue_xmit_accel(struct sk_buff *skb,
3012 struct net_device *sb_dev)
3013 {
3014 return __dev_queue_xmit(skb, sb_dev);
3015 }
3016
3017 static inline int dev_direct_xmit(struct sk_buff *skb, u16 queue_id)
3018 {
3019 int ret;
3020
3021 ret = __dev_direct_xmit(skb, queue_id);
3022 if (!dev_xmit_complete(ret))
3023 kfree_skb(skb);
3024 return ret;
3025 }
3026
3027 int register_netdevice(struct net_device *dev);
3028 void unregister_netdevice_queue(struct net_device *dev, struct list_head *head);
3029 void unregister_netdevice_many(struct list_head *head);
3030 static inline void unregister_netdevice(struct net_device *dev)
3031 {
3032 unregister_netdevice_queue(dev, NULL);
3033 }
3034
3035 int netdev_refcnt_read(const struct net_device *dev);
3036 void free_netdev(struct net_device *dev);
3037 void netdev_freemem(struct net_device *dev);
3038 int init_dummy_netdev(struct net_device *dev);
3039
3040 struct net_device *netdev_get_xmit_slave(struct net_device *dev,
3041 struct sk_buff *skb,
3042 bool all_slaves);
3043 struct net_device *netdev_sk_get_lowest_dev(struct net_device *dev,
3044 struct sock *sk);
3045 struct net_device *dev_get_by_index(struct net *net, int ifindex);
3046 struct net_device *__dev_get_by_index(struct net *net, int ifindex);
3047 struct net_device *dev_get_by_index_rcu(struct net *net, int ifindex);
3048 struct net_device *dev_get_by_napi_id(unsigned int napi_id);
3049 int dev_restart(struct net_device *dev);
3050
3051
3052 static inline int dev_hard_header(struct sk_buff *skb, struct net_device *dev,
3053 unsigned short type,
3054 const void *daddr, const void *saddr,
3055 unsigned int len)
3056 {
3057 if (!dev->header_ops || !dev->header_ops->create)
3058 return 0;
3059
3060 return dev->header_ops->create(skb, dev, type, daddr, saddr, len);
3061 }
3062
3063 static inline int dev_parse_header(const struct sk_buff *skb,
3064 unsigned char *haddr)
3065 {
3066 const struct net_device *dev = skb->dev;
3067
3068 if (!dev->header_ops || !dev->header_ops->parse)
3069 return 0;
3070 return dev->header_ops->parse(skb, haddr);
3071 }
3072
3073 static inline __be16 dev_parse_header_protocol(const struct sk_buff *skb)
3074 {
3075 const struct net_device *dev = skb->dev;
3076
3077 if (!dev->header_ops || !dev->header_ops->parse_protocol)
3078 return 0;
3079 return dev->header_ops->parse_protocol(skb);
3080 }
3081
3082
3083 static inline bool dev_validate_header(const struct net_device *dev,
3084 char *ll_header, int len)
3085 {
3086 if (likely(len >= dev->hard_header_len))
3087 return true;
3088 if (len < dev->min_header_len)
3089 return false;
3090
3091 if (capable(CAP_SYS_RAWIO)) {
3092 memset(ll_header + len, 0, dev->hard_header_len - len);
3093 return true;
3094 }
3095
3096 if (dev->header_ops && dev->header_ops->validate)
3097 return dev->header_ops->validate(ll_header, len);
3098
3099 return false;
3100 }
3101
3102 static inline bool dev_has_header(const struct net_device *dev)
3103 {
3104 return dev->header_ops && dev->header_ops->create;
3105 }
3106
3107
3108
3109
3110 struct softnet_data {
3111 struct list_head poll_list;
3112 struct sk_buff_head process_queue;
3113
3114
3115 unsigned int processed;
3116 unsigned int time_squeeze;
3117 unsigned int received_rps;
3118 #ifdef CONFIG_RPS
3119 struct softnet_data *rps_ipi_list;
3120 #endif
3121 #ifdef CONFIG_NET_FLOW_LIMIT
3122 struct sd_flow_limit __rcu *flow_limit;
3123 #endif
3124 struct Qdisc *output_queue;
3125 struct Qdisc **output_queue_tailp;
3126 struct sk_buff *completion_queue;
3127 #ifdef CONFIG_XFRM_OFFLOAD
3128 struct sk_buff_head xfrm_backlog;
3129 #endif
3130
3131 struct {
3132 u16 recursion;
3133 u8 more;
3134 #ifdef CONFIG_NET_EGRESS
3135 u8 skip_txqueue;
3136 #endif
3137 } xmit;
3138 #ifdef CONFIG_RPS
3139
3140
3141
3142 unsigned int input_queue_head ____cacheline_aligned_in_smp;
3143
3144
3145 call_single_data_t csd ____cacheline_aligned_in_smp;
3146 struct softnet_data *rps_ipi_next;
3147 unsigned int cpu;
3148 unsigned int input_queue_tail;
3149 #endif
3150 unsigned int dropped;
3151 struct sk_buff_head input_pkt_queue;
3152 struct napi_struct backlog;
3153
3154
3155 spinlock_t defer_lock ____cacheline_aligned_in_smp;
3156 int defer_count;
3157 int defer_ipi_scheduled;
3158 struct sk_buff *defer_list;
3159 call_single_data_t defer_csd;
3160 };
3161
3162 static inline void input_queue_head_incr(struct softnet_data *sd)
3163 {
3164 #ifdef CONFIG_RPS
3165 sd->input_queue_head++;
3166 #endif
3167 }
3168
3169 static inline void input_queue_tail_incr_save(struct softnet_data *sd,
3170 unsigned int *qtail)
3171 {
3172 #ifdef CONFIG_RPS
3173 *qtail = ++sd->input_queue_tail;
3174 #endif
3175 }
3176
3177 DECLARE_PER_CPU_ALIGNED(struct softnet_data, softnet_data);
3178
3179 static inline int dev_recursion_level(void)
3180 {
3181 return this_cpu_read(softnet_data.xmit.recursion);
3182 }
3183
3184 #define XMIT_RECURSION_LIMIT 8
3185 static inline bool dev_xmit_recursion(void)
3186 {
3187 return unlikely(__this_cpu_read(softnet_data.xmit.recursion) >
3188 XMIT_RECURSION_LIMIT);
3189 }
3190
3191 static inline void dev_xmit_recursion_inc(void)
3192 {
3193 __this_cpu_inc(softnet_data.xmit.recursion);
3194 }
3195
3196 static inline void dev_xmit_recursion_dec(void)
3197 {
3198 __this_cpu_dec(softnet_data.xmit.recursion);
3199 }
3200
3201 void __netif_schedule(struct Qdisc *q);
3202 void netif_schedule_queue(struct netdev_queue *txq);
3203
3204 static inline void netif_tx_schedule_all(struct net_device *dev)
3205 {
3206 unsigned int i;
3207
3208 for (i = 0; i < dev->num_tx_queues; i++)
3209 netif_schedule_queue(netdev_get_tx_queue(dev, i));
3210 }
3211
3212 static __always_inline void netif_tx_start_queue(struct netdev_queue *dev_queue)
3213 {
3214 clear_bit(__QUEUE_STATE_DRV_XOFF, &dev_queue->state);
3215 }
3216
3217
3218
3219
3220
3221
3222
3223 static inline void netif_start_queue(struct net_device *dev)
3224 {
3225 netif_tx_start_queue(netdev_get_tx_queue(dev, 0));
3226 }
3227
3228 static inline void netif_tx_start_all_queues(struct net_device *dev)
3229 {
3230 unsigned int i;
3231
3232 for (i = 0; i < dev->num_tx_queues; i++) {
3233 struct netdev_queue *txq = netdev_get_tx_queue(dev, i);
3234 netif_tx_start_queue(txq);
3235 }
3236 }
3237
3238 void netif_tx_wake_queue(struct netdev_queue *dev_queue);
3239
3240
3241
3242
3243
3244
3245
3246
3247 static inline void netif_wake_queue(struct net_device *dev)
3248 {
3249 netif_tx_wake_queue(netdev_get_tx_queue(dev, 0));
3250 }
3251
3252 static inline void netif_tx_wake_all_queues(struct net_device *dev)
3253 {
3254 unsigned int i;
3255
3256 for (i = 0; i < dev->num_tx_queues; i++) {
3257 struct netdev_queue *txq = netdev_get_tx_queue(dev, i);
3258 netif_tx_wake_queue(txq);
3259 }
3260 }
3261
3262 static __always_inline void netif_tx_stop_queue(struct netdev_queue *dev_queue)
3263 {
3264 set_bit(__QUEUE_STATE_DRV_XOFF, &dev_queue->state);
3265 }
3266
3267
3268
3269
3270
3271
3272
3273
3274 static inline void netif_stop_queue(struct net_device *dev)
3275 {
3276 netif_tx_stop_queue(netdev_get_tx_queue(dev, 0));
3277 }
3278
3279 void netif_tx_stop_all_queues(struct net_device *dev);
3280
3281 static inline bool netif_tx_queue_stopped(const struct netdev_queue *dev_queue)
3282 {
3283 return test_bit(__QUEUE_STATE_DRV_XOFF, &dev_queue->state);
3284 }
3285
3286
3287
3288
3289
3290
3291
3292 static inline bool netif_queue_stopped(const struct net_device *dev)
3293 {
3294 return netif_tx_queue_stopped(netdev_get_tx_queue(dev, 0));
3295 }
3296
3297 static inline bool netif_xmit_stopped(const struct netdev_queue *dev_queue)
3298 {
3299 return dev_queue->state & QUEUE_STATE_ANY_XOFF;
3300 }
3301
3302 static inline bool
3303 netif_xmit_frozen_or_stopped(const struct netdev_queue *dev_queue)
3304 {
3305 return dev_queue->state & QUEUE_STATE_ANY_XOFF_OR_FROZEN;
3306 }
3307
3308 static inline bool
3309 netif_xmit_frozen_or_drv_stopped(const struct netdev_queue *dev_queue)
3310 {
3311 return dev_queue->state & QUEUE_STATE_DRV_XOFF_OR_FROZEN;
3312 }
3313
3314
3315
3316
3317
3318
3319
3320
3321
3322
3323
3324 static inline void netdev_queue_set_dql_min_limit(struct netdev_queue *dev_queue,
3325 unsigned int min_limit)
3326 {
3327 #ifdef CONFIG_BQL
3328 dev_queue->dql.min_limit = min_limit;
3329 #endif
3330 }
3331
3332
3333
3334
3335
3336
3337
3338
3339 static inline void netdev_txq_bql_enqueue_prefetchw(struct netdev_queue *dev_queue)
3340 {
3341 #ifdef CONFIG_BQL
3342 prefetchw(&dev_queue->dql.num_queued);
3343 #endif
3344 }
3345
3346
3347
3348
3349
3350
3351
3352
3353 static inline void netdev_txq_bql_complete_prefetchw(struct netdev_queue *dev_queue)
3354 {
3355 #ifdef CONFIG_BQL
3356 prefetchw(&dev_queue->dql.limit);
3357 #endif
3358 }
3359
3360 static inline void netdev_tx_sent_queue(struct netdev_queue *dev_queue,
3361 unsigned int bytes)
3362 {
3363 #ifdef CONFIG_BQL
3364 dql_queued(&dev_queue->dql, bytes);
3365
3366 if (likely(dql_avail(&dev_queue->dql) >= 0))
3367 return;
3368
3369 set_bit(__QUEUE_STATE_STACK_XOFF, &dev_queue->state);
3370
3371
3372
3373
3374
3375
3376 smp_mb();
3377
3378
3379 if (unlikely(dql_avail(&dev_queue->dql) >= 0))
3380 clear_bit(__QUEUE_STATE_STACK_XOFF, &dev_queue->state);
3381 #endif
3382 }
3383
3384
3385
3386
3387
3388
3389
3390 static inline bool __netdev_tx_sent_queue(struct netdev_queue *dev_queue,
3391 unsigned int bytes,
3392 bool xmit_more)
3393 {
3394 if (xmit_more) {
3395 #ifdef CONFIG_BQL
3396 dql_queued(&dev_queue->dql, bytes);
3397 #endif
3398 return netif_tx_queue_stopped(dev_queue);
3399 }
3400 netdev_tx_sent_queue(dev_queue, bytes);
3401 return true;
3402 }
3403
3404
3405
3406
3407
3408
3409
3410
3411
3412
3413 static inline void netdev_sent_queue(struct net_device *dev, unsigned int bytes)
3414 {
3415 netdev_tx_sent_queue(netdev_get_tx_queue(dev, 0), bytes);
3416 }
3417
3418 static inline bool __netdev_sent_queue(struct net_device *dev,
3419 unsigned int bytes,
3420 bool xmit_more)
3421 {
3422 return __netdev_tx_sent_queue(netdev_get_tx_queue(dev, 0), bytes,
3423 xmit_more);
3424 }
3425
3426 static inline void netdev_tx_completed_queue(struct netdev_queue *dev_queue,
3427 unsigned int pkts, unsigned int bytes)
3428 {
3429 #ifdef CONFIG_BQL
3430 if (unlikely(!bytes))
3431 return;
3432
3433 dql_completed(&dev_queue->dql, bytes);
3434
3435
3436
3437
3438
3439
3440 smp_mb();
3441
3442 if (unlikely(dql_avail(&dev_queue->dql) < 0))
3443 return;
3444
3445 if (test_and_clear_bit(__QUEUE_STATE_STACK_XOFF, &dev_queue->state))
3446 netif_schedule_queue(dev_queue);
3447 #endif
3448 }
3449
3450
3451
3452
3453
3454
3455
3456
3457
3458
3459
3460 static inline void netdev_completed_queue(struct net_device *dev,
3461 unsigned int pkts, unsigned int bytes)
3462 {
3463 netdev_tx_completed_queue(netdev_get_tx_queue(dev, 0), pkts, bytes);
3464 }
3465
3466 static inline void netdev_tx_reset_queue(struct netdev_queue *q)
3467 {
3468 #ifdef CONFIG_BQL
3469 clear_bit(__QUEUE_STATE_STACK_XOFF, &q->state);
3470 dql_reset(&q->dql);
3471 #endif
3472 }
3473
3474
3475
3476
3477
3478
3479
3480
3481 static inline void netdev_reset_queue(struct net_device *dev_queue)
3482 {
3483 netdev_tx_reset_queue(netdev_get_tx_queue(dev_queue, 0));
3484 }
3485
3486
3487
3488
3489
3490
3491
3492
3493
3494 static inline u16 netdev_cap_txqueue(struct net_device *dev, u16 queue_index)
3495 {
3496 if (unlikely(queue_index >= dev->real_num_tx_queues)) {
3497 net_warn_ratelimited("%s selects TX queue %d, but real number of TX queues is %d\n",
3498 dev->name, queue_index,
3499 dev->real_num_tx_queues);
3500 return 0;
3501 }
3502
3503 return queue_index;
3504 }
3505
3506
3507
3508
3509
3510
3511
3512 static inline bool netif_running(const struct net_device *dev)
3513 {
3514 return test_bit(__LINK_STATE_START, &dev->state);
3515 }
3516
3517
3518
3519
3520
3521
3522
3523
3524
3525
3526
3527
3528
3529
3530
3531 static inline void netif_start_subqueue(struct net_device *dev, u16 queue_index)
3532 {
3533 struct netdev_queue *txq = netdev_get_tx_queue(dev, queue_index);
3534
3535 netif_tx_start_queue(txq);
3536 }
3537
3538
3539
3540
3541
3542
3543
3544
3545 static inline void netif_stop_subqueue(struct net_device *dev, u16 queue_index)
3546 {
3547 struct netdev_queue *txq = netdev_get_tx_queue(dev, queue_index);
3548 netif_tx_stop_queue(txq);
3549 }
3550
3551
3552
3553
3554
3555
3556
3557
3558 static inline bool __netif_subqueue_stopped(const struct net_device *dev,
3559 u16 queue_index)
3560 {
3561 struct netdev_queue *txq = netdev_get_tx_queue(dev, queue_index);
3562
3563 return netif_tx_queue_stopped(txq);
3564 }
3565
3566
3567
3568
3569
3570
3571
3572
3573 static inline bool netif_subqueue_stopped(const struct net_device *dev,
3574 struct sk_buff *skb)
3575 {
3576 return __netif_subqueue_stopped(dev, skb_get_queue_mapping(skb));
3577 }
3578
3579
3580
3581
3582
3583
3584
3585
3586 static inline void netif_wake_subqueue(struct net_device *dev, u16 queue_index)
3587 {
3588 struct netdev_queue *txq = netdev_get_tx_queue(dev, queue_index);
3589
3590 netif_tx_wake_queue(txq);
3591 }
3592
3593 #ifdef CONFIG_XPS
3594 int netif_set_xps_queue(struct net_device *dev, const struct cpumask *mask,
3595 u16 index);
3596 int __netif_set_xps_queue(struct net_device *dev, const unsigned long *mask,
3597 u16 index, enum xps_map_type type);
3598
3599
3600
3601
3602
3603
3604
3605
3606
3607 static inline bool netif_attr_test_mask(unsigned long j,
3608 const unsigned long *mask,
3609 unsigned int nr_bits)
3610 {
3611 cpu_max_bits_warn(j, nr_bits);
3612 return test_bit(j, mask);
3613 }
3614
3615
3616
3617
3618
3619
3620
3621
3622
3623 static inline bool netif_attr_test_online(unsigned long j,
3624 const unsigned long *online_mask,
3625 unsigned int nr_bits)
3626 {
3627 cpu_max_bits_warn(j, nr_bits);
3628
3629 if (online_mask)
3630 return test_bit(j, online_mask);
3631
3632 return (j < nr_bits);
3633 }
3634
3635
3636
3637
3638
3639
3640
3641
3642
3643 static inline unsigned int netif_attrmask_next(int n, const unsigned long *srcp,
3644 unsigned int nr_bits)
3645 {
3646
3647 if (n != -1)
3648 cpu_max_bits_warn(n, nr_bits);
3649
3650 if (srcp)
3651 return find_next_bit(srcp, nr_bits, n + 1);
3652
3653 return n + 1;
3654 }
3655
3656
3657
3658
3659
3660
3661
3662
3663
3664
3665 static inline int netif_attrmask_next_and(int n, const unsigned long *src1p,
3666 const unsigned long *src2p,
3667 unsigned int nr_bits)
3668 {
3669
3670 if (n != -1)
3671 cpu_max_bits_warn(n, nr_bits);
3672
3673 if (src1p && src2p)
3674 return find_next_and_bit(src1p, src2p, nr_bits, n + 1);
3675 else if (src1p)
3676 return find_next_bit(src1p, nr_bits, n + 1);
3677 else if (src2p)
3678 return find_next_bit(src2p, nr_bits, n + 1);
3679
3680 return n + 1;
3681 }
3682 #else
3683 static inline int netif_set_xps_queue(struct net_device *dev,
3684 const struct cpumask *mask,
3685 u16 index)
3686 {
3687 return 0;
3688 }
3689
3690 static inline int __netif_set_xps_queue(struct net_device *dev,
3691 const unsigned long *mask,
3692 u16 index, enum xps_map_type type)
3693 {
3694 return 0;
3695 }
3696 #endif
3697
3698
3699
3700
3701
3702
3703
3704 static inline bool netif_is_multiqueue(const struct net_device *dev)
3705 {
3706 return dev->num_tx_queues > 1;
3707 }
3708
3709 int netif_set_real_num_tx_queues(struct net_device *dev, unsigned int txq);
3710
3711 #ifdef CONFIG_SYSFS
3712 int netif_set_real_num_rx_queues(struct net_device *dev, unsigned int rxq);
3713 #else
3714 static inline int netif_set_real_num_rx_queues(struct net_device *dev,
3715 unsigned int rxqs)
3716 {
3717 dev->real_num_rx_queues = rxqs;
3718 return 0;
3719 }
3720 #endif
3721 int netif_set_real_num_queues(struct net_device *dev,
3722 unsigned int txq, unsigned int rxq);
3723
3724 static inline struct netdev_rx_queue *
3725 __netif_get_rx_queue(struct net_device *dev, unsigned int rxq)
3726 {
3727 return dev->_rx + rxq;
3728 }
3729
3730 #ifdef CONFIG_SYSFS
3731 static inline unsigned int get_netdev_rx_queue_index(
3732 struct netdev_rx_queue *queue)
3733 {
3734 struct net_device *dev = queue->dev;
3735 int index = queue - dev->_rx;
3736
3737 BUG_ON(index >= dev->num_rx_queues);
3738 return index;
3739 }
3740 #endif
3741
3742 int netif_get_num_default_rss_queues(void);
3743
3744 enum skb_free_reason {
3745 SKB_REASON_CONSUMED,
3746 SKB_REASON_DROPPED,
3747 };
3748
3749 void __dev_kfree_skb_irq(struct sk_buff *skb, enum skb_free_reason reason);
3750 void __dev_kfree_skb_any(struct sk_buff *skb, enum skb_free_reason reason);
3751
3752
3753
3754
3755
3756
3757
3758
3759
3760
3761
3762
3763
3764
3765
3766
3767
3768
3769
3770
3771 static inline void dev_kfree_skb_irq(struct sk_buff *skb)
3772 {
3773 __dev_kfree_skb_irq(skb, SKB_REASON_DROPPED);
3774 }
3775
3776 static inline void dev_consume_skb_irq(struct sk_buff *skb)
3777 {
3778 __dev_kfree_skb_irq(skb, SKB_REASON_CONSUMED);
3779 }
3780
3781 static inline void dev_kfree_skb_any(struct sk_buff *skb)
3782 {
3783 __dev_kfree_skb_any(skb, SKB_REASON_DROPPED);
3784 }
3785
3786 static inline void dev_consume_skb_any(struct sk_buff *skb)
3787 {
3788 __dev_kfree_skb_any(skb, SKB_REASON_CONSUMED);
3789 }
3790
3791 u32 bpf_prog_run_generic_xdp(struct sk_buff *skb, struct xdp_buff *xdp,
3792 struct bpf_prog *xdp_prog);
3793 void generic_xdp_tx(struct sk_buff *skb, struct bpf_prog *xdp_prog);
3794 int do_xdp_generic(struct bpf_prog *xdp_prog, struct sk_buff *skb);
3795 int netif_rx(struct sk_buff *skb);
3796 int __netif_rx(struct sk_buff *skb);
3797
3798 int netif_receive_skb(struct sk_buff *skb);
3799 int netif_receive_skb_core(struct sk_buff *skb);
3800 void netif_receive_skb_list_internal(struct list_head *head);
3801 void netif_receive_skb_list(struct list_head *head);
3802 gro_result_t napi_gro_receive(struct napi_struct *napi, struct sk_buff *skb);
3803 void napi_gro_flush(struct napi_struct *napi, bool flush_old);
3804 struct sk_buff *napi_get_frags(struct napi_struct *napi);
3805 gro_result_t napi_gro_frags(struct napi_struct *napi);
3806 struct packet_offload *gro_find_receive_by_type(__be16 type);
3807 struct packet_offload *gro_find_complete_by_type(__be16 type);
3808
3809 static inline void napi_free_frags(struct napi_struct *napi)
3810 {
3811 kfree_skb(napi->skb);
3812 napi->skb = NULL;
3813 }
3814
3815 bool netdev_is_rx_handler_busy(struct net_device *dev);
3816 int netdev_rx_handler_register(struct net_device *dev,
3817 rx_handler_func_t *rx_handler,
3818 void *rx_handler_data);
3819 void netdev_rx_handler_unregister(struct net_device *dev);
3820
3821 bool dev_valid_name(const char *name);
3822 static inline bool is_socket_ioctl_cmd(unsigned int cmd)
3823 {
3824 return _IOC_TYPE(cmd) == SOCK_IOC_TYPE;
3825 }
3826 int get_user_ifreq(struct ifreq *ifr, void __user **ifrdata, void __user *arg);
3827 int put_user_ifreq(struct ifreq *ifr, void __user *arg);
3828 int dev_ioctl(struct net *net, unsigned int cmd, struct ifreq *ifr,
3829 void __user *data, bool *need_copyout);
3830 int dev_ifconf(struct net *net, struct ifconf __user *ifc);
3831 int dev_ethtool(struct net *net, struct ifreq *ifr, void __user *userdata);
3832 unsigned int dev_get_flags(const struct net_device *);
3833 int __dev_change_flags(struct net_device *dev, unsigned int flags,
3834 struct netlink_ext_ack *extack);
3835 int dev_change_flags(struct net_device *dev, unsigned int flags,
3836 struct netlink_ext_ack *extack);
3837 void __dev_notify_flags(struct net_device *, unsigned int old_flags,
3838 unsigned int gchanges);
3839 int dev_set_alias(struct net_device *, const char *, size_t);
3840 int dev_get_alias(const struct net_device *, char *, size_t);
3841 int __dev_change_net_namespace(struct net_device *dev, struct net *net,
3842 const char *pat, int new_ifindex);
3843 static inline
3844 int dev_change_net_namespace(struct net_device *dev, struct net *net,
3845 const char *pat)
3846 {
3847 return __dev_change_net_namespace(dev, net, pat, 0);
3848 }
3849 int __dev_set_mtu(struct net_device *, int);
3850 int dev_set_mtu(struct net_device *, int);
3851 int dev_pre_changeaddr_notify(struct net_device *dev, const char *addr,
3852 struct netlink_ext_ack *extack);
3853 int dev_set_mac_address(struct net_device *dev, struct sockaddr *sa,
3854 struct netlink_ext_ack *extack);
3855 int dev_set_mac_address_user(struct net_device *dev, struct sockaddr *sa,
3856 struct netlink_ext_ack *extack);
3857 int dev_get_mac_address(struct sockaddr *sa, struct net *net, char *dev_name);
3858 int dev_get_port_parent_id(struct net_device *dev,
3859 struct netdev_phys_item_id *ppid, bool recurse);
3860 bool netdev_port_same_parent_id(struct net_device *a, struct net_device *b);
3861 struct sk_buff *validate_xmit_skb_list(struct sk_buff *skb, struct net_device *dev, bool *again);
3862 struct sk_buff *dev_hard_start_xmit(struct sk_buff *skb, struct net_device *dev,
3863 struct netdev_queue *txq, int *ret);
3864
3865 int bpf_xdp_link_attach(const union bpf_attr *attr, struct bpf_prog *prog);
3866 u8 dev_xdp_prog_count(struct net_device *dev);
3867 u32 dev_xdp_prog_id(struct net_device *dev, enum bpf_xdp_mode mode);
3868
3869 int __dev_forward_skb(struct net_device *dev, struct sk_buff *skb);
3870 int dev_forward_skb(struct net_device *dev, struct sk_buff *skb);
3871 int dev_forward_skb_nomtu(struct net_device *dev, struct sk_buff *skb);
3872 bool is_skb_forwardable(const struct net_device *dev,
3873 const struct sk_buff *skb);
3874
3875 static __always_inline bool __is_skb_forwardable(const struct net_device *dev,
3876 const struct sk_buff *skb,
3877 const bool check_mtu)
3878 {
3879 const u32 vlan_hdr_len = 4;
3880 unsigned int len;
3881
3882 if (!(dev->flags & IFF_UP))
3883 return false;
3884
3885 if (!check_mtu)
3886 return true;
3887
3888 len = dev->mtu + dev->hard_header_len + vlan_hdr_len;
3889 if (skb->len <= len)
3890 return true;
3891
3892
3893
3894
3895 if (skb_is_gso(skb))
3896 return true;
3897
3898 return false;
3899 }
3900
3901 struct net_device_core_stats __percpu *netdev_core_stats_alloc(struct net_device *dev);
3902
3903 static inline struct net_device_core_stats __percpu *dev_core_stats(struct net_device *dev)
3904 {
3905
3906 struct net_device_core_stats __percpu *p = READ_ONCE(dev->core_stats);
3907
3908 if (likely(p))
3909 return p;
3910
3911 return netdev_core_stats_alloc(dev);
3912 }
3913
3914 #define DEV_CORE_STATS_INC(FIELD) \
3915 static inline void dev_core_stats_##FIELD##_inc(struct net_device *dev) \
3916 { \
3917 struct net_device_core_stats __percpu *p; \
3918 \
3919 p = dev_core_stats(dev); \
3920 if (p) \
3921 this_cpu_inc(p->FIELD); \
3922 }
3923 DEV_CORE_STATS_INC(rx_dropped)
3924 DEV_CORE_STATS_INC(tx_dropped)
3925 DEV_CORE_STATS_INC(rx_nohandler)
3926 DEV_CORE_STATS_INC(rx_otherhost_dropped)
3927
3928 static __always_inline int ____dev_forward_skb(struct net_device *dev,
3929 struct sk_buff *skb,
3930 const bool check_mtu)
3931 {
3932 if (skb_orphan_frags(skb, GFP_ATOMIC) ||
3933 unlikely(!__is_skb_forwardable(dev, skb, check_mtu))) {
3934 dev_core_stats_rx_dropped_inc(dev);
3935 kfree_skb(skb);
3936 return NET_RX_DROP;
3937 }
3938
3939 skb_scrub_packet(skb, !net_eq(dev_net(dev), dev_net(skb->dev)));
3940 skb->priority = 0;
3941 return 0;
3942 }
3943
3944 bool dev_nit_active(struct net_device *dev);
3945 void dev_queue_xmit_nit(struct sk_buff *skb, struct net_device *dev);
3946
3947 static inline void __dev_put(struct net_device *dev)
3948 {
3949 if (dev) {
3950 #ifdef CONFIG_PCPU_DEV_REFCNT
3951 this_cpu_dec(*dev->pcpu_refcnt);
3952 #else
3953 refcount_dec(&dev->dev_refcnt);
3954 #endif
3955 }
3956 }
3957
3958 static inline void __dev_hold(struct net_device *dev)
3959 {
3960 if (dev) {
3961 #ifdef CONFIG_PCPU_DEV_REFCNT
3962 this_cpu_inc(*dev->pcpu_refcnt);
3963 #else
3964 refcount_inc(&dev->dev_refcnt);
3965 #endif
3966 }
3967 }
3968
3969 static inline void __netdev_tracker_alloc(struct net_device *dev,
3970 netdevice_tracker *tracker,
3971 gfp_t gfp)
3972 {
3973 #ifdef CONFIG_NET_DEV_REFCNT_TRACKER
3974 ref_tracker_alloc(&dev->refcnt_tracker, tracker, gfp);
3975 #endif
3976 }
3977
3978
3979
3980
3981 static inline void netdev_tracker_alloc(struct net_device *dev,
3982 netdevice_tracker *tracker, gfp_t gfp)
3983 {
3984 #ifdef CONFIG_NET_DEV_REFCNT_TRACKER
3985 refcount_dec(&dev->refcnt_tracker.no_tracker);
3986 __netdev_tracker_alloc(dev, tracker, gfp);
3987 #endif
3988 }
3989
3990 static inline void netdev_tracker_free(struct net_device *dev,
3991 netdevice_tracker *tracker)
3992 {
3993 #ifdef CONFIG_NET_DEV_REFCNT_TRACKER
3994 ref_tracker_free(&dev->refcnt_tracker, tracker);
3995 #endif
3996 }
3997
3998 static inline void netdev_hold(struct net_device *dev,
3999 netdevice_tracker *tracker, gfp_t gfp)
4000 {
4001 if (dev) {
4002 __dev_hold(dev);
4003 __netdev_tracker_alloc(dev, tracker, gfp);
4004 }
4005 }
4006
4007 static inline void netdev_put(struct net_device *dev,
4008 netdevice_tracker *tracker)
4009 {
4010 if (dev) {
4011 netdev_tracker_free(dev, tracker);
4012 __dev_put(dev);
4013 }
4014 }
4015
4016
4017
4018
4019
4020
4021
4022
4023 static inline void dev_hold(struct net_device *dev)
4024 {
4025 netdev_hold(dev, NULL, GFP_ATOMIC);
4026 }
4027
4028
4029
4030
4031
4032
4033
4034
4035 static inline void dev_put(struct net_device *dev)
4036 {
4037 netdev_put(dev, NULL);
4038 }
4039
4040 static inline void netdev_ref_replace(struct net_device *odev,
4041 struct net_device *ndev,
4042 netdevice_tracker *tracker,
4043 gfp_t gfp)
4044 {
4045 if (odev)
4046 netdev_tracker_free(odev, tracker);
4047
4048 __dev_hold(ndev);
4049 __dev_put(odev);
4050
4051 if (ndev)
4052 __netdev_tracker_alloc(ndev, tracker, gfp);
4053 }
4054
4055
4056
4057
4058
4059
4060
4061
4062
4063 void linkwatch_fire_event(struct net_device *dev);
4064
4065
4066
4067
4068
4069
4070
4071 static inline bool netif_carrier_ok(const struct net_device *dev)
4072 {
4073 return !test_bit(__LINK_STATE_NOCARRIER, &dev->state);
4074 }
4075
4076 unsigned long dev_trans_start(struct net_device *dev);
4077
4078 void __netdev_watchdog_up(struct net_device *dev);
4079
4080 void netif_carrier_on(struct net_device *dev);
4081 void netif_carrier_off(struct net_device *dev);
4082 void netif_carrier_event(struct net_device *dev);
4083
4084
4085
4086
4087
4088
4089
4090
4091
4092
4093
4094
4095
4096 static inline void netif_dormant_on(struct net_device *dev)
4097 {
4098 if (!test_and_set_bit(__LINK_STATE_DORMANT, &dev->state))
4099 linkwatch_fire_event(dev);
4100 }
4101
4102
4103
4104
4105
4106
4107
4108 static inline void netif_dormant_off(struct net_device *dev)
4109 {
4110 if (test_and_clear_bit(__LINK_STATE_DORMANT, &dev->state))
4111 linkwatch_fire_event(dev);
4112 }
4113
4114
4115
4116
4117
4118
4119
4120 static inline bool netif_dormant(const struct net_device *dev)
4121 {
4122 return test_bit(__LINK_STATE_DORMANT, &dev->state);
4123 }
4124
4125
4126
4127
4128
4129
4130
4131
4132
4133
4134
4135
4136 static inline void netif_testing_on(struct net_device *dev)
4137 {
4138 if (!test_and_set_bit(__LINK_STATE_TESTING, &dev->state))
4139 linkwatch_fire_event(dev);
4140 }
4141
4142
4143
4144
4145
4146
4147
4148 static inline void netif_testing_off(struct net_device *dev)
4149 {
4150 if (test_and_clear_bit(__LINK_STATE_TESTING, &dev->state))
4151 linkwatch_fire_event(dev);
4152 }
4153
4154
4155
4156
4157
4158
4159
4160 static inline bool netif_testing(const struct net_device *dev)
4161 {
4162 return test_bit(__LINK_STATE_TESTING, &dev->state);
4163 }
4164
4165
4166
4167
4168
4169
4170
4171
4172 static inline bool netif_oper_up(const struct net_device *dev)
4173 {
4174 return (dev->operstate == IF_OPER_UP ||
4175 dev->operstate == IF_OPER_UNKNOWN );
4176 }
4177
4178
4179
4180
4181
4182
4183
4184 static inline bool netif_device_present(const struct net_device *dev)
4185 {
4186 return test_bit(__LINK_STATE_PRESENT, &dev->state);
4187 }
4188
4189 void netif_device_detach(struct net_device *dev);
4190
4191 void netif_device_attach(struct net_device *dev);
4192
4193
4194
4195
4196
4197 enum {
4198 NETIF_MSG_DRV_BIT,
4199 NETIF_MSG_PROBE_BIT,
4200 NETIF_MSG_LINK_BIT,
4201 NETIF_MSG_TIMER_BIT,
4202 NETIF_MSG_IFDOWN_BIT,
4203 NETIF_MSG_IFUP_BIT,
4204 NETIF_MSG_RX_ERR_BIT,
4205 NETIF_MSG_TX_ERR_BIT,
4206 NETIF_MSG_TX_QUEUED_BIT,
4207 NETIF_MSG_INTR_BIT,
4208 NETIF_MSG_TX_DONE_BIT,
4209 NETIF_MSG_RX_STATUS_BIT,
4210 NETIF_MSG_PKTDATA_BIT,
4211 NETIF_MSG_HW_BIT,
4212 NETIF_MSG_WOL_BIT,
4213
4214
4215
4216
4217 NETIF_MSG_CLASS_COUNT,
4218 };
4219
4220 static_assert(NETIF_MSG_CLASS_COUNT <= 32);
4221
4222 #define __NETIF_MSG_BIT(bit) ((u32)1 << (bit))
4223 #define __NETIF_MSG(name) __NETIF_MSG_BIT(NETIF_MSG_ ## name ## _BIT)
4224
4225 #define NETIF_MSG_DRV __NETIF_MSG(DRV)
4226 #define NETIF_MSG_PROBE __NETIF_MSG(PROBE)
4227 #define NETIF_MSG_LINK __NETIF_MSG(LINK)
4228 #define NETIF_MSG_TIMER __NETIF_MSG(TIMER)
4229 #define NETIF_MSG_IFDOWN __NETIF_MSG(IFDOWN)
4230 #define NETIF_MSG_IFUP __NETIF_MSG(IFUP)
4231 #define NETIF_MSG_RX_ERR __NETIF_MSG(RX_ERR)
4232 #define NETIF_MSG_TX_ERR __NETIF_MSG(TX_ERR)
4233 #define NETIF_MSG_TX_QUEUED __NETIF_MSG(TX_QUEUED)
4234 #define NETIF_MSG_INTR __NETIF_MSG(INTR)
4235 #define NETIF_MSG_TX_DONE __NETIF_MSG(TX_DONE)
4236 #define NETIF_MSG_RX_STATUS __NETIF_MSG(RX_STATUS)
4237 #define NETIF_MSG_PKTDATA __NETIF_MSG(PKTDATA)
4238 #define NETIF_MSG_HW __NETIF_MSG(HW)
4239 #define NETIF_MSG_WOL __NETIF_MSG(WOL)
4240
4241 #define netif_msg_drv(p) ((p)->msg_enable & NETIF_MSG_DRV)
4242 #define netif_msg_probe(p) ((p)->msg_enable & NETIF_MSG_PROBE)
4243 #define netif_msg_link(p) ((p)->msg_enable & NETIF_MSG_LINK)
4244 #define netif_msg_timer(p) ((p)->msg_enable & NETIF_MSG_TIMER)
4245 #define netif_msg_ifdown(p) ((p)->msg_enable & NETIF_MSG_IFDOWN)
4246 #define netif_msg_ifup(p) ((p)->msg_enable & NETIF_MSG_IFUP)
4247 #define netif_msg_rx_err(p) ((p)->msg_enable & NETIF_MSG_RX_ERR)
4248 #define netif_msg_tx_err(p) ((p)->msg_enable & NETIF_MSG_TX_ERR)
4249 #define netif_msg_tx_queued(p) ((p)->msg_enable & NETIF_MSG_TX_QUEUED)
4250 #define netif_msg_intr(p) ((p)->msg_enable & NETIF_MSG_INTR)
4251 #define netif_msg_tx_done(p) ((p)->msg_enable & NETIF_MSG_TX_DONE)
4252 #define netif_msg_rx_status(p) ((p)->msg_enable & NETIF_MSG_RX_STATUS)
4253 #define netif_msg_pktdata(p) ((p)->msg_enable & NETIF_MSG_PKTDATA)
4254 #define netif_msg_hw(p) ((p)->msg_enable & NETIF_MSG_HW)
4255 #define netif_msg_wol(p) ((p)->msg_enable & NETIF_MSG_WOL)
4256
4257 static inline u32 netif_msg_init(int debug_value, int default_msg_enable_bits)
4258 {
4259
4260 if (debug_value < 0 || debug_value >= (sizeof(u32) * 8))
4261 return default_msg_enable_bits;
4262 if (debug_value == 0)
4263 return 0;
4264
4265 return (1U << debug_value) - 1;
4266 }
4267
4268 static inline void __netif_tx_lock(struct netdev_queue *txq, int cpu)
4269 {
4270 spin_lock(&txq->_xmit_lock);
4271
4272 WRITE_ONCE(txq->xmit_lock_owner, cpu);
4273 }
4274
4275 static inline bool __netif_tx_acquire(struct netdev_queue *txq)
4276 {
4277 __acquire(&txq->_xmit_lock);
4278 return true;
4279 }
4280
4281 static inline void __netif_tx_release(struct netdev_queue *txq)
4282 {
4283 __release(&txq->_xmit_lock);
4284 }
4285
4286 static inline void __netif_tx_lock_bh(struct netdev_queue *txq)
4287 {
4288 spin_lock_bh(&txq->_xmit_lock);
4289
4290 WRITE_ONCE(txq->xmit_lock_owner, smp_processor_id());
4291 }
4292
4293 static inline bool __netif_tx_trylock(struct netdev_queue *txq)
4294 {
4295 bool ok = spin_trylock(&txq->_xmit_lock);
4296
4297 if (likely(ok)) {
4298
4299 WRITE_ONCE(txq->xmit_lock_owner, smp_processor_id());
4300 }
4301 return ok;
4302 }
4303
4304 static inline void __netif_tx_unlock(struct netdev_queue *txq)
4305 {
4306
4307 WRITE_ONCE(txq->xmit_lock_owner, -1);
4308 spin_unlock(&txq->_xmit_lock);
4309 }
4310
4311 static inline void __netif_tx_unlock_bh(struct netdev_queue *txq)
4312 {
4313
4314 WRITE_ONCE(txq->xmit_lock_owner, -1);
4315 spin_unlock_bh(&txq->_xmit_lock);
4316 }
4317
4318
4319
4320
4321 static inline void txq_trans_update(struct netdev_queue *txq)
4322 {
4323 if (txq->xmit_lock_owner != -1)
4324 WRITE_ONCE(txq->trans_start, jiffies);
4325 }
4326
4327 static inline void txq_trans_cond_update(struct netdev_queue *txq)
4328 {
4329 unsigned long now = jiffies;
4330
4331 if (READ_ONCE(txq->trans_start) != now)
4332 WRITE_ONCE(txq->trans_start, now);
4333 }
4334
4335
4336 static inline void netif_trans_update(struct net_device *dev)
4337 {
4338 struct netdev_queue *txq = netdev_get_tx_queue(dev, 0);
4339
4340 txq_trans_cond_update(txq);
4341 }
4342
4343
4344
4345
4346
4347
4348
4349 void netif_tx_lock(struct net_device *dev);
4350
4351 static inline void netif_tx_lock_bh(struct net_device *dev)
4352 {
4353 local_bh_disable();
4354 netif_tx_lock(dev);
4355 }
4356
4357 void netif_tx_unlock(struct net_device *dev);
4358
4359 static inline void netif_tx_unlock_bh(struct net_device *dev)
4360 {
4361 netif_tx_unlock(dev);
4362 local_bh_enable();
4363 }
4364
4365 #define HARD_TX_LOCK(dev, txq, cpu) { \
4366 if ((dev->features & NETIF_F_LLTX) == 0) { \
4367 __netif_tx_lock(txq, cpu); \
4368 } else { \
4369 __netif_tx_acquire(txq); \
4370 } \
4371 }
4372
4373 #define HARD_TX_TRYLOCK(dev, txq) \
4374 (((dev->features & NETIF_F_LLTX) == 0) ? \
4375 __netif_tx_trylock(txq) : \
4376 __netif_tx_acquire(txq))
4377
4378 #define HARD_TX_UNLOCK(dev, txq) { \
4379 if ((dev->features & NETIF_F_LLTX) == 0) { \
4380 __netif_tx_unlock(txq); \
4381 } else { \
4382 __netif_tx_release(txq); \
4383 } \
4384 }
4385
4386 static inline void netif_tx_disable(struct net_device *dev)
4387 {
4388 unsigned int i;
4389 int cpu;
4390
4391 local_bh_disable();
4392 cpu = smp_processor_id();
4393 spin_lock(&dev->tx_global_lock);
4394 for (i = 0; i < dev->num_tx_queues; i++) {
4395 struct netdev_queue *txq = netdev_get_tx_queue(dev, i);
4396
4397 __netif_tx_lock(txq, cpu);
4398 netif_tx_stop_queue(txq);
4399 __netif_tx_unlock(txq);
4400 }
4401 spin_unlock(&dev->tx_global_lock);
4402 local_bh_enable();
4403 }
4404
4405 static inline void netif_addr_lock(struct net_device *dev)
4406 {
4407 unsigned char nest_level = 0;
4408
4409 #ifdef CONFIG_LOCKDEP
4410 nest_level = dev->nested_level;
4411 #endif
4412 spin_lock_nested(&dev->addr_list_lock, nest_level);
4413 }
4414
4415 static inline void netif_addr_lock_bh(struct net_device *dev)
4416 {
4417 unsigned char nest_level = 0;
4418
4419 #ifdef CONFIG_LOCKDEP
4420 nest_level = dev->nested_level;
4421 #endif
4422 local_bh_disable();
4423 spin_lock_nested(&dev->addr_list_lock, nest_level);
4424 }
4425
4426 static inline void netif_addr_unlock(struct net_device *dev)
4427 {
4428 spin_unlock(&dev->addr_list_lock);
4429 }
4430
4431 static inline void netif_addr_unlock_bh(struct net_device *dev)
4432 {
4433 spin_unlock_bh(&dev->addr_list_lock);
4434 }
4435
4436
4437
4438
4439
4440 #define for_each_dev_addr(dev, ha) \
4441 list_for_each_entry_rcu(ha, &dev->dev_addrs.list, list)
4442
4443
4444
4445 void ether_setup(struct net_device *dev);
4446
4447
4448 struct net_device *alloc_netdev_mqs(int sizeof_priv, const char *name,
4449 unsigned char name_assign_type,
4450 void (*setup)(struct net_device *),
4451 unsigned int txqs, unsigned int rxqs);
4452 #define alloc_netdev(sizeof_priv, name, name_assign_type, setup) \
4453 alloc_netdev_mqs(sizeof_priv, name, name_assign_type, setup, 1, 1)
4454
4455 #define alloc_netdev_mq(sizeof_priv, name, name_assign_type, setup, count) \
4456 alloc_netdev_mqs(sizeof_priv, name, name_assign_type, setup, count, \
4457 count)
4458
4459 int register_netdev(struct net_device *dev);
4460 void unregister_netdev(struct net_device *dev);
4461
4462 int devm_register_netdev(struct device *dev, struct net_device *ndev);
4463
4464
4465 int __hw_addr_sync(struct netdev_hw_addr_list *to_list,
4466 struct netdev_hw_addr_list *from_list, int addr_len);
4467 void __hw_addr_unsync(struct netdev_hw_addr_list *to_list,
4468 struct netdev_hw_addr_list *from_list, int addr_len);
4469 int __hw_addr_sync_dev(struct netdev_hw_addr_list *list,
4470 struct net_device *dev,
4471 int (*sync)(struct net_device *, const unsigned char *),
4472 int (*unsync)(struct net_device *,
4473 const unsigned char *));
4474 int __hw_addr_ref_sync_dev(struct netdev_hw_addr_list *list,
4475 struct net_device *dev,
4476 int (*sync)(struct net_device *,
4477 const unsigned char *, int),
4478 int (*unsync)(struct net_device *,
4479 const unsigned char *, int));
4480 void __hw_addr_ref_unsync_dev(struct netdev_hw_addr_list *list,
4481 struct net_device *dev,
4482 int (*unsync)(struct net_device *,
4483 const unsigned char *, int));
4484 void __hw_addr_unsync_dev(struct netdev_hw_addr_list *list,
4485 struct net_device *dev,
4486 int (*unsync)(struct net_device *,
4487 const unsigned char *));
4488 void __hw_addr_init(struct netdev_hw_addr_list *list);
4489
4490
4491 void dev_addr_mod(struct net_device *dev, unsigned int offset,
4492 const void *addr, size_t len);
4493
4494 static inline void
4495 __dev_addr_set(struct net_device *dev, const void *addr, size_t len)
4496 {
4497 dev_addr_mod(dev, 0, addr, len);
4498 }
4499
4500 static inline void dev_addr_set(struct net_device *dev, const u8 *addr)
4501 {
4502 __dev_addr_set(dev, addr, dev->addr_len);
4503 }
4504
4505 int dev_addr_add(struct net_device *dev, const unsigned char *addr,
4506 unsigned char addr_type);
4507 int dev_addr_del(struct net_device *dev, const unsigned char *addr,
4508 unsigned char addr_type);
4509
4510
4511 int dev_uc_add(struct net_device *dev, const unsigned char *addr);
4512 int dev_uc_add_excl(struct net_device *dev, const unsigned char *addr);
4513 int dev_uc_del(struct net_device *dev, const unsigned char *addr);
4514 int dev_uc_sync(struct net_device *to, struct net_device *from);
4515 int dev_uc_sync_multiple(struct net_device *to, struct net_device *from);
4516 void dev_uc_unsync(struct net_device *to, struct net_device *from);
4517 void dev_uc_flush(struct net_device *dev);
4518 void dev_uc_init(struct net_device *dev);
4519
4520
4521
4522
4523
4524
4525
4526
4527
4528
4529 static inline int __dev_uc_sync(struct net_device *dev,
4530 int (*sync)(struct net_device *,
4531 const unsigned char *),
4532 int (*unsync)(struct net_device *,
4533 const unsigned char *))
4534 {
4535 return __hw_addr_sync_dev(&dev->uc, dev, sync, unsync);
4536 }
4537
4538
4539
4540
4541
4542
4543
4544
4545 static inline void __dev_uc_unsync(struct net_device *dev,
4546 int (*unsync)(struct net_device *,
4547 const unsigned char *))
4548 {
4549 __hw_addr_unsync_dev(&dev->uc, dev, unsync);
4550 }
4551
4552
4553 int dev_mc_add(struct net_device *dev, const unsigned char *addr);
4554 int dev_mc_add_global(struct net_device *dev, const unsigned char *addr);
4555 int dev_mc_add_excl(struct net_device *dev, const unsigned char *addr);
4556 int dev_mc_del(struct net_device *dev, const unsigned char *addr);
4557 int dev_mc_del_global(struct net_device *dev, const unsigned char *addr);
4558 int dev_mc_sync(struct net_device *to, struct net_device *from);
4559 int dev_mc_sync_multiple(struct net_device *to, struct net_device *from);
4560 void dev_mc_unsync(struct net_device *to, struct net_device *from);
4561 void dev_mc_flush(struct net_device *dev);
4562 void dev_mc_init(struct net_device *dev);
4563
4564
4565
4566
4567
4568
4569
4570
4571
4572
4573 static inline int __dev_mc_sync(struct net_device *dev,
4574 int (*sync)(struct net_device *,
4575 const unsigned char *),
4576 int (*unsync)(struct net_device *,
4577 const unsigned char *))
4578 {
4579 return __hw_addr_sync_dev(&dev->mc, dev, sync, unsync);
4580 }
4581
4582
4583
4584
4585
4586
4587
4588
4589 static inline void __dev_mc_unsync(struct net_device *dev,
4590 int (*unsync)(struct net_device *,
4591 const unsigned char *))
4592 {
4593 __hw_addr_unsync_dev(&dev->mc, dev, unsync);
4594 }
4595
4596
4597 void dev_set_rx_mode(struct net_device *dev);
4598 int dev_set_promiscuity(struct net_device *dev, int inc);
4599 int dev_set_allmulti(struct net_device *dev, int inc);
4600 void netdev_state_change(struct net_device *dev);
4601 void __netdev_notify_peers(struct net_device *dev);
4602 void netdev_notify_peers(struct net_device *dev);
4603 void netdev_features_change(struct net_device *dev);
4604
4605 void dev_load(struct net *net, const char *name);
4606 struct rtnl_link_stats64 *dev_get_stats(struct net_device *dev,
4607 struct rtnl_link_stats64 *storage);
4608 void netdev_stats_to_stats64(struct rtnl_link_stats64 *stats64,
4609 const struct net_device_stats *netdev_stats);
4610 void dev_fetch_sw_netstats(struct rtnl_link_stats64 *s,
4611 const struct pcpu_sw_netstats __percpu *netstats);
4612 void dev_get_tstats64(struct net_device *dev, struct rtnl_link_stats64 *s);
4613
4614 extern int netdev_max_backlog;
4615 extern int dev_rx_weight;
4616 extern int dev_tx_weight;
4617 extern int gro_normal_batch;
4618
4619 enum {
4620 NESTED_SYNC_IMM_BIT,
4621 NESTED_SYNC_TODO_BIT,
4622 };
4623
4624 #define __NESTED_SYNC_BIT(bit) ((u32)1 << (bit))
4625 #define __NESTED_SYNC(name) __NESTED_SYNC_BIT(NESTED_SYNC_ ## name ## _BIT)
4626
4627 #define NESTED_SYNC_IMM __NESTED_SYNC(IMM)
4628 #define NESTED_SYNC_TODO __NESTED_SYNC(TODO)
4629
4630 struct netdev_nested_priv {
4631 unsigned char flags;
4632 void *data;
4633 };
4634
4635 bool netdev_has_upper_dev(struct net_device *dev, struct net_device *upper_dev);
4636 struct net_device *netdev_upper_get_next_dev_rcu(struct net_device *dev,
4637 struct list_head **iter);
4638
4639
4640 #define netdev_for_each_upper_dev_rcu(dev, updev, iter) \
4641 for (iter = &(dev)->adj_list.upper, \
4642 updev = netdev_upper_get_next_dev_rcu(dev, &(iter)); \
4643 updev; \
4644 updev = netdev_upper_get_next_dev_rcu(dev, &(iter)))
4645
4646 int netdev_walk_all_upper_dev_rcu(struct net_device *dev,
4647 int (*fn)(struct net_device *upper_dev,
4648 struct netdev_nested_priv *priv),
4649 struct netdev_nested_priv *priv);
4650
4651 bool netdev_has_upper_dev_all_rcu(struct net_device *dev,
4652 struct net_device *upper_dev);
4653
4654 bool netdev_has_any_upper_dev(struct net_device *dev);
4655
4656 void *netdev_lower_get_next_private(struct net_device *dev,
4657 struct list_head **iter);
4658 void *netdev_lower_get_next_private_rcu(struct net_device *dev,
4659 struct list_head **iter);
4660
4661 #define netdev_for_each_lower_private(dev, priv, iter) \
4662 for (iter = (dev)->adj_list.lower.next, \
4663 priv = netdev_lower_get_next_private(dev, &(iter)); \
4664 priv; \
4665 priv = netdev_lower_get_next_private(dev, &(iter)))
4666
4667 #define netdev_for_each_lower_private_rcu(dev, priv, iter) \
4668 for (iter = &(dev)->adj_list.lower, \
4669 priv = netdev_lower_get_next_private_rcu(dev, &(iter)); \
4670 priv; \
4671 priv = netdev_lower_get_next_private_rcu(dev, &(iter)))
4672
4673 void *netdev_lower_get_next(struct net_device *dev,
4674 struct list_head **iter);
4675
4676 #define netdev_for_each_lower_dev(dev, ldev, iter) \
4677 for (iter = (dev)->adj_list.lower.next, \
4678 ldev = netdev_lower_get_next(dev, &(iter)); \
4679 ldev; \
4680 ldev = netdev_lower_get_next(dev, &(iter)))
4681
4682 struct net_device *netdev_next_lower_dev_rcu(struct net_device *dev,
4683 struct list_head **iter);
4684 int netdev_walk_all_lower_dev(struct net_device *dev,
4685 int (*fn)(struct net_device *lower_dev,
4686 struct netdev_nested_priv *priv),
4687 struct netdev_nested_priv *priv);
4688 int netdev_walk_all_lower_dev_rcu(struct net_device *dev,
4689 int (*fn)(struct net_device *lower_dev,
4690 struct netdev_nested_priv *priv),
4691 struct netdev_nested_priv *priv);
4692
4693 void *netdev_adjacent_get_private(struct list_head *adj_list);
4694 void *netdev_lower_get_first_private_rcu(struct net_device *dev);
4695 struct net_device *netdev_master_upper_dev_get(struct net_device *dev);
4696 struct net_device *netdev_master_upper_dev_get_rcu(struct net_device *dev);
4697 int netdev_upper_dev_link(struct net_device *dev, struct net_device *upper_dev,
4698 struct netlink_ext_ack *extack);
4699 int netdev_master_upper_dev_link(struct net_device *dev,
4700 struct net_device *upper_dev,
4701 void *upper_priv, void *upper_info,
4702 struct netlink_ext_ack *extack);
4703 void netdev_upper_dev_unlink(struct net_device *dev,
4704 struct net_device *upper_dev);
4705 int netdev_adjacent_change_prepare(struct net_device *old_dev,
4706 struct net_device *new_dev,
4707 struct net_device *dev,
4708 struct netlink_ext_ack *extack);
4709 void netdev_adjacent_change_commit(struct net_device *old_dev,
4710 struct net_device *new_dev,
4711 struct net_device *dev);
4712 void netdev_adjacent_change_abort(struct net_device *old_dev,
4713 struct net_device *new_dev,
4714 struct net_device *dev);
4715 void netdev_adjacent_rename_links(struct net_device *dev, char *oldname);
4716 void *netdev_lower_dev_get_private(struct net_device *dev,
4717 struct net_device *lower_dev);
4718 void netdev_lower_state_changed(struct net_device *lower_dev,
4719 void *lower_state_info);
4720
4721
4722 #define NETDEV_RSS_KEY_LEN 52
4723 extern u8 netdev_rss_key[NETDEV_RSS_KEY_LEN] __read_mostly;
4724 void netdev_rss_key_fill(void *buffer, size_t len);
4725
4726 int skb_checksum_help(struct sk_buff *skb);
4727 int skb_crc32c_csum_help(struct sk_buff *skb);
4728 int skb_csum_hwoffload_help(struct sk_buff *skb,
4729 const netdev_features_t features);
4730
4731 struct sk_buff *__skb_gso_segment(struct sk_buff *skb,
4732 netdev_features_t features, bool tx_path);
4733 struct sk_buff *skb_eth_gso_segment(struct sk_buff *skb,
4734 netdev_features_t features, __be16 type);
4735 struct sk_buff *skb_mac_gso_segment(struct sk_buff *skb,
4736 netdev_features_t features);
4737
4738 struct netdev_bonding_info {
4739 ifslave slave;
4740 ifbond master;
4741 };
4742
4743 struct netdev_notifier_bonding_info {
4744 struct netdev_notifier_info info;
4745 struct netdev_bonding_info bonding_info;
4746 };
4747
4748 void netdev_bonding_info_change(struct net_device *dev,
4749 struct netdev_bonding_info *bonding_info);
4750
4751 #if IS_ENABLED(CONFIG_ETHTOOL_NETLINK)
4752 void ethtool_notify(struct net_device *dev, unsigned int cmd, const void *data);
4753 #else
4754 static inline void ethtool_notify(struct net_device *dev, unsigned int cmd,
4755 const void *data)
4756 {
4757 }
4758 #endif
4759
4760 static inline
4761 struct sk_buff *skb_gso_segment(struct sk_buff *skb, netdev_features_t features)
4762 {
4763 return __skb_gso_segment(skb, features, true);
4764 }
4765 __be16 skb_network_protocol(struct sk_buff *skb, int *depth);
4766
4767 static inline bool can_checksum_protocol(netdev_features_t features,
4768 __be16 protocol)
4769 {
4770 if (protocol == htons(ETH_P_FCOE))
4771 return !!(features & NETIF_F_FCOE_CRC);
4772
4773
4774
4775 if (features & NETIF_F_HW_CSUM) {
4776
4777 return true;
4778 }
4779
4780 switch (protocol) {
4781 case htons(ETH_P_IP):
4782 return !!(features & NETIF_F_IP_CSUM);
4783 case htons(ETH_P_IPV6):
4784 return !!(features & NETIF_F_IPV6_CSUM);
4785 default:
4786 return false;
4787 }
4788 }
4789
4790 #ifdef CONFIG_BUG
4791 void netdev_rx_csum_fault(struct net_device *dev, struct sk_buff *skb);
4792 #else
4793 static inline void netdev_rx_csum_fault(struct net_device *dev,
4794 struct sk_buff *skb)
4795 {
4796 }
4797 #endif
4798
4799 void net_enable_timestamp(void);
4800 void net_disable_timestamp(void);
4801
4802 static inline ktime_t netdev_get_tstamp(struct net_device *dev,
4803 const struct skb_shared_hwtstamps *hwtstamps,
4804 bool cycles)
4805 {
4806 const struct net_device_ops *ops = dev->netdev_ops;
4807
4808 if (ops->ndo_get_tstamp)
4809 return ops->ndo_get_tstamp(dev, hwtstamps, cycles);
4810
4811 return hwtstamps->hwtstamp;
4812 }
4813
4814 static inline netdev_tx_t __netdev_start_xmit(const struct net_device_ops *ops,
4815 struct sk_buff *skb, struct net_device *dev,
4816 bool more)
4817 {
4818 __this_cpu_write(softnet_data.xmit.more, more);
4819 return ops->ndo_start_xmit(skb, dev);
4820 }
4821
4822 static inline bool netdev_xmit_more(void)
4823 {
4824 return __this_cpu_read(softnet_data.xmit.more);
4825 }
4826
4827 static inline netdev_tx_t netdev_start_xmit(struct sk_buff *skb, struct net_device *dev,
4828 struct netdev_queue *txq, bool more)
4829 {
4830 const struct net_device_ops *ops = dev->netdev_ops;
4831 netdev_tx_t rc;
4832
4833 rc = __netdev_start_xmit(ops, skb, dev, more);
4834 if (rc == NETDEV_TX_OK)
4835 txq_trans_update(txq);
4836
4837 return rc;
4838 }
4839
4840 int netdev_class_create_file_ns(const struct class_attribute *class_attr,
4841 const void *ns);
4842 void netdev_class_remove_file_ns(const struct class_attribute *class_attr,
4843 const void *ns);
4844
4845 extern const struct kobj_ns_type_operations net_ns_type_operations;
4846
4847 const char *netdev_drivername(const struct net_device *dev);
4848
4849 static inline netdev_features_t netdev_intersect_features(netdev_features_t f1,
4850 netdev_features_t f2)
4851 {
4852 if ((f1 ^ f2) & NETIF_F_HW_CSUM) {
4853 if (f1 & NETIF_F_HW_CSUM)
4854 f1 |= (NETIF_F_IP_CSUM|NETIF_F_IPV6_CSUM);
4855 else
4856 f2 |= (NETIF_F_IP_CSUM|NETIF_F_IPV6_CSUM);
4857 }
4858
4859 return f1 & f2;
4860 }
4861
4862 static inline netdev_features_t netdev_get_wanted_features(
4863 struct net_device *dev)
4864 {
4865 return (dev->features & ~dev->hw_features) | dev->wanted_features;
4866 }
4867 netdev_features_t netdev_increment_features(netdev_features_t all,
4868 netdev_features_t one, netdev_features_t mask);
4869
4870
4871
4872
4873
4874 static inline netdev_features_t netdev_add_tso_features(netdev_features_t features,
4875 netdev_features_t mask)
4876 {
4877 return netdev_increment_features(features, NETIF_F_ALL_TSO, mask);
4878 }
4879
4880 int __netdev_update_features(struct net_device *dev);
4881 void netdev_update_features(struct net_device *dev);
4882 void netdev_change_features(struct net_device *dev);
4883
4884 void netif_stacked_transfer_operstate(const struct net_device *rootdev,
4885 struct net_device *dev);
4886
4887 netdev_features_t passthru_features_check(struct sk_buff *skb,
4888 struct net_device *dev,
4889 netdev_features_t features);
4890 netdev_features_t netif_skb_features(struct sk_buff *skb);
4891
4892 static inline bool net_gso_ok(netdev_features_t features, int gso_type)
4893 {
4894 netdev_features_t feature = (netdev_features_t)gso_type << NETIF_F_GSO_SHIFT;
4895
4896
4897 BUILD_BUG_ON(SKB_GSO_TCPV4 != (NETIF_F_TSO >> NETIF_F_GSO_SHIFT));
4898 BUILD_BUG_ON(SKB_GSO_DODGY != (NETIF_F_GSO_ROBUST >> NETIF_F_GSO_SHIFT));
4899 BUILD_BUG_ON(SKB_GSO_TCP_ECN != (NETIF_F_TSO_ECN >> NETIF_F_GSO_SHIFT));
4900 BUILD_BUG_ON(SKB_GSO_TCP_FIXEDID != (NETIF_F_TSO_MANGLEID >> NETIF_F_GSO_SHIFT));
4901 BUILD_BUG_ON(SKB_GSO_TCPV6 != (NETIF_F_TSO6 >> NETIF_F_GSO_SHIFT));
4902 BUILD_BUG_ON(SKB_GSO_FCOE != (NETIF_F_FSO >> NETIF_F_GSO_SHIFT));
4903 BUILD_BUG_ON(SKB_GSO_GRE != (NETIF_F_GSO_GRE >> NETIF_F_GSO_SHIFT));
4904 BUILD_BUG_ON(SKB_GSO_GRE_CSUM != (NETIF_F_GSO_GRE_CSUM >> NETIF_F_GSO_SHIFT));
4905 BUILD_BUG_ON(SKB_GSO_IPXIP4 != (NETIF_F_GSO_IPXIP4 >> NETIF_F_GSO_SHIFT));
4906 BUILD_BUG_ON(SKB_GSO_IPXIP6 != (NETIF_F_GSO_IPXIP6 >> NETIF_F_GSO_SHIFT));
4907 BUILD_BUG_ON(SKB_GSO_UDP_TUNNEL != (NETIF_F_GSO_UDP_TUNNEL >> NETIF_F_GSO_SHIFT));
4908 BUILD_BUG_ON(SKB_GSO_UDP_TUNNEL_CSUM != (NETIF_F_GSO_UDP_TUNNEL_CSUM >> NETIF_F_GSO_SHIFT));
4909 BUILD_BUG_ON(SKB_GSO_PARTIAL != (NETIF_F_GSO_PARTIAL >> NETIF_F_GSO_SHIFT));
4910 BUILD_BUG_ON(SKB_GSO_TUNNEL_REMCSUM != (NETIF_F_GSO_TUNNEL_REMCSUM >> NETIF_F_GSO_SHIFT));
4911 BUILD_BUG_ON(SKB_GSO_SCTP != (NETIF_F_GSO_SCTP >> NETIF_F_GSO_SHIFT));
4912 BUILD_BUG_ON(SKB_GSO_ESP != (NETIF_F_GSO_ESP >> NETIF_F_GSO_SHIFT));
4913 BUILD_BUG_ON(SKB_GSO_UDP != (NETIF_F_GSO_UDP >> NETIF_F_GSO_SHIFT));
4914 BUILD_BUG_ON(SKB_GSO_UDP_L4 != (NETIF_F_GSO_UDP_L4 >> NETIF_F_GSO_SHIFT));
4915 BUILD_BUG_ON(SKB_GSO_FRAGLIST != (NETIF_F_GSO_FRAGLIST >> NETIF_F_GSO_SHIFT));
4916
4917 return (features & feature) == feature;
4918 }
4919
4920 static inline bool skb_gso_ok(struct sk_buff *skb, netdev_features_t features)
4921 {
4922 return net_gso_ok(features, skb_shinfo(skb)->gso_type) &&
4923 (!skb_has_frag_list(skb) || (features & NETIF_F_FRAGLIST));
4924 }
4925
4926 static inline bool netif_needs_gso(struct sk_buff *skb,
4927 netdev_features_t features)
4928 {
4929 return skb_is_gso(skb) && (!skb_gso_ok(skb, features) ||
4930 unlikely((skb->ip_summed != CHECKSUM_PARTIAL) &&
4931 (skb->ip_summed != CHECKSUM_UNNECESSARY)));
4932 }
4933
4934 void netif_set_tso_max_size(struct net_device *dev, unsigned int size);
4935 void netif_set_tso_max_segs(struct net_device *dev, unsigned int segs);
4936 void netif_inherit_tso_max(struct net_device *to,
4937 const struct net_device *from);
4938
4939 static inline void skb_gso_error_unwind(struct sk_buff *skb, __be16 protocol,
4940 int pulled_hlen, u16 mac_offset,
4941 int mac_len)
4942 {
4943 skb->protocol = protocol;
4944 skb->encapsulation = 1;
4945 skb_push(skb, pulled_hlen);
4946 skb_reset_transport_header(skb);
4947 skb->mac_header = mac_offset;
4948 skb->network_header = skb->mac_header + mac_len;
4949 skb->mac_len = mac_len;
4950 }
4951
4952 static inline bool netif_is_macsec(const struct net_device *dev)
4953 {
4954 return dev->priv_flags & IFF_MACSEC;
4955 }
4956
4957 static inline bool netif_is_macvlan(const struct net_device *dev)
4958 {
4959 return dev->priv_flags & IFF_MACVLAN;
4960 }
4961
4962 static inline bool netif_is_macvlan_port(const struct net_device *dev)
4963 {
4964 return dev->priv_flags & IFF_MACVLAN_PORT;
4965 }
4966
4967 static inline bool netif_is_bond_master(const struct net_device *dev)
4968 {
4969 return dev->flags & IFF_MASTER && dev->priv_flags & IFF_BONDING;
4970 }
4971
4972 static inline bool netif_is_bond_slave(const struct net_device *dev)
4973 {
4974 return dev->flags & IFF_SLAVE && dev->priv_flags & IFF_BONDING;
4975 }
4976
4977 static inline bool netif_supports_nofcs(struct net_device *dev)
4978 {
4979 return dev->priv_flags & IFF_SUPP_NOFCS;
4980 }
4981
4982 static inline bool netif_has_l3_rx_handler(const struct net_device *dev)
4983 {
4984 return dev->priv_flags & IFF_L3MDEV_RX_HANDLER;
4985 }
4986
4987 static inline bool netif_is_l3_master(const struct net_device *dev)
4988 {
4989 return dev->priv_flags & IFF_L3MDEV_MASTER;
4990 }
4991
4992 static inline bool netif_is_l3_slave(const struct net_device *dev)
4993 {
4994 return dev->priv_flags & IFF_L3MDEV_SLAVE;
4995 }
4996
4997 static inline bool netif_is_bridge_master(const struct net_device *dev)
4998 {
4999 return dev->priv_flags & IFF_EBRIDGE;
5000 }
5001
5002 static inline bool netif_is_bridge_port(const struct net_device *dev)
5003 {
5004 return dev->priv_flags & IFF_BRIDGE_PORT;
5005 }
5006
5007 static inline bool netif_is_ovs_master(const struct net_device *dev)
5008 {
5009 return dev->priv_flags & IFF_OPENVSWITCH;
5010 }
5011
5012 static inline bool netif_is_ovs_port(const struct net_device *dev)
5013 {
5014 return dev->priv_flags & IFF_OVS_DATAPATH;
5015 }
5016
5017 static inline bool netif_is_any_bridge_port(const struct net_device *dev)
5018 {
5019 return netif_is_bridge_port(dev) || netif_is_ovs_port(dev);
5020 }
5021
5022 static inline bool netif_is_team_master(const struct net_device *dev)
5023 {
5024 return dev->priv_flags & IFF_TEAM;
5025 }
5026
5027 static inline bool netif_is_team_port(const struct net_device *dev)
5028 {
5029 return dev->priv_flags & IFF_TEAM_PORT;
5030 }
5031
5032 static inline bool netif_is_lag_master(const struct net_device *dev)
5033 {
5034 return netif_is_bond_master(dev) || netif_is_team_master(dev);
5035 }
5036
5037 static inline bool netif_is_lag_port(const struct net_device *dev)
5038 {
5039 return netif_is_bond_slave(dev) || netif_is_team_port(dev);
5040 }
5041
5042 static inline bool netif_is_rxfh_configured(const struct net_device *dev)
5043 {
5044 return dev->priv_flags & IFF_RXFH_CONFIGURED;
5045 }
5046
5047 static inline bool netif_is_failover(const struct net_device *dev)
5048 {
5049 return dev->priv_flags & IFF_FAILOVER;
5050 }
5051
5052 static inline bool netif_is_failover_slave(const struct net_device *dev)
5053 {
5054 return dev->priv_flags & IFF_FAILOVER_SLAVE;
5055 }
5056
5057
5058 static inline void netif_keep_dst(struct net_device *dev)
5059 {
5060 dev->priv_flags &= ~(IFF_XMIT_DST_RELEASE | IFF_XMIT_DST_RELEASE_PERM);
5061 }
5062
5063
5064 static inline bool netif_reduces_vlan_mtu(struct net_device *dev)
5065 {
5066
5067 return netif_is_macsec(dev);
5068 }
5069
5070 extern struct pernet_operations __net_initdata loopback_net_ops;
5071
5072
5073
5074
5075
5076 static inline const char *netdev_name(const struct net_device *dev)
5077 {
5078 if (!dev->name[0] || strchr(dev->name, '%'))
5079 return "(unnamed net_device)";
5080 return dev->name;
5081 }
5082
5083 static inline bool netdev_unregistering(const struct net_device *dev)
5084 {
5085 return dev->reg_state == NETREG_UNREGISTERING;
5086 }
5087
5088 static inline const char *netdev_reg_state(const struct net_device *dev)
5089 {
5090 switch (dev->reg_state) {
5091 case NETREG_UNINITIALIZED: return " (uninitialized)";
5092 case NETREG_REGISTERED: return "";
5093 case NETREG_UNREGISTERING: return " (unregistering)";
5094 case NETREG_UNREGISTERED: return " (unregistered)";
5095 case NETREG_RELEASED: return " (released)";
5096 case NETREG_DUMMY: return " (dummy)";
5097 }
5098
5099 WARN_ONCE(1, "%s: unknown reg_state %d\n", dev->name, dev->reg_state);
5100 return " (unknown)";
5101 }
5102
5103 #define MODULE_ALIAS_NETDEV(device) \
5104 MODULE_ALIAS("netdev-" device)
5105
5106
5107
5108
5109
5110
5111 #define netdev_WARN(dev, format, args...) \
5112 WARN(1, "netdevice: %s%s: " format, netdev_name(dev), \
5113 netdev_reg_state(dev), ##args)
5114
5115 #define netdev_WARN_ONCE(dev, format, args...) \
5116 WARN_ONCE(1, "netdevice: %s%s: " format, netdev_name(dev), \
5117 netdev_reg_state(dev), ##args)
5118
5119
5120
5121
5122
5123
5124
5125
5126
5127
5128
5129
5130
5131
5132
5133
5134
5135
5136
5137
5138 #define PTYPE_HASH_SIZE (16)
5139 #define PTYPE_HASH_MASK (PTYPE_HASH_SIZE - 1)
5140
5141 extern struct list_head ptype_all __read_mostly;
5142 extern struct list_head ptype_base[PTYPE_HASH_SIZE] __read_mostly;
5143
5144 extern struct net_device *blackhole_netdev;
5145
5146 #endif