blob: 8db5a0b4b52061afe9cd44dead33cc69523d15df [file] [log] [blame]
Linus Torvalds1da177e2005-04-16 15:20:36 -07001/*
2 * NET3 Protocol independent device support routines.
3 *
4 * This program is free software; you can redistribute it and/or
5 * modify it under the terms of the GNU General Public License
6 * as published by the Free Software Foundation; either version
7 * 2 of the License, or (at your option) any later version.
8 *
9 * Derived from the non IP parts of dev.c 1.0.19
Jesper Juhl02c30a82005-05-05 16:16:16 -070010 * Authors: Ross Biro
Linus Torvalds1da177e2005-04-16 15:20:36 -070011 * Fred N. van Kempen, <waltje@uWalt.NL.Mugnet.ORG>
12 * Mark Evans, <evansmp@uhura.aston.ac.uk>
13 *
14 * Additional Authors:
15 * Florian la Roche <rzsfl@rz.uni-sb.de>
16 * Alan Cox <gw4pts@gw4pts.ampr.org>
17 * David Hinds <dahinds@users.sourceforge.net>
18 * Alexey Kuznetsov <kuznet@ms2.inr.ac.ru>
19 * Adam Sulmicki <adam@cfar.umd.edu>
20 * Pekka Riikonen <priikone@poesidon.pspt.fi>
21 *
22 * Changes:
23 * D.J. Barrow : Fixed bug where dev->refcnt gets set
24 * to 2 if register_netdev gets called
25 * before net_dev_init & also removed a
26 * few lines of code in the process.
27 * Alan Cox : device private ioctl copies fields back.
28 * Alan Cox : Transmit queue code does relevant
29 * stunts to keep the queue safe.
30 * Alan Cox : Fixed double lock.
31 * Alan Cox : Fixed promisc NULL pointer trap
32 * ???????? : Support the full private ioctl range
33 * Alan Cox : Moved ioctl permission check into
34 * drivers
35 * Tim Kordas : SIOCADDMULTI/SIOCDELMULTI
36 * Alan Cox : 100 backlog just doesn't cut it when
37 * you start doing multicast video 8)
38 * Alan Cox : Rewrote net_bh and list manager.
39 * Alan Cox : Fix ETH_P_ALL echoback lengths.
40 * Alan Cox : Took out transmit every packet pass
41 * Saved a few bytes in the ioctl handler
42 * Alan Cox : Network driver sets packet type before
43 * calling netif_rx. Saves a function
44 * call a packet.
45 * Alan Cox : Hashed net_bh()
46 * Richard Kooijman: Timestamp fixes.
47 * Alan Cox : Wrong field in SIOCGIFDSTADDR
48 * Alan Cox : Device lock protection.
49 * Alan Cox : Fixed nasty side effect of device close
50 * changes.
51 * Rudi Cilibrasi : Pass the right thing to
52 * set_mac_address()
53 * Dave Miller : 32bit quantity for the device lock to
54 * make it work out on a Sparc.
55 * Bjorn Ekwall : Added KERNELD hack.
56 * Alan Cox : Cleaned up the backlog initialise.
57 * Craig Metz : SIOCGIFCONF fix if space for under
58 * 1 device.
59 * Thomas Bogendoerfer : Return ENODEV for dev_open, if there
60 * is no device open function.
61 * Andi Kleen : Fix error reporting for SIOCGIFCONF
62 * Michael Chastain : Fix signed/unsigned for SIOCGIFCONF
63 * Cyrus Durgin : Cleaned for KMOD
64 * Adam Sulmicki : Bug Fix : Network Device Unload
65 * A network device unload needs to purge
66 * the backlog queue.
67 * Paul Rusty Russell : SIOCSIFNAME
68 * Pekka Riikonen : Netdev boot-time settings code
69 * Andrew Morton : Make unregister_netdevice wait
70 * indefinitely on dev->refcnt
71 * J Hadi Salim : - Backlog queue sampling
72 * - netif_rx() feedback
73 */
74
Linus Torvalds7c0f6ba2016-12-24 11:46:01 -080075#include <linux/uaccess.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -070076#include <linux/bitops.h>
Randy Dunlap4fc268d2006-01-11 12:17:47 -080077#include <linux/capability.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -070078#include <linux/cpu.h>
79#include <linux/types.h>
80#include <linux/kernel.h>
stephen hemminger08e98972009-11-10 07:20:34 +000081#include <linux/hash.h>
Tejun Heo5a0e3ad2010-03-24 17:04:11 +090082#include <linux/slab.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -070083#include <linux/sched.h>
Arjan van de Ven4a3e2f72006-03-20 22:33:17 -080084#include <linux/mutex.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -070085#include <linux/string.h>
86#include <linux/mm.h>
87#include <linux/socket.h>
88#include <linux/sockios.h>
89#include <linux/errno.h>
90#include <linux/interrupt.h>
91#include <linux/if_ether.h>
92#include <linux/netdevice.h>
93#include <linux/etherdevice.h>
Ben Hutchings0187bdf2008-06-19 16:15:47 -070094#include <linux/ethtool.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -070095#include <linux/notifier.h>
96#include <linux/skbuff.h>
Brenden Blancoa7862b42016-07-19 12:16:48 -070097#include <linux/bpf.h>
Eric W. Biederman457c4cb2007-09-12 12:01:34 +020098#include <net/net_namespace.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -070099#include <net/sock.h>
Eric Dumazet02d62e82015-11-18 06:30:52 -0800100#include <net/busy_poll.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -0700101#include <linux/rtnetlink.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -0700102#include <linux/stat.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -0700103#include <net/dst.h>
Pravin B Shelarfc4099f2015-10-22 18:17:16 -0700104#include <net/dst_metadata.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -0700105#include <net/pkt_sched.h>
106#include <net/checksum.h>
Arnd Bergmann44540962009-11-26 06:07:08 +0000107#include <net/xfrm.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -0700108#include <linux/highmem.h>
109#include <linux/init.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -0700110#include <linux/module.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -0700111#include <linux/netpoll.h>
112#include <linux/rcupdate.h>
113#include <linux/delay.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -0700114#include <net/iw_handler.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -0700115#include <asm/current.h>
Steve Grubb5bdb9882005-12-03 08:39:35 -0500116#include <linux/audit.h>
Chris Leechdb217332006-06-17 21:24:58 -0700117#include <linux/dmaengine.h>
Herbert Xuf6a78bf2006-06-22 02:57:17 -0700118#include <linux/err.h>
David S. Millerc7fa9d12006-08-15 16:34:13 -0700119#include <linux/ctype.h>
Jarek Poplawski723e98b2007-05-15 22:46:18 -0700120#include <linux/if_arp.h>
Ben Hutchings6de329e2008-06-16 17:02:28 -0700121#include <linux/if_vlan.h>
David S. Miller8f0f2222008-07-15 03:47:03 -0700122#include <linux/ip.h>
Alexander Duyckad55dca2008-09-20 22:05:50 -0700123#include <net/ip.h>
Simon Horman25cd9ba2014-10-06 05:05:13 -0700124#include <net/mpls.h>
David S. Miller8f0f2222008-07-15 03:47:03 -0700125#include <linux/ipv6.h>
126#include <linux/in.h>
David S. Millerb6b2fed2008-07-21 09:48:06 -0700127#include <linux/jhash.h>
128#include <linux/random.h>
David S. Miller9cbc1cb2009-06-15 03:02:23 -0700129#include <trace/events/napi.h>
Koki Sanagicf66ba52010-08-23 18:45:02 +0900130#include <trace/events/net.h>
Koki Sanagi07dc22e2010-08-23 18:46:12 +0900131#include <trace/events/skb.h>
FUJITA Tomonori5acbbd42010-03-30 22:35:50 +0000132#include <linux/pci.h>
Stephen Rothwellcaeda9b2010-09-16 21:39:16 -0700133#include <linux/inetdevice.h>
Ben Hutchingsc4454772011-01-19 11:03:53 +0000134#include <linux/cpu_rmap.h>
Ingo Molnarc5905af2012-02-24 08:31:31 +0100135#include <linux/static_key.h>
Eliezer Tamiraf12fa62013-06-10 11:39:41 +0300136#include <linux/hashtable.h>
Eric Dumazet60877a32013-06-20 01:15:51 -0700137#include <linux/vmalloc.h>
Michal Kubeček529d0482013-11-15 06:18:50 +0100138#include <linux/if_macvlan.h>
Willem de Bruijne7fd2882014-08-04 22:11:48 -0400139#include <linux/errqueue.h>
Eric Dumazet3b47d302014-11-06 21:09:44 -0800140#include <linux/hrtimer.h>
Pablo Neirae687ad62015-05-13 18:19:38 +0200141#include <linux/netfilter_ingress.h>
Hariprasad Shenai40e4e712016-06-08 18:09:08 +0530142#include <linux/crash_dump.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -0700143
Pavel Emelyanov342709e2007-10-23 21:14:45 -0700144#include "net-sysfs.h"
145
Herbert Xud565b0a2008-12-15 23:38:52 -0800146/* Instead of increasing this, you should create a hash table. */
147#define MAX_GRO_SKBS 8
148
Herbert Xu5d38a072009-01-04 16:13:40 -0800149/* This should be increased if a protocol with a bigger head is added. */
150#define GRO_MAX_HEAD (MAX_HEADER + 128)
151
Linus Torvalds1da177e2005-04-16 15:20:36 -0700152static DEFINE_SPINLOCK(ptype_lock);
Vlad Yasevich62532da2012-11-15 08:49:10 +0000153static DEFINE_SPINLOCK(offload_lock);
Cong Wang900ff8c2013-02-18 19:20:33 +0000154struct list_head ptype_base[PTYPE_HASH_SIZE] __read_mostly;
155struct list_head ptype_all __read_mostly; /* Taps */
Vlad Yasevich62532da2012-11-15 08:49:10 +0000156static struct list_head offload_base __read_mostly;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700157
Ben Hutchingsae78dbf2014-01-10 22:17:24 +0000158static int netif_rx_internal(struct sk_buff *skb);
Loic Prylli54951192014-07-01 21:39:43 -0700159static int call_netdevice_notifiers_info(unsigned long val,
160 struct net_device *dev,
161 struct netdev_notifier_info *info);
Ben Hutchingsae78dbf2014-01-10 22:17:24 +0000162
Linus Torvalds1da177e2005-04-16 15:20:36 -0700163/*
Pavel Emelianov7562f872007-05-03 15:13:45 -0700164 * The @dev_base_head list is protected by @dev_base_lock and the rtnl
Linus Torvalds1da177e2005-04-16 15:20:36 -0700165 * semaphore.
166 *
Eric Dumazetc6d14c82009-11-04 05:43:23 -0800167 * Pure readers hold dev_base_lock for reading, or rcu_read_lock()
Linus Torvalds1da177e2005-04-16 15:20:36 -0700168 *
169 * Writers must hold the rtnl semaphore while they loop through the
Pavel Emelianov7562f872007-05-03 15:13:45 -0700170 * dev_base_head list, and hold dev_base_lock for writing when they do the
Linus Torvalds1da177e2005-04-16 15:20:36 -0700171 * actual updates. This allows pure readers to access the list even
172 * while a writer is preparing to update it.
173 *
174 * To put it another way, dev_base_lock is held for writing only to
175 * protect against pure readers; the rtnl semaphore provides the
176 * protection against other writers.
177 *
178 * See, for example usages, register_netdevice() and
179 * unregister_netdevice(), which must be called with the rtnl
180 * semaphore held.
181 */
Linus Torvalds1da177e2005-04-16 15:20:36 -0700182DEFINE_RWLOCK(dev_base_lock);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700183EXPORT_SYMBOL(dev_base_lock);
184
Eliezer Tamiraf12fa62013-06-10 11:39:41 +0300185/* protects napi_hash addition/deletion and napi_gen_id */
186static DEFINE_SPINLOCK(napi_hash_lock);
187
Eric Dumazet52bd2d62015-11-18 06:30:50 -0800188static unsigned int napi_gen_id = NR_CPUS;
Eric Dumazet6180d9d2015-11-18 06:31:01 -0800189static DEFINE_READ_MOSTLY_HASHTABLE(napi_hash, 8);
Eliezer Tamiraf12fa62013-06-10 11:39:41 +0300190
Thomas Gleixner18afa4b2013-07-23 16:13:17 +0200191static seqcount_t devnet_rename_seq;
Brian Haleyc91f6df2012-11-26 05:21:08 +0000192
Thomas Graf4e985ad2011-06-21 03:11:20 +0000193static inline void dev_base_seq_inc(struct net *net)
194{
195 while (++net->dev_base_seq == 0);
196}
197
Eric W. Biederman881d9662007-09-17 11:56:21 -0700198static inline struct hlist_head *dev_name_hash(struct net *net, const char *name)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700199{
Linus Torvalds8387ff22016-06-10 07:51:30 -0700200 unsigned int hash = full_name_hash(net, name, strnlen(name, IFNAMSIZ));
Eric Dumazet95c96172012-04-15 05:58:06 +0000201
stephen hemminger08e98972009-11-10 07:20:34 +0000202 return &net->dev_name_head[hash_32(hash, NETDEV_HASHBITS)];
Linus Torvalds1da177e2005-04-16 15:20:36 -0700203}
204
Eric W. Biederman881d9662007-09-17 11:56:21 -0700205static inline struct hlist_head *dev_index_hash(struct net *net, int ifindex)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700206{
Eric Dumazet7c28bd02009-10-24 06:13:17 -0700207 return &net->dev_index_head[ifindex & (NETDEV_HASHENTRIES - 1)];
Linus Torvalds1da177e2005-04-16 15:20:36 -0700208}
209
Eric Dumazete36fa2f2010-04-19 21:17:14 +0000210static inline void rps_lock(struct softnet_data *sd)
Changli Gao152102c2010-03-30 20:16:22 +0000211{
212#ifdef CONFIG_RPS
Eric Dumazete36fa2f2010-04-19 21:17:14 +0000213 spin_lock(&sd->input_pkt_queue.lock);
Changli Gao152102c2010-03-30 20:16:22 +0000214#endif
215}
216
Eric Dumazete36fa2f2010-04-19 21:17:14 +0000217static inline void rps_unlock(struct softnet_data *sd)
Changli Gao152102c2010-03-30 20:16:22 +0000218{
219#ifdef CONFIG_RPS
Eric Dumazete36fa2f2010-04-19 21:17:14 +0000220 spin_unlock(&sd->input_pkt_queue.lock);
Changli Gao152102c2010-03-30 20:16:22 +0000221#endif
222}
223
Eric W. Biedermance286d32007-09-12 13:53:49 +0200224/* Device list insertion */
dingtianhong53759be2013-04-17 22:17:50 +0000225static void list_netdevice(struct net_device *dev)
Eric W. Biedermance286d32007-09-12 13:53:49 +0200226{
YOSHIFUJI Hideakic346dca2008-03-25 21:47:49 +0900227 struct net *net = dev_net(dev);
Eric W. Biedermance286d32007-09-12 13:53:49 +0200228
229 ASSERT_RTNL();
230
231 write_lock_bh(&dev_base_lock);
Eric Dumazetc6d14c82009-11-04 05:43:23 -0800232 list_add_tail_rcu(&dev->dev_list, &net->dev_base_head);
Eric Dumazet72c95282009-10-30 07:11:27 +0000233 hlist_add_head_rcu(&dev->name_hlist, dev_name_hash(net, dev->name));
Eric Dumazetfb699dfd2009-10-19 19:18:49 +0000234 hlist_add_head_rcu(&dev->index_hlist,
235 dev_index_hash(net, dev->ifindex));
Eric W. Biedermance286d32007-09-12 13:53:49 +0200236 write_unlock_bh(&dev_base_lock);
Thomas Graf4e985ad2011-06-21 03:11:20 +0000237
238 dev_base_seq_inc(net);
Eric W. Biedermance286d32007-09-12 13:53:49 +0200239}
240
Eric Dumazetfb699dfd2009-10-19 19:18:49 +0000241/* Device list removal
242 * caller must respect a RCU grace period before freeing/reusing dev
243 */
Eric W. Biedermance286d32007-09-12 13:53:49 +0200244static void unlist_netdevice(struct net_device *dev)
245{
246 ASSERT_RTNL();
247
248 /* Unlink dev from the device chain */
249 write_lock_bh(&dev_base_lock);
Eric Dumazetc6d14c82009-11-04 05:43:23 -0800250 list_del_rcu(&dev->dev_list);
Eric Dumazet72c95282009-10-30 07:11:27 +0000251 hlist_del_rcu(&dev->name_hlist);
Eric Dumazetfb699dfd2009-10-19 19:18:49 +0000252 hlist_del_rcu(&dev->index_hlist);
Eric W. Biedermance286d32007-09-12 13:53:49 +0200253 write_unlock_bh(&dev_base_lock);
Thomas Graf4e985ad2011-06-21 03:11:20 +0000254
255 dev_base_seq_inc(dev_net(dev));
Eric W. Biedermance286d32007-09-12 13:53:49 +0200256}
257
Linus Torvalds1da177e2005-04-16 15:20:36 -0700258/*
259 * Our notifier list
260 */
261
Alan Sternf07d5b92006-05-09 15:23:03 -0700262static RAW_NOTIFIER_HEAD(netdev_chain);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700263
264/*
265 * Device drivers call our routines to queue packets here. We empty the
266 * queue in the local softnet handler.
267 */
Stephen Hemmingerbea33482007-10-03 16:41:36 -0700268
Eric Dumazet9958da02010-04-17 04:17:02 +0000269DEFINE_PER_CPU_ALIGNED(struct softnet_data, softnet_data);
Eric Dumazetd1b19df2009-09-03 01:29:39 -0700270EXPORT_PER_CPU_SYMBOL(softnet_data);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700271
David S. Millercf508b12008-07-22 14:16:42 -0700272#ifdef CONFIG_LOCKDEP
Jarek Poplawski723e98b2007-05-15 22:46:18 -0700273/*
David S. Millerc773e842008-07-08 23:13:53 -0700274 * register_netdevice() inits txq->_xmit_lock and sets lockdep class
Jarek Poplawski723e98b2007-05-15 22:46:18 -0700275 * according to dev->type
276 */
277static const unsigned short netdev_lock_type[] =
278 {ARPHRD_NETROM, ARPHRD_ETHER, ARPHRD_EETHER, ARPHRD_AX25,
279 ARPHRD_PRONET, ARPHRD_CHAOS, ARPHRD_IEEE802, ARPHRD_ARCNET,
280 ARPHRD_APPLETLK, ARPHRD_DLCI, ARPHRD_ATM, ARPHRD_METRICOM,
281 ARPHRD_IEEE1394, ARPHRD_EUI64, ARPHRD_INFINIBAND, ARPHRD_SLIP,
282 ARPHRD_CSLIP, ARPHRD_SLIP6, ARPHRD_CSLIP6, ARPHRD_RSRVD,
283 ARPHRD_ADAPT, ARPHRD_ROSE, ARPHRD_X25, ARPHRD_HWX25,
284 ARPHRD_PPP, ARPHRD_CISCO, ARPHRD_LAPB, ARPHRD_DDCMP,
285 ARPHRD_RAWHDLC, ARPHRD_TUNNEL, ARPHRD_TUNNEL6, ARPHRD_FRAD,
286 ARPHRD_SKIP, ARPHRD_LOOPBACK, ARPHRD_LOCALTLK, ARPHRD_FDDI,
287 ARPHRD_BIF, ARPHRD_SIT, ARPHRD_IPDDP, ARPHRD_IPGRE,
288 ARPHRD_PIMREG, ARPHRD_HIPPI, ARPHRD_ASH, ARPHRD_ECONET,
289 ARPHRD_IRDA, ARPHRD_FCPP, ARPHRD_FCAL, ARPHRD_FCPL,
Paul Gortmaker211ed862012-05-10 17:14:35 -0400290 ARPHRD_FCFABRIC, ARPHRD_IEEE80211, ARPHRD_IEEE80211_PRISM,
291 ARPHRD_IEEE80211_RADIOTAP, ARPHRD_PHONET, ARPHRD_PHONET_PIPE,
292 ARPHRD_IEEE802154, ARPHRD_VOID, ARPHRD_NONE};
Jarek Poplawski723e98b2007-05-15 22:46:18 -0700293
Jan Engelhardt36cbd3d2009-08-05 10:42:58 -0700294static const char *const netdev_lock_name[] =
Jarek Poplawski723e98b2007-05-15 22:46:18 -0700295 {"_xmit_NETROM", "_xmit_ETHER", "_xmit_EETHER", "_xmit_AX25",
296 "_xmit_PRONET", "_xmit_CHAOS", "_xmit_IEEE802", "_xmit_ARCNET",
297 "_xmit_APPLETLK", "_xmit_DLCI", "_xmit_ATM", "_xmit_METRICOM",
298 "_xmit_IEEE1394", "_xmit_EUI64", "_xmit_INFINIBAND", "_xmit_SLIP",
299 "_xmit_CSLIP", "_xmit_SLIP6", "_xmit_CSLIP6", "_xmit_RSRVD",
300 "_xmit_ADAPT", "_xmit_ROSE", "_xmit_X25", "_xmit_HWX25",
301 "_xmit_PPP", "_xmit_CISCO", "_xmit_LAPB", "_xmit_DDCMP",
302 "_xmit_RAWHDLC", "_xmit_TUNNEL", "_xmit_TUNNEL6", "_xmit_FRAD",
303 "_xmit_SKIP", "_xmit_LOOPBACK", "_xmit_LOCALTLK", "_xmit_FDDI",
304 "_xmit_BIF", "_xmit_SIT", "_xmit_IPDDP", "_xmit_IPGRE",
305 "_xmit_PIMREG", "_xmit_HIPPI", "_xmit_ASH", "_xmit_ECONET",
306 "_xmit_IRDA", "_xmit_FCPP", "_xmit_FCAL", "_xmit_FCPL",
Paul Gortmaker211ed862012-05-10 17:14:35 -0400307 "_xmit_FCFABRIC", "_xmit_IEEE80211", "_xmit_IEEE80211_PRISM",
308 "_xmit_IEEE80211_RADIOTAP", "_xmit_PHONET", "_xmit_PHONET_PIPE",
309 "_xmit_IEEE802154", "_xmit_VOID", "_xmit_NONE"};
Jarek Poplawski723e98b2007-05-15 22:46:18 -0700310
311static struct lock_class_key netdev_xmit_lock_key[ARRAY_SIZE(netdev_lock_type)];
David S. Millercf508b12008-07-22 14:16:42 -0700312static struct lock_class_key netdev_addr_lock_key[ARRAY_SIZE(netdev_lock_type)];
Jarek Poplawski723e98b2007-05-15 22:46:18 -0700313
314static inline unsigned short netdev_lock_pos(unsigned short dev_type)
315{
316 int i;
317
318 for (i = 0; i < ARRAY_SIZE(netdev_lock_type); i++)
319 if (netdev_lock_type[i] == dev_type)
320 return i;
321 /* the last key is used by default */
322 return ARRAY_SIZE(netdev_lock_type) - 1;
323}
324
David S. Millercf508b12008-07-22 14:16:42 -0700325static inline void netdev_set_xmit_lockdep_class(spinlock_t *lock,
326 unsigned short dev_type)
Jarek Poplawski723e98b2007-05-15 22:46:18 -0700327{
328 int i;
329
330 i = netdev_lock_pos(dev_type);
331 lockdep_set_class_and_name(lock, &netdev_xmit_lock_key[i],
332 netdev_lock_name[i]);
333}
David S. Millercf508b12008-07-22 14:16:42 -0700334
335static inline void netdev_set_addr_lockdep_class(struct net_device *dev)
336{
337 int i;
338
339 i = netdev_lock_pos(dev->type);
340 lockdep_set_class_and_name(&dev->addr_list_lock,
341 &netdev_addr_lock_key[i],
342 netdev_lock_name[i]);
343}
Jarek Poplawski723e98b2007-05-15 22:46:18 -0700344#else
David S. Millercf508b12008-07-22 14:16:42 -0700345static inline void netdev_set_xmit_lockdep_class(spinlock_t *lock,
346 unsigned short dev_type)
347{
348}
349static inline void netdev_set_addr_lockdep_class(struct net_device *dev)
Jarek Poplawski723e98b2007-05-15 22:46:18 -0700350{
351}
352#endif
Linus Torvalds1da177e2005-04-16 15:20:36 -0700353
354/*******************************************************************************
355
356 Protocol management and registration routines
357
358*******************************************************************************/
359
360/*
Linus Torvalds1da177e2005-04-16 15:20:36 -0700361 * Add a protocol ID to the list. Now that the input handler is
362 * smarter we can dispense with all the messy stuff that used to be
363 * here.
364 *
365 * BEWARE!!! Protocol handlers, mangling input packets,
366 * MUST BE last in hash buckets and checking protocol handlers
367 * MUST start from promiscuous ptype_all chain in net_bh.
368 * It is true now, do not change it.
369 * Explanation follows: if protocol handler, mangling packet, will
370 * be the first on list, it is not able to sense, that packet
371 * is cloned and should be copied-on-write, so that it will
372 * change it and subsequent readers will get broken packet.
373 * --ANK (980803)
374 */
375
Eric Dumazetc07b68e2010-09-02 03:53:46 +0000376static inline struct list_head *ptype_head(const struct packet_type *pt)
377{
378 if (pt->type == htons(ETH_P_ALL))
Salam Noureddine7866a622015-01-27 11:35:48 -0800379 return pt->dev ? &pt->dev->ptype_all : &ptype_all;
Eric Dumazetc07b68e2010-09-02 03:53:46 +0000380 else
Salam Noureddine7866a622015-01-27 11:35:48 -0800381 return pt->dev ? &pt->dev->ptype_specific :
382 &ptype_base[ntohs(pt->type) & PTYPE_HASH_MASK];
Eric Dumazetc07b68e2010-09-02 03:53:46 +0000383}
384
Linus Torvalds1da177e2005-04-16 15:20:36 -0700385/**
386 * dev_add_pack - add packet handler
387 * @pt: packet type declaration
388 *
389 * Add a protocol handler to the networking stack. The passed &packet_type
390 * is linked into kernel lists and may not be freed until it has been
391 * removed from the kernel lists.
392 *
YOSHIFUJI Hideaki4ec93ed2007-02-09 23:24:36 +0900393 * This call does not sleep therefore it can not
Linus Torvalds1da177e2005-04-16 15:20:36 -0700394 * guarantee all CPU's that are in middle of receiving packets
395 * will see the new packet type (until the next received packet).
396 */
397
398void dev_add_pack(struct packet_type *pt)
399{
Eric Dumazetc07b68e2010-09-02 03:53:46 +0000400 struct list_head *head = ptype_head(pt);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700401
Eric Dumazetc07b68e2010-09-02 03:53:46 +0000402 spin_lock(&ptype_lock);
403 list_add_rcu(&pt->list, head);
404 spin_unlock(&ptype_lock);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700405}
Eric Dumazetd1b19df2009-09-03 01:29:39 -0700406EXPORT_SYMBOL(dev_add_pack);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700407
Linus Torvalds1da177e2005-04-16 15:20:36 -0700408/**
409 * __dev_remove_pack - remove packet handler
410 * @pt: packet type declaration
411 *
412 * Remove a protocol handler that was previously added to the kernel
413 * protocol handlers by dev_add_pack(). The passed &packet_type is removed
414 * from the kernel lists and can be freed or reused once this function
YOSHIFUJI Hideaki4ec93ed2007-02-09 23:24:36 +0900415 * returns.
Linus Torvalds1da177e2005-04-16 15:20:36 -0700416 *
417 * The packet type might still be in use by receivers
418 * and must not be freed until after all the CPU's have gone
419 * through a quiescent state.
420 */
421void __dev_remove_pack(struct packet_type *pt)
422{
Eric Dumazetc07b68e2010-09-02 03:53:46 +0000423 struct list_head *head = ptype_head(pt);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700424 struct packet_type *pt1;
425
Eric Dumazetc07b68e2010-09-02 03:53:46 +0000426 spin_lock(&ptype_lock);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700427
428 list_for_each_entry(pt1, head, list) {
429 if (pt == pt1) {
430 list_del_rcu(&pt->list);
431 goto out;
432 }
433 }
434
Joe Perches7b6cd1c2012-02-01 10:54:43 +0000435 pr_warn("dev_remove_pack: %p not found\n", pt);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700436out:
Eric Dumazetc07b68e2010-09-02 03:53:46 +0000437 spin_unlock(&ptype_lock);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700438}
Eric Dumazetd1b19df2009-09-03 01:29:39 -0700439EXPORT_SYMBOL(__dev_remove_pack);
440
Linus Torvalds1da177e2005-04-16 15:20:36 -0700441/**
442 * dev_remove_pack - remove packet handler
443 * @pt: packet type declaration
444 *
445 * Remove a protocol handler that was previously added to the kernel
446 * protocol handlers by dev_add_pack(). The passed &packet_type is removed
447 * from the kernel lists and can be freed or reused once this function
448 * returns.
449 *
450 * This call sleeps to guarantee that no CPU is looking at the packet
451 * type after return.
452 */
453void dev_remove_pack(struct packet_type *pt)
454{
455 __dev_remove_pack(pt);
YOSHIFUJI Hideaki4ec93ed2007-02-09 23:24:36 +0900456
Linus Torvalds1da177e2005-04-16 15:20:36 -0700457 synchronize_net();
458}
Eric Dumazetd1b19df2009-09-03 01:29:39 -0700459EXPORT_SYMBOL(dev_remove_pack);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700460
Vlad Yasevich62532da2012-11-15 08:49:10 +0000461
462/**
463 * dev_add_offload - register offload handlers
464 * @po: protocol offload declaration
465 *
466 * Add protocol offload handlers to the networking stack. The passed
467 * &proto_offload is linked into kernel lists and may not be freed until
468 * it has been removed from the kernel lists.
469 *
470 * This call does not sleep therefore it can not
471 * guarantee all CPU's that are in middle of receiving packets
472 * will see the new offload handlers (until the next received packet).
473 */
474void dev_add_offload(struct packet_offload *po)
475{
David S. Millerbdef7de2015-06-01 14:56:09 -0700476 struct packet_offload *elem;
Vlad Yasevich62532da2012-11-15 08:49:10 +0000477
478 spin_lock(&offload_lock);
David S. Millerbdef7de2015-06-01 14:56:09 -0700479 list_for_each_entry(elem, &offload_base, list) {
480 if (po->priority < elem->priority)
481 break;
482 }
483 list_add_rcu(&po->list, elem->list.prev);
Vlad Yasevich62532da2012-11-15 08:49:10 +0000484 spin_unlock(&offload_lock);
485}
486EXPORT_SYMBOL(dev_add_offload);
487
488/**
489 * __dev_remove_offload - remove offload handler
490 * @po: packet offload declaration
491 *
492 * Remove a protocol offload handler that was previously added to the
493 * kernel offload handlers by dev_add_offload(). The passed &offload_type
494 * is removed from the kernel lists and can be freed or reused once this
495 * function returns.
496 *
497 * The packet type might still be in use by receivers
498 * and must not be freed until after all the CPU's have gone
499 * through a quiescent state.
500 */
stephen hemminger1d143d92013-12-29 14:01:29 -0800501static void __dev_remove_offload(struct packet_offload *po)
Vlad Yasevich62532da2012-11-15 08:49:10 +0000502{
503 struct list_head *head = &offload_base;
504 struct packet_offload *po1;
505
Eric Dumazetc53aa502012-11-16 08:08:23 +0000506 spin_lock(&offload_lock);
Vlad Yasevich62532da2012-11-15 08:49:10 +0000507
508 list_for_each_entry(po1, head, list) {
509 if (po == po1) {
510 list_del_rcu(&po->list);
511 goto out;
512 }
513 }
514
515 pr_warn("dev_remove_offload: %p not found\n", po);
516out:
Eric Dumazetc53aa502012-11-16 08:08:23 +0000517 spin_unlock(&offload_lock);
Vlad Yasevich62532da2012-11-15 08:49:10 +0000518}
Vlad Yasevich62532da2012-11-15 08:49:10 +0000519
520/**
521 * dev_remove_offload - remove packet offload handler
522 * @po: packet offload declaration
523 *
524 * Remove a packet offload handler that was previously added to the kernel
525 * offload handlers by dev_add_offload(). The passed &offload_type is
526 * removed from the kernel lists and can be freed or reused once this
527 * function returns.
528 *
529 * This call sleeps to guarantee that no CPU is looking at the packet
530 * type after return.
531 */
532void dev_remove_offload(struct packet_offload *po)
533{
534 __dev_remove_offload(po);
535
536 synchronize_net();
537}
538EXPORT_SYMBOL(dev_remove_offload);
539
Linus Torvalds1da177e2005-04-16 15:20:36 -0700540/******************************************************************************
541
542 Device Boot-time Settings Routines
543
544*******************************************************************************/
545
546/* Boot time configuration table */
547static struct netdev_boot_setup dev_boot_setup[NETDEV_BOOT_SETUP_MAX];
548
549/**
550 * netdev_boot_setup_add - add new setup entry
551 * @name: name of the device
552 * @map: configured settings for the device
553 *
554 * Adds new setup entry to the dev_boot_setup list. The function
555 * returns 0 on error and 1 on success. This is a generic routine to
556 * all netdevices.
557 */
558static int netdev_boot_setup_add(char *name, struct ifmap *map)
559{
560 struct netdev_boot_setup *s;
561 int i;
562
563 s = dev_boot_setup;
564 for (i = 0; i < NETDEV_BOOT_SETUP_MAX; i++) {
565 if (s[i].name[0] == '\0' || s[i].name[0] == ' ') {
566 memset(s[i].name, 0, sizeof(s[i].name));
Wang Chen93b3cff2008-07-01 19:57:19 -0700567 strlcpy(s[i].name, name, IFNAMSIZ);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700568 memcpy(&s[i].map, map, sizeof(s[i].map));
569 break;
570 }
571 }
572
573 return i >= NETDEV_BOOT_SETUP_MAX ? 0 : 1;
574}
575
576/**
577 * netdev_boot_setup_check - check boot time settings
578 * @dev: the netdevice
579 *
580 * Check boot time settings for the device.
581 * The found settings are set for the device to be used
582 * later in the device probing.
583 * Returns 0 if no settings found, 1 if they are.
584 */
585int netdev_boot_setup_check(struct net_device *dev)
586{
587 struct netdev_boot_setup *s = dev_boot_setup;
588 int i;
589
590 for (i = 0; i < NETDEV_BOOT_SETUP_MAX; i++) {
591 if (s[i].name[0] != '\0' && s[i].name[0] != ' ' &&
Wang Chen93b3cff2008-07-01 19:57:19 -0700592 !strcmp(dev->name, s[i].name)) {
Linus Torvalds1da177e2005-04-16 15:20:36 -0700593 dev->irq = s[i].map.irq;
594 dev->base_addr = s[i].map.base_addr;
595 dev->mem_start = s[i].map.mem_start;
596 dev->mem_end = s[i].map.mem_end;
597 return 1;
598 }
599 }
600 return 0;
601}
Eric Dumazetd1b19df2009-09-03 01:29:39 -0700602EXPORT_SYMBOL(netdev_boot_setup_check);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700603
604
605/**
606 * netdev_boot_base - get address from boot time settings
607 * @prefix: prefix for network device
608 * @unit: id for network device
609 *
610 * Check boot time settings for the base address of device.
611 * The found settings are set for the device to be used
612 * later in the device probing.
613 * Returns 0 if no settings found.
614 */
615unsigned long netdev_boot_base(const char *prefix, int unit)
616{
617 const struct netdev_boot_setup *s = dev_boot_setup;
618 char name[IFNAMSIZ];
619 int i;
620
621 sprintf(name, "%s%d", prefix, unit);
622
623 /*
624 * If device already registered then return base of 1
625 * to indicate not to probe for this interface
626 */
Eric W. Biederman881d9662007-09-17 11:56:21 -0700627 if (__dev_get_by_name(&init_net, name))
Linus Torvalds1da177e2005-04-16 15:20:36 -0700628 return 1;
629
630 for (i = 0; i < NETDEV_BOOT_SETUP_MAX; i++)
631 if (!strcmp(name, s[i].name))
632 return s[i].map.base_addr;
633 return 0;
634}
635
636/*
637 * Saves at boot time configured settings for any netdevice.
638 */
639int __init netdev_boot_setup(char *str)
640{
641 int ints[5];
642 struct ifmap map;
643
644 str = get_options(str, ARRAY_SIZE(ints), ints);
645 if (!str || !*str)
646 return 0;
647
648 /* Save settings */
649 memset(&map, 0, sizeof(map));
650 if (ints[0] > 0)
651 map.irq = ints[1];
652 if (ints[0] > 1)
653 map.base_addr = ints[2];
654 if (ints[0] > 2)
655 map.mem_start = ints[3];
656 if (ints[0] > 3)
657 map.mem_end = ints[4];
658
659 /* Add new entry to the list */
660 return netdev_boot_setup_add(str, &map);
661}
662
663__setup("netdev=", netdev_boot_setup);
664
665/*******************************************************************************
666
667 Device Interface Subroutines
668
669*******************************************************************************/
670
671/**
Nicolas Dichtela54acb32015-04-02 17:07:00 +0200672 * dev_get_iflink - get 'iflink' value of a interface
673 * @dev: targeted interface
674 *
675 * Indicates the ifindex the interface is linked to.
676 * Physical interfaces have the same 'ifindex' and 'iflink' values.
677 */
678
679int dev_get_iflink(const struct net_device *dev)
680{
681 if (dev->netdev_ops && dev->netdev_ops->ndo_get_iflink)
682 return dev->netdev_ops->ndo_get_iflink(dev);
683
Nicolas Dichtel7a66bbc2015-04-02 17:07:09 +0200684 return dev->ifindex;
Nicolas Dichtela54acb32015-04-02 17:07:00 +0200685}
686EXPORT_SYMBOL(dev_get_iflink);
687
688/**
Pravin B Shelarfc4099f2015-10-22 18:17:16 -0700689 * dev_fill_metadata_dst - Retrieve tunnel egress information.
690 * @dev: targeted interface
691 * @skb: The packet.
692 *
693 * For better visibility of tunnel traffic OVS needs to retrieve
694 * egress tunnel information for a packet. Following API allows
695 * user to get this info.
696 */
697int dev_fill_metadata_dst(struct net_device *dev, struct sk_buff *skb)
698{
699 struct ip_tunnel_info *info;
700
701 if (!dev->netdev_ops || !dev->netdev_ops->ndo_fill_metadata_dst)
702 return -EINVAL;
703
704 info = skb_tunnel_info_unclone(skb);
705 if (!info)
706 return -ENOMEM;
707 if (unlikely(!(info->mode & IP_TUNNEL_INFO_TX)))
708 return -EINVAL;
709
710 return dev->netdev_ops->ndo_fill_metadata_dst(dev, skb);
711}
712EXPORT_SYMBOL_GPL(dev_fill_metadata_dst);
713
714/**
Linus Torvalds1da177e2005-04-16 15:20:36 -0700715 * __dev_get_by_name - find a device by its name
Randy Dunlapc4ea43c2007-10-12 21:17:49 -0700716 * @net: the applicable net namespace
Linus Torvalds1da177e2005-04-16 15:20:36 -0700717 * @name: name to find
718 *
719 * Find an interface by name. Must be called under RTNL semaphore
720 * or @dev_base_lock. If the name is found a pointer to the device
721 * is returned. If the name is not found then %NULL is returned. The
722 * reference counters are not incremented so the caller must be
723 * careful with locks.
724 */
725
Eric W. Biederman881d9662007-09-17 11:56:21 -0700726struct net_device *__dev_get_by_name(struct net *net, const char *name)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700727{
Eric Dumazet0bd8d532009-10-30 01:40:11 -0700728 struct net_device *dev;
729 struct hlist_head *head = dev_name_hash(net, name);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700730
Sasha Levinb67bfe02013-02-27 17:06:00 -0800731 hlist_for_each_entry(dev, head, name_hlist)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700732 if (!strncmp(dev->name, name, IFNAMSIZ))
733 return dev;
Eric Dumazet0bd8d532009-10-30 01:40:11 -0700734
Linus Torvalds1da177e2005-04-16 15:20:36 -0700735 return NULL;
736}
Eric Dumazetd1b19df2009-09-03 01:29:39 -0700737EXPORT_SYMBOL(__dev_get_by_name);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700738
739/**
Eric Dumazet72c95282009-10-30 07:11:27 +0000740 * dev_get_by_name_rcu - find a device by its name
741 * @net: the applicable net namespace
742 * @name: name to find
743 *
744 * Find an interface by name.
745 * If the name is found a pointer to the device is returned.
746 * If the name is not found then %NULL is returned.
747 * The reference counters are not incremented so the caller must be
748 * careful with locks. The caller must hold RCU lock.
749 */
750
751struct net_device *dev_get_by_name_rcu(struct net *net, const char *name)
752{
Eric Dumazet72c95282009-10-30 07:11:27 +0000753 struct net_device *dev;
754 struct hlist_head *head = dev_name_hash(net, name);
755
Sasha Levinb67bfe02013-02-27 17:06:00 -0800756 hlist_for_each_entry_rcu(dev, head, name_hlist)
Eric Dumazet72c95282009-10-30 07:11:27 +0000757 if (!strncmp(dev->name, name, IFNAMSIZ))
758 return dev;
759
760 return NULL;
761}
762EXPORT_SYMBOL(dev_get_by_name_rcu);
763
764/**
Linus Torvalds1da177e2005-04-16 15:20:36 -0700765 * dev_get_by_name - find a device by its name
Randy Dunlapc4ea43c2007-10-12 21:17:49 -0700766 * @net: the applicable net namespace
Linus Torvalds1da177e2005-04-16 15:20:36 -0700767 * @name: name to find
768 *
769 * Find an interface by name. This can be called from any
770 * context and does its own locking. The returned handle has
771 * the usage count incremented and the caller must use dev_put() to
772 * release it when it is no longer needed. %NULL is returned if no
773 * matching device is found.
774 */
775
Eric W. Biederman881d9662007-09-17 11:56:21 -0700776struct net_device *dev_get_by_name(struct net *net, const char *name)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700777{
778 struct net_device *dev;
779
Eric Dumazet72c95282009-10-30 07:11:27 +0000780 rcu_read_lock();
781 dev = dev_get_by_name_rcu(net, name);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700782 if (dev)
783 dev_hold(dev);
Eric Dumazet72c95282009-10-30 07:11:27 +0000784 rcu_read_unlock();
Linus Torvalds1da177e2005-04-16 15:20:36 -0700785 return dev;
786}
Eric Dumazetd1b19df2009-09-03 01:29:39 -0700787EXPORT_SYMBOL(dev_get_by_name);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700788
789/**
790 * __dev_get_by_index - find a device by its ifindex
Randy Dunlapc4ea43c2007-10-12 21:17:49 -0700791 * @net: the applicable net namespace
Linus Torvalds1da177e2005-04-16 15:20:36 -0700792 * @ifindex: index of device
793 *
794 * Search for an interface by index. Returns %NULL if the device
795 * is not found or a pointer to the device. The device has not
796 * had its reference counter increased so the caller must be careful
797 * about locking. The caller must hold either the RTNL semaphore
798 * or @dev_base_lock.
799 */
800
Eric W. Biederman881d9662007-09-17 11:56:21 -0700801struct net_device *__dev_get_by_index(struct net *net, int ifindex)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700802{
Eric Dumazet0bd8d532009-10-30 01:40:11 -0700803 struct net_device *dev;
804 struct hlist_head *head = dev_index_hash(net, ifindex);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700805
Sasha Levinb67bfe02013-02-27 17:06:00 -0800806 hlist_for_each_entry(dev, head, index_hlist)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700807 if (dev->ifindex == ifindex)
808 return dev;
Eric Dumazet0bd8d532009-10-30 01:40:11 -0700809
Linus Torvalds1da177e2005-04-16 15:20:36 -0700810 return NULL;
811}
Eric Dumazetd1b19df2009-09-03 01:29:39 -0700812EXPORT_SYMBOL(__dev_get_by_index);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700813
Eric Dumazetfb699dfd2009-10-19 19:18:49 +0000814/**
815 * dev_get_by_index_rcu - find a device by its ifindex
816 * @net: the applicable net namespace
817 * @ifindex: index of device
818 *
819 * Search for an interface by index. Returns %NULL if the device
820 * is not found or a pointer to the device. The device has not
821 * had its reference counter increased so the caller must be careful
822 * about locking. The caller must hold RCU lock.
823 */
824
825struct net_device *dev_get_by_index_rcu(struct net *net, int ifindex)
826{
Eric Dumazetfb699dfd2009-10-19 19:18:49 +0000827 struct net_device *dev;
828 struct hlist_head *head = dev_index_hash(net, ifindex);
829
Sasha Levinb67bfe02013-02-27 17:06:00 -0800830 hlist_for_each_entry_rcu(dev, head, index_hlist)
Eric Dumazetfb699dfd2009-10-19 19:18:49 +0000831 if (dev->ifindex == ifindex)
832 return dev;
833
834 return NULL;
835}
836EXPORT_SYMBOL(dev_get_by_index_rcu);
837
Linus Torvalds1da177e2005-04-16 15:20:36 -0700838
839/**
840 * dev_get_by_index - find a device by its ifindex
Randy Dunlapc4ea43c2007-10-12 21:17:49 -0700841 * @net: the applicable net namespace
Linus Torvalds1da177e2005-04-16 15:20:36 -0700842 * @ifindex: index of device
843 *
844 * Search for an interface by index. Returns NULL if the device
845 * is not found or a pointer to the device. The device returned has
846 * had a reference added and the pointer is safe until the user calls
847 * dev_put to indicate they have finished with it.
848 */
849
Eric W. Biederman881d9662007-09-17 11:56:21 -0700850struct net_device *dev_get_by_index(struct net *net, int ifindex)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700851{
852 struct net_device *dev;
853
Eric Dumazetfb699dfd2009-10-19 19:18:49 +0000854 rcu_read_lock();
855 dev = dev_get_by_index_rcu(net, ifindex);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700856 if (dev)
857 dev_hold(dev);
Eric Dumazetfb699dfd2009-10-19 19:18:49 +0000858 rcu_read_unlock();
Linus Torvalds1da177e2005-04-16 15:20:36 -0700859 return dev;
860}
Eric Dumazetd1b19df2009-09-03 01:29:39 -0700861EXPORT_SYMBOL(dev_get_by_index);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700862
863/**
Nicolas Schichan5dbe7c12013-06-26 17:23:42 +0200864 * netdev_get_name - get a netdevice name, knowing its ifindex.
865 * @net: network namespace
866 * @name: a pointer to the buffer where the name will be stored.
867 * @ifindex: the ifindex of the interface to get the name from.
868 *
869 * The use of raw_seqcount_begin() and cond_resched() before
870 * retrying is required as we want to give the writers a chance
871 * to complete when CONFIG_PREEMPT is not set.
872 */
873int netdev_get_name(struct net *net, char *name, int ifindex)
874{
875 struct net_device *dev;
876 unsigned int seq;
877
878retry:
879 seq = raw_seqcount_begin(&devnet_rename_seq);
880 rcu_read_lock();
881 dev = dev_get_by_index_rcu(net, ifindex);
882 if (!dev) {
883 rcu_read_unlock();
884 return -ENODEV;
885 }
886
887 strcpy(name, dev->name);
888 rcu_read_unlock();
889 if (read_seqcount_retry(&devnet_rename_seq, seq)) {
890 cond_resched();
891 goto retry;
892 }
893
894 return 0;
895}
896
897/**
Eric Dumazet941666c2010-12-05 01:23:53 +0000898 * dev_getbyhwaddr_rcu - find a device by its hardware address
Randy Dunlapc4ea43c2007-10-12 21:17:49 -0700899 * @net: the applicable net namespace
Linus Torvalds1da177e2005-04-16 15:20:36 -0700900 * @type: media type of device
901 * @ha: hardware address
902 *
903 * Search for an interface by MAC address. Returns NULL if the device
Eric Dumazetc5066532011-01-24 13:16:16 -0800904 * is not found or a pointer to the device.
905 * The caller must hold RCU or RTNL.
Eric Dumazet941666c2010-12-05 01:23:53 +0000906 * The returned device has not had its ref count increased
Linus Torvalds1da177e2005-04-16 15:20:36 -0700907 * and the caller must therefore be careful about locking
908 *
Linus Torvalds1da177e2005-04-16 15:20:36 -0700909 */
910
Eric Dumazet941666c2010-12-05 01:23:53 +0000911struct net_device *dev_getbyhwaddr_rcu(struct net *net, unsigned short type,
912 const char *ha)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700913{
914 struct net_device *dev;
915
Eric Dumazet941666c2010-12-05 01:23:53 +0000916 for_each_netdev_rcu(net, dev)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700917 if (dev->type == type &&
918 !memcmp(dev->dev_addr, ha, dev->addr_len))
Pavel Emelianov7562f872007-05-03 15:13:45 -0700919 return dev;
920
921 return NULL;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700922}
Eric Dumazet941666c2010-12-05 01:23:53 +0000923EXPORT_SYMBOL(dev_getbyhwaddr_rcu);
Jochen Friedrichcf309e32005-09-22 04:44:55 -0300924
Eric W. Biederman881d9662007-09-17 11:56:21 -0700925struct net_device *__dev_getfirstbyhwtype(struct net *net, unsigned short type)
Patrick McHardy4e9cac22007-05-03 03:28:13 -0700926{
927 struct net_device *dev;
928
929 ASSERT_RTNL();
Eric W. Biederman881d9662007-09-17 11:56:21 -0700930 for_each_netdev(net, dev)
Patrick McHardy4e9cac22007-05-03 03:28:13 -0700931 if (dev->type == type)
Pavel Emelianov7562f872007-05-03 15:13:45 -0700932 return dev;
933
934 return NULL;
Patrick McHardy4e9cac22007-05-03 03:28:13 -0700935}
Patrick McHardy4e9cac22007-05-03 03:28:13 -0700936EXPORT_SYMBOL(__dev_getfirstbyhwtype);
937
Eric W. Biederman881d9662007-09-17 11:56:21 -0700938struct net_device *dev_getfirstbyhwtype(struct net *net, unsigned short type)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700939{
Eric Dumazet99fe3c32010-03-18 11:27:25 +0000940 struct net_device *dev, *ret = NULL;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700941
Eric Dumazet99fe3c32010-03-18 11:27:25 +0000942 rcu_read_lock();
943 for_each_netdev_rcu(net, dev)
944 if (dev->type == type) {
945 dev_hold(dev);
946 ret = dev;
947 break;
948 }
949 rcu_read_unlock();
950 return ret;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700951}
Linus Torvalds1da177e2005-04-16 15:20:36 -0700952EXPORT_SYMBOL(dev_getfirstbyhwtype);
953
954/**
WANG Cong6c555492014-09-11 15:35:09 -0700955 * __dev_get_by_flags - find any device with given flags
Randy Dunlapc4ea43c2007-10-12 21:17:49 -0700956 * @net: the applicable net namespace
Linus Torvalds1da177e2005-04-16 15:20:36 -0700957 * @if_flags: IFF_* values
958 * @mask: bitmask of bits in if_flags to check
959 *
960 * Search for any interface with the given flags. Returns NULL if a device
Eric Dumazetbb69ae02010-06-07 11:42:13 +0000961 * is not found or a pointer to the device. Must be called inside
WANG Cong6c555492014-09-11 15:35:09 -0700962 * rtnl_lock(), and result refcount is unchanged.
Linus Torvalds1da177e2005-04-16 15:20:36 -0700963 */
964
WANG Cong6c555492014-09-11 15:35:09 -0700965struct net_device *__dev_get_by_flags(struct net *net, unsigned short if_flags,
966 unsigned short mask)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700967{
Pavel Emelianov7562f872007-05-03 15:13:45 -0700968 struct net_device *dev, *ret;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700969
WANG Cong6c555492014-09-11 15:35:09 -0700970 ASSERT_RTNL();
971
Pavel Emelianov7562f872007-05-03 15:13:45 -0700972 ret = NULL;
WANG Cong6c555492014-09-11 15:35:09 -0700973 for_each_netdev(net, dev) {
Linus Torvalds1da177e2005-04-16 15:20:36 -0700974 if (((dev->flags ^ if_flags) & mask) == 0) {
Pavel Emelianov7562f872007-05-03 15:13:45 -0700975 ret = dev;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700976 break;
977 }
978 }
Pavel Emelianov7562f872007-05-03 15:13:45 -0700979 return ret;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700980}
WANG Cong6c555492014-09-11 15:35:09 -0700981EXPORT_SYMBOL(__dev_get_by_flags);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700982
983/**
984 * dev_valid_name - check if name is okay for network device
985 * @name: name string
986 *
987 * Network device names need to be valid file names to
David S. Millerc7fa9d12006-08-15 16:34:13 -0700988 * to allow sysfs to work. We also disallow any kind of
989 * whitespace.
Linus Torvalds1da177e2005-04-16 15:20:36 -0700990 */
David S. Miller95f050b2012-03-06 16:12:15 -0500991bool dev_valid_name(const char *name)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700992{
David S. Millerc7fa9d12006-08-15 16:34:13 -0700993 if (*name == '\0')
David S. Miller95f050b2012-03-06 16:12:15 -0500994 return false;
Stephen Hemmingerb6fe17d2006-08-29 17:06:13 -0700995 if (strlen(name) >= IFNAMSIZ)
David S. Miller95f050b2012-03-06 16:12:15 -0500996 return false;
David S. Millerc7fa9d12006-08-15 16:34:13 -0700997 if (!strcmp(name, ".") || !strcmp(name, ".."))
David S. Miller95f050b2012-03-06 16:12:15 -0500998 return false;
David S. Millerc7fa9d12006-08-15 16:34:13 -0700999
1000 while (*name) {
Matthew Thodea4176a92015-02-17 18:31:57 -06001001 if (*name == '/' || *name == ':' || isspace(*name))
David S. Miller95f050b2012-03-06 16:12:15 -05001002 return false;
David S. Millerc7fa9d12006-08-15 16:34:13 -07001003 name++;
1004 }
David S. Miller95f050b2012-03-06 16:12:15 -05001005 return true;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001006}
Eric Dumazetd1b19df2009-09-03 01:29:39 -07001007EXPORT_SYMBOL(dev_valid_name);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001008
1009/**
Eric W. Biedermanb267b172007-09-12 13:48:45 +02001010 * __dev_alloc_name - allocate a name for a device
1011 * @net: network namespace to allocate the device name in
Linus Torvalds1da177e2005-04-16 15:20:36 -07001012 * @name: name format string
Eric W. Biedermanb267b172007-09-12 13:48:45 +02001013 * @buf: scratch buffer and result name string
Linus Torvalds1da177e2005-04-16 15:20:36 -07001014 *
1015 * Passed a format string - eg "lt%d" it will try and find a suitable
Stephen Hemminger3041a062006-05-26 13:25:24 -07001016 * id. It scans list of devices to build up a free map, then chooses
1017 * the first empty slot. The caller must hold the dev_base or rtnl lock
1018 * while allocating the name and adding the device in order to avoid
1019 * duplicates.
1020 * Limited to bits_per_byte * page size devices (ie 32K on most platforms).
1021 * Returns the number of the unit assigned or a negative errno code.
Linus Torvalds1da177e2005-04-16 15:20:36 -07001022 */
1023
Eric W. Biedermanb267b172007-09-12 13:48:45 +02001024static int __dev_alloc_name(struct net *net, const char *name, char *buf)
Linus Torvalds1da177e2005-04-16 15:20:36 -07001025{
1026 int i = 0;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001027 const char *p;
1028 const int max_netdevices = 8*PAGE_SIZE;
Stephen Hemmingercfcabdc2007-10-09 01:59:42 -07001029 unsigned long *inuse;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001030 struct net_device *d;
1031
1032 p = strnchr(name, IFNAMSIZ-1, '%');
1033 if (p) {
1034 /*
1035 * Verify the string as this thing may have come from
1036 * the user. There must be either one "%d" and no other "%"
1037 * characters.
1038 */
1039 if (p[1] != 'd' || strchr(p + 2, '%'))
1040 return -EINVAL;
1041
1042 /* Use one page as a bit array of possible slots */
Stephen Hemmingercfcabdc2007-10-09 01:59:42 -07001043 inuse = (unsigned long *) get_zeroed_page(GFP_ATOMIC);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001044 if (!inuse)
1045 return -ENOMEM;
1046
Eric W. Biederman881d9662007-09-17 11:56:21 -07001047 for_each_netdev(net, d) {
Linus Torvalds1da177e2005-04-16 15:20:36 -07001048 if (!sscanf(d->name, name, &i))
1049 continue;
1050 if (i < 0 || i >= max_netdevices)
1051 continue;
1052
1053 /* avoid cases where sscanf is not exact inverse of printf */
Eric W. Biedermanb267b172007-09-12 13:48:45 +02001054 snprintf(buf, IFNAMSIZ, name, i);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001055 if (!strncmp(buf, d->name, IFNAMSIZ))
1056 set_bit(i, inuse);
1057 }
1058
1059 i = find_first_zero_bit(inuse, max_netdevices);
1060 free_page((unsigned long) inuse);
1061 }
1062
Octavian Purdilad9031022009-11-18 02:36:59 +00001063 if (buf != name)
1064 snprintf(buf, IFNAMSIZ, name, i);
Eric W. Biedermanb267b172007-09-12 13:48:45 +02001065 if (!__dev_get_by_name(net, buf))
Linus Torvalds1da177e2005-04-16 15:20:36 -07001066 return i;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001067
1068 /* It is possible to run out of possible slots
1069 * when the name is long and there isn't enough space left
1070 * for the digits, or if all bits are used.
1071 */
1072 return -ENFILE;
1073}
1074
Eric W. Biedermanb267b172007-09-12 13:48:45 +02001075/**
1076 * dev_alloc_name - allocate a name for a device
1077 * @dev: device
1078 * @name: name format string
1079 *
1080 * Passed a format string - eg "lt%d" it will try and find a suitable
1081 * id. It scans list of devices to build up a free map, then chooses
1082 * the first empty slot. The caller must hold the dev_base or rtnl lock
1083 * while allocating the name and adding the device in order to avoid
1084 * duplicates.
1085 * Limited to bits_per_byte * page size devices (ie 32K on most platforms).
1086 * Returns the number of the unit assigned or a negative errno code.
1087 */
1088
1089int dev_alloc_name(struct net_device *dev, const char *name)
1090{
1091 char buf[IFNAMSIZ];
1092 struct net *net;
1093 int ret;
1094
YOSHIFUJI Hideakic346dca2008-03-25 21:47:49 +09001095 BUG_ON(!dev_net(dev));
1096 net = dev_net(dev);
Eric W. Biedermanb267b172007-09-12 13:48:45 +02001097 ret = __dev_alloc_name(net, name, buf);
1098 if (ret >= 0)
1099 strlcpy(dev->name, buf, IFNAMSIZ);
1100 return ret;
1101}
Eric Dumazetd1b19df2009-09-03 01:29:39 -07001102EXPORT_SYMBOL(dev_alloc_name);
Eric W. Biedermanb267b172007-09-12 13:48:45 +02001103
Gao feng828de4f2012-09-13 20:58:27 +00001104static int dev_alloc_name_ns(struct net *net,
1105 struct net_device *dev,
1106 const char *name)
Octavian Purdilad9031022009-11-18 02:36:59 +00001107{
Gao feng828de4f2012-09-13 20:58:27 +00001108 char buf[IFNAMSIZ];
1109 int ret;
Daniel Lezcano8ce6cebc2010-05-19 10:12:19 +00001110
Gao feng828de4f2012-09-13 20:58:27 +00001111 ret = __dev_alloc_name(net, name, buf);
1112 if (ret >= 0)
1113 strlcpy(dev->name, buf, IFNAMSIZ);
1114 return ret;
1115}
1116
1117static int dev_get_valid_name(struct net *net,
1118 struct net_device *dev,
1119 const char *name)
1120{
1121 BUG_ON(!net);
Daniel Lezcano8ce6cebc2010-05-19 10:12:19 +00001122
Octavian Purdilad9031022009-11-18 02:36:59 +00001123 if (!dev_valid_name(name))
1124 return -EINVAL;
1125
Jiri Pirko1c5cae82011-04-30 01:21:32 +00001126 if (strchr(name, '%'))
Gao feng828de4f2012-09-13 20:58:27 +00001127 return dev_alloc_name_ns(net, dev, name);
Octavian Purdilad9031022009-11-18 02:36:59 +00001128 else if (__dev_get_by_name(net, name))
1129 return -EEXIST;
Daniel Lezcano8ce6cebc2010-05-19 10:12:19 +00001130 else if (dev->name != name)
1131 strlcpy(dev->name, name, IFNAMSIZ);
Octavian Purdilad9031022009-11-18 02:36:59 +00001132
1133 return 0;
1134}
Linus Torvalds1da177e2005-04-16 15:20:36 -07001135
1136/**
1137 * dev_change_name - change name of a device
1138 * @dev: device
1139 * @newname: name (or format string) must be at least IFNAMSIZ
1140 *
1141 * Change name of a device, can pass format strings "eth%d".
1142 * for wildcarding.
1143 */
Stephen Hemmingercf04a4c72008-09-30 02:22:14 -07001144int dev_change_name(struct net_device *dev, const char *newname)
Linus Torvalds1da177e2005-04-16 15:20:36 -07001145{
Tom Gundersen238fa362014-07-14 16:37:23 +02001146 unsigned char old_assign_type;
Herbert Xufcc5a032007-07-30 17:03:38 -07001147 char oldname[IFNAMSIZ];
Linus Torvalds1da177e2005-04-16 15:20:36 -07001148 int err = 0;
Herbert Xufcc5a032007-07-30 17:03:38 -07001149 int ret;
Eric W. Biederman881d9662007-09-17 11:56:21 -07001150 struct net *net;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001151
1152 ASSERT_RTNL();
YOSHIFUJI Hideakic346dca2008-03-25 21:47:49 +09001153 BUG_ON(!dev_net(dev));
Linus Torvalds1da177e2005-04-16 15:20:36 -07001154
YOSHIFUJI Hideakic346dca2008-03-25 21:47:49 +09001155 net = dev_net(dev);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001156 if (dev->flags & IFF_UP)
1157 return -EBUSY;
1158
Eric Dumazet30e6c9f2012-12-20 17:25:08 +00001159 write_seqcount_begin(&devnet_rename_seq);
Brian Haleyc91f6df2012-11-26 05:21:08 +00001160
1161 if (strncmp(newname, dev->name, IFNAMSIZ) == 0) {
Eric Dumazet30e6c9f2012-12-20 17:25:08 +00001162 write_seqcount_end(&devnet_rename_seq);
Stephen Hemmingerc8d90dc2007-10-26 03:53:42 -07001163 return 0;
Brian Haleyc91f6df2012-11-26 05:21:08 +00001164 }
Stephen Hemmingerc8d90dc2007-10-26 03:53:42 -07001165
Herbert Xufcc5a032007-07-30 17:03:38 -07001166 memcpy(oldname, dev->name, IFNAMSIZ);
1167
Gao feng828de4f2012-09-13 20:58:27 +00001168 err = dev_get_valid_name(net, dev, newname);
Brian Haleyc91f6df2012-11-26 05:21:08 +00001169 if (err < 0) {
Eric Dumazet30e6c9f2012-12-20 17:25:08 +00001170 write_seqcount_end(&devnet_rename_seq);
Octavian Purdilad9031022009-11-18 02:36:59 +00001171 return err;
Brian Haleyc91f6df2012-11-26 05:21:08 +00001172 }
Linus Torvalds1da177e2005-04-16 15:20:36 -07001173
Veaceslav Falico6fe82a32014-07-17 20:33:32 +02001174 if (oldname[0] && !strchr(oldname, '%'))
1175 netdev_info(dev, "renamed from %s\n", oldname);
1176
Tom Gundersen238fa362014-07-14 16:37:23 +02001177 old_assign_type = dev->name_assign_type;
1178 dev->name_assign_type = NET_NAME_RENAMED;
1179
Herbert Xufcc5a032007-07-30 17:03:38 -07001180rollback:
Eric W. Biedermana1b3f592010-05-04 17:36:49 -07001181 ret = device_rename(&dev->dev, dev->name);
1182 if (ret) {
1183 memcpy(dev->name, oldname, IFNAMSIZ);
Tom Gundersen238fa362014-07-14 16:37:23 +02001184 dev->name_assign_type = old_assign_type;
Eric Dumazet30e6c9f2012-12-20 17:25:08 +00001185 write_seqcount_end(&devnet_rename_seq);
Eric W. Biedermana1b3f592010-05-04 17:36:49 -07001186 return ret;
Stephen Hemmingerdcc99772008-05-14 22:33:38 -07001187 }
Herbert Xu7f988ea2007-07-30 16:35:46 -07001188
Eric Dumazet30e6c9f2012-12-20 17:25:08 +00001189 write_seqcount_end(&devnet_rename_seq);
Brian Haleyc91f6df2012-11-26 05:21:08 +00001190
Veaceslav Falico5bb025f2014-01-14 21:58:51 +01001191 netdev_adjacent_rename_links(dev, oldname);
1192
Herbert Xu7f988ea2007-07-30 16:35:46 -07001193 write_lock_bh(&dev_base_lock);
Eric Dumazet372b2312011-05-17 13:56:59 -04001194 hlist_del_rcu(&dev->name_hlist);
Eric Dumazet72c95282009-10-30 07:11:27 +00001195 write_unlock_bh(&dev_base_lock);
1196
1197 synchronize_rcu();
1198
1199 write_lock_bh(&dev_base_lock);
1200 hlist_add_head_rcu(&dev->name_hlist, dev_name_hash(net, dev->name));
Herbert Xu7f988ea2007-07-30 16:35:46 -07001201 write_unlock_bh(&dev_base_lock);
1202
Pavel Emelyanov056925a2007-09-16 15:42:43 -07001203 ret = call_netdevice_notifiers(NETDEV_CHANGENAME, dev);
Herbert Xufcc5a032007-07-30 17:03:38 -07001204 ret = notifier_to_errno(ret);
1205
1206 if (ret) {
Eric Dumazet91e9c07b2009-11-15 23:30:24 +00001207 /* err >= 0 after dev_alloc_name() or stores the first errno */
1208 if (err >= 0) {
Herbert Xufcc5a032007-07-30 17:03:38 -07001209 err = ret;
Eric Dumazet30e6c9f2012-12-20 17:25:08 +00001210 write_seqcount_begin(&devnet_rename_seq);
Herbert Xufcc5a032007-07-30 17:03:38 -07001211 memcpy(dev->name, oldname, IFNAMSIZ);
Veaceslav Falico5bb025f2014-01-14 21:58:51 +01001212 memcpy(oldname, newname, IFNAMSIZ);
Tom Gundersen238fa362014-07-14 16:37:23 +02001213 dev->name_assign_type = old_assign_type;
1214 old_assign_type = NET_NAME_RENAMED;
Herbert Xufcc5a032007-07-30 17:03:38 -07001215 goto rollback;
Eric Dumazet91e9c07b2009-11-15 23:30:24 +00001216 } else {
Joe Perches7b6cd1c2012-02-01 10:54:43 +00001217 pr_err("%s: name change rollback failed: %d\n",
Eric Dumazet91e9c07b2009-11-15 23:30:24 +00001218 dev->name, ret);
Herbert Xufcc5a032007-07-30 17:03:38 -07001219 }
1220 }
Linus Torvalds1da177e2005-04-16 15:20:36 -07001221
1222 return err;
1223}
1224
1225/**
Stephen Hemminger0b815a12008-09-22 21:28:11 -07001226 * dev_set_alias - change ifalias of a device
1227 * @dev: device
1228 * @alias: name up to IFALIASZ
Stephen Hemmingerf0db2752008-09-30 02:23:58 -07001229 * @len: limit of bytes to copy from info
Stephen Hemminger0b815a12008-09-22 21:28:11 -07001230 *
1231 * Set ifalias for a device,
1232 */
1233int dev_set_alias(struct net_device *dev, const char *alias, size_t len)
1234{
Alexey Khoroshilov7364e442012-08-08 00:33:25 +00001235 char *new_ifalias;
1236
Stephen Hemminger0b815a12008-09-22 21:28:11 -07001237 ASSERT_RTNL();
1238
1239 if (len >= IFALIASZ)
1240 return -EINVAL;
1241
Oliver Hartkopp96ca4a22008-09-23 21:23:19 -07001242 if (!len) {
Sachin Kamat388dfc22012-11-20 00:57:04 +00001243 kfree(dev->ifalias);
1244 dev->ifalias = NULL;
Oliver Hartkopp96ca4a22008-09-23 21:23:19 -07001245 return 0;
1246 }
1247
Alexey Khoroshilov7364e442012-08-08 00:33:25 +00001248 new_ifalias = krealloc(dev->ifalias, len + 1, GFP_KERNEL);
1249 if (!new_ifalias)
Stephen Hemminger0b815a12008-09-22 21:28:11 -07001250 return -ENOMEM;
Alexey Khoroshilov7364e442012-08-08 00:33:25 +00001251 dev->ifalias = new_ifalias;
Stephen Hemminger0b815a12008-09-22 21:28:11 -07001252
1253 strlcpy(dev->ifalias, alias, len+1);
1254 return len;
1255}
1256
1257
1258/**
Stephen Hemminger3041a062006-05-26 13:25:24 -07001259 * netdev_features_change - device changes features
Stephen Hemmingerd8a33ac2005-05-29 14:13:47 -07001260 * @dev: device to cause notification
1261 *
1262 * Called to indicate a device has changed features.
1263 */
1264void netdev_features_change(struct net_device *dev)
1265{
Pavel Emelyanov056925a2007-09-16 15:42:43 -07001266 call_netdevice_notifiers(NETDEV_FEAT_CHANGE, dev);
Stephen Hemmingerd8a33ac2005-05-29 14:13:47 -07001267}
1268EXPORT_SYMBOL(netdev_features_change);
1269
1270/**
Linus Torvalds1da177e2005-04-16 15:20:36 -07001271 * netdev_state_change - device changes state
1272 * @dev: device to cause notification
1273 *
1274 * Called to indicate a device has changed state. This function calls
1275 * the notifier chains for netdev_chain and sends a NEWLINK message
1276 * to the routing socket.
1277 */
1278void netdev_state_change(struct net_device *dev)
1279{
1280 if (dev->flags & IFF_UP) {
Loic Prylli54951192014-07-01 21:39:43 -07001281 struct netdev_notifier_change_info change_info;
1282
1283 change_info.flags_changed = 0;
1284 call_netdevice_notifiers_info(NETDEV_CHANGE, dev,
1285 &change_info.info);
Alexei Starovoitov7f294052013-10-23 16:02:42 -07001286 rtmsg_ifinfo(RTM_NEWLINK, dev, 0, GFP_KERNEL);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001287 }
1288}
Eric Dumazetd1b19df2009-09-03 01:29:39 -07001289EXPORT_SYMBOL(netdev_state_change);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001290
Amerigo Wangee89bab2012-08-09 22:14:56 +00001291/**
1292 * netdev_notify_peers - notify network peers about existence of @dev
1293 * @dev: network device
1294 *
1295 * Generate traffic such that interested network peers are aware of
1296 * @dev, such as by generating a gratuitous ARP. This may be used when
1297 * a device wants to inform the rest of the network about some sort of
1298 * reconfiguration such as a failover event or virtual machine
1299 * migration.
1300 */
1301void netdev_notify_peers(struct net_device *dev)
Or Gerlitzc1da4ac2008-06-13 18:12:00 -07001302{
Amerigo Wangee89bab2012-08-09 22:14:56 +00001303 rtnl_lock();
1304 call_netdevice_notifiers(NETDEV_NOTIFY_PEERS, dev);
1305 rtnl_unlock();
Or Gerlitzc1da4ac2008-06-13 18:12:00 -07001306}
Amerigo Wangee89bab2012-08-09 22:14:56 +00001307EXPORT_SYMBOL(netdev_notify_peers);
Or Gerlitzc1da4ac2008-06-13 18:12:00 -07001308
Patrick McHardybd380812010-02-26 06:34:53 +00001309static int __dev_open(struct net_device *dev)
Linus Torvalds1da177e2005-04-16 15:20:36 -07001310{
Stephen Hemmingerd3147742008-11-19 21:32:24 -08001311 const struct net_device_ops *ops = dev->netdev_ops;
Johannes Berg3b8bcfd2009-05-30 01:39:53 +02001312 int ret;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001313
Ben Hutchingse46b66b2008-05-08 02:53:17 -07001314 ASSERT_RTNL();
1315
Linus Torvalds1da177e2005-04-16 15:20:36 -07001316 if (!netif_device_present(dev))
1317 return -ENODEV;
1318
Neil Hormanca99ca12013-02-05 08:05:43 +00001319 /* Block netpoll from trying to do any rx path servicing.
1320 * If we don't do this there is a chance ndo_poll_controller
1321 * or ndo_poll may be running while we open the device
1322 */
Eric W. Biederman66b55522014-03-27 15:39:03 -07001323 netpoll_poll_disable(dev);
Neil Hormanca99ca12013-02-05 08:05:43 +00001324
Johannes Berg3b8bcfd2009-05-30 01:39:53 +02001325 ret = call_netdevice_notifiers(NETDEV_PRE_UP, dev);
1326 ret = notifier_to_errno(ret);
1327 if (ret)
1328 return ret;
1329
Linus Torvalds1da177e2005-04-16 15:20:36 -07001330 set_bit(__LINK_STATE_START, &dev->state);
Jeff Garzikbada3392007-10-23 20:19:37 -07001331
Stephen Hemmingerd3147742008-11-19 21:32:24 -08001332 if (ops->ndo_validate_addr)
1333 ret = ops->ndo_validate_addr(dev);
Jeff Garzikbada3392007-10-23 20:19:37 -07001334
Stephen Hemmingerd3147742008-11-19 21:32:24 -08001335 if (!ret && ops->ndo_open)
1336 ret = ops->ndo_open(dev);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001337
Eric W. Biederman66b55522014-03-27 15:39:03 -07001338 netpoll_poll_enable(dev);
Neil Hormanca99ca12013-02-05 08:05:43 +00001339
Jeff Garzikbada3392007-10-23 20:19:37 -07001340 if (ret)
1341 clear_bit(__LINK_STATE_START, &dev->state);
1342 else {
Linus Torvalds1da177e2005-04-16 15:20:36 -07001343 dev->flags |= IFF_UP;
Patrick McHardy4417da62007-06-27 01:28:10 -07001344 dev_set_rx_mode(dev);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001345 dev_activate(dev);
Theodore Ts'o7bf23572012-07-04 21:23:25 -04001346 add_device_randomness(dev->dev_addr, dev->addr_len);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001347 }
Jeff Garzikbada3392007-10-23 20:19:37 -07001348
Linus Torvalds1da177e2005-04-16 15:20:36 -07001349 return ret;
1350}
Patrick McHardybd380812010-02-26 06:34:53 +00001351
1352/**
1353 * dev_open - prepare an interface for use.
1354 * @dev: device to open
1355 *
1356 * Takes a device from down to up state. The device's private open
1357 * function is invoked and then the multicast lists are loaded. Finally
1358 * the device is moved into the up state and a %NETDEV_UP message is
1359 * sent to the netdev notifier chain.
1360 *
1361 * Calling this function on an active interface is a nop. On a failure
1362 * a negative errno code is returned.
1363 */
1364int dev_open(struct net_device *dev)
1365{
1366 int ret;
1367
Patrick McHardybd380812010-02-26 06:34:53 +00001368 if (dev->flags & IFF_UP)
1369 return 0;
1370
Patrick McHardybd380812010-02-26 06:34:53 +00001371 ret = __dev_open(dev);
1372 if (ret < 0)
1373 return ret;
1374
Alexei Starovoitov7f294052013-10-23 16:02:42 -07001375 rtmsg_ifinfo(RTM_NEWLINK, dev, IFF_UP|IFF_RUNNING, GFP_KERNEL);
Patrick McHardybd380812010-02-26 06:34:53 +00001376 call_netdevice_notifiers(NETDEV_UP, dev);
1377
1378 return ret;
1379}
Eric Dumazetd1b19df2009-09-03 01:29:39 -07001380EXPORT_SYMBOL(dev_open);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001381
Octavian Purdila44345722010-12-13 12:44:07 +00001382static int __dev_close_many(struct list_head *head)
Linus Torvalds1da177e2005-04-16 15:20:36 -07001383{
Octavian Purdila44345722010-12-13 12:44:07 +00001384 struct net_device *dev;
Patrick McHardybd380812010-02-26 06:34:53 +00001385
Ben Hutchingse46b66b2008-05-08 02:53:17 -07001386 ASSERT_RTNL();
David S. Miller9d5010d2007-09-12 14:33:25 +02001387 might_sleep();
1388
Eric W. Biederman5cde2822013-10-05 19:26:05 -07001389 list_for_each_entry(dev, head, close_list) {
Eric W. Biederman3f4df202014-03-27 15:38:17 -07001390 /* Temporarily disable netpoll until the interface is down */
Eric W. Biederman66b55522014-03-27 15:39:03 -07001391 netpoll_poll_disable(dev);
Eric W. Biederman3f4df202014-03-27 15:38:17 -07001392
Octavian Purdila44345722010-12-13 12:44:07 +00001393 call_netdevice_notifiers(NETDEV_GOING_DOWN, dev);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001394
Octavian Purdila44345722010-12-13 12:44:07 +00001395 clear_bit(__LINK_STATE_START, &dev->state);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001396
Octavian Purdila44345722010-12-13 12:44:07 +00001397 /* Synchronize to scheduled poll. We cannot touch poll list, it
1398 * can be even on different cpu. So just clear netif_running().
1399 *
1400 * dev->stop() will invoke napi_disable() on all of it's
1401 * napi_struct instances on this device.
1402 */
Peter Zijlstra4e857c52014-03-17 18:06:10 +01001403 smp_mb__after_atomic(); /* Commit netif_running(). */
Octavian Purdila44345722010-12-13 12:44:07 +00001404 }
Linus Torvalds1da177e2005-04-16 15:20:36 -07001405
Octavian Purdila44345722010-12-13 12:44:07 +00001406 dev_deactivate_many(head);
1407
Eric W. Biederman5cde2822013-10-05 19:26:05 -07001408 list_for_each_entry(dev, head, close_list) {
Octavian Purdila44345722010-12-13 12:44:07 +00001409 const struct net_device_ops *ops = dev->netdev_ops;
1410
1411 /*
1412 * Call the device specific close. This cannot fail.
1413 * Only if device is UP
1414 *
1415 * We allow it to be called even after a DETACH hot-plug
1416 * event.
1417 */
1418 if (ops->ndo_stop)
1419 ops->ndo_stop(dev);
1420
Octavian Purdila44345722010-12-13 12:44:07 +00001421 dev->flags &= ~IFF_UP;
Eric W. Biederman66b55522014-03-27 15:39:03 -07001422 netpoll_poll_enable(dev);
Octavian Purdila44345722010-12-13 12:44:07 +00001423 }
1424
1425 return 0;
1426}
1427
1428static int __dev_close(struct net_device *dev)
1429{
Linus Torvaldsf87e6f42011-02-17 22:54:38 +00001430 int retval;
Octavian Purdila44345722010-12-13 12:44:07 +00001431 LIST_HEAD(single);
1432
Eric W. Biederman5cde2822013-10-05 19:26:05 -07001433 list_add(&dev->close_list, &single);
Linus Torvaldsf87e6f42011-02-17 22:54:38 +00001434 retval = __dev_close_many(&single);
1435 list_del(&single);
Neil Hormanca99ca12013-02-05 08:05:43 +00001436
Linus Torvaldsf87e6f42011-02-17 22:54:38 +00001437 return retval;
Octavian Purdila44345722010-12-13 12:44:07 +00001438}
1439
David S. Miller99c4a262015-03-18 22:52:33 -04001440int dev_close_many(struct list_head *head, bool unlink)
Octavian Purdila44345722010-12-13 12:44:07 +00001441{
1442 struct net_device *dev, *tmp;
Octavian Purdila44345722010-12-13 12:44:07 +00001443
Eric W. Biederman5cde2822013-10-05 19:26:05 -07001444 /* Remove the devices that don't need to be closed */
1445 list_for_each_entry_safe(dev, tmp, head, close_list)
Octavian Purdila44345722010-12-13 12:44:07 +00001446 if (!(dev->flags & IFF_UP))
Eric W. Biederman5cde2822013-10-05 19:26:05 -07001447 list_del_init(&dev->close_list);
Octavian Purdila44345722010-12-13 12:44:07 +00001448
1449 __dev_close_many(head);
Matti Linnanvuorid8b2a4d2008-02-12 23:10:11 -08001450
Eric W. Biederman5cde2822013-10-05 19:26:05 -07001451 list_for_each_entry_safe(dev, tmp, head, close_list) {
Alexei Starovoitov7f294052013-10-23 16:02:42 -07001452 rtmsg_ifinfo(RTM_NEWLINK, dev, IFF_UP|IFF_RUNNING, GFP_KERNEL);
Octavian Purdila44345722010-12-13 12:44:07 +00001453 call_netdevice_notifiers(NETDEV_DOWN, dev);
David S. Miller99c4a262015-03-18 22:52:33 -04001454 if (unlink)
1455 list_del_init(&dev->close_list);
Octavian Purdila44345722010-12-13 12:44:07 +00001456 }
Linus Torvalds1da177e2005-04-16 15:20:36 -07001457
Linus Torvalds1da177e2005-04-16 15:20:36 -07001458 return 0;
1459}
David S. Miller99c4a262015-03-18 22:52:33 -04001460EXPORT_SYMBOL(dev_close_many);
Patrick McHardybd380812010-02-26 06:34:53 +00001461
1462/**
1463 * dev_close - shutdown an interface.
1464 * @dev: device to shutdown
1465 *
1466 * This function moves an active device into down state. A
1467 * %NETDEV_GOING_DOWN is sent to the netdev notifier chain. The device
1468 * is then deactivated and finally a %NETDEV_DOWN is sent to the notifier
1469 * chain.
1470 */
1471int dev_close(struct net_device *dev)
1472{
Eric Dumazete14a5992011-05-10 12:26:06 -07001473 if (dev->flags & IFF_UP) {
1474 LIST_HEAD(single);
Patrick McHardybd380812010-02-26 06:34:53 +00001475
Eric W. Biederman5cde2822013-10-05 19:26:05 -07001476 list_add(&dev->close_list, &single);
David S. Miller99c4a262015-03-18 22:52:33 -04001477 dev_close_many(&single, true);
Eric Dumazete14a5992011-05-10 12:26:06 -07001478 list_del(&single);
1479 }
dingtianhongda6e3782013-05-27 19:53:31 +00001480 return 0;
Patrick McHardybd380812010-02-26 06:34:53 +00001481}
Eric Dumazetd1b19df2009-09-03 01:29:39 -07001482EXPORT_SYMBOL(dev_close);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001483
1484
Ben Hutchings0187bdf2008-06-19 16:15:47 -07001485/**
1486 * dev_disable_lro - disable Large Receive Offload on a device
1487 * @dev: device
1488 *
1489 * Disable Large Receive Offload (LRO) on a net device. Must be
1490 * called under RTNL. This is needed if received packets may be
1491 * forwarded to another interface.
1492 */
1493void dev_disable_lro(struct net_device *dev)
1494{
Michal Kubečekfbe168b2014-11-13 07:54:50 +01001495 struct net_device *lower_dev;
1496 struct list_head *iter;
Michal Kubeček529d0482013-11-15 06:18:50 +01001497
Michał Mirosławbc5787c62011-11-15 15:29:55 +00001498 dev->wanted_features &= ~NETIF_F_LRO;
1499 netdev_update_features(dev);
Michał Mirosław27660512011-03-18 16:56:34 +00001500
Michał Mirosław22d59692011-04-21 12:42:15 +00001501 if (unlikely(dev->features & NETIF_F_LRO))
1502 netdev_WARN(dev, "failed to disable LRO!\n");
Michal Kubečekfbe168b2014-11-13 07:54:50 +01001503
1504 netdev_for_each_lower_dev(dev, lower_dev, iter)
1505 dev_disable_lro(lower_dev);
Ben Hutchings0187bdf2008-06-19 16:15:47 -07001506}
1507EXPORT_SYMBOL(dev_disable_lro);
1508
Jiri Pirko351638e2013-05-28 01:30:21 +00001509static int call_netdevice_notifier(struct notifier_block *nb, unsigned long val,
1510 struct net_device *dev)
1511{
1512 struct netdev_notifier_info info;
1513
1514 netdev_notifier_info_init(&info, dev);
1515 return nb->notifier_call(nb, val, &info);
1516}
Ben Hutchings0187bdf2008-06-19 16:15:47 -07001517
Eric W. Biederman881d9662007-09-17 11:56:21 -07001518static int dev_boot_phase = 1;
1519
Linus Torvalds1da177e2005-04-16 15:20:36 -07001520/**
1521 * register_netdevice_notifier - register a network notifier block
1522 * @nb: notifier
1523 *
1524 * Register a notifier to be called when network device events occur.
1525 * The notifier passed is linked into the kernel structures and must
1526 * not be reused until it has been unregistered. A negative errno code
1527 * is returned on a failure.
1528 *
1529 * When registered all registration and up events are replayed
YOSHIFUJI Hideaki4ec93ed2007-02-09 23:24:36 +09001530 * to the new notifier to allow device to have a race free
Linus Torvalds1da177e2005-04-16 15:20:36 -07001531 * view of the network device list.
1532 */
1533
1534int register_netdevice_notifier(struct notifier_block *nb)
1535{
1536 struct net_device *dev;
Herbert Xufcc5a032007-07-30 17:03:38 -07001537 struct net_device *last;
Eric W. Biederman881d9662007-09-17 11:56:21 -07001538 struct net *net;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001539 int err;
1540
1541 rtnl_lock();
Alan Sternf07d5b92006-05-09 15:23:03 -07001542 err = raw_notifier_chain_register(&netdev_chain, nb);
Herbert Xufcc5a032007-07-30 17:03:38 -07001543 if (err)
1544 goto unlock;
Eric W. Biederman881d9662007-09-17 11:56:21 -07001545 if (dev_boot_phase)
1546 goto unlock;
1547 for_each_net(net) {
1548 for_each_netdev(net, dev) {
Jiri Pirko351638e2013-05-28 01:30:21 +00001549 err = call_netdevice_notifier(nb, NETDEV_REGISTER, dev);
Eric W. Biederman881d9662007-09-17 11:56:21 -07001550 err = notifier_to_errno(err);
1551 if (err)
1552 goto rollback;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001553
Eric W. Biederman881d9662007-09-17 11:56:21 -07001554 if (!(dev->flags & IFF_UP))
1555 continue;
Herbert Xufcc5a032007-07-30 17:03:38 -07001556
Jiri Pirko351638e2013-05-28 01:30:21 +00001557 call_netdevice_notifier(nb, NETDEV_UP, dev);
Eric W. Biederman881d9662007-09-17 11:56:21 -07001558 }
Linus Torvalds1da177e2005-04-16 15:20:36 -07001559 }
Herbert Xufcc5a032007-07-30 17:03:38 -07001560
1561unlock:
Linus Torvalds1da177e2005-04-16 15:20:36 -07001562 rtnl_unlock();
1563 return err;
Herbert Xufcc5a032007-07-30 17:03:38 -07001564
1565rollback:
1566 last = dev;
Eric W. Biederman881d9662007-09-17 11:56:21 -07001567 for_each_net(net) {
1568 for_each_netdev(net, dev) {
1569 if (dev == last)
RongQing.Li8f891482011-11-30 23:43:07 -05001570 goto outroll;
Herbert Xufcc5a032007-07-30 17:03:38 -07001571
Eric W. Biederman881d9662007-09-17 11:56:21 -07001572 if (dev->flags & IFF_UP) {
Jiri Pirko351638e2013-05-28 01:30:21 +00001573 call_netdevice_notifier(nb, NETDEV_GOING_DOWN,
1574 dev);
1575 call_netdevice_notifier(nb, NETDEV_DOWN, dev);
Eric W. Biederman881d9662007-09-17 11:56:21 -07001576 }
Jiri Pirko351638e2013-05-28 01:30:21 +00001577 call_netdevice_notifier(nb, NETDEV_UNREGISTER, dev);
Herbert Xufcc5a032007-07-30 17:03:38 -07001578 }
Herbert Xufcc5a032007-07-30 17:03:38 -07001579 }
Pavel Emelyanovc67625a2007-11-14 15:53:16 -08001580
RongQing.Li8f891482011-11-30 23:43:07 -05001581outroll:
Pavel Emelyanovc67625a2007-11-14 15:53:16 -08001582 raw_notifier_chain_unregister(&netdev_chain, nb);
Herbert Xufcc5a032007-07-30 17:03:38 -07001583 goto unlock;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001584}
Eric Dumazetd1b19df2009-09-03 01:29:39 -07001585EXPORT_SYMBOL(register_netdevice_notifier);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001586
1587/**
1588 * unregister_netdevice_notifier - unregister a network notifier block
1589 * @nb: notifier
1590 *
1591 * Unregister a notifier previously registered by
1592 * register_netdevice_notifier(). The notifier is unlinked into the
1593 * kernel structures and may then be reused. A negative errno code
1594 * is returned on a failure.
Eric W. Biederman7d3d43d2012-04-06 15:33:35 +00001595 *
1596 * After unregistering unregister and down device events are synthesized
1597 * for all devices on the device list to the removed notifier to remove
1598 * the need for special case cleanup code.
Linus Torvalds1da177e2005-04-16 15:20:36 -07001599 */
1600
1601int unregister_netdevice_notifier(struct notifier_block *nb)
1602{
Eric W. Biederman7d3d43d2012-04-06 15:33:35 +00001603 struct net_device *dev;
1604 struct net *net;
Herbert Xu9f514952006-03-25 01:24:25 -08001605 int err;
1606
1607 rtnl_lock();
Alan Sternf07d5b92006-05-09 15:23:03 -07001608 err = raw_notifier_chain_unregister(&netdev_chain, nb);
Eric W. Biederman7d3d43d2012-04-06 15:33:35 +00001609 if (err)
1610 goto unlock;
1611
1612 for_each_net(net) {
1613 for_each_netdev(net, dev) {
1614 if (dev->flags & IFF_UP) {
Jiri Pirko351638e2013-05-28 01:30:21 +00001615 call_netdevice_notifier(nb, NETDEV_GOING_DOWN,
1616 dev);
1617 call_netdevice_notifier(nb, NETDEV_DOWN, dev);
Eric W. Biederman7d3d43d2012-04-06 15:33:35 +00001618 }
Jiri Pirko351638e2013-05-28 01:30:21 +00001619 call_netdevice_notifier(nb, NETDEV_UNREGISTER, dev);
Eric W. Biederman7d3d43d2012-04-06 15:33:35 +00001620 }
1621 }
1622unlock:
Herbert Xu9f514952006-03-25 01:24:25 -08001623 rtnl_unlock();
1624 return err;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001625}
Eric Dumazetd1b19df2009-09-03 01:29:39 -07001626EXPORT_SYMBOL(unregister_netdevice_notifier);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001627
1628/**
Jiri Pirko351638e2013-05-28 01:30:21 +00001629 * call_netdevice_notifiers_info - call all network notifier blocks
1630 * @val: value passed unmodified to notifier function
1631 * @dev: net_device pointer passed unmodified to notifier function
1632 * @info: notifier information data
1633 *
1634 * Call all network notifier blocks. Parameters and return value
1635 * are as for raw_notifier_call_chain().
1636 */
1637
stephen hemminger1d143d92013-12-29 14:01:29 -08001638static int call_netdevice_notifiers_info(unsigned long val,
1639 struct net_device *dev,
1640 struct netdev_notifier_info *info)
Jiri Pirko351638e2013-05-28 01:30:21 +00001641{
1642 ASSERT_RTNL();
1643 netdev_notifier_info_init(info, dev);
1644 return raw_notifier_call_chain(&netdev_chain, val, info);
1645}
Jiri Pirko351638e2013-05-28 01:30:21 +00001646
1647/**
Linus Torvalds1da177e2005-04-16 15:20:36 -07001648 * call_netdevice_notifiers - call all network notifier blocks
1649 * @val: value passed unmodified to notifier function
Randy Dunlapc4ea43c2007-10-12 21:17:49 -07001650 * @dev: net_device pointer passed unmodified to notifier function
Linus Torvalds1da177e2005-04-16 15:20:36 -07001651 *
1652 * Call all network notifier blocks. Parameters and return value
Alan Sternf07d5b92006-05-09 15:23:03 -07001653 * are as for raw_notifier_call_chain().
Linus Torvalds1da177e2005-04-16 15:20:36 -07001654 */
1655
Eric W. Biedermanad7379d2007-09-16 15:33:32 -07001656int call_netdevice_notifiers(unsigned long val, struct net_device *dev)
Linus Torvalds1da177e2005-04-16 15:20:36 -07001657{
Jiri Pirko351638e2013-05-28 01:30:21 +00001658 struct netdev_notifier_info info;
1659
1660 return call_netdevice_notifiers_info(val, dev, &info);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001661}
stephen hemmingeredf947f2011-03-24 13:24:01 +00001662EXPORT_SYMBOL(call_netdevice_notifiers);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001663
Pablo Neira1cf519002015-05-13 18:19:37 +02001664#ifdef CONFIG_NET_INGRESS
Daniel Borkmann45771392015-04-10 23:07:54 +02001665static struct static_key ingress_needed __read_mostly;
1666
1667void net_inc_ingress_queue(void)
1668{
1669 static_key_slow_inc(&ingress_needed);
1670}
1671EXPORT_SYMBOL_GPL(net_inc_ingress_queue);
1672
1673void net_dec_ingress_queue(void)
1674{
1675 static_key_slow_dec(&ingress_needed);
1676}
1677EXPORT_SYMBOL_GPL(net_dec_ingress_queue);
1678#endif
1679
Daniel Borkmann1f211a12016-01-07 22:29:47 +01001680#ifdef CONFIG_NET_EGRESS
1681static struct static_key egress_needed __read_mostly;
1682
1683void net_inc_egress_queue(void)
1684{
1685 static_key_slow_inc(&egress_needed);
1686}
1687EXPORT_SYMBOL_GPL(net_inc_egress_queue);
1688
1689void net_dec_egress_queue(void)
1690{
1691 static_key_slow_dec(&egress_needed);
1692}
1693EXPORT_SYMBOL_GPL(net_dec_egress_queue);
1694#endif
1695
Ingo Molnarc5905af2012-02-24 08:31:31 +01001696static struct static_key netstamp_needed __read_mostly;
Eric Dumazetb90e5792011-11-28 11:16:50 +00001697#ifdef HAVE_JUMP_LABEL
Ingo Molnarc5905af2012-02-24 08:31:31 +01001698/* We are not allowed to call static_key_slow_dec() from irq context
Eric Dumazetb90e5792011-11-28 11:16:50 +00001699 * If net_disable_timestamp() is called from irq context, defer the
Ingo Molnarc5905af2012-02-24 08:31:31 +01001700 * static_key_slow_dec() calls.
Eric Dumazetb90e5792011-11-28 11:16:50 +00001701 */
1702static atomic_t netstamp_needed_deferred;
1703#endif
Linus Torvalds1da177e2005-04-16 15:20:36 -07001704
1705void net_enable_timestamp(void)
1706{
Eric Dumazetb90e5792011-11-28 11:16:50 +00001707#ifdef HAVE_JUMP_LABEL
1708 int deferred = atomic_xchg(&netstamp_needed_deferred, 0);
1709
1710 if (deferred) {
1711 while (--deferred)
Ingo Molnarc5905af2012-02-24 08:31:31 +01001712 static_key_slow_dec(&netstamp_needed);
Eric Dumazetb90e5792011-11-28 11:16:50 +00001713 return;
1714 }
1715#endif
Ingo Molnarc5905af2012-02-24 08:31:31 +01001716 static_key_slow_inc(&netstamp_needed);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001717}
Eric Dumazetd1b19df2009-09-03 01:29:39 -07001718EXPORT_SYMBOL(net_enable_timestamp);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001719
1720void net_disable_timestamp(void)
1721{
Eric Dumazetb90e5792011-11-28 11:16:50 +00001722#ifdef HAVE_JUMP_LABEL
1723 if (in_interrupt()) {
1724 atomic_inc(&netstamp_needed_deferred);
1725 return;
1726 }
1727#endif
Ingo Molnarc5905af2012-02-24 08:31:31 +01001728 static_key_slow_dec(&netstamp_needed);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001729}
Eric Dumazetd1b19df2009-09-03 01:29:39 -07001730EXPORT_SYMBOL(net_disable_timestamp);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001731
Eric Dumazet3b098e22010-05-15 23:57:10 -07001732static inline void net_timestamp_set(struct sk_buff *skb)
Linus Torvalds1da177e2005-04-16 15:20:36 -07001733{
Thomas Gleixner2456e852016-12-25 11:38:40 +01001734 skb->tstamp = 0;
Ingo Molnarc5905af2012-02-24 08:31:31 +01001735 if (static_key_false(&netstamp_needed))
Patrick McHardya61bbcf2005-08-14 17:24:31 -07001736 __net_timestamp(skb);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001737}
1738
Eric Dumazet588f0332011-11-15 04:12:55 +00001739#define net_timestamp_check(COND, SKB) \
Ingo Molnarc5905af2012-02-24 08:31:31 +01001740 if (static_key_false(&netstamp_needed)) { \
Thomas Gleixner2456e852016-12-25 11:38:40 +01001741 if ((COND) && !(SKB)->tstamp) \
Eric Dumazet588f0332011-11-15 04:12:55 +00001742 __net_timestamp(SKB); \
1743 } \
Eric Dumazet3b098e22010-05-15 23:57:10 -07001744
Nikolay Aleksandrovf4b05d22016-04-28 17:59:28 +02001745bool is_skb_forwardable(const struct net_device *dev, const struct sk_buff *skb)
Daniel Lezcano79b569f2011-03-30 02:42:17 -07001746{
1747 unsigned int len;
1748
1749 if (!(dev->flags & IFF_UP))
1750 return false;
1751
1752 len = dev->mtu + dev->hard_header_len + VLAN_HLEN;
1753 if (skb->len <= len)
1754 return true;
1755
1756 /* if TSO is enabled, we don't care about the length as the packet
1757 * could be forwarded without being segmented before
1758 */
1759 if (skb_is_gso(skb))
1760 return true;
1761
1762 return false;
1763}
Vlad Yasevich1ee481f2014-03-27 17:32:29 -04001764EXPORT_SYMBOL_GPL(is_skb_forwardable);
Daniel Lezcano79b569f2011-03-30 02:42:17 -07001765
Herbert Xua0265d22014-04-17 13:45:03 +08001766int __dev_forward_skb(struct net_device *dev, struct sk_buff *skb)
1767{
Martin KaFai Lau4e3264d2016-11-09 15:36:33 -08001768 int ret = ____dev_forward_skb(dev, skb);
1769
1770 if (likely(!ret)) {
1771 skb->protocol = eth_type_trans(skb, dev);
1772 skb_postpull_rcsum(skb, eth_hdr(skb), ETH_HLEN);
Herbert Xua0265d22014-04-17 13:45:03 +08001773 }
1774
Martin KaFai Lau4e3264d2016-11-09 15:36:33 -08001775 return ret;
Herbert Xua0265d22014-04-17 13:45:03 +08001776}
1777EXPORT_SYMBOL_GPL(__dev_forward_skb);
1778
Arnd Bergmann44540962009-11-26 06:07:08 +00001779/**
1780 * dev_forward_skb - loopback an skb to another netif
1781 *
1782 * @dev: destination network device
1783 * @skb: buffer to forward
1784 *
1785 * return values:
1786 * NET_RX_SUCCESS (no congestion)
Eric Dumazet6ec82562010-05-06 00:53:53 -07001787 * NET_RX_DROP (packet was dropped, but freed)
Arnd Bergmann44540962009-11-26 06:07:08 +00001788 *
1789 * dev_forward_skb can be used for injecting an skb from the
1790 * start_xmit function of one device into the receive queue
1791 * of another device.
1792 *
1793 * The receiving device may be in another namespace, so
1794 * we have to clear all information in the skb that could
1795 * impact namespace isolation.
1796 */
1797int dev_forward_skb(struct net_device *dev, struct sk_buff *skb)
1798{
Herbert Xua0265d22014-04-17 13:45:03 +08001799 return __dev_forward_skb(dev, skb) ?: netif_rx_internal(skb);
Arnd Bergmann44540962009-11-26 06:07:08 +00001800}
1801EXPORT_SYMBOL_GPL(dev_forward_skb);
1802
Changli Gao71d9dec2010-12-15 19:57:25 +00001803static inline int deliver_skb(struct sk_buff *skb,
1804 struct packet_type *pt_prev,
1805 struct net_device *orig_dev)
1806{
Michael S. Tsirkin1080e512012-07-20 09:23:17 +00001807 if (unlikely(skb_orphan_frags(skb, GFP_ATOMIC)))
1808 return -ENOMEM;
Changli Gao71d9dec2010-12-15 19:57:25 +00001809 atomic_inc(&skb->users);
1810 return pt_prev->func(skb, skb->dev, pt_prev, orig_dev);
1811}
1812
Salam Noureddine7866a622015-01-27 11:35:48 -08001813static inline void deliver_ptype_list_skb(struct sk_buff *skb,
1814 struct packet_type **pt,
Jiri Pirkofbcb2172015-03-30 16:56:01 +02001815 struct net_device *orig_dev,
1816 __be16 type,
Salam Noureddine7866a622015-01-27 11:35:48 -08001817 struct list_head *ptype_list)
1818{
1819 struct packet_type *ptype, *pt_prev = *pt;
1820
1821 list_for_each_entry_rcu(ptype, ptype_list, list) {
1822 if (ptype->type != type)
1823 continue;
1824 if (pt_prev)
Jiri Pirkofbcb2172015-03-30 16:56:01 +02001825 deliver_skb(skb, pt_prev, orig_dev);
Salam Noureddine7866a622015-01-27 11:35:48 -08001826 pt_prev = ptype;
1827 }
1828 *pt = pt_prev;
1829}
1830
Eric Leblondc0de08d2012-08-16 22:02:58 +00001831static inline bool skb_loop_sk(struct packet_type *ptype, struct sk_buff *skb)
1832{
Eric Leblonda3d744e2012-11-06 02:10:10 +00001833 if (!ptype->af_packet_priv || !skb->sk)
Eric Leblondc0de08d2012-08-16 22:02:58 +00001834 return false;
1835
1836 if (ptype->id_match)
1837 return ptype->id_match(ptype, skb->sk);
1838 else if ((struct sock *)ptype->af_packet_priv == skb->sk)
1839 return true;
1840
1841 return false;
1842}
1843
Linus Torvalds1da177e2005-04-16 15:20:36 -07001844/*
1845 * Support routine. Sends outgoing frames to any network
1846 * taps currently in use.
1847 */
1848
David Ahern74b20582016-05-10 11:19:50 -07001849void dev_queue_xmit_nit(struct sk_buff *skb, struct net_device *dev)
Linus Torvalds1da177e2005-04-16 15:20:36 -07001850{
1851 struct packet_type *ptype;
Changli Gao71d9dec2010-12-15 19:57:25 +00001852 struct sk_buff *skb2 = NULL;
1853 struct packet_type *pt_prev = NULL;
Salam Noureddine7866a622015-01-27 11:35:48 -08001854 struct list_head *ptype_list = &ptype_all;
Patrick McHardya61bbcf2005-08-14 17:24:31 -07001855
Linus Torvalds1da177e2005-04-16 15:20:36 -07001856 rcu_read_lock();
Salam Noureddine7866a622015-01-27 11:35:48 -08001857again:
1858 list_for_each_entry_rcu(ptype, ptype_list, list) {
Linus Torvalds1da177e2005-04-16 15:20:36 -07001859 /* Never send packets back to the socket
1860 * they originated from - MvS (miquels@drinkel.ow.org)
1861 */
Salam Noureddine7866a622015-01-27 11:35:48 -08001862 if (skb_loop_sk(ptype, skb))
1863 continue;
Changli Gao71d9dec2010-12-15 19:57:25 +00001864
Salam Noureddine7866a622015-01-27 11:35:48 -08001865 if (pt_prev) {
1866 deliver_skb(skb2, pt_prev, skb->dev);
Changli Gao71d9dec2010-12-15 19:57:25 +00001867 pt_prev = ptype;
Salam Noureddine7866a622015-01-27 11:35:48 -08001868 continue;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001869 }
Salam Noureddine7866a622015-01-27 11:35:48 -08001870
1871 /* need to clone skb, done only once */
1872 skb2 = skb_clone(skb, GFP_ATOMIC);
1873 if (!skb2)
1874 goto out_unlock;
1875
1876 net_timestamp_set(skb2);
1877
1878 /* skb->nh should be correctly
1879 * set by sender, so that the second statement is
1880 * just protection against buggy protocols.
1881 */
1882 skb_reset_mac_header(skb2);
1883
1884 if (skb_network_header(skb2) < skb2->data ||
1885 skb_network_header(skb2) > skb_tail_pointer(skb2)) {
1886 net_crit_ratelimited("protocol %04x is buggy, dev %s\n",
1887 ntohs(skb2->protocol),
1888 dev->name);
1889 skb_reset_network_header(skb2);
1890 }
1891
1892 skb2->transport_header = skb2->network_header;
1893 skb2->pkt_type = PACKET_OUTGOING;
1894 pt_prev = ptype;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001895 }
Salam Noureddine7866a622015-01-27 11:35:48 -08001896
1897 if (ptype_list == &ptype_all) {
1898 ptype_list = &dev->ptype_all;
1899 goto again;
1900 }
1901out_unlock:
Changli Gao71d9dec2010-12-15 19:57:25 +00001902 if (pt_prev)
1903 pt_prev->func(skb2, skb->dev, pt_prev, skb->dev);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001904 rcu_read_unlock();
1905}
David Ahern74b20582016-05-10 11:19:50 -07001906EXPORT_SYMBOL_GPL(dev_queue_xmit_nit);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001907
Ben Hutchings2c530402012-07-10 10:55:09 +00001908/**
1909 * netif_setup_tc - Handle tc mappings on real_num_tx_queues change
John Fastabend4f57c082011-01-17 08:06:04 +00001910 * @dev: Network device
1911 * @txq: number of queues available
1912 *
1913 * If real_num_tx_queues is changed the tc mappings may no longer be
1914 * valid. To resolve this verify the tc mapping remains valid and if
1915 * not NULL the mapping. With no priorities mapping to this
1916 * offset/count pair it will no longer be used. In the worst case TC0
1917 * is invalid nothing can be done so disable priority mappings. If is
1918 * expected that drivers will fix this mapping if they can before
1919 * calling netif_set_real_num_tx_queues.
1920 */
Eric Dumazetbb134d22011-01-20 19:18:08 +00001921static void netif_setup_tc(struct net_device *dev, unsigned int txq)
John Fastabend4f57c082011-01-17 08:06:04 +00001922{
1923 int i;
1924 struct netdev_tc_txq *tc = &dev->tc_to_txq[0];
1925
1926 /* If TC0 is invalidated disable TC mapping */
1927 if (tc->offset + tc->count > txq) {
Joe Perches7b6cd1c2012-02-01 10:54:43 +00001928 pr_warn("Number of in use tx queues changed invalidating tc mappings. Priority traffic classification disabled!\n");
John Fastabend4f57c082011-01-17 08:06:04 +00001929 dev->num_tc = 0;
1930 return;
1931 }
1932
1933 /* Invalidated prio to tc mappings set to TC0 */
1934 for (i = 1; i < TC_BITMASK + 1; i++) {
1935 int q = netdev_get_prio_tc_map(dev, i);
1936
1937 tc = &dev->tc_to_txq[q];
1938 if (tc->offset + tc->count > txq) {
Joe Perches7b6cd1c2012-02-01 10:54:43 +00001939 pr_warn("Number of in use tx queues changed. Priority %i to tc mapping %i is no longer valid. Setting map to 0\n",
1940 i, q);
John Fastabend4f57c082011-01-17 08:06:04 +00001941 netdev_set_prio_tc_map(dev, i, 0);
1942 }
1943 }
1944}
1945
Alexander Duyck8d059b02016-10-28 11:43:49 -04001946int netdev_txq_to_tc(struct net_device *dev, unsigned int txq)
1947{
1948 if (dev->num_tc) {
1949 struct netdev_tc_txq *tc = &dev->tc_to_txq[0];
1950 int i;
1951
1952 for (i = 0; i < TC_MAX_QUEUE; i++, tc++) {
1953 if ((txq - tc->offset) < tc->count)
1954 return i;
1955 }
1956
1957 return -1;
1958 }
1959
1960 return 0;
1961}
1962
Alexander Duyck537c00d2013-01-10 08:57:02 +00001963#ifdef CONFIG_XPS
1964static DEFINE_MUTEX(xps_map_mutex);
1965#define xmap_dereference(P) \
1966 rcu_dereference_protected((P), lockdep_is_held(&xps_map_mutex))
1967
Alexander Duyck6234f872016-10-28 11:46:49 -04001968static bool remove_xps_queue(struct xps_dev_maps *dev_maps,
1969 int tci, u16 index)
Alexander Duyck10cdc3f2013-01-10 08:57:17 +00001970{
1971 struct xps_map *map = NULL;
1972 int pos;
1973
1974 if (dev_maps)
Alexander Duyck6234f872016-10-28 11:46:49 -04001975 map = xmap_dereference(dev_maps->cpu_map[tci]);
1976 if (!map)
1977 return false;
Alexander Duyck10cdc3f2013-01-10 08:57:17 +00001978
Alexander Duyck6234f872016-10-28 11:46:49 -04001979 for (pos = map->len; pos--;) {
1980 if (map->queues[pos] != index)
1981 continue;
1982
1983 if (map->len > 1) {
1984 map->queues[pos] = map->queues[--map->len];
Alexander Duyck10cdc3f2013-01-10 08:57:17 +00001985 break;
1986 }
Alexander Duyck6234f872016-10-28 11:46:49 -04001987
1988 RCU_INIT_POINTER(dev_maps->cpu_map[tci], NULL);
1989 kfree_rcu(map, rcu);
1990 return false;
Alexander Duyck10cdc3f2013-01-10 08:57:17 +00001991 }
1992
Alexander Duyck6234f872016-10-28 11:46:49 -04001993 return true;
Alexander Duyck10cdc3f2013-01-10 08:57:17 +00001994}
1995
Alexander Duyck6234f872016-10-28 11:46:49 -04001996static bool remove_xps_queue_cpu(struct net_device *dev,
1997 struct xps_dev_maps *dev_maps,
1998 int cpu, u16 offset, u16 count)
1999{
Alexander Duyck184c4492016-10-28 11:50:13 -04002000 int num_tc = dev->num_tc ? : 1;
2001 bool active = false;
2002 int tci;
Alexander Duyck6234f872016-10-28 11:46:49 -04002003
Alexander Duyck184c4492016-10-28 11:50:13 -04002004 for (tci = cpu * num_tc; num_tc--; tci++) {
2005 int i, j;
2006
2007 for (i = count, j = offset; i--; j++) {
2008 if (!remove_xps_queue(dev_maps, cpu, j))
2009 break;
2010 }
2011
2012 active |= i < 0;
Alexander Duyck6234f872016-10-28 11:46:49 -04002013 }
2014
Alexander Duyck184c4492016-10-28 11:50:13 -04002015 return active;
Alexander Duyck6234f872016-10-28 11:46:49 -04002016}
2017
2018static void netif_reset_xps_queues(struct net_device *dev, u16 offset,
2019 u16 count)
Alexander Duyck537c00d2013-01-10 08:57:02 +00002020{
2021 struct xps_dev_maps *dev_maps;
Alexander Duyck024e9672013-01-10 08:57:46 +00002022 int cpu, i;
Alexander Duyck10cdc3f2013-01-10 08:57:17 +00002023 bool active = false;
Alexander Duyck537c00d2013-01-10 08:57:02 +00002024
2025 mutex_lock(&xps_map_mutex);
2026 dev_maps = xmap_dereference(dev->xps_maps);
2027
2028 if (!dev_maps)
2029 goto out_no_maps;
2030
Alexander Duyck6234f872016-10-28 11:46:49 -04002031 for_each_possible_cpu(cpu)
2032 active |= remove_xps_queue_cpu(dev, dev_maps, cpu,
2033 offset, count);
Alexander Duyck537c00d2013-01-10 08:57:02 +00002034
Alexander Duyck10cdc3f2013-01-10 08:57:17 +00002035 if (!active) {
Alexander Duyck537c00d2013-01-10 08:57:02 +00002036 RCU_INIT_POINTER(dev->xps_maps, NULL);
2037 kfree_rcu(dev_maps, rcu);
2038 }
2039
Alexander Duyck6234f872016-10-28 11:46:49 -04002040 for (i = offset + (count - 1); count--; i--)
Alexander Duyck024e9672013-01-10 08:57:46 +00002041 netdev_queue_numa_node_write(netdev_get_tx_queue(dev, i),
2042 NUMA_NO_NODE);
2043
Alexander Duyck537c00d2013-01-10 08:57:02 +00002044out_no_maps:
2045 mutex_unlock(&xps_map_mutex);
2046}
2047
Alexander Duyck6234f872016-10-28 11:46:49 -04002048static void netif_reset_xps_queues_gt(struct net_device *dev, u16 index)
2049{
2050 netif_reset_xps_queues(dev, index, dev->num_tx_queues - index);
2051}
2052
Alexander Duyck01c5f862013-01-10 08:57:35 +00002053static struct xps_map *expand_xps_map(struct xps_map *map,
2054 int cpu, u16 index)
2055{
2056 struct xps_map *new_map;
2057 int alloc_len = XPS_MIN_MAP_ALLOC;
2058 int i, pos;
2059
2060 for (pos = 0; map && pos < map->len; pos++) {
2061 if (map->queues[pos] != index)
2062 continue;
2063 return map;
2064 }
2065
2066 /* Need to add queue to this CPU's existing map */
2067 if (map) {
2068 if (pos < map->alloc_len)
2069 return map;
2070
2071 alloc_len = map->alloc_len * 2;
2072 }
2073
2074 /* Need to allocate new map to store queue on this CPU's map */
2075 new_map = kzalloc_node(XPS_MAP_SIZE(alloc_len), GFP_KERNEL,
2076 cpu_to_node(cpu));
2077 if (!new_map)
2078 return NULL;
2079
2080 for (i = 0; i < pos; i++)
2081 new_map->queues[i] = map->queues[i];
2082 new_map->alloc_len = alloc_len;
2083 new_map->len = pos;
2084
2085 return new_map;
2086}
2087
Michael S. Tsirkin35735402013-10-02 09:14:06 +03002088int netif_set_xps_queue(struct net_device *dev, const struct cpumask *mask,
2089 u16 index)
Alexander Duyck537c00d2013-01-10 08:57:02 +00002090{
Alexander Duyck01c5f862013-01-10 08:57:35 +00002091 struct xps_dev_maps *dev_maps, *new_dev_maps = NULL;
Alexander Duyck184c4492016-10-28 11:50:13 -04002092 int i, cpu, tci, numa_node_id = -2;
2093 int maps_sz, num_tc = 1, tc = 0;
Alexander Duyck537c00d2013-01-10 08:57:02 +00002094 struct xps_map *map, *new_map;
Alexander Duyck01c5f862013-01-10 08:57:35 +00002095 bool active = false;
Alexander Duyck537c00d2013-01-10 08:57:02 +00002096
Alexander Duyck184c4492016-10-28 11:50:13 -04002097 if (dev->num_tc) {
2098 num_tc = dev->num_tc;
2099 tc = netdev_txq_to_tc(dev, index);
2100 if (tc < 0)
2101 return -EINVAL;
2102 }
2103
2104 maps_sz = XPS_DEV_MAPS_SIZE(num_tc);
2105 if (maps_sz < L1_CACHE_BYTES)
2106 maps_sz = L1_CACHE_BYTES;
2107
Alexander Duyck537c00d2013-01-10 08:57:02 +00002108 mutex_lock(&xps_map_mutex);
2109
2110 dev_maps = xmap_dereference(dev->xps_maps);
2111
Alexander Duyck01c5f862013-01-10 08:57:35 +00002112 /* allocate memory for queue storage */
Alexander Duyck184c4492016-10-28 11:50:13 -04002113 for_each_cpu_and(cpu, cpu_online_mask, mask) {
Alexander Duyck01c5f862013-01-10 08:57:35 +00002114 if (!new_dev_maps)
2115 new_dev_maps = kzalloc(maps_sz, GFP_KERNEL);
Alexander Duyck2bb60cb2013-02-22 06:38:44 +00002116 if (!new_dev_maps) {
2117 mutex_unlock(&xps_map_mutex);
Alexander Duyck01c5f862013-01-10 08:57:35 +00002118 return -ENOMEM;
Alexander Duyck2bb60cb2013-02-22 06:38:44 +00002119 }
Alexander Duyck01c5f862013-01-10 08:57:35 +00002120
Alexander Duyck184c4492016-10-28 11:50:13 -04002121 tci = cpu * num_tc + tc;
2122 map = dev_maps ? xmap_dereference(dev_maps->cpu_map[tci]) :
Alexander Duyck01c5f862013-01-10 08:57:35 +00002123 NULL;
2124
2125 map = expand_xps_map(map, cpu, index);
2126 if (!map)
2127 goto error;
2128
Alexander Duyck184c4492016-10-28 11:50:13 -04002129 RCU_INIT_POINTER(new_dev_maps->cpu_map[tci], map);
Alexander Duyck01c5f862013-01-10 08:57:35 +00002130 }
2131
2132 if (!new_dev_maps)
2133 goto out_no_new_maps;
2134
2135 for_each_possible_cpu(cpu) {
Alexander Duyck184c4492016-10-28 11:50:13 -04002136 /* copy maps belonging to foreign traffic classes */
2137 for (i = tc, tci = cpu * num_tc; dev_maps && i--; tci++) {
2138 /* fill in the new device map from the old device map */
2139 map = xmap_dereference(dev_maps->cpu_map[tci]);
2140 RCU_INIT_POINTER(new_dev_maps->cpu_map[tci], map);
2141 }
2142
2143 /* We need to explicitly update tci as prevous loop
2144 * could break out early if dev_maps is NULL.
2145 */
2146 tci = cpu * num_tc + tc;
2147
Alexander Duyck01c5f862013-01-10 08:57:35 +00002148 if (cpumask_test_cpu(cpu, mask) && cpu_online(cpu)) {
2149 /* add queue to CPU maps */
2150 int pos = 0;
2151
Alexander Duyck184c4492016-10-28 11:50:13 -04002152 map = xmap_dereference(new_dev_maps->cpu_map[tci]);
Alexander Duyck01c5f862013-01-10 08:57:35 +00002153 while ((pos < map->len) && (map->queues[pos] != index))
2154 pos++;
2155
2156 if (pos == map->len)
2157 map->queues[map->len++] = index;
Alexander Duyck537c00d2013-01-10 08:57:02 +00002158#ifdef CONFIG_NUMA
Alexander Duyck537c00d2013-01-10 08:57:02 +00002159 if (numa_node_id == -2)
2160 numa_node_id = cpu_to_node(cpu);
2161 else if (numa_node_id != cpu_to_node(cpu))
2162 numa_node_id = -1;
Alexander Duyck537c00d2013-01-10 08:57:02 +00002163#endif
Alexander Duyck01c5f862013-01-10 08:57:35 +00002164 } else if (dev_maps) {
2165 /* fill in the new device map from the old device map */
Alexander Duyck184c4492016-10-28 11:50:13 -04002166 map = xmap_dereference(dev_maps->cpu_map[tci]);
2167 RCU_INIT_POINTER(new_dev_maps->cpu_map[tci], map);
Alexander Duyck537c00d2013-01-10 08:57:02 +00002168 }
Alexander Duyck01c5f862013-01-10 08:57:35 +00002169
Alexander Duyck184c4492016-10-28 11:50:13 -04002170 /* copy maps belonging to foreign traffic classes */
2171 for (i = num_tc - tc, tci++; dev_maps && --i; tci++) {
2172 /* fill in the new device map from the old device map */
2173 map = xmap_dereference(dev_maps->cpu_map[tci]);
2174 RCU_INIT_POINTER(new_dev_maps->cpu_map[tci], map);
2175 }
Alexander Duyck537c00d2013-01-10 08:57:02 +00002176 }
2177
Alexander Duyck01c5f862013-01-10 08:57:35 +00002178 rcu_assign_pointer(dev->xps_maps, new_dev_maps);
2179
Alexander Duyck537c00d2013-01-10 08:57:02 +00002180 /* Cleanup old maps */
Alexander Duyck184c4492016-10-28 11:50:13 -04002181 if (!dev_maps)
2182 goto out_no_old_maps;
2183
2184 for_each_possible_cpu(cpu) {
2185 for (i = num_tc, tci = cpu * num_tc; i--; tci++) {
2186 new_map = xmap_dereference(new_dev_maps->cpu_map[tci]);
2187 map = xmap_dereference(dev_maps->cpu_map[tci]);
Alexander Duyck01c5f862013-01-10 08:57:35 +00002188 if (map && map != new_map)
2189 kfree_rcu(map, rcu);
2190 }
Alexander Duyck01c5f862013-01-10 08:57:35 +00002191 }
Alexander Duyck537c00d2013-01-10 08:57:02 +00002192
Alexander Duyck184c4492016-10-28 11:50:13 -04002193 kfree_rcu(dev_maps, rcu);
2194
2195out_no_old_maps:
Alexander Duyck01c5f862013-01-10 08:57:35 +00002196 dev_maps = new_dev_maps;
2197 active = true;
2198
2199out_no_new_maps:
2200 /* update Tx queue numa node */
Alexander Duyck537c00d2013-01-10 08:57:02 +00002201 netdev_queue_numa_node_write(netdev_get_tx_queue(dev, index),
2202 (numa_node_id >= 0) ? numa_node_id :
2203 NUMA_NO_NODE);
2204
Alexander Duyck01c5f862013-01-10 08:57:35 +00002205 if (!dev_maps)
2206 goto out_no_maps;
2207
2208 /* removes queue from unused CPUs */
2209 for_each_possible_cpu(cpu) {
Alexander Duyck184c4492016-10-28 11:50:13 -04002210 for (i = tc, tci = cpu * num_tc; i--; tci++)
2211 active |= remove_xps_queue(dev_maps, tci, index);
2212 if (!cpumask_test_cpu(cpu, mask) || !cpu_online(cpu))
2213 active |= remove_xps_queue(dev_maps, tci, index);
2214 for (i = num_tc - tc, tci++; --i; tci++)
2215 active |= remove_xps_queue(dev_maps, tci, index);
Alexander Duyck01c5f862013-01-10 08:57:35 +00002216 }
2217
2218 /* free map if not active */
2219 if (!active) {
2220 RCU_INIT_POINTER(dev->xps_maps, NULL);
2221 kfree_rcu(dev_maps, rcu);
2222 }
2223
2224out_no_maps:
Alexander Duyck537c00d2013-01-10 08:57:02 +00002225 mutex_unlock(&xps_map_mutex);
2226
2227 return 0;
2228error:
Alexander Duyck01c5f862013-01-10 08:57:35 +00002229 /* remove any maps that we added */
2230 for_each_possible_cpu(cpu) {
Alexander Duyck184c4492016-10-28 11:50:13 -04002231 for (i = num_tc, tci = cpu * num_tc; i--; tci++) {
2232 new_map = xmap_dereference(new_dev_maps->cpu_map[tci]);
2233 map = dev_maps ?
2234 xmap_dereference(dev_maps->cpu_map[tci]) :
2235 NULL;
2236 if (new_map && new_map != map)
2237 kfree(new_map);
2238 }
Alexander Duyck01c5f862013-01-10 08:57:35 +00002239 }
2240
Alexander Duyck537c00d2013-01-10 08:57:02 +00002241 mutex_unlock(&xps_map_mutex);
2242
Alexander Duyck537c00d2013-01-10 08:57:02 +00002243 kfree(new_dev_maps);
2244 return -ENOMEM;
2245}
2246EXPORT_SYMBOL(netif_set_xps_queue);
2247
2248#endif
Alexander Duyck9cf1f6a2016-10-28 11:43:20 -04002249void netdev_reset_tc(struct net_device *dev)
2250{
Alexander Duyck6234f872016-10-28 11:46:49 -04002251#ifdef CONFIG_XPS
2252 netif_reset_xps_queues_gt(dev, 0);
2253#endif
Alexander Duyck9cf1f6a2016-10-28 11:43:20 -04002254 dev->num_tc = 0;
2255 memset(dev->tc_to_txq, 0, sizeof(dev->tc_to_txq));
2256 memset(dev->prio_tc_map, 0, sizeof(dev->prio_tc_map));
2257}
2258EXPORT_SYMBOL(netdev_reset_tc);
2259
2260int netdev_set_tc_queue(struct net_device *dev, u8 tc, u16 count, u16 offset)
2261{
2262 if (tc >= dev->num_tc)
2263 return -EINVAL;
2264
Alexander Duyck6234f872016-10-28 11:46:49 -04002265#ifdef CONFIG_XPS
2266 netif_reset_xps_queues(dev, offset, count);
2267#endif
Alexander Duyck9cf1f6a2016-10-28 11:43:20 -04002268 dev->tc_to_txq[tc].count = count;
2269 dev->tc_to_txq[tc].offset = offset;
2270 return 0;
2271}
2272EXPORT_SYMBOL(netdev_set_tc_queue);
2273
2274int netdev_set_num_tc(struct net_device *dev, u8 num_tc)
2275{
2276 if (num_tc > TC_MAX_QUEUE)
2277 return -EINVAL;
2278
Alexander Duyck6234f872016-10-28 11:46:49 -04002279#ifdef CONFIG_XPS
2280 netif_reset_xps_queues_gt(dev, 0);
2281#endif
Alexander Duyck9cf1f6a2016-10-28 11:43:20 -04002282 dev->num_tc = num_tc;
2283 return 0;
2284}
2285EXPORT_SYMBOL(netdev_set_num_tc);
2286
John Fastabendf0796d52010-07-01 13:21:57 +00002287/*
2288 * Routine to help set real_num_tx_queues. To avoid skbs mapped to queues
2289 * greater then real_num_tx_queues stale skbs on the qdisc must be flushed.
2290 */
Tom Herberte6484932010-10-18 18:04:39 +00002291int netif_set_real_num_tx_queues(struct net_device *dev, unsigned int txq)
John Fastabendf0796d52010-07-01 13:21:57 +00002292{
Tom Herbert1d24eb42010-11-21 13:17:27 +00002293 int rc;
2294
Tom Herberte6484932010-10-18 18:04:39 +00002295 if (txq < 1 || txq > dev->num_tx_queues)
2296 return -EINVAL;
John Fastabendf0796d52010-07-01 13:21:57 +00002297
Ben Hutchings5c565802011-02-15 19:39:21 +00002298 if (dev->reg_state == NETREG_REGISTERED ||
2299 dev->reg_state == NETREG_UNREGISTERING) {
Tom Herberte6484932010-10-18 18:04:39 +00002300 ASSERT_RTNL();
2301
Tom Herbert1d24eb42010-11-21 13:17:27 +00002302 rc = netdev_queue_update_kobjects(dev, dev->real_num_tx_queues,
2303 txq);
Tom Herbertbf264142010-11-26 08:36:09 +00002304 if (rc)
2305 return rc;
2306
John Fastabend4f57c082011-01-17 08:06:04 +00002307 if (dev->num_tc)
2308 netif_setup_tc(dev, txq);
2309
Alexander Duyck024e9672013-01-10 08:57:46 +00002310 if (txq < dev->real_num_tx_queues) {
Tom Herberte6484932010-10-18 18:04:39 +00002311 qdisc_reset_all_tx_gt(dev, txq);
Alexander Duyck024e9672013-01-10 08:57:46 +00002312#ifdef CONFIG_XPS
2313 netif_reset_xps_queues_gt(dev, txq);
2314#endif
2315 }
John Fastabendf0796d52010-07-01 13:21:57 +00002316 }
Tom Herberte6484932010-10-18 18:04:39 +00002317
2318 dev->real_num_tx_queues = txq;
2319 return 0;
John Fastabendf0796d52010-07-01 13:21:57 +00002320}
2321EXPORT_SYMBOL(netif_set_real_num_tx_queues);
Denis Vlasenko56079432006-03-29 15:57:29 -08002322
Michael Daltona953be52014-01-16 22:23:28 -08002323#ifdef CONFIG_SYSFS
Ben Hutchings62fe0b42010-09-27 08:24:33 +00002324/**
2325 * netif_set_real_num_rx_queues - set actual number of RX queues used
2326 * @dev: Network device
2327 * @rxq: Actual number of RX queues
2328 *
2329 * This must be called either with the rtnl_lock held or before
2330 * registration of the net device. Returns 0 on success, or a
Ben Hutchings4e7f7952010-10-08 10:33:39 -07002331 * negative error code. If called before registration, it always
2332 * succeeds.
Ben Hutchings62fe0b42010-09-27 08:24:33 +00002333 */
2334int netif_set_real_num_rx_queues(struct net_device *dev, unsigned int rxq)
2335{
2336 int rc;
2337
Tom Herbertbd25fa72010-10-18 18:00:16 +00002338 if (rxq < 1 || rxq > dev->num_rx_queues)
2339 return -EINVAL;
2340
Ben Hutchings62fe0b42010-09-27 08:24:33 +00002341 if (dev->reg_state == NETREG_REGISTERED) {
2342 ASSERT_RTNL();
2343
Ben Hutchings62fe0b42010-09-27 08:24:33 +00002344 rc = net_rx_queue_update_kobjects(dev, dev->real_num_rx_queues,
2345 rxq);
2346 if (rc)
2347 return rc;
Ben Hutchings62fe0b42010-09-27 08:24:33 +00002348 }
2349
2350 dev->real_num_rx_queues = rxq;
2351 return 0;
2352}
2353EXPORT_SYMBOL(netif_set_real_num_rx_queues);
2354#endif
2355
Ben Hutchings2c530402012-07-10 10:55:09 +00002356/**
2357 * netif_get_num_default_rss_queues - default number of RSS queues
Yuval Mintz16917b82012-07-01 03:18:50 +00002358 *
2359 * This routine should set an upper limit on the number of RSS queues
2360 * used by default by multiqueue devices.
2361 */
Ben Hutchingsa55b1382012-07-10 10:54:38 +00002362int netif_get_num_default_rss_queues(void)
Yuval Mintz16917b82012-07-01 03:18:50 +00002363{
Hariprasad Shenai40e4e712016-06-08 18:09:08 +05302364 return is_kdump_kernel() ?
2365 1 : min_t(int, DEFAULT_MAX_NUM_RSS_QUEUES, num_online_cpus());
Yuval Mintz16917b82012-07-01 03:18:50 +00002366}
2367EXPORT_SYMBOL(netif_get_num_default_rss_queues);
2368
Eric Dumazet3bcb8462016-06-04 20:02:28 -07002369static void __netif_reschedule(struct Qdisc *q)
Jarek Poplawskidef82a12008-08-17 21:54:43 -07002370{
2371 struct softnet_data *sd;
2372 unsigned long flags;
2373
2374 local_irq_save(flags);
Christoph Lameter903ceff2014-08-17 12:30:35 -05002375 sd = this_cpu_ptr(&softnet_data);
Changli Gaoa9cbd582010-04-26 23:06:24 +00002376 q->next_sched = NULL;
2377 *sd->output_queue_tailp = q;
2378 sd->output_queue_tailp = &q->next_sched;
Jarek Poplawskidef82a12008-08-17 21:54:43 -07002379 raise_softirq_irqoff(NET_TX_SOFTIRQ);
2380 local_irq_restore(flags);
2381}
2382
David S. Miller37437bb2008-07-16 02:15:04 -07002383void __netif_schedule(struct Qdisc *q)
Denis Vlasenko56079432006-03-29 15:57:29 -08002384{
Jarek Poplawskidef82a12008-08-17 21:54:43 -07002385 if (!test_and_set_bit(__QDISC_STATE_SCHED, &q->state))
2386 __netif_reschedule(q);
Denis Vlasenko56079432006-03-29 15:57:29 -08002387}
2388EXPORT_SYMBOL(__netif_schedule);
2389
Eric Dumazete6247022013-12-05 04:45:08 -08002390struct dev_kfree_skb_cb {
2391 enum skb_free_reason reason;
2392};
2393
2394static struct dev_kfree_skb_cb *get_kfree_skb_cb(const struct sk_buff *skb)
Denis Vlasenko56079432006-03-29 15:57:29 -08002395{
Eric Dumazete6247022013-12-05 04:45:08 -08002396 return (struct dev_kfree_skb_cb *)skb->cb;
Denis Vlasenko56079432006-03-29 15:57:29 -08002397}
Denis Vlasenko56079432006-03-29 15:57:29 -08002398
John Fastabend46e5da42014-09-12 20:04:52 -07002399void netif_schedule_queue(struct netdev_queue *txq)
2400{
2401 rcu_read_lock();
2402 if (!(txq->state & QUEUE_STATE_ANY_XOFF)) {
2403 struct Qdisc *q = rcu_dereference(txq->qdisc);
2404
2405 __netif_schedule(q);
2406 }
2407 rcu_read_unlock();
2408}
2409EXPORT_SYMBOL(netif_schedule_queue);
2410
2411/**
2412 * netif_wake_subqueue - allow sending packets on subqueue
2413 * @dev: network device
2414 * @queue_index: sub queue index
2415 *
2416 * Resume individual transmit queue of a device with multiple transmit queues.
2417 */
2418void netif_wake_subqueue(struct net_device *dev, u16 queue_index)
2419{
2420 struct netdev_queue *txq = netdev_get_tx_queue(dev, queue_index);
2421
2422 if (test_and_clear_bit(__QUEUE_STATE_DRV_XOFF, &txq->state)) {
2423 struct Qdisc *q;
2424
2425 rcu_read_lock();
2426 q = rcu_dereference(txq->qdisc);
2427 __netif_schedule(q);
2428 rcu_read_unlock();
2429 }
2430}
2431EXPORT_SYMBOL(netif_wake_subqueue);
2432
2433void netif_tx_wake_queue(struct netdev_queue *dev_queue)
2434{
2435 if (test_and_clear_bit(__QUEUE_STATE_DRV_XOFF, &dev_queue->state)) {
2436 struct Qdisc *q;
2437
2438 rcu_read_lock();
2439 q = rcu_dereference(dev_queue->qdisc);
2440 __netif_schedule(q);
2441 rcu_read_unlock();
2442 }
2443}
2444EXPORT_SYMBOL(netif_tx_wake_queue);
2445
Eric Dumazete6247022013-12-05 04:45:08 -08002446void __dev_kfree_skb_irq(struct sk_buff *skb, enum skb_free_reason reason)
2447{
2448 unsigned long flags;
2449
2450 if (likely(atomic_read(&skb->users) == 1)) {
2451 smp_rmb();
2452 atomic_set(&skb->users, 0);
2453 } else if (likely(!atomic_dec_and_test(&skb->users))) {
2454 return;
2455 }
2456 get_kfree_skb_cb(skb)->reason = reason;
2457 local_irq_save(flags);
2458 skb->next = __this_cpu_read(softnet_data.completion_queue);
2459 __this_cpu_write(softnet_data.completion_queue, skb);
2460 raise_softirq_irqoff(NET_TX_SOFTIRQ);
2461 local_irq_restore(flags);
2462}
2463EXPORT_SYMBOL(__dev_kfree_skb_irq);
2464
2465void __dev_kfree_skb_any(struct sk_buff *skb, enum skb_free_reason reason)
Denis Vlasenko56079432006-03-29 15:57:29 -08002466{
2467 if (in_irq() || irqs_disabled())
Eric Dumazete6247022013-12-05 04:45:08 -08002468 __dev_kfree_skb_irq(skb, reason);
Denis Vlasenko56079432006-03-29 15:57:29 -08002469 else
2470 dev_kfree_skb(skb);
2471}
Eric Dumazete6247022013-12-05 04:45:08 -08002472EXPORT_SYMBOL(__dev_kfree_skb_any);
Denis Vlasenko56079432006-03-29 15:57:29 -08002473
2474
Stephen Hemmingerbea33482007-10-03 16:41:36 -07002475/**
2476 * netif_device_detach - mark device as removed
2477 * @dev: network device
2478 *
2479 * Mark device as removed from system and therefore no longer available.
2480 */
Denis Vlasenko56079432006-03-29 15:57:29 -08002481void netif_device_detach(struct net_device *dev)
2482{
2483 if (test_and_clear_bit(__LINK_STATE_PRESENT, &dev->state) &&
2484 netif_running(dev)) {
Alexander Duyckd5431032009-04-08 13:15:22 +00002485 netif_tx_stop_all_queues(dev);
Denis Vlasenko56079432006-03-29 15:57:29 -08002486 }
2487}
2488EXPORT_SYMBOL(netif_device_detach);
2489
Stephen Hemmingerbea33482007-10-03 16:41:36 -07002490/**
2491 * netif_device_attach - mark device as attached
2492 * @dev: network device
2493 *
2494 * Mark device as attached from system and restart if needed.
2495 */
Denis Vlasenko56079432006-03-29 15:57:29 -08002496void netif_device_attach(struct net_device *dev)
2497{
2498 if (!test_and_set_bit(__LINK_STATE_PRESENT, &dev->state) &&
2499 netif_running(dev)) {
Alexander Duyckd5431032009-04-08 13:15:22 +00002500 netif_tx_wake_all_queues(dev);
YOSHIFUJI Hideaki4ec93ed2007-02-09 23:24:36 +09002501 __netdev_watchdog_up(dev);
Denis Vlasenko56079432006-03-29 15:57:29 -08002502 }
2503}
2504EXPORT_SYMBOL(netif_device_attach);
2505
Jiri Pirko5605c762015-05-12 14:56:12 +02002506/*
2507 * Returns a Tx hash based on the given packet descriptor a Tx queues' number
2508 * to be used as a distribution range.
2509 */
2510u16 __skb_tx_hash(const struct net_device *dev, struct sk_buff *skb,
2511 unsigned int num_tx_queues)
2512{
2513 u32 hash;
2514 u16 qoffset = 0;
2515 u16 qcount = num_tx_queues;
2516
2517 if (skb_rx_queue_recorded(skb)) {
2518 hash = skb_get_rx_queue(skb);
2519 while (unlikely(hash >= num_tx_queues))
2520 hash -= num_tx_queues;
2521 return hash;
2522 }
2523
2524 if (dev->num_tc) {
2525 u8 tc = netdev_get_prio_tc_map(dev, skb->priority);
2526 qoffset = dev->tc_to_txq[tc].offset;
2527 qcount = dev->tc_to_txq[tc].count;
2528 }
2529
2530 return (u16) reciprocal_scale(skb_get_hash(skb), qcount) + qoffset;
2531}
2532EXPORT_SYMBOL(__skb_tx_hash);
2533
Ben Hutchings36c92472012-01-17 07:57:56 +00002534static void skb_warn_bad_offload(const struct sk_buff *skb)
2535{
Wei Tang84d15ae2016-06-16 21:17:49 +08002536 static const netdev_features_t null_features;
Ben Hutchings36c92472012-01-17 07:57:56 +00002537 struct net_device *dev = skb->dev;
Bjørn Mork88ad4172015-11-16 19:16:40 +01002538 const char *name = "";
Ben Hutchings36c92472012-01-17 07:57:56 +00002539
Ben Greearc846ad92013-04-19 10:45:52 +00002540 if (!net_ratelimit())
2541 return;
2542
Bjørn Mork88ad4172015-11-16 19:16:40 +01002543 if (dev) {
2544 if (dev->dev.parent)
2545 name = dev_driver_string(dev->dev.parent);
2546 else
2547 name = netdev_name(dev);
2548 }
Ben Hutchings36c92472012-01-17 07:57:56 +00002549 WARN(1, "%s: caps=(%pNF, %pNF) len=%d data_len=%d gso_size=%d "
2550 "gso_type=%d ip_summed=%d\n",
Bjørn Mork88ad4172015-11-16 19:16:40 +01002551 name, dev ? &dev->features : &null_features,
Michał Mirosław65e9d2f2012-01-17 10:00:40 +00002552 skb->sk ? &skb->sk->sk_route_caps : &null_features,
Ben Hutchings36c92472012-01-17 07:57:56 +00002553 skb->len, skb->data_len, skb_shinfo(skb)->gso_size,
2554 skb_shinfo(skb)->gso_type, skb->ip_summed);
2555}
2556
Linus Torvalds1da177e2005-04-16 15:20:36 -07002557/*
2558 * Invalidate hardware checksum when packet is to be mangled, and
2559 * complete checksum manually on outgoing path.
2560 */
Patrick McHardy84fa7932006-08-29 16:44:56 -07002561int skb_checksum_help(struct sk_buff *skb)
Linus Torvalds1da177e2005-04-16 15:20:36 -07002562{
Al Virod3bc23e2006-11-14 21:24:49 -08002563 __wsum csum;
Herbert Xu663ead32007-04-09 11:59:07 -07002564 int ret = 0, offset;
Linus Torvalds1da177e2005-04-16 15:20:36 -07002565
Patrick McHardy84fa7932006-08-29 16:44:56 -07002566 if (skb->ip_summed == CHECKSUM_COMPLETE)
Herbert Xua430a432006-07-08 13:34:56 -07002567 goto out_set_summed;
2568
2569 if (unlikely(skb_shinfo(skb)->gso_size)) {
Ben Hutchings36c92472012-01-17 07:57:56 +00002570 skb_warn_bad_offload(skb);
2571 return -EINVAL;
Linus Torvalds1da177e2005-04-16 15:20:36 -07002572 }
2573
Eric Dumazetcef401d2013-01-25 20:34:37 +00002574 /* Before computing a checksum, we should make sure no frag could
2575 * be modified by an external entity : checksum could be wrong.
2576 */
2577 if (skb_has_shared_frag(skb)) {
2578 ret = __skb_linearize(skb);
2579 if (ret)
2580 goto out;
2581 }
2582
Michał Mirosław55508d62010-12-14 15:24:08 +00002583 offset = skb_checksum_start_offset(skb);
Herbert Xua0308472007-10-15 01:47:15 -07002584 BUG_ON(offset >= skb_headlen(skb));
2585 csum = skb_checksum(skb, offset, skb->len - offset, 0);
2586
2587 offset += skb->csum_offset;
2588 BUG_ON(offset + sizeof(__sum16) > skb_headlen(skb));
2589
2590 if (skb_cloned(skb) &&
2591 !skb_clone_writable(skb, offset + sizeof(__sum16))) {
Linus Torvalds1da177e2005-04-16 15:20:36 -07002592 ret = pskb_expand_head(skb, 0, 0, GFP_ATOMIC);
2593 if (ret)
2594 goto out;
2595 }
2596
Eric Dumazet4f2e4ad2016-10-29 11:02:36 -07002597 *(__sum16 *)(skb->data + offset) = csum_fold(csum) ?: CSUM_MANGLED_0;
Herbert Xua430a432006-07-08 13:34:56 -07002598out_set_summed:
Linus Torvalds1da177e2005-04-16 15:20:36 -07002599 skb->ip_summed = CHECKSUM_NONE;
YOSHIFUJI Hideaki4ec93ed2007-02-09 23:24:36 +09002600out:
Linus Torvalds1da177e2005-04-16 15:20:36 -07002601 return ret;
2602}
Eric Dumazetd1b19df2009-09-03 01:29:39 -07002603EXPORT_SYMBOL(skb_checksum_help);
Linus Torvalds1da177e2005-04-16 15:20:36 -07002604
Vlad Yasevich53d64712014-03-27 17:26:18 -04002605__be16 skb_network_protocol(struct sk_buff *skb, int *depth)
Pravin B Shelarec5f0612013-03-07 09:28:01 +00002606{
2607 __be16 type = skb->protocol;
2608
Pravin B Shelar19acc322013-05-07 20:41:07 +00002609 /* Tunnel gso handlers can set protocol to ethernet. */
2610 if (type == htons(ETH_P_TEB)) {
2611 struct ethhdr *eth;
2612
2613 if (unlikely(!pskb_may_pull(skb, sizeof(struct ethhdr))))
2614 return 0;
2615
2616 eth = (struct ethhdr *)skb_mac_header(skb);
2617 type = eth->h_proto;
2618 }
2619
Toshiaki Makitad4bcef32015-01-29 20:37:07 +09002620 return __vlan_get_protocol(skb, type, depth);
Pravin B Shelarec5f0612013-03-07 09:28:01 +00002621}
2622
Pravin B Shelar05e8ef42013-02-14 09:44:55 +00002623/**
2624 * skb_mac_gso_segment - mac layer segmentation handler.
2625 * @skb: buffer to segment
2626 * @features: features for the output path (see dev->features)
2627 */
2628struct sk_buff *skb_mac_gso_segment(struct sk_buff *skb,
2629 netdev_features_t features)
2630{
2631 struct sk_buff *segs = ERR_PTR(-EPROTONOSUPPORT);
2632 struct packet_offload *ptype;
Vlad Yasevich53d64712014-03-27 17:26:18 -04002633 int vlan_depth = skb->mac_len;
2634 __be16 type = skb_network_protocol(skb, &vlan_depth);
Pravin B Shelar05e8ef42013-02-14 09:44:55 +00002635
Pravin B Shelarec5f0612013-03-07 09:28:01 +00002636 if (unlikely(!type))
2637 return ERR_PTR(-EINVAL);
Pravin B Shelar05e8ef42013-02-14 09:44:55 +00002638
Vlad Yasevich53d64712014-03-27 17:26:18 -04002639 __skb_pull(skb, vlan_depth);
Pravin B Shelar05e8ef42013-02-14 09:44:55 +00002640
2641 rcu_read_lock();
2642 list_for_each_entry_rcu(ptype, &offload_base, list) {
2643 if (ptype->type == type && ptype->callbacks.gso_segment) {
Pravin B Shelar05e8ef42013-02-14 09:44:55 +00002644 segs = ptype->callbacks.gso_segment(skb, features);
2645 break;
2646 }
2647 }
2648 rcu_read_unlock();
2649
2650 __skb_push(skb, skb->data - skb_mac_header(skb));
2651
2652 return segs;
2653}
2654EXPORT_SYMBOL(skb_mac_gso_segment);
2655
2656
Cong Wang12b00042013-02-05 16:36:38 +00002657/* openvswitch calls this on rx path, so we need a different check.
2658 */
2659static inline bool skb_needs_check(struct sk_buff *skb, bool tx_path)
2660{
2661 if (tx_path)
2662 return skb->ip_summed != CHECKSUM_PARTIAL;
2663 else
2664 return skb->ip_summed == CHECKSUM_NONE;
2665}
2666
Herbert Xuf6a78bf2006-06-22 02:57:17 -07002667/**
Cong Wang12b00042013-02-05 16:36:38 +00002668 * __skb_gso_segment - Perform segmentation on skb.
Herbert Xuf6a78bf2006-06-22 02:57:17 -07002669 * @skb: buffer to segment
Herbert Xu576a30e2006-06-27 13:22:38 -07002670 * @features: features for the output path (see dev->features)
Cong Wang12b00042013-02-05 16:36:38 +00002671 * @tx_path: whether it is called in TX path
Herbert Xuf6a78bf2006-06-22 02:57:17 -07002672 *
2673 * This function segments the given skb and returns a list of segments.
Herbert Xu576a30e2006-06-27 13:22:38 -07002674 *
2675 * It may return NULL if the skb requires no segmentation. This is
2676 * only possible when GSO is used for verifying header integrity.
Konstantin Khlebnikov9207f9d2016-01-08 15:21:46 +03002677 *
2678 * Segmentation preserves SKB_SGO_CB_OFFSET bytes of previous skb cb.
Herbert Xuf6a78bf2006-06-22 02:57:17 -07002679 */
Cong Wang12b00042013-02-05 16:36:38 +00002680struct sk_buff *__skb_gso_segment(struct sk_buff *skb,
2681 netdev_features_t features, bool tx_path)
Herbert Xuf6a78bf2006-06-22 02:57:17 -07002682{
Cong Wang12b00042013-02-05 16:36:38 +00002683 if (unlikely(skb_needs_check(skb, tx_path))) {
Pravin B Shelar05e8ef42013-02-14 09:44:55 +00002684 int err;
2685
Ben Hutchings36c92472012-01-17 07:57:56 +00002686 skb_warn_bad_offload(skb);
Herbert Xu67fd1a72009-01-19 16:26:44 -08002687
françois romieua40e0a62014-07-15 23:55:35 +02002688 err = skb_cow_head(skb, 0);
2689 if (err < 0)
Herbert Xua430a432006-07-08 13:34:56 -07002690 return ERR_PTR(err);
2691 }
2692
Alexander Duyck802ab552016-04-10 21:45:03 -04002693 /* Only report GSO partial support if it will enable us to
2694 * support segmentation on this frame without needing additional
2695 * work.
2696 */
2697 if (features & NETIF_F_GSO_PARTIAL) {
2698 netdev_features_t partial_features = NETIF_F_GSO_ROBUST;
2699 struct net_device *dev = skb->dev;
2700
2701 partial_features |= dev->features & dev->gso_partial_features;
2702 if (!skb_gso_ok(skb, features | partial_features))
2703 features &= ~NETIF_F_GSO_PARTIAL;
2704 }
2705
Konstantin Khlebnikov9207f9d2016-01-08 15:21:46 +03002706 BUILD_BUG_ON(SKB_SGO_CB_OFFSET +
2707 sizeof(*SKB_GSO_CB(skb)) > sizeof(skb->cb));
2708
Pravin B Shelar68c33162013-02-14 14:02:41 +00002709 SKB_GSO_CB(skb)->mac_offset = skb_headroom(skb);
Eric Dumazet3347c962013-10-19 11:42:56 -07002710 SKB_GSO_CB(skb)->encap_level = 0;
2711
Pravin B Shelar05e8ef42013-02-14 09:44:55 +00002712 skb_reset_mac_header(skb);
2713 skb_reset_mac_len(skb);
Herbert Xuf6a78bf2006-06-22 02:57:17 -07002714
Pravin B Shelar05e8ef42013-02-14 09:44:55 +00002715 return skb_mac_gso_segment(skb, features);
Herbert Xuf6a78bf2006-06-22 02:57:17 -07002716}
Cong Wang12b00042013-02-05 16:36:38 +00002717EXPORT_SYMBOL(__skb_gso_segment);
Herbert Xuf6a78bf2006-06-22 02:57:17 -07002718
Herbert Xufb286bb2005-11-10 13:01:24 -08002719/* Take action when hardware reception checksum errors are detected. */
2720#ifdef CONFIG_BUG
2721void netdev_rx_csum_fault(struct net_device *dev)
2722{
2723 if (net_ratelimit()) {
Joe Perches7b6cd1c2012-02-01 10:54:43 +00002724 pr_err("%s: hw csum failure\n", dev ? dev->name : "<unknown>");
Herbert Xufb286bb2005-11-10 13:01:24 -08002725 dump_stack();
2726 }
2727}
2728EXPORT_SYMBOL(netdev_rx_csum_fault);
2729#endif
2730
Linus Torvalds1da177e2005-04-16 15:20:36 -07002731/* Actually, we should eliminate this check as soon as we know, that:
2732 * 1. IOMMU is present and allows to map all the memory.
2733 * 2. No high memory really exists on this machine.
2734 */
2735
Florian Westphalc1e756b2014-05-05 15:00:44 +02002736static int illegal_highdma(struct net_device *dev, struct sk_buff *skb)
Linus Torvalds1da177e2005-04-16 15:20:36 -07002737{
Herbert Xu3d3a8532006-06-27 13:33:10 -07002738#ifdef CONFIG_HIGHMEM
Linus Torvalds1da177e2005-04-16 15:20:36 -07002739 int i;
FUJITA Tomonori5acbbd42010-03-30 22:35:50 +00002740 if (!(dev->features & NETIF_F_HIGHDMA)) {
Ian Campbellea2ab692011-08-22 23:44:58 +00002741 for (i = 0; i < skb_shinfo(skb)->nr_frags; i++) {
2742 skb_frag_t *frag = &skb_shinfo(skb)->frags[i];
2743 if (PageHighMem(skb_frag_page(frag)))
FUJITA Tomonori5acbbd42010-03-30 22:35:50 +00002744 return 1;
Ian Campbellea2ab692011-08-22 23:44:58 +00002745 }
FUJITA Tomonori5acbbd42010-03-30 22:35:50 +00002746 }
Linus Torvalds1da177e2005-04-16 15:20:36 -07002747
FUJITA Tomonori5acbbd42010-03-30 22:35:50 +00002748 if (PCI_DMA_BUS_IS_PHYS) {
2749 struct device *pdev = dev->dev.parent;
Linus Torvalds1da177e2005-04-16 15:20:36 -07002750
Eric Dumazet9092c652010-04-02 13:34:49 -07002751 if (!pdev)
2752 return 0;
FUJITA Tomonori5acbbd42010-03-30 22:35:50 +00002753 for (i = 0; i < skb_shinfo(skb)->nr_frags; i++) {
Ian Campbellea2ab692011-08-22 23:44:58 +00002754 skb_frag_t *frag = &skb_shinfo(skb)->frags[i];
2755 dma_addr_t addr = page_to_phys(skb_frag_page(frag));
FUJITA Tomonori5acbbd42010-03-30 22:35:50 +00002756 if (!pdev->dma_mask || addr + PAGE_SIZE - 1 > *pdev->dma_mask)
2757 return 1;
2758 }
2759 }
Herbert Xu3d3a8532006-06-27 13:33:10 -07002760#endif
Linus Torvalds1da177e2005-04-16 15:20:36 -07002761 return 0;
2762}
Linus Torvalds1da177e2005-04-16 15:20:36 -07002763
Simon Horman3b392dd2014-06-04 08:53:17 +09002764/* If MPLS offload request, verify we are testing hardware MPLS features
2765 * instead of standard features for the netdev.
2766 */
Pravin B Shelard0edc7b2014-12-23 16:20:11 -08002767#if IS_ENABLED(CONFIG_NET_MPLS_GSO)
Simon Horman3b392dd2014-06-04 08:53:17 +09002768static netdev_features_t net_mpls_features(struct sk_buff *skb,
2769 netdev_features_t features,
2770 __be16 type)
2771{
Simon Horman25cd9ba2014-10-06 05:05:13 -07002772 if (eth_p_mpls(type))
Simon Horman3b392dd2014-06-04 08:53:17 +09002773 features &= skb->dev->mpls_features;
2774
2775 return features;
2776}
2777#else
2778static netdev_features_t net_mpls_features(struct sk_buff *skb,
2779 netdev_features_t features,
2780 __be16 type)
2781{
2782 return features;
2783}
2784#endif
2785
Michał Mirosławc8f44af2011-11-15 15:29:55 +00002786static netdev_features_t harmonize_features(struct sk_buff *skb,
Florian Westphalc1e756b2014-05-05 15:00:44 +02002787 netdev_features_t features)
Jesse Grossf01a5232011-01-09 06:23:31 +00002788{
Vlad Yasevich53d64712014-03-27 17:26:18 -04002789 int tmp;
Simon Horman3b392dd2014-06-04 08:53:17 +09002790 __be16 type;
2791
2792 type = skb_network_protocol(skb, &tmp);
2793 features = net_mpls_features(skb, features, type);
Vlad Yasevich53d64712014-03-27 17:26:18 -04002794
Ed Cashinc0d680e2012-09-19 15:49:00 +00002795 if (skb->ip_summed != CHECKSUM_NONE &&
Simon Horman3b392dd2014-06-04 08:53:17 +09002796 !can_checksum_protocol(features, type)) {
Alexander Duyck996e8022016-05-02 09:25:10 -07002797 features &= ~(NETIF_F_CSUM_MASK | NETIF_F_GSO_MASK);
Florian Westphalc1e756b2014-05-05 15:00:44 +02002798 } else if (illegal_highdma(skb->dev, skb)) {
Jesse Grossf01a5232011-01-09 06:23:31 +00002799 features &= ~NETIF_F_SG;
2800 }
2801
2802 return features;
2803}
2804
Toshiaki Makitae38f3022015-03-27 14:31:13 +09002805netdev_features_t passthru_features_check(struct sk_buff *skb,
2806 struct net_device *dev,
2807 netdev_features_t features)
2808{
2809 return features;
2810}
2811EXPORT_SYMBOL(passthru_features_check);
2812
Toshiaki Makita8cb65d02015-03-27 14:31:12 +09002813static netdev_features_t dflt_features_check(const struct sk_buff *skb,
2814 struct net_device *dev,
2815 netdev_features_t features)
2816{
2817 return vlan_features_check(skb, features);
2818}
2819
Alexander Duyckcbc53e02016-04-10 21:44:51 -04002820static netdev_features_t gso_features_check(const struct sk_buff *skb,
2821 struct net_device *dev,
2822 netdev_features_t features)
2823{
2824 u16 gso_segs = skb_shinfo(skb)->gso_segs;
2825
2826 if (gso_segs > dev->gso_max_segs)
2827 return features & ~NETIF_F_GSO_MASK;
2828
Alexander Duyck802ab552016-04-10 21:45:03 -04002829 /* Support for GSO partial features requires software
2830 * intervention before we can actually process the packets
2831 * so we need to strip support for any partial features now
2832 * and we can pull them back in after we have partially
2833 * segmented the frame.
2834 */
2835 if (!(skb_shinfo(skb)->gso_type & SKB_GSO_PARTIAL))
2836 features &= ~dev->gso_partial_features;
2837
2838 /* Make sure to clear the IPv4 ID mangling feature if the
2839 * IPv4 header has the potential to be fragmented.
Alexander Duyckcbc53e02016-04-10 21:44:51 -04002840 */
2841 if (skb_shinfo(skb)->gso_type & SKB_GSO_TCPV4) {
2842 struct iphdr *iph = skb->encapsulation ?
2843 inner_ip_hdr(skb) : ip_hdr(skb);
2844
2845 if (!(iph->frag_off & htons(IP_DF)))
2846 features &= ~NETIF_F_TSO_MANGLEID;
2847 }
2848
2849 return features;
2850}
2851
Florian Westphalc1e756b2014-05-05 15:00:44 +02002852netdev_features_t netif_skb_features(struct sk_buff *skb)
Jesse Gross58e998c2010-10-29 12:14:55 +00002853{
Jesse Gross5f352272014-12-23 22:37:26 -08002854 struct net_device *dev = skb->dev;
Eric Dumazetfcbeb972014-10-05 10:11:27 -07002855 netdev_features_t features = dev->features;
Jesse Gross58e998c2010-10-29 12:14:55 +00002856
Alexander Duyckcbc53e02016-04-10 21:44:51 -04002857 if (skb_is_gso(skb))
2858 features = gso_features_check(skb, dev, features);
Ben Hutchings30b678d2012-07-30 15:57:00 +00002859
Jesse Gross5f352272014-12-23 22:37:26 -08002860 /* If encapsulation offload request, verify we are testing
2861 * hardware encapsulation features instead of standard
2862 * features for the netdev
2863 */
2864 if (skb->encapsulation)
2865 features &= dev->hw_enc_features;
2866
Toshiaki Makitaf5a7fb82015-03-27 14:31:11 +09002867 if (skb_vlan_tagged(skb))
2868 features = netdev_intersect_features(features,
2869 dev->vlan_features |
2870 NETIF_F_HW_VLAN_CTAG_TX |
2871 NETIF_F_HW_VLAN_STAG_TX);
Jesse Gross58e998c2010-10-29 12:14:55 +00002872
Jesse Gross5f352272014-12-23 22:37:26 -08002873 if (dev->netdev_ops->ndo_features_check)
2874 features &= dev->netdev_ops->ndo_features_check(skb, dev,
2875 features);
Toshiaki Makita8cb65d02015-03-27 14:31:12 +09002876 else
2877 features &= dflt_features_check(skb, dev, features);
Jesse Gross5f352272014-12-23 22:37:26 -08002878
Florian Westphalc1e756b2014-05-05 15:00:44 +02002879 return harmonize_features(skb, features);
Jesse Gross58e998c2010-10-29 12:14:55 +00002880}
Florian Westphalc1e756b2014-05-05 15:00:44 +02002881EXPORT_SYMBOL(netif_skb_features);
Jesse Gross58e998c2010-10-29 12:14:55 +00002882
David S. Miller2ea25512014-08-29 21:10:01 -07002883static int xmit_one(struct sk_buff *skb, struct net_device *dev,
David S. Miller95f6b3d2014-08-29 21:57:30 -07002884 struct netdev_queue *txq, bool more)
Herbert Xuf6a78bf2006-06-22 02:57:17 -07002885{
David S. Miller2ea25512014-08-29 21:10:01 -07002886 unsigned int len;
2887 int rc;
Stephen Hemminger00829822008-11-20 20:14:53 -08002888
Salam Noureddine7866a622015-01-27 11:35:48 -08002889 if (!list_empty(&ptype_all) || !list_empty(&dev->ptype_all))
David S. Miller2ea25512014-08-29 21:10:01 -07002890 dev_queue_xmit_nit(skb, dev);
Jesse Grossfc741212011-01-09 06:23:32 +00002891
David S. Miller2ea25512014-08-29 21:10:01 -07002892 len = skb->len;
2893 trace_net_dev_start_xmit(skb, dev);
David S. Miller95f6b3d2014-08-29 21:57:30 -07002894 rc = netdev_start_xmit(skb, dev, txq, more);
David S. Miller2ea25512014-08-29 21:10:01 -07002895 trace_net_dev_xmit(skb, rc, dev, len);
Eric Dumazetadf30902009-06-02 05:19:30 +00002896
Patrick McHardy572a9d72009-11-10 06:14:14 +00002897 return rc;
Herbert Xuf6a78bf2006-06-22 02:57:17 -07002898}
David S. Miller2ea25512014-08-29 21:10:01 -07002899
David S. Miller8dcda222014-09-01 15:06:40 -07002900struct sk_buff *dev_hard_start_xmit(struct sk_buff *first, struct net_device *dev,
2901 struct netdev_queue *txq, int *ret)
David S. Miller7f2e8702014-08-29 21:19:14 -07002902{
2903 struct sk_buff *skb = first;
2904 int rc = NETDEV_TX_OK;
2905
2906 while (skb) {
2907 struct sk_buff *next = skb->next;
2908
2909 skb->next = NULL;
David S. Miller95f6b3d2014-08-29 21:57:30 -07002910 rc = xmit_one(skb, dev, txq, next != NULL);
David S. Miller7f2e8702014-08-29 21:19:14 -07002911 if (unlikely(!dev_xmit_complete(rc))) {
2912 skb->next = next;
2913 goto out;
2914 }
2915
2916 skb = next;
2917 if (netif_xmit_stopped(txq) && skb) {
2918 rc = NETDEV_TX_BUSY;
2919 break;
2920 }
2921 }
2922
2923out:
2924 *ret = rc;
2925 return skb;
2926}
2927
Eric Dumazet1ff0dc92014-10-06 11:26:27 -07002928static struct sk_buff *validate_xmit_vlan(struct sk_buff *skb,
2929 netdev_features_t features)
David S. Millereae3f882014-08-30 15:17:13 -07002930{
Jiri Pirkodf8a39d2015-01-13 17:13:44 +01002931 if (skb_vlan_tag_present(skb) &&
Jiri Pirko59682502014-11-19 14:04:59 +01002932 !vlan_hw_offload_capable(features, skb->vlan_proto))
2933 skb = __vlan_hwaccel_push_inside(skb);
David S. Millereae3f882014-08-30 15:17:13 -07002934 return skb;
2935}
2936
Eric Dumazet55a93b32014-10-03 15:31:07 -07002937static struct sk_buff *validate_xmit_skb(struct sk_buff *skb, struct net_device *dev)
David S. Millereae3f882014-08-30 15:17:13 -07002938{
2939 netdev_features_t features;
2940
David S. Millereae3f882014-08-30 15:17:13 -07002941 features = netif_skb_features(skb);
2942 skb = validate_xmit_vlan(skb, features);
2943 if (unlikely(!skb))
2944 goto out_null;
2945
Johannes Berg8b86a612015-04-17 15:45:04 +02002946 if (netif_needs_gso(skb, features)) {
David S. Millerce937182014-08-30 19:22:20 -07002947 struct sk_buff *segs;
2948
2949 segs = skb_gso_segment(skb, features);
Jason Wangcecda692014-09-19 16:04:38 +08002950 if (IS_ERR(segs)) {
Jason Wangaf6dabc2014-12-19 11:09:13 +08002951 goto out_kfree_skb;
Jason Wangcecda692014-09-19 16:04:38 +08002952 } else if (segs) {
2953 consume_skb(skb);
2954 skb = segs;
2955 }
David S. Millereae3f882014-08-30 15:17:13 -07002956 } else {
2957 if (skb_needs_linearize(skb, features) &&
2958 __skb_linearize(skb))
2959 goto out_kfree_skb;
2960
2961 /* If packet is not checksummed and device does not
2962 * support checksumming for this protocol, complete
2963 * checksumming here.
2964 */
2965 if (skb->ip_summed == CHECKSUM_PARTIAL) {
2966 if (skb->encapsulation)
2967 skb_set_inner_transport_header(skb,
2968 skb_checksum_start_offset(skb));
2969 else
2970 skb_set_transport_header(skb,
2971 skb_checksum_start_offset(skb));
Tom Herberta1882222015-12-14 11:19:43 -08002972 if (!(features & NETIF_F_CSUM_MASK) &&
David S. Millereae3f882014-08-30 15:17:13 -07002973 skb_checksum_help(skb))
2974 goto out_kfree_skb;
2975 }
2976 }
2977
2978 return skb;
2979
2980out_kfree_skb:
2981 kfree_skb(skb);
2982out_null:
Eric Dumazetd21fd632016-04-12 21:50:07 -07002983 atomic_long_inc(&dev->tx_dropped);
David S. Millereae3f882014-08-30 15:17:13 -07002984 return NULL;
2985}
2986
Eric Dumazet55a93b32014-10-03 15:31:07 -07002987struct sk_buff *validate_xmit_skb_list(struct sk_buff *skb, struct net_device *dev)
2988{
2989 struct sk_buff *next, *head = NULL, *tail;
2990
Eric Dumazetbec3cfd2014-10-03 20:59:19 -07002991 for (; skb != NULL; skb = next) {
Eric Dumazet55a93b32014-10-03 15:31:07 -07002992 next = skb->next;
2993 skb->next = NULL;
Eric Dumazet55a93b32014-10-03 15:31:07 -07002994
Eric Dumazetbec3cfd2014-10-03 20:59:19 -07002995 /* in case skb wont be segmented, point to itself */
2996 skb->prev = skb;
2997
2998 skb = validate_xmit_skb(skb, dev);
2999 if (!skb)
3000 continue;
3001
3002 if (!head)
3003 head = skb;
3004 else
3005 tail->next = skb;
3006 /* If skb was segmented, skb->prev points to
3007 * the last segment. If not, it still contains skb.
3008 */
3009 tail = skb->prev;
Eric Dumazet55a93b32014-10-03 15:31:07 -07003010 }
3011 return head;
3012}
Willem de Bruijn104ba782016-10-26 11:23:07 -04003013EXPORT_SYMBOL_GPL(validate_xmit_skb_list);
Herbert Xuf6a78bf2006-06-22 02:57:17 -07003014
Eric Dumazet1def9232013-01-10 12:36:42 +00003015static void qdisc_pkt_len_init(struct sk_buff *skb)
3016{
3017 const struct skb_shared_info *shinfo = skb_shinfo(skb);
3018
3019 qdisc_skb_cb(skb)->pkt_len = skb->len;
3020
3021 /* To get more precise estimation of bytes sent on wire,
3022 * we add to pkt_len the headers size of all segments
3023 */
3024 if (shinfo->gso_size) {
Eric Dumazet757b8b12013-01-15 21:14:21 -08003025 unsigned int hdr_len;
Jason Wang15e5a032013-03-25 20:19:59 +00003026 u16 gso_segs = shinfo->gso_segs;
Eric Dumazet1def9232013-01-10 12:36:42 +00003027
Eric Dumazet757b8b12013-01-15 21:14:21 -08003028 /* mac layer + network layer */
3029 hdr_len = skb_transport_header(skb) - skb_mac_header(skb);
3030
3031 /* + transport layer */
Eric Dumazet1def9232013-01-10 12:36:42 +00003032 if (likely(shinfo->gso_type & (SKB_GSO_TCPV4 | SKB_GSO_TCPV6)))
3033 hdr_len += tcp_hdrlen(skb);
3034 else
3035 hdr_len += sizeof(struct udphdr);
Jason Wang15e5a032013-03-25 20:19:59 +00003036
3037 if (shinfo->gso_type & SKB_GSO_DODGY)
3038 gso_segs = DIV_ROUND_UP(skb->len - hdr_len,
3039 shinfo->gso_size);
3040
3041 qdisc_skb_cb(skb)->pkt_len += (gso_segs - 1) * hdr_len;
Eric Dumazet1def9232013-01-10 12:36:42 +00003042 }
3043}
3044
Krishna Kumarbbd8a0d2009-08-06 01:44:21 +00003045static inline int __dev_xmit_skb(struct sk_buff *skb, struct Qdisc *q,
3046 struct net_device *dev,
3047 struct netdev_queue *txq)
3048{
3049 spinlock_t *root_lock = qdisc_lock(q);
Eric Dumazet520ac302016-06-21 23:16:49 -07003050 struct sk_buff *to_free = NULL;
Eric Dumazeta2da5702011-01-20 03:48:19 +00003051 bool contended;
Krishna Kumarbbd8a0d2009-08-06 01:44:21 +00003052 int rc;
3053
Eric Dumazeta2da5702011-01-20 03:48:19 +00003054 qdisc_calculate_pkt_len(skb, q);
Eric Dumazet79640a42010-06-02 05:09:29 -07003055 /*
3056 * Heuristic to force contended enqueues to serialize on a
3057 * separate lock before trying to get qdisc main lock.
Eric Dumazetf9eb8ae2016-06-06 09:37:15 -07003058 * This permits qdisc->running owner to get the lock more
Ying Xue9bf2b8c2014-06-26 15:56:31 +08003059 * often and dequeue packets faster.
Eric Dumazet79640a42010-06-02 05:09:29 -07003060 */
Eric Dumazeta2da5702011-01-20 03:48:19 +00003061 contended = qdisc_is_running(q);
Eric Dumazet79640a42010-06-02 05:09:29 -07003062 if (unlikely(contended))
3063 spin_lock(&q->busylock);
3064
Krishna Kumarbbd8a0d2009-08-06 01:44:21 +00003065 spin_lock(root_lock);
3066 if (unlikely(test_bit(__QDISC_STATE_DEACTIVATED, &q->state))) {
Eric Dumazet520ac302016-06-21 23:16:49 -07003067 __qdisc_drop(skb, &to_free);
Krishna Kumarbbd8a0d2009-08-06 01:44:21 +00003068 rc = NET_XMIT_DROP;
3069 } else if ((q->flags & TCQ_F_CAN_BYPASS) && !qdisc_qlen(q) &&
Eric Dumazetbc135b22010-06-02 03:23:51 -07003070 qdisc_run_begin(q)) {
Krishna Kumarbbd8a0d2009-08-06 01:44:21 +00003071 /*
3072 * This is a work-conserving queue; there are no old skbs
3073 * waiting to be sent out; and the qdisc is not running -
3074 * xmit the skb directly.
3075 */
Eric Dumazetbfe0d022011-01-09 08:30:54 +00003076
Eric Dumazetbfe0d022011-01-09 08:30:54 +00003077 qdisc_bstats_update(q, skb);
3078
Eric Dumazet55a93b32014-10-03 15:31:07 -07003079 if (sch_direct_xmit(skb, q, dev, txq, root_lock, true)) {
Eric Dumazet79640a42010-06-02 05:09:29 -07003080 if (unlikely(contended)) {
3081 spin_unlock(&q->busylock);
3082 contended = false;
3083 }
Krishna Kumarbbd8a0d2009-08-06 01:44:21 +00003084 __qdisc_run(q);
Eric Dumazet79640a42010-06-02 05:09:29 -07003085 } else
Eric Dumazetbc135b22010-06-02 03:23:51 -07003086 qdisc_run_end(q);
Krishna Kumarbbd8a0d2009-08-06 01:44:21 +00003087
3088 rc = NET_XMIT_SUCCESS;
3089 } else {
Eric Dumazet520ac302016-06-21 23:16:49 -07003090 rc = q->enqueue(skb, q, &to_free) & NET_XMIT_MASK;
Eric Dumazet79640a42010-06-02 05:09:29 -07003091 if (qdisc_run_begin(q)) {
3092 if (unlikely(contended)) {
3093 spin_unlock(&q->busylock);
3094 contended = false;
3095 }
3096 __qdisc_run(q);
3097 }
Krishna Kumarbbd8a0d2009-08-06 01:44:21 +00003098 }
3099 spin_unlock(root_lock);
Eric Dumazet520ac302016-06-21 23:16:49 -07003100 if (unlikely(to_free))
3101 kfree_skb_list(to_free);
Eric Dumazet79640a42010-06-02 05:09:29 -07003102 if (unlikely(contended))
3103 spin_unlock(&q->busylock);
Krishna Kumarbbd8a0d2009-08-06 01:44:21 +00003104 return rc;
3105}
3106
Daniel Borkmann86f85152013-12-29 17:27:11 +01003107#if IS_ENABLED(CONFIG_CGROUP_NET_PRIO)
Neil Horman5bc14212011-11-22 05:10:51 +00003108static void skb_update_prio(struct sk_buff *skb)
3109{
Igor Maravic6977a792011-11-25 07:44:54 +00003110 struct netprio_map *map = rcu_dereference_bh(skb->dev->priomap);
Neil Horman5bc14212011-11-22 05:10:51 +00003111
Eric Dumazet91c68ce2012-07-08 21:45:10 +00003112 if (!skb->priority && skb->sk && map) {
Tejun Heo2a56a1f2015-12-07 17:38:52 -05003113 unsigned int prioidx =
3114 sock_cgroup_prioidx(&skb->sk->sk_cgrp_data);
Eric Dumazet91c68ce2012-07-08 21:45:10 +00003115
3116 if (prioidx < map->priomap_len)
3117 skb->priority = map->priomap[prioidx];
3118 }
Neil Horman5bc14212011-11-22 05:10:51 +00003119}
3120#else
3121#define skb_update_prio(skb)
3122#endif
3123
hannes@stressinduktion.orgf60e5992015-04-01 17:07:44 +02003124DEFINE_PER_CPU(int, xmit_recursion);
3125EXPORT_SYMBOL(xmit_recursion);
3126
Dave Jonesd29f7492008-07-22 14:09:06 -07003127/**
Michel Machado95603e22012-06-12 10:16:35 +00003128 * dev_loopback_xmit - loop back @skb
Eric W. Biederman0c4b51f2015-09-15 20:04:18 -05003129 * @net: network namespace this loopback is happening in
3130 * @sk: sk needed to be a netfilter okfn
Michel Machado95603e22012-06-12 10:16:35 +00003131 * @skb: buffer to transmit
3132 */
Eric W. Biederman0c4b51f2015-09-15 20:04:18 -05003133int dev_loopback_xmit(struct net *net, struct sock *sk, struct sk_buff *skb)
Michel Machado95603e22012-06-12 10:16:35 +00003134{
3135 skb_reset_mac_header(skb);
3136 __skb_pull(skb, skb_network_offset(skb));
3137 skb->pkt_type = PACKET_LOOPBACK;
3138 skb->ip_summed = CHECKSUM_UNNECESSARY;
3139 WARN_ON(!skb_dst(skb));
3140 skb_dst_force(skb);
3141 netif_rx_ni(skb);
3142 return 0;
3143}
3144EXPORT_SYMBOL(dev_loopback_xmit);
3145
Daniel Borkmann1f211a12016-01-07 22:29:47 +01003146#ifdef CONFIG_NET_EGRESS
3147static struct sk_buff *
3148sch_handle_egress(struct sk_buff *skb, int *ret, struct net_device *dev)
3149{
3150 struct tcf_proto *cl = rcu_dereference_bh(dev->egress_cl_list);
3151 struct tcf_result cl_res;
3152
3153 if (!cl)
3154 return skb;
3155
3156 /* skb->tc_verd and qdisc_skb_cb(skb)->pkt_len were already set
3157 * earlier by the caller.
3158 */
3159 qdisc_bstats_cpu_update(cl->q, skb);
3160
3161 switch (tc_classify(skb, cl, &cl_res, false)) {
3162 case TC_ACT_OK:
3163 case TC_ACT_RECLASSIFY:
3164 skb->tc_index = TC_H_MIN(cl_res.classid);
3165 break;
3166 case TC_ACT_SHOT:
3167 qdisc_qstats_cpu_drop(cl->q);
3168 *ret = NET_XMIT_DROP;
Daniel Borkmann7e2c3ae2016-05-15 23:28:29 +02003169 kfree_skb(skb);
3170 return NULL;
Daniel Borkmann1f211a12016-01-07 22:29:47 +01003171 case TC_ACT_STOLEN:
3172 case TC_ACT_QUEUED:
3173 *ret = NET_XMIT_SUCCESS;
Daniel Borkmann7e2c3ae2016-05-15 23:28:29 +02003174 consume_skb(skb);
Daniel Borkmann1f211a12016-01-07 22:29:47 +01003175 return NULL;
3176 case TC_ACT_REDIRECT:
3177 /* No need to push/pop skb's mac_header here on egress! */
3178 skb_do_redirect(skb);
3179 *ret = NET_XMIT_SUCCESS;
3180 return NULL;
3181 default:
3182 break;
3183 }
3184
3185 return skb;
3186}
3187#endif /* CONFIG_NET_EGRESS */
3188
Jiri Pirko638b2a62015-05-12 14:56:13 +02003189static inline int get_xps_queue(struct net_device *dev, struct sk_buff *skb)
3190{
3191#ifdef CONFIG_XPS
3192 struct xps_dev_maps *dev_maps;
3193 struct xps_map *map;
3194 int queue_index = -1;
3195
3196 rcu_read_lock();
3197 dev_maps = rcu_dereference(dev->xps_maps);
3198 if (dev_maps) {
Alexander Duyck184c4492016-10-28 11:50:13 -04003199 unsigned int tci = skb->sender_cpu - 1;
3200
3201 if (dev->num_tc) {
3202 tci *= dev->num_tc;
3203 tci += netdev_get_prio_tc_map(dev, skb->priority);
3204 }
3205
3206 map = rcu_dereference(dev_maps->cpu_map[tci]);
Jiri Pirko638b2a62015-05-12 14:56:13 +02003207 if (map) {
3208 if (map->len == 1)
3209 queue_index = map->queues[0];
3210 else
3211 queue_index = map->queues[reciprocal_scale(skb_get_hash(skb),
3212 map->len)];
3213 if (unlikely(queue_index >= dev->real_num_tx_queues))
3214 queue_index = -1;
3215 }
3216 }
3217 rcu_read_unlock();
3218
3219 return queue_index;
3220#else
3221 return -1;
3222#endif
3223}
3224
3225static u16 __netdev_pick_tx(struct net_device *dev, struct sk_buff *skb)
3226{
3227 struct sock *sk = skb->sk;
3228 int queue_index = sk_tx_queue_get(sk);
3229
3230 if (queue_index < 0 || skb->ooo_okay ||
3231 queue_index >= dev->real_num_tx_queues) {
3232 int new_index = get_xps_queue(dev, skb);
3233 if (new_index < 0)
3234 new_index = skb_tx_hash(dev, skb);
3235
3236 if (queue_index != new_index && sk &&
Eric Dumazet004a5d02015-10-04 21:08:10 -07003237 sk_fullsock(sk) &&
Jiri Pirko638b2a62015-05-12 14:56:13 +02003238 rcu_access_pointer(sk->sk_dst_cache))
3239 sk_tx_queue_set(sk, new_index);
3240
3241 queue_index = new_index;
3242 }
3243
3244 return queue_index;
3245}
3246
3247struct netdev_queue *netdev_pick_tx(struct net_device *dev,
3248 struct sk_buff *skb,
3249 void *accel_priv)
3250{
3251 int queue_index = 0;
3252
3253#ifdef CONFIG_XPS
Eric Dumazet52bd2d62015-11-18 06:30:50 -08003254 u32 sender_cpu = skb->sender_cpu - 1;
3255
3256 if (sender_cpu >= (u32)NR_CPUS)
Jiri Pirko638b2a62015-05-12 14:56:13 +02003257 skb->sender_cpu = raw_smp_processor_id() + 1;
3258#endif
3259
3260 if (dev->real_num_tx_queues != 1) {
3261 const struct net_device_ops *ops = dev->netdev_ops;
3262 if (ops->ndo_select_queue)
3263 queue_index = ops->ndo_select_queue(dev, skb, accel_priv,
3264 __netdev_pick_tx);
3265 else
3266 queue_index = __netdev_pick_tx(dev, skb);
3267
3268 if (!accel_priv)
3269 queue_index = netdev_cap_txqueue(dev, queue_index);
3270 }
3271
3272 skb_set_queue_mapping(skb, queue_index);
3273 return netdev_get_tx_queue(dev, queue_index);
3274}
3275
Michel Machado95603e22012-06-12 10:16:35 +00003276/**
Jason Wang9d08dd32014-01-20 11:25:13 +08003277 * __dev_queue_xmit - transmit a buffer
Dave Jonesd29f7492008-07-22 14:09:06 -07003278 * @skb: buffer to transmit
Jason Wang9d08dd32014-01-20 11:25:13 +08003279 * @accel_priv: private data used for L2 forwarding offload
Dave Jonesd29f7492008-07-22 14:09:06 -07003280 *
3281 * Queue a buffer for transmission to a network device. The caller must
3282 * have set the device and priority and built the buffer before calling
3283 * this function. The function can be called from an interrupt.
3284 *
3285 * A negative errno code is returned on a failure. A success does not
3286 * guarantee the frame will be transmitted as it may be dropped due
3287 * to congestion or traffic shaping.
3288 *
3289 * -----------------------------------------------------------------------------------
3290 * I notice this method can also return errors from the queue disciplines,
3291 * including NET_XMIT_DROP, which is a positive value. So, errors can also
3292 * be positive.
3293 *
3294 * Regardless of the return value, the skb is consumed, so it is currently
3295 * difficult to retry a send to this method. (You can bump the ref count
3296 * before sending to hold a reference for retry if you are careful.)
3297 *
3298 * When calling this method, interrupts MUST be enabled. This is because
3299 * the BH enable code must have IRQs enabled so that it will not deadlock.
3300 * --BLG
3301 */
Rashika Kheria0a59f3a2014-02-09 20:26:25 +05303302static int __dev_queue_xmit(struct sk_buff *skb, void *accel_priv)
Linus Torvalds1da177e2005-04-16 15:20:36 -07003303{
3304 struct net_device *dev = skb->dev;
David S. Millerdc2b4842008-07-08 17:18:23 -07003305 struct netdev_queue *txq;
Linus Torvalds1da177e2005-04-16 15:20:36 -07003306 struct Qdisc *q;
3307 int rc = -ENOMEM;
3308
Eric Dumazet6d1ccff2013-02-05 20:22:20 +00003309 skb_reset_mac_header(skb);
3310
Willem de Bruijne7fd2882014-08-04 22:11:48 -04003311 if (unlikely(skb_shinfo(skb)->tx_flags & SKBTX_SCHED_TSTAMP))
3312 __skb_tstamp_tx(skb, NULL, skb->sk, SCM_TSTAMP_SCHED);
3313
YOSHIFUJI Hideaki4ec93ed2007-02-09 23:24:36 +09003314 /* Disable soft irqs for various locks below. Also
3315 * stops preemption for RCU.
Linus Torvalds1da177e2005-04-16 15:20:36 -07003316 */
YOSHIFUJI Hideaki4ec93ed2007-02-09 23:24:36 +09003317 rcu_read_lock_bh();
Linus Torvalds1da177e2005-04-16 15:20:36 -07003318
Neil Horman5bc14212011-11-22 05:10:51 +00003319 skb_update_prio(skb);
3320
Daniel Borkmann1f211a12016-01-07 22:29:47 +01003321 qdisc_pkt_len_init(skb);
3322#ifdef CONFIG_NET_CLS_ACT
3323 skb->tc_verd = SET_TC_AT(skb->tc_verd, AT_EGRESS);
3324# ifdef CONFIG_NET_EGRESS
3325 if (static_key_false(&egress_needed)) {
3326 skb = sch_handle_egress(skb, &rc, dev);
3327 if (!skb)
3328 goto out;
3329 }
3330# endif
3331#endif
Eric Dumazet02875872014-10-05 18:38:35 -07003332 /* If device/qdisc don't need skb->dst, release it right now while
3333 * its hot in this cpu cache.
3334 */
3335 if (dev->priv_flags & IFF_XMIT_DST_RELEASE)
3336 skb_dst_drop(skb);
3337 else
3338 skb_dst_force(skb);
3339
Jason Wangf663dd92014-01-10 16:18:26 +08003340 txq = netdev_pick_tx(dev, skb, accel_priv);
Paul E. McKenneya898def2010-02-22 17:04:49 -08003341 q = rcu_dereference_bh(txq->qdisc);
David S. Miller37437bb2008-07-16 02:15:04 -07003342
Koki Sanagicf66ba52010-08-23 18:45:02 +09003343 trace_net_dev_queue(skb);
Linus Torvalds1da177e2005-04-16 15:20:36 -07003344 if (q->enqueue) {
Krishna Kumarbbd8a0d2009-08-06 01:44:21 +00003345 rc = __dev_xmit_skb(skb, q, dev, txq);
David S. Miller37437bb2008-07-16 02:15:04 -07003346 goto out;
Linus Torvalds1da177e2005-04-16 15:20:36 -07003347 }
3348
3349 /* The device has no queue. Common case for software devices:
3350 loopback, all the sorts of tunnels...
3351
Herbert Xu932ff272006-06-09 12:20:56 -07003352 Really, it is unlikely that netif_tx_lock protection is necessary
3353 here. (f.e. loopback and IP tunnels are clean ignoring statistics
Linus Torvalds1da177e2005-04-16 15:20:36 -07003354 counters.)
3355 However, it is possible, that they rely on protection
3356 made by us here.
3357
3358 Check this and shot the lock. It is not prone from deadlocks.
3359 Either shot noqueue qdisc, it is even simpler 8)
3360 */
3361 if (dev->flags & IFF_UP) {
3362 int cpu = smp_processor_id(); /* ok because BHs are off */
3363
David S. Millerc773e842008-07-08 23:13:53 -07003364 if (txq->xmit_lock_owner != cpu) {
Daniel Borkmanna70b5062016-06-10 21:19:06 +02003365 if (unlikely(__this_cpu_read(xmit_recursion) >
3366 XMIT_RECURSION_LIMIT))
Eric Dumazet745e20f2010-09-29 13:23:09 -07003367 goto recursion_alert;
3368
Jesper Dangaard Brouer1f595332014-09-03 17:56:09 +02003369 skb = validate_xmit_skb(skb, dev);
3370 if (!skb)
Eric Dumazetd21fd632016-04-12 21:50:07 -07003371 goto out;
Jesper Dangaard Brouer1f595332014-09-03 17:56:09 +02003372
David S. Millerc773e842008-07-08 23:13:53 -07003373 HARD_TX_LOCK(dev, txq, cpu);
Linus Torvalds1da177e2005-04-16 15:20:36 -07003374
Tom Herbert734664982011-11-28 16:32:44 +00003375 if (!netif_xmit_stopped(txq)) {
Eric Dumazet745e20f2010-09-29 13:23:09 -07003376 __this_cpu_inc(xmit_recursion);
David S. Millerce937182014-08-30 19:22:20 -07003377 skb = dev_hard_start_xmit(skb, dev, txq, &rc);
Eric Dumazet745e20f2010-09-29 13:23:09 -07003378 __this_cpu_dec(xmit_recursion);
Patrick McHardy572a9d72009-11-10 06:14:14 +00003379 if (dev_xmit_complete(rc)) {
David S. Millerc773e842008-07-08 23:13:53 -07003380 HARD_TX_UNLOCK(dev, txq);
Linus Torvalds1da177e2005-04-16 15:20:36 -07003381 goto out;
3382 }
3383 }
David S. Millerc773e842008-07-08 23:13:53 -07003384 HARD_TX_UNLOCK(dev, txq);
Joe Perchese87cc472012-05-13 21:56:26 +00003385 net_crit_ratelimited("Virtual device %s asks to queue packet!\n",
3386 dev->name);
Linus Torvalds1da177e2005-04-16 15:20:36 -07003387 } else {
3388 /* Recursion is detected! It is possible,
Eric Dumazet745e20f2010-09-29 13:23:09 -07003389 * unfortunately
3390 */
3391recursion_alert:
Joe Perchese87cc472012-05-13 21:56:26 +00003392 net_crit_ratelimited("Dead loop on virtual device %s, fix it urgently!\n",
3393 dev->name);
Linus Torvalds1da177e2005-04-16 15:20:36 -07003394 }
3395 }
3396
3397 rc = -ENETDOWN;
Herbert Xud4828d82006-06-22 02:28:18 -07003398 rcu_read_unlock_bh();
Linus Torvalds1da177e2005-04-16 15:20:36 -07003399
Eric Dumazet015f0682014-03-27 08:45:56 -07003400 atomic_long_inc(&dev->tx_dropped);
Jesper Dangaard Brouer1f595332014-09-03 17:56:09 +02003401 kfree_skb_list(skb);
Linus Torvalds1da177e2005-04-16 15:20:36 -07003402 return rc;
3403out:
Herbert Xud4828d82006-06-22 02:28:18 -07003404 rcu_read_unlock_bh();
Linus Torvalds1da177e2005-04-16 15:20:36 -07003405 return rc;
3406}
Jason Wangf663dd92014-01-10 16:18:26 +08003407
Eric W. Biederman2b4aa3c2015-09-15 20:04:07 -05003408int dev_queue_xmit(struct sk_buff *skb)
Jason Wangf663dd92014-01-10 16:18:26 +08003409{
3410 return __dev_queue_xmit(skb, NULL);
3411}
Eric W. Biederman2b4aa3c2015-09-15 20:04:07 -05003412EXPORT_SYMBOL(dev_queue_xmit);
Linus Torvalds1da177e2005-04-16 15:20:36 -07003413
Jason Wangf663dd92014-01-10 16:18:26 +08003414int dev_queue_xmit_accel(struct sk_buff *skb, void *accel_priv)
3415{
3416 return __dev_queue_xmit(skb, accel_priv);
3417}
3418EXPORT_SYMBOL(dev_queue_xmit_accel);
3419
Linus Torvalds1da177e2005-04-16 15:20:36 -07003420
3421/*=======================================================================
3422 Receiver routines
3423 =======================================================================*/
3424
Stephen Hemminger6b2bedc2007-03-12 14:33:50 -07003425int netdev_max_backlog __read_mostly = 1000;
Eric Dumazetc9e6bc62012-09-27 19:29:05 +00003426EXPORT_SYMBOL(netdev_max_backlog);
3427
Eric Dumazet3b098e22010-05-15 23:57:10 -07003428int netdev_tstamp_prequeue __read_mostly = 1;
Stephen Hemminger6b2bedc2007-03-12 14:33:50 -07003429int netdev_budget __read_mostly = 300;
3430int weight_p __read_mostly = 64; /* old backlog weight */
Linus Torvalds1da177e2005-04-16 15:20:36 -07003431
Eric Dumazeteecfd7c2010-05-06 22:07:48 -07003432/* Called with irq disabled */
3433static inline void ____napi_schedule(struct softnet_data *sd,
3434 struct napi_struct *napi)
3435{
3436 list_add_tail(&napi->poll_list, &sd->poll_list);
3437 __raise_softirq_irqoff(NET_RX_SOFTIRQ);
3438}
3439
Eric Dumazetdf334542010-03-24 19:13:54 +00003440#ifdef CONFIG_RPS
Tom Herbertfec5e652010-04-16 16:01:27 -07003441
3442/* One global table that all flow-based protocols share. */
Eric Dumazet6e3f7fa2010-10-25 03:02:02 +00003443struct rps_sock_flow_table __rcu *rps_sock_flow_table __read_mostly;
Tom Herbertfec5e652010-04-16 16:01:27 -07003444EXPORT_SYMBOL(rps_sock_flow_table);
Eric Dumazet567e4b72015-02-06 12:59:01 -08003445u32 rps_cpu_mask __read_mostly;
3446EXPORT_SYMBOL(rps_cpu_mask);
Tom Herbertfec5e652010-04-16 16:01:27 -07003447
Ingo Molnarc5905af2012-02-24 08:31:31 +01003448struct static_key rps_needed __read_mostly;
Jason Wang3df97ba2016-04-25 23:13:42 -04003449EXPORT_SYMBOL(rps_needed);
Eric Dumazet13bfff22016-12-07 08:29:10 -08003450struct static_key rfs_needed __read_mostly;
3451EXPORT_SYMBOL(rfs_needed);
Eric Dumazetadc93002011-11-17 03:13:26 +00003452
Ben Hutchingsc4454772011-01-19 11:03:53 +00003453static struct rps_dev_flow *
3454set_rps_cpu(struct net_device *dev, struct sk_buff *skb,
3455 struct rps_dev_flow *rflow, u16 next_cpu)
3456{
Eric Dumazeta31196b2015-04-25 09:35:24 -07003457 if (next_cpu < nr_cpu_ids) {
Ben Hutchingsc4454772011-01-19 11:03:53 +00003458#ifdef CONFIG_RFS_ACCEL
3459 struct netdev_rx_queue *rxqueue;
3460 struct rps_dev_flow_table *flow_table;
3461 struct rps_dev_flow *old_rflow;
3462 u32 flow_id;
3463 u16 rxq_index;
3464 int rc;
3465
3466 /* Should we steer this flow to a different hardware queue? */
Ben Hutchings69a19ee2011-02-15 20:32:04 +00003467 if (!skb_rx_queue_recorded(skb) || !dev->rx_cpu_rmap ||
3468 !(dev->features & NETIF_F_NTUPLE))
Ben Hutchingsc4454772011-01-19 11:03:53 +00003469 goto out;
3470 rxq_index = cpu_rmap_lookup_index(dev->rx_cpu_rmap, next_cpu);
3471 if (rxq_index == skb_get_rx_queue(skb))
3472 goto out;
3473
3474 rxqueue = dev->_rx + rxq_index;
3475 flow_table = rcu_dereference(rxqueue->rps_flow_table);
3476 if (!flow_table)
3477 goto out;
Tom Herbert61b905d2014-03-24 15:34:47 -07003478 flow_id = skb_get_hash(skb) & flow_table->mask;
Ben Hutchingsc4454772011-01-19 11:03:53 +00003479 rc = dev->netdev_ops->ndo_rx_flow_steer(dev, skb,
3480 rxq_index, flow_id);
3481 if (rc < 0)
3482 goto out;
3483 old_rflow = rflow;
3484 rflow = &flow_table->flows[flow_id];
Ben Hutchingsc4454772011-01-19 11:03:53 +00003485 rflow->filter = rc;
3486 if (old_rflow->filter == rflow->filter)
3487 old_rflow->filter = RPS_NO_FILTER;
3488 out:
3489#endif
3490 rflow->last_qtail =
Ben Hutchings09994d12011-10-03 04:42:46 +00003491 per_cpu(softnet_data, next_cpu).input_queue_head;
Ben Hutchingsc4454772011-01-19 11:03:53 +00003492 }
3493
Ben Hutchings09994d12011-10-03 04:42:46 +00003494 rflow->cpu = next_cpu;
Ben Hutchingsc4454772011-01-19 11:03:53 +00003495 return rflow;
3496}
3497
Tom Herbert0a9627f2010-03-16 08:03:29 +00003498/*
3499 * get_rps_cpu is called from netif_receive_skb and returns the target
3500 * CPU from the RPS map of the receiving queue for a given skb.
Eric Dumazetb0e28f12010-04-15 00:14:07 -07003501 * rcu_read_lock must be held on entry.
Tom Herbert0a9627f2010-03-16 08:03:29 +00003502 */
Tom Herbertfec5e652010-04-16 16:01:27 -07003503static int get_rps_cpu(struct net_device *dev, struct sk_buff *skb,
3504 struct rps_dev_flow **rflowp)
Tom Herbert0a9627f2010-03-16 08:03:29 +00003505{
Eric Dumazet567e4b72015-02-06 12:59:01 -08003506 const struct rps_sock_flow_table *sock_flow_table;
3507 struct netdev_rx_queue *rxqueue = dev->_rx;
Tom Herbertfec5e652010-04-16 16:01:27 -07003508 struct rps_dev_flow_table *flow_table;
Eric Dumazet567e4b72015-02-06 12:59:01 -08003509 struct rps_map *map;
Tom Herbert0a9627f2010-03-16 08:03:29 +00003510 int cpu = -1;
Eric Dumazet567e4b72015-02-06 12:59:01 -08003511 u32 tcpu;
Tom Herbert61b905d2014-03-24 15:34:47 -07003512 u32 hash;
Tom Herbert0a9627f2010-03-16 08:03:29 +00003513
Tom Herbert0a9627f2010-03-16 08:03:29 +00003514 if (skb_rx_queue_recorded(skb)) {
3515 u16 index = skb_get_rx_queue(skb);
Eric Dumazet567e4b72015-02-06 12:59:01 -08003516
Ben Hutchings62fe0b42010-09-27 08:24:33 +00003517 if (unlikely(index >= dev->real_num_rx_queues)) {
3518 WARN_ONCE(dev->real_num_rx_queues > 1,
3519 "%s received packet on queue %u, but number "
3520 "of RX queues is %u\n",
3521 dev->name, index, dev->real_num_rx_queues);
Tom Herbert0a9627f2010-03-16 08:03:29 +00003522 goto done;
3523 }
Eric Dumazet567e4b72015-02-06 12:59:01 -08003524 rxqueue += index;
Tom Herbert0a9627f2010-03-16 08:03:29 +00003525 }
3526
Eric Dumazet567e4b72015-02-06 12:59:01 -08003527 /* Avoid computing hash if RFS/RPS is not active for this rxqueue */
3528
3529 flow_table = rcu_dereference(rxqueue->rps_flow_table);
3530 map = rcu_dereference(rxqueue->rps_map);
3531 if (!flow_table && !map)
3532 goto done;
3533
Changli Gao2d47b452010-08-17 19:00:56 +00003534 skb_reset_network_header(skb);
Tom Herbert61b905d2014-03-24 15:34:47 -07003535 hash = skb_get_hash(skb);
3536 if (!hash)
Tom Herbert0a9627f2010-03-16 08:03:29 +00003537 goto done;
Tom Herbert0a9627f2010-03-16 08:03:29 +00003538
Tom Herbertfec5e652010-04-16 16:01:27 -07003539 sock_flow_table = rcu_dereference(rps_sock_flow_table);
3540 if (flow_table && sock_flow_table) {
Tom Herbertfec5e652010-04-16 16:01:27 -07003541 struct rps_dev_flow *rflow;
Eric Dumazet567e4b72015-02-06 12:59:01 -08003542 u32 next_cpu;
3543 u32 ident;
Tom Herbertfec5e652010-04-16 16:01:27 -07003544
Eric Dumazet567e4b72015-02-06 12:59:01 -08003545 /* First check into global flow table if there is a match */
3546 ident = sock_flow_table->ents[hash & sock_flow_table->mask];
3547 if ((ident ^ hash) & ~rps_cpu_mask)
3548 goto try_rps;
3549
3550 next_cpu = ident & rps_cpu_mask;
3551
3552 /* OK, now we know there is a match,
3553 * we can look at the local (per receive queue) flow table
3554 */
Tom Herbert61b905d2014-03-24 15:34:47 -07003555 rflow = &flow_table->flows[hash & flow_table->mask];
Tom Herbertfec5e652010-04-16 16:01:27 -07003556 tcpu = rflow->cpu;
3557
Tom Herbertfec5e652010-04-16 16:01:27 -07003558 /*
3559 * If the desired CPU (where last recvmsg was done) is
3560 * different from current CPU (one in the rx-queue flow
3561 * table entry), switch if one of the following holds:
Eric Dumazeta31196b2015-04-25 09:35:24 -07003562 * - Current CPU is unset (>= nr_cpu_ids).
Tom Herbertfec5e652010-04-16 16:01:27 -07003563 * - Current CPU is offline.
3564 * - The current CPU's queue tail has advanced beyond the
3565 * last packet that was enqueued using this table entry.
3566 * This guarantees that all previous packets for the flow
3567 * have been dequeued, thus preserving in order delivery.
3568 */
3569 if (unlikely(tcpu != next_cpu) &&
Eric Dumazeta31196b2015-04-25 09:35:24 -07003570 (tcpu >= nr_cpu_ids || !cpu_online(tcpu) ||
Tom Herbertfec5e652010-04-16 16:01:27 -07003571 ((int)(per_cpu(softnet_data, tcpu).input_queue_head -
Tom Herbertbaefa312012-11-16 09:04:15 +00003572 rflow->last_qtail)) >= 0)) {
3573 tcpu = next_cpu;
Ben Hutchingsc4454772011-01-19 11:03:53 +00003574 rflow = set_rps_cpu(dev, skb, rflow, next_cpu);
Tom Herbertbaefa312012-11-16 09:04:15 +00003575 }
Ben Hutchingsc4454772011-01-19 11:03:53 +00003576
Eric Dumazeta31196b2015-04-25 09:35:24 -07003577 if (tcpu < nr_cpu_ids && cpu_online(tcpu)) {
Tom Herbertfec5e652010-04-16 16:01:27 -07003578 *rflowp = rflow;
3579 cpu = tcpu;
3580 goto done;
3581 }
3582 }
3583
Eric Dumazet567e4b72015-02-06 12:59:01 -08003584try_rps:
3585
Tom Herbert0a9627f2010-03-16 08:03:29 +00003586 if (map) {
Daniel Borkmann8fc54f62014-08-23 20:58:54 +02003587 tcpu = map->cpus[reciprocal_scale(hash, map->len)];
Tom Herbert0a9627f2010-03-16 08:03:29 +00003588 if (cpu_online(tcpu)) {
3589 cpu = tcpu;
3590 goto done;
3591 }
3592 }
3593
3594done:
Tom Herbert0a9627f2010-03-16 08:03:29 +00003595 return cpu;
3596}
3597
Ben Hutchingsc4454772011-01-19 11:03:53 +00003598#ifdef CONFIG_RFS_ACCEL
3599
3600/**
3601 * rps_may_expire_flow - check whether an RFS hardware filter may be removed
3602 * @dev: Device on which the filter was set
3603 * @rxq_index: RX queue index
3604 * @flow_id: Flow ID passed to ndo_rx_flow_steer()
3605 * @filter_id: Filter ID returned by ndo_rx_flow_steer()
3606 *
3607 * Drivers that implement ndo_rx_flow_steer() should periodically call
3608 * this function for each installed filter and remove the filters for
3609 * which it returns %true.
3610 */
3611bool rps_may_expire_flow(struct net_device *dev, u16 rxq_index,
3612 u32 flow_id, u16 filter_id)
3613{
3614 struct netdev_rx_queue *rxqueue = dev->_rx + rxq_index;
3615 struct rps_dev_flow_table *flow_table;
3616 struct rps_dev_flow *rflow;
3617 bool expire = true;
Eric Dumazeta31196b2015-04-25 09:35:24 -07003618 unsigned int cpu;
Ben Hutchingsc4454772011-01-19 11:03:53 +00003619
3620 rcu_read_lock();
3621 flow_table = rcu_dereference(rxqueue->rps_flow_table);
3622 if (flow_table && flow_id <= flow_table->mask) {
3623 rflow = &flow_table->flows[flow_id];
3624 cpu = ACCESS_ONCE(rflow->cpu);
Eric Dumazeta31196b2015-04-25 09:35:24 -07003625 if (rflow->filter == filter_id && cpu < nr_cpu_ids &&
Ben Hutchingsc4454772011-01-19 11:03:53 +00003626 ((int)(per_cpu(softnet_data, cpu).input_queue_head -
3627 rflow->last_qtail) <
3628 (int)(10 * flow_table->mask)))
3629 expire = false;
3630 }
3631 rcu_read_unlock();
3632 return expire;
3633}
3634EXPORT_SYMBOL(rps_may_expire_flow);
3635
3636#endif /* CONFIG_RFS_ACCEL */
3637
Tom Herbert0a9627f2010-03-16 08:03:29 +00003638/* Called from hardirq (IPI) context */
Eric Dumazete36fa2f2010-04-19 21:17:14 +00003639static void rps_trigger_softirq(void *data)
Tom Herbert0a9627f2010-03-16 08:03:29 +00003640{
Eric Dumazete36fa2f2010-04-19 21:17:14 +00003641 struct softnet_data *sd = data;
3642
Eric Dumazeteecfd7c2010-05-06 22:07:48 -07003643 ____napi_schedule(sd, &sd->backlog);
Changli Gaodee42872010-05-02 05:42:16 +00003644 sd->received_rps++;
Tom Herbert0a9627f2010-03-16 08:03:29 +00003645}
Eric Dumazete36fa2f2010-04-19 21:17:14 +00003646
Tom Herbertfec5e652010-04-16 16:01:27 -07003647#endif /* CONFIG_RPS */
Tom Herbert0a9627f2010-03-16 08:03:29 +00003648
3649/*
Eric Dumazete36fa2f2010-04-19 21:17:14 +00003650 * Check if this softnet_data structure is another cpu one
3651 * If yes, queue it to our IPI list and return 1
3652 * If no, return 0
3653 */
3654static int rps_ipi_queued(struct softnet_data *sd)
3655{
3656#ifdef CONFIG_RPS
Christoph Lameter903ceff2014-08-17 12:30:35 -05003657 struct softnet_data *mysd = this_cpu_ptr(&softnet_data);
Eric Dumazete36fa2f2010-04-19 21:17:14 +00003658
3659 if (sd != mysd) {
3660 sd->rps_ipi_next = mysd->rps_ipi_list;
3661 mysd->rps_ipi_list = sd;
3662
3663 __raise_softirq_irqoff(NET_RX_SOFTIRQ);
3664 return 1;
3665 }
3666#endif /* CONFIG_RPS */
3667 return 0;
3668}
3669
Willem de Bruijn99bbc702013-05-20 04:02:32 +00003670#ifdef CONFIG_NET_FLOW_LIMIT
3671int netdev_flow_limit_table_len __read_mostly = (1 << 12);
3672#endif
3673
3674static bool skb_flow_limit(struct sk_buff *skb, unsigned int qlen)
3675{
3676#ifdef CONFIG_NET_FLOW_LIMIT
3677 struct sd_flow_limit *fl;
3678 struct softnet_data *sd;
3679 unsigned int old_flow, new_flow;
3680
3681 if (qlen < (netdev_max_backlog >> 1))
3682 return false;
3683
Christoph Lameter903ceff2014-08-17 12:30:35 -05003684 sd = this_cpu_ptr(&softnet_data);
Willem de Bruijn99bbc702013-05-20 04:02:32 +00003685
3686 rcu_read_lock();
3687 fl = rcu_dereference(sd->flow_limit);
3688 if (fl) {
Tom Herbert3958afa1b2013-12-15 22:12:06 -08003689 new_flow = skb_get_hash(skb) & (fl->num_buckets - 1);
Willem de Bruijn99bbc702013-05-20 04:02:32 +00003690 old_flow = fl->history[fl->history_head];
3691 fl->history[fl->history_head] = new_flow;
3692
3693 fl->history_head++;
3694 fl->history_head &= FLOW_LIMIT_HISTORY - 1;
3695
3696 if (likely(fl->buckets[old_flow]))
3697 fl->buckets[old_flow]--;
3698
3699 if (++fl->buckets[new_flow] > (FLOW_LIMIT_HISTORY >> 1)) {
3700 fl->count++;
3701 rcu_read_unlock();
3702 return true;
3703 }
3704 }
3705 rcu_read_unlock();
3706#endif
3707 return false;
3708}
3709
Eric Dumazete36fa2f2010-04-19 21:17:14 +00003710/*
Tom Herbert0a9627f2010-03-16 08:03:29 +00003711 * enqueue_to_backlog is called to queue an skb to a per CPU backlog
3712 * queue (may be a remote CPU queue).
3713 */
Tom Herbertfec5e652010-04-16 16:01:27 -07003714static int enqueue_to_backlog(struct sk_buff *skb, int cpu,
3715 unsigned int *qtail)
Tom Herbert0a9627f2010-03-16 08:03:29 +00003716{
Eric Dumazete36fa2f2010-04-19 21:17:14 +00003717 struct softnet_data *sd;
Tom Herbert0a9627f2010-03-16 08:03:29 +00003718 unsigned long flags;
Willem de Bruijn99bbc702013-05-20 04:02:32 +00003719 unsigned int qlen;
Tom Herbert0a9627f2010-03-16 08:03:29 +00003720
Eric Dumazete36fa2f2010-04-19 21:17:14 +00003721 sd = &per_cpu(softnet_data, cpu);
Tom Herbert0a9627f2010-03-16 08:03:29 +00003722
3723 local_irq_save(flags);
Tom Herbert0a9627f2010-03-16 08:03:29 +00003724
Eric Dumazete36fa2f2010-04-19 21:17:14 +00003725 rps_lock(sd);
Julian Anastasove9e4dd32015-07-09 09:59:09 +03003726 if (!netif_running(skb->dev))
3727 goto drop;
Willem de Bruijn99bbc702013-05-20 04:02:32 +00003728 qlen = skb_queue_len(&sd->input_pkt_queue);
3729 if (qlen <= netdev_max_backlog && !skb_flow_limit(skb, qlen)) {
Li RongQinge008f3f2014-12-08 09:42:55 +08003730 if (qlen) {
Tom Herbert0a9627f2010-03-16 08:03:29 +00003731enqueue:
Eric Dumazete36fa2f2010-04-19 21:17:14 +00003732 __skb_queue_tail(&sd->input_pkt_queue, skb);
Tom Herbert76cc8b12010-05-20 18:37:59 +00003733 input_queue_tail_incr_save(sd, qtail);
Eric Dumazete36fa2f2010-04-19 21:17:14 +00003734 rps_unlock(sd);
Changli Gao152102c2010-03-30 20:16:22 +00003735 local_irq_restore(flags);
Tom Herbert0a9627f2010-03-16 08:03:29 +00003736 return NET_RX_SUCCESS;
3737 }
3738
Eric Dumazetebda37c22010-05-06 23:51:21 +00003739 /* Schedule NAPI for backlog device
3740 * We can use non atomic operation since we own the queue lock
3741 */
3742 if (!__test_and_set_bit(NAPI_STATE_SCHED, &sd->backlog.state)) {
Eric Dumazete36fa2f2010-04-19 21:17:14 +00003743 if (!rps_ipi_queued(sd))
Eric Dumazeteecfd7c2010-05-06 22:07:48 -07003744 ____napi_schedule(sd, &sd->backlog);
Tom Herbert0a9627f2010-03-16 08:03:29 +00003745 }
3746 goto enqueue;
3747 }
3748
Julian Anastasove9e4dd32015-07-09 09:59:09 +03003749drop:
Changli Gaodee42872010-05-02 05:42:16 +00003750 sd->dropped++;
Eric Dumazete36fa2f2010-04-19 21:17:14 +00003751 rps_unlock(sd);
Tom Herbert0a9627f2010-03-16 08:03:29 +00003752
Tom Herbert0a9627f2010-03-16 08:03:29 +00003753 local_irq_restore(flags);
3754
Eric Dumazetcaf586e2010-09-30 21:06:55 +00003755 atomic_long_inc(&skb->dev->rx_dropped);
Tom Herbert0a9627f2010-03-16 08:03:29 +00003756 kfree_skb(skb);
3757 return NET_RX_DROP;
3758}
Linus Torvalds1da177e2005-04-16 15:20:36 -07003759
Ben Hutchingsae78dbf2014-01-10 22:17:24 +00003760static int netif_rx_internal(struct sk_buff *skb)
Linus Torvalds1da177e2005-04-16 15:20:36 -07003761{
Eric Dumazetb0e28f12010-04-15 00:14:07 -07003762 int ret;
Linus Torvalds1da177e2005-04-16 15:20:36 -07003763
Eric Dumazet588f0332011-11-15 04:12:55 +00003764 net_timestamp_check(netdev_tstamp_prequeue, skb);
Linus Torvalds1da177e2005-04-16 15:20:36 -07003765
Koki Sanagicf66ba52010-08-23 18:45:02 +09003766 trace_netif_rx(skb);
Eric Dumazetdf334542010-03-24 19:13:54 +00003767#ifdef CONFIG_RPS
Ingo Molnarc5905af2012-02-24 08:31:31 +01003768 if (static_key_false(&rps_needed)) {
Tom Herbertfec5e652010-04-16 16:01:27 -07003769 struct rps_dev_flow voidflow, *rflow = &voidflow;
Eric Dumazetb0e28f12010-04-15 00:14:07 -07003770 int cpu;
Linus Torvalds1da177e2005-04-16 15:20:36 -07003771
Changli Gaocece1942010-08-07 20:35:43 -07003772 preempt_disable();
Eric Dumazetb0e28f12010-04-15 00:14:07 -07003773 rcu_read_lock();
Tom Herbertfec5e652010-04-16 16:01:27 -07003774
3775 cpu = get_rps_cpu(skb->dev, skb, &rflow);
Eric Dumazetb0e28f12010-04-15 00:14:07 -07003776 if (cpu < 0)
3777 cpu = smp_processor_id();
Tom Herbertfec5e652010-04-16 16:01:27 -07003778
3779 ret = enqueue_to_backlog(skb, cpu, &rflow->last_qtail);
3780
Eric Dumazetb0e28f12010-04-15 00:14:07 -07003781 rcu_read_unlock();
Changli Gaocece1942010-08-07 20:35:43 -07003782 preempt_enable();
Eric Dumazetadc93002011-11-17 03:13:26 +00003783 } else
3784#endif
Tom Herbertfec5e652010-04-16 16:01:27 -07003785 {
3786 unsigned int qtail;
3787 ret = enqueue_to_backlog(skb, get_cpu(), &qtail);
3788 put_cpu();
3789 }
Eric Dumazetb0e28f12010-04-15 00:14:07 -07003790 return ret;
Linus Torvalds1da177e2005-04-16 15:20:36 -07003791}
Ben Hutchingsae78dbf2014-01-10 22:17:24 +00003792
3793/**
3794 * netif_rx - post buffer to the network code
3795 * @skb: buffer to post
3796 *
3797 * This function receives a packet from a device driver and queues it for
3798 * the upper (protocol) levels to process. It always succeeds. The buffer
3799 * may be dropped during processing for congestion control or by the
3800 * protocol layers.
3801 *
3802 * return values:
3803 * NET_RX_SUCCESS (no congestion)
3804 * NET_RX_DROP (packet was dropped)
3805 *
3806 */
3807
3808int netif_rx(struct sk_buff *skb)
3809{
3810 trace_netif_rx_entry(skb);
3811
3812 return netif_rx_internal(skb);
3813}
Eric Dumazetd1b19df2009-09-03 01:29:39 -07003814EXPORT_SYMBOL(netif_rx);
Linus Torvalds1da177e2005-04-16 15:20:36 -07003815
3816int netif_rx_ni(struct sk_buff *skb)
3817{
3818 int err;
3819
Ben Hutchingsae78dbf2014-01-10 22:17:24 +00003820 trace_netif_rx_ni_entry(skb);
3821
Linus Torvalds1da177e2005-04-16 15:20:36 -07003822 preempt_disable();
Ben Hutchingsae78dbf2014-01-10 22:17:24 +00003823 err = netif_rx_internal(skb);
Linus Torvalds1da177e2005-04-16 15:20:36 -07003824 if (local_softirq_pending())
3825 do_softirq();
3826 preempt_enable();
3827
3828 return err;
3829}
Linus Torvalds1da177e2005-04-16 15:20:36 -07003830EXPORT_SYMBOL(netif_rx_ni);
3831
Emese Revfy0766f782016-06-20 20:42:34 +02003832static __latent_entropy void net_tx_action(struct softirq_action *h)
Linus Torvalds1da177e2005-04-16 15:20:36 -07003833{
Christoph Lameter903ceff2014-08-17 12:30:35 -05003834 struct softnet_data *sd = this_cpu_ptr(&softnet_data);
Linus Torvalds1da177e2005-04-16 15:20:36 -07003835
3836 if (sd->completion_queue) {
3837 struct sk_buff *clist;
3838
3839 local_irq_disable();
3840 clist = sd->completion_queue;
3841 sd->completion_queue = NULL;
3842 local_irq_enable();
3843
3844 while (clist) {
3845 struct sk_buff *skb = clist;
3846 clist = clist->next;
3847
Ilpo Järvinen547b7922008-07-25 21:43:18 -07003848 WARN_ON(atomic_read(&skb->users));
Eric Dumazete6247022013-12-05 04:45:08 -08003849 if (likely(get_kfree_skb_cb(skb)->reason == SKB_REASON_CONSUMED))
3850 trace_consume_skb(skb);
3851 else
3852 trace_kfree_skb(skb, net_tx_action);
Jesper Dangaard Brouer15fad712016-02-08 13:15:04 +01003853
3854 if (skb->fclone != SKB_FCLONE_UNAVAILABLE)
3855 __kfree_skb(skb);
3856 else
3857 __kfree_skb_defer(skb);
Linus Torvalds1da177e2005-04-16 15:20:36 -07003858 }
Jesper Dangaard Brouer15fad712016-02-08 13:15:04 +01003859
3860 __kfree_skb_flush();
Linus Torvalds1da177e2005-04-16 15:20:36 -07003861 }
3862
3863 if (sd->output_queue) {
David S. Miller37437bb2008-07-16 02:15:04 -07003864 struct Qdisc *head;
Linus Torvalds1da177e2005-04-16 15:20:36 -07003865
3866 local_irq_disable();
3867 head = sd->output_queue;
3868 sd->output_queue = NULL;
Changli Gaoa9cbd582010-04-26 23:06:24 +00003869 sd->output_queue_tailp = &sd->output_queue;
Linus Torvalds1da177e2005-04-16 15:20:36 -07003870 local_irq_enable();
3871
3872 while (head) {
David S. Miller37437bb2008-07-16 02:15:04 -07003873 struct Qdisc *q = head;
3874 spinlock_t *root_lock;
3875
Linus Torvalds1da177e2005-04-16 15:20:36 -07003876 head = head->next_sched;
3877
David S. Miller5fb66222008-08-02 20:02:43 -07003878 root_lock = qdisc_lock(q);
Eric Dumazet3bcb8462016-06-04 20:02:28 -07003879 spin_lock(root_lock);
3880 /* We need to make sure head->next_sched is read
3881 * before clearing __QDISC_STATE_SCHED
3882 */
3883 smp_mb__before_atomic();
3884 clear_bit(__QDISC_STATE_SCHED, &q->state);
3885 qdisc_run(q);
3886 spin_unlock(root_lock);
Linus Torvalds1da177e2005-04-16 15:20:36 -07003887 }
3888 }
3889}
3890
Javier Martinez Canillas181402a2016-09-09 08:43:15 -04003891#if IS_ENABLED(CONFIG_BRIDGE) && IS_ENABLED(CONFIG_ATM_LANE)
Michał Mirosławda678292009-06-05 05:35:28 +00003892/* This hook is defined here for ATM LANE */
3893int (*br_fdb_test_addr_hook)(struct net_device *dev,
3894 unsigned char *addr) __read_mostly;
Stephen Hemminger4fb019a2009-09-11 11:50:08 -07003895EXPORT_SYMBOL_GPL(br_fdb_test_addr_hook);
Michał Mirosławda678292009-06-05 05:35:28 +00003896#endif
Linus Torvalds1da177e2005-04-16 15:20:36 -07003897
Daniel Borkmann1f211a12016-01-07 22:29:47 +01003898static inline struct sk_buff *
3899sch_handle_ingress(struct sk_buff *skb, struct packet_type **pt_prev, int *ret,
3900 struct net_device *orig_dev)
Herbert Xuf697c3e2007-10-14 00:38:47 -07003901{
Daniel Borkmanne7582ba2015-05-19 22:33:25 +02003902#ifdef CONFIG_NET_CLS_ACT
Daniel Borkmannd2788d32015-05-09 22:51:32 +02003903 struct tcf_proto *cl = rcu_dereference_bh(skb->dev->ingress_cl_list);
3904 struct tcf_result cl_res;
Eric Dumazet24824a02010-10-02 06:11:55 +00003905
Daniel Borkmannc9e99fd2015-05-09 22:51:31 +02003906 /* If there's at least one ingress present somewhere (so
3907 * we get here via enabled static key), remaining devices
3908 * that are not configured with an ingress qdisc will bail
Daniel Borkmannd2788d32015-05-09 22:51:32 +02003909 * out here.
Daniel Borkmannc9e99fd2015-05-09 22:51:31 +02003910 */
Daniel Borkmannd2788d32015-05-09 22:51:32 +02003911 if (!cl)
Daniel Borkmann45771392015-04-10 23:07:54 +02003912 return skb;
Herbert Xuf697c3e2007-10-14 00:38:47 -07003913 if (*pt_prev) {
3914 *ret = deliver_skb(skb, *pt_prev, orig_dev);
3915 *pt_prev = NULL;
Herbert Xuf697c3e2007-10-14 00:38:47 -07003916 }
3917
Florian Westphal33654952015-05-14 00:36:28 +02003918 qdisc_skb_cb(skb)->pkt_len = skb->len;
Daniel Borkmannc9e99fd2015-05-09 22:51:31 +02003919 skb->tc_verd = SET_TC_AT(skb->tc_verd, AT_INGRESS);
Eric Dumazet24ea5912015-07-06 05:18:03 -07003920 qdisc_bstats_cpu_update(cl->q, skb);
Daniel Borkmannc9e99fd2015-05-09 22:51:31 +02003921
Daniel Borkmann3b3ae882015-08-26 23:00:06 +02003922 switch (tc_classify(skb, cl, &cl_res, false)) {
Daniel Borkmannd2788d32015-05-09 22:51:32 +02003923 case TC_ACT_OK:
3924 case TC_ACT_RECLASSIFY:
3925 skb->tc_index = TC_H_MIN(cl_res.classid);
3926 break;
3927 case TC_ACT_SHOT:
Eric Dumazet24ea5912015-07-06 05:18:03 -07003928 qdisc_qstats_cpu_drop(cl->q);
Eric Dumazet8a3a4c62016-05-06 15:55:50 -07003929 kfree_skb(skb);
3930 return NULL;
Daniel Borkmannd2788d32015-05-09 22:51:32 +02003931 case TC_ACT_STOLEN:
3932 case TC_ACT_QUEUED:
Eric Dumazet8a3a4c62016-05-06 15:55:50 -07003933 consume_skb(skb);
Daniel Borkmannd2788d32015-05-09 22:51:32 +02003934 return NULL;
Alexei Starovoitov27b29f62015-09-15 23:05:43 -07003935 case TC_ACT_REDIRECT:
3936 /* skb_mac_header check was done by cls/act_bpf, so
3937 * we can safely push the L2 header back before
3938 * redirecting to another netdev
3939 */
3940 __skb_push(skb, skb->mac_len);
3941 skb_do_redirect(skb);
3942 return NULL;
Daniel Borkmannd2788d32015-05-09 22:51:32 +02003943 default:
3944 break;
Herbert Xuf697c3e2007-10-14 00:38:47 -07003945 }
Daniel Borkmanne7582ba2015-05-19 22:33:25 +02003946#endif /* CONFIG_NET_CLS_ACT */
Herbert Xuf697c3e2007-10-14 00:38:47 -07003947 return skb;
3948}
Linus Torvalds1da177e2005-04-16 15:20:36 -07003949
Jiri Pirkoab95bfe2010-06-01 21:52:08 +00003950/**
Mahesh Bandewar24b27fc2016-09-01 22:18:34 -07003951 * netdev_is_rx_handler_busy - check if receive handler is registered
3952 * @dev: device to check
3953 *
3954 * Check if a receive handler is already registered for a given device.
3955 * Return true if there one.
3956 *
3957 * The caller must hold the rtnl_mutex.
3958 */
3959bool netdev_is_rx_handler_busy(struct net_device *dev)
3960{
3961 ASSERT_RTNL();
3962 return dev && rtnl_dereference(dev->rx_handler);
3963}
3964EXPORT_SYMBOL_GPL(netdev_is_rx_handler_busy);
3965
3966/**
Jiri Pirkoab95bfe2010-06-01 21:52:08 +00003967 * netdev_rx_handler_register - register receive handler
3968 * @dev: device to register a handler for
3969 * @rx_handler: receive handler to register
Jiri Pirko93e2c322010-06-10 03:34:59 +00003970 * @rx_handler_data: data pointer that is used by rx handler
Jiri Pirkoab95bfe2010-06-01 21:52:08 +00003971 *
Masanari Iidae2278672014-02-18 22:54:36 +09003972 * Register a receive handler for a device. This handler will then be
Jiri Pirkoab95bfe2010-06-01 21:52:08 +00003973 * called from __netif_receive_skb. A negative errno code is returned
3974 * on a failure.
3975 *
3976 * The caller must hold the rtnl_mutex.
Jiri Pirko8a4eb572011-03-12 03:14:39 +00003977 *
3978 * For a general description of rx_handler, see enum rx_handler_result.
Jiri Pirkoab95bfe2010-06-01 21:52:08 +00003979 */
3980int netdev_rx_handler_register(struct net_device *dev,
Jiri Pirko93e2c322010-06-10 03:34:59 +00003981 rx_handler_func_t *rx_handler,
3982 void *rx_handler_data)
Jiri Pirkoab95bfe2010-06-01 21:52:08 +00003983{
3984 ASSERT_RTNL();
3985
3986 if (dev->rx_handler)
3987 return -EBUSY;
3988
Eric Dumazet00cfec32013-03-29 03:01:22 +00003989 /* Note: rx_handler_data must be set before rx_handler */
Jiri Pirko93e2c322010-06-10 03:34:59 +00003990 rcu_assign_pointer(dev->rx_handler_data, rx_handler_data);
Jiri Pirkoab95bfe2010-06-01 21:52:08 +00003991 rcu_assign_pointer(dev->rx_handler, rx_handler);
3992
3993 return 0;
3994}
3995EXPORT_SYMBOL_GPL(netdev_rx_handler_register);
3996
3997/**
3998 * netdev_rx_handler_unregister - unregister receive handler
3999 * @dev: device to unregister a handler from
4000 *
Kusanagi Kouichi166ec362013-03-18 02:59:52 +00004001 * Unregister a receive handler from a device.
Jiri Pirkoab95bfe2010-06-01 21:52:08 +00004002 *
4003 * The caller must hold the rtnl_mutex.
4004 */
4005void netdev_rx_handler_unregister(struct net_device *dev)
4006{
4007
4008 ASSERT_RTNL();
Stephen Hemmingera9b3cd72011-08-01 16:19:00 +00004009 RCU_INIT_POINTER(dev->rx_handler, NULL);
Eric Dumazet00cfec32013-03-29 03:01:22 +00004010 /* a reader seeing a non NULL rx_handler in a rcu_read_lock()
4011 * section has a guarantee to see a non NULL rx_handler_data
4012 * as well.
4013 */
4014 synchronize_net();
Stephen Hemmingera9b3cd72011-08-01 16:19:00 +00004015 RCU_INIT_POINTER(dev->rx_handler_data, NULL);
Jiri Pirkoab95bfe2010-06-01 21:52:08 +00004016}
4017EXPORT_SYMBOL_GPL(netdev_rx_handler_unregister);
4018
Mel Gormanb4b9e352012-07-31 16:44:26 -07004019/*
4020 * Limit the use of PFMEMALLOC reserves to those protocols that implement
4021 * the special handling of PFMEMALLOC skbs.
4022 */
4023static bool skb_pfmemalloc_protocol(struct sk_buff *skb)
4024{
4025 switch (skb->protocol) {
Joe Perches2b8837a2014-03-12 10:04:17 -07004026 case htons(ETH_P_ARP):
4027 case htons(ETH_P_IP):
4028 case htons(ETH_P_IPV6):
4029 case htons(ETH_P_8021Q):
4030 case htons(ETH_P_8021AD):
Mel Gormanb4b9e352012-07-31 16:44:26 -07004031 return true;
4032 default:
4033 return false;
4034 }
4035}
4036
Pablo Neirae687ad62015-05-13 18:19:38 +02004037static inline int nf_ingress(struct sk_buff *skb, struct packet_type **pt_prev,
4038 int *ret, struct net_device *orig_dev)
4039{
Daniel Borkmanne7582ba2015-05-19 22:33:25 +02004040#ifdef CONFIG_NETFILTER_INGRESS
Pablo Neirae687ad62015-05-13 18:19:38 +02004041 if (nf_hook_ingress_active(skb)) {
Aaron Conole2c1e2702016-09-21 11:35:03 -04004042 int ingress_retval;
4043
Pablo Neirae687ad62015-05-13 18:19:38 +02004044 if (*pt_prev) {
4045 *ret = deliver_skb(skb, *pt_prev, orig_dev);
4046 *pt_prev = NULL;
4047 }
4048
Aaron Conole2c1e2702016-09-21 11:35:03 -04004049 rcu_read_lock();
4050 ingress_retval = nf_hook_ingress(skb);
4051 rcu_read_unlock();
4052 return ingress_retval;
Pablo Neirae687ad62015-05-13 18:19:38 +02004053 }
Daniel Borkmanne7582ba2015-05-19 22:33:25 +02004054#endif /* CONFIG_NETFILTER_INGRESS */
Pablo Neirae687ad62015-05-13 18:19:38 +02004055 return 0;
4056}
Pablo Neirae687ad62015-05-13 18:19:38 +02004057
David S. Miller9754e292013-02-14 15:57:38 -05004058static int __netif_receive_skb_core(struct sk_buff *skb, bool pfmemalloc)
Linus Torvalds1da177e2005-04-16 15:20:36 -07004059{
4060 struct packet_type *ptype, *pt_prev;
Jiri Pirkoab95bfe2010-06-01 21:52:08 +00004061 rx_handler_func_t *rx_handler;
David S. Millerf2ccd8f2005-08-09 19:34:12 -07004062 struct net_device *orig_dev;
Jiri Pirko8a4eb572011-03-12 03:14:39 +00004063 bool deliver_exact = false;
Linus Torvalds1da177e2005-04-16 15:20:36 -07004064 int ret = NET_RX_DROP;
Al Viro252e3342006-11-14 20:48:11 -08004065 __be16 type;
Linus Torvalds1da177e2005-04-16 15:20:36 -07004066
Eric Dumazet588f0332011-11-15 04:12:55 +00004067 net_timestamp_check(!netdev_tstamp_prequeue, skb);
Eric Dumazet81bbb3d2009-09-30 16:42:42 -07004068
Koki Sanagicf66ba52010-08-23 18:45:02 +09004069 trace_netif_receive_skb(skb);
Patrick McHardy9b22ea52008-11-04 14:49:57 -08004070
Joe Eykholtcc9bd5c2008-07-02 18:22:00 -07004071 orig_dev = skb->dev;
Jiri Pirko1765a572011-02-12 06:48:36 +00004072
Arnaldo Carvalho de Meloc1d2bbe2007-04-10 20:45:18 -07004073 skb_reset_network_header(skb);
Eric Dumazetfda55ec2013-01-07 09:28:21 +00004074 if (!skb_transport_header_was_set(skb))
4075 skb_reset_transport_header(skb);
Jiri Pirko0b5c9db2011-06-10 06:56:58 +00004076 skb_reset_mac_len(skb);
Linus Torvalds1da177e2005-04-16 15:20:36 -07004077
4078 pt_prev = NULL;
4079
David S. Miller63d8ea72011-02-28 10:48:59 -08004080another_round:
David S. Millerb6858172012-07-23 16:27:54 -07004081 skb->skb_iif = skb->dev->ifindex;
David S. Miller63d8ea72011-02-28 10:48:59 -08004082
4083 __this_cpu_inc(softnet_data.processed);
4084
Patrick McHardy8ad227f2013-04-19 02:04:31 +00004085 if (skb->protocol == cpu_to_be16(ETH_P_8021Q) ||
4086 skb->protocol == cpu_to_be16(ETH_P_8021AD)) {
Vlad Yasevich0d5501c2014-08-08 14:42:13 -04004087 skb = skb_vlan_untag(skb);
Jiri Pirkobcc6d472011-04-07 19:48:33 +00004088 if (unlikely(!skb))
Julian Anastasov2c17d272015-07-09 09:59:10 +03004089 goto out;
Jiri Pirkobcc6d472011-04-07 19:48:33 +00004090 }
4091
Linus Torvalds1da177e2005-04-16 15:20:36 -07004092#ifdef CONFIG_NET_CLS_ACT
4093 if (skb->tc_verd & TC_NCLS) {
4094 skb->tc_verd = CLR_TC_NCLS(skb->tc_verd);
4095 goto ncls;
4096 }
4097#endif
4098
David S. Miller9754e292013-02-14 15:57:38 -05004099 if (pfmemalloc)
Mel Gormanb4b9e352012-07-31 16:44:26 -07004100 goto skip_taps;
4101
Linus Torvalds1da177e2005-04-16 15:20:36 -07004102 list_for_each_entry_rcu(ptype, &ptype_all, list) {
Salam Noureddine7866a622015-01-27 11:35:48 -08004103 if (pt_prev)
4104 ret = deliver_skb(skb, pt_prev, orig_dev);
4105 pt_prev = ptype;
4106 }
4107
4108 list_for_each_entry_rcu(ptype, &skb->dev->ptype_all, list) {
4109 if (pt_prev)
4110 ret = deliver_skb(skb, pt_prev, orig_dev);
4111 pt_prev = ptype;
Linus Torvalds1da177e2005-04-16 15:20:36 -07004112 }
4113
Mel Gormanb4b9e352012-07-31 16:44:26 -07004114skip_taps:
Pablo Neira1cf519002015-05-13 18:19:37 +02004115#ifdef CONFIG_NET_INGRESS
Daniel Borkmann45771392015-04-10 23:07:54 +02004116 if (static_key_false(&ingress_needed)) {
Daniel Borkmann1f211a12016-01-07 22:29:47 +01004117 skb = sch_handle_ingress(skb, &pt_prev, &ret, orig_dev);
Daniel Borkmann45771392015-04-10 23:07:54 +02004118 if (!skb)
Julian Anastasov2c17d272015-07-09 09:59:10 +03004119 goto out;
Pablo Neirae687ad62015-05-13 18:19:38 +02004120
4121 if (nf_ingress(skb, &pt_prev, &ret, orig_dev) < 0)
Julian Anastasov2c17d272015-07-09 09:59:10 +03004122 goto out;
Daniel Borkmann45771392015-04-10 23:07:54 +02004123 }
Pablo Neira1cf519002015-05-13 18:19:37 +02004124#endif
4125#ifdef CONFIG_NET_CLS_ACT
Daniel Borkmann45771392015-04-10 23:07:54 +02004126 skb->tc_verd = 0;
Linus Torvalds1da177e2005-04-16 15:20:36 -07004127ncls:
4128#endif
David S. Miller9754e292013-02-14 15:57:38 -05004129 if (pfmemalloc && !skb_pfmemalloc_protocol(skb))
Mel Gormanb4b9e352012-07-31 16:44:26 -07004130 goto drop;
4131
Jiri Pirkodf8a39d2015-01-13 17:13:44 +01004132 if (skb_vlan_tag_present(skb)) {
John Fastabend24257172011-10-10 09:16:41 +00004133 if (pt_prev) {
4134 ret = deliver_skb(skb, pt_prev, orig_dev);
4135 pt_prev = NULL;
4136 }
Florian Zumbiehl48cc32d32012-10-07 15:51:58 +00004137 if (vlan_do_receive(&skb))
John Fastabend24257172011-10-10 09:16:41 +00004138 goto another_round;
4139 else if (unlikely(!skb))
Julian Anastasov2c17d272015-07-09 09:59:10 +03004140 goto out;
John Fastabend24257172011-10-10 09:16:41 +00004141 }
4142
Florian Zumbiehl48cc32d32012-10-07 15:51:58 +00004143 rx_handler = rcu_dereference(skb->dev->rx_handler);
Jiri Pirkoab95bfe2010-06-01 21:52:08 +00004144 if (rx_handler) {
4145 if (pt_prev) {
4146 ret = deliver_skb(skb, pt_prev, orig_dev);
4147 pt_prev = NULL;
4148 }
Jiri Pirko8a4eb572011-03-12 03:14:39 +00004149 switch (rx_handler(&skb)) {
4150 case RX_HANDLER_CONSUMED:
Cristian Bercaru3bc1b1a2013-03-08 07:03:38 +00004151 ret = NET_RX_SUCCESS;
Julian Anastasov2c17d272015-07-09 09:59:10 +03004152 goto out;
Jiri Pirko8a4eb572011-03-12 03:14:39 +00004153 case RX_HANDLER_ANOTHER:
David S. Miller63d8ea72011-02-28 10:48:59 -08004154 goto another_round;
Jiri Pirko8a4eb572011-03-12 03:14:39 +00004155 case RX_HANDLER_EXACT:
4156 deliver_exact = true;
4157 case RX_HANDLER_PASS:
4158 break;
4159 default:
4160 BUG();
4161 }
Jiri Pirkoab95bfe2010-06-01 21:52:08 +00004162 }
Linus Torvalds1da177e2005-04-16 15:20:36 -07004163
Jiri Pirkodf8a39d2015-01-13 17:13:44 +01004164 if (unlikely(skb_vlan_tag_present(skb))) {
4165 if (skb_vlan_tag_get_id(skb))
Eric Dumazetd4b812d2013-07-18 07:19:26 -07004166 skb->pkt_type = PACKET_OTHERHOST;
4167 /* Note: we might in the future use prio bits
4168 * and set skb->priority like in vlan_do_receive()
4169 * For the time being, just ignore Priority Code Point
4170 */
4171 skb->vlan_tci = 0;
4172 }
Florian Zumbiehl48cc32d32012-10-07 15:51:58 +00004173
Linus Torvalds1da177e2005-04-16 15:20:36 -07004174 type = skb->protocol;
Salam Noureddine7866a622015-01-27 11:35:48 -08004175
4176 /* deliver only exact match when indicated */
4177 if (likely(!deliver_exact)) {
4178 deliver_ptype_list_skb(skb, &pt_prev, orig_dev, type,
4179 &ptype_base[ntohs(type) &
4180 PTYPE_HASH_MASK]);
4181 }
4182
4183 deliver_ptype_list_skb(skb, &pt_prev, orig_dev, type,
4184 &orig_dev->ptype_specific);
4185
4186 if (unlikely(skb->dev != orig_dev)) {
4187 deliver_ptype_list_skb(skb, &pt_prev, orig_dev, type,
4188 &skb->dev->ptype_specific);
Linus Torvalds1da177e2005-04-16 15:20:36 -07004189 }
4190
4191 if (pt_prev) {
Michael S. Tsirkin1080e512012-07-20 09:23:17 +00004192 if (unlikely(skb_orphan_frags(skb, GFP_ATOMIC)))
Michael S. Tsirkin0e698bf2012-09-15 22:44:16 +00004193 goto drop;
Michael S. Tsirkin1080e512012-07-20 09:23:17 +00004194 else
4195 ret = pt_prev->func(skb, skb->dev, pt_prev, orig_dev);
Linus Torvalds1da177e2005-04-16 15:20:36 -07004196 } else {
Mel Gormanb4b9e352012-07-31 16:44:26 -07004197drop:
Jarod Wilson6e7333d2016-02-01 18:51:05 -05004198 if (!deliver_exact)
4199 atomic_long_inc(&skb->dev->rx_dropped);
4200 else
4201 atomic_long_inc(&skb->dev->rx_nohandler);
Linus Torvalds1da177e2005-04-16 15:20:36 -07004202 kfree_skb(skb);
4203 /* Jamal, now you will not able to escape explaining
4204 * me how you were going to use this. :-)
4205 */
4206 ret = NET_RX_DROP;
4207 }
4208
Julian Anastasov2c17d272015-07-09 09:59:10 +03004209out:
David S. Miller9754e292013-02-14 15:57:38 -05004210 return ret;
4211}
4212
4213static int __netif_receive_skb(struct sk_buff *skb)
4214{
4215 int ret;
4216
4217 if (sk_memalloc_socks() && skb_pfmemalloc(skb)) {
4218 unsigned long pflags = current->flags;
4219
4220 /*
4221 * PFMEMALLOC skbs are special, they should
4222 * - be delivered to SOCK_MEMALLOC sockets only
4223 * - stay away from userspace
4224 * - have bounded memory usage
4225 *
4226 * Use PF_MEMALLOC as this saves us from propagating the allocation
4227 * context down to all allocation sites.
4228 */
4229 current->flags |= PF_MEMALLOC;
4230 ret = __netif_receive_skb_core(skb, true);
4231 tsk_restore_flags(current, pflags, PF_MEMALLOC);
4232 } else
4233 ret = __netif_receive_skb_core(skb, false);
4234
Linus Torvalds1da177e2005-04-16 15:20:36 -07004235 return ret;
4236}
Tom Herbert0a9627f2010-03-16 08:03:29 +00004237
Ben Hutchingsae78dbf2014-01-10 22:17:24 +00004238static int netif_receive_skb_internal(struct sk_buff *skb)
Tom Herbert0a9627f2010-03-16 08:03:29 +00004239{
Julian Anastasov2c17d272015-07-09 09:59:10 +03004240 int ret;
4241
Eric Dumazet588f0332011-11-15 04:12:55 +00004242 net_timestamp_check(netdev_tstamp_prequeue, skb);
Eric Dumazet3b098e22010-05-15 23:57:10 -07004243
Richard Cochranc1f19b52010-07-17 08:49:36 +00004244 if (skb_defer_rx_timestamp(skb))
4245 return NET_RX_SUCCESS;
4246
Julian Anastasov2c17d272015-07-09 09:59:10 +03004247 rcu_read_lock();
4248
Eric Dumazetdf334542010-03-24 19:13:54 +00004249#ifdef CONFIG_RPS
Ingo Molnarc5905af2012-02-24 08:31:31 +01004250 if (static_key_false(&rps_needed)) {
Eric Dumazet3b098e22010-05-15 23:57:10 -07004251 struct rps_dev_flow voidflow, *rflow = &voidflow;
Julian Anastasov2c17d272015-07-09 09:59:10 +03004252 int cpu = get_rps_cpu(skb->dev, skb, &rflow);
Tom Herbertfec5e652010-04-16 16:01:27 -07004253
Eric Dumazet3b098e22010-05-15 23:57:10 -07004254 if (cpu >= 0) {
4255 ret = enqueue_to_backlog(skb, cpu, &rflow->last_qtail);
4256 rcu_read_unlock();
Eric Dumazetadc93002011-11-17 03:13:26 +00004257 return ret;
Eric Dumazet3b098e22010-05-15 23:57:10 -07004258 }
Tom Herbertfec5e652010-04-16 16:01:27 -07004259 }
Tom Herbert1e94d722010-03-18 17:45:44 -07004260#endif
Julian Anastasov2c17d272015-07-09 09:59:10 +03004261 ret = __netif_receive_skb(skb);
4262 rcu_read_unlock();
4263 return ret;
Tom Herbert0a9627f2010-03-16 08:03:29 +00004264}
Ben Hutchingsae78dbf2014-01-10 22:17:24 +00004265
4266/**
4267 * netif_receive_skb - process receive buffer from network
4268 * @skb: buffer to process
4269 *
4270 * netif_receive_skb() is the main receive data processing function.
4271 * It always succeeds. The buffer may be dropped during processing
4272 * for congestion control or by the protocol layers.
4273 *
4274 * This function may only be called from softirq context and interrupts
4275 * should be enabled.
4276 *
4277 * Return values (usually ignored):
4278 * NET_RX_SUCCESS: no congestion
4279 * NET_RX_DROP: packet was dropped
4280 */
Eric W. Biederman04eb4482015-09-15 20:04:15 -05004281int netif_receive_skb(struct sk_buff *skb)
Ben Hutchingsae78dbf2014-01-10 22:17:24 +00004282{
4283 trace_netif_receive_skb_entry(skb);
4284
4285 return netif_receive_skb_internal(skb);
4286}
Eric W. Biederman04eb4482015-09-15 20:04:15 -05004287EXPORT_SYMBOL(netif_receive_skb);
Linus Torvalds1da177e2005-04-16 15:20:36 -07004288
Eric Dumazet41852492016-08-26 12:50:39 -07004289DEFINE_PER_CPU(struct work_struct, flush_works);
Paolo Abeni145dd5f2016-08-25 15:58:44 +02004290
4291/* Network device is going away, flush any packets still pending */
4292static void flush_backlog(struct work_struct *work)
4293{
Paolo Abeni145dd5f2016-08-25 15:58:44 +02004294 struct sk_buff *skb, *tmp;
4295 struct softnet_data *sd;
4296
4297 local_bh_disable();
4298 sd = this_cpu_ptr(&softnet_data);
4299
4300 local_irq_disable();
Eric Dumazete36fa2f2010-04-19 21:17:14 +00004301 rps_lock(sd);
Changli Gao6e7676c2010-04-27 15:07:33 -07004302 skb_queue_walk_safe(&sd->input_pkt_queue, skb, tmp) {
Eric Dumazet41852492016-08-26 12:50:39 -07004303 if (skb->dev->reg_state == NETREG_UNREGISTERING) {
Eric Dumazete36fa2f2010-04-19 21:17:14 +00004304 __skb_unlink(skb, &sd->input_pkt_queue);
Stephen Hemminger6e583ce2008-08-03 21:29:57 -07004305 kfree_skb(skb);
Tom Herbert76cc8b12010-05-20 18:37:59 +00004306 input_queue_head_incr(sd);
Stephen Hemminger6e583ce2008-08-03 21:29:57 -07004307 }
Changli Gao6e7676c2010-04-27 15:07:33 -07004308 }
Eric Dumazete36fa2f2010-04-19 21:17:14 +00004309 rps_unlock(sd);
Paolo Abeni145dd5f2016-08-25 15:58:44 +02004310 local_irq_enable();
Changli Gao6e7676c2010-04-27 15:07:33 -07004311
4312 skb_queue_walk_safe(&sd->process_queue, skb, tmp) {
Eric Dumazet41852492016-08-26 12:50:39 -07004313 if (skb->dev->reg_state == NETREG_UNREGISTERING) {
Changli Gao6e7676c2010-04-27 15:07:33 -07004314 __skb_unlink(skb, &sd->process_queue);
4315 kfree_skb(skb);
Tom Herbert76cc8b12010-05-20 18:37:59 +00004316 input_queue_head_incr(sd);
Changli Gao6e7676c2010-04-27 15:07:33 -07004317 }
4318 }
Paolo Abeni145dd5f2016-08-25 15:58:44 +02004319 local_bh_enable();
4320}
4321
Eric Dumazet41852492016-08-26 12:50:39 -07004322static void flush_all_backlogs(void)
Paolo Abeni145dd5f2016-08-25 15:58:44 +02004323{
4324 unsigned int cpu;
4325
4326 get_online_cpus();
4327
Eric Dumazet41852492016-08-26 12:50:39 -07004328 for_each_online_cpu(cpu)
4329 queue_work_on(cpu, system_highpri_wq,
4330 per_cpu_ptr(&flush_works, cpu));
Paolo Abeni145dd5f2016-08-25 15:58:44 +02004331
4332 for_each_online_cpu(cpu)
Eric Dumazet41852492016-08-26 12:50:39 -07004333 flush_work(per_cpu_ptr(&flush_works, cpu));
Paolo Abeni145dd5f2016-08-25 15:58:44 +02004334
4335 put_online_cpus();
Stephen Hemminger6e583ce2008-08-03 21:29:57 -07004336}
4337
Herbert Xud565b0a2008-12-15 23:38:52 -08004338static int napi_gro_complete(struct sk_buff *skb)
4339{
Vlad Yasevich22061d82012-11-15 08:49:11 +00004340 struct packet_offload *ptype;
Herbert Xud565b0a2008-12-15 23:38:52 -08004341 __be16 type = skb->protocol;
Vlad Yasevich22061d82012-11-15 08:49:11 +00004342 struct list_head *head = &offload_base;
Herbert Xud565b0a2008-12-15 23:38:52 -08004343 int err = -ENOENT;
4344
Eric Dumazetc3c7c252012-12-06 13:54:59 +00004345 BUILD_BUG_ON(sizeof(struct napi_gro_cb) > sizeof(skb->cb));
4346
Herbert Xufc59f9a2009-04-14 15:11:06 -07004347 if (NAPI_GRO_CB(skb)->count == 1) {
4348 skb_shinfo(skb)->gso_size = 0;
Herbert Xud565b0a2008-12-15 23:38:52 -08004349 goto out;
Herbert Xufc59f9a2009-04-14 15:11:06 -07004350 }
Herbert Xud565b0a2008-12-15 23:38:52 -08004351
4352 rcu_read_lock();
4353 list_for_each_entry_rcu(ptype, head, list) {
Vlad Yasevichf191a1d2012-11-15 08:49:23 +00004354 if (ptype->type != type || !ptype->callbacks.gro_complete)
Herbert Xud565b0a2008-12-15 23:38:52 -08004355 continue;
4356
Jerry Chu299603e82013-12-11 20:53:45 -08004357 err = ptype->callbacks.gro_complete(skb, 0);
Herbert Xud565b0a2008-12-15 23:38:52 -08004358 break;
4359 }
4360 rcu_read_unlock();
4361
4362 if (err) {
4363 WARN_ON(&ptype->list == head);
4364 kfree_skb(skb);
4365 return NET_RX_SUCCESS;
4366 }
4367
4368out:
Ben Hutchingsae78dbf2014-01-10 22:17:24 +00004369 return netif_receive_skb_internal(skb);
Herbert Xud565b0a2008-12-15 23:38:52 -08004370}
4371
Eric Dumazet2e71a6f2012-10-06 08:08:49 +00004372/* napi->gro_list contains packets ordered by age.
4373 * youngest packets at the head of it.
4374 * Complete skbs in reverse order to reduce latencies.
4375 */
4376void napi_gro_flush(struct napi_struct *napi, bool flush_old)
Herbert Xud565b0a2008-12-15 23:38:52 -08004377{
Eric Dumazet2e71a6f2012-10-06 08:08:49 +00004378 struct sk_buff *skb, *prev = NULL;
Herbert Xud565b0a2008-12-15 23:38:52 -08004379
Eric Dumazet2e71a6f2012-10-06 08:08:49 +00004380 /* scan list and build reverse chain */
4381 for (skb = napi->gro_list; skb != NULL; skb = skb->next) {
4382 skb->prev = prev;
4383 prev = skb;
Herbert Xud565b0a2008-12-15 23:38:52 -08004384 }
4385
Eric Dumazet2e71a6f2012-10-06 08:08:49 +00004386 for (skb = prev; skb; skb = prev) {
4387 skb->next = NULL;
4388
4389 if (flush_old && NAPI_GRO_CB(skb)->age == jiffies)
4390 return;
4391
4392 prev = skb->prev;
4393 napi_gro_complete(skb);
4394 napi->gro_count--;
4395 }
4396
Herbert Xud565b0a2008-12-15 23:38:52 -08004397 napi->gro_list = NULL;
4398}
Eric Dumazet86cac582010-08-31 18:25:32 +00004399EXPORT_SYMBOL(napi_gro_flush);
Herbert Xud565b0a2008-12-15 23:38:52 -08004400
Eric Dumazet89c5fa32012-12-10 13:28:16 +00004401static void gro_list_prepare(struct napi_struct *napi, struct sk_buff *skb)
4402{
4403 struct sk_buff *p;
4404 unsigned int maclen = skb->dev->hard_header_len;
Tom Herbert0b4cec82014-01-15 08:58:06 -08004405 u32 hash = skb_get_hash_raw(skb);
Eric Dumazet89c5fa32012-12-10 13:28:16 +00004406
4407 for (p = napi->gro_list; p; p = p->next) {
4408 unsigned long diffs;
4409
Tom Herbert0b4cec82014-01-15 08:58:06 -08004410 NAPI_GRO_CB(p)->flush = 0;
4411
4412 if (hash != skb_get_hash_raw(p)) {
4413 NAPI_GRO_CB(p)->same_flow = 0;
4414 continue;
4415 }
4416
Eric Dumazet89c5fa32012-12-10 13:28:16 +00004417 diffs = (unsigned long)p->dev ^ (unsigned long)skb->dev;
4418 diffs |= p->vlan_tci ^ skb->vlan_tci;
Jesse Grossce87fc62016-01-20 17:59:49 -08004419 diffs |= skb_metadata_dst_cmp(p, skb);
Eric Dumazet89c5fa32012-12-10 13:28:16 +00004420 if (maclen == ETH_HLEN)
4421 diffs |= compare_ether_header(skb_mac_header(p),
Eric Dumazeta50e2332014-03-29 21:28:21 -07004422 skb_mac_header(skb));
Eric Dumazet89c5fa32012-12-10 13:28:16 +00004423 else if (!diffs)
4424 diffs = memcmp(skb_mac_header(p),
Eric Dumazeta50e2332014-03-29 21:28:21 -07004425 skb_mac_header(skb),
Eric Dumazet89c5fa32012-12-10 13:28:16 +00004426 maclen);
4427 NAPI_GRO_CB(p)->same_flow = !diffs;
Eric Dumazet89c5fa32012-12-10 13:28:16 +00004428 }
4429}
4430
Jerry Chu299603e82013-12-11 20:53:45 -08004431static void skb_gro_reset_offset(struct sk_buff *skb)
4432{
4433 const struct skb_shared_info *pinfo = skb_shinfo(skb);
4434 const skb_frag_t *frag0 = &pinfo->frags[0];
4435
4436 NAPI_GRO_CB(skb)->data_offset = 0;
4437 NAPI_GRO_CB(skb)->frag0 = NULL;
4438 NAPI_GRO_CB(skb)->frag0_len = 0;
4439
4440 if (skb_mac_header(skb) == skb_tail_pointer(skb) &&
4441 pinfo->nr_frags &&
4442 !PageHighMem(skb_frag_page(frag0))) {
4443 NAPI_GRO_CB(skb)->frag0 = skb_frag_address(frag0);
4444 NAPI_GRO_CB(skb)->frag0_len = skb_frag_size(frag0);
Herbert Xud565b0a2008-12-15 23:38:52 -08004445 }
4446}
4447
Eric Dumazeta50e2332014-03-29 21:28:21 -07004448static void gro_pull_from_frag0(struct sk_buff *skb, int grow)
4449{
4450 struct skb_shared_info *pinfo = skb_shinfo(skb);
4451
4452 BUG_ON(skb->end - skb->tail < grow);
4453
4454 memcpy(skb_tail_pointer(skb), NAPI_GRO_CB(skb)->frag0, grow);
4455
4456 skb->data_len -= grow;
4457 skb->tail += grow;
4458
4459 pinfo->frags[0].page_offset += grow;
4460 skb_frag_size_sub(&pinfo->frags[0], grow);
4461
4462 if (unlikely(!skb_frag_size(&pinfo->frags[0]))) {
4463 skb_frag_unref(skb, 0);
4464 memmove(pinfo->frags, pinfo->frags + 1,
4465 --pinfo->nr_frags * sizeof(pinfo->frags[0]));
4466 }
4467}
4468
Rami Rosenbb728822012-11-28 21:55:25 +00004469static enum gro_result dev_gro_receive(struct napi_struct *napi, struct sk_buff *skb)
Herbert Xud565b0a2008-12-15 23:38:52 -08004470{
4471 struct sk_buff **pp = NULL;
Vlad Yasevich22061d82012-11-15 08:49:11 +00004472 struct packet_offload *ptype;
Herbert Xud565b0a2008-12-15 23:38:52 -08004473 __be16 type = skb->protocol;
Vlad Yasevich22061d82012-11-15 08:49:11 +00004474 struct list_head *head = &offload_base;
Herbert Xu0da2afd52008-12-26 14:57:42 -08004475 int same_flow;
Ben Hutchings5b252f02009-10-29 07:17:09 +00004476 enum gro_result ret;
Eric Dumazeta50e2332014-03-29 21:28:21 -07004477 int grow;
Herbert Xud565b0a2008-12-15 23:38:52 -08004478
Eric W. Biederman9c62a682014-03-14 20:51:52 -07004479 if (!(skb->dev->features & NETIF_F_GRO))
Herbert Xud565b0a2008-12-15 23:38:52 -08004480 goto normal;
4481
Eric Dumazetd61d0722016-11-07 11:12:27 -08004482 if (skb->csum_bad)
Herbert Xuf17f5c92009-01-14 14:36:12 -08004483 goto normal;
4484
Eric Dumazet89c5fa32012-12-10 13:28:16 +00004485 gro_list_prepare(napi, skb);
4486
Herbert Xud565b0a2008-12-15 23:38:52 -08004487 rcu_read_lock();
4488 list_for_each_entry_rcu(ptype, head, list) {
Vlad Yasevichf191a1d2012-11-15 08:49:23 +00004489 if (ptype->type != type || !ptype->callbacks.gro_receive)
Herbert Xud565b0a2008-12-15 23:38:52 -08004490 continue;
4491
Herbert Xu86911732009-01-29 14:19:50 +00004492 skb_set_network_header(skb, skb_gro_offset(skb));
Eric Dumazetefd94502013-02-14 17:31:48 +00004493 skb_reset_mac_len(skb);
Herbert Xud565b0a2008-12-15 23:38:52 -08004494 NAPI_GRO_CB(skb)->same_flow = 0;
Eric Dumazetd61d0722016-11-07 11:12:27 -08004495 NAPI_GRO_CB(skb)->flush = skb_is_gso(skb) || skb_has_frag_list(skb);
Herbert Xu5d38a072009-01-04 16:13:40 -08004496 NAPI_GRO_CB(skb)->free = 0;
Jesse Grossfac8e0f2016-03-19 09:32:01 -07004497 NAPI_GRO_CB(skb)->encap_mark = 0;
Sabrina Dubrocafcd91dd2016-10-20 15:58:02 +02004498 NAPI_GRO_CB(skb)->recursion_counter = 0;
Alexander Duycka0ca1532016-04-05 09:13:39 -07004499 NAPI_GRO_CB(skb)->is_fou = 0;
Alexander Duyck15305452016-04-10 21:44:57 -04004500 NAPI_GRO_CB(skb)->is_atomic = 1;
Tom Herbert15e23962015-02-10 16:30:31 -08004501 NAPI_GRO_CB(skb)->gro_remcsum_start = 0;
Herbert Xud565b0a2008-12-15 23:38:52 -08004502
Tom Herbert662880f2014-08-27 21:26:56 -07004503 /* Setup for GRO checksum validation */
4504 switch (skb->ip_summed) {
4505 case CHECKSUM_COMPLETE:
4506 NAPI_GRO_CB(skb)->csum = skb->csum;
4507 NAPI_GRO_CB(skb)->csum_valid = 1;
4508 NAPI_GRO_CB(skb)->csum_cnt = 0;
4509 break;
4510 case CHECKSUM_UNNECESSARY:
4511 NAPI_GRO_CB(skb)->csum_cnt = skb->csum_level + 1;
4512 NAPI_GRO_CB(skb)->csum_valid = 0;
4513 break;
4514 default:
4515 NAPI_GRO_CB(skb)->csum_cnt = 0;
4516 NAPI_GRO_CB(skb)->csum_valid = 0;
4517 }
Herbert Xud565b0a2008-12-15 23:38:52 -08004518
Vlad Yasevichf191a1d2012-11-15 08:49:23 +00004519 pp = ptype->callbacks.gro_receive(&napi->gro_list, skb);
Herbert Xud565b0a2008-12-15 23:38:52 -08004520 break;
4521 }
4522 rcu_read_unlock();
4523
4524 if (&ptype->list == head)
4525 goto normal;
4526
Herbert Xu0da2afd52008-12-26 14:57:42 -08004527 same_flow = NAPI_GRO_CB(skb)->same_flow;
Herbert Xu5d0d9be2009-01-29 14:19:48 +00004528 ret = NAPI_GRO_CB(skb)->free ? GRO_MERGED_FREE : GRO_MERGED;
Herbert Xu0da2afd52008-12-26 14:57:42 -08004529
Herbert Xud565b0a2008-12-15 23:38:52 -08004530 if (pp) {
4531 struct sk_buff *nskb = *pp;
4532
4533 *pp = nskb->next;
4534 nskb->next = NULL;
4535 napi_gro_complete(nskb);
Herbert Xu4ae55442009-02-08 18:00:36 +00004536 napi->gro_count--;
Herbert Xud565b0a2008-12-15 23:38:52 -08004537 }
4538
Herbert Xu0da2afd52008-12-26 14:57:42 -08004539 if (same_flow)
Herbert Xud565b0a2008-12-15 23:38:52 -08004540 goto ok;
4541
Eric Dumazet600adc12014-01-09 14:12:19 -08004542 if (NAPI_GRO_CB(skb)->flush)
Herbert Xud565b0a2008-12-15 23:38:52 -08004543 goto normal;
Herbert Xud565b0a2008-12-15 23:38:52 -08004544
Eric Dumazet600adc12014-01-09 14:12:19 -08004545 if (unlikely(napi->gro_count >= MAX_GRO_SKBS)) {
4546 struct sk_buff *nskb = napi->gro_list;
4547
4548 /* locate the end of the list to select the 'oldest' flow */
4549 while (nskb->next) {
4550 pp = &nskb->next;
4551 nskb = *pp;
4552 }
4553 *pp = NULL;
4554 nskb->next = NULL;
4555 napi_gro_complete(nskb);
4556 } else {
4557 napi->gro_count++;
4558 }
Herbert Xud565b0a2008-12-15 23:38:52 -08004559 NAPI_GRO_CB(skb)->count = 1;
Eric Dumazet2e71a6f2012-10-06 08:08:49 +00004560 NAPI_GRO_CB(skb)->age = jiffies;
Eric Dumazet29e98242014-05-16 11:34:37 -07004561 NAPI_GRO_CB(skb)->last = skb;
Herbert Xu86911732009-01-29 14:19:50 +00004562 skb_shinfo(skb)->gso_size = skb_gro_len(skb);
Herbert Xud565b0a2008-12-15 23:38:52 -08004563 skb->next = napi->gro_list;
4564 napi->gro_list = skb;
Herbert Xu5d0d9be2009-01-29 14:19:48 +00004565 ret = GRO_HELD;
Herbert Xud565b0a2008-12-15 23:38:52 -08004566
Herbert Xuad0f9902009-02-01 01:24:55 -08004567pull:
Eric Dumazeta50e2332014-03-29 21:28:21 -07004568 grow = skb_gro_offset(skb) - skb_headlen(skb);
4569 if (grow > 0)
4570 gro_pull_from_frag0(skb, grow);
Herbert Xud565b0a2008-12-15 23:38:52 -08004571ok:
Herbert Xu5d0d9be2009-01-29 14:19:48 +00004572 return ret;
Herbert Xud565b0a2008-12-15 23:38:52 -08004573
4574normal:
Herbert Xuad0f9902009-02-01 01:24:55 -08004575 ret = GRO_NORMAL;
4576 goto pull;
Herbert Xu5d38a072009-01-04 16:13:40 -08004577}
Herbert Xu96e93ea2009-01-06 10:49:34 -08004578
Jerry Chubf5a7552014-01-07 10:23:19 -08004579struct packet_offload *gro_find_receive_by_type(__be16 type)
4580{
4581 struct list_head *offload_head = &offload_base;
4582 struct packet_offload *ptype;
4583
4584 list_for_each_entry_rcu(ptype, offload_head, list) {
4585 if (ptype->type != type || !ptype->callbacks.gro_receive)
4586 continue;
4587 return ptype;
4588 }
4589 return NULL;
4590}
Or Gerlitze27a2f82014-01-20 13:59:20 +02004591EXPORT_SYMBOL(gro_find_receive_by_type);
Jerry Chubf5a7552014-01-07 10:23:19 -08004592
4593struct packet_offload *gro_find_complete_by_type(__be16 type)
4594{
4595 struct list_head *offload_head = &offload_base;
4596 struct packet_offload *ptype;
4597
4598 list_for_each_entry_rcu(ptype, offload_head, list) {
4599 if (ptype->type != type || !ptype->callbacks.gro_complete)
4600 continue;
4601 return ptype;
4602 }
4603 return NULL;
4604}
Or Gerlitze27a2f82014-01-20 13:59:20 +02004605EXPORT_SYMBOL(gro_find_complete_by_type);
Herbert Xu96e93ea2009-01-06 10:49:34 -08004606
Rami Rosenbb728822012-11-28 21:55:25 +00004607static gro_result_t napi_skb_finish(gro_result_t ret, struct sk_buff *skb)
Herbert Xu5d38a072009-01-04 16:13:40 -08004608{
Herbert Xu5d0d9be2009-01-29 14:19:48 +00004609 switch (ret) {
4610 case GRO_NORMAL:
Ben Hutchingsae78dbf2014-01-10 22:17:24 +00004611 if (netif_receive_skb_internal(skb))
Ben Hutchingsc7c4b3b2009-10-29 21:36:53 -07004612 ret = GRO_DROP;
4613 break;
Herbert Xu5d38a072009-01-04 16:13:40 -08004614
Herbert Xu5d0d9be2009-01-29 14:19:48 +00004615 case GRO_DROP:
Herbert Xu5d38a072009-01-04 16:13:40 -08004616 kfree_skb(skb);
4617 break;
Ben Hutchings5b252f02009-10-29 07:17:09 +00004618
Eric Dumazetdaa86542012-04-19 07:07:40 +00004619 case GRO_MERGED_FREE:
Jesse Grossce87fc62016-01-20 17:59:49 -08004620 if (NAPI_GRO_CB(skb)->free == NAPI_GRO_FREE_STOLEN_HEAD) {
4621 skb_dst_drop(skb);
Eric Dumazetd7e88832012-04-30 08:10:34 +00004622 kmem_cache_free(skbuff_head_cache, skb);
Jesse Grossce87fc62016-01-20 17:59:49 -08004623 } else {
Eric Dumazetd7e88832012-04-30 08:10:34 +00004624 __kfree_skb(skb);
Jesse Grossce87fc62016-01-20 17:59:49 -08004625 }
Eric Dumazetdaa86542012-04-19 07:07:40 +00004626 break;
4627
Ben Hutchings5b252f02009-10-29 07:17:09 +00004628 case GRO_HELD:
4629 case GRO_MERGED:
4630 break;
Herbert Xu5d38a072009-01-04 16:13:40 -08004631 }
4632
Ben Hutchingsc7c4b3b2009-10-29 21:36:53 -07004633 return ret;
Herbert Xu5d0d9be2009-01-29 14:19:48 +00004634}
Herbert Xu5d0d9be2009-01-29 14:19:48 +00004635
Ben Hutchingsc7c4b3b2009-10-29 21:36:53 -07004636gro_result_t napi_gro_receive(struct napi_struct *napi, struct sk_buff *skb)
Herbert Xu5d0d9be2009-01-29 14:19:48 +00004637{
Eric Dumazet93f93a42015-11-18 06:30:59 -08004638 skb_mark_napi_id(skb, napi);
Ben Hutchingsae78dbf2014-01-10 22:17:24 +00004639 trace_napi_gro_receive_entry(skb);
Herbert Xu86911732009-01-29 14:19:50 +00004640
Eric Dumazeta50e2332014-03-29 21:28:21 -07004641 skb_gro_reset_offset(skb);
4642
Eric Dumazet89c5fa32012-12-10 13:28:16 +00004643 return napi_skb_finish(dev_gro_receive(napi, skb), skb);
Herbert Xud565b0a2008-12-15 23:38:52 -08004644}
4645EXPORT_SYMBOL(napi_gro_receive);
4646
stephen hemmingerd0c2b0d2010-10-19 07:12:10 +00004647static void napi_reuse_skb(struct napi_struct *napi, struct sk_buff *skb)
Herbert Xu96e93ea2009-01-06 10:49:34 -08004648{
Eric Dumazet93a35f52014-10-23 06:30:30 -07004649 if (unlikely(skb->pfmemalloc)) {
4650 consume_skb(skb);
4651 return;
4652 }
Herbert Xu96e93ea2009-01-06 10:49:34 -08004653 __skb_pull(skb, skb_headlen(skb));
Eric Dumazet2a2a4592012-03-21 06:58:03 +00004654 /* restore the reserve we had after netdev_alloc_skb_ip_align() */
4655 skb_reserve(skb, NET_SKB_PAD + NET_IP_ALIGN - skb_headroom(skb));
Jesse Gross3701e512010-10-20 13:56:06 +00004656 skb->vlan_tci = 0;
Herbert Xu66c46d72011-01-29 20:44:54 -08004657 skb->dev = napi->dev;
Andy Gospodarek6d152e22011-02-02 14:53:25 -08004658 skb->skb_iif = 0;
Jerry Chuc3caf112014-07-14 15:54:46 -07004659 skb->encapsulation = 0;
4660 skb_shinfo(skb)->gso_type = 0;
Eric Dumazete33d0ba2014-04-03 09:28:10 -07004661 skb->truesize = SKB_TRUESIZE(skb_end_offset(skb));
Herbert Xu96e93ea2009-01-06 10:49:34 -08004662
4663 napi->skb = skb;
4664}
Herbert Xu96e93ea2009-01-06 10:49:34 -08004665
Herbert Xu76620aa2009-04-16 02:02:07 -07004666struct sk_buff *napi_get_frags(struct napi_struct *napi)
Herbert Xu5d38a072009-01-04 16:13:40 -08004667{
Herbert Xu5d38a072009-01-04 16:13:40 -08004668 struct sk_buff *skb = napi->skb;
Herbert Xu5d38a072009-01-04 16:13:40 -08004669
4670 if (!skb) {
Alexander Duyckfd11a832014-12-09 19:40:49 -08004671 skb = napi_alloc_skb(napi, GRO_MAX_HEAD);
Eric Dumazete2f9dc32015-11-19 12:11:23 -08004672 if (skb) {
4673 napi->skb = skb;
4674 skb_mark_napi_id(skb, napi);
4675 }
Herbert Xu5d38a072009-01-04 16:13:40 -08004676 }
Herbert Xu96e93ea2009-01-06 10:49:34 -08004677 return skb;
4678}
Herbert Xu76620aa2009-04-16 02:02:07 -07004679EXPORT_SYMBOL(napi_get_frags);
Herbert Xu96e93ea2009-01-06 10:49:34 -08004680
Eric Dumazeta50e2332014-03-29 21:28:21 -07004681static gro_result_t napi_frags_finish(struct napi_struct *napi,
4682 struct sk_buff *skb,
4683 gro_result_t ret)
Herbert Xu5d0d9be2009-01-29 14:19:48 +00004684{
Herbert Xu5d0d9be2009-01-29 14:19:48 +00004685 switch (ret) {
4686 case GRO_NORMAL:
Eric Dumazeta50e2332014-03-29 21:28:21 -07004687 case GRO_HELD:
4688 __skb_push(skb, ETH_HLEN);
4689 skb->protocol = eth_type_trans(skb, skb->dev);
4690 if (ret == GRO_NORMAL && netif_receive_skb_internal(skb))
Ben Hutchingsc7c4b3b2009-10-29 21:36:53 -07004691 ret = GRO_DROP;
Herbert Xu86911732009-01-29 14:19:50 +00004692 break;
Herbert Xu5d0d9be2009-01-29 14:19:48 +00004693
4694 case GRO_DROP:
Herbert Xu5d0d9be2009-01-29 14:19:48 +00004695 case GRO_MERGED_FREE:
4696 napi_reuse_skb(napi, skb);
4697 break;
Ben Hutchings5b252f02009-10-29 07:17:09 +00004698
4699 case GRO_MERGED:
4700 break;
Herbert Xu5d0d9be2009-01-29 14:19:48 +00004701 }
4702
Ben Hutchingsc7c4b3b2009-10-29 21:36:53 -07004703 return ret;
Herbert Xu5d0d9be2009-01-29 14:19:48 +00004704}
Herbert Xu5d0d9be2009-01-29 14:19:48 +00004705
Eric Dumazeta50e2332014-03-29 21:28:21 -07004706/* Upper GRO stack assumes network header starts at gro_offset=0
4707 * Drivers could call both napi_gro_frags() and napi_gro_receive()
4708 * We copy ethernet header into skb->data to have a common layout.
4709 */
Eric Dumazet4adb9c42012-05-18 20:49:06 +00004710static struct sk_buff *napi_frags_skb(struct napi_struct *napi)
Herbert Xu96e93ea2009-01-06 10:49:34 -08004711{
Herbert Xu76620aa2009-04-16 02:02:07 -07004712 struct sk_buff *skb = napi->skb;
Eric Dumazeta50e2332014-03-29 21:28:21 -07004713 const struct ethhdr *eth;
4714 unsigned int hlen = sizeof(*eth);
Herbert Xu76620aa2009-04-16 02:02:07 -07004715
4716 napi->skb = NULL;
4717
Eric Dumazeta50e2332014-03-29 21:28:21 -07004718 skb_reset_mac_header(skb);
4719 skb_gro_reset_offset(skb);
4720
4721 eth = skb_gro_header_fast(skb, 0);
4722 if (unlikely(skb_gro_header_hard(skb, hlen))) {
4723 eth = skb_gro_header_slow(skb, hlen, 0);
4724 if (unlikely(!eth)) {
Aaron Conole4da46ce2016-04-02 15:26:43 -04004725 net_warn_ratelimited("%s: dropping impossible skb from %s\n",
4726 __func__, napi->dev->name);
Eric Dumazeta50e2332014-03-29 21:28:21 -07004727 napi_reuse_skb(napi, skb);
4728 return NULL;
4729 }
4730 } else {
4731 gro_pull_from_frag0(skb, hlen);
4732 NAPI_GRO_CB(skb)->frag0 += hlen;
4733 NAPI_GRO_CB(skb)->frag0_len -= hlen;
Herbert Xu76620aa2009-04-16 02:02:07 -07004734 }
Eric Dumazeta50e2332014-03-29 21:28:21 -07004735 __skb_pull(skb, hlen);
4736
4737 /*
4738 * This works because the only protocols we care about don't require
4739 * special handling.
4740 * We'll fix it up properly in napi_frags_finish()
4741 */
4742 skb->protocol = eth->h_proto;
Herbert Xu76620aa2009-04-16 02:02:07 -07004743
Herbert Xu76620aa2009-04-16 02:02:07 -07004744 return skb;
4745}
Herbert Xu76620aa2009-04-16 02:02:07 -07004746
Ben Hutchingsc7c4b3b2009-10-29 21:36:53 -07004747gro_result_t napi_gro_frags(struct napi_struct *napi)
Herbert Xu76620aa2009-04-16 02:02:07 -07004748{
4749 struct sk_buff *skb = napi_frags_skb(napi);
Herbert Xu96e93ea2009-01-06 10:49:34 -08004750
4751 if (!skb)
Ben Hutchingsc7c4b3b2009-10-29 21:36:53 -07004752 return GRO_DROP;
Herbert Xu96e93ea2009-01-06 10:49:34 -08004753
Ben Hutchingsae78dbf2014-01-10 22:17:24 +00004754 trace_napi_gro_frags_entry(skb);
4755
Eric Dumazet89c5fa32012-12-10 13:28:16 +00004756 return napi_frags_finish(napi, skb, dev_gro_receive(napi, skb));
Herbert Xu5d38a072009-01-04 16:13:40 -08004757}
4758EXPORT_SYMBOL(napi_gro_frags);
4759
Tom Herbert573e8fc2014-08-22 13:33:47 -07004760/* Compute the checksum from gro_offset and return the folded value
4761 * after adding in any pseudo checksum.
4762 */
4763__sum16 __skb_gro_checksum_complete(struct sk_buff *skb)
4764{
4765 __wsum wsum;
4766 __sum16 sum;
4767
4768 wsum = skb_checksum(skb, skb_gro_offset(skb), skb_gro_len(skb), 0);
4769
4770 /* NAPI_GRO_CB(skb)->csum holds pseudo checksum */
4771 sum = csum_fold(csum_add(NAPI_GRO_CB(skb)->csum, wsum));
4772 if (likely(!sum)) {
4773 if (unlikely(skb->ip_summed == CHECKSUM_COMPLETE) &&
4774 !skb->csum_complete_sw)
4775 netdev_rx_csum_fault(skb->dev);
4776 }
4777
4778 NAPI_GRO_CB(skb)->csum = wsum;
4779 NAPI_GRO_CB(skb)->csum_valid = 1;
4780
4781 return sum;
4782}
4783EXPORT_SYMBOL(__skb_gro_checksum_complete);
4784
Eric Dumazete326bed2010-04-22 00:22:45 -07004785/*
Zhi Yong Wu855abcf2014-01-01 04:34:50 +08004786 * net_rps_action_and_irq_enable sends any pending IPI's for rps.
Eric Dumazete326bed2010-04-22 00:22:45 -07004787 * Note: called with local irq disabled, but exits with local irq enabled.
4788 */
4789static void net_rps_action_and_irq_enable(struct softnet_data *sd)
4790{
4791#ifdef CONFIG_RPS
4792 struct softnet_data *remsd = sd->rps_ipi_list;
4793
4794 if (remsd) {
4795 sd->rps_ipi_list = NULL;
4796
4797 local_irq_enable();
4798
4799 /* Send pending IPI's to kick RPS processing on remote cpus. */
4800 while (remsd) {
4801 struct softnet_data *next = remsd->rps_ipi_next;
4802
4803 if (cpu_online(remsd->cpu))
Frederic Weisbeckerc46fff22014-02-24 16:40:02 +01004804 smp_call_function_single_async(remsd->cpu,
Frederic Weisbeckerfce8ad12014-02-24 16:40:01 +01004805 &remsd->csd);
Eric Dumazete326bed2010-04-22 00:22:45 -07004806 remsd = next;
4807 }
4808 } else
4809#endif
4810 local_irq_enable();
4811}
4812
Eric Dumazetd75b1ad2014-11-02 06:19:33 -08004813static bool sd_has_rps_ipi_waiting(struct softnet_data *sd)
4814{
4815#ifdef CONFIG_RPS
4816 return sd->rps_ipi_list != NULL;
4817#else
4818 return false;
4819#endif
4820}
4821
Stephen Hemmingerbea33482007-10-03 16:41:36 -07004822static int process_backlog(struct napi_struct *napi, int quota)
Linus Torvalds1da177e2005-04-16 15:20:36 -07004823{
Eric Dumazeteecfd7c2010-05-06 22:07:48 -07004824 struct softnet_data *sd = container_of(napi, struct softnet_data, backlog);
Paolo Abeni145dd5f2016-08-25 15:58:44 +02004825 bool again = true;
4826 int work = 0;
Linus Torvalds1da177e2005-04-16 15:20:36 -07004827
Eric Dumazete326bed2010-04-22 00:22:45 -07004828 /* Check if we have pending ipi, its better to send them now,
4829 * not waiting net_rx_action() end.
4830 */
Eric Dumazetd75b1ad2014-11-02 06:19:33 -08004831 if (sd_has_rps_ipi_waiting(sd)) {
Eric Dumazete326bed2010-04-22 00:22:45 -07004832 local_irq_disable();
4833 net_rps_action_and_irq_enable(sd);
4834 }
Eric Dumazetd75b1ad2014-11-02 06:19:33 -08004835
Stephen Hemmingerbea33482007-10-03 16:41:36 -07004836 napi->weight = weight_p;
Paolo Abeni145dd5f2016-08-25 15:58:44 +02004837 while (again) {
Linus Torvalds1da177e2005-04-16 15:20:36 -07004838 struct sk_buff *skb;
Linus Torvalds1da177e2005-04-16 15:20:36 -07004839
Changli Gao6e7676c2010-04-27 15:07:33 -07004840 while ((skb = __skb_dequeue(&sd->process_queue))) {
Julian Anastasov2c17d272015-07-09 09:59:10 +03004841 rcu_read_lock();
Changli Gao6e7676c2010-04-27 15:07:33 -07004842 __netif_receive_skb(skb);
Julian Anastasov2c17d272015-07-09 09:59:10 +03004843 rcu_read_unlock();
Tom Herbert76cc8b12010-05-20 18:37:59 +00004844 input_queue_head_incr(sd);
Paolo Abeni145dd5f2016-08-25 15:58:44 +02004845 if (++work >= quota)
Tom Herbert76cc8b12010-05-20 18:37:59 +00004846 return work;
Paolo Abeni145dd5f2016-08-25 15:58:44 +02004847
Stephen Hemmingerbea33482007-10-03 16:41:36 -07004848 }
Linus Torvalds1da177e2005-04-16 15:20:36 -07004849
Paolo Abeni145dd5f2016-08-25 15:58:44 +02004850 local_irq_disable();
Changli Gao6e7676c2010-04-27 15:07:33 -07004851 rps_lock(sd);
Tom Herbert11ef7a82014-06-30 09:50:40 -07004852 if (skb_queue_empty(&sd->input_pkt_queue)) {
Eric Dumazeteecfd7c2010-05-06 22:07:48 -07004853 /*
4854 * Inline a custom version of __napi_complete().
4855 * only current cpu owns and manipulates this napi,
Tom Herbert11ef7a82014-06-30 09:50:40 -07004856 * and NAPI_STATE_SCHED is the only possible flag set
4857 * on backlog.
4858 * We can use a plain write instead of clear_bit(),
Eric Dumazeteecfd7c2010-05-06 22:07:48 -07004859 * and we dont need an smp_mb() memory barrier.
4860 */
Eric Dumazeteecfd7c2010-05-06 22:07:48 -07004861 napi->state = 0;
Paolo Abeni145dd5f2016-08-25 15:58:44 +02004862 again = false;
4863 } else {
4864 skb_queue_splice_tail_init(&sd->input_pkt_queue,
4865 &sd->process_queue);
Changli Gao6e7676c2010-04-27 15:07:33 -07004866 }
4867 rps_unlock(sd);
Paolo Abeni145dd5f2016-08-25 15:58:44 +02004868 local_irq_enable();
Changli Gao6e7676c2010-04-27 15:07:33 -07004869 }
Linus Torvalds1da177e2005-04-16 15:20:36 -07004870
Stephen Hemmingerbea33482007-10-03 16:41:36 -07004871 return work;
Linus Torvalds1da177e2005-04-16 15:20:36 -07004872}
4873
Stephen Hemmingerbea33482007-10-03 16:41:36 -07004874/**
4875 * __napi_schedule - schedule for receive
Randy Dunlapc4ea43c2007-10-12 21:17:49 -07004876 * @n: entry to schedule
Stephen Hemmingerbea33482007-10-03 16:41:36 -07004877 *
Eric Dumazetbc9ad162014-10-28 18:05:13 -07004878 * The entry's receive function will be scheduled to run.
4879 * Consider using __napi_schedule_irqoff() if hard irqs are masked.
Stephen Hemmingerbea33482007-10-03 16:41:36 -07004880 */
Harvey Harrisonb5606c22008-02-13 15:03:16 -08004881void __napi_schedule(struct napi_struct *n)
Stephen Hemmingerbea33482007-10-03 16:41:36 -07004882{
4883 unsigned long flags;
4884
4885 local_irq_save(flags);
Christoph Lameter903ceff2014-08-17 12:30:35 -05004886 ____napi_schedule(this_cpu_ptr(&softnet_data), n);
Stephen Hemmingerbea33482007-10-03 16:41:36 -07004887 local_irq_restore(flags);
4888}
4889EXPORT_SYMBOL(__napi_schedule);
4890
Eric Dumazetbc9ad162014-10-28 18:05:13 -07004891/**
4892 * __napi_schedule_irqoff - schedule for receive
4893 * @n: entry to schedule
4894 *
4895 * Variant of __napi_schedule() assuming hard irqs are masked
4896 */
4897void __napi_schedule_irqoff(struct napi_struct *n)
4898{
4899 ____napi_schedule(this_cpu_ptr(&softnet_data), n);
4900}
4901EXPORT_SYMBOL(__napi_schedule_irqoff);
4902
Eric Dumazet364b6052016-11-15 10:15:13 -08004903bool __napi_complete(struct napi_struct *n)
Herbert Xud565b0a2008-12-15 23:38:52 -08004904{
4905 BUG_ON(!test_bit(NAPI_STATE_SCHED, &n->state));
Herbert Xud565b0a2008-12-15 23:38:52 -08004906
Eric Dumazet217f6972016-11-15 10:15:11 -08004907 /* Some drivers call us directly, instead of calling
4908 * napi_complete_done().
4909 */
4910 if (unlikely(test_bit(NAPI_STATE_IN_BUSY_POLL, &n->state)))
Eric Dumazet364b6052016-11-15 10:15:13 -08004911 return false;
Eric Dumazet217f6972016-11-15 10:15:11 -08004912
Eric Dumazetd75b1ad2014-11-02 06:19:33 -08004913 list_del_init(&n->poll_list);
Peter Zijlstra4e857c52014-03-17 18:06:10 +01004914 smp_mb__before_atomic();
Herbert Xud565b0a2008-12-15 23:38:52 -08004915 clear_bit(NAPI_STATE_SCHED, &n->state);
Eric Dumazet364b6052016-11-15 10:15:13 -08004916 return true;
Herbert Xud565b0a2008-12-15 23:38:52 -08004917}
4918EXPORT_SYMBOL(__napi_complete);
4919
Eric Dumazet364b6052016-11-15 10:15:13 -08004920bool napi_complete_done(struct napi_struct *n, int work_done)
Herbert Xud565b0a2008-12-15 23:38:52 -08004921{
4922 unsigned long flags;
4923
4924 /*
Eric Dumazet217f6972016-11-15 10:15:11 -08004925 * 1) Don't let napi dequeue from the cpu poll list
4926 * just in case its running on a different cpu.
4927 * 2) If we are busy polling, do nothing here, we have
4928 * the guarantee we will be called later.
Herbert Xud565b0a2008-12-15 23:38:52 -08004929 */
Eric Dumazet217f6972016-11-15 10:15:11 -08004930 if (unlikely(n->state & (NAPIF_STATE_NPSVC |
4931 NAPIF_STATE_IN_BUSY_POLL)))
Eric Dumazet364b6052016-11-15 10:15:13 -08004932 return false;
Herbert Xud565b0a2008-12-15 23:38:52 -08004933
Eric Dumazet3b47d302014-11-06 21:09:44 -08004934 if (n->gro_list) {
4935 unsigned long timeout = 0;
Eric Dumazetd75b1ad2014-11-02 06:19:33 -08004936
Eric Dumazet3b47d302014-11-06 21:09:44 -08004937 if (work_done)
4938 timeout = n->dev->gro_flush_timeout;
4939
4940 if (timeout)
4941 hrtimer_start(&n->timer, ns_to_ktime(timeout),
4942 HRTIMER_MODE_REL_PINNED);
4943 else
4944 napi_gro_flush(n, false);
4945 }
Eric Dumazetd75b1ad2014-11-02 06:19:33 -08004946 if (likely(list_empty(&n->poll_list))) {
4947 WARN_ON_ONCE(!test_and_clear_bit(NAPI_STATE_SCHED, &n->state));
4948 } else {
4949 /* If n->poll_list is not empty, we need to mask irqs */
4950 local_irq_save(flags);
4951 __napi_complete(n);
4952 local_irq_restore(flags);
4953 }
Eric Dumazet364b6052016-11-15 10:15:13 -08004954 return true;
Herbert Xud565b0a2008-12-15 23:38:52 -08004955}
Eric Dumazet3b47d302014-11-06 21:09:44 -08004956EXPORT_SYMBOL(napi_complete_done);
Herbert Xud565b0a2008-12-15 23:38:52 -08004957
Eliezer Tamiraf12fa62013-06-10 11:39:41 +03004958/* must be called under rcu_read_lock(), as we dont take a reference */
Eric Dumazet02d62e82015-11-18 06:30:52 -08004959static struct napi_struct *napi_by_id(unsigned int napi_id)
Eliezer Tamiraf12fa62013-06-10 11:39:41 +03004960{
4961 unsigned int hash = napi_id % HASH_SIZE(napi_hash);
4962 struct napi_struct *napi;
4963
4964 hlist_for_each_entry_rcu(napi, &napi_hash[hash], napi_hash_node)
4965 if (napi->napi_id == napi_id)
4966 return napi;
4967
4968 return NULL;
4969}
Eric Dumazet02d62e82015-11-18 06:30:52 -08004970
4971#if defined(CONFIG_NET_RX_BUSY_POLL)
Eric Dumazet217f6972016-11-15 10:15:11 -08004972
Eric Dumazetce6aea92015-11-18 06:30:54 -08004973#define BUSY_POLL_BUDGET 8
Eric Dumazet217f6972016-11-15 10:15:11 -08004974
4975static void busy_poll_stop(struct napi_struct *napi, void *have_poll_lock)
4976{
4977 int rc;
4978
4979 clear_bit(NAPI_STATE_IN_BUSY_POLL, &napi->state);
4980
4981 local_bh_disable();
4982
4983 /* All we really want here is to re-enable device interrupts.
4984 * Ideally, a new ndo_busy_poll_stop() could avoid another round.
4985 */
4986 rc = napi->poll(napi, BUSY_POLL_BUDGET);
4987 netpoll_poll_unlock(have_poll_lock);
4988 if (rc == BUSY_POLL_BUDGET)
4989 __napi_schedule(napi);
4990 local_bh_enable();
4991 if (local_softirq_pending())
4992 do_softirq();
4993}
4994
Eric Dumazet02d62e82015-11-18 06:30:52 -08004995bool sk_busy_loop(struct sock *sk, int nonblock)
4996{
4997 unsigned long end_time = !nonblock ? sk_busy_loop_end_time(sk) : 0;
Eric Dumazet217f6972016-11-15 10:15:11 -08004998 int (*napi_poll)(struct napi_struct *napi, int budget);
Eric Dumazetce6aea92015-11-18 06:30:54 -08004999 int (*busy_poll)(struct napi_struct *dev);
Eric Dumazet217f6972016-11-15 10:15:11 -08005000 void *have_poll_lock = NULL;
Eric Dumazet02d62e82015-11-18 06:30:52 -08005001 struct napi_struct *napi;
Eric Dumazet217f6972016-11-15 10:15:11 -08005002 int rc;
5003
5004restart:
5005 rc = false;
5006 napi_poll = NULL;
Eric Dumazet02d62e82015-11-18 06:30:52 -08005007
Eric Dumazet2a028ec2015-11-18 06:30:53 -08005008 rcu_read_lock();
Eric Dumazet02d62e82015-11-18 06:30:52 -08005009
5010 napi = napi_by_id(sk->sk_napi_id);
5011 if (!napi)
5012 goto out;
5013
Eric Dumazetce6aea92015-11-18 06:30:54 -08005014 /* Note: ndo_busy_poll method is optional in linux-4.5 */
5015 busy_poll = napi->dev->netdev_ops->ndo_busy_poll;
Eric Dumazet02d62e82015-11-18 06:30:52 -08005016
Eric Dumazet217f6972016-11-15 10:15:11 -08005017 preempt_disable();
5018 for (;;) {
Eric Dumazetce6aea92015-11-18 06:30:54 -08005019 rc = 0;
Eric Dumazet2a028ec2015-11-18 06:30:53 -08005020 local_bh_disable();
Eric Dumazetce6aea92015-11-18 06:30:54 -08005021 if (busy_poll) {
5022 rc = busy_poll(napi);
Eric Dumazet217f6972016-11-15 10:15:11 -08005023 goto count;
Eric Dumazetce6aea92015-11-18 06:30:54 -08005024 }
Eric Dumazet217f6972016-11-15 10:15:11 -08005025 if (!napi_poll) {
5026 unsigned long val = READ_ONCE(napi->state);
5027
5028 /* If multiple threads are competing for this napi,
5029 * we avoid dirtying napi->state as much as we can.
5030 */
5031 if (val & (NAPIF_STATE_DISABLE | NAPIF_STATE_SCHED |
5032 NAPIF_STATE_IN_BUSY_POLL))
5033 goto count;
5034 if (cmpxchg(&napi->state, val,
5035 val | NAPIF_STATE_IN_BUSY_POLL |
5036 NAPIF_STATE_SCHED) != val)
5037 goto count;
5038 have_poll_lock = netpoll_poll_lock(napi);
5039 napi_poll = napi->poll;
5040 }
5041 rc = napi_poll(napi, BUSY_POLL_BUDGET);
5042 trace_napi_poll(napi, rc, BUSY_POLL_BUDGET);
5043count:
Eric Dumazet2a028ec2015-11-18 06:30:53 -08005044 if (rc > 0)
Eric Dumazet02a1d6e2016-04-27 16:44:39 -07005045 __NET_ADD_STATS(sock_net(sk),
5046 LINUX_MIB_BUSYPOLLRXPACKETS, rc);
Eric Dumazet2a028ec2015-11-18 06:30:53 -08005047 local_bh_enable();
Eric Dumazet02d62e82015-11-18 06:30:52 -08005048
5049 if (rc == LL_FLUSH_FAILED)
5050 break; /* permanent failure */
5051
Eric Dumazet217f6972016-11-15 10:15:11 -08005052 if (nonblock || !skb_queue_empty(&sk->sk_receive_queue) ||
5053 busy_loop_timeout(end_time))
5054 break;
Eric Dumazet02d62e82015-11-18 06:30:52 -08005055
Eric Dumazet217f6972016-11-15 10:15:11 -08005056 if (unlikely(need_resched())) {
5057 if (napi_poll)
5058 busy_poll_stop(napi, have_poll_lock);
5059 preempt_enable();
5060 rcu_read_unlock();
5061 cond_resched();
5062 rc = !skb_queue_empty(&sk->sk_receive_queue);
5063 if (rc || busy_loop_timeout(end_time))
5064 return rc;
5065 goto restart;
5066 }
Linus Torvalds6cdf89b2016-12-12 10:48:02 -08005067 cpu_relax();
Eric Dumazet217f6972016-11-15 10:15:11 -08005068 }
5069 if (napi_poll)
5070 busy_poll_stop(napi, have_poll_lock);
5071 preempt_enable();
Eric Dumazet02d62e82015-11-18 06:30:52 -08005072 rc = !skb_queue_empty(&sk->sk_receive_queue);
5073out:
Eric Dumazet2a028ec2015-11-18 06:30:53 -08005074 rcu_read_unlock();
Eric Dumazet02d62e82015-11-18 06:30:52 -08005075 return rc;
5076}
5077EXPORT_SYMBOL(sk_busy_loop);
5078
5079#endif /* CONFIG_NET_RX_BUSY_POLL */
Eliezer Tamiraf12fa62013-06-10 11:39:41 +03005080
Eric Dumazet149d6ad2016-11-08 11:07:28 -08005081static void napi_hash_add(struct napi_struct *napi)
Eliezer Tamiraf12fa62013-06-10 11:39:41 +03005082{
Eric Dumazetd64b5e82015-11-18 06:31:00 -08005083 if (test_bit(NAPI_STATE_NO_BUSY_POLL, &napi->state) ||
5084 test_and_set_bit(NAPI_STATE_HASHED, &napi->state))
Eric Dumazet52bd2d62015-11-18 06:30:50 -08005085 return;
Eliezer Tamiraf12fa62013-06-10 11:39:41 +03005086
Eric Dumazet52bd2d62015-11-18 06:30:50 -08005087 spin_lock(&napi_hash_lock);
Eliezer Tamiraf12fa62013-06-10 11:39:41 +03005088
Eric Dumazet52bd2d62015-11-18 06:30:50 -08005089 /* 0..NR_CPUS+1 range is reserved for sender_cpu use */
5090 do {
5091 if (unlikely(++napi_gen_id < NR_CPUS + 1))
5092 napi_gen_id = NR_CPUS + 1;
5093 } while (napi_by_id(napi_gen_id));
5094 napi->napi_id = napi_gen_id;
Eliezer Tamiraf12fa62013-06-10 11:39:41 +03005095
Eric Dumazet52bd2d62015-11-18 06:30:50 -08005096 hlist_add_head_rcu(&napi->napi_hash_node,
5097 &napi_hash[napi->napi_id % HASH_SIZE(napi_hash)]);
Eliezer Tamiraf12fa62013-06-10 11:39:41 +03005098
Eric Dumazet52bd2d62015-11-18 06:30:50 -08005099 spin_unlock(&napi_hash_lock);
Eliezer Tamiraf12fa62013-06-10 11:39:41 +03005100}
Eliezer Tamiraf12fa62013-06-10 11:39:41 +03005101
5102/* Warning : caller is responsible to make sure rcu grace period
5103 * is respected before freeing memory containing @napi
5104 */
Eric Dumazet34cbe272015-11-18 06:31:02 -08005105bool napi_hash_del(struct napi_struct *napi)
Eliezer Tamiraf12fa62013-06-10 11:39:41 +03005106{
Eric Dumazet34cbe272015-11-18 06:31:02 -08005107 bool rcu_sync_needed = false;
5108
Eliezer Tamiraf12fa62013-06-10 11:39:41 +03005109 spin_lock(&napi_hash_lock);
5110
Eric Dumazet34cbe272015-11-18 06:31:02 -08005111 if (test_and_clear_bit(NAPI_STATE_HASHED, &napi->state)) {
5112 rcu_sync_needed = true;
Eliezer Tamiraf12fa62013-06-10 11:39:41 +03005113 hlist_del_rcu(&napi->napi_hash_node);
Eric Dumazet34cbe272015-11-18 06:31:02 -08005114 }
Eliezer Tamiraf12fa62013-06-10 11:39:41 +03005115 spin_unlock(&napi_hash_lock);
Eric Dumazet34cbe272015-11-18 06:31:02 -08005116 return rcu_sync_needed;
Eliezer Tamiraf12fa62013-06-10 11:39:41 +03005117}
5118EXPORT_SYMBOL_GPL(napi_hash_del);
5119
Eric Dumazet3b47d302014-11-06 21:09:44 -08005120static enum hrtimer_restart napi_watchdog(struct hrtimer *timer)
5121{
5122 struct napi_struct *napi;
5123
5124 napi = container_of(timer, struct napi_struct, timer);
5125 if (napi->gro_list)
5126 napi_schedule(napi);
5127
5128 return HRTIMER_NORESTART;
5129}
5130
Herbert Xud565b0a2008-12-15 23:38:52 -08005131void netif_napi_add(struct net_device *dev, struct napi_struct *napi,
5132 int (*poll)(struct napi_struct *, int), int weight)
5133{
5134 INIT_LIST_HEAD(&napi->poll_list);
Eric Dumazet3b47d302014-11-06 21:09:44 -08005135 hrtimer_init(&napi->timer, CLOCK_MONOTONIC, HRTIMER_MODE_REL_PINNED);
5136 napi->timer.function = napi_watchdog;
Herbert Xu4ae55442009-02-08 18:00:36 +00005137 napi->gro_count = 0;
Herbert Xud565b0a2008-12-15 23:38:52 -08005138 napi->gro_list = NULL;
Herbert Xu5d38a072009-01-04 16:13:40 -08005139 napi->skb = NULL;
Herbert Xud565b0a2008-12-15 23:38:52 -08005140 napi->poll = poll;
Eric Dumazet82dc3c62013-03-05 15:57:22 +00005141 if (weight > NAPI_POLL_WEIGHT)
5142 pr_err_once("netif_napi_add() called with weight %d on device %s\n",
5143 weight, dev->name);
Herbert Xud565b0a2008-12-15 23:38:52 -08005144 napi->weight = weight;
5145 list_add(&napi->dev_list, &dev->napi_list);
Herbert Xud565b0a2008-12-15 23:38:52 -08005146 napi->dev = dev;
Herbert Xu5d38a072009-01-04 16:13:40 -08005147#ifdef CONFIG_NETPOLL
Herbert Xud565b0a2008-12-15 23:38:52 -08005148 napi->poll_owner = -1;
5149#endif
5150 set_bit(NAPI_STATE_SCHED, &napi->state);
Eric Dumazet93d05d42015-11-18 06:31:03 -08005151 napi_hash_add(napi);
Herbert Xud565b0a2008-12-15 23:38:52 -08005152}
5153EXPORT_SYMBOL(netif_napi_add);
5154
Eric Dumazet3b47d302014-11-06 21:09:44 -08005155void napi_disable(struct napi_struct *n)
5156{
5157 might_sleep();
5158 set_bit(NAPI_STATE_DISABLE, &n->state);
5159
5160 while (test_and_set_bit(NAPI_STATE_SCHED, &n->state))
5161 msleep(1);
Neil Horman2d8bff12015-09-23 14:57:58 -04005162 while (test_and_set_bit(NAPI_STATE_NPSVC, &n->state))
5163 msleep(1);
Eric Dumazet3b47d302014-11-06 21:09:44 -08005164
5165 hrtimer_cancel(&n->timer);
5166
5167 clear_bit(NAPI_STATE_DISABLE, &n->state);
5168}
5169EXPORT_SYMBOL(napi_disable);
5170
Eric Dumazet93d05d42015-11-18 06:31:03 -08005171/* Must be called in process context */
Herbert Xud565b0a2008-12-15 23:38:52 -08005172void netif_napi_del(struct napi_struct *napi)
5173{
Eric Dumazet93d05d42015-11-18 06:31:03 -08005174 might_sleep();
5175 if (napi_hash_del(napi))
5176 synchronize_net();
Peter P Waskiewicz Jrd7b06632008-12-26 01:35:35 -08005177 list_del_init(&napi->dev_list);
Herbert Xu76620aa2009-04-16 02:02:07 -07005178 napi_free_frags(napi);
Herbert Xud565b0a2008-12-15 23:38:52 -08005179
Eric Dumazet289dccb2013-12-20 14:29:08 -08005180 kfree_skb_list(napi->gro_list);
Herbert Xud565b0a2008-12-15 23:38:52 -08005181 napi->gro_list = NULL;
Herbert Xu4ae55442009-02-08 18:00:36 +00005182 napi->gro_count = 0;
Herbert Xud565b0a2008-12-15 23:38:52 -08005183}
5184EXPORT_SYMBOL(netif_napi_del);
5185
Herbert Xu726ce702014-12-21 07:16:21 +11005186static int napi_poll(struct napi_struct *n, struct list_head *repoll)
5187{
5188 void *have;
5189 int work, weight;
5190
5191 list_del_init(&n->poll_list);
5192
5193 have = netpoll_poll_lock(n);
5194
5195 weight = n->weight;
5196
5197 /* This NAPI_STATE_SCHED test is for avoiding a race
5198 * with netpoll's poll_napi(). Only the entity which
5199 * obtains the lock and sees NAPI_STATE_SCHED set will
5200 * actually make the ->poll() call. Therefore we avoid
5201 * accidentally calling ->poll() when NAPI is not scheduled.
5202 */
5203 work = 0;
5204 if (test_bit(NAPI_STATE_SCHED, &n->state)) {
5205 work = n->poll(n, weight);
Jesper Dangaard Brouer1db19db2016-07-07 18:01:32 +02005206 trace_napi_poll(n, work, weight);
Herbert Xu726ce702014-12-21 07:16:21 +11005207 }
5208
5209 WARN_ON_ONCE(work > weight);
5210
5211 if (likely(work < weight))
5212 goto out_unlock;
5213
5214 /* Drivers must not modify the NAPI state if they
5215 * consume the entire weight. In such cases this code
5216 * still "owns" the NAPI instance and therefore can
5217 * move the instance around on the list at-will.
5218 */
5219 if (unlikely(napi_disable_pending(n))) {
5220 napi_complete(n);
5221 goto out_unlock;
5222 }
5223
5224 if (n->gro_list) {
5225 /* flush too old packets
5226 * If HZ < 1000, flush all packets.
5227 */
5228 napi_gro_flush(n, HZ >= 1000);
5229 }
5230
Herbert Xu001ce542014-12-21 07:16:22 +11005231 /* Some drivers may have called napi_schedule
5232 * prior to exhausting their budget.
5233 */
5234 if (unlikely(!list_empty(&n->poll_list))) {
5235 pr_warn_once("%s: Budget exhausted after napi rescheduled\n",
5236 n->dev ? n->dev->name : "backlog");
5237 goto out_unlock;
5238 }
5239
Herbert Xu726ce702014-12-21 07:16:21 +11005240 list_add_tail(&n->poll_list, repoll);
5241
5242out_unlock:
5243 netpoll_poll_unlock(have);
5244
5245 return work;
5246}
5247
Emese Revfy0766f782016-06-20 20:42:34 +02005248static __latent_entropy void net_rx_action(struct softirq_action *h)
Linus Torvalds1da177e2005-04-16 15:20:36 -07005249{
Christoph Lameter903ceff2014-08-17 12:30:35 -05005250 struct softnet_data *sd = this_cpu_ptr(&softnet_data);
Stephen Hemminger24f8b232008-11-03 17:14:38 -08005251 unsigned long time_limit = jiffies + 2;
Stephen Hemminger51b0bde2005-06-23 20:14:40 -07005252 int budget = netdev_budget;
Eric Dumazetd75b1ad2014-11-02 06:19:33 -08005253 LIST_HEAD(list);
5254 LIST_HEAD(repoll);
Matt Mackall53fb95d2005-08-11 19:27:43 -07005255
Linus Torvalds1da177e2005-04-16 15:20:36 -07005256 local_irq_disable();
Eric Dumazetd75b1ad2014-11-02 06:19:33 -08005257 list_splice_init(&sd->poll_list, &list);
5258 local_irq_enable();
Linus Torvalds1da177e2005-04-16 15:20:36 -07005259
Herbert Xuceb8d5b2014-12-21 07:16:25 +11005260 for (;;) {
Stephen Hemmingerbea33482007-10-03 16:41:36 -07005261 struct napi_struct *n;
Linus Torvalds1da177e2005-04-16 15:20:36 -07005262
Herbert Xuceb8d5b2014-12-21 07:16:25 +11005263 if (list_empty(&list)) {
5264 if (!sd_has_rps_ipi_waiting(sd) && list_empty(&repoll))
Eric Dumazetf52dffe2016-11-23 08:44:56 -08005265 goto out;
Herbert Xuceb8d5b2014-12-21 07:16:25 +11005266 break;
5267 }
5268
Herbert Xu6bd373e2014-12-21 07:16:24 +11005269 n = list_first_entry(&list, struct napi_struct, poll_list);
5270 budget -= napi_poll(n, &repoll);
5271
Eric Dumazetd75b1ad2014-11-02 06:19:33 -08005272 /* If softirq window is exhausted then punt.
Stephen Hemminger24f8b232008-11-03 17:14:38 -08005273 * Allow this to run for 2 jiffies since which will allow
5274 * an average latency of 1.5/HZ.
Stephen Hemmingerbea33482007-10-03 16:41:36 -07005275 */
Herbert Xuceb8d5b2014-12-21 07:16:25 +11005276 if (unlikely(budget <= 0 ||
5277 time_after_eq(jiffies, time_limit))) {
5278 sd->time_squeeze++;
5279 break;
5280 }
Linus Torvalds1da177e2005-04-16 15:20:36 -07005281 }
Eric Dumazetd75b1ad2014-11-02 06:19:33 -08005282
Eric Dumazetd75b1ad2014-11-02 06:19:33 -08005283 local_irq_disable();
5284
5285 list_splice_tail_init(&sd->poll_list, &list);
5286 list_splice_tail(&repoll, &list);
5287 list_splice(&list, &sd->poll_list);
5288 if (!list_empty(&sd->poll_list))
5289 __raise_softirq_irqoff(NET_RX_SOFTIRQ);
5290
Eric Dumazete326bed2010-04-22 00:22:45 -07005291 net_rps_action_and_irq_enable(sd);
Eric Dumazetf52dffe2016-11-23 08:44:56 -08005292out:
5293 __kfree_skb_flush();
Linus Torvalds1da177e2005-04-16 15:20:36 -07005294}
5295
Veaceslav Falicoaa9d8562013-08-28 23:25:04 +02005296struct netdev_adjacent {
Jiri Pirko9ff162a2013-01-03 22:48:49 +00005297 struct net_device *dev;
Veaceslav Falico5d261912013-08-28 23:25:05 +02005298
5299 /* upper master flag, there can only be one master device per list */
Jiri Pirko9ff162a2013-01-03 22:48:49 +00005300 bool master;
Veaceslav Falico5d261912013-08-28 23:25:05 +02005301
Veaceslav Falico5d261912013-08-28 23:25:05 +02005302 /* counter for the number of times this device was added to us */
5303 u16 ref_nr;
5304
Veaceslav Falico402dae92013-09-25 09:20:09 +02005305 /* private field for the users */
5306 void *private;
5307
Jiri Pirko9ff162a2013-01-03 22:48:49 +00005308 struct list_head list;
5309 struct rcu_head rcu;
Jiri Pirko9ff162a2013-01-03 22:48:49 +00005310};
5311
Michal Kubeček6ea29da2015-09-24 10:59:05 +02005312static struct netdev_adjacent *__netdev_find_adj(struct net_device *adj_dev,
Veaceslav Falico2f268f12013-09-25 09:20:07 +02005313 struct list_head *adj_list)
Jiri Pirko9ff162a2013-01-03 22:48:49 +00005314{
Veaceslav Falico5d261912013-08-28 23:25:05 +02005315 struct netdev_adjacent *adj;
Veaceslav Falico5d261912013-08-28 23:25:05 +02005316
Veaceslav Falico2f268f12013-09-25 09:20:07 +02005317 list_for_each_entry(adj, adj_list, list) {
Veaceslav Falico5d261912013-08-28 23:25:05 +02005318 if (adj->dev == adj_dev)
5319 return adj;
Jiri Pirko9ff162a2013-01-03 22:48:49 +00005320 }
5321 return NULL;
5322}
5323
David Ahernf1170fd2016-10-17 19:15:51 -07005324static int __netdev_has_upper_dev(struct net_device *upper_dev, void *data)
5325{
5326 struct net_device *dev = data;
5327
5328 return upper_dev == dev;
5329}
5330
Jiri Pirko9ff162a2013-01-03 22:48:49 +00005331/**
5332 * netdev_has_upper_dev - Check if device is linked to an upper device
5333 * @dev: device
5334 * @upper_dev: upper device to check
5335 *
5336 * Find out if a device is linked to specified upper device and return true
5337 * in case it is. Note that this checks only immediate upper device,
5338 * not through a complete stack of devices. The caller must hold the RTNL lock.
5339 */
5340bool netdev_has_upper_dev(struct net_device *dev,
5341 struct net_device *upper_dev)
5342{
5343 ASSERT_RTNL();
5344
David Ahernf1170fd2016-10-17 19:15:51 -07005345 return netdev_walk_all_upper_dev_rcu(dev, __netdev_has_upper_dev,
5346 upper_dev);
Jiri Pirko9ff162a2013-01-03 22:48:49 +00005347}
5348EXPORT_SYMBOL(netdev_has_upper_dev);
5349
5350/**
David Ahern1a3f0602016-10-17 19:15:44 -07005351 * netdev_has_upper_dev_all - Check if device is linked to an upper device
5352 * @dev: device
5353 * @upper_dev: upper device to check
5354 *
5355 * Find out if a device is linked to specified upper device and return true
5356 * in case it is. Note that this checks the entire upper device chain.
5357 * The caller must hold rcu lock.
5358 */
5359
David Ahern1a3f0602016-10-17 19:15:44 -07005360bool netdev_has_upper_dev_all_rcu(struct net_device *dev,
5361 struct net_device *upper_dev)
5362{
5363 return !!netdev_walk_all_upper_dev_rcu(dev, __netdev_has_upper_dev,
5364 upper_dev);
5365}
5366EXPORT_SYMBOL(netdev_has_upper_dev_all_rcu);
5367
5368/**
Jiri Pirko9ff162a2013-01-03 22:48:49 +00005369 * netdev_has_any_upper_dev - Check if device is linked to some device
5370 * @dev: device
5371 *
5372 * Find out if a device is linked to an upper device and return true in case
5373 * it is. The caller must hold the RTNL lock.
5374 */
stephen hemminger1d143d92013-12-29 14:01:29 -08005375static bool netdev_has_any_upper_dev(struct net_device *dev)
Jiri Pirko9ff162a2013-01-03 22:48:49 +00005376{
5377 ASSERT_RTNL();
5378
David Ahernf1170fd2016-10-17 19:15:51 -07005379 return !list_empty(&dev->adj_list.upper);
Jiri Pirko9ff162a2013-01-03 22:48:49 +00005380}
Jiri Pirko9ff162a2013-01-03 22:48:49 +00005381
5382/**
5383 * netdev_master_upper_dev_get - Get master upper device
5384 * @dev: device
5385 *
5386 * Find a master upper device and return pointer to it or NULL in case
5387 * it's not there. The caller must hold the RTNL lock.
5388 */
5389struct net_device *netdev_master_upper_dev_get(struct net_device *dev)
5390{
Veaceslav Falicoaa9d8562013-08-28 23:25:04 +02005391 struct netdev_adjacent *upper;
Jiri Pirko9ff162a2013-01-03 22:48:49 +00005392
5393 ASSERT_RTNL();
5394
Veaceslav Falico2f268f12013-09-25 09:20:07 +02005395 if (list_empty(&dev->adj_list.upper))
Jiri Pirko9ff162a2013-01-03 22:48:49 +00005396 return NULL;
5397
Veaceslav Falico2f268f12013-09-25 09:20:07 +02005398 upper = list_first_entry(&dev->adj_list.upper,
Veaceslav Falicoaa9d8562013-08-28 23:25:04 +02005399 struct netdev_adjacent, list);
Jiri Pirko9ff162a2013-01-03 22:48:49 +00005400 if (likely(upper->master))
5401 return upper->dev;
5402 return NULL;
5403}
5404EXPORT_SYMBOL(netdev_master_upper_dev_get);
5405
David Ahern0f524a82016-10-17 19:15:52 -07005406/**
5407 * netdev_has_any_lower_dev - Check if device is linked to some device
5408 * @dev: device
5409 *
5410 * Find out if a device is linked to a lower device and return true in case
5411 * it is. The caller must hold the RTNL lock.
5412 */
5413static bool netdev_has_any_lower_dev(struct net_device *dev)
5414{
5415 ASSERT_RTNL();
5416
5417 return !list_empty(&dev->adj_list.lower);
5418}
5419
Veaceslav Falicob6ccba42013-09-25 09:20:23 +02005420void *netdev_adjacent_get_private(struct list_head *adj_list)
5421{
5422 struct netdev_adjacent *adj;
5423
5424 adj = list_entry(adj_list, struct netdev_adjacent, list);
5425
5426 return adj->private;
5427}
5428EXPORT_SYMBOL(netdev_adjacent_get_private);
5429
Veaceslav Falico31088a12013-09-25 09:20:12 +02005430/**
Vlad Yasevich44a40852014-05-16 17:20:38 -04005431 * netdev_upper_get_next_dev_rcu - Get the next dev from upper list
5432 * @dev: device
5433 * @iter: list_head ** of the current position
5434 *
5435 * Gets the next device from the dev's upper list, starting from iter
5436 * position. The caller must hold RCU read lock.
5437 */
5438struct net_device *netdev_upper_get_next_dev_rcu(struct net_device *dev,
5439 struct list_head **iter)
5440{
5441 struct netdev_adjacent *upper;
5442
5443 WARN_ON_ONCE(!rcu_read_lock_held() && !lockdep_rtnl_is_held());
5444
5445 upper = list_entry_rcu((*iter)->next, struct netdev_adjacent, list);
5446
5447 if (&upper->list == &dev->adj_list.upper)
5448 return NULL;
5449
5450 *iter = &upper->list;
5451
5452 return upper->dev;
5453}
5454EXPORT_SYMBOL(netdev_upper_get_next_dev_rcu);
5455
David Ahern1a3f0602016-10-17 19:15:44 -07005456static struct net_device *netdev_next_upper_dev_rcu(struct net_device *dev,
5457 struct list_head **iter)
5458{
5459 struct netdev_adjacent *upper;
5460
5461 WARN_ON_ONCE(!rcu_read_lock_held() && !lockdep_rtnl_is_held());
5462
5463 upper = list_entry_rcu((*iter)->next, struct netdev_adjacent, list);
5464
5465 if (&upper->list == &dev->adj_list.upper)
5466 return NULL;
5467
5468 *iter = &upper->list;
5469
5470 return upper->dev;
5471}
5472
5473int netdev_walk_all_upper_dev_rcu(struct net_device *dev,
5474 int (*fn)(struct net_device *dev,
5475 void *data),
5476 void *data)
5477{
5478 struct net_device *udev;
5479 struct list_head *iter;
5480 int ret;
5481
5482 for (iter = &dev->adj_list.upper,
5483 udev = netdev_next_upper_dev_rcu(dev, &iter);
5484 udev;
5485 udev = netdev_next_upper_dev_rcu(dev, &iter)) {
5486 /* first is the upper device itself */
5487 ret = fn(udev, data);
5488 if (ret)
5489 return ret;
5490
5491 /* then look at all of its upper devices */
5492 ret = netdev_walk_all_upper_dev_rcu(udev, fn, data);
5493 if (ret)
5494 return ret;
5495 }
5496
5497 return 0;
5498}
5499EXPORT_SYMBOL_GPL(netdev_walk_all_upper_dev_rcu);
5500
Jiri Pirko9ff162a2013-01-03 22:48:49 +00005501/**
Veaceslav Falico31088a12013-09-25 09:20:12 +02005502 * netdev_lower_get_next_private - Get the next ->private from the
5503 * lower neighbour list
5504 * @dev: device
5505 * @iter: list_head ** of the current position
5506 *
5507 * Gets the next netdev_adjacent->private from the dev's lower neighbour
5508 * list, starting from iter position. The caller must hold either hold the
5509 * RTNL lock or its own locking that guarantees that the neighbour lower
subashab@codeaurora.orgb4691392015-07-24 03:03:29 +00005510 * list will remain unchanged.
Veaceslav Falico31088a12013-09-25 09:20:12 +02005511 */
5512void *netdev_lower_get_next_private(struct net_device *dev,
5513 struct list_head **iter)
5514{
5515 struct netdev_adjacent *lower;
5516
5517 lower = list_entry(*iter, struct netdev_adjacent, list);
5518
5519 if (&lower->list == &dev->adj_list.lower)
5520 return NULL;
5521
Veaceslav Falico6859e7d2014-04-07 11:25:12 +02005522 *iter = lower->list.next;
Veaceslav Falico31088a12013-09-25 09:20:12 +02005523
5524 return lower->private;
5525}
5526EXPORT_SYMBOL(netdev_lower_get_next_private);
5527
5528/**
5529 * netdev_lower_get_next_private_rcu - Get the next ->private from the
5530 * lower neighbour list, RCU
5531 * variant
5532 * @dev: device
5533 * @iter: list_head ** of the current position
5534 *
5535 * Gets the next netdev_adjacent->private from the dev's lower neighbour
5536 * list, starting from iter position. The caller must hold RCU read lock.
5537 */
5538void *netdev_lower_get_next_private_rcu(struct net_device *dev,
5539 struct list_head **iter)
5540{
5541 struct netdev_adjacent *lower;
5542
5543 WARN_ON_ONCE(!rcu_read_lock_held());
5544
5545 lower = list_entry_rcu((*iter)->next, struct netdev_adjacent, list);
5546
5547 if (&lower->list == &dev->adj_list.lower)
5548 return NULL;
5549
Veaceslav Falico6859e7d2014-04-07 11:25:12 +02005550 *iter = &lower->list;
Veaceslav Falico31088a12013-09-25 09:20:12 +02005551
5552 return lower->private;
5553}
5554EXPORT_SYMBOL(netdev_lower_get_next_private_rcu);
5555
5556/**
Vlad Yasevich4085ebe2014-05-16 17:04:53 -04005557 * netdev_lower_get_next - Get the next device from the lower neighbour
5558 * list
5559 * @dev: device
5560 * @iter: list_head ** of the current position
5561 *
5562 * Gets the next netdev_adjacent from the dev's lower neighbour
5563 * list, starting from iter position. The caller must hold RTNL lock or
5564 * its own locking that guarantees that the neighbour lower
subashab@codeaurora.orgb4691392015-07-24 03:03:29 +00005565 * list will remain unchanged.
Vlad Yasevich4085ebe2014-05-16 17:04:53 -04005566 */
5567void *netdev_lower_get_next(struct net_device *dev, struct list_head **iter)
5568{
5569 struct netdev_adjacent *lower;
5570
Nikolay Aleksandrovcfdd28b2016-02-17 18:00:31 +01005571 lower = list_entry(*iter, struct netdev_adjacent, list);
Vlad Yasevich4085ebe2014-05-16 17:04:53 -04005572
5573 if (&lower->list == &dev->adj_list.lower)
5574 return NULL;
5575
Nikolay Aleksandrovcfdd28b2016-02-17 18:00:31 +01005576 *iter = lower->list.next;
Vlad Yasevich4085ebe2014-05-16 17:04:53 -04005577
5578 return lower->dev;
5579}
5580EXPORT_SYMBOL(netdev_lower_get_next);
5581
David Ahern1a3f0602016-10-17 19:15:44 -07005582static struct net_device *netdev_next_lower_dev(struct net_device *dev,
5583 struct list_head **iter)
5584{
5585 struct netdev_adjacent *lower;
5586
David Ahern46b5ab12016-10-26 13:21:33 -07005587 lower = list_entry((*iter)->next, struct netdev_adjacent, list);
David Ahern1a3f0602016-10-17 19:15:44 -07005588
5589 if (&lower->list == &dev->adj_list.lower)
5590 return NULL;
5591
David Ahern46b5ab12016-10-26 13:21:33 -07005592 *iter = &lower->list;
David Ahern1a3f0602016-10-17 19:15:44 -07005593
5594 return lower->dev;
5595}
5596
5597int netdev_walk_all_lower_dev(struct net_device *dev,
5598 int (*fn)(struct net_device *dev,
5599 void *data),
5600 void *data)
5601{
5602 struct net_device *ldev;
5603 struct list_head *iter;
5604 int ret;
5605
5606 for (iter = &dev->adj_list.lower,
5607 ldev = netdev_next_lower_dev(dev, &iter);
5608 ldev;
5609 ldev = netdev_next_lower_dev(dev, &iter)) {
5610 /* first is the lower device itself */
5611 ret = fn(ldev, data);
5612 if (ret)
5613 return ret;
5614
5615 /* then look at all of its lower devices */
5616 ret = netdev_walk_all_lower_dev(ldev, fn, data);
5617 if (ret)
5618 return ret;
5619 }
5620
5621 return 0;
5622}
5623EXPORT_SYMBOL_GPL(netdev_walk_all_lower_dev);
5624
David Ahern1a3f0602016-10-17 19:15:44 -07005625static struct net_device *netdev_next_lower_dev_rcu(struct net_device *dev,
5626 struct list_head **iter)
5627{
5628 struct netdev_adjacent *lower;
5629
5630 lower = list_entry_rcu((*iter)->next, struct netdev_adjacent, list);
5631 if (&lower->list == &dev->adj_list.lower)
5632 return NULL;
5633
5634 *iter = &lower->list;
5635
5636 return lower->dev;
5637}
5638
5639int netdev_walk_all_lower_dev_rcu(struct net_device *dev,
5640 int (*fn)(struct net_device *dev,
5641 void *data),
5642 void *data)
5643{
5644 struct net_device *ldev;
5645 struct list_head *iter;
5646 int ret;
5647
5648 for (iter = &dev->adj_list.lower,
5649 ldev = netdev_next_lower_dev_rcu(dev, &iter);
5650 ldev;
5651 ldev = netdev_next_lower_dev_rcu(dev, &iter)) {
5652 /* first is the lower device itself */
5653 ret = fn(ldev, data);
5654 if (ret)
5655 return ret;
5656
5657 /* then look at all of its lower devices */
5658 ret = netdev_walk_all_lower_dev_rcu(ldev, fn, data);
5659 if (ret)
5660 return ret;
5661 }
5662
5663 return 0;
5664}
5665EXPORT_SYMBOL_GPL(netdev_walk_all_lower_dev_rcu);
5666
Jiri Pirko7ce856a2016-07-04 08:23:12 +02005667/**
dingtianhonge001bfa2013-12-13 10:19:55 +08005668 * netdev_lower_get_first_private_rcu - Get the first ->private from the
5669 * lower neighbour list, RCU
5670 * variant
5671 * @dev: device
5672 *
5673 * Gets the first netdev_adjacent->private from the dev's lower neighbour
5674 * list. The caller must hold RCU read lock.
5675 */
5676void *netdev_lower_get_first_private_rcu(struct net_device *dev)
5677{
5678 struct netdev_adjacent *lower;
5679
5680 lower = list_first_or_null_rcu(&dev->adj_list.lower,
5681 struct netdev_adjacent, list);
5682 if (lower)
5683 return lower->private;
5684 return NULL;
5685}
5686EXPORT_SYMBOL(netdev_lower_get_first_private_rcu);
5687
5688/**
Jiri Pirko9ff162a2013-01-03 22:48:49 +00005689 * netdev_master_upper_dev_get_rcu - Get master upper device
5690 * @dev: device
5691 *
5692 * Find a master upper device and return pointer to it or NULL in case
5693 * it's not there. The caller must hold the RCU read lock.
5694 */
5695struct net_device *netdev_master_upper_dev_get_rcu(struct net_device *dev)
5696{
Veaceslav Falicoaa9d8562013-08-28 23:25:04 +02005697 struct netdev_adjacent *upper;
Jiri Pirko9ff162a2013-01-03 22:48:49 +00005698
Veaceslav Falico2f268f12013-09-25 09:20:07 +02005699 upper = list_first_or_null_rcu(&dev->adj_list.upper,
Veaceslav Falicoaa9d8562013-08-28 23:25:04 +02005700 struct netdev_adjacent, list);
Jiri Pirko9ff162a2013-01-03 22:48:49 +00005701 if (upper && likely(upper->master))
5702 return upper->dev;
5703 return NULL;
5704}
5705EXPORT_SYMBOL(netdev_master_upper_dev_get_rcu);
5706
Rashika Kheria0a59f3a2014-02-09 20:26:25 +05305707static int netdev_adjacent_sysfs_add(struct net_device *dev,
Veaceslav Falico3ee32702014-01-14 21:58:50 +01005708 struct net_device *adj_dev,
5709 struct list_head *dev_list)
5710{
5711 char linkname[IFNAMSIZ+7];
5712 sprintf(linkname, dev_list == &dev->adj_list.upper ?
5713 "upper_%s" : "lower_%s", adj_dev->name);
5714 return sysfs_create_link(&(dev->dev.kobj), &(adj_dev->dev.kobj),
5715 linkname);
5716}
Rashika Kheria0a59f3a2014-02-09 20:26:25 +05305717static void netdev_adjacent_sysfs_del(struct net_device *dev,
Veaceslav Falico3ee32702014-01-14 21:58:50 +01005718 char *name,
5719 struct list_head *dev_list)
5720{
5721 char linkname[IFNAMSIZ+7];
5722 sprintf(linkname, dev_list == &dev->adj_list.upper ?
5723 "upper_%s" : "lower_%s", name);
5724 sysfs_remove_link(&(dev->dev.kobj), linkname);
5725}
5726
Alexander Y. Fomichev7ce64c72014-09-15 14:22:35 +04005727static inline bool netdev_adjacent_is_neigh_list(struct net_device *dev,
5728 struct net_device *adj_dev,
5729 struct list_head *dev_list)
5730{
5731 return (dev_list == &dev->adj_list.upper ||
5732 dev_list == &dev->adj_list.lower) &&
5733 net_eq(dev_net(dev), dev_net(adj_dev));
5734}
Veaceslav Falico3ee32702014-01-14 21:58:50 +01005735
Veaceslav Falico5d261912013-08-28 23:25:05 +02005736static int __netdev_adjacent_dev_insert(struct net_device *dev,
5737 struct net_device *adj_dev,
Veaceslav Falico7863c052013-09-25 09:20:06 +02005738 struct list_head *dev_list,
Veaceslav Falico402dae92013-09-25 09:20:09 +02005739 void *private, bool master)
Veaceslav Falico5d261912013-08-28 23:25:05 +02005740{
5741 struct netdev_adjacent *adj;
Veaceslav Falico842d67a2013-09-25 09:20:31 +02005742 int ret;
Veaceslav Falico5d261912013-08-28 23:25:05 +02005743
Michal Kubeček6ea29da2015-09-24 10:59:05 +02005744 adj = __netdev_find_adj(adj_dev, dev_list);
Veaceslav Falico5d261912013-08-28 23:25:05 +02005745
5746 if (adj) {
David Ahern790510d2016-10-17 19:15:43 -07005747 adj->ref_nr += 1;
David Ahern67b62f92016-10-17 19:15:53 -07005748 pr_debug("Insert adjacency: dev %s adj_dev %s adj->ref_nr %d\n",
5749 dev->name, adj_dev->name, adj->ref_nr);
5750
Veaceslav Falico5d261912013-08-28 23:25:05 +02005751 return 0;
5752 }
5753
5754 adj = kmalloc(sizeof(*adj), GFP_KERNEL);
5755 if (!adj)
5756 return -ENOMEM;
5757
5758 adj->dev = adj_dev;
5759 adj->master = master;
David Ahern790510d2016-10-17 19:15:43 -07005760 adj->ref_nr = 1;
Veaceslav Falico402dae92013-09-25 09:20:09 +02005761 adj->private = private;
Veaceslav Falico5d261912013-08-28 23:25:05 +02005762 dev_hold(adj_dev);
Veaceslav Falico2f268f12013-09-25 09:20:07 +02005763
David Ahern67b62f92016-10-17 19:15:53 -07005764 pr_debug("Insert adjacency: dev %s adj_dev %s adj->ref_nr %d; dev_hold on %s\n",
5765 dev->name, adj_dev->name, adj->ref_nr, adj_dev->name);
Veaceslav Falico5d261912013-08-28 23:25:05 +02005766
Alexander Y. Fomichev7ce64c72014-09-15 14:22:35 +04005767 if (netdev_adjacent_is_neigh_list(dev, adj_dev, dev_list)) {
Veaceslav Falico3ee32702014-01-14 21:58:50 +01005768 ret = netdev_adjacent_sysfs_add(dev, adj_dev, dev_list);
Veaceslav Falico5831d662013-09-25 09:20:32 +02005769 if (ret)
5770 goto free_adj;
5771 }
5772
Veaceslav Falico7863c052013-09-25 09:20:06 +02005773 /* Ensure that master link is always the first item in list. */
Veaceslav Falico842d67a2013-09-25 09:20:31 +02005774 if (master) {
5775 ret = sysfs_create_link(&(dev->dev.kobj),
5776 &(adj_dev->dev.kobj), "master");
5777 if (ret)
Veaceslav Falico5831d662013-09-25 09:20:32 +02005778 goto remove_symlinks;
Veaceslav Falico842d67a2013-09-25 09:20:31 +02005779
Veaceslav Falico7863c052013-09-25 09:20:06 +02005780 list_add_rcu(&adj->list, dev_list);
Veaceslav Falico842d67a2013-09-25 09:20:31 +02005781 } else {
Veaceslav Falico7863c052013-09-25 09:20:06 +02005782 list_add_tail_rcu(&adj->list, dev_list);
Veaceslav Falico842d67a2013-09-25 09:20:31 +02005783 }
Veaceslav Falico5d261912013-08-28 23:25:05 +02005784
5785 return 0;
Veaceslav Falico842d67a2013-09-25 09:20:31 +02005786
Veaceslav Falico5831d662013-09-25 09:20:32 +02005787remove_symlinks:
Alexander Y. Fomichev7ce64c72014-09-15 14:22:35 +04005788 if (netdev_adjacent_is_neigh_list(dev, adj_dev, dev_list))
Veaceslav Falico3ee32702014-01-14 21:58:50 +01005789 netdev_adjacent_sysfs_del(dev, adj_dev->name, dev_list);
Veaceslav Falico842d67a2013-09-25 09:20:31 +02005790free_adj:
5791 kfree(adj);
Nikolay Aleksandrov974daef2013-10-23 15:28:56 +02005792 dev_put(adj_dev);
Veaceslav Falico842d67a2013-09-25 09:20:31 +02005793
5794 return ret;
Veaceslav Falico5d261912013-08-28 23:25:05 +02005795}
5796
stephen hemminger1d143d92013-12-29 14:01:29 -08005797static void __netdev_adjacent_dev_remove(struct net_device *dev,
5798 struct net_device *adj_dev,
Andrew Collins93409032016-10-03 13:43:02 -06005799 u16 ref_nr,
stephen hemminger1d143d92013-12-29 14:01:29 -08005800 struct list_head *dev_list)
Veaceslav Falico5d261912013-08-28 23:25:05 +02005801{
5802 struct netdev_adjacent *adj;
5803
David Ahern67b62f92016-10-17 19:15:53 -07005804 pr_debug("Remove adjacency: dev %s adj_dev %s ref_nr %d\n",
5805 dev->name, adj_dev->name, ref_nr);
5806
Michal Kubeček6ea29da2015-09-24 10:59:05 +02005807 adj = __netdev_find_adj(adj_dev, dev_list);
Veaceslav Falico5d261912013-08-28 23:25:05 +02005808
Veaceslav Falico2f268f12013-09-25 09:20:07 +02005809 if (!adj) {
David Ahern67b62f92016-10-17 19:15:53 -07005810 pr_err("Adjacency does not exist for device %s from %s\n",
Veaceslav Falico2f268f12013-09-25 09:20:07 +02005811 dev->name, adj_dev->name);
David Ahern67b62f92016-10-17 19:15:53 -07005812 WARN_ON(1);
5813 return;
Veaceslav Falico2f268f12013-09-25 09:20:07 +02005814 }
Veaceslav Falico5d261912013-08-28 23:25:05 +02005815
Andrew Collins93409032016-10-03 13:43:02 -06005816 if (adj->ref_nr > ref_nr) {
David Ahern67b62f92016-10-17 19:15:53 -07005817 pr_debug("adjacency: %s to %s ref_nr - %d = %d\n",
5818 dev->name, adj_dev->name, ref_nr,
5819 adj->ref_nr - ref_nr);
Andrew Collins93409032016-10-03 13:43:02 -06005820 adj->ref_nr -= ref_nr;
Veaceslav Falico5d261912013-08-28 23:25:05 +02005821 return;
5822 }
5823
Veaceslav Falico842d67a2013-09-25 09:20:31 +02005824 if (adj->master)
5825 sysfs_remove_link(&(dev->dev.kobj), "master");
5826
Alexander Y. Fomichev7ce64c72014-09-15 14:22:35 +04005827 if (netdev_adjacent_is_neigh_list(dev, adj_dev, dev_list))
Veaceslav Falico3ee32702014-01-14 21:58:50 +01005828 netdev_adjacent_sysfs_del(dev, adj_dev->name, dev_list);
Veaceslav Falico5831d662013-09-25 09:20:32 +02005829
Veaceslav Falico5d261912013-08-28 23:25:05 +02005830 list_del_rcu(&adj->list);
David Ahern67b62f92016-10-17 19:15:53 -07005831 pr_debug("adjacency: dev_put for %s, because link removed from %s to %s\n",
Veaceslav Falico2f268f12013-09-25 09:20:07 +02005832 adj_dev->name, dev->name, adj_dev->name);
Veaceslav Falico5d261912013-08-28 23:25:05 +02005833 dev_put(adj_dev);
5834 kfree_rcu(adj, rcu);
5835}
5836
stephen hemminger1d143d92013-12-29 14:01:29 -08005837static int __netdev_adjacent_dev_link_lists(struct net_device *dev,
5838 struct net_device *upper_dev,
5839 struct list_head *up_list,
5840 struct list_head *down_list,
5841 void *private, bool master)
Veaceslav Falico5d261912013-08-28 23:25:05 +02005842{
5843 int ret;
5844
David Ahern790510d2016-10-17 19:15:43 -07005845 ret = __netdev_adjacent_dev_insert(dev, upper_dev, up_list,
Andrew Collins93409032016-10-03 13:43:02 -06005846 private, master);
Veaceslav Falico5d261912013-08-28 23:25:05 +02005847 if (ret)
5848 return ret;
5849
David Ahern790510d2016-10-17 19:15:43 -07005850 ret = __netdev_adjacent_dev_insert(upper_dev, dev, down_list,
Andrew Collins93409032016-10-03 13:43:02 -06005851 private, false);
Veaceslav Falico5d261912013-08-28 23:25:05 +02005852 if (ret) {
David Ahern790510d2016-10-17 19:15:43 -07005853 __netdev_adjacent_dev_remove(dev, upper_dev, 1, up_list);
Veaceslav Falico5d261912013-08-28 23:25:05 +02005854 return ret;
5855 }
5856
5857 return 0;
5858}
5859
stephen hemminger1d143d92013-12-29 14:01:29 -08005860static void __netdev_adjacent_dev_unlink_lists(struct net_device *dev,
5861 struct net_device *upper_dev,
Andrew Collins93409032016-10-03 13:43:02 -06005862 u16 ref_nr,
stephen hemminger1d143d92013-12-29 14:01:29 -08005863 struct list_head *up_list,
5864 struct list_head *down_list)
Veaceslav Falico5d261912013-08-28 23:25:05 +02005865{
Andrew Collins93409032016-10-03 13:43:02 -06005866 __netdev_adjacent_dev_remove(dev, upper_dev, ref_nr, up_list);
5867 __netdev_adjacent_dev_remove(upper_dev, dev, ref_nr, down_list);
Veaceslav Falico5d261912013-08-28 23:25:05 +02005868}
5869
stephen hemminger1d143d92013-12-29 14:01:29 -08005870static int __netdev_adjacent_dev_link_neighbour(struct net_device *dev,
5871 struct net_device *upper_dev,
5872 void *private, bool master)
Veaceslav Falico2f268f12013-09-25 09:20:07 +02005873{
David Ahernf1170fd2016-10-17 19:15:51 -07005874 return __netdev_adjacent_dev_link_lists(dev, upper_dev,
5875 &dev->adj_list.upper,
5876 &upper_dev->adj_list.lower,
5877 private, master);
Veaceslav Falico2f268f12013-09-25 09:20:07 +02005878}
5879
stephen hemminger1d143d92013-12-29 14:01:29 -08005880static void __netdev_adjacent_dev_unlink_neighbour(struct net_device *dev,
5881 struct net_device *upper_dev)
Veaceslav Falico2f268f12013-09-25 09:20:07 +02005882{
Andrew Collins93409032016-10-03 13:43:02 -06005883 __netdev_adjacent_dev_unlink_lists(dev, upper_dev, 1,
Veaceslav Falico2f268f12013-09-25 09:20:07 +02005884 &dev->adj_list.upper,
5885 &upper_dev->adj_list.lower);
5886}
Veaceslav Falico5d261912013-08-28 23:25:05 +02005887
Jiri Pirko9ff162a2013-01-03 22:48:49 +00005888static int __netdev_upper_dev_link(struct net_device *dev,
Veaceslav Falico402dae92013-09-25 09:20:09 +02005889 struct net_device *upper_dev, bool master,
Jiri Pirko29bf24a2015-12-03 12:12:11 +01005890 void *upper_priv, void *upper_info)
Jiri Pirko9ff162a2013-01-03 22:48:49 +00005891{
Jiri Pirko0e4ead92015-08-27 09:31:18 +02005892 struct netdev_notifier_changeupper_info changeupper_info;
Veaceslav Falico5d261912013-08-28 23:25:05 +02005893 int ret = 0;
Jiri Pirko9ff162a2013-01-03 22:48:49 +00005894
5895 ASSERT_RTNL();
5896
5897 if (dev == upper_dev)
5898 return -EBUSY;
5899
5900 /* To prevent loops, check if dev is not upper device to upper_dev. */
David Ahernf1170fd2016-10-17 19:15:51 -07005901 if (netdev_has_upper_dev(upper_dev, dev))
Jiri Pirko9ff162a2013-01-03 22:48:49 +00005902 return -EBUSY;
5903
David Ahernf1170fd2016-10-17 19:15:51 -07005904 if (netdev_has_upper_dev(dev, upper_dev))
Jiri Pirko9ff162a2013-01-03 22:48:49 +00005905 return -EEXIST;
5906
5907 if (master && netdev_master_upper_dev_get(dev))
5908 return -EBUSY;
5909
Jiri Pirko0e4ead92015-08-27 09:31:18 +02005910 changeupper_info.upper_dev = upper_dev;
5911 changeupper_info.master = master;
5912 changeupper_info.linking = true;
Jiri Pirko29bf24a2015-12-03 12:12:11 +01005913 changeupper_info.upper_info = upper_info;
Jiri Pirko0e4ead92015-08-27 09:31:18 +02005914
Jiri Pirko573c7ba2015-10-16 14:01:22 +02005915 ret = call_netdevice_notifiers_info(NETDEV_PRECHANGEUPPER, dev,
5916 &changeupper_info.info);
5917 ret = notifier_to_errno(ret);
5918 if (ret)
5919 return ret;
5920
Jiri Pirko6dffb042015-12-03 12:12:10 +01005921 ret = __netdev_adjacent_dev_link_neighbour(dev, upper_dev, upper_priv,
Veaceslav Falico402dae92013-09-25 09:20:09 +02005922 master);
Veaceslav Falico5d261912013-08-28 23:25:05 +02005923 if (ret)
5924 return ret;
Jiri Pirko9ff162a2013-01-03 22:48:49 +00005925
Ido Schimmelb03804e2015-12-03 12:12:03 +01005926 ret = call_netdevice_notifiers_info(NETDEV_CHANGEUPPER, dev,
5927 &changeupper_info.info);
5928 ret = notifier_to_errno(ret);
5929 if (ret)
David Ahernf1170fd2016-10-17 19:15:51 -07005930 goto rollback;
Ido Schimmelb03804e2015-12-03 12:12:03 +01005931
Jiri Pirko9ff162a2013-01-03 22:48:49 +00005932 return 0;
Veaceslav Falico5d261912013-08-28 23:25:05 +02005933
David Ahernf1170fd2016-10-17 19:15:51 -07005934rollback:
Veaceslav Falico2f268f12013-09-25 09:20:07 +02005935 __netdev_adjacent_dev_unlink_neighbour(dev, upper_dev);
Veaceslav Falico5d261912013-08-28 23:25:05 +02005936
5937 return ret;
Jiri Pirko9ff162a2013-01-03 22:48:49 +00005938}
5939
5940/**
5941 * netdev_upper_dev_link - Add a link to the upper device
5942 * @dev: device
5943 * @upper_dev: new upper device
5944 *
5945 * Adds a link to device which is upper to this one. The caller must hold
5946 * the RTNL lock. On a failure a negative errno code is returned.
5947 * On success the reference counts are adjusted and the function
5948 * returns zero.
5949 */
5950int netdev_upper_dev_link(struct net_device *dev,
5951 struct net_device *upper_dev)
5952{
Jiri Pirko29bf24a2015-12-03 12:12:11 +01005953 return __netdev_upper_dev_link(dev, upper_dev, false, NULL, NULL);
Jiri Pirko9ff162a2013-01-03 22:48:49 +00005954}
5955EXPORT_SYMBOL(netdev_upper_dev_link);
5956
5957/**
5958 * netdev_master_upper_dev_link - Add a master link to the upper device
5959 * @dev: device
5960 * @upper_dev: new upper device
Jiri Pirko6dffb042015-12-03 12:12:10 +01005961 * @upper_priv: upper device private
Jiri Pirko29bf24a2015-12-03 12:12:11 +01005962 * @upper_info: upper info to be passed down via notifier
Jiri Pirko9ff162a2013-01-03 22:48:49 +00005963 *
5964 * Adds a link to device which is upper to this one. In this case, only
5965 * one master upper device can be linked, although other non-master devices
5966 * might be linked as well. The caller must hold the RTNL lock.
5967 * On a failure a negative errno code is returned. On success the reference
5968 * counts are adjusted and the function returns zero.
5969 */
5970int netdev_master_upper_dev_link(struct net_device *dev,
Jiri Pirko6dffb042015-12-03 12:12:10 +01005971 struct net_device *upper_dev,
Jiri Pirko29bf24a2015-12-03 12:12:11 +01005972 void *upper_priv, void *upper_info)
Jiri Pirko9ff162a2013-01-03 22:48:49 +00005973{
Jiri Pirko29bf24a2015-12-03 12:12:11 +01005974 return __netdev_upper_dev_link(dev, upper_dev, true,
5975 upper_priv, upper_info);
Jiri Pirko9ff162a2013-01-03 22:48:49 +00005976}
5977EXPORT_SYMBOL(netdev_master_upper_dev_link);
5978
5979/**
5980 * netdev_upper_dev_unlink - Removes a link to upper device
5981 * @dev: device
5982 * @upper_dev: new upper device
5983 *
5984 * Removes a link to device which is upper to this one. The caller must hold
5985 * the RTNL lock.
5986 */
5987void netdev_upper_dev_unlink(struct net_device *dev,
5988 struct net_device *upper_dev)
5989{
Jiri Pirko0e4ead92015-08-27 09:31:18 +02005990 struct netdev_notifier_changeupper_info changeupper_info;
Jiri Pirko9ff162a2013-01-03 22:48:49 +00005991 ASSERT_RTNL();
5992
Jiri Pirko0e4ead92015-08-27 09:31:18 +02005993 changeupper_info.upper_dev = upper_dev;
5994 changeupper_info.master = netdev_master_upper_dev_get(dev) == upper_dev;
5995 changeupper_info.linking = false;
5996
Jiri Pirko573c7ba2015-10-16 14:01:22 +02005997 call_netdevice_notifiers_info(NETDEV_PRECHANGEUPPER, dev,
5998 &changeupper_info.info);
5999
Veaceslav Falico2f268f12013-09-25 09:20:07 +02006000 __netdev_adjacent_dev_unlink_neighbour(dev, upper_dev);
Veaceslav Falico5d261912013-08-28 23:25:05 +02006001
Jiri Pirko0e4ead92015-08-27 09:31:18 +02006002 call_netdevice_notifiers_info(NETDEV_CHANGEUPPER, dev,
6003 &changeupper_info.info);
Jiri Pirko9ff162a2013-01-03 22:48:49 +00006004}
6005EXPORT_SYMBOL(netdev_upper_dev_unlink);
6006
Moni Shoua61bd3852015-02-03 16:48:29 +02006007/**
6008 * netdev_bonding_info_change - Dispatch event about slave change
6009 * @dev: device
Masanari Iida4a26e4532015-02-14 22:26:34 +09006010 * @bonding_info: info to dispatch
Moni Shoua61bd3852015-02-03 16:48:29 +02006011 *
6012 * Send NETDEV_BONDING_INFO to netdev notifiers with info.
6013 * The caller must hold the RTNL lock.
6014 */
6015void netdev_bonding_info_change(struct net_device *dev,
6016 struct netdev_bonding_info *bonding_info)
6017{
6018 struct netdev_notifier_bonding_info info;
6019
6020 memcpy(&info.bonding_info, bonding_info,
6021 sizeof(struct netdev_bonding_info));
6022 call_netdevice_notifiers_info(NETDEV_BONDING_INFO, dev,
6023 &info.info);
6024}
6025EXPORT_SYMBOL(netdev_bonding_info_change);
6026
Eric Dumazet2ce1ee12015-02-04 13:37:44 -08006027static void netdev_adjacent_add_links(struct net_device *dev)
Alexander Y. Fomichev4c754312014-08-25 16:26:45 +04006028{
6029 struct netdev_adjacent *iter;
6030
6031 struct net *net = dev_net(dev);
6032
6033 list_for_each_entry(iter, &dev->adj_list.upper, list) {
Wei Tangbe4da0e2016-06-16 21:30:12 +08006034 if (!net_eq(net, dev_net(iter->dev)))
Alexander Y. Fomichev4c754312014-08-25 16:26:45 +04006035 continue;
6036 netdev_adjacent_sysfs_add(iter->dev, dev,
6037 &iter->dev->adj_list.lower);
6038 netdev_adjacent_sysfs_add(dev, iter->dev,
6039 &dev->adj_list.upper);
6040 }
6041
6042 list_for_each_entry(iter, &dev->adj_list.lower, list) {
Wei Tangbe4da0e2016-06-16 21:30:12 +08006043 if (!net_eq(net, dev_net(iter->dev)))
Alexander Y. Fomichev4c754312014-08-25 16:26:45 +04006044 continue;
6045 netdev_adjacent_sysfs_add(iter->dev, dev,
6046 &iter->dev->adj_list.upper);
6047 netdev_adjacent_sysfs_add(dev, iter->dev,
6048 &dev->adj_list.lower);
6049 }
6050}
6051
Eric Dumazet2ce1ee12015-02-04 13:37:44 -08006052static void netdev_adjacent_del_links(struct net_device *dev)
Alexander Y. Fomichev4c754312014-08-25 16:26:45 +04006053{
6054 struct netdev_adjacent *iter;
6055
6056 struct net *net = dev_net(dev);
6057
6058 list_for_each_entry(iter, &dev->adj_list.upper, list) {
Wei Tangbe4da0e2016-06-16 21:30:12 +08006059 if (!net_eq(net, dev_net(iter->dev)))
Alexander Y. Fomichev4c754312014-08-25 16:26:45 +04006060 continue;
6061 netdev_adjacent_sysfs_del(iter->dev, dev->name,
6062 &iter->dev->adj_list.lower);
6063 netdev_adjacent_sysfs_del(dev, iter->dev->name,
6064 &dev->adj_list.upper);
6065 }
6066
6067 list_for_each_entry(iter, &dev->adj_list.lower, list) {
Wei Tangbe4da0e2016-06-16 21:30:12 +08006068 if (!net_eq(net, dev_net(iter->dev)))
Alexander Y. Fomichev4c754312014-08-25 16:26:45 +04006069 continue;
6070 netdev_adjacent_sysfs_del(iter->dev, dev->name,
6071 &iter->dev->adj_list.upper);
6072 netdev_adjacent_sysfs_del(dev, iter->dev->name,
6073 &dev->adj_list.lower);
6074 }
6075}
6076
Veaceslav Falico5bb025f2014-01-14 21:58:51 +01006077void netdev_adjacent_rename_links(struct net_device *dev, char *oldname)
Veaceslav Falico402dae92013-09-25 09:20:09 +02006078{
Veaceslav Falico5bb025f2014-01-14 21:58:51 +01006079 struct netdev_adjacent *iter;
Veaceslav Falico402dae92013-09-25 09:20:09 +02006080
Alexander Y. Fomichev4c754312014-08-25 16:26:45 +04006081 struct net *net = dev_net(dev);
6082
Veaceslav Falico5bb025f2014-01-14 21:58:51 +01006083 list_for_each_entry(iter, &dev->adj_list.upper, list) {
Wei Tangbe4da0e2016-06-16 21:30:12 +08006084 if (!net_eq(net, dev_net(iter->dev)))
Alexander Y. Fomichev4c754312014-08-25 16:26:45 +04006085 continue;
Veaceslav Falico5bb025f2014-01-14 21:58:51 +01006086 netdev_adjacent_sysfs_del(iter->dev, oldname,
6087 &iter->dev->adj_list.lower);
6088 netdev_adjacent_sysfs_add(iter->dev, dev,
6089 &iter->dev->adj_list.lower);
6090 }
Veaceslav Falico402dae92013-09-25 09:20:09 +02006091
Veaceslav Falico5bb025f2014-01-14 21:58:51 +01006092 list_for_each_entry(iter, &dev->adj_list.lower, list) {
Wei Tangbe4da0e2016-06-16 21:30:12 +08006093 if (!net_eq(net, dev_net(iter->dev)))
Alexander Y. Fomichev4c754312014-08-25 16:26:45 +04006094 continue;
Veaceslav Falico5bb025f2014-01-14 21:58:51 +01006095 netdev_adjacent_sysfs_del(iter->dev, oldname,
6096 &iter->dev->adj_list.upper);
6097 netdev_adjacent_sysfs_add(iter->dev, dev,
6098 &iter->dev->adj_list.upper);
6099 }
Veaceslav Falico402dae92013-09-25 09:20:09 +02006100}
Veaceslav Falico402dae92013-09-25 09:20:09 +02006101
6102void *netdev_lower_dev_get_private(struct net_device *dev,
6103 struct net_device *lower_dev)
6104{
6105 struct netdev_adjacent *lower;
6106
6107 if (!lower_dev)
6108 return NULL;
Michal Kubeček6ea29da2015-09-24 10:59:05 +02006109 lower = __netdev_find_adj(lower_dev, &dev->adj_list.lower);
Veaceslav Falico402dae92013-09-25 09:20:09 +02006110 if (!lower)
6111 return NULL;
6112
6113 return lower->private;
6114}
6115EXPORT_SYMBOL(netdev_lower_dev_get_private);
6116
Vlad Yasevich4085ebe2014-05-16 17:04:53 -04006117
Sabrina Dubroca952fcfd2016-08-12 16:10:33 +02006118int dev_get_nest_level(struct net_device *dev)
Vlad Yasevich4085ebe2014-05-16 17:04:53 -04006119{
6120 struct net_device *lower = NULL;
6121 struct list_head *iter;
6122 int max_nest = -1;
6123 int nest;
6124
6125 ASSERT_RTNL();
6126
6127 netdev_for_each_lower_dev(dev, lower, iter) {
Sabrina Dubroca952fcfd2016-08-12 16:10:33 +02006128 nest = dev_get_nest_level(lower);
Vlad Yasevich4085ebe2014-05-16 17:04:53 -04006129 if (max_nest < nest)
6130 max_nest = nest;
6131 }
6132
Sabrina Dubroca952fcfd2016-08-12 16:10:33 +02006133 return max_nest + 1;
Vlad Yasevich4085ebe2014-05-16 17:04:53 -04006134}
6135EXPORT_SYMBOL(dev_get_nest_level);
6136
Jiri Pirko04d48262015-12-03 12:12:15 +01006137/**
6138 * netdev_lower_change - Dispatch event about lower device state change
6139 * @lower_dev: device
6140 * @lower_state_info: state to dispatch
6141 *
6142 * Send NETDEV_CHANGELOWERSTATE to netdev notifiers with info.
6143 * The caller must hold the RTNL lock.
6144 */
6145void netdev_lower_state_changed(struct net_device *lower_dev,
6146 void *lower_state_info)
6147{
6148 struct netdev_notifier_changelowerstate_info changelowerstate_info;
6149
6150 ASSERT_RTNL();
6151 changelowerstate_info.lower_state_info = lower_state_info;
6152 call_netdevice_notifiers_info(NETDEV_CHANGELOWERSTATE, lower_dev,
6153 &changelowerstate_info.info);
6154}
6155EXPORT_SYMBOL(netdev_lower_state_changed);
6156
Jiri Pirko18bfb922016-07-05 11:27:38 +02006157int netdev_default_l2upper_neigh_construct(struct net_device *dev,
6158 struct neighbour *n)
6159{
6160 struct net_device *lower_dev, *stop_dev;
6161 struct list_head *iter;
6162 int err;
6163
6164 netdev_for_each_lower_dev(dev, lower_dev, iter) {
6165 if (!lower_dev->netdev_ops->ndo_neigh_construct)
6166 continue;
6167 err = lower_dev->netdev_ops->ndo_neigh_construct(lower_dev, n);
6168 if (err) {
6169 stop_dev = lower_dev;
6170 goto rollback;
6171 }
6172 }
6173 return 0;
6174
6175rollback:
6176 netdev_for_each_lower_dev(dev, lower_dev, iter) {
6177 if (lower_dev == stop_dev)
6178 break;
6179 if (!lower_dev->netdev_ops->ndo_neigh_destroy)
6180 continue;
6181 lower_dev->netdev_ops->ndo_neigh_destroy(lower_dev, n);
6182 }
6183 return err;
6184}
6185EXPORT_SYMBOL_GPL(netdev_default_l2upper_neigh_construct);
6186
6187void netdev_default_l2upper_neigh_destroy(struct net_device *dev,
6188 struct neighbour *n)
6189{
6190 struct net_device *lower_dev;
6191 struct list_head *iter;
6192
6193 netdev_for_each_lower_dev(dev, lower_dev, iter) {
6194 if (!lower_dev->netdev_ops->ndo_neigh_destroy)
6195 continue;
6196 lower_dev->netdev_ops->ndo_neigh_destroy(lower_dev, n);
6197 }
6198}
6199EXPORT_SYMBOL_GPL(netdev_default_l2upper_neigh_destroy);
6200
Patrick McHardyb6c40d62008-10-07 15:26:48 -07006201static void dev_change_rx_flags(struct net_device *dev, int flags)
6202{
Stephen Hemmingerd3147742008-11-19 21:32:24 -08006203 const struct net_device_ops *ops = dev->netdev_ops;
6204
Vlad Yasevichd2615bf2013-11-19 20:47:15 -05006205 if (ops->ndo_change_rx_flags)
Stephen Hemmingerd3147742008-11-19 21:32:24 -08006206 ops->ndo_change_rx_flags(dev, flags);
Patrick McHardyb6c40d62008-10-07 15:26:48 -07006207}
6208
Nicolas Dichtel991fb3f2013-09-25 12:02:45 +02006209static int __dev_set_promiscuity(struct net_device *dev, int inc, bool notify)
Patrick McHardy4417da62007-06-27 01:28:10 -07006210{
Eric Dumazetb536db92011-11-30 21:42:26 +00006211 unsigned int old_flags = dev->flags;
Eric W. Biedermand04a48b2012-05-23 17:01:57 -06006212 kuid_t uid;
6213 kgid_t gid;
Patrick McHardy4417da62007-06-27 01:28:10 -07006214
Patrick McHardy24023452007-07-14 18:51:31 -07006215 ASSERT_RTNL();
6216
Wang Chendad9b332008-06-18 01:48:28 -07006217 dev->flags |= IFF_PROMISC;
6218 dev->promiscuity += inc;
6219 if (dev->promiscuity == 0) {
6220 /*
6221 * Avoid overflow.
6222 * If inc causes overflow, untouch promisc and return error.
6223 */
6224 if (inc < 0)
6225 dev->flags &= ~IFF_PROMISC;
6226 else {
6227 dev->promiscuity -= inc;
Joe Perches7b6cd1c2012-02-01 10:54:43 +00006228 pr_warn("%s: promiscuity touches roof, set promiscuity failed. promiscuity feature of device might be broken.\n",
6229 dev->name);
Wang Chendad9b332008-06-18 01:48:28 -07006230 return -EOVERFLOW;
6231 }
6232 }
Patrick McHardy4417da62007-06-27 01:28:10 -07006233 if (dev->flags != old_flags) {
Joe Perches7b6cd1c2012-02-01 10:54:43 +00006234 pr_info("device %s %s promiscuous mode\n",
6235 dev->name,
6236 dev->flags & IFF_PROMISC ? "entered" : "left");
David Howells8192b0c2008-11-14 10:39:10 +11006237 if (audit_enabled) {
6238 current_uid_gid(&uid, &gid);
Klaus Heinrich Kiwi7759db82008-01-23 22:57:45 -05006239 audit_log(current->audit_context, GFP_ATOMIC,
6240 AUDIT_ANOM_PROMISCUOUS,
6241 "dev=%s prom=%d old_prom=%d auid=%u uid=%u gid=%u ses=%u",
6242 dev->name, (dev->flags & IFF_PROMISC),
6243 (old_flags & IFF_PROMISC),
Eric W. Biedermane1760bd2012-09-10 22:39:43 -07006244 from_kuid(&init_user_ns, audit_get_loginuid(current)),
Eric W. Biedermand04a48b2012-05-23 17:01:57 -06006245 from_kuid(&init_user_ns, uid),
6246 from_kgid(&init_user_ns, gid),
Klaus Heinrich Kiwi7759db82008-01-23 22:57:45 -05006247 audit_get_sessionid(current));
David Howells8192b0c2008-11-14 10:39:10 +11006248 }
Patrick McHardy24023452007-07-14 18:51:31 -07006249
Patrick McHardyb6c40d62008-10-07 15:26:48 -07006250 dev_change_rx_flags(dev, IFF_PROMISC);
Patrick McHardy4417da62007-06-27 01:28:10 -07006251 }
Nicolas Dichtel991fb3f2013-09-25 12:02:45 +02006252 if (notify)
6253 __dev_notify_flags(dev, old_flags, IFF_PROMISC);
Wang Chendad9b332008-06-18 01:48:28 -07006254 return 0;
Patrick McHardy4417da62007-06-27 01:28:10 -07006255}
6256
Linus Torvalds1da177e2005-04-16 15:20:36 -07006257/**
6258 * dev_set_promiscuity - update promiscuity count on a device
6259 * @dev: device
6260 * @inc: modifier
6261 *
Stephen Hemminger3041a062006-05-26 13:25:24 -07006262 * Add or remove promiscuity from a device. While the count in the device
Linus Torvalds1da177e2005-04-16 15:20:36 -07006263 * remains above zero the interface remains promiscuous. Once it hits zero
6264 * the device reverts back to normal filtering operation. A negative inc
6265 * value is used to drop promiscuity on the device.
Wang Chendad9b332008-06-18 01:48:28 -07006266 * Return 0 if successful or a negative errno code on error.
Linus Torvalds1da177e2005-04-16 15:20:36 -07006267 */
Wang Chendad9b332008-06-18 01:48:28 -07006268int dev_set_promiscuity(struct net_device *dev, int inc)
Linus Torvalds1da177e2005-04-16 15:20:36 -07006269{
Eric Dumazetb536db92011-11-30 21:42:26 +00006270 unsigned int old_flags = dev->flags;
Wang Chendad9b332008-06-18 01:48:28 -07006271 int err;
Linus Torvalds1da177e2005-04-16 15:20:36 -07006272
Nicolas Dichtel991fb3f2013-09-25 12:02:45 +02006273 err = __dev_set_promiscuity(dev, inc, true);
Patrick McHardy4b5a6982008-07-06 15:49:08 -07006274 if (err < 0)
Wang Chendad9b332008-06-18 01:48:28 -07006275 return err;
Patrick McHardy4417da62007-06-27 01:28:10 -07006276 if (dev->flags != old_flags)
6277 dev_set_rx_mode(dev);
Wang Chendad9b332008-06-18 01:48:28 -07006278 return err;
Linus Torvalds1da177e2005-04-16 15:20:36 -07006279}
Eric Dumazetd1b19df2009-09-03 01:29:39 -07006280EXPORT_SYMBOL(dev_set_promiscuity);
Linus Torvalds1da177e2005-04-16 15:20:36 -07006281
Nicolas Dichtel991fb3f2013-09-25 12:02:45 +02006282static int __dev_set_allmulti(struct net_device *dev, int inc, bool notify)
Linus Torvalds1da177e2005-04-16 15:20:36 -07006283{
Nicolas Dichtel991fb3f2013-09-25 12:02:45 +02006284 unsigned int old_flags = dev->flags, old_gflags = dev->gflags;
Linus Torvalds1da177e2005-04-16 15:20:36 -07006285
Patrick McHardy24023452007-07-14 18:51:31 -07006286 ASSERT_RTNL();
6287
Linus Torvalds1da177e2005-04-16 15:20:36 -07006288 dev->flags |= IFF_ALLMULTI;
Wang Chendad9b332008-06-18 01:48:28 -07006289 dev->allmulti += inc;
6290 if (dev->allmulti == 0) {
6291 /*
6292 * Avoid overflow.
6293 * If inc causes overflow, untouch allmulti and return error.
6294 */
6295 if (inc < 0)
6296 dev->flags &= ~IFF_ALLMULTI;
6297 else {
6298 dev->allmulti -= inc;
Joe Perches7b6cd1c2012-02-01 10:54:43 +00006299 pr_warn("%s: allmulti touches roof, set allmulti failed. allmulti feature of device might be broken.\n",
6300 dev->name);
Wang Chendad9b332008-06-18 01:48:28 -07006301 return -EOVERFLOW;
6302 }
6303 }
Patrick McHardy24023452007-07-14 18:51:31 -07006304 if (dev->flags ^ old_flags) {
Patrick McHardyb6c40d62008-10-07 15:26:48 -07006305 dev_change_rx_flags(dev, IFF_ALLMULTI);
Patrick McHardy4417da62007-06-27 01:28:10 -07006306 dev_set_rx_mode(dev);
Nicolas Dichtel991fb3f2013-09-25 12:02:45 +02006307 if (notify)
6308 __dev_notify_flags(dev, old_flags,
6309 dev->gflags ^ old_gflags);
Patrick McHardy24023452007-07-14 18:51:31 -07006310 }
Wang Chendad9b332008-06-18 01:48:28 -07006311 return 0;
Patrick McHardy4417da62007-06-27 01:28:10 -07006312}
Nicolas Dichtel991fb3f2013-09-25 12:02:45 +02006313
6314/**
6315 * dev_set_allmulti - update allmulti count on a device
6316 * @dev: device
6317 * @inc: modifier
6318 *
6319 * Add or remove reception of all multicast frames to a device. While the
6320 * count in the device remains above zero the interface remains listening
6321 * to all interfaces. Once it hits zero the device reverts back to normal
6322 * filtering operation. A negative @inc value is used to drop the counter
6323 * when releasing a resource needing all multicasts.
6324 * Return 0 if successful or a negative errno code on error.
6325 */
6326
6327int dev_set_allmulti(struct net_device *dev, int inc)
6328{
6329 return __dev_set_allmulti(dev, inc, true);
6330}
Eric Dumazetd1b19df2009-09-03 01:29:39 -07006331EXPORT_SYMBOL(dev_set_allmulti);
Patrick McHardy4417da62007-06-27 01:28:10 -07006332
6333/*
6334 * Upload unicast and multicast address lists to device and
6335 * configure RX filtering. When the device doesn't support unicast
Joe Perches53ccaae2007-12-20 14:02:06 -08006336 * filtering it is put in promiscuous mode while unicast addresses
Patrick McHardy4417da62007-06-27 01:28:10 -07006337 * are present.
6338 */
6339void __dev_set_rx_mode(struct net_device *dev)
6340{
Stephen Hemmingerd3147742008-11-19 21:32:24 -08006341 const struct net_device_ops *ops = dev->netdev_ops;
6342
Patrick McHardy4417da62007-06-27 01:28:10 -07006343 /* dev_open will call this function so the list will stay sane. */
6344 if (!(dev->flags&IFF_UP))
6345 return;
6346
6347 if (!netif_device_present(dev))
YOSHIFUJI Hideaki40b77c92007-07-19 10:43:23 +09006348 return;
Patrick McHardy4417da62007-06-27 01:28:10 -07006349
Jiri Pirko01789342011-08-16 06:29:00 +00006350 if (!(dev->priv_flags & IFF_UNICAST_FLT)) {
Patrick McHardy4417da62007-06-27 01:28:10 -07006351 /* Unicast addresses changes may only happen under the rtnl,
6352 * therefore calling __dev_set_promiscuity here is safe.
6353 */
Jiri Pirko32e7bfc2010-01-25 13:36:10 -08006354 if (!netdev_uc_empty(dev) && !dev->uc_promisc) {
Nicolas Dichtel991fb3f2013-09-25 12:02:45 +02006355 __dev_set_promiscuity(dev, 1, false);
Joe Perches2d348d12011-07-25 16:17:35 -07006356 dev->uc_promisc = true;
Jiri Pirko32e7bfc2010-01-25 13:36:10 -08006357 } else if (netdev_uc_empty(dev) && dev->uc_promisc) {
Nicolas Dichtel991fb3f2013-09-25 12:02:45 +02006358 __dev_set_promiscuity(dev, -1, false);
Joe Perches2d348d12011-07-25 16:17:35 -07006359 dev->uc_promisc = false;
Patrick McHardy4417da62007-06-27 01:28:10 -07006360 }
Patrick McHardy4417da62007-06-27 01:28:10 -07006361 }
Jiri Pirko01789342011-08-16 06:29:00 +00006362
6363 if (ops->ndo_set_rx_mode)
6364 ops->ndo_set_rx_mode(dev);
Patrick McHardy4417da62007-06-27 01:28:10 -07006365}
6366
6367void dev_set_rx_mode(struct net_device *dev)
6368{
David S. Millerb9e40852008-07-15 00:15:08 -07006369 netif_addr_lock_bh(dev);
Patrick McHardy4417da62007-06-27 01:28:10 -07006370 __dev_set_rx_mode(dev);
David S. Millerb9e40852008-07-15 00:15:08 -07006371 netif_addr_unlock_bh(dev);
Linus Torvalds1da177e2005-04-16 15:20:36 -07006372}
6373
Stephen Hemmingerf0db2752008-09-30 02:23:58 -07006374/**
6375 * dev_get_flags - get flags reported to userspace
6376 * @dev: device
6377 *
6378 * Get the combination of flag bits exported through APIs to userspace.
6379 */
Eric Dumazet95c96172012-04-15 05:58:06 +00006380unsigned int dev_get_flags(const struct net_device *dev)
Linus Torvalds1da177e2005-04-16 15:20:36 -07006381{
Eric Dumazet95c96172012-04-15 05:58:06 +00006382 unsigned int flags;
Linus Torvalds1da177e2005-04-16 15:20:36 -07006383
6384 flags = (dev->flags & ~(IFF_PROMISC |
6385 IFF_ALLMULTI |
Stefan Rompfb00055a2006-03-20 17:09:11 -08006386 IFF_RUNNING |
6387 IFF_LOWER_UP |
6388 IFF_DORMANT)) |
Linus Torvalds1da177e2005-04-16 15:20:36 -07006389 (dev->gflags & (IFF_PROMISC |
6390 IFF_ALLMULTI));
6391
Stefan Rompfb00055a2006-03-20 17:09:11 -08006392 if (netif_running(dev)) {
6393 if (netif_oper_up(dev))
6394 flags |= IFF_RUNNING;
6395 if (netif_carrier_ok(dev))
6396 flags |= IFF_LOWER_UP;
6397 if (netif_dormant(dev))
6398 flags |= IFF_DORMANT;
6399 }
Linus Torvalds1da177e2005-04-16 15:20:36 -07006400
6401 return flags;
6402}
Eric Dumazetd1b19df2009-09-03 01:29:39 -07006403EXPORT_SYMBOL(dev_get_flags);
Linus Torvalds1da177e2005-04-16 15:20:36 -07006404
Patrick McHardybd380812010-02-26 06:34:53 +00006405int __dev_change_flags(struct net_device *dev, unsigned int flags)
Linus Torvalds1da177e2005-04-16 15:20:36 -07006406{
Eric Dumazetb536db92011-11-30 21:42:26 +00006407 unsigned int old_flags = dev->flags;
Patrick McHardybd380812010-02-26 06:34:53 +00006408 int ret;
Linus Torvalds1da177e2005-04-16 15:20:36 -07006409
Patrick McHardy24023452007-07-14 18:51:31 -07006410 ASSERT_RTNL();
6411
Linus Torvalds1da177e2005-04-16 15:20:36 -07006412 /*
6413 * Set the flags on our device.
6414 */
6415
6416 dev->flags = (flags & (IFF_DEBUG | IFF_NOTRAILERS | IFF_NOARP |
6417 IFF_DYNAMIC | IFF_MULTICAST | IFF_PORTSEL |
6418 IFF_AUTOMEDIA)) |
6419 (dev->flags & (IFF_UP | IFF_VOLATILE | IFF_PROMISC |
6420 IFF_ALLMULTI));
6421
6422 /*
6423 * Load in the correct multicast list now the flags have changed.
6424 */
6425
Patrick McHardyb6c40d62008-10-07 15:26:48 -07006426 if ((old_flags ^ flags) & IFF_MULTICAST)
6427 dev_change_rx_flags(dev, IFF_MULTICAST);
Patrick McHardy24023452007-07-14 18:51:31 -07006428
Patrick McHardy4417da62007-06-27 01:28:10 -07006429 dev_set_rx_mode(dev);
Linus Torvalds1da177e2005-04-16 15:20:36 -07006430
6431 /*
6432 * Have we downed the interface. We handle IFF_UP ourselves
6433 * according to user attempts to set it, rather than blindly
6434 * setting it.
6435 */
6436
6437 ret = 0;
Peter Pan(潘卫平)d215d102014-06-16 21:57:22 +08006438 if ((old_flags ^ flags) & IFF_UP)
Patrick McHardybd380812010-02-26 06:34:53 +00006439 ret = ((old_flags & IFF_UP) ? __dev_close : __dev_open)(dev);
Linus Torvalds1da177e2005-04-16 15:20:36 -07006440
Linus Torvalds1da177e2005-04-16 15:20:36 -07006441 if ((flags ^ dev->gflags) & IFF_PROMISC) {
Eric Dumazetd1b19df2009-09-03 01:29:39 -07006442 int inc = (flags & IFF_PROMISC) ? 1 : -1;
Nicolas Dichtel991fb3f2013-09-25 12:02:45 +02006443 unsigned int old_flags = dev->flags;
Eric Dumazetd1b19df2009-09-03 01:29:39 -07006444
Linus Torvalds1da177e2005-04-16 15:20:36 -07006445 dev->gflags ^= IFF_PROMISC;
Nicolas Dichtel991fb3f2013-09-25 12:02:45 +02006446
6447 if (__dev_set_promiscuity(dev, inc, false) >= 0)
6448 if (dev->flags != old_flags)
6449 dev_set_rx_mode(dev);
Linus Torvalds1da177e2005-04-16 15:20:36 -07006450 }
6451
6452 /* NOTE: order of synchronization of IFF_PROMISC and IFF_ALLMULTI
6453 is important. Some (broken) drivers set IFF_PROMISC, when
6454 IFF_ALLMULTI is requested not asking us and not reporting.
6455 */
6456 if ((flags ^ dev->gflags) & IFF_ALLMULTI) {
Eric Dumazetd1b19df2009-09-03 01:29:39 -07006457 int inc = (flags & IFF_ALLMULTI) ? 1 : -1;
6458
Linus Torvalds1da177e2005-04-16 15:20:36 -07006459 dev->gflags ^= IFF_ALLMULTI;
Nicolas Dichtel991fb3f2013-09-25 12:02:45 +02006460 __dev_set_allmulti(dev, inc, false);
Linus Torvalds1da177e2005-04-16 15:20:36 -07006461 }
6462
Patrick McHardybd380812010-02-26 06:34:53 +00006463 return ret;
6464}
6465
Nicolas Dichtela528c212013-09-25 12:02:44 +02006466void __dev_notify_flags(struct net_device *dev, unsigned int old_flags,
6467 unsigned int gchanges)
Patrick McHardybd380812010-02-26 06:34:53 +00006468{
6469 unsigned int changes = dev->flags ^ old_flags;
6470
Nicolas Dichtela528c212013-09-25 12:02:44 +02006471 if (gchanges)
Alexei Starovoitov7f294052013-10-23 16:02:42 -07006472 rtmsg_ifinfo(RTM_NEWLINK, dev, gchanges, GFP_ATOMIC);
Nicolas Dichtela528c212013-09-25 12:02:44 +02006473
Patrick McHardybd380812010-02-26 06:34:53 +00006474 if (changes & IFF_UP) {
6475 if (dev->flags & IFF_UP)
6476 call_netdevice_notifiers(NETDEV_UP, dev);
6477 else
6478 call_netdevice_notifiers(NETDEV_DOWN, dev);
6479 }
6480
6481 if (dev->flags & IFF_UP &&
Jiri Pirkobe9efd32013-05-28 01:30:22 +00006482 (changes & ~(IFF_UP | IFF_PROMISC | IFF_ALLMULTI | IFF_VOLATILE))) {
6483 struct netdev_notifier_change_info change_info;
6484
6485 change_info.flags_changed = changes;
6486 call_netdevice_notifiers_info(NETDEV_CHANGE, dev,
6487 &change_info.info);
6488 }
Patrick McHardybd380812010-02-26 06:34:53 +00006489}
6490
6491/**
6492 * dev_change_flags - change device settings
6493 * @dev: device
6494 * @flags: device state flags
6495 *
6496 * Change settings on device based state flags. The flags are
6497 * in the userspace exported format.
6498 */
Eric Dumazetb536db92011-11-30 21:42:26 +00006499int dev_change_flags(struct net_device *dev, unsigned int flags)
Patrick McHardybd380812010-02-26 06:34:53 +00006500{
Eric Dumazetb536db92011-11-30 21:42:26 +00006501 int ret;
Nicolas Dichtel991fb3f2013-09-25 12:02:45 +02006502 unsigned int changes, old_flags = dev->flags, old_gflags = dev->gflags;
Patrick McHardybd380812010-02-26 06:34:53 +00006503
6504 ret = __dev_change_flags(dev, flags);
6505 if (ret < 0)
6506 return ret;
6507
Nicolas Dichtel991fb3f2013-09-25 12:02:45 +02006508 changes = (old_flags ^ dev->flags) | (old_gflags ^ dev->gflags);
Nicolas Dichtela528c212013-09-25 12:02:44 +02006509 __dev_notify_flags(dev, old_flags, changes);
Linus Torvalds1da177e2005-04-16 15:20:36 -07006510 return ret;
6511}
Eric Dumazetd1b19df2009-09-03 01:29:39 -07006512EXPORT_SYMBOL(dev_change_flags);
Linus Torvalds1da177e2005-04-16 15:20:36 -07006513
Veaceslav Falico2315dc92014-01-10 16:56:25 +01006514static int __dev_set_mtu(struct net_device *dev, int new_mtu)
6515{
6516 const struct net_device_ops *ops = dev->netdev_ops;
6517
6518 if (ops->ndo_change_mtu)
6519 return ops->ndo_change_mtu(dev, new_mtu);
6520
6521 dev->mtu = new_mtu;
6522 return 0;
6523}
6524
Stephen Hemmingerf0db2752008-09-30 02:23:58 -07006525/**
6526 * dev_set_mtu - Change maximum transfer unit
6527 * @dev: device
6528 * @new_mtu: new transfer unit
6529 *
6530 * Change the maximum transfer size of the network device.
6531 */
Linus Torvalds1da177e2005-04-16 15:20:36 -07006532int dev_set_mtu(struct net_device *dev, int new_mtu)
6533{
Veaceslav Falico2315dc92014-01-10 16:56:25 +01006534 int err, orig_mtu;
Linus Torvalds1da177e2005-04-16 15:20:36 -07006535
6536 if (new_mtu == dev->mtu)
6537 return 0;
6538
Jarod Wilson61e84622016-10-07 22:04:33 -04006539 /* MTU must be positive, and in range */
6540 if (new_mtu < 0 || new_mtu < dev->min_mtu) {
6541 net_err_ratelimited("%s: Invalid MTU %d requested, hw min %d\n",
6542 dev->name, new_mtu, dev->min_mtu);
Linus Torvalds1da177e2005-04-16 15:20:36 -07006543 return -EINVAL;
Jarod Wilson61e84622016-10-07 22:04:33 -04006544 }
6545
6546 if (dev->max_mtu > 0 && new_mtu > dev->max_mtu) {
6547 net_err_ratelimited("%s: Invalid MTU %d requested, hw max %d\n",
Jakub Kicinskia0e65de2016-10-17 18:02:22 +01006548 dev->name, new_mtu, dev->max_mtu);
Jarod Wilson61e84622016-10-07 22:04:33 -04006549 return -EINVAL;
6550 }
Linus Torvalds1da177e2005-04-16 15:20:36 -07006551
6552 if (!netif_device_present(dev))
6553 return -ENODEV;
6554
Veaceslav Falico1d486bf2014-01-16 00:02:18 +01006555 err = call_netdevice_notifiers(NETDEV_PRECHANGEMTU, dev);
6556 err = notifier_to_errno(err);
6557 if (err)
6558 return err;
Stephen Hemmingerd3147742008-11-19 21:32:24 -08006559
Veaceslav Falico2315dc92014-01-10 16:56:25 +01006560 orig_mtu = dev->mtu;
6561 err = __dev_set_mtu(dev, new_mtu);
Linus Torvalds1da177e2005-04-16 15:20:36 -07006562
Veaceslav Falico2315dc92014-01-10 16:56:25 +01006563 if (!err) {
6564 err = call_netdevice_notifiers(NETDEV_CHANGEMTU, dev);
6565 err = notifier_to_errno(err);
6566 if (err) {
6567 /* setting mtu back and notifying everyone again,
6568 * so that they have a chance to revert changes.
6569 */
6570 __dev_set_mtu(dev, orig_mtu);
6571 call_netdevice_notifiers(NETDEV_CHANGEMTU, dev);
6572 }
6573 }
Linus Torvalds1da177e2005-04-16 15:20:36 -07006574 return err;
6575}
Eric Dumazetd1b19df2009-09-03 01:29:39 -07006576EXPORT_SYMBOL(dev_set_mtu);
Linus Torvalds1da177e2005-04-16 15:20:36 -07006577
Stephen Hemmingerf0db2752008-09-30 02:23:58 -07006578/**
Vlad Dogarucbda10f2011-01-13 23:38:30 +00006579 * dev_set_group - Change group this device belongs to
6580 * @dev: device
6581 * @new_group: group this device should belong to
6582 */
6583void dev_set_group(struct net_device *dev, int new_group)
6584{
6585 dev->group = new_group;
6586}
6587EXPORT_SYMBOL(dev_set_group);
6588
6589/**
Stephen Hemmingerf0db2752008-09-30 02:23:58 -07006590 * dev_set_mac_address - Change Media Access Control Address
6591 * @dev: device
6592 * @sa: new address
6593 *
6594 * Change the hardware (MAC) address of the device
6595 */
Linus Torvalds1da177e2005-04-16 15:20:36 -07006596int dev_set_mac_address(struct net_device *dev, struct sockaddr *sa)
6597{
Stephen Hemmingerd3147742008-11-19 21:32:24 -08006598 const struct net_device_ops *ops = dev->netdev_ops;
Linus Torvalds1da177e2005-04-16 15:20:36 -07006599 int err;
6600
Stephen Hemmingerd3147742008-11-19 21:32:24 -08006601 if (!ops->ndo_set_mac_address)
Linus Torvalds1da177e2005-04-16 15:20:36 -07006602 return -EOPNOTSUPP;
6603 if (sa->sa_family != dev->type)
6604 return -EINVAL;
6605 if (!netif_device_present(dev))
6606 return -ENODEV;
Stephen Hemmingerd3147742008-11-19 21:32:24 -08006607 err = ops->ndo_set_mac_address(dev, sa);
Jiri Pirkof6521512013-01-01 03:30:14 +00006608 if (err)
6609 return err;
Jiri Pirkofbdeca22013-01-01 03:30:16 +00006610 dev->addr_assign_type = NET_ADDR_SET;
Jiri Pirkof6521512013-01-01 03:30:14 +00006611 call_netdevice_notifiers(NETDEV_CHANGEADDR, dev);
Theodore Ts'o7bf23572012-07-04 21:23:25 -04006612 add_device_randomness(dev->dev_addr, dev->addr_len);
Jiri Pirkof6521512013-01-01 03:30:14 +00006613 return 0;
Linus Torvalds1da177e2005-04-16 15:20:36 -07006614}
Eric Dumazetd1b19df2009-09-03 01:29:39 -07006615EXPORT_SYMBOL(dev_set_mac_address);
Linus Torvalds1da177e2005-04-16 15:20:36 -07006616
Jiri Pirko4bf84c32012-12-27 23:49:37 +00006617/**
6618 * dev_change_carrier - Change device carrier
6619 * @dev: device
Randy Dunlap691b3b72013-03-04 12:32:43 +00006620 * @new_carrier: new value
Jiri Pirko4bf84c32012-12-27 23:49:37 +00006621 *
6622 * Change device carrier
6623 */
6624int dev_change_carrier(struct net_device *dev, bool new_carrier)
6625{
6626 const struct net_device_ops *ops = dev->netdev_ops;
6627
6628 if (!ops->ndo_change_carrier)
6629 return -EOPNOTSUPP;
6630 if (!netif_device_present(dev))
6631 return -ENODEV;
6632 return ops->ndo_change_carrier(dev, new_carrier);
6633}
6634EXPORT_SYMBOL(dev_change_carrier);
6635
Linus Torvalds1da177e2005-04-16 15:20:36 -07006636/**
Jiri Pirko66b52b02013-07-29 18:16:49 +02006637 * dev_get_phys_port_id - Get device physical port ID
6638 * @dev: device
6639 * @ppid: port ID
6640 *
6641 * Get device physical port ID
6642 */
6643int dev_get_phys_port_id(struct net_device *dev,
Jiri Pirko02637fc2014-11-28 14:34:16 +01006644 struct netdev_phys_item_id *ppid)
Jiri Pirko66b52b02013-07-29 18:16:49 +02006645{
6646 const struct net_device_ops *ops = dev->netdev_ops;
6647
6648 if (!ops->ndo_get_phys_port_id)
6649 return -EOPNOTSUPP;
6650 return ops->ndo_get_phys_port_id(dev, ppid);
6651}
6652EXPORT_SYMBOL(dev_get_phys_port_id);
6653
6654/**
David Aherndb24a902015-03-17 20:23:15 -06006655 * dev_get_phys_port_name - Get device physical port name
6656 * @dev: device
6657 * @name: port name
Luis de Bethencourted49e652016-03-21 16:31:14 +00006658 * @len: limit of bytes to copy to name
David Aherndb24a902015-03-17 20:23:15 -06006659 *
6660 * Get device physical port name
6661 */
6662int dev_get_phys_port_name(struct net_device *dev,
6663 char *name, size_t len)
6664{
6665 const struct net_device_ops *ops = dev->netdev_ops;
6666
6667 if (!ops->ndo_get_phys_port_name)
6668 return -EOPNOTSUPP;
6669 return ops->ndo_get_phys_port_name(dev, name, len);
6670}
6671EXPORT_SYMBOL(dev_get_phys_port_name);
6672
6673/**
Anuradha Karuppiahd746d702015-07-14 13:43:19 -07006674 * dev_change_proto_down - update protocol port state information
6675 * @dev: device
6676 * @proto_down: new value
6677 *
6678 * This info can be used by switch drivers to set the phys state of the
6679 * port.
6680 */
6681int dev_change_proto_down(struct net_device *dev, bool proto_down)
6682{
6683 const struct net_device_ops *ops = dev->netdev_ops;
6684
6685 if (!ops->ndo_change_proto_down)
6686 return -EOPNOTSUPP;
6687 if (!netif_device_present(dev))
6688 return -ENODEV;
6689 return ops->ndo_change_proto_down(dev, proto_down);
6690}
6691EXPORT_SYMBOL(dev_change_proto_down);
6692
6693/**
Brenden Blancoa7862b42016-07-19 12:16:48 -07006694 * dev_change_xdp_fd - set or clear a bpf program for a device rx path
6695 * @dev: device
6696 * @fd: new program fd or negative value to clear
Daniel Borkmann85de8572016-11-28 23:16:54 +01006697 * @flags: xdp-related flags
Brenden Blancoa7862b42016-07-19 12:16:48 -07006698 *
6699 * Set or clear a bpf program for a device
6700 */
Daniel Borkmann85de8572016-11-28 23:16:54 +01006701int dev_change_xdp_fd(struct net_device *dev, int fd, u32 flags)
Brenden Blancoa7862b42016-07-19 12:16:48 -07006702{
6703 const struct net_device_ops *ops = dev->netdev_ops;
6704 struct bpf_prog *prog = NULL;
Daniel Borkmann85de8572016-11-28 23:16:54 +01006705 struct netdev_xdp xdp;
Brenden Blancoa7862b42016-07-19 12:16:48 -07006706 int err;
6707
Daniel Borkmann85de8572016-11-28 23:16:54 +01006708 ASSERT_RTNL();
6709
Brenden Blancoa7862b42016-07-19 12:16:48 -07006710 if (!ops->ndo_xdp)
6711 return -EOPNOTSUPP;
6712 if (fd >= 0) {
Daniel Borkmann85de8572016-11-28 23:16:54 +01006713 if (flags & XDP_FLAGS_UPDATE_IF_NOEXIST) {
6714 memset(&xdp, 0, sizeof(xdp));
6715 xdp.command = XDP_QUERY_PROG;
6716
6717 err = ops->ndo_xdp(dev, &xdp);
6718 if (err < 0)
6719 return err;
6720 if (xdp.prog_attached)
6721 return -EBUSY;
6722 }
6723
Brenden Blancoa7862b42016-07-19 12:16:48 -07006724 prog = bpf_prog_get_type(fd, BPF_PROG_TYPE_XDP);
6725 if (IS_ERR(prog))
6726 return PTR_ERR(prog);
6727 }
6728
Daniel Borkmann85de8572016-11-28 23:16:54 +01006729 memset(&xdp, 0, sizeof(xdp));
Brenden Blancoa7862b42016-07-19 12:16:48 -07006730 xdp.command = XDP_SETUP_PROG;
6731 xdp.prog = prog;
Daniel Borkmann85de8572016-11-28 23:16:54 +01006732
Brenden Blancoa7862b42016-07-19 12:16:48 -07006733 err = ops->ndo_xdp(dev, &xdp);
6734 if (err < 0 && prog)
6735 bpf_prog_put(prog);
6736
6737 return err;
6738}
6739EXPORT_SYMBOL(dev_change_xdp_fd);
6740
6741/**
Linus Torvalds1da177e2005-04-16 15:20:36 -07006742 * dev_new_index - allocate an ifindex
Randy Dunlapc4ea43c2007-10-12 21:17:49 -07006743 * @net: the applicable net namespace
Linus Torvalds1da177e2005-04-16 15:20:36 -07006744 *
6745 * Returns a suitable unique value for a new device interface
6746 * number. The caller must hold the rtnl semaphore or the
6747 * dev_base_lock to be sure it remains unique.
6748 */
Eric W. Biederman881d9662007-09-17 11:56:21 -07006749static int dev_new_index(struct net *net)
Linus Torvalds1da177e2005-04-16 15:20:36 -07006750{
Pavel Emelyanovaa79e662012-08-08 21:53:19 +00006751 int ifindex = net->ifindex;
Linus Torvalds1da177e2005-04-16 15:20:36 -07006752 for (;;) {
6753 if (++ifindex <= 0)
6754 ifindex = 1;
Eric W. Biederman881d9662007-09-17 11:56:21 -07006755 if (!__dev_get_by_index(net, ifindex))
Pavel Emelyanovaa79e662012-08-08 21:53:19 +00006756 return net->ifindex = ifindex;
Linus Torvalds1da177e2005-04-16 15:20:36 -07006757 }
6758}
6759
Linus Torvalds1da177e2005-04-16 15:20:36 -07006760/* Delayed registration/unregisteration */
Denis Cheng3b5b34f2007-12-07 00:49:17 -08006761static LIST_HEAD(net_todo_list);
Cong Wang200b9162014-05-12 15:11:20 -07006762DECLARE_WAIT_QUEUE_HEAD(netdev_unregistering_wq);
Linus Torvalds1da177e2005-04-16 15:20:36 -07006763
Stephen Hemminger6f05f622007-03-08 20:46:03 -08006764static void net_set_todo(struct net_device *dev)
Linus Torvalds1da177e2005-04-16 15:20:36 -07006765{
Linus Torvalds1da177e2005-04-16 15:20:36 -07006766 list_add_tail(&dev->todo_list, &net_todo_list);
Eric W. Biederman50624c92013-09-23 21:19:49 -07006767 dev_net(dev)->dev_unreg_count++;
Linus Torvalds1da177e2005-04-16 15:20:36 -07006768}
6769
Eric Dumazet9b5e3832009-10-27 07:04:19 +00006770static void rollback_registered_many(struct list_head *head)
Daniel Lezcano93ee31f2007-10-30 15:38:18 -07006771{
Krishna Kumare93737b2009-12-08 22:26:02 +00006772 struct net_device *dev, *tmp;
Eric W. Biederman5cde2822013-10-05 19:26:05 -07006773 LIST_HEAD(close_head);
Eric Dumazet9b5e3832009-10-27 07:04:19 +00006774
Daniel Lezcano93ee31f2007-10-30 15:38:18 -07006775 BUG_ON(dev_boot_phase);
6776 ASSERT_RTNL();
6777
Krishna Kumare93737b2009-12-08 22:26:02 +00006778 list_for_each_entry_safe(dev, tmp, head, unreg_list) {
Eric Dumazet9b5e3832009-10-27 07:04:19 +00006779 /* Some devices call without registering
Krishna Kumare93737b2009-12-08 22:26:02 +00006780 * for initialization unwind. Remove those
6781 * devices and proceed with the remaining.
Eric Dumazet9b5e3832009-10-27 07:04:19 +00006782 */
6783 if (dev->reg_state == NETREG_UNINITIALIZED) {
Joe Perches7b6cd1c2012-02-01 10:54:43 +00006784 pr_debug("unregister_netdevice: device %s/%p never was registered\n",
6785 dev->name, dev);
Daniel Lezcano93ee31f2007-10-30 15:38:18 -07006786
Eric Dumazet9b5e3832009-10-27 07:04:19 +00006787 WARN_ON(1);
Krishna Kumare93737b2009-12-08 22:26:02 +00006788 list_del(&dev->unreg_list);
6789 continue;
Eric Dumazet9b5e3832009-10-27 07:04:19 +00006790 }
Eric Dumazet449f4542011-05-19 12:24:16 +00006791 dev->dismantle = true;
Eric Dumazet9b5e3832009-10-27 07:04:19 +00006792 BUG_ON(dev->reg_state != NETREG_REGISTERED);
Octavian Purdila44345722010-12-13 12:44:07 +00006793 }
Eric Dumazet9b5e3832009-10-27 07:04:19 +00006794
Octavian Purdila44345722010-12-13 12:44:07 +00006795 /* If device is running, close it first. */
Eric W. Biederman5cde2822013-10-05 19:26:05 -07006796 list_for_each_entry(dev, head, unreg_list)
6797 list_add_tail(&dev->close_list, &close_head);
David S. Miller99c4a262015-03-18 22:52:33 -04006798 dev_close_many(&close_head, true);
Eric Dumazet9b5e3832009-10-27 07:04:19 +00006799
Octavian Purdila44345722010-12-13 12:44:07 +00006800 list_for_each_entry(dev, head, unreg_list) {
Eric Dumazet9b5e3832009-10-27 07:04:19 +00006801 /* And unlink it from device chain. */
6802 unlist_netdevice(dev);
6803
6804 dev->reg_state = NETREG_UNREGISTERING;
Daniel Lezcano93ee31f2007-10-30 15:38:18 -07006805 }
Eric Dumazet41852492016-08-26 12:50:39 -07006806 flush_all_backlogs();
Daniel Lezcano93ee31f2007-10-30 15:38:18 -07006807
Eric Dumazet9b5e3832009-10-27 07:04:19 +00006808 synchronize_net();
Daniel Lezcano93ee31f2007-10-30 15:38:18 -07006809
Eric Dumazet9b5e3832009-10-27 07:04:19 +00006810 list_for_each_entry(dev, head, unreg_list) {
Mahesh Bandewar395eea62014-12-03 13:46:24 -08006811 struct sk_buff *skb = NULL;
6812
Eric Dumazet9b5e3832009-10-27 07:04:19 +00006813 /* Shutdown queueing discipline. */
6814 dev_shutdown(dev);
Daniel Lezcano93ee31f2007-10-30 15:38:18 -07006815
Daniel Lezcano93ee31f2007-10-30 15:38:18 -07006816
Eric Dumazet9b5e3832009-10-27 07:04:19 +00006817 /* Notify protocols, that we are about to destroy
6818 this device. They should clean all the things.
6819 */
6820 call_netdevice_notifiers(NETDEV_UNREGISTER, dev);
6821
Mahesh Bandewar395eea62014-12-03 13:46:24 -08006822 if (!dev->rtnl_link_ops ||
6823 dev->rtnl_link_state == RTNL_LINK_INITIALIZED)
6824 skb = rtmsg_ifinfo_build_skb(RTM_DELLINK, dev, ~0U,
6825 GFP_KERNEL);
6826
Eric Dumazet9b5e3832009-10-27 07:04:19 +00006827 /*
6828 * Flush the unicast and multicast chains
6829 */
Jiri Pirkoa748ee22010-04-01 21:22:09 +00006830 dev_uc_flush(dev);
Jiri Pirko22bedad32010-04-01 21:22:57 +00006831 dev_mc_flush(dev);
Eric Dumazet9b5e3832009-10-27 07:04:19 +00006832
6833 if (dev->netdev_ops->ndo_uninit)
6834 dev->netdev_ops->ndo_uninit(dev);
6835
Mahesh Bandewar395eea62014-12-03 13:46:24 -08006836 if (skb)
6837 rtmsg_ifinfo_send(skb, dev, GFP_KERNEL);
Roopa Prabhu56bfa7e2014-05-01 11:40:30 -07006838
Jiri Pirko9ff162a2013-01-03 22:48:49 +00006839 /* Notifier chain MUST detach us all upper devices. */
6840 WARN_ON(netdev_has_any_upper_dev(dev));
David Ahern0f524a82016-10-17 19:15:52 -07006841 WARN_ON(netdev_has_any_lower_dev(dev));
Eric Dumazet9b5e3832009-10-27 07:04:19 +00006842
6843 /* Remove entries from kobject tree */
6844 netdev_unregister_kobject(dev);
Alexander Duyck024e9672013-01-10 08:57:46 +00006845#ifdef CONFIG_XPS
6846 /* Remove XPS queueing entries */
6847 netif_reset_xps_queues_gt(dev, 0);
6848#endif
Eric Dumazet9b5e3832009-10-27 07:04:19 +00006849 }
Daniel Lezcano93ee31f2007-10-30 15:38:18 -07006850
Eric W. Biederman850a5452011-10-13 22:25:23 +00006851 synchronize_net();
Daniel Lezcano93ee31f2007-10-30 15:38:18 -07006852
Eric W. Biedermana5ee1552009-11-29 15:45:58 +00006853 list_for_each_entry(dev, head, unreg_list)
Eric Dumazet9b5e3832009-10-27 07:04:19 +00006854 dev_put(dev);
6855}
Daniel Lezcano93ee31f2007-10-30 15:38:18 -07006856
Eric Dumazet9b5e3832009-10-27 07:04:19 +00006857static void rollback_registered(struct net_device *dev)
6858{
6859 LIST_HEAD(single);
Daniel Lezcano93ee31f2007-10-30 15:38:18 -07006860
Eric Dumazet9b5e3832009-10-27 07:04:19 +00006861 list_add(&dev->unreg_list, &single);
6862 rollback_registered_many(&single);
Eric Dumazetceaaec92011-02-17 22:59:19 +00006863 list_del(&single);
Daniel Lezcano93ee31f2007-10-30 15:38:18 -07006864}
6865
Jarod Wilsonfd867d52015-11-02 21:55:59 -05006866static netdev_features_t netdev_sync_upper_features(struct net_device *lower,
6867 struct net_device *upper, netdev_features_t features)
6868{
6869 netdev_features_t upper_disables = NETIF_F_UPPER_DISABLES;
6870 netdev_features_t feature;
Jarod Wilson5ba3f7d2015-11-03 10:15:59 -05006871 int feature_bit;
Jarod Wilsonfd867d52015-11-02 21:55:59 -05006872
Jarod Wilson5ba3f7d2015-11-03 10:15:59 -05006873 for_each_netdev_feature(&upper_disables, feature_bit) {
6874 feature = __NETIF_F_BIT(feature_bit);
Jarod Wilsonfd867d52015-11-02 21:55:59 -05006875 if (!(upper->wanted_features & feature)
6876 && (features & feature)) {
6877 netdev_dbg(lower, "Dropping feature %pNF, upper dev %s has it off.\n",
6878 &feature, upper->name);
6879 features &= ~feature;
6880 }
6881 }
6882
6883 return features;
6884}
6885
6886static void netdev_sync_lower_features(struct net_device *upper,
6887 struct net_device *lower, netdev_features_t features)
6888{
6889 netdev_features_t upper_disables = NETIF_F_UPPER_DISABLES;
6890 netdev_features_t feature;
Jarod Wilson5ba3f7d2015-11-03 10:15:59 -05006891 int feature_bit;
Jarod Wilsonfd867d52015-11-02 21:55:59 -05006892
Jarod Wilson5ba3f7d2015-11-03 10:15:59 -05006893 for_each_netdev_feature(&upper_disables, feature_bit) {
6894 feature = __NETIF_F_BIT(feature_bit);
Jarod Wilsonfd867d52015-11-02 21:55:59 -05006895 if (!(features & feature) && (lower->features & feature)) {
6896 netdev_dbg(upper, "Disabling feature %pNF on lower dev %s.\n",
6897 &feature, lower->name);
6898 lower->wanted_features &= ~feature;
6899 netdev_update_features(lower);
6900
6901 if (unlikely(lower->features & feature))
6902 netdev_WARN(upper, "failed to disable %pNF on %s!\n",
6903 &feature, lower->name);
6904 }
6905 }
6906}
6907
Michał Mirosławc8f44af2011-11-15 15:29:55 +00006908static netdev_features_t netdev_fix_features(struct net_device *dev,
6909 netdev_features_t features)
Herbert Xub63365a2008-10-23 01:11:29 -07006910{
Michał Mirosław57422dc2011-01-22 12:14:12 +00006911 /* Fix illegal checksum combinations */
6912 if ((features & NETIF_F_HW_CSUM) &&
6913 (features & (NETIF_F_IP_CSUM|NETIF_F_IPV6_CSUM))) {
Michał Mirosław6f404e42011-05-16 15:14:21 -04006914 netdev_warn(dev, "mixed HW and IP checksum settings.\n");
Michał Mirosław57422dc2011-01-22 12:14:12 +00006915 features &= ~(NETIF_F_IP_CSUM|NETIF_F_IPV6_CSUM);
6916 }
6917
Herbert Xub63365a2008-10-23 01:11:29 -07006918 /* TSO requires that SG is present as well. */
Ben Hutchingsea2d3682011-04-12 14:38:37 +00006919 if ((features & NETIF_F_ALL_TSO) && !(features & NETIF_F_SG)) {
Michał Mirosław6f404e42011-05-16 15:14:21 -04006920 netdev_dbg(dev, "Dropping TSO features since no SG feature.\n");
Ben Hutchingsea2d3682011-04-12 14:38:37 +00006921 features &= ~NETIF_F_ALL_TSO;
Herbert Xub63365a2008-10-23 01:11:29 -07006922 }
6923
Pravin B Shelarec5f0612013-03-07 09:28:01 +00006924 if ((features & NETIF_F_TSO) && !(features & NETIF_F_HW_CSUM) &&
6925 !(features & NETIF_F_IP_CSUM)) {
6926 netdev_dbg(dev, "Dropping TSO features since no CSUM feature.\n");
6927 features &= ~NETIF_F_TSO;
6928 features &= ~NETIF_F_TSO_ECN;
6929 }
6930
6931 if ((features & NETIF_F_TSO6) && !(features & NETIF_F_HW_CSUM) &&
6932 !(features & NETIF_F_IPV6_CSUM)) {
6933 netdev_dbg(dev, "Dropping TSO6 features since no CSUM feature.\n");
6934 features &= ~NETIF_F_TSO6;
6935 }
6936
Alexander Duyckb1dc4972016-05-02 09:38:24 -07006937 /* TSO with IPv4 ID mangling requires IPv4 TSO be enabled */
6938 if ((features & NETIF_F_TSO_MANGLEID) && !(features & NETIF_F_TSO))
6939 features &= ~NETIF_F_TSO_MANGLEID;
6940
Ben Hutchings31d8b9e2011-04-12 14:47:15 +00006941 /* TSO ECN requires that TSO is present as well. */
6942 if ((features & NETIF_F_ALL_TSO) == NETIF_F_TSO_ECN)
6943 features &= ~NETIF_F_TSO_ECN;
6944
Michał Mirosław212b5732011-02-15 16:59:16 +00006945 /* Software GSO depends on SG. */
6946 if ((features & NETIF_F_GSO) && !(features & NETIF_F_SG)) {
Michał Mirosław6f404e42011-05-16 15:14:21 -04006947 netdev_dbg(dev, "Dropping NETIF_F_GSO since no SG feature.\n");
Michał Mirosław212b5732011-02-15 16:59:16 +00006948 features &= ~NETIF_F_GSO;
6949 }
6950
Michał Mirosławacd11302011-01-24 15:45:15 -08006951 /* UFO needs SG and checksumming */
Herbert Xub63365a2008-10-23 01:11:29 -07006952 if (features & NETIF_F_UFO) {
Michał Mirosław79032642010-11-30 06:38:00 +00006953 /* maybe split UFO into V4 and V6? */
Tom Herbertc8cd0982015-12-14 11:19:44 -08006954 if (!(features & NETIF_F_HW_CSUM) &&
6955 ((features & (NETIF_F_IP_CSUM | NETIF_F_IPV6_CSUM)) !=
6956 (NETIF_F_IP_CSUM | NETIF_F_IPV6_CSUM))) {
Michał Mirosław6f404e42011-05-16 15:14:21 -04006957 netdev_dbg(dev,
Michał Mirosławacd11302011-01-24 15:45:15 -08006958 "Dropping NETIF_F_UFO since no checksum offload features.\n");
Herbert Xub63365a2008-10-23 01:11:29 -07006959 features &= ~NETIF_F_UFO;
6960 }
6961
6962 if (!(features & NETIF_F_SG)) {
Michał Mirosław6f404e42011-05-16 15:14:21 -04006963 netdev_dbg(dev,
Michał Mirosławacd11302011-01-24 15:45:15 -08006964 "Dropping NETIF_F_UFO since no NETIF_F_SG feature.\n");
Herbert Xub63365a2008-10-23 01:11:29 -07006965 features &= ~NETIF_F_UFO;
6966 }
6967 }
6968
Alexander Duyck802ab552016-04-10 21:45:03 -04006969 /* GSO partial features require GSO partial be set */
6970 if ((features & dev->gso_partial_features) &&
6971 !(features & NETIF_F_GSO_PARTIAL)) {
6972 netdev_dbg(dev,
6973 "Dropping partially supported GSO features since no GSO partial.\n");
6974 features &= ~dev->gso_partial_features;
6975 }
6976
Jiri Pirkod0290212014-04-02 23:09:31 +02006977#ifdef CONFIG_NET_RX_BUSY_POLL
6978 if (dev->netdev_ops->ndo_busy_poll)
6979 features |= NETIF_F_BUSY_POLL;
6980 else
6981#endif
6982 features &= ~NETIF_F_BUSY_POLL;
6983
Herbert Xub63365a2008-10-23 01:11:29 -07006984 return features;
6985}
Herbert Xub63365a2008-10-23 01:11:29 -07006986
Michał Mirosław6cb6a272011-04-02 22:48:47 -07006987int __netdev_update_features(struct net_device *dev)
Michał Mirosław5455c692011-02-15 16:59:17 +00006988{
Jarod Wilsonfd867d52015-11-02 21:55:59 -05006989 struct net_device *upper, *lower;
Michał Mirosławc8f44af2011-11-15 15:29:55 +00006990 netdev_features_t features;
Jarod Wilsonfd867d52015-11-02 21:55:59 -05006991 struct list_head *iter;
Jarod Wilsone7868a82015-11-03 23:09:32 -05006992 int err = -1;
Michał Mirosław5455c692011-02-15 16:59:17 +00006993
Michał Mirosław87267482011-04-12 09:56:38 +00006994 ASSERT_RTNL();
6995
Michał Mirosław5455c692011-02-15 16:59:17 +00006996 features = netdev_get_wanted_features(dev);
6997
6998 if (dev->netdev_ops->ndo_fix_features)
6999 features = dev->netdev_ops->ndo_fix_features(dev, features);
7000
7001 /* driver might be less strict about feature dependencies */
7002 features = netdev_fix_features(dev, features);
7003
Jarod Wilsonfd867d52015-11-02 21:55:59 -05007004 /* some features can't be enabled if they're off an an upper device */
7005 netdev_for_each_upper_dev_rcu(dev, upper, iter)
7006 features = netdev_sync_upper_features(dev, upper, features);
7007
Michał Mirosław5455c692011-02-15 16:59:17 +00007008 if (dev->features == features)
Jarod Wilsone7868a82015-11-03 23:09:32 -05007009 goto sync_lower;
Michał Mirosław5455c692011-02-15 16:59:17 +00007010
Michał Mirosławc8f44af2011-11-15 15:29:55 +00007011 netdev_dbg(dev, "Features changed: %pNF -> %pNF\n",
7012 &dev->features, &features);
Michał Mirosław5455c692011-02-15 16:59:17 +00007013
7014 if (dev->netdev_ops->ndo_set_features)
7015 err = dev->netdev_ops->ndo_set_features(dev, features);
Nikolay Aleksandrov5f8dc332015-11-13 14:54:01 +01007016 else
7017 err = 0;
Michał Mirosław5455c692011-02-15 16:59:17 +00007018
Michał Mirosław6cb6a272011-04-02 22:48:47 -07007019 if (unlikely(err < 0)) {
Michał Mirosław5455c692011-02-15 16:59:17 +00007020 netdev_err(dev,
Michał Mirosławc8f44af2011-11-15 15:29:55 +00007021 "set_features() failed (%d); wanted %pNF, left %pNF\n",
7022 err, &features, &dev->features);
Nikolay Aleksandrov17b85d22015-11-17 15:49:06 +01007023 /* return non-0 since some features might have changed and
7024 * it's better to fire a spurious notification than miss it
7025 */
7026 return -1;
Michał Mirosław6cb6a272011-04-02 22:48:47 -07007027 }
7028
Jarod Wilsone7868a82015-11-03 23:09:32 -05007029sync_lower:
Jarod Wilsonfd867d52015-11-02 21:55:59 -05007030 /* some features must be disabled on lower devices when disabled
7031 * on an upper device (think: bonding master or bridge)
7032 */
7033 netdev_for_each_lower_dev(dev, lower, iter)
7034 netdev_sync_lower_features(dev, lower, features);
7035
Michał Mirosław6cb6a272011-04-02 22:48:47 -07007036 if (!err)
7037 dev->features = features;
7038
Jarod Wilsone7868a82015-11-03 23:09:32 -05007039 return err < 0 ? 0 : 1;
Michał Mirosław6cb6a272011-04-02 22:48:47 -07007040}
7041
Michał Mirosławafe12cc2011-05-07 03:22:17 +00007042/**
7043 * netdev_update_features - recalculate device features
7044 * @dev: the device to check
7045 *
7046 * Recalculate dev->features set and send notifications if it
7047 * has changed. Should be called after driver or hardware dependent
7048 * conditions might have changed that influence the features.
7049 */
Michał Mirosław6cb6a272011-04-02 22:48:47 -07007050void netdev_update_features(struct net_device *dev)
7051{
7052 if (__netdev_update_features(dev))
7053 netdev_features_change(dev);
Michał Mirosław5455c692011-02-15 16:59:17 +00007054}
7055EXPORT_SYMBOL(netdev_update_features);
7056
Linus Torvalds1da177e2005-04-16 15:20:36 -07007057/**
Michał Mirosławafe12cc2011-05-07 03:22:17 +00007058 * netdev_change_features - recalculate device features
7059 * @dev: the device to check
7060 *
7061 * Recalculate dev->features set and send notifications even
7062 * if they have not changed. Should be called instead of
7063 * netdev_update_features() if also dev->vlan_features might
7064 * have changed to allow the changes to be propagated to stacked
7065 * VLAN devices.
7066 */
7067void netdev_change_features(struct net_device *dev)
7068{
7069 __netdev_update_features(dev);
7070 netdev_features_change(dev);
7071}
7072EXPORT_SYMBOL(netdev_change_features);
7073
7074/**
Patrick Mullaneyfc4a7482009-12-03 15:59:22 -08007075 * netif_stacked_transfer_operstate - transfer operstate
7076 * @rootdev: the root or lower level device to transfer state from
7077 * @dev: the device to transfer operstate to
7078 *
7079 * Transfer operational state from root to device. This is normally
7080 * called when a stacking relationship exists between the root
7081 * device and the device(a leaf device).
7082 */
7083void netif_stacked_transfer_operstate(const struct net_device *rootdev,
7084 struct net_device *dev)
7085{
7086 if (rootdev->operstate == IF_OPER_DORMANT)
7087 netif_dormant_on(dev);
7088 else
7089 netif_dormant_off(dev);
7090
7091 if (netif_carrier_ok(rootdev)) {
7092 if (!netif_carrier_ok(dev))
7093 netif_carrier_on(dev);
7094 } else {
7095 if (netif_carrier_ok(dev))
7096 netif_carrier_off(dev);
7097 }
7098}
7099EXPORT_SYMBOL(netif_stacked_transfer_operstate);
7100
Michael Daltona953be52014-01-16 22:23:28 -08007101#ifdef CONFIG_SYSFS
Eric Dumazet1b4bf462010-09-23 17:26:35 +00007102static int netif_alloc_rx_queues(struct net_device *dev)
7103{
Eric Dumazet1b4bf462010-09-23 17:26:35 +00007104 unsigned int i, count = dev->num_rx_queues;
Tom Herbertbd25fa72010-10-18 18:00:16 +00007105 struct netdev_rx_queue *rx;
Pankaj Gupta10595902015-01-12 11:41:28 +05307106 size_t sz = count * sizeof(*rx);
Eric Dumazet1b4bf462010-09-23 17:26:35 +00007107
Tom Herbertbd25fa72010-10-18 18:00:16 +00007108 BUG_ON(count < 1);
Eric Dumazet1b4bf462010-09-23 17:26:35 +00007109
Pankaj Gupta10595902015-01-12 11:41:28 +05307110 rx = kzalloc(sz, GFP_KERNEL | __GFP_NOWARN | __GFP_REPEAT);
7111 if (!rx) {
7112 rx = vzalloc(sz);
7113 if (!rx)
7114 return -ENOMEM;
7115 }
Tom Herbertbd25fa72010-10-18 18:00:16 +00007116 dev->_rx = rx;
7117
Tom Herbertbd25fa72010-10-18 18:00:16 +00007118 for (i = 0; i < count; i++)
Tom Herbertfe822242010-11-09 10:47:38 +00007119 rx[i].dev = dev;
Eric Dumazet1b4bf462010-09-23 17:26:35 +00007120 return 0;
7121}
Tom Herbertbf264142010-11-26 08:36:09 +00007122#endif
Eric Dumazet1b4bf462010-09-23 17:26:35 +00007123
Changli Gaoaa942102010-12-04 02:31:41 +00007124static void netdev_init_one_queue(struct net_device *dev,
7125 struct netdev_queue *queue, void *_unused)
7126{
7127 /* Initialize queue lock */
7128 spin_lock_init(&queue->_xmit_lock);
7129 netdev_set_xmit_lockdep_class(&queue->_xmit_lock, dev->type);
7130 queue->xmit_lock_owner = -1;
Changli Gaob236da62010-12-14 03:09:15 +00007131 netdev_queue_numa_node_write(queue, NUMA_NO_NODE);
Changli Gaoaa942102010-12-04 02:31:41 +00007132 queue->dev = dev;
Tom Herbert114cf582011-11-28 16:33:09 +00007133#ifdef CONFIG_BQL
7134 dql_init(&queue->dql, HZ);
7135#endif
Changli Gaoaa942102010-12-04 02:31:41 +00007136}
7137
Eric Dumazet60877a32013-06-20 01:15:51 -07007138static void netif_free_tx_queues(struct net_device *dev)
7139{
WANG Cong4cb28972014-06-02 15:55:22 -07007140 kvfree(dev->_tx);
Eric Dumazet60877a32013-06-20 01:15:51 -07007141}
7142
Tom Herberte6484932010-10-18 18:04:39 +00007143static int netif_alloc_netdev_queues(struct net_device *dev)
7144{
7145 unsigned int count = dev->num_tx_queues;
7146 struct netdev_queue *tx;
Eric Dumazet60877a32013-06-20 01:15:51 -07007147 size_t sz = count * sizeof(*tx);
Tom Herberte6484932010-10-18 18:04:39 +00007148
Eric Dumazetd3397272015-07-06 17:13:26 +02007149 if (count < 1 || count > 0xffff)
7150 return -EINVAL;
Tom Herberte6484932010-10-18 18:04:39 +00007151
Eric Dumazet60877a32013-06-20 01:15:51 -07007152 tx = kzalloc(sz, GFP_KERNEL | __GFP_NOWARN | __GFP_REPEAT);
7153 if (!tx) {
7154 tx = vzalloc(sz);
7155 if (!tx)
7156 return -ENOMEM;
7157 }
Tom Herberte6484932010-10-18 18:04:39 +00007158 dev->_tx = tx;
Tom Herbert1d24eb42010-11-21 13:17:27 +00007159
Tom Herberte6484932010-10-18 18:04:39 +00007160 netdev_for_each_tx_queue(dev, netdev_init_one_queue, NULL);
7161 spin_lock_init(&dev->tx_global_lock);
Changli Gaoaa942102010-12-04 02:31:41 +00007162
7163 return 0;
Tom Herberte6484932010-10-18 18:04:39 +00007164}
7165
Denys Vlasenkoa2029242015-05-11 21:17:53 +02007166void netif_tx_stop_all_queues(struct net_device *dev)
7167{
7168 unsigned int i;
7169
7170 for (i = 0; i < dev->num_tx_queues; i++) {
7171 struct netdev_queue *txq = netdev_get_tx_queue(dev, i);
7172 netif_tx_stop_queue(txq);
7173 }
7174}
7175EXPORT_SYMBOL(netif_tx_stop_all_queues);
7176
Patrick Mullaneyfc4a7482009-12-03 15:59:22 -08007177/**
Linus Torvalds1da177e2005-04-16 15:20:36 -07007178 * register_netdevice - register a network device
7179 * @dev: device to register
7180 *
7181 * Take a completed network device structure and add it to the kernel
7182 * interfaces. A %NETDEV_REGISTER message is sent to the netdev notifier
7183 * chain. 0 is returned on success. A negative errno code is returned
7184 * on a failure to set up the device, or if the name is a duplicate.
7185 *
7186 * Callers must hold the rtnl semaphore. You may want
7187 * register_netdev() instead of this.
7188 *
7189 * BUGS:
7190 * The locking appears insufficient to guarantee two parallel registers
7191 * will not get the same name.
7192 */
7193
7194int register_netdevice(struct net_device *dev)
7195{
Linus Torvalds1da177e2005-04-16 15:20:36 -07007196 int ret;
Stephen Hemmingerd3147742008-11-19 21:32:24 -08007197 struct net *net = dev_net(dev);
Linus Torvalds1da177e2005-04-16 15:20:36 -07007198
7199 BUG_ON(dev_boot_phase);
7200 ASSERT_RTNL();
7201
Stephen Hemmingerb17a7c12006-05-10 13:21:17 -07007202 might_sleep();
7203
Linus Torvalds1da177e2005-04-16 15:20:36 -07007204 /* When net_device's are persistent, this will be fatal. */
7205 BUG_ON(dev->reg_state != NETREG_UNINITIALIZED);
Stephen Hemmingerd3147742008-11-19 21:32:24 -08007206 BUG_ON(!net);
Linus Torvalds1da177e2005-04-16 15:20:36 -07007207
David S. Millerf1f28aa2008-07-15 00:08:33 -07007208 spin_lock_init(&dev->addr_list_lock);
David S. Millercf508b12008-07-22 14:16:42 -07007209 netdev_set_addr_lockdep_class(dev);
Linus Torvalds1da177e2005-04-16 15:20:36 -07007210
Gao feng828de4f2012-09-13 20:58:27 +00007211 ret = dev_get_valid_name(net, dev, dev->name);
Peter Pan(潘卫平)0696c3a2011-05-12 15:46:56 +00007212 if (ret < 0)
7213 goto out;
7214
Linus Torvalds1da177e2005-04-16 15:20:36 -07007215 /* Init, if this function is available */
Stephen Hemmingerd3147742008-11-19 21:32:24 -08007216 if (dev->netdev_ops->ndo_init) {
7217 ret = dev->netdev_ops->ndo_init(dev);
Linus Torvalds1da177e2005-04-16 15:20:36 -07007218 if (ret) {
7219 if (ret > 0)
7220 ret = -EIO;
Adrian Bunk90833aa2006-11-13 16:02:22 -08007221 goto out;
Linus Torvalds1da177e2005-04-16 15:20:36 -07007222 }
7223 }
YOSHIFUJI Hideaki4ec93ed2007-02-09 23:24:36 +09007224
Patrick McHardyf6469682013-04-19 02:04:27 +00007225 if (((dev->hw_features | dev->features) &
7226 NETIF_F_HW_VLAN_CTAG_FILTER) &&
Michał Mirosławd2ed2732013-01-29 15:14:16 +00007227 (!dev->netdev_ops->ndo_vlan_rx_add_vid ||
7228 !dev->netdev_ops->ndo_vlan_rx_kill_vid)) {
7229 netdev_WARN(dev, "Buggy VLAN acceleration in driver!\n");
7230 ret = -EINVAL;
7231 goto err_uninit;
7232 }
7233
Pavel Emelyanov9c7dafb2012-08-08 21:52:46 +00007234 ret = -EBUSY;
7235 if (!dev->ifindex)
7236 dev->ifindex = dev_new_index(net);
7237 else if (__dev_get_by_index(net, dev->ifindex))
7238 goto err_uninit;
7239
Michał Mirosław5455c692011-02-15 16:59:17 +00007240 /* Transfer changeable features to wanted_features and enable
7241 * software offloads (GSO and GRO).
7242 */
7243 dev->hw_features |= NETIF_F_SOFT_FEATURES;
Michał Mirosław14d12322011-02-22 16:52:28 +00007244 dev->features |= NETIF_F_SOFT_FEATURES;
7245 dev->wanted_features = dev->features & dev->hw_features;
Linus Torvalds1da177e2005-04-16 15:20:36 -07007246
Alexander Duyckcbc53e02016-04-10 21:44:51 -04007247 if (!(dev->flags & IFF_LOOPBACK))
Michał Mirosław34324dc2011-11-15 15:29:55 +00007248 dev->hw_features |= NETIF_F_NOCACHE_COPY;
Alexander Duyckcbc53e02016-04-10 21:44:51 -04007249
Alexander Duyck7f348a62016-04-20 16:51:00 -04007250 /* If IPv4 TCP segmentation offload is supported we should also
7251 * allow the device to enable segmenting the frame with the option
7252 * of ignoring a static IP ID value. This doesn't enable the
7253 * feature itself but allows the user to enable it later.
7254 */
Alexander Duyckcbc53e02016-04-10 21:44:51 -04007255 if (dev->hw_features & NETIF_F_TSO)
7256 dev->hw_features |= NETIF_F_TSO_MANGLEID;
Alexander Duyck7f348a62016-04-20 16:51:00 -04007257 if (dev->vlan_features & NETIF_F_TSO)
7258 dev->vlan_features |= NETIF_F_TSO_MANGLEID;
7259 if (dev->mpls_features & NETIF_F_TSO)
7260 dev->mpls_features |= NETIF_F_TSO_MANGLEID;
7261 if (dev->hw_enc_features & NETIF_F_TSO)
7262 dev->hw_enc_features |= NETIF_F_TSO_MANGLEID;
Tom Herbertc6e1a0d2011-04-04 22:30:30 -07007263
Michał Mirosław1180e7d2011-07-14 14:41:11 -07007264 /* Make NETIF_F_HIGHDMA inheritable to VLAN devices.
Brandon Philips16c3ea72010-09-15 09:24:24 +00007265 */
Michał Mirosław1180e7d2011-07-14 14:41:11 -07007266 dev->vlan_features |= NETIF_F_HIGHDMA;
Brandon Philips16c3ea72010-09-15 09:24:24 +00007267
Pravin B Shelaree579672013-03-07 09:28:08 +00007268 /* Make NETIF_F_SG inheritable to tunnel devices.
7269 */
Alexander Duyck802ab552016-04-10 21:45:03 -04007270 dev->hw_enc_features |= NETIF_F_SG | NETIF_F_GSO_PARTIAL;
Pravin B Shelaree579672013-03-07 09:28:08 +00007271
Simon Horman0d89d202013-05-23 21:02:52 +00007272 /* Make NETIF_F_SG inheritable to MPLS.
7273 */
7274 dev->mpls_features |= NETIF_F_SG;
7275
Johannes Berg7ffbe3f2009-10-02 05:15:27 +00007276 ret = call_netdevice_notifiers(NETDEV_POST_INIT, dev);
7277 ret = notifier_to_errno(ret);
7278 if (ret)
7279 goto err_uninit;
7280
Eric W. Biederman8b41d182007-09-26 22:02:53 -07007281 ret = netdev_register_kobject(dev);
Stephen Hemmingerb17a7c12006-05-10 13:21:17 -07007282 if (ret)
Herbert Xu7ce1b0e2007-07-30 16:29:40 -07007283 goto err_uninit;
Stephen Hemmingerb17a7c12006-05-10 13:21:17 -07007284 dev->reg_state = NETREG_REGISTERED;
7285
Michał Mirosław6cb6a272011-04-02 22:48:47 -07007286 __netdev_update_features(dev);
Michał Mirosław8e9b59b2011-02-22 16:52:28 +00007287
Linus Torvalds1da177e2005-04-16 15:20:36 -07007288 /*
7289 * Default initial state at registry is that the
7290 * device is present.
7291 */
7292
7293 set_bit(__LINK_STATE_PRESENT, &dev->state);
7294
Ben Hutchings8f4cccb2012-08-20 22:16:51 +01007295 linkwatch_init_dev(dev);
7296
Linus Torvalds1da177e2005-04-16 15:20:36 -07007297 dev_init_scheduler(dev);
Linus Torvalds1da177e2005-04-16 15:20:36 -07007298 dev_hold(dev);
Eric W. Biedermance286d32007-09-12 13:53:49 +02007299 list_netdevice(dev);
Theodore Ts'o7bf23572012-07-04 21:23:25 -04007300 add_device_randomness(dev->dev_addr, dev->addr_len);
Linus Torvalds1da177e2005-04-16 15:20:36 -07007301
Jiri Pirko948b3372013-01-08 01:38:25 +00007302 /* If the device has permanent device address, driver should
7303 * set dev_addr and also addr_assign_type should be set to
7304 * NET_ADDR_PERM (default value).
7305 */
7306 if (dev->addr_assign_type == NET_ADDR_PERM)
7307 memcpy(dev->perm_addr, dev->dev_addr, dev->addr_len);
7308
Linus Torvalds1da177e2005-04-16 15:20:36 -07007309 /* Notify protocols, that a new device appeared. */
Pavel Emelyanov056925a2007-09-16 15:42:43 -07007310 ret = call_netdevice_notifiers(NETDEV_REGISTER, dev);
Herbert Xufcc5a032007-07-30 17:03:38 -07007311 ret = notifier_to_errno(ret);
Daniel Lezcano93ee31f2007-10-30 15:38:18 -07007312 if (ret) {
7313 rollback_registered(dev);
7314 dev->reg_state = NETREG_UNREGISTERED;
7315 }
Eric W. Biedermand90a9092009-12-12 22:11:15 +00007316 /*
7317 * Prevent userspace races by waiting until the network
7318 * device is fully setup before sending notifications.
7319 */
Patrick McHardya2835762010-02-26 06:34:51 +00007320 if (!dev->rtnl_link_ops ||
7321 dev->rtnl_link_state == RTNL_LINK_INITIALIZED)
Alexei Starovoitov7f294052013-10-23 16:02:42 -07007322 rtmsg_ifinfo(RTM_NEWLINK, dev, ~0U, GFP_KERNEL);
Linus Torvalds1da177e2005-04-16 15:20:36 -07007323
7324out:
7325 return ret;
Herbert Xu7ce1b0e2007-07-30 16:29:40 -07007326
7327err_uninit:
Stephen Hemmingerd3147742008-11-19 21:32:24 -08007328 if (dev->netdev_ops->ndo_uninit)
7329 dev->netdev_ops->ndo_uninit(dev);
Herbert Xu7ce1b0e2007-07-30 16:29:40 -07007330 goto out;
Linus Torvalds1da177e2005-04-16 15:20:36 -07007331}
Eric Dumazetd1b19df2009-09-03 01:29:39 -07007332EXPORT_SYMBOL(register_netdevice);
Linus Torvalds1da177e2005-04-16 15:20:36 -07007333
7334/**
Benjamin Herrenschmidt937f1ba2009-01-14 21:05:05 -08007335 * init_dummy_netdev - init a dummy network device for NAPI
7336 * @dev: device to init
7337 *
7338 * This takes a network device structure and initialize the minimum
7339 * amount of fields so it can be used to schedule NAPI polls without
7340 * registering a full blown interface. This is to be used by drivers
7341 * that need to tie several hardware interfaces to a single NAPI
7342 * poll scheduler due to HW limitations.
7343 */
7344int init_dummy_netdev(struct net_device *dev)
7345{
7346 /* Clear everything. Note we don't initialize spinlocks
7347 * are they aren't supposed to be taken by any of the
7348 * NAPI code and this dummy netdev is supposed to be
7349 * only ever used for NAPI polls
7350 */
7351 memset(dev, 0, sizeof(struct net_device));
7352
7353 /* make sure we BUG if trying to hit standard
7354 * register/unregister code path
7355 */
7356 dev->reg_state = NETREG_DUMMY;
7357
Benjamin Herrenschmidt937f1ba2009-01-14 21:05:05 -08007358 /* NAPI wants this */
7359 INIT_LIST_HEAD(&dev->napi_list);
7360
7361 /* a dummy interface is started by default */
7362 set_bit(__LINK_STATE_PRESENT, &dev->state);
7363 set_bit(__LINK_STATE_START, &dev->state);
7364
Eric Dumazet29b44332010-10-11 10:22:12 +00007365 /* Note : We dont allocate pcpu_refcnt for dummy devices,
7366 * because users of this 'device' dont need to change
7367 * its refcount.
7368 */
7369
Benjamin Herrenschmidt937f1ba2009-01-14 21:05:05 -08007370 return 0;
7371}
7372EXPORT_SYMBOL_GPL(init_dummy_netdev);
7373
7374
7375/**
Linus Torvalds1da177e2005-04-16 15:20:36 -07007376 * register_netdev - register a network device
7377 * @dev: device to register
7378 *
7379 * Take a completed network device structure and add it to the kernel
7380 * interfaces. A %NETDEV_REGISTER message is sent to the netdev notifier
7381 * chain. 0 is returned on success. A negative errno code is returned
7382 * on a failure to set up the device, or if the name is a duplicate.
7383 *
Borislav Petkov38b4da32007-04-20 22:14:10 -07007384 * This is a wrapper around register_netdevice that takes the rtnl semaphore
Linus Torvalds1da177e2005-04-16 15:20:36 -07007385 * and expands the device name if you passed a format string to
7386 * alloc_netdev.
7387 */
7388int register_netdev(struct net_device *dev)
7389{
7390 int err;
7391
7392 rtnl_lock();
Linus Torvalds1da177e2005-04-16 15:20:36 -07007393 err = register_netdevice(dev);
Linus Torvalds1da177e2005-04-16 15:20:36 -07007394 rtnl_unlock();
7395 return err;
7396}
7397EXPORT_SYMBOL(register_netdev);
7398
Eric Dumazet29b44332010-10-11 10:22:12 +00007399int netdev_refcnt_read(const struct net_device *dev)
7400{
7401 int i, refcnt = 0;
7402
7403 for_each_possible_cpu(i)
7404 refcnt += *per_cpu_ptr(dev->pcpu_refcnt, i);
7405 return refcnt;
7406}
7407EXPORT_SYMBOL(netdev_refcnt_read);
7408
Ben Hutchings2c530402012-07-10 10:55:09 +00007409/**
Linus Torvalds1da177e2005-04-16 15:20:36 -07007410 * netdev_wait_allrefs - wait until all references are gone.
Randy Dunlap3de7a372012-08-18 14:36:44 +00007411 * @dev: target net_device
Linus Torvalds1da177e2005-04-16 15:20:36 -07007412 *
7413 * This is called when unregistering network devices.
7414 *
7415 * Any protocol or device that holds a reference should register
7416 * for netdevice notification, and cleanup and put back the
7417 * reference if they receive an UNREGISTER event.
7418 * We can get stuck here if buggy protocols don't correctly
YOSHIFUJI Hideaki4ec93ed2007-02-09 23:24:36 +09007419 * call dev_put.
Linus Torvalds1da177e2005-04-16 15:20:36 -07007420 */
7421static void netdev_wait_allrefs(struct net_device *dev)
7422{
7423 unsigned long rebroadcast_time, warning_time;
Eric Dumazet29b44332010-10-11 10:22:12 +00007424 int refcnt;
Linus Torvalds1da177e2005-04-16 15:20:36 -07007425
Eric Dumazete014deb2009-11-17 05:59:21 +00007426 linkwatch_forget_dev(dev);
7427
Linus Torvalds1da177e2005-04-16 15:20:36 -07007428 rebroadcast_time = warning_time = jiffies;
Eric Dumazet29b44332010-10-11 10:22:12 +00007429 refcnt = netdev_refcnt_read(dev);
7430
7431 while (refcnt != 0) {
Linus Torvalds1da177e2005-04-16 15:20:36 -07007432 if (time_after(jiffies, rebroadcast_time + 1 * HZ)) {
Stephen Hemminger6756ae42006-03-20 22:23:58 -08007433 rtnl_lock();
Linus Torvalds1da177e2005-04-16 15:20:36 -07007434
7435 /* Rebroadcast unregister notification */
Pavel Emelyanov056925a2007-09-16 15:42:43 -07007436 call_netdevice_notifiers(NETDEV_UNREGISTER, dev);
Linus Torvalds1da177e2005-04-16 15:20:36 -07007437
Eric Dumazet748e2d92012-08-22 21:50:59 +00007438 __rtnl_unlock();
Eric Dumazet0115e8e2012-08-22 17:19:46 +00007439 rcu_barrier();
Eric Dumazet748e2d92012-08-22 21:50:59 +00007440 rtnl_lock();
7441
Eric Dumazet0115e8e2012-08-22 17:19:46 +00007442 call_netdevice_notifiers(NETDEV_UNREGISTER_FINAL, dev);
Linus Torvalds1da177e2005-04-16 15:20:36 -07007443 if (test_bit(__LINK_STATE_LINKWATCH_PENDING,
7444 &dev->state)) {
7445 /* We must not have linkwatch events
7446 * pending on unregister. If this
7447 * happens, we simply run the queue
7448 * unscheduled, resulting in a noop
7449 * for this device.
7450 */
7451 linkwatch_run_queue();
7452 }
7453
Stephen Hemminger6756ae42006-03-20 22:23:58 -08007454 __rtnl_unlock();
Linus Torvalds1da177e2005-04-16 15:20:36 -07007455
7456 rebroadcast_time = jiffies;
7457 }
7458
7459 msleep(250);
7460
Eric Dumazet29b44332010-10-11 10:22:12 +00007461 refcnt = netdev_refcnt_read(dev);
7462
Linus Torvalds1da177e2005-04-16 15:20:36 -07007463 if (time_after(jiffies, warning_time + 10 * HZ)) {
Joe Perches7b6cd1c2012-02-01 10:54:43 +00007464 pr_emerg("unregister_netdevice: waiting for %s to become free. Usage count = %d\n",
7465 dev->name, refcnt);
Linus Torvalds1da177e2005-04-16 15:20:36 -07007466 warning_time = jiffies;
7467 }
7468 }
7469}
7470
7471/* The sequence is:
7472 *
7473 * rtnl_lock();
7474 * ...
7475 * register_netdevice(x1);
7476 * register_netdevice(x2);
7477 * ...
7478 * unregister_netdevice(y1);
7479 * unregister_netdevice(y2);
7480 * ...
7481 * rtnl_unlock();
7482 * free_netdev(y1);
7483 * free_netdev(y2);
7484 *
Herbert Xu58ec3b42008-10-07 15:50:03 -07007485 * We are invoked by rtnl_unlock().
Linus Torvalds1da177e2005-04-16 15:20:36 -07007486 * This allows us to deal with problems:
Stephen Hemmingerb17a7c12006-05-10 13:21:17 -07007487 * 1) We can delete sysfs objects which invoke hotplug
Linus Torvalds1da177e2005-04-16 15:20:36 -07007488 * without deadlocking with linkwatch via keventd.
7489 * 2) Since we run with the RTNL semaphore not held, we can sleep
7490 * safely in order to wait for the netdev refcnt to drop to zero.
Herbert Xu58ec3b42008-10-07 15:50:03 -07007491 *
7492 * We must not return until all unregister events added during
7493 * the interval the lock was held have been completed.
Linus Torvalds1da177e2005-04-16 15:20:36 -07007494 */
Linus Torvalds1da177e2005-04-16 15:20:36 -07007495void netdev_run_todo(void)
7496{
Oleg Nesterov626ab0e2006-06-23 02:05:55 -07007497 struct list_head list;
Linus Torvalds1da177e2005-04-16 15:20:36 -07007498
Linus Torvalds1da177e2005-04-16 15:20:36 -07007499 /* Snapshot list, allow later requests */
Oleg Nesterov626ab0e2006-06-23 02:05:55 -07007500 list_replace_init(&net_todo_list, &list);
Herbert Xu58ec3b42008-10-07 15:50:03 -07007501
7502 __rtnl_unlock();
Oleg Nesterov626ab0e2006-06-23 02:05:55 -07007503
Eric Dumazet0115e8e2012-08-22 17:19:46 +00007504
7505 /* Wait for rcu callbacks to finish before next phase */
Eric W. Biederman850a5452011-10-13 22:25:23 +00007506 if (!list_empty(&list))
7507 rcu_barrier();
7508
Linus Torvalds1da177e2005-04-16 15:20:36 -07007509 while (!list_empty(&list)) {
7510 struct net_device *dev
stephen hemmingere5e26d72010-02-24 14:01:38 +00007511 = list_first_entry(&list, struct net_device, todo_list);
Linus Torvalds1da177e2005-04-16 15:20:36 -07007512 list_del(&dev->todo_list);
7513
Eric Dumazet748e2d92012-08-22 21:50:59 +00007514 rtnl_lock();
Eric Dumazet0115e8e2012-08-22 17:19:46 +00007515 call_netdevice_notifiers(NETDEV_UNREGISTER_FINAL, dev);
Eric Dumazet748e2d92012-08-22 21:50:59 +00007516 __rtnl_unlock();
Eric Dumazet0115e8e2012-08-22 17:19:46 +00007517
Stephen Hemmingerb17a7c12006-05-10 13:21:17 -07007518 if (unlikely(dev->reg_state != NETREG_UNREGISTERING)) {
Joe Perches7b6cd1c2012-02-01 10:54:43 +00007519 pr_err("network todo '%s' but state %d\n",
Linus Torvalds1da177e2005-04-16 15:20:36 -07007520 dev->name, dev->reg_state);
Stephen Hemmingerb17a7c12006-05-10 13:21:17 -07007521 dump_stack();
7522 continue;
Linus Torvalds1da177e2005-04-16 15:20:36 -07007523 }
Stephen Hemmingerb17a7c12006-05-10 13:21:17 -07007524
Stephen Hemmingerb17a7c12006-05-10 13:21:17 -07007525 dev->reg_state = NETREG_UNREGISTERED;
7526
7527 netdev_wait_allrefs(dev);
7528
7529 /* paranoia */
Eric Dumazet29b44332010-10-11 10:22:12 +00007530 BUG_ON(netdev_refcnt_read(dev));
Salam Noureddine7866a622015-01-27 11:35:48 -08007531 BUG_ON(!list_empty(&dev->ptype_all));
7532 BUG_ON(!list_empty(&dev->ptype_specific));
Eric Dumazet33d480c2011-08-11 19:30:52 +00007533 WARN_ON(rcu_access_pointer(dev->ip_ptr));
7534 WARN_ON(rcu_access_pointer(dev->ip6_ptr));
Ilpo Järvinen547b7922008-07-25 21:43:18 -07007535 WARN_ON(dev->dn_ptr);
Stephen Hemmingerb17a7c12006-05-10 13:21:17 -07007536
Stephen Hemmingerb17a7c12006-05-10 13:21:17 -07007537 if (dev->destructor)
7538 dev->destructor(dev);
Stephen Hemminger9093bbb2007-05-19 15:39:25 -07007539
Eric W. Biederman50624c92013-09-23 21:19:49 -07007540 /* Report a network device has been unregistered */
7541 rtnl_lock();
7542 dev_net(dev)->dev_unreg_count--;
7543 __rtnl_unlock();
7544 wake_up(&netdev_unregistering_wq);
7545
Stephen Hemminger9093bbb2007-05-19 15:39:25 -07007546 /* Free network device */
7547 kobject_put(&dev->dev.kobj);
Linus Torvalds1da177e2005-04-16 15:20:36 -07007548 }
Linus Torvalds1da177e2005-04-16 15:20:36 -07007549}
7550
Jarod Wilson92566452016-02-01 18:51:04 -05007551/* Convert net_device_stats to rtnl_link_stats64. rtnl_link_stats64 has
7552 * all the same fields in the same order as net_device_stats, with only
7553 * the type differing, but rtnl_link_stats64 may have additional fields
7554 * at the end for newer counters.
Ben Hutchings3cfde792010-07-09 09:11:52 +00007555 */
Eric Dumazet77a1abf2012-03-05 04:50:09 +00007556void netdev_stats_to_stats64(struct rtnl_link_stats64 *stats64,
7557 const struct net_device_stats *netdev_stats)
Ben Hutchings3cfde792010-07-09 09:11:52 +00007558{
7559#if BITS_PER_LONG == 64
Jarod Wilson92566452016-02-01 18:51:04 -05007560 BUILD_BUG_ON(sizeof(*stats64) < sizeof(*netdev_stats));
Eric Dumazet77a1abf2012-03-05 04:50:09 +00007561 memcpy(stats64, netdev_stats, sizeof(*stats64));
Jarod Wilson92566452016-02-01 18:51:04 -05007562 /* zero out counters that only exist in rtnl_link_stats64 */
7563 memset((char *)stats64 + sizeof(*netdev_stats), 0,
7564 sizeof(*stats64) - sizeof(*netdev_stats));
Ben Hutchings3cfde792010-07-09 09:11:52 +00007565#else
Jarod Wilson92566452016-02-01 18:51:04 -05007566 size_t i, n = sizeof(*netdev_stats) / sizeof(unsigned long);
Ben Hutchings3cfde792010-07-09 09:11:52 +00007567 const unsigned long *src = (const unsigned long *)netdev_stats;
7568 u64 *dst = (u64 *)stats64;
7569
Jarod Wilson92566452016-02-01 18:51:04 -05007570 BUILD_BUG_ON(n > sizeof(*stats64) / sizeof(u64));
Ben Hutchings3cfde792010-07-09 09:11:52 +00007571 for (i = 0; i < n; i++)
7572 dst[i] = src[i];
Jarod Wilson92566452016-02-01 18:51:04 -05007573 /* zero out counters that only exist in rtnl_link_stats64 */
7574 memset((char *)stats64 + n * sizeof(u64), 0,
7575 sizeof(*stats64) - n * sizeof(u64));
Ben Hutchings3cfde792010-07-09 09:11:52 +00007576#endif
7577}
Eric Dumazet77a1abf2012-03-05 04:50:09 +00007578EXPORT_SYMBOL(netdev_stats_to_stats64);
Ben Hutchings3cfde792010-07-09 09:11:52 +00007579
Eric Dumazetd83345a2009-11-16 03:36:51 +00007580/**
Stephen Hemmingereeda3fd2008-11-19 21:40:23 -08007581 * dev_get_stats - get network device statistics
7582 * @dev: device to get statistics from
Eric Dumazet28172732010-07-07 14:58:56 -07007583 * @storage: place to store stats
Stephen Hemmingereeda3fd2008-11-19 21:40:23 -08007584 *
Ben Hutchingsd7753512010-07-09 09:12:41 +00007585 * Get network statistics from device. Return @storage.
7586 * The device driver may provide its own method by setting
7587 * dev->netdev_ops->get_stats64 or dev->netdev_ops->get_stats;
7588 * otherwise the internal statistics structure is used.
Stephen Hemmingereeda3fd2008-11-19 21:40:23 -08007589 */
Ben Hutchingsd7753512010-07-09 09:12:41 +00007590struct rtnl_link_stats64 *dev_get_stats(struct net_device *dev,
7591 struct rtnl_link_stats64 *storage)
Eric Dumazet7004bf22009-05-18 00:34:33 +00007592{
Stephen Hemmingereeda3fd2008-11-19 21:40:23 -08007593 const struct net_device_ops *ops = dev->netdev_ops;
7594
Eric Dumazet28172732010-07-07 14:58:56 -07007595 if (ops->ndo_get_stats64) {
7596 memset(storage, 0, sizeof(*storage));
Eric Dumazetcaf586e2010-09-30 21:06:55 +00007597 ops->ndo_get_stats64(dev, storage);
7598 } else if (ops->ndo_get_stats) {
Ben Hutchings3cfde792010-07-09 09:11:52 +00007599 netdev_stats_to_stats64(storage, ops->ndo_get_stats(dev));
Eric Dumazetcaf586e2010-09-30 21:06:55 +00007600 } else {
7601 netdev_stats_to_stats64(storage, &dev->stats);
Eric Dumazet28172732010-07-07 14:58:56 -07007602 }
Eric Dumazetcaf586e2010-09-30 21:06:55 +00007603 storage->rx_dropped += atomic_long_read(&dev->rx_dropped);
Eric Dumazet015f0682014-03-27 08:45:56 -07007604 storage->tx_dropped += atomic_long_read(&dev->tx_dropped);
Jarod Wilson6e7333d2016-02-01 18:51:05 -05007605 storage->rx_nohandler += atomic_long_read(&dev->rx_nohandler);
Eric Dumazet28172732010-07-07 14:58:56 -07007606 return storage;
Rusty Russellc45d2862007-03-28 14:29:08 -07007607}
Stephen Hemmingereeda3fd2008-11-19 21:40:23 -08007608EXPORT_SYMBOL(dev_get_stats);
Rusty Russellc45d2862007-03-28 14:29:08 -07007609
Eric Dumazet24824a02010-10-02 06:11:55 +00007610struct netdev_queue *dev_ingress_queue_create(struct net_device *dev)
David S. Millerdc2b4842008-07-08 17:18:23 -07007611{
Eric Dumazet24824a02010-10-02 06:11:55 +00007612 struct netdev_queue *queue = dev_ingress_queue(dev);
David S. Millerdc2b4842008-07-08 17:18:23 -07007613
Eric Dumazet24824a02010-10-02 06:11:55 +00007614#ifdef CONFIG_NET_CLS_ACT
7615 if (queue)
7616 return queue;
7617 queue = kzalloc(sizeof(*queue), GFP_KERNEL);
7618 if (!queue)
7619 return NULL;
7620 netdev_init_one_queue(dev, queue, NULL);
Eric Dumazet2ce1ee12015-02-04 13:37:44 -08007621 RCU_INIT_POINTER(queue->qdisc, &noop_qdisc);
Eric Dumazet24824a02010-10-02 06:11:55 +00007622 queue->qdisc_sleeping = &noop_qdisc;
7623 rcu_assign_pointer(dev->ingress_queue, queue);
7624#endif
7625 return queue;
David S. Millerbb949fb2008-07-08 16:55:56 -07007626}
7627
Eric Dumazet2c60db02012-09-16 09:17:26 +00007628static const struct ethtool_ops default_ethtool_ops;
7629
Stanislaw Gruszkad07d7502013-01-10 23:19:10 +00007630void netdev_set_default_ethtool_ops(struct net_device *dev,
7631 const struct ethtool_ops *ops)
7632{
7633 if (dev->ethtool_ops == &default_ethtool_ops)
7634 dev->ethtool_ops = ops;
7635}
7636EXPORT_SYMBOL_GPL(netdev_set_default_ethtool_ops);
7637
Eric Dumazet74d332c2013-10-30 13:10:44 -07007638void netdev_freemem(struct net_device *dev)
7639{
7640 char *addr = (char *)dev - dev->padded;
7641
WANG Cong4cb28972014-06-02 15:55:22 -07007642 kvfree(addr);
Eric Dumazet74d332c2013-10-30 13:10:44 -07007643}
7644
Linus Torvalds1da177e2005-04-16 15:20:36 -07007645/**
Tom Herbert36909ea2011-01-09 19:36:31 +00007646 * alloc_netdev_mqs - allocate network device
Tom Gundersenc835a672014-07-14 16:37:24 +02007647 * @sizeof_priv: size of private data to allocate space for
7648 * @name: device name format string
7649 * @name_assign_type: origin of device name
7650 * @setup: callback to initialize device
7651 * @txqs: the number of TX subqueues to allocate
7652 * @rxqs: the number of RX subqueues to allocate
Linus Torvalds1da177e2005-04-16 15:20:36 -07007653 *
7654 * Allocates a struct net_device with private data area for driver use
Li Zhong90e51ad2013-11-22 15:04:46 +08007655 * and performs basic initialization. Also allocates subqueue structs
Tom Herbert36909ea2011-01-09 19:36:31 +00007656 * for each queue on the device.
Linus Torvalds1da177e2005-04-16 15:20:36 -07007657 */
Tom Herbert36909ea2011-01-09 19:36:31 +00007658struct net_device *alloc_netdev_mqs(int sizeof_priv, const char *name,
Tom Gundersenc835a672014-07-14 16:37:24 +02007659 unsigned char name_assign_type,
Tom Herbert36909ea2011-01-09 19:36:31 +00007660 void (*setup)(struct net_device *),
7661 unsigned int txqs, unsigned int rxqs)
Linus Torvalds1da177e2005-04-16 15:20:36 -07007662{
Linus Torvalds1da177e2005-04-16 15:20:36 -07007663 struct net_device *dev;
Stephen Hemminger79439862008-07-21 13:28:44 -07007664 size_t alloc_size;
Eric Dumazet1ce8e7b2009-05-27 04:42:37 +00007665 struct net_device *p;
Linus Torvalds1da177e2005-04-16 15:20:36 -07007666
Stephen Hemmingerb6fe17d2006-08-29 17:06:13 -07007667 BUG_ON(strlen(name) >= sizeof(dev->name));
7668
Tom Herbert36909ea2011-01-09 19:36:31 +00007669 if (txqs < 1) {
Joe Perches7b6cd1c2012-02-01 10:54:43 +00007670 pr_err("alloc_netdev: Unable to allocate device with zero queues\n");
Tom Herbert55513fb2010-10-18 17:55:58 +00007671 return NULL;
7672 }
7673
Michael Daltona953be52014-01-16 22:23:28 -08007674#ifdef CONFIG_SYSFS
Tom Herbert36909ea2011-01-09 19:36:31 +00007675 if (rxqs < 1) {
Joe Perches7b6cd1c2012-02-01 10:54:43 +00007676 pr_err("alloc_netdev: Unable to allocate device with zero RX queues\n");
Tom Herbert36909ea2011-01-09 19:36:31 +00007677 return NULL;
7678 }
7679#endif
7680
David S. Millerfd2ea0a2008-07-17 01:56:23 -07007681 alloc_size = sizeof(struct net_device);
Alexey Dobriyand1643d22008-04-18 15:43:32 -07007682 if (sizeof_priv) {
7683 /* ensure 32-byte alignment of private area */
Eric Dumazet1ce8e7b2009-05-27 04:42:37 +00007684 alloc_size = ALIGN(alloc_size, NETDEV_ALIGN);
Alexey Dobriyand1643d22008-04-18 15:43:32 -07007685 alloc_size += sizeof_priv;
7686 }
7687 /* ensure 32-byte alignment of whole construct */
Eric Dumazet1ce8e7b2009-05-27 04:42:37 +00007688 alloc_size += NETDEV_ALIGN - 1;
Linus Torvalds1da177e2005-04-16 15:20:36 -07007689
Eric Dumazet74d332c2013-10-30 13:10:44 -07007690 p = kzalloc(alloc_size, GFP_KERNEL | __GFP_NOWARN | __GFP_REPEAT);
7691 if (!p)
7692 p = vzalloc(alloc_size);
Joe Perches62b59422013-02-04 16:48:16 +00007693 if (!p)
Linus Torvalds1da177e2005-04-16 15:20:36 -07007694 return NULL;
Linus Torvalds1da177e2005-04-16 15:20:36 -07007695
Eric Dumazet1ce8e7b2009-05-27 04:42:37 +00007696 dev = PTR_ALIGN(p, NETDEV_ALIGN);
Linus Torvalds1da177e2005-04-16 15:20:36 -07007697 dev->padded = (char *)dev - (char *)p;
Jiri Pirkoab9c73c2009-05-08 13:30:17 +00007698
Eric Dumazet29b44332010-10-11 10:22:12 +00007699 dev->pcpu_refcnt = alloc_percpu(int);
7700 if (!dev->pcpu_refcnt)
Eric Dumazet74d332c2013-10-30 13:10:44 -07007701 goto free_dev;
Jiri Pirkoab9c73c2009-05-08 13:30:17 +00007702
Linus Torvalds1da177e2005-04-16 15:20:36 -07007703 if (dev_addr_init(dev))
Eric Dumazet29b44332010-10-11 10:22:12 +00007704 goto free_pcpu;
Linus Torvalds1da177e2005-04-16 15:20:36 -07007705
Jiri Pirko22bedad32010-04-01 21:22:57 +00007706 dev_mc_init(dev);
Jiri Pirkoa748ee22010-04-01 21:22:09 +00007707 dev_uc_init(dev);
Jiri Pirkoccffad252009-05-22 23:22:17 +00007708
YOSHIFUJI Hideakic346dca2008-03-25 21:47:49 +09007709 dev_net_set(dev, &init_net);
Linus Torvalds1da177e2005-04-16 15:20:36 -07007710
Peter P Waskiewicz Jr82cc1a72008-03-21 03:43:19 -07007711 dev->gso_max_size = GSO_MAX_SIZE;
Ben Hutchings30b678d2012-07-30 15:57:00 +00007712 dev->gso_max_segs = GSO_MAX_SEGS;
Linus Torvalds1da177e2005-04-16 15:20:36 -07007713
Herbert Xud565b0a2008-12-15 23:38:52 -08007714 INIT_LIST_HEAD(&dev->napi_list);
Eric W. Biederman9fdce092009-10-30 14:51:13 +00007715 INIT_LIST_HEAD(&dev->unreg_list);
Eric W. Biederman5cde2822013-10-05 19:26:05 -07007716 INIT_LIST_HEAD(&dev->close_list);
Eric Dumazete014deb2009-11-17 05:59:21 +00007717 INIT_LIST_HEAD(&dev->link_watch_list);
Veaceslav Falico2f268f12013-09-25 09:20:07 +02007718 INIT_LIST_HEAD(&dev->adj_list.upper);
7719 INIT_LIST_HEAD(&dev->adj_list.lower);
Salam Noureddine7866a622015-01-27 11:35:48 -08007720 INIT_LIST_HEAD(&dev->ptype_all);
7721 INIT_LIST_HEAD(&dev->ptype_specific);
Jiri Kosina59cc1f62016-08-10 11:05:15 +02007722#ifdef CONFIG_NET_SCHED
7723 hash_init(dev->qdisc_hash);
7724#endif
Eric Dumazet02875872014-10-05 18:38:35 -07007725 dev->priv_flags = IFF_XMIT_DST_RELEASE | IFF_XMIT_DST_RELEASE_PERM;
Linus Torvalds1da177e2005-04-16 15:20:36 -07007726 setup(dev);
David S. Miller8d3bdbd2011-02-08 15:02:50 -08007727
Phil Suttera8131042016-02-17 15:37:43 +01007728 if (!dev->tx_queue_len) {
Phil Sutterf84bb1e2015-08-27 21:21:36 +02007729 dev->priv_flags |= IFF_NO_QUEUE;
Jesper Dangaard Brouer11597082016-11-03 14:56:06 +01007730 dev->tx_queue_len = DEFAULT_TX_QUEUE_LEN;
Phil Suttera8131042016-02-17 15:37:43 +01007731 }
Phil Sutter906470c2015-08-18 10:30:48 +02007732
David S. Miller8d3bdbd2011-02-08 15:02:50 -08007733 dev->num_tx_queues = txqs;
7734 dev->real_num_tx_queues = txqs;
7735 if (netif_alloc_netdev_queues(dev))
7736 goto free_all;
7737
Michael Daltona953be52014-01-16 22:23:28 -08007738#ifdef CONFIG_SYSFS
David S. Miller8d3bdbd2011-02-08 15:02:50 -08007739 dev->num_rx_queues = rxqs;
7740 dev->real_num_rx_queues = rxqs;
7741 if (netif_alloc_rx_queues(dev))
7742 goto free_all;
7743#endif
7744
Linus Torvalds1da177e2005-04-16 15:20:36 -07007745 strcpy(dev->name, name);
Tom Gundersenc835a672014-07-14 16:37:24 +02007746 dev->name_assign_type = name_assign_type;
Vlad Dogarucbda10f2011-01-13 23:38:30 +00007747 dev->group = INIT_NETDEV_GROUP;
Eric Dumazet2c60db02012-09-16 09:17:26 +00007748 if (!dev->ethtool_ops)
7749 dev->ethtool_ops = &default_ethtool_ops;
Pablo Neirae687ad62015-05-13 18:19:38 +02007750
7751 nf_hook_ingress_init(dev);
7752
Linus Torvalds1da177e2005-04-16 15:20:36 -07007753 return dev;
Jiri Pirkoab9c73c2009-05-08 13:30:17 +00007754
David S. Miller8d3bdbd2011-02-08 15:02:50 -08007755free_all:
7756 free_netdev(dev);
7757 return NULL;
7758
Eric Dumazet29b44332010-10-11 10:22:12 +00007759free_pcpu:
7760 free_percpu(dev->pcpu_refcnt);
Eric Dumazet74d332c2013-10-30 13:10:44 -07007761free_dev:
7762 netdev_freemem(dev);
Jiri Pirkoab9c73c2009-05-08 13:30:17 +00007763 return NULL;
Linus Torvalds1da177e2005-04-16 15:20:36 -07007764}
Tom Herbert36909ea2011-01-09 19:36:31 +00007765EXPORT_SYMBOL(alloc_netdev_mqs);
Linus Torvalds1da177e2005-04-16 15:20:36 -07007766
7767/**
7768 * free_netdev - free network device
7769 * @dev: device
7770 *
YOSHIFUJI Hideaki4ec93ed2007-02-09 23:24:36 +09007771 * This function does the last stage of destroying an allocated device
7772 * interface. The reference to the device object is released.
Linus Torvalds1da177e2005-04-16 15:20:36 -07007773 * If this is the last reference then it will be freed.
Eric Dumazet93d05d42015-11-18 06:31:03 -08007774 * Must be called in process context.
Linus Torvalds1da177e2005-04-16 15:20:36 -07007775 */
7776void free_netdev(struct net_device *dev)
7777{
Herbert Xud565b0a2008-12-15 23:38:52 -08007778 struct napi_struct *p, *n;
7779
Eric Dumazet93d05d42015-11-18 06:31:03 -08007780 might_sleep();
Eric Dumazet60877a32013-06-20 01:15:51 -07007781 netif_free_tx_queues(dev);
Michael Daltona953be52014-01-16 22:23:28 -08007782#ifdef CONFIG_SYSFS
Pankaj Gupta10595902015-01-12 11:41:28 +05307783 kvfree(dev->_rx);
Tom Herbertfe822242010-11-09 10:47:38 +00007784#endif
David S. Millere8a04642008-07-17 00:34:19 -07007785
Eric Dumazet33d480c2011-08-11 19:30:52 +00007786 kfree(rcu_dereference_protected(dev->ingress_queue, 1));
Eric Dumazet24824a02010-10-02 06:11:55 +00007787
Jiri Pirkof001fde2009-05-05 02:48:28 +00007788 /* Flush device addresses */
7789 dev_addr_flush(dev);
7790
Herbert Xud565b0a2008-12-15 23:38:52 -08007791 list_for_each_entry_safe(p, n, &dev->napi_list, dev_list)
7792 netif_napi_del(p);
7793
Eric Dumazet29b44332010-10-11 10:22:12 +00007794 free_percpu(dev->pcpu_refcnt);
7795 dev->pcpu_refcnt = NULL;
7796
Stephen Hemminger3041a062006-05-26 13:25:24 -07007797 /* Compatibility with error handling in drivers */
Linus Torvalds1da177e2005-04-16 15:20:36 -07007798 if (dev->reg_state == NETREG_UNINITIALIZED) {
Eric Dumazet74d332c2013-10-30 13:10:44 -07007799 netdev_freemem(dev);
Linus Torvalds1da177e2005-04-16 15:20:36 -07007800 return;
7801 }
7802
7803 BUG_ON(dev->reg_state != NETREG_UNREGISTERED);
7804 dev->reg_state = NETREG_RELEASED;
7805
Greg Kroah-Hartman43cb76d2002-04-09 12:14:34 -07007806 /* will free via device release */
7807 put_device(&dev->dev);
Linus Torvalds1da177e2005-04-16 15:20:36 -07007808}
Eric Dumazetd1b19df2009-09-03 01:29:39 -07007809EXPORT_SYMBOL(free_netdev);
YOSHIFUJI Hideaki4ec93ed2007-02-09 23:24:36 +09007810
Stephen Hemmingerf0db2752008-09-30 02:23:58 -07007811/**
7812 * synchronize_net - Synchronize with packet receive processing
7813 *
7814 * Wait for packets currently being received to be done.
7815 * Does not block later packets from starting.
7816 */
YOSHIFUJI Hideaki4ec93ed2007-02-09 23:24:36 +09007817void synchronize_net(void)
Linus Torvalds1da177e2005-04-16 15:20:36 -07007818{
7819 might_sleep();
Eric Dumazetbe3fc412011-05-23 23:07:32 +00007820 if (rtnl_is_locked())
7821 synchronize_rcu_expedited();
7822 else
7823 synchronize_rcu();
Linus Torvalds1da177e2005-04-16 15:20:36 -07007824}
Eric Dumazetd1b19df2009-09-03 01:29:39 -07007825EXPORT_SYMBOL(synchronize_net);
Linus Torvalds1da177e2005-04-16 15:20:36 -07007826
7827/**
Eric Dumazet44a08732009-10-27 07:03:04 +00007828 * unregister_netdevice_queue - remove device from the kernel
Linus Torvalds1da177e2005-04-16 15:20:36 -07007829 * @dev: device
Eric Dumazet44a08732009-10-27 07:03:04 +00007830 * @head: list
Jaswinder Singh Rajput6ebfbc02009-11-22 20:43:13 -08007831 *
Linus Torvalds1da177e2005-04-16 15:20:36 -07007832 * This function shuts down a device interface and removes it
Wang Chend59b54b2007-12-11 02:28:03 -08007833 * from the kernel tables.
Eric Dumazet44a08732009-10-27 07:03:04 +00007834 * If head not NULL, device is queued to be unregistered later.
Linus Torvalds1da177e2005-04-16 15:20:36 -07007835 *
7836 * Callers must hold the rtnl semaphore. You may want
7837 * unregister_netdev() instead of this.
7838 */
7839
Eric Dumazet44a08732009-10-27 07:03:04 +00007840void unregister_netdevice_queue(struct net_device *dev, struct list_head *head)
Linus Torvalds1da177e2005-04-16 15:20:36 -07007841{
Herbert Xua6620712007-12-12 19:21:56 -08007842 ASSERT_RTNL();
7843
Eric Dumazet44a08732009-10-27 07:03:04 +00007844 if (head) {
Eric W. Biederman9fdce092009-10-30 14:51:13 +00007845 list_move_tail(&dev->unreg_list, head);
Eric Dumazet44a08732009-10-27 07:03:04 +00007846 } else {
7847 rollback_registered(dev);
7848 /* Finish processing unregister after unlock */
7849 net_set_todo(dev);
7850 }
Linus Torvalds1da177e2005-04-16 15:20:36 -07007851}
Eric Dumazet44a08732009-10-27 07:03:04 +00007852EXPORT_SYMBOL(unregister_netdevice_queue);
Linus Torvalds1da177e2005-04-16 15:20:36 -07007853
7854/**
Eric Dumazet9b5e3832009-10-27 07:04:19 +00007855 * unregister_netdevice_many - unregister many devices
7856 * @head: list of devices
Eric Dumazet87757a92014-06-06 06:44:03 -07007857 *
7858 * Note: As most callers use a stack allocated list_head,
7859 * we force a list_del() to make sure stack wont be corrupted later.
Eric Dumazet9b5e3832009-10-27 07:04:19 +00007860 */
7861void unregister_netdevice_many(struct list_head *head)
7862{
7863 struct net_device *dev;
7864
7865 if (!list_empty(head)) {
7866 rollback_registered_many(head);
7867 list_for_each_entry(dev, head, unreg_list)
7868 net_set_todo(dev);
Eric Dumazet87757a92014-06-06 06:44:03 -07007869 list_del(head);
Eric Dumazet9b5e3832009-10-27 07:04:19 +00007870 }
7871}
Eric Dumazet63c80992009-10-27 07:06:49 +00007872EXPORT_SYMBOL(unregister_netdevice_many);
Eric Dumazet9b5e3832009-10-27 07:04:19 +00007873
7874/**
Linus Torvalds1da177e2005-04-16 15:20:36 -07007875 * unregister_netdev - remove device from the kernel
7876 * @dev: device
7877 *
7878 * This function shuts down a device interface and removes it
Wang Chend59b54b2007-12-11 02:28:03 -08007879 * from the kernel tables.
Linus Torvalds1da177e2005-04-16 15:20:36 -07007880 *
7881 * This is just a wrapper for unregister_netdevice that takes
7882 * the rtnl semaphore. In general you want to use this and not
7883 * unregister_netdevice.
7884 */
7885void unregister_netdev(struct net_device *dev)
7886{
7887 rtnl_lock();
7888 unregister_netdevice(dev);
7889 rtnl_unlock();
7890}
Linus Torvalds1da177e2005-04-16 15:20:36 -07007891EXPORT_SYMBOL(unregister_netdev);
7892
Eric W. Biedermance286d32007-09-12 13:53:49 +02007893/**
7894 * dev_change_net_namespace - move device to different nethost namespace
7895 * @dev: device
7896 * @net: network namespace
7897 * @pat: If not NULL name pattern to try if the current device name
7898 * is already taken in the destination network namespace.
7899 *
7900 * This function shuts down a device interface and moves it
7901 * to a new network namespace. On success 0 is returned, on
7902 * a failure a netagive errno code is returned.
7903 *
7904 * Callers must hold the rtnl semaphore.
7905 */
7906
7907int dev_change_net_namespace(struct net_device *dev, struct net *net, const char *pat)
7908{
Eric W. Biedermance286d32007-09-12 13:53:49 +02007909 int err;
7910
7911 ASSERT_RTNL();
7912
7913 /* Don't allow namespace local devices to be moved. */
7914 err = -EINVAL;
7915 if (dev->features & NETIF_F_NETNS_LOCAL)
7916 goto out;
7917
7918 /* Ensure the device has been registrered */
Eric W. Biedermance286d32007-09-12 13:53:49 +02007919 if (dev->reg_state != NETREG_REGISTERED)
7920 goto out;
7921
7922 /* Get out if there is nothing todo */
7923 err = 0;
YOSHIFUJI Hideaki878628f2008-03-26 03:57:35 +09007924 if (net_eq(dev_net(dev), net))
Eric W. Biedermance286d32007-09-12 13:53:49 +02007925 goto out;
7926
7927 /* Pick the destination device name, and ensure
7928 * we can use it in the destination network namespace.
7929 */
7930 err = -EEXIST;
Octavian Purdilad9031022009-11-18 02:36:59 +00007931 if (__dev_get_by_name(net, dev->name)) {
Eric W. Biedermance286d32007-09-12 13:53:49 +02007932 /* We get here if we can't use the current device name */
7933 if (!pat)
7934 goto out;
Gao feng828de4f2012-09-13 20:58:27 +00007935 if (dev_get_valid_name(net, dev, pat) < 0)
Eric W. Biedermance286d32007-09-12 13:53:49 +02007936 goto out;
7937 }
7938
7939 /*
7940 * And now a mini version of register_netdevice unregister_netdevice.
7941 */
7942
7943 /* If device is running close it first. */
Pavel Emelyanov9b772652007-10-10 02:49:09 -07007944 dev_close(dev);
Eric W. Biedermance286d32007-09-12 13:53:49 +02007945
7946 /* And unlink it from device chain */
7947 err = -ENODEV;
7948 unlist_netdevice(dev);
7949
7950 synchronize_net();
7951
7952 /* Shutdown queueing discipline. */
7953 dev_shutdown(dev);
7954
7955 /* Notify protocols, that we are about to destroy
7956 this device. They should clean all the things.
David Lamparter3b27e102010-09-17 03:22:19 +00007957
7958 Note that dev->reg_state stays at NETREG_REGISTERED.
7959 This is wanted because this way 8021q and macvlan know
7960 the device is just moving and can keep their slaves up.
Eric W. Biedermance286d32007-09-12 13:53:49 +02007961 */
7962 call_netdevice_notifiers(NETDEV_UNREGISTER, dev);
Gao feng6549dd42012-08-23 15:36:55 +00007963 rcu_barrier();
7964 call_netdevice_notifiers(NETDEV_UNREGISTER_FINAL, dev);
Alexei Starovoitov7f294052013-10-23 16:02:42 -07007965 rtmsg_ifinfo(RTM_DELLINK, dev, ~0U, GFP_KERNEL);
Eric W. Biedermance286d32007-09-12 13:53:49 +02007966
7967 /*
7968 * Flush the unicast and multicast chains
7969 */
Jiri Pirkoa748ee22010-04-01 21:22:09 +00007970 dev_uc_flush(dev);
Jiri Pirko22bedad32010-04-01 21:22:57 +00007971 dev_mc_flush(dev);
Eric W. Biedermance286d32007-09-12 13:53:49 +02007972
Serge Hallyn4e66ae22012-12-03 16:17:12 +00007973 /* Send a netdev-removed uevent to the old namespace */
7974 kobject_uevent(&dev->dev.kobj, KOBJ_REMOVE);
Alexander Y. Fomichev4c754312014-08-25 16:26:45 +04007975 netdev_adjacent_del_links(dev);
Serge Hallyn4e66ae22012-12-03 16:17:12 +00007976
Eric W. Biedermance286d32007-09-12 13:53:49 +02007977 /* Actually switch the network namespace */
YOSHIFUJI Hideakic346dca2008-03-25 21:47:49 +09007978 dev_net_set(dev, net);
Eric W. Biedermance286d32007-09-12 13:53:49 +02007979
Eric W. Biedermance286d32007-09-12 13:53:49 +02007980 /* If there is an ifindex conflict assign a new one */
Nicolas Dichtel7a66bbc2015-04-02 17:07:09 +02007981 if (__dev_get_by_index(net, dev->ifindex))
Eric W. Biedermance286d32007-09-12 13:53:49 +02007982 dev->ifindex = dev_new_index(net);
Eric W. Biedermance286d32007-09-12 13:53:49 +02007983
Serge Hallyn4e66ae22012-12-03 16:17:12 +00007984 /* Send a netdev-add uevent to the new namespace */
7985 kobject_uevent(&dev->dev.kobj, KOBJ_ADD);
Alexander Y. Fomichev4c754312014-08-25 16:26:45 +04007986 netdev_adjacent_add_links(dev);
Serge Hallyn4e66ae22012-12-03 16:17:12 +00007987
Eric W. Biederman8b41d182007-09-26 22:02:53 -07007988 /* Fixup kobjects */
Eric W. Biedermana1b3f592010-05-04 17:36:49 -07007989 err = device_rename(&dev->dev, dev->name);
Eric W. Biederman8b41d182007-09-26 22:02:53 -07007990 WARN_ON(err);
Eric W. Biedermance286d32007-09-12 13:53:49 +02007991
7992 /* Add the device back in the hashes */
7993 list_netdevice(dev);
7994
7995 /* Notify protocols, that a new device appeared. */
7996 call_netdevice_notifiers(NETDEV_REGISTER, dev);
7997
Eric W. Biedermand90a9092009-12-12 22:11:15 +00007998 /*
7999 * Prevent userspace races by waiting until the network
8000 * device is fully setup before sending notifications.
8001 */
Alexei Starovoitov7f294052013-10-23 16:02:42 -07008002 rtmsg_ifinfo(RTM_NEWLINK, dev, ~0U, GFP_KERNEL);
Eric W. Biedermand90a9092009-12-12 22:11:15 +00008003
Eric W. Biedermance286d32007-09-12 13:53:49 +02008004 synchronize_net();
8005 err = 0;
8006out:
8007 return err;
8008}
Johannes Berg463d0182009-07-14 00:33:35 +02008009EXPORT_SYMBOL_GPL(dev_change_net_namespace);
Eric W. Biedermance286d32007-09-12 13:53:49 +02008010
Sebastian Andrzej Siewiorf0bf90d2016-11-03 15:50:04 +01008011static int dev_cpu_dead(unsigned int oldcpu)
Linus Torvalds1da177e2005-04-16 15:20:36 -07008012{
8013 struct sk_buff **list_skb;
Linus Torvalds1da177e2005-04-16 15:20:36 -07008014 struct sk_buff *skb;
Sebastian Andrzej Siewiorf0bf90d2016-11-03 15:50:04 +01008015 unsigned int cpu;
Linus Torvalds1da177e2005-04-16 15:20:36 -07008016 struct softnet_data *sd, *oldsd;
8017
Linus Torvalds1da177e2005-04-16 15:20:36 -07008018 local_irq_disable();
8019 cpu = smp_processor_id();
8020 sd = &per_cpu(softnet_data, cpu);
8021 oldsd = &per_cpu(softnet_data, oldcpu);
8022
8023 /* Find end of our completion_queue. */
8024 list_skb = &sd->completion_queue;
8025 while (*list_skb)
8026 list_skb = &(*list_skb)->next;
8027 /* Append completion queue from offline CPU. */
8028 *list_skb = oldsd->completion_queue;
8029 oldsd->completion_queue = NULL;
8030
Linus Torvalds1da177e2005-04-16 15:20:36 -07008031 /* Append output queue from offline CPU. */
Changli Gaoa9cbd582010-04-26 23:06:24 +00008032 if (oldsd->output_queue) {
8033 *sd->output_queue_tailp = oldsd->output_queue;
8034 sd->output_queue_tailp = oldsd->output_queue_tailp;
8035 oldsd->output_queue = NULL;
8036 oldsd->output_queue_tailp = &oldsd->output_queue;
8037 }
Eric Dumazetac64da02015-01-15 17:04:22 -08008038 /* Append NAPI poll list from offline CPU, with one exception :
8039 * process_backlog() must be called by cpu owning percpu backlog.
8040 * We properly handle process_queue & input_pkt_queue later.
8041 */
8042 while (!list_empty(&oldsd->poll_list)) {
8043 struct napi_struct *napi = list_first_entry(&oldsd->poll_list,
8044 struct napi_struct,
8045 poll_list);
8046
8047 list_del_init(&napi->poll_list);
8048 if (napi->poll == process_backlog)
8049 napi->state = 0;
8050 else
8051 ____napi_schedule(sd, napi);
Heiko Carstens264524d2011-06-06 20:50:03 +00008052 }
Linus Torvalds1da177e2005-04-16 15:20:36 -07008053
8054 raise_softirq_irqoff(NET_TX_SOFTIRQ);
8055 local_irq_enable();
8056
8057 /* Process offline CPU's input_pkt_queue */
Tom Herbert76cc8b12010-05-20 18:37:59 +00008058 while ((skb = __skb_dequeue(&oldsd->process_queue))) {
Eric Dumazet91e83132015-02-05 14:58:14 -08008059 netif_rx_ni(skb);
Tom Herbert76cc8b12010-05-20 18:37:59 +00008060 input_queue_head_incr(oldsd);
8061 }
Eric Dumazetac64da02015-01-15 17:04:22 -08008062 while ((skb = skb_dequeue(&oldsd->input_pkt_queue))) {
Eric Dumazet91e83132015-02-05 14:58:14 -08008063 netif_rx_ni(skb);
Tom Herbert76cc8b12010-05-20 18:37:59 +00008064 input_queue_head_incr(oldsd);
Tom Herbertfec5e652010-04-16 16:01:27 -07008065 }
Linus Torvalds1da177e2005-04-16 15:20:36 -07008066
Sebastian Andrzej Siewiorf0bf90d2016-11-03 15:50:04 +01008067 return 0;
Linus Torvalds1da177e2005-04-16 15:20:36 -07008068}
Linus Torvalds1da177e2005-04-16 15:20:36 -07008069
Herbert Xu7f353bf2007-08-10 15:47:58 -07008070/**
Herbert Xub63365a2008-10-23 01:11:29 -07008071 * netdev_increment_features - increment feature set by one
8072 * @all: current feature set
8073 * @one: new feature set
8074 * @mask: mask feature set
Herbert Xu7f353bf2007-08-10 15:47:58 -07008075 *
8076 * Computes a new feature set after adding a device with feature set
Herbert Xub63365a2008-10-23 01:11:29 -07008077 * @one to the master device with current feature set @all. Will not
8078 * enable anything that is off in @mask. Returns the new feature set.
Herbert Xu7f353bf2007-08-10 15:47:58 -07008079 */
Michał Mirosławc8f44af2011-11-15 15:29:55 +00008080netdev_features_t netdev_increment_features(netdev_features_t all,
8081 netdev_features_t one, netdev_features_t mask)
Herbert Xu7f353bf2007-08-10 15:47:58 -07008082{
Tom Herbertc8cd0982015-12-14 11:19:44 -08008083 if (mask & NETIF_F_HW_CSUM)
Tom Herberta1882222015-12-14 11:19:43 -08008084 mask |= NETIF_F_CSUM_MASK;
Michał Mirosław1742f182011-04-22 06:31:16 +00008085 mask |= NETIF_F_VLAN_CHALLENGED;
8086
Tom Herberta1882222015-12-14 11:19:43 -08008087 all |= one & (NETIF_F_ONE_FOR_ALL | NETIF_F_CSUM_MASK) & mask;
Michał Mirosław1742f182011-04-22 06:31:16 +00008088 all &= one | ~NETIF_F_ALL_FOR_ALL;
8089
Michał Mirosław1742f182011-04-22 06:31:16 +00008090 /* If one device supports hw checksumming, set for all. */
Tom Herbertc8cd0982015-12-14 11:19:44 -08008091 if (all & NETIF_F_HW_CSUM)
8092 all &= ~(NETIF_F_CSUM_MASK & ~NETIF_F_HW_CSUM);
Herbert Xu7f353bf2007-08-10 15:47:58 -07008093
8094 return all;
8095}
Herbert Xub63365a2008-10-23 01:11:29 -07008096EXPORT_SYMBOL(netdev_increment_features);
Herbert Xu7f353bf2007-08-10 15:47:58 -07008097
Baruch Siach430f03c2013-06-02 20:43:55 +00008098static struct hlist_head * __net_init netdev_create_hash(void)
Pavel Emelyanov30d97d32007-09-16 15:40:33 -07008099{
8100 int i;
8101 struct hlist_head *hash;
8102
8103 hash = kmalloc(sizeof(*hash) * NETDEV_HASHENTRIES, GFP_KERNEL);
8104 if (hash != NULL)
8105 for (i = 0; i < NETDEV_HASHENTRIES; i++)
8106 INIT_HLIST_HEAD(&hash[i]);
8107
8108 return hash;
8109}
8110
Eric W. Biederman881d9662007-09-17 11:56:21 -07008111/* Initialize per network namespace state */
Pavel Emelyanov46650792007-10-08 20:38:39 -07008112static int __net_init netdev_init(struct net *net)
Eric W. Biederman881d9662007-09-17 11:56:21 -07008113{
Rustad, Mark D734b6542012-07-18 09:06:07 +00008114 if (net != &init_net)
8115 INIT_LIST_HEAD(&net->dev_base_head);
Eric W. Biederman881d9662007-09-17 11:56:21 -07008116
Pavel Emelyanov30d97d32007-09-16 15:40:33 -07008117 net->dev_name_head = netdev_create_hash();
8118 if (net->dev_name_head == NULL)
8119 goto err_name;
Eric W. Biederman881d9662007-09-17 11:56:21 -07008120
Pavel Emelyanov30d97d32007-09-16 15:40:33 -07008121 net->dev_index_head = netdev_create_hash();
8122 if (net->dev_index_head == NULL)
8123 goto err_idx;
Eric W. Biederman881d9662007-09-17 11:56:21 -07008124
8125 return 0;
Pavel Emelyanov30d97d32007-09-16 15:40:33 -07008126
8127err_idx:
8128 kfree(net->dev_name_head);
8129err_name:
8130 return -ENOMEM;
Eric W. Biederman881d9662007-09-17 11:56:21 -07008131}
8132
Stephen Hemmingerf0db2752008-09-30 02:23:58 -07008133/**
8134 * netdev_drivername - network driver for the device
8135 * @dev: network device
Stephen Hemmingerf0db2752008-09-30 02:23:58 -07008136 *
8137 * Determine network driver for device.
8138 */
David S. Miller3019de12011-06-06 16:41:33 -07008139const char *netdev_drivername(const struct net_device *dev)
Arjan van de Ven6579e572008-07-21 13:31:48 -07008140{
Stephen Hemmingercf04a4c72008-09-30 02:22:14 -07008141 const struct device_driver *driver;
8142 const struct device *parent;
David S. Miller3019de12011-06-06 16:41:33 -07008143 const char *empty = "";
Arjan van de Ven6579e572008-07-21 13:31:48 -07008144
8145 parent = dev->dev.parent;
Arjan van de Ven6579e572008-07-21 13:31:48 -07008146 if (!parent)
David S. Miller3019de12011-06-06 16:41:33 -07008147 return empty;
Arjan van de Ven6579e572008-07-21 13:31:48 -07008148
8149 driver = parent->driver;
8150 if (driver && driver->name)
David S. Miller3019de12011-06-06 16:41:33 -07008151 return driver->name;
8152 return empty;
Arjan van de Ven6579e572008-07-21 13:31:48 -07008153}
8154
Joe Perches6ea754e2014-09-22 11:10:50 -07008155static void __netdev_printk(const char *level, const struct net_device *dev,
8156 struct va_format *vaf)
Joe Perches256df2f2010-06-27 01:02:35 +00008157{
Joe Perchesb004ff42012-09-12 20:12:19 -07008158 if (dev && dev->dev.parent) {
Joe Perches6ea754e2014-09-22 11:10:50 -07008159 dev_printk_emit(level[1] - '0',
8160 dev->dev.parent,
8161 "%s %s %s%s: %pV",
8162 dev_driver_string(dev->dev.parent),
8163 dev_name(dev->dev.parent),
8164 netdev_name(dev), netdev_reg_state(dev),
8165 vaf);
Joe Perchesb004ff42012-09-12 20:12:19 -07008166 } else if (dev) {
Joe Perches6ea754e2014-09-22 11:10:50 -07008167 printk("%s%s%s: %pV",
8168 level, netdev_name(dev), netdev_reg_state(dev), vaf);
Joe Perchesb004ff42012-09-12 20:12:19 -07008169 } else {
Joe Perches6ea754e2014-09-22 11:10:50 -07008170 printk("%s(NULL net_device): %pV", level, vaf);
Joe Perchesb004ff42012-09-12 20:12:19 -07008171 }
Joe Perches256df2f2010-06-27 01:02:35 +00008172}
8173
Joe Perches6ea754e2014-09-22 11:10:50 -07008174void netdev_printk(const char *level, const struct net_device *dev,
8175 const char *format, ...)
Joe Perches256df2f2010-06-27 01:02:35 +00008176{
8177 struct va_format vaf;
8178 va_list args;
Joe Perches256df2f2010-06-27 01:02:35 +00008179
8180 va_start(args, format);
8181
8182 vaf.fmt = format;
8183 vaf.va = &args;
8184
Joe Perches6ea754e2014-09-22 11:10:50 -07008185 __netdev_printk(level, dev, &vaf);
Joe Perchesb004ff42012-09-12 20:12:19 -07008186
Joe Perches256df2f2010-06-27 01:02:35 +00008187 va_end(args);
Joe Perches256df2f2010-06-27 01:02:35 +00008188}
8189EXPORT_SYMBOL(netdev_printk);
8190
8191#define define_netdev_printk_level(func, level) \
Joe Perches6ea754e2014-09-22 11:10:50 -07008192void func(const struct net_device *dev, const char *fmt, ...) \
Joe Perches256df2f2010-06-27 01:02:35 +00008193{ \
Joe Perches256df2f2010-06-27 01:02:35 +00008194 struct va_format vaf; \
8195 va_list args; \
8196 \
8197 va_start(args, fmt); \
8198 \
8199 vaf.fmt = fmt; \
8200 vaf.va = &args; \
8201 \
Joe Perches6ea754e2014-09-22 11:10:50 -07008202 __netdev_printk(level, dev, &vaf); \
Joe Perchesb004ff42012-09-12 20:12:19 -07008203 \
Joe Perches256df2f2010-06-27 01:02:35 +00008204 va_end(args); \
Joe Perches256df2f2010-06-27 01:02:35 +00008205} \
8206EXPORT_SYMBOL(func);
8207
8208define_netdev_printk_level(netdev_emerg, KERN_EMERG);
8209define_netdev_printk_level(netdev_alert, KERN_ALERT);
8210define_netdev_printk_level(netdev_crit, KERN_CRIT);
8211define_netdev_printk_level(netdev_err, KERN_ERR);
8212define_netdev_printk_level(netdev_warn, KERN_WARNING);
8213define_netdev_printk_level(netdev_notice, KERN_NOTICE);
8214define_netdev_printk_level(netdev_info, KERN_INFO);
8215
Pavel Emelyanov46650792007-10-08 20:38:39 -07008216static void __net_exit netdev_exit(struct net *net)
Eric W. Biederman881d9662007-09-17 11:56:21 -07008217{
8218 kfree(net->dev_name_head);
8219 kfree(net->dev_index_head);
8220}
8221
Denis V. Lunev022cbae2007-11-13 03:23:50 -08008222static struct pernet_operations __net_initdata netdev_net_ops = {
Eric W. Biederman881d9662007-09-17 11:56:21 -07008223 .init = netdev_init,
8224 .exit = netdev_exit,
8225};
8226
Pavel Emelyanov46650792007-10-08 20:38:39 -07008227static void __net_exit default_device_exit(struct net *net)
Eric W. Biedermance286d32007-09-12 13:53:49 +02008228{
Eric W. Biedermane008b5f2009-11-29 22:25:30 +00008229 struct net_device *dev, *aux;
Eric W. Biedermance286d32007-09-12 13:53:49 +02008230 /*
Eric W. Biedermane008b5f2009-11-29 22:25:30 +00008231 * Push all migratable network devices back to the
Eric W. Biedermance286d32007-09-12 13:53:49 +02008232 * initial network namespace
8233 */
8234 rtnl_lock();
Eric W. Biedermane008b5f2009-11-29 22:25:30 +00008235 for_each_netdev_safe(net, dev, aux) {
Eric W. Biedermance286d32007-09-12 13:53:49 +02008236 int err;
Pavel Emelyanovaca51392008-05-08 01:24:25 -07008237 char fb_name[IFNAMSIZ];
Eric W. Biedermance286d32007-09-12 13:53:49 +02008238
8239 /* Ignore unmoveable devices (i.e. loopback) */
8240 if (dev->features & NETIF_F_NETNS_LOCAL)
8241 continue;
8242
Eric W. Biedermane008b5f2009-11-29 22:25:30 +00008243 /* Leave virtual devices for the generic cleanup */
8244 if (dev->rtnl_link_ops)
8245 continue;
Eric W. Biedermand0c082c2008-11-05 15:59:38 -08008246
Lucas De Marchi25985ed2011-03-30 22:57:33 -03008247 /* Push remaining network devices to init_net */
Pavel Emelyanovaca51392008-05-08 01:24:25 -07008248 snprintf(fb_name, IFNAMSIZ, "dev%d", dev->ifindex);
8249 err = dev_change_net_namespace(dev, &init_net, fb_name);
Eric W. Biedermance286d32007-09-12 13:53:49 +02008250 if (err) {
Joe Perches7b6cd1c2012-02-01 10:54:43 +00008251 pr_emerg("%s: failed to move %s to init_net: %d\n",
8252 __func__, dev->name, err);
Pavel Emelyanovaca51392008-05-08 01:24:25 -07008253 BUG();
Eric W. Biedermance286d32007-09-12 13:53:49 +02008254 }
8255 }
8256 rtnl_unlock();
8257}
8258
Eric W. Biederman50624c92013-09-23 21:19:49 -07008259static void __net_exit rtnl_lock_unregistering(struct list_head *net_list)
8260{
8261 /* Return with the rtnl_lock held when there are no network
8262 * devices unregistering in any network namespace in net_list.
8263 */
8264 struct net *net;
8265 bool unregistering;
Peter Zijlstraff960a72014-10-29 17:04:56 +01008266 DEFINE_WAIT_FUNC(wait, woken_wake_function);
Eric W. Biederman50624c92013-09-23 21:19:49 -07008267
Peter Zijlstraff960a72014-10-29 17:04:56 +01008268 add_wait_queue(&netdev_unregistering_wq, &wait);
Eric W. Biederman50624c92013-09-23 21:19:49 -07008269 for (;;) {
Eric W. Biederman50624c92013-09-23 21:19:49 -07008270 unregistering = false;
8271 rtnl_lock();
8272 list_for_each_entry(net, net_list, exit_list) {
8273 if (net->dev_unreg_count > 0) {
8274 unregistering = true;
8275 break;
8276 }
8277 }
8278 if (!unregistering)
8279 break;
8280 __rtnl_unlock();
Peter Zijlstraff960a72014-10-29 17:04:56 +01008281
8282 wait_woken(&wait, TASK_UNINTERRUPTIBLE, MAX_SCHEDULE_TIMEOUT);
Eric W. Biederman50624c92013-09-23 21:19:49 -07008283 }
Peter Zijlstraff960a72014-10-29 17:04:56 +01008284 remove_wait_queue(&netdev_unregistering_wq, &wait);
Eric W. Biederman50624c92013-09-23 21:19:49 -07008285}
8286
Eric W. Biederman04dc7f6b2009-12-03 02:29:04 +00008287static void __net_exit default_device_exit_batch(struct list_head *net_list)
8288{
8289 /* At exit all network devices most be removed from a network
Uwe Kleine-Königb5950762010-11-01 15:38:34 -04008290 * namespace. Do this in the reverse order of registration.
Eric W. Biederman04dc7f6b2009-12-03 02:29:04 +00008291 * Do this across as many network namespaces as possible to
8292 * improve batching efficiency.
8293 */
8294 struct net_device *dev;
8295 struct net *net;
8296 LIST_HEAD(dev_kill_list);
8297
Eric W. Biederman50624c92013-09-23 21:19:49 -07008298 /* To prevent network device cleanup code from dereferencing
8299 * loopback devices or network devices that have been freed
8300 * wait here for all pending unregistrations to complete,
8301 * before unregistring the loopback device and allowing the
8302 * network namespace be freed.
8303 *
8304 * The netdev todo list containing all network devices
8305 * unregistrations that happen in default_device_exit_batch
8306 * will run in the rtnl_unlock() at the end of
8307 * default_device_exit_batch.
8308 */
8309 rtnl_lock_unregistering(net_list);
Eric W. Biederman04dc7f6b2009-12-03 02:29:04 +00008310 list_for_each_entry(net, net_list, exit_list) {
8311 for_each_netdev_reverse(net, dev) {
Jiri Pirkob0ab2fa2014-06-26 09:58:25 +02008312 if (dev->rtnl_link_ops && dev->rtnl_link_ops->dellink)
Eric W. Biederman04dc7f6b2009-12-03 02:29:04 +00008313 dev->rtnl_link_ops->dellink(dev, &dev_kill_list);
8314 else
8315 unregister_netdevice_queue(dev, &dev_kill_list);
8316 }
8317 }
8318 unregister_netdevice_many(&dev_kill_list);
8319 rtnl_unlock();
8320}
8321
Denis V. Lunev022cbae2007-11-13 03:23:50 -08008322static struct pernet_operations __net_initdata default_device_ops = {
Eric W. Biedermance286d32007-09-12 13:53:49 +02008323 .exit = default_device_exit,
Eric W. Biederman04dc7f6b2009-12-03 02:29:04 +00008324 .exit_batch = default_device_exit_batch,
Eric W. Biedermance286d32007-09-12 13:53:49 +02008325};
8326
Linus Torvalds1da177e2005-04-16 15:20:36 -07008327/*
8328 * Initialize the DEV module. At boot time this walks the device list and
8329 * unhooks any devices that fail to initialise (normally hardware not
8330 * present) and leaves us with a valid list of present and active devices.
8331 *
8332 */
8333
8334/*
8335 * This is called single threaded during boot, so no need
8336 * to take the rtnl semaphore.
8337 */
8338static int __init net_dev_init(void)
8339{
8340 int i, rc = -ENOMEM;
8341
8342 BUG_ON(!dev_boot_phase);
8343
Linus Torvalds1da177e2005-04-16 15:20:36 -07008344 if (dev_proc_init())
8345 goto out;
8346
Eric W. Biederman8b41d182007-09-26 22:02:53 -07008347 if (netdev_kobject_init())
Linus Torvalds1da177e2005-04-16 15:20:36 -07008348 goto out;
8349
8350 INIT_LIST_HEAD(&ptype_all);
Pavel Emelyanov82d8a8672007-11-26 20:12:58 +08008351 for (i = 0; i < PTYPE_HASH_SIZE; i++)
Linus Torvalds1da177e2005-04-16 15:20:36 -07008352 INIT_LIST_HEAD(&ptype_base[i]);
8353
Vlad Yasevich62532da2012-11-15 08:49:10 +00008354 INIT_LIST_HEAD(&offload_base);
8355
Eric W. Biederman881d9662007-09-17 11:56:21 -07008356 if (register_pernet_subsys(&netdev_net_ops))
8357 goto out;
Linus Torvalds1da177e2005-04-16 15:20:36 -07008358
8359 /*
8360 * Initialise the packet receive queues.
8361 */
8362
KAMEZAWA Hiroyuki6f912042006-04-10 22:52:50 -07008363 for_each_possible_cpu(i) {
Eric Dumazet41852492016-08-26 12:50:39 -07008364 struct work_struct *flush = per_cpu_ptr(&flush_works, i);
Eric Dumazete36fa2f2010-04-19 21:17:14 +00008365 struct softnet_data *sd = &per_cpu(softnet_data, i);
Linus Torvalds1da177e2005-04-16 15:20:36 -07008366
Eric Dumazet41852492016-08-26 12:50:39 -07008367 INIT_WORK(flush, flush_backlog);
8368
Eric Dumazete36fa2f2010-04-19 21:17:14 +00008369 skb_queue_head_init(&sd->input_pkt_queue);
Changli Gao6e7676c2010-04-27 15:07:33 -07008370 skb_queue_head_init(&sd->process_queue);
Eric Dumazete36fa2f2010-04-19 21:17:14 +00008371 INIT_LIST_HEAD(&sd->poll_list);
Changli Gaoa9cbd582010-04-26 23:06:24 +00008372 sd->output_queue_tailp = &sd->output_queue;
Eric Dumazetdf334542010-03-24 19:13:54 +00008373#ifdef CONFIG_RPS
Eric Dumazete36fa2f2010-04-19 21:17:14 +00008374 sd->csd.func = rps_trigger_softirq;
8375 sd->csd.info = sd;
Eric Dumazete36fa2f2010-04-19 21:17:14 +00008376 sd->cpu = i;
Tom Herbert1e94d722010-03-18 17:45:44 -07008377#endif
Tom Herbert0a9627f2010-03-16 08:03:29 +00008378
Eric Dumazete36fa2f2010-04-19 21:17:14 +00008379 sd->backlog.poll = process_backlog;
8380 sd->backlog.weight = weight_p;
Linus Torvalds1da177e2005-04-16 15:20:36 -07008381 }
8382
Linus Torvalds1da177e2005-04-16 15:20:36 -07008383 dev_boot_phase = 0;
8384
Eric W. Biederman505d4f72008-11-07 22:54:20 -08008385 /* The loopback device is special if any other network devices
8386 * is present in a network namespace the loopback device must
8387 * be present. Since we now dynamically allocate and free the
8388 * loopback device ensure this invariant is maintained by
8389 * keeping the loopback device as the first device on the
8390 * list of network devices. Ensuring the loopback devices
8391 * is the first device that appears and the last network device
8392 * that disappears.
8393 */
8394 if (register_pernet_device(&loopback_net_ops))
8395 goto out;
8396
8397 if (register_pernet_device(&default_device_ops))
8398 goto out;
8399
Carlos R. Mafra962cf362008-05-15 11:15:37 -03008400 open_softirq(NET_TX_SOFTIRQ, net_tx_action);
8401 open_softirq(NET_RX_SOFTIRQ, net_rx_action);
Linus Torvalds1da177e2005-04-16 15:20:36 -07008402
Sebastian Andrzej Siewiorf0bf90d2016-11-03 15:50:04 +01008403 rc = cpuhp_setup_state_nocalls(CPUHP_NET_DEV_DEAD, "net/dev:dead",
8404 NULL, dev_cpu_dead);
8405 WARN_ON(rc < 0);
Thomas Graff38a9eb2015-07-21 10:43:56 +02008406 dst_subsys_init();
Linus Torvalds1da177e2005-04-16 15:20:36 -07008407 rc = 0;
8408out:
8409 return rc;
8410}
8411
8412subsys_initcall(net_dev_init);