mirror of
https://mirrors.bfsu.edu.cn/git/linux.git
synced 2024-11-11 12:28:41 +08:00
net: move ptype_all into net_hotdata
ptype_all is used in rx/tx fast paths. Move it to net_hotdata for better cache locality. Signed-off-by: Eric Dumazet <edumazet@google.com> Acked-by: Soheil Hassas Yeganeh <soheil@google.com> Reviewed-by: David Ahern <dsahern@kernel.org> Link: https://lore.kernel.org/r/20240306160031.874438-5-edumazet@google.com Signed-off-by: Jakub Kicinski <kuba@kernel.org>
This commit is contained in:
parent
f59b5416c3
commit
0b91fa4bfb
@ -5306,7 +5306,6 @@ static inline const char *netdev_reg_state(const struct net_device *dev)
|
||||
#define PTYPE_HASH_SIZE (16)
|
||||
#define PTYPE_HASH_MASK (PTYPE_HASH_SIZE - 1)
|
||||
|
||||
extern struct list_head ptype_all __read_mostly;
|
||||
extern struct list_head ptype_base[PTYPE_HASH_SIZE] __read_mostly;
|
||||
|
||||
extern struct net_device *blackhole_netdev;
|
||||
|
@ -7,6 +7,7 @@
|
||||
/* Read mostly data used in network fast paths. */
|
||||
struct net_hotdata {
|
||||
struct list_head offload_base;
|
||||
struct list_head ptype_all;
|
||||
int gro_normal_batch;
|
||||
int netdev_budget;
|
||||
int netdev_budget_usecs;
|
||||
|
@ -161,7 +161,6 @@
|
||||
|
||||
static DEFINE_SPINLOCK(ptype_lock);
|
||||
struct list_head ptype_base[PTYPE_HASH_SIZE] __read_mostly;
|
||||
struct list_head ptype_all __read_mostly; /* Taps */
|
||||
|
||||
static int netif_rx_internal(struct sk_buff *skb);
|
||||
static int call_netdevice_notifiers_extack(unsigned long val,
|
||||
@ -540,7 +539,7 @@ static inline void netdev_set_addr_lockdep_class(struct net_device *dev)
|
||||
static inline struct list_head *ptype_head(const struct packet_type *pt)
|
||||
{
|
||||
if (pt->type == htons(ETH_P_ALL))
|
||||
return pt->dev ? &pt->dev->ptype_all : &ptype_all;
|
||||
return pt->dev ? &pt->dev->ptype_all : &net_hotdata.ptype_all;
|
||||
else
|
||||
return pt->dev ? &pt->dev->ptype_specific :
|
||||
&ptype_base[ntohs(pt->type) & PTYPE_HASH_MASK];
|
||||
@ -2226,7 +2225,8 @@ static inline bool skb_loop_sk(struct packet_type *ptype, struct sk_buff *skb)
|
||||
*/
|
||||
bool dev_nit_active(struct net_device *dev)
|
||||
{
|
||||
return !list_empty(&ptype_all) || !list_empty(&dev->ptype_all);
|
||||
return !list_empty(&net_hotdata.ptype_all) ||
|
||||
!list_empty(&dev->ptype_all);
|
||||
}
|
||||
EXPORT_SYMBOL_GPL(dev_nit_active);
|
||||
|
||||
@ -2237,10 +2237,9 @@ EXPORT_SYMBOL_GPL(dev_nit_active);
|
||||
|
||||
void dev_queue_xmit_nit(struct sk_buff *skb, struct net_device *dev)
|
||||
{
|
||||
struct packet_type *ptype;
|
||||
struct list_head *ptype_list = &net_hotdata.ptype_all;
|
||||
struct packet_type *ptype, *pt_prev = NULL;
|
||||
struct sk_buff *skb2 = NULL;
|
||||
struct packet_type *pt_prev = NULL;
|
||||
struct list_head *ptype_list = &ptype_all;
|
||||
|
||||
rcu_read_lock();
|
||||
again:
|
||||
@ -2286,7 +2285,7 @@ again:
|
||||
pt_prev = ptype;
|
||||
}
|
||||
|
||||
if (ptype_list == &ptype_all) {
|
||||
if (ptype_list == &net_hotdata.ptype_all) {
|
||||
ptype_list = &dev->ptype_all;
|
||||
goto again;
|
||||
}
|
||||
@ -5387,7 +5386,7 @@ another_round:
|
||||
if (pfmemalloc)
|
||||
goto skip_taps;
|
||||
|
||||
list_for_each_entry_rcu(ptype, &ptype_all, list) {
|
||||
list_for_each_entry_rcu(ptype, &net_hotdata.ptype_all, list) {
|
||||
if (pt_prev)
|
||||
ret = deliver_skb(skb, pt_prev, orig_dev);
|
||||
pt_prev = ptype;
|
||||
@ -11749,7 +11748,6 @@ static int __init net_dev_init(void)
|
||||
if (netdev_kobject_init())
|
||||
goto out;
|
||||
|
||||
INIT_LIST_HEAD(&ptype_all);
|
||||
for (i = 0; i < PTYPE_HASH_SIZE; i++)
|
||||
INIT_LIST_HEAD(&ptype_base[i]);
|
||||
|
||||
|
@ -7,6 +7,7 @@
|
||||
|
||||
struct net_hotdata net_hotdata __cacheline_aligned = {
|
||||
.offload_base = LIST_HEAD_INIT(net_hotdata.offload_base),
|
||||
.ptype_all = LIST_HEAD_INIT(net_hotdata.ptype_all),
|
||||
.gro_normal_batch = 8,
|
||||
|
||||
.netdev_budget = 300,
|
||||
|
@ -3,6 +3,7 @@
|
||||
#include <linux/proc_fs.h>
|
||||
#include <linux/seq_file.h>
|
||||
#include <net/wext.h>
|
||||
#include <net/hotdata.h>
|
||||
|
||||
#include "dev.h"
|
||||
|
||||
@ -183,7 +184,7 @@ static void *ptype_get_idx(struct seq_file *seq, loff_t pos)
|
||||
}
|
||||
}
|
||||
|
||||
list_for_each_entry_rcu(pt, &ptype_all, list) {
|
||||
list_for_each_entry_rcu(pt, &net_hotdata.ptype_all, list) {
|
||||
if (i == pos)
|
||||
return pt;
|
||||
++i;
|
||||
@ -231,13 +232,13 @@ static void *ptype_seq_next(struct seq_file *seq, void *v, loff_t *pos)
|
||||
}
|
||||
}
|
||||
|
||||
nxt = ptype_all.next;
|
||||
nxt = net_hotdata.ptype_all.next;
|
||||
goto ptype_all;
|
||||
}
|
||||
|
||||
if (pt->type == htons(ETH_P_ALL)) {
|
||||
ptype_all:
|
||||
if (nxt != &ptype_all)
|
||||
if (nxt != &net_hotdata.ptype_all)
|
||||
goto found;
|
||||
hash = 0;
|
||||
nxt = ptype_base[0].next;
|
||||
|
Loading…
Reference in New Issue
Block a user