mirror of
https://mirrors.bfsu.edu.cn/git/linux.git
synced 2024-11-11 12:28:41 +08:00
d288a162dd
dst_entry::__refcnt is highly contended in scenarios where many connections happen from and to the same IP. The reference count is an atomic_t, so the reference count operations have to take the cache-line exclusive. Aside of the unavoidable reference count contention there is another significant problem which is caused by that: False sharing. perf top identified two affected read accesses. dst_entry::lwtstate and rtable::rt_genid. dst_entry:__refcnt is located at offset 64 of dst_entry, which puts it into a seperate cacheline vs. the read mostly members located at the beginning of the struct. That prevents false sharing vs. the struct members in the first 64 bytes of the structure, but there is also dst_entry::lwtstate which is located after the reference count and in the same cache line. This member is read after a reference count has been acquired. struct rtable embeds a struct dst_entry at offset 0. struct dst_entry has a size of 112 bytes, which means that the struct members of rtable which follow the dst member share the same cache line as dst_entry::__refcnt. Especially rtable::rt_genid is also read by the contexts which have a reference count acquired already. When dst_entry:__refcnt is incremented or decremented via an atomic operation these read accesses stall. This was found when analysing the memtier benchmark in 1:100 mode, which amplifies the problem extremly. Move the rt[6i]_uncached[_list] members out of struct rtable and struct rt6_info into struct dst_entry to provide padding and move the lwtstate member after that so it ends up in the same cache line. The resulting improvement depends on the micro-architecture and the number of CPUs. It ranges from +20% to +120% with a localhost memtier/memcached benchmark. [ tglx: Rearrange struct ] Signed-off-by: Wangyang Guo <wangyang.guo@intel.com> Signed-off-by: Arjan van de Ven <arjan@linux.intel.com> Signed-off-by: Thomas Gleixner <tglx@linutronix.de> Reviewed-by: Eric Dumazet <edumazet@google.com> Reviewed-by: David Ahern <dsahern@kernel.org> Link: https://lore.kernel.org/r/20230323102800.042297517@linutronix.de Signed-off-by: Jakub Kicinski <kuba@kernel.org>
260 lines
5.9 KiB
C
260 lines
5.9 KiB
C
// SPDX-License-Identifier: GPL-2.0
|
|
/*
|
|
* xfrm4_policy.c
|
|
*
|
|
* Changes:
|
|
* Kazunori MIYAZAWA @USAGI
|
|
* YOSHIFUJI Hideaki @USAGI
|
|
* Split up af-specific portion
|
|
*
|
|
*/
|
|
|
|
#include <linux/err.h>
|
|
#include <linux/kernel.h>
|
|
#include <linux/inetdevice.h>
|
|
#include <net/dst.h>
|
|
#include <net/xfrm.h>
|
|
#include <net/ip.h>
|
|
#include <net/l3mdev.h>
|
|
|
|
static struct dst_entry *__xfrm4_dst_lookup(struct net *net, struct flowi4 *fl4,
|
|
int tos, int oif,
|
|
const xfrm_address_t *saddr,
|
|
const xfrm_address_t *daddr,
|
|
u32 mark)
|
|
{
|
|
struct rtable *rt;
|
|
|
|
memset(fl4, 0, sizeof(*fl4));
|
|
fl4->daddr = daddr->a4;
|
|
fl4->flowi4_tos = tos;
|
|
fl4->flowi4_l3mdev = l3mdev_master_ifindex_by_index(net, oif);
|
|
fl4->flowi4_mark = mark;
|
|
if (saddr)
|
|
fl4->saddr = saddr->a4;
|
|
|
|
rt = __ip_route_output_key(net, fl4);
|
|
if (!IS_ERR(rt))
|
|
return &rt->dst;
|
|
|
|
return ERR_CAST(rt);
|
|
}
|
|
|
|
static struct dst_entry *xfrm4_dst_lookup(struct net *net, int tos, int oif,
|
|
const xfrm_address_t *saddr,
|
|
const xfrm_address_t *daddr,
|
|
u32 mark)
|
|
{
|
|
struct flowi4 fl4;
|
|
|
|
return __xfrm4_dst_lookup(net, &fl4, tos, oif, saddr, daddr, mark);
|
|
}
|
|
|
|
static int xfrm4_get_saddr(struct net *net, int oif,
|
|
xfrm_address_t *saddr, xfrm_address_t *daddr,
|
|
u32 mark)
|
|
{
|
|
struct dst_entry *dst;
|
|
struct flowi4 fl4;
|
|
|
|
dst = __xfrm4_dst_lookup(net, &fl4, 0, oif, NULL, daddr, mark);
|
|
if (IS_ERR(dst))
|
|
return -EHOSTUNREACH;
|
|
|
|
saddr->a4 = fl4.saddr;
|
|
dst_release(dst);
|
|
return 0;
|
|
}
|
|
|
|
static int xfrm4_fill_dst(struct xfrm_dst *xdst, struct net_device *dev,
|
|
const struct flowi *fl)
|
|
{
|
|
struct rtable *rt = (struct rtable *)xdst->route;
|
|
const struct flowi4 *fl4 = &fl->u.ip4;
|
|
|
|
xdst->u.rt.rt_iif = fl4->flowi4_iif;
|
|
|
|
xdst->u.dst.dev = dev;
|
|
netdev_hold(dev, &xdst->u.dst.dev_tracker, GFP_ATOMIC);
|
|
|
|
/* Sheit... I remember I did this right. Apparently,
|
|
* it was magically lost, so this code needs audit */
|
|
xdst->u.rt.rt_is_input = rt->rt_is_input;
|
|
xdst->u.rt.rt_flags = rt->rt_flags & (RTCF_BROADCAST | RTCF_MULTICAST |
|
|
RTCF_LOCAL);
|
|
xdst->u.rt.rt_type = rt->rt_type;
|
|
xdst->u.rt.rt_uses_gateway = rt->rt_uses_gateway;
|
|
xdst->u.rt.rt_gw_family = rt->rt_gw_family;
|
|
if (rt->rt_gw_family == AF_INET)
|
|
xdst->u.rt.rt_gw4 = rt->rt_gw4;
|
|
else if (rt->rt_gw_family == AF_INET6)
|
|
xdst->u.rt.rt_gw6 = rt->rt_gw6;
|
|
xdst->u.rt.rt_pmtu = rt->rt_pmtu;
|
|
xdst->u.rt.rt_mtu_locked = rt->rt_mtu_locked;
|
|
INIT_LIST_HEAD(&xdst->u.rt.dst.rt_uncached);
|
|
rt_add_uncached_list(&xdst->u.rt);
|
|
|
|
return 0;
|
|
}
|
|
|
|
static void xfrm4_update_pmtu(struct dst_entry *dst, struct sock *sk,
|
|
struct sk_buff *skb, u32 mtu,
|
|
bool confirm_neigh)
|
|
{
|
|
struct xfrm_dst *xdst = (struct xfrm_dst *)dst;
|
|
struct dst_entry *path = xdst->route;
|
|
|
|
path->ops->update_pmtu(path, sk, skb, mtu, confirm_neigh);
|
|
}
|
|
|
|
static void xfrm4_redirect(struct dst_entry *dst, struct sock *sk,
|
|
struct sk_buff *skb)
|
|
{
|
|
struct xfrm_dst *xdst = (struct xfrm_dst *)dst;
|
|
struct dst_entry *path = xdst->route;
|
|
|
|
path->ops->redirect(path, sk, skb);
|
|
}
|
|
|
|
static void xfrm4_dst_destroy(struct dst_entry *dst)
|
|
{
|
|
struct xfrm_dst *xdst = (struct xfrm_dst *)dst;
|
|
|
|
dst_destroy_metrics_generic(dst);
|
|
if (xdst->u.rt.dst.rt_uncached_list)
|
|
rt_del_uncached_list(&xdst->u.rt);
|
|
xfrm_dst_destroy(xdst);
|
|
}
|
|
|
|
static void xfrm4_dst_ifdown(struct dst_entry *dst, struct net_device *dev,
|
|
int unregister)
|
|
{
|
|
if (!unregister)
|
|
return;
|
|
|
|
xfrm_dst_ifdown(dst, dev);
|
|
}
|
|
|
|
static struct dst_ops xfrm4_dst_ops_template = {
|
|
.family = AF_INET,
|
|
.update_pmtu = xfrm4_update_pmtu,
|
|
.redirect = xfrm4_redirect,
|
|
.cow_metrics = dst_cow_metrics_generic,
|
|
.destroy = xfrm4_dst_destroy,
|
|
.ifdown = xfrm4_dst_ifdown,
|
|
.local_out = __ip_local_out,
|
|
.gc_thresh = 32768,
|
|
};
|
|
|
|
static const struct xfrm_policy_afinfo xfrm4_policy_afinfo = {
|
|
.dst_ops = &xfrm4_dst_ops_template,
|
|
.dst_lookup = xfrm4_dst_lookup,
|
|
.get_saddr = xfrm4_get_saddr,
|
|
.fill_dst = xfrm4_fill_dst,
|
|
.blackhole_route = ipv4_blackhole_route,
|
|
};
|
|
|
|
#ifdef CONFIG_SYSCTL
|
|
static struct ctl_table xfrm4_policy_table[] = {
|
|
{
|
|
.procname = "xfrm4_gc_thresh",
|
|
.data = &init_net.xfrm.xfrm4_dst_ops.gc_thresh,
|
|
.maxlen = sizeof(int),
|
|
.mode = 0644,
|
|
.proc_handler = proc_dointvec,
|
|
},
|
|
{ }
|
|
};
|
|
|
|
static __net_init int xfrm4_net_sysctl_init(struct net *net)
|
|
{
|
|
struct ctl_table *table;
|
|
struct ctl_table_header *hdr;
|
|
|
|
table = xfrm4_policy_table;
|
|
if (!net_eq(net, &init_net)) {
|
|
table = kmemdup(table, sizeof(xfrm4_policy_table), GFP_KERNEL);
|
|
if (!table)
|
|
goto err_alloc;
|
|
|
|
table[0].data = &net->xfrm.xfrm4_dst_ops.gc_thresh;
|
|
}
|
|
|
|
hdr = register_net_sysctl(net, "net/ipv4", table);
|
|
if (!hdr)
|
|
goto err_reg;
|
|
|
|
net->ipv4.xfrm4_hdr = hdr;
|
|
return 0;
|
|
|
|
err_reg:
|
|
if (!net_eq(net, &init_net))
|
|
kfree(table);
|
|
err_alloc:
|
|
return -ENOMEM;
|
|
}
|
|
|
|
static __net_exit void xfrm4_net_sysctl_exit(struct net *net)
|
|
{
|
|
struct ctl_table *table;
|
|
|
|
if (!net->ipv4.xfrm4_hdr)
|
|
return;
|
|
|
|
table = net->ipv4.xfrm4_hdr->ctl_table_arg;
|
|
unregister_net_sysctl_table(net->ipv4.xfrm4_hdr);
|
|
if (!net_eq(net, &init_net))
|
|
kfree(table);
|
|
}
|
|
#else /* CONFIG_SYSCTL */
|
|
static inline int xfrm4_net_sysctl_init(struct net *net)
|
|
{
|
|
return 0;
|
|
}
|
|
|
|
static inline void xfrm4_net_sysctl_exit(struct net *net)
|
|
{
|
|
}
|
|
#endif
|
|
|
|
static int __net_init xfrm4_net_init(struct net *net)
|
|
{
|
|
int ret;
|
|
|
|
memcpy(&net->xfrm.xfrm4_dst_ops, &xfrm4_dst_ops_template,
|
|
sizeof(xfrm4_dst_ops_template));
|
|
ret = dst_entries_init(&net->xfrm.xfrm4_dst_ops);
|
|
if (ret)
|
|
return ret;
|
|
|
|
ret = xfrm4_net_sysctl_init(net);
|
|
if (ret)
|
|
dst_entries_destroy(&net->xfrm.xfrm4_dst_ops);
|
|
|
|
return ret;
|
|
}
|
|
|
|
static void __net_exit xfrm4_net_exit(struct net *net)
|
|
{
|
|
xfrm4_net_sysctl_exit(net);
|
|
dst_entries_destroy(&net->xfrm.xfrm4_dst_ops);
|
|
}
|
|
|
|
static struct pernet_operations __net_initdata xfrm4_net_ops = {
|
|
.init = xfrm4_net_init,
|
|
.exit = xfrm4_net_exit,
|
|
};
|
|
|
|
static void __init xfrm4_policy_init(void)
|
|
{
|
|
xfrm_policy_register_afinfo(&xfrm4_policy_afinfo, AF_INET);
|
|
}
|
|
|
|
void __init xfrm4_init(void)
|
|
{
|
|
xfrm4_state_init();
|
|
xfrm4_policy_init();
|
|
xfrm4_protocol_init();
|
|
register_pernet_subsys(&xfrm4_net_ops);
|
|
}
|