korg/linux
0
0
Fork 0
mirror of https://mirrors.bfsu.edu.cn/git/linux.git synced 2024-11-26 05:34:13 +08:00
Code Issues Packages Projects Releases Wiki Activity

netfilter: nf_tables: avoid retpoline overhead for some ct expression calls

Browse Source 
nft_ct expression cannot be made builtin to nf_tables without also
forcing the conntrack itself to be builtin.

However, this can be avoided by splitting retrieval of a few
selector keys that only need to access the nf_conn structure,
i.e. no function calls to nf_conntrack code.

Many rulesets start with something like
"ct status established,related accept"

With this change, this no longer requires an indirect call, which
gives about 1.8% more throughput with a simple conntrack-enabled
forwarding test (retpoline thunk used).

Signed-off-by: Florian Westphal <fw@strlen.de>
This commit is contained in:
Florian Westphal 2023-01-03 13:47:17 +01:00
parent 2032e907d8
commit d9e7891476
5 changed files with 104 additions and 12 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

12
include/net/netfilter/nf_tables_core.h
View File

@ -61,6 +61,16 @@ struct nft_immediate_expr {
extern const struct nft_expr_ops nft_cmp_fast_ops;
extern const struct nft_expr_ops nft_cmp16_fast_ops;
struct nft_ct {
enum nft_ct_keys key:8;
enum ip_conntrack_dir dir:8;
u8 len;
union {
u8 dreg;
u8 sreg;
};
};
struct nft_payload {
enum nft_payload_bases base:8;
u8 offset;
@ -140,6 +150,8 @@ void nft_rt_get_eval(const struct nft_expr *expr,
struct nft_regs *regs, const struct nft_pktinfo *pkt);
void nft_counter_eval(const struct nft_expr *expr, struct nft_regs *regs,
const struct nft_pktinfo *pkt);
void nft_ct_get_fast_eval(const struct nft_expr *expr,
struct nft_regs *regs, const struct nft_pktinfo *pkt);
enum {
NFT_PAYLOAD_CTX_INNER_TUN = (1 << 0),

6
net/netfilter/Makefile
View File

@ -98,6 +98,12 @@ nf_tables-objs += nft_set_pipapo_avx2.o
endif
endif
ifdef CONFIG_NFT_CT
ifdef CONFIG_RETPOLINE
nf_tables-objs += nft_ct_fast.o
endif
endif
obj-$(CONFIG_NF_TABLES) += nf_tables.o
obj-$(CONFIG_NFT_COMPAT) += nft_compat.o
obj-$(CONFIG_NFT_CONNLIMIT) += nft_connlimit.o

3
net/netfilter/nf_tables_core.c
View File

@ -228,6 +228,9 @@ static void expr_call_ops_eval(const struct nft_expr *expr,
X(e, nft_counter_eval);
X(e, nft_meta_get_eval);
X(e, nft_lookup_eval);
#if IS_ENABLED(CONFIG_NFT_CT)
X(e, nft_ct_get_fast_eval);
#endif
X(e, nft_range_eval);
X(e, nft_immediate_eval);
X(e, nft_byteorder_eval);

39
net/netfilter/nft_ct.c
View File

@ -12,7 +12,7 @@
#include <linux/netlink.h>
#include <linux/netfilter.h>
#include <linux/netfilter/nf_tables.h>
#include <net/netfilter/nf_tables.h>
#include <net/netfilter/nf_tables_core.h>
#include <net/netfilter/nf_conntrack.h>
#include <net/netfilter/nf_conntrack_acct.h>
#include <net/netfilter/nf_conntrack_tuple.h>
@ -23,16 +23,6 @@
#include <net/netfilter/nf_conntrack_l4proto.h>
#include <net/netfilter/nf_conntrack_expect.h>
struct nft_ct {
enum nft_ct_keys key:8;
enum ip_conntrack_dir dir:8;
u8 len;
union {
u8 dreg;
u8 sreg;
};
};
struct nft_ct_helper_obj {
struct nf_conntrack_helper *helper4;
struct nf_conntrack_helper *helper6;
@ -759,6 +749,18 @@ static bool nft_ct_set_reduce(struct nft_regs_track *track,
return false;
}
#ifdef CONFIG_RETPOLINE
static const struct nft_expr_ops nft_ct_get_fast_ops = {
.type = &nft_ct_type,
.size = NFT_EXPR_SIZE(sizeof(struct nft_ct)),
.eval = nft_ct_get_fast_eval,
.init = nft_ct_get_init,
.destroy = nft_ct_get_destroy,
.dump = nft_ct_get_dump,
.reduce = nft_ct_set_reduce,
};
#endif
static const struct nft_expr_ops nft_ct_set_ops = {
.type = &nft_ct_type,
.size = NFT_EXPR_SIZE(sizeof(struct nft_ct)),
@ -791,8 +793,21 @@ nft_ct_select_ops(const struct nft_ctx *ctx,
if (tb[NFTA_CT_DREG] && tb[NFTA_CT_SREG])
return ERR_PTR(-EINVAL);
if (tb[NFTA_CT_DREG])
if (tb[NFTA_CT_DREG]) {
#ifdef CONFIG_RETPOLINE
u32 k = ntohl(nla_get_be32(tb[NFTA_CT_KEY]));
switch (k) {
case NFT_CT_STATE:
case NFT_CT_DIRECTION:
case NFT_CT_STATUS:
case NFT_CT_MARK:
case NFT_CT_SECMARK:
return &nft_ct_get_fast_ops;
}
#endif
return &nft_ct_get_ops;
}
if (tb[NFTA_CT_SREG]) {
#ifdef CONFIG_NF_CONNTRACK_ZONES

56
net/netfilter/nft_ct_fast.c Normal file
View File

@ -0,0 +1,56 @@
// SPDX-License-Identifier: GPL-2.0-only
#if IS_ENABLED(CONFIG_NFT_CT)
#include <linux/netfilter/nf_tables.h>
#include <net/netfilter/nf_tables_core.h>
#include <net/netfilter/nf_conntrack.h>
void nft_ct_get_fast_eval(const struct nft_expr *expr,
struct nft_regs *regs,
const struct nft_pktinfo *pkt)
{
const struct nft_ct *priv = nft_expr_priv(expr);
u32 *dest = &regs->data[priv->dreg];
enum ip_conntrack_info ctinfo;
const struct nf_conn *ct;
unsigned int state;
ct = nf_ct_get(pkt->skb, &ctinfo);
if (!ct) {
regs->verdict.code = NFT_BREAK;
return;
}
switch (priv->key) {
case NFT_CT_STATE:
if (ct)
state = NF_CT_STATE_BIT(ctinfo);
else if (ctinfo == IP_CT_UNTRACKED)
state = NF_CT_STATE_UNTRACKED_BIT;
else
state = NF_CT_STATE_INVALID_BIT;
*dest = state;
return;
case NFT_CT_DIRECTION:
nft_reg_store8(dest, CTINFO2DIR(ctinfo));
return;
case NFT_CT_STATUS:
*dest = ct->status;
return;
#ifdef CONFIG_NF_CONNTRACK_MARK
case NFT_CT_MARK:
*dest = ct->mark;
return;
#endif
#ifdef CONFIG_NF_CONNTRACK_SECMARK
case NFT_CT_SECMARK:
*dest = ct->secmark;
return;
#endif
default:
WARN_ON_ONCE(1);
regs->verdict.code = NFT_BREAK;
break;
}
}
EXPORT_SYMBOL_GPL(nft_ct_get_fast_eval);
#endif