renegade-project/linux-next
2
0
Fork 0
mirror of https://github.com/edk2-porting/linux-next.git synced 2024-12-22 20:23:57 +08:00
Code

nft_set_pipapo: Generalise group size for buckets

Browse Source 
Get rid of all hardcoded assumptions that buckets in lookup tables
correspond to four-bit groups, and replace them with appropriate
calculations based on a variable group size, now stored in struct
field.

The group size could now be in principle any divisor of eight. Note,
though, that lookup and get functions need an implementation
intimately depending on the group size, and the only supported size
there, currently, is four bits, which is also the initial and only
used size at the moment.

While at it, drop 'groups' from struct nft_pipapo: it was never used.

Signed-off-by: Stefano Brivio <sbrivio@redhat.com>
Signed-off-by: Pablo Neira Ayuso <pablo@netfilter.org>
This commit is contained in:
Stefano Brivio 2020-03-07 17:52:32 +01:00 committed by Pablo Neira Ayuso
parent 88bf6e4114
commit e807b13cb3
1 changed files with 111 additions and 95 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

206
net/netfilter/nft_set_pipapo.c
View File

@ -350,16 +350,18 @@
/* Number of bits to be grouped together in lookup table buckets, arbitrary */ /* Number of bits to be grouped together in lookup table buckets, arbitrary */
#define NFT_PIPAPO_GROUP_BITS 4 #define NFT_PIPAPO_GROUP_BITS 4
#define NFT_PIPAPO_GROUPS_PER_BYTE (BITS_PER_BYTE / NFT_PIPAPO_GROUP_BITS)
#define NFT_PIPAPO_GROUPS_PER_BYTE(f) (BITS_PER_BYTE / (f)->bb)
/* Fields are padded to 32 bits in input registers */ /* Fields are padded to 32 bits in input registers */
#define NFT_PIPAPO_GROUPS_PADDED_SIZE(x) \ #define NFT_PIPAPO_GROUPS_PADDED_SIZE(f) \
(round_up((x) / NFT_PIPAPO_GROUPS_PER_BYTE, sizeof(u32))) (round_up((f)->groups / NFT_PIPAPO_GROUPS_PER_BYTE(f), sizeof(u32)))
#define NFT_PIPAPO_GROUPS_PADDING(x) \ #define NFT_PIPAPO_GROUPS_PADDING(f) \
(NFT_PIPAPO_GROUPS_PADDED_SIZE((x)) - (x) / NFT_PIPAPO_GROUPS_PER_BYTE) (NFT_PIPAPO_GROUPS_PADDED_SIZE(f) - (f)->groups / \
NFT_PIPAPO_GROUPS_PER_BYTE(f))
/* Number of buckets, given by 2 ^ n, with n grouped bits */ /* Number of buckets given by 2 ^ n, with n bucket bits */
#define NFT_PIPAPO_BUCKETS (1 << NFT_PIPAPO_GROUP_BITS) #define NFT_PIPAPO_BUCKETS(bb) (1 << (bb))
/* Each n-bit range maps to up to n * 2 rules */ /* Each n-bit range maps to up to n * 2 rules */
#define NFT_PIPAPO_MAP_NBITS (const_ilog2(NFT_PIPAPO_MAX_BITS * 2)) #define NFT_PIPAPO_MAP_NBITS (const_ilog2(NFT_PIPAPO_MAX_BITS * 2))
@ -406,16 +408,18 @@ union nft_pipapo_map_bucket {
/** /**
* struct nft_pipapo_field - Lookup, mapping tables and related data for a field * struct nft_pipapo_field - Lookup, mapping tables and related data for a field
* @groups: Amount of 4-bit groups * @groups: Amount of bit groups
* @rules: Number of inserted rules * @rules: Number of inserted rules
* @bsize: Size of each bucket in lookup table, in longs * @bsize: Size of each bucket in lookup table, in longs
* @lt: Lookup table: 'groups' rows of NFT_PIPAPO_BUCKETS buckets * @bb: Number of bits grouped together in lookup table buckets
* @lt: Lookup table: 'groups' rows of buckets
* @mt: Mapping table: one bucket per rule * @mt: Mapping table: one bucket per rule
*/ */
struct nft_pipapo_field { struct nft_pipapo_field {
int groups; int groups;
unsigned long rules; unsigned long rules;
size_t bsize; size_t bsize;
int bb;
unsigned long *lt; unsigned long *lt;
union nft_pipapo_map_bucket *mt; union nft_pipapo_map_bucket *mt;
}; };
@ -443,7 +447,6 @@ static DEFINE_PER_CPU(bool, nft_pipapo_scratch_index);
* struct nft_pipapo - Representation of a set * struct nft_pipapo - Representation of a set
* @match: Currently in-use matching data * @match: Currently in-use matching data
* @clone: Copy where pending insertions and deletions are kept * @clone: Copy where pending insertions and deletions are kept
* @groups: Total amount of 4-bit groups for fields in this set
* @width: Total bytes to be matched for one packet, including padding * @width: Total bytes to be matched for one packet, including padding
* @dirty: Working copy has pending insertions or deletions * @dirty: Working copy has pending insertions or deletions
* @last_gc: Timestamp of last garbage collection run, jiffies * @last_gc: Timestamp of last garbage collection run, jiffies
@ -451,7 +454,6 @@ static DEFINE_PER_CPU(bool, nft_pipapo_scratch_index);
struct nft_pipapo { struct nft_pipapo {
struct nft_pipapo_match __rcu *match; struct nft_pipapo_match __rcu *match;
struct nft_pipapo_match *clone; struct nft_pipapo_match *clone;
int groups;
int width; int width;
bool dirty; bool dirty;
unsigned long last_gc; unsigned long last_gc;
@ -520,6 +522,34 @@ static int pipapo_refill(unsigned long *map, int len, int rules,
return ret; return ret;
} }
/**
* pipapo_and_field_buckets_4bit() - Intersect buckets for 4-bit groups
* @f: Field including lookup table
* @dst: Area to store result
* @data: Input data selecting table buckets
*/
static void pipapo_and_field_buckets_4bit(struct nft_pipapo_field *f,
unsigned long *dst,
const u8 *data)
{
unsigned long *lt = f->lt;
int group;
for (group = 0; group < f->groups; group += BITS_PER_BYTE / 4, data++) {
u8 v;
v = *data >> 4;
__bitmap_and(dst, dst, lt + v * f->bsize,
f->bsize * BITS_PER_LONG);
lt += f->bsize * NFT_PIPAPO_BUCKETS(4);
v = *data & 0x0f;
__bitmap_and(dst, dst, lt + v * f->bsize,
f->bsize * BITS_PER_LONG);
lt += f->bsize * NFT_PIPAPO_BUCKETS(4);
}
}
/** /**
* nft_pipapo_lookup() - Lookup function * nft_pipapo_lookup() - Lookup function
* @net: Network namespace * @net: Network namespace
@ -559,26 +589,15 @@ static bool nft_pipapo_lookup(const struct net *net, const struct nft_set *set,
nft_pipapo_for_each_field(f, i, m) { nft_pipapo_for_each_field(f, i, m) {
bool last = i == m->field_count - 1; bool last = i == m->field_count - 1;
unsigned long *lt = f->lt; int b;
int b, group;
/* For each 4-bit group: select lookup table bucket depending on /* For each bit group: select lookup table bucket depending on
* packet bytes value, then AND bucket value * packet bytes value, then AND bucket value
*/ */
for (group = 0; group < f->groups; group += 2) { pipapo_and_field_buckets_4bit(f, res_map, rp);
u8 v; BUILD_BUG_ON(NFT_PIPAPO_GROUP_BITS != 4);
v = *rp >> 4; rp += f->groups / NFT_PIPAPO_GROUPS_PER_BYTE(f);
__bitmap_and(res_map, res_map, lt + v * f->bsize,
f->bsize * BITS_PER_LONG);
lt += f->bsize * NFT_PIPAPO_BUCKETS;
v = *rp & 0x0f;
rp++;
__bitmap_and(res_map, res_map, lt + v * f->bsize,
f->bsize * BITS_PER_LONG);
lt += f->bsize * NFT_PIPAPO_BUCKETS;
}
/* Now populate the bitmap for the next field, unless this is /* Now populate the bitmap for the next field, unless this is
* the last field, in which case return the matched 'ext' * the last field, in which case return the matched 'ext'
@ -621,7 +640,7 @@ next_match:
map_index = !map_index; map_index = !map_index;
swap(res_map, fill_map); swap(res_map, fill_map);
rp += NFT_PIPAPO_GROUPS_PADDING(f->groups); rp += NFT_PIPAPO_GROUPS_PADDING(f);
} }
out: out:
@ -669,26 +688,17 @@ static struct nft_pipapo_elem *pipapo_get(const struct net *net,
nft_pipapo_for_each_field(f, i, m) { nft_pipapo_for_each_field(f, i, m) {
bool last = i == m->field_count - 1; bool last = i == m->field_count - 1;
unsigned long *lt = f->lt; int b;
int b, group;
/* For each 4-bit group: select lookup table bucket depending on /* For each bit group: select lookup table bucket depending on
* packet bytes value, then AND bucket value * packet bytes value, then AND bucket value
*/ */
for (group = 0; group < f->groups; group++) { if (f->bb == 4)
u8 v; pipapo_and_field_buckets_4bit(f, res_map, data);
else
BUG();
if (group % 2) { data += f->groups / NFT_PIPAPO_GROUPS_PER_BYTE(f);
v = *data & 0x0f;
data++;
} else {
v = *data >> 4;
}
__bitmap_and(res_map, res_map, lt + v * f->bsize,
f->bsize * BITS_PER_LONG);
lt += f->bsize * NFT_PIPAPO_BUCKETS;
}
/* Now populate the bitmap for the next field, unless this is /* Now populate the bitmap for the next field, unless this is
* the last field, in which case return the matched 'ext' * the last field, in which case return the matched 'ext'
@ -713,7 +723,7 @@ next_match:
goto out; goto out;
} }
data += NFT_PIPAPO_GROUPS_PADDING(f->groups); data += NFT_PIPAPO_GROUPS_PADDING(f);
/* Swap bitmap indices: fill_map will be the initial bitmap for /* Swap bitmap indices: fill_map will be the initial bitmap for
* the next field (i.e. the new res_map), and res_map is * the next field (i.e. the new res_map), and res_map is
@ -772,15 +782,15 @@ static int pipapo_resize(struct nft_pipapo_field *f, int old_rules, int rules)
else else
copy = new_bucket_size; copy = new_bucket_size;
new_lt = kvzalloc(f->groups * NFT_PIPAPO_BUCKETS * new_bucket_size * new_lt = kvzalloc(f->groups * NFT_PIPAPO_BUCKETS(f->bb) *
sizeof(*new_lt), GFP_KERNEL); new_bucket_size * sizeof(*new_lt), GFP_KERNEL);
if (!new_lt) if (!new_lt)
return -ENOMEM; return -ENOMEM;
new_p = new_lt; new_p = new_lt;
old_p = old_lt; old_p = old_lt;
for (group = 0; group < f->groups; group++) { for (group = 0; group < f->groups; group++) {
for (bucket = 0; bucket < NFT_PIPAPO_BUCKETS; bucket++) { for (bucket = 0; bucket < NFT_PIPAPO_BUCKETS(f->bb); bucket++) {
memcpy(new_p, old_p, copy * sizeof(*new_p)); memcpy(new_p, old_p, copy * sizeof(*new_p));
new_p += copy; new_p += copy;
old_p += copy; old_p += copy;
@ -829,7 +839,7 @@ static void pipapo_bucket_set(struct nft_pipapo_field *f, int rule, int group,
{ {
unsigned long *pos; unsigned long *pos;
pos = f->lt + f->bsize * NFT_PIPAPO_BUCKETS * group; pos = f->lt + f->bsize * NFT_PIPAPO_BUCKETS(f->bb) * group;
pos += f->bsize * v; pos += f->bsize * v;
__set_bit(rule, pos); __set_bit(rule, pos);
@ -849,7 +859,7 @@ static void pipapo_bucket_set(struct nft_pipapo_field *f, int rule, int group,
static int pipapo_insert(struct nft_pipapo_field *f, const uint8_t *k, static int pipapo_insert(struct nft_pipapo_field *f, const uint8_t *k,
int mask_bits) int mask_bits)
{ {
int rule = f->rules++, group, ret; int rule = f->rules++, group, ret, bit_offset = 0;
ret = pipapo_resize(f, f->rules - 1, f->rules); ret = pipapo_resize(f, f->rules - 1, f->rules);
if (ret) if (ret)
@ -859,22 +869,25 @@ static int pipapo_insert(struct nft_pipapo_field *f, const uint8_t *k,
int i, v; int i, v;
u8 mask; u8 mask;
if (group % 2) v = k[group / (BITS_PER_BYTE / f->bb)];
v = k[group / 2] & 0x0f; v &= GENMASK(BITS_PER_BYTE - bit_offset - 1, 0);
else v >>= (BITS_PER_BYTE - bit_offset) - f->bb;
v = k[group / 2] >> 4;
if (mask_bits >= (group + 1) * 4) { bit_offset += f->bb;
bit_offset %= BITS_PER_BYTE;
if (mask_bits >= (group + 1) * f->bb) {
/* Not masked */ /* Not masked */
pipapo_bucket_set(f, rule, group, v); pipapo_bucket_set(f, rule, group, v);
} else if (mask_bits <= group * 4) { } else if (mask_bits <= group * f->bb) {
/* Completely masked */ /* Completely masked */
for (i = 0; i < NFT_PIPAPO_BUCKETS; i++) for (i = 0; i < NFT_PIPAPO_BUCKETS(f->bb); i++)
pipapo_bucket_set(f, rule, group, i); pipapo_bucket_set(f, rule, group, i);
} else { } else {
/* The mask limit falls on this group */ /* The mask limit falls on this group */
mask = 0x0f >> (mask_bits - group * 4); mask = GENMASK(f->bb - 1, 0);
for (i = 0; i < NFT_PIPAPO_BUCKETS; i++) { mask >>= mask_bits - group * f->bb;
for (i = 0; i < NFT_PIPAPO_BUCKETS(f->bb); i++) {
if ((i & ~mask) == (v & ~mask)) if ((i & ~mask) == (v & ~mask))
pipapo_bucket_set(f, rule, group, i); pipapo_bucket_set(f, rule, group, i);
} }
@ -1123,11 +1136,11 @@ static int nft_pipapo_insert(const struct net *net, const struct nft_set *set,
return -ENOSPC; return -ENOSPC;
if (memcmp(start_p, end_p, if (memcmp(start_p, end_p,
f->groups / NFT_PIPAPO_GROUPS_PER_BYTE) > 0) f->groups / NFT_PIPAPO_GROUPS_PER_BYTE(f)) > 0)
return -EINVAL; return -EINVAL;
start_p += NFT_PIPAPO_GROUPS_PADDED_SIZE(f->groups); start_p += NFT_PIPAPO_GROUPS_PADDED_SIZE(f);
end_p += NFT_PIPAPO_GROUPS_PADDED_SIZE(f->groups); end_p += NFT_PIPAPO_GROUPS_PADDED_SIZE(f);
} }
/* Insert */ /* Insert */
@ -1141,22 +1154,19 @@ static int nft_pipapo_insert(const struct net *net, const struct nft_set *set,
rulemap[i].to = f->rules; rulemap[i].to = f->rules;
ret = memcmp(start, end, ret = memcmp(start, end,
f->groups / NFT_PIPAPO_GROUPS_PER_BYTE); f->groups / NFT_PIPAPO_GROUPS_PER_BYTE(f));
if (!ret) { if (!ret)
ret = pipapo_insert(f, start, ret = pipapo_insert(f, start, f->groups * f->bb);
f->groups * NFT_PIPAPO_GROUP_BITS); else
} else { ret = pipapo_expand(f, start, end, f->groups * f->bb);
ret = pipapo_expand(f, start, end,
f->groups * NFT_PIPAPO_GROUP_BITS);
}
if (f->bsize > bsize_max) if (f->bsize > bsize_max)
bsize_max = f->bsize; bsize_max = f->bsize;
rulemap[i].n = ret; rulemap[i].n = ret;
start += NFT_PIPAPO_GROUPS_PADDED_SIZE(f->groups); start += NFT_PIPAPO_GROUPS_PADDED_SIZE(f);
end += NFT_PIPAPO_GROUPS_PADDED_SIZE(f->groups); end += NFT_PIPAPO_GROUPS_PADDED_SIZE(f);
} }
if (!*this_cpu_ptr(m->scratch) || bsize_max > m->bsize_max) { if (!*this_cpu_ptr(m->scratch) || bsize_max > m->bsize_max) {
@ -1208,7 +1218,7 @@ static struct nft_pipapo_match *pipapo_clone(struct nft_pipapo_match *old)
for (i = 0; i < old->field_count; i++) { for (i = 0; i < old->field_count; i++) {
memcpy(dst, src, offsetof(struct nft_pipapo_field, lt)); memcpy(dst, src, offsetof(struct nft_pipapo_field, lt));
dst->lt = kvzalloc(src->groups * NFT_PIPAPO_BUCKETS * dst->lt = kvzalloc(src->groups * NFT_PIPAPO_BUCKETS(src->bb) *
src->bsize * sizeof(*dst->lt), src->bsize * sizeof(*dst->lt),
GFP_KERNEL); GFP_KERNEL);
if (!dst->lt) if (!dst->lt)
@ -1216,7 +1226,7 @@ static struct nft_pipapo_match *pipapo_clone(struct nft_pipapo_match *old)
memcpy(dst->lt, src->lt, memcpy(dst->lt, src->lt,
src->bsize * sizeof(*dst->lt) * src->bsize * sizeof(*dst->lt) *
src->groups * NFT_PIPAPO_BUCKETS); src->groups * NFT_PIPAPO_BUCKETS(src->bb));
dst->mt = kvmalloc(src->rules * sizeof(*src->mt), GFP_KERNEL); dst->mt = kvmalloc(src->rules * sizeof(*src->mt), GFP_KERNEL);
if (!dst->mt) if (!dst->mt)
@ -1394,9 +1404,9 @@ static void pipapo_drop(struct nft_pipapo_match *m,
unsigned long *pos; unsigned long *pos;
int b; int b;
pos = f->lt + g * NFT_PIPAPO_BUCKETS * f->bsize; pos = f->lt + g * NFT_PIPAPO_BUCKETS(f->bb) * f->bsize;
for (b = 0; b < NFT_PIPAPO_BUCKETS; b++) { for (b = 0; b < NFT_PIPAPO_BUCKETS(f->bb); b++) {
bitmap_cut(pos, pos, rulemap[i].to, bitmap_cut(pos, pos, rulemap[i].to,
rulemap[i].n, rulemap[i].n,
f->bsize * BITS_PER_LONG); f->bsize * BITS_PER_LONG);
@ -1690,30 +1700,33 @@ static bool nft_pipapo_flush(const struct net *net, const struct nft_set *set,
static int pipapo_get_boundaries(struct nft_pipapo_field *f, int first_rule, static int pipapo_get_boundaries(struct nft_pipapo_field *f, int first_rule,
int rule_count, u8 *left, u8 *right) int rule_count, u8 *left, u8 *right)
{ {
int g, mask_len = 0, bit_offset = 0;
u8 *l = left, *r = right; u8 *l = left, *r = right;
int g, mask_len = 0;
for (g = 0; g < f->groups; g++) { for (g = 0; g < f->groups; g++) {
int b, x0, x1; int b, x0, x1;
x0 = -1; x0 = -1;
x1 = -1; x1 = -1;
for (b = 0; b < NFT_PIPAPO_BUCKETS; b++) { for (b = 0; b < NFT_PIPAPO_BUCKETS(f->bb); b++) {
unsigned long *pos; unsigned long *pos;
pos = f->lt + (g * NFT_PIPAPO_BUCKETS + b) * f->bsize; pos = f->lt + (g * NFT_PIPAPO_BUCKETS(f->bb) + b) *
f->bsize;
if (test_bit(first_rule, pos) && x0 == -1) if (test_bit(first_rule, pos) && x0 == -1)
x0 = b; x0 = b;
if (test_bit(first_rule + rule_count - 1, pos)) if (test_bit(first_rule + rule_count - 1, pos))
x1 = b; x1 = b;
} }
if (g % 2) { *l |= x0 << (BITS_PER_BYTE - f->bb - bit_offset);
*(l++) |= x0 & 0x0f; *r |= x1 << (BITS_PER_BYTE - f->bb - bit_offset);
*(r++) |= x1 & 0x0f;
} else { bit_offset += f->bb;
*l |= x0 << 4; if (bit_offset >= BITS_PER_BYTE) {
*r |= x1 << 4; bit_offset %= BITS_PER_BYTE;
l++;
r++;
} }
if (x1 - x0 == 0) if (x1 - x0 == 0)
@ -1748,8 +1761,9 @@ static bool pipapo_match_field(struct nft_pipapo_field *f,
pipapo_get_boundaries(f, first_rule, rule_count, left, right); pipapo_get_boundaries(f, first_rule, rule_count, left, right);
return !memcmp(start, left, f->groups / NFT_PIPAPO_GROUPS_PER_BYTE) && return !memcmp(start, left,
!memcmp(end, right, f->groups / NFT_PIPAPO_GROUPS_PER_BYTE); f->groups / NFT_PIPAPO_GROUPS_PER_BYTE(f)) &&
!memcmp(end, right, f->groups / NFT_PIPAPO_GROUPS_PER_BYTE(f));
} }
/** /**
@ -1801,8 +1815,8 @@ static void nft_pipapo_remove(const struct net *net, const struct nft_set *set,
rules_fx = f->mt[start].n; rules_fx = f->mt[start].n;
start = f->mt[start].to; start = f->mt[start].to;
match_start += NFT_PIPAPO_GROUPS_PADDED_SIZE(f->groups); match_start += NFT_PIPAPO_GROUPS_PADDED_SIZE(f);
match_end += NFT_PIPAPO_GROUPS_PADDED_SIZE(f->groups); match_end += NFT_PIPAPO_GROUPS_PADDED_SIZE(f);
} }
if (i == m->field_count) { if (i == m->field_count) {
@ -1895,9 +1909,9 @@ static u64 nft_pipapo_privsize(const struct nlattr * const nla[],
* case here. * case here.
* *
* In general, for a non-ranged entry or a single composing netmask, we need * In general, for a non-ranged entry or a single composing netmask, we need
* one bit in each of the sixteen NFT_PIPAPO_BUCKETS, for each 4-bit group (that * one bit in each of the sixteen buckets, for each 4-bit group (that is, each
* is, each input bit needs four bits of matching data), plus a bucket in the * input bit needs four bits of matching data), plus a bucket in the mapping
* mapping table for each field. * table for each field.
* *
* Return: true only for compatible range concatenations * Return: true only for compatible range concatenations
*/ */
@ -1921,7 +1935,9 @@ static bool nft_pipapo_estimate(const struct nft_set_desc *desc, u32 features,
* each rule also needs a mapping bucket. * each rule also needs a mapping bucket.
*/ */
rules = ilog2(desc->field_len[i] * BITS_PER_BYTE) * 2; rules = ilog2(desc->field_len[i] * BITS_PER_BYTE) * 2;
entry_size += rules * NFT_PIPAPO_BUCKETS / BITS_PER_BYTE; entry_size += rules *
NFT_PIPAPO_BUCKETS(NFT_PIPAPO_GROUP_BITS) /
BITS_PER_BYTE;
entry_size += rules * sizeof(union nft_pipapo_map_bucket); entry_size += rules * sizeof(union nft_pipapo_map_bucket);
} }
@ -1985,8 +2001,8 @@ static int nft_pipapo_init(const struct nft_set *set,
rcu_head_init(&m->rcu); rcu_head_init(&m->rcu);
nft_pipapo_for_each_field(f, i, m) { nft_pipapo_for_each_field(f, i, m) {
f->groups = desc->field_len[i] * NFT_PIPAPO_GROUPS_PER_BYTE; f->bb = NFT_PIPAPO_GROUP_BITS;
priv->groups += f->groups; f->groups = desc->field_len[i] * NFT_PIPAPO_GROUPS_PER_BYTE(f);
priv->width += round_up(desc->field_len[i], sizeof(u32)); priv->width += round_up(desc->field_len[i], sizeof(u32));