2019-05-31 16:09:24 +08:00
|
|
|
// SPDX-License-Identifier: GPL-2.0-only
|
2017-10-16 18:19:28 +08:00
|
|
|
/* bpf/cpumap.c
|
|
|
|
*
|
|
|
|
* Copyright (c) 2017 Jesper Dangaard Brouer, Red Hat Inc.
|
|
|
|
*/
|
|
|
|
|
|
|
|
/* The 'cpumap' is primarily used as a backend map for XDP BPF helper
|
|
|
|
* call bpf_redirect_map() and XDP_REDIRECT action, like 'devmap'.
|
|
|
|
*
|
|
|
|
* Unlike devmap which redirects XDP frames out another NIC device,
|
|
|
|
* this map type redirects raw XDP frames to another CPU. The remote
|
|
|
|
* CPU will do SKB-allocation and call the normal network stack.
|
|
|
|
*
|
|
|
|
* This is a scalability and isolation mechanism, that allow
|
|
|
|
* separating the early driver network XDP layer, from the rest of the
|
|
|
|
* netstack, and assigning dedicated CPUs for this stage. This
|
|
|
|
* basically allows for 10G wirespeed pre-filtering via bpf.
|
|
|
|
*/
|
|
|
|
#include <linux/bpf.h>
|
|
|
|
#include <linux/filter.h>
|
|
|
|
#include <linux/ptr_ring.h>
|
2018-04-17 22:45:26 +08:00
|
|
|
#include <net/xdp.h>
|
2017-10-16 18:19:28 +08:00
|
|
|
|
|
|
|
#include <linux/sched.h>
|
|
|
|
#include <linux/workqueue.h>
|
|
|
|
#include <linux/kthread.h>
|
|
|
|
#include <linux/capability.h>
|
2017-10-16 18:19:44 +08:00
|
|
|
#include <trace/events/xdp.h>
|
2017-10-16 18:19:28 +08:00
|
|
|
|
bpf: cpumap xdp_buff to skb conversion and allocation
This patch makes cpumap functional, by adding SKB allocation and
invoking the network stack on the dequeuing CPU.
For constructing the SKB on the remote CPU, the xdp_buff in converted
into a struct xdp_pkt, and it mapped into the top headroom of the
packet, to avoid allocating separate mem. For now, struct xdp_pkt is
just a cpumap internal data structure, with info carried between
enqueue to dequeue.
If a driver doesn't have enough headroom it is simply dropped, with
return code -EOVERFLOW. This will be picked up the xdp tracepoint
infrastructure, to allow users to catch this.
V2: take into account xdp->data_meta
V4:
- Drop busypoll tricks, keeping it more simple.
- Skip RPS and Generic-XDP-recursive-reinjection, suggested by Alexei
V5: correct RCU read protection around __netif_receive_skb_core.
V6: Setting TASK_RUNNING vs TASK_INTERRUPTIBLE based on talk with Rik van Riel
Signed-off-by: Jesper Dangaard Brouer <brouer@redhat.com>
Signed-off-by: David S. Miller <davem@davemloft.net>
2017-10-16 18:19:39 +08:00
|
|
|
#include <linux/netdevice.h> /* netif_receive_skb_core */
|
|
|
|
#include <linux/etherdevice.h> /* eth_type_trans */
|
|
|
|
|
2017-10-16 18:19:28 +08:00
|
|
|
/* General idea: XDP packets getting XDP redirected to another CPU,
|
|
|
|
* will maximum be stored/queued for one driver ->poll() call. It is
|
2019-06-28 17:12:34 +08:00
|
|
|
* guaranteed that queueing the frame and the flush operation happen on
|
2017-10-16 18:19:28 +08:00
|
|
|
* same CPU. Thus, cpu_map_flush operation can deduct via this_cpu_ptr()
|
|
|
|
* which queue in bpf_cpu_map_entry contains packets.
|
|
|
|
*/
|
|
|
|
|
|
|
|
#define CPU_MAP_BULK_SIZE 8 /* 8 == one cacheline on 64-bit archs */
|
2019-06-28 17:12:34 +08:00
|
|
|
struct bpf_cpu_map_entry;
|
|
|
|
struct bpf_cpu_map;
|
|
|
|
|
2017-10-16 18:19:28 +08:00
|
|
|
struct xdp_bulk_queue {
|
|
|
|
void *q[CPU_MAP_BULK_SIZE];
|
2019-06-28 17:12:34 +08:00
|
|
|
struct list_head flush_node;
|
|
|
|
struct bpf_cpu_map_entry *obj;
|
2017-10-16 18:19:28 +08:00
|
|
|
unsigned int count;
|
|
|
|
};
|
|
|
|
|
|
|
|
/* Struct for every remote "destination" CPU in map */
|
|
|
|
struct bpf_cpu_map_entry {
|
2017-10-16 18:19:44 +08:00
|
|
|
u32 cpu; /* kthread CPU and map index */
|
|
|
|
int map_id; /* Back reference to map */
|
2017-10-16 18:19:28 +08:00
|
|
|
|
|
|
|
/* XDP can run multiple RX-ring queues, need __percpu enqueue store */
|
|
|
|
struct xdp_bulk_queue __percpu *bulkq;
|
|
|
|
|
2019-06-28 17:12:34 +08:00
|
|
|
struct bpf_cpu_map *cmap;
|
|
|
|
|
2017-10-16 18:19:28 +08:00
|
|
|
/* Queue with potential multi-producers, and single-consumer kthread */
|
|
|
|
struct ptr_ring *queue;
|
|
|
|
struct task_struct *kthread;
|
2020-07-14 21:56:37 +08:00
|
|
|
|
|
|
|
struct bpf_cpumap_val value;
|
2020-07-14 21:56:38 +08:00
|
|
|
struct bpf_prog *prog;
|
2017-10-16 18:19:28 +08:00
|
|
|
|
|
|
|
atomic_t refcnt; /* Control when this struct can be free'ed */
|
|
|
|
struct rcu_head rcu;
|
2020-07-14 21:56:37 +08:00
|
|
|
|
|
|
|
struct work_struct kthread_stop_wq;
|
2017-10-16 18:19:28 +08:00
|
|
|
};
|
|
|
|
|
|
|
|
struct bpf_cpu_map {
|
|
|
|
struct bpf_map map;
|
|
|
|
/* Below members specific for map type */
|
|
|
|
struct bpf_cpu_map_entry **cpu_map;
|
|
|
|
};
|
|
|
|
|
2019-12-19 14:10:04 +08:00
|
|
|
static DEFINE_PER_CPU(struct list_head, cpu_map_flush_list);
|
|
|
|
|
2019-12-19 14:10:00 +08:00
|
|
|
static int bq_flush_to_queue(struct xdp_bulk_queue *bq);
|
2017-10-16 18:19:28 +08:00
|
|
|
|
|
|
|
static struct bpf_map *cpu_map_alloc(union bpf_attr *attr)
|
|
|
|
{
|
2020-07-14 21:56:38 +08:00
|
|
|
u32 value_size = attr->value_size;
|
2017-10-16 18:19:28 +08:00
|
|
|
struct bpf_cpu_map *cmap;
|
|
|
|
int err = -ENOMEM;
|
|
|
|
u64 cost;
|
2019-12-19 14:10:04 +08:00
|
|
|
int ret;
|
2017-10-16 18:19:28 +08:00
|
|
|
|
2020-05-14 07:03:54 +08:00
|
|
|
if (!bpf_capable())
|
2017-10-16 18:19:28 +08:00
|
|
|
return ERR_PTR(-EPERM);
|
|
|
|
|
|
|
|
/* check sanity of attributes */
|
|
|
|
if (attr->max_entries == 0 || attr->key_size != 4 ||
|
2020-07-14 21:56:38 +08:00
|
|
|
(value_size != offsetofend(struct bpf_cpumap_val, qsize) &&
|
|
|
|
value_size != offsetofend(struct bpf_cpumap_val, bpf_prog.fd)) ||
|
|
|
|
attr->map_flags & ~BPF_F_NUMA_NODE)
|
2017-10-16 18:19:28 +08:00
|
|
|
return ERR_PTR(-EINVAL);
|
|
|
|
|
|
|
|
cmap = kzalloc(sizeof(*cmap), GFP_USER);
|
|
|
|
if (!cmap)
|
|
|
|
return ERR_PTR(-ENOMEM);
|
|
|
|
|
2018-01-12 12:29:06 +08:00
|
|
|
bpf_map_init_from_attr(&cmap->map, attr);
|
2017-10-16 18:19:28 +08:00
|
|
|
|
|
|
|
/* Pre-limit array size based on NR_CPUS, not final CPU check */
|
|
|
|
if (cmap->map.max_entries > NR_CPUS) {
|
|
|
|
err = -E2BIG;
|
|
|
|
goto free_cmap;
|
|
|
|
}
|
|
|
|
|
|
|
|
/* make sure page count doesn't overflow */
|
|
|
|
cost = (u64) cmap->map.max_entries * sizeof(struct bpf_cpu_map_entry *);
|
|
|
|
|
|
|
|
/* Notice returns -EPERM on if map size is larger than memlock limit */
|
2019-05-30 09:03:59 +08:00
|
|
|
ret = bpf_map_charge_init(&cmap->map.memory, cost);
|
2017-10-16 18:19:28 +08:00
|
|
|
if (ret) {
|
|
|
|
err = ret;
|
|
|
|
goto free_cmap;
|
|
|
|
}
|
|
|
|
|
|
|
|
/* Alloc array for possible remote "destination" CPUs */
|
|
|
|
cmap->cpu_map = bpf_map_area_alloc(cmap->map.max_entries *
|
|
|
|
sizeof(struct bpf_cpu_map_entry *),
|
|
|
|
cmap->map.numa_node);
|
|
|
|
if (!cmap->cpu_map)
|
2019-12-19 14:10:04 +08:00
|
|
|
goto free_charge;
|
2017-10-16 18:19:28 +08:00
|
|
|
|
|
|
|
return &cmap->map;
|
2019-05-30 09:03:58 +08:00
|
|
|
free_charge:
|
|
|
|
bpf_map_charge_finish(&cmap->map.memory);
|
2017-10-16 18:19:28 +08:00
|
|
|
free_cmap:
|
|
|
|
kfree(cmap);
|
|
|
|
return ERR_PTR(err);
|
|
|
|
}
|
|
|
|
|
|
|
|
static void get_cpu_map_entry(struct bpf_cpu_map_entry *rcpu)
|
|
|
|
{
|
|
|
|
atomic_inc(&rcpu->refcnt);
|
|
|
|
}
|
|
|
|
|
|
|
|
/* called from workqueue, to workaround syscall using preempt_disable */
|
|
|
|
static void cpu_map_kthread_stop(struct work_struct *work)
|
|
|
|
{
|
|
|
|
struct bpf_cpu_map_entry *rcpu;
|
|
|
|
|
|
|
|
rcpu = container_of(work, struct bpf_cpu_map_entry, kthread_stop_wq);
|
|
|
|
|
|
|
|
/* Wait for flush in __cpu_map_entry_free(), via full RCU barrier,
|
|
|
|
* as it waits until all in-flight call_rcu() callbacks complete.
|
|
|
|
*/
|
|
|
|
rcu_barrier();
|
|
|
|
|
|
|
|
/* kthread_stop will wake_up_process and wait for it to complete */
|
|
|
|
kthread_stop(rcpu->kthread);
|
|
|
|
}
|
|
|
|
|
2018-01-16 19:27:05 +08:00
|
|
|
static struct sk_buff *cpu_map_build_skb(struct bpf_cpu_map_entry *rcpu,
|
2019-04-12 23:07:43 +08:00
|
|
|
struct xdp_frame *xdpf,
|
|
|
|
struct sk_buff *skb)
|
bpf: cpumap xdp_buff to skb conversion and allocation
This patch makes cpumap functional, by adding SKB allocation and
invoking the network stack on the dequeuing CPU.
For constructing the SKB on the remote CPU, the xdp_buff in converted
into a struct xdp_pkt, and it mapped into the top headroom of the
packet, to avoid allocating separate mem. For now, struct xdp_pkt is
just a cpumap internal data structure, with info carried between
enqueue to dequeue.
If a driver doesn't have enough headroom it is simply dropped, with
return code -EOVERFLOW. This will be picked up the xdp tracepoint
infrastructure, to allow users to catch this.
V2: take into account xdp->data_meta
V4:
- Drop busypoll tricks, keeping it more simple.
- Skip RPS and Generic-XDP-recursive-reinjection, suggested by Alexei
V5: correct RCU read protection around __netif_receive_skb_core.
V6: Setting TASK_RUNNING vs TASK_INTERRUPTIBLE based on talk with Rik van Riel
Signed-off-by: Jesper Dangaard Brouer <brouer@redhat.com>
Signed-off-by: David S. Miller <davem@davemloft.net>
2017-10-16 18:19:39 +08:00
|
|
|
{
|
xdp: fix cpumap redirect SKB creation bug
We want to avoid leaking pointer info from xdp_frame (that is placed in
top of frame) like commit 6dfb970d3dbd ("xdp: avoid leaking info stored in
frame data on page reuse"), and followup commit 97e19cce05e5 ("bpf:
reserve xdp_frame size in xdp headroom") that reserve this headroom.
These changes also affected how cpumap constructed SKBs, as xdpf->headroom
size changed, the skb data starting point were in-effect shifted with 32
bytes (sizeof xdp_frame). This was still okay, as the cpumap frame_size
calculation also included xdpf->headroom which were reduced by same amount.
A bug was introduced in commit 77ea5f4cbe20 ("bpf/cpumap: make sure
frame_size for build_skb is aligned if headroom isn't"), where the
xdpf->headroom became part of the SKB_DATA_ALIGN rounding up. This
round-up to find the frame_size is in principle still correct as it does
not exceed the 2048 bytes frame_size (which is max for ixgbe and i40e),
but the 32 bytes offset of pkt_data_start puts this over the 2048 bytes
limit. This cause skb_shared_info to spill into next frame. It is a little
hard to trigger, as the SKB need to use above 15 skb_shinfo->frags[] as
far as I calculate. This does happen in practise for TCP streams when
skb_try_coalesce() kicks in.
KASAN can be used to detect these wrong memory accesses, I've seen:
BUG: KASAN: use-after-free in skb_try_coalesce+0x3cb/0x760
BUG: KASAN: wild-memory-access in skb_release_data+0xe2/0x250
Driver veth also construct a SKB from xdp_frame in this way, but is not
affected, as it doesn't reserve/deduct the room (used by xdp_frame) from
the SKB headroom. Instead is clears the pointers via xdp_scrub_frame(),
and allows SKB to use this area.
The fix in this patch is to do like veth and instead allow SKB to (re)use
the area occupied by xdp_frame, by clearing via xdp_scrub_frame(). (This
does kill the idea of the SKB being able to access (mem) info from this
area, but I guess it was a bad idea anyhow, and it was already killed by
the veth changes.)
Fixes: 77ea5f4cbe20 ("bpf/cpumap: make sure frame_size for build_skb is aligned if headroom isn't")
Signed-off-by: Jesper Dangaard Brouer <brouer@redhat.com>
Signed-off-by: Alexei Starovoitov <ast@kernel.org>
2019-03-29 17:18:00 +08:00
|
|
|
unsigned int hard_start_headroom;
|
bpf: cpumap xdp_buff to skb conversion and allocation
This patch makes cpumap functional, by adding SKB allocation and
invoking the network stack on the dequeuing CPU.
For constructing the SKB on the remote CPU, the xdp_buff in converted
into a struct xdp_pkt, and it mapped into the top headroom of the
packet, to avoid allocating separate mem. For now, struct xdp_pkt is
just a cpumap internal data structure, with info carried between
enqueue to dequeue.
If a driver doesn't have enough headroom it is simply dropped, with
return code -EOVERFLOW. This will be picked up the xdp tracepoint
infrastructure, to allow users to catch this.
V2: take into account xdp->data_meta
V4:
- Drop busypoll tricks, keeping it more simple.
- Skip RPS and Generic-XDP-recursive-reinjection, suggested by Alexei
V5: correct RCU read protection around __netif_receive_skb_core.
V6: Setting TASK_RUNNING vs TASK_INTERRUPTIBLE based on talk with Rik van Riel
Signed-off-by: Jesper Dangaard Brouer <brouer@redhat.com>
Signed-off-by: David S. Miller <davem@davemloft.net>
2017-10-16 18:19:39 +08:00
|
|
|
unsigned int frame_size;
|
|
|
|
void *pkt_data_start;
|
|
|
|
|
xdp: fix cpumap redirect SKB creation bug
We want to avoid leaking pointer info from xdp_frame (that is placed in
top of frame) like commit 6dfb970d3dbd ("xdp: avoid leaking info stored in
frame data on page reuse"), and followup commit 97e19cce05e5 ("bpf:
reserve xdp_frame size in xdp headroom") that reserve this headroom.
These changes also affected how cpumap constructed SKBs, as xdpf->headroom
size changed, the skb data starting point were in-effect shifted with 32
bytes (sizeof xdp_frame). This was still okay, as the cpumap frame_size
calculation also included xdpf->headroom which were reduced by same amount.
A bug was introduced in commit 77ea5f4cbe20 ("bpf/cpumap: make sure
frame_size for build_skb is aligned if headroom isn't"), where the
xdpf->headroom became part of the SKB_DATA_ALIGN rounding up. This
round-up to find the frame_size is in principle still correct as it does
not exceed the 2048 bytes frame_size (which is max for ixgbe and i40e),
but the 32 bytes offset of pkt_data_start puts this over the 2048 bytes
limit. This cause skb_shared_info to spill into next frame. It is a little
hard to trigger, as the SKB need to use above 15 skb_shinfo->frags[] as
far as I calculate. This does happen in practise for TCP streams when
skb_try_coalesce() kicks in.
KASAN can be used to detect these wrong memory accesses, I've seen:
BUG: KASAN: use-after-free in skb_try_coalesce+0x3cb/0x760
BUG: KASAN: wild-memory-access in skb_release_data+0xe2/0x250
Driver veth also construct a SKB from xdp_frame in this way, but is not
affected, as it doesn't reserve/deduct the room (used by xdp_frame) from
the SKB headroom. Instead is clears the pointers via xdp_scrub_frame(),
and allows SKB to use this area.
The fix in this patch is to do like veth and instead allow SKB to (re)use
the area occupied by xdp_frame, by clearing via xdp_scrub_frame(). (This
does kill the idea of the SKB being able to access (mem) info from this
area, but I guess it was a bad idea anyhow, and it was already killed by
the veth changes.)
Fixes: 77ea5f4cbe20 ("bpf/cpumap: make sure frame_size for build_skb is aligned if headroom isn't")
Signed-off-by: Jesper Dangaard Brouer <brouer@redhat.com>
Signed-off-by: Alexei Starovoitov <ast@kernel.org>
2019-03-29 17:18:00 +08:00
|
|
|
/* Part of headroom was reserved to xdpf */
|
|
|
|
hard_start_headroom = sizeof(struct xdp_frame) + xdpf->headroom;
|
|
|
|
|
2020-05-14 18:49:38 +08:00
|
|
|
/* Memory size backing xdp_frame data already have reserved
|
|
|
|
* room for build_skb to place skb_shared_info in tailroom.
|
bpf: cpumap xdp_buff to skb conversion and allocation
This patch makes cpumap functional, by adding SKB allocation and
invoking the network stack on the dequeuing CPU.
For constructing the SKB on the remote CPU, the xdp_buff in converted
into a struct xdp_pkt, and it mapped into the top headroom of the
packet, to avoid allocating separate mem. For now, struct xdp_pkt is
just a cpumap internal data structure, with info carried between
enqueue to dequeue.
If a driver doesn't have enough headroom it is simply dropped, with
return code -EOVERFLOW. This will be picked up the xdp tracepoint
infrastructure, to allow users to catch this.
V2: take into account xdp->data_meta
V4:
- Drop busypoll tricks, keeping it more simple.
- Skip RPS and Generic-XDP-recursive-reinjection, suggested by Alexei
V5: correct RCU read protection around __netif_receive_skb_core.
V6: Setting TASK_RUNNING vs TASK_INTERRUPTIBLE based on talk with Rik van Riel
Signed-off-by: Jesper Dangaard Brouer <brouer@redhat.com>
Signed-off-by: David S. Miller <davem@davemloft.net>
2017-10-16 18:19:39 +08:00
|
|
|
*/
|
2020-05-14 18:49:38 +08:00
|
|
|
frame_size = xdpf->frame_sz;
|
bpf: cpumap xdp_buff to skb conversion and allocation
This patch makes cpumap functional, by adding SKB allocation and
invoking the network stack on the dequeuing CPU.
For constructing the SKB on the remote CPU, the xdp_buff in converted
into a struct xdp_pkt, and it mapped into the top headroom of the
packet, to avoid allocating separate mem. For now, struct xdp_pkt is
just a cpumap internal data structure, with info carried between
enqueue to dequeue.
If a driver doesn't have enough headroom it is simply dropped, with
return code -EOVERFLOW. This will be picked up the xdp tracepoint
infrastructure, to allow users to catch this.
V2: take into account xdp->data_meta
V4:
- Drop busypoll tricks, keeping it more simple.
- Skip RPS and Generic-XDP-recursive-reinjection, suggested by Alexei
V5: correct RCU read protection around __netif_receive_skb_core.
V6: Setting TASK_RUNNING vs TASK_INTERRUPTIBLE based on talk with Rik van Riel
Signed-off-by: Jesper Dangaard Brouer <brouer@redhat.com>
Signed-off-by: David S. Miller <davem@davemloft.net>
2017-10-16 18:19:39 +08:00
|
|
|
|
xdp: fix cpumap redirect SKB creation bug
We want to avoid leaking pointer info from xdp_frame (that is placed in
top of frame) like commit 6dfb970d3dbd ("xdp: avoid leaking info stored in
frame data on page reuse"), and followup commit 97e19cce05e5 ("bpf:
reserve xdp_frame size in xdp headroom") that reserve this headroom.
These changes also affected how cpumap constructed SKBs, as xdpf->headroom
size changed, the skb data starting point were in-effect shifted with 32
bytes (sizeof xdp_frame). This was still okay, as the cpumap frame_size
calculation also included xdpf->headroom which were reduced by same amount.
A bug was introduced in commit 77ea5f4cbe20 ("bpf/cpumap: make sure
frame_size for build_skb is aligned if headroom isn't"), where the
xdpf->headroom became part of the SKB_DATA_ALIGN rounding up. This
round-up to find the frame_size is in principle still correct as it does
not exceed the 2048 bytes frame_size (which is max for ixgbe and i40e),
but the 32 bytes offset of pkt_data_start puts this over the 2048 bytes
limit. This cause skb_shared_info to spill into next frame. It is a little
hard to trigger, as the SKB need to use above 15 skb_shinfo->frags[] as
far as I calculate. This does happen in practise for TCP streams when
skb_try_coalesce() kicks in.
KASAN can be used to detect these wrong memory accesses, I've seen:
BUG: KASAN: use-after-free in skb_try_coalesce+0x3cb/0x760
BUG: KASAN: wild-memory-access in skb_release_data+0xe2/0x250
Driver veth also construct a SKB from xdp_frame in this way, but is not
affected, as it doesn't reserve/deduct the room (used by xdp_frame) from
the SKB headroom. Instead is clears the pointers via xdp_scrub_frame(),
and allows SKB to use this area.
The fix in this patch is to do like veth and instead allow SKB to (re)use
the area occupied by xdp_frame, by clearing via xdp_scrub_frame(). (This
does kill the idea of the SKB being able to access (mem) info from this
area, but I guess it was a bad idea anyhow, and it was already killed by
the veth changes.)
Fixes: 77ea5f4cbe20 ("bpf/cpumap: make sure frame_size for build_skb is aligned if headroom isn't")
Signed-off-by: Jesper Dangaard Brouer <brouer@redhat.com>
Signed-off-by: Alexei Starovoitov <ast@kernel.org>
2019-03-29 17:18:00 +08:00
|
|
|
pkt_data_start = xdpf->data - hard_start_headroom;
|
2019-04-12 23:07:43 +08:00
|
|
|
skb = build_skb_around(skb, pkt_data_start, frame_size);
|
|
|
|
if (unlikely(!skb))
|
bpf: cpumap xdp_buff to skb conversion and allocation
This patch makes cpumap functional, by adding SKB allocation and
invoking the network stack on the dequeuing CPU.
For constructing the SKB on the remote CPU, the xdp_buff in converted
into a struct xdp_pkt, and it mapped into the top headroom of the
packet, to avoid allocating separate mem. For now, struct xdp_pkt is
just a cpumap internal data structure, with info carried between
enqueue to dequeue.
If a driver doesn't have enough headroom it is simply dropped, with
return code -EOVERFLOW. This will be picked up the xdp tracepoint
infrastructure, to allow users to catch this.
V2: take into account xdp->data_meta
V4:
- Drop busypoll tricks, keeping it more simple.
- Skip RPS and Generic-XDP-recursive-reinjection, suggested by Alexei
V5: correct RCU read protection around __netif_receive_skb_core.
V6: Setting TASK_RUNNING vs TASK_INTERRUPTIBLE based on talk with Rik van Riel
Signed-off-by: Jesper Dangaard Brouer <brouer@redhat.com>
Signed-off-by: David S. Miller <davem@davemloft.net>
2017-10-16 18:19:39 +08:00
|
|
|
return NULL;
|
|
|
|
|
xdp: fix cpumap redirect SKB creation bug
We want to avoid leaking pointer info from xdp_frame (that is placed in
top of frame) like commit 6dfb970d3dbd ("xdp: avoid leaking info stored in
frame data on page reuse"), and followup commit 97e19cce05e5 ("bpf:
reserve xdp_frame size in xdp headroom") that reserve this headroom.
These changes also affected how cpumap constructed SKBs, as xdpf->headroom
size changed, the skb data starting point were in-effect shifted with 32
bytes (sizeof xdp_frame). This was still okay, as the cpumap frame_size
calculation also included xdpf->headroom which were reduced by same amount.
A bug was introduced in commit 77ea5f4cbe20 ("bpf/cpumap: make sure
frame_size for build_skb is aligned if headroom isn't"), where the
xdpf->headroom became part of the SKB_DATA_ALIGN rounding up. This
round-up to find the frame_size is in principle still correct as it does
not exceed the 2048 bytes frame_size (which is max for ixgbe and i40e),
but the 32 bytes offset of pkt_data_start puts this over the 2048 bytes
limit. This cause skb_shared_info to spill into next frame. It is a little
hard to trigger, as the SKB need to use above 15 skb_shinfo->frags[] as
far as I calculate. This does happen in practise for TCP streams when
skb_try_coalesce() kicks in.
KASAN can be used to detect these wrong memory accesses, I've seen:
BUG: KASAN: use-after-free in skb_try_coalesce+0x3cb/0x760
BUG: KASAN: wild-memory-access in skb_release_data+0xe2/0x250
Driver veth also construct a SKB from xdp_frame in this way, but is not
affected, as it doesn't reserve/deduct the room (used by xdp_frame) from
the SKB headroom. Instead is clears the pointers via xdp_scrub_frame(),
and allows SKB to use this area.
The fix in this patch is to do like veth and instead allow SKB to (re)use
the area occupied by xdp_frame, by clearing via xdp_scrub_frame(). (This
does kill the idea of the SKB being able to access (mem) info from this
area, but I guess it was a bad idea anyhow, and it was already killed by
the veth changes.)
Fixes: 77ea5f4cbe20 ("bpf/cpumap: make sure frame_size for build_skb is aligned if headroom isn't")
Signed-off-by: Jesper Dangaard Brouer <brouer@redhat.com>
Signed-off-by: Alexei Starovoitov <ast@kernel.org>
2019-03-29 17:18:00 +08:00
|
|
|
skb_reserve(skb, hard_start_headroom);
|
2018-04-17 22:45:57 +08:00
|
|
|
__skb_put(skb, xdpf->len);
|
|
|
|
if (xdpf->metasize)
|
|
|
|
skb_metadata_set(skb, xdpf->metasize);
|
bpf: cpumap xdp_buff to skb conversion and allocation
This patch makes cpumap functional, by adding SKB allocation and
invoking the network stack on the dequeuing CPU.
For constructing the SKB on the remote CPU, the xdp_buff in converted
into a struct xdp_pkt, and it mapped into the top headroom of the
packet, to avoid allocating separate mem. For now, struct xdp_pkt is
just a cpumap internal data structure, with info carried between
enqueue to dequeue.
If a driver doesn't have enough headroom it is simply dropped, with
return code -EOVERFLOW. This will be picked up the xdp tracepoint
infrastructure, to allow users to catch this.
V2: take into account xdp->data_meta
V4:
- Drop busypoll tricks, keeping it more simple.
- Skip RPS and Generic-XDP-recursive-reinjection, suggested by Alexei
V5: correct RCU read protection around __netif_receive_skb_core.
V6: Setting TASK_RUNNING vs TASK_INTERRUPTIBLE based on talk with Rik van Riel
Signed-off-by: Jesper Dangaard Brouer <brouer@redhat.com>
Signed-off-by: David S. Miller <davem@davemloft.net>
2017-10-16 18:19:39 +08:00
|
|
|
|
|
|
|
/* Essential SKB info: protocol and skb->dev */
|
2018-04-17 22:45:57 +08:00
|
|
|
skb->protocol = eth_type_trans(skb, xdpf->dev_rx);
|
bpf: cpumap xdp_buff to skb conversion and allocation
This patch makes cpumap functional, by adding SKB allocation and
invoking the network stack on the dequeuing CPU.
For constructing the SKB on the remote CPU, the xdp_buff in converted
into a struct xdp_pkt, and it mapped into the top headroom of the
packet, to avoid allocating separate mem. For now, struct xdp_pkt is
just a cpumap internal data structure, with info carried between
enqueue to dequeue.
If a driver doesn't have enough headroom it is simply dropped, with
return code -EOVERFLOW. This will be picked up the xdp tracepoint
infrastructure, to allow users to catch this.
V2: take into account xdp->data_meta
V4:
- Drop busypoll tricks, keeping it more simple.
- Skip RPS and Generic-XDP-recursive-reinjection, suggested by Alexei
V5: correct RCU read protection around __netif_receive_skb_core.
V6: Setting TASK_RUNNING vs TASK_INTERRUPTIBLE based on talk with Rik van Riel
Signed-off-by: Jesper Dangaard Brouer <brouer@redhat.com>
Signed-off-by: David S. Miller <davem@davemloft.net>
2017-10-16 18:19:39 +08:00
|
|
|
|
|
|
|
/* Optional SKB info, currently missing:
|
|
|
|
* - HW checksum info (skb->ip_summed)
|
|
|
|
* - HW RX hash (skb_set_hash)
|
|
|
|
* - RX ring dev queue index (skb_record_rx_queue)
|
|
|
|
*/
|
|
|
|
|
2019-06-18 21:05:27 +08:00
|
|
|
/* Until page_pool get SKB return path, release DMA here */
|
|
|
|
xdp_release_frame(xdpf);
|
|
|
|
|
xdp: fix cpumap redirect SKB creation bug
We want to avoid leaking pointer info from xdp_frame (that is placed in
top of frame) like commit 6dfb970d3dbd ("xdp: avoid leaking info stored in
frame data on page reuse"), and followup commit 97e19cce05e5 ("bpf:
reserve xdp_frame size in xdp headroom") that reserve this headroom.
These changes also affected how cpumap constructed SKBs, as xdpf->headroom
size changed, the skb data starting point were in-effect shifted with 32
bytes (sizeof xdp_frame). This was still okay, as the cpumap frame_size
calculation also included xdpf->headroom which were reduced by same amount.
A bug was introduced in commit 77ea5f4cbe20 ("bpf/cpumap: make sure
frame_size for build_skb is aligned if headroom isn't"), where the
xdpf->headroom became part of the SKB_DATA_ALIGN rounding up. This
round-up to find the frame_size is in principle still correct as it does
not exceed the 2048 bytes frame_size (which is max for ixgbe and i40e),
but the 32 bytes offset of pkt_data_start puts this over the 2048 bytes
limit. This cause skb_shared_info to spill into next frame. It is a little
hard to trigger, as the SKB need to use above 15 skb_shinfo->frags[] as
far as I calculate. This does happen in practise for TCP streams when
skb_try_coalesce() kicks in.
KASAN can be used to detect these wrong memory accesses, I've seen:
BUG: KASAN: use-after-free in skb_try_coalesce+0x3cb/0x760
BUG: KASAN: wild-memory-access in skb_release_data+0xe2/0x250
Driver veth also construct a SKB from xdp_frame in this way, but is not
affected, as it doesn't reserve/deduct the room (used by xdp_frame) from
the SKB headroom. Instead is clears the pointers via xdp_scrub_frame(),
and allows SKB to use this area.
The fix in this patch is to do like veth and instead allow SKB to (re)use
the area occupied by xdp_frame, by clearing via xdp_scrub_frame(). (This
does kill the idea of the SKB being able to access (mem) info from this
area, but I guess it was a bad idea anyhow, and it was already killed by
the veth changes.)
Fixes: 77ea5f4cbe20 ("bpf/cpumap: make sure frame_size for build_skb is aligned if headroom isn't")
Signed-off-by: Jesper Dangaard Brouer <brouer@redhat.com>
Signed-off-by: Alexei Starovoitov <ast@kernel.org>
2019-03-29 17:18:00 +08:00
|
|
|
/* Allow SKB to reuse area used by xdp_frame */
|
|
|
|
xdp_scrub_frame(xdpf);
|
|
|
|
|
bpf: cpumap xdp_buff to skb conversion and allocation
This patch makes cpumap functional, by adding SKB allocation and
invoking the network stack on the dequeuing CPU.
For constructing the SKB on the remote CPU, the xdp_buff in converted
into a struct xdp_pkt, and it mapped into the top headroom of the
packet, to avoid allocating separate mem. For now, struct xdp_pkt is
just a cpumap internal data structure, with info carried between
enqueue to dequeue.
If a driver doesn't have enough headroom it is simply dropped, with
return code -EOVERFLOW. This will be picked up the xdp tracepoint
infrastructure, to allow users to catch this.
V2: take into account xdp->data_meta
V4:
- Drop busypoll tricks, keeping it more simple.
- Skip RPS and Generic-XDP-recursive-reinjection, suggested by Alexei
V5: correct RCU read protection around __netif_receive_skb_core.
V6: Setting TASK_RUNNING vs TASK_INTERRUPTIBLE based on talk with Rik van Riel
Signed-off-by: Jesper Dangaard Brouer <brouer@redhat.com>
Signed-off-by: David S. Miller <davem@davemloft.net>
2017-10-16 18:19:39 +08:00
|
|
|
return skb;
|
|
|
|
}
|
|
|
|
|
2018-04-17 22:45:26 +08:00
|
|
|
static void __cpu_map_ring_cleanup(struct ptr_ring *ring)
|
|
|
|
{
|
|
|
|
/* The tear-down procedure should have made sure that queue is
|
|
|
|
* empty. See __cpu_map_entry_replace() and work-queue
|
|
|
|
* invoked cpu_map_kthread_stop(). Catch any broken behaviour
|
|
|
|
* gracefully and warn once.
|
|
|
|
*/
|
2018-04-17 22:45:57 +08:00
|
|
|
struct xdp_frame *xdpf;
|
2018-04-17 22:45:26 +08:00
|
|
|
|
2018-04-17 22:45:57 +08:00
|
|
|
while ((xdpf = ptr_ring_consume(ring)))
|
|
|
|
if (WARN_ON_ONCE(xdpf))
|
xdp: transition into using xdp_frame for return API
Changing API xdp_return_frame() to take struct xdp_frame as argument,
seems like a natural choice. But there are some subtle performance
details here that needs extra care, which is a deliberate choice.
When de-referencing xdp_frame on a remote CPU during DMA-TX
completion, result in the cache-line is change to "Shared"
state. Later when the page is reused for RX, then this xdp_frame
cache-line is written, which change the state to "Modified".
This situation already happens (naturally) for, virtio_net, tun and
cpumap as the xdp_frame pointer is the queued object. In tun and
cpumap, the ptr_ring is used for efficiently transferring cache-lines
(with pointers) between CPUs. Thus, the only option is to
de-referencing xdp_frame.
It is only the ixgbe driver that had an optimization, in which it can
avoid doing the de-reference of xdp_frame. The driver already have
TX-ring queue, which (in case of remote DMA-TX completion) have to be
transferred between CPUs anyhow. In this data area, we stored a
struct xdp_mem_info and a data pointer, which allowed us to avoid
de-referencing xdp_frame.
To compensate for this, a prefetchw is used for telling the cache
coherency protocol about our access pattern. My benchmarks show that
this prefetchw is enough to compensate the ixgbe driver.
V7: Adjust for commit d9314c474d4f ("i40e: add support for XDP_REDIRECT")
V8: Adjust for commit bd658dda4237 ("net/mlx5e: Separate dma base address
and offset in dma_sync call")
Signed-off-by: Jesper Dangaard Brouer <brouer@redhat.com>
Signed-off-by: David S. Miller <davem@davemloft.net>
2018-04-17 22:46:32 +08:00
|
|
|
xdp_return_frame(xdpf);
|
2018-04-17 22:45:26 +08:00
|
|
|
}
|
|
|
|
|
|
|
|
static void put_cpu_map_entry(struct bpf_cpu_map_entry *rcpu)
|
|
|
|
{
|
|
|
|
if (atomic_dec_and_test(&rcpu->refcnt)) {
|
2020-07-14 21:56:38 +08:00
|
|
|
if (rcpu->prog)
|
|
|
|
bpf_prog_put(rcpu->prog);
|
2018-04-17 22:45:26 +08:00
|
|
|
/* The queue should be empty at this point */
|
|
|
|
__cpu_map_ring_cleanup(rcpu->queue);
|
|
|
|
ptr_ring_cleanup(rcpu->queue, NULL);
|
|
|
|
kfree(rcpu->queue);
|
|
|
|
kfree(rcpu);
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2020-07-14 21:56:38 +08:00
|
|
|
static int cpu_map_bpf_prog_run_xdp(struct bpf_cpu_map_entry *rcpu,
|
|
|
|
void **frames, int n,
|
|
|
|
struct xdp_cpumap_stats *stats)
|
|
|
|
{
|
|
|
|
struct xdp_rxq_info rxq;
|
|
|
|
struct xdp_buff xdp;
|
|
|
|
int i, nframes = 0;
|
|
|
|
|
|
|
|
if (!rcpu->prog)
|
|
|
|
return n;
|
|
|
|
|
|
|
|
rcu_read_lock();
|
|
|
|
|
|
|
|
xdp_set_return_frame_no_direct();
|
|
|
|
xdp.rxq = &rxq;
|
|
|
|
|
|
|
|
for (i = 0; i < n; i++) {
|
|
|
|
struct xdp_frame *xdpf = frames[i];
|
|
|
|
u32 act;
|
|
|
|
int err;
|
|
|
|
|
|
|
|
rxq.dev = xdpf->dev_rx;
|
|
|
|
rxq.mem = xdpf->mem;
|
|
|
|
/* TODO: report queue_index to xdp_rxq_info */
|
|
|
|
|
|
|
|
xdp_convert_frame_to_buff(xdpf, &xdp);
|
|
|
|
|
|
|
|
act = bpf_prog_run_xdp(rcpu->prog, &xdp);
|
|
|
|
switch (act) {
|
|
|
|
case XDP_PASS:
|
|
|
|
err = xdp_update_frame_from_buff(&xdp, xdpf);
|
|
|
|
if (err < 0) {
|
|
|
|
xdp_return_frame(xdpf);
|
|
|
|
stats->drop++;
|
|
|
|
} else {
|
|
|
|
frames[nframes++] = xdpf;
|
|
|
|
stats->pass++;
|
|
|
|
}
|
|
|
|
break;
|
|
|
|
default:
|
|
|
|
bpf_warn_invalid_xdp_action(act);
|
|
|
|
/* fallthrough */
|
|
|
|
case XDP_DROP:
|
|
|
|
xdp_return_frame(xdpf);
|
|
|
|
stats->drop++;
|
|
|
|
break;
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
xdp_clear_return_frame_no_direct();
|
|
|
|
|
|
|
|
rcu_read_unlock();
|
|
|
|
|
|
|
|
return nframes;
|
|
|
|
}
|
|
|
|
|
2019-04-12 23:07:32 +08:00
|
|
|
#define CPUMAP_BATCH 8
|
|
|
|
|
2017-10-16 18:19:28 +08:00
|
|
|
static int cpu_map_kthread_run(void *data)
|
|
|
|
{
|
|
|
|
struct bpf_cpu_map_entry *rcpu = data;
|
|
|
|
|
|
|
|
set_current_state(TASK_INTERRUPTIBLE);
|
|
|
|
|
|
|
|
/* When kthread gives stop order, then rcpu have been disconnected
|
|
|
|
* from map, thus no new packets can enter. Remaining in-flight
|
|
|
|
* per CPU stored packets are flushed to this queue. Wait honoring
|
|
|
|
* kthread_stop signal until queue is empty.
|
|
|
|
*/
|
|
|
|
while (!kthread_should_stop() || !__ptr_ring_empty(rcpu->queue)) {
|
2020-07-14 21:56:38 +08:00
|
|
|
struct xdp_cpumap_stats stats = {}; /* zero stats */
|
|
|
|
gfp_t gfp = __GFP_ZERO | GFP_ATOMIC;
|
2019-04-12 23:07:32 +08:00
|
|
|
unsigned int drops = 0, sched = 0;
|
|
|
|
void *frames[CPUMAP_BATCH];
|
2019-04-12 23:07:43 +08:00
|
|
|
void *skbs[CPUMAP_BATCH];
|
2020-07-14 21:56:38 +08:00
|
|
|
int i, n, m, nframes;
|
2017-10-16 18:19:28 +08:00
|
|
|
|
bpf: cpumap xdp_buff to skb conversion and allocation
This patch makes cpumap functional, by adding SKB allocation and
invoking the network stack on the dequeuing CPU.
For constructing the SKB on the remote CPU, the xdp_buff in converted
into a struct xdp_pkt, and it mapped into the top headroom of the
packet, to avoid allocating separate mem. For now, struct xdp_pkt is
just a cpumap internal data structure, with info carried between
enqueue to dequeue.
If a driver doesn't have enough headroom it is simply dropped, with
return code -EOVERFLOW. This will be picked up the xdp tracepoint
infrastructure, to allow users to catch this.
V2: take into account xdp->data_meta
V4:
- Drop busypoll tricks, keeping it more simple.
- Skip RPS and Generic-XDP-recursive-reinjection, suggested by Alexei
V5: correct RCU read protection around __netif_receive_skb_core.
V6: Setting TASK_RUNNING vs TASK_INTERRUPTIBLE based on talk with Rik van Riel
Signed-off-by: Jesper Dangaard Brouer <brouer@redhat.com>
Signed-off-by: David S. Miller <davem@davemloft.net>
2017-10-16 18:19:39 +08:00
|
|
|
/* Release CPU reschedule checks */
|
|
|
|
if (__ptr_ring_empty(rcpu->queue)) {
|
2017-10-24 01:39:28 +08:00
|
|
|
set_current_state(TASK_INTERRUPTIBLE);
|
|
|
|
/* Recheck to avoid lost wake-up */
|
|
|
|
if (__ptr_ring_empty(rcpu->queue)) {
|
|
|
|
schedule();
|
|
|
|
sched = 1;
|
|
|
|
} else {
|
|
|
|
__set_current_state(TASK_RUNNING);
|
|
|
|
}
|
bpf: cpumap xdp_buff to skb conversion and allocation
This patch makes cpumap functional, by adding SKB allocation and
invoking the network stack on the dequeuing CPU.
For constructing the SKB on the remote CPU, the xdp_buff in converted
into a struct xdp_pkt, and it mapped into the top headroom of the
packet, to avoid allocating separate mem. For now, struct xdp_pkt is
just a cpumap internal data structure, with info carried between
enqueue to dequeue.
If a driver doesn't have enough headroom it is simply dropped, with
return code -EOVERFLOW. This will be picked up the xdp tracepoint
infrastructure, to allow users to catch this.
V2: take into account xdp->data_meta
V4:
- Drop busypoll tricks, keeping it more simple.
- Skip RPS and Generic-XDP-recursive-reinjection, suggested by Alexei
V5: correct RCU read protection around __netif_receive_skb_core.
V6: Setting TASK_RUNNING vs TASK_INTERRUPTIBLE based on talk with Rik van Riel
Signed-off-by: Jesper Dangaard Brouer <brouer@redhat.com>
Signed-off-by: David S. Miller <davem@davemloft.net>
2017-10-16 18:19:39 +08:00
|
|
|
} else {
|
2017-10-16 18:19:44 +08:00
|
|
|
sched = cond_resched();
|
2017-10-16 18:19:28 +08:00
|
|
|
}
|
bpf: cpumap xdp_buff to skb conversion and allocation
This patch makes cpumap functional, by adding SKB allocation and
invoking the network stack on the dequeuing CPU.
For constructing the SKB on the remote CPU, the xdp_buff in converted
into a struct xdp_pkt, and it mapped into the top headroom of the
packet, to avoid allocating separate mem. For now, struct xdp_pkt is
just a cpumap internal data structure, with info carried between
enqueue to dequeue.
If a driver doesn't have enough headroom it is simply dropped, with
return code -EOVERFLOW. This will be picked up the xdp tracepoint
infrastructure, to allow users to catch this.
V2: take into account xdp->data_meta
V4:
- Drop busypoll tricks, keeping it more simple.
- Skip RPS and Generic-XDP-recursive-reinjection, suggested by Alexei
V5: correct RCU read protection around __netif_receive_skb_core.
V6: Setting TASK_RUNNING vs TASK_INTERRUPTIBLE based on talk with Rik van Riel
Signed-off-by: Jesper Dangaard Brouer <brouer@redhat.com>
Signed-off-by: David S. Miller <davem@davemloft.net>
2017-10-16 18:19:39 +08:00
|
|
|
|
|
|
|
/*
|
|
|
|
* The bpf_cpu_map_entry is single consumer, with this
|
|
|
|
* kthread CPU pinned. Lockless access to ptr_ring
|
|
|
|
* consume side valid as no-resize allowed of queue.
|
|
|
|
*/
|
2020-07-14 21:56:38 +08:00
|
|
|
n = __ptr_ring_consume_batched(rcpu->queue, frames,
|
|
|
|
CPUMAP_BATCH);
|
2019-04-12 23:07:48 +08:00
|
|
|
for (i = 0; i < n; i++) {
|
|
|
|
void *f = frames[i];
|
|
|
|
struct page *page = virt_to_page(f);
|
|
|
|
|
|
|
|
/* Bring struct page memory area to curr CPU. Read by
|
|
|
|
* build_skb_around via page_is_pfmemalloc(), and when
|
|
|
|
* freed written by page_frag_free call.
|
|
|
|
*/
|
|
|
|
prefetchw(page);
|
|
|
|
}
|
|
|
|
|
2020-07-14 21:56:38 +08:00
|
|
|
/* Support running another XDP prog on this CPU */
|
|
|
|
nframes = cpu_map_bpf_prog_run_xdp(rcpu, frames, n, &stats);
|
|
|
|
if (nframes) {
|
|
|
|
m = kmem_cache_alloc_bulk(skbuff_head_cache, gfp, nframes, skbs);
|
|
|
|
if (unlikely(m == 0)) {
|
|
|
|
for (i = 0; i < nframes; i++)
|
|
|
|
skbs[i] = NULL; /* effect: xdp_return_frame */
|
|
|
|
drops += nframes;
|
|
|
|
}
|
2019-04-12 23:07:43 +08:00
|
|
|
}
|
2019-04-12 23:07:32 +08:00
|
|
|
|
|
|
|
local_bh_disable();
|
2020-07-14 21:56:38 +08:00
|
|
|
for (i = 0; i < nframes; i++) {
|
2019-04-12 23:07:32 +08:00
|
|
|
struct xdp_frame *xdpf = frames[i];
|
2019-04-12 23:07:43 +08:00
|
|
|
struct sk_buff *skb = skbs[i];
|
bpf: cpumap xdp_buff to skb conversion and allocation
This patch makes cpumap functional, by adding SKB allocation and
invoking the network stack on the dequeuing CPU.
For constructing the SKB on the remote CPU, the xdp_buff in converted
into a struct xdp_pkt, and it mapped into the top headroom of the
packet, to avoid allocating separate mem. For now, struct xdp_pkt is
just a cpumap internal data structure, with info carried between
enqueue to dequeue.
If a driver doesn't have enough headroom it is simply dropped, with
return code -EOVERFLOW. This will be picked up the xdp tracepoint
infrastructure, to allow users to catch this.
V2: take into account xdp->data_meta
V4:
- Drop busypoll tricks, keeping it more simple.
- Skip RPS and Generic-XDP-recursive-reinjection, suggested by Alexei
V5: correct RCU read protection around __netif_receive_skb_core.
V6: Setting TASK_RUNNING vs TASK_INTERRUPTIBLE based on talk with Rik van Riel
Signed-off-by: Jesper Dangaard Brouer <brouer@redhat.com>
Signed-off-by: David S. Miller <davem@davemloft.net>
2017-10-16 18:19:39 +08:00
|
|
|
int ret;
|
|
|
|
|
2019-04-12 23:07:43 +08:00
|
|
|
skb = cpu_map_build_skb(rcpu, xdpf, skb);
|
bpf: cpumap xdp_buff to skb conversion and allocation
This patch makes cpumap functional, by adding SKB allocation and
invoking the network stack on the dequeuing CPU.
For constructing the SKB on the remote CPU, the xdp_buff in converted
into a struct xdp_pkt, and it mapped into the top headroom of the
packet, to avoid allocating separate mem. For now, struct xdp_pkt is
just a cpumap internal data structure, with info carried between
enqueue to dequeue.
If a driver doesn't have enough headroom it is simply dropped, with
return code -EOVERFLOW. This will be picked up the xdp tracepoint
infrastructure, to allow users to catch this.
V2: take into account xdp->data_meta
V4:
- Drop busypoll tricks, keeping it more simple.
- Skip RPS and Generic-XDP-recursive-reinjection, suggested by Alexei
V5: correct RCU read protection around __netif_receive_skb_core.
V6: Setting TASK_RUNNING vs TASK_INTERRUPTIBLE based on talk with Rik van Riel
Signed-off-by: Jesper Dangaard Brouer <brouer@redhat.com>
Signed-off-by: David S. Miller <davem@davemloft.net>
2017-10-16 18:19:39 +08:00
|
|
|
if (!skb) {
|
xdp: transition into using xdp_frame for return API
Changing API xdp_return_frame() to take struct xdp_frame as argument,
seems like a natural choice. But there are some subtle performance
details here that needs extra care, which is a deliberate choice.
When de-referencing xdp_frame on a remote CPU during DMA-TX
completion, result in the cache-line is change to "Shared"
state. Later when the page is reused for RX, then this xdp_frame
cache-line is written, which change the state to "Modified".
This situation already happens (naturally) for, virtio_net, tun and
cpumap as the xdp_frame pointer is the queued object. In tun and
cpumap, the ptr_ring is used for efficiently transferring cache-lines
(with pointers) between CPUs. Thus, the only option is to
de-referencing xdp_frame.
It is only the ixgbe driver that had an optimization, in which it can
avoid doing the de-reference of xdp_frame. The driver already have
TX-ring queue, which (in case of remote DMA-TX completion) have to be
transferred between CPUs anyhow. In this data area, we stored a
struct xdp_mem_info and a data pointer, which allowed us to avoid
de-referencing xdp_frame.
To compensate for this, a prefetchw is used for telling the cache
coherency protocol about our access pattern. My benchmarks show that
this prefetchw is enough to compensate the ixgbe driver.
V7: Adjust for commit d9314c474d4f ("i40e: add support for XDP_REDIRECT")
V8: Adjust for commit bd658dda4237 ("net/mlx5e: Separate dma base address
and offset in dma_sync call")
Signed-off-by: Jesper Dangaard Brouer <brouer@redhat.com>
Signed-off-by: David S. Miller <davem@davemloft.net>
2018-04-17 22:46:32 +08:00
|
|
|
xdp_return_frame(xdpf);
|
bpf: cpumap xdp_buff to skb conversion and allocation
This patch makes cpumap functional, by adding SKB allocation and
invoking the network stack on the dequeuing CPU.
For constructing the SKB on the remote CPU, the xdp_buff in converted
into a struct xdp_pkt, and it mapped into the top headroom of the
packet, to avoid allocating separate mem. For now, struct xdp_pkt is
just a cpumap internal data structure, with info carried between
enqueue to dequeue.
If a driver doesn't have enough headroom it is simply dropped, with
return code -EOVERFLOW. This will be picked up the xdp tracepoint
infrastructure, to allow users to catch this.
V2: take into account xdp->data_meta
V4:
- Drop busypoll tricks, keeping it more simple.
- Skip RPS and Generic-XDP-recursive-reinjection, suggested by Alexei
V5: correct RCU read protection around __netif_receive_skb_core.
V6: Setting TASK_RUNNING vs TASK_INTERRUPTIBLE based on talk with Rik van Riel
Signed-off-by: Jesper Dangaard Brouer <brouer@redhat.com>
Signed-off-by: David S. Miller <davem@davemloft.net>
2017-10-16 18:19:39 +08:00
|
|
|
continue;
|
|
|
|
}
|
|
|
|
|
|
|
|
/* Inject into network stack */
|
|
|
|
ret = netif_receive_skb_core(skb);
|
|
|
|
if (ret == NET_RX_DROP)
|
|
|
|
drops++;
|
|
|
|
}
|
2017-10-16 18:19:44 +08:00
|
|
|
/* Feedback loop via tracepoint */
|
2020-07-14 21:56:38 +08:00
|
|
|
trace_xdp_cpumap_kthread(rcpu->map_id, n, drops, sched, &stats);
|
2017-10-16 18:19:44 +08:00
|
|
|
|
bpf: cpumap xdp_buff to skb conversion and allocation
This patch makes cpumap functional, by adding SKB allocation and
invoking the network stack on the dequeuing CPU.
For constructing the SKB on the remote CPU, the xdp_buff in converted
into a struct xdp_pkt, and it mapped into the top headroom of the
packet, to avoid allocating separate mem. For now, struct xdp_pkt is
just a cpumap internal data structure, with info carried between
enqueue to dequeue.
If a driver doesn't have enough headroom it is simply dropped, with
return code -EOVERFLOW. This will be picked up the xdp tracepoint
infrastructure, to allow users to catch this.
V2: take into account xdp->data_meta
V4:
- Drop busypoll tricks, keeping it more simple.
- Skip RPS and Generic-XDP-recursive-reinjection, suggested by Alexei
V5: correct RCU read protection around __netif_receive_skb_core.
V6: Setting TASK_RUNNING vs TASK_INTERRUPTIBLE based on talk with Rik van Riel
Signed-off-by: Jesper Dangaard Brouer <brouer@redhat.com>
Signed-off-by: David S. Miller <davem@davemloft.net>
2017-10-16 18:19:39 +08:00
|
|
|
local_bh_enable(); /* resched point, may call do_softirq() */
|
2017-10-16 18:19:28 +08:00
|
|
|
}
|
|
|
|
__set_current_state(TASK_RUNNING);
|
|
|
|
|
|
|
|
put_cpu_map_entry(rcpu);
|
|
|
|
return 0;
|
|
|
|
}
|
|
|
|
|
2020-07-14 21:56:38 +08:00
|
|
|
bool cpu_map_prog_allowed(struct bpf_map *map)
|
|
|
|
{
|
|
|
|
return map->map_type == BPF_MAP_TYPE_CPUMAP &&
|
|
|
|
map->value_size != offsetofend(struct bpf_cpumap_val, qsize);
|
|
|
|
}
|
|
|
|
|
|
|
|
static int __cpu_map_load_bpf_program(struct bpf_cpu_map_entry *rcpu, int fd)
|
|
|
|
{
|
|
|
|
struct bpf_prog *prog;
|
|
|
|
|
|
|
|
prog = bpf_prog_get_type(fd, BPF_PROG_TYPE_XDP);
|
|
|
|
if (IS_ERR(prog))
|
|
|
|
return PTR_ERR(prog);
|
|
|
|
|
|
|
|
if (prog->expected_attach_type != BPF_XDP_CPUMAP) {
|
|
|
|
bpf_prog_put(prog);
|
|
|
|
return -EINVAL;
|
|
|
|
}
|
|
|
|
|
|
|
|
rcpu->value.bpf_prog.id = prog->aux->id;
|
|
|
|
rcpu->prog = prog;
|
|
|
|
|
|
|
|
return 0;
|
|
|
|
}
|
|
|
|
|
2020-07-14 21:56:37 +08:00
|
|
|
static struct bpf_cpu_map_entry *
|
|
|
|
__cpu_map_entry_alloc(struct bpf_cpumap_val *value, u32 cpu, int map_id)
|
2017-10-16 18:19:28 +08:00
|
|
|
{
|
2020-07-14 21:56:38 +08:00
|
|
|
int numa, err, i, fd = value->bpf_prog.fd;
|
2018-02-14 22:17:34 +08:00
|
|
|
gfp_t gfp = GFP_KERNEL | __GFP_NOWARN;
|
2017-10-16 18:19:28 +08:00
|
|
|
struct bpf_cpu_map_entry *rcpu;
|
2019-06-28 17:12:34 +08:00
|
|
|
struct xdp_bulk_queue *bq;
|
2017-10-16 18:19:28 +08:00
|
|
|
|
|
|
|
/* Have map->numa_node, but choose node of redirect target CPU */
|
|
|
|
numa = cpu_to_node(cpu);
|
|
|
|
|
|
|
|
rcpu = kzalloc_node(sizeof(*rcpu), gfp, numa);
|
|
|
|
if (!rcpu)
|
|
|
|
return NULL;
|
|
|
|
|
|
|
|
/* Alloc percpu bulkq */
|
|
|
|
rcpu->bulkq = __alloc_percpu_gfp(sizeof(*rcpu->bulkq),
|
|
|
|
sizeof(void *), gfp);
|
|
|
|
if (!rcpu->bulkq)
|
|
|
|
goto free_rcu;
|
|
|
|
|
2019-06-28 17:12:34 +08:00
|
|
|
for_each_possible_cpu(i) {
|
|
|
|
bq = per_cpu_ptr(rcpu->bulkq, i);
|
|
|
|
bq->obj = rcpu;
|
|
|
|
}
|
|
|
|
|
2017-10-16 18:19:28 +08:00
|
|
|
/* Alloc queue */
|
|
|
|
rcpu->queue = kzalloc_node(sizeof(*rcpu->queue), gfp, numa);
|
|
|
|
if (!rcpu->queue)
|
|
|
|
goto free_bulkq;
|
|
|
|
|
2020-07-14 21:56:37 +08:00
|
|
|
err = ptr_ring_init(rcpu->queue, value->qsize, gfp);
|
2017-10-16 18:19:28 +08:00
|
|
|
if (err)
|
|
|
|
goto free_queue;
|
|
|
|
|
2017-10-16 18:19:44 +08:00
|
|
|
rcpu->cpu = cpu;
|
|
|
|
rcpu->map_id = map_id;
|
2020-07-14 21:56:37 +08:00
|
|
|
rcpu->value.qsize = value->qsize;
|
2017-10-16 18:19:28 +08:00
|
|
|
|
|
|
|
/* Setup kthread */
|
|
|
|
rcpu->kthread = kthread_create_on_node(cpu_map_kthread_run, rcpu, numa,
|
|
|
|
"cpumap/%d/map:%d", cpu, map_id);
|
|
|
|
if (IS_ERR(rcpu->kthread))
|
|
|
|
goto free_ptr_ring;
|
|
|
|
|
|
|
|
get_cpu_map_entry(rcpu); /* 1-refcnt for being in cmap->cpu_map[] */
|
|
|
|
get_cpu_map_entry(rcpu); /* 1-refcnt for kthread */
|
|
|
|
|
2020-07-14 21:56:38 +08:00
|
|
|
if (fd > 0 && __cpu_map_load_bpf_program(rcpu, fd))
|
|
|
|
goto free_ptr_ring;
|
|
|
|
|
2017-10-16 18:19:28 +08:00
|
|
|
/* Make sure kthread runs on a single CPU */
|
|
|
|
kthread_bind(rcpu->kthread, cpu);
|
|
|
|
wake_up_process(rcpu->kthread);
|
|
|
|
|
|
|
|
return rcpu;
|
|
|
|
|
|
|
|
free_ptr_ring:
|
|
|
|
ptr_ring_cleanup(rcpu->queue, NULL);
|
|
|
|
free_queue:
|
|
|
|
kfree(rcpu->queue);
|
|
|
|
free_bulkq:
|
|
|
|
free_percpu(rcpu->bulkq);
|
|
|
|
free_rcu:
|
|
|
|
kfree(rcpu);
|
|
|
|
return NULL;
|
|
|
|
}
|
|
|
|
|
2018-01-16 19:27:05 +08:00
|
|
|
static void __cpu_map_entry_free(struct rcu_head *rcu)
|
2017-10-16 18:19:28 +08:00
|
|
|
{
|
|
|
|
struct bpf_cpu_map_entry *rcpu;
|
|
|
|
|
|
|
|
/* This cpu_map_entry have been disconnected from map and one
|
2019-12-19 14:10:01 +08:00
|
|
|
* RCU grace-period have elapsed. Thus, XDP cannot queue any
|
2017-10-16 18:19:28 +08:00
|
|
|
* new packets and cannot change/set flush_needed that can
|
|
|
|
* find this entry.
|
|
|
|
*/
|
|
|
|
rcpu = container_of(rcu, struct bpf_cpu_map_entry, rcu);
|
|
|
|
|
|
|
|
free_percpu(rcpu->bulkq);
|
|
|
|
/* Cannot kthread_stop() here, last put free rcpu resources */
|
|
|
|
put_cpu_map_entry(rcpu);
|
|
|
|
}
|
|
|
|
|
|
|
|
/* After xchg pointer to bpf_cpu_map_entry, use the call_rcu() to
|
|
|
|
* ensure any driver rcu critical sections have completed, but this
|
|
|
|
* does not guarantee a flush has happened yet. Because driver side
|
|
|
|
* rcu_read_lock/unlock only protects the running XDP program. The
|
|
|
|
* atomic xchg and NULL-ptr check in __cpu_map_flush() makes sure a
|
|
|
|
* pending flush op doesn't fail.
|
|
|
|
*
|
|
|
|
* The bpf_cpu_map_entry is still used by the kthread, and there can
|
|
|
|
* still be pending packets (in queue and percpu bulkq). A refcnt
|
|
|
|
* makes sure to last user (kthread_stop vs. call_rcu) free memory
|
|
|
|
* resources.
|
|
|
|
*
|
|
|
|
* The rcu callback __cpu_map_entry_free flush remaining packets in
|
|
|
|
* percpu bulkq to queue. Due to caller map_delete_elem() disable
|
|
|
|
* preemption, cannot call kthread_stop() to make sure queue is empty.
|
|
|
|
* Instead a work_queue is started for stopping kthread,
|
2019-12-19 14:10:01 +08:00
|
|
|
* cpu_map_kthread_stop, which waits for an RCU grace period before
|
2017-10-16 18:19:28 +08:00
|
|
|
* stopping kthread, emptying the queue.
|
|
|
|
*/
|
2018-01-16 19:27:05 +08:00
|
|
|
static void __cpu_map_entry_replace(struct bpf_cpu_map *cmap,
|
|
|
|
u32 key_cpu, struct bpf_cpu_map_entry *rcpu)
|
2017-10-16 18:19:28 +08:00
|
|
|
{
|
|
|
|
struct bpf_cpu_map_entry *old_rcpu;
|
|
|
|
|
|
|
|
old_rcpu = xchg(&cmap->cpu_map[key_cpu], rcpu);
|
|
|
|
if (old_rcpu) {
|
|
|
|
call_rcu(&old_rcpu->rcu, __cpu_map_entry_free);
|
|
|
|
INIT_WORK(&old_rcpu->kthread_stop_wq, cpu_map_kthread_stop);
|
|
|
|
schedule_work(&old_rcpu->kthread_stop_wq);
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2018-01-16 19:27:05 +08:00
|
|
|
static int cpu_map_delete_elem(struct bpf_map *map, void *key)
|
2017-10-16 18:19:28 +08:00
|
|
|
{
|
|
|
|
struct bpf_cpu_map *cmap = container_of(map, struct bpf_cpu_map, map);
|
|
|
|
u32 key_cpu = *(u32 *)key;
|
|
|
|
|
|
|
|
if (key_cpu >= map->max_entries)
|
|
|
|
return -EINVAL;
|
|
|
|
|
|
|
|
/* notice caller map_delete_elem() use preempt_disable() */
|
|
|
|
__cpu_map_entry_replace(cmap, key_cpu, NULL);
|
|
|
|
return 0;
|
|
|
|
}
|
|
|
|
|
2018-01-16 19:27:05 +08:00
|
|
|
static int cpu_map_update_elem(struct bpf_map *map, void *key, void *value,
|
|
|
|
u64 map_flags)
|
2017-10-16 18:19:28 +08:00
|
|
|
{
|
|
|
|
struct bpf_cpu_map *cmap = container_of(map, struct bpf_cpu_map, map);
|
2020-07-14 21:56:37 +08:00
|
|
|
struct bpf_cpumap_val cpumap_value = {};
|
2017-10-16 18:19:28 +08:00
|
|
|
struct bpf_cpu_map_entry *rcpu;
|
|
|
|
/* Array index key correspond to CPU number */
|
|
|
|
u32 key_cpu = *(u32 *)key;
|
2020-07-14 21:56:37 +08:00
|
|
|
|
|
|
|
memcpy(&cpumap_value, value, map->value_size);
|
2017-10-16 18:19:28 +08:00
|
|
|
|
|
|
|
if (unlikely(map_flags > BPF_EXIST))
|
|
|
|
return -EINVAL;
|
|
|
|
if (unlikely(key_cpu >= cmap->map.max_entries))
|
|
|
|
return -E2BIG;
|
|
|
|
if (unlikely(map_flags == BPF_NOEXIST))
|
|
|
|
return -EEXIST;
|
2020-07-14 21:56:37 +08:00
|
|
|
if (unlikely(cpumap_value.qsize > 16384)) /* sanity limit on qsize */
|
2017-10-16 18:19:28 +08:00
|
|
|
return -EOVERFLOW;
|
|
|
|
|
|
|
|
/* Make sure CPU is a valid possible cpu */
|
2020-04-16 16:31:20 +08:00
|
|
|
if (key_cpu >= nr_cpumask_bits || !cpu_possible(key_cpu))
|
2017-10-16 18:19:28 +08:00
|
|
|
return -ENODEV;
|
|
|
|
|
2020-07-14 21:56:37 +08:00
|
|
|
if (cpumap_value.qsize == 0) {
|
2017-10-16 18:19:28 +08:00
|
|
|
rcpu = NULL; /* Same as deleting */
|
|
|
|
} else {
|
|
|
|
/* Updating qsize cause re-allocation of bpf_cpu_map_entry */
|
2020-07-14 21:56:37 +08:00
|
|
|
rcpu = __cpu_map_entry_alloc(&cpumap_value, key_cpu, map->id);
|
2017-10-16 18:19:28 +08:00
|
|
|
if (!rcpu)
|
|
|
|
return -ENOMEM;
|
2019-06-28 17:12:34 +08:00
|
|
|
rcpu->cmap = cmap;
|
2017-10-16 18:19:28 +08:00
|
|
|
}
|
|
|
|
rcu_read_lock();
|
|
|
|
__cpu_map_entry_replace(cmap, key_cpu, rcpu);
|
|
|
|
rcu_read_unlock();
|
|
|
|
return 0;
|
|
|
|
}
|
|
|
|
|
2018-01-16 19:27:05 +08:00
|
|
|
static void cpu_map_free(struct bpf_map *map)
|
2017-10-16 18:19:28 +08:00
|
|
|
{
|
|
|
|
struct bpf_cpu_map *cmap = container_of(map, struct bpf_cpu_map, map);
|
|
|
|
u32 i;
|
|
|
|
|
|
|
|
/* At this point bpf_prog->aux->refcnt == 0 and this map->refcnt == 0,
|
|
|
|
* so the bpf programs (can be more than one that used this map) were
|
|
|
|
* disconnected from events. Wait for outstanding critical sections in
|
|
|
|
* these programs to complete. The rcu critical section only guarantees
|
|
|
|
* no further "XDP/bpf-side" reads against bpf_cpu_map->cpu_map.
|
|
|
|
* It does __not__ ensure pending flush operations (if any) are
|
|
|
|
* complete.
|
|
|
|
*/
|
2018-08-18 05:26:14 +08:00
|
|
|
|
|
|
|
bpf_clear_redirect_map(map);
|
2017-10-16 18:19:28 +08:00
|
|
|
synchronize_rcu();
|
|
|
|
|
|
|
|
/* For cpu_map the remote CPUs can still be using the entries
|
|
|
|
* (struct bpf_cpu_map_entry).
|
|
|
|
*/
|
|
|
|
for (i = 0; i < cmap->map.max_entries; i++) {
|
|
|
|
struct bpf_cpu_map_entry *rcpu;
|
|
|
|
|
|
|
|
rcpu = READ_ONCE(cmap->cpu_map[i]);
|
|
|
|
if (!rcpu)
|
|
|
|
continue;
|
|
|
|
|
2019-12-19 14:10:01 +08:00
|
|
|
/* bq flush and cleanup happens after RCU grace-period */
|
2017-10-16 18:19:28 +08:00
|
|
|
__cpu_map_entry_replace(cmap, i, NULL); /* call_rcu */
|
|
|
|
}
|
|
|
|
bpf_map_area_free(cmap->cpu_map);
|
|
|
|
kfree(cmap);
|
|
|
|
}
|
|
|
|
|
|
|
|
struct bpf_cpu_map_entry *__cpu_map_lookup_elem(struct bpf_map *map, u32 key)
|
|
|
|
{
|
|
|
|
struct bpf_cpu_map *cmap = container_of(map, struct bpf_cpu_map, map);
|
|
|
|
struct bpf_cpu_map_entry *rcpu;
|
|
|
|
|
|
|
|
if (key >= map->max_entries)
|
|
|
|
return NULL;
|
|
|
|
|
|
|
|
rcpu = READ_ONCE(cmap->cpu_map[key]);
|
|
|
|
return rcpu;
|
|
|
|
}
|
|
|
|
|
|
|
|
static void *cpu_map_lookup_elem(struct bpf_map *map, void *key)
|
|
|
|
{
|
|
|
|
struct bpf_cpu_map_entry *rcpu =
|
|
|
|
__cpu_map_lookup_elem(map, *(u32 *)key);
|
|
|
|
|
2020-07-14 21:56:37 +08:00
|
|
|
return rcpu ? &rcpu->value : NULL;
|
2017-10-16 18:19:28 +08:00
|
|
|
}
|
|
|
|
|
|
|
|
static int cpu_map_get_next_key(struct bpf_map *map, void *key, void *next_key)
|
|
|
|
{
|
|
|
|
struct bpf_cpu_map *cmap = container_of(map, struct bpf_cpu_map, map);
|
|
|
|
u32 index = key ? *(u32 *)key : U32_MAX;
|
|
|
|
u32 *next = next_key;
|
|
|
|
|
|
|
|
if (index >= cmap->map.max_entries) {
|
|
|
|
*next = 0;
|
|
|
|
return 0;
|
|
|
|
}
|
|
|
|
|
|
|
|
if (index == cmap->map.max_entries - 1)
|
|
|
|
return -ENOENT;
|
|
|
|
*next = index + 1;
|
|
|
|
return 0;
|
|
|
|
}
|
|
|
|
|
2020-06-20 05:11:44 +08:00
|
|
|
static int cpu_map_btf_id;
|
2017-10-16 18:19:28 +08:00
|
|
|
const struct bpf_map_ops cpu_map_ops = {
|
|
|
|
.map_alloc = cpu_map_alloc,
|
|
|
|
.map_free = cpu_map_free,
|
|
|
|
.map_delete_elem = cpu_map_delete_elem,
|
|
|
|
.map_update_elem = cpu_map_update_elem,
|
|
|
|
.map_lookup_elem = cpu_map_lookup_elem,
|
|
|
|
.map_get_next_key = cpu_map_get_next_key,
|
2018-08-12 07:59:17 +08:00
|
|
|
.map_check_btf = map_check_no_btf,
|
2020-06-20 05:11:44 +08:00
|
|
|
.map_btf_name = "bpf_cpu_map",
|
|
|
|
.map_btf_id = &cpu_map_btf_id,
|
2017-10-16 18:19:28 +08:00
|
|
|
};
|
|
|
|
|
2019-12-19 14:10:00 +08:00
|
|
|
static int bq_flush_to_queue(struct xdp_bulk_queue *bq)
|
2017-10-16 18:19:28 +08:00
|
|
|
{
|
2019-06-28 17:12:34 +08:00
|
|
|
struct bpf_cpu_map_entry *rcpu = bq->obj;
|
2017-10-16 18:19:44 +08:00
|
|
|
unsigned int processed = 0, drops = 0;
|
|
|
|
const int to_cpu = rcpu->cpu;
|
2017-10-16 18:19:28 +08:00
|
|
|
struct ptr_ring *q;
|
|
|
|
int i;
|
|
|
|
|
|
|
|
if (unlikely(!bq->count))
|
|
|
|
return 0;
|
|
|
|
|
|
|
|
q = rcpu->queue;
|
|
|
|
spin_lock(&q->producer_lock);
|
|
|
|
|
|
|
|
for (i = 0; i < bq->count; i++) {
|
2018-04-17 22:45:57 +08:00
|
|
|
struct xdp_frame *xdpf = bq->q[i];
|
2017-10-16 18:19:28 +08:00
|
|
|
int err;
|
|
|
|
|
2018-04-17 22:45:57 +08:00
|
|
|
err = __ptr_ring_produce(q, xdpf);
|
2017-10-16 18:19:28 +08:00
|
|
|
if (err) {
|
2017-10-16 18:19:44 +08:00
|
|
|
drops++;
|
2019-12-19 14:10:00 +08:00
|
|
|
xdp_return_frame_rx_napi(xdpf);
|
2017-10-16 18:19:28 +08:00
|
|
|
}
|
2017-10-16 18:19:44 +08:00
|
|
|
processed++;
|
2017-10-16 18:19:28 +08:00
|
|
|
}
|
|
|
|
bq->count = 0;
|
|
|
|
spin_unlock(&q->producer_lock);
|
|
|
|
|
2019-06-28 17:12:34 +08:00
|
|
|
__list_del_clearprev(&bq->flush_node);
|
|
|
|
|
2017-10-16 18:19:44 +08:00
|
|
|
/* Feedback loop via tracepoints */
|
|
|
|
trace_xdp_cpumap_enqueue(rcpu->map_id, processed, drops, to_cpu);
|
2017-10-16 18:19:28 +08:00
|
|
|
return 0;
|
|
|
|
}
|
|
|
|
|
|
|
|
/* Runs under RCU-read-side, plus in softirq under NAPI protection.
|
|
|
|
* Thus, safe percpu variable access.
|
|
|
|
*/
|
2018-04-17 22:45:57 +08:00
|
|
|
static int bq_enqueue(struct bpf_cpu_map_entry *rcpu, struct xdp_frame *xdpf)
|
2017-10-16 18:19:28 +08:00
|
|
|
{
|
2019-12-19 14:10:04 +08:00
|
|
|
struct list_head *flush_list = this_cpu_ptr(&cpu_map_flush_list);
|
2017-10-16 18:19:28 +08:00
|
|
|
struct xdp_bulk_queue *bq = this_cpu_ptr(rcpu->bulkq);
|
|
|
|
|
|
|
|
if (unlikely(bq->count == CPU_MAP_BULK_SIZE))
|
2019-12-19 14:10:00 +08:00
|
|
|
bq_flush_to_queue(bq);
|
2017-10-16 18:19:28 +08:00
|
|
|
|
|
|
|
/* Notice, xdp_buff/page MUST be queued here, long enough for
|
|
|
|
* driver to code invoking us to finished, due to driver
|
|
|
|
* (e.g. ixgbe) recycle tricks based on page-refcnt.
|
|
|
|
*
|
2018-04-17 22:45:57 +08:00
|
|
|
* Thus, incoming xdp_frame is always queued here (else we race
|
2017-10-16 18:19:28 +08:00
|
|
|
* with another CPU on page-refcnt and remaining driver code).
|
|
|
|
* Queue time is very short, as driver will invoke flush
|
|
|
|
* operation, when completing napi->poll call.
|
|
|
|
*/
|
2018-04-17 22:45:57 +08:00
|
|
|
bq->q[bq->count++] = xdpf;
|
2019-06-28 17:12:34 +08:00
|
|
|
|
|
|
|
if (!bq->flush_node.prev)
|
|
|
|
list_add(&bq->flush_node, flush_list);
|
|
|
|
|
2017-10-16 18:19:28 +08:00
|
|
|
return 0;
|
|
|
|
}
|
|
|
|
|
2017-10-16 18:19:34 +08:00
|
|
|
int cpu_map_enqueue(struct bpf_cpu_map_entry *rcpu, struct xdp_buff *xdp,
|
|
|
|
struct net_device *dev_rx)
|
|
|
|
{
|
2018-04-17 22:45:57 +08:00
|
|
|
struct xdp_frame *xdpf;
|
2017-10-16 18:19:34 +08:00
|
|
|
|
2020-05-29 04:47:29 +08:00
|
|
|
xdpf = xdp_convert_buff_to_frame(xdp);
|
2018-04-17 22:45:57 +08:00
|
|
|
if (unlikely(!xdpf))
|
bpf: cpumap xdp_buff to skb conversion and allocation
This patch makes cpumap functional, by adding SKB allocation and
invoking the network stack on the dequeuing CPU.
For constructing the SKB on the remote CPU, the xdp_buff in converted
into a struct xdp_pkt, and it mapped into the top headroom of the
packet, to avoid allocating separate mem. For now, struct xdp_pkt is
just a cpumap internal data structure, with info carried between
enqueue to dequeue.
If a driver doesn't have enough headroom it is simply dropped, with
return code -EOVERFLOW. This will be picked up the xdp tracepoint
infrastructure, to allow users to catch this.
V2: take into account xdp->data_meta
V4:
- Drop busypoll tricks, keeping it more simple.
- Skip RPS and Generic-XDP-recursive-reinjection, suggested by Alexei
V5: correct RCU read protection around __netif_receive_skb_core.
V6: Setting TASK_RUNNING vs TASK_INTERRUPTIBLE based on talk with Rik van Riel
Signed-off-by: Jesper Dangaard Brouer <brouer@redhat.com>
Signed-off-by: David S. Miller <davem@davemloft.net>
2017-10-16 18:19:39 +08:00
|
|
|
return -EOVERFLOW;
|
2017-10-16 18:19:34 +08:00
|
|
|
|
bpf: cpumap xdp_buff to skb conversion and allocation
This patch makes cpumap functional, by adding SKB allocation and
invoking the network stack on the dequeuing CPU.
For constructing the SKB on the remote CPU, the xdp_buff in converted
into a struct xdp_pkt, and it mapped into the top headroom of the
packet, to avoid allocating separate mem. For now, struct xdp_pkt is
just a cpumap internal data structure, with info carried between
enqueue to dequeue.
If a driver doesn't have enough headroom it is simply dropped, with
return code -EOVERFLOW. This will be picked up the xdp tracepoint
infrastructure, to allow users to catch this.
V2: take into account xdp->data_meta
V4:
- Drop busypoll tricks, keeping it more simple.
- Skip RPS and Generic-XDP-recursive-reinjection, suggested by Alexei
V5: correct RCU read protection around __netif_receive_skb_core.
V6: Setting TASK_RUNNING vs TASK_INTERRUPTIBLE based on talk with Rik van Riel
Signed-off-by: Jesper Dangaard Brouer <brouer@redhat.com>
Signed-off-by: David S. Miller <davem@davemloft.net>
2017-10-16 18:19:39 +08:00
|
|
|
/* Info needed when constructing SKB on remote CPU */
|
2018-04-17 22:45:57 +08:00
|
|
|
xdpf->dev_rx = dev_rx;
|
2017-10-16 18:19:34 +08:00
|
|
|
|
2018-04-17 22:45:57 +08:00
|
|
|
bq_enqueue(rcpu, xdpf);
|
2017-10-16 18:19:34 +08:00
|
|
|
return 0;
|
|
|
|
}
|
|
|
|
|
2019-12-19 14:10:04 +08:00
|
|
|
void __cpu_map_flush(void)
|
2017-10-16 18:19:28 +08:00
|
|
|
{
|
2019-12-19 14:10:04 +08:00
|
|
|
struct list_head *flush_list = this_cpu_ptr(&cpu_map_flush_list);
|
2019-06-28 17:12:34 +08:00
|
|
|
struct xdp_bulk_queue *bq, *tmp;
|
2017-10-16 18:19:28 +08:00
|
|
|
|
2019-06-28 17:12:34 +08:00
|
|
|
list_for_each_entry_safe(bq, tmp, flush_list, flush_node) {
|
2019-12-19 14:10:00 +08:00
|
|
|
bq_flush_to_queue(bq);
|
2017-10-16 18:19:28 +08:00
|
|
|
|
|
|
|
/* If already running, costs spin_lock_irqsave + smb_mb */
|
2019-06-28 17:12:34 +08:00
|
|
|
wake_up_process(bq->obj->kthread);
|
2017-10-16 18:19:28 +08:00
|
|
|
}
|
|
|
|
}
|
2019-12-19 14:10:04 +08:00
|
|
|
|
|
|
|
static int __init cpu_map_init(void)
|
|
|
|
{
|
|
|
|
int cpu;
|
|
|
|
|
|
|
|
for_each_possible_cpu(cpu)
|
|
|
|
INIT_LIST_HEAD(&per_cpu(cpu_map_flush_list, cpu));
|
|
|
|
return 0;
|
|
|
|
}
|
|
|
|
|
|
|
|
subsys_initcall(cpu_map_init);
|