mirror of
https://github.com/edk2-porting/linux-next.git
synced 2024-12-16 01:04:08 +08:00
net/mlx5e: Move XDP related code into new XDP files
Take XDP code out of the general EN header and RX file into new XDP files. Currently, XDP-SQ resides only within an RQ and used from a single flow (XDP_TX) triggered upon RX completions. In a downstream patch, additional type of XDP-SQ instances will be presented and used for the XDP_REDIRECT flow, totally unrelated to the RX context. Signed-off-by: Tariq Toukan <tariqt@mellanox.com> Signed-off-by: Saeed Mahameed <saeedm@mellanox.com>
This commit is contained in:
parent
a26a5bdf3e
commit
159d213134
@ -14,7 +14,7 @@ mlx5_core-$(CONFIG_MLX5_FPGA) += fpga/cmd.o fpga/core.o fpga/conn.o fpga/sdk.o \
|
||||
fpga/ipsec.o fpga/tls.o
|
||||
|
||||
mlx5_core-$(CONFIG_MLX5_CORE_EN) += en_main.o en_common.o en_fs.o en_ethtool.o \
|
||||
en_tx.o en_rx.o en_dim.o en_txrx.o en_stats.o vxlan.o \
|
||||
en_tx.o en_rx.o en_dim.o en_txrx.o en/xdp.o en_stats.o vxlan.o \
|
||||
en_arfs.o en_fs_ethtool.o en_selftest.o en/port.o
|
||||
|
||||
mlx5_core-$(CONFIG_MLX5_MPFS) += lib/mpfs.o
|
||||
|
@ -72,8 +72,6 @@ struct page_pool;
|
||||
#define MLX5_RX_HEADROOM NET_SKB_PAD
|
||||
#define MLX5_SKB_FRAG_SZ(len) (SKB_DATA_ALIGN(len) + \
|
||||
SKB_DATA_ALIGN(sizeof(struct skb_shared_info)))
|
||||
#define MLX5E_XDP_MAX_MTU ((int)(PAGE_SIZE - \
|
||||
MLX5_SKB_FRAG_SZ(XDP_PACKET_HEADROOM)))
|
||||
|
||||
#define MLX5_MPWRQ_MIN_LOG_STRIDE_SZ(mdev) \
|
||||
(6 + MLX5_CAP_GEN(mdev, cache_line_128byte)) /* HW restriction */
|
||||
@ -149,10 +147,6 @@ struct page_pool;
|
||||
(DIV_ROUND_UP(MLX5E_UMR_WQE_INLINE_SZ, MLX5_SEND_WQE_BB))
|
||||
#define MLX5E_ICOSQ_MAX_WQEBBS MLX5E_UMR_WQEBBS
|
||||
|
||||
#define MLX5E_XDP_MIN_INLINE (ETH_HLEN + VLAN_HLEN)
|
||||
#define MLX5E_XDP_TX_DS_COUNT \
|
||||
((sizeof(struct mlx5e_tx_wqe) / MLX5_SEND_WQE_DS) + 1 /* SG DS */)
|
||||
|
||||
#define MLX5E_NUM_MAIN_GROUPS 9
|
||||
|
||||
#define MLX5E_MSG_LEVEL NETIF_MSG_LINK
|
||||
@ -878,14 +872,13 @@ void mlx5e_cq_error_event(struct mlx5_core_cq *mcq, enum mlx5_event event);
|
||||
int mlx5e_napi_poll(struct napi_struct *napi, int budget);
|
||||
bool mlx5e_poll_tx_cq(struct mlx5e_cq *cq, int napi_budget);
|
||||
int mlx5e_poll_rx_cq(struct mlx5e_cq *cq, int budget);
|
||||
bool mlx5e_poll_xdpsq_cq(struct mlx5e_cq *cq);
|
||||
void mlx5e_free_txqsq_descs(struct mlx5e_txqsq *sq);
|
||||
void mlx5e_free_xdpsq_descs(struct mlx5e_xdpsq *sq);
|
||||
|
||||
bool mlx5e_check_fragmented_striding_rq_cap(struct mlx5_core_dev *mdev);
|
||||
bool mlx5e_striding_rq_possible(struct mlx5_core_dev *mdev,
|
||||
struct mlx5e_params *params);
|
||||
|
||||
void mlx5e_page_dma_unmap(struct mlx5e_rq *rq, struct mlx5e_dma_info *dma_info);
|
||||
void mlx5e_page_release(struct mlx5e_rq *rq, struct mlx5e_dma_info *dma_info,
|
||||
bool recycle);
|
||||
void mlx5e_handle_rx_cqe(struct mlx5e_rq *rq, struct mlx5_cqe64 *cqe);
|
||||
|
225
drivers/net/ethernet/mellanox/mlx5/core/en/xdp.c
Normal file
225
drivers/net/ethernet/mellanox/mlx5/core/en/xdp.c
Normal file
@ -0,0 +1,225 @@
|
||||
/*
|
||||
* Copyright (c) 2018, Mellanox Technologies. All rights reserved.
|
||||
*
|
||||
* This software is available to you under a choice of one of two
|
||||
* licenses. You may choose to be licensed under the terms of the GNU
|
||||
* General Public License (GPL) Version 2, available from the file
|
||||
* COPYING in the main directory of this source tree, or the
|
||||
* OpenIB.org BSD license below:
|
||||
*
|
||||
* Redistribution and use in source and binary forms, with or
|
||||
* without modification, are permitted provided that the following
|
||||
* conditions are met:
|
||||
*
|
||||
* - Redistributions of source code must retain the above
|
||||
* copyright notice, this list of conditions and the following
|
||||
* disclaimer.
|
||||
*
|
||||
* - Redistributions in binary form must reproduce the above
|
||||
* copyright notice, this list of conditions and the following
|
||||
* disclaimer in the documentation and/or other materials
|
||||
* provided with the distribution.
|
||||
*
|
||||
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
|
||||
* EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
|
||||
* MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
|
||||
* NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
|
||||
* BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
|
||||
* ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
|
||||
* CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
||||
* SOFTWARE.
|
||||
*/
|
||||
|
||||
#include <linux/bpf_trace.h>
|
||||
#include "en/xdp.h"
|
||||
|
||||
/* returns true if packet was consumed by xdp */
|
||||
bool mlx5e_xdp_handle(struct mlx5e_rq *rq, struct mlx5e_dma_info *di,
|
||||
void *va, u16 *rx_headroom, u32 *len)
|
||||
{
|
||||
struct bpf_prog *prog = READ_ONCE(rq->xdp_prog);
|
||||
struct xdp_buff xdp;
|
||||
u32 act;
|
||||
int err;
|
||||
|
||||
if (!prog)
|
||||
return false;
|
||||
|
||||
xdp.data = va + *rx_headroom;
|
||||
xdp_set_data_meta_invalid(&xdp);
|
||||
xdp.data_end = xdp.data + *len;
|
||||
xdp.data_hard_start = va;
|
||||
xdp.rxq = &rq->xdp_rxq;
|
||||
|
||||
act = bpf_prog_run_xdp(prog, &xdp);
|
||||
switch (act) {
|
||||
case XDP_PASS:
|
||||
*rx_headroom = xdp.data - xdp.data_hard_start;
|
||||
*len = xdp.data_end - xdp.data;
|
||||
return false;
|
||||
case XDP_TX:
|
||||
if (unlikely(!mlx5e_xmit_xdp_frame(rq, di, &xdp)))
|
||||
trace_xdp_exception(rq->netdev, prog, act);
|
||||
return true;
|
||||
case XDP_REDIRECT:
|
||||
/* When XDP enabled then page-refcnt==1 here */
|
||||
err = xdp_do_redirect(rq->netdev, &xdp, prog);
|
||||
if (!err) {
|
||||
__set_bit(MLX5E_RQ_FLAG_XDP_XMIT, rq->flags);
|
||||
rq->xdpsq.db.redirect_flush = true;
|
||||
mlx5e_page_dma_unmap(rq, di);
|
||||
}
|
||||
return true;
|
||||
default:
|
||||
bpf_warn_invalid_xdp_action(act);
|
||||
case XDP_ABORTED:
|
||||
trace_xdp_exception(rq->netdev, prog, act);
|
||||
case XDP_DROP:
|
||||
rq->stats->xdp_drop++;
|
||||
return true;
|
||||
}
|
||||
}
|
||||
|
||||
bool mlx5e_xmit_xdp_frame(struct mlx5e_rq *rq, struct mlx5e_dma_info *di,
|
||||
const struct xdp_buff *xdp)
|
||||
{
|
||||
struct mlx5e_xdpsq *sq = &rq->xdpsq;
|
||||
struct mlx5_wq_cyc *wq = &sq->wq;
|
||||
u16 pi = mlx5_wq_cyc_ctr2ix(wq, sq->pc);
|
||||
struct mlx5e_tx_wqe *wqe = mlx5_wq_cyc_get_wqe(wq, pi);
|
||||
|
||||
struct mlx5_wqe_ctrl_seg *cseg = &wqe->ctrl;
|
||||
struct mlx5_wqe_eth_seg *eseg = &wqe->eth;
|
||||
struct mlx5_wqe_data_seg *dseg;
|
||||
|
||||
ptrdiff_t data_offset = xdp->data - xdp->data_hard_start;
|
||||
dma_addr_t dma_addr = di->addr + data_offset;
|
||||
unsigned int dma_len = xdp->data_end - xdp->data;
|
||||
|
||||
struct mlx5e_rq_stats *stats = rq->stats;
|
||||
|
||||
prefetchw(wqe);
|
||||
|
||||
if (unlikely(dma_len < MLX5E_XDP_MIN_INLINE || rq->hw_mtu < dma_len)) {
|
||||
stats->xdp_drop++;
|
||||
return false;
|
||||
}
|
||||
|
||||
if (unlikely(!mlx5e_wqc_has_room_for(wq, sq->cc, sq->pc, 1))) {
|
||||
if (sq->db.doorbell) {
|
||||
/* SQ is full, ring doorbell */
|
||||
mlx5e_xmit_xdp_doorbell(sq);
|
||||
sq->db.doorbell = false;
|
||||
}
|
||||
stats->xdp_tx_full++;
|
||||
return false;
|
||||
}
|
||||
|
||||
dma_sync_single_for_device(sq->pdev, dma_addr, dma_len, PCI_DMA_TODEVICE);
|
||||
|
||||
cseg->fm_ce_se = 0;
|
||||
|
||||
dseg = (struct mlx5_wqe_data_seg *)eseg + 1;
|
||||
|
||||
/* copy the inline part if required */
|
||||
if (sq->min_inline_mode != MLX5_INLINE_MODE_NONE) {
|
||||
memcpy(eseg->inline_hdr.start, xdp->data, MLX5E_XDP_MIN_INLINE);
|
||||
eseg->inline_hdr.sz = cpu_to_be16(MLX5E_XDP_MIN_INLINE);
|
||||
dma_len -= MLX5E_XDP_MIN_INLINE;
|
||||
dma_addr += MLX5E_XDP_MIN_INLINE;
|
||||
dseg++;
|
||||
}
|
||||
|
||||
/* write the dma part */
|
||||
dseg->addr = cpu_to_be64(dma_addr);
|
||||
dseg->byte_count = cpu_to_be32(dma_len);
|
||||
|
||||
cseg->opmod_idx_opcode = cpu_to_be32((sq->pc << 8) | MLX5_OPCODE_SEND);
|
||||
|
||||
/* move page to reference to sq responsibility,
|
||||
* and mark so it's not put back in page-cache.
|
||||
*/
|
||||
__set_bit(MLX5E_RQ_FLAG_XDP_XMIT, rq->flags); /* non-atomic */
|
||||
sq->db.di[pi] = *di;
|
||||
sq->pc++;
|
||||
|
||||
sq->db.doorbell = true;
|
||||
|
||||
stats->xdp_tx++;
|
||||
return true;
|
||||
}
|
||||
|
||||
bool mlx5e_poll_xdpsq_cq(struct mlx5e_cq *cq)
|
||||
{
|
||||
struct mlx5e_xdpsq *sq;
|
||||
struct mlx5_cqe64 *cqe;
|
||||
struct mlx5e_rq *rq;
|
||||
u16 sqcc;
|
||||
int i;
|
||||
|
||||
sq = container_of(cq, struct mlx5e_xdpsq, cq);
|
||||
|
||||
if (unlikely(!test_bit(MLX5E_SQ_STATE_ENABLED, &sq->state)))
|
||||
return false;
|
||||
|
||||
cqe = mlx5_cqwq_get_cqe(&cq->wq);
|
||||
if (!cqe)
|
||||
return false;
|
||||
|
||||
rq = container_of(sq, struct mlx5e_rq, xdpsq);
|
||||
|
||||
/* sq->cc must be updated only after mlx5_cqwq_update_db_record(),
|
||||
* otherwise a cq overrun may occur
|
||||
*/
|
||||
sqcc = sq->cc;
|
||||
|
||||
i = 0;
|
||||
do {
|
||||
u16 wqe_counter;
|
||||
bool last_wqe;
|
||||
|
||||
mlx5_cqwq_pop(&cq->wq);
|
||||
|
||||
wqe_counter = be16_to_cpu(cqe->wqe_counter);
|
||||
|
||||
do {
|
||||
struct mlx5e_dma_info *di;
|
||||
u16 ci;
|
||||
|
||||
last_wqe = (sqcc == wqe_counter);
|
||||
|
||||
ci = mlx5_wq_cyc_ctr2ix(&sq->wq, sqcc);
|
||||
di = &sq->db.di[ci];
|
||||
|
||||
sqcc++;
|
||||
/* Recycle RX page */
|
||||
mlx5e_page_release(rq, di, true);
|
||||
} while (!last_wqe);
|
||||
} while ((++i < MLX5E_TX_CQ_POLL_BUDGET) && (cqe = mlx5_cqwq_get_cqe(&cq->wq)));
|
||||
|
||||
rq->stats->xdp_tx_cqe += i;
|
||||
|
||||
mlx5_cqwq_update_db_record(&cq->wq);
|
||||
|
||||
/* ensure cq space is freed before enabling more cqes */
|
||||
wmb();
|
||||
|
||||
sq->cc = sqcc;
|
||||
return (i == MLX5E_TX_CQ_POLL_BUDGET);
|
||||
}
|
||||
|
||||
void mlx5e_free_xdpsq_descs(struct mlx5e_xdpsq *sq)
|
||||
{
|
||||
struct mlx5e_rq *rq = container_of(sq, struct mlx5e_rq, xdpsq);
|
||||
struct mlx5e_dma_info *di;
|
||||
u16 ci;
|
||||
|
||||
while (sq->cc != sq->pc) {
|
||||
ci = mlx5_wq_cyc_ctr2ix(&sq->wq, sq->cc);
|
||||
di = &sq->db.di[ci];
|
||||
sq->cc++;
|
||||
|
||||
mlx5e_page_release(rq, di, false);
|
||||
}
|
||||
}
|
||||
|
62
drivers/net/ethernet/mellanox/mlx5/core/en/xdp.h
Normal file
62
drivers/net/ethernet/mellanox/mlx5/core/en/xdp.h
Normal file
@ -0,0 +1,62 @@
|
||||
/*
|
||||
* Copyright (c) 2018, Mellanox Technologies. All rights reserved.
|
||||
*
|
||||
* This software is available to you under a choice of one of two
|
||||
* licenses. You may choose to be licensed under the terms of the GNU
|
||||
* General Public License (GPL) Version 2, available from the file
|
||||
* COPYING in the main directory of this source tree, or the
|
||||
* OpenIB.org BSD license below:
|
||||
*
|
||||
* Redistribution and use in source and binary forms, with or
|
||||
* without modification, are permitted provided that the following
|
||||
* conditions are met:
|
||||
*
|
||||
* - Redistributions of source code must retain the above
|
||||
* copyright notice, this list of conditions and the following
|
||||
* disclaimer.
|
||||
*
|
||||
* - Redistributions in binary form must reproduce the above
|
||||
* copyright notice, this list of conditions and the following
|
||||
* disclaimer in the documentation and/or other materials
|
||||
* provided with the distribution.
|
||||
*
|
||||
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
|
||||
* EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
|
||||
* MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
|
||||
* NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
|
||||
* BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
|
||||
* ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
|
||||
* CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
||||
* SOFTWARE.
|
||||
*/
|
||||
#ifndef __MLX5_EN_XDP_H__
|
||||
#define __MLX5_EN_XDP_H__
|
||||
|
||||
#include "en.h"
|
||||
|
||||
#define MLX5E_XDP_MAX_MTU ((int)(PAGE_SIZE - \
|
||||
MLX5_SKB_FRAG_SZ(XDP_PACKET_HEADROOM)))
|
||||
#define MLX5E_XDP_MIN_INLINE (ETH_HLEN + VLAN_HLEN)
|
||||
#define MLX5E_XDP_TX_DS_COUNT \
|
||||
((sizeof(struct mlx5e_tx_wqe) / MLX5_SEND_WQE_DS) + 1 /* SG DS */)
|
||||
|
||||
bool mlx5e_xdp_handle(struct mlx5e_rq *rq, struct mlx5e_dma_info *di,
|
||||
void *va, u16 *rx_headroom, u32 *len);
|
||||
bool mlx5e_poll_xdpsq_cq(struct mlx5e_cq *cq);
|
||||
void mlx5e_free_xdpsq_descs(struct mlx5e_xdpsq *sq);
|
||||
|
||||
bool mlx5e_xmit_xdp_frame(struct mlx5e_rq *rq, struct mlx5e_dma_info *di,
|
||||
const struct xdp_buff *xdp);
|
||||
|
||||
static inline void mlx5e_xmit_xdp_doorbell(struct mlx5e_xdpsq *sq)
|
||||
{
|
||||
struct mlx5_wq_cyc *wq = &sq->wq;
|
||||
struct mlx5e_tx_wqe *wqe;
|
||||
u16 pi = mlx5_wq_cyc_ctr2ix(wq, sq->pc - 1); /* last pi */
|
||||
|
||||
wqe = mlx5_wq_cyc_get_wqe(wq, pi);
|
||||
|
||||
mlx5e_notify_hw(wq, sq->pc, sq->uar_map, &wqe->ctrl);
|
||||
}
|
||||
|
||||
#endif
|
@ -47,6 +47,7 @@
|
||||
#include "accel/tls.h"
|
||||
#include "vxlan.h"
|
||||
#include "en/port.h"
|
||||
#include "en/xdp.h"
|
||||
|
||||
struct mlx5e_rq_param {
|
||||
u32 rqc[MLX5_ST_SZ_DW(rqc)];
|
||||
|
@ -34,7 +34,6 @@
|
||||
#include <linux/ip.h>
|
||||
#include <linux/ipv6.h>
|
||||
#include <linux/tcp.h>
|
||||
#include <linux/bpf_trace.h>
|
||||
#include <net/busy_poll.h>
|
||||
#include <net/ip6_checksum.h>
|
||||
#include <net/page_pool.h>
|
||||
@ -46,6 +45,7 @@
|
||||
#include "en_accel/ipsec_rxtx.h"
|
||||
#include "en_accel/tls_rxtx.h"
|
||||
#include "lib/clock.h"
|
||||
#include "en/xdp.h"
|
||||
|
||||
static inline bool mlx5e_rx_hw_stamp(struct hwtstamp_config *config)
|
||||
{
|
||||
@ -239,8 +239,7 @@ static inline int mlx5e_page_alloc_mapped(struct mlx5e_rq *rq,
|
||||
return 0;
|
||||
}
|
||||
|
||||
static void mlx5e_page_dma_unmap(struct mlx5e_rq *rq,
|
||||
struct mlx5e_dma_info *dma_info)
|
||||
void mlx5e_page_dma_unmap(struct mlx5e_rq *rq, struct mlx5e_dma_info *dma_info)
|
||||
{
|
||||
dma_unmap_page(rq->pdev, dma_info->addr, PAGE_SIZE, rq->buff.map_dir);
|
||||
}
|
||||
@ -850,135 +849,6 @@ static inline void mlx5e_complete_rx_cqe(struct mlx5e_rq *rq,
|
||||
mlx5e_build_rx_skb(cqe, cqe_bcnt, rq, skb);
|
||||
}
|
||||
|
||||
static inline void mlx5e_xmit_xdp_doorbell(struct mlx5e_xdpsq *sq)
|
||||
{
|
||||
struct mlx5_wq_cyc *wq = &sq->wq;
|
||||
struct mlx5e_tx_wqe *wqe;
|
||||
u16 pi = mlx5_wq_cyc_ctr2ix(wq, sq->pc - 1); /* last pi */
|
||||
|
||||
wqe = mlx5_wq_cyc_get_wqe(wq, pi);
|
||||
|
||||
mlx5e_notify_hw(wq, sq->pc, sq->uar_map, &wqe->ctrl);
|
||||
}
|
||||
|
||||
static inline bool mlx5e_xmit_xdp_frame(struct mlx5e_rq *rq,
|
||||
struct mlx5e_dma_info *di,
|
||||
const struct xdp_buff *xdp)
|
||||
{
|
||||
struct mlx5e_xdpsq *sq = &rq->xdpsq;
|
||||
struct mlx5_wq_cyc *wq = &sq->wq;
|
||||
u16 pi = mlx5_wq_cyc_ctr2ix(wq, sq->pc);
|
||||
struct mlx5e_tx_wqe *wqe = mlx5_wq_cyc_get_wqe(wq, pi);
|
||||
|
||||
struct mlx5_wqe_ctrl_seg *cseg = &wqe->ctrl;
|
||||
struct mlx5_wqe_eth_seg *eseg = &wqe->eth;
|
||||
struct mlx5_wqe_data_seg *dseg;
|
||||
|
||||
ptrdiff_t data_offset = xdp->data - xdp->data_hard_start;
|
||||
dma_addr_t dma_addr = di->addr + data_offset;
|
||||
unsigned int dma_len = xdp->data_end - xdp->data;
|
||||
|
||||
struct mlx5e_rq_stats *stats = rq->stats;
|
||||
|
||||
prefetchw(wqe);
|
||||
|
||||
if (unlikely(dma_len < MLX5E_XDP_MIN_INLINE || rq->hw_mtu < dma_len)) {
|
||||
stats->xdp_drop++;
|
||||
return false;
|
||||
}
|
||||
|
||||
if (unlikely(!mlx5e_wqc_has_room_for(wq, sq->cc, sq->pc, 1))) {
|
||||
if (sq->db.doorbell) {
|
||||
/* SQ is full, ring doorbell */
|
||||
mlx5e_xmit_xdp_doorbell(sq);
|
||||
sq->db.doorbell = false;
|
||||
}
|
||||
stats->xdp_tx_full++;
|
||||
return false;
|
||||
}
|
||||
|
||||
dma_sync_single_for_device(sq->pdev, dma_addr, dma_len, PCI_DMA_TODEVICE);
|
||||
|
||||
cseg->fm_ce_se = 0;
|
||||
|
||||
dseg = (struct mlx5_wqe_data_seg *)eseg + 1;
|
||||
|
||||
/* copy the inline part if required */
|
||||
if (sq->min_inline_mode != MLX5_INLINE_MODE_NONE) {
|
||||
memcpy(eseg->inline_hdr.start, xdp->data, MLX5E_XDP_MIN_INLINE);
|
||||
eseg->inline_hdr.sz = cpu_to_be16(MLX5E_XDP_MIN_INLINE);
|
||||
dma_len -= MLX5E_XDP_MIN_INLINE;
|
||||
dma_addr += MLX5E_XDP_MIN_INLINE;
|
||||
dseg++;
|
||||
}
|
||||
|
||||
/* write the dma part */
|
||||
dseg->addr = cpu_to_be64(dma_addr);
|
||||
dseg->byte_count = cpu_to_be32(dma_len);
|
||||
|
||||
cseg->opmod_idx_opcode = cpu_to_be32((sq->pc << 8) | MLX5_OPCODE_SEND);
|
||||
|
||||
/* move page to reference to sq responsibility,
|
||||
* and mark so it's not put back in page-cache.
|
||||
*/
|
||||
__set_bit(MLX5E_RQ_FLAG_XDP_XMIT, rq->flags); /* non-atomic */
|
||||
sq->db.di[pi] = *di;
|
||||
sq->pc++;
|
||||
|
||||
sq->db.doorbell = true;
|
||||
|
||||
stats->xdp_tx++;
|
||||
return true;
|
||||
}
|
||||
|
||||
/* returns true if packet was consumed by xdp */
|
||||
static inline bool mlx5e_xdp_handle(struct mlx5e_rq *rq,
|
||||
struct mlx5e_dma_info *di,
|
||||
void *va, u16 *rx_headroom, u32 *len)
|
||||
{
|
||||
struct bpf_prog *prog = READ_ONCE(rq->xdp_prog);
|
||||
struct xdp_buff xdp;
|
||||
u32 act;
|
||||
int err;
|
||||
|
||||
if (!prog)
|
||||
return false;
|
||||
|
||||
xdp.data = va + *rx_headroom;
|
||||
xdp_set_data_meta_invalid(&xdp);
|
||||
xdp.data_end = xdp.data + *len;
|
||||
xdp.data_hard_start = va;
|
||||
xdp.rxq = &rq->xdp_rxq;
|
||||
|
||||
act = bpf_prog_run_xdp(prog, &xdp);
|
||||
switch (act) {
|
||||
case XDP_PASS:
|
||||
*rx_headroom = xdp.data - xdp.data_hard_start;
|
||||
*len = xdp.data_end - xdp.data;
|
||||
return false;
|
||||
case XDP_TX:
|
||||
if (unlikely(!mlx5e_xmit_xdp_frame(rq, di, &xdp)))
|
||||
trace_xdp_exception(rq->netdev, prog, act);
|
||||
return true;
|
||||
case XDP_REDIRECT:
|
||||
/* When XDP enabled then page-refcnt==1 here */
|
||||
err = xdp_do_redirect(rq->netdev, &xdp, prog);
|
||||
if (!err) {
|
||||
__set_bit(MLX5E_RQ_FLAG_XDP_XMIT, rq->flags);
|
||||
rq->xdpsq.db.redirect_flush = true;
|
||||
mlx5e_page_dma_unmap(rq, di);
|
||||
}
|
||||
return true;
|
||||
default:
|
||||
bpf_warn_invalid_xdp_action(act);
|
||||
case XDP_ABORTED:
|
||||
trace_xdp_exception(rq->netdev, prog, act);
|
||||
case XDP_DROP:
|
||||
rq->stats->xdp_drop++;
|
||||
return true;
|
||||
}
|
||||
}
|
||||
|
||||
static inline
|
||||
struct sk_buff *mlx5e_build_linear_skb(struct mlx5e_rq *rq, void *va,
|
||||
u32 frag_size, u16 headroom,
|
||||
@ -1349,80 +1219,6 @@ int mlx5e_poll_rx_cq(struct mlx5e_cq *cq, int budget)
|
||||
return work_done;
|
||||
}
|
||||
|
||||
bool mlx5e_poll_xdpsq_cq(struct mlx5e_cq *cq)
|
||||
{
|
||||
struct mlx5e_xdpsq *sq;
|
||||
struct mlx5_cqe64 *cqe;
|
||||
struct mlx5e_rq *rq;
|
||||
u16 sqcc;
|
||||
int i;
|
||||
|
||||
sq = container_of(cq, struct mlx5e_xdpsq, cq);
|
||||
|
||||
if (unlikely(!test_bit(MLX5E_SQ_STATE_ENABLED, &sq->state)))
|
||||
return false;
|
||||
|
||||
cqe = mlx5_cqwq_get_cqe(&cq->wq);
|
||||
if (!cqe)
|
||||
return false;
|
||||
|
||||
rq = container_of(sq, struct mlx5e_rq, xdpsq);
|
||||
|
||||
/* sq->cc must be updated only after mlx5_cqwq_update_db_record(),
|
||||
* otherwise a cq overrun may occur
|
||||
*/
|
||||
sqcc = sq->cc;
|
||||
|
||||
i = 0;
|
||||
do {
|
||||
u16 wqe_counter;
|
||||
bool last_wqe;
|
||||
|
||||
mlx5_cqwq_pop(&cq->wq);
|
||||
|
||||
wqe_counter = be16_to_cpu(cqe->wqe_counter);
|
||||
|
||||
do {
|
||||
struct mlx5e_dma_info *di;
|
||||
u16 ci;
|
||||
|
||||
last_wqe = (sqcc == wqe_counter);
|
||||
|
||||
ci = mlx5_wq_cyc_ctr2ix(&sq->wq, sqcc);
|
||||
di = &sq->db.di[ci];
|
||||
|
||||
sqcc++;
|
||||
/* Recycle RX page */
|
||||
mlx5e_page_release(rq, di, true);
|
||||
} while (!last_wqe);
|
||||
} while ((++i < MLX5E_TX_CQ_POLL_BUDGET) && (cqe = mlx5_cqwq_get_cqe(&cq->wq)));
|
||||
|
||||
rq->stats->xdp_tx_cqe += i;
|
||||
|
||||
mlx5_cqwq_update_db_record(&cq->wq);
|
||||
|
||||
/* ensure cq space is freed before enabling more cqes */
|
||||
wmb();
|
||||
|
||||
sq->cc = sqcc;
|
||||
return (i == MLX5E_TX_CQ_POLL_BUDGET);
|
||||
}
|
||||
|
||||
void mlx5e_free_xdpsq_descs(struct mlx5e_xdpsq *sq)
|
||||
{
|
||||
struct mlx5e_rq *rq = container_of(sq, struct mlx5e_rq, xdpsq);
|
||||
struct mlx5e_dma_info *di;
|
||||
u16 ci;
|
||||
|
||||
while (sq->cc != sq->pc) {
|
||||
ci = mlx5_wq_cyc_ctr2ix(&sq->wq, sq->cc);
|
||||
di = &sq->db.di[ci];
|
||||
sq->cc++;
|
||||
|
||||
mlx5e_page_release(rq, di, false);
|
||||
}
|
||||
}
|
||||
|
||||
#ifdef CONFIG_MLX5_CORE_IPOIB
|
||||
|
||||
#define MLX5_IB_GRH_DGID_OFFSET 24
|
||||
|
@ -32,6 +32,7 @@
|
||||
|
||||
#include <linux/irq.h>
|
||||
#include "en.h"
|
||||
#include "en/xdp.h"
|
||||
|
||||
static inline bool mlx5e_channel_no_affinity_change(struct mlx5e_channel *c)
|
||||
{
|
||||
|
Loading…
Reference in New Issue
Block a user