korg/linux
0
0
Fork 0
mirror of https://mirrors.bfsu.edu.cn/git/linux.git synced 2024-12-24 03:24:55 +08:00
Code Issues Packages Projects Releases Wiki Activity

RDMA/bnxt_re: Enable SRIOV VF support on Broadcom's 57500 adapter series

Browse Source 
Broadcom's 575xx adapter series has support for SRIOV VFs.  Making changes
to enable SRIOV VF support. There are two major area where changes are
done:

 - Added new DB location for control-path and data-path DB ring

 - New devices do not need to issue the sriov-config slow-path command
   thus, skipping to call that firmware command.

For now enabling support for 64 RoCE VFs.

Link: https://lore.kernel.org/r/1570081715-14301-1-git-send-email-devesh.sharma@broadcom.com
Signed-off-by: Devesh Sharma <devesh.sharma@broadcom.com>
Signed-off-by: Jason Gunthorpe <jgg@mellanox.com>
This commit is contained in:
Devesh Sharma 2019-10-03 01:48:35 -04:00 committed by Jason Gunthorpe
parent b2e872f451
commit 39c48c5146
3 changed files with 82 additions and 57 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
drivers/infiniband/hw/bnxt_re/bnxt_re.h
View File

@ -108,6 +108,7 @@ struct bnxt_re_sqp_entries {
#define BNXT_RE_MAX_MSIX 9 #define BNXT_RE_MAX_MSIX 9
#define BNXT_RE_AEQ_IDX 0 #define BNXT_RE_AEQ_IDX 0
#define BNXT_RE_NQ_IDX 1 #define BNXT_RE_NQ_IDX 1
#define BNXT_RE_GEN_P5_MAX_VF 64
struct bnxt_re_dev { struct bnxt_re_dev {
struct ib_device ibdev; struct ib_device ibdev;

101
drivers/infiniband/hw/bnxt_re/main.c
View File

@ -119,61 +119,76 @@ static void bnxt_re_get_sriov_func_type(struct bnxt_re_dev *rdev)
* reserved for the function. The driver may choose to allocate fewer * reserved for the function. The driver may choose to allocate fewer
* resources than the firmware maximum. * resources than the firmware maximum.
*/ */
static void bnxt_re_set_resource_limits(struct bnxt_re_dev *rdev) static void bnxt_re_limit_pf_res(struct bnxt_re_dev *rdev)
{ {
u32 vf_qps = 0, vf_srqs = 0, vf_cqs = 0, vf_mrws = 0, vf_gids = 0; struct bnxt_qplib_dev_attr *attr;
u32 i; struct bnxt_qplib_ctx *ctx;
u32 vf_pct; int i;
u32 num_vfs;
struct bnxt_qplib_dev_attr *dev_attr = &rdev->dev_attr;
rdev->qplib_ctx.qpc_count = min_t(u32, BNXT_RE_MAX_QPC_COUNT, attr = &rdev->dev_attr;
dev_attr->max_qp); ctx = &rdev->qplib_ctx;
rdev->qplib_ctx.mrw_count = BNXT_RE_MAX_MRW_COUNT_256K; ctx->qpc_count = min_t(u32, BNXT_RE_MAX_QPC_COUNT,
attr->max_qp);
ctx->mrw_count = BNXT_RE_MAX_MRW_COUNT_256K;
/* Use max_mr from fw since max_mrw does not get set */ /* Use max_mr from fw since max_mrw does not get set */
rdev->qplib_ctx.mrw_count = min_t(u32, rdev->qplib_ctx.mrw_count, ctx->mrw_count = min_t(u32, ctx->mrw_count, attr->max_mr);
dev_attr->max_mr); ctx->srqc_count = min_t(u32, BNXT_RE_MAX_SRQC_COUNT,
rdev->qplib_ctx.srqc_count = min_t(u32, BNXT_RE_MAX_SRQC_COUNT, attr->max_srq);
dev_attr->max_srq); ctx->cq_count = min_t(u32, BNXT_RE_MAX_CQ_COUNT, attr->max_cq);
rdev->qplib_ctx.cq_count = min_t(u32, BNXT_RE_MAX_CQ_COUNT, if (!bnxt_qplib_is_chip_gen_p5(&rdev->chip_ctx))
dev_attr->max_cq);
for (i = 0; i < MAX_TQM_ALLOC_REQ; i++) for (i = 0; i < MAX_TQM_ALLOC_REQ; i++)
rdev->qplib_ctx.tqm_count[i] = rdev->qplib_ctx.tqm_count[i] =
rdev->dev_attr.tqm_alloc_reqs[i]; rdev->dev_attr.tqm_alloc_reqs[i];
}
if (rdev->num_vfs) { static void bnxt_re_limit_vf_res(struct bnxt_qplib_ctx *qplib_ctx, u32 num_vf)
{
struct bnxt_qplib_vf_res *vf_res;
u32 mrws = 0;
u32 vf_pct;
u32 nvfs;
vf_res = &qplib_ctx->vf_res;
/* /*
* Reserve a set of resources for the PF. Divide the remaining * Reserve a set of resources for the PF. Divide the remaining
* resources among the VFs * resources among the VFs
*/ */
vf_pct = 100 - BNXT_RE_PCT_RSVD_FOR_PF; vf_pct = 100 - BNXT_RE_PCT_RSVD_FOR_PF;
num_vfs = 100 * rdev->num_vfs; nvfs = num_vf;
vf_qps = (rdev->qplib_ctx.qpc_count * vf_pct) / num_vfs; num_vf = 100 * num_vf;
vf_srqs = (rdev->qplib_ctx.srqc_count * vf_pct) / num_vfs; vf_res->max_qp_per_vf = (qplib_ctx->qpc_count * vf_pct) / num_vf;
vf_cqs = (rdev->qplib_ctx.cq_count * vf_pct) / num_vfs; vf_res->max_srq_per_vf = (qplib_ctx->srqc_count * vf_pct) / num_vf;
vf_res->max_cq_per_vf = (qplib_ctx->cq_count * vf_pct) / num_vf;
/* /*
* The driver allows many more MRs than other resources. If the * The driver allows many more MRs than other resources. If the
* firmware does also, then reserve a fixed amount for the PF * firmware does also, then reserve a fixed amount for the PF and
* and divide the rest among VFs. VFs may use many MRs for NFS * divide the rest among VFs. VFs may use many MRs for NFS
* mounts, ISER, NVME applications, etc. If the firmware * mounts, ISER, NVME applications, etc. If the firmware severely
* severely restricts the number of MRs, then let PF have * restricts the number of MRs, then let PF have half and divide
* half and divide the rest among VFs, as for the other * the rest among VFs, as for the other resource types.
* resource types.
*/ */
if (rdev->qplib_ctx.mrw_count < BNXT_RE_MAX_MRW_COUNT_64K) if (qplib_ctx->mrw_count < BNXT_RE_MAX_MRW_COUNT_64K) {
vf_mrws = rdev->qplib_ctx.mrw_count * vf_pct / num_vfs; mrws = qplib_ctx->mrw_count * vf_pct;
else nvfs = num_vf;
vf_mrws = (rdev->qplib_ctx.mrw_count - } else {
BNXT_RE_RESVD_MR_FOR_PF) / rdev->num_vfs; mrws = qplib_ctx->mrw_count - BNXT_RE_RESVD_MR_FOR_PF;
vf_gids = BNXT_RE_MAX_GID_PER_VF;
} }
rdev->qplib_ctx.vf_res.max_mrw_per_vf = vf_mrws; vf_res->max_mrw_per_vf = (mrws / nvfs);
rdev->qplib_ctx.vf_res.max_gid_per_vf = vf_gids; vf_res->max_gid_per_vf = BNXT_RE_MAX_GID_PER_VF;
rdev->qplib_ctx.vf_res.max_qp_per_vf = vf_qps; }
rdev->qplib_ctx.vf_res.max_srq_per_vf = vf_srqs;
rdev->qplib_ctx.vf_res.max_cq_per_vf = vf_cqs; static void bnxt_re_set_resource_limits(struct bnxt_re_dev *rdev)
{
u32 num_vfs;
memset(&rdev->qplib_ctx.vf_res, 0, sizeof(struct bnxt_qplib_vf_res));
bnxt_re_limit_pf_res(rdev);
num_vfs = bnxt_qplib_is_chip_gen_p5(&rdev->chip_ctx) ?
BNXT_RE_GEN_P5_MAX_VF : rdev->num_vfs;
if (num_vfs)
bnxt_re_limit_vf_res(&rdev->qplib_ctx, num_vfs);
} }
/* for handling bnxt_en callbacks later */ /* for handling bnxt_en callbacks later */
@ -193,9 +208,11 @@ static void bnxt_re_sriov_config(void *p, int num_vfs)
return; return;
rdev->num_vfs = num_vfs; rdev->num_vfs = num_vfs;
if (!bnxt_qplib_is_chip_gen_p5(&rdev->chip_ctx)) {
bnxt_re_set_resource_limits(rdev); bnxt_re_set_resource_limits(rdev);
bnxt_qplib_set_func_resources(&rdev->qplib_res, &rdev->rcfw, bnxt_qplib_set_func_resources(&rdev->qplib_res, &rdev->rcfw,
&rdev->qplib_ctx); &rdev->qplib_ctx);
}
} }
static void bnxt_re_shutdown(void *p) static void bnxt_re_shutdown(void *p)
@ -894,10 +911,14 @@ static int bnxt_re_cqn_handler(struct bnxt_qplib_nq *nq,
return 0; return 0;
} }
#define BNXT_RE_GEN_P5_PF_NQ_DB 0x10000
#define BNXT_RE_GEN_P5_VF_NQ_DB 0x4000
static u32 bnxt_re_get_nqdb_offset(struct bnxt_re_dev *rdev, u16 indx) static u32 bnxt_re_get_nqdb_offset(struct bnxt_re_dev *rdev, u16 indx)
{ {
return bnxt_qplib_is_chip_gen_p5(&rdev->chip_ctx) ? return bnxt_qplib_is_chip_gen_p5(&rdev->chip_ctx) ?
0x10000 : rdev->msix_entries[indx].db_offset; (rdev->is_virtfn ? BNXT_RE_GEN_P5_VF_NQ_DB :
BNXT_RE_GEN_P5_PF_NQ_DB) :
rdev->msix_entries[indx].db_offset;
} }
static void bnxt_re_cleanup_res(struct bnxt_re_dev *rdev) static void bnxt_re_cleanup_res(struct bnxt_re_dev *rdev)
@ -1407,7 +1428,7 @@ static int bnxt_re_ib_reg(struct bnxt_re_dev *rdev)
rdev->is_virtfn); rdev->is_virtfn);
if (rc) if (rc)
goto disable_rcfw; goto disable_rcfw;
if (!rdev->is_virtfn)
bnxt_re_set_resource_limits(rdev); bnxt_re_set_resource_limits(rdev);
rc = bnxt_qplib_alloc_ctx(rdev->en_dev->pdev, &rdev->qplib_ctx, 0, rc = bnxt_qplib_alloc_ctx(rdev->en_dev->pdev, &rdev->qplib_ctx, 0,

5
drivers/infiniband/hw/bnxt_re/qplib_rcfw.c
View File

@ -494,8 +494,10 @@ int bnxt_qplib_init_rcfw(struct bnxt_qplib_rcfw *rcfw,
* shall setup this area for VF. Skipping the * shall setup this area for VF. Skipping the
* HW programming * HW programming
*/ */
if (is_virtfn || bnxt_qplib_is_chip_gen_p5(rcfw->res->cctx)) if (is_virtfn)
goto skip_ctx_setup; goto skip_ctx_setup;
if (bnxt_qplib_is_chip_gen_p5(rcfw->res->cctx))
goto config_vf_res;
level = ctx->qpc_tbl.level; level = ctx->qpc_tbl.level;
req.qpc_pg_size_qpc_lvl = (level << CMDQ_INITIALIZE_FW_QPC_LVL_SFT) | req.qpc_pg_size_qpc_lvl = (level << CMDQ_INITIALIZE_FW_QPC_LVL_SFT) |
@ -540,6 +542,7 @@ int bnxt_qplib_init_rcfw(struct bnxt_qplib_rcfw *rcfw,
req.number_of_srq = cpu_to_le32(ctx->srqc_tbl.max_elements); req.number_of_srq = cpu_to_le32(ctx->srqc_tbl.max_elements);
req.number_of_cq = cpu_to_le32(ctx->cq_tbl.max_elements); req.number_of_cq = cpu_to_le32(ctx->cq_tbl.max_elements);
config_vf_res:
req.max_qp_per_vf = cpu_to_le32(ctx->vf_res.max_qp_per_vf); req.max_qp_per_vf = cpu_to_le32(ctx->vf_res.max_qp_per_vf);
req.max_mrw_per_vf = cpu_to_le32(ctx->vf_res.max_mrw_per_vf); req.max_mrw_per_vf = cpu_to_le32(ctx->vf_res.max_mrw_per_vf);
req.max_srq_per_vf = cpu_to_le32(ctx->vf_res.max_srq_per_vf); req.max_srq_per_vf = cpu_to_le32(ctx->vf_res.max_srq_per_vf);