renegade-project/linux-next
2
0
Fork 0
mirror of https://github.com/edk2-porting/linux-next.git synced 2024-12-28 23:23:55 +08:00
Code

IB/mlx5: Support IB_WR_REG_SIG_MR

Browse Source 
This patch implements IB_WR_REG_SIG_MR posted by the user.

Baisically this WR involves 3 WQEs in order to prepare and properly
register the signature layout:

1. post UMR WR to register the sig_mr in one of two possible ways:
    * In case the user registered a single MR for data so the UMR data segment
      consists of:
      - single klm (data MR) passed by the user
      - BSF with signature attributes requested by the user.
    * In case the user registered 2 MRs, one for data and one for protection,
      the UMR consists of:
      - strided block format which includes data and protection MRs and
        their repetitive block format.
      - BSF with signature attributes requested by the user.

2. post SET_PSV in order to set the memory domain initial
   signature parameters passed by the user.
   SET_PSV is not signaled and solicited CQE.

3. post SET_PSV in order to set the wire domain initial
   signature parameters passed by the user.
   SET_PSV is not signaled and solicited CQE.

* After this compound WR we place a small fence for next WR to come.

This patch also introduces some helper functions to set the BSF correctly
and determining the signature format selectors.

Signed-off-by: Sagi Grimberg <sagig@mellanox.com>
Signed-off-by: Roland Dreier <roland@purestorage.com>
This commit is contained in:
Sagi Grimberg 2014-02-23 14:19:11 +02:00 committed by Roland Dreier
parent 3bcdb17a5e
commit e6631814fb
2 changed files with 483 additions and 0 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

422
drivers/infiniband/hw/mlx5/qp.c
View File

@ -1777,6 +1777,26 @@ static __be64 frwr_mkey_mask(void)
return cpu_to_be64(result);
}
static __be64 sig_mkey_mask(void)
{
u64 result;
result = MLX5_MKEY_MASK_LEN |
MLX5_MKEY_MASK_PAGE_SIZE |
MLX5_MKEY_MASK_START_ADDR |
MLX5_MKEY_MASK_EN_RINVAL |
MLX5_MKEY_MASK_KEY |
MLX5_MKEY_MASK_LR |
MLX5_MKEY_MASK_LW |
MLX5_MKEY_MASK_RR |
MLX5_MKEY_MASK_RW |
MLX5_MKEY_MASK_SMALL_FENCE |
MLX5_MKEY_MASK_FREE |
MLX5_MKEY_MASK_BSF_EN;
return cpu_to_be64(result);
}
static void set_frwr_umr_segment(struct mlx5_wqe_umr_ctrl_seg *umr,
struct ib_send_wr *wr, int li)
{
@ -1961,6 +1981,339 @@ static int set_data_inl_seg(struct mlx5_ib_qp *qp, struct ib_send_wr *wr,
return 0;
}
static u16 prot_field_size(enum ib_signature_type type)
{
switch (type) {
case IB_SIG_TYPE_T10_DIF:
return MLX5_DIF_SIZE;
default:
return 0;
}
}
static u8 bs_selector(int block_size)
{
switch (block_size) {
case 512: return 0x1;
case 520: return 0x2;
case 4096: return 0x3;
case 4160: return 0x4;
case 1073741824: return 0x5;
default: return 0;
}
}
static int format_selector(struct ib_sig_attrs *attr,
struct ib_sig_domain *domain,
int *selector)
{
#define FORMAT_DIF_NONE 0
#define FORMAT_DIF_CRC_INC 8
#define FORMAT_DIF_CRC_NO_INC 12
#define FORMAT_DIF_CSUM_INC 13
#define FORMAT_DIF_CSUM_NO_INC 14
switch (domain->sig.dif.type) {
case IB_T10DIF_NONE:
/* No DIF */
*selector = FORMAT_DIF_NONE;
break;
case IB_T10DIF_TYPE1: /* Fall through */
case IB_T10DIF_TYPE2:
switch (domain->sig.dif.bg_type) {
case IB_T10DIF_CRC:
*selector = FORMAT_DIF_CRC_INC;
break;
case IB_T10DIF_CSUM:
*selector = FORMAT_DIF_CSUM_INC;
break;
default:
return 1;
}
break;
case IB_T10DIF_TYPE3:
switch (domain->sig.dif.bg_type) {
case IB_T10DIF_CRC:
*selector = domain->sig.dif.type3_inc_reftag ?
FORMAT_DIF_CRC_INC :
FORMAT_DIF_CRC_NO_INC;
break;
case IB_T10DIF_CSUM:
*selector = domain->sig.dif.type3_inc_reftag ?
FORMAT_DIF_CSUM_INC :
FORMAT_DIF_CSUM_NO_INC;
break;
default:
return 1;
}
break;
default:
return 1;
}
return 0;
}
static int mlx5_set_bsf(struct ib_mr *sig_mr,
struct ib_sig_attrs *sig_attrs,
struct mlx5_bsf *bsf, u32 data_size)
{
struct mlx5_core_sig_ctx *msig = to_mmr(sig_mr)->sig;
struct mlx5_bsf_basic *basic = &bsf->basic;
struct ib_sig_domain *mem = &sig_attrs->mem;
struct ib_sig_domain *wire = &sig_attrs->wire;
int ret, selector;
switch (sig_attrs->mem.sig_type) {
case IB_SIG_TYPE_T10_DIF:
if (sig_attrs->wire.sig_type != IB_SIG_TYPE_T10_DIF)
return -EINVAL;
/* Input domain check byte mask */
basic->check_byte_mask = sig_attrs->check_mask;
if (mem->sig.dif.pi_interval == wire->sig.dif.pi_interval &&
mem->sig.dif.type == wire->sig.dif.type) {
/* Same block structure */
basic->bsf_size_sbs = 1 << 4;
if (mem->sig.dif.bg_type == wire->sig.dif.bg_type)
basic->wire.copy_byte_mask = 0xff;
else
basic->wire.copy_byte_mask = 0x3f;
} else
basic->wire.bs_selector = bs_selector(wire->sig.dif.pi_interval);
basic->mem.bs_selector = bs_selector(mem->sig.dif.pi_interval);
basic->raw_data_size = cpu_to_be32(data_size);
ret = format_selector(sig_attrs, mem, &selector);
if (ret)
return -EINVAL;
basic->m_bfs_psv = cpu_to_be32(selector << 24 |
msig->psv_memory.psv_idx);
ret = format_selector(sig_attrs, wire, &selector);
if (ret)
return -EINVAL;
basic->w_bfs_psv = cpu_to_be32(selector << 24 |
msig->psv_wire.psv_idx);
break;
default:
return -EINVAL;
}
return 0;
}
static int set_sig_data_segment(struct ib_send_wr *wr, struct mlx5_ib_qp *qp,
void **seg, int *size)
{
struct ib_sig_attrs *sig_attrs = wr->wr.sig_handover.sig_attrs;
struct ib_mr *sig_mr = wr->wr.sig_handover.sig_mr;
struct mlx5_bsf *bsf;
u32 data_len = wr->sg_list->length;
u32 data_key = wr->sg_list->lkey;
u64 data_va = wr->sg_list->addr;
int ret;
int wqe_size;
if (!wr->wr.sig_handover.prot) {
/**
* Source domain doesn't contain signature information
* So need construct:
* ------------------
* | data_klm |
* ------------------
* | BSF |
* ------------------
**/
struct mlx5_klm *data_klm = *seg;
data_klm->bcount = cpu_to_be32(data_len);
data_klm->key = cpu_to_be32(data_key);
data_klm->va = cpu_to_be64(data_va);
wqe_size = ALIGN(sizeof(*data_klm), 64);
} else {
/**
* Source domain contains signature information
* So need construct a strided block format:
* ---------------------------
* | stride_block_ctrl |
* ---------------------------
* | data_klm |
* ---------------------------
* | prot_klm |
* ---------------------------
* | BSF |
* ---------------------------
**/
struct mlx5_stride_block_ctrl_seg *sblock_ctrl;
struct mlx5_stride_block_entry *data_sentry;
struct mlx5_stride_block_entry *prot_sentry;
u32 prot_key = wr->wr.sig_handover.prot->lkey;
u64 prot_va = wr->wr.sig_handover.prot->addr;
u16 block_size = sig_attrs->mem.sig.dif.pi_interval;
int prot_size;
sblock_ctrl = *seg;
data_sentry = (void *)sblock_ctrl + sizeof(*sblock_ctrl);
prot_sentry = (void *)data_sentry + sizeof(*data_sentry);
prot_size = prot_field_size(sig_attrs->mem.sig_type);
if (!prot_size) {
pr_err("Bad block size given: %u\n", block_size);
return -EINVAL;
}
sblock_ctrl->bcount_per_cycle = cpu_to_be32(block_size +
prot_size);
sblock_ctrl->op = cpu_to_be32(MLX5_STRIDE_BLOCK_OP);
sblock_ctrl->repeat_count = cpu_to_be32(data_len / block_size);
sblock_ctrl->num_entries = cpu_to_be16(2);
data_sentry->bcount = cpu_to_be16(block_size);
data_sentry->key = cpu_to_be32(data_key);
data_sentry->va = cpu_to_be64(data_va);
prot_sentry->bcount = cpu_to_be16(prot_size);
prot_sentry->key = cpu_to_be32(prot_key);
if (prot_key == data_key && prot_va == data_va) {
/**
* The data and protection are interleaved
* in a single memory region
**/
prot_sentry->va = cpu_to_be64(data_va + block_size);
prot_sentry->stride = cpu_to_be16(block_size + prot_size);
data_sentry->stride = prot_sentry->stride;
} else {
/* The data and protection are two different buffers */
prot_sentry->va = cpu_to_be64(prot_va);
data_sentry->stride = cpu_to_be16(block_size);
prot_sentry->stride = cpu_to_be16(prot_size);
}
wqe_size = ALIGN(sizeof(*sblock_ctrl) + sizeof(*data_sentry) +
sizeof(*prot_sentry), 64);
}
*seg += wqe_size;
*size += wqe_size / 16;
if (unlikely((*seg == qp->sq.qend)))
*seg = mlx5_get_send_wqe(qp, 0);
bsf = *seg;
ret = mlx5_set_bsf(sig_mr, sig_attrs, bsf, data_len);
if (ret)
return -EINVAL;
*seg += sizeof(*bsf);
*size += sizeof(*bsf) / 16;
if (unlikely((*seg == qp->sq.qend)))
*seg = mlx5_get_send_wqe(qp, 0);
return 0;
}
static void set_sig_mkey_segment(struct mlx5_mkey_seg *seg,
struct ib_send_wr *wr, u32 nelements,
u32 length, u32 pdn)
{
struct ib_mr *sig_mr = wr->wr.sig_handover.sig_mr;
u32 sig_key = sig_mr->rkey;
memset(seg, 0, sizeof(*seg));
seg->flags = get_umr_flags(wr->wr.sig_handover.access_flags) |
MLX5_ACCESS_MODE_KLM;
seg->qpn_mkey7_0 = cpu_to_be32((sig_key & 0xff) | 0xffffff00);
seg->flags_pd = cpu_to_be32(MLX5_MKEY_REMOTE_INVAL |
MLX5_MKEY_BSF_EN | pdn);
seg->len = cpu_to_be64(length);
seg->xlt_oct_size = cpu_to_be32(be16_to_cpu(get_klm_octo(nelements)));
seg->bsfs_octo_size = cpu_to_be32(MLX5_MKEY_BSF_OCTO_SIZE);
}
static void set_sig_umr_segment(struct mlx5_wqe_umr_ctrl_seg *umr,
struct ib_send_wr *wr, u32 nelements)
{
memset(umr, 0, sizeof(*umr));
umr->flags = MLX5_FLAGS_INLINE | MLX5_FLAGS_CHECK_FREE;
umr->klm_octowords = get_klm_octo(nelements);
umr->bsf_octowords = cpu_to_be16(MLX5_MKEY_BSF_OCTO_SIZE);
umr->mkey_mask = sig_mkey_mask();
}
static int set_sig_umr_wr(struct ib_send_wr *wr, struct mlx5_ib_qp *qp,
void **seg, int *size)
{
struct mlx5_ib_mr *sig_mr = to_mmr(wr->wr.sig_handover.sig_mr);
u32 pdn = get_pd(qp)->pdn;
u32 klm_oct_size;
int region_len, ret;
if (unlikely(wr->num_sge != 1) ||
unlikely(wr->wr.sig_handover.access_flags &
IB_ACCESS_REMOTE_ATOMIC) ||
unlikely(!sig_mr->sig) || unlikely(!qp->signature_en))
return -EINVAL;
/* length of the protected region, data + protection */
region_len = wr->sg_list->length;
if (wr->wr.sig_handover.prot)
region_len += wr->wr.sig_handover.prot->length;
/**
* KLM octoword size - if protection was provided
* then we use strided block format (3 octowords),
* else we use single KLM (1 octoword)
**/
klm_oct_size = wr->wr.sig_handover.prot ? 3 : 1;
set_sig_umr_segment(*seg, wr, klm_oct_size);
*seg += sizeof(struct mlx5_wqe_umr_ctrl_seg);
*size += sizeof(struct mlx5_wqe_umr_ctrl_seg) / 16;
if (unlikely((*seg == qp->sq.qend)))
*seg = mlx5_get_send_wqe(qp, 0);
set_sig_mkey_segment(*seg, wr, klm_oct_size, region_len, pdn);
*seg += sizeof(struct mlx5_mkey_seg);
*size += sizeof(struct mlx5_mkey_seg) / 16;
if (unlikely((*seg == qp->sq.qend)))
*seg = mlx5_get_send_wqe(qp, 0);
ret = set_sig_data_segment(wr, qp, seg, size);
if (ret)
return ret;
return 0;
}
static int set_psv_wr(struct ib_sig_domain *domain,
u32 psv_idx, void **seg, int *size)
{
struct mlx5_seg_set_psv *psv_seg = *seg;
memset(psv_seg, 0, sizeof(*psv_seg));
psv_seg->psv_num = cpu_to_be32(psv_idx);
switch (domain->sig_type) {
case IB_SIG_TYPE_T10_DIF:
psv_seg->transient_sig = cpu_to_be32(domain->sig.dif.bg << 16 |
domain->sig.dif.app_tag);
psv_seg->ref_tag = cpu_to_be32(domain->sig.dif.ref_tag);
*seg += sizeof(*psv_seg);
*size += sizeof(*psv_seg) / 16;
break;
default:
pr_err("Bad signature type given.\n");
return 1;
}
return 0;
}
static int set_frwr_li_wr(void **seg, struct ib_send_wr *wr, int *size,
struct mlx5_core_dev *mdev, struct mlx5_ib_pd *pd, struct mlx5_ib_qp *qp)
{
@ -2108,6 +2461,7 @@ int mlx5_ib_post_send(struct ib_qp *ibqp, struct ib_send_wr *wr,
struct mlx5_ib_dev *dev = to_mdev(ibqp->device);
struct mlx5_core_dev *mdev = &dev->mdev;
struct mlx5_ib_qp *qp = to_mqp(ibqp);
struct mlx5_ib_mr *mr;
struct mlx5_wqe_data_seg *dpseg;
struct mlx5_wqe_xrc_seg *xrc;
struct mlx5_bf *bf = qp->bf;
@ -2203,6 +2557,73 @@ int mlx5_ib_post_send(struct ib_qp *ibqp, struct ib_send_wr *wr,
num_sge = 0;
break;
case IB_WR_REG_SIG_MR:
qp->sq.wr_data[idx] = IB_WR_REG_SIG_MR;
mr = to_mmr(wr->wr.sig_handover.sig_mr);
ctrl->imm = cpu_to_be32(mr->ibmr.rkey);
err = set_sig_umr_wr(wr, qp, &seg, &size);
if (err) {
mlx5_ib_warn(dev, "\n");
*bad_wr = wr;
goto out;
}
finish_wqe(qp, ctrl, size, idx, wr->wr_id,
nreq, get_fence(fence, wr),
next_fence, MLX5_OPCODE_UMR);
/*
* SET_PSV WQEs are not signaled and solicited
* on error
*/
wr->send_flags &= ~IB_SEND_SIGNALED;
wr->send_flags |= IB_SEND_SOLICITED;
err = begin_wqe(qp, &seg, &ctrl, wr,
&idx, &size, nreq);
if (err) {
mlx5_ib_warn(dev, "\n");
err = -ENOMEM;
*bad_wr = wr;
goto out;
}
err = set_psv_wr(&wr->wr.sig_handover.sig_attrs->mem,
mr->sig->psv_memory.psv_idx, &seg,
&size);
if (err) {
mlx5_ib_warn(dev, "\n");
*bad_wr = wr;
goto out;
}
finish_wqe(qp, ctrl, size, idx, wr->wr_id,
nreq, get_fence(fence, wr),
next_fence, MLX5_OPCODE_SET_PSV);
err = begin_wqe(qp, &seg, &ctrl, wr,
&idx, &size, nreq);
if (err) {
mlx5_ib_warn(dev, "\n");
err = -ENOMEM;
*bad_wr = wr;
goto out;
}
next_fence = MLX5_FENCE_MODE_INITIATOR_SMALL;
err = set_psv_wr(&wr->wr.sig_handover.sig_attrs->wire,
mr->sig->psv_wire.psv_idx, &seg,
&size);
if (err) {
mlx5_ib_warn(dev, "\n");
*bad_wr = wr;
goto out;
}
finish_wqe(qp, ctrl, size, idx, wr->wr_id,
nreq, get_fence(fence, wr),
next_fence, MLX5_OPCODE_SET_PSV);
num_sge = 0;
goto skip_psv;
default:
break;
}
@ -2286,6 +2707,7 @@ int mlx5_ib_post_send(struct ib_qp *ibqp, struct ib_send_wr *wr,
finish_wqe(qp, ctrl, size, idx, wr->wr_id, nreq,
get_fence(fence, wr), next_fence,
mlx5_ib_opcode[wr->opcode]);
skip_psv:
if (0)
dump_wqe(qp, idx, size);
}

61
include/linux/mlx5/qp.h
View File

@ -38,6 +38,8 @@
#define MLX5_INVALID_LKEY 0x100
#define MLX5_SIG_WQE_SIZE (MLX5_SEND_WQE_BB * 5)
#define MLX5_DIF_SIZE 8
#define MLX5_STRIDE_BLOCK_OP 0x400
enum mlx5_qp_optpar {
MLX5_QP_OPTPAR_ALT_ADDR_PATH = 1 << 0,
@ -152,6 +154,11 @@ enum {
MLX5_SND_DBR = 1,
};
enum {
MLX5_FLAGS_INLINE = 1<<7,
MLX5_FLAGS_CHECK_FREE = 1<<5,
};
struct mlx5_wqe_fmr_seg {
__be32 flags;
__be32 mem_key;
@ -279,6 +286,60 @@ struct mlx5_wqe_inline_seg {
__be32 byte_count;
};
struct mlx5_bsf {
struct mlx5_bsf_basic {
u8 bsf_size_sbs;
u8 check_byte_mask;
union {
u8 copy_byte_mask;
u8 bs_selector;
u8 rsvd_wflags;
} wire;
union {
u8 bs_selector;
u8 rsvd_mflags;
} mem;
__be32 raw_data_size;
__be32 w_bfs_psv;
__be32 m_bfs_psv;
} basic;
struct mlx5_bsf_ext {
__be32 t_init_gen_pro_size;
__be32 rsvd_epi_size;
__be32 w_tfs_psv;
__be32 m_tfs_psv;
} ext;
struct mlx5_bsf_inl {
__be32 w_inl_vld;
__be32 w_rsvd;
__be64 w_block_format;
__be32 m_inl_vld;
__be32 m_rsvd;
__be64 m_block_format;
} inl;
};
struct mlx5_klm {
__be32 bcount;
__be32 key;
__be64 va;
};
struct mlx5_stride_block_entry {
__be16 stride;
__be16 bcount;
__be32 key;
__be64 va;
};
struct mlx5_stride_block_ctrl_seg {
__be32 bcount_per_cycle;
__be32 op;
__be32 repeat_count;
u16 rsvd;
__be16 num_entries;
};
struct mlx5_core_qp {
void (*event) (struct mlx5_core_qp *, int);
int qpn;