korg/linux
0
0
Fork 0
mirror of https://mirrors.bfsu.edu.cn/git/linux.git synced 2024-11-13 14:24:11 +08:00
Code Issues Packages Projects Releases Wiki Activity

nvme updates for Linux 6.2

Browse Source 
- fix and cleanup nvme-fc req allocation (Chaitanya Kulkarni)
  - use the common tagset helpers in nvme-pci driver (Christoph Hellwig)
  - cleanup the nvme-pci removal path (Christoph Hellwig)
  - use kstrtobool() instead of strtobool (Christophe JAILLET)
  - allow unprivileged passthrough of Identify Controller (Joel Granados)
  - support io stats on the mpath device (Sagi Grimberg)
  - minor nvmet cleanup (Sagi Grimberg)
 -----BEGIN PGP SIGNATURE-----
 
 iQI/BAABCgApFiEEgdbnc3r/njty3Iq9D55TZVIEUYMFAmOQ2s4LHGhjaEBsc3Qu
 ZGUACgkQD55TZVIEUYNt9w//StxG9XxAD6aqZvDxzYG03qYDOFNLbXRevsGMtw/K
 TSkEwzbXkiRoWdCoqQv5coB0MzaT0R6X9/Ni+tZbws+wjWCeVKMVik92SpYt+kra
 b3rk+XUtcKDIjjbda9C4Yahl+VOF4iiR+gWGbH3SQ1WOAKZvb+oF82grzsRBvMfo
 khNAc2YKgGsJLCDTox2PlSg0x3zg9b7ZkXkOW/r0l71EGULTIogqdpIEaFU3z1qw
 vWi08DH80Ce1D+2R3u8Yg2hML/E2XZCKqdB2sXTzRdJOpNFvS2ODQby8oe6Lqvg5
 /wZGMhKKrcKbJOXOMjad+aToqK09JC7Kv5DdRleLGPQNaZ15ucVDpLUsaxRo3mzW
 zoZGYQCb1S7S3CAECPm9ctJ4uB1IolCVITEV5ZdkB56UPfHp8u2qpaPK+kQANAf9
 IiXLCWIzWndxkxNT1HemuCSvf1oLJBB9/zYnnJe7UEBJaC1GC0KPysk37+kMhvWj
 SxH8HJn68/fDG8nhZ0ptf9S5/xP6xJN8rBXzLxtuv9aAJjyXGxzxVLCBaTCre8I8
 706LZKKSrUe2J5HuaS4nju19yjNgtF+rq65qokhot/ANSIQfDo20FdNP2LjcfzJE
 ShwpwTv77uXOtxebtqbpufIavdCf73OhboThFo2GY4aELUaE7JS6u1yneq/4ALbZ
 RzE=
 =3ncj
 -----END PGP SIGNATURE-----

Merge tag 'nvme-6.2-2022-12-07' of git://git.infradead.org/nvme into for-6.2/block

Pull NVMe updates from Christoph:

"nvme updates for Linux 6.2

 - fix and cleanup nvme-fc req allocation (Chaitanya Kulkarni)
 - use the common tagset helpers in nvme-pci driver (Christoph Hellwig)
 - cleanup the nvme-pci removal path (Christoph Hellwig)
 - use kstrtobool() instead of strtobool (Christophe JAILLET)
 - allow unprivileged passthrough of Identify Controller (Joel Granados)
 - support io stats on the mpath device (Sagi Grimberg)
 - minor nvmet cleanup (Sagi Grimberg)"

* tag 'nvme-6.2-2022-12-07' of git://git.infradead.org/nvme: (22 commits)
  nvmet: don't open-code NVME_NS_ATTR_RO enumeration
  nvme-pci: use the tagset alloc/free helpers
  nvme: add the Apple shared tag workaround to nvme_alloc_io_tag_set
  nvme: only set reserved_tags in nvme_alloc_io_tag_set for fabrics controllers
  nvme: consolidate setting the tagset flags
  nvme: pass nr_maps explicitly to nvme_alloc_io_tag_set
  nvme-pci: split out a nvme_pci_ctrl_is_dead helper
  nvme-pci: return early on ctrl state mismatch in nvme_reset_work
  nvme-pci: rename nvme_disable_io_queues
  nvme-pci: cleanup nvme_suspend_queue
  nvme-pci: remove nvme_pci_disable
  nvme-pci: remove nvme_disable_admin_queue
  nvme: merge nvme_shutdown_ctrl into nvme_disable_ctrl
  nvme: use nvme_wait_ready in nvme_shutdown_ctrl
  nvme-apple: fix controller shutdown in apple_nvme_disable
  nvme-fc: move common code into helper
  nvme-fc: avoid null pointer dereference
  nvme: allow unprivileged passthrough of Identify Controller
  nvme-multipath: support io stats on the mpath device
  nvme: introduce nvme_start_request
  ...
This commit is contained in:
Jens Axboe 2022-12-07 12:39:37 -07:00
commit 8f415307c3
12 changed files with 214 additions and 238 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
drivers/nvme/host/apple.c
View File

@ -763,7 +763,7 @@ static blk_status_t apple_nvme_queue_rq(struct blk_mq_hw_ctx *hctx,
goto out_free_cmd;
}
blk_mq_start_request(req);
nvme_start_request(req);
apple_nvme_submit_cmd(q, cmnd);
return BLK_STS_OK;
@ -829,9 +829,7 @@ static void apple_nvme_disable(struct apple_nvme *anv, bool shutdown)
apple_nvme_remove_cq(anv);
}
if (shutdown)
nvme_shutdown_ctrl(&anv->ctrl);
nvme_disable_ctrl(&anv->ctrl);
nvme_disable_ctrl(&anv->ctrl, shutdown);
}
WRITE_ONCE(anv->ioq.enabled, false);

98
drivers/nvme/host/core.c
View File

@ -384,6 +384,8 @@ static inline void nvme_end_req(struct request *req)
nvme_log_error(req);
nvme_end_req_zoned(req);
nvme_trace_bio_complete(req);
if (req->cmd_flags & REQ_NVME_MPATH)
nvme_mpath_end_request(req);
blk_mq_end_request(req, status);
}
@ -2250,16 +2252,17 @@ static const struct block_device_operations nvme_bdev_ops = {
.pr_ops = &nvme_pr_ops,
};
static int nvme_wait_ready(struct nvme_ctrl *ctrl, u32 timeout, bool enabled)
static int nvme_wait_ready(struct nvme_ctrl *ctrl, u32 mask, u32 val,
u32 timeout, const char *op)
{
unsigned long timeout_jiffies = ((timeout + 1) * HZ / 2) + jiffies;
u32 csts, bit = enabled ? NVME_CSTS_RDY : 0;
unsigned long timeout_jiffies = jiffies + timeout * HZ;
u32 csts;
int ret;
while ((ret = ctrl->ops->reg_read32(ctrl, NVME_REG_CSTS, &csts)) == 0) {
if (csts == ~0)
return -ENODEV;
if ((csts & NVME_CSTS_RDY) == bit)
if ((csts & mask) == val)
break;
usleep_range(1000, 2000);
@ -2268,7 +2271,7 @@ static int nvme_wait_ready(struct nvme_ctrl *ctrl, u32 timeout, bool enabled)
if (time_after(jiffies, timeout_jiffies)) {
dev_err(ctrl->device,
"Device not ready; aborting %s, CSTS=0x%x\n",
enabled ? "initialisation" : "reset", csts);
op, csts);
return -ENODEV;
}
}
@ -2276,27 +2279,29 @@ static int nvme_wait_ready(struct nvme_ctrl *ctrl, u32 timeout, bool enabled)
return ret;
}
/*
* If the device has been passed off to us in an enabled state, just clear
* the enabled bit. The spec says we should set the 'shutdown notification
* bits', but doing so may cause the device to complete commands to the
* admin queue ... and we don't know what memory that might be pointing at!
*/
int nvme_disable_ctrl(struct nvme_ctrl *ctrl)
int nvme_disable_ctrl(struct nvme_ctrl *ctrl, bool shutdown)
{
int ret;
ctrl->ctrl_config &= ~NVME_CC_SHN_MASK;
ctrl->ctrl_config &= ~NVME_CC_ENABLE;
if (shutdown)
ctrl->ctrl_config |= NVME_CC_SHN_NORMAL;
else
ctrl->ctrl_config &= ~NVME_CC_ENABLE;
ret = ctrl->ops->reg_write32(ctrl, NVME_REG_CC, ctrl->ctrl_config);
if (ret)
return ret;
if (shutdown) {
return nvme_wait_ready(ctrl, NVME_CSTS_SHST_MASK,
NVME_CSTS_SHST_CMPLT,
ctrl->shutdown_timeout, "shutdown");
}
if (ctrl->quirks & NVME_QUIRK_DELAY_BEFORE_CHK_RDY)
msleep(NVME_QUIRK_DELAY_AMOUNT);
return nvme_wait_ready(ctrl, NVME_CAP_TIMEOUT(ctrl->cap), false);
return nvme_wait_ready(ctrl, NVME_CSTS_RDY, 0,
(NVME_CAP_TIMEOUT(ctrl->cap) + 1) / 2, "reset");
}
EXPORT_SYMBOL_GPL(nvme_disable_ctrl);
@ -2361,41 +2366,11 @@ int nvme_enable_ctrl(struct nvme_ctrl *ctrl)
ret = ctrl->ops->reg_write32(ctrl, NVME_REG_CC, ctrl->ctrl_config);
if (ret)
return ret;
return nvme_wait_ready(ctrl, timeout, true);
return nvme_wait_ready(ctrl, NVME_CSTS_RDY, NVME_CSTS_RDY,
(timeout + 1) / 2, "initialisation");
}
EXPORT_SYMBOL_GPL(nvme_enable_ctrl);
int nvme_shutdown_ctrl(struct nvme_ctrl *ctrl)
{
unsigned long timeout = jiffies + (ctrl->shutdown_timeout * HZ);
u32 csts;
int ret;
ctrl->ctrl_config &= ~NVME_CC_SHN_MASK;
ctrl->ctrl_config |= NVME_CC_SHN_NORMAL;
ret = ctrl->ops->reg_write32(ctrl, NVME_REG_CC, ctrl->ctrl_config);
if (ret)
return ret;
while ((ret = ctrl->ops->reg_read32(ctrl, NVME_REG_CSTS, &csts)) == 0) {
if ((csts & NVME_CSTS_SHST_MASK) == NVME_CSTS_SHST_CMPLT)
break;
msleep(100);
if (fatal_signal_pending(current))
return -EINTR;
if (time_after(jiffies, timeout)) {
dev_err(ctrl->device,
"Device shutdown incomplete; abort shutdown\n");
return -ENODEV;
}
}
return ret;
}
EXPORT_SYMBOL_GPL(nvme_shutdown_ctrl);
static int nvme_configure_timestamp(struct nvme_ctrl *ctrl)
{
__le64 ts;
@ -4856,8 +4831,7 @@ void nvme_complete_async_event(struct nvme_ctrl *ctrl, __le16 status,
EXPORT_SYMBOL_GPL(nvme_complete_async_event);
int nvme_alloc_admin_tag_set(struct nvme_ctrl *ctrl, struct blk_mq_tag_set *set,
const struct blk_mq_ops *ops, unsigned int flags,
unsigned int cmd_size)
const struct blk_mq_ops *ops, unsigned int cmd_size)
{
int ret;
@ -4867,7 +4841,9 @@ int nvme_alloc_admin_tag_set(struct nvme_ctrl *ctrl, struct blk_mq_tag_set *set,
if (ctrl->ops->flags & NVME_F_FABRICS)
set->reserved_tags = NVMF_RESERVED_TAGS;
set->numa_node = ctrl->numa_node;
set->flags = flags;
set->flags = BLK_MQ_F_NO_SCHED;
if (ctrl->ops->flags & NVME_F_BLOCKING)
set->flags |= BLK_MQ_F_BLOCKING;
set->cmd_size = cmd_size;
set->driver_data = ctrl;
set->nr_hw_queues = 1;
@ -4915,7 +4891,7 @@ void nvme_remove_admin_tag_set(struct nvme_ctrl *ctrl)
EXPORT_SYMBOL_GPL(nvme_remove_admin_tag_set);
int nvme_alloc_io_tag_set(struct nvme_ctrl *ctrl, struct blk_mq_tag_set *set,
const struct blk_mq_ops *ops, unsigned int flags,
const struct blk_mq_ops *ops, unsigned int nr_maps,
unsigned int cmd_size)
{
int ret;
@ -4923,15 +4899,23 @@ int nvme_alloc_io_tag_set(struct nvme_ctrl *ctrl, struct blk_mq_tag_set *set,
memset(set, 0, sizeof(*set));
set->ops = ops;
set->queue_depth = ctrl->sqsize + 1;
set->reserved_tags = NVMF_RESERVED_TAGS;
/*
* Some Apple controllers requires tags to be unique across admin and
* the (only) I/O queue, so reserve the first 32 tags of the I/O queue.
*/
if (ctrl->quirks & NVME_QUIRK_SHARED_TAGS)
set->reserved_tags = NVME_AQ_DEPTH;
else if (ctrl->ops->flags & NVME_F_FABRICS)
set->reserved_tags = NVMF_RESERVED_TAGS;
set->numa_node = ctrl->numa_node;
set->flags = flags;
set->flags = BLK_MQ_F_SHOULD_MERGE;
if (ctrl->ops->flags & NVME_F_BLOCKING)
set->flags |= BLK_MQ_F_BLOCKING;
set->cmd_size = cmd_size,
set->driver_data = ctrl;
set->nr_hw_queues = ctrl->queue_count - 1;
set->timeout = NVME_IO_TIMEOUT;
if (ops->map_queues)
set->nr_maps = ctrl->opts->nr_poll_queues ? HCTX_MAX_TYPES : 2;
set->nr_maps = nr_maps;
ret = blk_mq_alloc_tag_set(set);
if (ret)
return ret;
@ -5215,6 +5199,8 @@ EXPORT_SYMBOL_GPL(nvme_start_freeze);
void nvme_quiesce_io_queues(struct nvme_ctrl *ctrl)
{
if (!ctrl->tagset)
return;
if (!test_and_set_bit(NVME_CTRL_STOPPED, &ctrl->flags))
blk_mq_quiesce_tagset(ctrl->tagset);
else
@ -5224,6 +5210,8 @@ EXPORT_SYMBOL_GPL(nvme_quiesce_io_queues);
void nvme_unquiesce_io_queues(struct nvme_ctrl *ctrl)
{
if (!ctrl->tagset)
return;
if (test_and_clear_bit(NVME_CTRL_STOPPED, &ctrl->flags))
blk_mq_unquiesce_tagset(ctrl->tagset);
}

30
drivers/nvme/host/fc.c
View File

@ -1701,6 +1701,15 @@ restart:
spin_unlock_irqrestore(&rport->lock, flags);
}
static
void nvme_fc_rcv_ls_req_err_msg(struct nvme_fc_lport *lport,
struct fcnvme_ls_rqst_w0 *w0)
{
dev_info(lport->dev, "RCV %s LS failed: No memory\n",
(w0->ls_cmd <= NVME_FC_LAST_LS_CMD_VALUE) ?
nvmefc_ls_names[w0->ls_cmd] : "");
}
/**
* nvme_fc_rcv_ls_req - transport entry point called by an LLDD
* upon the reception of a NVME LS request.
@ -1754,13 +1763,16 @@ nvme_fc_rcv_ls_req(struct nvme_fc_remote_port *portptr,
}
lsop = kzalloc(sizeof(*lsop), GFP_KERNEL);
if (!lsop) {
nvme_fc_rcv_ls_req_err_msg(lport, w0);
ret = -ENOMEM;
goto out_put;
}
lsop->rqstbuf = kzalloc(sizeof(*lsop->rqstbuf), GFP_KERNEL);
lsop->rspbuf = kzalloc(sizeof(*lsop->rspbuf), GFP_KERNEL);
if (!lsop || !lsop->rqstbuf || !lsop->rspbuf) {
dev_info(lport->dev,
"RCV %s LS failed: No memory\n",
(w0->ls_cmd <= NVME_FC_LAST_LS_CMD_VALUE) ?
nvmefc_ls_names[w0->ls_cmd] : "");
if (!lsop->rqstbuf || !lsop->rspbuf) {
nvme_fc_rcv_ls_req_err_msg(lport, w0);
ret = -ENOMEM;
goto out_free;
}
@ -2506,7 +2518,7 @@ __nvme_fc_abort_outstanding_ios(struct nvme_fc_ctrl *ctrl, bool start_queues)
* Other transports, which don't have link-level contexts bound
* to sqe's, would try to gracefully shutdown the controller by
* writing the registers for shutdown and polling (call
* nvme_shutdown_ctrl()). Given a bunch of i/o was potentially
* nvme_disable_ctrl()). Given a bunch of i/o was potentially
* just aborted and we will wait on those contexts, and given
* there was no indication of how live the controlelr is on the
* link, don't send more io to create more contexts for the
@ -2733,7 +2745,7 @@ nvme_fc_start_fcp_op(struct nvme_fc_ctrl *ctrl, struct nvme_fc_queue *queue,
atomic_set(&op->state, FCPOP_STATE_ACTIVE);
if (!(op->flags & FCOP_FLAGS_AEN))
blk_mq_start_request(op->rq);
nvme_start_request(op->rq);
cmdiu->csn = cpu_to_be32(atomic_inc_return(&queue->csn));
ret = ctrl->lport->ops->fcp_io(&ctrl->lport->localport,
@ -2904,7 +2916,7 @@ nvme_fc_create_io_queues(struct nvme_fc_ctrl *ctrl)
nvme_fc_init_io_queues(ctrl);
ret = nvme_alloc_io_tag_set(&ctrl->ctrl, &ctrl->tag_set,
&nvme_fc_mq_ops, BLK_MQ_F_SHOULD_MERGE,
&nvme_fc_mq_ops, 1,
struct_size((struct nvme_fcp_op_w_sgl *)NULL, priv,
ctrl->lport->ops->fcprqst_priv_sz));
if (ret)
@ -3510,7 +3522,7 @@ nvme_fc_init_ctrl(struct device *dev, struct nvmf_ctrl_options *opts,
nvme_fc_init_queue(ctrl, 0);
ret = nvme_alloc_admin_tag_set(&ctrl->ctrl, &ctrl->admin_tag_set,
&nvme_fc_admin_mq_ops, BLK_MQ_F_NO_SCHED,
&nvme_fc_admin_mq_ops,
struct_size((struct nvme_fcp_op_w_sgl *)NULL, priv,
ctrl->lport->ops->fcprqst_priv_sz));
if (ret)

2
drivers/nvme/host/ioctl.c
View File

@ -34,6 +34,8 @@ static bool nvme_cmd_allowed(struct nvme_ns *ns, struct nvme_command *c,
case NVME_ID_CNS_NS:
case NVME_ID_CNS_CS_NS:
case NVME_ID_CNS_NS_CS_INDEP:
case NVME_ID_CNS_CS_CTRL:
case NVME_ID_CNS_CTRL:
return true;
}
}

26
drivers/nvme/host/multipath.c
View File

@ -114,6 +114,31 @@ void nvme_failover_req(struct request *req)
kblockd_schedule_work(&ns->head->requeue_work);
}
void nvme_mpath_start_request(struct request *rq)
{
struct nvme_ns *ns = rq->q->queuedata;
struct gendisk *disk = ns->head->disk;
if (!blk_queue_io_stat(disk->queue) || blk_rq_is_passthrough(rq))
return;
nvme_req(rq)->flags |= NVME_MPATH_IO_STATS;
nvme_req(rq)->start_time = bdev_start_io_acct(disk->part0,
blk_rq_bytes(rq) >> SECTOR_SHIFT,
req_op(rq), jiffies);
}
EXPORT_SYMBOL_GPL(nvme_mpath_start_request);
void nvme_mpath_end_request(struct request *rq)
{
struct nvme_ns *ns = rq->q->queuedata;
if (!(nvme_req(rq)->flags & NVME_MPATH_IO_STATS))
return;
bdev_end_io_acct(ns->head->disk->part0, req_op(rq),
nvme_req(rq)->start_time);
}
void nvme_kick_requeue_lists(struct nvme_ctrl *ctrl)
{
struct nvme_ns *ns;
@ -503,6 +528,7 @@ int nvme_mpath_alloc_disk(struct nvme_ctrl *ctrl, struct nvme_ns_head *head)
blk_queue_flag_set(QUEUE_FLAG_NONROT, head->disk->queue);
blk_queue_flag_set(QUEUE_FLAG_NOWAIT, head->disk->queue);
blk_queue_flag_set(QUEUE_FLAG_IO_STAT, head->disk->queue);
/*
* This assumes all controllers that refer to a namespace either
* support poll queues or not. That is not a strict guarantee,

29
drivers/nvme/host/nvme.h
View File

@ -162,6 +162,9 @@ struct nvme_request {
u8 retries;
u8 flags;
u16 status;
#ifdef CONFIG_NVME_MULTIPATH
unsigned long start_time;
#endif
struct nvme_ctrl *ctrl;
};
@ -173,6 +176,7 @@ struct nvme_request {
enum {
NVME_REQ_CANCELLED = (1 << 0),
NVME_REQ_USERCMD = (1 << 1),
NVME_MPATH_IO_STATS = (1 << 2),
};
static inline struct nvme_request *nvme_req(struct request *req)
@ -508,6 +512,8 @@ struct nvme_ctrl_ops {
unsigned int flags;
#define NVME_F_FABRICS (1 << 0)
#define NVME_F_METADATA_SUPPORTED (1 << 1)
#define NVME_F_BLOCKING (1 << 2)
const struct attribute_group **dev_attr_groups;
int (*reg_read32)(struct nvme_ctrl *ctrl, u32 off, u32 *val);
int (*reg_write32)(struct nvme_ctrl *ctrl, u32 off, u32 val);
@ -729,9 +735,8 @@ void nvme_cancel_tagset(struct nvme_ctrl *ctrl);
void nvme_cancel_admin_tagset(struct nvme_ctrl *ctrl);
bool nvme_change_ctrl_state(struct nvme_ctrl *ctrl,
enum nvme_ctrl_state new_state);
int nvme_disable_ctrl(struct nvme_ctrl *ctrl);
int nvme_disable_ctrl(struct nvme_ctrl *ctrl, bool shutdown);
int nvme_enable_ctrl(struct nvme_ctrl *ctrl);
int nvme_shutdown_ctrl(struct nvme_ctrl *ctrl);
int nvme_init_ctrl(struct nvme_ctrl *ctrl, struct device *dev,
const struct nvme_ctrl_ops *ops, unsigned long quirks);
void nvme_uninit_ctrl(struct nvme_ctrl *ctrl);
@ -739,11 +744,10 @@ void nvme_start_ctrl(struct nvme_ctrl *ctrl);
void nvme_stop_ctrl(struct nvme_ctrl *ctrl);
int nvme_init_ctrl_finish(struct nvme_ctrl *ctrl, bool was_suspended);
int nvme_alloc_admin_tag_set(struct nvme_ctrl *ctrl, struct blk_mq_tag_set *set,
const struct blk_mq_ops *ops, unsigned int flags,
unsigned int cmd_size);
const struct blk_mq_ops *ops, unsigned int cmd_size);
void nvme_remove_admin_tag_set(struct nvme_ctrl *ctrl);
int nvme_alloc_io_tag_set(struct nvme_ctrl *ctrl, struct blk_mq_tag_set *set,
const struct blk_mq_ops *ops, unsigned int flags,
const struct blk_mq_ops *ops, unsigned int nr_maps,
unsigned int cmd_size);
void nvme_remove_io_tag_set(struct nvme_ctrl *ctrl);
@ -882,6 +886,8 @@ bool nvme_mpath_clear_current_path(struct nvme_ns *ns);
void nvme_mpath_revalidate_paths(struct nvme_ns *ns);
void nvme_mpath_clear_ctrl_paths(struct nvme_ctrl *ctrl);
void nvme_mpath_shutdown_disk(struct nvme_ns_head *head);
void nvme_mpath_start_request(struct request *rq);
void nvme_mpath_end_request(struct request *rq);
static inline void nvme_trace_bio_complete(struct request *req)
{
@ -967,6 +973,12 @@ static inline void nvme_mpath_start_freeze(struct nvme_subsystem *subsys)
static inline void nvme_mpath_default_iopolicy(struct nvme_subsystem *subsys)
{
}
static inline void nvme_mpath_start_request(struct request *rq)
{
}
static inline void nvme_mpath_end_request(struct request *rq)
{
}
#endif /* CONFIG_NVME_MULTIPATH */
int nvme_revalidate_zones(struct nvme_ns *ns);
@ -1012,6 +1024,13 @@ static inline void nvme_hwmon_exit(struct nvme_ctrl *ctrl)
}
#endif
static inline void nvme_start_request(struct request *rq)
{
if (rq->cmd_flags & REQ_NVME_MPATH)
nvme_mpath_start_request(rq);
blk_mq_start_request(rq);
}
static inline bool nvme_ctrl_sgl_supported(struct nvme_ctrl *ctrl)
{
return ctrl->sgls & ((1 << 0) | (1 << 1));

209
drivers/nvme/host/pci.c
View File

@ -15,6 +15,7 @@
#include <linux/init.h>
#include <linux/interrupt.h>
#include <linux/io.h>
#include <linux/kstrtox.h>
#include <linux/memremap.h>
#include <linux/mm.h>
#include <linux/module.h>
@ -108,7 +109,7 @@ struct nvme_dev;
struct nvme_queue;
static void nvme_dev_disable(struct nvme_dev *dev, bool shutdown);
static bool __nvme_disable_io_queues(struct nvme_dev *dev, u8 opcode);
static void nvme_delete_io_queues(struct nvme_dev *dev);
/*
* Represents an NVM Express device. Each nvme_dev is a PCI function.
@ -397,7 +398,7 @@ static int nvme_pci_npages_sgl(void)
static int nvme_admin_init_hctx(struct blk_mq_hw_ctx *hctx, void *data,
unsigned int hctx_idx)
{
struct nvme_dev *dev = data;
struct nvme_dev *dev = to_nvme_dev(data);
struct nvme_queue *nvmeq = &dev->queues[0];
WARN_ON(hctx_idx != 0);
@ -410,7 +411,7 @@ static int nvme_admin_init_hctx(struct blk_mq_hw_ctx *hctx, void *data,
static int nvme_init_hctx(struct blk_mq_hw_ctx *hctx, void *data,
unsigned int hctx_idx)
{
struct nvme_dev *dev = data;
struct nvme_dev *dev = to_nvme_dev(data);
struct nvme_queue *nvmeq = &dev->queues[hctx_idx + 1];
WARN_ON(dev->tagset.tags[hctx_idx] != hctx->tags);
@ -422,7 +423,7 @@ static int nvme_pci_init_request(struct blk_mq_tag_set *set,
struct request *req, unsigned int hctx_idx,
unsigned int numa_node)
{
struct nvme_dev *dev = set->driver_data;
struct nvme_dev *dev = to_nvme_dev(set->driver_data);
struct nvme_iod *iod = blk_mq_rq_to_pdu(req);
nvme_req(req)->ctrl = &dev->ctrl;
@ -441,7 +442,7 @@ static int queue_irq_offset(struct nvme_dev *dev)
static void nvme_pci_map_queues(struct blk_mq_tag_set *set)
{
struct nvme_dev *dev = set->driver_data;
struct nvme_dev *dev = to_nvme_dev(set->driver_data);
int i, qoff, offset;
offset = queue_irq_offset(dev);
@ -906,7 +907,7 @@ static blk_status_t nvme_prep_rq(struct nvme_dev *dev, struct request *req)
goto out_unmap_data;
}
blk_mq_start_request(req);
nvme_start_request(req);
return BLK_STS_OK;
out_unmap_data:
nvme_unmap_data(dev, req);
@ -1467,14 +1468,12 @@ static void nvme_free_queues(struct nvme_dev *dev, int lowest)
}
}
/**
* nvme_suspend_queue - put queue into suspended state
* @nvmeq: queue to suspend
*/
static int nvme_suspend_queue(struct nvme_queue *nvmeq)
static void nvme_suspend_queue(struct nvme_dev *dev, unsigned int qid)
{
struct nvme_queue *nvmeq = &dev->queues[qid];
if (!test_and_clear_bit(NVMEQ_ENABLED, &nvmeq->flags))
return 1;
return;
/* ensure that nvme_queue_rq() sees NVMEQ_ENABLED cleared */
mb();
@ -1483,8 +1482,7 @@ static int nvme_suspend_queue(struct nvme_queue *nvmeq)
if (!nvmeq->qid && nvmeq->dev->ctrl.admin_q)
nvme_quiesce_admin_queue(&nvmeq->dev->ctrl);
if (!test_and_clear_bit(NVMEQ_POLLED, &nvmeq->flags))
pci_free_irq(to_pci_dev(nvmeq->dev->dev), nvmeq->cq_vector, nvmeq);
return 0;
pci_free_irq(to_pci_dev(dev->dev), nvmeq->cq_vector, nvmeq);
}
static void nvme_suspend_io_queues(struct nvme_dev *dev)
@ -1492,19 +1490,7 @@ static void nvme_suspend_io_queues(struct nvme_dev *dev)
int i;
for (i = dev->ctrl.queue_count - 1; i > 0; i--)
nvme_suspend_queue(&dev->queues[i]);
}
static void nvme_disable_admin_queue(struct nvme_dev *dev, bool shutdown)
{
struct nvme_queue *nvmeq = &dev->queues[0];
if (shutdown)
nvme_shutdown_ctrl(&dev->ctrl);
else
nvme_disable_ctrl(&dev->ctrl);
nvme_poll_irqdisable(nvmeq);
nvme_suspend_queue(dev, i);
}
/*
@ -1742,39 +1728,10 @@ static void nvme_dev_remove_admin(struct nvme_dev *dev)
* queue to flush these to completion.
*/
nvme_unquiesce_admin_queue(&dev->ctrl);
blk_mq_destroy_queue(dev->ctrl.admin_q);
blk_put_queue(dev->ctrl.admin_q);
blk_mq_free_tag_set(&dev->admin_tagset);
nvme_remove_admin_tag_set(&dev->ctrl);
}
}
static int nvme_pci_alloc_admin_tag_set(struct nvme_dev *dev)
{
struct blk_mq_tag_set *set = &dev->admin_tagset;
set->ops = &nvme_mq_admin_ops;
set->nr_hw_queues = 1;
set->queue_depth = NVME_AQ_MQ_TAG_DEPTH;
set->timeout = NVME_ADMIN_TIMEOUT;
set->numa_node = dev->ctrl.numa_node;
set->cmd_size = sizeof(struct nvme_iod);
set->flags = BLK_MQ_F_NO_SCHED;
set->driver_data = dev;
if (blk_mq_alloc_tag_set(set))
return -ENOMEM;
dev->ctrl.admin_tagset = set;
dev->ctrl.admin_q = blk_mq_init_queue(set);
if (IS_ERR(dev->ctrl.admin_q)) {
blk_mq_free_tag_set(set);
dev->ctrl.admin_q = NULL;
return -ENOMEM;
}
return 0;
}
static unsigned long db_bar_size(struct nvme_dev *dev, unsigned nr_io_queues)
{
return NVME_REG_DBS + ((nr_io_queues + 1) * 8 * dev->db_stride);
@ -1818,7 +1775,14 @@ static int nvme_pci_configure_admin_queue(struct nvme_dev *dev)
(readl(dev->bar + NVME_REG_CSTS) & NVME_CSTS_NSSRO))
writel(NVME_CSTS_NSSRO, dev->bar + NVME_REG_CSTS);
result = nvme_disable_ctrl(&dev->ctrl);
/*
* If the device has been passed off to us in an enabled state, just
* clear the enabled bit. The spec says we should set the 'shutdown
* notification bits', but doing so may cause the device to complete
* commands to the admin queue ... and we don't know what memory that
* might be pointing at!
*/
result = nvme_disable_ctrl(&dev->ctrl, false);
if (result < 0)
return result;
@ -2184,7 +2148,7 @@ static ssize_t hmb_store(struct device *dev, struct device_attribute *attr,
bool new;
int ret;
if (strtobool(buf, &new) < 0)
if (kstrtobool(buf, &new) < 0)
return -EINVAL;
if (new == ndev->hmb)
@ -2317,12 +2281,6 @@ static int nvme_setup_irqs(struct nvme_dev *dev, unsigned int nr_io_queues)
PCI_IRQ_ALL_TYPES | PCI_IRQ_AFFINITY, &affd);
}
static void nvme_disable_io_queues(struct nvme_dev *dev)
{
if (__nvme_disable_io_queues(dev, nvme_admin_delete_sq))
__nvme_disable_io_queues(dev, nvme_admin_delete_cq);
}
static unsigned int nvme_max_io_queues(struct nvme_dev *dev)
{
/*
@ -2430,7 +2388,7 @@ static int nvme_setup_io_queues(struct nvme_dev *dev)
if (dev->online_queues - 1 < dev->max_qid) {
nr_io_queues = dev->online_queues - 1;
nvme_disable_io_queues(dev);
nvme_delete_io_queues(dev);
result = nvme_setup_io_queues_trylock(dev);
if (result)
return result;
@ -2494,7 +2452,7 @@ static int nvme_delete_queue(struct nvme_queue *nvmeq, u8 opcode)
return 0;
}
static bool __nvme_disable_io_queues(struct nvme_dev *dev, u8 opcode)
static bool __nvme_delete_io_queues(struct nvme_dev *dev, u8 opcode)
{
int nr_queues = dev->online_queues - 1, sent = 0;
unsigned long timeout;
@ -2522,40 +2480,19 @@ static bool __nvme_disable_io_queues(struct nvme_dev *dev, u8 opcode)
return true;
}
static void nvme_pci_alloc_tag_set(struct nvme_dev *dev)
static void nvme_delete_io_queues(struct nvme_dev *dev)
{
struct blk_mq_tag_set * set = &dev->tagset;
int ret;
if (__nvme_delete_io_queues(dev, nvme_admin_delete_sq))
__nvme_delete_io_queues(dev, nvme_admin_delete_cq);
}
set->ops = &nvme_mq_ops;
set->nr_hw_queues = dev->online_queues - 1;
set->nr_maps = 1;
if (dev->io_queues[HCTX_TYPE_READ])
set->nr_maps = 2;
static unsigned int nvme_pci_nr_maps(struct nvme_dev *dev)
{
if (dev->io_queues[HCTX_TYPE_POLL])
set->nr_maps = 3;
set->timeout = NVME_IO_TIMEOUT;
set->numa_node = dev->ctrl.numa_node;
set->queue_depth = min_t(unsigned, dev->q_depth, BLK_MQ_MAX_DEPTH) - 1;
set->cmd_size = sizeof(struct nvme_iod);
set->flags = BLK_MQ_F_SHOULD_MERGE;
set->driver_data = dev;
/*
* Some Apple controllers requires tags to be unique
* across admin and IO queue, so reserve the first 32
* tags of the IO queue.
*/
if (dev->ctrl.quirks & NVME_QUIRK_SHARED_TAGS)
set->reserved_tags = NVME_AQ_DEPTH;
ret = blk_mq_alloc_tag_set(set);
if (ret) {
dev_warn(dev->ctrl.device,
"IO queues tagset allocation failed %d\n", ret);
return;
}
dev->ctrl.tagset = set;
return 3;
if (dev->io_queues[HCTX_TYPE_READ])
return 2;
return 1;
}
static void nvme_pci_update_nr_queues(struct nvme_dev *dev)
@ -2661,57 +2598,53 @@ static void nvme_dev_unmap(struct nvme_dev *dev)
pci_release_mem_regions(to_pci_dev(dev->dev));
}
static void nvme_pci_disable(struct nvme_dev *dev)
static bool nvme_pci_ctrl_is_dead(struct nvme_dev *dev)
{
struct pci_dev *pdev = to_pci_dev(dev->dev);
u32 csts;
pci_free_irq_vectors(pdev);
if (!pci_is_enabled(pdev) || !pci_device_is_present(pdev))
return true;
if (pdev->error_state != pci_channel_io_normal)
return true;
if (pci_is_enabled(pdev)) {
pci_disable_pcie_error_reporting(pdev);
pci_disable_device(pdev);
}
csts = readl(dev->bar + NVME_REG_CSTS);
return (csts & NVME_CSTS_CFS) || !(csts & NVME_CSTS_RDY);
}
static void nvme_dev_disable(struct nvme_dev *dev, bool shutdown)
{
bool dead = true, freeze = false;
struct pci_dev *pdev = to_pci_dev(dev->dev);
bool dead;
mutex_lock(&dev->shutdown_lock);
if (pci_is_enabled(pdev)) {
u32 csts;
if (pci_device_is_present(pdev))
csts = readl(dev->bar + NVME_REG_CSTS);
else
csts = ~0;
if (dev->ctrl.state == NVME_CTRL_LIVE ||
dev->ctrl.state == NVME_CTRL_RESETTING) {
freeze = true;
dead = nvme_pci_ctrl_is_dead(dev);
if (dev->ctrl.state == NVME_CTRL_LIVE ||
dev->ctrl.state == NVME_CTRL_RESETTING) {
if (pci_is_enabled(pdev))
nvme_start_freeze(&dev->ctrl);
}
dead = !!((csts & NVME_CSTS_CFS) || !(csts & NVME_CSTS_RDY) ||
pdev->error_state != pci_channel_io_normal);
/*
* Give the controller a chance to complete all entered requests
* if doing a safe shutdown.
*/
if (!dead && shutdown)
nvme_wait_freeze_timeout(&dev->ctrl, NVME_IO_TIMEOUT);
}
/*
* Give the controller a chance to complete all entered requests if
* doing a safe shutdown.
*/
if (!dead && shutdown && freeze)
nvme_wait_freeze_timeout(&dev->ctrl, NVME_IO_TIMEOUT);
nvme_quiesce_io_queues(&dev->ctrl);
if (!dead && dev->ctrl.queue_count > 0) {
nvme_disable_io_queues(dev);
nvme_disable_admin_queue(dev, shutdown);
nvme_delete_io_queues(dev);
nvme_disable_ctrl(&dev->ctrl, shutdown);
nvme_poll_irqdisable(&dev->queues[0]);
}
nvme_suspend_io_queues(dev);
nvme_suspend_queue(&dev->queues[0]);
nvme_pci_disable(dev);
nvme_suspend_queue(dev, 0);
pci_free_irq_vectors(pdev);
if (pci_is_enabled(pdev)) {
pci_disable_pcie_error_reporting(pdev);
pci_disable_device(pdev);
}
nvme_reap_pending_cqes(dev);
nvme_cancel_tagset(&dev->ctrl);
@ -2781,7 +2714,7 @@ static int nvme_pci_alloc_iod_mempool(struct nvme_dev *dev)
static void nvme_free_tagset(struct nvme_dev *dev)
{
if (dev->tagset.tags)
blk_mq_free_tag_set(&dev->tagset);
nvme_remove_io_tag_set(&dev->ctrl);
dev->ctrl.tagset = NULL;
}
@ -2806,8 +2739,7 @@ static void nvme_reset_work(struct work_struct *work)
if (dev->ctrl.state != NVME_CTRL_RESETTING) {
dev_warn(dev->ctrl.device, "ctrl state %d is not RESETTING\n",
dev->ctrl.state);
result = -ENODEV;
goto out;
return;
}
/*
@ -3113,7 +3045,8 @@ static int nvme_probe(struct pci_dev *pdev, const struct pci_device_id *id)
if (result)
goto out_release_iod_mempool;
result = nvme_pci_alloc_admin_tag_set(dev);
result = nvme_alloc_admin_tag_set(&dev->ctrl, &dev->admin_tagset,
&nvme_mq_admin_ops, sizeof(struct nvme_iod));
if (result)
goto out_disable;
@ -3143,12 +3076,14 @@ static int nvme_probe(struct pci_dev *pdev, const struct pci_device_id *id)
goto out_disable;
if (dev->online_queues > 1) {
nvme_pci_alloc_tag_set(dev);
nvme_alloc_io_tag_set(&dev->ctrl, &dev->tagset, &nvme_mq_ops,
nvme_pci_nr_maps(dev), sizeof(struct nvme_iod));
nvme_dbbuf_set(dev);
} else {
dev_warn(dev->ctrl.device, "IO queues not created\n");
}
if (!dev->ctrl.tagset)
dev_warn(dev->ctrl.device, "IO queues not created\n");
if (!nvme_change_ctrl_state(&dev->ctrl, NVME_CTRL_LIVE)) {
dev_warn(dev->ctrl.device,
"failed to mark controller live state\n");

12
drivers/nvme/host/rdma.c
View File

@ -798,7 +798,9 @@ static int nvme_rdma_alloc_tag_set(struct nvme_ctrl *ctrl)
NVME_RDMA_METADATA_SGL_SIZE;
return nvme_alloc_io_tag_set(ctrl, &to_rdma_ctrl(ctrl)->tag_set,
&nvme_rdma_mq_ops, BLK_MQ_F_SHOULD_MERGE, cmd_size);
&nvme_rdma_mq_ops,
ctrl->opts->nr_poll_queues ? HCTX_MAX_TYPES : 2,
cmd_size);
}
static void nvme_rdma_destroy_admin_queue(struct nvme_rdma_ctrl *ctrl)
@ -846,7 +848,6 @@ static int nvme_rdma_configure_admin_queue(struct nvme_rdma_ctrl *ctrl,
if (new) {
error = nvme_alloc_admin_tag_set(&ctrl->ctrl,
&ctrl->admin_tag_set, &nvme_rdma_admin_mq_ops,
BLK_MQ_F_NO_SCHED,
sizeof(struct nvme_rdma_request) +
NVME_RDMA_DATA_SGL_SIZE);
if (error)
@ -2040,7 +2041,7 @@ static blk_status_t nvme_rdma_queue_rq(struct blk_mq_hw_ctx *hctx,
if (ret)
goto unmap_qe;
blk_mq_start_request(rq);
nvme_start_request(rq);
if (IS_ENABLED(CONFIG_BLK_DEV_INTEGRITY) &&
queue->pi_support &&
@ -2208,10 +2209,7 @@ static void nvme_rdma_shutdown_ctrl(struct nvme_rdma_ctrl *ctrl, bool shutdown)
{
nvme_rdma_teardown_io_queues(ctrl, shutdown);
nvme_quiesce_admin_queue(&ctrl->ctrl);
if (shutdown)
nvme_shutdown_ctrl(&ctrl->ctrl);
else
nvme_disable_ctrl(&ctrl->ctrl);
nvme_disable_ctrl(&ctrl->ctrl, shutdown);
nvme_rdma_teardown_admin_queue(ctrl, shutdown);
}

13
drivers/nvme/host/tcp.c
View File

@ -1858,7 +1858,7 @@ static int nvme_tcp_configure_io_queues(struct nvme_ctrl *ctrl, bool new)
if (new) {
ret = nvme_alloc_io_tag_set(ctrl, &to_tcp_ctrl(ctrl)->tag_set,
&nvme_tcp_mq_ops,
BLK_MQ_F_SHOULD_MERGE | BLK_MQ_F_BLOCKING,
ctrl->opts->nr_poll_queues ? HCTX_MAX_TYPES : 2,
sizeof(struct nvme_tcp_request));
if (ret)
goto out_free_io_queues;
@ -1933,7 +1933,7 @@ static int nvme_tcp_configure_admin_queue(struct nvme_ctrl *ctrl, bool new)
if (new) {
error = nvme_alloc_admin_tag_set(ctrl,
&to_tcp_ctrl(ctrl)->admin_tag_set,
&nvme_tcp_admin_mq_ops, BLK_MQ_F_BLOCKING,
&nvme_tcp_admin_mq_ops,
sizeof(struct nvme_tcp_request));
if (error)
goto out_free_queue;
@ -2142,10 +2142,7 @@ static void nvme_tcp_teardown_ctrl(struct nvme_ctrl *ctrl, bool shutdown)
{
nvme_tcp_teardown_io_queues(ctrl, shutdown);
nvme_quiesce_admin_queue(ctrl);
if (shutdown)
nvme_shutdown_ctrl(ctrl);
else
nvme_disable_ctrl(ctrl);
nvme_disable_ctrl(ctrl, shutdown);
nvme_tcp_teardown_admin_queue(ctrl, shutdown);
}
@ -2405,7 +2402,7 @@ static blk_status_t nvme_tcp_queue_rq(struct blk_mq_hw_ctx *hctx,
if (unlikely(ret))
return ret;
blk_mq_start_request(rq);
nvme_start_request(rq);
nvme_tcp_queue_request(req, true, bd->last);
@ -2514,7 +2511,7 @@ static const struct blk_mq_ops nvme_tcp_admin_mq_ops = {
static const struct nvme_ctrl_ops nvme_tcp_ctrl_ops = {
.name = "tcp",
.module = THIS_MODULE,
.flags = NVME_F_FABRICS,
.flags = NVME_F_FABRICS | NVME_F_BLOCKING,
.reg_read32 = nvmf_reg_read32,
.reg_read64 = nvmf_reg_read64,
.reg_write32 = nvmf_reg_write32,

2
drivers/nvme/target/admin-cmd.c
View File

@ -561,7 +561,7 @@ static void nvmet_execute_identify_ns(struct nvmet_req *req)
}
if (req->ns->readonly)
id->nsattr |= (1 << 0);
id->nsattr |= NVME_NS_ATTR_RO;
done:
if (!status)
status = nvmet_copy_to_sgl(req, 0, id, sizeof(*id));

17
drivers/nvme/target/configfs.c
View File

@ -4,6 +4,7 @@
* Copyright (c) 2015-2016 HGST, a Western Digital Company.
*/
#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
#include <linux/kstrtox.h>
#include <linux/kernel.h>
#include <linux/module.h>
#include <linux/slab.h>
@ -267,7 +268,7 @@ static ssize_t nvmet_param_pi_enable_store(struct config_item *item,
struct nvmet_port *port = to_nvmet_port(item);
bool val;
if (strtobool(page, &val))
if (kstrtobool(page, &val))
return -EINVAL;
if (nvmet_is_port_enabled(port, __func__))
@ -532,7 +533,7 @@ static ssize_t nvmet_ns_enable_store(struct config_item *item,
bool enable;
int ret = 0;
if (strtobool(page, &enable))
if (kstrtobool(page, &enable))
return -EINVAL;
if (enable)
@ -556,7 +557,7 @@ static ssize_t nvmet_ns_buffered_io_store(struct config_item *item,
struct nvmet_ns *ns = to_nvmet_ns(item);
bool val;
if (strtobool(page, &val))
if (kstrtobool(page, &val))
return -EINVAL;
mutex_lock(&ns->subsys->lock);
@ -579,7 +580,7 @@ static ssize_t nvmet_ns_revalidate_size_store(struct config_item *item,
struct nvmet_ns *ns = to_nvmet_ns(item);
bool val;
if (strtobool(page, &val))
if (kstrtobool(page, &val))
return -EINVAL;
if (!val)
@ -728,7 +729,7 @@ static ssize_t nvmet_passthru_enable_store(struct config_item *item,
bool enable;
int ret = 0;
if (strtobool(page, &enable))
if (kstrtobool(page, &enable))
return -EINVAL;
if (enable)
@ -995,7 +996,7 @@ static ssize_t nvmet_subsys_attr_allow_any_host_store(struct config_item *item,
bool allow_any_host;
int ret = 0;
if (strtobool(page, &allow_any_host))
if (kstrtobool(page, &allow_any_host))
return -EINVAL;
down_write(&nvmet_config_sem);
@ -1382,7 +1383,7 @@ static ssize_t nvmet_subsys_attr_pi_enable_store(struct config_item *item,
struct nvmet_subsys *subsys = to_subsys(item);
bool pi_enable;
if (strtobool(page, &pi_enable))
if (kstrtobool(page, &pi_enable))
return -EINVAL;
subsys->pi_support = pi_enable;
@ -1511,7 +1512,7 @@ static ssize_t nvmet_referral_enable_store(struct config_item *item,
struct nvmet_port *port = to_nvmet_port(item);
bool enable;
if (strtobool(page, &enable))
if (kstrtobool(page, &enable))
goto inval;
if (enable)

8
drivers/nvme/target/loop.c
View File

@ -145,7 +145,7 @@ static blk_status_t nvme_loop_queue_rq(struct blk_mq_hw_ctx *hctx,
if (ret)
return ret;
blk_mq_start_request(req);
nvme_start_request(req);
iod->cmd.common.flags |= NVME_CMD_SGL_METABUF;
iod->req.port = queue->ctrl->port;
if (!nvmet_req_init(&iod->req, &queue->nvme_cq,
@ -353,7 +353,7 @@ static int nvme_loop_configure_admin_queue(struct nvme_loop_ctrl *ctrl)
ctrl->ctrl.queue_count = 1;
error = nvme_alloc_admin_tag_set(&ctrl->ctrl, &ctrl->admin_tag_set,
&nvme_loop_admin_mq_ops, BLK_MQ_F_NO_SCHED,
&nvme_loop_admin_mq_ops,
sizeof(struct nvme_loop_iod) +
NVME_INLINE_SG_CNT * sizeof(struct scatterlist));
if (error)
@ -401,7 +401,7 @@ static void nvme_loop_shutdown_ctrl(struct nvme_loop_ctrl *ctrl)
nvme_quiesce_admin_queue(&ctrl->ctrl);
if (ctrl->ctrl.state == NVME_CTRL_LIVE)
nvme_shutdown_ctrl(&ctrl->ctrl);
nvme_disable_ctrl(&ctrl->ctrl, true);
nvme_cancel_admin_tagset(&ctrl->ctrl);
nvme_loop_destroy_admin_queue(ctrl);
@ -494,7 +494,7 @@ static int nvme_loop_create_io_queues(struct nvme_loop_ctrl *ctrl)
return ret;
ret = nvme_alloc_io_tag_set(&ctrl->ctrl, &ctrl->tag_set,
&nvme_loop_mq_ops, BLK_MQ_F_SHOULD_MERGE,
&nvme_loop_mq_ops, 1,
sizeof(struct nvme_loop_iod) +
NVME_INLINE_SG_CNT * sizeof(struct scatterlist));
if (ret)