renegade-project/linux-next
2
0
Fork 0
mirror of https://github.com/edk2-porting/linux-next.git synced 2024-12-23 04:34:11 +08:00
Code

nvme-pci: split the nvme queue lock into submission and completion locks

Browse Source 
This is now feasible. We protect the submission queue ring with
->sq_lock, and the completion side with ->cq_lock.

Reviewed-by: Christoph Hellwig <hch@lst.de>
Signed-off-by: Jens Axboe <axboe@kernel.dk>
Signed-off-by: Christoph Hellwig <hch@lst.de>
This commit is contained in:
Jens Axboe 2018-05-17 18:31:51 +02:00 committed by Keith Busch
parent 5cb525c831
commit 1ab0cd6966
1 changed files with 23 additions and 21 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

44
drivers/nvme/host/pci.c
View File

@ -147,9 +147,10 @@ static inline struct nvme_dev *to_nvme_dev(struct nvme_ctrl *ctrl)
struct nvme_queue { struct nvme_queue {
struct device *q_dmadev; struct device *q_dmadev;
struct nvme_dev *dev; struct nvme_dev *dev;
spinlock_t q_lock; spinlock_t sq_lock;
struct nvme_command *sq_cmds; struct nvme_command *sq_cmds;
struct nvme_command __iomem *sq_cmds_io; struct nvme_command __iomem *sq_cmds_io;
spinlock_t cq_lock ____cacheline_aligned_in_smp;
volatile struct nvme_completion *cqes; volatile struct nvme_completion *cqes;
struct blk_mq_tags **tags; struct blk_mq_tags **tags;
dma_addr_t sq_dma_addr; dma_addr_t sq_dma_addr;
@ -894,9 +895,9 @@ static blk_status_t nvme_queue_rq(struct blk_mq_hw_ctx *hctx,
blk_mq_start_request(req); blk_mq_start_request(req);
spin_lock_irq(&nvmeq->q_lock); spin_lock_irq(&nvmeq->sq_lock);
__nvme_submit_cmd(nvmeq, &cmnd); __nvme_submit_cmd(nvmeq, &cmnd);
spin_unlock_irq(&nvmeq->q_lock); spin_unlock_irq(&nvmeq->sq_lock);
return BLK_STS_OK; return BLK_STS_OK;
out_cleanup_iod: out_cleanup_iod:
nvme_free_iod(dev, req); nvme_free_iod(dev, req);
@ -1000,9 +1001,9 @@ static irqreturn_t nvme_irq(int irq, void *data)
struct nvme_queue *nvmeq = data; struct nvme_queue *nvmeq = data;
u16 start, end; u16 start, end;
spin_lock(&nvmeq->q_lock); spin_lock(&nvmeq->cq_lock);
nvme_process_cq(nvmeq, &start, &end, -1); nvme_process_cq(nvmeq, &start, &end, -1);
spin_unlock(&nvmeq->q_lock); spin_unlock(&nvmeq->cq_lock);
if (start == end) if (start == end)
return IRQ_NONE; return IRQ_NONE;
@ -1026,9 +1027,9 @@ static int __nvme_poll(struct nvme_queue *nvmeq, unsigned int tag)
if (!nvme_cqe_pending(nvmeq)) if (!nvme_cqe_pending(nvmeq))
return 0; return 0;
spin_lock_irq(&nvmeq->q_lock); spin_lock_irq(&nvmeq->cq_lock);
found = nvme_process_cq(nvmeq, &start, &end, tag); found = nvme_process_cq(nvmeq, &start, &end, tag);
spin_unlock_irq(&nvmeq->q_lock); spin_unlock_irq(&nvmeq->cq_lock);
nvme_complete_cqes(nvmeq, start, end); nvme_complete_cqes(nvmeq, start, end);
return found; return found;
@ -1051,9 +1052,9 @@ static void nvme_pci_submit_async_event(struct nvme_ctrl *ctrl)
c.common.opcode = nvme_admin_async_event; c.common.opcode = nvme_admin_async_event;
c.common.command_id = NVME_AQ_BLK_MQ_DEPTH; c.common.command_id = NVME_AQ_BLK_MQ_DEPTH;
spin_lock_irq(&nvmeq->q_lock); spin_lock_irq(&nvmeq->sq_lock);
__nvme_submit_cmd(nvmeq, &c); __nvme_submit_cmd(nvmeq, &c);
spin_unlock_irq(&nvmeq->q_lock); spin_unlock_irq(&nvmeq->sq_lock);
} }
static int adapter_delete_queue(struct nvme_dev *dev, u8 opcode, u16 id) static int adapter_delete_queue(struct nvme_dev *dev, u8 opcode, u16 id)
@ -1310,15 +1311,15 @@ static int nvme_suspend_queue(struct nvme_queue *nvmeq)
{ {
int vector; int vector;
spin_lock_irq(&nvmeq->q_lock); spin_lock_irq(&nvmeq->cq_lock);
if (nvmeq->cq_vector == -1) { if (nvmeq->cq_vector == -1) {
spin_unlock_irq(&nvmeq->q_lock); spin_unlock_irq(&nvmeq->cq_lock);
return 1; return 1;
} }
vector = nvmeq->cq_vector; vector = nvmeq->cq_vector;
nvmeq->dev->online_queues--; nvmeq->dev->online_queues--;
nvmeq->cq_vector = -1; nvmeq->cq_vector = -1;
spin_unlock_irq(&nvmeq->q_lock); spin_unlock_irq(&nvmeq->cq_lock);
/* /*
* Ensure that nvme_queue_rq() sees it ->cq_vector == -1 without * Ensure that nvme_queue_rq() sees it ->cq_vector == -1 without
@ -1344,9 +1345,9 @@ static void nvme_disable_admin_queue(struct nvme_dev *dev, bool shutdown)
else else
nvme_disable_ctrl(&dev->ctrl, dev->ctrl.cap); nvme_disable_ctrl(&dev->ctrl, dev->ctrl.cap);
spin_lock_irq(&nvmeq->q_lock); spin_lock_irq(&nvmeq->cq_lock);
nvme_process_cq(nvmeq, &start, &end, -1); nvme_process_cq(nvmeq, &start, &end, -1);
spin_unlock_irq(&nvmeq->q_lock); spin_unlock_irq(&nvmeq->cq_lock);
nvme_complete_cqes(nvmeq, start, end); nvme_complete_cqes(nvmeq, start, end);
} }
@ -1406,7 +1407,8 @@ static int nvme_alloc_queue(struct nvme_dev *dev, int qid, int depth)
nvmeq->q_dmadev = dev->dev; nvmeq->q_dmadev = dev->dev;
nvmeq->dev = dev; nvmeq->dev = dev;
spin_lock_init(&nvmeq->q_lock); spin_lock_init(&nvmeq->sq_lock);
spin_lock_init(&nvmeq->cq_lock);
nvmeq->cq_head = 0; nvmeq->cq_head = 0;
nvmeq->cq_phase = 1; nvmeq->cq_phase = 1;
nvmeq->q_db = &dev->dbs[qid * 2 * dev->db_stride]; nvmeq->q_db = &dev->dbs[qid * 2 * dev->db_stride];
@ -1442,7 +1444,7 @@ static void nvme_init_queue(struct nvme_queue *nvmeq, u16 qid)
{ {
struct nvme_dev *dev = nvmeq->dev; struct nvme_dev *dev = nvmeq->dev;
spin_lock_irq(&nvmeq->q_lock); spin_lock_irq(&nvmeq->cq_lock);
nvmeq->sq_tail = 0; nvmeq->sq_tail = 0;
nvmeq->cq_head = 0; nvmeq->cq_head = 0;
nvmeq->cq_phase = 1; nvmeq->cq_phase = 1;
@ -1450,7 +1452,7 @@ static void nvme_init_queue(struct nvme_queue *nvmeq, u16 qid)
memset((void *)nvmeq->cqes, 0, CQ_SIZE(nvmeq->q_depth)); memset((void *)nvmeq->cqes, 0, CQ_SIZE(nvmeq->q_depth));
nvme_dbbuf_init(dev, nvmeq, qid); nvme_dbbuf_init(dev, nvmeq, qid);
dev->online_queues++; dev->online_queues++;
spin_unlock_irq(&nvmeq->q_lock); spin_unlock_irq(&nvmeq->cq_lock);
} }
static int nvme_create_queue(struct nvme_queue *nvmeq, int qid) static int nvme_create_queue(struct nvme_queue *nvmeq, int qid)
@ -2001,14 +2003,14 @@ static void nvme_del_cq_end(struct request *req, blk_status_t error)
unsigned long flags; unsigned long flags;
/* /*
* We might be called with the AQ q_lock held * We might be called with the AQ cq_lock held
* and the I/O queue q_lock should always * and the I/O queue cq_lock should always
* nest inside the AQ one. * nest inside the AQ one.
*/ */
spin_lock_irqsave_nested(&nvmeq->q_lock, flags, spin_lock_irqsave_nested(&nvmeq->cq_lock, flags,
SINGLE_DEPTH_NESTING); SINGLE_DEPTH_NESTING);
nvme_process_cq(nvmeq, &start, &end, -1); nvme_process_cq(nvmeq, &start, &end, -1);
spin_unlock_irqrestore(&nvmeq->q_lock, flags); spin_unlock_irqrestore(&nvmeq->cq_lock, flags);
nvme_complete_cqes(nvmeq, start, end); nvme_complete_cqes(nvmeq, start, end);
} }