renegade-project/linux-next
2
0
Fork 0
mirror of https://github.com/edk2-porting/linux-next.git synced 2025-01-01 18:24:23 +08:00
Code

io_uring: don't propagate io_comp_state

Browse Source 
There is no reason to drag io_comp_state into opcode handlers, we just
need a flag and the actual work will be done in __io_queue_sqe().

Signed-off-by: Pavel Begunkov <asml.silence@gmail.com>
Signed-off-by: Jens Axboe <axboe@kernel.dk>
This commit is contained in:
Pavel Begunkov 2021-02-10 00:03:09 +00:00 committed by Jens Axboe
parent 61e9820304
commit 889fca7328
1 changed files with 73 additions and 91 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

164
fs/io_uring.c
View File

@ -189,6 +189,7 @@ struct io_rings {
enum io_uring_cmd_flags { enum io_uring_cmd_flags {
IO_URING_F_NONBLOCK = 1, IO_URING_F_NONBLOCK = 1,
IO_URING_F_COMPLETE_DEFER = 2,
}; };
struct io_mapped_ubuf { struct io_mapped_ubuf {
@ -1018,7 +1019,7 @@ static void init_fixed_file_ref_node(struct io_ring_ctx *ctx,
struct fixed_rsrc_ref_node *ref_node); struct fixed_rsrc_ref_node *ref_node);
static void __io_complete_rw(struct io_kiocb *req, long res, long res2, static void __io_complete_rw(struct io_kiocb *req, long res, long res2,
struct io_comp_state *cs); unsigned int issue_flags);
static void io_cqring_fill_event(struct io_kiocb *req, long res); static void io_cqring_fill_event(struct io_kiocb *req, long res);
static void io_put_req(struct io_kiocb *req); static void io_put_req(struct io_kiocb *req);
static void io_put_req_deferred(struct io_kiocb *req, int nr); static void io_put_req_deferred(struct io_kiocb *req, int nr);
@ -1957,7 +1958,7 @@ static void io_submit_flush_completions(struct io_comp_state *cs)
} }
static void io_req_complete_state(struct io_kiocb *req, long res, static void io_req_complete_state(struct io_kiocb *req, long res,
unsigned int cflags, struct io_comp_state *cs) unsigned int cflags)
{ {
io_clean_op(req); io_clean_op(req);
req->result = res; req->result = res;
@ -1965,18 +1966,18 @@ static void io_req_complete_state(struct io_kiocb *req, long res,
req->flags |= REQ_F_COMPLETE_INLINE; req->flags |= REQ_F_COMPLETE_INLINE;
} }
static inline void __io_req_complete(struct io_kiocb *req, long res, static inline void __io_req_complete(struct io_kiocb *req, unsigned issue_flags,
unsigned cflags, struct io_comp_state *cs) long res, unsigned cflags)
{ {
if (!cs) if (issue_flags & IO_URING_F_COMPLETE_DEFER)
io_req_complete_nostate(req, res, cflags); io_req_complete_state(req, res, cflags);
else else
io_req_complete_state(req, res, cflags, cs); io_req_complete_nostate(req, res, cflags);
} }
static inline void io_req_complete(struct io_kiocb *req, long res) static inline void io_req_complete(struct io_kiocb *req, long res)
{ {
__io_req_complete(req, res, 0, NULL); __io_req_complete(req, 0, res, 0);
} }
static inline bool io_is_fallback_req(struct io_kiocb *req) static inline bool io_is_fallback_req(struct io_kiocb *req)
@ -2449,7 +2450,7 @@ static void io_iopoll_queue(struct list_head *again)
do { do {
req = list_first_entry(again, struct io_kiocb, inflight_entry); req = list_first_entry(again, struct io_kiocb, inflight_entry);
list_del(&req->inflight_entry); list_del(&req->inflight_entry);
__io_complete_rw(req, -EAGAIN, 0, NULL); __io_complete_rw(req, -EAGAIN, 0, 0);
} while (!list_empty(again)); } while (!list_empty(again));
} }
@ -2662,7 +2663,7 @@ static void kiocb_end_write(struct io_kiocb *req)
} }
static void io_complete_rw_common(struct kiocb *kiocb, long res, static void io_complete_rw_common(struct kiocb *kiocb, long res,
struct io_comp_state *cs) unsigned int issue_flags)
{ {
struct io_kiocb *req = container_of(kiocb, struct io_kiocb, rw.kiocb); struct io_kiocb *req = container_of(kiocb, struct io_kiocb, rw.kiocb);
int cflags = 0; int cflags = 0;
@ -2674,7 +2675,7 @@ static void io_complete_rw_common(struct kiocb *kiocb, long res,
req_set_fail_links(req); req_set_fail_links(req);
if (req->flags & REQ_F_BUFFER_SELECTED) if (req->flags & REQ_F_BUFFER_SELECTED)
cflags = io_put_rw_kbuf(req); cflags = io_put_rw_kbuf(req);
__io_req_complete(req, res, cflags, cs); __io_req_complete(req, issue_flags, res, cflags);
} }
#ifdef CONFIG_BLOCK #ifdef CONFIG_BLOCK
@ -2741,17 +2742,17 @@ static bool io_rw_reissue(struct io_kiocb *req, long res)
} }
static void __io_complete_rw(struct io_kiocb *req, long res, long res2, static void __io_complete_rw(struct io_kiocb *req, long res, long res2,
struct io_comp_state *cs) unsigned int issue_flags)
{ {
if (!io_rw_reissue(req, res)) if (!io_rw_reissue(req, res))
io_complete_rw_common(&req->rw.kiocb, res, cs); io_complete_rw_common(&req->rw.kiocb, res, issue_flags);
} }
static void io_complete_rw(struct kiocb *kiocb, long res, long res2) static void io_complete_rw(struct kiocb *kiocb, long res, long res2)
{ {
struct io_kiocb *req = container_of(kiocb, struct io_kiocb, rw.kiocb); struct io_kiocb *req = container_of(kiocb, struct io_kiocb, rw.kiocb);
__io_complete_rw(req, res, res2, NULL); __io_complete_rw(req, res, res2, 0);
} }
static void io_complete_rw_iopoll(struct kiocb *kiocb, long res, long res2) static void io_complete_rw_iopoll(struct kiocb *kiocb, long res, long res2)
@ -2970,7 +2971,7 @@ static inline void io_rw_done(struct kiocb *kiocb, ssize_t ret)
} }
static void kiocb_done(struct kiocb *kiocb, ssize_t ret, static void kiocb_done(struct kiocb *kiocb, ssize_t ret,
struct io_comp_state *cs) unsigned int issue_flags)
{ {
struct io_kiocb *req = container_of(kiocb, struct io_kiocb, rw.kiocb); struct io_kiocb *req = container_of(kiocb, struct io_kiocb, rw.kiocb);
struct io_async_rw *io = req->async_data; struct io_async_rw *io = req->async_data;
@ -2986,7 +2987,7 @@ static void kiocb_done(struct kiocb *kiocb, ssize_t ret,
if (req->flags & REQ_F_CUR_POS) if (req->flags & REQ_F_CUR_POS)
req->file->f_pos = kiocb->ki_pos; req->file->f_pos = kiocb->ki_pos;
if (ret >= 0 && kiocb->ki_complete == io_complete_rw) if (ret >= 0 && kiocb->ki_complete == io_complete_rw)
__io_complete_rw(req, ret, 0, cs); __io_complete_rw(req, ret, 0, issue_flags);
else else
io_rw_done(kiocb, ret); io_rw_done(kiocb, ret);
} }
@ -3481,8 +3482,7 @@ static int io_iter_do_read(struct io_kiocb *req, struct iov_iter *iter)
return -EINVAL; return -EINVAL;
} }
static int io_read(struct io_kiocb *req, unsigned int issue_flags, static int io_read(struct io_kiocb *req, unsigned int issue_flags)
struct io_comp_state *cs)
{ {
struct iovec inline_vecs[UIO_FASTIOV], *iovec = inline_vecs; struct iovec inline_vecs[UIO_FASTIOV], *iovec = inline_vecs;
struct kiocb *kiocb = &req->rw.kiocb; struct kiocb *kiocb = &req->rw.kiocb;
@ -3572,7 +3572,7 @@ static int io_read(struct io_kiocb *req, unsigned int issue_flags,
/* we got some bytes, but not all. retry. */ /* we got some bytes, but not all. retry. */
} while (ret > 0 && ret < io_size); } while (ret > 0 && ret < io_size);
done: done:
kiocb_done(kiocb, ret, cs); kiocb_done(kiocb, ret, issue_flags);
return 0; return 0;
} }
@ -3593,8 +3593,7 @@ static int io_write_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe)
return io_rw_prep_async(req, WRITE); return io_rw_prep_async(req, WRITE);
} }
static int io_write(struct io_kiocb *req, unsigned int issue_flags, static int io_write(struct io_kiocb *req, unsigned int issue_flags)
struct io_comp_state *cs)
{ {
struct iovec inline_vecs[UIO_FASTIOV], *iovec = inline_vecs; struct iovec inline_vecs[UIO_FASTIOV], *iovec = inline_vecs;
struct kiocb *kiocb = &req->rw.kiocb; struct kiocb *kiocb = &req->rw.kiocb;
@ -3668,7 +3667,7 @@ static int io_write(struct io_kiocb *req, unsigned int issue_flags,
if ((req->ctx->flags & IORING_SETUP_IOPOLL) && ret2 == -EAGAIN) if ((req->ctx->flags & IORING_SETUP_IOPOLL) && ret2 == -EAGAIN)
goto copy_iov; goto copy_iov;
done: done:
kiocb_done(kiocb, ret2, cs); kiocb_done(kiocb, ret2, issue_flags);
} else { } else {
copy_iov: copy_iov:
/* some cases will consume bytes even on error returns */ /* some cases will consume bytes even on error returns */
@ -3917,15 +3916,14 @@ static int io_splice(struct io_kiocb *req, unsigned int issue_flags)
/* /*
* IORING_OP_NOP just posts a completion event, nothing else. * IORING_OP_NOP just posts a completion event, nothing else.
*/ */
static int io_nop(struct io_kiocb *req, unsigned int issue_flags, static int io_nop(struct io_kiocb *req, unsigned int issue_flags)
struct io_comp_state *cs)
{ {
struct io_ring_ctx *ctx = req->ctx; struct io_ring_ctx *ctx = req->ctx;
if (unlikely(ctx->flags & IORING_SETUP_IOPOLL)) if (unlikely(ctx->flags & IORING_SETUP_IOPOLL))
return -EINVAL; return -EINVAL;
__io_req_complete(req, 0, 0, cs); __io_req_complete(req, issue_flags, 0, 0);
return 0; return 0;
} }
@ -4166,8 +4164,7 @@ static int __io_remove_buffers(struct io_ring_ctx *ctx, struct io_buffer *buf,
return i; return i;
} }
static int io_remove_buffers(struct io_kiocb *req, unsigned int issue_flags, static int io_remove_buffers(struct io_kiocb *req, unsigned int issue_flags)
struct io_comp_state *cs)
{ {
struct io_provide_buf *p = &req->pbuf; struct io_provide_buf *p = &req->pbuf;
struct io_ring_ctx *ctx = req->ctx; struct io_ring_ctx *ctx = req->ctx;
@ -4188,11 +4185,11 @@ static int io_remove_buffers(struct io_kiocb *req, unsigned int issue_flags,
/* need to hold the lock to complete IOPOLL requests */ /* need to hold the lock to complete IOPOLL requests */
if (ctx->flags & IORING_SETUP_IOPOLL) { if (ctx->flags & IORING_SETUP_IOPOLL) {
__io_req_complete(req, ret, 0, cs); __io_req_complete(req, issue_flags, ret, 0);
io_ring_submit_unlock(ctx, !force_nonblock); io_ring_submit_unlock(ctx, !force_nonblock);
} else { } else {
io_ring_submit_unlock(ctx, !force_nonblock); io_ring_submit_unlock(ctx, !force_nonblock);
__io_req_complete(req, ret, 0, cs); __io_req_complete(req, issue_flags, ret, 0);
} }
return 0; return 0;
} }
@ -4251,8 +4248,7 @@ static int io_add_buffers(struct io_provide_buf *pbuf, struct io_buffer **head)
return i ? i : -ENOMEM; return i ? i : -ENOMEM;
} }
static int io_provide_buffers(struct io_kiocb *req, unsigned int issue_flags, static int io_provide_buffers(struct io_kiocb *req, unsigned int issue_flags)
struct io_comp_state *cs)
{ {
struct io_provide_buf *p = &req->pbuf; struct io_provide_buf *p = &req->pbuf;
struct io_ring_ctx *ctx = req->ctx; struct io_ring_ctx *ctx = req->ctx;
@ -4284,11 +4280,11 @@ out:
/* need to hold the lock to complete IOPOLL requests */ /* need to hold the lock to complete IOPOLL requests */
if (ctx->flags & IORING_SETUP_IOPOLL) { if (ctx->flags & IORING_SETUP_IOPOLL) {
__io_req_complete(req, ret, 0, cs); __io_req_complete(req, issue_flags, ret, 0);
io_ring_submit_unlock(ctx, !force_nonblock); io_ring_submit_unlock(ctx, !force_nonblock);
} else { } else {
io_ring_submit_unlock(ctx, !force_nonblock); io_ring_submit_unlock(ctx, !force_nonblock);
__io_req_complete(req, ret, 0, cs); __io_req_complete(req, issue_flags, ret, 0);
} }
return 0; return 0;
} }
@ -4320,8 +4316,7 @@ static int io_epoll_ctl_prep(struct io_kiocb *req,
#endif #endif
} }
static int io_epoll_ctl(struct io_kiocb *req, unsigned int issue_flags, static int io_epoll_ctl(struct io_kiocb *req, unsigned int issue_flags)
struct io_comp_state *cs)
{ {
#if defined(CONFIG_EPOLL) #if defined(CONFIG_EPOLL)
struct io_epoll *ie = &req->epoll; struct io_epoll *ie = &req->epoll;
@ -4334,7 +4329,7 @@ static int io_epoll_ctl(struct io_kiocb *req, unsigned int issue_flags,
if (ret < 0) if (ret < 0)
req_set_fail_links(req); req_set_fail_links(req);
__io_req_complete(req, ret, 0, cs); __io_req_complete(req, issue_flags, ret, 0);
return 0; return 0;
#else #else
return -EOPNOTSUPP; return -EOPNOTSUPP;
@ -4466,8 +4461,7 @@ static int io_close_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe)
return 0; return 0;
} }
static int io_close(struct io_kiocb *req, unsigned int issue_flags, static int io_close(struct io_kiocb *req, unsigned int issue_flags)
struct io_comp_state *cs)
{ {
struct files_struct *files = current->files; struct files_struct *files = current->files;
struct io_close *close = &req->close; struct io_close *close = &req->close;
@ -4516,7 +4510,7 @@ err:
req_set_fail_links(req); req_set_fail_links(req);
if (file) if (file)
fput(file); fput(file);
__io_req_complete(req, ret, 0, cs); __io_req_complete(req, issue_flags, ret, 0);
return 0; return 0;
} }
@ -4612,8 +4606,7 @@ static int io_sendmsg_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe)
return ret; return ret;
} }
static int io_sendmsg(struct io_kiocb *req, unsigned int issue_flags, static int io_sendmsg(struct io_kiocb *req, unsigned int issue_flags)
struct io_comp_state *cs)
{ {
struct io_async_msghdr iomsg, *kmsg; struct io_async_msghdr iomsg, *kmsg;
struct socket *sock; struct socket *sock;
@ -4650,12 +4643,11 @@ static int io_sendmsg(struct io_kiocb *req, unsigned int issue_flags,
req->flags &= ~REQ_F_NEED_CLEANUP; req->flags &= ~REQ_F_NEED_CLEANUP;
if (ret < 0) if (ret < 0)
req_set_fail_links(req); req_set_fail_links(req);
__io_req_complete(req, ret, 0, cs); __io_req_complete(req, issue_flags, ret, 0);
return 0; return 0;
} }
static int io_send(struct io_kiocb *req, unsigned int issue_flags, static int io_send(struct io_kiocb *req, unsigned int issue_flags)
struct io_comp_state *cs)
{ {
struct io_sr_msg *sr = &req->sr_msg; struct io_sr_msg *sr = &req->sr_msg;
struct msghdr msg; struct msghdr msg;
@ -4692,7 +4684,7 @@ static int io_send(struct io_kiocb *req, unsigned int issue_flags,
if (ret < 0) if (ret < 0)
req_set_fail_links(req); req_set_fail_links(req);
__io_req_complete(req, ret, 0, cs); __io_req_complete(req, issue_flags, ret, 0);
return 0; return 0;
} }
@ -4833,8 +4825,7 @@ static int io_recvmsg_prep(struct io_kiocb *req,
return ret; return ret;
} }
static int io_recvmsg(struct io_kiocb *req, unsigned int issue_flags, static int io_recvmsg(struct io_kiocb *req, unsigned int issue_flags)
struct io_comp_state *cs)
{ {
struct io_async_msghdr iomsg, *kmsg; struct io_async_msghdr iomsg, *kmsg;
struct socket *sock; struct socket *sock;
@ -4886,12 +4877,11 @@ static int io_recvmsg(struct io_kiocb *req, unsigned int issue_flags,
req->flags &= ~REQ_F_NEED_CLEANUP; req->flags &= ~REQ_F_NEED_CLEANUP;
if (ret < 0) if (ret < 0)
req_set_fail_links(req); req_set_fail_links(req);
__io_req_complete(req, ret, cflags, cs); __io_req_complete(req, issue_flags, ret, cflags);
return 0; return 0;
} }
static int io_recv(struct io_kiocb *req, unsigned int issue_flags, static int io_recv(struct io_kiocb *req, unsigned int issue_flags)
struct io_comp_state *cs)
{ {
struct io_buffer *kbuf; struct io_buffer *kbuf;
struct io_sr_msg *sr = &req->sr_msg; struct io_sr_msg *sr = &req->sr_msg;
@ -4941,7 +4931,7 @@ out_free:
cflags = io_put_recv_kbuf(req); cflags = io_put_recv_kbuf(req);
if (ret < 0) if (ret < 0)
req_set_fail_links(req); req_set_fail_links(req);
__io_req_complete(req, ret, cflags, cs); __io_req_complete(req, issue_flags, ret, cflags);
return 0; return 0;
} }
@ -4961,8 +4951,7 @@ static int io_accept_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe)
return 0; return 0;
} }
static int io_accept(struct io_kiocb *req, unsigned int issue_flags, static int io_accept(struct io_kiocb *req, unsigned int issue_flags)
struct io_comp_state *cs)
{ {
struct io_accept *accept = &req->accept; struct io_accept *accept = &req->accept;
bool force_nonblock = issue_flags & IO_URING_F_NONBLOCK; bool force_nonblock = issue_flags & IO_URING_F_NONBLOCK;
@ -4982,7 +4971,7 @@ static int io_accept(struct io_kiocb *req, unsigned int issue_flags,
ret = -EINTR; ret = -EINTR;
req_set_fail_links(req); req_set_fail_links(req);
} }
__io_req_complete(req, ret, 0, cs); __io_req_complete(req, issue_flags, ret, 0);
return 0; return 0;
} }
@ -5006,8 +4995,7 @@ static int io_connect_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe)
&io->address); &io->address);
} }
static int io_connect(struct io_kiocb *req, unsigned int issue_flags, static int io_connect(struct io_kiocb *req, unsigned int issue_flags)
struct io_comp_state *cs)
{ {
struct io_async_connect __io, *io; struct io_async_connect __io, *io;
unsigned file_flags; unsigned file_flags;
@ -5045,7 +5033,7 @@ static int io_connect(struct io_kiocb *req, unsigned int issue_flags,
out: out:
if (ret < 0) if (ret < 0)
req_set_fail_links(req); req_set_fail_links(req);
__io_req_complete(req, ret, 0, cs); __io_req_complete(req, issue_flags, ret, 0);
return 0; return 0;
} }
#else /* !CONFIG_NET */ #else /* !CONFIG_NET */
@ -5054,14 +5042,12 @@ static int io_sendmsg_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe)
return -EOPNOTSUPP; return -EOPNOTSUPP;
} }
static int io_sendmsg(struct io_kiocb *req, unsigned int issue_flags, static int io_sendmsg(struct io_kiocb *req, unsigned int issue_flags)
struct io_comp_state *cs)
{ {
return -EOPNOTSUPP; return -EOPNOTSUPP;
} }
static int io_send(struct io_kiocb *req, unsigned int issue_flags, static int io_send(struct io_kiocb *req, unsigned int issue_flags)
struct io_comp_state *cs)
{ {
return -EOPNOTSUPP; return -EOPNOTSUPP;
} }
@ -5072,14 +5058,12 @@ static int io_recvmsg_prep(struct io_kiocb *req,
return -EOPNOTSUPP; return -EOPNOTSUPP;
} }
static int io_recvmsg(struct io_kiocb *req, unsigned int issue_flags, static int io_recvmsg(struct io_kiocb *req, unsigned int issue_flags)
struct io_comp_state *cs)
{ {
return -EOPNOTSUPP; return -EOPNOTSUPP;
} }
static int io_recv(struct io_kiocb *req, unsigned int issue_flags, static int io_recv(struct io_kiocb *req, unsigned int issue_flags)
struct io_comp_state *cs)
{ {
return -EOPNOTSUPP; return -EOPNOTSUPP;
} }
@ -5089,8 +5073,7 @@ static int io_accept_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe)
return -EOPNOTSUPP; return -EOPNOTSUPP;
} }
static int io_accept(struct io_kiocb *req, unsigned int issue_flags, static int io_accept(struct io_kiocb *req, unsigned int issue_flags)
struct io_comp_state *cs)
{ {
return -EOPNOTSUPP; return -EOPNOTSUPP;
} }
@ -5100,8 +5083,7 @@ static int io_connect_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe)
return -EOPNOTSUPP; return -EOPNOTSUPP;
} }
static int io_connect(struct io_kiocb *req, unsigned int issue_flags, static int io_connect(struct io_kiocb *req, unsigned int issue_flags)
struct io_comp_state *cs)
{ {
return -EOPNOTSUPP; return -EOPNOTSUPP;
} }
@ -5978,8 +5960,7 @@ static int io_rsrc_update_prep(struct io_kiocb *req,
return 0; return 0;
} }
static int io_files_update(struct io_kiocb *req, unsigned int issue_flags, static int io_files_update(struct io_kiocb *req, unsigned int issue_flags)
struct io_comp_state *cs)
{ {
struct io_ring_ctx *ctx = req->ctx; struct io_ring_ctx *ctx = req->ctx;
struct io_uring_rsrc_update up; struct io_uring_rsrc_update up;
@ -5997,7 +5978,7 @@ static int io_files_update(struct io_kiocb *req, unsigned int issue_flags,
if (ret < 0) if (ret < 0)
req_set_fail_links(req); req_set_fail_links(req);
__io_req_complete(req, ret, 0, cs); __io_req_complete(req, issue_flags, ret, 0);
return 0; return 0;
} }
@ -6204,25 +6185,24 @@ static void __io_clean_op(struct io_kiocb *req)
} }
} }
static int io_issue_sqe(struct io_kiocb *req, unsigned int issue_flags, static int io_issue_sqe(struct io_kiocb *req, unsigned int issue_flags)
struct io_comp_state *cs)
{ {
struct io_ring_ctx *ctx = req->ctx; struct io_ring_ctx *ctx = req->ctx;
int ret; int ret;
switch (req->opcode) { switch (req->opcode) {
case IORING_OP_NOP: case IORING_OP_NOP:
ret = io_nop(req, issue_flags, cs); ret = io_nop(req, issue_flags);
break; break;
case IORING_OP_READV: case IORING_OP_READV:
case IORING_OP_READ_FIXED: case IORING_OP_READ_FIXED:
case IORING_OP_READ: case IORING_OP_READ:
ret = io_read(req, issue_flags, cs); ret = io_read(req, issue_flags);
break; break;
case IORING_OP_WRITEV: case IORING_OP_WRITEV:
case IORING_OP_WRITE_FIXED: case IORING_OP_WRITE_FIXED:
case IORING_OP_WRITE: case IORING_OP_WRITE:
ret = io_write(req, issue_flags, cs); ret = io_write(req, issue_flags);
break; break;
case IORING_OP_FSYNC: case IORING_OP_FSYNC:
ret = io_fsync(req, issue_flags); ret = io_fsync(req, issue_flags);
@ -6237,16 +6217,16 @@ static int io_issue_sqe(struct io_kiocb *req, unsigned int issue_flags,
ret = io_sync_file_range(req, issue_flags); ret = io_sync_file_range(req, issue_flags);
break; break;
case IORING_OP_SENDMSG: case IORING_OP_SENDMSG:
ret = io_sendmsg(req, issue_flags, cs); ret = io_sendmsg(req, issue_flags);
break; break;
case IORING_OP_SEND: case IORING_OP_SEND:
ret = io_send(req, issue_flags, cs); ret = io_send(req, issue_flags);
break; break;
case IORING_OP_RECVMSG: case IORING_OP_RECVMSG:
ret = io_recvmsg(req, issue_flags, cs); ret = io_recvmsg(req, issue_flags);
break; break;
case IORING_OP_RECV: case IORING_OP_RECV:
ret = io_recv(req, issue_flags, cs); ret = io_recv(req, issue_flags);
break; break;
case IORING_OP_TIMEOUT: case IORING_OP_TIMEOUT:
ret = io_timeout(req, issue_flags); ret = io_timeout(req, issue_flags);
@ -6255,10 +6235,10 @@ static int io_issue_sqe(struct io_kiocb *req, unsigned int issue_flags,
ret = io_timeout_remove(req, issue_flags); ret = io_timeout_remove(req, issue_flags);
break; break;
case IORING_OP_ACCEPT: case IORING_OP_ACCEPT:
ret = io_accept(req, issue_flags, cs); ret = io_accept(req, issue_flags);
break; break;
case IORING_OP_CONNECT: case IORING_OP_CONNECT:
ret = io_connect(req, issue_flags, cs); ret = io_connect(req, issue_flags);
break; break;
case IORING_OP_ASYNC_CANCEL: case IORING_OP_ASYNC_CANCEL:
ret = io_async_cancel(req, issue_flags); ret = io_async_cancel(req, issue_flags);
@ -6270,10 +6250,10 @@ static int io_issue_sqe(struct io_kiocb *req, unsigned int issue_flags,
ret = io_openat(req, issue_flags); ret = io_openat(req, issue_flags);
break; break;
case IORING_OP_CLOSE: case IORING_OP_CLOSE:
ret = io_close(req, issue_flags, cs); ret = io_close(req, issue_flags);
break; break;
case IORING_OP_FILES_UPDATE: case IORING_OP_FILES_UPDATE:
ret = io_files_update(req, issue_flags, cs); ret = io_files_update(req, issue_flags);
break; break;
case IORING_OP_STATX: case IORING_OP_STATX:
ret = io_statx(req, issue_flags); ret = io_statx(req, issue_flags);
@ -6288,16 +6268,16 @@ static int io_issue_sqe(struct io_kiocb *req, unsigned int issue_flags,
ret = io_openat2(req, issue_flags); ret = io_openat2(req, issue_flags);
break; break;
case IORING_OP_EPOLL_CTL: case IORING_OP_EPOLL_CTL:
ret = io_epoll_ctl(req, issue_flags, cs); ret = io_epoll_ctl(req, issue_flags);
break; break;
case IORING_OP_SPLICE: case IORING_OP_SPLICE:
ret = io_splice(req, issue_flags); ret = io_splice(req, issue_flags);
break; break;
case IORING_OP_PROVIDE_BUFFERS: case IORING_OP_PROVIDE_BUFFERS:
ret = io_provide_buffers(req, issue_flags, cs); ret = io_provide_buffers(req, issue_flags);
break; break;
case IORING_OP_REMOVE_BUFFERS: case IORING_OP_REMOVE_BUFFERS:
ret = io_remove_buffers(req, issue_flags, cs); ret = io_remove_buffers(req, issue_flags);
break; break;
case IORING_OP_TEE: case IORING_OP_TEE:
ret = io_tee(req, issue_flags); ret = io_tee(req, issue_flags);
@ -6351,7 +6331,7 @@ static void io_wq_submit_work(struct io_wq_work *work)
if (!ret) { if (!ret) {
do { do {
ret = io_issue_sqe(req, 0, NULL); ret = io_issue_sqe(req, 0);
/* /*
* We can get EAGAIN for polled IO even though we're * We can get EAGAIN for polled IO even though we're
* forcing a sync submission from here, since we can't * forcing a sync submission from here, since we can't
@ -6498,8 +6478,10 @@ static void __io_queue_sqe(struct io_kiocb *req, struct io_comp_state *cs)
{ {
struct io_kiocb *linked_timeout; struct io_kiocb *linked_timeout;
const struct cred *old_creds = NULL; const struct cred *old_creds = NULL;
int ret; int ret, issue_flags = IO_URING_F_NONBLOCK;
if (cs)
issue_flags |= IO_URING_F_COMPLETE_DEFER;
again: again:
linked_timeout = io_prep_linked_timeout(req); linked_timeout = io_prep_linked_timeout(req);
@ -6514,7 +6496,7 @@ again:
old_creds = override_creds(req->work.identity->creds); old_creds = override_creds(req->work.identity->creds);
} }
ret = io_issue_sqe(req, IO_URING_F_NONBLOCK, cs); ret = io_issue_sqe(req, issue_flags);
/* /*
* We async punt it if the file wasn't marked NOWAIT, or if the file * We async punt it if the file wasn't marked NOWAIT, or if the file