From ab454438aa8dc9eb113df7d00f2cf9ec628a26ce Mon Sep 17 00:00:00 2001 From: Pavel Begunkov Date: Mon, 22 Mar 2021 01:58:33 +0000 Subject: [PATCH] io_uring: hide iter revert in resubmit_prep Move iov_iter_revert() resetting iterator in case of -EIOCBQUEUED into io_resubmit_prep(), so we don't do heavy revert in hot path, also saves a couple of checks. Signed-off-by: Pavel Begunkov Signed-off-by: Jens Axboe --- fs/io_uring.c | 17 ++++++++--------- 1 file changed, 8 insertions(+), 9 deletions(-) diff --git a/fs/io_uring.c b/fs/io_uring.c index 9362c9228540..a7239c86326e 100644 --- a/fs/io_uring.c +++ b/fs/io_uring.c @@ -2445,8 +2445,13 @@ static void kiocb_end_write(struct io_kiocb *req) #ifdef CONFIG_BLOCK static bool io_resubmit_prep(struct io_kiocb *req) { - /* either already prepared or successfully done */ - return req->async_data || !io_req_prep_async(req); + struct io_async_rw *rw = req->async_data; + + if (!rw) + return !io_req_prep_async(req); + /* may have left rw->iter inconsistent on -EIOCBQUEUED */ + iov_iter_revert(&rw->iter, req->result - iov_iter_count(&rw->iter)); + return true; } static bool io_rw_should_reissue(struct io_kiocb *req) @@ -2505,14 +2510,8 @@ static void io_complete_rw_iopoll(struct kiocb *kiocb, long res, long res2) struct io_kiocb *req = container_of(kiocb, struct io_kiocb, rw.kiocb); #ifdef CONFIG_BLOCK - /* Rewind iter, if we have one. iopoll path resubmits as usual */ if (res == -EAGAIN && io_rw_should_reissue(req)) { - struct io_async_rw *rw = req->async_data; - - if (rw) - iov_iter_revert(&rw->iter, - req->result - iov_iter_count(&rw->iter)); - else if (!io_resubmit_prep(req)) + if (!io_resubmit_prep(req)) req->flags |= REQ_F_DONT_REISSUE; } #endif