mirror of
https://mirrors.bfsu.edu.cn/git/linux.git
synced 2024-11-27 06:04:23 +08:00
io_uring/rsrc: keep one global dummy_ubuf
We set empty registered buffers to dummy_ubuf as an optimisation. Currently, we allocate the dummy entry for each ring, whenever we can simply have one global instance. We're casting out const on assignment, it's fine as we're not going to change the content of the dummy, the constness gives us an extra layer of protection if sth ever goes wrong. Signed-off-by: Pavel Begunkov <asml.silence@gmail.com> Link: https://lore.kernel.org/r/e4a96dda35ab755914bc43f6781bba0df97ac489.1691757663.git.asml.silence@gmail.com Signed-off-by: Jens Axboe <axboe@kernel.dk>
This commit is contained in:
parent
b6b2bb58a7
commit
19a63c4021
@ -290,13 +290,6 @@ static __cold struct io_ring_ctx *io_ring_ctx_alloc(struct io_uring_params *p)
|
||||
goto err;
|
||||
if (io_alloc_hash_table(&ctx->cancel_table_locked, hash_bits))
|
||||
goto err;
|
||||
|
||||
ctx->dummy_ubuf = kzalloc(sizeof(*ctx->dummy_ubuf), GFP_KERNEL);
|
||||
if (!ctx->dummy_ubuf)
|
||||
goto err;
|
||||
/* set invalid range, so io_import_fixed() fails meeting it */
|
||||
ctx->dummy_ubuf->ubuf = -1UL;
|
||||
|
||||
if (percpu_ref_init(&ctx->refs, io_ring_ctx_ref_free,
|
||||
0, GFP_KERNEL))
|
||||
goto err;
|
||||
@ -335,7 +328,6 @@ static __cold struct io_ring_ctx *io_ring_ctx_alloc(struct io_uring_params *p)
|
||||
INIT_WQ_LIST(&ctx->submit_state.compl_reqs);
|
||||
return ctx;
|
||||
err:
|
||||
kfree(ctx->dummy_ubuf);
|
||||
kfree(ctx->cancel_table.hbs);
|
||||
kfree(ctx->cancel_table_locked.hbs);
|
||||
kfree(ctx->io_bl);
|
||||
@ -2897,7 +2889,6 @@ static __cold void io_ring_ctx_free(struct io_ring_ctx *ctx)
|
||||
io_wq_put_hash(ctx->hash_map);
|
||||
kfree(ctx->cancel_table.hbs);
|
||||
kfree(ctx->cancel_table_locked.hbs);
|
||||
kfree(ctx->dummy_ubuf);
|
||||
kfree(ctx->io_bl);
|
||||
xa_destroy(&ctx->io_bl_xa);
|
||||
kfree(ctx);
|
||||
|
@ -33,6 +33,12 @@ static int io_sqe_buffer_register(struct io_ring_ctx *ctx, struct iovec *iov,
|
||||
#define IORING_MAX_FIXED_FILES (1U << 20)
|
||||
#define IORING_MAX_REG_BUFFERS (1U << 14)
|
||||
|
||||
static const struct io_mapped_ubuf dummy_ubuf = {
|
||||
/* set invalid range, so io_import_fixed() fails meeting it */
|
||||
.ubuf = -1UL,
|
||||
.ubuf_end = 0,
|
||||
};
|
||||
|
||||
int __io_account_mem(struct user_struct *user, unsigned long nr_pages)
|
||||
{
|
||||
unsigned long page_limit, cur_pages, new_pages;
|
||||
@ -132,7 +138,7 @@ static void io_buffer_unmap(struct io_ring_ctx *ctx, struct io_mapped_ubuf **slo
|
||||
struct io_mapped_ubuf *imu = *slot;
|
||||
unsigned int i;
|
||||
|
||||
if (imu != ctx->dummy_ubuf) {
|
||||
if (imu != &dummy_ubuf) {
|
||||
for (i = 0; i < imu->nr_bvecs; i++)
|
||||
unpin_user_page(imu->bvec[i].bv_page);
|
||||
if (imu->acct_pages)
|
||||
@ -459,14 +465,14 @@ static int __io_sqe_buffers_update(struct io_ring_ctx *ctx,
|
||||
break;
|
||||
|
||||
i = array_index_nospec(up->offset + done, ctx->nr_user_bufs);
|
||||
if (ctx->user_bufs[i] != ctx->dummy_ubuf) {
|
||||
if (ctx->user_bufs[i] != &dummy_ubuf) {
|
||||
err = io_queue_rsrc_removal(ctx->buf_data, i,
|
||||
ctx->user_bufs[i]);
|
||||
if (unlikely(err)) {
|
||||
io_buffer_unmap(ctx, &imu);
|
||||
break;
|
||||
}
|
||||
ctx->user_bufs[i] = ctx->dummy_ubuf;
|
||||
ctx->user_bufs[i] = (struct io_mapped_ubuf *)&dummy_ubuf;
|
||||
}
|
||||
|
||||
ctx->user_bufs[i] = imu;
|
||||
@ -1077,7 +1083,7 @@ static int io_sqe_buffer_register(struct io_ring_ctx *ctx, struct iovec *iov,
|
||||
int ret, nr_pages, i;
|
||||
struct folio *folio = NULL;
|
||||
|
||||
*pimu = ctx->dummy_ubuf;
|
||||
*pimu = (struct io_mapped_ubuf *)&dummy_ubuf;
|
||||
if (!iov->iov_base)
|
||||
return 0;
|
||||
|
||||
|
Loading…
Reference in New Issue
Block a user