SUNRPC: Optimize xdr_reserve_space()

Transitioning between encode buffers is quite infrequent. It happens
about 1 time in 400 calls to xdr_reserve_space(), measured on NFSD
with a typical build/test workload.

Force the compiler to remove that code from xdr_reserve_space(),
which is a hot path on both the server and the client. This change
reduces the size of xdr_reserve_space() from 10 cache lines to 2
when compiled with -Os.

Signed-off-by: Chuck Lever <chuck.lever@oracle.com>
Reviewed-by: J. Bruce Fields <bfields@fieldses.org>
This commit is contained in:
Chuck Lever 2022-06-07 16:47:58 -04:00
parent 6c254bf3b6
commit 62ed448cc5
2 changed files with 25 additions and 8 deletions

View File

@ -243,7 +243,7 @@ extern void xdr_init_encode(struct xdr_stream *xdr, struct xdr_buf *buf,
extern __be32 *xdr_reserve_space(struct xdr_stream *xdr, size_t nbytes);
extern int xdr_reserve_space_vec(struct xdr_stream *xdr, struct kvec *vec,
size_t nbytes);
extern void xdr_commit_encode(struct xdr_stream *xdr);
extern void __xdr_commit_encode(struct xdr_stream *xdr);
extern void xdr_truncate_encode(struct xdr_stream *xdr, size_t len);
extern int xdr_restrict_buflen(struct xdr_stream *xdr, int newbuflen);
extern void xdr_write_pages(struct xdr_stream *xdr, struct page **pages,
@ -306,6 +306,20 @@ xdr_reset_scratch_buffer(struct xdr_stream *xdr)
xdr_set_scratch_buffer(xdr, NULL, 0);
}
/**
* xdr_commit_encode - Ensure all data is written to xdr->buf
* @xdr: pointer to xdr_stream
*
* Handle encoding across page boundaries by giving the caller a
* temporary location to write to, then later copying the data into
* place. __xdr_commit_encode() does that copying.
*/
static inline void xdr_commit_encode(struct xdr_stream *xdr)
{
if (unlikely(xdr->scratch.iov_len))
__xdr_commit_encode(xdr);
}
/**
* xdr_stream_remaining - Return the number of bytes remaining in the stream
* @xdr: pointer to struct xdr_stream

View File

@ -919,7 +919,7 @@ void xdr_init_encode(struct xdr_stream *xdr, struct xdr_buf *buf, __be32 *p,
EXPORT_SYMBOL_GPL(xdr_init_encode);
/**
* xdr_commit_encode - Ensure all data is written to buffer
* __xdr_commit_encode - Ensure all data is written to buffer
* @xdr: pointer to xdr_stream
*
* We handle encoding across page boundaries by giving the caller a
@ -931,22 +931,25 @@ EXPORT_SYMBOL_GPL(xdr_init_encode);
* required at the end of encoding, or any other time when the xdr_buf
* data might be read.
*/
inline void xdr_commit_encode(struct xdr_stream *xdr)
void __xdr_commit_encode(struct xdr_stream *xdr)
{
int shift = xdr->scratch.iov_len;
void *page;
if (shift == 0)
return;
page = page_address(*xdr->page_ptr);
memcpy(xdr->scratch.iov_base, page, shift);
memmove(page, page + shift, (void *)xdr->p - page);
xdr_reset_scratch_buffer(xdr);
}
EXPORT_SYMBOL_GPL(xdr_commit_encode);
EXPORT_SYMBOL_GPL(__xdr_commit_encode);
static __be32 *xdr_get_next_encode_buffer(struct xdr_stream *xdr,
size_t nbytes)
/*
* The buffer space to be reserved crosses the boundary between
* xdr->buf->head and xdr->buf->pages, or between two pages
* in xdr->buf->pages.
*/
static noinline __be32 *xdr_get_next_encode_buffer(struct xdr_stream *xdr,
size_t nbytes)
{
__be32 *p;
int space_left;