korg/linux
0
0
Fork 0
mirror of https://mirrors.bfsu.edu.cn/git/linux.git synced 2024-11-28 06:34:12 +08:00
Code Issues Packages Projects Releases Wiki Activity

iov_iter: streamline iovec/bvec alignment iteration

Browse Source 
Rewrite the alignment checking iterators for iovec and bvec to be easier
to read, and also significantly more compact in terms of generated code.
This saves 270 bytes of text on x86-64 for me (with clang-18) and 224
bytes on arm64 (with gcc-13).

In profiles, also saves a bit of time as well for the same workload:

     0.81%     -0.18%  [kernel.vmlinux]  [k] iov_iter_aligned_bvec
     0.48%     -0.09%  [kernel.vmlinux]  [k] iov_iter_is_aligned

which is a nice side benefit as well.

Signed-off-by: Jens Axboe <axboe@kernel.dk>
Link: https://lore.kernel.org/r/544b31f7-6d4b-42f5-a544-1420501f081f@kernel.dk
Reviewed-by: Keith Busch <kbusch@kernel.org>
Signed-off-by: Christian Brauner <brauner@kernel.org>

v2: do the other half of the iterators too, as suggested by Keith.
    This further saves some text.
This commit is contained in:
Jens Axboe 2024-01-23 15:24:46 -07:00 committed by Christian Brauner
parent 0000ff2523
commit 2263639f96
1 changed files with 28 additions and 27 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

55
lib/iov_iter.c
View File

@ -714,12 +714,11 @@ EXPORT_SYMBOL(iov_iter_discard);
static bool iov_iter_aligned_iovec(const struct iov_iter *i, unsigned addr_mask,
unsigned len_mask)
{
const struct iovec *iov = iter_iov(i);
size_t size = i->count;
size_t skip = i->iov_offset;
unsigned k;
for (k = 0; k < i->nr_segs; k++, skip = 0) {
const struct iovec *iov = iter_iov(i) + k;
do {
size_t len = iov->iov_len - skip;
if (len > size)
@ -729,34 +728,36 @@ static bool iov_iter_aligned_iovec(const struct iov_iter *i, unsigned addr_mask,
if ((unsigned long)(iov->iov_base + skip) & addr_mask)
return false;
iov++;
size -= len;
if (!size)
break;
}
skip = 0;
} while (size);
return true;
}
static bool iov_iter_aligned_bvec(const struct iov_iter *i, unsigned addr_mask,
unsigned len_mask)
{
size_t size = i->count;
const struct bio_vec *bvec = i->bvec;
unsigned skip = i->iov_offset;
unsigned k;
size_t size = i->count;
for (k = 0; k < i->nr_segs; k++, skip = 0) {
size_t len = i->bvec[k].bv_len - skip;
do {
size_t len = bvec->bv_len;
if (len > size)
len = size;
if (len & len_mask)
return false;
if ((unsigned long)(i->bvec[k].bv_offset + skip) & addr_mask)
if ((unsigned long)(bvec->bv_offset + skip) & addr_mask)
return false;
bvec++;
size -= len;
if (!size)
break;
}
skip = 0;
} while (size);
return true;
}
@ -800,13 +801,12 @@ EXPORT_SYMBOL_GPL(iov_iter_is_aligned);
static unsigned long iov_iter_alignment_iovec(const struct iov_iter *i)
{
const struct iovec *iov = iter_iov(i);
unsigned long res = 0;
size_t size = i->count;
size_t skip = i->iov_offset;
unsigned k;
for (k = 0; k < i->nr_segs; k++, skip = 0) {
const struct iovec *iov = iter_iov(i) + k;
do {
size_t len = iov->iov_len - skip;
if (len) {
res |= (unsigned long)iov->iov_base + skip;
@ -814,30 +814,31 @@ static unsigned long iov_iter_alignment_iovec(const struct iov_iter *i)
len = size;
res |= len;
size -= len;
if (!size)
break;
}
}
iov++;
skip = 0;
} while (size);
return res;
}
static unsigned long iov_iter_alignment_bvec(const struct iov_iter *i)
{
const struct bio_vec *bvec = i->bvec;
unsigned res = 0;
size_t size = i->count;
unsigned skip = i->iov_offset;
unsigned k;
for (k = 0; k < i->nr_segs; k++, skip = 0) {
size_t len = i->bvec[k].bv_len - skip;
res |= (unsigned long)i->bvec[k].bv_offset + skip;
do {
size_t len = bvec->bv_len - skip;
res |= (unsigned long)bvec->bv_offset + skip;
if (len > size)
len = size;
res |= len;
bvec++;
size -= len;
if (!size)
break;
}
skip = 0;
} while (size);
return res;
}