2019-05-19 20:08:55 +08:00
|
|
|
// SPDX-License-Identifier: GPL-2.0-only
|
2005-04-17 06:20:36 +08:00
|
|
|
/*
|
|
|
|
* linux/fs/buffer.c
|
|
|
|
*
|
|
|
|
* Copyright (C) 1991, 1992, 2002 Linus Torvalds
|
|
|
|
*/
|
|
|
|
|
|
|
|
/*
|
|
|
|
* Start bdflush() with kernel_thread not syscall - Paul Gortmaker, 12/95
|
|
|
|
*
|
|
|
|
* Removed a lot of unnecessary code and simplified things now that
|
|
|
|
* the buffer cache isn't our primary cache - Andrew Tridgell 12/96
|
|
|
|
*
|
|
|
|
* Speed up hash, lru, and free list operations. Use gfp() for allocating
|
|
|
|
* hash table, use SLAB cache for buffer heads. SMP threading. -DaveM
|
|
|
|
*
|
|
|
|
* Added 32k buffer block sizes - these are required older ARM systems. - RMK
|
|
|
|
*
|
|
|
|
* async buffer flushing, 1999 Andrea Arcangeli <andrea@suse.de>
|
|
|
|
*/
|
|
|
|
|
|
|
|
#include <linux/kernel.h>
|
2017-02-04 06:47:37 +08:00
|
|
|
#include <linux/sched/signal.h>
|
2005-04-17 06:20:36 +08:00
|
|
|
#include <linux/syscalls.h>
|
|
|
|
#include <linux/fs.h>
|
2016-06-21 07:23:11 +08:00
|
|
|
#include <linux/iomap.h>
|
2005-04-17 06:20:36 +08:00
|
|
|
#include <linux/mm.h>
|
|
|
|
#include <linux/percpu.h>
|
|
|
|
#include <linux/slab.h>
|
2006-01-12 04:17:46 +08:00
|
|
|
#include <linux/capability.h>
|
2005-04-17 06:20:36 +08:00
|
|
|
#include <linux/blkdev.h>
|
|
|
|
#include <linux/file.h>
|
|
|
|
#include <linux/quotaops.h>
|
|
|
|
#include <linux/highmem.h>
|
2011-11-17 12:57:37 +08:00
|
|
|
#include <linux/export.h>
|
2015-06-02 22:37:23 +08:00
|
|
|
#include <linux/backing-dev.h>
|
2005-04-17 06:20:36 +08:00
|
|
|
#include <linux/writeback.h>
|
|
|
|
#include <linux/hash.h>
|
|
|
|
#include <linux/suspend.h>
|
|
|
|
#include <linux/buffer_head.h>
|
2006-12-10 18:19:27 +08:00
|
|
|
#include <linux/task_io_accounting_ops.h>
|
2005-04-17 06:20:36 +08:00
|
|
|
#include <linux/bio.h>
|
|
|
|
#include <linux/cpu.h>
|
|
|
|
#include <linux/bitops.h>
|
|
|
|
#include <linux/mpage.h>
|
[PATCH] spinlock consolidation
This patch (written by me and also containing many suggestions of Arjan van
de Ven) does a major cleanup of the spinlock code. It does the following
things:
- consolidates and enhances the spinlock/rwlock debugging code
- simplifies the asm/spinlock.h files
- encapsulates the raw spinlock type and moves generic spinlock
features (such as ->break_lock) into the generic code.
- cleans up the spinlock code hierarchy to get rid of the spaghetti.
Most notably there's now only a single variant of the debugging code,
located in lib/spinlock_debug.c. (previously we had one SMP debugging
variant per architecture, plus a separate generic one for UP builds)
Also, i've enhanced the rwlock debugging facility, it will now track
write-owners. There is new spinlock-owner/CPU-tracking on SMP builds too.
All locks have lockup detection now, which will work for both soft and hard
spin/rwlock lockups.
The arch-level include files now only contain the minimally necessary
subset of the spinlock code - all the rest that can be generalized now
lives in the generic headers:
include/asm-i386/spinlock_types.h | 16
include/asm-x86_64/spinlock_types.h | 16
I have also split up the various spinlock variants into separate files,
making it easier to see which does what. The new layout is:
SMP | UP
----------------------------|-----------------------------------
asm/spinlock_types_smp.h | linux/spinlock_types_up.h
linux/spinlock_types.h | linux/spinlock_types.h
asm/spinlock_smp.h | linux/spinlock_up.h
linux/spinlock_api_smp.h | linux/spinlock_api_up.h
linux/spinlock.h | linux/spinlock.h
/*
* here's the role of the various spinlock/rwlock related include files:
*
* on SMP builds:
*
* asm/spinlock_types.h: contains the raw_spinlock_t/raw_rwlock_t and the
* initializers
*
* linux/spinlock_types.h:
* defines the generic type and initializers
*
* asm/spinlock.h: contains the __raw_spin_*()/etc. lowlevel
* implementations, mostly inline assembly code
*
* (also included on UP-debug builds:)
*
* linux/spinlock_api_smp.h:
* contains the prototypes for the _spin_*() APIs.
*
* linux/spinlock.h: builds the final spin_*() APIs.
*
* on UP builds:
*
* linux/spinlock_type_up.h:
* contains the generic, simplified UP spinlock type.
* (which is an empty structure on non-debug builds)
*
* linux/spinlock_types.h:
* defines the generic type and initializers
*
* linux/spinlock_up.h:
* contains the __raw_spin_*()/etc. version of UP
* builds. (which are NOPs on non-debug, non-preempt
* builds)
*
* (included on UP-non-debug builds:)
*
* linux/spinlock_api_up.h:
* builds the _spin_*() APIs.
*
* linux/spinlock.h: builds the final spin_*() APIs.
*/
All SMP and UP architectures are converted by this patch.
arm, i386, ia64, ppc, ppc64, s390/s390x, x64 was build-tested via
crosscompilers. m32r, mips, sh, sparc, have not been tested yet, but should
be mostly fine.
From: Grant Grundler <grundler@parisc-linux.org>
Booted and lightly tested on a500-44 (64-bit, SMP kernel, dual CPU).
Builds 32-bit SMP kernel (not booted or tested). I did not try to build
non-SMP kernels. That should be trivial to fix up later if necessary.
I converted bit ops atomic_hash lock to raw_spinlock_t. Doing so avoids
some ugly nesting of linux/*.h and asm/*.h files. Those particular locks
are well tested and contained entirely inside arch specific code. I do NOT
expect any new issues to arise with them.
If someone does ever need to use debug/metrics with them, then they will
need to unravel this hairball between spinlocks, atomic ops, and bit ops
that exist only because parisc has exactly one atomic instruction: LDCW
(load and clear word).
From: "Luck, Tony" <tony.luck@intel.com>
ia64 fix
Signed-off-by: Ingo Molnar <mingo@elte.hu>
Signed-off-by: Arjan van de Ven <arjanv@infradead.org>
Signed-off-by: Grant Grundler <grundler@parisc-linux.org>
Cc: Matthew Wilcox <willy@debian.org>
Signed-off-by: Hirokazu Takata <takata@linux-m32r.org>
Signed-off-by: Mikael Pettersson <mikpe@csd.uu.se>
Signed-off-by: Benoit Boissinot <benoit.boissinot@ens-lyon.org>
Signed-off-by: Andrew Morton <akpm@osdl.org>
Signed-off-by: Linus Torvalds <torvalds@osdl.org>
2005-09-10 15:25:56 +08:00
|
|
|
#include <linux/bit_spinlock.h>
|
2016-11-05 01:08:11 +08:00
|
|
|
#include <linux/pagevec.h>
|
2018-08-18 06:46:44 +08:00
|
|
|
#include <linux/sched/mm.h>
|
2013-01-12 05:06:36 +08:00
|
|
|
#include <trace/events/block.h>
|
2019-10-23 11:33:11 +08:00
|
|
|
#include <linux/fscrypt.h>
|
2022-12-24 04:36:37 +08:00
|
|
|
#include <linux/fsverity.h>
|
fs/buffer.c: disable per-CPU buffer_head cache for isolated CPUs
For certain types of applications (for example PLC software or
RAN processing), upon occurrence of an event, it is necessary to
complete a certain task in a maximum amount of time (deadline).
One way to express this requirement is with a pair of numbers,
deadline time and execution time, where:
* deadline time: length of time between event and deadline.
* execution time: length of time it takes for processing of event
to occur on a particular hardware platform
(uninterrupted).
The particular values depend on use-case. For the case
where the realtime application executes in a virtualized
guest, an IPI which must be serviced in the host will cause
the following sequence of events:
1) VM-exit
2) execution of IPI (and function call)
3) VM-entry
Which causes an excess of 50us latency as observed by cyclictest
(this violates the latency requirement of vRAN application with 1ms TTI,
for example).
invalidate_bh_lrus calls an IPI on each CPU that has non empty
per-CPU cache:
on_each_cpu_cond(has_bh_in_lru, invalidate_bh_lru, NULL, 1);
The performance when using the per-CPU LRU cache is as follows:
42 ns per __find_get_block
68 ns per __find_get_block_slow
Given that the main use cases for latency sensitive applications
do not involve block I/O (data necessary for program operation is
locked in RAM), disable per-CPU buffer_head caches for isolated CPUs.
Signed-off-by: Marcelo Tosatti <mtosatti@redhat.com>
Acked-by: Frederic Weisbecker <frederic@kernel.org>
Message-Id: <ZJtBrybavtb1x45V@tpad>
Signed-off-by: Christian Brauner <brauner@kernel.org>
2023-06-28 04:08:15 +08:00
|
|
|
#include <linux/sched/isolation.h>
|
2005-04-17 06:20:36 +08:00
|
|
|
|
2019-12-01 09:49:18 +08:00
|
|
|
#include "internal.h"
|
|
|
|
|
2005-04-17 06:20:36 +08:00
|
|
|
static int fsync_buffers_list(spinlock_t *lock, struct list_head *list);
|
2022-08-18 13:04:40 +08:00
|
|
|
static void submit_bh_wbc(blk_opf_t opf, struct buffer_head *bh,
|
2024-02-03 04:39:25 +08:00
|
|
|
enum rw_hint hint, struct writeback_control *wbc);
|
2005-04-17 06:20:36 +08:00
|
|
|
|
|
|
|
#define BH_ENTRY(list) list_entry((list), struct buffer_head, b_assoc_buffers)
|
|
|
|
|
2013-01-12 05:06:35 +08:00
|
|
|
inline void touch_buffer(struct buffer_head *bh)
|
|
|
|
{
|
2013-01-12 05:06:36 +08:00
|
|
|
trace_block_touch_buffer(bh);
|
2022-12-16 05:43:53 +08:00
|
|
|
folio_mark_accessed(bh->b_folio);
|
2013-01-12 05:06:35 +08:00
|
|
|
}
|
|
|
|
EXPORT_SYMBOL(touch_buffer);
|
|
|
|
|
2008-02-08 20:19:52 +08:00
|
|
|
void __lock_buffer(struct buffer_head *bh)
|
2005-04-17 06:20:36 +08:00
|
|
|
{
|
sched: Remove proliferation of wait_on_bit() action functions
The current "wait_on_bit" interface requires an 'action'
function to be provided which does the actual waiting.
There are over 20 such functions, many of them identical.
Most cases can be satisfied by one of just two functions, one
which uses io_schedule() and one which just uses schedule().
So:
Rename wait_on_bit and wait_on_bit_lock to
wait_on_bit_action and wait_on_bit_lock_action
to make it explicit that they need an action function.
Introduce new wait_on_bit{,_lock} and wait_on_bit{,_lock}_io
which are *not* given an action function but implicitly use
a standard one.
The decision to error-out if a signal is pending is now made
based on the 'mode' argument rather than being encoded in the action
function.
All instances of the old wait_on_bit and wait_on_bit_lock which
can use the new version have been changed accordingly and their
action functions have been discarded.
wait_on_bit{_lock} does not return any specific error code in the
event of a signal so the caller must check for non-zero and
interpolate their own error code as appropriate.
The wait_on_bit() call in __fscache_wait_on_invalidate() was
ambiguous as it specified TASK_UNINTERRUPTIBLE but used
fscache_wait_bit_interruptible as an action function.
David Howells confirms this should be uniformly
"uninterruptible"
The main remaining user of wait_on_bit{,_lock}_action is NFS
which needs to use a freezer-aware schedule() call.
A comment in fs/gfs2/glock.c notes that having multiple 'action'
functions is useful as they display differently in the 'wchan'
field of 'ps'. (and /proc/$PID/wchan).
As the new bit_wait{,_io} functions are tagged "__sched", they
will not show up at all, but something higher in the stack. So
the distinction will still be visible, only with different
function names (gds2_glock_wait versus gfs2_glock_dq_wait in the
gfs2/glock.c case).
Since first version of this patch (against 3.15) two new action
functions appeared, on in NFS and one in CIFS. CIFS also now
uses an action function that makes the same freezer aware
schedule call as NFS.
Signed-off-by: NeilBrown <neilb@suse.de>
Acked-by: David Howells <dhowells@redhat.com> (fscache, keys)
Acked-by: Steven Whitehouse <swhiteho@redhat.com> (gfs2)
Acked-by: Peter Zijlstra <peterz@infradead.org>
Cc: Oleg Nesterov <oleg@redhat.com>
Cc: Steve French <sfrench@samba.org>
Cc: Linus Torvalds <torvalds@linux-foundation.org>
Link: http://lkml.kernel.org/r/20140707051603.28027.72349.stgit@notabene.brown
Signed-off-by: Ingo Molnar <mingo@kernel.org>
2014-07-07 13:16:04 +08:00
|
|
|
wait_on_bit_lock_io(&bh->b_state, BH_Lock, TASK_UNINTERRUPTIBLE);
|
2005-04-17 06:20:36 +08:00
|
|
|
}
|
|
|
|
EXPORT_SYMBOL(__lock_buffer);
|
|
|
|
|
2008-02-08 20:19:52 +08:00
|
|
|
void unlock_buffer(struct buffer_head *bh)
|
2005-04-17 06:20:36 +08:00
|
|
|
{
|
2008-10-19 11:27:00 +08:00
|
|
|
clear_bit_unlock(BH_Lock, &bh->b_state);
|
2014-03-18 01:06:10 +08:00
|
|
|
smp_mb__after_atomic();
|
2005-04-17 06:20:36 +08:00
|
|
|
wake_up_bit(&bh->b_state, BH_Lock);
|
|
|
|
}
|
2009-09-23 07:43:51 +08:00
|
|
|
EXPORT_SYMBOL(unlock_buffer);
|
2005-04-17 06:20:36 +08:00
|
|
|
|
2013-07-04 06:02:05 +08:00
|
|
|
/*
|
2022-01-18 03:35:22 +08:00
|
|
|
* Returns if the folio has dirty or writeback buffers. If all the buffers
|
|
|
|
* are unlocked and clean then the folio_test_dirty information is stale. If
|
|
|
|
* any of the buffers are locked, it is assumed they are locked for IO.
|
2013-07-04 06:02:05 +08:00
|
|
|
*/
|
2022-01-18 03:35:22 +08:00
|
|
|
void buffer_check_dirty_writeback(struct folio *folio,
|
2013-07-04 06:02:05 +08:00
|
|
|
bool *dirty, bool *writeback)
|
|
|
|
{
|
|
|
|
struct buffer_head *head, *bh;
|
|
|
|
*dirty = false;
|
|
|
|
*writeback = false;
|
|
|
|
|
2022-01-18 03:35:22 +08:00
|
|
|
BUG_ON(!folio_test_locked(folio));
|
2013-07-04 06:02:05 +08:00
|
|
|
|
2022-01-18 03:35:22 +08:00
|
|
|
head = folio_buffers(folio);
|
|
|
|
if (!head)
|
2013-07-04 06:02:05 +08:00
|
|
|
return;
|
|
|
|
|
2022-01-18 03:35:22 +08:00
|
|
|
if (folio_test_writeback(folio))
|
2013-07-04 06:02:05 +08:00
|
|
|
*writeback = true;
|
|
|
|
|
|
|
|
bh = head;
|
|
|
|
do {
|
|
|
|
if (buffer_locked(bh))
|
|
|
|
*writeback = true;
|
|
|
|
|
|
|
|
if (buffer_dirty(bh))
|
|
|
|
*dirty = true;
|
|
|
|
|
|
|
|
bh = bh->b_this_page;
|
|
|
|
} while (bh != head);
|
|
|
|
}
|
|
|
|
|
2005-04-17 06:20:36 +08:00
|
|
|
/*
|
|
|
|
* Block until a buffer comes unlocked. This doesn't stop it
|
|
|
|
* from becoming locked again - you have to lock it yourself
|
|
|
|
* if you want to preserve its state.
|
|
|
|
*/
|
|
|
|
void __wait_on_buffer(struct buffer_head * bh)
|
|
|
|
{
|
sched: Remove proliferation of wait_on_bit() action functions
The current "wait_on_bit" interface requires an 'action'
function to be provided which does the actual waiting.
There are over 20 such functions, many of them identical.
Most cases can be satisfied by one of just two functions, one
which uses io_schedule() and one which just uses schedule().
So:
Rename wait_on_bit and wait_on_bit_lock to
wait_on_bit_action and wait_on_bit_lock_action
to make it explicit that they need an action function.
Introduce new wait_on_bit{,_lock} and wait_on_bit{,_lock}_io
which are *not* given an action function but implicitly use
a standard one.
The decision to error-out if a signal is pending is now made
based on the 'mode' argument rather than being encoded in the action
function.
All instances of the old wait_on_bit and wait_on_bit_lock which
can use the new version have been changed accordingly and their
action functions have been discarded.
wait_on_bit{_lock} does not return any specific error code in the
event of a signal so the caller must check for non-zero and
interpolate their own error code as appropriate.
The wait_on_bit() call in __fscache_wait_on_invalidate() was
ambiguous as it specified TASK_UNINTERRUPTIBLE but used
fscache_wait_bit_interruptible as an action function.
David Howells confirms this should be uniformly
"uninterruptible"
The main remaining user of wait_on_bit{,_lock}_action is NFS
which needs to use a freezer-aware schedule() call.
A comment in fs/gfs2/glock.c notes that having multiple 'action'
functions is useful as they display differently in the 'wchan'
field of 'ps'. (and /proc/$PID/wchan).
As the new bit_wait{,_io} functions are tagged "__sched", they
will not show up at all, but something higher in the stack. So
the distinction will still be visible, only with different
function names (gds2_glock_wait versus gfs2_glock_dq_wait in the
gfs2/glock.c case).
Since first version of this patch (against 3.15) two new action
functions appeared, on in NFS and one in CIFS. CIFS also now
uses an action function that makes the same freezer aware
schedule call as NFS.
Signed-off-by: NeilBrown <neilb@suse.de>
Acked-by: David Howells <dhowells@redhat.com> (fscache, keys)
Acked-by: Steven Whitehouse <swhiteho@redhat.com> (gfs2)
Acked-by: Peter Zijlstra <peterz@infradead.org>
Cc: Oleg Nesterov <oleg@redhat.com>
Cc: Steve French <sfrench@samba.org>
Cc: Linus Torvalds <torvalds@linux-foundation.org>
Link: http://lkml.kernel.org/r/20140707051603.28027.72349.stgit@notabene.brown
Signed-off-by: Ingo Molnar <mingo@kernel.org>
2014-07-07 13:16:04 +08:00
|
|
|
wait_on_bit_io(&bh->b_state, BH_Lock, TASK_UNINTERRUPTIBLE);
|
2005-04-17 06:20:36 +08:00
|
|
|
}
|
2009-09-23 07:43:51 +08:00
|
|
|
EXPORT_SYMBOL(__wait_on_buffer);
|
2005-04-17 06:20:36 +08:00
|
|
|
|
fs: merge I/O error prints into one line
buffer.c uses two printk calls to print these messages:
[67353.422338] Buffer I/O error on device sdr, logical block 212868488
[67353.422338] lost page write due to I/O error on sdr
In a busy system, they may be interleaved with other prints,
losing the context for the second message. Merge them into
one line with one printk call so the prints are atomic.
Also, differentiate between async page writes, sync page writes, and
async page reads.
Also, shorten "device" to "dev" to match the block layer prints:
[67353.467906] blk_update_request: critical target error, dev sdr, sector
1707107328
Also, use %llu rather than %Lu.
Resulting prints look like:
[ 1356.437006] blk_update_request: critical target error, dev sdr, sector 1719693992
[ 1361.383522] quiet_error: 659876 callbacks suppressed
[ 1361.385816] Buffer I/O error on dev sdr, logical block 256902912, lost async page write
[ 1361.385819] Buffer I/O error on dev sdr, logical block 256903644, lost async page write
Signed-off-by: Robert Elliott <elliott@hp.com>
Reviewed-by: Webb Scales <webbnh@hp.com>
Signed-off-by: Jens Axboe <axboe@fb.com>
2014-10-22 03:55:09 +08:00
|
|
|
static void buffer_io_error(struct buffer_head *bh, char *msg)
|
2005-04-17 06:20:36 +08:00
|
|
|
{
|
fs: clarify rate limit suppressed buffer I/O errors
When quiet_error applies rate limiting to buffer_io_error calls, what the
they apply to is unclear because the name is so generic, particularly
if the messages are interleaved with others:
[ 1936.063572] quiet_error: 664293 callbacks suppressed
[ 1936.065297] Buffer I/O error on dev sdr, logical block 257429952, lost async page write
[ 1936.067814] Buffer I/O error on dev sdr, logical block 257429953, lost async page write
Also, the function uses printk_ratelimit(), although printk.h includes a
comment advising "Please don't use... Instead use printk_ratelimited()."
Change buffer_io_error to check the BH_Quiet bit itself, drop the
printk_ratelimit call, and print using printk_ratelimited.
This makes the messages look like:
[ 387.208839] buffer_io_error: 676394 callbacks suppressed
[ 387.210693] Buffer I/O error on dev sdr, logical block 211291776, lost async page write
[ 387.213432] Buffer I/O error on dev sdr, logical block 211291777, lost async page write
Signed-off-by: Robert Elliott <elliott@hp.com>
Reviewed-by: Webb Scales <webbnh@hp.com>
Signed-off-by: Jens Axboe <axboe@fb.com>
2014-10-22 03:55:11 +08:00
|
|
|
if (!test_bit(BH_Quiet, &bh->b_state))
|
|
|
|
printk_ratelimited(KERN_ERR
|
2015-04-13 20:31:37 +08:00
|
|
|
"Buffer I/O error on dev %pg, logical block %llu%s\n",
|
|
|
|
bh->b_bdev, (unsigned long long)bh->b_blocknr, msg);
|
2005-04-17 06:20:36 +08:00
|
|
|
}
|
|
|
|
|
|
|
|
/*
|
2007-10-16 16:24:47 +08:00
|
|
|
* End-of-IO handler helper function which does not touch the bh after
|
|
|
|
* unlocking it.
|
|
|
|
* Note: unlock_buffer() sort-of does touch the bh after unlocking it, but
|
|
|
|
* a race there is benign: unlock_buffer() only use the bh's address for
|
|
|
|
* hashing after unlocking the buffer, so it doesn't actually touch the bh
|
|
|
|
* itself.
|
2005-04-17 06:20:36 +08:00
|
|
|
*/
|
2007-10-16 16:24:47 +08:00
|
|
|
static void __end_buffer_read_notouch(struct buffer_head *bh, int uptodate)
|
2005-04-17 06:20:36 +08:00
|
|
|
{
|
|
|
|
if (uptodate) {
|
|
|
|
set_buffer_uptodate(bh);
|
|
|
|
} else {
|
2016-07-19 17:28:41 +08:00
|
|
|
/* This happens, due to failed read-ahead attempts. */
|
2005-04-17 06:20:36 +08:00
|
|
|
clear_buffer_uptodate(bh);
|
|
|
|
}
|
|
|
|
unlock_buffer(bh);
|
2007-10-16 16:24:47 +08:00
|
|
|
}
|
|
|
|
|
|
|
|
/*
|
|
|
|
* Default synchronous end-of-IO handler.. Just mark it up-to-date and
|
2022-09-01 21:35:03 +08:00
|
|
|
* unlock the buffer.
|
2007-10-16 16:24:47 +08:00
|
|
|
*/
|
|
|
|
void end_buffer_read_sync(struct buffer_head *bh, int uptodate)
|
|
|
|
{
|
|
|
|
__end_buffer_read_notouch(bh, uptodate);
|
2005-04-17 06:20:36 +08:00
|
|
|
put_bh(bh);
|
|
|
|
}
|
2009-09-23 07:43:51 +08:00
|
|
|
EXPORT_SYMBOL(end_buffer_read_sync);
|
2005-04-17 06:20:36 +08:00
|
|
|
|
|
|
|
void end_buffer_write_sync(struct buffer_head *bh, int uptodate)
|
|
|
|
{
|
|
|
|
if (uptodate) {
|
|
|
|
set_buffer_uptodate(bh);
|
|
|
|
} else {
|
fs: clarify rate limit suppressed buffer I/O errors
When quiet_error applies rate limiting to buffer_io_error calls, what the
they apply to is unclear because the name is so generic, particularly
if the messages are interleaved with others:
[ 1936.063572] quiet_error: 664293 callbacks suppressed
[ 1936.065297] Buffer I/O error on dev sdr, logical block 257429952, lost async page write
[ 1936.067814] Buffer I/O error on dev sdr, logical block 257429953, lost async page write
Also, the function uses printk_ratelimit(), although printk.h includes a
comment advising "Please don't use... Instead use printk_ratelimited()."
Change buffer_io_error to check the BH_Quiet bit itself, drop the
printk_ratelimit call, and print using printk_ratelimited.
This makes the messages look like:
[ 387.208839] buffer_io_error: 676394 callbacks suppressed
[ 387.210693] Buffer I/O error on dev sdr, logical block 211291776, lost async page write
[ 387.213432] Buffer I/O error on dev sdr, logical block 211291777, lost async page write
Signed-off-by: Robert Elliott <elliott@hp.com>
Reviewed-by: Webb Scales <webbnh@hp.com>
Signed-off-by: Jens Axboe <axboe@fb.com>
2014-10-22 03:55:11 +08:00
|
|
|
buffer_io_error(bh, ", lost sync page write");
|
2017-07-06 19:02:21 +08:00
|
|
|
mark_buffer_write_io_error(bh);
|
2005-04-17 06:20:36 +08:00
|
|
|
clear_buffer_uptodate(bh);
|
|
|
|
}
|
|
|
|
unlock_buffer(bh);
|
|
|
|
put_bh(bh);
|
|
|
|
}
|
2009-09-23 07:43:51 +08:00
|
|
|
EXPORT_SYMBOL(end_buffer_write_sync);
|
2005-04-17 06:20:36 +08:00
|
|
|
|
|
|
|
/*
|
|
|
|
* Various filesystems appear to want __find_get_block to be non-blocking.
|
|
|
|
* But it's the page lock which protects the buffers. To get around this,
|
|
|
|
* we get exclusion from try_to_free_buffers with the blockdev mapping's
|
2023-11-18 05:58:23 +08:00
|
|
|
* i_private_lock.
|
2005-04-17 06:20:36 +08:00
|
|
|
*
|
2023-11-18 05:58:23 +08:00
|
|
|
* Hack idea: for the blockdev mapping, i_private_lock contention
|
2005-04-17 06:20:36 +08:00
|
|
|
* may be quite high. This code could TryLock the page, and if that
|
2023-11-18 05:58:23 +08:00
|
|
|
* succeeds, there is no need to take i_private_lock.
|
2005-04-17 06:20:36 +08:00
|
|
|
*/
|
|
|
|
static struct buffer_head *
|
2005-11-07 16:59:39 +08:00
|
|
|
__find_get_block_slow(struct block_device *bdev, sector_t block)
|
2005-04-17 06:20:36 +08:00
|
|
|
{
|
|
|
|
struct inode *bd_inode = bdev->bd_inode;
|
|
|
|
struct address_space *bd_mapping = bd_inode->i_mapping;
|
|
|
|
struct buffer_head *ret = NULL;
|
|
|
|
pgoff_t index;
|
|
|
|
struct buffer_head *bh;
|
|
|
|
struct buffer_head *head;
|
2023-06-13 05:01:40 +08:00
|
|
|
struct folio *folio;
|
2005-04-17 06:20:36 +08:00
|
|
|
int all_mapped = 1;
|
fs: ratelimit __find_get_block_slow() failure message.
When something let __find_get_block_slow() hit all_mapped path, it calls
printk() for 100+ times per a second. But there is no need to print same
message with such high frequency; it is just asking for stall warning, or
at least bloating log files.
[ 399.866302][T15342] __find_get_block_slow() failed. block=1, b_blocknr=8
[ 399.873324][T15342] b_state=0x00000029, b_size=512
[ 399.878403][T15342] device loop0 blocksize: 4096
[ 399.883296][T15342] __find_get_block_slow() failed. block=1, b_blocknr=8
[ 399.890400][T15342] b_state=0x00000029, b_size=512
[ 399.895595][T15342] device loop0 blocksize: 4096
[ 399.900556][T15342] __find_get_block_slow() failed. block=1, b_blocknr=8
[ 399.907471][T15342] b_state=0x00000029, b_size=512
[ 399.912506][T15342] device loop0 blocksize: 4096
This patch reduces frequency to up to once per a second, in addition to
concatenating three lines into one.
[ 399.866302][T15342] __find_get_block_slow() failed. block=1, b_blocknr=8, b_state=0x00000029, b_size=512, device loop0 blocksize: 4096
Signed-off-by: Tetsuo Handa <penguin-kernel@I-love.SAKURA.ne.jp>
Reviewed-by: Jan Kara <jack@suse.cz>
Cc: Dmitry Vyukov <dvyukov@google.com>
Signed-off-by: Jens Axboe <axboe@kernel.dk>
2019-01-21 21:49:37 +08:00
|
|
|
static DEFINE_RATELIMIT_STATE(last_warned, HZ, 1);
|
2005-04-17 06:20:36 +08:00
|
|
|
|
2023-11-10 05:06:06 +08:00
|
|
|
index = ((loff_t)block << bd_inode->i_blkbits) / PAGE_SIZE;
|
2023-06-13 05:01:40 +08:00
|
|
|
folio = __filemap_get_folio(bd_mapping, index, FGP_ACCESSED, 0);
|
|
|
|
if (IS_ERR(folio))
|
2005-04-17 06:20:36 +08:00
|
|
|
goto out;
|
|
|
|
|
2023-11-18 05:58:23 +08:00
|
|
|
spin_lock(&bd_mapping->i_private_lock);
|
2023-06-13 05:01:40 +08:00
|
|
|
head = folio_buffers(folio);
|
|
|
|
if (!head)
|
2005-04-17 06:20:36 +08:00
|
|
|
goto out_unlock;
|
|
|
|
bh = head;
|
|
|
|
do {
|
2009-04-03 07:56:46 +08:00
|
|
|
if (!buffer_mapped(bh))
|
|
|
|
all_mapped = 0;
|
|
|
|
else if (bh->b_blocknr == block) {
|
2005-04-17 06:20:36 +08:00
|
|
|
ret = bh;
|
|
|
|
get_bh(bh);
|
|
|
|
goto out_unlock;
|
|
|
|
}
|
|
|
|
bh = bh->b_this_page;
|
|
|
|
} while (bh != head);
|
|
|
|
|
|
|
|
/* we might be here because some of the buffers on this page are
|
|
|
|
* not mapped. This is due to various races between
|
|
|
|
* file io on the block device and getblk. It gets dealt with
|
|
|
|
* elsewhere, don't buffer_error if we had some unmapped buffers
|
|
|
|
*/
|
fs: ratelimit __find_get_block_slow() failure message.
When something let __find_get_block_slow() hit all_mapped path, it calls
printk() for 100+ times per a second. But there is no need to print same
message with such high frequency; it is just asking for stall warning, or
at least bloating log files.
[ 399.866302][T15342] __find_get_block_slow() failed. block=1, b_blocknr=8
[ 399.873324][T15342] b_state=0x00000029, b_size=512
[ 399.878403][T15342] device loop0 blocksize: 4096
[ 399.883296][T15342] __find_get_block_slow() failed. block=1, b_blocknr=8
[ 399.890400][T15342] b_state=0x00000029, b_size=512
[ 399.895595][T15342] device loop0 blocksize: 4096
[ 399.900556][T15342] __find_get_block_slow() failed. block=1, b_blocknr=8
[ 399.907471][T15342] b_state=0x00000029, b_size=512
[ 399.912506][T15342] device loop0 blocksize: 4096
This patch reduces frequency to up to once per a second, in addition to
concatenating three lines into one.
[ 399.866302][T15342] __find_get_block_slow() failed. block=1, b_blocknr=8, b_state=0x00000029, b_size=512, device loop0 blocksize: 4096
Signed-off-by: Tetsuo Handa <penguin-kernel@I-love.SAKURA.ne.jp>
Reviewed-by: Jan Kara <jack@suse.cz>
Cc: Dmitry Vyukov <dvyukov@google.com>
Signed-off-by: Jens Axboe <axboe@kernel.dk>
2019-01-21 21:49:37 +08:00
|
|
|
ratelimit_set_flags(&last_warned, RATELIMIT_MSG_ON_RELEASE);
|
|
|
|
if (all_mapped && __ratelimit(&last_warned)) {
|
|
|
|
printk("__find_get_block_slow() failed. block=%llu, "
|
|
|
|
"b_blocknr=%llu, b_state=0x%08lx, b_size=%zu, "
|
|
|
|
"device %pg blocksize: %d\n",
|
|
|
|
(unsigned long long)block,
|
|
|
|
(unsigned long long)bh->b_blocknr,
|
|
|
|
bh->b_state, bh->b_size, bdev,
|
|
|
|
1 << bd_inode->i_blkbits);
|
2005-04-17 06:20:36 +08:00
|
|
|
}
|
|
|
|
out_unlock:
|
2023-11-18 05:58:23 +08:00
|
|
|
spin_unlock(&bd_mapping->i_private_lock);
|
2023-06-13 05:01:40 +08:00
|
|
|
folio_put(folio);
|
2005-04-17 06:20:36 +08:00
|
|
|
out:
|
|
|
|
return ret;
|
|
|
|
}
|
|
|
|
|
|
|
|
static void end_buffer_async_read(struct buffer_head *bh, int uptodate)
|
|
|
|
{
|
|
|
|
unsigned long flags;
|
2005-07-08 08:56:56 +08:00
|
|
|
struct buffer_head *first;
|
2005-04-17 06:20:36 +08:00
|
|
|
struct buffer_head *tmp;
|
2022-12-16 05:43:54 +08:00
|
|
|
struct folio *folio;
|
|
|
|
int folio_uptodate = 1;
|
2005-04-17 06:20:36 +08:00
|
|
|
|
|
|
|
BUG_ON(!buffer_async_read(bh));
|
|
|
|
|
2022-12-16 05:43:54 +08:00
|
|
|
folio = bh->b_folio;
|
2005-04-17 06:20:36 +08:00
|
|
|
if (uptodate) {
|
|
|
|
set_buffer_uptodate(bh);
|
|
|
|
} else {
|
|
|
|
clear_buffer_uptodate(bh);
|
fs: clarify rate limit suppressed buffer I/O errors
When quiet_error applies rate limiting to buffer_io_error calls, what the
they apply to is unclear because the name is so generic, particularly
if the messages are interleaved with others:
[ 1936.063572] quiet_error: 664293 callbacks suppressed
[ 1936.065297] Buffer I/O error on dev sdr, logical block 257429952, lost async page write
[ 1936.067814] Buffer I/O error on dev sdr, logical block 257429953, lost async page write
Also, the function uses printk_ratelimit(), although printk.h includes a
comment advising "Please don't use... Instead use printk_ratelimited()."
Change buffer_io_error to check the BH_Quiet bit itself, drop the
printk_ratelimit call, and print using printk_ratelimited.
This makes the messages look like:
[ 387.208839] buffer_io_error: 676394 callbacks suppressed
[ 387.210693] Buffer I/O error on dev sdr, logical block 211291776, lost async page write
[ 387.213432] Buffer I/O error on dev sdr, logical block 211291777, lost async page write
Signed-off-by: Robert Elliott <elliott@hp.com>
Reviewed-by: Webb Scales <webbnh@hp.com>
Signed-off-by: Jens Axboe <axboe@fb.com>
2014-10-22 03:55:11 +08:00
|
|
|
buffer_io_error(bh, ", async page read");
|
2022-12-16 05:43:54 +08:00
|
|
|
folio_set_error(folio);
|
2005-04-17 06:20:36 +08:00
|
|
|
}
|
|
|
|
|
|
|
|
/*
|
|
|
|
* Be _very_ careful from here on. Bad things can happen if
|
|
|
|
* two buffer heads end IO at almost the same time and both
|
|
|
|
* decide that the page is now completely done.
|
|
|
|
*/
|
2022-12-16 05:43:54 +08:00
|
|
|
first = folio_buffers(folio);
|
2019-11-18 21:28:24 +08:00
|
|
|
spin_lock_irqsave(&first->b_uptodate_lock, flags);
|
2005-04-17 06:20:36 +08:00
|
|
|
clear_buffer_async_read(bh);
|
|
|
|
unlock_buffer(bh);
|
|
|
|
tmp = bh;
|
|
|
|
do {
|
|
|
|
if (!buffer_uptodate(tmp))
|
2022-12-16 05:43:54 +08:00
|
|
|
folio_uptodate = 0;
|
2005-04-17 06:20:36 +08:00
|
|
|
if (buffer_async_read(tmp)) {
|
|
|
|
BUG_ON(!buffer_locked(tmp));
|
|
|
|
goto still_busy;
|
|
|
|
}
|
|
|
|
tmp = tmp->b_this_page;
|
|
|
|
} while (tmp != bh);
|
2019-11-18 21:28:24 +08:00
|
|
|
spin_unlock_irqrestore(&first->b_uptodate_lock, flags);
|
2005-04-17 06:20:36 +08:00
|
|
|
|
2023-10-05 00:53:05 +08:00
|
|
|
folio_end_read(folio, folio_uptodate);
|
2005-04-17 06:20:36 +08:00
|
|
|
return;
|
|
|
|
|
|
|
|
still_busy:
|
2019-11-18 21:28:24 +08:00
|
|
|
spin_unlock_irqrestore(&first->b_uptodate_lock, flags);
|
2005-04-17 06:20:36 +08:00
|
|
|
return;
|
|
|
|
}
|
|
|
|
|
2022-12-24 04:36:37 +08:00
|
|
|
struct postprocess_bh_ctx {
|
2019-10-23 11:33:11 +08:00
|
|
|
struct work_struct work;
|
|
|
|
struct buffer_head *bh;
|
|
|
|
};
|
|
|
|
|
2022-12-24 04:36:37 +08:00
|
|
|
static void verify_bh(struct work_struct *work)
|
|
|
|
{
|
|
|
|
struct postprocess_bh_ctx *ctx =
|
|
|
|
container_of(work, struct postprocess_bh_ctx, work);
|
|
|
|
struct buffer_head *bh = ctx->bh;
|
|
|
|
bool valid;
|
|
|
|
|
2023-02-25 07:25:30 +08:00
|
|
|
valid = fsverity_verify_blocks(bh->b_folio, bh->b_size, bh_offset(bh));
|
2022-12-24 04:36:37 +08:00
|
|
|
end_buffer_async_read(bh, valid);
|
|
|
|
kfree(ctx);
|
|
|
|
}
|
|
|
|
|
|
|
|
static bool need_fsverity(struct buffer_head *bh)
|
|
|
|
{
|
2023-02-25 07:25:30 +08:00
|
|
|
struct folio *folio = bh->b_folio;
|
|
|
|
struct inode *inode = folio->mapping->host;
|
2022-12-24 04:36:37 +08:00
|
|
|
|
|
|
|
return fsverity_active(inode) &&
|
|
|
|
/* needed by ext4 */
|
2023-02-25 07:25:30 +08:00
|
|
|
folio->index < DIV_ROUND_UP(inode->i_size, PAGE_SIZE);
|
2022-12-24 04:36:37 +08:00
|
|
|
}
|
|
|
|
|
2019-10-23 11:33:11 +08:00
|
|
|
static void decrypt_bh(struct work_struct *work)
|
|
|
|
{
|
2022-12-24 04:36:37 +08:00
|
|
|
struct postprocess_bh_ctx *ctx =
|
|
|
|
container_of(work, struct postprocess_bh_ctx, work);
|
2019-10-23 11:33:11 +08:00
|
|
|
struct buffer_head *bh = ctx->bh;
|
|
|
|
int err;
|
|
|
|
|
2023-02-25 07:25:03 +08:00
|
|
|
err = fscrypt_decrypt_pagecache_blocks(bh->b_folio, bh->b_size,
|
|
|
|
bh_offset(bh));
|
2022-12-24 04:36:37 +08:00
|
|
|
if (err == 0 && need_fsverity(bh)) {
|
|
|
|
/*
|
|
|
|
* We use different work queues for decryption and for verity
|
|
|
|
* because verity may require reading metadata pages that need
|
|
|
|
* decryption, and we shouldn't recurse to the same workqueue.
|
|
|
|
*/
|
|
|
|
INIT_WORK(&ctx->work, verify_bh);
|
|
|
|
fsverity_enqueue_verify_work(&ctx->work);
|
|
|
|
return;
|
|
|
|
}
|
2019-10-23 11:33:11 +08:00
|
|
|
end_buffer_async_read(bh, err == 0);
|
|
|
|
kfree(ctx);
|
|
|
|
}
|
|
|
|
|
|
|
|
/*
|
2022-04-29 22:40:40 +08:00
|
|
|
* I/O completion handler for block_read_full_folio() - pages
|
2019-10-23 11:33:11 +08:00
|
|
|
* which come unlocked at the end of I/O.
|
|
|
|
*/
|
|
|
|
static void end_buffer_async_read_io(struct buffer_head *bh, int uptodate)
|
|
|
|
{
|
- Daniel Verkamp has contributed a memfd series ("mm/memfd: add
F_SEAL_EXEC") which permits the setting of the memfd execute bit at
memfd creation time, with the option of sealing the state of the X bit.
- Peter Xu adds a patch series ("mm/hugetlb: Make huge_pte_offset()
thread-safe for pmd unshare") which addresses a rare race condition
related to PMD unsharing.
- Several folioification patch serieses from Matthew Wilcox, Vishal
Moola, Sidhartha Kumar and Lorenzo Stoakes
- Johannes Weiner has a series ("mm: push down lock_page_memcg()") which
does perform some memcg maintenance and cleanup work.
- SeongJae Park has added DAMOS filtering to DAMON, with the series
"mm/damon/core: implement damos filter". These filters provide users
with finer-grained control over DAMOS's actions. SeongJae has also done
some DAMON cleanup work.
- Kairui Song adds a series ("Clean up and fixes for swap").
- Vernon Yang contributed the series "Clean up and refinement for maple
tree".
- Yu Zhao has contributed the "mm: multi-gen LRU: memcg LRU" series. It
adds to MGLRU an LRU of memcgs, to improve the scalability of global
reclaim.
- David Hildenbrand has added some userfaultfd cleanup work in the
series "mm: uffd-wp + change_protection() cleanups".
- Christoph Hellwig has removed the generic_writepages() library
function in the series "remove generic_writepages".
- Baolin Wang has performed some maintenance on the compaction code in
his series "Some small improvements for compaction".
- Sidhartha Kumar is doing some maintenance work on struct page in his
series "Get rid of tail page fields".
- David Hildenbrand contributed some cleanup, bugfixing and
generalization of pte management and of pte debugging in his series "mm:
support __HAVE_ARCH_PTE_SWP_EXCLUSIVE on all architectures with swap
PTEs".
- Mel Gorman and Neil Brown have removed the __GFP_ATOMIC allocation
flag in the series "Discard __GFP_ATOMIC".
- Sergey Senozhatsky has improved zsmalloc's memory utilization with his
series "zsmalloc: make zspage chain size configurable".
- Joey Gouly has added prctl() support for prohibiting the creation of
writeable+executable mappings. The previous BPF-based approach had
shortcomings. See "mm: In-kernel support for memory-deny-write-execute
(MDWE)".
- Waiman Long did some kmemleak cleanup and bugfixing in the series
"mm/kmemleak: Simplify kmemleak_cond_resched() & fix UAF".
- T.J. Alumbaugh has contributed some MGLRU cleanup work in his series
"mm: multi-gen LRU: improve".
- Jiaqi Yan has provided some enhancements to our memory error
statistics reporting, mainly by presenting the statistics on a per-node
basis. See the series "Introduce per NUMA node memory error
statistics".
- Mel Gorman has a second and hopefully final shot at fixing a CPU-hog
regression in compaction via his series "Fix excessive CPU usage during
compaction".
- Christoph Hellwig does some vmalloc maintenance work in the series
"cleanup vfree and vunmap".
- Christoph Hellwig has removed block_device_operations.rw_page() in ths
series "remove ->rw_page".
- We get some maple_tree improvements and cleanups in Liam Howlett's
series "VMA tree type safety and remove __vma_adjust()".
- Suren Baghdasaryan has done some work on the maintainability of our
vm_flags handling in the series "introduce vm_flags modifier functions".
- Some pagemap cleanup and generalization work in Mike Rapoport's series
"mm, arch: add generic implementation of pfn_valid() for FLATMEM" and
"fixups for generic implementation of pfn_valid()"
- Baoquan He has done some work to make /proc/vmallocinfo and
/proc/kcore better represent the real state of things in his series
"mm/vmalloc.c: allow vread() to read out vm_map_ram areas".
- Jason Gunthorpe rationalized the GUP system's interface to the rest of
the kernel in the series "Simplify the external interface for GUP".
- SeongJae Park wishes to migrate people from DAMON's debugfs interface
over to its sysfs interface. To support this, we'll temporarily be
printing warnings when people use the debugfs interface. See the series
"mm/damon: deprecate DAMON debugfs interface".
- Andrey Konovalov provided the accurately named "lib/stackdepot: fixes
and clean-ups" series.
- Huang Ying has provided a dramatic reduction in migration's TLB flush
IPI rates with the series "migrate_pages(): batch TLB flushing".
- Arnd Bergmann has some objtool fixups in "objtool warning fixes".
-----BEGIN PGP SIGNATURE-----
iHUEABYIAB0WIQTTMBEPP41GrTpTJgfdBJ7gKXxAjgUCY/PoPQAKCRDdBJ7gKXxA
jlvpAPsFECUBBl20qSue2zCYWnHC7Yk4q9ytTkPB/MMDrFEN9wD/SNKEm2UoK6/K
DmxHkn0LAitGgJRS/W9w81yrgig9tAQ=
=MlGs
-----END PGP SIGNATURE-----
Merge tag 'mm-stable-2023-02-20-13-37' of git://git.kernel.org/pub/scm/linux/kernel/git/akpm/mm
Pull MM updates from Andrew Morton:
- Daniel Verkamp has contributed a memfd series ("mm/memfd: add
F_SEAL_EXEC") which permits the setting of the memfd execute bit at
memfd creation time, with the option of sealing the state of the X
bit.
- Peter Xu adds a patch series ("mm/hugetlb: Make huge_pte_offset()
thread-safe for pmd unshare") which addresses a rare race condition
related to PMD unsharing.
- Several folioification patch serieses from Matthew Wilcox, Vishal
Moola, Sidhartha Kumar and Lorenzo Stoakes
- Johannes Weiner has a series ("mm: push down lock_page_memcg()")
which does perform some memcg maintenance and cleanup work.
- SeongJae Park has added DAMOS filtering to DAMON, with the series
"mm/damon/core: implement damos filter".
These filters provide users with finer-grained control over DAMOS's
actions. SeongJae has also done some DAMON cleanup work.
- Kairui Song adds a series ("Clean up and fixes for swap").
- Vernon Yang contributed the series "Clean up and refinement for maple
tree".
- Yu Zhao has contributed the "mm: multi-gen LRU: memcg LRU" series. It
adds to MGLRU an LRU of memcgs, to improve the scalability of global
reclaim.
- David Hildenbrand has added some userfaultfd cleanup work in the
series "mm: uffd-wp + change_protection() cleanups".
- Christoph Hellwig has removed the generic_writepages() library
function in the series "remove generic_writepages".
- Baolin Wang has performed some maintenance on the compaction code in
his series "Some small improvements for compaction".
- Sidhartha Kumar is doing some maintenance work on struct page in his
series "Get rid of tail page fields".
- David Hildenbrand contributed some cleanup, bugfixing and
generalization of pte management and of pte debugging in his series
"mm: support __HAVE_ARCH_PTE_SWP_EXCLUSIVE on all architectures with
swap PTEs".
- Mel Gorman and Neil Brown have removed the __GFP_ATOMIC allocation
flag in the series "Discard __GFP_ATOMIC".
- Sergey Senozhatsky has improved zsmalloc's memory utilization with
his series "zsmalloc: make zspage chain size configurable".
- Joey Gouly has added prctl() support for prohibiting the creation of
writeable+executable mappings.
The previous BPF-based approach had shortcomings. See "mm: In-kernel
support for memory-deny-write-execute (MDWE)".
- Waiman Long did some kmemleak cleanup and bugfixing in the series
"mm/kmemleak: Simplify kmemleak_cond_resched() & fix UAF".
- T.J. Alumbaugh has contributed some MGLRU cleanup work in his series
"mm: multi-gen LRU: improve".
- Jiaqi Yan has provided some enhancements to our memory error
statistics reporting, mainly by presenting the statistics on a
per-node basis. See the series "Introduce per NUMA node memory error
statistics".
- Mel Gorman has a second and hopefully final shot at fixing a CPU-hog
regression in compaction via his series "Fix excessive CPU usage
during compaction".
- Christoph Hellwig does some vmalloc maintenance work in the series
"cleanup vfree and vunmap".
- Christoph Hellwig has removed block_device_operations.rw_page() in
ths series "remove ->rw_page".
- We get some maple_tree improvements and cleanups in Liam Howlett's
series "VMA tree type safety and remove __vma_adjust()".
- Suren Baghdasaryan has done some work on the maintainability of our
vm_flags handling in the series "introduce vm_flags modifier
functions".
- Some pagemap cleanup and generalization work in Mike Rapoport's
series "mm, arch: add generic implementation of pfn_valid() for
FLATMEM" and "fixups for generic implementation of pfn_valid()"
- Baoquan He has done some work to make /proc/vmallocinfo and
/proc/kcore better represent the real state of things in his series
"mm/vmalloc.c: allow vread() to read out vm_map_ram areas".
- Jason Gunthorpe rationalized the GUP system's interface to the rest
of the kernel in the series "Simplify the external interface for
GUP".
- SeongJae Park wishes to migrate people from DAMON's debugfs interface
over to its sysfs interface. To support this, we'll temporarily be
printing warnings when people use the debugfs interface. See the
series "mm/damon: deprecate DAMON debugfs interface".
- Andrey Konovalov provided the accurately named "lib/stackdepot: fixes
and clean-ups" series.
- Huang Ying has provided a dramatic reduction in migration's TLB flush
IPI rates with the series "migrate_pages(): batch TLB flushing".
- Arnd Bergmann has some objtool fixups in "objtool warning fixes".
* tag 'mm-stable-2023-02-20-13-37' of git://git.kernel.org/pub/scm/linux/kernel/git/akpm/mm: (505 commits)
include/linux/migrate.h: remove unneeded externs
mm/memory_hotplug: cleanup return value handing in do_migrate_range()
mm/uffd: fix comment in handling pte markers
mm: change to return bool for isolate_movable_page()
mm: hugetlb: change to return bool for isolate_hugetlb()
mm: change to return bool for isolate_lru_page()
mm: change to return bool for folio_isolate_lru()
objtool: add UACCESS exceptions for __tsan_volatile_read/write
kmsan: disable ftrace in kmsan core code
kasan: mark addr_has_metadata __always_inline
mm: memcontrol: rename memcg_kmem_enabled()
sh: initialize max_mapnr
m68k/nommu: add missing definition of ARCH_PFN_OFFSET
mm: percpu: fix incorrect size in pcpu_obj_full_size()
maple_tree: reduce stack usage with gcc-9 and earlier
mm: page_alloc: call panic() when memoryless node allocation fails
mm: multi-gen LRU: avoid futile retries
migrate_pages: move THP/hugetlb migration support check to simplify code
migrate_pages: batch flushing TLB
migrate_pages: share more code between _unmap and _move
...
2023-02-24 09:09:35 +08:00
|
|
|
struct inode *inode = bh->b_folio->mapping->host;
|
2022-12-24 04:36:37 +08:00
|
|
|
bool decrypt = fscrypt_inode_uses_fs_layer_crypto(inode);
|
|
|
|
bool verify = need_fsverity(bh);
|
|
|
|
|
|
|
|
/* Decrypt (with fscrypt) and/or verify (with fsverity) if needed. */
|
|
|
|
if (uptodate && (decrypt || verify)) {
|
|
|
|
struct postprocess_bh_ctx *ctx =
|
|
|
|
kmalloc(sizeof(*ctx), GFP_ATOMIC);
|
2019-10-23 11:33:11 +08:00
|
|
|
|
|
|
|
if (ctx) {
|
|
|
|
ctx->bh = bh;
|
2022-12-24 04:36:37 +08:00
|
|
|
if (decrypt) {
|
|
|
|
INIT_WORK(&ctx->work, decrypt_bh);
|
|
|
|
fscrypt_enqueue_decrypt_work(&ctx->work);
|
|
|
|
} else {
|
|
|
|
INIT_WORK(&ctx->work, verify_bh);
|
|
|
|
fsverity_enqueue_verify_work(&ctx->work);
|
|
|
|
}
|
2019-10-23 11:33:11 +08:00
|
|
|
return;
|
|
|
|
}
|
|
|
|
uptodate = 0;
|
|
|
|
}
|
|
|
|
end_buffer_async_read(bh, uptodate);
|
|
|
|
}
|
|
|
|
|
2005-04-17 06:20:36 +08:00
|
|
|
/*
|
2023-12-16 04:02:45 +08:00
|
|
|
* Completion handler for block_write_full_folio() - folios which are unlocked
|
|
|
|
* during I/O, and which have the writeback flag cleared upon I/O completion.
|
2005-04-17 06:20:36 +08:00
|
|
|
*/
|
2023-12-16 04:02:45 +08:00
|
|
|
static void end_buffer_async_write(struct buffer_head *bh, int uptodate)
|
2005-04-17 06:20:36 +08:00
|
|
|
{
|
|
|
|
unsigned long flags;
|
2005-07-08 08:56:56 +08:00
|
|
|
struct buffer_head *first;
|
2005-04-17 06:20:36 +08:00
|
|
|
struct buffer_head *tmp;
|
2022-12-16 05:43:55 +08:00
|
|
|
struct folio *folio;
|
2005-04-17 06:20:36 +08:00
|
|
|
|
|
|
|
BUG_ON(!buffer_async_write(bh));
|
|
|
|
|
2022-12-16 05:43:55 +08:00
|
|
|
folio = bh->b_folio;
|
2005-04-17 06:20:36 +08:00
|
|
|
if (uptodate) {
|
|
|
|
set_buffer_uptodate(bh);
|
|
|
|
} else {
|
fs: clarify rate limit suppressed buffer I/O errors
When quiet_error applies rate limiting to buffer_io_error calls, what the
they apply to is unclear because the name is so generic, particularly
if the messages are interleaved with others:
[ 1936.063572] quiet_error: 664293 callbacks suppressed
[ 1936.065297] Buffer I/O error on dev sdr, logical block 257429952, lost async page write
[ 1936.067814] Buffer I/O error on dev sdr, logical block 257429953, lost async page write
Also, the function uses printk_ratelimit(), although printk.h includes a
comment advising "Please don't use... Instead use printk_ratelimited()."
Change buffer_io_error to check the BH_Quiet bit itself, drop the
printk_ratelimit call, and print using printk_ratelimited.
This makes the messages look like:
[ 387.208839] buffer_io_error: 676394 callbacks suppressed
[ 387.210693] Buffer I/O error on dev sdr, logical block 211291776, lost async page write
[ 387.213432] Buffer I/O error on dev sdr, logical block 211291777, lost async page write
Signed-off-by: Robert Elliott <elliott@hp.com>
Reviewed-by: Webb Scales <webbnh@hp.com>
Signed-off-by: Jens Axboe <axboe@fb.com>
2014-10-22 03:55:11 +08:00
|
|
|
buffer_io_error(bh, ", lost async page write");
|
2017-07-06 19:02:21 +08:00
|
|
|
mark_buffer_write_io_error(bh);
|
2005-04-17 06:20:36 +08:00
|
|
|
clear_buffer_uptodate(bh);
|
2022-12-16 05:43:55 +08:00
|
|
|
folio_set_error(folio);
|
2005-04-17 06:20:36 +08:00
|
|
|
}
|
|
|
|
|
2022-12-16 05:43:55 +08:00
|
|
|
first = folio_buffers(folio);
|
2019-11-18 21:28:24 +08:00
|
|
|
spin_lock_irqsave(&first->b_uptodate_lock, flags);
|
2005-07-08 08:56:56 +08:00
|
|
|
|
2005-04-17 06:20:36 +08:00
|
|
|
clear_buffer_async_write(bh);
|
|
|
|
unlock_buffer(bh);
|
|
|
|
tmp = bh->b_this_page;
|
|
|
|
while (tmp != bh) {
|
|
|
|
if (buffer_async_write(tmp)) {
|
|
|
|
BUG_ON(!buffer_locked(tmp));
|
|
|
|
goto still_busy;
|
|
|
|
}
|
|
|
|
tmp = tmp->b_this_page;
|
|
|
|
}
|
2019-11-18 21:28:24 +08:00
|
|
|
spin_unlock_irqrestore(&first->b_uptodate_lock, flags);
|
2022-12-16 05:43:55 +08:00
|
|
|
folio_end_writeback(folio);
|
2005-04-17 06:20:36 +08:00
|
|
|
return;
|
|
|
|
|
|
|
|
still_busy:
|
2019-11-18 21:28:24 +08:00
|
|
|
spin_unlock_irqrestore(&first->b_uptodate_lock, flags);
|
2005-04-17 06:20:36 +08:00
|
|
|
return;
|
|
|
|
}
|
|
|
|
|
|
|
|
/*
|
|
|
|
* If a page's buffers are under async readin (end_buffer_async_read
|
|
|
|
* completion) then there is a possibility that another thread of
|
|
|
|
* control could lock one of the buffers after it has completed
|
|
|
|
* but while some of the other buffers have not completed. This
|
|
|
|
* locked buffer would confuse end_buffer_async_read() into not unlocking
|
|
|
|
* the page. So the absence of BH_Async_Read tells end_buffer_async_read()
|
|
|
|
* that this buffer is not under async I/O.
|
|
|
|
*
|
|
|
|
* The page comes unlocked when it has no locked buffer_async buffers
|
|
|
|
* left.
|
|
|
|
*
|
|
|
|
* PageLocked prevents anyone starting new async I/O reads any of
|
|
|
|
* the buffers.
|
|
|
|
*
|
|
|
|
* PageWriteback is used to prevent simultaneous writeout of the same
|
|
|
|
* page.
|
|
|
|
*
|
|
|
|
* PageLocked prevents anyone from starting writeback of a page which is
|
|
|
|
* under read I/O (PageWriteback is only ever set against a locked page).
|
|
|
|
*/
|
|
|
|
static void mark_buffer_async_read(struct buffer_head *bh)
|
|
|
|
{
|
2019-10-23 11:33:11 +08:00
|
|
|
bh->b_end_io = end_buffer_async_read_io;
|
2005-04-17 06:20:36 +08:00
|
|
|
set_buffer_async_read(bh);
|
|
|
|
}
|
|
|
|
|
2009-09-23 07:43:51 +08:00
|
|
|
static void mark_buffer_async_write_endio(struct buffer_head *bh,
|
|
|
|
bh_end_io_t *handler)
|
2005-04-17 06:20:36 +08:00
|
|
|
{
|
2009-04-16 01:22:38 +08:00
|
|
|
bh->b_end_io = handler;
|
2005-04-17 06:20:36 +08:00
|
|
|
set_buffer_async_write(bh);
|
|
|
|
}
|
2009-04-16 01:22:38 +08:00
|
|
|
|
|
|
|
void mark_buffer_async_write(struct buffer_head *bh)
|
|
|
|
{
|
|
|
|
mark_buffer_async_write_endio(bh, end_buffer_async_write);
|
|
|
|
}
|
2005-04-17 06:20:36 +08:00
|
|
|
EXPORT_SYMBOL(mark_buffer_async_write);
|
|
|
|
|
|
|
|
|
|
|
|
/*
|
|
|
|
* fs/buffer.c contains helper functions for buffer-backed address space's
|
|
|
|
* fsync functions. A common requirement for buffer-based filesystems is
|
|
|
|
* that certain data from the backing blockdev needs to be written out for
|
|
|
|
* a successful fsync(). For example, ext2 indirect blocks need to be
|
|
|
|
* written back and waited upon before fsync() returns.
|
|
|
|
*
|
2024-01-09 01:20:40 +08:00
|
|
|
* The functions mark_buffer_dirty_inode(), fsync_inode_buffers(),
|
2005-04-17 06:20:36 +08:00
|
|
|
* inode_has_buffers() and invalidate_inode_buffers() are provided for the
|
2023-11-18 05:58:23 +08:00
|
|
|
* management of a list of dependent buffers at ->i_mapping->i_private_list.
|
2005-04-17 06:20:36 +08:00
|
|
|
*
|
|
|
|
* Locking is a little subtle: try_to_free_buffers() will remove buffers
|
|
|
|
* from their controlling inode's queue when they are being freed. But
|
|
|
|
* try_to_free_buffers() will be operating against the *blockdev* mapping
|
|
|
|
* at the time, not against the S_ISREG file which depends on those buffers.
|
2023-11-18 05:58:23 +08:00
|
|
|
* So the locking for i_private_list is via the i_private_lock in the address_space
|
2005-04-17 06:20:36 +08:00
|
|
|
* which backs the buffers. Which is different from the address_space
|
|
|
|
* against which the buffers are listed. So for a particular address_space,
|
2023-11-18 05:58:23 +08:00
|
|
|
* mapping->i_private_lock does *not* protect mapping->i_private_list! In fact,
|
|
|
|
* mapping->i_private_list will always be protected by the backing blockdev's
|
|
|
|
* ->i_private_lock.
|
2005-04-17 06:20:36 +08:00
|
|
|
*
|
|
|
|
* Which introduces a requirement: all buffers on an address_space's
|
2023-11-18 05:58:23 +08:00
|
|
|
* ->i_private_list must be from the same address_space: the blockdev's.
|
2005-04-17 06:20:36 +08:00
|
|
|
*
|
2023-11-18 05:58:23 +08:00
|
|
|
* address_spaces which do not place buffers at ->i_private_list via these
|
|
|
|
* utility functions are free to use i_private_lock and i_private_list for
|
|
|
|
* whatever they want. The only requirement is that list_empty(i_private_list)
|
2005-04-17 06:20:36 +08:00
|
|
|
* be true at clear_inode() time.
|
|
|
|
*
|
|
|
|
* FIXME: clear_inode should not call invalidate_inode_buffers(). The
|
|
|
|
* filesystems should do that. invalidate_inode_buffers() should just go
|
|
|
|
* BUG_ON(!list_empty).
|
|
|
|
*
|
|
|
|
* FIXME: mark_buffer_dirty_inode() is a data-plane operation. It should
|
|
|
|
* take an address_space, not an inode. And it should be called
|
|
|
|
* mark_buffer_dirty_fsync() to clearly define why those buffers are being
|
|
|
|
* queued up.
|
|
|
|
*
|
|
|
|
* FIXME: mark_buffer_dirty_inode() doesn't need to add the buffer to the
|
|
|
|
* list if it is already on a list. Because if the buffer is on a list,
|
|
|
|
* it *must* already be on the right one. If not, the filesystem is being
|
|
|
|
* silly. This will save a ton of locking. But first we have to ensure
|
|
|
|
* that buffers are taken *off* the old inode's list when they are freed
|
|
|
|
* (presumably in truncate). That requires careful auditing of all
|
|
|
|
* filesystems (do it inside bforget()). It could also be done by bringing
|
|
|
|
* b_inode back.
|
|
|
|
*/
|
|
|
|
|
|
|
|
/*
|
2023-11-18 05:58:23 +08:00
|
|
|
* The buffer's backing address_space's i_private_lock must be held
|
2005-04-17 06:20:36 +08:00
|
|
|
*/
|
2008-07-30 13:33:47 +08:00
|
|
|
static void __remove_assoc_queue(struct buffer_head *bh)
|
2005-04-17 06:20:36 +08:00
|
|
|
{
|
|
|
|
list_del_init(&bh->b_assoc_buffers);
|
2006-10-17 15:10:19 +08:00
|
|
|
WARN_ON(!bh->b_assoc_map);
|
|
|
|
bh->b_assoc_map = NULL;
|
2005-04-17 06:20:36 +08:00
|
|
|
}
|
|
|
|
|
|
|
|
int inode_has_buffers(struct inode *inode)
|
|
|
|
{
|
2023-11-18 05:58:23 +08:00
|
|
|
return !list_empty(&inode->i_data.i_private_list);
|
2005-04-17 06:20:36 +08:00
|
|
|
}
|
|
|
|
|
|
|
|
/*
|
|
|
|
* osync is designed to support O_SYNC io. It waits synchronously for
|
|
|
|
* all already-submitted IO to complete, but does not queue any new
|
|
|
|
* writes to the disk.
|
|
|
|
*
|
2022-09-01 21:35:03 +08:00
|
|
|
* To do O_SYNC writes, just queue the buffer writes with write_dirty_buffer
|
|
|
|
* as you dirty the buffers, and then use osync_inode_buffers to wait for
|
2005-04-17 06:20:36 +08:00
|
|
|
* completion. Any other dirty buffers which are not yet queued for
|
|
|
|
* write will not be flushed to disk by the osync.
|
|
|
|
*/
|
|
|
|
static int osync_buffers_list(spinlock_t *lock, struct list_head *list)
|
|
|
|
{
|
|
|
|
struct buffer_head *bh;
|
|
|
|
struct list_head *p;
|
|
|
|
int err = 0;
|
|
|
|
|
|
|
|
spin_lock(lock);
|
|
|
|
repeat:
|
|
|
|
list_for_each_prev(p, list) {
|
|
|
|
bh = BH_ENTRY(p);
|
|
|
|
if (buffer_locked(bh)) {
|
|
|
|
get_bh(bh);
|
|
|
|
spin_unlock(lock);
|
|
|
|
wait_on_buffer(bh);
|
|
|
|
if (!buffer_uptodate(bh))
|
|
|
|
err = -EIO;
|
|
|
|
brelse(bh);
|
|
|
|
spin_lock(lock);
|
|
|
|
goto repeat;
|
|
|
|
}
|
|
|
|
}
|
|
|
|
spin_unlock(lock);
|
|
|
|
return err;
|
|
|
|
}
|
|
|
|
|
|
|
|
/**
|
2008-03-01 14:02:31 +08:00
|
|
|
* sync_mapping_buffers - write out & wait upon a mapping's "associated" buffers
|
2005-05-01 23:59:26 +08:00
|
|
|
* @mapping: the mapping which wants those buffers written
|
2005-04-17 06:20:36 +08:00
|
|
|
*
|
2023-11-18 05:58:23 +08:00
|
|
|
* Starts I/O against the buffers at mapping->i_private_list, and waits upon
|
2005-04-17 06:20:36 +08:00
|
|
|
* that I/O.
|
|
|
|
*
|
2005-05-01 23:59:26 +08:00
|
|
|
* Basically, this is a convenience function for fsync().
|
|
|
|
* @mapping is a file or directory which needs those buffers to be written for
|
|
|
|
* a successful fsync().
|
2005-04-17 06:20:36 +08:00
|
|
|
*/
|
|
|
|
int sync_mapping_buffers(struct address_space *mapping)
|
|
|
|
{
|
2023-11-18 05:58:23 +08:00
|
|
|
struct address_space *buffer_mapping = mapping->i_private_data;
|
2005-04-17 06:20:36 +08:00
|
|
|
|
2023-11-18 05:58:23 +08:00
|
|
|
if (buffer_mapping == NULL || list_empty(&mapping->i_private_list))
|
2005-04-17 06:20:36 +08:00
|
|
|
return 0;
|
|
|
|
|
2023-11-18 05:58:23 +08:00
|
|
|
return fsync_buffers_list(&buffer_mapping->i_private_lock,
|
|
|
|
&mapping->i_private_list);
|
2005-04-17 06:20:36 +08:00
|
|
|
}
|
|
|
|
EXPORT_SYMBOL(sync_mapping_buffers);
|
|
|
|
|
2023-04-21 17:46:12 +08:00
|
|
|
/**
|
|
|
|
* generic_buffers_fsync_noflush - generic buffer fsync implementation
|
|
|
|
* for simple filesystems with no inode lock
|
|
|
|
*
|
|
|
|
* @file: file to synchronize
|
|
|
|
* @start: start offset in bytes
|
|
|
|
* @end: end offset in bytes (inclusive)
|
|
|
|
* @datasync: only synchronize essential metadata if true
|
|
|
|
*
|
|
|
|
* This is a generic implementation of the fsync method for simple
|
|
|
|
* filesystems which track all non-inode metadata in the buffers list
|
|
|
|
* hanging off the address_space structure.
|
|
|
|
*/
|
|
|
|
int generic_buffers_fsync_noflush(struct file *file, loff_t start, loff_t end,
|
|
|
|
bool datasync)
|
|
|
|
{
|
|
|
|
struct inode *inode = file->f_mapping->host;
|
|
|
|
int err;
|
|
|
|
int ret;
|
|
|
|
|
|
|
|
err = file_write_and_wait_range(file, start, end);
|
|
|
|
if (err)
|
|
|
|
return err;
|
|
|
|
|
|
|
|
ret = sync_mapping_buffers(inode->i_mapping);
|
|
|
|
if (!(inode->i_state & I_DIRTY_ALL))
|
|
|
|
goto out;
|
|
|
|
if (datasync && !(inode->i_state & I_DIRTY_DATASYNC))
|
|
|
|
goto out;
|
|
|
|
|
|
|
|
err = sync_inode_metadata(inode, 1);
|
|
|
|
if (ret == 0)
|
|
|
|
ret = err;
|
|
|
|
|
|
|
|
out:
|
|
|
|
/* check and advance again to catch errors after syncing out buffers */
|
|
|
|
err = file_check_and_advance_wb_err(file);
|
|
|
|
if (ret == 0)
|
|
|
|
ret = err;
|
|
|
|
return ret;
|
|
|
|
}
|
|
|
|
EXPORT_SYMBOL(generic_buffers_fsync_noflush);
|
|
|
|
|
|
|
|
/**
|
|
|
|
* generic_buffers_fsync - generic buffer fsync implementation
|
|
|
|
* for simple filesystems with no inode lock
|
|
|
|
*
|
|
|
|
* @file: file to synchronize
|
|
|
|
* @start: start offset in bytes
|
|
|
|
* @end: end offset in bytes (inclusive)
|
|
|
|
* @datasync: only synchronize essential metadata if true
|
|
|
|
*
|
|
|
|
* This is a generic implementation of the fsync method for simple
|
|
|
|
* filesystems which track all non-inode metadata in the buffers list
|
|
|
|
* hanging off the address_space structure. This also makes sure that
|
|
|
|
* a device cache flush operation is called at the end.
|
|
|
|
*/
|
|
|
|
int generic_buffers_fsync(struct file *file, loff_t start, loff_t end,
|
|
|
|
bool datasync)
|
|
|
|
{
|
|
|
|
struct inode *inode = file->f_mapping->host;
|
|
|
|
int ret;
|
|
|
|
|
|
|
|
ret = generic_buffers_fsync_noflush(file, start, end, datasync);
|
|
|
|
if (!ret)
|
|
|
|
ret = blkdev_issue_flush(inode->i_sb->s_bdev);
|
|
|
|
return ret;
|
|
|
|
}
|
|
|
|
EXPORT_SYMBOL(generic_buffers_fsync);
|
|
|
|
|
2005-04-17 06:20:36 +08:00
|
|
|
/*
|
|
|
|
* Called when we've recently written block `bblock', and it is known that
|
|
|
|
* `bblock' was for a buffer_boundary() buffer. This means that the block at
|
|
|
|
* `bblock + 1' is probably a dirty indirect block. Hunt it down and, if it's
|
|
|
|
* dirty, schedule it for IO. So that indirects merge nicely with their data.
|
|
|
|
*/
|
|
|
|
void write_boundary_block(struct block_device *bdev,
|
|
|
|
sector_t bblock, unsigned blocksize)
|
|
|
|
{
|
|
|
|
struct buffer_head *bh = __find_get_block(bdev, bblock + 1, blocksize);
|
|
|
|
if (bh) {
|
|
|
|
if (buffer_dirty(bh))
|
2022-09-01 21:34:54 +08:00
|
|
|
write_dirty_buffer(bh, 0);
|
2005-04-17 06:20:36 +08:00
|
|
|
put_bh(bh);
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
void mark_buffer_dirty_inode(struct buffer_head *bh, struct inode *inode)
|
|
|
|
{
|
|
|
|
struct address_space *mapping = inode->i_mapping;
|
2022-12-16 05:43:52 +08:00
|
|
|
struct address_space *buffer_mapping = bh->b_folio->mapping;
|
2005-04-17 06:20:36 +08:00
|
|
|
|
|
|
|
mark_buffer_dirty(bh);
|
2023-11-18 05:58:23 +08:00
|
|
|
if (!mapping->i_private_data) {
|
|
|
|
mapping->i_private_data = buffer_mapping;
|
2005-04-17 06:20:36 +08:00
|
|
|
} else {
|
2023-11-18 05:58:23 +08:00
|
|
|
BUG_ON(mapping->i_private_data != buffer_mapping);
|
2005-04-17 06:20:36 +08:00
|
|
|
}
|
2008-02-08 20:21:59 +08:00
|
|
|
if (!bh->b_assoc_map) {
|
2023-11-18 05:58:23 +08:00
|
|
|
spin_lock(&buffer_mapping->i_private_lock);
|
2005-04-17 06:20:36 +08:00
|
|
|
list_move_tail(&bh->b_assoc_buffers,
|
2023-11-18 05:58:23 +08:00
|
|
|
&mapping->i_private_list);
|
2006-10-17 15:10:19 +08:00
|
|
|
bh->b_assoc_map = mapping;
|
2023-11-18 05:58:23 +08:00
|
|
|
spin_unlock(&buffer_mapping->i_private_lock);
|
2005-04-17 06:20:36 +08:00
|
|
|
}
|
|
|
|
}
|
|
|
|
EXPORT_SYMBOL(mark_buffer_dirty_inode);
|
|
|
|
|
|
|
|
/*
|
|
|
|
* Add a page to the dirty page list.
|
|
|
|
*
|
|
|
|
* It is a sad fact of life that this function is called from several places
|
|
|
|
* deeply under spinlocking. It may not sleep.
|
|
|
|
*
|
|
|
|
* If the page has buffers, the uptodate buffers are set dirty, to preserve
|
|
|
|
* dirty-state coherency between the page and the buffers. It the page does
|
|
|
|
* not have buffers then when they are later attached they will all be set
|
|
|
|
* dirty.
|
|
|
|
*
|
|
|
|
* The buffers are dirtied before the page is dirtied. There's a small race
|
|
|
|
* window in which a writepage caller may see the page cleanness but not the
|
|
|
|
* buffer dirtiness. That's fine. If this code were to set the page dirty
|
|
|
|
* before the buffers, a concurrent writepage caller could clear the page dirty
|
|
|
|
* bit, see a bunch of clean buffers and we'd end up with dirty buffers/clean
|
|
|
|
* page on the dirty page list.
|
|
|
|
*
|
2023-11-18 05:58:23 +08:00
|
|
|
* We use i_private_lock to lock against try_to_free_buffers while using the
|
2005-04-17 06:20:36 +08:00
|
|
|
* page's buffer list. Also use this to protect against clean buffers being
|
|
|
|
* added to the page after it was set dirty.
|
|
|
|
*
|
|
|
|
* FIXME: may need to call ->reservepage here as well. That's rather up to the
|
|
|
|
* address_space though.
|
|
|
|
*/
|
2022-02-10 04:22:12 +08:00
|
|
|
bool block_dirty_folio(struct address_space *mapping, struct folio *folio)
|
2005-04-17 06:20:36 +08:00
|
|
|
{
|
2022-02-10 04:22:12 +08:00
|
|
|
struct buffer_head *head;
|
|
|
|
bool newly_dirty;
|
2005-04-17 06:20:36 +08:00
|
|
|
|
2023-11-18 05:58:23 +08:00
|
|
|
spin_lock(&mapping->i_private_lock);
|
2022-02-10 04:22:12 +08:00
|
|
|
head = folio_buffers(folio);
|
|
|
|
if (head) {
|
2005-04-17 06:20:36 +08:00
|
|
|
struct buffer_head *bh = head;
|
|
|
|
|
|
|
|
do {
|
|
|
|
set_buffer_dirty(bh);
|
|
|
|
bh = bh->b_this_page;
|
|
|
|
} while (bh != head);
|
|
|
|
}
|
memcg: add per cgroup dirty page accounting
When modifying PG_Dirty on cached file pages, update the new
MEM_CGROUP_STAT_DIRTY counter. This is done in the same places where
global NR_FILE_DIRTY is managed. The new memcg stat is visible in the
per memcg memory.stat cgroupfs file. The most recent past attempt at
this was http://thread.gmane.org/gmane.linux.kernel.cgroups/8632
The new accounting supports future efforts to add per cgroup dirty
page throttling and writeback. It also helps an administrator break
down a container's memory usage and provides evidence to understand
memcg oom kills (the new dirty count is included in memcg oom kill
messages).
The ability to move page accounting between memcg
(memory.move_charge_at_immigrate) makes this accounting more
complicated than the global counter. The existing
mem_cgroup_{begin,end}_page_stat() lock is used to serialize move
accounting with stat updates.
Typical update operation:
memcg = mem_cgroup_begin_page_stat(page)
if (TestSetPageDirty()) {
[...]
mem_cgroup_update_page_stat(memcg)
}
mem_cgroup_end_page_stat(memcg)
Summary of mem_cgroup_end_page_stat() overhead:
- Without CONFIG_MEMCG it's a no-op
- With CONFIG_MEMCG and no inter memcg task movement, it's just
rcu_read_lock()
- With CONFIG_MEMCG and inter memcg task movement, it's
rcu_read_lock() + spin_lock_irqsave()
A memcg parameter is added to several routines because their callers
now grab mem_cgroup_begin_page_stat() which returns the memcg later
needed by for mem_cgroup_update_page_stat().
Because mem_cgroup_begin_page_stat() may disable interrupts, some
adjustments are needed:
- move __mark_inode_dirty() from __set_page_dirty() to its caller.
__mark_inode_dirty() locking does not want interrupts disabled.
- use spin_lock_irqsave(tree_lock) rather than spin_lock_irq() in
__delete_from_page_cache(), replace_page_cache_page(),
invalidate_complete_page2(), and __remove_mapping().
text data bss dec hex filename
8925147 1774832 1785856 12485835 be84cb vmlinux-!CONFIG_MEMCG-before
8925339 1774832 1785856 12486027 be858b vmlinux-!CONFIG_MEMCG-after
+192 text bytes
8965977 1784992 1785856 12536825 bf4bf9 vmlinux-CONFIG_MEMCG-before
8966750 1784992 1785856 12537598 bf4efe vmlinux-CONFIG_MEMCG-after
+773 text bytes
Performance tests run on v4.0-rc1-36-g4f671fe2f952. Lower is better for
all metrics, they're all wall clock or cycle counts. The read and write
fault benchmarks just measure fault time, they do not include I/O time.
* CONFIG_MEMCG not set:
baseline patched
kbuild 1m25.030000(+-0.088% 3 samples) 1m25.426667(+-0.120% 3 samples)
dd write 100 MiB 0.859211561 +-15.10% 0.874162885 +-15.03%
dd write 200 MiB 1.670653105 +-17.87% 1.669384764 +-11.99%
dd write 1000 MiB 8.434691190 +-14.15% 8.474733215 +-14.77%
read fault cycles 254.0(+-0.000% 10 samples) 253.0(+-0.000% 10 samples)
write fault cycles 2021.2(+-3.070% 10 samples) 1984.5(+-1.036% 10 samples)
* CONFIG_MEMCG=y root_memcg:
baseline patched
kbuild 1m25.716667(+-0.105% 3 samples) 1m25.686667(+-0.153% 3 samples)
dd write 100 MiB 0.855650830 +-14.90% 0.887557919 +-14.90%
dd write 200 MiB 1.688322953 +-12.72% 1.667682724 +-13.33%
dd write 1000 MiB 8.418601605 +-14.30% 8.673532299 +-15.00%
read fault cycles 266.0(+-0.000% 10 samples) 266.0(+-0.000% 10 samples)
write fault cycles 2051.7(+-1.349% 10 samples) 2049.6(+-1.686% 10 samples)
* CONFIG_MEMCG=y non-root_memcg:
baseline patched
kbuild 1m26.120000(+-0.273% 3 samples) 1m25.763333(+-0.127% 3 samples)
dd write 100 MiB 0.861723964 +-15.25% 0.818129350 +-14.82%
dd write 200 MiB 1.669887569 +-13.30% 1.698645885 +-13.27%
dd write 1000 MiB 8.383191730 +-14.65% 8.351742280 +-14.52%
read fault cycles 265.7(+-0.172% 10 samples) 267.0(+-0.000% 10 samples)
write fault cycles 2070.6(+-1.512% 10 samples) 2084.4(+-2.148% 10 samples)
As expected anon page faults are not affected by this patch.
tj: Updated to apply on top of the recent cancel_dirty_page() changes.
Signed-off-by: Sha Zhengju <handai.szj@gmail.com>
Signed-off-by: Greg Thelen <gthelen@google.com>
Signed-off-by: Tejun Heo <tj@kernel.org>
Signed-off-by: Jens Axboe <axboe@fb.com>
2015-05-23 05:13:16 +08:00
|
|
|
/*
|
mm: memcontrol: Use helpers to read page's memcg data
Patch series "mm: allow mapping accounted kernel pages to userspace", v6.
Currently a non-slab kernel page which has been charged to a memory cgroup
can't be mapped to userspace. The underlying reason is simple: PageKmemcg
flag is defined as a page type (like buddy, offline, etc), so it takes a
bit from a page->mapped counter. Pages with a type set can't be mapped to
userspace.
But in general the kmemcg flag has nothing to do with mapping to
userspace. It only means that the page has been accounted by the page
allocator, so it has to be properly uncharged on release.
Some bpf maps are mapping the vmalloc-based memory to userspace, and their
memory can't be accounted because of this implementation detail.
This patchset removes this limitation by moving the PageKmemcg flag into
one of the free bits of the page->mem_cgroup pointer. Also it formalizes
accesses to the page->mem_cgroup and page->obj_cgroups using new helpers,
adds several checks and removes a couple of obsolete functions. As the
result the code became more robust with fewer open-coded bit tricks.
This patch (of 4):
Currently there are many open-coded reads of the page->mem_cgroup pointer,
as well as a couple of read helpers, which are barely used.
It creates an obstacle on a way to reuse some bits of the pointer for
storing additional bits of information. In fact, we already do this for
slab pages, where the last bit indicates that a pointer has an attached
vector of objcg pointers instead of a regular memcg pointer.
This commits uses 2 existing helpers and introduces a new helper to
converts all read sides to calls of these helpers:
struct mem_cgroup *page_memcg(struct page *page);
struct mem_cgroup *page_memcg_rcu(struct page *page);
struct mem_cgroup *page_memcg_check(struct page *page);
page_memcg_check() is intended to be used in cases when the page can be a
slab page and have a memcg pointer pointing at objcg vector. It does
check the lowest bit, and if set, returns NULL. page_memcg() contains a
VM_BUG_ON_PAGE() check for the page not being a slab page.
To make sure nobody uses a direct access, struct page's
mem_cgroup/obj_cgroups is converted to unsigned long memcg_data.
Signed-off-by: Roman Gushchin <guro@fb.com>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
Signed-off-by: Alexei Starovoitov <ast@kernel.org>
Reviewed-by: Shakeel Butt <shakeelb@google.com>
Acked-by: Johannes Weiner <hannes@cmpxchg.org>
Acked-by: Michal Hocko <mhocko@suse.com>
Link: https://lkml.kernel.org/r/20201027001657.3398190-1-guro@fb.com
Link: https://lkml.kernel.org/r/20201027001657.3398190-2-guro@fb.com
Link: https://lore.kernel.org/bpf/20201201215900.3569844-2-guro@fb.com
2020-12-02 05:58:27 +08:00
|
|
|
* Lock out page's memcg migration to keep PageDirty
|
2016-03-16 05:57:04 +08:00
|
|
|
* synchronized with per-memcg dirty page counters.
|
memcg: add per cgroup dirty page accounting
When modifying PG_Dirty on cached file pages, update the new
MEM_CGROUP_STAT_DIRTY counter. This is done in the same places where
global NR_FILE_DIRTY is managed. The new memcg stat is visible in the
per memcg memory.stat cgroupfs file. The most recent past attempt at
this was http://thread.gmane.org/gmane.linux.kernel.cgroups/8632
The new accounting supports future efforts to add per cgroup dirty
page throttling and writeback. It also helps an administrator break
down a container's memory usage and provides evidence to understand
memcg oom kills (the new dirty count is included in memcg oom kill
messages).
The ability to move page accounting between memcg
(memory.move_charge_at_immigrate) makes this accounting more
complicated than the global counter. The existing
mem_cgroup_{begin,end}_page_stat() lock is used to serialize move
accounting with stat updates.
Typical update operation:
memcg = mem_cgroup_begin_page_stat(page)
if (TestSetPageDirty()) {
[...]
mem_cgroup_update_page_stat(memcg)
}
mem_cgroup_end_page_stat(memcg)
Summary of mem_cgroup_end_page_stat() overhead:
- Without CONFIG_MEMCG it's a no-op
- With CONFIG_MEMCG and no inter memcg task movement, it's just
rcu_read_lock()
- With CONFIG_MEMCG and inter memcg task movement, it's
rcu_read_lock() + spin_lock_irqsave()
A memcg parameter is added to several routines because their callers
now grab mem_cgroup_begin_page_stat() which returns the memcg later
needed by for mem_cgroup_update_page_stat().
Because mem_cgroup_begin_page_stat() may disable interrupts, some
adjustments are needed:
- move __mark_inode_dirty() from __set_page_dirty() to its caller.
__mark_inode_dirty() locking does not want interrupts disabled.
- use spin_lock_irqsave(tree_lock) rather than spin_lock_irq() in
__delete_from_page_cache(), replace_page_cache_page(),
invalidate_complete_page2(), and __remove_mapping().
text data bss dec hex filename
8925147 1774832 1785856 12485835 be84cb vmlinux-!CONFIG_MEMCG-before
8925339 1774832 1785856 12486027 be858b vmlinux-!CONFIG_MEMCG-after
+192 text bytes
8965977 1784992 1785856 12536825 bf4bf9 vmlinux-CONFIG_MEMCG-before
8966750 1784992 1785856 12537598 bf4efe vmlinux-CONFIG_MEMCG-after
+773 text bytes
Performance tests run on v4.0-rc1-36-g4f671fe2f952. Lower is better for
all metrics, they're all wall clock or cycle counts. The read and write
fault benchmarks just measure fault time, they do not include I/O time.
* CONFIG_MEMCG not set:
baseline patched
kbuild 1m25.030000(+-0.088% 3 samples) 1m25.426667(+-0.120% 3 samples)
dd write 100 MiB 0.859211561 +-15.10% 0.874162885 +-15.03%
dd write 200 MiB 1.670653105 +-17.87% 1.669384764 +-11.99%
dd write 1000 MiB 8.434691190 +-14.15% 8.474733215 +-14.77%
read fault cycles 254.0(+-0.000% 10 samples) 253.0(+-0.000% 10 samples)
write fault cycles 2021.2(+-3.070% 10 samples) 1984.5(+-1.036% 10 samples)
* CONFIG_MEMCG=y root_memcg:
baseline patched
kbuild 1m25.716667(+-0.105% 3 samples) 1m25.686667(+-0.153% 3 samples)
dd write 100 MiB 0.855650830 +-14.90% 0.887557919 +-14.90%
dd write 200 MiB 1.688322953 +-12.72% 1.667682724 +-13.33%
dd write 1000 MiB 8.418601605 +-14.30% 8.673532299 +-15.00%
read fault cycles 266.0(+-0.000% 10 samples) 266.0(+-0.000% 10 samples)
write fault cycles 2051.7(+-1.349% 10 samples) 2049.6(+-1.686% 10 samples)
* CONFIG_MEMCG=y non-root_memcg:
baseline patched
kbuild 1m26.120000(+-0.273% 3 samples) 1m25.763333(+-0.127% 3 samples)
dd write 100 MiB 0.861723964 +-15.25% 0.818129350 +-14.82%
dd write 200 MiB 1.669887569 +-13.30% 1.698645885 +-13.27%
dd write 1000 MiB 8.383191730 +-14.65% 8.351742280 +-14.52%
read fault cycles 265.7(+-0.172% 10 samples) 267.0(+-0.000% 10 samples)
write fault cycles 2070.6(+-1.512% 10 samples) 2084.4(+-2.148% 10 samples)
As expected anon page faults are not affected by this patch.
tj: Updated to apply on top of the recent cancel_dirty_page() changes.
Signed-off-by: Sha Zhengju <handai.szj@gmail.com>
Signed-off-by: Greg Thelen <gthelen@google.com>
Signed-off-by: Tejun Heo <tj@kernel.org>
Signed-off-by: Jens Axboe <axboe@fb.com>
2015-05-23 05:13:16 +08:00
|
|
|
*/
|
2022-02-10 04:22:12 +08:00
|
|
|
folio_memcg_lock(folio);
|
|
|
|
newly_dirty = !folio_test_set_dirty(folio);
|
2023-11-18 05:58:23 +08:00
|
|
|
spin_unlock(&mapping->i_private_lock);
|
2005-04-17 06:20:36 +08:00
|
|
|
|
2009-03-20 02:32:05 +08:00
|
|
|
if (newly_dirty)
|
2022-02-10 04:22:12 +08:00
|
|
|
__folio_mark_dirty(folio, mapping, 1);
|
memcg: add per cgroup dirty page accounting
When modifying PG_Dirty on cached file pages, update the new
MEM_CGROUP_STAT_DIRTY counter. This is done in the same places where
global NR_FILE_DIRTY is managed. The new memcg stat is visible in the
per memcg memory.stat cgroupfs file. The most recent past attempt at
this was http://thread.gmane.org/gmane.linux.kernel.cgroups/8632
The new accounting supports future efforts to add per cgroup dirty
page throttling and writeback. It also helps an administrator break
down a container's memory usage and provides evidence to understand
memcg oom kills (the new dirty count is included in memcg oom kill
messages).
The ability to move page accounting between memcg
(memory.move_charge_at_immigrate) makes this accounting more
complicated than the global counter. The existing
mem_cgroup_{begin,end}_page_stat() lock is used to serialize move
accounting with stat updates.
Typical update operation:
memcg = mem_cgroup_begin_page_stat(page)
if (TestSetPageDirty()) {
[...]
mem_cgroup_update_page_stat(memcg)
}
mem_cgroup_end_page_stat(memcg)
Summary of mem_cgroup_end_page_stat() overhead:
- Without CONFIG_MEMCG it's a no-op
- With CONFIG_MEMCG and no inter memcg task movement, it's just
rcu_read_lock()
- With CONFIG_MEMCG and inter memcg task movement, it's
rcu_read_lock() + spin_lock_irqsave()
A memcg parameter is added to several routines because their callers
now grab mem_cgroup_begin_page_stat() which returns the memcg later
needed by for mem_cgroup_update_page_stat().
Because mem_cgroup_begin_page_stat() may disable interrupts, some
adjustments are needed:
- move __mark_inode_dirty() from __set_page_dirty() to its caller.
__mark_inode_dirty() locking does not want interrupts disabled.
- use spin_lock_irqsave(tree_lock) rather than spin_lock_irq() in
__delete_from_page_cache(), replace_page_cache_page(),
invalidate_complete_page2(), and __remove_mapping().
text data bss dec hex filename
8925147 1774832 1785856 12485835 be84cb vmlinux-!CONFIG_MEMCG-before
8925339 1774832 1785856 12486027 be858b vmlinux-!CONFIG_MEMCG-after
+192 text bytes
8965977 1784992 1785856 12536825 bf4bf9 vmlinux-CONFIG_MEMCG-before
8966750 1784992 1785856 12537598 bf4efe vmlinux-CONFIG_MEMCG-after
+773 text bytes
Performance tests run on v4.0-rc1-36-g4f671fe2f952. Lower is better for
all metrics, they're all wall clock or cycle counts. The read and write
fault benchmarks just measure fault time, they do not include I/O time.
* CONFIG_MEMCG not set:
baseline patched
kbuild 1m25.030000(+-0.088% 3 samples) 1m25.426667(+-0.120% 3 samples)
dd write 100 MiB 0.859211561 +-15.10% 0.874162885 +-15.03%
dd write 200 MiB 1.670653105 +-17.87% 1.669384764 +-11.99%
dd write 1000 MiB 8.434691190 +-14.15% 8.474733215 +-14.77%
read fault cycles 254.0(+-0.000% 10 samples) 253.0(+-0.000% 10 samples)
write fault cycles 2021.2(+-3.070% 10 samples) 1984.5(+-1.036% 10 samples)
* CONFIG_MEMCG=y root_memcg:
baseline patched
kbuild 1m25.716667(+-0.105% 3 samples) 1m25.686667(+-0.153% 3 samples)
dd write 100 MiB 0.855650830 +-14.90% 0.887557919 +-14.90%
dd write 200 MiB 1.688322953 +-12.72% 1.667682724 +-13.33%
dd write 1000 MiB 8.418601605 +-14.30% 8.673532299 +-15.00%
read fault cycles 266.0(+-0.000% 10 samples) 266.0(+-0.000% 10 samples)
write fault cycles 2051.7(+-1.349% 10 samples) 2049.6(+-1.686% 10 samples)
* CONFIG_MEMCG=y non-root_memcg:
baseline patched
kbuild 1m26.120000(+-0.273% 3 samples) 1m25.763333(+-0.127% 3 samples)
dd write 100 MiB 0.861723964 +-15.25% 0.818129350 +-14.82%
dd write 200 MiB 1.669887569 +-13.30% 1.698645885 +-13.27%
dd write 1000 MiB 8.383191730 +-14.65% 8.351742280 +-14.52%
read fault cycles 265.7(+-0.172% 10 samples) 267.0(+-0.000% 10 samples)
write fault cycles 2070.6(+-1.512% 10 samples) 2084.4(+-2.148% 10 samples)
As expected anon page faults are not affected by this patch.
tj: Updated to apply on top of the recent cancel_dirty_page() changes.
Signed-off-by: Sha Zhengju <handai.szj@gmail.com>
Signed-off-by: Greg Thelen <gthelen@google.com>
Signed-off-by: Tejun Heo <tj@kernel.org>
Signed-off-by: Jens Axboe <axboe@fb.com>
2015-05-23 05:13:16 +08:00
|
|
|
|
2022-02-10 04:22:12 +08:00
|
|
|
folio_memcg_unlock(folio);
|
memcg: add per cgroup dirty page accounting
When modifying PG_Dirty on cached file pages, update the new
MEM_CGROUP_STAT_DIRTY counter. This is done in the same places where
global NR_FILE_DIRTY is managed. The new memcg stat is visible in the
per memcg memory.stat cgroupfs file. The most recent past attempt at
this was http://thread.gmane.org/gmane.linux.kernel.cgroups/8632
The new accounting supports future efforts to add per cgroup dirty
page throttling and writeback. It also helps an administrator break
down a container's memory usage and provides evidence to understand
memcg oom kills (the new dirty count is included in memcg oom kill
messages).
The ability to move page accounting between memcg
(memory.move_charge_at_immigrate) makes this accounting more
complicated than the global counter. The existing
mem_cgroup_{begin,end}_page_stat() lock is used to serialize move
accounting with stat updates.
Typical update operation:
memcg = mem_cgroup_begin_page_stat(page)
if (TestSetPageDirty()) {
[...]
mem_cgroup_update_page_stat(memcg)
}
mem_cgroup_end_page_stat(memcg)
Summary of mem_cgroup_end_page_stat() overhead:
- Without CONFIG_MEMCG it's a no-op
- With CONFIG_MEMCG and no inter memcg task movement, it's just
rcu_read_lock()
- With CONFIG_MEMCG and inter memcg task movement, it's
rcu_read_lock() + spin_lock_irqsave()
A memcg parameter is added to several routines because their callers
now grab mem_cgroup_begin_page_stat() which returns the memcg later
needed by for mem_cgroup_update_page_stat().
Because mem_cgroup_begin_page_stat() may disable interrupts, some
adjustments are needed:
- move __mark_inode_dirty() from __set_page_dirty() to its caller.
__mark_inode_dirty() locking does not want interrupts disabled.
- use spin_lock_irqsave(tree_lock) rather than spin_lock_irq() in
__delete_from_page_cache(), replace_page_cache_page(),
invalidate_complete_page2(), and __remove_mapping().
text data bss dec hex filename
8925147 1774832 1785856 12485835 be84cb vmlinux-!CONFIG_MEMCG-before
8925339 1774832 1785856 12486027 be858b vmlinux-!CONFIG_MEMCG-after
+192 text bytes
8965977 1784992 1785856 12536825 bf4bf9 vmlinux-CONFIG_MEMCG-before
8966750 1784992 1785856 12537598 bf4efe vmlinux-CONFIG_MEMCG-after
+773 text bytes
Performance tests run on v4.0-rc1-36-g4f671fe2f952. Lower is better for
all metrics, they're all wall clock or cycle counts. The read and write
fault benchmarks just measure fault time, they do not include I/O time.
* CONFIG_MEMCG not set:
baseline patched
kbuild 1m25.030000(+-0.088% 3 samples) 1m25.426667(+-0.120% 3 samples)
dd write 100 MiB 0.859211561 +-15.10% 0.874162885 +-15.03%
dd write 200 MiB 1.670653105 +-17.87% 1.669384764 +-11.99%
dd write 1000 MiB 8.434691190 +-14.15% 8.474733215 +-14.77%
read fault cycles 254.0(+-0.000% 10 samples) 253.0(+-0.000% 10 samples)
write fault cycles 2021.2(+-3.070% 10 samples) 1984.5(+-1.036% 10 samples)
* CONFIG_MEMCG=y root_memcg:
baseline patched
kbuild 1m25.716667(+-0.105% 3 samples) 1m25.686667(+-0.153% 3 samples)
dd write 100 MiB 0.855650830 +-14.90% 0.887557919 +-14.90%
dd write 200 MiB 1.688322953 +-12.72% 1.667682724 +-13.33%
dd write 1000 MiB 8.418601605 +-14.30% 8.673532299 +-15.00%
read fault cycles 266.0(+-0.000% 10 samples) 266.0(+-0.000% 10 samples)
write fault cycles 2051.7(+-1.349% 10 samples) 2049.6(+-1.686% 10 samples)
* CONFIG_MEMCG=y non-root_memcg:
baseline patched
kbuild 1m26.120000(+-0.273% 3 samples) 1m25.763333(+-0.127% 3 samples)
dd write 100 MiB 0.861723964 +-15.25% 0.818129350 +-14.82%
dd write 200 MiB 1.669887569 +-13.30% 1.698645885 +-13.27%
dd write 1000 MiB 8.383191730 +-14.65% 8.351742280 +-14.52%
read fault cycles 265.7(+-0.172% 10 samples) 267.0(+-0.000% 10 samples)
write fault cycles 2070.6(+-1.512% 10 samples) 2084.4(+-2.148% 10 samples)
As expected anon page faults are not affected by this patch.
tj: Updated to apply on top of the recent cancel_dirty_page() changes.
Signed-off-by: Sha Zhengju <handai.szj@gmail.com>
Signed-off-by: Greg Thelen <gthelen@google.com>
Signed-off-by: Tejun Heo <tj@kernel.org>
Signed-off-by: Jens Axboe <axboe@fb.com>
2015-05-23 05:13:16 +08:00
|
|
|
|
|
|
|
if (newly_dirty)
|
|
|
|
__mark_inode_dirty(mapping->host, I_DIRTY_PAGES);
|
|
|
|
|
2009-03-20 02:32:05 +08:00
|
|
|
return newly_dirty;
|
2005-04-17 06:20:36 +08:00
|
|
|
}
|
2022-02-10 04:22:12 +08:00
|
|
|
EXPORT_SYMBOL(block_dirty_folio);
|
2005-04-17 06:20:36 +08:00
|
|
|
|
|
|
|
/*
|
|
|
|
* Write out and wait upon a list of buffers.
|
|
|
|
*
|
|
|
|
* We have conflicting pressures: we want to make sure that all
|
|
|
|
* initially dirty buffers get waited on, but that any subsequently
|
|
|
|
* dirtied buffers don't. After all, we don't want fsync to last
|
|
|
|
* forever if somebody is actively writing to the file.
|
|
|
|
*
|
|
|
|
* Do this in two main stages: first we copy dirty buffers to a
|
|
|
|
* temporary inode list, queueing the writes as we go. Then we clean
|
|
|
|
* up, waiting for those writes to complete.
|
|
|
|
*
|
|
|
|
* During this second stage, any subsequent updates to the file may end
|
|
|
|
* up refiling the buffer on the original inode's dirty list again, so
|
|
|
|
* there is a chance we will end up with a buffer queued for write but
|
|
|
|
* not yet completed on that list. So, as a final cleanup we go through
|
|
|
|
* the osync code to catch these locked, dirty buffers without requeuing
|
|
|
|
* any newly dirty buffers for write.
|
|
|
|
*/
|
|
|
|
static int fsync_buffers_list(spinlock_t *lock, struct list_head *list)
|
|
|
|
{
|
|
|
|
struct buffer_head *bh;
|
|
|
|
struct list_head tmp;
|
2011-03-10 15:52:07 +08:00
|
|
|
struct address_space *mapping;
|
2005-04-17 06:20:36 +08:00
|
|
|
int err = 0, err2;
|
2011-03-17 17:51:40 +08:00
|
|
|
struct blk_plug plug;
|
2005-04-17 06:20:36 +08:00
|
|
|
|
|
|
|
INIT_LIST_HEAD(&tmp);
|
2011-03-17 17:51:40 +08:00
|
|
|
blk_start_plug(&plug);
|
2005-04-17 06:20:36 +08:00
|
|
|
|
|
|
|
spin_lock(lock);
|
|
|
|
while (!list_empty(list)) {
|
|
|
|
bh = BH_ENTRY(list->next);
|
2008-02-08 20:21:59 +08:00
|
|
|
mapping = bh->b_assoc_map;
|
2006-10-17 15:10:19 +08:00
|
|
|
__remove_assoc_queue(bh);
|
2008-02-08 20:21:59 +08:00
|
|
|
/* Avoid race with mark_buffer_dirty_inode() which does
|
|
|
|
* a lockless check and we rely on seeing the dirty bit */
|
|
|
|
smp_mb();
|
2005-04-17 06:20:36 +08:00
|
|
|
if (buffer_dirty(bh) || buffer_locked(bh)) {
|
|
|
|
list_add(&bh->b_assoc_buffers, &tmp);
|
2008-02-08 20:21:59 +08:00
|
|
|
bh->b_assoc_map = mapping;
|
2005-04-17 06:20:36 +08:00
|
|
|
if (buffer_dirty(bh)) {
|
|
|
|
get_bh(bh);
|
|
|
|
spin_unlock(lock);
|
|
|
|
/*
|
|
|
|
* Ensure any pending I/O completes so that
|
2010-08-11 23:06:24 +08:00
|
|
|
* write_dirty_buffer() actually writes the
|
|
|
|
* current contents - it is a noop if I/O is
|
|
|
|
* still in flight on potentially older
|
|
|
|
* contents.
|
2005-04-17 06:20:36 +08:00
|
|
|
*/
|
2016-11-01 21:40:10 +08:00
|
|
|
write_dirty_buffer(bh, REQ_SYNC);
|
2009-04-06 20:48:03 +08:00
|
|
|
|
|
|
|
/*
|
|
|
|
* Kick off IO for the previous mapping. Note
|
|
|
|
* that we will not run the very last mapping,
|
|
|
|
* wait_on_buffer() will do that for us
|
|
|
|
* through sync_buffer().
|
|
|
|
*/
|
2005-04-17 06:20:36 +08:00
|
|
|
brelse(bh);
|
|
|
|
spin_lock(lock);
|
|
|
|
}
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2011-03-17 17:51:40 +08:00
|
|
|
spin_unlock(lock);
|
|
|
|
blk_finish_plug(&plug);
|
|
|
|
spin_lock(lock);
|
|
|
|
|
2005-04-17 06:20:36 +08:00
|
|
|
while (!list_empty(&tmp)) {
|
|
|
|
bh = BH_ENTRY(tmp.prev);
|
|
|
|
get_bh(bh);
|
2008-02-08 20:21:59 +08:00
|
|
|
mapping = bh->b_assoc_map;
|
|
|
|
__remove_assoc_queue(bh);
|
|
|
|
/* Avoid race with mark_buffer_dirty_inode() which does
|
|
|
|
* a lockless check and we rely on seeing the dirty bit */
|
|
|
|
smp_mb();
|
|
|
|
if (buffer_dirty(bh)) {
|
|
|
|
list_add(&bh->b_assoc_buffers,
|
2023-11-18 05:58:23 +08:00
|
|
|
&mapping->i_private_list);
|
2008-02-08 20:21:59 +08:00
|
|
|
bh->b_assoc_map = mapping;
|
|
|
|
}
|
2005-04-17 06:20:36 +08:00
|
|
|
spin_unlock(lock);
|
|
|
|
wait_on_buffer(bh);
|
|
|
|
if (!buffer_uptodate(bh))
|
|
|
|
err = -EIO;
|
|
|
|
brelse(bh);
|
|
|
|
spin_lock(lock);
|
|
|
|
}
|
|
|
|
|
|
|
|
spin_unlock(lock);
|
|
|
|
err2 = osync_buffers_list(lock, list);
|
|
|
|
if (err)
|
|
|
|
return err;
|
|
|
|
else
|
|
|
|
return err2;
|
|
|
|
}
|
|
|
|
|
|
|
|
/*
|
|
|
|
* Invalidate any and all dirty buffers on a given inode. We are
|
|
|
|
* probably unmounting the fs, but that doesn't mean we have already
|
|
|
|
* done a sync(). Just drop the buffers from the inode list.
|
|
|
|
*
|
2023-11-18 05:58:23 +08:00
|
|
|
* NOTE: we take the inode's blockdev's mapping's i_private_lock. Which
|
2005-04-17 06:20:36 +08:00
|
|
|
* assumes that all the buffers are against the blockdev. Not true
|
|
|
|
* for reiserfs.
|
|
|
|
*/
|
|
|
|
void invalidate_inode_buffers(struct inode *inode)
|
|
|
|
{
|
|
|
|
if (inode_has_buffers(inode)) {
|
|
|
|
struct address_space *mapping = &inode->i_data;
|
2023-11-18 05:58:23 +08:00
|
|
|
struct list_head *list = &mapping->i_private_list;
|
|
|
|
struct address_space *buffer_mapping = mapping->i_private_data;
|
2005-04-17 06:20:36 +08:00
|
|
|
|
2023-11-18 05:58:23 +08:00
|
|
|
spin_lock(&buffer_mapping->i_private_lock);
|
2005-04-17 06:20:36 +08:00
|
|
|
while (!list_empty(list))
|
|
|
|
__remove_assoc_queue(BH_ENTRY(list->next));
|
2023-11-18 05:58:23 +08:00
|
|
|
spin_unlock(&buffer_mapping->i_private_lock);
|
2005-04-17 06:20:36 +08:00
|
|
|
}
|
|
|
|
}
|
2008-09-24 00:24:08 +08:00
|
|
|
EXPORT_SYMBOL(invalidate_inode_buffers);
|
2005-04-17 06:20:36 +08:00
|
|
|
|
|
|
|
/*
|
|
|
|
* Remove any clean buffers from the inode's buffer list. This is called
|
|
|
|
* when we're trying to free the inode itself. Those buffers can pin it.
|
|
|
|
*
|
|
|
|
* Returns true if all buffers were removed.
|
|
|
|
*/
|
|
|
|
int remove_inode_buffers(struct inode *inode)
|
|
|
|
{
|
|
|
|
int ret = 1;
|
|
|
|
|
|
|
|
if (inode_has_buffers(inode)) {
|
|
|
|
struct address_space *mapping = &inode->i_data;
|
2023-11-18 05:58:23 +08:00
|
|
|
struct list_head *list = &mapping->i_private_list;
|
|
|
|
struct address_space *buffer_mapping = mapping->i_private_data;
|
2005-04-17 06:20:36 +08:00
|
|
|
|
2023-11-18 05:58:23 +08:00
|
|
|
spin_lock(&buffer_mapping->i_private_lock);
|
2005-04-17 06:20:36 +08:00
|
|
|
while (!list_empty(list)) {
|
|
|
|
struct buffer_head *bh = BH_ENTRY(list->next);
|
|
|
|
if (buffer_dirty(bh)) {
|
|
|
|
ret = 0;
|
|
|
|
break;
|
|
|
|
}
|
|
|
|
__remove_assoc_queue(bh);
|
|
|
|
}
|
2023-11-18 05:58:23 +08:00
|
|
|
spin_unlock(&buffer_mapping->i_private_lock);
|
2005-04-17 06:20:36 +08:00
|
|
|
}
|
|
|
|
return ret;
|
|
|
|
}
|
|
|
|
|
|
|
|
/*
|
2023-04-17 20:36:16 +08:00
|
|
|
* Create the appropriate buffers when given a folio for data area and
|
2005-04-17 06:20:36 +08:00
|
|
|
* the size of each buffer.. Use the bh->b_this_page linked list to
|
|
|
|
* follow the buffers created. Return NULL if unable to create more
|
|
|
|
* buffers.
|
|
|
|
*
|
|
|
|
* The retry flag is used to differentiate async IO (paging, swapping)
|
|
|
|
* which may not fail from ordinary buffer allocations.
|
|
|
|
*/
|
2023-04-17 20:36:16 +08:00
|
|
|
struct buffer_head *folio_alloc_buffers(struct folio *folio, unsigned long size,
|
2023-09-14 23:00:04 +08:00
|
|
|
gfp_t gfp)
|
2005-04-17 06:20:36 +08:00
|
|
|
{
|
|
|
|
struct buffer_head *bh, *head;
|
|
|
|
long offset;
|
2020-10-18 07:13:40 +08:00
|
|
|
struct mem_cgroup *memcg, *old_memcg;
|
2005-04-17 06:20:36 +08:00
|
|
|
|
2023-04-17 20:36:16 +08:00
|
|
|
/* The folio lock pins the memcg */
|
|
|
|
memcg = folio_memcg(folio);
|
2020-10-18 07:13:40 +08:00
|
|
|
old_memcg = set_active_memcg(memcg);
|
2018-08-18 06:46:44 +08:00
|
|
|
|
2005-04-17 06:20:36 +08:00
|
|
|
head = NULL;
|
2023-04-17 20:36:16 +08:00
|
|
|
offset = folio_size(folio);
|
2005-04-17 06:20:36 +08:00
|
|
|
while ((offset -= size) >= 0) {
|
2017-09-27 19:40:16 +08:00
|
|
|
bh = alloc_buffer_head(gfp);
|
2005-04-17 06:20:36 +08:00
|
|
|
if (!bh)
|
|
|
|
goto no_grow;
|
|
|
|
|
|
|
|
bh->b_this_page = head;
|
|
|
|
bh->b_blocknr = -1;
|
|
|
|
head = bh;
|
|
|
|
|
|
|
|
bh->b_size = size;
|
|
|
|
|
2023-04-17 20:36:16 +08:00
|
|
|
/* Link the buffer to its folio */
|
|
|
|
folio_set_bh(bh, folio, offset);
|
2005-04-17 06:20:36 +08:00
|
|
|
}
|
2018-08-18 06:46:44 +08:00
|
|
|
out:
|
2020-10-18 07:13:40 +08:00
|
|
|
set_active_memcg(old_memcg);
|
2005-04-17 06:20:36 +08:00
|
|
|
return head;
|
|
|
|
/*
|
|
|
|
* In case anything failed, we just free everything we got.
|
|
|
|
*/
|
|
|
|
no_grow:
|
|
|
|
if (head) {
|
|
|
|
do {
|
|
|
|
bh = head;
|
|
|
|
head = head->b_this_page;
|
|
|
|
free_buffer_head(bh);
|
|
|
|
} while (head);
|
|
|
|
}
|
|
|
|
|
2018-08-18 06:46:44 +08:00
|
|
|
goto out;
|
2005-04-17 06:20:36 +08:00
|
|
|
}
|
2023-04-17 20:36:16 +08:00
|
|
|
EXPORT_SYMBOL_GPL(folio_alloc_buffers);
|
|
|
|
|
|
|
|
struct buffer_head *alloc_page_buffers(struct page *page, unsigned long size,
|
|
|
|
bool retry)
|
|
|
|
{
|
2023-09-14 23:00:04 +08:00
|
|
|
gfp_t gfp = GFP_NOFS | __GFP_ACCOUNT;
|
|
|
|
if (retry)
|
|
|
|
gfp |= __GFP_NOFAIL;
|
|
|
|
|
|
|
|
return folio_alloc_buffers(page_folio(page), size, gfp);
|
2023-04-17 20:36:16 +08:00
|
|
|
}
|
2005-04-17 06:20:36 +08:00
|
|
|
EXPORT_SYMBOL_GPL(alloc_page_buffers);
|
|
|
|
|
2023-06-13 05:01:39 +08:00
|
|
|
static inline void link_dev_buffers(struct folio *folio,
|
|
|
|
struct buffer_head *head)
|
2005-04-17 06:20:36 +08:00
|
|
|
{
|
|
|
|
struct buffer_head *bh, *tail;
|
|
|
|
|
|
|
|
bh = head;
|
|
|
|
do {
|
|
|
|
tail = bh;
|
|
|
|
bh = bh->b_this_page;
|
|
|
|
} while (bh);
|
|
|
|
tail->b_this_page = head;
|
2023-06-13 05:01:39 +08:00
|
|
|
folio_attach_private(folio, head);
|
2005-04-17 06:20:36 +08:00
|
|
|
}
|
|
|
|
|
2012-11-30 04:31:52 +08:00
|
|
|
static sector_t blkdev_max_block(struct block_device *bdev, unsigned int size)
|
|
|
|
{
|
|
|
|
sector_t retval = ~((sector_t)0);
|
2021-10-18 18:11:09 +08:00
|
|
|
loff_t sz = bdev_nr_bytes(bdev);
|
2012-11-30 04:31:52 +08:00
|
|
|
|
|
|
|
if (sz) {
|
|
|
|
unsigned int sizebits = blksize_bits(size);
|
|
|
|
retval = (sz >> sizebits);
|
|
|
|
}
|
|
|
|
return retval;
|
|
|
|
}
|
|
|
|
|
2005-04-17 06:20:36 +08:00
|
|
|
/*
|
2023-06-13 05:01:38 +08:00
|
|
|
* Initialise the state of a blockdev folio's buffers.
|
2005-04-17 06:20:36 +08:00
|
|
|
*/
|
2023-06-13 05:01:38 +08:00
|
|
|
static sector_t folio_init_buffers(struct folio *folio,
|
2023-11-10 05:06:03 +08:00
|
|
|
struct block_device *bdev, unsigned size)
|
2005-04-17 06:20:36 +08:00
|
|
|
{
|
2023-06-13 05:01:38 +08:00
|
|
|
struct buffer_head *head = folio_buffers(folio);
|
2005-04-17 06:20:36 +08:00
|
|
|
struct buffer_head *bh = head;
|
2023-06-13 05:01:38 +08:00
|
|
|
bool uptodate = folio_test_uptodate(folio);
|
2023-11-10 05:06:03 +08:00
|
|
|
sector_t block = div_u64(folio_pos(folio), size);
|
2021-10-18 18:11:10 +08:00
|
|
|
sector_t end_block = blkdev_max_block(bdev, size);
|
2005-04-17 06:20:36 +08:00
|
|
|
|
|
|
|
do {
|
|
|
|
if (!buffer_mapped(bh)) {
|
2018-01-17 14:25:12 +08:00
|
|
|
bh->b_end_io = NULL;
|
|
|
|
bh->b_private = NULL;
|
2005-04-17 06:20:36 +08:00
|
|
|
bh->b_bdev = bdev;
|
|
|
|
bh->b_blocknr = block;
|
|
|
|
if (uptodate)
|
|
|
|
set_buffer_uptodate(bh);
|
block: don't mark buffers beyond end of disk as mapped
Hi,
We have a bug report open where a squashfs image mounted on ppc64 would
exhibit errors due to trying to read beyond the end of the disk. It can
easily be reproduced by doing the following:
[root@ibm-p750e-02-lp3 ~]# ls -l install.img
-rw-r--r-- 1 root root 142032896 Apr 30 16:46 install.img
[root@ibm-p750e-02-lp3 ~]# mount -o loop ./install.img /mnt/test
[root@ibm-p750e-02-lp3 ~]# dd if=/dev/loop0 of=/dev/null
dd: reading `/dev/loop0': Input/output error
277376+0 records in
277376+0 records out
142016512 bytes (142 MB) copied, 0.9465 s, 150 MB/s
In dmesg, you'll find the following:
squashfs: version 4.0 (2009/01/31) Phillip Lougher
[ 43.106012] attempt to access beyond end of device
[ 43.106029] loop0: rw=0, want=277410, limit=277408
[ 43.106039] Buffer I/O error on device loop0, logical block 138704
[ 43.106053] attempt to access beyond end of device
[ 43.106057] loop0: rw=0, want=277412, limit=277408
[ 43.106061] Buffer I/O error on device loop0, logical block 138705
[ 43.106066] attempt to access beyond end of device
[ 43.106070] loop0: rw=0, want=277414, limit=277408
[ 43.106073] Buffer I/O error on device loop0, logical block 138706
[ 43.106078] attempt to access beyond end of device
[ 43.106081] loop0: rw=0, want=277416, limit=277408
[ 43.106085] Buffer I/O error on device loop0, logical block 138707
[ 43.106089] attempt to access beyond end of device
[ 43.106093] loop0: rw=0, want=277418, limit=277408
[ 43.106096] Buffer I/O error on device loop0, logical block 138708
[ 43.106101] attempt to access beyond end of device
[ 43.106104] loop0: rw=0, want=277420, limit=277408
[ 43.106108] Buffer I/O error on device loop0, logical block 138709
[ 43.106112] attempt to access beyond end of device
[ 43.106116] loop0: rw=0, want=277422, limit=277408
[ 43.106120] Buffer I/O error on device loop0, logical block 138710
[ 43.106124] attempt to access beyond end of device
[ 43.106128] loop0: rw=0, want=277424, limit=277408
[ 43.106131] Buffer I/O error on device loop0, logical block 138711
[ 43.106135] attempt to access beyond end of device
[ 43.106139] loop0: rw=0, want=277426, limit=277408
[ 43.106143] Buffer I/O error on device loop0, logical block 138712
[ 43.106147] attempt to access beyond end of device
[ 43.106151] loop0: rw=0, want=277428, limit=277408
[ 43.106154] Buffer I/O error on device loop0, logical block 138713
[ 43.106158] attempt to access beyond end of device
[ 43.106162] loop0: rw=0, want=277430, limit=277408
[ 43.106166] attempt to access beyond end of device
[ 43.106169] loop0: rw=0, want=277432, limit=277408
...
[ 43.106307] attempt to access beyond end of device
[ 43.106311] loop0: rw=0, want=277470, limit=2774
Squashfs manages to read in the end block(s) of the disk during the
mount operation. Then, when dd reads the block device, it leads to
block_read_full_page being called with buffers that are beyond end of
disk, but are marked as mapped. Thus, it would end up submitting read
I/O against them, resulting in the errors mentioned above. I fixed the
problem by modifying init_page_buffers to only set the buffer mapped if
it fell inside of i_size.
Cheers,
Jeff
Signed-off-by: Jeff Moyer <jmoyer@redhat.com>
Acked-by: Nick Piggin <npiggin@kernel.dk>
--
Changes from v1->v2: re-used max_block, as suggested by Nick Piggin.
Signed-off-by: Jens Axboe <axboe@kernel.dk>
2012-05-11 22:34:10 +08:00
|
|
|
if (block < end_block)
|
|
|
|
set_buffer_mapped(bh);
|
2005-04-17 06:20:36 +08:00
|
|
|
}
|
|
|
|
block++;
|
|
|
|
bh = bh->b_this_page;
|
|
|
|
} while (bh != head);
|
block: replace __getblk_slow misfix by grow_dev_page fix
Commit 91f68c89d8f3 ("block: fix infinite loop in __getblk_slow")
is not good: a successful call to grow_buffers() cannot guarantee
that the page won't be reclaimed before the immediate next call to
__find_get_block(), which is why there was always a loop there.
Yesterday I got "EXT4-fs error (device loop0): __ext4_get_inode_loc:3595:
inode #19278: block 664: comm cc1: unable to read itable block" on console,
which pointed to this commit.
I've been trying to bisect for weeks, why kbuild-on-ext4-on-loop-on-tmpfs
sometimes fails from a missing header file, under memory pressure on
ppc G5. I've never seen this on x86, and I've never seen it on 3.5-rc7
itself, despite that commit being in there: bisection pointed to an
irrelevant pinctrl merge, but hard to tell when failure takes between
18 minutes and 38 hours (but so far it's happened quicker on 3.6-rc2).
(I've since found such __ext4_get_inode_loc errors in /var/log/messages
from previous weeks: why the message never appeared on console until
yesterday morning is a mystery for another day.)
Revert 91f68c89d8f3, restoring __getblk_slow() to how it was (plus
a checkpatch nitfix). Simplify the interface between grow_buffers()
and grow_dev_page(), and avoid the infinite loop beyond end of device
by instead checking init_page_buffers()'s end_block there (I presume
that's more efficient than a repeated call to blkdev_max_block()),
returning -ENXIO to __getblk_slow() in that case.
And remove akpm's ten-year-old "__getblk() cannot fail ... weird"
comment, but that is worrying: are all users of __getblk() really
now prepared for a NULL bh beyond end of device, or will some oops??
Signed-off-by: Hugh Dickins <hughd@google.com>
Cc: stable@vger.kernel.org # 3.0 3.2 3.4 3.5
Signed-off-by: Jens Axboe <axboe@kernel.dk>
2012-08-23 18:17:36 +08:00
|
|
|
|
|
|
|
/*
|
|
|
|
* Caller needs to validate requested block against end of device.
|
|
|
|
*/
|
|
|
|
return end_block;
|
2005-04-17 06:20:36 +08:00
|
|
|
}
|
|
|
|
|
|
|
|
/*
|
2023-11-10 05:06:02 +08:00
|
|
|
* Create the page-cache folio that contains the requested block.
|
2005-04-17 06:20:36 +08:00
|
|
|
*
|
block: replace __getblk_slow misfix by grow_dev_page fix
Commit 91f68c89d8f3 ("block: fix infinite loop in __getblk_slow")
is not good: a successful call to grow_buffers() cannot guarantee
that the page won't be reclaimed before the immediate next call to
__find_get_block(), which is why there was always a loop there.
Yesterday I got "EXT4-fs error (device loop0): __ext4_get_inode_loc:3595:
inode #19278: block 664: comm cc1: unable to read itable block" on console,
which pointed to this commit.
I've been trying to bisect for weeks, why kbuild-on-ext4-on-loop-on-tmpfs
sometimes fails from a missing header file, under memory pressure on
ppc G5. I've never seen this on x86, and I've never seen it on 3.5-rc7
itself, despite that commit being in there: bisection pointed to an
irrelevant pinctrl merge, but hard to tell when failure takes between
18 minutes and 38 hours (but so far it's happened quicker on 3.6-rc2).
(I've since found such __ext4_get_inode_loc errors in /var/log/messages
from previous weeks: why the message never appeared on console until
yesterday morning is a mystery for another day.)
Revert 91f68c89d8f3, restoring __getblk_slow() to how it was (plus
a checkpatch nitfix). Simplify the interface between grow_buffers()
and grow_dev_page(), and avoid the infinite loop beyond end of device
by instead checking init_page_buffers()'s end_block there (I presume
that's more efficient than a repeated call to blkdev_max_block()),
returning -ENXIO to __getblk_slow() in that case.
And remove akpm's ten-year-old "__getblk() cannot fail ... weird"
comment, but that is worrying: are all users of __getblk() really
now prepared for a NULL bh beyond end of device, or will some oops??
Signed-off-by: Hugh Dickins <hughd@google.com>
Cc: stable@vger.kernel.org # 3.0 3.2 3.4 3.5
Signed-off-by: Jens Axboe <axboe@kernel.dk>
2012-08-23 18:17:36 +08:00
|
|
|
* This is used purely for blockdev mappings.
|
2023-11-10 05:06:02 +08:00
|
|
|
*
|
2024-01-01 17:38:48 +08:00
|
|
|
* Returns false if we have a failure which cannot be cured by retrying
|
|
|
|
* without sleeping. Returns true if we succeeded, or the caller should retry.
|
2005-04-17 06:20:36 +08:00
|
|
|
*/
|
2023-11-10 05:06:02 +08:00
|
|
|
static bool grow_dev_folio(struct block_device *bdev, sector_t block,
|
2023-11-10 05:06:03 +08:00
|
|
|
pgoff_t index, unsigned size, gfp_t gfp)
|
2005-04-17 06:20:36 +08:00
|
|
|
{
|
|
|
|
struct inode *inode = bdev->bd_inode;
|
2023-06-13 05:01:37 +08:00
|
|
|
struct folio *folio;
|
2005-04-17 06:20:36 +08:00
|
|
|
struct buffer_head *bh;
|
2023-11-10 05:06:02 +08:00
|
|
|
sector_t end_block = 0;
|
2013-10-17 04:47:00 +08:00
|
|
|
|
2023-06-13 05:01:37 +08:00
|
|
|
folio = __filemap_get_folio(inode->i_mapping, index,
|
2023-09-14 23:00:05 +08:00
|
|
|
FGP_LOCK | FGP_ACCESSED | FGP_CREAT, gfp);
|
|
|
|
if (IS_ERR(folio))
|
2023-11-10 05:06:02 +08:00
|
|
|
return false;
|
2005-04-17 06:20:36 +08:00
|
|
|
|
2023-06-13 05:01:37 +08:00
|
|
|
bh = folio_buffers(folio);
|
|
|
|
if (bh) {
|
2005-04-17 06:20:36 +08:00
|
|
|
if (bh->b_size == size) {
|
2023-11-10 05:06:03 +08:00
|
|
|
end_block = folio_init_buffers(folio, bdev, size);
|
2023-11-10 05:06:02 +08:00
|
|
|
goto unlock;
|
2005-04-17 06:20:36 +08:00
|
|
|
}
|
2023-11-10 05:06:02 +08:00
|
|
|
|
2024-01-01 17:38:48 +08:00
|
|
|
/*
|
|
|
|
* Retrying may succeed; for example the folio may finish
|
|
|
|
* writeback, or buffers may be cleaned. This should not
|
|
|
|
* happen very often; maybe we have old buffers attached to
|
|
|
|
* this blockdev's page cache and we're trying to change
|
|
|
|
* the block size?
|
|
|
|
*/
|
|
|
|
if (!try_to_free_buffers(folio)) {
|
|
|
|
end_block = ~0ULL;
|
2023-11-10 05:06:02 +08:00
|
|
|
goto unlock;
|
2005-04-17 06:20:36 +08:00
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2023-09-14 23:00:05 +08:00
|
|
|
bh = folio_alloc_buffers(folio, size, gfp | __GFP_ACCOUNT);
|
|
|
|
if (!bh)
|
2023-11-10 05:06:02 +08:00
|
|
|
goto unlock;
|
2005-04-17 06:20:36 +08:00
|
|
|
|
|
|
|
/*
|
2023-06-13 05:01:37 +08:00
|
|
|
* Link the folio to the buffers and initialise them. Take the
|
2005-04-17 06:20:36 +08:00
|
|
|
* lock to be atomic wrt __find_get_block(), which does not
|
2023-06-13 05:01:37 +08:00
|
|
|
* run under the folio lock.
|
2005-04-17 06:20:36 +08:00
|
|
|
*/
|
2023-11-18 05:58:23 +08:00
|
|
|
spin_lock(&inode->i_mapping->i_private_lock);
|
2023-06-13 05:01:39 +08:00
|
|
|
link_dev_buffers(folio, bh);
|
2023-11-10 05:06:03 +08:00
|
|
|
end_block = folio_init_buffers(folio, bdev, size);
|
2023-11-18 05:58:23 +08:00
|
|
|
spin_unlock(&inode->i_mapping->i_private_lock);
|
2023-11-10 05:06:02 +08:00
|
|
|
unlock:
|
2023-06-13 05:01:37 +08:00
|
|
|
folio_unlock(folio);
|
|
|
|
folio_put(folio);
|
2023-11-10 05:06:02 +08:00
|
|
|
return block < end_block;
|
2005-04-17 06:20:36 +08:00
|
|
|
}
|
|
|
|
|
|
|
|
/*
|
2023-11-10 05:06:02 +08:00
|
|
|
* Create buffers for the specified block device block's folio. If
|
|
|
|
* that folio was dirty, the buffers are set dirty also. Returns false
|
|
|
|
* if we've hit a permanent error.
|
2005-04-17 06:20:36 +08:00
|
|
|
*/
|
2023-11-10 05:06:02 +08:00
|
|
|
static bool grow_buffers(struct block_device *bdev, sector_t block,
|
|
|
|
unsigned size, gfp_t gfp)
|
2005-04-17 06:20:36 +08:00
|
|
|
{
|
2023-11-10 05:06:04 +08:00
|
|
|
loff_t pos;
|
2005-04-17 06:20:36 +08:00
|
|
|
|
2006-10-11 16:21:46 +08:00
|
|
|
/*
|
2023-11-10 05:06:04 +08:00
|
|
|
* Check for a block which lies outside our maximum possible
|
|
|
|
* pagecache index.
|
2006-10-11 16:21:46 +08:00
|
|
|
*/
|
2023-11-10 05:06:04 +08:00
|
|
|
if (check_mul_overflow(block, (sector_t)size, &pos) || pos > MAX_LFS_FILESIZE) {
|
|
|
|
printk(KERN_ERR "%s: requested out-of-range block %llu for device %pg\n",
|
2008-04-30 15:55:09 +08:00
|
|
|
__func__, (unsigned long long)block,
|
2015-04-13 20:31:37 +08:00
|
|
|
bdev);
|
2023-11-10 05:06:02 +08:00
|
|
|
return false;
|
2006-10-11 16:21:46 +08:00
|
|
|
}
|
block: replace __getblk_slow misfix by grow_dev_page fix
Commit 91f68c89d8f3 ("block: fix infinite loop in __getblk_slow")
is not good: a successful call to grow_buffers() cannot guarantee
that the page won't be reclaimed before the immediate next call to
__find_get_block(), which is why there was always a loop there.
Yesterday I got "EXT4-fs error (device loop0): __ext4_get_inode_loc:3595:
inode #19278: block 664: comm cc1: unable to read itable block" on console,
which pointed to this commit.
I've been trying to bisect for weeks, why kbuild-on-ext4-on-loop-on-tmpfs
sometimes fails from a missing header file, under memory pressure on
ppc G5. I've never seen this on x86, and I've never seen it on 3.5-rc7
itself, despite that commit being in there: bisection pointed to an
irrelevant pinctrl merge, but hard to tell when failure takes between
18 minutes and 38 hours (but so far it's happened quicker on 3.6-rc2).
(I've since found such __ext4_get_inode_loc errors in /var/log/messages
from previous weeks: why the message never appeared on console until
yesterday morning is a mystery for another day.)
Revert 91f68c89d8f3, restoring __getblk_slow() to how it was (plus
a checkpatch nitfix). Simplify the interface between grow_buffers()
and grow_dev_page(), and avoid the infinite loop beyond end of device
by instead checking init_page_buffers()'s end_block there (I presume
that's more efficient than a repeated call to blkdev_max_block()),
returning -ENXIO to __getblk_slow() in that case.
And remove akpm's ten-year-old "__getblk() cannot fail ... weird"
comment, but that is worrying: are all users of __getblk() really
now prepared for a NULL bh beyond end of device, or will some oops??
Signed-off-by: Hugh Dickins <hughd@google.com>
Cc: stable@vger.kernel.org # 3.0 3.2 3.4 3.5
Signed-off-by: Jens Axboe <axboe@kernel.dk>
2012-08-23 18:17:36 +08:00
|
|
|
|
2023-11-10 05:06:02 +08:00
|
|
|
/* Create a folio with the proper size buffers */
|
2023-11-10 05:06:04 +08:00
|
|
|
return grow_dev_folio(bdev, block, pos / PAGE_SIZE, size, gfp);
|
2005-04-17 06:20:36 +08:00
|
|
|
}
|
|
|
|
|
2016-09-13 04:30:41 +08:00
|
|
|
static struct buffer_head *
|
2014-09-05 10:04:42 +08:00
|
|
|
__getblk_slow(struct block_device *bdev, sector_t block,
|
|
|
|
unsigned size, gfp_t gfp)
|
2005-04-17 06:20:36 +08:00
|
|
|
{
|
|
|
|
/* Size must be multiple of hard sectorsize */
|
2009-05-23 05:17:49 +08:00
|
|
|
if (unlikely(size & (bdev_logical_block_size(bdev)-1) ||
|
2005-04-17 06:20:36 +08:00
|
|
|
(size < 512 || size > PAGE_SIZE))) {
|
|
|
|
printk(KERN_ERR "getblk(): invalid block size %d requested\n",
|
|
|
|
size);
|
2009-05-23 05:17:49 +08:00
|
|
|
printk(KERN_ERR "logical block size: %d\n",
|
|
|
|
bdev_logical_block_size(bdev));
|
2005-04-17 06:20:36 +08:00
|
|
|
|
|
|
|
dump_stack();
|
|
|
|
return NULL;
|
|
|
|
}
|
|
|
|
|
block: replace __getblk_slow misfix by grow_dev_page fix
Commit 91f68c89d8f3 ("block: fix infinite loop in __getblk_slow")
is not good: a successful call to grow_buffers() cannot guarantee
that the page won't be reclaimed before the immediate next call to
__find_get_block(), which is why there was always a loop there.
Yesterday I got "EXT4-fs error (device loop0): __ext4_get_inode_loc:3595:
inode #19278: block 664: comm cc1: unable to read itable block" on console,
which pointed to this commit.
I've been trying to bisect for weeks, why kbuild-on-ext4-on-loop-on-tmpfs
sometimes fails from a missing header file, under memory pressure on
ppc G5. I've never seen this on x86, and I've never seen it on 3.5-rc7
itself, despite that commit being in there: bisection pointed to an
irrelevant pinctrl merge, but hard to tell when failure takes between
18 minutes and 38 hours (but so far it's happened quicker on 3.6-rc2).
(I've since found such __ext4_get_inode_loc errors in /var/log/messages
from previous weeks: why the message never appeared on console until
yesterday morning is a mystery for another day.)
Revert 91f68c89d8f3, restoring __getblk_slow() to how it was (plus
a checkpatch nitfix). Simplify the interface between grow_buffers()
and grow_dev_page(), and avoid the infinite loop beyond end of device
by instead checking init_page_buffers()'s end_block there (I presume
that's more efficient than a repeated call to blkdev_max_block()),
returning -ENXIO to __getblk_slow() in that case.
And remove akpm's ten-year-old "__getblk() cannot fail ... weird"
comment, but that is worrying: are all users of __getblk() really
now prepared for a NULL bh beyond end of device, or will some oops??
Signed-off-by: Hugh Dickins <hughd@google.com>
Cc: stable@vger.kernel.org # 3.0 3.2 3.4 3.5
Signed-off-by: Jens Axboe <axboe@kernel.dk>
2012-08-23 18:17:36 +08:00
|
|
|
for (;;) {
|
|
|
|
struct buffer_head *bh;
|
2005-04-17 06:20:36 +08:00
|
|
|
|
|
|
|
bh = __find_get_block(bdev, block, size);
|
|
|
|
if (bh)
|
|
|
|
return bh;
|
block: replace __getblk_slow misfix by grow_dev_page fix
Commit 91f68c89d8f3 ("block: fix infinite loop in __getblk_slow")
is not good: a successful call to grow_buffers() cannot guarantee
that the page won't be reclaimed before the immediate next call to
__find_get_block(), which is why there was always a loop there.
Yesterday I got "EXT4-fs error (device loop0): __ext4_get_inode_loc:3595:
inode #19278: block 664: comm cc1: unable to read itable block" on console,
which pointed to this commit.
I've been trying to bisect for weeks, why kbuild-on-ext4-on-loop-on-tmpfs
sometimes fails from a missing header file, under memory pressure on
ppc G5. I've never seen this on x86, and I've never seen it on 3.5-rc7
itself, despite that commit being in there: bisection pointed to an
irrelevant pinctrl merge, but hard to tell when failure takes between
18 minutes and 38 hours (but so far it's happened quicker on 3.6-rc2).
(I've since found such __ext4_get_inode_loc errors in /var/log/messages
from previous weeks: why the message never appeared on console until
yesterday morning is a mystery for another day.)
Revert 91f68c89d8f3, restoring __getblk_slow() to how it was (plus
a checkpatch nitfix). Simplify the interface between grow_buffers()
and grow_dev_page(), and avoid the infinite loop beyond end of device
by instead checking init_page_buffers()'s end_block there (I presume
that's more efficient than a repeated call to blkdev_max_block()),
returning -ENXIO to __getblk_slow() in that case.
And remove akpm's ten-year-old "__getblk() cannot fail ... weird"
comment, but that is worrying: are all users of __getblk() really
now prepared for a NULL bh beyond end of device, or will some oops??
Signed-off-by: Hugh Dickins <hughd@google.com>
Cc: stable@vger.kernel.org # 3.0 3.2 3.4 3.5
Signed-off-by: Jens Axboe <axboe@kernel.dk>
2012-08-23 18:17:36 +08:00
|
|
|
|
2023-11-10 05:06:02 +08:00
|
|
|
if (!grow_buffers(bdev, block, size, gfp))
|
block: replace __getblk_slow misfix by grow_dev_page fix
Commit 91f68c89d8f3 ("block: fix infinite loop in __getblk_slow")
is not good: a successful call to grow_buffers() cannot guarantee
that the page won't be reclaimed before the immediate next call to
__find_get_block(), which is why there was always a loop there.
Yesterday I got "EXT4-fs error (device loop0): __ext4_get_inode_loc:3595:
inode #19278: block 664: comm cc1: unable to read itable block" on console,
which pointed to this commit.
I've been trying to bisect for weeks, why kbuild-on-ext4-on-loop-on-tmpfs
sometimes fails from a missing header file, under memory pressure on
ppc G5. I've never seen this on x86, and I've never seen it on 3.5-rc7
itself, despite that commit being in there: bisection pointed to an
irrelevant pinctrl merge, but hard to tell when failure takes between
18 minutes and 38 hours (but so far it's happened quicker on 3.6-rc2).
(I've since found such __ext4_get_inode_loc errors in /var/log/messages
from previous weeks: why the message never appeared on console until
yesterday morning is a mystery for another day.)
Revert 91f68c89d8f3, restoring __getblk_slow() to how it was (plus
a checkpatch nitfix). Simplify the interface between grow_buffers()
and grow_dev_page(), and avoid the infinite loop beyond end of device
by instead checking init_page_buffers()'s end_block there (I presume
that's more efficient than a repeated call to blkdev_max_block()),
returning -ENXIO to __getblk_slow() in that case.
And remove akpm's ten-year-old "__getblk() cannot fail ... weird"
comment, but that is worrying: are all users of __getblk() really
now prepared for a NULL bh beyond end of device, or will some oops??
Signed-off-by: Hugh Dickins <hughd@google.com>
Cc: stable@vger.kernel.org # 3.0 3.2 3.4 3.5
Signed-off-by: Jens Axboe <axboe@kernel.dk>
2012-08-23 18:17:36 +08:00
|
|
|
return NULL;
|
2005-04-17 06:20:36 +08:00
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
/*
|
|
|
|
* The relationship between dirty buffers and dirty pages:
|
|
|
|
*
|
|
|
|
* Whenever a page has any dirty buffers, the page's dirty bit is set, and
|
2017-12-04 23:40:41 +08:00
|
|
|
* the page is tagged dirty in the page cache.
|
2005-04-17 06:20:36 +08:00
|
|
|
*
|
|
|
|
* At all times, the dirtiness of the buffers represents the dirtiness of
|
|
|
|
* subsections of the page. If the page has buffers, the page dirty bit is
|
|
|
|
* merely a hint about the true dirty state.
|
|
|
|
*
|
|
|
|
* When a page is set dirty in its entirety, all its buffers are marked dirty
|
|
|
|
* (if the page has buffers).
|
|
|
|
*
|
|
|
|
* When a buffer is marked dirty, its page is dirtied, but the page's other
|
|
|
|
* buffers are not.
|
|
|
|
*
|
|
|
|
* Also. When blockdev buffers are explicitly read with bread(), they
|
|
|
|
* individually become uptodate. But their backing page remains not
|
|
|
|
* uptodate - even if all of its buffers are uptodate. A subsequent
|
2022-04-29 22:40:40 +08:00
|
|
|
* block_read_full_folio() against that folio will discover all the uptodate
|
|
|
|
* buffers, will set the folio uptodate and will perform no I/O.
|
2005-04-17 06:20:36 +08:00
|
|
|
*/
|
|
|
|
|
|
|
|
/**
|
|
|
|
* mark_buffer_dirty - mark a buffer_head as needing writeout
|
2005-05-01 23:59:26 +08:00
|
|
|
* @bh: the buffer_head to mark dirty
|
2005-04-17 06:20:36 +08:00
|
|
|
*
|
2017-12-04 23:40:41 +08:00
|
|
|
* mark_buffer_dirty() will set the dirty bit against the buffer, then set
|
|
|
|
* its backing page dirty, then tag the page as dirty in the page cache
|
|
|
|
* and then attach the address_space's inode to its superblock's dirty
|
2005-04-17 06:20:36 +08:00
|
|
|
* inode list.
|
|
|
|
*
|
2023-11-18 05:58:23 +08:00
|
|
|
* mark_buffer_dirty() is atomic. It takes bh->b_folio->mapping->i_private_lock,
|
2018-04-11 07:36:56 +08:00
|
|
|
* i_pages lock and mapping->host->i_lock.
|
2005-04-17 06:20:36 +08:00
|
|
|
*/
|
2008-02-08 20:19:52 +08:00
|
|
|
void mark_buffer_dirty(struct buffer_head *bh)
|
2005-04-17 06:20:36 +08:00
|
|
|
{
|
2007-07-17 19:03:34 +08:00
|
|
|
WARN_ON_ONCE(!buffer_uptodate(bh));
|
2008-04-05 05:38:17 +08:00
|
|
|
|
2013-01-12 05:06:36 +08:00
|
|
|
trace_block_dirty_buffer(bh);
|
|
|
|
|
2008-04-05 05:38:17 +08:00
|
|
|
/*
|
|
|
|
* Very *carefully* optimize the it-is-already-dirty case.
|
|
|
|
*
|
|
|
|
* Don't let the final "is it dirty" escape to before we
|
|
|
|
* perhaps modified the buffer.
|
|
|
|
*/
|
|
|
|
if (buffer_dirty(bh)) {
|
|
|
|
smp_mb();
|
|
|
|
if (buffer_dirty(bh))
|
|
|
|
return;
|
|
|
|
}
|
|
|
|
|
2009-03-20 02:32:05 +08:00
|
|
|
if (!test_set_buffer_dirty(bh)) {
|
2022-12-16 05:43:57 +08:00
|
|
|
struct folio *folio = bh->b_folio;
|
memcg: add per cgroup dirty page accounting
When modifying PG_Dirty on cached file pages, update the new
MEM_CGROUP_STAT_DIRTY counter. This is done in the same places where
global NR_FILE_DIRTY is managed. The new memcg stat is visible in the
per memcg memory.stat cgroupfs file. The most recent past attempt at
this was http://thread.gmane.org/gmane.linux.kernel.cgroups/8632
The new accounting supports future efforts to add per cgroup dirty
page throttling and writeback. It also helps an administrator break
down a container's memory usage and provides evidence to understand
memcg oom kills (the new dirty count is included in memcg oom kill
messages).
The ability to move page accounting between memcg
(memory.move_charge_at_immigrate) makes this accounting more
complicated than the global counter. The existing
mem_cgroup_{begin,end}_page_stat() lock is used to serialize move
accounting with stat updates.
Typical update operation:
memcg = mem_cgroup_begin_page_stat(page)
if (TestSetPageDirty()) {
[...]
mem_cgroup_update_page_stat(memcg)
}
mem_cgroup_end_page_stat(memcg)
Summary of mem_cgroup_end_page_stat() overhead:
- Without CONFIG_MEMCG it's a no-op
- With CONFIG_MEMCG and no inter memcg task movement, it's just
rcu_read_lock()
- With CONFIG_MEMCG and inter memcg task movement, it's
rcu_read_lock() + spin_lock_irqsave()
A memcg parameter is added to several routines because their callers
now grab mem_cgroup_begin_page_stat() which returns the memcg later
needed by for mem_cgroup_update_page_stat().
Because mem_cgroup_begin_page_stat() may disable interrupts, some
adjustments are needed:
- move __mark_inode_dirty() from __set_page_dirty() to its caller.
__mark_inode_dirty() locking does not want interrupts disabled.
- use spin_lock_irqsave(tree_lock) rather than spin_lock_irq() in
__delete_from_page_cache(), replace_page_cache_page(),
invalidate_complete_page2(), and __remove_mapping().
text data bss dec hex filename
8925147 1774832 1785856 12485835 be84cb vmlinux-!CONFIG_MEMCG-before
8925339 1774832 1785856 12486027 be858b vmlinux-!CONFIG_MEMCG-after
+192 text bytes
8965977 1784992 1785856 12536825 bf4bf9 vmlinux-CONFIG_MEMCG-before
8966750 1784992 1785856 12537598 bf4efe vmlinux-CONFIG_MEMCG-after
+773 text bytes
Performance tests run on v4.0-rc1-36-g4f671fe2f952. Lower is better for
all metrics, they're all wall clock or cycle counts. The read and write
fault benchmarks just measure fault time, they do not include I/O time.
* CONFIG_MEMCG not set:
baseline patched
kbuild 1m25.030000(+-0.088% 3 samples) 1m25.426667(+-0.120% 3 samples)
dd write 100 MiB 0.859211561 +-15.10% 0.874162885 +-15.03%
dd write 200 MiB 1.670653105 +-17.87% 1.669384764 +-11.99%
dd write 1000 MiB 8.434691190 +-14.15% 8.474733215 +-14.77%
read fault cycles 254.0(+-0.000% 10 samples) 253.0(+-0.000% 10 samples)
write fault cycles 2021.2(+-3.070% 10 samples) 1984.5(+-1.036% 10 samples)
* CONFIG_MEMCG=y root_memcg:
baseline patched
kbuild 1m25.716667(+-0.105% 3 samples) 1m25.686667(+-0.153% 3 samples)
dd write 100 MiB 0.855650830 +-14.90% 0.887557919 +-14.90%
dd write 200 MiB 1.688322953 +-12.72% 1.667682724 +-13.33%
dd write 1000 MiB 8.418601605 +-14.30% 8.673532299 +-15.00%
read fault cycles 266.0(+-0.000% 10 samples) 266.0(+-0.000% 10 samples)
write fault cycles 2051.7(+-1.349% 10 samples) 2049.6(+-1.686% 10 samples)
* CONFIG_MEMCG=y non-root_memcg:
baseline patched
kbuild 1m26.120000(+-0.273% 3 samples) 1m25.763333(+-0.127% 3 samples)
dd write 100 MiB 0.861723964 +-15.25% 0.818129350 +-14.82%
dd write 200 MiB 1.669887569 +-13.30% 1.698645885 +-13.27%
dd write 1000 MiB 8.383191730 +-14.65% 8.351742280 +-14.52%
read fault cycles 265.7(+-0.172% 10 samples) 267.0(+-0.000% 10 samples)
write fault cycles 2070.6(+-1.512% 10 samples) 2084.4(+-2.148% 10 samples)
As expected anon page faults are not affected by this patch.
tj: Updated to apply on top of the recent cancel_dirty_page() changes.
Signed-off-by: Sha Zhengju <handai.szj@gmail.com>
Signed-off-by: Greg Thelen <gthelen@google.com>
Signed-off-by: Tejun Heo <tj@kernel.org>
Signed-off-by: Jens Axboe <axboe@fb.com>
2015-05-23 05:13:16 +08:00
|
|
|
struct address_space *mapping = NULL;
|
|
|
|
|
2022-12-16 05:43:57 +08:00
|
|
|
folio_memcg_lock(folio);
|
|
|
|
if (!folio_test_set_dirty(folio)) {
|
|
|
|
mapping = folio->mapping;
|
2009-08-22 08:40:08 +08:00
|
|
|
if (mapping)
|
2022-12-16 05:43:57 +08:00
|
|
|
__folio_mark_dirty(folio, mapping, 0);
|
2009-08-22 08:40:08 +08:00
|
|
|
}
|
2022-12-16 05:43:57 +08:00
|
|
|
folio_memcg_unlock(folio);
|
memcg: add per cgroup dirty page accounting
When modifying PG_Dirty on cached file pages, update the new
MEM_CGROUP_STAT_DIRTY counter. This is done in the same places where
global NR_FILE_DIRTY is managed. The new memcg stat is visible in the
per memcg memory.stat cgroupfs file. The most recent past attempt at
this was http://thread.gmane.org/gmane.linux.kernel.cgroups/8632
The new accounting supports future efforts to add per cgroup dirty
page throttling and writeback. It also helps an administrator break
down a container's memory usage and provides evidence to understand
memcg oom kills (the new dirty count is included in memcg oom kill
messages).
The ability to move page accounting between memcg
(memory.move_charge_at_immigrate) makes this accounting more
complicated than the global counter. The existing
mem_cgroup_{begin,end}_page_stat() lock is used to serialize move
accounting with stat updates.
Typical update operation:
memcg = mem_cgroup_begin_page_stat(page)
if (TestSetPageDirty()) {
[...]
mem_cgroup_update_page_stat(memcg)
}
mem_cgroup_end_page_stat(memcg)
Summary of mem_cgroup_end_page_stat() overhead:
- Without CONFIG_MEMCG it's a no-op
- With CONFIG_MEMCG and no inter memcg task movement, it's just
rcu_read_lock()
- With CONFIG_MEMCG and inter memcg task movement, it's
rcu_read_lock() + spin_lock_irqsave()
A memcg parameter is added to several routines because their callers
now grab mem_cgroup_begin_page_stat() which returns the memcg later
needed by for mem_cgroup_update_page_stat().
Because mem_cgroup_begin_page_stat() may disable interrupts, some
adjustments are needed:
- move __mark_inode_dirty() from __set_page_dirty() to its caller.
__mark_inode_dirty() locking does not want interrupts disabled.
- use spin_lock_irqsave(tree_lock) rather than spin_lock_irq() in
__delete_from_page_cache(), replace_page_cache_page(),
invalidate_complete_page2(), and __remove_mapping().
text data bss dec hex filename
8925147 1774832 1785856 12485835 be84cb vmlinux-!CONFIG_MEMCG-before
8925339 1774832 1785856 12486027 be858b vmlinux-!CONFIG_MEMCG-after
+192 text bytes
8965977 1784992 1785856 12536825 bf4bf9 vmlinux-CONFIG_MEMCG-before
8966750 1784992 1785856 12537598 bf4efe vmlinux-CONFIG_MEMCG-after
+773 text bytes
Performance tests run on v4.0-rc1-36-g4f671fe2f952. Lower is better for
all metrics, they're all wall clock or cycle counts. The read and write
fault benchmarks just measure fault time, they do not include I/O time.
* CONFIG_MEMCG not set:
baseline patched
kbuild 1m25.030000(+-0.088% 3 samples) 1m25.426667(+-0.120% 3 samples)
dd write 100 MiB 0.859211561 +-15.10% 0.874162885 +-15.03%
dd write 200 MiB 1.670653105 +-17.87% 1.669384764 +-11.99%
dd write 1000 MiB 8.434691190 +-14.15% 8.474733215 +-14.77%
read fault cycles 254.0(+-0.000% 10 samples) 253.0(+-0.000% 10 samples)
write fault cycles 2021.2(+-3.070% 10 samples) 1984.5(+-1.036% 10 samples)
* CONFIG_MEMCG=y root_memcg:
baseline patched
kbuild 1m25.716667(+-0.105% 3 samples) 1m25.686667(+-0.153% 3 samples)
dd write 100 MiB 0.855650830 +-14.90% 0.887557919 +-14.90%
dd write 200 MiB 1.688322953 +-12.72% 1.667682724 +-13.33%
dd write 1000 MiB 8.418601605 +-14.30% 8.673532299 +-15.00%
read fault cycles 266.0(+-0.000% 10 samples) 266.0(+-0.000% 10 samples)
write fault cycles 2051.7(+-1.349% 10 samples) 2049.6(+-1.686% 10 samples)
* CONFIG_MEMCG=y non-root_memcg:
baseline patched
kbuild 1m26.120000(+-0.273% 3 samples) 1m25.763333(+-0.127% 3 samples)
dd write 100 MiB 0.861723964 +-15.25% 0.818129350 +-14.82%
dd write 200 MiB 1.669887569 +-13.30% 1.698645885 +-13.27%
dd write 1000 MiB 8.383191730 +-14.65% 8.351742280 +-14.52%
read fault cycles 265.7(+-0.172% 10 samples) 267.0(+-0.000% 10 samples)
write fault cycles 2070.6(+-1.512% 10 samples) 2084.4(+-2.148% 10 samples)
As expected anon page faults are not affected by this patch.
tj: Updated to apply on top of the recent cancel_dirty_page() changes.
Signed-off-by: Sha Zhengju <handai.szj@gmail.com>
Signed-off-by: Greg Thelen <gthelen@google.com>
Signed-off-by: Tejun Heo <tj@kernel.org>
Signed-off-by: Jens Axboe <axboe@fb.com>
2015-05-23 05:13:16 +08:00
|
|
|
if (mapping)
|
|
|
|
__mark_inode_dirty(mapping->host, I_DIRTY_PAGES);
|
2009-03-20 02:32:05 +08:00
|
|
|
}
|
2005-04-17 06:20:36 +08:00
|
|
|
}
|
2009-09-23 07:43:51 +08:00
|
|
|
EXPORT_SYMBOL(mark_buffer_dirty);
|
2005-04-17 06:20:36 +08:00
|
|
|
|
2017-07-06 19:02:21 +08:00
|
|
|
void mark_buffer_write_io_error(struct buffer_head *bh)
|
|
|
|
{
|
|
|
|
set_buffer_write_io_error(bh);
|
|
|
|
/* FIXME: do we need to set this in both places? */
|
2022-12-16 05:43:52 +08:00
|
|
|
if (bh->b_folio && bh->b_folio->mapping)
|
|
|
|
mapping_set_error(bh->b_folio->mapping, -EIO);
|
2023-08-07 19:26:22 +08:00
|
|
|
if (bh->b_assoc_map) {
|
2017-07-06 19:02:21 +08:00
|
|
|
mapping_set_error(bh->b_assoc_map, -EIO);
|
2023-08-07 19:26:22 +08:00
|
|
|
errseq_set(&bh->b_assoc_map->host->i_sb->s_wb_err, -EIO);
|
|
|
|
}
|
2017-07-06 19:02:21 +08:00
|
|
|
}
|
|
|
|
EXPORT_SYMBOL(mark_buffer_write_io_error);
|
|
|
|
|
2005-04-17 06:20:36 +08:00
|
|
|
/*
|
|
|
|
* Decrement a buffer_head's reference count. If all buffers against a page
|
|
|
|
* have zero reference count, are clean and unlocked, and if the page is clean
|
|
|
|
* and unlocked then try_to_free_buffers() may strip the buffers from the page
|
|
|
|
* in preparation for freeing it (sometimes, rarely, buffers are removed from
|
|
|
|
* a page but it ends up not being freed, and buffers may later be reattached).
|
|
|
|
*/
|
|
|
|
void __brelse(struct buffer_head * buf)
|
|
|
|
{
|
|
|
|
if (atomic_read(&buf->b_count)) {
|
|
|
|
put_bh(buf);
|
|
|
|
return;
|
|
|
|
}
|
2008-07-26 10:45:40 +08:00
|
|
|
WARN(1, KERN_ERR "VFS: brelse: Trying to free free buffer\n");
|
2005-04-17 06:20:36 +08:00
|
|
|
}
|
2009-09-23 07:43:51 +08:00
|
|
|
EXPORT_SYMBOL(__brelse);
|
2005-04-17 06:20:36 +08:00
|
|
|
|
|
|
|
/*
|
|
|
|
* bforget() is like brelse(), except it discards any
|
|
|
|
* potentially dirty data.
|
|
|
|
*/
|
|
|
|
void __bforget(struct buffer_head *bh)
|
|
|
|
{
|
|
|
|
clear_buffer_dirty(bh);
|
2008-02-08 20:21:59 +08:00
|
|
|
if (bh->b_assoc_map) {
|
2022-12-16 05:43:52 +08:00
|
|
|
struct address_space *buffer_mapping = bh->b_folio->mapping;
|
2005-04-17 06:20:36 +08:00
|
|
|
|
2023-11-18 05:58:23 +08:00
|
|
|
spin_lock(&buffer_mapping->i_private_lock);
|
2005-04-17 06:20:36 +08:00
|
|
|
list_del_init(&bh->b_assoc_buffers);
|
2006-10-17 15:10:19 +08:00
|
|
|
bh->b_assoc_map = NULL;
|
2023-11-18 05:58:23 +08:00
|
|
|
spin_unlock(&buffer_mapping->i_private_lock);
|
2005-04-17 06:20:36 +08:00
|
|
|
}
|
|
|
|
__brelse(bh);
|
|
|
|
}
|
2009-09-23 07:43:51 +08:00
|
|
|
EXPORT_SYMBOL(__bforget);
|
2005-04-17 06:20:36 +08:00
|
|
|
|
|
|
|
static struct buffer_head *__bread_slow(struct buffer_head *bh)
|
|
|
|
{
|
|
|
|
lock_buffer(bh);
|
|
|
|
if (buffer_uptodate(bh)) {
|
|
|
|
unlock_buffer(bh);
|
|
|
|
return bh;
|
|
|
|
} else {
|
|
|
|
get_bh(bh);
|
|
|
|
bh->b_end_io = end_buffer_read_sync;
|
2022-07-15 02:07:13 +08:00
|
|
|
submit_bh(REQ_OP_READ, bh);
|
2005-04-17 06:20:36 +08:00
|
|
|
wait_on_buffer(bh);
|
|
|
|
if (buffer_uptodate(bh))
|
|
|
|
return bh;
|
|
|
|
}
|
|
|
|
brelse(bh);
|
|
|
|
return NULL;
|
|
|
|
}
|
|
|
|
|
|
|
|
/*
|
|
|
|
* Per-cpu buffer LRU implementation. To reduce the cost of __find_get_block().
|
|
|
|
* The bhs[] array is sorted - newest buffer is at bhs[0]. Buffers have their
|
|
|
|
* refcount elevated by one when they're in an LRU. A buffer can only appear
|
|
|
|
* once in a particular CPU's LRU. A single buffer can be present in multiple
|
|
|
|
* CPU's LRUs at the same time.
|
|
|
|
*
|
|
|
|
* This is a transparent caching front-end to sb_bread(), sb_getblk() and
|
|
|
|
* sb_find_get_block().
|
|
|
|
*
|
|
|
|
* The LRUs themselves only need locking against invalidate_bh_lrus. We use
|
|
|
|
* a local interrupt disable for that.
|
|
|
|
*/
|
|
|
|
|
2014-10-10 06:29:38 +08:00
|
|
|
#define BH_LRU_SIZE 16
|
2005-04-17 06:20:36 +08:00
|
|
|
|
|
|
|
struct bh_lru {
|
|
|
|
struct buffer_head *bhs[BH_LRU_SIZE];
|
|
|
|
};
|
|
|
|
|
|
|
|
static DEFINE_PER_CPU(struct bh_lru, bh_lrus) = {{ NULL }};
|
|
|
|
|
|
|
|
#ifdef CONFIG_SMP
|
|
|
|
#define bh_lru_lock() local_irq_disable()
|
|
|
|
#define bh_lru_unlock() local_irq_enable()
|
|
|
|
#else
|
|
|
|
#define bh_lru_lock() preempt_disable()
|
|
|
|
#define bh_lru_unlock() preempt_enable()
|
|
|
|
#endif
|
|
|
|
|
|
|
|
static inline void check_irqs_on(void)
|
|
|
|
{
|
|
|
|
#ifdef irqs_disabled
|
|
|
|
BUG_ON(irqs_disabled());
|
|
|
|
#endif
|
|
|
|
}
|
|
|
|
|
|
|
|
/*
|
2017-07-11 06:47:29 +08:00
|
|
|
* Install a buffer_head into this cpu's LRU. If not already in the LRU, it is
|
|
|
|
* inserted at the front, and the buffer_head at the back if any is evicted.
|
|
|
|
* Or, if already in the LRU it is moved to the front.
|
2005-04-17 06:20:36 +08:00
|
|
|
*/
|
|
|
|
static void bh_lru_install(struct buffer_head *bh)
|
|
|
|
{
|
2017-07-11 06:47:29 +08:00
|
|
|
struct buffer_head *evictee = bh;
|
|
|
|
struct bh_lru *b;
|
|
|
|
int i;
|
2005-04-17 06:20:36 +08:00
|
|
|
|
|
|
|
check_irqs_on();
|
2022-03-23 05:39:34 +08:00
|
|
|
bh_lru_lock();
|
|
|
|
|
2021-05-05 09:37:00 +08:00
|
|
|
/*
|
|
|
|
* the refcount of buffer_head in bh_lru prevents dropping the
|
|
|
|
* attached page(i.e., try_to_free_buffers) so it could cause
|
|
|
|
* failing page migration.
|
|
|
|
* Skip putting upcoming bh into bh_lru until migration is done.
|
|
|
|
*/
|
fs/buffer.c: disable per-CPU buffer_head cache for isolated CPUs
For certain types of applications (for example PLC software or
RAN processing), upon occurrence of an event, it is necessary to
complete a certain task in a maximum amount of time (deadline).
One way to express this requirement is with a pair of numbers,
deadline time and execution time, where:
* deadline time: length of time between event and deadline.
* execution time: length of time it takes for processing of event
to occur on a particular hardware platform
(uninterrupted).
The particular values depend on use-case. For the case
where the realtime application executes in a virtualized
guest, an IPI which must be serviced in the host will cause
the following sequence of events:
1) VM-exit
2) execution of IPI (and function call)
3) VM-entry
Which causes an excess of 50us latency as observed by cyclictest
(this violates the latency requirement of vRAN application with 1ms TTI,
for example).
invalidate_bh_lrus calls an IPI on each CPU that has non empty
per-CPU cache:
on_each_cpu_cond(has_bh_in_lru, invalidate_bh_lru, NULL, 1);
The performance when using the per-CPU LRU cache is as follows:
42 ns per __find_get_block
68 ns per __find_get_block_slow
Given that the main use cases for latency sensitive applications
do not involve block I/O (data necessary for program operation is
locked in RAM), disable per-CPU buffer_head caches for isolated CPUs.
Signed-off-by: Marcelo Tosatti <mtosatti@redhat.com>
Acked-by: Frederic Weisbecker <frederic@kernel.org>
Message-Id: <ZJtBrybavtb1x45V@tpad>
Signed-off-by: Christian Brauner <brauner@kernel.org>
2023-06-28 04:08:15 +08:00
|
|
|
if (lru_cache_disabled() || cpu_is_isolated(smp_processor_id())) {
|
2022-03-23 05:39:34 +08:00
|
|
|
bh_lru_unlock();
|
2021-05-05 09:37:00 +08:00
|
|
|
return;
|
2022-03-23 05:39:34 +08:00
|
|
|
}
|
2005-04-17 06:20:36 +08:00
|
|
|
|
2017-07-11 06:47:29 +08:00
|
|
|
b = this_cpu_ptr(&bh_lrus);
|
|
|
|
for (i = 0; i < BH_LRU_SIZE; i++) {
|
|
|
|
swap(evictee, b->bhs[i]);
|
|
|
|
if (evictee == bh) {
|
|
|
|
bh_lru_unlock();
|
|
|
|
return;
|
2005-04-17 06:20:36 +08:00
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2017-07-11 06:47:29 +08:00
|
|
|
get_bh(bh);
|
|
|
|
bh_lru_unlock();
|
|
|
|
brelse(evictee);
|
2005-04-17 06:20:36 +08:00
|
|
|
}
|
|
|
|
|
|
|
|
/*
|
|
|
|
* Look up the bh in this cpu's LRU. If it's there, move it to the head.
|
|
|
|
*/
|
2006-01-15 05:20:43 +08:00
|
|
|
static struct buffer_head *
|
2007-02-12 16:52:14 +08:00
|
|
|
lookup_bh_lru(struct block_device *bdev, sector_t block, unsigned size)
|
2005-04-17 06:20:36 +08:00
|
|
|
{
|
|
|
|
struct buffer_head *ret = NULL;
|
2007-02-12 16:52:14 +08:00
|
|
|
unsigned int i;
|
2005-04-17 06:20:36 +08:00
|
|
|
|
|
|
|
check_irqs_on();
|
|
|
|
bh_lru_lock();
|
fs/buffer.c: disable per-CPU buffer_head cache for isolated CPUs
For certain types of applications (for example PLC software or
RAN processing), upon occurrence of an event, it is necessary to
complete a certain task in a maximum amount of time (deadline).
One way to express this requirement is with a pair of numbers,
deadline time and execution time, where:
* deadline time: length of time between event and deadline.
* execution time: length of time it takes for processing of event
to occur on a particular hardware platform
(uninterrupted).
The particular values depend on use-case. For the case
where the realtime application executes in a virtualized
guest, an IPI which must be serviced in the host will cause
the following sequence of events:
1) VM-exit
2) execution of IPI (and function call)
3) VM-entry
Which causes an excess of 50us latency as observed by cyclictest
(this violates the latency requirement of vRAN application with 1ms TTI,
for example).
invalidate_bh_lrus calls an IPI on each CPU that has non empty
per-CPU cache:
on_each_cpu_cond(has_bh_in_lru, invalidate_bh_lru, NULL, 1);
The performance when using the per-CPU LRU cache is as follows:
42 ns per __find_get_block
68 ns per __find_get_block_slow
Given that the main use cases for latency sensitive applications
do not involve block I/O (data necessary for program operation is
locked in RAM), disable per-CPU buffer_head caches for isolated CPUs.
Signed-off-by: Marcelo Tosatti <mtosatti@redhat.com>
Acked-by: Frederic Weisbecker <frederic@kernel.org>
Message-Id: <ZJtBrybavtb1x45V@tpad>
Signed-off-by: Christian Brauner <brauner@kernel.org>
2023-06-28 04:08:15 +08:00
|
|
|
if (cpu_is_isolated(smp_processor_id())) {
|
|
|
|
bh_lru_unlock();
|
|
|
|
return NULL;
|
|
|
|
}
|
2005-04-17 06:20:36 +08:00
|
|
|
for (i = 0; i < BH_LRU_SIZE; i++) {
|
2010-12-07 01:16:28 +08:00
|
|
|
struct buffer_head *bh = __this_cpu_read(bh_lrus.bhs[i]);
|
2005-04-17 06:20:36 +08:00
|
|
|
|
fs: check bh blocknr earlier when searching lru
It's very common for the buffer heads in the lru to have different block
numbers. By comparing the blocknr before the bdev and size we can
reduce the cost of searching in the very common case where all the
entries have the same bdev and size.
In quick hot cache cycle counting tests on a single fs workstation this
cut the cost of a miss by about 20%.
A diff of the disassembly shows the reordering of the bdev and blocknr
comparisons. This is in such a tiny loop that skipping one comparison
is a meaningful portion of the total work being done:
1628: 83 c1 01 add $0x1,%ecx
162b: 83 f9 08 cmp $0x8,%ecx
162e: 74 60 je 1690 <__find_get_block+0xa0>
1630: 89 c8 mov %ecx,%eax
1632: 65 4c 8b 04 c5 00 00 mov %gs:0x0(,%rax,8),%r8
1639: 00 00
163b: 4d 85 c0 test %r8,%r8
163e: 4c 89 c3 mov %r8,%rbx
1641: 74 e5 je 1628 <__find_get_block+0x38>
- 1643: 4d 3b 68 30 cmp 0x30(%r8),%r13
+ 1643: 4d 3b 68 18 cmp 0x18(%r8),%r13
1647: 75 df jne 1628 <__find_get_block+0x38>
- 1649: 4d 3b 60 18 cmp 0x18(%r8),%r12
+ 1649: 4d 3b 60 30 cmp 0x30(%r8),%r12
164d: 75 d9 jne 1628 <__find_get_block+0x38>
164f: 49 39 50 20 cmp %rdx,0x20(%r8)
1653: 75 d3 jne 1628 <__find_get_block+0x38>
Signed-off-by: Zach Brown <zab@zabbo.net>
Cc: Al Viro <viro@zeniv.linux.org.uk>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
2014-10-14 06:55:05 +08:00
|
|
|
if (bh && bh->b_blocknr == block && bh->b_bdev == bdev &&
|
|
|
|
bh->b_size == size) {
|
2005-04-17 06:20:36 +08:00
|
|
|
if (i) {
|
|
|
|
while (i) {
|
2010-12-07 01:16:28 +08:00
|
|
|
__this_cpu_write(bh_lrus.bhs[i],
|
|
|
|
__this_cpu_read(bh_lrus.bhs[i - 1]));
|
2005-04-17 06:20:36 +08:00
|
|
|
i--;
|
|
|
|
}
|
2010-12-07 01:16:28 +08:00
|
|
|
__this_cpu_write(bh_lrus.bhs[0], bh);
|
2005-04-17 06:20:36 +08:00
|
|
|
}
|
|
|
|
get_bh(bh);
|
|
|
|
ret = bh;
|
|
|
|
break;
|
|
|
|
}
|
|
|
|
}
|
|
|
|
bh_lru_unlock();
|
|
|
|
return ret;
|
|
|
|
}
|
|
|
|
|
|
|
|
/*
|
|
|
|
* Perform a pagecache lookup for the matching buffer. If it's there, refresh
|
|
|
|
* it in the LRU and mark it as accessed. If it is not present then return
|
|
|
|
* NULL
|
|
|
|
*/
|
|
|
|
struct buffer_head *
|
2007-02-12 16:52:14 +08:00
|
|
|
__find_get_block(struct block_device *bdev, sector_t block, unsigned size)
|
2005-04-17 06:20:36 +08:00
|
|
|
{
|
|
|
|
struct buffer_head *bh = lookup_bh_lru(bdev, block, size);
|
|
|
|
|
|
|
|
if (bh == NULL) {
|
2014-06-05 07:10:31 +08:00
|
|
|
/* __find_get_block_slow will mark the page accessed */
|
2005-11-07 16:59:39 +08:00
|
|
|
bh = __find_get_block_slow(bdev, block);
|
2005-04-17 06:20:36 +08:00
|
|
|
if (bh)
|
|
|
|
bh_lru_install(bh);
|
2014-06-05 07:10:31 +08:00
|
|
|
} else
|
2005-04-17 06:20:36 +08:00
|
|
|
touch_buffer(bh);
|
2014-06-05 07:10:31 +08:00
|
|
|
|
2005-04-17 06:20:36 +08:00
|
|
|
return bh;
|
|
|
|
}
|
|
|
|
EXPORT_SYMBOL(__find_get_block);
|
|
|
|
|
2023-09-14 23:00:05 +08:00
|
|
|
/**
|
|
|
|
* bdev_getblk - Get a buffer_head in a block device's buffer cache.
|
|
|
|
* @bdev: The block device.
|
|
|
|
* @block: The block number.
|
|
|
|
* @size: The size of buffer_heads for this @bdev.
|
|
|
|
* @gfp: The memory allocation flags to use.
|
|
|
|
*
|
|
|
|
* Return: The buffer head, or NULL if memory could not be allocated.
|
|
|
|
*/
|
|
|
|
struct buffer_head *bdev_getblk(struct block_device *bdev, sector_t block,
|
|
|
|
unsigned size, gfp_t gfp)
|
|
|
|
{
|
|
|
|
struct buffer_head *bh = __find_get_block(bdev, block, size);
|
|
|
|
|
|
|
|
might_alloc(gfp);
|
|
|
|
if (bh)
|
|
|
|
return bh;
|
|
|
|
|
|
|
|
return __getblk_slow(bdev, block, size, gfp);
|
|
|
|
}
|
|
|
|
EXPORT_SYMBOL(bdev_getblk);
|
|
|
|
|
2005-04-17 06:20:36 +08:00
|
|
|
/*
|
|
|
|
* Do async read-ahead on a buffer..
|
|
|
|
*/
|
2007-02-12 16:52:14 +08:00
|
|
|
void __breadahead(struct block_device *bdev, sector_t block, unsigned size)
|
2005-04-17 06:20:36 +08:00
|
|
|
{
|
2023-09-14 23:00:07 +08:00
|
|
|
struct buffer_head *bh = bdev_getblk(bdev, block, size,
|
|
|
|
GFP_NOWAIT | __GFP_MOVABLE);
|
|
|
|
|
2005-10-31 07:03:15 +08:00
|
|
|
if (likely(bh)) {
|
2022-09-01 21:34:54 +08:00
|
|
|
bh_readahead(bh, REQ_RAHEAD);
|
2005-10-31 07:03:15 +08:00
|
|
|
brelse(bh);
|
|
|
|
}
|
2005-04-17 06:20:36 +08:00
|
|
|
}
|
|
|
|
EXPORT_SYMBOL(__breadahead);
|
|
|
|
|
|
|
|
/**
|
2014-09-05 10:04:42 +08:00
|
|
|
* __bread_gfp() - reads a specified block and returns the bh
|
2005-05-01 23:59:26 +08:00
|
|
|
* @bdev: the block_device to read from
|
2005-04-17 06:20:36 +08:00
|
|
|
* @block: number of block
|
|
|
|
* @size: size (in bytes) to read
|
2014-09-05 10:04:42 +08:00
|
|
|
* @gfp: page allocation flag
|
|
|
|
*
|
2005-04-17 06:20:36 +08:00
|
|
|
* Reads a specified block, and returns buffer head that contains it.
|
2014-09-05 10:04:42 +08:00
|
|
|
* The page cache can be allocated from non-movable area
|
|
|
|
* not to prevent page migration if you set gfp to zero.
|
2005-04-17 06:20:36 +08:00
|
|
|
* It returns NULL if the block was unreadable.
|
|
|
|
*/
|
|
|
|
struct buffer_head *
|
2014-09-05 10:04:42 +08:00
|
|
|
__bread_gfp(struct block_device *bdev, sector_t block,
|
|
|
|
unsigned size, gfp_t gfp)
|
2005-04-17 06:20:36 +08:00
|
|
|
{
|
2023-09-14 23:00:11 +08:00
|
|
|
struct buffer_head *bh;
|
|
|
|
|
|
|
|
gfp |= mapping_gfp_constraint(bdev->bd_inode->i_mapping, ~__GFP_FS);
|
|
|
|
|
|
|
|
/*
|
|
|
|
* Prefer looping in the allocator rather than here, at least that
|
|
|
|
* code knows what it's doing.
|
|
|
|
*/
|
|
|
|
gfp |= __GFP_NOFAIL;
|
|
|
|
|
|
|
|
bh = bdev_getblk(bdev, block, size, gfp);
|
2005-04-17 06:20:36 +08:00
|
|
|
|
2005-10-31 07:03:15 +08:00
|
|
|
if (likely(bh) && !buffer_uptodate(bh))
|
2005-04-17 06:20:36 +08:00
|
|
|
bh = __bread_slow(bh);
|
|
|
|
return bh;
|
|
|
|
}
|
2014-09-05 10:04:42 +08:00
|
|
|
EXPORT_SYMBOL(__bread_gfp);
|
2005-04-17 06:20:36 +08:00
|
|
|
|
2021-05-05 09:37:00 +08:00
|
|
|
static void __invalidate_bh_lrus(struct bh_lru *b)
|
|
|
|
{
|
|
|
|
int i;
|
|
|
|
|
|
|
|
for (i = 0; i < BH_LRU_SIZE; i++) {
|
|
|
|
brelse(b->bhs[i]);
|
|
|
|
b->bhs[i] = NULL;
|
|
|
|
}
|
|
|
|
}
|
2005-04-17 06:20:36 +08:00
|
|
|
/*
|
|
|
|
* invalidate_bh_lrus() is called rarely - but not only at unmount.
|
|
|
|
* This doesn't race because it runs in each cpu either in irq
|
|
|
|
* or with preempt disabled.
|
|
|
|
*/
|
|
|
|
static void invalidate_bh_lru(void *arg)
|
|
|
|
{
|
|
|
|
struct bh_lru *b = &get_cpu_var(bh_lrus);
|
|
|
|
|
2021-05-05 09:37:00 +08:00
|
|
|
__invalidate_bh_lrus(b);
|
2005-04-17 06:20:36 +08:00
|
|
|
put_cpu_var(bh_lrus);
|
|
|
|
}
|
2012-03-29 05:42:45 +08:00
|
|
|
|
2021-05-05 09:37:00 +08:00
|
|
|
bool has_bh_in_lru(int cpu, void *dummy)
|
2012-03-29 05:42:45 +08:00
|
|
|
{
|
|
|
|
struct bh_lru *b = per_cpu_ptr(&bh_lrus, cpu);
|
|
|
|
int i;
|
2005-04-17 06:20:36 +08:00
|
|
|
|
2012-03-29 05:42:45 +08:00
|
|
|
for (i = 0; i < BH_LRU_SIZE; i++) {
|
|
|
|
if (b->bhs[i])
|
2019-12-01 09:49:15 +08:00
|
|
|
return true;
|
2012-03-29 05:42:45 +08:00
|
|
|
}
|
|
|
|
|
2019-12-01 09:49:15 +08:00
|
|
|
return false;
|
2012-03-29 05:42:45 +08:00
|
|
|
}
|
|
|
|
|
2007-05-07 05:49:55 +08:00
|
|
|
void invalidate_bh_lrus(void)
|
2005-04-17 06:20:36 +08:00
|
|
|
{
|
2020-01-17 17:01:37 +08:00
|
|
|
on_each_cpu_cond(has_bh_in_lru, invalidate_bh_lru, NULL, 1);
|
2005-04-17 06:20:36 +08:00
|
|
|
}
|
rewrite rd
This is a rewrite of the ramdisk block device driver.
The old one is really difficult because it effectively implements a block
device which serves data out of its own buffer cache. It relies on the dirty
bit being set, to pin its backing store in cache, however there are non
trivial paths which can clear the dirty bit (eg. try_to_free_buffers()),
which had recently lead to data corruption. And in general it is completely
wrong for a block device driver to do this.
The new one is more like a regular block device driver. It has no idea about
vm/vfs stuff. It's backing store is similar to the buffer cache (a simple
radix-tree of pages), but it doesn't know anything about page cache (the pages
in the radix tree are not pagecache pages).
There is one slight downside -- direct block device access and filesystem
metadata access goes through an extra copy and gets stored in RAM twice.
However, this downside is only slight, because the real buffercache of the
device is now reclaimable (because we're not playing crazy games with it), so
under memory intensive situations, footprint should effectively be the same --
maybe even a slight advantage to the new driver because it can also reclaim
buffer heads.
The fact that it now goes through all the regular vm/fs paths makes it
much more useful for testing, too.
text data bss dec hex filename
2837 849 384 4070 fe6 drivers/block/rd.o
3528 371 12 3911 f47 drivers/block/brd.o
Text is larger, but data and bss are smaller, making total size smaller.
A few other nice things about it:
- Similar structure and layout to the new loop device handlinag.
- Dynamic ramdisk creation.
- Runtime flexible buffer head size (because it is no longer part of the
ramdisk code).
- Boot / load time flexible ramdisk size, which could easily be extended
to a per-ramdisk runtime changeable size (eg. with an ioctl).
- Can use highmem for the backing store.
[akpm@linux-foundation.org: fix build]
[byron.bbradley@gmail.com: make rd_size non-static]
Signed-off-by: Nick Piggin <npiggin@suse.de>
Signed-off-by: Byron Bradley <byron.bbradley@gmail.com>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
2008-02-08 20:19:49 +08:00
|
|
|
EXPORT_SYMBOL_GPL(invalidate_bh_lrus);
|
2005-04-17 06:20:36 +08:00
|
|
|
|
2021-09-25 06:43:47 +08:00
|
|
|
/*
|
|
|
|
* It's called from workqueue context so we need a bh_lru_lock to close
|
|
|
|
* the race with preemption/irq.
|
|
|
|
*/
|
|
|
|
void invalidate_bh_lrus_cpu(void)
|
2021-05-05 09:37:00 +08:00
|
|
|
{
|
|
|
|
struct bh_lru *b;
|
|
|
|
|
|
|
|
bh_lru_lock();
|
2021-09-25 06:43:47 +08:00
|
|
|
b = this_cpu_ptr(&bh_lrus);
|
2021-05-05 09:37:00 +08:00
|
|
|
__invalidate_bh_lrus(b);
|
|
|
|
bh_lru_unlock();
|
|
|
|
}
|
|
|
|
|
2023-04-17 20:36:15 +08:00
|
|
|
void folio_set_bh(struct buffer_head *bh, struct folio *folio,
|
|
|
|
unsigned long offset)
|
|
|
|
{
|
|
|
|
bh->b_folio = folio;
|
|
|
|
BUG_ON(offset >= folio_size(folio));
|
|
|
|
if (folio_test_highmem(folio))
|
|
|
|
/*
|
|
|
|
* This catches illegal uses and preserves the offset:
|
|
|
|
*/
|
|
|
|
bh->b_data = (char *)(0 + offset);
|
|
|
|
else
|
|
|
|
bh->b_data = folio_address(folio) + offset;
|
|
|
|
}
|
|
|
|
EXPORT_SYMBOL(folio_set_bh);
|
|
|
|
|
2005-04-17 06:20:36 +08:00
|
|
|
/*
|
|
|
|
* Called when truncating a buffer on a page completely.
|
|
|
|
*/
|
2014-06-05 07:10:29 +08:00
|
|
|
|
|
|
|
/* Bits that are cleared during an invalidate */
|
|
|
|
#define BUFFER_FLAGS_DISCARD \
|
|
|
|
(1 << BH_Mapped | 1 << BH_New | 1 << BH_Req | \
|
|
|
|
1 << BH_Delay | 1 << BH_Unwritten)
|
|
|
|
|
2006-01-15 05:20:43 +08:00
|
|
|
static void discard_buffer(struct buffer_head * bh)
|
2005-04-17 06:20:36 +08:00
|
|
|
{
|
2022-07-15 01:16:53 +08:00
|
|
|
unsigned long b_state;
|
2014-06-05 07:10:29 +08:00
|
|
|
|
2005-04-17 06:20:36 +08:00
|
|
|
lock_buffer(bh);
|
|
|
|
clear_buffer_dirty(bh);
|
|
|
|
bh->b_bdev = NULL;
|
2022-07-15 01:16:53 +08:00
|
|
|
b_state = READ_ONCE(bh->b_state);
|
|
|
|
do {
|
|
|
|
} while (!try_cmpxchg(&bh->b_state, &b_state,
|
|
|
|
b_state & ~BUFFER_FLAGS_DISCARD));
|
2005-04-17 06:20:36 +08:00
|
|
|
unlock_buffer(bh);
|
|
|
|
}
|
|
|
|
|
|
|
|
/**
|
2022-02-10 04:21:34 +08:00
|
|
|
* block_invalidate_folio - Invalidate part or all of a buffer-backed folio.
|
|
|
|
* @folio: The folio which is affected.
|
2013-05-22 11:17:23 +08:00
|
|
|
* @offset: start of the range to invalidate
|
|
|
|
* @length: length of the range to invalidate
|
2005-04-17 06:20:36 +08:00
|
|
|
*
|
2022-02-10 04:21:34 +08:00
|
|
|
* block_invalidate_folio() is called when all or part of the folio has been
|
2011-09-01 08:22:57 +08:00
|
|
|
* invalidated by a truncate operation.
|
2005-04-17 06:20:36 +08:00
|
|
|
*
|
2022-02-10 04:21:34 +08:00
|
|
|
* block_invalidate_folio() does not have to release all buffers, but it must
|
2005-04-17 06:20:36 +08:00
|
|
|
* ensure that no dirty buffer is left outside @offset and that no I/O
|
|
|
|
* is underway against any of the blocks which are outside the truncation
|
|
|
|
* point. Because the caller is about to free (and possibly reuse) those
|
|
|
|
* blocks on-disk.
|
|
|
|
*/
|
2022-02-10 04:21:34 +08:00
|
|
|
void block_invalidate_folio(struct folio *folio, size_t offset, size_t length)
|
2005-04-17 06:20:36 +08:00
|
|
|
{
|
|
|
|
struct buffer_head *head, *bh, *next;
|
2022-02-10 04:21:34 +08:00
|
|
|
size_t curr_off = 0;
|
|
|
|
size_t stop = length + offset;
|
2005-04-17 06:20:36 +08:00
|
|
|
|
2022-02-10 04:21:34 +08:00
|
|
|
BUG_ON(!folio_test_locked(folio));
|
2005-04-17 06:20:36 +08:00
|
|
|
|
2013-05-22 11:17:23 +08:00
|
|
|
/*
|
|
|
|
* Check for overflow
|
|
|
|
*/
|
2022-02-10 04:21:34 +08:00
|
|
|
BUG_ON(stop > folio_size(folio) || stop < length);
|
|
|
|
|
|
|
|
head = folio_buffers(folio);
|
|
|
|
if (!head)
|
|
|
|
return;
|
2013-05-22 11:17:23 +08:00
|
|
|
|
2005-04-17 06:20:36 +08:00
|
|
|
bh = head;
|
|
|
|
do {
|
2022-02-10 04:21:34 +08:00
|
|
|
size_t next_off = curr_off + bh->b_size;
|
2005-04-17 06:20:36 +08:00
|
|
|
next = bh->b_this_page;
|
|
|
|
|
2013-05-22 11:17:23 +08:00
|
|
|
/*
|
|
|
|
* Are we still fully in range ?
|
|
|
|
*/
|
|
|
|
if (next_off > stop)
|
|
|
|
goto out;
|
|
|
|
|
2005-04-17 06:20:36 +08:00
|
|
|
/*
|
|
|
|
* is this block fully invalidated?
|
|
|
|
*/
|
|
|
|
if (offset <= curr_off)
|
|
|
|
discard_buffer(bh);
|
|
|
|
curr_off = next_off;
|
|
|
|
bh = next;
|
|
|
|
} while (bh != head);
|
|
|
|
|
|
|
|
/*
|
2022-02-10 04:21:34 +08:00
|
|
|
* We release buffers only if the entire folio is being invalidated.
|
2005-04-17 06:20:36 +08:00
|
|
|
* The get_block cached value has been unconditionally invalidated,
|
|
|
|
* so real IO is not possible anymore.
|
|
|
|
*/
|
2022-02-10 04:21:34 +08:00
|
|
|
if (length == folio_size(folio))
|
|
|
|
filemap_release_folio(folio, 0);
|
2005-04-17 06:20:36 +08:00
|
|
|
out:
|
2006-03-26 17:37:18 +08:00
|
|
|
return;
|
2005-04-17 06:20:36 +08:00
|
|
|
}
|
2022-02-10 04:21:34 +08:00
|
|
|
EXPORT_SYMBOL(block_invalidate_folio);
|
2005-04-17 06:20:36 +08:00
|
|
|
|
|
|
|
/*
|
|
|
|
* We attach and possibly dirty the buffers atomically wrt
|
2023-11-18 05:58:23 +08:00
|
|
|
* block_dirty_folio() via i_private_lock. try_to_free_buffers
|
2023-04-17 20:36:17 +08:00
|
|
|
* is already excluded via the folio lock.
|
2005-04-17 06:20:36 +08:00
|
|
|
*/
|
2023-10-17 04:11:14 +08:00
|
|
|
struct buffer_head *create_empty_buffers(struct folio *folio,
|
2023-10-17 04:10:49 +08:00
|
|
|
unsigned long blocksize, unsigned long b_state)
|
2005-04-17 06:20:36 +08:00
|
|
|
{
|
|
|
|
struct buffer_head *bh, *head, *tail;
|
2023-09-14 23:00:04 +08:00
|
|
|
gfp_t gfp = GFP_NOFS | __GFP_ACCOUNT | __GFP_NOFAIL;
|
2005-04-17 06:20:36 +08:00
|
|
|
|
2023-09-14 23:00:04 +08:00
|
|
|
head = folio_alloc_buffers(folio, blocksize, gfp);
|
2005-04-17 06:20:36 +08:00
|
|
|
bh = head;
|
|
|
|
do {
|
|
|
|
bh->b_state |= b_state;
|
|
|
|
tail = bh;
|
|
|
|
bh = bh->b_this_page;
|
|
|
|
} while (bh);
|
|
|
|
tail->b_this_page = head;
|
|
|
|
|
2023-11-18 05:58:23 +08:00
|
|
|
spin_lock(&folio->mapping->i_private_lock);
|
2023-04-17 20:36:17 +08:00
|
|
|
if (folio_test_uptodate(folio) || folio_test_dirty(folio)) {
|
2005-04-17 06:20:36 +08:00
|
|
|
bh = head;
|
|
|
|
do {
|
2023-04-17 20:36:17 +08:00
|
|
|
if (folio_test_dirty(folio))
|
2005-04-17 06:20:36 +08:00
|
|
|
set_buffer_dirty(bh);
|
2023-04-17 20:36:17 +08:00
|
|
|
if (folio_test_uptodate(folio))
|
2005-04-17 06:20:36 +08:00
|
|
|
set_buffer_uptodate(bh);
|
|
|
|
bh = bh->b_this_page;
|
|
|
|
} while (bh != head);
|
|
|
|
}
|
2023-04-17 20:36:17 +08:00
|
|
|
folio_attach_private(folio, head);
|
2023-11-18 05:58:23 +08:00
|
|
|
spin_unlock(&folio->mapping->i_private_lock);
|
2023-10-17 04:10:49 +08:00
|
|
|
|
|
|
|
return head;
|
2023-04-17 20:36:17 +08:00
|
|
|
}
|
2005-04-17 06:20:36 +08:00
|
|
|
EXPORT_SYMBOL(create_empty_buffers);
|
|
|
|
|
2016-11-05 01:08:11 +08:00
|
|
|
/**
|
|
|
|
* clean_bdev_aliases: clean a range of buffers in block device
|
|
|
|
* @bdev: Block device to clean buffers in
|
|
|
|
* @block: Start of a range of blocks to clean
|
|
|
|
* @len: Number of blocks to clean
|
2005-04-17 06:20:36 +08:00
|
|
|
*
|
2016-11-05 01:08:11 +08:00
|
|
|
* We are taking a range of blocks for data and we don't want writeback of any
|
|
|
|
* buffer-cache aliases starting from return from this function and until the
|
|
|
|
* moment when something will explicitly mark the buffer dirty (hopefully that
|
|
|
|
* will not happen until we will free that block ;-) We don't even need to mark
|
|
|
|
* it not-uptodate - nobody can expect anything from a newly allocated buffer
|
|
|
|
* anyway. We used to use unmap_buffer() for such invalidation, but that was
|
|
|
|
* wrong. We definitely don't want to mark the alias unmapped, for example - it
|
|
|
|
* would confuse anyone who might pick it with bread() afterwards...
|
|
|
|
*
|
|
|
|
* Also.. Note that bforget() doesn't lock the buffer. So there can be
|
|
|
|
* writeout I/O going on against recently-freed buffers. We don't wait on that
|
|
|
|
* I/O in bforget() - it's more efficient to wait on the I/O only if we really
|
|
|
|
* need to. That happens here.
|
2005-04-17 06:20:36 +08:00
|
|
|
*/
|
2016-11-05 01:08:11 +08:00
|
|
|
void clean_bdev_aliases(struct block_device *bdev, sector_t block, sector_t len)
|
2005-04-17 06:20:36 +08:00
|
|
|
{
|
2016-11-05 01:08:11 +08:00
|
|
|
struct inode *bd_inode = bdev->bd_inode;
|
|
|
|
struct address_space *bd_mapping = bd_inode->i_mapping;
|
2022-06-05 04:24:22 +08:00
|
|
|
struct folio_batch fbatch;
|
2023-11-10 05:06:06 +08:00
|
|
|
pgoff_t index = ((loff_t)block << bd_inode->i_blkbits) / PAGE_SIZE;
|
2016-11-05 01:08:11 +08:00
|
|
|
pgoff_t end;
|
2017-09-07 07:21:24 +08:00
|
|
|
int i, count;
|
2016-11-05 01:08:11 +08:00
|
|
|
struct buffer_head *bh;
|
|
|
|
struct buffer_head *head;
|
2005-04-17 06:20:36 +08:00
|
|
|
|
2023-11-10 05:06:06 +08:00
|
|
|
end = ((loff_t)(block + len - 1) << bd_inode->i_blkbits) / PAGE_SIZE;
|
2022-06-05 04:24:22 +08:00
|
|
|
folio_batch_init(&fbatch);
|
|
|
|
while (filemap_get_folios(bd_mapping, &index, end, &fbatch)) {
|
|
|
|
count = folio_batch_count(&fbatch);
|
2017-09-07 07:21:24 +08:00
|
|
|
for (i = 0; i < count; i++) {
|
2022-06-05 04:24:22 +08:00
|
|
|
struct folio *folio = fbatch.folios[i];
|
2005-04-17 06:20:36 +08:00
|
|
|
|
2022-06-05 04:24:22 +08:00
|
|
|
if (!folio_buffers(folio))
|
2016-11-05 01:08:11 +08:00
|
|
|
continue;
|
|
|
|
/*
|
2023-11-18 05:58:23 +08:00
|
|
|
* We use folio lock instead of bd_mapping->i_private_lock
|
2016-11-05 01:08:11 +08:00
|
|
|
* to pin buffers here since we can afford to sleep and
|
|
|
|
* it scales better than a global spinlock lock.
|
|
|
|
*/
|
2022-06-05 04:24:22 +08:00
|
|
|
folio_lock(folio);
|
|
|
|
/* Recheck when the folio is locked which pins bhs */
|
|
|
|
head = folio_buffers(folio);
|
|
|
|
if (!head)
|
2016-11-05 01:08:11 +08:00
|
|
|
goto unlock_page;
|
|
|
|
bh = head;
|
|
|
|
do {
|
2016-12-25 21:31:03 +08:00
|
|
|
if (!buffer_mapped(bh) || (bh->b_blocknr < block))
|
2016-11-05 01:08:11 +08:00
|
|
|
goto next;
|
|
|
|
if (bh->b_blocknr >= block + len)
|
|
|
|
break;
|
|
|
|
clear_buffer_dirty(bh);
|
|
|
|
wait_on_buffer(bh);
|
|
|
|
clear_buffer_req(bh);
|
|
|
|
next:
|
|
|
|
bh = bh->b_this_page;
|
|
|
|
} while (bh != head);
|
|
|
|
unlock_page:
|
2022-06-05 04:24:22 +08:00
|
|
|
folio_unlock(folio);
|
2016-11-05 01:08:11 +08:00
|
|
|
}
|
2022-06-05 04:24:22 +08:00
|
|
|
folio_batch_release(&fbatch);
|
2016-11-05 01:08:11 +08:00
|
|
|
cond_resched();
|
2017-09-07 07:21:24 +08:00
|
|
|
/* End of range already reached? */
|
|
|
|
if (index > end || !index)
|
|
|
|
break;
|
2005-04-17 06:20:36 +08:00
|
|
|
}
|
|
|
|
}
|
2016-11-05 01:08:11 +08:00
|
|
|
EXPORT_SYMBOL(clean_bdev_aliases);
|
2005-04-17 06:20:36 +08:00
|
|
|
|
2023-04-17 20:36:18 +08:00
|
|
|
static struct buffer_head *folio_create_buffers(struct folio *folio,
|
|
|
|
struct inode *inode,
|
|
|
|
unsigned int b_state)
|
2012-11-30 02:21:43 +08:00
|
|
|
{
|
2023-10-17 04:10:49 +08:00
|
|
|
struct buffer_head *bh;
|
|
|
|
|
2023-04-17 20:36:18 +08:00
|
|
|
BUG_ON(!folio_test_locked(folio));
|
2012-11-30 02:21:43 +08:00
|
|
|
|
2023-10-17 04:10:49 +08:00
|
|
|
bh = folio_buffers(folio);
|
|
|
|
if (!bh)
|
2023-10-17 04:11:14 +08:00
|
|
|
bh = create_empty_buffers(folio,
|
2023-10-17 04:10:49 +08:00
|
|
|
1 << READ_ONCE(inode->i_blkbits), b_state);
|
|
|
|
return bh;
|
2012-11-30 02:21:43 +08:00
|
|
|
}
|
|
|
|
|
2005-04-17 06:20:36 +08:00
|
|
|
/*
|
|
|
|
* NOTE! All mapped/uptodate combinations are valid:
|
|
|
|
*
|
|
|
|
* Mapped Uptodate Meaning
|
|
|
|
*
|
|
|
|
* No No "unknown" - must do get_block()
|
|
|
|
* No Yes "hole" - zero-filled
|
|
|
|
* Yes No "allocated" - allocated on disk, not read in
|
|
|
|
* Yes Yes "valid" - allocated and up-to-date in memory.
|
|
|
|
*
|
|
|
|
* "Dirty" is valid only with the last case (mapped+uptodate).
|
|
|
|
*/
|
|
|
|
|
|
|
|
/*
|
2023-12-16 04:02:44 +08:00
|
|
|
* While block_write_full_folio is writing back the dirty buffers under
|
2005-04-17 06:20:36 +08:00
|
|
|
* the page lock, whoever dirtied the buffers may decide to clean them
|
|
|
|
* again at any time. We handle that by only looking at the buffer
|
|
|
|
* state inside lock_buffer().
|
|
|
|
*
|
2023-12-16 04:02:44 +08:00
|
|
|
* If block_write_full_folio() is called for regular writeback
|
2005-04-17 06:20:36 +08:00
|
|
|
* (wbc->sync_mode == WB_SYNC_NONE) then it will redirty a page which has a
|
|
|
|
* locked buffer. This only can happen if someone has written the buffer
|
|
|
|
* directly, with submit_bh(). At the address_space level PageWriteback
|
|
|
|
* prevents this contention from occurring.
|
2009-04-08 06:12:43 +08:00
|
|
|
*
|
2023-12-16 04:02:44 +08:00
|
|
|
* If block_write_full_folio() is called with wbc->sync_mode ==
|
2016-11-01 21:40:10 +08:00
|
|
|
* WB_SYNC_ALL, the writes are posted using REQ_SYNC; this
|
2011-03-09 18:56:30 +08:00
|
|
|
* causes the writes to be flagged as synchronous writes.
|
2005-04-17 06:20:36 +08:00
|
|
|
*/
|
2023-06-13 05:01:31 +08:00
|
|
|
int __block_write_full_folio(struct inode *inode, struct folio *folio,
|
2023-12-16 04:02:45 +08:00
|
|
|
get_block_t *get_block, struct writeback_control *wbc)
|
2005-04-17 06:20:36 +08:00
|
|
|
{
|
|
|
|
int err;
|
|
|
|
sector_t block;
|
|
|
|
sector_t last_block;
|
2005-05-06 07:15:48 +08:00
|
|
|
struct buffer_head *bh, *head;
|
2023-11-10 05:06:08 +08:00
|
|
|
size_t blocksize;
|
2005-04-17 06:20:36 +08:00
|
|
|
int nr_underway = 0;
|
2022-07-15 02:07:12 +08:00
|
|
|
blk_opf_t write_flags = wbc_to_write_flags(wbc);
|
2005-04-17 06:20:36 +08:00
|
|
|
|
2023-06-13 05:01:31 +08:00
|
|
|
head = folio_create_buffers(folio, inode,
|
2023-04-17 20:36:18 +08:00
|
|
|
(1 << BH_Dirty) | (1 << BH_Uptodate));
|
2005-04-17 06:20:36 +08:00
|
|
|
|
|
|
|
/*
|
2022-02-10 04:22:12 +08:00
|
|
|
* Be very careful. We have no exclusion from block_dirty_folio
|
2005-04-17 06:20:36 +08:00
|
|
|
* here, and the (potentially unmapped) buffers may become dirty at
|
|
|
|
* any time. If a buffer becomes dirty here after we've inspected it
|
2023-06-13 05:01:31 +08:00
|
|
|
* then we just miss that fact, and the folio stays dirty.
|
2005-04-17 06:20:36 +08:00
|
|
|
*
|
2022-02-10 04:22:12 +08:00
|
|
|
* Buffers outside i_size may be dirtied by block_dirty_folio;
|
2005-04-17 06:20:36 +08:00
|
|
|
* handle that here by just cleaning them.
|
|
|
|
*/
|
|
|
|
|
|
|
|
bh = head;
|
2012-11-30 02:21:43 +08:00
|
|
|
blocksize = bh->b_size;
|
|
|
|
|
2023-11-10 05:06:08 +08:00
|
|
|
block = div_u64(folio_pos(folio), blocksize);
|
|
|
|
last_block = div_u64(i_size_read(inode) - 1, blocksize);
|
2005-04-17 06:20:36 +08:00
|
|
|
|
|
|
|
/*
|
|
|
|
* Get all the dirty buffers mapped to disk addresses and
|
|
|
|
* handle any aliases from the underlying blockdev's mapping.
|
|
|
|
*/
|
|
|
|
do {
|
|
|
|
if (block > last_block) {
|
|
|
|
/*
|
|
|
|
* mapped buffers outside i_size will occur, because
|
2023-06-13 05:01:31 +08:00
|
|
|
* this folio can be outside i_size when there is a
|
2005-04-17 06:20:36 +08:00
|
|
|
* truncate in progress.
|
|
|
|
*/
|
|
|
|
/*
|
2023-12-16 04:02:44 +08:00
|
|
|
* The buffer was zeroed by block_write_full_folio()
|
2005-04-17 06:20:36 +08:00
|
|
|
*/
|
|
|
|
clear_buffer_dirty(bh);
|
|
|
|
set_buffer_uptodate(bh);
|
2008-07-12 07:27:31 +08:00
|
|
|
} else if ((!buffer_mapped(bh) || buffer_delay(bh)) &&
|
|
|
|
buffer_dirty(bh)) {
|
2006-03-26 17:38:00 +08:00
|
|
|
WARN_ON(bh->b_size != blocksize);
|
2005-04-17 06:20:36 +08:00
|
|
|
err = get_block(inode, block, bh, 1);
|
|
|
|
if (err)
|
|
|
|
goto recover;
|
2008-07-12 07:27:31 +08:00
|
|
|
clear_buffer_delay(bh);
|
2005-04-17 06:20:36 +08:00
|
|
|
if (buffer_new(bh)) {
|
|
|
|
/* blockdev mappings never come here */
|
|
|
|
clear_buffer_new(bh);
|
2016-11-05 01:08:15 +08:00
|
|
|
clean_bdev_bh_alias(bh);
|
2005-04-17 06:20:36 +08:00
|
|
|
}
|
|
|
|
}
|
|
|
|
bh = bh->b_this_page;
|
|
|
|
block++;
|
|
|
|
} while (bh != head);
|
|
|
|
|
|
|
|
do {
|
|
|
|
if (!buffer_mapped(bh))
|
|
|
|
continue;
|
|
|
|
/*
|
|
|
|
* If it's a fully non-blocking write attempt and we cannot
|
2023-06-13 05:01:31 +08:00
|
|
|
* lock the buffer then redirty the folio. Note that this can
|
2009-09-24 01:37:09 +08:00
|
|
|
* potentially cause a busy-wait loop from writeback threads
|
|
|
|
* and kswapd activity, but those code paths have their own
|
|
|
|
* higher-level throttling.
|
2005-04-17 06:20:36 +08:00
|
|
|
*/
|
2010-10-27 05:21:26 +08:00
|
|
|
if (wbc->sync_mode != WB_SYNC_NONE) {
|
2005-04-17 06:20:36 +08:00
|
|
|
lock_buffer(bh);
|
2008-08-02 18:02:13 +08:00
|
|
|
} else if (!trylock_buffer(bh)) {
|
2023-06-13 05:01:31 +08:00
|
|
|
folio_redirty_for_writepage(wbc, folio);
|
2005-04-17 06:20:36 +08:00
|
|
|
continue;
|
|
|
|
}
|
|
|
|
if (test_clear_buffer_dirty(bh)) {
|
2023-12-16 04:02:45 +08:00
|
|
|
mark_buffer_async_write_endio(bh,
|
|
|
|
end_buffer_async_write);
|
2005-04-17 06:20:36 +08:00
|
|
|
} else {
|
|
|
|
unlock_buffer(bh);
|
|
|
|
}
|
|
|
|
} while ((bh = bh->b_this_page) != head);
|
|
|
|
|
|
|
|
/*
|
2023-06-13 05:01:31 +08:00
|
|
|
* The folio and its buffers are protected by the writeback flag,
|
|
|
|
* so we can drop the bh refcounts early.
|
2005-04-17 06:20:36 +08:00
|
|
|
*/
|
2023-06-13 05:01:31 +08:00
|
|
|
BUG_ON(folio_test_writeback(folio));
|
|
|
|
folio_start_writeback(folio);
|
2005-04-17 06:20:36 +08:00
|
|
|
|
|
|
|
do {
|
|
|
|
struct buffer_head *next = bh->b_this_page;
|
|
|
|
if (buffer_async_write(bh)) {
|
2024-02-03 04:39:25 +08:00
|
|
|
submit_bh_wbc(REQ_OP_WRITE | write_flags, bh,
|
|
|
|
inode->i_write_hint, wbc);
|
2005-04-17 06:20:36 +08:00
|
|
|
nr_underway++;
|
|
|
|
}
|
|
|
|
bh = next;
|
|
|
|
} while (bh != head);
|
2023-06-13 05:01:31 +08:00
|
|
|
folio_unlock(folio);
|
2005-04-17 06:20:36 +08:00
|
|
|
|
|
|
|
err = 0;
|
|
|
|
done:
|
|
|
|
if (nr_underway == 0) {
|
|
|
|
/*
|
2023-06-13 05:01:31 +08:00
|
|
|
* The folio was marked dirty, but the buffers were
|
2005-04-17 06:20:36 +08:00
|
|
|
* clean. Someone wrote them back by hand with
|
2022-09-01 21:35:03 +08:00
|
|
|
* write_dirty_buffer/submit_bh. A rare case.
|
2005-04-17 06:20:36 +08:00
|
|
|
*/
|
2023-06-13 05:01:31 +08:00
|
|
|
folio_end_writeback(folio);
|
2007-05-07 05:49:05 +08:00
|
|
|
|
2005-04-17 06:20:36 +08:00
|
|
|
/*
|
2023-06-13 05:01:31 +08:00
|
|
|
* The folio and buffer_heads can be released at any time from
|
2005-04-17 06:20:36 +08:00
|
|
|
* here on.
|
|
|
|
*/
|
|
|
|
}
|
|
|
|
return err;
|
|
|
|
|
|
|
|
recover:
|
|
|
|
/*
|
|
|
|
* ENOSPC, or some other error. We may already have added some
|
|
|
|
* blocks to the file, so we need to write these out to avoid
|
|
|
|
* exposing stale data.
|
2023-06-13 05:01:31 +08:00
|
|
|
* The folio is currently locked and not marked for writeback
|
2005-04-17 06:20:36 +08:00
|
|
|
*/
|
|
|
|
bh = head;
|
|
|
|
/* Recovery: lock and submit the mapped buffers */
|
|
|
|
do {
|
2008-07-12 07:27:31 +08:00
|
|
|
if (buffer_mapped(bh) && buffer_dirty(bh) &&
|
|
|
|
!buffer_delay(bh)) {
|
2005-04-17 06:20:36 +08:00
|
|
|
lock_buffer(bh);
|
2023-12-16 04:02:45 +08:00
|
|
|
mark_buffer_async_write_endio(bh,
|
|
|
|
end_buffer_async_write);
|
2005-04-17 06:20:36 +08:00
|
|
|
} else {
|
|
|
|
/*
|
|
|
|
* The buffer may have been set dirty during
|
2023-06-13 05:01:31 +08:00
|
|
|
* attachment to a dirty folio.
|
2005-04-17 06:20:36 +08:00
|
|
|
*/
|
|
|
|
clear_buffer_dirty(bh);
|
|
|
|
}
|
|
|
|
} while ((bh = bh->b_this_page) != head);
|
2023-06-13 05:01:31 +08:00
|
|
|
folio_set_error(folio);
|
|
|
|
BUG_ON(folio_test_writeback(folio));
|
|
|
|
mapping_set_error(folio->mapping, err);
|
|
|
|
folio_start_writeback(folio);
|
2005-04-17 06:20:36 +08:00
|
|
|
do {
|
|
|
|
struct buffer_head *next = bh->b_this_page;
|
|
|
|
if (buffer_async_write(bh)) {
|
|
|
|
clear_buffer_dirty(bh);
|
2024-02-03 04:39:25 +08:00
|
|
|
submit_bh_wbc(REQ_OP_WRITE | write_flags, bh,
|
|
|
|
inode->i_write_hint, wbc);
|
2005-04-17 06:20:36 +08:00
|
|
|
nr_underway++;
|
|
|
|
}
|
|
|
|
bh = next;
|
|
|
|
} while (bh != head);
|
2023-06-13 05:01:31 +08:00
|
|
|
folio_unlock(folio);
|
2005-04-17 06:20:36 +08:00
|
|
|
goto done;
|
|
|
|
}
|
2023-06-13 05:01:31 +08:00
|
|
|
EXPORT_SYMBOL(__block_write_full_folio);
|
2005-04-17 06:20:36 +08:00
|
|
|
|
2007-10-16 16:25:01 +08:00
|
|
|
/*
|
2023-06-13 05:01:36 +08:00
|
|
|
* If a folio has any new buffers, zero them out here, and mark them uptodate
|
2007-10-16 16:25:01 +08:00
|
|
|
* and dirty so they'll be written out (in order to prevent uninitialised
|
|
|
|
* block data from leaking). And clear the new bit.
|
|
|
|
*/
|
2023-06-13 05:01:36 +08:00
|
|
|
void folio_zero_new_buffers(struct folio *folio, size_t from, size_t to)
|
2007-10-16 16:25:01 +08:00
|
|
|
{
|
2023-06-13 05:01:36 +08:00
|
|
|
size_t block_start, block_end;
|
2007-10-16 16:25:01 +08:00
|
|
|
struct buffer_head *head, *bh;
|
|
|
|
|
2023-06-13 05:01:36 +08:00
|
|
|
BUG_ON(!folio_test_locked(folio));
|
|
|
|
head = folio_buffers(folio);
|
|
|
|
if (!head)
|
2007-10-16 16:25:01 +08:00
|
|
|
return;
|
|
|
|
|
2023-06-13 05:01:36 +08:00
|
|
|
bh = head;
|
2007-10-16 16:25:01 +08:00
|
|
|
block_start = 0;
|
|
|
|
do {
|
|
|
|
block_end = block_start + bh->b_size;
|
|
|
|
|
|
|
|
if (buffer_new(bh)) {
|
|
|
|
if (block_end > from && block_start < to) {
|
2023-06-13 05:01:36 +08:00
|
|
|
if (!folio_test_uptodate(folio)) {
|
|
|
|
size_t start, xend;
|
2007-10-16 16:25:01 +08:00
|
|
|
|
|
|
|
start = max(from, block_start);
|
2023-06-13 05:01:36 +08:00
|
|
|
xend = min(to, block_end);
|
2007-10-16 16:25:01 +08:00
|
|
|
|
2023-06-13 05:01:36 +08:00
|
|
|
folio_zero_segment(folio, start, xend);
|
2007-10-16 16:25:01 +08:00
|
|
|
set_buffer_uptodate(bh);
|
|
|
|
}
|
|
|
|
|
|
|
|
clear_buffer_new(bh);
|
|
|
|
mark_buffer_dirty(bh);
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
block_start = block_end;
|
|
|
|
bh = bh->b_this_page;
|
|
|
|
} while (bh != head);
|
|
|
|
}
|
2023-06-13 05:01:36 +08:00
|
|
|
EXPORT_SYMBOL(folio_zero_new_buffers);
|
2007-10-16 16:25:01 +08:00
|
|
|
|
2023-09-13 01:05:48 +08:00
|
|
|
static int
|
2016-06-21 07:23:11 +08:00
|
|
|
iomap_to_bh(struct inode *inode, sector_t block, struct buffer_head *bh,
|
2021-08-11 09:33:05 +08:00
|
|
|
const struct iomap *iomap)
|
2016-06-21 07:23:11 +08:00
|
|
|
{
|
2023-11-10 05:06:05 +08:00
|
|
|
loff_t offset = (loff_t)block << inode->i_blkbits;
|
2016-06-21 07:23:11 +08:00
|
|
|
|
|
|
|
bh->b_bdev = iomap->bdev;
|
|
|
|
|
|
|
|
/*
|
|
|
|
* Block points to offset in file we need to map, iomap contains
|
|
|
|
* the offset at which the map starts. If the map ends before the
|
|
|
|
* current block, then do not map the buffer and let the caller
|
|
|
|
* handle it.
|
|
|
|
*/
|
2023-09-13 01:05:48 +08:00
|
|
|
if (offset >= iomap->offset + iomap->length)
|
|
|
|
return -EIO;
|
2016-06-21 07:23:11 +08:00
|
|
|
|
|
|
|
switch (iomap->type) {
|
|
|
|
case IOMAP_HOLE:
|
|
|
|
/*
|
|
|
|
* If the buffer is not up to date or beyond the current EOF,
|
|
|
|
* we need to mark it as new to ensure sub-block zeroing is
|
|
|
|
* executed if necessary.
|
|
|
|
*/
|
|
|
|
if (!buffer_uptodate(bh) ||
|
|
|
|
(offset >= i_size_read(inode)))
|
|
|
|
set_buffer_new(bh);
|
2023-09-13 01:05:48 +08:00
|
|
|
return 0;
|
2016-06-21 07:23:11 +08:00
|
|
|
case IOMAP_DELALLOC:
|
|
|
|
if (!buffer_uptodate(bh) ||
|
|
|
|
(offset >= i_size_read(inode)))
|
|
|
|
set_buffer_new(bh);
|
|
|
|
set_buffer_uptodate(bh);
|
|
|
|
set_buffer_mapped(bh);
|
|
|
|
set_buffer_delay(bh);
|
2023-09-13 01:05:48 +08:00
|
|
|
return 0;
|
2016-06-21 07:23:11 +08:00
|
|
|
case IOMAP_UNWRITTEN:
|
|
|
|
/*
|
2018-06-20 06:10:55 +08:00
|
|
|
* For unwritten regions, we always need to ensure that regions
|
|
|
|
* in the block we are not writing to are zeroed. Mark the
|
|
|
|
* buffer as new to ensure this.
|
2016-06-21 07:23:11 +08:00
|
|
|
*/
|
|
|
|
set_buffer_new(bh);
|
|
|
|
set_buffer_unwritten(bh);
|
2020-08-24 06:36:59 +08:00
|
|
|
fallthrough;
|
2016-06-21 07:23:11 +08:00
|
|
|
case IOMAP_MAPPED:
|
2018-06-20 06:10:55 +08:00
|
|
|
if ((iomap->flags & IOMAP_F_NEW) ||
|
2023-09-25 23:54:45 +08:00
|
|
|
offset >= i_size_read(inode)) {
|
|
|
|
/*
|
|
|
|
* This can happen if truncating the block device races
|
|
|
|
* with the check in the caller as i_size updates on
|
|
|
|
* block devices aren't synchronized by i_rwsem for
|
|
|
|
* block devices.
|
|
|
|
*/
|
|
|
|
if (S_ISBLK(inode->i_mode))
|
|
|
|
return -EIO;
|
2016-06-21 07:23:11 +08:00
|
|
|
set_buffer_new(bh);
|
2023-09-25 23:54:45 +08:00
|
|
|
}
|
2017-10-02 05:55:54 +08:00
|
|
|
bh->b_blocknr = (iomap->addr + offset - iomap->offset) >>
|
|
|
|
inode->i_blkbits;
|
2016-06-21 07:23:11 +08:00
|
|
|
set_buffer_mapped(bh);
|
2023-09-13 01:05:48 +08:00
|
|
|
return 0;
|
|
|
|
default:
|
|
|
|
WARN_ON_ONCE(1);
|
|
|
|
return -EIO;
|
2016-06-21 07:23:11 +08:00
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2021-11-04 02:05:47 +08:00
|
|
|
int __block_write_begin_int(struct folio *folio, loff_t pos, unsigned len,
|
2021-08-11 09:33:05 +08:00
|
|
|
get_block_t *get_block, const struct iomap *iomap)
|
2005-04-17 06:20:36 +08:00
|
|
|
{
|
2023-11-10 05:06:07 +08:00
|
|
|
size_t from = offset_in_folio(folio, pos);
|
|
|
|
size_t to = from + len;
|
2021-11-04 02:05:47 +08:00
|
|
|
struct inode *inode = folio->mapping->host;
|
2023-11-10 05:06:07 +08:00
|
|
|
size_t block_start, block_end;
|
2005-04-17 06:20:36 +08:00
|
|
|
sector_t block;
|
|
|
|
int err = 0;
|
2023-11-10 05:06:07 +08:00
|
|
|
size_t blocksize;
|
2005-04-17 06:20:36 +08:00
|
|
|
struct buffer_head *bh, *head, *wait[2], **wait_bh=wait;
|
|
|
|
|
2021-11-04 02:05:47 +08:00
|
|
|
BUG_ON(!folio_test_locked(folio));
|
2023-11-10 05:06:07 +08:00
|
|
|
BUG_ON(to > folio_size(folio));
|
2005-04-17 06:20:36 +08:00
|
|
|
BUG_ON(from > to);
|
|
|
|
|
2023-04-17 20:36:18 +08:00
|
|
|
head = folio_create_buffers(folio, inode, 0);
|
2012-11-30 02:21:43 +08:00
|
|
|
blocksize = head->b_size;
|
2023-11-10 05:06:07 +08:00
|
|
|
block = div_u64(folio_pos(folio), blocksize);
|
2005-04-17 06:20:36 +08:00
|
|
|
|
2023-11-10 05:06:07 +08:00
|
|
|
for (bh = head, block_start = 0; bh != head || !block_start;
|
2005-04-17 06:20:36 +08:00
|
|
|
block++, block_start=block_end, bh = bh->b_this_page) {
|
|
|
|
block_end = block_start + blocksize;
|
|
|
|
if (block_end <= from || block_start >= to) {
|
2021-11-04 02:05:47 +08:00
|
|
|
if (folio_test_uptodate(folio)) {
|
2005-04-17 06:20:36 +08:00
|
|
|
if (!buffer_uptodate(bh))
|
|
|
|
set_buffer_uptodate(bh);
|
|
|
|
}
|
|
|
|
continue;
|
|
|
|
}
|
|
|
|
if (buffer_new(bh))
|
|
|
|
clear_buffer_new(bh);
|
|
|
|
if (!buffer_mapped(bh)) {
|
2006-03-26 17:38:00 +08:00
|
|
|
WARN_ON(bh->b_size != blocksize);
|
2023-09-13 01:05:48 +08:00
|
|
|
if (get_block)
|
2016-06-21 07:23:11 +08:00
|
|
|
err = get_block(inode, block, bh, 1);
|
2023-09-13 01:05:48 +08:00
|
|
|
else
|
|
|
|
err = iomap_to_bh(inode, block, bh, iomap);
|
|
|
|
if (err)
|
|
|
|
break;
|
2016-06-21 07:23:11 +08:00
|
|
|
|
2005-04-17 06:20:36 +08:00
|
|
|
if (buffer_new(bh)) {
|
2016-11-05 01:08:15 +08:00
|
|
|
clean_bdev_bh_alias(bh);
|
2021-11-04 02:05:47 +08:00
|
|
|
if (folio_test_uptodate(folio)) {
|
2007-10-16 16:25:00 +08:00
|
|
|
clear_buffer_new(bh);
|
2005-04-17 06:20:36 +08:00
|
|
|
set_buffer_uptodate(bh);
|
2007-10-16 16:25:00 +08:00
|
|
|
mark_buffer_dirty(bh);
|
2005-04-17 06:20:36 +08:00
|
|
|
continue;
|
|
|
|
}
|
2008-02-05 14:28:29 +08:00
|
|
|
if (block_end > to || block_start < from)
|
2021-11-04 02:05:47 +08:00
|
|
|
folio_zero_segments(folio,
|
2008-02-05 14:28:29 +08:00
|
|
|
to, block_end,
|
|
|
|
block_start, from);
|
2005-04-17 06:20:36 +08:00
|
|
|
continue;
|
|
|
|
}
|
|
|
|
}
|
2021-11-04 02:05:47 +08:00
|
|
|
if (folio_test_uptodate(folio)) {
|
2005-04-17 06:20:36 +08:00
|
|
|
if (!buffer_uptodate(bh))
|
|
|
|
set_buffer_uptodate(bh);
|
|
|
|
continue;
|
|
|
|
}
|
|
|
|
if (!buffer_uptodate(bh) && !buffer_delay(bh) &&
|
2007-02-12 16:51:41 +08:00
|
|
|
!buffer_unwritten(bh) &&
|
2005-04-17 06:20:36 +08:00
|
|
|
(block_start < from || block_end > to)) {
|
2022-09-01 21:34:54 +08:00
|
|
|
bh_read_nowait(bh, 0);
|
2005-04-17 06:20:36 +08:00
|
|
|
*wait_bh++=bh;
|
|
|
|
}
|
|
|
|
}
|
|
|
|
/*
|
|
|
|
* If we issued read requests - let them complete.
|
|
|
|
*/
|
|
|
|
while(wait_bh > wait) {
|
|
|
|
wait_on_buffer(*--wait_bh);
|
|
|
|
if (!buffer_uptodate(*wait_bh))
|
2005-05-06 07:15:45 +08:00
|
|
|
err = -EIO;
|
2005-04-17 06:20:36 +08:00
|
|
|
}
|
2011-06-14 06:58:27 +08:00
|
|
|
if (unlikely(err))
|
2023-06-13 05:01:36 +08:00
|
|
|
folio_zero_new_buffers(folio, from, to);
|
2005-04-17 06:20:36 +08:00
|
|
|
return err;
|
|
|
|
}
|
2016-06-21 07:23:11 +08:00
|
|
|
|
|
|
|
int __block_write_begin(struct page *page, loff_t pos, unsigned len,
|
|
|
|
get_block_t *get_block)
|
|
|
|
{
|
2021-11-04 02:05:47 +08:00
|
|
|
return __block_write_begin_int(page_folio(page), pos, len, get_block,
|
|
|
|
NULL);
|
2016-06-21 07:23:11 +08:00
|
|
|
}
|
2010-10-06 16:47:23 +08:00
|
|
|
EXPORT_SYMBOL(__block_write_begin);
|
2005-04-17 06:20:36 +08:00
|
|
|
|
2023-06-26 13:55:18 +08:00
|
|
|
static void __block_commit_write(struct folio *folio, size_t from, size_t to)
|
2005-04-17 06:20:36 +08:00
|
|
|
{
|
2023-06-13 05:01:35 +08:00
|
|
|
size_t block_start, block_end;
|
|
|
|
bool partial = false;
|
2005-04-17 06:20:36 +08:00
|
|
|
unsigned blocksize;
|
|
|
|
struct buffer_head *bh, *head;
|
|
|
|
|
2023-06-13 05:01:35 +08:00
|
|
|
bh = head = folio_buffers(folio);
|
2012-11-30 02:21:43 +08:00
|
|
|
blocksize = bh->b_size;
|
2005-04-17 06:20:36 +08:00
|
|
|
|
2012-11-30 02:21:43 +08:00
|
|
|
block_start = 0;
|
|
|
|
do {
|
2005-04-17 06:20:36 +08:00
|
|
|
block_end = block_start + blocksize;
|
|
|
|
if (block_end <= from || block_start >= to) {
|
|
|
|
if (!buffer_uptodate(bh))
|
2023-06-13 05:01:35 +08:00
|
|
|
partial = true;
|
2005-04-17 06:20:36 +08:00
|
|
|
} else {
|
|
|
|
set_buffer_uptodate(bh);
|
|
|
|
mark_buffer_dirty(bh);
|
|
|
|
}
|
2021-02-25 04:02:48 +08:00
|
|
|
if (buffer_new(bh))
|
|
|
|
clear_buffer_new(bh);
|
2012-11-30 02:21:43 +08:00
|
|
|
|
|
|
|
block_start = block_end;
|
|
|
|
bh = bh->b_this_page;
|
|
|
|
} while (bh != head);
|
2005-04-17 06:20:36 +08:00
|
|
|
|
|
|
|
/*
|
|
|
|
* If this is a partial write which happened to make all buffers
|
2022-04-29 22:40:40 +08:00
|
|
|
* uptodate then we can optimize away a bogus read_folio() for
|
2023-06-13 05:01:35 +08:00
|
|
|
* the next read(). Here we 'discover' whether the folio went
|
2005-04-17 06:20:36 +08:00
|
|
|
* uptodate as a result of this (potentially partial) write.
|
|
|
|
*/
|
|
|
|
if (!partial)
|
2023-06-13 05:01:35 +08:00
|
|
|
folio_mark_uptodate(folio);
|
2005-04-17 06:20:36 +08:00
|
|
|
}
|
|
|
|
|
2007-10-16 16:25:01 +08:00
|
|
|
/*
|
2010-06-04 17:29:58 +08:00
|
|
|
* block_write_begin takes care of the basic task of block allocation and
|
|
|
|
* bringing partial write blocks uptodate first.
|
|
|
|
*
|
fs: introduce new truncate sequence
Introduce a new truncate calling sequence into fs/mm subsystems. Rather than
setattr > vmtruncate > truncate, have filesystems call their truncate sequence
from ->setattr if filesystem specific operations are required. vmtruncate is
deprecated, and truncate_pagecache and inode_newsize_ok helpers introduced
previously should be used.
simple_setattr is introduced for simple in-ram filesystems to implement
the new truncate sequence. Eventually all filesystems should be converted
to implement a setattr, and the default code in notify_change should go
away.
simple_setsize is also introduced to perform just the ATTR_SIZE portion
of simple_setattr (ie. changing i_size and trimming pagecache).
To implement the new truncate sequence:
- filesystem specific manipulations (eg freeing blocks) must be done in
the setattr method rather than ->truncate.
- vmtruncate can not be used by core code to trim blocks past i_size in
the event of write failure after allocation, so this must be performed
in the fs code.
- convert usage of helpers block_write_begin, nobh_write_begin,
cont_write_begin, and *blockdev_direct_IO* to use _newtrunc postfixed
variants. These avoid calling vmtruncate to trim blocks (see previous).
- inode_setattr should not be used. generic_setattr is a new function
to be used to copy simple attributes into the generic inode.
- make use of the better opportunity to handle errors with the new sequence.
Big problem with the previous calling sequence: the filesystem is not called
until i_size has already changed. This means it is not allowed to fail the
call, and also it does not know what the previous i_size was. Also, generic
code calling vmtruncate to truncate allocated blocks in case of error had
no good way to return a meaningful error (or, for example, atomically handle
block deallocation).
Cc: Christoph Hellwig <hch@lst.de>
Acked-by: Jan Kara <jack@suse.cz>
Signed-off-by: Nick Piggin <npiggin@suse.de>
Signed-off-by: Al Viro <viro@zeniv.linux.org.uk>
2010-05-26 23:05:33 +08:00
|
|
|
* The filesystem needs to handle block truncation upon failure.
|
2007-10-16 16:25:01 +08:00
|
|
|
*/
|
2010-06-04 17:29:58 +08:00
|
|
|
int block_write_begin(struct address_space *mapping, loff_t pos, unsigned len,
|
2022-02-23 00:25:12 +08:00
|
|
|
struct page **pagep, get_block_t *get_block)
|
2007-10-16 16:25:01 +08:00
|
|
|
{
|
mm, fs: get rid of PAGE_CACHE_* and page_cache_{get,release} macros
PAGE_CACHE_{SIZE,SHIFT,MASK,ALIGN} macros were introduced *long* time
ago with promise that one day it will be possible to implement page
cache with bigger chunks than PAGE_SIZE.
This promise never materialized. And unlikely will.
We have many places where PAGE_CACHE_SIZE assumed to be equal to
PAGE_SIZE. And it's constant source of confusion on whether
PAGE_CACHE_* or PAGE_* constant should be used in a particular case,
especially on the border between fs and mm.
Global switching to PAGE_CACHE_SIZE != PAGE_SIZE would cause to much
breakage to be doable.
Let's stop pretending that pages in page cache are special. They are
not.
The changes are pretty straight-forward:
- <foo> << (PAGE_CACHE_SHIFT - PAGE_SHIFT) -> <foo>;
- <foo> >> (PAGE_CACHE_SHIFT - PAGE_SHIFT) -> <foo>;
- PAGE_CACHE_{SIZE,SHIFT,MASK,ALIGN} -> PAGE_{SIZE,SHIFT,MASK,ALIGN};
- page_cache_get() -> get_page();
- page_cache_release() -> put_page();
This patch contains automated changes generated with coccinelle using
script below. For some reason, coccinelle doesn't patch header files.
I've called spatch for them manually.
The only adjustment after coccinelle is revert of changes to
PAGE_CAHCE_ALIGN definition: we are going to drop it later.
There are few places in the code where coccinelle didn't reach. I'll
fix them manually in a separate patch. Comments and documentation also
will be addressed with the separate patch.
virtual patch
@@
expression E;
@@
- E << (PAGE_CACHE_SHIFT - PAGE_SHIFT)
+ E
@@
expression E;
@@
- E >> (PAGE_CACHE_SHIFT - PAGE_SHIFT)
+ E
@@
@@
- PAGE_CACHE_SHIFT
+ PAGE_SHIFT
@@
@@
- PAGE_CACHE_SIZE
+ PAGE_SIZE
@@
@@
- PAGE_CACHE_MASK
+ PAGE_MASK
@@
expression E;
@@
- PAGE_CACHE_ALIGN(E)
+ PAGE_ALIGN(E)
@@
expression E;
@@
- page_cache_get(E)
+ get_page(E)
@@
expression E;
@@
- page_cache_release(E)
+ put_page(E)
Signed-off-by: Kirill A. Shutemov <kirill.shutemov@linux.intel.com>
Acked-by: Michal Hocko <mhocko@suse.com>
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
2016-04-01 20:29:47 +08:00
|
|
|
pgoff_t index = pos >> PAGE_SHIFT;
|
2007-10-16 16:25:01 +08:00
|
|
|
struct page *page;
|
2010-06-04 17:29:57 +08:00
|
|
|
int status;
|
2007-10-16 16:25:01 +08:00
|
|
|
|
2022-02-23 00:25:12 +08:00
|
|
|
page = grab_cache_page_write_begin(mapping, index);
|
2010-06-04 17:29:57 +08:00
|
|
|
if (!page)
|
|
|
|
return -ENOMEM;
|
2007-10-16 16:25:01 +08:00
|
|
|
|
2010-06-04 17:29:57 +08:00
|
|
|
status = __block_write_begin(page, pos, len, get_block);
|
2007-10-16 16:25:01 +08:00
|
|
|
if (unlikely(status)) {
|
2010-06-04 17:29:57 +08:00
|
|
|
unlock_page(page);
|
mm, fs: get rid of PAGE_CACHE_* and page_cache_{get,release} macros
PAGE_CACHE_{SIZE,SHIFT,MASK,ALIGN} macros were introduced *long* time
ago with promise that one day it will be possible to implement page
cache with bigger chunks than PAGE_SIZE.
This promise never materialized. And unlikely will.
We have many places where PAGE_CACHE_SIZE assumed to be equal to
PAGE_SIZE. And it's constant source of confusion on whether
PAGE_CACHE_* or PAGE_* constant should be used in a particular case,
especially on the border between fs and mm.
Global switching to PAGE_CACHE_SIZE != PAGE_SIZE would cause to much
breakage to be doable.
Let's stop pretending that pages in page cache are special. They are
not.
The changes are pretty straight-forward:
- <foo> << (PAGE_CACHE_SHIFT - PAGE_SHIFT) -> <foo>;
- <foo> >> (PAGE_CACHE_SHIFT - PAGE_SHIFT) -> <foo>;
- PAGE_CACHE_{SIZE,SHIFT,MASK,ALIGN} -> PAGE_{SIZE,SHIFT,MASK,ALIGN};
- page_cache_get() -> get_page();
- page_cache_release() -> put_page();
This patch contains automated changes generated with coccinelle using
script below. For some reason, coccinelle doesn't patch header files.
I've called spatch for them manually.
The only adjustment after coccinelle is revert of changes to
PAGE_CAHCE_ALIGN definition: we are going to drop it later.
There are few places in the code where coccinelle didn't reach. I'll
fix them manually in a separate patch. Comments and documentation also
will be addressed with the separate patch.
virtual patch
@@
expression E;
@@
- E << (PAGE_CACHE_SHIFT - PAGE_SHIFT)
+ E
@@
expression E;
@@
- E >> (PAGE_CACHE_SHIFT - PAGE_SHIFT)
+ E
@@
@@
- PAGE_CACHE_SHIFT
+ PAGE_SHIFT
@@
@@
- PAGE_CACHE_SIZE
+ PAGE_SIZE
@@
@@
- PAGE_CACHE_MASK
+ PAGE_MASK
@@
expression E;
@@
- PAGE_CACHE_ALIGN(E)
+ PAGE_ALIGN(E)
@@
expression E;
@@
- page_cache_get(E)
+ get_page(E)
@@
expression E;
@@
- page_cache_release(E)
+ put_page(E)
Signed-off-by: Kirill A. Shutemov <kirill.shutemov@linux.intel.com>
Acked-by: Michal Hocko <mhocko@suse.com>
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
2016-04-01 20:29:47 +08:00
|
|
|
put_page(page);
|
2010-06-04 17:29:57 +08:00
|
|
|
page = NULL;
|
2007-10-16 16:25:01 +08:00
|
|
|
}
|
|
|
|
|
2010-06-04 17:29:57 +08:00
|
|
|
*pagep = page;
|
2007-10-16 16:25:01 +08:00
|
|
|
return status;
|
|
|
|
}
|
|
|
|
EXPORT_SYMBOL(block_write_begin);
|
|
|
|
|
|
|
|
int block_write_end(struct file *file, struct address_space *mapping,
|
|
|
|
loff_t pos, unsigned len, unsigned copied,
|
|
|
|
struct page *page, void *fsdata)
|
|
|
|
{
|
2023-06-13 05:01:35 +08:00
|
|
|
struct folio *folio = page_folio(page);
|
|
|
|
size_t start = pos - folio_pos(folio);
|
2007-10-16 16:25:01 +08:00
|
|
|
|
|
|
|
if (unlikely(copied < len)) {
|
|
|
|
/*
|
2022-04-29 22:40:40 +08:00
|
|
|
* The buffers that were written will now be uptodate, so
|
|
|
|
* we don't have to worry about a read_folio reading them
|
|
|
|
* and overwriting a partial write. However if we have
|
|
|
|
* encountered a short write and only partially written
|
|
|
|
* into a buffer, it will not be marked uptodate, so a
|
|
|
|
* read_folio might come in and destroy our partial write.
|
2007-10-16 16:25:01 +08:00
|
|
|
*
|
|
|
|
* Do the simplest thing, and just treat any short write to a
|
2023-06-13 05:01:35 +08:00
|
|
|
* non uptodate folio as a zero-length write, and force the
|
2007-10-16 16:25:01 +08:00
|
|
|
* caller to redo the whole thing.
|
|
|
|
*/
|
2023-06-13 05:01:35 +08:00
|
|
|
if (!folio_test_uptodate(folio))
|
2007-10-16 16:25:01 +08:00
|
|
|
copied = 0;
|
|
|
|
|
2023-06-13 05:01:36 +08:00
|
|
|
folio_zero_new_buffers(folio, start+copied, start+len);
|
2007-10-16 16:25:01 +08:00
|
|
|
}
|
2023-06-13 05:01:35 +08:00
|
|
|
flush_dcache_folio(folio);
|
2007-10-16 16:25:01 +08:00
|
|
|
|
|
|
|
/* This could be a short (even 0-length) commit */
|
2023-06-26 13:55:17 +08:00
|
|
|
__block_commit_write(folio, start, start + copied);
|
2007-10-16 16:25:01 +08:00
|
|
|
|
|
|
|
return copied;
|
|
|
|
}
|
|
|
|
EXPORT_SYMBOL(block_write_end);
|
|
|
|
|
|
|
|
int generic_write_end(struct file *file, struct address_space *mapping,
|
|
|
|
loff_t pos, unsigned len, unsigned copied,
|
|
|
|
struct page *page, void *fsdata)
|
|
|
|
{
|
2019-06-28 08:28:40 +08:00
|
|
|
struct inode *inode = mapping->host;
|
|
|
|
loff_t old_size = inode->i_size;
|
|
|
|
bool i_size_changed = false;
|
|
|
|
|
2007-10-16 16:25:01 +08:00
|
|
|
copied = block_write_end(file, mapping, pos, len, copied, page, fsdata);
|
2019-06-28 08:28:40 +08:00
|
|
|
|
|
|
|
/*
|
|
|
|
* No need to use i_size_read() here, the i_size cannot change under us
|
|
|
|
* because we hold i_rwsem.
|
|
|
|
*
|
|
|
|
* But it's important to update i_size while still holding page lock:
|
|
|
|
* page writeout could otherwise come in and zero beyond i_size.
|
|
|
|
*/
|
|
|
|
if (pos + copied > inode->i_size) {
|
|
|
|
i_size_write(inode, pos + copied);
|
|
|
|
i_size_changed = true;
|
|
|
|
}
|
|
|
|
|
|
|
|
unlock_page(page);
|
2019-04-30 23:45:34 +08:00
|
|
|
put_page(page);
|
2019-06-28 08:28:40 +08:00
|
|
|
|
|
|
|
if (old_size < pos)
|
|
|
|
pagecache_isize_extended(inode, old_size, pos);
|
|
|
|
/*
|
|
|
|
* Don't mark the inode dirty under page lock. First, it unnecessarily
|
|
|
|
* makes the holding time of page lock longer. Second, it forces lock
|
|
|
|
* ordering of page lock and transaction start for journaling
|
|
|
|
* filesystems.
|
|
|
|
*/
|
|
|
|
if (i_size_changed)
|
|
|
|
mark_inode_dirty(inode);
|
2019-04-30 23:45:33 +08:00
|
|
|
return copied;
|
2007-10-16 16:25:01 +08:00
|
|
|
}
|
|
|
|
EXPORT_SYMBOL(generic_write_end);
|
|
|
|
|
vfs: pagecache usage optimization for pagesize!=blocksize
When we read some part of a file through pagecache, if there is a
pagecache of corresponding index but this page is not uptodate, read IO
is issued and this page will be uptodate.
I think this is good for pagesize == blocksize environment but there is
room for improvement on pagesize != blocksize environment. Because in
this case a page can have multiple buffers and even if a page is not
uptodate, some buffers can be uptodate.
So I suggest that when all buffers which correspond to a part of a file
that we want to read are uptodate, use this pagecache and copy data from
this pagecache to user buffer even if a page is not uptodate. This can
reduce read IO and improve system throughput.
I wrote a benchmark program and got result number with this program.
This benchmark do:
1: mount and open a test file.
2: create a 512MB file.
3: close a file and umount.
4: mount and again open a test file.
5: pwrite randomly 300000 times on a test file. offset is aligned
by IO size(1024bytes).
6: measure time of preading randomly 100000 times on a test file.
The result was:
2.6.26
330 sec
2.6.26-patched
226 sec
Arch:i386
Filesystem:ext3
Blocksize:1024 bytes
Memory: 1GB
On ext3/4, a file is written through buffer/block. So random read/write
mixed workloads or random read after random write workloads are optimized
with this patch under pagesize != blocksize environment. This test result
showed this.
The benchmark program is as follows:
#include <stdio.h>
#include <sys/types.h>
#include <sys/stat.h>
#include <fcntl.h>
#include <unistd.h>
#include <time.h>
#include <stdlib.h>
#include <string.h>
#include <sys/mount.h>
#define LEN 1024
#define LOOP 1024*512 /* 512MB */
main(void)
{
unsigned long i, offset, filesize;
int fd;
char buf[LEN];
time_t t1, t2;
if (mount("/dev/sda1", "/root/test1/", "ext3", 0, 0) < 0) {
perror("cannot mount\n");
exit(1);
}
memset(buf, 0, LEN);
fd = open("/root/test1/testfile", O_CREAT|O_RDWR|O_TRUNC);
if (fd < 0) {
perror("cannot open file\n");
exit(1);
}
for (i = 0; i < LOOP; i++)
write(fd, buf, LEN);
close(fd);
if (umount("/root/test1/") < 0) {
perror("cannot umount\n");
exit(1);
}
if (mount("/dev/sda1", "/root/test1/", "ext3", 0, 0) < 0) {
perror("cannot mount\n");
exit(1);
}
fd = open("/root/test1/testfile", O_RDWR);
if (fd < 0) {
perror("cannot open file\n");
exit(1);
}
filesize = LEN * LOOP;
for (i = 0; i < 300000; i++){
offset = (random() % filesize) & (~(LEN - 1));
pwrite(fd, buf, LEN, offset);
}
printf("start test\n");
time(&t1);
for (i = 0; i < 100000; i++){
offset = (random() % filesize) & (~(LEN - 1));
pread(fd, buf, LEN, offset);
}
time(&t2);
printf("%ld sec\n", t2-t1);
close(fd);
if (umount("/root/test1/") < 0) {
perror("cannot umount\n");
exit(1);
}
}
Signed-off-by: Hisashi Hifumi <hifumi.hisashi@oss.ntt.co.jp>
Cc: Nick Piggin <nickpiggin@yahoo.com.au>
Cc: Christoph Hellwig <hch@infradead.org>
Cc: Jan Kara <jack@ucw.cz>
Cc: <linux-ext4@vger.kernel.org>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
2008-07-29 06:46:36 +08:00
|
|
|
/*
|
2022-02-10 04:21:27 +08:00
|
|
|
* block_is_partially_uptodate checks whether buffers within a folio are
|
vfs: pagecache usage optimization for pagesize!=blocksize
When we read some part of a file through pagecache, if there is a
pagecache of corresponding index but this page is not uptodate, read IO
is issued and this page will be uptodate.
I think this is good for pagesize == blocksize environment but there is
room for improvement on pagesize != blocksize environment. Because in
this case a page can have multiple buffers and even if a page is not
uptodate, some buffers can be uptodate.
So I suggest that when all buffers which correspond to a part of a file
that we want to read are uptodate, use this pagecache and copy data from
this pagecache to user buffer even if a page is not uptodate. This can
reduce read IO and improve system throughput.
I wrote a benchmark program and got result number with this program.
This benchmark do:
1: mount and open a test file.
2: create a 512MB file.
3: close a file and umount.
4: mount and again open a test file.
5: pwrite randomly 300000 times on a test file. offset is aligned
by IO size(1024bytes).
6: measure time of preading randomly 100000 times on a test file.
The result was:
2.6.26
330 sec
2.6.26-patched
226 sec
Arch:i386
Filesystem:ext3
Blocksize:1024 bytes
Memory: 1GB
On ext3/4, a file is written through buffer/block. So random read/write
mixed workloads or random read after random write workloads are optimized
with this patch under pagesize != blocksize environment. This test result
showed this.
The benchmark program is as follows:
#include <stdio.h>
#include <sys/types.h>
#include <sys/stat.h>
#include <fcntl.h>
#include <unistd.h>
#include <time.h>
#include <stdlib.h>
#include <string.h>
#include <sys/mount.h>
#define LEN 1024
#define LOOP 1024*512 /* 512MB */
main(void)
{
unsigned long i, offset, filesize;
int fd;
char buf[LEN];
time_t t1, t2;
if (mount("/dev/sda1", "/root/test1/", "ext3", 0, 0) < 0) {
perror("cannot mount\n");
exit(1);
}
memset(buf, 0, LEN);
fd = open("/root/test1/testfile", O_CREAT|O_RDWR|O_TRUNC);
if (fd < 0) {
perror("cannot open file\n");
exit(1);
}
for (i = 0; i < LOOP; i++)
write(fd, buf, LEN);
close(fd);
if (umount("/root/test1/") < 0) {
perror("cannot umount\n");
exit(1);
}
if (mount("/dev/sda1", "/root/test1/", "ext3", 0, 0) < 0) {
perror("cannot mount\n");
exit(1);
}
fd = open("/root/test1/testfile", O_RDWR);
if (fd < 0) {
perror("cannot open file\n");
exit(1);
}
filesize = LEN * LOOP;
for (i = 0; i < 300000; i++){
offset = (random() % filesize) & (~(LEN - 1));
pwrite(fd, buf, LEN, offset);
}
printf("start test\n");
time(&t1);
for (i = 0; i < 100000; i++){
offset = (random() % filesize) & (~(LEN - 1));
pread(fd, buf, LEN, offset);
}
time(&t2);
printf("%ld sec\n", t2-t1);
close(fd);
if (umount("/root/test1/") < 0) {
perror("cannot umount\n");
exit(1);
}
}
Signed-off-by: Hisashi Hifumi <hifumi.hisashi@oss.ntt.co.jp>
Cc: Nick Piggin <nickpiggin@yahoo.com.au>
Cc: Christoph Hellwig <hch@infradead.org>
Cc: Jan Kara <jack@ucw.cz>
Cc: <linux-ext4@vger.kernel.org>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
2008-07-29 06:46:36 +08:00
|
|
|
* uptodate or not.
|
|
|
|
*
|
2022-02-10 04:21:27 +08:00
|
|
|
* Returns true if all buffers which correspond to the specified part
|
|
|
|
* of the folio are uptodate.
|
vfs: pagecache usage optimization for pagesize!=blocksize
When we read some part of a file through pagecache, if there is a
pagecache of corresponding index but this page is not uptodate, read IO
is issued and this page will be uptodate.
I think this is good for pagesize == blocksize environment but there is
room for improvement on pagesize != blocksize environment. Because in
this case a page can have multiple buffers and even if a page is not
uptodate, some buffers can be uptodate.
So I suggest that when all buffers which correspond to a part of a file
that we want to read are uptodate, use this pagecache and copy data from
this pagecache to user buffer even if a page is not uptodate. This can
reduce read IO and improve system throughput.
I wrote a benchmark program and got result number with this program.
This benchmark do:
1: mount and open a test file.
2: create a 512MB file.
3: close a file and umount.
4: mount and again open a test file.
5: pwrite randomly 300000 times on a test file. offset is aligned
by IO size(1024bytes).
6: measure time of preading randomly 100000 times on a test file.
The result was:
2.6.26
330 sec
2.6.26-patched
226 sec
Arch:i386
Filesystem:ext3
Blocksize:1024 bytes
Memory: 1GB
On ext3/4, a file is written through buffer/block. So random read/write
mixed workloads or random read after random write workloads are optimized
with this patch under pagesize != blocksize environment. This test result
showed this.
The benchmark program is as follows:
#include <stdio.h>
#include <sys/types.h>
#include <sys/stat.h>
#include <fcntl.h>
#include <unistd.h>
#include <time.h>
#include <stdlib.h>
#include <string.h>
#include <sys/mount.h>
#define LEN 1024
#define LOOP 1024*512 /* 512MB */
main(void)
{
unsigned long i, offset, filesize;
int fd;
char buf[LEN];
time_t t1, t2;
if (mount("/dev/sda1", "/root/test1/", "ext3", 0, 0) < 0) {
perror("cannot mount\n");
exit(1);
}
memset(buf, 0, LEN);
fd = open("/root/test1/testfile", O_CREAT|O_RDWR|O_TRUNC);
if (fd < 0) {
perror("cannot open file\n");
exit(1);
}
for (i = 0; i < LOOP; i++)
write(fd, buf, LEN);
close(fd);
if (umount("/root/test1/") < 0) {
perror("cannot umount\n");
exit(1);
}
if (mount("/dev/sda1", "/root/test1/", "ext3", 0, 0) < 0) {
perror("cannot mount\n");
exit(1);
}
fd = open("/root/test1/testfile", O_RDWR);
if (fd < 0) {
perror("cannot open file\n");
exit(1);
}
filesize = LEN * LOOP;
for (i = 0; i < 300000; i++){
offset = (random() % filesize) & (~(LEN - 1));
pwrite(fd, buf, LEN, offset);
}
printf("start test\n");
time(&t1);
for (i = 0; i < 100000; i++){
offset = (random() % filesize) & (~(LEN - 1));
pread(fd, buf, LEN, offset);
}
time(&t2);
printf("%ld sec\n", t2-t1);
close(fd);
if (umount("/root/test1/") < 0) {
perror("cannot umount\n");
exit(1);
}
}
Signed-off-by: Hisashi Hifumi <hifumi.hisashi@oss.ntt.co.jp>
Cc: Nick Piggin <nickpiggin@yahoo.com.au>
Cc: Christoph Hellwig <hch@infradead.org>
Cc: Jan Kara <jack@ucw.cz>
Cc: <linux-ext4@vger.kernel.org>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
2008-07-29 06:46:36 +08:00
|
|
|
*/
|
2022-02-10 04:21:27 +08:00
|
|
|
bool block_is_partially_uptodate(struct folio *folio, size_t from, size_t count)
|
vfs: pagecache usage optimization for pagesize!=blocksize
When we read some part of a file through pagecache, if there is a
pagecache of corresponding index but this page is not uptodate, read IO
is issued and this page will be uptodate.
I think this is good for pagesize == blocksize environment but there is
room for improvement on pagesize != blocksize environment. Because in
this case a page can have multiple buffers and even if a page is not
uptodate, some buffers can be uptodate.
So I suggest that when all buffers which correspond to a part of a file
that we want to read are uptodate, use this pagecache and copy data from
this pagecache to user buffer even if a page is not uptodate. This can
reduce read IO and improve system throughput.
I wrote a benchmark program and got result number with this program.
This benchmark do:
1: mount and open a test file.
2: create a 512MB file.
3: close a file and umount.
4: mount and again open a test file.
5: pwrite randomly 300000 times on a test file. offset is aligned
by IO size(1024bytes).
6: measure time of preading randomly 100000 times on a test file.
The result was:
2.6.26
330 sec
2.6.26-patched
226 sec
Arch:i386
Filesystem:ext3
Blocksize:1024 bytes
Memory: 1GB
On ext3/4, a file is written through buffer/block. So random read/write
mixed workloads or random read after random write workloads are optimized
with this patch under pagesize != blocksize environment. This test result
showed this.
The benchmark program is as follows:
#include <stdio.h>
#include <sys/types.h>
#include <sys/stat.h>
#include <fcntl.h>
#include <unistd.h>
#include <time.h>
#include <stdlib.h>
#include <string.h>
#include <sys/mount.h>
#define LEN 1024
#define LOOP 1024*512 /* 512MB */
main(void)
{
unsigned long i, offset, filesize;
int fd;
char buf[LEN];
time_t t1, t2;
if (mount("/dev/sda1", "/root/test1/", "ext3", 0, 0) < 0) {
perror("cannot mount\n");
exit(1);
}
memset(buf, 0, LEN);
fd = open("/root/test1/testfile", O_CREAT|O_RDWR|O_TRUNC);
if (fd < 0) {
perror("cannot open file\n");
exit(1);
}
for (i = 0; i < LOOP; i++)
write(fd, buf, LEN);
close(fd);
if (umount("/root/test1/") < 0) {
perror("cannot umount\n");
exit(1);
}
if (mount("/dev/sda1", "/root/test1/", "ext3", 0, 0) < 0) {
perror("cannot mount\n");
exit(1);
}
fd = open("/root/test1/testfile", O_RDWR);
if (fd < 0) {
perror("cannot open file\n");
exit(1);
}
filesize = LEN * LOOP;
for (i = 0; i < 300000; i++){
offset = (random() % filesize) & (~(LEN - 1));
pwrite(fd, buf, LEN, offset);
}
printf("start test\n");
time(&t1);
for (i = 0; i < 100000; i++){
offset = (random() % filesize) & (~(LEN - 1));
pread(fd, buf, LEN, offset);
}
time(&t2);
printf("%ld sec\n", t2-t1);
close(fd);
if (umount("/root/test1/") < 0) {
perror("cannot umount\n");
exit(1);
}
}
Signed-off-by: Hisashi Hifumi <hifumi.hisashi@oss.ntt.co.jp>
Cc: Nick Piggin <nickpiggin@yahoo.com.au>
Cc: Christoph Hellwig <hch@infradead.org>
Cc: Jan Kara <jack@ucw.cz>
Cc: <linux-ext4@vger.kernel.org>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
2008-07-29 06:46:36 +08:00
|
|
|
{
|
|
|
|
unsigned block_start, block_end, blocksize;
|
|
|
|
unsigned to;
|
|
|
|
struct buffer_head *bh, *head;
|
2022-02-10 04:21:27 +08:00
|
|
|
bool ret = true;
|
vfs: pagecache usage optimization for pagesize!=blocksize
When we read some part of a file through pagecache, if there is a
pagecache of corresponding index but this page is not uptodate, read IO
is issued and this page will be uptodate.
I think this is good for pagesize == blocksize environment but there is
room for improvement on pagesize != blocksize environment. Because in
this case a page can have multiple buffers and even if a page is not
uptodate, some buffers can be uptodate.
So I suggest that when all buffers which correspond to a part of a file
that we want to read are uptodate, use this pagecache and copy data from
this pagecache to user buffer even if a page is not uptodate. This can
reduce read IO and improve system throughput.
I wrote a benchmark program and got result number with this program.
This benchmark do:
1: mount and open a test file.
2: create a 512MB file.
3: close a file and umount.
4: mount and again open a test file.
5: pwrite randomly 300000 times on a test file. offset is aligned
by IO size(1024bytes).
6: measure time of preading randomly 100000 times on a test file.
The result was:
2.6.26
330 sec
2.6.26-patched
226 sec
Arch:i386
Filesystem:ext3
Blocksize:1024 bytes
Memory: 1GB
On ext3/4, a file is written through buffer/block. So random read/write
mixed workloads or random read after random write workloads are optimized
with this patch under pagesize != blocksize environment. This test result
showed this.
The benchmark program is as follows:
#include <stdio.h>
#include <sys/types.h>
#include <sys/stat.h>
#include <fcntl.h>
#include <unistd.h>
#include <time.h>
#include <stdlib.h>
#include <string.h>
#include <sys/mount.h>
#define LEN 1024
#define LOOP 1024*512 /* 512MB */
main(void)
{
unsigned long i, offset, filesize;
int fd;
char buf[LEN];
time_t t1, t2;
if (mount("/dev/sda1", "/root/test1/", "ext3", 0, 0) < 0) {
perror("cannot mount\n");
exit(1);
}
memset(buf, 0, LEN);
fd = open("/root/test1/testfile", O_CREAT|O_RDWR|O_TRUNC);
if (fd < 0) {
perror("cannot open file\n");
exit(1);
}
for (i = 0; i < LOOP; i++)
write(fd, buf, LEN);
close(fd);
if (umount("/root/test1/") < 0) {
perror("cannot umount\n");
exit(1);
}
if (mount("/dev/sda1", "/root/test1/", "ext3", 0, 0) < 0) {
perror("cannot mount\n");
exit(1);
}
fd = open("/root/test1/testfile", O_RDWR);
if (fd < 0) {
perror("cannot open file\n");
exit(1);
}
filesize = LEN * LOOP;
for (i = 0; i < 300000; i++){
offset = (random() % filesize) & (~(LEN - 1));
pwrite(fd, buf, LEN, offset);
}
printf("start test\n");
time(&t1);
for (i = 0; i < 100000; i++){
offset = (random() % filesize) & (~(LEN - 1));
pread(fd, buf, LEN, offset);
}
time(&t2);
printf("%ld sec\n", t2-t1);
close(fd);
if (umount("/root/test1/") < 0) {
perror("cannot umount\n");
exit(1);
}
}
Signed-off-by: Hisashi Hifumi <hifumi.hisashi@oss.ntt.co.jp>
Cc: Nick Piggin <nickpiggin@yahoo.com.au>
Cc: Christoph Hellwig <hch@infradead.org>
Cc: Jan Kara <jack@ucw.cz>
Cc: <linux-ext4@vger.kernel.org>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
2008-07-29 06:46:36 +08:00
|
|
|
|
2022-02-10 04:21:27 +08:00
|
|
|
head = folio_buffers(folio);
|
|
|
|
if (!head)
|
|
|
|
return false;
|
2012-11-30 02:21:43 +08:00
|
|
|
blocksize = head->b_size;
|
2022-02-10 04:21:27 +08:00
|
|
|
to = min_t(unsigned, folio_size(folio) - from, count);
|
vfs: pagecache usage optimization for pagesize!=blocksize
When we read some part of a file through pagecache, if there is a
pagecache of corresponding index but this page is not uptodate, read IO
is issued and this page will be uptodate.
I think this is good for pagesize == blocksize environment but there is
room for improvement on pagesize != blocksize environment. Because in
this case a page can have multiple buffers and even if a page is not
uptodate, some buffers can be uptodate.
So I suggest that when all buffers which correspond to a part of a file
that we want to read are uptodate, use this pagecache and copy data from
this pagecache to user buffer even if a page is not uptodate. This can
reduce read IO and improve system throughput.
I wrote a benchmark program and got result number with this program.
This benchmark do:
1: mount and open a test file.
2: create a 512MB file.
3: close a file and umount.
4: mount and again open a test file.
5: pwrite randomly 300000 times on a test file. offset is aligned
by IO size(1024bytes).
6: measure time of preading randomly 100000 times on a test file.
The result was:
2.6.26
330 sec
2.6.26-patched
226 sec
Arch:i386
Filesystem:ext3
Blocksize:1024 bytes
Memory: 1GB
On ext3/4, a file is written through buffer/block. So random read/write
mixed workloads or random read after random write workloads are optimized
with this patch under pagesize != blocksize environment. This test result
showed this.
The benchmark program is as follows:
#include <stdio.h>
#include <sys/types.h>
#include <sys/stat.h>
#include <fcntl.h>
#include <unistd.h>
#include <time.h>
#include <stdlib.h>
#include <string.h>
#include <sys/mount.h>
#define LEN 1024
#define LOOP 1024*512 /* 512MB */
main(void)
{
unsigned long i, offset, filesize;
int fd;
char buf[LEN];
time_t t1, t2;
if (mount("/dev/sda1", "/root/test1/", "ext3", 0, 0) < 0) {
perror("cannot mount\n");
exit(1);
}
memset(buf, 0, LEN);
fd = open("/root/test1/testfile", O_CREAT|O_RDWR|O_TRUNC);
if (fd < 0) {
perror("cannot open file\n");
exit(1);
}
for (i = 0; i < LOOP; i++)
write(fd, buf, LEN);
close(fd);
if (umount("/root/test1/") < 0) {
perror("cannot umount\n");
exit(1);
}
if (mount("/dev/sda1", "/root/test1/", "ext3", 0, 0) < 0) {
perror("cannot mount\n");
exit(1);
}
fd = open("/root/test1/testfile", O_RDWR);
if (fd < 0) {
perror("cannot open file\n");
exit(1);
}
filesize = LEN * LOOP;
for (i = 0; i < 300000; i++){
offset = (random() % filesize) & (~(LEN - 1));
pwrite(fd, buf, LEN, offset);
}
printf("start test\n");
time(&t1);
for (i = 0; i < 100000; i++){
offset = (random() % filesize) & (~(LEN - 1));
pread(fd, buf, LEN, offset);
}
time(&t2);
printf("%ld sec\n", t2-t1);
close(fd);
if (umount("/root/test1/") < 0) {
perror("cannot umount\n");
exit(1);
}
}
Signed-off-by: Hisashi Hifumi <hifumi.hisashi@oss.ntt.co.jp>
Cc: Nick Piggin <nickpiggin@yahoo.com.au>
Cc: Christoph Hellwig <hch@infradead.org>
Cc: Jan Kara <jack@ucw.cz>
Cc: <linux-ext4@vger.kernel.org>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
2008-07-29 06:46:36 +08:00
|
|
|
to = from + to;
|
2022-02-10 04:21:27 +08:00
|
|
|
if (from < blocksize && to > folio_size(folio) - blocksize)
|
|
|
|
return false;
|
vfs: pagecache usage optimization for pagesize!=blocksize
When we read some part of a file through pagecache, if there is a
pagecache of corresponding index but this page is not uptodate, read IO
is issued and this page will be uptodate.
I think this is good for pagesize == blocksize environment but there is
room for improvement on pagesize != blocksize environment. Because in
this case a page can have multiple buffers and even if a page is not
uptodate, some buffers can be uptodate.
So I suggest that when all buffers which correspond to a part of a file
that we want to read are uptodate, use this pagecache and copy data from
this pagecache to user buffer even if a page is not uptodate. This can
reduce read IO and improve system throughput.
I wrote a benchmark program and got result number with this program.
This benchmark do:
1: mount and open a test file.
2: create a 512MB file.
3: close a file and umount.
4: mount and again open a test file.
5: pwrite randomly 300000 times on a test file. offset is aligned
by IO size(1024bytes).
6: measure time of preading randomly 100000 times on a test file.
The result was:
2.6.26
330 sec
2.6.26-patched
226 sec
Arch:i386
Filesystem:ext3
Blocksize:1024 bytes
Memory: 1GB
On ext3/4, a file is written through buffer/block. So random read/write
mixed workloads or random read after random write workloads are optimized
with this patch under pagesize != blocksize environment. This test result
showed this.
The benchmark program is as follows:
#include <stdio.h>
#include <sys/types.h>
#include <sys/stat.h>
#include <fcntl.h>
#include <unistd.h>
#include <time.h>
#include <stdlib.h>
#include <string.h>
#include <sys/mount.h>
#define LEN 1024
#define LOOP 1024*512 /* 512MB */
main(void)
{
unsigned long i, offset, filesize;
int fd;
char buf[LEN];
time_t t1, t2;
if (mount("/dev/sda1", "/root/test1/", "ext3", 0, 0) < 0) {
perror("cannot mount\n");
exit(1);
}
memset(buf, 0, LEN);
fd = open("/root/test1/testfile", O_CREAT|O_RDWR|O_TRUNC);
if (fd < 0) {
perror("cannot open file\n");
exit(1);
}
for (i = 0; i < LOOP; i++)
write(fd, buf, LEN);
close(fd);
if (umount("/root/test1/") < 0) {
perror("cannot umount\n");
exit(1);
}
if (mount("/dev/sda1", "/root/test1/", "ext3", 0, 0) < 0) {
perror("cannot mount\n");
exit(1);
}
fd = open("/root/test1/testfile", O_RDWR);
if (fd < 0) {
perror("cannot open file\n");
exit(1);
}
filesize = LEN * LOOP;
for (i = 0; i < 300000; i++){
offset = (random() % filesize) & (~(LEN - 1));
pwrite(fd, buf, LEN, offset);
}
printf("start test\n");
time(&t1);
for (i = 0; i < 100000; i++){
offset = (random() % filesize) & (~(LEN - 1));
pread(fd, buf, LEN, offset);
}
time(&t2);
printf("%ld sec\n", t2-t1);
close(fd);
if (umount("/root/test1/") < 0) {
perror("cannot umount\n");
exit(1);
}
}
Signed-off-by: Hisashi Hifumi <hifumi.hisashi@oss.ntt.co.jp>
Cc: Nick Piggin <nickpiggin@yahoo.com.au>
Cc: Christoph Hellwig <hch@infradead.org>
Cc: Jan Kara <jack@ucw.cz>
Cc: <linux-ext4@vger.kernel.org>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
2008-07-29 06:46:36 +08:00
|
|
|
|
|
|
|
bh = head;
|
|
|
|
block_start = 0;
|
|
|
|
do {
|
|
|
|
block_end = block_start + blocksize;
|
|
|
|
if (block_end > from && block_start < to) {
|
|
|
|
if (!buffer_uptodate(bh)) {
|
2022-02-10 04:21:27 +08:00
|
|
|
ret = false;
|
vfs: pagecache usage optimization for pagesize!=blocksize
When we read some part of a file through pagecache, if there is a
pagecache of corresponding index but this page is not uptodate, read IO
is issued and this page will be uptodate.
I think this is good for pagesize == blocksize environment but there is
room for improvement on pagesize != blocksize environment. Because in
this case a page can have multiple buffers and even if a page is not
uptodate, some buffers can be uptodate.
So I suggest that when all buffers which correspond to a part of a file
that we want to read are uptodate, use this pagecache and copy data from
this pagecache to user buffer even if a page is not uptodate. This can
reduce read IO and improve system throughput.
I wrote a benchmark program and got result number with this program.
This benchmark do:
1: mount and open a test file.
2: create a 512MB file.
3: close a file and umount.
4: mount and again open a test file.
5: pwrite randomly 300000 times on a test file. offset is aligned
by IO size(1024bytes).
6: measure time of preading randomly 100000 times on a test file.
The result was:
2.6.26
330 sec
2.6.26-patched
226 sec
Arch:i386
Filesystem:ext3
Blocksize:1024 bytes
Memory: 1GB
On ext3/4, a file is written through buffer/block. So random read/write
mixed workloads or random read after random write workloads are optimized
with this patch under pagesize != blocksize environment. This test result
showed this.
The benchmark program is as follows:
#include <stdio.h>
#include <sys/types.h>
#include <sys/stat.h>
#include <fcntl.h>
#include <unistd.h>
#include <time.h>
#include <stdlib.h>
#include <string.h>
#include <sys/mount.h>
#define LEN 1024
#define LOOP 1024*512 /* 512MB */
main(void)
{
unsigned long i, offset, filesize;
int fd;
char buf[LEN];
time_t t1, t2;
if (mount("/dev/sda1", "/root/test1/", "ext3", 0, 0) < 0) {
perror("cannot mount\n");
exit(1);
}
memset(buf, 0, LEN);
fd = open("/root/test1/testfile", O_CREAT|O_RDWR|O_TRUNC);
if (fd < 0) {
perror("cannot open file\n");
exit(1);
}
for (i = 0; i < LOOP; i++)
write(fd, buf, LEN);
close(fd);
if (umount("/root/test1/") < 0) {
perror("cannot umount\n");
exit(1);
}
if (mount("/dev/sda1", "/root/test1/", "ext3", 0, 0) < 0) {
perror("cannot mount\n");
exit(1);
}
fd = open("/root/test1/testfile", O_RDWR);
if (fd < 0) {
perror("cannot open file\n");
exit(1);
}
filesize = LEN * LOOP;
for (i = 0; i < 300000; i++){
offset = (random() % filesize) & (~(LEN - 1));
pwrite(fd, buf, LEN, offset);
}
printf("start test\n");
time(&t1);
for (i = 0; i < 100000; i++){
offset = (random() % filesize) & (~(LEN - 1));
pread(fd, buf, LEN, offset);
}
time(&t2);
printf("%ld sec\n", t2-t1);
close(fd);
if (umount("/root/test1/") < 0) {
perror("cannot umount\n");
exit(1);
}
}
Signed-off-by: Hisashi Hifumi <hifumi.hisashi@oss.ntt.co.jp>
Cc: Nick Piggin <nickpiggin@yahoo.com.au>
Cc: Christoph Hellwig <hch@infradead.org>
Cc: Jan Kara <jack@ucw.cz>
Cc: <linux-ext4@vger.kernel.org>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
2008-07-29 06:46:36 +08:00
|
|
|
break;
|
|
|
|
}
|
|
|
|
if (block_end >= to)
|
|
|
|
break;
|
|
|
|
}
|
|
|
|
block_start = block_end;
|
|
|
|
bh = bh->b_this_page;
|
|
|
|
} while (bh != head);
|
|
|
|
|
|
|
|
return ret;
|
|
|
|
}
|
|
|
|
EXPORT_SYMBOL(block_is_partially_uptodate);
|
|
|
|
|
2005-04-17 06:20:36 +08:00
|
|
|
/*
|
2022-04-29 22:40:40 +08:00
|
|
|
* Generic "read_folio" function for block devices that have the normal
|
2005-04-17 06:20:36 +08:00
|
|
|
* get_block functionality. This is most of the block device filesystems.
|
2022-04-29 22:40:40 +08:00
|
|
|
* Reads the folio asynchronously --- the unlock_buffer() and
|
2005-04-17 06:20:36 +08:00
|
|
|
* set/clear_buffer_uptodate() functions propagate buffer state into the
|
2022-04-29 22:40:40 +08:00
|
|
|
* folio once IO has completed.
|
2005-04-17 06:20:36 +08:00
|
|
|
*/
|
2022-04-29 22:40:40 +08:00
|
|
|
int block_read_full_folio(struct folio *folio, get_block_t *get_block)
|
2005-04-17 06:20:36 +08:00
|
|
|
{
|
2022-04-29 22:40:40 +08:00
|
|
|
struct inode *inode = folio->mapping->host;
|
2005-04-17 06:20:36 +08:00
|
|
|
sector_t iblock, lblock;
|
|
|
|
struct buffer_head *bh, *head, *arr[MAX_BUF_PER_PAGE];
|
2023-11-10 05:06:08 +08:00
|
|
|
size_t blocksize;
|
2005-04-17 06:20:36 +08:00
|
|
|
int nr, i;
|
|
|
|
int fully_mapped = 1;
|
2022-05-27 11:19:49 +08:00
|
|
|
bool page_error = false;
|
2022-12-24 04:36:37 +08:00
|
|
|
loff_t limit = i_size_read(inode);
|
|
|
|
|
|
|
|
/* This is needed for ext4. */
|
|
|
|
if (IS_ENABLED(CONFIG_FS_VERITY) && IS_VERITY(inode))
|
|
|
|
limit = inode->i_sb->s_maxbytes;
|
2005-04-17 06:20:36 +08:00
|
|
|
|
2022-04-29 22:40:40 +08:00
|
|
|
VM_BUG_ON_FOLIO(folio_test_large(folio), folio);
|
|
|
|
|
2023-04-17 20:36:18 +08:00
|
|
|
head = folio_create_buffers(folio, inode, 0);
|
2012-11-30 02:21:43 +08:00
|
|
|
blocksize = head->b_size;
|
2005-04-17 06:20:36 +08:00
|
|
|
|
2023-11-10 05:06:08 +08:00
|
|
|
iblock = div_u64(folio_pos(folio), blocksize);
|
|
|
|
lblock = div_u64(limit + blocksize - 1, blocksize);
|
2005-04-17 06:20:36 +08:00
|
|
|
bh = head;
|
|
|
|
nr = 0;
|
|
|
|
i = 0;
|
|
|
|
|
|
|
|
do {
|
|
|
|
if (buffer_uptodate(bh))
|
|
|
|
continue;
|
|
|
|
|
|
|
|
if (!buffer_mapped(bh)) {
|
2005-05-17 12:53:49 +08:00
|
|
|
int err = 0;
|
|
|
|
|
2005-04-17 06:20:36 +08:00
|
|
|
fully_mapped = 0;
|
|
|
|
if (iblock < lblock) {
|
2006-03-26 17:38:00 +08:00
|
|
|
WARN_ON(bh->b_size != blocksize);
|
2005-05-17 12:53:49 +08:00
|
|
|
err = get_block(inode, iblock, bh, 0);
|
2022-05-27 11:19:49 +08:00
|
|
|
if (err) {
|
2022-04-29 22:40:40 +08:00
|
|
|
folio_set_error(folio);
|
2022-05-27 11:19:49 +08:00
|
|
|
page_error = true;
|
|
|
|
}
|
2005-04-17 06:20:36 +08:00
|
|
|
}
|
|
|
|
if (!buffer_mapped(bh)) {
|
2022-04-29 22:40:40 +08:00
|
|
|
folio_zero_range(folio, i * blocksize,
|
|
|
|
blocksize);
|
2005-05-17 12:53:49 +08:00
|
|
|
if (!err)
|
|
|
|
set_buffer_uptodate(bh);
|
2005-04-17 06:20:36 +08:00
|
|
|
continue;
|
|
|
|
}
|
|
|
|
/*
|
|
|
|
* get_block() might have updated the buffer
|
|
|
|
* synchronously
|
|
|
|
*/
|
|
|
|
if (buffer_uptodate(bh))
|
|
|
|
continue;
|
|
|
|
}
|
|
|
|
arr[nr++] = bh;
|
|
|
|
} while (i++, iblock++, (bh = bh->b_this_page) != head);
|
|
|
|
|
|
|
|
if (fully_mapped)
|
2022-04-29 22:40:40 +08:00
|
|
|
folio_set_mappedtodisk(folio);
|
2005-04-17 06:20:36 +08:00
|
|
|
|
|
|
|
if (!nr) {
|
|
|
|
/*
|
2023-10-05 00:53:05 +08:00
|
|
|
* All buffers are uptodate or get_block() returned an
|
|
|
|
* error when trying to map them - we can finish the read.
|
2005-04-17 06:20:36 +08:00
|
|
|
*/
|
2023-10-05 00:53:05 +08:00
|
|
|
folio_end_read(folio, !page_error);
|
2005-04-17 06:20:36 +08:00
|
|
|
return 0;
|
|
|
|
}
|
|
|
|
|
|
|
|
/* Stage two: lock the buffers */
|
|
|
|
for (i = 0; i < nr; i++) {
|
|
|
|
bh = arr[i];
|
|
|
|
lock_buffer(bh);
|
|
|
|
mark_buffer_async_read(bh);
|
|
|
|
}
|
|
|
|
|
|
|
|
/*
|
|
|
|
* Stage 3: start the IO. Check for uptodateness
|
|
|
|
* inside the buffer lock in case another process reading
|
|
|
|
* the underlying blockdev brought it uptodate (the sct fix).
|
|
|
|
*/
|
|
|
|
for (i = 0; i < nr; i++) {
|
|
|
|
bh = arr[i];
|
|
|
|
if (buffer_uptodate(bh))
|
|
|
|
end_buffer_async_read(bh, 1);
|
|
|
|
else
|
2022-07-15 02:07:13 +08:00
|
|
|
submit_bh(REQ_OP_READ, bh);
|
2005-04-17 06:20:36 +08:00
|
|
|
}
|
|
|
|
return 0;
|
|
|
|
}
|
2022-04-29 22:40:40 +08:00
|
|
|
EXPORT_SYMBOL(block_read_full_folio);
|
2005-04-17 06:20:36 +08:00
|
|
|
|
|
|
|
/* utility function for filesystems that need to do work on expanding
|
2007-10-16 16:25:07 +08:00
|
|
|
* truncates. Uses filesystem pagecache writes to allow the filesystem to
|
2005-04-17 06:20:36 +08:00
|
|
|
* deal with the hole.
|
|
|
|
*/
|
2007-10-16 16:25:07 +08:00
|
|
|
int generic_cont_expand_simple(struct inode *inode, loff_t size)
|
2005-04-17 06:20:36 +08:00
|
|
|
{
|
|
|
|
struct address_space *mapping = inode->i_mapping;
|
2022-03-04 02:35:20 +08:00
|
|
|
const struct address_space_operations *aops = mapping->a_ops;
|
2005-04-17 06:20:36 +08:00
|
|
|
struct page *page;
|
2022-09-15 23:04:16 +08:00
|
|
|
void *fsdata = NULL;
|
2005-04-17 06:20:36 +08:00
|
|
|
int err;
|
|
|
|
|
2009-08-21 00:35:06 +08:00
|
|
|
err = inode_newsize_ok(inode, size);
|
|
|
|
if (err)
|
2005-04-17 06:20:36 +08:00
|
|
|
goto out;
|
|
|
|
|
2022-03-04 02:35:20 +08:00
|
|
|
err = aops->write_begin(NULL, mapping, size, 0, &page, &fsdata);
|
2007-10-16 16:25:07 +08:00
|
|
|
if (err)
|
2006-01-08 17:02:13 +08:00
|
|
|
goto out;
|
|
|
|
|
2022-03-04 02:35:20 +08:00
|
|
|
err = aops->write_end(NULL, mapping, size, 0, 0, page, fsdata);
|
2007-10-16 16:25:07 +08:00
|
|
|
BUG_ON(err > 0);
|
2006-01-08 17:02:13 +08:00
|
|
|
|
2005-04-17 06:20:36 +08:00
|
|
|
out:
|
|
|
|
return err;
|
|
|
|
}
|
2009-09-23 07:43:51 +08:00
|
|
|
EXPORT_SYMBOL(generic_cont_expand_simple);
|
2005-04-17 06:20:36 +08:00
|
|
|
|
2008-04-29 15:59:01 +08:00
|
|
|
static int cont_expand_zero(struct file *file, struct address_space *mapping,
|
|
|
|
loff_t pos, loff_t *bytes)
|
2005-04-17 06:20:36 +08:00
|
|
|
{
|
|
|
|
struct inode *inode = mapping->host;
|
2022-03-04 02:35:20 +08:00
|
|
|
const struct address_space_operations *aops = mapping->a_ops;
|
2017-02-28 06:28:32 +08:00
|
|
|
unsigned int blocksize = i_blocksize(inode);
|
2007-10-16 16:25:07 +08:00
|
|
|
struct page *page;
|
2022-09-15 23:04:16 +08:00
|
|
|
void *fsdata = NULL;
|
2007-10-16 16:25:07 +08:00
|
|
|
pgoff_t index, curidx;
|
|
|
|
loff_t curpos;
|
|
|
|
unsigned zerofrom, offset, len;
|
|
|
|
int err = 0;
|
2005-04-17 06:20:36 +08:00
|
|
|
|
mm, fs: get rid of PAGE_CACHE_* and page_cache_{get,release} macros
PAGE_CACHE_{SIZE,SHIFT,MASK,ALIGN} macros were introduced *long* time
ago with promise that one day it will be possible to implement page
cache with bigger chunks than PAGE_SIZE.
This promise never materialized. And unlikely will.
We have many places where PAGE_CACHE_SIZE assumed to be equal to
PAGE_SIZE. And it's constant source of confusion on whether
PAGE_CACHE_* or PAGE_* constant should be used in a particular case,
especially on the border between fs and mm.
Global switching to PAGE_CACHE_SIZE != PAGE_SIZE would cause to much
breakage to be doable.
Let's stop pretending that pages in page cache are special. They are
not.
The changes are pretty straight-forward:
- <foo> << (PAGE_CACHE_SHIFT - PAGE_SHIFT) -> <foo>;
- <foo> >> (PAGE_CACHE_SHIFT - PAGE_SHIFT) -> <foo>;
- PAGE_CACHE_{SIZE,SHIFT,MASK,ALIGN} -> PAGE_{SIZE,SHIFT,MASK,ALIGN};
- page_cache_get() -> get_page();
- page_cache_release() -> put_page();
This patch contains automated changes generated with coccinelle using
script below. For some reason, coccinelle doesn't patch header files.
I've called spatch for them manually.
The only adjustment after coccinelle is revert of changes to
PAGE_CAHCE_ALIGN definition: we are going to drop it later.
There are few places in the code where coccinelle didn't reach. I'll
fix them manually in a separate patch. Comments and documentation also
will be addressed with the separate patch.
virtual patch
@@
expression E;
@@
- E << (PAGE_CACHE_SHIFT - PAGE_SHIFT)
+ E
@@
expression E;
@@
- E >> (PAGE_CACHE_SHIFT - PAGE_SHIFT)
+ E
@@
@@
- PAGE_CACHE_SHIFT
+ PAGE_SHIFT
@@
@@
- PAGE_CACHE_SIZE
+ PAGE_SIZE
@@
@@
- PAGE_CACHE_MASK
+ PAGE_MASK
@@
expression E;
@@
- PAGE_CACHE_ALIGN(E)
+ PAGE_ALIGN(E)
@@
expression E;
@@
- page_cache_get(E)
+ get_page(E)
@@
expression E;
@@
- page_cache_release(E)
+ put_page(E)
Signed-off-by: Kirill A. Shutemov <kirill.shutemov@linux.intel.com>
Acked-by: Michal Hocko <mhocko@suse.com>
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
2016-04-01 20:29:47 +08:00
|
|
|
index = pos >> PAGE_SHIFT;
|
|
|
|
offset = pos & ~PAGE_MASK;
|
2007-10-16 16:25:07 +08:00
|
|
|
|
mm, fs: get rid of PAGE_CACHE_* and page_cache_{get,release} macros
PAGE_CACHE_{SIZE,SHIFT,MASK,ALIGN} macros were introduced *long* time
ago with promise that one day it will be possible to implement page
cache with bigger chunks than PAGE_SIZE.
This promise never materialized. And unlikely will.
We have many places where PAGE_CACHE_SIZE assumed to be equal to
PAGE_SIZE. And it's constant source of confusion on whether
PAGE_CACHE_* or PAGE_* constant should be used in a particular case,
especially on the border between fs and mm.
Global switching to PAGE_CACHE_SIZE != PAGE_SIZE would cause to much
breakage to be doable.
Let's stop pretending that pages in page cache are special. They are
not.
The changes are pretty straight-forward:
- <foo> << (PAGE_CACHE_SHIFT - PAGE_SHIFT) -> <foo>;
- <foo> >> (PAGE_CACHE_SHIFT - PAGE_SHIFT) -> <foo>;
- PAGE_CACHE_{SIZE,SHIFT,MASK,ALIGN} -> PAGE_{SIZE,SHIFT,MASK,ALIGN};
- page_cache_get() -> get_page();
- page_cache_release() -> put_page();
This patch contains automated changes generated with coccinelle using
script below. For some reason, coccinelle doesn't patch header files.
I've called spatch for them manually.
The only adjustment after coccinelle is revert of changes to
PAGE_CAHCE_ALIGN definition: we are going to drop it later.
There are few places in the code where coccinelle didn't reach. I'll
fix them manually in a separate patch. Comments and documentation also
will be addressed with the separate patch.
virtual patch
@@
expression E;
@@
- E << (PAGE_CACHE_SHIFT - PAGE_SHIFT)
+ E
@@
expression E;
@@
- E >> (PAGE_CACHE_SHIFT - PAGE_SHIFT)
+ E
@@
@@
- PAGE_CACHE_SHIFT
+ PAGE_SHIFT
@@
@@
- PAGE_CACHE_SIZE
+ PAGE_SIZE
@@
@@
- PAGE_CACHE_MASK
+ PAGE_MASK
@@
expression E;
@@
- PAGE_CACHE_ALIGN(E)
+ PAGE_ALIGN(E)
@@
expression E;
@@
- page_cache_get(E)
+ get_page(E)
@@
expression E;
@@
- page_cache_release(E)
+ put_page(E)
Signed-off-by: Kirill A. Shutemov <kirill.shutemov@linux.intel.com>
Acked-by: Michal Hocko <mhocko@suse.com>
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
2016-04-01 20:29:47 +08:00
|
|
|
while (index > (curidx = (curpos = *bytes)>>PAGE_SHIFT)) {
|
|
|
|
zerofrom = curpos & ~PAGE_MASK;
|
2005-04-17 06:20:36 +08:00
|
|
|
if (zerofrom & (blocksize-1)) {
|
|
|
|
*bytes |= (blocksize-1);
|
|
|
|
(*bytes)++;
|
|
|
|
}
|
mm, fs: get rid of PAGE_CACHE_* and page_cache_{get,release} macros
PAGE_CACHE_{SIZE,SHIFT,MASK,ALIGN} macros were introduced *long* time
ago with promise that one day it will be possible to implement page
cache with bigger chunks than PAGE_SIZE.
This promise never materialized. And unlikely will.
We have many places where PAGE_CACHE_SIZE assumed to be equal to
PAGE_SIZE. And it's constant source of confusion on whether
PAGE_CACHE_* or PAGE_* constant should be used in a particular case,
especially on the border between fs and mm.
Global switching to PAGE_CACHE_SIZE != PAGE_SIZE would cause to much
breakage to be doable.
Let's stop pretending that pages in page cache are special. They are
not.
The changes are pretty straight-forward:
- <foo> << (PAGE_CACHE_SHIFT - PAGE_SHIFT) -> <foo>;
- <foo> >> (PAGE_CACHE_SHIFT - PAGE_SHIFT) -> <foo>;
- PAGE_CACHE_{SIZE,SHIFT,MASK,ALIGN} -> PAGE_{SIZE,SHIFT,MASK,ALIGN};
- page_cache_get() -> get_page();
- page_cache_release() -> put_page();
This patch contains automated changes generated with coccinelle using
script below. For some reason, coccinelle doesn't patch header files.
I've called spatch for them manually.
The only adjustment after coccinelle is revert of changes to
PAGE_CAHCE_ALIGN definition: we are going to drop it later.
There are few places in the code where coccinelle didn't reach. I'll
fix them manually in a separate patch. Comments and documentation also
will be addressed with the separate patch.
virtual patch
@@
expression E;
@@
- E << (PAGE_CACHE_SHIFT - PAGE_SHIFT)
+ E
@@
expression E;
@@
- E >> (PAGE_CACHE_SHIFT - PAGE_SHIFT)
+ E
@@
@@
- PAGE_CACHE_SHIFT
+ PAGE_SHIFT
@@
@@
- PAGE_CACHE_SIZE
+ PAGE_SIZE
@@
@@
- PAGE_CACHE_MASK
+ PAGE_MASK
@@
expression E;
@@
- PAGE_CACHE_ALIGN(E)
+ PAGE_ALIGN(E)
@@
expression E;
@@
- page_cache_get(E)
+ get_page(E)
@@
expression E;
@@
- page_cache_release(E)
+ put_page(E)
Signed-off-by: Kirill A. Shutemov <kirill.shutemov@linux.intel.com>
Acked-by: Michal Hocko <mhocko@suse.com>
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
2016-04-01 20:29:47 +08:00
|
|
|
len = PAGE_SIZE - zerofrom;
|
2005-04-17 06:20:36 +08:00
|
|
|
|
2022-03-04 02:35:20 +08:00
|
|
|
err = aops->write_begin(file, mapping, curpos, len,
|
2017-05-09 06:58:59 +08:00
|
|
|
&page, &fsdata);
|
2007-10-16 16:25:07 +08:00
|
|
|
if (err)
|
|
|
|
goto out;
|
2008-02-05 14:28:29 +08:00
|
|
|
zero_user(page, zerofrom, len);
|
2022-03-04 02:35:20 +08:00
|
|
|
err = aops->write_end(file, mapping, curpos, len, len,
|
2007-10-16 16:25:07 +08:00
|
|
|
page, fsdata);
|
|
|
|
if (err < 0)
|
|
|
|
goto out;
|
|
|
|
BUG_ON(err != len);
|
|
|
|
err = 0;
|
2008-04-28 17:16:28 +08:00
|
|
|
|
|
|
|
balance_dirty_pages_ratelimited(mapping);
|
2014-07-28 01:00:41 +08:00
|
|
|
|
2019-01-04 07:28:58 +08:00
|
|
|
if (fatal_signal_pending(current)) {
|
2014-07-28 01:00:41 +08:00
|
|
|
err = -EINTR;
|
|
|
|
goto out;
|
|
|
|
}
|
2007-10-16 16:25:07 +08:00
|
|
|
}
|
2005-04-17 06:20:36 +08:00
|
|
|
|
2007-10-16 16:25:07 +08:00
|
|
|
/* page covers the boundary, find the boundary offset */
|
|
|
|
if (index == curidx) {
|
mm, fs: get rid of PAGE_CACHE_* and page_cache_{get,release} macros
PAGE_CACHE_{SIZE,SHIFT,MASK,ALIGN} macros were introduced *long* time
ago with promise that one day it will be possible to implement page
cache with bigger chunks than PAGE_SIZE.
This promise never materialized. And unlikely will.
We have many places where PAGE_CACHE_SIZE assumed to be equal to
PAGE_SIZE. And it's constant source of confusion on whether
PAGE_CACHE_* or PAGE_* constant should be used in a particular case,
especially on the border between fs and mm.
Global switching to PAGE_CACHE_SIZE != PAGE_SIZE would cause to much
breakage to be doable.
Let's stop pretending that pages in page cache are special. They are
not.
The changes are pretty straight-forward:
- <foo> << (PAGE_CACHE_SHIFT - PAGE_SHIFT) -> <foo>;
- <foo> >> (PAGE_CACHE_SHIFT - PAGE_SHIFT) -> <foo>;
- PAGE_CACHE_{SIZE,SHIFT,MASK,ALIGN} -> PAGE_{SIZE,SHIFT,MASK,ALIGN};
- page_cache_get() -> get_page();
- page_cache_release() -> put_page();
This patch contains automated changes generated with coccinelle using
script below. For some reason, coccinelle doesn't patch header files.
I've called spatch for them manually.
The only adjustment after coccinelle is revert of changes to
PAGE_CAHCE_ALIGN definition: we are going to drop it later.
There are few places in the code where coccinelle didn't reach. I'll
fix them manually in a separate patch. Comments and documentation also
will be addressed with the separate patch.
virtual patch
@@
expression E;
@@
- E << (PAGE_CACHE_SHIFT - PAGE_SHIFT)
+ E
@@
expression E;
@@
- E >> (PAGE_CACHE_SHIFT - PAGE_SHIFT)
+ E
@@
@@
- PAGE_CACHE_SHIFT
+ PAGE_SHIFT
@@
@@
- PAGE_CACHE_SIZE
+ PAGE_SIZE
@@
@@
- PAGE_CACHE_MASK
+ PAGE_MASK
@@
expression E;
@@
- PAGE_CACHE_ALIGN(E)
+ PAGE_ALIGN(E)
@@
expression E;
@@
- page_cache_get(E)
+ get_page(E)
@@
expression E;
@@
- page_cache_release(E)
+ put_page(E)
Signed-off-by: Kirill A. Shutemov <kirill.shutemov@linux.intel.com>
Acked-by: Michal Hocko <mhocko@suse.com>
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
2016-04-01 20:29:47 +08:00
|
|
|
zerofrom = curpos & ~PAGE_MASK;
|
2005-04-17 06:20:36 +08:00
|
|
|
/* if we will expand the thing last block will be filled */
|
2007-10-16 16:25:07 +08:00
|
|
|
if (offset <= zerofrom) {
|
|
|
|
goto out;
|
|
|
|
}
|
|
|
|
if (zerofrom & (blocksize-1)) {
|
2005-04-17 06:20:36 +08:00
|
|
|
*bytes |= (blocksize-1);
|
|
|
|
(*bytes)++;
|
|
|
|
}
|
2007-10-16 16:25:07 +08:00
|
|
|
len = offset - zerofrom;
|
2005-04-17 06:20:36 +08:00
|
|
|
|
2022-03-04 02:35:20 +08:00
|
|
|
err = aops->write_begin(file, mapping, curpos, len,
|
2017-05-09 06:58:59 +08:00
|
|
|
&page, &fsdata);
|
2007-10-16 16:25:07 +08:00
|
|
|
if (err)
|
|
|
|
goto out;
|
2008-02-05 14:28:29 +08:00
|
|
|
zero_user(page, zerofrom, len);
|
2022-03-04 02:35:20 +08:00
|
|
|
err = aops->write_end(file, mapping, curpos, len, len,
|
2007-10-16 16:25:07 +08:00
|
|
|
page, fsdata);
|
|
|
|
if (err < 0)
|
|
|
|
goto out;
|
|
|
|
BUG_ON(err != len);
|
|
|
|
err = 0;
|
2005-04-17 06:20:36 +08:00
|
|
|
}
|
2007-10-16 16:25:07 +08:00
|
|
|
out:
|
|
|
|
return err;
|
|
|
|
}
|
|
|
|
|
|
|
|
/*
|
|
|
|
* For moronic filesystems that do not allow holes in file.
|
|
|
|
* We may have to extend the file.
|
|
|
|
*/
|
2010-06-04 17:29:55 +08:00
|
|
|
int cont_write_begin(struct file *file, struct address_space *mapping,
|
2022-02-23 00:25:12 +08:00
|
|
|
loff_t pos, unsigned len,
|
2007-10-16 16:25:07 +08:00
|
|
|
struct page **pagep, void **fsdata,
|
|
|
|
get_block_t *get_block, loff_t *bytes)
|
|
|
|
{
|
|
|
|
struct inode *inode = mapping->host;
|
2017-02-28 06:28:32 +08:00
|
|
|
unsigned int blocksize = i_blocksize(inode);
|
|
|
|
unsigned int zerofrom;
|
2007-10-16 16:25:07 +08:00
|
|
|
int err;
|
|
|
|
|
|
|
|
err = cont_expand_zero(file, mapping, pos, bytes);
|
|
|
|
if (err)
|
2010-06-04 17:29:58 +08:00
|
|
|
return err;
|
2007-10-16 16:25:07 +08:00
|
|
|
|
mm, fs: get rid of PAGE_CACHE_* and page_cache_{get,release} macros
PAGE_CACHE_{SIZE,SHIFT,MASK,ALIGN} macros were introduced *long* time
ago with promise that one day it will be possible to implement page
cache with bigger chunks than PAGE_SIZE.
This promise never materialized. And unlikely will.
We have many places where PAGE_CACHE_SIZE assumed to be equal to
PAGE_SIZE. And it's constant source of confusion on whether
PAGE_CACHE_* or PAGE_* constant should be used in a particular case,
especially on the border between fs and mm.
Global switching to PAGE_CACHE_SIZE != PAGE_SIZE would cause to much
breakage to be doable.
Let's stop pretending that pages in page cache are special. They are
not.
The changes are pretty straight-forward:
- <foo> << (PAGE_CACHE_SHIFT - PAGE_SHIFT) -> <foo>;
- <foo> >> (PAGE_CACHE_SHIFT - PAGE_SHIFT) -> <foo>;
- PAGE_CACHE_{SIZE,SHIFT,MASK,ALIGN} -> PAGE_{SIZE,SHIFT,MASK,ALIGN};
- page_cache_get() -> get_page();
- page_cache_release() -> put_page();
This patch contains automated changes generated with coccinelle using
script below. For some reason, coccinelle doesn't patch header files.
I've called spatch for them manually.
The only adjustment after coccinelle is revert of changes to
PAGE_CAHCE_ALIGN definition: we are going to drop it later.
There are few places in the code where coccinelle didn't reach. I'll
fix them manually in a separate patch. Comments and documentation also
will be addressed with the separate patch.
virtual patch
@@
expression E;
@@
- E << (PAGE_CACHE_SHIFT - PAGE_SHIFT)
+ E
@@
expression E;
@@
- E >> (PAGE_CACHE_SHIFT - PAGE_SHIFT)
+ E
@@
@@
- PAGE_CACHE_SHIFT
+ PAGE_SHIFT
@@
@@
- PAGE_CACHE_SIZE
+ PAGE_SIZE
@@
@@
- PAGE_CACHE_MASK
+ PAGE_MASK
@@
expression E;
@@
- PAGE_CACHE_ALIGN(E)
+ PAGE_ALIGN(E)
@@
expression E;
@@
- page_cache_get(E)
+ get_page(E)
@@
expression E;
@@
- page_cache_release(E)
+ put_page(E)
Signed-off-by: Kirill A. Shutemov <kirill.shutemov@linux.intel.com>
Acked-by: Michal Hocko <mhocko@suse.com>
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
2016-04-01 20:29:47 +08:00
|
|
|
zerofrom = *bytes & ~PAGE_MASK;
|
2007-10-16 16:25:07 +08:00
|
|
|
if (pos+len > *bytes && zerofrom & (blocksize-1)) {
|
|
|
|
*bytes |= (blocksize-1);
|
|
|
|
(*bytes)++;
|
2005-04-17 06:20:36 +08:00
|
|
|
}
|
|
|
|
|
2022-02-23 00:25:12 +08:00
|
|
|
return block_write_begin(mapping, pos, len, pagep, get_block);
|
2005-04-17 06:20:36 +08:00
|
|
|
}
|
2009-09-23 07:43:51 +08:00
|
|
|
EXPORT_SYMBOL(cont_write_begin);
|
2005-04-17 06:20:36 +08:00
|
|
|
|
2023-06-26 13:55:18 +08:00
|
|
|
void block_commit_write(struct page *page, unsigned from, unsigned to)
|
2005-04-17 06:20:36 +08:00
|
|
|
{
|
2023-06-13 05:01:35 +08:00
|
|
|
struct folio *folio = page_folio(page);
|
2023-06-26 13:55:17 +08:00
|
|
|
__block_commit_write(folio, from, to);
|
2005-04-17 06:20:36 +08:00
|
|
|
}
|
2009-09-23 07:43:51 +08:00
|
|
|
EXPORT_SYMBOL(block_commit_write);
|
2005-04-17 06:20:36 +08:00
|
|
|
|
2007-07-19 15:39:55 +08:00
|
|
|
/*
|
|
|
|
* block_page_mkwrite() is not allowed to change the file size as it gets
|
|
|
|
* called from a page fault handler when a page is first dirtied. Hence we must
|
|
|
|
* be careful to check for EOF conditions here. We set the page up correctly
|
|
|
|
* for a written page which means we get ENOSPC checking when writing into
|
|
|
|
* holes and correct delalloc and unwritten extent mapping on filesystems that
|
|
|
|
* support these features.
|
|
|
|
*
|
|
|
|
* We are not allowed to take the i_mutex here so we have to play games to
|
|
|
|
* protect against truncate races as the page could now be beyond EOF. Because
|
fs: introduce new truncate sequence
Introduce a new truncate calling sequence into fs/mm subsystems. Rather than
setattr > vmtruncate > truncate, have filesystems call their truncate sequence
from ->setattr if filesystem specific operations are required. vmtruncate is
deprecated, and truncate_pagecache and inode_newsize_ok helpers introduced
previously should be used.
simple_setattr is introduced for simple in-ram filesystems to implement
the new truncate sequence. Eventually all filesystems should be converted
to implement a setattr, and the default code in notify_change should go
away.
simple_setsize is also introduced to perform just the ATTR_SIZE portion
of simple_setattr (ie. changing i_size and trimming pagecache).
To implement the new truncate sequence:
- filesystem specific manipulations (eg freeing blocks) must be done in
the setattr method rather than ->truncate.
- vmtruncate can not be used by core code to trim blocks past i_size in
the event of write failure after allocation, so this must be performed
in the fs code.
- convert usage of helpers block_write_begin, nobh_write_begin,
cont_write_begin, and *blockdev_direct_IO* to use _newtrunc postfixed
variants. These avoid calling vmtruncate to trim blocks (see previous).
- inode_setattr should not be used. generic_setattr is a new function
to be used to copy simple attributes into the generic inode.
- make use of the better opportunity to handle errors with the new sequence.
Big problem with the previous calling sequence: the filesystem is not called
until i_size has already changed. This means it is not allowed to fail the
call, and also it does not know what the previous i_size was. Also, generic
code calling vmtruncate to truncate allocated blocks in case of error had
no good way to return a meaningful error (or, for example, atomically handle
block deallocation).
Cc: Christoph Hellwig <hch@lst.de>
Acked-by: Jan Kara <jack@suse.cz>
Signed-off-by: Nick Piggin <npiggin@suse.de>
Signed-off-by: Al Viro <viro@zeniv.linux.org.uk>
2010-05-26 23:05:33 +08:00
|
|
|
* truncate writes the inode size before removing pages, once we have the
|
2007-07-19 15:39:55 +08:00
|
|
|
* page lock we can determine safely if the page is beyond EOF. If it is not
|
|
|
|
* beyond EOF, then the page is guaranteed safe against truncation until we
|
|
|
|
* unlock the page.
|
2011-05-24 06:23:35 +08:00
|
|
|
*
|
2012-06-12 22:20:37 +08:00
|
|
|
* Direct callers of this function should protect against filesystem freezing
|
2015-10-14 06:51:02 +08:00
|
|
|
* using sb_start_pagefault() - sb_end_pagefault() functions.
|
2007-07-19 15:39:55 +08:00
|
|
|
*/
|
2015-10-14 06:51:02 +08:00
|
|
|
int block_page_mkwrite(struct vm_area_struct *vma, struct vm_fault *vmf,
|
2011-05-24 06:23:34 +08:00
|
|
|
get_block_t get_block)
|
2007-07-19 15:39:55 +08:00
|
|
|
{
|
2023-06-13 05:01:34 +08:00
|
|
|
struct folio *folio = page_folio(vmf->page);
|
2013-01-24 06:07:38 +08:00
|
|
|
struct inode *inode = file_inode(vma->vm_file);
|
2007-07-19 15:39:55 +08:00
|
|
|
unsigned long end;
|
|
|
|
loff_t size;
|
2011-05-24 06:23:34 +08:00
|
|
|
int ret;
|
2007-07-19 15:39:55 +08:00
|
|
|
|
2023-06-13 05:01:34 +08:00
|
|
|
folio_lock(folio);
|
2007-07-19 15:39:55 +08:00
|
|
|
size = i_size_read(inode);
|
2023-06-13 05:01:34 +08:00
|
|
|
if ((folio->mapping != inode->i_mapping) ||
|
|
|
|
(folio_pos(folio) >= size)) {
|
2011-05-24 06:23:34 +08:00
|
|
|
/* We overload EFAULT to mean page got truncated */
|
|
|
|
ret = -EFAULT;
|
|
|
|
goto out_unlock;
|
2007-07-19 15:39:55 +08:00
|
|
|
}
|
|
|
|
|
2023-06-13 05:01:34 +08:00
|
|
|
end = folio_size(folio);
|
|
|
|
/* folio is wholly or partially inside EOF */
|
|
|
|
if (folio_pos(folio) + end > size)
|
|
|
|
end = size - folio_pos(folio);
|
2007-07-19 15:39:55 +08:00
|
|
|
|
2023-06-13 05:01:34 +08:00
|
|
|
ret = __block_write_begin_int(folio, 0, end, get_block, NULL);
|
2023-06-26 13:55:18 +08:00
|
|
|
if (unlikely(ret))
|
2011-05-24 06:23:34 +08:00
|
|
|
goto out_unlock;
|
2023-06-26 13:55:18 +08:00
|
|
|
|
|
|
|
__block_commit_write(folio, 0, end);
|
|
|
|
|
2023-06-13 05:01:34 +08:00
|
|
|
folio_mark_dirty(folio);
|
|
|
|
folio_wait_stable(folio);
|
2011-05-24 06:23:34 +08:00
|
|
|
return 0;
|
|
|
|
out_unlock:
|
2023-06-13 05:01:34 +08:00
|
|
|
folio_unlock(folio);
|
2007-07-19 15:39:55 +08:00
|
|
|
return ret;
|
2011-05-24 06:23:34 +08:00
|
|
|
}
|
2009-09-23 07:43:51 +08:00
|
|
|
EXPORT_SYMBOL(block_page_mkwrite);
|
2005-04-17 06:20:36 +08:00
|
|
|
|
|
|
|
int block_truncate_page(struct address_space *mapping,
|
|
|
|
loff_t from, get_block_t *get_block)
|
|
|
|
{
|
mm, fs: get rid of PAGE_CACHE_* and page_cache_{get,release} macros
PAGE_CACHE_{SIZE,SHIFT,MASK,ALIGN} macros were introduced *long* time
ago with promise that one day it will be possible to implement page
cache with bigger chunks than PAGE_SIZE.
This promise never materialized. And unlikely will.
We have many places where PAGE_CACHE_SIZE assumed to be equal to
PAGE_SIZE. And it's constant source of confusion on whether
PAGE_CACHE_* or PAGE_* constant should be used in a particular case,
especially on the border between fs and mm.
Global switching to PAGE_CACHE_SIZE != PAGE_SIZE would cause to much
breakage to be doable.
Let's stop pretending that pages in page cache are special. They are
not.
The changes are pretty straight-forward:
- <foo> << (PAGE_CACHE_SHIFT - PAGE_SHIFT) -> <foo>;
- <foo> >> (PAGE_CACHE_SHIFT - PAGE_SHIFT) -> <foo>;
- PAGE_CACHE_{SIZE,SHIFT,MASK,ALIGN} -> PAGE_{SIZE,SHIFT,MASK,ALIGN};
- page_cache_get() -> get_page();
- page_cache_release() -> put_page();
This patch contains automated changes generated with coccinelle using
script below. For some reason, coccinelle doesn't patch header files.
I've called spatch for them manually.
The only adjustment after coccinelle is revert of changes to
PAGE_CAHCE_ALIGN definition: we are going to drop it later.
There are few places in the code where coccinelle didn't reach. I'll
fix them manually in a separate patch. Comments and documentation also
will be addressed with the separate patch.
virtual patch
@@
expression E;
@@
- E << (PAGE_CACHE_SHIFT - PAGE_SHIFT)
+ E
@@
expression E;
@@
- E >> (PAGE_CACHE_SHIFT - PAGE_SHIFT)
+ E
@@
@@
- PAGE_CACHE_SHIFT
+ PAGE_SHIFT
@@
@@
- PAGE_CACHE_SIZE
+ PAGE_SIZE
@@
@@
- PAGE_CACHE_MASK
+ PAGE_MASK
@@
expression E;
@@
- PAGE_CACHE_ALIGN(E)
+ PAGE_ALIGN(E)
@@
expression E;
@@
- page_cache_get(E)
+ get_page(E)
@@
expression E;
@@
- page_cache_release(E)
+ put_page(E)
Signed-off-by: Kirill A. Shutemov <kirill.shutemov@linux.intel.com>
Acked-by: Michal Hocko <mhocko@suse.com>
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
2016-04-01 20:29:47 +08:00
|
|
|
pgoff_t index = from >> PAGE_SHIFT;
|
2005-04-17 06:20:36 +08:00
|
|
|
unsigned blocksize;
|
2006-01-08 17:03:05 +08:00
|
|
|
sector_t iblock;
|
2023-06-13 05:01:41 +08:00
|
|
|
size_t offset, length, pos;
|
2005-04-17 06:20:36 +08:00
|
|
|
struct inode *inode = mapping->host;
|
2023-06-13 05:01:41 +08:00
|
|
|
struct folio *folio;
|
2005-04-17 06:20:36 +08:00
|
|
|
struct buffer_head *bh;
|
2023-03-23 10:32:59 +08:00
|
|
|
int err = 0;
|
2005-04-17 06:20:36 +08:00
|
|
|
|
2017-02-28 06:28:32 +08:00
|
|
|
blocksize = i_blocksize(inode);
|
2023-06-13 05:01:41 +08:00
|
|
|
length = from & (blocksize - 1);
|
2005-04-17 06:20:36 +08:00
|
|
|
|
|
|
|
/* Block boundary? Nothing to do */
|
|
|
|
if (!length)
|
|
|
|
return 0;
|
|
|
|
|
|
|
|
length = blocksize - length;
|
2023-11-10 05:06:06 +08:00
|
|
|
iblock = ((loff_t)index * PAGE_SIZE) >> inode->i_blkbits;
|
|
|
|
|
2023-06-13 05:01:41 +08:00
|
|
|
folio = filemap_grab_folio(mapping, index);
|
|
|
|
if (IS_ERR(folio))
|
|
|
|
return PTR_ERR(folio);
|
2005-04-17 06:20:36 +08:00
|
|
|
|
2023-06-13 05:01:41 +08:00
|
|
|
bh = folio_buffers(folio);
|
2023-10-17 04:10:49 +08:00
|
|
|
if (!bh)
|
2023-10-17 04:11:14 +08:00
|
|
|
bh = create_empty_buffers(folio, blocksize, 0);
|
2005-04-17 06:20:36 +08:00
|
|
|
|
|
|
|
/* Find the buffer that contains "offset" */
|
2023-06-13 05:01:41 +08:00
|
|
|
offset = offset_in_folio(folio, from);
|
2005-04-17 06:20:36 +08:00
|
|
|
pos = blocksize;
|
|
|
|
while (offset >= pos) {
|
|
|
|
bh = bh->b_this_page;
|
|
|
|
iblock++;
|
|
|
|
pos += blocksize;
|
|
|
|
}
|
|
|
|
|
|
|
|
if (!buffer_mapped(bh)) {
|
2006-03-26 17:38:00 +08:00
|
|
|
WARN_ON(bh->b_size != blocksize);
|
2005-04-17 06:20:36 +08:00
|
|
|
err = get_block(inode, iblock, bh, 0);
|
|
|
|
if (err)
|
|
|
|
goto unlock;
|
|
|
|
/* unmapped? It's a hole - nothing to do */
|
|
|
|
if (!buffer_mapped(bh))
|
|
|
|
goto unlock;
|
|
|
|
}
|
|
|
|
|
|
|
|
/* Ok, it's mapped. Make sure it's up-to-date */
|
2023-06-13 05:01:41 +08:00
|
|
|
if (folio_test_uptodate(folio))
|
2005-04-17 06:20:36 +08:00
|
|
|
set_buffer_uptodate(bh);
|
|
|
|
|
2007-02-12 16:51:41 +08:00
|
|
|
if (!buffer_uptodate(bh) && !buffer_delay(bh) && !buffer_unwritten(bh)) {
|
2022-09-01 21:34:54 +08:00
|
|
|
err = bh_read(bh, 0);
|
2005-04-17 06:20:36 +08:00
|
|
|
/* Uhhuh. Read error. Complain and punt. */
|
2022-09-01 21:34:54 +08:00
|
|
|
if (err < 0)
|
2005-04-17 06:20:36 +08:00
|
|
|
goto unlock;
|
|
|
|
}
|
|
|
|
|
2023-06-13 05:01:41 +08:00
|
|
|
folio_zero_range(folio, offset, length);
|
2005-04-17 06:20:36 +08:00
|
|
|
mark_buffer_dirty(bh);
|
|
|
|
|
|
|
|
unlock:
|
2023-06-13 05:01:41 +08:00
|
|
|
folio_unlock(folio);
|
|
|
|
folio_put(folio);
|
2023-03-23 10:32:59 +08:00
|
|
|
|
2005-04-17 06:20:36 +08:00
|
|
|
return err;
|
|
|
|
}
|
2009-09-23 07:43:51 +08:00
|
|
|
EXPORT_SYMBOL(block_truncate_page);
|
2005-04-17 06:20:36 +08:00
|
|
|
|
|
|
|
/*
|
|
|
|
* The generic ->writepage function for buffer-backed address_spaces
|
|
|
|
*/
|
2023-12-16 04:02:44 +08:00
|
|
|
int block_write_full_folio(struct folio *folio, struct writeback_control *wbc,
|
|
|
|
void *get_block)
|
2005-04-17 06:20:36 +08:00
|
|
|
{
|
2023-06-13 05:01:33 +08:00
|
|
|
struct inode * const inode = folio->mapping->host;
|
2005-04-17 06:20:36 +08:00
|
|
|
loff_t i_size = i_size_read(inode);
|
|
|
|
|
2023-06-13 05:01:33 +08:00
|
|
|
/* Is the folio fully inside i_size? */
|
|
|
|
if (folio_pos(folio) + folio_size(folio) <= i_size)
|
2023-12-16 04:02:45 +08:00
|
|
|
return __block_write_full_folio(inode, folio, get_block, wbc);
|
2005-04-17 06:20:36 +08:00
|
|
|
|
2023-06-13 05:01:33 +08:00
|
|
|
/* Is the folio fully outside i_size? (truncate in progress) */
|
|
|
|
if (folio_pos(folio) >= i_size) {
|
2023-06-13 05:01:31 +08:00
|
|
|
folio_unlock(folio);
|
2005-04-17 06:20:36 +08:00
|
|
|
return 0; /* don't care */
|
|
|
|
}
|
|
|
|
|
|
|
|
/*
|
2023-06-13 05:01:33 +08:00
|
|
|
* The folio straddles i_size. It must be zeroed out on each and every
|
2009-12-12 05:35:40 +08:00
|
|
|
* writepage invocation because it may be mmapped. "A file is mapped
|
2005-04-17 06:20:36 +08:00
|
|
|
* in multiples of the page size. For a file that is not a multiple of
|
2023-06-13 05:01:33 +08:00
|
|
|
* the page size, the remaining memory is zeroed when mapped, and
|
2005-04-17 06:20:36 +08:00
|
|
|
* writes to that region are not written out to the file."
|
|
|
|
*/
|
2023-06-13 05:01:33 +08:00
|
|
|
folio_zero_segment(folio, offset_in_folio(folio, i_size),
|
|
|
|
folio_size(folio));
|
2023-12-16 04:02:45 +08:00
|
|
|
return __block_write_full_folio(inode, folio, get_block, wbc);
|
2009-04-16 01:22:38 +08:00
|
|
|
}
|
|
|
|
|
2005-04-17 06:20:36 +08:00
|
|
|
sector_t generic_block_bmap(struct address_space *mapping, sector_t block,
|
|
|
|
get_block_t *get_block)
|
|
|
|
{
|
|
|
|
struct inode *inode = mapping->host;
|
2017-07-05 12:56:21 +08:00
|
|
|
struct buffer_head tmp = {
|
|
|
|
.b_size = i_blocksize(inode),
|
|
|
|
};
|
|
|
|
|
2005-04-17 06:20:36 +08:00
|
|
|
get_block(inode, block, &tmp, 0);
|
|
|
|
return tmp.b_blocknr;
|
|
|
|
}
|
2009-09-23 07:43:51 +08:00
|
|
|
EXPORT_SYMBOL(generic_block_bmap);
|
2005-04-17 06:20:36 +08:00
|
|
|
|
2015-07-20 21:29:37 +08:00
|
|
|
static void end_bio_bh_io_sync(struct bio *bio)
|
2005-04-17 06:20:36 +08:00
|
|
|
{
|
|
|
|
struct buffer_head *bh = bio->bi_private;
|
|
|
|
|
2015-07-25 02:37:59 +08:00
|
|
|
if (unlikely(bio_flagged(bio, BIO_QUIET)))
|
block: Supress Buffer I/O errors when SCSI REQ_QUIET flag set
Allow the scsi request REQ_QUIET flag to be propagated to the buffer
file system layer. The basic ideas is to pass the flag from the scsi
request to the bio (block IO) and then to the buffer layer. The buffer
layer can then suppress needless printks.
This patch declutters the kernel log by removed the 40-50 (per lun)
buffer io error messages seen during a boot in my multipath setup . It
is a good chance any real errors will be missed in the "noise" it the
logs without this patch.
During boot I see blocks of messages like
"
__ratelimit: 211 callbacks suppressed
Buffer I/O error on device sdm, logical block 5242879
Buffer I/O error on device sdm, logical block 5242879
Buffer I/O error on device sdm, logical block 5242847
Buffer I/O error on device sdm, logical block 1
Buffer I/O error on device sdm, logical block 5242878
Buffer I/O error on device sdm, logical block 5242879
Buffer I/O error on device sdm, logical block 5242879
Buffer I/O error on device sdm, logical block 5242879
Buffer I/O error on device sdm, logical block 5242879
Buffer I/O error on device sdm, logical block 5242872
"
in my logs.
My disk environment is multipath fiber channel using the SCSI_DH_RDAC
code and multipathd. This topology includes an "active" and "ghost"
path for each lun. IO's to the "ghost" path will never complete and the
SCSI layer, via the scsi device handler rdac code, quick returns the IOs
to theses paths and sets the REQ_QUIET scsi flag to suppress the scsi
layer messages.
I am wanting to extend the QUIET behavior to include the buffer file
system layer to deal with these errors as well. I have been running this
patch for a while now on several boxes without issue. A few runs of
bonnie++ show no noticeable difference in performance in my setup.
Thanks for John Stultz for the quiet_error finalization.
Submitted-by: Keith Mannthey <kmannth@us.ibm.com>
Signed-off-by: Jens Axboe <jens.axboe@oracle.com>
2008-11-25 17:24:35 +08:00
|
|
|
set_bit(BH_Quiet, &bh->b_state);
|
|
|
|
|
2017-06-03 15:38:06 +08:00
|
|
|
bh->b_end_io(bh, !bio->bi_status);
|
2005-04-17 06:20:36 +08:00
|
|
|
bio_put(bio);
|
|
|
|
}
|
|
|
|
|
2022-08-18 13:04:40 +08:00
|
|
|
static void submit_bh_wbc(blk_opf_t opf, struct buffer_head *bh,
|
2024-02-03 04:39:25 +08:00
|
|
|
enum rw_hint write_hint,
|
2022-08-18 13:04:40 +08:00
|
|
|
struct writeback_control *wbc)
|
2005-04-17 06:20:36 +08:00
|
|
|
{
|
2022-07-15 02:07:13 +08:00
|
|
|
const enum req_op op = opf & REQ_OP_MASK;
|
2005-04-17 06:20:36 +08:00
|
|
|
struct bio *bio;
|
|
|
|
|
|
|
|
BUG_ON(!buffer_locked(bh));
|
|
|
|
BUG_ON(!buffer_mapped(bh));
|
|
|
|
BUG_ON(!bh->b_end_io);
|
2009-05-13 04:22:37 +08:00
|
|
|
BUG_ON(buffer_delay(bh));
|
|
|
|
BUG_ON(buffer_unwritten(bh));
|
2005-04-17 06:20:36 +08:00
|
|
|
|
|
|
|
/*
|
2008-08-22 16:00:36 +08:00
|
|
|
* Only clear out a write error when rewriting
|
2005-04-17 06:20:36 +08:00
|
|
|
*/
|
2016-06-06 03:31:43 +08:00
|
|
|
if (test_set_buffer_req(bh) && (op == REQ_OP_WRITE))
|
2005-04-17 06:20:36 +08:00
|
|
|
clear_buffer_write_io_error(bh);
|
|
|
|
|
2022-01-24 17:11:05 +08:00
|
|
|
if (buffer_meta(bh))
|
2022-07-15 02:07:13 +08:00
|
|
|
opf |= REQ_META;
|
2022-01-24 17:11:05 +08:00
|
|
|
if (buffer_prio(bh))
|
2022-07-15 02:07:13 +08:00
|
|
|
opf |= REQ_PRIO;
|
2022-01-24 17:11:05 +08:00
|
|
|
|
2022-07-15 02:07:13 +08:00
|
|
|
bio = bio_alloc(bh->b_bdev, 1, opf, GFP_NOIO);
|
2005-04-17 06:20:36 +08:00
|
|
|
|
2020-07-02 09:56:07 +08:00
|
|
|
fscrypt_set_bio_crypt_ctx_bh(bio, bh, GFP_NOIO);
|
|
|
|
|
2013-10-12 06:44:27 +08:00
|
|
|
bio->bi_iter.bi_sector = bh->b_blocknr * (bh->b_size >> 9);
|
2024-02-03 04:39:25 +08:00
|
|
|
bio->bi_write_hint = write_hint;
|
2005-04-17 06:20:36 +08:00
|
|
|
|
2023-05-31 19:50:27 +08:00
|
|
|
__bio_add_page(bio, bh->b_page, bh->b_size, bh_offset(bh));
|
2005-04-17 06:20:36 +08:00
|
|
|
|
|
|
|
bio->bi_end_io = end_bio_bh_io_sync;
|
|
|
|
bio->bi_private = bh;
|
|
|
|
|
2020-01-05 09:41:14 +08:00
|
|
|
/* Take care of bh's that straddle the end of the device */
|
|
|
|
guard_bio_eod(bio);
|
|
|
|
|
2018-12-06 01:10:34 +08:00
|
|
|
if (wbc) {
|
|
|
|
wbc_init_bio(wbc, bio);
|
2019-06-28 04:39:49 +08:00
|
|
|
wbc_account_cgroup_owner(wbc, bh->b_page, bh->b_size);
|
2018-12-06 01:10:34 +08:00
|
|
|
}
|
|
|
|
|
2016-06-06 03:31:41 +08:00
|
|
|
submit_bio(bio);
|
2005-04-17 06:20:36 +08:00
|
|
|
}
|
2015-06-02 22:37:23 +08:00
|
|
|
|
2022-08-18 13:04:40 +08:00
|
|
|
void submit_bh(blk_opf_t opf, struct buffer_head *bh)
|
2015-06-02 22:37:23 +08:00
|
|
|
{
|
2024-02-03 04:39:25 +08:00
|
|
|
submit_bh_wbc(opf, bh, WRITE_LIFE_NOT_SET, NULL);
|
2013-04-30 06:07:25 +08:00
|
|
|
}
|
2009-09-23 07:43:51 +08:00
|
|
|
EXPORT_SYMBOL(submit_bh);
|
2005-04-17 06:20:36 +08:00
|
|
|
|
2022-07-15 02:07:12 +08:00
|
|
|
void write_dirty_buffer(struct buffer_head *bh, blk_opf_t op_flags)
|
2010-08-11 23:06:24 +08:00
|
|
|
{
|
|
|
|
lock_buffer(bh);
|
|
|
|
if (!test_clear_buffer_dirty(bh)) {
|
|
|
|
unlock_buffer(bh);
|
|
|
|
return;
|
|
|
|
}
|
|
|
|
bh->b_end_io = end_buffer_write_sync;
|
|
|
|
get_bh(bh);
|
2022-07-15 02:07:13 +08:00
|
|
|
submit_bh(REQ_OP_WRITE | op_flags, bh);
|
2010-08-11 23:06:24 +08:00
|
|
|
}
|
|
|
|
EXPORT_SYMBOL(write_dirty_buffer);
|
|
|
|
|
2005-04-17 06:20:36 +08:00
|
|
|
/*
|
|
|
|
* For a data-integrity writeout, we need to wait upon any in-progress I/O
|
|
|
|
* and then start new I/O and then wait upon it. The caller must have a ref on
|
|
|
|
* the buffer_head.
|
|
|
|
*/
|
2022-07-15 02:07:12 +08:00
|
|
|
int __sync_dirty_buffer(struct buffer_head *bh, blk_opf_t op_flags)
|
2005-04-17 06:20:36 +08:00
|
|
|
{
|
|
|
|
WARN_ON(atomic_read(&bh->b_count) < 1);
|
|
|
|
lock_buffer(bh);
|
|
|
|
if (test_clear_buffer_dirty(bh)) {
|
fs: prevent BUG_ON in submit_bh_wbc()
If a device is hot-removed --- for example, when a physical device is
unplugged from pcie slot or a nbd device's network is shutdown ---
this can result in a BUG_ON() crash in submit_bh_wbc(). This is
because the when the block device dies, the buffer heads will have
their Buffer_Mapped flag get cleared, leading to the crash in
submit_bh_wbc.
We had attempted to work around this problem in commit a17712c8
("ext4: check superblock mapped prior to committing"). Unfortunately,
it's still possible to hit the BUG_ON(!buffer_mapped(bh)) if the
device dies between when the work-around check in ext4_commit_super()
and when submit_bh_wbh() is finally called:
Code path:
ext4_commit_super
judge if 'buffer_mapped(sbh)' is false, return <== commit a17712c8
lock_buffer(sbh)
...
unlock_buffer(sbh)
__sync_dirty_buffer(sbh,...
lock_buffer(sbh)
judge if 'buffer_mapped(sbh))' is false, return <== added by this patch
submit_bh(...,sbh)
submit_bh_wbc(...,sbh,...)
[100722.966497] kernel BUG at fs/buffer.c:3095! <== BUG_ON(!buffer_mapped(bh))' in submit_bh_wbc()
[100722.966503] invalid opcode: 0000 [#1] SMP
[100722.966566] task: ffff8817e15a9e40 task.stack: ffffc90024744000
[100722.966574] RIP: 0010:submit_bh_wbc+0x180/0x190
[100722.966575] RSP: 0018:ffffc90024747a90 EFLAGS: 00010246
[100722.966576] RAX: 0000000000620005 RBX: ffff8818a80603a8 RCX: 0000000000000000
[100722.966576] RDX: ffff8818a80603a8 RSI: 0000000000020800 RDI: 0000000000000001
[100722.966577] RBP: ffffc90024747ac0 R08: 0000000000000000 R09: ffff88207f94170d
[100722.966578] R10: 00000000000437c8 R11: 0000000000000001 R12: 0000000000020800
[100722.966578] R13: 0000000000000001 R14: 000000000bf9a438 R15: ffff88195f333000
[100722.966580] FS: 00007fa2eee27700(0000) GS:ffff88203d840000(0000) knlGS:0000000000000000
[100722.966580] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033
[100722.966581] CR2: 0000000000f0b008 CR3: 000000201a622003 CR4: 00000000007606e0
[100722.966582] DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000
[100722.966583] DR3: 0000000000000000 DR6: 00000000fffe0ff0 DR7: 0000000000000400
[100722.966583] PKRU: 55555554
[100722.966583] Call Trace:
[100722.966588] __sync_dirty_buffer+0x6e/0xd0
[100722.966614] ext4_commit_super+0x1d8/0x290 [ext4]
[100722.966626] __ext4_std_error+0x78/0x100 [ext4]
[100722.966635] ? __ext4_journal_get_write_access+0xca/0x120 [ext4]
[100722.966646] ext4_reserve_inode_write+0x58/0xb0 [ext4]
[100722.966655] ? ext4_dirty_inode+0x48/0x70 [ext4]
[100722.966663] ext4_mark_inode_dirty+0x53/0x1e0 [ext4]
[100722.966671] ? __ext4_journal_start_sb+0x6d/0xf0 [ext4]
[100722.966679] ext4_dirty_inode+0x48/0x70 [ext4]
[100722.966682] __mark_inode_dirty+0x17f/0x350
[100722.966686] generic_update_time+0x87/0xd0
[100722.966687] touch_atime+0xa9/0xd0
[100722.966690] generic_file_read_iter+0xa09/0xcd0
[100722.966694] ? page_cache_tree_insert+0xb0/0xb0
[100722.966704] ext4_file_read_iter+0x4a/0x100 [ext4]
[100722.966707] ? __inode_security_revalidate+0x4f/0x60
[100722.966709] __vfs_read+0xec/0x160
[100722.966711] vfs_read+0x8c/0x130
[100722.966712] SyS_pread64+0x87/0xb0
[100722.966716] do_syscall_64+0x67/0x1b0
[100722.966719] entry_SYSCALL64_slow_path+0x25/0x25
To address this, add the check of 'buffer_mapped(bh)' to
__sync_dirty_buffer(). This also has the benefit of fixing this for
other file systems.
With this addition, we can drop the workaround in ext4_commit_supper().
[ Commit description rewritten by tytso. ]
Signed-off-by: Xianting Tian <xianting_tian@126.com>
Link: https://lore.kernel.org/r/1596211825-8750-1-git-send-email-xianting_tian@126.com
Signed-off-by: Theodore Ts'o <tytso@mit.edu>
2020-08-01 00:10:25 +08:00
|
|
|
/*
|
|
|
|
* The bh should be mapped, but it might not be if the
|
|
|
|
* device was hot-removed. Not much we can do but fail the I/O.
|
|
|
|
*/
|
|
|
|
if (!buffer_mapped(bh)) {
|
|
|
|
unlock_buffer(bh);
|
|
|
|
return -EIO;
|
|
|
|
}
|
|
|
|
|
2005-04-17 06:20:36 +08:00
|
|
|
get_bh(bh);
|
|
|
|
bh->b_end_io = end_buffer_write_sync;
|
2022-08-18 13:04:39 +08:00
|
|
|
submit_bh(REQ_OP_WRITE | op_flags, bh);
|
2005-04-17 06:20:36 +08:00
|
|
|
wait_on_buffer(bh);
|
2022-08-18 13:04:39 +08:00
|
|
|
if (!buffer_uptodate(bh))
|
|
|
|
return -EIO;
|
2005-04-17 06:20:36 +08:00
|
|
|
} else {
|
|
|
|
unlock_buffer(bh);
|
|
|
|
}
|
2022-08-18 13:04:39 +08:00
|
|
|
return 0;
|
2005-04-17 06:20:36 +08:00
|
|
|
}
|
2010-08-11 23:05:45 +08:00
|
|
|
EXPORT_SYMBOL(__sync_dirty_buffer);
|
|
|
|
|
|
|
|
int sync_dirty_buffer(struct buffer_head *bh)
|
|
|
|
{
|
2016-11-01 21:40:10 +08:00
|
|
|
return __sync_dirty_buffer(bh, REQ_SYNC);
|
2010-08-11 23:05:45 +08:00
|
|
|
}
|
2009-09-23 07:43:51 +08:00
|
|
|
EXPORT_SYMBOL(sync_dirty_buffer);
|
2005-04-17 06:20:36 +08:00
|
|
|
|
|
|
|
/*
|
2022-05-01 13:08:08 +08:00
|
|
|
* try_to_free_buffers() checks if all the buffers on this particular folio
|
2005-04-17 06:20:36 +08:00
|
|
|
* are unused, and releases them if so.
|
|
|
|
*
|
|
|
|
* Exclusion against try_to_free_buffers may be obtained by either
|
2023-11-18 05:58:23 +08:00
|
|
|
* locking the folio or by holding its mapping's i_private_lock.
|
2005-04-17 06:20:36 +08:00
|
|
|
*
|
2022-05-01 13:08:08 +08:00
|
|
|
* If the folio is dirty but all the buffers are clean then we need to
|
|
|
|
* be sure to mark the folio clean as well. This is because the folio
|
2005-04-17 06:20:36 +08:00
|
|
|
* may be against a block device, and a later reattachment of buffers
|
2022-05-01 13:08:08 +08:00
|
|
|
* to a dirty folio will set *all* buffers dirty. Which would corrupt
|
2005-04-17 06:20:36 +08:00
|
|
|
* filesystem data on the same device.
|
|
|
|
*
|
2022-05-01 13:08:08 +08:00
|
|
|
* The same applies to regular filesystem folios: if all the buffers are
|
|
|
|
* clean then we set the folio clean and proceed. To do that, we require
|
2022-02-10 04:22:12 +08:00
|
|
|
* total exclusion from block_dirty_folio(). That is obtained with
|
2023-11-18 05:58:23 +08:00
|
|
|
* i_private_lock.
|
2005-04-17 06:20:36 +08:00
|
|
|
*
|
|
|
|
* try_to_free_buffers() is non-blocking.
|
|
|
|
*/
|
|
|
|
static inline int buffer_busy(struct buffer_head *bh)
|
|
|
|
{
|
|
|
|
return atomic_read(&bh->b_count) |
|
|
|
|
(bh->b_state & ((1 << BH_Dirty) | (1 << BH_Lock)));
|
|
|
|
}
|
|
|
|
|
2022-05-01 18:39:50 +08:00
|
|
|
static bool
|
|
|
|
drop_buffers(struct folio *folio, struct buffer_head **buffers_to_free)
|
2005-04-17 06:20:36 +08:00
|
|
|
{
|
2022-05-01 18:39:50 +08:00
|
|
|
struct buffer_head *head = folio_buffers(folio);
|
2005-04-17 06:20:36 +08:00
|
|
|
struct buffer_head *bh;
|
|
|
|
|
|
|
|
bh = head;
|
|
|
|
do {
|
|
|
|
if (buffer_busy(bh))
|
|
|
|
goto failed;
|
|
|
|
bh = bh->b_this_page;
|
|
|
|
} while (bh != head);
|
|
|
|
|
|
|
|
do {
|
|
|
|
struct buffer_head *next = bh->b_this_page;
|
|
|
|
|
2008-02-08 20:21:59 +08:00
|
|
|
if (bh->b_assoc_map)
|
2005-04-17 06:20:36 +08:00
|
|
|
__remove_assoc_queue(bh);
|
|
|
|
bh = next;
|
|
|
|
} while (bh != head);
|
|
|
|
*buffers_to_free = head;
|
2022-05-01 18:39:50 +08:00
|
|
|
folio_detach_private(folio);
|
|
|
|
return true;
|
2005-04-17 06:20:36 +08:00
|
|
|
failed:
|
2022-05-01 18:39:50 +08:00
|
|
|
return false;
|
2005-04-17 06:20:36 +08:00
|
|
|
}
|
|
|
|
|
2022-05-01 13:08:08 +08:00
|
|
|
bool try_to_free_buffers(struct folio *folio)
|
2005-04-17 06:20:36 +08:00
|
|
|
{
|
2022-05-01 13:08:08 +08:00
|
|
|
struct address_space * const mapping = folio->mapping;
|
2005-04-17 06:20:36 +08:00
|
|
|
struct buffer_head *buffers_to_free = NULL;
|
2022-05-01 13:08:08 +08:00
|
|
|
bool ret = 0;
|
2005-04-17 06:20:36 +08:00
|
|
|
|
2022-05-01 13:08:08 +08:00
|
|
|
BUG_ON(!folio_test_locked(folio));
|
|
|
|
if (folio_test_writeback(folio))
|
|
|
|
return false;
|
2005-04-17 06:20:36 +08:00
|
|
|
|
|
|
|
if (mapping == NULL) { /* can this still happen? */
|
2022-05-01 18:39:50 +08:00
|
|
|
ret = drop_buffers(folio, &buffers_to_free);
|
2005-04-17 06:20:36 +08:00
|
|
|
goto out;
|
|
|
|
}
|
|
|
|
|
2023-11-18 05:58:23 +08:00
|
|
|
spin_lock(&mapping->i_private_lock);
|
2022-05-01 18:39:50 +08:00
|
|
|
ret = drop_buffers(folio, &buffers_to_free);
|
Resurrect 'try_to_free_buffers()' VM hackery
It's not pretty, but it appears that ext3 with data=journal will clean
pages without ever actually telling the VM that they are clean. This,
in turn, will result in the VM (and balance_dirty_pages() in particular)
to never realize that the pages got cleaned, and wait forever for an
event that already happened.
Technically, this seems to be a problem with ext3 itself, but it used to
be hidden by 'try_to_free_buffers()' noticing this situation on its own,
and just working around the filesystem problem.
This commit re-instates that hack, in order to avoid a regression for
the 2.6.20 release. This fixes bugzilla 7844:
http://bugzilla.kernel.org/show_bug.cgi?id=7844
Peter Zijlstra points out that we should probably retain the debugging
code that this removes from cancel_dirty_page(), and I agree, but for
the imminent release we might as well just silence the warning too
(since it's not a new bug: anything that triggers that warning has been
around forever).
Acked-by: Randy Dunlap <rdunlap@xenotime.net>
Acked-by: Jens Axboe <jens.axboe@oracle.com>
Acked-by: Peter Zijlstra <a.p.zijlstra@chello.nl>
Cc: Andrew Morton <akpm@osdl.org>
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
2007-01-27 04:47:06 +08:00
|
|
|
|
|
|
|
/*
|
|
|
|
* If the filesystem writes its buffers by hand (eg ext3)
|
2022-05-01 13:08:08 +08:00
|
|
|
* then we can have clean buffers against a dirty folio. We
|
|
|
|
* clean the folio here; otherwise the VM will never notice
|
Resurrect 'try_to_free_buffers()' VM hackery
It's not pretty, but it appears that ext3 with data=journal will clean
pages without ever actually telling the VM that they are clean. This,
in turn, will result in the VM (and balance_dirty_pages() in particular)
to never realize that the pages got cleaned, and wait forever for an
event that already happened.
Technically, this seems to be a problem with ext3 itself, but it used to
be hidden by 'try_to_free_buffers()' noticing this situation on its own,
and just working around the filesystem problem.
This commit re-instates that hack, in order to avoid a regression for
the 2.6.20 release. This fixes bugzilla 7844:
http://bugzilla.kernel.org/show_bug.cgi?id=7844
Peter Zijlstra points out that we should probably retain the debugging
code that this removes from cancel_dirty_page(), and I agree, but for
the imminent release we might as well just silence the warning too
(since it's not a new bug: anything that triggers that warning has been
around forever).
Acked-by: Randy Dunlap <rdunlap@xenotime.net>
Acked-by: Jens Axboe <jens.axboe@oracle.com>
Acked-by: Peter Zijlstra <a.p.zijlstra@chello.nl>
Cc: Andrew Morton <akpm@osdl.org>
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
2007-01-27 04:47:06 +08:00
|
|
|
* that the filesystem did any IO at all.
|
|
|
|
*
|
|
|
|
* Also, during truncate, discard_buffer will have marked all
|
2022-05-01 13:08:08 +08:00
|
|
|
* the folio's buffers clean. We discover that here and clean
|
|
|
|
* the folio also.
|
2007-01-30 11:36:27 +08:00
|
|
|
*
|
2023-11-18 05:58:23 +08:00
|
|
|
* i_private_lock must be held over this entire operation in order
|
2022-02-10 04:22:12 +08:00
|
|
|
* to synchronise against block_dirty_folio and prevent the
|
2007-01-30 11:36:27 +08:00
|
|
|
* dirty bit from being lost.
|
Resurrect 'try_to_free_buffers()' VM hackery
It's not pretty, but it appears that ext3 with data=journal will clean
pages without ever actually telling the VM that they are clean. This,
in turn, will result in the VM (and balance_dirty_pages() in particular)
to never realize that the pages got cleaned, and wait forever for an
event that already happened.
Technically, this seems to be a problem with ext3 itself, but it used to
be hidden by 'try_to_free_buffers()' noticing this situation on its own,
and just working around the filesystem problem.
This commit re-instates that hack, in order to avoid a regression for
the 2.6.20 release. This fixes bugzilla 7844:
http://bugzilla.kernel.org/show_bug.cgi?id=7844
Peter Zijlstra points out that we should probably retain the debugging
code that this removes from cancel_dirty_page(), and I agree, but for
the imminent release we might as well just silence the warning too
(since it's not a new bug: anything that triggers that warning has been
around forever).
Acked-by: Randy Dunlap <rdunlap@xenotime.net>
Acked-by: Jens Axboe <jens.axboe@oracle.com>
Acked-by: Peter Zijlstra <a.p.zijlstra@chello.nl>
Cc: Andrew Morton <akpm@osdl.org>
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
2007-01-27 04:47:06 +08:00
|
|
|
*/
|
2015-05-23 05:13:15 +08:00
|
|
|
if (ret)
|
2022-05-01 13:08:08 +08:00
|
|
|
folio_cancel_dirty(folio);
|
2023-11-18 05:58:23 +08:00
|
|
|
spin_unlock(&mapping->i_private_lock);
|
2005-04-17 06:20:36 +08:00
|
|
|
out:
|
|
|
|
if (buffers_to_free) {
|
|
|
|
struct buffer_head *bh = buffers_to_free;
|
|
|
|
|
|
|
|
do {
|
|
|
|
struct buffer_head *next = bh->b_this_page;
|
|
|
|
free_buffer_head(bh);
|
|
|
|
bh = next;
|
|
|
|
} while (bh != buffers_to_free);
|
|
|
|
}
|
|
|
|
return ret;
|
|
|
|
}
|
|
|
|
EXPORT_SYMBOL(try_to_free_buffers);
|
|
|
|
|
|
|
|
/*
|
|
|
|
* Buffer-head allocation
|
|
|
|
*/
|
2023-10-12 00:55:00 +08:00
|
|
|
static struct kmem_cache *bh_cachep __ro_after_init;
|
2005-04-17 06:20:36 +08:00
|
|
|
|
|
|
|
/*
|
|
|
|
* Once the number of bh's in the machine exceeds this level, we start
|
|
|
|
* stripping them in writeback.
|
|
|
|
*/
|
2023-10-12 00:55:00 +08:00
|
|
|
static unsigned long max_buffer_heads __ro_after_init;
|
2005-04-17 06:20:36 +08:00
|
|
|
|
|
|
|
int buffer_heads_over_limit;
|
|
|
|
|
|
|
|
struct bh_accounting {
|
|
|
|
int nr; /* Number of live bh's */
|
|
|
|
int ratelimit; /* Limit cacheline bouncing */
|
|
|
|
};
|
|
|
|
|
|
|
|
static DEFINE_PER_CPU(struct bh_accounting, bh_accounting) = {0, 0};
|
|
|
|
|
|
|
|
static void recalc_bh_state(void)
|
|
|
|
{
|
|
|
|
int i;
|
|
|
|
int tot = 0;
|
|
|
|
|
2010-12-07 01:40:05 +08:00
|
|
|
if (__this_cpu_inc_return(bh_accounting.ratelimit) - 1 < 4096)
|
2005-04-17 06:20:36 +08:00
|
|
|
return;
|
2010-12-07 01:16:28 +08:00
|
|
|
__this_cpu_write(bh_accounting.ratelimit, 0);
|
2006-03-24 19:18:10 +08:00
|
|
|
for_each_online_cpu(i)
|
2005-04-17 06:20:36 +08:00
|
|
|
tot += per_cpu(bh_accounting, i).nr;
|
|
|
|
buffer_heads_over_limit = (tot > max_buffer_heads);
|
|
|
|
}
|
2010-12-07 01:16:28 +08:00
|
|
|
|
2005-10-07 14:46:04 +08:00
|
|
|
struct buffer_head *alloc_buffer_head(gfp_t gfp_flags)
|
2005-04-17 06:20:36 +08:00
|
|
|
{
|
2010-03-11 07:20:33 +08:00
|
|
|
struct buffer_head *ret = kmem_cache_zalloc(bh_cachep, gfp_flags);
|
2005-04-17 06:20:36 +08:00
|
|
|
if (ret) {
|
2007-05-17 13:10:57 +08:00
|
|
|
INIT_LIST_HEAD(&ret->b_assoc_buffers);
|
2019-11-18 21:28:24 +08:00
|
|
|
spin_lock_init(&ret->b_uptodate_lock);
|
2010-12-07 01:16:28 +08:00
|
|
|
preempt_disable();
|
|
|
|
__this_cpu_inc(bh_accounting.nr);
|
2005-04-17 06:20:36 +08:00
|
|
|
recalc_bh_state();
|
2010-12-07 01:16:28 +08:00
|
|
|
preempt_enable();
|
2005-04-17 06:20:36 +08:00
|
|
|
}
|
|
|
|
return ret;
|
|
|
|
}
|
|
|
|
EXPORT_SYMBOL(alloc_buffer_head);
|
|
|
|
|
|
|
|
void free_buffer_head(struct buffer_head *bh)
|
|
|
|
{
|
|
|
|
BUG_ON(!list_empty(&bh->b_assoc_buffers));
|
|
|
|
kmem_cache_free(bh_cachep, bh);
|
2010-12-07 01:16:28 +08:00
|
|
|
preempt_disable();
|
|
|
|
__this_cpu_dec(bh_accounting.nr);
|
2005-04-17 06:20:36 +08:00
|
|
|
recalc_bh_state();
|
2010-12-07 01:16:28 +08:00
|
|
|
preempt_enable();
|
2005-04-17 06:20:36 +08:00
|
|
|
}
|
|
|
|
EXPORT_SYMBOL(free_buffer_head);
|
|
|
|
|
2016-11-03 22:49:57 +08:00
|
|
|
static int buffer_exit_cpu_dead(unsigned int cpu)
|
2005-04-17 06:20:36 +08:00
|
|
|
{
|
|
|
|
int i;
|
|
|
|
struct bh_lru *b = &per_cpu(bh_lrus, cpu);
|
|
|
|
|
|
|
|
for (i = 0; i < BH_LRU_SIZE; i++) {
|
|
|
|
brelse(b->bhs[i]);
|
|
|
|
b->bhs[i] = NULL;
|
|
|
|
}
|
2010-12-07 01:16:28 +08:00
|
|
|
this_cpu_add(bh_accounting.nr, per_cpu(bh_accounting, cpu).nr);
|
2006-03-24 19:18:10 +08:00
|
|
|
per_cpu(bh_accounting, cpu).nr = 0;
|
2016-11-03 22:49:57 +08:00
|
|
|
return 0;
|
2005-04-17 06:20:36 +08:00
|
|
|
}
|
|
|
|
|
2008-01-29 12:58:26 +08:00
|
|
|
/**
|
2008-03-20 08:01:00 +08:00
|
|
|
* bh_uptodate_or_lock - Test whether the buffer is uptodate
|
2008-01-29 12:58:26 +08:00
|
|
|
* @bh: struct buffer_head
|
|
|
|
*
|
|
|
|
* Return true if the buffer is up-to-date and false,
|
|
|
|
* with the buffer locked, if not.
|
|
|
|
*/
|
|
|
|
int bh_uptodate_or_lock(struct buffer_head *bh)
|
|
|
|
{
|
|
|
|
if (!buffer_uptodate(bh)) {
|
|
|
|
lock_buffer(bh);
|
|
|
|
if (!buffer_uptodate(bh))
|
|
|
|
return 0;
|
|
|
|
unlock_buffer(bh);
|
|
|
|
}
|
|
|
|
return 1;
|
|
|
|
}
|
|
|
|
EXPORT_SYMBOL(bh_uptodate_or_lock);
|
|
|
|
|
|
|
|
/**
|
2022-09-01 21:34:53 +08:00
|
|
|
* __bh_read - Submit read for a locked buffer
|
2008-01-29 12:58:26 +08:00
|
|
|
* @bh: struct buffer_head
|
2022-09-01 21:34:53 +08:00
|
|
|
* @op_flags: appending REQ_OP_* flags besides REQ_OP_READ
|
|
|
|
* @wait: wait until reading finish
|
2008-01-29 12:58:26 +08:00
|
|
|
*
|
2022-09-01 21:34:53 +08:00
|
|
|
* Returns zero on success or don't wait, and -EIO on error.
|
2008-01-29 12:58:26 +08:00
|
|
|
*/
|
2022-09-01 21:34:53 +08:00
|
|
|
int __bh_read(struct buffer_head *bh, blk_opf_t op_flags, bool wait)
|
2008-01-29 12:58:26 +08:00
|
|
|
{
|
2022-09-01 21:34:53 +08:00
|
|
|
int ret = 0;
|
2008-01-29 12:58:26 +08:00
|
|
|
|
2022-09-01 21:34:53 +08:00
|
|
|
BUG_ON(!buffer_locked(bh));
|
2008-01-29 12:58:26 +08:00
|
|
|
|
|
|
|
get_bh(bh);
|
|
|
|
bh->b_end_io = end_buffer_read_sync;
|
2022-09-01 21:34:53 +08:00
|
|
|
submit_bh(REQ_OP_READ | op_flags, bh);
|
|
|
|
if (wait) {
|
|
|
|
wait_on_buffer(bh);
|
|
|
|
if (!buffer_uptodate(bh))
|
|
|
|
ret = -EIO;
|
|
|
|
}
|
|
|
|
return ret;
|
|
|
|
}
|
|
|
|
EXPORT_SYMBOL(__bh_read);
|
|
|
|
|
|
|
|
/**
|
|
|
|
* __bh_read_batch - Submit read for a batch of unlocked buffers
|
|
|
|
* @nr: entry number of the buffer batch
|
|
|
|
* @bhs: a batch of struct buffer_head
|
|
|
|
* @op_flags: appending REQ_OP_* flags besides REQ_OP_READ
|
|
|
|
* @force_lock: force to get a lock on the buffer if set, otherwise drops any
|
|
|
|
* buffer that cannot lock.
|
|
|
|
*
|
|
|
|
* Returns zero on success or don't wait, and -EIO on error.
|
|
|
|
*/
|
|
|
|
void __bh_read_batch(int nr, struct buffer_head *bhs[],
|
|
|
|
blk_opf_t op_flags, bool force_lock)
|
|
|
|
{
|
|
|
|
int i;
|
|
|
|
|
|
|
|
for (i = 0; i < nr; i++) {
|
|
|
|
struct buffer_head *bh = bhs[i];
|
|
|
|
|
|
|
|
if (buffer_uptodate(bh))
|
|
|
|
continue;
|
|
|
|
|
|
|
|
if (force_lock)
|
|
|
|
lock_buffer(bh);
|
|
|
|
else
|
|
|
|
if (!trylock_buffer(bh))
|
|
|
|
continue;
|
|
|
|
|
|
|
|
if (buffer_uptodate(bh)) {
|
|
|
|
unlock_buffer(bh);
|
|
|
|
continue;
|
|
|
|
}
|
|
|
|
|
|
|
|
bh->b_end_io = end_buffer_read_sync;
|
|
|
|
get_bh(bh);
|
|
|
|
submit_bh(REQ_OP_READ | op_flags, bh);
|
|
|
|
}
|
2008-01-29 12:58:26 +08:00
|
|
|
}
|
2022-09-01 21:34:53 +08:00
|
|
|
EXPORT_SYMBOL(__bh_read_batch);
|
2008-01-29 12:58:26 +08:00
|
|
|
|
2005-04-17 06:20:36 +08:00
|
|
|
void __init buffer_init(void)
|
|
|
|
{
|
2013-02-23 08:35:46 +08:00
|
|
|
unsigned long nrpages;
|
2016-11-03 22:49:57 +08:00
|
|
|
int ret;
|
2005-04-17 06:20:36 +08:00
|
|
|
|
2024-01-16 17:11:37 +08:00
|
|
|
bh_cachep = KMEM_CACHE(buffer_head,
|
2024-02-24 21:53:15 +08:00
|
|
|
SLAB_RECLAIM_ACCOUNT|SLAB_PANIC);
|
2005-04-17 06:20:36 +08:00
|
|
|
/*
|
|
|
|
* Limit the bh occupancy to 10% of ZONE_NORMAL
|
|
|
|
*/
|
|
|
|
nrpages = (nr_free_buffer_pages() * 10) / 100;
|
|
|
|
max_buffer_heads = nrpages * (PAGE_SIZE / sizeof(struct buffer_head));
|
2016-11-03 22:49:57 +08:00
|
|
|
ret = cpuhp_setup_state_nocalls(CPUHP_FS_BUFF_DEAD, "fs/buffer:dead",
|
|
|
|
NULL, buffer_exit_cpu_dead);
|
|
|
|
WARN_ON(ret < 0);
|
2005-04-17 06:20:36 +08:00
|
|
|
}
|