mirror of
https://mirrors.bfsu.edu.cn/git/linux.git
synced 2024-12-05 10:04:12 +08:00
70d9329857
The current notifiers have the following error handling pattern all over the place: int err, nr; err = __foo_notifier_call_chain(&chain, val_up, v, -1, &nr); if (err & NOTIFIER_STOP_MASK) __foo_notifier_call_chain(&chain, val_down, v, nr-1, NULL) And aside from the endless repetition thereof, it is broken. Consider blocking notifiers; both calls take and drop the rwsem, this means that the notifier list can change in between the two calls, making @nr meaningless. Fix this by replacing all the __foo_notifier_call_chain() functions with foo_notifier_call_chain_robust() that embeds the above pattern, but ensures it is inside a single lock region. Note: I switched atomic_notifier_call_chain_robust() to use the spinlock, since RCU cannot provide the guarantee required for the recovery. Note: software_resume() error handling was broken afaict. Signed-off-by: Peter Zijlstra (Intel) <peterz@infradead.org> Signed-off-by: Ingo Molnar <mingo@kernel.org> Acked-by: Rafael J. Wysocki <rafael.j.wysocki@intel.com> Link: https://lore.kernel.org/r/20200818135804.325626653@infradead.org
563 lines
16 KiB
C
563 lines
16 KiB
C
// SPDX-License-Identifier: GPL-2.0-only
|
|
#include <linux/kdebug.h>
|
|
#include <linux/kprobes.h>
|
|
#include <linux/export.h>
|
|
#include <linux/notifier.h>
|
|
#include <linux/rcupdate.h>
|
|
#include <linux/vmalloc.h>
|
|
#include <linux/reboot.h>
|
|
|
|
/*
|
|
* Notifier list for kernel code which wants to be called
|
|
* at shutdown. This is used to stop any idling DMA operations
|
|
* and the like.
|
|
*/
|
|
BLOCKING_NOTIFIER_HEAD(reboot_notifier_list);
|
|
|
|
/*
|
|
* Notifier chain core routines. The exported routines below
|
|
* are layered on top of these, with appropriate locking added.
|
|
*/
|
|
|
|
static int notifier_chain_register(struct notifier_block **nl,
|
|
struct notifier_block *n)
|
|
{
|
|
while ((*nl) != NULL) {
|
|
if (unlikely((*nl) == n)) {
|
|
WARN(1, "double register detected");
|
|
return 0;
|
|
}
|
|
if (n->priority > (*nl)->priority)
|
|
break;
|
|
nl = &((*nl)->next);
|
|
}
|
|
n->next = *nl;
|
|
rcu_assign_pointer(*nl, n);
|
|
return 0;
|
|
}
|
|
|
|
static int notifier_chain_unregister(struct notifier_block **nl,
|
|
struct notifier_block *n)
|
|
{
|
|
while ((*nl) != NULL) {
|
|
if ((*nl) == n) {
|
|
rcu_assign_pointer(*nl, n->next);
|
|
return 0;
|
|
}
|
|
nl = &((*nl)->next);
|
|
}
|
|
return -ENOENT;
|
|
}
|
|
|
|
/**
|
|
* notifier_call_chain - Informs the registered notifiers about an event.
|
|
* @nl: Pointer to head of the blocking notifier chain
|
|
* @val: Value passed unmodified to notifier function
|
|
* @v: Pointer passed unmodified to notifier function
|
|
* @nr_to_call: Number of notifier functions to be called. Don't care
|
|
* value of this parameter is -1.
|
|
* @nr_calls: Records the number of notifications sent. Don't care
|
|
* value of this field is NULL.
|
|
* @returns: notifier_call_chain returns the value returned by the
|
|
* last notifier function called.
|
|
*/
|
|
static int notifier_call_chain(struct notifier_block **nl,
|
|
unsigned long val, void *v,
|
|
int nr_to_call, int *nr_calls)
|
|
{
|
|
int ret = NOTIFY_DONE;
|
|
struct notifier_block *nb, *next_nb;
|
|
|
|
nb = rcu_dereference_raw(*nl);
|
|
|
|
while (nb && nr_to_call) {
|
|
next_nb = rcu_dereference_raw(nb->next);
|
|
|
|
#ifdef CONFIG_DEBUG_NOTIFIERS
|
|
if (unlikely(!func_ptr_is_kernel_text(nb->notifier_call))) {
|
|
WARN(1, "Invalid notifier called!");
|
|
nb = next_nb;
|
|
continue;
|
|
}
|
|
#endif
|
|
ret = nb->notifier_call(nb, val, v);
|
|
|
|
if (nr_calls)
|
|
(*nr_calls)++;
|
|
|
|
if (ret & NOTIFY_STOP_MASK)
|
|
break;
|
|
nb = next_nb;
|
|
nr_to_call--;
|
|
}
|
|
return ret;
|
|
}
|
|
NOKPROBE_SYMBOL(notifier_call_chain);
|
|
|
|
/**
|
|
* notifier_call_chain_robust - Inform the registered notifiers about an event
|
|
* and rollback on error.
|
|
* @nl: Pointer to head of the blocking notifier chain
|
|
* @val_up: Value passed unmodified to the notifier function
|
|
* @val_down: Value passed unmodified to the notifier function when recovering
|
|
* from an error on @val_up
|
|
* @v Pointer passed unmodified to the notifier function
|
|
*
|
|
* NOTE: It is important the @nl chain doesn't change between the two
|
|
* invocations of notifier_call_chain() such that we visit the
|
|
* exact same notifier callbacks; this rules out any RCU usage.
|
|
*
|
|
* Returns: the return value of the @val_up call.
|
|
*/
|
|
static int notifier_call_chain_robust(struct notifier_block **nl,
|
|
unsigned long val_up, unsigned long val_down,
|
|
void *v)
|
|
{
|
|
int ret, nr = 0;
|
|
|
|
ret = notifier_call_chain(nl, val_up, v, -1, &nr);
|
|
if (ret & NOTIFY_STOP_MASK)
|
|
notifier_call_chain(nl, val_down, v, nr-1, NULL);
|
|
|
|
return ret;
|
|
}
|
|
|
|
/*
|
|
* Atomic notifier chain routines. Registration and unregistration
|
|
* use a spinlock, and call_chain is synchronized by RCU (no locks).
|
|
*/
|
|
|
|
/**
|
|
* atomic_notifier_chain_register - Add notifier to an atomic notifier chain
|
|
* @nh: Pointer to head of the atomic notifier chain
|
|
* @n: New entry in notifier chain
|
|
*
|
|
* Adds a notifier to an atomic notifier chain.
|
|
*
|
|
* Currently always returns zero.
|
|
*/
|
|
int atomic_notifier_chain_register(struct atomic_notifier_head *nh,
|
|
struct notifier_block *n)
|
|
{
|
|
unsigned long flags;
|
|
int ret;
|
|
|
|
spin_lock_irqsave(&nh->lock, flags);
|
|
ret = notifier_chain_register(&nh->head, n);
|
|
spin_unlock_irqrestore(&nh->lock, flags);
|
|
return ret;
|
|
}
|
|
EXPORT_SYMBOL_GPL(atomic_notifier_chain_register);
|
|
|
|
/**
|
|
* atomic_notifier_chain_unregister - Remove notifier from an atomic notifier chain
|
|
* @nh: Pointer to head of the atomic notifier chain
|
|
* @n: Entry to remove from notifier chain
|
|
*
|
|
* Removes a notifier from an atomic notifier chain.
|
|
*
|
|
* Returns zero on success or %-ENOENT on failure.
|
|
*/
|
|
int atomic_notifier_chain_unregister(struct atomic_notifier_head *nh,
|
|
struct notifier_block *n)
|
|
{
|
|
unsigned long flags;
|
|
int ret;
|
|
|
|
spin_lock_irqsave(&nh->lock, flags);
|
|
ret = notifier_chain_unregister(&nh->head, n);
|
|
spin_unlock_irqrestore(&nh->lock, flags);
|
|
synchronize_rcu();
|
|
return ret;
|
|
}
|
|
EXPORT_SYMBOL_GPL(atomic_notifier_chain_unregister);
|
|
|
|
int atomic_notifier_call_chain_robust(struct atomic_notifier_head *nh,
|
|
unsigned long val_up, unsigned long val_down, void *v)
|
|
{
|
|
unsigned long flags;
|
|
int ret;
|
|
|
|
/*
|
|
* Musn't use RCU; because then the notifier list can
|
|
* change between the up and down traversal.
|
|
*/
|
|
spin_lock_irqsave(&nh->lock, flags);
|
|
ret = notifier_call_chain_robust(&nh->head, val_up, val_down, v);
|
|
spin_unlock_irqrestore(&nh->lock, flags);
|
|
|
|
return ret;
|
|
}
|
|
EXPORT_SYMBOL_GPL(atomic_notifier_call_chain_robust);
|
|
NOKPROBE_SYMBOL(atomic_notifier_call_chain_robust);
|
|
|
|
/**
|
|
* atomic_notifier_call_chain - Call functions in an atomic notifier chain
|
|
* @nh: Pointer to head of the atomic notifier chain
|
|
* @val: Value passed unmodified to notifier function
|
|
* @v: Pointer passed unmodified to notifier function
|
|
*
|
|
* Calls each function in a notifier chain in turn. The functions
|
|
* run in an atomic context, so they must not block.
|
|
* This routine uses RCU to synchronize with changes to the chain.
|
|
*
|
|
* If the return value of the notifier can be and'ed
|
|
* with %NOTIFY_STOP_MASK then atomic_notifier_call_chain()
|
|
* will return immediately, with the return value of
|
|
* the notifier function which halted execution.
|
|
* Otherwise the return value is the return value
|
|
* of the last notifier function called.
|
|
*/
|
|
int atomic_notifier_call_chain(struct atomic_notifier_head *nh,
|
|
unsigned long val, void *v)
|
|
{
|
|
int ret;
|
|
|
|
rcu_read_lock();
|
|
ret = notifier_call_chain(&nh->head, val, v, -1, NULL);
|
|
rcu_read_unlock();
|
|
|
|
return ret;
|
|
}
|
|
EXPORT_SYMBOL_GPL(atomic_notifier_call_chain);
|
|
NOKPROBE_SYMBOL(atomic_notifier_call_chain);
|
|
|
|
/*
|
|
* Blocking notifier chain routines. All access to the chain is
|
|
* synchronized by an rwsem.
|
|
*/
|
|
|
|
/**
|
|
* blocking_notifier_chain_register - Add notifier to a blocking notifier chain
|
|
* @nh: Pointer to head of the blocking notifier chain
|
|
* @n: New entry in notifier chain
|
|
*
|
|
* Adds a notifier to a blocking notifier chain.
|
|
* Must be called in process context.
|
|
*
|
|
* Currently always returns zero.
|
|
*/
|
|
int blocking_notifier_chain_register(struct blocking_notifier_head *nh,
|
|
struct notifier_block *n)
|
|
{
|
|
int ret;
|
|
|
|
/*
|
|
* This code gets used during boot-up, when task switching is
|
|
* not yet working and interrupts must remain disabled. At
|
|
* such times we must not call down_write().
|
|
*/
|
|
if (unlikely(system_state == SYSTEM_BOOTING))
|
|
return notifier_chain_register(&nh->head, n);
|
|
|
|
down_write(&nh->rwsem);
|
|
ret = notifier_chain_register(&nh->head, n);
|
|
up_write(&nh->rwsem);
|
|
return ret;
|
|
}
|
|
EXPORT_SYMBOL_GPL(blocking_notifier_chain_register);
|
|
|
|
/**
|
|
* blocking_notifier_chain_unregister - Remove notifier from a blocking notifier chain
|
|
* @nh: Pointer to head of the blocking notifier chain
|
|
* @n: Entry to remove from notifier chain
|
|
*
|
|
* Removes a notifier from a blocking notifier chain.
|
|
* Must be called from process context.
|
|
*
|
|
* Returns zero on success or %-ENOENT on failure.
|
|
*/
|
|
int blocking_notifier_chain_unregister(struct blocking_notifier_head *nh,
|
|
struct notifier_block *n)
|
|
{
|
|
int ret;
|
|
|
|
/*
|
|
* This code gets used during boot-up, when task switching is
|
|
* not yet working and interrupts must remain disabled. At
|
|
* such times we must not call down_write().
|
|
*/
|
|
if (unlikely(system_state == SYSTEM_BOOTING))
|
|
return notifier_chain_unregister(&nh->head, n);
|
|
|
|
down_write(&nh->rwsem);
|
|
ret = notifier_chain_unregister(&nh->head, n);
|
|
up_write(&nh->rwsem);
|
|
return ret;
|
|
}
|
|
EXPORT_SYMBOL_GPL(blocking_notifier_chain_unregister);
|
|
|
|
int blocking_notifier_call_chain_robust(struct blocking_notifier_head *nh,
|
|
unsigned long val_up, unsigned long val_down, void *v)
|
|
{
|
|
int ret = NOTIFY_DONE;
|
|
|
|
/*
|
|
* We check the head outside the lock, but if this access is
|
|
* racy then it does not matter what the result of the test
|
|
* is, we re-check the list after having taken the lock anyway:
|
|
*/
|
|
if (rcu_access_pointer(nh->head)) {
|
|
down_read(&nh->rwsem);
|
|
ret = notifier_call_chain_robust(&nh->head, val_up, val_down, v);
|
|
up_read(&nh->rwsem);
|
|
}
|
|
return ret;
|
|
}
|
|
EXPORT_SYMBOL_GPL(blocking_notifier_call_chain_robust);
|
|
|
|
/**
|
|
* blocking_notifier_call_chain - Call functions in a blocking notifier chain
|
|
* @nh: Pointer to head of the blocking notifier chain
|
|
* @val: Value passed unmodified to notifier function
|
|
* @v: Pointer passed unmodified to notifier function
|
|
*
|
|
* Calls each function in a notifier chain in turn. The functions
|
|
* run in a process context, so they are allowed to block.
|
|
*
|
|
* If the return value of the notifier can be and'ed
|
|
* with %NOTIFY_STOP_MASK then blocking_notifier_call_chain()
|
|
* will return immediately, with the return value of
|
|
* the notifier function which halted execution.
|
|
* Otherwise the return value is the return value
|
|
* of the last notifier function called.
|
|
*/
|
|
int blocking_notifier_call_chain(struct blocking_notifier_head *nh,
|
|
unsigned long val, void *v)
|
|
{
|
|
int ret = NOTIFY_DONE;
|
|
|
|
/*
|
|
* We check the head outside the lock, but if this access is
|
|
* racy then it does not matter what the result of the test
|
|
* is, we re-check the list after having taken the lock anyway:
|
|
*/
|
|
if (rcu_access_pointer(nh->head)) {
|
|
down_read(&nh->rwsem);
|
|
ret = notifier_call_chain(&nh->head, val, v, -1, NULL);
|
|
up_read(&nh->rwsem);
|
|
}
|
|
return ret;
|
|
}
|
|
EXPORT_SYMBOL_GPL(blocking_notifier_call_chain);
|
|
|
|
/*
|
|
* Raw notifier chain routines. There is no protection;
|
|
* the caller must provide it. Use at your own risk!
|
|
*/
|
|
|
|
/**
|
|
* raw_notifier_chain_register - Add notifier to a raw notifier chain
|
|
* @nh: Pointer to head of the raw notifier chain
|
|
* @n: New entry in notifier chain
|
|
*
|
|
* Adds a notifier to a raw notifier chain.
|
|
* All locking must be provided by the caller.
|
|
*
|
|
* Currently always returns zero.
|
|
*/
|
|
int raw_notifier_chain_register(struct raw_notifier_head *nh,
|
|
struct notifier_block *n)
|
|
{
|
|
return notifier_chain_register(&nh->head, n);
|
|
}
|
|
EXPORT_SYMBOL_GPL(raw_notifier_chain_register);
|
|
|
|
/**
|
|
* raw_notifier_chain_unregister - Remove notifier from a raw notifier chain
|
|
* @nh: Pointer to head of the raw notifier chain
|
|
* @n: Entry to remove from notifier chain
|
|
*
|
|
* Removes a notifier from a raw notifier chain.
|
|
* All locking must be provided by the caller.
|
|
*
|
|
* Returns zero on success or %-ENOENT on failure.
|
|
*/
|
|
int raw_notifier_chain_unregister(struct raw_notifier_head *nh,
|
|
struct notifier_block *n)
|
|
{
|
|
return notifier_chain_unregister(&nh->head, n);
|
|
}
|
|
EXPORT_SYMBOL_GPL(raw_notifier_chain_unregister);
|
|
|
|
int raw_notifier_call_chain_robust(struct raw_notifier_head *nh,
|
|
unsigned long val_up, unsigned long val_down, void *v)
|
|
{
|
|
return notifier_call_chain_robust(&nh->head, val_up, val_down, v);
|
|
}
|
|
EXPORT_SYMBOL_GPL(raw_notifier_call_chain_robust);
|
|
|
|
/**
|
|
* raw_notifier_call_chain - Call functions in a raw notifier chain
|
|
* @nh: Pointer to head of the raw notifier chain
|
|
* @val: Value passed unmodified to notifier function
|
|
* @v: Pointer passed unmodified to notifier function
|
|
*
|
|
* Calls each function in a notifier chain in turn. The functions
|
|
* run in an undefined context.
|
|
* All locking must be provided by the caller.
|
|
*
|
|
* If the return value of the notifier can be and'ed
|
|
* with %NOTIFY_STOP_MASK then raw_notifier_call_chain()
|
|
* will return immediately, with the return value of
|
|
* the notifier function which halted execution.
|
|
* Otherwise the return value is the return value
|
|
* of the last notifier function called.
|
|
*/
|
|
int raw_notifier_call_chain(struct raw_notifier_head *nh,
|
|
unsigned long val, void *v)
|
|
{
|
|
return notifier_call_chain(&nh->head, val, v, -1, NULL);
|
|
}
|
|
EXPORT_SYMBOL_GPL(raw_notifier_call_chain);
|
|
|
|
#ifdef CONFIG_SRCU
|
|
/*
|
|
* SRCU notifier chain routines. Registration and unregistration
|
|
* use a mutex, and call_chain is synchronized by SRCU (no locks).
|
|
*/
|
|
|
|
/**
|
|
* srcu_notifier_chain_register - Add notifier to an SRCU notifier chain
|
|
* @nh: Pointer to head of the SRCU notifier chain
|
|
* @n: New entry in notifier chain
|
|
*
|
|
* Adds a notifier to an SRCU notifier chain.
|
|
* Must be called in process context.
|
|
*
|
|
* Currently always returns zero.
|
|
*/
|
|
int srcu_notifier_chain_register(struct srcu_notifier_head *nh,
|
|
struct notifier_block *n)
|
|
{
|
|
int ret;
|
|
|
|
/*
|
|
* This code gets used during boot-up, when task switching is
|
|
* not yet working and interrupts must remain disabled. At
|
|
* such times we must not call mutex_lock().
|
|
*/
|
|
if (unlikely(system_state == SYSTEM_BOOTING))
|
|
return notifier_chain_register(&nh->head, n);
|
|
|
|
mutex_lock(&nh->mutex);
|
|
ret = notifier_chain_register(&nh->head, n);
|
|
mutex_unlock(&nh->mutex);
|
|
return ret;
|
|
}
|
|
EXPORT_SYMBOL_GPL(srcu_notifier_chain_register);
|
|
|
|
/**
|
|
* srcu_notifier_chain_unregister - Remove notifier from an SRCU notifier chain
|
|
* @nh: Pointer to head of the SRCU notifier chain
|
|
* @n: Entry to remove from notifier chain
|
|
*
|
|
* Removes a notifier from an SRCU notifier chain.
|
|
* Must be called from process context.
|
|
*
|
|
* Returns zero on success or %-ENOENT on failure.
|
|
*/
|
|
int srcu_notifier_chain_unregister(struct srcu_notifier_head *nh,
|
|
struct notifier_block *n)
|
|
{
|
|
int ret;
|
|
|
|
/*
|
|
* This code gets used during boot-up, when task switching is
|
|
* not yet working and interrupts must remain disabled. At
|
|
* such times we must not call mutex_lock().
|
|
*/
|
|
if (unlikely(system_state == SYSTEM_BOOTING))
|
|
return notifier_chain_unregister(&nh->head, n);
|
|
|
|
mutex_lock(&nh->mutex);
|
|
ret = notifier_chain_unregister(&nh->head, n);
|
|
mutex_unlock(&nh->mutex);
|
|
synchronize_srcu(&nh->srcu);
|
|
return ret;
|
|
}
|
|
EXPORT_SYMBOL_GPL(srcu_notifier_chain_unregister);
|
|
|
|
/**
|
|
* srcu_notifier_call_chain - Call functions in an SRCU notifier chain
|
|
* @nh: Pointer to head of the SRCU notifier chain
|
|
* @val: Value passed unmodified to notifier function
|
|
* @v: Pointer passed unmodified to notifier function
|
|
*
|
|
* Calls each function in a notifier chain in turn. The functions
|
|
* run in a process context, so they are allowed to block.
|
|
*
|
|
* If the return value of the notifier can be and'ed
|
|
* with %NOTIFY_STOP_MASK then srcu_notifier_call_chain()
|
|
* will return immediately, with the return value of
|
|
* the notifier function which halted execution.
|
|
* Otherwise the return value is the return value
|
|
* of the last notifier function called.
|
|
*/
|
|
int srcu_notifier_call_chain(struct srcu_notifier_head *nh,
|
|
unsigned long val, void *v)
|
|
{
|
|
int ret;
|
|
int idx;
|
|
|
|
idx = srcu_read_lock(&nh->srcu);
|
|
ret = notifier_call_chain(&nh->head, val, v, -1, NULL);
|
|
srcu_read_unlock(&nh->srcu, idx);
|
|
return ret;
|
|
}
|
|
EXPORT_SYMBOL_GPL(srcu_notifier_call_chain);
|
|
|
|
/**
|
|
* srcu_init_notifier_head - Initialize an SRCU notifier head
|
|
* @nh: Pointer to head of the srcu notifier chain
|
|
*
|
|
* Unlike other sorts of notifier heads, SRCU notifier heads require
|
|
* dynamic initialization. Be sure to call this routine before
|
|
* calling any of the other SRCU notifier routines for this head.
|
|
*
|
|
* If an SRCU notifier head is deallocated, it must first be cleaned
|
|
* up by calling srcu_cleanup_notifier_head(). Otherwise the head's
|
|
* per-cpu data (used by the SRCU mechanism) will leak.
|
|
*/
|
|
void srcu_init_notifier_head(struct srcu_notifier_head *nh)
|
|
{
|
|
mutex_init(&nh->mutex);
|
|
if (init_srcu_struct(&nh->srcu) < 0)
|
|
BUG();
|
|
nh->head = NULL;
|
|
}
|
|
EXPORT_SYMBOL_GPL(srcu_init_notifier_head);
|
|
|
|
#endif /* CONFIG_SRCU */
|
|
|
|
static ATOMIC_NOTIFIER_HEAD(die_chain);
|
|
|
|
int notrace notify_die(enum die_val val, const char *str,
|
|
struct pt_regs *regs, long err, int trap, int sig)
|
|
{
|
|
struct die_args args = {
|
|
.regs = regs,
|
|
.str = str,
|
|
.err = err,
|
|
.trapnr = trap,
|
|
.signr = sig,
|
|
|
|
};
|
|
RCU_LOCKDEP_WARN(!rcu_is_watching(),
|
|
"notify_die called but RCU thinks we're quiescent");
|
|
return atomic_notifier_call_chain(&die_chain, val, &args);
|
|
}
|
|
NOKPROBE_SYMBOL(notify_die);
|
|
|
|
int register_die_notifier(struct notifier_block *nb)
|
|
{
|
|
return atomic_notifier_chain_register(&die_chain, nb);
|
|
}
|
|
EXPORT_SYMBOL_GPL(register_die_notifier);
|
|
|
|
int unregister_die_notifier(struct notifier_block *nb)
|
|
{
|
|
return atomic_notifier_chain_unregister(&die_chain, nb);
|
|
}
|
|
EXPORT_SYMBOL_GPL(unregister_die_notifier);
|