2005-04-17 06:20:36 +08:00
|
|
|
/*
|
2009-09-19 01:28:19 +08:00
|
|
|
* Read-Copy Update mechanism for mutual exclusion
|
2005-04-17 06:20:36 +08:00
|
|
|
*
|
|
|
|
* This program is free software; you can redistribute it and/or modify
|
|
|
|
* it under the terms of the GNU General Public License as published by
|
|
|
|
* the Free Software Foundation; either version 2 of the License, or
|
|
|
|
* (at your option) any later version.
|
|
|
|
*
|
|
|
|
* This program is distributed in the hope that it will be useful,
|
|
|
|
* but WITHOUT ANY WARRANTY; without even the implied warranty of
|
|
|
|
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
|
|
|
* GNU General Public License for more details.
|
|
|
|
*
|
|
|
|
* You should have received a copy of the GNU General Public License
|
|
|
|
* along with this program; if not, write to the Free Software
|
|
|
|
* Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
|
|
|
|
*
|
2008-01-26 04:08:24 +08:00
|
|
|
* Copyright IBM Corporation, 2001
|
2005-04-17 06:20:36 +08:00
|
|
|
*
|
|
|
|
* Author: Dipankar Sarma <dipankar@in.ibm.com>
|
2009-09-19 01:28:19 +08:00
|
|
|
*
|
2006-10-04 17:17:21 +08:00
|
|
|
* Based on the original work by Paul McKenney <paulmck@us.ibm.com>
|
2005-04-17 06:20:36 +08:00
|
|
|
* and inputs from Rusty Russell, Andrea Arcangeli and Andi Kleen.
|
|
|
|
* Papers:
|
|
|
|
* http://www.rdrop.com/users/paulmck/paper/rclockpdcsproof.pdf
|
|
|
|
* http://lse.sourceforge.net/locking/rclock_OLS.2001.05.01c.sc.pdf (OLS2001)
|
|
|
|
*
|
|
|
|
* For detailed explanation of Read-Copy Update mechanism see -
|
2009-09-19 01:28:19 +08:00
|
|
|
* http://lse.sourceforge.net/locking/rcupdate.html
|
2005-04-17 06:20:36 +08:00
|
|
|
*
|
|
|
|
*/
|
|
|
|
|
|
|
|
#ifndef __LINUX_RCUPDATE_H
|
|
|
|
#define __LINUX_RCUPDATE_H
|
|
|
|
|
|
|
|
#include <linux/cache.h>
|
|
|
|
#include <linux/spinlock.h>
|
|
|
|
#include <linux/threads.h>
|
|
|
|
#include <linux/cpumask.h>
|
|
|
|
#include <linux/seqlock.h>
|
2007-10-12 04:11:12 +08:00
|
|
|
#include <linux/lockdep.h>
|
rcu: add call_rcu_sched()
Fourth cut of patch to provide the call_rcu_sched(). This is again to
synchronize_sched() as call_rcu() is to synchronize_rcu().
Should be fine for experimental and -rt use, but not ready for inclusion.
With some luck, I will be able to tell Andrew to come out of hiding on
the next round.
Passes multi-day rcutorture sessions with concurrent CPU hotplugging.
Fixes since the first version include a bug that could result in
indefinite blocking (spotted by Gautham Shenoy), better resiliency
against CPU-hotplug operations, and other minor fixes.
Fixes since the second version include reworking grace-period detection
to avoid deadlocks that could happen when running concurrently with
CPU hotplug, adding Mathieu's fix to avoid the softlockup messages,
as well as Mathieu's fix to allow use earlier in boot.
Fixes since the third version include a wrong-CPU bug spotted by
Andrew, getting rid of the obsolete synchronize_kernel API that somehow
snuck back in, merging spin_unlock() and local_irq_restore() in a
few places, commenting the code that checks for quiescent states based
on interrupting from user-mode execution or the idle loop, removing
some inline attributes, and some code-style changes.
Known/suspected shortcomings:
o I still do not entirely trust the sleep/wakeup logic. Next step
will be to use a private snapshot of the CPU online mask in
rcu_sched_grace_period() -- if the CPU wasn't there at the start
of the grace period, we don't need to hear from it. And the
bit about accounting for changes in online CPUs inside of
rcu_sched_grace_period() is ugly anyway.
o It might be good for rcu_sched_grace_period() to invoke
resched_cpu() when a given CPU wasn't responding quickly,
but resched_cpu() is declared static...
This patch also fixes a long-standing bug in the earlier preemptable-RCU
implementation of synchronize_rcu() that could result in loss of
concurrent external changes to a task's CPU affinity mask. I still cannot
remember who reported this...
Signed-off-by: Paul E. McKenney <paulmck@linux.vnet.ibm.com>
Signed-off-by: Mathieu Desnoyers <mathieu.desnoyers@polymtl.ca>
Signed-off-by: Ingo Molnar <mingo@elte.hu>
Signed-off-by: Thomas Gleixner <tglx@linutronix.de>
2008-05-13 03:21:05 +08:00
|
|
|
#include <linux/completion.h>
|
2010-04-17 20:48:42 +08:00
|
|
|
#include <linux/debugobjects.h>
|
2010-04-29 05:39:09 +08:00
|
|
|
#include <linux/compiler.h>
|
2005-04-17 06:20:36 +08:00
|
|
|
|
2010-03-11 07:24:05 +08:00
|
|
|
#ifdef CONFIG_RCU_TORTURE_TEST
|
|
|
|
extern int rcutorture_runnable; /* for sysctl */
|
|
|
|
#endif /* #ifdef CONFIG_RCU_TORTURE_TEST */
|
|
|
|
|
2005-04-17 06:20:36 +08:00
|
|
|
/**
|
|
|
|
* struct rcu_head - callback structure for use with RCU
|
|
|
|
* @next: next update requests in a list
|
|
|
|
* @func: actual update function to call after the grace period.
|
|
|
|
*/
|
|
|
|
struct rcu_head {
|
|
|
|
struct rcu_head *next;
|
|
|
|
void (*func)(struct rcu_head *head);
|
|
|
|
};
|
|
|
|
|
2009-06-26 00:08:16 +08:00
|
|
|
/* Exported common interfaces */
|
|
|
|
extern void rcu_barrier_bh(void);
|
|
|
|
extern void rcu_barrier_sched(void);
|
|
|
|
extern void synchronize_sched_expedited(void);
|
|
|
|
extern int sched_expedited_torture_stats(char *page);
|
|
|
|
|
|
|
|
/* Internal to kernel */
|
|
|
|
extern void rcu_init(void);
|
rcu: Teach RCU that idle task is not quiscent state at boot
This patch fixes a bug located by Vegard Nossum with the aid of
kmemcheck, updated based on review comments from Nick Piggin,
Ingo Molnar, and Andrew Morton. And cleans up the variable-name
and function-name language. ;-)
The boot CPU runs in the context of its idle thread during boot-up.
During this time, idle_cpu(0) will always return nonzero, which will
fool Classic and Hierarchical RCU into deciding that a large chunk of
the boot-up sequence is a big long quiescent state. This in turn causes
RCU to prematurely end grace periods during this time.
This patch changes the rcutree.c and rcuclassic.c rcu_check_callbacks()
function to ignore the idle task as a quiescent state until the
system has started up the scheduler in rest_init(), introducing a
new non-API function rcu_idle_now_means_idle() to inform RCU of this
transition. RCU maintains an internal rcu_idle_cpu_truthful variable
to track this state, which is then used by rcu_check_callback() to
determine if it should believe idle_cpu().
Because this patch has the effect of disallowing RCU grace periods
during long stretches of the boot-up sequence, this patch also introduces
Josh Triplett's UP-only optimization that makes synchronize_rcu() be a
no-op if num_online_cpus() returns 1. This allows boot-time code that
calls synchronize_rcu() to proceed normally. Note, however, that RCU
callbacks registered by call_rcu() will likely queue up until later in
the boot sequence. Although rcuclassic and rcutree can also use this
same optimization after boot completes, rcupreempt must restrict its
use of this optimization to the portion of the boot sequence before the
scheduler starts up, given that an rcupreempt RCU read-side critical
section may be preeempted.
In addition, this patch takes Nick Piggin's suggestion to make the
system_state global variable be __read_mostly.
Changes since v4:
o Changes the name of the introduced function and variable to
be less emotional. ;-)
Changes since v3:
o WARN_ON(nr_context_switches() > 0) to verify that RCU
switches out of boot-time mode before the first context
switch, as suggested by Nick Piggin.
Changes since v2:
o Created rcu_blocking_is_gp() internal-to-RCU API that
determines whether a call to synchronize_rcu() is itself
a grace period.
o The definition of rcu_blocking_is_gp() for rcuclassic and
rcutree checks to see if but a single CPU is online.
o The definition of rcu_blocking_is_gp() for rcupreempt
checks to see both if but a single CPU is online and if
the system is still in early boot.
This allows rcupreempt to again work correctly if running
on a single CPU after booting is complete.
o Added check to rcupreempt's synchronize_sched() for there
being but one online CPU.
Tested all three variants both SMP and !SMP, booted fine, passed a short
rcutorture test on both x86 and Power.
Located-by: Vegard Nossum <vegard.nossum@gmail.com>
Tested-by: Vegard Nossum <vegard.nossum@gmail.com>
Tested-by: Paul E. McKenney <paulmck@linux.vnet.ibm.com>
Signed-off-by: Paul E. McKenney <paulmck@linux.vnet.ibm.com>
Signed-off-by: Ingo Molnar <mingo@elte.hu>
2009-02-26 10:03:42 +08:00
|
|
|
|
rcu: Merge preemptable-RCU functionality into hierarchical RCU
Create a kernel/rcutree_plugin.h file that contains definitions
for preemptable RCU (or, under the #else branch of the #ifdef,
empty definitions for the classic non-preemptable semantics).
These definitions fit into plugins defined in kernel/rcutree.c
for this purpose.
This variant of preemptable RCU uses a new algorithm whose
read-side expense is roughly that of classic hierarchical RCU
under CONFIG_PREEMPT. This new algorithm's update-side expense
is similar to that of classic hierarchical RCU, and, in absence
of read-side preemption or blocking, is exactly that of classic
hierarchical RCU. Perhaps more important, this new algorithm
has a much simpler implementation, saving well over 1,000 lines
of code compared to mainline's implementation of preemptable
RCU, which will hopefully be retired in favor of this new
algorithm.
The simplifications are obtained by maintaining per-task
nesting state for running tasks, and using a simple
lock-protected algorithm to handle accounting when tasks block
within RCU read-side critical sections, making use of lessons
learned while creating numerous user-level RCU implementations
over the past 18 months.
Signed-off-by: Paul E. McKenney <paulmck@linux.vnet.ibm.com>
Cc: laijs@cn.fujitsu.com
Cc: dipankar@in.ibm.com
Cc: akpm@linux-foundation.org
Cc: mathieu.desnoyers@polymtl.ca
Cc: josht@linux.vnet.ibm.com
Cc: dvhltc@us.ibm.com
Cc: niv@us.ibm.com
Cc: peterz@infradead.org
Cc: rostedt@goodmis.org
LKML-Reference: <12509746134003-git-send-email->
Signed-off-by: Ingo Molnar <mingo@elte.hu>
2009-08-23 04:56:52 +08:00
|
|
|
#if defined(CONFIG_TREE_RCU) || defined(CONFIG_TREE_PREEMPT_RCU)
|
"Tree RCU": scalable classic RCU implementation
This patch fixes a long-standing performance bug in classic RCU that
results in massive internal-to-RCU lock contention on systems with
more than a few hundred CPUs. Although this patch creates a separate
flavor of RCU for ease of review and patch maintenance, it is intended
to replace classic RCU.
This patch still handles stress better than does mainline, so I am still
calling it ready for inclusion. This patch is against the -tip tree.
Nevertheless, experience on an actual 1000+ CPU machine would still be
most welcome.
Most of the changes noted below were found while creating an rcutiny
(which should permit ejecting the current rcuclassic) and while doing
detailed line-by-line documentation.
Updates from v9 (http://lkml.org/lkml/2008/12/2/334):
o Fixes from remainder of line-by-line code walkthrough,
including comment spelling, initialization, undesirable
narrowing due to type conversion, removing redundant memory
barriers, removing redundant local-variable initialization,
and removing redundant local variables.
I do not believe that any of these fixes address the CPU-hotplug
issues that Andi Kleen was seeing, but please do give it a whirl
in case the machine is smarter than I am.
A writeup from the walkthrough may be found at the following
URL, in case you are suffering from terminal insomnia or
masochism:
http://www.kernel.org/pub/linux/kernel/people/paulmck/tmp/rcutree-walkthrough.2008.12.16a.pdf
o Made rcutree tracing use seq_file, as suggested some time
ago by Lai Jiangshan.
o Added a .csv variant of the rcudata debugfs trace file, to allow
people having thousands of CPUs to drop the data into
a spreadsheet. Tested with oocalc and gnumeric. Updated
documentation to suit.
Updates from v8 (http://lkml.org/lkml/2008/11/15/139):
o Fix a theoretical race between grace-period initialization and
force_quiescent_state() that could occur if more than three
jiffies were required to carry out the grace-period
initialization. Which it might, if you had enough CPUs.
o Apply Ingo's printk-standardization patch.
o Substitute local variables for repeated accesses to global
variables.
o Fix comment misspellings and redundant (but harmless) increments
of ->n_rcu_pending (this latter after having explicitly added it).
o Apply checkpatch fixes.
Updates from v7 (http://lkml.org/lkml/2008/10/10/291):
o Fixed a number of problems noted by Gautham Shenoy, including
the cpu-stall-detection bug that he was having difficulty
convincing me was real. ;-)
o Changed cpu-stall detection to wait for ten seconds rather than
three in order to reduce false positive, as suggested by Ingo
Molnar.
o Produced a design document (http://lwn.net/Articles/305782/).
The act of writing this document uncovered a number of both
theoretical and "here and now" bugs as noted below.
o Fix dynticks_nesting accounting confusion, simplify WARN_ON()
condition, fix kerneldoc comments, and add memory barriers
in dynticks interface functions.
o Add more data to tracing.
o Remove unused "rcu_barrier" field from rcu_data structure.
o Count calls to rcu_pending() from scheduling-clock interrupt
to use as a surrogate timebase should jiffies stop counting.
o Fix a theoretical race between force_quiescent_state() and
grace-period initialization. Yes, initialization does have to
go on for some jiffies for this race to occur, but given enough
CPUs...
Updates from v6 (http://lkml.org/lkml/2008/9/23/448):
o Fix a number of checkpatch.pl complaints.
o Apply review comments from Ingo Molnar and Lai Jiangshan
on the stall-detection code.
o Fix several bugs in !CONFIG_SMP builds.
o Fix a misspelled config-parameter name so that RCU now announces
at boot time if stall detection is configured.
o Run tests on numerous combinations of configurations parameters,
which after the fixes above, now build and run correctly.
Updates from v5 (http://lkml.org/lkml/2008/9/15/92, bad subject line):
o Fix a compiler error in the !CONFIG_FANOUT_EXACT case (blew a
changeset some time ago, and finally got around to retesting
this option).
o Fix some tracing bugs in rcupreempt that caused incorrect
totals to be printed.
o I now test with a more brutal random-selection online/offline
script (attached). Probably more brutal than it needs to be
on the people reading it as well, but so it goes.
o A number of optimizations and usability improvements:
o Make rcu_pending() ignore the grace-period timeout when
there is no grace period in progress.
o Make force_quiescent_state() avoid going for a global
lock in the case where there is no grace period in
progress.
o Rearrange struct fields to improve struct layout.
o Make call_rcu() initiate a grace period if RCU was
idle, rather than waiting for the next scheduling
clock interrupt.
o Invoke rcu_irq_enter() and rcu_irq_exit() only when
idle, as suggested by Andi Kleen. I still don't
completely trust this change, and might back it out.
o Make CONFIG_RCU_TRACE be the single config variable
manipulated for all forms of RCU, instead of the prior
confusion.
o Document tracing files and formats for both rcupreempt
and rcutree.
Updates from v4 for those missing v5 given its bad subject line:
o Separated dynticks interface so that NMIs and irqs call separate
functions, greatly simplifying it. In particular, this code
no longer requires a proof of correctness. ;-)
o Separated dynticks state out into its own per-CPU structure,
avoiding the duplicated accounting.
o The case where a dynticks-idle CPU runs an irq handler that
invokes call_rcu() is now correctly handled, forcing that CPU
out of dynticks-idle mode.
o Review comments have been applied (thank you all!!!).
For but one example, fixed the dynticks-ordering issue that
Manfred pointed out, saving me much debugging. ;-)
o Adjusted rcuclassic and rcupreempt to handle dynticks changes.
Attached is an updated patch to Classic RCU that applies a hierarchy,
greatly reducing the contention on the top-level lock for large machines.
This passes 10-hour concurrent rcutorture and online-offline testing on
128-CPU ppc64 without dynticks enabled, and exposes some timekeeping
bugs in presence of dynticks (exciting working on a system where
"sleep 1" hangs until interrupted...), which were fixed in the
2.6.27 kernel. It is getting more reliable than mainline by some
measures, so the next version will be against -tip for inclusion.
See also Manfred Spraul's recent patches (or his earlier work from
2004 at http://marc.info/?l=linux-kernel&m=108546384711797&w=2).
We will converge onto a common patch in the fullness of time, but are
currently exploring different regions of the design space. That said,
I have already gratefully stolen quite a few of Manfred's ideas.
This patch provides CONFIG_RCU_FANOUT, which controls the bushiness
of the RCU hierarchy. Defaults to 32 on 32-bit machines and 64 on
64-bit machines. If CONFIG_NR_CPUS is less than CONFIG_RCU_FANOUT,
there is no hierarchy. By default, the RCU initialization code will
adjust CONFIG_RCU_FANOUT to balance the hierarchy, so strongly NUMA
architectures may choose to set CONFIG_RCU_FANOUT_EXACT to disable
this balancing, allowing the hierarchy to be exactly aligned to the
underlying hardware. Up to two levels of hierarchy are permitted
(in addition to the root node), allowing up to 16,384 CPUs on 32-bit
systems and up to 262,144 CPUs on 64-bit systems. I just know that I
am going to regret saying this, but this seems more than sufficient
for the foreseeable future. (Some architectures might wish to set
CONFIG_RCU_FANOUT=4, which would limit such architectures to 64 CPUs.
If this becomes a real problem, additional levels can be added, but I
doubt that it will make a significant difference on real hardware.)
In the common case, a given CPU will manipulate its private rcu_data
structure and the rcu_node structure that it shares with its immediate
neighbors. This can reduce both lock and memory contention by multiple
orders of magnitude, which should eliminate the need for the strange
manipulations that are reported to be required when running Linux on
very large systems.
Some shortcomings:
o More bugs will probably surface as a result of an ongoing
line-by-line code inspection.
Patches will be provided as required.
o There are probably hangs, rcutorture failures, &c. Seems
quite stable on a 128-CPU machine, but that is kind of small
compared to 4096 CPUs. However, seems to do better than
mainline.
Patches will be provided as required.
o The memory footprint of this version is several KB larger
than rcuclassic.
A separate UP-only rcutiny patch will be provided, which will
reduce the memory footprint significantly, even compared
to the old rcuclassic. One such patch passes light testing,
and has a memory footprint smaller even than rcuclassic.
Initial reaction from various embedded guys was "it is not
worth it", so am putting it aside.
Credits:
o Manfred Spraul for ideas, review comments, and bugs spotted,
as well as some good friendly competition. ;-)
o Josh Triplett, Ingo Molnar, Peter Zijlstra, Mathieu Desnoyers,
Lai Jiangshan, Andi Kleen, Andy Whitcroft, and Andrew Morton
for reviews and comments.
o Thomas Gleixner for much-needed help with some timer issues
(see patches below).
o Jon M. Tollefson, Tim Pepper, Andrew Theurer, Jose R. Santos,
Andy Whitcroft, Darrick Wong, Nishanth Aravamudan, Anton
Blanchard, Dave Kleikamp, and Nathan Lynch for keeping machines
alive despite my heavy abuse^Wtesting.
Signed-off-by: Paul E. McKenney <paulmck@linux.vnet.ibm.com>
Signed-off-by: Ingo Molnar <mingo@elte.hu>
2008-12-19 04:55:32 +08:00
|
|
|
#include <linux/rcutree.h>
|
2010-06-30 07:49:16 +08:00
|
|
|
#elif defined(CONFIG_TINY_RCU) || defined(CONFIG_TINY_PREEMPT_RCU)
|
rcu: "Tiny RCU", The Bloatwatch Edition
This patch is a version of RCU designed for !SMP provided for a
small-footprint RCU implementation. In particular, the
implementation of synchronize_rcu() is extremely lightweight and
high performance. It passes rcutorture testing in each of the
four relevant configurations (combinations of NO_HZ and PREEMPT)
on x86. This saves about 1K bytes compared to old Classic RCU
(which is no longer in mainline), and more than three kilobytes
compared to Hierarchical RCU (updated to 2.6.30):
CONFIG_TREE_RCU:
text data bss dec filename
183 4 0 187 kernel/rcupdate.o
2783 520 36 3339 kernel/rcutree.o
3526 Total (vs 4565 for v7)
CONFIG_TREE_PREEMPT_RCU:
text data bss dec filename
263 4 0 267 kernel/rcupdate.o
4594 776 52 5422 kernel/rcutree.o
5689 Total (6155 for v7)
CONFIG_TINY_RCU:
text data bss dec filename
96 4 0 100 kernel/rcupdate.o
734 24 0 758 kernel/rcutiny.o
858 Total (vs 848 for v7)
The above is for x86. Your mileage may vary on other platforms.
Further compression is possible, but is being procrastinated.
Changes from v7 (http://lkml.org/lkml/2009/10/9/388)
o Apply Lai Jiangshan's review comments (aside from
might_sleep() in synchronize_sched(), which is covered by SMP builds).
o Fix up expedited primitives.
Changes from v6 (http://lkml.org/lkml/2009/9/23/293).
o Forward ported to put it into the 2.6.33 stream.
o Added lockdep support.
o Make lightweight rcu_barrier.
Changes from v5 (http://lkml.org/lkml/2009/6/23/12).
o Ported to latest pre-2.6.32 merge window kernel.
- Renamed rcu_qsctr_inc() to rcu_sched_qs().
- Renamed rcu_bh_qsctr_inc() to rcu_bh_qs().
- Provided trivial rcu_cpu_notify().
- Provided trivial exit_rcu().
- Provided trivial rcu_needs_cpu().
- Fixed up the rcu_*_enter/exit() functions in linux/hardirq.h.
o Removed the dependence on EMBEDDED, with a view to making
TINY_RCU default for !SMP at some time in the future.
o Added (trivial) support for expedited grace periods.
Changes from v4 (http://lkml.org/lkml/2009/5/2/91) include:
o Squeeze the size down a bit further by removing the
->completed field from struct rcu_ctrlblk.
o This permits synchronize_rcu() to become the empty function.
Previous concerns about rcutorture were unfounded, as
rcutorture correctly handles a constant value from
rcu_batches_completed() and rcu_batches_completed_bh().
Changes from v3 (http://lkml.org/lkml/2009/3/29/221) include:
o Changed rcu_batches_completed(), rcu_batches_completed_bh()
rcu_enter_nohz(), rcu_exit_nohz(), rcu_nmi_enter(), and
rcu_nmi_exit(), to be static inlines, as suggested by David
Howells. Doing this saves about 100 bytes from rcutiny.o.
(The numbers between v3 and this v4 of the patch are not directly
comparable, since they are against different versions of Linux.)
Changes from v2 (http://lkml.org/lkml/2009/2/3/333) include:
o Fix whitespace issues.
o Change short-circuit "||" operator to instead be "+" in order
to fix performance bug noted by "kraai" on LWN.
(http://lwn.net/Articles/324348/)
Changes from v1 (http://lkml.org/lkml/2009/1/13/440) include:
o This version depends on EMBEDDED as well as !SMP, as suggested
by Ingo.
o Updated rcu_needs_cpu() to unconditionally return zero,
permitting the CPU to enter dynticks-idle mode at any time.
This works because callbacks can be invoked upon entry to
dynticks-idle mode.
o Paul is now OK with this being included, based on a poll at
the Kernel Miniconf at linux.conf.au, where about ten people said
that they cared about saving 900 bytes on single-CPU systems.
o Applies to both mainline and tip/core/rcu.
Signed-off-by: Paul E. McKenney <paulmck@linux.vnet.ibm.com>
Acked-by: David Howells <dhowells@redhat.com>
Acked-by: Josh Triplett <josh@joshtriplett.org>
Reviewed-by: Lai Jiangshan <laijs@cn.fujitsu.com>
Cc: dipankar@in.ibm.com
Cc: mathieu.desnoyers@polymtl.ca
Cc: dvhltc@us.ibm.com
Cc: niv@us.ibm.com
Cc: peterz@infradead.org
Cc: rostedt@goodmis.org
Cc: Valdis.Kletnieks@vt.edu
Cc: avi@redhat.com
Cc: mtosatti@redhat.com
LKML-Reference: <12565226351355-git-send-email->
Signed-off-by: Ingo Molnar <mingo@elte.hu>
2009-10-26 10:03:50 +08:00
|
|
|
#include <linux/rcutiny.h>
|
"Tree RCU": scalable classic RCU implementation
This patch fixes a long-standing performance bug in classic RCU that
results in massive internal-to-RCU lock contention on systems with
more than a few hundred CPUs. Although this patch creates a separate
flavor of RCU for ease of review and patch maintenance, it is intended
to replace classic RCU.
This patch still handles stress better than does mainline, so I am still
calling it ready for inclusion. This patch is against the -tip tree.
Nevertheless, experience on an actual 1000+ CPU machine would still be
most welcome.
Most of the changes noted below were found while creating an rcutiny
(which should permit ejecting the current rcuclassic) and while doing
detailed line-by-line documentation.
Updates from v9 (http://lkml.org/lkml/2008/12/2/334):
o Fixes from remainder of line-by-line code walkthrough,
including comment spelling, initialization, undesirable
narrowing due to type conversion, removing redundant memory
barriers, removing redundant local-variable initialization,
and removing redundant local variables.
I do not believe that any of these fixes address the CPU-hotplug
issues that Andi Kleen was seeing, but please do give it a whirl
in case the machine is smarter than I am.
A writeup from the walkthrough may be found at the following
URL, in case you are suffering from terminal insomnia or
masochism:
http://www.kernel.org/pub/linux/kernel/people/paulmck/tmp/rcutree-walkthrough.2008.12.16a.pdf
o Made rcutree tracing use seq_file, as suggested some time
ago by Lai Jiangshan.
o Added a .csv variant of the rcudata debugfs trace file, to allow
people having thousands of CPUs to drop the data into
a spreadsheet. Tested with oocalc and gnumeric. Updated
documentation to suit.
Updates from v8 (http://lkml.org/lkml/2008/11/15/139):
o Fix a theoretical race between grace-period initialization and
force_quiescent_state() that could occur if more than three
jiffies were required to carry out the grace-period
initialization. Which it might, if you had enough CPUs.
o Apply Ingo's printk-standardization patch.
o Substitute local variables for repeated accesses to global
variables.
o Fix comment misspellings and redundant (but harmless) increments
of ->n_rcu_pending (this latter after having explicitly added it).
o Apply checkpatch fixes.
Updates from v7 (http://lkml.org/lkml/2008/10/10/291):
o Fixed a number of problems noted by Gautham Shenoy, including
the cpu-stall-detection bug that he was having difficulty
convincing me was real. ;-)
o Changed cpu-stall detection to wait for ten seconds rather than
three in order to reduce false positive, as suggested by Ingo
Molnar.
o Produced a design document (http://lwn.net/Articles/305782/).
The act of writing this document uncovered a number of both
theoretical and "here and now" bugs as noted below.
o Fix dynticks_nesting accounting confusion, simplify WARN_ON()
condition, fix kerneldoc comments, and add memory barriers
in dynticks interface functions.
o Add more data to tracing.
o Remove unused "rcu_barrier" field from rcu_data structure.
o Count calls to rcu_pending() from scheduling-clock interrupt
to use as a surrogate timebase should jiffies stop counting.
o Fix a theoretical race between force_quiescent_state() and
grace-period initialization. Yes, initialization does have to
go on for some jiffies for this race to occur, but given enough
CPUs...
Updates from v6 (http://lkml.org/lkml/2008/9/23/448):
o Fix a number of checkpatch.pl complaints.
o Apply review comments from Ingo Molnar and Lai Jiangshan
on the stall-detection code.
o Fix several bugs in !CONFIG_SMP builds.
o Fix a misspelled config-parameter name so that RCU now announces
at boot time if stall detection is configured.
o Run tests on numerous combinations of configurations parameters,
which after the fixes above, now build and run correctly.
Updates from v5 (http://lkml.org/lkml/2008/9/15/92, bad subject line):
o Fix a compiler error in the !CONFIG_FANOUT_EXACT case (blew a
changeset some time ago, and finally got around to retesting
this option).
o Fix some tracing bugs in rcupreempt that caused incorrect
totals to be printed.
o I now test with a more brutal random-selection online/offline
script (attached). Probably more brutal than it needs to be
on the people reading it as well, but so it goes.
o A number of optimizations and usability improvements:
o Make rcu_pending() ignore the grace-period timeout when
there is no grace period in progress.
o Make force_quiescent_state() avoid going for a global
lock in the case where there is no grace period in
progress.
o Rearrange struct fields to improve struct layout.
o Make call_rcu() initiate a grace period if RCU was
idle, rather than waiting for the next scheduling
clock interrupt.
o Invoke rcu_irq_enter() and rcu_irq_exit() only when
idle, as suggested by Andi Kleen. I still don't
completely trust this change, and might back it out.
o Make CONFIG_RCU_TRACE be the single config variable
manipulated for all forms of RCU, instead of the prior
confusion.
o Document tracing files and formats for both rcupreempt
and rcutree.
Updates from v4 for those missing v5 given its bad subject line:
o Separated dynticks interface so that NMIs and irqs call separate
functions, greatly simplifying it. In particular, this code
no longer requires a proof of correctness. ;-)
o Separated dynticks state out into its own per-CPU structure,
avoiding the duplicated accounting.
o The case where a dynticks-idle CPU runs an irq handler that
invokes call_rcu() is now correctly handled, forcing that CPU
out of dynticks-idle mode.
o Review comments have been applied (thank you all!!!).
For but one example, fixed the dynticks-ordering issue that
Manfred pointed out, saving me much debugging. ;-)
o Adjusted rcuclassic and rcupreempt to handle dynticks changes.
Attached is an updated patch to Classic RCU that applies a hierarchy,
greatly reducing the contention on the top-level lock for large machines.
This passes 10-hour concurrent rcutorture and online-offline testing on
128-CPU ppc64 without dynticks enabled, and exposes some timekeeping
bugs in presence of dynticks (exciting working on a system where
"sleep 1" hangs until interrupted...), which were fixed in the
2.6.27 kernel. It is getting more reliable than mainline by some
measures, so the next version will be against -tip for inclusion.
See also Manfred Spraul's recent patches (or his earlier work from
2004 at http://marc.info/?l=linux-kernel&m=108546384711797&w=2).
We will converge onto a common patch in the fullness of time, but are
currently exploring different regions of the design space. That said,
I have already gratefully stolen quite a few of Manfred's ideas.
This patch provides CONFIG_RCU_FANOUT, which controls the bushiness
of the RCU hierarchy. Defaults to 32 on 32-bit machines and 64 on
64-bit machines. If CONFIG_NR_CPUS is less than CONFIG_RCU_FANOUT,
there is no hierarchy. By default, the RCU initialization code will
adjust CONFIG_RCU_FANOUT to balance the hierarchy, so strongly NUMA
architectures may choose to set CONFIG_RCU_FANOUT_EXACT to disable
this balancing, allowing the hierarchy to be exactly aligned to the
underlying hardware. Up to two levels of hierarchy are permitted
(in addition to the root node), allowing up to 16,384 CPUs on 32-bit
systems and up to 262,144 CPUs on 64-bit systems. I just know that I
am going to regret saying this, but this seems more than sufficient
for the foreseeable future. (Some architectures might wish to set
CONFIG_RCU_FANOUT=4, which would limit such architectures to 64 CPUs.
If this becomes a real problem, additional levels can be added, but I
doubt that it will make a significant difference on real hardware.)
In the common case, a given CPU will manipulate its private rcu_data
structure and the rcu_node structure that it shares with its immediate
neighbors. This can reduce both lock and memory contention by multiple
orders of magnitude, which should eliminate the need for the strange
manipulations that are reported to be required when running Linux on
very large systems.
Some shortcomings:
o More bugs will probably surface as a result of an ongoing
line-by-line code inspection.
Patches will be provided as required.
o There are probably hangs, rcutorture failures, &c. Seems
quite stable on a 128-CPU machine, but that is kind of small
compared to 4096 CPUs. However, seems to do better than
mainline.
Patches will be provided as required.
o The memory footprint of this version is several KB larger
than rcuclassic.
A separate UP-only rcutiny patch will be provided, which will
reduce the memory footprint significantly, even compared
to the old rcuclassic. One such patch passes light testing,
and has a memory footprint smaller even than rcuclassic.
Initial reaction from various embedded guys was "it is not
worth it", so am putting it aside.
Credits:
o Manfred Spraul for ideas, review comments, and bugs spotted,
as well as some good friendly competition. ;-)
o Josh Triplett, Ingo Molnar, Peter Zijlstra, Mathieu Desnoyers,
Lai Jiangshan, Andi Kleen, Andy Whitcroft, and Andrew Morton
for reviews and comments.
o Thomas Gleixner for much-needed help with some timer issues
(see patches below).
o Jon M. Tollefson, Tim Pepper, Andrew Theurer, Jose R. Santos,
Andy Whitcroft, Darrick Wong, Nishanth Aravamudan, Anton
Blanchard, Dave Kleikamp, and Nathan Lynch for keeping machines
alive despite my heavy abuse^Wtesting.
Signed-off-by: Paul E. McKenney <paulmck@linux.vnet.ibm.com>
Signed-off-by: Ingo Molnar <mingo@elte.hu>
2008-12-19 04:55:32 +08:00
|
|
|
#else
|
|
|
|
#error "Unknown RCU implementation specified to kernel configuration"
|
2009-08-23 04:56:53 +08:00
|
|
|
#endif
|
2008-01-26 04:08:24 +08:00
|
|
|
|
2010-04-17 20:48:42 +08:00
|
|
|
/*
|
|
|
|
* init_rcu_head_on_stack()/destroy_rcu_head_on_stack() are needed for dynamic
|
|
|
|
* initialization and destruction of rcu_head on the stack. rcu_head structures
|
|
|
|
* allocated dynamically in the heap or defined statically don't need any
|
|
|
|
* initialization.
|
|
|
|
*/
|
|
|
|
#ifdef CONFIG_DEBUG_OBJECTS_RCU_HEAD
|
|
|
|
extern void init_rcu_head_on_stack(struct rcu_head *head);
|
|
|
|
extern void destroy_rcu_head_on_stack(struct rcu_head *head);
|
|
|
|
#else /* !CONFIG_DEBUG_OBJECTS_RCU_HEAD */
|
2010-04-17 20:48:39 +08:00
|
|
|
static inline void init_rcu_head_on_stack(struct rcu_head *head)
|
|
|
|
{
|
|
|
|
}
|
|
|
|
|
|
|
|
static inline void destroy_rcu_head_on_stack(struct rcu_head *head)
|
|
|
|
{
|
|
|
|
}
|
2010-04-17 20:48:42 +08:00
|
|
|
#endif /* #else !CONFIG_DEBUG_OBJECTS_RCU_HEAD */
|
2010-04-17 20:48:39 +08:00
|
|
|
|
2009-08-23 04:56:47 +08:00
|
|
|
#ifdef CONFIG_DEBUG_LOCK_ALLOC
|
rcu: Introduce lockdep-based checking to RCU read-side primitives
Inspection is proving insufficient to catch all RCU misuses,
which is understandable given that rcu_dereference() might be
protected by any of four different flavors of RCU (RCU, RCU-bh,
RCU-sched, and SRCU), and might also/instead be protected by any
of a number of locking primitives. It is therefore time to
enlist the aid of lockdep.
This set of patches is inspired by earlier work by Peter
Zijlstra and Thomas Gleixner, and takes the following approach:
o Set up separate lockdep classes for RCU, RCU-bh, and RCU-sched.
o Set up separate lockdep classes for each instance of SRCU.
o Create primitives that check for being in an RCU read-side
critical section. These return exact answers if lockdep is
fully enabled, but if unsure, report being in an RCU read-side
critical section. (We want to avoid false positives!)
The primitives are:
For RCU: rcu_read_lock_held(void)
For RCU-bh: rcu_read_lock_bh_held(void)
For RCU-sched: rcu_read_lock_sched_held(void)
For SRCU: srcu_read_lock_held(struct srcu_struct *sp)
o Add rcu_dereference_check(), which takes a second argument
in which one places a boolean expression based on the above
primitives and/or lockdep_is_held().
o A new kernel configuration parameter, CONFIG_PROVE_RCU, enables
rcu_dereference_check(). This depends on CONFIG_PROVE_LOCKING,
and should be quite helpful during the transition period while
CONFIG_PROVE_RCU-unaware patches are in flight.
The existing rcu_dereference() primitive does no checking, but
upcoming patches will change that.
Signed-off-by: Paul E. McKenney <paulmck@linux.vnet.ibm.com>
Cc: laijs@cn.fujitsu.com
Cc: dipankar@in.ibm.com
Cc: mathieu.desnoyers@polymtl.ca
Cc: josh@joshtriplett.org
Cc: dvhltc@us.ibm.com
Cc: niv@us.ibm.com
Cc: peterz@infradead.org
Cc: rostedt@goodmis.org
Cc: Valdis.Kletnieks@vt.edu
Cc: dhowells@redhat.com
LKML-Reference: <1266887105-1528-1-git-send-email-paulmck@linux.vnet.ibm.com>
Signed-off-by: Ingo Molnar <mingo@elte.hu>
2010-02-23 09:04:45 +08:00
|
|
|
|
2009-08-23 04:56:47 +08:00
|
|
|
extern struct lockdep_map rcu_lock_map;
|
rcu: Introduce lockdep-based checking to RCU read-side primitives
Inspection is proving insufficient to catch all RCU misuses,
which is understandable given that rcu_dereference() might be
protected by any of four different flavors of RCU (RCU, RCU-bh,
RCU-sched, and SRCU), and might also/instead be protected by any
of a number of locking primitives. It is therefore time to
enlist the aid of lockdep.
This set of patches is inspired by earlier work by Peter
Zijlstra and Thomas Gleixner, and takes the following approach:
o Set up separate lockdep classes for RCU, RCU-bh, and RCU-sched.
o Set up separate lockdep classes for each instance of SRCU.
o Create primitives that check for being in an RCU read-side
critical section. These return exact answers if lockdep is
fully enabled, but if unsure, report being in an RCU read-side
critical section. (We want to avoid false positives!)
The primitives are:
For RCU: rcu_read_lock_held(void)
For RCU-bh: rcu_read_lock_bh_held(void)
For RCU-sched: rcu_read_lock_sched_held(void)
For SRCU: srcu_read_lock_held(struct srcu_struct *sp)
o Add rcu_dereference_check(), which takes a second argument
in which one places a boolean expression based on the above
primitives and/or lockdep_is_held().
o A new kernel configuration parameter, CONFIG_PROVE_RCU, enables
rcu_dereference_check(). This depends on CONFIG_PROVE_LOCKING,
and should be quite helpful during the transition period while
CONFIG_PROVE_RCU-unaware patches are in flight.
The existing rcu_dereference() primitive does no checking, but
upcoming patches will change that.
Signed-off-by: Paul E. McKenney <paulmck@linux.vnet.ibm.com>
Cc: laijs@cn.fujitsu.com
Cc: dipankar@in.ibm.com
Cc: mathieu.desnoyers@polymtl.ca
Cc: josh@joshtriplett.org
Cc: dvhltc@us.ibm.com
Cc: niv@us.ibm.com
Cc: peterz@infradead.org
Cc: rostedt@goodmis.org
Cc: Valdis.Kletnieks@vt.edu
Cc: dhowells@redhat.com
LKML-Reference: <1266887105-1528-1-git-send-email-paulmck@linux.vnet.ibm.com>
Signed-off-by: Ingo Molnar <mingo@elte.hu>
2010-02-23 09:04:45 +08:00
|
|
|
# define rcu_read_acquire() \
|
|
|
|
lock_acquire(&rcu_lock_map, 0, 0, 2, 1, NULL, _THIS_IP_)
|
2009-08-23 04:56:47 +08:00
|
|
|
# define rcu_read_release() lock_release(&rcu_lock_map, 1, _THIS_IP_)
|
rcu: Introduce lockdep-based checking to RCU read-side primitives
Inspection is proving insufficient to catch all RCU misuses,
which is understandable given that rcu_dereference() might be
protected by any of four different flavors of RCU (RCU, RCU-bh,
RCU-sched, and SRCU), and might also/instead be protected by any
of a number of locking primitives. It is therefore time to
enlist the aid of lockdep.
This set of patches is inspired by earlier work by Peter
Zijlstra and Thomas Gleixner, and takes the following approach:
o Set up separate lockdep classes for RCU, RCU-bh, and RCU-sched.
o Set up separate lockdep classes for each instance of SRCU.
o Create primitives that check for being in an RCU read-side
critical section. These return exact answers if lockdep is
fully enabled, but if unsure, report being in an RCU read-side
critical section. (We want to avoid false positives!)
The primitives are:
For RCU: rcu_read_lock_held(void)
For RCU-bh: rcu_read_lock_bh_held(void)
For RCU-sched: rcu_read_lock_sched_held(void)
For SRCU: srcu_read_lock_held(struct srcu_struct *sp)
o Add rcu_dereference_check(), which takes a second argument
in which one places a boolean expression based on the above
primitives and/or lockdep_is_held().
o A new kernel configuration parameter, CONFIG_PROVE_RCU, enables
rcu_dereference_check(). This depends on CONFIG_PROVE_LOCKING,
and should be quite helpful during the transition period while
CONFIG_PROVE_RCU-unaware patches are in flight.
The existing rcu_dereference() primitive does no checking, but
upcoming patches will change that.
Signed-off-by: Paul E. McKenney <paulmck@linux.vnet.ibm.com>
Cc: laijs@cn.fujitsu.com
Cc: dipankar@in.ibm.com
Cc: mathieu.desnoyers@polymtl.ca
Cc: josh@joshtriplett.org
Cc: dvhltc@us.ibm.com
Cc: niv@us.ibm.com
Cc: peterz@infradead.org
Cc: rostedt@goodmis.org
Cc: Valdis.Kletnieks@vt.edu
Cc: dhowells@redhat.com
LKML-Reference: <1266887105-1528-1-git-send-email-paulmck@linux.vnet.ibm.com>
Signed-off-by: Ingo Molnar <mingo@elte.hu>
2010-02-23 09:04:45 +08:00
|
|
|
|
|
|
|
extern struct lockdep_map rcu_bh_lock_map;
|
|
|
|
# define rcu_read_acquire_bh() \
|
|
|
|
lock_acquire(&rcu_bh_lock_map, 0, 0, 2, 1, NULL, _THIS_IP_)
|
|
|
|
# define rcu_read_release_bh() lock_release(&rcu_bh_lock_map, 1, _THIS_IP_)
|
|
|
|
|
|
|
|
extern struct lockdep_map rcu_sched_lock_map;
|
|
|
|
# define rcu_read_acquire_sched() \
|
|
|
|
lock_acquire(&rcu_sched_lock_map, 0, 0, 2, 1, NULL, _THIS_IP_)
|
|
|
|
# define rcu_read_release_sched() \
|
|
|
|
lock_release(&rcu_sched_lock_map, 1, _THIS_IP_)
|
|
|
|
|
2010-04-16 03:50:39 +08:00
|
|
|
extern int debug_lockdep_rcu_enabled(void);
|
2010-03-03 23:46:57 +08:00
|
|
|
|
rcu: Introduce lockdep-based checking to RCU read-side primitives
Inspection is proving insufficient to catch all RCU misuses,
which is understandable given that rcu_dereference() might be
protected by any of four different flavors of RCU (RCU, RCU-bh,
RCU-sched, and SRCU), and might also/instead be protected by any
of a number of locking primitives. It is therefore time to
enlist the aid of lockdep.
This set of patches is inspired by earlier work by Peter
Zijlstra and Thomas Gleixner, and takes the following approach:
o Set up separate lockdep classes for RCU, RCU-bh, and RCU-sched.
o Set up separate lockdep classes for each instance of SRCU.
o Create primitives that check for being in an RCU read-side
critical section. These return exact answers if lockdep is
fully enabled, but if unsure, report being in an RCU read-side
critical section. (We want to avoid false positives!)
The primitives are:
For RCU: rcu_read_lock_held(void)
For RCU-bh: rcu_read_lock_bh_held(void)
For RCU-sched: rcu_read_lock_sched_held(void)
For SRCU: srcu_read_lock_held(struct srcu_struct *sp)
o Add rcu_dereference_check(), which takes a second argument
in which one places a boolean expression based on the above
primitives and/or lockdep_is_held().
o A new kernel configuration parameter, CONFIG_PROVE_RCU, enables
rcu_dereference_check(). This depends on CONFIG_PROVE_LOCKING,
and should be quite helpful during the transition period while
CONFIG_PROVE_RCU-unaware patches are in flight.
The existing rcu_dereference() primitive does no checking, but
upcoming patches will change that.
Signed-off-by: Paul E. McKenney <paulmck@linux.vnet.ibm.com>
Cc: laijs@cn.fujitsu.com
Cc: dipankar@in.ibm.com
Cc: mathieu.desnoyers@polymtl.ca
Cc: josh@joshtriplett.org
Cc: dvhltc@us.ibm.com
Cc: niv@us.ibm.com
Cc: peterz@infradead.org
Cc: rostedt@goodmis.org
Cc: Valdis.Kletnieks@vt.edu
Cc: dhowells@redhat.com
LKML-Reference: <1266887105-1528-1-git-send-email-paulmck@linux.vnet.ibm.com>
Signed-off-by: Ingo Molnar <mingo@elte.hu>
2010-02-23 09:04:45 +08:00
|
|
|
/**
|
2010-04-29 05:39:09 +08:00
|
|
|
* rcu_read_lock_held() - might we be in RCU read-side critical section?
|
rcu: Introduce lockdep-based checking to RCU read-side primitives
Inspection is proving insufficient to catch all RCU misuses,
which is understandable given that rcu_dereference() might be
protected by any of four different flavors of RCU (RCU, RCU-bh,
RCU-sched, and SRCU), and might also/instead be protected by any
of a number of locking primitives. It is therefore time to
enlist the aid of lockdep.
This set of patches is inspired by earlier work by Peter
Zijlstra and Thomas Gleixner, and takes the following approach:
o Set up separate lockdep classes for RCU, RCU-bh, and RCU-sched.
o Set up separate lockdep classes for each instance of SRCU.
o Create primitives that check for being in an RCU read-side
critical section. These return exact answers if lockdep is
fully enabled, but if unsure, report being in an RCU read-side
critical section. (We want to avoid false positives!)
The primitives are:
For RCU: rcu_read_lock_held(void)
For RCU-bh: rcu_read_lock_bh_held(void)
For RCU-sched: rcu_read_lock_sched_held(void)
For SRCU: srcu_read_lock_held(struct srcu_struct *sp)
o Add rcu_dereference_check(), which takes a second argument
in which one places a boolean expression based on the above
primitives and/or lockdep_is_held().
o A new kernel configuration parameter, CONFIG_PROVE_RCU, enables
rcu_dereference_check(). This depends on CONFIG_PROVE_LOCKING,
and should be quite helpful during the transition period while
CONFIG_PROVE_RCU-unaware patches are in flight.
The existing rcu_dereference() primitive does no checking, but
upcoming patches will change that.
Signed-off-by: Paul E. McKenney <paulmck@linux.vnet.ibm.com>
Cc: laijs@cn.fujitsu.com
Cc: dipankar@in.ibm.com
Cc: mathieu.desnoyers@polymtl.ca
Cc: josh@joshtriplett.org
Cc: dvhltc@us.ibm.com
Cc: niv@us.ibm.com
Cc: peterz@infradead.org
Cc: rostedt@goodmis.org
Cc: Valdis.Kletnieks@vt.edu
Cc: dhowells@redhat.com
LKML-Reference: <1266887105-1528-1-git-send-email-paulmck@linux.vnet.ibm.com>
Signed-off-by: Ingo Molnar <mingo@elte.hu>
2010-02-23 09:04:45 +08:00
|
|
|
*
|
2010-03-31 01:52:21 +08:00
|
|
|
* If CONFIG_DEBUG_LOCK_ALLOC is selected, returns nonzero iff in an RCU
|
|
|
|
* read-side critical section. In absence of CONFIG_DEBUG_LOCK_ALLOC,
|
rcu: Introduce lockdep-based checking to RCU read-side primitives
Inspection is proving insufficient to catch all RCU misuses,
which is understandable given that rcu_dereference() might be
protected by any of four different flavors of RCU (RCU, RCU-bh,
RCU-sched, and SRCU), and might also/instead be protected by any
of a number of locking primitives. It is therefore time to
enlist the aid of lockdep.
This set of patches is inspired by earlier work by Peter
Zijlstra and Thomas Gleixner, and takes the following approach:
o Set up separate lockdep classes for RCU, RCU-bh, and RCU-sched.
o Set up separate lockdep classes for each instance of SRCU.
o Create primitives that check for being in an RCU read-side
critical section. These return exact answers if lockdep is
fully enabled, but if unsure, report being in an RCU read-side
critical section. (We want to avoid false positives!)
The primitives are:
For RCU: rcu_read_lock_held(void)
For RCU-bh: rcu_read_lock_bh_held(void)
For RCU-sched: rcu_read_lock_sched_held(void)
For SRCU: srcu_read_lock_held(struct srcu_struct *sp)
o Add rcu_dereference_check(), which takes a second argument
in which one places a boolean expression based on the above
primitives and/or lockdep_is_held().
o A new kernel configuration parameter, CONFIG_PROVE_RCU, enables
rcu_dereference_check(). This depends on CONFIG_PROVE_LOCKING,
and should be quite helpful during the transition period while
CONFIG_PROVE_RCU-unaware patches are in flight.
The existing rcu_dereference() primitive does no checking, but
upcoming patches will change that.
Signed-off-by: Paul E. McKenney <paulmck@linux.vnet.ibm.com>
Cc: laijs@cn.fujitsu.com
Cc: dipankar@in.ibm.com
Cc: mathieu.desnoyers@polymtl.ca
Cc: josh@joshtriplett.org
Cc: dvhltc@us.ibm.com
Cc: niv@us.ibm.com
Cc: peterz@infradead.org
Cc: rostedt@goodmis.org
Cc: Valdis.Kletnieks@vt.edu
Cc: dhowells@redhat.com
LKML-Reference: <1266887105-1528-1-git-send-email-paulmck@linux.vnet.ibm.com>
Signed-off-by: Ingo Molnar <mingo@elte.hu>
2010-02-23 09:04:45 +08:00
|
|
|
* this assumes we are in an RCU read-side critical section unless it can
|
2010-04-29 05:39:09 +08:00
|
|
|
* prove otherwise. This is useful for debug checks in functions that
|
|
|
|
* require that they be called within an RCU read-side critical section.
|
2010-03-03 23:46:57 +08:00
|
|
|
*
|
2010-04-29 05:39:09 +08:00
|
|
|
* Checks debug_lockdep_rcu_enabled() to prevent false positives during boot
|
2010-03-31 01:59:28 +08:00
|
|
|
* and while lockdep is disabled.
|
rcu: Introduce lockdep-based checking to RCU read-side primitives
Inspection is proving insufficient to catch all RCU misuses,
which is understandable given that rcu_dereference() might be
protected by any of four different flavors of RCU (RCU, RCU-bh,
RCU-sched, and SRCU), and might also/instead be protected by any
of a number of locking primitives. It is therefore time to
enlist the aid of lockdep.
This set of patches is inspired by earlier work by Peter
Zijlstra and Thomas Gleixner, and takes the following approach:
o Set up separate lockdep classes for RCU, RCU-bh, and RCU-sched.
o Set up separate lockdep classes for each instance of SRCU.
o Create primitives that check for being in an RCU read-side
critical section. These return exact answers if lockdep is
fully enabled, but if unsure, report being in an RCU read-side
critical section. (We want to avoid false positives!)
The primitives are:
For RCU: rcu_read_lock_held(void)
For RCU-bh: rcu_read_lock_bh_held(void)
For RCU-sched: rcu_read_lock_sched_held(void)
For SRCU: srcu_read_lock_held(struct srcu_struct *sp)
o Add rcu_dereference_check(), which takes a second argument
in which one places a boolean expression based on the above
primitives and/or lockdep_is_held().
o A new kernel configuration parameter, CONFIG_PROVE_RCU, enables
rcu_dereference_check(). This depends on CONFIG_PROVE_LOCKING,
and should be quite helpful during the transition period while
CONFIG_PROVE_RCU-unaware patches are in flight.
The existing rcu_dereference() primitive does no checking, but
upcoming patches will change that.
Signed-off-by: Paul E. McKenney <paulmck@linux.vnet.ibm.com>
Cc: laijs@cn.fujitsu.com
Cc: dipankar@in.ibm.com
Cc: mathieu.desnoyers@polymtl.ca
Cc: josh@joshtriplett.org
Cc: dvhltc@us.ibm.com
Cc: niv@us.ibm.com
Cc: peterz@infradead.org
Cc: rostedt@goodmis.org
Cc: Valdis.Kletnieks@vt.edu
Cc: dhowells@redhat.com
LKML-Reference: <1266887105-1528-1-git-send-email-paulmck@linux.vnet.ibm.com>
Signed-off-by: Ingo Molnar <mingo@elte.hu>
2010-02-23 09:04:45 +08:00
|
|
|
*/
|
|
|
|
static inline int rcu_read_lock_held(void)
|
|
|
|
{
|
2010-03-03 23:46:57 +08:00
|
|
|
if (!debug_lockdep_rcu_enabled())
|
|
|
|
return 1;
|
|
|
|
return lock_is_held(&rcu_lock_map);
|
rcu: Introduce lockdep-based checking to RCU read-side primitives
Inspection is proving insufficient to catch all RCU misuses,
which is understandable given that rcu_dereference() might be
protected by any of four different flavors of RCU (RCU, RCU-bh,
RCU-sched, and SRCU), and might also/instead be protected by any
of a number of locking primitives. It is therefore time to
enlist the aid of lockdep.
This set of patches is inspired by earlier work by Peter
Zijlstra and Thomas Gleixner, and takes the following approach:
o Set up separate lockdep classes for RCU, RCU-bh, and RCU-sched.
o Set up separate lockdep classes for each instance of SRCU.
o Create primitives that check for being in an RCU read-side
critical section. These return exact answers if lockdep is
fully enabled, but if unsure, report being in an RCU read-side
critical section. (We want to avoid false positives!)
The primitives are:
For RCU: rcu_read_lock_held(void)
For RCU-bh: rcu_read_lock_bh_held(void)
For RCU-sched: rcu_read_lock_sched_held(void)
For SRCU: srcu_read_lock_held(struct srcu_struct *sp)
o Add rcu_dereference_check(), which takes a second argument
in which one places a boolean expression based on the above
primitives and/or lockdep_is_held().
o A new kernel configuration parameter, CONFIG_PROVE_RCU, enables
rcu_dereference_check(). This depends on CONFIG_PROVE_LOCKING,
and should be quite helpful during the transition period while
CONFIG_PROVE_RCU-unaware patches are in flight.
The existing rcu_dereference() primitive does no checking, but
upcoming patches will change that.
Signed-off-by: Paul E. McKenney <paulmck@linux.vnet.ibm.com>
Cc: laijs@cn.fujitsu.com
Cc: dipankar@in.ibm.com
Cc: mathieu.desnoyers@polymtl.ca
Cc: josh@joshtriplett.org
Cc: dvhltc@us.ibm.com
Cc: niv@us.ibm.com
Cc: peterz@infradead.org
Cc: rostedt@goodmis.org
Cc: Valdis.Kletnieks@vt.edu
Cc: dhowells@redhat.com
LKML-Reference: <1266887105-1528-1-git-send-email-paulmck@linux.vnet.ibm.com>
Signed-off-by: Ingo Molnar <mingo@elte.hu>
2010-02-23 09:04:45 +08:00
|
|
|
}
|
|
|
|
|
2010-03-16 08:03:43 +08:00
|
|
|
/*
|
|
|
|
* rcu_read_lock_bh_held() is defined out of line to avoid #include-file
|
|
|
|
* hell.
|
rcu: Introduce lockdep-based checking to RCU read-side primitives
Inspection is proving insufficient to catch all RCU misuses,
which is understandable given that rcu_dereference() might be
protected by any of four different flavors of RCU (RCU, RCU-bh,
RCU-sched, and SRCU), and might also/instead be protected by any
of a number of locking primitives. It is therefore time to
enlist the aid of lockdep.
This set of patches is inspired by earlier work by Peter
Zijlstra and Thomas Gleixner, and takes the following approach:
o Set up separate lockdep classes for RCU, RCU-bh, and RCU-sched.
o Set up separate lockdep classes for each instance of SRCU.
o Create primitives that check for being in an RCU read-side
critical section. These return exact answers if lockdep is
fully enabled, but if unsure, report being in an RCU read-side
critical section. (We want to avoid false positives!)
The primitives are:
For RCU: rcu_read_lock_held(void)
For RCU-bh: rcu_read_lock_bh_held(void)
For RCU-sched: rcu_read_lock_sched_held(void)
For SRCU: srcu_read_lock_held(struct srcu_struct *sp)
o Add rcu_dereference_check(), which takes a second argument
in which one places a boolean expression based on the above
primitives and/or lockdep_is_held().
o A new kernel configuration parameter, CONFIG_PROVE_RCU, enables
rcu_dereference_check(). This depends on CONFIG_PROVE_LOCKING,
and should be quite helpful during the transition period while
CONFIG_PROVE_RCU-unaware patches are in flight.
The existing rcu_dereference() primitive does no checking, but
upcoming patches will change that.
Signed-off-by: Paul E. McKenney <paulmck@linux.vnet.ibm.com>
Cc: laijs@cn.fujitsu.com
Cc: dipankar@in.ibm.com
Cc: mathieu.desnoyers@polymtl.ca
Cc: josh@joshtriplett.org
Cc: dvhltc@us.ibm.com
Cc: niv@us.ibm.com
Cc: peterz@infradead.org
Cc: rostedt@goodmis.org
Cc: Valdis.Kletnieks@vt.edu
Cc: dhowells@redhat.com
LKML-Reference: <1266887105-1528-1-git-send-email-paulmck@linux.vnet.ibm.com>
Signed-off-by: Ingo Molnar <mingo@elte.hu>
2010-02-23 09:04:45 +08:00
|
|
|
*/
|
2010-03-16 08:03:43 +08:00
|
|
|
extern int rcu_read_lock_bh_held(void);
|
rcu: Introduce lockdep-based checking to RCU read-side primitives
Inspection is proving insufficient to catch all RCU misuses,
which is understandable given that rcu_dereference() might be
protected by any of four different flavors of RCU (RCU, RCU-bh,
RCU-sched, and SRCU), and might also/instead be protected by any
of a number of locking primitives. It is therefore time to
enlist the aid of lockdep.
This set of patches is inspired by earlier work by Peter
Zijlstra and Thomas Gleixner, and takes the following approach:
o Set up separate lockdep classes for RCU, RCU-bh, and RCU-sched.
o Set up separate lockdep classes for each instance of SRCU.
o Create primitives that check for being in an RCU read-side
critical section. These return exact answers if lockdep is
fully enabled, but if unsure, report being in an RCU read-side
critical section. (We want to avoid false positives!)
The primitives are:
For RCU: rcu_read_lock_held(void)
For RCU-bh: rcu_read_lock_bh_held(void)
For RCU-sched: rcu_read_lock_sched_held(void)
For SRCU: srcu_read_lock_held(struct srcu_struct *sp)
o Add rcu_dereference_check(), which takes a second argument
in which one places a boolean expression based on the above
primitives and/or lockdep_is_held().
o A new kernel configuration parameter, CONFIG_PROVE_RCU, enables
rcu_dereference_check(). This depends on CONFIG_PROVE_LOCKING,
and should be quite helpful during the transition period while
CONFIG_PROVE_RCU-unaware patches are in flight.
The existing rcu_dereference() primitive does no checking, but
upcoming patches will change that.
Signed-off-by: Paul E. McKenney <paulmck@linux.vnet.ibm.com>
Cc: laijs@cn.fujitsu.com
Cc: dipankar@in.ibm.com
Cc: mathieu.desnoyers@polymtl.ca
Cc: josh@joshtriplett.org
Cc: dvhltc@us.ibm.com
Cc: niv@us.ibm.com
Cc: peterz@infradead.org
Cc: rostedt@goodmis.org
Cc: Valdis.Kletnieks@vt.edu
Cc: dhowells@redhat.com
LKML-Reference: <1266887105-1528-1-git-send-email-paulmck@linux.vnet.ibm.com>
Signed-off-by: Ingo Molnar <mingo@elte.hu>
2010-02-23 09:04:45 +08:00
|
|
|
|
|
|
|
/**
|
2010-04-29 05:39:09 +08:00
|
|
|
* rcu_read_lock_sched_held() - might we be in RCU-sched read-side critical section?
|
rcu: Introduce lockdep-based checking to RCU read-side primitives
Inspection is proving insufficient to catch all RCU misuses,
which is understandable given that rcu_dereference() might be
protected by any of four different flavors of RCU (RCU, RCU-bh,
RCU-sched, and SRCU), and might also/instead be protected by any
of a number of locking primitives. It is therefore time to
enlist the aid of lockdep.
This set of patches is inspired by earlier work by Peter
Zijlstra and Thomas Gleixner, and takes the following approach:
o Set up separate lockdep classes for RCU, RCU-bh, and RCU-sched.
o Set up separate lockdep classes for each instance of SRCU.
o Create primitives that check for being in an RCU read-side
critical section. These return exact answers if lockdep is
fully enabled, but if unsure, report being in an RCU read-side
critical section. (We want to avoid false positives!)
The primitives are:
For RCU: rcu_read_lock_held(void)
For RCU-bh: rcu_read_lock_bh_held(void)
For RCU-sched: rcu_read_lock_sched_held(void)
For SRCU: srcu_read_lock_held(struct srcu_struct *sp)
o Add rcu_dereference_check(), which takes a second argument
in which one places a boolean expression based on the above
primitives and/or lockdep_is_held().
o A new kernel configuration parameter, CONFIG_PROVE_RCU, enables
rcu_dereference_check(). This depends on CONFIG_PROVE_LOCKING,
and should be quite helpful during the transition period while
CONFIG_PROVE_RCU-unaware patches are in flight.
The existing rcu_dereference() primitive does no checking, but
upcoming patches will change that.
Signed-off-by: Paul E. McKenney <paulmck@linux.vnet.ibm.com>
Cc: laijs@cn.fujitsu.com
Cc: dipankar@in.ibm.com
Cc: mathieu.desnoyers@polymtl.ca
Cc: josh@joshtriplett.org
Cc: dvhltc@us.ibm.com
Cc: niv@us.ibm.com
Cc: peterz@infradead.org
Cc: rostedt@goodmis.org
Cc: Valdis.Kletnieks@vt.edu
Cc: dhowells@redhat.com
LKML-Reference: <1266887105-1528-1-git-send-email-paulmck@linux.vnet.ibm.com>
Signed-off-by: Ingo Molnar <mingo@elte.hu>
2010-02-23 09:04:45 +08:00
|
|
|
*
|
2010-03-31 01:52:21 +08:00
|
|
|
* If CONFIG_DEBUG_LOCK_ALLOC is selected, returns nonzero iff in an
|
|
|
|
* RCU-sched read-side critical section. In absence of
|
|
|
|
* CONFIG_DEBUG_LOCK_ALLOC, this assumes we are in an RCU-sched read-side
|
|
|
|
* critical section unless it can prove otherwise. Note that disabling
|
|
|
|
* of preemption (including disabling irqs) counts as an RCU-sched
|
2010-04-29 05:39:09 +08:00
|
|
|
* read-side critical section. This is useful for debug checks in functions
|
|
|
|
* that required that they be called within an RCU-sched read-side
|
|
|
|
* critical section.
|
2010-03-03 23:46:57 +08:00
|
|
|
*
|
2010-03-31 01:59:28 +08:00
|
|
|
* Check debug_lockdep_rcu_enabled() to prevent false positives during boot
|
|
|
|
* and while lockdep is disabled.
|
rcu: Introduce lockdep-based checking to RCU read-side primitives
Inspection is proving insufficient to catch all RCU misuses,
which is understandable given that rcu_dereference() might be
protected by any of four different flavors of RCU (RCU, RCU-bh,
RCU-sched, and SRCU), and might also/instead be protected by any
of a number of locking primitives. It is therefore time to
enlist the aid of lockdep.
This set of patches is inspired by earlier work by Peter
Zijlstra and Thomas Gleixner, and takes the following approach:
o Set up separate lockdep classes for RCU, RCU-bh, and RCU-sched.
o Set up separate lockdep classes for each instance of SRCU.
o Create primitives that check for being in an RCU read-side
critical section. These return exact answers if lockdep is
fully enabled, but if unsure, report being in an RCU read-side
critical section. (We want to avoid false positives!)
The primitives are:
For RCU: rcu_read_lock_held(void)
For RCU-bh: rcu_read_lock_bh_held(void)
For RCU-sched: rcu_read_lock_sched_held(void)
For SRCU: srcu_read_lock_held(struct srcu_struct *sp)
o Add rcu_dereference_check(), which takes a second argument
in which one places a boolean expression based on the above
primitives and/or lockdep_is_held().
o A new kernel configuration parameter, CONFIG_PROVE_RCU, enables
rcu_dereference_check(). This depends on CONFIG_PROVE_LOCKING,
and should be quite helpful during the transition period while
CONFIG_PROVE_RCU-unaware patches are in flight.
The existing rcu_dereference() primitive does no checking, but
upcoming patches will change that.
Signed-off-by: Paul E. McKenney <paulmck@linux.vnet.ibm.com>
Cc: laijs@cn.fujitsu.com
Cc: dipankar@in.ibm.com
Cc: mathieu.desnoyers@polymtl.ca
Cc: josh@joshtriplett.org
Cc: dvhltc@us.ibm.com
Cc: niv@us.ibm.com
Cc: peterz@infradead.org
Cc: rostedt@goodmis.org
Cc: Valdis.Kletnieks@vt.edu
Cc: dhowells@redhat.com
LKML-Reference: <1266887105-1528-1-git-send-email-paulmck@linux.vnet.ibm.com>
Signed-off-by: Ingo Molnar <mingo@elte.hu>
2010-02-23 09:04:45 +08:00
|
|
|
*/
|
2010-03-04 09:50:16 +08:00
|
|
|
#ifdef CONFIG_PREEMPT
|
rcu: Introduce lockdep-based checking to RCU read-side primitives
Inspection is proving insufficient to catch all RCU misuses,
which is understandable given that rcu_dereference() might be
protected by any of four different flavors of RCU (RCU, RCU-bh,
RCU-sched, and SRCU), and might also/instead be protected by any
of a number of locking primitives. It is therefore time to
enlist the aid of lockdep.
This set of patches is inspired by earlier work by Peter
Zijlstra and Thomas Gleixner, and takes the following approach:
o Set up separate lockdep classes for RCU, RCU-bh, and RCU-sched.
o Set up separate lockdep classes for each instance of SRCU.
o Create primitives that check for being in an RCU read-side
critical section. These return exact answers if lockdep is
fully enabled, but if unsure, report being in an RCU read-side
critical section. (We want to avoid false positives!)
The primitives are:
For RCU: rcu_read_lock_held(void)
For RCU-bh: rcu_read_lock_bh_held(void)
For RCU-sched: rcu_read_lock_sched_held(void)
For SRCU: srcu_read_lock_held(struct srcu_struct *sp)
o Add rcu_dereference_check(), which takes a second argument
in which one places a boolean expression based on the above
primitives and/or lockdep_is_held().
o A new kernel configuration parameter, CONFIG_PROVE_RCU, enables
rcu_dereference_check(). This depends on CONFIG_PROVE_LOCKING,
and should be quite helpful during the transition period while
CONFIG_PROVE_RCU-unaware patches are in flight.
The existing rcu_dereference() primitive does no checking, but
upcoming patches will change that.
Signed-off-by: Paul E. McKenney <paulmck@linux.vnet.ibm.com>
Cc: laijs@cn.fujitsu.com
Cc: dipankar@in.ibm.com
Cc: mathieu.desnoyers@polymtl.ca
Cc: josh@joshtriplett.org
Cc: dvhltc@us.ibm.com
Cc: niv@us.ibm.com
Cc: peterz@infradead.org
Cc: rostedt@goodmis.org
Cc: Valdis.Kletnieks@vt.edu
Cc: dhowells@redhat.com
LKML-Reference: <1266887105-1528-1-git-send-email-paulmck@linux.vnet.ibm.com>
Signed-off-by: Ingo Molnar <mingo@elte.hu>
2010-02-23 09:04:45 +08:00
|
|
|
static inline int rcu_read_lock_sched_held(void)
|
|
|
|
{
|
|
|
|
int lockdep_opinion = 0;
|
|
|
|
|
2010-03-03 23:46:57 +08:00
|
|
|
if (!debug_lockdep_rcu_enabled())
|
|
|
|
return 1;
|
rcu: Introduce lockdep-based checking to RCU read-side primitives
Inspection is proving insufficient to catch all RCU misuses,
which is understandable given that rcu_dereference() might be
protected by any of four different flavors of RCU (RCU, RCU-bh,
RCU-sched, and SRCU), and might also/instead be protected by any
of a number of locking primitives. It is therefore time to
enlist the aid of lockdep.
This set of patches is inspired by earlier work by Peter
Zijlstra and Thomas Gleixner, and takes the following approach:
o Set up separate lockdep classes for RCU, RCU-bh, and RCU-sched.
o Set up separate lockdep classes for each instance of SRCU.
o Create primitives that check for being in an RCU read-side
critical section. These return exact answers if lockdep is
fully enabled, but if unsure, report being in an RCU read-side
critical section. (We want to avoid false positives!)
The primitives are:
For RCU: rcu_read_lock_held(void)
For RCU-bh: rcu_read_lock_bh_held(void)
For RCU-sched: rcu_read_lock_sched_held(void)
For SRCU: srcu_read_lock_held(struct srcu_struct *sp)
o Add rcu_dereference_check(), which takes a second argument
in which one places a boolean expression based on the above
primitives and/or lockdep_is_held().
o A new kernel configuration parameter, CONFIG_PROVE_RCU, enables
rcu_dereference_check(). This depends on CONFIG_PROVE_LOCKING,
and should be quite helpful during the transition period while
CONFIG_PROVE_RCU-unaware patches are in flight.
The existing rcu_dereference() primitive does no checking, but
upcoming patches will change that.
Signed-off-by: Paul E. McKenney <paulmck@linux.vnet.ibm.com>
Cc: laijs@cn.fujitsu.com
Cc: dipankar@in.ibm.com
Cc: mathieu.desnoyers@polymtl.ca
Cc: josh@joshtriplett.org
Cc: dvhltc@us.ibm.com
Cc: niv@us.ibm.com
Cc: peterz@infradead.org
Cc: rostedt@goodmis.org
Cc: Valdis.Kletnieks@vt.edu
Cc: dhowells@redhat.com
LKML-Reference: <1266887105-1528-1-git-send-email-paulmck@linux.vnet.ibm.com>
Signed-off-by: Ingo Molnar <mingo@elte.hu>
2010-02-23 09:04:45 +08:00
|
|
|
if (debug_locks)
|
|
|
|
lockdep_opinion = lock_is_held(&rcu_sched_lock_map);
|
2010-03-19 03:25:33 +08:00
|
|
|
return lockdep_opinion || preempt_count() != 0 || irqs_disabled();
|
rcu: Introduce lockdep-based checking to RCU read-side primitives
Inspection is proving insufficient to catch all RCU misuses,
which is understandable given that rcu_dereference() might be
protected by any of four different flavors of RCU (RCU, RCU-bh,
RCU-sched, and SRCU), and might also/instead be protected by any
of a number of locking primitives. It is therefore time to
enlist the aid of lockdep.
This set of patches is inspired by earlier work by Peter
Zijlstra and Thomas Gleixner, and takes the following approach:
o Set up separate lockdep classes for RCU, RCU-bh, and RCU-sched.
o Set up separate lockdep classes for each instance of SRCU.
o Create primitives that check for being in an RCU read-side
critical section. These return exact answers if lockdep is
fully enabled, but if unsure, report being in an RCU read-side
critical section. (We want to avoid false positives!)
The primitives are:
For RCU: rcu_read_lock_held(void)
For RCU-bh: rcu_read_lock_bh_held(void)
For RCU-sched: rcu_read_lock_sched_held(void)
For SRCU: srcu_read_lock_held(struct srcu_struct *sp)
o Add rcu_dereference_check(), which takes a second argument
in which one places a boolean expression based on the above
primitives and/or lockdep_is_held().
o A new kernel configuration parameter, CONFIG_PROVE_RCU, enables
rcu_dereference_check(). This depends on CONFIG_PROVE_LOCKING,
and should be quite helpful during the transition period while
CONFIG_PROVE_RCU-unaware patches are in flight.
The existing rcu_dereference() primitive does no checking, but
upcoming patches will change that.
Signed-off-by: Paul E. McKenney <paulmck@linux.vnet.ibm.com>
Cc: laijs@cn.fujitsu.com
Cc: dipankar@in.ibm.com
Cc: mathieu.desnoyers@polymtl.ca
Cc: josh@joshtriplett.org
Cc: dvhltc@us.ibm.com
Cc: niv@us.ibm.com
Cc: peterz@infradead.org
Cc: rostedt@goodmis.org
Cc: Valdis.Kletnieks@vt.edu
Cc: dhowells@redhat.com
LKML-Reference: <1266887105-1528-1-git-send-email-paulmck@linux.vnet.ibm.com>
Signed-off-by: Ingo Molnar <mingo@elte.hu>
2010-02-23 09:04:45 +08:00
|
|
|
}
|
2010-03-04 09:50:16 +08:00
|
|
|
#else /* #ifdef CONFIG_PREEMPT */
|
|
|
|
static inline int rcu_read_lock_sched_held(void)
|
|
|
|
{
|
|
|
|
return 1;
|
rcu: Introduce lockdep-based checking to RCU read-side primitives
Inspection is proving insufficient to catch all RCU misuses,
which is understandable given that rcu_dereference() might be
protected by any of four different flavors of RCU (RCU, RCU-bh,
RCU-sched, and SRCU), and might also/instead be protected by any
of a number of locking primitives. It is therefore time to
enlist the aid of lockdep.
This set of patches is inspired by earlier work by Peter
Zijlstra and Thomas Gleixner, and takes the following approach:
o Set up separate lockdep classes for RCU, RCU-bh, and RCU-sched.
o Set up separate lockdep classes for each instance of SRCU.
o Create primitives that check for being in an RCU read-side
critical section. These return exact answers if lockdep is
fully enabled, but if unsure, report being in an RCU read-side
critical section. (We want to avoid false positives!)
The primitives are:
For RCU: rcu_read_lock_held(void)
For RCU-bh: rcu_read_lock_bh_held(void)
For RCU-sched: rcu_read_lock_sched_held(void)
For SRCU: srcu_read_lock_held(struct srcu_struct *sp)
o Add rcu_dereference_check(), which takes a second argument
in which one places a boolean expression based on the above
primitives and/or lockdep_is_held().
o A new kernel configuration parameter, CONFIG_PROVE_RCU, enables
rcu_dereference_check(). This depends on CONFIG_PROVE_LOCKING,
and should be quite helpful during the transition period while
CONFIG_PROVE_RCU-unaware patches are in flight.
The existing rcu_dereference() primitive does no checking, but
upcoming patches will change that.
Signed-off-by: Paul E. McKenney <paulmck@linux.vnet.ibm.com>
Cc: laijs@cn.fujitsu.com
Cc: dipankar@in.ibm.com
Cc: mathieu.desnoyers@polymtl.ca
Cc: josh@joshtriplett.org
Cc: dvhltc@us.ibm.com
Cc: niv@us.ibm.com
Cc: peterz@infradead.org
Cc: rostedt@goodmis.org
Cc: Valdis.Kletnieks@vt.edu
Cc: dhowells@redhat.com
LKML-Reference: <1266887105-1528-1-git-send-email-paulmck@linux.vnet.ibm.com>
Signed-off-by: Ingo Molnar <mingo@elte.hu>
2010-02-23 09:04:45 +08:00
|
|
|
}
|
2010-03-04 09:50:16 +08:00
|
|
|
#endif /* #else #ifdef CONFIG_PREEMPT */
|
rcu: Introduce lockdep-based checking to RCU read-side primitives
Inspection is proving insufficient to catch all RCU misuses,
which is understandable given that rcu_dereference() might be
protected by any of four different flavors of RCU (RCU, RCU-bh,
RCU-sched, and SRCU), and might also/instead be protected by any
of a number of locking primitives. It is therefore time to
enlist the aid of lockdep.
This set of patches is inspired by earlier work by Peter
Zijlstra and Thomas Gleixner, and takes the following approach:
o Set up separate lockdep classes for RCU, RCU-bh, and RCU-sched.
o Set up separate lockdep classes for each instance of SRCU.
o Create primitives that check for being in an RCU read-side
critical section. These return exact answers if lockdep is
fully enabled, but if unsure, report being in an RCU read-side
critical section. (We want to avoid false positives!)
The primitives are:
For RCU: rcu_read_lock_held(void)
For RCU-bh: rcu_read_lock_bh_held(void)
For RCU-sched: rcu_read_lock_sched_held(void)
For SRCU: srcu_read_lock_held(struct srcu_struct *sp)
o Add rcu_dereference_check(), which takes a second argument
in which one places a boolean expression based on the above
primitives and/or lockdep_is_held().
o A new kernel configuration parameter, CONFIG_PROVE_RCU, enables
rcu_dereference_check(). This depends on CONFIG_PROVE_LOCKING,
and should be quite helpful during the transition period while
CONFIG_PROVE_RCU-unaware patches are in flight.
The existing rcu_dereference() primitive does no checking, but
upcoming patches will change that.
Signed-off-by: Paul E. McKenney <paulmck@linux.vnet.ibm.com>
Cc: laijs@cn.fujitsu.com
Cc: dipankar@in.ibm.com
Cc: mathieu.desnoyers@polymtl.ca
Cc: josh@joshtriplett.org
Cc: dvhltc@us.ibm.com
Cc: niv@us.ibm.com
Cc: peterz@infradead.org
Cc: rostedt@goodmis.org
Cc: Valdis.Kletnieks@vt.edu
Cc: dhowells@redhat.com
LKML-Reference: <1266887105-1528-1-git-send-email-paulmck@linux.vnet.ibm.com>
Signed-off-by: Ingo Molnar <mingo@elte.hu>
2010-02-23 09:04:45 +08:00
|
|
|
|
|
|
|
#else /* #ifdef CONFIG_DEBUG_LOCK_ALLOC */
|
|
|
|
|
|
|
|
# define rcu_read_acquire() do { } while (0)
|
|
|
|
# define rcu_read_release() do { } while (0)
|
|
|
|
# define rcu_read_acquire_bh() do { } while (0)
|
|
|
|
# define rcu_read_release_bh() do { } while (0)
|
|
|
|
# define rcu_read_acquire_sched() do { } while (0)
|
|
|
|
# define rcu_read_release_sched() do { } while (0)
|
|
|
|
|
|
|
|
static inline int rcu_read_lock_held(void)
|
|
|
|
{
|
|
|
|
return 1;
|
|
|
|
}
|
|
|
|
|
|
|
|
static inline int rcu_read_lock_bh_held(void)
|
|
|
|
{
|
|
|
|
return 1;
|
|
|
|
}
|
|
|
|
|
2010-03-04 09:50:16 +08:00
|
|
|
#ifdef CONFIG_PREEMPT
|
rcu: Introduce lockdep-based checking to RCU read-side primitives
Inspection is proving insufficient to catch all RCU misuses,
which is understandable given that rcu_dereference() might be
protected by any of four different flavors of RCU (RCU, RCU-bh,
RCU-sched, and SRCU), and might also/instead be protected by any
of a number of locking primitives. It is therefore time to
enlist the aid of lockdep.
This set of patches is inspired by earlier work by Peter
Zijlstra and Thomas Gleixner, and takes the following approach:
o Set up separate lockdep classes for RCU, RCU-bh, and RCU-sched.
o Set up separate lockdep classes for each instance of SRCU.
o Create primitives that check for being in an RCU read-side
critical section. These return exact answers if lockdep is
fully enabled, but if unsure, report being in an RCU read-side
critical section. (We want to avoid false positives!)
The primitives are:
For RCU: rcu_read_lock_held(void)
For RCU-bh: rcu_read_lock_bh_held(void)
For RCU-sched: rcu_read_lock_sched_held(void)
For SRCU: srcu_read_lock_held(struct srcu_struct *sp)
o Add rcu_dereference_check(), which takes a second argument
in which one places a boolean expression based on the above
primitives and/or lockdep_is_held().
o A new kernel configuration parameter, CONFIG_PROVE_RCU, enables
rcu_dereference_check(). This depends on CONFIG_PROVE_LOCKING,
and should be quite helpful during the transition period while
CONFIG_PROVE_RCU-unaware patches are in flight.
The existing rcu_dereference() primitive does no checking, but
upcoming patches will change that.
Signed-off-by: Paul E. McKenney <paulmck@linux.vnet.ibm.com>
Cc: laijs@cn.fujitsu.com
Cc: dipankar@in.ibm.com
Cc: mathieu.desnoyers@polymtl.ca
Cc: josh@joshtriplett.org
Cc: dvhltc@us.ibm.com
Cc: niv@us.ibm.com
Cc: peterz@infradead.org
Cc: rostedt@goodmis.org
Cc: Valdis.Kletnieks@vt.edu
Cc: dhowells@redhat.com
LKML-Reference: <1266887105-1528-1-git-send-email-paulmck@linux.vnet.ibm.com>
Signed-off-by: Ingo Molnar <mingo@elte.hu>
2010-02-23 09:04:45 +08:00
|
|
|
static inline int rcu_read_lock_sched_held(void)
|
|
|
|
{
|
2010-04-03 07:17:17 +08:00
|
|
|
return preempt_count() != 0 || irqs_disabled();
|
rcu: Introduce lockdep-based checking to RCU read-side primitives
Inspection is proving insufficient to catch all RCU misuses,
which is understandable given that rcu_dereference() might be
protected by any of four different flavors of RCU (RCU, RCU-bh,
RCU-sched, and SRCU), and might also/instead be protected by any
of a number of locking primitives. It is therefore time to
enlist the aid of lockdep.
This set of patches is inspired by earlier work by Peter
Zijlstra and Thomas Gleixner, and takes the following approach:
o Set up separate lockdep classes for RCU, RCU-bh, and RCU-sched.
o Set up separate lockdep classes for each instance of SRCU.
o Create primitives that check for being in an RCU read-side
critical section. These return exact answers if lockdep is
fully enabled, but if unsure, report being in an RCU read-side
critical section. (We want to avoid false positives!)
The primitives are:
For RCU: rcu_read_lock_held(void)
For RCU-bh: rcu_read_lock_bh_held(void)
For RCU-sched: rcu_read_lock_sched_held(void)
For SRCU: srcu_read_lock_held(struct srcu_struct *sp)
o Add rcu_dereference_check(), which takes a second argument
in which one places a boolean expression based on the above
primitives and/or lockdep_is_held().
o A new kernel configuration parameter, CONFIG_PROVE_RCU, enables
rcu_dereference_check(). This depends on CONFIG_PROVE_LOCKING,
and should be quite helpful during the transition period while
CONFIG_PROVE_RCU-unaware patches are in flight.
The existing rcu_dereference() primitive does no checking, but
upcoming patches will change that.
Signed-off-by: Paul E. McKenney <paulmck@linux.vnet.ibm.com>
Cc: laijs@cn.fujitsu.com
Cc: dipankar@in.ibm.com
Cc: mathieu.desnoyers@polymtl.ca
Cc: josh@joshtriplett.org
Cc: dvhltc@us.ibm.com
Cc: niv@us.ibm.com
Cc: peterz@infradead.org
Cc: rostedt@goodmis.org
Cc: Valdis.Kletnieks@vt.edu
Cc: dhowells@redhat.com
LKML-Reference: <1266887105-1528-1-git-send-email-paulmck@linux.vnet.ibm.com>
Signed-off-by: Ingo Molnar <mingo@elte.hu>
2010-02-23 09:04:45 +08:00
|
|
|
}
|
2010-03-04 09:50:16 +08:00
|
|
|
#else /* #ifdef CONFIG_PREEMPT */
|
|
|
|
static inline int rcu_read_lock_sched_held(void)
|
|
|
|
{
|
|
|
|
return 1;
|
rcu: Introduce lockdep-based checking to RCU read-side primitives
Inspection is proving insufficient to catch all RCU misuses,
which is understandable given that rcu_dereference() might be
protected by any of four different flavors of RCU (RCU, RCU-bh,
RCU-sched, and SRCU), and might also/instead be protected by any
of a number of locking primitives. It is therefore time to
enlist the aid of lockdep.
This set of patches is inspired by earlier work by Peter
Zijlstra and Thomas Gleixner, and takes the following approach:
o Set up separate lockdep classes for RCU, RCU-bh, and RCU-sched.
o Set up separate lockdep classes for each instance of SRCU.
o Create primitives that check for being in an RCU read-side
critical section. These return exact answers if lockdep is
fully enabled, but if unsure, report being in an RCU read-side
critical section. (We want to avoid false positives!)
The primitives are:
For RCU: rcu_read_lock_held(void)
For RCU-bh: rcu_read_lock_bh_held(void)
For RCU-sched: rcu_read_lock_sched_held(void)
For SRCU: srcu_read_lock_held(struct srcu_struct *sp)
o Add rcu_dereference_check(), which takes a second argument
in which one places a boolean expression based on the above
primitives and/or lockdep_is_held().
o A new kernel configuration parameter, CONFIG_PROVE_RCU, enables
rcu_dereference_check(). This depends on CONFIG_PROVE_LOCKING,
and should be quite helpful during the transition period while
CONFIG_PROVE_RCU-unaware patches are in flight.
The existing rcu_dereference() primitive does no checking, but
upcoming patches will change that.
Signed-off-by: Paul E. McKenney <paulmck@linux.vnet.ibm.com>
Cc: laijs@cn.fujitsu.com
Cc: dipankar@in.ibm.com
Cc: mathieu.desnoyers@polymtl.ca
Cc: josh@joshtriplett.org
Cc: dvhltc@us.ibm.com
Cc: niv@us.ibm.com
Cc: peterz@infradead.org
Cc: rostedt@goodmis.org
Cc: Valdis.Kletnieks@vt.edu
Cc: dhowells@redhat.com
LKML-Reference: <1266887105-1528-1-git-send-email-paulmck@linux.vnet.ibm.com>
Signed-off-by: Ingo Molnar <mingo@elte.hu>
2010-02-23 09:04:45 +08:00
|
|
|
}
|
2010-03-04 09:50:16 +08:00
|
|
|
#endif /* #else #ifdef CONFIG_PREEMPT */
|
rcu: Introduce lockdep-based checking to RCU read-side primitives
Inspection is proving insufficient to catch all RCU misuses,
which is understandable given that rcu_dereference() might be
protected by any of four different flavors of RCU (RCU, RCU-bh,
RCU-sched, and SRCU), and might also/instead be protected by any
of a number of locking primitives. It is therefore time to
enlist the aid of lockdep.
This set of patches is inspired by earlier work by Peter
Zijlstra and Thomas Gleixner, and takes the following approach:
o Set up separate lockdep classes for RCU, RCU-bh, and RCU-sched.
o Set up separate lockdep classes for each instance of SRCU.
o Create primitives that check for being in an RCU read-side
critical section. These return exact answers if lockdep is
fully enabled, but if unsure, report being in an RCU read-side
critical section. (We want to avoid false positives!)
The primitives are:
For RCU: rcu_read_lock_held(void)
For RCU-bh: rcu_read_lock_bh_held(void)
For RCU-sched: rcu_read_lock_sched_held(void)
For SRCU: srcu_read_lock_held(struct srcu_struct *sp)
o Add rcu_dereference_check(), which takes a second argument
in which one places a boolean expression based on the above
primitives and/or lockdep_is_held().
o A new kernel configuration parameter, CONFIG_PROVE_RCU, enables
rcu_dereference_check(). This depends on CONFIG_PROVE_LOCKING,
and should be quite helpful during the transition period while
CONFIG_PROVE_RCU-unaware patches are in flight.
The existing rcu_dereference() primitive does no checking, but
upcoming patches will change that.
Signed-off-by: Paul E. McKenney <paulmck@linux.vnet.ibm.com>
Cc: laijs@cn.fujitsu.com
Cc: dipankar@in.ibm.com
Cc: mathieu.desnoyers@polymtl.ca
Cc: josh@joshtriplett.org
Cc: dvhltc@us.ibm.com
Cc: niv@us.ibm.com
Cc: peterz@infradead.org
Cc: rostedt@goodmis.org
Cc: Valdis.Kletnieks@vt.edu
Cc: dhowells@redhat.com
LKML-Reference: <1266887105-1528-1-git-send-email-paulmck@linux.vnet.ibm.com>
Signed-off-by: Ingo Molnar <mingo@elte.hu>
2010-02-23 09:04:45 +08:00
|
|
|
|
|
|
|
#endif /* #else #ifdef CONFIG_DEBUG_LOCK_ALLOC */
|
|
|
|
|
|
|
|
#ifdef CONFIG_PROVE_RCU
|
|
|
|
|
2010-05-07 00:28:41 +08:00
|
|
|
extern int rcu_my_thread_group_empty(void);
|
|
|
|
|
2010-06-26 00:08:19 +08:00
|
|
|
/**
|
|
|
|
* rcu_lockdep_assert - emit lockdep splat if specified condition not met
|
|
|
|
* @c: condition to check
|
|
|
|
*/
|
|
|
|
#define rcu_lockdep_assert(c) \
|
2010-04-20 16:23:07 +08:00
|
|
|
do { \
|
|
|
|
static bool __warned; \
|
|
|
|
if (debug_lockdep_rcu_enabled() && !__warned && !(c)) { \
|
|
|
|
__warned = true; \
|
|
|
|
lockdep_rcu_dereference(__FILE__, __LINE__); \
|
|
|
|
} \
|
|
|
|
} while (0)
|
|
|
|
|
2010-04-29 05:39:09 +08:00
|
|
|
#else /* #ifdef CONFIG_PROVE_RCU */
|
|
|
|
|
2010-06-26 00:08:19 +08:00
|
|
|
#define rcu_lockdep_assert(c) do { } while (0)
|
2010-04-29 05:39:09 +08:00
|
|
|
|
|
|
|
#endif /* #else #ifdef CONFIG_PROVE_RCU */
|
|
|
|
|
|
|
|
/*
|
|
|
|
* Helper functions for rcu_dereference_check(), rcu_dereference_protected()
|
|
|
|
* and rcu_assign_pointer(). Some of these could be folded into their
|
|
|
|
* callers, but they are left separate in order to ease introduction of
|
|
|
|
* multiple flavors of pointers to match the multiple flavors of RCU
|
|
|
|
* (e.g., __rcu_bh, * __rcu_sched, and __srcu), should this make sense in
|
|
|
|
* the future.
|
|
|
|
*/
|
|
|
|
#define __rcu_access_pointer(p, space) \
|
|
|
|
({ \
|
|
|
|
typeof(*p) *_________p1 = (typeof(*p)*__force )ACCESS_ONCE(p); \
|
|
|
|
(void) (((typeof (*p) space *)p) == p); \
|
|
|
|
((typeof(*p) __force __kernel *)(_________p1)); \
|
|
|
|
})
|
|
|
|
#define __rcu_dereference_check(p, c, space) \
|
|
|
|
({ \
|
|
|
|
typeof(*p) *_________p1 = (typeof(*p)*__force )ACCESS_ONCE(p); \
|
2010-06-26 00:08:19 +08:00
|
|
|
rcu_lockdep_assert(c); \
|
2010-04-29 05:39:09 +08:00
|
|
|
(void) (((typeof (*p) space *)p) == p); \
|
|
|
|
smp_read_barrier_depends(); \
|
|
|
|
((typeof(*p) __force __kernel *)(_________p1)); \
|
|
|
|
})
|
|
|
|
#define __rcu_dereference_protected(p, c, space) \
|
|
|
|
({ \
|
2010-06-26 00:08:19 +08:00
|
|
|
rcu_lockdep_assert(c); \
|
2010-04-29 05:39:09 +08:00
|
|
|
(void) (((typeof (*p) space *)p) == p); \
|
|
|
|
((typeof(*p) __force __kernel *)(p)); \
|
|
|
|
})
|
|
|
|
|
|
|
|
#define __rcu_dereference_index_check(p, c) \
|
|
|
|
({ \
|
|
|
|
typeof(p) _________p1 = ACCESS_ONCE(p); \
|
2010-06-26 00:08:19 +08:00
|
|
|
rcu_lockdep_assert(c); \
|
2010-04-29 05:39:09 +08:00
|
|
|
smp_read_barrier_depends(); \
|
|
|
|
(_________p1); \
|
|
|
|
})
|
|
|
|
#define __rcu_assign_pointer(p, v, space) \
|
|
|
|
({ \
|
|
|
|
if (!__builtin_constant_p(v) || \
|
|
|
|
((v) != NULL)) \
|
|
|
|
smp_wmb(); \
|
|
|
|
(p) = (typeof(*v) __force space *)(v); \
|
|
|
|
})
|
|
|
|
|
|
|
|
|
|
|
|
/**
|
|
|
|
* rcu_access_pointer() - fetch RCU pointer with no dereferencing
|
|
|
|
* @p: The pointer to read
|
|
|
|
*
|
|
|
|
* Return the value of the specified RCU-protected pointer, but omit the
|
|
|
|
* smp_read_barrier_depends() and keep the ACCESS_ONCE(). This is useful
|
|
|
|
* when the value of this pointer is accessed, but the pointer is not
|
|
|
|
* dereferenced, for example, when testing an RCU-protected pointer against
|
|
|
|
* NULL. Although rcu_access_pointer() may also be used in cases where
|
|
|
|
* update-side locks prevent the value of the pointer from changing, you
|
|
|
|
* should instead use rcu_dereference_protected() for this use case.
|
|
|
|
*/
|
|
|
|
#define rcu_access_pointer(p) __rcu_access_pointer((p), __rcu)
|
|
|
|
|
rcu: Introduce lockdep-based checking to RCU read-side primitives
Inspection is proving insufficient to catch all RCU misuses,
which is understandable given that rcu_dereference() might be
protected by any of four different flavors of RCU (RCU, RCU-bh,
RCU-sched, and SRCU), and might also/instead be protected by any
of a number of locking primitives. It is therefore time to
enlist the aid of lockdep.
This set of patches is inspired by earlier work by Peter
Zijlstra and Thomas Gleixner, and takes the following approach:
o Set up separate lockdep classes for RCU, RCU-bh, and RCU-sched.
o Set up separate lockdep classes for each instance of SRCU.
o Create primitives that check for being in an RCU read-side
critical section. These return exact answers if lockdep is
fully enabled, but if unsure, report being in an RCU read-side
critical section. (We want to avoid false positives!)
The primitives are:
For RCU: rcu_read_lock_held(void)
For RCU-bh: rcu_read_lock_bh_held(void)
For RCU-sched: rcu_read_lock_sched_held(void)
For SRCU: srcu_read_lock_held(struct srcu_struct *sp)
o Add rcu_dereference_check(), which takes a second argument
in which one places a boolean expression based on the above
primitives and/or lockdep_is_held().
o A new kernel configuration parameter, CONFIG_PROVE_RCU, enables
rcu_dereference_check(). This depends on CONFIG_PROVE_LOCKING,
and should be quite helpful during the transition period while
CONFIG_PROVE_RCU-unaware patches are in flight.
The existing rcu_dereference() primitive does no checking, but
upcoming patches will change that.
Signed-off-by: Paul E. McKenney <paulmck@linux.vnet.ibm.com>
Cc: laijs@cn.fujitsu.com
Cc: dipankar@in.ibm.com
Cc: mathieu.desnoyers@polymtl.ca
Cc: josh@joshtriplett.org
Cc: dvhltc@us.ibm.com
Cc: niv@us.ibm.com
Cc: peterz@infradead.org
Cc: rostedt@goodmis.org
Cc: Valdis.Kletnieks@vt.edu
Cc: dhowells@redhat.com
LKML-Reference: <1266887105-1528-1-git-send-email-paulmck@linux.vnet.ibm.com>
Signed-off-by: Ingo Molnar <mingo@elte.hu>
2010-02-23 09:04:45 +08:00
|
|
|
/**
|
2010-04-29 05:39:09 +08:00
|
|
|
* rcu_dereference_check() - rcu_dereference with debug checking
|
2010-04-10 06:39:11 +08:00
|
|
|
* @p: The pointer to read, prior to dereferencing
|
|
|
|
* @c: The conditions under which the dereference will take place
|
rcu: Introduce lockdep-based checking to RCU read-side primitives
Inspection is proving insufficient to catch all RCU misuses,
which is understandable given that rcu_dereference() might be
protected by any of four different flavors of RCU (RCU, RCU-bh,
RCU-sched, and SRCU), and might also/instead be protected by any
of a number of locking primitives. It is therefore time to
enlist the aid of lockdep.
This set of patches is inspired by earlier work by Peter
Zijlstra and Thomas Gleixner, and takes the following approach:
o Set up separate lockdep classes for RCU, RCU-bh, and RCU-sched.
o Set up separate lockdep classes for each instance of SRCU.
o Create primitives that check for being in an RCU read-side
critical section. These return exact answers if lockdep is
fully enabled, but if unsure, report being in an RCU read-side
critical section. (We want to avoid false positives!)
The primitives are:
For RCU: rcu_read_lock_held(void)
For RCU-bh: rcu_read_lock_bh_held(void)
For RCU-sched: rcu_read_lock_sched_held(void)
For SRCU: srcu_read_lock_held(struct srcu_struct *sp)
o Add rcu_dereference_check(), which takes a second argument
in which one places a boolean expression based on the above
primitives and/or lockdep_is_held().
o A new kernel configuration parameter, CONFIG_PROVE_RCU, enables
rcu_dereference_check(). This depends on CONFIG_PROVE_LOCKING,
and should be quite helpful during the transition period while
CONFIG_PROVE_RCU-unaware patches are in flight.
The existing rcu_dereference() primitive does no checking, but
upcoming patches will change that.
Signed-off-by: Paul E. McKenney <paulmck@linux.vnet.ibm.com>
Cc: laijs@cn.fujitsu.com
Cc: dipankar@in.ibm.com
Cc: mathieu.desnoyers@polymtl.ca
Cc: josh@joshtriplett.org
Cc: dvhltc@us.ibm.com
Cc: niv@us.ibm.com
Cc: peterz@infradead.org
Cc: rostedt@goodmis.org
Cc: Valdis.Kletnieks@vt.edu
Cc: dhowells@redhat.com
LKML-Reference: <1266887105-1528-1-git-send-email-paulmck@linux.vnet.ibm.com>
Signed-off-by: Ingo Molnar <mingo@elte.hu>
2010-02-23 09:04:45 +08:00
|
|
|
*
|
2010-04-10 06:39:11 +08:00
|
|
|
* Do an rcu_dereference(), but check that the conditions under which the
|
2010-04-29 05:39:09 +08:00
|
|
|
* dereference will take place are correct. Typically the conditions
|
|
|
|
* indicate the various locking conditions that should be held at that
|
|
|
|
* point. The check should return true if the conditions are satisfied.
|
|
|
|
* An implicit check for being in an RCU read-side critical section
|
|
|
|
* (rcu_read_lock()) is included.
|
2010-04-10 06:39:11 +08:00
|
|
|
*
|
|
|
|
* For example:
|
|
|
|
*
|
2010-04-29 05:39:09 +08:00
|
|
|
* bar = rcu_dereference_check(foo->bar, lockdep_is_held(&foo->lock));
|
2010-04-10 06:39:11 +08:00
|
|
|
*
|
|
|
|
* could be used to indicate to lockdep that foo->bar may only be dereferenced
|
2010-04-29 05:39:09 +08:00
|
|
|
* if either rcu_read_lock() is held, or that the lock required to replace
|
2010-04-10 06:39:11 +08:00
|
|
|
* the bar struct at foo->bar is held.
|
|
|
|
*
|
|
|
|
* Note that the list of conditions may also include indications of when a lock
|
|
|
|
* need not be held, for example during initialisation or destruction of the
|
|
|
|
* target struct:
|
|
|
|
*
|
2010-04-29 05:39:09 +08:00
|
|
|
* bar = rcu_dereference_check(foo->bar, lockdep_is_held(&foo->lock) ||
|
2010-04-10 06:39:11 +08:00
|
|
|
* atomic_read(&foo->usage) == 0);
|
2010-04-29 05:39:09 +08:00
|
|
|
*
|
|
|
|
* Inserts memory barriers on architectures that require them
|
|
|
|
* (currently only the Alpha), prevents the compiler from refetching
|
|
|
|
* (and from merging fetches), and, more importantly, documents exactly
|
|
|
|
* which pointers are protected by RCU and checks that the pointer is
|
|
|
|
* annotated as __rcu.
|
rcu: Introduce lockdep-based checking to RCU read-side primitives
Inspection is proving insufficient to catch all RCU misuses,
which is understandable given that rcu_dereference() might be
protected by any of four different flavors of RCU (RCU, RCU-bh,
RCU-sched, and SRCU), and might also/instead be protected by any
of a number of locking primitives. It is therefore time to
enlist the aid of lockdep.
This set of patches is inspired by earlier work by Peter
Zijlstra and Thomas Gleixner, and takes the following approach:
o Set up separate lockdep classes for RCU, RCU-bh, and RCU-sched.
o Set up separate lockdep classes for each instance of SRCU.
o Create primitives that check for being in an RCU read-side
critical section. These return exact answers if lockdep is
fully enabled, but if unsure, report being in an RCU read-side
critical section. (We want to avoid false positives!)
The primitives are:
For RCU: rcu_read_lock_held(void)
For RCU-bh: rcu_read_lock_bh_held(void)
For RCU-sched: rcu_read_lock_sched_held(void)
For SRCU: srcu_read_lock_held(struct srcu_struct *sp)
o Add rcu_dereference_check(), which takes a second argument
in which one places a boolean expression based on the above
primitives and/or lockdep_is_held().
o A new kernel configuration parameter, CONFIG_PROVE_RCU, enables
rcu_dereference_check(). This depends on CONFIG_PROVE_LOCKING,
and should be quite helpful during the transition period while
CONFIG_PROVE_RCU-unaware patches are in flight.
The existing rcu_dereference() primitive does no checking, but
upcoming patches will change that.
Signed-off-by: Paul E. McKenney <paulmck@linux.vnet.ibm.com>
Cc: laijs@cn.fujitsu.com
Cc: dipankar@in.ibm.com
Cc: mathieu.desnoyers@polymtl.ca
Cc: josh@joshtriplett.org
Cc: dvhltc@us.ibm.com
Cc: niv@us.ibm.com
Cc: peterz@infradead.org
Cc: rostedt@goodmis.org
Cc: Valdis.Kletnieks@vt.edu
Cc: dhowells@redhat.com
LKML-Reference: <1266887105-1528-1-git-send-email-paulmck@linux.vnet.ibm.com>
Signed-off-by: Ingo Molnar <mingo@elte.hu>
2010-02-23 09:04:45 +08:00
|
|
|
*/
|
|
|
|
#define rcu_dereference_check(p, c) \
|
2010-04-29 05:39:09 +08:00
|
|
|
__rcu_dereference_check((p), rcu_read_lock_held() || (c), __rcu)
|
|
|
|
|
|
|
|
/**
|
|
|
|
* rcu_dereference_bh_check() - rcu_dereference_bh with debug checking
|
|
|
|
* @p: The pointer to read, prior to dereferencing
|
|
|
|
* @c: The conditions under which the dereference will take place
|
|
|
|
*
|
|
|
|
* This is the RCU-bh counterpart to rcu_dereference_check().
|
|
|
|
*/
|
|
|
|
#define rcu_dereference_bh_check(p, c) \
|
|
|
|
__rcu_dereference_check((p), rcu_read_lock_bh_held() || (c), __rcu)
|
rcu: Introduce lockdep-based checking to RCU read-side primitives
Inspection is proving insufficient to catch all RCU misuses,
which is understandable given that rcu_dereference() might be
protected by any of four different flavors of RCU (RCU, RCU-bh,
RCU-sched, and SRCU), and might also/instead be protected by any
of a number of locking primitives. It is therefore time to
enlist the aid of lockdep.
This set of patches is inspired by earlier work by Peter
Zijlstra and Thomas Gleixner, and takes the following approach:
o Set up separate lockdep classes for RCU, RCU-bh, and RCU-sched.
o Set up separate lockdep classes for each instance of SRCU.
o Create primitives that check for being in an RCU read-side
critical section. These return exact answers if lockdep is
fully enabled, but if unsure, report being in an RCU read-side
critical section. (We want to avoid false positives!)
The primitives are:
For RCU: rcu_read_lock_held(void)
For RCU-bh: rcu_read_lock_bh_held(void)
For RCU-sched: rcu_read_lock_sched_held(void)
For SRCU: srcu_read_lock_held(struct srcu_struct *sp)
o Add rcu_dereference_check(), which takes a second argument
in which one places a boolean expression based on the above
primitives and/or lockdep_is_held().
o A new kernel configuration parameter, CONFIG_PROVE_RCU, enables
rcu_dereference_check(). This depends on CONFIG_PROVE_LOCKING,
and should be quite helpful during the transition period while
CONFIG_PROVE_RCU-unaware patches are in flight.
The existing rcu_dereference() primitive does no checking, but
upcoming patches will change that.
Signed-off-by: Paul E. McKenney <paulmck@linux.vnet.ibm.com>
Cc: laijs@cn.fujitsu.com
Cc: dipankar@in.ibm.com
Cc: mathieu.desnoyers@polymtl.ca
Cc: josh@joshtriplett.org
Cc: dvhltc@us.ibm.com
Cc: niv@us.ibm.com
Cc: peterz@infradead.org
Cc: rostedt@goodmis.org
Cc: Valdis.Kletnieks@vt.edu
Cc: dhowells@redhat.com
LKML-Reference: <1266887105-1528-1-git-send-email-paulmck@linux.vnet.ibm.com>
Signed-off-by: Ingo Molnar <mingo@elte.hu>
2010-02-23 09:04:45 +08:00
|
|
|
|
2010-04-10 06:39:10 +08:00
|
|
|
/**
|
2010-04-29 05:39:09 +08:00
|
|
|
* rcu_dereference_sched_check() - rcu_dereference_sched with debug checking
|
|
|
|
* @p: The pointer to read, prior to dereferencing
|
|
|
|
* @c: The conditions under which the dereference will take place
|
|
|
|
*
|
|
|
|
* This is the RCU-sched counterpart to rcu_dereference_check().
|
|
|
|
*/
|
|
|
|
#define rcu_dereference_sched_check(p, c) \
|
|
|
|
__rcu_dereference_check((p), rcu_read_lock_sched_held() || (c), \
|
|
|
|
__rcu)
|
|
|
|
|
|
|
|
#define rcu_dereference_raw(p) rcu_dereference_check(p, 1) /*@@@ needed? @@@*/
|
|
|
|
|
|
|
|
/**
|
|
|
|
* rcu_dereference_index_check() - rcu_dereference for indices with debug checking
|
|
|
|
* @p: The pointer to read, prior to dereferencing
|
|
|
|
* @c: The conditions under which the dereference will take place
|
|
|
|
*
|
|
|
|
* Similar to rcu_dereference_check(), but omits the sparse checking.
|
|
|
|
* This allows rcu_dereference_index_check() to be used on integers,
|
|
|
|
* which can then be used as array indices. Attempting to use
|
|
|
|
* rcu_dereference_check() on an integer will give compiler warnings
|
|
|
|
* because the sparse address-space mechanism relies on dereferencing
|
|
|
|
* the RCU-protected pointer. Dereferencing integers is not something
|
|
|
|
* that even gcc will put up with.
|
|
|
|
*
|
|
|
|
* Note that this function does not implicitly check for RCU read-side
|
|
|
|
* critical sections. If this function gains lots of uses, it might
|
|
|
|
* make sense to provide versions for each flavor of RCU, but it does
|
|
|
|
* not make sense as of early 2010.
|
|
|
|
*/
|
|
|
|
#define rcu_dereference_index_check(p, c) \
|
|
|
|
__rcu_dereference_index_check((p), (c))
|
|
|
|
|
|
|
|
/**
|
|
|
|
* rcu_dereference_protected() - fetch RCU pointer when updates prevented
|
|
|
|
* @p: The pointer to read, prior to dereferencing
|
|
|
|
* @c: The conditions under which the dereference will take place
|
2010-04-10 06:39:10 +08:00
|
|
|
*
|
|
|
|
* Return the value of the specified RCU-protected pointer, but omit
|
|
|
|
* both the smp_read_barrier_depends() and the ACCESS_ONCE(). This
|
|
|
|
* is useful in cases where update-side locks prevent the value of the
|
|
|
|
* pointer from changing. Please note that this primitive does -not-
|
|
|
|
* prevent the compiler from repeating this reference or combining it
|
|
|
|
* with other references, so it should not be used without protection
|
|
|
|
* of appropriate locks.
|
2010-04-29 05:39:09 +08:00
|
|
|
*
|
|
|
|
* This function is only for update-side use. Using this function
|
|
|
|
* when protected only by rcu_read_lock() will result in infrequent
|
|
|
|
* but very ugly failures.
|
2010-04-10 06:39:10 +08:00
|
|
|
*/
|
|
|
|
#define rcu_dereference_protected(p, c) \
|
2010-04-29 05:39:09 +08:00
|
|
|
__rcu_dereference_protected((p), (c), __rcu)
|
2010-04-10 06:39:10 +08:00
|
|
|
|
2010-04-29 05:39:09 +08:00
|
|
|
/**
|
|
|
|
* rcu_dereference_bh_protected() - fetch RCU-bh pointer when updates prevented
|
|
|
|
* @p: The pointer to read, prior to dereferencing
|
|
|
|
* @c: The conditions under which the dereference will take place
|
|
|
|
*
|
|
|
|
* This is the RCU-bh counterpart to rcu_dereference_protected().
|
|
|
|
*/
|
|
|
|
#define rcu_dereference_bh_protected(p, c) \
|
|
|
|
__rcu_dereference_protected((p), (c), __rcu)
|
rcu: Introduce lockdep-based checking to RCU read-side primitives
Inspection is proving insufficient to catch all RCU misuses,
which is understandable given that rcu_dereference() might be
protected by any of four different flavors of RCU (RCU, RCU-bh,
RCU-sched, and SRCU), and might also/instead be protected by any
of a number of locking primitives. It is therefore time to
enlist the aid of lockdep.
This set of patches is inspired by earlier work by Peter
Zijlstra and Thomas Gleixner, and takes the following approach:
o Set up separate lockdep classes for RCU, RCU-bh, and RCU-sched.
o Set up separate lockdep classes for each instance of SRCU.
o Create primitives that check for being in an RCU read-side
critical section. These return exact answers if lockdep is
fully enabled, but if unsure, report being in an RCU read-side
critical section. (We want to avoid false positives!)
The primitives are:
For RCU: rcu_read_lock_held(void)
For RCU-bh: rcu_read_lock_bh_held(void)
For RCU-sched: rcu_read_lock_sched_held(void)
For SRCU: srcu_read_lock_held(struct srcu_struct *sp)
o Add rcu_dereference_check(), which takes a second argument
in which one places a boolean expression based on the above
primitives and/or lockdep_is_held().
o A new kernel configuration parameter, CONFIG_PROVE_RCU, enables
rcu_dereference_check(). This depends on CONFIG_PROVE_LOCKING,
and should be quite helpful during the transition period while
CONFIG_PROVE_RCU-unaware patches are in flight.
The existing rcu_dereference() primitive does no checking, but
upcoming patches will change that.
Signed-off-by: Paul E. McKenney <paulmck@linux.vnet.ibm.com>
Cc: laijs@cn.fujitsu.com
Cc: dipankar@in.ibm.com
Cc: mathieu.desnoyers@polymtl.ca
Cc: josh@joshtriplett.org
Cc: dvhltc@us.ibm.com
Cc: niv@us.ibm.com
Cc: peterz@infradead.org
Cc: rostedt@goodmis.org
Cc: Valdis.Kletnieks@vt.edu
Cc: dhowells@redhat.com
LKML-Reference: <1266887105-1528-1-git-send-email-paulmck@linux.vnet.ibm.com>
Signed-off-by: Ingo Molnar <mingo@elte.hu>
2010-02-23 09:04:45 +08:00
|
|
|
|
2010-04-29 05:39:09 +08:00
|
|
|
/**
|
|
|
|
* rcu_dereference_sched_protected() - fetch RCU-sched pointer when updates prevented
|
|
|
|
* @p: The pointer to read, prior to dereferencing
|
|
|
|
* @c: The conditions under which the dereference will take place
|
|
|
|
*
|
|
|
|
* This is the RCU-sched counterpart to rcu_dereference_protected().
|
|
|
|
*/
|
|
|
|
#define rcu_dereference_sched_protected(p, c) \
|
|
|
|
__rcu_dereference_protected((p), (c), __rcu)
|
rcu: Introduce lockdep-based checking to RCU read-side primitives
Inspection is proving insufficient to catch all RCU misuses,
which is understandable given that rcu_dereference() might be
protected by any of four different flavors of RCU (RCU, RCU-bh,
RCU-sched, and SRCU), and might also/instead be protected by any
of a number of locking primitives. It is therefore time to
enlist the aid of lockdep.
This set of patches is inspired by earlier work by Peter
Zijlstra and Thomas Gleixner, and takes the following approach:
o Set up separate lockdep classes for RCU, RCU-bh, and RCU-sched.
o Set up separate lockdep classes for each instance of SRCU.
o Create primitives that check for being in an RCU read-side
critical section. These return exact answers if lockdep is
fully enabled, but if unsure, report being in an RCU read-side
critical section. (We want to avoid false positives!)
The primitives are:
For RCU: rcu_read_lock_held(void)
For RCU-bh: rcu_read_lock_bh_held(void)
For RCU-sched: rcu_read_lock_sched_held(void)
For SRCU: srcu_read_lock_held(struct srcu_struct *sp)
o Add rcu_dereference_check(), which takes a second argument
in which one places a boolean expression based on the above
primitives and/or lockdep_is_held().
o A new kernel configuration parameter, CONFIG_PROVE_RCU, enables
rcu_dereference_check(). This depends on CONFIG_PROVE_LOCKING,
and should be quite helpful during the transition period while
CONFIG_PROVE_RCU-unaware patches are in flight.
The existing rcu_dereference() primitive does no checking, but
upcoming patches will change that.
Signed-off-by: Paul E. McKenney <paulmck@linux.vnet.ibm.com>
Cc: laijs@cn.fujitsu.com
Cc: dipankar@in.ibm.com
Cc: mathieu.desnoyers@polymtl.ca
Cc: josh@joshtriplett.org
Cc: dvhltc@us.ibm.com
Cc: niv@us.ibm.com
Cc: peterz@infradead.org
Cc: rostedt@goodmis.org
Cc: Valdis.Kletnieks@vt.edu
Cc: dhowells@redhat.com
LKML-Reference: <1266887105-1528-1-git-send-email-paulmck@linux.vnet.ibm.com>
Signed-off-by: Ingo Molnar <mingo@elte.hu>
2010-02-23 09:04:45 +08:00
|
|
|
|
2009-08-23 04:56:47 +08:00
|
|
|
|
2010-04-10 06:39:10 +08:00
|
|
|
/**
|
2010-04-29 05:39:09 +08:00
|
|
|
* rcu_dereference() - fetch RCU-protected pointer for dereferencing
|
|
|
|
* @p: The pointer to read, prior to dereferencing
|
2010-04-10 06:39:10 +08:00
|
|
|
*
|
2010-04-29 05:39:09 +08:00
|
|
|
* This is a simple wrapper around rcu_dereference_check().
|
2010-04-10 06:39:10 +08:00
|
|
|
*/
|
2010-04-29 05:39:09 +08:00
|
|
|
#define rcu_dereference(p) rcu_dereference_check(p, 0)
|
2010-04-10 06:39:10 +08:00
|
|
|
|
2005-04-17 06:20:36 +08:00
|
|
|
/**
|
2010-04-29 05:39:09 +08:00
|
|
|
* rcu_dereference_bh() - fetch an RCU-bh-protected pointer for dereferencing
|
|
|
|
* @p: The pointer to read, prior to dereferencing
|
|
|
|
*
|
|
|
|
* Makes rcu_dereference_check() do the dirty work.
|
|
|
|
*/
|
|
|
|
#define rcu_dereference_bh(p) rcu_dereference_bh_check(p, 0)
|
|
|
|
|
|
|
|
/**
|
|
|
|
* rcu_dereference_sched() - fetch RCU-sched-protected pointer for dereferencing
|
|
|
|
* @p: The pointer to read, prior to dereferencing
|
|
|
|
*
|
|
|
|
* Makes rcu_dereference_check() do the dirty work.
|
|
|
|
*/
|
|
|
|
#define rcu_dereference_sched(p) rcu_dereference_sched_check(p, 0)
|
|
|
|
|
|
|
|
/**
|
|
|
|
* rcu_read_lock() - mark the beginning of an RCU read-side critical section
|
2005-04-17 06:20:36 +08:00
|
|
|
*
|
2005-05-01 23:59:04 +08:00
|
|
|
* When synchronize_rcu() is invoked on one CPU while other CPUs
|
2005-04-17 06:20:36 +08:00
|
|
|
* are within RCU read-side critical sections, then the
|
2005-05-01 23:59:04 +08:00
|
|
|
* synchronize_rcu() is guaranteed to block until after all the other
|
2005-04-17 06:20:36 +08:00
|
|
|
* CPUs exit their critical sections. Similarly, if call_rcu() is invoked
|
|
|
|
* on one CPU while other CPUs are within RCU read-side critical
|
|
|
|
* sections, invocation of the corresponding RCU callback is deferred
|
|
|
|
* until after the all the other CPUs exit their critical sections.
|
|
|
|
*
|
|
|
|
* Note, however, that RCU callbacks are permitted to run concurrently
|
2010-07-09 08:38:59 +08:00
|
|
|
* with new RCU read-side critical sections. One way that this can happen
|
2005-04-17 06:20:36 +08:00
|
|
|
* is via the following sequence of events: (1) CPU 0 enters an RCU
|
|
|
|
* read-side critical section, (2) CPU 1 invokes call_rcu() to register
|
|
|
|
* an RCU callback, (3) CPU 0 exits the RCU read-side critical section,
|
|
|
|
* (4) CPU 2 enters a RCU read-side critical section, (5) the RCU
|
|
|
|
* callback is invoked. This is legal, because the RCU read-side critical
|
|
|
|
* section that was running concurrently with the call_rcu() (and which
|
|
|
|
* therefore might be referencing something that the corresponding RCU
|
|
|
|
* callback would free up) has completed before the corresponding
|
|
|
|
* RCU callback is invoked.
|
|
|
|
*
|
|
|
|
* RCU read-side critical sections may be nested. Any deferred actions
|
|
|
|
* will be deferred until the outermost RCU read-side critical section
|
|
|
|
* completes.
|
|
|
|
*
|
2010-08-08 12:59:54 +08:00
|
|
|
* You can avoid reading and understanding the next paragraph by
|
|
|
|
* following this rule: don't put anything in an rcu_read_lock() RCU
|
|
|
|
* read-side critical section that would block in a !PREEMPT kernel.
|
|
|
|
* But if you want the full story, read on!
|
|
|
|
*
|
|
|
|
* In non-preemptible RCU implementations (TREE_RCU and TINY_RCU), it
|
|
|
|
* is illegal to block while in an RCU read-side critical section. In
|
|
|
|
* preemptible RCU implementations (TREE_PREEMPT_RCU and TINY_PREEMPT_RCU)
|
|
|
|
* in CONFIG_PREEMPT kernel builds, RCU read-side critical sections may
|
|
|
|
* be preempted, but explicit blocking is illegal. Finally, in preemptible
|
|
|
|
* RCU implementations in real-time (CONFIG_PREEMPT_RT) kernel builds,
|
|
|
|
* RCU read-side critical sections may be preempted and they may also
|
|
|
|
* block, but only when acquiring spinlocks that are subject to priority
|
|
|
|
* inheritance.
|
2005-04-17 06:20:36 +08:00
|
|
|
*/
|
2009-08-23 04:56:47 +08:00
|
|
|
static inline void rcu_read_lock(void)
|
|
|
|
{
|
|
|
|
__rcu_read_lock();
|
|
|
|
__acquire(RCU);
|
|
|
|
rcu_read_acquire();
|
|
|
|
}
|
2005-04-17 06:20:36 +08:00
|
|
|
|
|
|
|
/*
|
|
|
|
* So where is rcu_write_lock()? It does not exist, as there is no
|
|
|
|
* way for writers to lock out RCU readers. This is a feature, not
|
|
|
|
* a bug -- this property is what provides RCU's performance benefits.
|
|
|
|
* Of course, writers must coordinate with each other. The normal
|
|
|
|
* spinlock primitives work well for this, but any other technique may be
|
|
|
|
* used as well. RCU does not care how the writers keep out of each
|
|
|
|
* others' way, as long as they do so.
|
|
|
|
*/
|
2009-09-28 22:46:32 +08:00
|
|
|
|
|
|
|
/**
|
2010-04-29 05:39:09 +08:00
|
|
|
* rcu_read_unlock() - marks the end of an RCU read-side critical section.
|
2009-09-28 22:46:32 +08:00
|
|
|
*
|
|
|
|
* See rcu_read_lock() for more information.
|
|
|
|
*/
|
2009-08-23 04:56:47 +08:00
|
|
|
static inline void rcu_read_unlock(void)
|
|
|
|
{
|
|
|
|
rcu_read_release();
|
|
|
|
__release(RCU);
|
|
|
|
__rcu_read_unlock();
|
|
|
|
}
|
2005-04-17 06:20:36 +08:00
|
|
|
|
|
|
|
/**
|
2010-04-29 05:39:09 +08:00
|
|
|
* rcu_read_lock_bh() - mark the beginning of an RCU-bh critical section
|
2005-04-17 06:20:36 +08:00
|
|
|
*
|
|
|
|
* This is equivalent of rcu_read_lock(), but to be used when updates
|
2010-04-29 05:39:09 +08:00
|
|
|
* are being done using call_rcu_bh() or synchronize_rcu_bh(). Since
|
|
|
|
* both call_rcu_bh() and synchronize_rcu_bh() consider completion of a
|
|
|
|
* softirq handler to be a quiescent state, a process in RCU read-side
|
|
|
|
* critical section must be protected by disabling softirqs. Read-side
|
|
|
|
* critical sections in interrupt context can use just rcu_read_lock(),
|
|
|
|
* though this should at least be commented to avoid confusing people
|
|
|
|
* reading the code.
|
2005-04-17 06:20:36 +08:00
|
|
|
*/
|
2009-08-23 04:56:47 +08:00
|
|
|
static inline void rcu_read_lock_bh(void)
|
|
|
|
{
|
|
|
|
__rcu_read_lock_bh();
|
|
|
|
__acquire(RCU_BH);
|
rcu: Introduce lockdep-based checking to RCU read-side primitives
Inspection is proving insufficient to catch all RCU misuses,
which is understandable given that rcu_dereference() might be
protected by any of four different flavors of RCU (RCU, RCU-bh,
RCU-sched, and SRCU), and might also/instead be protected by any
of a number of locking primitives. It is therefore time to
enlist the aid of lockdep.
This set of patches is inspired by earlier work by Peter
Zijlstra and Thomas Gleixner, and takes the following approach:
o Set up separate lockdep classes for RCU, RCU-bh, and RCU-sched.
o Set up separate lockdep classes for each instance of SRCU.
o Create primitives that check for being in an RCU read-side
critical section. These return exact answers if lockdep is
fully enabled, but if unsure, report being in an RCU read-side
critical section. (We want to avoid false positives!)
The primitives are:
For RCU: rcu_read_lock_held(void)
For RCU-bh: rcu_read_lock_bh_held(void)
For RCU-sched: rcu_read_lock_sched_held(void)
For SRCU: srcu_read_lock_held(struct srcu_struct *sp)
o Add rcu_dereference_check(), which takes a second argument
in which one places a boolean expression based on the above
primitives and/or lockdep_is_held().
o A new kernel configuration parameter, CONFIG_PROVE_RCU, enables
rcu_dereference_check(). This depends on CONFIG_PROVE_LOCKING,
and should be quite helpful during the transition period while
CONFIG_PROVE_RCU-unaware patches are in flight.
The existing rcu_dereference() primitive does no checking, but
upcoming patches will change that.
Signed-off-by: Paul E. McKenney <paulmck@linux.vnet.ibm.com>
Cc: laijs@cn.fujitsu.com
Cc: dipankar@in.ibm.com
Cc: mathieu.desnoyers@polymtl.ca
Cc: josh@joshtriplett.org
Cc: dvhltc@us.ibm.com
Cc: niv@us.ibm.com
Cc: peterz@infradead.org
Cc: rostedt@goodmis.org
Cc: Valdis.Kletnieks@vt.edu
Cc: dhowells@redhat.com
LKML-Reference: <1266887105-1528-1-git-send-email-paulmck@linux.vnet.ibm.com>
Signed-off-by: Ingo Molnar <mingo@elte.hu>
2010-02-23 09:04:45 +08:00
|
|
|
rcu_read_acquire_bh();
|
2009-08-23 04:56:47 +08:00
|
|
|
}
|
2005-04-17 06:20:36 +08:00
|
|
|
|
|
|
|
/*
|
|
|
|
* rcu_read_unlock_bh - marks the end of a softirq-only RCU critical section
|
|
|
|
*
|
|
|
|
* See rcu_read_lock_bh() for more information.
|
|
|
|
*/
|
2009-08-23 04:56:47 +08:00
|
|
|
static inline void rcu_read_unlock_bh(void)
|
|
|
|
{
|
rcu: Introduce lockdep-based checking to RCU read-side primitives
Inspection is proving insufficient to catch all RCU misuses,
which is understandable given that rcu_dereference() might be
protected by any of four different flavors of RCU (RCU, RCU-bh,
RCU-sched, and SRCU), and might also/instead be protected by any
of a number of locking primitives. It is therefore time to
enlist the aid of lockdep.
This set of patches is inspired by earlier work by Peter
Zijlstra and Thomas Gleixner, and takes the following approach:
o Set up separate lockdep classes for RCU, RCU-bh, and RCU-sched.
o Set up separate lockdep classes for each instance of SRCU.
o Create primitives that check for being in an RCU read-side
critical section. These return exact answers if lockdep is
fully enabled, but if unsure, report being in an RCU read-side
critical section. (We want to avoid false positives!)
The primitives are:
For RCU: rcu_read_lock_held(void)
For RCU-bh: rcu_read_lock_bh_held(void)
For RCU-sched: rcu_read_lock_sched_held(void)
For SRCU: srcu_read_lock_held(struct srcu_struct *sp)
o Add rcu_dereference_check(), which takes a second argument
in which one places a boolean expression based on the above
primitives and/or lockdep_is_held().
o A new kernel configuration parameter, CONFIG_PROVE_RCU, enables
rcu_dereference_check(). This depends on CONFIG_PROVE_LOCKING,
and should be quite helpful during the transition period while
CONFIG_PROVE_RCU-unaware patches are in flight.
The existing rcu_dereference() primitive does no checking, but
upcoming patches will change that.
Signed-off-by: Paul E. McKenney <paulmck@linux.vnet.ibm.com>
Cc: laijs@cn.fujitsu.com
Cc: dipankar@in.ibm.com
Cc: mathieu.desnoyers@polymtl.ca
Cc: josh@joshtriplett.org
Cc: dvhltc@us.ibm.com
Cc: niv@us.ibm.com
Cc: peterz@infradead.org
Cc: rostedt@goodmis.org
Cc: Valdis.Kletnieks@vt.edu
Cc: dhowells@redhat.com
LKML-Reference: <1266887105-1528-1-git-send-email-paulmck@linux.vnet.ibm.com>
Signed-off-by: Ingo Molnar <mingo@elte.hu>
2010-02-23 09:04:45 +08:00
|
|
|
rcu_read_release_bh();
|
2009-08-23 04:56:47 +08:00
|
|
|
__release(RCU_BH);
|
|
|
|
__rcu_read_unlock_bh();
|
|
|
|
}
|
2005-04-17 06:20:36 +08:00
|
|
|
|
2008-09-29 23:06:46 +08:00
|
|
|
/**
|
2010-04-29 05:39:09 +08:00
|
|
|
* rcu_read_lock_sched() - mark the beginning of a RCU-sched critical section
|
2008-09-29 23:06:46 +08:00
|
|
|
*
|
2010-04-29 05:39:09 +08:00
|
|
|
* This is equivalent of rcu_read_lock(), but to be used when updates
|
|
|
|
* are being done using call_rcu_sched() or synchronize_rcu_sched().
|
|
|
|
* Read-side critical sections can also be introduced by anything that
|
|
|
|
* disables preemption, including local_irq_disable() and friends.
|
2008-09-29 23:06:46 +08:00
|
|
|
*/
|
2009-08-23 04:56:46 +08:00
|
|
|
static inline void rcu_read_lock_sched(void)
|
|
|
|
{
|
|
|
|
preempt_disable();
|
2009-08-23 04:56:47 +08:00
|
|
|
__acquire(RCU_SCHED);
|
rcu: Introduce lockdep-based checking to RCU read-side primitives
Inspection is proving insufficient to catch all RCU misuses,
which is understandable given that rcu_dereference() might be
protected by any of four different flavors of RCU (RCU, RCU-bh,
RCU-sched, and SRCU), and might also/instead be protected by any
of a number of locking primitives. It is therefore time to
enlist the aid of lockdep.
This set of patches is inspired by earlier work by Peter
Zijlstra and Thomas Gleixner, and takes the following approach:
o Set up separate lockdep classes for RCU, RCU-bh, and RCU-sched.
o Set up separate lockdep classes for each instance of SRCU.
o Create primitives that check for being in an RCU read-side
critical section. These return exact answers if lockdep is
fully enabled, but if unsure, report being in an RCU read-side
critical section. (We want to avoid false positives!)
The primitives are:
For RCU: rcu_read_lock_held(void)
For RCU-bh: rcu_read_lock_bh_held(void)
For RCU-sched: rcu_read_lock_sched_held(void)
For SRCU: srcu_read_lock_held(struct srcu_struct *sp)
o Add rcu_dereference_check(), which takes a second argument
in which one places a boolean expression based on the above
primitives and/or lockdep_is_held().
o A new kernel configuration parameter, CONFIG_PROVE_RCU, enables
rcu_dereference_check(). This depends on CONFIG_PROVE_LOCKING,
and should be quite helpful during the transition period while
CONFIG_PROVE_RCU-unaware patches are in flight.
The existing rcu_dereference() primitive does no checking, but
upcoming patches will change that.
Signed-off-by: Paul E. McKenney <paulmck@linux.vnet.ibm.com>
Cc: laijs@cn.fujitsu.com
Cc: dipankar@in.ibm.com
Cc: mathieu.desnoyers@polymtl.ca
Cc: josh@joshtriplett.org
Cc: dvhltc@us.ibm.com
Cc: niv@us.ibm.com
Cc: peterz@infradead.org
Cc: rostedt@goodmis.org
Cc: Valdis.Kletnieks@vt.edu
Cc: dhowells@redhat.com
LKML-Reference: <1266887105-1528-1-git-send-email-paulmck@linux.vnet.ibm.com>
Signed-off-by: Ingo Molnar <mingo@elte.hu>
2010-02-23 09:04:45 +08:00
|
|
|
rcu_read_acquire_sched();
|
2009-08-23 04:56:46 +08:00
|
|
|
}
|
2009-09-24 00:50:42 +08:00
|
|
|
|
|
|
|
/* Used by lockdep and tracing: cannot be traced, cannot call lockdep. */
|
2009-08-25 00:42:00 +08:00
|
|
|
static inline notrace void rcu_read_lock_sched_notrace(void)
|
2009-08-23 04:56:46 +08:00
|
|
|
{
|
|
|
|
preempt_disable_notrace();
|
2009-08-23 04:56:47 +08:00
|
|
|
__acquire(RCU_SCHED);
|
2009-08-23 04:56:46 +08:00
|
|
|
}
|
2008-09-29 23:06:46 +08:00
|
|
|
|
|
|
|
/*
|
|
|
|
* rcu_read_unlock_sched - marks the end of a RCU-classic critical section
|
|
|
|
*
|
|
|
|
* See rcu_read_lock_sched for more information.
|
|
|
|
*/
|
2009-08-23 04:56:46 +08:00
|
|
|
static inline void rcu_read_unlock_sched(void)
|
|
|
|
{
|
rcu: Introduce lockdep-based checking to RCU read-side primitives
Inspection is proving insufficient to catch all RCU misuses,
which is understandable given that rcu_dereference() might be
protected by any of four different flavors of RCU (RCU, RCU-bh,
RCU-sched, and SRCU), and might also/instead be protected by any
of a number of locking primitives. It is therefore time to
enlist the aid of lockdep.
This set of patches is inspired by earlier work by Peter
Zijlstra and Thomas Gleixner, and takes the following approach:
o Set up separate lockdep classes for RCU, RCU-bh, and RCU-sched.
o Set up separate lockdep classes for each instance of SRCU.
o Create primitives that check for being in an RCU read-side
critical section. These return exact answers if lockdep is
fully enabled, but if unsure, report being in an RCU read-side
critical section. (We want to avoid false positives!)
The primitives are:
For RCU: rcu_read_lock_held(void)
For RCU-bh: rcu_read_lock_bh_held(void)
For RCU-sched: rcu_read_lock_sched_held(void)
For SRCU: srcu_read_lock_held(struct srcu_struct *sp)
o Add rcu_dereference_check(), which takes a second argument
in which one places a boolean expression based on the above
primitives and/or lockdep_is_held().
o A new kernel configuration parameter, CONFIG_PROVE_RCU, enables
rcu_dereference_check(). This depends on CONFIG_PROVE_LOCKING,
and should be quite helpful during the transition period while
CONFIG_PROVE_RCU-unaware patches are in flight.
The existing rcu_dereference() primitive does no checking, but
upcoming patches will change that.
Signed-off-by: Paul E. McKenney <paulmck@linux.vnet.ibm.com>
Cc: laijs@cn.fujitsu.com
Cc: dipankar@in.ibm.com
Cc: mathieu.desnoyers@polymtl.ca
Cc: josh@joshtriplett.org
Cc: dvhltc@us.ibm.com
Cc: niv@us.ibm.com
Cc: peterz@infradead.org
Cc: rostedt@goodmis.org
Cc: Valdis.Kletnieks@vt.edu
Cc: dhowells@redhat.com
LKML-Reference: <1266887105-1528-1-git-send-email-paulmck@linux.vnet.ibm.com>
Signed-off-by: Ingo Molnar <mingo@elte.hu>
2010-02-23 09:04:45 +08:00
|
|
|
rcu_read_release_sched();
|
2009-08-23 04:56:47 +08:00
|
|
|
__release(RCU_SCHED);
|
2009-08-23 04:56:46 +08:00
|
|
|
preempt_enable();
|
|
|
|
}
|
2009-09-24 00:50:42 +08:00
|
|
|
|
|
|
|
/* Used by lockdep and tracing: cannot be traced, cannot call lockdep. */
|
2009-08-25 00:42:00 +08:00
|
|
|
static inline notrace void rcu_read_unlock_sched_notrace(void)
|
2009-08-23 04:56:46 +08:00
|
|
|
{
|
2009-08-23 04:56:47 +08:00
|
|
|
__release(RCU_SCHED);
|
2009-08-23 04:56:46 +08:00
|
|
|
preempt_enable_notrace();
|
|
|
|
}
|
2008-09-29 23:06:46 +08:00
|
|
|
|
2005-04-17 06:20:36 +08:00
|
|
|
/**
|
2010-04-29 05:39:09 +08:00
|
|
|
* rcu_assign_pointer() - assign to RCU-protected pointer
|
|
|
|
* @p: pointer to assign to
|
|
|
|
* @v: value to assign (publish)
|
2010-02-23 09:04:46 +08:00
|
|
|
*
|
2010-04-29 05:39:09 +08:00
|
|
|
* Assigns the specified value to the specified RCU-protected
|
|
|
|
* pointer, ensuring that any concurrent RCU readers will see
|
|
|
|
* any prior initialization. Returns the value assigned.
|
2005-04-17 06:20:36 +08:00
|
|
|
*
|
|
|
|
* Inserts memory barriers on architectures that require them
|
|
|
|
* (pretty much all of them other than x86), and also prevents
|
|
|
|
* the compiler from reordering the code that initializes the
|
|
|
|
* structure after the pointer assignment. More importantly, this
|
|
|
|
* call documents which pointers will be dereferenced by RCU read-side
|
|
|
|
* code.
|
|
|
|
*/
|
2008-02-06 17:37:25 +08:00
|
|
|
#define rcu_assign_pointer(p, v) \
|
2010-04-29 05:39:09 +08:00
|
|
|
__rcu_assign_pointer((p), (v), __rcu)
|
|
|
|
|
|
|
|
/**
|
|
|
|
* RCU_INIT_POINTER() - initialize an RCU protected pointer
|
|
|
|
*
|
|
|
|
* Initialize an RCU-protected pointer in such a way to avoid RCU-lockdep
|
|
|
|
* splats.
|
|
|
|
*/
|
|
|
|
#define RCU_INIT_POINTER(p, v) \
|
|
|
|
p = (typeof(*v) __force __rcu *)(v)
|
2005-04-17 06:20:36 +08:00
|
|
|
|
rcu: add call_rcu_sched()
Fourth cut of patch to provide the call_rcu_sched(). This is again to
synchronize_sched() as call_rcu() is to synchronize_rcu().
Should be fine for experimental and -rt use, but not ready for inclusion.
With some luck, I will be able to tell Andrew to come out of hiding on
the next round.
Passes multi-day rcutorture sessions with concurrent CPU hotplugging.
Fixes since the first version include a bug that could result in
indefinite blocking (spotted by Gautham Shenoy), better resiliency
against CPU-hotplug operations, and other minor fixes.
Fixes since the second version include reworking grace-period detection
to avoid deadlocks that could happen when running concurrently with
CPU hotplug, adding Mathieu's fix to avoid the softlockup messages,
as well as Mathieu's fix to allow use earlier in boot.
Fixes since the third version include a wrong-CPU bug spotted by
Andrew, getting rid of the obsolete synchronize_kernel API that somehow
snuck back in, merging spin_unlock() and local_irq_restore() in a
few places, commenting the code that checks for quiescent states based
on interrupting from user-mode execution or the idle loop, removing
some inline attributes, and some code-style changes.
Known/suspected shortcomings:
o I still do not entirely trust the sleep/wakeup logic. Next step
will be to use a private snapshot of the CPU online mask in
rcu_sched_grace_period() -- if the CPU wasn't there at the start
of the grace period, we don't need to hear from it. And the
bit about accounting for changes in online CPUs inside of
rcu_sched_grace_period() is ugly anyway.
o It might be good for rcu_sched_grace_period() to invoke
resched_cpu() when a given CPU wasn't responding quickly,
but resched_cpu() is declared static...
This patch also fixes a long-standing bug in the earlier preemptable-RCU
implementation of synchronize_rcu() that could result in loss of
concurrent external changes to a task's CPU affinity mask. I still cannot
remember who reported this...
Signed-off-by: Paul E. McKenney <paulmck@linux.vnet.ibm.com>
Signed-off-by: Mathieu Desnoyers <mathieu.desnoyers@polymtl.ca>
Signed-off-by: Ingo Molnar <mingo@elte.hu>
Signed-off-by: Thomas Gleixner <tglx@linutronix.de>
2008-05-13 03:21:05 +08:00
|
|
|
/* Infrastructure to implement the synchronize_() primitives. */
|
|
|
|
|
|
|
|
struct rcu_synchronize {
|
|
|
|
struct rcu_head head;
|
|
|
|
struct completion completion;
|
|
|
|
};
|
|
|
|
|
|
|
|
extern void wakeme_after_rcu(struct rcu_head *head);
|
|
|
|
|
2008-01-26 04:08:24 +08:00
|
|
|
/**
|
2010-04-29 05:39:09 +08:00
|
|
|
* call_rcu() - Queue an RCU callback for invocation after a grace period.
|
2008-01-26 04:08:24 +08:00
|
|
|
* @head: structure to be used for queueing the RCU updates.
|
2010-07-09 08:38:59 +08:00
|
|
|
* @func: actual callback function to be invoked after the grace period
|
2008-01-26 04:08:24 +08:00
|
|
|
*
|
2010-07-09 08:38:59 +08:00
|
|
|
* The callback function will be invoked some time after a full grace
|
|
|
|
* period elapses, in other words after all pre-existing RCU read-side
|
|
|
|
* critical sections have completed. However, the callback function
|
|
|
|
* might well execute concurrently with RCU read-side critical sections
|
|
|
|
* that started after call_rcu() was invoked. RCU read-side critical
|
2008-01-26 04:08:24 +08:00
|
|
|
* sections are delimited by rcu_read_lock() and rcu_read_unlock(),
|
|
|
|
* and may be nested.
|
|
|
|
*/
|
|
|
|
extern void call_rcu(struct rcu_head *head,
|
|
|
|
void (*func)(struct rcu_head *head));
|
|
|
|
|
|
|
|
/**
|
2010-04-29 05:39:09 +08:00
|
|
|
* call_rcu_bh() - Queue an RCU for invocation after a quicker grace period.
|
2008-01-26 04:08:24 +08:00
|
|
|
* @head: structure to be used for queueing the RCU updates.
|
2010-07-09 08:38:59 +08:00
|
|
|
* @func: actual callback function to be invoked after the grace period
|
2008-01-26 04:08:24 +08:00
|
|
|
*
|
2010-07-09 08:38:59 +08:00
|
|
|
* The callback function will be invoked some time after a full grace
|
2008-01-26 04:08:24 +08:00
|
|
|
* period elapses, in other words after all currently executing RCU
|
|
|
|
* read-side critical sections have completed. call_rcu_bh() assumes
|
|
|
|
* that the read-side critical sections end on completion of a softirq
|
|
|
|
* handler. This means that read-side critical sections in process
|
|
|
|
* context must not be interrupted by softirqs. This interface is to be
|
|
|
|
* used when most of the read-side critical sections are in softirq context.
|
|
|
|
* RCU read-side critical sections are delimited by :
|
|
|
|
* - rcu_read_lock() and rcu_read_unlock(), if in interrupt context.
|
|
|
|
* OR
|
|
|
|
* - rcu_read_lock_bh() and rcu_read_unlock_bh(), if in process context.
|
|
|
|
* These may be nested.
|
|
|
|
*/
|
|
|
|
extern void call_rcu_bh(struct rcu_head *head,
|
|
|
|
void (*func)(struct rcu_head *head));
|
|
|
|
|
2010-04-17 20:48:42 +08:00
|
|
|
/*
|
|
|
|
* debug_rcu_head_queue()/debug_rcu_head_unqueue() are used internally
|
|
|
|
* by call_rcu() and rcu callback execution, and are therefore not part of the
|
|
|
|
* RCU API. Leaving in rcupdate.h because they are used by all RCU flavors.
|
|
|
|
*/
|
|
|
|
|
|
|
|
#ifdef CONFIG_DEBUG_OBJECTS_RCU_HEAD
|
|
|
|
# define STATE_RCU_HEAD_READY 0
|
|
|
|
# define STATE_RCU_HEAD_QUEUED 1
|
|
|
|
|
|
|
|
extern struct debug_obj_descr rcuhead_debug_descr;
|
|
|
|
|
|
|
|
static inline void debug_rcu_head_queue(struct rcu_head *head)
|
|
|
|
{
|
|
|
|
debug_object_activate(head, &rcuhead_debug_descr);
|
|
|
|
debug_object_active_state(head, &rcuhead_debug_descr,
|
|
|
|
STATE_RCU_HEAD_READY,
|
|
|
|
STATE_RCU_HEAD_QUEUED);
|
|
|
|
}
|
|
|
|
|
|
|
|
static inline void debug_rcu_head_unqueue(struct rcu_head *head)
|
|
|
|
{
|
|
|
|
debug_object_active_state(head, &rcuhead_debug_descr,
|
|
|
|
STATE_RCU_HEAD_QUEUED,
|
|
|
|
STATE_RCU_HEAD_READY);
|
|
|
|
debug_object_deactivate(head, &rcuhead_debug_descr);
|
|
|
|
}
|
|
|
|
#else /* !CONFIG_DEBUG_OBJECTS_RCU_HEAD */
|
|
|
|
static inline void debug_rcu_head_queue(struct rcu_head *head)
|
|
|
|
{
|
|
|
|
}
|
|
|
|
|
|
|
|
static inline void debug_rcu_head_unqueue(struct rcu_head *head)
|
|
|
|
{
|
|
|
|
}
|
|
|
|
#endif /* #else !CONFIG_DEBUG_OBJECTS_RCU_HEAD */
|
|
|
|
|
2005-04-17 06:20:36 +08:00
|
|
|
#endif /* __LINUX_RCUPDATE_H */
|