mirror of
https://github.com/edk2-porting/linux-next.git
synced 2024-12-23 20:53:53 +08:00
0f1ac8fd25
When function tracing with either debug locks enabled or tracing
preempt disabled, the add_preempt_count() is traced. This is an
issue with lockdep and function tracing. As function tracing
can disable interrupts, and lockdep records that change,
lockdep may not be able to handle this recursion if it happens from
an NMI context.
The first thing that an NMI does is:
#define nmi_enter() \
do { \
ftrace_nmi_enter(); \
BUG_ON(in_nmi()); \
add_preempt_count(NMI_OFFSET + HARDIRQ_OFFSET); \
lockdep_off(); \
rcu_nmi_enter(); \
trace_hardirq_enter(); \
} while (0)
When the add_preempt_count() is traced, and the tracing callback
disables interrupts, it will jump into the lockdep code. There's
some places in lockdep that can't handle this re-entrance, and
causes lockdep to fail.
As the lockdep_off() (and lockdep_on) is a simple:
void lockdep_off(void)
{
current->lockdep_recursion++;
}
and is never traced, it can be called first in nmi_enter()
and lockdep_on() last in nmi_exit().
Cc: Peter Zijlstra <a.p.zijlstra@chello.nl>
Signed-off-by: Steven Rostedt <rostedt@goodmis.org>
202 lines
5.5 KiB
C
202 lines
5.5 KiB
C
#ifndef LINUX_HARDIRQ_H
|
|
#define LINUX_HARDIRQ_H
|
|
|
|
#include <linux/preempt.h>
|
|
#include <linux/lockdep.h>
|
|
#include <linux/ftrace_irq.h>
|
|
#include <linux/vtime.h>
|
|
#include <asm/hardirq.h>
|
|
|
|
/*
|
|
* We put the hardirq and softirq counter into the preemption
|
|
* counter. The bitmask has the following meaning:
|
|
*
|
|
* - bits 0-7 are the preemption count (max preemption depth: 256)
|
|
* - bits 8-15 are the softirq count (max # of softirqs: 256)
|
|
*
|
|
* The hardirq count can in theory reach the same as NR_IRQS.
|
|
* In reality, the number of nested IRQS is limited to the stack
|
|
* size as well. For archs with over 1000 IRQS it is not practical
|
|
* to expect that they will all nest. We give a max of 10 bits for
|
|
* hardirq nesting. An arch may choose to give less than 10 bits.
|
|
* m68k expects it to be 8.
|
|
*
|
|
* - bits 16-25 are the hardirq count (max # of nested hardirqs: 1024)
|
|
* - bit 26 is the NMI_MASK
|
|
* - bit 27 is the PREEMPT_ACTIVE flag
|
|
*
|
|
* PREEMPT_MASK: 0x000000ff
|
|
* SOFTIRQ_MASK: 0x0000ff00
|
|
* HARDIRQ_MASK: 0x03ff0000
|
|
* NMI_MASK: 0x04000000
|
|
*/
|
|
#define PREEMPT_BITS 8
|
|
#define SOFTIRQ_BITS 8
|
|
#define NMI_BITS 1
|
|
|
|
#define MAX_HARDIRQ_BITS 10
|
|
|
|
#ifndef HARDIRQ_BITS
|
|
# define HARDIRQ_BITS MAX_HARDIRQ_BITS
|
|
#endif
|
|
|
|
#if HARDIRQ_BITS > MAX_HARDIRQ_BITS
|
|
#error HARDIRQ_BITS too high!
|
|
#endif
|
|
|
|
#define PREEMPT_SHIFT 0
|
|
#define SOFTIRQ_SHIFT (PREEMPT_SHIFT + PREEMPT_BITS)
|
|
#define HARDIRQ_SHIFT (SOFTIRQ_SHIFT + SOFTIRQ_BITS)
|
|
#define NMI_SHIFT (HARDIRQ_SHIFT + HARDIRQ_BITS)
|
|
|
|
#define __IRQ_MASK(x) ((1UL << (x))-1)
|
|
|
|
#define PREEMPT_MASK (__IRQ_MASK(PREEMPT_BITS) << PREEMPT_SHIFT)
|
|
#define SOFTIRQ_MASK (__IRQ_MASK(SOFTIRQ_BITS) << SOFTIRQ_SHIFT)
|
|
#define HARDIRQ_MASK (__IRQ_MASK(HARDIRQ_BITS) << HARDIRQ_SHIFT)
|
|
#define NMI_MASK (__IRQ_MASK(NMI_BITS) << NMI_SHIFT)
|
|
|
|
#define PREEMPT_OFFSET (1UL << PREEMPT_SHIFT)
|
|
#define SOFTIRQ_OFFSET (1UL << SOFTIRQ_SHIFT)
|
|
#define HARDIRQ_OFFSET (1UL << HARDIRQ_SHIFT)
|
|
#define NMI_OFFSET (1UL << NMI_SHIFT)
|
|
|
|
#define SOFTIRQ_DISABLE_OFFSET (2 * SOFTIRQ_OFFSET)
|
|
|
|
#ifndef PREEMPT_ACTIVE
|
|
#define PREEMPT_ACTIVE_BITS 1
|
|
#define PREEMPT_ACTIVE_SHIFT (NMI_SHIFT + NMI_BITS)
|
|
#define PREEMPT_ACTIVE (__IRQ_MASK(PREEMPT_ACTIVE_BITS) << PREEMPT_ACTIVE_SHIFT)
|
|
#endif
|
|
|
|
#if PREEMPT_ACTIVE < (1 << (NMI_SHIFT + NMI_BITS))
|
|
#error PREEMPT_ACTIVE is too low!
|
|
#endif
|
|
|
|
#define hardirq_count() (preempt_count() & HARDIRQ_MASK)
|
|
#define softirq_count() (preempt_count() & SOFTIRQ_MASK)
|
|
#define irq_count() (preempt_count() & (HARDIRQ_MASK | SOFTIRQ_MASK \
|
|
| NMI_MASK))
|
|
|
|
/*
|
|
* Are we doing bottom half or hardware interrupt processing?
|
|
* Are we in a softirq context? Interrupt context?
|
|
* in_softirq - Are we currently processing softirq or have bh disabled?
|
|
* in_serving_softirq - Are we currently processing softirq?
|
|
*/
|
|
#define in_irq() (hardirq_count())
|
|
#define in_softirq() (softirq_count())
|
|
#define in_interrupt() (irq_count())
|
|
#define in_serving_softirq() (softirq_count() & SOFTIRQ_OFFSET)
|
|
|
|
/*
|
|
* Are we in NMI context?
|
|
*/
|
|
#define in_nmi() (preempt_count() & NMI_MASK)
|
|
|
|
#if defined(CONFIG_PREEMPT_COUNT)
|
|
# define PREEMPT_CHECK_OFFSET 1
|
|
#else
|
|
# define PREEMPT_CHECK_OFFSET 0
|
|
#endif
|
|
|
|
/*
|
|
* Are we running in atomic context? WARNING: this macro cannot
|
|
* always detect atomic context; in particular, it cannot know about
|
|
* held spinlocks in non-preemptible kernels. Thus it should not be
|
|
* used in the general case to determine whether sleeping is possible.
|
|
* Do not use in_atomic() in driver code.
|
|
*/
|
|
#define in_atomic() ((preempt_count() & ~PREEMPT_ACTIVE) != 0)
|
|
|
|
/*
|
|
* Check whether we were atomic before we did preempt_disable():
|
|
* (used by the scheduler, *after* releasing the kernel lock)
|
|
*/
|
|
#define in_atomic_preempt_off() \
|
|
((preempt_count() & ~PREEMPT_ACTIVE) != PREEMPT_CHECK_OFFSET)
|
|
|
|
#ifdef CONFIG_PREEMPT_COUNT
|
|
# define preemptible() (preempt_count() == 0 && !irqs_disabled())
|
|
# define IRQ_EXIT_OFFSET (HARDIRQ_OFFSET-1)
|
|
#else
|
|
# define preemptible() 0
|
|
# define IRQ_EXIT_OFFSET HARDIRQ_OFFSET
|
|
#endif
|
|
|
|
#if defined(CONFIG_SMP) || defined(CONFIG_GENERIC_HARDIRQS)
|
|
extern void synchronize_irq(unsigned int irq);
|
|
#else
|
|
# define synchronize_irq(irq) barrier()
|
|
#endif
|
|
|
|
#if defined(CONFIG_TINY_RCU) || defined(CONFIG_TINY_PREEMPT_RCU)
|
|
|
|
static inline void rcu_nmi_enter(void)
|
|
{
|
|
}
|
|
|
|
static inline void rcu_nmi_exit(void)
|
|
{
|
|
}
|
|
|
|
#else
|
|
extern void rcu_nmi_enter(void);
|
|
extern void rcu_nmi_exit(void);
|
|
#endif
|
|
|
|
/*
|
|
* It is safe to do non-atomic ops on ->hardirq_context,
|
|
* because NMI handlers may not preempt and the ops are
|
|
* always balanced, so the interrupted value of ->hardirq_context
|
|
* will always be restored.
|
|
*/
|
|
#define __irq_enter() \
|
|
do { \
|
|
vtime_account_irq_enter(current); \
|
|
add_preempt_count(HARDIRQ_OFFSET); \
|
|
trace_hardirq_enter(); \
|
|
} while (0)
|
|
|
|
/*
|
|
* Enter irq context (on NO_HZ, update jiffies):
|
|
*/
|
|
extern void irq_enter(void);
|
|
|
|
/*
|
|
* Exit irq context without processing softirqs:
|
|
*/
|
|
#define __irq_exit() \
|
|
do { \
|
|
trace_hardirq_exit(); \
|
|
vtime_account_irq_exit(current); \
|
|
sub_preempt_count(HARDIRQ_OFFSET); \
|
|
} while (0)
|
|
|
|
/*
|
|
* Exit irq context and process softirqs if needed:
|
|
*/
|
|
extern void irq_exit(void);
|
|
|
|
#define nmi_enter() \
|
|
do { \
|
|
lockdep_off(); \
|
|
ftrace_nmi_enter(); \
|
|
BUG_ON(in_nmi()); \
|
|
add_preempt_count(NMI_OFFSET + HARDIRQ_OFFSET); \
|
|
rcu_nmi_enter(); \
|
|
trace_hardirq_enter(); \
|
|
} while (0)
|
|
|
|
#define nmi_exit() \
|
|
do { \
|
|
trace_hardirq_exit(); \
|
|
rcu_nmi_exit(); \
|
|
BUG_ON(!in_nmi()); \
|
|
sub_preempt_count(NMI_OFFSET + HARDIRQ_OFFSET); \
|
|
ftrace_nmi_exit(); \
|
|
lockdep_on(); \
|
|
} while (0)
|
|
|
|
#endif /* LINUX_HARDIRQ_H */
|