mirror of
https://mirrors.bfsu.edu.cn/git/linux.git
synced 2024-12-17 07:54:54 +08:00
707e59ba49
The following commit:
1fb3a8b2cf
("xen/spinlock: Fix locking path engaging too soon under PVHVM.")
... moved the initalization of the kicker interrupt until after
native_cpu_up() is called.
However, when using qspinlocks, a CPU may try to kick another CPU that is
spinning (because it has not yet initialized its kicker interrupt), resulting
in the following crash during boot:
kernel BUG at /build/linux-Ay7j_C/linux-4.4.0/drivers/xen/events/events_base.c:1210!
invalid opcode: 0000 [#1] SMP
...
RIP: 0010:[<ffffffff814c97c9>] [<ffffffff814c97c9>] xen_send_IPI_one+0x59/0x60
...
Call Trace:
[<ffffffff8102be9e>] xen_qlock_kick+0xe/0x10
[<ffffffff810cabc2>] __pv_queued_spin_unlock+0xb2/0xf0
[<ffffffff810ca6d1>] ? __raw_callee_save___pv_queued_spin_unlock+0x11/0x20
[<ffffffff81052936>] ? check_tsc_warp+0x76/0x150
[<ffffffff81052aa6>] check_tsc_sync_source+0x96/0x160
[<ffffffff81051e28>] native_cpu_up+0x3d8/0x9f0
[<ffffffff8102b315>] xen_hvm_cpu_up+0x35/0x80
[<ffffffff8108198c>] _cpu_up+0x13c/0x180
[<ffffffff81081a4a>] cpu_up+0x7a/0xa0
[<ffffffff81f80dfc>] smp_init+0x7f/0x81
[<ffffffff81f5a121>] kernel_init_freeable+0xef/0x212
[<ffffffff81817f30>] ? rest_init+0x80/0x80
[<ffffffff81817f3e>] kernel_init+0xe/0xe0
[<ffffffff8182488f>] ret_from_fork+0x3f/0x70
[<ffffffff81817f30>] ? rest_init+0x80/0x80
To fix this, only send the kick if the target CPU's interrupt has been
initialized. This check isn't racy, because the target is waiting for
the spinlock, so it won't have initialized the interrupt in the
meantime.
Signed-off-by: Ross Lagerwall <ross.lagerwall@citrix.com>
Reviewed-by: Boris Ostrovsky <boris.ostrovsky@oracle.com>
Cc: David Vrabel <david.vrabel@citrix.com>
Cc: Juergen Gross <jgross@suse.com>
Cc: Konrad Rzeszutek Wilk <konrad.wilk@oracle.com>
Cc: Linus Torvalds <torvalds@linux-foundation.org>
Cc: Peter Zijlstra <peterz@infradead.org>
Cc: Thomas Gleixner <tglx@linutronix.de>
Cc: linux-kernel@vger.kernel.org
Cc: xen-devel@lists.xenproject.org
Signed-off-by: Ingo Molnar <mingo@kernel.org>
416 lines
9.8 KiB
C
416 lines
9.8 KiB
C
/*
|
|
* Split spinlock implementation out into its own file, so it can be
|
|
* compiled in a FTRACE-compatible way.
|
|
*/
|
|
#include <linux/kernel_stat.h>
|
|
#include <linux/spinlock.h>
|
|
#include <linux/debugfs.h>
|
|
#include <linux/log2.h>
|
|
#include <linux/gfp.h>
|
|
#include <linux/slab.h>
|
|
|
|
#include <asm/paravirt.h>
|
|
|
|
#include <xen/interface/xen.h>
|
|
#include <xen/events.h>
|
|
|
|
#include "xen-ops.h"
|
|
#include "debugfs.h"
|
|
|
|
static DEFINE_PER_CPU(int, lock_kicker_irq) = -1;
|
|
static DEFINE_PER_CPU(char *, irq_name);
|
|
static bool xen_pvspin = true;
|
|
|
|
#ifdef CONFIG_QUEUED_SPINLOCKS
|
|
|
|
#include <asm/qspinlock.h>
|
|
|
|
static void xen_qlock_kick(int cpu)
|
|
{
|
|
int irq = per_cpu(lock_kicker_irq, cpu);
|
|
|
|
/* Don't kick if the target's kicker interrupt is not initialized. */
|
|
if (irq == -1)
|
|
return;
|
|
|
|
xen_send_IPI_one(cpu, XEN_SPIN_UNLOCK_VECTOR);
|
|
}
|
|
|
|
/*
|
|
* Halt the current CPU & release it back to the host
|
|
*/
|
|
static void xen_qlock_wait(u8 *byte, u8 val)
|
|
{
|
|
int irq = __this_cpu_read(lock_kicker_irq);
|
|
|
|
/* If kicker interrupts not initialized yet, just spin */
|
|
if (irq == -1)
|
|
return;
|
|
|
|
/* clear pending */
|
|
xen_clear_irq_pending(irq);
|
|
barrier();
|
|
|
|
/*
|
|
* We check the byte value after clearing pending IRQ to make sure
|
|
* that we won't miss a wakeup event because of the clearing.
|
|
*
|
|
* The sync_clear_bit() call in xen_clear_irq_pending() is atomic.
|
|
* So it is effectively a memory barrier for x86.
|
|
*/
|
|
if (READ_ONCE(*byte) != val)
|
|
return;
|
|
|
|
/*
|
|
* If an interrupt happens here, it will leave the wakeup irq
|
|
* pending, which will cause xen_poll_irq() to return
|
|
* immediately.
|
|
*/
|
|
|
|
/* Block until irq becomes pending (or perhaps a spurious wakeup) */
|
|
xen_poll_irq(irq);
|
|
}
|
|
|
|
#else /* CONFIG_QUEUED_SPINLOCKS */
|
|
|
|
enum xen_contention_stat {
|
|
TAKEN_SLOW,
|
|
TAKEN_SLOW_PICKUP,
|
|
TAKEN_SLOW_SPURIOUS,
|
|
RELEASED_SLOW,
|
|
RELEASED_SLOW_KICKED,
|
|
NR_CONTENTION_STATS
|
|
};
|
|
|
|
|
|
#ifdef CONFIG_XEN_DEBUG_FS
|
|
#define HISTO_BUCKETS 30
|
|
static struct xen_spinlock_stats
|
|
{
|
|
u32 contention_stats[NR_CONTENTION_STATS];
|
|
u32 histo_spin_blocked[HISTO_BUCKETS+1];
|
|
u64 time_blocked;
|
|
} spinlock_stats;
|
|
|
|
static u8 zero_stats;
|
|
|
|
static inline void check_zero(void)
|
|
{
|
|
u8 ret;
|
|
u8 old = READ_ONCE(zero_stats);
|
|
if (unlikely(old)) {
|
|
ret = cmpxchg(&zero_stats, old, 0);
|
|
/* This ensures only one fellow resets the stat */
|
|
if (ret == old)
|
|
memset(&spinlock_stats, 0, sizeof(spinlock_stats));
|
|
}
|
|
}
|
|
|
|
static inline void add_stats(enum xen_contention_stat var, u32 val)
|
|
{
|
|
check_zero();
|
|
spinlock_stats.contention_stats[var] += val;
|
|
}
|
|
|
|
static inline u64 spin_time_start(void)
|
|
{
|
|
return xen_clocksource_read();
|
|
}
|
|
|
|
static void __spin_time_accum(u64 delta, u32 *array)
|
|
{
|
|
unsigned index = ilog2(delta);
|
|
|
|
check_zero();
|
|
|
|
if (index < HISTO_BUCKETS)
|
|
array[index]++;
|
|
else
|
|
array[HISTO_BUCKETS]++;
|
|
}
|
|
|
|
static inline void spin_time_accum_blocked(u64 start)
|
|
{
|
|
u32 delta = xen_clocksource_read() - start;
|
|
|
|
__spin_time_accum(delta, spinlock_stats.histo_spin_blocked);
|
|
spinlock_stats.time_blocked += delta;
|
|
}
|
|
#else /* !CONFIG_XEN_DEBUG_FS */
|
|
static inline void add_stats(enum xen_contention_stat var, u32 val)
|
|
{
|
|
}
|
|
|
|
static inline u64 spin_time_start(void)
|
|
{
|
|
return 0;
|
|
}
|
|
|
|
static inline void spin_time_accum_blocked(u64 start)
|
|
{
|
|
}
|
|
#endif /* CONFIG_XEN_DEBUG_FS */
|
|
|
|
struct xen_lock_waiting {
|
|
struct arch_spinlock *lock;
|
|
__ticket_t want;
|
|
};
|
|
|
|
static DEFINE_PER_CPU(struct xen_lock_waiting, lock_waiting);
|
|
static cpumask_t waiting_cpus;
|
|
|
|
__visible void xen_lock_spinning(struct arch_spinlock *lock, __ticket_t want)
|
|
{
|
|
int irq = __this_cpu_read(lock_kicker_irq);
|
|
struct xen_lock_waiting *w = this_cpu_ptr(&lock_waiting);
|
|
int cpu = smp_processor_id();
|
|
u64 start;
|
|
__ticket_t head;
|
|
unsigned long flags;
|
|
|
|
/* If kicker interrupts not initialized yet, just spin */
|
|
if (irq == -1)
|
|
return;
|
|
|
|
start = spin_time_start();
|
|
|
|
/*
|
|
* Make sure an interrupt handler can't upset things in a
|
|
* partially setup state.
|
|
*/
|
|
local_irq_save(flags);
|
|
/*
|
|
* We don't really care if we're overwriting some other
|
|
* (lock,want) pair, as that would mean that we're currently
|
|
* in an interrupt context, and the outer context had
|
|
* interrupts enabled. That has already kicked the VCPU out
|
|
* of xen_poll_irq(), so it will just return spuriously and
|
|
* retry with newly setup (lock,want).
|
|
*
|
|
* The ordering protocol on this is that the "lock" pointer
|
|
* may only be set non-NULL if the "want" ticket is correct.
|
|
* If we're updating "want", we must first clear "lock".
|
|
*/
|
|
w->lock = NULL;
|
|
smp_wmb();
|
|
w->want = want;
|
|
smp_wmb();
|
|
w->lock = lock;
|
|
|
|
/* This uses set_bit, which atomic and therefore a barrier */
|
|
cpumask_set_cpu(cpu, &waiting_cpus);
|
|
add_stats(TAKEN_SLOW, 1);
|
|
|
|
/* clear pending */
|
|
xen_clear_irq_pending(irq);
|
|
|
|
/* Only check lock once pending cleared */
|
|
barrier();
|
|
|
|
/*
|
|
* Mark entry to slowpath before doing the pickup test to make
|
|
* sure we don't deadlock with an unlocker.
|
|
*/
|
|
__ticket_enter_slowpath(lock);
|
|
|
|
/* make sure enter_slowpath, which is atomic does not cross the read */
|
|
smp_mb__after_atomic();
|
|
|
|
/*
|
|
* check again make sure it didn't become free while
|
|
* we weren't looking
|
|
*/
|
|
head = READ_ONCE(lock->tickets.head);
|
|
if (__tickets_equal(head, want)) {
|
|
add_stats(TAKEN_SLOW_PICKUP, 1);
|
|
goto out;
|
|
}
|
|
|
|
/* Allow interrupts while blocked */
|
|
local_irq_restore(flags);
|
|
|
|
/*
|
|
* If an interrupt happens here, it will leave the wakeup irq
|
|
* pending, which will cause xen_poll_irq() to return
|
|
* immediately.
|
|
*/
|
|
|
|
/* Block until irq becomes pending (or perhaps a spurious wakeup) */
|
|
xen_poll_irq(irq);
|
|
add_stats(TAKEN_SLOW_SPURIOUS, !xen_test_irq_pending(irq));
|
|
|
|
local_irq_save(flags);
|
|
|
|
kstat_incr_irq_this_cpu(irq);
|
|
out:
|
|
cpumask_clear_cpu(cpu, &waiting_cpus);
|
|
w->lock = NULL;
|
|
|
|
local_irq_restore(flags);
|
|
|
|
spin_time_accum_blocked(start);
|
|
}
|
|
PV_CALLEE_SAVE_REGS_THUNK(xen_lock_spinning);
|
|
|
|
static void xen_unlock_kick(struct arch_spinlock *lock, __ticket_t next)
|
|
{
|
|
int cpu;
|
|
|
|
add_stats(RELEASED_SLOW, 1);
|
|
|
|
for_each_cpu(cpu, &waiting_cpus) {
|
|
const struct xen_lock_waiting *w = &per_cpu(lock_waiting, cpu);
|
|
|
|
/* Make sure we read lock before want */
|
|
if (READ_ONCE(w->lock) == lock &&
|
|
READ_ONCE(w->want) == next) {
|
|
add_stats(RELEASED_SLOW_KICKED, 1);
|
|
xen_send_IPI_one(cpu, XEN_SPIN_UNLOCK_VECTOR);
|
|
break;
|
|
}
|
|
}
|
|
}
|
|
#endif /* CONFIG_QUEUED_SPINLOCKS */
|
|
|
|
static irqreturn_t dummy_handler(int irq, void *dev_id)
|
|
{
|
|
BUG();
|
|
return IRQ_HANDLED;
|
|
}
|
|
|
|
void xen_init_lock_cpu(int cpu)
|
|
{
|
|
int irq;
|
|
char *name;
|
|
|
|
if (!xen_pvspin)
|
|
return;
|
|
|
|
WARN(per_cpu(lock_kicker_irq, cpu) >= 0, "spinlock on CPU%d exists on IRQ%d!\n",
|
|
cpu, per_cpu(lock_kicker_irq, cpu));
|
|
|
|
name = kasprintf(GFP_KERNEL, "spinlock%d", cpu);
|
|
irq = bind_ipi_to_irqhandler(XEN_SPIN_UNLOCK_VECTOR,
|
|
cpu,
|
|
dummy_handler,
|
|
IRQF_PERCPU|IRQF_NOBALANCING,
|
|
name,
|
|
NULL);
|
|
|
|
if (irq >= 0) {
|
|
disable_irq(irq); /* make sure it's never delivered */
|
|
per_cpu(lock_kicker_irq, cpu) = irq;
|
|
per_cpu(irq_name, cpu) = name;
|
|
}
|
|
|
|
printk("cpu %d spinlock event irq %d\n", cpu, irq);
|
|
}
|
|
|
|
void xen_uninit_lock_cpu(int cpu)
|
|
{
|
|
if (!xen_pvspin)
|
|
return;
|
|
|
|
unbind_from_irqhandler(per_cpu(lock_kicker_irq, cpu), NULL);
|
|
per_cpu(lock_kicker_irq, cpu) = -1;
|
|
kfree(per_cpu(irq_name, cpu));
|
|
per_cpu(irq_name, cpu) = NULL;
|
|
}
|
|
|
|
|
|
/*
|
|
* Our init of PV spinlocks is split in two init functions due to us
|
|
* using paravirt patching and jump labels patching and having to do
|
|
* all of this before SMP code is invoked.
|
|
*
|
|
* The paravirt patching needs to be done _before_ the alternative asm code
|
|
* is started, otherwise we would not patch the core kernel code.
|
|
*/
|
|
void __init xen_init_spinlocks(void)
|
|
{
|
|
|
|
if (!xen_pvspin) {
|
|
printk(KERN_DEBUG "xen: PV spinlocks disabled\n");
|
|
return;
|
|
}
|
|
printk(KERN_DEBUG "xen: PV spinlocks enabled\n");
|
|
#ifdef CONFIG_QUEUED_SPINLOCKS
|
|
__pv_init_lock_hash();
|
|
pv_lock_ops.queued_spin_lock_slowpath = __pv_queued_spin_lock_slowpath;
|
|
pv_lock_ops.queued_spin_unlock = PV_CALLEE_SAVE(__pv_queued_spin_unlock);
|
|
pv_lock_ops.wait = xen_qlock_wait;
|
|
pv_lock_ops.kick = xen_qlock_kick;
|
|
#else
|
|
pv_lock_ops.lock_spinning = PV_CALLEE_SAVE(xen_lock_spinning);
|
|
pv_lock_ops.unlock_kick = xen_unlock_kick;
|
|
#endif
|
|
}
|
|
|
|
/*
|
|
* While the jump_label init code needs to happend _after_ the jump labels are
|
|
* enabled and before SMP is started. Hence we use pre-SMP initcall level
|
|
* init. We cannot do it in xen_init_spinlocks as that is done before
|
|
* jump labels are activated.
|
|
*/
|
|
static __init int xen_init_spinlocks_jump(void)
|
|
{
|
|
if (!xen_pvspin)
|
|
return 0;
|
|
|
|
if (!xen_domain())
|
|
return 0;
|
|
|
|
static_key_slow_inc(¶virt_ticketlocks_enabled);
|
|
return 0;
|
|
}
|
|
early_initcall(xen_init_spinlocks_jump);
|
|
|
|
static __init int xen_parse_nopvspin(char *arg)
|
|
{
|
|
xen_pvspin = false;
|
|
return 0;
|
|
}
|
|
early_param("xen_nopvspin", xen_parse_nopvspin);
|
|
|
|
#if defined(CONFIG_XEN_DEBUG_FS) && !defined(CONFIG_QUEUED_SPINLOCKS)
|
|
|
|
static struct dentry *d_spin_debug;
|
|
|
|
static int __init xen_spinlock_debugfs(void)
|
|
{
|
|
struct dentry *d_xen = xen_init_debugfs();
|
|
|
|
if (d_xen == NULL)
|
|
return -ENOMEM;
|
|
|
|
if (!xen_pvspin)
|
|
return 0;
|
|
|
|
d_spin_debug = debugfs_create_dir("spinlocks", d_xen);
|
|
|
|
debugfs_create_u8("zero_stats", 0644, d_spin_debug, &zero_stats);
|
|
|
|
debugfs_create_u32("taken_slow", 0444, d_spin_debug,
|
|
&spinlock_stats.contention_stats[TAKEN_SLOW]);
|
|
debugfs_create_u32("taken_slow_pickup", 0444, d_spin_debug,
|
|
&spinlock_stats.contention_stats[TAKEN_SLOW_PICKUP]);
|
|
debugfs_create_u32("taken_slow_spurious", 0444, d_spin_debug,
|
|
&spinlock_stats.contention_stats[TAKEN_SLOW_SPURIOUS]);
|
|
|
|
debugfs_create_u32("released_slow", 0444, d_spin_debug,
|
|
&spinlock_stats.contention_stats[RELEASED_SLOW]);
|
|
debugfs_create_u32("released_slow_kicked", 0444, d_spin_debug,
|
|
&spinlock_stats.contention_stats[RELEASED_SLOW_KICKED]);
|
|
|
|
debugfs_create_u64("time_blocked", 0444, d_spin_debug,
|
|
&spinlock_stats.time_blocked);
|
|
|
|
debugfs_create_u32_array("histo_blocked", 0444, d_spin_debug,
|
|
spinlock_stats.histo_spin_blocked, HISTO_BUCKETS + 1);
|
|
|
|
return 0;
|
|
}
|
|
fs_initcall(xen_spinlock_debugfs);
|
|
|
|
#endif /* CONFIG_XEN_DEBUG_FS */
|